├── go.mod ├── .gitignore ├── README.md ├── internal └── internal.go ├── LICENSE ├── pipeline ├── parnode.go ├── example_wordcount_test.go ├── filter.go ├── strictordnode.go ├── lparnode.go ├── seqnode.go ├── source.go ├── filters.go └── pipeline.go ├── go.sum ├── doc.go ├── sort ├── example_interface_test.go ├── quicksort.go ├── mergesort.go ├── sort.go └── sort_test.go ├── parallel ├── example_heatdistribution_test.go ├── parallel_test.go └── parallel.go ├── sequential └── sequential.go └── speculative └── speculative.go /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/exascience/pargo 2 | 3 | go 1.14 4 | 5 | require gonum.org/v1/gonum v0.7.0 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | 13 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 14 | .glide/ 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pargo 2 | ## A library for parallel programming in Go 3 | 4 | Package pargo provides functions and data structures for expressing 5 | parallel algorithms. While Go is primarily designed for concurrent 6 | programming, it is also usable to some extent for parallel 7 | programming, and this library provides convenience functionality to 8 | turn otherwise sequential algorithms into parallel algorithms, with 9 | the goal to improve performance. 10 | 11 | Documentation: [http://godoc.org/github.com/ExaScience/pargo](http://godoc.org/github.com/ExaScience/pargo) 12 | and [http://github.com/ExaScience/pargo/wiki](http://github.com/ExaScience/pargo/wiki) 13 | -------------------------------------------------------------------------------- /internal/internal.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "runtime" 7 | "runtime/debug" 8 | ) 9 | 10 | // ComputeNofBatches divides the size of the range (high - low) by n. If n is 0, 11 | // a default is used that takes runtime.GOMAXPROCS(0) into account. 12 | func ComputeNofBatches(low, high, n int) (batches int) { 13 | switch size := high - low; { 14 | case size > 0: 15 | switch { 16 | case n == 0: 17 | batches = 2 * runtime.GOMAXPROCS(0) 18 | case n > 0: 19 | batches = n 20 | default: 21 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 22 | } 23 | if batches > size { 24 | batches = size 25 | } 26 | case size == 0: 27 | batches = 1 28 | default: 29 | panic(fmt.Sprintf("invalid range: %v:%v", low, high)) 30 | } 31 | return 32 | } 33 | 34 | type runtimeError struct{ error } 35 | 36 | func (runtimeError) RuntimeError() {} 37 | 38 | // WrapPanic adds stack trace information to a recovered panic. 39 | func WrapPanic(p interface{}) interface{} { 40 | if p != nil { 41 | s := fmt.Sprintf("%v\n%s\nrethrown at", p, debug.Stack()) 42 | if _, isError := p.(error); isError { 43 | r := errors.New(s) 44 | if _, isRuntimeError := p.(runtime.Error); isRuntimeError { 45 | return runtimeError{r} 46 | } 47 | return r 48 | } 49 | return s 50 | } 51 | return nil 52 | } 53 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017, Imec 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /pipeline/parnode.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | type parnode struct { 8 | waitGroup sync.WaitGroup 9 | filters []Filter 10 | receivers []Receiver 11 | finalizers []Finalizer 12 | } 13 | 14 | // Par creates a parallel node with the given filters. 15 | func Par(filters ...Filter) Node { 16 | return &parnode{filters: filters} 17 | } 18 | 19 | // Implements the TryMerge method of the Node interface. 20 | func (node *parnode) TryMerge(next Node) bool { 21 | if nxt, merge := next.(*parnode); merge { 22 | node.filters = append(node.filters, nxt.filters...) 23 | node.receivers = append(node.receivers, nxt.receivers...) 24 | node.finalizers = append(node.finalizers, nxt.finalizers...) 25 | return true 26 | } 27 | return false 28 | } 29 | 30 | // Implements the Begin method of the Node interface. 31 | func (node *parnode) Begin(p *Pipeline, _ int, dataSize *int) (keep bool) { 32 | node.receivers, node.finalizers = ComposeFilters(p, Parallel, dataSize, node.filters) 33 | node.filters = nil 34 | keep = (len(node.receivers) > 0) || (len(node.finalizers) > 0) 35 | return 36 | } 37 | 38 | // Implements the Feed method of the Node interface. 39 | func (node *parnode) Feed(p *Pipeline, index int, seqNo int, data interface{}) { 40 | node.waitGroup.Add(1) 41 | go func() { 42 | defer node.waitGroup.Done() 43 | select { 44 | case <-p.ctx.Done(): 45 | return 46 | default: 47 | feed(p, node.receivers, index, seqNo, data) 48 | } 49 | }() 50 | } 51 | 52 | // Implements the End method of the Node interface. 53 | func (node *parnode) End() { 54 | node.waitGroup.Wait() 55 | for _, finalize := range node.finalizers { 56 | finalize() 57 | } 58 | node.receivers = nil 59 | node.finalizers = nil 60 | } 61 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= 2 | github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= 3 | github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= 4 | github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= 5 | golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 6 | golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 7 | golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2 h1:y102fOLFqhV41b+4GPiJoa0k/x+pJcEi2/HB1Y5T6fU= 8 | golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 9 | golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= 10 | golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 11 | golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 12 | gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= 13 | gonum.org/v1/gonum v0.7.0 h1:Hdks0L0hgznZLG9nzXb8vZ0rRvqNvAcgAp84y7Mwkgw= 14 | gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM= 15 | gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= 16 | gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= 17 | gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= 18 | rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= 19 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package pargo provides functions and data structures for expressing parallel 2 | // algorithms. While Go is primarily designed for concurrent programming, it is 3 | // also usable to some extent for parallel programming, and this library 4 | // provides convenience functionality to turn otherwise sequential algorithms 5 | // into parallel algorithms, with the goal to improve performance. 6 | // 7 | // For documentation that provides a more structured overview than is possible 8 | // with Godoc, see the wiki at https://github.com/exascience/pargo/wiki 9 | // 10 | // Pargo provides the following subpackages: 11 | // 12 | // pargo/parallel provides simple functions for executing series of thunks or 13 | // predicates, as well as thunks, predicates, or reducers over ranges in 14 | // parallel. See also https://github.com/ExaScience/pargo/wiki/TaskParallelism 15 | // 16 | // pargo/speculative provides speculative implementations of most of the 17 | // functions from pargo/parallel. These implementations not only execute in 18 | // parallel, but also attempt to terminate early as soon as the final result is 19 | // known. See also https://github.com/ExaScience/pargo/wiki/TaskParallelism 20 | // 21 | // pargo/sequential provides sequential implementations of all functions from 22 | // pargo/parallel, for testing and debugging purposes. 23 | // 24 | // pargo/sort provides parallel sorting algorithms. 25 | // 26 | // pargo/sync provides an efficient parallel map implementation. 27 | // 28 | // pargo/pipeline provides functions and data structures to construct and 29 | // execute parallel pipelines. 30 | // 31 | // Pargo has been influenced to various extents by ideas from Cilk, Threading 32 | // Building Blocks, and Java's java.util.concurrent and java.util.stream 33 | // packages. See http://supertech.csail.mit.edu/papers/steal.pdf for some 34 | // theoretical background, and the sample chapter at 35 | // https://mitpress.mit.edu/books/introduction-algorithms for a more practical 36 | // overview of the underlying concepts. 37 | package pargo 38 | -------------------------------------------------------------------------------- /sort/example_interface_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The Go Authors. All rights reserved. Use of this source code 2 | // is governed by a BSD-style license that can be found in the LICENSE file. 3 | 4 | // Adapted by Pascal Costanza for the Pargo package. 5 | 6 | package sort_test 7 | 8 | import ( 9 | "fmt" 10 | stdsort "sort" 11 | 12 | sort "github.com/exascience/pargo/sort" 13 | ) 14 | 15 | type Person struct { 16 | Name string 17 | Age int 18 | } 19 | 20 | func (p Person) String() string { 21 | return fmt.Sprintf("%s: %d", p.Name, p.Age) 22 | } 23 | 24 | // ByAge implements sort.SequentialSorter, sort.Sorter, and sort.StableSorter 25 | // for []Person based on the Age field. 26 | type ByAge []Person 27 | 28 | func (a ByAge) SequentialSort(i, j int) { 29 | stdsort.SliceStable(a, func(i, j int) bool { 30 | return a[i].Age < a[j].Age 31 | }) 32 | } 33 | 34 | func (a ByAge) Len() int { return len(a) } 35 | func (a ByAge) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 36 | func (a ByAge) Less(i, j int) bool { return a[i].Age < a[j].Age } 37 | 38 | func (a ByAge) NewTemp() sort.StableSorter { return make(ByAge, len(a)) } 39 | 40 | func (this ByAge) Assign(that sort.StableSorter) func(i, j, len int) { 41 | dst, src := this, that.(ByAge) 42 | return func(i, j, len int) { 43 | for k := 0; k < len; k++ { 44 | dst[i+k] = src[j+k] 45 | } 46 | } 47 | } 48 | 49 | func Example() { 50 | people := []Person{ 51 | {"Bob", 31}, 52 | {"John", 42}, 53 | {"Michael", 17}, 54 | {"Jenny", 26}, 55 | } 56 | 57 | fmt.Println(people) 58 | sort.Sort(ByAge(people)) 59 | fmt.Println(people) 60 | 61 | people = []Person{ 62 | {"Bob", 31}, 63 | {"John", 42}, 64 | {"Michael", 17}, 65 | {"Jenny", 26}, 66 | } 67 | 68 | fmt.Println(people) 69 | sort.StableSort(ByAge(people)) 70 | fmt.Println(people) 71 | 72 | // Output: 73 | // [Bob: 31 John: 42 Michael: 17 Jenny: 26] 74 | // [Michael: 17 Jenny: 26 Bob: 31 John: 42] 75 | // [Bob: 31 John: 42 Michael: 17 Jenny: 26] 76 | // [Michael: 17 Jenny: 26 Bob: 31 John: 42] 77 | } 78 | -------------------------------------------------------------------------------- /sort/quicksort.go: -------------------------------------------------------------------------------- 1 | package sort 2 | 3 | import ( 4 | "sort" 5 | 6 | "github.com/exascience/pargo/parallel" 7 | ) 8 | 9 | const qsortGrainSize = 0x500 10 | 11 | // Sorter is a type, typically a collection, that can be sorted by Sort in this 12 | // package. The methods require that (ranges of) elements of the collection can 13 | // be enumerated by integer indices. 14 | type Sorter interface { 15 | SequentialSorter 16 | sort.Interface 17 | } 18 | 19 | func medianOfThree(data sort.Interface, l, m, r int) int { 20 | if data.Less(l, m) { 21 | if data.Less(m, r) { 22 | return m 23 | } else if data.Less(l, r) { 24 | return r 25 | } 26 | } else if data.Less(r, m) { 27 | return m 28 | } else if data.Less(r, l) { 29 | return r 30 | } 31 | return l 32 | } 33 | 34 | func pseudoMedianOfNine(data sort.Interface, index, size int) int { 35 | offset := size / 8 36 | return medianOfThree(data, 37 | medianOfThree(data, index, index+offset, index+offset*2), 38 | medianOfThree(data, index+offset*3, index+offset*4, index+offset*5), 39 | medianOfThree(data, index+offset*6, index+offset*7, index+size-1), 40 | ) 41 | } 42 | 43 | // Sort uses a parallel quicksort implementation. 44 | // 45 | // It is good for small core counts and small collection sizes. 46 | func Sort(data Sorter) { 47 | size := data.Len() 48 | sSort := data.SequentialSort 49 | if size < qsortGrainSize { 50 | sSort(0, size) 51 | return 52 | } 53 | var pSort func(int, int) 54 | pSort = func(index, size int) { 55 | if size < qsortGrainSize { 56 | sSort(index, index+size) 57 | } else { 58 | m := pseudoMedianOfNine(data, index, size) 59 | if m > index { 60 | data.Swap(index, m) 61 | } 62 | i, j := index, index+size 63 | outer: 64 | for { 65 | for { 66 | j-- 67 | if !data.Less(index, j) { 68 | break 69 | } 70 | } 71 | for { 72 | if i == j { 73 | break outer 74 | } 75 | i++ 76 | if !data.Less(i, index) { 77 | break 78 | } 79 | } 80 | if i == j { 81 | break outer 82 | } 83 | data.Swap(i, j) 84 | } 85 | data.Swap(j, index) 86 | i = j + 1 87 | parallel.Do( 88 | func() { pSort(index, j-index) }, 89 | func() { pSort(i, index+size-i) }, 90 | ) 91 | } 92 | } 93 | if !IsSorted(data) { 94 | pSort(0, size) 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /pipeline/example_wordcount_test.go: -------------------------------------------------------------------------------- 1 | package pipeline_test 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "runtime" 8 | "strings" 9 | 10 | "github.com/exascience/pargo/pipeline" 11 | "github.com/exascience/pargo/sort" 12 | "github.com/exascience/pargo/sync" 13 | ) 14 | 15 | type Word string 16 | 17 | func (w Word) Hash() (hash uint64) { 18 | // DJBX33A 19 | hash = 5381 20 | for _, b := range w { 21 | hash = ((hash << 5) + hash) + uint64(b) 22 | } 23 | return 24 | } 25 | 26 | func WordCount(r io.Reader) *sync.Map { 27 | result := sync.NewMap(16 * runtime.GOMAXPROCS(0)) 28 | scanner := pipeline.NewScanner(r) 29 | scanner.Split(bufio.ScanWords) 30 | var p pipeline.Pipeline 31 | p.Source(scanner) 32 | p.Add( 33 | pipeline.Par(pipeline.Receive( 34 | func(_ int, data interface{}) interface{} { 35 | var uniqueWords []string 36 | for _, s := range data.([]string) { 37 | newValue, _ := result.Modify(Word(s), func(value interface{}, ok bool) (newValue interface{}, store bool) { 38 | if ok { 39 | newValue = value.(int) + 1 40 | } else { 41 | newValue = 1 42 | } 43 | store = true 44 | return 45 | }) 46 | if newValue.(int) == 1 { 47 | uniqueWords = append(uniqueWords, s) 48 | } 49 | } 50 | return uniqueWords 51 | }, 52 | )), 53 | pipeline.Ord(pipeline.ReceiveAndFinalize( 54 | func(_ int, data interface{}) interface{} { 55 | // print unique words as encountered first at the source 56 | for _, s := range data.([]string) { 57 | fmt.Print(s, " ") 58 | } 59 | return data 60 | }, 61 | func() { fmt.Println(".") }, 62 | )), 63 | ) 64 | p.Run() 65 | return result 66 | } 67 | 68 | func Example_wordCount() { 69 | r := strings.NewReader("The big black bug bit the big black bear but the big black bear bit the big black bug back") 70 | counts := WordCount(r) 71 | words := make(sort.StringSlice, 0) 72 | counts.Range(func(key, _ interface{}) bool { 73 | words = append(words, string(key.(Word))) 74 | return true 75 | }) 76 | sort.Sort(words) 77 | for _, word := range words { 78 | count, _ := counts.Load(Word(word)) 79 | fmt.Println(word, count.(int)) 80 | } 81 | 82 | // Output: 83 | // The big black bug bit the bear but back . 84 | // The 1 85 | // back 1 86 | // bear 2 87 | // big 4 88 | // bit 2 89 | // black 4 90 | // bug 2 91 | // but 1 92 | // the 3 93 | } 94 | -------------------------------------------------------------------------------- /pipeline/filter.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | // A NodeKind reperesents the different kinds of nodes. 4 | type NodeKind int 5 | 6 | const ( 7 | // Ordered nodes receive batches in encounter order. 8 | Ordered NodeKind = iota 9 | 10 | // Sequential nodes receive batches in arbitrary sequential order. 11 | Sequential 12 | 13 | // Parallel nodes receives batches in parallel. 14 | Parallel 15 | ) 16 | 17 | // A Filter is a function that returns a Receiver and a Finalizer to be added to 18 | // a node. It receives a pipeline, the kind of node it will be added to, and the 19 | // expected total data size that the receiver will be asked to process. 20 | // 21 | // The dataSize parameter is either positive, in which case it indicates the 22 | // expected total size of all batches that will eventually be passed to this 23 | // filter's receiver, or it is negative, in which case the expected size is 24 | // either unknown or too difficult to determine. The dataSize parameter is a 25 | // pointer whose contents can be modified by the filter, for example if this 26 | // filter increases or decreases the total size for subsequent filters, or if 27 | // this filter can change dataSize from an unknown to a known value, or vice 28 | // versa, must change it from a known to an unknown value. 29 | // 30 | // Either the receiver or the finalizer or both can be nil, in which case they 31 | // will not be added to the current node. 32 | type Filter func(pipeline *Pipeline, kind NodeKind, dataSize *int) (Receiver, Finalizer) 33 | 34 | // A Receiver is called for every data batch, and returns a potentially modified 35 | // data batch. The seqNo parameter indicates the order in which the data batch 36 | // was encountered at the current pipeline's data source. 37 | type Receiver func(seqNo int, data interface{}) (filteredData interface{}) 38 | 39 | // A Finalizer is called once after the corresponding receiver has been called 40 | // for all data batches in the current pipeline. 41 | type Finalizer func() 42 | 43 | // ComposeFilters takes a number of filters, calls them with the given pipeline, 44 | // kind, and dataSize parameters in order, and appends the returned receivers 45 | // and finalizers (except for nil values) to the result slices. 46 | // 47 | // ComposeFilters is used in Node implementations. User programs typically do 48 | // not call ComposeFilters. 49 | func ComposeFilters(pipeline *Pipeline, kind NodeKind, dataSize *int, filters []Filter) (receivers []Receiver, finalizers []Finalizer) { 50 | for _, filter := range filters { 51 | receiver, finalizer := filter(pipeline, kind, dataSize) 52 | if receiver != nil { 53 | receivers = append(receivers, receiver) 54 | } 55 | if finalizer != nil { 56 | finalizers = append(finalizers, finalizer) 57 | } 58 | } 59 | return 60 | } 61 | 62 | func feed(p *Pipeline, receivers []Receiver, index int, seqNo int, data interface{}) { 63 | for _, receive := range receivers { 64 | data = receive(seqNo, data) 65 | } 66 | p.FeedForward(index, seqNo, data) 67 | } 68 | -------------------------------------------------------------------------------- /pipeline/strictordnode.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | type strictordnode struct { 8 | cond *sync.Cond 9 | channel chan dataBatch 10 | waitGroup sync.WaitGroup 11 | run int 12 | filters []Filter 13 | receivers []Receiver 14 | finalizers []Finalizer 15 | } 16 | 17 | // StrictOrd creates an ordered node with the given filters. 18 | func StrictOrd(filters ...Filter) Node { 19 | return &strictordnode{filters: filters} 20 | } 21 | 22 | // Implements the TryMerge method of the Node interface. 23 | func (node *strictordnode) TryMerge(next Node) bool { 24 | switch nxt := next.(type) { 25 | case *seqnode: 26 | node.filters = append(node.filters, nxt.filters...) 27 | node.receivers = append(node.receivers, nxt.receivers...) 28 | node.finalizers = append(node.finalizers, nxt.finalizers...) 29 | return true 30 | case *strictordnode: 31 | node.filters = append(node.filters, nxt.filters...) 32 | node.receivers = append(node.receivers, nxt.receivers...) 33 | node.finalizers = append(node.finalizers, nxt.finalizers...) 34 | return true 35 | default: 36 | return false 37 | } 38 | } 39 | 40 | //Implements the Begin method of the Node interface. 41 | func (node *strictordnode) Begin(p *Pipeline, index int, dataSize *int) (keep bool) { 42 | node.receivers, node.finalizers = ComposeFilters(p, Ordered, dataSize, node.filters) 43 | node.filters = nil 44 | if keep = (len(node.receivers) > 0) || (len(node.finalizers) > 0); keep { 45 | node.cond = sync.NewCond(&sync.Mutex{}) 46 | node.channel = make(chan dataBatch) 47 | node.waitGroup.Add(1) 48 | go func() { 49 | defer node.waitGroup.Done() 50 | for { 51 | select { 52 | case <-p.ctx.Done(): 53 | node.cond.Broadcast() 54 | return 55 | case batch, ok := <-node.channel: 56 | if !ok { 57 | return 58 | } 59 | node.cond.L.Lock() 60 | if batch.seqNo != node.run { 61 | panic("Invalid receive order in a strictly ordered pipeline node.") 62 | } 63 | node.run++ 64 | node.cond.L.Unlock() 65 | node.cond.Broadcast() 66 | feed(p, node.receivers, index, batch.seqNo, batch.data) 67 | } 68 | } 69 | }() 70 | } 71 | return 72 | } 73 | 74 | // Implements the Feed method of the Node interface. 75 | func (node *strictordnode) Feed(p *Pipeline, _ int, seqNo int, data interface{}) { 76 | node.cond.L.Lock() 77 | defer node.cond.L.Unlock() 78 | for { 79 | if node.run == seqNo { 80 | select { 81 | case <-p.ctx.Done(): 82 | return 83 | case node.channel <- dataBatch{seqNo, data}: 84 | return 85 | } 86 | } 87 | select { 88 | case <-p.ctx.Done(): 89 | return 90 | default: 91 | node.cond.Wait() 92 | } 93 | } 94 | } 95 | 96 | // Implements the End method of the Node interface. 97 | func (node *strictordnode) End() { 98 | close(node.channel) 99 | node.waitGroup.Wait() 100 | for _, finalize := range node.finalizers { 101 | finalize() 102 | } 103 | node.receivers = nil 104 | node.finalizers = nil 105 | } 106 | -------------------------------------------------------------------------------- /parallel/example_heatdistribution_test.go: -------------------------------------------------------------------------------- 1 | package parallel_test 2 | 3 | // This is a simplified version of a heat distribution simulation, based on an 4 | // implementation by Wilfried Verachtert. 5 | // 6 | // See https://en.wikipedia.org/wiki/Heat_equation for some theoretical 7 | // background. 8 | 9 | import ( 10 | "fmt" 11 | "math" 12 | 13 | "gonum.org/v1/gonum/mat" 14 | 15 | "github.com/exascience/pargo/parallel" 16 | ) 17 | 18 | const ε = 0.001 19 | 20 | func maxDiff(m1, m2 *mat.Dense) (result float64) { 21 | rows, cols := m1.Dims() 22 | result = parallel.RangeReduceFloat64( 23 | 1, rows-1, 0, 24 | func(low, high int) (result float64) { 25 | for row := low; row < high; row++ { 26 | r1 := m1.RawRowView(row) 27 | r2 := m2.RawRowView(row) 28 | for col := 1; col < cols-1; col++ { 29 | result = math.Max(result, math.Abs(r1[col]-r2[col])) 30 | } 31 | } 32 | return 33 | }, 34 | math.Max, 35 | ) 36 | return 37 | } 38 | 39 | func HeatDistributionStep(u, v *mat.Dense) { 40 | rows, cols := u.Dims() 41 | parallel.Range(1, rows-1, 0, 42 | func(low, high int) { 43 | for row := low; row < high; row++ { 44 | uRow := u.RawRowView(row) 45 | vRow := v.RawRowView(row) 46 | vRowUp := v.RawRowView(row - 1) 47 | vRowDn := v.RawRowView(row + 1) 48 | for col := 1; col < cols-1; col++ { 49 | uRow[col] = (vRowUp[col] + vRowDn[col] + vRow[col-1] + vRow[col+1]) / 4.0 50 | } 51 | } 52 | }, 53 | ) 54 | } 55 | 56 | func HeatDistributionSimulation(M, N int, init, t, r, b, l float64) { 57 | // ensure a border 58 | M += 2 59 | N += 2 60 | 61 | // set up the input matrix 62 | data := make([]float64, M*N) 63 | for i := range data { 64 | data[i] = init 65 | } 66 | u := mat.NewDense(M, N, data) 67 | 68 | // set up the border for the input matrix 69 | for i := 0; i < N; i++ { 70 | u.Set(0, i, t) 71 | u.Set(M-1, i, b) 72 | } 73 | for i := 0; i < M; i++ { 74 | u.Set(i, 0, l) 75 | u.Set(i, N-1, r) 76 | } 77 | 78 | // create a secondary working matrix 79 | v := mat.NewDense(M, N, nil) 80 | v.Copy(u) 81 | 82 | // run the simulation 83 | for δ, iterations := ε+1.0, 0; δ >= ε; { 84 | for step := 0; step < 1000; step++ { 85 | HeatDistributionStep(v, u) 86 | HeatDistributionStep(u, v) 87 | } 88 | iterations += 2000 89 | δ = maxDiff(u, v) 90 | fmt.Printf("iterations: %6d, δ: %08.6f, u[8][8]: %10.8f\n", iterations, δ, u.At(8, 8)) 91 | } 92 | } 93 | 94 | func Example_heatDistributionSimulation() { 95 | HeatDistributionSimulation(1024, 1024, 75, 0, 100, 100, 100) 96 | 97 | // Output: 98 | // iterations: 2000, δ: 0.009073, u[8][8]: 50.99678108 99 | // iterations: 4000, δ: 0.004537, u[8][8]: 50.50380048 100 | // iterations: 6000, δ: 0.003025, u[8][8]: 50.33708179 101 | // iterations: 8000, δ: 0.002268, u[8][8]: 50.25326869 102 | // iterations: 10000, δ: 0.001815, u[8][8]: 50.20283493 103 | // iterations: 12000, δ: 0.001512, u[8][8]: 50.16915148 104 | // iterations: 14000, δ: 0.001296, u[8][8]: 50.14506197 105 | // iterations: 16000, δ: 0.001134, u[8][8]: 50.12697847 106 | // iterations: 18000, δ: 0.001008, u[8][8]: 50.11290381 107 | // iterations: 20000, δ: 0.000907, u[8][8]: 50.10163797 108 | } 109 | -------------------------------------------------------------------------------- /parallel/parallel_test.go: -------------------------------------------------------------------------------- 1 | package parallel_test 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "runtime" 7 | 8 | "github.com/exascience/pargo/parallel" 9 | ) 10 | 11 | func ExampleDo() { 12 | var fib func(int) (int, error) 13 | 14 | fib = func(n int) (result int, err error) { 15 | if n < 0 { 16 | err = errors.New("invalid argument") 17 | } else if n < 2 { 18 | result = n 19 | } else { 20 | var n1, n2 int 21 | n1, err = fib(n - 1) 22 | if err != nil { 23 | return 24 | } 25 | n2, err = fib(n - 2) 26 | result = n1 + n2 27 | } 28 | return 29 | } 30 | 31 | type intErr struct { 32 | n int 33 | err error 34 | } 35 | 36 | var parallelFib func(int) intErr 37 | 38 | parallelFib = func(n int) (result intErr) { 39 | if n < 0 { 40 | result.err = errors.New("invalid argument") 41 | } else if n < 20 { 42 | result.n, result.err = fib(n) 43 | } else { 44 | var n1, n2 intErr 45 | parallel.Do( 46 | func() { n1 = parallelFib(n - 1) }, 47 | func() { n2 = parallelFib(n - 2) }, 48 | ) 49 | result.n = n1.n + n2.n 50 | if n1.err != nil { 51 | result.err = n1.err 52 | } else { 53 | result.err = n2.err 54 | } 55 | } 56 | return 57 | } 58 | 59 | if result := parallelFib(-1); result.err != nil { 60 | fmt.Println(result.err) 61 | } else { 62 | fmt.Println(result.n) 63 | } 64 | 65 | // Output: 66 | // invalid argument 67 | } 68 | 69 | func ExampleRangeReduceIntSum() { 70 | numDivisors := func(n int) int { 71 | return parallel.RangeReduceIntSum( 72 | 1, n+1, runtime.GOMAXPROCS(0), 73 | func(low, high int) int { 74 | var sum int 75 | for i := low; i < high; i++ { 76 | if (n % i) == 0 { 77 | sum++ 78 | } 79 | } 80 | return sum 81 | }, 82 | ) 83 | } 84 | 85 | fmt.Println(numDivisors(12)) 86 | 87 | // Output: 88 | // 6 89 | } 90 | 91 | func numDivisors(n int) int { 92 | return parallel.RangeReduceIntSum( 93 | 1, n+1, runtime.GOMAXPROCS(0), 94 | func(low, high int) int { 95 | var sum int 96 | for i := low; i < high; i++ { 97 | if (n % i) == 0 { 98 | sum++ 99 | } 100 | } 101 | return sum 102 | }, 103 | ) 104 | } 105 | 106 | func ExampleRangeReduce() { 107 | findPrimes := func(n int) []int { 108 | result := parallel.RangeReduce( 109 | 2, n, 4*runtime.GOMAXPROCS(0), 110 | func(low, high int) interface{} { 111 | var slice []int 112 | for i := low; i < high; i++ { 113 | if numDivisors(i) == 2 { // see RangeReduceInt example 114 | slice = append(slice, i) 115 | } 116 | } 117 | return slice 118 | }, 119 | func(x, y interface{}) interface{} { 120 | return append(x.([]int), y.([]int)...) 121 | }, 122 | ) 123 | return result.([]int) 124 | } 125 | 126 | fmt.Println(findPrimes(20)) 127 | 128 | // Output: 129 | // [2 3 5 7 11 13 17 19] 130 | } 131 | 132 | func ExampleRangeReduceFloat64Sum() { 133 | sumFloat64s := func(f []float64) float64 { 134 | result := parallel.RangeReduceFloat64Sum( 135 | 0, len(f), runtime.GOMAXPROCS(0), 136 | func(low, high int) float64 { 137 | var sum float64 138 | for i := low; i < high; i++ { 139 | sum += f[i] 140 | } 141 | return sum 142 | }, 143 | ) 144 | return result 145 | } 146 | 147 | fmt.Println(sumFloat64s([]float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10})) 148 | 149 | // Output: 150 | // 55 151 | } 152 | -------------------------------------------------------------------------------- /pipeline/lparnode.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "runtime" 5 | "sync" 6 | ) 7 | 8 | type lparnode struct { 9 | limit int 10 | ordered bool 11 | cond *sync.Cond 12 | channel chan dataBatch 13 | waitGroup sync.WaitGroup 14 | run int 15 | filters []Filter 16 | receivers []Receiver 17 | finalizers []Finalizer 18 | } 19 | 20 | // LimitedPar creates a parallel node with the given filters. The node uses at 21 | // most limit goroutines at the same time. If limit is 0, a reasonable default 22 | // is used instead. Even if limit is 0, the node is still limited. For unlimited 23 | // nodes, use Par instead. 24 | func LimitedPar(limit int, filters ...Filter) Node { 25 | if limit <= 0 { 26 | limit = runtime.GOMAXPROCS(0) 27 | } 28 | if limit == 1 { 29 | return &seqnode{kind: Sequential, filters: filters} 30 | } 31 | return &lparnode{limit: limit, filters: filters} 32 | } 33 | 34 | func (node *lparnode) makeOrdered() { 35 | node.ordered = true 36 | node.cond = sync.NewCond(&sync.Mutex{}) 37 | } 38 | 39 | // Implements the TryMerge method of the Node interface. 40 | func (node *lparnode) TryMerge(next Node) bool { 41 | if nxt, merge := next.(*lparnode); merge && (nxt.limit == node.limit) { 42 | node.filters = append(node.filters, nxt.filters...) 43 | node.receivers = append(node.receivers, nxt.receivers...) 44 | node.finalizers = append(node.finalizers, nxt.finalizers...) 45 | return true 46 | } 47 | return false 48 | } 49 | 50 | // Implements the Begin method of the Node interface. 51 | func (node *lparnode) Begin(p *Pipeline, index int, dataSize *int) (keep bool) { 52 | node.receivers, node.finalizers = ComposeFilters(p, Parallel, dataSize, node.filters) 53 | node.filters = nil 54 | if keep = (len(node.receivers) > 0) || (len(node.finalizers) > 0); keep { 55 | node.channel = make(chan dataBatch) 56 | node.waitGroup.Add(node.limit) 57 | for i := 0; i < node.limit; i++ { 58 | go func() { 59 | defer node.waitGroup.Done() 60 | for { 61 | select { 62 | case <-p.ctx.Done(): 63 | if node.ordered { 64 | node.cond.Broadcast() 65 | } 66 | return 67 | case batch, ok := <-node.channel: 68 | if !ok { 69 | return 70 | } 71 | if node.ordered { 72 | node.cond.L.Lock() 73 | if batch.seqNo != node.run { 74 | panic("Invalid receive order in an ordered limited parallel pipeline node.") 75 | } 76 | node.run++ 77 | node.cond.L.Unlock() 78 | node.cond.Broadcast() 79 | } 80 | feed(p, node.receivers, index, batch.seqNo, batch.data) 81 | } 82 | } 83 | }() 84 | } 85 | } 86 | return 87 | } 88 | 89 | // Implements the Feed method of the Node interface. 90 | func (node *lparnode) Feed(p *Pipeline, _ int, seqNo int, data interface{}) { 91 | if node.ordered { 92 | node.cond.L.Lock() 93 | defer node.cond.L.Unlock() 94 | for { 95 | if node.run == seqNo { 96 | select { 97 | case <-p.ctx.Done(): 98 | return 99 | case node.channel <- dataBatch{seqNo, data}: 100 | return 101 | } 102 | } 103 | select { 104 | case <-p.ctx.Done(): 105 | return 106 | default: 107 | node.cond.Wait() 108 | } 109 | } 110 | } 111 | select { 112 | case <-p.ctx.Done(): 113 | return 114 | case node.channel <- dataBatch{seqNo, data}: 115 | return 116 | } 117 | } 118 | 119 | // Implements the End method of the Node interface. 120 | func (node *lparnode) End() { 121 | close(node.channel) 122 | node.waitGroup.Wait() 123 | for _, finalize := range node.finalizers { 124 | finalize() 125 | } 126 | node.receivers = nil 127 | node.finalizers = nil 128 | } 129 | -------------------------------------------------------------------------------- /pipeline/seqnode.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | type ( 8 | dataBatch struct { 9 | seqNo int 10 | data interface{} 11 | } 12 | 13 | seqnode struct { 14 | kind NodeKind 15 | channel chan dataBatch 16 | waitGroup sync.WaitGroup 17 | filters []Filter 18 | receivers []Receiver 19 | finalizers []Finalizer 20 | } 21 | ) 22 | 23 | // Ord creates an ordered node with the given filters. 24 | func Ord(filters ...Filter) Node { 25 | return &seqnode{kind: Ordered, filters: filters} 26 | } 27 | 28 | // Seq creates a sequential node with the given filters. 29 | func Seq(filters ...Filter) Node { 30 | return &seqnode{kind: Sequential, filters: filters} 31 | } 32 | 33 | // Implements the TryMerge method of the Node interface. 34 | func (node *seqnode) TryMerge(next Node) bool { 35 | if nxt, merge := next.(*seqnode); merge && (len(nxt.filters) > 0) { 36 | if nxt.kind == Ordered { 37 | node.kind = Ordered 38 | } 39 | node.filters = append(node.filters, nxt.filters...) 40 | node.receivers = append(node.receivers, nxt.receivers...) 41 | node.finalizers = append(node.finalizers, nxt.finalizers...) 42 | return true 43 | } 44 | return false 45 | } 46 | 47 | // Implements the Begin method of the Node interface. 48 | func (node *seqnode) Begin(p *Pipeline, index int, dataSize *int) (keep bool) { 49 | node.receivers, node.finalizers = ComposeFilters(p, node.kind, dataSize, node.filters) 50 | node.filters = nil 51 | if keep = (len(node.receivers) > 0) || (len(node.finalizers) > 0); keep { 52 | node.channel = make(chan dataBatch) 53 | node.waitGroup.Add(1) 54 | switch node.kind { 55 | case Sequential: 56 | go func() { 57 | defer node.waitGroup.Done() 58 | for { 59 | select { 60 | case <-p.ctx.Done(): 61 | return 62 | case batch, ok := <-node.channel: 63 | if !ok { 64 | return 65 | } 66 | feed(p, node.receivers, index, batch.seqNo, batch.data) 67 | } 68 | } 69 | }() 70 | case Ordered: 71 | go func() { 72 | defer node.waitGroup.Done() 73 | stash := make(map[int]interface{}) 74 | run := 0 75 | for { 76 | select { 77 | case <-p.ctx.Done(): 78 | return 79 | case batch, ok := <-node.channel: 80 | switch { 81 | case !ok: 82 | return 83 | case batch.seqNo < run: 84 | panic("Invalid receive order in an ordered pipeline node.") 85 | case batch.seqNo > run: 86 | stash[batch.seqNo] = batch.data 87 | default: 88 | feed(p, node.receivers, index, batch.seqNo, batch.data) 89 | checkStash: 90 | for { 91 | select { 92 | case <-p.ctx.Done(): 93 | return 94 | default: 95 | run++ 96 | data, ok := stash[run] 97 | if !ok { 98 | break checkStash 99 | } 100 | delete(stash, run) 101 | feed(p, node.receivers, index, run, data) 102 | } 103 | } 104 | } 105 | } 106 | } 107 | }() 108 | default: 109 | panic("Invalid NodeKind in a sequential pipeline node.") 110 | } 111 | } 112 | return 113 | } 114 | 115 | // Implements the Feed method of the Node interface. 116 | func (node *seqnode) Feed(p *Pipeline, _ int, seqNo int, data interface{}) { 117 | select { 118 | case <-p.ctx.Done(): 119 | return 120 | case node.channel <- dataBatch{seqNo, data}: 121 | return 122 | } 123 | } 124 | 125 | // Implements the End method of the Node interface. 126 | func (node *seqnode) End() { 127 | close(node.channel) 128 | node.waitGroup.Wait() 129 | for _, finalize := range node.finalizers { 130 | finalize() 131 | } 132 | node.receivers = nil 133 | node.finalizers = nil 134 | } 135 | -------------------------------------------------------------------------------- /sort/mergesort.go: -------------------------------------------------------------------------------- 1 | package sort 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/exascience/pargo/parallel" 7 | ) 8 | 9 | const msortGrainSize = 0x3000 10 | 11 | // StableSorter is a type, typically a collection, that can be sorted by 12 | // StableSort in this package. The methods require that ranges of elements of 13 | // the collection can be enumerated by integer indices. 14 | type StableSorter interface { 15 | SequentialSorter 16 | 17 | // NewTemp creates a new collection that can hold as many elements as the 18 | // original collection. This is temporary memory needed by StableSort, but 19 | // not needed anymore afterwards. The temporary collection does not need to 20 | // be initialized. 21 | NewTemp() StableSorter 22 | 23 | // Len is the number of elements in the collection. 24 | Len() int 25 | 26 | // Less reports whether the element with index i should sort before the 27 | // element with index j. 28 | Less(i, j int) bool 29 | 30 | // Assign returns a function that assigns ranges from source to the receiver 31 | // collection. The element with index i is the first element in the receiver 32 | // to assign to, and the element with index j is the first element in the 33 | // source collection to assign from, with len determining the number of 34 | // elements to assign. The effect should be the same as receiver[i:i+len] = 35 | // source[j:j+len]. 36 | Assign(source StableSorter) func(i, j, len int) 37 | } 38 | 39 | type sorter struct { 40 | less func(i, j int) bool 41 | assign func(i, j, len int) 42 | } 43 | 44 | func binarySearchEq(x int, T *sorter, p, r int) int { 45 | low, high := p, r+1 46 | if low > high { 47 | return low 48 | } 49 | for low < high { 50 | mid := (low + high) / 2 51 | if !T.less(mid, x) { 52 | high = mid 53 | } else { 54 | low = mid + 1 55 | } 56 | } 57 | return high 58 | } 59 | 60 | func binarySearchNeq(x int, T *sorter, p, r int) int { 61 | low, high := p, r+1 62 | if low > high { 63 | return low 64 | } 65 | for low < high { 66 | mid := (low + high) / 2 67 | if T.less(x, mid) { 68 | high = mid 69 | } else { 70 | low = mid + 1 71 | } 72 | } 73 | return high 74 | } 75 | 76 | func sMerge(T *sorter, p1, r1, p2, r2 int, A *sorter, p3 int) { 77 | for { 78 | if p2 > r2 { 79 | A.assign(p3, p1, r1+1-p1) 80 | return 81 | } 82 | 83 | q1 := p1 84 | for (p1 <= r1) && !T.less(p2, p1) { 85 | p1++ 86 | } 87 | n1 := p1 - q1 88 | A.assign(p3, q1, n1) 89 | p3 += n1 90 | 91 | if p1 > r1 { 92 | A.assign(p3, p2, r2+1-p2) 93 | return 94 | } 95 | 96 | q2 := p2 97 | for (p2 <= r2) && T.less(p2, p1) { 98 | p2++ 99 | } 100 | n2 := p2 - q2 101 | A.assign(p3, q2, n2) 102 | p3 += n2 103 | } 104 | } 105 | 106 | func pMerge(T *sorter, p1, r1, p2, r2 int, A *sorter, p3 int) { 107 | n1 := r1 - p1 + 1 108 | n2 := r2 - p2 + 1 109 | if (n1 + n2) < msortGrainSize { 110 | sMerge(T, p1, r1, p2, r2, A, p3) 111 | return 112 | } 113 | if n1 > n2 { 114 | if n1 == 0 { 115 | return 116 | } 117 | q1 := (p1 + r1) / 2 118 | q2 := binarySearchEq(q1, T, p2, r2) 119 | q3 := p3 + (q1 - p1) + (q2 - p2) 120 | A.assign(q3, q1, 1) 121 | parallel.Do( 122 | func() { pMerge(T, p1, q1-1, p2, q2-1, A, p3) }, 123 | func() { pMerge(T, q1+1, r1, q2, r2, A, q3+1) }, 124 | ) 125 | } else { 126 | if n2 == 0 { 127 | return 128 | } 129 | q2 := (p2 + r2) / 2 130 | q1 := binarySearchNeq(q2, T, p1, r1) 131 | q3 := p3 + (q1 - p1) + (q2 - p2) 132 | A.assign(q3, q2, 1) 133 | parallel.Do( 134 | func() { pMerge(T, p1, q1-1, p2, q2-1, A, p3) }, 135 | func() { pMerge(T, q1, r1, q2+1, r2, A, q3+1) }, 136 | ) 137 | } 138 | } 139 | 140 | // StableSort uses a parallel implementation of merge sort, also known as 141 | // cilksort. 142 | // 143 | // StableSort is only stable if data's SequentialSort method is stable. 144 | // 145 | // StableSort is good for large core counts and large collection sizes, but 146 | // needs a shallow copy of the data collection as additional temporary memory. 147 | func StableSort(data StableSorter) { 148 | // See https://en.wikipedia.org/wiki/Introduction_to_Algorithms and 149 | // https://www.clear.rice.edu/comp422/lecture-notes/ for details on the algorithm. 150 | size := data.Len() 151 | sSort := data.SequentialSort 152 | if size < msortGrainSize { 153 | sSort(0, size) 154 | return 155 | } 156 | var T, A *sorter 157 | var temp sync.WaitGroup 158 | temp.Add(1) 159 | go func() { 160 | defer temp.Done() 161 | a := data.NewTemp() 162 | T = &sorter{data.Less, data.Assign(a)} 163 | A = &sorter{a.Less, a.Assign(data)} 164 | }() 165 | var pSort func(int, int) 166 | pSort = func(index, size int) { 167 | if size < msortGrainSize { 168 | sSort(index, index+size) 169 | } else { 170 | q1 := size / 4 171 | q2 := q1 + q1 172 | q3 := q2 + q1 173 | parallel.Do( 174 | func() { pSort(index, q1) }, 175 | func() { pSort(index+q1, q1) }, 176 | func() { pSort(index+q2, q1) }, 177 | func() { pSort(index+q3, size-q3) }, 178 | ) 179 | temp.Wait() 180 | parallel.Do( 181 | func() { pMerge(T, index, index+q1-1, index+q1, index+q2-1, A, index) }, 182 | func() { pMerge(T, index+q2, index+q3-1, index+q3, index+size-1, A, index+q2) }, 183 | ) 184 | pMerge(A, index, index+q2-1, index+q2, index+size-1, T, index) 185 | } 186 | } 187 | pSort(0, size) 188 | } 189 | -------------------------------------------------------------------------------- /sort/sort.go: -------------------------------------------------------------------------------- 1 | // Package sort provides implementations of parallel sorting algorithms. 2 | package sort 3 | 4 | import ( 5 | "sort" 6 | "sync/atomic" 7 | 8 | "github.com/exascience/pargo/speculative" 9 | ) 10 | 11 | // SequentialSorter is a type, typically a collection, that can be sequentially 12 | // sorted. This is needed as a base case for the parallel sorting algorithms in 13 | // this package. It is recommended to implement this interface by using the 14 | // functions in the sort package of Go's standard library. 15 | type SequentialSorter interface { 16 | // Sort the range that starts at index i and ends at index j. If the 17 | // collection that is represented by this interface is a slice, then the 18 | // slice expression collection[i:j] returns the correct slice to be sorted. 19 | SequentialSort(i, j int) 20 | } 21 | 22 | const serialCutoff = 10 23 | 24 | // IsSorted determines in parallel whether data is already sorted. It attempts 25 | // to terminate early when the return value is false. 26 | func IsSorted(data sort.Interface) bool { 27 | size := data.Len() 28 | if size < qsortGrainSize { 29 | return sort.IsSorted(data) 30 | } 31 | for i := 1; i < serialCutoff; i++ { 32 | if data.Less(i, i-1) { 33 | return false 34 | } 35 | } 36 | var done int32 37 | defer atomic.StoreInt32(&done, 1) 38 | var pTest func(int, int) bool 39 | pTest = func(index, size int) bool { 40 | if size < qsortGrainSize { 41 | for i := index; i < index+size; i++ { 42 | if ((i % 1024) == 0) && (atomic.LoadInt32(&done) != 0) { 43 | return false 44 | } 45 | if data.Less(i, i-1) { 46 | return false 47 | } 48 | } 49 | return true 50 | } 51 | half := size / 2 52 | result := speculative.And( 53 | func() bool { return pTest(index, half) }, 54 | func() bool { return pTest(index+half, size-half) }, 55 | ) 56 | return result 57 | } 58 | return pTest(serialCutoff, size-serialCutoff) 59 | } 60 | 61 | // IntSlice attaches the methods of sort.Interface, SequentialSorter, Sorter, 62 | // and StableSorter to []int, sorting in increasing order. 63 | type IntSlice []int 64 | 65 | // SequentialSort implements the method of of the SequentialSorter interface. 66 | func (s IntSlice) SequentialSort(i, j int) { 67 | sort.Stable(sort.IntSlice(s[i:j])) 68 | } 69 | 70 | func (s IntSlice) Len() int { 71 | return len(s) 72 | } 73 | 74 | func (s IntSlice) Less(i, j int) bool { 75 | return s[i] < s[j] 76 | } 77 | 78 | func (s IntSlice) Swap(i, j int) { 79 | s[i], s[j] = s[j], s[i] 80 | } 81 | 82 | // NewTemp implements the method of the StableSorter interface. 83 | func (s IntSlice) NewTemp() StableSorter { 84 | return IntSlice(make([]int, len(s))) 85 | } 86 | 87 | // Assign implements the method of the StableSorter interface. 88 | func (s IntSlice) Assign(source StableSorter) func(i, j, len int) { 89 | dst, src := s, source.(IntSlice) 90 | return func(i, j, len int) { 91 | copy(dst[i:i+len], src[j:j+len]) 92 | } 93 | } 94 | 95 | // IntsAreSorted determines in parallel whether a slice of ints is already 96 | // sorted in increasing order. It attempts to terminate early when the return 97 | // value is false. 98 | func IntsAreSorted(a []int) bool { 99 | return IsSorted(IntSlice(a)) 100 | } 101 | 102 | // Float64Slice attaches the methods of sort.Interface, SequentialSorter, 103 | // Sorter, and StableSorter to []float64, sorting in increasing order. 104 | type Float64Slice []float64 105 | 106 | // SequentialSort implements the method of the SequentialSorter interface. 107 | func (s Float64Slice) SequentialSort(i, j int) { 108 | sort.Stable(sort.Float64Slice(s[i:j])) 109 | } 110 | 111 | func (s Float64Slice) Len() int { 112 | return len(s) 113 | } 114 | 115 | func (s Float64Slice) Less(i, j int) bool { 116 | return s[i] < s[j] 117 | } 118 | 119 | func (s Float64Slice) Swap(i, j int) { 120 | s[i], s[j] = s[j], s[i] 121 | } 122 | 123 | // NewTemp implements the method of the StableSorter interface. 124 | func (s Float64Slice) NewTemp() StableSorter { 125 | return Float64Slice(make([]float64, len(s))) 126 | } 127 | 128 | // Assign implements the method of the StableSorter interface. 129 | func (s Float64Slice) Assign(source StableSorter) func(i, j, len int) { 130 | dst, src := s, source.(Float64Slice) 131 | return func(i, j, len int) { 132 | copy(dst[i:i+len], src[j:j+len]) 133 | } 134 | } 135 | 136 | // Float64sAreSorted determines in parallel whether a slice of float64s is 137 | // already sorted in increasing order. It attempts to terminate early when the 138 | // return value is false. 139 | func Float64sAreSorted(a []float64) bool { 140 | return IsSorted(Float64Slice(a)) 141 | } 142 | 143 | // StringSlice attaches the methods of sort.Interface, SequentialSorter, Sorter, 144 | // and StableSorter to []string, sorting in increasing order. 145 | type StringSlice []string 146 | 147 | // SequentialSort implements the method of the SequentialSorter interface. 148 | func (s StringSlice) SequentialSort(i, j int) { 149 | sort.Stable(sort.StringSlice(s[i:j])) 150 | } 151 | 152 | func (s StringSlice) Len() int { 153 | return len(s) 154 | } 155 | 156 | func (s StringSlice) Less(i, j int) bool { 157 | return s[i] < s[j] 158 | } 159 | 160 | func (s StringSlice) Swap(i, j int) { 161 | s[i], s[j] = s[j], s[i] 162 | } 163 | 164 | // NewTemp implements the method of the StableSorter interface. 165 | func (s StringSlice) NewTemp() StableSorter { 166 | return StringSlice(make([]string, len(s))) 167 | } 168 | 169 | // Assign implements the method of the StableSorter interface. 170 | func (s StringSlice) Assign(source StableSorter) func(i, j, len int) { 171 | dst, src := s, source.(StringSlice) 172 | return func(i, j, len int) { 173 | copy(dst[i:i+len], src[j:j+len]) 174 | } 175 | } 176 | 177 | // StringsAreSorted determines in parallel whether a slice of strings is already 178 | // sorted in increasing order. It attempts to terminate early when the return 179 | // value is false. 180 | func StringsAreSorted(a []string) bool { 181 | return IsSorted(StringSlice(a)) 182 | } 183 | -------------------------------------------------------------------------------- /sort/sort_test.go: -------------------------------------------------------------------------------- 1 | package sort 2 | 3 | import ( 4 | "bytes" 5 | "math/rand" 6 | "sort" 7 | "testing" 8 | ) 9 | 10 | type ( 11 | By func(i, j int) bool 12 | 13 | IntSliceSorter struct { 14 | slice []int 15 | by By 16 | } 17 | ) 18 | 19 | func (s IntSliceSorter) NewTemp() StableSorter { 20 | return IntSliceSorter{make([]int, len(s.slice)), s.by} 21 | } 22 | 23 | func (s IntSliceSorter) Len() int { 24 | return len(s.slice) 25 | } 26 | 27 | func (s IntSliceSorter) Less(i, j int) bool { 28 | return s.by(s.slice[i], s.slice[j]) 29 | } 30 | 31 | func (s IntSliceSorter) Swap(i, j int) { 32 | s.slice[i], s.slice[j] = s.slice[j], s.slice[i] 33 | } 34 | 35 | func (s IntSliceSorter) Assign(t StableSorter) func(i, j, len int) { 36 | dst, src := s.slice, t.(IntSliceSorter).slice 37 | return func(i, j, len int) { 38 | for k := 0; k < len; k++ { 39 | dst[i+k] = src[j+k] 40 | } 41 | } 42 | } 43 | 44 | func (s IntSliceSorter) SequentialSort(i, j int) { 45 | slice, by := s.slice[i:j], s.by 46 | sort.Slice(slice, func(i, j int) bool { 47 | return by(slice[i], slice[j]) 48 | }) 49 | } 50 | 51 | func (by By) SequentialSort(slice []int) { 52 | sort.Sort(IntSliceSorter{slice, by}) 53 | } 54 | 55 | func (by By) ParallelStableSort(slice []int) { 56 | StableSort(IntSliceSorter{slice, by}) 57 | } 58 | 59 | func (by By) ParallelSort(slice []int) { 60 | Sort(IntSliceSorter{slice, by}) 61 | } 62 | 63 | func (by By) IsSorted(slice []int) bool { 64 | return sort.IsSorted(IntSliceSorter{slice, by}) 65 | } 66 | 67 | func makeRandomSlice(size, limit int) []int { 68 | result := make([]int, size) 69 | for i := 0; i < size; i++ { 70 | result[i] = rand.Intn(limit) 71 | } 72 | return result 73 | } 74 | 75 | func TestSort(t *testing.T) { 76 | orgSlice := makeRandomSlice(100*0x6000, 100*100*0x6000) 77 | s1 := make([]int, len(orgSlice)) 78 | s2 := make([]int, len(orgSlice)) 79 | copy(s1, orgSlice) 80 | copy(s2, orgSlice) 81 | 82 | t.Run("ParallelStableSort", func(t *testing.T) { 83 | By(func(i, j int) bool { return i < j }).ParallelStableSort(s1) 84 | if !By(func(i, j int) bool { return i < j }).IsSorted(s1) { 85 | t.Errorf("parallel stable sort incorrect") 86 | } 87 | }) 88 | 89 | t.Run("ParallelSort", func(t *testing.T) { 90 | By(func(i, j int) bool { return i < j }).ParallelSort(s2) 91 | if !By(func(i, j int) bool { return i < j }).IsSorted(s2) { 92 | t.Errorf("parallel sort incorrect") 93 | } 94 | }) 95 | } 96 | 97 | func TestIntSort(t *testing.T) { 98 | orgSlice := makeRandomSlice(100*0x6000, 100*100*0x6000) 99 | s1 := make([]int, len(orgSlice)) 100 | s2 := make([]int, len(orgSlice)) 101 | copy(s1, orgSlice) 102 | copy(s2, orgSlice) 103 | 104 | t.Run("ParallelStableSort IntSlice", func(t *testing.T) { 105 | StableSort(IntSlice(s1)) 106 | if !sort.IntsAreSorted(s1) { 107 | t.Errorf("parallel stable sort on IntSlice incorrect") 108 | } 109 | if !IntsAreSorted(s1) { 110 | t.Errorf("parallel IntsAreSorted incorrect") 111 | } 112 | }) 113 | 114 | t.Run("ParallelSort IntSlice", func(t *testing.T) { 115 | Sort(IntSlice(s2)) 116 | if !sort.IntsAreSorted(s2) { 117 | t.Errorf("parallel sort on IntSlice incorrect") 118 | } 119 | if !IntsAreSorted(s2) { 120 | t.Errorf("parallel IntsAreSorted incorrect") 121 | } 122 | }) 123 | } 124 | 125 | func makeRandomFloat64Slice(size int) []float64 { 126 | result := make([]float64, size) 127 | for i := 0; i < size; i++ { 128 | result[i] = rand.NormFloat64() 129 | } 130 | return result 131 | } 132 | 133 | func TestFloat64Sort(t *testing.T) { 134 | orgSlice := makeRandomFloat64Slice(100 * 0x6000) 135 | s1 := make([]float64, len(orgSlice)) 136 | s2 := make([]float64, len(orgSlice)) 137 | copy(s1, orgSlice) 138 | copy(s2, orgSlice) 139 | 140 | t.Run("ParallelStableSort Float64Slice", func(t *testing.T) { 141 | StableSort(Float64Slice(s1)) 142 | if !sort.Float64sAreSorted(s1) { 143 | t.Errorf("parallel stable sort on Float64Slice incorrect") 144 | } 145 | if !Float64sAreSorted(s1) { 146 | t.Errorf("parallel Float64sAreSorted incorrect") 147 | } 148 | }) 149 | 150 | t.Run("ParallelSort Float64Slice", func(t *testing.T) { 151 | Sort(Float64Slice(s2)) 152 | if !sort.Float64sAreSorted(s2) { 153 | t.Errorf("parallel sort on Float64Slice incorrect") 154 | } 155 | if !Float64sAreSorted(s2) { 156 | t.Errorf("parallel Float64sAreSorted incorrect") 157 | } 158 | }) 159 | } 160 | 161 | func makeRandomStringSlice(size, lenlimit int, limit int32) []string { 162 | result := make([]string, size) 163 | for i := 0; i < size; i++ { 164 | var buf bytes.Buffer 165 | len := rand.Intn(lenlimit) 166 | for j := 0; j < len; j++ { 167 | buf.WriteRune(rand.Int31n(limit)) 168 | } 169 | result[i] = buf.String() 170 | } 171 | return result 172 | } 173 | 174 | func TestStringSort(t *testing.T) { 175 | orgSlice := makeRandomStringSlice(100*0x6000, 256, 16384) 176 | s1 := make([]string, len(orgSlice)) 177 | s2 := make([]string, len(orgSlice)) 178 | copy(s1, orgSlice) 179 | copy(s2, orgSlice) 180 | 181 | t.Run("ParallelStableSort StringSlice", func(t *testing.T) { 182 | StableSort(StringSlice(s1)) 183 | if !sort.StringsAreSorted(s1) { 184 | t.Errorf("parallel stable sort on StringSlice incorrect") 185 | } 186 | if !StringsAreSorted(s1) { 187 | t.Errorf("parallel StringsAreSorted incorrect") 188 | } 189 | }) 190 | 191 | t.Run("ParallelSort StringSlice", func(t *testing.T) { 192 | Sort(StringSlice(s2)) 193 | if !sort.StringsAreSorted(s2) { 194 | t.Errorf("parallel sort on StringSlice incorrect") 195 | } 196 | if !StringsAreSorted(s2) { 197 | t.Errorf("parallel StringsAreSorted incorrect") 198 | } 199 | }) 200 | } 201 | 202 | type ( 203 | box struct { 204 | primary, secondary int 205 | } 206 | 207 | boxSlice []box 208 | ) 209 | 210 | func makeRandomBoxSlice(size int) boxSlice { 211 | result := make([]box, size) 212 | half := ((size - 1) / 2) + 1 213 | for i := 0; i < size; i++ { 214 | result[i].primary = rand.Intn(half) 215 | result[i].secondary = i + 1 216 | } 217 | return result 218 | } 219 | 220 | func (s boxSlice) NewTemp() StableSorter { 221 | return boxSlice(make([]box, len(s))) 222 | } 223 | 224 | func (s boxSlice) Len() int { 225 | return len(s) 226 | } 227 | 228 | func (s boxSlice) Less(i, j int) bool { 229 | return s[i].primary < s[j].primary 230 | } 231 | 232 | func (s boxSlice) Assign(source StableSorter) func(i, j, len int) { 233 | dst, src := s, source.(boxSlice) 234 | return func(i, j, len int) { 235 | for k := 0; k < len; k++ { 236 | dst[i+k] = src[j+k] 237 | } 238 | } 239 | } 240 | 241 | func (s boxSlice) SequentialSort(i, j int) { 242 | slice := s[i:j] 243 | sort.SliceStable(slice, func(i, j int) bool { 244 | return slice[i].primary < slice[j].primary 245 | }) 246 | } 247 | 248 | func checkStable(b boxSlice) bool { 249 | m := make(map[int]int) 250 | for _, el := range b { 251 | if m[el.primary] < el.secondary { 252 | m[el.primary] = el.secondary 253 | } else { 254 | return false 255 | } 256 | } 257 | return true 258 | } 259 | 260 | func TestStableSort(t *testing.T) { 261 | orgSlice := makeRandomBoxSlice(100 * 0x6000) 262 | s1 := make(boxSlice, len(orgSlice)) 263 | copy(s1, orgSlice) 264 | 265 | t.Run("ParallelStableSort boxSlice", func(t *testing.T) { 266 | StableSort(s1) 267 | if !sort.SliceIsSorted(s1, func(i, j int) bool { 268 | return s1[i].primary < s1[j].primary 269 | }) { 270 | t.Errorf("parallel stable sort on boxSlice incorrect") 271 | } 272 | }) 273 | 274 | t.Run("CheckStable ParallelStableSort boxSlice", func(t *testing.T) { 275 | if !checkStable(s1) { 276 | t.Errorf("parallel stable sort on boxSlice not stable") 277 | } 278 | }) 279 | } 280 | 281 | func BenchmarkSort(b *testing.B) { 282 | orgSlice := makeRandomSlice(100*0x6000, 100*100*0x6000) 283 | s1 := make([]int, len(orgSlice)) 284 | s2 := make([]int, len(orgSlice)) 285 | s3 := make([]int, len(orgSlice)) 286 | 287 | b.Run("SequentialSort", func(b *testing.B) { 288 | for i := 0; i < b.N; i++ { 289 | b.StopTimer() 290 | copy(s1, orgSlice) 291 | b.StartTimer() 292 | By(func(i, j int) bool { return i < j }).SequentialSort(s1) 293 | } 294 | }) 295 | 296 | b.Run("ParallelStableSort", func(b *testing.B) { 297 | for i := 0; i < b.N; i++ { 298 | b.StopTimer() 299 | copy(s2, orgSlice) 300 | b.StartTimer() 301 | By(func(i, j int) bool { return i < j }).ParallelStableSort(s2) 302 | } 303 | }) 304 | 305 | b.Run("ParallelSort", func(b *testing.B) { 306 | for i := 0; i < b.N; i++ { 307 | b.StopTimer() 308 | copy(s3, orgSlice) 309 | b.StartTimer() 310 | By(func(i, j int) bool { return i < j }).ParallelSort(s3) 311 | } 312 | }) 313 | } 314 | -------------------------------------------------------------------------------- /pipeline/source.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "io" 7 | "reflect" 8 | ) 9 | 10 | // A Source represents an object that can generate data batches for pipelines. 11 | type Source interface { 12 | // Err returns an error value or nil 13 | Err() error 14 | 15 | // Prepare receives a pipeline context and informs the pipeline what the 16 | // total expected size of all data batches is. The return value is -1 if the 17 | // total size is unknown or difficult to determine. 18 | Prepare(ctx context.Context) (size int) 19 | 20 | // Fetch gets a data batch of the requested size from the source. It returns 21 | // the size of the data batch that it was actually able to fetch. It returns 22 | // 0 if there is no more data to be fetched from the source; the pipeline 23 | // will then make no further attempts to fetch more elements. 24 | Fetch(size int) (fetched int) 25 | 26 | // Data returns the last fetched data batch. 27 | Data() interface{} 28 | } 29 | 30 | type sliceSource struct { 31 | value reflect.Value 32 | size int 33 | data interface{} 34 | } 35 | 36 | func newSliceSource(value reflect.Value) *sliceSource { 37 | size := value.Len() 38 | return &sliceSource{value: value.Slice(0, size), size: size} 39 | } 40 | 41 | func (src *sliceSource) Err() error { 42 | return nil 43 | } 44 | 45 | func (src *sliceSource) Prepare(_ context.Context) int { 46 | return src.size 47 | } 48 | 49 | func (src *sliceSource) Fetch(n int) (fetched int) { 50 | switch { 51 | case src.size == 0: 52 | src.data = nil 53 | case n >= src.size: 54 | fetched = src.size 55 | src.data = src.value.Interface() 56 | src.value = reflect.ValueOf(nil) 57 | src.size = 0 58 | default: 59 | fetched = n 60 | src.data = src.value.Slice(0, n).Interface() 61 | src.value = src.value.Slice(n, src.size) 62 | src.size -= n 63 | } 64 | return 65 | } 66 | 67 | func (src *sliceSource) Data() interface{} { 68 | return src.data 69 | } 70 | 71 | type chanSource struct { 72 | cases []reflect.SelectCase 73 | zero reflect.Value 74 | data interface{} 75 | } 76 | 77 | func newChanSource(value reflect.Value) *chanSource { 78 | zeroElem := value.Type().Elem() 79 | return &chanSource{ 80 | cases: []reflect.SelectCase{{Dir: reflect.SelectRecv, Chan: value}}, 81 | zero: reflect.Zero(reflect.SliceOf(zeroElem)), 82 | } 83 | } 84 | 85 | func (src *chanSource) Err() error { 86 | return nil 87 | } 88 | 89 | func (src *chanSource) Prepare(ctx context.Context) (size int) { 90 | src.cases = append(src.cases, reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(ctx.Done())}) 91 | return -1 92 | } 93 | 94 | func (src *chanSource) Fetch(n int) (fetched int) { 95 | data := src.zero 96 | for fetched = 0; fetched < n; fetched++ { 97 | if chosen, element, ok := reflect.Select(src.cases); (chosen == 0) && ok { 98 | data = reflect.Append(data, element) 99 | } else { 100 | break 101 | } 102 | } 103 | src.data = data.Interface() 104 | return 105 | } 106 | 107 | func (src *chanSource) Data() interface{} { 108 | return src.data 109 | } 110 | 111 | func reflectSource(source interface{}) Source { 112 | switch value := reflect.ValueOf(source); value.Kind() { 113 | case reflect.Array, reflect.Slice, reflect.String: 114 | return newSliceSource(value) 115 | case reflect.Chan: 116 | return newChanSource(value) 117 | default: 118 | panic("A default pipeline source is not of kind Array, Slice, String, or Chan.") 119 | } 120 | } 121 | 122 | // Scanner is a wrapper around bufio.Scanner so it can act as a data source for 123 | // pipelines. It fetches strings. 124 | type Scanner struct { 125 | *bufio.Scanner 126 | data interface{} 127 | } 128 | 129 | // NewScanner returns a new Scanner to read from r. The split function defaults 130 | // to bufio.ScanLines. 131 | func NewScanner(r io.Reader) *Scanner { 132 | return &Scanner{Scanner: bufio.NewScanner(r)} 133 | } 134 | 135 | // Prepare implements the method of the Source interface. 136 | func (src *Scanner) Prepare(_ context.Context) (size int) { 137 | return -1 138 | } 139 | 140 | // Fetch implements the method of the Source interface. 141 | func (src *Scanner) Fetch(n int) (fetched int) { 142 | var data []string 143 | for fetched = 0; fetched < n; fetched++ { 144 | if src.Scan() { 145 | data = append(data, src.Text()) 146 | } else { 147 | break 148 | } 149 | } 150 | src.data = data 151 | return 152 | } 153 | 154 | // Data implements the method of the Source interface. 155 | func (src *Scanner) Data() interface{} { 156 | return src.data 157 | } 158 | 159 | // BytesScanner is a wrapper around bufio.Scanner so it can act as a data source 160 | // for pipelines. It fetches slices of bytes. 161 | type BytesScanner struct { 162 | *bufio.Scanner 163 | data interface{} 164 | } 165 | 166 | // NewBytesScanner returns a new Scanner to read from r. The split function 167 | // defaults to bufio.ScanLines. 168 | func NewBytesScanner(r io.Reader) *BytesScanner { 169 | return &BytesScanner{Scanner: bufio.NewScanner(r)} 170 | } 171 | 172 | // Prepare implements the method of the Source interface. 173 | func (src *BytesScanner) Prepare(_ context.Context) (size int) { 174 | return -1 175 | } 176 | 177 | // Fetch implements the method of the Source interface. 178 | func (src *BytesScanner) Fetch(n int) (fetched int) { 179 | var data [][]byte 180 | for fetched = 0; fetched < n; fetched++ { 181 | if src.Scan() { 182 | data = append(data, append([]byte(nil), src.Bytes()...)) 183 | } else { 184 | break 185 | } 186 | } 187 | src.data = data 188 | return 189 | } 190 | 191 | // Data implements the method of the Source interface. 192 | func (src *BytesScanner) Data() interface{} { 193 | return src.data 194 | } 195 | 196 | // Func is a generic source that generates data batches 197 | // by repeatedly calling a function. 198 | type Func struct { 199 | data interface{} 200 | err error 201 | size int 202 | fetch func(size int) (data interface{}, fetched int, err error) 203 | } 204 | 205 | // NewFunc returns a new Func to generate data batches 206 | // by repeatedly calling fetch. 207 | // 208 | // The size parameter informs the pipeline what the total 209 | // expected size of all data batches is. Pass -1 if the 210 | // total size is unknown or difficult to determine. 211 | // 212 | // The fetch function returns a data batch of the requested 213 | // size. It returns the size of the data batch that it was 214 | // actually able to fetch. It returns 0 if there is no more 215 | // data to be fetched from the source; the pipeline will 216 | // then make no further attempts to fetch more elements. 217 | // 218 | // The fetch function can also return an error if necessary. 219 | func NewFunc(size int, fetch func(size int) (data interface{}, fetched int, err error)) *Func { 220 | return &Func{size: size, fetch: fetch} 221 | } 222 | 223 | // Err implements the method of the Source interface. 224 | func (f *Func) Err() error { 225 | return f.err 226 | } 227 | 228 | // Prepare implements the method of the Source interface. 229 | func (f *Func) Prepare(_ context.Context) int { 230 | return f.size 231 | } 232 | 233 | // Fetch implements the method of the Source interface. 234 | func (f *Func) Fetch(size int) (fetched int) { 235 | f.data, fetched, f.err = f.fetch(size) 236 | return 237 | } 238 | 239 | // Data implements the method of the Source interface. 240 | func (f *Func) Data() interface{} { 241 | return f.data 242 | } 243 | 244 | // SingletonChan is similar to a regular chan source, 245 | // except it only accepts and passes through single 246 | // elements instead of creating slices of elements 247 | // from the input channel. 248 | type SingletonChan struct { 249 | cases []reflect.SelectCase 250 | zero reflect.Value 251 | data interface{} 252 | } 253 | 254 | // NewSingletonChan returns a new SingletonChan to read from 255 | // the given channel. 256 | func NewSingletonChan(channel interface{}) *SingletonChan { 257 | value := reflect.ValueOf(channel) 258 | if value.Kind() != reflect.Chan { 259 | panic("parameter for pargo.pipeline.NewSingletonChan is not a channel") 260 | } 261 | return &SingletonChan{ 262 | cases: []reflect.SelectCase{{Dir: reflect.SelectRecv, Chan: value}}, 263 | zero: reflect.Zero(value.Type().Elem()), 264 | } 265 | } 266 | 267 | // Err implements the method of the Source interface. 268 | func (src *SingletonChan) Err() error { 269 | return nil 270 | } 271 | 272 | // Prepare implements the method of the Source interface. 273 | func (src *SingletonChan) Prepare(ctx context.Context) (size int) { 274 | src.cases = append(src.cases, reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(ctx.Done())}) 275 | return -1 276 | } 277 | 278 | // Fetch implements the method of the Source interface. 279 | func (src *SingletonChan) Fetch(n int) (fetched int) { 280 | if chosen, element, ok := reflect.Select(src.cases); (chosen == 0) && ok { 281 | src.data = element.Interface() 282 | return 1 283 | } 284 | src.data = src.zero 285 | return 0 286 | } 287 | 288 | // Data implements the method of the Source interface. 289 | func (src *SingletonChan) Data() interface{} { 290 | return src.data 291 | } 292 | -------------------------------------------------------------------------------- /pipeline/filters.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "errors" 5 | "reflect" 6 | "sync" 7 | "sync/atomic" 8 | ) 9 | 10 | // NewNode creates a node of the given kind, with the given filters. 11 | // 12 | // It is often more convenient to use one of the Ord, Seq, or Par methods. 13 | func NewNode(kind NodeKind, filters ...Filter) Node { 14 | switch kind { 15 | case Ordered, Sequential: 16 | return &seqnode{kind: kind, filters: filters} 17 | case Parallel: 18 | return &parnode{filters: filters} 19 | default: 20 | panic("Invalid NodeKind in pipeline.NewNode.") 21 | } 22 | } 23 | 24 | // Identity is a filter that passes data batches through unmodified. 25 | // This filter will be optimized away in a pipeline, so it 26 | // does not hurt to add it. 27 | func Identity(_ *Pipeline, _ NodeKind, _ *int) (_ Receiver, _ Finalizer) { 28 | return 29 | } 30 | 31 | // Receive creates a Filter that returns the given receiver and a nil finalizer. 32 | func Receive(receive Receiver) Filter { 33 | return func(_ *Pipeline, _ NodeKind, _ *int) (receiver Receiver, _ Finalizer) { 34 | receiver = receive 35 | return 36 | } 37 | } 38 | 39 | // Finalize creates a filter that returns a nil receiver and the given 40 | // finalizer. 41 | func Finalize(finalize Finalizer) Filter { 42 | return func(_ *Pipeline, _ NodeKind, _ *int) (_ Receiver, finalizer Finalizer) { 43 | finalizer = finalize 44 | return 45 | } 46 | } 47 | 48 | // ReceiveAndFinalize creates a filter that returns the given filter and 49 | // receiver. 50 | func ReceiveAndFinalize(receive Receiver, finalize Finalizer) Filter { 51 | return func(_ *Pipeline, _ NodeKind, _ *int) (receiver Receiver, finalizer Finalizer) { 52 | receiver = receive 53 | finalizer = finalize 54 | return 55 | } 56 | } 57 | 58 | // A Predicate is a function that is passed a data batch and returns a boolean 59 | // value. 60 | // 61 | // In most cases, it will cast the data parameter to a specific slice type and 62 | // check a predicate on each element of the slice. 63 | type Predicate func(data interface{}) bool 64 | 65 | // Every creates a filter that sets the result pointer to true if the given 66 | // predicate returns true for every data batch. If cancelWhenKnown is true, this 67 | // filter cancels the pipeline as soon as the predicate returns false on a data 68 | // batch. 69 | func Every(result *bool, cancelWhenKnown bool, predicate Predicate) Filter { 70 | *result = true 71 | return func(pipeline *Pipeline, kind NodeKind, _ *int) (receiver Receiver, finalizer Finalizer) { 72 | switch kind { 73 | case Parallel: 74 | res := int32(1) 75 | receiver = func(_ int, data interface{}) interface{} { 76 | if !predicate(data) { 77 | atomic.StoreInt32(&res, 0) 78 | if cancelWhenKnown { 79 | pipeline.Cancel() 80 | } 81 | } 82 | return data 83 | } 84 | finalizer = func() { 85 | if atomic.LoadInt32(&res) == 0 { 86 | *result = false 87 | } 88 | } 89 | default: 90 | receiver = func(_ int, data interface{}) interface{} { 91 | if !predicate(data) { 92 | *result = false 93 | if cancelWhenKnown { 94 | pipeline.Cancel() 95 | } 96 | } 97 | return data 98 | } 99 | } 100 | return 101 | } 102 | } 103 | 104 | // NotEvery creates a filter that sets the result pointer to true if the given 105 | // predicate returns false for at least one of the data batches it is passed. If 106 | // cancelWhenKnown is true, this filter cancels the pipeline as soon as the 107 | // predicate returns false on a data batch. 108 | func NotEvery(result *bool, cancelWhenKnown bool, predicate Predicate) Filter { 109 | *result = false 110 | return func(pipeline *Pipeline, kind NodeKind, _ *int) (receiver Receiver, finalizer Finalizer) { 111 | switch kind { 112 | case Parallel: 113 | res := int32(0) 114 | receiver = func(_ int, data interface{}) interface{} { 115 | if !predicate(data) { 116 | atomic.StoreInt32(&res, 1) 117 | if cancelWhenKnown { 118 | pipeline.Cancel() 119 | } 120 | } 121 | return data 122 | } 123 | finalizer = func() { 124 | if atomic.LoadInt32(&res) == 1 { 125 | *result = true 126 | } 127 | } 128 | default: 129 | receiver = func(_ int, data interface{}) interface{} { 130 | if !predicate(data) { 131 | *result = true 132 | if cancelWhenKnown { 133 | pipeline.Cancel() 134 | } 135 | } 136 | return data 137 | } 138 | } 139 | return 140 | } 141 | } 142 | 143 | // Some creates a filter that sets the result pointer to true if the given 144 | // predicate returns true for at least one of the data batches it is passed. If 145 | // cancelWhenKnown is true, this filter cancels the pipeline as soon as the 146 | // predicate returns true on a data batch. 147 | func Some(result *bool, cancelWhenKnown bool, predicate Predicate) Filter { 148 | *result = false 149 | return func(pipeline *Pipeline, kind NodeKind, _ *int) (receiver Receiver, finalizer Finalizer) { 150 | switch kind { 151 | case Parallel: 152 | res := int32(0) 153 | receiver = func(_ int, data interface{}) interface{} { 154 | if predicate(data) { 155 | atomic.StoreInt32(&res, 1) 156 | if cancelWhenKnown { 157 | pipeline.Cancel() 158 | } 159 | } 160 | return data 161 | } 162 | finalizer = func() { 163 | if atomic.LoadInt32(&res) == 1 { 164 | *result = true 165 | } 166 | } 167 | default: 168 | receiver = func(_ int, data interface{}) interface{} { 169 | if predicate(data) { 170 | *result = true 171 | if cancelWhenKnown { 172 | pipeline.Cancel() 173 | } 174 | } 175 | return data 176 | } 177 | } 178 | return 179 | } 180 | } 181 | 182 | // NotAny creates a filter that sets the result pointer to true if the given 183 | // predicate returns false for every data batch. If cancelWhenKnown is true, 184 | // this filter cancels the pipeline as soon as the predicate returns true on a 185 | // data batch. 186 | func NotAny(result *bool, cancelWhenKnown bool, predicate Predicate) Filter { 187 | *result = true 188 | return func(pipeline *Pipeline, kind NodeKind, _ *int) (receiver Receiver, finalizer Finalizer) { 189 | switch kind { 190 | case Parallel: 191 | res := int32(1) 192 | receiver = func(_ int, data interface{}) interface{} { 193 | if predicate(data) { 194 | atomic.StoreInt32(&res, 0) 195 | if cancelWhenKnown { 196 | pipeline.Cancel() 197 | } 198 | } 199 | return data 200 | } 201 | finalizer = func() { 202 | if atomic.LoadInt32(&res) == 0 { 203 | *result = false 204 | } 205 | } 206 | default: 207 | receiver = func(_ int, data interface{}) interface{} { 208 | if predicate(data) { 209 | *result = false 210 | if cancelWhenKnown { 211 | pipeline.Cancel() 212 | } 213 | } 214 | return data 215 | } 216 | } 217 | return 218 | } 219 | } 220 | 221 | // Slice creates a filter that appends all the data batches it sees to the 222 | // result. The result must represent a settable slice, for example by using the 223 | // address operator & on a given slice. 224 | func Slice(result interface{}) Filter { 225 | res := reflect.ValueOf(result).Elem() 226 | return func(pipeline *Pipeline, kind NodeKind, _ *int) (receiver Receiver, finalizer Finalizer) { 227 | if (res.Kind() != reflect.Slice) && !res.CanSet() { 228 | pipeline.SetErr(errors.New("result is not a settable slice in Pipeline.ToSlice")) 229 | return 230 | } 231 | switch kind { 232 | case Parallel: 233 | var m sync.Mutex 234 | receiver = func(_ int, data interface{}) interface{} { 235 | if data != nil { 236 | d := reflect.ValueOf(data) 237 | m.Lock() 238 | defer m.Unlock() 239 | res.Set(reflect.AppendSlice(res, d)) 240 | } 241 | return data 242 | } 243 | default: 244 | receiver = func(_ int, data interface{}) interface{} { 245 | if data != nil { 246 | res.Set(reflect.AppendSlice(res, reflect.ValueOf(data))) 247 | } 248 | return data 249 | } 250 | } 251 | return 252 | } 253 | } 254 | 255 | // Count creates a filter that sets the result pointer to the total size of all 256 | // data batches it sees. 257 | func Count(result *int) Filter { 258 | return func(pipeline *Pipeline, kind NodeKind, size *int) (receiver Receiver, finalizer Finalizer) { 259 | switch { 260 | case *size >= 0: 261 | *result = *size 262 | case kind == Parallel: 263 | var res = int64(0) 264 | receiver = func(_ int, data interface{}) interface{} { 265 | if data != nil { 266 | d := reflect.ValueOf(data) 267 | atomic.AddInt64(&res, int64(d.Len())) 268 | } 269 | return data 270 | } 271 | finalizer = func() { 272 | *result = int(atomic.LoadInt64(&res)) 273 | } 274 | default: 275 | receiver = func(_ int, data interface{}) interface{} { 276 | if data != nil { 277 | d := reflect.ValueOf(data) 278 | *result += d.Len() 279 | } 280 | return data 281 | } 282 | } 283 | return 284 | } 285 | } 286 | 287 | // Limit creates an ordered node with a filter that caps the total size of all 288 | // data batches it passes to the next filter in the pipeline to the given limit. 289 | // If cancelWhenKnown is true, this filter cancels the pipeline as soon as the 290 | // limit is reached. If limit is negative, all data is passed through 291 | // unmodified. 292 | func Limit(limit int, cancelWhenReached bool) Node { 293 | return Ord(func(pipeline *Pipeline, _ NodeKind, size *int) (receiver Receiver, _ Finalizer) { 294 | switch { 295 | case limit < 0: // unlimited 296 | case limit == 0: 297 | *size = 0 298 | if cancelWhenReached { 299 | pipeline.Cancel() 300 | } 301 | receiver = func(_ int, _ interface{}) interface{} { return nil } 302 | case (*size < 0) || (*size > limit): 303 | if *size > 0 { 304 | *size = limit 305 | } 306 | seen := 0 307 | receiver = func(_ int, data interface{}) (result interface{}) { 308 | if seen >= limit { 309 | return 310 | } 311 | d := reflect.ValueOf(data) 312 | l := d.Len() 313 | if (seen + l) > limit { 314 | result = d.Slice(0, limit-seen).Interface() 315 | seen = limit 316 | } else { 317 | result = data 318 | seen += l 319 | } 320 | if cancelWhenReached && (seen == limit) { 321 | pipeline.Cancel() 322 | } 323 | return 324 | } 325 | } 326 | return 327 | }) 328 | } 329 | 330 | // Skip creates an ordered node with a filter that skips the first n elements 331 | // from the data batches it passes to the next filter in the pipeline. If n is 332 | // negative, no data is passed through, and the error value of the pipeline is 333 | // set to a non-nil value. 334 | func Skip(n int) Node { 335 | return Ord(func(pipeline *Pipeline, _ NodeKind, size *int) (receiver Receiver, _ Finalizer) { 336 | switch { 337 | case n < 0: // skip everything 338 | *size = 0 339 | pipeline.SetErr(errors.New("skip filter with unknown size")) 340 | receiver = func(_ int, _ interface{}) interface{} { return nil } 341 | case n == 0: // nothing to skip 342 | case (*size < 0) || (*size > n): 343 | if *size > 0 { 344 | *size = n 345 | } 346 | seen := 0 347 | receiver = func(_ int, data interface{}) (result interface{}) { 348 | if seen >= n { 349 | result = data 350 | return 351 | } 352 | d := reflect.ValueOf(data) 353 | l := d.Len() 354 | if (seen + l) > n { 355 | result = d.Slice(n-seen, l).Interface() 356 | seen = n 357 | } else { 358 | seen += l 359 | } 360 | return 361 | } 362 | case *size <= n: 363 | *size = 0 364 | receiver = func(_ int, _ interface{}) interface{} { return nil } 365 | } 366 | return 367 | }) 368 | } 369 | -------------------------------------------------------------------------------- /pipeline/pipeline.go: -------------------------------------------------------------------------------- 1 | // Package pipeline provides means to construct and execute parallel pipelines. 2 | // 3 | // A Pipeline feeds batches of data through several functions that can be 4 | // specified to be executed in encounter order, in arbitrary sequential order, 5 | // or in parallel. Ordered, sequential, or parallel stages can arbitrarily 6 | // alternate. 7 | // 8 | // A Pipeline consists of a Source object, and several Node objects. 9 | // 10 | // Source objects that are supported by this implementation are arrays, slices, 11 | // strings, channels, and bufio.Scanner objects, but other kinds of Source 12 | // objects can be added by user programs. 13 | // 14 | // Node objects can be specified to receive batches from the input source either 15 | // sequentially in encounter order, which is always the same order in which they 16 | // were originally encountered at the source; sequentially, but in arbitrary 17 | // order; or in parallel. Ordered nodes always receive batches in encounter 18 | // order even if they are preceded by arbitrary sequential, or even parallel 19 | // nodes. 20 | // 21 | // Node objects consist of filters, which are pairs of receiver and finalizer 22 | // functions. Each batch is passed to each receiver function, which can 23 | // transform and modify the batch for the next receiver function in the 24 | // pipeline. Each finalizer function is called once when all batches have been 25 | // passed through all receiver functions. 26 | // 27 | // Pipelines do not have an explicit representation for sinks. Instead, filters 28 | // can use side effects to generate results. 29 | // 30 | // Pipelines also support cancelation by way of the context package of Go's 31 | // standard library. 32 | // 33 | // An application of pipelines can be found in the elPrep tool at 34 | // https://github.com/exascience/elprep - specifically in 35 | // https://github.com/ExaScience/elprep/blob/master/sam/filter-pipeline.go 36 | package pipeline 37 | 38 | import ( 39 | "context" 40 | "runtime" 41 | "sync" 42 | ) 43 | 44 | // A Node object represents a sequence of filters which are together executed 45 | // either in encounter order, in arbitrary sequential order, or in parallel. 46 | // 47 | // The methods of this interface are typically not called by user programs, but 48 | // rather implemented by specific node types and called by pipelines. Ordered, 49 | // sequential, and parallel nodes are also implemented in this package, so that 50 | // user programs are typically not concerned with Node methods at all. 51 | type Node interface { 52 | 53 | // TryMerge tries to merge node with the current node by appending its 54 | // filters to the filters of the current node, which succeeds if both nodes 55 | // are either sequential or parallel. The return value merged indicates 56 | // whether merging succeeded. 57 | TryMerge(node Node) (merged bool) 58 | 59 | // Begin informs this node that the pipeline is going to start to feed 60 | // batches of data to this node. The pipeline, the index of this node among 61 | // all the nodes in the pipeline, and the expected total size of all batches 62 | // combined are passed as parameters. 63 | // 64 | // The dataSize parameter is either positive, in which case it indicates the 65 | // expected total size of all batches that will eventually be passed to this 66 | // node's Feed method, or it is negative, in which case the expected size is 67 | // either unknown or too difficult to determine. The dataSize parameter is a 68 | // pointer whose contents can be modified by Begin, for example if this node 69 | // increases or decreases the total size for subsequent nodes, or if this 70 | // node can change dataSize from an unknown to a known value, or vice versa, 71 | // must change it from a known to an unknown value. 72 | // 73 | // A node may decide that, based on the given information, it will actually 74 | // not need to see any of the batches that are normally going to be passed 75 | // to it. In that case, it can return false as a result, and its Feed and 76 | // End method will not be called anymore. Otherwise, it should return true 77 | // by default. 78 | Begin(p *Pipeline, index int, dataSize *int) (keep bool) 79 | 80 | // Feed is called for each batch of data. The pipeline, the index of this 81 | // node among all the nodes in the pipeline (which may be different from the 82 | // index number seen by Begin), the sequence number of the batch (according 83 | // to the encounter order), and the actual batch of data are passed as 84 | // parameters. 85 | // 86 | // The data parameter contains the batch of data, which is usually a slice 87 | // of a particular type. After the data has been processed by all filters of 88 | // this node, the node must call p.FeedForward with exactly the same index 89 | // and sequence numbers, but a potentially modified batch of data. 90 | // FeedForward must be called even when the data batch is or becomes empty, 91 | // to ensure that all sequence numbers are seen by subsequent nodes. 92 | Feed(p *Pipeline, index int, seqNo int, data interface{}) 93 | 94 | // End is called after all batches have been passed to Feed. This allows the 95 | // node to release resources and call the finalizers of its filters. 96 | End() 97 | } 98 | 99 | // A Pipeline is a parallel pipeline that can feed batches of data fetched from 100 | // a source through several nodes that are ordered, sequential, or parallel. 101 | // 102 | // The zero Pipeline is valid and empty. 103 | // 104 | // A Pipeline must not be copied after first use. 105 | type Pipeline struct { 106 | mutex sync.RWMutex 107 | err error 108 | ctx context.Context 109 | cancel context.CancelFunc 110 | source Source 111 | nodes []Node 112 | nofBatches int 113 | batchInc int 114 | maxBatchSize int 115 | } 116 | 117 | // Err returns the current error value for this pipeline, which may be nil if no 118 | // error has occurred so far. 119 | // 120 | // Err and SetErr are safe to be concurrently invoked. 121 | func (p *Pipeline) Err() (err error) { 122 | p.mutex.RLock() 123 | err = p.err 124 | p.mutex.RUnlock() 125 | return err 126 | } 127 | 128 | // SetErr attempts to set a new error value for this pipeline, unless it already 129 | // has a non-nil error value. If the attempt is successful, SetErr also cancels 130 | // the pipeline, and returns true. If the attempt is not successful, SetErr 131 | // returns false. 132 | // 133 | // SetErr and Err are safe to be concurrently invoked, for example from the 134 | // different goroutines executing filters of parallel nodes in this pipeline. 135 | func (p *Pipeline) SetErr(err error) bool { 136 | p.mutex.Lock() 137 | if p.err == nil { 138 | p.err = err 139 | p.mutex.Unlock() 140 | p.cancel() 141 | return true 142 | } 143 | p.mutex.Unlock() 144 | return false 145 | } 146 | 147 | // Context returns this pipeline's context. 148 | func (p *Pipeline) Context() context.Context { 149 | return p.ctx 150 | } 151 | 152 | // Cancel calls the cancel function of this pipeline's context. 153 | func (p *Pipeline) Cancel() { 154 | p.cancel() 155 | } 156 | 157 | // Source sets the data source for this pipeline. 158 | // 159 | // If source does not implement the Source interface, the pipeline uses 160 | // reflection to create a proper source for arrays, slices, strings, or 161 | // channels. 162 | // 163 | // It is safe to call Source multiple times before Run or RunWithContext is 164 | // called, in which case only the last call to Source is effective. 165 | func (p *Pipeline) Source(source interface{}) { 166 | switch src := source.(type) { 167 | case Source: 168 | p.source = src 169 | default: 170 | p.source = reflectSource(source) 171 | } 172 | } 173 | 174 | // Add appends nodes to the end of this pipeline. 175 | func (p *Pipeline) Add(nodes ...Node) { 176 | for _, node := range nodes { 177 | if l := len(p.nodes); (l == 0) || !p.nodes[l-1].TryMerge(node) { 178 | p.nodes = append(p.nodes, node) 179 | } 180 | } 181 | } 182 | 183 | // NofBatches sets or gets the number of batches that are created from the data 184 | // source for this pipeline, if the expected total size for this pipeline's data 185 | // source is known or can be determined easily. 186 | // 187 | // NofBatches can be called safely by user programs before Run or RunWithContext 188 | // is called. 189 | // 190 | // If user programs do not call NofBatches, or call them with a value < 1, then 191 | // the pipeline will choose a reasonable default value that takes 192 | // runtime.GOMAXPROCS(0) into account. 193 | // 194 | // If the expected total size for this pipeline's data source is unknown, or is 195 | // difficult to determine, use SetVariableBatchSize to influence batch sizes. 196 | func (p *Pipeline) NofBatches(n int) (nofBatches int) { 197 | if n < 1 { 198 | nofBatches = p.nofBatches 199 | if nofBatches < 1 { 200 | nofBatches = 2 * runtime.GOMAXPROCS(0) 201 | p.nofBatches = nofBatches 202 | } 203 | } else { 204 | nofBatches = n 205 | p.nofBatches = n 206 | } 207 | return 208 | } 209 | 210 | const ( 211 | defaultBatchInc = 1024 212 | defaultMaxBatchSize = 0x2000000 213 | ) 214 | 215 | // SetVariableBatchSize sets the batch size(s) for the batches that are created 216 | // from the data source for this pipeline, if the expected total size for this 217 | // pipeline's data source is unknown or difficult to determine. 218 | // 219 | // SetVariableBatchSize can be called safely by user programs before Run or 220 | // RunWithContext is called. 221 | // 222 | // If user programs do not call SetVariableBatchSize, or pass a value < 1 to any 223 | // of the two parameters, then the pipeline will choose a reasonable default 224 | // value for that respective parameter. 225 | // 226 | // The pipeline will start with batchInc as a batch size, and increase the batch 227 | // size for every subsequent batch by batchInc to accomodate data sources of 228 | // different total sizes. The batch size will never be larger than maxBatchSize, 229 | // though. 230 | // 231 | // If the expected total size for this pipeline's data source is known, or can 232 | // be determined easily, use NofBatches to influence the batch size. 233 | func (p *Pipeline) SetVariableBatchSize(batchInc, maxBatchSize int) { 234 | p.batchInc = batchInc 235 | p.maxBatchSize = maxBatchSize 236 | } 237 | 238 | func (p *Pipeline) finalizeVariableBatchSize() { 239 | if p.batchInc < 1 { 240 | p.batchInc = defaultBatchInc 241 | } 242 | if p.maxBatchSize < 1 { 243 | p.maxBatchSize = defaultMaxBatchSize 244 | } 245 | } 246 | 247 | func (p *Pipeline) nextBatchSize(batchSize int) (result int) { 248 | result = batchSize + p.batchInc 249 | if result > p.maxBatchSize { 250 | result = p.maxBatchSize 251 | } 252 | return 253 | } 254 | 255 | // RunWithContext initiates pipeline execution. 256 | // 257 | // It expects a context and a cancel function as parameters, for example from 258 | // context.WithCancel(context.Background()). It does not ensure that the cancel 259 | // function is called at least once, so this must be ensured by the function 260 | // calling RunWithContext. 261 | // 262 | // RunWithContext should only be called after a data source has been set using 263 | // the Source method, and one or more Node objects have been added to the 264 | // pipeline using the Add method. NofBatches can be called before RunWithContext 265 | // to deal with load imbalance, but this is not necessary since RunWithContext 266 | // chooses a reasonable default value. 267 | // 268 | // RunWithContext prepares the data source, tells each node that batches are 269 | // going to be sent to them by calling Begin, and then fetches batches from the 270 | // data source and sends them to the nodes. Once the data source is depleted, 271 | // the nodes are informed that the end of the data source has been reached. 272 | func (p *Pipeline) RunWithContext(ctx context.Context, cancel context.CancelFunc) { 273 | if p.err != nil { 274 | return 275 | } 276 | p.ctx, p.cancel = ctx, cancel 277 | dataSize := p.source.Prepare(p.ctx) 278 | filteredSize := dataSize 279 | for index := 0; index < len(p.nodes); { 280 | if p.nodes[index].Begin(p, index, &filteredSize) { 281 | index++ 282 | } else { 283 | p.nodes = append(p.nodes[:index], p.nodes[index+1:]...) 284 | } 285 | } 286 | if p.err != nil { 287 | return 288 | } 289 | if len(p.nodes) > 0 { 290 | for index := 0; index < len(p.nodes)-1; { 291 | if p.nodes[index].TryMerge(p.nodes[index+1]) { 292 | p.nodes = append(p.nodes[:index+1], p.nodes[index+2:]...) 293 | } else { 294 | index++ 295 | } 296 | } 297 | for index := len(p.nodes) - 1; index >= 0; index-- { 298 | if _, ok := p.nodes[index].(*strictordnode); ok { 299 | for index = index - 1; index >= 0; index-- { 300 | switch node := p.nodes[index].(type) { 301 | case *seqnode: 302 | node.kind = Ordered 303 | case *lparnode: 304 | node.makeOrdered() 305 | } 306 | } 307 | break 308 | } 309 | } 310 | if dataSize < 0 { 311 | p.finalizeVariableBatchSize() 312 | for seqNo, batchSize := 0, p.batchInc; p.source.Fetch(batchSize) > 0; seqNo, batchSize = seqNo+1, p.nextBatchSize(batchSize) { 313 | p.nodes[0].Feed(p, 0, seqNo, p.source.Data()) 314 | if err := p.source.Err(); err != nil { 315 | p.SetErr(err) 316 | return 317 | } else if p.Err() != nil { 318 | return 319 | } 320 | } 321 | } else { 322 | batchSize := ((dataSize - 1) / p.NofBatches(0)) + 1 323 | if batchSize == 0 { 324 | batchSize = 1 325 | } 326 | for seqNo := 0; p.source.Fetch(batchSize) > 0; seqNo++ { 327 | p.nodes[0].Feed(p, 0, seqNo, p.source.Data()) 328 | if err := p.source.Err(); err != nil { 329 | p.SetErr(err) 330 | return 331 | } else if p.Err() != nil { 332 | return 333 | } 334 | } 335 | } 336 | } 337 | for _, node := range p.nodes { 338 | node.End() 339 | } 340 | if p.err == nil { 341 | p.err = p.source.Err() 342 | } 343 | } 344 | 345 | // Run initiates pipeline execution by calling 346 | // RunWithContext(context.WithCancel(context.Background())), and ensures that 347 | // the cancel function is called at least once when the pipeline is done. 348 | // 349 | // Run should only be called after a data source has been set using the Source 350 | // method, and one or more Node objects have been added to the pipeline using 351 | // the Add method. NofBatches can be called before Run to deal with load 352 | // imbalance, but this is not necessary since Run chooses a reasonable default 353 | // value. 354 | // 355 | // Run prepares the data source, tells each node that batches are going to be 356 | // sent to them by calling Begin, and then fetches batches from the data source 357 | // and sends them to the nodes. Once the data source is depleted, the nodes are 358 | // informed that the end of the data source has been reached. 359 | func (p *Pipeline) Run() { 360 | ctx, cancel := context.WithCancel(context.Background()) 361 | defer cancel() 362 | p.RunWithContext(ctx, cancel) 363 | } 364 | 365 | // FeedForward must be called in the Feed method of a node to forward a 366 | // potentially modified data batch to the next node in the current pipeline. 367 | // 368 | // FeedForward is used in Node implementations. User programs typically do not 369 | // call FeedForward. 370 | // 371 | // FeedForward must be called with the pipeline received as a parameter by Feed, 372 | // and must pass the same index and seqNo received by Feed. The data parameter 373 | // can be either a modified or an unmodified data batch. FeedForward must always 374 | // be called, even if the data batch is unmodified, and even if the data batch 375 | // is or becomes empty. 376 | func (p *Pipeline) FeedForward(index int, seqNo int, data interface{}) { 377 | if index++; index < len(p.nodes) { 378 | p.nodes[index].Feed(p, index, seqNo, data) 379 | } 380 | } 381 | -------------------------------------------------------------------------------- /sequential/sequential.go: -------------------------------------------------------------------------------- 1 | // Package sequential provides sequential implementations of the functions 2 | // provided by the parallel and speculative packages. This is useful for testing 3 | // and debugging. 4 | // 5 | // It is not recommended to use the implementations of this package for any 6 | // other purpose, because they are almost certainly too inefficient for regular 7 | // sequential programs. 8 | package sequential 9 | 10 | import ( 11 | "fmt" 12 | 13 | "github.com/exascience/pargo/internal" 14 | ) 15 | 16 | // Reduce receives one or more functions, executes them sequentially, and 17 | // combines their results sequentially. 18 | // 19 | // Partial results are combined with the join function. 20 | func Reduce( 21 | join func(x, y interface{}) interface{}, 22 | firstFunction func() interface{}, 23 | moreFunctions ...func() interface{}, 24 | ) interface{} { 25 | result := firstFunction() 26 | for _, f := range moreFunctions { 27 | result = join(result, f()) 28 | } 29 | return result 30 | } 31 | 32 | // ReduceFloat64 receives one or more functions, executes them sequentially, and 33 | // combines their results sequentially. 34 | // 35 | // Partial results are combined with the join function. 36 | func ReduceFloat64( 37 | join func(x, y float64) float64, 38 | firstFunction func() float64, 39 | moreFunctions ...func() float64, 40 | ) float64 { 41 | result := firstFunction() 42 | for _, f := range moreFunctions { 43 | result = join(result, f()) 44 | } 45 | return result 46 | } 47 | 48 | // ReduceFloat64Sum receives zero or more functions, executes them sequentially, 49 | // and adds their results sequentially. 50 | func ReduceFloat64Sum(functions ...func() float64) float64 { 51 | result := float64(0) 52 | for _, f := range functions { 53 | result += f() 54 | } 55 | return result 56 | } 57 | 58 | // ReduceFloat64Product receives zero or more functions, executes them 59 | // sequentially, and multiplies their results sequentially. 60 | func ReduceFloat64Product(functions ...func() float64) float64 { 61 | result := float64(1) 62 | for _, f := range functions { 63 | result *= f() 64 | } 65 | return result 66 | } 67 | 68 | // ReduceInt receives one or more functions, executes them sequentially, and 69 | // combines their results sequentially. 70 | // 71 | // Partial results are combined with the join function. 72 | func ReduceInt( 73 | join func(x, y int) int, 74 | firstFunction func() int, 75 | moreFunctions ...func() int, 76 | ) int { 77 | result := firstFunction() 78 | for _, f := range moreFunctions { 79 | result = join(result, f()) 80 | } 81 | return result 82 | } 83 | 84 | // ReduceIntSum receives zero or more functions, executes them sequentially, and 85 | // adds their results sequentially. 86 | func ReduceIntSum(functions ...func() int) int { 87 | result := 0 88 | for _, f := range functions { 89 | result += f() 90 | } 91 | return result 92 | } 93 | 94 | // ReduceIntProduct receives zero or more functions, executes them sequentially, 95 | // and multiplies their results sequentially. 96 | func ReduceIntProduct(functions ...func() int) int { 97 | result := 1 98 | for _, f := range functions { 99 | result *= f() 100 | } 101 | return result 102 | } 103 | 104 | // ReduceString receives one or more functions, executes them in parallel, and 105 | // combines their results in parallel. 106 | // 107 | // Partial results are combined with the join function. 108 | func ReduceString( 109 | join func(x, y string) string, 110 | firstFunction func() string, 111 | moreFunctions ...func() string, 112 | ) string { 113 | result := firstFunction() 114 | for _, f := range moreFunctions { 115 | result = join(result, f()) 116 | } 117 | return result 118 | } 119 | 120 | // ReduceStringSum receives zero or more functions, executes them in parallel, 121 | // and concatenates their results in parallel. 122 | func ReduceStringSum(functions ...func() string) string { 123 | result := "" 124 | for _, f := range functions { 125 | result += f() 126 | } 127 | return result 128 | } 129 | 130 | // Do receives zero or more thunks and executes them sequentially. 131 | func Do(thunks ...func()) { 132 | for _, thunk := range thunks { 133 | thunk() 134 | } 135 | } 136 | 137 | // And receives zero or more predicate functions and executes them sequentially, 138 | // combining all return values with the && operator, with true as the default 139 | // return value. 140 | func And(predicates ...func() bool) bool { 141 | result := true 142 | for _, predicate := range predicates { 143 | result = result && predicate() 144 | } 145 | return result 146 | } 147 | 148 | // Or receives zero or more predicate functions and executes them sequentially, 149 | // combining all return values with the || operator, with false as the default 150 | // return value. 151 | func Or(predicates ...func() bool) bool { 152 | result := false 153 | for _, predicate := range predicates { 154 | result = result || predicate() 155 | } 156 | return result 157 | } 158 | 159 | // Range receives a range, a batch count n, and a range function f, divides the 160 | // range into batches, and invokes the range function for each of these batches 161 | // sequentially, covering the half-open interval from low to high, including low 162 | // but excluding high. 163 | // 164 | // The range is specified by a low and high integer, with low <= high. The 165 | // batches are determined by dividing up the size of the range (high - low) by 166 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 167 | // into account. 168 | // 169 | // Range panics if high < low, or if n < 0. 170 | func Range( 171 | low, high, n int, 172 | f func(low, high int), 173 | ) { 174 | var recur func(int, int, int) 175 | recur = func(low, high, n int) { 176 | switch { 177 | case n == 1: 178 | f(low, high) 179 | case n > 1: 180 | batchSize := ((high - low - 1) / n) + 1 181 | half := n / 2 182 | mid := low + batchSize*half 183 | if mid >= high { 184 | f(low, high) 185 | return 186 | } 187 | recur(low, mid, half) 188 | recur(mid, high, n-half) 189 | default: 190 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 191 | } 192 | } 193 | recur(low, high, internal.ComputeNofBatches(low, high, n)) 194 | } 195 | 196 | // RangeAnd receives a range, a batch count n, and a range predicate function f, 197 | // divides the range into batches, and invokes the range predicate for each of 198 | // these batches sequentially, covering the half-open interval from low to high, 199 | // including low but excluding high. 200 | // 201 | // The range is specified by a low and high integer, with low <= high. The 202 | // batches are determined by dividing up the size of the range (high - low) by 203 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 204 | // into account. 205 | // 206 | // RangeAnd returns by combining all return values with the && operator. 207 | // 208 | // RangeAnd panics if high < low, or if n < 0. 209 | func RangeAnd( 210 | low, high, n int, 211 | f func(low, high int) bool, 212 | ) bool { 213 | var recur func(int, int, int) bool 214 | recur = func(low, high, n int) bool { 215 | switch { 216 | case n == 1: 217 | return f(low, high) 218 | case n > 1: 219 | batchSize := ((high - low - 1) / n) + 1 220 | half := n / 2 221 | mid := low + batchSize*half 222 | if mid >= high { 223 | return f(low, high) 224 | } 225 | b0 := recur(low, mid, half) 226 | b1 := recur(mid, high, n-half) 227 | return b0 && b1 228 | default: 229 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 230 | } 231 | } 232 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 233 | } 234 | 235 | // RangeOr receives a range, a batch count n, and a range predicate function f, 236 | // divides the range into batches, and invokes the range predicate for each of 237 | // these batches sequentially, covering the half-open interval from low to high, 238 | // including low but excluding high. 239 | // 240 | // The range is specified by a low and high integer, with low <= high. The 241 | // batches are determined by dividing up the size of the range (high - low) by 242 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 243 | // into account. 244 | // 245 | // RangeOr returns by combining all return values with the || operator. 246 | // 247 | // RangeOr panics if high < low, or if n < 0. 248 | func RangeOr( 249 | low, high, n int, 250 | f func(low, high int) bool, 251 | ) bool { 252 | var recur func(int, int, int) bool 253 | recur = func(low, high, n int) bool { 254 | switch { 255 | case n == 1: 256 | return f(low, high) 257 | case n > 1: 258 | batchSize := ((high - low - 1) / n) + 1 259 | half := n / 2 260 | mid := low + batchSize*half 261 | if mid >= high { 262 | return f(low, high) 263 | } 264 | b0 := recur(low, mid, half) 265 | b1 := recur(mid, high, n-half) 266 | return b0 || b1 267 | default: 268 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 269 | } 270 | } 271 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 272 | } 273 | 274 | // RangeReduce receives a range, a batch count, a range reduce function, and a 275 | // join function, divides the range into batches, and invokes the range reducer 276 | // for each of these batches sequentially, covering the half-open interval from 277 | // low to high, including low but excluding high. The results of the range 278 | // reducer invocations are then combined by repeated invocations of join. 279 | // 280 | // The range is specified by a low and high integer, with low <= high. The 281 | // batches are determined by dividing up the size of the range (high - low) by 282 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 283 | // into account. 284 | // 285 | // RangeReduce panics if high < low, or if n < 0. 286 | func RangeReduce( 287 | low, high, n int, 288 | reduce func(low, high int) interface{}, 289 | join func(x, y interface{}) interface{}, 290 | ) interface{} { 291 | var recur func(int, int, int) interface{} 292 | recur = func(low, high, n int) interface{} { 293 | switch { 294 | case n == 1: 295 | return reduce(low, high) 296 | case n > 1: 297 | batchSize := ((high - low - 1) / n) + 1 298 | half := n / 2 299 | mid := low + batchSize*half 300 | if mid >= high { 301 | return reduce(low, high) 302 | } 303 | left := recur(low, mid, half) 304 | right := recur(mid, high, n-half) 305 | return join(left, right) 306 | default: 307 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 308 | } 309 | } 310 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 311 | } 312 | 313 | // RangeReduceInt receives a range, a batch count n, a range reducer function, 314 | // and a join function, divides the range into batches, and invokes the range 315 | // reducer for each of these batches sequentially, covering the half-open 316 | // interval from low to high, including low but excluding high. The results of 317 | // the range reducer invocations are then combined by repeated invocations of 318 | // join. 319 | // 320 | // The range is specified by a low and high integer, with low <= high. The 321 | // batches are determined by dividing up the size of the range (high - low) by 322 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 323 | // into account. 324 | // 325 | // IntRangeReduce panics if high < low, or if n < 0. 326 | func RangeReduceInt( 327 | low, high, n int, 328 | reduce func(low, high int) int, 329 | join func(x, y int) int, 330 | ) int { 331 | var recur func(int, int, int) int 332 | recur = func(low, high, n int) int { 333 | switch { 334 | case n == 1: 335 | return reduce(low, high) 336 | case n > 1: 337 | batchSize := ((high - low - 1) / n) + 1 338 | half := n / 2 339 | mid := low + batchSize*half 340 | if mid >= high { 341 | return reduce(low, high) 342 | } 343 | left := recur(low, mid, half) 344 | right := recur(mid, high, n-half) 345 | return join(left, right) 346 | default: 347 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 348 | } 349 | } 350 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 351 | } 352 | 353 | // RangeReduceIntSum receives a range, a batch count n, and a range reducer 354 | // function, divides the range into batches, and invokes the range reducer for 355 | // each of these batches sequentially, covering the half-open interval from low 356 | // to high, including low but excluding high. The results of the range reducer 357 | // invocations are then added together. 358 | // 359 | // The range is specified by a low and high integer, with low <= high. The 360 | // batches are determined by dividing up the size of the range (high - low) by 361 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 362 | // into account. 363 | // 364 | // RangeReduceIntSum panics if high < low, or if n < 0. 365 | func RangeReduceIntSum( 366 | low, high, n int, 367 | reduce func(low, high int) int, 368 | ) int { 369 | var recur func(int, int, int) int 370 | recur = func(low, high, n int) int { 371 | switch { 372 | case n == 1: 373 | return reduce(low, high) 374 | case n > 1: 375 | batchSize := ((high - low - 1) / n) + 1 376 | half := n / 2 377 | mid := low + batchSize*half 378 | if mid >= high { 379 | return reduce(low, high) 380 | } 381 | left := recur(low, mid, half) 382 | right := recur(mid, high, n-half) 383 | return left + right 384 | default: 385 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 386 | } 387 | } 388 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 389 | } 390 | 391 | // RangeReduceIntProduct receives a range, a batch count n, and a range reducer 392 | // function, divides the range into batches, and invokes the range reducer for 393 | // each of these batches sequentially, covering the half-open interval from low 394 | // to high, including low but excluding high. The results of the range reducer 395 | // invocations are then multiplied with each other. 396 | // 397 | // The range is specified by a low and high integer, with low <= high. The 398 | // batches are determined by dividing up the size of the range (high - low) by 399 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 400 | // into account. 401 | // 402 | // RangeReduceIntProduct panics if high < low, or if n < 0. 403 | func RangeReduceIntProduct( 404 | low, high, n int, 405 | reduce func(low, high int) int, 406 | ) int { 407 | var recur func(int, int, int) int 408 | recur = func(low, high, n int) int { 409 | switch { 410 | case n == 1: 411 | return reduce(low, high) 412 | case n > 1: 413 | batchSize := ((high - low - 1) / n) + 1 414 | half := n / 2 415 | mid := low + batchSize*half 416 | if mid >= high { 417 | return reduce(low, high) 418 | } 419 | left := recur(low, mid, half) 420 | right := recur(mid, high, n-half) 421 | return left * right 422 | default: 423 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 424 | } 425 | } 426 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 427 | } 428 | 429 | // RangeReduceFloat64 receives a range, a batch count n, a range reducer 430 | // function, and a join function, divides the range into batches, and invokes 431 | // the range reducer for each of these batches sequentially, covering the 432 | // half-open interval from low to high, including low but excluding high. The 433 | // results of the range reducer invocations are then combined by repeated 434 | // invocations of join. 435 | // 436 | // The range is specified by a low and high integer, with low <= high. The 437 | // batches are determined by dividing up the size of the range (high - low) by 438 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 439 | // into account. 440 | // 441 | // Float64RangeReduce panics if high < low, or if n < 0. 442 | func RangeReduceFloat64( 443 | low, high, n int, 444 | reduce func(low, high int) float64, 445 | join func(x, y float64) float64, 446 | ) float64 { 447 | var recur func(int, int, int) float64 448 | recur = func(low, high, n int) float64 { 449 | switch { 450 | case n == 1: 451 | return reduce(low, high) 452 | case n > 1: 453 | batchSize := ((high - low - 1) / n) + 1 454 | half := n / 2 455 | mid := low + batchSize*half 456 | if mid >= high { 457 | return reduce(low, high) 458 | } 459 | left := recur(low, mid, half) 460 | right := recur(mid, high, n-half) 461 | return join(left, right) 462 | default: 463 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 464 | } 465 | } 466 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 467 | } 468 | 469 | // RangeReduceFloat64Sum receives a range, a batch count n, and a range reducer 470 | // function, divides the range into batches, and invokes the range reducer for 471 | // each of these batches sequentially, covering the half-open interval from low 472 | // to high, including low but excluding high. The results of the range reducer 473 | // invocations are then added together. 474 | // 475 | // The range is specified by a low and high integer, with low <= high. The 476 | // batches are determined by dividing up the size of the range (high - low) by 477 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 478 | // into account. 479 | // 480 | // RangeReduceFloat64Sum panics if high < low, or if n < 0. 481 | func RangeReduceFloat64Sum( 482 | low, high, n int, 483 | reduce func(low, high int) float64, 484 | ) float64 { 485 | var recur func(int, int, int) float64 486 | recur = func(low, high, n int) float64 { 487 | switch { 488 | case n == 1: 489 | return reduce(low, high) 490 | case n > 1: 491 | batchSize := ((high - low - 1) / n) + 1 492 | half := n / 2 493 | mid := low + batchSize*half 494 | if mid >= high { 495 | return reduce(low, high) 496 | } 497 | left := recur(low, mid, half) 498 | right := recur(mid, high, n-half) 499 | return left + right 500 | default: 501 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 502 | } 503 | } 504 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 505 | } 506 | 507 | // RangeReduceFloat64Product receives a range, a batch count n, and a range 508 | // reducer function, divides the range into batches, and invokes the range 509 | // reducer for each of these batches sequentially, covering the half-open 510 | // interval from low to high, including low but excluding high. The results of 511 | // the range reducer invocations are then multiplied with each other. 512 | // 513 | // The range is specified by a low and high integer, with low <= high. The 514 | // batches are determined by dividing up the size of the range (high - low) by 515 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 516 | // into account. 517 | // 518 | // RangeReduceFloat64Product panics if high < low, or if n < 0. 519 | func RangeReduceFloat64Product( 520 | low, high, n int, 521 | reduce func(low, high int) float64, 522 | ) float64 { 523 | var recur func(int, int, int) float64 524 | recur = func(low, high, n int) float64 { 525 | switch { 526 | case n == 1: 527 | return reduce(low, high) 528 | case n > 1: 529 | batchSize := ((high - low - 1) / n) + 1 530 | half := n / 2 531 | mid := low + batchSize*half 532 | if mid >= high { 533 | return reduce(low, high) 534 | } 535 | left := recur(low, mid, half) 536 | right := recur(mid, high, n-half) 537 | return left * right 538 | default: 539 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 540 | } 541 | } 542 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 543 | } 544 | 545 | // RangeReduceString receives a range, a batch count n, a range reducer 546 | // function, and a join function, divides the range into batches, and invokes 547 | // the range reducer for each of these batches sequentially, covering the 548 | // half-open interval from low to high, including low but excluding high. The 549 | // results of the range reducer invocations are then combined by repeated 550 | // invocations of join. 551 | // 552 | // The range is specified by a low and high integer, with low <= high. The 553 | // batches are determined by dividing up the size of the range (high - low) by 554 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 555 | // into account. 556 | // 557 | // StringRangeReduce panics if high < low, or if n < 0. 558 | func RangeReduceString( 559 | low, high, n int, 560 | reduce func(low, high int) string, 561 | join func(x, y string) string, 562 | ) string { 563 | var recur func(int, int, int) string 564 | recur = func(low, high, n int) string { 565 | switch { 566 | case n == 1: 567 | return reduce(low, high) 568 | case n > 1: 569 | batchSize := ((high - low - 1) / n) + 1 570 | half := n / 2 571 | mid := low + batchSize*half 572 | if mid >= high { 573 | return reduce(low, high) 574 | } 575 | left := recur(low, mid, half) 576 | right := recur(mid, high, n-half) 577 | return join(left, right) 578 | default: 579 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 580 | } 581 | } 582 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 583 | } 584 | 585 | // RangeReduceStringSum receives a range, a batch count n, and a range reducer 586 | // function, divides the range into batches, and invokes the range reducer for 587 | // each of these batches sequentially, covering the half-open interval from low 588 | // to high, including low but excluding high. The results of the range reducer 589 | // invocations are then concatenated together. 590 | // 591 | // The range is specified by a low and high integer, with low <= high. The 592 | // batches are determined by dividing up the size of the range (high - low) by 593 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 594 | // into account. 595 | // 596 | // RangeReduceStringSum panics if high < low, or if n < 0. 597 | func RangeReduceStringSum( 598 | low, high, n int, 599 | reduce func(low, high int) string, 600 | ) string { 601 | var recur func(int, int, int) string 602 | recur = func(low, high, n int) string { 603 | switch { 604 | case n == 1: 605 | return reduce(low, high) 606 | case n > 1: 607 | batchSize := ((high - low - 1) / n) + 1 608 | half := n / 2 609 | mid := low + batchSize*half 610 | if mid >= high { 611 | return reduce(low, high) 612 | } 613 | left := recur(low, mid, half) 614 | right := recur(mid, high, n-half) 615 | return left + right 616 | default: 617 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 618 | } 619 | } 620 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 621 | } 622 | -------------------------------------------------------------------------------- /speculative/speculative.go: -------------------------------------------------------------------------------- 1 | // Package speculative provides functions for expressing parallel algorithms, 2 | // similar to the functions in package parallel, except that the implementations 3 | // here terminate early when they can. 4 | // 5 | // See https://github.com/ExaScience/pargo/wiki/TaskParallelism for a general 6 | // overview. 7 | package speculative 8 | 9 | import ( 10 | "fmt" 11 | "sync" 12 | 13 | "github.com/exascience/pargo/internal" 14 | ) 15 | 16 | // Reduce receives one or more functions, executes them in parallel, and 17 | // combines their results with the join function in parallel. 18 | // 19 | // Each function is invoked in its own goroutine. Reduce returns either when all 20 | // functions have terminated with a second return value of false; or when one or 21 | // more functions return a second return value of true. In the latter case, the 22 | // first return value of the left-most function that returned true as a second 23 | // return value becomes the final result, without waiting for the other 24 | // functions to terminate. 25 | // 26 | // If one or more functions panic, the corresponding goroutines recover the 27 | // panics, and Reduce eventually panics with the left-most recovered panic 28 | // value. 29 | func Reduce( 30 | join func(x, y interface{}) (interface{}, bool), 31 | firstFunction func() (interface{}, bool), 32 | moreFunctions ...func() (interface{}, bool), 33 | ) (interface{}, bool) { 34 | if len(moreFunctions) == 0 { 35 | return firstFunction() 36 | } 37 | var left, right interface{} 38 | var b0, b1 bool 39 | var p interface{} 40 | var wg sync.WaitGroup 41 | wg.Add(1) 42 | if len(moreFunctions) == 1 { 43 | go func() { 44 | defer func() { 45 | p = internal.WrapPanic(recover()) 46 | wg.Done() 47 | }() 48 | right, b1 = moreFunctions[0]() 49 | }() 50 | left, b0 = firstFunction() 51 | } else { 52 | half := (len(moreFunctions) + 1) / 2 53 | go func() { 54 | defer func() { 55 | p = internal.WrapPanic(recover()) 56 | wg.Done() 57 | }() 58 | right, b1 = Reduce(join, moreFunctions[half], moreFunctions[half+1:]...) 59 | }() 60 | left, b0 = Reduce(join, firstFunction, moreFunctions[:half]...) 61 | } 62 | if b0 { 63 | return left, true 64 | } 65 | wg.Wait() 66 | if p != nil { 67 | panic(p) 68 | } 69 | if b1 { 70 | return right, true 71 | } 72 | return join(left, right) 73 | } 74 | 75 | // ReduceFloat64 receives one or more functions, executes them in parallel, and 76 | // combines their results with the join function in parallel. 77 | // 78 | // Each function is invoked in its own goroutine. ReduceFloat64 returns either 79 | // when all functions have terminated with a second return value of false; or 80 | // when one or more functions return a second return value of true. In the 81 | // latter case, the first return value of the left-most function that returned 82 | // true as a second return value becomes the final result, without waiting for 83 | // the other functions to terminate. 84 | // 85 | // If one or more functions panic, the corresponding goroutines recover the 86 | // panics, and ReduceFloat64 eventually panics with the left-most recovered 87 | // panic value. 88 | func ReduceFloat64( 89 | join func(x, y float64) (float64, bool), 90 | firstFunction func() (float64, bool), 91 | moreFunctions ...func() (float64, bool), 92 | ) (float64, bool) { 93 | if len(moreFunctions) == 0 { 94 | return firstFunction() 95 | } 96 | var left, right float64 97 | var b0, b1 bool 98 | var p interface{} 99 | var wg sync.WaitGroup 100 | wg.Add(1) 101 | if len(moreFunctions) == 1 { 102 | go func() { 103 | defer func() { 104 | p = internal.WrapPanic(recover()) 105 | wg.Done() 106 | }() 107 | right, b1 = moreFunctions[0]() 108 | }() 109 | left, b0 = firstFunction() 110 | } else { 111 | half := (len(moreFunctions) + 1) / 2 112 | go func() { 113 | defer func() { 114 | p = internal.WrapPanic(recover()) 115 | wg.Done() 116 | }() 117 | right, b1 = ReduceFloat64(join, moreFunctions[half], moreFunctions[half+1:]...) 118 | }() 119 | left, b0 = ReduceFloat64(join, firstFunction, moreFunctions[:half]...) 120 | } 121 | if b0 { 122 | return left, true 123 | } 124 | wg.Wait() 125 | if p != nil { 126 | panic(p) 127 | } 128 | if b1 { 129 | return right, true 130 | } 131 | return join(left, right) 132 | } 133 | 134 | // ReduceInt receives one or more functions, executes them in parallel, and 135 | // combines their results with the join function in parallel. 136 | // 137 | // Each function is invoked in its own goroutine. ReduceInt returns either when 138 | // all functions have terminated with a second return value of false; or when 139 | // one or more functions return a second return value of true. In the latter 140 | // case, the first return value of the left-most function that returned true as 141 | // a second return value becomes the final result, without waiting for the other 142 | // functions to terminate. 143 | // 144 | // If one or more functions panic, the corresponding goroutines recover the 145 | // panics, and ReduceInt eventually panics with the left-most recovered panic 146 | // value. 147 | func ReduceInt( 148 | join func(x, y int) (int, bool), 149 | firstFunction func() (int, bool), 150 | moreFunctions ...func() (int, bool), 151 | ) (int, bool) { 152 | if len(moreFunctions) == 0 { 153 | return firstFunction() 154 | } 155 | var left, right int 156 | var b0, b1 bool 157 | var p interface{} 158 | var wg sync.WaitGroup 159 | wg.Add(1) 160 | if len(moreFunctions) == 1 { 161 | go func() { 162 | defer func() { 163 | p = internal.WrapPanic(recover()) 164 | wg.Done() 165 | }() 166 | right, b1 = moreFunctions[0]() 167 | }() 168 | left, b0 = firstFunction() 169 | } else { 170 | half := (len(moreFunctions) + 1) / 2 171 | go func() { 172 | defer func() { 173 | p = internal.WrapPanic(recover()) 174 | wg.Done() 175 | }() 176 | right, b1 = ReduceInt(join, moreFunctions[half], moreFunctions[half+1:]...) 177 | }() 178 | left, b0 = ReduceInt(join, firstFunction, moreFunctions[:half]...) 179 | } 180 | if b0 { 181 | return left, true 182 | } 183 | wg.Wait() 184 | if p != nil { 185 | panic(p) 186 | } 187 | if b1 { 188 | return right, true 189 | } 190 | return join(left, right) 191 | } 192 | 193 | // ReduceString receives one or more functions, executes them in parallel, and 194 | // combines their results with the join function in parallel. 195 | // 196 | // Each function is invoked in its own goroutine. ReduceString returns either 197 | // when all functions have terminated with a second return value of false; or 198 | // when one or more functions return a second return value of true. In the 199 | // latter case, the first return value of the left-most function that returned 200 | // true as a second return value becomes the final result, without waiting for 201 | // the other functions to terminate. 202 | // 203 | // If one or more functions panic, the corresponding goroutines recover the 204 | // panics, and ReduceString eventually panics with the left-most recovered panic 205 | // value. 206 | func ReduceString( 207 | join func(x, y string) (string, bool), 208 | firstFunction func() (string, bool), 209 | moreFunctions ...func() (string, bool), 210 | ) (string, bool) { 211 | if len(moreFunctions) == 0 { 212 | return firstFunction() 213 | } 214 | var left, right string 215 | var b0, b1 bool 216 | var p interface{} 217 | var wg sync.WaitGroup 218 | wg.Add(1) 219 | if len(moreFunctions) == 1 { 220 | go func() { 221 | defer func() { 222 | p = internal.WrapPanic(recover()) 223 | wg.Done() 224 | }() 225 | right, b1 = moreFunctions[0]() 226 | }() 227 | left, b0 = firstFunction() 228 | } else { 229 | half := (len(moreFunctions) + 1) / 2 230 | go func() { 231 | defer func() { 232 | p = internal.WrapPanic(recover()) 233 | wg.Done() 234 | }() 235 | right, b1 = ReduceString(join, moreFunctions[half], moreFunctions[half+1:]...) 236 | }() 237 | left, b0 = ReduceString(join, firstFunction, moreFunctions[:half]...) 238 | } 239 | if b0 { 240 | return left, true 241 | } 242 | wg.Wait() 243 | if p != nil { 244 | panic(p) 245 | } 246 | if b1 { 247 | return right, true 248 | } 249 | return join(left, right) 250 | } 251 | 252 | // Do receives zero or more thunks and executes them in parallel. 253 | // 254 | // Each function is invoked in its own goroutine. Do returns either when all 255 | // functions have terminated with a return value of false; or when one or more 256 | // functions return true, without waiting for the other functions to terminate. 257 | // 258 | // If one or more thunks panic, the corresponding goroutines recover the panics, 259 | // and Do may eventually panic with the left-most recovered panic value. 260 | func Do(thunks ...func() bool) bool { 261 | switch len(thunks) { 262 | case 0: 263 | return false 264 | case 1: 265 | return thunks[0]() 266 | } 267 | var b0, b1 bool 268 | var p interface{} 269 | var wg sync.WaitGroup 270 | wg.Add(1) 271 | switch len(thunks) { 272 | case 2: 273 | go func() { 274 | defer func() { 275 | p = internal.WrapPanic(recover()) 276 | wg.Done() 277 | }() 278 | b1 = thunks[1]() 279 | }() 280 | b0 = thunks[0]() 281 | default: 282 | half := len(thunks) / 2 283 | go func() { 284 | defer func() { 285 | p = internal.WrapPanic(recover()) 286 | wg.Done() 287 | }() 288 | b1 = Do(thunks[half:]...) 289 | }() 290 | b0 = Do(thunks[:half]...) 291 | } 292 | if b0 { 293 | return true 294 | } 295 | wg.Wait() 296 | if p != nil { 297 | panic(p) 298 | } 299 | return b1 300 | } 301 | 302 | // And receives zero or more predicate functions and executes them in parallel. 303 | // 304 | // Each predicate is invoked in its own goroutine, and And returns true if all 305 | // of them return true; or And returns false when at least one of them returns 306 | // false, without waiting for the other predicates to terminate. 307 | // 308 | // If one or more predicates panic, the corresponding goroutines recover the 309 | // panics, and And may eventually panic with the left-most recovered panic 310 | // value. 311 | func And(predicates ...func() bool) bool { 312 | switch len(predicates) { 313 | case 0: 314 | return true 315 | case 1: 316 | return predicates[0]() 317 | } 318 | var b0, b1 bool 319 | var p interface{} 320 | var wg sync.WaitGroup 321 | wg.Add(1) 322 | switch len(predicates) { 323 | case 2: 324 | go func() { 325 | defer func() { 326 | p = internal.WrapPanic(recover()) 327 | wg.Done() 328 | }() 329 | b1 = predicates[1]() 330 | }() 331 | b0 = predicates[0]() 332 | default: 333 | half := len(predicates) / 2 334 | go func() { 335 | defer func() { 336 | p = internal.WrapPanic(recover()) 337 | wg.Done() 338 | }() 339 | b1 = And(predicates[half:]...) 340 | }() 341 | b0 = And(predicates[:half]...) 342 | } 343 | if !b0 { 344 | return false 345 | } 346 | wg.Wait() 347 | if p != nil { 348 | panic(p) 349 | } 350 | return b1 351 | } 352 | 353 | // Or receives zero or more predicate functions and executes them in parallel. 354 | // 355 | // Each predicate is invoked in its own goroutine, and Or returns false if all 356 | // of them return false; or Or returns true when at least one of them returns 357 | // true, without waiting for the other predicates to terminate. 358 | // 359 | // If one or more predicates panic, the corresponding goroutines recover the 360 | // panics, and Or may eventually panic with the left-most recovered panic value. 361 | func Or(predicates ...func() bool) bool { 362 | switch len(predicates) { 363 | case 0: 364 | return false 365 | case 1: 366 | return predicates[0]() 367 | } 368 | var b0, b1 bool 369 | var p interface{} 370 | var wg sync.WaitGroup 371 | wg.Add(1) 372 | switch len(predicates) { 373 | case 2: 374 | go func() { 375 | defer func() { 376 | p = internal.WrapPanic(recover()) 377 | wg.Done() 378 | }() 379 | b1 = predicates[1]() 380 | }() 381 | b0 = predicates[0]() 382 | default: 383 | half := len(predicates) / 2 384 | go func() { 385 | defer func() { 386 | p = internal.WrapPanic(recover()) 387 | wg.Done() 388 | }() 389 | b1 = Or(predicates[half:]...) 390 | }() 391 | b0 = Or(predicates[:half]...) 392 | } 393 | if b0 { 394 | return true 395 | } 396 | wg.Wait() 397 | if p != nil { 398 | panic(p) 399 | } 400 | return b1 401 | } 402 | 403 | // Range receives a range, a batch count n, and a range function f, divides the 404 | // range into batches, and invokes the range function for each of these batches 405 | // in parallel, covering the half-open interval from low to high, including low 406 | // but excluding high. 407 | // 408 | // The range is specified by a low and high integer, with low <= high. The 409 | // batches are determined by dividing up the size of the range (high - low) by 410 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 411 | // into account. 412 | // 413 | // The range function is invoked for each batch in its own goroutine, with 0 <= 414 | // low <= high, and Range returns either when all range functions have 415 | // terminated with a return value of true; or when one or more range functions 416 | // return true, without waiting for the other range functions to terminate. 417 | // 418 | // Range panics if high < low, or if n < 0. 419 | // 420 | // If one or more range functions panic, the corresponding goroutines recover 421 | // the panics, and Range may eventually panic with the left-most recovered panic 422 | // value. If both non-nil error values are returned and panics occur, then the 423 | // left-most of these events take precedence. 424 | func Range( 425 | low, high, n int, 426 | f func(low, high int) bool, 427 | ) bool { 428 | var recur func(int, int, int) bool 429 | recur = func(low, high, n int) bool { 430 | switch { 431 | case n == 1: 432 | return f(low, high) 433 | case n > 1: 434 | batchSize := ((high - low - 1) / n) + 1 435 | half := n / 2 436 | mid := low + batchSize*half 437 | if mid >= high { 438 | return f(low, high) 439 | } 440 | var b1 bool 441 | var p interface{} 442 | var wg sync.WaitGroup 443 | wg.Add(1) 444 | go func() { 445 | defer func() { 446 | p = internal.WrapPanic(recover()) 447 | wg.Done() 448 | }() 449 | b1 = recur(mid, high, n-half) 450 | }() 451 | if recur(low, mid, half) { 452 | return true 453 | } 454 | wg.Wait() 455 | if p != nil { 456 | panic(p) 457 | } 458 | return b1 459 | default: 460 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 461 | } 462 | } 463 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 464 | } 465 | 466 | // RangeAnd receives a range, a batch count n, and a range predicate function f, 467 | // divides the range into batches, and invokes the range predicate for each of 468 | // these batches in parallel, covering the half-open interval from low to high, 469 | // including low but excluding high. 470 | // 471 | // The range is specified by a low and high integer, with low <= high. The 472 | // batches are determined by dividing up the size of the range (high - low) by 473 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 474 | // into account. 475 | // 476 | // The range predicate is invoked for each batch in its own goroutine, with 0 <= 477 | // low <= high, and RangeAnd returns true if all of them return true; or 478 | // RangeAnd returns false when at least one of them returns false, without 479 | // waiting for the other range predicates to terminate. 480 | // 481 | // RangeAnd panics if high < low, or if n < 0. 482 | // 483 | // If one or more range predicates panic, the corresponding goroutines recover 484 | // the panics, and RangeAnd may eventually panic with the left-most recovered 485 | // panic value. 486 | func RangeAnd( 487 | low, high, n int, 488 | f func(low, high int) bool, 489 | ) bool { 490 | var recur func(int, int, int) bool 491 | recur = func(low, high, n int) bool { 492 | switch { 493 | case n == 1: 494 | return f(low, high) 495 | case n > 1: 496 | batchSize := ((high - low - 1) / n) + 1 497 | half := n / 2 498 | mid := low + batchSize*half 499 | if mid >= high { 500 | return f(low, high) 501 | } 502 | var b1 bool 503 | var p interface{} 504 | var wg sync.WaitGroup 505 | wg.Add(1) 506 | go func() { 507 | defer func() { 508 | p = internal.WrapPanic(recover()) 509 | wg.Done() 510 | }() 511 | b1 = recur(mid, high, n-half) 512 | }() 513 | if !recur(low, mid, half) { 514 | return false 515 | } 516 | wg.Wait() 517 | if p != nil { 518 | panic(p) 519 | } 520 | return b1 521 | default: 522 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 523 | } 524 | } 525 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 526 | } 527 | 528 | // RangeOr receives a range, a batch count n, and a range predicate function f, 529 | // divides the range into batches, and invokes the range predicate for each of 530 | // these batches in parallel, covering the half-open interval from low to high, 531 | // including low but excluding high. 532 | // 533 | // The range is specified by a low and high integer, with low <= high. The 534 | // batches are determined by dividing up the size of the range (high - low) by 535 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 536 | // into account. 537 | // 538 | // The range predicate is invoked for each batch in its own goroutine, with 0 <= 539 | // low <= high, and RangeOr returns false if all of them return false; or 540 | // RangeOr returns true when at least one of them returns true, without waiting 541 | // for the other range predicates to terminate. 542 | // 543 | // RangeOr panics if high < low, or if n < 0. 544 | // 545 | // If one or more range predicates panic, the corresponding goroutines recover 546 | // the panics, and RangeOr may eventually panic with the left-most recovered 547 | // panic value. 548 | func RangeOr( 549 | low, high, n int, 550 | f func(low, high int) bool, 551 | ) bool { 552 | var recur func(int, int, int) bool 553 | recur = func(low, high, n int) bool { 554 | switch { 555 | case n == 1: 556 | return f(low, high) 557 | case n > 1: 558 | batchSize := ((high - low - 1) / n) + 1 559 | half := n / 2 560 | mid := low + batchSize*half 561 | if mid >= high { 562 | return f(low, high) 563 | } 564 | var b1 bool 565 | var p interface{} 566 | var wg sync.WaitGroup 567 | wg.Add(1) 568 | go func() { 569 | defer func() { 570 | p = internal.WrapPanic(recover()) 571 | wg.Done() 572 | }() 573 | b1 = recur(mid, high, n-half) 574 | }() 575 | if recur(low, mid, half) { 576 | return true 577 | } 578 | wg.Wait() 579 | if p != nil { 580 | panic(p) 581 | } 582 | return b1 583 | default: 584 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 585 | } 586 | } 587 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 588 | } 589 | 590 | // RangeReduce receives a range, a batch count n, a range reducer function, and 591 | // a join function, divides the range into batches, and invokes the range 592 | // reducer for each of these batches in parallel, covering the half-open 593 | // interval from low to high, including low but excluding high. The results of 594 | // the range reducer invocations are then combined by repeated invocations of 595 | // join. 596 | // 597 | // The range is specified by a low and high integer, with low <= high. The 598 | // batches are determined by dividing up the size of the range (high - low) by 599 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 600 | // into account. 601 | // 602 | // The range reducer is invoked for each batch in its own goroutine, with 0 <= 603 | // low <= high, and RangeReduce returns either when all range reducers and joins 604 | // have terminated with a second return value of false; or when one or more 605 | // range or join functions return a second return value of true. In the latter 606 | // case, the first return value of the left-most function that returned true as 607 | // a second return value becomes the final result, without waiting for the other 608 | // range and pair reducers to terminate. 609 | // 610 | // RangeReduce panics if high < low, or if n < 0. 611 | // 612 | // If one or more reducer invocations panic, the corresponding goroutines 613 | // recover the panics, and RangeReduce eventually panics with the left-most 614 | // recovered panic value. 615 | func RangeReduce( 616 | low, high, n int, 617 | reduce func(low, high int) (interface{}, bool), 618 | join func(x, y interface{}) (interface{}, bool), 619 | ) (interface{}, bool) { 620 | var recur func(int, int, int) (interface{}, bool) 621 | recur = func(low, high, n int) (interface{}, bool) { 622 | switch { 623 | case n == 1: 624 | return reduce(low, high) 625 | case n > 1: 626 | batchSize := ((high - low - 1) / n) + 1 627 | half := n / 2 628 | mid := low + batchSize*half 629 | if mid >= high { 630 | return reduce(low, high) 631 | } 632 | var left, right interface{} 633 | var b0, b1 bool 634 | var p interface{} 635 | var wg sync.WaitGroup 636 | wg.Add(1) 637 | go func() { 638 | defer func() { 639 | p = internal.WrapPanic(recover()) 640 | wg.Done() 641 | }() 642 | right, b1 = recur(mid, high, n-half) 643 | }() 644 | left, b0 = recur(low, mid, half) 645 | if b0 { 646 | return left, true 647 | } 648 | wg.Wait() 649 | if p != nil { 650 | panic(p) 651 | } 652 | if b1 { 653 | return right, true 654 | } 655 | return join(left, right) 656 | default: 657 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 658 | } 659 | } 660 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 661 | } 662 | 663 | // RangeReduceInt receives a range, a batch count n, a range reducer function, 664 | // and a join function, divides the range into batches, and invokes the range 665 | // reducer for each of these batches in parallel, covering the half-open 666 | // interval from low to high, including low but excluding high. The results of 667 | // the range reducer invocations are then combined by repeated invocations of 668 | // join. 669 | // 670 | // The range is specified by a low and high integer, with low <= high. The 671 | // batches are determined by dividing up the size of the range (high - low) by 672 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 673 | // into account. 674 | // 675 | // The range reducer is invoked for each batch in its own goroutine, with 0 <= 676 | // low <= high, and RangeReduceInt returns either when all range reducers and 677 | // joins have terminated with a second return value of false; or when one or 678 | // more range or join functions return a second return value of true. In the 679 | // latter case, the first return value of the left-most function that returned 680 | // true as a second return value becomes the final result, without waiting for 681 | // the other range and pair reducers to terminate. 682 | // 683 | // RangeReduceInt panics if high < low, or if n < 0. 684 | // 685 | // If one or more reducer invocations panic, the corresponding goroutines 686 | // recover the panics, and RangeReduceInt eventually panics with the left-most 687 | // recovered panic value. 688 | func RangeReduceInt( 689 | low, high, n int, 690 | reduce func(low, high int) (int, bool), 691 | join func(x, y int) (int, bool), 692 | ) (int, bool) { 693 | var recur func(int, int, int) (int, bool) 694 | recur = func(low, high, n int) (int, bool) { 695 | switch { 696 | case n == 1: 697 | return reduce(low, high) 698 | case n > 1: 699 | batchSize := ((high - low - 1) / n) + 1 700 | half := n / 2 701 | mid := low + batchSize*half 702 | if mid >= high { 703 | return reduce(low, high) 704 | } 705 | var left, right int 706 | var b0, b1 bool 707 | var p interface{} 708 | var wg sync.WaitGroup 709 | wg.Add(1) 710 | go func() { 711 | defer func() { 712 | p = internal.WrapPanic(recover()) 713 | wg.Done() 714 | }() 715 | right, b1 = recur(mid, high, n-half) 716 | }() 717 | left, b0 = recur(low, mid, half) 718 | if b0 { 719 | return left, true 720 | } 721 | wg.Wait() 722 | if p != nil { 723 | panic(p) 724 | } 725 | if b1 { 726 | return right, true 727 | } 728 | return join(left, right) 729 | default: 730 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 731 | } 732 | } 733 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 734 | } 735 | 736 | // RangeReduceFloat64 receives a range, a batch count n, a range reducer 737 | // function, and a join function, divides the range into batches, and invokes 738 | // the range reducer for each of these batches in parallel, covering the 739 | // half-open interval from low to high, including low but excluding high. The 740 | // results of the range reducer invocations are then combined by repeated 741 | // invocations of join. 742 | // 743 | // The range is specified by a low and high integer, with low <= high. The 744 | // batches are determined by dividing up the size of the range (high - low) by 745 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 746 | // into account. 747 | // 748 | // The range reducer is invoked for each batch in its own goroutine, with 0 <= 749 | // low <= high, and RangeReduceFloat64 returns either when all range reducers 750 | // and joins have terminated with a second return value of false; or when one or 751 | // more range or join functions return a second return value of true. In the 752 | // latter case, the first return value of the left-most function that returned 753 | // true as a second return value becomes the final result, without waiting for 754 | // the other range and pair reducers to terminate. 755 | // 756 | // RangeReduceFloat64 panics if high < low, or if n < 0. 757 | // 758 | // If one or more reducer invocations panic, the corresponding goroutines 759 | // recover the panics, and RangeReduceFloat64 eventually panics with the 760 | // left-most recovered panic value. 761 | func RangeReduceFloat64( 762 | low, high, n int, 763 | reduce func(low, high int) (float64, bool), 764 | join func(x, y float64) (float64, bool), 765 | ) (float64, bool) { 766 | var recur func(int, int, int) (float64, bool) 767 | recur = func(low, high, n int) (float64, bool) { 768 | switch { 769 | case n == 1: 770 | return reduce(low, high) 771 | case n > 1: 772 | batchSize := ((high - low - 1) / n) + 1 773 | half := n / 2 774 | mid := low + batchSize*half 775 | if mid >= high { 776 | return reduce(low, high) 777 | } 778 | var left, right float64 779 | var b0, b1 bool 780 | var p interface{} 781 | var wg sync.WaitGroup 782 | wg.Add(1) 783 | go func() { 784 | defer func() { 785 | p = internal.WrapPanic(recover()) 786 | wg.Done() 787 | }() 788 | right, b1 = recur(mid, high, n-half) 789 | }() 790 | left, b0 = recur(low, mid, half) 791 | if b0 { 792 | return left, true 793 | } 794 | wg.Wait() 795 | if p != nil { 796 | panic(p) 797 | } 798 | if b1 { 799 | return right, true 800 | } 801 | return join(left, right) 802 | default: 803 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 804 | } 805 | } 806 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 807 | } 808 | 809 | // RangeReduceString receives a range, a batch count n, a range reducer 810 | // function, and a join function, divides the range into batches, and invokes 811 | // the range reducer for each of these batches in parallel, covering the 812 | // half-open interval from low to high, including low but excluding high. The 813 | // results of the range reducer invocations are then combined by repeated 814 | // invocations of join. 815 | // 816 | // The range is specified by a low and high integer, with low <= high. The 817 | // batches are determined by dividing up the size of the range (high - low) by 818 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 819 | // into account. 820 | // 821 | // The range reducer is invoked for each batch in its own goroutine, with 0 <= 822 | // low <= high, and RangeReduceString returns either when all range reducers and 823 | // joins have terminated with a second return value of false; or when one or 824 | // more range or join functions return a second return value of true. In the 825 | // latter case, the first return value of the left-most function that returned 826 | // true as a second return value becomes the final result, without waiting for 827 | // the other range and pair reducers to terminate. 828 | // 829 | // RangeReduceString panics if high < low, or if n < 0. 830 | // 831 | // If one or more reducer invocations panic, the corresponding goroutines 832 | // recover the panics, and RangeReduceString eventually panics with the 833 | // left-most recovered panic value. 834 | func RangeReduceString( 835 | low, high, n int, 836 | reduce func(low, high int) (string, bool), 837 | join func(x, y string) (string, bool), 838 | ) (string, bool) { 839 | var recur func(int, int, int) (string, bool) 840 | recur = func(low, high, n int) (string, bool) { 841 | switch { 842 | case n == 1: 843 | return reduce(low, high) 844 | case n > 1: 845 | batchSize := ((high - low - 1) / n) + 1 846 | half := n / 2 847 | mid := low + batchSize*half 848 | if mid >= high { 849 | return reduce(low, high) 850 | } 851 | var left, right string 852 | var b0, b1 bool 853 | var p interface{} 854 | var wg sync.WaitGroup 855 | wg.Add(1) 856 | go func() { 857 | defer func() { 858 | p = internal.WrapPanic(recover()) 859 | wg.Done() 860 | }() 861 | right, b1 = recur(mid, high, n-half) 862 | }() 863 | left, b0 = recur(low, mid, half) 864 | if b0 { 865 | return left, true 866 | } 867 | wg.Wait() 868 | if p != nil { 869 | panic(p) 870 | } 871 | if b1 { 872 | return right, true 873 | } 874 | return join(left, right) 875 | default: 876 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 877 | } 878 | } 879 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 880 | } 881 | -------------------------------------------------------------------------------- /parallel/parallel.go: -------------------------------------------------------------------------------- 1 | // Package parallel provides functions for expressing parallel algorithms. 2 | // 3 | // See https://github.com/ExaScience/pargo/wiki/TaskParallelism for a general 4 | // overview. 5 | package parallel 6 | 7 | import ( 8 | "fmt" 9 | "sync" 10 | 11 | "github.com/exascience/pargo/internal" 12 | ) 13 | 14 | // Reduce receives one or more functions, executes them in parallel, and 15 | // combines their results with the join function in parallel. 16 | // 17 | // Each function is invoked in its own goroutine, and Reduce returns only when 18 | // all functions have terminated. 19 | // 20 | // If one or more functions panic, the corresponding goroutines recover the 21 | // panics, and Reduce eventually panics with the left-most recovered panic 22 | // value. 23 | func Reduce( 24 | join func(x, y interface{}) interface{}, 25 | firstFunction func() interface{}, 26 | moreFunctions ...func() interface{}, 27 | ) interface{} { 28 | if len(moreFunctions) == 0 { 29 | return firstFunction() 30 | } 31 | var left, right interface{} 32 | var p interface{} 33 | var wg sync.WaitGroup 34 | wg.Add(1) 35 | if len(moreFunctions) == 1 { 36 | go func() { 37 | defer func() { 38 | p = internal.WrapPanic(recover()) 39 | wg.Done() 40 | }() 41 | right = moreFunctions[0]() 42 | }() 43 | left = firstFunction() 44 | } else { 45 | half := (len(moreFunctions) + 1) / 2 46 | go func() { 47 | defer func() { 48 | p = internal.WrapPanic(recover()) 49 | wg.Done() 50 | }() 51 | right = Reduce(join, moreFunctions[half], moreFunctions[half+1:]...) 52 | }() 53 | left = Reduce(join, firstFunction, moreFunctions[:half]...) 54 | } 55 | wg.Wait() 56 | if p != nil { 57 | panic(p) 58 | } 59 | return join(left, right) 60 | } 61 | 62 | // ReduceFloat64 receives one or more functions, executes them in parallel, and 63 | // combines their results with the join function in parallel. 64 | // 65 | // Each function is invoked in its own goroutine, and ReduceFloat64 returns only 66 | // when all functions have terminated. 67 | // 68 | // If one or more functions panic, the corresponding goroutines recover the 69 | // panics, and ReduceFloat64 eventually panics with the left-most recovered 70 | // panic value. 71 | func ReduceFloat64( 72 | join func(x, y float64) float64, 73 | firstFunction func() float64, 74 | moreFunctions ...func() float64, 75 | ) float64 { 76 | if len(moreFunctions) == 0 { 77 | return firstFunction() 78 | } 79 | var left, right float64 80 | var p interface{} 81 | var wg sync.WaitGroup 82 | wg.Add(1) 83 | if len(moreFunctions) == 1 { 84 | go func() { 85 | defer func() { 86 | p = internal.WrapPanic(recover()) 87 | wg.Done() 88 | }() 89 | right = moreFunctions[0]() 90 | }() 91 | left = firstFunction() 92 | } else { 93 | half := (len(moreFunctions) + 1) / 2 94 | go func() { 95 | defer func() { 96 | p = internal.WrapPanic(recover()) 97 | wg.Done() 98 | }() 99 | right = ReduceFloat64(join, moreFunctions[half], moreFunctions[half+1:]...) 100 | }() 101 | left = ReduceFloat64(join, firstFunction, moreFunctions[:half]...) 102 | } 103 | wg.Wait() 104 | if p != nil { 105 | panic(p) 106 | } 107 | return join(left, right) 108 | } 109 | 110 | // ReduceFloat64Sum receives zero or more functions, executes them in parallel, 111 | // and adds their results in parallel. 112 | // 113 | // Each function is invoked in its own goroutine, and ReduceFloat64Sum returns 114 | // only when all functions have terminated. 115 | // 116 | // If one or more functions panic, the corresponding goroutines recover the 117 | // panics, and ReduceFloat64Sum eventually panics with the left-most recovered 118 | // panic value. 119 | func ReduceFloat64Sum(functions ...func() float64) float64 { 120 | switch len(functions) { 121 | case 0: 122 | return 0 123 | case 1: 124 | return functions[0]() 125 | } 126 | var left, right float64 127 | var p interface{} 128 | var wg sync.WaitGroup 129 | wg.Add(1) 130 | switch len(functions) { 131 | case 2: 132 | go func() { 133 | defer func() { 134 | p = internal.WrapPanic(recover()) 135 | wg.Done() 136 | }() 137 | right = functions[1]() 138 | }() 139 | left = functions[0]() 140 | default: 141 | half := len(functions) / 2 142 | go func() { 143 | defer func() { 144 | p = internal.WrapPanic(recover()) 145 | wg.Done() 146 | }() 147 | right = ReduceFloat64Sum(functions[half:]...) 148 | }() 149 | left = ReduceFloat64Sum(functions[:half]...) 150 | } 151 | wg.Wait() 152 | if p != nil { 153 | panic(p) 154 | } 155 | return left + right 156 | } 157 | 158 | // ReduceFloat64Product receives zero or more functions, executes them in 159 | // parallel, and multiplies their results in parallel. 160 | // 161 | // Each function is invoked in its own goroutine, and ReduceFloat64Product 162 | // returns only when all functions have terminated. 163 | // 164 | // If one or more functions panic, the corresponding goroutines recover the 165 | // panics, and ReduceFloat64Product eventually panics with the left-most 166 | // recovered panic value. 167 | func ReduceFloat64Product(functions ...func() float64) float64 { 168 | switch len(functions) { 169 | case 0: 170 | return 1 171 | case 1: 172 | return functions[0]() 173 | } 174 | var left, right float64 175 | var p interface{} 176 | var wg sync.WaitGroup 177 | wg.Add(1) 178 | switch len(functions) { 179 | case 2: 180 | go func() { 181 | defer func() { 182 | p = internal.WrapPanic(recover()) 183 | wg.Done() 184 | }() 185 | right = functions[1]() 186 | }() 187 | left = functions[0]() 188 | default: 189 | half := len(functions) / 2 190 | go func() { 191 | defer func() { 192 | p = internal.WrapPanic(recover()) 193 | wg.Done() 194 | }() 195 | right = ReduceFloat64Product(functions[half:]...) 196 | }() 197 | left = ReduceFloat64Product(functions[:half]...) 198 | } 199 | wg.Wait() 200 | if p != nil { 201 | panic(p) 202 | } 203 | return left * right 204 | } 205 | 206 | // ReduceInt receives zero or more functions, executes them in parallel, and 207 | // combines their results with the join function in parallel. 208 | // 209 | // Each function is invoked in its own goroutine, and ReduceInt returns only 210 | // when all functions have terminated. 211 | // 212 | // If one or more functions panic, the corresponding goroutines recover the 213 | // panics, and ReduceInt eventually panics with the left-most recovered panic 214 | // value. 215 | func ReduceInt( 216 | join func(x, y int) int, 217 | firstFunction func() int, 218 | moreFunctions ...func() int, 219 | ) int { 220 | if len(moreFunctions) == 0 { 221 | return firstFunction() 222 | } 223 | var left, right int 224 | var p interface{} 225 | var wg sync.WaitGroup 226 | wg.Add(1) 227 | if len(moreFunctions) == 1 { 228 | go func() { 229 | defer func() { 230 | p = internal.WrapPanic(recover()) 231 | wg.Done() 232 | }() 233 | right = moreFunctions[0]() 234 | }() 235 | left = firstFunction() 236 | } else { 237 | half := (len(moreFunctions) + 1) / 2 238 | go func() { 239 | defer func() { 240 | p = internal.WrapPanic(recover()) 241 | wg.Done() 242 | }() 243 | right = ReduceInt(join, moreFunctions[half], moreFunctions[half+1:]...) 244 | }() 245 | left = ReduceInt(join, firstFunction, moreFunctions[:half]...) 246 | } 247 | wg.Wait() 248 | if p != nil { 249 | panic(p) 250 | } 251 | return join(left, right) 252 | } 253 | 254 | // ReduceIntSum receives zero or more functions, executes them in parallel, and 255 | // adds their results in parallel. 256 | // 257 | // Each function is invoked in its own goroutine, and ReduceIntSum returns only 258 | // when all functions have terminated. 259 | // 260 | // If one or more functions panic, the corresponding goroutines recover the 261 | // panics, and ReduceIntSum eventually panics with the left-most recovered panic 262 | // value. 263 | func ReduceIntSum(functions ...func() int) int { 264 | switch len(functions) { 265 | case 0: 266 | return 0 267 | case 1: 268 | return functions[0]() 269 | } 270 | var left, right int 271 | var p interface{} 272 | var wg sync.WaitGroup 273 | wg.Add(1) 274 | switch len(functions) { 275 | case 2: 276 | go func() { 277 | defer func() { 278 | p = internal.WrapPanic(recover()) 279 | wg.Done() 280 | }() 281 | right = functions[1]() 282 | }() 283 | left = functions[0]() 284 | default: 285 | half := len(functions) / 2 286 | go func() { 287 | defer func() { 288 | p = internal.WrapPanic(recover()) 289 | wg.Done() 290 | }() 291 | right = ReduceIntSum(functions[half:]...) 292 | }() 293 | left = ReduceIntSum(functions[:half]...) 294 | } 295 | wg.Wait() 296 | if p != nil { 297 | panic(p) 298 | } 299 | return left + right 300 | } 301 | 302 | // ReduceIntProduct receives zero or more functions, executes them in parallel, 303 | // and multiplies their results in parallel. 304 | // 305 | // Each function is invoked in its own goroutine, and ReduceIntProduct returns 306 | // only when all functions have terminated. 307 | // 308 | // If one or more functions panic, the corresponding goroutines recover the 309 | // panics, and ReduceIntProduct eventually panics with the left-most recovered 310 | // panic value. 311 | func ReduceIntProduct(functions ...func() int) int { 312 | switch len(functions) { 313 | case 0: 314 | return 1 315 | case 1: 316 | return functions[0]() 317 | } 318 | var left, right int 319 | var p interface{} 320 | var wg sync.WaitGroup 321 | wg.Add(1) 322 | switch len(functions) { 323 | case 2: 324 | go func() { 325 | defer func() { 326 | p = internal.WrapPanic(recover()) 327 | wg.Done() 328 | }() 329 | right = functions[1]() 330 | }() 331 | left = functions[0]() 332 | default: 333 | half := len(functions) / 2 334 | go func() { 335 | defer func() { 336 | p = internal.WrapPanic(recover()) 337 | wg.Done() 338 | }() 339 | right = ReduceIntProduct(functions[half:]...) 340 | }() 341 | left = ReduceIntProduct(functions[:half]...) 342 | } 343 | wg.Wait() 344 | if p != nil { 345 | panic(p) 346 | } 347 | return left * right 348 | } 349 | 350 | // ReduceString receives zero or more functions, executes them in parallel, and 351 | // combines their results with the join function in parallel. 352 | // 353 | // Each function is invoked in its own goroutine, and ReduceString returns only 354 | // when all functions have terminated. 355 | // 356 | // If one or more functions panic, the corresponding goroutines recover the 357 | // panics, and ReduceString eventually panics with the left-most recovered panic 358 | // value. 359 | func ReduceString( 360 | join func(x, y string) string, 361 | firstFunction func() string, 362 | moreFunctions ...func() string, 363 | ) string { 364 | if len(moreFunctions) == 0 { 365 | return firstFunction() 366 | } 367 | var left, right string 368 | var p interface{} 369 | var wg sync.WaitGroup 370 | wg.Add(1) 371 | if len(moreFunctions) == 1 { 372 | go func() { 373 | defer func() { 374 | p = internal.WrapPanic(recover()) 375 | wg.Done() 376 | }() 377 | right = moreFunctions[0]() 378 | }() 379 | left = firstFunction() 380 | } else { 381 | half := (len(moreFunctions) + 1) / 2 382 | go func() { 383 | defer func() { 384 | p = internal.WrapPanic(recover()) 385 | wg.Done() 386 | }() 387 | right = ReduceString(join, moreFunctions[half], moreFunctions[half+1:]...) 388 | }() 389 | left = ReduceString(join, firstFunction, moreFunctions[:half]...) 390 | } 391 | wg.Wait() 392 | if p != nil { 393 | panic(p) 394 | } 395 | return join(left, right) 396 | } 397 | 398 | // ReduceStringSum receives zero or more functions, executes them in parallel, 399 | // and concatenates their results in parallel. 400 | // 401 | // Each function is invoked in its own goroutine, and ReduceStringSum returns 402 | // only when all functions have terminated. 403 | // 404 | // If one or more functions panic, the corresponding goroutines recover the 405 | // panics, and ReduceStringSum eventually panics with the left-most recovered 406 | // panic value. 407 | func ReduceStringSum(functions ...func() string) string { 408 | switch len(functions) { 409 | case 0: 410 | return "" 411 | case 1: 412 | return functions[0]() 413 | } 414 | var left, right string 415 | var p interface{} 416 | var wg sync.WaitGroup 417 | wg.Add(1) 418 | switch len(functions) { 419 | case 2: 420 | go func() { 421 | defer func() { 422 | p = internal.WrapPanic(recover()) 423 | wg.Done() 424 | }() 425 | right = functions[1]() 426 | }() 427 | left = functions[0]() 428 | default: 429 | half := len(functions) / 2 430 | go func() { 431 | defer func() { 432 | p = internal.WrapPanic(recover()) 433 | wg.Done() 434 | }() 435 | right = ReduceStringSum(functions[half:]...) 436 | }() 437 | left = ReduceStringSum(functions[:half]...) 438 | } 439 | wg.Wait() 440 | if p != nil { 441 | panic(p) 442 | } 443 | return left + right 444 | } 445 | 446 | // Do receives zero or more thunks and executes them in parallel. 447 | // 448 | // Each thunk is invoked in its own goroutine, and Do returns only when all 449 | // thunks have terminated. 450 | // 451 | // If one or more thunks panic, the corresponding goroutines recover the panics, 452 | // and Do eventually panics with the left-most recovered panic value. 453 | func Do(thunks ...func()) { 454 | switch len(thunks) { 455 | case 0: 456 | return 457 | case 1: 458 | thunks[0]() 459 | return 460 | } 461 | var p interface{} 462 | var wg sync.WaitGroup 463 | wg.Add(1) 464 | switch len(thunks) { 465 | case 2: 466 | go func() { 467 | defer func() { 468 | p = internal.WrapPanic(recover()) 469 | wg.Done() 470 | }() 471 | thunks[1]() 472 | }() 473 | thunks[0]() 474 | default: 475 | half := len(thunks) / 2 476 | go func() { 477 | defer func() { 478 | p = internal.WrapPanic(recover()) 479 | wg.Done() 480 | }() 481 | Do(thunks[half:]...) 482 | }() 483 | Do(thunks[:half]...) 484 | } 485 | wg.Wait() 486 | if p != nil { 487 | panic(p) 488 | } 489 | } 490 | 491 | // And receives zero or more predicate functions and executes them in parallel. 492 | // 493 | // Each predicate is invoked in its own goroutine, and And returns only when all 494 | // predicates have terminated, combining all return values with the && operator, 495 | // with true as the default return value. 496 | // 497 | // If one or more predicates panic, the corresponding goroutines recover the 498 | // panics, and And eventually panics with the left-most recovered panic value. 499 | func And(predicates ...func() bool) bool { 500 | switch len(predicates) { 501 | case 0: 502 | return true 503 | case 1: 504 | return predicates[0]() 505 | } 506 | var b0, b1 bool 507 | var p interface{} 508 | var wg sync.WaitGroup 509 | wg.Add(1) 510 | switch len(predicates) { 511 | case 2: 512 | go func() { 513 | defer func() { 514 | p = internal.WrapPanic(recover()) 515 | wg.Done() 516 | }() 517 | b1 = predicates[1]() 518 | }() 519 | b0 = predicates[0]() 520 | default: 521 | half := len(predicates) / 2 522 | go func() { 523 | defer func() { 524 | p = internal.WrapPanic(recover()) 525 | wg.Done() 526 | }() 527 | b1 = And(predicates[half:]...) 528 | }() 529 | b0 = And(predicates[:half]...) 530 | } 531 | wg.Wait() 532 | if p != nil { 533 | panic(p) 534 | } 535 | return b0 && b1 536 | } 537 | 538 | // Or receives zero or more predicate functions and executes them in parallel. 539 | // 540 | // Each predicate is invoked in its own goroutine, and Or returns only when all 541 | // predicates have terminated, combining all return values with the || operator, 542 | // with false as the default return value. 543 | // 544 | // If one or more predicates panic, the corresponding goroutines recover the 545 | // panics, and Or eventually panics with the left-most recovered panic value. 546 | func Or(predicates ...func() bool) bool { 547 | switch len(predicates) { 548 | case 0: 549 | return false 550 | case 1: 551 | return predicates[0]() 552 | } 553 | var b0, b1 bool 554 | var p interface{} 555 | var wg sync.WaitGroup 556 | wg.Add(1) 557 | switch len(predicates) { 558 | case 2: 559 | go func() { 560 | defer func() { 561 | p = internal.WrapPanic(recover()) 562 | wg.Done() 563 | }() 564 | b1 = predicates[1]() 565 | }() 566 | b0 = predicates[0]() 567 | default: 568 | half := len(predicates) / 2 569 | go func() { 570 | defer func() { 571 | p = internal.WrapPanic(recover()) 572 | wg.Done() 573 | }() 574 | b1 = Or(predicates[half:]...) 575 | }() 576 | b0 = Or(predicates[:half]...) 577 | } 578 | wg.Wait() 579 | if p != nil { 580 | panic(p) 581 | } 582 | return b0 || b1 583 | } 584 | 585 | // Range receives a range, a batch count n, and a range function f, divides the 586 | // range into batches, and invokes the range function for each of these batches 587 | // in parallel, covering the half-open interval from low to high, including low 588 | // but excluding high. 589 | // 590 | // The range is specified by a low and high integer, with low <= high. The 591 | // batches are determined by dividing up the size of the range (high - low) by 592 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 593 | // into account. 594 | // 595 | // The range function is invoked for each batch in its own goroutine, with 0 <= 596 | // low <= high, and Range returns only when all range functions have terminated. 597 | // 598 | // Range panics if high < low, or if n < 0. 599 | // 600 | // If one or more range function invocations panic, the corresponding goroutines 601 | // recover the panics, and Range eventually panics with the left-most recovered 602 | // panic value. 603 | func Range( 604 | low, high, n int, 605 | f func(low, high int), 606 | ) { 607 | var recur func(int, int, int) 608 | recur = func(low, high, n int) { 609 | switch { 610 | case n == 1: 611 | f(low, high) 612 | case n > 1: 613 | batchSize := ((high - low - 1) / n) + 1 614 | half := n / 2 615 | mid := low + batchSize*half 616 | if mid >= high { 617 | f(low, high) 618 | return 619 | } 620 | var p interface{} 621 | var wg sync.WaitGroup 622 | wg.Add(1) 623 | go func() { 624 | defer func() { 625 | p = internal.WrapPanic(recover()) 626 | wg.Done() 627 | }() 628 | recur(mid, high, n-half) 629 | }() 630 | recur(low, mid, half) 631 | wg.Wait() 632 | if p != nil { 633 | panic(p) 634 | } 635 | return 636 | default: 637 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 638 | } 639 | } 640 | recur(low, high, internal.ComputeNofBatches(low, high, n)) 641 | } 642 | 643 | // RangeAnd receives a range, a batch count n, and a range predicate function f, 644 | // divides the range into batches, and invokes the range predicate for each of 645 | // these batches in parallel, covering the half-open interval from low to high, 646 | // including low but excluding high. 647 | // 648 | // The range is specified by a low and high integer, with low <= high. The 649 | // batches are determined by dividing up the size of the range (high - low) by 650 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 651 | // into account. 652 | // 653 | // The range predicate is invoked for each batch in its own goroutine, with 0 <= 654 | // low <= high, and RangeAnd returns only when all range predicates have 655 | // terminated, combining all return values with the && operator. 656 | // 657 | // RangeAnd panics if high < low, or if n < 0. 658 | // 659 | // If one or more range predicate invocations panic, the corresponding 660 | // goroutines recover the panics, and RangeAnd eventually panics with the 661 | // left-most recovered panic value. 662 | func RangeAnd( 663 | low, high, n int, 664 | f func(low, high int) bool, 665 | ) bool { 666 | var recur func(int, int, int) bool 667 | recur = func(low, high, n int) bool { 668 | switch { 669 | case n == 1: 670 | return f(low, high) 671 | case n > 1: 672 | batchSize := ((high - low - 1) / n) + 1 673 | half := n / 2 674 | mid := low + batchSize*half 675 | if mid >= high { 676 | return f(low, high) 677 | } 678 | var b0, b1 bool 679 | var p interface{} 680 | var wg sync.WaitGroup 681 | wg.Add(1) 682 | go func() { 683 | defer func() { 684 | p = internal.WrapPanic(recover()) 685 | wg.Done() 686 | }() 687 | b1 = recur(mid, high, n-half) 688 | }() 689 | b0 = recur(low, mid, half) 690 | wg.Wait() 691 | if p != nil { 692 | panic(p) 693 | } 694 | return b0 && b1 695 | default: 696 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 697 | } 698 | } 699 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 700 | } 701 | 702 | // RangeOr receives a range, a batch count n, and a range predicate function f, 703 | // divides the range into batches, and invokes the range predicate for each of 704 | // these batches in parallel, covering the half-open interval from low to high, 705 | // including low but excluding high. 706 | // 707 | // The range is specified by a low and high integer, with low <= high. The 708 | // batches are determined by dividing up the size of the range (high - low) by 709 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 710 | // into account. 711 | // 712 | // The range predicate is invoked for each batch in its own goroutine, with 0 <= 713 | // low <= high, and RangeOr returns only when all range predicates have 714 | // terminated, combining all return values with the || operator. 715 | // 716 | // RangeOr panics if high < low, or if n < 0. 717 | // 718 | // If one or more range predicate invocations panic, the corresponding 719 | // goroutines recover the panics, and RangeOr eventually panics with the 720 | // left-most recovered panic value. 721 | func RangeOr( 722 | low, high, n int, 723 | f func(low, high int) bool, 724 | ) bool { 725 | var recur func(int, int, int) bool 726 | recur = func(low, high, n int) bool { 727 | switch { 728 | case n == 1: 729 | return f(low, high) 730 | case n > 1: 731 | batchSize := ((high - low - 1) / n) + 1 732 | half := n / 2 733 | mid := low + batchSize*half 734 | if mid >= high { 735 | return f(low, high) 736 | } 737 | var b0, b1 bool 738 | var p interface{} 739 | var wg sync.WaitGroup 740 | wg.Add(1) 741 | go func() { 742 | defer func() { 743 | p = internal.WrapPanic(recover()) 744 | wg.Done() 745 | }() 746 | b1 = recur(mid, high, n-half) 747 | }() 748 | b0 = recur(low, mid, half) 749 | wg.Wait() 750 | if p != nil { 751 | panic(p) 752 | } 753 | return b0 || b1 754 | default: 755 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 756 | } 757 | } 758 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 759 | } 760 | 761 | // RangeReduce receives a range, a batch count, a range reduce function, and a 762 | // join function, divides the range into batches, and invokes the range reducer 763 | // for each of these batches in parallel, covering the half-open interval from 764 | // low to high, including low but excluding high. The results of the range 765 | // reducer invocations are then combined by repeated invocations of join. 766 | // 767 | // The range is specified by a low and high integer, with low <= high. The 768 | // batches are determined by dividing up the size of the range (high - low) by 769 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 770 | // into account. 771 | // 772 | // The range reducer is invoked for each batch in its own goroutine, with 0 <= 773 | // low <= high, and RangeReduce returns only when all range reducers and pair 774 | // reducers have terminated. 775 | // 776 | // RangeReduce panics if high < low, or if n < 0. 777 | // 778 | // If one or more reducer invocations panic, the corresponding goroutines 779 | // recover the panics, and RangeReduce eventually panics with the left-most 780 | // recovered panic value. 781 | func RangeReduce( 782 | low, high, n int, 783 | reduce func(low, high int) interface{}, 784 | join func(x, y interface{}) interface{}, 785 | ) interface{} { 786 | var recur func(int, int, int) interface{} 787 | recur = func(low, high, n int) interface{} { 788 | switch { 789 | case n == 1: 790 | return reduce(low, high) 791 | case n > 1: 792 | batchSize := ((high - low - 1) / n) + 1 793 | half := n / 2 794 | mid := low + batchSize*half 795 | if mid >= high { 796 | return reduce(low, high) 797 | } 798 | var left, right interface{} 799 | var p interface{} 800 | var wg sync.WaitGroup 801 | wg.Add(1) 802 | go func() { 803 | defer func() { 804 | p = internal.WrapPanic(recover()) 805 | wg.Done() 806 | }() 807 | right = recur(mid, high, n-half) 808 | }() 809 | left = recur(low, mid, half) 810 | wg.Wait() 811 | if p != nil { 812 | panic(p) 813 | } 814 | return join(left, right) 815 | default: 816 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 817 | } 818 | } 819 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 820 | } 821 | 822 | // RangeReduceInt receives a range, a batch count n, a range reducer function, 823 | // and a join function, divides the range into batches, and invokes the range 824 | // reducer for each of these batches in parallel, covering the half-open 825 | // interval from low to high, including low but excluding high. The results of 826 | // the range reducer invocations are then combined by repeated invocations of 827 | // join. 828 | // 829 | // The range is specified by a low and high integer, with low <= high. The 830 | // batches are determined by dividing up the size of the range (high - low) by 831 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 832 | // into account. 833 | // 834 | // The range reducer is invoked for each batch in its own goroutine, with 0 <= 835 | // low <= high, and RangeReduceInt returns only when all range reducers and pair 836 | // reducers have terminated. 837 | // 838 | // RangeReduceInt panics if high < low, or if n < 0. 839 | // 840 | // If one or more reducer invocations panic, the corresponding goroutines 841 | // recover the panics, and RangeReduceInt eventually panics with the left-most 842 | // recovered panic value. 843 | func RangeReduceInt( 844 | low, high, n int, 845 | reduce func(low, high int) int, 846 | join func(x, y int) int, 847 | ) int { 848 | var recur func(int, int, int) int 849 | recur = func(low, high, n int) int { 850 | switch { 851 | case n == 1: 852 | return reduce(low, high) 853 | case n > 1: 854 | batchSize := ((high - low - 1) / n) + 1 855 | half := n / 2 856 | mid := low + batchSize*half 857 | if mid >= high { 858 | return reduce(low, high) 859 | } 860 | var left, right int 861 | var p interface{} 862 | var wg sync.WaitGroup 863 | wg.Add(1) 864 | go func() { 865 | defer func() { 866 | p = internal.WrapPanic(recover()) 867 | wg.Done() 868 | }() 869 | right = recur(mid, high, n-half) 870 | }() 871 | left = recur(low, mid, half) 872 | wg.Wait() 873 | if p != nil { 874 | panic(p) 875 | } 876 | return join(left, right) 877 | default: 878 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 879 | } 880 | } 881 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 882 | } 883 | 884 | // RangeReduceIntSum receives a range, a batch count n, and a range reducer 885 | // function, divides the range into batches, and invokes the range reducer for 886 | // each of these batches in parallel, covering the half-open interval from low 887 | // to high, including low but excluding high. The results of the range reducer 888 | // invocations are then added together. 889 | // 890 | // The range is specified by a low and high integer, with low <= high. The 891 | // batches are determined by dividing up the size of the range (high - low) by 892 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 893 | // into account. 894 | // 895 | // The range reducer is invoked for each batch in its own goroutine, with 0 <= 896 | // low <= high, and RangeReduceIntSum returns only when all range reducers and 897 | // pair reducers have terminated. 898 | // 899 | // RangeReduceIntSum panics if high < low, or if n < 0. 900 | // 901 | // If one or more reducer invocations panic, the corresponding goroutines 902 | // recover the panics, and RangeReduceIntSum eventually panics with the 903 | // left-most recovered panic value. 904 | func RangeReduceIntSum( 905 | low, high, n int, 906 | reduce func(low, high int) int, 907 | ) int { 908 | var recur func(int, int, int) int 909 | recur = func(low, high, n int) int { 910 | switch { 911 | case n == 1: 912 | return reduce(low, high) 913 | case n > 1: 914 | batchSize := ((high - low - 1) / n) + 1 915 | half := n / 2 916 | mid := low + batchSize*half 917 | if mid >= high { 918 | return reduce(low, high) 919 | } 920 | var left, right int 921 | var p interface{} 922 | var wg sync.WaitGroup 923 | wg.Add(1) 924 | go func() { 925 | defer func() { 926 | p = internal.WrapPanic(recover()) 927 | wg.Done() 928 | }() 929 | right = recur(mid, high, n-half) 930 | }() 931 | left = recur(low, mid, half) 932 | wg.Wait() 933 | if p != nil { 934 | panic(p) 935 | } 936 | return left + right 937 | default: 938 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 939 | } 940 | } 941 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 942 | } 943 | 944 | // RangeReduceIntProduct receives a range, a batch count n, and a range reducer 945 | // function, divides the range into batches, and invokes the range reducer for 946 | // each of these batches in parallel, covering the half-open interval from low 947 | // to high, including low but excluding high. The results of the range reducer 948 | // invocations are then multiplied with each other. 949 | // 950 | // The range is specified by a low and high integer, with low <= high. The 951 | // batches are determined by dividing up the size of the range (high - low) by 952 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 953 | // into account. 954 | // 955 | // The range reducer is invoked for each batch in its own goroutine, with 0 <= 956 | // low <= high, and RangeReduceIntProduct returns only when all range reducers 957 | // and pair reducers have terminated. 958 | // 959 | // RangeReduceIntProduct panics if high < low, or if n < 0. 960 | // 961 | // If one or more reducer invocations panic, the corresponding goroutines 962 | // recover the panics, and RangeReduceIntProducet eventually panics with the 963 | // left-most recovered panic value. 964 | func RangeReduceIntProduct( 965 | low, high, n int, 966 | reduce func(low, high int) int, 967 | ) int { 968 | var recur func(int, int, int) int 969 | recur = func(low, high, n int) int { 970 | switch { 971 | case n == 1: 972 | return reduce(low, high) 973 | case n > 1: 974 | batchSize := ((high - low - 1) / n) + 1 975 | half := n / 2 976 | mid := low + batchSize*half 977 | if mid >= high { 978 | return reduce(low, high) 979 | } 980 | var left, right int 981 | var p interface{} 982 | var wg sync.WaitGroup 983 | wg.Add(1) 984 | go func() { 985 | defer func() { 986 | p = internal.WrapPanic(recover()) 987 | wg.Done() 988 | }() 989 | right = recur(mid, high, n-half) 990 | }() 991 | left = recur(low, mid, half) 992 | wg.Wait() 993 | if p != nil { 994 | panic(p) 995 | } 996 | return left * right 997 | default: 998 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 999 | } 1000 | } 1001 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 1002 | } 1003 | 1004 | // RangeReduceFloat64 receives a range, a batch count n, a range reducer 1005 | // function, and a join function, divides the range into batches, and invokes 1006 | // the range reducer for each of these batches in parallel, covering the 1007 | // half-open interval from low to high, including low but excluding high. The 1008 | // results of the range reducer invocations are then combined by repeated 1009 | // invocations of join. 1010 | // 1011 | // The range is specified by a low and high integer, with low <= high. The 1012 | // batches are determined by dividing up the size of the range (high - low) by 1013 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 1014 | // into account. 1015 | // 1016 | // The range reducer is invoked for each batch in its own goroutine, with 0 <= 1017 | // low <= high, and RangeReduceFloat64 returns only when all range reducers and 1018 | // pair reducers have terminated. 1019 | // 1020 | // RangeReduceFloat64 panics if high < low, or if n < 0. 1021 | // 1022 | // If one or more reducer invocations panic, the corresponding goroutines 1023 | // recover the panics, and RangeReduceFloat64 eventually panics with the 1024 | // left-most recovered panic value. 1025 | func RangeReduceFloat64( 1026 | low, high, n int, 1027 | reduce func(low, high int) float64, 1028 | join func(x, y float64) float64, 1029 | ) float64 { 1030 | var recur func(int, int, int) float64 1031 | recur = func(low, high, n int) float64 { 1032 | switch { 1033 | case n == 1: 1034 | return reduce(low, high) 1035 | case n > 1: 1036 | batchSize := ((high - low - 1) / n) + 1 1037 | half := n / 2 1038 | mid := low + batchSize*half 1039 | if mid >= high { 1040 | return reduce(low, high) 1041 | } 1042 | var left, right float64 1043 | var p interface{} 1044 | var wg sync.WaitGroup 1045 | wg.Add(1) 1046 | go func() { 1047 | defer func() { 1048 | p = internal.WrapPanic(recover()) 1049 | wg.Done() 1050 | }() 1051 | right = recur(mid, high, n-half) 1052 | }() 1053 | left = recur(low, mid, half) 1054 | wg.Wait() 1055 | if p != nil { 1056 | panic(p) 1057 | } 1058 | return join(left, right) 1059 | default: 1060 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 1061 | } 1062 | } 1063 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 1064 | } 1065 | 1066 | // RangeReduceFloat64Sum receives a range, a batch count n, and a range reducer 1067 | // function, divides the range into batches, and invokes the range reducer for 1068 | // each of these batches in parallel, covering the half-open interval from low 1069 | // to high, including low but excluding high. The results of the range reducer 1070 | // invocations are then added together. 1071 | // 1072 | // The range is specified by a low and high integer, with low <= high. The 1073 | // batches are determined by dividing up the size of the range (high - low) by 1074 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 1075 | // into account. 1076 | // 1077 | // The range reducer is invoked for each batch in its own goroutine, with 0 <= 1078 | // low <= high, and RangeReduceFloat64Sum returns only when all range reducers 1079 | // and pair reducers have terminated. 1080 | // 1081 | // RangeReduceFloat64Sum panics if high < low, or if n < 0. 1082 | // 1083 | // If one or more reducer invocations panic, the corresponding goroutines 1084 | // recover the panics, and RangeReduceFloat64Sum eventually panics with the 1085 | // left-most recovered panic value. 1086 | func RangeReduceFloat64Sum( 1087 | low, high, n int, 1088 | reduce func(low, high int) float64, 1089 | ) float64 { 1090 | var recur func(int, int, int) float64 1091 | recur = func(low, high, n int) float64 { 1092 | switch { 1093 | case n == 1: 1094 | return reduce(low, high) 1095 | case n > 1: 1096 | batchSize := ((high - low - 1) / n) + 1 1097 | half := n / 2 1098 | mid := low + batchSize*half 1099 | if mid >= high { 1100 | return reduce(low, high) 1101 | } 1102 | var left, right float64 1103 | var p interface{} 1104 | var wg sync.WaitGroup 1105 | wg.Add(1) 1106 | go func() { 1107 | defer func() { 1108 | p = internal.WrapPanic(recover()) 1109 | wg.Done() 1110 | }() 1111 | right = recur(mid, high, n-half) 1112 | }() 1113 | left = recur(low, mid, half) 1114 | wg.Wait() 1115 | if p != nil { 1116 | panic(p) 1117 | } 1118 | return left + right 1119 | default: 1120 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 1121 | } 1122 | } 1123 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 1124 | } 1125 | 1126 | // RangeReduceFloat64Product receives a range, a batch count n, and a range 1127 | // reducer function, divides the range into batches, and invokes the range 1128 | // reducer for each of these batches in parallel, covering the half-open 1129 | // interval from low to high, including low but excluding high. The results of 1130 | // the range reducer invocations are then multiplied with each other. 1131 | // 1132 | // The range is specified by a low and high integer, with low <= high. The 1133 | // batches are determined by dividing up the size of the range (high - low) by 1134 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 1135 | // into account. 1136 | // 1137 | // The range reducer is invoked for each batch in its own goroutine, with 0 <= 1138 | // low <= high, and RangeReduceFloat64Product returns only when all range 1139 | // reducers and pair reducers have terminated. 1140 | // 1141 | // RangeReduceFloat64Product panics if high < low, or if n < 0. 1142 | // 1143 | // If one or more reducer invocations panic, the corresponding goroutines 1144 | // recover the panics, and RangeReduceFloat64Producet eventually panics with the 1145 | // left-most recovered panic value. 1146 | func RangeReduceFloat64Product( 1147 | low, high, n int, 1148 | reduce func(low, high int) float64, 1149 | ) float64 { 1150 | var recur func(int, int, int) float64 1151 | recur = func(low, high, n int) float64 { 1152 | switch { 1153 | case n == 1: 1154 | return reduce(low, high) 1155 | case n > 1: 1156 | batchSize := ((high - low - 1) / n) + 1 1157 | half := n / 2 1158 | mid := low + batchSize*half 1159 | if mid >= high { 1160 | return reduce(low, high) 1161 | } 1162 | var left, right float64 1163 | var p interface{} 1164 | var wg sync.WaitGroup 1165 | wg.Add(1) 1166 | go func() { 1167 | defer func() { 1168 | p = internal.WrapPanic(recover()) 1169 | wg.Done() 1170 | }() 1171 | right = recur(mid, high, n-half) 1172 | }() 1173 | left = recur(low, mid, half) 1174 | wg.Wait() 1175 | if p != nil { 1176 | panic(p) 1177 | } 1178 | return left * right 1179 | default: 1180 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 1181 | } 1182 | } 1183 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 1184 | } 1185 | 1186 | // RangeReduceString receives a range, a batch count n, a range reducer 1187 | // function, and a join function, divides the range into batches, and invokes 1188 | // the range reducer for each of these batches in parallel, covering the 1189 | // half-open interval from low to high, including low but excluding high. The 1190 | // results of the range reducer invocations are then combined by repeated 1191 | // invocations of join. 1192 | // 1193 | // The range is specified by a low and high integer, with low <= high. The 1194 | // batches are determined by dividing up the size of the range (high - low) by 1195 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 1196 | // into account. 1197 | // 1198 | // The range reducer is invoked for each batch in its own goroutine, with 0 <= 1199 | // low <= high, and RangeReduceString returns only when all range reducers and 1200 | // pair reducers have terminated. 1201 | // 1202 | // RangeReduceString panics if high < low, or if n < 0. 1203 | // 1204 | // If one or more reducer invocations panic, the corresponding goroutines 1205 | // recover the panics, and RangeReduceString eventually panics with the 1206 | // left-most recovered panic value. 1207 | func RangeReduceString( 1208 | low, high, n int, 1209 | reduce func(low, high int) string, 1210 | join func(x, y string) string, 1211 | ) string { 1212 | var recur func(int, int, int) string 1213 | recur = func(low, high, n int) string { 1214 | switch { 1215 | case n == 1: 1216 | return reduce(low, high) 1217 | case n > 1: 1218 | batchSize := ((high - low - 1) / n) + 1 1219 | half := n / 2 1220 | mid := low + batchSize*half 1221 | if mid >= high { 1222 | return reduce(low, high) 1223 | } 1224 | var left, right string 1225 | var p interface{} 1226 | var wg sync.WaitGroup 1227 | wg.Add(1) 1228 | go func() { 1229 | defer func() { 1230 | p = internal.WrapPanic(recover()) 1231 | wg.Done() 1232 | }() 1233 | right = recur(mid, high, n-half) 1234 | }() 1235 | left = recur(low, mid, half) 1236 | wg.Wait() 1237 | if p != nil { 1238 | panic(p) 1239 | } 1240 | return join(left, right) 1241 | default: 1242 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 1243 | } 1244 | } 1245 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 1246 | } 1247 | 1248 | // RangeReduceStringSum receives a range, a batch count n, and a range reducer 1249 | // function, divides the range into batches, and invokes the range reducer for 1250 | // each of these batches in parallel, covering the half-open interval from low 1251 | // to high, including low but excluding high. The results of the range reducer 1252 | // invocations are then concatenated together. 1253 | // 1254 | // The range is specified by a low and high integer, with low <= high. The 1255 | // batches are determined by dividing up the size of the range (high - low) by 1256 | // n. If n is 0, a reasonable default is used that takes runtime.GOMAXPROCS(0) 1257 | // into account. 1258 | // 1259 | // The range reducer is invoked for each batch in its own goroutine, with 0 <= 1260 | // low <= high, and RangeReduceStringSum returns only when all range reducers 1261 | // and pair reducers have terminated. 1262 | // 1263 | // RangeReduceStringSum panics if high < low, or if n < 0. 1264 | // 1265 | // If one or more reducer invocations panic, the corresponding goroutines 1266 | // recover the panics, and RangeReduceStringSum eventually panics with the 1267 | // left-most recovered panic value. 1268 | func RangeReduceStringSum( 1269 | low, high, n int, 1270 | reduce func(low, high int) string, 1271 | ) string { 1272 | var recur func(int, int, int) string 1273 | recur = func(low, high, n int) string { 1274 | switch { 1275 | case n == 1: 1276 | return reduce(low, high) 1277 | case n > 1: 1278 | batchSize := ((high - low - 1) / n) + 1 1279 | half := n / 2 1280 | mid := low + batchSize*half 1281 | if mid >= high { 1282 | return reduce(low, high) 1283 | } 1284 | var left, right string 1285 | var p interface{} 1286 | var wg sync.WaitGroup 1287 | wg.Add(1) 1288 | go func() { 1289 | defer func() { 1290 | p = internal.WrapPanic(recover()) 1291 | wg.Done() 1292 | }() 1293 | right = recur(mid, high, n-half) 1294 | }() 1295 | left = recur(low, mid, half) 1296 | wg.Wait() 1297 | if p != nil { 1298 | panic(p) 1299 | } 1300 | return left + right 1301 | default: 1302 | panic(fmt.Sprintf("invalid number of batches: %v", n)) 1303 | } 1304 | } 1305 | return recur(low, high, internal.ComputeNofBatches(low, high, n)) 1306 | } 1307 | --------------------------------------------------------------------------------