├── .travis.yml ├── README.md ├── doc └── parallel.png ├── parallel.go ├── parallel_n.go ├── parallel_n_test.go └── parallel_test.go /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.0 4 | - 1.1 5 | - tip 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Parallel 2 | 3 | ![logo](doc/parallel.png) 4 | [![Build Status](https://travis-ci.org/wangkuiyi/parallel.png?branch=master)](https://travis-ci.org/wangkuiyi/parallel) 5 | 6 | ## What Is It? 7 | 8 | This package provides OpenMP like syntax for Go. 9 | 10 | ## For 11 | 12 | The function `parallel.For` mimics the `for` structure, but the loop 13 | body is executed in parallel. For example, the following code snippet 14 | fills slice `m` by numbers `{1,0,3,0}`. 15 | 16 | m := make([]int, 4) 17 | for i := 0; i < 4; i += 2 { 18 | m[i] = i+1 19 | } 20 | 21 | The following snippet does the same thing, but in parallel: 22 | 23 | parallel.For(0, 4, 2, func(i int) { m[i] = i+1 }) 24 | 25 | ## Do 26 | 27 | The function `parallel.Do` accepts a variable number of parameters, 28 | each should be a function with no parameter nor return value. It 29 | executes these functions in parallel. For example, the following code 30 | snippet 31 | 32 | m := make([]int, 4) 33 | Do( 34 | func() { m[0] = 0 }, 35 | func() { m[1] = 1 }, 36 | func() { m[2] = 2 }, 37 | func() { m[3] = 3 }) 38 | 39 | fills `m` by `{0,1,2,3}`. 40 | -------------------------------------------------------------------------------- /doc/parallel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangkuiyi/parallel/92b5cec6e73ec1ed9af283f69070b42a63fb94da/doc/parallel.png -------------------------------------------------------------------------------- /parallel.go: -------------------------------------------------------------------------------- 1 | // The package parallel provides OpenMP like syntax for Go. 2 | // 3 | // For 4 | // 5 | // The function parallel.For mimics the for structure, but the loop 6 | // body is executed in parallel. For example, the following code snippet 7 | // fills slice m by numbers {1,0,3,0}. 8 | // m := make([]int, 4) 9 | // for i := 0; i < 4; i += 2 { 10 | // m[i] = i+1 11 | // } 12 | // The following snippet does the same thing, but in parallel: 13 | // parallel.For(0, 4, 2, func(i int) { m[i] = i+1 }) 14 | // 15 | // Do 16 | // 17 | // The function parallel.Do accepts a variable number of parameters, 18 | // each should be a function with no parameter nor return value. It 19 | // executes these functions in parallel. For example, the following code 20 | // snippet 21 | // m := make([]int, 4) 22 | // Do( 23 | // func() { m[0] = 0 }, 24 | // func() { m[1] = 1 }, 25 | // func() { m[2] = 2 }, 26 | // func() { m[3] = 3 }) 27 | // fills m by {0,1,2,3}. 28 | // 29 | package parallel 30 | 31 | import ( 32 | "errors" 33 | "fmt" 34 | "reflect" 35 | "sync" 36 | ) 37 | 38 | // For accepts a worker function with an integer paramter and returns 39 | // either nothing or an error. It then makes (high-low)/step 40 | // simultaneous invocations to the worker. These simultaneous 41 | // invocations mimic the for loop: i := low; i < high; i += step, 42 | // where i is the parameter passed to worker. For examples, please 43 | // refer to unit tests. 44 | func For(low, high, step int, worker interface{}) error { 45 | if low > high { 46 | return fmt.Errorf("low (%d) > high (%d)", low, high) 47 | } 48 | if step <= 0 { 49 | return fmt.Errorf("step (%d) must be positive", step) 50 | } 51 | 52 | t := reflect.TypeOf(worker) 53 | if t.Kind() != reflect.Func { 54 | return errors.New("parallel.For worker must be a function.") 55 | } 56 | if t.NumIn() != 1 { 57 | return errors.New("parallel.For worker must have 1 parameter") 58 | } 59 | if t.In(0).Kind() != reflect.Int { 60 | return errors.New("parallel.For worker must have a int param") 61 | } 62 | if t.NumOut() > 1 { 63 | return errors.New("parallel.For worker must return nothing or error") 64 | } 65 | if t.NumOut() == 1 && t.Out(0).Name() != "error" { 66 | return errors.New("parallel.For worker's return type must be error") 67 | } 68 | 69 | // sem has sufficiently large buffer that prevents blocking. 70 | sem := make(chan int, high-low) 71 | v := reflect.ValueOf(worker) 72 | var errs string 73 | var mutex sync.Mutex 74 | 75 | for i := low; i < high; i += step { 76 | go func(v reflect.Value, i int) { 77 | if t.NumOut() == 0 { // worker returns nothing 78 | v.Call([]reflect.Value{reflect.ValueOf(i)}) 79 | } else { // worker returns an error 80 | r := v.Call([]reflect.Value{reflect.ValueOf(i)}) 81 | if r[0].Interface() != nil { 82 | mutex.Lock() 83 | defer mutex.Unlock() 84 | errs += fmt.Sprintf("%v\n", r[0].Interface().(error)) 85 | } 86 | } 87 | sem <- 1 88 | }(v, i) 89 | } 90 | 91 | for i := low; i < high; i += step { 92 | <-sem 93 | } 94 | 95 | if len(errs) > 0 { 96 | return errors.New(errs) 97 | } 98 | return nil 99 | } 100 | 101 | // Do accepts a varadic parameter of functions and execute them in 102 | // parallel. These functions must have no parameter, and return 103 | // either nothing or an error. For examples, please refer to 104 | // corresponding unit test. 105 | func Do(functions ...interface{}) error { 106 | t := make([]reflect.Type, len(functions)) 107 | 108 | for i, f := range functions { 109 | t[i] = reflect.TypeOf(f) 110 | if t[i].Kind() != reflect.Func { 111 | return fmt.Errorf("The #%d param of Do is not a function", i+1) 112 | } 113 | if t[i].NumIn() != 0 { 114 | return fmt.Errorf( 115 | "The #%d param of Do is not a function with out param", i+1) 116 | } 117 | if t[i].NumOut() > 1 { 118 | return fmt.Errorf( 119 | "The #%d param of Do must return nothing or an error", i+1) 120 | } 121 | if t[i].NumOut() == 1 && t[i].Out(0).Name() != "error" { 122 | return errors.New("Return type is not error") 123 | } 124 | } 125 | 126 | // sem has sufficiently large buffer that prevents blocking. 127 | sem := make(chan int, len(functions)) 128 | errs := make([]error, len(functions)) 129 | 130 | for i, f := range functions { 131 | v := reflect.ValueOf(f) 132 | go func(v reflect.Value, i int) { 133 | if t[i].NumOut() == 0 { // f returns nothing 134 | v.Call(nil) 135 | } else { // f returns an error 136 | r := v.Call(nil) 137 | if r[0].Interface() != nil { 138 | errs[i] = r[0].Interface().(error) 139 | } 140 | } 141 | sem <- 1 142 | }(v, i) 143 | } 144 | 145 | for i := 0; i < len(functions); i++ { 146 | <-sem 147 | } 148 | 149 | r := "" 150 | for _, e := range errs { 151 | if e != nil { 152 | r = r + fmt.Sprintf("%v\n", e) 153 | } 154 | } 155 | 156 | if len(r) > 0 { 157 | return errors.New(r) 158 | } 159 | return nil 160 | } 161 | 162 | // RangeMap calls f for each key-value pairs in map m, and collect 163 | // non-nil errors returned by f as its return value. 164 | func RangeMap(m interface{}, f func(k, v reflect.Value) error) error { 165 | if reflect.TypeOf(m).Kind() != reflect.Map { 166 | panic(fmt.Sprintf("%+v is not a map", m)) 167 | } 168 | 169 | keys := reflect.ValueOf(m).MapKeys() 170 | es := make([]error, len(keys)) 171 | 172 | var wg sync.WaitGroup 173 | for i, k := range keys { 174 | wg.Add(1) // Do not put this line into go func(k,v,i); 175 | // otherwise wg.Wait might be executed before 176 | // wg.Add(1) 177 | go func(k, v reflect.Value, i int) { 178 | defer wg.Done() 179 | es[i] = f(k, v) 180 | }(k, reflect.ValueOf(m).MapIndex(k), i) 181 | } 182 | wg.Wait() 183 | 184 | r := "" 185 | for _, e := range es { 186 | if e != nil { 187 | r += fmt.Sprintf("%v\n", e) 188 | } 189 | } 190 | if len(r) > 0 { 191 | return errors.New(r) 192 | } 193 | return nil 194 | } 195 | -------------------------------------------------------------------------------- /parallel_n.go: -------------------------------------------------------------------------------- 1 | package parallel 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "reflect" 7 | "sync" 8 | ) 9 | 10 | // ForN is similar to For, but constraint the maximum number of 11 | // parallel goroutines. 12 | func ForN(low, high, step, parallelism int, worker interface{}) error { 13 | if low > high { 14 | return fmt.Errorf("low (%d) > high (%d)", low, high) 15 | } 16 | if step <= 0 { 17 | return fmt.Errorf("step (%d) must be positive", step) 18 | } 19 | 20 | typ := reflect.TypeOf(worker) 21 | if typ.Kind() != reflect.Func { 22 | return errors.New("parallel.ForN worker must be a function.") 23 | } 24 | if typ.NumIn() != 1 { 25 | return errors.New("parallel.ForN worker must have 1 parameter") 26 | } 27 | if typ.In(0).Kind() != reflect.Int { 28 | return errors.New("parallel.ForN worker must have a int param") 29 | } 30 | if typ.NumOut() > 1 { 31 | return errors.New("parallel.ForN worker must return nothing or error") 32 | } 33 | if typ.NumOut() == 1 && typ.Out(0).Name() != "error" { 34 | return errors.New("parallel.ForN worker's return type must be error") 35 | } 36 | 37 | chin := make(chan int) 38 | chout := make(chan error) 39 | var wg sync.WaitGroup 40 | wg.Add(parallelism) 41 | var errs string 42 | 43 | go func() { 44 | for i := low; i < high; i += step { 45 | chin <- i 46 | } 47 | close(chin) 48 | }() 49 | 50 | val := reflect.ValueOf(worker) 51 | for i := 0; i < parallelism; i++ { 52 | go func(val reflect.Value) { 53 | defer wg.Done() 54 | for input := range chin { 55 | if typ.NumOut() == 0 { // worker returns nothing 56 | val.Call([]reflect.Value{reflect.ValueOf(input)}) 57 | } else { // worker returns an error 58 | r := val.Call([]reflect.Value{reflect.ValueOf(input)}) 59 | if r[0].Interface() != nil { 60 | chout <- r[0].Interface().(error) 61 | } 62 | } 63 | } 64 | }(val) 65 | } 66 | 67 | go func() { 68 | wg.Wait() 69 | close(chout) 70 | }() 71 | 72 | for e := range chout { 73 | errs += e.Error() + "\n" 74 | } 75 | if len(errs) > 0 { 76 | return errors.New(errs) 77 | } 78 | return nil 79 | } 80 | -------------------------------------------------------------------------------- /parallel_n_test.go: -------------------------------------------------------------------------------- 1 | package parallel 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | ) 7 | 8 | func TestForN(t *testing.T) { 9 | m := make([]int, 40000) 10 | ForN(0, len(m), 1, 10, func(i int) { 11 | if i%2 == 0 { 12 | m[i] = 1 13 | } 14 | }) 15 | sum := 0 16 | for i := 0; i < len(m); i++ { 17 | sum += m[i] 18 | } 19 | if sum != len(m)/2 { 20 | t.Errorf("Expecting %d, got %d", len(m)/2, sum) 21 | } 22 | } 23 | 24 | func TestForNOutput(t *testing.T) { 25 | loop := 40000 26 | e := ForN(0, loop, 1, 10, func(i int) error { 27 | if i%2 == 0 { 28 | return errors.New("A") 29 | } 30 | return nil 31 | }) 32 | if l := len(e.Error()); l != loop { 33 | t.Errorf("Expecting %d, got %d", loop, l) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /parallel_test.go: -------------------------------------------------------------------------------- 1 | package parallel 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "testing" 7 | ) 8 | 9 | func TestFor(t *testing.T) { 10 | m := make([]int, 4) 11 | For(0, 4, 1, func(index int) { m[index] = index }) 12 | if m[0] != 0 || m[1] != 1 || m[2] != 2 || m[3] != 3 { 13 | t.Errorf("Failed setting arry m using For\n") 14 | } 15 | } 16 | 17 | func TestForWithErrors(t *testing.T) { 18 | m := make([]int, 4) 19 | e := For(0, 4, 1, func(index int) error { 20 | m[index] = index 21 | if index < 2 { 22 | return fmt.Errorf("Error %d", index) 23 | } 24 | return nil 25 | }) 26 | 27 | if m[0] != 0 || m[1] != 1 || m[2] != 2 || m[3] != 3 { 28 | t.Errorf("Failed setting arry m using For\n") 29 | } 30 | 31 | if e.Error() != "Error 0\nError 1\n" && 32 | e.Error() != "Error 1\nError 0\n" { 33 | t.Errorf("Got unexpected errors:%v", e) 34 | } 35 | } 36 | 37 | func TestNestedFor(t *testing.T) { 38 | m := make([]int, 4) 39 | For(0, 2, 1, func(i int) { 40 | For(0, 2, 1, func(j int) { 41 | m[i*2+j] = i*2 + j 42 | }) 43 | }) 44 | 45 | if m[0] != 0 || m[1] != 1 || m[2] != 2 || m[3] != 3 { 46 | t.Errorf("Failed setting arry m using For\n") 47 | } 48 | } 49 | 50 | func TestDo(t *testing.T) { 51 | m := make([]int, 4) 52 | 53 | e := Do( 54 | func() error { m[0] = 0; return fmt.Errorf("First error") }, 55 | func() error { m[1] = 1; return nil }, 56 | func() error { m[2] = 2; return fmt.Errorf("Second error") }, 57 | func() { m[3] = 3 }) 58 | 59 | if e == nil { 60 | t.Errorf("Failed capturing errors") 61 | } 62 | 63 | if e.Error() != "First error\nSecond error\n" { 64 | t.Errorf("Captured wrong errors") 65 | } 66 | 67 | if m[0] != 0 || m[1] != 1 || m[2] != 2 || m[3] != 3 { 68 | t.Errorf("Failed setting arry m using Do. m=%v\n", m) 69 | } 70 | } 71 | 72 | func ExampleRangeMap() error { 73 | wc := map[string]int{ 74 | "apple": 2, 75 | "orange": 1, 76 | } 77 | return RangeMap(wc, func(k, v reflect.Value) error { 78 | return fmt.Errorf("%s:%d", k.String(), v.Int()) 79 | }) 80 | } 81 | 82 | func TestRangeMap(t *testing.T) { 83 | e := ExampleRangeMap() 84 | if fmt.Sprint(e) != "apple:2\norange:1\n" && 85 | fmt.Sprint(e) != "orange:1\napple:2\n" { 86 | t.Errorf("Unexpected return %s", fmt.Sprint(e)) 87 | } 88 | } 89 | --------------------------------------------------------------------------------