├── .github ├── FUNDING.yml └── workflows │ └── test.yml ├── go.mod ├── .gitignore ├── README.md ├── workers_test.go ├── LICENSE ├── go.sum └── workers.go /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [bep] -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/bep/workers 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/frankban/quicktest v1.13.0 7 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c 8 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 9 | ) 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Tests](https://github.com/bep/workers/workflows/Test/badge.svg)](https://github.com/bep/workers/actions?query=workflow%3ATest) 2 | [![Go Report Card](https://goreportcard.com/badge/github.com/bep/workers)](https://goreportcard.com/report/github.com/bep/workers) 3 | [![GoDoc](https://godoc.org/github.com/bep/workers?status.svg)](https://godoc.org/github.com/bep/workers) 4 | 5 | 6 | A simple Go library to set up tasks to be executed in parallel. 7 | 8 | ```go 9 | package main 10 | 11 | import ( 12 | "context" 13 | "log" 14 | 15 | "github.com/bep/workers" 16 | ) 17 | 18 | func main() { 19 | // Max 4 tasks to be executed in parallel. 20 | w := workers.New(4) 21 | r, _ := w.Start(context.Background()) 22 | 23 | r.Run(func() error { 24 | return nil 25 | }) 26 | 27 | // ... run more tasks. 28 | 29 | if err := r.Wait(); err != nil { 30 | log.Fatal(err) 31 | } 32 | } 33 | ``` -------------------------------------------------------------------------------- /workers_test.go: -------------------------------------------------------------------------------- 1 | package workers 2 | 3 | import ( 4 | "context" 5 | "runtime" 6 | "sort" 7 | "sync" 8 | "testing" 9 | 10 | qt "github.com/frankban/quicktest" 11 | ) 12 | 13 | func TestPara(t *testing.T) { 14 | if runtime.NumCPU() < 2 { 15 | t.Skipf("skip para test, CPU count is %d", runtime.NumCPU()) 16 | } 17 | 18 | c := qt.New(t) 19 | 20 | n := 1024 21 | ints := make([]int, n) 22 | for i := 0; i < n; i++ { 23 | ints[i] = i 24 | } 25 | 26 | p := New(4) 27 | r, _ := p.Start(context.Background()) 28 | 29 | var result []int 30 | var mu sync.Mutex 31 | for i := 0; i < n; i++ { 32 | i := i 33 | r.Run(func() error { 34 | mu.Lock() 35 | defer mu.Unlock() 36 | result = append(result, i) 37 | return nil 38 | }) 39 | } 40 | 41 | c.Assert(r.Wait(), qt.IsNil) 42 | c.Assert(result, qt.HasLen, len(ints)) 43 | c.Assert(sort.IntsAreSorted(result), qt.Equals, false, qt.Commentf("Para does not seem to be parallel")) 44 | sort.Ints(result) 45 | c.Assert(result, qt.DeepEquals, ints) 46 | } 47 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Bjørn Erik Pedersen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: [ main ] 4 | pull_request: 5 | name: Test 6 | jobs: 7 | test: 8 | strategy: 9 | matrix: 10 | go-version: [1.18.x,1.19.x] 11 | platform: [ubuntu-latest, macos-latest, windows-latest] 12 | runs-on: ${{ matrix.platform }} 13 | steps: 14 | - name: Install Go 15 | uses: actions/setup-go@v3 16 | with: 17 | go-version: ${{ matrix.go-version }} 18 | - name: Install staticcheck 19 | run: go install honnef.co/go/tools/cmd/staticcheck@latest 20 | shell: bash 21 | - name: Install golint 22 | run: go install golang.org/x/lint/golint@latest 23 | shell: bash 24 | - name: Update PATH 25 | run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH 26 | shell: bash 27 | - name: Checkout code 28 | uses: actions/checkout@v1 29 | - name: Fmt 30 | if: matrix.platform != 'windows-latest' # :( 31 | run: "diff <(gofmt -d .) <(printf '')" 32 | shell: bash 33 | - name: Vet 34 | run: go vet ./... 35 | - name: Staticcheck 36 | run: staticcheck ./... 37 | - name: Lint 38 | run: golint ./... 39 | - name: Test 40 | run: go test -race ./... 41 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/frankban/quicktest v1.13.0 h1:yNZif1OkDfNoDfb9zZa9aXIpejNR4F23Wely0c+Qdqk= 2 | github.com/frankban/quicktest v1.13.0/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU= 3 | github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= 4 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 5 | github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= 6 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 7 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 8 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 9 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 10 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= 11 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 12 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 13 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 14 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 15 | -------------------------------------------------------------------------------- /workers.go: -------------------------------------------------------------------------------- 1 | // Package workers implements a parallel task executor. 2 | package workers 3 | 4 | import ( 5 | "context" 6 | 7 | "golang.org/x/sync/errgroup" 8 | ) 9 | 10 | // Workforce configures a task executor with the most number of tasks to be executed in parallel. 11 | type Workforce struct { 12 | sem chan struct{} 13 | } 14 | 15 | // Runner wraps the lifecycle methods of a new task set. 16 | // 17 | // Run wil block until a worker is available or the context is cancelled, 18 | // and then run the given func in a new goroutine. 19 | // Wait will wait for all the running goroutines to finish. 20 | type Runner interface { 21 | Run(func() error) 22 | Wait() error 23 | } 24 | 25 | type errGroupRunner struct { 26 | *errgroup.Group 27 | w *Workforce 28 | ctx context.Context 29 | } 30 | 31 | func (g *errGroupRunner) Run(fn func() error) { 32 | select { 33 | case g.w.sem <- struct{}{}: 34 | case <-g.ctx.Done(): 35 | return 36 | } 37 | 38 | g.Go(func() error { 39 | err := fn() 40 | <-g.w.sem 41 | return err 42 | }) 43 | } 44 | 45 | // New creates a new Workforce with the given number of workers. 46 | func New(numWorkers int) *Workforce { 47 | return &Workforce{ 48 | sem: make(chan struct{}, numWorkers), 49 | } 50 | } 51 | 52 | // Start starts a new Runner. 53 | func (w *Workforce) Start(ctx context.Context) (Runner, context.Context) { 54 | g, ctx := errgroup.WithContext(ctx) 55 | return &errGroupRunner{ 56 | Group: g, 57 | ctx: ctx, 58 | w: w, 59 | }, ctx 60 | } 61 | --------------------------------------------------------------------------------