├── .gitignore ├── go.mod ├── readme.md └── wpool ├── job.go ├── exec.go ├── job_test.go └── exec_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/godoylucase/workers-pool 2 | 3 | go 1.16 4 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # GO Concurrency - Worker Pool Pattern 2 | 3 | ![WorkerPools](https://user-images.githubusercontent.com/12415822/123521580-f3908680-d68d-11eb-8891-29ed2b316677.png) 4 | -------------------------------------------------------------------------------- /wpool/job.go: -------------------------------------------------------------------------------- 1 | package wpool 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | type JobID string 8 | type jobType string 9 | type jobMetadata map[string]interface{} 10 | 11 | type ExecutionFn func(ctx context.Context, args interface{}) (interface{}, error) 12 | 13 | type JobDescriptor struct { 14 | ID JobID 15 | JType jobType 16 | Metadata map[string]interface{} 17 | } 18 | 19 | type Result struct { 20 | Value interface{} 21 | Err error 22 | Descriptor JobDescriptor 23 | } 24 | 25 | type Job struct { 26 | Descriptor JobDescriptor 27 | ExecFn ExecutionFn 28 | Args interface{} 29 | } 30 | 31 | func (j Job) execute(ctx context.Context) Result { 32 | value, err := j.ExecFn(ctx, j.Args) 33 | if err != nil { 34 | return Result{ 35 | Err: err, 36 | Descriptor: j.Descriptor, 37 | } 38 | } 39 | 40 | return Result{ 41 | Value: value, 42 | Descriptor: j.Descriptor, 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /wpool/exec.go: -------------------------------------------------------------------------------- 1 | package wpool 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | ) 8 | 9 | func worker(ctx context.Context, wg *sync.WaitGroup, jobs <-chan Job, results chan<- Result) { 10 | defer wg.Done() 11 | for { 12 | select { 13 | case job, ok := <-jobs: 14 | if !ok { 15 | return 16 | } 17 | // fan-in job execution multiplexing results into the results channel 18 | results <- job.execute(ctx) 19 | case <-ctx.Done(): 20 | fmt.Printf("cancelled worker. Error detail: %v\n", ctx.Err()) 21 | results <- Result{ 22 | Err: ctx.Err(), 23 | } 24 | return 25 | } 26 | } 27 | } 28 | 29 | type WorkerPool struct { 30 | workersCount int 31 | jobs chan Job 32 | results chan Result 33 | Done chan struct{} 34 | } 35 | 36 | func New(wcount int) WorkerPool { 37 | return WorkerPool{ 38 | workersCount: wcount, 39 | jobs: make(chan Job, wcount), 40 | results: make(chan Result, wcount), 41 | Done: make(chan struct{}), 42 | } 43 | } 44 | 45 | func (wp WorkerPool) Run(ctx context.Context) { 46 | var wg sync.WaitGroup 47 | 48 | for i := 0; i < wp.workersCount; i++ { 49 | wg.Add(1) 50 | // fan out worker goroutines 51 | //reading from jobs channel and 52 | //pushing calcs into results channel 53 | go worker(ctx, &wg, wp.jobs, wp.results) 54 | } 55 | 56 | wg.Wait() 57 | close(wp.Done) 58 | close(wp.results) 59 | } 60 | 61 | func (wp WorkerPool) Results() <-chan Result { 62 | return wp.results 63 | } 64 | 65 | func (wp WorkerPool) GenerateFrom(jobsBulk []Job) { 66 | for i, _ := range jobsBulk { 67 | wp.jobs <- jobsBulk[i] 68 | } 69 | close(wp.jobs) 70 | } 71 | -------------------------------------------------------------------------------- /wpool/job_test.go: -------------------------------------------------------------------------------- 1 | package wpool 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "reflect" 7 | "testing" 8 | ) 9 | 10 | var ( 11 | errDefault = errors.New("wrong argument type") 12 | descriptor = JobDescriptor{ 13 | ID: JobID("1"), 14 | JType: jobType("anyType"), 15 | Metadata: jobMetadata{ 16 | "foo": "foo", 17 | "bar": "bar", 18 | }, 19 | } 20 | execFn = func(ctx context.Context, args interface{}) (interface{}, error) { 21 | argVal, ok := args.(int) 22 | if !ok { 23 | return nil, errDefault 24 | } 25 | 26 | return argVal * 2, nil 27 | } 28 | ) 29 | 30 | func Test_job_Execute(t *testing.T) { 31 | ctx := context.TODO() 32 | 33 | type fields struct { 34 | descriptor JobDescriptor 35 | execFn ExecutionFn 36 | args interface{} 37 | } 38 | tests := []struct { 39 | name string 40 | fields fields 41 | want Result 42 | }{ 43 | { 44 | name: "job execution success", 45 | fields: fields{ 46 | descriptor: descriptor, 47 | execFn: execFn, 48 | args: 10, 49 | }, 50 | want: Result{ 51 | Value: 20, 52 | Descriptor: descriptor, 53 | }, 54 | }, 55 | { 56 | name: "job execution failure", 57 | fields: fields{ 58 | descriptor: descriptor, 59 | execFn: execFn, 60 | args: "10", 61 | }, 62 | want: Result{ 63 | Err: errDefault, 64 | Descriptor: descriptor, 65 | }, 66 | }, 67 | } 68 | for _, tt := range tests { 69 | t.Run(tt.name, func(t *testing.T) { 70 | j := Job{ 71 | Descriptor: tt.fields.descriptor, 72 | ExecFn: tt.fields.execFn, 73 | Args: tt.fields.args, 74 | } 75 | 76 | got := j.execute(ctx) 77 | if tt.want.Err != nil { 78 | if !reflect.DeepEqual(got.Err, tt.want.Err) { 79 | t.Errorf("execute() = %v, wantError %v", got.Err, tt.want.Err) 80 | } 81 | return 82 | } 83 | 84 | if !reflect.DeepEqual(got, tt.want) { 85 | t.Errorf("execute() = %v, want %v", got, tt.want) 86 | } 87 | }) 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /wpool/exec_test.go: -------------------------------------------------------------------------------- 1 | package wpool 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strconv" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | const ( 12 | jobsCount = 10 13 | workerCount = 2 14 | ) 15 | 16 | func TestWorkerPool(t *testing.T) { 17 | wp := New(workerCount) 18 | 19 | ctx, cancel := context.WithCancel(context.TODO()) 20 | defer cancel() 21 | 22 | go wp.GenerateFrom(testJobs()) 23 | 24 | go wp.Run(ctx) 25 | 26 | for { 27 | select { 28 | case r, ok := <-wp.Results(): 29 | if !ok { 30 | continue 31 | } 32 | 33 | i, err := strconv.ParseInt(string(r.Descriptor.ID), 10, 64) 34 | if err != nil { 35 | t.Fatalf("unexpected error: %v", err) 36 | } 37 | 38 | val := r.Value.(int) 39 | if val != int(i)*2 { 40 | t.Fatalf("wrong value %v; expected %v", val, int(i)*2) 41 | } 42 | case <-wp.Done: 43 | return 44 | default: 45 | } 46 | } 47 | } 48 | 49 | func TestWorkerPool_TimeOut(t *testing.T) { 50 | wp := New(workerCount) 51 | 52 | ctx, cancel := context.WithTimeout(context.TODO(), time.Nanosecond*10) 53 | defer cancel() 54 | 55 | go wp.Run(ctx) 56 | 57 | for { 58 | select { 59 | case r := <-wp.Results(): 60 | if r.Err != nil && r.Err != context.DeadlineExceeded { 61 | t.Fatalf("expected error: %v; got: %v", context.DeadlineExceeded, r.Err) 62 | } 63 | case <-wp.Done: 64 | return 65 | default: 66 | } 67 | } 68 | } 69 | 70 | func TestWorkerPool_Cancel(t *testing.T) { 71 | wp := New(workerCount) 72 | 73 | ctx, cancel := context.WithCancel(context.TODO()) 74 | 75 | go wp.Run(ctx) 76 | cancel() 77 | 78 | for { 79 | select { 80 | case r := <-wp.Results(): 81 | if r.Err != nil && r.Err != context.Canceled { 82 | t.Fatalf("expected error: %v; got: %v", context.Canceled, r.Err) 83 | } 84 | case <-wp.Done: 85 | return 86 | default: 87 | } 88 | } 89 | } 90 | 91 | func testJobs() []Job { 92 | jobs := make([]Job, jobsCount) 93 | for i := 0; i < jobsCount; i++ { 94 | jobs[i] = Job{ 95 | Descriptor: JobDescriptor{ 96 | ID: JobID(fmt.Sprintf("%v", i)), 97 | JType: "anyType", 98 | Metadata: nil, 99 | }, 100 | ExecFn: execFn, 101 | Args: i, 102 | } 103 | } 104 | return jobs 105 | } 106 | --------------------------------------------------------------------------------