├── .gitignore ├── README.md ├── cmd └── main.go ├── consumer.go ├── producer.go └── worker.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-workerpool 2 | Worker pool implementation. Target is `Make it be a high performance golang module, used in any way` 3 | ## Introduction 4 | Inspiration by below link, want to implement a common library to support mass throughput system arch. 5 | 6 | http://marcio.io/2015/07/handling-1-million-requests-per-minute-with-golang/ 7 | 8 | ## Prototype 9 | 10 | Producer -> Generate Data 11 | 12 | Consumer -> Processing Data 13 | 14 | Worker   -> Do the real job 15 | -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | workerPool "github.com/qianguozheng/go-workerpool" 7 | ) 8 | 9 | func main() { 10 | // generate worker to do job 11 | dispatcher := workerPool.NewDispatcher(3) 12 | dispatcher.Run() 13 | 14 | // produce job to be done 15 | producer := workerPool.NewProducer(40) 16 | producer.Run() 17 | 18 | time.Sleep(time.Second * 100) 19 | 20 | } 21 | -------------------------------------------------------------------------------- /consumer.go: -------------------------------------------------------------------------------- 1 | package workerPool 2 | 3 | import "fmt" 4 | 5 | type Dispatcher struct { 6 | // A pool of workers channels that are registered with the dispatcher 7 | WorkerPool chan chan Job 8 | Len int 9 | } 10 | 11 | func NewDispatcher(maxWorkers int) *Dispatcher{ 12 | pool := make(chan chan Job, maxWorkers) 13 | return &Dispatcher{WorkerPool:pool, Len:maxWorkers} 14 | } 15 | 16 | func (d *Dispatcher) Run() { 17 | // starting n number of workers 18 | fmt.Println("len of workerPool", len(d.WorkerPool)) 19 | for i:=0; i < d.Len; i++{ 20 | fmt.Println("Processor generate worker to do job ", i) 21 | worker := NewWorker(d.WorkerPool) 22 | fmt.Println("Generate NewWorker done ", i) 23 | worker.Start() 24 | fmt.Println("Worker started", i) 25 | } 26 | go d.dispatch() 27 | } 28 | 29 | func (d *Dispatcher) dispatch() { 30 | for{ 31 | select{ 32 | case job:= <-JobQueue: 33 | // a job request has been received 34 | fmt.Println("Store a job into jobChannel") 35 | go func(job Job){ 36 | //try to obtain a worker job channel that is available. 37 | //this will block until a worker is idle 38 | jobChannel := <- d.WorkerPool 39 | //dispatch the job to the worker job channel 40 | jobChannel <- job 41 | }(job) 42 | } 43 | } 44 | } 45 | 46 | -------------------------------------------------------------------------------- /producer.go: -------------------------------------------------------------------------------- 1 | package workerPool 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type Producer struct { 8 | job Job 9 | } 10 | 11 | func NewProducer(maxJob int) (*Producer) { 12 | job := Job{Payload:Payload(maxJob)} 13 | return &Producer{job: job} 14 | } 15 | 16 | func (p Producer) Run() { 17 | for i:=1; i < int(p.job.Payload); i++{ 18 | work:= Job{Payload: Payload(i)} 19 | fmt.Println("Producer: job " , i) 20 | JobQueue <- work 21 | //time.Sleep(time.Second*1) 22 | } 23 | } 24 | 25 | 26 | //func payloadHandler(){ 27 | // //Go through each payload and queue items individually to be posted to S3 28 | // //for _, payload := range 29 | // payload := Payload(4) 30 | // work := Job{Payload: payload} 31 | // 32 | // //Push the work onto the queue. 33 | // JobQueue <- work 34 | //} -------------------------------------------------------------------------------- /worker.go: -------------------------------------------------------------------------------- 1 | package workerPool 2 | 3 | 4 | import ( 5 | "fmt" 6 | ) 7 | 8 | var ( 9 | //MaxWorker = os.Getenv("MAX_WORKERS") 10 | //MaxQueue = os.Getenv("MAX_QUEUE") 11 | MaxWorker = 5 12 | MaxQueue = 1 13 | ) 14 | 15 | //For example usage 16 | type Payload int 17 | 18 | func (p Payload) Do() (err error){ 19 | //Do a job sample 20 | fmt.Println("I am working Do", int(p)) 21 | err = nil 22 | return 23 | } 24 | 25 | // Job represents the job to be run 26 | type Job struct { 27 | Payload Payload 28 | } 29 | 30 | // A buffered channel that we can send work requests on. 31 | 32 | var JobQueue chan Job 33 | func init(){ 34 | JobQueue = make(chan Job, 1) 35 | } 36 | // Worker represents the worker that executes the job 37 | 38 | type Worker struct{ 39 | WorkerPool chan chan Job 40 | JobChannel chan Job 41 | quit chan bool 42 | } 43 | 44 | func NewWorker(workerPool chan chan Job) Worker{ 45 | return Worker{ 46 | WorkerPool: workerPool, 47 | JobChannel: make(chan Job), 48 | quit: make(chan bool)} 49 | } 50 | 51 | // Start method starts the run loop for the worker, listening for a quit channel 52 | // in case we need to stop it 53 | func (w Worker) Start(){ 54 | 55 | go func(){ 56 | 57 | for{ 58 | //register the current worker into the worker queue. 59 | w.WorkerPool <- w.JobChannel 60 | 61 | select { 62 | case job:= <- w.JobChannel: 63 | // we have received a work request. 64 | if err := job.Payload.Do(); err != nil{ 65 | fmt.Printf("Error do payload function ：%s", err.Error()) 66 | } 67 | case <- w.quit: 68 | // we have received a signal to stop 69 | return 70 | } 71 | } 72 | }() 73 | } 74 | 75 | // Stop signals the worker to stop listening for work requests. 76 | func (w Worker) Stop() { 77 | go func() { 78 | w.quit <-true 79 | }() 80 | } --------------------------------------------------------------------------------