├── .gitignore ├── LICENSE ├── README.md ├── actor.go ├── dispatcher.go ├── main.go └── worker.go /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.swap 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, the spawn developers. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # spawn 2 | 3 | *Process-based parallelism for the Go programming language* 4 | 5 | Package spawn provides multiprocessing funcionality for the 6 | Go programming language. It offers a small set of functions that 7 | allow spawning processes and distributing tasks among them. 8 | 9 | ## Quickstart 10 | 11 | The following example shows how to calculate the squares of the 12 | first 1000 natural numbers across 4 processes. 13 | 14 | ```go 15 | package main 16 | 17 | import ( 18 | "fmt" 19 | "github.com/slyrz/spawn" 20 | ) 21 | 22 | func main() { 23 | var err error 24 | 25 | // We are going to pass ints as tasks in this example, 26 | // so we have to register type int here. 27 | err = spawn.Register(new(int)) 28 | if err != nil { 29 | panic(err) 30 | } 31 | 32 | // The dispatch function generates tasks and submits them to 33 | // the worker processes. 34 | spawn.Dispatch(func() { 35 | for i := 1; i <= 1000; i++ { 36 | spawn.Task <- i 37 | } 38 | close(spawn.Task) 39 | }) 40 | 41 | // The work function runs in the spawned processes. It receives tasks, 42 | // performs a heavy computation (x^2) and sends the processed tasks 43 | // as results back to our main process. 44 | spawn.Work(func() { 45 | for task := range spawn.Task { 46 | val := task.(int) 47 | spawn.Result <- (val * val) 48 | } 49 | }) 50 | 51 | // Let's spawn 4 worker processes. 52 | err = spawn.Run(4) 53 | if err != nil { 54 | panic(err) 55 | } 56 | 57 | // Receive all results in a semi-random order. 58 | for res := range spawn.Result { 59 | fmt.Println(res) 60 | } 61 | } 62 | ``` 63 | 64 | ### License 65 | 66 | spawn is released under MIT license. 67 | You can find a copy of the MIT License in the [LICENSE](./LICENSE) file. 68 | 69 | -------------------------------------------------------------------------------- /actor.go: -------------------------------------------------------------------------------- 1 | package spawn 2 | 3 | import ( 4 | "os" 5 | "strconv" 6 | ) 7 | 8 | type pipe struct { 9 | readEnd *os.File 10 | writeEnd *os.File 11 | } 12 | 13 | // Create a new pipe using the os.Pipe function. 14 | func newPipe() *pipe { 15 | pr, pw, err := os.Pipe() 16 | if err != nil { 17 | panic(err) 18 | } 19 | return &pipe{pr, pw} 20 | } 21 | 22 | // Create a pipe from already opened file descriptors. 23 | func newPipeFromExtraFiles(i, j int) *pipe { 24 | return &pipe{ 25 | os.NewFile(uintptr(i), strconv.Itoa(i)), 26 | os.NewFile(uintptr(j), strconv.Itoa(j)), 27 | } 28 | } 29 | 30 | func (p *pipe) close() { 31 | p.readEnd.Close() 32 | p.writeEnd.Close() 33 | } 34 | 35 | type actor struct { 36 | input *pipe 37 | output *pipe 38 | } 39 | 40 | func newActor() *actor { 41 | return &actor{ 42 | newPipe(), 43 | newPipe(), 44 | } 45 | } 46 | 47 | func newActorFromExtraFiles(off int) *actor { 48 | return &actor{ 49 | newPipeFromExtraFiles(off, 1+off), 50 | newPipeFromExtraFiles(2+off, 3+off), 51 | } 52 | } 53 | 54 | func (a *actor) getFiles() []*os.File { 55 | return []*os.File{ 56 | a.input.readEnd, 57 | a.input.writeEnd, 58 | a.output.readEnd, 59 | a.output.writeEnd, 60 | } 61 | } 62 | 63 | func (a *actor) close() { 64 | a.input.close() 65 | a.output.close() 66 | } 67 | -------------------------------------------------------------------------------- /dispatcher.go: -------------------------------------------------------------------------------- 1 | package spawn 2 | 3 | import ( 4 | "encoding/gob" 5 | "fmt" 6 | "os" 7 | "os/exec" 8 | ) 9 | 10 | // The childProcess struct combines several objects used to handle a spawned 11 | // child process. 12 | type childProcess struct { 13 | *actor // input/output pipe of child process. 14 | *exec.Cmd // command to start/stop process. 15 | *gob.Encoder // encoder to write on the child's input pipe. 16 | *gob.Decoder // decoder to read from the child's output pipe. 17 | } 18 | 19 | // newChildProcess creates a new child process, but doesn't start it yet. 20 | func newChildProcess() *childProcess { 21 | act := newActor() 22 | cmd := exec.Command(os.Args[0]) 23 | 24 | // Show the output of our child processes on the dispatcher's terminal. 25 | cmd.Stdout = os.Stdout 26 | cmd.Stderr = os.Stderr 27 | 28 | // To make a long story short: Go's multithreaded nature forbids using 29 | // plain fork. So we have to use fork + exec instead. This starts 30 | // a completely new process. We use an envirnment variable to tell this 31 | // process that it is a child, not a dispatcher. To allow communication, 32 | // we pass the already created pipes as ExtraFiles to the new process. 33 | // The child process inherits these ExtraFiles as already open files. 34 | cmd.Env = os.Environ() 35 | cmd.Env = append(cmd.Env, "SPAWN_WORKER=yes") 36 | 37 | // Child receives the same command-line arguments as the dispatcher. 38 | cmd.Args = os.Args 39 | 40 | // Child inherites these file descriptors as already opened files. 41 | cmd.ExtraFiles = make([]*os.File, 0, 8) 42 | cmd.ExtraFiles = append(cmd.ExtraFiles, self.getFiles()...) 43 | cmd.ExtraFiles = append(cmd.ExtraFiles, act.getFiles()...) 44 | 45 | // Encoder is used to send tasks to child processes. 46 | // Decoder is used to receive finished tasks from child processes. 47 | enc := gob.NewEncoder(act.input.writeEnd) 48 | dec := gob.NewDecoder(act.output.readEnd) 49 | 50 | return &childProcess{act, cmd, enc, dec} 51 | } 52 | 53 | // Closes pipes and kill process. 54 | func (p *childProcess) kill() { 55 | p.actor.close() 56 | p.Process.Kill() 57 | } 58 | 59 | func runDispatcher(n int) { 60 | pool := make(map[int]*childProcess) 61 | for i := 0; i < n; i++ { 62 | proc := newChildProcess() 63 | proc.Start() 64 | pool[proc.Process.Pid] = proc 65 | } 66 | 67 | // tasksSent: number of tasks sent to our childs 68 | // tasksDone: number of finished tasks returned from our childs 69 | tasksSent := 0 70 | tasksDone := 0 71 | 72 | // Idle channel contains the pids of idle child processes. 73 | Idle := make(chan int, n) 74 | 75 | // Goroutine which reads pids from the process' input pipe. 76 | // Whenever a child process finishes a task, it writes its pid to our 77 | // input pipe and the processed task to its own output pipe. We read the 78 | // pids here, lookup the corresponding child process and read the processed 79 | // task from the child process' output pipe. 80 | go func() { 81 | var pid int 82 | 83 | dec := gob.NewDecoder(self.input.readEnd) 84 | for { 85 | err := dec.Decode(&pid) 86 | if err != nil { 87 | fmt.Println(err) 88 | break 89 | } 90 | 91 | // If our process pool doesn't contain the pid we just read, 92 | // something went terribly wrong. 93 | proc, ok := pool[pid] 94 | if !ok { 95 | panic("Received unkown Pid.") 96 | } 97 | 98 | proc.Decoder.DecodeValue(taskVal) 99 | Result <- taskVal.Interface() 100 | Idle <- pid 101 | tasksDone++ 102 | } 103 | close(Idle) 104 | }() 105 | 106 | // Goroutine dispatch() sends tasks over the Task channel. We distribute 107 | // them among our child processes. 108 | go dispatch() 109 | 110 | // Goroutine which receives tasks from dispatch() and distributes each 111 | // task to an idle child process. For every received task it waits till it 112 | // receives a pid on the Idle channel and sends the task to the 113 | // corresponding child process. After distributing all tasks, it waits 114 | // for all results and finally kills the child processes. 115 | go func() { 116 | for task := range Task { 117 | pid := <-Idle 118 | 119 | // The pids we receive from Idle should always exist in our 120 | // process pool 121 | pool[pid].Encode(task) 122 | tasksSent++ 123 | } 124 | 125 | // We distributed all tasks. Now we wait until we received all results. 126 | for tasksDone < tasksSent { 127 | <-Idle 128 | } 129 | 130 | // Workers don't terminate on their own. They just block on a read, so 131 | // we have to kill them now. 132 | for _, proc := range pool { 133 | proc.kill() 134 | } 135 | close(Result) 136 | }() 137 | 138 | // Ready, set, go! Start distributing tasks. 139 | for pid := range pool { 140 | Idle <- pid 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014, The Spawn Developers. 2 | // Released under MIT License. 3 | 4 | // Package spawn provides multiprocessing funcionality for the 5 | // Go programming language. It offers a small set of functions that 6 | // allow spawning processes and distributing tasks among them. 7 | 8 | package spawn 9 | 10 | import ( 11 | "errors" 12 | "os" 13 | "reflect" 14 | ) 15 | 16 | // Result channel is used to receive processed tasks. 17 | var Result chan interface{} 18 | 19 | // Task channel is used to distribute unprocessed tasks. 20 | var Task chan interface{} 21 | 22 | // self stores the input/output pipes of the current process. 23 | // parent stores the input/output pipes of the process' parent. It's only set 24 | // in child processes and contains nil in the dispatcher process. 25 | var ( 26 | self *actor // pipes of current process, always set. 27 | parent *actor // pipes of process' parent, only set in children. 28 | ) 29 | 30 | // The dispatch function is a user defined function that generates tasks and 31 | // writes them to the Task channel. 32 | // The work function is a user defined function that processes tasks and writes 33 | // them to the Result channel. 34 | var ( 35 | dispatch func() 36 | work func() 37 | ) 38 | 39 | // If we are the dispatcher process, ExpandEnv should return an empty string 40 | // because this environment variable should be set for child processes only. 41 | var isDispatcher = os.ExpandEnv("${SPAWN_WORKER}") == "" 42 | 43 | func init() { 44 | if isDispatcher { 45 | parent = nil 46 | self = newActor() 47 | } else { 48 | parent = newActorFromExtraFiles(3) 49 | self = newActorFromExtraFiles(7) 50 | } 51 | } 52 | 53 | var taskVal reflect.Value 54 | var taskPtr reflect.Value 55 | 56 | // Register records the type of tasks. It accepts a value or a pointer of 57 | // a value for the preferred task type as argument. Only values of that type 58 | // should be sent to the Task and Result channels. 59 | func Register(v interface{}) error { 60 | val := reflect.ValueOf(v) 61 | if val.Kind() == reflect.Ptr { 62 | if val.IsNil() { 63 | return errors.New("nil pointer") 64 | } 65 | taskPtr = val 66 | taskVal = val.Elem() 67 | } else { 68 | taskVal = val 69 | } 70 | return nil 71 | } 72 | 73 | // Dispatch registers the user defined dispatch function. The user defined 74 | // dispatch function gets executed in the parent process. It should send tasks 75 | // to the Task channel and close the Task channel upon exit. 76 | func Dispatch(f func()) { 77 | dispatch = f 78 | } 79 | 80 | // Work registers the user defined work function. The user defined work 81 | // function gets executed in the child processes. It should receive tasks from 82 | // the Tasks channel and send the processed results to the Result channel. 83 | func Work(f func()) { 84 | work = f 85 | } 86 | 87 | // Run creates n child processes and distributes the tasks created by the 88 | // dispatch function among them. It receives the processed tasks and sends 89 | // them to the Result channel. 90 | func Run(n int) error { 91 | if dispatch == nil { 92 | return errors.New("no dispatch function") 93 | } 94 | if work == nil { 95 | return errors.New("no work function") 96 | } 97 | if !taskVal.IsValid() { 98 | return errors.New("no task type registered") 99 | } 100 | 101 | Task = make(chan interface{}) 102 | Result = make(chan interface{}) 103 | 104 | if isDispatcher { 105 | runDispatcher(n) 106 | } else { 107 | runWorker() 108 | } 109 | return nil 110 | } 111 | -------------------------------------------------------------------------------- /worker.go: -------------------------------------------------------------------------------- 1 | package spawn 2 | 3 | import ( 4 | "encoding/gob" 5 | "os" 6 | ) 7 | 8 | func runWorker() { 9 | // Goroutine which reads tasks from the process' input pipe and sends 10 | // them to the Task channel. 11 | go func() { 12 | dec := gob.NewDecoder(self.input.readEnd) 13 | for { 14 | err := dec.DecodeValue(taskVal) 15 | if err != nil { 16 | panic(err) 17 | } 18 | Task <- taskVal.Interface() 19 | } 20 | }() 21 | 22 | // Goroutine work() processes tasks from Task channel and sends them 23 | // to the Result channel. 24 | go work() 25 | 26 | // encPid: encoder to write pids to input pipe of parent. 27 | // encRes: encoder to write finished tasks to our own output pipe. 28 | encPid := gob.NewEncoder(parent.input.writeEnd) 29 | encRes := gob.NewEncoder(self.output.writeEnd) 30 | 31 | pid := os.Getpid() 32 | 33 | for res := range Result { 34 | var err error 35 | 36 | // Tell our parent that we finished processing the task. 37 | err = encPid.Encode(pid) 38 | if err != nil { 39 | panic(err) 40 | } 41 | 42 | // Write process task to our own output pipe. The parent process is 43 | // going to read it from there. 44 | err = encRes.Encode(res) 45 | if err != nil { 46 | panic(err) 47 | } 48 | } 49 | } 50 | --------------------------------------------------------------------------------