├── .gitignore ├── example_test.go ├── LICENSE ├── README.md ├── runner.go └── runner_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | 13 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 14 | .glide/ 15 | -------------------------------------------------------------------------------- /example_test.go: -------------------------------------------------------------------------------- 1 | package dag_test 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | "github.com/natessilva/dag" 8 | ) 9 | 10 | func ExampleRunner() { 11 | var r dag.Runner 12 | 13 | r.AddVertex("one", func() error { 14 | fmt.Println("one and two will run in parallel before three") 15 | return nil 16 | }) 17 | r.AddVertex("two", func() error { 18 | fmt.Println("one and two will run in parallel before three") 19 | return nil 20 | }) 21 | r.AddVertex("three", func() error { 22 | fmt.Println("three will run before four") 23 | return errors.New("three is broken") 24 | }) 25 | r.AddVertex("four", func() error { 26 | fmt.Println("four will never run") 27 | return nil 28 | }) 29 | 30 | r.AddEdge("one", "three") 31 | r.AddEdge("two", "three") 32 | 33 | r.AddEdge("three", "four") 34 | 35 | fmt.Printf("the runner terminated with: %v\n", r.Run()) 36 | // Output: 37 | // one and two will run in parallel before three 38 | // one and two will run in parallel before three 39 | // three will run before four 40 | // the runner terminated with: three is broken 41 | } 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 natessilva 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dag 2 | 3 | [![GoDoc](https://godoc.org/github.com/natessilva/dag?status.svg)](https://godoc.org/github.com/natessilva/dag) 4 | 5 | dag.Runner is a mechanism to orchestrate goroutines and the order in which they run, using the semantics of a directed acyclic graph. 6 | 7 | Create a zero value dag.Runner, add vertices (functions) to it, add edges or dependencies between vertices, and finally invoke Run. Run will run all of the vertices in parallel topological order returning after either all vertices complete, or an error gets returned. 8 | 9 | ## Example 10 | 11 | ```go 12 | var r dag.Runner 13 | 14 | r.AddVertex("one", func() error { 15 | fmt.Println("one and two will run in parallel before three") 16 | return nil 17 | }) 18 | r.AddVertex("two", func() error { 19 | fmt.Println("one and two will run in parallel before three") 20 | return nil 21 | }) 22 | r.AddVertex("three", func() error { 23 | fmt.Println("three will run before four") 24 | return errors.New("three is broken") 25 | }) 26 | r.AddVertex("four", func() error { 27 | fmt.Println("four will never run") 28 | return nil 29 | }) 30 | 31 | r.AddEdge("one", "three") 32 | r.AddEdge("two", "three") 33 | 34 | r.AddEdge("three", "four") 35 | 36 | fmt.Printf("the runner terminated with: %v\n", r.Run()) 37 | ``` -------------------------------------------------------------------------------- /runner.go: -------------------------------------------------------------------------------- 1 | // Package dag implements a directed acyclic graph task runner with deterministic teardown. 2 | // it is similar to package errgroup, in that it runs multiple tasks in parallel and returns 3 | // the first error it encounters. Users define a Runner as a set vertices (functions) and edges 4 | // between them. During Run, the directed acyclec graph will be validated and each vertex 5 | // will run in parallel as soon as it's dependencies have been resolved. The Runner will only 6 | // return after all running goroutines have stopped. 7 | package dag 8 | 9 | import ( 10 | "errors" 11 | ) 12 | 13 | // Runner collects functions and arranges them as vertices and edges of a directed acyclic graph. 14 | // Upon validation of the graph, functions are run in parallel topological order. The zero value 15 | // is useful. 16 | type Runner struct { 17 | fns map[string]func() error 18 | graph map[string][]string 19 | } 20 | 21 | var errMissingVertex = errors.New("missing vertex") 22 | var errCycleDetected = errors.New("dependency cycle detected") 23 | 24 | // AddVertex adds a function as a vertex in the graph. Only functions which have been added in this 25 | // way will be executed during Run. 26 | func (d *Runner) AddVertex(name string, fn func() error) { 27 | if d.fns == nil { 28 | d.fns = make(map[string]func() error) 29 | } 30 | d.fns[name] = fn 31 | } 32 | 33 | // AddEdge establishes a dependency between two vertices in the graph. Both from and to must exist 34 | // in the graph, or Run will err. The vertex at from will execute before the vertex at to. 35 | func (d *Runner) AddEdge(from, to string) { 36 | if d.graph == nil { 37 | d.graph = make(map[string][]string) 38 | } 39 | d.graph[from] = append(d.graph[from], to) 40 | } 41 | 42 | type result struct { 43 | name string 44 | err error 45 | } 46 | 47 | func (d *Runner) detectCycles() bool { 48 | visited := make(map[string]bool) 49 | recStack := make(map[string]bool) 50 | 51 | for vertex := range d.graph { 52 | if !visited[vertex] { 53 | if d.detectCyclesHelper(vertex, visited, recStack) { 54 | return true 55 | } 56 | } 57 | } 58 | return false 59 | } 60 | 61 | func (d *Runner) detectCyclesHelper(vertex string, visited, recStack map[string]bool) bool { 62 | visited[vertex] = true 63 | recStack[vertex] = true 64 | 65 | for _, v := range d.graph[vertex] { 66 | // only check cycles on a vertex one time 67 | if !visited[v] { 68 | if d.detectCyclesHelper(v, visited, recStack) { 69 | return true 70 | } 71 | // if we've visited this vertex in this recursion stack, then we have a cycle 72 | } else if recStack[v] { 73 | return true 74 | } 75 | 76 | } 77 | recStack[vertex] = false 78 | return false 79 | } 80 | 81 | // Run will validate that all edges in the graph point to existing vertices, and that there are 82 | // no dependency cycles. After validation, each vertex will be run, deterministically, in parallel 83 | // topological order. If any vertex returns an error, no more vertices will be scheduled and 84 | // Run will exit and return that error once all in-flight functions finish execution. 85 | func (d *Runner) Run() error { 86 | // sanity check 87 | if len(d.fns) == 0 { 88 | return nil 89 | } 90 | // count how many deps each vertex has 91 | deps := make(map[string]int) 92 | for vertex, edges := range d.graph { 93 | // every vertex along every edge must have an associated fn 94 | if _, ok := d.fns[vertex]; !ok { 95 | return errMissingVertex 96 | } 97 | for _, vertex := range edges { 98 | if _, ok := d.fns[vertex]; !ok { 99 | return errMissingVertex 100 | } 101 | deps[vertex]++ 102 | } 103 | } 104 | 105 | if d.detectCycles() { 106 | return errCycleDetected 107 | } 108 | 109 | running := 0 110 | resc := make(chan result, len(d.fns)) 111 | var err error 112 | 113 | // start any vertex that has no deps 114 | for name := range d.fns { 115 | if deps[name] == 0 { 116 | running++ 117 | start(name, d.fns[name], resc) 118 | } 119 | } 120 | 121 | // wait for all running work to complete 122 | for running > 0 { 123 | res := <-resc 124 | running-- 125 | 126 | // capture the first error 127 | if res.err != nil && err == nil { 128 | err = res.err 129 | } 130 | 131 | // don't enqueue any more work on if there's been an error 132 | if err != nil { 133 | continue 134 | } 135 | 136 | // start any vertex whose deps are fully resolved 137 | for _, vertex := range d.graph[res.name] { 138 | if deps[vertex]--; deps[vertex] == 0 { 139 | running++ 140 | start(vertex, d.fns[vertex], resc) 141 | } 142 | } 143 | } 144 | return err 145 | } 146 | 147 | func start(name string, fn func() error, resc chan<- result) { 148 | go func() { 149 | resc <- result{ 150 | name: name, 151 | err: fn(), 152 | } 153 | }() 154 | } 155 | -------------------------------------------------------------------------------- /runner_test.go: -------------------------------------------------------------------------------- 1 | package dag 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestZero(t *testing.T) { 10 | var r Runner 11 | res := make(chan error) 12 | go func() { res <- r.Run() }() 13 | select { 14 | case err := <-res: 15 | if err != nil { 16 | t.Errorf("%v", err) 17 | } 18 | case <-time.After(100 * time.Millisecond): 19 | t.Error("timeout") 20 | } 21 | } 22 | 23 | func TestOne(t *testing.T) { 24 | myError := errors.New("error") 25 | var r Runner 26 | r.AddVertex("one", func() error { return myError }) 27 | res := make(chan error) 28 | go func() { res <- r.Run() }() 29 | select { 30 | case err := <-res: 31 | if want, have := myError, err; want != have { 32 | t.Errorf("want %v, have %v", want, have) 33 | } 34 | case <-time.After(100 * time.Millisecond): 35 | t.Error("timeout") 36 | } 37 | } 38 | 39 | func TestManyNoDeps(t *testing.T) { 40 | myError := errors.New("error") 41 | var r Runner 42 | r.AddVertex("one", func() error { return myError }) 43 | r.AddVertex("two", func() error { return nil }) 44 | r.AddVertex("three", func() error { return nil }) 45 | r.AddVertex("fout", func() error { return nil }) 46 | res := make(chan error) 47 | go func() { res <- r.Run() }() 48 | select { 49 | case err := <-res: 50 | if want, have := myError, err; want != have { 51 | t.Errorf("want %v, have %v", want, have) 52 | } 53 | case <-time.After(100 * time.Millisecond): 54 | t.Error("timeout") 55 | } 56 | } 57 | 58 | func TestManyWithCycle(t *testing.T) { 59 | var r Runner 60 | r.AddVertex("one", func() error { return nil }) 61 | r.AddVertex("two", func() error { return nil }) 62 | r.AddVertex("three", func() error { return nil }) 63 | r.AddVertex("four", func() error { return nil }) 64 | 65 | r.AddEdge("one", "two") 66 | r.AddEdge("two", "three") 67 | r.AddEdge("three", "four") 68 | r.AddEdge("three", "one") 69 | res := make(chan error) 70 | go func() { res <- r.Run() }() 71 | select { 72 | case err := <-res: 73 | if want, have := errCycleDetected, err; want != have { 74 | t.Errorf("want %v, have %v", want, have) 75 | } 76 | case <-time.After(100 * time.Millisecond): 77 | t.Error("timeout") 78 | } 79 | } 80 | 81 | func TestInvalidToVertex(t *testing.T) { 82 | var r Runner 83 | r.AddVertex("one", func() error { return nil }) 84 | r.AddVertex("two", func() error { return nil }) 85 | r.AddVertex("three", func() error { return nil }) 86 | r.AddVertex("four", func() error { return nil }) 87 | 88 | r.AddEdge("one", "two") 89 | r.AddEdge("two", "three") 90 | r.AddEdge("three", "four") 91 | r.AddEdge("three", "definitely-not-a-valid-vertex") 92 | res := make(chan error) 93 | go func() { res <- r.Run() }() 94 | select { 95 | case err := <-res: 96 | if want, have := errMissingVertex, err; want != have { 97 | t.Errorf("want %v, have %v", want, have) 98 | } 99 | case <-time.After(100 * time.Millisecond): 100 | t.Error("timeout") 101 | } 102 | } 103 | 104 | func TestInvalidFromVertex(t *testing.T) { 105 | var r Runner 106 | r.AddVertex("one", func() error { return nil }) 107 | r.AddVertex("two", func() error { return nil }) 108 | r.AddVertex("three", func() error { return nil }) 109 | r.AddVertex("four", func() error { return nil }) 110 | 111 | r.AddEdge("one", "two") 112 | r.AddEdge("two", "three") 113 | r.AddEdge("three", "four") 114 | r.AddEdge("definitely-not-a-valid-vertex", "three") 115 | res := make(chan error) 116 | go func() { res <- r.Run() }() 117 | select { 118 | case err := <-res: 119 | if want, have := errMissingVertex, err; want != have { 120 | t.Errorf("want %v, have %v", want, have) 121 | } 122 | case <-time.After(100 * time.Millisecond): 123 | t.Error("timeout") 124 | } 125 | } 126 | 127 | func TestManyWithDepsSuccess(t *testing.T) { 128 | resc := make(chan string, 7) 129 | 130 | var r Runner 131 | r.AddVertex("one", func() error { 132 | resc <- "one" 133 | return nil 134 | }) 135 | r.AddVertex("two", func() error { 136 | resc <- "two" 137 | return nil 138 | }) 139 | r.AddVertex("three", func() error { 140 | resc <- "three" 141 | return nil 142 | }) 143 | r.AddVertex("four", func() error { 144 | resc <- "four" 145 | return nil 146 | }) 147 | r.AddVertex("five", func() error { 148 | resc <- "five" 149 | return nil 150 | }) 151 | r.AddVertex("six", func() error { 152 | resc <- "six" 153 | return nil 154 | }) 155 | r.AddVertex("seven", func() error { 156 | resc <- "seven" 157 | return nil 158 | }) 159 | 160 | r.AddEdge("one", "two") 161 | r.AddEdge("one", "three") 162 | 163 | r.AddEdge("two", "four") 164 | r.AddEdge("two", "seven") 165 | 166 | r.AddEdge("five", "six") 167 | 168 | res := make(chan error) 169 | go func() { res <- r.Run() }() 170 | select { 171 | case err := <-res: 172 | if want, have := error(nil), err; want != have { 173 | t.Errorf("want %v, have %v", want, have) 174 | } 175 | case <-time.After(100 * time.Millisecond): 176 | t.Error("timeout") 177 | } 178 | 179 | results := make([]string, 7) 180 | timeoutc := time.After(100 * time.Millisecond) 181 | for i := range results { 182 | select { 183 | case results[i] = <-resc: 184 | case <-timeoutc: 185 | t.Error("timeout") 186 | } 187 | } 188 | 189 | checkOrder("one", "two", results, t) 190 | checkOrder("one", "three", results, t) 191 | 192 | checkOrder("two", "four", results, t) 193 | checkOrder("two", "seven", results, t) 194 | 195 | checkOrder("five", "six", results, t) 196 | } 197 | 198 | func checkOrder(from, to string, results []string, t *testing.T) { 199 | var fromIndex, toIndex int 200 | for i := range results { 201 | if results[i] == from { 202 | fromIndex = i 203 | } 204 | if results[i] == to { 205 | toIndex = i 206 | } 207 | } 208 | if fromIndex > toIndex { 209 | t.Errorf("from vertex: %s came after to vertex: %s", from, to) 210 | } 211 | } 212 | --------------------------------------------------------------------------------