├── README.md ├── dag.go ├── dag1_test.go ├── dag_test.go └── go.mod /README.md: -------------------------------------------------------------------------------- 1 | # 分布式任务调度系统之任务编排及工作流实现原理与 golang 实践 2 | 3 | 定时调度系统（定时任务、定时执行）算是工作中经常依赖的中间件系统，简单使用操作系统的 crontab，或基于 Quartz，xxl-job 来搭建任务调度平台，行业有很多优秀的开源产品和中间件。了解其工作和设计原理，有助于我们完善或定制一套适合公司业务场景的任务调度中间件。 4 | 5 | 今天我们对调度任务的依赖关系及编排展开分析，实现一套工作流，来满足任务间的复杂依赖的场景。 6 | 7 | 本章内容提要： 8 | - 任务调度依赖 & 工作流 9 | - 图相关知识 10 | - golang 并发相关 11 | 12 | ### 阅读全文链接 13 | [任务编排及工作流实现](https://mp.weixin.qq.com/s?__biz=MzIyMzMxNjYwNw==&mid=2247483920&idx=1&sn=922d3d3e6ff07951fd45d629a960dca3&chksm=e8215d00df56d41679227c299134e7cea9cb9f93094f8597abfd3675a19116da628e02e28042&token=1776095493&lang=zh_CN#rd) 14 | 15 | 扫码关注微信订阅号支持： 16 | 17 | 技术岁月 18 | 19 | techyears 20 | 21 | ![技术岁月](https://raw.githubusercontent.com/skyhackvip/ratelimit/master/techyears.jpg) 22 | -------------------------------------------------------------------------------- /dag.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | //图结构 4 | type DAG struct { 5 | Vertexs []*Vertex 6 | } 7 | 8 | //顶点 9 | type Vertex struct { 10 | Key string 11 | Value interface{} 12 | Parents []*Vertex 13 | Children []*Vertex 14 | } 15 | 16 | //添加顶点 17 | func (dag *DAG) AddVertex(v *Vertex) { 18 | dag.Vertexs = append(dag.Vertexs, v) 19 | } 20 | 21 | //添加边 22 | func (dag *DAG) AddEdge(from, to *Vertex) { 23 | from.Children = append(from.Children, to) 24 | to.Parents = append(to.Parents, from) 25 | } 26 | 27 | //生成图，返回dag和其根顶点 28 | func NewDAG() (*DAG, *Vertex) { 29 | var dag = &DAG{} 30 | for k, v := range vertexs { 31 | va := &Vertex{Key: "a", Value: "1"} 32 | vb := &Vertex{Key: "b", Value: "2"} 33 | vc := &Vertex{Key: "c", Value: "3"} 34 | vd := &Vertex{Key: "d", Value: "4"} 35 | ve := &Vertex{Key: "e", Value: "5"} 36 | vf := &Vertex{Key: "f", Value: "6"} 37 | vg := &Vertex{Key: "g", Value: "7"} 38 | vh := &Vertex{Key: "h", Value: "8"} 39 | vi := &Vertex{Key: "i", Value: "9"} 40 | vx := &Vertex{Key: "x", Value: "10"} 41 | vy := &Vertex{Key: "y", Value: "11"} 42 | } 43 | dag.AddEdge(va, vb) 44 | dag.AddEdge(va, vc) 45 | dag.AddEdge(va, vd) 46 | dag.AddEdge(vb, ve) 47 | dag.AddEdge(vb, vh) 48 | dag.AddEdge(vb, vf) 49 | dag.AddEdge(vc, vf) 50 | dag.AddEdge(vc, vg) 51 | dag.AddEdge(vd, vg) 52 | dag.AddEdge(vh, vi) 53 | dag.AddEdge(ve, vi) 54 | dag.AddEdge(vf, vi) 55 | dag.AddEdge(vg, vi) 56 | dag.AddEdge(vx, vd) 57 | dag.AddEdge(vy, vi) 58 | return dag, va 59 | } 60 | -------------------------------------------------------------------------------- /dag1_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/eapache/queue" 6 | "sync" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestDagNew(t *testing.T) { 12 | _, root := NewDAG() 13 | all := BFSNew(root) 14 | 15 | for _, layer := range all { 16 | fmt.Println("------------------") 17 | doTasksNew(layer) 18 | } 19 | } 20 | 21 | //广度遍历 返回双层结构 22 | func BFSNew(root *Vertex) [][]*Vertex { 23 | 24 | q := queue.New() 25 | q.Add(root) 26 | visited := make(map[string]*Vertex) 27 | all := make([][]*Vertex, 0) 28 | 29 | for q.Length() > 0 { 30 | qSize := q.Length() 31 | tmp := make([]*Vertex, 0) 32 | for i := 0; i < qSize; i++ { 33 | //pop vertex 34 | currVert := q.Remove().(*Vertex) 35 | if _, ok := visited[currVert.Key]; ok { 36 | continue 37 | } 38 | visited[currVert.Key] = currVert 39 | tmp = append(tmp, currVert) 40 | 41 | //fmt.Println(level, currVert.Key, currVert.Value) 42 | 43 | for _, val := range currVert.Children { 44 | if _, ok := visited[val.Key]; !ok { 45 | q.Add(val) //add child 46 | } 47 | } 48 | } 49 | all = append([][]*Vertex{tmp}, all...) 50 | } 51 | return all 52 | } 53 | 54 | //并发执行 55 | func doTasksNew(vertexs []*Vertex) { 56 | var wg sync.WaitGroup 57 | for _, v := range vertexs { 58 | wg.Add(1) 59 | go func(v *Vertex) { 60 | defer wg.Done() 61 | time.Sleep(5 * time.Second) 62 | fmt.Printf("do %v, result is %v \n", v.Key, v.Value) 63 | }(v) //notice 64 | } 65 | wg.Wait() 66 | } 67 | -------------------------------------------------------------------------------- /dag_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/eapache/queue" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestDag(t *testing.T) { 11 | _, root := NewDAG() 12 | all := BFS(root) 13 | doTasks(all) 14 | } 15 | 16 | //广度遍历 17 | func BFS(root *Vertex) []*Vertex { 18 | 19 | q := queue.New() 20 | q.Add(root) 21 | visited := make(map[string]*Vertex) 22 | all := make([]*Vertex, 0) 23 | 24 | for q.Length() > 0 { 25 | qSize := q.Length() 26 | for i := 0; i < qSize; i++ { 27 | //pop vertex 28 | currVert := q.Remove().(*Vertex) 29 | if _, ok := visited[currVert.Key]; ok { 30 | continue 31 | } 32 | visited[currVert.Key] = currVert 33 | all = append([]*Vertex{currVert}, all...) 34 | for _, val := range currVert.Children { 35 | if _, ok := visited[val.Key]; !ok { 36 | q.Add(val) //add child 37 | } 38 | } 39 | } 40 | } 41 | return all 42 | } 43 | 44 | //并发执行 45 | func doTasks(vertexs []*Vertex) { 46 | for _, v := range vertexs { 47 | time.Sleep(5 * time.Second) 48 | fmt.Printf("do %v, result is %v \n", v.Key, v.Value) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/skyhackvip 2 | 3 | go 1.13 4 | 5 | require github.com/eapache/queue v1.1.0 6 | --------------------------------------------------------------------------------