├── .github └── workflows │ └── go.yml ├── .gitignore ├── .golangci.yml ├── LICENSE ├── README.md ├── benchmark └── benchmark_test.go ├── examples ├── conditional │ └── condition.go ├── fibonacci │ └── fibonacci.go ├── loop │ └── loop.go ├── parallel_merge_sort │ └── parallel_merge_sort.go ├── priority │ └── priority.go ├── simple │ └── simple.go └── word_count │ └── word_count.go ├── executor.go ├── executor_test.go ├── flow.go ├── go.mod ├── go.sum ├── graph.go ├── image ├── condition.svg ├── desc.svg ├── fl.svg ├── loop.svg ├── simple.svg └── subflow.svg ├── node.go ├── profiler.go ├── profiler_test.go ├── task.go ├── taskflow.go ├── taskflow_test.go ├── utils ├── copool.go ├── copool_test.go ├── obj_pool.go ├── pprof.go ├── pprof_test.go ├── queue.go ├── queue_test.go ├── utils.go └── utils_test.go ├── visualizer.go ├── visualizer_dot.go └── visualizer_dot_test.go /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Generate code coverage badge 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | label: 8 | types: 9 | - created 10 | push: 11 | branches: 12 | - main 13 | jobs: 14 | test: 15 | runs-on: ubuntu-latest 16 | name: Update coverage badge 17 | timeout-minutes: 30 # 添加超时设置（单位：分钟） 18 | 19 | steps: 20 | - name: Checkout 21 | uses: actions/checkout@v4 22 | with: 23 | ref: ${{ github.event.pull_request.head.sha }} 24 | persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal access token. 25 | fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. 26 | 27 | - name: Setup go 28 | uses: actions/setup-go@v4 29 | with: 30 | go-version-file: 'go.mod' 31 | 32 | - name: Run Test 33 | run: | 34 | go test -count=100 -timeout=1800s -v ./... -covermode=count -coverprofile=coverage.out 35 | 36 | - name: Upload coverage to Codecov 37 | uses: codecov/codecov-action@v4 38 | with: 39 | token: ${{ secrets.CODECOV_TOKEN }} 40 | files: ./coverage.out # optional 41 | flags: unittests # optional 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # If you prefer the allow list template instead of the deny list, see community template: 2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 3 | # 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | bin/ 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | *.dot 17 | 18 | # Dependency directories (remove the comment below to include it) 19 | # vendor/ 20 | 21 | # Go workspace file 22 | go.work 23 | *.prof 24 | example 25 | .vscode/* -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | timeout: "10m" 3 | skip-files: 4 | - "*/_test\\.go$" # 使用双反斜杠转义点 5 | 6 | linters: 7 | disable-all: true 8 | enable: 9 | # basic 10 | - govet 11 | - staticcheck 12 | - errcheck 13 | - ineffassign 14 | - gosimple 15 | - unused 16 | # style 17 | - gofmt 18 | - goimports 19 | - misspell 20 | - stylecheck 21 | - dupl 22 | - wsl 23 | - goconst 24 | # complexity 25 | - funlen 26 | - gocyclo 27 | - lll 28 | # security 29 | - gosec 30 | 31 | linters-settings: 32 | funlen: 33 | # Checks the number of lines in a function. 34 | lines: 80 35 | # Checks the number of statements in a function. 36 | statements: 40 37 | # Ignore comments when counting lines. 38 | ignore-comments: true 39 | lines-in-file: 800 40 | gocyclo: 41 | # Minimal code complexity to report. 42 | # Default: 30 (but we recommend 10-20) 43 | min-complexity: 15 44 | lll: 45 | # Max line length, lines longer will be reported. 46 | # Default: 120. 47 | line-length: 120 48 | dupl: 49 | # Tokens count to trigger issue. 50 | # Default: 150 51 | threshold: 100 52 | 53 | output: 54 | format: colored-line-number 55 | print-issued-lines: true 56 | print-config: true -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-taskflow 2 | 3 | [![codecov](https://codecov.io/github/noneback/go-taskflow/graph/badge.svg?token=CITXYA10C6)](https://codecov.io/github/noneback/go-taskflow) 4 | [![Go Reference](https://pkg.go.dev/badge/github.com/noneback/go-taskflow.svg)](https://pkg.go.dev/github.com/noneback/go-taskflow) 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/noneback/go-taskflow)](https://goreportcard.com/report/github.com/noneback/go-taskflow) 6 | [![Mentioned in Awesome Go](https://awesome.re/mentioned-badge.svg)](https://github.com/avelino/awesome-go) 7 | [![DeepWiki][deepwiki-image]][deepwiki-url] 8 | 9 | [deepwiki-url]: https://deepwiki.com/noneback/go-taskflow 10 | [deepwiki-image]: https://img.shields.io/badge/Chat%20with-DeepWiki%20🤖-20B2AA 11 | 12 | ![go-taskflow](https://socialify.git.ci/noneback/go-taskflow/image?description=1&language=1&name=1&pattern=Solid&theme=Auto) 13 | 14 | go-taskflow is a general-purpose task-parallel programming framework for Go, inspired by [taskflow-cpp](https://github.com/taskflow/taskflow). It leverages Go's native capabilities and simplicity, making it ideal for managing complex dependencies in concurrent tasks. 15 | 16 | ## Features 17 | 18 | - **High Extensibility**: Easily extend the framework to adapt to various specific use cases. 19 | - **Native Go Concurrency Model**: Leverages Go's goroutines for efficient concurrent task execution. 20 | - **User-Friendly Programming Interface**: Simplifies complex task dependency management in Go. 21 | - **Static, Subflow, Conditional, and Cyclic Tasking**: Define static tasks, condition nodes, nested subflows, and cyclic flows to enhance modularity and programmability. 22 | 23 | | Static | Subflow | Condition | Cyclic | 24 | |:-----------|:------------:|------------:|------------:| 25 | | ![](image/simple.svg) | ![](image/subflow.svg) | ![](image/condition.svg) | ![](image/loop.svg) | 26 | 27 | - **Priority Task Scheduling**: Assign task priorities to ensure higher-priority tasks are executed first. 28 | - **Built-in Visualization and Profiling Tools**: Generate visual representations of tasks and profile task execution performance using integrated tools, simplifying debugging and optimization. 29 | 30 | ## Use Cases 31 | 32 | - **Data Pipelines**: Orchestrate data processing stages with complex dependencies. 33 | - **AI Agent Workflow Automation**: Define and execute AI agent workflows with clear sequences and dependency structures. 34 | - **Parallel Graph Tasking**: Execute graph-based tasks concurrently to maximize CPU utilization. 35 | 36 | ## Installation 37 | 38 | Import the latest version of go-taskflow using: 39 | 40 | ```bash 41 | go get -u github.com/noneback/go-taskflow 42 | ``` 43 | ## Documentation 44 | [DeepWiki Page](https://deepwiki.com/noneback/go-taskflow) 45 | ## Example 46 | 47 | Below is an example of using go-taskflow to implement a parallel merge sort: 48 | 49 | ```go 50 | package main 51 | 52 | import ( 53 | "fmt" 54 | "log" 55 | "math/rand" 56 | "os" 57 | "slices" 58 | "strconv" 59 | "sync" 60 | 61 | gtf "github.com/noneback/go-taskflow" 62 | ) 63 | 64 | // mergeInto merges a sorted source array into a sorted destination array. 65 | func mergeInto(dest, src []int) []int { 66 | size := len(dest) + len(src) 67 | tmp := make([]int, 0, size) 68 | i, j := 0, 0 69 | for i < len(dest) && j < len(src) { 70 | if dest[i] < src[j] { 71 | tmp = append(tmp, dest[i]) 72 | i++ 73 | } else { 74 | tmp = append(tmp, src[j]) 75 | j++ 76 | } 77 | } 78 | 79 | if i < len(dest) { 80 | tmp = append(tmp, dest[i:]...) 81 | } else { 82 | tmp = append(tmp, src[j:]...) 83 | } 84 | 85 | return tmp 86 | } 87 | 88 | func main() { 89 | size := 100 90 | randomArr := make([][]int, 10) 91 | sortedArr := make([]int, 0, 10*size) 92 | mutex := &sync.Mutex{} 93 | 94 | for i := 0; i < 10; i++ { 95 | for j := 0; j < size; j++ { 96 | randomArr[i] = append(randomArr[i], rand.Int()) 97 | } 98 | } 99 | 100 | sortTasks := make([]*gtf.Task, 10) 101 | tf := gtf.NewTaskFlow("merge sort") 102 | done := tf.NewTask("Done", func() { 103 | if !slices.IsSorted(sortedArr) { 104 | log.Fatal("Sorting failed") 105 | } 106 | fmt.Println("Sorted successfully") 107 | fmt.Println(sortedArr[:1000]) 108 | }) 109 | 110 | for i := 0; i < 10; i++ { 111 | sortTasks[i] = tf.NewTask("sort_"+strconv.Itoa(i), func() { 112 | arr := randomArr[i] 113 | slices.Sort(arr) 114 | mutex.Lock() 115 | defer mutex.Unlock() 116 | sortedArr = mergeInto(sortedArr, arr) 117 | }) 118 | } 119 | done.Succeed(sortTasks...) 120 | 121 | executor := gtf.NewExecutor(1000) 122 | 123 | executor.Run(tf).Wait() 124 | 125 | if err := tf.Dump(os.Stdout); err != nil { 126 | log.Fatal("Error dumping taskflow:", err) 127 | } 128 | 129 | if err := executor.Profile(os.Stdout); err != nil { 130 | log.Fatal("Error profiling taskflow:", err) 131 | } 132 | } 133 | ``` 134 | 135 | For more examples, visit the [examples directory](https://github.com/noneback/go-taskflow/tree/main/examples). 136 | 137 | ## Benchmark 138 | 139 | The following benchmark provides a rough estimate of performance. Note that most realistic workloads are I/O-bound, and their performance cannot be accurately reflected by these results. For CPU-intensive tasks, consider using [taskflow-cpp](https://github.com/taskflow/taskflow). 140 | 141 | ```plaintext 142 | $ go test -bench=. -benchmem 143 | goos: linux 144 | goarch: amd64 145 | pkg: github.com/noneback/go-taskflow/benchmark 146 | cpu: Intel(R) Xeon(R) Platinum 8269CY CPU @ 2.50GHz 147 | BenchmarkC32-4 23282 51891 ns/op 7295 B/op 227 allocs/op 148 | BenchmarkS32-4 7047 160199 ns/op 6907 B/op 255 allocs/op 149 | BenchmarkC6-4 66397 18289 ns/op 1296 B/op 47 allocs/op 150 | BenchmarkC8x8-4 7946 143474 ns/op 16914 B/op 504 allocs/op 151 | PASS 152 | ok github.com/noneback/go-taskflow/benchmark 5.606s 153 | ``` 154 | 155 | ## Understanding Conditional Tasks 156 | 157 | Conditional nodes in go-taskflow behave similarly to those in [taskflow-cpp](https://github.com/taskflow/taskflow). They participate in both conditional control and looping. To avoid common pitfalls, refer to the [Conditional Tasking documentation](https://taskflow.github.io/taskflow/ConditionalTasking.html). 158 | 159 | ## Error Handling in go-taskflow 160 | 161 | In Go, `errors` are values, and it is the user's responsibility to handle them appropriately. Only unrecovered `panic` events are managed by the framework. If a `panic` occurs, the entire parent graph is canceled, leaving the remaining tasks incomplete. This behavior may evolve in the future. If you have suggestions, feel free to share them. 162 | 163 | To prevent interruptions caused by `panic`, you can handle them manually when registering tasks: 164 | 165 | ```go 166 | tf.NewTask("not interrupt", func() { 167 | defer func() { 168 | if r := recover(); r != nil { 169 | // Handle the panic. 170 | } 171 | }() 172 | // User-defined logic. 173 | }) 174 | ``` 175 | 176 | ## Visualizing Taskflows 177 | 178 | To generate a visual representation of a taskflow, use the `Dump` method: 179 | 180 | ```go 181 | if err := tf.Dump(os.Stdout); err != nil { 182 | log.Fatal(err) 183 | } 184 | ``` 185 | 186 | The `Dump` method generates raw strings in DOT format. Use the `dot` tool to create a graph SVG. 187 | 188 | ![dot](image/desc.svg) 189 | 190 | ## Profiling Taskflows 191 | 192 | To profile a taskflow, use the `Profile` method: 193 | 194 | ```go 195 | if err := executor.Profile(os.Stdout); err != nil { 196 | log.Fatal(err) 197 | } 198 | ``` 199 | 200 | The `Profile` method generates raw strings in flamegraph format. Use the `flamegraph` tool to create a flamegraph SVG. 201 | 202 | ![flg](image/fl.svg) 203 | 204 | ## Stargazer 205 | 206 | [![Star History Chart](https://api.star-history.com/svg?repos=noneback/go-taskflow&type=Date)](https://star-history.com/#noneback/go-taskflow&Date) 207 | 208 | -------------------------------------------------------------------------------- /benchmark/benchmark_test.go: -------------------------------------------------------------------------------- 1 | package benchmark 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | gotaskflow "github.com/noneback/go-taskflow" 8 | ) 9 | 10 | var executor = gotaskflow.NewExecutor(6400) 11 | 12 | func BenchmarkC32(b *testing.B) { 13 | tf := gotaskflow.NewTaskFlow("G") 14 | for i := 0; i < 32; i++ { 15 | tf.NewTask(fmt.Sprintf("N%d", i), func() {}) 16 | } 17 | 18 | for i := 0; i < b.N; i++ { 19 | executor.Run(tf).Wait() 20 | } 21 | } 22 | 23 | func BenchmarkS32(b *testing.B) { 24 | tf := gotaskflow.NewTaskFlow("G") 25 | prev := tf.NewTask("N0", func() {}) 26 | for i := 1; i < 32; i++ { 27 | next := tf.NewTask(fmt.Sprintf("N%d", i), func() {}) 28 | prev.Precede(next) 29 | prev = next 30 | } 31 | 32 | for i := 0; i < b.N; i++ { 33 | executor.Run(tf).Wait() 34 | } 35 | } 36 | 37 | func BenchmarkC6(b *testing.B) { 38 | tf := gotaskflow.NewTaskFlow("G") 39 | n0 := tf.NewTask("N0", func() {}) 40 | n1 := tf.NewTask("N1", func() {}) 41 | n2 := tf.NewTask("N2", func() {}) 42 | n3 := tf.NewTask("N3", func() {}) 43 | n4 := tf.NewTask("N4", func() {}) 44 | n5 := tf.NewTask("N5", func() {}) 45 | 46 | n0.Precede(n1, n2) 47 | n1.Precede(n3) 48 | n2.Precede(n4) 49 | n5.Succeed(n3, n4) 50 | 51 | for i := 0; i < b.N; i++ { 52 | executor.Run(tf).Wait() 53 | } 54 | } 55 | 56 | func BenchmarkC8x8(b *testing.B) { 57 | tf := gotaskflow.NewTaskFlow("G") 58 | 59 | layersCount := 8 60 | layerNodesCount := 8 61 | 62 | var curLayer, upperLayer []*gotaskflow.Task 63 | 64 | for i := 0; i < layersCount; i++ { 65 | for j := 0; j < layerNodesCount; j++ { 66 | task := tf.NewTask(fmt.Sprintf("N%d", i*layersCount+j), func() {}) 67 | 68 | for i := range upperLayer { 69 | upperLayer[i].Precede(task) 70 | } 71 | 72 | curLayer = append(curLayer, task) 73 | } 74 | 75 | upperLayer = curLayer 76 | curLayer = []*gotaskflow.Task{} 77 | } 78 | 79 | for i := 0; i < b.N; i++ { 80 | executor.Run(tf).Wait() 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /examples/conditional/condition.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "runtime" 8 | "time" 9 | 10 | gotaskflow "github.com/noneback/go-taskflow" 11 | ) 12 | 13 | func main() { 14 | executor := gotaskflow.NewExecutor(uint(runtime.NumCPU()-1) * 10000) 15 | tf := gotaskflow.NewTaskFlow("G") 16 | A, B, C := 17 | tf.NewTask("A", func() { 18 | fmt.Println("A") 19 | }), 20 | tf.NewTask("B", func() { 21 | fmt.Println("B") 22 | }), 23 | tf.NewTask("C", func() { 24 | fmt.Println("C") 25 | }) 26 | 27 | A1, B1, _ := 28 | tf.NewTask("A1", func() { 29 | fmt.Println("A1") 30 | }), 31 | tf.NewTask("B1", func() { 32 | fmt.Println("B1") 33 | }), 34 | tf.NewTask("C1", func() { 35 | fmt.Println("C1") 36 | }) 37 | A.Precede(B) 38 | C.Precede(B) 39 | A1.Precede(B) 40 | C.Succeed(A1) 41 | C.Succeed(B1) 42 | 43 | subflow := tf.NewSubflow("sub1", func(sf *gotaskflow.Subflow) { 44 | A2, B2, C2 := 45 | sf.NewTask("A2", func() { 46 | fmt.Println("A2") 47 | }), 48 | sf.NewTask("B2", func() { 49 | fmt.Println("B2") 50 | }), 51 | sf.NewTask("C2", func() { 52 | fmt.Println("C2") 53 | }) 54 | A2.Precede(B2) 55 | C2.Precede(B2) 56 | }) 57 | 58 | subflow2 := tf.NewSubflow("sub2", func(sf *gotaskflow.Subflow) { 59 | A3, B3, C3 := 60 | sf.NewTask("A3", func() { 61 | fmt.Println("A3") 62 | }), 63 | sf.NewTask("B3", func() { 64 | fmt.Println("B3") 65 | }), 66 | sf.NewTask("C3", func() { 67 | fmt.Println("C3") 68 | // time.Sleep(10 * time.Second) 69 | }) 70 | A3.Precede(B3) 71 | C3.Precede(B3) 72 | }) 73 | 74 | cond := tf.NewCondition("binary", func() uint { 75 | return uint(time.Now().Second() % 2) 76 | }) 77 | B.Precede(cond) 78 | cond.Precede(subflow, subflow2) 79 | executor.Run(tf).Wait() 80 | fmt.Println("Print DOT") 81 | if err := tf.Dump(os.Stdout); err != nil { 82 | log.Fatal(err) 83 | } 84 | fmt.Println("Print Flamegraph") 85 | if err := executor.Profile(os.Stdout); err != nil { 86 | log.Fatal(err) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /examples/fibonacci/fibonacci.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "runtime" 8 | 9 | gotaskflow "github.com/noneback/go-taskflow" 10 | ) 11 | 12 | func main() { 13 | n := 10 14 | fib := make([]int, n+1) 15 | 16 | executor := gotaskflow.NewExecutor(uint(runtime.NumCPU())) 17 | tf := gotaskflow.NewTaskFlow("fibonacci") 18 | 19 | f0 := tf.NewTask("F0", func() { fib[0] = 1 }) 20 | f1 := tf.NewTask("F1", func() { fib[1] = 1 }) 21 | 22 | tasks := []*gotaskflow.Task{f0, f1} 23 | for k := 2; k <= n; k++ { 24 | k := k 25 | task := tf.NewTask(fmt.Sprintf("F%d", k), func() { 26 | fib[k] = fib[k-1] + fib[k-2] 27 | }) 28 | tasks[k-1].Precede(task) // F(n-1) -> F(n) 29 | tasks[k-2].Precede(task) // F(n-2) -> F(n) 30 | tasks = append(tasks, task) 31 | } 32 | if err := tf.Dump(os.Stdout); err != nil { 33 | log.Fatal(err) 34 | } 35 | executor.Run(tf).Wait() 36 | 37 | fmt.Printf("F(%d) = %d\n", n+1, fib[n]) 38 | } 39 | -------------------------------------------------------------------------------- /examples/loop/loop.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "runtime" 8 | "time" 9 | 10 | gotaskflow "github.com/noneback/go-taskflow" 11 | ) 12 | 13 | func main() { 14 | 15 | executor := gotaskflow.NewExecutor(uint(runtime.NumCPU()) * 100) 16 | i := 0 17 | tf := gotaskflow.NewTaskFlow("G") 18 | init, cond, body, back, done := 19 | tf.NewTask("init", func() { 20 | i = 0 21 | fmt.Println("i=0") 22 | }), 23 | tf.NewCondition("while i < 5", func() uint { 24 | time.Sleep(100 * time.Millisecond) 25 | if i < 5 { 26 | return 0 27 | } else { 28 | return 1 29 | } 30 | }), 31 | tf.NewTask("body", func() { 32 | i += 1 33 | fmt.Println("i++ =", i) 34 | }), 35 | tf.NewCondition("back", func() uint { 36 | fmt.Println("back") 37 | return 0 38 | }), 39 | tf.NewTask("done", func() { 40 | fmt.Println("done") 41 | }) 42 | 43 | init.Precede(cond) 44 | cond.Precede(body, done) 45 | body.Precede(back) 46 | back.Precede(cond) 47 | 48 | executor.Run(tf).Wait() 49 | if i < 5 { 50 | log.Fatal("i < 5") 51 | } 52 | 53 | if err := tf.Dump(os.Stdout); err != nil { 54 | log.Fatal(err) 55 | } 56 | executor.Profile(os.Stdout) 57 | } 58 | -------------------------------------------------------------------------------- /examples/parallel_merge_sort/parallel_merge_sort.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "math/rand" 7 | "os" 8 | "slices" 9 | "strconv" 10 | "sync" 11 | 12 | gtf "github.com/noneback/go-taskflow" 13 | "github.com/noneback/go-taskflow/utils" 14 | ) 15 | 16 | // merge sorted src to sorted dest 17 | func mergeInto(dest, src []int) []int { 18 | size := len(dest) + len(src) 19 | tmp := make([]int, 0, size) 20 | i, j := 0, 0 21 | for i < len(dest) && j < len(src) { 22 | if dest[i] < src[j] { 23 | tmp = append(tmp, dest[i]) 24 | i++ 25 | } else { 26 | tmp = append(tmp, src[j]) 27 | j++ 28 | } 29 | } 30 | 31 | if i < len(dest) { 32 | tmp = append(tmp, dest[i:]...) 33 | } else { 34 | tmp = append(tmp, src[j:]...) 35 | } 36 | 37 | return tmp 38 | } 39 | func main() { 40 | pprof := utils.NewPprofUtils(utils.CPU, "./out.prof") 41 | pprof.StartProfile() 42 | defer pprof.StopProfile() 43 | 44 | size := 10000 45 | share := 1000 46 | randomArr := make([][]int, share) 47 | sortedArr := make([]int, 0, share*size) 48 | mutex := &sync.Mutex{} 49 | 50 | for i := 0; i < share; i++ { 51 | for j := 0; j < size; j++ { 52 | randomArr[i] = append(randomArr[i], rand.Int()) 53 | } 54 | } 55 | 56 | sortTasks := make([]*gtf.Task, share) 57 | tf := gtf.NewTaskFlow("merge sort") 58 | done := tf.NewTask("Done", func() { 59 | if !slices.IsSorted(sortedArr) { 60 | log.Fatal("Failed") 61 | } 62 | fmt.Println("Sorted") 63 | fmt.Println(sortedArr[:1000]) 64 | }) 65 | 66 | for i := 0; i < share; i++ { 67 | idx := i 68 | sortTasks[idx] = tf.NewTask("sort_"+strconv.Itoa(idx), func() { 69 | arr := randomArr[idx] 70 | slices.Sort(arr) 71 | mutex.Lock() 72 | defer mutex.Unlock() 73 | sortedArr = mergeInto(sortedArr, arr) 74 | }) 75 | } 76 | done.Succeed(sortTasks...) 77 | 78 | executor := gtf.NewExecutor(1000000) 79 | 80 | executor.Run(tf).Wait() 81 | 82 | if err := tf.Dump(os.Stdout); err != nil { 83 | log.Fatal("V->", err) 84 | } 85 | 86 | if err := executor.Profile(os.Stdout); err != nil { 87 | log.Fatal("P->", err) 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /examples/priority/priority.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | gotaskflow "github.com/noneback/go-taskflow" 7 | "github.com/noneback/go-taskflow/utils" 8 | ) 9 | 10 | func main() { 11 | exector := gotaskflow.NewExecutor(uint(2)) 12 | q := utils.NewQueue[byte](true) 13 | tf := gotaskflow.NewTaskFlow("G") 14 | 15 | tf.NewTask("B", func() { 16 | fmt.Println("B") 17 | q.Put('B') 18 | }).Priority(gotaskflow.NORMAL) 19 | tf.NewTask("C", func() { 20 | fmt.Println("C") 21 | q.Put('C') 22 | }).Priority(gotaskflow.HIGH) 23 | tf.NewSubflow("sub1", func(sf *gotaskflow.Subflow) { 24 | sf.NewTask("A2", func() { 25 | fmt.Println("A2") 26 | q.Put('a') 27 | }).Priority(gotaskflow.LOW) 28 | sf.NewTask("B2", func() { 29 | fmt.Println("B2") 30 | q.Put('b') 31 | }).Priority(gotaskflow.HIGH) 32 | sf.NewTask("C2", func() { 33 | fmt.Println("C2") 34 | q.Put('c') 35 | }).Priority(gotaskflow.NORMAL) 36 | 37 | }).Priority(gotaskflow.LOW) 38 | 39 | exector.Run(tf).Wait() 40 | } 41 | -------------------------------------------------------------------------------- /examples/simple/simple.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "runtime" 8 | 9 | gotaskflow "github.com/noneback/go-taskflow" 10 | ) 11 | 12 | func main() { 13 | executor := gotaskflow.NewExecutor(uint(runtime.NumCPU()-1) * 10000) 14 | 15 | tf := gotaskflow.NewTaskFlow("G") 16 | A, B, C := 17 | tf.NewTask("A", func() { 18 | fmt.Println("A") 19 | }), 20 | tf.NewTask("B", func() { 21 | fmt.Println("B") 22 | }), 23 | tf.NewTask("C", func() { 24 | fmt.Println("C") 25 | }) 26 | 27 | A1, B1, _ := 28 | tf.NewTask("A1", func() { 29 | fmt.Println("A1") 30 | }), 31 | tf.NewTask("B1", func() { 32 | fmt.Println("B1") 33 | }), 34 | tf.NewTask("C1", func() { 35 | fmt.Println("C1") 36 | }) 37 | A.Precede(B) 38 | C.Precede(B) 39 | A1.Precede(B) 40 | C.Succeed(A1) 41 | C.Succeed(B1) 42 | 43 | subflow := tf.NewSubflow("sub1", func(sf *gotaskflow.Subflow) { 44 | A2, B2, C2 := 45 | sf.NewTask("A2", func() { 46 | fmt.Println("A2") 47 | }), 48 | sf.NewTask("B2", func() { 49 | fmt.Println("B2") 50 | }), 51 | sf.NewTask("C2", func() { 52 | fmt.Println("C2") 53 | }) 54 | A2.Precede(B2) 55 | C2.Precede(B2) 56 | }) 57 | 58 | subflow2 := tf.NewSubflow("sub2", func(sf *gotaskflow.Subflow) { 59 | A3, B3, C3 := 60 | sf.NewTask("A3", func() { 61 | fmt.Println("A3") 62 | }), 63 | sf.NewTask("B3", func() { 64 | fmt.Println("B3") 65 | }), 66 | sf.NewTask("C3", func() { 67 | fmt.Println("C3") 68 | // time.Sleep(10 * time.Second) 69 | }) 70 | A3.Precede(B3) 71 | C3.Precede(B3) 72 | }) 73 | 74 | subflow.Precede(B) 75 | subflow.Precede(subflow2) 76 | executor.Run(tf).Wait() 77 | fmt.Println("Print DOT") 78 | if err := tf.Dump(os.Stdout); err != nil { 79 | log.Fatal(err) 80 | } 81 | fmt.Println("Print Flamegraph") 82 | if err := executor.Profile(os.Stdout); err != nil { 83 | log.Fatal(err) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /examples/word_count/word_count.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "fmt" 7 | "log" 8 | "os" 9 | "path/filepath" 10 | "runtime" 11 | "sort" 12 | "strings" 13 | 14 | gotaskflow "github.com/noneback/go-taskflow" 15 | ) 16 | 17 | // 配置参数结构体 18 | type MRConfig struct { 19 | NumMappers int 20 | NumReducers int 21 | ChunkSize int 22 | TempDir string 23 | OutputPath string 24 | } 25 | 26 | type WordCount struct { 27 | Key string 28 | Value int 29 | } 30 | 31 | type MapReduce struct { 32 | cfg MRConfig 33 | executor gotaskflow.Executor 34 | input string 35 | mapOutputs [][]string 36 | } 37 | 38 | func NewMapReduce(input string, cfg MRConfig) *MapReduce { 39 | os.MkdirAll(cfg.TempDir, 0755) 40 | return &MapReduce{ 41 | cfg: cfg, 42 | executor: gotaskflow.NewExecutor(uint(runtime.NumCPU())), 43 | input: input, 44 | } 45 | } 46 | func (mr *MapReduce) Run() { 47 | tf := gotaskflow.NewTaskFlow("wordcount") 48 | mapper := make([][]int, mr.cfg.NumMappers) 49 | // spilt doc 50 | splitTask := tf.NewTask("split_input", func() { 51 | chunks := splitString(mr.input, mr.cfg.ChunkSize) 52 | for i, chunk := range chunks { 53 | path := filepath.Join(mr.cfg.TempDir, fmt.Sprintf("input_%d.txt", i)) 54 | if err := os.WriteFile(path, []byte(chunk), 0644); err != nil { 55 | log.Fatal(err) 56 | } 57 | mapper[i%mr.cfg.NumMappers] = append(mapper[i%mr.cfg.NumMappers], i) 58 | } 59 | }) 60 | 61 | mapTasks := make([]*gotaskflow.Task, mr.cfg.NumMappers) 62 | mr.mapOutputs = make([][]string, mr.cfg.NumMappers) 63 | 64 | for i := 0; i < mr.cfg.NumMappers; i++ { 65 | // scan input 66 | idx := i 67 | mapTasks[idx] = tf.NewTask(fmt.Sprintf("map_%d", idx), func() { 68 | for _, val := range mapper[idx] { 69 | mr.processMap(idx, val) 70 | } 71 | }) 72 | } 73 | splitTask.Precede(mapTasks...) 74 | 75 | reduceTasks := make([]*gotaskflow.Task, mr.cfg.NumReducers) 76 | for r := 0; r < mr.cfg.NumReducers; r++ { 77 | r := r 78 | reduceTasks[r] = tf.NewTask(fmt.Sprintf("reduce_%d", r), func() { 79 | log.Println("reduce:", r) 80 | mr.processReduce(r) 81 | }) 82 | } 83 | 84 | for _, mapTask := range mapTasks { 85 | mapTask.Precede(reduceTasks...) 86 | } 87 | 88 | mergeTask := tf.NewTask("merge_results", mr.mergeResults) 89 | for _, rt := range reduceTasks { 90 | rt.Precede(mergeTask) 91 | } 92 | 93 | if err := tf.Dump(os.Stdout); err != nil { 94 | log.Fatal(err) 95 | } 96 | mr.executor.Run(tf).Wait() 97 | } 98 | 99 | func (mr *MapReduce) processMap(mapID, inputID int) { 100 | inputPath := filepath.Join(mr.cfg.TempDir, fmt.Sprintf("input_%d.txt", inputID)) 101 | data, err := os.ReadFile(inputPath) 102 | if err != nil { 103 | log.Fatal(err) 104 | } 105 | fmt.Println(mapID, "process", inputPath) 106 | 107 | intermediate := make([]map[string]int, mr.cfg.NumReducers) 108 | for i := range intermediate { 109 | intermediate[i] = make(map[string]int) 110 | } 111 | 112 | scanner := bufio.NewScanner(strings.NewReader(string(data))) 113 | scanner.Split(bufio.ScanWords) 114 | 115 | for scanner.Scan() { 116 | word := scanner.Text() 117 | r := hash(word) % mr.cfg.NumReducers 118 | intermediate[r][word]++ 119 | } 120 | 121 | outputs := make([]string, mr.cfg.NumReducers) 122 | for r := 0; r < mr.cfg.NumReducers; r++ { 123 | fpath := filepath.Join(mr.cfg.TempDir, fmt.Sprintf("map-%d-reduce-%d.json", mapID, r)) 124 | file, err := os.OpenFile(fpath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 125 | if err != nil { 126 | log.Fatal(err) 127 | } 128 | defer file.Close() 129 | 130 | enc := json.NewEncoder(file) 131 | for word, count := range intermediate[r] { 132 | enc.Encode(WordCount{word, count}) 133 | } 134 | outputs[r] = fpath 135 | } 136 | mr.mapOutputs[mapID] = outputs 137 | } 138 | 139 | func (mr *MapReduce) processReduce(reduceID int) { 140 | var intermediate []WordCount 141 | 142 | for m := 0; m < mr.cfg.NumMappers; m++ { 143 | fpath := filepath.Join(mr.cfg.TempDir, fmt.Sprintf("map-%d-reduce-%d.json", m, reduceID)) 144 | file, err := os.Open(fpath) 145 | if err != nil { 146 | continue 147 | } 148 | defer file.Close() 149 | 150 | dec := json.NewDecoder(file) 151 | for dec.More() { 152 | var wc WordCount 153 | if err := dec.Decode(&wc); err != nil { 154 | log.Fatal(err) 155 | } 156 | intermediate = append(intermediate, wc) 157 | } 158 | } 159 | 160 | results := make(map[string]int) 161 | for _, item := range intermediate { 162 | results[item.Key] += item.Value 163 | } 164 | 165 | outputPath := filepath.Join(mr.cfg.TempDir, fmt.Sprintf("reduce-out-%d.txt", reduceID)) 166 | file, err := os.Create(outputPath) 167 | if err != nil { 168 | log.Fatal(err) 169 | } 170 | defer file.Close() 171 | 172 | for word, count := range results { 173 | fmt.Fprintf(file, "%s\t%d\n", word, count) 174 | } 175 | } 176 | 177 | func (mr *MapReduce) mergeResults() { 178 | var finalResults map[string]int = make(map[string]int) 179 | 180 | for r := 0; r < mr.cfg.NumReducers; r++ { 181 | path := filepath.Join(mr.cfg.TempDir, fmt.Sprintf("reduce-out-%d.txt", r)) 182 | file, err := os.Open(path) 183 | if err != nil { 184 | continue 185 | } 186 | defer file.Close() 187 | 188 | scanner := bufio.NewScanner(file) 189 | for scanner.Scan() { 190 | var word string 191 | var count int 192 | fmt.Sscanf(scanner.Text(), "%s\t%d", &word, &count) 193 | finalResults[word] += count 194 | } 195 | } 196 | 197 | keys := make([]string, 0, len(finalResults)) 198 | for k := range finalResults { 199 | keys = append(keys, k) 200 | } 201 | sort.Strings(keys) 202 | 203 | outputFile, err := os.Create(mr.cfg.OutputPath) 204 | if err != nil { 205 | log.Fatal(err) 206 | } 207 | defer outputFile.Close() 208 | 209 | for _, word := range keys { 210 | fmt.Fprintf(outputFile, "%s\t%d\n", word, finalResults[word]) 211 | } 212 | } 213 | 214 | func splitString(s string, chunkSize int) []string { 215 | var chunks []string 216 | for len(s) > 0 { 217 | if len(s) < chunkSize { 218 | chunks = append(chunks, s) 219 | break 220 | } 221 | chunk := s[:chunkSize] 222 | if lastSpace := strings.LastIndex(chunk, " "); lastSpace != -1 { 223 | chunk = chunk[:lastSpace] 224 | s = s[lastSpace+1:] 225 | } else { 226 | s = s[chunkSize:] 227 | } 228 | chunks = append(chunks, chunk) 229 | } 230 | return chunks 231 | } 232 | 233 | func hash(s string) int { 234 | h := 0 235 | for _, c := range s { 236 | h = 31*h + int(c) 237 | } 238 | if h < 0 { 239 | h = -h 240 | } 241 | return h 242 | } 243 | 244 | func main() { 245 | log.SetFlags(log.Llongfile) 246 | input := `Navigating This Book Now that you know who you’ll be hearing from, the next logical step would be to find out what you’ll be hearing about, which brings us to the second thing I wanted to mention. There are conceptually two major parts to this book, each with four chapters, and each followed up by a chapter that stands relatively independently on its own. 247 | The fun begins with Part I, The Beam Model (Chapters 1–4), which focuses on the high-level batch plus streaming data processing model originally developed for Google Cloud Dataflow, later donated to the Apache Software Foundation as Apache Beam, and also now seen in whole or in part across most other systems in the industry. It’s composed of four chapters: Chapter 1, Streaming 101, which covers the basics of stream processing, establishing some terminology, discussing the capabilities of streaming systems, distinguishing between two important domains of time (processing time and event time), and finally looking at some common data processing patterns. 248 | Chapter 2, The What, Where, When, and How of Data Processing, which covers in detail the core concepts of robust stream processing over out-of-order data, each analyzed within the context of a concrete running example and with animated diagrams to highlight the dimension of time. 249 | Chapter 3, Watermarks (written by Slava), which provides a deep survey of temporal progress metrics, how they are created, and how they propagate through pipelines. It ends by examining the details of two real-world watermark implementations. 250 | Chapter 4, Advanced Windowing, which picks up where Chapter 2 left off, diving into some advanced windowing and triggering concepts like processing-time windows, sessions, and continuation triggers. 251 | Between Parts I and II, providing an interlude as timely as the details contained therein are important, stands Chapter 5, Exactly-Once and Side Effects (written by Reuven). In it, he enumerates the challenges of providing end-to-end exactly-once (or effectively-once) processing semantics and walks through the implementation details of three different approaches to exactlyonce processing: Apache Flink, Apache Spark, and Google Cloud Dataflow. 252 | Next begins Part II, Streams and Tables (Chapters 6–9), which dives deeper into the conceptual and investigates the lower-level “streams and tables” way of thinking about stream processing, recently popularized by some upstanding citizens in the Apache Kafka community but, of course, invented decades ago by folks in the database community, because wasn’t everything? It too is composed of four chapters: Chapter 6, Streams and Tables, which introduces the basic idea of streams and tables, analyzes the classic MapReduce approach through a streams-and-tables lens, and then constructs a theory of streams and tables sufficiently general to encompass the full breadth of the Beam Model (and beyond). 253 | Chapter 7, The Practicalities of Persistent State, which considers the motivations for persistent state in streaming pipelines, looks at two common types of implicit state, and then analyzes a practical use case (advertising attribution) to inform the necessary characteristics of a general state management mechanism. 254 | Chapter 8, Streaming SQL, which investigates the meaning of streaming within the context of relational algebra and SQL, contrasts the inherent stream and table biases within the Beam Model and classic SQL as they exist today, and proposes a set of possible paths forward toward incorporating robust streaming semantics in SQL. 255 | Chapter 9, Streaming Joins, which surveys a variety of different join types, analyzes their behavior within the context of streaming, and finally looks in detail at a useful but ill-supported streaming join use case: temporal validity windows. 256 | Finally, closing out the book is Chapter 10, The Evolution of Large-Scale Data Processing, which strolls through a focused history of the MapReduce lineage of data processing systems, examining some of the important contributions that have evolved streaming systems into what they are today.` 257 | 258 | // 配置参数 259 | cfg := MRConfig{ 260 | NumMappers: 4, 261 | NumReducers: 2, 262 | ChunkSize: 300, 263 | TempDir: "./mr-tmp", 264 | OutputPath: "final-count.txt", 265 | } 266 | 267 | mr := NewMapReduce(input, cfg) 268 | mr.Run() 269 | 270 | fmt.Println("Final word count:") 271 | data, _ := os.ReadFile(cfg.OutputPath) 272 | fmt.Println(string(data)) 273 | } 274 | -------------------------------------------------------------------------------- /executor.go: -------------------------------------------------------------------------------- 1 | package gotaskflow 2 | 3 | import ( 4 | "cmp" 5 | "fmt" 6 | "io" 7 | "log" 8 | "runtime/debug" 9 | "slices" 10 | "sync" 11 | "time" 12 | 13 | "github.com/noneback/go-taskflow/utils" 14 | ) 15 | 16 | // Executor schedule and execute taskflow 17 | type Executor interface { 18 | Wait() // Wait block until all tasks finished 19 | Profile(w io.Writer) error // Profile write flame graph raw text into w 20 | Run(tf *TaskFlow) Executor // Run start to schedule and execute taskflow 21 | } 22 | 23 | type innerExecutorImpl struct { 24 | concurrency uint 25 | pool *utils.Copool 26 | wq *utils.Queue[*innerNode] 27 | wg *sync.WaitGroup 28 | profiler *profiler 29 | mu *sync.Mutex 30 | } 31 | 32 | // NewExecutor return a Executor with a specified max goroutine concurrency(recommend a value bigger than Runtime.NumCPU, **MUST** bigger than num(subflows). ) 33 | func NewExecutor(concurrency uint) Executor { 34 | if concurrency == 0 { 35 | panic("executor concurrency cannot be zero") 36 | } 37 | t := newProfiler() 38 | return &innerExecutorImpl{ 39 | concurrency: concurrency, 40 | pool: utils.NewCopool(concurrency), 41 | wq: utils.NewQueue[*innerNode](false), 42 | wg: &sync.WaitGroup{}, 43 | profiler: t, 44 | mu: &sync.Mutex{}, 45 | } 46 | } 47 | 48 | // Run start to schedule and execute taskflow 49 | func (e *innerExecutorImpl) Run(tf *TaskFlow) Executor { 50 | tf.frozen = true 51 | e.scheduleGraph(nil, tf.graph, nil) 52 | return e 53 | } 54 | 55 | func (e *innerExecutorImpl) invokeGraph(g *eGraph, parentSpan *span) bool { 56 | for { 57 | g.scheCond.L.Lock() 58 | e.mu.Lock() 59 | for !g.recyclable() && e.wq.Len() == 0 && !g.canceled.Load() { 60 | e.mu.Unlock() 61 | g.scheCond.Wait() 62 | e.mu.Lock() 63 | } 64 | 65 | g.scheCond.L.Unlock() 66 | 67 | // tasks can only be executed after sched, and joinCounter incr when sched, so here no need to lock up. 68 | if g.recyclable() || g.canceled.Load() { 69 | e.mu.Unlock() 70 | break 71 | } 72 | node := e.wq.Pop() 73 | e.mu.Unlock() 74 | 75 | e.invokeNode(node, parentSpan) 76 | } 77 | return !g.canceled.Load() 78 | } 79 | 80 | func (e *innerExecutorImpl) sche_successors(node *innerNode) { 81 | candidate := make([]*innerNode, 0, len(node.successors)) 82 | 83 | for _, n := range node.successors { 84 | n.mu.Lock() 85 | if (n.recyclable() && n.state.Load() == kNodeStateIdle) || n.Typ == nodeCondition { 86 | // deps all done or condition node or task has been sched. 87 | n.state.Store(kNodeStateWaiting) 88 | candidate = append(candidate, n) 89 | } 90 | n.mu.Unlock() 91 | } 92 | 93 | slices.SortFunc(candidate, func(i, j *innerNode) int { 94 | return cmp.Compare(i.priority, j.priority) 95 | }) 96 | node.setup() // make node repeatable 97 | e.schedule(candidate...) 98 | } 99 | 100 | func (e *innerExecutorImpl) invokeStatic(node *innerNode, parentSpan *span, p *Static) func() { 101 | return func() { 102 | span := span{extra: attr{ 103 | typ: nodeStatic, 104 | name: node.name, 105 | }, begin: time.Now(), parent: parentSpan} 106 | 107 | defer func() { 108 | span.cost = time.Since(span.begin) 109 | if r := recover(); r != nil { 110 | node.g.canceled.Store(true) 111 | log.Printf("graph %v is canceled, since static node %v panics", node.g.name, node.name) 112 | log.Printf("[recovered] static node %s, panic: %v, stack: %s", node.name, r, debug.Stack()) 113 | } else { 114 | e.profiler.AddSpan(&span) // remove canceled node span 115 | } 116 | 117 | node.drop() 118 | e.sche_successors(node) 119 | node.g.deref() 120 | e.wg.Done() 121 | }() 122 | if !node.g.canceled.Load() { 123 | node.state.Store(kNodeStateRunning) 124 | p.handle() 125 | node.state.Store(kNodeStateFinished) 126 | } 127 | } 128 | } 129 | 130 | func (e *innerExecutorImpl) invokeSubflow(node *innerNode, parentSpan *span, p *Subflow) func() { 131 | return func() { 132 | span := span{extra: attr{ 133 | typ: nodeSubflow, 134 | name: node.name, 135 | }, begin: time.Now(), parent: parentSpan} 136 | 137 | defer func() { 138 | span.cost = time.Since(span.begin) 139 | if r := recover(); r != nil { 140 | log.Printf("graph %v is canceled, since subflow %v panics", node.g.name, node.name) 141 | log.Printf("[recovered] subflow %s, panic: %v, stack: %s", node.name, r, debug.Stack()) 142 | node.g.canceled.Store(true) 143 | p.g.canceled.Store(true) 144 | } else { 145 | e.profiler.AddSpan(&span) // remove canceled node span 146 | } 147 | 148 | e.scheduleGraph(node.g, p.g, &span) 149 | node.drop() 150 | e.sche_successors(node) 151 | node.g.deref() 152 | e.wg.Done() 153 | }() 154 | 155 | if !node.g.canceled.Load() { 156 | node.state.Store(kNodeStateRunning) 157 | if !p.g.instantiated { 158 | p.handle(p) 159 | } 160 | p.g.instantiated = true 161 | node.state.Store(kNodeStateFinished) 162 | } 163 | } 164 | } 165 | 166 | func (e *innerExecutorImpl) invokeCondition(node *innerNode, parentSpan *span, p *Condition) func() { 167 | return func() { 168 | span := span{extra: attr{ 169 | typ: nodeCondition, 170 | name: node.name, 171 | }, begin: time.Now(), parent: parentSpan} 172 | 173 | defer func() { 174 | span.cost = time.Since(span.begin) 175 | if r := recover(); r != nil { 176 | node.g.canceled.Store(true) 177 | log.Printf("graph %v is canceled, since condition node %v panics", node.g.name, node.name) 178 | log.Printf("[recovered] condition node %s, panic: %v, stack: %s", node.name, r, debug.Stack()) 179 | } else { 180 | e.profiler.AddSpan(&span) // remove canceled node span 181 | } 182 | node.drop() 183 | // e.sche_successors(node) 184 | node.g.deref() 185 | node.setup() 186 | e.wg.Done() 187 | }() 188 | 189 | if !node.g.canceled.Load() { 190 | node.state.Store(kNodeStateRunning) 191 | 192 | choice := p.handle() 193 | if choice > uint(len(p.mapper)) { 194 | panic(fmt.Sprintln("condition task failed, successors of condition should be more than precondition choice", p.handle())) 195 | } 196 | // do choice and cancel others 197 | node.state.Store(kNodeStateFinished) 198 | e.schedule(p.mapper[choice]) 199 | } 200 | } 201 | } 202 | 203 | func (e *innerExecutorImpl) invokeNode(node *innerNode, parentSpan *span) { 204 | switch p := node.ptr.(type) { 205 | case *Static: 206 | e.pool.Go(e.invokeStatic(node, parentSpan, p)) 207 | case *Subflow: 208 | e.pool.Go(e.invokeSubflow(node, parentSpan, p)) 209 | case *Condition: 210 | e.pool.Go(e.invokeCondition(node, parentSpan, p)) 211 | default: 212 | panic("unsupported node") 213 | } 214 | } 215 | 216 | func (e *innerExecutorImpl) pushIntoQueue(node *innerNode) { 217 | e.mu.Lock() 218 | defer e.mu.Unlock() 219 | e.wq.Put(node) 220 | } 221 | 222 | func (e *innerExecutorImpl) schedule(nodes ...*innerNode) { 223 | for _, node := range nodes { 224 | if node.g.canceled.Load() { 225 | // no need 226 | node.g.scheCond.L.Lock() 227 | node.g.scheCond.Signal() 228 | node.g.scheCond.L.Unlock() 229 | log.Printf("node %v is not scheduled, since graph %v is canceled\n", node.name, node.g.name) 230 | return 231 | } 232 | e.wg.Add(1) 233 | node.g.scheCond.L.Lock() 234 | 235 | node.g.ref() 236 | e.pushIntoQueue(node) 237 | 238 | node.g.scheCond.Signal() 239 | node.g.scheCond.L.Unlock() 240 | } 241 | } 242 | 243 | func (e *innerExecutorImpl) scheduleGraph(parentg, g *eGraph, parentSpan *span) { 244 | g.setup() 245 | slices.SortFunc(g.entries, func(i, j *innerNode) int { 246 | return cmp.Compare(i.priority, j.priority) 247 | }) 248 | e.schedule(g.entries...) 249 | if !e.invokeGraph(g, parentSpan) && parentg != nil { 250 | parentg.canceled.Store(true) 251 | log.Printf("graph %s canceled, since subgraph %s is canceled\n", parentg.name, g.name) 252 | } 253 | 254 | g.scheCond.Signal() 255 | } 256 | 257 | // Wait: block until all tasks finished 258 | func (e *innerExecutorImpl) Wait() { 259 | e.wg.Wait() 260 | } 261 | 262 | // Profile write flame graph raw text into w 263 | func (e *innerExecutorImpl) Profile(w io.Writer) error { 264 | return e.profiler.draw(w) 265 | } 266 | -------------------------------------------------------------------------------- /executor_test.go: -------------------------------------------------------------------------------- 1 | package gotaskflow_test 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "runtime" 7 | "testing" 8 | "time" 9 | 10 | gotaskflow "github.com/noneback/go-taskflow" 11 | ) 12 | 13 | func TestExecutor(t *testing.T) { 14 | executor := gotaskflow.NewExecutor(uint(runtime.NumCPU())) 15 | tf := gotaskflow.NewTaskFlow("G") 16 | A, B, C := 17 | tf.NewTask("A", func() { 18 | fmt.Println("A") 19 | }), 20 | tf.NewTask("B", func() { 21 | fmt.Println("B") 22 | }), 23 | tf.NewTask("C", func() { 24 | fmt.Println("C") 25 | }) 26 | 27 | A1, B1, _ := 28 | tf.NewTask("A1", func() { 29 | fmt.Println("A1") 30 | }), 31 | tf.NewTask("B1", func() { 32 | fmt.Println("B1") 33 | }), 34 | tf.NewTask("C1", func() { 35 | fmt.Println("C1") 36 | }) 37 | A.Precede(B) 38 | C.Precede(B) 39 | A1.Precede(B) 40 | C.Succeed(A1) 41 | C.Succeed(B1) 42 | 43 | executor.Run(tf).Wait() 44 | executor.Profile(os.Stdout) 45 | } 46 | 47 | func TestPanicInSubflow(t *testing.T) { 48 | executor := gotaskflow.NewExecutor(100000) 49 | tf := gotaskflow.NewTaskFlow("G") 50 | copy_gcc_source_file := tf.NewTask("copy_gcc_source_file", func() { 51 | time.Sleep(1 * time.Second) 52 | fmt.Println("copy_gcc_source_file") 53 | }) 54 | tar_gcc_source_file := tf.NewTask("tar_gcc_source_file", func() { 55 | time.Sleep(1 * time.Second) 56 | fmt.Println("tar_gcc_source_file") 57 | }) 58 | download_prerequisites := tf.NewSubflow("download_prerequisites", func(sf *gotaskflow.Subflow) { 59 | sf.NewTask("", func() { 60 | time.Sleep(1 * time.Second) 61 | fmt.Println("download_prerequisites") 62 | panic(1) 63 | }) 64 | }) 65 | yum_install_dependency_package := tf.NewTask("yum_install_dependency_package", func() { 66 | time.Sleep(1 * time.Second) 67 | fmt.Println("yum_install_dependency_package") 68 | }) 69 | mkdir_and_prepare_build := tf.NewTask("mkdir_and_prepare_build", func() { 70 | time.Sleep(1 * time.Second) 71 | fmt.Println("mkdir_and_prepare_build") 72 | }) 73 | make_build := tf.NewTask("make_build", func() { 74 | time.Sleep(1 * time.Second) 75 | fmt.Println("make_build") 76 | }) 77 | make_install := tf.NewTask("make_install", func() { 78 | time.Sleep(1 * time.Second) 79 | fmt.Println("make_install") 80 | }) 81 | relink := tf.NewTask("relink", func() { 82 | time.Sleep(1 * time.Second) 83 | fmt.Println("relink") 84 | }) 85 | copy_gcc_source_file.Precede(tar_gcc_source_file) 86 | yum_install_dependency_package.Precede(download_prerequisites) 87 | tar_gcc_source_file.Precede(download_prerequisites) 88 | download_prerequisites.Precede(mkdir_and_prepare_build) 89 | mkdir_and_prepare_build.Precede(make_build) 90 | make_build.Precede(make_install) 91 | make_install.Precede(relink) 92 | executor.Run(tf).Wait() 93 | } 94 | -------------------------------------------------------------------------------- /flow.go: -------------------------------------------------------------------------------- 1 | package gotaskflow 2 | 3 | var builder = flowBuilder{} 4 | 5 | type flowBuilder struct{} 6 | 7 | // Condition Wrapper 8 | type Condition struct { 9 | handle func() uint 10 | mapper map[uint]*innerNode 11 | } 12 | 13 | // Static Wrapper 14 | type Static struct { 15 | handle func() 16 | } 17 | 18 | // Subflow Wrapper 19 | type Subflow struct { 20 | handle func(sf *Subflow) 21 | g *eGraph 22 | } 23 | 24 | // Push pushs all tasks into subflow 25 | func (sf *Subflow) push(tasks ...*Task) { 26 | for _, task := range tasks { 27 | sf.g.push(task.node) 28 | } 29 | } 30 | 31 | func (tf *flowBuilder) NewStatic(name string, f func()) *innerNode { 32 | node := newNode(name) 33 | node.ptr = &Static{ 34 | handle: f, 35 | } 36 | node.Typ = nodeStatic 37 | return node 38 | } 39 | 40 | func (fb *flowBuilder) NewSubflow(name string, f func(sf *Subflow)) *innerNode { 41 | node := newNode(name) 42 | node.ptr = &Subflow{ 43 | handle: f, 44 | g: newGraph(name), 45 | } 46 | node.Typ = nodeSubflow 47 | return node 48 | } 49 | 50 | func (fb *flowBuilder) NewCondition(name string, f func() uint) *innerNode { 51 | node := newNode(name) 52 | node.ptr = &Condition{ 53 | handle: f, 54 | mapper: make(map[uint]*innerNode), 55 | } 56 | node.Typ = nodeCondition 57 | return node 58 | } 59 | 60 | // NewStaticTask returns a static task 61 | func (sf *Subflow) NewTask(name string, f func()) *Task { 62 | task := &Task{ 63 | node: builder.NewStatic(name, f), 64 | } 65 | sf.push(task) 66 | return task 67 | } 68 | 69 | // NewSubflow returns a subflow task 70 | func (sf *Subflow) NewSubflow(name string, f func(sf *Subflow)) *Task { 71 | task := &Task{ 72 | node: builder.NewSubflow(name, f), 73 | } 74 | sf.push(task) 75 | return task 76 | } 77 | 78 | // NewCondition returns a condition task. The predict func return value determines its successor. 79 | func (sf *Subflow) NewCondition(name string, predict func() uint) *Task { 80 | task := &Task{ 81 | node: builder.NewCondition(name, predict), 82 | } 83 | sf.push(task) 84 | return task 85 | } 86 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/noneback/go-taskflow 2 | 3 | go 1.21.6 4 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/noneback/go-taskflow/51d5d70e0308a3ece6025de3ee6860a41a4d1866/go.sum -------------------------------------------------------------------------------- /graph.go: -------------------------------------------------------------------------------- 1 | package gotaskflow 2 | 3 | import ( 4 | "sync" 5 | "sync/atomic" 6 | ) 7 | 8 | type eGraph struct { // execution graph 9 | name string 10 | nodes []*innerNode 11 | joinCounter atomic.Int32 12 | entries []*innerNode 13 | scheCond *sync.Cond 14 | instantiated bool 15 | canceled atomic.Bool // only changes when task in graph panic 16 | } 17 | 18 | func newGraph(name string) *eGraph { 19 | return &eGraph{ 20 | name: name, 21 | nodes: make([]*innerNode, 0), 22 | scheCond: sync.NewCond(&sync.Mutex{}), 23 | joinCounter: atomic.Int32{}, 24 | } 25 | } 26 | 27 | func (g *eGraph) ref() { 28 | g.joinCounter.Add(1) 29 | } 30 | 31 | func (g *eGraph) deref() { 32 | g.scheCond.L.Lock() 33 | defer g.scheCond.L.Unlock() 34 | defer g.scheCond.Signal() 35 | 36 | g.joinCounter.Add(-1) 37 | } 38 | 39 | func (g *eGraph) reset() { 40 | g.joinCounter.Store(0) 41 | g.entries = g.entries[:0] 42 | for _, n := range g.nodes { 43 | n.joinCounter.Store(0) 44 | } 45 | } 46 | 47 | func (g *eGraph) push(n ...*innerNode) { 48 | g.nodes = append(g.nodes, n...) 49 | for _, node := range n { 50 | node.g = g 51 | } 52 | } 53 | 54 | func (g *eGraph) setup() { 55 | g.reset() 56 | 57 | for _, node := range g.nodes { 58 | node.setup() 59 | 60 | if len(node.dependents) == 0 { 61 | g.entries = append(g.entries, node) 62 | } 63 | } 64 | } 65 | 66 | func (g *eGraph) recyclable() bool { 67 | return g.joinCounter.Load() == 0 68 | } 69 | -------------------------------------------------------------------------------- /image/condition.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | G 11 | 12 | 13 | cluster_sub2 14 | 15 | sub2 16 | 17 | 18 | cluster_sub1 19 | 20 | sub1 21 | 22 | 23 | 24 | A3 25 | 26 | A3 27 | 28 | 29 | 30 | B3 31 | 32 | B3 33 | 34 | 35 | 36 | A3->B3 37 | 38 | 39 | 40 | 41 | 42 | C3 43 | 44 | C3 45 | 46 | 47 | 48 | C3->B3 49 | 50 | 51 | 52 | 53 | 54 | sub2 55 | 56 | 57 | 58 | 59 | A2 60 | 61 | A2 62 | 63 | 64 | 65 | B2 66 | 67 | B2 68 | 69 | 70 | 71 | A2->B2 72 | 73 | 74 | 75 | 76 | 77 | C2 78 | 79 | C2 80 | 81 | 82 | 83 | C2->B2 84 | 85 | 86 | 87 | 88 | 89 | sub1 90 | 91 | 92 | 93 | 94 | A 95 | 96 | A 97 | 98 | 99 | 100 | B 101 | 102 | B 103 | 104 | 105 | 106 | A->B 107 | 108 | 109 | 110 | 111 | 112 | cond 113 | 114 | cond 115 | 116 | 117 | 118 | B->cond 119 | 120 | 121 | 122 | 123 | 124 | cond->sub2 125 | 126 | 127 | 1 128 | 129 | 130 | 131 | cond->sub1 132 | 133 | 134 | 0 135 | 136 | 137 | 138 | C 139 | 140 | C 141 | 142 | 143 | 144 | C->B 145 | 146 | 147 | 148 | 149 | 150 | A1 151 | 152 | A1 153 | 154 | 155 | 156 | A1->B 157 | 158 | 159 | 160 | 161 | 162 | A1->C 163 | 164 | 165 | 166 | 167 | 168 | B1 169 | 170 | B1 171 | 172 | 173 | 174 | B1->C 175 | 176 | 177 | 178 | 179 | 180 | C1 181 | 182 | C1 183 | 184 | 185 | -------------------------------------------------------------------------------- /image/desc.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | G 11 | 12 | 13 | cluster_sub2 14 | 15 | sub2 16 | 17 | 18 | cluster_sub1 19 | 20 | sub1 21 | 22 | 23 | cluster_sub in sub 24 | 25 | sub in sub 26 | 27 | 28 | 29 | A3 30 | 31 | A3 32 | 33 | 34 | 35 | B3 36 | 37 | B3 38 | 39 | 40 | 41 | A3->B3 42 | 43 | 44 | 45 | 46 | 47 | C3 48 | 49 | C3 50 | 51 | 52 | 53 | C3->B3 54 | 55 | 56 | 57 | 58 | 59 | sub2 60 | 61 | 62 | 63 | 64 | done 65 | 66 | done 67 | 68 | 69 | 70 | sub in sub 71 | 72 | 73 | 74 | 75 | A2 76 | 77 | A2 78 | 79 | 80 | 81 | B2 82 | 83 | B2 84 | 85 | 86 | 87 | A2->B2 88 | 89 | 90 | 91 | 92 | 93 | C2 94 | 95 | C2 96 | 97 | 98 | 99 | C2->B2 100 | 101 | 102 | 103 | 104 | 105 | cond 106 | 107 | cond 108 | 109 | 110 | 111 | cond->sub in sub 112 | 113 | 114 | 0 115 | 116 | 117 | 118 | cond->cond 119 | 120 | 121 | 1 122 | 123 | 124 | 125 | sub1 126 | 127 | 128 | 129 | 130 | sub1->sub2 131 | 132 | 133 | 134 | 135 | 136 | A 137 | 138 | A 139 | 140 | 141 | 142 | B 143 | 144 | B 145 | 146 | 147 | 148 | A->B 149 | 150 | 151 | 152 | 153 | 154 | C 155 | 156 | C 157 | 158 | 159 | 160 | C->B 161 | 162 | 163 | 164 | 165 | 166 | C1 167 | 168 | C1 169 | 170 | 171 | 172 | C->C1 173 | 174 | 175 | 176 | 177 | 178 | C1->sub1 179 | 180 | 181 | 182 | 183 | 184 | A1 185 | 186 | A1 187 | 188 | 189 | 190 | A1->C 191 | 192 | 193 | 194 | 195 | 196 | B1 197 | 198 | B1 199 | 200 | 201 | 202 | B1->C 203 | 204 | 205 | 206 | 207 | -------------------------------------------------------------------------------- /image/fl.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 23 | 424 | 425 | Flame Graph 426 | 427 | Reset Zoom 428 | Search 429 | ic 430 | 431 | 432 | 433 | static,B,cost 11µs (11 samples, 3.44%) 434 | sta.. 435 | 436 | 437 | static,B3,cost 3µs480ns (3 samples, 0.94%) 438 | 439 | 440 | 441 | static,B2,cost 2µs800ns (2 samples, 0.62%) 442 | 443 | 444 | 445 | static,C,cost 3µs278ns (3 samples, 0.94%) 446 | 447 | 448 | 449 | subflow,sub2,cost 2µs789ns (38 samples, 11.88%) 450 | subflow,sub2,cost.. 451 | 452 | 453 | subflow,sub1,cost 2µs250ns (62 samples, 19.38%) 454 | subflow,sub1,cost 2µs250ns 455 | 456 | 457 | static,C1,cost 70µs619ns (70 samples, 21.88%) 458 | static,C1,cost 70µs619ns 459 | 460 | 461 | static,A,cost 13µs991ns (13 samples, 4.06%) 462 | stat.. 463 | 464 | 465 | static,B1,cost 71µs394ns (71 samples, 22.19%) 466 | static,B1,cost 71µs394ns 467 | 468 | 469 | static,A1,cost 63µs495ns (63 samples, 19.69%) 470 | static,A1,cost 63µs495ns 471 | 472 | 473 | static,C3,cost 7µs120ns (7 samples, 2.19%) 474 | s.. 475 | 476 | 477 | static,A2,cost 55µs376ns (55 samples, 17.19%) 478 | static,A2,cost 55µs376ns 479 | 480 | 481 | static,C2,cost 5µs27ns (5 samples, 1.56%) 482 | 483 | 484 | 485 | static,A3,cost 17µs306ns (17 samples, 5.31%) 486 | static.. 487 | 488 | 489 | all (320 samples, 100%) 490 | 491 | 492 | 493 | -------------------------------------------------------------------------------- /image/loop.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | G 11 | 12 | 13 | 14 | init 15 | 16 | init 17 | 18 | 19 | 20 | while i < 5 21 | 22 | while i < 5 23 | 24 | 25 | 26 | init->while i < 5 27 | 28 | 29 | 30 | 31 | 32 | body 33 | 34 | body 35 | 36 | 37 | 38 | while i < 5->body 39 | 40 | 41 | 0 42 | 43 | 44 | 45 | done 46 | 47 | done 48 | 49 | 50 | 51 | while i < 5->done 52 | 53 | 54 | 1 55 | 56 | 57 | 58 | back 59 | 60 | back 61 | 62 | 63 | 64 | body->back 65 | 66 | 67 | 68 | 69 | 70 | back->while i < 5 71 | 72 | 73 | 0 74 | 75 | 76 | -------------------------------------------------------------------------------- /image/simple.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | wordcount 4 | 5 | 6 | 7 | split_input 8 | 9 | split_input 10 | 11 | 12 | 13 | map_0 14 | 15 | map_0 16 | 17 | 18 | 19 | split_input->map_0 20 | 21 | 22 | 23 | 24 | 25 | map_1 26 | 27 | map_1 28 | 29 | 30 | 31 | split_input->map_1 32 | 33 | 34 | 35 | 36 | 37 | map_2 38 | 39 | map_2 40 | 41 | 42 | 43 | split_input->map_2 44 | 45 | 46 | 47 | 48 | 49 | map_3 50 | 51 | map_3 52 | 53 | 54 | 55 | split_input->map_3 56 | 57 | 58 | 59 | 60 | 61 | reduce_0 62 | 63 | reduce_0 64 | 65 | 66 | 67 | map_0->reduce_0 68 | 69 | 70 | 71 | 72 | 73 | reduce_1 74 | 75 | reduce_1 76 | 77 | 78 | 79 | map_0->reduce_1 80 | 81 | 82 | 83 | 84 | 85 | map_1->reduce_0 86 | 87 | 88 | 89 | 90 | 91 | map_1->reduce_1 92 | 93 | 94 | 95 | 96 | 97 | map_2->reduce_0 98 | 99 | 100 | 101 | 102 | 103 | map_2->reduce_1 104 | 105 | 106 | 107 | 108 | 109 | map_3->reduce_0 110 | 111 | 112 | 113 | 114 | 115 | map_3->reduce_1 116 | 117 | 118 | 119 | 120 | 121 | merge_results 122 | 123 | merge_results 124 | 125 | 126 | 127 | reduce_0->merge_results 128 | 129 | 130 | 131 | 132 | 133 | reduce_1->merge_results 134 | 135 | 136 | 137 | 138 | -------------------------------------------------------------------------------- /image/subflow.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | G 11 | 12 | 13 | cluster_sub2 14 | 15 | sub2 16 | 17 | 18 | cluster_sub1 19 | 20 | sub1 21 | 22 | 23 | 24 | A3 25 | 26 | A3 27 | 28 | 29 | 30 | B3 31 | 32 | B3 33 | 34 | 35 | 36 | A3->B3 37 | 38 | 39 | 40 | 41 | 42 | C3 43 | 44 | C3 45 | 46 | 47 | 48 | C3->B3 49 | 50 | 51 | 52 | 53 | 54 | sub2 55 | 56 | 57 | 58 | 59 | A2 60 | 61 | A2 62 | 63 | 64 | 65 | B2 66 | 67 | B2 68 | 69 | 70 | 71 | A2->B2 72 | 73 | 74 | 75 | 76 | 77 | C2 78 | 79 | C2 80 | 81 | 82 | 83 | C2->B2 84 | 85 | 86 | 87 | 88 | 89 | sub1 90 | 91 | 92 | 93 | 94 | sub1->sub2 95 | 96 | 97 | 98 | 99 | 100 | B 101 | 102 | B 103 | 104 | 105 | 106 | sub1->B 107 | 108 | 109 | 110 | 111 | 112 | A 113 | 114 | A 115 | 116 | 117 | 118 | A->B 119 | 120 | 121 | 122 | 123 | 124 | C 125 | 126 | C 127 | 128 | 129 | 130 | C->B 131 | 132 | 133 | 134 | 135 | 136 | A1 137 | 138 | A1 139 | 140 | 141 | 142 | A1->B 143 | 144 | 145 | 146 | 147 | 148 | A1->C 149 | 150 | 151 | 152 | 153 | 154 | B1 155 | 156 | B1 157 | 158 | 159 | 160 | B1->C 161 | 162 | 163 | 164 | 165 | 166 | C1 167 | 168 | C1 169 | 170 | 171 | -------------------------------------------------------------------------------- /node.go: -------------------------------------------------------------------------------- 1 | package gotaskflow 2 | 3 | import ( 4 | "strconv" 5 | "sync" 6 | "sync/atomic" 7 | "time" 8 | ) 9 | 10 | const ( 11 | kNodeStateIdle = int32(iota + 1) 12 | kNodeStateWaiting 13 | kNodeStateRunning 14 | kNodeStateFinished 15 | ) 16 | 17 | type nodeType string 18 | 19 | const ( 20 | nodeSubflow nodeType = "subflow" // subflow 21 | nodeStatic nodeType = "static" // static 22 | nodeCondition nodeType = "condition" // static 23 | ) 24 | 25 | type innerNode struct { 26 | name string 27 | successors []*innerNode 28 | dependents []*innerNode 29 | Typ nodeType 30 | ptr interface{} 31 | mu *sync.Mutex 32 | state atomic.Int32 33 | joinCounter atomic.Int32 34 | g *eGraph 35 | priority TaskPriority 36 | } 37 | 38 | func (n *innerNode) recyclable() bool { 39 | return n.joinCounter.Load() == 0 40 | } 41 | 42 | func (n *innerNode) ref() { 43 | n.joinCounter.Add(1) 44 | } 45 | 46 | func (n *innerNode) deref() { 47 | n.joinCounter.Add(-1) 48 | } 49 | 50 | func (n *innerNode) setup() { 51 | n.mu.Lock() 52 | defer n.mu.Unlock() 53 | n.state.Store(kNodeStateIdle) 54 | for _, dep := range n.dependents { 55 | if dep.Typ == nodeCondition { 56 | continue 57 | } 58 | 59 | n.ref() 60 | } 61 | } 62 | func (n *innerNode) drop() { 63 | // release every deps 64 | for _, node := range n.successors { 65 | if n.Typ != nodeCondition { 66 | node.deref() 67 | } 68 | } 69 | } 70 | 71 | // set dependency： V deps on N, V is input node 72 | func (n *innerNode) precede(v *innerNode) { 73 | n.successors = append(n.successors, v) 74 | v.dependents = append(v.dependents, n) 75 | } 76 | 77 | func newNode(name string) *innerNode { 78 | if len(name) == 0 { 79 | name = "N_" + strconv.Itoa(time.Now().Nanosecond()) 80 | } 81 | return &innerNode{ 82 | name: name, 83 | state: atomic.Int32{}, 84 | successors: make([]*innerNode, 0), 85 | dependents: make([]*innerNode, 0), 86 | mu: &sync.Mutex{}, 87 | priority: NORMAL, 88 | joinCounter: atomic.Int32{}, 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /profiler.go: -------------------------------------------------------------------------------- 1 | package gotaskflow 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "sync" 7 | "time" 8 | 9 | "github.com/noneback/go-taskflow/utils" 10 | ) 11 | 12 | type profiler struct { 13 | spans map[attr]*span 14 | 15 | mu *sync.Mutex 16 | } 17 | 18 | func newProfiler() *profiler { 19 | return &profiler{ 20 | spans: make(map[attr]*span), 21 | mu: &sync.Mutex{}, 22 | } 23 | } 24 | 25 | func (t *profiler) AddSpan(s *span) { 26 | t.mu.Lock() 27 | defer t.mu.Unlock() 28 | 29 | if span, ok := t.spans[s.extra]; ok { 30 | s.cost += span.cost 31 | } 32 | t.spans[s.extra] = s 33 | } 34 | 35 | type attr struct { 36 | typ nodeType 37 | name string 38 | } 39 | 40 | type span struct { 41 | extra attr 42 | begin time.Time 43 | cost time.Duration 44 | parent *span 45 | } 46 | 47 | func (s *span) String() string { 48 | return fmt.Sprintf("%s,%s,cost %v", s.extra.typ, s.extra.name, utils.NormalizeDuration(s.cost)) 49 | } 50 | 51 | func (t *profiler) draw(w io.Writer) error { 52 | // compact spans base on name 53 | t.mu.Lock() 54 | defer t.mu.Unlock() 55 | 56 | for _, s := range t.spans { 57 | path := "" 58 | if s.extra.typ != nodeSubflow { 59 | path = s.String() 60 | cur := s 61 | 62 | for cur.parent != nil { 63 | path = cur.parent.String() + ";" + path 64 | cur = cur.parent 65 | } 66 | msg := fmt.Sprintf("%s %v\n", path, s.cost.Microseconds()) 67 | 68 | if _, err := w.Write([]byte(msg)); err != nil { 69 | return fmt.Errorf("write profile -> %w", err) 70 | } 71 | } 72 | 73 | } 74 | return nil 75 | } 76 | -------------------------------------------------------------------------------- /profiler_test.go: -------------------------------------------------------------------------------- 1 | package gotaskflow 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | "time" 7 | 8 | "github.com/noneback/go-taskflow/utils" 9 | ) 10 | 11 | func TestProfilerAddSpan(t *testing.T) { 12 | profiler := newProfiler() 13 | mark := attr{ 14 | typ: nodeStatic, 15 | name: "test-span", 16 | } 17 | span := &span{ 18 | extra: mark, 19 | begin: time.Now(), 20 | cost: 5 * time.Millisecond, 21 | } 22 | profiler.AddSpan(span) 23 | 24 | if len(profiler.spans) != 1 { 25 | t.Errorf("expected 1 span, got %d", len(profiler.spans)) 26 | } 27 | 28 | if profiler.spans[mark] != span { 29 | t.Errorf("expected span to be added correctly, got %v", profiler.spans[mark]) 30 | } 31 | } 32 | 33 | func TestSpanString(t *testing.T) { 34 | now := time.Now() 35 | span := &span{ 36 | extra: attr{ 37 | typ: nodeStatic, 38 | name: "test-span", 39 | }, 40 | begin: now, 41 | cost: 10 * time.Millisecond, 42 | } 43 | 44 | expected := "static,test-span,cost " + utils.NormalizeDuration(10*time.Millisecond) 45 | actual := span.String() 46 | 47 | if actual != expected { 48 | t.Errorf("expected %s, got %s", expected, actual) 49 | } 50 | } 51 | 52 | func TestProfilerDraw(t *testing.T) { 53 | profiler := newProfiler() 54 | now := time.Now() 55 | parentSpan := &span{ 56 | extra: attr{ 57 | typ: nodeStatic, 58 | name: "parent", 59 | }, 60 | begin: now, 61 | cost: 10 * time.Millisecond, 62 | } 63 | 64 | childSpan := &span{ 65 | extra: attr{ 66 | typ: nodeStatic, 67 | name: "child", 68 | }, 69 | begin: now, 70 | cost: 5 * time.Millisecond, 71 | parent: parentSpan, 72 | } 73 | 74 | profiler.AddSpan(parentSpan) 75 | profiler.AddSpan(childSpan) 76 | 77 | var buf bytes.Buffer 78 | err := profiler.draw(&buf) 79 | if err != nil { 80 | t.Errorf("unexpected error: %v", err) 81 | } 82 | 83 | output := buf.String() 84 | if len(output) == 0 { 85 | t.Errorf("expected output, got empty string") 86 | } 87 | 88 | expectedOutput := "static,parent,cost 10ms 10000\nstatic,parent,cost 10ms;static,child,cost 5ms 5000\n" 89 | expectedOutput2 := "static,parent,cost 10ms;static,child,cost 5ms 5000\nstatic,parent,cost 10ms 10000\n" 90 | if output != expectedOutput && output != expectedOutput2 { 91 | t.Errorf("expected output: %v\ngot: %v", expectedOutput, output) 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /task.go: -------------------------------------------------------------------------------- 1 | package gotaskflow 2 | 3 | // Basic component of Taskflow 4 | type Task struct { 5 | node *innerNode 6 | } 7 | 8 | // Precede: Tasks all depend on *this*. 9 | // In Addition, order of tasks is correspond to predict result, ranging from 0...len(tasks) 10 | func (t *Task) Precede(tasks ...*Task) { 11 | if cond, ok := t.node.ptr.(*Condition); ok { 12 | for i, task := range tasks { 13 | cond.mapper[uint(i)] = task.node 14 | } 15 | } 16 | 17 | for _, task := range tasks { 18 | t.node.precede(task.node) 19 | } 20 | } 21 | 22 | // Succeed: *this* deps on tasks 23 | func (t *Task) Succeed(tasks ...*Task) { 24 | for _, task := range tasks { 25 | task.node.precede(t.node) 26 | } 27 | } 28 | 29 | func (t *Task) Name() string { 30 | return t.node.name 31 | } 32 | 33 | // Priority sets task's sche priority. Noted that due to goroutine concurrent mode, it can only assure task schedule priority, rather than its execution. 34 | func (t *Task) Priority(p TaskPriority) *Task { 35 | t.node.priority = p 36 | return t 37 | } 38 | 39 | // Task sche priority 40 | type TaskPriority uint 41 | 42 | const ( 43 | HIGH = TaskPriority(iota + 1) 44 | NORMAL 45 | LOW 46 | ) 47 | -------------------------------------------------------------------------------- /taskflow.go: -------------------------------------------------------------------------------- 1 | package gotaskflow 2 | 3 | import "io" 4 | 5 | // TaskFlow represents a series of tasks 6 | type TaskFlow struct { 7 | graph *eGraph 8 | frozen bool 9 | } 10 | 11 | // Reset resets taskflow 12 | func (tf *TaskFlow) Reset() { 13 | // tf.graph.reset() 14 | tf.frozen = false 15 | } 16 | 17 | // NewTaskFlow returns a taskflow struct 18 | func NewTaskFlow(name string) *TaskFlow { 19 | return &TaskFlow{ 20 | graph: newGraph(name), 21 | } 22 | } 23 | 24 | // Push pushs all task into taskflow 25 | func (tf *TaskFlow) push(tasks ...*Task) { 26 | if tf.frozen { 27 | panic("Taskflow is frozen, cannot new tasks") 28 | } 29 | 30 | for _, task := range tasks { 31 | tf.graph.push(task.node) 32 | } 33 | } 34 | 35 | func (tf *TaskFlow) Name() string { 36 | return tf.graph.name 37 | } 38 | 39 | // NewStaticTask returns a attached static task 40 | func (tf *TaskFlow) NewTask(name string, f func()) *Task { 41 | task := &Task{ 42 | node: builder.NewStatic(name, f), 43 | } 44 | tf.push(task) 45 | return task 46 | } 47 | 48 | // NewSubflow returns a attached subflow task 49 | // NOTICE: instantiate will be invoke only once to instantiate itself 50 | func (tf *TaskFlow) NewSubflow(name string, instantiate func(sf *Subflow)) *Task { 51 | task := &Task{ 52 | node: builder.NewSubflow(name, instantiate), 53 | } 54 | tf.push(task) 55 | return task 56 | } 57 | 58 | // NewCondition returns a attached condition task. NOTICE: The predict func return value determines its successor. 59 | func (tf *TaskFlow) NewCondition(name string, predict func() uint) *Task { 60 | task := &Task{ 61 | node: builder.NewCondition(name, predict), 62 | } 63 | tf.push(task) 64 | return task 65 | } 66 | 67 | // Dump writes graph dot data into writer 68 | func (tf *TaskFlow) Dump(writer io.Writer) error { 69 | return dot.Visualize(tf, writer) 70 | } 71 | -------------------------------------------------------------------------------- /taskflow_test.go: -------------------------------------------------------------------------------- 1 | package gotaskflow_test 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | _ "net/http/pprof" 7 | "os" 8 | "sync/atomic" 9 | "testing" 10 | "time" 11 | 12 | gotaskflow "github.com/noneback/go-taskflow" 13 | "github.com/noneback/go-taskflow/utils" 14 | ) 15 | 16 | type rgChain[R comparable] struct { 17 | rgs []*rgroup[R] 18 | } 19 | 20 | func newRgChain[R comparable]() *rgChain[R] { 21 | return &rgChain[R]{ 22 | rgs: make([]*rgroup[R], 0), 23 | } 24 | } 25 | 26 | func (c *rgChain[R]) grouping(rs ...R) { 27 | g := newRg[R]() 28 | g.push(rs...) 29 | c.rgs = append(c.rgs, g) 30 | } 31 | 32 | // result group 33 | type rgroup[R comparable] struct { 34 | pre, next *rgroup[R] 35 | elems map[R]struct{} 36 | } 37 | 38 | func newRg[R comparable]() *rgroup[R] { 39 | return &rgroup[R]{ 40 | elems: make(map[R]struct{}), 41 | } 42 | } 43 | 44 | func (g *rgroup[R]) push(rs ...R) { 45 | for _, r := range rs { 46 | g.elems[r] = struct{}{} 47 | } 48 | } 49 | 50 | func (g *rgroup[R]) chain(successor *rgroup[R]) { 51 | g.next = successor 52 | successor.pre = g.next 53 | } 54 | 55 | func (g *rgroup[R]) contains(r R) bool { 56 | _, ok := g.elems[r] 57 | return ok 58 | } 59 | 60 | func checkTopology[R comparable](t *testing.T, q *utils.Queue[R], chain *rgChain[R]) { 61 | for _, g := range chain.rgs { 62 | for len(g.elems) != 0 { 63 | node := q.Pop() 64 | if g.contains(node) { 65 | delete(g.elems, node) 66 | } else { 67 | fmt.Println("failed in", node) 68 | t.Fail() 69 | } 70 | } 71 | } 72 | } 73 | 74 | var executor = gotaskflow.NewExecutor(10) 75 | 76 | func TestTaskFlow(t *testing.T) { 77 | q := utils.NewQueue[string](true) 78 | tf := gotaskflow.NewTaskFlow("G") 79 | A, B, C := 80 | tf.NewTask("A", func() { 81 | fmt.Println("A") 82 | q.Put("A") 83 | }), 84 | tf.NewTask("B", func() { 85 | fmt.Println("B") 86 | q.Put("B") 87 | }), 88 | tf.NewTask("C", func() { 89 | fmt.Println("C") 90 | q.Put("C") 91 | }) 92 | 93 | A1, B1, _ := 94 | tf.NewTask("A1", func() { 95 | fmt.Println("A1") 96 | q.Put("A1") 97 | }), 98 | tf.NewTask("B1", func() { 99 | fmt.Println("B1") 100 | q.Put("B1") 101 | }), 102 | tf.NewTask("C1", func() { 103 | fmt.Println("C1") 104 | q.Put("C1") 105 | }) 106 | chains := newRgChain[string]() 107 | chains.grouping("C1", "A1", "B1", "A", "C") 108 | chains.grouping("B") 109 | 110 | A.Precede(B) 111 | C.Precede(B) 112 | A1.Precede(B) 113 | C.Succeed(A1) 114 | C.Succeed(B1) 115 | 116 | t.Run("TestViz", func(t *testing.T) { 117 | if err := tf.Dump(os.Stdout); err != nil { 118 | panic(err) 119 | } 120 | }) 121 | 122 | executor.Run(tf).Wait() 123 | if q.Len() != 6 { 124 | t.Fail() 125 | } 126 | 127 | // checkTopology(t, q, chains) 128 | } 129 | 130 | func TestSubflow(t *testing.T) { 131 | q := utils.NewQueue[string](true) 132 | // chains := newRgChain[string]() 133 | tf := gotaskflow.NewTaskFlow("G") 134 | A, B, C := 135 | tf.NewTask("A", func() { 136 | fmt.Println("A") 137 | q.Put("A") 138 | }), 139 | tf.NewTask("B", func() { 140 | fmt.Println("B") 141 | q.Put("B") 142 | }), 143 | tf.NewTask("C", func() { 144 | fmt.Println("C") 145 | q.Put("C") 146 | }) 147 | 148 | A1, B1, C1 := 149 | tf.NewTask("A1", func() { 150 | fmt.Println("A1") 151 | q.Put("A1") 152 | }), 153 | tf.NewTask("B1", func() { 154 | fmt.Println("B1") 155 | q.Put("B1") 156 | }), 157 | tf.NewTask("C1", func() { 158 | fmt.Println("C1") 159 | q.Put("C1") 160 | }) 161 | A.Precede(B) 162 | C.Precede(B) 163 | C.Succeed(A1) 164 | C.Succeed(B1) 165 | 166 | subflow := tf.NewSubflow("sub1", func(sf *gotaskflow.Subflow) { 167 | A2, B2, C2 := 168 | sf.NewTask("A2", func() { 169 | fmt.Println("A2") 170 | q.Put("A2") 171 | }), 172 | sf.NewTask("B2", func() { 173 | fmt.Println("B2") 174 | q.Put("B2") 175 | }), 176 | sf.NewTask("C2", func() { 177 | fmt.Println("C2") 178 | q.Put("C2") 179 | }) 180 | A2.Precede(B2) 181 | C2.Precede(B2) 182 | cond := sf.NewCondition("cond", func() uint { 183 | return 0 184 | }) 185 | 186 | ssub := sf.NewSubflow("sub in sub", func(sf *gotaskflow.Subflow) { 187 | sf.NewTask("done", func() { 188 | fmt.Println("done") 189 | }) 190 | }) 191 | 192 | cond.Precede(ssub, cond) 193 | 194 | }) 195 | 196 | subflow2 := tf.NewSubflow("sub2", func(sf *gotaskflow.Subflow) { 197 | A3, B3, C3 := 198 | sf.NewTask("A3", func() { 199 | fmt.Println("A3") 200 | q.Put("A3") 201 | }), 202 | sf.NewTask("B3", func() { 203 | fmt.Println("B3") 204 | q.Put("B3") 205 | }), 206 | sf.NewTask("C3", func() { 207 | fmt.Println("C3") 208 | q.Put("C3") 209 | // time.Sleep(10 * time.Second) 210 | }) 211 | A3.Precede(B3) 212 | C3.Precede(B3) 213 | 214 | }) 215 | 216 | subflow.Precede(subflow2) 217 | C1.Precede(subflow) 218 | C1.Succeed(C) 219 | 220 | executor.Run(tf) 221 | executor.Wait() 222 | if err := tf.Dump(os.Stdout); err != nil { 223 | log.Fatal(err) 224 | } 225 | executor.Profile(os.Stdout) 226 | 227 | chain := newRgChain[string]() 228 | 229 | // Group 1 - Top-level nodes 230 | chain.grouping("A1", "B1", "A") 231 | chain.grouping("C") 232 | chain.grouping("B", "C1") 233 | chain.grouping("A2", "C2") 234 | chain.grouping("B2") 235 | 236 | // Group 2 - Connections under A, B, C 237 | chain.grouping("A3", "C3") 238 | chain.grouping("B3") 239 | 240 | // validate 241 | if q.Len() != 12 { 242 | t.Fail() 243 | } 244 | // checkTopology(t, q, chain) 245 | } 246 | 247 | // ERROR robust testing 248 | func TestTaskflowPanic(t *testing.T) { 249 | tf := gotaskflow.NewTaskFlow("G") 250 | A, B, C := 251 | tf.NewTask("A", func() { 252 | fmt.Println("A") 253 | }), 254 | tf.NewTask("B", func() { 255 | fmt.Println("B") 256 | }), 257 | tf.NewTask("C", func() { 258 | fmt.Println("C") 259 | panic("panic C") 260 | }) 261 | A.Precede(B) 262 | C.Precede(B) 263 | 264 | executor.Run(tf).Wait() 265 | } 266 | 267 | func TestSubflowPanic(t *testing.T) { 268 | tf := gotaskflow.NewTaskFlow("G") 269 | A, B, C := 270 | tf.NewTask("A", func() { 271 | fmt.Println("A") 272 | }), 273 | tf.NewTask("B", func() { 274 | fmt.Println("B") 275 | }), 276 | tf.NewTask("C", func() { 277 | fmt.Println("C") 278 | }) 279 | A.Precede(B) 280 | C.Precede(B) 281 | 282 | subflow := tf.NewSubflow("sub1", func(sf *gotaskflow.Subflow) { 283 | A2, B2, C2 := 284 | tf.NewTask("A2", func() { 285 | fmt.Println("A2") 286 | time.Sleep(1 * time.Second) 287 | }), 288 | tf.NewTask("B2", func() { 289 | fmt.Println("B2") 290 | }), 291 | tf.NewTask("C2", func() { 292 | fmt.Println("C2") 293 | panic("C2 panicked") 294 | }) 295 | A2.Precede(B2) 296 | panic("subflow panic") 297 | C2.Precede(B2) 298 | }) 299 | 300 | subflow.Precede(B) 301 | executor.Run(tf) 302 | executor.Wait() 303 | if err := tf.Dump(os.Stdout); err != nil { 304 | fmt.Errorf("%v", err) 305 | } 306 | executor.Profile(os.Stdout) 307 | } 308 | 309 | func TestTaskflowCondition(t *testing.T) { 310 | q := utils.NewQueue[string](true) 311 | chain := newRgChain[string]() 312 | tf := gotaskflow.NewTaskFlow("G") 313 | t.Run("normal", func(t *testing.T) { 314 | A, B, C := 315 | tf.NewTask("A", func() { 316 | fmt.Println("A") 317 | q.Put("A") 318 | }), 319 | tf.NewTask("B", func() { 320 | fmt.Println("B") 321 | q.Put("B") 322 | }), 323 | tf.NewTask("C", func() { 324 | fmt.Println("C") 325 | q.Put("C") 326 | }) 327 | A.Precede(B) 328 | C.Precede(B) 329 | 330 | fail, success := tf.NewTask("failed", func() { 331 | fmt.Println("Failed") 332 | q.Put("failed") 333 | t.Fail() 334 | }), tf.NewTask("success", func() { 335 | fmt.Println("success") 336 | q.Put("success") 337 | }) 338 | 339 | cond := tf.NewCondition("cond", func() uint { 340 | q.Put("cond") 341 | return 0 342 | }) 343 | B.Precede(cond) 344 | cond.Precede(success, fail) 345 | 346 | suc := tf.NewSubflow("sub1", func(sf *gotaskflow.Subflow) { 347 | A2, B2, C2 := 348 | sf.NewTask("A2", func() { 349 | fmt.Println("A2") 350 | q.Put("A2") 351 | }), 352 | sf.NewTask("B2", func() { 353 | fmt.Println("B2") 354 | q.Put("B2") 355 | }), 356 | sf.NewTask("C2", func() { 357 | fmt.Println("C2") 358 | q.Put("C2") 359 | }) 360 | A2.Precede(B2) 361 | C2.Precede(B2) 362 | }).Priority(gotaskflow.HIGH) 363 | fs := tf.NewTask("fail_single", func() { 364 | fmt.Println("it should be canceled") 365 | q.Put("fail_single") 366 | }) 367 | fail.Precede(fs, suc) 368 | // success.Precede(suc) 369 | if err := tf.Dump(os.Stdout); err != nil { 370 | fmt.Errorf("%v", err) 371 | } 372 | executor.Run(tf).Wait() 373 | 374 | executor.Profile(os.Stdout) 375 | chain.grouping("A", "C") 376 | chain.grouping("B") 377 | chain.grouping("cond") 378 | chain.grouping("success") 379 | 380 | checkTopology(t, q, chain) 381 | 382 | }) 383 | 384 | t.Run("start with condition node", func(t *testing.T) { 385 | i := 0 386 | tf := gotaskflow.NewTaskFlow("G") 387 | 388 | cond := tf.NewCondition("cond", func() uint { 389 | if i == 0 { 390 | return 0 391 | } else { 392 | return 1 393 | } 394 | }) 395 | 396 | zero, one := tf.NewTask("zero", func() { 397 | fmt.Println("zero") 398 | }), tf.NewTask("one", func() { 399 | fmt.Println("one") 400 | }) 401 | cond.Precede(zero, one) 402 | 403 | executor.Run(tf).Wait() 404 | 405 | if err := tf.Dump(os.Stdout); err != nil { 406 | log.Fatal(err) 407 | } 408 | executor.Profile(os.Stdout) 409 | 410 | }) 411 | 412 | } 413 | 414 | func TestTaskflowLoop(t *testing.T) { 415 | t.Run("normal", func(t *testing.T) { 416 | i := 0 417 | tf := gotaskflow.NewTaskFlow("G") 418 | init, cond, body, back, done := 419 | tf.NewTask("init", func() { 420 | i = 0 421 | fmt.Println("i=0") 422 | }), 423 | tf.NewCondition("while i < 5", func() uint { 424 | if i < 5 { 425 | return 0 426 | } else { 427 | return 1 428 | } 429 | }), 430 | tf.NewTask("body", func() { 431 | i += 1 432 | fmt.Println("i++ =", i) 433 | }), 434 | tf.NewCondition("back", func() uint { 435 | fmt.Println("back") 436 | return 0 437 | }), 438 | tf.NewTask("done", func() { 439 | fmt.Println("done") 440 | }) 441 | 442 | init.Precede(cond) 443 | cond.Precede(body, done) 444 | body.Precede(back) 445 | back.Precede(cond) 446 | if err := tf.Dump(os.Stdout); err != nil { 447 | // log.Fatal(err) 448 | } 449 | executor.Run(tf).Wait() 450 | if i < 5 { 451 | t.Fail() 452 | } 453 | 454 | executor.Profile(os.Stdout) 455 | }) 456 | 457 | t.Run("simple loop", func(t *testing.T) { 458 | i := 0 459 | tf := gotaskflow.NewTaskFlow("G") 460 | init := tf.NewTask("init", func() { 461 | i = 0 462 | }) 463 | cond := tf.NewCondition("cond", func() uint { 464 | i++ 465 | fmt.Println("i++ =", i) 466 | if i > 2 { 467 | return 0 468 | } else { 469 | return 1 470 | } 471 | }) 472 | 473 | done := tf.NewTask("done", func() { 474 | fmt.Println("done") 475 | }) 476 | 477 | init.Precede(cond) 478 | cond.Precede(done, cond) 479 | 480 | executor.Run(tf).Wait() 481 | if i <= 2 { 482 | t.Fail() 483 | } 484 | 485 | if err := tf.Dump(os.Stdout); err != nil { 486 | // log.Fatal(err) 487 | } 488 | executor.Profile(os.Stdout) 489 | }) 490 | } 491 | 492 | func TestTaskflowPriority(t *testing.T) { 493 | executor := gotaskflow.NewExecutor(uint(1)) 494 | q := utils.NewQueue[byte](true) 495 | tf := gotaskflow.NewTaskFlow("G") 496 | tf.NewTask("B", func() { 497 | fmt.Println("B") 498 | q.Put('B') 499 | }).Priority(gotaskflow.NORMAL) 500 | 501 | tf.NewTask("C", func() { 502 | fmt.Println("C") 503 | q.Put('C') 504 | }).Priority(gotaskflow.HIGH) 505 | A := tf.NewTask("A", func() { 506 | fmt.Println("A") 507 | q.Put('A') 508 | }).Priority(gotaskflow.LOW) 509 | 510 | A.Precede(tf.NewTask("A2", func() { 511 | fmt.Println("A2") 512 | q.Put('a') 513 | }).Priority(gotaskflow.LOW), 514 | tf.NewTask("B2", func() { 515 | fmt.Println("B2") 516 | q.Put('b') 517 | }).Priority(gotaskflow.HIGH), 518 | tf.NewTask("C2", func() { 519 | fmt.Println("C2") 520 | q.Put('c') 521 | }).Priority(gotaskflow.NORMAL)) 522 | 523 | executor.Run(tf).Wait() 524 | tf.Dump(os.Stdout) 525 | fmt.Println("validate") 526 | for _, val := range []byte{'C', 'B', 'A', 'b', 'c', 'a'} { 527 | real := q.Pop() 528 | fmt.Printf("%c, ", real) 529 | if val != real { 530 | t.Fatal("[FAILED]", string(val), string(real)) 531 | t.FailNow() 532 | } 533 | } 534 | } 535 | 536 | func TestTaskflowNotInFlow(t *testing.T) { 537 | tf := gotaskflow.NewTaskFlow("tf") 538 | task := tf.NewTask("init", func() { 539 | fmt.Println("task init") 540 | }) 541 | cnt := 0 542 | for i := 0; i < 10; i++ { 543 | task.Precede(tf.NewTask("test", func() { 544 | fmt.Println(cnt) 545 | cnt++ 546 | })) 547 | } 548 | 549 | executor.Run(tf).Wait() 550 | } 551 | 552 | func TestTaskflowFrozen(t *testing.T) { 553 | tf := gotaskflow.NewTaskFlow("G") 554 | A, B, C := 555 | tf.NewTask("A", func() { 556 | fmt.Println("A") 557 | }), 558 | tf.NewTask("B", func() { 559 | fmt.Println("B") 560 | }), 561 | tf.NewTask("C", func() { 562 | fmt.Println("C") 563 | }) 564 | A.Precede(B) 565 | C.Precede(B) 566 | 567 | executor.Run(tf).Wait() 568 | utils.AssertPanics(t, "frozen", func() { 569 | tf.NewTask("tt", func() { 570 | fmt.Println("should not") 571 | }) 572 | }) 573 | } 574 | 575 | func TestLoopRunManyTimes(t *testing.T) { 576 | tf := gotaskflow.NewTaskFlow("G") 577 | var count atomic.Int32 578 | // add := func() { 579 | // count.Add(1) 580 | // } 581 | add := func(name string) func() { 582 | return func() { 583 | fmt.Println(name) 584 | count.Add(1) 585 | } 586 | } 587 | A, B, C := 588 | tf.NewTask("A", add("A")), 589 | tf.NewTask("B", add("B")), 590 | tf.NewTask("C", add("C")) 591 | A.Precede(B) 592 | C.Precede(B) 593 | t.Run("static", func(t *testing.T) { 594 | for i := 0; i < 10000; i++ { 595 | log.Println("static iter ---> ", i) 596 | if cnt := count.Load(); cnt%3 != 0 { 597 | t.Error("static unexpected count", cnt) 598 | return 599 | } 600 | executor.Run(tf).Wait() 601 | } 602 | }) 603 | // tf.Dump(os.Stdout) 604 | 605 | tf.Reset() 606 | count.Store(0) 607 | 608 | sf := tf.NewSubflow("sub", func(sf *gotaskflow.Subflow) { 609 | fmt.Println("sub") 610 | A1, B1, C1 := 611 | sf.NewTask("A1", add("A1")), 612 | sf.NewTask("B1", add("B1")), 613 | sf.NewTask("C1", add("C1")) 614 | A1.Precede(B1) 615 | C1.Precede(B1) 616 | }) 617 | additional := tf.NewTask("additional", add("Additional")) 618 | B.Precede(sf) 619 | additional.Precede(sf) 620 | executor.Run(tf).Wait() 621 | // dot, _ := os.OpenFile("./dot.data", os.O_RDWR, os.ModeAppend) 622 | 623 | tf.Dump(os.Stdout) 624 | t.Run("subflow", func(t *testing.T) { 625 | for i := 0; i < 10000; i++ { 626 | log.Println("subflow iter ---> ", i) 627 | if cnt := count.Load(); cnt%7 != 0 { 628 | t.Error("subflow unexpected count", cnt) 629 | return 630 | } 631 | executor.Run(tf).Wait() 632 | } 633 | }) 634 | 635 | tf.Reset() 636 | count.Store(0) 637 | 638 | cond := tf.NewCondition("if count %10 % 7 == 0", func() uint { 639 | if count.Load()%7 == 0 { 640 | return 0 641 | } else { 642 | return 1 643 | } 644 | }) 645 | plus7 := tf.NewTask("7 plus 7", func() { 646 | count.Add(7) 647 | }) 648 | cond.Precede(plus7, tf.NewTask("7 minus 3", func() { 649 | log.Println(count.Load()) 650 | log.Println("should not minus 3") 651 | })) 652 | sf.Precede(cond) 653 | 654 | t.Run("condition", func(t *testing.T) { 655 | for i := 0; i < 10000; i++ { 656 | log.Println("condition iter ---> ", i) 657 | if cnt := count.Load(); cnt%7 != 0 { 658 | t.Error("condition unexpect count", cnt) 659 | return 660 | } 661 | executor.Run(tf).Wait() 662 | } 663 | }) 664 | 665 | tf.Reset() 666 | count.Store(0) 667 | cond2 := tf.NewCondition("bigger than 10000", func() uint { 668 | if count.Load() > 10000 { 669 | return 1 670 | } else { 671 | return 0 672 | } 673 | }) 674 | 675 | plus7.Precede(cond2) 676 | done := tf.NewTask("done", func() { 677 | if count.Load() < 10000 { 678 | t.Fail() 679 | } 680 | }) 681 | 682 | new_plus7 := tf.NewTask("new plus 7", func() { 683 | count.Add(7) 684 | }) 685 | cond2.Precede(new_plus7, done) 686 | new_plus7.Precede(cond2) 687 | 688 | tf.Dump(os.Stdout) 689 | t.Run("loop", func(t *testing.T) { 690 | for i := 0; i < 10000; i++ { 691 | log.Println("loop iter ---> ", i) 692 | if cnt := count.Load(); cnt%7 != 0 { 693 | log.Println(cnt) 694 | t.Error("loop unexpect count", cnt) 695 | return 696 | } 697 | executor.Run(tf).Wait() 698 | } 699 | }) 700 | } 701 | 702 | func TestSequencialTaskingPanic(t *testing.T) { 703 | exe := gotaskflow.NewExecutor(1) 704 | tfl := gotaskflow.NewTaskFlow("test") 705 | q := utils.NewQueue[string](true) 706 | tfl.NewTask("task1", func() { 707 | q.Put("panic") 708 | fmt.Println("task1") 709 | panic(1) 710 | }) 711 | tfl.NewTask("task2", func() { 712 | q.Put("2") 713 | fmt.Println("task2") 714 | }) 715 | tfl.NewTask("task3", func() { 716 | q.Put("3") 717 | fmt.Println("task3") 718 | }) 719 | exe.Run(tfl).Wait() 720 | if q.Top() != "panic" { 721 | t.Fail() 722 | } 723 | } 724 | -------------------------------------------------------------------------------- /utils/copool.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "runtime/debug" 8 | "sync" 9 | "sync/atomic" 10 | ) 11 | 12 | type cotask struct { 13 | ctx *context.Context 14 | f func() 15 | } 16 | 17 | func (ct *cotask) zero() { 18 | ct.ctx = nil 19 | ct.f = nil 20 | } 21 | 22 | type Copool struct { 23 | panicHandler func(*context.Context, interface{}) 24 | cap uint 25 | taskQ *Queue[*cotask] 26 | corun atomic.Int32 27 | coworker atomic.Int32 28 | mu *sync.Mutex 29 | taskObjPool *ObjectPool[*cotask] 30 | } 31 | 32 | // NewCopool return a goroutine pool with specified cap 33 | func NewCopool(cap uint) *Copool { 34 | return &Copool{ 35 | panicHandler: nil, 36 | taskQ: NewQueue[*cotask](false), 37 | cap: cap, 38 | corun: atomic.Int32{}, 39 | coworker: atomic.Int32{}, 40 | mu: &sync.Mutex{}, 41 | taskObjPool: NewObjectPool(func() *cotask { 42 | return &cotask{} 43 | }), 44 | } 45 | } 46 | 47 | // Go executes f. 48 | func (cp *Copool) Go(f func()) { 49 | ctx := context.Background() 50 | cp.CtxGo(&ctx, f) 51 | } 52 | 53 | // CtxGo executes f and accepts the context. 54 | func (cp *Copool) CtxGo(ctx *context.Context, f func()) { 55 | cp.corun.Add(1) 56 | task := cp.taskObjPool.Get() 57 | task.f = func() { 58 | defer func() { 59 | if r := recover(); r != nil { 60 | if cp.panicHandler != nil { 61 | cp.panicHandler(ctx, r) 62 | } else { 63 | msg := fmt.Sprintf("[panic] copool: %v: %s", r, debug.Stack()) 64 | fmt.Println(msg) 65 | os.Exit(-1) 66 | } 67 | } 68 | }() 69 | defer cp.corun.Add(-1) 70 | f() 71 | } 72 | 73 | task.ctx = ctx 74 | cp.mu.Lock() 75 | cp.taskQ.Put(task) 76 | 77 | if cp.coworker.Load() == 0 || cp.taskQ.Len() != 0 && uint(cp.coworker.Load()) < uint(cp.cap) { 78 | cp.mu.Unlock() 79 | cp.coworker.Add(1) 80 | 81 | go func() { 82 | defer cp.coworker.Add(-1) 83 | 84 | for { 85 | cp.mu.Lock() 86 | if cp.taskQ.Len() == 0 { 87 | cp.mu.Unlock() 88 | return 89 | } 90 | 91 | task := cp.taskQ.Pop() 92 | cp.mu.Unlock() 93 | task.f() 94 | task.zero() 95 | cp.taskObjPool.Put(task) 96 | } 97 | 98 | }() 99 | } else { 100 | cp.mu.Unlock() 101 | } 102 | } 103 | 104 | // SetPanicHandler sets the panic handler. 105 | func (cp *Copool) SetPanicHandler(f func(*context.Context, interface{})) *Copool { 106 | cp.panicHandler = f 107 | return cp 108 | } 109 | -------------------------------------------------------------------------------- /utils/copool_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "sync/atomic" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | const benchmarkTimes = 100000 13 | 14 | func DoCopyStack(a, b int) int { 15 | if b < 100 { 16 | return DoCopyStack(0, b+1) 17 | } 18 | return 0 19 | } 20 | 21 | func testFunc() { 22 | DoCopyStack(0, 0) 23 | } 24 | 25 | func TestPool(t *testing.T) { 26 | p := NewCopool(10000) 27 | var n int32 28 | var wg sync.WaitGroup 29 | for i := 0; i < 2000; i++ { 30 | // fmt.Print(i) 31 | wg.Add(1) 32 | p.Go(func() { 33 | defer wg.Done() 34 | atomic.AddInt32(&n, 1) 35 | }) 36 | } 37 | wg.Wait() 38 | if n != 2000 { 39 | t.Error(n) 40 | } 41 | } 42 | 43 | func testPanic() { 44 | n := 0 45 | fmt.Println(1 / n) 46 | } 47 | 48 | func TestPoolPanic(t *testing.T) { 49 | p := NewCopool(10000) 50 | p.SetPanicHandler(func(ctx *context.Context, i interface{}) { 51 | fmt.Sprintln(i) 52 | }) 53 | var wg sync.WaitGroup 54 | p.Go(testPanic) 55 | wg.Wait() 56 | time.Sleep(time.Second) 57 | } 58 | 59 | func TestPoolSequentialExec(t *testing.T) { 60 | p := NewCopool(1) 61 | q := make([]int, 0, 10000) 62 | // mutex := &sync.Mutex{} 63 | idx := 0 64 | 65 | for i := 0; i < 10000; i++ { 66 | p.Go(func() { 67 | q = append(q, idx) 68 | idx++ 69 | }) 70 | } 71 | time.Sleep(1 * time.Second) 72 | 73 | fmt.Println("len", len(q)) 74 | 75 | for idx, v := range q { 76 | if idx != v { 77 | fmt.Println(idx, v) 78 | t.Fail() 79 | } 80 | } 81 | 82 | } 83 | 84 | func BenchmarkCopool(b *testing.B) { 85 | p := NewCopool(10000) 86 | var wg sync.WaitGroup 87 | b.ReportAllocs() 88 | b.ResetTimer() 89 | for i := 0; i < b.N; i++ { 90 | wg.Add(benchmarkTimes) 91 | for j := 0; j < benchmarkTimes; j++ { 92 | p.Go(func() { 93 | testFunc() 94 | wg.Done() 95 | }) 96 | } 97 | wg.Wait() 98 | } 99 | } 100 | func BenchmarkGo(b *testing.B) { 101 | var wg sync.WaitGroup 102 | b.ReportAllocs() 103 | b.ResetTimer() 104 | for i := 0; i < b.N; i++ { 105 | wg.Add(benchmarkTimes) 106 | for j := 0; j < benchmarkTimes; j++ { 107 | go func() { 108 | testFunc() 109 | wg.Done() 110 | }() 111 | } 112 | wg.Wait() 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /utils/obj_pool.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "sync" 4 | 5 | // ObjectPool with Type 6 | type ObjectPool[T any] struct { 7 | pool sync.Pool 8 | } 9 | 10 | func NewObjectPool[T any](creator func() T) *ObjectPool[T] { 11 | return &ObjectPool[T]{ 12 | pool: sync.Pool{ 13 | New: func() any { return creator() }, 14 | }, 15 | } 16 | } 17 | 18 | func (p *ObjectPool[T]) Get() T { 19 | return p.pool.Get().(T) 20 | } 21 | 22 | func (p *ObjectPool[T]) Put(x T) { 23 | p.pool.Put(x) 24 | } 25 | -------------------------------------------------------------------------------- /utils/pprof.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "runtime/pprof" 6 | ) 7 | 8 | type ProfileType int 9 | 10 | type PprofUtils struct { 11 | f *os.File 12 | profile ProfileType 13 | } 14 | 15 | func NewPprofUtils(profile ProfileType, output string) *PprofUtils { 16 | p := &PprofUtils{} 17 | f, err := os.Create(output) 18 | if err != nil { 19 | panic(err) 20 | } 21 | p.f = f 22 | p.profile = profile 23 | 24 | return p 25 | } 26 | 27 | const ( 28 | CPU ProfileType = iota 29 | HEAP 30 | ) 31 | 32 | func (p *PprofUtils) StartProfile() { 33 | switch p.profile { 34 | case CPU: 35 | if err := pprof.StartCPUProfile(p.f); err != nil { 36 | panic(err) 37 | } 38 | case HEAP: 39 | if err := pprof.WriteHeapProfile(p.f); err != nil { 40 | panic(err) 41 | } 42 | default: 43 | panic("unsupported profile type") 44 | } 45 | 46 | } 47 | 48 | func (p *PprofUtils) StopProfile() { 49 | defer p.f.Close() 50 | 51 | switch p.profile { 52 | case CPU: 53 | pprof.StopCPUProfile() 54 | case HEAP: 55 | default: 56 | panic("unsupported profile type") 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /utils/pprof_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | func TestCPUProfile(t *testing.T) { 10 | tmpFile, err := os.CreateTemp("", "cpu_profile.*.prof") 11 | if err != nil { 12 | t.Fatalf("Failed to create temp file: %v", err) 13 | } 14 | defer os.Remove(tmpFile.Name()) 15 | tmpFile.Close() 16 | 17 | profiler := NewPprofUtils(CPU, tmpFile.Name()) 18 | defer profiler.StopProfile() 19 | 20 | profiler.StartProfile() 21 | 22 | doCPUWork() 23 | } 24 | 25 | func TestHeapProfile(t *testing.T) { 26 | tmpFile, err := ioutil.TempFile("", "heap_profile.*.prof") 27 | if err != nil { 28 | t.Fatalf("Failed to create temp file: %v", err) 29 | } 30 | defer os.Remove(tmpFile.Name()) 31 | tmpFile.Close() 32 | 33 | profiler := NewPprofUtils(HEAP, tmpFile.Name()) 34 | defer profiler.StopProfile() 35 | profiler.StartProfile() 36 | 37 | doMemoryWork() 38 | } 39 | 40 | func TestInvalidProfileType(t *testing.T) { 41 | defer func() { 42 | if r := recover(); r == nil { 43 | t.Error("Expected panic for invalid profile type, but got none") 44 | } 45 | }() 46 | 47 | p := NewPprofUtils(ProfileType(99), "invalid.prof") 48 | p.StartProfile() 49 | defer p.StopProfile() 50 | } 51 | 52 | func TestFileCreateError(t *testing.T) { 53 | defer func() { 54 | if r := recover(); r == nil { 55 | t.Error("Expected panic when file creation fails, but got none") 56 | } 57 | }() 58 | 59 | _ = NewPprofUtils(CPU, "/nonexistent/cpu.prof") 60 | } 61 | 62 | func doCPUWork() { 63 | for i := 0; i < 1e6; i++ { 64 | _ = i * i 65 | } 66 | } 67 | 68 | func doMemoryWork() { 69 | data := make([]byte, 10<<20) 70 | _ = data 71 | } 72 | -------------------------------------------------------------------------------- /utils/queue.go: -------------------------------------------------------------------------------- 1 | // NOTE: CODE BASE IS COPIED FROM https://github.com/eapache/queue/blob/main/v2/queue.go, modified to make it thread safe 2 | 3 | package utils 4 | 5 | import ( 6 | "sync" 7 | ) 8 | 9 | // minQueueLen is smallest capacity that queue may have. 10 | // Must be power of 2 for bitwise modulus: x % n == x & (n - 1). 11 | const minQueueLen = 16 12 | 13 | // Queue represents a single instance of the queue data structure. 14 | type Queue[V any] struct { 15 | buf []*V 16 | head, tail, count int 17 | rw *sync.RWMutex 18 | tsafe bool 19 | } 20 | 21 | // New constructs and returns a new Queue. 22 | func NewQueue[V any](threadSafe bool) *Queue[V] { 23 | return &Queue[V]{ 24 | buf: make([]*V, minQueueLen), 25 | rw: &sync.RWMutex{}, 26 | tsafe: threadSafe, 27 | } 28 | } 29 | 30 | // Length returns the number of elements currently stored in the queue. 31 | func (q *Queue[V]) Len() int { 32 | if q.tsafe { 33 | q.rw.RLock() 34 | defer q.rw.RUnlock() 35 | } 36 | 37 | return q.count 38 | } 39 | 40 | // resizes the queue to fit exactly twice its current contents 41 | // this can result in shrinking if the queue is less than half-full 42 | func (q *Queue[V]) resize() { 43 | newBuf := make([]*V, q.count<<1) 44 | 45 | if q.tail > q.head { 46 | copy(newBuf, q.buf[q.head:q.tail]) 47 | } else { 48 | n := copy(newBuf, q.buf[q.head:]) 49 | copy(newBuf[n:], q.buf[:q.tail]) 50 | } 51 | 52 | q.head = 0 53 | q.tail = q.count 54 | q.buf = newBuf 55 | } 56 | 57 | // Add puts an element on the end of the queue. 58 | func (q *Queue[V]) Put(elem V) { 59 | if q.tsafe { 60 | q.rw.Lock() 61 | defer q.rw.Unlock() 62 | } 63 | 64 | if q.count == len(q.buf) { 65 | q.resize() 66 | } 67 | 68 | q.buf[q.tail] = &elem 69 | // bitwise modulus 70 | q.tail = (q.tail + 1) & (len(q.buf) - 1) 71 | q.count++ 72 | } 73 | 74 | // Top returns the element at the head of the queue. This call panics 75 | // if the queue is empty. 76 | func (q *Queue[V]) Top() V { 77 | if q.tsafe { 78 | q.rw.RLock() 79 | defer q.rw.RUnlock() 80 | } 81 | 82 | if q.count <= 0 { 83 | panic("queue: Peek() called on empty queue") 84 | } 85 | return *(q.buf[q.head]) 86 | } 87 | 88 | // Get returns the element at index i in the queue. If the index is 89 | // invalid, the call will panic. This method accepts both positive and 90 | // negative index values. Index 0 refers to the first element, and 91 | // index -1 refers to the last. 92 | func (q *Queue[V]) Get(i int) V { 93 | if q.tsafe { 94 | q.rw.RLock() 95 | defer q.rw.RUnlock() 96 | } 97 | 98 | // If indexing backwards, convert to positive index. 99 | if i < 0 { 100 | i += q.count 101 | } 102 | if i < 0 || i >= q.count { 103 | panic("queue: Get() called with index out of range") 104 | } 105 | // bitwise modulus 106 | return *(q.buf[(q.head+i)&(len(q.buf)-1)]) 107 | } 108 | 109 | // Remove removes and returns the element from the front of the queue. If the 110 | // queue is empty, the call will panic. 111 | func (q *Queue[V]) Pop() V { 112 | if q.tsafe { 113 | q.rw.Lock() 114 | defer q.rw.Unlock() 115 | } 116 | 117 | if q.count <= 0 { 118 | panic("queue: Remove() called on empty queue") 119 | } 120 | ret := q.buf[q.head] 121 | q.buf[q.head] = nil 122 | // bitwise modulus 123 | q.head = (q.head + 1) & (len(q.buf) - 1) 124 | q.count-- 125 | // Resize down if buffer 1/4 full. 126 | if len(q.buf) > minQueueLen && (q.count<<2) == len(q.buf) { 127 | q.resize() 128 | } 129 | return *ret 130 | } 131 | 132 | // Remove removes and returns the element from the front of the queue. If the 133 | // queue is empty, the call will panic. 134 | func (q *Queue[V]) TryPop() (V, bool) { 135 | if q.tsafe { 136 | q.rw.Lock() 137 | defer q.rw.Unlock() 138 | } 139 | 140 | if q.count <= 0 { 141 | var tmp V 142 | return tmp, false 143 | } 144 | ret := q.buf[q.head] 145 | q.buf[q.head] = nil 146 | // bitwise modulus 147 | q.head = (q.head + 1) & (len(q.buf) - 1) 148 | q.count-- 149 | // Resize down if buffer 1/4 full. 150 | if len(q.buf) > minQueueLen && (q.count<<2) == len(q.buf) { 151 | q.resize() 152 | } 153 | 154 | return *ret, true 155 | } 156 | -------------------------------------------------------------------------------- /utils/queue_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | NOTE: CODE BASE IS COPIED FROM https://github.com/eapache/queue/blob/main/v2/queue.go, modified to make it thread safe 3 | 4 | Package queue provides a fast, ring-buffer queue based on the version suggested by Dariusz Górecki. 5 | Using this instead of other, simpler, queue implementations (slice+append or linked list) provides 6 | substantial memory and time benefits, and fewer GC pauses. 7 | 8 | The queue implemented here is as fast as it is for an additional reason: it is *not* thread-safe. 9 | */ 10 | 11 | package utils 12 | 13 | import "testing" 14 | 15 | func TestQueueSimple(t *testing.T) { 16 | q := NewQueue[int](false) 17 | 18 | for i := 0; i < minQueueLen; i++ { 19 | q.Put(i) 20 | } 21 | for i := 0; i < minQueueLen; i++ { 22 | if q.Top() != i { 23 | t.Error("peek", i, "had value", q.Top()) 24 | } 25 | x := q.Pop() 26 | if x != i { 27 | t.Error("remove", i, "had value", x) 28 | } 29 | } 30 | } 31 | 32 | func TestQueueWrapping(t *testing.T) { 33 | q := NewQueue[int](false) 34 | 35 | for i := 0; i < minQueueLen; i++ { 36 | q.Put(i) 37 | } 38 | for i := 0; i < 3; i++ { 39 | q.Pop() 40 | q.Put(minQueueLen + i) 41 | } 42 | 43 | for i := 0; i < minQueueLen; i++ { 44 | if q.Top() != i+3 { 45 | t.Error("peek", i, "had value", q.Top()) 46 | } 47 | q.Pop() 48 | } 49 | } 50 | 51 | func TestQueueLen(t *testing.T) { 52 | q := NewQueue[int](false) 53 | 54 | if q.Len() != 0 { 55 | t.Error("empty queue length not 0") 56 | } 57 | 58 | for i := 0; i < 1000; i++ { 59 | q.Put(i) 60 | if q.Len() != i+1 { 61 | t.Error("adding: queue with", i, "elements has length", q.Len()) 62 | } 63 | } 64 | for i := 0; i < 1000; i++ { 65 | q.Pop() 66 | if q.Len() != 1000-i-1 { 67 | t.Error("removing: queue with", 1000-i-i, "elements has length", q.Len()) 68 | } 69 | } 70 | } 71 | 72 | func TestQueueGet(t *testing.T) { 73 | q := NewQueue[int](false) 74 | 75 | for i := 0; i < 1000; i++ { 76 | q.Put(i) 77 | for j := 0; j < q.Len(); j++ { 78 | if q.Get(j) != j { 79 | t.Errorf("index %d doesn't contain %d", j, j) 80 | } 81 | } 82 | } 83 | } 84 | 85 | func TestQueueGetNegative(t *testing.T) { 86 | q := NewQueue[int](false) 87 | 88 | for i := 0; i < 1000; i++ { 89 | q.Put(i) 90 | for j := 1; j <= q.Len(); j++ { 91 | if q.Get(-j) != q.Len()-j { 92 | t.Errorf("index %d doesn't contain %d", -j, q.Len()-j) 93 | } 94 | } 95 | } 96 | } 97 | 98 | func TestQueueGetOutOfRangePanics(t *testing.T) { 99 | q := NewQueue[int](false) 100 | 101 | q.Put(1) 102 | q.Put(2) 103 | q.Put(3) 104 | 105 | AssertPanics(t, "should panic when negative index", func() { 106 | q.Get(-4) 107 | }) 108 | 109 | AssertPanics(t, "should panic when index greater than length", func() { 110 | q.Get(4) 111 | }) 112 | } 113 | 114 | func TestQueuePeekOutOfRangePanics(t *testing.T) { 115 | q := NewQueue[int](false) 116 | 117 | AssertPanics(t, "should panic when peeking empty queue", func() { 118 | q.Top() 119 | }) 120 | 121 | q.Put(1) 122 | q.Pop() 123 | 124 | AssertPanics(t, "should panic when peeking emptied queue", func() { 125 | q.Top() 126 | }) 127 | } 128 | 129 | func TestQueuePopOutOfRangePanics(t *testing.T) { 130 | q := NewQueue[int](false) 131 | 132 | AssertPanics(t, "should panic when removing empty queue", func() { 133 | q.Pop() 134 | }) 135 | 136 | q.Put(1) 137 | q.Pop() 138 | 139 | AssertPanics(t, "should panic when removing emptied queue", func() { 140 | q.Pop() 141 | }) 142 | } 143 | 144 | // WARNING: Go's benchmark utility (go test -bench .) increases the number of 145 | // iterations until the benchmarks take a reasonable amount of time to run; memory usage 146 | // is *NOT* considered. On a fast CPU, these benchmarks can fill hundreds of GB of memory 147 | // (and then hang when they start to swap). You can manually control the number of iterations 148 | // with the `-benchtime` argument. Passing `-benchtime 1000000x` seems to be about right. 149 | 150 | func BenchmarkQueueSerial(b *testing.B) { 151 | q := NewQueue[int](false) 152 | for i := 0; i < b.N; i++ { 153 | q.Put(0) 154 | } 155 | for i := 0; i < b.N; i++ { 156 | q.Top() 157 | q.Pop() 158 | } 159 | } 160 | 161 | func BenchmarkQueueGet(b *testing.B) { 162 | q := NewQueue[int](false) 163 | for i := 0; i < b.N; i++ { 164 | q.Put(i) 165 | } 166 | b.ResetTimer() 167 | for i := 0; i < b.N; i++ { 168 | q.Get(i) 169 | } 170 | } 171 | 172 | func BenchmarkQueueTickTock(b *testing.B) { 173 | q := NewQueue[int](false) 174 | for i := 0; i < b.N; i++ { 175 | q.Put(0) 176 | q.Top() 177 | q.Pop() 178 | } 179 | } 180 | -------------------------------------------------------------------------------- /utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | "unsafe" 8 | ) 9 | 10 | func UnsafeToString(b []byte) string { 11 | return unsafe.String(unsafe.SliceData(b), len(b)) 12 | } 13 | 14 | func UnsafeToBytes(s string) []byte { 15 | if len(s) == 0 { 16 | return []byte{} 17 | } 18 | 19 | ptr := unsafe.StringData(s) 20 | return unsafe.Slice(ptr, len(s)) 21 | } 22 | 23 | // NormalizeDuration normalize duration 24 | func NormalizeDuration(d time.Duration) string { 25 | ns := d.Nanoseconds() 26 | hours := int(d.Hours()) 27 | minutes := int(d.Minutes()) % 60 28 | seconds := int(d.Seconds()) % 60 29 | milliseconds := int(d.Milliseconds()) % 1000 30 | microseconds := int(d.Microseconds()) % 1000 31 | nanoseconds := ns % int64(time.Microsecond) 32 | 33 | var parts []byte 34 | 35 | if hours > 0 { 36 | parts = append(parts, fmt.Sprintf("%dh", hours)...) 37 | } 38 | if minutes > 0 { 39 | parts = append(parts, fmt.Sprintf("%dm", minutes)...) 40 | } 41 | if seconds > 0 { 42 | parts = append(parts, fmt.Sprintf("%ds", seconds)...) 43 | } 44 | if milliseconds > 0 { 45 | parts = append(parts, fmt.Sprintf("%dms", milliseconds)...) 46 | } 47 | if microseconds > 0 { 48 | parts = append(parts, fmt.Sprintf("%dµs", microseconds)...) 49 | } 50 | if nanoseconds > 0 || len(parts) == 0 { 51 | parts = append(parts, fmt.Sprintf("%dns", nanoseconds)...) 52 | } 53 | 54 | return UnsafeToString(parts) 55 | } 56 | 57 | func AssertPanics(t *testing.T, name string, f func()) { 58 | defer func() { 59 | if r := recover(); r == nil { 60 | t.Errorf("%s: didn't panic as expected", name) 61 | } 62 | }() 63 | 64 | f() 65 | } 66 | -------------------------------------------------------------------------------- /utils/utils_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestUnsafeToString(t *testing.T) { 10 | original := "Hello, World!" 11 | b := []byte(original) 12 | s := UnsafeToString(b) 13 | 14 | if s != original { 15 | t.Errorf("Expected %q but got %q", original, s) 16 | } 17 | } 18 | 19 | func TestUnsafeToBytes(t *testing.T) { 20 | original := "Hello, World!" 21 | b := UnsafeToBytes(original) 22 | 23 | if string(b) != original { 24 | t.Errorf("Expected %q but got %q", original, string(b)) 25 | } 26 | } 27 | 28 | func TestPanic(t *testing.T) { 29 | f := func() { 30 | defer func() { 31 | if r := recover(); r != nil { 32 | fmt.Println("Recovered in causePanic:", r) 33 | } 34 | fmt.Println("1") 35 | }() 36 | 37 | fmt.Println("result") 38 | } 39 | f() 40 | } 41 | 42 | func TestNormalizeDuration(t *testing.T) { 43 | tests := []struct { 44 | input time.Duration 45 | expected string 46 | }{ 47 | {time.Duration(0), "0ns"}, 48 | {time.Second, "1s"}, 49 | {time.Duration(2 * time.Second), "2s"}, 50 | {time.Minute, "1m"}, 51 | {time.Duration(61 * time.Second), "1m1s"}, 52 | {time.Hour, "1h"}, 53 | {time.Duration(3601 * time.Second), "1h1s"}, 54 | {time.Duration(1*time.Hour + 30*time.Minute + 15*time.Second), "1h30m15s"}, 55 | {time.Duration(1*time.Minute + 500*time.Millisecond), "1m500ms"}, 56 | {time.Duration(500 * time.Microsecond), "500µs"}, 57 | {time.Duration(1 * time.Nanosecond), "1ns"}, 58 | } 59 | for _, test := range tests { 60 | t.Run(test.input.String(), func(t *testing.T) { 61 | result := NormalizeDuration(test.input) 62 | if result != test.expected { 63 | t.Errorf("NormalizeDuration(%v) = %v; want %v", test.input, result, test.expected) 64 | } 65 | }) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /visualizer.go: -------------------------------------------------------------------------------- 1 | package gotaskflow 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | var dot = dotVizer{} 8 | 9 | type Visualizer interface { 10 | // Visualize generate raw dag text in dot format and write to writer 11 | Visualize(tf *TaskFlow, writer io.Writer) error 12 | } 13 | -------------------------------------------------------------------------------- /visualizer_dot.go: -------------------------------------------------------------------------------- 1 | package gotaskflow 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "strings" 7 | ) 8 | 9 | type dotVizer struct{} 10 | 11 | // DotGraph represents a graph in DOT format 12 | type DotGraph struct { 13 | name string 14 | isSubgraph bool 15 | nodes map[string]*DotNode 16 | edges []*DotEdge 17 | attributes map[string]string 18 | subgraphs []*DotGraph 19 | indent string 20 | } 21 | 22 | // DotNode represents a node in DOT format 23 | type DotNode struct { 24 | id string 25 | attributes map[string]string 26 | } 27 | 28 | // DotEdge represents an edge in DOT format 29 | type DotEdge struct { 30 | from *DotNode 31 | to *DotNode 32 | attributes map[string]string 33 | } 34 | 35 | func NewDotGraph(name string) *DotGraph { 36 | return &DotGraph{ 37 | name: name, 38 | isSubgraph: false, 39 | nodes: make(map[string]*DotNode), 40 | edges: make([]*DotEdge, 0), 41 | attributes: make(map[string]string), 42 | subgraphs: make([]*DotGraph, 0), 43 | indent: "", 44 | } 45 | } 46 | 47 | func (g *DotGraph) CreateNode(name string) *DotNode { 48 | if node, exists := g.nodes[name]; exists { 49 | return node 50 | } 51 | 52 | node := &DotNode{ 53 | id: name, 54 | attributes: make(map[string]string), 55 | } 56 | g.nodes[name] = node 57 | return node 58 | } 59 | 60 | func (g *DotGraph) CreateEdge(from, to *DotNode, label string) *DotEdge { 61 | edge := &DotEdge{ 62 | from: from, 63 | to: to, 64 | attributes: make(map[string]string), 65 | } 66 | if label != "" { 67 | edge.attributes["label"] = label 68 | } 69 | g.edges = append(g.edges, edge) 70 | return edge 71 | } 72 | 73 | func (g *DotGraph) SubGraph(name string) *DotGraph { 74 | subgraph := &DotGraph{ 75 | name: name, 76 | isSubgraph: true, 77 | nodes: make(map[string]*DotNode), 78 | edges: make([]*DotEdge, 0), 79 | attributes: make(map[string]string), 80 | subgraphs: make([]*DotGraph, 0), 81 | indent: g.indent + " ", 82 | } 83 | g.subgraphs = append(g.subgraphs, subgraph) 84 | return subgraph 85 | } 86 | 87 | func (g *DotGraph) String() string { 88 | var sb strings.Builder 89 | 90 | if g.isSubgraph { 91 | sb.WriteString(g.indent + "subgraph " + quote("cluster_"+g.name) + " {\n") 92 | } else { 93 | sb.WriteString(g.indent + "digraph " + quote(g.name) + " {\n") 94 | } 95 | 96 | for k, v := range g.attributes { 97 | sb.WriteString(g.indent + " " + k + "=" + quote(v) + ";\n") 98 | } 99 | 100 | for _, node := range g.nodes { 101 | sb.WriteString(node.Format(g.indent + " ")) 102 | } 103 | 104 | for _, edge := range g.edges { 105 | sb.WriteString(edge.Format(g.indent + " ")) 106 | } 107 | 108 | for _, subgraph := range g.subgraphs { 109 | sb.WriteString(subgraph.String()) 110 | } 111 | 112 | sb.WriteString(g.indent + "}\n") 113 | return sb.String() 114 | } 115 | 116 | func (node *DotNode) Format(indent string) string { 117 | attrs := formatAttributes(node.attributes) 118 | 119 | if attrs == "" { 120 | return indent + quote(node.id) + ";\n" 121 | } 122 | 123 | return indent + quote(node.id) + " [" + attrs + "];\n" 124 | } 125 | 126 | func (edge *DotEdge) Format(indent string) string { 127 | from := edge.from.id 128 | to := edge.to.id 129 | 130 | attrs := formatAttributes(edge.attributes) 131 | 132 | if attrs == "" { 133 | return indent + quote(from) + " -> " + quote(to) + ";\n" 134 | } 135 | 136 | return indent + quote(from) + " -> " + quote(to) + " [" + attrs + "];\n" 137 | } 138 | 139 | func quote(s string) string { 140 | return "\"" + s + "\"" 141 | } 142 | 143 | func formatAttributes(attrs map[string]string) string { 144 | if len(attrs) == 0 { 145 | return "" 146 | } 147 | 148 | result := make([]string, 0, len(attrs)) 149 | for k, v := range attrs { 150 | result = append(result, k+"="+quote(v)) 151 | } 152 | return strings.Join(result, ", ") 153 | } 154 | 155 | // visualizeG recursively visualizes the graph and its subgraphs in DOT format 156 | func (v *dotVizer) visualizeG(g *eGraph, parentGraph *DotGraph) error { 157 | graph := parentGraph 158 | graph.attributes["rankdir"] = "LR" 159 | 160 | nodeMap := make(map[string]*DotNode) 161 | 162 | for _, node := range g.nodes { 163 | color := "black" 164 | if node.priority == HIGH { 165 | color = "#f5427b" 166 | } else if node.priority == LOW { 167 | color = "purple" 168 | } 169 | 170 | switch p := node.ptr.(type) { 171 | case *Static: 172 | dotNode := graph.CreateNode(node.name) 173 | dotNode.attributes["color"] = color 174 | nodeMap[node.name] = dotNode 175 | 176 | case *Condition: 177 | dotNode := graph.CreateNode(node.name) 178 | dotNode.attributes["shape"] = "diamond" 179 | dotNode.attributes["color"] = "green" 180 | nodeMap[node.name] = dotNode 181 | 182 | case *Subflow: 183 | subgraph := graph.SubGraph(node.name) 184 | subgraph.attributes["label"] = node.name 185 | subgraph.attributes["style"] = "dashed" 186 | subgraph.attributes["rankdir"] = "LR" 187 | subgraph.attributes["bgcolor"] = "#F5F5F5" 188 | subgraph.attributes["fontcolor"] = color 189 | 190 | subgraphDot := subgraph.CreateNode(node.name) 191 | subgraphDot.attributes["shape"] = "point" 192 | subgraphDot.attributes["height"] = "0.05" 193 | subgraphDot.attributes["width"] = "0.05" 194 | 195 | nodeMap[node.name] = subgraphDot 196 | 197 | err := v.visualizeG(p.g, subgraph) 198 | if err != nil { 199 | errorNodeName := "unvisualized_subflow_" + p.g.name 200 | dotNode := graph.CreateNode(errorNodeName) 201 | dotNode.attributes["color"] = "#a10212" 202 | dotNode.attributes["comment"] = "cannot visualize due to instantiate panic or failed" 203 | nodeMap[node.name] = dotNode 204 | } 205 | } 206 | } 207 | 208 | for _, node := range g.nodes { 209 | for idx, deps := range node.successors { 210 | if from, ok := nodeMap[node.name]; ok { 211 | if to, ok := nodeMap[deps.name]; ok { 212 | label := "" 213 | style := "solid" 214 | if _, ok := node.ptr.(*Condition); ok { 215 | label = fmt.Sprintf("%d", idx) 216 | style = "dashed" 217 | } 218 | 219 | edge := graph.CreateEdge(from, to, label) 220 | if style != "solid" { 221 | edge.attributes["style"] = style 222 | } 223 | } 224 | } 225 | } 226 | } 227 | 228 | return nil 229 | } 230 | 231 | // Visualize generates raw dag text in dot format and writes to writer 232 | func (v *dotVizer) Visualize(tf *TaskFlow, writer io.Writer) error { 233 | graph := NewDotGraph(tf.graph.name) 234 | err := v.visualizeG(tf.graph, graph) 235 | if err != nil { 236 | return fmt.Errorf("visualize %v -> %w", tf.graph.name, err) 237 | } 238 | 239 | _, err = writer.Write([]byte(graph.String())) 240 | if err != nil { 241 | return fmt.Errorf("write dot output -> %w", err) 242 | } 243 | 244 | return nil 245 | } 246 | -------------------------------------------------------------------------------- /visualizer_dot_test.go: -------------------------------------------------------------------------------- 1 | package gotaskflow 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | "testing" 7 | ) 8 | 9 | func TestDotGraph_String(t *testing.T) { 10 | graph := NewDotGraph("test_graph") 11 | graph.attributes["rankdir"] = "LR" 12 | 13 | nodeA := graph.CreateNode("A") 14 | nodeA.attributes["color"] = "black" 15 | 16 | nodeB := graph.CreateNode("B") 17 | nodeB.attributes["shape"] = "diamond" 18 | 19 | edge := graph.CreateEdge(nodeA, nodeB, "edge_label") 20 | edge.attributes["style"] = "dashed" 21 | 22 | result := graph.String() 23 | 24 | expectedParts := []string{ 25 | `digraph "test_graph" {`, 26 | `rankdir="LR";`, 27 | `"A" [color="black"];`, 28 | `"B" [shape="diamond"];`, 29 | `"A" -> "B"`, 30 | `}`, 31 | } 32 | 33 | for _, part := range expectedParts { 34 | if !strings.Contains(result, part) { 35 | t.Errorf("Expected DOT output to contain %q, but it didn't.\nActual output:\n%s", part, result) 36 | } 37 | } 38 | } 39 | 40 | func TestDotGraph_SubGraph(t *testing.T) { 41 | graph := NewDotGraph("main_graph") 42 | 43 | nodeA := graph.CreateNode("A") 44 | nodeB := graph.CreateNode("B") 45 | graph.CreateEdge(nodeA, nodeB, "") 46 | 47 | subgraph := graph.SubGraph("sub_graph") 48 | subgraph.attributes["style"] = "dashed" 49 | 50 | nodeC := subgraph.CreateNode("C") 51 | nodeD := subgraph.CreateNode("D") 52 | subgraph.CreateEdge(nodeC, nodeD, "") 53 | 54 | result := graph.String() 55 | 56 | expectedParts := []string{ 57 | `digraph "main_graph" {`, 58 | `"A" -> "B";`, 59 | `subgraph "cluster_sub_graph" {`, 60 | `style="dashed";`, 61 | `"C";`, 62 | `"D";`, 63 | `"C" -> "D";`, 64 | `}`, 65 | } 66 | 67 | for _, part := range expectedParts { 68 | if !strings.Contains(result, part) { 69 | t.Errorf("Expected DOT output to contain %q, but it didn't.\nActual output:\n%s", part, result) 70 | } 71 | } 72 | } 73 | 74 | func TestDotVizer_Visualize(t *testing.T) { 75 | tf := NewTaskFlow("test_flow") 76 | 77 | taskA := tf.NewTask("A", func() {}) 78 | taskB := tf.NewTask("B", func() {}) 79 | taskC := tf.NewTask("C", func() {}) 80 | 81 | taskA.Precede(taskB) 82 | taskC.Precede(taskB) 83 | 84 | var buf bytes.Buffer 85 | 86 | vizer := &dotVizer{} 87 | err := vizer.Visualize(tf, &buf) 88 | 89 | if err != nil { 90 | t.Fatalf("Visualize returned an error: %v", err) 91 | } 92 | 93 | result := buf.String() 94 | 95 | expectedParts := []string{ 96 | `digraph "test_flow" {`, 97 | `rankdir="LR";`, 98 | `"A"`, 99 | `"B"`, 100 | `"C"`, 101 | `"A" -> "B"`, 102 | `"C" -> "B"`, 103 | } 104 | 105 | for _, part := range expectedParts { 106 | if !strings.Contains(result, part) { 107 | t.Errorf("Expected DOT output to contain %q, but it didn't.\nActual output:\n%s", part, result) 108 | } 109 | } 110 | } 111 | 112 | func TestDotVizer_VisualizeComplex(t *testing.T) { 113 | tf := NewTaskFlow("complex_flow") 114 | 115 | taskA := tf.NewTask("A", func() {}) 116 | taskB := tf.NewTask("B", func() {}) 117 | 118 | condTask := tf.NewCondition("cond", func() uint { return 0 }) 119 | 120 | subTask := tf.NewSubflow("sub", func(sf *Subflow) { 121 | subA := sf.NewTask("subA", func() {}) 122 | subB := sf.NewTask("subB", func() {}) 123 | subA.Precede(subB) 124 | }) 125 | 126 | taskA.Precede(taskB) 127 | taskB.Precede(condTask) 128 | condTask.Precede(subTask) 129 | 130 | var buf bytes.Buffer 131 | 132 | vizer := &dotVizer{} 133 | err := vizer.Visualize(tf, &buf) 134 | 135 | if err != nil { 136 | t.Fatalf("Visualize returned an error: %v", err) 137 | } 138 | 139 | result := buf.String() 140 | 141 | expectedParts := []string{ 142 | `digraph "complex_flow" {`, 143 | `rankdir="LR";`, 144 | `"A"`, 145 | `"B"`, 146 | `"cond" `, 147 | `subgraph "cluster_sub" {`, 148 | `"sub"`, 149 | `"A" -> "B"`, 150 | `"B" -> "cond"`, 151 | `"cond" -> "sub"`, 152 | } 153 | 154 | for _, part := range expectedParts { 155 | if !strings.Contains(result, part) { 156 | t.Errorf("Expected DOT output to contain %q, but it didn't.\nActual output:\n%s", part, result) 157 | } 158 | } 159 | } 160 | 161 | func TestDotNode_Format(t *testing.T) { 162 | node := &DotNode{ 163 | id: "test_node", 164 | attributes: make(map[string]string), 165 | } 166 | 167 | result := node.Format(" ") 168 | expected := ` "test_node";` + "\n" 169 | if result != expected { 170 | t.Errorf("Expected %q, got %q", expected, result) 171 | } 172 | 173 | node.attributes["color"] = "red" 174 | result = node.Format(" ") 175 | expected = ` "test_node" [color="red"];` + "\n" 176 | if result != expected { 177 | t.Errorf("Expected %q, got %q", expected, result) 178 | } 179 | } 180 | 181 | func TestDotEdge_Format(t *testing.T) { 182 | from := &DotNode{id: "from"} 183 | to := &DotNode{id: "to"} 184 | 185 | edge := &DotEdge{ 186 | from: from, 187 | to: to, 188 | attributes: make(map[string]string), 189 | } 190 | 191 | result := edge.Format(" ") 192 | expected := ` "from" -> "to";` + "\n" 193 | if result != expected { 194 | t.Errorf("Expected %q, got %q", expected, result) 195 | } 196 | 197 | edge.attributes["style"] = "dashed" 198 | result = edge.Format(" ") 199 | expected = ` "from" -> "to" [style="dashed"];` + "\n" 200 | if result != expected { 201 | t.Errorf("Expected %q, got %q", expected, result) 202 | } 203 | } 204 | 205 | func TestFormatAttributes(t *testing.T) { 206 | attrs := make(map[string]string) 207 | result := formatAttributes(attrs) 208 | if result != "" { 209 | t.Errorf("Expected empty string for empty attributes, got %q", result) 210 | } 211 | 212 | attrs["color"] = "red" 213 | result = formatAttributes(attrs) 214 | expected := `color="red"` 215 | if result != expected { 216 | t.Errorf("Expected %q for single attribute, got %q", expected, result) 217 | } 218 | 219 | attrs["shape"] = "box" 220 | result = formatAttributes(attrs) 221 | 222 | option1 := `color="red", shape="box"` 223 | option2 := `shape="box", color="red"` 224 | if result != option1 && result != option2 { 225 | t.Errorf("Expected either %q or %q for multiple attributes, got %q", option1, option2, result) 226 | } 227 | } 228 | 229 | func TestQuote(t *testing.T) { 230 | testCases := []struct { 231 | input string 232 | expected string 233 | }{ 234 | {"test", `"test"`}, 235 | {"", `""`}, 236 | {`"quoted"`, `""quoted""`}, 237 | } 238 | 239 | for _, tc := range testCases { 240 | result := quote(tc.input) 241 | if result != tc.expected { 242 | t.Errorf("quote(%q) = %q, expected %q", tc.input, result, tc.expected) 243 | } 244 | } 245 | } 246 | --------------------------------------------------------------------------------