├── .gitignore ├── LICENSE ├── README.md ├── data.go ├── examples ├── basic.go ├── branch.go ├── err_group.go └── loop.go ├── flow.go ├── go.mod ├── go.sum ├── loop_test.go ├── node.go ├── queue.go ├── task.go ├── vertex.go └── vertex_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | flow.iml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Sujit Baniya 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | This package provides simple graph to execute functions in a group. 3 | 4 | ## Features 5 | - Define nodes of different types: Vertex, Branch and Loop 6 | - Define branch for conditional nodes 7 | 8 | 9 | ## Installation 10 | > go get github.com/sujit-baniya/flow 11 | 12 | ## Usage 13 | 14 | ### Basic Flow 15 | ```go 16 | package main 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | "github.com/sujit-baniya/flow" 22 | ) 23 | 24 | func Message(ctx context.Context, d flow.Data) (flow.Data, error) { 25 | d.Payload = flow.Payload(fmt.Sprintf("message %s", d.Payload)) 26 | return d, nil 27 | } 28 | 29 | func Send(ctx context.Context, d flow.Data) (flow.Data, error) { 30 | d.Payload = flow.Payload(fmt.Sprintf("This is send %s", d.Payload)) 31 | return d, nil 32 | } 33 | 34 | func basicFlow() { 35 | flow1 := flow.New() 36 | flow1.AddNode("message", Message) 37 | flow1.AddNode("send", Send) 38 | flow1.Edge("message", "send") 39 | response, e := flow1.Build().Process(context.Background(), flow.Data{ 40 | Payload: flow.Payload("Payload"), 41 | }) 42 | if e != nil { 43 | panic(e) 44 | } 45 | fmt.Println(response.ToString()) 46 | } 47 | 48 | func basicRawFlow() { 49 | rawFlow := []byte(`{ 50 | "edges": [ 51 | ["message", "send"] 52 | ] 53 | }`) 54 | flow1 := flow.New(rawFlow) 55 | flow1.AddNode("message", Message) 56 | flow1.AddNode("send", Send) 57 | response, e := flow1.Process(context.Background(), flow.Data{ 58 | Payload: flow.Payload("Payload"), 59 | }) 60 | if e != nil { 61 | panic(e) 62 | } 63 | fmt.Println(response.ToString()) 64 | } 65 | 66 | func main() { 67 | basicFlow() 68 | basicRawFlow() 69 | } 70 | 71 | ``` 72 | 73 | ### Branch Flow 74 | ```go 75 | package main 76 | 77 | import ( 78 | "context" 79 | "fmt" 80 | "encoding/json" 81 | "github.com/sujit-baniya/flow" 82 | ) 83 | 84 | func GetRegistration(ctx context.Context, d flow.Data) (flow.Data, error) { 85 | return d, nil 86 | } 87 | 88 | // VerifyUser Conditional Vertex 89 | func VerifyUser(ctx context.Context, d flow.Data) (flow.Data, error) { 90 | var reg Registration 91 | d.ConvertTo(®) 92 | if _, ok := registeredEmail[reg.Email]; !ok { 93 | d.Status = "pass" 94 | } else { 95 | d.Status = "fail" 96 | } 97 | return d, nil 98 | } 99 | 100 | func CreateUser(ctx context.Context, d flow.Data) (flow.Data, error) { 101 | d.Payload = flow.Payload(fmt.Sprintf("create user %s", d.Payload)) 102 | return d, nil 103 | } 104 | 105 | func CancelRegistration(ctx context.Context, d flow.Data) (flow.Data, error) { 106 | d.Payload = flow.Payload(fmt.Sprintf("cancel user %s", d.Payload)) 107 | return d, nil 108 | } 109 | 110 | type Registration struct { 111 | Email string 112 | Password string 113 | } 114 | 115 | var registeredEmail = map[string]bool{"test@gmail.com": true} 116 | 117 | func basicRegistrationFlow() { 118 | flow1 := flow.New() 119 | flow1.AddNode("get-registration", GetRegistration) 120 | flow1.AddNode("create-user", CreateUser) 121 | flow1.AddNode("cancel-registration", CancelRegistration) 122 | flow1.AddNode("verify-user", VerifyUser) 123 | flow1.ConditionalNode("verify-user", map[string]string{ 124 | "pass": "create-user", 125 | "fail": "cancel-registration", 126 | }) 127 | flow1.Edge("get-registration", "verify-user") 128 | 129 | registration1 := Registration{ 130 | Email: "test@gmail.com", 131 | Password: "admin", 132 | } 133 | reg1, _ := json.Marshal(registration1) 134 | 135 | registration2 := Registration{ 136 | Email: "test1@gmail.com", 137 | Password: "admin", 138 | } 139 | reg2, _ := json.Marshal(registration2) 140 | response, e := flow1.Process(context.Background(), flow.Data{ 141 | Payload: reg1, 142 | }) 143 | if e != nil { 144 | panic(e) 145 | } 146 | fmt.Println(response.ToString()) 147 | response, e = flow1.Process(context.Background(), flow.Data{ 148 | Payload: reg2, 149 | }) 150 | if e != nil { 151 | panic(e) 152 | } 153 | fmt.Println(response.ToString()) 154 | } 155 | 156 | func basicRegistrationRawFlow() { 157 | rawFlow := []byte(`{ 158 | "edges": [ 159 | ["get-registration", "verify-user"] 160 | ], 161 | "branches":[ 162 | { 163 | "key": "verify-user", 164 | "conditional_nodes": { 165 | "pass": "create-user", 166 | "fail": "cancel-registration" 167 | } 168 | } 169 | ] 170 | }`) 171 | flow1 := flow.New(rawFlow) 172 | flow1.AddNode("get-registration", GetRegistration) 173 | flow1.AddNode("create-user", CreateUser) 174 | flow1.AddNode("cancel-registration", CancelRegistration) 175 | flow1.AddNode("verify-user", VerifyUser) 176 | registration1 := Registration{ 177 | Email: "test@gmail.com", 178 | Password: "admin", 179 | } 180 | reg1, _ := json.Marshal(registration1) 181 | 182 | registration2 := Registration{ 183 | Email: "test1@gmail.com", 184 | Password: "admin", 185 | } 186 | reg2, _ := json.Marshal(registration2) 187 | response, e := flow1.Process(context.Background(), flow.Data{ 188 | Payload: reg1, 189 | }) 190 | if e != nil { 191 | panic(e) 192 | } 193 | fmt.Println(response.ToString()) 194 | response, e = flow1.Process(context.Background(), flow.Data{ 195 | Payload: reg2, 196 | }) 197 | if e != nil { 198 | panic(e) 199 | } 200 | fmt.Println(response.ToString()) 201 | } 202 | 203 | func main() { 204 | basicRegistrationFlow() 205 | basicRegistrationRawFlow() 206 | } 207 | 208 | ``` 209 | 210 | ### Loop Flow 211 | ```go 212 | package main 213 | 214 | import ( 215 | "context" 216 | "fmt" 217 | "encoding/json" 218 | "github.com/sujit-baniya/flow" 219 | "strings" 220 | ) 221 | 222 | func GetSentence(ctx context.Context, d flow.Data) (flow.Data, error) { 223 | words := strings.Split(d.ToString(), ` `) 224 | bt, _ := json.Marshal(words) 225 | d.Payload = bt 226 | return d, nil 227 | } 228 | 229 | func ForEachWord(ctx context.Context, d flow.Data) (flow.Data, error) { 230 | return d, nil 231 | } 232 | 233 | func WordUpperCase(ctx context.Context, d flow.Data) (flow.Data, error) { 234 | var word string 235 | _ = json.Unmarshal(d.Payload, &word) 236 | bt, _ := json.Marshal(strings.Title(strings.ToLower(word))) 237 | d.Payload = bt 238 | return d, nil 239 | } 240 | 241 | func AppendString(ctx context.Context, d flow.Data) (flow.Data, error) { 242 | var word string 243 | _ = json.Unmarshal(d.Payload, &word) 244 | bt, _ := json.Marshal("Upper Case: " + word) 245 | d.Payload = bt 246 | return d, nil 247 | } 248 | 249 | func main() { 250 | flow1 := flow.New() 251 | flow1.AddNode("get-sentence", GetSentence) 252 | flow1.AddNode("for-each-word", ForEachWord) 253 | flow1.AddNode("upper-case", WordUpperCase) 254 | flow1.AddNode("append-string", AppendString) 255 | flow1.Loop("for-each-word", "upper-case") 256 | flow1.Edge("get-sentence", "for-each-word") 257 | flow1.Edge("upper-case", "append-string") 258 | resp, e := flow1.Process(context.Background(), flow.Data{ 259 | Payload: flow.Payload("this is a sentence"), 260 | }) 261 | if e != nil { 262 | panic(e) 263 | } 264 | fmt.Println(resp.ToString()) 265 | } 266 | 267 | ``` 268 | ## ToDo List 269 | - Implement async flow and nodes 270 | - Implement distributed nodes 271 | -------------------------------------------------------------------------------- /data.go: -------------------------------------------------------------------------------- 1 | package flow 2 | 3 | import ( 4 | "encoding/json" 5 | ) 6 | 7 | type Payload []byte 8 | 9 | type Attachment struct { 10 | Data []byte 11 | File string 12 | MimeType string 13 | } 14 | 15 | type Data struct { 16 | RequestID string `json:"request_id"` 17 | Payload Payload `json:"payload"` 18 | Status string `json:"status"` 19 | Flow string `json:"flow"` 20 | Operation string `json:"operation"` 21 | FailedReason error `json:"failed_reason"` 22 | UserID uint `json:"user_id"` 23 | TimeStamp int64 `json:"time_stamp"` 24 | Download bool `json:"download"` 25 | FileName string `json:"file_name"` 26 | Attachments []Attachment `json:"attachments"` 27 | } 28 | 29 | func (d *Data) UnmarshalBinary(data []byte) error { 30 | return json.Unmarshal(data, d) 31 | } 32 | 33 | func (d *Data) MarshalBinary() ([]byte, error) { 34 | return json.Marshal(d) 35 | } 36 | 37 | func (d *Data) ConvertTo(rs interface{}) error { 38 | return json.Unmarshal(d.Payload, rs) 39 | } 40 | 41 | func (d *Data) ToString() string { 42 | return string(d.Payload) 43 | } 44 | 45 | func (d *Data) GetStatus() string { 46 | return d.Status 47 | } 48 | 49 | func (d *Data) Log() error { 50 | return nil 51 | } 52 | 53 | func (d *Data) LogRecords(count ...int64) error { 54 | 55 | return nil 56 | } 57 | 58 | func (d *Data) logUserCount(prefix, date string, count ...int64) error { 59 | return nil 60 | } 61 | 62 | func (d *Data) logUserFlowCount(prefix, date string, count ...int64) error { 63 | return nil 64 | } 65 | 66 | func (d *Data) logUserFlowOperationCount(prefix, date string) error { 67 | return nil 68 | } 69 | 70 | func (d *Data) logUserFlowOperation(prefix string) error { 71 | return nil 72 | } 73 | -------------------------------------------------------------------------------- /examples/basic.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/sujit-baniya/flow" 7 | ) 8 | 9 | func Message(ctx context.Context, d flow.Data) (flow.Data, error) { 10 | d.Payload = flow.Payload(fmt.Sprintf("message %s", d.Payload)) 11 | return d, nil 12 | } 13 | 14 | func Send(ctx context.Context, d flow.Data) (flow.Data, error) { 15 | d.Payload = flow.Payload(fmt.Sprintf("This is send %s", d.Payload)) 16 | return d, nil 17 | } 18 | 19 | func basicNodes(flow1 *flow.Flow) { 20 | flow1.AddNode("message", Message) 21 | flow1.AddNode("send", Send) 22 | } 23 | 24 | func basicFlow() { 25 | flow1 := flow.New() 26 | basicNodes(flow1) 27 | flow1.Edge("message", "send") 28 | response, e := flow1.Build().Process(context.Background(), flow.Data{ 29 | Payload: flow.Payload("Payload"), 30 | }) 31 | if e != nil { 32 | panic(e) 33 | } 34 | fmt.Println(response.ToString()) 35 | } 36 | 37 | func basicRawFlow() { 38 | rawFlow := []byte(`{ 39 | "edges": [ 40 | ["message", "send"] 41 | ] 42 | }`) 43 | flow1 := flow.New(rawFlow) 44 | basicNodes(flow1) 45 | response, e := flow1.Process(context.Background(), flow.Data{ 46 | Payload: flow.Payload("Payload"), 47 | }) 48 | if e != nil { 49 | panic(e) 50 | } 51 | fmt.Println(response.ToString()) 52 | } 53 | 54 | func main() { 55 | basicFlow() 56 | basicRawFlow() 57 | } 58 | -------------------------------------------------------------------------------- /examples/branch.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/sujit-baniya/flow" 8 | ) 9 | 10 | func GetRegistration(ctx context.Context, d flow.Data) (flow.Data, error) { 11 | return d, nil 12 | } 13 | 14 | // Conditional Vertex 15 | func VerifyUser(ctx context.Context, d flow.Data) (flow.Data, error) { 16 | var reg Registration 17 | d.ConvertTo(®) 18 | if _, ok := registeredEmail[reg.Email]; !ok { 19 | d.Status = "pass" 20 | } else { 21 | d.Status = "fail" 22 | } 23 | return d, nil 24 | } 25 | 26 | func CreateUser(ctx context.Context, d flow.Data) (flow.Data, error) { 27 | d.Payload = flow.Payload(fmt.Sprintf("create user %s", d.Payload)) 28 | return d, nil 29 | } 30 | 31 | func CancelRegistration(ctx context.Context, d flow.Data) (flow.Data, error) { 32 | d.Payload = flow.Payload(fmt.Sprintf("cancel user %s", d.Payload)) 33 | return d, nil 34 | } 35 | 36 | func registrationNodes(flow1 *flow.Flow) { 37 | flow1.AddNode("get-registration", GetRegistration) 38 | flow1.AddNode("create-user", CreateUser) 39 | flow1.AddNode("cancel-registration", CancelRegistration) 40 | flow1.AddNode("verify-user", VerifyUser) 41 | } 42 | 43 | type Registration struct { 44 | Email string 45 | Password string 46 | } 47 | 48 | var registeredEmail = map[string]bool{"test@gmail.com": true} 49 | 50 | func basicRegistrationFlow() { 51 | flow1 := flow.New() 52 | registrationNodes(flow1) 53 | flow1.ConditionalNode("verify-user", map[string]string{ 54 | "pass": "create-user", 55 | "fail": "cancel-registration", 56 | }) 57 | flow1.Edge("get-registration", "verify-user") 58 | 59 | registration1 := Registration{ 60 | Email: "test@gmail.com", 61 | Password: "admin", 62 | } 63 | reg1, _ := json.Marshal(registration1) 64 | 65 | registration2 := Registration{ 66 | Email: "test1@gmail.com", 67 | Password: "admin", 68 | } 69 | reg2, _ := json.Marshal(registration2) 70 | response, e := flow1.Process(context.Background(), flow.Data{ 71 | Payload: reg1, 72 | }) 73 | if e != nil { 74 | panic(e) 75 | } 76 | fmt.Println(response.ToString()) 77 | response, e = flow1.Process(context.Background(), flow.Data{ 78 | Payload: reg2, 79 | }) 80 | if e != nil { 81 | panic(e) 82 | } 83 | fmt.Println(response.ToString()) 84 | } 85 | 86 | func basicRegistrationRawFlow() { 87 | rawFlow := []byte(`{ 88 | "edges": [ 89 | ["get-registration", "verify-user"] 90 | ], 91 | "branches":[ 92 | { 93 | "key": "verify-user", 94 | "conditional_nodes": { 95 | "pass": "create-user", 96 | "fail": "cancel-registration" 97 | } 98 | } 99 | ] 100 | }`) 101 | flow1 := flow.New(rawFlow) 102 | registrationNodes(flow1) 103 | registration1 := Registration{ 104 | Email: "test@gmail.com", 105 | Password: "admin", 106 | } 107 | reg1, _ := json.Marshal(registration1) 108 | 109 | registration2 := Registration{ 110 | Email: "test1@gmail.com", 111 | Password: "admin", 112 | } 113 | reg2, _ := json.Marshal(registration2) 114 | response, e := flow1.Process(context.Background(), flow.Data{ 115 | Payload: reg1, 116 | }) 117 | if e != nil { 118 | panic(e) 119 | } 120 | fmt.Println(response.ToString()) 121 | response, e = flow1.Process(context.Background(), flow.Data{ 122 | Payload: reg2, 123 | }) 124 | if e != nil { 125 | panic(e) 126 | } 127 | fmt.Println(response.ToString()) 128 | } 129 | 130 | func main() { 131 | basicRegistrationFlow() 132 | basicRegistrationRawFlow() 133 | } 134 | -------------------------------------------------------------------------------- /examples/err_group.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "crypto/md5" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | "path/filepath" 11 | 12 | "golang.org/x/sync/errgroup" 13 | ) 14 | 15 | func main() { 16 | m, err := MD5All(context.Background(), ".") 17 | if err != nil { 18 | log.Fatal(err) 19 | } 20 | 21 | for k, sum := range m { 22 | fmt.Printf("%s:\t%x\n", k, sum) 23 | } 24 | } 25 | 26 | type result struct { 27 | path string 28 | sum [md5.Size]byte 29 | } 30 | 31 | func MD5All(ctx context.Context, root string) (map[string][md5.Size]byte, error) { 32 | g, ctx := errgroup.WithContext(ctx) 33 | paths := make(chan string) 34 | 35 | g.Go(func() error { 36 | defer close(paths) 37 | return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { 38 | if err != nil { 39 | return err 40 | } 41 | if !info.Mode().IsRegular() { 42 | return nil 43 | } 44 | select { 45 | case paths <- path: 46 | case <-ctx.Done(): 47 | return ctx.Err() 48 | } 49 | return nil 50 | }) 51 | }) 52 | 53 | c := make(chan result) 54 | const numDigesters = 20 55 | for i := 0; i < numDigesters; i++ { 56 | g.Go(func() error { 57 | for path := range paths { 58 | data, err := ioutil.ReadFile(path) 59 | if err != nil { 60 | return err 61 | } 62 | select { 63 | case c <- result{path, md5.Sum(data)}: 64 | case <-ctx.Done(): 65 | return ctx.Err() 66 | } 67 | } 68 | return nil 69 | }) 70 | } 71 | go func() { 72 | g.Wait() 73 | close(c) 74 | }() 75 | 76 | m := make(map[string][md5.Size]byte) 77 | for r := range c { 78 | m[r.path] = r.sum 79 | } 80 | if err := g.Wait(); err != nil { 81 | return nil, err 82 | } 83 | return m, nil 84 | } 85 | -------------------------------------------------------------------------------- /examples/loop.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/sujit-baniya/flow" 8 | "strings" 9 | ) 10 | 11 | func GetSentence(ctx context.Context, d flow.Data) (flow.Data, error) { 12 | words := strings.Split(d.ToString(), ` `) 13 | bt, _ := json.Marshal(words) 14 | d.Payload = bt 15 | return d, nil 16 | } 17 | 18 | func ForEachWord(ctx context.Context, d flow.Data) (flow.Data, error) { 19 | return d, nil 20 | } 21 | 22 | func WordUpperCase(ctx context.Context, d flow.Data) (flow.Data, error) { 23 | var word string 24 | _ = json.Unmarshal(d.Payload, &word) 25 | d.Payload = flow.Payload(strings.ToTitle(strings.ToLower(word))) 26 | return d, nil 27 | } 28 | 29 | func AppendString(ctx context.Context, d flow.Data) (flow.Data, error) { 30 | d.Payload = flow.Payload("Upper Case: " + string(d.Payload)) 31 | fmt.Println(d.ToString()) 32 | return d, nil 33 | } 34 | 35 | func wordNodes(flow1 *flow.Flow) { 36 | flow1.AddNode("get-sentence", GetSentence) 37 | flow1.AddNode("for-each-word", ForEachWord) 38 | flow1.AddNode("upper-case", WordUpperCase) 39 | flow1.AddNode("append-string", AppendString) 40 | } 41 | 42 | func main() { 43 | flow1 := flow.New() 44 | wordNodes(flow1) 45 | flow1.Loop("for-each-word", "upper-case") 46 | flow1.Edge("get-sentence", "for-each-word") 47 | flow1.Edge("upper-case", "append-string") 48 | _, e := flow1.Process(context.Background(), flow.Data{ 49 | Payload: flow.Payload("this is a sentence"), 50 | }) 51 | if e != nil { 52 | panic(e) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /flow.go: -------------------------------------------------------------------------------- 1 | package flow 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | ) 9 | 10 | type Flow struct { 11 | Key string `json:"key"` 12 | Error error `json:"error"` 13 | Status string `json:"status"` 14 | firstNode Node 15 | lastNode Node 16 | rawNodes map[string]Handler 17 | nodes map[string]Node 18 | inVertex map[string]bool 19 | outVertex map[string]bool 20 | raw *RawFlow 21 | } 22 | 23 | type RawFlow struct { 24 | RunInBackground bool `json:"run_in_background"` 25 | ProcessOperationCount int `json:"process_operation_count"` 26 | FirstNode string `json:"first_node,omitempty"` 27 | LastNode string `json:"last_node,omitempty"` 28 | Nodes []string `json:"nodes,omitempty"` 29 | Loops [][]string `json:"loops,omitempty"` 30 | ForEach []ForEach `json:"for_each,omitempty"` 31 | Branches []Branch `json:"branches,omitempty"` 32 | Edges [][]string `json:"edges,omitempty"` 33 | } 34 | 35 | type Branch struct { 36 | Key string `json:"key"` 37 | ConditionalNodes map[string]string `json:"conditional_nodes"` 38 | } 39 | 40 | type ForEach struct { 41 | InVertex string `json:"in_vertex"` 42 | ChildVertex []string `json:"child_vertex"` 43 | } 44 | 45 | func New(raw ...Payload) *Flow { 46 | rawFlow := &RawFlow{} 47 | f := &Flow{ 48 | nodes: make(map[string]Node), 49 | inVertex: make(map[string]bool), 50 | outVertex: make(map[string]bool), 51 | rawNodes: make(map[string]Handler), 52 | raw: rawFlow, 53 | } 54 | if len(raw) == 0 { 55 | return f 56 | } 57 | err := json.Unmarshal(raw[0], &rawFlow) 58 | if err != nil { 59 | f.Error = err 60 | return f 61 | } 62 | f.raw = rawFlow 63 | return f 64 | } 65 | 66 | func NewRaw(flow *RawFlow) *Flow { 67 | return &Flow{ 68 | nodes: make(map[string]Node), 69 | inVertex: make(map[string]bool), 70 | outVertex: make(map[string]bool), 71 | rawNodes: make(map[string]Handler), 72 | raw: flow, 73 | } 74 | } 75 | 76 | func (f *Flow) Node(vertex string) *Flow { 77 | f.raw.Nodes = append(f.raw.Nodes, vertex) 78 | return f 79 | } 80 | 81 | func (f *Flow) AddNode(node string, handler Handler) *Flow { 82 | f.rawNodes[node] = handler 83 | f.Node(node) 84 | return f 85 | } 86 | 87 | func (f *Flow) Edge(inVertex, outVertex string) *Flow { 88 | f.raw.Edges = append(f.raw.Edges, []string{inVertex, outVertex}) 89 | return f 90 | } 91 | 92 | func (f *Flow) ConditionalNode(vertex string, conditions map[string]string) *Flow { 93 | branch := Branch{ 94 | Key: vertex, 95 | ConditionalNodes: conditions, 96 | } 97 | f.raw.Branches = append(f.raw.Branches, branch) 98 | return f 99 | } 100 | 101 | func (f *Flow) Loop(inVertex string, childVertex ...string) *Flow { 102 | v := []string{inVertex} 103 | v = append(v, childVertex...) 104 | f.raw.Loops = append(f.raw.Loops, v) 105 | return f 106 | } 107 | 108 | func (f *Flow) ForEach(inVertex string, childVertex ...string) *Flow { 109 | forEach := ForEach{ 110 | InVertex: inVertex, 111 | ChildVertex: childVertex, 112 | } 113 | f.raw.ForEach = append(f.raw.ForEach, forEach) 114 | return f 115 | } 116 | 117 | func (f *Flow) Process(ctx context.Context, data Data) (Data, error) { 118 | if f.Error != nil { 119 | return data, f.Error 120 | } 121 | f.Status = "PROCESSING" 122 | if f.firstNode != nil { 123 | d, err := f.firstNode.Process(ctx, data) 124 | if err != nil { 125 | return d, err 126 | } 127 | if f.lastNode != nil { 128 | return f.lastNode.Process(ctx, d) 129 | } 130 | return d, nil 131 | } 132 | t := f.Build() 133 | if t.Error != nil { 134 | return data, t.Error 135 | } 136 | if f.firstNode != nil { 137 | d, err := f.firstNode.Process(ctx, data) 138 | if err != nil { 139 | return d, err 140 | } 141 | if f.lastNode != nil { 142 | return f.lastNode.Process(ctx, d) 143 | } 144 | return d, nil 145 | } 146 | for _, n := range f.nodes { 147 | f.firstNode = n 148 | d, err := f.firstNode.Process(ctx, data) 149 | if err != nil { 150 | return d, err 151 | } 152 | if f.lastNode != nil { 153 | return f.lastNode.Process(ctx, d) 154 | } 155 | return d, nil 156 | } 157 | return data, errors.New("no edges defined") 158 | } 159 | 160 | func (f *Flow) GetType() string { 161 | return "Flow" 162 | } 163 | 164 | func (f *Flow) GetKey() string { 165 | return f.Key 166 | } 167 | 168 | func (f *Flow) AddEdge(node Node) { 169 | f.nodes[node.GetKey()] = node 170 | } 171 | 172 | func (f *Flow) WithRaw(raw *RawFlow) *Flow { 173 | f.raw = raw 174 | return f 175 | } 176 | 177 | func (f *Flow) GetNodeHandler(node string) Handler { 178 | return f.rawNodes[node] 179 | } 180 | 181 | func (f *Flow) Build() *Flow { 182 | var noNodes, noEdges bool 183 | for _, node := range f.raw.Nodes { 184 | f.addNode(node) 185 | } 186 | if len(f.raw.Edges) == 0 { 187 | noEdges = true 188 | } 189 | for _, edge := range f.raw.Edges { 190 | f.addNode(edge[0]) 191 | f.addNode(edge[1]) 192 | } 193 | if f.raw.FirstNode != "" { 194 | f.firstNode = f.nodes[f.raw.FirstNode] 195 | } 196 | if f.raw.LastNode != "" { 197 | f.lastNode = f.nodes[f.raw.LastNode] 198 | } 199 | for _, loop := range f.raw.Loops { 200 | loopHandler := f.GetNodeHandler(loop[0]) 201 | childVertexes := loop[1:] 202 | for _, v := range childVertexes { 203 | f.addNode(v) 204 | } 205 | if loopHandler != nil { 206 | f.loop(loop[0], loopHandler, childVertexes...) 207 | } 208 | } 209 | for _, branch := range f.raw.Branches { 210 | branchHandler := f.GetNodeHandler(branch.Key) 211 | if branchHandler == nil { 212 | f.Error = errors.New(fmt.Sprintf("No branch handler defined for key '%s'", branch.Key)) 213 | return f 214 | } 215 | f.conditionalNode(branch.Key, branchHandler, branch.ConditionalNodes) 216 | } 217 | 218 | for _, edge := range f.raw.Edges { 219 | f.edge(edge[0], edge[1]) 220 | } 221 | if noEdges && noNodes { 222 | f.Error = errors.New("no vertex or edges are defined") 223 | } 224 | Add(f.Key, f) 225 | return f 226 | } 227 | 228 | func (f *Flow) OperationCountByType(optType string) int { 229 | return f.raw.ProcessOperationCount 230 | } 231 | 232 | func (f *Flow) RunInBackground() bool { 233 | return f.raw.RunInBackground 234 | } 235 | 236 | func (f *Flow) addNode(node string) { 237 | handler := f.GetNodeHandler(node) 238 | if handler != nil { 239 | f.node(node, handler) 240 | } 241 | } 242 | 243 | func (f *Flow) conditionalNode(vertex string, handler Handler, conditions map[string]string) *Flow { 244 | branches := make(map[string]Node) 245 | if n, ok := f.nodes[vertex]; ok { 246 | node := n.(*Vertex) 247 | for condition, nodeKey := range conditions { 248 | f.outVertex[nodeKey] = true 249 | if n, ok := f.nodes[nodeKey]; ok { 250 | branches[condition] = n 251 | } 252 | } 253 | node.branches = branches 254 | f.nodes[vertex] = node 255 | } else { 256 | node := &Vertex{ 257 | Key: vertex, 258 | Type: "Branch", 259 | handler: handler, 260 | ConditionalNodes: conditions, 261 | } 262 | for condition, nodeKey := range conditions { 263 | f.outVertex[nodeKey] = true 264 | if n, ok := f.nodes[nodeKey]; ok { 265 | branches[condition] = n 266 | } 267 | } 268 | node.branches = branches 269 | f.nodes[vertex] = node 270 | } 271 | return f 272 | } 273 | 274 | func (f *Flow) node(vertex string, handler Handler) *Flow { 275 | if _, ok := f.nodes[vertex]; !ok { 276 | f.nodes[vertex] = &Vertex{ 277 | Key: vertex, 278 | Type: "Vertex", 279 | ConditionalNodes: make(map[string]string), 280 | handler: handler, 281 | edges: make(map[string]Node), 282 | branches: make(map[string]Node), 283 | } 284 | } 285 | return f 286 | } 287 | 288 | func (f *Flow) edge(inVertex, outVertex string) *Flow { 289 | var outNode, inNode Node 290 | var okOutNode, okInNode bool 291 | outNode, okOutNode = f.nodes[outVertex] 292 | inNode, okInNode = f.nodes[inVertex] 293 | if !okOutNode { 294 | f.Error = errors.New(fmt.Sprintf("Output Vertex with key %s doesn't exist", outVertex)) 295 | return f 296 | } 297 | if !okInNode { 298 | f.Error = errors.New(fmt.Sprintf("Input Vertex with key %s doesn't exist", inVertex)) 299 | return f 300 | } 301 | f.inVertex[inVertex] = true 302 | f.outVertex[outVertex] = true 303 | inOk := f.inVertex[inVertex] 304 | outOk := f.outVertex[inVertex] 305 | if f.firstNode == nil && inOk && !outOk { 306 | f.firstNode = f.nodes[inVertex] 307 | } 308 | if okInNode && okOutNode { 309 | inNode.AddEdge(outNode) 310 | } 311 | return f 312 | } 313 | 314 | func (f *Flow) loop(inVertex string, inHandler Handler, childVertex ...string) *Flow { 315 | childVertexes := make(map[string]Node) 316 | for _, v := range childVertex { 317 | f.outVertex[v] = true 318 | childVertexes[v] = f.nodes[v] 319 | } 320 | 321 | loop := &Vertex{ 322 | Key: inVertex, 323 | Type: "Loop", 324 | loops: childVertexes, 325 | handler: inHandler, 326 | } 327 | f.nodes[inVertex] = loop 328 | return f 329 | } 330 | 331 | var flowList = map[string]*Flow{} 332 | 333 | func Add(key string, flow *Flow) { 334 | flowList[key] = flow 335 | } 336 | 337 | func Get(key string) *Flow { 338 | return flowList[key] 339 | } 340 | 341 | func All() map[string]*Flow { 342 | return flowList 343 | } 344 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/sujit-baniya/flow 2 | 3 | go 1.18 4 | 5 | require golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8= 2 | golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 3 | -------------------------------------------------------------------------------- /loop_test.go: -------------------------------------------------------------------------------- 1 | package flow 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | func GetSentence(ctx context.Context, d Data) (Data, error) { 11 | words := strings.Split(d.ToString(), ` `) 12 | bt, _ := json.Marshal(words) 13 | d.Payload = bt 14 | return d, nil 15 | } 16 | 17 | func ForEachWord(ctx context.Context, d Data) (Data, error) { 18 | return d, nil 19 | } 20 | 21 | func WordUpperCase(ctx context.Context, d Data) (Data, error) { 22 | var word string 23 | _ = json.Unmarshal(d.Payload, &word) 24 | bt, _ := json.Marshal(strings.Title(strings.ToLower(word))) 25 | d.Payload = bt 26 | return d, nil 27 | } 28 | 29 | func AppendIP(ctx context.Context, d Data) (Data, error) { 30 | var word string 31 | _ = json.Unmarshal(d.Payload, &word) 32 | bt, _ := json.Marshal("IP: " + word) 33 | d.Payload = bt 34 | return d, nil 35 | } 36 | 37 | func AppendString(ctx context.Context, d Data) (Data, error) { 38 | var word string 39 | _ = json.Unmarshal(d.Payload, &word) 40 | bt, _ := json.Marshal("Upper Case: " + word) 41 | d.Payload = bt 42 | return d, nil 43 | } 44 | 45 | func BenchmarkFlow_Loop(b *testing.B) { 46 | flow1 := New() 47 | flow1.AddNode("get-sentence", GetSentence) 48 | flow1.AddNode("for-each-word", ForEachWord) 49 | flow1.AddNode("upper-case", WordUpperCase) 50 | flow1.AddNode("append-string", AppendString) 51 | flow1.AddNode("append-ip", AppendIP) 52 | flow1.Loop("for-each-word", "append-ip", "upper-case") 53 | flow1.Edge("get-sentence", "for-each-word") 54 | flow1.Edge("upper-case", "append-string") 55 | for i := 0; i < b.N; i++ { 56 | flow1.Process(context.Background(), Data{ 57 | Payload: Payload("this is a sentence"), 58 | }) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /node.go: -------------------------------------------------------------------------------- 1 | package flow 2 | 3 | import "context" 4 | 5 | type Node interface { 6 | Process(ctx context.Context, data Data) (Data, error) 7 | AddEdge(node Node) 8 | GetType() string 9 | GetKey() string 10 | } 11 | 12 | type Handler func(ctx context.Context, data Data) (Data, error) 13 | -------------------------------------------------------------------------------- /queue.go: -------------------------------------------------------------------------------- 1 | package flow 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "sync" 7 | "sync/atomic" 8 | "time" 9 | ) 10 | 11 | var ErrQueueShuttingDown = errors.New("queue is shutting down; new tasks are not being accepted") 12 | 13 | type Queue struct { 14 | mutex sync.Mutex 15 | name string 16 | next time.Time 17 | now func() time.Time 18 | tasks []*Task 19 | wg sync.WaitGroup 20 | 21 | accept int32 22 | shutdown chan struct{} 23 | started chan struct{} 24 | wake chan struct{} 25 | } 26 | 27 | // NewQueue Creates a new task queue. The name of the task queue is used in Prometheus 28 | // label names and must match [a-zA-Z0-9:_] (snake case is used by convention). 29 | func NewQueue(name string) *Queue { 30 | return &Queue{ 31 | name: name, 32 | now: func() time.Time { 33 | return time.Now().UTC() 34 | }, 35 | accept: 1, 36 | shutdown: make(chan struct{}), 37 | started: make(chan struct{}, 1), 38 | } 39 | } 40 | 41 | // Now Sets the function the queue will use to obtain the current time. 42 | func (q *Queue) Now(now func() time.Time) { 43 | q.now = now 44 | } 45 | 46 | // Enqueue Enqueues a task. 47 | // 48 | // An error will only be returned if the queue has been shut down. 49 | func (q *Queue) Enqueue(t *Task) error { 50 | if atomic.LoadInt32(&q.accept) == 0 { 51 | return ErrQueueShuttingDown 52 | } 53 | 54 | q.mutex.Lock() 55 | q.tasks = append(q.tasks, t) 56 | if q.wake != nil { 57 | // Runs asynchronously to avoid deadlocking if a task submits another task 58 | go func() { 59 | q.wake <- struct{}{} 60 | }() 61 | } 62 | q.mutex.Unlock() 63 | return nil 64 | } 65 | 66 | // Submit Creates and enqueues a new task, returning the new task. Note that the 67 | // caller cannot customize settings on the task without creating a race 68 | // condition; so attempting to will panic. See NewTask and (*Queue).Enqueue to 69 | // create tasks with customized options. 70 | // 71 | // An error will only be returned if the queue has been shut down. 72 | func (q *Queue) Submit(fn TaskFunc) (*Task, error) { 73 | t := NewTask(fn) 74 | t.immutable = true 75 | err := q.Enqueue(t) 76 | return t, err 77 | } 78 | 79 | // Dispatch Attempts any tasks which are due and updates the task schedule. Returns true 80 | // if there is more work to do. 81 | func (q *Queue) Dispatch(ctx context.Context) bool { 82 | next := time.Unix(1<<63-62135596801, 999999999) // "max" time 83 | now := q.now() 84 | 85 | // In order to avoid deadlocking if a task queues another task, we make a 86 | // copy of the task list and release the mutex while executing them. 87 | q.mutex.Lock() 88 | tasks := make([]*Task, len(q.tasks)) 89 | copy(tasks, q.tasks) 90 | q.mutex.Unlock() 91 | 92 | for _, task := range tasks { 93 | due := task.NextAttempt().Before(now) 94 | if due { 95 | _, _ = task.Attempt(ctx) 96 | } 97 | if !task.Done() && task.NextAttempt().Before(next) { 98 | next = task.NextAttempt() 99 | } 100 | } 101 | 102 | q.mutex.Lock() 103 | newTasks := make([]*Task, 0, len(q.tasks)) 104 | for _, task := range q.tasks { 105 | if !task.Done() { 106 | newTasks = append(newTasks, task) 107 | } 108 | } 109 | q.tasks = newTasks 110 | q.mutex.Unlock() 111 | 112 | q.next = next 113 | return len(newTasks) != 0 114 | } 115 | 116 | func (q *Queue) run(ctx context.Context) { 117 | q.mutex.Lock() 118 | if q.wake != nil { 119 | panic(errors.New("this queue is already running on another goroutine")) 120 | } 121 | 122 | q.wake = make(chan struct{}) 123 | q.mutex.Unlock() 124 | 125 | for { 126 | more := q.Dispatch(ctx) 127 | if atomic.LoadInt32(&q.accept) == 0 && !more { 128 | return 129 | } 130 | 131 | select { 132 | case <-time.After(q.next.Sub(q.now())): 133 | break 134 | case <-ctx.Done(): 135 | return 136 | case <-q.wake: 137 | break 138 | case <-q.shutdown: 139 | atomic.StoreInt32(&q.accept, 0) 140 | break 141 | } 142 | } 143 | } 144 | 145 | // Run the task queue. Blocks until the context is cancelled. 146 | func (q *Queue) Run(ctx context.Context) { 147 | select { 148 | case <-q.started: 149 | panic(errors.New("this queue is already started on another goroutine")) 150 | default: 151 | q.run(ctx) 152 | } 153 | } 154 | 155 | // Start the task queue in the background. If you wish to use the warm 156 | // shutdown feature, you must use Start, not Run. 157 | func (q *Queue) Start(ctx context.Context) { 158 | q.wg.Add(1) 159 | 160 | select { 161 | case q.started <- struct{}{}: 162 | go func() { 163 | q.run(ctx) 164 | q.wg.Done() 165 | }() 166 | default: 167 | panic(errors.New("this queue is already started on another goroutine")) 168 | } 169 | } 170 | 171 | // Shutdown Stops accepting new tasks and blocks until all already-queued tasks are 172 | // complete. The queue must have been started with Start, not Run. 173 | func (q *Queue) Shutdown() { 174 | select { 175 | case <-q.started: 176 | q.shutdown <- struct{}{} 177 | q.wg.Wait() 178 | default: 179 | panic(errors.New("attempted warm shutdown on queue which was not run with queue.Start(ctx)")) 180 | } 181 | } 182 | 183 | // Join Shuts down any number of work queues in parallel and blocks until they're 184 | // all finished. 185 | func Join(queues ...*Queue) { 186 | var wg sync.WaitGroup 187 | wg.Add(len(queues)) 188 | for _, q := range queues { 189 | go func(q *Queue) { 190 | q.Shutdown() 191 | wg.Done() 192 | }(q) 193 | } 194 | wg.Wait() 195 | } 196 | -------------------------------------------------------------------------------- /task.go: -------------------------------------------------------------------------------- 1 | package flow 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "log" 8 | "math" 9 | "math/rand" 10 | "time" 11 | ) 12 | 13 | var ( 14 | // ErrAlreadyComplete Returned when a task is attempted which was already successfully completed. 15 | ErrAlreadyComplete = errors.New("this task was already successfully completed once") 16 | 17 | // ErrDoNotReattempt If this is returned from a task function, the task shall not be re-attempted. 18 | ErrDoNotReattempt = errors.New("this task should not be re-attempted") 19 | 20 | // ErrMaxRetriesExceeded This task has been attempted too many times. 21 | ErrMaxRetriesExceeded = errors.New("the maximum retries for this task has been exceeded") 22 | 23 | // Now Set this function to influence the clock that will be used for 24 | // scheduling re-attempts. 25 | Now = func() time.Time { 26 | return time.Now().UTC() 27 | } 28 | ) 29 | 30 | type TaskFunc func(ctx context.Context) error 31 | 32 | // Task Stores state for a task which shall be or has been executed. Each task may 33 | // only be executed successfully once. 34 | type Task struct { 35 | Metadata map[string]interface{} 36 | 37 | after func(ctx context.Context, task *Task) 38 | attempts int 39 | done bool 40 | err error 41 | fn TaskFunc 42 | nextAttempt time.Time 43 | 44 | baseDuration time.Duration 45 | immutable bool 46 | jitter bool 47 | maxAttempts int 48 | maxTimeout time.Duration 49 | sleepDuration time.Duration 50 | within time.Duration 51 | } 52 | 53 | // NewTask Creates a new task for a given function. 54 | func NewTask(fn TaskFunc) *Task { 55 | return &Task{ 56 | Metadata: make(map[string]interface{}), 57 | 58 | baseDuration: time.Minute, 59 | fn: fn, 60 | jitter: true, 61 | maxAttempts: 1, 62 | maxTimeout: 30 * time.Minute, 63 | sleepDuration: time.Minute, 64 | } 65 | } 66 | 67 | // Attempt to execute this task. 68 | // 69 | // If successful, the zero time and nil are returned. 70 | // 71 | // Otherwise, the error returned from the task function is returned to the 72 | // caller. If an error is returned for which errors.Is(err, ErrDoNotReattempt) 73 | // is true, the caller should not call Attempt again. 74 | func (t *Task) Attempt(ctx context.Context) (time.Time, error) { 75 | if t.done { 76 | if t.err == nil { 77 | return time.Time{}, ErrAlreadyComplete 78 | } 79 | return time.Time{}, t.err 80 | } 81 | 82 | t.attempts += 1 83 | if t.attempts > t.maxAttempts { 84 | t.err = ErrMaxRetriesExceeded 85 | t.done = true 86 | if t.after != nil { 87 | t.after(ctx, t) 88 | t.after = nil 89 | } 90 | return time.Time{}, ErrMaxRetriesExceeded 91 | } 92 | 93 | if errors.Is(t.err, ErrDoNotReattempt) { 94 | return time.Time{}, t.err 95 | } 96 | 97 | if t.within != time.Duration(0) { 98 | var cancel context.CancelFunc 99 | ctx, cancel = context.WithTimeout(ctx, t.within) 100 | defer cancel() 101 | } 102 | 103 | func() { 104 | defer func() { 105 | if err := recover(); err != nil { 106 | t.err = fmt.Errorf("panic: %v", err) 107 | } 108 | }() 109 | t.err = t.fn(ctx) 110 | }() 111 | 112 | if t.err == nil { 113 | t.done = true 114 | if t.after != nil { 115 | t.after(ctx, t) 116 | t.after = nil 117 | } 118 | return time.Time{}, nil 119 | } 120 | 121 | if t.jitter { 122 | rand.Seed(time.Now().UnixNano()) 123 | max := int((t.sleepDuration*3 - t.baseDuration + time.Minute).Minutes()) 124 | sleep := time.Duration(rand.Intn(max))*time.Minute + t.baseDuration 125 | if t.sleepDuration = sleep; t.sleepDuration > t.maxTimeout { 126 | t.sleepDuration = t.maxTimeout 127 | } 128 | } else { 129 | sleep := math.Pow(2, float64(t.attempts)) 130 | if sleep > t.maxTimeout.Minutes() || sleep < 0 { 131 | t.sleepDuration = t.maxTimeout 132 | } else { 133 | t.sleepDuration = time.Duration(sleep) * time.Minute 134 | } 135 | } 136 | 137 | t.nextAttempt = Now().Add(t.sleepDuration) 138 | 139 | log.Printf("Attempt %d/%d failed (%v), retrying in %s", 140 | t.Attempts(), t.maxAttempts, t.err, t.sleepDuration.String()) 141 | return t.nextAttempt, t.err 142 | } 143 | 144 | // Retries Set the maximum number of retries on failure, or -1 to attempt indefinitely. 145 | // By default, tasks are not retried on failure. 146 | func (t *Task) Retries(n int) *Task { 147 | if n < -1 { 148 | panic(errors.New("invalid input to Task.Retries")) 149 | } 150 | if t.immutable { 151 | panic(errors.New("attempted to configure immutable task")) 152 | } 153 | t.maxAttempts = n 154 | return t 155 | } 156 | 157 | // MaxTimeout Sets the maximum timeout between retries, or zero to exponentially increase 158 | // the timeout indefinitely. Defaults to 30 minutes. 159 | func (t *Task) MaxTimeout(d time.Duration) *Task { 160 | if d < 0 { 161 | panic(errors.New("invalid timeout provided to Task.MaxTimeout")) 162 | } 163 | if t.immutable { 164 | panic(errors.New("attempted to configure immutable task")) 165 | } 166 | t.maxTimeout = d 167 | return t 168 | } 169 | 170 | // After Sets a function which will be executed once the task is completed, 171 | // successfully or not. The final result (nil or an error) is passed to the 172 | // callee. 173 | func (t *Task) After(fn func(ctx context.Context, task *Task)) *Task { 174 | if t.after != nil { 175 | panic(errors.New("this task already has an 'After' function assigned")) 176 | } 177 | if t.immutable { 178 | panic(errors.New("attempted to configure immutable task")) 179 | } 180 | t.after = fn 181 | return t 182 | } 183 | 184 | // Within Specifies an upper limit for the duration of each attempt. 185 | func (t *Task) Within(deadline time.Duration) *Task { 186 | if t.immutable { 187 | panic(errors.New("attempted to configure immutable task")) 188 | } 189 | t.within = deadline 190 | return t 191 | } 192 | 193 | // NotBefore Specifies the earliest possible time of the first execution. 194 | func (t *Task) NotBefore(date time.Time) *Task { 195 | if t.immutable { 196 | panic(errors.New("attempted to configure immutable task")) 197 | } 198 | t.nextAttempt = date 199 | return t 200 | } 201 | 202 | // NoJitter Specifies that randomness should not be introduced into the exponential 203 | // backoff algorithm. 204 | func (t *Task) NoJitter() *Task { 205 | t.jitter = false 206 | return t 207 | } 208 | 209 | // Result Returns the result of the task. The task must have been completed for this 210 | // to be valid. 211 | func (t *Task) Result() error { 212 | if !t.done { 213 | panic(errors.New("(*Task).Result() called on incomplete task")) 214 | } 215 | return t.err 216 | } 217 | 218 | // Attempts Returns the number of times this task has been attempted 219 | func (t *Task) Attempts() int { 220 | return t.attempts 221 | } 222 | 223 | // NextAttempt Returns the time the next attempt is scheduled for, or the zero value if it 224 | // has not been attempted before. 225 | func (t *Task) NextAttempt() time.Time { 226 | return t.nextAttempt 227 | } 228 | 229 | // Done Returns true if this task was completed, successfully or not. 230 | func (t *Task) Done() bool { 231 | return t.done 232 | } 233 | -------------------------------------------------------------------------------- /vertex.go: -------------------------------------------------------------------------------- 1 | package flow 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "reflect" 8 | 9 | "golang.org/x/sync/errgroup" 10 | ) 11 | 12 | type Vertex struct { 13 | Key string `json:"key"` 14 | Type string `json:"type"` 15 | ConditionalNodes map[string]string `json:"conditional_nodes"` 16 | handler Handler 17 | edges map[string]Node 18 | branches map[string]Node 19 | loops map[string]Node 20 | } 21 | 22 | func merge(map1 map[string]interface{}, map2 map[string]interface{}) map[string]interface{} { 23 | for k, m := range map2 { 24 | if _, ok := map1[k]; !ok { 25 | map1[k] = m 26 | } 27 | } 28 | return map1 29 | } 30 | 31 | func (v *Vertex) loop(ctx context.Context, loops map[string]Node, data Data, response Data) ([]interface{}, error) { 32 | g, ctx := errgroup.WithContext(ctx) 33 | result := make(chan interface{}) 34 | var rs, results []interface{} 35 | err := json.Unmarshal(response.Payload, &rs) 36 | if err != nil { 37 | return nil, err 38 | } 39 | for _, single := range rs { 40 | single := single 41 | g.Go(func() error { 42 | var payload []byte 43 | currentData := make(map[string]interface{}) 44 | switch s := single.(type) { 45 | case map[string]interface{}: 46 | currentData = s 47 | } 48 | if currentData != nil { 49 | payload, err = json.Marshal(currentData) 50 | if err != nil { 51 | return err 52 | } 53 | } else { 54 | payload, err = json.Marshal(single) 55 | if err != nil { 56 | return err 57 | } 58 | } 59 | dataPayload := data 60 | dataPayload.Payload = payload 61 | var responseData map[string]interface{} 62 | for _, loop := range loops { 63 | resp, err := loop.Process(ctx, dataPayload) 64 | resp.FailedReason = err 65 | if err != nil { 66 | return err 67 | } 68 | err = json.Unmarshal(resp.Payload, &responseData) 69 | if err != nil { 70 | return err 71 | } 72 | currentData = merge(currentData, responseData) 73 | } 74 | payload, err = json.Marshal(currentData) 75 | if err != nil { 76 | return err 77 | } 78 | dataPayload.Payload = payload 79 | err = json.Unmarshal(dataPayload.Payload, &single) 80 | if err != nil { 81 | return err 82 | } 83 | select { 84 | case result <- single: 85 | case <-ctx.Done(): 86 | return ctx.Err() 87 | } 88 | return nil 89 | }) 90 | } 91 | go func() { 92 | g.Wait() 93 | close(result) 94 | }() 95 | for ch := range result { 96 | results = append(results, ch) 97 | } 98 | if err := g.Wait(); err != nil { 99 | return nil, err 100 | } 101 | return results, nil 102 | } 103 | 104 | func (v *Vertex) Process(ctx context.Context, data Data) (Data, error) { 105 | if v.GetType() == "Branch" && len(v.ConditionalNodes) == 0 { 106 | return data, errors.New("required at least one condition for branch") 107 | } 108 | response, err := v.handler(ctx, data) 109 | if err != nil { 110 | return data, err 111 | } 112 | if v.Type == "Loop" { 113 | result, err := v.loop(ctx, v.loops, data, response) 114 | if err != nil { 115 | return data, err 116 | } 117 | tmp, err := json.Marshal(result) 118 | if err != nil { 119 | return data, err 120 | } 121 | response.Payload = tmp 122 | } 123 | if val, ok := v.branches[response.GetStatus()]; ok { 124 | response, err = val.Process(ctx, response) 125 | response.FailedReason = err 126 | } 127 | for _, edge := range v.edges { 128 | response, err = edge.Process(ctx, response) 129 | response.FailedReason = err 130 | if err != nil { 131 | return data, err 132 | } 133 | } 134 | 135 | return response, err 136 | } 137 | 138 | func (v *Vertex) GetType() string { 139 | return v.Type 140 | } 141 | 142 | func (v *Vertex) GetKey() string { 143 | return v.Key 144 | } 145 | 146 | func (v *Vertex) AddEdge(node Node) { 147 | if v.edges == nil { 148 | v.edges = make(map[string]Node) 149 | } 150 | v.edges[node.GetKey()] = node 151 | } 152 | 153 | func clone(data interface{}) interface{} { 154 | if reflect.TypeOf(data).Kind() == reflect.Ptr { 155 | return reflect.New(reflect.ValueOf(data).Elem().Type()).Interface() 156 | } 157 | return reflect.New(reflect.TypeOf(data)).Elem().Interface() 158 | } 159 | -------------------------------------------------------------------------------- /vertex_test.go: -------------------------------------------------------------------------------- 1 | package flow 2 | --------------------------------------------------------------------------------