├── .gitignore ├── 10-port-definition ├── go.mod └── definitions.go ├── 12-process-definition ├── go.mod └── definitions.go ├── 09-component-definition ├── go.mod └── definitions.go ├── 11-connection-definition ├── go.mod └── definitions.go ├── 00-adhoc ├── go.mod ├── go.sum └── main.go ├── 14-generic ├── go.mod ├── go.sum ├── main.go └── flow │ └── network.go ├── 01-adhoc-context ├── go.mod ├── go.sum └── main.go ├── 02-component ├── go.mod ├── go.sum └── main.go ├── 05-interface ├── go.mod ├── go.sum ├── network.go └── main.go ├── 03-component-connect ├── go.mod ├── go.sum └── main.go ├── 04-component-context ├── go.mod ├── go.sum ├── network.go └── main.go ├── 06-generic-processor ├── go.mod ├── go.sum ├── network.go └── main.go ├── 07-connection-chan ├── go.mod ├── go.sum ├── network.go └── main.go ├── 08-convenience-reflection ├── go.mod ├── go.sum ├── main.go └── flow │ └── graph.go ├── README.md ├── LICENSE └── 13-information-packet-definition └── definitions.go /.gitignore: -------------------------------------------------------------------------------- 1 | *.sublime-project 2 | *.sublime-workspace -------------------------------------------------------------------------------- /10-port-definition/go.mod: -------------------------------------------------------------------------------- 1 | module fbp.example 2 | 3 | go 1.17 4 | -------------------------------------------------------------------------------- /12-process-definition/go.mod: -------------------------------------------------------------------------------- 1 | module fbp.example 2 | 3 | go 1.17 4 | -------------------------------------------------------------------------------- /09-component-definition/go.mod: -------------------------------------------------------------------------------- 1 | module fbp.example 2 | 3 | go 1.17 4 | -------------------------------------------------------------------------------- /11-connection-definition/go.mod: -------------------------------------------------------------------------------- 1 | module fbp.example 2 | 3 | go 1.17 4 | -------------------------------------------------------------------------------- /00-adhoc/go.mod: -------------------------------------------------------------------------------- 1 | module fbp.example 2 | 3 | go 1.17 4 | 5 | require golang.org/x/sync v0.0.0-20210220032951-036812b2e83c 6 | -------------------------------------------------------------------------------- /14-generic/go.mod: -------------------------------------------------------------------------------- 1 | module fbp.example 2 | 3 | go 1.18 4 | 5 | require golang.org/x/sync v0.0.0-20210220032951-036812b2e83c 6 | -------------------------------------------------------------------------------- /01-adhoc-context/go.mod: -------------------------------------------------------------------------------- 1 | module fbp.example 2 | 3 | go 1.17 4 | 5 | require golang.org/x/sync v0.0.0-20210220032951-036812b2e83c 6 | -------------------------------------------------------------------------------- /02-component/go.mod: -------------------------------------------------------------------------------- 1 | module fbp.example 2 | 3 | go 1.17 4 | 5 | require golang.org/x/sync v0.0.0-20210220032951-036812b2e83c 6 | -------------------------------------------------------------------------------- /05-interface/go.mod: -------------------------------------------------------------------------------- 1 | module fbp.example 2 | 3 | go 1.17 4 | 5 | require golang.org/x/sync v0.0.0-20210220032951-036812b2e83c 6 | -------------------------------------------------------------------------------- /03-component-connect/go.mod: -------------------------------------------------------------------------------- 1 | module fbp.example 2 | 3 | go 1.17 4 | 5 | require golang.org/x/sync v0.0.0-20210220032951-036812b2e83c 6 | -------------------------------------------------------------------------------- /04-component-context/go.mod: -------------------------------------------------------------------------------- 1 | module fbp.example 2 | 3 | go 1.17 4 | 5 | require golang.org/x/sync v0.0.0-20210220032951-036812b2e83c 6 | -------------------------------------------------------------------------------- /06-generic-processor/go.mod: -------------------------------------------------------------------------------- 1 | module fbp.example 2 | 3 | go 1.17 4 | 5 | require golang.org/x/sync v0.0.0-20210220032951-036812b2e83c 6 | -------------------------------------------------------------------------------- /07-connection-chan/go.mod: -------------------------------------------------------------------------------- 1 | module fbp.example 2 | 3 | go 1.17 4 | 5 | require golang.org/x/sync v0.0.0-20210220032951-036812b2e83c 6 | -------------------------------------------------------------------------------- /08-convenience-reflection/go.mod: -------------------------------------------------------------------------------- 1 | module fbp.example 2 | 3 | go 1.17 4 | 5 | require golang.org/x/sync v0.0.0-20210220032951-036812b2e83c 6 | -------------------------------------------------------------------------------- /00-adhoc/go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= 2 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 3 | -------------------------------------------------------------------------------- /02-component/go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= 2 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 3 | -------------------------------------------------------------------------------- /05-interface/go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= 2 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 3 | -------------------------------------------------------------------------------- /14-generic/go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= 2 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 3 | -------------------------------------------------------------------------------- /01-adhoc-context/go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= 2 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 3 | -------------------------------------------------------------------------------- /07-connection-chan/go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= 2 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 3 | -------------------------------------------------------------------------------- /03-component-connect/go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= 2 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 3 | -------------------------------------------------------------------------------- /04-component-context/go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= 2 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 3 | -------------------------------------------------------------------------------- /06-generic-processor/go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= 2 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 3 | -------------------------------------------------------------------------------- /08-convenience-reflection/go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= 2 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Flow Based Programming 2 | 3 | This repository contains fragments and ideas related to Flow Based Programming. 4 | It shows different ways of implementing different features. 5 | 6 | Each example is progressively more complex, but also supports more features. 7 | Similarly, it's designed to show the design-space not necessarily the classical FBP. 8 | 9 | Currently this is mostly a brain-dump and not very coherent. -------------------------------------------------------------------------------- /04-component-context/network.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | 6 | "golang.org/x/sync/errgroup" 7 | ) 8 | 9 | type Component = func(context.Context) error 10 | 11 | type Network struct { 12 | list []Component 13 | } 14 | 15 | func (net *Network) Add(proc Component) { 16 | net.list = append(net.list, proc) 17 | } 18 | 19 | func (net *Network) Run(ctx context.Context) error { 20 | var group errgroup.Group 21 | 22 | for _, proc := range net.list { 23 | proc := proc 24 | group.Go(func() error { 25 | return proc(ctx) 26 | }) 27 | } 28 | 29 | return group.Wait() 30 | } 31 | -------------------------------------------------------------------------------- /05-interface/network.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | 6 | "golang.org/x/sync/errgroup" 7 | ) 8 | 9 | type Component interface { 10 | Name() string 11 | Run(context.Context) error 12 | } 13 | 14 | type Network struct { 15 | list []Component 16 | } 17 | 18 | func (net *Network) Add(com Component) { 19 | net.list = append(net.list, com) 20 | } 21 | 22 | func (net *Network) Run(ctx context.Context) error { 23 | var group errgroup.Group 24 | 25 | for _, com := range net.list { 26 | com := com 27 | group.Go(func() error { 28 | return com.Run(ctx) 29 | }) 30 | } 31 | 32 | return group.Wait() 33 | } 34 | -------------------------------------------------------------------------------- /07-connection-chan/network.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | 6 | "golang.org/x/sync/errgroup" 7 | ) 8 | 9 | type Component interface { 10 | Name() string 11 | Run(context.Context) error 12 | } 13 | 14 | type Network struct { 15 | list []Component 16 | } 17 | 18 | func (net *Network) Add(com Component) { 19 | net.list = append(net.list, com) 20 | } 21 | 22 | func (net *Network) Run(ctx context.Context) error { 23 | var group errgroup.Group 24 | 25 | for _, com := range net.list { 26 | com := com 27 | group.Go(func() error { 28 | return com.Run(ctx) 29 | }) 30 | } 31 | 32 | return group.Wait() 33 | } 34 | -------------------------------------------------------------------------------- /06-generic-processor/network.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | 6 | "golang.org/x/sync/errgroup" 7 | ) 8 | 9 | type Component interface { 10 | Name() string 11 | Run(context.Context) error 12 | } 13 | 14 | type Network struct { 15 | list []Component 16 | } 17 | 18 | func (net *Network) Add(com Component) { 19 | net.list = append(net.list, com) 20 | } 21 | 22 | func (net *Network) Run(ctx context.Context) error { 23 | var group errgroup.Group 24 | 25 | for _, com := range net.list { 26 | com := com 27 | group.Go(func() error { 28 | return com.Run(ctx) 29 | }) 30 | } 31 | 32 | return group.Wait() 33 | } 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /13-information-packet-definition/definitions.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "time" 4 | 5 | /* 6 | So far we have been sending strings back-and forth. 7 | However we can structure messages better. 8 | */ 9 | 10 | /* 11 | First approach would be use different types for 12 | different messages. 13 | */ 14 | 15 | type User struct { 16 | Name string 17 | } 18 | 19 | /* 20 | We could also use a structured format, something json like. 21 | 22 | These messages would be trivial to send over the network. 23 | */ 24 | 25 | type Kind byte 26 | 27 | const ( 28 | StringKind = Kind(0) 29 | IntKind = Kind(1) 30 | TreeKind = Kind(2) 31 | ) 32 | 33 | type Node struct { 34 | Kind Kind 35 | String string 36 | Int int64 37 | Children []Node 38 | } 39 | 40 | /* 41 | To add headers to messages we can use two approaches. 42 | 43 | 1. Wrapping 44 | 2. Field 45 | 3. Embedding 46 | */ 47 | 48 | // Wrapping apprach -- with generics it would allow to constrain the type. 49 | type Message struct { 50 | SentAt time.Time 51 | TTL time.Time 52 | Content interface{} // arbitrary type 53 | } 54 | 55 | // Using a field. This approach has been used extensively in https://github.com/miekg/dns/blob/master/types.go. 56 | type Header2 struct { 57 | SentAt time.Time 58 | TTL time.Time 59 | } 60 | 61 | type User2 struct { 62 | Hdr Header2 63 | Name string 64 | } 65 | 66 | // Embedding, as a varation on using a field. This can provide some convenience benefits. 67 | type Header3 struct { 68 | SentAt time.Time 69 | TTL time.Time 70 | } 71 | 72 | type User3 struct { 73 | Header3 74 | Name string 75 | } 76 | -------------------------------------------------------------------------------- /02-component/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "golang.org/x/sync/errgroup" 8 | ) 9 | 10 | /* 11 | 12 | This is pretty much the previous implementation, but with a helper to setup connections. 13 | 14 | */ 15 | 16 | func main() { 17 | // Use a errgroup for managing goroutines to avoid needing to use sync.WaitGroup etc. 18 | var processes errgroup.Group 19 | defer processes.Wait() 20 | 21 | // create components 22 | hello := &Hello{Count: 10} 23 | upper := &Upper{} 24 | printer := &Printer{} 25 | 26 | // create connections between components 27 | helloToUpper := make(chan string) 28 | hello.Out = helloToUpper 29 | upper.In = helloToUpper 30 | 31 | upperToPrinter := make(chan string) 32 | upper.Out = upperToPrinter 33 | printer.In = upperToPrinter 34 | 35 | // start components 36 | processes.Go(hello.Run) 37 | processes.Go(upper.Run) 38 | processes.Go(printer.Run) 39 | } 40 | 41 | // Hello components generates Count hellos. 42 | type Hello struct { 43 | Count int 44 | Out chan<- string 45 | } 46 | 47 | func (hello *Hello) Run() error { 48 | defer close(hello.Out) 49 | for i := 0; i < hello.Count; i++ { 50 | hello.Out <- fmt.Sprintf("Hello %d", i) 51 | } 52 | return nil 53 | } 54 | 55 | // Upper component upper-cases the strings. 56 | type Upper struct { 57 | In <-chan string 58 | Out chan<- string 59 | } 60 | 61 | func (upper *Upper) Run() error { 62 | defer close(upper.Out) 63 | for value := range upper.In { 64 | upper.Out <- strings.ToUpper(value) 65 | } 66 | return nil 67 | } 68 | 69 | // Printer prints the input values. 70 | type Printer struct { 71 | In <-chan string 72 | } 73 | 74 | func (printer *Printer) Run() error { 75 | for value := range printer.In { 76 | fmt.Println(value) 77 | } 78 | return nil 79 | } 80 | -------------------------------------------------------------------------------- /00-adhoc/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "golang.org/x/sync/errgroup" 8 | ) 9 | 10 | /* 11 | 12 | This demonstrates an ad-hoc FBP like system in Go. This approach is useful when 13 | you want a fixed pipeline in an existing system. 14 | 15 | Obviously there are a bunch of problems with this system: 16 | 17 | 1. There's no explicit notion of a network, process nor component. 18 | 2. It's easy to mess up channel usage, 19 | e.g. when one of the components never closes it's output, 20 | it leaves all other components running. 21 | 3. There's no way to change connections live. 22 | 4. There's no way to start/stop components. 23 | 5. There's no way to reconfigure components live. 24 | 6. It doesn't respond to signals (e.g. CTRL-C gracefully) 25 | 26 | It's probably much closer to CSP, but nevertheless, it demonstrates a good 27 | starting point for FBP. 28 | 29 | */ 30 | 31 | func main() { 32 | // Use a errgroup for managing goroutines to avoid needing to use sync.WaitGroup etc. 33 | var processes errgroup.Group 34 | defer processes.Wait() 35 | 36 | // Setup connections between components. 37 | upcaseIn := make(chan string) 38 | upcaseOut := make(chan string) 39 | printerIn := upcaseOut 40 | 41 | // Start input component. 42 | processes.Go(func() error { 43 | defer close(upcaseIn) 44 | for i := 0; i < 10; i++ { 45 | upcaseIn <- fmt.Sprintf("Hello %d", i) 46 | } 47 | return nil 48 | }) 49 | 50 | // Start upcase component. 51 | processes.Go(func() error { 52 | defer close(upcaseOut) 53 | for value := range upcaseIn { 54 | upcaseOut <- strings.ToUpper(value) 55 | } 56 | return nil 57 | }) 58 | 59 | // Start output component 60 | processes.Go(func() error { 61 | for value := range printerIn { 62 | fmt.Println(value) 63 | } 64 | return nil 65 | }) 66 | } 67 | -------------------------------------------------------------------------------- /03-component-connect/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "golang.org/x/sync/errgroup" 8 | ) 9 | 10 | /* 11 | 12 | This is pretty much the previous implementation, but with a helper to setup connections. 13 | 14 | */ 15 | 16 | func main() { 17 | // Use a errgroup for managing goroutines to avoid needing to use sync.WaitGroup etc. 18 | var processes errgroup.Group 19 | defer processes.Wait() 20 | 21 | // create components 22 | hello := &Hello{Count: 10} 23 | upper := &Upper{} 24 | printer := &Printer{} 25 | 26 | // create connections between components 27 | hello.Out, upper.In = StringConnection() 28 | upper.Out, printer.In = StringConnection() 29 | 30 | // start components 31 | processes.Go(hello.Run) 32 | processes.Go(upper.Run) 33 | processes.Go(printer.Run) 34 | } 35 | 36 | // StringConnection creates a new channel with input and output channels. 37 | func StringConnection() (out chan<- string, in <-chan string) { 38 | ch := make(chan string) 39 | return ch, ch 40 | } 41 | 42 | // Hello components generates Count hellos. 43 | type Hello struct { 44 | Count int 45 | Out chan<- string 46 | } 47 | 48 | func (hello *Hello) Run() error { 49 | defer close(hello.Out) 50 | for i := 0; i < hello.Count; i++ { 51 | hello.Out <- fmt.Sprintf("Hello %d", i) 52 | } 53 | return nil 54 | } 55 | 56 | // Upper component upper-cases the strings. 57 | type Upper struct { 58 | In <-chan string 59 | Out chan<- string 60 | } 61 | 62 | func (upper *Upper) Run() error { 63 | defer close(upper.Out) 64 | for value := range upper.In { 65 | upper.Out <- strings.ToUpper(value) 66 | } 67 | return nil 68 | } 69 | 70 | // Printer prints the input values. 71 | type Printer struct { 72 | In <-chan string 73 | } 74 | 75 | func (printer *Printer) Run() error { 76 | for value := range printer.In { 77 | fmt.Println(value) 78 | } 79 | return nil 80 | } 81 | -------------------------------------------------------------------------------- /01-adhoc-context/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "os/signal" 8 | "strings" 9 | 10 | "golang.org/x/sync/errgroup" 11 | ) 12 | 13 | /* 14 | 15 | This is pretty similar to 00-adhoc, however has added proper cancellation. 16 | This means you can tear-down the network with Ctrl-C and handle each component 17 | stopping gracefully. 18 | 19 | Otherwise, it has pretty similar problems as previous... and also, the code 20 | isn't as clear as before. 21 | 22 | */ 23 | 24 | func main() { 25 | // Setup cancellable context. 26 | ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) 27 | defer cancel() 28 | 29 | // Use a errgroup for managing goroutines to avoid needing to use sync.WaitGroup etc. 30 | var processes errgroup.Group 31 | defer processes.Wait() 32 | 33 | // Setup connections between components. 34 | upcaseIn := make(chan string) 35 | upcaseOut := make(chan string) 36 | printerIn := upcaseOut 37 | 38 | // Start input component. 39 | processes.Go(func() error { 40 | defer close(upcaseIn) 41 | for i := 0; i < 10; i++ { 42 | select { 43 | case <-ctx.Done(): 44 | return ctx.Err() 45 | case upcaseIn <- fmt.Sprintf("Hello %d", i): 46 | } 47 | } 48 | return nil 49 | }) 50 | 51 | // Start upcase component. 52 | processes.Go(func() error { 53 | defer close(upcaseOut) 54 | 55 | for { 56 | select { 57 | case <-ctx.Done(): 58 | return ctx.Err() 59 | case value, ok := <-upcaseIn: 60 | if !ok { 61 | return nil 62 | } 63 | 64 | value = strings.ToUpper(value) 65 | 66 | select { 67 | case <-ctx.Done(): 68 | return ctx.Err() 69 | case upcaseOut <- value: 70 | } 71 | } 72 | } 73 | }) 74 | 75 | // Start output component 76 | processes.Go(func() error { 77 | for { 78 | select { 79 | case <-ctx.Done(): 80 | return ctx.Err() 81 | case value, ok := <-printerIn: 82 | if !ok { 83 | return nil 84 | } 85 | fmt.Println(value) 86 | } 87 | } 88 | }) 89 | } 90 | -------------------------------------------------------------------------------- /08-convenience-reflection/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "fbp.example/flow" 8 | ) 9 | 10 | /* 11 | 12 | Now a complete change of the approach. 13 | 14 | In principle we can take ideas from the previous and combine them into 15 | a easier to use and more minimal usage. 16 | 17 | For example, this approach uses reflection heavily and has a basic DSL 18 | for defining the networks. 19 | 20 | This was one of the ideas we discussed with Samuel Lampa a long while ago 21 | in the thread https://groups.google.com/forum/#!msg/golang-nuts/vgj_d-MjUHA/T9sE64Yrcq0J. 22 | 23 | I do not recommend this approach due to the heavy use of reflection, however, 24 | it does demonstrate some of the possible ways that can be implemented. 25 | */ 26 | 27 | type Comm struct{ In, Out chan string } 28 | 29 | func main() { 30 | comm := &Comm{} 31 | graph := flow.New(comm) 32 | 33 | graph.Registry = flow.Registry{ 34 | "Split": NewSplit, 35 | "Lower": NewLower, 36 | "Upper": NewUpper, 37 | } 38 | 39 | graph.Setup(` 40 | : s Split 41 | : l Lower 42 | : u Upper 43 | 44 | $.In -> s.In 45 | s.Left -> l.In 46 | s.Right -> u.In 47 | 48 | l.Out -> $.Out 49 | u.Out -> $.Out 50 | `) 51 | graph.Start() 52 | 53 | for i := range []int{1, 2, 3, 4, 5} { 54 | comm.In <- fmt.Sprintf("Hello %v", i) 55 | } 56 | close(comm.In) 57 | 58 | for v := range comm.Out { 59 | fmt.Printf("%v\n", v) 60 | } 61 | } 62 | 63 | type Split struct{ In, Left, Right chan string } 64 | 65 | func NewSplit() flow.Node { return &Split{} } 66 | 67 | func (node *Split) Run() error { 68 | defer close(node.Left) 69 | defer close(node.Right) 70 | for v := range node.In { 71 | m := len(v) / 2 72 | node.Left <- v[:m] 73 | node.Right <- v[m:] 74 | } 75 | return nil 76 | } 77 | 78 | type Lower struct{ In, Out chan string } 79 | 80 | func NewLower() flow.Node { return &Lower{} } 81 | 82 | func (node *Lower) Run() error { 83 | defer close(node.Out) 84 | for v := range node.In { 85 | node.Out <- strings.ToLower(v) 86 | } 87 | return nil 88 | } 89 | 90 | type Upper struct{ In, Out chan string } 91 | 92 | func NewUpper() flow.Node { return &Upper{} } 93 | 94 | func (node *Upper) Run() error { 95 | defer close(node.Out) 96 | for v := range node.In { 97 | node.Out <- strings.ToUpper(v) 98 | } 99 | return nil 100 | } 101 | -------------------------------------------------------------------------------- /14-generic/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strconv" 7 | "strings" 8 | "time" 9 | 10 | "fbp.example/flow" 11 | ) 12 | 13 | /* 14 | NOTE: this code requires Go.tip (Go 1.18) 15 | 16 | This example shows a version that uses generics and implements several features: 17 | 18 | * Dynamically modifying connections. 19 | * Typesafe communication. 20 | 21 | TODO: 22 | 23 | * Stop/Start processes / components. 24 | * Connect components live. 25 | * Disconnect components live. 26 | * Multi-connect 27 | */ 28 | 29 | type Hello struct { 30 | Out flow.Out[string] 31 | 32 | count int 33 | } 34 | 35 | func (e *Hello) Run(ctx context.Context) error { 36 | for { 37 | err := e.Out.Send(ctx, "Hello " + strconv.Itoa(e.count)) 38 | if err != nil { 39 | return err 40 | } 41 | 42 | e.count++ 43 | time.Sleep(500*time.Millisecond) 44 | } 45 | } 46 | 47 | type Upper struct { 48 | In flow.In[string] 49 | Out flow.Out[string] 50 | } 51 | 52 | func (u *Upper) Run(ctx context.Context) error { 53 | for { 54 | v, err := u.In.Recv(ctx) 55 | if err != nil { 56 | return err 57 | } 58 | 59 | v = strings.ToUpper(v) 60 | 61 | err = u.Out.Send(ctx, v) 62 | if err != nil { 63 | return err 64 | } 65 | } 66 | } 67 | 68 | type Lower struct { 69 | In flow.In[string] 70 | Out flow.Out[string] 71 | } 72 | 73 | func (u *Lower) Run(ctx context.Context) error { 74 | for { 75 | v, err := u.In.Recv(ctx) 76 | if err != nil { 77 | return err 78 | } 79 | 80 | v = strings.ToLower(v) 81 | 82 | err = u.Out.Send(ctx, v) 83 | if err != nil { 84 | return err 85 | } 86 | } 87 | } 88 | type Printer[T any] struct { 89 | In flow.In[T] 90 | } 91 | 92 | func (p *Printer[T]) Run(ctx context.Context) error { 93 | for { 94 | v, err := p.In.Recv(ctx) 95 | if err != nil { 96 | return err 97 | } 98 | 99 | fmt.Println(v) 100 | } 101 | } 102 | 103 | func main() { 104 | var net flow.Network 105 | 106 | var ( 107 | hello Hello 108 | upper Upper 109 | lower Lower 110 | printer Printer[string] 111 | ) 112 | 113 | net.Add(&hello, &upper, &lower, &printer) 114 | 115 | go net.Run(context.Background()) 116 | 117 | { 118 | first := flow.Connect(&hello.Out, &upper.In) 119 | second := flow.Connect(&upper.Out, &printer.In) 120 | time.Sleep(3 * time.Second) 121 | first.Disconnect() 122 | second.Disconnect() 123 | } 124 | 125 | { 126 | first := flow.Connect(&hello.Out, &lower.In) 127 | second := flow.Connect(&lower.Out, &printer.In) 128 | time.Sleep(3 * time.Second) 129 | first.Disconnect() 130 | second.Disconnect() 131 | } 132 | } -------------------------------------------------------------------------------- /04-component-context/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "os/signal" 8 | "strings" 9 | ) 10 | 11 | /* 12 | 13 | This introduces a new concept for tracking the whole network. 14 | Similarly this adds handling of stopping the network. 15 | 16 | Currently this uses a an arbitrary func as the component, however, 17 | this probably isn't specific enough in many cases. 18 | 19 | */ 20 | 21 | func main() { 22 | // Setup cancellable context. 23 | ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) 24 | defer cancel() 25 | 26 | // Use a custom network to avoid needing to use sync.WaitGroup etc. 27 | var network Network 28 | 29 | // create components 30 | hello := &Hello{Count: 10} 31 | upper := &Upper{} 32 | printer := &Printer{} 33 | 34 | // create connections between components 35 | hello.Out, upper.In = StringConnection() 36 | upper.Out, printer.In = StringConnection() 37 | 38 | // add components 39 | network.Add(hello.Run) 40 | network.Add(upper.Run) 41 | network.Add(printer.Run) 42 | 43 | // start the network 44 | err := network.Run(ctx) 45 | if err != nil { 46 | fmt.Fprintln(os.Stderr, err) 47 | } 48 | } 49 | 50 | // StringConnection creates a new channel with input and output channels. 51 | func StringConnection() (out chan<- string, in <-chan string) { 52 | ch := make(chan string) 53 | return ch, ch 54 | } 55 | 56 | // Hello components generates Count hellos. 57 | type Hello struct { 58 | Count int 59 | Out chan<- string 60 | } 61 | 62 | func (hello *Hello) Run(ctx context.Context) error { 63 | defer close(hello.Out) 64 | for i := 0; i < hello.Count; i++ { 65 | select { 66 | case <-ctx.Done(): 67 | return ctx.Err() 68 | case hello.Out <- fmt.Sprintf("Hello %d", i): 69 | } 70 | } 71 | return nil 72 | } 73 | 74 | // Upper component upper-cases the strings. 75 | type Upper struct { 76 | In <-chan string 77 | Out chan<- string 78 | } 79 | 80 | func (upper *Upper) Run(ctx context.Context) error { 81 | defer close(upper.Out) 82 | 83 | for { 84 | select { 85 | case <-ctx.Done(): 86 | return ctx.Err() 87 | case value, ok := <-upper.In: 88 | if !ok { 89 | return nil 90 | } 91 | 92 | value = strings.ToUpper(value) 93 | 94 | select { 95 | case <-ctx.Done(): 96 | return ctx.Err() 97 | case upper.Out <- value: 98 | } 99 | } 100 | } 101 | 102 | return nil 103 | } 104 | 105 | // Printer prints the input values. 106 | type Printer struct { 107 | In <-chan string 108 | } 109 | 110 | func (printer *Printer) Run(ctx context.Context) error { 111 | for { 112 | select { 113 | case <-ctx.Done(): 114 | return ctx.Err() 115 | case value, ok := <-printer.In: 116 | if !ok { 117 | return nil 118 | } 119 | fmt.Println(value) 120 | } 121 | } 122 | return nil 123 | } 124 | -------------------------------------------------------------------------------- /05-interface/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "os/signal" 8 | "strings" 9 | ) 10 | 11 | /* 12 | 13 | Instead of using a plain `func(context.Context) error` as the component definition. 14 | It switches to using an interface: 15 | 16 | type Component interface { 17 | Name() string 18 | Run(context.Context) error 19 | } 20 | 21 | This allows to put more constraints on the component itself. 22 | */ 23 | 24 | func main() { 25 | // Setup cancellable context. 26 | ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) 27 | defer cancel() 28 | 29 | // Use a custom network to avoid needing to use sync.WaitGroup etc. 30 | var network Network 31 | 32 | // create components 33 | hello := &Hello{Count: 10} 34 | upper := &Upper{} 35 | printer := &Printer{} 36 | 37 | // create connections between components 38 | hello.Out, upper.In = StringConnection() 39 | upper.Out, printer.In = StringConnection() 40 | 41 | // add components 42 | network.Add(hello) 43 | network.Add(upper) 44 | network.Add(printer) 45 | 46 | // start the network 47 | err := network.Run(ctx) 48 | if err != nil { 49 | fmt.Fprintln(os.Stderr, err) 50 | } 51 | } 52 | 53 | // StringConnection creates a new channel with input and output channels. 54 | func StringConnection() (out chan<- string, in <-chan string) { 55 | ch := make(chan string) 56 | return ch, ch 57 | } 58 | 59 | // Hello components generates Count hellos. 60 | type Hello struct { 61 | Count int 62 | Out chan<- string 63 | } 64 | 65 | func (*Hello) Name() string { return "Hello" } 66 | 67 | func (hello *Hello) Run(ctx context.Context) error { 68 | defer close(hello.Out) 69 | for i := 0; i < hello.Count; i++ { 70 | select { 71 | case <-ctx.Done(): 72 | return ctx.Err() 73 | case hello.Out <- fmt.Sprintf("Hello %d", i): 74 | } 75 | } 76 | return nil 77 | } 78 | 79 | // Upper component upper-cases the strings. 80 | type Upper struct { 81 | In <-chan string 82 | Out chan<- string 83 | } 84 | 85 | func (*Upper) Name() string { return "Upper" } 86 | 87 | func (upper *Upper) Run(ctx context.Context) error { 88 | defer close(upper.Out) 89 | 90 | for { 91 | select { 92 | case <-ctx.Done(): 93 | return ctx.Err() 94 | case value, ok := <-upper.In: 95 | if !ok { 96 | return nil 97 | } 98 | 99 | value = strings.ToUpper(value) 100 | 101 | select { 102 | case <-ctx.Done(): 103 | return ctx.Err() 104 | case upper.Out <- value: 105 | } 106 | } 107 | } 108 | 109 | return nil 110 | } 111 | 112 | // Printer prints the input values. 113 | type Printer struct { 114 | In <-chan string 115 | } 116 | 117 | func (*Printer) Name() string { return "Printer" } 118 | 119 | func (printer *Printer) Run(ctx context.Context) error { 120 | for { 121 | select { 122 | case <-ctx.Done(): 123 | return ctx.Err() 124 | case value, ok := <-printer.In: 125 | if !ok { 126 | return nil 127 | } 128 | fmt.Println(value) 129 | } 130 | } 131 | return nil 132 | } 133 | -------------------------------------------------------------------------------- /11-connection-definition/definitions.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | /* 6 | Here we'll look at different ways of writing connections. 7 | */ 8 | 9 | /* 10 | 11 | A really good starting point for Go is using a channel. 12 | 13 | However it does have a few limitations, it doesn't support prioritization 14 | similarly it's difficult to manipulate or observe what's in the current 15 | channel at all. 16 | 17 | Similarly, channels require to have a specific length which cannot be dynamically 18 | changed. Or at least not trivially. 19 | 20 | It also doesn't keep track to which ports it's being connected to. 21 | 22 | The sends on such a channel can also become a bottleneck. A good estimate is that 23 | a send/recv pair will take ~300ns. 24 | */ 25 | 26 | func ExampleChannel() { 27 | var in <-chan string 28 | var out chan<- string 29 | 30 | connection := make(chan string) 31 | in, out = connection, connection 32 | 33 | _, _ = in, out 34 | } 35 | 36 | /* 37 | An extension to basic channel would be to add tracking the Ports. 38 | 39 | This would give us capability to cut the connection by referencing 40 | the connection. Of course, the ports would need to support it. 41 | */ 42 | 43 | type InPort1 struct { 44 | Data <-chan string 45 | } 46 | 47 | type OutPort1 struct { 48 | Data chan<- string 49 | } 50 | 51 | type Connection1 struct { 52 | From *OutPort1 53 | Data chan string 54 | To *InPort1 55 | } 56 | 57 | /* 58 | A custom SPSC / MPSC queue could be used that has better performance characteristics. 59 | 60 | This can reduce the send/recv overhead to ~40ns. 61 | 62 | For example https://github.com/loov/queue/tree/master/extqueue contains several implementations. 63 | 64 | Of course, using a custom queue wouldn't mesh well with Go channels, which is 65 | a more common way to handle communication. 66 | */ 67 | 68 | /* 69 | We could also avoid concurrency altogether or make it optional by using a 70 | more event / callback based approach. 71 | 72 | This would of course require writing components in a completely different 73 | manner. 74 | */ 75 | 76 | type Printer3 struct{} 77 | 78 | func (*Printer3) Setup(p *Process3) { 79 | p.On("IN", func(message string) { 80 | fmt.Println(message) 81 | }) 82 | } 83 | 84 | type Process3 struct{} // stub to make things work 85 | 86 | func (*Process3) On(msg string, callback func(message string)) {} 87 | 88 | /* 89 | A similar version would be to implement a Re-Actor approach instead. 90 | This approach can be much more performant compared to the concurrent version. 91 | See "Development and Deployment of Multiplayer Online Games" for more details. 92 | 93 | Of course, since it's not going to be concurrent it's going further from 94 | the ideals of FBP. 95 | */ 96 | 97 | type Printer4 struct{} 98 | type Outbox4 struct{} 99 | 100 | func (*Outbox4) Send(port string, message string) {} 101 | 102 | func (*Printer4) Handle(port, msg string, out Outbox4) { 103 | switch port { 104 | case "IN": 105 | fmt.Println(msg) 106 | out.Send("OUT", msg) // we'll repeat it to out 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /14-generic/flow/network.go: -------------------------------------------------------------------------------- 1 | package flow 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | "golang.org/x/sync/errgroup" 8 | ) 9 | 10 | type Network struct { 11 | components []Component 12 | } 13 | 14 | func (net *Network) Add(components ...Component) { 15 | net.components = append(net.components, components...) 16 | } 17 | 18 | func (net *Network) Run(ctx context.Context) error { 19 | var g errgroup.Group 20 | for _, c := range net.components { 21 | c := c 22 | g.Go(func() error { 23 | return c.Run(ctx) 24 | }) 25 | } 26 | return g.Wait() 27 | } 28 | 29 | type Component interface { 30 | Run(ctx context.Context) error 31 | } 32 | 33 | type Conn[T any] struct { 34 | from *Out[T] 35 | to *In[T] 36 | } 37 | 38 | func Connect[T any](from *Out[T], to *In[T]) *Conn[T] { 39 | conn := Conn[T]{} 40 | conn.from = from 41 | conn.to = to 42 | 43 | data := make(chan T) 44 | conn.from.swap(data) 45 | conn.to.swap(data) 46 | 47 | return &conn 48 | } 49 | 50 | func (conn *Conn[T]) Disconnect() { 51 | conn.from.swap(nil) 52 | conn.to.swap(nil) 53 | } 54 | 55 | type In[T any] struct { 56 | // TODO: support multiple inbound channels 57 | 58 | mu sync.Mutex 59 | data chan T 60 | ping chan struct{} 61 | 62 | create sync.Once 63 | } 64 | 65 | func (in *In[T]) init() { in.create.Do(func() { in.ping = make(chan struct{}) })} 66 | 67 | func (in *In[T]) swap(data chan T) { 68 | in.init() 69 | 70 | in.mu.Lock() 71 | in.data = data 72 | in.mu.Unlock() 73 | 74 | select{ 75 | case in.ping<-struct{}{}: 76 | default: 77 | } 78 | } 79 | 80 | func (in *In[T]) current() chan T { 81 | in.mu.Lock() 82 | defer in.mu.Unlock() 83 | return in.data 84 | } 85 | 86 | func (in *In[T]) Recv(ctx context.Context) (T, error) { 87 | var zero T 88 | if err := ctx.Err(); err != nil { 89 | return zero, err 90 | } 91 | in.init() 92 | 93 | for { 94 | select { 95 | case <-in.ping: 96 | default: 97 | } 98 | 99 | select { 100 | case <-ctx.Done(): 101 | return zero, ctx.Err() 102 | case v := <-in.current(): 103 | return v, nil 104 | case <-in.ping: 105 | } 106 | } 107 | } 108 | 109 | type Out[T any] struct { 110 | mu sync.Mutex 111 | data chan T 112 | ping chan struct{} 113 | 114 | create sync.Once 115 | } 116 | 117 | func (out *Out[T]) init() { out.create.Do(func() { out.ping = make(chan struct{}) })} 118 | 119 | func (out *Out[T]) swap(data chan T) { 120 | out.init() 121 | 122 | out.mu.Lock() 123 | out.data = data 124 | out.mu.Unlock() 125 | 126 | select{ 127 | case out.ping<-struct{}{}: 128 | default: 129 | } 130 | } 131 | 132 | func (out *Out[T]) current() chan T { 133 | out.mu.Lock() 134 | defer out.mu.Unlock() 135 | return out.data 136 | } 137 | 138 | func (out *Out[T]) Send(ctx context.Context, v T) error { 139 | if err := ctx.Err(); err != nil { 140 | return err 141 | } 142 | 143 | out.init() 144 | 145 | for { 146 | select { 147 | case <-out.ping: 148 | default: 149 | } 150 | 151 | select { 152 | case <-ctx.Done(): 153 | return ctx.Err() 154 | case out.current() <- v: 155 | return nil 156 | case <-out.ping: 157 | } 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /06-generic-processor/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "os/signal" 8 | "strings" 9 | ) 10 | 11 | /* 12 | 13 | If we wish to add more string processor components, each of them would end up 14 | much longer. However we can write a type such as StringProcessor for simplifying our code. 15 | 16 | This would mean that the upper implementation can be reduced to: 17 | 18 | func NewUpper() *StringProcessor { return NewStringProcessor("Upper", strings.ToUpper) } 19 | 20 | */ 21 | 22 | func main() { 23 | // Setup cancellable context. 24 | ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) 25 | defer cancel() 26 | 27 | // Use a custom network to avoid needing to use sync.WaitGroup etc. 28 | var network Network 29 | 30 | // create components 31 | hello := &Hello{Count: 10} 32 | upper := NewUpper() 33 | printer := &Printer{} 34 | 35 | // create connections between components 36 | hello.Out, upper.In = StringConnection() 37 | upper.Out, printer.In = StringConnection() 38 | 39 | // add components 40 | network.Add(hello) 41 | network.Add(upper) 42 | network.Add(printer) 43 | 44 | // start the network 45 | err := network.Run(ctx) 46 | if err != nil { 47 | fmt.Fprintln(os.Stderr, err) 48 | } 49 | } 50 | 51 | // StringConnection creates a new channel with input and output channels. 52 | func StringConnection() (out chan<- string, in <-chan string) { 53 | ch := make(chan string) 54 | return ch, ch 55 | } 56 | 57 | // Hello components generates Count hellos. 58 | type Hello struct { 59 | Count int 60 | Out chan<- string 61 | } 62 | 63 | func (*Hello) Name() string { return "Hello" } 64 | 65 | func (hello *Hello) Run(ctx context.Context) error { 66 | defer close(hello.Out) 67 | for i := 0; i < hello.Count; i++ { 68 | select { 69 | case <-ctx.Done(): 70 | return ctx.Err() 71 | case hello.Out <- fmt.Sprintf("Hello %d", i): 72 | } 73 | } 74 | return nil 75 | } 76 | 77 | func NewUpper() *StringProcessor { return NewStringProcessor("Upper", strings.ToUpper) } 78 | func NewLower() *StringProcessor { return NewStringProcessor("Lower", strings.ToLower) } 79 | 80 | // StringProcessor implements a generic component that can be used to process strings. 81 | type StringProcessor struct { 82 | In <-chan string 83 | Out chan<- string 84 | 85 | name string 86 | process func(string) string 87 | } 88 | 89 | func NewStringProcessor(name string, process func(string) string) *StringProcessor { 90 | return &StringProcessor{ 91 | name: name, 92 | process: process, 93 | } 94 | } 95 | 96 | func (p *StringProcessor) Name() string { return p.name } 97 | 98 | func (p *StringProcessor) Run(ctx context.Context) error { 99 | defer close(p.Out) 100 | 101 | for { 102 | select { 103 | case <-ctx.Done(): 104 | return ctx.Err() 105 | case value, ok := <-p.In: 106 | if !ok { 107 | return nil 108 | } 109 | 110 | if p.process != nil { 111 | value = p.process(value) 112 | } 113 | 114 | select { 115 | case <-ctx.Done(): 116 | return ctx.Err() 117 | case p.Out <- value: 118 | } 119 | } 120 | } 121 | 122 | return nil 123 | } 124 | 125 | // Printer prints the input values. 126 | type Printer struct { 127 | In <-chan string 128 | } 129 | 130 | func (*Printer) Name() string { return "Printer" } 131 | 132 | func (printer *Printer) Run(ctx context.Context) error { 133 | for { 134 | select { 135 | case <-ctx.Done(): 136 | return ctx.Err() 137 | case value, ok := <-printer.In: 138 | if !ok { 139 | return nil 140 | } 141 | fmt.Println(value) 142 | } 143 | } 144 | return nil 145 | } 146 | -------------------------------------------------------------------------------- /12-process-definition/definitions.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | /* 6 | Here we'll look at different ways of writing a process. 7 | 8 | The implementation will depend highly on how components and ports have been defined. 9 | 10 | Overall we need to make decisions: 11 | 12 | 1. who owns the ports: process or component or connection 13 | 2. who owns the data: process or component 14 | 3. is it concurrent 15 | 4. how do we stop/start the process 16 | 5. do we combine the process and component (with/without embedding) 17 | */ 18 | 19 | /* 20 | Who owns the ports, is mainly a syntactic issue. 21 | */ 22 | 23 | /* 24 | Who owns the data is a question also on how to write components. 25 | 26 | By having component own the data, it becomes obvious what data 27 | it is working on, however the component definition becomes longer. 28 | 29 | Having process own the data makes it slightly easier to inspect 30 | and the components end up being shorter to write (in some cases). 31 | */ 32 | 33 | /* 34 | Is it concurrent is a question about the performance of the system. 35 | It might seem counter-intuitive, but a concurrent system is not necessarily 36 | faster, but it can be slower due to the communication overhead. 37 | 38 | If the components are large-grained, such that they don't have to 39 | process that many information packets, then it probably doesn't make 40 | a significant difference. 41 | 42 | However, let's say you need to process 1e6 information packets per second 43 | then a communication cost (assuming 50ns per packet) would end up 44 | as 0.05second. Which would be a significant portion of the second. 45 | This isn't accounting the thread/goroutine scheduling and cache trashing 46 | that may happen with large graphs. 47 | 48 | Similarly, since much of the servers handle requests from many 49 | users. It might make sense to run a single network in a separate 50 | thread/goroutine rather than each component. It would still get the 51 | benefit of parallelism, without the communication overhead. 52 | */ 53 | 54 | /* 55 | Stopping and starting the process is a question on whether it should 56 | handle things with context.Context or some other mechanism. 57 | 58 | Similarly, how do you introduce exit points for the components. 59 | One approach would be to handle exiting with in and out ports. 60 | 61 | This makes the implementation easy, however, it creates a question on 62 | what do you do with inflight messages that's being currently processed. 63 | 64 | Alternatively a component could have a "hard-stop" and "graceful-stop" 65 | distinction, where the graceful-stop is specially handled. 66 | 67 | Using context.Context would allow nicer integration with Go, for example 68 | the components could use it to make http requests and hence have cancellable 69 | requests as well. 70 | 71 | This context.Context could be an explicit parameter or could be 72 | integrated into Process itself -- i.e. Process itself is a context.Context. 73 | 74 | However, trying to fit context.Context into the system can introduce additional 75 | complexity. If the system is short-lived then there might not be any significant 76 | benefit to it. 77 | */ 78 | 79 | /* 80 | We could flip the dependency and make component embed a process. 81 | 82 | This would mean that the network has to deal with the generic structure 83 | and interface. Similarly, it could make handling the process level control 84 | from the network more difficult. 85 | 86 | The difficulty and complexity arises, because the network needs to control processes 87 | not components. By pushing the process a level deeper, means there needs to be a way 88 | to access the internal process. Or it would need to expose all the behavior and control 89 | parts. 90 | 91 | Although this approach could allow for interesting varations where some processes 92 | are concurrent and some are reactive. 93 | */ 94 | 95 | type Process struct { 96 | In map[string]chan string 97 | } 98 | 99 | type Printer struct { 100 | Process 101 | } 102 | 103 | func (printer *Printer) Execute() { 104 | for value := range printer.In["in"] { 105 | fmt.Println(value) 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /09-component-definition/definitions.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | /* 6 | 7 | Here we'll look at different ways of writing components. 8 | 9 | We'll use `chan string` as the port, we'll look the connections 10 | in a separate folder. 11 | 12 | */ 13 | 14 | /* 15 | First of all the usual verison, you have a struct per component, 16 | where fields may define configuration. 17 | 18 | The `In` could be hooked up manually or via reflection. 19 | */ 20 | 21 | type Printer struct { 22 | In <-chan string 23 | } 24 | 25 | func (printer *Printer) Execute() { 26 | for value := range printer.In { 27 | fmt.Println(value) 28 | } 29 | } 30 | 31 | /* 32 | Then we could do the port lookup inside the component constructor: 33 | */ 34 | 35 | type Printer2 struct { 36 | in <-chan string 37 | } 38 | 39 | func NewPrinter2(p *Process) *Printer2 { 40 | return &Printer2{ 41 | in: p.In("IN"), 42 | } 43 | } 44 | 45 | func (printer *Printer2) Execute(p *Process) { 46 | for value := range printer.in { 47 | fmt.Println(value) 48 | } 49 | } 50 | 51 | /* 52 | Alternatively, it could be done as part of Execute: 53 | */ 54 | 55 | type Printer3 struct { 56 | in <-chan string 57 | } 58 | 59 | func (printer *Printer3) Execute(p *Process) { 60 | printer.in = p.In("IN") 61 | 62 | for value := range printer.in { 63 | fmt.Println(value) 64 | } 65 | } 66 | 67 | /* 68 | One common approach is to use closures to define functionality. 69 | 70 | This return the execute function. 71 | */ 72 | 73 | func Printer4(p *Process) (execute func()) { 74 | in := p.In("IN") 75 | 76 | return func() { 77 | for value := range in { 78 | fmt.Println(value) 79 | } 80 | } 81 | } 82 | 83 | /* 84 | Now via reflection it would also be possible to define components as functions 85 | and the ports as arguments. 86 | 87 | One of the issues is that with reflection it's not possible to figure out the 88 | argument names. 89 | */ 90 | 91 | func Printer5(in <-chan string) { 92 | for value := range in { 93 | fmt.Println(value) 94 | } 95 | } 96 | 97 | /* 98 | One option to capture the names is to use a struct instead. 99 | 100 | Of course, both versions using reflection will have some overhead. 101 | 102 | To gain persitence across runs it would either need to persist the arguments. 103 | */ 104 | 105 | func Printer6(port *struct { 106 | In <-chan string 107 | }) { 108 | for value := range port.In { 109 | fmt.Println(value) 110 | } 111 | } 112 | 113 | /* 114 | Although in principle it doesn't differ much from this definition 115 | that uses reflection to fill in the ports. 116 | 117 | This version would be preferred over the previous ones, because it's slightly 118 | clearer how it works. 119 | */ 120 | 121 | type Printer7 struct { 122 | In <-chan string 123 | } 124 | 125 | func (p *Printer7) Execute() { 126 | for value := range p.In { 127 | fmt.Println(value) 128 | } 129 | } 130 | 131 | /* 132 | It's also possible to treat components as just functionality and no state at all. 133 | 134 | The first example uses a `map[string]string` to hang data to the process. 135 | This is quite similar to 136 | */ 137 | 138 | func Printer8(p *Process) { 139 | in := p.In("IN") 140 | data := p.Data() 141 | for value := range in { 142 | data[value] = "found" 143 | } 144 | } 145 | 146 | /* 147 | It's also possible to treat components as just functionality and no state at all. 148 | 149 | We could also use an approach similar to sync.Pool to persist data. 150 | */ 151 | 152 | func Printer9(p *Process) { 153 | in := p.In("IN") 154 | 155 | type Data struct { 156 | Counter int 157 | } 158 | 159 | data := p.Data2("default", func() interface{} { 160 | return &Data{} 161 | }).(*Data) 162 | 163 | for value := range in { 164 | fmt.Println(value) 165 | data.Counter++ 166 | } 167 | } 168 | 169 | /* 170 | To prevent name collisions between components a tag type can be used instead 171 | of a string. 172 | */ 173 | 174 | func Printer10(p *Process) { 175 | in := p.In("IN") 176 | 177 | type Tag struct{} 178 | type Data struct { 179 | Counter int 180 | } 181 | data := p.Data2(Tag{}, func() interface{} { 182 | return &Data{} 183 | }).(*Data) 184 | 185 | for value := range in { 186 | fmt.Println(value) 187 | data.Counter++ 188 | } 189 | } 190 | 191 | /* stub to make compilation work */ 192 | 193 | type Process struct{} 194 | 195 | func (p *Process) In(name string) <-chan string { 196 | // TODO: 197 | return nil 198 | } 199 | 200 | func (p *Process) Data() map[string]string { 201 | // TODO: 202 | return nil 203 | } 204 | 205 | func (p *Process) Data2(tag interface{}, create func() interface{}) interface{} { 206 | // TODO: 207 | return nil 208 | } 209 | -------------------------------------------------------------------------------- /07-connection-chan/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "os/signal" 8 | "strings" 9 | "sync" 10 | "time" 11 | 12 | "golang.org/x/sync/errgroup" 13 | ) 14 | 15 | /* 16 | 17 | So far we haven't had an explicit idea of a Connection. 18 | 19 | We'll start one which we can reconfigure while the network is running. 20 | 21 | We'll use a separate goroutine to pump messages from one channel to another. 22 | 23 | One of the problems with this approach is that when a connection is disconnected 24 | while a message is in that "connection" it would either need to stall the disconnecting 25 | or drop the message. Dropping the message definitely seems like a bug, but so 26 | does stalling. 27 | */ 28 | 29 | func main() { 30 | // Setup cancellable context. 31 | ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) 32 | defer cancel() 33 | 34 | // Use a custom network to avoid needing to use sync.WaitGroup etc. 35 | var network Network 36 | 37 | // create components 38 | hello := NewHello(time.Second) 39 | upper := NewUpper() 40 | lower := NewLower() 41 | printer := NewPrinter() 42 | 43 | // add components to the network 44 | network.Add(hello) 45 | network.Add(upper) 46 | network.Add(lower) 47 | network.Add(printer) 48 | 49 | var group errgroup.Group 50 | group.Go(func() error { return network.Run(ctx) }) 51 | 52 | // configure the network live: 53 | group.Go(func() error { 54 | // connect both upper and lower to the printer 55 | ConnectString(upper.Out, printer.In) 56 | ConnectString(lower.Out, printer.In) 57 | 58 | // This starts changing the Hello connection between upper and lower. 59 | for { 60 | helloToUpper := ConnectString(hello.Out, upper.In) 61 | 62 | select { 63 | case <-time.After(3 * time.Second): 64 | case <-ctx.Done(): 65 | return ctx.Err() 66 | } 67 | 68 | helloToUpper.Cut() 69 | 70 | helloToLower := ConnectString(hello.Out, lower.In) 71 | select { 72 | case <-time.After(3 * time.Second): 73 | case <-ctx.Done(): 74 | return ctx.Err() 75 | } 76 | helloToLower.Cut() 77 | } 78 | return nil 79 | }) 80 | 81 | // start the network 82 | err := group.Wait() 83 | if err != nil { 84 | fmt.Fprintln(os.Stderr, err) 85 | } 86 | } 87 | 88 | // Connection pumps message from one port to another. 89 | func ConnectString(out chan string, in chan string) *StringConnection { 90 | var conn StringConnection 91 | conn.pump(out, in) 92 | return &conn 93 | } 94 | 95 | type StringConnection struct { 96 | cut sync.Once 97 | stop chan struct{} 98 | exited chan struct{} 99 | } 100 | 101 | func (conn *StringConnection) pump(out, in chan string) { 102 | conn.stop = make(chan struct{}) 103 | conn.exited = make(chan struct{}) 104 | 105 | go func() { 106 | defer close(conn.exited) 107 | 108 | for { 109 | select { 110 | case <-conn.stop: 111 | return 112 | case val := <-out: 113 | select { 114 | case <-conn.stop: 115 | // BUG: the message should be atomically moved from one 116 | // component to the other. 117 | return 118 | case in <- val: 119 | } 120 | } 121 | } 122 | }() 123 | } 124 | 125 | func (conn *StringConnection) Cut() { 126 | conn.cut.Do(func() { close(conn.stop) }) 127 | <-conn.exited 128 | } 129 | 130 | // Hello components generates Count hellos. 131 | type Hello struct { 132 | Interval time.Duration 133 | Out chan string 134 | } 135 | 136 | func NewHello(interval time.Duration) *Hello { 137 | return &Hello{ 138 | Interval: interval, 139 | Out: make(chan string), 140 | } 141 | } 142 | 143 | func (*Hello) Name() string { return "Hello" } 144 | 145 | func (hello *Hello) Run(ctx context.Context) error { 146 | for count := 0; ; count++ { 147 | select { 148 | case <-ctx.Done(): 149 | return ctx.Err() 150 | case hello.Out <- fmt.Sprintf("Hello %d", count): 151 | } 152 | 153 | select { 154 | case <-ctx.Done(): 155 | return ctx.Err() 156 | case <-time.After(hello.Interval): 157 | } 158 | } 159 | return nil 160 | } 161 | 162 | // Upper component upper-cases the strings. 163 | func NewUpper() *StringProcessor { return NewStringProcessor("Upper", strings.ToUpper) } 164 | 165 | // Lower component lower-cases the strings. 166 | func NewLower() *StringProcessor { return NewStringProcessor("Lower", strings.ToLower) } 167 | 168 | // StringProcessor implements a generic component that can be used to process strings. 169 | type StringProcessor struct { 170 | In chan string 171 | Out chan string 172 | 173 | name string 174 | process func(string) string 175 | } 176 | 177 | func NewStringProcessor(name string, process func(string) string) *StringProcessor { 178 | return &StringProcessor{ 179 | In: make(chan string), 180 | Out: make(chan string), 181 | 182 | name: name, 183 | process: process, 184 | } 185 | } 186 | 187 | func (p *StringProcessor) Name() string { return p.name } 188 | 189 | func (p *StringProcessor) Run(ctx context.Context) error { 190 | for { 191 | select { 192 | case <-ctx.Done(): 193 | return ctx.Err() 194 | case value, ok := <-p.In: 195 | if !ok { 196 | return nil 197 | } 198 | 199 | if p.process != nil { 200 | value = p.process(value) 201 | } 202 | 203 | select { 204 | case <-ctx.Done(): 205 | return ctx.Err() 206 | case p.Out <- value: 207 | } 208 | } 209 | } 210 | 211 | return nil 212 | } 213 | 214 | // Printer prints the input values. 215 | type Printer struct { 216 | In chan string 217 | } 218 | 219 | func NewPrinter() *Printer { 220 | return &Printer{ 221 | In: make(chan string), 222 | } 223 | } 224 | 225 | func (*Printer) Name() string { return "Printer" } 226 | 227 | func (printer *Printer) Run(ctx context.Context) error { 228 | for { 229 | select { 230 | case <-ctx.Done(): 231 | return ctx.Err() 232 | case value, ok := <-printer.In: 233 | if !ok { 234 | return nil 235 | } 236 | fmt.Println(value) 237 | } 238 | } 239 | return nil 240 | } 241 | -------------------------------------------------------------------------------- /10-port-definition/definitions.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | ) 7 | 8 | /* 9 | Here we'll look at different ways of writing ports. 10 | */ 11 | 12 | /* 13 | The version which most Go users would lean towrads is 14 | using channels for the configuration. 15 | 16 | There are few problems with this approach though. 17 | It doesn't allow dynamically changing the connections. 18 | 19 | Similarly handling context cancellations and component 20 | stopping is going to introduce a bunch of code. 21 | */ 22 | 23 | type Printer struct { 24 | In <-chan string 25 | } 26 | 27 | func (printer *Printer) Execute() { 28 | for value := range printer.In { 29 | fmt.Println(value) 30 | } 31 | } 32 | 33 | /* 34 | The other question about using ports is, which party is 35 | responsible for creating the ports. 36 | 37 | First, we could push such logic into the Process and 38 | let the Process track the ports. 39 | 40 | As a benefit, the component doesn't have to make the port 41 | public. 42 | */ 43 | 44 | type Printer2 struct { 45 | in <-chan string 46 | } 47 | 48 | func NewPrinter2(p *Process) *Printer2 { 49 | return &Printer2{ 50 | in: p.In("IN"), 51 | } 52 | } 53 | 54 | /* 55 | We could have the component itself do it. 56 | However, in this case it would end up introducing a lot of 57 | duplication in components. 58 | 59 | Similarly, you wouldn't be able to easily change the connections 60 | dynamically. 61 | 62 | It's going to be also confusing on who is responsible for closing 63 | the channels. This of course isn't a problem with a fixed-form network. 64 | */ 65 | 66 | type Printer3 struct { 67 | In chan string 68 | } 69 | 70 | func NewPrinter3() *Printer3 { 71 | return &Printer3{ 72 | In: make(chan string), 73 | } 74 | } 75 | 76 | /* 77 | Alternatively, the setup could be done as part of Execute, 78 | either pulling it from the process or creating one yourself. 79 | */ 80 | 81 | type Printer4 struct { 82 | in <-chan string 83 | } 84 | 85 | func (printer *Printer4) Execute(p *Process) { 86 | printer.in = p.In("IN") 87 | 88 | for value := range printer.in { 89 | fmt.Println(value) 90 | } 91 | } 92 | 93 | /* 94 | Then it's possible to do it via reflection. 95 | The process would walk over the ports and fill them in. 96 | */ 97 | 98 | type Printer5 struct { 99 | In <-chan string 100 | } 101 | 102 | func (printer *Printer5) Execute() { 103 | for value := range printer.In { 104 | fmt.Println(value) 105 | } 106 | } 107 | 108 | /* 109 | The port filling could be done via descriptors. 110 | 111 | This approach is somewhat similar to the "reflection" version, 112 | however it avoids using reflect package. 113 | 114 | This approach however allows to easily create systems where 115 | the component has dynamic number of ports. 116 | */ 117 | 118 | type Printer6 struct { 119 | in chan string 120 | } 121 | 122 | type Port6 struct { 123 | name string 124 | in *chan string 125 | } 126 | 127 | func (printer *Printer6) Ports() []Port6 { 128 | return []Port6{ 129 | {"in", &printer.in}, 130 | } 131 | } 132 | 133 | func (printer *Printer6) Execute() { 134 | for value := range printer.in { 135 | fmt.Println(value) 136 | } 137 | } 138 | 139 | /* 140 | The component could keep track ports in a separate struct. 141 | 142 | It probably isn't significantly better than previous, 143 | however it might have some specific use-cases. 144 | */ 145 | 146 | type Ports7 struct { 147 | In map[string]chan string 148 | Out map[string]chan string 149 | } 150 | 151 | type Printer7 struct { 152 | Ports7 153 | } 154 | 155 | func (printer *Printer7) Execute() { 156 | for value := range printer.In["in"] { 157 | fmt.Println(value) 158 | } 159 | } 160 | 161 | /* 162 | There are few variations on how to component stopping. 163 | 164 | First, using the context approach. 165 | While it does work, it's rather verbose. 166 | */ 167 | 168 | type Printer8 struct { 169 | In <-chan string 170 | } 171 | 172 | func (printer *Printer8) Execute(ctx context.Context) error { 173 | for { 174 | select { 175 | case <-ctx.Done(): 176 | return ctx.Err() 177 | case value := <-printer.In: 178 | fmt.Println(value) 179 | } 180 | } 181 | } 182 | 183 | /* 184 | We can help the situation by adding a wrapper type 185 | that provides context support. 186 | */ 187 | 188 | type Port9 struct { 189 | Data chan string 190 | } 191 | 192 | func (port *Port9) Recv(ctx context.Context) (value string, ok bool) { 193 | select { 194 | case <-ctx.Done(): 195 | return "", false 196 | case value, ok := <-port.Data: 197 | return value, ok 198 | } 199 | } 200 | 201 | type Printer9 struct { 202 | In *Port9 203 | } 204 | 205 | func (printer *Printer9) Execute(ctx context.Context) { 206 | for { 207 | value, ok := printer.In.Recv(ctx) 208 | if !ok { 209 | return 210 | } 211 | 212 | fmt.Println(value) 213 | } 214 | } 215 | 216 | /* 217 | Instead of using context.Context, we can 218 | provide a custom cancellation inside the process. 219 | 220 | This of course wouldn't mesh as well with the rest of Go, 221 | however, it might be beneficial for writing a framework. 222 | */ 223 | 224 | type Port10 struct { 225 | Process *Process 226 | Data chan string 227 | } 228 | 229 | func (port *Port10) Recv() (value string, ok bool) { 230 | select { 231 | case <-port.Process.Stop: 232 | return "", false 233 | case value, ok := <-port.Data: 234 | return value, ok 235 | } 236 | } 237 | 238 | /* 239 | By providing a custom port type it would be possible to allow 240 | swapping out the port. 241 | */ 242 | 243 | type Port11ChangeRequest struct { 244 | New chan string 245 | Swapped func() bool 246 | } 247 | 248 | type Port11 struct { 249 | Process *Process 250 | Data chan string 251 | Swap chan *Port11ChangeRequest 252 | } 253 | 254 | func (port *Port11) Recv() (value string, ok bool) { 255 | retry: 256 | select { 257 | case swap := <-port.Swap: 258 | port.Data = swap.New 259 | swap.Swapped() 260 | goto retry 261 | case <-port.Process.Stop: 262 | return "", false 263 | case value, ok := <-port.Data: 264 | return value, ok 265 | } 266 | } 267 | 268 | /* 269 | With all these wrapper ports, one of the problem is having properly 270 | typed messages. Currently the main approaches how to avoid the problems 271 | would be to make the channels use interface{} (or something based on it). 272 | 273 | Alternatively, implement different port types, e.g. StringInPort. 274 | 275 | However, with Go 1.18 it would be possible to write a generic port. 276 | 277 | type Port[T any] struct { 278 | Process *Process[T] 279 | Data chan T 280 | } 281 | */ 282 | 283 | /* stub to make compilation work */ 284 | 285 | type Process struct { 286 | Stop chan struct{} 287 | } 288 | 289 | func (p *Process) In(name string) <-chan string { 290 | // TODO: 291 | return nil 292 | } 293 | -------------------------------------------------------------------------------- /08-convenience-reflection/flow/graph.go: -------------------------------------------------------------------------------- 1 | package flow 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "errors" 7 | "fmt" 8 | "reflect" 9 | "regexp" 10 | "strings" 11 | "sync/atomic" 12 | ) 13 | 14 | const BufferSize = 0 15 | 16 | type Registry map[Type]MakeFn 17 | type MakeFn func() Node 18 | 19 | type Graph struct { 20 | // Comm is used for input/output from the Graph 21 | Comm interface{} 22 | // Registry contains all the constructors for nodes 23 | Registry Registry 24 | // Nodes contains created node information 25 | Nodes map[Name]Node 26 | // Ports contains all tha ports 27 | Ports map[string]*Port 28 | } 29 | 30 | type Node interface{} 31 | 32 | type Runnable interface { 33 | Run() error 34 | } 35 | 36 | type Port struct { 37 | Name string 38 | Chan reflect.Value 39 | Refs int32 40 | } 41 | 42 | func NewPort(name string, v reflect.Value) *Port { return &Port{name, v, 0} } 43 | 44 | func (c *Port) Acquire() { atomic.AddInt32(&c.Refs, 1) } 45 | func (c *Port) Release() { 46 | if atomic.AddInt32(&c.Refs, -1) == 0 { 47 | c.Chan.Close() 48 | } 49 | } 50 | 51 | func New(comm interface{}) *Graph { 52 | return &Graph{ 53 | Comm: comm, 54 | 55 | Registry: make(Registry), 56 | Nodes: make(map[Name]Node), 57 | Ports: make(map[string]*Port), 58 | } 59 | } 60 | 61 | // Setup is convenience for parsing the wiring and wiring up the graph 62 | func (g *Graph) Setup(def string) error { 63 | wiring, err := ParseWiring(def) 64 | if err != nil { 65 | return err 66 | } 67 | return g.WireUp(wiring) 68 | } 69 | 70 | func (g *Graph) WireUp(w *Wiring) error { 71 | // we add comm as a node to the graph, this simplifies all the wiring 72 | // I can use $ in the wiring and avoid multiple lookup mechanisms 73 | g.Nodes["$"] = g.Comm 74 | 75 | // create all the nodes 76 | for name, typ := range w.Decls { 77 | mk, ok := g.Registry[typ] 78 | if !ok { 79 | return fmt.Errorf("cannot create %s type %s does not exist", name, typ) 80 | } 81 | g.Nodes[name] = mk() 82 | } 83 | 84 | for _, wire := range w.Wires { 85 | // lookup the source node 86 | from, ok := g.Nodes[wire.From] 87 | if !ok { 88 | return fmt.Errorf("source node %s does not exist", wire.From) 89 | } 90 | 91 | // lookup the destination node 92 | to, ok := g.Nodes[wire.To] 93 | if !ok { 94 | return fmt.Errorf("target node %s does not exist", wire.To) 95 | } 96 | 97 | // find the actual source node struct 98 | rfrom := reflect.ValueOf(from) 99 | // deref pointers 100 | // we need to have the actual struct for the FieldByName to work 101 | for rfrom.Kind() == reflect.Ptr { 102 | rfrom = rfrom.Elem() 103 | } 104 | // same as previous, but for destination node 105 | rto := reflect.ValueOf(to) 106 | for rto.Kind() == reflect.Ptr { 107 | rto = rto.Elem() 108 | } 109 | 110 | // find the channel fields from nodes 111 | rsrc := rfrom.FieldByName(string(wire.Src)) 112 | rdst := rto.FieldByName(string(wire.Dst)) 113 | 114 | // sanity checks 115 | switch { 116 | case !rsrc.IsValid(): 117 | return fmt.Errorf("source node %s does not have port %s", wire.From, wire.Src) 118 | case rsrc.Kind() != reflect.Chan: 119 | return fmt.Errorf("source %s.%s is not a chan", wire.From, wire.Src) 120 | case !rdst.IsValid(): 121 | return fmt.Errorf("target node %s does not have port %s", wire.To, wire.Dst) 122 | case rdst.Kind() != reflect.Chan: 123 | return fmt.Errorf("target %s.%s is not a chan", wire.To, wire.Dst) 124 | } 125 | 126 | // recreate the full port names 127 | srcname := string(wire.From) + "." + string(wire.Src) 128 | dstname := string(wire.To) + "." + string(wire.Dst) 129 | 130 | var src, dst *Port 131 | // have attached the channel to the node already? 132 | if rsrc.IsNil() { 133 | // create new channel with correct type 134 | ch := reflect.MakeChan(rsrc.Type(), BufferSize) 135 | // add it to the struct 136 | rsrc.Set(ch) 137 | // create a port for it 138 | src = NewPort(srcname, ch) 139 | // add it to graph 140 | g.Ports[srcname] = src 141 | } else { 142 | // look up the port 143 | src, ok = g.Ports[srcname] 144 | // sanity check 145 | if !ok { 146 | panic("uninitialized src " + srcname) 147 | } 148 | } 149 | 150 | // have attached the channel to the node already? 151 | if rdst.IsNil() { 152 | // create new channel with correct type 153 | ch := reflect.MakeChan(rsrc.Type(), BufferSize) 154 | // add it to the struct 155 | rdst.Set(ch) 156 | // create a port for it 157 | dst = NewPort(dstname, ch) 158 | // add it to graph 159 | g.Ports[dstname] = dst 160 | } else { 161 | // look up the port 162 | dst, ok = g.Ports[dstname] 163 | // sanity check 164 | if !ok { 165 | panic("uninitialized dst " + dstname) 166 | } 167 | } 168 | 169 | // we acquire the destination port 170 | dst.Acquire() 171 | 172 | // create a copying routine 173 | go func() { 174 | // when the source finishes we release the destination port 175 | // this way when the counter hits 0 i.e. there are no more incoming 176 | // values to the In port of a node then it can be closed 177 | defer dst.Release() 178 | for { 179 | // pull out a value from the output of a node 180 | v, ok := src.Chan.Recv() 181 | if !ok { 182 | return 183 | } 184 | // put it into result 185 | dst.Chan.Send(v) 186 | } 187 | }() 188 | } 189 | return nil 190 | } 191 | 192 | // starts all the nodes 193 | func (g *Graph) Start() { 194 | for _, n := range g.Nodes { 195 | r, ok := n.(Runnable) 196 | if ok { 197 | go func() { 198 | //TODO: do something smarter with errors 199 | if err := r.Run(); err != nil { 200 | panic(err) 201 | } 202 | }() 203 | } 204 | } 205 | } 206 | 207 | func ParseWiring(def string) (*Wiring, error) { 208 | wiring := &Wiring{Decls: make(map[Name]Type)} 209 | 210 | // really stupid hacky parsing 211 | rxDecl := regexp.MustCompile(`:\s+([$a-zA-Z]+)\s+([a-zA-Z]+)`) 212 | rxPipe := regexp.MustCompile(`([\$a-zA-Z]+)\.([a-zA-Z]+)\s*->\s*([\$a-zA-Z]+)\.([a-zA-Z]+)`) 213 | 214 | line := bufio.NewScanner(bytes.NewBufferString(def)) 215 | for line.Scan() { 216 | stmt := strings.TrimSpace(line.Text()) 217 | if len(stmt) == 0 { 218 | continue 219 | } 220 | 221 | if stmt[0] == ':' { 222 | xs := rxDecl.FindAllStringSubmatch(stmt, -1) 223 | if len(xs) != 1 { 224 | return nil, errors.New("invalid line: " + stmt) 225 | } 226 | 227 | wiring.Decls[Name(xs[0][1])] = Type(xs[0][2]) 228 | } else { 229 | xs := rxPipe.FindAllStringSubmatch(stmt, -1) 230 | if len(xs) != 1 { 231 | return nil, errors.New("invalid line: " + stmt) 232 | } 233 | 234 | wiring.Wires = append(wiring.Wires, Wire{ 235 | From: Name(xs[0][1]), 236 | Src: PortName(xs[0][2]), 237 | To: Name(xs[0][3]), 238 | Dst: PortName(xs[0][4]), 239 | }) 240 | } 241 | } 242 | 243 | return wiring, nil 244 | } 245 | 246 | type Name string 247 | type Type string 248 | type PortName string 249 | 250 | type Wiring struct { 251 | Decls map[Name]Type 252 | Wires []Wire 253 | } 254 | 255 | type Wire struct { 256 | From Name 257 | Src PortName 258 | To Name 259 | Dst PortName 260 | } 261 | --------------------------------------------------------------------------------