├── .github └── workflows │ └── go.yml ├── .gitignore ├── LICENSE ├── README.md ├── async.go ├── async_test.go ├── cmd ├── generators │ ├── redis_stream.go │ ├── rest_api.go │ └── sql.go ├── main.go ├── pipeline_example.go └── pipeline_fan_out.go ├── diag ├── flow-chart.png └── small-chart.png ├── go.mod ├── go.sum ├── list.go ├── list_test.go ├── pipeline.go ├── pipeline_test.go ├── pipeline_tree.go ├── stepper.go ├── util.go └── util_test.go /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a golang project 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go 3 | 4 | name: Go 5 | 6 | on: 7 | push: 8 | branches: [ "main" ] 9 | pull_request: 10 | branches: [ "main" ] 11 | 12 | jobs: 13 | 14 | build: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v4 18 | 19 | - name: Set up Go 20 | uses: actions/setup-go@v4 21 | with: 22 | go-version: '1.23' 23 | 24 | - name: Build 25 | run: go build -v ./... 26 | 27 | - name: Test 28 | run: go test -v ./... 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | vendor -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 arrno 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Gliter ✨ 2 | 3 | [![Go Reference](https://pkg.go.dev/badge/github.com/arrno/gliter.svg)](https://pkg.go.dev/github.com/arrno/gliter) 4 | [![Go Build](https://github.com/arrno/gliter/actions/workflows/go.yml/badge.svg)](https://github.com/arrno/gliter/actions/workflows/go.yml) 5 | 6 | **Go lang iter tools! ʕ◔ϖ◔ʔ** 7 | 8 | Download: 9 | 10 | ``` 11 | go get github.com/arrno/gliter 12 | ``` 13 | 14 | Import: 15 | 16 | ```go 17 | import "github.com/arrno/gliter" 18 | ``` 19 | 20 | > The mission of this project is to make it easy to compose normal business logic into complex async patterns. Ideally, we should spend most our mental energy solving our core problems instead of worrying about race conditions, deadlocks, channel states, and go-routine leaks. The patterns published here are all intended to serve that goal. 21 | 22 | ## Pipelines 23 | 24 | Orchestrate a series of functions into a branching async pipeline with the Pipeline type. 25 | 26 | ```go 27 | gliter.NewPipeline(streamTransactionsFromKafka). 28 | Stage( 29 | preprocessFeatures, // Normalize, extract relevant fields 30 | ). 31 | Stage( 32 | runFraudModel, // Model inference on transactions 33 | checkBusinessRules, // Non-ML heuristic checks 34 | ). 35 | Merge( 36 | aggregateResults, // Combine outputs from ML & rules 37 | ). 38 | Stage( 39 | sendToAlertSystem, // Notify if fraud detected 40 | storeInDatabase, // Log for later analysis 41 | ). 42 | Run() 43 | ``` 44 | 45 | ### Example 46 | 47 | Start with a generator function and some simple transformers: 48 | 49 | ```go 50 | func exampleGen() func() (int, bool, error) { 51 | data := []int{1, 2, 3, 4, 5} 52 | index := -1 53 | return func() (int, bool, error) { 54 | index++ 55 | if index == len(data) { 56 | return 0, false, nil 57 | } 58 | return data[index], true, nil 59 | } 60 | } 61 | 62 | func exampleMid(i int) (int, error) { 63 | return i * 2, nil 64 | } 65 | 66 | func exampleAlt(i int) (int, error) { 67 | return i * i, nil 68 | } 69 | 70 | func exampleEnd(i int) (int, error) { 71 | return i * i, nil 72 | } 73 | ``` 74 | 75 | Assemble the functions into a new async pipeline using `NewPipeline`, `Stage`, and `Run`: 76 | 77 | ```go 78 | gliter.NewPipeline(exampleGen()). 79 | Stage( 80 | exampleMid, 81 | ). 82 | Stage( 83 | exampleEnd, 84 | ). 85 | Run() 86 | ``` 87 | 88 | Note that: 89 | 90 | - Data always flows downstream from generator through stages sequentially. 91 | - There is no distinct end stage. Any side-effects/outputs like db writes or API posts should be handled inside a Stage function wherever appropriate. 92 | - Any error encountered at any stage will short-circuit the pipeline. 93 | 94 | ### Fork 95 | 96 | Stages do automatic branching like so: 97 | 98 | ```go 99 | gliter.NewPipeline(exampleGen()). 100 | Stage( 101 | exampleMid, // branch A 102 | exampleAlt, // branch B 103 | ). 104 | Stage( 105 | exampleEnd, // Downstream of fork, exists in both branches 106 | ). 107 | Run() 108 | ``` 109 | 110 | **Any time we choose to add multiple handlers in a single stage, we are forking the pipeline that many times.** When a fork occurs, all downstream stages are implicitly duplicated to exist in each stream and each transmitted record is emitted on all available downstream paths doubling the number of processed records/streams. 111 | 112 | **If you fork a stream that processes pointers, you should clone the record in downstream branches before mutating it.** 113 | 114 | **What if I want to branch out a stage without duplicating records/streams?** In that case, check out the **Option** stage or the **InParallel** utility documented below. InParallel also demonstrated in `./cmd/pipeline_fan_out.go`. 115 | 116 | ### Throttle 117 | 118 | What if our end stage results in a high number of concurrent output streams that overwhelms a destination DB or API? Use the throttle stage to rein in concurrent streams like this: 119 | 120 | ```go 121 | // With concurrency throttling 122 | gliter.NewPipeline(exampleGen()). 123 | Stage( 124 | exampleMid, // branch A 125 | exampleMid, // branch B 126 | ). 127 | Stage( 128 | exampleMid, // branches A.C, B.C 129 | exampleMid, // branches A.D, B.D 130 | exampleMid, // branches A.E, B.E 131 | ). 132 | Throttle(2). // throttle into branches X, Z 133 | Stage( 134 | exampleEnd, 135 | ). 136 | Run() 137 | ``` 138 | 139 | Here is a visual diagram of the pipeline the code produces: 140 | 141 | ![Alt text](./diag/small-chart.png?raw=true "Title") 142 | 143 | ### Merge 144 | 145 | A merge stage is like a throttle but with a transform function to handle merging records from upstream sibling branches. Once all upstream branches are ready to emit, the merge handler receives one element from each via the slice argument. The handler can emit one or more resulting records which gliter converts back into a stream. 146 | 147 | ```go 148 | gliter.NewPipeline(exampleGen()). 149 | Stage( 150 | exampleMid, // branch A 151 | exampleMid, // branch B 152 | ). 153 | Stage( 154 | exampleMid, // branches A.C, B.C 155 | exampleMid, // branches A.D, B.D 156 | exampleMid, // branches A.E, B.E 157 | ). 158 | Merge( // merges into branch Z 159 | func(items []int) ([]int, error) { 160 | sum := 0 161 | for _, item := range items { 162 | sum += item 163 | } 164 | return []int{sum}, nil 165 | }, 166 | ). 167 | Stage( 168 | exampleEnd, 169 | ). 170 | Run() 171 | ``` 172 | 173 | ### Batch 174 | 175 | What if one of our stages does something slow, like a DB write, that could be optimized with batching? Use a special batch stage to pool items together for bulk processing: 176 | 177 | ```go 178 | func exampleBatch(items []int) ([]int, error) { 179 | // A slow/expensive operation 180 | if err := storeToDB(items); err != nil { 181 | return nil, err 182 | } 183 | return items, nil 184 | } 185 | 186 | gliter.NewPipeline(exampleGen()). 187 | Stage(exampleMid). 188 | Batch(100, exampleBatch). 189 | Stage(exampleEnd). 190 | Run() 191 | ``` 192 | 193 | gliter will handle converting the input-stream to batch and output-batch to stream for you which means batch stages are composable with normal stages. 194 | 195 | ### Option 196 | 197 | An option stage resembles a regular forking stage but **does not fork/clone pipeline streams.** Each handler is an optional route for an inbound record and only one is selected. Multiple handlers can process records concurrently which means an option stage is also effectively a buffer. **A record has an equal chance of being emitted on any handler that is available.** That means faster handlers will process more records than slower handlers. 198 | 199 | ```go 200 | gliter.NewPipeline(exampleGen()). 201 | Stage(exampleMid). 202 | Option( // each record will enter one of these 203 | func(item int) (int, error) { 204 | return 1 + item, nil 205 | }, 206 | func(item int) (int, error) { 207 | return 2 + item, nil 208 | }, 209 | func(item int) (int, error) { 210 | return 3 + item, nil 211 | }, 212 | ). 213 | Stage(exampleEnd). 214 | Run() 215 | ``` 216 | 217 | ### Buffer 218 | 219 | In certain situations, you may want to introduce a buffer before a slow/expensive stage. Doing so **will not increase the overall performance of your pipeline** but may aid in faster response signalling to an upstream caller. Add a buffer like this: 220 | 221 | ```go 222 | gliter.NewPipeline(exampleGen()). 223 | Stage(exampleMid). 224 | Buffer(5). 225 | Stage(exampleEnd). 226 | Run() 227 | ``` 228 | 229 | In this example, `exampleMid` can process/emit up to 5 results while exampleEnd is busy. Once the buffer is full, `exampleMid` is blocked again until `exampleEnd` pulls from the buffer. 230 | 231 | ### Tally 232 | 233 | There are two ways to tally items processed by the pipeline. 234 | 235 | - Toggle on config to get all the node counts returned: 236 | 237 | ```go 238 | counts, err := gliter. 239 | NewPipeline(exampleGen()). 240 | Config(gliter.PLConfig{ReturnCount: true}). 241 | Run() 242 | 243 | if err != nil { 244 | panic(err) 245 | } 246 | 247 | for _, count := range counts { 248 | fmt.Printf("Node: %s\nCount: %d\n\n", count.NodeID, count.Count) 249 | } 250 | ``` 251 | 252 | - For more granular control, use the `Tally` channel like this: 253 | 254 | ```go 255 | pipeline := NewPipeline(exampleGen()) 256 | tally := pipeline.Tally() 257 | 258 | endWithTally := func(i int) (int, error) { 259 | tally <- 1 // any integer 260 | return exampleEnd(i) 261 | } 262 | 263 | // Produces one `PLNodeCount` for node "tally" 264 | count, err := pipeline. 265 | Stage(endWithTally). 266 | Run() 267 | 268 | if err != nil { 269 | panic(err) 270 | } 271 | 272 | // All integers sent to tally are summed 273 | fmt.Printf("Node: %s\nCount: %d\n", count[0].NodeID, count[0].Count) 274 | ``` 275 | 276 | This is helpful if slices/maps are passed through the pipeline and you want to tally the total number or individual records processed. **Note that the tally channel is listened to while the pipeline is running and is closed when all pipeline stages exit.** For that reason, tally can always be written to freely within a stage function. Writes to tally outside of a stage function however is not recommended. 277 | 278 | ### Insight 279 | 280 | It may be helpful during testing to audit what is happening inside a pipeline. 281 | 282 | To do so, optionally set pipeline logging via one of the following modes in `pipeline.Config(gliter.PLConfig{...})`: 283 | 284 | - `LogCount` - Log result count table (start here) 285 | - `LogEmit` - Log every emit 286 | - `LogAll` - Log emit and count 287 | - `LogStep` - Interactive CLI stepper 288 | 289 | ```go 290 | gliter.NewPipeline(exampleGen()). // with logging 291 | Config(gliter.PLConfig{LogAll: true}). 292 | Stage( 293 | exampleMid, 294 | exampleAlt, 295 | ). 296 | Stage( 297 | exampleEnd, 298 | ). 299 | Run() 300 | ``` 301 | 302 | Output: 303 | 304 | ``` 305 | Emit -> 4 306 | Emit -> 16 307 | ... 308 | Emit -> 100 309 | +-------+-------+-------+ 310 | | node | count | value | 311 | +-------+-------+-------+ 312 | | GEN | 5 | 5 | 313 | | 0:0:0 | 5 | 10 | 314 | | 0:0:1 | 5 | 10 | 315 | | 1:0:0 | 5 | 100 | 316 | | 1:1:0 | 5 | 100 | 317 | +-------+-------+-------+ 318 | ``` 319 | 320 | The node id shown in the table is constructed as **stage-index** : **parent-index** : **func-index**. For example, node id `4:1:2` would indicate the third function (idx 2) of the fifth stage (idx 4) branched from the second parent node (idx 1). 321 | 322 | "The second parent node" in this context can also be called the func at index 1 of the fourth stage. 323 | 324 | Throttle, buffer, and merge stages are logged a bit differently as **[THROTTLE]**, **[BUFFER]**, and **[MERGE]** respectively. 325 | 326 | > Note, you only pay for what you use. If logging it not enabled, these states are not tracked. 327 | 328 | ### Examples 329 | 330 | - Get a boost with pre-built generators for common vendors in [./cmd/generators](https://github.com/arrno/gliter/blob/main/cmd/generators) 331 | - For a more realistic pipeline example, see [./cmd/pipeline_example.go](https://github.com/arrno/gliter/blob/main/cmd/pipeline_example.go) 332 | - For an example of composing pipeline patterns with `InParallel` AKA Fan-in/Fan-out, see [./cmd/pipeline_fan_out.go](https://github.com/arrno/gliter/blob/main/cmd/pipeline_fan_out.go) 333 | - For an example with pipeline benchmarking, see [this repository](https://github.com/arrno/benchmark-gliter) 334 | 335 | ## In parallel 336 | 337 | A Fan-in/Fan-out utility to run a series of functions in parallel and collect results **preserving order at no cost.**. 338 | 339 | InParallel complement the Pipeline pattern in the following ways: 340 | 341 | - Use `InParallel` to run a set of unique pipelines concurrently 342 | - Call `InParallel` from inside a slow pipeline stage to fan-out/fan-in the expensive task 343 | 344 | ```go 345 | tasks := []func() (string, error){ 346 | func() (string, error) { 347 | return "Hello", nil 348 | }, 349 | func() (string, error) { 350 | return ", ", nil 351 | }, 352 | func() (string, error) { 353 | return "Async!", nil 354 | }, 355 | } 356 | 357 | // Run all tasks at the same time and collect results/err 358 | results, err := gliter.InParallel(tasks) 359 | if err != nil { 360 | panic(err) 361 | } 362 | 363 | // Always prints "Hello, Async!" 364 | for _, result := range results { 365 | fmt.Print(result) 366 | } 367 | ``` 368 | 369 | ## Misc 370 | 371 | Other async helpers in this library include: 372 | 373 | - Do your own throttling with `ThrottleBy` 374 | - Do your own channel forking with `TeeBy` 375 | - `ReadOrDone` + `WriteOrDone` 376 | - `IterDone` - iterate until read or done channel is closed 377 | - `Any` - take one from any, consolidating "done" channels 378 | 379 | Also included are some synchronous iter tools that may be helpful: 380 | 381 | The `List` Type 382 | 383 | ```go 384 | list := gliter.List(0, 1, 2, 3, 4) 385 | list.Pop() // removes/returns `4` 386 | list.Push(8) // appends `8` 387 | ``` 388 | 389 | Chaining functions on a list 390 | 391 | ```go 392 | value := gliter. 393 | List(0, 1, 2, 3, 4). 394 | Filter(func(i int) bool { 395 | return i%2 == 0 396 | }). // []int{0, 2, 4} 397 | Map(func(val int) int { 398 | return val * 2 399 | }). // []int{0, 4, 8} 400 | Reduce(func(acc *int, val int) { 401 | *acc += val 402 | }) // 12 403 | ``` 404 | 405 | Includes: 406 | 407 | - `list.Filter(func (i int) bool { return i%2 == 0 })` 408 | - `list.Map(func (i int) int { return i * 2 })` 409 | - `list.Reduce(func(acc *int, val int) { *acc += val })` 410 | - `list.Find(func (i int) bool { return i%2 == 0 })` 411 | - `list.Len()` 412 | - `list.Reverse()` 413 | - `list.At(i)` 414 | - `list.FPop()` 415 | - `list.Slice(start, stop)` 416 | - `list.Delete(index)` 417 | - `list.Insert(index, "value")` 418 | 419 | Unwrap back into slice via `list.Iter()` or `list.Unwrap()` 420 | 421 | ```go 422 | list := gliter.List(0, 1, 2, 3, 4) 423 | for _, item := range list.Iter() { 424 | fmt.Println(item) 425 | } 426 | ``` 427 | 428 | Map to alt type via `Map(list, func(in) out)` 429 | 430 | Something missing? Open a PR. **Contributions welcome!** 431 | -------------------------------------------------------------------------------- /async.go: -------------------------------------------------------------------------------- 1 | package gliter 2 | 3 | import "sync" 4 | 5 | // InParallel runs all the functions asynchronously and returns the results in order or the first error. 6 | func InParallel[T any](funcs []func() (T, error)) ([]T, error) { 7 | type orderedResult struct { 8 | result T 9 | order int 10 | } 11 | dataChan := make(chan orderedResult, len(funcs)) 12 | errChan := make(chan error, len(funcs)) 13 | for i, f := range funcs { 14 | go func(order int) { 15 | result, err := f() 16 | if err != nil { 17 | errChan <- err 18 | } else { 19 | dataChan <- orderedResult{result, order} 20 | } 21 | }(i) 22 | } 23 | results := make([]T, len(funcs)) 24 | for range len(funcs) { 25 | select { 26 | case err := <-errChan: 27 | return nil, err 28 | case res := <-dataChan: 29 | results[res.order] = res.result 30 | } 31 | } 32 | return results, nil 33 | } 34 | 35 | // ThrottleBy merges the output of the provided channels into n output channels. 36 | // This function returns when 'in' channels are closed or signal is received on 'done'. 37 | func ThrottleBy[T any](in []chan T, done <-chan interface{}, n int) (out []chan T) { 38 | out = make([]chan T, n) 39 | for i := range n { 40 | out[i] = make(chan T) 41 | } 42 | 43 | go func() { 44 | defer func() { 45 | for _, ch := range out { 46 | close(ch) 47 | } 48 | }() 49 | var wg sync.WaitGroup 50 | // a signal on any inbound channel has equal chance or emitting on any outbound channel 51 | for _, inChan := range in { 52 | for _, outChan := range out { 53 | wg.Add(1) 54 | go func() { 55 | defer wg.Done() 56 | for { 57 | if val, ok := ReadOrDone(inChan, done); !ok { 58 | return 59 | } else if !WriteOrDone(val, outChan, done) { 60 | return 61 | } 62 | } 63 | }() 64 | } 65 | } 66 | wg.Wait() 67 | }() 68 | 69 | return 70 | } 71 | 72 | // TeeBy broadcasts all received signals on provided channel into n output channels. 73 | // This function returns when 'in' channel is closed or signal is received on 'done'. 74 | func TeeBy[T any](in <-chan T, done <-chan interface{}, n int) (outR []<-chan T) { 75 | if n < 2 { 76 | return []<-chan T{in} 77 | } 78 | outW := make([]chan T, n) 79 | outR = make([]<-chan T, n) 80 | for i := range n { 81 | ch := make(chan T) 82 | outW[i] = ch 83 | outR[i] = ch 84 | } 85 | 86 | go func() { 87 | defer func() { 88 | for _, ch := range outW { 89 | close(ch) 90 | } 91 | }() 92 | for { 93 | select { 94 | case val, ok := <-in: 95 | if ok { 96 | var wg sync.WaitGroup 97 | wg.Add(len(outW)) 98 | for _, ch := range outW { 99 | go func() { 100 | defer wg.Done() 101 | WriteOrDone(val, ch, done) 102 | }() 103 | } 104 | wg.Wait() 105 | } else { 106 | return 107 | } 108 | case <-done: 109 | return 110 | } 111 | } 112 | }() 113 | 114 | return 115 | } 116 | 117 | // WriteOrDone blocks until it sends to 'write' or receives from 'done' and returns the boolean result. 118 | func WriteOrDone[T any](val T, write chan<- T, done <-chan any) bool { 119 | select { 120 | case write <- val: 121 | return true 122 | case <-done: 123 | return false 124 | } 125 | } 126 | 127 | // ReadOrDone blocks until it receives from 'read' or receives from 'done' and returns the boolean result. 128 | func ReadOrDone[T any](read <-chan T, done <-chan any) (T, bool) { 129 | select { 130 | case val, ok := <-read: 131 | return val, ok 132 | case <-done: 133 | var zero T 134 | return zero, false 135 | } 136 | } 137 | 138 | // Any consolidates a set of 'done' channels into one done channel. 139 | func Any(channels ...<-chan any) <-chan any { 140 | switch len(channels) { 141 | case 0: 142 | return nil 143 | case 1: 144 | return channels[0] 145 | } 146 | orDone := make(chan any) 147 | go func() { 148 | defer close(orDone) 149 | switch len(channels) { 150 | case 2: 151 | select { 152 | case <-channels[0]: 153 | case <-channels[1]: 154 | } 155 | default: 156 | select { 157 | case <-channels[0]: 158 | case <-channels[1]: 159 | case <-channels[2]: 160 | case <-Any(append(channels[3:], orDone)...): 161 | } 162 | } 163 | }() 164 | return orDone 165 | } 166 | 167 | // IterDone combines a read and done channel for convenient iterating. 168 | // Iterate over the return channel knowing the loop will exit when either read or done 169 | // are closed. 170 | func IterDone[T any](read <-chan T, done <-chan any) <-chan T { 171 | out := make(chan T) 172 | go func() { 173 | defer close(out) 174 | for { 175 | val, ok := ReadOrDone(read, done) 176 | if !ok { 177 | return 178 | } 179 | out <- val 180 | } 181 | }() 182 | return out 183 | } 184 | -------------------------------------------------------------------------------- /async_test.go: -------------------------------------------------------------------------------- 1 | package gliter 2 | 3 | import ( 4 | "errors" 5 | "reflect" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestInParallel(t *testing.T) { 12 | tasks := []func() (string, error){ 13 | func() (string, error) { 14 | return "Hello", nil 15 | }, 16 | func() (string, error) { 17 | return ", ", nil 18 | }, 19 | func() (string, error) { 20 | return "Async!", nil 21 | }, 22 | } 23 | results, err := InParallel(tasks) 24 | assert.Nil(t, err) 25 | assert.True(t, reflect.DeepEqual([]string{"Hello", ", ", "Async!"}, results)) 26 | } 27 | 28 | func TestInParallelErr(t *testing.T) { 29 | tasks := []func() (string, error){ 30 | func() (string, error) { 31 | return "Hello", nil 32 | }, 33 | func() (string, error) { 34 | return ",", errors.New("error") 35 | }, 36 | func() (string, error) { 37 | return "Async!", nil 38 | }, 39 | } 40 | _, err := InParallel(tasks) 41 | assert.NotNil(t, err) 42 | assert.Equal(t, err.Error(), "error") 43 | } 44 | 45 | func TestIterDoneLong(t *testing.T) { 46 | done := make(chan any) 47 | data := make(chan int) 48 | go func() { 49 | defer close(data) 50 | for i := range 100 { 51 | data <- i 52 | } 53 | }() 54 | 55 | results := make([]int, 100) 56 | expected := make([]int, 100) 57 | 58 | for i := range IterDone(data, done) { 59 | results[i] = i 60 | } 61 | for i := range 100 { 62 | expected[i] = i 63 | } 64 | assert.True(t, reflect.DeepEqual(expected, results)) 65 | } 66 | 67 | func TestIterDoneShort(t *testing.T) { 68 | done := make(chan any) 69 | final := make(chan any) 70 | data := make(chan int) 71 | go func() { 72 | defer close(data) 73 | for i := range 21 { 74 | data <- i 75 | } 76 | close(done) // simulate disrupt 77 | <-final // hang so data is not closed 78 | }() 79 | 80 | results := make([]int, 100) 81 | expected := make([]int, 100) 82 | 83 | for i := range IterDone(data, done) { 84 | results[i] = i 85 | } // can only exit due to done being closed 86 | for i := range 21 { 87 | expected[i] = i 88 | } 89 | assert.True(t, reflect.DeepEqual(expected, results)) 90 | // close final so goroutine can dismount and close data 91 | close(final) 92 | } 93 | -------------------------------------------------------------------------------- /cmd/generators/redis_stream.go: -------------------------------------------------------------------------------- 1 | package generators 2 | 3 | // Commented out because we don't include dependencies 4 | 5 | // import ( 6 | // "fmt" 7 | // "log" 8 | // "time" 9 | 10 | // "github.com/go-redis/redis/v8" 11 | // "golang.org/x/net/context" 12 | // ) 13 | 14 | // type RedisGenerator struct { 15 | // client *redis.Client 16 | // pageSize int64 17 | // streamKey string 18 | // lastID string 19 | // done <-chan any 20 | // } 21 | 22 | // func NewRedisGenerator(pageSize int64, streamKey string, client *redis.Client, done <-chan any) func() ([]map[string]any, bool, error) { 23 | // if client == nil { 24 | // panic("Nil client") 25 | // } 26 | // gen := &RedisGenerator{ 27 | // client: client, 28 | // pageSize: pageSize, 29 | // streamKey: streamKey, 30 | // done: done, 31 | // } 32 | // return gen.fetch 33 | // } 34 | 35 | // func (g *RedisGenerator) fetch() ([]map[string]any, bool, error) { 36 | 37 | // // The only way to exit 38 | // select { 39 | // case <-g.done: 40 | // return nil, false, nil 41 | // default: 42 | // } 43 | 44 | // // Fetch the next batch of messages from the stream 45 | // ctx := context.Background() 46 | // messages, err := g.client.XRead(ctx, &redis.XReadArgs{ 47 | // Streams: []string{g.streamKey, g.lastID}, 48 | // Block: 0, // block indefinitely 49 | // Count: g.pageSize, 50 | // }).Result() 51 | 52 | // if err != nil { 53 | // if err == redis.Nil { 54 | // // No new messages in the stream, continue 55 | // // This should never happen if blocking indefinitely on read 56 | // return []map[string]any{}, true, nil 57 | // } 58 | // return nil, false, err 59 | // } 60 | 61 | // var results []map[string]any 62 | // for _, message := range messages { 63 | // for _, xMessage := range message.Messages { 64 | // messageData := make(map[string]any) 65 | // for key, value := range xMessage.Values { 66 | // messageData[key] = value 67 | // } 68 | // results = append(results, messageData) 69 | // g.lastID = xMessage.ID 70 | // } 71 | // } 72 | 73 | // return results, true, nil 74 | 75 | // } 76 | 77 | // func main() { 78 | // client := redis.NewClient(&redis.Options{ 79 | // Addr: "localhost:6379", // Redis server address 80 | // }) 81 | 82 | // done := make(chan any) 83 | // go func() { 84 | // <-time.After(time.Duration(5) * time.Minute) 85 | // close(done) 86 | // }() 87 | 88 | // // Create the generator function 89 | // gen := NewRedisGenerator(10, "myStream", client, done) 90 | 91 | // // Example of consuming the stream 92 | // for { 93 | // data, hasData, err := gen() 94 | // if err != nil { 95 | // log.Fatal("Error reading from Redis stream: ", err) 96 | // } 97 | // if !hasData { 98 | // // No more data in the stream 99 | // break 100 | // } 101 | 102 | // // Process the received data 103 | // fmt.Println("Received data:", data) 104 | // } 105 | // } 106 | -------------------------------------------------------------------------------- /cmd/generators/rest_api.go: -------------------------------------------------------------------------------- 1 | package generators 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | ) 9 | 10 | type APIGenerator struct { 11 | url string 12 | client *http.Client 13 | page int 14 | pageSize int 15 | hasMore bool 16 | } 17 | 18 | func NewAPIGenerator(url string, pageSize int, client *http.Client) func() ([]map[string]any, bool, error) { 19 | if client == nil { 20 | client = http.DefaultClient 21 | } 22 | gen := &APIGenerator{ 23 | url: url, 24 | client: client, 25 | page: 0, 26 | pageSize: pageSize, 27 | hasMore: true, 28 | } 29 | return gen.fetch 30 | } 31 | 32 | func (g *APIGenerator) fetch() ([]map[string]any, bool, error) { 33 | if !g.hasMore { 34 | return nil, false, nil 35 | } 36 | g.page++ 37 | resp, err := g.client.Get(fmt.Sprintf("%s?page=%d&pageSize=%d", g.url, g.page, g.pageSize)) 38 | if err != nil { 39 | return nil, false, err 40 | } 41 | defer resp.Body.Close() 42 | if resp.StatusCode != http.StatusOK { 43 | return nil, false, fmt.Errorf("HTTP error: %d", resp.StatusCode) 44 | } 45 | body, err := io.ReadAll(resp.Body) 46 | if err != nil { 47 | return nil, false, err 48 | } 49 | var data []map[string]any 50 | if err := json.Unmarshal(body, &data); err != nil { 51 | return nil, false, err 52 | } 53 | g.hasMore = len(data) == g.pageSize 54 | return data, g.hasMore, nil 55 | } 56 | 57 | // func main() { 58 | // apiGen := NewAPIGenerator("https://api.example.com/data", 10, nil) 59 | // for { 60 | // data, hasMore, err := apiGen() 61 | // if err != nil { 62 | // fmt.Println("Error:", err) 63 | // break 64 | // } 65 | // if !hasMore { 66 | // break 67 | // } 68 | // fmt.Println("Fetched:", data) 69 | // } 70 | // } 71 | -------------------------------------------------------------------------------- /cmd/generators/sql.go: -------------------------------------------------------------------------------- 1 | package generators 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | // mysql 7 | // _ "github.com/go-sql-driver/mysql" 8 | // postgres 9 | // _ "github.com/lib/pq" 10 | ) 11 | 12 | type DBGenerator struct { 13 | db *sql.DB 14 | query string 15 | pageSize int 16 | offset int 17 | hasMore bool 18 | } 19 | 20 | func NewDBGenerator(db *sql.DB, query string, pageSize int) func() ([]map[string]any, bool, error) { 21 | gen := &DBGenerator{ 22 | db: db, 23 | query: query, 24 | pageSize: pageSize, 25 | offset: 0, 26 | hasMore: true, 27 | } 28 | return gen.fetch 29 | } 30 | 31 | func (g *DBGenerator) fetch() ([]map[string]any, bool, error) { 32 | if !g.hasMore { 33 | return nil, false, nil 34 | } 35 | 36 | rows, err := g.db.Query(fmt.Sprintf("%s LIMIT %d OFFSET %d", g.query, g.pageSize, g.offset)) 37 | if err != nil { 38 | return nil, false, err 39 | } 40 | defer rows.Close() 41 | 42 | columns, err := rows.Columns() 43 | if err != nil { 44 | return nil, false, err 45 | } 46 | 47 | var results []map[string]any 48 | for rows.Next() { 49 | scanArgs := make([]interface{}, len(columns)) 50 | for i := range scanArgs { 51 | var v interface{} 52 | scanArgs[i] = &v 53 | } 54 | if err := rows.Scan(scanArgs...); err != nil { 55 | return nil, false, err 56 | } 57 | 58 | rowMap := make(map[string]any) 59 | for i, colName := range columns { 60 | rowMap[colName] = *(scanArgs[i].(*interface{})) 61 | } 62 | results = append(results, rowMap) 63 | } 64 | 65 | g.hasMore = len(results) == g.pageSize 66 | g.offset += g.pageSize 67 | 68 | return results, g.hasMore, nil 69 | } 70 | 71 | // func main() { 72 | // // mysql 73 | // db, err := sql.Open("mysql", "user:password@tcp(localhost:3306)/dbname") 74 | // // postgres 75 | // db, err := sql.Open("postgres", "user=password dbname=yourdb sslmode=disable") 76 | // if err != nil { 77 | // panic(err) 78 | // } 79 | // defer db.Close() 80 | 81 | // dbGen := NewDBGenerator(db, "SELECT * FROM your_table", 10) 82 | // for { 83 | // data, hasMore, err := dbGen() 84 | // if err != nil { 85 | // fmt.Println("Error:", err) 86 | // break 87 | // } 88 | // if !hasMore { 89 | // break 90 | // } 91 | // fmt.Println("Fetched:", data) 92 | // } 93 | // } 94 | -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | func main() { 6 | fmt.Println("Example main:") 7 | ExampleMain() 8 | fmt.Println("Example fan out:") 9 | ExampleFanOut() 10 | } 11 | -------------------------------------------------------------------------------- /cmd/pipeline_example.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "math" 7 | "math/rand/v2" 8 | "time" 9 | 10 | "github.com/arrno/gliter" 11 | ) 12 | 13 | /* 14 | In this example, we fetch paginated transaction records from an API, transform them 15 | by two distinct transform functions, and store the results in a database. 16 | 17 | By using a gliter Pipeline, we can reduce ingest latency by storing records to the DB 18 | while we're fetching the next page from the API (and transforming). By running all stages 19 | concurrently, the throughput is increased. 20 | 21 | Since we have two transformers stacked in the middle stage, our pipeline is also forked 22 | such that both downstream transform branches are writing to the DB concurrently. 23 | 24 | All of this with almost zero async primitives. 25 | */ 26 | 27 | // These are our interfaces for input/output. 28 | 29 | // API will be wrapped by our pipeline generator. 30 | type API[T any] interface { 31 | FetchPage(page uint) (results []T, err error) 32 | } 33 | 34 | // DB is where our last pipeline stage will store the output. 35 | type DB interface { 36 | BatchAdd(records []any) error 37 | } 38 | 39 | // Record is the pipeline input data type. 40 | type Record struct { 41 | CreatedAt time.Time 42 | TransactionNumber int 43 | AmountDollars float64 44 | } 45 | 46 | // MockAPI satisfied the API interface. 47 | type MockAPI[T any] struct { 48 | Pages [][]T 49 | } 50 | 51 | func NewMockAPI() *MockAPI[Record] { 52 | a := new(MockAPI[Record]) 53 | a.Pages = make([][]Record, 10) 54 | for i := range 10 { 55 | page := make([]Record, 10) 56 | for j := range len(page) { 57 | page[j] = Record{ 58 | CreatedAt: time.Now(), 59 | TransactionNumber: (j * i) + i, 60 | AmountDollars: math.Round(rand.Float64()*10000) / 100, 61 | } 62 | } 63 | a.Pages[i] = page 64 | } 65 | return a 66 | } 67 | 68 | // FetchPage returns a page of data that may be empty. If `page` is negative, an error is returned. 69 | func (a *MockAPI[Record]) FetchPage(page int) (results []Record, err error) { 70 | if page < 1 { 71 | return nil, fmt.Errorf("page number must be positive. Received %d", page) 72 | } else if page > len(a.Pages) { 73 | return nil, nil 74 | } 75 | return a.Pages[page-1], nil 76 | } 77 | 78 | // MockDB satisfied the DB interface. 79 | type MockDB struct { 80 | Records []any 81 | } 82 | 83 | func NewMockDB() *MockDB { 84 | return new(MockDB) 85 | } 86 | 87 | // BatchAdd stores the records. If `records` is empty, an error is returned. 88 | func (db *MockDB) BatchAdd(records []any) error { 89 | if len(records) == 0 { 90 | return errors.New("records should not be empty") 91 | } 92 | db.Records = append(db.Records, records...) 93 | return nil 94 | } 95 | 96 | // RecordCents is a transform variant of the original `Record` type. 97 | type RecordCents struct { 98 | CreatedAt time.Time 99 | UpdatedAt time.Time 100 | TransactionNumber int 101 | AmountCents int 102 | } 103 | 104 | // ConvertToCents is our first transform function which converts a `Record` into a `RecordCents`. 105 | func ConvertToCents(data any) (any, error) { 106 | records, ok := data.([]Record) 107 | if !ok { 108 | return data, errors.New("expected page in dollars") 109 | } 110 | results := make([]RecordCents, len(records)) 111 | for i, r := range records { 112 | results[i] = RecordCents{ 113 | CreatedAt: r.CreatedAt, 114 | UpdatedAt: time.Now(), 115 | TransactionNumber: r.TransactionNumber, 116 | AmountCents: int(r.AmountDollars * 100), 117 | } 118 | } 119 | return results, nil 120 | } 121 | 122 | // ApplyEvenFee is our second transform function which applies a $10 fee to even-numbered Record transactions. 123 | func ApplyEvenFee(data any) (any, error) { 124 | records, ok := data.([]Record) 125 | if !ok { 126 | return data, errors.New("expected page in dollars") 127 | } 128 | results := make([]Record, len(records)) 129 | for i, r := range records { 130 | // Clone ptr to mutate 131 | results[i] = Record{ 132 | CreatedAt: r.CreatedAt, 133 | TransactionNumber: r.TransactionNumber, 134 | AmountDollars: r.AmountDollars - 10.0, 135 | } 136 | if r.TransactionNumber%2 != 0 { 137 | results[i].AmountDollars -= 10 138 | } 139 | } 140 | return results, nil 141 | } 142 | 143 | // ExampleMain is the `Main` function for our example. Here we will build and run our pipeline. 144 | func ExampleMain() { 145 | 146 | // First we create a generator that wraps our API service. 147 | gen := func() func() (any, bool, error) { 148 | api := NewMockAPI() 149 | var page int 150 | return func() (any, bool, error) { 151 | page++ 152 | results, err := api.FetchPage(page) 153 | if err != nil { 154 | return struct{}{}, false, err 155 | } 156 | if len(results) == 0 { 157 | return results, false, nil 158 | } else { 159 | return results, true, nil 160 | } 161 | } 162 | } 163 | 164 | // Next we create a DB service and a `store` method. 165 | // The `store` method wraps the db `BatchAdd` function and will serve 166 | // as the final stage of our pipeline. Note that `store` also takes a 167 | // tally channel for granular processed-record counting. 168 | db := NewMockDB() 169 | store := func(inbound any, tally chan<- int) (any, error) { 170 | var results []any 171 | if records, ok := inbound.([]Record); ok { 172 | results = make([]any, len(records)) 173 | for i, r := range records { 174 | results[i] = r 175 | } 176 | } else if records, ok := inbound.([]RecordCents); ok { 177 | results = make([]any, len(records)) 178 | for i, r := range records { 179 | results[i] = r 180 | } 181 | } 182 | tally <- len(results) // count total records processed 183 | if err := db.BatchAdd(results); err != nil { 184 | return struct{}{}, err 185 | } 186 | return struct{}{}, nil 187 | } 188 | 189 | // We create a new pipeline out of our generator. 190 | pipeline := gliter.NewPipeline(gen()) 191 | 192 | // Let's initialize a tally channel for better control over the processed count. 193 | tally := pipeline.Tally() 194 | // `storeWithTally` is a simple closure that captures `tally` and calls `store`. 195 | storeWithTally := func(inbound any) (any, error) { 196 | return store(inbound, tally) 197 | } 198 | 199 | // Lastly, we assemble our pipeline stages, enable logging, and run the pipeline. 200 | count, err := pipeline. 201 | Config(gliter.PLConfig{LogCount: true}). 202 | Stage( 203 | ConvertToCents, 204 | ApplyEvenFee, 205 | ). 206 | Stage( 207 | storeWithTally, 208 | ). 209 | Run() 210 | 211 | if err != nil { 212 | panic(err) 213 | } 214 | 215 | // We expect 200 records to have been processed by the sum of all `store` function instances. 216 | fmt.Printf("Node: %s\nCount: %d\n", count[0].NodeID, count[0].Count) 217 | 218 | // That's it! 219 | } 220 | -------------------------------------------------------------------------------- /cmd/pipeline_fan_out.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "math/rand/v2" 7 | "time" 8 | 9 | "github.com/arrno/gliter" 10 | ) 11 | 12 | // To simplify the example, we use a generic API that fetches pages of []any 13 | 14 | func NewMockAPIAny() *MockAPI[any] { 15 | a := new(MockAPI[any]) 16 | a.Pages = make([][]any, 10) 17 | for i := range 10 { 18 | page := make([]any, 10) 19 | for j := range len(page) { 20 | page[j] = Record{ 21 | CreatedAt: time.Now(), 22 | TransactionNumber: (j * i) + i, 23 | AmountDollars: math.Round(rand.Float64()*10000) / 100, 24 | } 25 | } 26 | a.Pages[i] = page 27 | } 28 | return a 29 | } 30 | 31 | func (a *MockAPI[any]) FetchPageAny(page int) (results []any, err error) { 32 | if page < 1 { 33 | return nil, fmt.Errorf("page number must be positive. Received %d", page) 34 | } else if page > len(a.Pages) { 35 | return nil, nil 36 | } 37 | return a.Pages[page-1], nil 38 | } 39 | 40 | // ExampleFanOut is the `Main` function for our example. Here we will build and run our pipeline. 41 | func ExampleFanOut() { 42 | 43 | // First we create a generator that wraps our API service. 44 | gen := func() func() ([]any, bool, error) { 45 | api := NewMockAPIAny() 46 | var page int 47 | return func() ([]any, bool, error) { 48 | page++ 49 | results, err := api.FetchPage(page) 50 | if err != nil { 51 | return nil, false, err 52 | } 53 | if len(results) == 0 { 54 | return results, false, nil 55 | } else { 56 | return results, true, nil 57 | } 58 | } 59 | } 60 | 61 | // Next, we create a simple time stamping function for our middle stage 62 | wrap := func(inbound []any) ([]any, error) { 63 | wrappedData := make([]any, len(inbound)) 64 | for i, data := range inbound { 65 | wrappedData[i] = map[string]any{ 66 | "processedAt": time.Now(), 67 | "data": data, 68 | } 69 | } 70 | return wrappedData, nil 71 | } 72 | 73 | // init mock db 74 | db := NewMockDB() 75 | 76 | // Finally, the interesting part. In our store function, we chunk the records 77 | // and write all chunks in parallel. 78 | fanOutStore := func(inbound []any) ([]any, error) { 79 | chunks := gliter.ChunkBy(inbound, 5) 80 | funcs := make([]func() (any, error), len(chunks)) 81 | for i, chunk := range chunks { 82 | funcs[i] = func() (any, error) { 83 | // We only care about the error 84 | return nil, db.BatchAdd(chunk) 85 | } 86 | } 87 | _, err := gliter.InParallel(funcs) 88 | return nil, err 89 | } 90 | 91 | // assemble and run the pipeline 92 | _, err := gliter.NewPipeline(gen()). 93 | Config(gliter.PLConfig{LogCount: true}). 94 | Stage(wrap). 95 | Stage(fanOutStore). 96 | Run() 97 | 98 | if err != nil { 99 | panic(err) 100 | } 101 | 102 | // Unlike in pipeline_example.go, this pipeline only processes 10pages * 10records = 100 records (no forking here). 103 | // Even so, the final stage called 1:0:0 is writing two concurrent chunks of 5records each on every invocation... 104 | // so while GEN, 0:0:0, and 1:0:0 each have only 10 invocations, 1:0:0 has spawned a total of 20 child go routines. 105 | // +-------+-------+----------------------+ 106 | // | node | count | value | 107 | // +-------+-------+----------------------+ 108 | // | GEN | 10 | [{2025-02-03 12:45.. | 109 | // | 0:0:0 | 10 | [map[data:{2025-02.. | 110 | // | 1:0:0 | 10 | [] | 111 | // +-------+-------+----------------------+ 112 | 113 | // That's it! 114 | } 115 | -------------------------------------------------------------------------------- /diag/flow-chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arrno/gliter/0ccc578ee9c442d00daea0a254d4e5b40bdb9cba/diag/flow-chart.png -------------------------------------------------------------------------------- /diag/small-chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arrno/gliter/0ccc578ee9c442d00daea0a254d4e5b40bdb9cba/diag/small-chart.png -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/arrno/gliter 2 | 3 | go 1.22.0 4 | 5 | require github.com/stretchr/testify v1.10.0 6 | 7 | require ( 8 | github.com/davecgh/go-spew v1.1.1 // indirect 9 | github.com/pmezard/go-difflib v1.0.0 // indirect 10 | gopkg.in/yaml.v3 v3.0.1 // indirect 11 | ) 12 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 4 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 5 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 6 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 7 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 8 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 9 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 10 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 11 | -------------------------------------------------------------------------------- /list.go: -------------------------------------------------------------------------------- 1 | package gliter 2 | 3 | import "slices" 4 | 5 | type Iter[T any] struct { 6 | items []T 7 | } 8 | 9 | func NewList[T any]() *Iter[T] { 10 | return &Iter[T]{} 11 | } 12 | 13 | func MakeList[T any](length uint, capacity uint) *Iter[T] { 14 | return &Iter[T]{ 15 | items: make([]T, length, capacity), 16 | } 17 | } 18 | 19 | func List[T any](vals ...T) *Iter[T] { 20 | return &Iter[T]{ 21 | items: vals, 22 | } 23 | } 24 | 25 | func SliceToList[T any](s []T) *Iter[T] { 26 | return &Iter[T]{ 27 | items: s, 28 | } 29 | } 30 | 31 | func (l Iter[T]) Unwrap() []T { 32 | return l.items 33 | } 34 | 35 | func (l *Iter[T]) Iter() []T { 36 | return l.items 37 | } 38 | 39 | func (l *Iter[T]) Push(val T) { 40 | l.items = append(l.items, val) 41 | } 42 | 43 | func (l *Iter[T]) Pop() T { 44 | var val T 45 | if len(l.items) > 0 { 46 | val = l.items[len(l.items)-1] 47 | l.items = l.items[:len(l.items)-1] 48 | } 49 | return val 50 | } 51 | 52 | // FPop inserts at the front of the list. 53 | func (l *Iter[T]) FPop() T { 54 | var val T 55 | if len(l.items) > 0 { 56 | val = l.items[0] 57 | l.items = l.items[1:len(l.items)] 58 | } 59 | return val 60 | 61 | } 62 | 63 | // Map transforms the values of Iter without changing the type. To change 64 | // the type, see non-receiver alternative 'Map' function. 65 | func (l Iter[T]) Map(f func(val T) T) Iter[T] { 66 | for i, item := range l.items { 67 | l.items[i] = f(item) 68 | } 69 | return l 70 | } 71 | 72 | func (l Iter[T]) Filter(f func(val T) bool) *Iter[T] { 73 | filtered := []T{} 74 | for _, item := range l.items { 75 | if f(item) { 76 | filtered = append(filtered, item) 77 | } 78 | } 79 | return List(filtered...) 80 | } 81 | 82 | func (l Iter[T]) Find(f func(val T) bool) (T, bool) { 83 | for _, item := range l.items { 84 | if f(item) { 85 | return item, true 86 | } 87 | } 88 | var zero T 89 | return zero, false 90 | } 91 | 92 | func (l Iter[T]) Reduce(f func(acc *T, val T)) *T { 93 | acc := new(T) 94 | for _, item := range l.items { 95 | f(acc, item) 96 | } 97 | return acc 98 | } 99 | 100 | func (l Iter[T]) Reverse() Iter[T] { 101 | for i, j := 0, len(l.items)-1; i < j; i, j = i+1, j-1 { 102 | l.items[i], l.items[j] = l.items[j], l.items[i] 103 | } 104 | return l 105 | } 106 | 107 | func (l Iter[T]) Len() int { 108 | return len(l.items) 109 | } 110 | 111 | func (l Iter[T]) At(index int) T { 112 | return l.items[index] 113 | } 114 | 115 | func (l Iter[T]) Delete(index int) Iter[T] { 116 | ret := make([]T, 0) 117 | ret = append(ret, l.items[:index]...) 118 | l.items = append(ret, l.items[index+1:]...) 119 | return l 120 | } 121 | 122 | func (l Iter[T]) Insert(index int, val T) Iter[T] { 123 | l.items = slices.Insert(l.items, index, val) 124 | return l 125 | } 126 | 127 | func (l Iter[T]) Slice(start, stop int) Iter[T] { 128 | l.items = l.items[start:stop] 129 | return l 130 | } 131 | 132 | // Map converts and transforms Iter[T] to Iter[R]. 133 | func Map[T, R any](list Iter[T], f func(T) R) *Iter[R] { 134 | result := make([]R, len(list.items)) 135 | for i, item := range list.items { 136 | result[i] = f(item) 137 | } 138 | return List(result...) 139 | } 140 | -------------------------------------------------------------------------------- /list_test.go: -------------------------------------------------------------------------------- 1 | package gliter 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | type Thing struct { 11 | Num int 12 | Char rune 13 | } 14 | 15 | func TestList(t *testing.T) { 16 | list := List(0, 1, 2, 3, 4) 17 | list.Pop() // removes/returns `4` 18 | list.Push(8) // appends `8` 19 | assert.True(t, reflect.DeepEqual([]int{0, 1, 2, 3, 8}, list.items)) 20 | 21 | value := List(0, 1, 2, 3, 4). 22 | Filter(func(i int) bool { 23 | return i%2 == 0 24 | }). // []int{0, 2, 4} 25 | Map(func(val int) int { 26 | return val * 2 27 | }). // []int{0, 4, 8} 28 | Reduce(func(acc *int, val int) { 29 | *acc += val 30 | }) // 12 31 | assert.Equal(t, 12, *value) 32 | 33 | item, ok := List(Thing{1, 'a'}, Thing{2, 'b'}, Thing{3, 'c'}).Find(func(val Thing) bool { 34 | return val.Num > 1 35 | }) 36 | assert.True(t, ok) 37 | assert.True(t, reflect.DeepEqual(Thing{2, 'b'}, item)) 38 | 39 | _, ok = List(Thing{1, 'a'}, Thing{2, 'b'}, Thing{3, 'c'}).Find(func(val Thing) bool { 40 | return val.Num == 7 41 | }) 42 | assert.False(t, ok) 43 | 44 | assert.Equal(t, 3, List(1, 2, 3, 4, 5).At(2)) 45 | 46 | assert.True(t, reflect.DeepEqual( 47 | List(1, 2, 3, 4, 5).Reverse().Unwrap(), 48 | []int{5, 4, 3, 2, 1})) 49 | 50 | assert.True(t, reflect.DeepEqual( 51 | List(1, 2, 3, 4, 5).Delete(1).Unwrap(), 52 | []int{1, 3, 4, 5})) 53 | 54 | assert.True(t, reflect.DeepEqual( 55 | List(1, 2, 3, 4, 5). 56 | Insert(1, 7). 57 | Insert(0, 0). 58 | Unwrap(), 59 | []int{0, 1, 7, 2, 3, 4, 5})) 60 | 61 | newList := Map(*List(1, 2, 3), func(i int) Thing { 62 | return Thing{ 63 | Num: i, 64 | Char: 'a', 65 | } 66 | }) 67 | assert.True(t, reflect.DeepEqual([]Thing{{1, 'a'}, {2, 'a'}, {3, 'a'}}, newList.Unwrap())) 68 | } 69 | -------------------------------------------------------------------------------- /pipeline.go: -------------------------------------------------------------------------------- 1 | package gliter 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "sync" 7 | ) 8 | 9 | type stageType int 10 | 11 | const ( 12 | FORK stageType = iota 13 | THROTTLE 14 | BATCH 15 | BUFFER 16 | MERGE 17 | OPTION 18 | ) 19 | 20 | var ( 21 | ErrNoGenerator error = errors.New("Pipeline run error. Invalid pipeline: No generator provided") 22 | ErrEmptyStage error = errors.New("Pipeline run error. Empty stage: No functions provided in stage") 23 | ErrEmptyThrottle error = errors.New("Pipeline run error. Empty throttle: Throttle must be a positive value") 24 | ErrInvalidThrottle error = errors.New("Pipeline run error. Invalid throttle: Throttle value cannot be higher than channel count.") 25 | ErrNilFunc error = errors.New("Pipeline run error. Invalid stage: Nil function.") 26 | ) 27 | 28 | // PLConfig controls limited pipeline behavior. 29 | type PLConfig struct { 30 | LogAll bool 31 | LogEmit bool 32 | LogCount bool 33 | ReturnCount bool 34 | LogStep bool 35 | } 36 | 37 | func (c PLConfig) keepCount() bool { 38 | return c.LogAll || c.LogCount || c.LogStep || c.ReturnCount 39 | } 40 | 41 | type stage[T any] struct { 42 | stageType stageType 43 | handlers []func(T) (T, error) 44 | } 45 | 46 | type batch[T any] struct { 47 | stage int 48 | size int 49 | handler func([]T) ([]T, error) 50 | } 51 | 52 | // Pipeline spawns threads for all stage functions and orchestrates channel signals between them. 53 | type Pipeline[T any] struct { 54 | generator func() (T, bool, error) 55 | stages []stage[T] 56 | batches map[int]batch[T] 57 | merges map[int]func([]T) ([]T, error) 58 | buffers map[int]uint 59 | optionForks map[int][]func(T) (T, error) 60 | tally <-chan int 61 | config PLConfig 62 | } 63 | 64 | func NewPipeline[T any](gen func() (T, bool, error)) *Pipeline[T] { 65 | return &Pipeline[T]{ 66 | batches: map[int]batch[T]{}, 67 | buffers: map[int]uint{}, 68 | merges: map[int]func(data []T) ([]T, error){}, 69 | optionForks: map[int][]func(T) (T, error){}, 70 | generator: gen, 71 | } 72 | } 73 | 74 | func (p *Pipeline[T]) Tally() chan<- int { 75 | tally := make(chan int) 76 | p.tally = tally 77 | return tally 78 | } 79 | 80 | // Stage pushes a new stage onto the pipeline. A stage should have > 0 transform functions. Each 81 | // transform function beyond the first forks the pipeline into an additional downstream branch. 82 | func (p *Pipeline[T]) Stage(fs ...func(data T) (T, error)) *Pipeline[T] { 83 | st := stage[T]{ 84 | stageType: FORK, 85 | handlers: fs, 86 | } 87 | p.stages = append(p.stages, st) 88 | return p 89 | } 90 | 91 | // Batch pushes a special batch stage onto the pipeline of size n. A batch allows you to operate on a set of items. 92 | // This is helpful for expensive operations such as DB writes. 93 | func (p *Pipeline[T]) Batch(n int, f func(set []T) ([]T, error)) *Pipeline[T] { 94 | p.batches[len(p.stages)] = batch[T]{stage: len(p.stages), size: n, handler: f} 95 | p.stages = append(p.stages, stage[T]{stageType: BATCH, handlers: make([]func(data T) (T, error), 1)}) 96 | return p 97 | } 98 | 99 | // Buffer pushes a special buffer stage onto the pipeline of size n. A buffer stage is effectively a buffered channel of 100 | // size n in between the previous and next stage. 101 | func (p *Pipeline[T]) Buffer(n uint) *Pipeline[T] { 102 | p.buffers[len(p.stages)] = n 103 | p.stages = append(p.stages, stage[T]{stageType: BUFFER, handlers: make([]func(data T) (T, error), 1)}) 104 | return p 105 | } 106 | 107 | // Throttle pushes a special throttle stage onto the pipeline. A throttle stage will converge all upstream 108 | // branches into n downstream branches. 109 | func (p *Pipeline[T]) Throttle(n uint) *Pipeline[T] { 110 | if n < 1 { 111 | n = 1 112 | } 113 | p.stages = append(p.stages, stage[T]{stageType: THROTTLE, handlers: make([]func(data T) (T, error), n)}) 114 | return p 115 | } 116 | 117 | // Merge pushes a special merge stage onto the pipeline. A merge stage will merge all upstream 118 | // branches into a single downstream branch with a merge function. Output is converted back into a stream. 119 | func (p *Pipeline[T]) Merge(f func(data []T) ([]T, error)) *Pipeline[T] { 120 | p.merges[len(p.stages)] = f 121 | p.stages = append(p.stages, stage[T]{stageType: MERGE, handlers: make([]func(data T) (T, error), 1)}) 122 | return p 123 | } 124 | 125 | // Option pushes a special option stage onto the pipeline. An option stage is effectively a buffer 126 | // of size N where N is the number of handler functions. A record emitted from an upstream branch 127 | // has an equal chance of entering any one of the option functions which are ready to receive. 128 | func (p *Pipeline[T]) Option(fs ...func(data T) (T, error)) *Pipeline[T] { 129 | p.optionForks[len(p.stages)] = fs 130 | p.stages = append(p.stages, stage[T]{stageType: OPTION, handlers: make([]func(data T) (T, error), 1)}) 131 | return p 132 | } 133 | 134 | func (p *Pipeline[T]) Config(config PLConfig) *Pipeline[T] { 135 | p.config = config 136 | return p 137 | } 138 | 139 | func (p *Pipeline[T]) handleLog(val T) { 140 | if p.config.LogAll || p.config.LogEmit { 141 | fmt.Printf("Emit -> %v\n", val) 142 | } 143 | } 144 | 145 | func (p *Pipeline[T]) handleBufferFunc( 146 | id string, 147 | inChan <-chan T, 148 | index int, 149 | done <-chan interface{}, 150 | wg *sync.WaitGroup, 151 | stepSignal chan<- any, 152 | stepDone <-chan any, 153 | ) (outChan chan T, errChan chan error, node *PLNode[T]) { 154 | 155 | wg.Add(1) 156 | errChan = make(chan error) 157 | outChan = make(chan T, p.buffers[index]) 158 | var val T 159 | 160 | node = NewPLNodeAs(id, val) 161 | keepCount := p.config.keepCount() 162 | if stepDone != nil { 163 | done = Any(done, stepDone) 164 | } 165 | 166 | go func() { 167 | defer func() { 168 | close(outChan) 169 | close(errChan) 170 | wg.Done() 171 | }() 172 | 173 | for { 174 | val, ok := ReadOrDone(inChan, done) 175 | if !ok { 176 | return 177 | } 178 | if keepCount { 179 | node.IncAs(val) 180 | } 181 | if stepSignal != nil { 182 | var T any 183 | if !WriteOrDone(T, stepSignal, done) { 184 | return 185 | } 186 | } 187 | if !WriteOrDone(val, outChan, done) { 188 | return 189 | } 190 | } 191 | }() 192 | 193 | return 194 | } 195 | 196 | func (p *Pipeline[T]) handleStageFunc( 197 | id string, 198 | inChan <-chan T, 199 | f func(data T) (T, error), 200 | done <-chan interface{}, 201 | wg *sync.WaitGroup, 202 | stepSignal chan<- any, 203 | stepDone <-chan any, 204 | ) (outChan chan T, errChan chan error, node *PLNode[T]) { 205 | 206 | wg.Add(1) 207 | errChan = make(chan error) 208 | outChan = make(chan T) 209 | 210 | var val T 211 | node = NewPLNodeAs(id, val) 212 | keepCount := p.config.keepCount() 213 | 214 | if stepDone != nil { 215 | done = Any(done, stepDone) 216 | } 217 | 218 | go func() { 219 | defer func() { 220 | close(outChan) 221 | close(errChan) 222 | wg.Done() 223 | }() 224 | 225 | if f == nil { 226 | WriteOrDone(ErrNilFunc, errChan, done) 227 | return 228 | } 229 | 230 | for { 231 | if val, ok := ReadOrDone(inChan, done); ok { 232 | out, err := f(val) 233 | if err != nil { 234 | WriteOrDone(err, errChan, done) 235 | return 236 | } 237 | if keepCount { 238 | node.IncAs(out) 239 | } 240 | if stepSignal != nil { 241 | var T any 242 | if !WriteOrDone(T, stepSignal, done) { 243 | return 244 | } 245 | } 246 | if !WriteOrDone(out, outChan, done) { 247 | return 248 | } 249 | } else { 250 | return 251 | } 252 | } 253 | }() 254 | return 255 | } 256 | 257 | func (p *Pipeline[T]) handleBatchFunc( 258 | id string, 259 | inChan <-chan T, 260 | index int, 261 | done <-chan interface{}, 262 | wg *sync.WaitGroup, 263 | stepSignal chan<- any, 264 | stepDone <-chan any, 265 | ) (outChan chan T, errChan chan error, node *PLNode[T]) { 266 | 267 | wg.Add(1) 268 | errChan = make(chan error) 269 | outChan = make(chan T) 270 | 271 | var val T 272 | node = NewPLNodeAs(id, val) 273 | keepCount := p.config.keepCount() 274 | 275 | if stepDone != nil { 276 | done = Any(done, stepDone) 277 | } 278 | 279 | batch := p.batches[index] 280 | queue := make([]T, 0, batch.size) 281 | 282 | handleBatch := func() bool { 283 | outSet, err := batch.handler(queue) 284 | if err != nil { 285 | WriteOrDone(err, errChan, done) 286 | return false 287 | } 288 | if keepCount { 289 | node.IncAsBatch(outSet) 290 | } 291 | if stepSignal != nil { 292 | var T any 293 | if !WriteOrDone(T, stepSignal, done) { 294 | return false 295 | } 296 | } 297 | for _, out := range outSet { 298 | if !WriteOrDone(out, outChan, done) { 299 | return false 300 | } 301 | } 302 | return true 303 | } 304 | 305 | go func() { 306 | defer func() { 307 | close(outChan) 308 | close(errChan) 309 | wg.Done() 310 | }() 311 | 312 | if batch.handler == nil { 313 | WriteOrDone(ErrNilFunc, errChan, done) 314 | return 315 | } 316 | 317 | for { 318 | if val, ok := ReadOrDone(inChan, done); ok { 319 | queue = append(queue, val) 320 | if len(queue) >= batch.size { 321 | if !handleBatch() { 322 | return 323 | } 324 | queue = nil 325 | } 326 | } else { 327 | if len(queue) > 0 && !handleBatch() { 328 | return 329 | } 330 | queue = nil 331 | return 332 | } 333 | } 334 | }() 335 | 336 | return 337 | } 338 | 339 | func (p *Pipeline[T]) handleOptionFunc( 340 | id string, 341 | inChan <-chan T, 342 | index int, 343 | done <-chan interface{}, 344 | wg *sync.WaitGroup, 345 | stepSignal chan<- any, 346 | stepDone <-chan any, 347 | ) (outChan chan T, errChan chan error, node *PLNode[T]) { 348 | 349 | wg.Add(1) 350 | errChan = make(chan error) 351 | outChan = make(chan T) 352 | 353 | var val T 354 | 355 | node = NewPLNodeAs(id, val) 356 | keepCount := p.config.keepCount() 357 | if stepDone != nil { 358 | done = Any(done, stepDone) 359 | } 360 | 361 | optionFuncs := p.optionForks[index] 362 | 363 | buffer := make(chan T, len(optionFuncs)) 364 | nilFunc := false 365 | var workerWg sync.WaitGroup 366 | 367 | // When all our workers exit, close downstream channels 368 | go func() { 369 | defer wg.Done() 370 | workerWg.Wait() 371 | close(outChan) 372 | close(errChan) 373 | }() 374 | 375 | for _, f := range optionFuncs { 376 | 377 | if f == nil { 378 | nilFunc = true 379 | break 380 | } 381 | workerWg.Add(1) 382 | 383 | // worker 384 | go func(handler func(T) (T, error)) { 385 | defer workerWg.Done() 386 | for { 387 | if val, ok := ReadOrDone(buffer, done); ok { 388 | result, err := handler(val) 389 | if err != nil { 390 | WriteOrDone(err, errChan, done) 391 | return 392 | } 393 | if keepCount { 394 | node.IncAsAtomic(result) // single node multi writers 395 | } 396 | if stepSignal != nil { 397 | var T any 398 | if !WriteOrDone(T, stepSignal, done) { 399 | return 400 | } 401 | } 402 | if !WriteOrDone(result, outChan, done) { 403 | return 404 | } 405 | } else { 406 | return 407 | } 408 | } 409 | }(f) 410 | } 411 | 412 | go func() { 413 | defer func() { 414 | close(buffer) 415 | }() 416 | 417 | if nilFunc { 418 | WriteOrDone(ErrNilFunc, errChan, done) 419 | return 420 | } 421 | 422 | for { 423 | if val, ok := ReadOrDone(inChan, done); ok { 424 | if !WriteOrDone(val, buffer, done) { 425 | return 426 | } 427 | } else { 428 | return 429 | } 430 | } 431 | }() 432 | 433 | return 434 | } 435 | 436 | func (p *Pipeline[T]) handleMergeFunc( 437 | id string, 438 | inChans []chan T, 439 | index int, 440 | done <-chan interface{}, 441 | wg *sync.WaitGroup, 442 | stepSignal chan<- any, 443 | stepDone <-chan any, 444 | ) (outChan chan T, errChan chan error, node *PLNode[T]) { 445 | 446 | wg.Add(1) 447 | errChan = make(chan error) 448 | outChan = make(chan T) 449 | 450 | var val T 451 | 452 | node = NewPLNodeAs(id, val) 453 | keepCount := p.config.keepCount() 454 | if stepDone != nil { 455 | done = Any(done, stepDone) 456 | } 457 | 458 | mergeFunc := p.merges[index] 459 | queue := make([]T, 0, len(inChans)) 460 | 461 | handleBatch := func() bool { 462 | outSet, err := mergeFunc(queue) 463 | if err != nil { 464 | WriteOrDone(err, errChan, done) 465 | return false 466 | } 467 | if keepCount { 468 | node.IncAsBatch(outSet) 469 | } 470 | if stepSignal != nil { 471 | var T any 472 | if !WriteOrDone(T, stepSignal, done) { 473 | return false 474 | } 475 | } 476 | for _, out := range outSet { 477 | if !WriteOrDone(out, outChan, done) { 478 | return false 479 | } 480 | } 481 | return true 482 | } 483 | 484 | go func() { 485 | defer func() { 486 | close(outChan) 487 | close(errChan) 488 | wg.Done() 489 | }() 490 | 491 | if mergeFunc == nil { 492 | WriteOrDone(ErrNilFunc, errChan, done) 493 | return 494 | } 495 | 496 | for { 497 | // read one per chan 498 | for _, inChan := range inChans { 499 | if val, ok := ReadOrDone(inChan, done); ok { 500 | queue = append(queue, val) 501 | } else { 502 | queue = nil 503 | return 504 | } 505 | } 506 | if !handleBatch() { 507 | return 508 | } 509 | queue = nil 510 | 511 | } 512 | }() 513 | 514 | return 515 | } 516 | 517 | // Run builds and launches all the pipeline stages. 518 | func (p *Pipeline[T]) Run() ([]PLNodeCount, error) { 519 | 520 | if p.generator == nil { 521 | return nil, errors.New("pipeline run error. Invalid pipeline: No generator provided") 522 | } 523 | 524 | // Init logging helpers 525 | var val T 526 | root := NewPLNodeAs[T]("GEN", val) 527 | var stepSignal chan<- any 528 | var stepDone <-chan any 529 | 530 | if p.config.LogStep { 531 | stepSignal, stepDone = NewStepper(root).Run() 532 | defer close(stepSignal) 533 | } 534 | 535 | keepCount := p.config.keepCount() 536 | 537 | // Init async helpers 538 | done := make(chan any) 539 | var anyDone <-chan any 540 | anyDone = done 541 | 542 | if stepDone != nil { 543 | anyDone = Any(done, stepDone) 544 | } 545 | 546 | dataChan := make(chan T) 547 | errChan := make(chan error) 548 | var wg sync.WaitGroup 549 | 550 | // Init generator 551 | wg.Add(1) 552 | go func() { 553 | 554 | defer func() { 555 | close(errChan) 556 | close(dataChan) 557 | wg.Done() 558 | }() 559 | 560 | val, con, err := p.generator() 561 | if err != nil { 562 | WriteOrDone(err, errChan, done) 563 | } 564 | 565 | for con { 566 | if keepCount { 567 | root.IncAs(val) 568 | } 569 | if WriteOrDone(val, dataChan, anyDone) { 570 | val, con, err = p.generator() 571 | if err != nil { 572 | WriteOrDone(err, errChan, done) 573 | } 574 | } else { 575 | return 576 | } 577 | } 578 | }() 579 | 580 | // Stage async helpers 581 | errChans := List(errChan) 582 | prevOuts := List(dataChan) 583 | prevNodes := List(root) 584 | 585 | // Chain stages 586 | for idx, stage := range p.stages { 587 | if len(stage.handlers) == 0 { 588 | continue 589 | } 590 | 591 | size := prevOuts.Len() * len(stage.handlers) 592 | outChans := MakeList[chan T](0, uint(size)) 593 | outNodes := MakeList[*PLNode[T]](0, uint(size)) 594 | 595 | if stage.stageType == THROTTLE { 596 | // Invalid throttle length 597 | if len(stage.handlers) > prevOuts.Len() { 598 | throttleErr := make(chan error, 1) 599 | throttleErr <- ErrInvalidThrottle 600 | errChans.Push(throttleErr) 601 | break 602 | } 603 | 604 | // child nodes of throttle don't exactly line up as you would expect in node tree 605 | // that's ok so long as throttle size is smaller than previous stage. 606 | // PLNode tree is just for logging. 607 | prevOuts = SliceToList(ThrottleBy(prevOuts.Iter(), anyDone, len(stage.handlers))) 608 | for i := range len(stage.handlers) { 609 | node := NewPLNodeAs("[THROTTLE]", val) 610 | node.encap = true 611 | prevNodes.At(i).SpawnAs(node) 612 | outNodes.Push(node) 613 | } 614 | 615 | prevNodes = outNodes 616 | continue 617 | } 618 | 619 | if stage.stageType == MERGE { 620 | outChan, errChan, node := p.handleMergeFunc( 621 | "[MERGE]", 622 | prevOuts.Unwrap(), 623 | idx, 624 | anyDone, 625 | &wg, 626 | stepSignal, 627 | stepDone, 628 | ) 629 | 630 | prevNodes.At(0).SpawnAs(node) 631 | errChans.Push(errChan) 632 | prevOuts = List(outChan) 633 | prevNodes = List(node) 634 | continue 635 | } 636 | 637 | for j, po := range prevOuts.Iter() { // honor cumulative of prev forks 638 | parentNode := prevNodes.At(j) 639 | forkOut := TeeBy(po, anyDone, len(stage.handlers)) 640 | 641 | for i, f := range stage.handlers { // for current stage 642 | // var outChan, errChan, node 643 | var outChan chan T 644 | var errChan chan error 645 | var node *PLNode[T] 646 | 647 | if stage.stageType == BUFFER { 648 | outChan, errChan, node = p.handleBufferFunc( 649 | "[BUFFER]", 650 | forkOut[i], 651 | idx, 652 | anyDone, 653 | &wg, 654 | stepSignal, 655 | stepDone, 656 | ) 657 | } else if stage.stageType == BATCH { 658 | outChan, errChan, node = p.handleBatchFunc( 659 | fmt.Sprintf("%d:%d:%d", idx, j, i), 660 | forkOut[i], 661 | idx, 662 | anyDone, 663 | &wg, 664 | stepSignal, 665 | stepDone, 666 | ) 667 | } else if stage.stageType == OPTION { 668 | outChan, errChan, node = p.handleOptionFunc( 669 | fmt.Sprintf("%d:%d:%d", idx, j, i), 670 | forkOut[i], 671 | idx, 672 | anyDone, 673 | &wg, 674 | stepSignal, 675 | stepDone, 676 | ) 677 | } else { 678 | outChan, errChan, node = p.handleStageFunc( 679 | fmt.Sprintf("%d:%d:%d", idx, j, i), 680 | forkOut[i], 681 | f, 682 | anyDone, 683 | &wg, 684 | stepSignal, 685 | stepDone, 686 | ) 687 | } 688 | 689 | errChans.Push(errChan) 690 | outChans.Push(outChan) 691 | outNodes.Push(node) 692 | parentNode.SpawnAs(node) 693 | } 694 | } 695 | 696 | prevOuts = outChans 697 | prevNodes = outNodes 698 | } 699 | 700 | // Listen for errors 701 | errBuff := make(chan error, 1) 702 | for _, errChan := range errChans.Iter() { 703 | wg.Add(1) 704 | go func() { 705 | defer wg.Done() 706 | if err, ok := ReadOrDone(errChan, anyDone); ok { 707 | if WriteOrDone(err, errBuff, anyDone) { 708 | // Will only reach once since err buffer has cap of 1 709 | close(done) 710 | } 711 | } 712 | }() 713 | } 714 | 715 | // Drain end of pipeline 716 | for _, prevOut := range prevOuts.Iter() { 717 | wg.Add(1) 718 | go func() { 719 | defer wg.Done() 720 | for { 721 | if val, ok := ReadOrDone(prevOut, anyDone); ok { 722 | p.handleLog(val) 723 | } else { 724 | return 725 | } 726 | } 727 | }() 728 | } 729 | 730 | // Conditional tally 731 | var tallyChan chan int 732 | var tallyDone chan any // tally needs to stay open until all stages exit 733 | 734 | // if tally was created 735 | if p.tally != nil { 736 | tallyChan = make(chan int, 1) 737 | tallyDone = make(chan any) 738 | go func() { 739 | // anyAnyDone := Any(anyDone, tallyDone) 740 | tallyCount := 0 741 | defer func() { tallyChan <- tallyCount }() 742 | for { 743 | if val, ok := ReadOrDone(p.tally, tallyDone); ok { 744 | tallyCount += val 745 | } else { 746 | return 747 | } 748 | } 749 | }() 750 | } 751 | 752 | // Capture, display/return results 753 | tallyResult := 0 754 | wg.Wait() // wait for all stages to exit 755 | 756 | if p.tally != nil { 757 | close(tallyDone) 758 | tallyResult = <-tallyChan 759 | } 760 | 761 | var err error 762 | select { 763 | case err = <-errBuff: 764 | default: 765 | } 766 | 767 | if p.config.LogAll || p.config.LogCount { 768 | root.PrintFullBF() 769 | } 770 | if p.tally != nil { 771 | return []PLNodeCount{{NodeID: "tally", Count: tallyResult}}, err 772 | } 773 | if p.config.ReturnCount { 774 | return root.CollectCount(), err 775 | } 776 | 777 | return nil, err 778 | } 779 | -------------------------------------------------------------------------------- /pipeline_test.go: -------------------------------------------------------------------------------- 1 | package gliter 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "reflect" 7 | "sort" 8 | "strconv" 9 | "sync" 10 | "testing" 11 | 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func TestPipeline(t *testing.T) { 16 | col, exampleEnd := makeEnd() 17 | _, err := NewPipeline(exampleGen(5)). 18 | Stage( 19 | exampleMid, // branch A 20 | exampleMid, // branch B 21 | ). 22 | Stage( 23 | exampleEnd, 24 | ). 25 | Run() 26 | assert.Nil(t, err) 27 | expected := []int{4, 4, 16, 16, 36, 36, 64, 64, 100, 100} 28 | actual := col.items 29 | sort.Slice(actual, func(i, j int) bool { 30 | return actual[i] < actual[j] 31 | }) 32 | assert.True(t, reflect.DeepEqual(expected, actual)) 33 | } 34 | 35 | func TestPipelineErr(t *testing.T) { 36 | _, exampleEnd := makeEnd() 37 | _, err := NewPipeline(exampleGen(5)). 38 | Stage( 39 | exampleMid, // branch A 40 | exampleMidErr, // branch B 41 | ). 42 | Stage( 43 | exampleEnd, 44 | ). 45 | Run() 46 | assert.NotNil(t, err) 47 | 48 | // With throttle 49 | _, exampleEnd = makeEnd() 50 | _, err = NewPipeline(exampleGen(5)). 51 | Stage( 52 | exampleMid, // branch A 53 | exampleMidErr, // branch B 54 | ). 55 | Stage( 56 | exampleEnd, 57 | ). 58 | Throttle(1). 59 | Run() 60 | assert.NotNil(t, err) 61 | } 62 | 63 | func TestPipelineGenErr(t *testing.T) { 64 | for _, gen := range []func() (int, bool, error){ 65 | exampleGenErrOne(), 66 | exampleGenErrTwo(), 67 | exampleGenErrThree(), 68 | } { 69 | _, exampleEnd := makeEnd() 70 | _, err := NewPipeline(gen). 71 | Stage( 72 | exampleMid, 73 | ). 74 | Stage( 75 | exampleEnd, 76 | ). 77 | Run() 78 | assert.NotNil(t, err) 79 | } 80 | } 81 | 82 | func TestPipelineEndErr(t *testing.T) { 83 | _, endErr := makeEndErr() 84 | _, err := NewPipeline(exampleGen(5)). 85 | Stage( 86 | exampleMid, 87 | ). 88 | Stage( 89 | endErr, 90 | ). 91 | Run() 92 | assert.NotNil(t, err) 93 | 94 | _, endBatchErr := makeEndBatchErr() 95 | _, err = NewPipeline(exampleGen(5)). 96 | Batch( 97 | 10, 98 | endBatchErr, 99 | ). 100 | Run() 101 | assert.NotNil(t, err) 102 | } 103 | 104 | func TestPipelineTallyErr(t *testing.T) { 105 | _, exampleEnd := makeEnd() 106 | 107 | pipeline := NewPipeline(exampleGenErrFour()) 108 | tally := pipeline.Tally() 109 | 110 | endWithTally := func(i int) (int, error) { 111 | tally <- 1 112 | return exampleEnd(i) 113 | } 114 | 115 | _, err := pipeline. 116 | Stage( 117 | exampleMid, // branch A 118 | exampleMid, // branch B 119 | ). 120 | Stage( 121 | exampleMid, // branches A.C, B.C 122 | exampleMid, // branches A.D, B.D 123 | ). 124 | Stage( 125 | endWithTally, 126 | endWithTally, 127 | endWithTally, 128 | ). 129 | Run() 130 | 131 | assert.NotNil(t, err) 132 | } 133 | 134 | func TestPipelineFork(t *testing.T) { 135 | col, exampleEnd := makeEnd() 136 | _, err := NewPipeline(exampleGen(5)). 137 | Stage( 138 | exampleMid, // branch A 139 | exampleMid, // branch B 140 | ). 141 | Stage( 142 | exampleMid, // branches A.C, B.C 143 | exampleMid, // branches A.D, B.D 144 | exampleMid, // branches A.E, B.E 145 | ). 146 | Stage( 147 | exampleEnd, 148 | ). 149 | Run() 150 | // 1, 2, 3, 4, 5 151 | // 16, 64, 144, 246, 400 152 | assert.Nil(t, err) 153 | expected := []int{ 154 | 16, 16, 16, 16, 16, 16, 155 | 64, 64, 64, 64, 64, 64, 156 | 144, 144, 144, 144, 144, 144, 157 | 256, 256, 256, 256, 256, 256, 158 | 400, 400, 400, 400, 400, 400, 159 | } 160 | actual := col.items 161 | sort.Slice(actual, func(i, j int) bool { 162 | return actual[i] < actual[j] 163 | }) 164 | assert.True(t, reflect.DeepEqual(expected, actual)) 165 | } 166 | 167 | func TestPipelineThrottle(t *testing.T) { 168 | col, exampleEnd := makeEnd() 169 | _, err := NewPipeline(exampleGen(5)). 170 | Stage( 171 | exampleMid, // branch A 172 | exampleMid, // branch B 173 | ). 174 | Stage( 175 | exampleMid, // branches A.C, B.C 176 | exampleMid, // branches A.D, B.D 177 | exampleMid, // branches A.E, B.E 178 | ). 179 | Throttle(2). // merge into branches X, Z 180 | Stage( 181 | exampleEnd, 182 | ). 183 | Run() 184 | // 1, 2, 3, 4, 5 185 | // 16, 64, 144, 246, 400 186 | assert.Nil(t, err) 187 | expected := []int{ 188 | 16, 16, 16, 16, 16, 16, 189 | 64, 64, 64, 64, 64, 64, 190 | 144, 144, 144, 144, 144, 144, 191 | 256, 256, 256, 256, 256, 256, 192 | 400, 400, 400, 400, 400, 400, 193 | } 194 | actual := col.items 195 | sort.Slice(actual, func(i, j int) bool { 196 | return actual[i] < actual[j] 197 | }) 198 | assert.True(t, reflect.DeepEqual(expected, actual)) 199 | } 200 | 201 | func TestPipelineThrottleErr(t *testing.T) { 202 | _, exampleEnd := makeEnd() 203 | _, err := NewPipeline(exampleGen(5)). 204 | Stage( 205 | exampleMid, // branch A 206 | exampleMid, // branch B 207 | ). 208 | Throttle(3). 209 | Stage( 210 | exampleEnd, 211 | ). 212 | Run() 213 | assert.Equal(t, err, ErrInvalidThrottle) 214 | 215 | _, err = NewPipeline(exampleGen(5)). 216 | Stage( 217 | exampleMid, // branch A 218 | exampleMid, // branch B 219 | ). 220 | Stage( 221 | exampleMid, // branch A 222 | exampleMid, // branch B 223 | ). 224 | Throttle(4). 225 | Stage( 226 | exampleEnd, 227 | ). 228 | Run() 229 | assert.Nil(t, err) 230 | 231 | _, err = NewPipeline(exampleGen(5)). 232 | Stage( 233 | exampleMid, // branch A 234 | exampleMid, // branch B 235 | ). 236 | Stage( 237 | exampleMid, // branch A 238 | exampleMid, // branch B 239 | ). 240 | Throttle(5). 241 | Stage( 242 | exampleEnd, 243 | ). 244 | Run() 245 | assert.Equal(t, err, ErrInvalidThrottle) 246 | } 247 | 248 | func TestPipelineMerge(t *testing.T) { 249 | col, exampleEnd := makeNoopEnd() 250 | counts, err := NewPipeline(exampleGen(5)). 251 | Config(PLConfig{ReturnCount: true}). 252 | Stage( 253 | exampleMid, // branch A 254 | exampleMid, // branch B 255 | ). 256 | Stage( 257 | exampleMid, // branches A.C, B.C 258 | exampleMid, // branches A.D, B.D 259 | exampleMid, // branches A.E, B.E 260 | ). 261 | Merge( 262 | func(data []int) ([]int, error) { 263 | str := "" 264 | for _, n := range data { 265 | str += string([]rune(fmt.Sprintf("%d", n))[0]) 266 | } 267 | i, err := strconv.ParseInt(str, 10, 64) 268 | if err != nil { 269 | return nil, err 270 | } 271 | return []int{int(i)}, nil 272 | }, 273 | ). 274 | Stage( 275 | exampleEnd, 276 | ). 277 | Run() 278 | // 1, 2, 3, 4, 5 279 | // 4, 8, 12, 16, 20 280 | assert.Nil(t, err) 281 | expected := []int{ 282 | 111111, 283 | 111111, 284 | 222222, 285 | 444444, 286 | 888888, 287 | } 288 | actual := col.items 289 | sort.Slice(actual, func(i, j int) bool { 290 | return actual[i] < actual[j] 291 | }) 292 | assert.True(t, reflect.DeepEqual(expected, actual)) 293 | assert.Equal(t, 5, counts[len(counts)-1].Count) 294 | 295 | // With no forking 296 | col, exampleEnd = makeNoopEnd() 297 | counts, err = NewPipeline(exampleGen(5)). 298 | Config(PLConfig{ReturnCount: true}). 299 | Stage( 300 | exampleMid, // branch A 301 | ). 302 | Merge( 303 | func(items []int) ([]int, error) { 304 | sum := 0 305 | for _, item := range items { 306 | sum += item 307 | } 308 | return []int{sum}, nil 309 | }, 310 | ). 311 | Stage( 312 | exampleEnd, 313 | ). 314 | Run() 315 | 316 | expected = []int{2, 4, 6, 8, 10} 317 | 318 | actual = col.items 319 | sort.Slice(actual, func(i, j int) bool { 320 | return actual[i] < actual[j] 321 | }) 322 | assert.True(t, reflect.DeepEqual(expected, actual)) 323 | assert.Equal(t, 5, counts[len(counts)-1].Count) 324 | } 325 | 326 | func TestPipelineMergeErr(t *testing.T) { 327 | _, exampleEnd := makeNoopEnd() 328 | _, err := NewPipeline(exampleGen(5)). 329 | Stage( 330 | exampleMid, // branch A 331 | exampleMid, // branch B 332 | ). 333 | Merge( 334 | func(data []int) ([]int, error) { 335 | return nil, errors.New("Oh no") 336 | }, 337 | ). 338 | Stage( 339 | exampleEnd, 340 | ). 341 | Run() 342 | assert.NotNil(t, err) 343 | } 344 | 345 | func TestPipelineOption(t *testing.T) { 346 | _, exampleEnd := makeNoopEnd() 347 | counts, err := NewPipeline(exampleGen(5)). 348 | Config(PLConfig{ReturnCount: true}). 349 | Stage( 350 | exampleMid, // branch A 351 | exampleMid, // branch B 352 | ). 353 | Option( 354 | func(item int) (int, error) { 355 | return 1, nil 356 | }, 357 | func(item int) (int, error) { 358 | return 2, nil 359 | }, 360 | func(item int) (int, error) { 361 | return 3, nil 362 | }, 363 | ). 364 | Stage( 365 | exampleEnd, 366 | ). 367 | Run() 368 | assert.Nil(t, err) 369 | expectedNodeIds := []string{ 370 | "GEN", 371 | "0:0:0", 372 | "0:0:1", 373 | "1:0:0", 374 | "1:1:0", 375 | "2:0:0", 376 | "2:1:0", 377 | } 378 | expectedNodeCounts := []int{5, 5, 5, 5, 5, 5, 5} 379 | resultNodeIds := make([]string, 7) 380 | resultNodeCounts := make([]int, 7) 381 | for i, count := range counts { 382 | resultNodeIds[i] = count.NodeID 383 | resultNodeCounts[i] = count.Count 384 | } 385 | assert.Equal(t, expectedNodeIds, resultNodeIds) 386 | assert.Equal(t, expectedNodeCounts, resultNodeCounts) 387 | } 388 | 389 | func TestPipelineOptionErr(t *testing.T) { 390 | _, exampleEnd := makeNoopEnd() 391 | _, err := NewPipeline(exampleGen(5)). 392 | Stage( 393 | exampleMid, // branch A 394 | exampleMid, // branch B 395 | ). 396 | Option( 397 | func(data int) (int, error) { 398 | return 0, errors.New("Oh no") 399 | }, 400 | func(data int) (int, error) { 401 | return 0, errors.New("Oh no") 402 | }, 403 | ). 404 | Stage( 405 | exampleEnd, 406 | ). 407 | Run() 408 | assert.NotNil(t, err) 409 | } 410 | 411 | func TestPipelineTally(t *testing.T) { 412 | _, exampleEnd := makeEnd() 413 | 414 | pipeline := NewPipeline(exampleGen(5)) 415 | tally := pipeline.Tally() 416 | 417 | endWithTally := func(i int) (int, error) { 418 | tally <- 1 419 | return exampleEnd(i) 420 | } 421 | 422 | count, err := pipeline. 423 | Stage( 424 | exampleMid, // branch A 425 | exampleMid, // branch B 426 | ). 427 | Stage( 428 | exampleMid, // branches A.C, B.C 429 | exampleMid, // branches A.D, B.D 430 | exampleMid, // branches A.E, B.E 431 | ). 432 | Stage( 433 | endWithTally, 434 | ). 435 | Run() 436 | 437 | // 1, 2, 3, 4, 5 438 | // 16, 64, 144, 246, 400 439 | assert.Nil(t, err) 440 | assert.Equal(t, len(count), 1) 441 | assert.Equal(t, count[0].NodeID, "tally") 442 | assert.Equal(t, count[0].Count, 30) 443 | } 444 | 445 | func TestEmptyStage(t *testing.T) { 446 | count, err := NewPipeline(exampleGen(5)). 447 | Config(PLConfig{ReturnCount: true}). 448 | Stage(). 449 | Run() 450 | assert.Nil(t, err) 451 | assert.Equal(t, count[0].Count, 5) 452 | // 453 | count, err = NewPipeline(exampleGen(5)). 454 | Config(PLConfig{ReturnCount: true}). 455 | Throttle(0). 456 | Run() 457 | assert.Nil(t, err) 458 | assert.Equal(t, count[0].Count, 5) 459 | 460 | // Empty generator 461 | _, err = NewPipeline[int](nil). 462 | Config(PLConfig{ReturnCount: true}). 463 | Stage(exampleMid). 464 | Run() 465 | assert.NotNil(t, err) 466 | } 467 | 468 | func TestNilStage(t *testing.T) { 469 | _, exampleEnd := makeEnd() 470 | _, err := NewPipeline(exampleGen(5)). 471 | Stage( 472 | exampleMid, // branch A 473 | nil, 474 | ). 475 | Stage( 476 | exampleEnd, 477 | ). 478 | Run() 479 | assert.NotNil(t, err) 480 | assert.True(t, errors.Is(err, ErrNilFunc)) 481 | 482 | _, exampleEnd = makeEnd() 483 | _, err = NewPipeline(exampleGen(5)). 484 | Stage( 485 | exampleMid, // branch A 486 | exampleMid, 487 | ). 488 | Batch( 489 | 2, nil, 490 | ). 491 | Run() 492 | assert.NotNil(t, err) 493 | assert.True(t, errors.Is(err, ErrNilFunc)) 494 | 495 | _, exampleEnd = makeEnd() 496 | _, err = NewPipeline(exampleGen(5)). 497 | Stage( 498 | exampleMid, // branch A 499 | exampleMid, 500 | ). 501 | Merge(nil). 502 | Run() 503 | assert.NotNil(t, err) 504 | assert.True(t, errors.Is(err, ErrNilFunc)) 505 | 506 | _, exampleEnd = makeEnd() 507 | _, err = NewPipeline(exampleGen(5)). 508 | Stage( 509 | exampleMid, // branch A 510 | exampleMid, 511 | ). 512 | Option( 513 | exampleMid, 514 | nil, 515 | ). 516 | Run() 517 | assert.NotNil(t, err) 518 | assert.True(t, errors.Is(err, ErrNilFunc)) 519 | } 520 | 521 | func TestPipelineBatch(t *testing.T) { 522 | col, exampleEndBatch := makeEndBatch() 523 | _, err := NewPipeline(exampleGen(23)). 524 | Batch( 525 | 10, 526 | exampleEndBatch, 527 | ). 528 | Run() 529 | assert.Nil(t, err) 530 | expected := make([]int, 23) 531 | for i := range 23 { 532 | expected[i] = (i + 1) * 2 533 | } 534 | actual := col.items 535 | sort.Slice(actual, func(i, j int) bool { 536 | return actual[i] < actual[j] 537 | }) 538 | assert.True(t, reflect.DeepEqual(expected, actual)) 539 | 540 | col, exampleEnd := makeEnd() 541 | _, err = NewPipeline(exampleGen(23)). 542 | Batch( 543 | 10, 544 | exampleMidBatch, 545 | ). 546 | Stage( 547 | exampleEnd, 548 | ). 549 | Run() 550 | assert.Nil(t, err) 551 | actual = col.items 552 | sort.Slice(actual, func(i, j int) bool { 553 | return actual[i] < actual[j] 554 | }) 555 | expected = make([]int, 23) 556 | for i := range 23 { 557 | v := (i + 1) * 2 558 | expected[i] = v * v 559 | } 560 | assert.True(t, reflect.DeepEqual(expected, actual)) 561 | } 562 | 563 | func TestPipelineBatchBranch(t *testing.T) { 564 | col, exampleEnd := makeEnd() 565 | _, err := NewPipeline(exampleGen(5)). 566 | Stage( 567 | exampleMid, // branch A 568 | exampleMid, // branch B 569 | ). 570 | Batch(2, exampleMidBatch). 571 | Stage( 572 | exampleEnd, 573 | ). 574 | Run() 575 | assert.Nil(t, err) 576 | expected := []int{16, 16, 64, 64, 144, 144, 256, 256, 400, 400} 577 | actual := col.items 578 | sort.Slice(actual, func(i, j int) bool { 579 | return actual[i] < actual[j] 580 | }) 581 | assert.True(t, reflect.DeepEqual(expected, actual)) 582 | } 583 | 584 | func TestPipelineBuffer(t *testing.T) { 585 | col, exampleEnd := makeEnd() 586 | _, err := NewPipeline(exampleGen(5)). 587 | Stage( 588 | exampleMid, // branch A 589 | exampleMid, // branch B 590 | ). 591 | Buffer(5). 592 | Stage( 593 | exampleEnd, 594 | ). 595 | Run() 596 | assert.Nil(t, err) 597 | expected := []int{4, 4, 16, 16, 36, 36, 64, 64, 100, 100} 598 | actual := col.items 599 | sort.Slice(actual, func(i, j int) bool { 600 | return actual[i] < actual[j] 601 | }) 602 | assert.True(t, reflect.DeepEqual(expected, actual)) 603 | } 604 | 605 | func TestPipelineMix(t *testing.T) { 606 | _, exampleEnd := makeEnd() 607 | count, err := NewPipeline(exampleGen(5)). 608 | Config(PLConfig{ReturnCount: true}). 609 | Stage( 610 | exampleMid, // branch A 611 | exampleMid, // branch B 612 | ). 613 | Throttle(1). // merge into 1 branch 614 | Batch(5, exampleMidBatch). 615 | Buffer(2). 616 | Stage( 617 | exampleEnd, 618 | ). 619 | Run() 620 | expected := []PLNodeCount{ 621 | { 622 | NodeID: "GEN", 623 | Count: 5, // gen 5 624 | }, 625 | { 626 | NodeID: "0:0:0", 627 | Count: 5, // branch a 628 | }, 629 | { 630 | NodeID: "0:0:1", 631 | Count: 5, // branch b 632 | }, 633 | // throttle 1 634 | { 635 | NodeID: "[THROTTLE]", 636 | Count: -1, // throttle doesn't keep count 637 | }, 638 | { 639 | NodeID: "2:0:0", 640 | Count: 2, // 5 per branch is 10 batched by 5 is 2 641 | }, 642 | // buffer 643 | { 644 | NodeID: "[BUFFER]", 645 | Count: 10, 646 | }, 647 | { 648 | NodeID: "4:0:0", 649 | Count: 10, // 10 total items un-batched 650 | }, 651 | } 652 | assert.Nil(t, err) 653 | assert.True(t, reflect.DeepEqual(expected, count)) 654 | } 655 | 656 | type Collect[T any] struct { 657 | mu sync.Mutex 658 | items []T 659 | } 660 | 661 | func NewCollect[T any]() *Collect[T] { 662 | return &Collect[T]{ 663 | items: []T{}, 664 | } 665 | } 666 | 667 | func exampleGen(n int) func() (int, bool, error) { 668 | data := make([]int, n) 669 | for i := range n { 670 | data[i] = i + 1 671 | } 672 | index := -1 673 | return func() (int, bool, error) { 674 | index++ 675 | if index == len(data) { 676 | return 0, false, nil 677 | } 678 | return data[index], true, nil 679 | } 680 | } 681 | 682 | func exampleGenErrOne() func() (int, bool, error) { 683 | return func() (int, bool, error) { 684 | return 0, true, errors.New("err") 685 | } 686 | } 687 | 688 | func exampleGenErrTwo() func() (int, bool, error) { 689 | return func() (int, bool, error) { 690 | return 0, false, errors.New("err") 691 | } 692 | } 693 | 694 | func exampleGenErrThree() func() (int, bool, error) { 695 | data := []int{1, 2, 3, 4, 5} 696 | index := -1 697 | return func() (int, bool, error) { 698 | index++ 699 | if index > 1 { 700 | return 0, false, errors.New("err") 701 | } 702 | if index == len(data) { 703 | return 0, false, nil 704 | } 705 | return data[index], true, nil 706 | } 707 | } 708 | 709 | func exampleGenErrFour() func() (int, bool, error) { 710 | data := make([]int, 100) 711 | for i := range 100 { 712 | data[i] = i 713 | } 714 | index := -1 715 | return func() (int, bool, error) { 716 | index++ 717 | if index > 75 { 718 | return 0, false, errors.New("err") 719 | } 720 | if index == len(data) { 721 | return 0, false, nil 722 | } 723 | return data[index], true, nil 724 | } 725 | } 726 | 727 | func exampleMid(i int) (int, error) { 728 | return i * 2, nil 729 | } 730 | 731 | func exampleMidBatch(s []int) ([]int, error) { 732 | results := make([]int, 0, len(s)) 733 | for _, i := range s { 734 | results = append(results, i*2) 735 | } 736 | return results, nil 737 | } 738 | 739 | func makeEnd() (*Collect[int], func(i int) (int, error)) { 740 | col := NewCollect[int]() 741 | return col, func(i int) (int, error) { 742 | r := i * i 743 | col.mu.Lock() 744 | defer col.mu.Unlock() 745 | col.items = append(col.items, r) 746 | return r, nil 747 | } 748 | } 749 | 750 | func makeNoopEnd() (*Collect[int], func(i int) (int, error)) { 751 | col := NewCollect[int]() 752 | return col, func(i int) (int, error) { 753 | col.mu.Lock() 754 | defer col.mu.Unlock() 755 | col.items = append(col.items, i) 756 | return i, nil 757 | } 758 | } 759 | 760 | func makeEndErr() (*Collect[int], func(i int) (int, error)) { 761 | col := NewCollect[int]() 762 | return col, func(i int) (int, error) { 763 | if i > 2 { 764 | return 0, errors.New("err") 765 | } 766 | r := i * i 767 | col.mu.Lock() 768 | defer col.mu.Unlock() 769 | col.items = append(col.items, r) 770 | return r, nil 771 | } 772 | } 773 | 774 | func makeEndBatch() (*Collect[int], func(set []int) ([]int, error)) { 775 | col := NewCollect[int]() 776 | return col, func(set []int) ([]int, error) { 777 | for _, j := range set { 778 | r := j * 2 779 | col.mu.Lock() 780 | col.items = append(col.items, r) 781 | col.mu.Unlock() 782 | } 783 | return nil, nil 784 | } 785 | } 786 | 787 | func makeEndBatchErr() (*Collect[int], func(set []int) ([]int, error)) { 788 | col := NewCollect[int]() 789 | return col, func(set []int) ([]int, error) { 790 | for _, j := range set { 791 | if j > 2 { 792 | return nil, errors.New("err") 793 | } 794 | r := j * 2 795 | col.mu.Lock() 796 | col.items = append(col.items, r) 797 | col.mu.Unlock() 798 | } 799 | return nil, nil 800 | } 801 | } 802 | func exampleMidErr(i int) (int, error) { 803 | if i > 2 { 804 | return 0, errors.New("err") 805 | } 806 | return i * 2, nil 807 | } 808 | -------------------------------------------------------------------------------- /pipeline_tree.go: -------------------------------------------------------------------------------- 1 | package gliter 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | ) 7 | 8 | // PLNode is a representation of a pipeline node for logging/insight-tracking. 9 | type PLNode[T any] struct { 10 | mu sync.Mutex 11 | id string 12 | encap bool 13 | count uint 14 | val T 15 | children []*PLNode[T] 16 | } 17 | 18 | func NewPLNode[T any]() *PLNode[T] { 19 | return new(PLNode[T]) 20 | } 21 | 22 | func NewPLNodeAs[T any](id string, val T) *PLNode[T] { 23 | return &PLNode[T]{ 24 | id: id, 25 | val: val, 26 | } 27 | } 28 | 29 | func (n *PLNode[T]) Print() { 30 | if n.encap { 31 | fmt.Printf(n.id + "\n") 32 | return 33 | } 34 | fmt.Printf("| %s | +%d | --> %v\n", n.id, n.count, n.val) 35 | } 36 | 37 | func (n *PLNode[T]) State() (string, int, T) { 38 | if n.encap { 39 | var v T 40 | return n.id, -1, v 41 | } 42 | return n.id, int(n.count), n.val 43 | } 44 | 45 | func (n *PLNode[T]) StateArr() []string { 46 | if n.encap { 47 | return []string{n.id, "--", "--"} 48 | } 49 | return []string{n.id, fmt.Sprintf("%d", n.count), fmt.Sprintf("%v", n.val)} 50 | } 51 | 52 | func (n *PLNode[T]) Count() PLNodeCount { 53 | count := int(n.count) 54 | if n.encap { 55 | count = -1 56 | } 57 | return PLNodeCount{ 58 | NodeID: n.id, 59 | Count: count, 60 | } 61 | } 62 | 63 | // PrintFullBF will print the full PLNode tree breadth-first to stdout. 64 | func (n *PLNode[T]) PrintFullBF() { 65 | q := List[*PLNode[T]]() 66 | results := [][]string{ 67 | {"node", "count", "value"}, 68 | } 69 | dimensions := []int{len(results[0][0]), len(results[0][1]), len(results[0][2])} 70 | var collect func() 71 | collect = func() { 72 | if q.Len() == 0 { 73 | return 74 | } 75 | next := q.FPop() 76 | state := next.StateArr() 77 | for i := range len(dimensions) { 78 | dimensions[i] = min(max(dimensions[i], len(state[i])), MAX_PAD) 79 | } 80 | results = append(results, state) 81 | for _, child := range next.children { 82 | q.Push(child) 83 | } 84 | collect() 85 | } 86 | q.Push(n) 87 | collect() 88 | printSep := func() { 89 | fmt.Printf( 90 | "+%s+%s+%s+\n", 91 | sep(dimensions[0]+2), 92 | sep(dimensions[1]+2), 93 | sep(dimensions[2]+2), 94 | ) 95 | } 96 | printSep() 97 | for i, result := range results { 98 | fmt.Printf( 99 | "| %s | %s | %s |\n", 100 | pad(result[0], dimensions[0]), 101 | pad(result[1], dimensions[1]), 102 | pad(result[2], dimensions[2]), 103 | ) 104 | if i == 0 { 105 | printSep() 106 | } 107 | } 108 | printSep() 109 | } 110 | 111 | func (n *PLNode[T]) CollectCount() []PLNodeCount { 112 | q := List[*PLNode[T]]() 113 | results := []PLNodeCount{} 114 | var collect func() 115 | collect = func() { 116 | if q.Len() == 0 { 117 | return 118 | } 119 | next := q.FPop() 120 | results = append(results, next.Count()) 121 | for _, child := range next.children { 122 | q.Push(child) 123 | } 124 | collect() 125 | } 126 | q.Push(n) 127 | collect() 128 | return results 129 | } 130 | 131 | func (n *PLNode[T]) Set(val T) { 132 | n.val = val 133 | } 134 | 135 | func (n *PLNode[T]) Inc() { 136 | n.count++ 137 | } 138 | 139 | func (n *PLNode[T]) IncAs(val T) { 140 | n.val = val 141 | n.count++ 142 | } 143 | 144 | func (n *PLNode[T]) IncAsAtomic(val T) { 145 | n.mu.Lock() 146 | defer n.mu.Unlock() 147 | n.val = val 148 | n.count++ 149 | } 150 | 151 | func (n *PLNode[T]) IncAsBatch(val []T) { 152 | if len(val) > 0 { 153 | n.val = val[len(val)-1] 154 | } 155 | n.count++ 156 | } 157 | 158 | func (n *PLNode[T]) Spawn() *PLNode[T] { 159 | child := NewPLNode[T]() 160 | n.children = append(n.children, child) 161 | return child 162 | } 163 | 164 | func (n *PLNode[T]) SpawnAs(child *PLNode[T]) { 165 | n.children = append(n.children, child) 166 | } 167 | 168 | type PLNodeCount struct { 169 | NodeID string 170 | Count int 171 | } 172 | -------------------------------------------------------------------------------- /stepper.go: -------------------------------------------------------------------------------- 1 | package gliter 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | "runtime" 8 | "slices" 9 | "strings" 10 | ) 11 | 12 | // Stepper provides a central point to block all threads and await user intervention. 13 | type Stepper[T any] struct { 14 | signal <-chan any 15 | root *PLNode[T] 16 | } 17 | 18 | func NewStepper[T any](root *PLNode[T]) *Stepper[T] { 19 | s := new(Stepper[T]) 20 | s.root = root 21 | return s 22 | } 23 | 24 | // Run launches a stepper thread and provides a signal channel for external threads to write to. 25 | // If this function exists, the returned 'done' channel is closed. 26 | func (s *Stepper[T]) Run() (chan<- any, <-chan any) { 27 | signal := make(chan any) 28 | done := make(chan any) 29 | s.signal = signal 30 | go func() { 31 | defer close(done) 32 | for range s.signal { 33 | s.clearConsole() 34 | s.root.PrintFullBF() 35 | fmt.Print("Step [Y/n]:") 36 | var input string 37 | fmt.Scanln(&input) 38 | if slices.Contains([]string{"", "y"}, strings.ToLower(input)) { 39 | continue 40 | } 41 | return 42 | } 43 | }() 44 | return signal, done 45 | } 46 | 47 | func (s *Stepper[T]) clearConsole() { 48 | var cmd *exec.Cmd 49 | if runtime.GOOS == "windows" { 50 | cmd = exec.Command("cmd", "/c", "cls") 51 | } else { 52 | cmd = exec.Command("clear") 53 | } 54 | cmd.Stdout = os.Stdout 55 | cmd.Run() 56 | } 57 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package gliter 2 | 3 | import "strings" 4 | 5 | const MAX_PAD int = 20 6 | 7 | func pad(val string, dimension int) string { 8 | if len(val) == dimension { 9 | return val 10 | } else if len(val) > dimension { 11 | return string([]rune(val)[0:dimension-2]) + ".." 12 | } 13 | return val + strings.Repeat(" ", dimension-len(val)) 14 | } 15 | 16 | func sep(dimension int) string { 17 | return strings.Repeat("-", dimension) 18 | } 19 | 20 | func ChunkBy[T any](items []T, by int) [][]T { 21 | if by >= len(items) { 22 | return [][]T{items} 23 | } else if by < 1 { 24 | return nil 25 | } 26 | chunks := make([][]T, 0, int(len(items)/by)) 27 | chunkIndex := -1 28 | 29 | for i, item := range items { 30 | if i%by == 0 { 31 | // new chunk 32 | chunks = append(chunks, make([]T, 0, by)) 33 | chunkIndex++ 34 | } 35 | chunks[chunkIndex] = append(chunks[chunkIndex], item) 36 | } 37 | return chunks 38 | } 39 | 40 | func Flatten[T any](chunks [][]T) []T { 41 | if len(chunks) == 0 { 42 | return chunks[0] 43 | } 44 | results := make([]T, 0, len(chunks)*len(chunks[0])) 45 | for _, chunk := range chunks { 46 | results = append(results, chunk...) 47 | } 48 | return results 49 | } 50 | -------------------------------------------------------------------------------- /util_test.go: -------------------------------------------------------------------------------- 1 | package gliter 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestChunk(t *testing.T) { 11 | data := make([]int, 138) 12 | for i := range 138 { 13 | data[i] = i 14 | } 15 | chunks := ChunkBy(data, 20) 16 | expected := make([][]int, 7) 17 | idx := -1 18 | for i := range 7 { 19 | if i == 6 { 20 | for _ = range 18 { 21 | idx++ 22 | expected[i] = append(expected[i], idx) 23 | } 24 | } else { 25 | for _ = range 20 { 26 | idx++ 27 | expected[i] = append(expected[i], idx) 28 | } 29 | } 30 | } 31 | assert.True(t, reflect.DeepEqual(expected, chunks)) 32 | 33 | flat := Flatten(chunks) 34 | assert.True(t, reflect.DeepEqual(data, flat)) 35 | } 36 | --------------------------------------------------------------------------------