├── go.sum ├── .gitignore ├── go.mod ├── logo └── iocast.png ├── _example ├── db │ ├── go.mod │ ├── go.sum │ ├── tasks.go │ └── main.go ├── backoff │ ├── go.mod │ ├── go.sum │ ├── tasks.go │ └── main.go ├── simple │ ├── go.mod │ ├── go.sum │ ├── tasks.go │ └── main.go ├── pipeline │ ├── go.mod │ ├── go.sum │ ├── main.go │ └── tasks.go └── scheduler │ ├── go.mod │ ├── go.sum │ ├── tasks.go │ └── main.go ├── .github ├── ISSUE_TEMPLATE │ ├── custom.md │ └── feature_request.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ └── ci.yaml ├── SECURITY.md ├── db.go ├── scheduler_test.go ├── LICENSE ├── worker_test.go ├── CONTRIBUTING.md ├── worker.go ├── pipeline.go ├── pipeline_test.go ├── task_test.go ├── README.md ├── scheduler.go ├── task.go ├── CODE_OF_CONDUCT.md └── .golangci.yaml /go.sum: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/svaloumas/iocast 2 | 3 | go 1.23.4 4 | -------------------------------------------------------------------------------- /logo/iocast.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/svaloumas/iocast/HEAD/logo/iocast.png -------------------------------------------------------------------------------- /_example/db/go.mod: -------------------------------------------------------------------------------- 1 | module example 2 | 3 | go 1.23.4 4 | 5 | require github.com/svaloumas/iocast v0.3.1 6 | -------------------------------------------------------------------------------- /_example/backoff/go.mod: -------------------------------------------------------------------------------- 1 | module example 2 | 3 | go 1.23.4 4 | 5 | require github.com/svaloumas/iocast v0.3.1 6 | -------------------------------------------------------------------------------- /_example/simple/go.mod: -------------------------------------------------------------------------------- 1 | module example 2 | 3 | go 1.23.4 4 | 5 | require github.com/svaloumas/iocast v0.3.1 6 | -------------------------------------------------------------------------------- /_example/pipeline/go.mod: -------------------------------------------------------------------------------- 1 | module example 2 | 3 | go 1.23.4 4 | 5 | require github.com/svaloumas/iocast v0.3.1 6 | -------------------------------------------------------------------------------- /_example/scheduler/go.mod: -------------------------------------------------------------------------------- 1 | module example 2 | 3 | go 1.23.4 4 | 5 | require github.com/svaloumas/iocast v0.3.1 6 | -------------------------------------------------------------------------------- /_example/db/go.sum: -------------------------------------------------------------------------------- 1 | github.com/svaloumas/iocast v0.3.1 h1:Z8GOQkHcjYB/MyWDUGy7EuzqDB2xFB3N2UEPSTE3VT4= 2 | github.com/svaloumas/iocast v0.3.1/go.mod h1:27KO2w54pSIZrdhrlponlaYDADSr0caB6WgL4udHBrk= 3 | -------------------------------------------------------------------------------- /_example/simple/go.sum: -------------------------------------------------------------------------------- 1 | github.com/svaloumas/iocast v0.3.1 h1:Z8GOQkHcjYB/MyWDUGy7EuzqDB2xFB3N2UEPSTE3VT4= 2 | github.com/svaloumas/iocast v0.3.1/go.mod h1:27KO2w54pSIZrdhrlponlaYDADSr0caB6WgL4udHBrk= 3 | -------------------------------------------------------------------------------- /_example/backoff/go.sum: -------------------------------------------------------------------------------- 1 | github.com/svaloumas/iocast v0.3.1 h1:Z8GOQkHcjYB/MyWDUGy7EuzqDB2xFB3N2UEPSTE3VT4= 2 | github.com/svaloumas/iocast v0.3.1/go.mod h1:27KO2w54pSIZrdhrlponlaYDADSr0caB6WgL4udHBrk= 3 | -------------------------------------------------------------------------------- /_example/pipeline/go.sum: -------------------------------------------------------------------------------- 1 | github.com/svaloumas/iocast v0.3.1 h1:Z8GOQkHcjYB/MyWDUGy7EuzqDB2xFB3N2UEPSTE3VT4= 2 | github.com/svaloumas/iocast v0.3.1/go.mod h1:27KO2w54pSIZrdhrlponlaYDADSr0caB6WgL4udHBrk= 3 | -------------------------------------------------------------------------------- /_example/scheduler/go.sum: -------------------------------------------------------------------------------- 1 | github.com/svaloumas/iocast v0.3.1 h1:Z8GOQkHcjYB/MyWDUGy7EuzqDB2xFB3N2UEPSTE3VT4= 2 | github.com/svaloumas/iocast v0.3.1/go.mod h1:27KO2w54pSIZrdhrlponlaYDADSr0caB6WgL4udHBrk= 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/custom.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Custom issue template 3 | about: Describe this issue template's purpose here. 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | Only the last stable version at any given point. 6 | 7 | ## Reporting a Vulnerability 8 | 9 | Vulnerabilities can be disclosed via email to svaloumas@proton.me 10 | -------------------------------------------------------------------------------- /db.go: -------------------------------------------------------------------------------- 1 | package iocast 2 | 3 | import ( 4 | "encoding/json" 5 | "sync" 6 | ) 7 | 8 | // DB represents a storage. 9 | type DB interface { 10 | Write(string, Result[any]) error 11 | } 12 | 13 | type MemDB struct { 14 | db *sync.Map 15 | } 16 | 17 | // NewMemDB creates and returns a new memDB instance. 18 | func NewMemDB(db *sync.Map) DB { 19 | return &MemDB{ 20 | db: db, 21 | } 22 | } 23 | 24 | // Write stores the results to the database. 25 | func (w *MemDB) Write(id string, r Result[any]) error { 26 | data, err := json.Marshal(r) 27 | if err != nil { 28 | return err 29 | } 30 | w.db.Store(id, data) 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /scheduler_test.go: -------------------------------------------------------------------------------- 1 | package iocast 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestScheduler(t *testing.T) { 11 | p := NewWorkerPool(4, 8) 12 | p.Start(context.Background()) 13 | defer p.Stop() 14 | 15 | taskFn := NewTaskFunc(context.Background(), "args", testTaskFn) 16 | 17 | task := TaskBuilder("uuid", taskFn).Build() 18 | 19 | s := NewScheduler(p, 50*time.Millisecond) 20 | defer s.Stop() 21 | 22 | s.Dispatch() 23 | 24 | err := s.Schedule(task, time.Now().Add(50*time.Millisecond)) 25 | if err != nil { 26 | log.Printf("err: %v", err) 27 | } 28 | 29 | result := <-task.Wait() 30 | expected := "args" 31 | if result.Out != expected { 32 | t.Errorf("wrong result output: got %v want %v", result.Out, expected) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /_example/db/tasks.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | type Args struct { 9 | addr string 10 | id int 11 | } 12 | 13 | func fetchContent(addr string, id int) []byte { 14 | // do some heavy work 15 | time.Sleep(200 * time.Millisecond) 16 | return []byte("content") 17 | } 18 | 19 | func saveToDisk(content []byte) (string, error) { 20 | return "path/to/content", nil 21 | } 22 | 23 | func DownloadContent(ctx context.Context, args *Args) (string, error) { 24 | 25 | contentChan := make(chan []byte) 26 | go func() { 27 | contentChan <- fetchContent(args.addr, args.id) 28 | close(contentChan) 29 | }() 30 | select { 31 | case content := <-contentChan: 32 | return saveToDisk(content) 33 | case <-ctx.Done(): 34 | return "", ctx.Err() 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /_example/scheduler/tasks.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | type Args struct { 9 | addr string 10 | id int 11 | } 12 | 13 | func fetchContent(addr string, id int) []byte { 14 | // do some heavy work 15 | time.Sleep(200 * time.Millisecond) 16 | return []byte("content") 17 | } 18 | 19 | func saveToDisk(content []byte) (string, error) { 20 | return "path/to/content", nil 21 | } 22 | 23 | func DownloadContent(ctx context.Context, args *Args) (string, error) { 24 | 25 | contentChan := make(chan []byte) 26 | go func() { 27 | contentChan <- fetchContent(args.addr, args.id) 28 | close(contentChan) 29 | }() 30 | select { 31 | case content := <-contentChan: 32 | return saveToDisk(content) 33 | case <-ctx.Done(): 34 | return "", ctx.Err() 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /_example/simple/tasks.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | type Args struct { 9 | addr string 10 | id int 11 | } 12 | 13 | func fetchContent(addr string, id int) []byte { 14 | // do some heavy work 15 | time.Sleep(200 * time.Millisecond) 16 | return []byte("content") 17 | } 18 | 19 | func saveToDisk(content []byte) (string, error) { 20 | return "path/to/content", nil 21 | } 22 | 23 | func DownloadContent(ctx context.Context, args *Args) (string, error) { 24 | 25 | contentChan := make(chan []byte) 26 | go func() { 27 | contentChan <- fetchContent(args.addr, args.id) 28 | close(contentChan) 29 | }() 30 | select { 31 | case content := <-contentChan: 32 | return saveToDisk(content) 33 | case <-ctx.Done(): 34 | return "", ctx.Err() 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /_example/simple/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | 7 | "github.com/svaloumas/iocast" 8 | ) 9 | 10 | func main() { 11 | // create the worker pool 12 | p := iocast.NewWorkerPool(4, 8) 13 | p.Start(context.Background()) 14 | defer p.Stop() 15 | 16 | // create a task func 17 | args := &Args{addr: "http://somewhere.net", id: 1} 18 | taskFn := iocast.NewTaskFunc(context.Background(), args, DownloadContent) 19 | 20 | // create a wrapper task 21 | t := iocast.TaskBuilder("uuid", taskFn).MaxRetries(3).Build() 22 | 23 | // enqueue the task 24 | ok := p.Enqueue(t) 25 | if !ok { 26 | log.Fatal("queue is full") 27 | } 28 | 29 | m := t.Metadata() 30 | log.Printf("status: %s", m.Status) 31 | 32 | // wait for the result 33 | result := <-t.Wait() 34 | log.Printf("result: %+v\n", result) 35 | } 36 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## What type of PR is this? (check all applicable) 2 | 3 | - [ ] Refactor 4 | - [ ] Feature 5 | - [ ] Bug Fix 6 | - [ ] Optimization 7 | - [ ] Documentation Update 8 | 9 | ## Description 10 | 11 | ## Related Tickets & Documents 12 | 13 | - Related Issue # 14 | - Closes # 15 | 16 | ## QA Instructions, Screenshots 17 | 18 | _Please replace this line with instructions on how to test your changes, a note 19 | on the devices and browsers this has been tested on, as well as any relevant 20 | images for UI changes._ 21 | 22 | ## Added/updated tests? 23 | _We encourage you to keep the code coverage percentage at 80% and above._ 24 | 25 | - [ ] Yes 26 | - [ ] No, and this is why: _please replace this line with details on why tests 27 | have not been included_ 28 | - [ ] I need help with writing tests 29 | 30 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | jobs: 10 | lint: 11 | runs-on: ubuntu-24.04 12 | 13 | steps: 14 | - name: Checkout code 15 | uses: actions/checkout@v3 16 | 17 | - name: Set up Go 18 | uses: actions/setup-go@v4 19 | with: 20 | go-version: 1.23.4 21 | 22 | - name: golangci-lint 23 | uses: golangci/golangci-lint-action@v6 24 | with: 25 | version: v1.60 26 | 27 | test: 28 | runs-on: ubuntu-24.04 29 | 30 | steps: 31 | - name: Checkout code 32 | uses: actions/checkout@v3 33 | 34 | - name: Set up Go 35 | uses: actions/setup-go@v4 36 | with: 37 | go-version: 1.23.4 38 | 39 | - name: Run tests 40 | run: go test -v ./... 41 | -------------------------------------------------------------------------------- /_example/backoff/tasks.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "time" 8 | ) 9 | 10 | type Args struct { 11 | addr string 12 | id int 13 | } 14 | 15 | func fetchContent(addr string, id int) []byte { 16 | // do some heavy work 17 | time.Sleep(200 * time.Millisecond) 18 | return []byte("content") 19 | } 20 | 21 | func saveToDisk(content []byte) (string, error) { 22 | return "", errors.New("some error") 23 | } 24 | 25 | func DownloadContent(ctx context.Context, args *Args) (string, error) { 26 | fmt.Println("attempting to download content...") 27 | 28 | contentChan := make(chan []byte) 29 | go func() { 30 | contentChan <- fetchContent(args.addr, args.id) 31 | close(contentChan) 32 | }() 33 | select { 34 | case content := <-contentChan: 35 | return saveToDisk(content) 36 | case <-ctx.Done(): 37 | return "", ctx.Err() 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /_example/backoff/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "time" 7 | 8 | "github.com/svaloumas/iocast" 9 | ) 10 | 11 | func main() { 12 | // create the worker pool 13 | p := iocast.NewWorkerPool(4, 8) 14 | p.Start(context.Background()) 15 | defer p.Stop() 16 | 17 | // create a task func 18 | args := &Args{addr: "http://somewhere.net", id: 1} 19 | taskFn := iocast.NewTaskFunc(context.Background(), args, DownloadContent) 20 | 21 | // create a wrapper task 22 | t := iocast.TaskBuilder("uuid", taskFn). 23 | MaxRetries(2). 24 | BackOff([]time.Duration{2 * time.Second, 5 * time.Second}). 25 | Build() 26 | 27 | // enqueue the task 28 | ok := p.Enqueue(t) 29 | if !ok { 30 | log.Fatal("queue is full") 31 | } 32 | 33 | m := t.Metadata() 34 | log.Printf("status: %s", m.Status) 35 | 36 | // wait for the result 37 | result := <-t.Wait() 38 | log.Printf("result: %+v\n", result) 39 | } 40 | -------------------------------------------------------------------------------- /_example/scheduler/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "time" 7 | 8 | "github.com/svaloumas/iocast" 9 | ) 10 | 11 | func main() { 12 | // create the worker pool 13 | p := iocast.NewWorkerPool(4, 8) 14 | p.Start(context.Background()) 15 | defer p.Stop() 16 | 17 | // create a task func 18 | args := &Args{addr: "http://somewhere.net", id: 1} 19 | taskFn := iocast.NewTaskFunc(context.Background(), args, DownloadContent) 20 | 21 | // create a wrapper task 22 | t := iocast.TaskBuilder("uuid", taskFn).Build() 23 | 24 | // create the scheduler 25 | s := iocast.NewScheduler(p, 100*time.Millisecond) 26 | defer s.Stop() 27 | 28 | // run it 29 | s.Dispatch() 30 | 31 | // schedule the task 32 | err := s.Schedule(t, time.Now().Add(200*time.Millisecond)) 33 | if err != nil { 34 | log.Printf("err: %v", err) 35 | } 36 | 37 | // wait for the result 38 | result := <-t.Wait() 39 | log.Printf("result: %+v\n", result) 40 | } 41 | -------------------------------------------------------------------------------- /_example/db/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "sync" 7 | "time" 8 | 9 | "github.com/svaloumas/iocast" 10 | ) 11 | 12 | func main() { 13 | // create the worker pool 14 | p := iocast.NewWorkerPool(4, 8) 15 | p.Start(context.Background()) 16 | defer p.Stop() 17 | 18 | // create a task func 19 | args := &Args{addr: "http://somewhere.net", id: 1} 20 | taskFn := iocast.NewTaskFunc(context.Background(), args, DownloadContent) 21 | 22 | // create a wrapper task 23 | m := &sync.Map{} 24 | db := iocast.NewMemDB(m) 25 | t := iocast.TaskBuilder("uuid", taskFn).Database(db).Build() 26 | 27 | // enqueue the task 28 | ok := p.Enqueue(t) 29 | if !ok { 30 | log.Fatal("queue is full") 31 | } 32 | 33 | // wait for the result 34 | time.Sleep(500 * time.Millisecond) 35 | data, ok := m.Load("uuid") 36 | if !ok { 37 | log.Fatal("result not written") 38 | } 39 | bytes := data.([]byte) 40 | log.Printf("result: %+v\n", string(bytes)) 41 | } 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Stefanos Valoumas 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /worker_test.go: -------------------------------------------------------------------------------- 1 | package iocast 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | ) 7 | 8 | func TestWorkerPool(t *testing.T) { 9 | p := NewWorkerPool(1, 1) 10 | p.Start(context.Background()) 11 | defer p.Stop() 12 | 13 | p2 := NewWorkerPool(0, 0) // will refuse to enqueue 14 | p2.Start(context.Background()) 15 | defer p2.Stop() 16 | 17 | args := "test" 18 | taskFn := NewTaskFunc(context.Background(), args, testTaskFn) 19 | task := TaskBuilder("ok", taskFn).Build() 20 | task2 := TaskBuilder("full queue", taskFn).Build() 21 | 22 | tests := []struct { 23 | name string 24 | expected string 25 | p *WorkerPool 26 | }{ 27 | { 28 | "ok", 29 | args, 30 | p, 31 | }, 32 | { 33 | "full queue", 34 | args, 35 | p2, 36 | }, 37 | } 38 | 39 | for i, tt := range tests { 40 | t.Run(tt.name, func(t *testing.T) { 41 | if i == 0 { 42 | ok := tt.p.Enqueue(task) 43 | if !ok { 44 | t.Errorf("unexpected full queue") 45 | } 46 | } else { 47 | ok := tt.p.Enqueue(task2) 48 | if ok { 49 | t.Errorf("unexpected not full queue") 50 | } 51 | } 52 | 53 | if i == 0 { 54 | result := <-task.Wait() 55 | if result.Out != tt.expected { 56 | t.Errorf("unexpected result out: got %v want %v", result.Out, tt.expected) 57 | } 58 | } 59 | }) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to iocast 🚀 2 | 3 | Thank you for your interest in contributing to **iocast**! This project is young and growing, so your help means a lot. Whether it’s reporting a bug, suggesting an improvement, or contributing code, every bit helps! 💡 4 | 5 | ## Get Started 6 | 7 | 1. **Fork the repo** 8 | Click the "Fork" button on the top-right corner of the page to create your own copy of the project. 9 | 10 | 2. **Clone your fork** 11 | ```bash 12 | git clone https://github.com/your-username/iocast.git 13 | cd iocast 14 | ``` 15 | 16 | 3. **Create a new branch** 17 | ```bash 18 | git checkout -b my-new-feature 19 | ``` 20 | 21 | 4. **Make your changes** 22 | 23 | Keep the changes clean and focused. Try to follow the existing code style. 24 | 25 | 5. **Test your changes** 26 | 27 | If applicable, run the tests to make sure everything works as expected. 28 | 29 | 6. **Submit a Pull Request** 30 | 31 | Push your branch to your fork and open a PR. Don’t forget to make proper use of our Pull Request template. 32 | 33 | # Guidelines for Contributions 🛠️ 34 | 35 | * Be respectful: This project is a friendly space. Let’s keep it that way. 36 | * Keep it simple: If it’s complex, discuss it in an issue before starting. 37 | * Ask questions: New to contributing? Not sure about something? Open an issue, and let’s chat. 38 | -------------------------------------------------------------------------------- /worker.go: -------------------------------------------------------------------------------- 1 | package iocast 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "sync" 7 | ) 8 | 9 | type WorkerPool struct { 10 | queue chan Job 11 | workers int 12 | wg *sync.WaitGroup 13 | } 14 | 15 | // NewWorkerPool initializes and returns new workerpool instance. 16 | func NewWorkerPool(workers, capacity int) *WorkerPool { 17 | return &WorkerPool{ 18 | queue: make(chan Job, capacity), 19 | workers: workers, 20 | wg: &sync.WaitGroup{}, 21 | } 22 | } 23 | 24 | // Enqueue pushes a task to the queue. 25 | func (p WorkerPool) Enqueue(t Job) bool { 26 | select { 27 | case p.queue <- t: 28 | return true 29 | default: 30 | return false 31 | } 32 | } 33 | 34 | // Start starts the worker pool pattern. 35 | func (p WorkerPool) Start(ctx context.Context) { 36 | for _ = range p.workers { 37 | p.wg.Add(1) 38 | go func() { 39 | defer p.wg.Done() 40 | for { 41 | select { 42 | case j, ok := <-p.queue: 43 | if !ok { 44 | return 45 | } 46 | go func() { 47 | err := j.Write() 48 | if err != nil { 49 | log.Printf("error writing the result of task %s: %v", j.ID(), err) 50 | } 51 | }() 52 | j.Exec(ctx) 53 | case <-ctx.Done(): 54 | return 55 | } 56 | } 57 | }() 58 | } 59 | } 60 | 61 | // Stop closes the queue and the worker pool gracefully. 62 | func (p WorkerPool) Stop() { 63 | close(p.queue) 64 | // Wait for the workers to run their last tasks. 65 | p.wg.Wait() 66 | } 67 | -------------------------------------------------------------------------------- /_example/pipeline/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | 7 | "github.com/svaloumas/iocast" 8 | ) 9 | 10 | func main() { 11 | // create the worker pool 12 | q := iocast.NewWorkerPool(4, 8) 13 | q.Start(context.Background()) 14 | defer q.Stop() 15 | 16 | // create the task funcs 17 | downloadArgs := &DownloadArgs{addr: "http://somewhere.net", id: 1} 18 | downloadFn := iocast.NewTaskFunc(context.Background(), downloadArgs, DownloadContent) 19 | 20 | processArgs := &ProcessArgs{mode: "MODE_1"} 21 | processFn := iocast.NewTaskFuncWithPreviousResult(context.Background(), processArgs, ProcessContent) 22 | 23 | uploadArgs := &UploadArgs{addr: "http://storage.net/path/to/file"} 24 | uploadFn := iocast.NewTaskFuncWithPreviousResult(context.Background(), uploadArgs, UploadContent) 25 | 26 | // create the wrapper tasks 27 | downloadTask := iocast.TaskBuilder("download", downloadFn).MaxRetries(5).Build() 28 | processTask := iocast.TaskBuilder("process", processFn).MaxRetries(4).Build() 29 | uploadTask := iocast.TaskBuilder("upload", uploadFn).MaxRetries(3).Build() 30 | 31 | // create the pipeline 32 | p, err := iocast.NewPipeline("some id", downloadTask, processTask, uploadTask) 33 | if err != nil { 34 | log.Fatalf("error creating a pipeine: %s", err) 35 | } 36 | 37 | // enqueue the pipeline 38 | ok := q.Enqueue(p) 39 | if !ok { 40 | log.Fatal("queue is full") 41 | } 42 | 43 | // wait for the result 44 | result := <-p.Wait() 45 | log.Printf("result out: %+v", *result.Out) 46 | } 47 | -------------------------------------------------------------------------------- /pipeline.go: -------------------------------------------------------------------------------- 1 | package iocast 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | ) 7 | 8 | const ( 9 | minTasksNum = 2 10 | ) 11 | 12 | type Pipeline[T any] struct { 13 | id string 14 | head *Task[T] 15 | resultChan chan Result[T] 16 | } 17 | 18 | // NewPipeline links tasks together to execute them in order, returns a pipeline instance. 19 | func NewPipeline[T any](id string, tasks ...*Task[T]) (*Pipeline[T], error) { 20 | if len(tasks) < minTasksNum { 21 | return nil, errors.New("at least two tasks must be linked to create a pipeline") 22 | } 23 | head := tasks[0] 24 | for i, t := range tasks { 25 | if i < len(tasks)-1 { 26 | t.link(tasks[i+1]) 27 | } 28 | } 29 | return &Pipeline[T]{ 30 | id: id, 31 | head: head, 32 | resultChan: head.resultChan, 33 | }, nil 34 | } 35 | 36 | // Wait awaits for the final result of the pipeline (last task in the order). 37 | func (p *Pipeline[T]) Wait() <-chan Result[T] { 38 | return p.head.resultChan 39 | } 40 | 41 | // Exec executes the linked tasks of the pipeline. 42 | func (p *Pipeline[T]) Exec(ctx context.Context) { 43 | p.head.Exec(ctx) 44 | } 45 | 46 | // Write stores the results of the pipeline (head's result) to the database. 47 | func (p *Pipeline[T]) Write() error { 48 | return p.head.Write() 49 | } 50 | 51 | // ID is an ID geter. 52 | func (p *Pipeline[T]) ID() string { 53 | return p.id 54 | } 55 | 56 | // Metadata is a metadata getter. 57 | func (p *Pipeline[T]) Metadata() Metadata { 58 | p.head.mu.Lock() 59 | defer p.head.mu.Unlock() 60 | return p.head.metadata 61 | } 62 | -------------------------------------------------------------------------------- /pipeline_test.go: -------------------------------------------------------------------------------- 1 | package iocast 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | ) 7 | 8 | func testTaskPipedFn(_ context.Context, args string, previous Result[string]) (string, error) { 9 | return args + previous.Out, nil 10 | } 11 | 12 | func TestPipeline(t *testing.T) { 13 | args := "test" 14 | 15 | taskFn := NewTaskFunc(context.Background(), args, testTaskFn) 16 | task := TaskBuilder("head", taskFn).Build() 17 | 18 | taskPipedFn := NewTaskFuncWithPreviousResult(context.Background(), args, testTaskPipedFn) 19 | pipedTask := TaskBuilder("second", taskPipedFn).Build() 20 | 21 | p, err := NewPipeline("id", task, pipedTask) 22 | if err != nil { 23 | t.Errorf("NewPipeline returned unexpected error: %v", err) 24 | } 25 | 26 | go p.Exec(context.Background()) 27 | 28 | result := <-p.Wait() 29 | if result.Err != nil { 30 | t.Errorf("Wait returned unexpected result error: %v", result.Err) 31 | } 32 | expected := args + args 33 | if result.Out != expected { 34 | t.Errorf("Wait returned unexpected result output: got %v want %v", result.Out, expected) 35 | } 36 | } 37 | 38 | func TestPipelineWithLessThanTwoTasks(t *testing.T) { 39 | args := "test" 40 | 41 | taskFn := NewTaskFunc(context.Background(), args, testTaskFn) 42 | task := TaskBuilder("head", taskFn).Build() 43 | 44 | _, err := NewPipeline("id", task) 45 | expectedMsg := "at least two tasks must be linked to create a pipeline" 46 | if err == nil { 47 | t.Errorf("NewPipeline did not return expected error: got nil want %v", expectedMsg) 48 | } else if err.Error() != expectedMsg { 49 | t.Errorf("NewPipeline returned unexpected error: got %v want %v", err.Error(), expectedMsg) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /task_test.go: -------------------------------------------------------------------------------- 1 | package iocast 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "testing" 7 | ) 8 | 9 | var ( 10 | totalAttempts = 0 11 | ) 12 | 13 | func testTaskFn(_ context.Context, args string) (string, error) { 14 | return args, nil 15 | } 16 | 17 | func testFailingTaskFn(_ context.Context, _ string) (string, error) { 18 | totalAttempts++ 19 | return "", errors.New("something went wrong") 20 | } 21 | 22 | func TestTask(t *testing.T) { 23 | args := "test" 24 | 25 | taskFn := NewTaskFunc(context.Background(), args, testTaskFn) 26 | task := TaskBuilder("simple", taskFn).Build() 27 | 28 | taskFnWithRetries := NewTaskFunc(context.Background(), args, testFailingTaskFn) 29 | taskWithRetries := TaskBuilder("retries", taskFnWithRetries).MaxRetries(3).Build() 30 | 31 | tests := []struct { 32 | name string 33 | expected string 34 | job Job 35 | }{ 36 | { 37 | "simple task", 38 | args, 39 | task, 40 | }, 41 | { 42 | "task with retries", 43 | args, 44 | taskWithRetries, 45 | }, 46 | } 47 | 48 | for _, tt := range tests { 49 | t.Run(tt.name, func(t *testing.T) { 50 | tt.job.Exec(context.Background()) 51 | 52 | switch tt.name { 53 | case "simple task": 54 | result := <-task.Wait() 55 | if result.Out != args { 56 | t.Errorf("Exec returned unexpected result output: got %v want %v", result.Out, args) 57 | } 58 | case "task with retries": 59 | result := <-taskWithRetries.Wait() 60 | expectedMsg := "something went wrong" 61 | if result.Err.Error() != expectedMsg { 62 | t.Errorf("Exec returned unexpected result error: got %v want %v", result.Err.Error(), expectedMsg) 63 | } 64 | if totalAttempts != 4 { 65 | t.Error("unexpected total attempts made") 66 | } 67 | } 68 | }) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /_example/pipeline/tasks.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/svaloumas/iocast" 8 | ) 9 | 10 | type DownloadArgs struct { 11 | addr string 12 | id int 13 | } 14 | 15 | type ProcessArgs struct { 16 | mode string 17 | } 18 | 19 | type UploadArgs struct { 20 | addr string 21 | } 22 | 23 | type Output struct { 24 | path string 25 | processedPath string 26 | uploadedPath string 27 | } 28 | 29 | func fetchContent(addr string, id int) []byte { 30 | // do some heavy work 31 | time.Sleep(300 * time.Millisecond) 32 | return []byte("content") 33 | } 34 | 35 | func saveToDisk(content []byte) (string, error) { 36 | return "path/to/content", nil 37 | } 38 | 39 | func processContent(path, mode string) string { 40 | // do some heavy work 41 | time.Sleep(300 * time.Millisecond) 42 | return "path/to/processed/content" 43 | } 44 | 45 | func uploadContent(processedPath string) string { 46 | // do some heavy work 47 | time.Sleep(300 * time.Millisecond) 48 | return "path/to/uploaded/content" 49 | } 50 | 51 | func DownloadContent(ctx context.Context, args *DownloadArgs) (*Output, error) { 52 | 53 | contentChan := make(chan []byte) 54 | go func() { 55 | contentChan <- fetchContent(args.addr, args.id) 56 | close(contentChan) 57 | }() 58 | 59 | select { 60 | case content := <-contentChan: 61 | path, err := saveToDisk(content) 62 | if err != nil { 63 | return nil, err 64 | } 65 | return &Output{path: path}, nil 66 | case <-ctx.Done(): 67 | return nil, ctx.Err() 68 | } 69 | } 70 | 71 | func ProcessContent(ctx context.Context, args *ProcessArgs, previous iocast.Result[*Output]) (*Output, error) { 72 | 73 | pathChan := make(chan string) 74 | go func() { 75 | pathChan <- processContent(previous.Out.path, args.mode) 76 | close(pathChan) 77 | }() 78 | 79 | select { 80 | case processedPath := <-pathChan: 81 | return &Output{ 82 | path: previous.Out.path, 83 | processedPath: processedPath, 84 | }, nil 85 | case <-ctx.Done(): 86 | return nil, ctx.Err() 87 | } 88 | } 89 | 90 | func UploadContent(ctx context.Context, args *UploadArgs, previous iocast.Result[*Output]) (*Output, error) { 91 | 92 | pathChan := make(chan string) 93 | go func() { 94 | pathChan <- uploadContent(previous.Out.processedPath) 95 | close(pathChan) 96 | }() 97 | 98 | select { 99 | case uploadedPath := <-pathChan: 100 | return &Output{ 101 | path: previous.Out.path, 102 | processedPath: previous.Out.processedPath, 103 | uploadedPath: uploadedPath, 104 | }, nil 105 | case <-ctx.Done(): 106 | return nil, ctx.Err() 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 |

iocast

3 |

A zero-dependency async task running library that aims to be simple, easy to use and flexible.

4 |
5 | 6 |
7 | 8 |
9 | 10 |
11 | 12 | [![stars](https://img.shields.io/github/stars/svaloumas/iocast?style=social)](https://github.com/svaloumas/iocast/stargazers) 13 | [![forks](https://img.shields.io/github/forks/svaloumas/iocast?style=social)](https://github.com/svaloumas/iocast/network/members) 14 | [![issues](https://img.shields.io/github/issues/svaloumas/iocast?style=social&logo=github)](https://github.com/svaloumas/iocast/issues?q=is%3Aissue+is%3Aopen+) 15 | [![go_reportcard](https://img.shields.io/badge/go%20report-A+-brightgreen.svg?style=social&logo=github)](https://goreportcard.com/report/github.com/svaloumas/iocast) 16 |
17 | 18 | ## installation 19 | 20 | ```bash 21 | go get github.com/svaloumas/iocast 22 | ``` 23 | 24 | ## usage 25 | 26 | The module utilizes Go Generics internally, enabling the flexibility to define your custom structs to use as arguments and arbitrary result types in your tasks. 27 | 28 | ```go 29 | func DownloadContent(ctx context.Context, args *Args) (string, error) { 30 | 31 | contentChan := make(chan []byte) 32 | go func() { 33 | contentChan <- fetchContent(args.addr, args.id) 34 | close(contentChan) 35 | }() 36 | select { 37 | case content := <-contentChan: 38 | return saveToDisk(content) 39 | case <-ctx.Done(): 40 | return "", ctx.Err() 41 | } 42 | } 43 | 44 | func main() { 45 | numOfWorkers := runtime.NumCPU() 46 | p := iocast.NewWorkerPool(numOfWorkers, numOfWorkers*2) 47 | p.Start(context.Background()) 48 | defer p.Stop() 49 | 50 | args := &Args{addr: "http://somewhere.net", id: 1} 51 | taskFn := iocast.NewTaskFunc(context.Background(), args, DownloadContent) 52 | 53 | t := iocast.TaskBuilder(taskFn). 54 | MaxRetries(2). 55 | BackOff([]time.Duration{2*time.Second, 5*time.Second}). 56 | Build() 57 | 58 | p.Enqueue(t) 59 | 60 | m := t.Metadata() 61 | log.Printf("status: %s", m.Status) 62 | 63 | result := <-t.Wait() 64 | } 65 | ``` 66 | 67 | See [examples](_example/) for a detailed illustration of how to run simple tasks and linked tasks as pipelines. 68 | 69 | ## features 70 | 71 | - [x] Generic Task Arguments. Pass any built-in or custom type as an argument to your tasks. 72 | - [x] Flexible Task Results. Return any type of value from your tasks. 73 | - [x] Context Awareness. Optionally include a context when running tasks. 74 | - [x] Retry attemtps. Define the number of retry attempts for each task. 75 | - [x] Task Pipelines. Chain tasks to execute sequentially, with the option to pass the result of one task as the argument for the next. 76 | - [x] Database Interface. Use the built-in in-memory database or use custom drivers for other storage engines by implementing an one-func interface. 77 | - [x] Task Metadata. Retrieve metadata such as status, creation time, execution time, and elapsed time. Metadata is also stored with the task results. 78 | - [x] Scheduler: Schedule tasks to run at a specific timestamp. 79 | - [x] Retries backoff mechanism: Set the duration of the intervals between failed retry attempts. 80 | - [ ] Scheduler: Add support for periodic tasks. 81 | 82 | ## test 83 | 84 | ```bash 85 | go test -v ./... 86 | ``` 87 | -------------------------------------------------------------------------------- /scheduler.go: -------------------------------------------------------------------------------- 1 | package iocast 2 | 3 | import ( 4 | "errors" 5 | "log" 6 | "sync" 7 | "time" 8 | ) 9 | 10 | var ( 11 | ErrScheduledRunInThePast = errors.New("cannot schedule run in the past: when < now") 12 | ) 13 | 14 | type ScheduleDB struct { 15 | db *sync.Map 16 | } 17 | 18 | // Schedule is a task's schedule. 19 | type Schedule struct { 20 | job Job 21 | RunAt time.Time 22 | } 23 | 24 | type Scheduler struct { 25 | db *ScheduleDB 26 | wp *WorkerPool 27 | pollingInterval time.Duration 28 | done chan struct{} 29 | } 30 | 31 | // NewScheduler creates and returns a new scheduler instance. 32 | func NewScheduler(wp *WorkerPool, pollingInterval time.Duration) *Scheduler { 33 | return &Scheduler{ 34 | db: &ScheduleDB{ 35 | db: &sync.Map{}, 36 | }, 37 | wp: wp, 38 | pollingInterval: pollingInterval, 39 | done: make(chan struct{}), 40 | } 41 | } 42 | 43 | // ScheduleRun schedules a run for the task. 44 | func (s *Scheduler) Schedule(j Job, runAt time.Time) error { 45 | if err := s.validate(runAt); err != nil { 46 | return err 47 | } 48 | schedule := &Schedule{ 49 | job: j, 50 | RunAt: runAt, 51 | } 52 | return s.db.Store(j.ID(), schedule) 53 | } 54 | 55 | // Dispatch polls the databases for any due schedules and enqueues their tasks for execution. 56 | func (s *Scheduler) Dispatch() { 57 | ticker := time.NewTicker(s.pollingInterval) 58 | 59 | go func() { 60 | for { 61 | select { 62 | case <-ticker.C: 63 | s.dispatchDueTasks() 64 | case <-s.done: 65 | ticker.Stop() 66 | return 67 | } 68 | } 69 | }() 70 | } 71 | 72 | func (s *Scheduler) dispatchDueTasks() { 73 | schedules, err := s.db.FetchDue(time.Now()) 74 | if err != nil { 75 | log.Printf("failed to fetch due schedules: %v", err) 76 | return 77 | } 78 | 79 | for _, schedule := range schedules { 80 | ok := s.wp.Enqueue(schedule.job) 81 | if !ok { 82 | log.Printf("failed to enqueue task with id: %s", schedule.job.ID()) 83 | return 84 | } 85 | err = s.db.Delete(schedule.job.ID()) 86 | if err != nil { 87 | log.Printf("failed to delete due schedule: %v", err) 88 | return 89 | } 90 | } 91 | } 92 | 93 | // Stop stops the scheduler. 94 | func (s *Scheduler) Stop() { 95 | close(s.done) 96 | } 97 | 98 | func (s *Scheduler) validate(runAt time.Time) error { 99 | if runAt.Before(time.Now()) { 100 | return ErrScheduledRunInThePast 101 | } 102 | return nil 103 | } 104 | 105 | // Store stores the schedule in the database. 106 | func (m *ScheduleDB) Store(id string, s *Schedule) error { 107 | m.db.Store(id, s) 108 | return nil 109 | } 110 | 111 | // Delete removes a schedule from the database. 112 | func (m *ScheduleDB) Delete(id string) error { 113 | m.db.Delete(id) 114 | return nil 115 | } 116 | 117 | // FetchDue fetches the due schedules from the database. 118 | func (m *ScheduleDB) FetchDue(now time.Time) ([]*Schedule, error) { 119 | var dueSchedules []*Schedule 120 | m.db.Range(func(_, value any) bool { 121 | schedule, ok := value.(*Schedule) 122 | if !ok { 123 | return true // skip 124 | } 125 | 126 | runTime := time.Date( 127 | now.Year(), now.Month(), now.Day(), 128 | schedule.RunAt.Hour(), schedule.RunAt.Minute(), 129 | schedule.RunAt.Second(), 0, now.Location(), 130 | ) 131 | 132 | if now.After(runTime) { 133 | dueSchedules = append(dueSchedules, schedule) 134 | } 135 | return true 136 | }) 137 | return dueSchedules, nil 138 | } 139 | -------------------------------------------------------------------------------- /task.go: -------------------------------------------------------------------------------- 1 | package iocast 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "time" 8 | ) 9 | 10 | type status interface { 11 | status() 12 | } 13 | 14 | type taskStatus string 15 | 16 | func (taskStatus) status() {} 17 | 18 | var ( 19 | TaskStatusPending = taskStatus("PENDING") 20 | TaskStatusRunning = taskStatus("RUNNING") 21 | TaskStatusFailed = taskStatus("FAILED") 22 | TaskStatusSuccess = taskStatus("SUCCESS") 23 | ) 24 | 25 | // Job represents a task to be executed. 26 | type Job interface { 27 | ID() string 28 | Exec(context.Context) 29 | Write() error 30 | Metadata() Metadata 31 | } 32 | 33 | type Metadata struct { 34 | CreatetAt time.Time `json:"created_at"` 35 | StartedAt time.Time `json:"started_at"` 36 | Elapsed time.Duration `json:"elapsed"` 37 | Status status `json:"status"` 38 | } 39 | 40 | // Result is the output of a task's execution. 41 | type Result[T any] struct { 42 | Out T `json:"out"` 43 | Err error `json:"err"` 44 | Metadata Metadata `json:"metadata"` 45 | } 46 | 47 | type TaskFn[T any] func(previousResult Result[T]) Result[T] 48 | 49 | type Task[T any] struct { 50 | mu sync.RWMutex 51 | id string 52 | taskFn TaskFn[T] 53 | resultChan chan Result[T] 54 | next *Task[T] 55 | maxRetries int 56 | backoff []time.Duration 57 | db DB 58 | metadata Metadata 59 | } 60 | 61 | // NewTaskFunc initializes and returns a new task func. 62 | func NewTaskFunc[Arg, T any]( 63 | ctx context.Context, 64 | args Arg, 65 | fn func(ctx context.Context, args Arg) (T, error)) TaskFn[T] { 66 | return func(_ Result[T]) Result[T] { 67 | out, err := fn(ctx, args) 68 | return Result[T]{Out: out, Err: err} 69 | } 70 | } 71 | 72 | // NewTaskFuncWithPreviousResult initializes and returns a new task func that can use the precious task's result. 73 | func NewTaskFuncWithPreviousResult[Arg, T any]( 74 | ctx context.Context, 75 | args Arg, 76 | fn func(ctx context.Context, args Arg, previousResult Result[T]) (T, error)) TaskFn[T] { 77 | return func(previous Result[T]) Result[T] { 78 | out, err := fn(ctx, args, previous) 79 | return Result[T]{Out: out, Err: err} 80 | } 81 | } 82 | 83 | func (t *Task[T]) link(next *Task[T]) { 84 | t.next = next 85 | } 86 | 87 | func (t *Task[T]) markRunning() { 88 | t.mu.Lock() 89 | defer t.mu.Unlock() 90 | t.metadata.StartedAt = time.Now().UTC() 91 | t.metadata.Status = TaskStatusRunning 92 | } 93 | 94 | func (t *Task[T]) markFailed() { 95 | t.mu.Lock() 96 | defer t.mu.Unlock() 97 | t.metadata.Elapsed = time.Since(t.metadata.StartedAt) 98 | t.metadata.Status = TaskStatusFailed 99 | } 100 | 101 | func (t *Task[T]) markSuccess() { 102 | t.mu.Lock() 103 | defer t.mu.Unlock() 104 | t.metadata.Elapsed = time.Since(t.metadata.StartedAt) 105 | t.metadata.Status = TaskStatusSuccess 106 | } 107 | 108 | func (t *Task[T]) try(ctx context.Context, previous Result[T]) Result[T] { 109 | var result Result[T] 110 | 111 | t.markRunning() 112 | 113 | result = t.taskFn(previous) 114 | if result.Err == nil { 115 | t.markSuccess() 116 | return result 117 | } 118 | 119 | RETRY: 120 | for i := range t.maxRetries { 121 | var backoff time.Duration = 0 122 | if len(t.backoff) > 0 { 123 | backoff = t.backoff[i] 124 | } 125 | select { 126 | case <-time.After(backoff): 127 | result = t.taskFn(previous) 128 | if result.Err == nil { 129 | t.markSuccess() 130 | break RETRY 131 | } 132 | case <-ctx.Done(): 133 | // At least the first attempt has failed so result does exist 134 | break RETRY 135 | } 136 | } 137 | return result 138 | } 139 | 140 | // Wait blocks on the result channel of the task until it is ready. 141 | func (t *Task[T]) Wait() <-chan Result[T] { 142 | return t.resultChan 143 | } 144 | 145 | // ID is an ID geter. 146 | func (t *Task[T]) ID() string { 147 | return t.id 148 | } 149 | 150 | // Wait blocks on the result channel if there's a writer and writes the result when ready. 151 | func (t *Task[T]) Write() error { 152 | if t.db != nil { 153 | result, ok := <-t.resultChan 154 | if !ok { 155 | return nil 156 | } 157 | return t.db.Write(t.id, Result[any]{ 158 | Out: result.Out, 159 | Err: result.Err, 160 | Metadata: result.Metadata, 161 | }) 162 | } 163 | return nil 164 | } 165 | 166 | // Exec executes the task. 167 | func (t *Task[T]) Exec(ctx context.Context) { 168 | idx := 1 169 | var result Result[T] 170 | 171 | result = t.try(ctx, result) 172 | if result.Err != nil { 173 | // it's a pipeline so wrap the error 174 | if t.next != nil { 175 | result.Err = fmt.Errorf("error in task number %d: %w", idx, result.Err) 176 | } 177 | t.markFailed() 178 | t.resultChan <- result 179 | close(t.resultChan) 180 | return 181 | } 182 | for t.next != nil { 183 | idx++ 184 | 185 | result = t.next.try(ctx, result) 186 | if result.Err != nil { 187 | result.Err = fmt.Errorf("error in task number %d: %w", idx, result.Err) 188 | // mark the head of the pipeline 189 | t.markFailed() 190 | break 191 | } 192 | t.next = t.next.next 193 | } 194 | result.Metadata = t.metadata 195 | t.resultChan <- result 196 | close(t.resultChan) 197 | } 198 | 199 | // Metadata is a metadata getter. 200 | func (t *Task[T]) Metadata() Metadata { 201 | t.mu.Lock() 202 | defer t.mu.Unlock() 203 | return t.metadata 204 | } 205 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | . 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | # This code is licensed under the terms of the MIT license https://opensource.org/license/mit 2 | # Copyright (c) 2021 Marat Reymers 3 | 4 | ## Golden config for golangci-lint v1.61.0 5 | # 6 | # This is the best config for golangci-lint based on my experience and opinion. 7 | # It is very strict, but not extremely strict. 8 | # Feel free to adapt and change it for your needs. 9 | 10 | run: 11 | # Timeout for analysis, e.g. 30s, 5m. 12 | # Default: 1m 13 | timeout: 3m 14 | 15 | # This file contains only configs which differ from defaults. 16 | # All possible options can be found here https://github.com/golangci/golangci-lint/blob/master/.golangci.reference.yml 17 | linters-settings: 18 | cyclop: 19 | # The maximal code complexity to report. 20 | # Default: 10 21 | max-complexity: 30 22 | # The maximal average package complexity. 23 | # If it's higher than 0.0 (float) the check is enabled 24 | # Default: 0.0 25 | package-average: 10.0 26 | 27 | errcheck: 28 | # Report about not checking of errors in type assertions: `a := b.(MyStruct)`. 29 | # Such cases aren't reported by default. 30 | # Default: false 31 | check-type-assertions: true 32 | 33 | exhaustive: 34 | # Program elements to check for exhaustiveness. 35 | # Default: [ switch ] 36 | check: 37 | - switch 38 | - map 39 | 40 | exhaustruct: 41 | # List of regular expressions to exclude struct packages and their names from checks. 42 | # Regular expressions must match complete canonical struct package/name/structname. 43 | # Default: [] 44 | exclude: 45 | # std libs 46 | - "^net/http.Client$" 47 | - "^net/http.Cookie$" 48 | - "^net/http.Request$" 49 | - "^net/http.Response$" 50 | - "^net/http.Server$" 51 | - "^net/http.Transport$" 52 | - "^net/url.URL$" 53 | - "^os/exec.Cmd$" 54 | - "^reflect.StructField$" 55 | # public libs 56 | - "^github.com/Shopify/sarama.Config$" 57 | - "^github.com/Shopify/sarama.ProducerMessage$" 58 | - "^github.com/mitchellh/mapstructure.DecoderConfig$" 59 | - "^github.com/prometheus/client_golang/.+Opts$" 60 | - "^github.com/spf13/cobra.Command$" 61 | - "^github.com/spf13/cobra.CompletionOptions$" 62 | - "^github.com/stretchr/testify/mock.Mock$" 63 | - "^github.com/testcontainers/testcontainers-go.+Request$" 64 | - "^github.com/testcontainers/testcontainers-go.FromDockerfile$" 65 | - "^golang.org/x/tools/go/analysis.Analyzer$" 66 | - "^google.golang.org/protobuf/.+Options$" 67 | - "^gopkg.in/yaml.v3.Node$" 68 | 69 | funlen: 70 | # Checks the number of lines in a function. 71 | # If lower than 0, disable the check. 72 | # Default: 60 73 | lines: 100 74 | # Checks the number of statements in a function. 75 | # If lower than 0, disable the check. 76 | # Default: 40 77 | statements: 50 78 | # Ignore comments when counting lines. 79 | # Default false 80 | ignore-comments: true 81 | 82 | gocognit: 83 | # Minimal code complexity to report. 84 | # Default: 30 (but we recommend 10-20) 85 | min-complexity: 20 86 | 87 | gocritic: 88 | # Settings passed to gocritic. 89 | # The settings key is the name of a supported gocritic checker. 90 | # The list of supported checkers can be find in https://go-critic.github.io/overview. 91 | settings: 92 | captLocal: 93 | # Whether to restrict checker to params only. 94 | # Default: true 95 | paramsOnly: false 96 | underef: 97 | # Whether to skip (*x).method() calls where x is a pointer receiver. 98 | # Default: true 99 | skipRecvDeref: false 100 | 101 | gomodguard: 102 | blocked: 103 | # List of blocked modules. 104 | # Default: [] 105 | modules: 106 | - github.com/golang/protobuf: 107 | recommendations: 108 | - google.golang.org/protobuf 109 | reason: "see https://developers.google.com/protocol-buffers/docs/reference/go/faq#modules" 110 | - github.com/satori/go.uuid: 111 | recommendations: 112 | - github.com/google/uuid 113 | reason: "satori's package is not maintained" 114 | - github.com/gofrs/uuid: 115 | recommendations: 116 | - github.com/gofrs/uuid/v5 117 | reason: "gofrs' package was not go module before v5" 118 | 119 | govet: 120 | # Enable all analyzers. 121 | # Default: false 122 | enable-all: true 123 | # Disable analyzers by name. 124 | # Run `go tool vet help` to see all analyzers. 125 | # Default: [] 126 | disable: 127 | - fieldalignment # too strict 128 | # Settings per analyzer. 129 | settings: 130 | shadow: 131 | # Whether to be strict about shadowing; can be noisy. 132 | # Default: false 133 | strict: true 134 | 135 | inamedparam: 136 | # Skips check for interface methods with only a single parameter. 137 | # Default: false 138 | skip-single-param: true 139 | 140 | mnd: 141 | # List of function patterns to exclude from analysis. 142 | # Values always ignored: `time.Date`, 143 | # `strconv.FormatInt`, `strconv.FormatUint`, `strconv.FormatFloat`, 144 | # `strconv.ParseInt`, `strconv.ParseUint`, `strconv.ParseFloat`. 145 | # Default: [] 146 | ignored-functions: 147 | - args.Error 148 | - flag.Arg 149 | - flag.Duration.* 150 | - flag.Float.* 151 | - flag.Int.* 152 | - flag.Uint.* 153 | - os.Chmod 154 | - os.Mkdir.* 155 | - os.OpenFile 156 | - os.WriteFile 157 | - prometheus.ExponentialBuckets.* 158 | - prometheus.LinearBuckets 159 | 160 | nakedret: 161 | # Make an issue if func has more lines of code than this setting, and it has naked returns. 162 | # Default: 30 163 | max-func-lines: 0 164 | 165 | nolintlint: 166 | # Exclude following linters from requiring an explanation. 167 | # Default: [] 168 | allow-no-explanation: [funlen, gocognit, lll] 169 | # Enable to require an explanation of nonzero length after each nolint directive. 170 | # Default: false 171 | require-explanation: true 172 | # Enable to require nolint directives to mention the specific linter being suppressed. 173 | # Default: false 174 | require-specific: true 175 | 176 | perfsprint: 177 | # Optimizes into strings concatenation. 178 | # Default: true 179 | strconcat: false 180 | 181 | rowserrcheck: 182 | # database/sql is always checked 183 | # Default: [] 184 | packages: 185 | - github.com/jmoiron/sqlx 186 | 187 | sloglint: 188 | # Enforce not using global loggers. 189 | # Values: 190 | # - "": disabled 191 | # - "all": report all global loggers 192 | # - "default": report only the default slog logger 193 | # https://github.com/go-simpler/sloglint?tab=readme-ov-file#no-global 194 | # Default: "" 195 | no-global: "all" 196 | # Enforce using methods that accept a context. 197 | # Values: 198 | # - "": disabled 199 | # - "all": report all contextless calls 200 | # - "scope": report only if a context exists in the scope of the outermost function 201 | # https://github.com/go-simpler/sloglint?tab=readme-ov-file#context-only 202 | # Default: "" 203 | context: "scope" 204 | 205 | tenv: 206 | # The option `all` will run against whole test files (`_test.go`) regardless of method/function signatures. 207 | # Otherwise, only methods that take `*testing.T`, `*testing.B`, and `testing.TB` as arguments are checked. 208 | # Default: false 209 | all: true 210 | 211 | linters: 212 | disable-all: true 213 | enable: 214 | ## enabled by default 215 | - errcheck # checking for unchecked errors, these unchecked errors can be critical bugs in some cases 216 | - gosimple # specializes in simplifying a code 217 | - govet # reports suspicious constructs, such as Printf calls whose arguments do not align with the format string 218 | - ineffassign # detects when assignments to existing variables are not used 219 | - staticcheck # is a go vet on steroids, applying a ton of static analysis checks 220 | - typecheck # like the front-end of a Go compiler, parses and type-checks Go code 221 | - unused # checks for unused constants, variables, functions and types 222 | ## disabled by default 223 | - asasalint # checks for pass []any as any in variadic func(...any) 224 | - asciicheck # checks that your code does not contain non-ASCII identifiers 225 | - bidichk # checks for dangerous unicode character sequences 226 | - bodyclose # checks whether HTTP response body is closed successfully 227 | - canonicalheader # checks whether net/http.Header uses canonical header 228 | - copyloopvar # detects places where loop variables are copied (Go 1.22+) 229 | - cyclop # checks function and package cyclomatic complexity 230 | - dupl # tool for code clone detection 231 | - durationcheck # checks for two durations multiplied together 232 | - errname # checks that sentinel errors are prefixed with the Err and error types are suffixed with the Error 233 | - errorlint # finds code that will cause problems with the error wrapping scheme introduced in Go 1.13 234 | - exhaustive # checks exhaustiveness of enum switch statements 235 | - fatcontext # detects nested contexts in loops 236 | - forbidigo # forbids identifiers 237 | - funlen # tool for detection of long functions 238 | - gocheckcompilerdirectives # validates go compiler directive comments (//go:) 239 | # - gochecknoglobals # checks that no global variables exist 240 | - gochecknoinits # checks that no init functions are present in Go code 241 | - gochecksumtype # checks exhaustiveness on Go "sum types" 242 | - gocognit # computes and checks the cognitive complexity of functions 243 | - goconst # finds repeated strings that could be replaced by a constant 244 | - gocritic # provides diagnostics that check for bugs, performance and style issues 245 | - gocyclo # computes and checks the cyclomatic complexity of functions 246 | - godot # checks if comments end in a period 247 | - goimports # in addition to fixing imports, goimports also formats your code in the same style as gofmt 248 | - gomoddirectives # manages the use of 'replace', 'retract', and 'excludes' directives in go.mod 249 | - gomodguard # allow and block lists linter for direct Go module dependencies. This is different from depguard where there are different block types for example version constraints and module recommendations 250 | - goprintffuncname # checks that printf-like functions are named with f at the end 251 | - gosec # inspects source code for security problems 252 | - intrange # finds places where for loops could make use of an integer range 253 | - lll # reports long lines 254 | - loggercheck # checks key value pairs for common logger libraries (kitlog,klog,logr,zap) 255 | - makezero # finds slice declarations with non-zero initial length 256 | - mirror # reports wrong mirror patterns of bytes/strings usage 257 | - mnd # detects magic numbers 258 | - musttag # enforces field tags in (un)marshaled structs 259 | - nakedret # finds naked returns in functions greater than a specified function length 260 | - nestif # reports deeply nested if statements 261 | - nilerr # finds the code that returns nil even if it checks that the error is not nil 262 | - nilnil # checks that there is no simultaneous return of nil error and an invalid value 263 | - noctx # finds sending http request without context.Context 264 | - nolintlint # reports ill-formed or insufficient nolint directives 265 | - nonamedreturns # reports all named returns 266 | - nosprintfhostport # checks for misuse of Sprintf to construct a host with port in a URL 267 | - perfsprint # checks that fmt.Sprintf can be replaced with a faster alternative 268 | - predeclared # finds code that shadows one of Go's predeclared identifiers 269 | - promlinter # checks Prometheus metrics naming via promlint 270 | - protogetter # reports direct reads from proto message fields when getters should be used 271 | - reassign # checks that package variables are not reassigned 272 | # - revive # fast, configurable, extensible, flexible, and beautiful linter for Go, drop-in replacement of golint 273 | - rowserrcheck # checks whether Err of rows is checked successfully 274 | - sloglint # ensure consistent code style when using log/slog 275 | - spancheck # checks for mistakes with OpenTelemetry/Census spans 276 | - sqlclosecheck # checks that sql.Rows and sql.Stmt are closed 277 | - stylecheck # is a replacement for golint 278 | - tenv # detects using os.Setenv instead of t.Setenv since Go1.17 279 | - testableexamples # checks if examples are testable (have an expected output) 280 | - testifylint # checks usage of github.com/stretchr/testify 281 | # - testpackage # makes you use a separate _test package 282 | - tparallel # detects inappropriate usage of t.Parallel() method in your Go test codes 283 | - unconvert # removes unnecessary type conversions 284 | - unparam # reports unused function parameters 285 | - usestdlibvars # detects the possibility to use variables/constants from the Go standard library 286 | - wastedassign # finds wasted assignment statements 287 | - whitespace # detects leading and trailing whitespace 288 | 289 | ## you may want to enable 290 | #- decorder # checks declaration order and count of types, constants, variables and functions 291 | #- exhaustruct # [highly recommend to enable] checks if all structure fields are initialized 292 | #- gci # controls golang package import order and makes it always deterministic 293 | #- ginkgolinter # [if you use ginkgo/gomega] enforces standards of using ginkgo and gomega 294 | #- godox # detects FIXME, TODO and other comment keywords 295 | #- goheader # checks is file header matches to pattern 296 | #- inamedparam # [great idea, but too strict, need to ignore a lot of cases by default] reports interfaces with unnamed method parameters 297 | #- interfacebloat # checks the number of methods inside an interface 298 | #- ireturn # accept interfaces, return concrete types 299 | #- prealloc # [premature optimization, but can be used in some cases] finds slice declarations that could potentially be preallocated 300 | #- tagalign # checks that struct tags are well aligned 301 | #- varnamelen # [great idea, but too many false positives] checks that the length of a variable's name matches its scope 302 | #- wrapcheck # checks that errors returned from external packages are wrapped 303 | #- zerologlint # detects the wrong usage of zerolog that a user forgets to dispatch zerolog.Event 304 | 305 | ## disabled 306 | #- containedctx # detects struct contained context.Context field 307 | #- contextcheck # [too many false positives] checks the function whether use a non-inherited context 308 | #- depguard # [replaced by gomodguard] checks if package imports are in a list of acceptable packages 309 | #- dogsled # checks assignments with too many blank identifiers (e.g. x, _, _, _, := f()) 310 | #- dupword # [useless without config] checks for duplicate words in the source code 311 | #- err113 # [too strict] checks the errors handling expressions 312 | #- errchkjson # [don't see profit + I'm against of omitting errors like in the first example https://github.com/breml/errchkjson] checks types passed to the json encoding functions. Reports unsupported types and optionally reports occasions, where the check for the returned error can be omitted 313 | #- execinquery # [deprecated] checks query string in Query function which reads your Go src files and warning it finds 314 | #- exportloopref # [not necessary from Go 1.22] checks for pointers to enclosing loop variables 315 | #- forcetypeassert # [replaced by errcheck] finds forced type assertions 316 | #- gofmt # [replaced by goimports] checks whether code was gofmt-ed 317 | #- gofumpt # [replaced by goimports, gofumports is not available yet] checks whether code was gofumpt-ed 318 | #- gosmopolitan # reports certain i18n/l10n anti-patterns in your Go codebase 319 | #- grouper # analyzes expression groups 320 | #- importas # enforces consistent import aliases 321 | #- maintidx # measures the maintainability index of each function 322 | #- misspell # [useless] finds commonly misspelled English words in comments 323 | #- nlreturn # [too strict and mostly code is not more readable] checks for a new line before return and branch statements to increase code clarity 324 | #- paralleltest # [too many false positives] detects missing usage of t.Parallel() method in your Go test 325 | #- tagliatelle # checks the struct tags 326 | #- thelper # detects golang test helpers without t.Helper() call and checks the consistency of test helpers 327 | #- wsl # [too strict and mostly code is not more readable] whitespace linter forces you to use empty lines 328 | 329 | issues: 330 | # Maximum count of issues with the same text. 331 | # Set to 0 to disable. 332 | # Default: 3 333 | max-same-issues: 50 334 | 335 | exclude-rules: 336 | - source: "(noinspection|TODO)" 337 | linters: [godot] 338 | - source: "//noinspection" 339 | linters: [gocritic] 340 | - path: "_test\\.go" 341 | linters: 342 | - bodyclose 343 | - dupl 344 | - funlen 345 | - goconst 346 | - gosec 347 | - noctx 348 | - wrapcheck 349 | --------------------------------------------------------------------------------