├── .github └── workflows │ ├── go.yml │ └── goreleaser.yml ├── .gitignore ├── .goreleaser.yaml ├── LICENSE ├── README.md ├── deploy ├── grafana_config │ └── provisioning │ │ └── datasources │ │ └── datasource.yaml └── prometheus_config │ └── prometheus.yml ├── docker-compose.yaml ├── examples ├── complex-http-crawler │ ├── main.go │ └── pkg │ │ └── model │ │ ├── http.go │ │ └── task.go ├── load-s3-file-as-input │ ├── README.md │ └── main.go ├── load-s3-store-s3 │ ├── README.md │ └── main.go ├── metadata │ └── main.go ├── nopper │ └── main.go ├── prometheus │ └── main.go ├── random-error │ └── main.go ├── read-from-gzip-file │ ├── data.txt.gz │ └── main.go ├── result-channel │ └── main.go ├── simple-http-crawler │ └── main.go ├── sleeper │ └── main.go ├── stdio │ └── main.go ├── tcp-port-scanner │ ├── main.go │ ├── status.go │ └── task.go ├── ultimate-http-crawler │ └── README.md └── write-to-gzip-file │ ├── .gitignore │ └── main.go ├── go.mod ├── go.sum ├── pkg ├── runner │ └── runner.go ├── utils │ ├── capture.go │ ├── capture_test.go │ ├── channel.go │ ├── io.go │ ├── mapreduce.go │ ├── string.go │ ├── string_test.go │ ├── tee.go │ ├── timeout.go │ ├── timeout_test.go │ ├── url.go │ └── url_test.go └── version │ └── version.go ├── prometheus.go ├── scheduler.go ├── scheduler_test.go ├── status.go └── task.go /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a golang project 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go 3 | 4 | name: Go 5 | 6 | on: 7 | push: 8 | branches: [ "main" ] 9 | pull_request: 10 | branches: [ "main" ] 11 | 12 | jobs: 13 | 14 | build: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - 18 | name: Checkout 19 | uses: actions/checkout@v4 20 | - 21 | name: Set up Go 22 | uses: actions/setup-go@v5 23 | with: 24 | go-version: '1.22' 25 | 26 | - name: Build 27 | run: go build -v ./... 28 | 29 | - name: Test 30 | run: go test -v ./... 31 | 32 | - name: Run coverage 33 | run: go test -race -coverprofile=coverage.txt -covermode=atomic ./... 34 | 35 | - 36 | name: Upload coverage reports to Codecov 37 | uses: codecov/codecov-action@v4.0.1 38 | with: 39 | token: ${{ secrets.CODECOV_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/goreleaser.yml: -------------------------------------------------------------------------------- 1 | name: goreleaser 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*.*.*" 7 | workflow_dispatch: 8 | 9 | permissions: 10 | contents: write 11 | 12 | jobs: 13 | goreleaser: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - 17 | name: Checkout 18 | uses: actions/checkout@v4 19 | with: 20 | fetch-depth: 0 21 | - 22 | name: Set up Go 23 | uses: actions/setup-go@v5 24 | with: 25 | go-version: '1.22' 26 | - 27 | name: Run GoReleaser 28 | uses: goreleaser/goreleaser-action@v5 29 | with: 30 | distribution: goreleaser 31 | version: latest 32 | args: release --clean 33 | env: 34 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # If you prefer the allow list template instead of the deny list, see community template: 2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 3 | # 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Dependency directories (remove the comment below to include it) 18 | # vendor/ 19 | 20 | # Go workspace file 21 | go.work 22 | 23 | # Example data files 24 | data/ 25 | *.txt 26 | *.json 27 | 28 | # Goreleaser 29 | dist/ 30 | 31 | # Visual Studio Code 32 | .vscode/ 33 | 34 | # Environment files 35 | .env 36 | 37 | # Profile 38 | *.prof 39 | 40 | # Trace file 41 | *.trace -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | # This is an example .goreleaser.yml file with some sensible defaults. 2 | # Make sure to check the documentation at https://goreleaser.com 3 | 4 | # The lines below are called `modelines`. See `:help modeline` 5 | # Feel free to remove those if you don't want/need to use them. 6 | # yaml-language-server: $schema=https://goreleaser.com/static/schema.json 7 | # vim: set ts=2 sw=2 tw=0 fo=cnqoj 8 | 9 | version: 2 10 | 11 | before: 12 | hooks: 13 | # You may remove this if you don't use go modules. 14 | - go mod tidy 15 | # you may remove this if you don't need go generate 16 | - go generate ./... 17 | 18 | builds: 19 | - id: complex-http-crawler 20 | main: examples/complex-http-crawler/main.go 21 | env: 22 | - CGO_ENABLED=0 23 | goos: 24 | - linux 25 | goarch: 26 | - amd64 27 | ldflags: 28 | - -s -w 29 | - -X "github.com/WangYihang/gojob/pkg/version.Version={{.Version}}" 30 | - -X "github.com/WangYihang/gojob/pkg/version.Commit={{.Commit}}" 31 | - -X "github.com/WangYihang/gojob/pkg/version.Date={{.Date}}" 32 | 33 | archives: 34 | - format: tar.gz 35 | # this name template makes the OS and Arch compatible with the results of `uname`. 36 | name_template: >- 37 | {{ .ProjectName }}_ 38 | {{- title .Os }}_ 39 | {{- if eq .Arch "amd64" }}x86_64 40 | {{- else if eq .Arch "386" }}i386 41 | {{- else }}{{ .Arch }}{{ end }} 42 | {{- if .Arm }}v{{ .Arm }}{{ end }} 43 | # use zip for windows archives 44 | format_overrides: 45 | - goos: windows 46 | format: zip 47 | 48 | changelog: 49 | sort: asc 50 | use: github 51 | groups: 52 | - title: Features 53 | regexp: '^.*?feat(\([[:word:]]+\))??!?:.+$' 54 | order: 0 55 | - title: "Bug fixes" 56 | regexp: '^.*?fix(\([[:word:]]+\))??!?:.+$' 57 | order: 1 58 | - title: "Documentation" 59 | regexp: '^.*?doc?(\([[:word:]]+\))??!?:.+$' 60 | order: 2 61 | 62 | upx: 63 | - 64 | # Whether to enable it or not. 65 | enabled: true 66 | 67 | # Filter by build ID. 68 | ids: [ "complex-http-crawler"] 69 | 70 | # Compress argument. 71 | # Valid options are from '1' (faster) to '9' (better), and 'best'. 72 | compress: best 73 | 74 | # Whether to try LZMA (slower). 75 | lzma: true 76 | 77 | # Whether to try all methods and filters (slow). 78 | brute: false 79 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Yihang Wang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Go(od) Job 2 | 3 | [![Go Reference](https://pkg.go.dev/badge/github.com/WangYihang/gojob.svg)](https://pkg.go.dev/github.com/WangYihang/gojob) 4 | [![Go Report Card](https://goreportcard.com/badge/github.com/WangYihang/gojob)](https://goreportcard.com/report/github.com/WangYihang/gojob) 5 | [![codecov](https://codecov.io/gh/WangYihang/gojob/graph/badge.svg?token=FG1HT7FCKG)](https://codecov.io/gh/WangYihang/gojob) 6 | [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2FWangYihang%2Fgojob.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2FWangYihang%2Fgojob?ref=badge_shield) 7 | 8 | gojob is a simple job scheduler. 9 | 10 | ## Install 11 | 12 | ``` 13 | go get github.com/WangYihang/gojob 14 | ``` 15 | 16 | ## Usage 17 | 18 | Create a job scheduler with a worker pool of size 32. To do this, you need to implement the `Task` interface. 19 | 20 | ```go 21 | // Task is an interface that defines a task 22 | type Task interface { 23 | // Do starts the task, returns error if failed 24 | // If an error is returned, the task will be retried until MaxRetries 25 | // You can set MaxRetries by calling SetMaxRetries on the scheduler 26 | Do() error 27 | } 28 | ``` 29 | 30 | The whole [code](./examples/simple-http-crawler/main.go) looks like this (try it [online](https://go.dev/play/p/UiYextGte4v)). 31 | 32 | ```go 33 | package main 34 | 35 | import ( 36 | "fmt" 37 | "net/http" 38 | 39 | "github.com/WangYihang/gojob" 40 | ) 41 | 42 | type MyTask struct { 43 | Url string `json:"url"` 44 | StatusCode int `json:"status_code"` 45 | } 46 | 47 | func New(url string) *MyTask { 48 | return &MyTask{ 49 | Url: url, 50 | } 51 | } 52 | 53 | func (t *MyTask) Do() error { 54 | response, err := http.Get(t.Url) 55 | if err != nil { 56 | return err 57 | } 58 | t.StatusCode = response.StatusCode 59 | defer response.Body.Close() 60 | return nil 61 | } 62 | 63 | func main() { 64 | var numTotalTasks int64 = 256 65 | scheduler := gojob.New( 66 | gojob.WithNumWorkers(8), 67 | gojob.WithMaxRetries(4), 68 | gojob.WithMaxRuntimePerTaskSeconds(16), 69 | gojob.WithNumShards(4), 70 | gojob.WithShard(0), 71 | gojob.WithTotalTasks(numTotalTasks), 72 | gojob.WithStatusFilePath("status.json"), 73 | gojob.WithResultFilePath("result.json"), 74 | gojob.WithMetadataFilePath("metadata.json"), 75 | ). 76 | Start() 77 | for i := range numTotalTasks { 78 | scheduler.Submit(New(fmt.Sprintf("https://httpbin.org/task/%d", i))) 79 | } 80 | scheduler.Wait() 81 | } 82 | ``` 83 | 84 | ## Use Case 85 | 86 | ### http-crawler 87 | 88 | Let's say you have a bunch of URLs that you want to crawl and save the HTTP response to a file. You can use gojob to do that. 89 | Check [it](./examples/complex-http-crawler/) out for details. 90 | 91 | Try it out using the following command. 92 | 93 | ```bash 94 | $ go run github.com/WangYihang/gojob/examples/complex-http-crawler@latest --help 95 | Usage: 96 | main [OPTIONS] 97 | 98 | Application Options: 99 | -i, --input= input file path 100 | -o, --output= output file path 101 | -r, --max-retries= max retries (default: 3) 102 | -t, --max-runtime-per-task-seconds= max runtime per task seconds (default: 60) 103 | -n, --num-workers= number of workers (default: 32) 104 | 105 | Help Options: 106 | -h, --help Show this help message 107 | ``` 108 | 109 | ```bash 110 | $ cat urls.txt 111 | https://www.google.com/ 112 | https://www.facebook.com/ 113 | https://www.youtube.com/ 114 | ``` 115 | 116 | ``` 117 | $ go run github.com/WangYihang/gojob/examples/complex-http-crawler@latest -i input.txt -o output.txt -n 4 118 | ``` 119 | 120 | ```json 121 | $ tail -n 1 output.txt 122 | { 123 | "started_at": 1708934911909748, 124 | "finished_at": 1708934913160935, 125 | "num_tries": 1, 126 | "task": { 127 | "url": "https://www.google.com/", 128 | "http": { 129 | "request": { 130 | "method": "HEAD", 131 | "url": "https://www.google.com/", 132 | "host": "www.google.com", 133 | // details omitted for simplicity 134 | }, 135 | "response": { 136 | "status": "200 OK", 137 | "proto": "HTTP/2.0", 138 | "header": { 139 | "Alt-Svc": [ 140 | "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" 141 | ], 142 | }, 143 | // details omitted for simplicity 144 | "body": "", 145 | } 146 | } 147 | }, 148 | "error": "" 149 | } 150 | ``` 151 | 152 | ## Integration with Prometheus 153 | 154 | gojob provides metrics (`num_total`, `num_finshed`, `num_succeed`, `num_finished`) for Prometheus. You can use the following code to expose the metrics. 155 | 156 | ```go 157 | package main 158 | 159 | func main() { 160 | scheduler := gojob.New( 161 | // All you need to do is just adding the following option to the scheduler constructor 162 | gojob.WithPrometheusPushGateway("http://localhost:9091", "gojob"), 163 | ).Start() 164 | } 165 | ``` 166 | 167 | ## Coverage 168 | 169 | [![codecov-graph](https://codecov.io/gh/WangYihang/gojob/graphs/tree.svg?token=FG1HT7FCKG)](https://codecov.io/gh/WangYihang/gojob) 170 | 171 | 172 | ## License 173 | [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2FWangYihang%2Fgojob.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2FWangYihang%2Fgojob?ref=badge_large) -------------------------------------------------------------------------------- /deploy/grafana_config/provisioning/datasources/datasource.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | datasources: 4 | - name: Prometheus 5 | type: prometheus 6 | url: http://prometheus:9090 7 | isDefault: true 8 | access: proxy 9 | editable: true -------------------------------------------------------------------------------- /deploy/prometheus_config/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s 3 | scrape_timeout: 10s 4 | evaluation_interval: 15s 5 | alerting: 6 | alertmanagers: 7 | - static_configs: 8 | - targets: [] 9 | scheme: http 10 | timeout: 10s 11 | api_version: v1 12 | scrape_configs: 13 | - job_name: pushgateway 14 | honor_labels: true 15 | metrics_path: /metrics 16 | scheme: http 17 | static_configs: 18 | - targets: 19 | - pushgateway:9091 -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | services: 3 | prometheus: 4 | image: prom/prometheus 5 | container_name: prometheus 6 | restart: unless-stopped 7 | command: 8 | - '--config.file=/etc/prometheus/prometheus.yml' 9 | volumes: 10 | - ./deploy/prometheus_config:/etc/prometheus 11 | - prometheus_data:/prometheus 12 | 13 | pushgateway: 14 | image: prom/pushgateway 15 | container_name: pushgateway 16 | restart: unless-stopped 17 | ports: 18 | - 9091:9091 19 | 20 | grafana: 21 | image: grafana/grafana 22 | container_name: grafana 23 | ports: 24 | - 3000:3000 25 | environment: 26 | - GF_SECURITY_ADMIN_USER=${GF_SECURITY_ADMIN_USER:-admin} 27 | - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin} 28 | - GF_USERS_ALLOW_SIGN_UP=${GF_USERS_ALLOW_SIGN_UP:-false} 29 | restart: unless-stopped 30 | volumes: 31 | - grafana_data:/var/lib/grafana 32 | 33 | volumes: 34 | prometheus_data: 35 | grafana_data: -------------------------------------------------------------------------------- /examples/complex-http-crawler/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/WangYihang/gojob" 7 | "github.com/WangYihang/gojob/examples/complex-http-crawler/pkg/model" 8 | "github.com/WangYihang/gojob/pkg/utils" 9 | "github.com/WangYihang/gojob/pkg/version" 10 | "github.com/jessevdk/go-flags" 11 | ) 12 | 13 | type Options struct { 14 | InputFilePath string `short:"i" long:"input" description:"input file path" required:"true"` 15 | OutputFilePath string `short:"o" long:"output" description:"output file path" required:"true"` 16 | MaxRetries int `short:"r" long:"max-retries" description:"max retries" default:"3"` 17 | MaxRuntimePerTaskSeconds int `short:"t" long:"max-runtime-per-task-seconds" description:"max runtime per task seconds" default:"60"` 18 | NumWorkers int `short:"n" long:"num-workers" description:"number of workers" default:"32"` 19 | NumShards int `short:"s" long:"num-shards" description:"number of shards" default:"1"` 20 | Shard int `short:"d" long:"shard" description:"shard" default:"0"` 21 | Version func() `long:"version" description:"print version and exit" json:"-"` 22 | } 23 | 24 | var opts Options 25 | 26 | func init() { 27 | opts.Version = version.PrintVersion 28 | _, err := flags.Parse(&opts) 29 | if err != nil { 30 | os.Exit(1) 31 | } 32 | } 33 | 34 | func main() { 35 | total := utils.Count(utils.Cat(opts.InputFilePath)) 36 | scheduler := gojob.New( 37 | gojob.WithNumWorkers(opts.NumWorkers), 38 | gojob.WithMaxRetries(opts.MaxRetries), 39 | gojob.WithMaxRuntimePerTaskSeconds(opts.MaxRuntimePerTaskSeconds), 40 | gojob.WithNumShards(int64(opts.NumShards)), 41 | gojob.WithShard(int64(opts.Shard)), 42 | gojob.WithResultFilePath(opts.OutputFilePath), 43 | gojob.WithTotalTasks(total), 44 | gojob.WithStatusFilePath("status.json"), 45 | gojob.WithResultFilePath("result.json"), 46 | gojob.WithMetadataFilePath("metadata.json"), 47 | gojob.WithPrometheusPushGateway("http://localhost:9091", "gojob"), 48 | ). 49 | Start() 50 | for line := range utils.Cat(opts.InputFilePath) { 51 | scheduler.Submit(model.New(string(line))) 52 | } 53 | scheduler.Wait() 54 | } 55 | -------------------------------------------------------------------------------- /examples/complex-http-crawler/pkg/model/http.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "io" 5 | "log/slog" 6 | "mime/multipart" 7 | "net/http" 8 | "net/url" 9 | ) 10 | 11 | type HTTP struct { 12 | Request *HTTPRequest `json:"request"` 13 | Response *HTTPResponse `json:"response"` 14 | } 15 | 16 | type HTTPRequest struct { 17 | Method string `json:"method"` 18 | URL string `json:"url"` 19 | Host string `json:"host"` 20 | 21 | RemoteAddr string `json:"remote_addr"` 22 | RequestURI string `json:"request_uri"` 23 | 24 | Proto string `json:"proto"` 25 | ProtoMajor int `json:"proto_major"` 26 | ProtoMinor int `json:"proto_minor"` 27 | 28 | Header http.Header `json:"header"` 29 | 30 | ContentLength int64 `json:"content_length"` 31 | TransferEncoding []string `json:"transfer_encoding"` 32 | Close bool `json:"close"` 33 | 34 | Form url.Values `json:"form"` 35 | PostForm url.Values `json:"post_form"` 36 | MultipartForm *multipart.Form `json:"multipart_form"` 37 | 38 | Trailer http.Header `json:"trailer"` 39 | } 40 | 41 | func NewHTTPRequest(req *http.Request) (*HTTPRequest, error) { 42 | httpRequest := &HTTPRequest{ 43 | Method: req.Method, 44 | URL: req.URL.String(), 45 | Host: req.Host, 46 | RemoteAddr: req.RemoteAddr, 47 | RequestURI: req.RequestURI, 48 | Proto: req.Proto, 49 | ProtoMajor: req.ProtoMajor, 50 | ProtoMinor: req.ProtoMinor, 51 | Header: req.Header, 52 | ContentLength: req.ContentLength, 53 | TransferEncoding: req.TransferEncoding, 54 | Close: req.Close, 55 | Form: req.Form, 56 | PostForm: req.PostForm, 57 | MultipartForm: req.MultipartForm, 58 | Trailer: req.Trailer, 59 | } 60 | return httpRequest, nil 61 | } 62 | 63 | type HTTPResponse struct { 64 | Status string `json:"status"` 65 | StatusCode int `json:"status_code"` 66 | 67 | Proto string `json:"proto"` 68 | ProtoMajor int `json:"proto_major"` 69 | ProtoMinor int `json:"proto_minor"` 70 | 71 | Header http.Header `json:"header"` 72 | 73 | Body []byte `json:"body"` 74 | 75 | ContentLength int64 `json:"content_length"` 76 | TransferEncoding []string `json:"transfer_encoding"` 77 | Close bool `json:"close"` 78 | Uncompressed bool `json:"uncompressed"` 79 | Trailer http.Header `json:"trailer"` 80 | } 81 | 82 | func NewHTTPResponse(resp *http.Response) (*HTTPResponse, error) { 83 | httpResponse := &HTTPResponse{ 84 | Status: resp.Status, 85 | StatusCode: resp.StatusCode, 86 | Proto: resp.Proto, 87 | ProtoMajor: resp.ProtoMajor, 88 | ProtoMinor: resp.ProtoMinor, 89 | Header: resp.Header, 90 | Body: []byte{}, 91 | ContentLength: resp.ContentLength, 92 | TransferEncoding: resp.TransferEncoding, 93 | Close: resp.Close, 94 | Uncompressed: resp.Uncompressed, 95 | Trailer: resp.Trailer, 96 | } 97 | // Read response body 98 | body, err := io.ReadAll(resp.Body) 99 | if err != nil { 100 | slog.Warn("error occurred while reading response body", slog.String("error", err.Error())) 101 | return httpResponse, nil 102 | } 103 | httpResponse.Body = body 104 | return httpResponse, nil 105 | } 106 | -------------------------------------------------------------------------------- /examples/complex-http-crawler/pkg/model/task.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | ) 7 | 8 | const MAX_TRIES = 4 9 | 10 | type MyTask struct { 11 | Url string `json:"url"` 12 | HTTP HTTP `json:"http"` 13 | } 14 | 15 | func New(url string) *MyTask { 16 | return &MyTask{ 17 | Url: url, 18 | } 19 | } 20 | 21 | func (t *MyTask) Do() error { 22 | transport := &http.Transport{ 23 | DisableCompression: true, 24 | } 25 | client := &http.Client{ 26 | Transport: transport, 27 | CheckRedirect: func(req *http.Request, via []*http.Request) error { 28 | return http.ErrUseLastResponse 29 | }, 30 | Timeout: 4 * time.Second, 31 | } 32 | req, err := http.NewRequest(http.MethodHead, t.Url, nil) 33 | if err != nil { 34 | return err 35 | } 36 | httpRequest, err := NewHTTPRequest(req) 37 | if err != nil { 38 | return err 39 | } 40 | t.HTTP.Request = httpRequest 41 | resp, err := client.Do(req) 42 | if err != nil { 43 | return err 44 | } 45 | httpResponse, err := NewHTTPResponse(resp) 46 | if err != nil { 47 | return err 48 | } 49 | t.HTTP.Response = httpResponse 50 | return nil 51 | } 52 | -------------------------------------------------------------------------------- /examples/load-s3-file-as-input/README.md: -------------------------------------------------------------------------------- 1 | ## Usage 2 | 3 | Use the following path as the input filepath. 4 | 5 | ``` 6 | s3://default/data.json?region=us-west-2&endpoint=s3.amazonaws.com&access_key=********************&secret_key=**************************************** 7 | ``` -------------------------------------------------------------------------------- /examples/load-s3-file-as-input/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log/slog" 5 | 6 | "github.com/WangYihang/gojob/pkg/utils" 7 | ) 8 | 9 | func main() { 10 | for line := range utils.Cat( 11 | "s3://uio/example.txt" + 12 | "?endpoint=127.0.0.1:9000" + 13 | "&access_key=********************" + 14 | "&secret_key=********************************************" + 15 | "&insecure=true" + 16 | "&download=true", 17 | ) { 18 | slog.Info("s3", slog.String("line", line)) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /examples/load-s3-store-s3/README.md: -------------------------------------------------------------------------------- 1 | ## Usage 2 | 3 | Use the following path as the input filepath. 4 | 5 | ``` 6 | s3://default/data.json?region=us-west-2&endpoint=s3.amazonaws.com&access_key=********************&secret_key=**************************************** 7 | ``` -------------------------------------------------------------------------------- /examples/load-s3-store-s3/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log/slog" 5 | 6 | "github.com/WangYihang/gojob/pkg/utils" 7 | "github.com/WangYihang/uio" 8 | ) 9 | 10 | func main() { 11 | resultFd, err := uio.Open("s3://uio/output.txt" + 12 | "?endpoint=127.0.0.1:9000" + 13 | "&access_key=minioadmin" + 14 | "&secret_key=minioadmin" + 15 | "&insecure=true" + 16 | "&mode=write", 17 | ) 18 | if err != nil { 19 | slog.Error("failed to open result file", slog.String("error", err.Error())) 20 | return 21 | } 22 | defer resultFd.Close() 23 | for line := range utils.Cat( 24 | "s3://uio/input.txt" + 25 | "?endpoint=127.0.0.1:9000" + 26 | "&access_key=minioadmin" + 27 | "&secret_key=minioadmin" + 28 | "&insecure=true" + 29 | "&mode=read", 30 | ) { 31 | slog.Info("s3", slog.String("line", line)) 32 | resultFd.Write([]byte(line + "\n")) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /examples/metadata/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | 7 | "github.com/WangYihang/gojob" 8 | "github.com/WangYihang/gojob/pkg/runner" 9 | "github.com/WangYihang/gojob/pkg/utils" 10 | ) 11 | 12 | type MyTask struct{} 13 | 14 | func New() *MyTask { 15 | return &MyTask{} 16 | } 17 | 18 | func (t *MyTask) Do() error { 19 | time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond) 20 | return nil 21 | } 22 | 23 | func main() { 24 | inputFilePath := "data/input.txt" 25 | total := utils.Count(utils.Cat(inputFilePath)) 26 | scheduler := gojob.New( 27 | gojob.WithNumWorkers(8), 28 | gojob.WithMaxRetries(4), 29 | gojob.WithMaxRuntimePerTaskSeconds(16), 30 | gojob.WithNumShards(4), 31 | gojob.WithShard(0), 32 | gojob.WithResultFilePath("data/output.txt"), 33 | gojob.WithStatusFilePath("data/output.status"), 34 | gojob.WithTotalTasks(total), 35 | gojob.WithMetadata("a", "b"), 36 | gojob.WithMetadata("c", "d"), 37 | gojob.WithMetadata("runner", runner.Runner), 38 | ). 39 | Start() 40 | for range utils.Cat(inputFilePath) { 41 | scheduler.Submit(New()) 42 | } 43 | scheduler.Wait() 44 | } 45 | -------------------------------------------------------------------------------- /examples/nopper/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | 7 | "github.com/WangYihang/gojob" 8 | "github.com/WangYihang/gojob/pkg/utils" 9 | ) 10 | 11 | type MyTask struct{} 12 | 13 | func New() *MyTask { 14 | return &MyTask{} 15 | } 16 | 17 | func (t *MyTask) Do() error { 18 | time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond) 19 | return nil 20 | } 21 | 22 | func main() { 23 | inputFilePath := "data/input.txt" 24 | total := utils.Count(utils.Cat(inputFilePath)) 25 | scheduler := gojob.New( 26 | gojob.WithNumWorkers(8), 27 | gojob.WithMaxRetries(4), 28 | gojob.WithMaxRuntimePerTaskSeconds(16), 29 | gojob.WithNumShards(4), 30 | gojob.WithShard(0), 31 | gojob.WithResultFilePath("data/output.json"), 32 | gojob.WithStatusFilePath("data/status.json"), 33 | gojob.WithMetadataFilePath("data/metadata.json"), 34 | gojob.WithTotalTasks(total), 35 | ). 36 | Start() 37 | for range utils.Cat(inputFilePath) { 38 | scheduler.Submit(New()) 39 | } 40 | scheduler.Wait() 41 | } 42 | -------------------------------------------------------------------------------- /examples/prometheus/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | 7 | "github.com/WangYihang/gojob" 8 | ) 9 | 10 | type MyTask struct { 11 | Url string `json:"url"` 12 | StatusCode int `json:"status_code"` 13 | } 14 | 15 | func New(url string) *MyTask { 16 | return &MyTask{ 17 | Url: url, 18 | } 19 | } 20 | 21 | func (t *MyTask) Do() error { 22 | response, err := http.Get(t.Url) 23 | if err != nil { 24 | return err 25 | } 26 | t.StatusCode = response.StatusCode 27 | defer response.Body.Close() 28 | return nil 29 | } 30 | 31 | func main() { 32 | var numTotalTasks int64 = 256 33 | scheduler := gojob.New( 34 | gojob.WithNumWorkers(8), 35 | gojob.WithMaxRetries(4), 36 | gojob.WithMaxRuntimePerTaskSeconds(16), 37 | gojob.WithNumShards(4), 38 | gojob.WithShard(0), 39 | gojob.WithTotalTasks(numTotalTasks), 40 | gojob.WithStatusFilePath("status.json"), 41 | gojob.WithResultFilePath("result.json"), 42 | gojob.WithMetadataFilePath("metadata.json"), 43 | gojob.WithPrometheusPushGateway("http://localhost:9091/", "gojob"), 44 | ). 45 | Start() 46 | for i := range numTotalTasks { 47 | scheduler.Submit(New(fmt.Sprintf("https://httpbin.org/task/%d", i))) 48 | } 49 | scheduler.Wait() 50 | } 51 | -------------------------------------------------------------------------------- /examples/random-error/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "math/rand" 6 | "time" 7 | 8 | "github.com/WangYihang/gojob" 9 | ) 10 | 11 | type MyTask struct { 12 | Index int `json:"index"` 13 | SleepSeconds int `json:"sleep_seconds"` 14 | ErrorProbability float64 `json:"error_probability"` 15 | } 16 | 17 | func New(index int, sleepSeconds int) *MyTask { 18 | return &MyTask{ 19 | Index: index, 20 | SleepSeconds: sleepSeconds, 21 | ErrorProbability: 0.32, 22 | } 23 | } 24 | 25 | func (t *MyTask) Do() error { 26 | time.Sleep(time.Duration(t.SleepSeconds) * time.Second) 27 | if rand.Float64() < t.ErrorProbability { 28 | return errors.New("an error occurred") 29 | } 30 | return nil 31 | } 32 | 33 | func main() { 34 | total := 256 35 | scheduler := gojob.New( 36 | gojob.WithNumWorkers(8), 37 | gojob.WithMaxRetries(1), 38 | gojob.WithMaxRuntimePerTaskSeconds(16), 39 | gojob.WithShard(0), 40 | gojob.WithNumShards(1), 41 | gojob.WithTotalTasks(int64(total)), 42 | gojob.WithStatusFilePath("status.json"), 43 | gojob.WithResultFilePath("result.json"), 44 | gojob.WithMetadataFilePath("metadata.json"), 45 | ).Start() 46 | for i := 0; i < total; i++ { 47 | scheduler.Submit(New(i, rand.Intn(10))) 48 | } 49 | scheduler.Wait() 50 | } 51 | -------------------------------------------------------------------------------- /examples/read-from-gzip-file/data.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WangYihang/gojob/66ed0ba94af390ee2dee534a38590e076e65aa05/examples/read-from-gzip-file/data.txt.gz -------------------------------------------------------------------------------- /examples/read-from-gzip-file/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | 7 | "github.com/WangYihang/gojob" 8 | "github.com/WangYihang/gojob/pkg/utils" 9 | ) 10 | 11 | type MyTask struct { 12 | Line string 13 | } 14 | 15 | func New(line string) *MyTask { 16 | return &MyTask{ 17 | Line: line, 18 | } 19 | } 20 | 21 | func (t *MyTask) Do() error { 22 | time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond) 23 | return nil 24 | } 25 | 26 | func main() { 27 | scheduler := gojob.New( 28 | gojob.WithNumWorkers(8), 29 | gojob.WithMaxRetries(4), 30 | gojob.WithMaxRuntimePerTaskSeconds(16), 31 | gojob.WithResultFilePath("-"), 32 | gojob.WithStatusFilePath("status.json"), 33 | ). 34 | Start() 35 | for line := range utils.Cat("data.txt.gz") { 36 | scheduler.Submit(New(line)) 37 | } 38 | scheduler.Wait() 39 | } 40 | -------------------------------------------------------------------------------- /examples/result-channel/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log/slog" 7 | "net/http" 8 | 9 | "github.com/WangYihang/gojob" 10 | ) 11 | 12 | type MyTask struct { 13 | Url string `json:"url"` 14 | StatusCode int `json:"status_code"` 15 | } 16 | 17 | func New(url string) *MyTask { 18 | return &MyTask{ 19 | Url: url, 20 | } 21 | } 22 | 23 | func (t *MyTask) Do() error { 24 | response, err := http.Get(t.Url) 25 | if err != nil { 26 | return err 27 | } 28 | t.StatusCode = response.StatusCode 29 | defer response.Body.Close() 30 | return nil 31 | } 32 | 33 | func main() { 34 | var numTotalTasks int64 = 256 35 | scheduler := gojob.New( 36 | gojob.WithNumWorkers(8), 37 | gojob.WithMaxRetries(4), 38 | gojob.WithMaxRuntimePerTaskSeconds(16), 39 | gojob.WithNumShards(4), 40 | gojob.WithShard(0), 41 | gojob.WithTotalTasks(numTotalTasks), 42 | gojob.WithStatusFilePath("status.json"), 43 | gojob.WithResultFilePath("result.json"), 44 | gojob.WithMetadataFilePath("metadata.json"), 45 | ) 46 | go func() { 47 | for result := range scheduler.ResultChan() { 48 | data, err := json.Marshal(result) 49 | if err != nil { 50 | slog.Error("failed to marshal result", slog.String("error", err.Error())) 51 | continue 52 | } 53 | fmt.Println(string(data)) 54 | } 55 | }() 56 | for i := range numTotalTasks { 57 | scheduler.Submit(New(fmt.Sprintf("https://httpbin.org/task/%d", i))) 58 | } 59 | scheduler.Wait() 60 | } 61 | -------------------------------------------------------------------------------- /examples/simple-http-crawler/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | 7 | "github.com/WangYihang/gojob" 8 | ) 9 | 10 | type MyTask struct { 11 | Url string `json:"url"` 12 | StatusCode int `json:"status_code"` 13 | } 14 | 15 | func New(url string) *MyTask { 16 | return &MyTask{ 17 | Url: url, 18 | } 19 | } 20 | 21 | func (t *MyTask) Do() error { 22 | response, err := http.Get(t.Url) 23 | if err != nil { 24 | return err 25 | } 26 | t.StatusCode = response.StatusCode 27 | defer response.Body.Close() 28 | return nil 29 | } 30 | 31 | func main() { 32 | var numTotalTasks int64 = 256 33 | scheduler := gojob.New( 34 | gojob.WithNumWorkers(8), 35 | gojob.WithMaxRetries(4), 36 | gojob.WithMaxRuntimePerTaskSeconds(16), 37 | gojob.WithNumShards(4), 38 | gojob.WithShard(0), 39 | gojob.WithTotalTasks(numTotalTasks), 40 | gojob.WithStatusFilePath("status.json"), 41 | gojob.WithResultFilePath("result.json"), 42 | gojob.WithMetadataFilePath("metadata.json"), 43 | ). 44 | Start() 45 | for i := range numTotalTasks { 46 | scheduler.Submit(New(fmt.Sprintf("https://httpbin.org/task/%d", i))) 47 | } 48 | scheduler.Wait() 49 | } 50 | -------------------------------------------------------------------------------- /examples/sleeper/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | 7 | "github.com/WangYihang/gojob" 8 | ) 9 | 10 | type MyTask struct { 11 | Index int `json:"index"` 12 | SleepSeconds int `json:"sleep_seconds"` 13 | } 14 | 15 | func New(index int, sleepSeconds int) *MyTask { 16 | return &MyTask{ 17 | Index: index, 18 | SleepSeconds: sleepSeconds, 19 | } 20 | } 21 | 22 | func (t *MyTask) Do() error { 23 | time.Sleep(time.Duration(t.SleepSeconds) * time.Second) 24 | return nil 25 | } 26 | 27 | func main() { 28 | total := 256 29 | scheduler := gojob.New( 30 | gojob.WithNumWorkers(8), 31 | gojob.WithMaxRetries(4), 32 | gojob.WithMaxRuntimePerTaskSeconds(16), 33 | gojob.WithNumShards(4), 34 | gojob.WithShard(0), 35 | gojob.WithTotalTasks(int64(total)), 36 | gojob.WithStatusFilePath("-"), 37 | gojob.WithResultFilePath("-"), 38 | gojob.WithMetadataFilePath("-"), 39 | ).Start() 40 | for i := 0; i < total; i++ { 41 | scheduler.Submit(New(i, rand.Intn(10))) 42 | } 43 | scheduler.Wait() 44 | } 45 | -------------------------------------------------------------------------------- /examples/stdio/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/WangYihang/gojob" 7 | "github.com/WangYihang/gojob/pkg/utils" 8 | ) 9 | 10 | type MyPrinterTask struct { 11 | line string 12 | } 13 | 14 | func New(line string) *MyPrinterTask { 15 | return &MyPrinterTask{ 16 | line: line, 17 | } 18 | } 19 | 20 | func (t *MyPrinterTask) Do() error { 21 | fmt.Println(t.line) 22 | return nil 23 | } 24 | 25 | func main() { 26 | scheduler := gojob.New( 27 | gojob.WithNumWorkers(8), 28 | gojob.WithMaxRetries(4), 29 | gojob.WithMaxRuntimePerTaskSeconds(16), 30 | gojob.WithResultFilePath("-"), 31 | gojob.WithStatusFilePath("status.json"), 32 | ). 33 | Start() 34 | for line := range utils.Cat("-") { 35 | scheduler.Submit(New(line)) 36 | } 37 | scheduler.Wait() 38 | } 39 | -------------------------------------------------------------------------------- /examples/tcp-port-scanner/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/WangYihang/gojob" 5 | ) 6 | 7 | func main() { 8 | var commonPorts = []uint16{ 9 | 21, 22, 23, 25, 53, 80, 110, 135, 139, 143, 443, 445, 993, 995, 10 | 1723, 3306, 3389, 5900, 8080, 11 | } 12 | var ips = []string{ 13 | "192.168.200.1", 14 | "192.168.200.254", 15 | } 16 | scheduler := gojob.New( 17 | gojob.WithNumWorkers(8), 18 | gojob.WithMaxRetries(4), 19 | gojob.WithMaxRuntimePerTaskSeconds(16), 20 | gojob.WithNumShards(1), 21 | gojob.WithShard(0), 22 | gojob.WithTotalTasks(int64(len(ips)*len(commonPorts))), 23 | gojob.WithStatusFilePath("status.json"), 24 | gojob.WithResultFilePath("result.json"), 25 | gojob.WithMetadataFilePath("metadata.json"), 26 | ). 27 | Start() 28 | for _, ip := range ips { 29 | for _, port := range commonPorts { 30 | scheduler.Submit(New(ip, port)) 31 | } 32 | } 33 | scheduler.Wait() 34 | } 35 | -------------------------------------------------------------------------------- /examples/tcp-port-scanner/status.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | type Status uint8 4 | 5 | const ( 6 | Unknown Status = iota 7 | Open 8 | Closed 9 | ) 10 | 11 | func (s Status) String() string { 12 | statuses := map[Status]string{ 13 | Unknown: "Undefined", 14 | Open: "Open", 15 | Closed: "Closed", 16 | } 17 | return statuses[s] 18 | } 19 | 20 | func (s Status) MarshalJSON() ([]byte, error) { 21 | return []byte(`"` + s.String() + `"`), nil 22 | } 23 | -------------------------------------------------------------------------------- /examples/tcp-port-scanner/task.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "net" 4 | 5 | type MyTask struct { 6 | IP string `json:"ip"` 7 | Port uint16 `json:"port"` 8 | Status Status `json:"status"` 9 | } 10 | 11 | func New(ip string, port uint16) *MyTask { 12 | return &MyTask{ 13 | IP: ip, 14 | Port: port, 15 | Status: Unknown, 16 | } 17 | } 18 | 19 | func (t *MyTask) Do() error { 20 | conn, err := net.DialTCP("tcp", nil, &net.TCPAddr{ 21 | IP: net.ParseIP(t.IP), 22 | Port: int(t.Port), 23 | }) 24 | if err != nil { 25 | t.Status = Closed 26 | return nil 27 | } 28 | defer conn.Close() 29 | t.Status = Open 30 | return nil 31 | } 32 | -------------------------------------------------------------------------------- /examples/ultimate-http-crawler/README.md: -------------------------------------------------------------------------------- 1 | https://github.com/WangYihang/http-grab -------------------------------------------------------------------------------- /examples/write-to-gzip-file/.gitignore: -------------------------------------------------------------------------------- 1 | *.gz 2 | *.gzip -------------------------------------------------------------------------------- /examples/write-to-gzip-file/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "time" 7 | 8 | "github.com/WangYihang/gojob" 9 | ) 10 | 11 | type MyTask struct { 12 | Line string 13 | } 14 | 15 | func New(line string) *MyTask { 16 | return &MyTask{ 17 | Line: line, 18 | } 19 | } 20 | 21 | func (t *MyTask) Do() error { 22 | time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond) 23 | fmt.Println(t.Line) 24 | return nil 25 | } 26 | 27 | func main() { 28 | scheduler := gojob.New( 29 | gojob.WithNumWorkers(1), 30 | gojob.WithMaxRetries(4), 31 | gojob.WithMaxRuntimePerTaskSeconds(16), 32 | gojob.WithResultFilePath("result.txt.gz"), 33 | gojob.WithStatusFilePath("status.json"), 34 | gojob.WithMetadataFilePath("metadata.json"), 35 | ). 36 | Start() 37 | for line := range 16 { 38 | scheduler.Submit(New(fmt.Sprintf("line-%d", line))) 39 | } 40 | scheduler.Wait() 41 | } 42 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/WangYihang/gojob 2 | 3 | go 1.22.6 4 | 5 | require ( 6 | github.com/WangYihang/uio v0.0.0-20240910061712-086a0337cd43 7 | github.com/google/uuid v1.6.0 8 | github.com/jessevdk/go-flags v1.5.0 9 | github.com/prometheus/client_golang v1.19.0 10 | github.com/prometheus/client_model v0.6.1 11 | ) 12 | 13 | require ( 14 | github.com/beorn7/perks v1.0.1 // indirect 15 | github.com/caarlos0/env v3.5.0+incompatible // indirect 16 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 17 | github.com/dustin/go-humanize v1.0.1 // indirect 18 | github.com/go-ini/ini v1.67.0 // indirect 19 | github.com/goccy/go-json v0.10.3 // indirect 20 | github.com/klauspost/compress v1.17.9 // indirect 21 | github.com/klauspost/cpuid/v2 v2.2.8 // indirect 22 | github.com/minio/md5-simd v1.1.2 // indirect 23 | github.com/minio/minio-go/v7 v7.0.75 // indirect 24 | github.com/prometheus/common v0.48.0 // indirect 25 | github.com/prometheus/procfs v0.12.0 // indirect 26 | github.com/rs/xid v1.5.0 // indirect 27 | golang.org/x/crypto v0.24.0 // indirect 28 | golang.org/x/net v0.26.0 // indirect 29 | golang.org/x/sys v0.21.0 // indirect 30 | golang.org/x/text v0.16.0 // indirect 31 | google.golang.org/protobuf v1.33.0 // indirect 32 | ) 33 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/WangYihang/uio v0.0.0-20240905152743-19ba2df5e6a6 h1:4r+fK+dAO7ry5gtFV6emxvcwP9sE8QT8UnPfGqlaD6g= 2 | github.com/WangYihang/uio v0.0.0-20240905152743-19ba2df5e6a6/go.mod h1:5WoqViIAdldkfhEyOaceDjpfH4wazQDLz86aJVnHnGQ= 3 | github.com/WangYihang/uio v0.0.0-20240906030653-6e960648c0b6 h1:Ig0Hu89i+nOY6wgLzOY5x0KBBDqr417uwOtsvr9Pzz8= 4 | github.com/WangYihang/uio v0.0.0-20240906030653-6e960648c0b6/go.mod h1:5WoqViIAdldkfhEyOaceDjpfH4wazQDLz86aJVnHnGQ= 5 | github.com/WangYihang/uio v0.0.0-20240906032945-575b80997943 h1:e5MSNG/CiJXIK/wKAcPShZUqLkir9TUXnlypwGudWTM= 6 | github.com/WangYihang/uio v0.0.0-20240906032945-575b80997943/go.mod h1:5WoqViIAdldkfhEyOaceDjpfH4wazQDLz86aJVnHnGQ= 7 | github.com/WangYihang/uio v0.0.0-20240906040721-305cfedd121a h1:IGsqxTs+mCkAfJdm3p2rwjlxPhKLyWou80tsDtVSEKM= 8 | github.com/WangYihang/uio v0.0.0-20240906040721-305cfedd121a/go.mod h1:5WoqViIAdldkfhEyOaceDjpfH4wazQDLz86aJVnHnGQ= 9 | github.com/WangYihang/uio v0.0.0-20240910061712-086a0337cd43 h1:ecPV975eOq/GLtvuZrZjD3Vqly/Nsb9jzfMyhkisEm0= 10 | github.com/WangYihang/uio v0.0.0-20240910061712-086a0337cd43/go.mod h1:5WoqViIAdldkfhEyOaceDjpfH4wazQDLz86aJVnHnGQ= 11 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 12 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 13 | github.com/caarlos0/env v3.5.0+incompatible h1:Yy0UN8o9Wtr/jGHZDpCBLpNrzcFLLM2yixi/rBrKyJs= 14 | github.com/caarlos0/env v3.5.0+incompatible/go.mod h1:tdCsowwCzMLdkqRYDlHpZCp2UooDD3MspDBjZ2AD02Y= 15 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 16 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 17 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 18 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 19 | github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= 20 | github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= 21 | github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= 22 | github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= 23 | github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= 24 | github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= 25 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 26 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 27 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 28 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 29 | github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= 30 | github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= 31 | github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= 32 | github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= 33 | github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 34 | github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= 35 | github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= 36 | github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= 37 | github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= 38 | github.com/minio/minio-go/v7 v7.0.75 h1:0uLrB6u6teY2Jt+cJUVi9cTvDRuBKWSRzSAcznRkwlE= 39 | github.com/minio/minio-go/v7 v7.0.75/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= 40 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 41 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 42 | github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= 43 | github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= 44 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 45 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 46 | github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= 47 | github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= 48 | github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= 49 | github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= 50 | github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= 51 | github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= 52 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 53 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 54 | golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= 55 | golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= 56 | golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= 57 | golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= 58 | golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 59 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 60 | golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= 61 | golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 62 | golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= 63 | golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= 64 | google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= 65 | google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= 66 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 67 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 68 | -------------------------------------------------------------------------------- /pkg/runner/runner.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "encoding/json" 5 | "io" 6 | "log/slog" 7 | "net/http" 8 | "sync" 9 | ) 10 | 11 | var Runner *IPInfo 12 | var once sync.Once 13 | 14 | func init() { 15 | once.Do(func() { 16 | Runner = NewIPInfo() 17 | err := Runner.Get() 18 | if err != nil { 19 | slog.Error("error occurred while getting runner ip info", slog.String("error", err.Error())) 20 | return 21 | } 22 | }) 23 | } 24 | 25 | type IPInfo struct { 26 | IP string `json:"ip"` 27 | Hostname string `json:"hostname"` 28 | City string `json:"city"` 29 | Region string `json:"region"` 30 | Country string `json:"country"` 31 | Loc string `json:"loc"` 32 | Org string `json:"org"` 33 | Postal string `json:"postal"` 34 | Timezone string `json:"timezone"` 35 | } 36 | 37 | func NewIPInfo() *IPInfo { 38 | return &IPInfo{ 39 | IP: "unknown", 40 | Hostname: "unknown", 41 | City: "unknown", 42 | Region: "unknown", 43 | Country: "unknown", 44 | Loc: "unknown", 45 | Org: "unknown", 46 | Postal: "unknown", 47 | Timezone: "unknown", 48 | } 49 | } 50 | 51 | func (i *IPInfo) Get() error { 52 | req, err := http.Get("https://ipinfo.io/") 53 | if err != nil { 54 | return err 55 | } 56 | defer req.Body.Close() 57 | body, err := io.ReadAll(req.Body) 58 | if err != nil { 59 | return err 60 | } 61 | err = json.Unmarshal(body, i) 62 | if err != nil { 63 | return err 64 | } 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /pkg/utils/capture.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "os" 7 | ) 8 | 9 | type StdoutCapture struct { 10 | originalStdout *os.File 11 | r *os.File 12 | w *os.File 13 | buffer bytes.Buffer 14 | } 15 | 16 | func NewStdoutCapture() *StdoutCapture { 17 | return &StdoutCapture{} 18 | } 19 | 20 | func (oc *StdoutCapture) StartCapture() { 21 | oc.originalStdout = os.Stdout 22 | 23 | r, w, _ := os.Pipe() 24 | os.Stdout = w 25 | 26 | oc.r = r 27 | oc.w = w 28 | } 29 | 30 | func (oc *StdoutCapture) StopCapture() { 31 | os.Stdout = oc.originalStdout 32 | oc.w.Close() 33 | 34 | io.Copy(&oc.buffer, oc.r) 35 | oc.r.Close() 36 | } 37 | 38 | func (oc *StdoutCapture) GetCapturedOutput() string { 39 | return oc.buffer.String() 40 | } 41 | -------------------------------------------------------------------------------- /pkg/utils/capture_test.go: -------------------------------------------------------------------------------- 1 | package utils_test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/WangYihang/gojob/pkg/utils" 8 | ) 9 | 10 | func TestOutputCapture(t *testing.T) { 11 | oc := utils.NewStdoutCapture() 12 | oc.StartCapture() 13 | fmt.Println("hello") 14 | oc.StopCapture() 15 | if oc.GetCapturedOutput() != "hello\n" { 16 | t.Fail() 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /pkg/utils/channel.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "sync" 4 | 5 | // Fanin takes a slice of channels and returns a single channel that 6 | func Fanin[T interface{}](cs []chan T) chan T { 7 | var wg sync.WaitGroup 8 | out := make(chan T) 9 | output := func(c chan T) { 10 | for n := range c { 11 | out <- n 12 | } 13 | wg.Done() 14 | } 15 | wg.Add(len(cs)) 16 | for _, c := range cs { 17 | go output(c) 18 | } 19 | go func() { 20 | wg.Wait() 21 | close(out) 22 | }() 23 | return out 24 | } 25 | 26 | // Fanout takes a channel and returns a slice of channels 27 | // the item in the input channel will be distributed to the output channels 28 | func Fanout[T interface{}](in chan *T, n int) []chan *T { 29 | cs := make([]chan *T, n) 30 | for i := 0; i < n; i++ { 31 | cs[i] = make(chan *T) 32 | go func(c chan *T) { 33 | for n := range in { 34 | c <- n 35 | } 36 | close(c) 37 | }(cs[i]) 38 | } 39 | return cs 40 | } 41 | 42 | // Filter takes a channel and returns a channel with the items that pass the filter 43 | func Filter[T interface{}](in chan T, f func(T) bool) chan T { 44 | out := make(chan T) 45 | go func() { 46 | defer close(out) 47 | for line := range in { 48 | if f(line) { 49 | out <- line 50 | } 51 | } 52 | }() 53 | return out 54 | } 55 | -------------------------------------------------------------------------------- /pkg/utils/io.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "log/slog" 7 | "strings" 8 | 9 | "github.com/WangYihang/uio" 10 | ) 11 | 12 | // Head takes a channel and returns a channel with the first n items 13 | func Head[T interface{}](in <-chan T, max int) <-chan T { 14 | out := make(chan T) 15 | go func() { 16 | defer close(out) 17 | i := 0 18 | for line := range in { 19 | if i >= max { 20 | break 21 | } 22 | out <- line 23 | i++ 24 | } 25 | }() 26 | return out 27 | } 28 | 29 | // Tail takes a channel and returns a channel with the last n items 30 | func Tail[T interface{}](in <-chan T, max int) <-chan T { 31 | out := make(chan T) 32 | go func() { 33 | defer close(out) 34 | var lines []T 35 | for line := range in { 36 | lines = append(lines, line) 37 | if len(lines) > max { 38 | lines = lines[1:] 39 | } 40 | } 41 | for _, line := range lines { 42 | out <- line 43 | } 44 | }() 45 | return out 46 | } 47 | 48 | // Skip takes a channel and returns a channel with the first n items skipped 49 | func Skip[T any](in <-chan T, n int) <-chan T { 50 | out := make(chan T) 51 | go func() { 52 | defer close(out) 53 | for i := 0; i < n; i++ { 54 | <-in 55 | } 56 | for line := range in { 57 | out <- line 58 | } 59 | }() 60 | return out 61 | } 62 | 63 | // Cat takes a file path and returns a channel with the lines of the file 64 | // Spaces are trimmed from the beginning and end of each line 65 | func Cat(filePath string) <-chan string { 66 | out := make(chan string) 67 | 68 | go func() { 69 | defer close(out) // Ensure the channel is closed when the goroutine finishes 70 | 71 | // Open the file 72 | file, err := uio.Open(filePath) 73 | if err != nil { 74 | slog.Error("error occurred while opening file", slog.String("path", filePath), slog.String("error", err.Error())) 75 | return // Close the channel and exit the goroutine 76 | } 77 | defer file.Close() 78 | 79 | scanner := bufio.NewScanner(file.(io.Reader)) // Change the type of file to io.Reader 80 | for scanner.Scan() { 81 | out <- strings.TrimSpace(scanner.Text()) // Send the line to the channel 82 | } 83 | 84 | // Check for errors during Scan, excluding EOF 85 | if err := scanner.Err(); err != nil { 86 | slog.Error("error occurred while reading file", slog.String("path", filePath), slog.String("error", err.Error())) 87 | } 88 | }() 89 | 90 | return out 91 | } 92 | 93 | // Count takes a channel and returns the number of items 94 | func Count[T any](in <-chan T) (count int64) { 95 | for range in { 96 | count++ 97 | } 98 | return count 99 | } 100 | -------------------------------------------------------------------------------- /pkg/utils/mapreduce.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | // Map takes a channel and returns a channel with the items that pass the filter 4 | func Map[T interface{}, U interface{}](in chan T, f func(T) U) chan U { 5 | out := make(chan U) 6 | go func() { 7 | defer close(out) 8 | for line := range in { 9 | out <- f(line) 10 | } 11 | }() 12 | return out 13 | } 14 | 15 | // Reduce takes a channel and returns a channel with the items that pass the filter 16 | func Reduce[T interface{}](in chan T, f func(T, T) T) T { 17 | var result T 18 | for line := range in { 19 | result = f(result, line) 20 | } 21 | return result 22 | } 23 | -------------------------------------------------------------------------------- /pkg/utils/string.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "strings" 4 | 5 | func Sanitize(s string) string { 6 | builder := strings.Builder{} 7 | for _, c := range s { 8 | if 'a' <= c && c <= 'z' { 9 | builder.WriteRune(c) 10 | continue 11 | } 12 | if 'A' <= c && c <= 'Z' { 13 | builder.WriteRune(c - 'A' + 'a') 14 | continue 15 | } 16 | if '0' <= c && c <= '9' { 17 | builder.WriteRune(c) 18 | continue 19 | } 20 | if c == ' ' || c == '-' { 21 | builder.WriteString("-") 22 | } 23 | } 24 | return builder.String() 25 | } 26 | -------------------------------------------------------------------------------- /pkg/utils/string_test.go: -------------------------------------------------------------------------------- 1 | package utils_test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/WangYihang/gojob/pkg/utils" 8 | ) 9 | 10 | func ExampleSanitize() { 11 | sanitized := utils.Sanitize("New York") 12 | fmt.Println(sanitized) 13 | // Output: new-york 14 | } 15 | 16 | func TestSanitize(t *testing.T) { 17 | testcases := []struct { 18 | s string 19 | want string 20 | }{ 21 | {s: "New York", want: "new-york"}, 22 | {s: "New-York", want: "new-york"}, 23 | {s: "NewYork", want: "newyork"}, 24 | } 25 | for _, testcase := range testcases { 26 | got := utils.Sanitize(testcase.s) 27 | if got != testcase.want { 28 | t.Errorf("Sanitize(%#v), want: %#v, expected: %#v", testcase.s, testcase.want, got) 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /pkg/utils/tee.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | // TeeWriterCloser structure, used to write to and close multiple io.WriteClosers 8 | type TeeWriterCloser struct { 9 | writers []io.WriteCloser 10 | } 11 | 12 | // NewTeeWriterCloser creates a new instance of TeeWriterCloser 13 | func NewTeeWriterCloser(writers ...io.WriteCloser) *TeeWriterCloser { 14 | return &TeeWriterCloser{ 15 | writers: writers, 16 | } 17 | } 18 | 19 | // Write implements the io.Writer interface 20 | // It writes the given byte slice to all the contained writers 21 | func (t *TeeWriterCloser) Write(p []byte) (n int, err error) { 22 | for _, w := range t.writers { 23 | n, err = w.Write(p) 24 | if err != nil { 25 | return 26 | } 27 | } 28 | return len(p), nil 29 | } 30 | 31 | // Close implements the io.Closer interface 32 | // It closes all the contained writers and returns the first encountered error 33 | func (t *TeeWriterCloser) Close() (err error) { 34 | for _, w := range t.writers { 35 | if cerr := w.Close(); cerr != nil && err == nil { 36 | err = cerr // Record the first encountered error 37 | } 38 | } 39 | return 40 | } 41 | -------------------------------------------------------------------------------- /pkg/utils/timeout.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | func RunWithTimeout(f func() error, timeout time.Duration) error { 9 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 10 | defer cancel() 11 | 12 | done := make(chan error, 1) 13 | 14 | go func() { 15 | defer close(done) 16 | 17 | done <- f() 18 | }() 19 | 20 | select { 21 | case <-ctx.Done(): 22 | return ctx.Err() 23 | case err := <-done: 24 | return err 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /pkg/utils/timeout_test.go: -------------------------------------------------------------------------------- 1 | package utils_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/WangYihang/gojob/pkg/utils" 9 | ) 10 | 11 | type Task struct { 12 | ctx context.Context 13 | } 14 | 15 | func (t *Task) Do() error { 16 | time.Sleep(16 * time.Second) 17 | return nil 18 | } 19 | 20 | func TestRunWithTimeout(t *testing.T) { 21 | task := &Task{context.Background()} 22 | err := utils.RunWithTimeout(task.Do, 1*time.Second) 23 | if err == nil { 24 | t.Errorf("Expected timeout error, got nil") 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /pkg/utils/url.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "net/url" 4 | 5 | func ParseProtocol(uri string) (protocol string, err error) { 6 | parsed, err := url.Parse(uri) 7 | if parsed.Scheme == "" { 8 | return "file", nil 9 | } 10 | return parsed.Scheme, err 11 | } 12 | -------------------------------------------------------------------------------- /pkg/utils/url_test.go: -------------------------------------------------------------------------------- 1 | package utils_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/WangYihang/gojob/pkg/utils" 7 | ) 8 | 9 | func TestParseProtocol(t *testing.T) { 10 | testcases := []struct { 11 | uri string 12 | expected string 13 | }{ 14 | { 15 | uri: "http://example.com", 16 | expected: "http", 17 | }, 18 | { 19 | uri: "https://example.com", 20 | expected: "https", 21 | }, 22 | { 23 | uri: "ftp://example.com", 24 | expected: "ftp", 25 | }, 26 | { 27 | uri: "file://example.com", 28 | expected: "file", 29 | }, 30 | { 31 | uri: "s3://bucket/shakespeare.txt", 32 | expected: "s3", 33 | }, 34 | { 35 | uri: "/etc/passwd", 36 | expected: "file", 37 | }, 38 | } 39 | for _, tc := range testcases { 40 | protocol, err := utils.ParseProtocol(tc.uri) 41 | if err != nil { 42 | t.Errorf("unexpected error: %v", err) 43 | } 44 | if protocol != tc.expected { 45 | t.Errorf("expected %s, got %s", tc.expected, protocol) 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /pkg/version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | ) 8 | 9 | var ( 10 | Version string = "unknown" 11 | Commit string = "unknown" 12 | Date string = time.Now().Format("2006-01-02") 13 | ) 14 | 15 | func GetVersion() string { 16 | return fmt.Sprintf("Version: %s\nCommit: %s\nDate: %s\n", Version, Commit, Date) 17 | } 18 | 19 | func PrintVersion() { 20 | os.Stderr.WriteString(GetVersion()) 21 | os.Exit(0) 22 | } 23 | -------------------------------------------------------------------------------- /prometheus.go: -------------------------------------------------------------------------------- 1 | package gojob 2 | 3 | import ( 4 | "fmt" 5 | "log/slog" 6 | "sync" 7 | 8 | "github.com/WangYihang/gojob/pkg/runner" 9 | "github.com/WangYihang/gojob/pkg/utils" 10 | "github.com/WangYihang/gojob/pkg/version" 11 | "github.com/prometheus/client_golang/prometheus" 12 | "github.com/prometheus/client_golang/prometheus/collectors" 13 | "github.com/prometheus/client_golang/prometheus/push" 14 | io_prometheus_client "github.com/prometheus/client_model/go" 15 | ) 16 | 17 | var ( 18 | numTotal = prometheus.NewGauge( 19 | prometheus.GaugeOpts{ 20 | Name: "num_total", 21 | Help: "Total number of processed events", 22 | }, 23 | ) 24 | numFailed = prometheus.NewGauge( 25 | prometheus.GaugeOpts{ 26 | Name: "num_failed", 27 | Help: "Total number of failed events", 28 | }, 29 | ) 30 | numSucceed = prometheus.NewGauge( 31 | prometheus.GaugeOpts{ 32 | Name: "num_succeed", 33 | Help: "Total number of succeeded events", 34 | }, 35 | ) 36 | numFinished = prometheus.NewGauge( 37 | prometheus.GaugeOpts{ 38 | Name: "num_finished", 39 | Help: "Total number of finished events", 40 | }, 41 | ) 42 | ) 43 | 44 | type customMetricsRegistry struct { 45 | *prometheus.Registry 46 | customLabels []*io_prometheus_client.LabelPair 47 | } 48 | 49 | func NewRegistryWithLabels(labels map[string]string) *customMetricsRegistry { 50 | c := &customMetricsRegistry{ 51 | Registry: prometheus.NewRegistry(), 52 | } 53 | for k, v := range labels { 54 | c.customLabels = append(c.customLabels, &io_prometheus_client.LabelPair{ 55 | Name: &k, 56 | Value: &v, 57 | }) 58 | } 59 | return c 60 | } 61 | 62 | func (g *customMetricsRegistry) Gather() ([]*io_prometheus_client.MetricFamily, error) { 63 | metricFamilies, err := g.Registry.Gather() 64 | 65 | for _, metricFamily := range metricFamilies { 66 | metrics := metricFamily.Metric 67 | for _, metric := range metrics { 68 | metric.Label = append(metric.Label, g.customLabels...) 69 | } 70 | } 71 | 72 | return metricFamilies, err 73 | } 74 | 75 | func prometheusPusher(url, job string, statusChan <-chan Status, wg *sync.WaitGroup) { 76 | instance := fmt.Sprintf( 77 | "gojob-%s-%s-%s-%s", 78 | version.Version, 79 | utils.Sanitize(runner.Runner.Country), 80 | utils.Sanitize(runner.Runner.City), 81 | runner.Runner.IP, 82 | ) 83 | registry := NewRegistryWithLabels(map[string]string{ 84 | "gojob_version": version.Version, 85 | "gojob_runner_ip": runner.Runner.IP, 86 | "gojob_runner_country": runner.Runner.Country, 87 | "gojob_runner_region": runner.Runner.Region, 88 | "gojob_runner_city": runner.Runner.City, 89 | }) 90 | registry.MustRegister(numTotal, numFailed, numSucceed, numFinished) 91 | registry.MustRegister( 92 | collectors.NewGoCollector(), 93 | collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), 94 | ) 95 | go func() { 96 | for status := range statusChan { 97 | slog.Info("promehteus pusher", slog.Any("status", status)) 98 | numTotal.Set(float64(status.NumTotal)) 99 | numFailed.Set(float64(status.NumFailed)) 100 | numSucceed.Set(float64(status.NumSucceed)) 101 | numFinished.Set(float64(status.NumFinished)) 102 | if err := push.New(url, job).Grouping( 103 | "instance", instance, 104 | ).Gatherer(registry).Push(); err != nil { 105 | slog.Error("error occurred while pushing to prometheus", slog.String("error", err.Error())) 106 | } 107 | } 108 | wg.Done() 109 | }() 110 | } 111 | -------------------------------------------------------------------------------- /scheduler.go: -------------------------------------------------------------------------------- 1 | package gojob 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log/slog" 7 | "sync" 8 | "sync/atomic" 9 | "time" 10 | 11 | "github.com/WangYihang/gojob/pkg/utils" 12 | "github.com/WangYihang/uio" 13 | "github.com/google/uuid" 14 | ) 15 | 16 | type schedulerMetadata map[string]interface{} 17 | 18 | // Scheduler is a task scheduler 19 | type Scheduler struct { 20 | id string 21 | numWorkers int 22 | metadata schedulerMetadata 23 | 24 | maxRetries int 25 | maxRuntimePerTaskSeconds int 26 | 27 | numShards int64 28 | shard int64 29 | 30 | isStarted atomic.Bool 31 | currentIndex atomic.Int64 32 | 33 | statusManager *statusManager 34 | 35 | taskChan chan *basicTask 36 | resultChans []chan *basicTask 37 | 38 | resultFilePath string 39 | statusFilePath string 40 | metadataFilePath string 41 | prometheusPushGatewayUrl string 42 | prometheusPushGatewayJob string 43 | 44 | taskWg *sync.WaitGroup 45 | recorderWg *sync.WaitGroup 46 | } 47 | 48 | type schedulerOption func(*Scheduler) error 49 | 50 | func New(options ...schedulerOption) *Scheduler { 51 | id := uuid.New().String() 52 | svr := &Scheduler{ 53 | id: id, 54 | numWorkers: 1, 55 | metadata: schedulerMetadata{"id": id}, 56 | 57 | maxRetries: 4, 58 | maxRuntimePerTaskSeconds: 16, 59 | 60 | numShards: 1, 61 | shard: 0, 62 | 63 | isStarted: atomic.Bool{}, 64 | currentIndex: atomic.Int64{}, 65 | 66 | statusManager: newStatusManager(), 67 | 68 | taskChan: make(chan *basicTask), 69 | resultChans: []chan *basicTask{}, 70 | 71 | resultFilePath: "result.json", 72 | statusFilePath: "status.json", 73 | metadataFilePath: "metadata.json", 74 | 75 | prometheusPushGatewayUrl: "", 76 | prometheusPushGatewayJob: "gojob", 77 | 78 | taskWg: &sync.WaitGroup{}, 79 | recorderWg: &sync.WaitGroup{}, 80 | } 81 | for _, opt := range options { 82 | err := opt(svr) 83 | if err != nil { 84 | panic(err) 85 | } 86 | } 87 | go svr.statusManager.Start() 88 | svr.recorderWg.Add(4) 89 | chanRecorder(svr.resultFilePath, svr.ResultChan(), svr.recorderWg) 90 | chanRecorder(svr.statusFilePath, svr.StatusChan(), svr.recorderWg) 91 | chanRecorder("-", svr.StatusChan(), svr.recorderWg) 92 | metadataChan := make(chan schedulerMetadata) 93 | chanRecorder(svr.metadataFilePath, metadataChan, svr.recorderWg) 94 | metadataChan <- svr.metadata 95 | close(metadataChan) 96 | if svr.prometheusPushGatewayUrl != "" { 97 | svr.recorderWg.Add(1) 98 | prometheusPusher(svr.prometheusPushGatewayUrl, svr.prometheusPushGatewayJob, svr.StatusChan(), svr.recorderWg) 99 | } 100 | return svr 101 | } 102 | 103 | // SetNumShards sets the number of shards, default is 1 which means no sharding 104 | func WithNumShards(numShards int64) schedulerOption { 105 | return func(s *Scheduler) error { 106 | if numShards <= 0 { 107 | return fmt.Errorf("numShards must be greater than 0") 108 | } 109 | s.numShards = numShards 110 | return nil 111 | } 112 | } 113 | 114 | // SetShard sets the shard (from 0 to NumShards-1) 115 | func WithShard(shard int64) schedulerOption { 116 | return func(s *Scheduler) error { 117 | if shard < 0 || shard >= s.numShards { 118 | return fmt.Errorf("shard must be in [0, NumShards)") 119 | } 120 | s.shard = shard 121 | return nil 122 | } 123 | } 124 | 125 | // SetNumWorkers sets the number of workers 126 | func WithNumWorkers(numWorkers int) schedulerOption { 127 | return func(s *Scheduler) error { 128 | if numWorkers <= 0 { 129 | return fmt.Errorf("numWorkers must be greater than 0") 130 | } 131 | s.numWorkers = numWorkers 132 | return nil 133 | } 134 | } 135 | 136 | // SetMaxRetries sets the max retries 137 | func WithMaxRetries(maxRetries int) schedulerOption { 138 | return func(s *Scheduler) error { 139 | if maxRetries <= 0 { 140 | return fmt.Errorf("maxRetries must be greater than 0") 141 | } 142 | s.maxRetries = maxRetries 143 | return nil 144 | } 145 | } 146 | 147 | // SetMaxRuntimePerTaskSeconds sets the max runtime per task seconds 148 | func WithMaxRuntimePerTaskSeconds(maxRuntimePerTaskSeconds int) schedulerOption { 149 | return func(s *Scheduler) error { 150 | if maxRuntimePerTaskSeconds <= 0 { 151 | return fmt.Errorf("maxRuntimePerTaskSeconds must be greater than 0") 152 | } 153 | s.maxRuntimePerTaskSeconds = maxRuntimePerTaskSeconds 154 | return nil 155 | } 156 | } 157 | 158 | // WithTotalTasks sets the total number of tasks, and calculates the number of tasks for this shard 159 | func WithTotalTasks(numTotalTasks int64) schedulerOption { 160 | return func(s *Scheduler) error { 161 | // Check if NumShards is set and is greater than 0 162 | if s.numShards <= 0 { 163 | return fmt.Errorf("number of shards must be greater than 0") 164 | } 165 | 166 | // Check if Shard is set and is within the valid range [0, NumShards) 167 | if s.shard < 0 || s.shard >= s.numShards { 168 | return fmt.Errorf("shard must be within the range [0, NumShards)") 169 | } 170 | 171 | // Calculate the base number of tasks per shard 172 | baseTasksPerShard := numTotalTasks / int64(s.numShards) 173 | 174 | // Calculate the remainder 175 | remainder := numTotalTasks % int64(s.numShards) 176 | 177 | // Adjust task count for shards that need to handle an extra task due to the remainder 178 | if int64(s.shard) < remainder { 179 | baseTasksPerShard++ 180 | } 181 | 182 | // Store the number of tasks for this shard 183 | s.statusManager.SetTotal(baseTasksPerShard) 184 | return nil 185 | } 186 | } 187 | 188 | // AddMetadata adds metadata 189 | func WithMetadata(key string, value interface{}) schedulerOption { 190 | return func(s *Scheduler) error { 191 | s.metadata[key] = value 192 | return nil 193 | } 194 | } 195 | 196 | // WithResultFilePath sets the file path for results 197 | func WithResultFilePath(path string) schedulerOption { 198 | return func(s *Scheduler) error { 199 | s.resultFilePath = path 200 | return nil 201 | } 202 | } 203 | 204 | // WithStatusFilePath sets the file path for status 205 | func WithStatusFilePath(path string) schedulerOption { 206 | return func(s *Scheduler) error { 207 | s.statusFilePath = path 208 | return nil 209 | } 210 | } 211 | 212 | func WithPrometheusPushGateway(url string, job string) schedulerOption { 213 | return func(s *Scheduler) error { 214 | s.prometheusPushGatewayUrl = url 215 | s.prometheusPushGatewayJob = job 216 | return nil 217 | } 218 | } 219 | 220 | // WithMetadataFilePath sets the file path for metadata 221 | func WithMetadataFilePath(path string) schedulerOption { 222 | return func(s *Scheduler) error { 223 | s.metadataFilePath = path 224 | return nil 225 | } 226 | } 227 | 228 | // chanRecorder records the channel to a given file path 229 | func chanRecorder[T *basicTask | Status | schedulerMetadata](path string, ch <-chan T, wg *sync.WaitGroup) { 230 | fd, err := uio.Open(path) 231 | if err != nil { 232 | slog.Error("error occurred while opening file", slog.String("path", path), slog.String("error", err.Error())) 233 | return 234 | } 235 | go func() { 236 | encoder := json.NewEncoder(fd) 237 | for item := range ch { 238 | if err := encoder.Encode(item); err != nil { 239 | slog.Error("error occurred while serializing data", slog.String("path", path), slog.String("error", err.Error())) 240 | } 241 | } 242 | fd.Close() 243 | wg.Done() 244 | }() 245 | } 246 | 247 | // ResultChan returns a newly created channel to receive results 248 | // Everytime ResultChan is called, a new channel is created, and the results are written to all channels 249 | // This is useful for multiple consumers (e.g. writing to multiple files) 250 | func (s *Scheduler) ResultChan() <-chan *basicTask { 251 | c := make(chan *basicTask) 252 | s.resultChans = append(s.resultChans, c) 253 | return c 254 | } 255 | 256 | // StatusChan returns a newly created channel to receive status 257 | // Everytime StatusChan is called, a new channel is created, and the status are written to all channels 258 | // This is useful for multiple consumers (e.g. writing to multiple files, report to prometheus, etc.) 259 | func (s *Scheduler) StatusChan() <-chan Status { 260 | return s.statusManager.StatusChan() 261 | } 262 | 263 | func (s *Scheduler) Metadata() map[string]interface{} { 264 | return s.metadata 265 | } 266 | 267 | // Submit submits a task to the scheduler 268 | func (s *Scheduler) Submit(task Task) { 269 | if !s.isStarted.Load() { 270 | s.Start() 271 | } 272 | index := s.currentIndex.Load() 273 | if (index % s.numShards) == s.shard { 274 | s.taskWg.Add(1) 275 | s.taskChan <- newBasicTask(index, task) 276 | } 277 | s.currentIndex.Add(1) 278 | } 279 | 280 | // Start starts the scheduler 281 | func (s *Scheduler) Start() *Scheduler { 282 | for i := 0; i < s.numWorkers; i++ { 283 | go s.Worker() 284 | } 285 | s.isStarted.Store(true) 286 | return s 287 | } 288 | 289 | // Wait waits for all tasks to finish 290 | func (s *Scheduler) Wait() { 291 | // Wait for all tasks to finish 292 | s.taskWg.Wait() 293 | // Close task channel 294 | close(s.taskChan) 295 | // Close result channels 296 | for _, resultChan := range s.resultChans { 297 | close(resultChan) 298 | } 299 | // Wait for all recorders to finish 300 | s.statusManager.Stop() 301 | // Wait for all recorders to finish 302 | s.recorderWg.Wait() 303 | } 304 | 305 | func (s *Scheduler) Status() Status { 306 | return s.statusManager.Snapshot() 307 | } 308 | 309 | // Worker is a worker 310 | func (s *Scheduler) Worker() { 311 | for task := range s.taskChan { 312 | // Start task 313 | for i := 0; i < s.maxRetries; i++ { 314 | err := func() error { 315 | task.StartedAt = time.Now().UnixMicro() 316 | defer func() { 317 | task.NumTries++ 318 | task.FinishedAt = time.Now().UnixMicro() 319 | }() 320 | return utils.RunWithTimeout(task.Task.Do, time.Duration(s.maxRuntimePerTaskSeconds)*time.Second) 321 | }() 322 | if err != nil { 323 | task.Error = err.Error() 324 | } else { 325 | task.Error = "" 326 | break 327 | } 328 | } 329 | // Write log 330 | for _, resultChan := range s.resultChans { 331 | resultChan <- task 332 | } 333 | // Update status 334 | if task.Error != "" { 335 | s.statusManager.IncFailed() 336 | } else { 337 | s.statusManager.IncSucceed() 338 | } 339 | // Notify task is done 340 | s.taskWg.Done() 341 | } 342 | } 343 | -------------------------------------------------------------------------------- /scheduler_test.go: -------------------------------------------------------------------------------- 1 | package gojob_test 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "sort" 7 | "strconv" 8 | "strings" 9 | "sync" 10 | "testing" 11 | 12 | "github.com/WangYihang/gojob" 13 | ) 14 | 15 | type safeWriter struct { 16 | writer *strings.Builder 17 | lock sync.Mutex 18 | } 19 | 20 | func newSafeWriter() *safeWriter { 21 | return &safeWriter{ 22 | writer: new(strings.Builder), 23 | lock: sync.Mutex{}, 24 | } 25 | } 26 | 27 | func (sw *safeWriter) WriteString(s string) { 28 | sw.lock.Lock() 29 | defer sw.lock.Unlock() 30 | sw.writer.WriteString(s) 31 | } 32 | 33 | func (sw *safeWriter) String() string { 34 | return sw.writer.String() 35 | } 36 | 37 | type schedulerTestTask struct { 38 | I int 39 | writer *safeWriter 40 | } 41 | 42 | func newTask(i int, writer *safeWriter) *schedulerTestTask { 43 | return &schedulerTestTask{ 44 | I: i, 45 | writer: writer, 46 | } 47 | } 48 | 49 | func (t *schedulerTestTask) Do() error { 50 | t.writer.WriteString(fmt.Sprintf("%d\n", t.I)) 51 | return nil 52 | } 53 | 54 | func TestSharding(t *testing.T) { 55 | testcases := []struct { 56 | numShards int64 57 | shard int64 58 | expected []int 59 | }{ 60 | { 61 | numShards: 2, 62 | shard: 0, 63 | expected: []int{0, 2, 4, 6, 8, 10, 12, 14}, 64 | }, 65 | { 66 | numShards: 2, 67 | shard: 1, 68 | expected: []int{1, 3, 5, 7, 9, 11, 13, 15}, 69 | }, 70 | { 71 | numShards: 3, 72 | shard: 0, 73 | expected: []int{0, 3, 6, 9, 12, 15}, 74 | }, 75 | { 76 | numShards: 3, 77 | shard: 1, 78 | expected: []int{1, 4, 7, 10, 13}, 79 | }, 80 | { 81 | numShards: 3, 82 | shard: 2, 83 | expected: []int{2, 5, 8, 11, 14}, 84 | }, 85 | } 86 | for _, tc := range testcases { 87 | safeWriter := newSafeWriter() 88 | scheduler := gojob.New( 89 | gojob.WithNumShards(tc.numShards), 90 | gojob.WithShard(tc.shard), 91 | gojob.WithResultFilePath("-"), 92 | gojob.WithStatusFilePath("-"), 93 | gojob.WithMetadataFilePath("-"), 94 | ).Start() 95 | for i := 0; i < 16; i++ { 96 | scheduler.Submit(newTask(i, safeWriter)) 97 | } 98 | scheduler.Wait() 99 | output := safeWriter.String() 100 | lines := strings.Split(output, "\n") 101 | numbers := []int{} 102 | for _, line := range lines { 103 | if line == "" { 104 | continue 105 | } 106 | number, err := strconv.Atoi(line) 107 | if err != nil { 108 | t.Fatal(err) 109 | } 110 | numbers = append(numbers, number) 111 | } 112 | sort.Ints(numbers) 113 | if !reflect.DeepEqual(numbers, tc.expected) { 114 | t.Errorf("Expected %v, got %v", tc.expected, numbers) 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /status.go: -------------------------------------------------------------------------------- 1 | package gojob 2 | 3 | import ( 4 | "sync" 5 | "sync/atomic" 6 | "time" 7 | ) 8 | 9 | // Status represents the status of the job. 10 | type Status struct { 11 | Timestamp string `json:"timestamp"` 12 | NumFailed int64 `json:"num_failed"` 13 | NumSucceed int64 `json:"num_succeed"` 14 | NumFinished int64 `json:"num_done"` 15 | NumTotal int64 `json:"num_total"` 16 | } 17 | 18 | type statusManager struct { 19 | numFailed atomic.Int64 20 | numSucceed atomic.Int64 21 | numTotal atomic.Int64 22 | 23 | mutex sync.Mutex 24 | ticker *time.Ticker 25 | statusChans []chan Status 26 | } 27 | 28 | func newStatusManager() *statusManager { 29 | return &statusManager{ 30 | numFailed: atomic.Int64{}, 31 | numSucceed: atomic.Int64{}, 32 | numTotal: atomic.Int64{}, 33 | mutex: sync.Mutex{}, 34 | ticker: time.NewTicker(5 * time.Second), 35 | statusChans: []chan Status{}, 36 | } 37 | } 38 | 39 | func (sm *statusManager) notify() { 40 | status := sm.Snapshot() 41 | sm.mutex.Lock() 42 | for _, ch := range sm.statusChans { 43 | ch <- status 44 | } 45 | sm.mutex.Unlock() 46 | } 47 | 48 | // Start starts the status manager. 49 | // It will notify all the status channels every second. 50 | func (sm *statusManager) Start() { 51 | sm.notify() 52 | for range sm.ticker.C { 53 | sm.notify() 54 | } 55 | } 56 | 57 | // Stop stops the status manager. 58 | func (sm *statusManager) Stop() { 59 | sm.notify() 60 | sm.notify() 61 | sm.ticker.Stop() 62 | for _, ch := range sm.statusChans { 63 | close(ch) 64 | } 65 | } 66 | 67 | // IncFailed increments the number of failed jobs. 68 | func (sm *statusManager) IncFailed() { 69 | sm.numFailed.Add(1) 70 | } 71 | 72 | // IncSucceed increments the number of succeed jobs. 73 | func (sm *statusManager) IncSucceed() { 74 | sm.numSucceed.Add(1) 75 | } 76 | 77 | // SetTotal sets the total number of jobs. 78 | // It should be called before the job starts. 79 | func (sm *statusManager) SetTotal(total int64) { 80 | sm.numTotal.Store(total) 81 | } 82 | 83 | // StatusChan returns a channel that will receive the status of the job. 84 | // The status will be sent every second. It should be called before the job starts. 85 | // You can call it multiple times to get multiple channels. 86 | func (sm *statusManager) StatusChan() <-chan Status { 87 | ch := make(chan Status) 88 | sm.mutex.Lock() 89 | sm.statusChans = append(sm.statusChans, ch) 90 | sm.mutex.Unlock() 91 | return ch 92 | } 93 | 94 | // Snapshot returns the current status of the job. 95 | func (sm *statusManager) Snapshot() Status { 96 | sm.mutex.Lock() 97 | numFailed := sm.numFailed.Load() 98 | numSucceed := sm.numSucceed.Load() 99 | numTotal := sm.numTotal.Load() 100 | sm.mutex.Unlock() 101 | return Status{ 102 | Timestamp: time.Now().Format(time.RFC3339), 103 | NumFailed: numFailed, 104 | NumSucceed: numSucceed, 105 | NumFinished: numFailed + numSucceed, 106 | NumTotal: numTotal, 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /task.go: -------------------------------------------------------------------------------- 1 | package gojob 2 | 3 | import ( 4 | "github.com/google/uuid" 5 | ) 6 | 7 | // Task is an interface that defines a task 8 | type Task interface { 9 | // Do starts the task, returns error if failed 10 | // If an error is returned, the task will be retried until MaxRetries 11 | // You can set MaxRetries by calling SetMaxRetries on the scheduler 12 | Do() error 13 | } 14 | 15 | type basicTask struct { 16 | Index int64 `json:"index"` 17 | ID string `json:"id"` 18 | StartedAt int64 `json:"started_at"` 19 | FinishedAt int64 `json:"finished_at"` 20 | NumTries int `json:"num_tries"` 21 | Task Task `json:"task"` 22 | Error string `json:"error"` 23 | } 24 | 25 | func newBasicTask(index int64, task Task) *basicTask { 26 | return &basicTask{ 27 | Index: index, 28 | ID: uuid.New().String(), 29 | StartedAt: 0, 30 | FinishedAt: 0, 31 | NumTries: 0, 32 | Task: task, 33 | Error: "", 34 | } 35 | } 36 | --------------------------------------------------------------------------------