├── .dockerignore ├── .github └── workflows │ └── build.yml ├── .golangci.yml ├── .goreleaser.yml ├── Dockerfile ├── LICENSE ├── README.md ├── client.go ├── client_test.go ├── cluster_test.go ├── cmd └── dalga │ └── main.go ├── config.go ├── config.toml ├── dalga.go ├── dalga_test.go ├── example_endpoint.py ├── go.mod ├── go.sum ├── internal ├── clock │ └── clock.go ├── instance │ └── instance.go ├── jobmanager │ └── jobmanager.go ├── log │ └── log.go ├── retry │ ├── retry.go │ └── retry_test.go ├── scheduler │ ├── benchmark_test.go │ ├── scheduler.go │ └── scheduler_test.go ├── server │ └── server.go └── table │ ├── job.go │ ├── table.go │ └── table_test.go └── recur_test.go /.dockerignore: -------------------------------------------------------------------------------- 1 | .git/ 2 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | name: Test 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/setup-go@v2 11 | with: 12 | go-version: 1.15 13 | 14 | - uses: actions/checkout@v2 15 | 16 | - uses: actions/cache@v2 17 | with: 18 | path: ~/go/pkg/mod 19 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 20 | restore-keys: | 21 | ${{ runner.os }}-go- 22 | 23 | - name: Get dependencies 24 | run: go get -v -t -d ./... 25 | 26 | - name: Build 27 | run: go build -v ./... 28 | 29 | - name: Lint 30 | uses: golangci/golangci-lint-action@v2 31 | with: 32 | version: v1.30 33 | 34 | - name: Setup MySQL server 35 | run: | 36 | sudo systemctl start mysql.service 37 | mysqladmin -u root -p'root' password '' 38 | mysql -u root -e 'create database test' 39 | 40 | - name: Test 41 | run: go test -race -v -covermode atomic -coverprofile=covprofile ./... 42 | 43 | - name: Send coverage 44 | uses: shogo82148/actions-goveralls@v1 45 | with: 46 | path-to-profile: covprofile 47 | 48 | goreleaser: 49 | name: Build and release binaries 50 | needs: test 51 | runs-on: ubuntu-latest 52 | if: startsWith(github.ref, 'refs/tags/v') 53 | steps: 54 | - uses: actions/checkout@v2 55 | - uses: actions/setup-go@v2 56 | with: 57 | go-version: 1.15 58 | - uses: goreleaser/goreleaser-action@v2.2.0 59 | with: 60 | args: release --rm-dist 61 | env: 62 | GITHUB_TOKEN: ${{ secrets.github_token }} 63 | 64 | docker: 65 | name: Build and push Docker image 66 | needs: test 67 | runs-on: ubuntu-latest 68 | if: startsWith(github.ref, 'refs/tags/v') 69 | steps: 70 | - uses: actions/checkout@v2 71 | 72 | - name: Prepare 73 | id: prep 74 | run: | 75 | DOCKER_IMAGE=cenkalti/dalga 76 | VERSION=edge 77 | if [[ $GITHUB_REF == refs/tags/* ]]; then 78 | VERSION=${GITHUB_REF#refs/tags/v} 79 | fi 80 | TAGS="${DOCKER_IMAGE}:${VERSION}" 81 | if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then 82 | TAGS="$TAGS,${DOCKER_IMAGE}:latest" 83 | fi 84 | echo ::set-output name=version::${VERSION} 85 | echo ::set-output name=date::$(date --utc +'%Y-%m-%dT%H:%M:%SZ') 86 | echo ::set-output name=tags::${TAGS} 87 | 88 | - name: Set up Docker Buildx 89 | uses: docker/setup-buildx-action@v1 90 | 91 | - name: Login to DockerHub 92 | uses: docker/login-action@v1 93 | with: 94 | username: ${{ secrets.DOCKER_USERNAME }} 95 | password: ${{ secrets.DOCKER_PASSWORD }} 96 | 97 | - name: Build and push 98 | uses: docker/build-push-action@v2 99 | with: 100 | push: true 101 | tags: ${{ steps.prep.outputs.tags }} 102 | build-args: VERSION=${{ steps.prep.outputs.version }},COMMIT=${{ github.sha }},DATE=${{ steps.prep.outputs.date }} 103 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters: 2 | enable-all: true 3 | disable: 4 | - maligned 5 | - lll 6 | - gochecknoglobals 7 | - gochecknoinits 8 | - gocyclo 9 | - nakedret 10 | - funlen 11 | - wsl 12 | - gocognit 13 | - gomnd 14 | - nlreturn 15 | - goerr113 16 | issues: 17 | exclude-rules: 18 | - path: _test\.go 19 | linters: 20 | - goconst 21 | - errcheck 22 | - noctx 23 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | env: 2 | - GO111MODULE=on 3 | before: 4 | hooks: 5 | - go mod download 6 | builds: 7 | - env: 8 | - CGO_ENABLED=0 9 | ldflags: -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} 10 | goos: 11 | - linux 12 | goarch: 13 | - amd64 14 | main: ./cmd/dalga/main.go 15 | archives: 16 | - format: tar.gz 17 | name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}" 18 | checksum: 19 | name_template: 'checksums.txt' 20 | changelog: 21 | sort: asc 22 | filters: 23 | exclude: 24 | - '^docs:' 25 | - '^test:' 26 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.15 2 | 3 | WORKDIR /go/src/app 4 | 5 | COPY go.mod go.sum ./ 6 | RUN go mod download 7 | 8 | COPY . . 9 | ARG VERSION 10 | ARG COMMIT 11 | ARG DATE 12 | RUN CGO_ENABLED=0 go build -o /go/bin/dalga -ldflags="-s -w -X main.version=$VERSION -X main.commit=$COMMIT -X main.date=$DATE" ./cmd/dalga 13 | 14 | FROM alpine:latest 15 | RUN touch /etc/dalga.toml 16 | COPY --from=0 /go/bin/dalga /bin/dalga 17 | ENTRYPOINT ["/bin/dalga", "-config", "/etc/dalga.toml"] 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 Cenk Altı 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Dalga 2 | ===== 3 | 4 | Dalga is a job scheduler. It's like cron-as-a-service. 5 | 6 | - Can schedule periodic or one-off jobs. 7 | - Stores jobs in a MySQL table with location info. 8 | - Has an HTTP interface for scheduling and cancelling jobs. 9 | - Makes a POST request to the endpoint defined in config on the job's execution time. 10 | - Retries failed jobs with constant or exponential backoff. 11 | - Multiple instances can be run for high availability and scaling out. 12 | 13 | Install 14 | ------- 15 | 16 | Use pre-built Docker image: 17 | 18 | $ docker run -e DALGA_MYSQL_HOST=mysql.example.com cenkalti/dalga 19 | 20 | or download the latest binary from [releases page](https://github.com/cenkalti/dalga/releases). 21 | 22 | Usage 23 | ----- 24 | 25 | See [example config file](https://github.com/cenkalti/dalga/blob/v3/config.toml) for configuration options. 26 | TOML and YAML file formats are supported. 27 | Configuration values can also be set via environment variables with `DALGA_` prefix. 28 | 29 | First, you must create the table for storing jobs: 30 | 31 | $ dalga -config dalga.toml -create-tables 32 | 33 | Then, run the server: 34 | 35 | $ dalga -config dalga.toml 36 | 37 | Schedule a new job to run every 60 seconds: 38 | 39 | $ curl -i -X PUT 'http://127.0.0.1:34006/jobs/check_feed/1234?interval=60' 40 | HTTP/1.1 201 Created 41 | Content-Type: application/json; charset=utf-8 42 | Date: Tue, 11 Nov 2014 22:10:40 GMT 43 | Content-Length: 83 44 | 45 | {"path":"check_feed","body":"1234","interval":60,"next_run":"2014-11-11T22:11:40Z"} 46 | 47 | PUT always returns 201. If there is an existing job with path and body, it will be rescheduled. 48 | 49 | There are 4 options that you can pass to `Schedule` but not every combination is valid: 50 | 51 | | Param | Description | Type | Example | 52 | | ----- | ----------- | ---- | ------- | 53 | | interval | Run job at intervals | Integer or ISO 8601 interval | 60 or PT60S | 54 | | first-run | Do not run job until this time | RFC3339 Timestamp | 1985-04-12T23:20:50.52Z | 55 | | one-off | Run job only once | Boolean | true, false, 1, 0 | 56 | | immediate | Run job immediately as it is scheduled | Boolean | true, false, 1, 0 | 57 | 58 | 60 seconds later, Dalga makes a POST to your endpoint defined in config: 59 | 60 | Path: / 61 | Body: 62 | 63 | The endpoint must return 200 if the job is successful. 64 | 65 | The endpoint may return 204 if job is invalid. In this case Dalga will remove the job from the table. 66 | 67 | Anything other than 200 or 204 makes Dalga to retry the job indefinitely with an exponential backoff. 68 | 69 | Get the status of a job: 70 | 71 | $ curl -i -X GET 'http://127.0.0.1:34006/jobs/check_feed/1234' 72 | HTTP/1.1 200 OK 73 | Content-Type: application/json; charset=utf-8 74 | Date: Tue, 11 Nov 2014 22:12:21 GMT 75 | Content-Length: 83 76 | 77 | {"path":"check_feed","body":"1234","interval":60,"next_run":"2014-11-11T22:12:41Z"} 78 | 79 | GET may return 404 if job is not found. 80 | 81 | Cancel previously scheduled job: 82 | 83 | $ curl -i -X DELETE 'http://127.0.0.1:34006/jobs/check_feed/1234' 84 | HTTP/1.1 204 No Content 85 | Date: Tue, 11 Nov 2014 22:13:35 GMT 86 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | package dalga 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "net/http" 9 | "net/url" 10 | "strings" 11 | "time" 12 | 13 | "github.com/cenkalti/dalga/v3/internal/jobmanager" 14 | "github.com/cenkalti/dalga/v3/internal/server" 15 | "github.com/cenkalti/dalga/v3/internal/table" 16 | "github.com/senseyeio/duration" 17 | ) 18 | 19 | // ErrNotExist is returned when requested job does not exist. 20 | var ErrNotExist = table.ErrNotExist 21 | 22 | // ClientOpt is an option that can be provided to a Dalga client. 23 | type ClientOpt func(c *Client) 24 | 25 | // WithClient provides a specific HTTP client. 26 | func WithClient(clnt *http.Client) ClientOpt { 27 | return func(c *Client) { 28 | c.clnt = clnt 29 | } 30 | } 31 | 32 | // Client is used to interact with a Dalga cluster using REST. 33 | type Client struct { 34 | clnt *http.Client 35 | BaseURL string 36 | } 37 | 38 | // NewClient creates a REST Client for a Dalga cluster. 39 | func NewClient(baseURL string, opts ...ClientOpt) *Client { 40 | c := &Client{ 41 | BaseURL: strings.TrimSuffix(baseURL, "/"), 42 | clnt: http.DefaultClient, 43 | } 44 | for _, o := range opts { 45 | o(c) 46 | } 47 | return c 48 | } 49 | 50 | // Get retrieves the job with path and body. 51 | func (clnt *Client) Get(ctx context.Context, path, body string) (*Job, error) { 52 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, clnt.jobURL(path, body), nil) 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | resp, err := clnt.clnt.Do(req) 58 | if err != nil { 59 | select { 60 | case <-ctx.Done(): 61 | err = ctx.Err() 62 | default: 63 | } 64 | return nil, fmt.Errorf("cannot get job: %w", err) 65 | } 66 | defer resp.Body.Close() 67 | var buf bytes.Buffer 68 | _, _ = buf.ReadFrom(resp.Body) 69 | if resp.StatusCode == http.StatusNotFound { 70 | return nil, ErrNotExist 71 | } else if resp.StatusCode != http.StatusOK { 72 | return nil, fmt.Errorf("unexpected status code: %d, body: %q", resp.StatusCode, buf.String()) 73 | } 74 | 75 | var j Job 76 | dec := json.NewDecoder(&buf) 77 | if err := dec.Decode(&j); err != nil { 78 | return nil, fmt.Errorf("cannot unmarshal body: %q, cause: %w", buf.String(), err) 79 | } 80 | 81 | return &j, nil 82 | } 83 | 84 | // Schedule creates a new job with path and body, and the provided options. 85 | func (clnt *Client) Schedule(ctx context.Context, path, body string, opts ...ScheduleOpt) (*Job, error) { 86 | so := jobmanager.ScheduleOptions{} 87 | for _, o := range opts { 88 | o(&so) 89 | } 90 | 91 | values := make(url.Values) 92 | if !so.Interval.IsZero() { 93 | values.Set("interval", so.Interval.String()) 94 | } 95 | if so.Location != nil { 96 | values.Set("location", so.Location.String()) 97 | } 98 | if !so.FirstRun.IsZero() { 99 | values.Set("first-run", so.FirstRun.Format(time.RFC3339)) 100 | } 101 | if so.OneOff { 102 | values.Set("one-off", "true") 103 | } 104 | if so.Immediate { 105 | values.Set("immediate", "true") 106 | } 107 | 108 | req, err := http.NewRequestWithContext(ctx, http.MethodPut, clnt.jobURL(path, body), strings.NewReader(values.Encode())) 109 | if err != nil { 110 | return nil, err 111 | } 112 | req.Header.Set("Content-Type", "application/x-www-form-urlencoded") 113 | 114 | resp, err := clnt.clnt.Do(req) 115 | if err != nil { 116 | select { 117 | case <-ctx.Done(): 118 | err = ctx.Err() 119 | default: 120 | } 121 | return nil, fmt.Errorf("cannot schedule new job: %w", err) 122 | } 123 | defer resp.Body.Close() 124 | var buf bytes.Buffer 125 | _, _ = buf.ReadFrom(resp.Body) 126 | if resp.StatusCode != http.StatusCreated { 127 | return nil, fmt.Errorf("unexpected status code: %d, body: %q", resp.StatusCode, buf.String()) 128 | } 129 | 130 | var j Job 131 | dec := json.NewDecoder(&buf) 132 | if err := dec.Decode(&j); err != nil { 133 | return nil, fmt.Errorf("cannot unmarshal body: %q, cause: %w", buf.String(), err) 134 | } 135 | 136 | return &j, nil 137 | } 138 | 139 | // Disable stops the job with path and body from running at its scheduled times. 140 | func (clnt *Client) Disable(ctx context.Context, path, body string) (*Job, error) { 141 | return clnt.setEnabled(ctx, path, body, false) 142 | } 143 | 144 | // Enable allows the job with path and body to continue running at its scheduled times. 145 | // 146 | // If the next scheduled run is still in the future, the job will execute at that point. 147 | // If the scheduled run is now in the past, the behavior depends upon the value of 148 | // the FixedIntervals setting: 149 | // 150 | // If FixedIntervals is false, the job will run immediately. 151 | // 152 | // If FixedIntervals is true, the job will reschedule to the next appropriate point 153 | // in the future based on its interval setting, effectively skipping the scheduled 154 | // runs that were missed while the job was disabled. 155 | func (clnt *Client) Enable(ctx context.Context, path, body string) (*Job, error) { 156 | return clnt.setEnabled(ctx, path, body, true) 157 | } 158 | 159 | func (clnt *Client) setEnabled(ctx context.Context, path, body string, enabled bool) (*Job, error) { 160 | action := "enable" 161 | if !enabled { 162 | action = "disable" 163 | } 164 | req, err := http.NewRequestWithContext(ctx, http.MethodPatch, clnt.jobURL(path, body)+"?"+action+"=true", nil) 165 | if err != nil { 166 | return nil, err 167 | } 168 | 169 | resp, err := clnt.clnt.Do(req) 170 | if err != nil { 171 | select { 172 | case <-ctx.Done(): 173 | err = ctx.Err() 174 | default: 175 | } 176 | return nil, fmt.Errorf("cannot %s job: %w", action, err) 177 | } 178 | defer resp.Body.Close() 179 | var buf bytes.Buffer 180 | _, _ = buf.ReadFrom(resp.Body) 181 | if resp.StatusCode == http.StatusNotFound { 182 | return nil, ErrNotExist 183 | } else if resp.StatusCode != http.StatusOK { 184 | return nil, fmt.Errorf("unexpected status code: %d, body: %q", resp.StatusCode, buf.String()) 185 | } 186 | 187 | var j Job 188 | dec := json.NewDecoder(&buf) 189 | if err := dec.Decode(&j); err != nil { 190 | return nil, fmt.Errorf("cannot unmarshal body: %q, cause: %w", buf.String(), err) 191 | } 192 | 193 | return &j, nil 194 | } 195 | 196 | // Cancel deletes the job with path and body. 197 | func (clnt *Client) Cancel(ctx context.Context, path, body string) error { 198 | req, err := http.NewRequestWithContext(ctx, http.MethodDelete, clnt.jobURL(path, body), nil) 199 | if err != nil { 200 | return err 201 | } 202 | 203 | resp, err := clnt.clnt.Do(req) 204 | if err != nil { 205 | select { 206 | case <-ctx.Done(): 207 | err = ctx.Err() 208 | default: 209 | } 210 | return fmt.Errorf("cannot cancel job: %w", err) 211 | } 212 | defer resp.Body.Close() 213 | var buf bytes.Buffer 214 | _, _ = buf.ReadFrom(resp.Body) 215 | if resp.StatusCode != http.StatusNoContent { 216 | return fmt.Errorf("unexpected status code: %d, body: %q", resp.StatusCode, buf.String()) 217 | } 218 | 219 | return nil 220 | } 221 | 222 | // Status returns general information about the Dalga cluster. 223 | func (clnt *Client) Status(ctx context.Context) (*Status, error) { 224 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, clnt.BaseURL+"/status", nil) 225 | if err != nil { 226 | return nil, err 227 | } 228 | 229 | resp, err := clnt.clnt.Do(req) 230 | if err != nil { 231 | select { 232 | case <-ctx.Done(): 233 | err = ctx.Err() 234 | default: 235 | } 236 | return nil, fmt.Errorf("cannot get status: %w", err) 237 | } 238 | defer resp.Body.Close() 239 | var buf bytes.Buffer 240 | _, _ = buf.ReadFrom(resp.Body) 241 | if resp.StatusCode != http.StatusOK { 242 | return nil, fmt.Errorf("unexpected status code: %d, body: %q", resp.StatusCode, buf.String()) 243 | } 244 | 245 | var s Status 246 | dec := json.NewDecoder(&buf) 247 | if err := dec.Decode(&s); err != nil { 248 | return nil, fmt.Errorf("cannot unmarshal body: %q, cause: %w", buf.String(), err) 249 | } 250 | 251 | return &s, nil 252 | } 253 | 254 | func (clnt *Client) jobURL(path, body string) string { 255 | return fmt.Sprintf("%s/jobs/%s/%s", clnt.BaseURL, path, body) 256 | } 257 | 258 | // Status contains general information about a Dalga cluster. 259 | type Status = server.Status 260 | 261 | // ScheduleOpt is an option that can be provided to the Schedule method. 262 | type ScheduleOpt func(o *jobmanager.ScheduleOptions) 263 | 264 | // WithInterval specifies that a job should recur, with frequency 265 | // given as an ISO8601 duration as an interval: 266 | // https://en.wikipedia.org/wiki/ISO_8601#Time_intervals 267 | // 268 | // This option is incompatible with the WithOneOff option. 269 | func WithInterval(d duration.Duration) ScheduleOpt { 270 | return func(o *jobmanager.ScheduleOptions) { 271 | o.Interval = d 272 | } 273 | } 274 | 275 | // MustWithIntervalString is identical to WithInterval, except that it performs a parsing step. 276 | // It panics if s is not a valid ISO8601 duration. 277 | func MustWithIntervalString(s string) ScheduleOpt { 278 | d, err := duration.ParseISO8601(s) 279 | if err != nil { 280 | panic(err) 281 | } 282 | return WithInterval(d) 283 | } 284 | 285 | // WithLocation specifies what location a job's schedule should be relative to. 286 | // This is solely relevant for calculating intervals using an ISO8601 duration, 287 | // since "P1D" can mean 23 or 25 hours of real time if the job's location is 288 | // undergoing a daylight savings shift within that period. 289 | // 290 | // Note that Dalga will not double-execute a job if it's scheduled at a time that repeats 291 | // itself during a daylight savings shift, since it doesn't use wall clock time. 292 | // 293 | // If this option is omitted, the job will default to UTC as a location. 294 | func WithLocation(l *time.Location) ScheduleOpt { 295 | return func(o *jobmanager.ScheduleOptions) { 296 | o.Location = l 297 | } 298 | } 299 | 300 | // MustWithLocationName is identical to WithLocation, except that it performs a parsing step. 301 | // It panics if n is not a valid *time.Location name. 302 | func MustWithLocationName(n string) ScheduleOpt { 303 | l, err := time.LoadLocation(n) 304 | if err != nil { 305 | panic(err) 306 | } 307 | return WithLocation(l) 308 | } 309 | 310 | // WithFirstRun specifies the job's first scheduled execution time. 311 | // It's incompatible with the WithImmediate option. 312 | // 313 | // The timezone of t is used when computing the first execution's 314 | // instant in time, but subsequent intervals are computed within 315 | // the timezone specified by the job's location. 316 | // 317 | // If neither WithFirstRun or WithImmediate are used, 318 | // the job's initial run will occur after one interval has elapsed. 319 | func WithFirstRun(t time.Time) ScheduleOpt { 320 | return func(o *jobmanager.ScheduleOptions) { 321 | o.FirstRun = t 322 | } 323 | } 324 | 325 | // WithOneOff specifies that the job should run once and then delete itself. 326 | // It's incompatible with the WithInterval option. 327 | func WithOneOff() ScheduleOpt { 328 | return func(o *jobmanager.ScheduleOptions) { 329 | o.OneOff = true 330 | } 331 | } 332 | 333 | // WithImmediate specifies that the job should run immediately. 334 | // It's incompatible with the WithFirstRun option. 335 | // 336 | // If neither WithFirstRun or WithImmediate are used, 337 | // the job's initial run will occur after one interval has elapsed. 338 | func WithImmediate() ScheduleOpt { 339 | return func(o *jobmanager.ScheduleOptions) { 340 | o.Immediate = true 341 | } 342 | } 343 | 344 | // Job is the external representation of a job in Dalga. 345 | type Job = table.JobJSON 346 | -------------------------------------------------------------------------------- /client_test.go: -------------------------------------------------------------------------------- 1 | package dalga // nolint: testpackage 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "net/http" 7 | "net/http/httptest" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | // TestClient performs basic functionality tests. 13 | func TestClient(t *testing.T) { 14 | c := make(chan string) 15 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 16 | var buf bytes.Buffer 17 | buf.ReadFrom(r.Body) 18 | defer r.Body.Close() 19 | 20 | c <- buf.String() 21 | _, _ = w.Write([]byte("OK")) 22 | })) 23 | defer srv.Close() 24 | 25 | config := DefaultConfig 26 | config.Endpoint.BaseURL = "http://" + srv.Listener.Addr().String() + "/" 27 | config.MySQL.SkipLocked = false 28 | config.Listen.Port = 34007 29 | config.MySQL.Table = "test_client" 30 | d, lis, cleanup := newDalga(t, config) 31 | defer cleanup() 32 | 33 | ctx, cancel := context.WithCancel(context.Background()) 34 | go d.Run(ctx) 35 | defer func() { 36 | cancel() 37 | <-d.NotifyDone() 38 | }() 39 | 40 | callCtx := context.Background() 41 | 42 | clnt := NewClient("http://" + lis.Addr()) 43 | 44 | t.Run("get nonexistent", func(t *testing.T) { 45 | _, err := clnt.Get(callCtx, "what", "who") 46 | if err != ErrNotExist { 47 | t.Fatal("expected ErrNotExist") 48 | } 49 | }) 50 | 51 | t.Run("schedule", func(t *testing.T) { 52 | if j, err := clnt.Schedule(callCtx, "when", "where", MustWithIntervalString("PT1M")); err != nil { 53 | t.Fatal(err) 54 | } else if j.Body != "where" { 55 | t.Fatalf("unexpected body: %s", j.Body) 56 | } 57 | }) 58 | 59 | t.Run("get", func(t *testing.T) { 60 | if j, err := clnt.Get(callCtx, "when", "where"); err != nil { 61 | t.Fatal(err) 62 | } else if j.Body != "where" { 63 | t.Fatalf("unexpected body: %s", j.Body) 64 | } 65 | }) 66 | 67 | t.Run("can't disable nonexistent", func(t *testing.T) { 68 | if _, err := clnt.Disable(callCtx, "apple", "banana"); err != ErrNotExist { 69 | t.Fatalf("unexpected error: %v", err) 70 | } 71 | }) 72 | 73 | t.Run("disable", func(t *testing.T) { 74 | if j, err := clnt.Disable(callCtx, "when", "where"); err != nil { 75 | t.Fatal(err) 76 | } else if j.NextRun != nil { 77 | t.Fatalf("unexpected next_run: %v", j.NextRun) 78 | } 79 | }) 80 | 81 | t.Run("enable", func(t *testing.T) { 82 | if j, err := clnt.Enable(callCtx, "when", "where"); err != nil { 83 | t.Fatal(err) 84 | } else if j.NextRun == nil { 85 | t.Fatalf("unexpected next_run: %v", j.NextRun) 86 | } 87 | }) 88 | 89 | t.Run("cancel", func(t *testing.T) { 90 | if err := clnt.Cancel(callCtx, "when", "where"); err != nil { 91 | t.Fatal(err) 92 | } 93 | if _, err := clnt.Get(callCtx, "when", "where"); err != ErrNotExist { 94 | t.Fatalf("unexpected error: %v", err) 95 | } 96 | }) 97 | 98 | t.Run("idempotent cancel", func(t *testing.T) { 99 | if err := clnt.Cancel(callCtx, "when", "where"); err != nil { 100 | t.Fatal(err) 101 | } 102 | }) 103 | } 104 | 105 | func printJob(t *testing.T, j *Job) { 106 | t.Helper() 107 | var nextRun string 108 | if j.NextRun != nil { 109 | nextRun = *j.NextRun 110 | } 111 | t.Logf("Job: interval=%s, next_run=%s, next_sched=%s", j.Interval, nextRun, j.NextSched) 112 | } 113 | 114 | // TestEnableScheduling ensures that re-enabled jobs schedule their next run correctly. 115 | func TestEnableScheduling(t *testing.T) { 116 | loc, err := time.LoadLocation("America/Los_Angeles") 117 | if err != nil { 118 | t.Fatal(err) 119 | } 120 | start := time.Date(2020, 8, 29, 15, 47, 0, 0, loc) 121 | 122 | tests := []struct { 123 | name string 124 | fixed bool 125 | retry int 126 | start time.Time 127 | firstRun time.Time 128 | interval string 129 | success bool 130 | disableAt time.Time 131 | enableAt time.Time 132 | expectRunAt time.Time 133 | notes string 134 | }{ 135 | { 136 | name: "brief-pause", 137 | fixed: false, 138 | retry: 60, 139 | start: start, 140 | firstRun: start.Add(time.Hour), 141 | interval: "PT5H", 142 | success: true, 143 | disableAt: start.Add(time.Hour * 2), 144 | enableAt: start.Add(time.Hour * 3), 145 | expectRunAt: start.Add(time.Hour * 6), 146 | notes: "should have no effect", 147 | }, 148 | { 149 | name: "brief-pause-fixed", 150 | fixed: true, 151 | retry: 60, 152 | start: start, 153 | firstRun: start.Add(time.Hour), 154 | interval: "PT5H", 155 | success: true, 156 | disableAt: start.Add(time.Hour * 2), 157 | enableAt: start.Add(time.Hour * 3), 158 | expectRunAt: start.Add(time.Hour * 6), 159 | notes: "should have no effect", 160 | }, 161 | { 162 | name: "brief-pause-during-retry", 163 | fixed: false, 164 | retry: 60, 165 | start: start, 166 | firstRun: start.Add(time.Hour), 167 | interval: "PT5H", 168 | success: false, 169 | disableAt: start.Add(time.Hour + time.Second*30), 170 | enableAt: start.Add(time.Hour + time.Second*45), 171 | expectRunAt: start.Add(time.Hour + time.Minute), 172 | notes: "should have no effect", 173 | }, 174 | { 175 | name: "brief-pause-during-retry-fixed", 176 | fixed: true, 177 | retry: 60, 178 | start: start, 179 | firstRun: start.Add(time.Hour), 180 | interval: "PT5H", 181 | success: false, 182 | disableAt: start.Add(time.Hour + time.Second*30), 183 | enableAt: start.Add(time.Hour + time.Second*45), 184 | expectRunAt: start.Add(time.Hour * 6), 185 | notes: "should cancel retries and reschedule", 186 | }, 187 | { 188 | name: "pass-over-schedule-point", 189 | fixed: false, 190 | retry: 60, 191 | start: start, 192 | firstRun: start.Add(time.Hour), 193 | interval: "PT5H", 194 | success: true, 195 | disableAt: start.Add(time.Hour * 2), 196 | enableAt: start.Add(time.Hour * 7), 197 | expectRunAt: start.Add(time.Hour * 6), 198 | notes: "should have run point in the past", 199 | }, 200 | { 201 | name: "pass-over-schedule-point-fixed", 202 | fixed: true, 203 | retry: 60, 204 | start: start, 205 | firstRun: start.Add(time.Hour), 206 | interval: "PT5H", 207 | success: true, 208 | disableAt: start.Add(time.Hour * 2), 209 | enableAt: start.Add(time.Hour * 7), 210 | expectRunAt: start.Add(time.Hour * 11), 211 | notes: "should reschedule for the next future point", 212 | }, 213 | } 214 | 215 | config := DefaultConfig 216 | config.MySQL.SkipLocked = false 217 | config.Listen.Port = 34009 218 | config.MySQL.Table = "test_enable_scheduling" 219 | 220 | for _, test := range tests { 221 | test := test 222 | t.Run(test.name, func(t *testing.T) { 223 | c := make(chan string) 224 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 225 | var buf bytes.Buffer 226 | buf.ReadFrom(r.Body) 227 | defer r.Body.Close() 228 | 229 | c <- buf.String() 230 | if test.success { 231 | _, _ = w.Write([]byte("OK")) 232 | } else { 233 | http.Error(w, "failed", 400) 234 | } 235 | })) 236 | 237 | config.Endpoint.BaseURL = "http://" + srv.Listener.Addr().String() + "/" 238 | config.Jobs.FixedIntervals = test.fixed 239 | config.Jobs.RetryInterval = time.Duration(test.retry) * time.Second 240 | config.Jobs.RetryMaxInterval = time.Duration(test.retry) * time.Second 241 | config.Jobs.RetryMultiplier = 1 242 | 243 | d, lis, cleanup := newDalga(t, config) 244 | defer cleanup() 245 | 246 | t.Log("setting clock to:", test.start.String()) 247 | clk := d.UseClock(test.start) 248 | 249 | runCtx, cancel := context.WithCancel(context.Background()) 250 | go d.Run(runCtx) 251 | defer func() { 252 | cancel() 253 | <-d.NotifyDone() 254 | }() 255 | 256 | ctx := context.Background() 257 | clnt := NewClient("http://" + lis.Addr()) 258 | 259 | t.Log("scheduling test job") 260 | j, err := clnt.Schedule(ctx, "abc", test.name, 261 | WithFirstRun(test.firstRun), 262 | MustWithIntervalString(test.interval), 263 | WithLocation(loc), 264 | ) 265 | if err != nil { 266 | t.Fatal(err) 267 | } 268 | printJob(t, j) 269 | 270 | t.Log("setting clock to:", test.firstRun.Add(time.Second)) 271 | clk.Set(test.firstRun.Add(time.Second)) 272 | 273 | select { 274 | case body := <-c: 275 | if body != test.name { 276 | t.Fatalf("expected '%s' but found '%s'", test.name, body) 277 | } 278 | case <-time.After(time.Second * 3): 279 | t.Fatal("never received POST for 1st job execution") 280 | } 281 | <-time.After(time.Millisecond * 100) 282 | 283 | j, err = clnt.Get(ctx, j.Path, j.Body) 284 | if err != nil { 285 | t.Fatal(err) 286 | } 287 | printJob(t, j) 288 | 289 | t.Log("setting clock to:", test.disableAt.String()) 290 | clk.Set(test.disableAt) 291 | t.Log("disabling test job") 292 | j, err = clnt.Disable(ctx, "abc", test.name) 293 | if err != nil { 294 | t.Fatal(err) 295 | } 296 | printJob(t, j) 297 | if j.NextRun != nil { 298 | t.Fatalf("unexpected next run: %s", *j.NextRun) 299 | } 300 | 301 | t.Log("setting clock to:", test.enableAt.String()) 302 | clk.Set(test.enableAt) 303 | t.Log("disabling test job") 304 | j, err = clnt.Enable(ctx, "abc", test.name) 305 | if err != nil { 306 | t.Fatal(err) 307 | } 308 | printJob(t, j) 309 | 310 | if j.NextRun == nil { 311 | t.Fatalf("unexpected j.NextRun: %v", j.NextRun) 312 | } 313 | nextRun, err := time.Parse(time.RFC3339, *j.NextRun) 314 | if err != nil { 315 | t.Fatal(err) 316 | } 317 | if nextRun.After(test.expectRunAt.Add(time.Second)) || nextRun.Before(test.expectRunAt.Add(-time.Second)) { 318 | t.Fatalf("run at '%s' too different from expected value '%s'", nextRun.Format(time.RFC3339), test.expectRunAt.Format(time.RFC3339)) 319 | } 320 | }) 321 | } 322 | } 323 | 324 | // TestDisableRunningJob ensures that if a job is disabled after 325 | // it starts executing, the rescheduling action that occurs when 326 | // execution finishes will not inadvertently re-enable the job. 327 | func TestDisableRunningJob(t *testing.T) { 328 | c1 := make(chan struct{}) 329 | c2 := make(chan struct{}) 330 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 331 | c1 <- struct{}{} 332 | <-c2 333 | _, _ = w.Write([]byte("OK")) 334 | })) 335 | 336 | config := DefaultConfig 337 | config.MySQL.SkipLocked = false 338 | config.Listen.Port = 34010 339 | config.MySQL.Table = "test_disable_running_job" 340 | config.Endpoint.BaseURL = "http://" + srv.Listener.Addr().String() + "/" 341 | 342 | d, lis, cleanup := newDalga(t, config) 343 | defer cleanup() 344 | 345 | runCtx, cancel := context.WithCancel(context.Background()) 346 | go d.Run(runCtx) 347 | defer func() { 348 | cancel() 349 | <-d.NotifyDone() 350 | }() 351 | 352 | ctx := context.Background() 353 | clnt := NewClient("http://" + lis.Addr()) 354 | 355 | _, err := clnt.Schedule(ctx, "alpha", "beta", 356 | WithFirstRun(time.Now().Add(time.Second)), 357 | MustWithIntervalString("PT1H"), 358 | ) 359 | if err != nil { 360 | t.Fatal(err) 361 | } 362 | 363 | select { 364 | case <-c1: 365 | case <-time.After(time.Second * 3): 366 | t.Fatal("never received POST") 367 | } 368 | 369 | j, err := clnt.Disable(ctx, "alpha", "beta") 370 | if err != nil { 371 | t.Fatal(err) 372 | } 373 | if j.NextRun != nil { 374 | t.Fatalf("unexpected nextRun: %v", *j.NextRun) 375 | } 376 | 377 | c2 <- struct{}{} 378 | <-time.After(time.Millisecond * 100) 379 | 380 | j, err = clnt.Get(ctx, "alpha", "beta") 381 | if err != nil { 382 | t.Fatal(err) 383 | } 384 | if j.NextRun != nil { 385 | t.Fatalf("unexpected nextRun: %v", *j.NextRun) 386 | } 387 | } 388 | -------------------------------------------------------------------------------- /cluster_test.go: -------------------------------------------------------------------------------- 1 | package dalga // nolint: testpackage 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "net/http" 8 | "net/http/httptest" 9 | "strconv" 10 | "testing" 11 | "time" 12 | ) 13 | 14 | type callResult struct { 15 | body string 16 | instance string 17 | } 18 | 19 | func TestCluster(t *testing.T) { 20 | const numInstances = 10 21 | const jobCount = 100 22 | 23 | called := make(chan callResult) 24 | endpoint := func(w http.ResponseWriter, r *http.Request) { 25 | var buf bytes.Buffer 26 | buf.ReadFrom(r.Body) 27 | r.Body.Close() 28 | called <- callResult{ 29 | body: buf.String(), 30 | instance: r.Header.Get("dalga-instance"), 31 | } 32 | } 33 | 34 | mux := http.NewServeMux() 35 | mux.HandleFunc("/", endpoint) 36 | srv := httptest.NewServer(mux) 37 | defer srv.Close() 38 | 39 | instances := make([]*Dalga, numInstances) 40 | for i := 0; i < numInstances; i++ { 41 | config := DefaultConfig 42 | config.MySQL.SkipLocked = false 43 | config.MySQL.Table = "test_cluster" 44 | config.Jobs.FixedIntervals = true 45 | config.Jobs.ScanFrequency = 100 * time.Millisecond 46 | config.Endpoint.BaseURL = "http://" + srv.Listener.Addr().String() + "/" 47 | config.Listen.Port = 34100 + i 48 | 49 | d, _, cleanup := newDalga(t, config) 50 | instances[i] = d 51 | defer cleanup() 52 | } 53 | client := NewClient("http://" + instances[0].listener.Addr().String()) 54 | 55 | ctx, cancel := context.WithCancel(context.Background()) 56 | for _, inst := range instances { 57 | go inst.Run(ctx) 58 | } 59 | defer func() { 60 | cancel() 61 | for _, inst := range instances { 62 | <-inst.NotifyDone() 63 | } 64 | }() 65 | 66 | // This is a fairly crude test which attempts to ensure that, 67 | // with several instances running at once, two instances will not 68 | // grab the same job and execute it. It's hardly a Jepsen test, 69 | // more of a basic sanity check. 70 | t.Run("run at most once", func(t *testing.T) { 71 | start := time.Now() 72 | received := make(map[string]bool, jobCount) 73 | for i := 0; i < jobCount; i++ { 74 | key := fmt.Sprintf("job%d", i) 75 | _, err := client.Schedule(ctx, "apple", key, WithFirstRun(start), WithOneOff()) 76 | if err != nil { 77 | t.Fatal(err) 78 | } 79 | received[key] = false 80 | } 81 | 82 | for i := 0; i < jobCount; i++ { 83 | rcv := <-called 84 | if ok := received[rcv.body]; ok { 85 | t.Errorf("Received job %s twice!", rcv) 86 | } 87 | received[rcv.body] = true 88 | } 89 | 90 | for key, ok := range received { 91 | if !ok { 92 | t.Errorf("Did not receive job %s", key) 93 | } 94 | } 95 | }) 96 | 97 | t.Run("distributed amongst instances", func(t *testing.T) { 98 | start := time.Now() 99 | countsByInstance := map[string]int{} 100 | for i := 0; i < jobCount; i++ { 101 | key := fmt.Sprintf("job%d", i) 102 | _, err := client.Schedule(ctx, "banana", key, WithFirstRun(start), WithOneOff()) 103 | if err != nil { 104 | t.Fatal(err) 105 | } 106 | } 107 | 108 | for i := 0; i < jobCount; i++ { 109 | rcv := <-called 110 | countForInstance := countsByInstance[rcv.instance] 111 | countsByInstance[rcv.instance] = countForInstance + 1 112 | } 113 | 114 | t.Logf("Counts by instance: %+v", countsByInstance) 115 | 116 | if len(countsByInstance) != len(instances) { 117 | t.Fatalf("Expected each instance to have done some jobs.") 118 | } 119 | }) 120 | } 121 | 122 | func TestClusterSuccession(t *testing.T) { 123 | const numInstances = 3 124 | const testTimeout = 3 * time.Second 125 | instances := make([]*Dalga, numInstances) 126 | ctxes := make([]context.Context, numInstances) 127 | cancels := make([]func(), numInstances) 128 | for i := 0; i < numInstances; i++ { 129 | ctxes[i], cancels[i] = context.WithCancel(context.Background()) 130 | } 131 | killStart, killDone := make(chan string), make(chan string) 132 | kill := func(w http.ResponseWriter, r *http.Request) { 133 | instance := r.Header.Get("dalga-instance") 134 | killStart <- instance 135 | killDone <- instance 136 | } 137 | 138 | mux := http.NewServeMux() 139 | mux.HandleFunc("/kill", kill) 140 | srv := httptest.NewServer(mux) 141 | defer srv.Close() 142 | 143 | for i := 0; i < numInstances; i++ { 144 | config := DefaultConfig 145 | config.MySQL.SkipLocked = false 146 | config.MySQL.Table = "test_cluster_succession" 147 | config.Jobs.FixedIntervals = true 148 | config.Jobs.ScanFrequency = 100 * time.Millisecond 149 | config.Endpoint.BaseURL = "http://" + srv.Listener.Addr().String() + "/" 150 | config.Listen.Port = 34200 + i 151 | config.Listen.ShutdownTimeout = 2 * time.Second 152 | config.Endpoint.Timeout = time.Second 153 | 154 | d, _, cleanup := newDalga(t, config) 155 | instances[i] = d 156 | defer cleanup() 157 | } 158 | client := NewClient("http://" + instances[0].listener.Addr().String()) 159 | 160 | for i, inst := range instances { 161 | go inst.Run(ctxes[i]) 162 | } 163 | defer func() { 164 | for i, inst := range instances { 165 | if fn := cancels[i]; fn != nil { 166 | fn() 167 | } 168 | <-inst.NotifyDone() 169 | } 170 | }() 171 | 172 | t.Run("reclaim jobs", func(t *testing.T) { 173 | start := time.Now() 174 | _, err := client.Schedule(context.Background(), "kill", "bill", WithFirstRun(start), WithOneOff()) 175 | if err != nil { 176 | t.Fatal(err) 177 | } 178 | 179 | // An instance is going to run the job. 180 | var doomedInstance string 181 | select { 182 | case doomedInstance = <-killStart: 183 | case <-time.After(testTimeout): 184 | t.Fatal("Didn't get the kill start") 185 | } 186 | 187 | // Now we kill it before the cycle of action completes. 188 | t.Logf("Killing instance %s", doomedInstance) 189 | id, err := strconv.ParseUint(doomedInstance, 10, 32) 190 | if err != nil { 191 | t.Fatal(err) 192 | } 193 | var done chan struct{} 194 | for i, inst := range instances { 195 | if inst.instance.ID() == uint32(id) { 196 | done = inst.NotifyDone() 197 | cancels[i]() 198 | cancels[i] = nil 199 | break 200 | } 201 | } 202 | if done == nil { 203 | t.Fatalf("Couldn't locate instance with id %d", id) 204 | } 205 | select { 206 | case <-done: 207 | case <-time.After(testTimeout): 208 | t.Fatal("Didn't get the done signal from the dead Dalga") 209 | } 210 | select { 211 | case <-killDone: 212 | case <-time.After(testTimeout): 213 | t.Fatal("Didn't finish the kill") 214 | } 215 | 216 | // Now, another instance will pick up that job and try again. 217 | var replacement string 218 | select { 219 | case replacement = <-killStart: 220 | case <-time.After(testTimeout): 221 | t.Fatal("Job wasn't picked up by a replacement instance") 222 | } 223 | if replacement == doomedInstance { 224 | t.Fatal("Something is very wrong") 225 | } 226 | select { 227 | case finished := <-killDone: 228 | if finished != replacement { 229 | t.Fatalf("Something else is very wrong, finished is '%s' but should be '%s'", finished, replacement) 230 | } 231 | case <-time.After(testTimeout): 232 | t.Fatal("Job was never finished") 233 | } 234 | }) 235 | } 236 | -------------------------------------------------------------------------------- /cmd/dalga/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "os/signal" 9 | "path/filepath" 10 | "strings" 11 | "syscall" 12 | 13 | "github.com/cenkalti/dalga/v3" 14 | "github.com/cenkalti/dalga/v3/internal/log" 15 | "github.com/knadh/koanf" 16 | "github.com/knadh/koanf/parsers/toml" 17 | "github.com/knadh/koanf/parsers/yaml" 18 | "github.com/knadh/koanf/providers/env" 19 | "github.com/knadh/koanf/providers/file" 20 | ) 21 | 22 | // These variables are set by goreleaser on build. 23 | var ( 24 | version = "0.0.0" 25 | commit = "" 26 | date = "" 27 | ) 28 | 29 | var ( 30 | configFlag = flag.String("config", "dalga.toml", "config file") 31 | versionFlag = flag.Bool("version", false, "print version") 32 | createTables = flag.Bool("create-tables", false, "create table for storing jobs") 33 | debug = flag.Bool("debug", false, "turn on debug messages") 34 | ) 35 | 36 | func versionString() string { 37 | if len(commit) > 7 { 38 | commit = commit[:7] 39 | } 40 | return fmt.Sprintf("%s (%s) [%s]", version, commit, date) 41 | } 42 | 43 | func main() { 44 | flag.Parse() 45 | 46 | if *versionFlag { 47 | fmt.Println(versionString()) 48 | return 49 | } 50 | 51 | if *debug { 52 | log.EnableDebug() 53 | } 54 | 55 | c, err := readConfig() 56 | if err != nil { 57 | log.Fatal(err) 58 | } 59 | 60 | d, err := dalga.New(c) 61 | if err != nil { 62 | log.Fatal(err) 63 | } 64 | defer d.Close() 65 | 66 | if *createTables { 67 | if err := d.CreateTable(); err != nil { 68 | log.Fatal(err) 69 | } 70 | fmt.Println("Table created successfully") 71 | return 72 | } 73 | 74 | signals := make(chan os.Signal) 75 | signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) 76 | 77 | ctx, cancel := context.WithCancel(context.Background()) 78 | go func() { 79 | <-signals 80 | cancel() 81 | }() 82 | 83 | d.Run(ctx) 84 | } 85 | 86 | func readConfig() (c dalga.Config, err error) { 87 | c = dalga.DefaultConfig 88 | k := koanf.New(".") 89 | var parser koanf.Parser 90 | ext := filepath.Ext(*configFlag) 91 | if ext == ".yaml" || ext == ".yml" { 92 | parser = yaml.Parser() 93 | } else { 94 | parser = toml.Parser() 95 | } 96 | err = k.Load(file.Provider(*configFlag), parser) 97 | if err != nil { 98 | return 99 | } 100 | err = k.Load(env.Provider("DALGA_", ".", func(s string) string { 101 | return strings.Replace(strings.TrimPrefix(s, "DALGA_"), "_", ".", -1) 102 | }), nil) 103 | if err != nil { 104 | return 105 | } 106 | err = k.Unmarshal("", &c) 107 | return 108 | } 109 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package dalga 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "time" 7 | ) 8 | 9 | // DefaultConfig contains sensible defaults for Dalga instance. 10 | // For a simple deployment, you only need to override MySQL options. 11 | var DefaultConfig = Config{ 12 | Jobs: jobsConfig{ 13 | RetryInterval: time.Minute, 14 | RetryMultiplier: 1, 15 | RetryMaxInterval: time.Minute, 16 | ScanFrequency: time.Second, 17 | MaxRunning: 100, 18 | }, 19 | MySQL: mysqlConfig{ 20 | Host: "127.0.0.1", 21 | Port: 3306, 22 | DB: "test", 23 | Table: "dalga", 24 | User: "root", 25 | Password: "", 26 | MaxOpenConns: 50, 27 | SkipLocked: true, 28 | TransactionIsolationParamName: "transaction_isolation", 29 | DialTimeout: 30 * time.Second, 30 | ReadTimeout: 30 * time.Second, 31 | WriteTimeout: 30 * time.Second, 32 | }, 33 | Listen: listenConfig{ 34 | Host: "127.0.0.1", 35 | Port: 34006, 36 | ShutdownTimeout: 10 * time.Second, 37 | IdleTimeout: 60 * time.Second, 38 | ReadTimeout: 10 * time.Second, 39 | WriteTimeout: 10 * time.Second, 40 | }, 41 | Endpoint: endpointConfig{ 42 | BaseURL: "http://127.0.0.1:5000/", 43 | Timeout: 10 * time.Second, 44 | }, 45 | } 46 | 47 | // Config values for Dalga instance. 48 | type Config struct { 49 | Jobs jobsConfig 50 | MySQL mysqlConfig 51 | Listen listenConfig 52 | Endpoint endpointConfig 53 | } 54 | 55 | type jobsConfig struct { 56 | RandomizationFactor float64 57 | RetryInterval time.Duration 58 | RetryMultiplier float64 59 | RetryMaxInterval time.Duration 60 | RetryStopAfter time.Duration 61 | FixedIntervals bool 62 | ScanFrequency time.Duration 63 | MaxRunning int 64 | } 65 | 66 | type mysqlConfig struct { 67 | Host string 68 | Port int 69 | DB string 70 | Table string 71 | User string 72 | Password string 73 | MaxOpenConns int 74 | SkipLocked bool 75 | TransactionIsolationParamName string 76 | DialTimeout time.Duration 77 | ReadTimeout time.Duration 78 | WriteTimeout time.Duration 79 | } 80 | 81 | func (c mysqlConfig) DSN() string { 82 | v := url.Values{} 83 | v.Set("parseTime", "true") 84 | v.Set(c.TransactionIsolationParamName, "'READ-COMMITTED'") 85 | v.Set("timeout", c.DialTimeout.String()) 86 | v.Set("readTimeout", c.ReadTimeout.String()) 87 | v.Set("writeTimeout", c.WriteTimeout.String()) 88 | return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?", c.User, c.Password, c.Host, c.Port, c.DB) + v.Encode() 89 | } 90 | 91 | type listenConfig struct { 92 | Host string 93 | Port int 94 | ShutdownTimeout time.Duration 95 | IdleTimeout time.Duration 96 | ReadTimeout time.Duration 97 | WriteTimeout time.Duration 98 | } 99 | 100 | func (c listenConfig) Addr() string { 101 | return fmt.Sprintf("%s:%d", c.Host, c.Port) 102 | } 103 | 104 | type endpointConfig struct { 105 | BaseURL string 106 | Timeout time.Duration 107 | } 108 | -------------------------------------------------------------------------------- /config.toml: -------------------------------------------------------------------------------- 1 | # Example config file 2 | 3 | [jobs] 4 | # Next run time of periodic jobs are randomized on each run by this factor. 5 | # Must be a number between 0 and 1. 6 | randomizationFactor = 0.0 7 | 8 | # Failed jobs are retried after given duration in seconds. 9 | retryInterval = "60s" 10 | 11 | [mysql] 12 | host = "127.0.0.1" 13 | port = 3306 14 | db = "test" 15 | table = "dalga" 16 | user = "root" 17 | password = "" 18 | 19 | [listen] 20 | host = "127.0.0.1" 21 | port = 34006 22 | 23 | [endpoint] 24 | baseurl = "http://127.0.0.1:5000/" 25 | timeout = "10s" 26 | -------------------------------------------------------------------------------- /dalga.go: -------------------------------------------------------------------------------- 1 | package dalga 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "errors" 7 | "log" 8 | "net" 9 | "time" 10 | 11 | "github.com/cenkalti/dalga/v3/internal/clock" 12 | "github.com/cenkalti/dalga/v3/internal/instance" 13 | "github.com/cenkalti/dalga/v3/internal/jobmanager" 14 | "github.com/cenkalti/dalga/v3/internal/retry" 15 | "github.com/cenkalti/dalga/v3/internal/scheduler" 16 | "github.com/cenkalti/dalga/v3/internal/server" 17 | "github.com/cenkalti/dalga/v3/internal/table" 18 | ) 19 | 20 | // Dalga is a job scheduler. 21 | type Dalga struct { 22 | config Config 23 | db *sql.DB 24 | listener net.Listener 25 | table *table.Table 26 | instance *instance.Instance 27 | Jobs *jobmanager.JobManager 28 | scheduler *scheduler.Scheduler 29 | server *server.Server 30 | done chan struct{} 31 | } 32 | 33 | // New returns a new Dalga instance. Close must be called when disposing the object. 34 | func New(config Config) (*Dalga, error) { 35 | if config.Jobs.RandomizationFactor < 0 || config.Jobs.RandomizationFactor > 1 { 36 | return nil, errors.New("randomization factor must be between 0 and 1") 37 | } 38 | 39 | db, err := sql.Open("mysql", config.MySQL.DSN()) 40 | if err != nil { 41 | return nil, err 42 | } 43 | db.SetMaxOpenConns(config.MySQL.MaxOpenConns) 44 | 45 | lis, err := net.Listen("tcp", config.Listen.Addr()) 46 | if err != nil { 47 | db.Close() 48 | return nil, err 49 | } 50 | log.Println("listening", lis.Addr()) 51 | 52 | t := table.New(db, config.MySQL.Table) 53 | t.SkipLocked = config.MySQL.SkipLocked 54 | t.FixedIntervals = config.Jobs.FixedIntervals 55 | i := instance.New(t) 56 | r := &retry.Retry{ 57 | Interval: config.Jobs.RetryInterval, 58 | MaxInterval: config.Jobs.RetryMaxInterval, 59 | Multiplier: config.Jobs.RetryMultiplier, 60 | StopAfter: config.Jobs.RetryStopAfter, 61 | } 62 | s := scheduler.New(t, i.ID(), config.Endpoint.BaseURL, config.Endpoint.Timeout, r, config.Jobs.RandomizationFactor, config.Jobs.ScanFrequency, config.Jobs.MaxRunning) 63 | j := jobmanager.New(t, s) 64 | srv := server.New(j, t, i.ID(), lis, config.Listen.ShutdownTimeout, config.Listen.IdleTimeout, config.Listen.ReadTimeout, config.Listen.WriteTimeout) 65 | return &Dalga{ 66 | config: config, 67 | db: db, 68 | listener: lis, 69 | table: t, 70 | instance: i, 71 | scheduler: s, 72 | Jobs: j, 73 | server: srv, 74 | done: make(chan struct{}), 75 | }, nil 76 | } 77 | 78 | // Close database connections and HTTP listener. 79 | func (d *Dalga) Close() { 80 | d.listener.Close() 81 | d.db.Close() 82 | } 83 | 84 | // NotifyDone returns a channel that will be closed when Run method returns. 85 | func (d *Dalga) NotifyDone() chan struct{} { 86 | return d.done 87 | } 88 | 89 | // Run Dalga. This function is blocking. 90 | func (d *Dalga) Run(ctx context.Context) { 91 | defer close(d.done) 92 | 93 | go d.server.Run(ctx) 94 | go d.instance.Run(ctx) 95 | go d.scheduler.Run(ctx) 96 | 97 | <-ctx.Done() 98 | 99 | <-d.server.NotifyDone() 100 | <-d.instance.NotifyDone() 101 | <-d.scheduler.NotifyDone() 102 | } 103 | 104 | // CreateTable creates the table for storing jobs on database. 105 | func (d *Dalga) CreateTable() error { 106 | return d.table.Create(context.Background()) 107 | } 108 | 109 | // UseClock overrides Dalga's datetime to help test schedules, retry behavior, etc. 110 | // Use the returned "clock" to manually advance time and trigger jobs as desired. 111 | func (d *Dalga) UseClock(now time.Time) *clock.Clock { 112 | d.table.Clk = clock.New(now) 113 | return d.table.Clk 114 | } 115 | -------------------------------------------------------------------------------- /dalga_test.go: -------------------------------------------------------------------------------- 1 | package dalga // nolint: testpackage 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "database/sql" 7 | "net/http" 8 | "net/http/httptest" 9 | "net/url" 10 | "strings" 11 | "testing" 12 | "time" 13 | 14 | "github.com/cenkalti/dalga/v3/internal/log" 15 | ) 16 | 17 | func init() { 18 | log.EnableDebug() 19 | } 20 | 21 | const ( 22 | testBody = "testBody" 23 | testTimeout = 5 * time.Second 24 | ) 25 | 26 | func TestSchedule(t *testing.T) { 27 | called := make(chan string) 28 | endpoint := func(w http.ResponseWriter, r *http.Request) { 29 | var buf bytes.Buffer 30 | buf.ReadFrom(r.Body) 31 | r.Body.Close() 32 | called <- buf.String() 33 | } 34 | 35 | mux := http.NewServeMux() 36 | mux.HandleFunc("/", endpoint) 37 | srv := httptest.NewServer(mux) 38 | defer srv.Close() 39 | 40 | config := DefaultConfig 41 | config.MySQL.SkipLocked = false 42 | config.Endpoint.BaseURL = "http://" + srv.Listener.Addr().String() + "/" 43 | 44 | d, lis, cleanup := newDalga(t, config) 45 | defer cleanup() 46 | 47 | ctx, cancel := context.WithCancel(context.Background()) 48 | go d.Run(ctx) 49 | defer func() { 50 | cancel() 51 | <-d.NotifyDone() 52 | }() 53 | 54 | values := make(url.Values) 55 | values.Set("one-off", "true") 56 | values.Set("first-run", "1990-01-01T00:00:00Z") 57 | 58 | scheduleURL := "http://" + lis.Addr() + "/jobs/testPath/" + testBody 59 | req, err := http.NewRequest("PUT", scheduleURL, strings.NewReader(values.Encode())) 60 | if err != nil { 61 | t.Fatal(err) 62 | } 63 | req.Header.Set("Content-Type", "application/x-www-form-urlencoded") 64 | 65 | var client http.Client 66 | resp, err := client.Do(req) 67 | if err != nil { 68 | t.Fatalf("cannot schedule new job: %s", err.Error()) 69 | } 70 | defer resp.Body.Close() 71 | var buf bytes.Buffer 72 | buf.ReadFrom(resp.Body) 73 | if resp.StatusCode != 201 { 74 | t.Fatalf("unexpected status code: %d, body: %q", resp.StatusCode, buf.String()) 75 | } 76 | t.Log("PUT response:", buf.String()) 77 | 78 | t.Log("scheduled job") 79 | 80 | select { 81 | case body := <-called: 82 | t.Log("endpoint is called") 83 | if body != testBody { 84 | t.Fatalf("Invalid body: %s", body) 85 | } 86 | case <-time.After(testTimeout): 87 | t.Fatal("timeout") 88 | } 89 | time.Sleep(time.Second) 90 | } 91 | 92 | func newDalga(t *testing.T, config Config) (*Dalga, listenConfig, func()) { 93 | db, err := sql.Open("mysql", config.MySQL.DSN()) 94 | if err != nil { 95 | t.Fatal(err.Error()) 96 | } 97 | defer db.Close() 98 | 99 | err = db.Ping() 100 | if err != nil { 101 | t.Fatalf("cannot connect to mysql: %s", err.Error()) 102 | } 103 | t.Log("connected to db") 104 | 105 | d, err := New(config) 106 | if err != nil { 107 | t.Fatal(err) 108 | } 109 | 110 | err = d.table.Drop(context.Background()) 111 | if err != nil { 112 | t.Fatal(err) 113 | } 114 | t.Log("dropped table") 115 | 116 | err = d.CreateTable() 117 | if err != nil { 118 | t.Fatal(err) 119 | } 120 | t.Log("created table") 121 | 122 | return d, config.Listen, func() { 123 | d.Close() 124 | _ = d.table.Drop(context.Background()) 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /example_endpoint.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import flask 4 | 5 | app = flask.Flask(__name__) 6 | 7 | 8 | @app.route("/", methods=['POST']) 9 | def handle_job(path): 10 | print("job received at path:", path, "body:", flask.request.data.decode()) 11 | time.sleep(5) 12 | return "" 13 | 14 | 15 | app.run(use_reloader=True) 16 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/cenkalti/dalga/v3 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 7 | github.com/go-mysql/errors v0.0.0-20180603193453-03314bea68e0 8 | github.com/go-sql-driver/mysql v1.5.0 9 | github.com/knadh/koanf v0.12.1 10 | github.com/senseyeio/duration v0.0.0-20180430131211-7c2a214ada46 11 | ) 12 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 h1:y4B3+GPxKlrigF1ha5FFErxK+sr6sWxQovRMzwMhejo= 4 | github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= 9 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= 10 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 11 | github.com/go-mysql/errors v0.0.0-20180603193453-03314bea68e0 h1:meiLwrW6ukHHehydhoDxVHdQKQe7TFgEpH0A0hHBAWs= 12 | github.com/go-mysql/errors v0.0.0-20180603193453-03314bea68e0/go.mod h1:ZH8V0509n2OSZLMYTMHzcy4hqUB+rG8ghK1zsP4i5gE= 13 | github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= 14 | github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 15 | github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= 16 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 17 | github.com/knadh/koanf v0.12.1 h1:9N0asqPUvZ0jL9thTbMZ0FAT3VrGfR+aYm5JTInT+Gs= 18 | github.com/knadh/koanf v0.12.1/go.mod h1:31bzRSM7vS5Vm9LNLo7B2Re1zhLOZT6EQKeodixBikE= 19 | github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4= 20 | github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= 21 | github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= 22 | github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= 23 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 24 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 25 | github.com/rhnvrm/simples3 v0.5.0/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= 26 | github.com/senseyeio/duration v0.0.0-20180430131211-7c2a214ada46 h1:Dz0HrI1AtNSGCE8LXLLqoZU4iuOJXPWndenCsZfstA8= 27 | github.com/senseyeio/duration v0.0.0-20180430131211-7c2a214ada46/go.mod h1:is8FVkzSi7PYLWEXT5MgWhglFsyyiW8ffxAoJqfuFZo= 28 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 29 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 30 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 31 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 32 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 33 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 34 | golang.org/x/sys v0.0.0-20200331124033-c3d80250170d h1:nc5K6ox/4lTFbMVSL9WRR81ixkcwXThoiF6yf+R9scA= 35 | golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 36 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 37 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 38 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 39 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 40 | -------------------------------------------------------------------------------- /internal/clock/clock.go: -------------------------------------------------------------------------------- 1 | package clock 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | type Clock struct { 9 | t time.Time 10 | l sync.RWMutex 11 | } 12 | 13 | func New(t time.Time) *Clock { 14 | return &Clock{t: t} 15 | } 16 | 17 | func (clk *Clock) Set(t time.Time) { 18 | clk.l.Lock() 19 | clk.t = t 20 | clk.l.Unlock() 21 | } 22 | 23 | func (clk *Clock) Get() time.Time { 24 | clk.l.RLock() 25 | defer clk.l.RUnlock() 26 | return clk.t 27 | } 28 | 29 | func (clk *Clock) Add(d time.Duration) { 30 | clk.l.Lock() 31 | clk.t = clk.t.Add(d) 32 | clk.l.Unlock() 33 | } 34 | 35 | func (clk *Clock) NowUTC() *time.Time { 36 | if clk == nil { 37 | return nil 38 | } 39 | t := clk.Get().UTC() 40 | return &t 41 | } 42 | -------------------------------------------------------------------------------- /internal/instance/instance.go: -------------------------------------------------------------------------------- 1 | package instance 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "math/rand" 7 | "time" 8 | 9 | "github.com/cenkalti/dalga/v3/internal/table" 10 | ) 11 | 12 | type Instance struct { 13 | id uint32 14 | table *table.Table 15 | ready chan struct{} 16 | done chan struct{} 17 | } 18 | 19 | func New(t *table.Table) *Instance { 20 | s := rand.NewSource(time.Now().UnixNano()) 21 | r := rand.New(s) // nolint: gosec 22 | id := r.Uint32() 23 | return &Instance{ 24 | id: id, 25 | table: t, 26 | ready: make(chan struct{}), 27 | done: make(chan struct{}), 28 | } 29 | } 30 | 31 | func (i *Instance) ID() uint32 { 32 | return i.id 33 | } 34 | 35 | func (i *Instance) NotifyDone() chan struct{} { 36 | return i.done 37 | } 38 | 39 | func (i *Instance) NotifyReady() chan struct{} { 40 | return i.ready 41 | } 42 | 43 | func (i *Instance) Run(ctx context.Context) { 44 | defer close(i.done) 45 | i.updateInstance(ctx) 46 | close(i.ready) 47 | for { 48 | select { 49 | case <-time.After(time.Second): 50 | i.updateInstance(ctx) 51 | case <-ctx.Done(): 52 | shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 53 | defer cancel() 54 | err := i.table.DeleteInstance(shutdownCtx, i.id) 55 | if err != nil { 56 | log.Print("cannot delete instance from db:", err) 57 | } 58 | return 59 | } 60 | } 61 | } 62 | 63 | func (i *Instance) updateInstance(ctx context.Context) { 64 | err := i.table.UpdateInstance(ctx, i.id) 65 | if err != nil { 66 | log.Print("cannot update instance at db: ", err) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /internal/jobmanager/jobmanager.go: -------------------------------------------------------------------------------- 1 | package jobmanager 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "time" 7 | 8 | "github.com/cenkalti/dalga/v3/internal/log" 9 | "github.com/cenkalti/dalga/v3/internal/scheduler" 10 | "github.com/cenkalti/dalga/v3/internal/table" 11 | "github.com/senseyeio/duration" 12 | ) 13 | 14 | type JobManager struct { 15 | table *table.Table 16 | scheduler *scheduler.Scheduler 17 | } 18 | 19 | type ScheduleOptions struct { 20 | OneOff bool 21 | Immediate bool 22 | FirstRun time.Time 23 | Location *time.Location 24 | Interval duration.Duration 25 | } 26 | 27 | var ErrInvalidArgs = errors.New("invalid arguments") 28 | 29 | func New(t *table.Table, s *scheduler.Scheduler) *JobManager { 30 | return &JobManager{ 31 | table: t, 32 | scheduler: s, 33 | } 34 | } 35 | 36 | func (m *JobManager) Get(ctx context.Context, path, body string) (*table.Job, error) { 37 | return m.table.Get(ctx, path, body) 38 | } 39 | 40 | // Schedule inserts a new job to the table or replaces existing one. 41 | // Returns the created or replaced job. 42 | func (m *JobManager) Schedule(ctx context.Context, path, body string, opt ScheduleOptions) (*table.Job, error) { 43 | key := table.Key{ 44 | Path: path, 45 | Body: body, 46 | } 47 | 48 | var interval duration.Duration 49 | var delay duration.Duration 50 | var nextRun time.Time 51 | switch { 52 | case opt.OneOff && opt.Immediate: // one-off and immediate 53 | // both first-run and interval params must be zero 54 | if !opt.FirstRun.IsZero() || !opt.Interval.IsZero() { 55 | return nil, ErrInvalidArgs 56 | } 57 | case opt.OneOff && !opt.Immediate: // one-off but later 58 | // only one of from first-run and interval params must be set 59 | if (!opt.FirstRun.IsZero() && !opt.Interval.IsZero()) || (opt.FirstRun.IsZero() && opt.Interval.IsZero()) { 60 | return nil, ErrInvalidArgs 61 | } 62 | if !opt.Interval.IsZero() { 63 | delay = opt.Interval 64 | } else { 65 | nextRun = opt.FirstRun 66 | } 67 | case !opt.OneOff && opt.Immediate: // periodic and immediate 68 | if opt.Interval.IsZero() || !opt.FirstRun.IsZero() { 69 | return nil, ErrInvalidArgs 70 | } 71 | interval = opt.Interval 72 | default: // periodic 73 | if opt.Interval.IsZero() { 74 | return nil, ErrInvalidArgs 75 | } 76 | interval = opt.Interval 77 | if !opt.FirstRun.IsZero() { 78 | nextRun = opt.FirstRun 79 | } else { 80 | delay = opt.Interval 81 | } 82 | } 83 | log.Debugln("job is scheduled:", key.Path, key.Body) 84 | return m.table.AddJob(ctx, key, interval, delay, opt.Location, nextRun) 85 | } 86 | 87 | // Disable prevents a job from from running until re-enabled. 88 | // Disabling an I/P job does not cancel its current run. 89 | func (m *JobManager) Disable(ctx context.Context, path, body string) (*table.Job, error) { 90 | log.Debugln("job is disabled:", path, body) 91 | return m.table.DisableJob(ctx, table.Key{Path: path, Body: body}) 92 | } 93 | 94 | // Enable reschedules a disabled job so that it will run again. 95 | func (m *JobManager) Enable(ctx context.Context, path, body string) (*table.Job, error) { 96 | log.Debugln("job is enabled:", path, body) 97 | return m.table.EnableJob(ctx, table.Key{Path: path, Body: body}) 98 | } 99 | 100 | // Cancel deletes the job with path and body. 101 | func (m *JobManager) Cancel(ctx context.Context, path, body string) error { 102 | log.Debugln("job is cancelled:", path, body) 103 | return m.table.DeleteJob(ctx, table.Key{Path: path, Body: body}) 104 | } 105 | 106 | // Running returns the number of running jobs currently. 107 | func (m *JobManager) Running() int { 108 | return m.scheduler.Running() 109 | } 110 | -------------------------------------------------------------------------------- /internal/log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import stdlog "log" 4 | 5 | var ( 6 | debugging bool 7 | disabled bool 8 | ) 9 | 10 | func EnableDebug() { 11 | debugging = true 12 | } 13 | 14 | func Disable() { 15 | disabled = true 16 | } 17 | 18 | func Debugln(args ...interface{}) { 19 | if debugging && !disabled { 20 | stdlog.Println(args...) 21 | } 22 | } 23 | 24 | func Debugf(fmt string, args ...interface{}) { 25 | if debugging && !disabled { 26 | stdlog.Printf(fmt, args...) 27 | } 28 | } 29 | 30 | func Println(args ...interface{}) { 31 | if !disabled { 32 | stdlog.Println(args...) 33 | } 34 | } 35 | 36 | func Printf(fmt string, args ...interface{}) { 37 | if !disabled { 38 | stdlog.Printf(fmt, args...) 39 | } 40 | } 41 | 42 | func Fatal(msg interface{}) { 43 | if !disabled { 44 | stdlog.Fatal(msg) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /internal/retry/retry.go: -------------------------------------------------------------------------------- 1 | package retry 2 | 3 | import "time" 4 | 5 | type Retry struct { 6 | Interval time.Duration 7 | Multiplier float64 8 | MaxInterval time.Duration 9 | StopAfter time.Duration 10 | } 11 | 12 | // NextRun returns the time when the jobs must be retried again. 13 | func (r *Retry) NextRun(sched, now time.Time) time.Time { 14 | passed := now.Sub(sched) 15 | periods := float64(passed) / float64(r.Interval) 16 | delay := (r.Multiplier-1)*periods + 1 17 | interval := time.Duration(delay * float64(r.Interval)) 18 | if interval > r.MaxInterval { 19 | interval = r.MaxInterval 20 | } 21 | nextRun := now.Add(interval) 22 | if r.StopAfter > 0 && nextRun.Sub(sched) > r.StopAfter { 23 | return time.Time{} 24 | } 25 | return nextRun 26 | } 27 | -------------------------------------------------------------------------------- /internal/retry/retry_test.go: -------------------------------------------------------------------------------- 1 | package retry_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/cenkalti/dalga/v3/internal/retry" 8 | ) 9 | 10 | type Retry = retry.Retry 11 | 12 | func t0(sec float64) time.Time { 13 | const start = "2000-01-01T00:00:00Z" 14 | t, _ := time.Parse(time.RFC3339, start) 15 | return t.Add(time.Duration(sec * float64(time.Second))) 16 | } 17 | 18 | func TestNextRun(t *testing.T) { 19 | r1 := Retry{ 20 | Interval: time.Second, 21 | Multiplier: 1, 22 | MaxInterval: time.Second, 23 | } 24 | r2 := Retry{ 25 | Interval: time.Second, 26 | Multiplier: 2, 27 | MaxInterval: 10 * time.Second, 28 | } 29 | r3 := Retry{ 30 | Interval: time.Second, 31 | Multiplier: 3, 32 | MaxInterval: 100 * time.Second, 33 | } 34 | r4 := Retry{ 35 | Interval: 8 * time.Second, 36 | Multiplier: 1.5, 37 | MaxInterval: 100 * time.Second, 38 | } 39 | r5 := Retry{ 40 | Interval: 1500 * time.Millisecond, 41 | Multiplier: 1.5, 42 | MaxInterval: 100 * time.Second, 43 | } 44 | cases := []struct { 45 | Retry Retry 46 | Now float64 47 | NextRun float64 48 | }{ 49 | {r1, 0, 1}, 50 | {r1, 10, 11}, 51 | {r2, 0, 1}, 52 | {r2, 1, 3}, 53 | {r2, 3, 7}, 54 | {r2, 7, 15}, 55 | {r3, 0, 1}, 56 | {r3, 1, 4}, 57 | {r3, 4, 13}, 58 | {r3, 13, 40}, 59 | {r4, 0, 8}, 60 | {r4, 8, 20}, 61 | {r5, 0, 1.5}, 62 | {r5, 1.5, 3.75}, 63 | } 64 | for _, c := range cases { 65 | nextRun := c.Retry.NextRun(t0(0), t0(c.Now)) 66 | if !nextRun.Equal(t0(c.NextRun)) { 67 | t.Log(c.Retry.Interval, c.Retry.MaxInterval, c.Retry.Multiplier, c.Now, c.NextRun, nextRun.String()) 68 | t.FailNow() 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /internal/scheduler/benchmark_test.go: -------------------------------------------------------------------------------- 1 | package scheduler // nolint:testpackage 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "fmt" 7 | "strconv" 8 | "sync" 9 | "testing" 10 | "time" 11 | 12 | "github.com/cenkalti/dalga/v3/internal/instance" 13 | "github.com/cenkalti/dalga/v3/internal/log" 14 | "github.com/cenkalti/dalga/v3/internal/retry" 15 | "github.com/cenkalti/dalga/v3/internal/table" 16 | "github.com/senseyeio/duration" 17 | ) 18 | 19 | const tableName = "sched" 20 | 21 | var dsn = fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true&transaction_isolation=%%27READ-COMMITTED%%27", "root", "", "127.0.0.1", 3306, "test") 22 | 23 | // Run with command: go test -bench=Bench -run=x ./internal/scheduler 24 | // To see the output of a single test, add -v flag and comment out desired tests below. 25 | 26 | func BenchmarkScheduler(b *testing.B) { 27 | if testing.Verbose() { 28 | log.EnableDebug() 29 | } else { 30 | log.Disable() 31 | } 32 | 33 | const numJobs = 1000 34 | cleanup := prepareTable(b, dsn, numJobs) 35 | defer cleanup() 36 | 37 | testCases := []struct { 38 | numInstances int 39 | maxRunning int 40 | skipLocked bool 41 | }{ 42 | {1, 1, false}, 43 | {1, 0, false}, 44 | {10, 1, false}, 45 | {10, 0, false}, 46 | {1, 1, true}, 47 | {1, 0, true}, 48 | {10, 1, true}, 49 | {10, 0, true}, 50 | } 51 | 52 | for _, tc := range testCases { 53 | tc := tc 54 | b.Run(fmt.Sprintf("numInstances:%d maxRunning:%d skipLocked:%v", tc.numInstances, tc.maxRunning, tc.skipLocked), func(b *testing.B) { 55 | benchmarkScheduler(b, tc.numInstances, tc.maxRunning, tc.skipLocked) 56 | }) 57 | } 58 | } 59 | 60 | func benchmarkScheduler(b *testing.B, numInstances, maxRunning int, skipLocked bool) { 61 | var wg sync.WaitGroup 62 | 63 | // Craeate scheduler instances. 64 | schedulers := make([]*Scheduler, 0, numInstances) 65 | ctx, cancel := context.WithCancel(context.Background()) 66 | defer cancel() 67 | for i := 0; i < numInstances; i++ { 68 | s, cleanup := prepareInstance(ctx, b, dsn, skipLocked, maxRunning) 69 | defer cleanup() 70 | schedulers = append(schedulers, s) 71 | } 72 | 73 | // Run all schedulers but one in infinite loop to see how it affects performance. 74 | // Note that benchmark output shows only single instance throughput. 75 | // To find out the throughput of whole cluster, the result need to be multiplied by the number of instances. 76 | for i := 1; i < numInstances; i++ { 77 | wg.Add(1) 78 | go func(s *Scheduler) { 79 | for { 80 | if ok := s.runOnce(ctx); !ok { 81 | break 82 | } 83 | } 84 | wg.Done() 85 | }(schedulers[i]) 86 | } 87 | 88 | // Benchmark the first scheduler instance. 89 | s := schedulers[0] 90 | b.ResetTimer() 91 | for i := 0; i < b.N; i++ { 92 | s.runOnce(context.Background()) 93 | } 94 | 95 | cancel() 96 | wg.Wait() 97 | } 98 | 99 | func prepareInstance(ctx context.Context, b *testing.B, dsn string, skipLocked bool, maxRunning int) (*Scheduler, func()) { 100 | db, err := sql.Open("mysql", dsn) 101 | if err != nil { 102 | b.Fatal(err) 103 | } 104 | r := &retry.Retry{ 105 | Interval: time.Second, 106 | MaxInterval: time.Second, 107 | Multiplier: 1, 108 | } 109 | tbl := table.New(db, tableName) 110 | tbl.SkipLocked = skipLocked 111 | 112 | i := instance.New(tbl) 113 | go i.Run(ctx) 114 | <-i.NotifyReady() 115 | 116 | s := New(tbl, i.ID(), "http://example.com/", 4*time.Second, r, 0, 250*time.Millisecond, maxRunning) 117 | s.skipOneOffDelete = true 118 | s.skipPost = true 119 | 120 | return s, func() { 121 | <-i.NotifyDone() 122 | db.Close() 123 | } 124 | } 125 | 126 | func prepareTable(t *testing.B, dsn string, numJobs int) func() { 127 | db, err := sql.Open("mysql", dsn) 128 | if err != nil { 129 | t.Fatal(err) 130 | } 131 | defer db.Close() 132 | 133 | t.Log("creating tables") 134 | tbl := table.New(db, tableName) 135 | if err := tbl.Drop(context.Background()); err != nil { 136 | t.Fatal(err) 137 | } 138 | if err := tbl.Create(context.Background()); err != nil { 139 | t.Fatal(err) 140 | } 141 | 142 | // Create some number of one-off jobs. 143 | t.Logf("adding %d jobs", numJobs) 144 | nextRun := time.Now().UTC().Add(-time.Hour) // time in past, a job that is ready to run 145 | for i := 0; i < numJobs; i++ { 146 | _, err := tbl.AddJob(context.Background(), table.Key{Path: "path", Body: "body" + strconv.Itoa(i)}, duration.Duration{TS: 0}, duration.Duration{}, time.UTC, nextRun) 147 | if err != nil { 148 | t.Fatal(err) 149 | } 150 | } 151 | 152 | t.Log("job table is ready") 153 | return func() { tbl.Drop(context.Background()) } 154 | } 155 | -------------------------------------------------------------------------------- /internal/scheduler/scheduler.go: -------------------------------------------------------------------------------- 1 | package scheduler 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "fmt" 7 | "net/http" 8 | "strings" 9 | "sync" 10 | "sync/atomic" 11 | "time" 12 | 13 | "github.com/cenkalti/dalga/v3/internal/log" 14 | "github.com/cenkalti/dalga/v3/internal/retry" 15 | "github.com/cenkalti/dalga/v3/internal/table" 16 | "github.com/go-sql-driver/mysql" 17 | ) 18 | 19 | type Scheduler struct { 20 | table *table.Table 21 | instanceID uint32 22 | client http.Client 23 | baseURL string 24 | randomizationFactor float64 25 | retryParams *retry.Retry 26 | runningJobs int32 27 | scanFrequency time.Duration 28 | done chan struct{} 29 | wg sync.WaitGroup 30 | maxRunning chan struct{} 31 | 32 | // Flags to disable some functionality for running benchmarks. 33 | skipOneOffDelete bool 34 | skipPost bool 35 | } 36 | 37 | func New(t *table.Table, instanceID uint32, baseURL string, clientTimeout time.Duration, retryParams *retry.Retry, randomizationFactor float64, scanFrequency time.Duration, maxRunning int) *Scheduler { 38 | s := &Scheduler{ 39 | table: t, 40 | instanceID: instanceID, 41 | baseURL: baseURL, 42 | randomizationFactor: randomizationFactor, 43 | retryParams: retryParams, 44 | scanFrequency: scanFrequency, 45 | done: make(chan struct{}), 46 | client: http.Client{ 47 | Timeout: clientTimeout, 48 | }, 49 | } 50 | // Create a semaphore channel to limit max running jobs. 51 | if maxRunning > 0 { 52 | s.maxRunning = make(chan struct{}, maxRunning) 53 | } 54 | return s 55 | } 56 | 57 | func (s *Scheduler) NotifyDone() <-chan struct{} { 58 | return s.done 59 | } 60 | 61 | func (s *Scheduler) Running() int { 62 | return int(atomic.LoadInt32(&s.runningJobs)) 63 | } 64 | 65 | // Run runs a loop that reads the next Job from the queue and executees it in it's own goroutine. 66 | func (s *Scheduler) Run(ctx context.Context) { 67 | defer func() { 68 | s.wg.Wait() 69 | close(s.done) 70 | }() 71 | 72 | for { 73 | log.Debugln("---") 74 | ok := s.runOnce(ctx) 75 | if !ok { 76 | return 77 | } 78 | } 79 | } 80 | 81 | func (s *Scheduler) runOnce(ctx context.Context) bool { 82 | job, err := s.table.Front(ctx, s.instanceID) 83 | if err == context.Canceled { 84 | return false 85 | } 86 | if err == sql.ErrNoRows { 87 | log.Debugln("no scheduled jobs in the table") 88 | select { 89 | case <-time.After(s.scanFrequency): 90 | case <-ctx.Done(): 91 | return false 92 | } 93 | return true 94 | } 95 | if myErr, ok := err.(*mysql.MySQLError); ok && myErr.Number == 1146 { 96 | // Table doesn't exist 97 | log.Fatal(myErr) 98 | } 99 | if err != nil { 100 | log.Println("error while getting next job:", err) 101 | select { 102 | case <-time.After(s.scanFrequency): 103 | case <-ctx.Done(): 104 | return false 105 | } 106 | return true 107 | } 108 | if s.maxRunning != nil { 109 | // Wait for semaphore 110 | select { 111 | case s.maxRunning <- struct{}{}: 112 | case <-ctx.Done(): 113 | return false 114 | default: 115 | log.Printf("max running jobs (%d) has been reached", cap(s.maxRunning)) 116 | select { 117 | case s.maxRunning <- struct{}{}: 118 | case <-ctx.Done(): 119 | return false 120 | } 121 | } 122 | } 123 | s.wg.Add(1) 124 | go s.runJob(job) 125 | return true 126 | } 127 | 128 | func (s *Scheduler) runJob(job *table.Job) { 129 | atomic.AddInt32(&s.runningJobs, 1) 130 | if err := s.execute(job); err != nil { 131 | log.Printf("error on execution of %s: %s", job.String(), err) 132 | } 133 | if s.maxRunning != nil { 134 | <-s.maxRunning 135 | } 136 | atomic.AddInt32(&s.runningJobs, -1) 137 | s.wg.Done() 138 | } 139 | 140 | // execute makes a POST request to the endpoint and updates the Job's next run time. 141 | func (s *Scheduler) execute(j *table.Job) error { 142 | log.Debugln("executing:", j.String()) 143 | 144 | // Stopping the instance should not cancel running execution. 145 | // We rely on http.Client timeout and MySQL driver timeouts here. 146 | ctx := context.Background() 147 | 148 | code, err := s.postJob(ctx, j) 149 | if err != nil { 150 | log.Printf("error while doing http post for %s: %s", j.String(), err) 151 | return s.table.UpdateNextRun(ctx, j.Key, 0.0, s.retryParams) 152 | } 153 | if !s.skipOneOffDelete && j.OneOff() { 154 | log.Debugln("deleting one-off job") 155 | return s.table.DeleteJob(ctx, j.Key) 156 | } 157 | if code == 204 { 158 | log.Debugln("deleting not found job") 159 | return s.table.DeleteJob(ctx, j.Key) 160 | } 161 | return s.table.UpdateNextRun(ctx, j.Key, s.randomizationFactor, nil) 162 | } 163 | 164 | func (s *Scheduler) postJob(ctx context.Context, j *table.Job) (code int, err error) { 165 | if s.skipPost { 166 | return 200, nil 167 | } 168 | url := s.baseURL + j.Path 169 | log.Debugln("doing http post to", url) 170 | req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, strings.NewReader(j.Body)) 171 | if err != nil { 172 | return 173 | } 174 | req.Header.Set("content-type", "text/plain") 175 | req.Header.Set("dalga-sched", j.NextSched.Format(time.RFC3339)) 176 | req.Header.Set("dalga-instance", fmt.Sprintf("%d", s.instanceID)) 177 | resp, err := s.client.Do(req) 178 | if err != nil { 179 | return 180 | } 181 | defer resp.Body.Close() 182 | switch resp.StatusCode { 183 | case 200, 204: 184 | code = resp.StatusCode 185 | default: 186 | err = fmt.Errorf("endpoint error: %d", resp.StatusCode) 187 | } 188 | return 189 | } 190 | -------------------------------------------------------------------------------- /internal/scheduler/scheduler_test.go: -------------------------------------------------------------------------------- 1 | package scheduler_test 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "fmt" 7 | "net/http" 8 | "net/http/httptest" 9 | "testing" 10 | "time" 11 | 12 | "github.com/cenkalti/dalga/v3/internal/instance" 13 | "github.com/cenkalti/dalga/v3/internal/retry" 14 | "github.com/cenkalti/dalga/v3/internal/scheduler" 15 | "github.com/cenkalti/dalga/v3/internal/table" 16 | "github.com/senseyeio/duration" 17 | ) 18 | 19 | var dsn = fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true&multiStatements=true", "root", "", "127.0.0.1", 3306, "test") 20 | 21 | // TestSchedHeader verifies that when the scheduler executes a job, the 22 | // POST includes a header with the unix timestamp of the intended execution. 23 | // 24 | // Retries of a particular execution will preserve the timestamp of the 25 | // original execution, which receivers can use to ensure idempotency. 26 | func TestSchedHeader(t *testing.T) { 27 | rcv := make(chan string) 28 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 29 | rcv <- r.Header.Get("dalga-sched") 30 | http.Error(w, "job failed", 500) 31 | })) 32 | 33 | db, err := sql.Open("mysql", dsn) 34 | if err != nil { 35 | t.Fatal(err) 36 | } 37 | defer db.Close() 38 | 39 | tbl := table.New(db, "sched") 40 | if err := tbl.Drop(context.Background()); err != nil { 41 | t.Fatal(err) 42 | } 43 | if err := tbl.Create(context.Background()); err != nil { 44 | t.Fatal(err) 45 | } 46 | defer tbl.Drop(context.Background()) 47 | 48 | r := &retry.Retry{ 49 | Interval: time.Second, 50 | MaxInterval: time.Second, 51 | Multiplier: 1, 52 | } 53 | i := instance.New(tbl) 54 | s := scheduler.New(tbl, i.ID(), "http://"+srv.Listener.Addr().String()+"/", 4*time.Second, r, 0, 250*time.Millisecond, 0) 55 | 56 | ctx, cancel := context.WithCancel(context.Background()) 57 | defer func() { 58 | cancel() 59 | <-i.NotifyDone() 60 | <-s.NotifyDone() 61 | }() 62 | go i.Run(ctx) 63 | go s.Run(ctx) 64 | 65 | nextRun := time.Now().UTC().Truncate(time.Second) 66 | _, err = tbl.AddJob(context.Background(), table.Key{Path: "abc", Body: "def"}, duration.Duration{TS: 10}, duration.Duration{}, time.UTC, nextRun) 67 | if err != nil { 68 | t.Fatal(err) 69 | } 70 | 71 | expect := nextRun.Format(time.RFC3339) 72 | 73 | // First run 74 | select { 75 | case hdr := <-rcv: 76 | if hdr != expect { 77 | t.Fatalf("Expected header %s and found %s", expect, hdr) 78 | } 79 | case <-time.After(time.Second * 5): 80 | t.Fatal("Job never fired.") 81 | } 82 | 83 | // Retry must preserve original sched time 84 | select { 85 | case hdr := <-rcv: 86 | if hdr != expect { 87 | t.Fatalf("Expected header %s and found %s", expect, hdr) 88 | } 89 | case <-time.After(time.Second * 5): 90 | t.Fatal("Job is not retried.") 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /internal/server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "net" 7 | "net/http" 8 | "net/url" 9 | "strconv" 10 | "time" 11 | 12 | "github.com/bmizerany/pat" 13 | "github.com/cenkalti/dalga/v3/internal/jobmanager" 14 | "github.com/cenkalti/dalga/v3/internal/log" 15 | "github.com/cenkalti/dalga/v3/internal/table" 16 | "github.com/senseyeio/duration" 17 | ) 18 | 19 | type Server struct { 20 | shutdownTimeout time.Duration 21 | idleTimeout time.Duration 22 | readTimeout time.Duration 23 | writeTimeout time.Duration 24 | jobs *jobmanager.JobManager 25 | table *table.Table 26 | instanceID uint32 27 | listener net.Listener 28 | httpServer http.Server 29 | done chan struct{} 30 | } 31 | 32 | func New(j *jobmanager.JobManager, t *table.Table, instanceID uint32, l net.Listener, shutdownTimeout, idleTimeout, readTimeout, writeTimeout time.Duration) *Server { 33 | s := &Server{ 34 | shutdownTimeout: shutdownTimeout, 35 | idleTimeout: idleTimeout, 36 | readTimeout: readTimeout, 37 | writeTimeout: writeTimeout, 38 | jobs: j, 39 | table: t, 40 | instanceID: instanceID, 41 | listener: l, 42 | done: make(chan struct{}), 43 | } 44 | s.httpServer = s.createServer() 45 | return s 46 | } 47 | 48 | func (s *Server) NotifyDone() chan struct{} { 49 | return s.done 50 | } 51 | 52 | func (s *Server) Run(ctx context.Context) { 53 | defer close(s.done) 54 | shutdownDone := make(chan struct{}) 55 | go s.waitShutdown(ctx, shutdownDone) 56 | _ = s.httpServer.Serve(s.listener) 57 | <-shutdownDone 58 | } 59 | 60 | func (s *Server) waitShutdown(ctx context.Context, shutdownDone chan struct{}) { 61 | defer close(shutdownDone) 62 | select { 63 | case <-s.done: 64 | case <-ctx.Done(): 65 | shutdownCtx, cancel := context.WithTimeout(context.Background(), s.shutdownTimeout) 66 | defer cancel() 67 | _ = s.httpServer.Shutdown(shutdownCtx) 68 | } 69 | } 70 | 71 | func (s *Server) createServer() http.Server { 72 | const path = "/jobs/:jobPath/:jobBody" 73 | m := pat.New() 74 | m.Get(path, handler(s.handleGet)) 75 | m.Put(path, handler(s.handleSchedule)) 76 | m.Patch(path, handler(s.handlePatch)) 77 | m.Del(path, handler(s.handleCancel)) 78 | m.Get("/status", http.HandlerFunc(s.handleStatus)) 79 | return http.Server{ 80 | Handler: m, 81 | ReadTimeout: s.readTimeout, 82 | WriteTimeout: s.writeTimeout, 83 | IdleTimeout: s.idleTimeout, 84 | } 85 | } 86 | 87 | func handler(f func(w http.ResponseWriter, r *http.Request, jobPath, body string)) http.HandlerFunc { 88 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 89 | log.Debugln("http:", r.Method, r.RequestURI) 90 | var err error 91 | 92 | jobPath := r.URL.Query().Get(":jobPath") 93 | if jobPath == "" { 94 | http.Error(w, "empty routing key", http.StatusBadRequest) 95 | return 96 | } 97 | jobPath, err = url.QueryUnescape(jobPath) 98 | if err != nil { 99 | http.Error(w, err.Error(), http.StatusBadRequest) 100 | return 101 | } 102 | 103 | jobBody := r.URL.Query().Get(":jobBody") 104 | if jobBody == "" { 105 | http.Error(w, "empty job", http.StatusBadRequest) 106 | return 107 | } 108 | jobBody, err = url.QueryUnescape(jobBody) 109 | if err != nil { 110 | http.Error(w, err.Error(), http.StatusBadRequest) 111 | return 112 | } 113 | 114 | f(w, r, jobPath, jobBody) 115 | }) 116 | } 117 | 118 | func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path, body string) { 119 | job, err := s.jobs.Get(r.Context(), path, body) 120 | if err == table.ErrNotExist { 121 | http.Error(w, err.Error(), http.StatusNotFound) 122 | return 123 | } 124 | if err != nil { 125 | http.Error(w, err.Error(), http.StatusInternalServerError) 126 | return 127 | } 128 | data, err := json.Marshal(job) 129 | if err != nil { 130 | http.Error(w, err.Error(), http.StatusInternalServerError) 131 | return 132 | } 133 | w.Header().Set("Content-Type", "application/json") 134 | _, _ = w.Write(data) 135 | } 136 | 137 | func (s *Server) handleSchedule(w http.ResponseWriter, r *http.Request, path, body string) { 138 | var opt jobmanager.ScheduleOptions 139 | var err error 140 | 141 | oneOffParam := r.FormValue("one-off") 142 | if oneOffParam != "" { 143 | opt.OneOff, err = strconv.ParseBool(oneOffParam) 144 | if err != nil { 145 | http.Error(w, "cannot parse one-off", http.StatusBadRequest) 146 | return 147 | } 148 | } 149 | 150 | immediateParam := r.FormValue("immediate") 151 | if immediateParam != "" { 152 | opt.Immediate, err = strconv.ParseBool(immediateParam) 153 | if err != nil { 154 | http.Error(w, "cannot parse immediate", http.StatusBadRequest) 155 | return 156 | } 157 | } 158 | 159 | intervalParam := r.FormValue("interval") 160 | if intervalParam != "" { 161 | if intervalParam[0] == 'P' { 162 | opt.Interval, err = duration.ParseISO8601(intervalParam) 163 | } else { 164 | opt.Interval.TS, err = strconv.Atoi(intervalParam) 165 | } 166 | if err != nil { 167 | http.Error(w, "cannot parse interval", http.StatusBadRequest) 168 | return 169 | } 170 | } 171 | 172 | locationParam := r.FormValue("location") 173 | if locationParam != "" { 174 | opt.Location, err = time.LoadLocation(locationParam) 175 | if err != nil { 176 | http.Error(w, "cannot parse location", http.StatusBadRequest) 177 | return 178 | } 179 | } 180 | 181 | firstRunParam := r.FormValue("first-run") 182 | if firstRunParam != "" { 183 | opt.FirstRun, err = time.Parse(time.RFC3339, firstRunParam) 184 | if err != nil { 185 | http.Error(w, "cannot parse first-run", http.StatusBadRequest) 186 | return 187 | } 188 | } 189 | 190 | job, err := s.jobs.Schedule(r.Context(), path, body, opt) 191 | if err == jobmanager.ErrInvalidArgs { 192 | http.Error(w, "invalid params", http.StatusBadRequest) 193 | return 194 | } 195 | if err != nil { 196 | http.Error(w, err.Error(), http.StatusInternalServerError) 197 | return 198 | } 199 | 200 | data, err := json.Marshal(job) 201 | if err != nil { 202 | http.Error(w, err.Error(), http.StatusInternalServerError) 203 | return 204 | } 205 | w.Header().Set("Content-Type", "application/json") 206 | w.WriteHeader(http.StatusCreated) 207 | _, _ = w.Write(data) 208 | } 209 | 210 | func (s *Server) handlePatch(w http.ResponseWriter, r *http.Request, path, body string) { 211 | var job *table.Job 212 | var err error 213 | switch { 214 | case r.URL.Query().Get("disable") == "true": 215 | job, err = s.jobs.Disable(r.Context(), path, body) 216 | case r.URL.Query().Get("enable") == "true": 217 | job, err = s.jobs.Enable(r.Context(), path, body) 218 | default: 219 | http.Error(w, "pass enable=true or disable=true query params", http.StatusBadRequest) 220 | return 221 | } 222 | if err == table.ErrNotExist { 223 | http.Error(w, err.Error(), http.StatusNotFound) 224 | return 225 | } 226 | if err != nil { 227 | http.Error(w, err.Error(), http.StatusInternalServerError) 228 | return 229 | } 230 | data, err := json.Marshal(job) 231 | if err != nil { 232 | http.Error(w, err.Error(), http.StatusInternalServerError) 233 | return 234 | } 235 | w.Header().Set("Content-Type", "application/json") 236 | w.WriteHeader(http.StatusOK) 237 | _, _ = w.Write(data) 238 | } 239 | 240 | func (s *Server) handleCancel(w http.ResponseWriter, r *http.Request, path, body string) { 241 | err := s.jobs.Cancel(r.Context(), path, body) 242 | if err != nil { 243 | http.Error(w, err.Error(), http.StatusInternalServerError) 244 | return 245 | } 246 | w.WriteHeader(http.StatusNoContent) 247 | } 248 | 249 | func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { 250 | total, err := s.table.Count(r.Context()) 251 | if err != nil { 252 | http.Error(w, err.Error(), http.StatusInternalServerError) 253 | return 254 | } 255 | pending, err := s.table.Pending(r.Context()) 256 | if err != nil { 257 | http.Error(w, err.Error(), http.StatusInternalServerError) 258 | return 259 | } 260 | running, err := s.table.Running(r.Context()) 261 | if err != nil { 262 | http.Error(w, err.Error(), http.StatusInternalServerError) 263 | return 264 | } 265 | instances, err := s.table.Instances(r.Context()) 266 | if err != nil { 267 | http.Error(w, err.Error(), http.StatusInternalServerError) 268 | return 269 | } 270 | lag, err := s.table.Lag(r.Context()) 271 | if err != nil { 272 | http.Error(w, err.Error(), http.StatusInternalServerError) 273 | return 274 | } 275 | m := Status{ 276 | InstanceID: s.instanceID, 277 | InstanceRunningJobs: s.jobs.Running(), 278 | RunningJobs: running, 279 | TotalJobs: total, 280 | PendingJobs: pending, 281 | TotalInstances: instances, 282 | Lag: lag, 283 | } 284 | data, err := json.Marshal(m) 285 | if err != nil { 286 | http.Error(w, err.Error(), http.StatusInternalServerError) 287 | return 288 | } 289 | w.Header().Set("Content-Type", "application/json") 290 | _, _ = w.Write(data) 291 | } 292 | 293 | type Status struct { 294 | InstanceID uint32 `json:"instance_id"` 295 | InstanceRunningJobs int `json:"instance_running_jobs"` 296 | RunningJobs int64 `json:"running_jobs"` 297 | TotalJobs int64 `json:"total_jobs"` 298 | PendingJobs int64 `json:"pending_jobs"` 299 | TotalInstances int64 `json:"total_instances"` 300 | Lag int64 `json:"lag"` 301 | } 302 | -------------------------------------------------------------------------------- /internal/table/job.go: -------------------------------------------------------------------------------- 1 | package table 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/json" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/senseyeio/duration" 10 | ) 11 | 12 | // Job is the record stored in jobs table. 13 | // Primary key for the table is Key. 14 | type Job struct { 15 | Key 16 | // Interval is the duration between each POST to the endpoint. 17 | // Interval is "" for one-off jobs. 18 | Interval duration.Duration 19 | // Interval is relative to the Location. 20 | // Format is the tz database name, such as America/Los_Angeles. 21 | Location *time.Location 22 | // NextRun is the next run time of the job, including retries. 23 | NextRun sql.NullTime 24 | // NextSched is the next time the job is scheduled to run, regardless of retries. 25 | NextSched time.Time 26 | // Job is running if not nil. 27 | InstanceID *uint32 28 | } 29 | 30 | type Key struct { 31 | // Path is where the job is going to be POSTed when it's time came. 32 | Path string 33 | // Body of POST request. 34 | Body string 35 | } 36 | 37 | // String returns the job in human-readable form. 38 | func (j *Job) String() string { 39 | var id uint32 40 | if j.InstanceID != nil { 41 | id = *j.InstanceID 42 | } 43 | nextRun := "null" 44 | if j.NextRun.Valid { 45 | nextRun = j.NextRun.Time.Format(time.RFC3339) 46 | } 47 | return fmt.Sprintf("Job<%q, %q, %s, %s, %s, %s, %d>", j.Path, j.Body, j.Interval.String(), j.Location.String(), nextRun, j.NextSched.Format(time.RFC3339), id) 48 | } 49 | 50 | // OneOff returns true for one-off jobs. One-off jobs are stored with empty interval on jobs table. 51 | func (j *Job) OneOff() bool { 52 | return j.Interval.IsZero() 53 | } 54 | 55 | func (j *Job) Enabled() bool { 56 | return j.NextRun.Valid 57 | } 58 | 59 | func (j *Job) MarshalJSON() ([]byte, error) { 60 | var nextRun *string 61 | if j.NextRun.Valid { 62 | formatted := j.NextRun.Time.Format(time.RFC3339) 63 | nextRun = &formatted 64 | } 65 | return json.Marshal(JobJSON{ 66 | Path: j.Path, 67 | Body: j.Body, 68 | Interval: j.Interval.String(), 69 | Location: j.Location.String(), 70 | NextRun: nextRun, 71 | NextSched: j.NextSched.Format(time.RFC3339), 72 | InstanceID: j.InstanceID, 73 | }) 74 | } 75 | 76 | func (j *Job) setLocation(locationName string) (err error) { 77 | loc := time.UTC // Default to UTC in case it's omitted somehow in the database. 78 | if locationName != "" { 79 | loc, err = time.LoadLocation(locationName) 80 | if err != nil { 81 | return err 82 | } 83 | } 84 | j.Location = loc 85 | j.NextSched = j.NextSched.In(loc) 86 | if j.NextRun.Valid { 87 | j.NextRun.Time = j.NextRun.Time.In(loc) 88 | } 89 | return nil 90 | } 91 | 92 | type JobJSON struct { 93 | Path string `json:"path"` 94 | Body string `json:"body"` 95 | Interval string `json:"interval"` 96 | Location string `json:"location"` 97 | NextRun *string `json:"next_run"` 98 | NextSched string `json:"next_sched"` 99 | InstanceID *uint32 `json:"instance_id"` 100 | } 101 | -------------------------------------------------------------------------------- /internal/table/table.go: -------------------------------------------------------------------------------- 1 | package table 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "errors" 7 | "fmt" 8 | "math/rand" 9 | "strconv" 10 | "time" 11 | 12 | "github.com/cenkalti/dalga/v3/internal/clock" 13 | "github.com/cenkalti/dalga/v3/internal/log" 14 | "github.com/cenkalti/dalga/v3/internal/retry" 15 | my "github.com/go-mysql/errors" 16 | "github.com/go-sql-driver/mysql" 17 | "github.com/senseyeio/duration" 18 | ) 19 | 20 | const ( 21 | maxRetries = 10 22 | ) 23 | 24 | var ErrNotExist = errors.New("job does not exist") 25 | 26 | type Table struct { 27 | db *sql.DB 28 | name string 29 | SkipLocked bool 30 | FixedIntervals bool 31 | Clk *clock.Clock 32 | } 33 | 34 | func New(db *sql.DB, name string) *Table { 35 | return &Table{ 36 | db: db, 37 | name: name, 38 | } 39 | } 40 | 41 | // Create jobs table. 42 | func (t *Table) Create(ctx context.Context) error { 43 | const createTableSQL = "" + 44 | "CREATE TABLE `%s` (" + 45 | " `path` VARCHAR(255) NOT NULL," + 46 | " `body` VARCHAR(255) NOT NULL," + 47 | " `interval` VARCHAR(255) NOT NULL," + 48 | " `location` VARCHAR(255) NOT NULL," + 49 | " `next_run` DATETIME NULL," + 50 | " `next_sched` DATETIME NOT NULL," + 51 | " `instance_id` INT UNSIGNED," + 52 | " PRIMARY KEY (`path`, `body`)," + 53 | " KEY (`next_run`)," + 54 | " FOREIGN KEY (`instance_id`) REFERENCES `%s_instances` (`id`) ON DELETE SET NULL" + 55 | ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4" 56 | const createInstancesTableSQL = "" + 57 | "CREATE TABLE `%s_instances` (" + 58 | " `id` INT UNSIGNED NOT NULL," + 59 | " `updated_at` DATETIME NOT NULL," + 60 | " PRIMARY KEY (`id`)" + 61 | ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4" 62 | _, err := t.db.ExecContext(ctx, fmt.Sprintf(createInstancesTableSQL, t.name)) 63 | if err != nil { 64 | return err 65 | } 66 | _, err = t.db.ExecContext(ctx, fmt.Sprintf(createTableSQL, t.name, t.name)) 67 | return err 68 | } 69 | 70 | func (t *Table) Drop(ctx context.Context) error { 71 | dropSQL := "DROP TABLE " + t.name 72 | _, err := t.db.ExecContext(ctx, dropSQL) 73 | if err != nil { 74 | if myErr, ok := err.(*mysql.MySQLError); !ok || myErr.Number != 1051 { // Unknown table 75 | return err 76 | } 77 | } 78 | dropSQL = "DROP TABLE " + t.name + "_instances" 79 | _, err = t.db.ExecContext(ctx, dropSQL) 80 | if err != nil { 81 | if myErr, ok := err.(*mysql.MySQLError); !ok || myErr.Number != 1051 { // Unknown table 82 | return err 83 | } 84 | } 85 | return nil 86 | } 87 | 88 | // Get returns a job from the scheduler table, whether or not it is disabled. 89 | func (t *Table) Get(ctx context.Context, path, body string) (*Job, error) { 90 | s := "SELECT path, body, `interval`, location, next_run, next_sched, instance_id " + 91 | "FROM " + t.name + " " + 92 | "WHERE path = ? AND body = ?" 93 | row := t.db.QueryRowContext(ctx, s, path, body) 94 | j, _, err := t.scanJob(row, false) 95 | return &j, err 96 | } 97 | 98 | func (t *Table) getForUpdate(ctx context.Context, path, body string) (tx *sql.Tx, j Job, now time.Time, err error) { 99 | tx, err = t.db.BeginTx(ctx, nil) 100 | if err != nil { 101 | return 102 | } 103 | defer func() { 104 | if err != nil && tx != nil { 105 | tx.Rollback() // nolint: errcheck 106 | } 107 | }() 108 | s := "SELECT path, body, `interval`, location, next_run, next_sched, instance_id, IFNULL(CAST(? as DATETIME), UTC_TIMESTAMP()) " + 109 | "FROM " + t.name + " " + 110 | "WHERE path = ? AND body = ? FOR UPDATE" 111 | row := tx.QueryRowContext(ctx, s, t.Clk.NowUTC(), path, body) 112 | j, now, err = t.scanJob(row, true) 113 | return 114 | } 115 | 116 | func (t *Table) scanJob(row *sql.Row, withCurrentTime bool) (j Job, now time.Time, err error) { 117 | var interval, locationName string 118 | var instanceID sql.NullInt64 119 | if withCurrentTime { 120 | err = row.Scan(&j.Path, &j.Body, &interval, &locationName, &j.NextRun, &j.NextSched, &instanceID, &now) 121 | } else { 122 | err = row.Scan(&j.Path, &j.Body, &interval, &locationName, &j.NextRun, &j.NextSched, &instanceID) 123 | } 124 | if err == sql.ErrNoRows { 125 | err = ErrNotExist 126 | } 127 | if err != nil { 128 | return 129 | } 130 | err = j.setLocation(locationName) 131 | if err != nil { 132 | return 133 | } 134 | if interval != "" { 135 | j.Interval, err = duration.ParseISO8601(interval) 136 | if err != nil { 137 | return 138 | } 139 | } 140 | if instanceID.Valid { 141 | id := uint32(instanceID.Int64) 142 | j.InstanceID = &id 143 | } 144 | now = now.In(j.Location) 145 | return 146 | } 147 | 148 | // AddJob inserts a job into the scheduler table. 149 | func (t *Table) AddJob(ctx context.Context, key Key, interval, delay duration.Duration, location *time.Location, nextRun time.Time) (*Job, error) { 150 | if location == nil { 151 | location = time.UTC 152 | } 153 | tx, err := t.db.BeginTx(ctx, nil) 154 | if err != nil { 155 | return nil, err 156 | } 157 | defer tx.Rollback() // nolint: errcheck 158 | if nextRun.IsZero() { 159 | row := tx.QueryRowContext(ctx, "SELECT IFNULL(CAST(? as DATETIME), UTC_TIMESTAMP())", t.Clk.NowUTC()) 160 | var now time.Time 161 | if err := row.Scan(&now); err != nil { 162 | return nil, err 163 | } 164 | now = now.In(location) 165 | nextRun = delay.Shift(now) 166 | } 167 | s := "REPLACE INTO " + t.name + 168 | "(path, body, `interval`, location, next_run, next_sched) " + 169 | "VALUES (?, ?, ?, ?, ?, ?)" 170 | var locationName string 171 | if location != time.UTC { 172 | locationName = location.String() 173 | } 174 | var intervalString string 175 | if !interval.IsZero() { 176 | intervalString = interval.String() 177 | } 178 | _, err = tx.ExecContext(ctx, s, key.Path, key.Body, intervalString, locationName, nextRun.UTC(), nextRun.UTC()) 179 | if err != nil { 180 | return nil, err 181 | } 182 | job := &Job{ 183 | Key: key, 184 | Interval: interval, 185 | Location: location, 186 | NextRun: sql.NullTime{Valid: true, Time: nextRun}, 187 | NextSched: nextRun, 188 | } 189 | return job, tx.Commit() 190 | } 191 | 192 | // EnableJob marks the job as enabled by setting next_run to next_sched. 193 | // 194 | // If next_sched is in the past, the job will then be picked up for execution immediately. 195 | // 196 | // With FixedIntervals enabled, next_sched is advanced by the value of interval 197 | // until it's in the future and next_run matches it. 198 | func (t *Table) EnableJob(ctx context.Context, key Key) (*Job, error) { 199 | tx, j, now, err := t.getForUpdate(ctx, key.Path, key.Body) 200 | if err != nil { 201 | return nil, err 202 | } 203 | defer tx.Rollback() // nolint: errcheck 204 | if j.Enabled() { 205 | // Job is already enabled. 206 | return &j, nil 207 | } 208 | if t.FixedIntervals { 209 | for j.NextSched.Before(now) { 210 | j.NextSched = j.Interval.Shift(j.NextSched) 211 | } 212 | } 213 | j.NextRun.Time = j.NextSched 214 | j.NextRun.Valid = true 215 | s := "UPDATE " + t.name + " " + 216 | "SET next_run=?, next_sched=? " + 217 | "WHERE path = ? AND body = ?" 218 | _, err = tx.ExecContext(ctx, s, j.NextRun.Time.UTC(), j.NextSched.UTC(), key.Path, key.Body) 219 | if err != nil { 220 | return nil, fmt.Errorf("failed to enable job: %w", err) 221 | } 222 | return &j, tx.Commit() 223 | } 224 | 225 | // DisableJob prevents a job from running by setting next_run to NULL, 226 | // while preserving the value of next_sched. 227 | func (t *Table) DisableJob(ctx context.Context, key Key) (*Job, error) { 228 | tx, j, _, err := t.getForUpdate(ctx, key.Path, key.Body) 229 | if err != nil { 230 | return nil, err 231 | } 232 | defer tx.Rollback() // nolint: errcheck 233 | if !j.Enabled() { 234 | // Job is already disabled. 235 | return &j, nil 236 | } 237 | if !t.FixedIntervals { 238 | // Disabling then enabling the job has a side effect of resetting exponential backoff to initial value. 239 | // This is not a problem if RetryMultiplier=1. 240 | j.NextSched = j.NextRun.Time 241 | } 242 | s := "UPDATE " + t.name + " SET next_run=NULL, next_sched=? WHERE path = ? AND body = ?" 243 | _, err = tx.ExecContext(ctx, s, j.NextSched, key.Path, key.Body) 244 | if err != nil { 245 | return nil, fmt.Errorf("failed to disable job: %w", err) 246 | } 247 | j.NextRun.Valid = false 248 | return &j, tx.Commit() 249 | } 250 | 251 | // DeleteJob removes a job from scheduler table. 252 | func (t *Table) DeleteJob(ctx context.Context, key Key) error { 253 | return withRetries(maxRetries, func() error { 254 | return t.deleteJob(ctx, key) 255 | }) 256 | } 257 | 258 | func (t *Table) deleteJob(ctx context.Context, key Key) error { 259 | s := "DELETE FROM " + t.name + " WHERE path=? AND body=?" 260 | _, err := t.db.ExecContext(ctx, s, key.Path, key.Body) 261 | return err 262 | } 263 | 264 | // Front returns the next scheduled job from the table, 265 | // based on the value of next_run, and claims it for the calling instance. 266 | func (t *Table) Front(ctx context.Context, instanceID uint32) (*Job, error) { 267 | var job *Job 268 | err := withRetries(maxRetries, func() error { 269 | var err error 270 | job, err = t.front(ctx, instanceID) 271 | return err 272 | }) 273 | return job, err 274 | } 275 | 276 | func (t *Table) front(ctx context.Context, instanceID uint32) (*Job, error) { 277 | tx, err := t.db.BeginTx(ctx, nil) 278 | if err != nil { 279 | return nil, err 280 | } 281 | defer tx.Rollback() // nolint: errcheck 282 | s := "SELECT path, body, `interval`, location, next_run, next_sched " + 283 | "FROM " + t.name + " " + 284 | "WHERE next_run < IFNULL(CAST(? as DATETIME), UTC_TIMESTAMP()) " + 285 | "AND instance_id IS NULL " + 286 | "ORDER BY next_run ASC LIMIT 1 " + 287 | "FOR UPDATE" 288 | if t.SkipLocked { 289 | s += " SKIP LOCKED" 290 | } 291 | row := tx.QueryRowContext(ctx, s, t.Clk.NowUTC()) 292 | var j Job 293 | var interval, locationName string 294 | err = row.Scan(&j.Path, &j.Body, &interval, &locationName, &j.NextRun, &j.NextSched) 295 | if err != nil { 296 | return nil, err 297 | } 298 | if interval != "" { 299 | j.Interval, err = duration.ParseISO8601(interval) 300 | if err != nil { 301 | return nil, err 302 | } 303 | } 304 | err = j.setLocation(locationName) 305 | if err != nil { 306 | return nil, err 307 | } 308 | s = "UPDATE " + t.name + " SET instance_id=? WHERE path=? AND body=?" 309 | _, err = tx.ExecContext(ctx, s, instanceID, j.Path, j.Body) 310 | if err != nil { 311 | return nil, err 312 | } 313 | return &j, tx.Commit() 314 | } 315 | 316 | // UpdateNextRun sets next_run and next_sched, and unclaims it from an instance. 317 | // 318 | // With default settings, next_sched and next_run are set to now+delay. 319 | // 320 | // With FixedIntervals enabled, next_sched is advanced by the value of interval 321 | // until it's in the future and next_run matches it. 322 | // 323 | // If this is a retry, next_run is set to a value based on retry parameters and next_sched is not adjusted. 324 | // 325 | // If UpdateNextRun is called on a disabled job, as many happen when a job has 326 | // been disabled during execution, next_sched will advance but next_run will remain NULL. 327 | func (t *Table) UpdateNextRun(ctx context.Context, key Key, randFactor float64, retryParams *retry.Retry) error { 328 | return withRetries(maxRetries, func() error { 329 | return t.updateNextRun(ctx, key, randFactor, retryParams) 330 | }) 331 | } 332 | 333 | func (t *Table) updateNextRun(ctx context.Context, key Key, randFactor float64, retryParams *retry.Retry) error { 334 | tx, j, now, err := t.getForUpdate(ctx, key.Path, key.Body) 335 | if err != nil { 336 | return err 337 | } 338 | defer tx.Rollback() // nolint: errcheck 339 | switch { 340 | case retryParams != nil: 341 | j.NextRun.Time = retryParams.NextRun(j.NextSched, now) 342 | if j.NextRun.Time.IsZero() { 343 | j.NextRun.Valid = false 344 | } 345 | case t.FixedIntervals: 346 | for j.NextSched.Before(now) { 347 | j.NextSched = j.Interval.Shift(j.NextSched) 348 | } 349 | j.NextRun.Time = j.NextSched 350 | case !t.FixedIntervals: 351 | j.NextSched = j.Interval.Shift(now) 352 | if randFactor > 0 { 353 | diff := randomize(j.NextSched.Sub(now), randFactor) 354 | j.NextSched = now.Add(diff) 355 | } 356 | j.NextRun.Time = j.NextSched 357 | } 358 | s := "UPDATE " + t.name + " " + 359 | "SET next_run=?, next_sched=?, instance_id=NULL " + 360 | "WHERE path = ? AND body = ?" 361 | // Note that we are passing next_run as sql.NullTime value. 362 | // If next_run is already NULL (j.NextRun.Valid == false), it is not going to be updated. 363 | // This may happen when the job gets disabled while it is running. 364 | _, err = tx.ExecContext(ctx, s, j.NextRun, j.NextSched.UTC(), key.Path, key.Body) 365 | if err != nil { 366 | return fmt.Errorf("failed to update next run: %w", err) 367 | } 368 | return tx.Commit() 369 | } 370 | 371 | // UpdateInstanceID claims a job for an instance. 372 | func (t *Table) UpdateInstanceID(ctx context.Context, key Key, instanceID uint32) error { 373 | s := "UPDATE " + t.name + " " + 374 | "SET instance_id=? " + 375 | "WHERE path = ? AND body = ?" 376 | _, err := t.db.ExecContext(ctx, s, instanceID, key.Path, key.Body) 377 | return err 378 | } 379 | 380 | // Count returns the count of scheduled jobs in the table. 381 | func (t *Table) Count(ctx context.Context) (int64, error) { 382 | s := "SELECT COUNT(*) FROM " + t.name 383 | var count int64 384 | return count, t.db.QueryRowContext(ctx, s).Scan(&count) 385 | } 386 | 387 | // Pending returns the count of pending jobs in the table. 388 | func (t *Table) Pending(ctx context.Context) (int64, error) { 389 | s := "SELECT COUNT(*) FROM " + t.name + " " + 390 | "WHERE next_run < IFNULL(CAST(? as DATETIME), UTC_TIMESTAMP())" 391 | var count int64 392 | return count, t.db.QueryRowContext(ctx, s, t.Clk.NowUTC()).Scan(&count) 393 | } 394 | 395 | // Lag returns the number of seconds passed from the execution time of the oldest pending job. 396 | func (t *Table) Lag(ctx context.Context) (int64, error) { 397 | s := "SELECT TIMESTAMPDIFF(SECOND, next_run, IFNULL(CAST(? as DATETIME), UTC_TIMESTAMP())) FROM " + t.name + " " + 398 | "WHERE next_run < IFNULL(CAST(? as DATETIME), UTC_TIMESTAMP()) AND instance_id is NULL " + 399 | "ORDER BY next_run ASC LIMIT 1" 400 | now := t.Clk.NowUTC() 401 | var lag int64 402 | err := t.db.QueryRowContext(ctx, s, now, now).Scan(&lag) 403 | if err == sql.ErrNoRows { 404 | err = nil 405 | } 406 | return lag, err 407 | } 408 | 409 | // Running returns the count of total running jobs in the table. 410 | func (t *Table) Running(ctx context.Context) (int64, error) { 411 | s := "SELECT COUNT(*) FROM " + t.name + " " + 412 | "WHERE instance_id IS NOT NULL" 413 | var count int64 414 | return count, t.db.QueryRowContext(ctx, s).Scan(&count) 415 | } 416 | 417 | // Instances returns the count of running Dalga instances. 418 | func (t *Table) Instances(ctx context.Context) (int64, error) { 419 | s := "SELECT COUNT(*) FROM " + t.name + "_instances " 420 | var count int64 421 | return count, t.db.QueryRowContext(ctx, s).Scan(&count) 422 | } 423 | 424 | // UpdateInstance adds the instance to the list of active instances, 425 | // and clears out any inactive instances from the list, such as 426 | // instances that were unable to call DeleteInstance during shutdown. 427 | func (t *Table) UpdateInstance(ctx context.Context, id uint32) error { 428 | now := t.Clk.NowUTC() 429 | s1 := "INSERT INTO " + t.name + "_instances(id, updated_at) VALUES (" + strconv.FormatUint(uint64(id), 10) + ",IFNULL(CAST(? as DATETIME), UTC_TIMESTAMP())) ON DUPLICATE KEY UPDATE updated_at=IFNULL(?, UTC_TIMESTAMP())" 430 | if _, err := t.db.ExecContext(ctx, s1, now, now); err != nil { 431 | return err 432 | } 433 | s2 := "DELETE FROM " + t.name + "_instances WHERE updated_at < IFNULL(CAST(? as DATETIME), UTC_TIMESTAMP) - INTERVAL 1 MINUTE" 434 | if _, err := t.db.ExecContext(ctx, s2, now); err != nil { 435 | return err 436 | } 437 | return nil 438 | } 439 | 440 | // DeleteInstance removes an entry from the list of active instances. 441 | func (t *Table) DeleteInstance(ctx context.Context, id uint32) error { 442 | s := "DELETE FROM " + t.name + "_instances WHERE id=?" 443 | _, err := t.db.ExecContext(ctx, s, id) 444 | return err 445 | } 446 | 447 | func randomize(d time.Duration, f float64) time.Duration { 448 | delta := time.Duration(f * float64(d)) 449 | return d - delta + time.Duration(float64(2*delta)*rand.Float64()) // nolint: gosec 450 | } 451 | 452 | func withRetries(retryCount int, fn func() error) (err error) { 453 | for attempt := 0; attempt < retryCount; attempt++ { 454 | err = fn() 455 | if err == nil { 456 | return nil 457 | } 458 | var merr *mysql.MySQLError 459 | ok := errors.As(err, &merr) 460 | if !ok { 461 | return err 462 | } 463 | if dur := mysqlRetryInterval(merr); dur > 0 { 464 | log.Println("mysql error:", merr.Number, "sleeping", dur.String(), "before retry") 465 | time.Sleep(dur) 466 | continue 467 | } 468 | break 469 | } 470 | return 471 | } 472 | 473 | func mysqlRetryInterval(err error) time.Duration { 474 | if ok, myerr := my.Error(err); ok { // MySQL error 475 | if my.MySQLErrorCode(err) == 1213 { // deadlock 476 | return time.Millisecond * 10 477 | } 478 | if my.CanRetry(myerr) { 479 | return time.Second 480 | } 481 | } 482 | return 0 483 | } 484 | -------------------------------------------------------------------------------- /internal/table/table_test.go: -------------------------------------------------------------------------------- 1 | package table_test 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "testing" 7 | "time" 8 | 9 | "github.com/cenkalti/dalga/v3/internal/clock" 10 | "github.com/cenkalti/dalga/v3/internal/table" 11 | "github.com/senseyeio/duration" 12 | ) 13 | 14 | var dsn = "root:@tcp(127.0.0.1:3306)/test?parseTime=true&multiStatements=true" 15 | 16 | func TestAddJob(t *testing.T) { 17 | db, err := sql.Open("mysql", dsn) 18 | if err != nil { 19 | t.Fatal(err.Error()) 20 | } 21 | defer db.Close() 22 | 23 | if err = db.Ping(); err != nil { 24 | t.Fatalf("cannot connect to mysql: %s", err.Error()) 25 | } 26 | 27 | ctx := context.Background() 28 | 29 | now := time.Date(2020, time.August, 19, 11, 46, 0, 0, time.Local) 30 | firstRun := now.Add(time.Minute * 30) 31 | 32 | tbl := table.New(db, "test_jobs") 33 | if err := tbl.Drop(ctx); err != nil { 34 | t.Fatal(err) 35 | } 36 | if err := tbl.Create(ctx); err != nil { 37 | t.Fatal(err) 38 | } 39 | tbl.SkipLocked = false 40 | tbl.FixedIntervals = true 41 | 42 | tbl.Clk = clock.New(now) 43 | j, err := tbl.AddJob(ctx, table.Key{ 44 | Path: "abc", 45 | Body: "def", 46 | }, mustDuration("PT60M"), mustDuration("PT30M"), time.Local, time.Time{}) 47 | if err != nil { 48 | t.Fatal(err) 49 | } 50 | if !j.NextRun.Time.Equal(firstRun) { 51 | t.Fatalf("expected first run '%v' but found '%v'", firstRun, j.NextRun) 52 | } 53 | t.Run("AddJob returns timezoned job", func(t *testing.T) { 54 | if expect, found := firstRun.Format(time.RFC3339), j.NextRun.Time.Format(time.RFC3339); expect != found { 55 | t.Fatalf("expected first run '%s' but found '%s'", expect, found) 56 | } 57 | }) 58 | 59 | var instanceID uint32 = 123456 60 | if err := tbl.UpdateInstance(ctx, instanceID); err != nil { 61 | t.Fatal(err) 62 | } 63 | 64 | _, err = tbl.Front(ctx, instanceID) 65 | if err != sql.ErrNoRows { 66 | t.Fatalf("unexpected error: %s", err.Error()) 67 | } 68 | 69 | t.Run("Get returns timezoned job", func(t *testing.T) { 70 | j, err = tbl.Get(ctx, "abc", "def") 71 | if err != nil { 72 | t.Fatalf("unexpected error: %s", err.Error()) 73 | } 74 | if expect, found := firstRun.Format(time.RFC3339), j.NextRun.Time.Format(time.RFC3339); expect != found { 75 | t.Fatalf("expected first run '%s' but found '%s'", expect, found) 76 | } 77 | }) 78 | 79 | tbl.Clk.Add(time.Minute * 31) 80 | 81 | j, err = tbl.Front(ctx, instanceID) 82 | if err != nil { 83 | t.Fatal(err) 84 | } 85 | if j.Key.Path != "abc" || j.Key.Body != "def" { 86 | t.Fatalf("unexpected key %v", j.Key) 87 | } 88 | t.Run("Front returns timezoned job", func(t *testing.T) { 89 | if expect, found := firstRun.Format(time.RFC3339), j.NextRun.Time.Format(time.RFC3339); expect != found { 90 | t.Fatalf("expected first run '%s' but found '%s'", expect, found) 91 | } 92 | }) 93 | 94 | t.Run("Disable hides job", func(t *testing.T) { 95 | if err := tbl.UpdateNextRun(ctx, j.Key, 0, nil); err != nil { 96 | t.Fatal(err) 97 | } 98 | tbl.Clk.Set(j.Interval.Shift(tbl.Clk.Get()).Add(time.Minute)) 99 | if _, err := tbl.DisableJob(ctx, j.Key); err != nil { 100 | t.Fatal(err) 101 | } 102 | _, err := tbl.Front(ctx, instanceID) 103 | if err != sql.ErrNoRows { 104 | t.Fatalf("unexpected error: %s", err.Error()) 105 | } 106 | }) 107 | 108 | t.Run("Disabled jobs have no nextRun", func(t *testing.T) { 109 | j, err := tbl.Get(ctx, j.Key.Path, j.Key.Body) 110 | if err != nil { 111 | t.Fatal(err) 112 | } 113 | if j.NextRun.Valid { 114 | t.Fatalf("expected nextRun to be invalid: %v", j.NextRun) 115 | } 116 | }) 117 | 118 | t.Run("Generic rescheduling won't re-enable a job", func(t *testing.T) { 119 | if err := tbl.UpdateNextRun(ctx, j.Key, 0, nil); err != nil { 120 | t.Fatal(err) 121 | } 122 | tbl.Clk.Set(j.Interval.Shift(tbl.Clk.Get()).Add(time.Minute)) 123 | _, err = tbl.Front(ctx, instanceID) 124 | if err != sql.ErrNoRows { 125 | t.Fatalf("unexpected error: %s", err.Error()) 126 | } 127 | }) 128 | 129 | t.Run("Can re-enable", func(t *testing.T) { 130 | if _, err := tbl.EnableJob(ctx, j.Key); err != nil { 131 | t.Fatal(err) 132 | } 133 | _, err = tbl.Front(ctx, instanceID) 134 | if err != sql.ErrNoRows { 135 | t.Fatalf("unexpected error: %v", err) 136 | } 137 | tbl.Clk.Set(j.Interval.Shift(tbl.Clk.Get()).Add(time.Minute)) 138 | j, err = tbl.Front(ctx, instanceID) 139 | if err != nil { 140 | t.Fatal(err) 141 | } 142 | if j.Key.Path != "abc" || j.Key.Body != "def" { 143 | t.Fatalf("unexpected key %v", j.Key) 144 | } 145 | }) 146 | } 147 | 148 | func mustDuration(s string) duration.Duration { 149 | d, err := duration.ParseISO8601(s) 150 | if err != nil { 151 | panic(err) 152 | } 153 | return d 154 | } 155 | -------------------------------------------------------------------------------- /recur_test.go: -------------------------------------------------------------------------------- 1 | package dalga // nolint: testpackage 2 | 3 | // These test cases are adapted from job_recurring_test.go in https://github.com/ajvb/kala 4 | 5 | import ( 6 | "bytes" 7 | "context" 8 | "net/http" 9 | "net/http/httptest" 10 | "testing" 11 | "time" 12 | ) 13 | 14 | // This test works by using a series of checkpoints, spaced apart. 15 | // A job is scheduled 5 seconds after the first checkpoint. 16 | // By moving the clock to each checkpoint, and then 6 seconds later, 17 | // you can verify that the job hasn't run between the two checkpoints, 18 | // and only runs at the scheduled point. 19 | // 20 | // This is useful for ensuring that durations behave correctly on a grand scale. 21 | func TestRecur(t *testing.T) { 22 | tableTests := []struct { 23 | Name string 24 | Location string 25 | Start string 26 | Interval string 27 | Checkpoints []string 28 | }{ 29 | { 30 | Name: "Daily", 31 | Location: "America/Los_Angeles", 32 | Start: "2020-Jan-13 14:09", 33 | Interval: "P1D", 34 | Checkpoints: []string{ 35 | "2020-Jan-14 14:09", 36 | "2020-Jan-15 14:09", 37 | "2020-Jan-16 14:09", 38 | }, 39 | }, 40 | { 41 | Name: "Daily across DST boundary", 42 | Location: "America/Los_Angeles", 43 | Start: "2020-Mar-05 14:09", 44 | Interval: "P1D", 45 | Checkpoints: []string{ 46 | "2020-Mar-06 14:09", 47 | "2020-Mar-07 14:09", 48 | "2020-Mar-08 14:09", 49 | "2020-Mar-09 14:09", 50 | }, 51 | }, 52 | { 53 | Name: "Daily across DST boundary in UTC", 54 | Location: time.UTC.String(), 55 | Start: "2020-Mar-05 14:09", 56 | Interval: "P1D", 57 | Checkpoints: []string{ 58 | "2020-Mar-06 14:09", 59 | "2020-Mar-07 14:09", 60 | "2020-Mar-08 14:09", 61 | "2020-Mar-09 14:09", 62 | }, 63 | }, 64 | { 65 | Name: "24 Hourly across DST boundary", 66 | Location: "America/Los_Angeles", 67 | Start: "2020-Mar-05 14:09", 68 | Interval: "PT24H", 69 | Checkpoints: []string{ 70 | "2020-Mar-06 14:09", 71 | "2020-Mar-07 14:09", 72 | "2020-Mar-08 15:09", 73 | "2020-Mar-09 15:09", 74 | }, 75 | }, 76 | { 77 | Name: "Weekly", 78 | Location: "America/Los_Angeles", 79 | Start: "2020-Jan-13 14:09", 80 | Interval: "P1W", 81 | Checkpoints: []string{ 82 | "2020-Jan-20 14:09", 83 | "2020-Jan-27 14:09", 84 | "2020-Feb-03 14:09", 85 | }, 86 | }, 87 | { 88 | Name: "Monthly", 89 | Location: "America/Los_Angeles", 90 | Start: "2020-Jan-20 14:09", 91 | Interval: "P1M", 92 | Checkpoints: []string{ 93 | "2020-Feb-20 14:09", 94 | "2020-Mar-20 14:09", 95 | "2020-Apr-20 14:09", 96 | "2020-May-20 14:09", 97 | "2020-Jun-20 14:09", 98 | "2020-Jul-20 14:09", 99 | "2020-Aug-20 14:09", 100 | "2020-Sep-20 14:09", 101 | "2020-Oct-20 14:09", 102 | "2020-Nov-20 14:09", 103 | "2020-Dec-20 14:09", 104 | "2021-Jan-20 14:09", 105 | }, 106 | }, 107 | { 108 | Name: "Monthly with Normalization", 109 | Location: "America/Los_Angeles", 110 | Start: "2020-Jul-31 14:09", 111 | Interval: "P1M", 112 | Checkpoints: []string{ 113 | "2020-Aug-31 14:09", 114 | "2020-Oct-01 14:09", 115 | "2020-Nov-01 14:09", 116 | }, 117 | }, 118 | { 119 | Name: "Yearly across Leap Year boundary", 120 | Location: "America/Los_Angeles", 121 | Start: "2020-Jan-20 14:09", 122 | Interval: "P1Y", 123 | Checkpoints: []string{ 124 | "2021-Jan-20 14:09", 125 | "2022-Jan-20 14:09", 126 | "2023-Jan-20 14:09", 127 | "2024-Jan-20 14:09", 128 | "2025-Jan-20 14:09", 129 | }, 130 | }, 131 | } 132 | 133 | called := make(chan string) 134 | endpoint := func(w http.ResponseWriter, r *http.Request) { 135 | var buf bytes.Buffer 136 | buf.ReadFrom(r.Body) 137 | r.Body.Close() 138 | called <- buf.String() 139 | } 140 | 141 | mux := http.NewServeMux() 142 | mux.HandleFunc("/", endpoint) 143 | srv := httptest.NewServer(mux) 144 | defer srv.Close() 145 | 146 | config := DefaultConfig 147 | config.MySQL.SkipLocked = false 148 | config.Jobs.FixedIntervals = true 149 | config.Jobs.ScanFrequency = 100 * time.Millisecond 150 | config.Endpoint.BaseURL = "http://" + srv.Listener.Addr().String() + "/" 151 | config.Listen.Port = 34008 152 | 153 | d, lis, cleanup := newDalga(t, config) 154 | defer cleanup() 155 | 156 | clk := d.UseClock(time.Time{}) 157 | 158 | ctx, cancel := context.WithCancel(context.Background()) 159 | go d.Run(ctx) 160 | defer func() { 161 | cancel() 162 | <-d.NotifyDone() 163 | }() 164 | 165 | client := NewClient("http://" + lis.Addr()) 166 | 167 | for _, testStruct := range tableTests { 168 | testStruct := testStruct 169 | t.Run(testStruct.Name, func(t *testing.T) { 170 | ctx := context.Background() 171 | 172 | defer func() { 173 | if err := client.Cancel(ctx, "what", testStruct.Name); err != nil { 174 | t.Fatal(err) 175 | } 176 | }() 177 | 178 | now := parseTimeInLocation(t, testStruct.Start, testStruct.Location) 179 | clk.Set(now) 180 | 181 | start := now.Add(time.Second * 5) 182 | _, err := client.Schedule(ctx, "what", testStruct.Name, 183 | MustWithIntervalString(testStruct.Interval), 184 | WithFirstRun(start), 185 | MustWithLocationName(testStruct.Location), 186 | ) 187 | if err != nil { 188 | t.Fatal(err) 189 | } 190 | 191 | checkpoints := append([]string{testStruct.Start}, testStruct.Checkpoints...) 192 | 193 | for i, chk := range checkpoints { 194 | clk.Set(parseTimeInLocation(t, chk, testStruct.Location)) 195 | 196 | t.Logf("Checkpoint %d advanced time to: %s", i, clk.Get().Format(time.RFC3339)) 197 | 198 | select { 199 | case <-called: 200 | t.Fatalf("Expected job not run on checkpoint %d of test %s.", i, testStruct.Name) 201 | case <-time.After(time.Second): 202 | } 203 | 204 | clk.Add(time.Second * 6) 205 | 206 | t.Logf("Checkpoint %d advanced time to: %s", i, clk.Get().Format(time.RFC3339)) 207 | 208 | select { 209 | case v := <-called: 210 | if v != testStruct.Name { 211 | t.Fatalf("Expected body '%s' but found '%s'", testStruct.Name, v) 212 | } 213 | case <-time.After(testTimeout): 214 | t.Fatalf("Expected job to have run on checkpoint %d of test %s.", i, testStruct.Name) 215 | } 216 | 217 | time.Sleep(time.Millisecond * 500) 218 | } 219 | }) 220 | } 221 | } 222 | 223 | func parseTimeInLocation(t *testing.T, value string, location string) time.Time { 224 | loc, err := time.LoadLocation(location) 225 | if err != nil { 226 | t.Fatal(err) 227 | } 228 | now, err := time.ParseInLocation("2006-Jan-02 15:04", value, loc) 229 | if err != nil { 230 | t.Fatal(err) 231 | } 232 | return now 233 | } 234 | --------------------------------------------------------------------------------