├── .drone.yml ├── .gitignore ├── Dockerfile ├── README.md ├── _bin_ └── check_easyjson ├── client.go ├── errand-routes.go ├── errands-routes.go ├── errands-server.go ├── errands.db ├── glide.lock ├── glide.yaml ├── go.mod ├── go.sum ├── main.go ├── main_easyjson.go ├── memorydb └── memorydb.go ├── metrics └── metrics.go ├── misc └── illustration.png ├── pipeline-routes.go ├── postman └── Errands.postman_collection.json ├── schemas ├── errand.go ├── glide.yaml ├── pipeline.go ├── pipeline_test.go └── schemas_easyjson.go └── utils └── utils.go /.drone.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kind: pipeline 3 | type: docker 4 | name: review 5 | 6 | platform: 7 | os: linux 8 | arch: amd64 9 | 10 | steps: 11 | - name: vet 12 | image: golang:1.14 13 | commands: 14 | - go vet ./... 15 | volumes: 16 | - name: deps 17 | path: /go/pkg 18 | 19 | - name: lint 20 | pull: always 21 | image: polygonio/go-linter:v1.13 22 | environment: 23 | ADDITIONAL_ARGS: --new-from-rev 86548ee1cf5f1478bdd5ee285a2dd0d6f8406c76 24 | volumes: 25 | - name: deps 26 | path: /go/pkg 27 | 28 | - name: easyjson 29 | image: polygonio/tool-serialization:latest 30 | commands: 31 | - ./_bin_/check_easyjson 32 | volumes: 33 | - name: deps 34 | path: /go/pkg 35 | 36 | - name: test 37 | image: golang:1.14 38 | commands: 39 | - go test -v ./... 40 | environment: 41 | CGO_ENABLED: 0 42 | GOOS: linux 43 | volumes: 44 | - name: deps 45 | path: /go/pkg 46 | 47 | volumes: 48 | - name: deps 49 | temp: {} 50 | 51 | trigger: 52 | branch: 53 | - master 54 | event: 55 | - pull_request 56 | 57 | --- 58 | kind: pipeline 59 | type: docker 60 | name: latest docker image 61 | 62 | platform: 63 | os: linux 64 | arch: amd64 65 | 66 | steps: 67 | - name: build 68 | image: golang:1.14 69 | commands: 70 | - go build . 71 | 72 | - name: docker-push 73 | image: plugins/docker 74 | settings: 75 | password: 76 | from_secret: dockerhub_password 77 | repo: polygonio/errands-server 78 | tag: 79 | - ${DRONE_BRANCH/master/latest} 80 | - ${DRONE_COMMIT} 81 | username: 82 | from_secret: dockerhub_username 83 | 84 | trigger: 85 | branch: 86 | - master 87 | event: 88 | - push 89 | 90 | --- 91 | kind: pipeline 92 | type: docker 93 | name: release docker image 94 | 95 | platform: 96 | os: linux 97 | arch: amd64 98 | 99 | steps: 100 | - name: build 101 | image: golang:1.14 102 | commands: 103 | - go build . 104 | 105 | - name: docker-push 106 | image: plugins/docker 107 | settings: 108 | password: 109 | from_secret: dockerhub_password 110 | repo: polygonio/errands-server 111 | tag: 112 | - ${DRONE_TAG} 113 | username: 114 | from_secret: dockerhub_username 115 | 116 | trigger: 117 | event: 118 | - tag 119 | ref: 120 | - refs/tags/** 121 | 122 | --- 123 | kind: signature 124 | hmac: e92a73b237ff584c90348545cdc015ee1652451959565d48213cbe67158c3a78 125 | 126 | ... 127 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | vendor/ 15 | badger/ 16 | errands/ -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.14 2 | 3 | COPY errands-server /go/bin/errands-server 4 | 5 | ENTRYPOINT [ "errands-server" ] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 | 5 | # Errands Server 6 | Errands API Server. A language agnostic, HTTP based queue with server side events ( SSE ). Persistant storage using Badger for SSD performance. Concurrency safe for multiple workers to be processing errands off the queue. 7 | 8 | ## Client Libraries 9 | - JS - https://github.com/polygon-io/errands-js 10 | - Go - https://github.com/polygon-io/errands-go 11 | 12 | ## Optional Params: 13 | Errands server uses environment variables as a way to configure optional config params, here they are: 14 | - `ERRANDS_PORT=:4545` - Will change the listening port to 4545 15 | - `ERRANDS_STORAGE="/errands/"` - Will change the DB location to /errands/ 16 | 17 | ## Running: 18 | See the `postman` folder for the PostMan Collection which contains examples and tests for all the possible routes. 19 | 20 | You can run the API locally with docker: 21 | 22 | docker run -p 5555:5555 polygonio/errands-server 23 | 24 | ## Push Events 25 | There is an endpoint which will push server side events ( SSE ) to the client when changes happen on the errands server. 26 | 27 | /v1/errands/notifications?events={{events_to_listen_to}} 28 | 29 | The events are comma delimited. For example `?events=created,completed,failed`. The default is `*` which will send any and all events which happen on the server. 30 | 31 | Event Types: 32 | - `created` - When a new errand is created 33 | - `updated` - When a new errand is updated 34 | - `completed` - When a new errand is completed 35 | - `processing` - When a new errand is processing 36 | - `retry` - When a new errand is being retried 37 | - `failed` - When a new errand is failed 38 | 39 | Along with the event type, the errand which triggered the event will have its data included in the message. -------------------------------------------------------------------------------- /_bin_/check_easyjson: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eou pipefail 4 | 5 | pkgs=(schemas) 6 | 7 | for pkg in "${pkgs[@]}"; do 8 | cp ${pkg}/${pkg}_easyjson.go ${pkg}/${pkg}_easyjson.go.check 9 | 10 | easyjson -pkg ${pkg} 11 | failed=$(diff ${pkg}/${pkg}_easyjson.go.check ${pkg}/${pkg}_easyjson.go) || echo "Run \`easyjson -pkg ${pkg}\` and update HEAD" 12 | 13 | # restore easyjson code 14 | git checkout ${pkg}/${pkg}_easyjson.go 15 | rm ${pkg}/${pkg}_easyjson.go.check 16 | 17 | if [[ "$(echo -n "${failed}" | wc -w)" -ne "0" ]]; then 18 | exit 1 19 | fi 20 | done 21 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "strings" 6 | 7 | gin "github.com/gin-gonic/gin" 8 | ) 9 | 10 | type Client struct { 11 | ErrandsServer *ErrandsServer 12 | Notifications chan *Notification 13 | Gin *gin.Context 14 | EventSubs []string 15 | } 16 | 17 | func (s *ErrandsServer) RemoveClient(c *Client) { 18 | close(c.Notifications) 19 | s.UnregisterClient <- c 20 | } 21 | 22 | func (s *ErrandsServer) NewClient(c *gin.Context) (*Client, error) { 23 | obj := &Client{ 24 | Notifications: make(chan *Notification, 10), 25 | ErrandsServer: s, 26 | Gin: c, 27 | } 28 | events := c.DefaultQuery("events", "*") 29 | 30 | obj.EventSubs = strings.Split(events, ",") 31 | if len(obj.EventSubs) < 1 { 32 | return obj, errors.New("must have at least 1 event subscription") 33 | } 34 | s.RegisterClient <- obj 35 | 36 | return obj, nil 37 | } 38 | 39 | func (c *Client) Gone() { 40 | c.ErrandsServer.RemoveClient(c) 41 | } 42 | -------------------------------------------------------------------------------- /errand-routes.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/http" 7 | 8 | gin "github.com/gin-gonic/gin" 9 | "github.com/polygon-io/errands-server/metrics" 10 | schemas "github.com/polygon-io/errands-server/schemas" 11 | utils "github.com/polygon-io/errands-server/utils" 12 | ) 13 | 14 | const inactive = "inactive" 15 | const active = "active" 16 | 17 | type UpdateRequest struct { 18 | Progress float64 `json:"progress"` 19 | Logs []string `json:"logs"` 20 | } 21 | 22 | func (s *ErrandsServer) updateErrand(c *gin.Context) { 23 | var ( 24 | updatedErrand *schemas.Errand 25 | updateReq UpdateRequest 26 | ) 27 | 28 | if err := c.ShouldBind(&updateReq); err != nil { 29 | c.JSON(http.StatusBadRequest, gin.H{ 30 | "message": "Invalid Parameters", 31 | "error": err.Error(), 32 | }) 33 | 34 | return 35 | } 36 | 37 | updatedErrand, err := s.UpdateErrandByID(c.Param("id"), func(errand *schemas.Errand) error { 38 | if errand.Status != active { 39 | return errors.New("errand must be in active state to update progress") 40 | } 41 | // Update this errand attributes: 42 | if updateReq.Progress != 0 { 43 | if updateReq.Progress < 0 || updateReq.Progress >= 101 { 44 | return errors.New("progress must be between 0 - 100") 45 | } 46 | errand.Progress = updateReq.Progress 47 | } 48 | 49 | return nil 50 | }) 51 | if err != nil { 52 | c.JSON(http.StatusInternalServerError, gin.H{ 53 | "message": "Internal Server Error!", 54 | "error": err.Error(), 55 | }) 56 | 57 | return 58 | } 59 | 60 | s.AddNotification("updated", updatedErrand) 61 | c.JSON(http.StatusOK, gin.H{ 62 | "status": "OK", 63 | "results": updatedErrand, 64 | }) 65 | } 66 | 67 | type FailedRequest struct { 68 | Reason string `json:"reason" binding:"required"` 69 | } 70 | 71 | func (s *ErrandsServer) failedErrand(c *gin.Context) { 72 | var ( 73 | updatedErrand *schemas.Errand 74 | failedReq FailedRequest 75 | ) 76 | 77 | if err := c.ShouldBind(&failedReq); err != nil { 78 | c.JSON(http.StatusBadRequest, gin.H{ 79 | "message": "Invalid Parameters", 80 | "error": err.Error(), 81 | }) 82 | 83 | return 84 | } 85 | 86 | updatedErrand, err := s.UpdateErrandByID(c.Param("id"), func(errand *schemas.Errand) error { 87 | return failErrand(errand, failedReq) 88 | }) 89 | if err != nil { 90 | c.JSON(http.StatusInternalServerError, gin.H{ 91 | "message": "Internal Server Error!", 92 | "error": err.Error(), 93 | }) 94 | 95 | return 96 | } 97 | 98 | s.AddNotification("failed", updatedErrand) 99 | c.JSON(http.StatusOK, gin.H{ 100 | "status": "OK", 101 | "results": updatedErrand, 102 | }) 103 | } 104 | 105 | func failErrand(errand *schemas.Errand, failureRequest FailedRequest) error { 106 | // Update this errand attributes: 107 | if err := errand.AddToLogs("ERROR", failureRequest.Reason); err != nil { 108 | return err 109 | } 110 | errand.Failed = utils.GetTimestamp() 111 | errand.Status = schemas.StatusFailed 112 | errand.Progress = 0 113 | if errand.Options.Retries > 0 { 114 | // If we should retry this errand: 115 | if errand.Attempts <= errand.Options.Retries { 116 | errand.Status = inactive 117 | } else { 118 | // If this errand is out of retries 119 | metrics.ErrandFailed(errand.Type) 120 | } 121 | } else { 122 | // If this errand was not configured with retries 123 | metrics.ErrandFailed(errand.Type) 124 | } 125 | 126 | return nil 127 | } 128 | 129 | type CompletedRequest struct { 130 | Results *gin.H `json:"results"` 131 | } 132 | 133 | func (s *ErrandsServer) completeErrand(c *gin.Context) { 134 | var ( 135 | updatedErrand *schemas.Errand 136 | compReq CompletedRequest 137 | ) 138 | 139 | if err := c.ShouldBind(&compReq); err != nil { 140 | c.JSON(http.StatusBadRequest, gin.H{ 141 | "message": "Invalid Parameters", 142 | "error": err.Error(), 143 | }) 144 | 145 | return 146 | } 147 | 148 | shouldDelete := false 149 | 150 | updatedErrand, err := s.UpdateErrandByID(c.Param("id"), func(errand *schemas.Errand) error { 151 | // Update this errand attributes: 152 | if err := errand.AddToLogs("INFO", "Completed!"); err != nil { 153 | return err 154 | } 155 | errand.Completed = utils.GetTimestamp() 156 | errand.Status = schemas.StatusCompleted 157 | errand.Progress = 100 158 | // errand.Results = compReq.Results 159 | // If we should delete this errand upon completion: 160 | if errand.Options.DeleteOnCompleted { 161 | shouldDelete = true 162 | } 163 | 164 | metrics.ErrandCompleted(errand.Type) 165 | return nil 166 | }) 167 | if err == nil && shouldDelete && updatedErrand.ID != "" { 168 | s.deleteErrandByID(updatedErrand.ID) 169 | } 170 | 171 | if shouldDelete && updatedErrand.ID != "" { 172 | s.deleteErrandByID(updatedErrand.ID) 173 | } 174 | 175 | s.AddNotification("completed", updatedErrand) 176 | 177 | c.JSON(http.StatusOK, gin.H{ 178 | "status": "OK", 179 | "results": updatedErrand, 180 | }) 181 | } 182 | 183 | func (s *ErrandsServer) retryErrand(c *gin.Context) { 184 | var updatedErrand *schemas.Errand 185 | 186 | updatedErrand, err := s.UpdateErrandByID(c.Param("id"), func(errand *schemas.Errand) error { 187 | if errand.Status == inactive { 188 | return errors.New("cannot retry errand which is in inactive state") 189 | } 190 | // Update this errand attributes: 191 | if err := errand.AddToLogs("INFO", "Retrying!"); err != nil { 192 | return err 193 | } 194 | errand.Status = inactive 195 | errand.Progress = 0 196 | return nil 197 | }) 198 | if err != nil { 199 | c.JSON(http.StatusInternalServerError, gin.H{ 200 | "message": "Internal Server Error!", 201 | "error": err.Error(), 202 | }) 203 | 204 | return 205 | } 206 | 207 | s.AddNotification("retry", updatedErrand) 208 | c.JSON(http.StatusOK, gin.H{ 209 | "status": "OK", 210 | "results": updatedErrand, 211 | }) 212 | } 213 | 214 | func (s *ErrandsServer) logToErrand(c *gin.Context) { 215 | var logReq schemas.Log 216 | if err := c.ShouldBind(&logReq); err != nil { 217 | c.JSON(http.StatusBadRequest, gin.H{ 218 | "message": "Invalid Parameters", 219 | "error": err.Error(), 220 | }) 221 | 222 | return 223 | } 224 | 225 | updatedErrand, err := s.UpdateErrandByID(c.Param("id"), func(errand *schemas.Errand) error { 226 | if errand.Status != active { 227 | return errors.New("errand must be in active state to log to") 228 | } 229 | 230 | // Update this errand attributes: 231 | return errand.AddToLogs(logReq.Severity, logReq.Message) 232 | }) 233 | if err != nil { 234 | c.JSON(http.StatusInternalServerError, gin.H{ 235 | "message": "Internal Server Error!", 236 | "error": err.Error(), 237 | }) 238 | 239 | return 240 | } 241 | 242 | c.JSON(http.StatusOK, gin.H{ 243 | "status": "OK", 244 | "results": updatedErrand, 245 | }) 246 | } 247 | 248 | func (s *ErrandsServer) deleteErrand(c *gin.Context) { 249 | s.ErrandStore.Delete(c.Param("id")) 250 | 251 | s.deleteErrandByID(c.Param("id")) 252 | 253 | c.JSON(http.StatusOK, gin.H{ 254 | "status": "OK", 255 | }) 256 | } 257 | 258 | func (s *ErrandsServer) getErrand(c *gin.Context) { 259 | errandObj, found := s.ErrandStore.Get(c.Param("id")) 260 | if !found { 261 | c.JSON(http.StatusNotFound, nil) 262 | } 263 | errand := errandObj.(schemas.Errand) 264 | c.JSON(http.StatusOK, gin.H{ 265 | "status": "OK", 266 | "results": errand, 267 | }) 268 | } 269 | 270 | func (s *ErrandsServer) deleteErrandByID(id string) { 271 | s.ErrandStore.Delete(id) 272 | } 273 | 274 | // UpdateErrandByID Lets you pass in a function which will be called allowing you to update the errand. If no error is returned, the errand will be saved in the DB with the new attributes. 275 | func (s *ErrandsServer) UpdateErrandByID(id string, update func(*schemas.Errand) error) (*schemas.Errand, error) { 276 | errandObj, found := s.ErrandStore.Get(id) 277 | if !found { 278 | return nil, errors.New("errand with this ID not found") 279 | } 280 | 281 | errand := errandObj.(schemas.Errand) 282 | if err := update(&errand); err != nil { 283 | return nil, fmt.Errorf("error in given update function (fn): %w", err) 284 | } 285 | 286 | s.updateErrandInPipeline(&errand) 287 | 288 | s.ErrandStore.SetDefault(id, errand) 289 | 290 | return &errand, nil 291 | } 292 | 293 | func (s *ErrandsServer) UpdateErrandsByFilter(filter func(*schemas.Errand) bool, update func(*schemas.Errand) error) error { 294 | for _, itemObj := range s.ErrandStore.Items() { 295 | errand := itemObj.Object.(schemas.Errand) 296 | 297 | if filter(&errand) { 298 | if err := update(&errand); err != nil { 299 | return fmt.Errorf("error in given update function (fn): %w", err) 300 | } 301 | 302 | s.updateErrandInPipeline(&errand) 303 | 304 | s.ErrandStore.SetDefault(errand.ID, errand) 305 | } 306 | } 307 | 308 | return nil 309 | } 310 | -------------------------------------------------------------------------------- /errands-routes.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "io" 7 | "net/http" 8 | "sort" 9 | "time" 10 | 11 | log "github.com/sirupsen/logrus" 12 | 13 | gin "github.com/gin-gonic/gin" 14 | 15 | schemas "github.com/polygon-io/errands-server/schemas" 16 | utils "github.com/polygon-io/errands-server/utils" 17 | ) 18 | 19 | func (s *ErrandsServer) errandNotifications(c *gin.Context) { 20 | client, err := s.NewClient(c) 21 | if err != nil { 22 | c.JSON(http.StatusInternalServerError, gin.H{ 23 | "message": "Error Creating Subscription", 24 | "error": err.Error(), 25 | }) 26 | 27 | return 28 | } 29 | 30 | w := client.Gin.Writer 31 | 32 | w.Header().Set("Cache-Control", "no-cache") 33 | w.Header().Set("Connection", "keep-alive") 34 | w.Header().Set("Content-Type", "text/event-stream") 35 | w.Header().Set("X-Accel-Buffering", "no") 36 | 37 | clientGone := client.Gin.Writer.CloseNotify() 38 | 39 | client.Gin.Stream(func(wr io.Writer) bool { 40 | for { 41 | select { 42 | case <-clientGone: 43 | client.Gone() 44 | return false 45 | case t, ok := <-client.Notifications: 46 | if ok { 47 | // If we are subscribed to this event type: 48 | if utils.Contains(client.EventSubs, t.Event) || client.EventSubs[0] == "*" { 49 | jsonData, _ := json.Marshal(t) 50 | client.Gin.SSEvent("message", string(jsonData)) 51 | w.Flush() 52 | } 53 | return true 54 | } 55 | return false 56 | } 57 | } 58 | }) 59 | } 60 | 61 | func (s *ErrandsServer) createErrand(c *gin.Context) { 62 | log.Println("creating errand") 63 | 64 | var item schemas.Errand 65 | 66 | if err := c.ShouldBindJSON(&item); err != nil { 67 | c.JSON(http.StatusBadRequest, gin.H{ 68 | "message": "Errand validation failed!", 69 | "error": err.Error(), 70 | }) 71 | 72 | return 73 | } 74 | 75 | item.SetDefaults() 76 | s.ErrandStore.SetDefault(item.ID, item) 77 | s.AddNotification("created", &item) 78 | c.JSON(http.StatusOK, gin.H{ 79 | "status": "OK", 80 | "results": item, 81 | }) 82 | } 83 | 84 | func (s *ErrandsServer) saveErrand(errand *schemas.Errand) error { 85 | if !ContainsStatus(schemas.ErrandStatuses, errand.Status) { 86 | return errors.New("invalid errand status state") 87 | } 88 | 89 | s.ErrandStore.SetDefault(errand.ID, *errand) 90 | 91 | return nil 92 | } 93 | 94 | func (s *ErrandsServer) getAllErrands(c *gin.Context) { 95 | errands := s.GetErrandsBy(func(errand *schemas.Errand) bool { 96 | return true 97 | }) 98 | 99 | c.JSON(http.StatusOK, gin.H{ 100 | "status": "OK", 101 | "results": errands, 102 | }) 103 | } 104 | 105 | func (s *ErrandsServer) getFilteredErrands(c *gin.Context) { 106 | key := c.Param("key") 107 | value := c.Param("val") 108 | 109 | errands := s.GetErrandsBy(func(errand *schemas.Errand) bool { 110 | switch key { 111 | case "status": 112 | return string(errand.Status) == value 113 | case "type": 114 | return errand.Type == value 115 | default: 116 | return false 117 | } 118 | }) 119 | 120 | c.JSON(http.StatusOK, gin.H{ 121 | "status": "OK", 122 | "results": errands, 123 | }) 124 | } 125 | 126 | type filteredUpdateReq struct { 127 | Status string `json:"status"` 128 | Delete bool `json:"delete"` 129 | } 130 | 131 | func (s *ErrandsServer) updateFilteredErrands(c *gin.Context) { 132 | key := c.Param("key") 133 | value := c.Param("val") 134 | 135 | var updateReq filteredUpdateReq 136 | 137 | if err := c.ShouldBind(&updateReq); err != nil { 138 | c.JSON(http.StatusBadRequest, gin.H{ 139 | "message": "Invalid Parameters", 140 | "error": err.Error(), 141 | }) 142 | 143 | return 144 | } 145 | 146 | errands := s.GetErrandsBy(func(errand *schemas.Errand) bool { 147 | switch key { 148 | case "status": 149 | return string(errand.Status) == value 150 | case "type": 151 | return errand.Type == value 152 | default: 153 | return false 154 | } 155 | }) 156 | 157 | var err error 158 | 159 | for _, errand := range errands { 160 | if updateReq.Delete { 161 | s.deleteErrandByID(errand.ID) 162 | } else if updateReq.Status != "" { 163 | _, err = s.UpdateErrandByID(errand.ID, func(e *schemas.Errand) error { 164 | e.Status = schemas.Status(updateReq.Status) 165 | return nil 166 | }) 167 | if err != nil { 168 | break 169 | } 170 | } 171 | } 172 | 173 | if err != nil { 174 | c.JSON(http.StatusInternalServerError, gin.H{ 175 | "message": "Internal Server Error!", 176 | "error": err.Error(), 177 | }) 178 | 179 | return 180 | } 181 | 182 | c.JSON(http.StatusOK, gin.H{ 183 | "status": "OK", 184 | "count": len(errands), 185 | }) 186 | } 187 | 188 | //nolint:funlen // Only 1 line over 189 | func (s *ErrandsServer) processErrand(c *gin.Context) { 190 | var procErrand schemas.Errand 191 | 192 | errands := make([]schemas.Errand, 0) 193 | typeFilter := c.Param("type") 194 | 195 | for _, itemObj := range s.ErrandStore.Items() { 196 | item := itemObj.Object.(schemas.Errand) 197 | 198 | if item.Status != schemas.StatusInactive { 199 | continue 200 | } 201 | 202 | if item.Type != typeFilter { 203 | continue 204 | } 205 | // Add to list of errands we could possibly process: 206 | errands = append(errands, item) 207 | } 208 | 209 | if len(errands) == 0 { 210 | c.JSON(http.StatusNotFound, gin.H{ 211 | "message": "No jobs", 212 | }) 213 | 214 | return 215 | } 216 | 217 | // Of the possible errands to process, sort them by date & priority: 218 | sort.SliceStable(errands, func(i, j int) bool { 219 | return errands[i].Created < errands[j].Created 220 | }) 221 | sort.SliceStable(errands, func(i, j int) bool { 222 | return errands[i].Options.Priority > errands[j].Options.Priority 223 | }) 224 | 225 | procErrand = errands[0] 226 | 227 | updatedErrand, err := s.UpdateErrandByID(procErrand.ID, func(errand *schemas.Errand) error { 228 | errand.Started = utils.GetTimestamp() 229 | errand.Attempts++ 230 | errand.Status = schemas.StatusActive 231 | errand.Progress = 0.0 232 | 233 | _ = errand.AddToLogs("INFO", "Started!") 234 | return nil 235 | }) 236 | 237 | if err != nil { 238 | log.WithError(err).Warn("potentially no job found") 239 | 240 | c.JSON(http.StatusNotFound, gin.H{ 241 | "message": "No jobs", 242 | }) 243 | return 244 | } 245 | 246 | s.AddNotification("processing", updatedErrand) 247 | c.JSON(http.StatusOK, gin.H{ 248 | "status": "OK", 249 | "results": updatedErrand, 250 | }) 251 | } 252 | 253 | func (s *ErrandsServer) GetErrandsBy(fn func(*schemas.Errand) bool) []schemas.Errand { 254 | errands := make([]schemas.Errand, 0) 255 | 256 | for _, itemObj := range s.ErrandStore.Items() { 257 | errand := itemObj.Object.(schemas.Errand) 258 | if fn(&errand) { 259 | errands = append(errands, errand) 260 | } 261 | } 262 | 263 | return errands 264 | } 265 | 266 | func (s *ErrandsServer) clearErrands(c *gin.Context) { 267 | duration, err := time.ParseDuration(c.Param("duration")) 268 | if err != nil { 269 | c.JSON(http.StatusBadRequest, gin.H{ 270 | "message": "Invalid Duration", 271 | "error": err.Error(), 272 | }) 273 | 274 | return 275 | } 276 | 277 | threshold := time.Now().Add(-duration) 278 | errands := make([]schemas.Errand, 0) 279 | 280 | for _, itemObj := range s.ErrandStore.Items() { 281 | item := itemObj.Object.(schemas.Errand) 282 | 283 | var stoppedRunning int64 284 | 285 | switch item.Status { 286 | case "completed": 287 | stoppedRunning = item.Completed 288 | case "failed": 289 | stoppedRunning = item.Failed 290 | default: 291 | continue 292 | } 293 | 294 | if stoppedRunning >= threshold.UnixNano() { 295 | continue 296 | } 297 | 298 | errands = append(errands, item) 299 | } 300 | 301 | for _, errand := range errands { 302 | s.deleteErrandByID(errand.ID) 303 | } 304 | 305 | c.JSON(http.StatusOK, gin.H{ 306 | "status": "OK", 307 | "results": errands, 308 | }) 309 | } 310 | 311 | func ContainsStatus(slice []schemas.Status, status schemas.Status) bool { 312 | for _, s := range slice { 313 | if s == status { 314 | return true 315 | } 316 | } 317 | 318 | return false 319 | } 320 | -------------------------------------------------------------------------------- /errands-server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "path" 7 | "reflect" 8 | "time" 9 | 10 | "github.com/prometheus/client_golang/prometheus/promhttp" 11 | log "github.com/sirupsen/logrus" 12 | 13 | cors "github.com/gin-contrib/cors" 14 | gin "github.com/gin-gonic/gin" 15 | 16 | binding "github.com/gin-gonic/gin/binding" 17 | store "github.com/polygon-io/errands-server/memorydb" 18 | schemas "github.com/polygon-io/errands-server/schemas" 19 | validator "gopkg.in/go-playground/validator.v8" 20 | ) 21 | 22 | const ( 23 | errandsDBPathSuffix = "errands/" 24 | pipelinesDBPathSuffix = "pipelines/" 25 | ) 26 | 27 | //easyjson:json 28 | type Notification struct { 29 | Event string `json:"event"` 30 | Errand schemas.Errand `json:"errand,omitempty"` 31 | } 32 | 33 | type ErrandsServer struct { 34 | StorageDir string 35 | Port string 36 | ErrandStore *store.MemoryStore 37 | PipelineStore *store.MemoryStore 38 | Server *http.Server 39 | API *gin.Engine 40 | ErrandsRoutes *gin.RouterGroup 41 | ErrandRoutes *gin.RouterGroup 42 | Notifications chan *Notification 43 | StreamClients []*Client 44 | RegisterClient chan *Client 45 | UnregisterClient chan *Client 46 | periodicSave bool 47 | } 48 | 49 | func NewErrandsServer(cfg *Config) *ErrandsServer { 50 | obj := &ErrandsServer{ 51 | StorageDir: cfg.Storage, 52 | Port: cfg.Port, 53 | StreamClients: make([]*Client, 0), 54 | RegisterClient: make(chan *Client, 10), 55 | UnregisterClient: make(chan *Client, 10), 56 | Notifications: make(chan *Notification, 100), 57 | ErrandStore: store.New(), 58 | PipelineStore: store.New(), 59 | periodicSave: true, 60 | } 61 | 62 | go obj.createAPI() 63 | go obj.broadcastLoop() 64 | 65 | if err := obj.ErrandStore.LoadFile(path.Join(cfg.Storage, errandsDBPathSuffix)); err != nil { 66 | log.WithError(err).Error("unable to load errands db") 67 | log.Warning("Could not load errand data from previous DB file.") 68 | log.Warning("If this is your first time running, this is normal.") 69 | log.Warning("If not please check the contents of your file: ", cfg.Storage) 70 | } 71 | 72 | if err := obj.ErrandStore.LoadFile(path.Join(cfg.Storage, pipelinesDBPathSuffix)); err != nil { 73 | log.WithError(err).Error("unable to load pipelines db") 74 | log.Warning("Could not load pipeline data from previous DB file.") 75 | log.Warning("If this is your first time running, this is normal.") 76 | log.Warning("If not please check the contents of your file: ", cfg.Storage) 77 | } 78 | 79 | go obj.periodicallySaveDB() 80 | 81 | go obj.periodicallyCheckTTLs() 82 | 83 | return obj 84 | } 85 | 86 | func (s *ErrandsServer) periodicallySaveDB() { 87 | for { 88 | time.Sleep(60 * time.Second) 89 | 90 | if !s.periodicSave { 91 | return 92 | } 93 | 94 | log.Info("Checkpoint saving DB to file...") 95 | s.saveDBs() 96 | } 97 | } 98 | 99 | func (s *ErrandsServer) periodicallyCheckTTLs() { 100 | t := time.NewTicker(time.Minute) 101 | defer t.Stop() 102 | 103 | filter := func(errand *schemas.Errand) bool { 104 | if errand.Options.TTL <= 0 || errand.Status != schemas.StatusActive { 105 | return false 106 | } 107 | 108 | started := time.Unix(0, errand.Started*1_000_000) // ms to ns 109 | ttlDuration := time.Duration(errand.Options.TTL) * time.Minute // m to ns 110 | 111 | return time.Since(started) > ttlDuration 112 | } 113 | 114 | update := func(errand *schemas.Errand) error { 115 | if err := failErrand(errand, FailedRequest{Reason: "TTL Expired"}); err != nil { 116 | return fmt.Errorf("unable to fail errand: %s; %w", errand.ID, err) 117 | } 118 | 119 | s.AddNotification("failed", errand) 120 | return nil 121 | } 122 | 123 | for range t.C { 124 | if err := s.UpdateErrandsByFilter(filter, update); err != nil { 125 | log.WithError(err).Error("Unable to fail errand(s)") 126 | } 127 | } 128 | } 129 | 130 | func (s *ErrandsServer) saveDBs() { 131 | if err := s.ErrandStore.SaveFile(path.Join(s.StorageDir, errandsDBPathSuffix)); err != nil { 132 | log.Error("----- Error checkpoint saving the errand DB to file -----") 133 | log.Error(err) 134 | } 135 | 136 | if err := s.PipelineStore.SaveFile(path.Join(s.StorageDir, pipelinesDBPathSuffix)); err != nil { 137 | log.Error("----- Error checkpoint saving the pipeline DB to file -----") 138 | log.Error(err) 139 | } 140 | } 141 | 142 | func (s *ErrandsServer) AddNotification(event string, errand *schemas.Errand) { 143 | obj := &Notification{ 144 | Event: event, 145 | Errand: *errand, 146 | } 147 | s.Notifications <- obj 148 | } 149 | 150 | func (s *ErrandsServer) broadcastLoop() { 151 | for { 152 | select { 153 | case client := <-s.RegisterClient: 154 | s.StreamClients = append(s.StreamClients, client) 155 | case client := <-s.UnregisterClient: 156 | for i, c := range s.StreamClients { 157 | if c == client { 158 | s.StreamClients = append(s.StreamClients[:i], s.StreamClients[i+1:]...) 159 | } 160 | } 161 | case not := <-s.Notifications: 162 | for _, client := range s.StreamClients { 163 | notificationCopy := &Notification{} 164 | *notificationCopy = *not 165 | client.Notifications <- notificationCopy 166 | } 167 | } 168 | } 169 | } 170 | 171 | func (s *ErrandsServer) kill() { 172 | s.killAPI() 173 | 174 | for _, client := range s.StreamClients { 175 | client.Gone() 176 | } 177 | 178 | s.killDB() 179 | } 180 | 181 | func (s *ErrandsServer) killAPI() { 182 | log.Println("Closing the HTTP Server") 183 | 184 | _ = s.Server.Close() 185 | } 186 | 187 | func (s *ErrandsServer) killDB() { 188 | log.Println("Closing the DB") 189 | s.saveDBs() 190 | } 191 | 192 | func UserStructLevelValidation(v *validator.Validate, structLevel *validator.StructLevel) { 193 | errand := structLevel.CurrentStruct.Interface().(schemas.Errand) 194 | if errand.Options.TTL < 5 && errand.Options.TTL != 0 { 195 | structLevel.ReportError( 196 | reflect.ValueOf(errand.Options.TTL), "ttl", "ttl", "must be positive, and more than 5", 197 | ) 198 | } 199 | } 200 | 201 | //nolint:funlen // 8 Lines over, but this has all our routes 202 | func (s *ErrandsServer) createAPI() { 203 | s.API = gin.Default() 204 | 205 | CORSconfig := cors.DefaultConfig() 206 | CORSconfig.AllowCredentials = true 207 | CORSconfig.AllowOriginFunc = func(origin string) bool { 208 | return true 209 | } 210 | s.API.Use(cors.New(CORSconfig)) 211 | 212 | if v, ok := binding.Validator.Engine().(*validator.Validate); ok { 213 | v.RegisterStructValidation(UserStructLevelValidation, schemas.Errand{}) 214 | } 215 | 216 | // Singular errand Routes: 217 | s.ErrandRoutes = s.API.Group("/v1/errand") 218 | // Get an errand by id: 219 | s.ErrandRoutes.GET("/:id", s.getErrand) 220 | // Delete an errand by id: 221 | s.ErrandRoutes.DELETE("/:id", s.deleteErrand) 222 | // Update an errand by id: 223 | s.ErrandRoutes.PUT("/:id", s.updateErrand) 224 | s.ErrandRoutes.PUT("/:id/failed", s.failedErrand) 225 | s.ErrandRoutes.PUT("/:id/completed", s.completeErrand) 226 | s.ErrandRoutes.POST("/:id/log", s.logToErrand) 227 | s.ErrandRoutes.POST("/:id/retry", s.retryErrand) 228 | 229 | // Errands Routes 230 | s.ErrandsRoutes = s.API.Group("/v1/errands") 231 | // Create a new errand: 232 | s.ErrandsRoutes.POST("/", s.createErrand) 233 | // Get all errands: 234 | s.ErrandsRoutes.GET("/", s.getAllErrands) 235 | // Notifications: 236 | s.ErrandsRoutes.GET("/notifications", s.errandNotifications) 237 | // Ready to process an errand: 238 | s.ErrandsRoutes.POST("/process/:type", s.processErrand) 239 | // Get all errands in a current type or state: 240 | s.ErrandsRoutes.GET("/list/:key/:val", s.getFilteredErrands) 241 | // Update all errands in this state: 242 | s.ErrandsRoutes.POST("/update/:key/:val", s.updateFilteredErrands) 243 | // Clear all finished errands older than. 244 | s.ErrandsRoutes.POST("/clear/:duration", s.clearErrands) 245 | 246 | // Singular pipeline routes 247 | pipelineRoutes := s.API.Group("/v1/pipeline") 248 | pipelineRoutes.POST("/", s.createPipeline) 249 | pipelineRoutes.GET("/:id", s.getPipeline) 250 | pipelineRoutes.DELETE("/:id", s.deletePipeline) 251 | 252 | // Plural pipeline routes 253 | pipelinesRoutes := s.API.Group("/v1/pipelines") 254 | pipelinesRoutes.GET("/", s.listPipelines) 255 | 256 | // Prometheus metrics 257 | s.API.GET("/metrics", func(c *gin.Context) { 258 | promhttp.Handler().ServeHTTP(c.Writer, c.Request) 259 | }) 260 | 261 | s.Server = &http.Server{ 262 | Addr: s.Port, 263 | Handler: s.API, 264 | ReadHeaderTimeout: time.Second * 30, 265 | } 266 | 267 | log.Println("Starting server on port:", s.Port) 268 | 269 | if err := s.Server.ListenAndServe(); err != nil && err != http.ErrServerClosed { 270 | log.Fatalf("listen: %s\n", err) 271 | } 272 | } 273 | -------------------------------------------------------------------------------- /errands.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/polygon-io/errands-server/7cb3e6755a330e819c58af03bb3ff694d8faf6f8/errands.db -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: 611257d7204466869e78430a6fab8478dc479dcef4721262a9c16cb45e61710d 2 | updated: 2019-03-21T13:54:02.935666-04:00 3 | imports: 4 | - name: github.com/AndreasBriese/bbloom 5 | version: e2d15f34fcf99d5dbb871c820ec73f710fca9815 6 | - name: github.com/dgraph-io/badger 7 | version: 439fd464b155d419201a5c195c70d40618376776 8 | subpackages: 9 | - options 10 | - protos 11 | - skl 12 | - table 13 | - "y" 14 | - name: github.com/dgryski/go-farm 15 | version: 3adb47b1fb0f6d9efcc5051a6c62e2e413ac85a9 16 | - name: github.com/gin-contrib/cors 17 | version: cf4846e6a636a76237a28d9286f163c132e841bc 18 | - name: github.com/gin-contrib/gzip 19 | version: b07653c4b4b188bf0af293ae44b72e7ac9623a17 20 | - name: github.com/gin-contrib/sse 21 | version: 5545eab6dad3bbbd6c5ae9186383c2a9d23c0dae 22 | - name: github.com/gin-gonic/gin 23 | version: b869fe1415e4b9eb52f247441830d502aece2d4d 24 | subpackages: 25 | - binding 26 | - json 27 | - render 28 | - name: github.com/golang/protobuf 29 | version: d3c38a4eb4970272b87a425ae00ccc4548e2f9bb 30 | subpackages: 31 | - proto 32 | - name: github.com/google/uuid 33 | version: 0cd6bf5da1e1c83f8b45653022c74f71af0538a4 34 | - name: github.com/json-iterator/go 35 | version: 0ff49de124c6f76f8494e194af75bde0f1a49a29 36 | - name: github.com/kelseyhightower/envconfig 37 | version: f611eb38b3875cc3bd991ca91c51d06446afa14c 38 | - name: github.com/mailru/easyjson 39 | version: 1de009706dbeb9d05f18586f0735fcdb7c524481 40 | - name: github.com/mattn/go-isatty 41 | version: c2a7a6ca930a4cd0bc33a3f298eb71960732a3a7 42 | - name: github.com/modern-go/concurrent 43 | version: bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94 44 | - name: github.com/modern-go/reflect2 45 | version: 94122c33edd36123c84d5368cfb2b69df93a0ec8 46 | - name: github.com/pkg/errors 47 | version: 27936f6d90f9c8e1145f11ed52ffffbfdb9e0af7 48 | - name: github.com/ugorji/go 49 | version: 2dc34c0b87808deb8e2c69e1625db4c14e5d0d4a 50 | subpackages: 51 | - codec 52 | - name: golang.org/x/net 53 | version: 1272bf9dcd53ea65c09668fb4c76e65deb740072 54 | subpackages: 55 | - internal/timeseries 56 | - trace 57 | - name: golang.org/x/sys 58 | version: f7bb7a8bee54210937e93ec56d007d892c1f0580 59 | subpackages: 60 | - unix 61 | - name: gopkg.in/go-playground/validator.v8 62 | version: 5f1438d3fca68893a817e4a66806cea46a9e4ebf 63 | - name: gopkg.in/yaml.v2 64 | version: 51d6538a90f86fe93ac480b35f37b2be17fef232 65 | testImports: [] 66 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/polygon-io/errands-server 2 | import: 3 | - package: github.com/dgraph-io/badger 4 | version: ^2.0.0-rc1 5 | - package: github.com/gin-gonic/gin 6 | version: ^1.3.0 7 | - package: gopkg.in/go-playground/validator.v8 8 | version: ^8.18.2 9 | - package: github.com/google/uuid 10 | version: ^1.1.1 11 | - package: github.com/gin-contrib/cors 12 | version: ^1.2.0 13 | - package: github.com/gin-contrib/gzip 14 | version: ^0.0.1 15 | - package: github.com/kelseyhightower/envconfig 16 | version: ^1.3.0 17 | - package: github.com/mailru/easyjson 18 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/polygon-io/errands-server 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/gin-contrib/cors v0.0.0-20170318125340-cf4846e6a636 7 | github.com/gin-gonic/gin v1.6.3 8 | github.com/google/go-cmp v0.5.0 // indirect 9 | github.com/google/uuid v1.1.2 10 | github.com/kelseyhightower/envconfig v1.4.0 11 | github.com/kr/text v0.2.0 // indirect 12 | github.com/mailru/easyjson v0.7.1 13 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect 14 | github.com/patrickmn/go-cache v2.1.0+incompatible 15 | github.com/prometheus/client_golang v1.9.0 16 | github.com/sirupsen/logrus v1.6.0 17 | github.com/stretchr/testify v1.6.1 18 | github.com/ugorji/go v1.2.5 // indirect 19 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect 20 | gopkg.in/go-playground/assert.v1 v1.2.1 // indirect 21 | gopkg.in/go-playground/validator.v8 v8.18.2 22 | gopkg.in/yaml.v3 v3.0.0-20200601152816-913338de1bd2 // indirect 23 | ) 24 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 4 | github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= 5 | github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= 6 | github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= 7 | github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= 8 | github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= 9 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 10 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 11 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 12 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 13 | github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= 14 | github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= 15 | github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= 16 | github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= 17 | github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= 18 | github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= 19 | github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= 20 | github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= 21 | github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= 22 | github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= 23 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 24 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 25 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 26 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 27 | github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= 28 | github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= 29 | github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= 30 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 31 | github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= 32 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 33 | github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= 34 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 35 | github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= 36 | github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= 37 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 38 | github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 39 | github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= 40 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 41 | github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= 42 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 43 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 44 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 45 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 46 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 47 | github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 48 | github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= 49 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= 50 | github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= 51 | github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= 52 | github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= 53 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 54 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 55 | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 56 | github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= 57 | github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= 58 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 59 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 60 | github.com/gin-contrib/cors v0.0.0-20170318125340-cf4846e6a636 h1:oGgJA7DJphAc81EMHZ+2G7Ai2xyg5eoq7bbqzCsiWFc= 61 | github.com/gin-contrib/cors v0.0.0-20170318125340-cf4846e6a636/go.mod h1:cw+u9IsAkC16e42NtYYVCLsHYXE98nB3M7Dr9mLSeH4= 62 | github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= 63 | github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= 64 | github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= 65 | github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= 66 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 67 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 68 | github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= 69 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 70 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 71 | github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= 72 | github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= 73 | github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= 74 | github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= 75 | github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= 76 | github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= 77 | github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= 78 | github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= 79 | github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= 80 | github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= 81 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 82 | github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= 83 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 84 | github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 85 | github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= 86 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 87 | github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 88 | github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 89 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 90 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 91 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 92 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 93 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 94 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 95 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 96 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 97 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 98 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 99 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 100 | github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= 101 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 102 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 103 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 104 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 105 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 106 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 107 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 108 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 109 | github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= 110 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 111 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 112 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 113 | github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 114 | github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= 115 | github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 116 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 117 | github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= 118 | github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 119 | github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 120 | github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 121 | github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= 122 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= 123 | github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= 124 | github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= 125 | github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= 126 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 127 | github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= 128 | github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= 129 | github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= 130 | github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= 131 | github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= 132 | github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= 133 | github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= 134 | github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 135 | github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 136 | github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= 137 | github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= 138 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 139 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 140 | github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= 141 | github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= 142 | github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= 143 | github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= 144 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 145 | github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= 146 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 147 | github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= 148 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 149 | github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= 150 | github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= 151 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 152 | github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 153 | github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 154 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 155 | github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= 156 | github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 157 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 158 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 159 | github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= 160 | github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= 161 | github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= 162 | github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= 163 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 164 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 165 | github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= 166 | github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 167 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 168 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 169 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 170 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 171 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 172 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 173 | github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= 174 | github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= 175 | github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= 176 | github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= 177 | github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= 178 | github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= 179 | github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= 180 | github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= 181 | github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 182 | github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 183 | github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= 184 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 185 | github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= 186 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 187 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 188 | github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= 189 | github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= 190 | github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 191 | github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= 192 | github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= 193 | github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= 194 | github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 195 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 196 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 197 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 198 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 199 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 200 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 201 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 202 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 203 | github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 204 | github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= 205 | github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= 206 | github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= 207 | github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= 208 | github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= 209 | github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= 210 | github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= 211 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= 212 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= 213 | github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= 214 | github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= 215 | github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= 216 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 217 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 218 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 219 | github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= 220 | github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= 221 | github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= 222 | github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= 223 | github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= 224 | github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= 225 | github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= 226 | github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= 227 | github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= 228 | github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= 229 | github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= 230 | github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= 231 | github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= 232 | github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= 233 | github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= 234 | github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= 235 | github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= 236 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 237 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 238 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 239 | github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= 240 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 241 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 242 | github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= 243 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 244 | github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= 245 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 246 | github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= 247 | github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= 248 | github.com/prometheus/client_golang v1.9.0 h1:Rrch9mh17XcxvEu9D9DEpb4isxjGBtcevQjKvxPRQIU= 249 | github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= 250 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 251 | github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 252 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 253 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 254 | github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 255 | github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= 256 | github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 257 | github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 258 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 259 | github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= 260 | github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= 261 | github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM= 262 | github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= 263 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 264 | github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 265 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 266 | github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= 267 | github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= 268 | github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= 269 | github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= 270 | github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= 271 | github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= 272 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 273 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 274 | github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= 275 | github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= 276 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= 277 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 278 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 279 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 280 | github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= 281 | github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= 282 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 283 | github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= 284 | github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= 285 | github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= 286 | github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= 287 | github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 288 | github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= 289 | github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= 290 | github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= 291 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 292 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 293 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 294 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 295 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 296 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 297 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 298 | github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 299 | github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= 300 | github.com/ugorji/go v1.2.5 h1:NozRHfUeEta89taVkyfsDVSy2f7v89Frft4pjnWuGuc= 301 | github.com/ugorji/go v1.2.5/go.mod h1:gat2tIT8KJG8TVI8yv77nEO/KYT6dV7JE1gfUa8Xuls= 302 | github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= 303 | github.com/ugorji/go/codec v1.2.5 h1:8WobZKAk18Msm2CothY2jnztY56YVY8kF1oQrj21iis= 304 | github.com/ugorji/go/codec v1.2.5/go.mod h1:QPxoTbPKSEAlAHPYt02++xp/en9B/wUdwFCz+hj5caA= 305 | github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= 306 | github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= 307 | github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= 308 | go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= 309 | go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= 310 | go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= 311 | go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= 312 | go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 313 | go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 314 | go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= 315 | go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 316 | go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= 317 | go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= 318 | go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 319 | go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= 320 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 321 | golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 322 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 323 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 324 | golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 325 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 326 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 327 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 328 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 329 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 330 | golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 331 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 332 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 333 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 334 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 335 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 336 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 337 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 338 | golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 339 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 340 | golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 341 | golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 342 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 343 | golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 344 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 345 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 346 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 347 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 348 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 349 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 350 | golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 351 | golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 352 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 353 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 354 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 355 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 356 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 357 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 358 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 359 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 360 | golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 361 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 362 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 363 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 364 | golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 365 | golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 366 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 367 | golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 368 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 369 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 370 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 371 | golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 372 | golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 373 | golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 374 | golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 375 | golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 376 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 377 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 378 | golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 379 | golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 380 | golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e h1:AyodaIpKjppX+cBfTASF2E1US3H2JFBj920Ot3rtDjs= 381 | golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 382 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 383 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 384 | golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 385 | golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 386 | golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 387 | golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 388 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 389 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 390 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 391 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 392 | golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 393 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 394 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 395 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 396 | golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 397 | golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 398 | golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 399 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 400 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 401 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 402 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 403 | google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= 404 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 405 | google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 406 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 407 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 408 | google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 409 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 410 | google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= 411 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 412 | google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= 413 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 414 | google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= 415 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 416 | google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 417 | google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 418 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 419 | google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 420 | google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 421 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 422 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 423 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 424 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 425 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 426 | google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= 427 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 428 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 429 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 430 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 431 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 432 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= 433 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 434 | gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= 435 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 436 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 437 | gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= 438 | gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= 439 | gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= 440 | gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ= 441 | gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= 442 | gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= 443 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 444 | gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= 445 | gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= 446 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 447 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 448 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 449 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 450 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 451 | gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= 452 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 453 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 454 | gopkg.in/yaml.v3 v3.0.0-20200601152816-913338de1bd2 h1:VEmvx0P+GVTgkNu2EdTN988YCZPcD3lo9AoczZpucwc= 455 | gopkg.in/yaml.v3 v3.0.0-20200601152816-913338de1bd2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 456 | honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 457 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 458 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 459 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 460 | sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= 461 | sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= 462 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "os/signal" 6 | 7 | envconfig "github.com/kelseyhightower/envconfig" 8 | log "github.com/sirupsen/logrus" 9 | ) 10 | 11 | /* 12 | Config is 13 | ENVIRONMENT VARIABLES: 14 | ----------------------------- 15 | Set values via env variables, prefixed with ERRANDS_ 16 | eg: 17 | 18 | ERRANDS_PORT=:4545 - Will change the listening port to 4545 19 | ERRANDS_STORAGE="/errands/" - Will change the DB location to /errands/ 20 | */ 21 | type Config struct { 22 | Storage string `split_words:"true" default:"./errands.db"` 23 | Port string `split_words:"true" default:":5555"` 24 | } 25 | 26 | func main() { 27 | // Parse Env Vars: 28 | var cfg Config 29 | err := envconfig.Process("ERRANDS", &cfg) 30 | if err != nil { 31 | log.Fatal(err) 32 | } 33 | 34 | // trap SIGINT to trigger a shutdown. 35 | signals := make(chan os.Signal, 1) 36 | signal.Notify(signals, os.Interrupt) 37 | 38 | server := NewErrandsServer(&cfg) 39 | 40 | log.Info("listening for signals") 41 | 42 | <-signals 43 | log.Info("Exiting") 44 | server.kill() 45 | } 46 | -------------------------------------------------------------------------------- /main_easyjson.go: -------------------------------------------------------------------------------- 1 | // Code generated by easyjson for marshaling/unmarshaling. DO NOT EDIT. 2 | 3 | package main 4 | 5 | import ( 6 | json "encoding/json" 7 | easyjson "github.com/mailru/easyjson" 8 | jlexer "github.com/mailru/easyjson/jlexer" 9 | jwriter "github.com/mailru/easyjson/jwriter" 10 | ) 11 | 12 | // suppress unused package warning 13 | var ( 14 | _ *json.RawMessage 15 | _ *jlexer.Lexer 16 | _ *jwriter.Writer 17 | _ easyjson.Marshaler 18 | ) 19 | 20 | func easyjson89aae3efDecodeGithubComPolygonIoErrandsServer(in *jlexer.Lexer, out *Notification) { 21 | isTopLevel := in.IsStart() 22 | if in.IsNull() { 23 | if isTopLevel { 24 | in.Consumed() 25 | } 26 | in.Skip() 27 | return 28 | } 29 | in.Delim('{') 30 | for !in.IsDelim('}') { 31 | key := in.UnsafeString() 32 | in.WantColon() 33 | if in.IsNull() { 34 | in.Skip() 35 | in.WantComma() 36 | continue 37 | } 38 | switch key { 39 | case "event": 40 | out.Event = string(in.String()) 41 | case "errand": 42 | (out.Errand).UnmarshalEasyJSON(in) 43 | default: 44 | in.SkipRecursive() 45 | } 46 | in.WantComma() 47 | } 48 | in.Delim('}') 49 | if isTopLevel { 50 | in.Consumed() 51 | } 52 | } 53 | func easyjson89aae3efEncodeGithubComPolygonIoErrandsServer(out *jwriter.Writer, in Notification) { 54 | out.RawByte('{') 55 | first := true 56 | _ = first 57 | { 58 | const prefix string = ",\"event\":" 59 | if first { 60 | first = false 61 | out.RawString(prefix[1:]) 62 | } else { 63 | out.RawString(prefix) 64 | } 65 | out.String(string(in.Event)) 66 | } 67 | if true { 68 | const prefix string = ",\"errand\":" 69 | if first { 70 | first = false 71 | out.RawString(prefix[1:]) 72 | } else { 73 | out.RawString(prefix) 74 | } 75 | (in.Errand).MarshalEasyJSON(out) 76 | } 77 | out.RawByte('}') 78 | } 79 | 80 | // MarshalJSON supports json.Marshaler interface 81 | func (v Notification) MarshalJSON() ([]byte, error) { 82 | w := jwriter.Writer{} 83 | easyjson89aae3efEncodeGithubComPolygonIoErrandsServer(&w, v) 84 | return w.Buffer.BuildBytes(), w.Error 85 | } 86 | 87 | // MarshalEasyJSON supports easyjson.Marshaler interface 88 | func (v Notification) MarshalEasyJSON(w *jwriter.Writer) { 89 | easyjson89aae3efEncodeGithubComPolygonIoErrandsServer(w, v) 90 | } 91 | 92 | // UnmarshalJSON supports json.Unmarshaler interface 93 | func (v *Notification) UnmarshalJSON(data []byte) error { 94 | r := jlexer.Lexer{Data: data} 95 | easyjson89aae3efDecodeGithubComPolygonIoErrandsServer(&r, v) 96 | return r.Error() 97 | } 98 | 99 | // UnmarshalEasyJSON supports easyjson.Unmarshaler interface 100 | func (v *Notification) UnmarshalEasyJSON(l *jlexer.Lexer) { 101 | easyjson89aae3efDecodeGithubComPolygonIoErrandsServer(l, v) 102 | } 103 | -------------------------------------------------------------------------------- /memorydb/memorydb.go: -------------------------------------------------------------------------------- 1 | // Package memorydb provides a key value store. 2 | package memorydb 3 | 4 | import ( 5 | cache "github.com/patrickmn/go-cache" 6 | "github.com/sirupsen/logrus" 7 | ) 8 | 9 | /* 10 | 11 | In memory Key/Value store 12 | - Load from flat file 13 | - Store to flat file when exiting 14 | - Store to flat file periodically as backup 15 | 16 | Methods: 17 | - Create 18 | - Read/Get 19 | - Update 20 | - Delete 21 | 22 | */ 23 | 24 | type MemoryStore struct { 25 | *cache.Cache 26 | } 27 | 28 | func New() *MemoryStore { 29 | return &MemoryStore{ 30 | cache.New(cache.NoExpiration, 0), 31 | } 32 | } 33 | 34 | func LoadDBFrom(dbLocation string) error { 35 | logrus.Println("Loaded memory DB from file: ", dbLocation) 36 | return nil 37 | } 38 | -------------------------------------------------------------------------------- /metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import "github.com/prometheus/client_golang/prometheus" 4 | 5 | const ( 6 | labelErrandType = "errand_type" 7 | labelStatus = "status" 8 | ) 9 | 10 | var ( 11 | errandCompletedCounter *prometheus.CounterVec 12 | ) 13 | 14 | func init() { 15 | completedCounterOpts := prometheus.CounterOpts{ 16 | Namespace: "poly", 17 | Subsystem: "errands", 18 | Name: "completed_total", 19 | Help: "A counter of all the completed errands processed by this server", 20 | } 21 | errandCompletedCounter = prometheus.NewCounterVec(completedCounterOpts, []string{labelErrandType, labelStatus}) 22 | prometheus.MustRegister(errandCompletedCounter) 23 | } 24 | 25 | // ErrandCompleted increments the errand completed counter for a successfully completed errand. 26 | func ErrandCompleted(errandType string) { 27 | errandCompletedCounter.With(prometheus.Labels{labelErrandType: errandType, labelStatus: "completed"}).Inc() 28 | } 29 | 30 | // ErrandFailed increments the errand completed counter for an errand that failed (and is not going to retry anymore). 31 | func ErrandFailed(errandType string) { 32 | errandCompletedCounter.With(prometheus.Labels{labelErrandType: errandType, labelStatus: "failed"}).Inc() 33 | } 34 | -------------------------------------------------------------------------------- /misc/illustration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/polygon-io/errands-server/7cb3e6755a330e819c58af03bb3ff694d8faf6f8/misc/illustration.png -------------------------------------------------------------------------------- /pipeline-routes.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | "strconv" 6 | 7 | "github.com/gin-gonic/gin" 8 | "github.com/google/uuid" 9 | log "github.com/sirupsen/logrus" 10 | 11 | "github.com/polygon-io/errands-server/schemas" 12 | "github.com/polygon-io/errands-server/utils" 13 | ) 14 | 15 | const ( 16 | dryRunQueryParam = "dryRun" 17 | idParam = "id" 18 | statusFilterQueryParam = "status" 19 | ) 20 | 21 | func (s *ErrandsServer) createPipeline(c *gin.Context) { 22 | var pipeline schemas.Pipeline 23 | if err := c.ShouldBindJSON(&pipeline); err != nil { 24 | c.JSON(http.StatusBadRequest, gin.H{ 25 | "message": "pipeline validation failed", 26 | "error": err.Error(), 27 | }) 28 | 29 | return 30 | } 31 | 32 | if err := pipeline.Validate(); err != nil { 33 | c.JSON(http.StatusBadRequest, gin.H{ 34 | "message": "pipeline validation failed", 35 | "error": err.Error(), 36 | }) 37 | 38 | return 39 | } 40 | 41 | // If this was just a dry run, we're done 42 | dryRun, _ := strconv.ParseBool(c.Query(dryRunQueryParam)) 43 | if dryRun { 44 | c.JSON(http.StatusOK, gin.H{"message": "pipeline validated successfully"}) 45 | return 46 | } 47 | 48 | // Set the ID and status of the pipeline 49 | pipeline.ID = uuid.New().String() 50 | pipeline.Status = schemas.StatusInactive 51 | pipeline.StartedMillis = utils.GetTimestamp() 52 | 53 | // Initialize all the errands in the pipeline 54 | for _, errand := range pipeline.Errands { 55 | errand.SetDefaults() 56 | errand.PipelineID = pipeline.ID 57 | errand.Status = schemas.StatusBlocked 58 | 59 | if err := s.saveErrand(errand); err != nil { 60 | // This should never really happen 61 | log.WithError(err).Error("error saving errand") 62 | c.JSON(http.StatusInternalServerError, gin.H{"message": "internal store error"}) 63 | return 64 | } 65 | } 66 | 67 | // Kick off all the unblocked errands now 68 | for _, unblockedErrand := range pipeline.GetUnblockedErrands() { 69 | unblockedErrand.Status = schemas.StatusInactive 70 | s.ErrandStore.SetDefault(unblockedErrand.ID, *unblockedErrand) 71 | s.AddNotification("created", unblockedErrand) 72 | } 73 | 74 | s.PipelineStore.SetDefault(pipeline.ID, pipeline) 75 | 76 | c.JSON(http.StatusOK, gin.H{ 77 | "status": "OK", 78 | "results": pipeline, 79 | }) 80 | } 81 | 82 | func (s *ErrandsServer) deletePipeline(c *gin.Context) { 83 | pipelineID := c.Param(idParam) 84 | pipeline, exists := s.getPipelineFromStore(pipelineID) 85 | if !exists { 86 | c.JSON(http.StatusNotFound, gin.H{"status": "not_found"}) 87 | return 88 | } 89 | 90 | s.cascadeDeletePipeline(pipeline) 91 | c.JSON(http.StatusOK, gin.H{"status": "OK"}) 92 | } 93 | 94 | func (s *ErrandsServer) getPipeline(c *gin.Context) { 95 | pipelineID := c.Param(idParam) 96 | pipeline, exists := s.getPipelineFromStore(pipelineID) 97 | if !exists { 98 | c.JSON(http.StatusNotFound, gin.H{"status": "not_found"}) 99 | return 100 | } 101 | 102 | c.JSON(http.StatusOK, gin.H{ 103 | "status": "OK", 104 | "results": pipeline, 105 | }) 106 | } 107 | 108 | func (s *ErrandsServer) listPipelines(c *gin.Context) { 109 | statusFilter := c.Query(statusFilterQueryParam) 110 | filterFn := acceptAllPipelineFilter 111 | 112 | if statusFilter != "" { 113 | filterFn = statusPipelineFilter(schemas.Status(statusFilter)) 114 | } 115 | 116 | pipelines := s.getFilteredPipelinesFromStore(filterFn) 117 | 118 | // We only want to return an overview of each pipeline in the list API. 119 | // Strip out errands and dependencies from the pipelines to make the response smaller. 120 | // Users can get pipeline details by ID 121 | for i := range pipelines { 122 | pipelines[i].Errands = nil 123 | pipelines[i].Dependencies = nil 124 | } 125 | 126 | c.JSON(http.StatusOK, gin.H{ 127 | "status": "OK", 128 | "results": pipelines, 129 | }) 130 | } 131 | 132 | func (s *ErrandsServer) cascadeDeletePipeline(pipeline schemas.Pipeline) { 133 | // Delete all of the errands in the pipeline 134 | for _, errand := range pipeline.Errands { 135 | s.deleteErrandByID(errand.ID) 136 | } 137 | 138 | // Delete the pipeline itself 139 | s.PipelineStore.Delete(pipeline.ID) 140 | } 141 | 142 | func (s *ErrandsServer) getPipelineFromStore(pipelineID string) (schemas.Pipeline, bool) { 143 | pipeline, exists := s.PipelineStore.Get(pipelineID) 144 | if !exists { 145 | return schemas.Pipeline{}, false 146 | } 147 | 148 | return pipeline.(schemas.Pipeline), true 149 | } 150 | 151 | func (s *ErrandsServer) getFilteredPipelinesFromStore(filterFn func(pipeline schemas.Pipeline) bool) []schemas.Pipeline { 152 | var results []schemas.Pipeline 153 | 154 | for _, pipelineItem := range s.PipelineStore.Items() { 155 | pipeline := pipelineItem.Object.(schemas.Pipeline) 156 | if filterFn(pipeline) { 157 | results = append(results, pipeline) 158 | } 159 | } 160 | 161 | return results 162 | } 163 | 164 | func statusPipelineFilter(status schemas.Status) func(pipeline schemas.Pipeline) bool { 165 | return func(pipeline schemas.Pipeline) bool { 166 | return pipeline.Status == status 167 | } 168 | } 169 | 170 | func acceptAllPipelineFilter(pipeline schemas.Pipeline) bool { 171 | return true 172 | } 173 | 174 | func (s *ErrandsServer) updateErrandInPipeline(errand *schemas.Errand) { 175 | pipeline, exists := s.getPipelineFromStore(errand.PipelineID) 176 | if !exists { 177 | return 178 | } 179 | 180 | // Update the pipeline's internal representation of the errand 181 | for i, pipelineErrand := range pipeline.Errands { 182 | if errand.ID == pipelineErrand.ID { 183 | pipeline.Errands[i] = errand 184 | break 185 | } 186 | } 187 | 188 | // Check for any newly unblocked errands 189 | for _, unblockedErrand := range pipeline.GetUnblockedErrands() { 190 | // If this unblocked errand is already in progress, just continue 191 | if unblockedErrand.Status != schemas.StatusBlocked { 192 | continue 193 | } 194 | 195 | unblockedErrand.Status = schemas.StatusInactive 196 | if err := unblockedErrand.AddToLogs("INFO", "errand unblocked"); err != nil { 197 | // Log this but continue 198 | log.WithError(err).Error("unable to add to errand logs") 199 | } 200 | 201 | s.ErrandStore.SetDefault(unblockedErrand.ID, *unblockedErrand) 202 | } 203 | 204 | pipeline.RecalculateStatus() 205 | 206 | // If the pipeline just finished and is marked as deleteOnCompleted, delete it now 207 | if pipeline.Status == schemas.StatusCompleted && pipeline.DeleteOnCompleted { 208 | s.cascadeDeletePipeline(pipeline) 209 | } else { 210 | // Otherwise save the updated pipeline 211 | s.PipelineStore.SetDefault(pipeline.ID, pipeline) 212 | } 213 | } 214 | -------------------------------------------------------------------------------- /postman/Errands.postman_collection.json: -------------------------------------------------------------------------------- 1 | { 2 | "info": { 3 | "_postman_id": "8f651924-835c-43ec-bc36-f1293093a3da", 4 | "name": "Errands", 5 | "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" 6 | }, 7 | "item": [ 8 | { 9 | "name": "Create Errand", 10 | "event": [ 11 | { 12 | "listen": "test", 13 | "script": { 14 | "id": "9e1c8139-bc0c-4d7d-90c6-651d5fd0c035", 15 | "exec": [ 16 | "var jsonData = pm.response.json();", 17 | "pm.globals.set(\"errand_id\", jsonData.results.id);", 18 | "", 19 | "var schema = {", 20 | " type: \"object\",", 21 | " properties: {", 22 | " status: {", 23 | " type: \"string\"", 24 | " },", 25 | " results: {", 26 | " $ref: \"errand\",", 27 | " }", 28 | " }", 29 | "};", 30 | "pm.test(\"Status code is 200\", function (){ ", 31 | " pm.response.to.have.status(200);", 32 | "});", 33 | "pm.test('Schema is valid', function() {", 34 | " pm.expect(tv4.validate(jsonData, schema)).to.be.true;", 35 | "});", 36 | "pm.test(\"Errand status should be 'inactive'\", function (){ ", 37 | " pm.expect(jsonData.results.status).to.eql( 'inactive' );", 38 | "});" 39 | ], 40 | "type": "text/javascript" 41 | } 42 | } 43 | ], 44 | "request": { 45 | "method": "POST", 46 | "header": [ 47 | { 48 | "key": "Content-Type", 49 | "name": "Content-Type", 50 | "value": "application/json", 51 | "type": "text" 52 | } 53 | ], 54 | "body": { 55 | "mode": "raw", 56 | "raw": "{\n\t\"name\": \"Process: Prior 100\",\n\t\"type\": \"extract\",\n\t\"options\": {\n\t\t\"ttl\": 60,\n\t\t\"priority\": 500\n\t},\n\t\"data\": {\n\t\t\"file\":\"http://s3.domain.com/some/dir/file.ext\",\n\t\t\"user\": \"351351345151\",\n\t\t\"dbid\": \"899687969\"\n\t}\n}" 57 | }, 58 | "url": { 59 | "raw": "http://localhost:5555/v1/errands/", 60 | "protocol": "http", 61 | "host": [ 62 | "localhost" 63 | ], 64 | "port": "5555", 65 | "path": [ 66 | "v1", 67 | "errands", 68 | "" 69 | ] 70 | } 71 | }, 72 | "response": [] 73 | }, 74 | { 75 | "name": "Get Errands", 76 | "event": [ 77 | { 78 | "listen": "test", 79 | "script": { 80 | "id": "6c5b1ca0-0441-43c4-91c3-68f0e3fda5b8", 81 | "exec": [ 82 | "var schema = {", 83 | " type: \"object\",", 84 | " properties: {", 85 | " status: {", 86 | " type: \"string\"", 87 | " },", 88 | " results: {", 89 | " type: \"array\",", 90 | " items: {", 91 | " $ref: \"errand\",", 92 | " }", 93 | " }", 94 | " }", 95 | "};", 96 | "pm.test(\"Status code is 200\", function (){ ", 97 | " pm.response.to.have.status(200);", 98 | "});", 99 | "pm.test('Schema is valid', function() {", 100 | " pm.expect(tv4.validate(pm.response.json(), schema)).to.be.true;", 101 | "});" 102 | ], 103 | "type": "text/javascript" 104 | } 105 | } 106 | ], 107 | "request": { 108 | "method": "GET", 109 | "header": [], 110 | "body": { 111 | "mode": "raw", 112 | "raw": "" 113 | }, 114 | "url": { 115 | "raw": "http://localhost:5555/v1/errands", 116 | "protocol": "http", 117 | "host": [ 118 | "localhost" 119 | ], 120 | "port": "5555", 121 | "path": [ 122 | "v1", 123 | "errands" 124 | ] 125 | } 126 | }, 127 | "response": [] 128 | }, 129 | { 130 | "name": "Get Filtered Errands", 131 | "event": [ 132 | { 133 | "listen": "test", 134 | "script": { 135 | "id": "6c5b1ca0-0441-43c4-91c3-68f0e3fda5b8", 136 | "exec": [ 137 | "var schema = {", 138 | " type: \"object\",", 139 | " properties: {", 140 | " status: {", 141 | " type: \"string\"", 142 | " },", 143 | " results: {", 144 | " type: \"array\",", 145 | " items: {", 146 | " $ref: \"errand\",", 147 | " }", 148 | " }", 149 | " }", 150 | "};", 151 | "pm.test(\"Status code is 200\", function (){ ", 152 | " pm.response.to.have.status(200);", 153 | "});", 154 | "pm.test('Schema is valid', function() {", 155 | " pm.expect(tv4.validate(pm.response.json(), schema)).to.be.true;", 156 | "});" 157 | ], 158 | "type": "text/javascript" 159 | } 160 | } 161 | ], 162 | "request": { 163 | "method": "GET", 164 | "header": [], 165 | "body": { 166 | "mode": "raw", 167 | "raw": "" 168 | }, 169 | "url": { 170 | "raw": "http://localhost:5555/v1/errands/list/status/inactive", 171 | "protocol": "http", 172 | "host": [ 173 | "localhost" 174 | ], 175 | "port": "5555", 176 | "path": [ 177 | "v1", 178 | "errands", 179 | "list", 180 | "status", 181 | "inactive" 182 | ] 183 | } 184 | }, 185 | "response": [] 186 | }, 187 | { 188 | "name": "Process Errand - Start", 189 | "event": [ 190 | { 191 | "listen": "test", 192 | "script": { 193 | "id": "e79ba1c7-4b3b-4942-87a6-ba16fbf61aac", 194 | "exec": [ 195 | "var jsonData = pm.response.json();", 196 | "pm.globals.set(\"errand_id\", jsonData.results.id);", 197 | "", 198 | "var schema = {", 199 | " type: \"object\",", 200 | " properties: {", 201 | " status: {", 202 | " type: \"string\"", 203 | " },", 204 | " results: {", 205 | " $ref: \"errand\",", 206 | " }", 207 | " }", 208 | "};", 209 | "pm.test(\"Status code is 200\", function (){ ", 210 | " pm.response.to.have.status(200);", 211 | "});", 212 | "pm.test('Schema is valid', function() {", 213 | " pm.expect(tv4.validate(jsonData, schema)).to.be.true;", 214 | "});", 215 | "pm.test(\"Errand status should be 'active'\", function (){ ", 216 | " pm.expect(jsonData.results.status).to.eql( \"active\" );", 217 | "});" 218 | ], 219 | "type": "text/javascript" 220 | } 221 | } 222 | ], 223 | "request": { 224 | "method": "POST", 225 | "header": [], 226 | "body": { 227 | "mode": "raw", 228 | "raw": "" 229 | }, 230 | "url": { 231 | "raw": "http://localhost:5555/v1/errands/process?type=extract", 232 | "protocol": "http", 233 | "host": [ 234 | "localhost" 235 | ], 236 | "port": "5555", 237 | "path": [ 238 | "v1", 239 | "errands", 240 | "process" 241 | ], 242 | "query": [ 243 | { 244 | "key": "type", 245 | "value": "extract" 246 | } 247 | ] 248 | } 249 | }, 250 | "response": [] 251 | }, 252 | { 253 | "name": "Log to Errand", 254 | "event": [ 255 | { 256 | "listen": "test", 257 | "script": { 258 | "id": "e898eaac-a354-4b4f-8acb-af6695cdfb3a", 259 | "exec": [ 260 | "var schema = {", 261 | " type: \"object\",", 262 | " properties: {", 263 | " status: {", 264 | " type: \"string\"", 265 | " },", 266 | " results: {", 267 | " $ref: \"errand\",", 268 | " }", 269 | " }", 270 | "};", 271 | "pm.test(\"Status code is 200\", function (){ ", 272 | " pm.response.to.have.status(200);", 273 | "});", 274 | "pm.test('Schema is valid', function() {", 275 | " pm.expect(tv4.validate(pm.response.json(), schema)).to.be.true;", 276 | "});" 277 | ], 278 | "type": "text/javascript" 279 | } 280 | } 281 | ], 282 | "request": { 283 | "method": "POST", 284 | "header": [ 285 | { 286 | "key": "Content-Type", 287 | "name": "Content-Type", 288 | "value": "application/json", 289 | "type": "text" 290 | } 291 | ], 292 | "body": { 293 | "mode": "raw", 294 | "raw": "{\n\t\"severity\": \"INFO\",\n\t\"message\": \"This is an update to the log of this errand..\"\n}" 295 | }, 296 | "url": { 297 | "raw": "http://localhost:5555/v1/errand/{{errand_id}}/log", 298 | "protocol": "http", 299 | "host": [ 300 | "localhost" 301 | ], 302 | "port": "5555", 303 | "path": [ 304 | "v1", 305 | "errand", 306 | "{{errand_id}}", 307 | "log" 308 | ] 309 | } 310 | }, 311 | "response": [] 312 | }, 313 | { 314 | "name": "Update Errand", 315 | "event": [ 316 | { 317 | "listen": "test", 318 | "script": { 319 | "id": "e243f842-ec20-43f2-a7a9-a568b8251109", 320 | "exec": [ 321 | "var jsonData = pm.response.json();", 322 | "var schema = {", 323 | " type: \"object\",", 324 | " properties: {", 325 | " status: {", 326 | " type: \"string\"", 327 | " },", 328 | " results: {", 329 | " $ref: \"errand\",", 330 | " }", 331 | " }", 332 | "};", 333 | "pm.test(\"Status code is 200\", function (){ ", 334 | " pm.response.to.have.status(200);", 335 | "});", 336 | "pm.test('Schema is valid', function() {", 337 | " pm.expect(tv4.validate(jsonData, schema)).to.be.true;", 338 | "});", 339 | "pm.test(\"Errand progress should be 98.789\", function (){ ", 340 | " pm.expect(jsonData.results.progress).to.eql( 98.789 );", 341 | "});" 342 | ], 343 | "type": "text/javascript" 344 | } 345 | } 346 | ], 347 | "request": { 348 | "method": "PUT", 349 | "header": [ 350 | { 351 | "key": "Content-Type", 352 | "name": "Content-Type", 353 | "value": "application/json", 354 | "type": "text" 355 | } 356 | ], 357 | "body": { 358 | "mode": "raw", 359 | "raw": "{\n\t\"progress\": 98.789\n}" 360 | }, 361 | "url": { 362 | "raw": "http://localhost:5555/v1/errand/{{errand_id}}", 363 | "protocol": "http", 364 | "host": [ 365 | "localhost" 366 | ], 367 | "port": "5555", 368 | "path": [ 369 | "v1", 370 | "errand", 371 | "{{errand_id}}" 372 | ] 373 | } 374 | }, 375 | "response": [] 376 | }, 377 | { 378 | "name": "Fail Errand", 379 | "event": [ 380 | { 381 | "listen": "test", 382 | "script": { 383 | "id": "f62e5f49-85b3-4e70-86ac-7569e949b5a3", 384 | "exec": [ 385 | "var jsonData = pm.response.json();", 386 | "var schema = {", 387 | " type: \"object\",", 388 | " properties: {", 389 | " status: {", 390 | " type: \"string\"", 391 | " },", 392 | " results: {", 393 | " $ref: \"errand\",", 394 | " }", 395 | " }", 396 | "};", 397 | "pm.test(\"Status code is 200\", function (){ ", 398 | " pm.response.to.have.status(200);", 399 | "});", 400 | "pm.test('Schema is valid', function() {", 401 | " pm.expect(tv4.validate(jsonData, schema)).to.be.true;", 402 | "});", 403 | "pm.test(\"Errand Status should be 'failed'\", function (){ ", 404 | " pm.expect(jsonData.results.status).to.eql(\"failed\");", 405 | "});", 406 | "pm.test(\"Errand progress should be 0\", function (){ ", 407 | " pm.expect(jsonData.results.progress).to.eql( 0 );", 408 | "});" 409 | ], 410 | "type": "text/javascript" 411 | } 412 | } 413 | ], 414 | "request": { 415 | "method": "PUT", 416 | "header": [ 417 | { 418 | "key": "Content-Type", 419 | "name": "Content-Type", 420 | "type": "text", 421 | "value": "application/json" 422 | } 423 | ], 424 | "body": { 425 | "mode": "raw", 426 | "raw": "{\n\t\"reason\": \"Error processing this errand. Processing failed at line 101:42\"\n}" 427 | }, 428 | "url": { 429 | "raw": "http://localhost:5555/v1/errand/{{errand_id}}/failed", 430 | "protocol": "http", 431 | "host": [ 432 | "localhost" 433 | ], 434 | "port": "5555", 435 | "path": [ 436 | "v1", 437 | "errand", 438 | "{{errand_id}}", 439 | "failed" 440 | ] 441 | } 442 | }, 443 | "response": [] 444 | }, 445 | { 446 | "name": "Complete Errand", 447 | "event": [ 448 | { 449 | "listen": "test", 450 | "script": { 451 | "id": "ad665b0f-0c50-4996-bb31-2178d9eeaeb0", 452 | "exec": [ 453 | "var jsonData = pm.response.json();", 454 | "var schema = {", 455 | " type: \"object\",", 456 | " properties: {", 457 | " status: {", 458 | " type: \"string\"", 459 | " },", 460 | " results: {", 461 | " $ref: \"errand\",", 462 | " }", 463 | " }", 464 | "};", 465 | "pm.test(\"Status code is 200\", function (){ ", 466 | " pm.response.to.have.status(200);", 467 | "});", 468 | "pm.test('Schema is valid', function() {", 469 | " pm.expect(tv4.validate(jsonData, schema)).to.be.true;", 470 | "});", 471 | "pm.test(\"Errand Status should be 'completed'\", function (){ ", 472 | " pm.expect(jsonData.results.status).to.eql(\"completed\");", 473 | "});", 474 | "pm.test(\"Errand progress should be 0\", function (){ ", 475 | " pm.expect(jsonData.results.progress).to.eql( 100 );", 476 | "});" 477 | ], 478 | "type": "text/javascript" 479 | } 480 | } 481 | ], 482 | "request": { 483 | "method": "PUT", 484 | "header": [], 485 | "body": { 486 | "mode": "raw", 487 | "raw": "" 488 | }, 489 | "url": { 490 | "raw": "http://localhost:5555/v1/errand/{{errand_id}}/completed", 491 | "protocol": "http", 492 | "host": [ 493 | "localhost" 494 | ], 495 | "port": "5555", 496 | "path": [ 497 | "v1", 498 | "errand", 499 | "{{errand_id}}", 500 | "completed" 501 | ] 502 | } 503 | }, 504 | "response": [] 505 | }, 506 | { 507 | "name": "Retry Errand", 508 | "event": [ 509 | { 510 | "listen": "test", 511 | "script": { 512 | "id": "eb808d6e-06e8-4968-b9b2-e86c7915b299", 513 | "exec": [ 514 | "var jsonData = pm.response.json();", 515 | "var schema = {", 516 | " type: \"object\",", 517 | " properties: {", 518 | " status: {", 519 | " type: \"string\"", 520 | " },", 521 | " results: {", 522 | " $ref: \"errand\",", 523 | " }", 524 | " }", 525 | "};", 526 | "pm.test(\"Status code is 200\", function (){ ", 527 | " pm.response.to.have.status(200);", 528 | "});", 529 | "pm.test('Schema is valid', function() {", 530 | " pm.expect(tv4.validate(jsonData, schema)).to.be.true;", 531 | "});", 532 | "pm.test(\"Errand Status should be 'inactive'\", function (){ ", 533 | " pm.expect(jsonData.results.status).to.eql(\"inactive\");", 534 | "});", 535 | "pm.test(\"Errand progress should be 0\", function (){ ", 536 | " pm.expect(jsonData.results.progress).to.eql( 0 );", 537 | "});" 538 | ], 539 | "type": "text/javascript" 540 | } 541 | } 542 | ], 543 | "request": { 544 | "method": "POST", 545 | "header": [], 546 | "body": { 547 | "mode": "raw", 548 | "raw": "" 549 | }, 550 | "url": { 551 | "raw": "http://localhost:5555/v1/errand/{{errand_id}}/retry", 552 | "protocol": "http", 553 | "host": [ 554 | "localhost" 555 | ], 556 | "port": "5555", 557 | "path": [ 558 | "v1", 559 | "errand", 560 | "{{errand_id}}", 561 | "retry" 562 | ] 563 | } 564 | }, 565 | "response": [] 566 | }, 567 | { 568 | "name": "Delete Errand", 569 | "event": [ 570 | { 571 | "listen": "test", 572 | "script": { 573 | "id": "eb78c88a-144b-47ef-a34e-11f1a52816ad", 574 | "exec": [ 575 | "pm.test(\"Status code is 200\", function (){ ", 576 | " pm.response.to.have.status(200);", 577 | "});", 578 | "" 579 | ], 580 | "type": "text/javascript" 581 | } 582 | } 583 | ], 584 | "request": { 585 | "method": "DELETE", 586 | "header": [], 587 | "body": { 588 | "mode": "raw", 589 | "raw": "" 590 | }, 591 | "url": { 592 | "raw": "http://localhost:5555/v1/errand/{{errand_id}}", 593 | "protocol": "http", 594 | "host": [ 595 | "localhost" 596 | ], 597 | "port": "5555", 598 | "path": [ 599 | "v1", 600 | "errand", 601 | "{{errand_id}}" 602 | ] 603 | } 604 | }, 605 | "response": [] 606 | }, 607 | { 608 | "name": "Update All Errands ( filtered )", 609 | "event": [ 610 | { 611 | "listen": "test", 612 | "script": { 613 | "id": "e243f842-ec20-43f2-a7a9-a568b8251109", 614 | "exec": [ 615 | "var jsonData = pm.response.json();", 616 | "var schema = {", 617 | " type: \"object\",", 618 | " properties: {", 619 | " status: {", 620 | " type: \"string\"", 621 | " },", 622 | " count: \"number\"", 623 | " }", 624 | "};", 625 | "pm.test(\"Status code is 200\", function (){ ", 626 | " pm.response.to.have.status(200);", 627 | "});", 628 | "pm.test('Schema is valid', function() {", 629 | " pm.expect(tv4.validate(jsonData, schema)).to.be.true;", 630 | "});" 631 | ], 632 | "type": "text/javascript" 633 | } 634 | } 635 | ], 636 | "request": { 637 | "method": "POST", 638 | "header": [ 639 | { 640 | "key": "Content-Type", 641 | "name": "Content-Type", 642 | "type": "text", 643 | "value": "application/json" 644 | } 645 | ], 646 | "body": { 647 | "mode": "raw", 648 | "raw": "{\n\t\"status\": \"inactive\"\n}" 649 | }, 650 | "url": { 651 | "raw": "http://localhost:5555/v1/errands/update/status/active", 652 | "protocol": "http", 653 | "host": [ 654 | "localhost" 655 | ], 656 | "port": "5555", 657 | "path": [ 658 | "v1", 659 | "errands", 660 | "update", 661 | "status", 662 | "active" 663 | ] 664 | } 665 | }, 666 | "response": [] 667 | } 668 | ], 669 | "event": [ 670 | { 671 | "listen": "prerequest", 672 | "script": { 673 | "id": "f9876e9f-99ad-43cd-bc4b-32f20c813d34", 674 | "type": "text/javascript", 675 | "exec": [ 676 | "tv4.addSchema('errand', {", 677 | " type: \"object\",", 678 | " properties: {", 679 | " id: \"string\",", 680 | " name: \"string\",", 681 | " type: \"string\",", 682 | " options: {", 683 | " type: \"object\",", 684 | " properties: {", 685 | " ttl: \"number\",", 686 | " retries: \"number\",", 687 | " priority: \"number\",", 688 | " deleteOnCompleted: \"bool\"", 689 | " }", 690 | " },", 691 | " created: \"number\",", 692 | " status: { type: \"string\", enum:[\"inactive\", \"active\", \"failed\", \"completed\"] },", 693 | " progress: \"number\",", 694 | " attempts: \"number\",", 695 | " started: \"number\",", 696 | " failed: \"number\",", 697 | " logs: {", 698 | " type: \"array\",", 699 | " items: {", 700 | " type: \"object\",", 701 | " properties: {", 702 | " severity: {", 703 | " type: \"string\",", 704 | " enum: [\"INFO\", \"WARNING\", \"ERROR\"],", 705 | " },", 706 | " message: \"string\",", 707 | " timestamp: \"number\"", 708 | " }", 709 | " }", 710 | " }", 711 | " }", 712 | " });" 713 | ] 714 | } 715 | }, 716 | { 717 | "listen": "test", 718 | "script": { 719 | "id": "7e524ec1-0588-4bd0-bdd4-3a4bbd262595", 720 | "type": "text/javascript", 721 | "exec": [ 722 | "" 723 | ] 724 | } 725 | } 726 | ], 727 | "variable": [ 728 | { 729 | "id": "0d6d657d-4625-47e6-adcf-16aab5e3d107", 730 | "key": "errand", 731 | "value": "ERRAND_ID", 732 | "type": "string" 733 | } 734 | ] 735 | } 736 | -------------------------------------------------------------------------------- /schemas/errand.go: -------------------------------------------------------------------------------- 1 | // Package schemas provides model schemas for the errand server. 2 | package schemas 3 | 4 | import ( 5 | "fmt" 6 | 7 | "github.com/google/uuid" 8 | 9 | "github.com/polygon-io/errands-server/utils" 10 | ) 11 | 12 | type Status string 13 | 14 | // All the possible statuses for errands or pipelines. 15 | const ( 16 | StatusBlocked Status = "blocked" 17 | StatusInactive Status = "inactive" 18 | StatusActive Status = "active" 19 | StatusFailed Status = "failed" 20 | StatusCompleted Status = "completed" 21 | ) 22 | 23 | // ErrandStatuses is a slice of all valid statuses. 24 | // 25 | //nolint:gochecknoglobals // These are readonly. 26 | var ErrandStatuses = []Status{StatusBlocked, StatusInactive, StatusActive, StatusFailed, StatusCompleted} 27 | 28 | //easyjson:json 29 | type Errand struct { 30 | 31 | // General Attributes: 32 | ID string `json:"id"` 33 | Name string `json:"name" binding:"required"` 34 | Type string `json:"type" binding:"required"` 35 | Options ErrandOptions `json:"options"` 36 | Data map[string]interface{} `json:"data,omitempty"` 37 | Created int64 `json:"created"` 38 | Status Status `json:"status,omitempty"` 39 | Results map[string]interface{} `json:"results,omitempty"` 40 | 41 | // Internal attributes: 42 | Progress float64 `json:"progress"` 43 | Attempts int `json:"attempts"` 44 | Started int64 `json:"started,omitempty"` // Timestamp of last Start 45 | Failed int64 `json:"failed,omitempty"` // Timestamp of last Fail 46 | Completed int64 `json:"completed,omitempty"` // Timestamp of last Fail 47 | Logs []Log `json:"logs,omitempty"` 48 | 49 | // PipelineID is the ID of the pipeline that this errand belongs to (if any) 50 | PipelineID string `json:"pipeline,omitempty"` 51 | } 52 | 53 | // ErrandOptions holds various options tied to an errand. 54 | // 55 | //easyjson:json 56 | type ErrandOptions struct { 57 | // TTL is measured in minutes. 58 | TTL int `json:"ttl,omitempty"` 59 | Retries int `json:"retries,omitempty"` 60 | Priority int `json:"priority,omitempty"` 61 | DeleteOnCompleted bool `json:"deleteOnCompleted,omitempty"` 62 | } 63 | 64 | //easyjson:json 65 | type Log struct { 66 | Severity string `json:"severity" binding:"required"` 67 | Message string `json:"message" binding:"required"` 68 | Timestamp int64 `json:"timestamp"` 69 | } 70 | 71 | func NewErrand() *Errand { 72 | obj := &Errand{} 73 | obj.SetDefaults() 74 | 75 | return obj 76 | } 77 | 78 | func (e *Errand) SetDefaults() { 79 | uid := uuid.New() 80 | 81 | uidText, err := uid.MarshalText() 82 | if err != nil { 83 | panic(err) 84 | } 85 | 86 | e.ID = string(uidText) 87 | e.Status = "inactive" 88 | e.Created = utils.GetTimestamp() 89 | e.Logs = make([]Log, 0) 90 | } 91 | 92 | func (e *Errand) AddToLogs(severity, message string) error { 93 | validSeverities := []string{"INFO", "WARNING", "ERROR"} 94 | if !utils.Contains(validSeverities, severity) { 95 | return fmt.Errorf("invalid log severity: %s", severity) 96 | } 97 | 98 | obj := Log{ 99 | Severity: severity, 100 | Message: message, 101 | Timestamp: utils.GetTimestamp(), 102 | } 103 | e.Logs = append(e.Logs, obj) 104 | 105 | return nil 106 | } 107 | -------------------------------------------------------------------------------- /schemas/glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/polygon-io/errands-server/schemas 2 | import: 3 | # - package: github.com/gin-gonic/gin 4 | # version: ^1.3.0 5 | - package: github.com/google/uuid 6 | version: ^1.1.1 7 | - package: github.com/mailru/easyjson 8 | subpackages: 9 | - jlexer 10 | - jwriter 11 | - package: github.com/polygon-io/errands-server 12 | version: ^1.0.0 13 | subpackages: 14 | - utils 15 | -------------------------------------------------------------------------------- /schemas/pipeline.go: -------------------------------------------------------------------------------- 1 | package schemas 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | "github.com/polygon-io/errands-server/utils" 8 | ) 9 | 10 | // Pipeline represents an errand pipeline which consists of errands and dependencies between them. 11 | // Cyclic dependencies are not allowed. 12 | // 13 | //easyjson:json 14 | type Pipeline struct { 15 | Name string `json:"name" binding:"required"` 16 | DeleteOnCompleted bool `json:"deleteOnCompleted,omitempty"` 17 | 18 | Errands []*Errand `json:"errands,omitempty" binding:"required"` 19 | Dependencies []*PipelineDependency `json:"dependencies,omitempty"` 20 | 21 | // Attributes added by errands server 22 | ID string `json:"id"` 23 | Status Status `json:"status,omitempty"` 24 | 25 | // TODO: use ptime once that's public 26 | StartedMillis int64 `json:"startedMillis"` 27 | EndedMillis int64 `json:"endedMillis,omitempty"` 28 | } 29 | 30 | // PipelineDependency describes a dependency between errands within a pipeline. 31 | // 32 | //easyjson:json 33 | type PipelineDependency struct { 34 | // Target is the name of the errand within the pipeline that this dependency relates to 35 | Target string `json:"target" binding:"required"` 36 | 37 | // DependsOn is the name of the errand within the pipeline that the Target errand depends on 38 | DependsOn string `json:"dependsOn" binding:"required"` 39 | } 40 | 41 | type dependencyGraph struct { 42 | // errands maps errand names to the errands themselves for quick look-ups 43 | errands map[string]*Errand 44 | 45 | // dependencyToDependents maps an errand to a slice of errands that depend on it. 46 | // ie dependencyToDependents["A"] = []{"B"} -> "B" depends on "A" 47 | dependencyToDependents map[string][]*Errand 48 | 49 | // dependentToDependencies maps an errand to a slice of errands that it depends on. 50 | // This is the transpose of dependencyToDependents. 51 | // ie dependentToDependencies["B"] = []{"A"} -> "B" depends on "A" 52 | dependentToDependencies map[string][]*Errand 53 | } 54 | 55 | // Validate checks that the pipeline describes a valid dependency graph and returns a user-friendly error if the pipeline is invalid. 56 | func (p *Pipeline) Validate() error { 57 | if len(p.Errands) == 0 { 58 | return errors.New("no errands specified in pipeline") 59 | } 60 | 61 | // Map of errand name to errand schema 62 | errandMap := make(map[string]*Errand, len(p.Errands)) 63 | 64 | // Build up map of errand name to errand, and double check there are no errands with duplicate names 65 | for _, errand := range p.Errands { 66 | if _, exists := errandMap[errand.Name]; exists { 67 | return fmt.Errorf("duplicate name in errands list: %s", errand.Name) 68 | } 69 | 70 | errandMap[errand.Name] = errand 71 | } 72 | 73 | // Make sure dependencies all reference valid errands in the pipeline 74 | for _, dep := range p.Dependencies { 75 | if dep.Target == dep.DependsOn { 76 | return fmt.Errorf("errand cannot depend on itself: %s", dep.Target) 77 | } 78 | 79 | if _, exists := errandMap[dep.Target]; !exists { 80 | return fmt.Errorf("dependency references unknown errand name: %s", dep.Target) 81 | } 82 | 83 | if _, exists := errandMap[dep.DependsOn]; !exists { 84 | return fmt.Errorf("dependency references unknown errand name: %s", dep.DependsOn) 85 | } 86 | } 87 | 88 | // Check for dependency cycles 89 | return p.buildDependencyGraph().checkForDependencyCycles() 90 | } 91 | 92 | func (p *Pipeline) GetUnblockedErrands() []*Errand { 93 | return p.buildDependencyGraph().findUnblockedErrands() 94 | } 95 | 96 | func (p *Pipeline) RecalculateStatus() { 97 | var numCompleted int 98 | 99 | for _, errand := range p.Errands { 100 | // Failed takes precedence over everything 101 | if errand.Status == StatusFailed { 102 | p.Status = StatusFailed 103 | break 104 | } 105 | 106 | if p.Status == StatusInactive && errand.Status == StatusActive { 107 | p.Status = StatusActive 108 | } 109 | 110 | if errand.Status == StatusCompleted { 111 | numCompleted++ 112 | } 113 | } 114 | 115 | // If all the errands are completed, the pipeline is complete 116 | if numCompleted == len(p.Errands) { 117 | p.Status = StatusCompleted 118 | 119 | // Update the ended timestamp if it wasn't set already 120 | if p.EndedMillis == 0 { 121 | p.EndedMillis = utils.GetTimestamp() 122 | } 123 | } 124 | } 125 | 126 | // buildDependencyGraph constructs a dependencyGraph for this pipeline. 127 | // This function assumes the errands and dependencies have been validated already. 128 | func (p *Pipeline) buildDependencyGraph() dependencyGraph { 129 | g := dependencyGraph{ 130 | errands: make(map[string]*Errand, len(p.Errands)), 131 | dependencyToDependents: make(map[string][]*Errand, len(p.Errands)), 132 | dependentToDependencies: make(map[string][]*Errand, len(p.Errands)), 133 | } 134 | 135 | // Build up the errand map and initialize the graph with all errands and no dependencies 136 | for _, errand := range p.Errands { 137 | g.errands[errand.Name] = errand 138 | g.dependencyToDependents[errand.Name] = []*Errand{} 139 | g.dependentToDependencies[errand.Name] = []*Errand{} 140 | } 141 | 142 | // Fill in the dependencies 143 | for _, dep := range p.Dependencies { 144 | g.dependencyToDependents[dep.DependsOn] = append(g.dependencyToDependents[dep.DependsOn], g.errands[dep.Target]) 145 | g.dependentToDependencies[dep.Target] = append(g.dependentToDependencies[dep.Target], g.errands[dep.DependsOn]) 146 | } 147 | 148 | return g 149 | } 150 | 151 | // findUnblockedErrands returns a slice of errands that have no dependencies blocking them. 152 | func (g dependencyGraph) findUnblockedErrands() []*Errand { 153 | var independentErrands []*Errand 154 | 155 | for dependent, dependencies := range g.dependentToDependencies { 156 | var blocked bool 157 | 158 | // This dependent is blocked if any of its dependencies are not completed 159 | for _, dependency := range dependencies { 160 | if dependency.Status != StatusCompleted { 161 | blocked = true 162 | break 163 | } 164 | } 165 | 166 | if !blocked { 167 | independentErrands = append(independentErrands, g.errands[dependent]) 168 | } 169 | } 170 | 171 | return independentErrands 172 | } 173 | 174 | // checkForDependencyCycles returns an error if it finds a cyclic dependency in the dependencyGraph. 175 | // It determines cycles by doing the following: 176 | // For each node N, perform a depth first traversal of G starting at N. 177 | // If we ever encounter N again during the traversal, we've detected a cycle. 178 | func (g dependencyGraph) checkForDependencyCycles() error { 179 | independentErrands := g.findUnblockedErrands() 180 | if len(independentErrands) == 0 { 181 | return fmt.Errorf("dependency cycle found; all errands have dependencies") 182 | } 183 | 184 | for _, currentHomeErrand := range g.errands { 185 | toVisitStack := []*Errand{currentHomeErrand} 186 | currentTreeVisitedSet := make(map[string]struct{}, len(g.dependencyToDependents)) 187 | 188 | // While we have nodes to visit, keep traversing 189 | for len(toVisitStack) > 0 { 190 | topOfStackIndex := len(toVisitStack) - 1 191 | errand := toVisitStack[topOfStackIndex] 192 | toVisitStack = toVisitStack[:topOfStackIndex] // Pop off the last value from the stack 193 | 194 | // If we've seen this node before, we may have detected a cycle. 195 | if _, exists := currentTreeVisitedSet[errand.Name]; exists { 196 | // If the errand we've seen before is the currentHomeErrand then we know for certain 197 | // that we found a cycle. 198 | if errand.Name == currentHomeErrand.Name { 199 | return fmt.Errorf("dependency cycle found involving '%s'", errand.Name) 200 | } 201 | 202 | // If we've already visited this node, but it's not the start node, we _might_ have 203 | // found a cycle, but we can't be sure until we traverse starting from that node. 204 | // Until then, don't continue to add dependencies from this duplicate node to the visit stack. 205 | } else { 206 | // If we haven't seen this node before, Add add it to the visited set 207 | currentTreeVisitedSet[errand.Name] = struct{}{} 208 | 209 | // Add all of this errand's dependencies to the visit stack 210 | toVisitStack = append(toVisitStack, g.dependencyToDependents[errand.Name]...) 211 | } 212 | } 213 | } 214 | 215 | return nil 216 | } 217 | -------------------------------------------------------------------------------- /schemas/pipeline_test.go: -------------------------------------------------------------------------------- 1 | package schemas 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestPipelineValidate(t *testing.T) { 10 | t.Run("no errands", func(t *testing.T) { 11 | p := Pipeline{Name: "no errands!"} 12 | assert.Error(t, p.Validate()) 13 | }) 14 | 15 | t.Run("errands with duplicate name", func(t *testing.T) { 16 | p := Pipeline{ 17 | Name: "duplicate name", 18 | Errands: []*Errand{{ 19 | Name: "errand", 20 | }, { 21 | Name: "errand", 22 | }}, 23 | } 24 | 25 | assert.Error(t, p.Validate()) 26 | }) 27 | 28 | t.Run("errand with self dependency", func(t *testing.T) { 29 | p := Pipeline{ 30 | Name: "self dependency", 31 | Errands: []*Errand{{Name: "single errand"}}, 32 | Dependencies: []*PipelineDependency{{ 33 | Target: "single errand", 34 | DependsOn: "single errand", 35 | }}, 36 | } 37 | 38 | assert.Error(t, p.Validate()) 39 | }) 40 | 41 | t.Run("dependency with invalid target", func(t *testing.T) { 42 | p := Pipeline{ 43 | Name: "self dependency", 44 | Errands: []*Errand{{Name: "single errand"}}, 45 | Dependencies: []*PipelineDependency{{ 46 | Target: "not a real errand", 47 | DependsOn: "single errand", 48 | }}, 49 | } 50 | 51 | assert.Error(t, p.Validate()) 52 | }) 53 | 54 | t.Run("dependency with invalid dependsOn", func(t *testing.T) { 55 | p := Pipeline{ 56 | Name: "self dependency", 57 | Errands: []*Errand{{Name: "single errand"}}, 58 | Dependencies: []*PipelineDependency{{ 59 | Target: "single errand", 60 | DependsOn: "not a real dependency", 61 | }}, 62 | } 63 | 64 | assert.Error(t, p.Validate()) 65 | }) 66 | 67 | t.Run("simple dependency cycle | 2 errands", func(t *testing.T) { 68 | /* 69 | A <--> B 70 | */ 71 | p := Pipeline{ 72 | Name: "simple dependency cycle", 73 | Errands: []*Errand{ 74 | {Name: "A"}, 75 | {Name: "B"}, 76 | }, 77 | Dependencies: []*PipelineDependency{ 78 | {Target: "A", DependsOn: "B"}, 79 | {Target: "B", DependsOn: "A"}, 80 | }, 81 | } 82 | 83 | assert.Error(t, p.Validate()) 84 | }) 85 | 86 | t.Run("strongly connected subgraph cycle", func(t *testing.T) { 87 | /* 88 | A <--> B C 89 | */ 90 | p := Pipeline{ 91 | Name: "strongly connected cycle", 92 | Errands: []*Errand{ 93 | {Name: "A"}, 94 | {Name: "B"}, 95 | {Name: "C"}, 96 | }, 97 | Dependencies: []*PipelineDependency{ 98 | {Target: "A", DependsOn: "B"}, 99 | {Target: "B", DependsOn: "A"}, 100 | }, 101 | } 102 | 103 | assert.Error(t, p.Validate()) 104 | }) 105 | 106 | t.Run("single graph with cycle", func(t *testing.T) { 107 | /* 108 | A <--> B <-- C 109 | */ 110 | p := Pipeline{ 111 | Name: "single graph with cycle", 112 | Errands: []*Errand{ 113 | {Name: "A"}, 114 | {Name: "B"}, 115 | {Name: "C"}, 116 | }, 117 | Dependencies: []*PipelineDependency{ 118 | {Target: "A", DependsOn: "B"}, 119 | {Target: "B", DependsOn: "C"}, 120 | {Target: "B", DependsOn: "A"}, 121 | }, 122 | } 123 | 124 | assert.Error(t, p.Validate()) 125 | }) 126 | 127 | t.Run("multiple sub-graphs one with cycle", func(t *testing.T) { 128 | /* 129 | A <--> B <-- C D --> E --> F 130 | */ 131 | p := Pipeline{ 132 | Name: "strongly connected cycle", 133 | Errands: []*Errand{ 134 | {Name: "A"}, 135 | {Name: "B"}, 136 | {Name: "C"}, 137 | {Name: "D"}, 138 | {Name: "E"}, 139 | {Name: "F"}, 140 | }, 141 | Dependencies: []*PipelineDependency{ 142 | {Target: "A", DependsOn: "B"}, 143 | {Target: "B", DependsOn: "C"}, 144 | {Target: "B", DependsOn: "A"}, 145 | 146 | {Target: "E", DependsOn: "D"}, 147 | {Target: "F", DependsOn: "E"}, 148 | }, 149 | } 150 | 151 | assert.Error(t, p.Validate()) 152 | }) 153 | 154 | t.Run("multiple sub-graphs happy path", func(t *testing.T) { 155 | /* 156 | A --> B --> C D --> E --> F 157 | */ 158 | p := Pipeline{ 159 | Name: "strongly connected cycle", 160 | Errands: []*Errand{ 161 | {Name: "A"}, 162 | {Name: "B"}, 163 | {Name: "C"}, 164 | {Name: "D"}, 165 | {Name: "E"}, 166 | {Name: "F"}, 167 | }, 168 | Dependencies: []*PipelineDependency{ 169 | {Target: "A", DependsOn: "B"}, 170 | {Target: "B", DependsOn: "C"}, 171 | 172 | {Target: "E", DependsOn: "D"}, 173 | {Target: "F", DependsOn: "E"}, 174 | }, 175 | } 176 | 177 | assert.NoError(t, p.Validate()) 178 | }) 179 | 180 | t.Run("single graph happy path | diverging", func(t *testing.T) { 181 | /* 182 | |--> D 183 | A --| 184 | |--> B --> C 185 | */ 186 | p := Pipeline{ 187 | Name: "single graph with cycle", 188 | Errands: []*Errand{ 189 | {Name: "A"}, 190 | {Name: "B"}, 191 | {Name: "C"}, 192 | {Name: "D"}, 193 | }, 194 | Dependencies: []*PipelineDependency{ 195 | {Target: "B", DependsOn: "A"}, 196 | {Target: "C", DependsOn: "B"}, 197 | {Target: "D", DependsOn: "A"}, 198 | }, 199 | } 200 | 201 | assert.NoError(t, p.Validate()) 202 | }) 203 | 204 | t.Run("single graph happy path | converging", func(t *testing.T) { 205 | /* 206 | A --> B --| 207 | |--> C --> E 208 | D --| 209 | */ 210 | p := Pipeline{ 211 | Name: "single graph with cycle", 212 | Errands: []*Errand{ 213 | {Name: "A"}, 214 | {Name: "B"}, 215 | {Name: "C"}, 216 | {Name: "D"}, 217 | {Name: "E"}, 218 | }, 219 | Dependencies: []*PipelineDependency{ 220 | {Target: "B", DependsOn: "A"}, 221 | {Target: "C", DependsOn: "B"}, 222 | {Target: "C", DependsOn: "D"}, 223 | {Target: "E", DependsOn: "C"}, 224 | }, 225 | } 226 | 227 | assert.NoError(t, p.Validate()) 228 | }) 229 | 230 | t.Run("single graph happy path | converging and diverging", func(t *testing.T) { 231 | /* 232 | |--> B --| |--> E 233 | A -->| |--> C --| 234 | |--> D --| |--> F --> G 235 | */ 236 | p := Pipeline{ 237 | Name: "single graph with cycle", 238 | Errands: []*Errand{ 239 | {Name: "A"}, 240 | {Name: "B"}, 241 | {Name: "C"}, 242 | {Name: "D"}, 243 | {Name: "E"}, 244 | {Name: "F"}, 245 | {Name: "G"}, 246 | }, 247 | Dependencies: []*PipelineDependency{ 248 | {Target: "D", DependsOn: "A"}, 249 | {Target: "B", DependsOn: "A"}, 250 | {Target: "C", DependsOn: "B"}, 251 | {Target: "C", DependsOn: "D"}, 252 | {Target: "E", DependsOn: "C"}, 253 | {Target: "F", DependsOn: "C"}, 254 | {Target: "G", DependsOn: "F"}, 255 | }, 256 | } 257 | 258 | assert.NoError(t, p.Validate()) 259 | }) 260 | } 261 | -------------------------------------------------------------------------------- /schemas/schemas_easyjson.go: -------------------------------------------------------------------------------- 1 | // Code generated by easyjson for marshaling/unmarshaling. DO NOT EDIT. 2 | 3 | package schemas 4 | 5 | import ( 6 | json "encoding/json" 7 | easyjson "github.com/mailru/easyjson" 8 | jlexer "github.com/mailru/easyjson/jlexer" 9 | jwriter "github.com/mailru/easyjson/jwriter" 10 | ) 11 | 12 | // suppress unused package warning 13 | var ( 14 | _ *json.RawMessage 15 | _ *jlexer.Lexer 16 | _ *jwriter.Writer 17 | _ easyjson.Marshaler 18 | ) 19 | 20 | func easyjson2189435aDecodeGithubComPolygonIoErrandsServerSchemas(in *jlexer.Lexer, out *PipelineDependency) { 21 | isTopLevel := in.IsStart() 22 | if in.IsNull() { 23 | if isTopLevel { 24 | in.Consumed() 25 | } 26 | in.Skip() 27 | return 28 | } 29 | in.Delim('{') 30 | for !in.IsDelim('}') { 31 | key := in.UnsafeString() 32 | in.WantColon() 33 | if in.IsNull() { 34 | in.Skip() 35 | in.WantComma() 36 | continue 37 | } 38 | switch key { 39 | case "target": 40 | out.Target = string(in.String()) 41 | case "dependsOn": 42 | out.DependsOn = string(in.String()) 43 | default: 44 | in.SkipRecursive() 45 | } 46 | in.WantComma() 47 | } 48 | in.Delim('}') 49 | if isTopLevel { 50 | in.Consumed() 51 | } 52 | } 53 | func easyjson2189435aEncodeGithubComPolygonIoErrandsServerSchemas(out *jwriter.Writer, in PipelineDependency) { 54 | out.RawByte('{') 55 | first := true 56 | _ = first 57 | { 58 | const prefix string = ",\"target\":" 59 | out.RawString(prefix[1:]) 60 | out.String(string(in.Target)) 61 | } 62 | { 63 | const prefix string = ",\"dependsOn\":" 64 | out.RawString(prefix) 65 | out.String(string(in.DependsOn)) 66 | } 67 | out.RawByte('}') 68 | } 69 | 70 | // MarshalJSON supports json.Marshaler interface 71 | func (v PipelineDependency) MarshalJSON() ([]byte, error) { 72 | w := jwriter.Writer{} 73 | easyjson2189435aEncodeGithubComPolygonIoErrandsServerSchemas(&w, v) 74 | return w.Buffer.BuildBytes(), w.Error 75 | } 76 | 77 | // MarshalEasyJSON supports easyjson.Marshaler interface 78 | func (v PipelineDependency) MarshalEasyJSON(w *jwriter.Writer) { 79 | easyjson2189435aEncodeGithubComPolygonIoErrandsServerSchemas(w, v) 80 | } 81 | 82 | // UnmarshalJSON supports json.Unmarshaler interface 83 | func (v *PipelineDependency) UnmarshalJSON(data []byte) error { 84 | r := jlexer.Lexer{Data: data} 85 | easyjson2189435aDecodeGithubComPolygonIoErrandsServerSchemas(&r, v) 86 | return r.Error() 87 | } 88 | 89 | // UnmarshalEasyJSON supports easyjson.Unmarshaler interface 90 | func (v *PipelineDependency) UnmarshalEasyJSON(l *jlexer.Lexer) { 91 | easyjson2189435aDecodeGithubComPolygonIoErrandsServerSchemas(l, v) 92 | } 93 | func easyjson2189435aDecodeGithubComPolygonIoErrandsServerSchemas1(in *jlexer.Lexer, out *Pipeline) { 94 | isTopLevel := in.IsStart() 95 | if in.IsNull() { 96 | if isTopLevel { 97 | in.Consumed() 98 | } 99 | in.Skip() 100 | return 101 | } 102 | in.Delim('{') 103 | for !in.IsDelim('}') { 104 | key := in.UnsafeString() 105 | in.WantColon() 106 | if in.IsNull() { 107 | in.Skip() 108 | in.WantComma() 109 | continue 110 | } 111 | switch key { 112 | case "name": 113 | out.Name = string(in.String()) 114 | case "deleteOnCompleted": 115 | out.DeleteOnCompleted = bool(in.Bool()) 116 | case "errands": 117 | if in.IsNull() { 118 | in.Skip() 119 | out.Errands = nil 120 | } else { 121 | in.Delim('[') 122 | if out.Errands == nil { 123 | if !in.IsDelim(']') { 124 | out.Errands = make([]*Errand, 0, 8) 125 | } else { 126 | out.Errands = []*Errand{} 127 | } 128 | } else { 129 | out.Errands = (out.Errands)[:0] 130 | } 131 | for !in.IsDelim(']') { 132 | var v1 *Errand 133 | if in.IsNull() { 134 | in.Skip() 135 | v1 = nil 136 | } else { 137 | if v1 == nil { 138 | v1 = new(Errand) 139 | } 140 | (*v1).UnmarshalEasyJSON(in) 141 | } 142 | out.Errands = append(out.Errands, v1) 143 | in.WantComma() 144 | } 145 | in.Delim(']') 146 | } 147 | case "dependencies": 148 | if in.IsNull() { 149 | in.Skip() 150 | out.Dependencies = nil 151 | } else { 152 | in.Delim('[') 153 | if out.Dependencies == nil { 154 | if !in.IsDelim(']') { 155 | out.Dependencies = make([]*PipelineDependency, 0, 8) 156 | } else { 157 | out.Dependencies = []*PipelineDependency{} 158 | } 159 | } else { 160 | out.Dependencies = (out.Dependencies)[:0] 161 | } 162 | for !in.IsDelim(']') { 163 | var v2 *PipelineDependency 164 | if in.IsNull() { 165 | in.Skip() 166 | v2 = nil 167 | } else { 168 | if v2 == nil { 169 | v2 = new(PipelineDependency) 170 | } 171 | (*v2).UnmarshalEasyJSON(in) 172 | } 173 | out.Dependencies = append(out.Dependencies, v2) 174 | in.WantComma() 175 | } 176 | in.Delim(']') 177 | } 178 | case "id": 179 | out.ID = string(in.String()) 180 | case "status": 181 | out.Status = Status(in.String()) 182 | case "startedMillis": 183 | out.StartedMillis = int64(in.Int64()) 184 | case "endedMillis": 185 | out.EndedMillis = int64(in.Int64()) 186 | default: 187 | in.SkipRecursive() 188 | } 189 | in.WantComma() 190 | } 191 | in.Delim('}') 192 | if isTopLevel { 193 | in.Consumed() 194 | } 195 | } 196 | func easyjson2189435aEncodeGithubComPolygonIoErrandsServerSchemas1(out *jwriter.Writer, in Pipeline) { 197 | out.RawByte('{') 198 | first := true 199 | _ = first 200 | { 201 | const prefix string = ",\"name\":" 202 | out.RawString(prefix[1:]) 203 | out.String(string(in.Name)) 204 | } 205 | if in.DeleteOnCompleted { 206 | const prefix string = ",\"deleteOnCompleted\":" 207 | out.RawString(prefix) 208 | out.Bool(bool(in.DeleteOnCompleted)) 209 | } 210 | if len(in.Errands) != 0 { 211 | const prefix string = ",\"errands\":" 212 | out.RawString(prefix) 213 | { 214 | out.RawByte('[') 215 | for v3, v4 := range in.Errands { 216 | if v3 > 0 { 217 | out.RawByte(',') 218 | } 219 | if v4 == nil { 220 | out.RawString("null") 221 | } else { 222 | (*v4).MarshalEasyJSON(out) 223 | } 224 | } 225 | out.RawByte(']') 226 | } 227 | } 228 | if len(in.Dependencies) != 0 { 229 | const prefix string = ",\"dependencies\":" 230 | out.RawString(prefix) 231 | { 232 | out.RawByte('[') 233 | for v5, v6 := range in.Dependencies { 234 | if v5 > 0 { 235 | out.RawByte(',') 236 | } 237 | if v6 == nil { 238 | out.RawString("null") 239 | } else { 240 | (*v6).MarshalEasyJSON(out) 241 | } 242 | } 243 | out.RawByte(']') 244 | } 245 | } 246 | { 247 | const prefix string = ",\"id\":" 248 | out.RawString(prefix) 249 | out.String(string(in.ID)) 250 | } 251 | if in.Status != "" { 252 | const prefix string = ",\"status\":" 253 | out.RawString(prefix) 254 | out.String(string(in.Status)) 255 | } 256 | { 257 | const prefix string = ",\"startedMillis\":" 258 | out.RawString(prefix) 259 | out.Int64(int64(in.StartedMillis)) 260 | } 261 | if in.EndedMillis != 0 { 262 | const prefix string = ",\"endedMillis\":" 263 | out.RawString(prefix) 264 | out.Int64(int64(in.EndedMillis)) 265 | } 266 | out.RawByte('}') 267 | } 268 | 269 | // MarshalJSON supports json.Marshaler interface 270 | func (v Pipeline) MarshalJSON() ([]byte, error) { 271 | w := jwriter.Writer{} 272 | easyjson2189435aEncodeGithubComPolygonIoErrandsServerSchemas1(&w, v) 273 | return w.Buffer.BuildBytes(), w.Error 274 | } 275 | 276 | // MarshalEasyJSON supports easyjson.Marshaler interface 277 | func (v Pipeline) MarshalEasyJSON(w *jwriter.Writer) { 278 | easyjson2189435aEncodeGithubComPolygonIoErrandsServerSchemas1(w, v) 279 | } 280 | 281 | // UnmarshalJSON supports json.Unmarshaler interface 282 | func (v *Pipeline) UnmarshalJSON(data []byte) error { 283 | r := jlexer.Lexer{Data: data} 284 | easyjson2189435aDecodeGithubComPolygonIoErrandsServerSchemas1(&r, v) 285 | return r.Error() 286 | } 287 | 288 | // UnmarshalEasyJSON supports easyjson.Unmarshaler interface 289 | func (v *Pipeline) UnmarshalEasyJSON(l *jlexer.Lexer) { 290 | easyjson2189435aDecodeGithubComPolygonIoErrandsServerSchemas1(l, v) 291 | } 292 | func easyjson2189435aDecodeGithubComPolygonIoErrandsServerSchemas2(in *jlexer.Lexer, out *Log) { 293 | isTopLevel := in.IsStart() 294 | if in.IsNull() { 295 | if isTopLevel { 296 | in.Consumed() 297 | } 298 | in.Skip() 299 | return 300 | } 301 | in.Delim('{') 302 | for !in.IsDelim('}') { 303 | key := in.UnsafeString() 304 | in.WantColon() 305 | if in.IsNull() { 306 | in.Skip() 307 | in.WantComma() 308 | continue 309 | } 310 | switch key { 311 | case "severity": 312 | out.Severity = string(in.String()) 313 | case "message": 314 | out.Message = string(in.String()) 315 | case "timestamp": 316 | out.Timestamp = int64(in.Int64()) 317 | default: 318 | in.SkipRecursive() 319 | } 320 | in.WantComma() 321 | } 322 | in.Delim('}') 323 | if isTopLevel { 324 | in.Consumed() 325 | } 326 | } 327 | func easyjson2189435aEncodeGithubComPolygonIoErrandsServerSchemas2(out *jwriter.Writer, in Log) { 328 | out.RawByte('{') 329 | first := true 330 | _ = first 331 | { 332 | const prefix string = ",\"severity\":" 333 | out.RawString(prefix[1:]) 334 | out.String(string(in.Severity)) 335 | } 336 | { 337 | const prefix string = ",\"message\":" 338 | out.RawString(prefix) 339 | out.String(string(in.Message)) 340 | } 341 | { 342 | const prefix string = ",\"timestamp\":" 343 | out.RawString(prefix) 344 | out.Int64(int64(in.Timestamp)) 345 | } 346 | out.RawByte('}') 347 | } 348 | 349 | // MarshalJSON supports json.Marshaler interface 350 | func (v Log) MarshalJSON() ([]byte, error) { 351 | w := jwriter.Writer{} 352 | easyjson2189435aEncodeGithubComPolygonIoErrandsServerSchemas2(&w, v) 353 | return w.Buffer.BuildBytes(), w.Error 354 | } 355 | 356 | // MarshalEasyJSON supports easyjson.Marshaler interface 357 | func (v Log) MarshalEasyJSON(w *jwriter.Writer) { 358 | easyjson2189435aEncodeGithubComPolygonIoErrandsServerSchemas2(w, v) 359 | } 360 | 361 | // UnmarshalJSON supports json.Unmarshaler interface 362 | func (v *Log) UnmarshalJSON(data []byte) error { 363 | r := jlexer.Lexer{Data: data} 364 | easyjson2189435aDecodeGithubComPolygonIoErrandsServerSchemas2(&r, v) 365 | return r.Error() 366 | } 367 | 368 | // UnmarshalEasyJSON supports easyjson.Unmarshaler interface 369 | func (v *Log) UnmarshalEasyJSON(l *jlexer.Lexer) { 370 | easyjson2189435aDecodeGithubComPolygonIoErrandsServerSchemas2(l, v) 371 | } 372 | func easyjson2189435aDecodeGithubComPolygonIoErrandsServerSchemas3(in *jlexer.Lexer, out *ErrandOptions) { 373 | isTopLevel := in.IsStart() 374 | if in.IsNull() { 375 | if isTopLevel { 376 | in.Consumed() 377 | } 378 | in.Skip() 379 | return 380 | } 381 | in.Delim('{') 382 | for !in.IsDelim('}') { 383 | key := in.UnsafeString() 384 | in.WantColon() 385 | if in.IsNull() { 386 | in.Skip() 387 | in.WantComma() 388 | continue 389 | } 390 | switch key { 391 | case "ttl": 392 | out.TTL = int(in.Int()) 393 | case "retries": 394 | out.Retries = int(in.Int()) 395 | case "priority": 396 | out.Priority = int(in.Int()) 397 | case "deleteOnCompleted": 398 | out.DeleteOnCompleted = bool(in.Bool()) 399 | default: 400 | in.SkipRecursive() 401 | } 402 | in.WantComma() 403 | } 404 | in.Delim('}') 405 | if isTopLevel { 406 | in.Consumed() 407 | } 408 | } 409 | func easyjson2189435aEncodeGithubComPolygonIoErrandsServerSchemas3(out *jwriter.Writer, in ErrandOptions) { 410 | out.RawByte('{') 411 | first := true 412 | _ = first 413 | if in.TTL != 0 { 414 | const prefix string = ",\"ttl\":" 415 | first = false 416 | out.RawString(prefix[1:]) 417 | out.Int(int(in.TTL)) 418 | } 419 | if in.Retries != 0 { 420 | const prefix string = ",\"retries\":" 421 | if first { 422 | first = false 423 | out.RawString(prefix[1:]) 424 | } else { 425 | out.RawString(prefix) 426 | } 427 | out.Int(int(in.Retries)) 428 | } 429 | if in.Priority != 0 { 430 | const prefix string = ",\"priority\":" 431 | if first { 432 | first = false 433 | out.RawString(prefix[1:]) 434 | } else { 435 | out.RawString(prefix) 436 | } 437 | out.Int(int(in.Priority)) 438 | } 439 | if in.DeleteOnCompleted { 440 | const prefix string = ",\"deleteOnCompleted\":" 441 | if first { 442 | first = false 443 | out.RawString(prefix[1:]) 444 | } else { 445 | out.RawString(prefix) 446 | } 447 | out.Bool(bool(in.DeleteOnCompleted)) 448 | } 449 | out.RawByte('}') 450 | } 451 | 452 | // MarshalJSON supports json.Marshaler interface 453 | func (v ErrandOptions) MarshalJSON() ([]byte, error) { 454 | w := jwriter.Writer{} 455 | easyjson2189435aEncodeGithubComPolygonIoErrandsServerSchemas3(&w, v) 456 | return w.Buffer.BuildBytes(), w.Error 457 | } 458 | 459 | // MarshalEasyJSON supports easyjson.Marshaler interface 460 | func (v ErrandOptions) MarshalEasyJSON(w *jwriter.Writer) { 461 | easyjson2189435aEncodeGithubComPolygonIoErrandsServerSchemas3(w, v) 462 | } 463 | 464 | // UnmarshalJSON supports json.Unmarshaler interface 465 | func (v *ErrandOptions) UnmarshalJSON(data []byte) error { 466 | r := jlexer.Lexer{Data: data} 467 | easyjson2189435aDecodeGithubComPolygonIoErrandsServerSchemas3(&r, v) 468 | return r.Error() 469 | } 470 | 471 | // UnmarshalEasyJSON supports easyjson.Unmarshaler interface 472 | func (v *ErrandOptions) UnmarshalEasyJSON(l *jlexer.Lexer) { 473 | easyjson2189435aDecodeGithubComPolygonIoErrandsServerSchemas3(l, v) 474 | } 475 | func easyjson2189435aDecodeGithubComPolygonIoErrandsServerSchemas4(in *jlexer.Lexer, out *Errand) { 476 | isTopLevel := in.IsStart() 477 | if in.IsNull() { 478 | if isTopLevel { 479 | in.Consumed() 480 | } 481 | in.Skip() 482 | return 483 | } 484 | in.Delim('{') 485 | for !in.IsDelim('}') { 486 | key := in.UnsafeString() 487 | in.WantColon() 488 | if in.IsNull() { 489 | in.Skip() 490 | in.WantComma() 491 | continue 492 | } 493 | switch key { 494 | case "id": 495 | out.ID = string(in.String()) 496 | case "name": 497 | out.Name = string(in.String()) 498 | case "type": 499 | out.Type = string(in.String()) 500 | case "options": 501 | (out.Options).UnmarshalEasyJSON(in) 502 | case "data": 503 | if in.IsNull() { 504 | in.Skip() 505 | } else { 506 | in.Delim('{') 507 | if !in.IsDelim('}') { 508 | out.Data = make(map[string]interface{}) 509 | } else { 510 | out.Data = nil 511 | } 512 | for !in.IsDelim('}') { 513 | key := string(in.String()) 514 | in.WantColon() 515 | var v7 interface{} 516 | if m, ok := v7.(easyjson.Unmarshaler); ok { 517 | m.UnmarshalEasyJSON(in) 518 | } else if m, ok := v7.(json.Unmarshaler); ok { 519 | _ = m.UnmarshalJSON(in.Raw()) 520 | } else { 521 | v7 = in.Interface() 522 | } 523 | (out.Data)[key] = v7 524 | in.WantComma() 525 | } 526 | in.Delim('}') 527 | } 528 | case "created": 529 | out.Created = int64(in.Int64()) 530 | case "status": 531 | out.Status = Status(in.String()) 532 | case "results": 533 | if in.IsNull() { 534 | in.Skip() 535 | } else { 536 | in.Delim('{') 537 | if !in.IsDelim('}') { 538 | out.Results = make(map[string]interface{}) 539 | } else { 540 | out.Results = nil 541 | } 542 | for !in.IsDelim('}') { 543 | key := string(in.String()) 544 | in.WantColon() 545 | var v8 interface{} 546 | if m, ok := v8.(easyjson.Unmarshaler); ok { 547 | m.UnmarshalEasyJSON(in) 548 | } else if m, ok := v8.(json.Unmarshaler); ok { 549 | _ = m.UnmarshalJSON(in.Raw()) 550 | } else { 551 | v8 = in.Interface() 552 | } 553 | (out.Results)[key] = v8 554 | in.WantComma() 555 | } 556 | in.Delim('}') 557 | } 558 | case "progress": 559 | out.Progress = float64(in.Float64()) 560 | case "attempts": 561 | out.Attempts = int(in.Int()) 562 | case "started": 563 | out.Started = int64(in.Int64()) 564 | case "failed": 565 | out.Failed = int64(in.Int64()) 566 | case "completed": 567 | out.Completed = int64(in.Int64()) 568 | case "logs": 569 | if in.IsNull() { 570 | in.Skip() 571 | out.Logs = nil 572 | } else { 573 | in.Delim('[') 574 | if out.Logs == nil { 575 | if !in.IsDelim(']') { 576 | out.Logs = make([]Log, 0, 1) 577 | } else { 578 | out.Logs = []Log{} 579 | } 580 | } else { 581 | out.Logs = (out.Logs)[:0] 582 | } 583 | for !in.IsDelim(']') { 584 | var v9 Log 585 | (v9).UnmarshalEasyJSON(in) 586 | out.Logs = append(out.Logs, v9) 587 | in.WantComma() 588 | } 589 | in.Delim(']') 590 | } 591 | case "pipeline": 592 | out.PipelineID = string(in.String()) 593 | default: 594 | in.SkipRecursive() 595 | } 596 | in.WantComma() 597 | } 598 | in.Delim('}') 599 | if isTopLevel { 600 | in.Consumed() 601 | } 602 | } 603 | func easyjson2189435aEncodeGithubComPolygonIoErrandsServerSchemas4(out *jwriter.Writer, in Errand) { 604 | out.RawByte('{') 605 | first := true 606 | _ = first 607 | { 608 | const prefix string = ",\"id\":" 609 | out.RawString(prefix[1:]) 610 | out.String(string(in.ID)) 611 | } 612 | { 613 | const prefix string = ",\"name\":" 614 | out.RawString(prefix) 615 | out.String(string(in.Name)) 616 | } 617 | { 618 | const prefix string = ",\"type\":" 619 | out.RawString(prefix) 620 | out.String(string(in.Type)) 621 | } 622 | { 623 | const prefix string = ",\"options\":" 624 | out.RawString(prefix) 625 | (in.Options).MarshalEasyJSON(out) 626 | } 627 | if len(in.Data) != 0 { 628 | const prefix string = ",\"data\":" 629 | out.RawString(prefix) 630 | { 631 | out.RawByte('{') 632 | v10First := true 633 | for v10Name, v10Value := range in.Data { 634 | if v10First { 635 | v10First = false 636 | } else { 637 | out.RawByte(',') 638 | } 639 | out.String(string(v10Name)) 640 | out.RawByte(':') 641 | if m, ok := v10Value.(easyjson.Marshaler); ok { 642 | m.MarshalEasyJSON(out) 643 | } else if m, ok := v10Value.(json.Marshaler); ok { 644 | out.Raw(m.MarshalJSON()) 645 | } else { 646 | out.Raw(json.Marshal(v10Value)) 647 | } 648 | } 649 | out.RawByte('}') 650 | } 651 | } 652 | { 653 | const prefix string = ",\"created\":" 654 | out.RawString(prefix) 655 | out.Int64(int64(in.Created)) 656 | } 657 | if in.Status != "" { 658 | const prefix string = ",\"status\":" 659 | out.RawString(prefix) 660 | out.String(string(in.Status)) 661 | } 662 | if len(in.Results) != 0 { 663 | const prefix string = ",\"results\":" 664 | out.RawString(prefix) 665 | { 666 | out.RawByte('{') 667 | v11First := true 668 | for v11Name, v11Value := range in.Results { 669 | if v11First { 670 | v11First = false 671 | } else { 672 | out.RawByte(',') 673 | } 674 | out.String(string(v11Name)) 675 | out.RawByte(':') 676 | if m, ok := v11Value.(easyjson.Marshaler); ok { 677 | m.MarshalEasyJSON(out) 678 | } else if m, ok := v11Value.(json.Marshaler); ok { 679 | out.Raw(m.MarshalJSON()) 680 | } else { 681 | out.Raw(json.Marshal(v11Value)) 682 | } 683 | } 684 | out.RawByte('}') 685 | } 686 | } 687 | { 688 | const prefix string = ",\"progress\":" 689 | out.RawString(prefix) 690 | out.Float64(float64(in.Progress)) 691 | } 692 | { 693 | const prefix string = ",\"attempts\":" 694 | out.RawString(prefix) 695 | out.Int(int(in.Attempts)) 696 | } 697 | if in.Started != 0 { 698 | const prefix string = ",\"started\":" 699 | out.RawString(prefix) 700 | out.Int64(int64(in.Started)) 701 | } 702 | if in.Failed != 0 { 703 | const prefix string = ",\"failed\":" 704 | out.RawString(prefix) 705 | out.Int64(int64(in.Failed)) 706 | } 707 | if in.Completed != 0 { 708 | const prefix string = ",\"completed\":" 709 | out.RawString(prefix) 710 | out.Int64(int64(in.Completed)) 711 | } 712 | if len(in.Logs) != 0 { 713 | const prefix string = ",\"logs\":" 714 | out.RawString(prefix) 715 | { 716 | out.RawByte('[') 717 | for v12, v13 := range in.Logs { 718 | if v12 > 0 { 719 | out.RawByte(',') 720 | } 721 | (v13).MarshalEasyJSON(out) 722 | } 723 | out.RawByte(']') 724 | } 725 | } 726 | if in.PipelineID != "" { 727 | const prefix string = ",\"pipeline\":" 728 | out.RawString(prefix) 729 | out.String(string(in.PipelineID)) 730 | } 731 | out.RawByte('}') 732 | } 733 | 734 | // MarshalJSON supports json.Marshaler interface 735 | func (v Errand) MarshalJSON() ([]byte, error) { 736 | w := jwriter.Writer{} 737 | easyjson2189435aEncodeGithubComPolygonIoErrandsServerSchemas4(&w, v) 738 | return w.Buffer.BuildBytes(), w.Error 739 | } 740 | 741 | // MarshalEasyJSON supports easyjson.Marshaler interface 742 | func (v Errand) MarshalEasyJSON(w *jwriter.Writer) { 743 | easyjson2189435aEncodeGithubComPolygonIoErrandsServerSchemas4(w, v) 744 | } 745 | 746 | // UnmarshalJSON supports json.Unmarshaler interface 747 | func (v *Errand) UnmarshalJSON(data []byte) error { 748 | r := jlexer.Lexer{Data: data} 749 | easyjson2189435aDecodeGithubComPolygonIoErrandsServerSchemas4(&r, v) 750 | return r.Error() 751 | } 752 | 753 | // UnmarshalEasyJSON supports easyjson.Unmarshaler interface 754 | func (v *Errand) UnmarshalEasyJSON(l *jlexer.Lexer) { 755 | easyjson2189435aDecodeGithubComPolygonIoErrandsServerSchemas4(l, v) 756 | } 757 | -------------------------------------------------------------------------------- /utils/utils.go: -------------------------------------------------------------------------------- 1 | // Package utils provides basic utilities for the errand server. 2 | package utils 3 | 4 | import ( 5 | "time" 6 | ) 7 | 8 | func GetTimestamp() int64 { 9 | return time.Now().UnixNano() / 1_000_000 10 | } 11 | 12 | func Contains(s []string, e string) bool { 13 | for _, a := range s { 14 | if a == e { 15 | return true 16 | } 17 | } 18 | 19 | return false 20 | } 21 | --------------------------------------------------------------------------------