├── LICENSE ├── README.md ├── example ├── checkpoints.go ├── config.go ├── deleteJar.go ├── jobDetail.go ├── jobManagerConfig.go ├── jobManagerMetrics.go ├── jobs.go ├── jobsOverview.go ├── listJars.go ├── planJar.go ├── runJar.go ├── savepoints.go ├── shutdown.go ├── stopJob.go ├── stopJobWithSavepoint.go ├── testdata │ └── test.jar └── uploadJar.go ├── go.mod ├── jar.go ├── job.go └── request.go /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016-2018 Flink Club Authors. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Flink Monitoring API Golang library 2 | 3 | [![GoDoc](https://godoc.org/github.com/flink-go/api?status.svg)](https://godoc.org/github.com/flink-go/api) 4 | 5 | Detail doc: https://ci.apache.org/projects/flink/flink-docs-stable/monitoring/rest_api.html 6 | 7 | Status: Beta 8 | 9 | 10 | ``` 11 | package main 12 | 13 | import ( 14 | "fmt" 15 | 16 | "github.com/flink-go/api" 17 | ) 18 | 19 | func main() { 20 | // Your flink server HTTP API 21 | c, err := api.New("127.0.0.1:8081") 22 | if err != nil { 23 | panic(err) 24 | } 25 | 26 | // get cluster config 27 | config, err := c.Config() 28 | if err != nil { 29 | panic(err) 30 | } 31 | fmt.Println(config) 32 | } 33 | ``` 34 | 35 | More examples in [example](/example) dir. 36 | ### Cluster API 37 | 38 | * shutdown cluster 39 | * list config 40 | 41 | 42 | ### Jar File API 43 | 44 | * upload jar file 45 | * list jar files 46 | * delete jar file 47 | * plan jar file 48 | * run jar file 49 | 50 | ### Job API 51 | 52 | * job manager config 53 | * job manager metrics 54 | * list all jobs 55 | * stop a job 56 | * job overview 57 | * job detail 58 | 59 | ### checkpoints 60 | 61 | * get all checkpoints of a job 62 | * stop a job with a savepoint 63 | 64 | ### TODO: 65 | 66 | * vertices 67 | * checkpoints/config 68 | * /jobs/:jobid/checkpoints/details/:checkpointid 69 | * /jobs/:jobid/config 70 | * /jobs/:jobid/exceptions 71 | * /jobs/:jobid/execution-result 72 | * /jobs/:jobid/metrics 73 | * /jobs/:jobid/plan 74 | * /jobs/:jobid/rescaling 75 | * /jobs/:jobid/rescaling/:triggerid 76 | * overview 77 | * /savepoint-disposal 78 | * /taskmanagers 79 | 80 | -------------------------------------------------------------------------------- /example/checkpoints.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/flink-go/api" 8 | ) 9 | 10 | func main() { 11 | c, err := api.New(os.Getenv("FLINK_API")) 12 | if err != nil { 13 | panic(err) 14 | } 15 | 16 | // Checkpoints test 17 | v, err := c.Checkpoints("2bd452ba193d1575a4acc9ed09f896ea") 18 | if err != nil { 19 | panic(err) 20 | } 21 | fmt.Println(v) 22 | } 23 | -------------------------------------------------------------------------------- /example/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/flink-go/api" 8 | ) 9 | 10 | func main() { 11 | c, err := api.New(os.Getenv("FLINK_API")) 12 | if err != nil { 13 | panic(err) 14 | } 15 | 16 | // config test 17 | config, err := c.Config() 18 | if err != nil { 19 | panic(err) 20 | } 21 | fmt.Println(config) 22 | } 23 | -------------------------------------------------------------------------------- /example/deleteJar.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/flink-go/api" 7 | ) 8 | 9 | func main() { 10 | c, err := api.New(os.Getenv("FLINK_API")) 11 | if err != nil { 12 | panic(err) 13 | } 14 | 15 | // delete test 16 | err = c.DeleteJar("efb8367e-aa0d-4ceb-957f-0e8f46ed4b10_test.jar") 17 | if err != nil { 18 | panic(err) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /example/jobDetail.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/flink-go/api" 8 | ) 9 | 10 | func main() { 11 | c, err := api.New(os.Getenv("FLINK_API")) 12 | if err != nil { 13 | panic(err) 14 | } 15 | 16 | // jobs test 17 | jobs, err := c.Job("8ea123d2bdc3064f36b92889e43803ee") 18 | if err != nil { 19 | panic(err) 20 | } 21 | fmt.Println(jobs) 22 | } 23 | -------------------------------------------------------------------------------- /example/jobManagerConfig.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/flink-go/api" 8 | ) 9 | 10 | func main() { 11 | c, err := api.New(os.Getenv("FLINK_API")) 12 | if err != nil { 13 | panic(err) 14 | } 15 | 16 | // job manager config test 17 | config, err := c.JobManagerConfig() 18 | if err != nil { 19 | panic(err) 20 | } 21 | fmt.Println(config) 22 | } 23 | -------------------------------------------------------------------------------- /example/jobManagerMetrics.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/flink-go/api" 8 | ) 9 | 10 | func main() { 11 | c, err := api.New(os.Getenv("FLINK_API")) 12 | if err != nil { 13 | panic(err) 14 | } 15 | 16 | // job manager metrics test 17 | config, err := c.JobManagerMetrics() 18 | if err != nil { 19 | panic(err) 20 | } 21 | fmt.Println(config) 22 | } 23 | -------------------------------------------------------------------------------- /example/jobs.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/flink-go/api" 8 | ) 9 | 10 | func main() { 11 | c, err := api.New(os.Getenv("FLINK_API")) 12 | if err != nil { 13 | panic(err) 14 | } 15 | 16 | // jobs test 17 | jobs, err := c.Jobs() 18 | if err != nil { 19 | panic(err) 20 | } 21 | fmt.Println(jobs) 22 | } 23 | -------------------------------------------------------------------------------- /example/jobsOverview.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/flink-go/api" 8 | ) 9 | 10 | func main() { 11 | c, err := api.New(os.Getenv("FLINK_API")) 12 | if err != nil { 13 | panic(err) 14 | } 15 | 16 | // jobs overview test 17 | jobs, err := c.JobsOverview() 18 | if err != nil { 19 | panic(err) 20 | } 21 | fmt.Println(jobs) 22 | } 23 | -------------------------------------------------------------------------------- /example/listJars.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/flink-go/api" 8 | ) 9 | 10 | func main() { 11 | c, err := api.New(os.Getenv("FLINK_API")) 12 | if err != nil { 13 | panic(err) 14 | } 15 | 16 | // upload test 17 | resp, err := c.Jars() 18 | if err != nil { 19 | panic(err) 20 | } 21 | fmt.Println(resp) 22 | } 23 | -------------------------------------------------------------------------------- /example/planJar.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/flink-go/api" 8 | ) 9 | 10 | func main() { 11 | c, err := api.New(os.Getenv("FLINK_API")) 12 | if err != nil { 13 | panic(err) 14 | } 15 | 16 | // plan test 17 | resp, err := c.PlanJar("8c0c2226-b532-4d9b-b698-8aa649694bb9_test.jar") 18 | if err != nil { 19 | panic(err) 20 | } 21 | fmt.Println(resp) 22 | } 23 | -------------------------------------------------------------------------------- /example/runJar.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/flink-go/api" 8 | ) 9 | 10 | func main() { 11 | c, err := api.New(os.Getenv("FLINK_API")) 12 | if err != nil { 13 | panic(err) 14 | } 15 | 16 | opts := api.RunOpts{ 17 | JarID: "8c0c2226-b532-4d9b-b698-8aa649694bb9_test.jar", 18 | } 19 | // run test 20 | resp, err := c.RunJar(opts) 21 | if err != nil { 22 | panic(err) 23 | } 24 | fmt.Println(resp) 25 | } 26 | -------------------------------------------------------------------------------- /example/savepoints.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/flink-go/api" 8 | ) 9 | 10 | func main() { 11 | c, err := api.New(os.Getenv("FLINK_API")) 12 | if err != nil { 13 | panic(err) 14 | } 15 | 16 | // save points test 17 | v, err := c.SavePoints("2bd452ba193d1575a4acc9ed09f896ea", "test", false) 18 | if err != nil { 19 | panic(err) 20 | } 21 | fmt.Println(v) 22 | } 23 | -------------------------------------------------------------------------------- /example/shutdown.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/flink-go/api" 7 | ) 8 | 9 | func main() { 10 | c, err := api.New(os.Getenv("FLINK_API")) 11 | if err != nil { 12 | panic(err) 13 | } 14 | 15 | // shutdown test 16 | if err := c.Shutdown(); err != nil { 17 | panic(err) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /example/stopJob.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/flink-go/api" 7 | ) 8 | 9 | func main() { 10 | c, err := api.New(os.Getenv("FLINK_API")) 11 | if err != nil { 12 | panic(err) 13 | } 14 | 15 | // stop job test 16 | err = c.StopJob("8ea123d2bdc3064f36b92889e43803ee") 17 | if err != nil { 18 | panic(err) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /example/stopJobWithSavepoint.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/flink-go/api" 8 | ) 9 | 10 | func main() { 11 | c, err := api.New(os.Getenv("FLINK_API")) 12 | if err != nil { 13 | panic(err) 14 | } 15 | 16 | v, err := c.StopJobWithSavepoint("2bd452ba193d1575a4acc9ed09f896ea", "test", false) 17 | if err != nil { 18 | panic(err) 19 | } 20 | fmt.Println(v) 21 | } 22 | -------------------------------------------------------------------------------- /example/testdata/test.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flink-go/api/471e35b6d0ae1f6ce1029a6cae24249c1c42af77/example/testdata/test.jar -------------------------------------------------------------------------------- /example/uploadJar.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/flink-go/api" 8 | ) 9 | 10 | func main() { 11 | c, err := api.New(os.Getenv("FLINK_API")) 12 | if err != nil { 13 | panic(err) 14 | } 15 | 16 | // upload test 17 | resp, err := c.UploadJar("./testdata/test.jar") 18 | if err != nil { 19 | panic(err) 20 | } 21 | fmt.Println(resp) 22 | } 23 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/flink-go/api 2 | 3 | go 1.14 4 | -------------------------------------------------------------------------------- /jar.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "mime/multipart" 9 | "net/http" 10 | "os" 11 | "path/filepath" 12 | "strconv" 13 | "strings" 14 | ) 15 | 16 | // Client reprents flink REST API client 17 | type Client struct { 18 | // Addr reprents flink job manager server address 19 | Addr string 20 | 21 | client *httpClient 22 | } 23 | 24 | // New returns a flink client 25 | func New(addr string) (*Client, error) { 26 | return &Client{ 27 | Addr: addr, 28 | client: newHttpClient(), 29 | }, nil 30 | } 31 | 32 | func (c *Client) url(path string) string { 33 | if strings.HasPrefix(c.Addr, "http") { 34 | return fmt.Sprintf("%s%s", c.Addr, path) 35 | } 36 | return fmt.Sprintf("http://%s%s", c.Addr, path) 37 | } 38 | 39 | // Shutdown shutdown the flink cluster 40 | func (c *Client) Shutdown() error { 41 | req, err := http.NewRequest("DELETE", c.url("/cluster"), nil) 42 | if err != nil { 43 | return err 44 | } 45 | _, err = c.client.Do(req) 46 | return err 47 | } 48 | 49 | type configResp struct { 50 | RefreshInterval int64 `json:"refresh-interval"` 51 | TimezoneName string `json:"timezone-name"` 52 | TimezoneOffset int64 `json:"timezone-offset"` 53 | FlinkVersion string `json:"flink-version"` 54 | FlinkRevision string `json:"flink-revision"` 55 | Features features `json:"features"` 56 | } 57 | type features struct { 58 | WebSubmit bool `json:"web-submit"` 59 | } 60 | 61 | // Config returns the configuration of the WebUI 62 | func (c *Client) Config() (configResp, error) { 63 | var r configResp 64 | req, err := http.NewRequest("GET", c.url("/config"), nil) 65 | if err != nil { 66 | return r, err 67 | } 68 | b, err := c.client.Do(req) 69 | if err != nil { 70 | return r, err 71 | } 72 | err = json.Unmarshal(b, &r) 73 | return r, err 74 | } 75 | 76 | type uploadResp struct { 77 | FileName string `json:"filename"` 78 | Status string `json:"status"` 79 | } 80 | 81 | // Upload uploads jar file 82 | func (c *Client) UploadJar(fpath string) (uploadResp, error) { 83 | var r uploadResp 84 | file, err := os.Open(fpath) 85 | if err != nil { 86 | return r, err 87 | } 88 | defer file.Close() 89 | 90 | body := &bytes.Buffer{} 91 | writer := multipart.NewWriter(body) 92 | part, err := writer.CreateFormFile("jarfile", filepath.Base(file.Name())) 93 | io.Copy(part, file) 94 | writer.Close() 95 | 96 | req, err := http.NewRequest("POST", c.url("/jars/upload"), body) 97 | if err != nil { 98 | return r, err 99 | } 100 | req.Header.Add("Content-Type", writer.FormDataContentType()) 101 | b, err := c.client.Do(req) 102 | if err != nil { 103 | return r, err 104 | } 105 | err = json.Unmarshal(b, &r) 106 | return r, err 107 | } 108 | 109 | type jarsResp struct { 110 | Address string `json:"address"` 111 | Files []jarFile `json:"files"` 112 | } 113 | 114 | type jarFile struct { 115 | ID string `json:"id"` 116 | Name string `json:"name"` 117 | Uploaded int64 `json:"uploaded"` 118 | Entries []entry `json:"entry"` 119 | } 120 | 121 | type entry struct { 122 | Name string `json:"name"` 123 | Description string `json:"description"` 124 | } 125 | 126 | // Jars eturns a list of all jars previously uploaded 127 | // via '/jars/upload' 128 | func (c *Client) Jars() (jarsResp, error) { 129 | var r jarsResp 130 | req, err := http.NewRequest("GET", c.url("/jars"), nil) 131 | if err != nil { 132 | return r, err 133 | } 134 | b, err := c.client.Do(req) 135 | if err != nil { 136 | return r, err 137 | } 138 | err = json.Unmarshal(b, &r) 139 | return r, err 140 | } 141 | 142 | // DeleteJar deletes a jar file 143 | func (c *Client) DeleteJar(jarid string) error { 144 | uri := fmt.Sprintf("/jars/%s", jarid) 145 | req, err := http.NewRequest("DELETE", c.url(uri), nil) 146 | if err != nil { 147 | return err 148 | } 149 | _, err = c.client.Do(req) 150 | return err 151 | } 152 | 153 | type planResp struct { 154 | Plan plan `json:"plan"` 155 | } 156 | 157 | type plan struct { 158 | JID string `json:"jid"` 159 | Name string `json:"name"` 160 | Nodes []node `json:"nodes"` 161 | } 162 | 163 | type node struct { 164 | ID string `json:"id"` 165 | Parallelism int `json:"parallelism"` 166 | Operator string `json:"operator"` 167 | OperatorStrategy string `json:"operator_strategy"` 168 | Description string `json:"description"` 169 | Inputs []input `json:"inputs"` 170 | } 171 | 172 | type input struct { 173 | Num int `json:"num"` 174 | ID string `json:"id"` 175 | ShipStrategy string `json:"ship_strategy"` 176 | Exchange string `json:"exchange"` 177 | } 178 | 179 | // PlanJar returns the dataflow plan of a job contained 180 | // in a jar previously uploaded via '/jars/upload'. 181 | // Todo: support more args. 182 | func (c *Client) PlanJar(jarid string) (planResp, error) { 183 | var r planResp 184 | uri := fmt.Sprintf("/jars/%s/plan", jarid) 185 | req, err := http.NewRequest("GET", c.url(uri), nil) 186 | if err != nil { 187 | return r, err 188 | } 189 | b, err := c.client.Do(req) 190 | if err != nil { 191 | return r, err 192 | } 193 | err = json.Unmarshal(b, &r) 194 | return r, err 195 | } 196 | 197 | type runResp struct { 198 | ID string 199 | } 200 | 201 | type RunOpts struct { 202 | // JarID: String value that identifies a jar. When 203 | // uploading the jar a path is returned, where the 204 | // filename is the ID. 205 | JarID string 206 | 207 | // AllowNonRestoredState(optional): Boolean value that 208 | // specifies whether the job submission should be 209 | // rejected if the savepoint contains state that 210 | // cannot be mapped back to the job. 211 | AllowNonRestoredState bool 212 | 213 | // SavepointPath (optional): String value that 214 | // specifies the path of the savepoint to restore the 215 | // job from. 216 | SavepointPath string 217 | 218 | // programArg (optional): list of program arguments. 219 | ProgramArg []string 220 | 221 | // EntryClass (optional): String value that specifies 222 | // the fully qualified name of the entry point class. 223 | // Overrides the class defined in the jar file 224 | // manifest. 225 | EntryClass string 226 | 227 | // Parallelism (optional): Positive integer value that 228 | // specifies the desired parallelism for the job. 229 | Parallelism int 230 | } 231 | 232 | // RunJar submits a job by running a jar previously 233 | // uploaded via '/jars/upload'. 234 | func (c *Client) RunJar(opts RunOpts) (runResp, error) { 235 | var r runResp 236 | uri := fmt.Sprintf("/jars/%s/run", opts.JarID) 237 | req, err := http.NewRequest("POST", c.url(uri), nil) 238 | q := req.URL.Query() 239 | if opts.SavepointPath != "" { 240 | q.Add("savepointPath", opts.SavepointPath) 241 | q.Add("allowNonRestoredState", strconv.FormatBool(opts.AllowNonRestoredState)) 242 | } 243 | if len(opts.ProgramArg) > 0 { 244 | q.Add("programArg", strings.Join(opts.ProgramArg, ",")) 245 | } 246 | if opts.EntryClass != "" { 247 | q.Add("entry-class", opts.EntryClass) 248 | } 249 | if opts.Parallelism > 0 { 250 | q.Add("parallelism", strconv.Itoa(opts.Parallelism)) 251 | } 252 | req.URL.RawQuery = q.Encode() 253 | if err != nil { 254 | return r, err 255 | } 256 | b, err := c.client.Do(req) 257 | if err != nil { 258 | return r, err 259 | } 260 | err = json.Unmarshal(b, &r) 261 | return r, err 262 | } 263 | -------------------------------------------------------------------------------- /job.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "strings" 9 | ) 10 | 11 | type kv struct { 12 | Key string `json:"key"` 13 | Value string `json:"value"` 14 | } 15 | 16 | // JobManagerConfig returns the cluster configuration of 17 | // job manager server. 18 | func (c *Client) JobManagerConfig() ([]kv, error) { 19 | var r []kv 20 | req, err := http.NewRequest( 21 | "GET", 22 | c.url("/jobmanager/config"), 23 | nil, 24 | ) 25 | if err != nil { 26 | return r, err 27 | } 28 | b, err := c.client.Do(req) 29 | if err != nil { 30 | return r, err 31 | } 32 | err = json.Unmarshal(b, &r) 33 | return r, err 34 | } 35 | 36 | type metric struct { 37 | ID string `json:"id"` 38 | } 39 | 40 | // JobManagerMetrics provides access to job manager 41 | // metrics. 42 | func (c *Client) JobManagerMetrics() ([]metric, error) { 43 | var r []metric 44 | req, err := http.NewRequest( 45 | "GET", 46 | c.url("/jobmanager/metrics"), 47 | nil, 48 | ) 49 | if err != nil { 50 | return r, err 51 | } 52 | b, err := c.client.Do(req) 53 | if err != nil { 54 | return r, err 55 | } 56 | err = json.Unmarshal(b, &r) 57 | return r, err 58 | } 59 | 60 | type jobsResp struct { 61 | Jobs []job `json:"jobs"` 62 | } 63 | 64 | type job struct { 65 | ID string `json:"id"` 66 | Status string `json:"status"` 67 | } 68 | 69 | // Jobs returns an overview over all jobs and their 70 | // current state. 71 | func (c *Client) Jobs() (jobsResp, error) { 72 | var r jobsResp 73 | req, err := http.NewRequest( 74 | "GET", 75 | c.url("/jobs"), 76 | nil, 77 | ) 78 | if err != nil { 79 | return r, err 80 | } 81 | b, err := c.client.Do(req) 82 | if err != nil { 83 | return r, err 84 | } 85 | err = json.Unmarshal(b, &r) 86 | return r, err 87 | } 88 | 89 | // SubmitJob submits a job. 90 | func (c *Client) SubmitJob() error { 91 | return fmt.Errorf("not implement") 92 | } 93 | 94 | type JobMetricsOpts struct { 95 | // Metrics (optional): string values to select 96 | // specific metrics. 97 | Metrics []string 98 | 99 | // Agg (optional): list of aggregation modes which 100 | // should be calculated. Available aggregations are: 101 | // "min, max, sum, avg". 102 | Agg []string 103 | 104 | // Jobs (optional): job list of 32-character 105 | // hexadecimal strings to select specific jobs. 106 | Jobs []string 107 | } 108 | 109 | // JobMetrics provides access to aggregated job metrics. 110 | func (c *Client) JobMetrics(opts JobMetricsOpts) (map[string]interface{}, error) { 111 | var r map[string]interface{} 112 | req, err := http.NewRequest( 113 | "GET", 114 | c.url("/jobs/metrics"), 115 | nil, 116 | ) 117 | if err != nil { 118 | return r, err 119 | } 120 | q := req.URL.Query() 121 | if len(opts.Metrics) > 0 { 122 | q.Add("get", strings.Join(opts.Metrics, ",")) 123 | } 124 | 125 | if len(opts.Agg) > 0 { 126 | q.Add("agg", strings.Join(opts.Agg, ",")) 127 | } 128 | 129 | if len(opts.Jobs) > 0 { 130 | q.Add("jobs", strings.Join(opts.Jobs, ",")) 131 | } 132 | 133 | req.URL.RawQuery = q.Encode() 134 | 135 | b, err := c.client.Do(req) 136 | if err != nil { 137 | return r, err 138 | } 139 | err = json.Unmarshal(b, &r) 140 | return r, err 141 | } 142 | 143 | type overviewResp struct { 144 | Jobs []jobOverview `json:"jobs"` 145 | } 146 | 147 | type jobOverview struct { 148 | ID string `json:"jid"` 149 | Name string `json:"name"` 150 | State string `json:"state"` 151 | Start int64 `json:"start-time"` 152 | End int64 `json:"end-time"` 153 | Duration int64 `json:"duration"` 154 | LastModification int64 `json:"last-modification"` 155 | Tasks status `json:"tasks"` 156 | } 157 | 158 | type status struct { 159 | Total int `json:"total,omitempty"` 160 | Created int `json:"created"` 161 | Scheduled int `json:"scheduled"` 162 | Deploying int `json:"deploying"` 163 | Running int `json:"running"` 164 | Finished int `json:"finished"` 165 | Canceling int `json:"canceling"` 166 | Canceled int `json:"canceled"` 167 | Failed int `json:"failed"` 168 | Reconciling int `json:"reconciling"` 169 | } 170 | 171 | // JobsOverview returns an overview over all jobs. 172 | func (c *Client) JobsOverview() (overviewResp, error) { 173 | var r overviewResp 174 | req, err := http.NewRequest( 175 | "GET", 176 | c.url("/jobs/overview"), 177 | nil, 178 | ) 179 | if err != nil { 180 | return r, err 181 | } 182 | b, err := c.client.Do(req) 183 | if err != nil { 184 | return r, err 185 | } 186 | err = json.Unmarshal(b, &r) 187 | return r, err 188 | } 189 | 190 | type jobResp struct { 191 | ID string `json:"jid"` 192 | Name string `json:"name"` 193 | IsStoppable bool `json:"isStoppable"` 194 | State string `json:"state"` 195 | 196 | Start int64 `json:"start-time"` 197 | End int64 `json:"end-time"` 198 | Duration int64 `json:"duration"` 199 | Now int64 `json:"now"` 200 | 201 | Timestamps timestamps `json:"timestamps"` 202 | Vertices []vertice `json:"vertices"` 203 | StatusCounts status `json:"status-counts"` 204 | Plan plan `json:"plan"` 205 | } 206 | 207 | type timestamps struct { 208 | Canceled int64 `json:"CANCELED"` 209 | Suspended int64 `json:"SUSPENDED"` 210 | Finished int64 `json:"FINISHED"` 211 | Canceling int64 `json:"CANCELLING"` 212 | Running int64 `json:"RUNNING"` 213 | Restaring int64 `json:"RESTARTING"` 214 | Reconciling int64 `json:"RECONCILING"` 215 | Created int64 `json:"CREATED"` 216 | Failed int64 `json:"FAILED"` 217 | Failing int64 `json:"FAILING"` 218 | } 219 | 220 | type vertice struct { 221 | ID string `json:"id"` 222 | Name string `json:"name"` 223 | Status string `json:"status"` 224 | Parallelism int `json:"parallelism"` 225 | Start int64 `json:"start-time"` 226 | End int64 `json:"end-time"` 227 | Duration int64 `json:"duration"` 228 | Tasks status `json:"tasks"` 229 | Metrics map[string]interface{} `json:"metrics"` 230 | } 231 | 232 | // Job returns details of a job. 233 | func (c *Client) Job(jobID string) (jobResp, error) { 234 | var r jobResp 235 | uri := fmt.Sprintf("/jobs/%s", jobID) 236 | req, err := http.NewRequest( 237 | "GET", 238 | c.url(uri), 239 | nil, 240 | ) 241 | if err != nil { 242 | return r, err 243 | } 244 | b, err := c.client.Do(req) 245 | if err != nil { 246 | return r, err 247 | } 248 | err = json.Unmarshal(b, &r) 249 | return r, err 250 | } 251 | 252 | // StopJob terminates a job. 253 | func (c *Client) StopJob(jobID string) error { 254 | uri := fmt.Sprintf("/jobs/%s", jobID) 255 | req, err := http.NewRequest( 256 | "PATCH", 257 | c.url(uri), 258 | nil, 259 | ) 260 | if err != nil { 261 | return err 262 | } 263 | _, err = c.client.Do(req) 264 | return err 265 | } 266 | 267 | type checkpointsResp struct { 268 | Counts counts `json:"counts"` 269 | Summary summary `json:"summary"` 270 | Latest latest `json:"latest"` 271 | History []failedCheckpointsStatics `json:"history"` 272 | } 273 | 274 | type counts struct { 275 | Restored int `json:"restored"` 276 | Total int `json:"total"` 277 | InProgress int `json:"in_progress"` 278 | Completed int `json:"completed"` 279 | Failed int `json:"failed"` 280 | } 281 | 282 | type summary struct { 283 | StateSize statics `json:"state_size` 284 | End2EndDuration statics `json:"end_to_end_duration"` 285 | AlignmentBuffered statics `json:"alignment_buffered"` 286 | } 287 | 288 | type statics struct { 289 | Min int `json:"min"` 290 | Max int `json:"max"` 291 | Avg int `json:"avg"` 292 | } 293 | 294 | type latest struct { 295 | Completed completedCheckpointsStatics `json:"completed"` 296 | Savepoint savepointsStatics `json:"savepoint"` 297 | Failed failedCheckpointsStatics `json:"failed"` 298 | Restored restoredCheckpointsStatics `json:"restored"` 299 | } 300 | 301 | type completedCheckpointsStatics struct { 302 | ID string `json:"id"` 303 | Status string `json:"status"` 304 | IsSavepoint bool `json:"is_savepoint"` 305 | TriggerTimestamp int64 `json:"trigger_timestamp"` 306 | LatestAckTimestamp int64 `json:"latest_ack_timestamp"` 307 | StateSize int64 `json:"state_size"` 308 | End2EndDuration int64 `json:"end_to_end_duration"` 309 | AlignmentBuffered int64 `json:"alignment_buffered"` 310 | NumSubtasks int64 `json:"num_subtasks"` 311 | NumAcknowledgedSubtasks int64 `json:"num_acknowledged_subtasks"` 312 | tasks taskCheckpointsStatics `json:"tasks"` 313 | ExternalPath string `json:"external_path"` 314 | Discarded bool `json:"discarded"` 315 | } 316 | 317 | type savepointsStatics struct { 318 | ID int `json:"id"` 319 | Status string `json:"status"` 320 | IsSavepoint bool `json:"is_savepoint"` 321 | TriggerTimestamp int64 `json:"trigger_timestamp"` 322 | LatestAckTimestamp int64 `json:"latest_ack_timestamp"` 323 | StateSize int64 `json:"state_size"` 324 | End2EndDuration int64 `json:"end_to_end_duration"` 325 | AlignmentBuffered int64 `json:"alignment_buffered"` 326 | NumSubtasks int64 `json:"num_subtasks"` 327 | NumAcknowledgedSubtasks int64 `json:"num_acknowledged_subtasks"` 328 | tasks taskCheckpointsStatics `json:"tasks"` 329 | ExternalPath string `json:"external_path"` 330 | Discarded bool `json:"discarded"` 331 | } 332 | type taskCheckpointsStatics struct { 333 | ID string `json:"id"` 334 | Status string `json:"status"` 335 | 336 | LatestAckTimestamp int64 `json:"latest_ack_timestamp"` 337 | 338 | FailureTimestamp int64 `json:"failure_timestamp"` 339 | FailureMessage string `json:"failure_message"` 340 | 341 | StateSize int64 `json:"state_size"` 342 | End2EndDuration int64 `json:"end_to_end_duration"` 343 | AlignmentBuffered int64 `json:"alignment_buffered"` 344 | NumSubtasks int64 `json:"num_subtasks"` 345 | NumAcknowledgedSubtasks int64 `json:"num_acknowledged_subtasks"` 346 | } 347 | 348 | type failedCheckpointsStatics struct { 349 | ID int64 `json:"id"` 350 | Status string `json:"status"` 351 | IsSavepoint bool `json:"is_savepoint"` 352 | TriggerTimestamp int64 `json:"trigger_timestamp"` 353 | LatestAckTimestamp int64 `json:"latest_ack_timestamp"` 354 | StateSize int64 `json:"state_size"` 355 | End2EndDuration int64 `json:"end_to_end_duration"` 356 | AlignmentBuffered int64 `json:"alignment_buffered"` 357 | NumSubtasks int64 `json:"num_subtasks"` 358 | NumAcknowledgedSubtasks int64 `json:"num_acknowledged_subtasks"` 359 | tasks taskCheckpointsStatics `json:"tasks"` 360 | } 361 | 362 | type restoredCheckpointsStatics struct { 363 | ID int64 `json:"id"` 364 | RestoreTimestamp int64 `json:"restore_timestamp"` 365 | IsSavepoint bool `json:"is_savepoint"` 366 | ExternalPath string `json:"external_path"` 367 | } 368 | 369 | // Checkpoints returns checkpointing statistics for a job. 370 | func (c *Client) Checkpoints(jobID string) (checkpointsResp, error) { 371 | var r checkpointsResp 372 | uri := fmt.Sprintf("/jobs/%s/checkpoints", jobID) 373 | req, err := http.NewRequest( 374 | "GET", 375 | c.url(uri), 376 | nil, 377 | ) 378 | if err != nil { 379 | return r, err 380 | } 381 | b, err := c.client.Do(req) 382 | if err != nil { 383 | return r, err 384 | } 385 | err = json.Unmarshal(b, &r) 386 | return r, err 387 | } 388 | 389 | type savePointsResp struct { 390 | RequestID string `json:"request-id"` 391 | } 392 | 393 | // SavePoints triggers a savepoint, and optionally cancels the 394 | // job afterwards. This async operation would return a 395 | // 'triggerid' for further query identifier. 396 | func (c *Client) SavePoints(jobID string, saveDir string, cancleJob bool) (savePointsResp, error) { 397 | var r savePointsResp 398 | 399 | type savePointsReq struct { 400 | SaveDir string `json:"target-directory"` 401 | CancleJob bool `json:"cancel-job"` 402 | } 403 | 404 | d := savePointsReq{ 405 | SaveDir: saveDir, 406 | CancleJob: cancleJob, 407 | } 408 | data := new(bytes.Buffer) 409 | json.NewEncoder(data).Encode(d) 410 | uri := fmt.Sprintf("/jobs/%s/savepoints", jobID) 411 | req, err := http.NewRequest( 412 | "POST", 413 | c.url(uri), 414 | data, 415 | ) 416 | if err != nil { 417 | return r, err 418 | } 419 | b, err := c.client.Do(req) 420 | if err != nil { 421 | return r, err 422 | } 423 | err = json.Unmarshal(b, &r) 424 | return r, err 425 | } 426 | 427 | type stopJobResp struct { 428 | RequestID string `json:"request-id"` 429 | } 430 | 431 | // StopJob stops a job with a savepoint. Optionally, it can also 432 | // emit a MAX_WATERMARK before taking the savepoint to flush out 433 | // any state waiting for timers to fire. This async operation 434 | // would return a 'triggerid' for further query identifier. 435 | func (c *Client) StopJobWithSavepoint(jobID string, saveDir string, drain bool) (stopJobResp, error) { 436 | var r stopJobResp 437 | type stopJobReq struct { 438 | SaveDir string `json:"targetDirectory"` 439 | Drain bool `json:"drain"` 440 | } 441 | 442 | d := stopJobReq{ 443 | SaveDir: saveDir, 444 | Drain: drain, 445 | } 446 | data := new(bytes.Buffer) 447 | json.NewEncoder(data).Encode(d) 448 | uri := fmt.Sprintf("/jobs/%s/stop", jobID) 449 | req, err := http.NewRequest( 450 | "POST", 451 | c.url(uri), 452 | data, 453 | ) 454 | if err != nil { 455 | return r, err 456 | } 457 | b, err := c.client.Do(req) 458 | if err != nil { 459 | return r, err 460 | } 461 | err = json.Unmarshal(b, &r) 462 | return r, err 463 | } 464 | -------------------------------------------------------------------------------- /request.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | ) 8 | 9 | type httpClient struct { 10 | client http.Client 11 | } 12 | 13 | func newHttpClient() *httpClient { 14 | return &httpClient{ 15 | client: http.Client{}, 16 | } 17 | } 18 | 19 | func (c *httpClient) Do(req *http.Request) ([]byte, error) { 20 | resp, err := c.client.Do(req) 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | defer resp.Body.Close() 26 | body, err := ioutil.ReadAll(resp.Body) 27 | if err != nil { 28 | return nil, err 29 | } 30 | if int(resp.StatusCode/100) != 2 { 31 | return nil, fmt.Errorf("http status not 2xx: %d %s", resp.StatusCode, string(body)) 32 | } 33 | return body, nil 34 | } 35 | --------------------------------------------------------------------------------