├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── HISTORY.md ├── LICENSE ├── Makefile ├── README.md ├── client.go ├── client_test.go ├── cmd └── kafka-connect │ ├── doc.go │ ├── kafka-connect.go │ ├── kafka-connect_test.go │ └── suite_test.go ├── connect_suite_test.go ├── connectors.go ├── connectors_test.go ├── err-excludes.txt ├── errors.go ├── go.mod ├── go.sum ├── tools.go └── version.go /.gitignore: -------------------------------------------------------------------------------- 1 | /dist/ 2 | /man/ 3 | # Coverage profiles generated by `ginkgo -cover` 4 | *.coverprofile 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.11 4 | - 1.12 5 | - tip 6 | 7 | env: 8 | global: 9 | # Travis puts the checkout into GOPATH, so we need to force modules 10 | - GO111MODULE=on 11 | 12 | install: 13 | - make get-devtools 14 | - export PATH=$PATH:$GOPATH/bin 15 | 16 | script: 17 | - make check coverage 18 | 19 | after_success: 20 | - | 21 | bash <(curl -s https://codecov.io/bash) -f connect.coverprofile || \ 22 | echo "Codecov did not collect coverage reports" 23 | 24 | # Container-based infra for builds that start faster. 25 | sudo: false 26 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributing Guide 2 | ------------------ 3 | 4 | Thank you for your interest in contributing! Improvements and reports of bugs 5 | or documentation flaws are welcomed. 6 | 7 | This is a small and simple project, but as my first public Go project I've 8 | tried to establish good habits to follow. If you're unsure about anything, 9 | don't hesitate to ask. 10 | 11 | Please submit fixes or enhancements via GitHub pull requests, ensuring that 12 | changes have passing test coverage and a clean bill of health from `golint`, 13 | `gofmt`, and preferably `go vet`. The latter checks are not yet automated so 14 | your diligence is appreciated. 15 | 16 | You can inspect the `.travis.yml` file to see how the build is executed on CI, 17 | pull requests will be checked to pass the same procedure. The convenience `make 18 | get-devtools` target can install any tools that you're missing. 19 | 20 | If you wish to contribute a change that involves updating dependencies, please 21 | use Go 1.11 or later for your work so that [the `go` tool supports managing 22 | `go.mod`][mod]. 23 | 24 | [mod]: https://tip.golang.org/cmd/go/#hdr-Maintaining_module_requirements 25 | -------------------------------------------------------------------------------- /HISTORY.md: -------------------------------------------------------------------------------- 1 | Release Notes 2 | ============= 3 | 4 | The CLI tool and library are versioned independently and both adhere to 5 | [Semantic Versioning] policy. Compatibility with Kafka Connect API versions 6 | will be noted in the release history below. 7 | 8 | Status 9 | ------ 10 | 11 | The library is tentatively at 1.0 status pending a trial release, the API is 12 | expected to be stable without breaking changes for the Kafka 0.10.0.x series. 13 | The CLI's output, exit codes, etc. are provisional and subject to change until 14 | a 1.0 release—please consult the change log before installing new versions if 15 | you rely on these for scripting. 16 | 17 | Unreleased Changes 18 | ------------------ 19 | 20 | Updated the build to Go 1.11, and only this version in order to use modules and 21 | drop Glide and not keep dependencies vendored. No functional changes. 22 | 23 | kafka-connect CLI 24 | ----------------- 25 | 26 | ### v0.9.0 - 11 August, 2016 ### 27 | 28 | **Library version: v0.9.0** 29 | 30 | Initial release. Covers nearly all API functionality (minus connector plugins), 31 | but output may not yet be stable. 32 | 33 | go-kafka/connect Library 34 | ------------------------ 35 | 36 | ### v0.9.0 - 11 August, 2016 ### 37 | 38 | Initial release. Supports the Kafka Connect REST API as of Kafka v0.10.0.0. 39 | 40 | 41 | [Semantic Versioning]: http://semver.org/ 42 | 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Ches Martin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # This makefile probably requires GNU make >= 3.81 2 | 3 | GO ?= go 4 | # OS X, use sha256sum or gsha256sum elsewhere 5 | SHASUM := shasum -a 256 6 | VERSION := $(shell git describe) 7 | 8 | packages := ./... 9 | fordist := find * -type d -exec 10 | 11 | # Should use -mod=readonly in CI 12 | build: 13 | $(GO) build $(packages) 14 | 15 | install: 16 | $(GO) install $(packages) 17 | 18 | # Use `test all` now for CI? https://research.swtch.com/vgo-cmd 19 | # Re: ginkgo, https://github.com/onsi/ginkgo/issues/278 20 | test: 21 | $(GO) test $(packages) 22 | 23 | spec: 24 | ginkgo -r -v 25 | 26 | # TODO: coverage for CLI? https://github.com/onsi/ginkgo/issues/89 27 | coverage: 28 | $(GO) test --covermode count --coverprofile connect.coverprofile . 29 | $(GO) tool cover --func connect.coverprofile 30 | 31 | browse-coverage: coverage 32 | $(GO) tool cover --html connect.coverprofile 33 | 34 | check: lint errcheck 35 | 36 | lint: 37 | golint --set_exit_status ./... 38 | 39 | errcheck: 40 | errcheck --asserts --exclude=err-excludes.txt $(packages) 41 | 42 | zen: 43 | ginkgo watch -notify $(packages) 44 | 45 | get-devtools: 46 | @echo Getting golint... 47 | $(GO) install golang.org/x/lint/golint 48 | @echo Getting errcheck... 49 | $(GO) install github.com/kisielk/errcheck 50 | 51 | get-reltools: 52 | @echo Getting gox... 53 | $(GO) install github.com/mitchellh/gox 54 | 55 | dist: test 56 | @echo Cross-compiling binaries... 57 | gox -verbose \ 58 | -ldflags "-s -w" \ 59 | -os="darwin linux windows" \ 60 | -arch="amd64 386" \ 61 | -output="dist/{{.OS}}-{{.Arch}}/{{.Dir}}" ./cmd/... 62 | 63 | release: dist 64 | @echo Preparing distributions... 65 | @cd dist && \ 66 | $(fordist) sh -c 'gpg --detach-sign --armor {}/kafka-connect*' \; && \ 67 | $(fordist) cp ../LICENSE {} \; && \ 68 | $(fordist) cp ../README.md {} \; && \ 69 | $(fordist) cp ../HISTORY.md {} \; && \ 70 | $(fordist) tar -zcf kafka-connect-${VERSION}-{}.tar.gz {} \; && \ 71 | $(fordist) zip -r kafka-connect-${VERSION}-{}.zip {} \; && \ 72 | echo Computing checksums... && \ 73 | find . \( -name '*.tar.gz' -or -name '*.zip' \) -exec \ 74 | sh -c '$(SHASUM) {} > {}.sha256sum' \; && \ 75 | cd .. 76 | @echo Done 77 | 78 | man: install 79 | mkdir -p man 80 | kafka-connect --help-man > man/kafka-connect.1 81 | nroff -man man/kafka-connect.1 82 | @echo 83 | @echo ----------------------------------------- 84 | @echo Man page generated at man/kafka-connect.1 85 | @echo ----------------------------------------- 86 | 87 | clean: 88 | $(RM) *.coverprofile 89 | $(RM) -r man 90 | 91 | distclean: clean 92 | $(RM) -r dist/ 93 | $(GO) clean -i $(packages) 94 | 95 | .PHONY: build install test spec coverage browse-coverage 96 | .PHONY: check lint errcheck zen get-devtools 97 | .PHONY: dist get-reltools man release 98 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Kafka Connect CLI 2 | ================= 3 | 4 | [![Release][release-badge]][latest release] 5 | [![Build Status][travis-badge]][build status] 6 | [![Coverage Status][coverage-badge]][coverage status] 7 | [![Go Report Card][go-report-badge]][go report card] 8 | [![GoDoc][godoc-badge]][godoc] 9 | 10 | A fast, portable, self-documenting CLI tool to inspect and manage [Kafka 11 | Connect] connectors via the [REST API]. Because you don't want to be fumbling 12 | through runbooks of `curl` commands when something's going wrong, or ever 13 | really. 14 | 15 | This project also contains a Go library for the Kafka Connect API usable by 16 | other Go tools or applications. See [Using the Go 17 | Library](#using-the-go-library) for details. 18 | 19 | Usage 20 | ----- 21 | 22 | The tool is self-documenting: run `kafka-connect help` or `kafka-connect help 23 | ` when you need a reference. A summary of functionality: 24 | 25 | $ kafka-connect 26 | usage: kafka-connect [] [ ...] 27 | 28 | Command line utility for managing Kafka Connect. 29 | 30 | Flags: 31 | -h, --help Show context-sensitive help (also try --help-long and --help-man). 32 | --version Show application version. 33 | -H, --host=http://localhost:8083/ 34 | Host address for the Kafka Connect REST API instance. 35 | 36 | Commands: 37 | help [...] 38 | Show help. 39 | 40 | list 41 | Lists active connectors. Aliased as 'ls'. 42 | 43 | create [] [] 44 | Creates a new connector instance. 45 | 46 | update 47 | Updates a connector. 48 | 49 | delete 50 | Deletes a connector. Aliased as 'rm'. 51 | 52 | show 53 | Shows information about a connector and its tasks. 54 | 55 | config 56 | Displays configuration of a connector. 57 | 58 | tasks 59 | Displays tasks currently running for a connector. 60 | 61 | status 62 | Gets current status of a connector. 63 | 64 | pause 65 | Pause a connector and its tasks. 66 | 67 | resume 68 | Resume a paused connector. 69 | 70 | restart 71 | Restart a connector and its tasks. 72 | 73 | version 74 | Shows kafka-connect version information. 75 | 76 | For examples, see [the Godoc page for the command][cmd doc]. 77 | 78 | The process exits with a zero status when operations are successful and 79 | non-zero in the case of errors. 80 | 81 | [cmd doc]: https://godoc.org/github.com/go-kafka/connect/cmd/kafka-connect 82 | 83 | ### Manual Page ### 84 | 85 | If you'd like a `man` page, you can generate one and place it on your 86 | `MANPATH`: 87 | 88 | ```sh 89 | $ kafka-connect --help-man > /usr/local/share/man/man1/kafka-connect.1 90 | ``` 91 | 92 | ### Options ### 93 | 94 | Expanded details for select parameters: 95 | 96 | - `--host / -H`: API host address, default `http://localhost:8083/`. Can be set 97 | with environment variable `KAFKA_CONNECT_CLI_HOST`. Note that you can target 98 | any host in a Kafka Connect cluster. 99 | 100 | Installation 101 | ------------ 102 | 103 | Binary releases [are available on GitHub][releases], signed and with checksums. 104 | 105 | Fetch the appropriate version for your platform and place it somewhere on your 106 | `PATH`. The YOLO way: 107 | 108 | ```sh 109 | $ curl -L https://github.com/go-kafka/connect/releases/download/cli-v0.9.0/kafka-connect-v0.9.0-linux-amd64.zip 110 | $ unzip kafka-connect-v0.9.0-linux-amd64.zip 111 | $ mv linux-amd64/kafka-connect /usr/local/bin/ 112 | ``` 113 | 114 | The prudent way: 115 | 116 | ```sh 117 | $ curl -L https://github.com/go-kafka/connect/releases/download/cli-v0.9.0/kafka-connect-v0.9.0-linux-amd64.zip 118 | $ curl -L https://github.com/go-kafka/connect/releases/download/cli-v0.9.0/kafka-connect-v0.9.0-linux-amd64.zip.sha256sum 119 | # Verify integrity of the archive file, on OS X try shasum --check 120 | $ sha256sum --check kafka-connect-v0.9.0-linux-amd64.zip.sha256sum 121 | $ unzip kafka-connect-v0.9.0-linux-amd64.zip 122 | $ mv linux-amd64/kafka-connect /usr/local/bin/ 123 | ``` 124 | 125 | Or best of all, the careful way: 126 | 127 | ```sh 128 | $ curl -L https://github.com/go-kafka/connect/releases/download/cli-v0.9.0/kafka-connect-v0.9.0-linux-amd64.zip 129 | $ unzip kafka-connect-v0.9.0-linux-amd64.zip 130 | # Verify signature of the binary: 131 | $ gpg --verify linux-amd64/kafka-connect{.asc,} 132 | $ mv linux-amd64/kafka-connect /usr/local/bin/ 133 | ``` 134 | 135 | You can find my GPG key distributed on keyservers with ID `8638EE95`. The 136 | fingerprint is: 137 | 138 | 23D6 18B5 3AB8 209F F172 C070 6E5C D3ED 8638 EE95 139 | 140 | For a more detailed primer on GPG signatures and key authenticity, check out 141 | [the Apache Software Foundation's doc](http://www.apache.org/info/verification.html). 142 | 143 | *Cross-compiled binaries are possibly untested—please report any issues. If you 144 | would like a binary build for a platform that is not currently published, I'm 145 | happy to make one available as long as Go can cross-compile it without 146 | problem—please open an issue.* 147 | 148 | To build your own version from source, see the below [Building and 149 | Development](#building-and-development) section. 150 | 151 | ### Command Completion ### 152 | 153 | Shell completion is built in for bash and zsh, just add the following to your 154 | shell profile initialization (`~/.bash_profile` or the like): 155 | 156 | ```sh 157 | which kafka-connect >/dev/null && eval "$(kafka-connect --completion-script-bash)" 158 | ``` 159 | 160 | Predictably, use `--completion-script-zsh` for zsh. 161 | 162 | Building and Development 163 | ------------------------ 164 | 165 | This project is implemented in Go and uses [Go 1.11 modules] to achieve 166 | reproducible builds. 167 | 168 | Once you [have a working Go toolchain][write go], it is simple to build like 169 | any Go project that uses modules: 170 | 171 | ```sh 172 | # In someplace you'd like to keep your work: 173 | $ git clone git@github.com:go-kafka/connect.git 174 | $ cd connect 175 | $ go build # or 176 | $ go install # or 177 | $ go test # etc. 178 | ``` 179 | 180 | Note that you _do not_ need to use a workspace, i.e. `$GOPATH`. In fact, you 181 | should not, or else you'll need to set `GO111MODULE=on` in your shell 182 | environment to force module-aware mode on. 183 | 184 | Cross-compiling is again standard Go procedure: set `GOOS` and `GOARCH`. For 185 | example if you wanted to build a CLI tool binary for Linux on ARM: 186 | 187 | ```sh 188 | $ env GOOS=linux GOARCH=arm go build ./cmd/... 189 | $ file ./kafka-connect 190 | kafka-connect: ELF 32-bit LSB executable, ARM, version 1 (SYSV), statically linked, not stripped 191 | ``` 192 | 193 | #### Testing with Ginkgo #### 194 | 195 | This project uses the [Ginkgo] BDD testing library. You can run the tests 196 | normally with `go test` or `make test`. If you wish to use additional features 197 | of [the Ginkgo CLI tool][ginkgo cli] like `watch` mode or generating stub test 198 | files, etc. you'll need to install it using: 199 | 200 | $ go get github.com/onsi/ginkgo/ginkgo 201 | 202 | ### Using the Go Library ### 203 | 204 | [![GoDoc][godoc-badge]][godoc] 205 | 206 | To use the Go library, simply use `go get` and import it in your code as usual: 207 | 208 | ```sh 209 | $ go get github.com/go-kafka/connect 210 | ``` 211 | 212 | The library has no dependencies beyond the standard library. Dependencies in 213 | this repository's `go.mod` are for the CLI tool (the `cmd` sub-package, not 214 | installed unless you append `/...` to the `go get` command above). 215 | 216 | See the API documentation linked above for examples. 217 | 218 | Versions 219 | -------- 220 | 221 | For information about versioning policy and compatibility status please see 222 | [the release notes](HISTORY.md). 223 | 224 | Alternatives 225 | ------------ 226 | 227 | 228 | 229 | When I wanted a tool like this, I found this one. It's written in Scala—I <3 230 | Scala, but JVM start-up time is sluggish for CLI tools, and it's much easier to 231 | distribute self-contained native binaries to management hosts that don't 232 | require a JVM installed. 233 | 234 | Similar things can be said of Kafka's packaged management scripts, which are 235 | less ergonomic. Hence, I wrote this Go variant. 236 | 237 | Kudos to the kafka-connect-tools authors for inspiration. 238 | 239 | Contributing 240 | ------------ 241 | 242 | Please see [the Contributing Guide](CONTRIBUTING.md)! 243 | 244 | License 245 | ------- 246 | 247 | The library and CLI tool are made available under the terms of the MIT license, 248 | see the [LICENSE](LICENSE) file for full details. 249 | 250 | 251 | [Kafka Connect]: http://docs.confluent.io/current/connect/intro.html 252 | [REST API]: http://docs.confluent.io/current/connect/userguide.html#rest-interface 253 | [releases]: https://github.com/go-kafka/connect/releases 254 | [Go 1.11 modules]: https://github.com/golang/go/wiki/Modules 255 | [write go]: https://golang.org/doc/install 256 | [Ginkgo]: https://onsi.github.io/ginkgo/ 257 | [ginkgo cli]: https://onsi.github.io/ginkgo/#the-ginkgo-cli 258 | 259 | [release-badge]: https://img.shields.io/github/release/go-kafka/connect.svg 260 | [latest release]: https://github.com/go-kafka/connect/releases/latest 261 | [travis-badge]:https://travis-ci.org/go-kafka/connect.svg?branch=master 262 | [build status]: https://travis-ci.org/go-kafka/connect 263 | [coverage-badge]: https://codecov.io/gh/go-kafka/connect/branch/master/graph/badge.svg 264 | [coverage status]: https://codecov.io/gh/go-kafka/connect 265 | [go-report-badge]: https://goreportcard.com/badge/github.com/go-kafka/connect 266 | [go report card]: https://goreportcard.com/report/github.com/go-kafka/connect 267 | [godoc-badge]: http://img.shields.io/badge/godoc-reference-blue.svg?style=flat 268 | [godoc]: https://godoc.org/github.com/go-kafka/connect 269 | 270 | 271 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "net/http" 10 | "net/url" 11 | ) 12 | 13 | const ( 14 | // StatusUnprocessableEntity is the status code returned when sending a 15 | // request with invalid fields. 16 | StatusUnprocessableEntity = 422 17 | ) 18 | 19 | const ( 20 | // DefaultHostURL is the default HTTP host used for connecting to a Kafka 21 | // Connect REST API. 22 | DefaultHostURL = "http://localhost:8083/" 23 | userAgent = "go-kafka/0.9 connect/" + Version 24 | ) 25 | 26 | // A Client manages communication with the Kafka Connect REST API. 27 | type Client struct { 28 | host *url.URL // Base host URL for API requests. 29 | 30 | // HTTP client used to communicate with the API. By default 31 | // http.DefaultClient will be used. 32 | HTTPClient *http.Client 33 | 34 | // User agent used when communicating with the Kafka Connect API. 35 | UserAgent string 36 | } 37 | 38 | // NewClient returns a new Kafka Connect API client that communicates with the 39 | // optional host. If no host is given, DefaultHostURL (localhost) is used. 40 | func NewClient(host ...string) *Client { 41 | var hostURL *url.URL 42 | var err error 43 | 44 | switch len(host) { 45 | case 0: 46 | hostURL, _ = url.Parse(DefaultHostURL) 47 | case 1: 48 | hostURL, err = url.Parse(host[0]) 49 | if err != nil { 50 | panic(err.Error()) 51 | } 52 | default: 53 | panic("only one host URL can be given") 54 | } 55 | 56 | return &Client{host: hostURL, UserAgent: userAgent} 57 | } 58 | 59 | func (c *Client) httpClient() *http.Client { 60 | if c.HTTPClient == nil { 61 | return http.DefaultClient 62 | } 63 | return c.HTTPClient 64 | } 65 | 66 | // Host returns the API root URL the Client is configured to talk to. 67 | func (c *Client) Host() string { 68 | return c.host.String() 69 | } 70 | 71 | // NewRequest creates an API request. A relative URL can be provided in path, 72 | // in which case it is resolved relative to the BaseURL of the Client. 73 | // Relative URLs should always be specified without a preceding slash. If 74 | // specified, the value pointed to by body is JSON-encoded and included as the 75 | // request body. 76 | func (c *Client) NewRequest(method, path string, body interface{}) (*http.Request, error) { 77 | rel, err := url.Parse(path) 78 | if err != nil { 79 | return nil, err 80 | } 81 | 82 | url := c.host.ResolveReference(rel) 83 | 84 | var contentType string 85 | var buf io.ReadWriter 86 | if body != nil { 87 | buf = new(bytes.Buffer) 88 | err := json.NewEncoder(buf).Encode(body) 89 | if err != nil { 90 | return nil, err 91 | } 92 | contentType = "application/json" 93 | } 94 | 95 | request, err := http.NewRequest(method, url.String(), buf) 96 | if err != nil { 97 | return nil, err 98 | } 99 | 100 | request.Header.Set("Accept", "application/json") 101 | if contentType != "" { 102 | request.Header.Set("Content-Type", contentType) 103 | } 104 | if c.UserAgent != "" { 105 | request.Header.Set("User-Agent", c.UserAgent) 106 | } 107 | 108 | return request, nil 109 | } 110 | 111 | // Do sends an API request and returns the API response. The API response is 112 | // JSON-decoded and stored in the value pointed to by v, or returned as an 113 | // error if an API or HTTP error has occurred. 114 | func (c *Client) Do(req *http.Request, v interface{}) (*http.Response, error) { 115 | response, err := c.httpClient().Do(req) 116 | if err != nil { 117 | return nil, err 118 | } 119 | defer response.Body.Close() 120 | 121 | if response.StatusCode >= 400 { 122 | return response, buildError(req, response) 123 | } 124 | 125 | if v != nil { 126 | err = json.NewDecoder(response.Body).Decode(v) 127 | if err == io.EOF { 128 | err = nil // ignore EOF, empty response body 129 | } 130 | } 131 | 132 | return response, err 133 | } 134 | 135 | // Simple GET helper with no request body. 136 | func (c *Client) get(path string, v interface{}) (*http.Response, error) { 137 | return c.doRequest("GET", path, nil, v) 138 | } 139 | 140 | func (c *Client) delete(path string) (*http.Response, error) { 141 | return c.doRequest("DELETE", path, nil, nil) 142 | } 143 | 144 | func (c *Client) doRequest(method, path string, body, v interface{}) (*http.Response, error) { 145 | request, err := c.NewRequest(method, path, body) 146 | if err != nil { 147 | return nil, err 148 | } 149 | 150 | return c.Do(request, v) 151 | } 152 | 153 | func buildError(req *http.Request, resp *http.Response) error { 154 | apiError := APIError{Response: resp} 155 | data, err := ioutil.ReadAll(resp.Body) 156 | if err == nil && data != nil { 157 | _ = json.Unmarshal(data, &apiError) // Fall back on general error below 158 | } 159 | 160 | // Possibly a general HTTP error, e.g. we're not even talking to a valid 161 | // Kafka Connect API host 162 | if apiError.Code == 0 { 163 | return fmt.Errorf("HTTP %v on %v %v", resp.Status, req.Method, req.URL) 164 | } 165 | return apiError 166 | } 167 | -------------------------------------------------------------------------------- /client_test.go: -------------------------------------------------------------------------------- 1 | package connect_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | . "github.com/go-kafka/connect" 8 | ) 9 | 10 | var _ = Describe("NewClient", func() { 11 | It("uses the default host URL", func() { 12 | client := NewClient() 13 | Expect(client.Host()).To(Equal(DefaultHostURL)) 14 | }) 15 | 16 | It("uses a given host URL", func() { 17 | host := "http://example.com" 18 | client := NewClient(host) 19 | Expect(client.Host()).To(Equal(host)) 20 | }) 21 | 22 | Context("given an invalid host URL", func() { 23 | initClient := func() { 24 | NewClient("&*%$fdasj") 25 | } 26 | 27 | It("panics", func() { 28 | Expect(initClient).To(Panic()) 29 | }) 30 | }) 31 | 32 | Context("given multiple host arguments", func() { 33 | initClient := func() { 34 | NewClient("one", "another") 35 | } 36 | 37 | It("panics", func() { 38 | Expect(initClient).To(Panic()) 39 | }) 40 | }) 41 | }) 42 | -------------------------------------------------------------------------------- /cmd/kafka-connect/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | kafka-connect is a command line utility for managing Kafka Connect. 3 | 4 | With it, you can inspect the status of connector instances running in a Kafka 5 | cluster, start new connectors or update the configuration of existing ones, or 6 | invoke lifecycle operations like pausing or restarting connectors. 7 | 8 | For usage information, please see: 9 | 10 | kafka-connect help 11 | kafka-connect help 12 | 13 | The tool is mostly self-documenting in this manner, but note that you can also 14 | pass data for creating and updating connectors via standard input. These two 15 | invocations are equivalent, but one may be more convenient for a particular 16 | scripting task: 17 | 18 | kafka-connect create --from-file connector.json 19 | cat connector.json | kafka-connect create 20 | 21 | Updating works similarly: 22 | 23 | kafka-connect update connector-name --config config.json 24 | cat config.json | kafka-connect update connector-name 25 | 26 | In these examples, connector.json represents a JSON structure accepted by the 27 | Connect REST API for creating connectors, and config.json is only the config 28 | object of such a structure. The latter can be obtained for an existing connector 29 | with: 30 | 31 | kafka-connect config connector-name 32 | 33 | And the former is the output of the show command minus active tasks—using the jq 34 | tool: 35 | 36 | kafka-connect show connector-name | jq 'del(.tasks)' 37 | 38 | If you have configurations, you can also create new connector instances by 39 | specifying names for them on the command line: 40 | 41 | kafka-connect create new-connector --config config.json 42 | cat config.json | kafka-connect create new-connector 43 | 44 | API Host 45 | 46 | By default kafka-connect will attempt to make requests to a Kafka Connect API 47 | instance running on localhost and the default API port of 8083. This can be 48 | changed by giving a full URL with the --host (or -H) flag, or with the 49 | environment variable KAFKA_CONNECT_CLI_HOST. 50 | 51 | Putting it all together, you might migrate connectors from one cluster to 52 | another: 53 | 54 | kafka-connect show connector-name | jq 'del(.tasks)' | \ 55 | kafka-connect -H http://newcluster:8083 create 56 | kafka-connect delete connector-name 57 | 58 | For complete details of the data structures, see the REST API documentation: 59 | http://docs.confluent.io/latest/connect/userguide.html#connect-userguide-rest. 60 | */ 61 | package main 62 | -------------------------------------------------------------------------------- /cmd/kafka-connect/kafka-connect.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "net/url" 9 | "os" 10 | 11 | "gopkg.in/alecthomas/kingpin.v2" 12 | 13 | "github.com/go-kafka/connect" 14 | ) 15 | 16 | const ( 17 | // Version is the kafka-connect CLI version. 18 | Version = "0.9.0" 19 | versionString = "kafka-connect version " + Version + "\n" + 20 | "go-kafka/connect version " + connect.Version 21 | 22 | hostenv = "KAFKA_CONNECT_CLI_HOST" 23 | ) 24 | 25 | // ValidationError indicates that command arguments break an expected invariant. 26 | type ValidationError struct { 27 | Message string 28 | 29 | // Flag suggesting that a handler should display contextual usage help in 30 | // addition to the short error message (e.g. kingpin.FatalUsageContext). 31 | SuggestUsage bool 32 | } 33 | 34 | func (e ValidationError) Error() string { 35 | return e.Message 36 | } 37 | 38 | // A connectorAction is a function that performs an imperative action on a 39 | // Connector by name. 40 | type connectorAction func(name string) (*http.Response, error) 41 | 42 | var ( 43 | pipedinput bool 44 | 45 | host *url.URL 46 | connName string 47 | 48 | // For matching which execution we dispatch without proliferating strings 49 | listCmd, createCmd, updateCmd, deleteCmd *kingpin.CmdClause 50 | showCmd, configCmd, tasksCmd, statusCmd *kingpin.CmdClause 51 | pauseCmd, resumeCmd, restartCmd *kingpin.CmdClause 52 | versionCmd *kingpin.CmdClause 53 | 54 | newConnectorFilePath, connectorConfigPath string 55 | ) 56 | 57 | func init() { 58 | pipedinput = !isatty(os.Stdin) 59 | } 60 | 61 | // BuildApp constructs the kafka-connect command line interface. 62 | func BuildApp() *kingpin.Application { 63 | app := kingpin.New("kafka-connect", "Command line utility for managing Kafka Connect."). 64 | Version(Version). // man page template messed up by versionString... 65 | Author("Ches Martin"). 66 | UsageWriter(os.Stdout) 67 | 68 | app.HelpFlag.Short('h') 69 | 70 | app.Flag("host", "Host address for the Kafka Connect REST API instance."). 71 | Short('H'). 72 | Default(connect.DefaultHostURL). 73 | Envar(hostenv). 74 | URLVar(&host) 75 | 76 | // The modular style of Kingpin setup might cut down on the non-local vars, 77 | // but it feels pretty heavy and less declarative, so I'm undecided... 78 | listCmd = app.Command("list", "Lists active connectors. Aliased as 'ls'.").Alias("ls") 79 | createCmd = app.Command("create", "Creates a new connector instance.") 80 | updateCmd = app.Command("update", "Updates a connector.") 81 | deleteCmd = app.Command("delete", "Deletes a connector. Aliased as 'rm'.").Alias("rm") 82 | showCmd = app.Command("show", "Shows information about a connector and its tasks.") 83 | configCmd = app.Command("config", "Displays configuration of a connector.") 84 | tasksCmd = app.Command("tasks", "Displays tasks currently running for a connector.") 85 | statusCmd = app.Command("status", "Gets current status of a connector.") 86 | pauseCmd = app.Command("pause", "Pause a connector and its tasks.") 87 | resumeCmd = app.Command("resume", "Resume a paused connector.") 88 | restartCmd = app.Command("restart", "Restart a connector and its tasks.") 89 | versionCmd = app.Command("version", "Shows kafka-connect version information.") 90 | 91 | // TODO: New stuff 92 | // plugin subcommand: list (default), validate 93 | 94 | // Most commands need a connector name, reduce the boilerplate. 95 | addConnectorNameArg := func(cmdName, hint string, required bool) { 96 | command := app.GetCommand(cmdName) 97 | desc := fmt.Sprintf("Name of the connector to %v.", hint) 98 | if required { 99 | command.Arg("name", desc).Required().StringVar(&connName) 100 | } else { 101 | command.Arg("name", desc).StringVar(&connName) 102 | } 103 | } 104 | 105 | addConnectorNameArg("create", "create", false) 106 | hintedByName := []string{"update", "delete", "show", "pause", "resume", "restart"} 107 | for _, name := range hintedByName { 108 | addConnectorNameArg(name, name, true) 109 | } 110 | for _, name := range []string{"config", "tasks", "status"} { 111 | addConnectorNameArg(name, "look up", true) 112 | } 113 | 114 | createCmd.Flag("from-file", "A JSON file matching API request format, including connector name."). 115 | Short('f'). 116 | PlaceHolder("FILE"). 117 | ExistingFileVar(&newConnectorFilePath) 118 | createCmd.Flag("config", "A JSON file containing connector config."). 119 | Short('c'). 120 | PlaceHolder("FILE"). 121 | ExistingFileVar(&connectorConfigPath) 122 | 123 | updateCmd.Flag("config", "A JSON file containing connector config."). 124 | Short('c'). 125 | PlaceHolder("FILE"). 126 | ExistingFileVar(&connectorConfigPath) 127 | 128 | // Re-initialize global state for in-process tests, yeah kinda gross 129 | connName, newConnectorFilePath, connectorConfigPath = "", "", "" 130 | host = nil 131 | 132 | return app 133 | } 134 | 135 | // ValidateArgs parses and validates CLI arguments, isolated from execution 136 | // logic. Returns user's parsed subcommand if invocation is valid, else error. 137 | func ValidateArgs(app *kingpin.Application, argv []string) (subcommand string, err error) { 138 | // TODO: Employ kingpin Validate, but it's currently difficult to use: 139 | // https://github.com/alecthomas/kingpin/issues/125 140 | if subcommand, err = app.Parse(argv); err != nil { 141 | return 142 | } 143 | 144 | if !(*host).IsAbs() { 145 | msg := fmt.Sprintf("host %v is not a valid absolute URL", (*host).String()) 146 | if os.Getenv(hostenv) != "" { 147 | msg += fmt.Sprintf(" (set by %v)", hostenv) 148 | } 149 | err = ValidationError{msg, false} 150 | return 151 | } 152 | 153 | switch subcommand { 154 | case createCmd.FullCommand(): 155 | if pipedinput && (newConnectorFilePath != "" || connectorConfigPath != "") { 156 | err = ValidationError{"--from-file and --config cannot be used with input from stdin", false} 157 | return 158 | } 159 | 160 | if connName == "" { 161 | if connectorConfigPath != "" { 162 | err = ValidationError{"--config requires a connector name", true} 163 | return 164 | } 165 | if newConnectorFilePath == "" && !pipedinput { 166 | err = ValidationError{"either a connector name or --from-file is required", true} 167 | return 168 | } 169 | } else { 170 | if connectorConfigPath == "" && !pipedinput { 171 | err = ValidationError{"--config is required with a connector name", true} 172 | return 173 | } 174 | // Kingpin v3 might give us first-class mutual exclusivity support with 175 | // nice usage output: https://github.com/alecthomas/kingpin/issues/103 176 | if newConnectorFilePath != "" { 177 | err = ValidationError{"--from-file and --config are mutually exclusive", true} 178 | return 179 | } 180 | } 181 | case updateCmd.FullCommand(): 182 | if pipedinput && connectorConfigPath != "" { 183 | err = ValidationError{"--config cannot be used with input from stdin", false} 184 | return 185 | } 186 | if connectorConfigPath == "" && !pipedinput { 187 | err = ValidationError{"configuration input is required, try --config or pipe to stdin", true} 188 | return 189 | } 190 | } 191 | 192 | return 193 | } 194 | 195 | func main() { 196 | app := BuildApp() 197 | argv := os.Args[1:] 198 | subcommand, err := ValidateArgs(app, argv) 199 | 200 | if err != nil { 201 | if verr, ok := err.(ValidationError); ok && !verr.SuggestUsage { 202 | app.Fatalf(verr.Error()) 203 | } 204 | context, _ := app.ParseContext(argv) 205 | app.FatalUsageContext(context, err.Error()) 206 | } 207 | 208 | // Localize use of os.Exit because it doesn't run deferreds 209 | app.FatalIfError(run(subcommand), "") 210 | } 211 | 212 | func run(subcommand string) error { 213 | client := connect.NewClient(host.String()) 214 | 215 | // Dispatch subcommands 216 | switch subcommand { 217 | case listCmd.FullCommand(): 218 | return maybePrintAPIResult(client.ListConnectors()) 219 | 220 | case createCmd.FullCommand(): 221 | // TODO: verify/improve error output of 409 Conflict 222 | return createConnector(connName, client) 223 | 224 | case updateCmd.FullCommand(): 225 | var config connect.ConnectorConfig 226 | source := findInputSource() 227 | if err := decodeConnectorConfig(source, &config); err != nil { 228 | return err 229 | } 230 | return maybePrintAPIResult(client.UpdateConnectorConfig(connName, config)) 231 | 232 | case deleteCmd.FullCommand(): 233 | // TODO: verify error output of 409 Conflict 234 | return affectConnector(connName, client.DeleteConnector, "Deleted") 235 | 236 | case showCmd.FullCommand(): 237 | return maybePrintAPIResult(client.GetConnector(connName)) 238 | 239 | case configCmd.FullCommand(): 240 | return maybePrintAPIResult(client.GetConnectorConfig(connName)) 241 | 242 | case tasksCmd.FullCommand(): 243 | return maybePrintAPIResult(client.GetConnectorTasks(connName)) 244 | 245 | case statusCmd.FullCommand(): 246 | return maybePrintAPIResult(client.GetConnectorStatus(connName)) 247 | 248 | case pauseCmd.FullCommand(): 249 | return affectConnector(connName, client.PauseConnector, "Paused") 250 | 251 | case resumeCmd.FullCommand(): 252 | return affectConnector(connName, client.ResumeConnector, "Resumed") 253 | 254 | case restartCmd.FullCommand(): 255 | // TODO: verify error output of 409 Conflict 256 | return affectConnector(connName, client.RestartConnector, "Restarted") 257 | 258 | case versionCmd.FullCommand(): 259 | _, err := fmt.Println(versionString) 260 | return err 261 | 262 | default: // won't reach here, arg parsing handles unknown commands 263 | return fmt.Errorf("command `%v` is missing implementation", subcommand) 264 | } 265 | } 266 | 267 | func maybePrintAPIResult(data interface{}, resp *http.Response, err error) error { 268 | if err != nil { 269 | return err 270 | } 271 | 272 | if output, err := formatPrettyJSON(data); err == nil { 273 | fmt.Println(output) 274 | } 275 | 276 | return err 277 | } 278 | 279 | func affectConnector(name string, action connectorAction, desc string) error { 280 | _, err := action(name) 281 | if err == nil { 282 | fmt.Printf("%v connector %v.\n", desc, name) 283 | } 284 | 285 | return err 286 | } 287 | 288 | func createConnector(name string, client *connect.Client) (err error) { 289 | var connector connect.Connector 290 | source := findInputSource() 291 | 292 | if source == newConnectorFilePath || name == "" { 293 | // kafka-connect create --from-file 294 | // cat connector.json | kafka-connect create 295 | err = decodeConnectorConfig(source, &connector) 296 | } else { 297 | // kafka-connect create my-conn-name --config 298 | // cat config.json | kafka-connect create my-conn-name 299 | var config connect.ConnectorConfig 300 | err = decodeConnectorConfig(source, &config) 301 | connector = connect.Connector{Name: name, Config: config} 302 | } 303 | 304 | if err != nil { 305 | return 306 | } 307 | 308 | // Go's JSON decoding is not strict so it can succeed even if we got 309 | // something that isn't a Connector (e.g. a ConnectorConfig passed to 310 | // create). The API handles bad input poorly (500s instead of 422), so try 311 | // to give the user a better error than HTTP does. 312 | if connector.Config == nil { 313 | return fmt.Errorf("input was not a valid connector (%v)", source) 314 | } 315 | 316 | // The API dubiously allows creating connectors with blank names... That's 317 | // probably a mistake, let's try to avoid it. It also sometimes returns name 318 | // as an attribute of config, so this might be present in roundtrip 319 | // scripting. 320 | // TODO: could apply this workaround in the library, but I'd rather get 321 | // the behavior acknowledged as a bug and not do that. 322 | if connector.Name == "" && connector.Config["name"] != "" { 323 | connector.Name = connector.Config["name"] 324 | } 325 | 326 | if _, err = client.CreateConnector(&connector); err == nil { 327 | if output, err := formatPrettyJSON(connector); err == nil { 328 | fmt.Println(output) 329 | } 330 | } 331 | 332 | return 333 | } 334 | 335 | // Are we getting data via stdin, create --from-file, or --config? 336 | func findInputSource() (path string) { 337 | if pipedinput { 338 | return os.Stdin.Name() 339 | } 340 | if newConnectorFilePath != "" { 341 | return newConnectorFilePath 342 | } 343 | if connectorConfigPath != "" { 344 | return connectorConfigPath 345 | } 346 | return 347 | } 348 | 349 | // Attempts to unmarshal data from source into dst. This should be a pointer to 350 | // a Connector or ConnectorConfig expected to be found in source. 351 | func decodeConnectorConfig(source string, dst interface{}) error { 352 | // TODO: should really buffer stdin just in case... 353 | contents, err := ioutil.ReadFile(source) 354 | if err != nil { 355 | return err 356 | } 357 | if err := json.Unmarshal(contents, dst); err != nil { 358 | return fmt.Errorf("input was not a valid connector configuration (%v)", source) 359 | } 360 | 361 | return nil 362 | } 363 | 364 | // TODO: Some kind of formatter abstraction 365 | func formatPrettyJSON(v interface{}) (string, error) { 366 | pretty, err := json.MarshalIndent(v, "", " ") 367 | if err != nil { 368 | return "", err 369 | } 370 | return string(pretty), nil 371 | } 372 | 373 | // TODO: This probably doesn't work on Windows. 374 | // https://github.com/mattn/go-isatty 375 | func isatty(file *os.File) bool { 376 | stat, err := file.Stat() 377 | if err != nil { 378 | return false 379 | } 380 | return (stat.Mode() & os.ModeCharDevice) != 0 381 | } 382 | -------------------------------------------------------------------------------- /cmd/kafka-connect/kafka-connect_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "os/exec" 7 | 8 | . "github.com/onsi/ginkgo" 9 | . "github.com/onsi/gomega" 10 | . "github.com/onsi/gomega/gbytes" 11 | . "github.com/onsi/gomega/gexec" 12 | "gopkg.in/alecthomas/kingpin.v2" 13 | 14 | . "github.com/go-kafka/connect/cmd/kafka-connect" 15 | ) 16 | 17 | var _ = Describe("kafka-connect CLI", func() { 18 | var command *exec.Cmd 19 | 20 | Context("with no arguments", func() { 21 | BeforeEach(func() { 22 | command = exec.Command(pathToCLI) 23 | }) 24 | 25 | It("executes successfully", func() { 26 | session, err := Start(command, nil, nil) // Don't need to see this output with -v 27 | Expect(err).NotTo(HaveOccurred()) 28 | Eventually(session).Should(Exit(0)) 29 | }) 30 | 31 | It("outputs usage help", func() { 32 | session, _ := Start(command, nil, nil) 33 | Eventually(session).Should(Say("usage: kafka-connect")) 34 | }) 35 | }) 36 | }) 37 | 38 | var _ = Describe("Argument Validation", func() { 39 | var app *kingpin.Application 40 | var argv []string 41 | var subcommand string 42 | var err error 43 | 44 | JustBeforeEach(func() { 45 | app = BuildApp() 46 | subcommand, err = ValidateArgs(app, argv) 47 | }) 48 | 49 | Describe("for global flags", func() { 50 | Context("with a --host that is not an absolute URL", func() { 51 | BeforeEach(func() { argv = []string{"--host", "asdfjk", "list"} }) 52 | 53 | It("fails", func() { 54 | Expect(err).To(HaveOccurred()) 55 | Expect(err.Error()).To(Equal("host asdfjk is not a valid absolute URL")) 56 | Expect(subcommand).To(Equal("list")) 57 | }) 58 | 59 | It("does not suggest usage", func() { 60 | verr, _ := err.(ValidationError) 61 | Expect(verr.SuggestUsage).To(BeFalse()) 62 | }) 63 | }) 64 | }) 65 | 66 | Describe("for a nonexistent command", func() { 67 | BeforeEach(func() { argv = []string{"asdfjk"} }) 68 | 69 | It("fails", func() { 70 | Expect(err).To(HaveOccurred()) 71 | Expect(err.Error()).To(Equal("expected command but got \"asdfjk\"")) 72 | }) 73 | }) 74 | 75 | Describe("for create", func() { 76 | var existingFilepath string 77 | 78 | BeforeEach(func() { argv = []string{"create"} }) 79 | 80 | Context("without a connector name", func() { 81 | Context("without --from-file", func() { 82 | It("fails", func() { 83 | Expect(err).To(HaveOccurred()) 84 | Expect(err.Error()).To(Equal("either a connector name or --from-file is required")) 85 | }) 86 | 87 | It("suggests usage", func() { 88 | verr, _ := err.(ValidationError) 89 | Expect(verr.SuggestUsage).To(BeTrue()) 90 | }) 91 | }) 92 | 93 | Context("with --config", func() { 94 | BeforeEach(func() { 95 | tmpfile, _ := ioutil.TempFile("", "connector-config") 96 | existingFilepath = tmpfile.Name() 97 | argv = append(argv, "--config", existingFilepath) 98 | }) 99 | 100 | AfterEach(func() { 101 | _ = os.Remove(existingFilepath) 102 | }) 103 | 104 | It("fails", func() { 105 | Expect(err).To(HaveOccurred()) 106 | Expect(err.Error()).To(Equal("--config requires a connector name")) 107 | }) 108 | 109 | It("suggests usage", func() { 110 | verr, _ := err.(ValidationError) 111 | Expect(verr.SuggestUsage).To(BeTrue()) 112 | }) 113 | }) 114 | }) 115 | 116 | Context("with a connector name", func() { 117 | BeforeEach(func() { argv = append(argv, "a-name") }) 118 | 119 | Context("without --config", func() { 120 | It("fails", func() { 121 | Expect(err).To(HaveOccurred(), "argv: %v", argv) 122 | Expect(err.Error()).To(Equal("--config is required with a connector name")) 123 | }) 124 | 125 | It("suggests usage", func() { 126 | verr, _ := err.(ValidationError) 127 | Expect(verr.SuggestUsage).To(BeTrue()) 128 | }) 129 | }) 130 | 131 | Context("with --config", func() { 132 | BeforeEach(func() { 133 | tmpfile, _ := ioutil.TempFile("", "connector-config") 134 | existingFilepath = tmpfile.Name() 135 | argv = append(argv, "--config", existingFilepath) 136 | }) 137 | 138 | AfterEach(func() { 139 | _ = os.Remove(existingFilepath) 140 | }) 141 | 142 | Context("and --from-file", func() { 143 | BeforeEach(func() { 144 | argv = append(argv, "--from-file", existingFilepath) 145 | }) 146 | 147 | It("fails", func() { 148 | Expect(err).To(HaveOccurred()) 149 | Expect(err.Error()).To(Equal("--from-file and --config are mutually exclusive")) 150 | }) 151 | 152 | It("suggests usage", func() { 153 | verr, _ := err.(ValidationError) 154 | Expect(verr.SuggestUsage).To(BeTrue()) 155 | }) 156 | }) 157 | }) 158 | }) 159 | }) 160 | 161 | Describe("for update", func() { 162 | BeforeEach(func() { argv = []string{"update"} }) 163 | 164 | Context("without a connector name", func() { 165 | It("fails", func() { 166 | Expect(err).To(HaveOccurred()) 167 | Expect(err.Error()).To(Equal("required argument 'name' not provided")) 168 | }) 169 | }) 170 | 171 | Context("with a connector name", func() { 172 | BeforeEach(func() { argv = append(argv, "a-name") }) 173 | 174 | Context("without --config", func() { 175 | It("fails", func() { 176 | Expect(err).To(HaveOccurred()) 177 | Expect(err.Error()).To(ContainSubstring("configuration input is required")) 178 | }) 179 | 180 | It("suggests usage", func() { 181 | verr, _ := err.(ValidationError) 182 | Expect(verr.SuggestUsage).To(BeTrue()) 183 | }) 184 | }) 185 | }) 186 | }) 187 | }) 188 | -------------------------------------------------------------------------------- /cmd/kafka-connect/suite_test.go: -------------------------------------------------------------------------------- 1 | package main_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | "github.com/onsi/gomega/gexec" 9 | ) 10 | 11 | var pathToCLI string 12 | 13 | func TestKafkaConnectCLI(t *testing.T) { 14 | RegisterFailHandler(Fail) 15 | RunSpecs(t, "go-kafka/connect CLI Suite") 16 | } 17 | 18 | var _ = BeforeSuite(func() { 19 | var err error 20 | 21 | // Build the executable in a sandbox 22 | pathToCLI, err = gexec.Build("github.com/go-kafka/connect/cmd/kafka-connect") 23 | Expect(err).NotTo(HaveOccurred()) 24 | }) 25 | 26 | var _ = AfterSuite(func() { 27 | gexec.CleanupBuildArtifacts() 28 | }) 29 | -------------------------------------------------------------------------------- /connect_suite_test.go: -------------------------------------------------------------------------------- 1 | package connect_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestConnect(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "go-kafka/connect Library Suite") 13 | } 14 | -------------------------------------------------------------------------------- /connectors.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/http" 7 | ) 8 | 9 | // A Connector represents a Kafka Connect connector instance. 10 | // 11 | // See: http://docs.confluent.io/current/connect/userguide.html#connectors-tasks-and-workers 12 | type Connector struct { 13 | Name string `json:"name"` 14 | Config ConnectorConfig `json:"config,omitempty"` 15 | Tasks []TaskID `json:"tasks,omitempty"` 16 | } 17 | 18 | // ConnectorConfig is a key-value mapping of configuration for connectors, where 19 | // keys are in the form of Java properties. 20 | // 21 | // See: http://docs.confluent.io/current/connect/userguide.html#configuring-connectors 22 | type ConnectorConfig map[string]string 23 | 24 | // A Task is a unit of work dispatched by a Connector to parallelize the work of 25 | // a data copy job. 26 | // 27 | // See: http://docs.confluent.io/current/connect/userguide.html#connectors-tasks-and-workers 28 | type Task struct { 29 | ID TaskID `json:"id"` 30 | Config map[string]string `json:"config"` 31 | } 32 | 33 | // A TaskID has two components, a numerical ID and a connector name by which the 34 | // ID is scoped. 35 | type TaskID struct { 36 | ConnectorName string `json:"connector"` 37 | ID int `json:"task"` 38 | } 39 | 40 | // ConnectorStatus reflects the status of a Connector and state of its Tasks. 41 | // 42 | // Having connector name and a "connector" object at top level is a little 43 | // awkward and produces stuttering, but it's their design, not ours. 44 | type ConnectorStatus struct { 45 | Name string `json:"name"` 46 | Connector ConnectorState `json:"connector"` 47 | Tasks []TaskState `json:"tasks"` 48 | } 49 | 50 | // ConnectorState reflects the running state of a Connector and the worker where 51 | // it is running. 52 | type ConnectorState struct { 53 | State string `json:"state"` 54 | WorkerID string `json:"worker_id"` 55 | } 56 | 57 | // TaskState reflects the running state of a Task and the worker where it is 58 | // running. 59 | type TaskState struct { 60 | ID int `json:"id"` 61 | State string `json:"state"` 62 | WorkerID string `json:"worker_id"` 63 | Trace string `json:"trace,omitempty"` 64 | } 65 | 66 | // TODO: Probably need to URL-encode connector names 67 | 68 | // CreateConnector creates a new connector instance. If successful, conn is 69 | // updated with the connector's state returned by the API, including Tasks. 70 | // 71 | // Passing an object that already contains Tasks produces an error. 72 | // 73 | // See: http://docs.confluent.io/current/connect/userguide.html#post--connectors 74 | func (c *Client) CreateConnector(conn *Connector) (*http.Response, error) { 75 | if len(conn.Tasks) != 0 { 76 | return nil, errors.New("Cannot create Connector with existing Tasks") 77 | } 78 | path := "connectors" 79 | response, err := c.doRequest("POST", path, conn, conn) 80 | return response, err 81 | } 82 | 83 | // ListConnectors retrieves a list of active connector names. 84 | // 85 | // See: http://docs.confluent.io/current/connect/userguide.html#get--connectors 86 | func (c *Client) ListConnectors() ([]string, *http.Response, error) { 87 | path := "connectors" 88 | var names []string 89 | response, err := c.get(path, &names) 90 | return names, response, err 91 | } 92 | 93 | // GetConnector retrieves information about a connector with the given name. 94 | // 95 | // See: http://docs.confluent.io/current/connect/userguide.html#get--connectors-(string-name) 96 | func (c *Client) GetConnector(name string) (*Connector, *http.Response, error) { 97 | path := "connectors/" + name 98 | connector := new(Connector) 99 | response, err := c.get(path, connector) 100 | return connector, response, err 101 | } 102 | 103 | // GetConnectorConfig retrieves configuration for a connector with the given 104 | // name. 105 | // 106 | // See: http://docs.confluent.io/current/connect/userguide.html#get--connectors-(string-name)-config 107 | func (c *Client) GetConnectorConfig(name string) (ConnectorConfig, *http.Response, error) { 108 | path := fmt.Sprintf("connectors/%v/config", name) 109 | config := make(ConnectorConfig) 110 | response, err := c.get(path, &config) 111 | return config, response, err 112 | } 113 | 114 | // GetConnectorTasks retrieves a list of tasks currently running for a connector 115 | // with the given name. 116 | // 117 | // See: http://docs.confluent.io/current/connect/userguide.html#get--connectors-(string-name)-tasks 118 | func (c *Client) GetConnectorTasks(name string) ([]Task, *http.Response, error) { 119 | path := fmt.Sprintf("connectors/%v/tasks", name) 120 | var tasks []Task 121 | response, err := c.get(path, &tasks) 122 | return tasks, response, err 123 | } 124 | 125 | // GetConnectorStatus gets current status of the connector, including whether it 126 | // is running, failed or paused, which worker it is assigned to, error 127 | // information if it has failed, and the state of all its tasks. 128 | // 129 | // See: http://docs.confluent.io/current/connect/userguide.html#get--connectors-(string-name)-status 130 | func (c *Client) GetConnectorStatus(name string) (*ConnectorStatus, *http.Response, error) { 131 | path := fmt.Sprintf("connectors/%v/status", name) 132 | status := new(ConnectorStatus) 133 | response, err := c.get(path, status) 134 | return status, response, err 135 | } 136 | 137 | // UpdateConnectorConfig updates configuration for an existing connector with 138 | // the given name, returning the new state of the Connector. 139 | // 140 | // If the connector does not exist, it will be created, and the returned HTTP 141 | // response will indicate a 201 Created status. 142 | // 143 | // See: http://docs.confluent.io/current/connect/userguide.html#put--connectors-(string-name)-config 144 | func (c *Client) UpdateConnectorConfig(name string, config ConnectorConfig) (*Connector, *http.Response, error) { 145 | path := fmt.Sprintf("connectors/%v/config", name) 146 | connector := new(Connector) 147 | response, err := c.doRequest("PUT", path, config, connector) 148 | return connector, response, err 149 | } 150 | 151 | // DeleteConnector deletes a connector with the given name, halting all tasks 152 | // and deleting its configuration. 153 | // 154 | // See: http://docs.confluent.io/current/connect/userguide.html#delete--connectors-(string-name)- 155 | func (c *Client) DeleteConnector(name string) (*http.Response, error) { 156 | return c.delete("connectors/" + name) 157 | } 158 | 159 | // PauseConnector pauses a connector and its tasks, which stops message 160 | // processing until the connector is resumed. Tasks will transition to PAUSED 161 | // state asynchronously. 162 | // 163 | // See: http://docs.confluent.io/current/connect/userguide.html#put--connectors-(string-name)-pause 164 | func (c *Client) PauseConnector(name string) (*http.Response, error) { 165 | path := fmt.Sprintf("connectors/%v/pause", name) 166 | return c.doRequest("PUT", path, nil, nil) 167 | } 168 | 169 | // ResumeConnector resumes a paused connector. Tasks will transition to RUNNING 170 | // state asynchronously. 171 | // 172 | // See: http://docs.confluent.io/current/connect/userguide.html#put--connectors-(string-name)-resume 173 | func (c *Client) ResumeConnector(name string) (*http.Response, error) { 174 | path := fmt.Sprintf("connectors/%v/resume", name) 175 | return c.doRequest("PUT", path, nil, nil) 176 | } 177 | 178 | // RestartConnector restarts a connector and its tasks. 179 | // 180 | // See http://docs.confluent.io/current/connect/userguide.html#post--connectors-(string-name)-restart 181 | func (c *Client) RestartConnector(name string) (*http.Response, error) { 182 | path := fmt.Sprintf("connectors/%v/restart", name) 183 | return c.doRequest("POST", path, nil, nil) 184 | } 185 | -------------------------------------------------------------------------------- /connectors_test.go: -------------------------------------------------------------------------------- 1 | package connect_test 2 | 3 | import ( 4 | "net/http" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | "github.com/onsi/gomega/ghttp" 9 | 10 | . "github.com/go-kafka/connect" 11 | ) 12 | 13 | var ( 14 | client *Client 15 | server *ghttp.Server 16 | 17 | jsonAcceptHeader = http.Header{"Accept": []string{"application/json"}} 18 | jsonContentHeader = http.Header{"Content-Type": []string{"application/json"}} 19 | ) 20 | 21 | var _ = Describe("Connector CRUD", func() { 22 | BeforeEach(func() { 23 | server = ghttp.NewServer() 24 | client = NewClient(server.URL()) 25 | }) 26 | 27 | AfterEach(func() { 28 | server.Close() 29 | }) 30 | 31 | fileSourceConfig := ConnectorConfig{ 32 | "connector.class": "FileStreamSource", 33 | "file": "/tmp/test.txt", 34 | "tasks.max": "1", 35 | "topic": "go-kafka-connect-test", 36 | } 37 | 38 | Describe("CreateConnector", func() { 39 | var connector, resultConnector Connector 40 | var statusCode int 41 | 42 | BeforeEach(func() { 43 | connector = Connector{ 44 | Name: "local-file-source", 45 | Config: fileSourceConfig, 46 | } 47 | 48 | server.AppendHandlers( 49 | ghttp.CombineHandlers( 50 | ghttp.VerifyRequest("POST", "/connectors"), 51 | ghttp.VerifyHeader(jsonContentHeader), 52 | ghttp.VerifyHeader(jsonAcceptHeader), 53 | ghttp.VerifyJSONRepresenting(connector), 54 | ghttp.RespondWithJSONEncodedPtr(&statusCode, &resultConnector), 55 | ), 56 | ) 57 | }) 58 | 59 | Context("when a valid Connector is given", func() { 60 | BeforeEach(func() { 61 | statusCode = http.StatusCreated 62 | resultConnector = connector 63 | resultConnector.Tasks = []TaskID{{"local-file-source", 0}} 64 | Expect(connector).NotTo(Equal(resultConnector)) 65 | }) 66 | 67 | It("updates reference with state of newly-created instance", func() { 68 | _, err := client.CreateConnector(&connector) 69 | Expect(err).NotTo(HaveOccurred()) 70 | Expect(connector).To(Equal(resultConnector)) 71 | }) 72 | }) 73 | 74 | // The API ought to return a 422 but it currently returns 500 instead 75 | // (and the response is text/html despite Accept). 76 | // TODO: report this upstream as a bug 77 | Context("when an invalid Connector is given", func() { 78 | var origConnector Connector 79 | 80 | BeforeEach(func() { 81 | statusCode = http.StatusInternalServerError 82 | origConnector = connector 83 | }) 84 | 85 | // TODO: if 422 is returned in the future (see above), assert on 86 | // APIError value. 87 | It("returns an error", func() { 88 | resp, err := client.CreateConnector(&connector) 89 | Expect(err).To(HaveOccurred()) 90 | Expect(resp.StatusCode).To(Equal(http.StatusInternalServerError)) 91 | }) 92 | 93 | It("does not mutate Connector reference", func() { 94 | _, err := client.CreateConnector(&connector) 95 | Expect(err).To(HaveOccurred()) 96 | Expect(connector).To(Equal(origConnector)) 97 | }) 98 | }) 99 | 100 | Context("when a Connector with extant Tasks is given", func() { 101 | BeforeEach(func() { 102 | connector.Tasks = []TaskID{{"local-file-source", 0}} 103 | }) 104 | 105 | It("returns an error", func() { 106 | _, err := client.CreateConnector(&connector) 107 | Expect(err).To(HaveOccurred()) 108 | Expect(err.Error()).To(Equal("Cannot create Connector with existing Tasks")) 109 | }) 110 | }) 111 | }) 112 | 113 | Describe("ListConnectors", func() { 114 | BeforeEach(func() { 115 | server.AppendHandlers( 116 | ghttp.CombineHandlers( 117 | ghttp.VerifyRequest("GET", "/connectors"), 118 | ghttp.VerifyHeader(jsonAcceptHeader), 119 | ghttp.RespondWith(http.StatusOK, `["test", "phony-hdfs-sink"]`), 120 | ), 121 | ) 122 | }) 123 | 124 | It("returns list of connector names", func() { 125 | names, _, err := client.ListConnectors() 126 | Expect(err).NotTo(HaveOccurred()) 127 | Expect(names).To(Equal([]string{"test", "phony-hdfs-sink"})) 128 | }) 129 | }) 130 | 131 | Describe("GetConnector", func() { 132 | var resultConnector interface{} 133 | var statusCode int 134 | 135 | BeforeEach(func() { 136 | resultConnector = Connector{ 137 | Name: "local-file-source", 138 | Config: fileSourceConfig, 139 | Tasks: []TaskID{{"local-file-source", 0}}, 140 | } 141 | 142 | server.AppendHandlers( 143 | ghttp.CombineHandlers( 144 | ghttp.VerifyRequest("GET", "/connectors/local-file-source"), 145 | ghttp.VerifyHeader(jsonAcceptHeader), 146 | ghttp.RespondWithJSONEncodedPtr(&statusCode, &resultConnector), 147 | ), 148 | ) 149 | }) 150 | 151 | Context("when existing connector name is given", func() { 152 | BeforeEach(func() { 153 | statusCode = http.StatusOK 154 | }) 155 | 156 | It("returns a connector", func() { 157 | connector, _, err := client.GetConnector("local-file-source") 158 | Expect(err).NotTo(HaveOccurred()) 159 | Expect(*connector).To(Equal(resultConnector.(Connector))) 160 | }) 161 | }) 162 | 163 | Context("when nonexisting connector name is given", func() { 164 | apiError := APIError{ 165 | Code: 404, 166 | Message: "Connector local-file-source not found", 167 | } 168 | 169 | BeforeEach(func() { 170 | statusCode = http.StatusNotFound 171 | resultConnector = apiError 172 | }) 173 | 174 | It("returns an error response", func() { 175 | connector, resp, err := client.GetConnector("local-file-source") 176 | Expect(err).To(MatchError(err.(APIError))) 177 | Expect(*connector).To(BeZero()) 178 | Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) 179 | }) 180 | }) 181 | }) 182 | 183 | Describe("GetConnectorConfig", func() { 184 | var statusCode int 185 | 186 | BeforeEach(func() { 187 | server.AppendHandlers( 188 | ghttp.CombineHandlers( 189 | ghttp.VerifyRequest("GET", "/connectors/local-file-source/config"), 190 | ghttp.VerifyHeader(jsonAcceptHeader), 191 | ghttp.RespondWithJSONEncodedPtr(&statusCode, &fileSourceConfig), 192 | ), 193 | ) 194 | }) 195 | 196 | Context("when existing connector name is given", func() { 197 | BeforeEach(func() { 198 | statusCode = http.StatusOK 199 | }) 200 | 201 | It("returns configuration for connector", func() { 202 | config, _, err := client.GetConnectorConfig("local-file-source") 203 | Expect(err).NotTo(HaveOccurred()) 204 | Expect(config).To(Equal(fileSourceConfig)) 205 | }) 206 | }) 207 | 208 | Context("when nonexisting connector name is given", func() { 209 | BeforeEach(func() { 210 | statusCode = http.StatusNotFound 211 | }) 212 | 213 | It("returns an error response", func() { 214 | config, resp, err := client.GetConnectorConfig("local-file-source") 215 | Expect(err).To(HaveOccurred()) 216 | Expect(config).To(BeEmpty()) 217 | Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) 218 | }) 219 | }) 220 | }) 221 | 222 | Describe("GetConnectorTasks", func() { 223 | var resultTasks []Task 224 | var statusCode int 225 | 226 | BeforeEach(func() { 227 | resultTasks = []Task{ 228 | { 229 | ID: TaskID{"local-file-source", 0}, 230 | Config: map[string]string{ 231 | "file": "/tmp/test.txt", 232 | "task.class": "org.apache.kafka.connect.file.FileStreamSourceTask", 233 | "topic": "go-kafka-connect-test", 234 | }, 235 | }, 236 | } 237 | 238 | server.AppendHandlers( 239 | ghttp.CombineHandlers( 240 | ghttp.VerifyRequest("GET", "/connectors/local-file-source/tasks"), 241 | ghttp.VerifyHeader(jsonAcceptHeader), 242 | ghttp.RespondWithJSONEncodedPtr(&statusCode, &resultTasks), 243 | ), 244 | ) 245 | }) 246 | 247 | Context("when existing connector name is given", func() { 248 | BeforeEach(func() { 249 | statusCode = http.StatusOK 250 | }) 251 | 252 | It("returns tasks", func() { 253 | tasks, _, err := client.GetConnectorTasks("local-file-source") 254 | Expect(err).NotTo(HaveOccurred()) 255 | Expect(tasks).To(Equal(resultTasks)) 256 | }) 257 | }) 258 | 259 | Context("when nonexisting connector name is given", func() { 260 | BeforeEach(func() { 261 | statusCode = http.StatusNotFound 262 | }) 263 | 264 | It("returns an error response", func() { 265 | tasks, resp, err := client.GetConnectorTasks("local-file-source") 266 | Expect(err).To(HaveOccurred()) 267 | Expect(tasks).To(BeEmpty()) 268 | Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) 269 | }) 270 | }) 271 | }) 272 | 273 | Describe("GetConnectorStatus", func() { 274 | var resultStatus *ConnectorStatus 275 | var statusCode int 276 | 277 | BeforeEach(func() { 278 | resultStatus = &ConnectorStatus{ 279 | Name: "local-file-source", 280 | Connector: ConnectorState{ 281 | State: "RUNNING", 282 | WorkerID: "127.0.0.1:8083", 283 | }, 284 | Tasks: []TaskState{ 285 | { 286 | ID: 0, 287 | State: "RUNNING", 288 | WorkerID: "127.0.0.1:8083", 289 | }, 290 | }, 291 | } 292 | 293 | server.AppendHandlers( 294 | ghttp.CombineHandlers( 295 | ghttp.VerifyRequest("GET", "/connectors/local-file-source/status"), 296 | ghttp.VerifyHeader(jsonAcceptHeader), 297 | ghttp.RespondWithJSONEncodedPtr(&statusCode, &resultStatus), 298 | ), 299 | ) 300 | }) 301 | 302 | Context("when existing connector name is given", func() { 303 | BeforeEach(func() { 304 | statusCode = http.StatusOK 305 | }) 306 | 307 | It("returns connector status", func() { 308 | status, _, err := client.GetConnectorStatus("local-file-source") 309 | Expect(err).NotTo(HaveOccurred()) 310 | Expect(status).To(Equal(resultStatus)) 311 | }) 312 | }) 313 | 314 | Context("when nonexisting connector name is given", func() { 315 | BeforeEach(func() { 316 | statusCode = http.StatusNotFound 317 | }) 318 | 319 | It("returns an error response", func() { 320 | status, resp, err := client.GetConnectorStatus("local-file-source") 321 | Expect(err).To(HaveOccurred()) 322 | Expect(*status).To(BeZero()) 323 | Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) 324 | }) 325 | }) 326 | }) 327 | 328 | Describe("UpdateConnectorConfig", func() { 329 | var statusCode int 330 | 331 | resultConnector := Connector{ 332 | Name: "local-file-source", 333 | Config: fileSourceConfig, 334 | Tasks: []TaskID{{"local-file-source", 0}}, 335 | } 336 | 337 | BeforeEach(func() { 338 | server.AppendHandlers( 339 | ghttp.CombineHandlers( 340 | ghttp.VerifyRequest("PUT", "/connectors/local-file-source/config"), 341 | ghttp.VerifyHeader(jsonContentHeader), 342 | ghttp.VerifyHeader(jsonAcceptHeader), 343 | ghttp.VerifyJSONRepresenting(fileSourceConfig), 344 | ghttp.RespondWithJSONEncodedPtr(&statusCode, &resultConnector), 345 | ), 346 | ) 347 | }) 348 | 349 | Context("when existing connector name is given", func() { 350 | BeforeEach(func() { 351 | statusCode = http.StatusOK 352 | }) 353 | 354 | It("returns updated connector", func() { 355 | connector, resp, err := client.UpdateConnectorConfig("local-file-source", fileSourceConfig) 356 | Expect(err).NotTo(HaveOccurred()) 357 | Expect(connector.Config["file"]).To(Equal("/tmp/test.txt")) 358 | Expect(resp.StatusCode).To(Equal(http.StatusOK)) 359 | }) 360 | }) 361 | 362 | Context("when nonexisting connector name is given", func() { 363 | BeforeEach(func() { 364 | statusCode = http.StatusCreated 365 | }) 366 | 367 | It("returns newly created connector with a Created response", func() { 368 | connector, resp, err := client.UpdateConnectorConfig("local-file-source", fileSourceConfig) 369 | Expect(err).NotTo(HaveOccurred()) 370 | Expect(*connector).To(Equal(resultConnector)) 371 | Expect(resp.StatusCode).To(Equal(http.StatusCreated)) 372 | }) 373 | }) 374 | }) 375 | 376 | Describe("DeleteConnector", func() { 377 | var statusCode int 378 | 379 | BeforeEach(func() { 380 | server.AppendHandlers( 381 | ghttp.CombineHandlers( 382 | ghttp.VerifyRequest("DELETE", "/connectors/local-file-source"), 383 | ghttp.VerifyHeader(jsonAcceptHeader), 384 | ghttp.RespondWithPtr(&statusCode, nil), 385 | ), 386 | ) 387 | }) 388 | 389 | Context("when existing connector name is given", func() { 390 | BeforeEach(func() { 391 | statusCode = http.StatusNoContent 392 | }) 393 | 394 | It("deletes connector", func() { 395 | resp, err := client.DeleteConnector("local-file-source") 396 | Expect(err).NotTo(HaveOccurred()) 397 | Expect(resp.StatusCode).To(Equal(http.StatusNoContent)) 398 | }) 399 | 400 | Context("when rebalance is in process", func() { 401 | BeforeEach(func() { 402 | statusCode = http.StatusConflict 403 | }) 404 | 405 | It("returns error with a conflict response", func() { 406 | resp, err := client.DeleteConnector("local-file-source") 407 | Expect(err).To(HaveOccurred()) 408 | Expect(resp.StatusCode).To(Equal(http.StatusConflict)) 409 | }) 410 | }) 411 | }) 412 | 413 | Context("when nonexisting connector name is given", func() { 414 | BeforeEach(func() { 415 | statusCode = http.StatusNotFound 416 | }) 417 | 418 | It("returns error with a not found response", func() { 419 | resp, err := client.DeleteConnector("local-file-source") 420 | Expect(err).To(HaveOccurred()) 421 | Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) 422 | }) 423 | }) 424 | }) 425 | }) 426 | 427 | var _ = Describe("Connector Lifecycle", func() { 428 | BeforeEach(func() { 429 | server = ghttp.NewServer() 430 | client = NewClient(server.URL()) 431 | }) 432 | 433 | AfterEach(func() { 434 | server.Close() 435 | }) 436 | 437 | Describe("PauseConnector", func() { 438 | var statusCode int 439 | 440 | BeforeEach(func() { 441 | server.AppendHandlers( 442 | ghttp.CombineHandlers( 443 | ghttp.VerifyRequest("PUT", "/connectors/local-file-source/pause"), 444 | ghttp.VerifyHeader(jsonAcceptHeader), 445 | ghttp.RespondWithPtr(&statusCode, nil), 446 | ), 447 | ) 448 | }) 449 | 450 | Context("when existing connector name is given", func() { 451 | BeforeEach(func() { 452 | statusCode = http.StatusAccepted 453 | }) 454 | 455 | It("pauses connector", func() { 456 | resp, err := client.PauseConnector("local-file-source") 457 | Expect(err).NotTo(HaveOccurred()) 458 | Expect(resp.StatusCode).To(Equal(http.StatusAccepted)) 459 | }) 460 | }) 461 | 462 | Context("when nonexisting connector name is given", func() { 463 | BeforeEach(func() { 464 | statusCode = http.StatusNotFound 465 | }) 466 | 467 | It("returns error with a not found response", func() { 468 | resp, err := client.PauseConnector("local-file-source") 469 | Expect(err).To(HaveOccurred()) 470 | Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) 471 | }) 472 | }) 473 | }) 474 | 475 | Describe("ResumeConnector", func() { 476 | var statusCode int 477 | 478 | BeforeEach(func() { 479 | server.AppendHandlers( 480 | ghttp.CombineHandlers( 481 | ghttp.VerifyRequest("PUT", "/connectors/local-file-source/resume"), 482 | ghttp.VerifyHeader(jsonAcceptHeader), 483 | ghttp.RespondWithPtr(&statusCode, nil), 484 | ), 485 | ) 486 | }) 487 | 488 | Context("when existing connector name is given", func() { 489 | BeforeEach(func() { 490 | statusCode = http.StatusAccepted 491 | }) 492 | 493 | It("resumes connector", func() { 494 | resp, err := client.ResumeConnector("local-file-source") 495 | Expect(err).NotTo(HaveOccurred()) 496 | Expect(resp.StatusCode).To(Equal(http.StatusAccepted)) 497 | }) 498 | }) 499 | 500 | Context("when nonexisting connector name is given", func() { 501 | BeforeEach(func() { 502 | statusCode = http.StatusNotFound 503 | }) 504 | 505 | It("returns error with a not found response", func() { 506 | resp, err := client.ResumeConnector("local-file-source") 507 | Expect(err).To(HaveOccurred()) 508 | Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) 509 | }) 510 | }) 511 | }) 512 | 513 | Describe("RestartConnector", func() { 514 | var statusCode int 515 | 516 | BeforeEach(func() { 517 | server.AppendHandlers( 518 | ghttp.CombineHandlers( 519 | ghttp.VerifyRequest("POST", "/connectors/local-file-source/restart"), 520 | ghttp.VerifyHeader(jsonAcceptHeader), 521 | ghttp.RespondWithPtr(&statusCode, nil), 522 | ), 523 | ) 524 | }) 525 | 526 | Context("when existing connector name is given", func() { 527 | BeforeEach(func() { 528 | statusCode = http.StatusOK 529 | }) 530 | 531 | It("restarts connector", func() { 532 | resp, err := client.RestartConnector("local-file-source") 533 | Expect(err).NotTo(HaveOccurred()) 534 | Expect(resp.StatusCode).To(Equal(http.StatusOK)) 535 | }) 536 | 537 | Context("when rebalance is in process", func() { 538 | BeforeEach(func() { 539 | statusCode = http.StatusConflict 540 | }) 541 | 542 | It("returns error with a conflict response", func() { 543 | resp, err := client.RestartConnector("local-file-source") 544 | Expect(err).To(HaveOccurred()) 545 | Expect(resp.StatusCode).To(Equal(http.StatusConflict)) 546 | }) 547 | }) 548 | }) 549 | 550 | Context("when nonexisting connector name is given", func() { 551 | BeforeEach(func() { 552 | // The API actually throws a 500 on POST to nonexistent 553 | statusCode = http.StatusInternalServerError 554 | }) 555 | 556 | It("returns error with a server error response", func() { 557 | resp, err := client.RestartConnector("local-file-source") 558 | Expect(err).To(HaveOccurred()) 559 | Expect(resp.StatusCode).To(Equal(http.StatusInternalServerError)) 560 | }) 561 | }) 562 | }) 563 | }) 564 | -------------------------------------------------------------------------------- /err-excludes.txt: -------------------------------------------------------------------------------- 1 | (io.ReadCloser).Close 2 | -------------------------------------------------------------------------------- /errors.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | ) 7 | 8 | // APIError holds information returned from a Kafka Connect API instance about 9 | // why an API call failed. 10 | type APIError struct { 11 | Code int `json:"error_code"` 12 | Message string `json:"message"` 13 | Response *http.Response // HTTP response that caused this error 14 | } 15 | 16 | func (e APIError) Error() string { 17 | return fmt.Sprintf("%v (HTTP %d)", e.Message, e.Code) 18 | } 19 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/go-kafka/connect 2 | 3 | require ( 4 | github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 // indirect 5 | github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 // indirect 6 | github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 // indirect 7 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect 8 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect 9 | github.com/davecgh/go-spew v1.1.1 // indirect 10 | github.com/kisielk/errcheck v1.2.0 11 | github.com/mattn/go-isatty v0.0.4 // indirect 12 | github.com/mitchellh/gox v1.0.1 13 | github.com/onsi/ginkgo v1.6.0 14 | github.com/onsi/gomega v1.4.2 15 | github.com/pmezard/go-difflib v1.0.0 // indirect 16 | github.com/sergi/go-diff v1.0.0 // indirect 17 | github.com/stretchr/testify v1.2.2 // indirect 18 | golang.org/x/lint v0.0.0-20190409202823-959b441ac422 19 | gopkg.in/alecthomas/kingpin.v2 v2.2.2 20 | ) 21 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U= 2 | github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI= 3 | github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo= 4 | github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0= 5 | github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 h1:GDQdwm/gAcJcLAKQQZGOJ4knlw+7rfEQQcmwTbt4p5E= 6 | github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= 7 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= 8 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 9 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= 10 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 11 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 12 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 13 | github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= 14 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 15 | github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= 16 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 17 | github.com/hashicorp/go-version v1.0.0 h1:21MVWPKDphxa7ineQQTrCU5brh7OuVVAzGOCnnCPtE8= 18 | github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= 19 | github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= 20 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 21 | github.com/kisielk/errcheck v1.2.0 h1:reN85Pxc5larApoH1keMBiu2GWtPqXQ1nc9gx+jOU+E= 22 | github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= 23 | github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= 24 | github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 25 | github.com/mitchellh/gox v1.0.1 h1:x0jD3dcHk9a9xPSDN6YEL4xL6Qz0dvNYm8yZqui5chI= 26 | github.com/mitchellh/gox v1.0.1/go.mod h1:ED6BioOGXMswlXa2zxfh/xdd5QhwYliBFn9V18Ap4z4= 27 | github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWeY= 28 | github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= 29 | github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= 30 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 31 | github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I= 32 | github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 33 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 34 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 35 | github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= 36 | github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= 37 | github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= 38 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 39 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 40 | golang.org/x/lint v0.0.0-20190409202823-959b441ac422 h1:QzoH/1pFpZguR8NrRHLcO6jKqfv2zpuSqZLgdm7ZmjI= 41 | golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 42 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= 43 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 44 | golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= 45 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 46 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= 47 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 48 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs= 49 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 50 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= 51 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 52 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 53 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 54 | golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 55 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ= 56 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 57 | gopkg.in/alecthomas/kingpin.v2 v2.2.2 h1:VBV8OzdyP4EuRQy9lkr5gkIGaGt5FRC0JH/+TmQVfd8= 58 | gopkg.in/alecthomas/kingpin.v2 v2.2.2/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 59 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 60 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 61 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 62 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 63 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 64 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 65 | gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= 66 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 67 | -------------------------------------------------------------------------------- /tools.go: -------------------------------------------------------------------------------- 1 | // +build tools 2 | 3 | // Package tools manages development tool versions through the module system. 4 | // 5 | // See https://github.com/golang/go/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module 6 | package tools 7 | 8 | import ( 9 | _ "github.com/kisielk/errcheck" 10 | _ "github.com/mitchellh/gox" 11 | _ "golang.org/x/lint/golint" 12 | ) 13 | -------------------------------------------------------------------------------- /version.go: -------------------------------------------------------------------------------- 1 | // Package connect provides a client for the Kafka Connect REST API. 2 | package connect 3 | 4 | // Version is the go-kafka/connect library version. 5 | const Version = "0.9.0" 6 | --------------------------------------------------------------------------------