├── .circleci └── config.yml ├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── pull_request_template.md ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── .readthedocs.yml ├── .vscode └── launch.json ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── OSSMETADATA ├── README.md ├── SECURITY.md ├── cmd └── connectctl │ └── main.go ├── docs ├── FAQ.md ├── Makefile ├── _static │ └── custom.css ├── cli │ ├── connectctl.md │ ├── connectctl_connectors.md │ ├── connectctl_connectors_add.md │ ├── connectctl_connectors_list.md │ ├── connectctl_connectors_manage.md │ ├── connectctl_connectors_pause.md │ ├── connectctl_connectors_remove.md │ ├── connectctl_connectors_restart.md │ ├── connectctl_connectors_resume.md │ ├── connectctl_connectors_status.md │ ├── connectctl_plugins.md │ ├── connectctl_plugins_list.md │ ├── connectctl_plugins_validate.md │ ├── connectctl_version.md │ └── index.rst ├── conf.py ├── index.rst ├── installation.md └── requirements.txt ├── examples ├── connector1.json ├── connector2.json └── connectors.json ├── go.mod ├── go.sum ├── humans.txt ├── internal ├── ctl │ ├── connectors │ │ ├── add.go │ │ ├── connectors.go │ │ ├── list.go │ │ ├── manage.go │ │ ├── pause.go │ │ ├── remove.go │ │ ├── restart.go │ │ ├── resume.go │ │ ├── status.go │ │ └── status_test.go │ ├── flags.go │ ├── output.go │ ├── plugins │ │ ├── list.go │ │ ├── plugins.go │ │ └── validate.go │ ├── tasks │ │ ├── get.go │ │ ├── list.go │ │ ├── output.go │ │ ├── restart.go │ │ ├── status.go │ │ ├── tasks.go │ │ ├── tasks_test.go │ │ ├── utils.go │ │ └── utils_test.go │ ├── transform.go │ └── version │ │ └── version.go ├── healthcheck │ └── healthcheck.go ├── logging │ └── logging.go └── version │ └── version.go └── pkg ├── client └── connect │ ├── LICENSE │ ├── client.go │ ├── cluster.go │ ├── connectors.go │ ├── errors.go │ ├── errors_test.go │ └── plugins.go ├── manager ├── cluster.go ├── config.go ├── crud.go ├── logger.go ├── manage.go ├── manager.go ├── manager_test.go ├── mocks │ └── client.go ├── pause.go ├── plugins.go ├── restart.go ├── restart_policy.go ├── restart_policy_test.go ├── resume.go ├── status.go ├── tasks.go ├── tasks_test.go └── types.go ├── signal ├── signal.go ├── signal_posix.go └── signal_windows.go └── sources ├── sources.go └── sources_test.go /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | jobs: 4 | buildtest: 5 | docker: 6 | - image: circleci/golang:1.14.1 7 | steps: 8 | - checkout 9 | - run: make ci 10 | - save_cache: 11 | key: go-mod-{{ checksum "go.sum" }} 12 | paths: 13 | - "/go/pkg/mod" 14 | release: 15 | docker: 16 | - image: circleci/golang:1.14.1 17 | steps: 18 | - checkout 19 | - restore_cache: 20 | keys: 21 | - go-mod-{{ checksum "go.sum" }} 22 | - run: make release 23 | docs-build: 24 | docker: 25 | - image: python:3.7 26 | steps: 27 | - checkout 28 | - run: 29 | name: Install dependencies 30 | command: pip install -r docs/requirements.txt 31 | - run: 32 | name: Build docs 33 | command: cd docs/ && make html 34 | - persist_to_workspace: 35 | root: docs/_build 36 | paths: html 37 | docs-deploy: 38 | docker: 39 | - image: node:8.10.0 40 | steps: 41 | - checkout 42 | - attach_workspace: 43 | at: docs/_build 44 | - run: 45 | name: Install and configure dependencies 46 | command: | 47 | npm install -g gh-pages@2.1.1 48 | git config user.email "ci-build@90poe.io" 49 | git config user.name "ci-build" 50 | - add_ssh_keys: 51 | fingerprints: 52 | - "b2:1b:aa:03:24:de:cd:aa:aa:0d:f6:ad:be:c9:85:19" 53 | - run: 54 | name: Deploy docs to gh-pages branch 55 | command: gh-pages --dotfiles --message "chore - docs published to GitHub Pages" --dist docs/_build/html 56 | add-tag: 57 | docker: 58 | - image: circleci/golang:1.14.1 59 | steps: 60 | - checkout 61 | - add_ssh_keys: 62 | fingerprints: 63 | - "b2:1b:aa:03:24:de:cd:aa:aa:0d:f6:ad:be:c9:85:19" 64 | - run: 65 | name: Pushing git tag 66 | command: | 67 | git config --global user.name "90POE" 68 | git config --global user.email "bot@90poe.io" 69 | 70 | go get -u github.com/sv-tools/bumptag 71 | 72 | bumptag -p -a 73 | workflows: 74 | version: 2 75 | any-commit: 76 | jobs: 77 | - buildtest: 78 | filters: 79 | tags: 80 | ignore: /.*/ 81 | - docs-build: 82 | filters: 83 | tags: 84 | ignore: /.*/ 85 | - add-tag: 86 | requires: 87 | - buildtest 88 | - docs-build 89 | filters: 90 | tags: 91 | ignore: /.*/ 92 | branches: 93 | only: 94 | - master 95 | release: 96 | jobs: 97 | - release: 98 | filters: 99 | branches: 100 | ignore: /.*/ 101 | tags: 102 | only: /v[0-9]+(\.[0-9]+)*(-.*)*/ 103 | - docs-build: 104 | filters: 105 | branches: 106 | ignore: /.*/ 107 | tags: 108 | only: /v[0-9]+(\.[0-9]+)*(-.*)*/ 109 | - docs-deploy: 110 | requires: 111 | - release 112 | - docs-build 113 | filters: 114 | branches: 115 | ignore: /.*/ 116 | tags: 117 | only: /v[0-9]+(\.[0-9]+)*(-.*)*/ 118 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @richardcase @the4thamigo-uk -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Version Information** 24 | Please include the output of the following command: 25 | 26 | `connectctl version` 27 | 28 | **Screenshots** 29 | If applicable, add screenshots to help explain your problem. 30 | 31 | **Additional context** 32 | Add any other context about the problem here. 33 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | **IMPORTANT: Please do not create a Pull Request without creating an issue first.** 2 | 3 | *Any change needs to be discussed before proceeding. Failure to do so may result in the rejection of the pull request.* 4 | 5 | Please provide enough information so that others can review your pull request: 6 | 7 | 8 | 9 | Explain the **details** for making this change. What existing problem does the pull request solve? 10 | 11 | 12 | 13 | **Test plan (required)** 14 | 15 | Demonstrate the code is solid. Example: The exact commands you ran and their output, screenshots / videos if the pull request changes UI. 16 | 17 | 18 | 19 | **Code formatting** 20 | 21 | 22 | 23 | **Closing issues** 24 | 25 | Put `closes #XXXX` in your comment to auto-close the issue that your PR fixes (if such). 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | bin/ 15 | dist/ 16 | 17 | coverage.out 18 | 19 | docs/_build 20 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | # options for analysis running 2 | run: 3 | # default concurrency is a available CPU number 4 | concurrency: 4 5 | 6 | # timeout for analysis, e.g. 30s, 5m, default is 1m 7 | deadline: 5m 8 | 9 | # exit code when at least one issue was found, default is 1 10 | issues-exit-code: 1 11 | 12 | # include test files or not, default is true 13 | tests: false 14 | 15 | # list of build tags, all linters use it. Default is empty list. 16 | #build-tags: 17 | # - mytag 18 | 19 | # which dirs to skip: they won't be analyzed; 20 | # can use regexp here: generated.*, regexp is applied on full path; 21 | # default value is empty list, but next dirs are always skipped independently 22 | # from this option's value: 23 | # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ 24 | skip-dirs: 25 | - ^vendor$ 26 | - ^build$ 27 | - \/usr\/local\/go 28 | 29 | # which files to skip: they will be analyzed, but issues from them 30 | # won't be reported. Default value is empty list, but there is 31 | # no need to include all autogenerated files, we confidently recognize 32 | # autogenerated files. If it's not please let us know. 33 | skip-files: 34 | # - ".*\\.my\\.go$" 35 | # - lib/bad.go 36 | 37 | 38 | # output configuration options 39 | output: 40 | # colored-line-number|line-number|json|tab|checkstyle, default is "colored-line-number" 41 | format: tab 42 | 43 | # print lines of code with issue, default is true 44 | print-issued-lines: true 45 | 46 | # print linter name in the end of issue text, default is true 47 | print-linter-name: true 48 | 49 | 50 | # all available settings of specific linters 51 | linters-settings: 52 | errcheck: 53 | # report about not checking of errors in type assetions: `a := b.(MyStruct)`; 54 | # default is false: such cases aren't reported by default. 55 | check-type-assertions: false 56 | 57 | # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; 58 | # default is false: such cases aren't reported by default. 59 | check-blank: false 60 | govet: 61 | # report about shadowed variables 62 | check-shadowing: true 63 | 64 | # Obtain type information from installed (to $GOPATH/pkg) package files: 65 | # golangci-lint will execute `go install -i` and `go test -i` for analyzed packages 66 | # before analyzing them. 67 | # By default this option is disabled and govet gets type information by loader from source code. 68 | # Loading from source code is slow, but it's done only once for all linters. 69 | # Go-installing of packages first time is much slower than loading them from source code, 70 | # therefore this option is disabled by default. 71 | # But repeated installation is fast in go >= 1.10 because of build caching. 72 | # Enable this option only if all conditions are met: 73 | # 1. you use only "fast" linters (--fast e.g.): no program loading occurs 74 | # 2. you use go >= 1.10 75 | # 3. you do repeated runs (false for CI) or cache $GOPATH/pkg or `go env GOCACHE` dir in CI. 76 | use-installed-packages: false 77 | golint: 78 | # minimal confidence for issues, default is 0.8 79 | min-confidence: 0.8 80 | gofmt: 81 | # simplify code: gofmt with `-s` option, true by default 82 | simplify: false 83 | gocyclo: 84 | # minimal code complexity to report, 30 by default (but we recommend 10-20) 85 | min-complexity: 10 86 | maligned: 87 | # print struct with more effective memory layout or not, false by default 88 | suggest-new: true 89 | dupl: 90 | # tokens count to trigger issue, 150 by default 91 | threshold: 100 92 | goconst: 93 | # minimal length of string constant, 3 by default 94 | min-len: 3 95 | # minimal occurrences count to trigger, 3 by default 96 | min-occurrences: 3 97 | depguard: 98 | list-type: blacklist 99 | include-go-root: false 100 | packages: 101 | - github.com/golang/glog 102 | misspell: 103 | # Correct spellings using locale preferences for US or UK. 104 | # Default is to use a neutral variety of English. 105 | # Setting locale to US will correct the British spelling of 'colour' to 'color'. 106 | locale: UK 107 | lll: 108 | # max line length, lines longer will be reported. Default is 120. 109 | # '\t' is counted as 1 character by default, and can be changed with the tab-width option 110 | line-length: 120 111 | # tab width in spaces. Default to 1. 112 | tab-width: 1 113 | unused: 114 | # treat code as a program (not a library) and report unused exported identifiers; default is false. 115 | # XXX: if you enable thisconfig setting, unused will report a lot of false-positives in text editors: 116 | # if it's called for subdconfigir of a project it can't find funcs usages. All text editor integrations 117 | # with golangci-lint callconfig it on a directory with the changed file. 118 | check-exported: false 119 | unparam: 120 | # call graph construction algorithm (cha, rta). In general, use cha for libraries, 121 | # and rta for programs with main packages. Default is cha. 122 | algo: cha 123 | 124 | # Inspect exported functions, default is false. Set to true if no external program/library imports your code. 125 | # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: 126 | # if it's called for subdir of a project it can't find external interfaces. All text editor integrations 127 | # with golangci-lint call it on a directory with the changed file. 128 | check-exported: false 129 | nakedret: 130 | # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 131 | max-func-lines: 30 132 | prealloc: 133 | # XXX: we don't recommend using this linter before doing performance profiling. 134 | # For most programs usage of prealloc will be a premature optimization. 135 | 136 | # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them. 137 | # True by default. 138 | simple: true 139 | range-loops: true # Report preallocation suggestions on range loops, true by default 140 | for-loops: false # Report preallocation suggestions on for loops, false by default 141 | 142 | 143 | linters: 144 | enable-all: true 145 | disable: 146 | - goimports 147 | - maligned 148 | - prealloc 149 | - gocyclo 150 | - lll 151 | - gosec 152 | - dupl 153 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | # This is an example goreleaser.yaml file with some sane defaults. 2 | # Make sure to check the documentation at http://goreleaser.com 3 | before: 4 | hooks: 5 | - go mod download 6 | builds: 7 | - id: default 8 | main: ./cmd/connectctl 9 | binary: connectctl 10 | flags: 11 | - -tags 12 | - netgo release 13 | env: 14 | - CGO_ENABLED=0 15 | ldflags: 16 | - -s -w -X github.com/90poe/connectctl/internal/version.BuildDate={{.Date}} -X github.com/90poe/connectctl/internal/version.GitHash={{.Commit}} -X github.com/90poe/connectctl/internal/version.Version={{.Version}} 17 | goos: 18 | - windows 19 | - darwin 20 | - linux 21 | goarch: 22 | - amd64 23 | 24 | archives: 25 | - id: default 26 | builds: 27 | - default 28 | name_template: "{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}" 29 | replacements: 30 | darwin: Darwin 31 | linux: Linux 32 | windows: Windows 33 | amd64: x86_64 34 | format: tar.gz 35 | format_overrides: 36 | - goos: windows 37 | format: zip 38 | checksum: 39 | name_template: "{{ .ProjectName }}_checksums.txt" 40 | snapshot: 41 | name_template: "{{ .Tag }}-next" 42 | changelog: 43 | sort: asc 44 | filters: 45 | exclude: 46 | - '^docs:' 47 | - '^test:' 48 | - '^chore:' 49 | - '^style:' 50 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Build documentation in the docs/ directory with Sphinx 9 | sphinx: 10 | configuration: docs/conf.py 11 | 12 | # Build documentation with MkDocs 13 | #mkdocs: 14 | # configuration: mkdocs.yml 15 | 16 | # Optionally build your docs in additional formats such as PDF and ePub 17 | formats: all 18 | 19 | # Optionally set the version of Python and requirements required to build your docs 20 | python: 21 | version: 3.7 22 | install: 23 | - requirements: docs/requirements.txt -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Launch (file: connector1.json)", 9 | "type": "go", 10 | "request": "launch", 11 | "mode": "auto", 12 | "program": "${workspaceRoot}/cmd/connectctl", 13 | "env": {}, 14 | "args": [ 15 | "connectors", 16 | "manage", 17 | "--cluster", 18 | "http://localhost:8083", 19 | "--files", 20 | "${workspaceRoot}/examples/connector1.json", 21 | "--sync-period", 22 | "30s", 23 | "--loglevel", 24 | "TRACE", 25 | "--allow-purge" 26 | 27 | ] 28 | }, 29 | { 30 | "name": "Launch (file: connector2.json)", 31 | "type": "go", 32 | "request": "launch", 33 | "mode": "auto", 34 | "program": "${workspaceRoot}/cmd/connectctl", 35 | "env": {}, 36 | "args": [ 37 | "connectors", 38 | "manage", 39 | "--cluster", 40 | "http://localhost:8083", 41 | "--files", 42 | "${workspaceRoot}/examples/connector2.json", 43 | "--sync-period", 44 | "30s", 45 | "--loglevel", 46 | "TRACE", 47 | "--allow-purge" 48 | ] 49 | }, 50 | { 51 | "name": "Launch (dir: examples)", 52 | "type": "go", 53 | "request": "launch", 54 | "mode": "auto", 55 | "program": "${workspaceRoot}/cmd/connectctl", 56 | "env": {}, 57 | "args": [ 58 | "connectors", 59 | "manage", 60 | "--cluster", 61 | "http://localhost:8083", 62 | "--directory", 63 | "${workspaceRoot}/examples", 64 | "--sync-period", 65 | "30s", 66 | "--loglevel", 67 | "TRACE", 68 | "--allow-purge" 69 | ] 70 | }, 71 | { 72 | "name": "Launch (restart all)", 73 | "type": "go", 74 | "request": "launch", 75 | "mode": "auto", 76 | "program": "${workspaceRoot}/cmd/connectctl", 77 | "env": {}, 78 | "args": [ 79 | "connectors", 80 | "restart", 81 | "--cluster", 82 | "http://localhost:8083", 83 | "--loglevel", 84 | "TRACE" 85 | ] 86 | }, 87 | { 88 | "name": "Launch (list)", 89 | "type": "go", 90 | "request": "launch", 91 | "mode": "auto", 92 | "program": "${workspaceRoot}/cmd/connectctl", 93 | "env": {}, 94 | "args": [ 95 | "connectors", 96 | "list", 97 | "--cluster", 98 | "http://localhost:8083", 99 | "--loglevel", 100 | "TRACE", 101 | "--output", 102 | "table" 103 | ] 104 | }, 105 | { 106 | "name": "Launch (list plugins)", 107 | "type": "go", 108 | "request": "launch", 109 | "mode": "auto", 110 | "program": "${workspaceRoot}/cmd/connectctl", 111 | "env": {}, 112 | "args": [ 113 | "plugins", 114 | "list", 115 | "--cluster", 116 | "http://localhost:8083", 117 | "--loglevel", 118 | "TRACE", 119 | "--output", 120 | "table" 121 | ] 122 | } 123 | ] 124 | } -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to make participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies within all project spaces, and it also applies when 49 | an individual is representing the project or its community in public spaces. 50 | Examples of representing a project or community include using an official 51 | project e-mail address, posting via an official social media account, or acting 52 | as an appointed representative at an online or offline event. Representation of 53 | a project may be further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at [INSERT EMAIL ADDRESS]. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | 78 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to connectctl 2 | 3 | Firstly, thanks for considering contributing to *connectctl*. To make it a really 4 | great tool we need your help. 5 | 6 | *connectctl* is [Apache 2.0 licenced](LICENSE) and accepts contributions via GitHub 7 | pull requests. There are many ways to contribute, from writing tutorials or blog posts, 8 | improving the documentation, submitting bug reports and feature requests or writing code. 9 | 10 | 11 | ## Certificate of Origin 12 | 13 | By contributing to this project you agree to the [Developer Certificate of 14 | Origin](https://developercertificate.org/). This was created by the Linux 15 | Foundation and is a simple statement that you, as a contributor, have the legal 16 | right to make the contribution. 17 | 18 | To signify that you agree to the DCO you must signoff all commits: 19 | 20 | ```bash 21 | git commit --signoff 22 | ``` 23 | 24 | ## Getting Started 25 | 26 | - Fork the repository on GitHub 27 | - Read the [README](README.md) for getting started as a user and learn how/where to ask for help 28 | - If you want to contribute as a developer, continue reading this document for further instructions 29 | - Play with the project, submit bugs, submit pull requests! 30 | 31 | ### Contribution workflow 32 | 33 | #### 1. Set up your Go environment 34 | 35 | This project is written in Go. To be able to contribute you will need: 36 | 37 | 1. A working Go installation of Go >= 1.12. You can check the 38 | [official installation guide](https://golang.org/doc/install). 39 | 40 | 2. Make sure that `$(go env GOPATH)/bin` is in your shell's `PATH`. You can do so by 41 | running `export PATH="$(go env GOPATH)/bin:$PATH"` 42 | 43 | 3. Fork this repository and clone it by running: 44 | 45 | ```bash 46 | git clone git@github.com:/connectctl.git 47 | ``` 48 | 49 | > As the project uses modules its recommeneded that you NOT clone under the `GOPATH`. 50 | 51 | #### 2. Test and build 52 | 53 | Make sure you can run the tests and build the binary. 54 | 55 | ```bash 56 | make install-deps 57 | make test 58 | make build 59 | ``` 60 | 61 | #### 3. Find a feature to work on 62 | 63 | - Look at the existing [issues](https://github.com/90poe/connectctl/issues) to see if there is anything 64 | you would like to work on. If don't see anything then feel free to create your own feature request. 65 | 66 | - If you are a new contributor then take a look at the issues marked 67 | with [good first issue](https://github.com/90poe/connectctl/labels/good%20first%20issue). 68 | 69 | - Make your code changes within a feature branch: 70 | 71 | ```bash 72 | git checkout -b 73 | ``` 74 | 75 | - Add yourself to [humans.txt](humans.txt) if this is your first contribution. 76 | 77 | - Try to commit changes in logical units with a commit message in this [format](#format-of-the-commit-message). Remember 78 | to signoff your commits. 79 | 80 | - Don't forget to update the docs if relevent. The [README](README.md) or the [docs](docs/) folder is where docs usually live. 81 | 82 | - Make sure the tests pass and that there are no linting problems. 83 | 84 | #### 4. Create a pull request 85 | 86 | Push your changes to your fork and then create a pull request to origin. Where possible use the PR template. 87 | 88 | You can mark a PR as wotk in progress by prefixing the title of your PR with `WIP: `. 89 | 90 | 91 | ### Commit Message Format 92 | 93 | We would like to follow the **Conventional Commits** format for commit messsages. The full specification can be 94 | read [here](https://www.conventionalcommits.org/en/v1.0.0-beta.3/). The format is: 95 | 96 | ``` 97 | [optional scope]: 98 | 99 | [optional body] 100 | 101 | [optional footer] 102 | ``` 103 | 104 | Where `` is one of the following: 105 | * `feat` - a new feature 106 | * `fix` - a bug fix 107 | * `chore` - changes to the build pocess, code generation or anything that doesn't match elsewhere 108 | * `docs` - documentation only changes 109 | * `style` - changes that don't affect the meaning of the code (i.e. code formatting) 110 | * `refactor` - a change that doesn't fix a feature or bug 111 | * `test` - changes to tests only. 112 | 113 | The `scope` can be a pkg name but is optional. 114 | The `body` should include details of what changed and why. If there is a breaking change then the `body` should start with the 115 | following: `BREAKING CHANGE`. 116 | 117 | The footer should include any related github issue numbers. 118 | 119 | An example: 120 | 121 | ```text 122 | feat: Added connector status command 123 | 124 | A new command has been added to show the status of tasks within a connector. The 125 | command will return the number of failed tasks as the exit code. 126 | 127 | Fixes: #123 128 | ``` 129 | 130 | A tool like [Commitizen](https://github.com/commitizen/cz-cli) can be used to help with formatting commit messages. 131 | 132 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2019 90poe 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | built_at := $(shell date +%s) 2 | git_commit := $(shell git describe --dirty --always) 3 | 4 | BIN:=./bin 5 | 6 | OS := $(shell uname) 7 | GOLANGCI_LINT_VERSION?=1.19.1 8 | ifeq ($(OS),Darwin) 9 | GOLANGCI_LINT_ARCHIVE=golangci-lint-$(GOLANGCI_LINT_VERSION)-darwin-amd64.tar.gz 10 | else 11 | GOLANGCI_LINT_ARCHIVE=golangci-lint-$(GOLANGCI_LINT_VERSION)-linux-amd64.tar.gz 12 | endif 13 | 14 | .PHONY: build 15 | build: 16 | CGO_ENABLED=0 go build -ldflags "-X github.com/90poe/connectctl/internal/version.GitHash=$(git_commit) -X github.com/90poe/connectctl/internal/version.BuildDate=$(built_at)" ./cmd/connectctl 17 | 18 | .PHONY: local-release 19 | local-release: 20 | goreleaser --snapshot --skip-publish --rm-dist 21 | 22 | .PHONY: release 23 | release: test lint 24 | curl -sL https://git.io/goreleaser | bash 25 | 26 | .PHONY: test 27 | test: 28 | @go test -v -covermode=count -coverprofile=coverage.out ./... 29 | 30 | .PHONY: ci 31 | ci: build test lint 32 | 33 | .PHONY: lint 34 | lint: $(BIN)/golangci-lint/golangci-lint ## lint 35 | $(BIN)/golangci-lint/golangci-lint run 36 | 37 | $(BIN)/golangci-lint/golangci-lint: 38 | curl -OL https://github.com/golangci/golangci-lint/releases/download/v$(GOLANGCI_LINT_VERSION)/$(GOLANGCI_LINT_ARCHIVE) 39 | mkdir -p $(BIN)/golangci-lint/ 40 | tar -xf $(GOLANGCI_LINT_ARCHIVE) --strip-components=1 -C $(BIN)/golangci-lint/ 41 | chmod +x $(BIN)/golangci-lint 42 | rm -f $(GOLANGCI_LINT_ARCHIVE) 43 | 44 | .PHONY: mocks 45 | # generate mocks 46 | mocks: 47 | ifeq ("$(wildcard $(shell which counterfeiter))","") 48 | go get github.com/maxbrunsfeld/counterfeiter/v6 49 | endif 50 | counterfeiter -o=./pkg/manager/mocks/client.go ./pkg/manager/manager.go client 51 | -------------------------------------------------------------------------------- /OSSMETADATA: -------------------------------------------------------------------------------- 1 | osslifecycle=active -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## connectctl 2 | ![GitHub release (latest by date)](https://img.shields.io/github/v/release/90poe/connectctl) 3 | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 4 | [![CircleCI](https://circleci.com/gh/90poe/connectctl/tree/master.svg?style=svg)](https://circleci.com/gh/90poe/connectctl/tree/master) 5 | ![OSS Lifecycle](https://img.shields.io/osslifecycle/90poe/connectctl) 6 | [![Go Report Card](https://goreportcard.com/badge/github.com/90poe/connectctl)](https://goreportcard.com/report/github.com/90poe/connectctl) 7 | 8 | 9 | A CLI for working with [Kafka Connect](https://docs.confluent.io/current/connect/index.html). 10 | 11 | > This is a work in progress project. If you'd like to contribute please consider contrubiting. 12 | 13 | ### Getting started 14 | 15 | The latest release can be found on the [releases page](https://github.com/90poe/connectctl/releases) and 16 | install and how to install is covered [here](docs/installation.md). 17 | 18 | Details of the commands available can be read in the [cli documentation](docs/cli/connectctl.md). 19 | 20 | Documentation is also available via [ReadTheDocs](https://connectctl.readthedocs.io). 21 | 22 | ### Contributing 23 | 24 | We'd love you to contribute to the project. If you are interested in helping out please 25 | see the [contributing guide](CONTRIBUTING.md). 26 | 27 | ### Acknowledgements 28 | The code in `pkg/client/connect` is originally from [here](https://github.com/go-kafka/connect) but has been modified for use in this utility. 29 | 30 | ### License 31 | 32 | Copyright 2019 90poe. This project is licensed under the Apache 2.0 License. 33 | 34 | 35 | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 36 | [![CircleCI](https://circleci.com/gh/90poe/connectctl/tree/master.svg?style=svg)](https://circleci.com/gh/90poe/connectctl/tree/master) 37 | ![OSS Lifecycle](https://img.shields.io/osslifecycle/90poe/connectctl) 38 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | We are currently in active development so there are no supported versions yet. 6 | 7 | | Version | Supported | 8 | | ------- | ------------------ | 9 | | x.x.x | :x: | 10 | 11 | 12 | ## Reporting a Vulnerability 13 | 14 | The best way to report a vulnerability is to raise an issue [here](https://github.com/90poe/connectctl/issues) 15 | and add the *security* label. 16 | -------------------------------------------------------------------------------- /cmd/connectctl/main.go: -------------------------------------------------------------------------------- 1 | //nolint:gochecknoglobals 2 | package main 3 | 4 | import ( 5 | "fmt" 6 | "os" 7 | "strings" 8 | 9 | "github.com/90poe/connectctl/internal/ctl/tasks" 10 | "github.com/90poe/connectctl/pkg/client/connect" 11 | 12 | "github.com/90poe/connectctl/internal/ctl/connectors" 13 | "github.com/90poe/connectctl/internal/ctl/plugins" 14 | "github.com/90poe/connectctl/internal/ctl/version" 15 | 16 | homedir "github.com/mitchellh/go-homedir" 17 | "github.com/spf13/cobra" 18 | "github.com/spf13/viper" 19 | ) 20 | 21 | var ( 22 | cfgFile string 23 | ) 24 | 25 | func main() { 26 | rootCmd := &cobra.Command{ 27 | Use: "connectctl [command]", 28 | Short: "A kafka connect CLI", 29 | Long: "", 30 | RunE: func(c *cobra.Command, _ []string) error { 31 | return c.Help() 32 | }, 33 | } 34 | 35 | rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "", "", "Config file (default is $HOME/.connectctl.yaml)") 36 | _ = viper.BindPFlag("config", rootCmd.PersistentFlags().Lookup("config")) 37 | 38 | rootCmd.AddCommand(connectors.Command()) 39 | rootCmd.AddCommand(plugins.Command()) 40 | rootCmd.AddCommand(version.Command()) 41 | 42 | rootCmd.AddCommand(tasks.Command(&tasks.GenericOptions{ 43 | CreateClient: func(clusterURL string) (client tasks.Client, err error) { 44 | return connect.NewClient(clusterURL) 45 | }, 46 | })) 47 | 48 | cobra.OnInitialize(initConfig) 49 | 50 | if err := rootCmd.Execute(); err != nil { 51 | fmt.Printf("%s", err) 52 | os.Exit(1) 53 | } 54 | } 55 | 56 | func initConfig() { 57 | if cfgFile != "" { 58 | viper.SetConfigFile(cfgFile) 59 | } else { 60 | home, err := homedir.Dir() 61 | if err != nil { 62 | fmt.Println(err) 63 | os.Exit(1) 64 | } 65 | 66 | viper.AddConfigPath(home) 67 | viper.SetConfigFile(".connectctl.yaml") 68 | } 69 | 70 | replacer := strings.NewReplacer(".", "-") 71 | viper.SetEnvKeyReplacer(replacer) 72 | viper.AutomaticEnv() 73 | 74 | if err := viper.ReadInConfig(); err == nil { 75 | fmt.Println("Using config file:", viper.ConfigFileUsed()) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /docs/FAQ.md: -------------------------------------------------------------------------------- 1 | # FAQ 2 | 3 | Any questions realted to connecttl: 4 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/_static/custom.css: -------------------------------------------------------------------------------- 1 | @import url("https://fonts.googleapis.com/css?family=Montserrat&display=swap"); 2 | @import url("https://fonts.googleapis.com/css?family=Roboto+Mono&display=swap"); 3 | 4 | body { 5 | font-family: "Montserrat", sans-serif; 6 | } 7 | 8 | .rst-content pre.literal-block, .rst-content div[class^="highlight"] pre, .rst-content .linenodiv pre { 9 | font-family: "Roboto Mono", monospace; 10 | } 11 | 12 | code { 13 | font-family: "Roboto Mono", monospace; 14 | } -------------------------------------------------------------------------------- /docs/cli/connectctl.md: -------------------------------------------------------------------------------- 1 | ## connectctl 2 | 3 | connectctl: work with Kafka Connect easily 4 | 5 | ### Synopsis 6 | 7 | *connectctl* is a cli that makes working with kafka connect easier. It can be used to manage connectors and plugins and to also also actively manage/reconcile the state of a cluster. 8 | 9 | The operations you can perform are split into 2 subcommands: 10 | connectors Manage Kafka Connect connectors 11 | plugins Manage Kafka connect connector plugins 12 | 13 | Example usage: 14 | 15 | $ connectctl connectors add \ 16 | -c http://connect:8083 17 | $ connectctl connectors list -c http://connect:8083 18 | 19 | ### Options 20 | 21 | ``` 22 | -h, --help Help for connectctl 23 | -l, --loglevel loglevel Specify the loglevel for the program (default info) 24 | --logfile Specify a file to output logs to 25 | ``` 26 | 27 | ### SEE ALSO 28 | 29 | * [connectctl connectors](connectctl_connectors.md) - Perform connector operations against a Kafka Connect cluster 30 | * [connectctl plugins](connectctl_plugins.md) - Perform connector plugin operations against a Kafka Connect cluster 31 | * [connectctl version](connectctl_version.md) - Display version information 32 | -------------------------------------------------------------------------------- /docs/cli/connectctl_connectors.md: -------------------------------------------------------------------------------- 1 | ## connectctl connectors 2 | 3 | perform a connector operation against a Kafka Connect cluster 4 | 5 | ### Synopsis 6 | 7 | 8 | Perform operations against a Kafka Connect cluster that relate to connectors. 9 | Operations are always against a specific cluster and URL must be supplied. 10 | 11 | 12 | ``` 13 | connectctl connectors [flags] 14 | ``` 15 | 16 | ### Options 17 | 18 | None, all options are at the subcommand level 19 | 20 | ### Options inherited from parent commands 21 | 22 | ``` 23 | -h, --help Help for connectctl 24 | -l, --loglevel loglevel Specify the loglevel for the program (default info) 25 | --logfile Specify a file to output logs to 26 | ``` 27 | 28 | ### SEE ALSO 29 | 30 | * [connectctl](connectctl.md) - connectctl: work with Kafka Connect easily 31 | * [connectctl connectors status](connectctl_connectors_status.md) - Connectors status 32 | * [connectctl connectors add](connectctl_connectors_add.md) - Add connectors 33 | * [connectctl connectors remove](connectctl_connectors_remove.md) - Remove connectors 34 | * [connectctl connectors list](connectctl_connectors_list.md) - List connectors 35 | * [connectctl connectors restart](connectctl_connectors_restart.md) - Restart connectors 36 | * [connectctl connectors pause](connectctl_connectors_pause.md) - Pause connectors 37 | * [connectctl connectors resume](connectctl_connectors_resume.md) - Resume connectors 38 | * [connectctl connectors manage](connectctl_connectors_manage.md) - Actively manage connectors -------------------------------------------------------------------------------- /docs/cli/connectctl_connectors_add.md: -------------------------------------------------------------------------------- 1 | ## connectctl connectors add 2 | 3 | Add a connector 4 | 5 | ### Synopsis 6 | 7 | 8 | Creates a new connector based on a definition ina cluster. 9 | It can create one or more connectors in a single execution. 10 | 11 | 12 | ``` 13 | connectctl connectors add [flags] 14 | ``` 15 | 16 | ### Options 17 | 18 | ``` 19 | -h, --help help for add 20 | -c, --clusterURL the url of the kafka connect cluster to create the connectors in 21 | -f, --files the json file containing the connector definition. Multiple files can be specified 22 | either by comma separating file1.json,file2.json or by repeating the flag. 23 | -d, --directory a director that contains json files with the connector definitions to add 24 | ``` 25 | 26 | NOTE: the -d and -f options are mutually exclusive. 27 | 28 | ### Options inherited from parent commands 29 | 30 | ``` 31 | -l, --loglevel loglevel Specify the loglevel for the program (default info) 32 | --logfile Specify a file to output logs to 33 | ``` 34 | 35 | ### SEE ALSO 36 | 37 | * [connectctl connectors](connectctl_connectors.md) - Manage connectors -------------------------------------------------------------------------------- /docs/cli/connectctl_connectors_list.md: -------------------------------------------------------------------------------- 1 | ## connectctl connectors list 2 | 3 | List the connectors 4 | 5 | ### Synopsis 6 | 7 | 8 | Lists all the connectors in a given Kafka Connect cluster. 9 | The output includes the connecor status and the format can be 10 | specified. 11 | 12 | 13 | ``` 14 | connectctl connectors list [flags] 15 | ``` 16 | 17 | ### Options 18 | 19 | ``` 20 | -h, --help help for add 21 | -c, --clusterURL the url of the kafka connect cluster to remove the connectors from 22 | -o, --output specify the format of the list of connectors. Valid options 23 | are json and table. The default is json. 24 | 25 | ``` 26 | ### Options inherited from parent commands 27 | 28 | ``` 29 | -l, --loglevel loglevel Specify the loglevel for the program (default info) 30 | --logfile Specify a file to output logs to 31 | ``` 32 | 33 | ### SEE ALSO 34 | 35 | * [connectctl connectors](connectctl_connectors.md) - Manage connectors -------------------------------------------------------------------------------- /docs/cli/connectctl_connectors_manage.md: -------------------------------------------------------------------------------- 1 | ## connectctl connectors manage 2 | 3 | Actively manage connectors in a cluster 4 | 5 | ### Synopsis 6 | 7 | 8 | This command will actively manage connectors in a cluster by 9 | creating, updating, removing and restarting connectors. 10 | The command can be run once or it can run continously where it 11 | will sync the desired state and actually state on a periodic basis. 12 | 13 | 14 | ``` 15 | connectctl connectors manage [flags] 16 | ``` 17 | 18 | 19 | 20 | ### Options 21 | 22 | ``` 23 | -h, --help help for add 24 | -c, --clusterURL the url of the kafka connect cluster to avtively manage 25 | -f, --files the json file(s) containing the connector definition. Multiple files can be specified 26 | either by comma separating file1.json,file2.json or by repeating the flag. 27 | -d, --directory a director that contains json files with the connector definitions to add 28 | -s, --sync-period how often to check the current state of the connectors in the lcuster specified 29 | by -c and the desired stats of the connectors as specified by -f or -d. 30 | The default is 5 minutes. 31 | --allow-purge if specified then any connectors that are found in the cluster that aren't 32 | in the desired state (as spcified by -f or -d) will be deleted from the cluster. 33 | The default is false. 34 | --auto-restart if specified then connector tasks will be restarted if they are in a FAILED state 35 | The default is false. 36 | --once if specified the command will run once and then exit. The default is false. 37 | 38 | ``` 39 | 40 | NOTE: the -d and -f options are mutually exclusive. If you don't specify --once then the command will 41 | run continuosly and will try and synchronise the state of the cluster according the duration 42 | specific by the -s option. 43 | 44 | ### Options inherited from parent commands 45 | 46 | ``` 47 | -l, --loglevel loglevel Specify the loglevel for the program (default info) 48 | --logfile Specify a file to output logs to 49 | ``` 50 | 51 | ### SEE ALSO 52 | 53 | * [connectctl connectors](connectctl_connectors.md) - Manage connectors -------------------------------------------------------------------------------- /docs/cli/connectctl_connectors_pause.md: -------------------------------------------------------------------------------- 1 | ## connectctl connectors pause 2 | 3 | Pause connectors 4 | 5 | ### Synopsis 6 | 7 | 8 | Pauses connectors in a specified Kafka Connect cluster. 9 | It can pause one or more connectors in a single execution. 10 | 11 | 12 | ``` 13 | connectctl connectors pause [flags] 14 | ``` 15 | 16 | ### Options 17 | 18 | ``` 19 | -h, --help help for add 20 | -c, --clusterURL the url of the kafka connect cluster to pause connectors in 21 | -n, --connectors the names of the connectors to pause. Multiple connector names 22 | can be specified either by comma separating conn1,conn2 23 | or by repeating the flag --n conn1 --n conn2. If no name is 24 | supplied then ALL connectors will be paused. 25 | ``` 26 | ### Options inherited from parent commands 27 | 28 | ``` 29 | -l, --loglevel loglevel Specify the loglevel for the program (default info) 30 | --logfile Specify a file to output logs to 31 | ``` 32 | 33 | ### SEE ALSO 34 | 35 | * [connectctl connectors](connectctl_connectors.md) - Manage connectors -------------------------------------------------------------------------------- /docs/cli/connectctl_connectors_remove.md: -------------------------------------------------------------------------------- 1 | ## connectctl connectors remove 2 | 3 | Remove a connector 4 | 5 | ### Synopsis 6 | 7 | 8 | Removes a named connector from a cluster. 9 | It can remove one or more connectors in a single execution. 10 | 11 | 12 | ``` 13 | connectctl connectors remove [flags] 14 | ``` 15 | 16 | ### Options 17 | 18 | ``` 19 | -h, --help help for add 20 | -c, --clusterURL the url of the kafka connect cluster to remove the connectors from 21 | -n, --connectors the names of the connectors to remove. Multiple connector names 22 | can be specified either by comma separating conn1,conn2 23 | or by repeating the flag --n conn1 --n conn2. 24 | ``` 25 | ### Options inherited from parent commands 26 | ## connectctl connectors remove 27 | 28 | Remove a connector 29 | 30 | ### Synopsis 31 | 32 | 33 | Removes a named connector from a cluster. 34 | It can remove one or more connectors in a single execution. 35 | 36 | 37 | ``` 38 | connectctl connectors remove [flags] 39 | ``` 40 | 41 | ### Options 42 | 43 | ``` 44 | -h, --help help for add 45 | -c, --clusterURL the url of the kafka connect cluster to remove the connectors from 46 | -n, --connectors the names of the connectors to remove. Multiple connector names 47 | can be specified either by comma separating conn1,conn2 48 | or by repeating the flag --n conn1 --n conn2. 49 | ``` 50 | ### Options inherited from parent commands 51 | 52 | ``` 53 | -l, --loglevel loglevel Specify the loglevel for the program (default info) 54 | --logfile Specify a file to output logs to 55 | ``` 56 | 57 | ### SEE ALSO 58 | 59 | * [connectctl connectors](connectctl_connectors.md) - Manage connectors 60 | ``` 61 | -l, --loglevel loglevel Specify the loglevel for the program (default info) 62 | --logfile Specify a file to output logs to 63 | ``` 64 | 65 | ### SEE ALSO 66 | 67 | * [connectctl connectors](connectctl_connectors.md) - Manage connectors -------------------------------------------------------------------------------- /docs/cli/connectctl_connectors_restart.md: -------------------------------------------------------------------------------- 1 | ## connectctl connectors restart 2 | 3 | Restart connectors 4 | 5 | ### Synopsis 6 | 7 | 8 | Restart connectors in a specified Kafka Connect cluster. 9 | It can restart one or more connectors in a single execution. 10 | 11 | 12 | ``` 13 | connectctl connectors restart [flags] 14 | ``` 15 | 16 | ### Options 17 | 18 | ``` 19 | -h, --help help for add 20 | -c, --clusterURL the url of the kafka connect cluster to restart connectors in 21 | -n, --connectors the names of the connectors to restart. Multiple connector names 22 | can be specified either by comma separating conn1,conn2 23 | or by repeating the flag --n conn1 --n conn2. If no name is 24 | supplied then ALL connectors will be restarted. 25 | ``` 26 | ### Options inherited from parent commands 27 | 28 | ``` 29 | -l, --loglevel loglevel Specify the loglevel for the program (default info) 30 | --logfile Specify a file to output logs to 31 | ``` 32 | 33 | ### SEE ALSO 34 | 35 | * [connectctl connectors](connectctl_connectors.md) - Manage connectors -------------------------------------------------------------------------------- /docs/cli/connectctl_connectors_resume.md: -------------------------------------------------------------------------------- 1 | ## connectctl connectors resume 2 | 3 | Resume connectors 4 | 5 | ### Synopsis 6 | 7 | 8 | Resume connectors in a specified Kafka Connect cluster. 9 | It can resume one or more connectors in a single execution. 10 | 11 | 12 | ``` 13 | connectctl connectors resume [flags] 14 | ``` 15 | 16 | ### Options 17 | 18 | ``` 19 | -h, --help help for add 20 | -c, --clusterURL the url of the kafka connect cluster to resume connectors in 21 | -n, --connectors the names of the connectors to resume. Multiple connector names 22 | can be specified either by comma separating conn1,conn2 23 | or by repeating the flag --n conn1 --n conn2. If no name is 24 | supplied then ALL connectors will be resumed. 25 | ``` 26 | ### Options inherited from parent commands 27 | 28 | ``` 29 | -l, --loglevel loglevel Specify the loglevel for the program (default info) 30 | --logfile Specify a file to output logs to 31 | ``` 32 | 33 | ### SEE ALSO 34 | 35 | * [connectctl connectors](connectctl_connectors.md) - Manage connectors -------------------------------------------------------------------------------- /docs/cli/connectctl_connectors_status.md: -------------------------------------------------------------------------------- 1 | ## connectctl connectors status 2 | 3 | Status of connectors 4 | 5 | ### Synopsis 6 | 7 | 8 | Display status of selected connectors. 9 | If some tasks or connectors are failing, command will exit with code 1. 10 | 11 | 12 | ``` 13 | connectctl connectors status [flags] 14 | ``` 15 | 16 | ### Options 17 | 18 | ``` 19 | -h, --help help for add 20 | -c, --clusterURL the url of the kafka connect cluster 21 | -n, --connectors the names of the connectors. Multiple connector names 22 | can be specified either by comma separating conn1,conn2 23 | or by repeating the flag --n conn1 --n conn2. If no name is 24 | supplied status of ALL connectors will be displayed. 25 | -o, --output specify the output format (valid options: json, table) (default "json") 26 | -q, --quiet disable output logging 27 | ``` 28 | ### Options inherited from parent commands 29 | 30 | ``` 31 | -l, --loglevel loglevel Specify the loglevel for the program (default info) 32 | --logfile Specify a file to output logs to 33 | ``` 34 | 35 | ### SEE ALSO 36 | 37 | * [connectctl connectors](connectctl_connectors.md) - Manage connectors -------------------------------------------------------------------------------- /docs/cli/connectctl_plugins.md: -------------------------------------------------------------------------------- 1 | ## connectctl plugins 2 | 3 | Manage connector plugins 4 | 5 | ### Synopsis 6 | 7 | 8 | Perform operations against a Kafka Connect cluster that relate to connector plugins. 9 | Operations are always against a specific cluster and URL must be supplied. 10 | 11 | 12 | ``` 13 | connectctl plugins [flags] 14 | ``` 15 | 16 | ### Options 17 | 18 | None, all options are at the subcommand level 19 | 20 | ### Options inherited from parent commands 21 | 22 | ``` 23 | -h, --help Help for connectctl 24 | -l, --loglevel loglevel Specify the loglevel for the program (default info) 25 | --logfile Specify a file to output logs to 26 | ``` 27 | 28 | ### SEE ALSO 29 | 30 | * [connectctl](connectctl.md) - connectctl: work with Kafka Connect easily 31 | * [connectctl plugins list](connectctl_plugins_list.md) - List connector plugins 32 | * [connectctl plugins validate](connectctl_plugins_validate.md) - Validates connector config 33 | -------------------------------------------------------------------------------- /docs/cli/connectctl_plugins_list.md: -------------------------------------------------------------------------------- 1 | ## connectctl plugins list 2 | 3 | List connector plugins 4 | 5 | ### Synopsis 6 | 7 | 8 | Lists all the connector plugins installed on a given 9 | Kafka Connect cluster node. 10 | The output can be formatted as JSON or a table. 11 | 12 | 13 | ``` 14 | connectctl plugins list [flags] 15 | ``` 16 | 17 | ### Options 18 | 19 | ``` 20 | -h, --help help for add 21 | -c, --clusterURL the url of the kafka connect cluster to list the plugins from 22 | -o, --output specify the format of the list of plugins. Valid options 23 | are json and table. The default is json. 24 | 25 | ``` 26 | ### Options inherited from parent commands 27 | 28 | ``` 29 | -l, --loglevel loglevel Specify the loglevel for the program (default info) 30 | --logfile Specify a file to output logs to 31 | ``` 32 | 33 | ### SEE ALSO 34 | 35 | * [connectctl plugins](connectctl_plugins.md) - Manage plugins -------------------------------------------------------------------------------- /docs/cli/connectctl_plugins_validate.md: -------------------------------------------------------------------------------- 1 | ## connectctl plugins validate 2 | 3 | Validates connector config 4 | 5 | ### Synopsis 6 | 7 | Validate the provided configuration values against the configuration definition. This API performs per config validation, outputs suggested values and error messages during validation. 8 | It exits with code 1 if config is invalid. 9 | 10 | 11 | ``` 12 | connectctl plugins validate [flags] 13 | ``` 14 | 15 | ### Options 16 | 17 | ``` 18 | -c, --cluster string the URL of the connect cluster (required) 19 | -h, --help help for validate 20 | -i, --input string Input data in json format (required) 21 | -o, --output string specify the output format (valid options: json, table) (default "json") 22 | -q, --quiet disable output logging 23 | ``` 24 | ### Options inherited from parent commands 25 | 26 | ``` 27 | -l, --loglevel loglevel Specify the loglevel for the program (default info) 28 | --logfile Specify a file to output logs to 29 | ``` 30 | 31 | ### SEE ALSO 32 | 33 | * [connectctl plugins](connectctl_plugins.md) - Manage plugins -------------------------------------------------------------------------------- /docs/cli/connectctl_version.md: -------------------------------------------------------------------------------- 1 | ## connectctl version 2 | 3 | Print the version of connectctl 4 | 5 | ### Synopsis 6 | 7 | Prints version information of connectctl 8 | 9 | ``` 10 | connectctl version 11 | ``` 12 | 13 | ### Options 14 | 15 | ``` 16 | -h, --help help for version 17 | -c, --clusterURL the url of the kafka connect cluster 18 | ``` 19 | ### SEE ALSO 20 | 21 | * [connectctl](connectctl.md) - connectctl: work with Kafka Connect easily -------------------------------------------------------------------------------- /docs/cli/index.rst: -------------------------------------------------------------------------------- 1 | connectctl 2 | ============== 3 | 4 | connectctl: manage Kafka Connect 5 | 6 | .. toctree:: 7 | :glob: 8 | :titlesonly: 9 | :maxdepth: 0 10 | 11 | * -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | # import os 14 | # import sys 15 | # sys.path.insert(0, os.path.abspath('.')) 16 | 17 | 18 | # -- Project information ----------------------------------------------------- 19 | 20 | project = 'connectctl' 21 | copyright = '2019, 90poe & connectctl development tean' 22 | author = 'connectctl development team' 23 | 24 | 25 | # -- General configuration --------------------------------------------------- 26 | 27 | # Add any Sphinx extension module names here, as strings. They can be 28 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 29 | # ones. 30 | extensions = [ 31 | 'recommonmark', 32 | 'sphinx_markdown_tables', 33 | ] 34 | 35 | # Add any paths that contain templates here, relative to this directory. 36 | templates_path = ['_templates'] 37 | 38 | # List of patterns, relative to source directory, that match files and 39 | # directories to ignore when looking for source files. 40 | # This pattern also affects html_static_path and html_extra_path. 41 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 42 | 43 | 44 | # -- Options for HTML output ------------------------------------------------- 45 | 46 | # The theme to use for HTML and HTML Help pages. See the documentation for 47 | # a list of builtin themes. 48 | # 49 | html_theme = 'sphinx_rtd_theme' 50 | 51 | # Add any paths that contain custom static files (such as style sheets) here, 52 | # relative to this directory. They are copied after the builtin static files, 53 | # so a file named "default.css" will overwrite the builtin "default.css". 54 | html_static_path = ['_static'] 55 | 56 | # The suffix(es) of source filenames. 57 | # You can specify multiple suffix as a list of string: 58 | # 59 | source_suffix = ['.rst', '.md'] 60 | 61 | # The master toctree document. 62 | master_doc = 'index' 63 | 64 | pygments_style = 'sphinx' 65 | 66 | # Output file base name for HTML help builder. 67 | htmlhelp_basename = 'connectctldoc' 68 | 69 | # One entry per manual page. List of tuples 70 | # (source start file, name, description, authors, manual section). 71 | man_pages = [ 72 | (master_doc, 'connectctl', 'connectctl Documentation', 73 | [author], 1) 74 | ] 75 | 76 | # Grouping the document tree into Texinfo files. List of tuples 77 | # (source start file, target name, title, author, 78 | # dir menu entry, description, category) 79 | texinfo_documents = [ 80 | (master_doc, 'connectctl', 'connectctl Documentation', 81 | author, 'connectctl', 'One line description of project.', 82 | 'Miscellaneous'), 83 | ] 84 | 85 | def setup(app): 86 | app.add_stylesheet('custom.css') -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. connectctl documentation master file, created by 2 | sphinx-quickstart on Fri Aug 23 09:43:58 2019. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to connectctl's documentation! 7 | ===================================== 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :caption: Contents: 12 | 13 | installation 14 | FAQ 15 | 16 | .. toctree:: 17 | :caption: CLI Reference 18 | :titlesonly: 19 | :maxdepth: 0 20 | 21 | cli/index 22 | 23 | 24 | 25 | 26 | Indices and tables 27 | ================== 28 | 29 | * :ref:`genindex` 30 | * :ref:`modindex` 31 | * :ref:`search` 32 | -------------------------------------------------------------------------------- /docs/installation.md: -------------------------------------------------------------------------------- 1 | # connectctl installation 2 | 3 | The contains details of how to install and uninstall connectctl 4 | 5 | ## Requirements 6 | 7 | connectctl runs on Linux, Windows and MacOS. 8 | 9 | ## Download the binary 10 | 11 | The connectctl binary can be downloaded from the [releases section](https://github.com/90poe/connectctl/releases). Its advised that you make the binary available in your PATH. 12 | 13 | ## Future work 14 | 15 | We'll be supplying an install script and support for various package managers in the future. 16 | 17 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx == 2.1.2 2 | sphinx_markdown_tables == 0.0.9 3 | recommonmark == 0.5.0 4 | sphinx-rtd-theme == 0.4.3 -------------------------------------------------------------------------------- /examples/connector1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test-connector-1", 3 | "config": { 4 | "name": "test-connector-1", 5 | "connector.class": "FileStreamSource", 6 | "tasks.max": "1", 7 | "topic": "connect-test", 8 | "file": "test.txt" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /examples/connector2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test-connector-2", 3 | "config": { 4 | "name": "test-connector-2", 5 | "connector.class": "FileStreamSource", 6 | "tasks.max": "1", 7 | "topic": "connect-test", 8 | "file": "test2.txt" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /examples/connectors.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "test-connectors-1", 4 | "config": { 5 | "name": "test-connectors-1", 6 | "connector.class": "FileStreamSource", 7 | "tasks.max": "1", 8 | "topic": "connect-test", 9 | "file": "test_1.txt" 10 | } 11 | }, 12 | { 13 | "name": "test-connectors-2", 14 | "config": { 15 | "name": "test-connectors-2", 16 | "connector.class": "FileStreamSource", 17 | "tasks.max": "1", 18 | "topic": "connect-test", 19 | "file": "test_2.txt" 20 | } 21 | } 22 | ] 23 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/90poe/connectctl 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect 7 | github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 // indirect 8 | github.com/go-openapi/errors v0.19.2 // indirect 9 | github.com/go-openapi/strfmt v0.19.0 // indirect 10 | github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 11 | github.com/jedib0t/go-pretty v4.3.0+incompatible 12 | github.com/kelseyhightower/envconfig v1.4.0 13 | github.com/magiconair/bump v1.2.0 // indirect 14 | github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63 // indirect 15 | github.com/mattn/go-runewidth v0.0.4 // indirect 16 | github.com/maxbrunsfeld/counterfeiter/v6 v6.2.3 // indirect 17 | github.com/mitchellh/go-homedir v1.1.0 18 | github.com/pkg/errors v0.8.1 19 | github.com/sirupsen/logrus v1.4.2 20 | github.com/spf13/afero v1.2.2 // indirect 21 | github.com/spf13/cobra v0.0.6 22 | github.com/spf13/pflag v1.0.3 23 | github.com/spf13/viper v1.6.2 24 | github.com/stretchr/testify v1.5.1 25 | github.com/sv-tools/bumptag v1.5.1 // indirect 26 | github.com/urfave/cli v1.22.3 // indirect 27 | gopkg.in/matryer/try.v1 v1.0.0-20150601225556-312d2599e12e 28 | ) 29 | -------------------------------------------------------------------------------- /humans.txt: -------------------------------------------------------------------------------- 1 | __ __ .__ 2 | ____ ____ ____ ____ ____ _____/ |_ _____/ |_| | 3 | _/ ___\/ _ \ / \ / \_/ __ \_/ ___\ __\/ ___\ __\ | 4 | \ \__( <_> ) | \ | \ ___/\ \___| | \ \___| | | |__ 5 | \___ >____/|___| /___| /\___ >\___ >__| \___ >__| |____/ 6 | \/ \/ \/ \/ \/ \/ 7 | 8 | 9 | /* Humans */ 10 | Andy @the4thamigo-uk maintainer 11 | Richard @richardcase maintainer 12 | 13 | /* Organizations */ 14 | 90poe https://www.openocean.studio/ 15 | 16 | /* Thanks */ 17 | Thanks to 90poe for giving birth to this project and allowing it to be open sourced. 18 | 19 | Also, thanks to all contributors and users (assuming there are more) of connectctl. 20 | 21 | /* Site */ 22 | 23 | Last update: - 24 | Language: English 25 | Doctype: HTML5 26 | IDE: VIM/VsCode/Atom 27 | Generator: Jekyll (Ruby) 28 | SCM: Git 29 | Hosting: GitHub -------------------------------------------------------------------------------- /internal/ctl/connectors/add.go: -------------------------------------------------------------------------------- 1 | package connectors 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/90poe/connectctl/internal/ctl" 7 | "github.com/90poe/connectctl/internal/version" 8 | "github.com/90poe/connectctl/pkg/client/connect" 9 | "github.com/90poe/connectctl/pkg/manager" 10 | "github.com/pkg/errors" 11 | 12 | "github.com/spf13/cobra" 13 | ) 14 | 15 | type addConnectorsCmdParams struct { 16 | ClusterURL string 17 | Files []string 18 | Directory string 19 | EnvVar string 20 | } 21 | 22 | func addConnectorCmd() *cobra.Command { 23 | params := &addConnectorsCmdParams{} 24 | 25 | addCmd := &cobra.Command{ 26 | Use: "add", 27 | Short: "Add connectors to a connect cluster", 28 | Long: "", 29 | RunE: func(cmd *cobra.Command, _ []string) error { 30 | return doAddConnectors(cmd, params) 31 | }, 32 | } 33 | 34 | ctl.AddCommonConnectorsFlags(addCmd, ¶ms.ClusterURL) 35 | ctl.AddDefinitionFilesFlags(addCmd, ¶ms.Files, ¶ms.Directory, ¶ms.EnvVar) 36 | 37 | return addCmd 38 | } 39 | 40 | func doAddConnectors(cmd *cobra.Command, params *addConnectorsCmdParams) error { 41 | config := &manager.Config{ 42 | ClusterURL: params.ClusterURL, 43 | Version: version.Version, 44 | } 45 | 46 | source, err := findSource(params.Files, params.Directory, params.EnvVar, cmd) 47 | 48 | if err != nil { 49 | return err 50 | } 51 | 52 | connectors, err := source() 53 | if err != nil { 54 | return errors.Wrap(err, "error reading connector configuration from files") 55 | } 56 | 57 | userAgent := fmt.Sprintf("90poe.io/connectctl/%s", version.Version) 58 | 59 | client, err := connect.NewClient(params.ClusterURL, connect.WithUserAgent(userAgent)) 60 | if err != nil { 61 | return errors.Wrap(err, "error creating connect client") 62 | } 63 | 64 | mngr, err := manager.NewConnectorsManager(client, config) 65 | if err != nil { 66 | return errors.Wrap(err, "error creating connectors manager") 67 | } 68 | if err = mngr.Add(connectors); err != nil { 69 | return errors.Wrap(err, "error creating connectors") 70 | } 71 | fmt.Println("added connectors") 72 | return nil 73 | } 74 | -------------------------------------------------------------------------------- /internal/ctl/connectors/connectors.go: -------------------------------------------------------------------------------- 1 | package connectors 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | ) 6 | 7 | // Command creates the the management commands 8 | func Command() *cobra.Command { 9 | connectorsCmd := &cobra.Command{ 10 | Use: "connectors", 11 | Short: "Commands related to Kafka Connect Connectors", 12 | Long: "", 13 | RunE: func(cmd *cobra.Command, _ []string) error { 14 | return cmd.Help() 15 | }, 16 | } 17 | 18 | // Add subcommands 19 | connectorsCmd.AddCommand(manageConnectorsCmd()) 20 | connectorsCmd.AddCommand(restartConnectorsCmd()) 21 | connectorsCmd.AddCommand(listConnectorsCmd()) 22 | connectorsCmd.AddCommand(addConnectorCmd()) 23 | connectorsCmd.AddCommand(removeConnectorCmd()) 24 | connectorsCmd.AddCommand(pauseConnectorsCmd()) 25 | connectorsCmd.AddCommand(resumeConnectorsCmd()) 26 | connectorsCmd.AddCommand(connectorsStatusCmd()) 27 | 28 | return connectorsCmd 29 | } 30 | -------------------------------------------------------------------------------- /internal/ctl/connectors/list.go: -------------------------------------------------------------------------------- 1 | package connectors 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/90poe/connectctl/internal/ctl" 9 | "github.com/90poe/connectctl/internal/version" 10 | "github.com/90poe/connectctl/pkg/client/connect" 11 | "github.com/90poe/connectctl/pkg/manager" 12 | "github.com/pkg/errors" 13 | 14 | "github.com/jedib0t/go-pretty/table" 15 | "github.com/spf13/cobra" 16 | ) 17 | 18 | type listConnectorsCmdParams struct { 19 | ClusterURL string 20 | Output string 21 | } 22 | 23 | func listConnectorsCmd() *cobra.Command { 24 | params := &listConnectorsCmdParams{} 25 | 26 | listCmd := &cobra.Command{ 27 | Use: "list", 28 | Short: "List connectors in a cluster", 29 | Long: "", 30 | RunE: func(cmd *cobra.Command, _ []string) error { 31 | return doListConnectors(cmd, params) 32 | }, 33 | } 34 | 35 | ctl.AddCommonConnectorsFlags(listCmd, ¶ms.ClusterURL) 36 | ctl.AddOutputFlags(listCmd, ¶ms.Output) 37 | 38 | return listCmd 39 | } 40 | 41 | func doListConnectors(_ *cobra.Command, params *listConnectorsCmdParams) error { 42 | config := &manager.Config{ 43 | ClusterURL: params.ClusterURL, 44 | Version: version.Version, 45 | } 46 | 47 | userAgent := fmt.Sprintf("90poe.io/connectctl/%s", version.Version) 48 | 49 | client, err := connect.NewClient(params.ClusterURL, connect.WithUserAgent(userAgent)) 50 | if err != nil { 51 | return errors.Wrap(err, "error creating connect client") 52 | } 53 | 54 | mngr, err := manager.NewConnectorsManager(client, config) 55 | if err != nil { 56 | return errors.Wrap(err, "error creating connectors manager") 57 | } 58 | 59 | connectors, err := mngr.GetAllConnectors() 60 | if err != nil { 61 | return errors.Wrap(err, "error getting all connectors") 62 | } 63 | 64 | switch params.Output { 65 | case "json": 66 | err := printConnectorsAsJSON(connectors) 67 | if err != nil { 68 | return errors.Wrap(err, "error printing connectors as JSON") 69 | } 70 | case "table": 71 | printConnectorsAsTable(connectors) 72 | default: 73 | return fmt.Errorf("invalid output format specified: %s", params.Output) 74 | } 75 | return nil 76 | } 77 | 78 | func printConnectorsAsJSON(connectors []*manager.ConnectorWithState) error { 79 | b, err := json.MarshalIndent(connectors, "", " ") 80 | if err != nil { 81 | return err 82 | } 83 | 84 | os.Stdout.Write(b) 85 | return nil 86 | } 87 | 88 | func printConnectorsAsTable(connectors []*manager.ConnectorWithState) { 89 | t := table.NewWriter() 90 | t.SetOutputMirror(os.Stdout) 91 | t.AppendHeader(table.Row{"Name", "State", "WorkerId", "Tasks", "Config"}) 92 | 93 | for _, connector := range connectors { 94 | config := "" 95 | for key, val := range connector.Config { 96 | config += fmt.Sprintf("%s=%s\n", key, val) 97 | } 98 | 99 | tasks := "" 100 | for _, task := range connector.Tasks { 101 | tasks += fmt.Sprintf("%d(%s): %s\n", task.ID, task.WorkerID, task.State) 102 | } 103 | 104 | t.AppendRow(table.Row{ 105 | connector.Name, 106 | connector.ConnectorState.State, 107 | connector.ConnectorState.WorkerID, 108 | tasks, 109 | config, 110 | }) 111 | } 112 | t.Render() 113 | } 114 | -------------------------------------------------------------------------------- /internal/ctl/connectors/manage.go: -------------------------------------------------------------------------------- 1 | package connectors 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "time" 8 | 9 | "github.com/90poe/connectctl/internal/ctl" 10 | "github.com/90poe/connectctl/internal/healthcheck" 11 | "github.com/90poe/connectctl/internal/logging" 12 | "github.com/90poe/connectctl/internal/version" 13 | "github.com/90poe/connectctl/pkg/client/connect" 14 | "github.com/90poe/connectctl/pkg/manager" 15 | signals "github.com/90poe/connectctl/pkg/signal" 16 | "github.com/90poe/connectctl/pkg/sources" 17 | "github.com/pkg/errors" 18 | log "github.com/sirupsen/logrus" 19 | "github.com/spf13/cobra" 20 | "gopkg.in/matryer/try.v1" 21 | 22 | "github.com/kelseyhightower/envconfig" 23 | ) 24 | 25 | type manageDefaults struct { 26 | ClusterURL string `envconfig:"CLUSTER"` 27 | Files []string `envconfig:"FILES"` 28 | Directory string `envconfig:"DIRECTORY"` 29 | EnvVar string `envconfig:"ENV_VAR"` 30 | InitialWaitPeriod time.Duration `envconfig:"INITIAL_WAIT_PERIOD"` 31 | SyncPeriod time.Duration `envconfig:"SYNC_PERIOD"` 32 | SyncErrorRetryMax int `envconfig:"SYNC_ERROR_RETRY_MAX"` 33 | SyncErrorRetryPeriod time.Duration `envconfig:"SYNC_ERROR_RETRY_PERIOD"` 34 | AllowPurge bool `envconfig:"ALLOW_PURGE"` 35 | AutoRestart bool `envconfig:"AUTO_RESTART"` 36 | RunOnce bool `envconfig:"RUN_ONCE"` 37 | EnableHealthCheck bool `envconfig:"HEALTHCHECK_ENABLE"` 38 | HealthCheckAddress string `envconfig:"HEALTHCHECK_ADDRESS"` 39 | HTTPClientTimeout time.Duration `envconfig:"HTTP_CLIENT_TIMEOUT"` 40 | GlobalConnectorRestartsMax int `envconfig:"GLOBAL_CONNECTOR_RESTARTS_MAX"` 41 | GlobalConnectorRestartPeriod time.Duration `envconfig:"GLOBAL_CONNECTOR_RESTART_PERIOD"` 42 | GlobalTaskRestartsMax int `envconfig:"GLOBAL_TASK_RESTARTS_MAX"` 43 | GlobalTaskRestartPeriod time.Duration `envconfig:"GLOBAL_TASK_RESTART_PERIOD"` 44 | LogLevel string `envconfig:"LOG_LEVEL"` 45 | LogFile string `envconfig:"LOG_FILE"` 46 | LogFormat string `envconfig:"LOG_FORMAT"` 47 | } 48 | 49 | func manageConnectorsCmd() *cobra.Command { // nolint: funlen 50 | params := &manageDefaults{ 51 | InitialWaitPeriod: 3 * time.Minute, 52 | SyncPeriod: 5 * time.Minute, 53 | SyncErrorRetryMax: 10, 54 | SyncErrorRetryPeriod: 1 * time.Minute, 55 | HealthCheckAddress: ":9000", 56 | HTTPClientTimeout: 20 * time.Second, 57 | GlobalConnectorRestartsMax: 5, 58 | GlobalConnectorRestartPeriod: 10 * time.Second, 59 | GlobalTaskRestartsMax: 5, 60 | GlobalTaskRestartPeriod: 10 * time.Second, 61 | LogLevel: "INFO", 62 | LogFormat: "TEXT", 63 | } 64 | 65 | manageCmd := &cobra.Command{ 66 | Use: "manage", 67 | Short: "Actively manage connectors in a Kafka Connect cluster", 68 | Long: `This command will add/delete/update connectors in a destination 69 | Kafa Connect cluster based on a list of desired connectors which are specified 70 | as a list of files or all files in a directory. The command runs continuously and 71 | will sync desired state with actual state based on the --sync-period flag. But 72 | if you specify --once then it will sync once and then exit.`, 73 | RunE: func(cmd *cobra.Command, _ []string) error { 74 | err := envconfig.Process("CONNECTCTL", params) 75 | 76 | if err != nil { 77 | return errors.Wrap(err, "error processing environmental configuration") 78 | } 79 | 80 | return doManageConnectors(cmd, params) 81 | }, 82 | } 83 | 84 | ctl.AddCommonConnectorsFlags(manageCmd, ¶ms.ClusterURL) 85 | 86 | ctl.AddDefinitionFilesFlags(manageCmd, ¶ms.Files, ¶ms.Directory, ¶ms.EnvVar) 87 | 88 | ctl.BindDurationVarP(manageCmd.Flags(), ¶ms.SyncPeriod, params.SyncPeriod, "sync-period", "s", "how often to sync with the connect cluster") 89 | ctl.BindDurationVar(manageCmd.Flags(), ¶ms.InitialWaitPeriod, params.InitialWaitPeriod, "wait-period", "time period to wait before starting the first sync") 90 | 91 | ctl.BindBoolVar(manageCmd.Flags(), ¶ms.AllowPurge, false, "allow-purge", "if set connectctl will manage all connectors in a cluster. If connectors exist in the cluster that aren' t specified in --files then the connectors will be deleted") 92 | ctl.BindBoolVar(manageCmd.Flags(), ¶ms.AutoRestart, false, "auto-restart", "if set connectors and tasks that are failed with automatically be restarted") 93 | ctl.BindBoolVar(manageCmd.Flags(), ¶ms.RunOnce, false, "once", "if supplied sync will run once and command will exit") 94 | 95 | ctl.BindBoolVar(manageCmd.Flags(), ¶ms.EnableHealthCheck, false, "healthcheck-enable", "if true a healthcheck via http will be enabled") 96 | ctl.BindStringVar(manageCmd.Flags(), ¶ms.HealthCheckAddress, params.HealthCheckAddress, "healthcheck-address", "if enabled the healthchecks ('/live' and '/ready') will be available from this address") 97 | 98 | ctl.BindDurationVar(manageCmd.Flags(), ¶ms.HTTPClientTimeout, params.HTTPClientTimeout, "http-client-timeout", "HTTP client timeout") 99 | 100 | ctl.BindIntVar(manageCmd.Flags(), ¶ms.GlobalConnectorRestartsMax, params.GlobalConnectorRestartsMax, "global-connector-restarts-max", "maximum times a failed connector will be restarted") 101 | ctl.BindDurationVar(manageCmd.Flags(), ¶ms.GlobalConnectorRestartPeriod, params.GlobalConnectorRestartPeriod, "global-connector-restart-period", "period of time between failed connector restart attemots") 102 | ctl.BindIntVar(manageCmd.Flags(), ¶ms.GlobalTaskRestartsMax, params.GlobalTaskRestartsMax, "global-task-restarts-max", "maximum times a failed task will be restarted") 103 | ctl.BindDurationVar(manageCmd.Flags(), ¶ms.GlobalTaskRestartPeriod, params.GlobalTaskRestartPeriod, "global-task-restart-period", "period of time between failed task restarts") 104 | 105 | ctl.BindIntVar(manageCmd.Flags(), ¶ms.SyncErrorRetryMax, params.SyncErrorRetryMax, "sync-error-retry-max", "maximum times to ignore retryable errors whilst syncing") 106 | ctl.BindDurationVar(manageCmd.Flags(), ¶ms.SyncErrorRetryPeriod, params.SyncErrorRetryPeriod, "sync-error-retry-period", "period of time between retryable errors whilst syncing") 107 | 108 | ctl.BindStringVarP(manageCmd.Flags(), ¶ms.LogLevel, params.LogLevel, "loglevel", "l", "Log level for the CLI (Optional)") 109 | ctl.BindStringVar(manageCmd.Flags(), ¶ms.LogFile, params.LogFile, "logfile", "A file to use for log output (Optional)") 110 | ctl.BindStringVar(manageCmd.Flags(), ¶ms.LogFormat, params.LogFormat, "logformat", "Format for log output (Optional)") 111 | 112 | return manageCmd 113 | } 114 | 115 | func doManageConnectors(cmd *cobra.Command, params *manageDefaults) error { 116 | if err := logging.Configure(params.LogLevel, params.LogFile, params.LogFormat); err != nil { 117 | return errors.Wrap(err, "error configuring logging") 118 | } 119 | 120 | logger := log.WithFields(log.Fields{ 121 | "cluster": params.ClusterURL, 122 | "version": version.Version, 123 | }) 124 | logger.Debug("executing manage connectors command") 125 | 126 | if err := checkConfigSwitches(params.Files, params.Directory, params.EnvVar); err != nil { 127 | return errors.Wrap(err, "error with configuration") 128 | } 129 | 130 | config := &manager.Config{ 131 | ClusterURL: params.ClusterURL, 132 | InitialWaitPeriod: params.InitialWaitPeriod, 133 | SyncPeriod: params.SyncPeriod, 134 | AllowPurge: params.AllowPurge, 135 | AutoRestart: params.AutoRestart, 136 | Version: version.Version, 137 | GlobalConnectorRestartsMax: params.GlobalConnectorRestartsMax, 138 | GlobalConnectorRestartPeriod: params.GlobalConnectorRestartPeriod, 139 | GlobalTaskRestartsMax: params.GlobalTaskRestartsMax, 140 | GlobalTaskRestartPeriod: params.GlobalTaskRestartPeriod, 141 | } 142 | 143 | logger.WithField("config", config).Trace("manage connectors configuration") 144 | 145 | userAgent := fmt.Sprintf("90poe.io/connectctl/%s", version.Version) 146 | 147 | client, err := connect.NewClient(params.ClusterURL, connect.WithUserAgent(userAgent), connect.WithHTTPClient(&http.Client{Timeout: params.HTTPClientTimeout})) 148 | if err != nil { 149 | return errors.Wrap(err, "error creating connect client") 150 | } 151 | 152 | mngr, err := manager.NewConnectorsManager(client, config, manager.WithLogger(logger)) 153 | if err != nil { 154 | return errors.Wrap(err, "error creating connectors manager") 155 | } 156 | 157 | return syncOrManage(logger, params, cmd, mngr) 158 | } 159 | 160 | func syncOrManage(logger *log.Entry, params *manageDefaults, cmd *cobra.Command, mngr *manager.ConnectorManager) error { 161 | if params.EnableHealthCheck { 162 | healthCheckHandler := healthcheck.New(mngr) 163 | 164 | go func() { 165 | if err := healthCheckHandler.Start(params.HealthCheckAddress); err != nil { 166 | logger.WithError(err).Fatalln("error starting healthcheck") 167 | } 168 | }() 169 | // nolint 170 | defer healthCheckHandler.Shutdown(context.Background()) 171 | } 172 | 173 | source, err := findSource(params.Files, params.Directory, params.EnvVar, cmd) 174 | 175 | if err != nil { 176 | return err 177 | } 178 | 179 | stopCh := signals.SetupSignalHandler() 180 | 181 | try.MaxRetries = params.SyncErrorRetryMax 182 | 183 | return try.Do(func(attempt int) (bool, error) { 184 | lgr := logger.WithField("attempt", attempt) 185 | 186 | var ierr error 187 | if params.RunOnce { 188 | lgr.Info("running once") 189 | ierr = mngr.Sync(source) 190 | } else { 191 | lgr.Info("managing") 192 | ierr = mngr.Manage(source, stopCh) 193 | } 194 | 195 | if ierr != nil { 196 | lgr = logger.WithError(ierr) 197 | rootCause := errors.Cause(ierr) 198 | if connect.IsRetryable(rootCause) { 199 | lgr.WithField("attempt", attempt).Error("recoverable error when running manage") 200 | time.Sleep(params.SyncErrorRetryPeriod) 201 | return true, errors.New("retry please") 202 | } 203 | lgr.Error("non-recoverable error when running manage") 204 | return false, ierr 205 | } 206 | lgr.Info("attempt finished") 207 | return false, nil 208 | }) 209 | } 210 | func findSource(files []string, directory, envar string, cmd *cobra.Command) (manager.ConnectorSource, error) { 211 | switch { 212 | case len(files) > 0: 213 | if len(files) == 1 && files[0] == "-" { 214 | return sources.StdIn(cmd.InOrStdin()), nil 215 | } 216 | return sources.Files(files), nil 217 | 218 | case directory != "": 219 | return sources.Directory(directory), nil 220 | case envar != "": 221 | return sources.EnvVarValue(envar), nil 222 | } 223 | return nil, errors.New("error finding connector definitions from parameters") 224 | } 225 | 226 | func checkConfigSwitches(files []string, directory, envar string) error { 227 | paramsSet := 0 228 | 229 | if len(files) != 0 { 230 | paramsSet++ 231 | } 232 | if directory != "" { 233 | paramsSet++ 234 | } 235 | if envar != "" { 236 | paramsSet++ 237 | } 238 | 239 | if paramsSet == 1 { 240 | return nil 241 | } 242 | 243 | return errors.New("you must supply a list of files using --files or a directory that contains files using --directory or an environmental whose value is a JSON serialised connector or array of connectors") 244 | } 245 | -------------------------------------------------------------------------------- /internal/ctl/connectors/pause.go: -------------------------------------------------------------------------------- 1 | package connectors 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/90poe/connectctl/internal/ctl" 7 | "github.com/90poe/connectctl/internal/version" 8 | "github.com/90poe/connectctl/pkg/client/connect" 9 | "github.com/90poe/connectctl/pkg/manager" 10 | "github.com/pkg/errors" 11 | 12 | "github.com/spf13/cobra" 13 | ) 14 | 15 | type pauseConnectorsCmdParams struct { 16 | ClusterURL string 17 | Connectors []string 18 | } 19 | 20 | func pauseConnectorsCmd() *cobra.Command { 21 | params := &pauseConnectorsCmdParams{} 22 | 23 | pauseCmd := &cobra.Command{ 24 | Use: "pause", 25 | Short: "Pause connectors in a cluster", 26 | Long: "", 27 | RunE: func(cmd *cobra.Command, _ []string) error { 28 | return doPauseConnectors(cmd, params) 29 | }, 30 | } 31 | 32 | ctl.AddCommonConnectorsFlags(pauseCmd, ¶ms.ClusterURL) 33 | ctl.AddConnectorNamesFlags(pauseCmd, ¶ms.Connectors) 34 | 35 | return pauseCmd 36 | } 37 | 38 | func doPauseConnectors(_ *cobra.Command, params *pauseConnectorsCmdParams) error { 39 | config := &manager.Config{ 40 | ClusterURL: params.ClusterURL, 41 | Version: version.Version, 42 | } 43 | 44 | userAgent := fmt.Sprintf("90poe.io/connectctl/%s", version.Version) 45 | 46 | client, err := connect.NewClient(params.ClusterURL, connect.WithUserAgent(userAgent)) 47 | if err != nil { 48 | return errors.Wrap(err, "error creating connect client") 49 | } 50 | 51 | mngr, err := manager.NewConnectorsManager(client, config) 52 | if err != nil { 53 | return errors.Wrap(err, "error creating connectors manager") 54 | } 55 | 56 | if err = mngr.Pause(params.Connectors); err != nil { 57 | return errors.Wrap(err, "error pausing connectors") 58 | } 59 | 60 | fmt.Println("connectors paused successfully") 61 | return nil 62 | } 63 | -------------------------------------------------------------------------------- /internal/ctl/connectors/remove.go: -------------------------------------------------------------------------------- 1 | package connectors 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/90poe/connectctl/internal/ctl" 7 | "github.com/90poe/connectctl/internal/version" 8 | "github.com/90poe/connectctl/pkg/client/connect" 9 | "github.com/90poe/connectctl/pkg/manager" 10 | "github.com/pkg/errors" 11 | 12 | "github.com/spf13/cobra" 13 | ) 14 | 15 | type removeConnectorsCmdParams struct { 16 | ClusterURL string 17 | Connectors []string 18 | } 19 | 20 | func removeConnectorCmd() *cobra.Command { 21 | params := &removeConnectorsCmdParams{} 22 | 23 | removeCmd := &cobra.Command{ 24 | Use: "remove", 25 | Short: "Remove connectors from a connect cluster", 26 | Long: "", 27 | RunE: func(cmd *cobra.Command, _ []string) error { 28 | return doRemoveConnectors(cmd, params) 29 | }, 30 | } 31 | 32 | ctl.AddCommonConnectorsFlags(removeCmd, ¶ms.ClusterURL) 33 | ctl.AddConnectorNamesFlags(removeCmd, ¶ms.Connectors) 34 | 35 | return removeCmd 36 | } 37 | 38 | func doRemoveConnectors(_ *cobra.Command, params *removeConnectorsCmdParams) error { 39 | config := &manager.Config{ 40 | ClusterURL: params.ClusterURL, 41 | Version: version.Version, 42 | } 43 | 44 | userAgent := fmt.Sprintf("90poe.io/connectctl/%s", version.Version) 45 | 46 | client, err := connect.NewClient(params.ClusterURL, connect.WithUserAgent(userAgent)) 47 | if err != nil { 48 | return errors.Wrap(err, "error creating connect client") 49 | } 50 | 51 | mngr, err := manager.NewConnectorsManager(client, config) 52 | if err != nil { 53 | return errors.Wrap(err, "error creating connectors manager") 54 | } 55 | if err = mngr.Remove(params.Connectors); err != nil { 56 | return errors.Wrap(err, "error removing connectors") 57 | } 58 | 59 | fmt.Println("removed connectors") 60 | return nil 61 | } 62 | -------------------------------------------------------------------------------- /internal/ctl/connectors/restart.go: -------------------------------------------------------------------------------- 1 | package connectors 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/90poe/connectctl/internal/ctl" 7 | "github.com/90poe/connectctl/internal/version" 8 | "github.com/90poe/connectctl/pkg/client/connect" 9 | "github.com/90poe/connectctl/pkg/manager" 10 | "github.com/pkg/errors" 11 | 12 | "github.com/spf13/cobra" 13 | "github.com/spf13/viper" 14 | ) 15 | 16 | type restartConnectorsCmdParams struct { 17 | ClusterURL string 18 | Connectors []string 19 | ForceRestartTasks bool 20 | RestartTasks bool 21 | TaskIDs []int 22 | } 23 | 24 | // Command creates the the management commands 25 | func restartConnectorsCmd() *cobra.Command { 26 | params := &restartConnectorsCmdParams{} 27 | 28 | restartCmd := &cobra.Command{ 29 | Use: "restart", 30 | Short: "Restart connectors in a cluster", 31 | Long: "", 32 | RunE: func(cmd *cobra.Command, _ []string) error { 33 | return doRestartConnectors(cmd, params) 34 | }, 35 | } 36 | 37 | ctl.AddCommonConnectorsFlags(restartCmd, ¶ms.ClusterURL) 38 | ctl.AddConnectorNamesFlags(restartCmd, ¶ms.Connectors) 39 | 40 | restartCmd.Flags().BoolVar(¶ms.RestartTasks, "restart-tasks", true, "Whether to restart the connector tasks") 41 | _ = viper.BindPFlag("restart-tasks", restartCmd.PersistentFlags().Lookup("restart-tasks")) 42 | 43 | restartCmd.Flags().BoolVar(¶ms.ForceRestartTasks, "force-restart-tasks", false, "Whether to force restart the connector tasks") 44 | _ = viper.BindPFlag("force-restart-tasks", restartCmd.PersistentFlags().Lookup("force-restart-tasks")) 45 | 46 | restartCmd.Flags().IntSliceVarP(¶ms.TaskIDs, "tasks", "t", []int{}, "The task ids to restart (if no ids are specified, all connectors will be restarted)") 47 | _ = viper.BindPFlag("tasks", restartCmd.PersistentFlags().Lookup("tasks")) 48 | 49 | return restartCmd 50 | } 51 | 52 | func doRestartConnectors(_ *cobra.Command, params *restartConnectorsCmdParams) error { 53 | config := &manager.Config{ 54 | ClusterURL: params.ClusterURL, 55 | Version: version.Version, 56 | } 57 | 58 | userAgent := fmt.Sprintf("90poe.io/connectctl/%s", version.Version) 59 | 60 | client, err := connect.NewClient(params.ClusterURL, connect.WithUserAgent(userAgent)) 61 | if err != nil { 62 | return errors.Wrap(err, "error creating connect client") 63 | } 64 | 65 | mngr, err := manager.NewConnectorsManager(client, config) 66 | if err != nil { 67 | return errors.Wrap(err, "error creating connectors manager") 68 | } 69 | 70 | if err = mngr.Restart(params.Connectors, params.RestartTasks, params.ForceRestartTasks, params.TaskIDs); err != nil { 71 | return errors.Wrap(err, "error restarting connectors") 72 | } 73 | 74 | fmt.Printf("connectors restarted successfully") 75 | return nil 76 | } 77 | -------------------------------------------------------------------------------- /internal/ctl/connectors/resume.go: -------------------------------------------------------------------------------- 1 | package connectors 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/90poe/connectctl/internal/ctl" 7 | "github.com/90poe/connectctl/internal/version" 8 | "github.com/90poe/connectctl/pkg/client/connect" 9 | "github.com/90poe/connectctl/pkg/manager" 10 | "github.com/pkg/errors" 11 | 12 | "github.com/spf13/cobra" 13 | ) 14 | 15 | type resumeConnectorsCmdParams struct { 16 | ClusterURL string 17 | Connectors []string 18 | } 19 | 20 | func resumeConnectorsCmd() *cobra.Command { 21 | params := &resumeConnectorsCmdParams{} 22 | 23 | resumeCmd := &cobra.Command{ 24 | Use: "resume", 25 | Short: "Resume connectors in a cluster", 26 | Long: "", 27 | RunE: func(cmd *cobra.Command, _ []string) error { 28 | return doResumeConnectors(cmd, params) 29 | }, 30 | } 31 | 32 | ctl.AddCommonConnectorsFlags(resumeCmd, ¶ms.ClusterURL) 33 | ctl.AddConnectorNamesFlags(resumeCmd, ¶ms.Connectors) 34 | 35 | return resumeCmd 36 | } 37 | 38 | func doResumeConnectors(_ *cobra.Command, params *resumeConnectorsCmdParams) error { 39 | config := &manager.Config{ 40 | ClusterURL: params.ClusterURL, 41 | Version: version.Version, 42 | } 43 | 44 | userAgent := fmt.Sprintf("90poe.io/connectctl/%s", version.Version) 45 | 46 | client, err := connect.NewClient(params.ClusterURL, connect.WithUserAgent(userAgent)) 47 | if err != nil { 48 | return errors.Wrap(err, "error creating connect client") 49 | } 50 | 51 | mngr, err := manager.NewConnectorsManager(client, config) 52 | if err != nil { 53 | return errors.Wrap(err, "error creating connectors manager") 54 | } 55 | 56 | if err = mngr.Resume(params.Connectors); err != nil { 57 | return errors.Wrap(err, "error resuming connectors") 58 | } 59 | 60 | fmt.Println("connectors resumed successfully") 61 | return nil 62 | } 63 | -------------------------------------------------------------------------------- /internal/ctl/connectors/status.go: -------------------------------------------------------------------------------- 1 | package connectors 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/90poe/connectctl/internal/ctl" 9 | "github.com/90poe/connectctl/internal/version" 10 | "github.com/90poe/connectctl/pkg/client/connect" 11 | "github.com/90poe/connectctl/pkg/manager" 12 | "github.com/jedib0t/go-pretty/table" 13 | "github.com/pkg/errors" 14 | 15 | "github.com/spf13/cobra" 16 | ) 17 | 18 | type connectorsStatusCmdParams struct { 19 | ClusterURL string 20 | Connectors []string 21 | Output string 22 | Quiet bool 23 | } 24 | 25 | func connectorsStatusCmd() *cobra.Command { 26 | params := &connectorsStatusCmdParams{} 27 | 28 | statusCmd := &cobra.Command{ 29 | Use: "status", 30 | Short: "Get status for connectors in a cluster", 31 | Long: "", 32 | RunE: func(cmd *cobra.Command, _ []string) error { 33 | return doConnectorsStatus(cmd, params) 34 | }, 35 | } 36 | 37 | ctl.AddCommonConnectorsFlags(statusCmd, ¶ms.ClusterURL) 38 | ctl.AddConnectorNamesFlags(statusCmd, ¶ms.Connectors) 39 | ctl.AddOutputFlags(statusCmd, ¶ms.Output) 40 | ctl.AddQuietFlag(statusCmd, ¶ms.Quiet) 41 | 42 | return statusCmd 43 | } 44 | 45 | func doConnectorsStatus(_ *cobra.Command, params *connectorsStatusCmdParams) error { 46 | config := &manager.Config{ 47 | ClusterURL: params.ClusterURL, 48 | Version: version.Version, 49 | } 50 | 51 | userAgent := fmt.Sprintf("90poe.io/connectctl/%s", version.Version) 52 | 53 | client, err := connect.NewClient(params.ClusterURL, connect.WithUserAgent(userAgent)) 54 | if err != nil { 55 | return errors.Wrap(err, "error creating connect client") 56 | } 57 | 58 | mngr, err := manager.NewConnectorsManager(client, config) 59 | if err != nil { 60 | return errors.Wrap(err, "error creating connectors manager") 61 | } 62 | 63 | statusList, err := mngr.Status(params.Connectors) 64 | if err != nil { 65 | return errors.Wrap(err, "error getting connectors status") 66 | } 67 | 68 | if !params.Quiet { 69 | switch params.Output { 70 | case "json": 71 | if err = printAsJSON(statusList); err != nil { 72 | return errors.Wrap(err, "error printing connectors status as JSON") 73 | } 74 | 75 | case "table": 76 | printAsTable(statusList) 77 | 78 | default: 79 | return fmt.Errorf("invalid output format specified: %s", params.Output) 80 | } 81 | } 82 | 83 | failingConnectors, failingTasks := countFailing(statusList) 84 | 85 | if failingConnectors != 0 || failingTasks != 0 { 86 | return fmt.Errorf("%d connectors are failng, %d tasks are failing", failingConnectors, failingTasks) 87 | } 88 | 89 | return nil 90 | } 91 | 92 | func countFailing(statusList []*connect.ConnectorStatus) (int, int) { 93 | connectorCount := 0 94 | taskCount := 0 95 | 96 | for _, status := range statusList { 97 | if status.Connector.State == "FAILED" { 98 | connectorCount++ 99 | } 100 | 101 | taskCount += countFailingTasks(&status.Tasks) 102 | } 103 | 104 | return connectorCount, taskCount 105 | } 106 | 107 | func countFailingTasks(tasks *[]connect.TaskState) int { 108 | count := 0 109 | 110 | for _, task := range *tasks { 111 | if task.State == "FAILED" { 112 | count++ 113 | } 114 | } 115 | 116 | return count 117 | } 118 | 119 | func printAsJSON(statusList []*connect.ConnectorStatus) error { 120 | b, err := json.MarshalIndent(statusList, "", " ") 121 | if err != nil { 122 | return err 123 | } 124 | 125 | os.Stdout.Write(b) 126 | return nil 127 | } 128 | 129 | func printAsTable(statusList []*connect.ConnectorStatus) { 130 | t := table.NewWriter() 131 | t.SetOutputMirror(os.Stdout) 132 | t.AppendHeader(table.Row{"Name", "State", "WorkerId", "Tasks"}) 133 | 134 | for _, status := range statusList { 135 | tasks := "" 136 | for _, task := range status.Tasks { 137 | tasks += fmt.Sprintf("%d(%s): %s\n", task.ID, task.WorkerID, task.State) 138 | } 139 | 140 | t.AppendRow(table.Row{ 141 | status.Name, 142 | status.Connector.State, 143 | status.Connector.WorkerID, 144 | tasks, 145 | }) 146 | } 147 | 148 | t.Render() 149 | } 150 | -------------------------------------------------------------------------------- /internal/ctl/connectors/status_test.go: -------------------------------------------------------------------------------- 1 | package connectors 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/90poe/connectctl/pkg/client/connect" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestCountFailing(t *testing.T) { 11 | statusList := []*connect.ConnectorStatus{ 12 | { 13 | Connector: connect.ConnectorState{ 14 | State: "RUNNING", 15 | }, 16 | Tasks: []connect.TaskState{ 17 | { 18 | State: "RUNNING", 19 | }, 20 | { 21 | State: "FAILED", 22 | }, 23 | }, 24 | }, 25 | { 26 | Connector: connect.ConnectorState{ 27 | State: "FAILED", 28 | }, 29 | Tasks: []connect.TaskState{ 30 | { 31 | State: "FAILED", 32 | }, 33 | { 34 | State: "FAILED", 35 | }, 36 | }, 37 | }, 38 | } 39 | 40 | connectorsFailing, tasksFailing := countFailing(statusList) 41 | 42 | require.Equal(t, 1, connectorsFailing) 43 | require.Equal(t, 3, tasksFailing) 44 | 45 | } 46 | -------------------------------------------------------------------------------- /internal/ctl/flags.go: -------------------------------------------------------------------------------- 1 | package ctl 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/spf13/cobra" 8 | "github.com/spf13/pflag" 9 | "github.com/spf13/viper" 10 | ) 11 | 12 | func AddClusterFlag(cmd *cobra.Command, required bool, clusterURL *string) { 13 | description := "the URL of the connect cluster" 14 | 15 | if required { 16 | description = requiredDescription(&description) 17 | } 18 | 19 | BindStringVarP(cmd.Flags(), clusterURL, "", "cluster", "c", description) 20 | } 21 | 22 | func AddCommonConnectorsFlags(cmd *cobra.Command, clusterURL *string) { 23 | AddClusterFlag(cmd, true, clusterURL) 24 | } 25 | 26 | func AddOutputFlags(cmd *cobra.Command, output *string) { 27 | BindStringVarP(cmd.Flags(), output, "json", "output", "o", "specify the output format (valid options: json, table)") 28 | } 29 | 30 | func AddQuietFlag(cmd *cobra.Command, quiet *bool) { 31 | BindBoolVarP(cmd.Flags(), quiet, false, "quiet", "q", "disable output logging") 32 | } 33 | 34 | func AddInputFlag(cmd *cobra.Command, required bool, input *string) { 35 | description := "Input data in json format" 36 | 37 | if required { 38 | description = requiredDescription(&description) 39 | } 40 | 41 | BindStringVarP(cmd.Flags(), input, "", "input", "i", description) 42 | } 43 | 44 | func AddDefinitionFilesFlags(cmd *cobra.Command, files *[]string, directory *string, env *string) { 45 | BindStringArrayVarP(cmd.Flags(), files, []string{}, "files", "f", "the connector definitions files (Required if --directory or --env-var not specified)") 46 | BindStringVarP(cmd.Flags(), directory, "", "directory", "d", "the directory containing the connector definitions files (Required if --file or --env-vars not specified)") 47 | BindStringVarP(cmd.Flags(), env, "", "env-var", "e", "an environmental variable whose value is a singular or array of connectors serialised as JSON (Required if --files or --directory not specified)") 48 | } 49 | 50 | func AddConnectorNamesFlags(cmd *cobra.Command, names *[]string) { 51 | BindStringArrayVarP(cmd.Flags(), names, []string{}, "connectors", "n", "The connect names to perform action on (if not specified action will be performed on all connectors)") 52 | } 53 | 54 | func BindDurationVarP(f *pflag.FlagSet, p *time.Duration, value time.Duration, long, short, description string) { 55 | f.DurationVarP(p, long, short, value, description) 56 | _ = viper.BindPFlag(long, f.Lookup(long)) 57 | viper.SetDefault(long, value) 58 | } 59 | 60 | func BindDurationVar(f *pflag.FlagSet, p *time.Duration, value time.Duration, long, description string) { 61 | f.DurationVar(p, long, value, description) 62 | _ = viper.BindPFlag(long, f.Lookup(long)) 63 | viper.SetDefault(long, value) 64 | } 65 | 66 | func BindBoolVar(f *pflag.FlagSet, p *bool, value bool, long, description string) { 67 | f.BoolVar(p, long, value, description) 68 | _ = viper.BindPFlag(long, f.Lookup(long)) 69 | viper.SetDefault(long, value) 70 | } 71 | 72 | func BindBoolVarP(f *pflag.FlagSet, p *bool, value bool, long, short, description string) { 73 | f.BoolVarP(p, long, short, value, description) 74 | _ = viper.BindPFlag(long, f.Lookup(long)) 75 | viper.SetDefault(long, value) 76 | } 77 | 78 | func BindStringVarP(f *pflag.FlagSet, p *string, value, long, short, description string) { 79 | f.StringVarP(p, long, short, value, description) 80 | _ = viper.BindPFlag(long, f.Lookup(long)) 81 | viper.SetDefault(long, value) 82 | } 83 | 84 | func BindStringVar(f *pflag.FlagSet, p *string, value, long, description string) { 85 | f.StringVar(p, long, value, description) 86 | _ = viper.BindPFlag(long, f.Lookup(long)) 87 | viper.SetDefault(long, value) 88 | } 89 | 90 | func BindStringArrayVarP(f *pflag.FlagSet, p *[]string, value []string, long, short, description string) { 91 | f.StringArrayVarP(p, long, short, value, description) 92 | _ = viper.BindPFlag(long, f.Lookup(long)) 93 | viper.SetDefault(long, value) 94 | } 95 | 96 | func BindIntVar(f *pflag.FlagSet, p *int, value int, long, description string) { 97 | f.IntVar(p, long, value, description) 98 | _ = viper.BindPFlag(long, f.Lookup(long)) 99 | viper.SetDefault(long, value) 100 | } 101 | 102 | func requiredDescription(desc *string) string { 103 | return fmt.Sprintf("%s (required)", *desc) 104 | } 105 | -------------------------------------------------------------------------------- /internal/ctl/output.go: -------------------------------------------------------------------------------- 1 | package ctl 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | 7 | "github.com/jedib0t/go-pretty/table" 8 | ) 9 | 10 | func PrintAsJSON(data interface{}) error { 11 | b, err := json.MarshalIndent(data, "", " ") 12 | if err != nil { 13 | return err 14 | } 15 | 16 | os.Stdout.Write(b) 17 | return nil 18 | } 19 | 20 | func PrintAsTable(handler func(table.Writer)) { 21 | t := table.NewWriter() 22 | t.SetOutputMirror(os.Stdout) 23 | 24 | handler(t) 25 | 26 | t.Render() 27 | } 28 | -------------------------------------------------------------------------------- /internal/ctl/plugins/list.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/90poe/connectctl/internal/ctl" 9 | "github.com/90poe/connectctl/internal/version" 10 | "github.com/90poe/connectctl/pkg/client/connect" 11 | "github.com/90poe/connectctl/pkg/manager" 12 | "github.com/pkg/errors" 13 | 14 | "github.com/jedib0t/go-pretty/table" 15 | "github.com/spf13/cobra" 16 | ) 17 | 18 | type listPluginsCmdParams struct { 19 | ClusterURL string 20 | Output string 21 | } 22 | 23 | func listPluginsCmd() *cobra.Command { 24 | params := &listPluginsCmdParams{} 25 | 26 | listCmd := &cobra.Command{ 27 | Use: "list", 28 | Short: "List connector plugins in a cluster", 29 | Long: "", 30 | RunE: func(cmd *cobra.Command, _ []string) error { 31 | return doListPlugins(cmd, params) 32 | }, 33 | } 34 | 35 | ctl.AddCommonConnectorsFlags(listCmd, ¶ms.ClusterURL) 36 | ctl.AddOutputFlags(listCmd, ¶ms.Output) 37 | 38 | return listCmd 39 | } 40 | 41 | func doListPlugins(_ *cobra.Command, params *listPluginsCmdParams) error { 42 | config := &manager.Config{ 43 | ClusterURL: params.ClusterURL, 44 | Version: version.Version, 45 | } 46 | 47 | userAgent := fmt.Sprintf("90poe.io/connectctl/%s", version.Version) 48 | 49 | client, err := connect.NewClient(params.ClusterURL, connect.WithUserAgent(userAgent)) 50 | if err != nil { 51 | return errors.Wrap(err, "error creating connect client") 52 | } 53 | 54 | mngr, err := manager.NewConnectorsManager(client, config) 55 | if err != nil { 56 | return errors.Wrap(err, "error creating connectors manager") 57 | } 58 | 59 | plugins, err := mngr.GetAllPlugins() 60 | if err != nil { 61 | return errors.Wrap(err, "error getting all connector plguns") 62 | } 63 | 64 | switch params.Output { 65 | case "json": 66 | err = printPluginsAsJSON(plugins) 67 | if err != nil { 68 | return errors.Wrap(err, "error printing plugins as JSON") 69 | } 70 | case "table": 71 | printPluginsAsTable(plugins) 72 | default: 73 | return fmt.Errorf("invalid output format specified: %s", params.Output) 74 | } 75 | return nil 76 | } 77 | 78 | func printPluginsAsJSON(plugins []*connect.Plugin) error { 79 | b, err := json.MarshalIndent(plugins, "", " ") 80 | if err != nil { 81 | return err 82 | } 83 | 84 | os.Stdout.Write(b) 85 | return nil 86 | } 87 | 88 | func printPluginsAsTable(plugins []*connect.Plugin) { 89 | t := table.NewWriter() 90 | t.SetOutputMirror(os.Stdout) 91 | t.AppendHeader(table.Row{"Class", "Type", "Version"}) 92 | 93 | for _, plugin := range plugins { 94 | t.AppendRow(table.Row{ 95 | plugin.Class, 96 | plugin.Type, 97 | plugin.Version, 98 | }) 99 | } 100 | t.Render() 101 | } 102 | -------------------------------------------------------------------------------- /internal/ctl/plugins/plugins.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | ) 6 | 7 | // Command creates the plugins command (and subcommands) 8 | func Command() *cobra.Command { 9 | pluginsCmd := &cobra.Command{ 10 | Use: "plugins", 11 | Short: "Commands related to Kafka Connect plugins", 12 | Long: "", 13 | RunE: func(cmd *cobra.Command, _ []string) error { 14 | return cmd.Help() 15 | }, 16 | } 17 | 18 | // Add subcommands 19 | pluginsCmd.AddCommand(listPluginsCmd()) 20 | pluginsCmd.AddCommand(validatePluginsCmd()) 21 | 22 | return pluginsCmd 23 | } 24 | -------------------------------------------------------------------------------- /internal/ctl/plugins/validate.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/90poe/connectctl/internal/ctl" 9 | "github.com/90poe/connectctl/internal/version" 10 | "github.com/90poe/connectctl/pkg/client/connect" 11 | "github.com/90poe/connectctl/pkg/manager" 12 | "github.com/jedib0t/go-pretty/table" 13 | "github.com/pkg/errors" 14 | 15 | "github.com/spf13/cobra" 16 | ) 17 | 18 | type validatePluginsCmdParams struct { 19 | ClusterURL string 20 | Input string 21 | Output string 22 | Quiet bool 23 | } 24 | 25 | func validatePluginsCmd() *cobra.Command { 26 | params := &validatePluginsCmdParams{} 27 | 28 | validateCmd := &cobra.Command{ 29 | Use: "validate", 30 | Short: "Validates plugin config", 31 | Long: "", 32 | RunE: func(cmd *cobra.Command, _ []string) error { 33 | return doValidatePlugins(cmd, params) 34 | }, 35 | } 36 | 37 | ctl.AddClusterFlag(validateCmd, true, ¶ms.ClusterURL) 38 | ctl.AddInputFlag(validateCmd, true, ¶ms.Input) 39 | ctl.AddOutputFlags(validateCmd, ¶ms.Output) 40 | ctl.AddQuietFlag(validateCmd, ¶ms.Quiet) 41 | 42 | return validateCmd 43 | } 44 | 45 | func doValidatePlugins(_ *cobra.Command, params *validatePluginsCmdParams) error { 46 | var inputConfig connect.ConnectorConfig 47 | if err := json.Unmarshal([]byte(params.Input), &inputConfig); err != nil { 48 | return errors.Wrap(err, "error parsing input connector config") 49 | } 50 | 51 | config := &manager.Config{ 52 | ClusterURL: params.ClusterURL, 53 | Version: version.Version, 54 | } 55 | 56 | userAgent := fmt.Sprintf("90poe.io/connectctl/%s", version.Version) 57 | 58 | client, err := connect.NewClient(params.ClusterURL, connect.WithUserAgent(userAgent)) 59 | if err != nil { 60 | return errors.Wrap(err, "error creating connect client") 61 | } 62 | 63 | mngr, err := manager.NewConnectorsManager(client, config) 64 | if err != nil { 65 | return errors.Wrap(err, "error creating connectors manager") 66 | } 67 | 68 | validation, err := mngr.ValidatePlugins(inputConfig) 69 | if err != nil { 70 | return err 71 | } 72 | 73 | if !params.Quiet { 74 | switch params.Output { 75 | case "json": 76 | if err = ctl.PrintAsJSON(validation); err != nil { 77 | return errors.Wrap(err, "error printing validation results as JSON") 78 | } 79 | 80 | case "table": 81 | printAsTable(validation) 82 | 83 | default: 84 | return fmt.Errorf("invalid output format specified: %s", params.Output) 85 | } 86 | } 87 | 88 | if validation.ErrorCount > 0 { 89 | return fmt.Errorf("detected %d errors in the configuation", validation.ErrorCount) 90 | } 91 | 92 | return nil 93 | } 94 | 95 | func printAsTable(validation *connect.ConfigValidation) { 96 | ctl.PrintAsTable(func(t table.Writer) { 97 | t.Style().Options.SeparateRows = true 98 | t.AppendHeader(table.Row{"Name", "Spec", "Value", "Errors"}) 99 | 100 | for _, info := range validation.Configs { 101 | spec := fmt.Sprintf( 102 | "default: %s\nrequired: %v", 103 | ctl.StrPtrToStr(info.Definition.DefaultValue), 104 | info.Definition.Required, 105 | ) 106 | 107 | errors := strings.Join(info.Value.Errors, "\n") 108 | 109 | t.AppendRow(table.Row{ 110 | info.Definition.Name, 111 | spec, 112 | ctl.StrPtrToStr(info.Value.Value), 113 | errors, 114 | }) 115 | } 116 | }) 117 | } 118 | -------------------------------------------------------------------------------- /internal/ctl/tasks/get.go: -------------------------------------------------------------------------------- 1 | package tasks 2 | 3 | import ( 4 | "fmt" 5 | "github.com/90poe/connectctl/pkg/client/connect" 6 | "github.com/pkg/errors" 7 | "github.com/spf13/cobra" 8 | "io" 9 | "strconv" 10 | ) 11 | 12 | type GetOptions struct { 13 | *GenericOptions 14 | 15 | TaskID int 16 | Output string 17 | } 18 | 19 | func NewGetCommand(options *GenericOptions) *cobra.Command { 20 | var opts = GetOptions{ 21 | GenericOptions: options, 22 | } 23 | 24 | var cmd = cobra.Command{ 25 | Use: "get", 26 | Short: "Displays a single task currently running for the connector.", 27 | RunE: func(cmd *cobra.Command, args []string) error { 28 | if err := opts.Validate(); err != nil { 29 | return err 30 | } 31 | return opts.Run(cmd.OutOrStdout()) 32 | }, 33 | } 34 | 35 | cmd.PersistentFlags().StringVarP(&opts.Output, "output", "o", "json", "The output format. Possible values are 'json' and 'table'") 36 | 37 | cmd.PersistentFlags().IntVarP(&opts.TaskID, "id", "", -1, "The ID of the task to get") 38 | _ = cmd.MarkPersistentFlagRequired("id") 39 | 40 | return &cmd 41 | } 42 | 43 | func (o *GetOptions) Run(out io.Writer) error { 44 | client, err := o.CreateClient(o.ClusterURL) 45 | if err != nil { 46 | return errors.Wrap(err, "failed to create http client") 47 | } 48 | tasks, _, err := client.GetConnectorTasks(o.ConnectorName) 49 | if err != nil { 50 | return errors.Wrap(err, "failed to retrieve tasks") 51 | } 52 | task, ok := findTaskByID(tasks, o.TaskID) 53 | if !ok { 54 | return errors.New("no task found by id=" + strconv.Itoa(o.TaskID)) 55 | } 56 | return o.writeOutput(task, out) 57 | } 58 | 59 | func (o *GetOptions) writeOutput(task connect.Task, out io.Writer) error { 60 | var outputFn TaskListOutputFn 61 | var outputType = OutputType(o.Output) 62 | switch outputType { 63 | case OutputTypeJSON: 64 | outputFn = OutputTaskListAsJSON 65 | case OutputTypeTable: 66 | outputFn = OutputTaskListAsTable 67 | default: 68 | return fmt.Errorf("output type '%s' is not supported", o.Output) 69 | } 70 | output, err := outputFn([]connect.Task{task}) 71 | if err != nil { 72 | return fmt.Errorf("failed to form output for '%s' type", outputType) 73 | } 74 | if _, err := out.Write(output); err != nil { 75 | return errors.Wrap(err, "failed to write output") 76 | } 77 | return nil 78 | } 79 | -------------------------------------------------------------------------------- /internal/ctl/tasks/list.go: -------------------------------------------------------------------------------- 1 | package tasks 2 | 3 | import ( 4 | "fmt" 5 | "github.com/90poe/connectctl/pkg/client/connect" 6 | "github.com/pkg/errors" 7 | "github.com/spf13/cobra" 8 | "io" 9 | ) 10 | 11 | type ListOptions struct { 12 | // Use pointer here to make reference to the 'tasks' options. 13 | // Otherwise, the local copy of the config will be created an no 14 | // value from the flags will be set. It is caused by the global 15 | // scope nature of viper and multiple overrides of cluster flag 16 | // from the other commands. 17 | *GenericOptions 18 | 19 | Output string 20 | } 21 | 22 | func NewListCommand(options *GenericOptions) *cobra.Command { 23 | var opts = ListOptions{ 24 | GenericOptions: options, 25 | } 26 | 27 | var cmd = cobra.Command{ 28 | Use: "list", 29 | Short: "Displays a list of tasks currently running for the connector.", 30 | RunE: func(cmd *cobra.Command, args []string) error { 31 | if err := opts.Validate(); err != nil { 32 | return err 33 | } 34 | return opts.Run(cmd.OutOrStdout()) 35 | }, 36 | } 37 | 38 | cmd.PersistentFlags().StringVarP(&opts.Output, "output", "o", "json", "The output format. Possible values are 'json' and 'table'") 39 | 40 | return &cmd 41 | } 42 | 43 | func (o *ListOptions) Run(out io.Writer) error { 44 | client, err := o.CreateClient(o.ClusterURL) 45 | if err != nil { 46 | return errors.Wrap(err, "failed to create http client") 47 | } 48 | tasks, _, err := client.GetConnectorTasks(o.ConnectorName) 49 | if err != nil { 50 | return errors.Wrap(err, "failed to retrieve tasks") 51 | } 52 | return o.writeOutput(tasks, out) 53 | } 54 | 55 | func (o *ListOptions) writeOutput(tasks []connect.Task, out io.Writer) error { 56 | var outputFn TaskListOutputFn 57 | var outputType = OutputType(o.Output) 58 | switch outputType { 59 | case OutputTypeJSON: 60 | outputFn = OutputTaskListAsJSON 61 | case OutputTypeTable: 62 | outputFn = OutputTaskListAsTable 63 | default: 64 | return fmt.Errorf("output type '%s' is not supported", o.Output) 65 | } 66 | output, err := outputFn(tasks) 67 | if err != nil { 68 | return fmt.Errorf("failed to form output for '%s' type", outputType) 69 | } 70 | if _, err := out.Write(output); err != nil { 71 | return errors.Wrap(err, "failed to write output") 72 | } 73 | return nil 74 | } 75 | -------------------------------------------------------------------------------- /internal/ctl/tasks/output.go: -------------------------------------------------------------------------------- 1 | package tasks 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "github.com/90poe/connectctl/pkg/client/connect" 7 | "github.com/jedib0t/go-pretty/table" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | type OutputType string 12 | 13 | const ( 14 | OutputTypeJSON OutputType = "json" 15 | OutputTypeTable OutputType = "table" 16 | ) 17 | 18 | type TaskListOutputFn func([]connect.Task) ([]byte, error) 19 | 20 | func OutputTaskListAsJSON(tasks []connect.Task) ([]byte, error) { 21 | b, err := DefaultMarshalIndent(tasks) 22 | if err != nil { 23 | return nil, errors.Wrap(err, "failed to marshal tasks") 24 | } 25 | return b, nil 26 | } 27 | 28 | func OutputTaskListAsTable(tasks []connect.Task) ([]byte, error) { 29 | var buff bytes.Buffer 30 | t := table.NewWriter() 31 | t.Style().Options.SeparateRows = true 32 | t.SetOutputMirror(&buff) 33 | t.AppendHeader(table.Row{"Connector Name", "ID", "Config"}) 34 | for _, task := range tasks { 35 | configBytes, err := DefaultMarshalIndent(task.Config) 36 | if err != nil { 37 | return nil, errors.Wrapf(err, "failed to marshal config %v", task.Config) 38 | } 39 | t.AppendRow(table.Row{task.ID.ConnectorName, task.ID.ID, string(configBytes)}) 40 | } 41 | t.Render() 42 | return buff.Bytes(), nil 43 | } 44 | 45 | func DefaultMarshalIndent(value interface{}) ([]byte, error) { 46 | return json.MarshalIndent(value, "", " ") 47 | } 48 | 49 | type TaskStateOutputFn func(*connect.TaskState) ([]byte, error) 50 | 51 | func OutputTaskStateAsJSON(taskState *connect.TaskState) ([]byte, error) { 52 | b, err := DefaultMarshalIndent(taskState) 53 | if err != nil { 54 | return nil, errors.Wrap(err, "failed to marshal task state") 55 | } 56 | return b, nil 57 | } 58 | 59 | func OutputTaskStateAsTable(taskState *connect.TaskState) ([]byte, error) { 60 | var buff bytes.Buffer 61 | t := table.NewWriter() 62 | t.SetOutputMirror(&buff) 63 | t.AppendHeader(table.Row{"ID", "WorkerID", "State", "Trace"}) 64 | t.AppendRow(table.Row{taskState.ID, taskState.WorkerID, taskState.State, taskState.Trace}) 65 | t.Render() 66 | return buff.Bytes(), nil 67 | } 68 | -------------------------------------------------------------------------------- /internal/ctl/tasks/restart.go: -------------------------------------------------------------------------------- 1 | package tasks 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "github.com/spf13/cobra" 6 | ) 7 | 8 | type RestartOptions struct { 9 | *GenericOptions 10 | 11 | TaskID int 12 | } 13 | 14 | func NewRestartCommand(options *GenericOptions) *cobra.Command { 15 | var opts = RestartOptions{ 16 | GenericOptions: options, 17 | } 18 | 19 | var cmd = cobra.Command{ 20 | Use: "restart", 21 | Short: "Restart an individual task for the specified connector.", 22 | RunE: func(cmd *cobra.Command, args []string) error { 23 | if err := opts.Validate(); err != nil { 24 | return err 25 | } 26 | return opts.Run() 27 | }, 28 | } 29 | 30 | cmd.PersistentFlags().IntVarP(&opts.TaskID, "id", "", -1, "The ID of the task to restart") 31 | _ = cmd.MarkPersistentFlagRequired("id") 32 | 33 | return &cmd 34 | } 35 | 36 | func (o *RestartOptions) Run() error { 37 | client, err := o.CreateClient(o.ClusterURL) 38 | if err != nil { 39 | return errors.Wrap(err, "failed to create http client") 40 | } 41 | _, err = client.RestartConnectorTask(o.ConnectorName, o.TaskID) 42 | if err != nil { 43 | return errors.Wrapf(err, "failed to restart task '%d' for connector '%s'", o.TaskID, o.ConnectorName) 44 | } 45 | return nil 46 | } 47 | -------------------------------------------------------------------------------- /internal/ctl/tasks/status.go: -------------------------------------------------------------------------------- 1 | package tasks 2 | 3 | import ( 4 | "fmt" 5 | "github.com/90poe/connectctl/pkg/client/connect" 6 | "github.com/pkg/errors" 7 | "github.com/spf13/cobra" 8 | "io" 9 | ) 10 | 11 | type StatusOptions struct { 12 | *GenericOptions 13 | 14 | TaskID int 15 | Output string 16 | } 17 | 18 | func NewStatusCommand(options *GenericOptions) *cobra.Command { 19 | var opts = StatusOptions{ 20 | GenericOptions: options, 21 | } 22 | 23 | var cmd = cobra.Command{ 24 | Use: "status", 25 | Short: "Displays a status by individual task currently running for the connector.", 26 | RunE: func(cmd *cobra.Command, args []string) error { 27 | if err := opts.Validate(); err != nil { 28 | return err 29 | } 30 | return opts.Run(cmd.OutOrStdout()) 31 | }, 32 | } 33 | 34 | cmd.PersistentFlags().StringVarP(&opts.Output, "output", "o", "json", "The output format. Possible values are 'json' and 'table'") 35 | 36 | cmd.PersistentFlags().IntVarP(&opts.TaskID, "id", "", -1, "The ID of the task to get status for") 37 | _ = cmd.MarkPersistentFlagRequired("id") 38 | 39 | return &cmd 40 | } 41 | 42 | func (o *StatusOptions) Run(out io.Writer) error { 43 | client, err := o.CreateClient(o.ClusterURL) 44 | if err != nil { 45 | return errors.Wrap(err, "failed to create http client") 46 | } 47 | taskState, _, err := client.GetConnectorTaskStatus(o.ConnectorName, o.TaskID) 48 | if err != nil { 49 | return errors.Wrap(err, "failed to get task status") 50 | } 51 | if taskState == nil { 52 | return errors.New("task state response is nil") 53 | } 54 | return o.writeOutput(taskState, out) 55 | } 56 | 57 | func (o *StatusOptions) writeOutput(taskState *connect.TaskState, out io.Writer) error { 58 | var outputFn TaskStateOutputFn 59 | var outputType = OutputType(o.Output) 60 | switch outputType { 61 | case OutputTypeJSON: 62 | outputFn = OutputTaskStateAsJSON 63 | case OutputTypeTable: 64 | outputFn = OutputTaskStateAsTable 65 | default: 66 | return fmt.Errorf("output type '%s' is not supported", o.Output) 67 | } 68 | output, err := outputFn(taskState) 69 | if err != nil { 70 | return fmt.Errorf("failed to form output for '%s' type", outputType) 71 | } 72 | if _, err := out.Write(output); err != nil { 73 | return errors.Wrap(err, "failed to write output") 74 | } 75 | return nil 76 | } 77 | -------------------------------------------------------------------------------- /internal/ctl/tasks/tasks.go: -------------------------------------------------------------------------------- 1 | package tasks 2 | 3 | import ( 4 | "github.com/90poe/connectctl/pkg/client/connect" 5 | "github.com/pkg/errors" 6 | "github.com/spf13/cobra" 7 | "net/http" 8 | "net/url" 9 | "strings" 10 | ) 11 | 12 | type ClientFn func(string) (Client, error) 13 | 14 | type Client interface { 15 | GetConnectorTasks(name string) ([]connect.Task, *http.Response, error) 16 | GetConnectorTaskStatus(name string, taskID int) (*connect.TaskState, *http.Response, error) 17 | RestartConnectorTask(name string, taskID int) (*http.Response, error) 18 | } 19 | 20 | type GenericOptions struct { 21 | // Function for creating API client 22 | CreateClient ClientFn 23 | 24 | ClusterURL string 25 | ConnectorName string 26 | } 27 | 28 | func Command(opts *GenericOptions) *cobra.Command { 29 | var cmd = cobra.Command{ 30 | Use: "tasks", 31 | Short: "Commands related to kafka connector tasks", 32 | RunE: func(cmd *cobra.Command, args []string) error { 33 | if err := opts.Validate(); err != nil { 34 | return err 35 | } 36 | // return 'help' by default 37 | return cmd.Help() 38 | }, 39 | } 40 | 41 | // ClusterURL will be used in sub-commands 42 | cmd.PersistentFlags().StringVarP(&opts.ClusterURL, "cluster", "c", "", "the URL of the connect cluster to manage (required)") 43 | _ = cmd.MarkPersistentFlagRequired("cluster") 44 | 45 | // ConnectorName will be used in sub-commands 46 | cmd.PersistentFlags().StringVar(&opts.ConnectorName, "connector", "", "Connector name to get tasks for") 47 | _ = cmd.MarkPersistentFlagRequired("connector") 48 | 49 | // connectctl tasks list --cluster=... --connector=... 50 | cmd.AddCommand(NewListCommand(opts)) 51 | 52 | // connectctl task get --cluster=... --connector=... --id=... 53 | cmd.AddCommand(NewGetCommand(opts)) 54 | 55 | // connectctl task restart --cluster=... --connector=... --id=... 56 | cmd.AddCommand(NewRestartCommand(opts)) 57 | 58 | // connectctl task status --cluster=... --connector=... --id=... 59 | cmd.AddCommand(NewStatusCommand(opts)) 60 | 61 | return &cmd 62 | } 63 | 64 | func (o *GenericOptions) Validate() error { 65 | _, err := url.ParseRequestURI(o.ClusterURL) 66 | if err != nil { 67 | return errors.Wrap(err, "--cluster is not a valid URI") 68 | } 69 | if len(strings.TrimSpace(o.ConnectorName)) == 0 { 70 | return errors.New("--connector name is empty") 71 | } 72 | return nil 73 | } 74 | -------------------------------------------------------------------------------- /internal/ctl/tasks/tasks_test.go: -------------------------------------------------------------------------------- 1 | package tasks 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "testing" 6 | ) 7 | 8 | func TestGenericOptions_Validate(t *testing.T) { 9 | f := func(opts GenericOptions, expectedErr error) { 10 | err := opts.Validate() 11 | if err != nil && errors.Cause(err) == expectedErr { 12 | t.Fatalf("expected %#v, got %#v", expectedErr, err) 13 | } 14 | } 15 | 16 | const ( 17 | clusterURL = "http://localhost:8083" 18 | connectorName = "connector" 19 | ) 20 | 21 | var ( 22 | clusterErr = errors.New("--cluster is not a valid URI") 23 | connectorErr = errors.New("--connector name is empty") 24 | ) 25 | 26 | f( 27 | GenericOptions{ 28 | ClusterURL: "", 29 | }, 30 | clusterErr, 31 | ) 32 | f( 33 | GenericOptions{ 34 | ConnectorName: "", 35 | }, 36 | clusterErr, 37 | ) 38 | f( 39 | GenericOptions{ 40 | ClusterURL: "simple:string", 41 | }, 42 | clusterErr, 43 | ) 44 | f( 45 | GenericOptions{ 46 | ClusterURL: clusterURL, 47 | }, 48 | connectorErr, 49 | ) 50 | f( 51 | GenericOptions{ 52 | ClusterURL: clusterURL, 53 | ConnectorName: connectorName, 54 | }, 55 | nil, 56 | ) 57 | f( 58 | GenericOptions{ 59 | ClusterURL: "www.google.com", 60 | ConnectorName: connectorName, 61 | }, 62 | clusterErr, 63 | ) 64 | f( 65 | GenericOptions{ 66 | ClusterURL: "https://www.google.com", 67 | ConnectorName: connectorName, 68 | }, 69 | nil, 70 | ) 71 | } 72 | -------------------------------------------------------------------------------- /internal/ctl/tasks/utils.go: -------------------------------------------------------------------------------- 1 | package tasks 2 | 3 | import ( 4 | "github.com/90poe/connectctl/pkg/client/connect" 5 | ) 6 | 7 | func findTaskByID(tasks []connect.Task, id int) (connect.Task, bool) { 8 | for _, t := range tasks { 9 | if t.ID.ID == id { 10 | return t, true 11 | } 12 | } 13 | return connect.Task{}, false 14 | } 15 | -------------------------------------------------------------------------------- /internal/ctl/tasks/utils_test.go: -------------------------------------------------------------------------------- 1 | package tasks 2 | 3 | import ( 4 | "github.com/90poe/connectctl/pkg/client/connect" 5 | "reflect" 6 | "testing" 7 | ) 8 | 9 | func TestFindTaskByID(t *testing.T) { 10 | f := func(tasks []connect.Task, id int, expected connect.Task, found bool) { 11 | t.Helper() 12 | task, ok := findTaskByID(tasks, id) 13 | if ok != found { 14 | t.Fatalf("expected '%t', got '%t'", found, ok) 15 | } 16 | if !reflect.DeepEqual(task, expected) { 17 | t.Fatalf("expected %#v, got %#v", expected, task) 18 | } 19 | } 20 | 21 | empty := connect.Task{} 22 | 23 | tasks := []connect.Task{ 24 | { 25 | ID: connect.TaskID{ 26 | ConnectorName: "a", 27 | ID: 1, 28 | }, 29 | Config: nil, 30 | }, 31 | { 32 | ID: connect.TaskID{ 33 | ConnectorName: "b", 34 | ID: 3, 35 | }, 36 | Config: nil, 37 | }, 38 | { 39 | ID: connect.TaskID{ 40 | ConnectorName: "c", 41 | ID: 2, 42 | }, 43 | Config: nil, 44 | }, 45 | } 46 | 47 | f(nil, 1, empty, false) 48 | f([]connect.Task{}, 1, empty, false) 49 | f( 50 | tasks, 51 | 1, 52 | connect.Task{ 53 | ID: connect.TaskID{ 54 | ConnectorName: "a", 55 | ID: 1, 56 | }, 57 | Config: nil, 58 | }, 59 | true, 60 | ) 61 | f( 62 | tasks, 63 | 3, 64 | connect.Task{ 65 | ID: connect.TaskID{ 66 | ConnectorName: "b", 67 | ID: 3, 68 | }, 69 | Config: nil, 70 | }, 71 | true, 72 | ) 73 | f( 74 | tasks, 75 | 2, 76 | connect.Task{ 77 | ID: connect.TaskID{ 78 | ConnectorName: "c", 79 | ID: 2, 80 | }, 81 | Config: nil, 82 | }, 83 | true, 84 | ) 85 | f(tasks, -1, empty, false) 86 | f(tasks, 4, empty, false) 87 | } 88 | -------------------------------------------------------------------------------- /internal/ctl/transform.go: -------------------------------------------------------------------------------- 1 | package ctl 2 | 3 | func StrPtrToStr(str *string) string { 4 | if str == nil { 5 | return "null" 6 | } 7 | 8 | return *str 9 | } 10 | -------------------------------------------------------------------------------- /internal/ctl/version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | 7 | "github.com/pkg/errors" 8 | "github.com/spf13/cobra" 9 | 10 | "github.com/90poe/connectctl/internal/ctl" 11 | "github.com/90poe/connectctl/internal/version" 12 | "github.com/90poe/connectctl/pkg/client/connect" 13 | "github.com/90poe/connectctl/pkg/manager" 14 | ) 15 | 16 | type versionCmdParams struct { 17 | ClusterURL string 18 | } 19 | 20 | // Command creates the the management commands 21 | func Command() *cobra.Command { 22 | params := &versionCmdParams{} 23 | 24 | versionCmd := &cobra.Command{ 25 | Use: "version", 26 | Short: "Display version information", 27 | Long: "", 28 | RunE: func(cmd *cobra.Command, _ []string) error { 29 | return doVersion(cmd, params) 30 | }, 31 | } 32 | 33 | ctl.AddClusterFlag(versionCmd, false, ¶ms.ClusterURL) 34 | 35 | return versionCmd 36 | } 37 | 38 | func doVersion(_ *cobra.Command, params *versionCmdParams) error { 39 | var ( 40 | clusterInfo *connect.ClusterInfo 41 | err error 42 | ) 43 | 44 | if params.ClusterURL != "" { 45 | clusterInfo, err = getClusterInfo(params.ClusterURL) 46 | if err != nil { 47 | return err 48 | } 49 | } 50 | 51 | fmt.Printf("Version: %s\n", version.Version) 52 | fmt.Printf("Commit: %s\n", version.GitHash) 53 | fmt.Printf("Build Date: %s\n", version.BuildDate) 54 | fmt.Printf("GO Version: %s\n", runtime.Version()) 55 | fmt.Printf("GOOS: %s\n", runtime.GOOS) 56 | fmt.Printf("GOARCH: %s\n", runtime.GOARCH) 57 | 58 | if clusterInfo != nil { 59 | fmt.Printf("Connect Worker Version: %s\n", clusterInfo.Version) 60 | } 61 | 62 | return nil 63 | } 64 | 65 | func getClusterInfo(clusterURL string) (*connect.ClusterInfo, error) { 66 | config := &manager.Config{ 67 | ClusterURL: clusterURL, 68 | Version: version.Version, 69 | } 70 | 71 | userAgent := fmt.Sprintf("90poe.io/connectctl/%s", version.Version) 72 | 73 | client, err := connect.NewClient(config.ClusterURL, connect.WithUserAgent(userAgent)) 74 | if err != nil { 75 | return nil, errors.Wrap(err, "error creating connect client") 76 | } 77 | 78 | mngr, err := manager.NewConnectorsManager(client, config) 79 | if err != nil { 80 | return nil, errors.Wrap(err, "error creating connectors manager") 81 | } 82 | 83 | clusterInfo, err := mngr.GetClusterInfo() 84 | if err != nil { 85 | return nil, errors.Wrap(err, "error getting cluster info") 86 | } 87 | 88 | return clusterInfo, nil 89 | } 90 | -------------------------------------------------------------------------------- /internal/healthcheck/healthcheck.go: -------------------------------------------------------------------------------- 1 | package healthcheck 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | 7 | "github.com/heptiolabs/healthcheck" 8 | ) 9 | 10 | type health struct { 11 | healthcheck.Handler 12 | server *http.Server 13 | } 14 | 15 | // healthCheckable constrains the type accepted to the registerHealthChecks to 16 | // instances that implement a Readiness and a Liveness probe 17 | type healthCheckable interface { 18 | ReadinessCheck() (string, func() error) 19 | LivenessCheck() (string, func() error) 20 | } 21 | 22 | // New initiates the healthchecks as an HTTP server 23 | func New(healthCheckables ...healthCheckable) *health { // nolint 24 | h := &health{Handler: healthcheck.NewHandler()} 25 | h.Append(healthCheckables...) 26 | return h 27 | } 28 | 29 | // Append will register the healthchecks 30 | func (h *health) Append(healthCheckables ...healthCheckable) { 31 | for _, check := range healthCheckables { 32 | key, f := check.LivenessCheck() 33 | h.AddLivenessCheck(key, f) 34 | 35 | key, f = check.ReadinessCheck() 36 | h.AddReadinessCheck(key, f) 37 | } 38 | } 39 | 40 | // Start binds to the given address or returns an error 41 | // Will block so start in a go routine. 42 | func (h *health) Start(address string) error { 43 | h.server = &http.Server{Addr: address, Handler: h.Handler} 44 | 45 | if err := h.server.ListenAndServe(); err != http.ErrServerClosed { 46 | return err 47 | } 48 | 49 | return nil 50 | } 51 | 52 | // Shutdown will close the underlying http server 53 | func (h *health) Shutdown(ctx context.Context) error { 54 | return h.server.Shutdown(ctx) 55 | } 56 | -------------------------------------------------------------------------------- /internal/logging/logging.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "bufio" 5 | "os" 6 | "strings" 7 | 8 | "github.com/pkg/errors" 9 | "github.com/sirupsen/logrus" 10 | ) 11 | 12 | // Configure sets up the logger 13 | func Configure(logLevel, logFile, logFormat string) error { 14 | hostname, err := os.Hostname() 15 | if err != nil { 16 | return errors.Wrap(err, "getting hostname") 17 | } 18 | logrus.WithField("hostname", hostname) 19 | 20 | // use a file if you want 21 | if logFile != "" { 22 | f, errOpen := os.OpenFile(logFile, os.O_RDWR|os.O_APPEND, 0660) 23 | if errOpen != nil { 24 | return errors.Wrapf(errOpen, "opening log file %s", logFile) 25 | } 26 | logrus.SetOutput(bufio.NewWriter(f)) 27 | } 28 | 29 | if logLevel != "" { 30 | level, err := logrus.ParseLevel(strings.ToUpper(logLevel)) 31 | if err != nil { 32 | return errors.Wrapf(err, "setting log level to %s", level) 33 | } 34 | logrus.SetLevel(level) 35 | } 36 | 37 | if strings.ToUpper(logFormat) == "JSON" { 38 | logrus.SetFormatter(&logrus.JSONFormatter{}) 39 | } else { 40 | // always use the fulltimestamp 41 | logrus.SetFormatter(&logrus.TextFormatter{ 42 | FullTimestamp: true, 43 | }) 44 | } 45 | // Set log output to stderr 46 | logrus.SetOutput(os.Stderr) 47 | 48 | return nil 49 | } 50 | -------------------------------------------------------------------------------- /internal/version/version.go: -------------------------------------------------------------------------------- 1 | //nolint:gochecknoglobals 2 | package version 3 | 4 | import "fmt" 5 | 6 | // Version specifies the version 7 | var Version string 8 | 9 | // BuildDate specifies the date 10 | var BuildDate string 11 | 12 | // GitHash specifies the commit has associated with the build 13 | var GitHash string 14 | 15 | // ToString will convert the version information to a string 16 | func ToString() string { 17 | return fmt.Sprintf("Version: %s, Build Date: %s, Git Commit: %s", Version, BuildDate, GitHash) 18 | } 19 | -------------------------------------------------------------------------------- /pkg/client/connect/LICENSE: -------------------------------------------------------------------------------- 1 | The code in this package is originally (and now modified) from https://github.com/go-kafka/connect and its license is below: 2 | 3 | MIT License 4 | 5 | Copyright (c) 2016 Ches Martin 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy 8 | of this software and associated documentation files (the "Software"), to deal 9 | in the Software without restriction, including without limitation the rights 10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the Software is 12 | furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in all 15 | copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | SOFTWARE. -------------------------------------------------------------------------------- /pkg/client/connect/client.go: -------------------------------------------------------------------------------- 1 | // NOTE: Code originally from https://github.com/go-kafka/connect 2 | 3 | package connect 4 | 5 | import ( 6 | "bytes" 7 | "encoding/json" 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | "net/http" 12 | "net/url" 13 | 14 | "github.com/pkg/errors" 15 | ) 16 | 17 | const ( 18 | // StatusUnprocessableEntity is the status code returned when sending a 19 | // request with invalid fields. 20 | StatusUnprocessableEntity = 422 21 | ) 22 | 23 | const ( 24 | userAgentDefault = "90poe.io/connectctl/noversion" 25 | ) 26 | 27 | // Client manages communication with the Kafka Connect REST API. 28 | type Client struct { 29 | host *url.URL // Base host URL for API requests. 30 | 31 | // HTTP client used to communicate with the API. By default 32 | // http.DefaultClient will be used. 33 | httpClient *http.Client 34 | 35 | // User agent used when communicating with the Kafka Connect API. 36 | userAgent string 37 | } 38 | 39 | // Option can be supplied that override the default Clients properties 40 | type Option func(c *Client) 41 | 42 | // WithUserAgent allows the userAgent to be overridden 43 | func WithUserAgent(userAgent string) Option { 44 | return func(c *Client) { 45 | c.userAgent = userAgent 46 | } 47 | } 48 | 49 | // WithHTTPClient allows a specific http.Client to be set 50 | func WithHTTPClient(httpClient *http.Client) Option { 51 | return func(c *Client) { 52 | c.httpClient = httpClient 53 | } 54 | } 55 | 56 | // NewClient returns a new Kafka Connect API client that communicates host. 57 | func NewClient(host string, opts ...Option) (*Client, error) { 58 | hostURL, err := url.Parse(host) 59 | if err != nil { 60 | return nil, errors.Wrapf(err, "error parsing url %s", host) 61 | } 62 | 63 | c := &Client{ 64 | host: hostURL, 65 | userAgent: userAgentDefault, 66 | httpClient: http.DefaultClient, 67 | } 68 | 69 | for _, opt := range opts { 70 | opt(c) 71 | } 72 | 73 | return c, nil 74 | } 75 | 76 | // Host returns the API root URL the Client is configured to talk to. 77 | func (c *Client) Host() string { 78 | return c.host.String() 79 | } 80 | 81 | // NewRequest creates an API request. A relative URL can be provided in path, 82 | // in which case it is resolved relative to the BaseURL of the Client. 83 | // Relative URLs should always be specified without a preceding slash. If 84 | // specified, the value pointed to by body is JSON-encoded and included as the 85 | // request body. 86 | func (c *Client) NewRequest(method, path string, body interface{}) (*http.Request, error) { 87 | rel, err := url.Parse(path) 88 | if err != nil { 89 | return nil, err 90 | } 91 | 92 | url := c.host.ResolveReference(rel) 93 | 94 | var contentType string 95 | var buf io.ReadWriter 96 | if body != nil { 97 | buf = new(bytes.Buffer) 98 | errEnc := json.NewEncoder(buf).Encode(body) 99 | if errEnc != nil { 100 | return nil, errEnc 101 | } 102 | contentType = "application/json" 103 | } 104 | 105 | request, err := http.NewRequest(method, url.String(), buf) 106 | if err != nil { 107 | return nil, err 108 | } 109 | 110 | request.Header.Set("Accept", "application/json") 111 | if contentType != "" { 112 | request.Header.Set("Content-Type", contentType) 113 | } 114 | request.Header.Set("User-Agent", c.userAgent) 115 | 116 | return request, nil 117 | } 118 | 119 | // Do sends an API request and returns the API response. The API response is 120 | // JSON-decoded and stored in the value pointed to by v, or returned as an 121 | // error if an API or HTTP error has occurred. 122 | func (c *Client) Do(req *http.Request, v interface{}) (*http.Response, error) { 123 | response, err := c.httpClient.Do(req) 124 | if err != nil { 125 | return nil, err 126 | } 127 | defer response.Body.Close() 128 | 129 | if response.StatusCode >= 400 { 130 | return response, buildError(req, response) 131 | } 132 | 133 | if v != nil { 134 | err = json.NewDecoder(response.Body).Decode(v) 135 | if err == io.EOF { 136 | err = nil // ignore EOF, empty response body 137 | } 138 | } 139 | 140 | return response, err 141 | } 142 | 143 | // Simple GET helper with no request body. 144 | func (c *Client) get(path string, v interface{}) (*http.Response, error) { 145 | return c.doRequest("GET", path, nil, v) 146 | } 147 | 148 | func (c *Client) delete(path string) (*http.Response, error) { 149 | return c.doRequest("DELETE", path, nil, nil) 150 | } 151 | 152 | func (c *Client) doRequest(method, path string, body, v interface{}) (*http.Response, error) { 153 | request, err := c.NewRequest(method, path, body) 154 | if err != nil { 155 | return nil, err 156 | } 157 | 158 | return c.Do(request, v) 159 | } 160 | 161 | func buildError(req *http.Request, resp *http.Response) error { 162 | apiError := &APIError{Response: resp} 163 | data, err := ioutil.ReadAll(resp.Body) 164 | if err == nil && data != nil { 165 | _ = json.Unmarshal(data, &apiError) // Fall back on general error below 166 | } 167 | 168 | // Possibly a general HTTP error, e.g. we're not even talking to a valid 169 | // Kafka Connect API host 170 | if apiError.Code == 0 { 171 | return fmt.Errorf("HTTP %v on %v %v", resp.Status, req.Method, req.URL) 172 | } 173 | return apiError 174 | } 175 | -------------------------------------------------------------------------------- /pkg/client/connect/cluster.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | // ClusterInfo - this is new and not from the original author 8 | type ClusterInfo struct { 9 | Version string `json:"version"` 10 | Commit string `json:"commit"` 11 | KafkaClusterID string `json:"kafka_cluster_id"` 12 | } 13 | 14 | // GetClusterInfo retrieves information about a cluster 15 | // 16 | // See: https://docs.confluent.io/current/connect/references/restapi.html#kconnect-cluster 17 | func (c *Client) GetClusterInfo() (*ClusterInfo, *http.Response, error) { 18 | path := "/" 19 | clusterInfo := new(ClusterInfo) 20 | response, err := c.get(path, &clusterInfo) 21 | return clusterInfo, response, err 22 | } 23 | -------------------------------------------------------------------------------- /pkg/client/connect/connectors.go: -------------------------------------------------------------------------------- 1 | // NOTE: Code originally from https://github.com/go-kafka/connect 2 | 3 | package connect 4 | 5 | import ( 6 | "errors" 7 | "fmt" 8 | "net/http" 9 | "reflect" 10 | ) 11 | 12 | // Connector represents a Kafka Connect connector instance. 13 | // 14 | // See: http://docs.confluent.io/current/connect/userguide.html#connectors-tasks-and-workers 15 | type Connector struct { 16 | Name string `json:"name"` 17 | Config ConnectorConfig `json:"config,omitempty"` 18 | Tasks []TaskID `json:"tasks,omitempty"` 19 | } 20 | 21 | // ConfigEqual will compare 2 instances of a connector config 22 | // and return true or false if they are equal. 23 | // 24 | // Note: tasks are ignored in the comparison 25 | func (c *Connector) ConfigEqual(other Connector) bool { 26 | if c.Name != other.Name { 27 | return false 28 | } 29 | 30 | return reflect.DeepEqual(c.Config, other.Config) 31 | } 32 | 33 | // ConnectorConfig is a key-value mapping of configuration for connectors, where 34 | // keys are in the form of Java properties. 35 | // 36 | // See: http://docs.confluent.io/current/connect/userguide.html#configuring-connectors 37 | type ConnectorConfig map[string]string 38 | 39 | // Task is a unit of work dispatched by a Connector to parallelize the work of 40 | // a data copy job. 41 | // 42 | // See: http://docs.confluent.io/current/connect/userguide.html#connectors-tasks-and-workers 43 | type Task struct { 44 | ID TaskID `json:"id"` 45 | Config map[string]string `json:"config"` 46 | } 47 | 48 | // TaskID 49 | // NOTE: Code originally from https://github.com/go-kafka/connect as two components, a numerical ID and a connector name by which the 50 | // ID is scoped. 51 | type TaskID struct { 52 | ConnectorName string `json:"connector"` 53 | ID int `json:"task"` 54 | } 55 | 56 | // ConnectorStatus reflects the status of a Connector and state of its Tasks. 57 | // 58 | // Having connector name and a "connector" object at top level is a little 59 | // awkward and produces stuttering, but it's their design, not ours. 60 | type ConnectorStatus struct { 61 | Name string `json:"name"` 62 | Connector ConnectorState `json:"connector"` 63 | Tasks []TaskState `json:"tasks"` 64 | } 65 | 66 | // ConnectorState reflects the running state of a Connector and the worker where 67 | // it is running. 68 | type ConnectorState struct { 69 | State string `json:"state"` 70 | WorkerID string `json:"worker_id"` 71 | } 72 | 73 | // TaskState reflects the running state of a Task and the worker where it is 74 | // running. 75 | type TaskState struct { 76 | ID int `json:"id"` 77 | State string `json:"state"` 78 | WorkerID string `json:"worker_id"` 79 | Trace string `json:"trace,omitempty"` 80 | } 81 | 82 | // CreateConnector creates a new connector instance. If successful, conn is 83 | // updated with the connector's state returned by the API, including Tasks. 84 | // 85 | // Passing an object that already contains Tasks produces an error. 86 | // 87 | // See: http://docs.confluent.io/current/connect/userguide.html#post--connectors 88 | func (c *Client) CreateConnector(conn Connector) (*http.Response, error) { 89 | if len(conn.Tasks) != 0 { 90 | return nil, errors.New("cannot create Connector with existing Tasks") 91 | } 92 | path := "connectors" 93 | response, err := c.doRequest("POST", path, conn, &conn) 94 | return response, err 95 | } 96 | 97 | // ListConnectors retrieves a list of active connector names. 98 | // 99 | // See: http://docs.confluent.io/current/connect/userguide.html#get--connectors 100 | func (c *Client) ListConnectors() ([]string, *http.Response, error) { 101 | path := "connectors" 102 | var names []string 103 | response, err := c.get(path, &names) 104 | return names, response, err 105 | } 106 | 107 | // GetConnector retrieves information about a connector with the given name. 108 | // 109 | // See: http://docs.confluent.io/current/connect/userguide.html#get--connectors-(string-name) 110 | func (c *Client) GetConnector(name string) (*Connector, *http.Response, error) { 111 | path := "connectors/" + name 112 | connector := new(Connector) 113 | response, err := c.get(path, &connector) 114 | return connector, response, err 115 | } 116 | 117 | // GetConnectorConfig retrieves configuration for a connector with the given 118 | // name. 119 | // 120 | // See: http://docs.confluent.io/current/connect/userguide.html#get--connectors-(string-name)-config 121 | func (c *Client) GetConnectorConfig(name string) (ConnectorConfig, *http.Response, error) { 122 | path := fmt.Sprintf("connectors/%v/config", name) 123 | config := make(ConnectorConfig) 124 | response, err := c.get(path, &config) 125 | return config, response, err 126 | } 127 | 128 | // GetConnectorTasks retrieves a list of tasks currently running for a connector 129 | // with the given name. 130 | // 131 | // See: http://docs.confluent.io/current/connect/userguide.html#get--connectors-(string-name)-tasks 132 | func (c *Client) GetConnectorTasks(name string) ([]Task, *http.Response, error) { 133 | path := fmt.Sprintf("connectors/%v/tasks", name) 134 | var tasks []Task 135 | response, err := c.get(path, &tasks) 136 | return tasks, response, err 137 | } 138 | 139 | // GetConnectorStatus gets current status of the connector, including whether it 140 | // is running, failed or paused, which worker it is assigned to, error 141 | // information if it has failed, and the state of all its tasks. 142 | // 143 | // See: http://docs.confluent.io/current/connect/userguide.html#get--connectors-(string-name)-status 144 | func (c *Client) GetConnectorStatus(name string) (*ConnectorStatus, *http.Response, error) { 145 | path := fmt.Sprintf("connectors/%v/status", name) 146 | status := new(ConnectorStatus) 147 | response, err := c.get(path, status) 148 | return status, response, err 149 | } 150 | 151 | // GetConnectorTaskStatus gets the status of task for a connector. 152 | // 153 | // See: https://docs.confluent.io/current/connect/references/restapi.html#get--connectors-(string-name)-tasks-(int-taskid)-status 154 | func (c *Client) GetConnectorTaskStatus(name string, taskID int) (*TaskState, *http.Response, error) { 155 | path := fmt.Sprintf("connectors/%v/tasks/%v/status", name, taskID) 156 | status := new(TaskState) 157 | respnse, err := c.get(path, status) 158 | return status, respnse, err 159 | } 160 | 161 | // UpdateConnectorConfig updates configuration for an existing connector with 162 | // the given name, returning the new state of the Connector. 163 | // 164 | // If the connector does not exist, it will be created, and the returned HTTP 165 | // response will indicate a 201 Created status. 166 | // 167 | // See: http://docs.confluent.io/current/connect/userguide.html#put--connectors-(string-name)-config 168 | func (c *Client) UpdateConnectorConfig(name string, config ConnectorConfig) (*Connector, *http.Response, error) { 169 | path := fmt.Sprintf("connectors/%v/config", name) 170 | connector := new(Connector) 171 | response, err := c.doRequest("PUT", path, config, &connector) 172 | return connector, response, err 173 | } 174 | 175 | // DeleteConnector deletes a connector with the given name, halting all tasks 176 | // and deleting its configuration. 177 | // 178 | // See: http://docs.confluent.io/current/connect/userguide.html#delete--connectors-(string-name)- 179 | func (c *Client) DeleteConnector(name string) (*http.Response, error) { 180 | return c.delete("connectors/" + name) 181 | } 182 | 183 | // PauseConnector pauses a connector and its tasks, which stops message 184 | // processing until the connector is resumed. Tasks will transition to PAUSED 185 | // state asynchronously. 186 | // 187 | // See: http://docs.confluent.io/current/connect/userguide.html#put--connectors-(string-name)-pause 188 | func (c *Client) PauseConnector(name string) (*http.Response, error) { 189 | path := fmt.Sprintf("connectors/%v/pause", name) 190 | return c.doRequest("PUT", path, nil, nil) 191 | } 192 | 193 | // ResumeConnector resumes a paused connector. Tasks will transition to RUNNING 194 | // state asynchronously. 195 | // 196 | // See: http://docs.confluent.io/current/connect/userguide.html#put--connectors-(string-name)-resume 197 | func (c *Client) ResumeConnector(name string) (*http.Response, error) { 198 | path := fmt.Sprintf("connectors/%v/resume", name) 199 | return c.doRequest("PUT", path, nil, nil) 200 | } 201 | 202 | // RestartConnector restarts a connector and its tasks. 203 | // 204 | // See http://docs.confluent.io/current/connect/userguide.html#post--connectors-(string-name)-restart 205 | func (c *Client) RestartConnector(name string) (*http.Response, error) { 206 | path := fmt.Sprintf("connectors/%v/restart", name) 207 | return c.doRequest("POST", path, nil, nil) 208 | } 209 | 210 | // RestartConnectorTask restarts a tasks for a connector. 211 | // 212 | // See https://docs.confluent.io/current/connect/references/restapi.html#post--connectors-(string-name)-tasks-(int-taskid)-restart 213 | func (c *Client) RestartConnectorTask(name string, taskID int) (*http.Response, error) { 214 | path := fmt.Sprintf("connectors/%v/tasks/%v/restart", name, taskID) 215 | return c.doRequest("POST", path, nil, nil) 216 | } 217 | -------------------------------------------------------------------------------- /pkg/client/connect/errors.go: -------------------------------------------------------------------------------- 1 | // NOTE: Code originally from https://github.com/go-kafka/connect 2 | 3 | package connect 4 | 5 | import ( 6 | "fmt" 7 | "net/http" 8 | ) 9 | 10 | // APIError holds information returned from a Kafka Connect API instance about 11 | // why an API call failed. 12 | type APIError struct { 13 | Code int `json:"error_code"` 14 | Message string `json:"message"` 15 | Response *http.Response // HTTP response that caused this error 16 | } 17 | 18 | func (e APIError) Error() string { 19 | return fmt.Sprintf("%v (HTTP %d)", e.Message, e.Code) 20 | } 21 | 22 | // IsAPIError indicates if the error is an struct of type APIError 23 | func IsAPIError(err error) bool { 24 | _, ok := err.(*APIError) 25 | return ok 26 | } 27 | 28 | // IsNotFound indicates if the error represents an HTTP 404 status code 29 | func IsNotFound(err error) bool { 30 | apiErr, ok := err.(*APIError) 31 | if !ok { 32 | return false 33 | } 34 | return apiErr.Code == http.StatusNotFound 35 | } 36 | 37 | // IsRetryable indicates if the error could be retryed. 38 | // See https://github.com/apache/kafka/blob/master/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/resources/ConnectorsResource.java#L299-L325 39 | func IsRetryable(err error) bool { 40 | apiErr, ok := err.(*APIError) 41 | if !ok { 42 | return false 43 | } 44 | return apiErr.Code == http.StatusConflict 45 | } 46 | -------------------------------------------------------------------------------- /pkg/client/connect/errors_test.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "net/http" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func Test_IsRetryable(t *testing.T) { 13 | 14 | testCases := []struct { 15 | name string 16 | req *http.Request 17 | resp *http.Response 18 | result bool 19 | }{ 20 | { 21 | name: "401", 22 | result: true, 23 | resp: &http.Response{ 24 | Body: ioutil.NopCloser(bytes.NewBufferString(`{ "error_code" : 409 }`)), 25 | StatusCode: http.StatusConflict, 26 | }, 27 | req: &http.Request{}, 28 | }, 29 | { 30 | name: "404", 31 | result: false, 32 | resp: &http.Response{ 33 | Body: ioutil.NopCloser(bytes.NewBufferString(`{ "error_code" : 404 }`)), 34 | StatusCode: http.StatusNotFound, 35 | }, 36 | req: &http.Request{}, 37 | }, 38 | } 39 | 40 | for i := range testCases { 41 | tc := testCases[i] 42 | t.Run(tc.name, func(t *testing.T) { 43 | 44 | err := buildError(tc.req, tc.resp) 45 | require.True(t, IsAPIError(err)) 46 | 47 | result := IsRetryable(err) 48 | require.Equal(t, tc.result, result) 49 | 50 | }) 51 | } 52 | } 53 | 54 | func Test_IsNotFound(t *testing.T) { 55 | 56 | testCases := []struct { 57 | name string 58 | req *http.Request 59 | resp *http.Response 60 | result bool 61 | }{ 62 | { 63 | name: "404", 64 | result: true, 65 | resp: &http.Response{ 66 | Body: ioutil.NopCloser(bytes.NewBufferString(`{ "error_code" : 404 }`)), 67 | StatusCode: http.StatusNotFound, 68 | }, 69 | req: &http.Request{}, 70 | }, 71 | { 72 | name: "401", 73 | resp: &http.Response{ 74 | Body: ioutil.NopCloser(bytes.NewBufferString(`{ "error_code" : 409 }`)), 75 | StatusCode: http.StatusConflict, 76 | }, 77 | req: &http.Request{}, 78 | }, 79 | } 80 | 81 | for i := range testCases { 82 | tc := testCases[i] 83 | t.Run(tc.name, func(t *testing.T) { 84 | 85 | err := buildError(tc.req, tc.resp) 86 | require.True(t, IsAPIError(err)) 87 | 88 | result := IsNotFound(err) 89 | require.Equal(t, tc.result, result) 90 | 91 | }) 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /pkg/client/connect/plugins.go: -------------------------------------------------------------------------------- 1 | package connect 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/http" 7 | "strings" 8 | ) 9 | 10 | // This is new and not from the original author 11 | 12 | // Plugin represents a Kafka Connect connector plugin 13 | type Plugin struct { 14 | Class string `json:"class"` 15 | Type string `json:"type"` 16 | Version string `json:"version"` 17 | } 18 | 19 | type FieldDefinition struct { 20 | Name string `json:"name"` 21 | Type string `json:"type"` 22 | Required bool `json:"required"` 23 | DefaultValue *string `json:"default_value"` 24 | Importance string `json:"importance"` 25 | Documentation string `json:"documentation"` 26 | Group string `json:"group"` 27 | Width string `json:"width"` 28 | DisplayName string `json:"display_name"` 29 | Dependents []map[string]interface{} `json:"dependents"` //unknown type 30 | Order int `json:"order"` 31 | } 32 | 33 | type FieldValue struct { 34 | Name string `json:"name"` 35 | Value *string `json:"value"` 36 | RecommendedValues []*string `json:"recommended_values"` 37 | Errors []string `json:"errors"` 38 | Visible bool `json:"visible"` 39 | } 40 | 41 | type FieldValidation struct { 42 | Definition FieldDefinition `json:"definition"` 43 | Value FieldValue `json:"value"` 44 | } 45 | 46 | type ConfigValidation struct { 47 | Name string `json:"name"` 48 | ErrorCount int `json:"error_count"` 49 | Groups []string `json:"groups"` 50 | Configs []FieldValidation `json:"configs"` 51 | } 52 | 53 | // ListPlugins retrieves a list of the installed plugins. 54 | // Note that the API only checks for connectors on the worker 55 | // that handles the request, which means it is possible to see 56 | // inconsistent results, especially during a rolling upgrade if 57 | // you add new connector jars 58 | // See: https://docs.confluent.io/current/connect/references/restapi.html#get--connector-plugins- 59 | func (c *Client) ListPlugins() ([]*Plugin, *http.Response, error) { 60 | path := "connector-plugins" 61 | var names []*Plugin 62 | 63 | response, err := c.get(path, &names) 64 | return names, response, err 65 | } 66 | 67 | // ValidatePlugins validates the provided configuration values against the configuration definition. 68 | // See: https://docs.confluent.io/current/connect/references/restapi.html#put--connector-plugins-(string-name)-config-validate 69 | func (c *Client) ValidatePlugins(config ConnectorConfig) (*ConfigValidation, *http.Response, error) { 70 | connectorClass, ok := config["connector.class"] 71 | if !ok { 72 | return nil, nil, errors.New("missing required key in config: 'connector.class'") 73 | } 74 | 75 | tuple := strings.Split(connectorClass, ".") 76 | path := fmt.Sprintf("connector-plugins/%s/config/validate", tuple[len(tuple)-1]) 77 | 78 | var validation ConfigValidation 79 | response, err := c.doRequest("PUT", path, config, &validation) 80 | 81 | return &validation, response, err 82 | } 83 | -------------------------------------------------------------------------------- /pkg/manager/cluster.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | 6 | "github.com/90poe/connectctl/pkg/client/connect" 7 | ) 8 | 9 | // GetClusterInfo returns kafka cluster info 10 | func (c *ConnectorManager) GetClusterInfo() (*connect.ClusterInfo, error) { 11 | clusterInfo, _, err := c.client.GetClusterInfo() 12 | 13 | if err != nil { 14 | return nil, errors.Wrap(err, "error getting cluster info") 15 | } 16 | 17 | return clusterInfo, nil 18 | } 19 | -------------------------------------------------------------------------------- /pkg/manager/config.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // Config represents the connect manager configuration 8 | type Config struct { 9 | ClusterURL string `json:"cluster_url"` 10 | SyncPeriod time.Duration `json:"sync_period"` 11 | InitialWaitPeriod time.Duration `json:"initial_wait_period"` 12 | AllowPurge bool `json:"allow_purge"` 13 | AutoRestart bool `json:"auto_restart"` 14 | 15 | Version string `json:"version"` 16 | 17 | GlobalConnectorRestartsMax int `json:"global_connector_restarts_max"` 18 | GlobalConnectorRestartPeriod time.Duration `json:"global_connector_restart_period"` 19 | GlobalTaskRestartsMax int `json:"global_task_restarts_max"` 20 | GlobalTaskRestartPeriod time.Duration `json:"global_task_restart_period"` 21 | 22 | RestartOverrides *RestartPolicy `json:"restart_policy"` 23 | } 24 | 25 | // RestartPolicy lists each connectors maximum restart policy 26 | // If AutoRestart == true 27 | // If a policy does not exist for a connector the connector or task will be restarted once. 28 | // If a connector or task is restarted the count of failed attempts is reset. 29 | // If the number of unsuccessful restarts is reached the manager will return and connectctl will stop. 30 | type RestartPolicy struct { 31 | Connectors map[string]Policy `json:"connectors"` 32 | } 33 | 34 | // Policy contains a collection of values to be managed 35 | type Policy struct { 36 | ConnectorRestartsMax int `json:"connector_restarts_max"` 37 | ConnectorRestartPeriod time.Duration `json:"connector_restart_period"` 38 | TaskRestartsMax int `json:"task_restarts_max"` 39 | TaskRestartPeriod time.Duration `json:"task_restart_period"` 40 | } 41 | -------------------------------------------------------------------------------- /pkg/manager/crud.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | 6 | "github.com/90poe/connectctl/pkg/client/connect" 7 | ) 8 | 9 | // GetAllConnectors returns all the connectors in a cluster 10 | func (c *ConnectorManager) GetAllConnectors() ([]*ConnectorWithState, error) { 11 | existing, err := c.ListConnectors() 12 | if err != nil { 13 | return nil, err 14 | } 15 | 16 | connectors := make([]*ConnectorWithState, len(existing)) 17 | for i, connectorName := range existing { 18 | connector, err := c.GetConnector(connectorName) 19 | if err != nil { 20 | return nil, err 21 | } 22 | connectors[i] = connector 23 | } 24 | 25 | return connectors, nil 26 | } 27 | 28 | // GetConnector returns information about a named connector in the cluster 29 | func (c *ConnectorManager) GetConnector(connectorName string) (*ConnectorWithState, error) { 30 | connector, _, err := c.client.GetConnector(connectorName) 31 | if err != nil { 32 | return nil, errors.Wrapf(err, "getting connector %s", connectorName) 33 | } 34 | 35 | connectorStatus, _, err := c.client.GetConnectorStatus(connectorName) 36 | if err != nil { 37 | return nil, errors.Wrapf(err, "getting connector status %s", connectorName) 38 | } 39 | 40 | withState := &ConnectorWithState{ 41 | Name: connector.Name, 42 | ConnectorState: connectorStatus.Connector, 43 | Config: connector.Config, 44 | Tasks: connectorStatus.Tasks, 45 | } 46 | 47 | return withState, nil 48 | } 49 | 50 | // ListConnectors returns the names of all connectors in the cluster 51 | func (c *ConnectorManager) ListConnectors() ([]string, error) { 52 | connectors, _, err := c.client.ListConnectors() 53 | 54 | if err != nil { 55 | return nil, errors.Wrap(err, "getting existing connectors") 56 | } 57 | 58 | return connectors, nil 59 | } 60 | 61 | // Add will add connectors to a cluster 62 | func (c *ConnectorManager) Add(connectors []connect.Connector) error { 63 | for _, connector := range connectors { 64 | if _, err := c.client.CreateConnector(connector); err != nil { 65 | return errors.Wrapf(err, "error creating connector %s", connector.Name) 66 | } 67 | } 68 | 69 | return nil 70 | } 71 | 72 | // Remove will remove connectors from a cluster 73 | func (c *ConnectorManager) Remove(connectorNames []string) error { 74 | for _, connectorName := range connectorNames { 75 | if _, err := c.client.DeleteConnector(connectorName); err != nil { 76 | return errors.Wrapf(err, "error deleting connector %s", connectorName) 77 | } 78 | } 79 | 80 | return nil 81 | } 82 | -------------------------------------------------------------------------------- /pkg/manager/logger.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | // Logger is the interface to implement to get all of the great news/updates 4 | type Logger interface { 5 | Infof(message string, args ...interface{}) 6 | Warnf(message string, args ...interface{}) 7 | Debugf(message string, args ...interface{}) 8 | Errorf(message string, args ...interface{}) 9 | } 10 | 11 | type noopLogger struct{} 12 | 13 | func (n *noopLogger) Infof(string, ...interface{}) { 14 | // do nothing 15 | } 16 | func (n *noopLogger) Warnf(string, ...interface{}) { 17 | // do nothing 18 | } 19 | func (n *noopLogger) Debugf(string, ...interface{}) { 20 | // do nothing 21 | } 22 | func (n *noopLogger) Errorf(string, ...interface{}) { 23 | // do nothing 24 | } 25 | 26 | func newNoopLogger() *noopLogger { 27 | return &noopLogger{} 28 | } 29 | -------------------------------------------------------------------------------- /pkg/manager/manage.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/90poe/connectctl/pkg/client/connect" 8 | 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | // Manage will start the connector manager running and managing connectors 13 | func (c *ConnectorManager) Manage(source ConnectorSource, stopCH <-chan struct{}) error { 14 | // mark ourselves as having an unhealthy state until we have 15 | // successfully configured the kafka-connect instance 16 | c.readinessState = errorState 17 | 18 | syncChannel := time.NewTicker(c.config.InitialWaitPeriod).C 19 | for { 20 | select { 21 | case <-syncChannel: 22 | 23 | err := c.trySync(source) 24 | if err != nil { 25 | return errors.Wrap(err, "error synchronising connectors for source") 26 | } 27 | syncChannel = time.NewTicker(c.config.SyncPeriod).C 28 | 29 | case <-stopCH: 30 | return nil 31 | } 32 | } 33 | } 34 | 35 | func (c *ConnectorManager) trySync(source ConnectorSource) error { 36 | // we only want to try Syncing if we can contact the kafka-connect instance. 37 | // Using the LivenessCheck as a proxy for calculating the connection 38 | if c.livenessState == okState { 39 | err := c.Sync(source) 40 | if err != nil { 41 | // set back into an unhealthy state 42 | c.readinessState = errorState 43 | return err 44 | } 45 | // mark ourselves as being in an ok state as we have 46 | // started syncing without any error 47 | c.readinessState = okState 48 | return nil 49 | } 50 | c.logger.Infof("skipping sync as livenessState == %v", c.livenessState) 51 | return nil 52 | } 53 | 54 | // Sync will synchronise the desired and actual state of connectors in a cluster 55 | func (c *ConnectorManager) Sync(source ConnectorSource) error { 56 | c.logger.Infof("loading connectors") 57 | connectors, err := source() 58 | if err != nil { 59 | return errors.Wrap(err, "error getting connectors configuration") 60 | } 61 | c.logger.Infof("connectors loaded : %d", len(connectors)) 62 | // creating a runtime restart policy here, overriding with the supplied one (if any) 63 | // Ensuring that we have a policy defined for each connector we are manging here 64 | // dramatically simplifies the management and restart code 65 | policy := runtimePolicyFromConnectors(connectors, c.config) 66 | 67 | if err = c.reconcileConnectors(connectors, policy); err != nil { 68 | return errors.Wrap(err, "error synchronising connectors") 69 | } 70 | return nil 71 | } 72 | 73 | func (c *ConnectorManager) reconcileConnectors(connectors []connect.Connector, restartPolicy runtimeRestartPolicy) error { 74 | for _, connector := range connectors { 75 | if err := c.reconcileConnector(connector); err != nil { 76 | return errors.Wrapf(err, "error reconciling connector: %s", connector.Name) 77 | } 78 | } 79 | if c.config.AllowPurge { 80 | if err := c.checkAndDeleteUnmanaged(connectors); err != nil { 81 | return errors.Wrapf(err, "error checking for unmanaged connectors to purge") 82 | } 83 | } 84 | if c.config.AutoRestart { 85 | if err := c.autoRestart(connectors, restartPolicy); err != nil { 86 | return errors.Wrap(err, "error checking connector status") 87 | } 88 | } 89 | 90 | return nil 91 | } 92 | 93 | func (c *ConnectorManager) autoRestart(connectors []connect.Connector, restartPolicy runtimeRestartPolicy) error { 94 | for _, connector := range connectors { 95 | name := connector.Name 96 | err := c.retryRestartConnector(name, restartPolicy[name].ConnectorRestartsMax, restartPolicy[name].ConnectorRestartPeriod) 97 | if err != nil { 98 | return err 99 | } 100 | err = c.retryRestartConnectorTask(name, restartPolicy[name].TaskRestartsMax, restartPolicy[name].TaskRestartPeriod) 101 | if err != nil { 102 | return err 103 | } 104 | } 105 | return nil 106 | } 107 | 108 | func (c *ConnectorManager) retryRestartConnector(name string, retrys int, retryPeriod time.Duration) error { 109 | attempts := 0 110 | for attempts <= retrys { 111 | status, _, err := c.client.GetConnectorStatus(name) 112 | 113 | if err != nil { 114 | return errors.Wrapf(err, "error getting connector status: %s", name) 115 | } 116 | 117 | if isConnectorFailed(status.Connector) { 118 | c.logger.Warnf("connector not running: %s", name) 119 | 120 | if err = c.restartConnector(name); err != nil { 121 | c.logger.Warnf("restarting connector failed: %s", err.Error()) 122 | return errors.Wrapf(err, "error restarting connector: %s", name) 123 | } 124 | } else { 125 | return nil 126 | } 127 | 128 | attempts++ 129 | time.Sleep(retryPeriod) 130 | } 131 | c.logger.Warnf("error restarting connector: %s, retrys: %d", name, retrys) 132 | return fmt.Errorf("error restarting connector: %s, retrys: %d", name, retrys) 133 | } 134 | 135 | func isConnectorRunning(c connect.ConnectorState) bool { 136 | return c.State == "RUNNING" // nolint 137 | } 138 | 139 | func isConnectorFailed(c connect.ConnectorState) bool { 140 | return c.State == "FAILED" // nolint 141 | } 142 | 143 | func isTaskFailed(t connect.TaskState) bool { 144 | return t.State == "FAILED" //nolint 145 | } 146 | 147 | func (c *ConnectorManager) retryRestartConnectorTask(name string, retrys int, retryPeriod time.Duration) error { 148 | attempts := 0 149 | for attempts <= retrys { 150 | status, _, err := c.client.GetConnectorStatus(name) 151 | 152 | if err != nil { 153 | return errors.Wrapf(err, "error getting connector status: %s", name) 154 | } 155 | 156 | if isConnectorRunning(status.Connector) { 157 | runningTasks := 0 158 | 159 | for _, taskState := range status.Tasks { 160 | if isTaskFailed(taskState) { 161 | if taskState.Trace != "" { 162 | c.logger.Warnf("task not running: %s ( %d ) : %s", name, taskState.ID, taskState.Trace) 163 | } else { 164 | c.logger.Warnf("task not running: %s ( %d )", name, taskState.ID) 165 | } 166 | 167 | if _, err := c.client.RestartConnectorTask(name, taskState.ID); err != nil { 168 | c.logger.Warnf("restarting task failed: %s", err.Error()) 169 | return err 170 | } 171 | } else { 172 | runningTasks++ 173 | } 174 | } 175 | if runningTasks == len(status.Tasks) { 176 | return nil 177 | } 178 | } else { 179 | return nil 180 | } 181 | 182 | attempts++ 183 | time.Sleep(retryPeriod) 184 | } 185 | c.logger.Warnf("error restarting connector task: %s, retrys: %d", name, retrys) 186 | return fmt.Errorf("error restarting connector task: %s, retrys: %d", name, retrys) 187 | } 188 | 189 | func (c *ConnectorManager) checkAndDeleteUnmanaged(connectors []connect.Connector) error { 190 | existing, _, err := c.client.ListConnectors() 191 | if err != nil { 192 | return errors.Wrap(err, "error getting existing connectors") 193 | } 194 | 195 | var unmanaged []string 196 | for _, existingName := range existing { 197 | if !containsConnector(existingName, connectors) { 198 | unmanaged = append(unmanaged, existingName) 199 | } 200 | } 201 | 202 | if len(unmanaged) == 0 { 203 | return nil 204 | } 205 | 206 | if err := c.Remove(unmanaged); err != nil { 207 | return errors.Wrap(err, "error deleting unmanaged connectors") 208 | } 209 | return nil 210 | } 211 | 212 | func (c *ConnectorManager) reconcileConnector(connector connect.Connector) error { 213 | existingConnectors, _, err := c.client.GetConnector(connector.Name) 214 | 215 | if err != nil { 216 | if connect.IsNotFound(err) { 217 | return c.handleNewConnector(connector) 218 | } 219 | return errors.Wrap(err, "error getting existing connector from cluster") 220 | } 221 | 222 | if existingConnectors != nil { 223 | return c.handleExistingConnector(connector, existingConnectors) 224 | } 225 | return nil 226 | } 227 | 228 | func (c *ConnectorManager) handleExistingConnector(connector connect.Connector, existingConnector *connect.Connector) error { 229 | if existingConnector.ConfigEqual(connector) { 230 | return nil 231 | } 232 | 233 | if _, _, err := c.client.UpdateConnectorConfig(existingConnector.Name, connector.Config); err != nil { 234 | return errors.Wrap(err, "error updating connector config") 235 | } 236 | 237 | return nil 238 | } 239 | 240 | func (c *ConnectorManager) handleNewConnector(connector connect.Connector) error { 241 | if err := c.Add([]connect.Connector{connector}); err != nil { 242 | return errors.Wrap(err, "error creating connector") 243 | } 244 | 245 | return nil 246 | } 247 | 248 | func containsConnector(connectorName string, connectors []connect.Connector) bool { 249 | for _, c := range connectors { 250 | if c.Name == connectorName { 251 | return true 252 | } 253 | } 254 | return false 255 | } 256 | -------------------------------------------------------------------------------- /pkg/manager/manager.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "github.com/90poe/connectctl/pkg/client/connect" 8 | "github.com/heptiolabs/healthcheck" 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | // ConnectorSource will return a slice of the desired connector configuration 13 | type ConnectorSource func() ([]connect.Connector, error) 14 | 15 | type client interface { 16 | GetClusterInfo() (*connect.ClusterInfo, *http.Response, error) 17 | CreateConnector(conn connect.Connector) (*http.Response, error) 18 | ListConnectors() ([]string, *http.Response, error) 19 | GetConnector(name string) (*connect.Connector, *http.Response, error) 20 | ListPlugins() ([]*connect.Plugin, *http.Response, error) 21 | ValidatePlugins(config connect.ConnectorConfig) (*connect.ConfigValidation, *http.Response, error) 22 | GetConnectorStatus(name string) (*connect.ConnectorStatus, *http.Response, error) 23 | DeleteConnector(name string) (*http.Response, error) 24 | RestartConnectorTask(name string, taskID int) (*http.Response, error) 25 | UpdateConnectorConfig(name string, config connect.ConnectorConfig) (*connect.Connector, *http.Response, error) 26 | RestartConnector(name string) (*http.Response, error) 27 | ResumeConnector(name string) (*http.Response, error) 28 | PauseConnector(name string) (*http.Response, error) 29 | } 30 | 31 | // ConnectorManager manages connectors in a Kafka Connect cluster 32 | type ConnectorManager struct { 33 | config *Config 34 | client client 35 | logger Logger 36 | 37 | readinessState healthcheckState 38 | livenessState healthcheckState 39 | } 40 | 41 | // Option can be supplied that override the default ConnectorManager properties 42 | type Option func(c *ConnectorManager) 43 | 44 | // WithLogger allows for a logger of choice to be injected 45 | func WithLogger(l Logger) Option { 46 | return func(c *ConnectorManager) { 47 | c.logger = l 48 | } 49 | } 50 | 51 | // NewConnectorsManager creates a new ConnectorManager 52 | func NewConnectorsManager(client client, config *Config, opts ...Option) (*ConnectorManager, error) { 53 | cm := &ConnectorManager{ 54 | config: config, 55 | client: client, 56 | logger: newNoopLogger(), 57 | readinessState: unknownState, 58 | livenessState: unknownState, 59 | } 60 | 61 | for _, opt := range opts { 62 | opt(cm) 63 | } 64 | 65 | return cm, nil 66 | } 67 | 68 | type healthcheckState int 69 | 70 | const ( 71 | unknownState healthcheckState = iota 72 | okState 73 | errorState 74 | ) 75 | 76 | // ReadinessCheck checks if we have been able to start syncing with kafka-connect 77 | func (c *ConnectorManager) ReadinessCheck() (string, func() error) { 78 | return "connectctl-readiness-check", func() error { 79 | switch c.readinessState { 80 | case okState: 81 | c.logger.Infof("healthcheck: readiness: ok") 82 | return nil 83 | case unknownState, errorState: 84 | c.logger.Infof("healthcheck: readiness: not ready") 85 | return errors.New("connectctl is not ready") 86 | } 87 | 88 | return nil 89 | } 90 | } 91 | 92 | // LivenessCheck checks if the the kafka-connect instance is running. 93 | // The timeout of 2 seconds is arbitrary. 94 | func (c *ConnectorManager) LivenessCheck() (string, func() error) { 95 | check := func() error { 96 | err := healthcheck.HTTPGetCheck(c.config.ClusterURL, time.Second*2)() 97 | if err != nil { 98 | c.livenessState = errorState 99 | c.logger.Infof("healthcheck: liveness : %s", err.Error()) 100 | return err 101 | } 102 | c.livenessState = okState 103 | c.logger.Infof("healthcheck: liveness : ok") 104 | return nil 105 | } 106 | return "connectctl-liveness-check-kafka-connect-instance", check 107 | } 108 | -------------------------------------------------------------------------------- /pkg/manager/manager_test.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "net/http" 5 | "testing" 6 | "time" 7 | 8 | "github.com/90poe/connectctl/pkg/client/connect" 9 | "github.com/90poe/connectctl/pkg/manager/mocks" 10 | "github.com/pkg/errors" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func Test_Manage_MissingConnectorsAreAdded(t *testing.T) { 15 | t.Parallel() 16 | 17 | createCalled := false 18 | 19 | mock := &mocks.FakeClient{ 20 | GetConnectorStub: func(string) (*connect.Connector, *http.Response, error) { 21 | return nil, nil, &connect.APIError{Code: http.StatusNotFound} 22 | }, 23 | CreateConnectorStub: func(c connect.Connector) (*http.Response, error) { 24 | 25 | require.Equal(t, c.Name, "one") 26 | 27 | createCalled = true 28 | return nil, nil 29 | }, 30 | } 31 | config := &Config{} 32 | 33 | cm, err := NewConnectorsManager(mock, config) 34 | require.Nil(t, err) 35 | 36 | source := func() ([]connect.Connector, error) { 37 | return []connect.Connector{ 38 | connect.Connector{Name: "one"}, 39 | }, nil 40 | 41 | } 42 | 43 | err = cm.Sync(source) 44 | require.Nil(t, err) 45 | require.True(t, createCalled) 46 | 47 | } 48 | 49 | func Test_Manage_ExistingConnectorsAreRemovedIfNotListed(t *testing.T) { 50 | t.Parallel() 51 | 52 | deleteCalled := false 53 | 54 | mock := &mocks.FakeClient{ 55 | ListConnectorsStub: func() ([]string, *http.Response, error) { 56 | return []string{"delete-me"}, nil, nil 57 | }, 58 | DeleteConnectorStub: func(string) (*http.Response, error) { 59 | deleteCalled = true 60 | return nil, nil 61 | }, 62 | } 63 | config := &Config{ 64 | AllowPurge: true, 65 | } 66 | 67 | cm, err := NewConnectorsManager(mock, config) 68 | require.Nil(t, err) 69 | 70 | source := func() ([]connect.Connector, error) { 71 | return []connect.Connector{}, nil 72 | } 73 | 74 | err = cm.Sync(source) 75 | require.Nil(t, err) 76 | require.True(t, deleteCalled) 77 | 78 | } 79 | 80 | func Test_Manage_ErrorsAreAPIErrorsIfUnwrapped(t *testing.T) { 81 | t.Parallel() 82 | 83 | mock := &mocks.FakeClient{ 84 | GetConnectorStub: func(string) (*connect.Connector, *http.Response, error) { 85 | return nil, nil, &connect.APIError{Code: http.StatusInternalServerError} 86 | }, 87 | } 88 | config := &Config{} 89 | 90 | cm, err := NewConnectorsManager(mock, config) 91 | require.Nil(t, err) 92 | 93 | source := func() ([]connect.Connector, error) { 94 | return []connect.Connector{ 95 | connect.Connector{Name: "one"}, 96 | }, nil 97 | 98 | } 99 | 100 | err = cm.Sync(source) 101 | rootCause := errors.Cause(err) 102 | require.True(t, connect.IsAPIError(rootCause)) 103 | } 104 | 105 | func Test_Manage_ConnectorRunning_FailedTasksAreRestarted(t *testing.T) { 106 | t.Parallel() 107 | 108 | mock := &mocks.FakeClient{ 109 | GetConnectorStatusStub: func(string) (*connect.ConnectorStatus, *http.Response, error) { 110 | return &connect.ConnectorStatus{ 111 | Connector: connect.ConnectorState{ 112 | State: "RUNNING", 113 | }, 114 | Tasks: []connect.TaskState{ 115 | connect.TaskState{ 116 | State: "FAILED", 117 | }, 118 | }, 119 | }, nil, nil 120 | }} 121 | 122 | config := &Config{ 123 | AutoRestart: true, 124 | } 125 | 126 | cm, err := NewConnectorsManager(mock, config) 127 | require.Nil(t, err) 128 | 129 | source := func() ([]connect.Connector, error) { 130 | return []connect.Connector{ 131 | connect.Connector{Name: "foo"}, 132 | }, nil 133 | } 134 | 135 | err = cm.Sync(source) 136 | require.NotNil(t, err) 137 | require.Equal(t, 2, mock.RestartConnectorTaskCallCount()) 138 | } 139 | 140 | func Test_Manage_ConnectorFailed_IsRestarted(t *testing.T) { 141 | t.Parallel() 142 | count := 0 143 | 144 | mock := &mocks.FakeClient{ 145 | GetConnectorStatusStub: func(string) (*connect.ConnectorStatus, *http.Response, error) { 146 | if count == 0 { 147 | count++ 148 | return &connect.ConnectorStatus{ 149 | Connector: connect.ConnectorState{ 150 | State: "FAILED", 151 | }, 152 | Tasks: []connect.TaskState{ 153 | connect.TaskState{ 154 | State: "FAILED", 155 | }, 156 | }, 157 | }, nil, nil 158 | } else { 159 | return &connect.ConnectorStatus{ 160 | Connector: connect.ConnectorState{ 161 | State: "RUNNING", 162 | }, 163 | Tasks: []connect.TaskState{ 164 | connect.TaskState{ 165 | State: "RUNNING", 166 | }, 167 | }, 168 | }, nil, nil 169 | } 170 | }, 171 | } 172 | 173 | config := &Config{ 174 | AutoRestart: true, 175 | } 176 | 177 | cm, err := NewConnectorsManager(mock, config) 178 | require.Nil(t, err) 179 | 180 | source := func() ([]connect.Connector, error) { 181 | return []connect.Connector{ 182 | connect.Connector{Name: "foo"}, 183 | }, nil 184 | } 185 | 186 | err = cm.Sync(source) 187 | require.Nil(t, err) 188 | require.Equal(t, 1, mock.RestartConnectorCallCount()) 189 | } 190 | 191 | func Test_Manage_ConnectorFailed_IsRestarted_WithPolicy(t *testing.T) { 192 | t.Parallel() 193 | 194 | mock := &mocks.FakeClient{ 195 | GetConnectorStatusStub: func(string) (*connect.ConnectorStatus, *http.Response, error) { 196 | return &connect.ConnectorStatus{ 197 | Connector: connect.ConnectorState{ 198 | State: "FAILED", 199 | }, 200 | Tasks: []connect.TaskState{ 201 | connect.TaskState{ 202 | State: "FAILED", 203 | }, 204 | }, 205 | }, nil, nil 206 | }, 207 | RestartConnectorStub: func(string) (*http.Response, error) { 208 | return nil, nil 209 | }, 210 | } 211 | 212 | config := &Config{ 213 | AutoRestart: true, 214 | RestartOverrides: &RestartPolicy{ 215 | Connectors: map[string]Policy{ 216 | "foo": Policy{ 217 | ConnectorRestartsMax: 10, 218 | ConnectorRestartPeriod: time.Millisecond, 219 | }, 220 | }, 221 | }, 222 | } 223 | 224 | cm, err := NewConnectorsManager(mock, config) 225 | require.Nil(t, err) 226 | 227 | source := func() ([]connect.Connector, error) { 228 | return []connect.Connector{ 229 | connect.Connector{Name: "foo"}, 230 | }, nil 231 | } 232 | 233 | err = cm.Sync(source) 234 | require.NotNil(t, err) 235 | require.Equal(t, 11, mock.RestartConnectorCallCount()) 236 | require.Equal(t, 11, mock.GetConnectorStatusCallCount()) 237 | } 238 | 239 | func Test_Manage_ConnectorFailed_IsRestarted_WithPolicy_RestartWorks(t *testing.T) { 240 | t.Parallel() 241 | count := 0 242 | 243 | mock := &mocks.FakeClient{ 244 | GetConnectorStatusStub: func(string) (*connect.ConnectorStatus, *http.Response, error) { 245 | if count == 0 { 246 | count++ 247 | return &connect.ConnectorStatus{ 248 | Connector: connect.ConnectorState{ 249 | State: "FAILED", 250 | }, 251 | Tasks: []connect.TaskState{ 252 | connect.TaskState{ 253 | State: "FAILED", 254 | }, 255 | }, 256 | }, nil, nil 257 | } else { 258 | return &connect.ConnectorStatus{ 259 | Connector: connect.ConnectorState{ 260 | State: "RUNNING", 261 | }, 262 | Tasks: []connect.TaskState{ 263 | connect.TaskState{ 264 | State: "RUNNING", 265 | }, 266 | }, 267 | }, nil, nil 268 | 269 | } 270 | }, 271 | RestartConnectorStub: func(string) (*http.Response, error) { 272 | return nil, nil 273 | }, 274 | } 275 | 276 | config := &Config{ 277 | AutoRestart: true, 278 | RestartOverrides: &RestartPolicy{ 279 | Connectors: map[string]Policy{ 280 | "foo": Policy{ 281 | ConnectorRestartsMax: 10, 282 | ConnectorRestartPeriod: time.Millisecond, 283 | TaskRestartsMax: 0, 284 | TaskRestartPeriod: time.Millisecond, 285 | }, 286 | }, 287 | }, 288 | } 289 | 290 | cm, err := NewConnectorsManager(mock, config) 291 | require.Nil(t, err) 292 | 293 | source := func() ([]connect.Connector, error) { 294 | return []connect.Connector{ 295 | connect.Connector{Name: "foo"}, 296 | }, nil 297 | } 298 | 299 | err = cm.Sync(source) 300 | require.Nil(t, err) 301 | require.Equal(t, 1, mock.RestartConnectorCallCount()) 302 | require.Equal(t, 3, mock.GetConnectorStatusCallCount()) 303 | } 304 | -------------------------------------------------------------------------------- /pkg/manager/pause.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import "github.com/pkg/errors" 4 | 5 | // Pause will pause a number of connectors in a cluster 6 | func (c *ConnectorManager) Pause(connectors []string) error { 7 | if len(connectors) == 0 { 8 | return c.pauseAllConnectors() 9 | } 10 | 11 | return c.pauseSpecifiedConnectors(connectors) 12 | } 13 | 14 | func (c *ConnectorManager) pauseAllConnectors() error { 15 | existing, _, err := c.client.ListConnectors() 16 | if err != nil { 17 | return errors.Wrap(err, "error listing connectors") 18 | } 19 | 20 | return c.pauseSpecifiedConnectors(existing) 21 | } 22 | 23 | func (c *ConnectorManager) pauseSpecifiedConnectors(connectors []string) error { 24 | for _, connectorName := range connectors { 25 | if _, err := c.client.PauseConnector(connectorName); err != nil { 26 | return errors.Wrapf(err, "error pausing connector %s", connectorName) 27 | } 28 | } 29 | 30 | return nil 31 | } 32 | -------------------------------------------------------------------------------- /pkg/manager/plugins.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | 6 | "github.com/90poe/connectctl/pkg/client/connect" 7 | ) 8 | 9 | // GetAllPlugins returns all the connector plugins installed 10 | func (c *ConnectorManager) GetAllPlugins() ([]*connect.Plugin, error) { 11 | plugins, _, err := c.client.ListPlugins() 12 | 13 | if err != nil { 14 | return nil, errors.Wrap(err, "error listing plugins") 15 | } 16 | 17 | return plugins, nil 18 | } 19 | 20 | // ValidatePlugins returns validation results of a connector config 21 | func (c *ConnectorManager) ValidatePlugins(config connect.ConnectorConfig) (*connect.ConfigValidation, error) { 22 | validation, _, err := c.client.ValidatePlugins(config) 23 | 24 | if err != nil { 25 | return nil, errors.Wrap(err, "error validating plugins") 26 | } 27 | 28 | return validation, nil 29 | } 30 | -------------------------------------------------------------------------------- /pkg/manager/restart.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import "github.com/pkg/errors" 4 | 5 | // Restart will restart a number of connectors in a cluster 6 | func (c *ConnectorManager) Restart(connectorNames []string, restartTasks bool, 7 | forceRestartTasks bool, taskIDs []int) error { 8 | if len(connectorNames) > 0 { 9 | return c.restartConnectors(connectorNames, restartTasks, forceRestartTasks, taskIDs) 10 | } 11 | 12 | connectorNames, err := c.ListConnectors() 13 | if err != nil { 14 | return err 15 | } 16 | 17 | return c.restartConnectors(connectorNames, restartTasks, forceRestartTasks, taskIDs) 18 | } 19 | 20 | func (c *ConnectorManager) restartConnectors(connectorNames []string, restartTasks bool, 21 | forceRestartTasks bool, taskIDs []int) error { 22 | for _, connectorName := range connectorNames { 23 | if err := c.restartConnector(connectorName); err != nil { 24 | return errors.Wrapf(err, "error restarting connector : %s", connectorName) 25 | } 26 | 27 | if restartTasks { 28 | if err := c.restartConnectorTasks(connectorName, forceRestartTasks, taskIDs); err != nil { 29 | return errors.Wrapf(err, "error restarting task : %s", connectorName) 30 | } 31 | } 32 | } 33 | 34 | return nil 35 | } 36 | 37 | func (c *ConnectorManager) restartConnector(connectorName string) error { 38 | if _, err := c.client.RestartConnector(connectorName); err != nil { 39 | return errors.Wrapf(err, "error calling restart connector : %s", connectorName) 40 | } 41 | 42 | return nil 43 | } 44 | 45 | func (c *ConnectorManager) restartConnectorTasks(connectorName string, forceRestartTasks bool, taskIDs []int) error { 46 | connector, err := c.GetConnector(connectorName) 47 | if err != nil { 48 | return err 49 | } 50 | 51 | if len(taskIDs) == 0 { 52 | taskIDs = connector.Tasks.IDs() 53 | } 54 | 55 | tasks := connector.Tasks.Filter(ByID(taskIDs...)) 56 | 57 | if !forceRestartTasks { 58 | tasks = tasks.Filter(IsNotRunning) 59 | } 60 | 61 | for _, taskID := range tasks.IDs() { 62 | if _, err := c.client.RestartConnectorTask(connectorName, taskID); err != nil { 63 | return errors.Wrapf(err, "error calling restart task connector API for task %d", taskID) 64 | } 65 | } 66 | 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /pkg/manager/restart_policy.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/90poe/connectctl/pkg/client/connect" 7 | ) 8 | 9 | type runtimeRestartPolicy map[string]Policy 10 | 11 | const ( 12 | // period to use between restart attempts 13 | // 10 seconds chosen at random 14 | defaultRestartPeriod = time.Second * 10 15 | ) 16 | 17 | func runtimePolicyFromConnectors(connectors []connect.Connector, config *Config) runtimeRestartPolicy { 18 | // create restart policy here, overriding with any supplied values (if any) 19 | policy := runtimeRestartPolicy{} 20 | 21 | for _, c := range connectors { 22 | p := Policy{ 23 | ConnectorRestartsMax: 1, 24 | ConnectorRestartPeriod: defaultRestartPeriod, 25 | TaskRestartsMax: 1, 26 | TaskRestartPeriod: defaultRestartPeriod, 27 | } 28 | if config != nil { 29 | // apply globals (if any) 30 | if config.GlobalConnectorRestartsMax != 0 { 31 | p.ConnectorRestartsMax = config.GlobalConnectorRestartsMax 32 | } 33 | if config.GlobalTaskRestartsMax != 0 { 34 | p.TaskRestartsMax = config.GlobalTaskRestartsMax 35 | } 36 | if config.GlobalConnectorRestartPeriod != 0 { 37 | p.ConnectorRestartPeriod = config.GlobalConnectorRestartPeriod 38 | } 39 | if config.GlobalTaskRestartPeriod != 0 { 40 | p.TaskRestartPeriod = config.GlobalTaskRestartPeriod 41 | } 42 | } 43 | policy[c.Name] = p 44 | } 45 | 46 | // apply overrides (if any) 47 | if config != nil && config.RestartOverrides != nil { 48 | for k, v := range config.RestartOverrides.Connectors { 49 | p := policy[k] 50 | 51 | if v.ConnectorRestartsMax != 0 { 52 | p.ConnectorRestartsMax = v.ConnectorRestartsMax 53 | } 54 | if v.ConnectorRestartPeriod != 0 { 55 | p.ConnectorRestartPeriod = v.ConnectorRestartPeriod 56 | } 57 | if v.TaskRestartsMax != 0 { 58 | p.TaskRestartsMax = v.TaskRestartsMax 59 | } 60 | if v.TaskRestartPeriod != 0 { 61 | p.TaskRestartPeriod = v.TaskRestartPeriod 62 | } 63 | 64 | policy[k] = p 65 | } 66 | } 67 | return policy 68 | } 69 | -------------------------------------------------------------------------------- /pkg/manager/restart_policy_test.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/90poe/connectctl/pkg/client/connect" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func Test_RestartPolicy_Default(t *testing.T) { 13 | t.Parallel() 14 | 15 | connectors := []connect.Connector{ 16 | connect.Connector{Name: "foo"}, 17 | } 18 | 19 | policy := runtimePolicyFromConnectors(connectors, nil) 20 | 21 | require.Len(t, policy, 1) 22 | require.NotNil(t, policy["foo"]) 23 | 24 | foo := policy["foo"] 25 | 26 | require.Equal(t, 1, foo.ConnectorRestartsMax) 27 | require.Equal(t, 1, foo.TaskRestartsMax) 28 | require.Equal(t, defaultRestartPeriod, foo.ConnectorRestartPeriod) 29 | require.Equal(t, defaultRestartPeriod, foo.TaskRestartPeriod) 30 | } 31 | 32 | func Test_RestartPolicy_Globals(t *testing.T) { 33 | t.Parallel() 34 | 35 | connectors := []connect.Connector{ 36 | connect.Connector{Name: "foo"}, 37 | } 38 | 39 | policy := runtimePolicyFromConnectors(connectors, &Config{ 40 | GlobalConnectorRestartsMax: 97, 41 | GlobalConnectorRestartPeriod: time.Second * 98, 42 | GlobalTaskRestartsMax: 99, 43 | GlobalTaskRestartPeriod: time.Second * 100, 44 | }) 45 | 46 | require.Len(t, policy, 1) 47 | require.NotNil(t, policy["foo"]) 48 | 49 | foo := policy["foo"] 50 | 51 | require.Equal(t, 97, foo.ConnectorRestartsMax) 52 | require.Equal(t, time.Second*98, foo.ConnectorRestartPeriod) 53 | require.Equal(t, 99, foo.TaskRestartsMax) 54 | require.Equal(t, time.Second*100, foo.TaskRestartPeriod) 55 | } 56 | 57 | func Test_RestartPolicy_Override(t *testing.T) { 58 | t.Parallel() 59 | 60 | connectors := []connect.Connector{ 61 | connect.Connector{Name: "foo"}, 62 | } 63 | 64 | ovveride := RestartPolicy{ 65 | Connectors: map[string]Policy{ 66 | "foo": Policy{ 67 | ConnectorRestartsMax: 10, 68 | TaskRestartsMax: 11, 69 | TaskRestartPeriod: time.Second * 100, 70 | ConnectorRestartPeriod: time.Second * 101, 71 | }, 72 | }, 73 | } 74 | 75 | config := &Config{RestartOverrides: &ovveride} 76 | policy := runtimePolicyFromConnectors(connectors, config) 77 | 78 | require.Len(t, policy, 1) 79 | require.NotNil(t, policy["foo"]) 80 | 81 | foo := policy["foo"] 82 | 83 | require.Equal(t, 10, foo.ConnectorRestartsMax) 84 | require.Equal(t, 11, foo.TaskRestartsMax) 85 | require.Equal(t, time.Second*101, foo.ConnectorRestartPeriod) 86 | require.Equal(t, time.Second*100, foo.TaskRestartPeriod) 87 | } 88 | -------------------------------------------------------------------------------- /pkg/manager/resume.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import "github.com/pkg/errors" 4 | 5 | // Resume will resume a number of connectors in a cluster 6 | func (c *ConnectorManager) Resume(connectors []string) error { 7 | if len(connectors) == 0 { 8 | return c.resumeAllConnectors() 9 | } 10 | 11 | return c.resumeSpecifiedConnectors(connectors) 12 | } 13 | 14 | func (c *ConnectorManager) resumeAllConnectors() error { 15 | existing, _, err := c.client.ListConnectors() 16 | if err != nil { 17 | return errors.Wrap(err, "error listing connectors") 18 | } 19 | return c.resumeSpecifiedConnectors(existing) 20 | } 21 | 22 | func (c *ConnectorManager) resumeSpecifiedConnectors(connectors []string) error { 23 | for _, connectorName := range connectors { 24 | if _, err := c.client.ResumeConnector(connectorName); err != nil { 25 | return errors.Wrapf(err, "error resuming connector %s", connectorName) 26 | } 27 | } 28 | 29 | return nil 30 | } 31 | -------------------------------------------------------------------------------- /pkg/manager/status.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "github.com/90poe/connectctl/pkg/client/connect" 5 | "github.com/pkg/errors" 6 | ) 7 | 8 | // Status - gets status of specified (or all) connectors 9 | func (c *ConnectorManager) Status(connectors []string) ([]*connect.ConnectorStatus, error) { 10 | if len(connectors) == 0 { 11 | return c.allConnectorsStatus() 12 | } 13 | 14 | return c.specifiedConnectorsStatus(connectors) 15 | } 16 | 17 | func (c *ConnectorManager) allConnectorsStatus() ([]*connect.ConnectorStatus, error) { 18 | existing, _, err := c.client.ListConnectors() 19 | if err != nil { 20 | return nil, errors.Wrap(err, "error listing connectors") 21 | } 22 | 23 | return c.specifiedConnectorsStatus(existing) 24 | } 25 | 26 | func (c *ConnectorManager) specifiedConnectorsStatus(connectors []string) ([]*connect.ConnectorStatus, error) { 27 | statusList := make([]*connect.ConnectorStatus, len(connectors)) 28 | 29 | for idx, connectorName := range connectors { 30 | status, _, err := c.client.GetConnectorStatus(connectorName) 31 | if err != nil { 32 | return nil, errors.Wrapf(err, "error getting connector status for %s", connectorName) 33 | } 34 | 35 | statusList[idx] = status 36 | } 37 | 38 | return statusList, nil 39 | } 40 | -------------------------------------------------------------------------------- /pkg/manager/tasks.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "sort" 5 | 6 | "github.com/90poe/connectctl/pkg/client/connect" 7 | ) 8 | 9 | type Tasks []connect.TaskState 10 | 11 | // TaskPredicate is a function that performs some test on a connect.TaskState 12 | type TaskPredicate func(connect.TaskState) bool 13 | 14 | // IsRunning returns true if the connector task is in a RUNNING state 15 | func IsRunning(task connect.TaskState) bool { 16 | return task.State == "RUNNING" //nolint 17 | } 18 | 19 | // IsNotRunning returns true if the connector task is not in a RUNNING state 20 | func IsNotRunning(task connect.TaskState) bool { 21 | return task.State != "RUNNING" //nolint 22 | } 23 | 24 | // ByID returns a predicate that returns true if the connector task has one of the given task IDs 25 | func ByID(taskIDs ...int) TaskPredicate { 26 | sort.Ints(taskIDs) 27 | return func(task connect.TaskState) bool { 28 | found := sort.SearchInts(taskIDs, task.ID) 29 | return found < len(taskIDs) && taskIDs[found] == task.ID 30 | } 31 | } 32 | 33 | // IDs returns a subset of the Tasks for which the predicate returns true 34 | func (t Tasks) Filter(predicate TaskPredicate) Tasks { 35 | var found Tasks 36 | for _, task := range t { 37 | if predicate(task) { 38 | found = append(found, task) 39 | } 40 | } 41 | return found 42 | } 43 | 44 | // IDs returns the slice of task IDs 45 | func (t Tasks) IDs() []int { 46 | taskIDs := make([]int, len(t)) 47 | for i, task := range t { 48 | taskIDs[i] = task.ID 49 | } 50 | return taskIDs 51 | } 52 | -------------------------------------------------------------------------------- /pkg/manager/tasks_test.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/90poe/connectctl/pkg/client/connect" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestTasks_ByID_Found(t *testing.T) { 11 | f := ByID(3, 2, 1) 12 | assert.True(t, f(connect.TaskState{ID: 1})) 13 | assert.True(t, f(connect.TaskState{ID: 2})) 14 | assert.True(t, f(connect.TaskState{ID: 3})) 15 | assert.False(t, f(connect.TaskState{ID: 0})) 16 | assert.False(t, f(connect.TaskState{ID: 4})) 17 | } 18 | 19 | func TestTasks_IsRunning(t *testing.T) { 20 | f := IsRunning 21 | assert.True(t, f(connect.TaskState{State: "RUNNING"})) 22 | assert.False(t, f(connect.TaskState{State: "OTHER"})) 23 | } 24 | 25 | func TestTasks_IsNotRunning(t *testing.T) { 26 | f := IsNotRunning 27 | assert.False(t, f(connect.TaskState{State: "RUNNING"})) 28 | assert.True(t, f(connect.TaskState{State: "OTHER"})) 29 | } 30 | 31 | func TestTasks_IDsEmpty(t *testing.T) { 32 | var tasks Tasks 33 | assert.Equal(t, []int{}, tasks.IDs()) 34 | } 35 | 36 | func TestTasks_IDsNotEmpty(t *testing.T) { 37 | assert.Equal(t, []int{1, 2, 3}, 38 | Tasks{ 39 | connect.TaskState{ID: 1}, 40 | connect.TaskState{ID: 2}, 41 | connect.TaskState{ID: 3}, 42 | }.IDs()) 43 | } 44 | 45 | func TestTasks_Filter(t *testing.T) { 46 | assert.Equal(t, Tasks{ 47 | connect.TaskState{ID: 2}, 48 | connect.TaskState{ID: 1}, 49 | connect.TaskState{ID: 0}, 50 | }, 51 | Tasks{ 52 | connect.TaskState{ID: -2}, 53 | connect.TaskState{ID: 2}, 54 | connect.TaskState{ID: -1}, 55 | connect.TaskState{ID: 1}, 56 | connect.TaskState{ID: 0}, 57 | }.Filter(func(task connect.TaskState) bool { 58 | return task.ID >= 0 59 | })) 60 | } 61 | -------------------------------------------------------------------------------- /pkg/manager/types.go: -------------------------------------------------------------------------------- 1 | package manager 2 | 3 | import ( 4 | "github.com/90poe/connectctl/pkg/client/connect" 5 | ) 6 | 7 | // ConnectorWithState is the connect config and state 8 | type ConnectorWithState struct { 9 | Name string `json:"name"` 10 | Config connect.ConnectorConfig `json:"config,omitempty"` 11 | ConnectorState connect.ConnectorState `json:"connectorState,omitempty"` 12 | Tasks Tasks `json:"tasks,omitempty"` 13 | } 14 | -------------------------------------------------------------------------------- /pkg/signal/signal.go: -------------------------------------------------------------------------------- 1 | package signals 2 | 3 | import ( 4 | "os" 5 | "os/signal" 6 | ) 7 | 8 | //nolint:gochecknoglobals 9 | var onlyOneSignalHandler = make(chan struct{}) 10 | 11 | // SetupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned 12 | // which is closed on one of these signals. If a second signal is caught, the program 13 | // is terminated with exit code 1. 14 | func SetupSignalHandler() (stopCh <-chan struct{}) { 15 | close(onlyOneSignalHandler) // panics when called twice 16 | 17 | stop := make(chan struct{}) 18 | c := make(chan os.Signal, 2) 19 | signal.Notify(c, shutdownSignals...) 20 | go func() { 21 | <-c 22 | close(stop) 23 | <-c 24 | os.Exit(1) // second signal. Exit directly. 25 | }() 26 | 27 | return stop 28 | } 29 | -------------------------------------------------------------------------------- /pkg/signal/signal_posix.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | package signals 4 | 5 | import ( 6 | "os" 7 | "syscall" 8 | ) 9 | 10 | //nolint:gochecknoglobals 11 | var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} 12 | -------------------------------------------------------------------------------- /pkg/signal/signal_windows.go: -------------------------------------------------------------------------------- 1 | package signals 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | var shutdownSignals = []os.Signal{os.Interrupt} 8 | -------------------------------------------------------------------------------- /pkg/sources/sources.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | 13 | "github.com/pkg/errors" 14 | 15 | "github.com/90poe/connectctl/pkg/client/connect" 16 | ) 17 | 18 | // Files returns the aggregrated connectors loaded from a set of filepaths or an error 19 | func Files(files []string) func() ([]connect.Connector, error) { 20 | return func() ([]connect.Connector, error) { 21 | connectors := []connect.Connector{} 22 | 23 | for _, file := range files { 24 | bytes, err := ioutil.ReadFile(file) 25 | if err != nil { 26 | return nil, errors.Wrapf(err, "reading connector json %s", file) 27 | } 28 | 29 | c, err := processBytes(bytes) 30 | 31 | if err != nil { 32 | return nil, errors.Wrap(err, "unmarshalling connector from bytes") 33 | } 34 | 35 | connectors = append(connectors, c...) 36 | } 37 | 38 | return connectors, nil 39 | } 40 | } 41 | 42 | // Directory returns the aggregrated connectors loaded from a directory and its children or an error 43 | // Note - Files need to end with .json. 44 | func Directory(dir string) func() ([]connect.Connector, error) { 45 | return func() ([]connect.Connector, error) { 46 | var files []string 47 | 48 | err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { 49 | if filepath.Ext(path) == ".json" { 50 | files = append(files, path) 51 | } 52 | return nil 53 | }) 54 | if err != nil { 55 | return nil, errors.Wrapf(err, "list connector files in directory %s", dir) 56 | } 57 | 58 | return Files(files)() 59 | } 60 | } 61 | 62 | // EnvVarValue returns the connectors loaded from an environmental variable or an error 63 | func EnvVarValue(env string) func() ([]connect.Connector, error) { 64 | return func() ([]connect.Connector, error) { 65 | value, ok := os.LookupEnv(env) 66 | 67 | if !ok { 68 | return nil, fmt.Errorf("error resolving env var : %s", env) 69 | } 70 | 71 | value = strings.TrimSpace(value) 72 | return processBytes([]byte(value)) 73 | } 74 | } 75 | 76 | // StdIn returns the connectors piped via stdin or an error 77 | func StdIn(in io.Reader) func() ([]connect.Connector, error) { 78 | // read the input as io.Reader isn't re-readable 79 | data, err := ioutil.ReadAll(in) 80 | 81 | return func() ([]connect.Connector, error) { 82 | if err != nil { 83 | return nil, errors.Wrap(err, "error reading from StdIn") 84 | } 85 | 86 | return processBytes(data) 87 | } 88 | } 89 | 90 | func processBytes(data []byte) ([]connect.Connector, error) { 91 | if bytes.HasPrefix(data, []byte("[")) { // REVIEW : is there a better test for a JSON array? 92 | c, err := newConnectorsFromBytes(data) 93 | if err != nil { 94 | return nil, errors.Wrap(err, "error unmarshalling connectors from bytes") 95 | } 96 | return c, nil 97 | } 98 | 99 | c, err := newConnectorFromBytes(data) 100 | if err != nil { 101 | return nil, errors.Wrap(err, "error unmarshalling connector from bytes") 102 | } 103 | 104 | return []connect.Connector{c}, nil 105 | } 106 | 107 | func newConnectorFromBytes(bytes []byte) (connect.Connector, error) { 108 | c := connect.Connector{} 109 | err := json.Unmarshal(bytes, &c) 110 | return c, err 111 | } 112 | 113 | func newConnectorsFromBytes(bytes []byte) ([]connect.Connector, error) { 114 | c := []connect.Connector{} 115 | err := json.Unmarshal(bytes, &c) 116 | return c, err 117 | } 118 | -------------------------------------------------------------------------------- /pkg/sources/sources_test.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func Test_StdIn(t *testing.T) { 13 | 14 | testCases := []struct { 15 | input string 16 | hasErr bool 17 | count int 18 | }{ 19 | {input: "[{},{}]", count: 2}, 20 | {input: "[]", count: 0}, 21 | {input: "", count: 0, hasErr: true}, 22 | {input: "{}", count: 1}, 23 | {input: "{", count: 0, hasErr: true}, 24 | } 25 | 26 | for i := range testCases { 27 | tc := testCases[i] 28 | t.Run(tc.input, func(t *testing.T) { 29 | 30 | r := strings.NewReader(tc.input) 31 | f := StdIn(r) 32 | c, err := f() 33 | 34 | if tc.hasErr { 35 | require.NotNil(t, err) 36 | } else { 37 | require.Nil(t, err) 38 | } 39 | require.Equal(t, len(c), tc.count) 40 | 41 | }) 42 | } 43 | } 44 | 45 | func Test_EnvVarValue(t *testing.T) { 46 | 47 | testCases := []struct { 48 | name string 49 | input string 50 | hasErr bool 51 | count int 52 | }{ 53 | {name: "one", input: " [{},{}]", count: 2}, 54 | {name: "two", input: "[]", count: 0}, 55 | {name: "three", input: "", count: 0, hasErr: true}, 56 | {name: "four", input: "{ }", count: 1}, 57 | {name: "five", input: "{", count: 0, hasErr: true}, 58 | } 59 | 60 | for i := range testCases { 61 | tc := testCases[i] 62 | t.Run(tc.name, func(t *testing.T) { 63 | 64 | os.Setenv(tc.name, tc.input) 65 | c, err := EnvVarValue(tc.name)() 66 | 67 | if tc.hasErr { 68 | require.NotNil(t, err) 69 | } else { 70 | require.Nil(t, err) 71 | } 72 | require.Equal(t, len(c), tc.count) 73 | 74 | }) 75 | } 76 | } 77 | 78 | func Test_Directory(t *testing.T) { 79 | 80 | path := "../../examples/" 81 | 82 | c, err := Directory(path)() 83 | 84 | require.Nil(t, err) 85 | require.Len(t, c, 4) 86 | } 87 | --------------------------------------------------------------------------------