├── MANIFEST.in ├── setup.cfg ├── .gitignore ├── Makefile ├── setup.py ├── LICENSE ├── README.md └── kafka_connect └── __init__.py /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # Include Markdown files 2 | include *.md 3 | 4 | # Include the license file 5 | include LICENSE 6 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | # This includes the license file in the wheel. 3 | license_file = LICENSE 4 | 5 | [bdist_wheel] 6 | universal = 0 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # general things to ignore 2 | build/ 3 | dist/ 4 | *.egg-info/ 5 | *.egg 6 | *.py[cod] 7 | __pycache__/ 8 | *.so 9 | *~ 10 | 11 | # due to using tox and pytest 12 | .tox 13 | .cache 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: publish clean 2 | 3 | publish: 4 | git push origin && git push --tags origin 5 | $(MAKE) clean 6 | python setup.py sdist bdist_wheel 7 | twine upload dist/* 8 | $(MAKE) clean 9 | 10 | clean: 11 | @rm -Rf *.egg .cache .coverage .tox build dist docs/build htmlcov 12 | @find -depth -type d -name __pycache__ -exec rm -Rf {} \; 13 | @find -type f -name '*.pyc' -delete 14 | 15 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup(name='kafka-connect-python', 4 | version='0.0.3', 5 | description='Kafka Connect Module', 6 | url='http://github.com/sxnxl/kafka-connect-python', 7 | author='Senol Korkmaz', 8 | author_email='senol.korkmaz@gmail.com', 9 | license='MIT', 10 | packages=['kafka_connect'], 11 | classifiers=[ 12 | 'Development Status :: 3 - Alpha', 13 | 'Intended Audience :: Developers', 14 | 'License :: OSI Approved :: MIT License', 15 | 16 | 'Programming Language :: Python :: 3', 17 | ], 18 | ) 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Şenol Korkmaz 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kafka-connect-python 2 | Python module for Kafka Connect REST API 3 | 4 | 5 | ## Requirements 6 | * Python (3.6) 7 | 8 | ## Installation 9 | 10 | Install using `pip`... 11 | 12 | ```sh 13 | pip install kafka-connect-python 14 | ``` 15 | 16 | ## Examples 17 | 18 | ### Create KafkaConnect REST Interface 19 | ```python 20 | from kafka_connect import KafkaConnect 21 | 22 | connect = KafkaConnect(host='localhost', port=8083, scheme='http') 23 | 24 | print(connect.api.version) 25 | 26 | ``` 27 | 28 | ### Create a connector using config dictionary 29 | ```python 30 | config = { 31 | "connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector", 32 | "connection.url": "jdbc:postgresql://localhost/testdb?user=testuser&password=SecurePassword!", 33 | "key.converter": "io.confluent.connect.avro.AvroConverter", 34 | "key.converter.schema.registry.url": "http://localhost:8081", 35 | "value.converter": "io.confluent.connect.avro.AvroConverter", 36 | "value.converter.schema.registry.url": "http://localhost:8081", 37 | "table.whitelist": "sampletable", 38 | "mode": "timestamp", 39 | "timestamp.column.name": "lastupdated", 40 | "topic.prefix": "test-0-" 41 | } 42 | 43 | connect.connectors['sample-connector'] = config 44 | ``` 45 | 46 | ### Update connector config 47 | ```pyton 48 | connector = connect.connectors['sample-connector'] 49 | connector.config['poll.time.ms'] = 500 50 | ``` 51 | 52 | ### List connector names 53 | ```python 54 | list(map(lambda c: c.name, connect.connectors)) 55 | ``` 56 | 57 | ### Iterate over connectors and tasks 58 | ```python 59 | for connector in connect.connectors: 60 | for task in connector.tasks: 61 | task.restart() 62 | ``` 63 | 64 | ### Delete a connector 65 | ```python 66 | del connect.connectors['sample-connector'] 67 | ``` 68 | -------------------------------------------------------------------------------- /kafka_connect/__init__.py: -------------------------------------------------------------------------------- 1 | from collections import MutableMapping 2 | from urllib.request import Request, urlopen 3 | from urllib.error import URLError, HTTPError 4 | import json 5 | 6 | __all__ = ['KafkaConnect'] 7 | 8 | class Error(RuntimeError): 9 | retriable: bool = False 10 | code: int = 0 11 | 12 | def __str__(self): 13 | if not self.args: 14 | return self.__class__.__name__ 15 | return '{0}: {1}'.format(self.__class__.__name__, 16 | super().__str__()) 17 | 18 | class RebalanceError(Error): 19 | retriable = True 20 | code = 409 21 | 22 | class NotFoundError(Error): 23 | code = 404 24 | 25 | 26 | class TaskStatus: 27 | """ TaskStatus Type """ 28 | 29 | __slots__ = 'id', 'state', 'worker_id' 30 | 31 | def __init__(self, id, state, worker_id): 32 | self.id = id 33 | self.state = state 34 | self.worker_id = worker_id 35 | 36 | 37 | class Task: 38 | """ Task Type """ 39 | 40 | __slots__ = '_api', 'id', 'connector', 'config' 41 | 42 | def __init__(self, api, id, connector, config): 43 | self._api = api 44 | self.id = id 45 | self.connector = connector 46 | self.config = config 47 | 48 | @property 49 | def status(self): 50 | response = self._api.get( 51 | '/connectors/{}/tasks/{}/status'.format(self.connector, self.id)) 52 | id = response.get('id') 53 | state = response.get('state') 54 | worker_id = response.get('worker_id') 55 | return TaskStatus(id=id, state=state, worker_id=worker_id) 56 | 57 | def restart(self): 58 | self._api.post( 59 | 'connectors/{}/tasks/{}/restart'.format(self.connector, self.id)) 60 | 61 | 62 | class Config(MutableMapping): 63 | """ Connector Config """ 64 | 65 | __slots__ = '_api', '_config', '_connector' 66 | 67 | def __init__(self, api, connector): 68 | self._api = api 69 | self._connector = connector 70 | config = self._api.get("/connectors/{}/config".format(connector)) 71 | self._config = config 72 | 73 | def __getitem__(self, name): 74 | return self._config[name] 75 | 76 | def __setitem__(self, name, value): 77 | self._config[name] = value 78 | self._commit() 79 | 80 | def __delitem__(self, name): 81 | del self._config[name] 82 | self._commit() 83 | 84 | def _commit(self): 85 | self._api.put( 86 | "/connectors/{}/config".format(self._connector), data=self._config) 87 | 88 | def __iter__(self): 89 | return iter(self._config) 90 | 91 | def __len__(self): 92 | return len(self._config) 93 | 94 | def keys(self): 95 | return self._config.keys() 96 | 97 | 98 | class Connector: 99 | """ Connector Type """ 100 | 101 | __slots__ = '_name', '_api' 102 | 103 | def __init__(self, api, name): 104 | self._name = name 105 | self._api = api 106 | 107 | # TODO: create setter to rename if possible 108 | @property 109 | def name(self): 110 | return self._name 111 | 112 | @property 113 | def config(self): 114 | _config = Config(self._api, self.name) 115 | return _config 116 | 117 | @config.setter 118 | def config(self, value): 119 | self._api.put( 120 | "/connectors/{}/config".format(self.name), data=value) 121 | 122 | @property 123 | def status(self): 124 | self._api.get('/connectors/{}/status'.format(self.name)) 125 | 126 | def restart(self): 127 | self._api.post('/connectors/{}/restart'.format(self.name)) 128 | 129 | def pause(self): 130 | self._api.put('/connectors/{}/pause'.format(self.name)) 131 | 132 | def resume(self): 133 | self._api.put('/connectors/{}/resume'.format(self.name)) 134 | 135 | @property 136 | def tasks(self): 137 | response = self._api.get('/connectors/{}/tasks'.format(self.name)) 138 | _tasks = list() 139 | for task_dict in response: 140 | id_dict = task_dict.get('id') 141 | config = task_dict.get('config') 142 | id = id_dict.get('task') 143 | connector = id_dict.get('connector') 144 | task = Task(self._api, id, connector, config) 145 | _tasks.append(task) 146 | return _tasks 147 | 148 | 149 | class Connectors(MutableMapping): 150 | """ Connectors """ 151 | 152 | __slots__ = MutableMapping.__slots__ + ('_api',) 153 | 154 | def __init__(self, api): 155 | self._api = api 156 | 157 | def __getitem__(self, name): 158 | return Connector(self._api, name) 159 | 160 | def __setitem__(self, name, value): 161 | if isinstance(value, dict): 162 | config = value 163 | elif isinstance(value, Connector): 164 | config = value.config 165 | 166 | # TODO: test name in config 167 | if not 'name' in config: 168 | config['name'] = name 169 | 170 | if name in self.keys(): 171 | self._api.put("/connectors/{}/config".format(name), data=config) 172 | else: 173 | self._api.post( 174 | "/connectors", data={'name': name, 'config': config}) 175 | 176 | def __delitem__(self, name): 177 | self._api.delete("/connectors/{}".format(name)) 178 | 179 | def __iter__(self): 180 | for key in self.keys(): 181 | yield self.__getitem__(key) 182 | 183 | def __len__(self): 184 | return len(self.keys()) 185 | 186 | def keys(self): 187 | return self._api.get('/connectors') 188 | 189 | 190 | class API(object): 191 | """ Kafka Connect REST API Object """ 192 | 193 | __slots__ = 'host', 'port', 'url', 'autocommit', 'version', 'commit', 'kafka_cluster_id' 194 | 195 | def __init__(self, host='localhost', port=8083, scheme='http', autocommit=True): 196 | self.host = host 197 | self.port = port 198 | self.url = "{}://{}:{}".format(scheme, host, port) 199 | self.autocommit = autocommit 200 | self.ping() 201 | 202 | def ping(self): 203 | info = self.get() 204 | self.version = info.get('version') 205 | self.commit = info.get('commit') 206 | self.kafka_cluster_id = info.get('kafka_cluster_id') 207 | 208 | def request(self, endpoint='/', method='GET'): 209 | if not endpoint.startswith('/'): 210 | endpoint = '/{}'.format(endpoint) 211 | request_url = "{}{}".format(self.url, endpoint) 212 | request = Request(request_url, method=method) 213 | request.add_header('Accept', 'application/json') 214 | request.add_header('Content-Type', 'application/json') 215 | return request 216 | 217 | def response(self, request): 218 | try: 219 | response = urlopen(request) 220 | except HTTPError as e: 221 | code = e.code 222 | if code == 404: 223 | raise NotFoundError() 224 | elif code == 409: 225 | raise RebalanceError() 226 | else: 227 | raise Error() 228 | except (URLError, ConnectionError, ConnectionResetError, ConnectionAbortedError): 229 | raise ConnectionError() 230 | except: 231 | raise Error() 232 | 233 | response_data = response.read() 234 | if response_data: 235 | response_dict = json.loads(response_data) 236 | else: 237 | response_dict = {} 238 | return response_dict 239 | 240 | def get(self, endpoint='/'): 241 | request = self.request(endpoint) 242 | return self.response(request) 243 | 244 | def post(self, endpoint='/', data=None): 245 | request = self.request(endpoint, method='POST') 246 | if data: 247 | request.data = json.dumps(data).encode('utf-8') 248 | return self.response(request) 249 | 250 | def put(self, endpoint='/', data=None): 251 | request = self.request(endpoint, method='PUT') 252 | if data: 253 | request.data = json.dumps(data).encode('utf-8') 254 | return self.response(request) 255 | 256 | def delete(self, endpoint='/'): 257 | request = self.request(endpoint, method='DELETE') 258 | return self.response(request) 259 | 260 | 261 | class Plugin: 262 | """ Plugin Type """ 263 | 264 | __slots__ = '_api', 'class_name', 'type_name', 'version' 265 | 266 | def __init__(self, api, class_name, type_name, version): 267 | self._api = api 268 | self.class_name = class_name 269 | self.type_name = type_name 270 | self.version = version 271 | 272 | def __repr__(self): 273 | return self.class_name 274 | 275 | def validate(self, config): 276 | return self._api.put('/connector-plugins/{}/config/validate/'.format(self.class_name.split('.')[-1]), data=config) 277 | 278 | 279 | class KafkaConnect: 280 | """ Kafka Connect Object """ 281 | 282 | __slots__ = 'api', 'connectors' 283 | 284 | def __init__(self, host='localhost', port=8083, scheme='http'): 285 | self.api = API(host=host, port=port, scheme=scheme) 286 | self.connectors = Connectors(self.api) 287 | 288 | @property 289 | def plugins(self): # pylint: disable=unused-variable 290 | connector_plugins = self.api.get('/connector-plugins/') 291 | 292 | for plugin in connector_plugins: 293 | class_name = plugin.get('class') 294 | type_name = plugin.get('type') 295 | version = plugin.get('version') 296 | yield Plugin(self.api, class_name, type_name, version) 297 | --------------------------------------------------------------------------------