├── MANIFEST.in ├── tests ├── acceptance │ ├── kafka_consumer_manager.feature │ ├── config │ │ └── test.yaml │ ├── list_groups.feature │ ├── list_topics.feature │ ├── copy_group.feature │ ├── unsubscribe_topics.feature │ ├── delete_group.feature │ ├── offset_restore.feature │ ├── steps │ │ ├── __init__.py │ │ ├── unsubscribe_topics.py │ │ ├── delete_group.py │ │ ├── rename_group.py │ │ ├── copy_group.py │ │ ├── list_groups.py │ │ ├── list_topics.py │ │ ├── common.py │ │ ├── offset_get.py │ │ ├── min_isr.py │ │ ├── offset_save.py │ │ ├── dual_commit_fetch.py │ │ ├── offset_advance.py │ │ ├── offset_rewind.py │ │ ├── offset_restore.py │ │ ├── offset_set.py │ │ └── util.py │ ├── offset_set.feature │ ├── kafka_check.feature │ ├── environment.py │ ├── offset_rewind.feature │ ├── offset_advance.feature │ ├── offset_save.feature │ └── offset_get.feature ├── __init__.py ├── kafka_check │ ├── __init__.py │ └── test_command.py ├── kafka_cluster_manager │ ├── __init__.py │ ├── cmds │ │ └── __init__.py │ ├── helper.py │ ├── conftest.py │ ├── topic_test.py │ ├── util_test.py │ └── partition_test.py ├── kafka_consumer_manager │ ├── __init__.py │ ├── test_offset_get.py │ ├── test_offset_advance.py │ ├── test_offset_rewind.py │ ├── test_offset_restore.py │ ├── test_offset_save.py │ └── test_delete_group.py ├── kafka_rolling_restart │ ├── __init__.py │ └── test_main.py └── util │ └── test_protocol.py ├── requirements-dev.txt ├── scripts ├── kafka-utils ├── kafka-check ├── kafka-cluster-manager ├── kafka-consumer-manager └── kafka-rolling-restart ├── CHANGELOG.rst ├── docker ├── zookeeper │ └── Dockerfile ├── itest_0.8.2 │ ├── download-kafka.sh │ ├── run_tests.sh │ └── Dockerfile ├── itest_0.9.0 │ ├── download-kafka.sh │ ├── run_tests.sh │ └── Dockerfile ├── kafka_0.8.2 │ ├── download-kafka.sh │ └── Dockerfile └── kafka_0.9.0 │ ├── download-kafka.sh │ └── Dockerfile ├── docker-compose.yml ├── requirements.txt ├── Makefile ├── kafka_utils ├── kafka_check │ ├── __init__.py │ ├── commands │ │ ├── __init__.py │ │ ├── min_isr.py │ │ └── command.py │ ├── status_code.py │ └── main.py ├── kafka_cluster_manager │ ├── __init__.py │ ├── cmds │ │ ├── __init__.py │ │ ├── store_assignments.py │ │ ├── decommission.py │ │ └── replace.py │ └── cluster_info │ │ ├── __init__.py │ │ ├── error.py │ │ ├── replication_group_parser.py │ │ ├── topic.py │ │ ├── partition.py │ │ └── util.py ├── kafka_rolling_restart │ └── __init__.py ├── __init__.py ├── kafka_consumer_manager │ ├── commands │ │ ├── __init__.py │ │ ├── list_topics.py │ │ ├── delete_group.py │ │ ├── offset_rewind.py │ │ ├── offset_advance.py │ │ ├── copy_group.py │ │ ├── rename_group.py │ │ └── offset_set.py │ ├── __init__.py │ ├── main.py │ └── util.py ├── util │ ├── client.py │ ├── error.py │ ├── __init__.py │ └── protocol.py └── main.py ├── .gitignore ├── .pre-commit-config.yaml ├── .travis.yml ├── docs ├── source │ ├── index.rst │ ├── config.rst │ ├── kafka_check.rst │ └── kafka_rolling_restart.rst └── Makefile ├── tox.ini ├── setup.py └── README.md /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include kafka_utils *.py 2 | include README.md 3 | include LICENSE 4 | -------------------------------------------------------------------------------- /tests/acceptance/kafka_consumer_manager.feature: -------------------------------------------------------------------------------- 1 | Feature: kafka_consumer_manager 2 | 3 | 4 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | ipdb==0.10.0 2 | ipython==4.2.0 3 | ipython-genutils==0.1.0 4 | pre-commit==0.8.0 5 | -------------------------------------------------------------------------------- /scripts/kafka-utils: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from kafka_utils.main import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | -------------------------------------------------------------------------------- /scripts/kafka-check: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from kafka_utils.kafka_check.main import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | -------------------------------------------------------------------------------- /scripts/kafka-cluster-manager: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from kafka_utils.kafka_cluster_manager.main import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | -------------------------------------------------------------------------------- /scripts/kafka-consumer-manager: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from kafka_utils.kafka_consumer_manager.main import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | -------------------------------------------------------------------------------- /scripts/kafka-rolling-restart: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from kafka_utils.kafka_rolling_restart.main import run 4 | 5 | if __name__ == '__main__': 6 | run() 7 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | 0.1.1 (May 17, 2016) 2 | ---------------------- 3 | 4 | * Fix group-parser local import 5 | 6 | 0.1.0 (May 17, 2016) 7 | ---------------------- 8 | 9 | * Initial open-source release 10 | -------------------------------------------------------------------------------- /tests/acceptance/config/test.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | clusters: 3 | test_cluster: 4 | broker_list: 5 | - "kafka:9092" 6 | zookeeper: "zookeeper:2181" 7 | local_config: 8 | cluster: test_cluster 9 | -------------------------------------------------------------------------------- /docker/zookeeper/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | MAINTAINER Team Distributed Systems 3 | 4 | RUN apt-get update && apt-get -y install zookeeper 5 | 6 | CMD /usr/share/zookeeper/bin/zkServer.sh start-foreground 7 | -------------------------------------------------------------------------------- /tests/acceptance/list_groups.feature: -------------------------------------------------------------------------------- 1 | Feature: kafka_consumer_manager list_groups subcommand 2 | 3 | Scenario: Calling the list_groups command 4 | Given we have a set of existing consumer groups 5 | when we call the list_groups command 6 | then the groups will be listed 7 | -------------------------------------------------------------------------------- /tests/acceptance/list_topics.feature: -------------------------------------------------------------------------------- 1 | Feature: kafka_consumer_manager list_topics subcommand 2 | 3 | Scenario: Calling the list_topics command 4 | Given we have a set of existing topics and a consumer group 5 | when we call the list_topics command 6 | then the topics will be listed 7 | -------------------------------------------------------------------------------- /docker/itest_0.8.2/download-kafka.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | mirror=$(curl --stderr /dev/null https://www.apache.org/dyn/closer.cgi\?as_json\=1 | jq -r '.preferred') 4 | url="${mirror}kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" 5 | wget -q "${url}" -O "/tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" 6 | -------------------------------------------------------------------------------- /docker/itest_0.9.0/download-kafka.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | mirror=$(curl --stderr /dev/null https://www.apache.org/dyn/closer.cgi\?as_json\=1 | jq -r '.preferred') 4 | url="${mirror}kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" 5 | wget -q "${url}" -O "/tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" 6 | -------------------------------------------------------------------------------- /docker/kafka_0.8.2/download-kafka.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | mirror=$(curl --stderr /dev/null https://www.apache.org/dyn/closer.cgi\?as_json\=1 | jq -r '.preferred') 4 | url="${mirror}kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" 5 | wget -q "${url}" -O "/tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" 6 | -------------------------------------------------------------------------------- /docker/kafka_0.9.0/download-kafka.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | mirror=$(curl --stderr /dev/null https://www.apache.org/dyn/closer.cgi\?as_json\=1 | jq -r '.preferred') 4 | url="${mirror}kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" 5 | wget -q "${url}" -O "/tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" 6 | -------------------------------------------------------------------------------- /docker/itest_0.9.0/run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | function do_at_exit { 6 | exit_status=$? 7 | rm -rf build/ dist/ kafka_utils.egg-info/ 8 | rm -rf .tox/log .tox/dist .tox/acceptance 9 | find . -name '*.pyc' -delete 10 | find . -name '__pycache__' -delete 11 | exit $exit_status 12 | } 13 | 14 | # Clean up artifacts from tests 15 | trap do_at_exit EXIT INT TERM 16 | 17 | tox -e acceptance 18 | -------------------------------------------------------------------------------- /docker/itest_0.8.2/run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | function do_at_exit { 6 | exit_status=$? 7 | rm -rf build/ dist/ kafka_utils.egg-info/ 8 | rm -rf .tox/log .tox/dist .tox/acceptance 9 | find . -name '*.pyc' -delete 10 | find . -name '__pycache__' -print0 | xargs -0 rm -rf 11 | exit $exit_status 12 | } 13 | 14 | # Clean up artifacts from tests 15 | trap do_at_exit EXIT INT TERM 16 | 17 | tox -e acceptance 18 | -------------------------------------------------------------------------------- /tests/acceptance/copy_group.feature: -------------------------------------------------------------------------------- 1 | Feature: kafka_consumer_manager copy_group subcommand 2 | 3 | Scenario: Calling the copy_group command 4 | Given we have an existing kafka cluster with a topic 5 | when we produce some number of messages into the topic 6 | when we consume some number of messages from the topic 7 | when we call the copy_group command with a new groupid 8 | then the committed offsets in the new group will match the old group 9 | -------------------------------------------------------------------------------- /tests/acceptance/unsubscribe_topics.feature: -------------------------------------------------------------------------------- 1 | Feature: kafka_consumer_manager unsubscribe_topics subcommand 2 | 3 | Scenario: Calling the unsubscribe_topics command 4 | Given we have an existing kafka cluster with a topic 5 | when we produce some number of messages into the topic 6 | when we consume some number of messages from the topic 7 | when we call the unsubscribe_topics command 8 | then the committed offsets will no longer exist in zookeeper 9 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | itest: 5 | build: docker/itest_${KAFKA_VERSION} 6 | links: 7 | - kafka 8 | - zookeeper 9 | volumes: 10 | - .:/work 11 | command: echo "dummy command" 12 | 13 | kafka: 14 | build: docker/kafka_${KAFKA_VERSION} 15 | expose: 16 | - "9092" 17 | links: 18 | - zookeeper 19 | 20 | zookeeper: 21 | build: docker/zookeeper 22 | expose: 23 | - "2181" 24 | -------------------------------------------------------------------------------- /tests/acceptance/delete_group.feature: -------------------------------------------------------------------------------- 1 | Feature: kafka_consumer_manager delete_group subcommand 2 | 3 | Scenario: Calling the delete_group command 4 | Given we have an existing kafka cluster with a topic 5 | when we produce some number of messages into the topic 6 | when we consume some number of messages from the topic 7 | when we call the delete_group command 8 | when we call the offset_get command 9 | then the specified group will not be found 10 | -------------------------------------------------------------------------------- /tests/acceptance/offset_restore.feature: -------------------------------------------------------------------------------- 1 | Feature: kafka_consumer_manager offset_restore subcommand 2 | 3 | Scenario: Calling the offset_restore command 4 | Given we have an existing kafka cluster with a topic 5 | when we produce some number of messages into the topic 6 | when we consume some number of messages from the topic 7 | Given we have a json offsets file 8 | when we call the offset_restore command with the offsets file 9 | then the committed offsets will match the offsets file 10 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Fabric==1.8.1 2 | PyStaticConfiguration==0.9.0 3 | PyYAML==3.11 4 | argcomplete==0.8.1 5 | boto==2.38.0 6 | cffi==1.1.2 7 | cryptography==0.8.2 8 | ecdsa==0.11 9 | enum34==1.0.4 10 | futures==3.0.1 11 | jsonschema==2.4.0 12 | kafka-python==0.9.5 13 | kazoo==2.2 14 | ndg-httpsclient==0.3.3 15 | paramiko==1.16.0 16 | pyOpenSSL==0.15.1 17 | pyasn1==0.1.7 18 | pycparser==2.10 19 | pycrypto==2.6.1 20 | requests==2.7.0 21 | requests-futures==0.9.5 22 | setproctitle==1.1.8 23 | simplejson==3.6.5 24 | six==1.9.0 25 | wsgiref==0.1.2 26 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PACKAGE_VERSION := $(shell python setup.py --version) 2 | 3 | all: test 4 | 5 | clean: 6 | rm -rf build dist *.egg-info/ .tox/ virtualenv_run 7 | find . -name '*.pyc' -delete 8 | find . -name '__pycache__' -delete 9 | find . -name '*.deb' -delete 10 | find . -name '*.changes' -delete 11 | make -C docs clean 12 | 13 | test: 14 | tox 15 | 16 | acceptance: acceptance8 acceptance9 17 | 18 | acceptance8: 19 | tox -e docker_itest_8 20 | 21 | acceptance9: 22 | tox -e docker_itest_9 23 | 24 | coverage: 25 | tox -e coverage 26 | 27 | tag: 28 | git tag v${PACKAGE_VERSION} 29 | 30 | docs: 31 | tox -e docs 32 | 33 | .PHONY: all clean test coverage tag docs 34 | -------------------------------------------------------------------------------- /tests/kafka_check/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /kafka_utils/kafka_check/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /tests/acceptance/steps/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /tests/kafka_cluster_manager/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /tests/kafka_consumer_manager/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /tests/kafka_rolling_restart/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /kafka_utils/kafka_check/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /kafka_utils/kafka_cluster_manager/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /kafka_utils/kafka_rolling_restart/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /tests/kafka_cluster_manager/cmds/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /kafka_utils/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | __version__ = '0.1.1' 16 | -------------------------------------------------------------------------------- /kafka_utils/kafka_cluster_manager/cmds/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /kafka_utils/kafka_consumer_manager/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /kafka_utils/kafka_cluster_manager/cluster_info/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /kafka_utils/kafka_consumer_manager/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | __version__ = "0.1.0" 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Pycharm 9 | .idea/ 10 | 11 | # Distribution / packaging 12 | .Python 13 | env/ 14 | build/ 15 | dist/ 16 | dist 17 | eggs/ 18 | sdist/ 19 | *.egg-info/ 20 | .installed.cfg 21 | *.egg 22 | *.swp 23 | 24 | # PyInstaller 25 | # Usually these files are written by a python script from a template 26 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 27 | *.manifest 28 | *.spec 29 | 30 | # Installer logs 31 | pip-log.txt 32 | pip-delete-this-directory.txt 33 | 34 | # Unit test / coverage reports 35 | .tox/ 36 | .coverage 37 | .cache 38 | nosetests.xml 39 | coverage.xml 40 | 41 | # Sphinx documentation 42 | docs/_build/ 43 | 44 | # virtualenv 45 | virtualenv_run/ 46 | -------------------------------------------------------------------------------- /tests/acceptance/offset_set.feature: -------------------------------------------------------------------------------- 1 | Feature: kafka_consumer_manager offset_set subcommand 2 | 3 | Scenario: Calling the offset_set command 4 | Given we have an existing kafka cluster with a topic 5 | when we produce some number of messages into the topic 6 | when we consume some number of messages from the topic 7 | when we call the offset_set command with a groupid and offset data 8 | then the committed offsets will match the specified offsets 9 | 10 | Scenario: Calling the offset_set command when the group doesn't exist 11 | Given we have an existing kafka cluster with a topic 12 | when we produce some number of messages into the topic 13 | when we call the offset_set command with a new groupid and the force option 14 | then the committed offsets will match the specified offsets 15 | -------------------------------------------------------------------------------- /tests/acceptance/kafka_check.feature: -------------------------------------------------------------------------------- 1 | Feature: kafka_check 2 | 3 | Scenario: Calling the min_isr command on empty cluster 4 | Given we have an existing kafka cluster 5 | when we call the min_isr command 6 | then OK will be printed 7 | 8 | Scenario: Calling the min_isr command on a cluster with isr greater or equal to min.isr for each topic 9 | Given we have an existing kafka cluster with a topic 10 | when we change min.isr settings for a topic to 1 11 | when we call the min_isr command 12 | then OK will be printed 13 | 14 | Scenario: Calling the min_isr command on a cluster with isr smaller than min.isr 15 | Given we have an existing kafka cluster with a topic 16 | when we change min.isr settings for a topic to 2 17 | when we call the min_isr command 18 | then CRITICAL will be printed 19 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | - repo: https://github.com/pre-commit/pre-commit-hooks.git 2 | sha: 29bf11d13689a0a9a895c41eb3591c7e942d377d 3 | hooks: 4 | - id: check-added-large-files 5 | language_version: python2.7 6 | - id: check-merge-conflict 7 | language_version: python2.7 8 | - id: trailing-whitespace 9 | language_version: python2.7 10 | - id: end-of-file-fixer 11 | language_version: python2.7 12 | - id: autopep8-wrapper 13 | language_version: python2.7 14 | args: [--ignore=E501, --in-place] 15 | - id: flake8 16 | language_version: python2.7 17 | args: [--ignore=E501] 18 | 19 | - repo: https://github.com/asottile/reorder_python_imports.git 20 | sha: f3dfe379d2ea341c6cf54d926d4585b35dea9251 21 | hooks: 22 | - id: reorder-python-imports 23 | files: .*\.py$ 24 | language_version: python2.7 25 | -------------------------------------------------------------------------------- /tests/acceptance/environment.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from .steps.util import delete_topic 16 | from .steps.util import list_topics 17 | 18 | 19 | def before_scenario(context, scenario): 20 | """Remove all topics from Kafka before each test""" 21 | topics = list_topics() 22 | for topic in topics.split('\n'): 23 | delete_topic(topic) 24 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 2.7 3 | 4 | sudo: required 5 | 6 | services: 7 | - docker 8 | 9 | before_install: 10 | - sudo sh -c 'echo "deb https://apt.dockerproject.org/repo ubuntu-precise main" > /etc/apt/sources.list.d/docker.list' 11 | - sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D 12 | - sudo apt-get update 13 | - sudo apt-key update 14 | - sudo apt-get -qqy -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install docker-engine=1.11.1-0~precise 15 | - sudo rm /usr/local/bin/docker-compose 16 | - curl -L https://github.com/docker/compose/releases/download/1.7.0/docker-compose-`uname -s`-`uname -m` > docker-compose 17 | - chmod +x docker-compose 18 | - sudo mv docker-compose /usr/local/bin 19 | - docker-compose -v 20 | - docker -v 21 | 22 | install: 23 | - pip install tox 24 | 25 | script: 26 | - tox 27 | - tox -e docker_itest_8 28 | - tox -e docker_itest_9 29 | -------------------------------------------------------------------------------- /kafka_utils/kafka_check/status_code.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import print_function 16 | 17 | import sys 18 | 19 | OK = 0 20 | WARNING = 1 21 | CRITICAL = 2 22 | 23 | STATUS_STRING = { 24 | OK: 'OK', 25 | WARNING: 'WARNING', 26 | CRITICAL: 'CRITICAL', 27 | } 28 | 29 | 30 | def terminate(signal, msg): 31 | print('%s: %s' % (STATUS_STRING[signal], msg)) 32 | sys.exit(signal) 33 | -------------------------------------------------------------------------------- /tests/kafka_cluster_manager/helper.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from kafka_utils.kafka_cluster_manager.cluster_info.broker import Broker 16 | from kafka_utils.kafka_cluster_manager.cluster_info.partition import Partition 17 | 18 | 19 | def create_broker(broker_id, partitions): 20 | b = Broker(broker_id, partitions=set(partitions)) 21 | for p in partitions: 22 | p.add_replica(b) 23 | return b 24 | 25 | 26 | def create_and_attach_partition(topic, partition_id): 27 | partition = Partition(topic, partition_id) 28 | topic.add_partition(partition) 29 | return partition 30 | -------------------------------------------------------------------------------- /tests/kafka_check/test_command.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from kafka_utils.kafka_check.commands.command import parse_meta_properties_file 16 | 17 | 18 | META_PROPERTIES_CONTENT = [ 19 | "#\n", 20 | "#Thu May 12 17:59:12 BST 2016\n", 21 | "version=0\n", 22 | "broker.id=123\n", 23 | ] 24 | 25 | 26 | class TestParseMetaPropertiesFile(object): 27 | 28 | def test_parse_meta_properties_file(self): 29 | broker_id = parse_meta_properties_file(META_PROPERTIES_CONTENT) 30 | 31 | assert broker_id == 123 32 | 33 | def test_parse_meta_properties_file_empty(self): 34 | broker_id = parse_meta_properties_file([]) 35 | 36 | assert broker_id is None 37 | -------------------------------------------------------------------------------- /docker/kafka_0.8.2/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | MAINTAINER Team Distributed Systems 3 | 4 | # Install Java. 5 | ENV JAVA_HOME /usr/lib/jvm/java-8-oracle 6 | ENV PATH="$PATH:$JAVA_HOME/bin" 7 | RUN apt-get update && \ 8 | apt-get install software-properties-common -y 9 | RUN \ 10 | echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ 11 | add-apt-repository -y ppa:webupd8team/java && \ 12 | apt-get update && \ 13 | apt-get install -y oracle-java8-installer && \ 14 | rm -rf /var/lib/apt/lists/* && \ 15 | rm -rf /var/cache/oracle-jdk8-installer 16 | 17 | # Install Kafka. 18 | RUN apt-get update && apt-get install -y \ 19 | unzip \ 20 | wget \ 21 | curl \ 22 | jq \ 23 | coreutils 24 | ENV KAFKA_VERSION="0.8.2.1" 25 | ENV SCALA_VERSION="2.11" 26 | ENV KAFKA_HOME /opt/kafka_${SCALA_VERSION}-${KAFKA_VERSION} 27 | COPY download-kafka.sh /tmp/download-kafka.sh 28 | RUN chmod 755 /tmp/download-kafka.sh 29 | RUN /tmp/download-kafka.sh && tar xfz /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz -C /opt && rm /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz 30 | ENV PATH="$PATH:$KAFKA_HOME/bin" 31 | 32 | COPY config.properties /server.properties 33 | 34 | CMD echo "Kafka starting" && kafka-server-start.sh /server.properties 35 | -------------------------------------------------------------------------------- /docker/kafka_0.9.0/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | MAINTAINER Team Distributed Systems 3 | 4 | # Install Java. 5 | ENV JAVA_HOME /usr/lib/jvm/java-8-oracle 6 | ENV PATH="$PATH:$JAVA_HOME/bin" 7 | RUN apt-get update && \ 8 | apt-get install software-properties-common -y 9 | RUN \ 10 | echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ 11 | add-apt-repository -y ppa:webupd8team/java && \ 12 | apt-get update && \ 13 | apt-get install -y oracle-java8-installer && \ 14 | rm -rf /var/lib/apt/lists/* && \ 15 | rm -rf /var/cache/oracle-jdk8-installer 16 | 17 | # Install Kafka. 18 | RUN apt-get update && apt-get install -y \ 19 | unzip \ 20 | wget \ 21 | curl \ 22 | jq \ 23 | coreutils 24 | ENV KAFKA_VERSION="0.9.0.0" 25 | ENV SCALA_VERSION="2.11" 26 | ENV KAFKA_HOME /opt/kafka_${SCALA_VERSION}-${KAFKA_VERSION} 27 | COPY download-kafka.sh /tmp/download-kafka.sh 28 | RUN chmod 755 /tmp/download-kafka.sh 29 | RUN /tmp/download-kafka.sh && tar xfz /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz -C /opt && rm /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz 30 | ENV PATH="$PATH:$KAFKA_HOME/bin" 31 | 32 | COPY config.properties /server.properties 33 | 34 | CMD echo "Kafka starting" && kafka-server-start.sh /server.properties 35 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | Kafka-Utils v\ |version| 2 | ######################## 3 | 4 | Description 5 | *********** 6 | Kafka-Utils is a library containing tools to interact with kafka clusters and manage them. The tool provides utilities 7 | like listing of all the clusters, balancing the partition distribution across brokers and replication-groups, managing 8 | consumer groups, rolling-restart of the cluster, cluster healthchecks. 9 | 10 | For more information about Apache Kafka see the official `Kafka documentation`_. 11 | 12 | How to install 13 | ************** 14 | .. code-block:: bash 15 | 16 | $ pip install kafka-utils 17 | 18 | 19 | List available clusters. 20 | 21 | .. code-block:: bash 22 | 23 | $ kafka-utils 24 | Cluster type sample_type: 25 | Cluster name: cluster-1 26 | broker list: cluster-elb-1:9092 27 | zookeeper: 11.11.11.111:2181,11.11.11.112:2181,11.11.11.113:2181/kafka-1 28 | 29 | 30 | .. _Kafka documentation: http://kafka.apache.org/documentation.html#introduction 31 | 32 | .. toctree:: 33 | :maxdepth: -1 34 | 35 | config 36 | kafka_cluster_manager 37 | kafka_consumer_manager 38 | kafka_rolling_restart 39 | kafka_check 40 | 41 | 42 | Indices and tables 43 | ================== 44 | 45 | * :ref:`genindex` 46 | * :ref:`modindex` 47 | * :ref:`search` 48 | -------------------------------------------------------------------------------- /docs/source/config.rst: -------------------------------------------------------------------------------- 1 | Configuration 2 | ############# 3 | 4 | Kafka-Utils reads the cluster configuration needed to access Kafka clusters from yaml files. 5 | Each cluster is identified by *type* and *name*. 6 | Multiple clusters of the same type should be listed in the same `type.yaml` file. 7 | The yaml files are read from :code:`$KAFKA_DISCOVERY_DIR`, :code:`$HOME/.kafka_discovery` and :code:`/etc/kafka_discovery`, 8 | the former overrides the latter. 9 | 10 | Sample configuration for :code:`sample_type` cluster at :code:`/etc/kafka_discovery/sample_type.yaml` 11 | 12 | .. code-block:: yaml 13 | 14 | --- 15 | clusters: 16 | cluster-1: 17 | broker_list: 18 | - "cluster-elb-1:9092" 19 | zookeeper: "11.11.11.111:2181,11.11.11.112:2181,11.11.11.113:2181/kafka-1" 20 | cluster-2: 21 | broker_list: 22 | - "cluster-elb-2:9092" 23 | zookeeper: "11.11.11.211:2181,11.11.11.212:2181,11.11.11.213:2181/kafka-2" 24 | local_config: 25 | cluster: cluster-1 26 | 27 | For example the kafka-cluster-manager command: 28 | 29 | .. code:: bash 30 | 31 | $ kafka-cluster-manager --cluster-type sample_type stats 32 | 33 | will pick up default cluster `cluster-1` from the local_config at /etc/kafka_discovery/sample_type.yaml to display 34 | statistics of default kafka-configuration. 35 | -------------------------------------------------------------------------------- /tests/acceptance/offset_rewind.feature: -------------------------------------------------------------------------------- 1 | Feature: kafka_consumer_manager offset_rewind subcommand 2 | 3 | Scenario: Calling the offset_rewind command 4 | Given we have an existing kafka cluster with a topic 5 | when we produce some number of messages into the topic 6 | when we consume some number of messages from the topic 7 | when we call the offset_rewind command with a groupid and topic 8 | then the committed offsets will match the earliest message offsets 9 | 10 | @kafka9 11 | Scenario: Calling the offset_rewind command with kafka storage 12 | Given we have an existing kafka cluster with a topic 13 | Given we have initialized kafka offsets storage 14 | when we produce some number of messages into the topic 15 | when we consume some number of messages from the topic 16 | when we call the offset_rewind command and commit into kafka 17 | when we call the offset_get command 18 | then the earliest message offsets will be shown 19 | 20 | Scenario: Calling the offset_rewind command when the group doesn't exist 21 | Given we have an existing kafka cluster with a topic 22 | when we produce some number of messages into the topic 23 | when we call the offset_rewind command with a new groupid and the force option 24 | then the committed offsets will match the earliest message offsets 25 | -------------------------------------------------------------------------------- /tests/acceptance/offset_advance.feature: -------------------------------------------------------------------------------- 1 | Feature: kafka_consumer_manager offset_advance 2 | 3 | Scenario: Calling the offset_advance command 4 | Given we have an existing kafka cluster with a topic 5 | when we produce some number of messages into the topic 6 | when we consume some number of messages from the topic 7 | when we call the offset_advance command with a groupid and topic 8 | then the committed offsets will match the latest message offsets 9 | 10 | @kafka9 11 | Scenario: Calling the offset_advance command with kafka storage 12 | Given we have an existing kafka cluster with a topic 13 | Given we have initialized kafka offsets storage 14 | when we produce some number of messages into the topic 15 | when we consume some number of messages from the topic 16 | when we call the offset_advance command and commit into kafka 17 | when we call the offset_get command with kafka storage 18 | then the latest message offsets will be shown 19 | 20 | Scenario: Calling the offset_advance command when the group doesn't exist 21 | Given we have an existing kafka cluster with a topic 22 | when we produce some number of messages into the topic 23 | when we call the offset_advance command with a new groupid and the force option 24 | then the committed offsets will match the latest message offsets 25 | -------------------------------------------------------------------------------- /tests/kafka_cluster_manager/conftest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import pytest 16 | 17 | from kafka_utils.kafka_cluster_manager.cluster_info.partition import Partition 18 | from kafka_utils.kafka_cluster_manager.cluster_info.topic import Topic 19 | 20 | 21 | @pytest.fixture 22 | def create_partition(): 23 | """Fixture to create a partition and attach it to a topic""" 24 | topics = {} 25 | 26 | def _add_partition(topic_id, partition_id, replication_factor=1): 27 | if topic_id not in topics: 28 | topics[topic_id] = Topic(topic_id, replication_factor) 29 | topic = topics[topic_id] 30 | partition = Partition(topic, partition_id) 31 | topic.add_partition(partition) 32 | return partition 33 | 34 | return _add_partition 35 | -------------------------------------------------------------------------------- /docs/source/kafka_check.rst: -------------------------------------------------------------------------------- 1 | Kafka Check 2 | *********** 3 | 4 | Checking in-sync replicas 5 | ========================= 6 | This kafka tool provides the ability to check in-sync replicas for each topic-partition 7 | in the cluster. 8 | 9 | The :code:`min_isr` command checks if the number of in-sync replicas for a 10 | partition is equal or greater than the minimum number of in-sync replicas 11 | configured for the topic the partition belongs to. A topic specific 12 | :code:`min.insync.replicas` overrides the given default. 13 | 14 | The parameters for min_isr check are: 15 | 16 | * :code:`--default_min_isr DEFAULT_MIN_ISR`: Default min.isr value for cases without 17 | settings in Zookeeper for some topics. 18 | * :code:`--data-path DATA_PATH`: Path to the Kafka data folder. 19 | * :code:`--controller-only`: If this parameter is specified, it will do nothing and 20 | succeed on non-controller brokers. If :code:`--broker-id` is also set as -1 21 | then broker-id will be computed from given data-path. 22 | 23 | .. code-block:: bash 24 | 25 | $ kafka-check --cluster-type=sample_type min_isr 26 | OK: All replicas in sync. 27 | 28 | In case of min isr violations: 29 | 30 | .. code-block:: bash 31 | 32 | $ kafka-check --cluster-type=sample_type min_isr --default_min_isr 3 33 | 34 | isr=2 is lower than min_isr=3 for sample_topic:0 35 | CRITICAL: 1 partition(s) have the number of replicas in sync that is lower 36 | than the specified min ISR. 37 | 38 | -------------------------------------------------------------------------------- /tests/acceptance/offset_save.feature: -------------------------------------------------------------------------------- 1 | Feature: kafka_consumer_manager offset_save subcommand 2 | 3 | Scenario: Calling the offset_save command after consuming some messages 4 | Given we have an existing kafka cluster with a topic 5 | when we produce some number of messages into the topic 6 | when we consume some number of messages from the topic 7 | when we call the offset_save command with an offsets file 8 | then the correct offsets will be saved into the given file 9 | 10 | Scenario: Calling offset_save after offset_restore 11 | Given we have an existing kafka cluster with a topic 12 | when we produce some number of messages into the topic 13 | when we consume some number of messages from the topic 14 | Given we have a json offsets file 15 | when we call the offset_restore command with the offsets file 16 | when we call the offset_save command with an offsets file 17 | then the restored offsets will be saved into the given file 18 | 19 | @kafka9 20 | Scenario: Calling offset_save after offset_restore with kafka storage 21 | Given we have an existing kafka cluster with a topic 22 | Given we have initialized kafka offsets storage 23 | when we produce some number of messages into the topic 24 | when we consume some number of messages from the topic 25 | Given we have a json offsets file 26 | when we call the offset_restore command with the offsets file and kafka storage 27 | when we call the offset_save command with an offsets file and kafka storage 28 | then the restored offsets will be saved into the given file 29 | -------------------------------------------------------------------------------- /tests/acceptance/steps/unsubscribe_topics.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from behave import then 16 | from behave import when 17 | 18 | from .util import call_cmd 19 | from .util import get_cluster_config 20 | from kafka_utils.util.zookeeper import ZK 21 | 22 | 23 | def call_unsubscribe_topics(groupid): 24 | cmd = ['kafka-consumer-manager', 25 | '--cluster-type', 'test', 26 | '--cluster-name', 'test_cluster', 27 | '--discovery-base-path', 'tests/acceptance/config', 28 | 'unsubscribe_topics', 29 | groupid] 30 | return call_cmd(cmd) 31 | 32 | 33 | @when('we call the unsubscribe_topics command') 34 | def step_impl2(context): 35 | call_unsubscribe_topics(context.group) 36 | 37 | 38 | @then(u'the committed offsets will no longer exist in zookeeper') 39 | def step_impl4(context): 40 | cluster_config = get_cluster_config() 41 | with ZK(cluster_config) as zk: 42 | offsets = zk.get_group_offsets(context.group) 43 | assert context.topic not in offsets 44 | -------------------------------------------------------------------------------- /tests/acceptance/steps/delete_group.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from behave import then 16 | from behave import when 17 | 18 | from .util import call_cmd 19 | 20 | 21 | def call_delete_group(groupid, storage=None): 22 | cmd = ['kafka-consumer-manager', 23 | '--cluster-type', 'test', 24 | '--cluster-name', 'test_cluster', 25 | '--discovery-base-path', 'tests/acceptance/config', 26 | 'delete_group', 27 | groupid] 28 | if storage: 29 | cmd.extend(['--storage', storage]) 30 | return call_cmd(cmd) 31 | 32 | 33 | @when('we call the delete_group command') 34 | def step_impl2(context): 35 | call_delete_group(context.group) 36 | 37 | 38 | @when('we call the delete_group command with kafka storage') 39 | def step_impl2_2(context): 40 | call_delete_group(context.group, storage='kafka') 41 | 42 | 43 | @then(u'the specified group will not be found') 44 | def step_impl5(context): 45 | pattern = 'Error: Consumer Group ID {} does' \ 46 | ' not exist.'.format(context.group) 47 | assert pattern in context.output 48 | -------------------------------------------------------------------------------- /kafka_utils/kafka_cluster_manager/cluster_info/error.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from kafka_utils.util.error import KafkaToolError 16 | 17 | 18 | class InvalidBrokerIdError(KafkaToolError): 19 | """Raised when a broker id doesn't exist in the cluster.""" 20 | pass 21 | 22 | 23 | class InvalidPartitionError(KafkaToolError): 24 | """Raised when a partition tuple (topic, partition) doesn't exist in the cluster""" 25 | pass 26 | 27 | 28 | class EmptyReplicationGroupError(KafkaToolError): 29 | """Raised when there are no brokers in a replication group.""" 30 | pass 31 | 32 | 33 | class BrokerDecommissionError(KafkaToolError): 34 | """Raised if it is not possible to move partition out 35 | from decommissioned brokers. 36 | """ 37 | pass 38 | 39 | 40 | class NotEligibleGroupError(KafkaToolError): 41 | """Raised when there are no brokers eligible to acquire a certain partition 42 | in a replication group. 43 | """ 44 | pass 45 | 46 | 47 | class RebalanceError(KafkaToolError): 48 | """Raised when a rebalance operation is not possible.""" 49 | pass 50 | -------------------------------------------------------------------------------- /kafka_utils/util/client.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import print_function 17 | from __future__ import unicode_literals 18 | 19 | import functools 20 | 21 | from kafka import KafkaClient 22 | 23 | from kafka_utils.util.protocol import KafkaToolProtocol 24 | 25 | 26 | class KafkaToolClient(KafkaClient): 27 | ''' 28 | Extends the KafkaClient class, and includes a method for sending offset 29 | commit requests to Kafka. 30 | ''' 31 | 32 | def send_offset_commit_request_kafka( 33 | self, group, payloads=[], 34 | fail_on_error=True, callback=None): 35 | encoder = functools.partial( 36 | KafkaToolProtocol.encode_offset_commit_request_kafka, 37 | group=group, 38 | ) 39 | decoder = KafkaToolProtocol.decode_offset_commit_response 40 | resps = self._send_consumer_aware_request(group, payloads, encoder, decoder) 41 | 42 | return [resp if not callback else callback(resp) for resp in resps 43 | if not fail_on_error or not self._raise_on_response_error(resp)] 44 | -------------------------------------------------------------------------------- /docker/itest_0.8.2/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | MAINTAINER Team Distributed Systems 3 | 4 | # We need to install Java and Kafka in order to use Kafka CLI. The Kafka server 5 | # will never run in this container; the Kafka server will run in the "kafka" 6 | # container. 7 | 8 | # Install Java. 9 | ENV JAVA_HOME /usr/lib/jvm/java-8-oracle 10 | ENV PATH="$PATH:$JAVA_HOME/bin" 11 | RUN apt-get update && \ 12 | apt-get install software-properties-common -y 13 | RUN \ 14 | echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ 15 | add-apt-repository -y ppa:webupd8team/java && \ 16 | apt-get update && \ 17 | apt-get install -y oracle-java8-installer && \ 18 | rm -rf /var/lib/apt/lists/* && \ 19 | rm -rf /var/cache/oracle-jdk8-installer 20 | 21 | # Install Kafka. 22 | RUN apt-get update && apt-get install -y \ 23 | unzip \ 24 | wget \ 25 | curl \ 26 | jq \ 27 | coreutils 28 | ENV KAFKA_VERSION="0.8.2.1" 29 | ENV SCALA_VERSION="2.11" 30 | ENV KAFKA_HOME /opt/kafka_${SCALA_VERSION}-${KAFKA_VERSION} 31 | COPY download-kafka.sh /tmp/download-kafka.sh 32 | RUN chmod 755 /tmp/download-kafka.sh 33 | RUN /tmp/download-kafka.sh && tar xfz /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz -C /opt && rm /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz 34 | ENV PATH="$PATH:$KAFKA_HOME/bin" 35 | 36 | # Install Python. 37 | RUN apt-get update && apt-get install -y \ 38 | build-essential \ 39 | python2.7 \ 40 | python2.7-dev \ 41 | python-pip \ 42 | python-pkg-resources \ 43 | python-setuptools 44 | 45 | RUN pip install tox 46 | 47 | COPY run_tests.sh /scripts/run_tests.sh 48 | RUN chmod 755 /scripts/run_tests.sh 49 | 50 | WORKDIR /work 51 | -------------------------------------------------------------------------------- /docker/itest_0.9.0/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | MAINTAINER Team Distributed Systems 3 | 4 | # We need to install Java and Kafka in order to use Kafka CLI. The Kafka server 5 | # will never run in this container; the Kafka server will run in the "kafka" 6 | # container. 7 | 8 | # Install Java. 9 | ENV JAVA_HOME /usr/lib/jvm/java-8-oracle 10 | ENV PATH="$PATH:$JAVA_HOME/bin" 11 | RUN apt-get update && \ 12 | apt-get install software-properties-common -y 13 | RUN \ 14 | echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ 15 | add-apt-repository -y ppa:webupd8team/java && \ 16 | apt-get update && \ 17 | apt-get install -y oracle-java8-installer && \ 18 | rm -rf /var/lib/apt/lists/* && \ 19 | rm -rf /var/cache/oracle-jdk8-installer 20 | 21 | # Install Kafka. 22 | RUN apt-get update && apt-get install -y \ 23 | unzip \ 24 | wget \ 25 | curl \ 26 | jq \ 27 | coreutils 28 | ENV KAFKA_VERSION="0.9.0.0" 29 | ENV SCALA_VERSION="2.11" 30 | ENV KAFKA_HOME /opt/kafka_${SCALA_VERSION}-${KAFKA_VERSION} 31 | COPY download-kafka.sh /tmp/download-kafka.sh 32 | RUN chmod 755 /tmp/download-kafka.sh 33 | RUN /tmp/download-kafka.sh && tar xfz /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz -C /opt && rm /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz 34 | ENV PATH="$PATH:$KAFKA_HOME/bin" 35 | 36 | # Install Python. 37 | RUN apt-get update && apt-get install -y \ 38 | build-essential \ 39 | python2.7 \ 40 | python2.7-dev \ 41 | python-pip \ 42 | python-pkg-resources \ 43 | python-setuptools 44 | 45 | RUN pip install tox 46 | 47 | COPY run_tests.sh /scripts/run_tests.sh 48 | RUN chmod 755 /scripts/run_tests.sh 49 | 50 | WORKDIR /work 51 | -------------------------------------------------------------------------------- /tests/acceptance/steps/rename_group.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from behave import then 16 | from behave import when 17 | 18 | from .util import call_cmd 19 | from .util import get_cluster_config 20 | from kafka_utils.util.zookeeper import ZK 21 | 22 | 23 | NEW_GROUP = 'new_group' 24 | 25 | 26 | def call_rename_group(old_group, new_group): 27 | cmd = ['kafka-consumer-manager', 28 | '--cluster-type', 'test', 29 | '--cluster-name', 'test_cluster', 30 | '--discovery-base-path', 'tests/acceptance/config', 31 | 'rename_group', 32 | old_group, 33 | new_group] 34 | return call_cmd(cmd) 35 | 36 | 37 | @when(u'we call the rename_group command with a new groupid') 38 | def step_impl2(context): 39 | call_rename_group(NEW_GROUP) 40 | 41 | 42 | @then(u'the committed offsets in the new group will match the expected values') 43 | def step_impl4(context): 44 | cluster_config = get_cluster_config() 45 | with ZK(cluster_config) as zk: 46 | offsets = zk.get_group_offsets(context.group) 47 | new_offsets = zk.get_group_offsets(NEW_GROUP) 48 | assert offsets is None 49 | assert context.topic in new_offsets 50 | -------------------------------------------------------------------------------- /tests/acceptance/steps/copy_group.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from behave import then 16 | from behave import when 17 | 18 | from .util import call_cmd 19 | from .util import get_cluster_config 20 | from kafka_utils.util.zookeeper import ZK 21 | 22 | 23 | NEW_GROUP = 'new_group' 24 | 25 | 26 | def call_copy_group(old_group, new_group): 27 | cmd = ['kafka-consumer-manager', 28 | '--cluster-type', 'test', 29 | '--cluster-name', 'test_cluster', 30 | '--discovery-base-path', 'tests/acceptance/config', 31 | 'copy_group', 32 | old_group, 33 | new_group] 34 | return call_cmd(cmd) 35 | 36 | 37 | @when(u'we call the copy_group command with a new groupid') 38 | def step_impl2(context): 39 | call_copy_group(context.group, NEW_GROUP) 40 | 41 | 42 | @then(u'the committed offsets in the new group will match the old group') 43 | def step_impl4(context): 44 | cluster_config = get_cluster_config() 45 | with ZK(cluster_config) as zk: 46 | offsets = zk.get_group_offsets(context.group) 47 | new_offsets = zk.get_group_offsets(NEW_GROUP) 48 | assert context.topic in offsets 49 | assert new_offsets == offsets 50 | -------------------------------------------------------------------------------- /tests/acceptance/steps/list_groups.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from behave import given 16 | from behave import then 17 | from behave import when 18 | 19 | from .util import call_cmd 20 | from .util import create_consumer_group 21 | from .util import create_random_topic 22 | from .util import produce_example_msg 23 | 24 | test_groups = ['group1', 'group2', 'group3'] 25 | 26 | 27 | @given('we have a set of existing consumer groups') 28 | def step_impl1(context): 29 | topic = create_random_topic(1, 1) 30 | produce_example_msg(topic) 31 | 32 | for group in test_groups: 33 | create_consumer_group(topic, group) 34 | 35 | 36 | def call_list_groups(): 37 | cmd = ['kafka-consumer-manager', 38 | '--cluster-type', 'test', 39 | '--cluster-name', 'test_cluster', 40 | '--discovery-base-path', 'tests/acceptance/config', 41 | 'list_groups'] 42 | return call_cmd(cmd) 43 | 44 | 45 | @when('we call the list_groups command') 46 | def step_impl2(context): 47 | context.output = call_list_groups() 48 | 49 | 50 | @then('the groups will be listed') 51 | def step_impl3(context): 52 | for group in test_groups: 53 | assert group in context.output 54 | -------------------------------------------------------------------------------- /kafka_utils/kafka_cluster_manager/cluster_info/replication_group_parser.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | class ReplicationGroupParser(object): 18 | """Base class for replication group parsers""" 19 | 20 | def get_replication_group(self, broker): 21 | """Implement the logic to extract the replication group id of a broker. 22 | 23 | This method is called with a broker object as argument. 24 | 25 | Example: 26 | 27 | .. code-block:: python 28 | 29 | class MyParser(ReplicationGroupParser): 30 | 31 | def get_replication_group(self, broker): 32 | return broker.metadata['host'].split('.', 2)[1] 33 | 34 | :param broker: py:class:`kafka_utils.kafka_cluster_manager.cluster_info.broker.Broker` 35 | :returns: a string representing the replication group name of the broker 36 | """ 37 | raise NotImplementedError("Implement in subclass") 38 | 39 | 40 | class DefaultReplicationGroupParser(ReplicationGroupParser): 41 | 42 | def get_replication_group(self, broker): 43 | """Default group is None. All brokers are considered in the same group. 44 | 45 | TODO: Support kafka 0.10 and rack tag. 46 | """ 47 | return None 48 | -------------------------------------------------------------------------------- /tests/acceptance/steps/list_topics.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from behave import given 16 | from behave import then 17 | from behave import when 18 | 19 | from .util import call_cmd 20 | from .util import create_consumer_group 21 | from .util import create_random_topic 22 | from .util import produce_example_msg 23 | 24 | test_group = 'group1' 25 | test_topics = ['topic1', 'topic2', 'topic3'] 26 | 27 | 28 | @given('we have a set of existing topics and a consumer group') 29 | def step_impl1(context): 30 | for topic in test_topics: 31 | create_random_topic(1, 1, topic_name=topic) 32 | produce_example_msg(topic) 33 | 34 | create_consumer_group(topic, test_group) 35 | 36 | 37 | def call_list_topics(groupid): 38 | cmd = ['kafka-consumer-manager', 39 | '--cluster-type', 'test', 40 | '--cluster-name', 'test_cluster', 41 | '--discovery-base-path', 'tests/acceptance/config', 42 | 'list_topics', 43 | groupid] 44 | return call_cmd(cmd) 45 | 46 | 47 | @when('we call the list_topics command') 48 | def step_impl2(context): 49 | context.output = call_list_topics('group1') 50 | 51 | 52 | @then('the topics will be listed') 53 | def step_impl3(context): 54 | for topic in test_topics: 55 | assert topic in context.output 56 | -------------------------------------------------------------------------------- /tests/kafka_cluster_manager/topic_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import pytest 16 | from mock import Mock 17 | from mock import sentinel 18 | 19 | from kafka_utils.kafka_cluster_manager.cluster_info.partition import Partition 20 | from kafka_utils.kafka_cluster_manager.cluster_info.topic import Topic 21 | 22 | 23 | class TestTopic(object): 24 | 25 | @pytest.fixture 26 | def topic(self): 27 | return Topic('t0', 2, set([sentinel.p1, sentinel.p2])) 28 | 29 | def test_id(self, topic): 30 | assert topic.id == 't0' 31 | 32 | def test_replication_factor(self, topic): 33 | assert topic.replication_factor == 2 34 | 35 | def test_partitions(self, topic): 36 | assert topic.partitions == set([sentinel.p1, sentinel.p2]) 37 | 38 | def test_add_partition(self): 39 | mock_partitions = set([ 40 | Mock( 41 | spec=Partition, 42 | replicas=[sentinel.r1, sentinel.r2], 43 | ), 44 | Mock( 45 | spec=Partition, 46 | replicas=[sentinel.r4, sentinel.r3], 47 | ), 48 | ]) 49 | topic = Topic('t0', 2, mock_partitions) 50 | new_partition = Mock(spec=Partition, replicas=[sentinel.r2]) 51 | topic.add_partition(new_partition) 52 | assert topic.partitions == mock_partitions | set([new_partition]) 53 | -------------------------------------------------------------------------------- /tests/acceptance/steps/common.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from behave import given 16 | from behave import when 17 | 18 | from .util import create_consumer_group 19 | from .util import create_random_group_id 20 | from .util import create_random_topic 21 | from .util import initialize_kafka_offsets_topic 22 | from .util import produce_example_msg 23 | 24 | PRODUCED_MSG_COUNT = 82 25 | CONSUMED_MSG_COUNT = 39 26 | 27 | 28 | @given(u'we have an existing kafka cluster with a topic') 29 | def step_impl1(context): 30 | context.topic = create_random_topic(1, 1) 31 | 32 | 33 | @when(u'we produce some number of messages into the topic') 34 | def step_impl2(context): 35 | produce_example_msg(context.topic, num_messages=PRODUCED_MSG_COUNT) 36 | context.msgs_produced = PRODUCED_MSG_COUNT 37 | 38 | 39 | @when(u'we consume some number of messages from the topic') 40 | def step_impl3(context): 41 | context.group = create_random_group_id() 42 | context.consumer = create_consumer_group( 43 | context.topic, 44 | context.group, 45 | num_messages=CONSUMED_MSG_COUNT, 46 | ) 47 | context.msgs_consumed = CONSUMED_MSG_COUNT 48 | 49 | 50 | @given(u'we have initialized kafka offsets storage') 51 | def step_impl4(context): 52 | initialize_kafka_offsets_topic() 53 | 54 | 55 | @given(u'we have an existing kafka cluster') 56 | def step_impl5(context): 57 | pass 58 | -------------------------------------------------------------------------------- /tests/acceptance/steps/offset_get.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from behave import then 16 | from behave import when 17 | 18 | from .util import call_offset_get 19 | 20 | 21 | @when(u'we call the offset_get command') 22 | def step_impl4(context): 23 | context.output = call_offset_get(context.group) 24 | 25 | 26 | @when(u'we call the offset_get command with the dual storage option') 27 | def step_impl4_2(context): 28 | context.output = call_offset_get(context.group, storage='dual') 29 | 30 | 31 | @when(u'we call the offset_get command with kafka storage') 32 | def step_impl4_3(context): 33 | context.output = call_offset_get(context.group, storage='kafka') 34 | 35 | 36 | @when(u'we call the offset_get command with zookeeper storage') 37 | def step_impl4_4(context): 38 | context.output = call_offset_get(context.group, storage='zookeeper') 39 | 40 | 41 | @then(u'the correct offset will be shown') 42 | def step_impl5(context): 43 | offsets = context.consumer.offsets(group='commit') 44 | key = (context.topic, 0) 45 | offset = offsets[key] 46 | pattern = 'Current Offset: {}'.format(offset) 47 | assert pattern in context.output 48 | 49 | 50 | @then(u'the offset that was committed into Kafka will be shown') 51 | def step_impl5_2(context): 52 | offset = context.set_offset_kafka 53 | pattern = 'Current Offset: {}'.format(offset) 54 | assert pattern in context.output 55 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27 3 | 4 | [testenv] 5 | deps = 6 | -rrequirements-dev.txt 7 | flake8 8 | pytest 9 | mock 10 | commands = 11 | pre-commit install -f --install-hooks 12 | py.test -s {posargs} 13 | flake8 . 14 | 15 | [testenv:coverage] 16 | basepython = python2.7 17 | deps = 18 | {[testenv]deps} 19 | coverage 20 | commands = 21 | coverage run --source kafka_utils/ -m pytest --strict {posargs} 22 | coverage report -m 23 | flake8 . 24 | 25 | [testenv:acceptance] 26 | deps = 27 | {[testenv]deps} 28 | behave 29 | passenv = KAFKA_VERSION 30 | commands = 31 | /bin/bash -c "if [ $KAFKA_VERSION = '0.9.0.0' ]; then \ 32 | behave tests/acceptance --no-capture; \ 33 | else \ 34 | behave tests/acceptance --tags=-kafka9 --no-capture; \ 35 | fi" 36 | 37 | [testenv:docker_itest_8] 38 | deps = 39 | docker-compose==1.6.2 40 | basepython = python2.7 41 | whitelist_externals = /bin/bash 42 | commands = 43 | /bin/bash -c "export KAFKA_VERSION='0.8.2'; \ 44 | docker-compose rm --force && \ 45 | docker-compose build && \ 46 | docker-compose run itest /scripts/run_tests.sh; exit_status=$?; \ 47 | docker-compose stop; exit $exit_status" 48 | 49 | [testenv:docker_itest_9] 50 | deps = 51 | docker-compose==1.6.2 52 | basepython = python2.7 53 | whitelist_externals = /bin/bash 54 | commands = 55 | /bin/bash -c "export KAFKA_VERSION='0.9.0'; \ 56 | docker-compose rm --force && \ 57 | docker-compose build && \ 58 | docker-compose run itest /scripts/run_tests.sh; exit_status=$?; \ 59 | docker-compose stop; exit $exit_status" 60 | 61 | [testenv:docs] 62 | deps = 63 | {[testenv]deps} 64 | sphinx 65 | sphinx_rtd_theme 66 | changedir = docs 67 | commands = sphinx-build -b html -d build/doctrees source build/html 68 | 69 | [flake8] 70 | exclude = .svn,CVS,.bzr,.hg,.git,__pycache__,.tox,docs,virtualenv_run,.ropeproject 71 | ignore = E501 72 | 73 | [pytest] 74 | norecursedirs = .* virtualenv_run 75 | 76 | -------------------------------------------------------------------------------- /kafka_utils/util/error.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | class KafkaToolError(Exception): 18 | """Base class for kafka tool exceptions.""" 19 | pass 20 | 21 | 22 | class ConfigurationError(KafkaToolError): 23 | """Error in configuration. For example: missing configuration file 24 | or misformatted configuration.""" 25 | pass 26 | 27 | 28 | class MissingConfigurationError(ConfigurationError): 29 | """Missing configuration file.""" 30 | pass 31 | 32 | 33 | class InvalidConfigurationError(ConfigurationError): 34 | """Invalid configuration file.""" 35 | pass 36 | 37 | 38 | class InvalidOffsetStorageError(KafkaToolError): 39 | """Unknown source of offsets.""" 40 | pass 41 | 42 | 43 | class UnknownTopic(KafkaToolError): 44 | """Topic does not exist in kafka.""" 45 | pass 46 | 47 | 48 | class UnknownPartitions(KafkaToolError): 49 | """Partition doesn't exist in kafka.""" 50 | pass 51 | 52 | 53 | class OffsetCommitError(KafkaToolError): 54 | """Error during offset commit.""" 55 | 56 | def __init__(self, topic, partition, error): 57 | self.topic = topic 58 | self.partition = partition 59 | self.error = error 60 | 61 | def __eq__(self, other): 62 | if all([ 63 | self.topic == other.topic, 64 | self.partition == other.partition, 65 | self.error == other.error, 66 | ]): 67 | return True 68 | return False 69 | -------------------------------------------------------------------------------- /kafka_utils/kafka_cluster_manager/cluster_info/topic.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """This class contains information for a topic object. 16 | 17 | Useful as part of reassignment project when deciding upon moving 18 | partitions of same topic over different brokers. 19 | """ 20 | import logging 21 | 22 | 23 | class Topic(object): 24 | """Information of a topic object. 25 | 26 | :params 27 | id: Name of the given topic 28 | replication_factor: replication factor of a given topic 29 | weight: Relative load of topic compared to other topics 30 | For future use 31 | partitions: List of Partition objects 32 | """ 33 | 34 | def __init__(self, id, replication_factor=0, partitions=None, weight=1.0): 35 | self._id = id 36 | self._replication_factor = replication_factor 37 | self._partitions = partitions or set([]) 38 | self.weight = weight 39 | self.log = logging.getLogger(self.__class__.__name__) 40 | 41 | @property 42 | def id(self): 43 | return self._id 44 | 45 | @property 46 | def replication_factor(self): 47 | return self._replication_factor 48 | 49 | @property 50 | def partitions(self): 51 | return self._partitions 52 | 53 | def add_partition(self, partition): 54 | self._partitions.add(partition) 55 | 56 | def __str__(self): 57 | return "{0}".format(self._id) 58 | 59 | def __repr__(self): 60 | return "{0}".format(self) 61 | -------------------------------------------------------------------------------- /tests/acceptance/offset_get.feature: -------------------------------------------------------------------------------- 1 | Feature: kafka_consumer_manager offset_get subcommand 2 | 3 | Scenario: Calling the offset_get command 4 | Given we have an existing kafka cluster with a topic 5 | when we produce some number of messages into the topic 6 | when we consume some number of messages from the topic 7 | when we call the offset_get command 8 | then the correct offset will be shown 9 | 10 | @kafka9 11 | Scenario: Committing offsets into Kafka and fetching offsets with kafka option 12 | Given we have an existing kafka cluster with a topic 13 | Given we have initialized kafka offsets storage 14 | when we commit some offsets for a group into kafka 15 | when we fetch offsets for the group with the kafka option 16 | then the fetched offsets will match the committed offsets 17 | 18 | @kafka9 19 | Scenario: Committing offsets into Kafka and fetching offsets with dual option 20 | Given we have an existing kafka cluster with a topic 21 | Given we have initialized kafka offsets storage 22 | when we commit some offsets for a group into kafka 23 | when we fetch offsets for the group with the dual option 24 | then the fetched offsets will match the committed offsets 25 | 26 | @kafka9 27 | Scenario: Calling the offset_get command with dual storage 28 | Given we have an existing kafka cluster with a topic 29 | Given we have initialized kafka offsets storage 30 | when we produce some number of messages into the topic 31 | when we consume some number of messages from the topic 32 | when we call the offset_set command and commit into kafka 33 | when we call the offset_get command with the dual storage option 34 | then the offset that was committed into Kafka will be shown 35 | 36 | @kafka9 37 | Scenario: Calling the offset_get command with kafka storage 38 | Given we have an existing kafka cluster with a topic 39 | Given we have initialized kafka offsets storage 40 | when we produce some number of messages into the topic 41 | when we consume some number of messages from the topic 42 | when we call the offset_set command and commit into kafka 43 | when we call the offset_get command with kafka storage 44 | then the offset that was committed into Kafka will be shown 45 | -------------------------------------------------------------------------------- /kafka_utils/main.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import print_function 17 | from __future__ import unicode_literals 18 | 19 | import argparse 20 | import logging 21 | 22 | from kafka_utils import __version__ 23 | from kafka_utils.util.config import iter_configurations 24 | 25 | 26 | logging.getLogger().addHandler(logging.NullHandler()) 27 | 28 | 29 | def parse_args(): 30 | """Parse the arguments.""" 31 | parser = argparse.ArgumentParser( 32 | description='Show available clusters.' 33 | ) 34 | parser.add_argument( 35 | '-v', 36 | '--version', 37 | action='version', 38 | version="%(prog)s {0}".format(__version__), 39 | ) 40 | parser.add_argument( 41 | '--discovery-base-path', 42 | dest='discovery_base_path', 43 | type=str, 44 | help='Path of the directory containing the .yaml config.' 45 | ' Default try: ' 46 | '$KAFKA_DISCOVERY_DIR, $HOME/.kafka_discovery, /etc/kafka_discovery', 47 | ) 48 | 49 | return parser.parse_args() 50 | 51 | 52 | def run(): 53 | args = parse_args() 54 | 55 | for config in iter_configurations(args.discovery_base_path): 56 | print("cluster-type {type}:".format(type=config.cluster_type)) 57 | for cluster in config.get_all_clusters(): 58 | print( 59 | "\tcluster-name: {name}\n" 60 | "\tbroker-list: {b_list}\n" 61 | "\tzookeeper: {zk}".format( 62 | name=cluster.name, 63 | b_list=", ".join(cluster.broker_list), 64 | zk=cluster.zookeeper, 65 | ) 66 | ) 67 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import os 16 | 17 | from setuptools import find_packages 18 | from setuptools import setup 19 | 20 | from kafka_utils import __version__ 21 | 22 | 23 | with open( 24 | os.path.join( 25 | os.path.abspath(os.path.dirname(__file__)), 26 | "README.md" 27 | ) 28 | ) as f: 29 | README = f.read() 30 | 31 | 32 | setup( 33 | name="kafka-utils", 34 | version=__version__, 35 | author="Distributed Systems Team", 36 | author_email="team-dist-sys@yelp.com", 37 | description="Kafka management utils", 38 | packages=find_packages(exclude=["scripts", "tests"]), 39 | url="https://github.com/Yelp/kafka-utils", 40 | license="Apache License 2.0", 41 | long_description=README, 42 | keywords="apache kafka", 43 | scripts=[ 44 | "scripts/kafka-consumer-manager", 45 | "scripts/kafka-cluster-manager", 46 | "scripts/kafka-rolling-restart", 47 | "scripts/kafka-utils", 48 | "scripts/kafka-check", 49 | ], 50 | install_requires=[ 51 | "kazoo>=2.0,<3.0.0", 52 | "fabric>=1.8.0,<1.11.0", 53 | "PyYAML<4.0.0", 54 | "requests-futures>0.9.0", 55 | "kafka-python<1.0.0", 56 | "requests<3.0.0", 57 | ], 58 | classifiers=[ 59 | "Development Status :: 4 - Beta", 60 | "License :: OSI Approved :: Apache Software License", 61 | "Programming Language :: Python", 62 | "Programming Language :: Python :: 2.7", 63 | "Environment :: Console", 64 | "Intended Audience :: Developers", 65 | "Intended Audience :: System Administrators", 66 | "Operating System :: POSIX", 67 | "Operating System :: MacOS :: MacOS X", 68 | ], 69 | ) 70 | -------------------------------------------------------------------------------- /kafka_utils/util/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import print_function 16 | 17 | import json 18 | import sys 19 | from itertools import groupby 20 | 21 | 22 | def groupsortby(data, key): 23 | """Sort and group by the same key.""" 24 | return groupby(sorted(data, key=key), key) 25 | 26 | 27 | def dict_merge(set1, set2): 28 | """Joins two dictionaries.""" 29 | return dict(set1.items() + set2.items()) 30 | 31 | 32 | def to_h(num, suffix='B'): 33 | """Converts a byte value in human readable form.""" 34 | if num is None: # Show None when data is missing 35 | return "None" 36 | for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: 37 | if abs(num) < 1024.0: 38 | return "%3.1f%s%s" % (num, unit, suffix) 39 | num /= 1024.0 40 | return "%.1f%s%s" % (num, 'Yi', suffix) 41 | 42 | 43 | def to_int(num): 44 | """ 45 | Converts 'num' to int representation in string 46 | or to "None" in case of None. 47 | """ 48 | if num is None: 49 | return "None" 50 | return "{:.0f}".format(num) 51 | 52 | 53 | def to_float(num): 54 | """ 55 | Converts 'num' to float representation in string 56 | or to "None" in case of None. 57 | """ 58 | if num is None: 59 | return "None" 60 | return "{:.2f}".format(num) 61 | 62 | 63 | def format_to_json(data): 64 | """Converts `data` into json 65 | If stdout is a tty it performs a pretty print. 66 | """ 67 | if sys.stdout.isatty(): 68 | return json.dumps(data, indent=4, separators=(',', ': ')) 69 | else: 70 | return json.dumps(data) 71 | 72 | 73 | def print_json(data): 74 | """Converts `data` into json and prints it to stdout.""" 75 | print(format_to_json(data)) 76 | -------------------------------------------------------------------------------- /tests/acceptance/steps/min_isr.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from behave import then 16 | from behave import when 17 | from util import call_cmd 18 | from util import get_cluster_config 19 | 20 | from kafka_utils.util.zookeeper import ZK 21 | 22 | 23 | ISR_CONF_NAME = 'min.insync.replicas' 24 | CONF_PATH = '/config/topics/' 25 | 26 | 27 | def call_min_isr(): 28 | cmd = ['kafka-check', 29 | '--cluster-type', 'test', 30 | '--cluster-name', 'test_cluster', 31 | '--discovery-base-path', 'tests/acceptance/config', 32 | 'min_isr'] 33 | return call_cmd(cmd) 34 | 35 | 36 | def set_min_isr(topic, min_isr): 37 | cluster_config = get_cluster_config() 38 | with ZK(cluster_config) as zk: 39 | config = zk.get_topic_config(topic) 40 | config[ISR_CONF_NAME] = str(min_isr) 41 | zk.set_topic_config(topic, config) 42 | 43 | 44 | @when(u'we call the min_isr command') 45 | def step_impl2(context): 46 | context.min_isr_out = call_min_isr() 47 | 48 | 49 | @when(u'we change min.isr settings for a topic to 1') 50 | def step_impl3(context): 51 | set_min_isr(context.topic, 1) 52 | 53 | 54 | @when(u'we change min.isr settings for a topic to 2') 55 | def step_impl4(context): 56 | set_min_isr(context.topic, 2) 57 | 58 | 59 | @then(u'OK will be printed') 60 | def step_impl5(context): 61 | assert context.min_isr_out == 'OK: All replicas in sync.\n', context.min_isr_out 62 | 63 | 64 | @then(u'CRITICAL will be printed') 65 | def step_impl6(context): 66 | error_msg = ("isr=1 is lower than min_isr=2 for {topic}:0\n" 67 | "CRITICAL: 1 partition(s) have the number of " 68 | "replicas in sync that is lower than the specified min ISR.\n").format( 69 | topic=context.topic) 70 | assert context.min_isr_out == error_msg, context.min_isr_out 71 | -------------------------------------------------------------------------------- /kafka_utils/util/protocol.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import struct 16 | 17 | from kafka.protocol import KafkaProtocol 18 | from kafka.util import group_by_topic_and_partition 19 | from kafka.util import write_short_string 20 | 21 | 22 | class KafkaToolProtocol(KafkaProtocol): 23 | 24 | @classmethod 25 | def encode_offset_commit_request_kafka(cls, client_id, correlation_id, 26 | group, payloads): 27 | """ 28 | Encode some OffsetCommitRequest structs 29 | Arguments: 30 | client_id: string 31 | correlation_id: int 32 | group: string, the consumer group you are committing offsets for 33 | payloads: list of OffsetCommitRequest 34 | """ 35 | grouped_payloads = group_by_topic_and_partition(payloads) 36 | 37 | message = [] 38 | message.append(cls._encode_message_header( 39 | client_id, correlation_id, 40 | KafkaProtocol.OFFSET_COMMIT_KEY, 41 | version=2)) 42 | message.append(write_short_string(group)) 43 | message.append(struct.pack('>i', -1)) # ConsumerGroupGenerationId 44 | message.append(write_short_string('')) # ConsumerId 45 | message.append(struct.pack('>q', -1)) # Retention time 46 | message.append(struct.pack('>i', len(grouped_payloads))) 47 | 48 | for topic, topic_payloads in grouped_payloads.items(): 49 | message.append(write_short_string(topic)) 50 | message.append(struct.pack('>i', len(topic_payloads))) 51 | 52 | for partition, payload in topic_payloads.items(): 53 | message.append(struct.pack('>iq', partition, payload.offset)) 54 | message.append(write_short_string(payload.metadata)) 55 | 56 | msg = b''.join(message) 57 | return struct.pack('>i%ds' % len(msg), len(msg), msg) 58 | -------------------------------------------------------------------------------- /kafka_utils/kafka_consumer_manager/commands/list_topics.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | 17 | import sys 18 | 19 | from .offset_manager import OffsetManagerBase 20 | from kafka_utils.util.client import KafkaToolClient 21 | 22 | 23 | class ListTopics(OffsetManagerBase): 24 | 25 | @classmethod 26 | def setup_subparser(cls, subparsers): 27 | parser_list_topics = subparsers.add_parser( 28 | "list_topics", 29 | description="List topics by consumer group.", 30 | add_help=False 31 | ) 32 | parser_list_topics.add_argument( 33 | "-h", "--help", action="help", 34 | help="Show this help message and exit." 35 | ) 36 | parser_list_topics.add_argument( 37 | 'groupid', 38 | help="Consumer Group ID whose topics shall be fetched." 39 | ) 40 | parser_list_topics.set_defaults(command=cls.run) 41 | 42 | @classmethod 43 | def run(cls, args, cluster_config): 44 | # Setup the Kafka client 45 | client = KafkaToolClient(cluster_config.broker_list) 46 | client.load_metadata_for_topics() 47 | 48 | topics_dict = cls.preprocess_args( 49 | args.groupid, None, None, 50 | cluster_config, client, 51 | False 52 | ) 53 | if not topics_dict: 54 | print( 55 | "Consumer Group ID: {group} does not exist in " 56 | "Zookeeper".format( 57 | group=args.groupid 58 | ) 59 | ) 60 | sys.exit(1) 61 | 62 | print("Consumer Group ID: {groupid}".format(groupid=args.groupid)) 63 | for topic, partitions in topics_dict.iteritems(): 64 | print("\tTopic: {topic}".format(topic=topic)) 65 | print("\t\tPartitions: {partitions}".format(partitions=partitions)) 66 | -------------------------------------------------------------------------------- /tests/acceptance/steps/offset_save.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import json 16 | import tempfile 17 | 18 | from behave import then 19 | from behave import when 20 | 21 | from .util import call_cmd 22 | 23 | 24 | def create_saved_file(): 25 | return tempfile.NamedTemporaryFile() 26 | 27 | 28 | def call_offset_save(groupid, offsets_file, storage=None): 29 | cmd = ['kafka-consumer-manager', 30 | '--cluster-type', 'test', 31 | '--cluster-name', 'test_cluster', 32 | '--discovery-base-path', 'tests/acceptance/config', 33 | 'offset_save', 34 | groupid, 35 | offsets_file] 36 | if storage: 37 | cmd.extend(['--storage', storage]) 38 | return call_cmd(cmd) 39 | 40 | 41 | @when(u'we call the offset_save command with an offsets file') 42 | def step_impl2(context): 43 | context.offsets_file = create_saved_file() 44 | call_offset_save(context.group, context.offsets_file.name) 45 | 46 | 47 | @when(u'we call the offset_save command with an offsets file and kafka storage') 48 | def step_impl2_2(context): 49 | context.offsets_file = create_saved_file() 50 | call_offset_save(context.group, context.offsets_file.name, storage='kafka') 51 | 52 | 53 | @then(u'the correct offsets will be saved into the given file') 54 | def step_impl3(context): 55 | offsets = context.consumer.offsets(group='commit') 56 | key = (context.topic, 0) 57 | offset = offsets[key] 58 | 59 | data = json.loads(context.offsets_file.read()) 60 | assert offset == data['offsets'][context.topic]['0'] 61 | context.offsets_file.close() 62 | 63 | 64 | @then(u'the restored offsets will be saved into the given file') 65 | def step_impl3_2(context): 66 | offset = context.restored_offset 67 | 68 | data = json.loads(context.offsets_file.read()) 69 | assert offset == data['offsets'][context.topic]['0'] 70 | context.offsets_file.close() 71 | -------------------------------------------------------------------------------- /docs/source/kafka_rolling_restart.rst: -------------------------------------------------------------------------------- 1 | Rolling restart 2 | *************** 3 | 4 | The kafka-rolling-restart script can be used to safely restart an entire 5 | cluster, one server at a time. The script finds all the servers in a cluster, 6 | checks their health status and executes the restart. 7 | 8 | Cluster health 9 | ============== 10 | 11 | The health of the cluster is defined in terms of broker availability and under 12 | replicated partitions. Kafka-rolling-restart will check that all brokers are 13 | answering to JMX requests, and that the total numer of under replicated 14 | partitions is zero. If both conditions are fulfilled, the cluster is considered 15 | healthy and the next broker will be restarted. 16 | 17 | The JMX metrics are accessed via `Jolokia `_, which must be 18 | running on all brokers. 19 | 20 | .. note:: If a broker is not registered in Zookeeper when the tool is executed, 21 | it will not appear in the list of known brokers and it will be ignored. 22 | 23 | Parameters 24 | ========== 25 | 26 | The parameters specific for kafka-rolling-restart are: 27 | 28 | * :code:`--check-interval INTERVAL`: the number of seconds between each check. 29 | Default 10. 30 | * :code:`--check-count COUNT`: the number of consecutive checks that must result 31 | in cluster healthy before restarting the next server. Default 12. 32 | * :code:`--unhealthy-time-limit LIMIT`: the maximum time in seconds that a 33 | cluster can be unhealthy for. If the limit is reached, the script will 34 | terminate with an error. Default 600. 35 | * :code:`--jolokia-port PORT`: The Jolokia port. Default 8778. 36 | * :code:`--jolokia-prefix PREFIX`: The Jolokia prefix. Default "jolokia/". 37 | * :code:`--no-confirm`: If specified, the script will not ask for confirmation. 38 | * :code:`--skip N`: Skip the first N servers. Useful to recover from a partial 39 | rolling restart. Default 0. 40 | * :code:`--verbose`: Turn on verbose output. 41 | 42 | Examples 43 | ======== 44 | 45 | Restart the generic dev cluster, checking the JXM metrics every 30 seconds, and 46 | restarting the next broker after 5 consecutive checks have confirmed the health 47 | of the cluster: 48 | 49 | .. code-block:: bash 50 | 51 | kafka-rolling-restart --cluster-type generic --cluster-name dev --check-interval 30 --check-count 5 52 | 53 | Check the generic prod cluster. It will report an error if the cluster is 54 | unhealthy for more than 900 seconds: 55 | 56 | .. code-block:: bash 57 | 58 | kafka-rolling-restart --cluster-type generic --cluster-name prod --unhealthy-time-limit 900 59 | -------------------------------------------------------------------------------- /tests/kafka_consumer_manager/test_offset_get.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import mock 16 | import pytest 17 | 18 | from kafka_utils.kafka_consumer_manager. \ 19 | commands.offset_get import OffsetGet 20 | 21 | 22 | class TestOffsetGet(object): 23 | 24 | @pytest.yield_fixture 25 | def client(self): 26 | with mock.patch( 27 | 'kafka_utils.kafka_consumer_manager.' 28 | 'commands.offset_get.KafkaToolClient', 29 | autospec=True, 30 | ) as mock_client: 31 | yield mock_client 32 | 33 | def test_get_offsets(self, client): 34 | consumer_group = 'group1' 35 | topics = {'topic1': {0: 100}} 36 | 37 | with mock.patch( 38 | 'kafka_utils.util.offsets._verify_topics_and_partitions', 39 | return_value=topics, 40 | autospec=True, 41 | ): 42 | OffsetGet.get_offsets( 43 | client, 44 | consumer_group, 45 | topics, 46 | 'zookeeper', 47 | ) 48 | 49 | assert client.load_metadata_for_topics.call_count == 1 50 | assert client.send_offset_fetch_request.call_count == 1 51 | assert client.send_offset_fetch_request_kafka.call_count == 0 52 | 53 | def test_get_offsets_kafka(self, client): 54 | consumer_group = 'group1' 55 | topics = {'topic1': {0: 100}} 56 | 57 | with mock.patch( 58 | 'kafka_utils.util.offsets._verify_topics_and_partitions', 59 | return_value=topics, 60 | autospec=True, 61 | ): 62 | OffsetGet.get_offsets( 63 | client, 64 | consumer_group, 65 | topics, 66 | 'kafka', 67 | ) 68 | 69 | assert client.load_metadata_for_topics.call_count == 1 70 | assert client.send_offset_fetch_request.call_count == 0 71 | assert client.send_offset_fetch_request_kafka.call_count == 1 72 | -------------------------------------------------------------------------------- /tests/acceptance/steps/dual_commit_fetch.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from behave import then 16 | from behave import when 17 | 18 | from .util import create_random_group_id 19 | from .util import get_cluster_config 20 | from kafka_utils.util.client import KafkaToolClient 21 | from kafka_utils.util.monitoring import get_current_offsets 22 | from kafka_utils.util.offsets import set_consumer_offsets 23 | 24 | 25 | TEST_OFFSET = 56 26 | 27 | 28 | def commit_offsets(offsets, group, storage): 29 | # Setup the Kafka client 30 | config = get_cluster_config() 31 | client = KafkaToolClient(config.broker_list) 32 | set_consumer_offsets( 33 | client, 34 | group, 35 | offsets, 36 | offset_storage=storage, 37 | ) 38 | client.close() 39 | 40 | 41 | def fetch_offsets(group, topics, storage): 42 | # Setup the Kafka client 43 | config = get_cluster_config() 44 | client = KafkaToolClient(config.broker_list) 45 | offsets = get_current_offsets(client, group, topics, False, storage) 46 | client.close() 47 | return offsets 48 | 49 | 50 | @when(u'we commit some offsets for a group into kafka') 51 | def step_impl4(context): 52 | context.offsets = {context.topic: {0: TEST_OFFSET}} 53 | context.group = create_random_group_id() 54 | commit_offsets(context.offsets, context.group, 'kafka') 55 | 56 | 57 | @when(u'we fetch offsets for the group with the dual option') 58 | def step_impl4_2(context): 59 | topics = context.offsets.keys() 60 | context.fetched_offsets = fetch_offsets( 61 | context.group, 62 | topics, 63 | 'dual' 64 | ) 65 | 66 | 67 | @when(u'we fetch offsets for the group with the kafka option') 68 | def step_impl4_3(context): 69 | topics = context.offsets.keys() 70 | context.fetched_offsets = fetch_offsets( 71 | context.group, 72 | topics, 73 | 'kafka' 74 | ) 75 | 76 | 77 | @then(u'the fetched offsets will match the committed offsets') 78 | def step_impl5(context): 79 | assert context.fetched_offsets == context.offsets 80 | -------------------------------------------------------------------------------- /tests/acceptance/steps/offset_advance.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from behave import then 16 | from behave import when 17 | 18 | from .util import call_cmd 19 | from .util import get_cluster_config 20 | from kafka_utils.util.zookeeper import ZK 21 | 22 | 23 | def call_offset_advance(groupid, topic=None, storage=None, force=False): 24 | cmd = ['kafka-consumer-manager', 25 | '--cluster-type', 'test', 26 | '--cluster-name', 'test_cluster', 27 | '--discovery-base-path', 'tests/acceptance/config', 28 | 'offset_advance', 29 | groupid] 30 | if topic: 31 | cmd.extend(['--topic', topic]) 32 | if storage: 33 | cmd.extend(['--storage', storage]) 34 | if force: 35 | cmd.extend(['--force']) 36 | return call_cmd(cmd) 37 | 38 | 39 | @when(u'we call the offset_advance command with a groupid and topic') 40 | def step_impl3(context): 41 | call_offset_advance(context.group) 42 | 43 | 44 | @when(u'we call the offset_advance command and commit into kafka') 45 | def step_impl3_2(context): 46 | call_offset_advance( 47 | context.group, 48 | topic=context.topic, 49 | storage='kafka', 50 | force=True, 51 | ) 52 | 53 | 54 | @when(u'we call the offset_advance command with a new groupid and the force option') 55 | def step_impl2(context): 56 | context.group = 'offset_advance_test_group' 57 | call_offset_advance( 58 | context.group, 59 | topic=context.topic, 60 | force=True, 61 | ) 62 | 63 | 64 | @then(u'the committed offsets will match the latest message offsets') 65 | def step_impl4(context): 66 | cluster_config = get_cluster_config() 67 | with ZK(cluster_config) as zk: 68 | offsets = zk.get_group_offsets(context.group) 69 | assert offsets[context.topic]["0"] == context.msgs_produced 70 | 71 | 72 | @then(u'the latest message offsets will be shown') 73 | def step_impl5_2(context): 74 | offset = context.msgs_produced 75 | pattern = 'Current Offset: {}'.format(offset) 76 | assert pattern in context.output 77 | -------------------------------------------------------------------------------- /tests/acceptance/steps/offset_rewind.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from behave import then 16 | from behave import when 17 | 18 | from .util import call_cmd 19 | from .util import get_cluster_config 20 | from kafka_utils.util.zookeeper import ZK 21 | 22 | 23 | def offsets_data(topic, offset): 24 | return '''{topic}.{partition}={offset}'''.format( 25 | topic=topic, 26 | partition='0', 27 | offset=offset, 28 | ) 29 | 30 | 31 | def call_offset_rewind(groupid, topic, storage=None, force=False): 32 | cmd = ['kafka-consumer-manager', 33 | '--cluster-type', 'test', 34 | '--cluster-name', 'test_cluster', 35 | '--discovery-base-path', 'tests/acceptance/config', 36 | 'offset_rewind', 37 | groupid, 38 | '--topic', topic] 39 | if storage: 40 | cmd.extend(['--storage', storage]) 41 | if force: 42 | cmd.extend(['--force']) 43 | return call_cmd(cmd) 44 | 45 | 46 | @when(u'we call the offset_rewind command with a groupid and topic') 47 | def step_impl3(context): 48 | call_offset_rewind(context.group, context.topic) 49 | 50 | 51 | @when(u'we call the offset_rewind command and commit into kafka') 52 | def step_impl3_2(context): 53 | call_offset_rewind(context.group, context.topic, storage='kafka') 54 | 55 | 56 | @when(u'we call the offset_rewind command with a new groupid and the force option') 57 | def step_impl2(context): 58 | context.group = 'offset_advance_created_group' 59 | call_offset_rewind( 60 | context.group, 61 | topic=context.topic, 62 | force=True, 63 | ) 64 | 65 | 66 | @then(u'the committed offsets will match the earliest message offsets') 67 | def step_impl4(context): 68 | cluster_config = get_cluster_config() 69 | with ZK(cluster_config) as zk: 70 | offsets = zk.get_group_offsets(context.group) 71 | assert offsets[context.topic]["0"] == 0 72 | 73 | 74 | @then(u'the earliest message offsets will be shown') 75 | def step_impl5_2(context): 76 | offset = 0 77 | pattern = 'Current Offset: {}'.format(offset) 78 | assert pattern in context.output 79 | -------------------------------------------------------------------------------- /tests/kafka_consumer_manager/test_offset_advance.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import mock 16 | import pytest 17 | 18 | from kafka_utils.kafka_consumer_manager.commands. \ 19 | offset_advance import OffsetAdvance 20 | 21 | 22 | @mock.patch('kafka_utils.kafka_consumer_manager.' 23 | 'commands.offset_advance.KafkaToolClient') 24 | class TestOffsetAdvance(object): 25 | topics_partitions = { 26 | "topic1": [0, 1, 2], 27 | "topic2": [0, 1] 28 | } 29 | 30 | def test_run(self, mock_client): 31 | with mock.patch.object( 32 | OffsetAdvance, 33 | "preprocess_args", 34 | spec=OffsetAdvance.preprocess_args, 35 | return_value=self.topics_partitions, 36 | ), mock.patch( 37 | "kafka_utils.kafka_consumer_manager." 38 | "commands.offset_advance.advance_consumer_offsets", 39 | autospec=True 40 | ) as mock_advance: 41 | args = mock.Mock( 42 | groupid="some_group", 43 | topic=None, 44 | partitions=None, 45 | ) 46 | cluster_config = mock.Mock() 47 | OffsetAdvance.run(args, cluster_config) 48 | 49 | ordered_args, _ = mock_advance.call_args 50 | assert ordered_args[1] == args.groupid 51 | assert ordered_args[2] == self.topics_partitions 52 | mock_client.return_value.load_metadata_for_topics. \ 53 | assert_called_once_with() 54 | mock_client.return_value.close.assert_called_once_with() 55 | 56 | def test_run_type_error(self, mock_client): 57 | with mock.patch.object( 58 | OffsetAdvance, 59 | "preprocess_args", 60 | spec=OffsetAdvance.preprocess_args, 61 | return_value="some_string", 62 | ): 63 | args = mock.Mock( 64 | groupid="some_group", 65 | topic=None, 66 | partitions=None 67 | ) 68 | cluster_config = mock.Mock() 69 | with pytest.raises(TypeError): 70 | OffsetAdvance.run(args, cluster_config) 71 | -------------------------------------------------------------------------------- /kafka_utils/kafka_cluster_manager/cmds/store_assignments.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import json 16 | import logging 17 | 18 | from .command import ClusterManagerCmd 19 | from kafka_utils.util.validation import assignment_to_plan 20 | 21 | 22 | class StoreAssignmentsCmd(ClusterManagerCmd): 23 | 24 | def __init__(self): 25 | super(StoreAssignmentsCmd, self).__init__() 26 | self.log = logging.getLogger(self.__class__.__name__) 27 | 28 | def build_subparser(self, subparsers): 29 | subparser = subparsers.add_parser( 30 | 'store_assignments', 31 | description='Emit json encoding the current assignment of ' 32 | 'partitions to replicas.', 33 | help='''This command will not mutate the cluster\'s state. 34 | Output json is of this form: 35 | {"version":1,"partitions":[ 36 | {"topic": "foo1", "partition": 2, "replicas": [1, 2]}, 37 | {"topic": "foo1", "partition": 0, "replicas": [3, 4]}, 38 | {"topic": "foo2", "partition": 2, "replicas": [1, 2]}, 39 | {"topic": "foo2", "partition": 0, "replicas": [3, 4]}, 40 | {"topic": "foo1", "partition": 1, "replicas": [2, 3]}, 41 | {"topic": "foo2", "partition": 1, "replicas": [2, 3]}]}''' 42 | ) 43 | subparser.add_argument( 44 | '--json_out', 45 | type=str, 46 | help=('Path to json output file. ' 47 | 'Will output to stdout if not set. ' 48 | 'If file exists already, it will be clobbered.') 49 | ) 50 | return subparser 51 | 52 | def run_command(self, ct): 53 | plan_json = json.dumps(assignment_to_plan(ct.assignment)) 54 | if self.args.json_out: 55 | with open(self.args.json_out, 'w') as f: 56 | self.log.info( 57 | 'writing assignments as json to: %s', 58 | self.args.json_out, 59 | ) 60 | f.write(plan_json) 61 | else: 62 | self.log.info('writing assignments as json to stdout') 63 | print plan_json 64 | -------------------------------------------------------------------------------- /tests/kafka_cluster_manager/util_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import pytest 16 | 17 | from kafka_utils.kafka_cluster_manager.cluster_info.util import compute_optimum 18 | from kafka_utils.kafka_cluster_manager.cluster_info.util import separate_groups 19 | 20 | 21 | def test_compute_optimum(): 22 | optimal, extra = compute_optimum(3, 10) 23 | 24 | assert optimal == 3 25 | assert extra == 1 26 | 27 | 28 | def test_compute_optimum_zero_groups(): 29 | with pytest.raises(ZeroDivisionError): 30 | optimal, extra = compute_optimum(0, 10) 31 | 32 | 33 | def test_compute_optimum_zero_elements(): 34 | optimal, extra = compute_optimum(10, 0) 35 | 36 | assert optimal == 0 37 | assert extra == 0 38 | 39 | 40 | def test_separate_groups_balanced(): 41 | groups = [4, 4, 4] 42 | total = 12 43 | 44 | overloaded, underloaded = separate_groups(groups, lambda x: x, total) 45 | 46 | assert not overloaded 47 | assert not underloaded 48 | 49 | 50 | def test_separate_groups_almost_balanced(): 51 | groups = [5, 5, 4] 52 | total = 14 53 | 54 | overloaded, underloaded = separate_groups(groups, lambda x: x, total) 55 | 56 | assert not overloaded 57 | assert not underloaded 58 | 59 | 60 | def test_separate_groups_unbalanced(): 61 | groups = [4, 4, 3, 2] 62 | total = 13 63 | 64 | overloaded, underloaded = separate_groups(groups, lambda x: x, total) 65 | 66 | assert overloaded == [4, 4] 67 | assert underloaded == [2] 68 | 69 | 70 | def test_separate_groups_balanced_greater_total(): 71 | groups = [4, 4, 4] 72 | total = 13 73 | 74 | overloaded, underloaded = separate_groups(groups, lambda x: x, total) 75 | 76 | assert underloaded == [4] 77 | 78 | 79 | def test_separate_groups_balanced_much_greater_total(): 80 | groups = [4, 4, 4] 81 | total = 20 82 | 83 | overloaded, underloaded = separate_groups(groups, lambda x: x, total) 84 | 85 | assert underloaded == [4, 4, 4] 86 | 87 | 88 | def test_separate_groups_balanced_smaller_total(): 89 | groups = [4, 4, 4] 90 | total = 6 91 | 92 | overloaded, underloaded = separate_groups(groups, lambda x: x, total) 93 | 94 | assert overloaded == [4, 4, 4] 95 | -------------------------------------------------------------------------------- /tests/acceptance/steps/offset_restore.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import os 16 | import tempfile 17 | 18 | from behave import given 19 | from behave import then 20 | from behave import when 21 | 22 | from .util import call_cmd 23 | from .util import get_cluster_config 24 | from kafka_utils.util.zookeeper import ZK 25 | 26 | 27 | RESTORED_OFFSET = 55 28 | 29 | 30 | def create_restore_file(group, topic, offset): 31 | offset_restore_data = ''' 32 | {{ 33 | "groupid": "{group}", 34 | "offsets": {{ 35 | "{topic}": {{ 36 | "0": {offset} 37 | }} 38 | }} 39 | }} 40 | '''.format(group=group, topic=topic, offset=offset) 41 | 42 | f = tempfile.NamedTemporaryFile() 43 | f.write(offset_restore_data) 44 | f.flush() 45 | return f 46 | 47 | 48 | def call_offset_restore(offsets_file, storage=None): 49 | cmd = ['kafka-consumer-manager', 50 | '--cluster-type', 'test', 51 | '--cluster-name', 'test_cluster', 52 | '--discovery-base-path', 'tests/acceptance/config', 53 | 'offset_restore', 54 | offsets_file] 55 | if storage: 56 | cmd.extend(['--storage', storage]) 57 | return call_cmd(cmd) 58 | 59 | 60 | @given(u'we have a json offsets file') 61 | def step_impl2(context): 62 | context.restored_offset = RESTORED_OFFSET 63 | context.offsets_file = create_restore_file( 64 | context.group, 65 | context.topic, 66 | context.restored_offset, 67 | ) 68 | assert os.path.isfile(context.offsets_file.name) 69 | 70 | 71 | @when(u'we call the offset_restore command with the offsets file') 72 | def step_impl3(context): 73 | call_offset_restore(context.offsets_file.name) 74 | 75 | 76 | @when(u'we call the offset_restore command with the offsets file and kafka storage') 77 | def step_impl3_2(context): 78 | call_offset_restore(context.offsets_file.name, storage='kafka') 79 | 80 | 81 | @then(u'the committed offsets will match the offsets file') 82 | def step_impl4(context): 83 | cluster_config = get_cluster_config() 84 | with ZK(cluster_config) as zk: 85 | offsets = zk.get_group_offsets(context.group) 86 | assert offsets[context.topic]["0"] == RESTORED_OFFSET 87 | context.offsets_file.close() 88 | -------------------------------------------------------------------------------- /tests/acceptance/steps/offset_set.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from behave import then 16 | from behave import when 17 | 18 | from .util import call_cmd 19 | from .util import get_cluster_config 20 | from kafka_utils.util.zookeeper import ZK 21 | 22 | 23 | SET_OFFSET = 36 24 | SET_OFFSET_KAFKA = 65 25 | 26 | 27 | def offsets_data(topic, offset): 28 | return '''{topic}.{partition}={offset}'''.format( 29 | topic=topic, 30 | partition='0', 31 | offset=offset, 32 | ) 33 | 34 | 35 | def call_offset_set(groupid, offsets_data, storage=None, force=False): 36 | cmd = ['kafka-consumer-manager', 37 | '--cluster-type', 'test', 38 | '--cluster-name', 'test_cluster', 39 | '--discovery-base-path', 'tests/acceptance/config', 40 | 'offset_set', 41 | groupid, 42 | offsets_data] 43 | if storage: 44 | cmd.extend(['--storage', storage]) 45 | if force: 46 | cmd.extend(['--force']) 47 | return call_cmd(cmd) 48 | 49 | 50 | @when(u'we call the offset_set command with a groupid and offset data') 51 | def step_impl2(context): 52 | context.offsets = offsets_data(context.topic, SET_OFFSET) 53 | call_offset_set(context.group, context.offsets) 54 | 55 | 56 | @when(u'we call the offset_set command and commit into kafka') 57 | def step_impl2_2(context): 58 | if not hasattr(context, 'group'): 59 | context.group = 'test_kafka_offset_group' 60 | context.offsets = offsets_data(context.topic, SET_OFFSET_KAFKA) 61 | context.set_offset_kafka = SET_OFFSET_KAFKA 62 | call_offset_set(context.group, context.offsets, storage='kafka') 63 | 64 | 65 | @when(u'we call the offset_set command with a new groupid and the force option') 66 | def step_impl2_3(context): 67 | context.offsets = offsets_data(context.topic, SET_OFFSET) 68 | context.group = 'offset_set_created_group' 69 | call_offset_set(context.group, context.offsets, force=True) 70 | 71 | 72 | @then(u'the committed offsets will match the specified offsets') 73 | def step_impl3(context): 74 | cluster_config = get_cluster_config() 75 | with ZK(cluster_config) as zk: 76 | offsets = zk.get_group_offsets(context.group) 77 | assert offsets[context.topic]["0"] == SET_OFFSET 78 | -------------------------------------------------------------------------------- /tests/kafka_consumer_manager/test_offset_rewind.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import mock 16 | import pytest 17 | 18 | from kafka_utils.kafka_consumer_manager. \ 19 | commands.offset_rewind import OffsetRewind 20 | 21 | 22 | class TestOffsetRewind(object): 23 | topics_partitions = { 24 | "topic1": [0, 1, 2], 25 | "topic2": [0, 1] 26 | } 27 | 28 | @mock.patch('kafka_utils.kafka_consumer_manager.' 29 | 'commands.offset_rewind.KafkaToolClient') 30 | def test_run(self, mock_client): 31 | with mock.patch.object( 32 | OffsetRewind, 33 | "preprocess_args", 34 | spec=OffsetRewind.preprocess_args, 35 | return_value=self.topics_partitions, 36 | ), mock.patch( 37 | "kafka_utils.kafka_consumer_manager." 38 | "commands.offset_rewind.rewind_consumer_offsets", 39 | autospec=True 40 | ) as mock_rewind: 41 | args = mock.Mock( 42 | groupid="some_group", 43 | topic=None, 44 | partitions=None 45 | ) 46 | cluster_config = mock.Mock() 47 | OffsetRewind.run(args, cluster_config) 48 | 49 | ordered_args, _ = mock_rewind.call_args 50 | assert ordered_args[1] == args.groupid 51 | assert ordered_args[2] == self.topics_partitions 52 | mock_client.return_value.load_metadata_for_topics. \ 53 | assert_called_once_with() 54 | mock_client.return_value.close.assert_called_once_with() 55 | 56 | @mock.patch('kafka_utils.kafka_consumer_manager' 57 | '.commands.offset_rewind.KafkaToolClient') 58 | def test_run_type_error(self, mock_client): 59 | with mock.patch.object( 60 | OffsetRewind, 61 | "preprocess_args", 62 | spec=OffsetRewind.preprocess_args, 63 | return_value="some_string", 64 | ): 65 | args = mock.Mock( 66 | groupid="some_group", 67 | topic=None, 68 | partitions=None 69 | ) 70 | cluster_config = mock.Mock() 71 | with pytest.raises(TypeError): 72 | OffsetRewind.run(args, cluster_config) 73 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/Yelp/kafka-utils.svg?branch=master)](https://travis-ci.org/Yelp/kafka-utils) 2 | 3 | # Kafka-Utils 4 | 5 | A suite of python tools to interact and manage Apache Kafka clusters. 6 | Kafka-Utils currently runs on python2.7. 7 | 8 | ## Configuration 9 | 10 | Kafka-Utils reads cluster configuration needed to access Kafka clusters from yaml files. Each cluster is identified by *type* and *name*. 11 | Multiple clusters of the same type should be listed in the same `type.yaml` file. 12 | The yaml files are read from `$KAFKA_DISCOVERY_DIR`, `$HOME/.kafka_discovery` and `/etc/kafka_discovery`, the former overrides the latter. 13 | 14 | 15 | Sample configuration for `sample_type` cluster at `/etc/kafka_discovery/sample_type.yaml` 16 | 17 | ```yaml 18 | --- 19 | clusters: 20 | cluster-1: 21 | broker_list: 22 | - "cluster-elb-1:9092" 23 | zookeeper: "11.11.11.111:2181,11.11.11.112:2181,11.11.11.113:2181/kafka-1" 24 | cluster-2: 25 | broker_list: 26 | - "cluster-elb-2:9092" 27 | zookeeper: "11.11.11.211:2181,11.11.11.212:2181,11.11.11.213:2181/kafka-2" 28 | local_config: 29 | cluster: cluster-1 30 | ``` 31 | 32 | ## Install 33 | 34 | From PyPI: 35 | ```shell 36 | $ pip install kafka-utils 37 | ``` 38 | 39 | 40 | ## Kafka-Utils command-line interface 41 | 42 | ### List all clusters 43 | 44 | ```shell 45 | $ kafka-utils 46 | cluster-type sample_type: 47 | cluster-name: cluster-1 48 | broker-list: cluster-elb-1:9092 49 | zookeeper: 11.11.11.111:2181,11.11.11.112:2181,11.11.11.113:2181/kafka-1 50 | cluster-name: cluster-2 51 | broker-list: cluster-elb-2:9092 52 | zookeeper: 11.11.11.211:2181,11.11.11.212:2181,11.11.11.213:2181/kafka-2 53 | ``` 54 | 55 | ### Get consumer offsets 56 | 57 | ```shell 58 | $ kafka-consumer-manager --cluster-type sample_type offset_get sample_consumer 59 | ``` 60 | 61 | ### Rebalance cluster cluster1 of type sample_cluster 62 | 63 | ```shell 64 | $ kafka-cluster-manager --cluster-type sample_type --cluster-name cluster1 65 | --apply rebalance --brokers --leaders --max-partition-movements 10 66 | --max-leader-changes 15 67 | ``` 68 | 69 | ### Rolling-restart a cluster 70 | 71 | ```shell 72 | $ kafka-rolling-restart --cluster-type sample_type 73 | ``` 74 | 75 | ### Check in-sync replicas 76 | 77 | ```shell 78 | $ kafka-check --cluster-type sample_type min_isr 79 | ``` 80 | 81 | ## Documentation 82 | 83 | Read the documentation at [Read the Docs](http://kafka-utils.readthedocs.io/en/latest/). 84 | 85 | ## License 86 | 87 | Kafka-Utils is licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 88 | 89 | ## Contributing 90 | 91 | Everyone is encouraged to contribute to Kafka-Utils by forking the 92 | [Github repository](http://github.com/Yelp/kafka-utils) and making a pull request or opening an issue. 93 | -------------------------------------------------------------------------------- /tests/util/test_protocol.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import struct 16 | 17 | from kafka.common import OffsetCommitRequest 18 | 19 | from kafka_utils.util.protocol import KafkaToolProtocol 20 | 21 | 22 | class TestProtocol(object): 23 | 24 | def test_encode_offset_commit_request_kafka(self): 25 | 26 | header = b"".join([ 27 | struct.pack('>i', 113), # Total message length 28 | 29 | struct.pack('>h', 8), # Message type = offset commit 30 | struct.pack('>h', 2), # API version 31 | struct.pack('>i', 42), # Correlation ID 32 | struct.pack('>h9s', 9, b"client_id"), # The client ID 33 | struct.pack('>h8s', 8, b"group_id"), # The group to commit for 34 | struct.pack('>i', -1), # Consumer group generation id 35 | struct.pack(">h0s", 0, b""), # Consumer id 36 | struct.pack('>q', -1), # Retention time 37 | struct.pack('>i', 2), # Num topics 38 | ]) 39 | 40 | topic1 = b"".join([ 41 | struct.pack(">h6s", 6, b"topic1"), # Topic for the request 42 | struct.pack(">i", 2), # Two partitions 43 | struct.pack(">i", 0), # Partition 0 44 | struct.pack(">q", 123), # Offset 123 45 | struct.pack(">h", -1), # Null metadata 46 | struct.pack(">i", 1), # Partition 1 47 | struct.pack(">q", 234), # Offset 234 48 | struct.pack(">h", -1), # Null metadata 49 | ]) 50 | 51 | topic2 = b"".join([ 52 | struct.pack(">h6s", 6, b"topic2"), # Topic for the request 53 | struct.pack(">i", 1), # One partition 54 | struct.pack(">i", 2), # Partition 2 55 | struct.pack(">q", 345), # Offset 345 56 | struct.pack(">h", -1), # Null metadata 57 | ]) 58 | 59 | expected1 = b"".join([header, topic1, topic2]) 60 | expected2 = b"".join([header, topic2, topic1]) 61 | 62 | encoded = KafkaToolProtocol.encode_offset_commit_request_kafka(b"client_id", 42, b"group_id", [ 63 | OffsetCommitRequest(b"topic1", 0, 123, None), 64 | OffsetCommitRequest(b"topic1", 1, 234, None), 65 | OffsetCommitRequest(b"topic2", 2, 345, None), 66 | ]) 67 | 68 | assert encoded in [expected1, expected2] 69 | -------------------------------------------------------------------------------- /kafka_utils/kafka_check/commands/min_isr.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import print_function 17 | 18 | from kafka_utils.kafka_check import status_code 19 | from kafka_utils.kafka_check.commands.command import KafkaCheckCmd 20 | 21 | 22 | class MinIsrCmd(KafkaCheckCmd): 23 | 24 | def build_subparser(self, subparsers): 25 | subparser = subparsers.add_parser( 26 | 'min_isr', 27 | description='Check min isr number for each topic in the cluster.', 28 | help='This command will check actual number of insync replicas for each ' 29 | 'topic-partition in the cluster with configuration for that topic ' 30 | 'in Zookeeper or default min.isr param if it is specified and there ' 31 | 'is no settings in Zookeeper for partition.', 32 | ) 33 | subparser.add_argument( 34 | '--default_min_isr', 35 | type=int, 36 | default=1, 37 | help='Default min.isr value for cases without settings in Zookeeper ' 38 | 'for some topics. Default: %(default)s', 39 | ) 40 | return subparser 41 | 42 | def get_min_isr(self, topic): 43 | """Return the min-isr for topic, or None if not specified""" 44 | ISR_CONF_NAME = 'min.insync.replicas' 45 | config = self.zk.get_topic_config(topic) 46 | if ISR_CONF_NAME in config: 47 | return int(config[ISR_CONF_NAME]) 48 | else: 49 | return None 50 | 51 | def run_command(self): 52 | """Min_isr command, checks number of actual min-isr 53 | for each topic-partition with configuration for that topic.""" 54 | topics = self.zk.get_topics() 55 | not_in_sync = 0 56 | 57 | for name, topic_data in topics.items(): 58 | min_isr = self.get_min_isr(name) or self.args.default_min_isr 59 | if min_isr is None: 60 | continue 61 | for p_id, partition in topic_data['partitions'].items(): 62 | cur_isr = len(partition['isr']) 63 | if cur_isr < min_isr: 64 | print("isr={isr} is lower than min_isr={min_isr} for {topic}:{partition}".format( 65 | isr=cur_isr, min_isr=min_isr, topic=name, partition=p_id, 66 | )) 67 | not_in_sync += 1 68 | if not_in_sync == 0: 69 | return status_code.OK, "All replicas in sync." 70 | else: 71 | msg = ("{0} partition(s) have the number of replicas in " 72 | "sync that is lower than the specified min ISR.").format(not_in_sync) 73 | return status_code.CRITICAL, msg 74 | -------------------------------------------------------------------------------- /kafka_utils/kafka_consumer_manager/commands/delete_group.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import print_function 17 | 18 | import sys 19 | 20 | from .offset_manager import OffsetWriter 21 | from kafka_utils.util.client import KafkaToolClient 22 | from kafka_utils.util.offsets import nullify_offsets 23 | from kafka_utils.util.offsets import set_consumer_offsets 24 | from kafka_utils.util.zookeeper import ZK 25 | 26 | 27 | class DeleteGroup(OffsetWriter): 28 | 29 | @classmethod 30 | def setup_subparser(cls, subparsers): 31 | parser_delete_group = subparsers.add_parser( 32 | "delete_group", 33 | description="Delete a consumer group by groupid. This " 34 | "tool shall delete all group offset metadata from Zookeeper.", 35 | add_help=False 36 | ) 37 | parser_delete_group.add_argument( 38 | "-h", "--help", action="help", 39 | help="Show this help message and exit." 40 | ) 41 | parser_delete_group.add_argument( 42 | 'groupid', 43 | help="Consumer Group IDs whose metadata shall be deleted." 44 | ) 45 | parser_delete_group.add_argument( 46 | '--storage', choices=['zookeeper', 'kafka'], 47 | help="String describing where to store the committed offsets.", 48 | ) 49 | parser_delete_group.set_defaults(command=cls.run) 50 | 51 | @classmethod 52 | def run(cls, args, cluster_config): 53 | # Setup the Kafka client 54 | client = KafkaToolClient(cluster_config.broker_list) 55 | client.load_metadata_for_topics() 56 | 57 | topics_dict = cls.preprocess_args( 58 | args.groupid, None, None, cluster_config, client 59 | ) 60 | if not args.storage or args.storage == 'zookeeper': 61 | cls.delete_group_zk(cluster_config, args.groupid) 62 | elif args.storage == 'kafka': 63 | cls.delete_group_kafka(client, args.groupid, topics_dict) 64 | else: 65 | print( 66 | "Error: Invalid offset storage option: " 67 | "{}.".format(args.storage), 68 | file=sys.stderr, 69 | ) 70 | sys.exit(1) 71 | 72 | @classmethod 73 | def delete_group_zk(cls, cluster_config, group): 74 | with ZK(cluster_config) as zk: 75 | zk.delete_group(group) 76 | 77 | @classmethod 78 | def delete_group_kafka(cls, client, group, topics): 79 | new_offsets = nullify_offsets(topics) 80 | set_consumer_offsets( 81 | client, 82 | group, 83 | new_offsets, 84 | offset_storage='kafka', 85 | ) 86 | -------------------------------------------------------------------------------- /kafka_utils/kafka_consumer_manager/main.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import print_function 17 | from __future__ import unicode_literals 18 | 19 | import argparse 20 | import logging 21 | import sys 22 | 23 | from .commands.copy_group import CopyGroup 24 | from .commands.delete_group import DeleteGroup 25 | from .commands.list_groups import ListGroups 26 | from .commands.list_topics import ListTopics 27 | from .commands.offset_advance import OffsetAdvance 28 | from .commands.offset_get import OffsetGet 29 | from .commands.offset_restore import OffsetRestore 30 | from .commands.offset_rewind import OffsetRewind 31 | from .commands.offset_save import OffsetSave 32 | from .commands.offset_set import OffsetSet 33 | from .commands.rename_group import RenameGroup 34 | from .commands.unsubscribe_topics import UnsubscribeTopics 35 | from kafka_utils.util.config import get_cluster_config 36 | from kafka_utils.util.error import ConfigurationError 37 | 38 | 39 | def parse_args(): 40 | parser = argparse.ArgumentParser( 41 | description="kafka-consumer-manager provides to ability to view and " 42 | "manipulate consumer offsets for a specific consumer group.", 43 | ) 44 | parser.add_argument( 45 | '--cluster-type', dest='cluster_type', required=True, 46 | help='Type of Kafka cluster. This is a mandatory option.', 47 | ) 48 | parser.add_argument( 49 | '--cluster-name', dest='cluster_name', 50 | help='Kafka Cluster Name. If not specified, this defaults to the ' 51 | 'local cluster.', 52 | ) 53 | parser.add_argument( 54 | '--discovery-base-path', 55 | dest='discovery_base_path', 56 | type=str, 57 | help='Path of the directory containing the .yaml config', 58 | ) 59 | subparsers = parser.add_subparsers() 60 | 61 | OffsetGet.add_parser(subparsers) 62 | OffsetSave.add_parser(subparsers) 63 | OffsetSet.add_parser(subparsers) 64 | OffsetAdvance.add_parser(subparsers) 65 | OffsetRewind.add_parser(subparsers) 66 | ListTopics.add_parser(subparsers) 67 | ListGroups.add_parser(subparsers) 68 | UnsubscribeTopics.add_parser(subparsers) 69 | CopyGroup.add_parser(subparsers) 70 | DeleteGroup.add_parser(subparsers) 71 | RenameGroup.add_parser(subparsers) 72 | OffsetRestore.add_parser(subparsers) 73 | return parser.parse_args() 74 | 75 | 76 | def run(): 77 | logging.basicConfig(level=logging.ERROR) 78 | args = parse_args() 79 | try: 80 | conf = get_cluster_config( 81 | args.cluster_type, 82 | args.cluster_name, 83 | args.discovery_base_path, 84 | ) 85 | except ConfigurationError as e: 86 | print(e, file=sys.stderr) 87 | sys.exit(1) 88 | args.command(args, conf) 89 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 14 | 15 | .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " pickle to make pickle files" 22 | @echo " json to make JSON files" 23 | @echo " htmlhelp to make HTML files and a HTML help project" 24 | @echo " qthelp to make HTML files and a qthelp project" 25 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 26 | @echo " changes to make an overview of all changed/added/deprecated items" 27 | @echo " linkcheck to check all external links for integrity" 28 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 29 | 30 | clean: 31 | -rm -rf $(BUILDDIR)/* 32 | 33 | html: 34 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 35 | @echo 36 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 37 | 38 | dirhtml: 39 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 40 | @echo 41 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 42 | 43 | pickle: 44 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 45 | @echo 46 | @echo "Build finished; now you can process the pickle files." 47 | 48 | json: 49 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 50 | @echo 51 | @echo "Build finished; now you can process the JSON files." 52 | 53 | htmlhelp: 54 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 55 | @echo 56 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 57 | ".hhp project file in $(BUILDDIR)/htmlhelp." 58 | 59 | qthelp: 60 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 61 | @echo 62 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 63 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 64 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Kafka-Utils.qhcp" 65 | @echo "To view the help file:" 66 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Kafka-Utils.qhc" 67 | 68 | latex: 69 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 70 | @echo 71 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 72 | @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ 73 | "run these through (pdf)latex." 74 | 75 | changes: 76 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 77 | @echo 78 | @echo "The overview file is in $(BUILDDIR)/changes." 79 | 80 | linkcheck: 81 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 82 | @echo 83 | @echo "Link check complete; look for any errors in the above output " \ 84 | "or in $(BUILDDIR)/linkcheck/output.txt." 85 | 86 | doctest: 87 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 88 | @echo "Testing of doctests in the sources finished, look at the " \ 89 | "results in $(BUILDDIR)/doctest/output.txt." 90 | -------------------------------------------------------------------------------- /tests/kafka_cluster_manager/partition_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import pytest 16 | from mock import sentinel 17 | 18 | from kafka_utils.kafka_cluster_manager.cluster_info.partition import Partition 19 | 20 | 21 | class TestPartition(object): 22 | 23 | @pytest.fixture 24 | def partition(self): 25 | mock_topic = sentinel.t1 26 | mock_topic.id = 't1' 27 | return Partition( 28 | mock_topic, 29 | 0, 30 | [sentinel.r1, sentinel.r2], 31 | ) 32 | 33 | def test_name(self, partition): 34 | assert partition.name == ('t1', 0) 35 | 36 | def test_topic(self, partition): 37 | assert partition.topic == sentinel.t1 38 | 39 | def test_replicas(self, partition): 40 | assert partition.replicas == [sentinel.r1, sentinel.r2] 41 | 42 | def test_leader(self, partition): 43 | assert partition.leader == sentinel.r1 44 | 45 | def test_replication_factor(self, partition): 46 | assert partition.replication_factor == 2 47 | 48 | def test_partition_id(self, partition): 49 | assert partition.partition_id == 0 50 | 51 | def test_add_replica(self, partition): 52 | new_broker = sentinel.new_r 53 | partition.add_replica(new_broker) 54 | assert partition.replicas == [sentinel.r1, sentinel.r2, sentinel.new_r] 55 | 56 | def test_swap_leader(self, partition): 57 | b = sentinel.r2 58 | old_replicas = partition.replicas 59 | partition.swap_leader(b) 60 | 61 | # Verify leader changed to b 62 | assert partition.leader == b 63 | # Verify that replica set remains same 64 | assert sorted(old_replicas) == sorted(partition.replicas) 65 | 66 | def test_followers_1(self, partition): 67 | # Case:1 With followers 68 | assert partition.followers == [sentinel.r2] 69 | 70 | def test_followers_2(self): 71 | # Case:2 No-followers 72 | mock_topic = sentinel.t1 73 | mock_topic.id = 't1' 74 | p2 = Partition(mock_topic, 0, [sentinel.r1]) 75 | 76 | assert p2.followers == [] 77 | 78 | def test_count_siblings(self): 79 | t1, t1.id = sentinel.t1, 't1' 80 | t2, t2.id = sentinel.t2, 't2' 81 | p1, p3, p4 = Partition(t1, 0), Partition(t1, 1), Partition(t2, 0), 82 | 83 | # verify sibling count 84 | p_group = [p1, p4, p3] 85 | assert p3.count_siblings(p_group) == 2 86 | assert p4.count_siblings(p_group) == 1 87 | p_group = [p4] 88 | assert p1.count_siblings(p_group) == 0 89 | 90 | # Empty group 91 | p_group = [] 92 | assert p1.count_siblings(p_group) == 0 93 | 94 | def test_replace(self, partition): 95 | curr_broker = partition.replicas[0] 96 | partition.replace(curr_broker, sentinel.new_broker) 97 | 98 | assert partition.replicas[0] == sentinel.new_broker 99 | -------------------------------------------------------------------------------- /tests/kafka_consumer_manager/test_offset_restore.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import mock 16 | import pytest 17 | 18 | from kafka_utils.kafka_consumer_manager. \ 19 | commands.offset_restore import OffsetRestore 20 | from kafka_utils.util.client import KafkaToolClient 21 | from kafka_utils.util.monitoring import ConsumerPartitionOffsets 22 | 23 | 24 | class TestOffsetRestore(object): 25 | 26 | topics_partitions = { 27 | "topic1": [0, 1, 2], 28 | "topic2": [0, 1, 2, 3], 29 | "topic3": [0, 1], 30 | } 31 | consumer_offsets_metadata = { 32 | 'topic1': 33 | [ 34 | ConsumerPartitionOffsets(topic='topic1', partition=0, current=20, highmark=655, lowmark=655), 35 | ConsumerPartitionOffsets(topic='topic1', partition=1, current=10, highmark=655, lowmark=655) 36 | ] 37 | } 38 | parsed_consumer_offsets = {'groupid': 'group1', 'offsets': {'topic1': {0: 10, 1: 20}}} 39 | new_consumer_offsets = {'topic1': {0: 10, 1: 20}} 40 | kafka_consumer_offsets = {'topic1': [ 41 | ConsumerPartitionOffsets(topic='topic1', partition=0, current=30, highmark=40, lowmark=10), 42 | ConsumerPartitionOffsets(topic='topic1', partition=1, current=20, highmark=40, lowmark=10), 43 | ]} 44 | 45 | @pytest.fixture 46 | def mock_kafka_client(self): 47 | mock_kafka_client = mock.MagicMock( 48 | spec=KafkaToolClient 49 | ) 50 | mock_kafka_client.get_partition_ids_for_topic. \ 51 | side_effect = self.topics_partitions 52 | return mock_kafka_client 53 | 54 | def test_restore_offsets_zk(self, mock_kafka_client): 55 | with mock.patch( 56 | "kafka_utils.kafka_consumer_manager." 57 | "commands.offset_restore.set_consumer_offsets", 58 | return_value=[], 59 | autospec=True, 60 | ) as mock_set_offsets, mock.patch.object( 61 | OffsetRestore, 62 | "parse_consumer_offsets", 63 | spec=OffsetRestore.parse_consumer_offsets, 64 | return_value=self.parsed_consumer_offsets, 65 | ), mock.patch( 66 | "kafka_utils.kafka_consumer_manager." 67 | "commands.offset_restore.get_consumer_offsets_metadata", 68 | return_value=self.consumer_offsets_metadata, 69 | autospec=True, 70 | ): 71 | OffsetRestore.restore_offsets( 72 | mock_kafka_client, 73 | self.parsed_consumer_offsets, 74 | 'zookeeper', 75 | ) 76 | 77 | ordered_args, _ = mock_set_offsets.call_args 78 | assert ordered_args[1] == 'group1' 79 | assert ordered_args[2] == self.new_consumer_offsets 80 | 81 | def test_build_new_offsets(self, mock_kafka_client): 82 | new_offsets = OffsetRestore.build_new_offsets( 83 | mock_kafka_client, 84 | {'topic1': {0: 10, 1: 20}}, 85 | {'topic1': [0, 1]}, 86 | self.kafka_consumer_offsets, 87 | ) 88 | 89 | assert new_offsets == self.new_consumer_offsets 90 | -------------------------------------------------------------------------------- /kafka_utils/kafka_cluster_manager/cluster_info/partition.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | class Partition(object): 18 | """Class representing the partition object. 19 | It contains topic-partition_id tuple as name, topic and replicas 20 | (list of brokers). 21 | """ 22 | 23 | def __init__(self, topic, id, replicas=None): 24 | # Every partition name has (topic, partition) tuple 25 | self._name = (topic.id, id) 26 | self._replicas = replicas or [] 27 | self._topic = topic 28 | 29 | @property 30 | def name(self): 31 | "Name of partition, consisting of (topic_id, partition_id) tuple.""" 32 | return self._name 33 | 34 | @property 35 | def partition_id(self): 36 | """Partition id component of the partition-tuple.""" 37 | return int(self._name[1]) 38 | 39 | @property 40 | def topic(self): 41 | return self._topic 42 | 43 | @property 44 | def replicas(self): 45 | """List of brokers in partition.""" 46 | return self._replicas 47 | 48 | @property 49 | def leader(self): 50 | """Leader broker for the partition.""" 51 | return self._replicas[0] 52 | 53 | @property 54 | def replication_factor(self): 55 | return len(self._replicas) 56 | 57 | @property 58 | def followers(self): 59 | """Return list of brokers not as preferred leader 60 | for a particular partition. 61 | """ 62 | return self._replicas[1:] 63 | 64 | def add_replica(self, broker): 65 | """Add broker to existing set of replicas.""" 66 | self._replicas.append(broker) 67 | 68 | def swap_leader(self, new_leader): 69 | """Change the preferred leader with one of 70 | given replicas. 71 | 72 | Note: Leaders for all the replicas of current 73 | partition needs to be changed. 74 | """ 75 | # Replica set cannot be changed 76 | assert(new_leader in self._replicas) 77 | curr_leader = self.leader 78 | idx = self._replicas.index(new_leader) 79 | self._replicas[0], self._replicas[idx] = \ 80 | self._replicas[idx], self._replicas[0] 81 | return curr_leader 82 | 83 | def replace(self, source, dest): 84 | """Replace source broker with destination broker in replica set if found.""" 85 | for i, broker in enumerate(self.replicas): 86 | if broker == source: 87 | self.replicas[i] = dest 88 | return 89 | 90 | def count_siblings(self, partitions): 91 | """Count siblings of partition in given partition-list. 92 | 93 | :key-term: 94 | sibling: partitions with same topic 95 | """ 96 | count = sum( 97 | int(self.topic == partition.topic) 98 | for partition in partitions 99 | ) 100 | return count 101 | 102 | def __str__(self): 103 | return "{name}".format(name=self._name) 104 | 105 | def __repr__(self): 106 | return "{0}".format(self) 107 | -------------------------------------------------------------------------------- /kafka_utils/kafka_check/main.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """ 16 | Kafka checks module. 17 | Each check is separated subcommand for kafka-check. 18 | """ 19 | from __future__ import absolute_import 20 | from __future__ import print_function 21 | 22 | import argparse 23 | 24 | from kafka_utils.kafka_check import status_code 25 | from kafka_utils.kafka_check.commands.min_isr import MinIsrCmd 26 | from kafka_utils.kafka_check.status_code import terminate 27 | from kafka_utils.util import config 28 | from kafka_utils.util.error import ConfigurationError 29 | 30 | 31 | def convert_to_broker_id(string): 32 | """Convert string to kafka broker_id.""" 33 | error_msg = 'Positive integer or -1 required, {string} given.'.format(string=string) 34 | try: 35 | value = int(string) 36 | except ValueError: 37 | raise argparse.ArgumentTypeError(error_msg) 38 | if value <= 0 and value != -1: 39 | raise argparse.ArgumentTypeError(error_msg) 40 | return value 41 | 42 | 43 | def parse_args(): 44 | """Parse the command line arguments.""" 45 | parser = argparse.ArgumentParser( 46 | description='Check kafka current status', 47 | ) 48 | parser.add_argument( 49 | "--cluster-type", 50 | dest='cluster_type', 51 | required=True, 52 | help='Type of cluster', 53 | default=None, 54 | ) 55 | parser.add_argument( 56 | "--cluster-name", 57 | dest='cluster_name', 58 | help='Name of the cluster', 59 | ) 60 | parser.add_argument( 61 | '--discovery-base-path', 62 | dest='discovery_base_path', 63 | type=str, 64 | help='Path of the directory containing the .yaml config', 65 | ) 66 | parser.add_argument( 67 | "--broker-id", 68 | help='Kafka current broker id.', 69 | type=convert_to_broker_id, 70 | ) 71 | parser.add_argument( 72 | "--data-path", 73 | help='Path to the Kafka data folder.', 74 | ) 75 | parser.add_argument( 76 | '--controller-only', 77 | action="store_true", 78 | help='If this parameter is specified, it will do nothing and succeed on ' 79 | 'non-controller brokers. Set --broker-id to -1 to read broker-id from ' 80 | '--data-path.', 81 | ) 82 | 83 | subparsers = parser.add_subparsers() 84 | MinIsrCmd().add_subparser(subparsers) 85 | 86 | return parser.parse_args() 87 | 88 | 89 | def run(): 90 | """Verify command-line arguments and run commands""" 91 | args = parse_args() 92 | 93 | try: 94 | cluster_config = config.get_cluster_config( 95 | args.cluster_type, 96 | args.cluster_name, 97 | args.discovery_base_path, 98 | ) 99 | code, msg = args.command(cluster_config, args) 100 | except ConfigurationError as e: 101 | terminate(status_code.CRITICAL, "ConfigurationError {0}".format(e)) 102 | except Exception as e: 103 | terminate(status_code.CRITICAL, "Got Exception: {0}".format(e)) 104 | 105 | terminate(code, msg) 106 | -------------------------------------------------------------------------------- /kafka_utils/kafka_consumer_manager/commands/offset_rewind.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import print_function 17 | from __future__ import unicode_literals 18 | 19 | import sys 20 | 21 | from .offset_manager import OffsetWriter 22 | from kafka_utils.util.client import KafkaToolClient 23 | from kafka_utils.util.offsets import rewind_consumer_offsets 24 | 25 | 26 | class OffsetRewind(OffsetWriter): 27 | 28 | @classmethod 29 | def setup_subparser(cls, subparsers): 30 | parser_offset_rewind = subparsers.add_parser( 31 | "offset_rewind", 32 | description="Rewind consumer offsets for the specified consumer " 33 | "group to the earliest message in the topic partition", 34 | add_help=False 35 | ) 36 | parser_offset_rewind.add_argument( 37 | "-h", "--help", action="help", 38 | help="Show this help message and exit." 39 | ) 40 | parser_offset_rewind.add_argument( 41 | 'groupid', 42 | help="Consumer Group ID whose consumer offsets shall be rewinded." 43 | ) 44 | parser_offset_rewind.add_argument( 45 | "--topic", 46 | help="Kafka topic whose offsets shall be manipulated. If no topic is " 47 | "specified, offsets from all topics that the consumer is " 48 | "subscribed to, shall be rewinded." 49 | ) 50 | parser_offset_rewind.add_argument( 51 | "--partitions", nargs='+', type=int, 52 | help="List of partitions within the topic. If no partitions are " 53 | "specified, offsets from all partitions of the topic shall " 54 | "be rewinded." 55 | ) 56 | parser_offset_rewind.add_argument( 57 | '--storage', choices=['zookeeper', 'kafka'], 58 | help="String describing where to store the committed offsets." 59 | ) 60 | parser_offset_rewind.add_argument( 61 | '--force', 62 | action='store_true', 63 | help="Force the offset of the group to be committed even if " 64 | "it does not already exist." 65 | ) 66 | parser_offset_rewind.set_defaults(command=OffsetRewind.run) 67 | 68 | @classmethod 69 | def run(cls, args, cluster_config): 70 | # Setup the Kafka client 71 | client = KafkaToolClient(cluster_config.broker_list) 72 | client.load_metadata_for_topics() 73 | 74 | topics_dict = cls.preprocess_args( 75 | args.groupid, 76 | args.topic, 77 | args.partitions, 78 | cluster_config, 79 | client, 80 | force=args.force, 81 | ) 82 | try: 83 | rewind_consumer_offsets( 84 | client, 85 | args.groupid, 86 | topics_dict, 87 | args.storage, 88 | ) 89 | except TypeError: 90 | print( 91 | "Error: Badly formatted input, please re-run command " 92 | "with --help option.", file=sys.stderr 93 | ) 94 | raise 95 | 96 | client.close() 97 | -------------------------------------------------------------------------------- /kafka_utils/kafka_consumer_manager/commands/offset_advance.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import print_function 17 | from __future__ import unicode_literals 18 | 19 | import sys 20 | 21 | from .offset_manager import OffsetWriter 22 | from kafka_utils.util.client import KafkaToolClient 23 | from kafka_utils.util.offsets import advance_consumer_offsets 24 | 25 | 26 | class OffsetAdvance(OffsetWriter): 27 | 28 | @classmethod 29 | def setup_subparser(cls, subparsers): 30 | parser_offset_advance = subparsers.add_parser( 31 | "offset_advance", 32 | description="Advance consumer offsets for the specified consumer " 33 | "group to the latest message in the topic partition", 34 | add_help=False 35 | ) 36 | parser_offset_advance.add_argument( 37 | "-h", "--help", action="help", 38 | help="Show this help message and exit." 39 | ) 40 | parser_offset_advance.add_argument( 41 | 'groupid', 42 | help="Consumer Group ID whose consumer offsets shall be advanced." 43 | ) 44 | parser_offset_advance.add_argument( 45 | "--topic", 46 | help="Kafka topic whose offsets shall be manipulated. If no topic " 47 | "is specified, offsets from all topics that the consumer is " 48 | "subscribed to, shall be advanced." 49 | ) 50 | parser_offset_advance.add_argument( 51 | "--partitions", nargs='+', type=int, 52 | help="List of partitions within the topic. If no partitions are " 53 | "specified, offsets from all partitions of the topic shall " 54 | "be advanced." 55 | ) 56 | parser_offset_advance.add_argument( 57 | '--storage', choices=['zookeeper', 'kafka'], 58 | help="String describing where to store the committed offsets." 59 | ) 60 | parser_offset_advance.add_argument( 61 | '--force', 62 | action='store_true', 63 | help="Force the offset of the group to be committed even if " 64 | "it does not already exist." 65 | ) 66 | parser_offset_advance.set_defaults(command=cls.run) 67 | 68 | @classmethod 69 | def run(cls, args, cluster_config): 70 | # Setup the Kafka client 71 | client = KafkaToolClient(cluster_config.broker_list) 72 | client.load_metadata_for_topics() 73 | 74 | topics_dict = cls.preprocess_args( 75 | args.groupid, 76 | args.topic, 77 | args.partitions, 78 | cluster_config, 79 | client, 80 | force=args.force, 81 | ) 82 | try: 83 | advance_consumer_offsets( 84 | client, 85 | args.groupid, 86 | topics_dict, 87 | offset_storage=args.storage, 88 | ) 89 | except TypeError: 90 | print( 91 | "Error: Badly formatted input, please re-run command ", 92 | "with --help option.", file=sys.stderr 93 | ) 94 | raise 95 | 96 | client.close() 97 | -------------------------------------------------------------------------------- /tests/kafka_consumer_manager/test_offset_save.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import mock 16 | 17 | from kafka_utils.kafka_consumer_manager. \ 18 | commands.offset_save import OffsetSave 19 | from kafka_utils.util.monitoring import ConsumerPartitionOffsets 20 | 21 | 22 | class TestOffsetSave(object): 23 | topics_partitions = { 24 | "topic1": [0, 1, 2], 25 | "topic2": [0, 1, 2, 3], 26 | "topic3": [0, 1], 27 | } 28 | consumer_offsets_metadata = { 29 | 'topic1': 30 | [ 31 | ConsumerPartitionOffsets(topic='topic1', partition=0, current=10, highmark=655, lowmark=655), 32 | ConsumerPartitionOffsets(topic='topic1', partition=1, current=20, highmark=655, lowmark=655), 33 | ] 34 | } 35 | offset_data_file = {'groupid': 'group1', 'offsets': {'topic1': {0: 10, 1: 20}}} 36 | json_data = {'groupid': 'group1', 'offsets': {'topic1': {'0': 10, '1': 20}}} 37 | 38 | @mock.patch('kafka_utils.kafka_consumer_manager.' 39 | 'commands.offset_save.KafkaToolClient') 40 | def test_save_offsets(self, mock_client): 41 | with mock.patch.object( 42 | OffsetSave, 43 | "write_offsets_to_file", 44 | spec=OffsetSave.write_offsets_to_file, 45 | return_value=[], 46 | ) as mock_write_offsets: 47 | filename = 'offset_file' 48 | consumer_group = 'group1' 49 | OffsetSave.save_offsets( 50 | self.consumer_offsets_metadata, 51 | self.topics_partitions, 52 | filename, 53 | consumer_group, 54 | ) 55 | 56 | ordered_args, _ = mock_write_offsets.call_args 57 | assert ordered_args[0] == filename 58 | assert ordered_args[1] == self.offset_data_file 59 | 60 | @mock.patch('kafka_utils.kafka_consumer_manager.' 61 | 'commands.offset_save.KafkaToolClient') 62 | def test_run(self, mock_client): 63 | with mock.patch.object( 64 | OffsetSave, 65 | "preprocess_args", 66 | spec=OffsetSave.preprocess_args, 67 | return_value=self.topics_partitions, 68 | ), mock.patch( 69 | "kafka_utils.kafka_consumer_manager." 70 | "commands.offset_save.get_consumer_offsets_metadata", 71 | return_value=self.consumer_offsets_metadata, 72 | autospec=True, 73 | ), mock.patch.object( 74 | OffsetSave, 75 | "write_offsets_to_file", 76 | spec=OffsetSave.write_offsets_to_file, 77 | return_value=[], 78 | ) as mock_write_offsets: 79 | args = mock.Mock( 80 | groupid="group1", 81 | json_file="some_file", 82 | ) 83 | cluster_config = mock.Mock() 84 | OffsetSave.run(args, cluster_config) 85 | 86 | mock_client.return_value.load_metadata_for_topics. \ 87 | assert_called_once_with() 88 | mock_client.return_value.close.assert_called_once_with() 89 | ordered_args, _ = mock_write_offsets.call_args 90 | assert ordered_args[0] == "some_file" 91 | assert ordered_args[1] == self.offset_data_file 92 | -------------------------------------------------------------------------------- /tests/kafka_rolling_restart/test_main.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import mock 16 | import pytest 17 | import requests 18 | from requests.exceptions import RequestException 19 | 20 | from kafka_utils.kafka_rolling_restart import main 21 | 22 | 23 | @mock.patch.object(main.FuturesSession, 'get', autospec=True) 24 | def test_read_cluster_value_partitions(mock_get): 25 | response = mock.Mock(status_code=200, spec=requests.Response) 26 | response.json.return_value = {'value': 1} 27 | 28 | request = mock_get.return_value 29 | request.result.return_value = response 30 | 31 | p, b = main.read_cluster_status(["host1", "host2", "host3"], 80, "jolokia") 32 | 33 | assert p == 3 # 3 missing partitions 34 | assert b == 0 # 0 missing brokers 35 | 36 | 37 | @mock.patch.object(main.FuturesSession, 'get', autospec=True) 38 | def test_read_cluster_value_exit(mock_get): 39 | response = mock.Mock(status_code=404, spec=requests.Response) 40 | 41 | request = mock_get.return_value 42 | request.result.return_value = response 43 | 44 | with pytest.raises(SystemExit): 45 | p, b = main.read_cluster_status(["host1"], 80, "jolokia") 46 | 47 | 48 | @mock.patch.object(main.FuturesSession, 'get', autospec=True) 49 | def test_read_cluster_value_no_key(mock_get): 50 | response = mock.Mock(status_code=200, spec=requests.Response) 51 | response.json.return_value = {'wrong_key': 1} 52 | 53 | request = mock_get.return_value 54 | request.result.return_value = response 55 | 56 | p, b = main.read_cluster_status(["host1"], 80, "jolokia") 57 | 58 | assert p == 0 # 0 missing partitions 59 | assert b == 1 # 1 missing brokers 60 | 61 | 62 | @mock.patch.object(main.FuturesSession, 'get', autospec=True) 63 | def test_read_cluster_value_server_down(mock_get): 64 | request = mock_get.return_value 65 | request.result.side_effect = RequestException 66 | 67 | p, b = main.read_cluster_status(["host1"], 80, "jolokia") 68 | 69 | assert p == 0 # 0 missing partitions 70 | assert b == 1 # 1 missing brokers 71 | 72 | 73 | def read_cluster_state_values(first_part, repeat): 74 | for value in first_part: 75 | yield value 76 | while True: 77 | yield repeat 78 | 79 | 80 | @mock.patch.object( 81 | main, 82 | 'read_cluster_status', 83 | side_effect=read_cluster_state_values([(100, 1), (0, 1), (100, 0)], (0, 0)), 84 | autospec=True, 85 | ) 86 | @mock.patch('time.sleep', autospec=True) 87 | def test_wait_for_stable_cluster_success(mock_sleep, mock_read): 88 | main.wait_for_stable_cluster([], 1, "", 5, 3, 100) 89 | 90 | assert mock_read.call_count == 6 91 | assert mock_sleep.mock_calls == [mock.call(5)] * 5 92 | 93 | 94 | @mock.patch.object( 95 | main, 96 | 'read_cluster_status', 97 | side_effect=read_cluster_state_values([], (100, 0)), 98 | autospec=True, 99 | ) 100 | @mock.patch('time.sleep', autospec=True) 101 | def test_wait_for_stable_cluster_timeout(mock_sleep, mock_read): 102 | with pytest.raises(main.WaitTimeoutException): 103 | main.wait_for_stable_cluster([], 1, "", 5, 3, 100) 104 | 105 | assert mock_read.call_count == 21 106 | assert mock_sleep.mock_calls == [mock.call(5)] * 20 107 | -------------------------------------------------------------------------------- /tests/kafka_consumer_manager/test_delete_group.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import mock 16 | import pytest 17 | from kazoo.exceptions import ZookeeperError 18 | 19 | from kafka_utils.kafka_consumer_manager.commands. \ 20 | delete_group import DeleteGroup 21 | 22 | 23 | class TestDeleteGroup(object): 24 | 25 | @pytest.yield_fixture 26 | def client(self): 27 | with mock.patch( 28 | 'kafka_utils.kafka_consumer_manager.' 29 | 'commands.delete_group.KafkaToolClient', 30 | autospec=True, 31 | ) as mock_client: 32 | yield mock_client 33 | 34 | @pytest.yield_fixture 35 | def zk(self): 36 | with mock.patch( 37 | 'kafka_utils.kafka_consumer_manager.' 38 | 'commands.delete_group.ZK', 39 | autospec=True 40 | ) as mock_zk: 41 | mock_zk.return_value.__enter__.return_value = mock_zk.return_value 42 | yield mock_zk 43 | 44 | @pytest.yield_fixture 45 | def offsets(self): 46 | yield {'topic1': {0: 100}} 47 | 48 | def test_run_wipe_delete_group(self, client, zk, offsets): 49 | with mock.patch.object( 50 | DeleteGroup, 51 | 'preprocess_args', 52 | spec=DeleteGroup.preprocess_args, 53 | return_value=offsets, 54 | ): 55 | args = mock.Mock( 56 | groupid="some_group", 57 | storage="zookeeper", 58 | ) 59 | cluster_config = mock.Mock(zookeeper='some_ip') 60 | 61 | DeleteGroup.run(args, cluster_config) 62 | 63 | obj = zk.return_value 64 | assert obj.delete_group.call_args_list == [ 65 | mock.call(args.groupid), 66 | ] 67 | 68 | def test_run_wipe_delete_group_error(self, client, zk, offsets): 69 | with mock.patch.object( 70 | DeleteGroup, 71 | 'preprocess_args', 72 | spec=DeleteGroup.preprocess_args, 73 | return_value=offsets, 74 | ): 75 | obj = zk.return_value.__enter__.return_value 76 | obj.__exit__.return_value = False 77 | obj.delete_group.side_effect = ZookeeperError("Boom!") 78 | args = mock.Mock( 79 | groupid="some_group", 80 | storage="zookeeper", 81 | ) 82 | cluster_config = mock.Mock(zookeeper='some_ip') 83 | 84 | with pytest.raises(ZookeeperError): 85 | DeleteGroup.run(args, cluster_config) 86 | 87 | def test_delete_topic_kafka_storage(self, client, offsets): 88 | new_offsets = {'topic1': {0: -1}} 89 | 90 | with mock.patch( 91 | 'kafka_utils.kafka_consumer_manager.' 92 | 'commands.delete_group.set_consumer_offsets', 93 | autospec=True, 94 | ) as mock_set: 95 | DeleteGroup.delete_group_kafka(client, 'some_group', offsets) 96 | 97 | assert mock_set.call_count == 1 98 | assert mock_set.call_args_list == [ 99 | mock.call( 100 | client, 101 | 'some_group', 102 | new_offsets, 103 | offset_storage='kafka', 104 | ), 105 | ] 106 | -------------------------------------------------------------------------------- /kafka_utils/kafka_consumer_manager/commands/copy_group.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import print_function 17 | from __future__ import unicode_literals 18 | 19 | import sys 20 | 21 | from kazoo.exceptions import NoNodeError 22 | 23 | from .offset_manager import OffsetManagerBase 24 | from kafka_utils.kafka_consumer_manager.util import create_offsets 25 | from kafka_utils.kafka_consumer_manager.util import fetch_offsets 26 | from kafka_utils.kafka_consumer_manager.util import preprocess_topics 27 | from kafka_utils.util.client import KafkaToolClient 28 | from kafka_utils.util.zookeeper import ZK 29 | 30 | 31 | class CopyGroup(OffsetManagerBase): 32 | 33 | @classmethod 34 | def setup_subparser(cls, subparsers): 35 | parser_copy_group = subparsers.add_parser( 36 | "copy_group", 37 | description="Copy specified consumer group details to a new group.", 38 | ) 39 | parser_copy_group.add_argument( 40 | 'source_groupid', 41 | help="Consumer Group to be copied.", 42 | ) 43 | parser_copy_group.add_argument( 44 | 'dest_groupid', 45 | help="New name for the consumer group being copied to.", 46 | ) 47 | parser_copy_group.add_argument( 48 | "--topic", 49 | help="Kafka topic whose offsets will be copied into destination group" 50 | " If no topic is specificed all topic offsets will be copied.", 51 | ) 52 | parser_copy_group.add_argument( 53 | "--partitions", 54 | nargs='+', 55 | type=int, 56 | help="List of partitions within the topic. If no partitions are " 57 | "specified, offsets from all partitions of the topic shall " 58 | "be copied.", 59 | ) 60 | parser_copy_group.set_defaults(command=cls.run) 61 | 62 | @classmethod 63 | def run(cls, args, cluster_config): 64 | if args.source_groupid == args.dest_groupid: 65 | print( 66 | "Error: Source group ID and destination group ID are same.", 67 | file=sys.stderr, 68 | ) 69 | sys.exit(1) 70 | # Setup the Kafka client 71 | client = KafkaToolClient(cluster_config.broker_list) 72 | client.load_metadata_for_topics() 73 | source_topics = cls.preprocess_args( 74 | args.source_groupid, 75 | args.topic, 76 | args.partitions, 77 | cluster_config, 78 | client, 79 | ) 80 | with ZK(cluster_config) as zk: 81 | try: 82 | topics_dest_group = zk.get_children( 83 | "/consumers/{groupid}/offsets".format( 84 | groupid=args.dest_groupid, 85 | ) 86 | ) 87 | except NoNodeError: 88 | # Consumer Group ID doesn't exist. 89 | pass 90 | else: 91 | preprocess_topics( 92 | args.source_groupid, 93 | source_topics.keys(), 94 | args.dest_groupid, 95 | topics_dest_group, 96 | ) 97 | 98 | # Fetch offsets 99 | source_offsets = fetch_offsets(zk, args.source_groupid, source_topics) 100 | create_offsets(zk, args.dest_groupid, source_offsets) 101 | -------------------------------------------------------------------------------- /kafka_utils/kafka_consumer_manager/commands/rename_group.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import print_function 17 | from __future__ import unicode_literals 18 | 19 | import sys 20 | 21 | from kazoo.exceptions import NoNodeError 22 | 23 | from .offset_manager import OffsetManagerBase 24 | from kafka_utils.kafka_consumer_manager.util import create_offsets 25 | from kafka_utils.kafka_consumer_manager.util import fetch_offsets 26 | from kafka_utils.kafka_consumer_manager.util import preprocess_topics 27 | from kafka_utils.util.client import KafkaToolClient 28 | from kafka_utils.util.zookeeper import ZK 29 | 30 | 31 | class RenameGroup(OffsetManagerBase): 32 | 33 | @classmethod 34 | def setup_subparser(cls, subparsers): 35 | parser_rename_group = subparsers.add_parser( 36 | "rename_group", 37 | description="Rename specified consumer group ID to a new name. " 38 | "This tool shall migrate all offset metadata in Zookeeper.", 39 | add_help=False 40 | ) 41 | parser_rename_group.add_argument( 42 | "-h", "--help", action="help", 43 | help="Show this help message and exit." 44 | ) 45 | parser_rename_group.add_argument( 46 | 'old_groupid', 47 | help="Consumer Group ID to be renamed." 48 | ) 49 | parser_rename_group.add_argument( 50 | 'new_groupid', 51 | help="New name for the consumer group ID." 52 | ) 53 | parser_rename_group.set_defaults(command=cls.run) 54 | 55 | @classmethod 56 | def run(cls, args, cluster_config): 57 | if args.old_groupid == args.new_groupid: 58 | print( 59 | "Error: Old group ID and new group ID are the same.", 60 | file=sys.stderr, 61 | ) 62 | sys.exit(1) 63 | # Setup the Kafka client 64 | client = KafkaToolClient(cluster_config.broker_list) 65 | client.load_metadata_for_topics() 66 | 67 | topics_dict = cls.preprocess_args( 68 | args.old_groupid, None, None, cluster_config, client 69 | ) 70 | with ZK(cluster_config) as zk: 71 | try: 72 | topics = zk.get_children( 73 | "/consumers/{groupid}/offsets".format( 74 | groupid=args.new_groupid 75 | ) 76 | ) 77 | except NoNodeError: 78 | # Consumer Group ID doesn't exist. 79 | pass 80 | else: 81 | preprocess_topics( 82 | args.old_groupid, 83 | topics_dict.keys(), 84 | args.new_groupid, 85 | topics, 86 | ) 87 | 88 | old_offsets = fetch_offsets(zk, args.old_groupid, topics_dict) 89 | create_offsets(zk, args.new_groupid, old_offsets) 90 | try: 91 | old_base_path = "/consumers/{groupid}".format( 92 | groupid=args.old_groupid, 93 | ) 94 | zk.delete(old_base_path, recursive=True) 95 | except: 96 | print( 97 | "Error: Unable to migrate all metadata in Zookeeper. " 98 | "Please re-run the command.", 99 | file=sys.stderr 100 | ) 101 | raise 102 | -------------------------------------------------------------------------------- /kafka_utils/kafka_cluster_manager/cmds/decommission.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import print_function 16 | 17 | import logging 18 | import sys 19 | 20 | from .command import ClusterManagerCmd 21 | from kafka_utils.util.validation import assignment_to_plan 22 | from kafka_utils.util.validation import validate_plan 23 | 24 | 25 | DEFAULT_MAX_PARTITION_MOVEMENTS = 1 26 | DEFAULT_MAX_LEADER_CHANGES = 5 27 | 28 | 29 | class DecommissionCmd(ClusterManagerCmd): 30 | 31 | def __init__(self): 32 | super(DecommissionCmd, self).__init__() 33 | self.log = logging.getLogger(self.__class__.__name__) 34 | 35 | def build_subparser(self, subparsers): 36 | subparser = subparsers.add_parser( 37 | 'decommission', 38 | description='Decommission one or more brokers of the cluster.', 39 | help='This command is used to move all the replicas assigned to given ' 40 | 'brokers and redistribute them across all the other brokers while ' 41 | 'trying to keep the cluster balanced.', 42 | ) 43 | subparser.add_argument( 44 | 'broker_ids', 45 | nargs='+', 46 | type=int, 47 | help='Broker ids of the brokers to decommission.', 48 | ) 49 | subparser.add_argument( 50 | '--max-partition-movements', 51 | type=self.positive_int, 52 | default=DEFAULT_MAX_PARTITION_MOVEMENTS, 53 | help='Maximum number of partition-movements in final set of actions.' 54 | ' DEFAULT: %(default)s. RECOMMENDATION: Should be at least max ' 55 | 'replication-factor across the cluster.', 56 | ) 57 | subparser.add_argument( 58 | '--max-leader-changes', 59 | type=self.positive_int, 60 | default=DEFAULT_MAX_LEADER_CHANGES, 61 | help='Maximum number of actions with leader-only changes.' 62 | ' DEFAULT: %(default)s', 63 | ) 64 | return subparser 65 | 66 | def run_command(self, cluster_topology): 67 | base_assignment = cluster_topology.assignment 68 | cluster_topology.decommission_brokers(self.args.broker_ids) 69 | 70 | if not validate_plan( 71 | assignment_to_plan(cluster_topology.assignment), 72 | assignment_to_plan(base_assignment), 73 | ): 74 | self.log.error('Invalid assignment %s.', cluster_topology.assignment) 75 | print( 76 | 'Invalid assignment: {0}'.format(cluster_topology.assignment), 77 | file=sys.stderr, 78 | ) 79 | sys.exit(1) 80 | 81 | # Reduce the proposed assignment based on max_partition_movements 82 | # and max_leader_changes 83 | reduced_assignment = self.get_reduced_assignment( 84 | base_assignment, 85 | cluster_topology.assignment, 86 | self.args.max_partition_movements, 87 | self.args.max_leader_changes, 88 | ) 89 | if reduced_assignment: 90 | self.process_assignment(reduced_assignment) 91 | else: 92 | self.log.info( 93 | "Cluster already balanced. No more replicas in " 94 | "decommissioned brokers." 95 | ) 96 | print( 97 | "Cluster already balanced. No more replicas in " 98 | "decommissioned brokers." 99 | ) 100 | -------------------------------------------------------------------------------- /kafka_utils/kafka_cluster_manager/cmds/replace.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import print_function 16 | 17 | import logging 18 | import sys 19 | 20 | from .command import ClusterManagerCmd 21 | from kafka_utils.util.validation import assignment_to_plan 22 | from kafka_utils.util.validation import validate_plan 23 | 24 | 25 | DEFAULT_MAX_PARTITION_MOVEMENTS = 1 26 | DEFAULT_MAX_LEADER_CHANGES = 5 27 | 28 | 29 | class ReplaceBrokerCmd(ClusterManagerCmd): 30 | 31 | def __init__(self): 32 | super(ReplaceBrokerCmd, self).__init__() 33 | self.log = logging.getLogger(self.__class__.__name__) 34 | 35 | def build_subparser(self, subparsers): 36 | subparser = subparsers.add_parser( 37 | 'replace-broker', 38 | description='Replace the given broker with new broker by moving all partitions.', 39 | help='This command is used to move all the replicas assigned to a given ' 40 | 'broker to destination broker. No change in cluster-imbalance state', 41 | ) 42 | subparser.add_argument( 43 | '--source-broker', 44 | type=int, 45 | required=True, 46 | help='Broker id of source broker.', 47 | ) 48 | subparser.add_argument( 49 | '--dest-broker', 50 | type=int, 51 | required=True, 52 | help='Broker id of destination broker.', 53 | ) 54 | subparser.add_argument( 55 | '--max-partition-movements', 56 | type=self.positive_int, 57 | default=DEFAULT_MAX_PARTITION_MOVEMENTS, 58 | help='Maximum number of partition-movements in final set of actions.' 59 | ' DEFAULT: %(default)s. RECOMMENDATION: Should be at least max ' 60 | 'replication-factor across the cluster.', 61 | ) 62 | subparser.add_argument( 63 | '--max-leader-changes', 64 | type=self.positive_int, 65 | default=DEFAULT_MAX_LEADER_CHANGES, 66 | help='Maximum number of actions with leader-only changes.' 67 | ' DEFAULT: %(default)s', 68 | ) 69 | return subparser 70 | 71 | def run_command(self, cluster_topology): 72 | if self.args.source_broker == self.args.dest_broker: 73 | print("Error: Destination broker is same as source broker.") 74 | sys.exit() 75 | 76 | base_assignment = cluster_topology.assignment 77 | cluster_topology.replace_broker(self.args.source_broker, self.args.dest_broker) 78 | 79 | if not validate_plan( 80 | assignment_to_plan(cluster_topology.assignment), 81 | assignment_to_plan(base_assignment), 82 | ): 83 | self.log.error('Invalid assignment %s.', cluster_topology.assignment) 84 | print( 85 | 'Invalid assignment: {0}'.format(cluster_topology.assignment), 86 | file=sys.stderr, 87 | ) 88 | sys.exit(1) 89 | 90 | # Reduce the proposed assignment based on max_partition_movements 91 | # and max_leader_changes 92 | reduced_assignment = self.get_reduced_assignment( 93 | base_assignment, 94 | cluster_topology.assignment, 95 | self.args.max_partition_movements, 96 | self.args.max_leader_changes, 97 | ) 98 | if reduced_assignment: 99 | self.process_assignment(reduced_assignment) 100 | else: 101 | self.log.info("Broker already replaced. No more replicas in source broker.") 102 | print("Broker already replaced. No more replicas in source broker.") 103 | -------------------------------------------------------------------------------- /tests/acceptance/steps/util.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import subprocess 16 | import time 17 | import uuid 18 | 19 | from kafka import KafkaConsumer 20 | from kafka import SimpleProducer 21 | from kafka.common import LeaderNotAvailableError 22 | 23 | from kafka_utils.util import config 24 | from kafka_utils.util.client import KafkaToolClient 25 | 26 | 27 | ZOOKEEPER_URL = 'zookeeper:2181' 28 | KAFKA_URL = 'kafka:9092' 29 | 30 | 31 | def get_cluster_config(): 32 | return config.get_cluster_config( 33 | 'test', 34 | 'test_cluster', 35 | 'tests/acceptance/config', 36 | ) 37 | 38 | 39 | def create_topic(topic_name, replication_factor, partitions): 40 | cmd = ['kafka-topics.sh', '--create', 41 | '--zookeeper', ZOOKEEPER_URL, 42 | '--replication-factor', str(replication_factor), 43 | '--partitions', str(partitions), 44 | '--topic', topic_name] 45 | subprocess.check_call(cmd) 46 | 47 | # It may take a little moment for the topic to be ready for writing. 48 | time.sleep(1) 49 | 50 | 51 | def list_topics(): 52 | cmd = ['kafka-topics.sh', '--list', 53 | '--zookeeper', ZOOKEEPER_URL] 54 | 55 | return call_cmd(cmd) 56 | 57 | 58 | def delete_topic(topic_name): 59 | cmd = ['kafka-topics.sh', '--delete', 60 | '--zookeeper', ZOOKEEPER_URL, 61 | '--topic', topic_name] 62 | 63 | return call_cmd(cmd) 64 | 65 | 66 | def call_cmd(cmd): 67 | output = '' 68 | try: 69 | p = subprocess.Popen( 70 | cmd, 71 | stdin=subprocess.PIPE, 72 | stdout=subprocess.PIPE, 73 | stderr=subprocess.PIPE, 74 | ) 75 | out, err = p.communicate('y') 76 | if out: 77 | output += out 78 | if err: 79 | output += err 80 | except subprocess.CalledProcessError as e: 81 | output += e.output 82 | return output 83 | 84 | 85 | def create_random_topic(replication_factor, partitions, topic_name=None): 86 | if not topic_name: 87 | topic_name = str(uuid.uuid1()) 88 | create_topic(topic_name, replication_factor, partitions) 89 | return topic_name 90 | 91 | 92 | def create_random_group_id(): 93 | return str(uuid.uuid1()) 94 | 95 | 96 | def produce_example_msg(topic, num_messages=1): 97 | kafka = KafkaToolClient(KAFKA_URL) 98 | producer = SimpleProducer(kafka) 99 | for i in xrange(num_messages): 100 | try: 101 | producer.send_messages(topic, b'some message') 102 | except LeaderNotAvailableError: 103 | # Sometimes kafka takes a bit longer to assign a leader to a new 104 | # topic 105 | time.sleep(10) 106 | producer.send_messages(topic, b'some message') 107 | 108 | 109 | def create_consumer_group(topic, group_name, num_messages=1): 110 | consumer = KafkaConsumer( 111 | topic, 112 | group_id=group_name, 113 | auto_commit_enable=False, 114 | bootstrap_servers=[KAFKA_URL], 115 | auto_offset_reset='smallest') 116 | for i in xrange(num_messages): 117 | message = consumer.next() 118 | consumer.task_done(message) 119 | consumer.commit() 120 | return consumer 121 | 122 | 123 | def call_offset_get(group, storage=None): 124 | cmd = ['kafka-consumer-manager', 125 | '--cluster-type', 'test', 126 | '--cluster-name', 'test_cluster', 127 | '--discovery-base-path', 'tests/acceptance/config', 128 | 'offset_get', 129 | group] 130 | if storage: 131 | cmd.extend(['--storage', storage]) 132 | return call_cmd(cmd) 133 | 134 | 135 | def initialize_kafka_offsets_topic(): 136 | topic = create_random_topic(1, 1) 137 | produce_example_msg(topic, num_messages=1) 138 | create_consumer_group(topic, 'foo') 139 | call_offset_get('foo', storage='kafka') 140 | time.sleep(10) 141 | -------------------------------------------------------------------------------- /kafka_utils/kafka_cluster_manager/cluster_info/util.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | def get_partitions_per_broker(brokers): 18 | """Return partition count for each broker.""" 19 | return dict( 20 | (broker, len(broker.partitions)) 21 | for broker in brokers 22 | ) 23 | 24 | 25 | def get_leaders_per_broker(brokers): 26 | """Return count for each broker the number of times 27 | it is assigned as preferred leader. 28 | """ 29 | return dict( 30 | (broker, broker.count_preferred_replica()) 31 | for broker in brokers 32 | ) 33 | 34 | 35 | def compute_optimum(groups, elements): 36 | """Compute the number of elements per group and the reminder. 37 | 38 | :param elements: total number of elements 39 | :param groups: total number of groups 40 | """ 41 | return elements // groups, elements % groups 42 | 43 | 44 | def _smart_separate_groups(groups, key, total): 45 | """Given a list of group objects, and a function to extract the number of 46 | elements for each of them, return the list of groups that have an excessive 47 | number of elements (when compared to a uniform distribution), a list of 48 | groups with insufficient elements, and a list of groups that already have 49 | the optimal number of elements. 50 | 51 | :param list groups: list of group objects 52 | :param func key: function to retrieve the current number of elements from the group object 53 | :param int total: total number of elements to distribute 54 | 55 | Example: 56 | .. code-block:: python 57 | smart_separate_groups([11, 9, 10, 14], lambda g: g) => ([14], [10, 9], [11]) 58 | """ 59 | optimum, extra = compute_optimum(len(groups), total) 60 | over_loaded, under_loaded, optimal = [], [], [] 61 | for group in sorted(groups, key=key, reverse=True): 62 | n_elements = key(group) 63 | additional_element = 1 if extra else 0 64 | if n_elements > optimum + additional_element: 65 | over_loaded.append(group) 66 | elif n_elements == optimum + additional_element: 67 | optimal.append(group) 68 | elif n_elements < optimum + additional_element: 69 | under_loaded.append(group) 70 | extra -= additional_element 71 | return over_loaded, under_loaded, optimal 72 | 73 | 74 | def separate_groups(groups, key, total): 75 | """Separate the group into overloaded and under-loaded groups. 76 | 77 | The revised over-loaded groups increases the choice space for future 78 | selection of most suitable group based on search criteria. 79 | 80 | For example: 81 | Given the groups (a:4, b:4, c:3, d:2) where the number represents the number 82 | of elements for each group. 83 | smart_separate_groups sets 'a' and 'c' as optimal, 'b' as over-loaded 84 | and 'd' as under-loaded. 85 | 86 | separate-groups combines 'a' with 'b' as over-loaded, allowing to select 87 | between these two groups to transfer the element to 'd'. 88 | 89 | :param groups: list of groups 90 | :param key: function to retrieve element count from group 91 | :param total: total number of elements to distribute 92 | :returns: sorted lists of over loaded (descending) and under 93 | loaded (ascending) group 94 | """ 95 | optimum, _ = compute_optimum(len(groups), total) 96 | over_loaded, under_loaded, optimal = _smart_separate_groups(groups, key, total) 97 | # If every group is optimal return 98 | if not over_loaded: 99 | return over_loaded, under_loaded 100 | # Some groups in optimal may have a number of elements that is optimum + 1. 101 | # In this case they should be considered over_loaded. 102 | potential_over_loaded = [ 103 | group for group in optimal 104 | if key(group) > optimum 105 | ] 106 | revised_over_loaded = over_loaded + potential_over_loaded 107 | return ( 108 | sorted(revised_over_loaded, key=key, reverse=True), 109 | sorted(under_loaded, key=key), 110 | ) 111 | -------------------------------------------------------------------------------- /kafka_utils/kafka_consumer_manager/commands/offset_set.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import print_function 17 | from __future__ import unicode_literals 18 | 19 | import sys 20 | from collections import defaultdict 21 | 22 | from .offset_manager import OffsetWriter 23 | from kafka_utils.util.client import KafkaToolClient 24 | from kafka_utils.util.offsets import set_consumer_offsets 25 | 26 | 27 | class OffsetSet(OffsetWriter): 28 | new_offsets_dict = defaultdict(dict) 29 | 30 | @classmethod 31 | def topics_dict(cls, string): 32 | try: 33 | topic, partition_offset = string.rsplit(".", 1) 34 | partition, offset = partition_offset.split("=", 1) 35 | cls.new_offsets_dict[topic][int(partition)] = int(offset) 36 | except ValueError: 37 | print( 38 | "Error: Badly formatted input, please re-run command " 39 | "with --help option.", file=sys.stderr 40 | ) 41 | sys.exit(1) 42 | 43 | @classmethod 44 | def add_parser(cls, subparsers): 45 | parser_offset_set = subparsers.add_parser( 46 | "offset_set", 47 | description="Modify consumer offsets for the specified consumer " 48 | "group to the specified offset.", 49 | add_help=False, 50 | ) 51 | parser_offset_set.add_argument( 52 | "-h", "--help", action="help", 53 | help="Show this help message and exit.", 54 | ) 55 | parser_offset_set.add_argument( 56 | 'groupid', 57 | help="Consumer Group ID whose consumer offsets shall be modified.", 58 | ) 59 | 60 | parser_offset_set.add_argument( 61 | "newoffsets", nargs='+', metavar=('.='), 62 | type=cls.topics_dict, 63 | help="Tuple containing the Kafka topic, partition and " 64 | "the the intended " 65 | "new offset.", 66 | ) 67 | parser_offset_set.add_argument( 68 | '--storage', choices=['zookeeper', 'kafka'], 69 | help="String describing where to store the committed offsets.", 70 | ) 71 | parser_offset_set.add_argument( 72 | '--force', 73 | action='store_true', 74 | help="Force the offset of the group to be committed even if " 75 | "it does not already exist.", 76 | ) 77 | 78 | parser_offset_set.set_defaults(command=cls.run) 79 | 80 | @classmethod 81 | def run(cls, args, cluster_config): 82 | # Setup the Kafka client 83 | client = KafkaToolClient(cluster_config.broker_list) 84 | client.load_metadata_for_topics() 85 | 86 | # Let's verify that the consumer does exist in Zookeeper 87 | if not args.force: 88 | cls.get_topics_from_consumer_group_id( 89 | cluster_config, 90 | args.groupid, 91 | ) 92 | 93 | try: 94 | results = set_consumer_offsets( 95 | client, 96 | args.groupid, 97 | cls.new_offsets_dict, 98 | offset_storage=args.storage, 99 | ) 100 | except TypeError: 101 | print( 102 | "Error: Badly formatted input, please re-run command " 103 | "with --help option.", file=sys.stderr 104 | ) 105 | raise 106 | 107 | client.close() 108 | 109 | if results: 110 | final_error_str = ("Error: Unable to commit consumer offsets for:\n") 111 | for result in results: 112 | error_str = ( 113 | " Topic: {topic} Partition: {partition} Error: {error}\n".format( 114 | topic=result.topic, 115 | partition=result.partition, 116 | error=result.error 117 | ) 118 | ) 119 | final_error_str += error_str 120 | print(final_error_str, file=sys.stderr) 121 | sys.exit(1) 122 | -------------------------------------------------------------------------------- /kafka_utils/kafka_consumer_manager/util.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from __future__ import absolute_import 16 | from __future__ import print_function 17 | from __future__ import unicode_literals 18 | 19 | import sys 20 | from collections import defaultdict 21 | 22 | from kazoo.exceptions import NodeExistsError 23 | 24 | 25 | def preprocess_topics(source_groupid, source_topics, dest_groupid, topics_dest_group): 26 | """Pre-process the topics in source and destination group for duplicates.""" 27 | # Is the new consumer already subscribed to any of these topics? 28 | common_topics = [topic for topic in topics_dest_group if topic in source_topics] 29 | if common_topics: 30 | print( 31 | "Error: Consumer Group ID: {groupid} is already " 32 | "subscribed to following topics: {topic}.\nPlease delete this " 33 | "topics from new group before re-running the " 34 | "command.".format( 35 | groupid=dest_groupid, 36 | topic=', '.join(common_topics), 37 | ), 38 | file=sys.stderr, 39 | ) 40 | sys.exit(1) 41 | # Let's confirm what the user intends to do. 42 | if topics_dest_group: 43 | in_str = ( 44 | "New Consumer Group: {dest_groupid} already " 45 | "exists.\nTopics subscribed to by the consumer groups are listed " 46 | "below:\n{source_groupid}: {source_group_topics}\n" 47 | "{dest_groupid}: {dest_group_topics}\nDo you intend to copy into" 48 | "existing consumer destination-group? (y/n)".format( 49 | source_groupid=source_groupid, 50 | source_group_topics=source_topics, 51 | dest_groupid=dest_groupid, 52 | dest_group_topics=topics_dest_group, 53 | ) 54 | ) 55 | prompt_user_input(in_str) 56 | 57 | 58 | def create_offsets(zk, consumer_group, offsets): 59 | """Create path with offset value for each topic-partition of given consumer 60 | group. 61 | 62 | :param zk: Zookeeper client 63 | :param consumer_group: Consumer group id for given offsets 64 | :type consumer_group: int 65 | :param offsets: Offsets of all topic-partitions 66 | :type offsets: dict(topic, dict(partition, offset)) 67 | """ 68 | # Create new offsets 69 | for topic, partition_offsets in offsets.iteritems(): 70 | for partition, offset in partition_offsets.iteritems(): 71 | new_path = "/consumers/{groupid}/offsets/{topic}/{partition}".format( 72 | groupid=consumer_group, 73 | topic=topic, 74 | partition=partition, 75 | ) 76 | try: 77 | zk.create(new_path, value=bytes(offset), makepath=True) 78 | except NodeExistsError: 79 | print( 80 | "Error: Path {path} already exists. Please re-run the " 81 | "command.".format(path=new_path), 82 | file=sys.stderr, 83 | ) 84 | raise 85 | 86 | 87 | def fetch_offsets(zk, consumer_group, topics): 88 | """Fetch offsets for given topics of given consumer group. 89 | 90 | :param zk: Zookeeper client 91 | :param consumer_group: Consumer group id for given offsets 92 | :type consumer_group: int 93 | :rtype: dict(topic, dict(partition, offset)) 94 | """ 95 | source_offsets = defaultdict(dict) 96 | for topic, partitions in topics.iteritems(): 97 | for partition in partitions: 98 | offset, _ = zk.get( 99 | "/consumers/{groupid}/offsets/{topic}/{partition}".format( 100 | groupid=consumer_group, 101 | topic=topic, 102 | partition=partition, 103 | ) 104 | ) 105 | source_offsets[topic][partition] = offset 106 | return source_offsets 107 | 108 | 109 | def prompt_user_input(in_str): 110 | while(True): 111 | answer = raw_input(in_str + ' ') 112 | if answer == "n" or answer == "no": 113 | sys.exit(0) 114 | if answer == "y" or answer == "yes": 115 | return 116 | -------------------------------------------------------------------------------- /kafka_utils/kafka_check/commands/command.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2016 Yelp Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | from kafka_utils.kafka_check import status_code 16 | from kafka_utils.kafka_check.status_code import terminate 17 | from kafka_utils.util.zookeeper import ZK 18 | 19 | 20 | class KafkaCheckCmd(object): 21 | """Interface used by all kafka_check commands 22 | The attributes cluster_config, args and zk are initialized on run(). 23 | """ 24 | 25 | def __init__(self): 26 | self.cluster_config = None 27 | self.args = None 28 | self.zk = None 29 | 30 | def build_subparser(self, subparsers): 31 | """Build the command subparser. 32 | 33 | :param subparsers: argpars subparsers 34 | :returns: subparser 35 | """ 36 | raise NotImplementedError("Implement in subclass") 37 | 38 | def run_command(self): 39 | """Implement the command logic. 40 | When run_command is called cluster_config, args, and zk are already 41 | initialized. 42 | """ 43 | raise NotImplementedError("Implement in subclass") 44 | 45 | def run(self, cluster_config, args): 46 | self.cluster_config = cluster_config 47 | self.args = args 48 | with ZK(self.cluster_config) as self.zk: 49 | if args.controller_only: 50 | check_run_on_controller(self.zk, self.args) 51 | return self.run_command() 52 | 53 | def add_subparser(self, subparsers): 54 | self.build_subparser(subparsers).set_defaults(command=self.run) 55 | 56 | 57 | def get_controller_id(zk): 58 | return zk.get_json('/controller').get('brokerid') 59 | 60 | 61 | def parse_meta_properties_file(content): 62 | for line in content: 63 | parts = line.rstrip().split("=") 64 | if len(parts) == 2 and parts[0] == "broker.id": 65 | return int(parts[1]) 66 | return None 67 | 68 | 69 | def read_generated_broker_id(meta_properties_path): 70 | """reads broker_id from meta.properties file. 71 | 72 | :param string meta_properties_path: path for meta.properties file 73 | :returns int: broker_id from meta_properties_path 74 | """ 75 | try: 76 | with open(meta_properties_path, 'r') as f: 77 | broker_id = parse_meta_properties_file(f) 78 | except IOError: 79 | terminate( 80 | status_code.WARNING, 81 | "Cannot open meta.properties file: {path}".format( 82 | path=meta_properties_path, 83 | ), 84 | ) 85 | except ValueError: 86 | terminate(status_code.WARNING, "Broker id not valid") 87 | 88 | if broker_id is None: 89 | terminate(status_code.WARNING, "Autogenerated broker id missing from data directory") 90 | 91 | return broker_id 92 | 93 | 94 | def get_broker_id(data_path): 95 | """This function will look into the data folder to get the automatically created 96 | broker_id. 97 | 98 | :param string data_path: the path to the kafka data folder 99 | :returns int: the real broker_id 100 | """ 101 | 102 | # Path to the meta.properties file. This is used to read the automatic broker id 103 | # if the given broker id is -1 104 | META_FILE_PATH = "{data_path}/meta.properties" 105 | 106 | if not data_path: 107 | terminate(status_code.WARNING, "You need to specify the data_path if broker_id == -1") 108 | meta_properties_path = META_FILE_PATH.format(data_path=data_path) 109 | return read_generated_broker_id(meta_properties_path) 110 | 111 | 112 | def check_run_on_controller(zk, args): 113 | """Kafka 0.9 supports automatic broker ids. If args.broker_id is set to -1, 114 | it will call get_broker_id() for parse broker_id from meta.properties file. 115 | """ 116 | if args.broker_id is None: 117 | terminate(status_code.WARNING, "Broker id is not specified") 118 | 119 | if args.broker_id != -1: 120 | broker_id = args.broker_id 121 | else: 122 | broker_id = get_broker_id(args.data_path) 123 | 124 | controller_id = get_controller_id(zk) 125 | 126 | # This check is only executed by the controller 127 | if broker_id != controller_id: 128 | terminate(status_code.OK, 'Broker %s is not the controller, nothing to check' % (broker_id)) 129 | --------------------------------------------------------------------------------