├── .gitattributes ├── src ├── arm │ ├── HDInsight4.0 │ │ ├── hdinsight-kafka-2.1-vnet │ │ │ ├── azuredeploy.parameters.json │ │ │ ├── README.md │ │ │ └── azuredeploy.json │ │ ├── hdinsight-kafka-2.1-hadoop-3.1-vnet │ │ │ ├── azuredeploy.parameters.json │ │ │ ├── README.md │ │ │ └── azuredeploy.json │ │ └── hdinsight-kafka-2.1-spark-2.4-vnet │ │ │ ├── azuredeploy.parameters.json │ │ │ ├── README.md │ │ │ └── azuredeploy.json │ ├── HDInsight5.0 │ │ ├── hdinsight-kafka-2.4-vnet │ │ │ ├── azuredeploy.parameters.json │ │ │ ├── README.md │ │ │ └── azuredeploy.json │ │ └── hdinsight-kafka-2.4-spark-3.1-vnet │ │ │ ├── azuredeploy.parameters.json │ │ │ ├── README.md │ │ │ └── azuredeploy.json │ └── README.md ├── python │ ├── troubleshooting │ │ ├── kafka_topic_describe.py │ │ ├── kafka_get_pid_status.py │ │ ├── kafka_restart_controller.py │ │ ├── run_custom_commands.py │ │ ├── kafka_restart_brokers.py │ │ ├── kafka_broker_status.py │ │ ├── README.MD │ │ ├── kafka_perf_test.py │ │ └── kafka_utils.py │ ├── metrics │ │ └── metrics.py │ └── rebalance │ │ ├── README.md │ │ ├── test_rebalance_rackaware.py │ │ └── rebalance_rackaware.py └── powershell │ ├── Create-Kafka2.1Cluster.ps1 │ ├── Create-Kafka2.4Cluster.ps1 │ ├── Create-Kafka2.1Cluster-ExistingVnet.ps1 │ └── Create-Kafka2.4Cluster-ExistingVnet.ps1 ├── .gitignore └── README.md /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /src/arm/HDInsight4.0/hdinsight-kafka-2.1-vnet/azuredeploy.parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "clusterName": { 6 | "value": "" 7 | }, 8 | "clusterSize": { 9 | "value": 4 10 | }, 11 | "clusterLoginPassword": { 12 | "value": "" 13 | }, 14 | "sshUserName": { 15 | "value": "" 16 | }, 17 | "sshPassword": { 18 | "value": "" 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /src/arm/HDInsight5.0/hdinsight-kafka-2.4-vnet/azuredeploy.parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "clusterName": { 6 | "value": "" 7 | }, 8 | "clusterSize": { 9 | "value": 4 10 | }, 11 | "clusterLoginPassword": { 12 | "value": "" 13 | }, 14 | "sshUserName": { 15 | "value": "" 16 | }, 17 | "sshPassword": { 18 | "value": "" 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /src/arm/HDInsight4.0/hdinsight-kafka-2.1-hadoop-3.1-vnet/azuredeploy.parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "clusterNameSuffix": { 6 | "value": "" 7 | }, 8 | "clusterSize": { 9 | "value": 4 10 | }, 11 | "clusterLoginPassword": { 12 | "value": "" 13 | }, 14 | "sshUserName": { 15 | "value": "" 16 | }, 17 | "sshPassword": { 18 | "value": "" 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /src/arm/HDInsight4.0/hdinsight-kafka-2.1-spark-2.4-vnet/azuredeploy.parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "clusterNameSuffix": { 6 | "value": "" 7 | }, 8 | "clusterSize": { 9 | "value": 4 10 | }, 11 | "clusterLoginPassword": { 12 | "value": "" 13 | }, 14 | "sshUserName": { 15 | "value": "" 16 | }, 17 | "sshPassword": { 18 | "value": "" 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /src/arm/HDInsight5.0/hdinsight-kafka-2.4-spark-3.1-vnet/azuredeploy.parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "clusterNameSuffix": { 6 | "value": "" 7 | }, 8 | "clusterSize": { 9 | "value": 4 10 | }, 11 | "clusterLoginPassword": { 12 | "value": "" 13 | }, 14 | "sshUserName": { 15 | "value": "" 16 | }, 17 | "sshPassword": { 18 | "value": "" 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Windows image file caches 2 | Thumbs.db 3 | ehthumbs.db 4 | 5 | # Folder config file 6 | Desktop.ini 7 | 8 | # Recycle Bin used on file shares 9 | $RECYCLE.BIN/ 10 | 11 | # Windows Installer files 12 | *.cab 13 | *.msi 14 | *.msm 15 | *.msp 16 | 17 | # Windows shortcuts 18 | *.lnk 19 | 20 | # ========================= 21 | # Operating System Files 22 | # ========================= 23 | 24 | # OSX 25 | # ========================= 26 | 27 | .DS_Store 28 | .AppleDouble 29 | .LSOverride 30 | 31 | # Thumbnails 32 | ._* 33 | 34 | # Files that might appear on external disk 35 | .Spotlight-V100 36 | .Trashes 37 | 38 | # Directories potentially created on remote AFP share 39 | .AppleDB 40 | .AppleDesktop 41 | Network Trash Folder 42 | Temporary Items 43 | .apdisk 44 | 45 | .pyc -------------------------------------------------------------------------------- /src/arm/HDInsight4.0/hdinsight-kafka-2.1-vnet/README.md: -------------------------------------------------------------------------------- 1 | # Deploy a Linux-based HDInsight Kafka cluster with managed disks in a VNET. 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | This template allows you to create a Linux-based HDInsight Kafka cluster 2.1 (HDI 4.0) with managed disks in a VNET. -------------------------------------------------------------------------------- /src/arm/HDInsight5.0/hdinsight-kafka-2.4-vnet/README.md: -------------------------------------------------------------------------------- 1 | # Deploy a Linux-based HDInsight Kafka cluster with managed disks in a VNET. 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | This template allows you to create a Linux-based HDInsight Kafka cluster 2.4 (HDI 5.0) with managed disks in a VNET. This is not a recommended HDI version of Kafka 2.4 version. -------------------------------------------------------------------------------- /src/arm/HDInsight5.0/hdinsight-kafka-2.4-spark-3.1-vnet/README.md: -------------------------------------------------------------------------------- 1 | # Deploy a Linux-based HDInsight Kafka cluster with managed disks and HDInsight Spark cluster in the same virtual network. 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | This template allows you to create a Linux-based HDInsight Kafka cluster 2.4 (HDI 5.0) with managed disks and a Linux-based HDInsight Spark cluster 3.1 (HDI 5.0) in the same virtual network. -------------------------------------------------------------------------------- /src/arm/HDInsight4.0/hdinsight-kafka-2.1-spark-2.4-vnet/README.md: -------------------------------------------------------------------------------- 1 | # Deploy a Linux-based HDInsight Kafka cluster with managed disks and HDInsight Spark cluster in the same virtual network. 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | This template allows you to create a Linux-based HDInsight Kafka cluster 2.1 (HDI 4.0) with managed disks and a Linux-based HDInsight Spark cluster 2.4 (HDI 4.0) in the same virtual network. -------------------------------------------------------------------------------- /src/arm/HDInsight4.0/hdinsight-kafka-2.1-hadoop-3.1-vnet/README.md: -------------------------------------------------------------------------------- 1 | # Deploy a Linux-based HDInsight Kafka cluster with managed disks and HDInsight Hadoop cluster in the same virtual network. 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | This template allows you to create a Linux-based HDInsight Kafka cluster 2.1 (HDI 4.0) with managed disks and a Linux-based HDInsight Hadoop cluster 3.1 (HDI 4.0) in the same virtual network. -------------------------------------------------------------------------------- /src/python/troubleshooting/kafka_topic_describe.py: -------------------------------------------------------------------------------- 1 | import logging, sys, time 2 | from kafka_utils import KafkaUtils 3 | 4 | logger = logging.getLogger(__name__) 5 | 6 | 7 | def main(utils): 8 | topic_name = '' 9 | if len(sys.argv) > 1: 10 | topic_name = sys.argv[1] 11 | logger.info('topic_name = ' + topic_name) 12 | logger.info('Listing topics') 13 | kafka_version, hdi_version = utils.get_kafka_hdp_version() 14 | if kafka_version >= '3.2.0': 15 | (_, broker_server) = utils.get_brokers_from_ambari() 16 | logger.info(broker_server) 17 | shell_command = "/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --describe --bootstrap-server {0}".format( 18 | broker_server) 19 | else: 20 | zookeeper_quorum = utils.get_zookeeper_quorum() 21 | shell_command = "/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --describe --zookeeper {0}".format( 22 | zookeeper_quorum) 23 | if topic_name: 24 | shell_command += " --topic {0}".format(topic_name) 25 | 26 | stdout, stderr = utils.run_shell_command(shell_command) 27 | 28 | 29 | if __name__ == '__main__': 30 | utils = KafkaUtils(logger, "kafkatopicdescribe{0}.log".format(int(time.time()))) 31 | main(utils) 32 | -------------------------------------------------------------------------------- /src/python/troubleshooting/kafka_get_pid_status.py: -------------------------------------------------------------------------------- 1 | """Script to report the status of the Kafka process on this current host. 2 | """ 3 | import datetime 4 | import os 5 | import psutil 6 | 7 | kafka_pid_file = '/var/run/kafka/kafka.pid' 8 | 9 | if os.path.exists(kafka_pid_file): 10 | with open(kafka_pid_file, "r") as kpf: 11 | kafka_pid = kpf.readline().strip() 12 | int_kafka_pid = int(kafka_pid) 13 | if int_kafka_pid > 0 and psutil.pid_exists(int_kafka_pid): 14 | kafka_process = psutil.Process(int_kafka_pid) 15 | kafka_process_status = kafka_process.status() 16 | 17 | # Reboot if Kafka process has been zombie for more than max_kafka_zombie_secs seconds 18 | if kafka_process_status == psutil.STATUS_ZOMBIE: 19 | print('ERROR: Kafka process with pid [{0}] is in a zombie state: {1}'. 20 | format(kafka_pid, kafka_process_status)) 21 | else: 22 | print('SUCCESS: Verification of Kafka successful! Kafka pid: {0}, status: {1}, create_time: {2}'. 23 | format(kafka_pid, kafka_process_status, 24 | datetime.datetime.fromtimestamp(kafka_process.create_time()).strftime("%Y-%m-%d %H:%M:%S"))) 25 | else: 26 | print('ERROR: Kafka is not running. Invalid pid file: {0}, pid: {1}'.format(kafka_pid_file, int_kafka_pid)) 27 | else: 28 | print('ERROR: Kafka pid file: {0} not found, Kafka is not running'.format(kafka_pid_file)) 29 | -------------------------------------------------------------------------------- /src/python/troubleshooting/kafka_restart_controller.py: -------------------------------------------------------------------------------- 1 | """Script to restart the current Kafka Broker which is acting as the Controller 2 | """ 3 | import logging, pprint, time 4 | 5 | from kafka_utils import KafkaUtils 6 | from kafka_broker_status import get_kafka_broker_status, get_kafka_controller_status, str_kafka_controller_status 7 | 8 | logger = logging.getLogger(__name__) 9 | debug = False 10 | 11 | def main(utils): 12 | broker_hosts, zk_brokers, dead_broker_hosts = get_kafka_broker_status(utils) 13 | zk_controller = get_kafka_controller_status(utils, broker_hosts, zk_brokers) 14 | logger.info(str_kafka_controller_status(zk_controller)) 15 | logger.info('Restarting Kafka Controller at broker id {0} on host {1}'.format(zk_controller['controller_id'], zk_controller['controller_host'])) 16 | response = utils.restart_kafka_broker_from_ambari(zk_controller['controller_host']) 17 | logger.debug(pprint.pformat(response)) 18 | logger.info('Successfully restarted Kafka Controller at broker id {0} on host {1}'.format(zk_controller['controller_id'], zk_controller['controller_host'])) 19 | zk_controller = get_kafka_controller_status(utils, broker_hosts, zk_brokers) 20 | logger.info('\nNEW Kafka Controller is at broker id {0} on host {1}'.format(zk_controller['controller_id'], zk_controller['controller_host'])) 21 | logger.info(str_kafka_controller_status(zk_controller)) 22 | 23 | if __name__ == '__main__': 24 | utils = KafkaUtils(logger, "kafkarestartcontroller{0}.log".format(int(time.time())), debug) 25 | main(utils) 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HDInsight Kafka Tools 2 | 3 | This repository contains scripts, tools and ARM templates for Apache Kafka on HDInsight clusters. 4 | Please refer to the table below for different resources in this repository. 5 | 6 | | Location | Description | 7 | |-----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------| 8 | | [ARM Templates](src/arm/) | Azure ARM templates for HDInsight Kafka 2.1 and 2.4 and various other cluster types that can be deployed in the same VNET along with HDInsight Kafka. | 9 | | [Powershell](src/powershell) | Powershell scripts to create HDInsight Kafka 2.1 and 2.4 clusters. | 10 | | [Kafka Rebalance](src/python/rebalance) | Python script to rebalance (re-assign) Kafka Topics and Partitions across different Azure Fault Domains and Upgrade Domains for high availability. | 11 | | [Kafka Troubleshooting](src/python/troubleshooting) | Python scripts to check the status of Kafka brokers and restart brokers based on their health. | 12 | 13 | ## Other HDInsight Kafka Resources 14 | * [HDInsight Kafka - Getting Started](https://docs.microsoft.com/en-us/azure/hdinsight/kafka/apache-kafka-get-started) 15 | * [HDInsight FAQs](https://hdinsight.github.io/) 16 | * [HDInsight Kafka FAQs](https://hdinsight.github.io/kafka/kafka-landing) 17 | 18 | For feedback, please open new issues or write to us at hdikafka at microsoft dot com. -------------------------------------------------------------------------------- /src/arm/README.md: -------------------------------------------------------------------------------- 1 | # HDInsight Kafka ARM Templates 2 | 3 | This directory contains ARM templates for Apache Kafka on HDInsight clusters. 4 | 5 | | Directory | Description | 6 | |-----------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------| 7 | | [HDInsight Kafka 2.1 VNET](HDInsight4.0/hdinsight-kafka-2.1-vnet) | ARM Template for Linux-based HDInsight Kafka cluster 2.1 (HDI 4.0) with managed disks. | 8 | | [HDInsight Kafka 2.1 Spark 2.4 VNET](HDInsight4.0/hdinsight-kafka-2.1-spark-2.4-vnet) | ARM Template for Linux-based HDInsight Kafka cluster 2.1 (HDI 4.0) with managed disks and HDInsight Spark cluster 2.4 (HDI 4.0) in the same VNET. | 9 | | [HDInsight Kafka 2.1 Hadoop 3.1 VNET](HDInsight4.0/hdinsight-kafka-2.1-hadoop-3.1-vnet) | ARM Template for Linux-based HDInsight Kafka cluster 2.1 (HDI 4.0) with managed disks and HDInsight Hadoop cluster 3.1 (HDI 4.0) in the same VNET. | 10 | | [HDInsight Kafka 2.4 VNET](HDInsight5.0/hdinsight-kafka-2.4-vnet) | ARM Template for Linux-based HDInsight Kafka cluster 2.4 (HDI 5.0) with managed disks. This HDI version is recommended for Kafka 2.4 version. | 11 | | [HDInsight Kafka 2.4 Spark 2.4 VNET](HDInsight5.0/hdinsight-kafka-2.4-spark-3.1-vnet) | ARM Template for Linux-based HDInsight Kafka cluster 2.4 (HDI 5.0) with managed disks and HDInsight Spark cluster 3.1 (HDI 5.0) in the same VNET. | 12 | 13 | Additional HDInsight ARM templates can be found at: 14 | * [Azure Quickstart Templates](https://github.com/Azure/azure-quickstart-templates) 15 | * [Azure Quickstart Templates for HDInsight](https://azure.microsoft.com/en-us/resources/templates/?term=hdinsight) 16 | -------------------------------------------------------------------------------- /src/powershell/Create-Kafka2.1Cluster.ps1: -------------------------------------------------------------------------------- 1 | # This script is used to create a Hdinsight Kafka cluster. The script creates a resourge group, WASB storage account, storage account. 2 | # If these resources already exist, you can comment out the corresponding lines that create new resources. 3 | 4 | # Select the subscription to use 5 | $subscriptionID = "" 6 | 7 | # Cluster details 8 | $hdiversion = "4.0" 9 | $token ="" 10 | # Resource Group Name 11 | $resourceGroupName = $token + "rg" 12 | # Location 13 | $location = "West US" 14 | # Cluster Dns Name 15 | $clusterName = $token 16 | # Default WASB storage account associated with the cluster 17 | $defaultStorageAccountName = $token + "store" 18 | # Default container Name 19 | $defaultStorageContainerName = $token + "container" 20 | # Number of worker nodes (brokers) in the clusters 21 | $clusterNodes = 1 22 | 23 | $clusterCredential = Get-Credential -Message "Enter Cluster user credentials" -UserName "admin" 24 | $clusterSshCredential = Get-Credential -Message "Enter SSH user credentials" 25 | 26 | # Sign in to Azure 27 | Login-AzureRmAccount 28 | 29 | Select-AzureRmSubscription -SubscriptionId $subscriptionID 30 | 31 | # Create an Azure Resource Group 32 | New-AzureRmResourceGroup -Name $resourceGroupName -Location $location 33 | 34 | # Create an Azure Storage account and container used as the default storage 35 | New-AzureRmStorageAccount ` 36 | -ResourceGroupName $resourceGroupName ` 37 | -StorageAccountName $defaultStorageAccountName ` 38 | -Location $location ` 39 | -Type Standard_LRS 40 | $defaultStorageAccountKey = (Get-AzureRmStorageAccountKey -ResourceGroupName $resourceGroupName -Name $defaultStorageAccountName)[0].Value 41 | $destContext = New-AzureStorageContext -StorageAccountName $defaultStorageAccountName -StorageAccountKey $defaultStorageAccountKey 42 | New-AzureStorageContainer -Name $defaultStorageContainerName -Context $destContext 43 | 44 | # The location of the HDInsight cluster must be in the same data center as the Storage account. 45 | $location = Get-AzureRmStorageAccount -ResourceGroupName $resourceGroupName -StorageAccountName $defaultStorageAccountName | %{$_.Location} 46 | 47 | New-AzureRmHDInsightCluster ` 48 | -ClusterName $clusterName ` 49 | -ResourceGroupName $resourceGroupName ` 50 | -HttpCredential $clusterCredential ` 51 | -SshCredential $clusterSshCredential ` 52 | -Location $location ` 53 | -DefaultStorageAccountName "$defaultStorageAccountName.blob.core.windows.net" ` 54 | -DefaultStorageAccountKey $defaultStorageAccountKey ` 55 | -DefaultStorageContainer $defaultStorageContainerName ` 56 | -ClusterSizeInNodes $clusterNodes ` 57 | -ClusterType Kafka ` 58 | -OSType Linux ` 59 | -Version $hdiversion ` 60 | -HeadNodeSize "Standard_D3" ` 61 | -WorkerNodeSize "Standard_D3" -------------------------------------------------------------------------------- /src/powershell/Create-Kafka2.4Cluster.ps1: -------------------------------------------------------------------------------- 1 | # This script is used to create a Hdinsight Kafka cluster. The script creates a resourge group, WASB storage account, storage account. 2 | # If these resources already exist, you can comment out the corresponding lines that create new resources. 3 | 4 | # Select the subscription to use 5 | $subscriptionID = "" 6 | 7 | # Cluster details 8 | $hdiversion = "5.0" 9 | $token ="" 10 | # Resource Group Name 11 | $resourceGroupName = $token + "rg" 12 | # Location 13 | $location = "West US" 14 | # Cluster Dns Name 15 | $clusterName = $token 16 | # Default WASB storage account associated with the cluster 17 | $defaultStorageAccountName = $token + "store" 18 | # Default container Name 19 | $defaultStorageContainerName = $token + "container" 20 | # Number of worker nodes (brokers) in the clusters 21 | $clusterNodes = 1 22 | 23 | $clusterCredential = Get-Credential -Message "Enter Cluster user credentials" -UserName "admin" 24 | $clusterSshCredential = Get-Credential -Message "Enter SSH user credentials" 25 | 26 | # Sign in to Azure 27 | Login-AzureRmAccount 28 | 29 | Select-AzureRmSubscription -SubscriptionId $subscriptionID 30 | 31 | # Create an Azure Resource Group 32 | New-AzureRmResourceGroup -Name $resourceGroupName -Location $location 33 | 34 | # Create an Azure Storage account and container used as the default storage 35 | New-AzureRmStorageAccount ` 36 | -ResourceGroupName $resourceGroupName ` 37 | -StorageAccountName $defaultStorageAccountName ` 38 | -Location $location ` 39 | -Type Standard_LRS 40 | $defaultStorageAccountKey = (Get-AzureRmStorageAccountKey -ResourceGroupName $resourceGroupName -Name $defaultStorageAccountName)[0].Value 41 | $destContext = New-AzureStorageContext -StorageAccountName $defaultStorageAccountName -StorageAccountKey $defaultStorageAccountKey 42 | New-AzureStorageContainer -Name $defaultStorageContainerName -Context $destContext 43 | 44 | # The location of the HDInsight cluster must be in the same data center as the Storage account. 45 | $location = Get-AzureRmStorageAccount -ResourceGroupName $resourceGroupName -StorageAccountName $defaultStorageAccountName | %{$_.Location} 46 | 47 | New-AzureRmHDInsightCluster ` 48 | -ClusterName $clusterName ` 49 | -ResourceGroupName $resourceGroupName ` 50 | -HttpCredential $clusterCredential ` 51 | -SshCredential $clusterSshCredential ` 52 | -Location $location ` 53 | -DefaultStorageAccountName "$defaultStorageAccountName.blob.core.windows.net" ` 54 | -DefaultStorageAccountKey $defaultStorageAccountKey ` 55 | -DefaultStorageContainer $defaultStorageContainerName ` 56 | -ClusterSizeInNodes $clusterNodes ` 57 | -ClusterType Kafka ` 58 | -OSType Linux ` 59 | -Version $hdiversion ` 60 | -HeadNodeSize "Standard_D3" ` 61 | -WorkerNodeSize "Standard_D3" -------------------------------------------------------------------------------- /src/powershell/Create-Kafka2.1Cluster-ExistingVnet.ps1: -------------------------------------------------------------------------------- 1 | # This script is used to create a Hdinsight Kafka cluster. The script creates a resourge group, WASB storage account, storage accoun in an existing virtual network. 2 | # If these resources already exist, you can comment out the corresponding lines that create new resources. 3 | 4 | # Select the subscription to use 5 | $subscriptionID = "" 6 | 7 | # Cluster details 8 | $hdiversion = "4.0" 9 | $token ="" 10 | # Resource Group Name 11 | $resourceGroupName = $token + "rg" 12 | # Location 13 | $location = "West US" 14 | # Cluster Dns Name 15 | $clusterName = $token 16 | # Default WASB storage account associated with the cluster 17 | $defaultStorageAccountName = $token + "store" 18 | # Default container Name 19 | $defaultStorageContainerName = $token + "container" 20 | # Number of worker nodes (brokers) in the clusters 21 | $clusterNodes = 1 22 | # Existing virtual network's Id. Format: 23 | # "/subscriptions/YOUR_SUBSCRIPTION_ID/resourceGroups/RESOURCE_GROUP_NAME/providers/microsoft.network/virtualNetworks/VIRTUAL_NETWORK_NAME", 24 | $virtualNetworkId = "VIRTUAL_NETWORK_ID" 25 | # Subnet Name. Format: 26 | # "/subscriptions/YOUR_SUBSCRIPTION_ID/resourceGroups/RESOURCE_GROUP_NAME/providers/microsoft.network/virtualNetworks/VIRTUAL_NETWORK_NAME/subnets/SUBNET_NAME" 27 | $subnetName = "SUBNET_NAME" 28 | 29 | $clusterCredential = Get-Credential -Message "Enter Cluster user credentials" -UserName "admin" 30 | $clusterSshCredential = Get-Credential -Message "Enter SSH user credentials" 31 | 32 | # Sign in to Azure 33 | Login-AzureRmAccount 34 | 35 | Select-AzureRmSubscription -SubscriptionId $subscriptionID 36 | 37 | # Create an Azure Resource Group 38 | New-AzureRmResourceGroup -Name $resourceGroupName -Location $location 39 | 40 | # Create an Azure Storage account and container used as the default storage 41 | New-AzureRmStorageAccount ` 42 | -ResourceGroupName $resourceGroupName ` 43 | -StorageAccountName $defaultStorageAccountName ` 44 | -Location $location ` 45 | -Type Standard_LRS 46 | $defaultStorageAccountKey = (Get-AzureRmStorageAccountKey -ResourceGroupName $resourceGroupName -Name $defaultStorageAccountName)[0].Value 47 | $destContext = New-AzureStorageContext -StorageAccountName $defaultStorageAccountName -StorageAccountKey $defaultStorageAccountKey 48 | New-AzureStorageContainer -Name $defaultStorageContainerName -Context $destContext 49 | 50 | # The location of the HDInsight cluster must be in the same data center as the Storage account. 51 | $location = Get-AzureRmStorageAccount -ResourceGroupName $resourceGroupName -StorageAccountName $defaultStorageAccountName | %{$_.Location} 52 | 53 | New-AzureRmHDInsightCluster ` 54 | -ClusterName $clusterName ` 55 | -ResourceGroupName $resourceGroupName ` 56 | -HttpCredential $clusterCredential ` 57 | -SshCredential $clusterSshCredential ` 58 | -Location $location ` 59 | -DefaultStorageAccountName "$defaultStorageAccountName.blob.core.windows.net" ` 60 | -DefaultStorageAccountKey $defaultStorageAccountKey ` 61 | -DefaultStorageContainer $defaultStorageContainerName ` 62 | -ClusterSizeInNodes $clusterNodes ` 63 | -ClusterType Kafka ` 64 | -OSType Linux ` 65 | -Version $hdiversion ` 66 | -HeadNodeSize "Standard_D3" ` 67 | -WorkerNodeSize "Standard_D3" ` 68 | -VirtualNetworkId $virtualNetworkId ` 69 | -SubnetName $subnetName -------------------------------------------------------------------------------- /src/powershell/Create-Kafka2.4Cluster-ExistingVnet.ps1: -------------------------------------------------------------------------------- 1 | # This script is used to create a Hdinsight Kafka cluster. The script creates a resourge group, WASB storage account, storage accoun in an existing virtual network. 2 | # If these resources already exist, you can comment out the corresponding lines that create new resources. 3 | 4 | # Select the subscription to use 5 | $subscriptionID = "" 6 | 7 | # Cluster details 8 | $hdiversion = "5.0" 9 | $token ="" 10 | # Resource Group Name 11 | $resourceGroupName = $token + "rg" 12 | # Location 13 | $location = "West US" 14 | # Cluster Dns Name 15 | $clusterName = $token 16 | # Default WASB storage account associated with the cluster 17 | $defaultStorageAccountName = $token + "store" 18 | # Default container Name 19 | $defaultStorageContainerName = $token + "container" 20 | # Number of worker nodes (brokers) in the clusters 21 | $clusterNodes = 1 22 | # Existing virtual network's Id. Format: 23 | # "/subscriptions/YOUR_SUBSCRIPTION_ID/resourceGroups/RESOURCE_GROUP_NAME/providers/microsoft.network/virtualNetworks/VIRTUAL_NETWORK_NAME", 24 | $virtualNetworkId = "VIRTUAL_NETWORK_ID" 25 | # Subnet Name. Format: 26 | # "/subscriptions/YOUR_SUBSCRIPTION_ID/resourceGroups/RESOURCE_GROUP_NAME/providers/microsoft.network/virtualNetworks/VIRTUAL_NETWORK_NAME/subnets/SUBNET_NAME" 27 | $subnetName = "SUBNET_NAME" 28 | 29 | $clusterCredential = Get-Credential -Message "Enter Cluster user credentials" -UserName "admin" 30 | $clusterSshCredential = Get-Credential -Message "Enter SSH user credentials" 31 | 32 | # Sign in to Azure 33 | Login-AzureRmAccount 34 | 35 | Select-AzureRmSubscription -SubscriptionId $subscriptionID 36 | 37 | # Create an Azure Resource Group 38 | New-AzureRmResourceGroup -Name $resourceGroupName -Location $location 39 | 40 | # Create an Azure Storage account and container used as the default storage 41 | New-AzureRmStorageAccount ` 42 | -ResourceGroupName $resourceGroupName ` 43 | -StorageAccountName $defaultStorageAccountName ` 44 | -Location $location ` 45 | -Type Standard_LRS 46 | $defaultStorageAccountKey = (Get-AzureRmStorageAccountKey -ResourceGroupName $resourceGroupName -Name $defaultStorageAccountName)[0].Value 47 | $destContext = New-AzureStorageContext -StorageAccountName $defaultStorageAccountName -StorageAccountKey $defaultStorageAccountKey 48 | New-AzureStorageContainer -Name $defaultStorageContainerName -Context $destContext 49 | 50 | # The location of the HDInsight cluster must be in the same data center as the Storage account. 51 | $location = Get-AzureRmStorageAccount -ResourceGroupName $resourceGroupName -StorageAccountName $defaultStorageAccountName | %{$_.Location} 52 | 53 | New-AzureRmHDInsightCluster ` 54 | -ClusterName $clusterName ` 55 | -ResourceGroupName $resourceGroupName ` 56 | -HttpCredential $clusterCredential ` 57 | -SshCredential $clusterSshCredential ` 58 | -Location $location ` 59 | -DefaultStorageAccountName "$defaultStorageAccountName.blob.core.windows.net" ` 60 | -DefaultStorageAccountKey $defaultStorageAccountKey ` 61 | -DefaultStorageContainer $defaultStorageContainerName ` 62 | -ClusterSizeInNodes $clusterNodes ` 63 | -ClusterType Kafka ` 64 | -OSType Linux ` 65 | -Version $hdiversion ` 66 | -HeadNodeSize "Standard_D3" ` 67 | -WorkerNodeSize "Standard_D3" ` 68 | -VirtualNetworkId $virtualNetworkId ` 69 | -SubnetName $subnetName -------------------------------------------------------------------------------- /src/python/metrics/metrics.py: -------------------------------------------------------------------------------- 1 | """ 2 | OVERVIEW: 3 | ========= 4 | Get Kafka metrics from Ambari. 5 | 6 | RUNNING THE SCRIPT: 7 | =================== 8 | You need to run this script with sudo privilege due to permission issues on some python packages: 9 | sudo python metrics.py -h 10 | 11 | Tested on HDInsight 3.2 (Linux Ubuntu 12) with HDP 2.2.7.1 and Kafka 0.8.1.1 12 | 13 | REFERENCES: 14 | =========== 15 | https://cwiki.apache.org/confluence/display/AMBARI/Stack+Defined+Metrics 16 | https://cwiki.apache.org/confluence/display/AMBARI/Ambari+Metrics+API+specification 17 | """ 18 | 19 | import logging 20 | import json 21 | import os.path 22 | import calendar 23 | import time 24 | import argparse 25 | 26 | from hdinsight_common import hdinsightlogging 27 | from hdinsight_common.AmbariHelper import AmbariHelper 28 | 29 | logger = logging.getLogger(__name__) 30 | KAFKA_METRICS_DESCRIPTOR_URL = 'stacks/HDP/versions/2.2/services/KAFKA/artifacts/metrics_descriptor' 31 | KAFKA_METRICS_BASE_URL = 'clusters/{0}/services/KAFKA/components/KAFKA_BROKER?fields=' 32 | 33 | AMBARI_METRICS = 'AMBARI_METRICS' 34 | 35 | def execute(temporal): 36 | logger.info('Getting Kafka metrics descriptor from %s', KAFKA_METRICS_DESCRIPTOR_URL) 37 | a = AmbariHelper() 38 | kafka_metrics_descriptor = a.query_url(KAFKA_METRICS_DESCRIPTOR_URL) 39 | clusters_result = a.query_url('clusters') 40 | 41 | kafka_metrics = {} 42 | kafka_metrics_url = KAFKA_METRICS_BASE_URL.format(clusters_result['items'][0]['Clusters']['cluster_name']) 43 | idx = 0 44 | 45 | end_epoch = calendar.timegm(time.gmtime()) 46 | start_epoch = end_epoch - temporal 47 | 48 | for i in kafka_metrics_descriptor['artifact_data']['KAFKA']['KAFKA_BROKER']['Component'][0]['metrics']['default']: 49 | if ((temporal > 0) and kafka_metrics_descriptor['artifact_data']['KAFKA']['KAFKA_BROKER']['Component'][0]['metrics']['default'][i]['temporal']): 50 | metric = i + '[{0},{1},15]'.format(start_epoch, end_epoch) 51 | else: 52 | metric = i 53 | if metric: 54 | if idx > 0: 55 | kafka_metrics_url += ',' 56 | kafka_metrics_url += metric 57 | idx += 1 58 | 59 | if temporal > 0: 60 | kafka_metrics_url += '&_{0}'.format(1000*end_epoch) 61 | 62 | logger.info('Querying Metrics Url: %s', kafka_metrics_url) 63 | kafka_metrics = a.query_url(kafka_metrics_url) 64 | logger.info('Kafka Metrics:\r\n%s', kafka_metrics) 65 | 66 | if 'metrics' in kafka_metrics: 67 | return kafka_metrics 68 | else: 69 | error_msg = 'Kafka metrics unavailable. Please check the status of Ambari Metrics Server or trying doing a temporal query using --temporal parameter.' 70 | logger.error(error_msg) 71 | ambari_metrics_service = a.get_service_info(AMBARI_METRICS) 72 | logger.debug('Ambari metrics service:\r\n%s', ambari_metrics_service) 73 | raise RuntimeError(error_msg) 74 | 75 | def main(): 76 | parser = argparse.ArgumentParser(description='Get Kafka metrics from Ambari') 77 | parser.add_argument('-t', '--temporal', nargs='?', type=int, const='300', default='0', 78 | help='specify the time interval (in seconds) to get temporal metrics (default: last 5 minutes metrics are returned). NOT specifying this argument will return current point-in-time metrics') 79 | 80 | args = parser.parse_args() 81 | 82 | logger.info('Ambari metrics mode - Temporal: %s', args.temporal) 83 | kafka_metrics = execute(args.temporal) 84 | 85 | if __name__ == "__main__": 86 | hdinsightlogging.initialize_root_logger() 87 | main() 88 | -------------------------------------------------------------------------------- /src/python/troubleshooting/run_custom_commands.py: -------------------------------------------------------------------------------- 1 | import argparse, logging, os, sys, time, traceback 2 | from kafka_utils import KafkaUtils 3 | 4 | logger = logging.getLogger(__name__) 5 | debug = False 6 | 7 | def copy_files(broker_host, ssh_password_param, ssh_key_param, ssh_username, src, dest): 8 | stdout, stderr = utils.run_shell_command('{0}scp {1} -o StrictHostKeyChecking=no {4} {2}@{3}:{5}' 9 | .format(ssh_password_param, ssh_key_param, ssh_username, broker_host, src, dest)) 10 | return stdout, stderr 11 | 12 | def run_command(broker_host, ssh_password_param, ssh_key_param, ssh_username, cmd): 13 | stdout, stderr = utils.run_shell_command('{0}ssh {1} -o StrictHostKeyChecking=no {2}@{3} "{4}"' 14 | .format(ssh_password_param, ssh_key_param, ssh_username, broker_host, cmd)) 15 | return stdout, stderr 16 | 17 | def main(args, utils): 18 | script_dir = os.path.dirname(__file__) 19 | 20 | ssh_username = args.ssh_username 21 | logger.info('ssh_username = ' + ssh_username) 22 | 23 | ssh_key_param = '' 24 | ssh_password_param = '' 25 | 26 | ssh_password = args.ssh_password 27 | ssh_key_file = '' 28 | 29 | if os.path.exists(ssh_password): 30 | ssh_key_file = ssh_password 31 | elif os.path.exists(os.path.join(script_dir, ssh_password)): 32 | ssh_key_file = os.path.join(script_dir, ssh_password) 33 | 34 | if os.path.exists(ssh_key_file): 35 | logger.info('ssh_key_file = ' + ssh_key_file) 36 | stdout, stderr = utils.run_shell_command('chmod 600 {0}'.format(ssh_key_file)) 37 | ssh_key_param = '-i {0}'.format(ssh_key_file) 38 | else: 39 | logger.debug('ssh_password = ' + ssh_password) 40 | ssh_password_param = 'sshpass -p {0} '.format(ssh_password) 41 | stdout, stderr = utils.run_shell_command('sudo apt-get install sshpass') 42 | 43 | broker_hosts, brokers = utils.get_brokers_from_ambari() 44 | errored_brokers = [] 45 | for broker_host in broker_hosts: 46 | if broker_host: 47 | logger.info('\nPatching broker host: {0}\n-----------------------------------'.format(broker_host)) 48 | try: 49 | stdout, stderr = copy_files(broker_host, ssh_password_param, ssh_key_param, ssh_username, '~/*.txt', '~/') 50 | #cmd = 'sudo dpkg -i ~/*.deb' 51 | #cmd = 'sudo chage -I -1 -m 0 -M 99999 -E -1 {2}' 52 | cmd = 'sudo ls -alFh ~/' 53 | stdout, stderr = run_command(broker_host, ssh_password_param, ssh_key_param, ssh_username, cmd) 54 | logger.info('\nBroker host: {0} successfully patched\n==================================\n'.format(broker_host)) 55 | except: 56 | logger.error(traceback.print_exc()) 57 | errored_brokers.append(broker_host) 58 | #raise RuntimeError('Failing due to an execution error') 59 | logger.info('\nBroker host: {0} patching failed\n==================================\n'.format(broker_host)) 60 | 61 | if len(errored_brokers) > 0: 62 | logger.info('Errored brokers that were not be patched: {0}\n{1}\n'.format(len(errored_brokers), reduce(lambda x, y : x + y, map(lambda b : b + '\n', errored_brokers)))) 63 | 64 | if __name__ == '__main__': 65 | parser = argparse.ArgumentParser(description='Run custom commands on all Kafka Brokers hosts.') 66 | parser.add_argument('ssh_username', help='The SSH Username for the nodes.') 67 | parser.add_argument('ssh_password', help='The SSH Password or SSH Key File for the SSH User.') 68 | args = parser.parse_args() 69 | utils = KafkaUtils(logger, 'runcustomcommands{0}'.format(int(time.time())) + '.log') 70 | main(args, utils) 71 | -------------------------------------------------------------------------------- /src/python/troubleshooting/kafka_restart_brokers.py: -------------------------------------------------------------------------------- 1 | """Script to restart Kafka Brokers ensuring High Availability of brokers 2 | """ 3 | import argparse 4 | import logging 5 | import pprint 6 | import time 7 | 8 | from kafka_broker_status import get_kafka_broker_status, str_kafka_brokers_status 9 | from kafka_utils import KafkaUtils 10 | 11 | logger = logging.getLogger(__name__) 12 | debug = False 13 | 14 | TIMEOUT_SECS = 3600 15 | WAIT_SECS = 300 16 | SLEEP_SECS = 30 17 | 18 | 19 | def main(args, utils): 20 | force = args.force 21 | all = args.all 22 | if all: 23 | force = True 24 | 25 | broker_hosts, zk_brokers, dead_broker_hosts = get_kafka_broker_status(utils) 26 | 27 | if all: 28 | logger.info('Restarting all brokers irrespective of stale configs or dead brokers.') 29 | restart_broker_hosts = broker_hosts.keys() 30 | else: 31 | restart_broker_hosts = utils.get_stale_broker_hosts_from_ambari() 32 | 33 | if len(restart_broker_hosts) == 0: 34 | logger.debug('No Brokers with stale configs found to restart.') 35 | else: 36 | logger.info('Restarting brokers on following hosts: {0}\n{1}\n'.format(len(restart_broker_hosts), 37 | pprint.pformat(restart_broker_hosts))) 38 | 39 | for restart_broker_host in restart_broker_hosts: 40 | broker_hosts, zk_brokers, dead_broker_hosts = get_kafka_broker_status(utils) 41 | 42 | if len(dead_broker_hosts) > 0: 43 | logger.warn('Dead/Unregistered brokers: {0}\n{1}\n'. 44 | format(len(dead_broker_hosts), pprint.pformat(dead_broker_hosts))) 45 | if not force: 46 | err_msg = 'One or more brokers are not online, cannot proceed with broker restarts.' 47 | logger.error(err_msg) 48 | raise RuntimeError(err_msg) 49 | else: 50 | logger.info('Forcing restart of brokers inspite of dead or unregistered brokers.') 51 | else: 52 | logger.debug( 53 | 'All brokers are online and registered in Zookeeper, proceeding with restarts of stale Kafka Broker ' 54 | 'on {0}.'.format( 55 | restart_broker_host)) 56 | 57 | utils.restart_kafka_broker_from_ambari(restart_broker_host) 58 | zk_brokers = utils.get_brokers_from_zookeeper() 59 | now = time.time() 60 | timeout = now + TIMEOUT_SECS 61 | while restart_broker_host not in zk_brokers: 62 | if time.time() > timeout: 63 | err_msg = 'Kafka Broker on {0} failed to come online in Zookeeper in {1} secs. Please check the Kafka ' \ 64 | 'server log for potential issues'.format(restart_broker_host, TIMEOUT_SECS) 65 | logger.error(err_msg) 66 | raise RuntimeError(err_msg) 67 | logger.info('Kafka Broker on {0} is not yet online in Zookeeper. Sleeping for {1} seconds...'.format( 68 | restart_broker_host, SLEEP_SECS)) 69 | time.sleep(SLEEP_SECS) 70 | zk_brokers = utils.get_brokers_from_zookeeper() 71 | logger.info( 'Kafka Broker on {0} successfully restarted and is now online in Zookeeper. Waiting for {1} ' 72 | 'seconds before proceeding to the next broker.'.format(restart_broker_host, WAIT_SECS)) 73 | time.sleep(WAIT_SECS) 74 | 75 | logger.info('All brokers with stale configs have been restarted. Refreshing information...') 76 | restart_broker_hosts = utils.get_stale_broker_hosts_from_ambari() 77 | zk_brokers = utils.get_brokers_from_zookeeper() 78 | logger.info(str_kafka_brokers_status(broker_hosts, zk_brokers)) 79 | 80 | 81 | if __name__ == '__main__': 82 | parser = argparse.ArgumentParser( 83 | description='Restart Kafka Brokers one by one and wait for their Zookeeper state to be alive.') 84 | parser.add_argument('-f', '--force', action='store_true', 85 | help='Restart brokers with stale configs irrespective of their current health.') 86 | parser.add_argument('-a', '--all', action='store_true', 87 | help='Restart all brokers one by one irrespective of their current health.') 88 | args = parser.parse_args() 89 | utils = KafkaUtils(logger, 'kafkarestartbrokers{0}.log'.format(int(time.time())), debug) 90 | main(args, utils) 91 | -------------------------------------------------------------------------------- /src/python/rebalance/README.md: -------------------------------------------------------------------------------- 1 | # Kafka Partition Rebalance Tool 2 | 3 | ## Introduction 4 | Kafka is not aware of the cluster topology (not rack aware) and hence partitions are susceptible to data loss or unavailability in the event of faults or updates. 5 | 6 | This tool generates a reassignment plan that has two goals: 7 | - Redistribute the replicas of partitions of a topic across brokers in a manner such that all replicas of a partition are in separate Update Domains (UDs) & Fault Domains (FDs). 8 | - Balance the leader load across the cluster - The number of leaders assigned to each broker is more or less the same. 9 | 10 | Once the reassignment plan is generated the user can execute this plan. Upon execution, the tool updates the zookeeper path '/admin/reassign_partitions' with the list of topic partitions and (if specified in the Json file) the list of their new assigned replicas. The controller listens to the path above. When a data change update is triggered, the controller reads the list of topic partitions and their assigned replicas from zookeeper. The control handles starting new replicas in RAR and waititing until the new replicas are in sync with the leader. At this point, the old replicas are stopped and the partition is removed from the '/admin/reassignpartitions' path. Note that these steps are async operations. 11 | 12 | This tool is the best suitable for execution on each of the following events: 13 | - New cluster 14 | - Whenever a new topic/partition is created 15 | - Cluster is scaled up 16 | 17 | Note for execution on existing clusters: 18 | When executing on a cluster with large data sizes, there will be a performance degradation while reassignment is taking place, due to data movement. 19 | On large clusters, rebalance can take several hours. In such scenarios, it is recommended to execute rebalance by steps (by topic or by partitions of a topic). 20 | 21 | ## Prerequisites 22 | The script relies on various python modules. If not installed already, you must manually install these prior to execution: 23 | ``` 24 | sudo apt-get update 25 | sudo apt-get install libffi-dev libssl-dev 26 | sudo pip install --upgrade PyOpenSSL ndg-httpsclient pyasn1 retry pexpect 27 | sudo pip install kazoo requests[security] 28 | ``` 29 | 30 | ## How to use 31 | Copy the file to /usr/hdp/current/kafka-broker/bin, and run it as ROOT. 32 | 33 | ``` 34 | usage: rebalance_rackaware.py [-h] [--topics TOPICS] [--execute] [--verify] 35 | [--force] [--throttle THROTTLE] 36 | [--rebalancePlanDir REBALANCEPLANDIR] 37 | [--deadhosts DEADHOSTS] 38 | 39 | Rebalance Kafka Replicas. 40 | 41 | optional arguments: 42 | -h, --help show this help message and exit 43 | --topics TOPICS [TOPICS ...] 44 | Comma separated list of topics to reassign replicas. 45 | Use ALL|all to rebalance all topics 46 | --execute whether or not to execute the reassignment plan 47 | --verify Verify status of reassignment operation 48 | --force Force rebalance of all partitions in a topic, even if already balanaced. 49 | --throttle Upper bound on bandwidth used to move replicas from machine to machine. 50 | --rebalancePlanDir Directory where the rebalance plan should be saved or retrieved from. 51 | --deadhosts Comma separated list of hosts which have been removed from the cluster. 52 | ``` 53 | 54 | Without "--execute" this tool only scans the current assignment generates the replica reassignment file. 55 | 56 | ## Example Invocation: 57 | 58 | #### Generate reassignment plan for all topics on cluster: 59 | 60 | ```sudo python rebalance_rackaware.py --topics ALL --rebalancePlanDir /tmp/rebalance/``` 61 | 62 | The plan will be saved at /tmp/rebalance/kafkaRebalancePlan.json 63 | 64 | #### Execute reassignment: 65 | 66 | ```sudo python rebalance_rackaware.py --execute --rebalancePlanDir /tmp/rebalance/``` 67 | 68 | This will execute the plan saved in the above location. 69 | 70 | #### Verify progress of reassignment: 71 | 72 | ```sudo python rebalance_rackaware.py --verify --rebalancePlanDir /tmp/rebalance/``` 73 | 74 | 75 | ## Debugging 76 | Debug logs can be found /var/log/kafka/rebalance_log.log. 77 | The log file includes detailed information about the steps taken by the tool and can be used for troubleshooting. 78 | 79 | ## References 80 | [High availability of your data with Apache Kafka for HDInsight](https://docs.microsoft.com/en-us/azure/hdinsight/kafka/apache-kafka-high-availability) 81 | -------------------------------------------------------------------------------- /src/python/troubleshooting/kafka_broker_status.py: -------------------------------------------------------------------------------- 1 | """Script to report the current Kafka Broker status for this HDInsight Kafka cluster. 2 | """ 3 | import logging, pprint, time 4 | 5 | from kafka_utils import KafkaUtils 6 | 7 | logger = logging.getLogger(__name__) 8 | debug = False 9 | 10 | def str_kafka_brokers_status(broker_hosts, zk_brokers): 11 | """Returns the formatted Broker status as string. Takes Broker Hosts from Ambari and Zookeeper as the input. 12 | """ 13 | if len(zk_brokers) > 0: 14 | return ('Zookeeper registered brokers: {0}\n{1}\n{2}\n'. 15 | format( 16 | len(zk_brokers), 17 | '{0} {1} {2} {3} {4}'.format( 18 | 'broker.id'.ljust(16), 19 | 'broker.host'.ljust(66), 20 | 'broker.ip'.ljust(20), 21 | 'broker.timestamp'.ljust(30), 22 | 'broker.uptime'.ljust(30)), 23 | reduce(lambda b1, b2 : b1 + '\n' + b2, 24 | sorted( 25 | map(lambda b : '{0} {1} {2} {3} {4}'.format( 26 | str(zk_brokers[b]['id']).ljust(16), 27 | b.ljust(66), 28 | broker_hosts[b].ljust(20), 29 | zk_brokers[b]['datetime'].ljust(30), 30 | zk_brokers[b]['duration'].ljust(30)), 31 | zk_brokers 32 | ) 33 | ) 34 | ) 35 | ) 36 | ) 37 | else: 38 | return 'There are NO brokers registered in Zookeeper.' 39 | 40 | def str_kafka_controller_status(zk_controller): 41 | """Returns the formatted Kafka Controller status as string. Takes Controller data from Zookeeper as the input. 42 | """ 43 | return ('Zookeeper registered controller: \n{0}\n{1}\n'. 44 | format( 45 | '{0} {1} {2} {3} {4}'.format( 46 | 'controller.id'.ljust(16), 47 | 'controller.host'.ljust(70) , 48 | 'controller.ip'.ljust(20) , 49 | 'controller.timestamp'.ljust(30), 50 | 'controller.uptime'.ljust(30)), 51 | '{0} {1} {2} {3} {4}'.format( 52 | zk_controller['controller_id'].ljust(16), 53 | zk_controller['controller_host'].ljust(70), 54 | zk_controller['controller_ip'].ljust(20), 55 | zk_controller['datetime'].ljust(30), 56 | zk_controller['duration'].ljust(30)) 57 | )) 58 | 59 | def get_kafka_broker_status(utils): 60 | """Gets Kafka Broker status by querying Ambari and Zookeeper. 61 | 62 | Returns Broker Hosts in Ambari and Zookeeper, and Dead Brokers. All the returned objects are dictionaries of dictionaries with host name as the key. 63 | """ 64 | broker_hosts, brokers = utils.get_brokers_from_ambari() 65 | logger.info('Ambari brokers: {0}\n{1}\n'.format(len(broker_hosts), pprint.pformat(broker_hosts))) 66 | 67 | zk_brokers = utils.get_brokers_from_zookeeper() 68 | logger.info(str_kafka_brokers_status(broker_hosts, zk_brokers)) 69 | 70 | dead_broker_hosts = {} 71 | for expected_broker_host in broker_hosts: 72 | if not expected_broker_host in zk_brokers: 73 | dead_broker_hosts[expected_broker_host] = broker_hosts[expected_broker_host] 74 | 75 | if len(dead_broker_hosts) > 0: 76 | logger.warn('Dead/Unregistered brokers: {0}\n{1}\n'. 77 | format(len(dead_broker_hosts), pprint.pformat(dead_broker_hosts))) 78 | else: 79 | logger.info('There are no dead or unregistered brokers') 80 | 81 | return broker_hosts, zk_brokers, dead_broker_hosts 82 | 83 | def get_kafka_controller_status(utils, broker_hosts, zk_brokers): 84 | """Gets Kafka Controller status by querying Zookeeper. Returns a dictionary object for the controller. 85 | 86 | Returned object contains following keys: 87 | controller_id (broker_id) 88 | controller_host 89 | controller_ip 90 | """ 91 | zk_controller = utils.get_controller_from_zookeeper() 92 | if zk_brokers and zk_controller: 93 | zk_controller['controller_id'] = str(zk_controller['brokerid']) 94 | for zk_broker_host, zk_broker_info in zk_brokers.iteritems(): 95 | if zk_broker_info['id'] == zk_controller['controller_id']: 96 | zk_controller['controller_host'] = str(zk_broker_host) 97 | zk_controller['controller_ip'] = str(broker_hosts[zk_controller['controller_host']]) 98 | break 99 | if 'controller_host' not in zk_controller: 100 | logger.error('Unable to find controller host information') 101 | else: 102 | logger.info(str_kafka_controller_status(zk_controller)) 103 | else: 104 | err_msg = 'There are no brokers or controller online.' 105 | logger.error(err_msg) 106 | raise RuntimeError(err_msg) 107 | return zk_controller 108 | 109 | def main(utils): 110 | broker_hosts, zk_brokers, dead_broker_hosts = get_kafka_broker_status(utils) 111 | zk_controller = get_kafka_controller_status(utils, broker_hosts, zk_brokers) 112 | 113 | if __name__ == '__main__': 114 | utils = KafkaUtils(logger, "kafkabrokerstatus{0}.log".format(int(time.time())), debug) 115 | main(utils) 116 | -------------------------------------------------------------------------------- /src/python/troubleshooting/README.MD: -------------------------------------------------------------------------------- 1 | # HDInsight Kafka Troubleshooting Python Scripts 2 | 3 | Repository of Python scripts that can be executed on any HDInsight Kafka cluster node via SSH. 4 | Each of these scripts provides a support action which you can use to either get the health status of Kafka or perform certain actions like restarting Brokers and waiting for them to be healthy before proceeding to next one. 5 | 6 | The scripts use Ambari and Zookeeper APIs to gather the information and execute appropriate Ambari Actions using internal libraries, such that users do not need to provide any host names or credentials. 7 | Each script also generates a local log file whose name is also displayed on the console that can be shared with Microsoft support for further help/investigations. 8 | 9 | The scripts require HDInsight Clusters to be of versions HDI 3.5 (HDP 2.5/Kafka 0.9), HDI 3.6 (HDP 2.6/Kafka 0.10) or above. 10 | 11 | ## How to use this repository 12 | Clone the repository on any HDInsight cluster node using the following command: 13 | 14 | ``` 15 | git clone https://github.com/hdinsight/hdinsight-kafka-tools.git 16 | ``` 17 | 18 | The scripts are being located under ```hdinsight-kafka-tools/src/python/troubleshooting``` and can be executed from any directory. 19 | 20 | | Script | File | When to use | Description | 21 | |--------------------------|------------------------------------------------------------|---------------------------------------------------------------------|------------------------------------------------------------------------| 22 | | Kafka Performance Test | [kafka_perf_test.py](kafka_perf_test.py) | Anytime | A quick performance test for Kafka. | 23 | | Kafka Broker Status | [kafka_broker_status.py](kafka_broker_status.py) | Anytime | Reports Broker status for all Kafka Brokers. | 24 | | Kafka Restart Brokers | [kafka_restart_brokers.py](kafka_restart_brokers.py) | Post Kafka configuration update | Restarts Kafka brokers based on health check. | 25 | | Kafka Restart Controller | [kafka_restart_controller.py](kafka_restart_controller.py) | For clearing Kafka controller state for partition leadership issues | Restarts Kafka Broker that is the current controller. | 26 | | Kafka Topic Describe | [kafka_topic_describe.py](kafka_topic_describe.py) | Anytime | Wrapper script for Kafka Topic describe that takes care of all inputs. | 27 | 28 | 29 | ## Kafka Performance Test 30 | A simple script that uses Kafka Performance tools to create a topic, produce and consume data from the topic and then deletes the topic. All the parameters are taken care of by the script. 31 | The number of partitions will be equal to the number of brokers in the system and the script will send 1 million messages of 100 bytes each per partition and report the numbers for each of the steps. 32 | 33 | ```bash 34 | sudo python kafka_perf_test.py 35 | ``` 36 | 37 | ## Kafka Broker Status 38 | Handy script to list the uptime of all Kafka Brokers and also provide a list of hosts where Kafka Broker is down and not registered with the Zookeeper. 39 | The script also provides the broker id to host name and ip mapping. 40 | 41 | ```bash 42 | sudo python kafka_broker_status.py 43 | ``` 44 | 45 | Sample output: 46 | ``` 47 | 2017-12-29 04:55:00,514 - kafka_broker_status.py - __main__ - INFO - Zookeeper registered brokers: 3 48 | broker.id broker.host broker.ip broker.timestamp broker.uptime 49 | 1001 wn1-kafkar.averylargefqdnincloudexist.cx.internal.cloudapp.net 10.0.0.32 2017-12-29 01:20:00 3:35:00.440146 50 | 1002 wn0-kafkar.averylargefqdnincloudexist.cx.internal.cloudapp.net 10.0.0.33 2017-12-29 02:11:53 2:43:07.402321 51 | 1003 wn2-kafkar.averylargefqdnincloudexist.cx.internal.cloudapp.net 10.0.0.36 2017-12-29 01:21:37 3:33:23.469578 52 | 53 | 2017-12-29 04:55:00,514 - kafka_broker_status.py - __main__ - INFO - There are no dead or unregistered brokers 54 | 2017-12-29 04:55:00,595 - kafka_broker_status.py - __main__ - INFO - Zookeeper registered controller: 55 | controller.id controller.host controller.ip controller.timestamp controller.uptime 56 | 1001 wn1-kafkar.averylargefqdnincloudexist.cx.internal.cloudapp.net 10.0.0.32 2017-12-29 02:11:31 2:43:28.751835 57 | ``` 58 | 59 | ## Kafka Restart Brokers 60 | Restart Kafka Brokers that have stale configs i.e. brokers are pending restart in Ambari after configuration update. 61 | The script queries for brokers with stale configs from Ambari and then checks if all the brokers are alive in Zookeeper and then initiates restarts one by one. 62 | Post each restart of the Kafka Broker component on each host, it waits for the Kafka server startup to complete and check if the broker is registered in Zookeeper before proceeding to the next one. 63 | 64 | ```bash 65 | sudo python kafka_restart_brokers.py 66 | ``` 67 | 68 | Two additional options are available in the script to force restart of all or only stale brokers irrespective of their Zookeeper state. 69 | 70 | ``` 71 | usage: kafka_restart_brokers.py [-h] [-f] [-a] 72 | 73 | Restart Kafka Brokers one by one and wait for their Zookeeper state to be alive. 74 | 75 | optional arguments: 76 | -h, --help show this help message and exit 77 | -f, --force Restart brokers with stale configs irrespective of their current health. 78 | -a, --all Restart all brokers one by one irrespective of their current health. 79 | ``` 80 | 81 | ## Kafka Restart Controller 82 | Restart the Kafka Broker which is the current acting controller. 83 | The script checks the broker and controller information in the Zookeeper and then issues the restart of Kafka Broker via Ambari for the host that maps to that broker id. 84 | 85 | ```bash 86 | sudo python kafka_restart_controller.py 87 | ``` 88 | 89 | ## Kafka Topic Describe 90 | A simple script that describes all or some topics on the Kafka cluster. 91 | 92 | ```bash 93 | sudo python kafka_topic_describe.py 94 | ``` 95 | 96 | ## Kafka Get PID Status 97 | A simple script to report the detailed status of the Kafka process especially if the process became a zombie. Helpful in cases where it is not responding to any actions and Ambari sees it as healthy. 98 | This script has to be run on the node where the Kafka process is running. 99 | 100 | ```bash 101 | sudo python kafka_get_pid_status.py 102 | ``` 103 | 104 | ## Run Custom Commands 105 | This is a DIY patch the cluster script, where you can run any command or copy files across all Kafka Broker hosts by providing SSH creds. 106 | Some use cases include extending the password expiry for the current SSH user or checking the status of certain process. 107 | 108 | ```bash 109 | sudo python run_custom_commands.py 110 | ``` 111 | 112 | ``` 113 | usage: run_custom_commands.py [-h] ssh_username ssh_password 114 | 115 | Run custom commands on all Kafka Brokers hosts. 116 | 117 | positional arguments: 118 | ssh_username The SSH User name for the nodes. 119 | ssh_password The SSH Password or SSH Key File for the SSH User. 120 | 121 | optional arguments: 122 | -h, --help show this help message and exit 123 | ``` -------------------------------------------------------------------------------- /src/python/troubleshooting/kafka_perf_test.py: -------------------------------------------------------------------------------- 1 | """A script to perform a quick Kafka Performance test. 2 | """ 3 | import logging 4 | import time 5 | 6 | from kafka_utils import KafkaUtils 7 | 8 | logger = logging.getLogger(__name__) 9 | debug = False 10 | 11 | 12 | def get_kafka_shell_inputs(utils): 13 | """Gets the inputs for Kafka Shell. Takes KafkaUtils class object as the input. 14 | 15 | Returns the following: 16 | Zookeeper quorum 17 | Broker connection string 18 | Number of partitions = Number of broker hosts 19 | replicationfactor = 3 (default) 20 | messages = 1M per partition 21 | threads = min(4, partitions) 22 | messagesize = 100 Bytes 23 | batchsize = 10000 24 | thoughput = -1 25 | """ 26 | zookeepers = utils.get_zookeeper_quorum() 27 | broker_hosts, brokers = utils.get_brokers_from_ambari() 28 | broker_hosts_count = len(broker_hosts) 29 | 30 | utils.logger.info('Choosing defaults:') 31 | partitions = broker_hosts_count 32 | replicationfactor = min(3, broker_hosts_count) 33 | messages = (partitions * 1000000) 34 | threads = min(4, partitions) 35 | messagesize = 100 36 | batchsize = 10000 37 | throughput = -1 38 | 39 | logger.info('\n' + 40 | 'zookeepers = {0}\n'.format(zookeepers) + 41 | 'brokers = {0}\n'.format(brokers) + 42 | 'partitions = {0}\n'.format(partitions) + 43 | 'replicationfactor = {0}\n'.format(replicationfactor) + 44 | 'messages = {0}\n'.format(messages) + 45 | 'threads = {0}\n'.format(threads) + 46 | 'messagesize = {0}\n'.format(messagesize) + 47 | 'batchsize = {0}\n'.format(batchsize) + 48 | 'thoughput = {0}\n'.format(throughput) 49 | ) 50 | 51 | return zookeepers, brokers, partitions, replicationfactor, messages, threads, messagesize, batchsize, throughput 52 | 53 | 54 | def main(utils, topic): 55 | zookeepers, brokers, partitions, replicationfactor, messages, threads, messagesize, batchsize, throughput = \ 56 | get_kafka_shell_inputs(utils) 57 | kafka_version, hdi_version = utils.get_kafka_hdp_version() 58 | if kafka_version >= '3.2.0': 59 | # Create 60 | logger.info("Creating topic: {0}".format(topic)) 61 | shell_command = "/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --create --bootstrap-server {0} --topic {1} " \ 62 | "--partitions {2} --replication-factor {3}".format(brokers, topic, partitions, 63 | replicationfactor) 64 | utils.run_shell_command(shell_command) 65 | 66 | # List 67 | logger.info("Listing topics") 68 | shell_command = "/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --list --bootstrap-server {0}".format( 69 | brokers) 70 | utils.run_shell_command(shell_command) 71 | 72 | # Describe 73 | logger.info("Describing topic: {0}".format(topic)) 74 | shell_command = "/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --describe --bootstrap-servers {0} --topic " \ 75 | "{1}".format(brokers, topic) 76 | utils.run_shell_command(shell_command) 77 | 78 | # Produce 79 | logger.info("Producing {0} messages to topic {1}".format(messages, topic)) 80 | shell_command = "/usr/hdp/current/kafka-broker/bin/kafka-producer-perf-test.sh --topic {0} " \ 81 | "--num-records {1} --record-size {2} --throughput {3} --producer-props acks=0 " \ 82 | "bootstrap.servers={4} batch.size={5}".format(topic, messages, messagesize, throughput, 83 | brokers, batchsize) 84 | utils.run_shell_command(shell_command) 85 | 86 | # Offset 87 | partitions_list = ",".join(map(str, range(0, partitions))) 88 | logger.info("Listing offsets of partitions {0} of topic {1}".format(partitions_list, topic)) 89 | shell_command = "/usr/hdp/current/kafka-broker/bin/kafka-run-class.sh kafka.tools.GetOffsetShell " \ 90 | "--bootstrap-server {0} --topic {1} --partitions {2} --time -1".\ 91 | format(brokers, topic, partitions_list) 92 | utils.run_shell_command(shell_command) 93 | 94 | # Consume 95 | logger.info("Consuming {0} messages from topic {1}".format(messages, topic)) 96 | shell_command = "/usr/hdp/current/kafka-broker/bin/kafka-consumer-perf-test.sh --bootstrap-server {0} -messages" \ 97 | " {1} --topic {2} --threads {3}".format(brokers, messages, topic, threads) 98 | utils.run_shell_command(shell_command) 99 | 100 | # Delete 101 | logger.info("Deleting topic: {0}".format(topic)) 102 | shell_command = "/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --delete --bootstrap-server {0} --topic {1}" \ 103 | .format(brokers, topic) 104 | utils.run_shell_command(shell_command) 105 | else: 106 | # Create 107 | logger.info("Creating topic: {0}".format(topic)) 108 | shell_command = "/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --create --zookeeper {0} --topic {1} " \ 109 | "--partitions {2} --replication-factor {3}".format(zookeepers, topic, partitions, 110 | replicationfactor) 111 | utils.run_shell_command(shell_command) 112 | 113 | # List 114 | logger.info("Listing topics") 115 | shell_command = "/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --list --zookeeper {0}".format(zookeepers) 116 | utils.run_shell_command(shell_command) 117 | 118 | # Describe 119 | logger.info("Describing topic: {0}".format(topic)) 120 | shell_command = "/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --describe --zookeeper {0} --topic {1}". \ 121 | format(zookeepers, topic) 122 | utils.run_shell_command(shell_command) 123 | 124 | # Produce 125 | logger.info("Producing {0} messages to topic {1}".format(messages, topic)) 126 | shell_command = "/usr/hdp/current/kafka-broker/bin/kafka-producer-perf-test.sh --broker-list {0} --topics {1} " \ 127 | "--messages {2} --message-size {3} --batch-size {4} --request-num-acks 0 --compression-codec 0 " \ 128 | "--threads {5}".format(brokers, topic, messages, messagesize, batchsize, threads) 129 | utils.run_shell_command(shell_command) 130 | 131 | # Offset 132 | partitions_list = ",".join(map(str, range(0, partitions))) 133 | logger.info("Listing offsets of partitions {0} of topic {1}".format(partitions_list, topic)) 134 | shell_command = "/usr/hdp/current/kafka-broker/bin/kafka-run-class.sh kafka.tools.GetOffsetShell --broker-list " \ 135 | "{0} --topic {1} --partitions {2} --time -1 --offsets 1".format(brokers, topic, partitions_list) 136 | utils.run_shell_command(shell_command) 137 | 138 | # Consume 139 | logger.info("Consuming {0} messages from topic {1}".format(messages, topic)) 140 | shell_command = "/usr/hdp/current/kafka-broker/bin/kafka-consumer-perf-test.sh --zookeeper {0} -messages {1} " \ 141 | "--topic {2} --threads {3}".format(zookeepers, messages, topic, threads) 142 | utils.run_shell_command(shell_command) 143 | 144 | # Delete 145 | logger.info("Deleting topic: {0}".format(topic)) 146 | shell_command = "/usr/hdp/current/kafka-broker/bin/kafka-topics.sh --delete --zookeeper {0} --topic {1}".format( 147 | zookeepers, topic) 148 | utils.run_shell_command(shell_command) 149 | 150 | 151 | if __name__ == '__main__': 152 | topic = "kafkaperftest{0}".format(int(time.time())) 153 | utils = KafkaUtils(logger, topic + ".log", debug) 154 | main(utils, topic) 155 | -------------------------------------------------------------------------------- /src/arm/HDInsight4.0/hdinsight-kafka-2.1-vnet/azuredeploy.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "clusterName": { 6 | "type": "string", 7 | "metadata": { 8 | "description": "The name of the cluster to create." 9 | } 10 | }, 11 | "clusterVersion": { 12 | "type": "string", 13 | "defaultValue": "4.0", 14 | "metadata": { 15 | "description": "The HDInsight version to deploy." 16 | } 17 | }, 18 | "headNodeSize": { 19 | "type": "string", 20 | "defaultValue": "Standard_D4_V2", 21 | "metadata": { 22 | "description": "The VM size of the head nodes." 23 | } 24 | }, 25 | "workerNodeSize": { 26 | "type": "string", 27 | "defaultValue": "Standard_D4_V2", 28 | "metadata": { 29 | "description": "The VM size of the worker nodes." 30 | } 31 | }, 32 | "zookeeperNodeSize": { 33 | "type": "string", 34 | "defaultValue": "Standard_D3_V2", 35 | "metadata": { 36 | "description": "The VM size of the zookeeper nodes." 37 | } 38 | }, 39 | "clusterSize": { 40 | "type": "int", 41 | "defaultValue": 4, 42 | "metadata": { 43 | "description": "The number of worker nodes in the cluster." 44 | } 45 | }, 46 | "disksPerWorkerNode": { 47 | "type": "int", 48 | "defaultValue": 8, 49 | "metadata": { 50 | "description": "The number of disks per worker node." 51 | } 52 | }, 53 | "clusterLoginUserName": { 54 | "type": "string", 55 | "defaultValue": "admin", 56 | "metadata": { 57 | "description": "These credentials can be used to submit jobs to the cluster and to log into cluster dashboards." 58 | } 59 | }, 60 | "clusterLoginPassword": { 61 | "type": "securestring", 62 | "metadata": { 63 | "description": "The password must be at least 10 characters in length and must contain at least one digit, one non-alphanumeric character, and one upper or lower case letter." 64 | } 65 | }, 66 | "sshUserName": { 67 | "type": "string", 68 | "metadata": { 69 | "description": "These credentials can be used to remotely access the cluster." 70 | } 71 | }, 72 | "sshPassword": { 73 | "type": "securestring", 74 | "metadata": { 75 | "description": "The password must be at least 10 characters in length and must contain at least one digit, one non-alphanumeric character, and one upper or lower case letter." 76 | } 77 | } 78 | }, 79 | "variables": { 80 | "defaultApiVersion": "2015-06-15", 81 | "clusterApiVersion": "2015-03-01-preview", 82 | "defaultStorageName": "[concat('store', uniqueString(resourceGroup().id))]", 83 | "storageType": "Standard_LRS", 84 | "virtualNetworkName": "[concat('vnet', uniqueString(resourceGroup().id))]", 85 | "virtualNetworkPrefix": "10.0.0.0/16", 86 | "virtualNetworkDefaultSubnetName": "default", 87 | "virtualNetworkSubnet1Prefix": "10.0.0.0/20", 88 | "clusterType": "kafka", 89 | "componentVersion": "2.1" 90 | }, 91 | "resources": [ 92 | { 93 | "name": "[variables('defaultStorageName')]", 94 | "type": "Microsoft.Storage/storageAccounts", 95 | "location": "[resourceGroup().location]", 96 | "apiVersion": "2015-06-15", 97 | "dependsOn": [ ], 98 | "tags": { 99 | "displayName": "[variables('defaultStorageName')]" 100 | }, 101 | "properties": { 102 | "accountType": "[variables('storageType')]" 103 | } 104 | }, 105 | { 106 | "name": "[variables('virtualNetworkName')]", 107 | "type": "Microsoft.Network/virtualNetworks", 108 | "location": "[resourceGroup().location]", 109 | "apiVersion": "2015-06-15", 110 | "dependsOn": [ ], 111 | "tags": { 112 | "displayName": "[variables('virtualNetworkName')]" 113 | }, 114 | "properties": { 115 | "addressSpace": { 116 | "addressPrefixes": [ 117 | "[variables('virtualNetworkPrefix')]" 118 | ] 119 | }, 120 | "subnets": [ 121 | { 122 | "name": "[variables('virtualNetworkDefaultSubnetName')]", 123 | "properties": { 124 | "addressPrefix": "[variables('virtualNetworkSubnet1Prefix')]" 125 | } 126 | } 127 | ] 128 | } 129 | }, 130 | { 131 | "name": "[parameters('clusterName')]", 132 | "type": "Microsoft.HDInsight/clusters", 133 | "location": "[resourceGroup().location]", 134 | "apiVersion": "[variables('clusterApiVersion')]", 135 | "properties": { 136 | "clusterVersion": "[parameters('clusterVersion')]", 137 | "osType": "Linux", 138 | "clusterDefinition": { 139 | "kind": "[variables('clusterType')]", 140 | "componentVersion": { 141 | "Kafka": "[variables('componentVersion')]" 142 | }, 143 | "configurations": { 144 | "gateway": { 145 | "restAuthCredential.isEnabled": true, 146 | "restAuthCredential.username": "[parameters('clusterLoginUserName')]", 147 | "restAuthCredential.password": "[parameters('clusterLoginPassword')]" 148 | } 149 | } 150 | }, 151 | "storageProfile": { 152 | "storageaccounts": [ 153 | { 154 | "name": "[concat(variables('defaultStorageName'),'.blob.core.windows.net')]", 155 | "isDefault": true, 156 | "container": "[parameters('clusterName')]", 157 | "key": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('defaultStorageName')), variables('defaultApiVersion')).key1]" 158 | } 159 | ] 160 | }, 161 | "computeProfile": { 162 | "roles": [ 163 | { 164 | "name": "headnode", 165 | "targetInstanceCount": "2", 166 | "hardwareProfile": { 167 | "vmSize": "[parameters('headNodeSize')]" 168 | }, 169 | "osProfile": { 170 | "linuxOperatingSystemProfile": { 171 | "username": "[parameters('sshUserName')]", 172 | "password": "[parameters('sshPassword')]" 173 | } 174 | }, 175 | "virtualNetworkProfile": { 176 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 177 | "subnet": "[concat(resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName')), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 178 | } 179 | }, 180 | { 181 | "name": "workernode", 182 | "targetInstanceCount": "[parameters('clusterSize')]", 183 | "hardwareProfile": { 184 | "vmSize": "[parameters('workerNodeSize')]" 185 | }, 186 | "dataDisksGroups": [ 187 | { 188 | "disksPerNode": "[parameters('disksPerWorkerNode')]" 189 | } 190 | ], 191 | "osProfile": { 192 | "linuxOperatingSystemProfile": { 193 | "username": "[parameters('sshUserName')]", 194 | "password": "[parameters('sshPassword')]" 195 | } 196 | }, 197 | "virtualNetworkProfile": { 198 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 199 | "subnet": "[concat('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 200 | } 201 | }, 202 | { 203 | "name": "zookeepernode", 204 | "targetInstanceCount": 3, 205 | "hardwareProfile": { 206 | "vmSize": "[parameters('zookeeperNodeSize')]" 207 | }, 208 | "osProfile": { 209 | "linuxOperatingSystemProfile": { 210 | "username": "[parameters('sshUserName')]", 211 | "password": "[parameters('sshPassword')]" 212 | } 213 | }, 214 | "virtualNetworkProfile": { 215 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 216 | "subnet": "[concat('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 217 | } 218 | } 219 | ] 220 | } 221 | }, 222 | "dependsOn": [ 223 | "[variables('defaultStorageName')]", 224 | "[variables('virtualNetworkName')]" 225 | ] 226 | } 227 | ], 228 | "outputs": { 229 | "cluster": { 230 | "type": "object", 231 | "value": "[reference(resourceId('Microsoft.HDInsight/clusters', parameters('clusterName')))]" 232 | } 233 | } 234 | } 235 | -------------------------------------------------------------------------------- /src/arm/HDInsight5.0/hdinsight-kafka-2.4-vnet/azuredeploy.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "clusterName": { 6 | "type": "string", 7 | "metadata": { 8 | "description": "The name of the cluster to create." 9 | } 10 | }, 11 | "clusterVersion": { 12 | "type": "string", 13 | "defaultValue": "5.0", 14 | "metadata": { 15 | "description": "The HDInsight version to deploy." 16 | } 17 | }, 18 | "headNodeSize": { 19 | "type": "string", 20 | "defaultValue": "Standard_D4_V2", 21 | "metadata": { 22 | "description": "The VM size of the head nodes." 23 | } 24 | }, 25 | "workerNodeSize": { 26 | "type": "string", 27 | "defaultValue": "Standard_D4_V2", 28 | "metadata": { 29 | "description": "The VM size of the worker nodes." 30 | } 31 | }, 32 | "zookeeperNodeSize": { 33 | "type": "string", 34 | "defaultValue": "Standard_D3_V2", 35 | "metadata": { 36 | "description": "The VM size of the zookeeper nodes." 37 | } 38 | }, 39 | "clusterSize": { 40 | "type": "int", 41 | "defaultValue": 4, 42 | "metadata": { 43 | "description": "The number of worker nodes in the cluster." 44 | } 45 | }, 46 | "disksPerWorkerNode": { 47 | "type": "int", 48 | "defaultValue": 8, 49 | "metadata": { 50 | "description": "The number of disks per worker node." 51 | } 52 | }, 53 | "clusterLoginUserName": { 54 | "type": "string", 55 | "defaultValue": "admin", 56 | "metadata": { 57 | "description": "These credentials can be used to submit jobs to the cluster and to log into cluster dashboards." 58 | } 59 | }, 60 | "clusterLoginPassword": { 61 | "type": "securestring", 62 | "metadata": { 63 | "description": "The password must be at least 10 characters in length and must contain at least one digit, one non-alphanumeric character, and one upper or lower case letter." 64 | } 65 | }, 66 | "sshUserName": { 67 | "type": "string", 68 | "metadata": { 69 | "description": "These credentials can be used to remotely access the cluster." 70 | } 71 | }, 72 | "sshPassword": { 73 | "type": "securestring", 74 | "metadata": { 75 | "description": "The password must be at least 10 characters in length and must contain at least one digit, one non-alphanumeric character, and one upper or lower case letter." 76 | } 77 | } 78 | }, 79 | "variables": { 80 | "defaultApiVersion": "2015-06-15", 81 | "clusterApiVersion": "2015-03-01-preview", 82 | "defaultStorageName": "[concat('store', uniqueString(resourceGroup().id))]", 83 | "storageType": "Standard_LRS", 84 | "virtualNetworkName": "[concat('vnet', uniqueString(resourceGroup().id))]", 85 | "virtualNetworkPrefix": "10.0.0.0/16", 86 | "virtualNetworkDefaultSubnetName": "default", 87 | "virtualNetworkSubnet1Prefix": "10.0.0.0/20", 88 | "clusterType": "kafka", 89 | "componentVersion": "2.4" 90 | }, 91 | "resources": [ 92 | { 93 | "name": "[variables('defaultStorageName')]", 94 | "type": "Microsoft.Storage/storageAccounts", 95 | "location": "[resourceGroup().location]", 96 | "apiVersion": "2015-06-15", 97 | "dependsOn": [ ], 98 | "tags": { 99 | "displayName": "[variables('defaultStorageName')]" 100 | }, 101 | "properties": { 102 | "accountType": "[variables('storageType')]" 103 | } 104 | }, 105 | { 106 | "name": "[variables('virtualNetworkName')]", 107 | "type": "Microsoft.Network/virtualNetworks", 108 | "location": "[resourceGroup().location]", 109 | "apiVersion": "2015-06-15", 110 | "dependsOn": [ ], 111 | "tags": { 112 | "displayName": "[variables('virtualNetworkName')]" 113 | }, 114 | "properties": { 115 | "addressSpace": { 116 | "addressPrefixes": [ 117 | "[variables('virtualNetworkPrefix')]" 118 | ] 119 | }, 120 | "subnets": [ 121 | { 122 | "name": "[variables('virtualNetworkDefaultSubnetName')]", 123 | "properties": { 124 | "addressPrefix": "[variables('virtualNetworkSubnet1Prefix')]" 125 | } 126 | } 127 | ] 128 | } 129 | }, 130 | { 131 | "name": "[parameters('clusterName')]", 132 | "type": "Microsoft.HDInsight/clusters", 133 | "location": "[resourceGroup().location]", 134 | "apiVersion": "[variables('clusterApiVersion')]", 135 | "properties": { 136 | "clusterVersion": "[parameters('clusterVersion')]", 137 | "osType": "Linux", 138 | "clusterDefinition": { 139 | "kind": "[variables('clusterType')]", 140 | "componentVersion": { 141 | "Kafka": "[variables('componentVersion')]" 142 | }, 143 | "configurations": { 144 | "gateway": { 145 | "restAuthCredential.isEnabled": true, 146 | "restAuthCredential.username": "[parameters('clusterLoginUserName')]", 147 | "restAuthCredential.password": "[parameters('clusterLoginPassword')]" 148 | } 149 | } 150 | }, 151 | "storageProfile": { 152 | "storageaccounts": [ 153 | { 154 | "name": "[concat(variables('defaultStorageName'),'.blob.core.windows.net')]", 155 | "isDefault": true, 156 | "container": "[parameters('clusterName')]", 157 | "key": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('defaultStorageName')), variables('defaultApiVersion')).key1]" 158 | } 159 | ] 160 | }, 161 | "computeProfile": { 162 | "roles": [ 163 | { 164 | "name": "headnode", 165 | "targetInstanceCount": "2", 166 | "hardwareProfile": { 167 | "vmSize": "[parameters('headNodeSize')]" 168 | }, 169 | "osProfile": { 170 | "linuxOperatingSystemProfile": { 171 | "username": "[parameters('sshUserName')]", 172 | "password": "[parameters('sshPassword')]" 173 | } 174 | }, 175 | "virtualNetworkProfile": { 176 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 177 | "subnet": "[concat(resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName')), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 178 | } 179 | }, 180 | { 181 | "name": "workernode", 182 | "targetInstanceCount": "[parameters('clusterSize')]", 183 | "hardwareProfile": { 184 | "vmSize": "[parameters('workerNodeSize')]" 185 | }, 186 | "dataDisksGroups": [ 187 | { 188 | "disksPerNode": "[parameters('disksPerWorkerNode')]" 189 | } 190 | ], 191 | "osProfile": { 192 | "linuxOperatingSystemProfile": { 193 | "username": "[parameters('sshUserName')]", 194 | "password": "[parameters('sshPassword')]" 195 | } 196 | }, 197 | "virtualNetworkProfile": { 198 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 199 | "subnet": "[concat('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 200 | } 201 | }, 202 | { 203 | "name": "zookeepernode", 204 | "targetInstanceCount": 3, 205 | "hardwareProfile": { 206 | "vmSize": "[parameters('zookeeperNodeSize')]" 207 | }, 208 | "osProfile": { 209 | "linuxOperatingSystemProfile": { 210 | "username": "[parameters('sshUserName')]", 211 | "password": "[parameters('sshPassword')]" 212 | } 213 | }, 214 | "virtualNetworkProfile": { 215 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 216 | "subnet": "[concat('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 217 | } 218 | } 219 | ] 220 | } 221 | }, 222 | "dependsOn": [ 223 | "[variables('defaultStorageName')]", 224 | "[variables('virtualNetworkName')]" 225 | ] 226 | } 227 | ], 228 | "outputs": { 229 | "cluster": { 230 | "type": "object", 231 | "value": "[reference(resourceId('Microsoft.HDInsight/clusters', parameters('clusterName')))]" 232 | } 233 | } 234 | } 235 | -------------------------------------------------------------------------------- /src/python/troubleshooting/kafka_utils.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import json 3 | import logging 4 | import pprint 5 | import subprocess 6 | import sys 7 | import time 8 | 9 | from hdinsight_common import Constants 10 | from hdinsight_common.AmbariHelper import AmbariHelper 11 | from kazoo.client import KazooClient 12 | from kazoo.client import KazooState 13 | from retry import retry 14 | 15 | 16 | def zk_connection_loss_check(state): 17 | """Checks for Zookeeper connection loss 18 | """ 19 | if state == KazooState.LOST or state == KazooState.SUSPENDED: 20 | raise RuntimeError('Fatal error lost connection to zookeeper.') 21 | 22 | 23 | class KafkaUtils: 24 | """Utility class for Kafka troubleshooting scripts. 25 | 26 | The class provides methods and routines for various Ambari and Zookeeper operations. 27 | """ 28 | 29 | """Constants""" 30 | SERVICE_KAFKA = 'KAFKA' 31 | COMPONENT_KAFKA_BROKER = 'KAFKA_BROKER' 32 | ZK_KAFKA_BROKER_PATH = 'brokers/ids' 33 | ZK_KAFKA_CONTROLLER_PATH = 'controller' 34 | STATE_INSTALLED = 'INSTALLED' 35 | STATE_STARTED = 'STARTED' 36 | KAFKA_HOME = '/usr/hdp/current/kafka-broker/' 37 | KAFKA_LIBS_PATH = KAFKA_HOME + 'libs/' 38 | 39 | KAFKA_BROKER_PORT = '9092' 40 | ZOOKEEPER_PORT = '2181' 41 | 42 | TIMEOUT_SECS = 900 43 | SLEEP_SECS = 30 44 | 45 | def __init__(self, logger_arg, log_file, debug_mode=False): 46 | """Constructor for KafkaUtils that sets up the logging module and AmbariHelper. 47 | """ 48 | self.debug_mode = debug_mode 49 | self.logger = logger_arg 50 | self.logger.setLevel(logging.DEBUG) 51 | 52 | logging_format = '%(asctime)s - %(filename)s - %(name)s - %(levelname)s - %(message)s' 53 | # create console handler and set level to info 54 | handler = logging.StreamHandler() 55 | handler.setLevel(logging.INFO) 56 | formatter = logging.Formatter(logging_format) 57 | handler.setFormatter(formatter) 58 | self.logger.addHandler(handler) 59 | 60 | # create file handler and set level to debug 61 | handler = logging.FileHandler(log_file, 'a') 62 | handler.setLevel(logging.DEBUG) 63 | formatter = logging.Formatter(logging_format) 64 | handler.setFormatter(formatter) 65 | self.logger.addHandler(handler) 66 | 67 | self.logger.info('Log file: {0}'.format(log_file)) 68 | 69 | self.ambari_helper = AmbariHelper() 70 | self.cluster_manifest = self.ambari_helper.get_cluster_manifest() 71 | self.cluster_name = self.cluster_manifest.deployment.cluster_name 72 | 73 | def run_shell_command(self, shell_command, throw_on_error=True): 74 | """Runs a shell command locally and returns the stdout, stderr 75 | """ 76 | self.logger.info(shell_command) 77 | if not self.debug_mode: 78 | p = subprocess.Popen(shell_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 79 | stdout, stderr = p.communicate() 80 | 81 | if stdout: 82 | self.logger.info('\n' + stdout) 83 | 84 | if stderr: 85 | self.logger.error('\n' + stderr) 86 | 87 | if throw_on_error and (not (p.returncode == 0)): 88 | self.logger.error('Process returned non-zero exit code: {0}'.format(p.returncode)) 89 | sys.exit(p.returncode) 90 | 91 | return stdout, stderr 92 | 93 | def get_hosts_from_ambari(self): 94 | """Gets Hosts from Ambari. 95 | """ 96 | hosts_result = self.ambari_helper.request_url('clusters/{0}/hosts'.format(self.cluster_name), 'GET', 97 | {'fields': 'Hosts/ip'}) 98 | 99 | hosts = {} 100 | for host in hosts_result['items']: 101 | hosts[host['Hosts']['host_name']] = host['Hosts']['ip'] 102 | self.logger.debug('hosts: {0}\n'.format(pprint.pformat(hosts))) 103 | return hosts 104 | 105 | def get_zookeeper_hosts(self): 106 | """Gets Zookeeper hosts from Ambari. 107 | """ 108 | hosts = self.get_hosts_from_ambari() 109 | 110 | zk_hosts = {k: v for k, v in hosts.iteritems() if 111 | k.startswith(self.cluster_manifest.settings[Constants.ZOOKEEPER_VM_NAME_PREFIX_SETTING_KEY])} 112 | self.logger.debug('zk_hosts:\n{0}\n'.format(pprint.pformat(zk_hosts))) 113 | 114 | zk_quorum = reduce(lambda r1, r2: r1 + ',' + r2, 115 | map(lambda m: m + ':' + self.ZOOKEEPER_PORT, zk_hosts)) 116 | 117 | self.logger.debug('zk_quorum: {0}\n'.format(zk_quorum)) 118 | 119 | return zk_hosts, zk_quorum 120 | 121 | def get_zookeeper_quorum(self): 122 | """Gets the Zookeeper quorum. 123 | """ 124 | zk_hosts, zk_quorum = self.get_zookeeper_hosts() 125 | return zk_quorum 126 | 127 | def get_brokers_from_ambari(self): 128 | """Gets Broker Information. Returns Dictionary of Brokers hosts and the Broker Connection String. 129 | """ 130 | hosts = self.get_hosts_from_ambari() 131 | 132 | broker_hosts = {k: v for k, v in hosts.iteritems() if 133 | k.startswith(self.cluster_manifest.settings[Constants.WORKERNODE_VM_NAME_PREFIX_SETTING_KEY])} 134 | self.logger.debug('broker_hosts:\n{0}\n'.format(pprint.pformat(broker_hosts))) 135 | 136 | brokers = reduce(lambda r1, r2: r1 + ',' + r2, 137 | map(lambda m: m + ':' + self.KAFKA_BROKER_PORT, 138 | broker_hosts)) 139 | self.logger.debug('brokers: {0}\n'.format(brokers)) 140 | 141 | return broker_hosts, brokers 142 | 143 | def get_brokers_from_zookeeper(self): 144 | """Gets the Kafka Broker information from Zookeeper for live brokers. 145 | 146 | Returns a dictionary of Brokers with host name as the key, where each entry contains another dictionary for 147 | that broker information 148 | """ 149 | zk_quorum = self.get_zookeeper_quorum() 150 | zk = self.zk_connect(zk_quorum) 151 | 152 | zk_brokers_ids = zk.get_children(self.ZK_KAFKA_BROKER_PATH) 153 | 154 | zk_brokers_info = {} 155 | for zk_broker_id in zk_brokers_ids: 156 | zk_broker_id_data, stat = zk.get('{0}/{1}'.format(self.ZK_KAFKA_BROKER_PATH, zk_broker_id)) 157 | zk_broker_info = json.loads(zk_broker_id_data) 158 | zk_broker_info['id'] = zk_broker_id 159 | zk_broker_datetime = datetime.datetime.utcfromtimestamp(int(zk_broker_info['timestamp']) / 1000.0) 160 | zk_broker_info['datetime'] = zk_broker_datetime.strftime('%Y-%m-%d %H:%M:%S') 161 | zk_broker_info['duration'] = str((datetime.datetime.utcnow() - zk_broker_datetime)) 162 | zk_brokers_info[zk_broker_info['host']] = zk_broker_info 163 | 164 | zk.stop() 165 | return zk_brokers_info 166 | 167 | def get_controller_from_zookeeper(self): 168 | """Gets the Kafka controller information from Zookeeper as a dict. 169 | """ 170 | zk_quorum = self.get_zookeeper_quorum() 171 | zk = self.zk_connect(zk_quorum) 172 | zk_controller_data, stat = zk.get(self.ZK_KAFKA_CONTROLLER_PATH) 173 | zk.stop() 174 | 175 | zk_controller = json.loads(zk_controller_data) 176 | zk_controller_datetime = datetime.datetime.utcfromtimestamp(int(zk_controller['timestamp']) / 1000.0) 177 | zk_controller['datetime'] = zk_controller_datetime.strftime('%Y-%m-%d %H:%M:%S') 178 | zk_controller['duration'] = str((datetime.datetime.utcnow() - zk_controller_datetime)) 179 | 180 | return zk_controller 181 | 182 | def get_stale_broker_hosts_from_ambari(self): 183 | """Gets Brokers from Ambari with Stale configs. 184 | """ 185 | return self.get_stale_hosts_from_ambari(self.COMPONENT_KAFKA_BROKER) 186 | 187 | def get_stale_hosts_from_ambari(self, component_name): 188 | """Gets hosts from Ambari that have Stale configs for the given component_name 189 | """ 190 | params = {'HostRoles/stale_configs': 'true', 'HostRoles/component_name': component_name} 191 | hosts_result = self.ambari_helper.request_url('clusters/{0}/host_components'.format(self.cluster_name), 'GET', 192 | params) 193 | 194 | stale_hosts = map(lambda m: m['HostRoles']['host_name'], 195 | filter(lambda h: h['HostRoles']['host_name'].startswith( 196 | self.cluster_manifest.settings[Constants.WORKERNODE_VM_NAME_PREFIX_SETTING_KEY]), 197 | hosts_result['items'])) 198 | self.logger.info( 199 | 'Hosts with stale configs for component {0}: {1}\n{2}\n'.format(component_name, len(stale_hosts), 200 | pprint.pformat(stale_hosts))) 201 | return stale_hosts 202 | 203 | def restart_kafka_broker_from_ambari(self, host_name): 204 | """Restart the Kafka Broker on the host. 205 | """ 206 | return self.restart_component_from_ambari(host_name, self.SERVICE_KAFKA, self.COMPONENT_KAFKA_BROKER) 207 | 208 | def restart_component_from_ambari(self, host_name, service_name, component_name): 209 | """Restart the component for a service on the host. 210 | """ 211 | self.stop_component_from_ambari(host_name, service_name, component_name) 212 | self.start_component_from_ambari(host_name, service_name, component_name) 213 | 214 | def get_component_from_ambari(self, host_name, service_name, component_name): 215 | """Gets the component state from Ambari for the host. 216 | """ 217 | host_component_url = 'clusters/{0}/hosts/{1}/host_components/{2}'.format(self.cluster_name, host_name, 218 | component_name) 219 | return self.ambari_helper.query_url(host_component_url) 220 | 221 | def stop_component_from_ambari(self, host_name, service_name, component_name): 222 | """Stops the component on the host and waits until it stops. 223 | """ 224 | self.change_host_component_state_from_ambari(host_name, service_name, component_name, self.STATE_INSTALLED) 225 | return self.wait_for_component_state_from_ambari(host_name, service_name, component_name, self.STATE_INSTALLED) 226 | 227 | def start_component_from_ambari(self, host_name, service_name, component_name): 228 | """Starts the component on the hosts and waits until it is started. 229 | """ 230 | self.change_host_component_state_from_ambari(host_name, service_name, component_name, self.STATE_STARTED) 231 | return self.wait_for_component_state_from_ambari(host_name, service_name, component_name, self.STATE_STARTED) 232 | 233 | def wait_for_component_state_from_ambari(self, host_name, service_name, component_name, state): 234 | """Waits for the component to reach the desired state. 235 | """ 236 | now = time.time() 237 | timeout = now + self.TIMEOUT_SECS 238 | while time.time() < timeout: 239 | component = self.get_component_from_ambari(host_name, service_name, component_name) 240 | if component['HostRoles']['state'] == state: 241 | self.logger.debug( 242 | 'Component {0} on host {1} reached the desired state: {2}'.format(component_name, host_name, state)) 243 | return component 244 | else: 245 | self.logger.debug( 246 | 'Component {0} on host {1} has not yet reached the desired state: {2}. Current state: {3}, ' 247 | 'Time Elapsed: {4:.2f}'.format( 248 | component_name, host_name, state, component['HostRoles']['state'], (time.time() - now))) 249 | time.sleep(self.SLEEP_SECS) 250 | err_msg = 'Component {0} on host {1} did not reach the desired state: {2} in {3} secs.'.format(component_name, 251 | host_name, state, 252 | self.TIMEOUT_SECS) 253 | self.logger.error(err_msg) 254 | raise RuntimeError(err_msg) 255 | 256 | @retry(exceptions=BaseException, tries=Constants.MAX_RETRIES, delay=Constants.RETRY_INTERVAL_DELAY, 257 | backoff=Constants.RETRY_INTERVAL_BACKOFF) 258 | def change_host_component_state_from_ambari(self, host_name, service_name, component_name, state): 259 | """ 260 | Convenience method for changing the state of a particular host component. State values can be 'STARTED' or 'INSTALLED' 261 | """ 262 | host_component_url = 'clusters/{0}/hosts/{1}/host_components/{2}'.format(self.cluster_name, host_name, 263 | component_name) 264 | payload = { 265 | 'RequestInfo': { 266 | 'context': 'Change host component state for host {0} and component {1} to state {2} via REST'.format( 267 | host_name, component_name, state) 268 | }, 269 | 'Body': { 270 | 'HostRoles': { 271 | 'state': state 272 | } 273 | } 274 | } 275 | json_payload = json.dumps(payload) 276 | return self.ambari_helper.put_url(host_component_url, json_payload) 277 | 278 | def zk_connection_loss_check(self, state): 279 | """Checks for Zookeeper connection loss 280 | """ 281 | if state == KazooState.LOST or state == KazooState.SUSPENDED: 282 | raise RuntimeError('Fatal error lost connection to zookeeper.') 283 | 284 | @retry(exceptions=BaseException, tries=3, delay=1, backoff=2) 285 | def zk_connect(self, zk_quorum): 286 | """Connect to the specified Zookeeper quorum using the connection string. 287 | """ 288 | self.logger.debug('Connecting to zookeeper quorum at: {0}'.format(zk_quorum)) 289 | zk = KazooClient(hosts=zk_quorum) 290 | zk.start() 291 | zk.add_listener(self.zk_connection_loss_check) 292 | return zk 293 | 294 | def get_kafka_hdp_version(self): 295 | p1 = subprocess.Popen(["find {0} -name \*kafka_\*".format(self.KAFKA_LIBS_PATH)], shell=True, stdout=subprocess.PIPE) 296 | data = p1.stdout.readline() 297 | assert p1.wait() == 0 298 | data = data.split('\n')[0].split('/')[-1].split('-') 299 | splits = data[1].split('.') 300 | kafka_version = ".".join(splits[:3]) 301 | 302 | hdp_version = ".".join(splits[3:]) 303 | return kafka_version, hdp_version 304 | -------------------------------------------------------------------------------- /src/arm/HDInsight4.0/hdinsight-kafka-2.1-hadoop-3.1-vnet/azuredeploy.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "clusterNameSuffix": { 6 | "type": "string", 7 | "metadata": { 8 | "description": "The cluster name suffix, cluster type is the prefix" 9 | } 10 | }, 11 | "clusterVersion": { 12 | "type": "string", 13 | "defaultValue": "4.0", 14 | "metadata": { 15 | "description": "The HDInsight version to deploy." 16 | } 17 | }, 18 | "headNodeSize": { 19 | "type": "string", 20 | "defaultValue": "Standard_D4_V2", 21 | "metadata": { 22 | "description": "The VM size of the head nodes." 23 | } 24 | }, 25 | "workerNodeSize": { 26 | "type": "string", 27 | "defaultValue": "Standard_D4_V2", 28 | "metadata": { 29 | "description": "The VM size of the worker nodes." 30 | } 31 | }, 32 | "zookeeperNodeSize": { 33 | "type": "string", 34 | "defaultValue": "Standard_D3_V2", 35 | "metadata": { 36 | "description": "The VM size of the zookeeper nodes." 37 | } 38 | }, 39 | "clusterSize": { 40 | "type": "int", 41 | "defaultValue": 4, 42 | "metadata": { 43 | "description": "The number of worker nodes in the cluster." 44 | } 45 | }, 46 | "disksPerWorkerNode": { 47 | "type": "int", 48 | "defaultValue": 8, 49 | "metadata": { 50 | "description": "The number of disks per worker node." 51 | } 52 | }, 53 | "clusterLoginUserName": { 54 | "type": "string", 55 | "defaultValue": "admin", 56 | "metadata": { 57 | "description": "These credentials can be used to submit jobs to the cluster and to log into cluster dashboards." 58 | } 59 | }, 60 | "clusterLoginPassword": { 61 | "type": "securestring", 62 | "metadata": { 63 | "description": "The password must be at least 10 characters in length and must contain at least one digit, one non-alphanumeric character, and one upper or lower case letter." 64 | } 65 | }, 66 | "sshUserName": { 67 | "type": "string", 68 | "metadata": { 69 | "description": "These credentials can be used to remotely access the cluster." 70 | } 71 | }, 72 | "sshPassword": { 73 | "type": "securestring", 74 | "metadata": { 75 | "description": "The password must be at least 10 characters in length and must contain at least one digit, one non-alphanumeric character, and one upper or lower case letter." 76 | } 77 | } 78 | }, 79 | "variables": { 80 | "defaultApiVersion": "2015-06-15", 81 | "clusterApiVersion": "2015-03-01-preview", 82 | "defaultStorageName": "[concat('store', uniqueString(resourceGroup().id))]", 83 | "storageType": "Standard_LRS", 84 | "virtualNetworkName": "[concat('vnet', uniqueString(resourceGroup().id))]", 85 | "virtualNetworkPrefix": "10.0.0.0/16", 86 | "virtualNetworkDefaultSubnetName": "default", 87 | "virtualNetworkSubnet1Prefix": "10.0.0.0/20", 88 | "clusterType": "kafka", 89 | "clusterName": "[concat(variables('clusterType'), parameters('clusterNameSuffix'))]", 90 | "hadoopClusterType": "hadoop", 91 | "hadoopClusterName": "[concat(variables('hadoopClusterType'), parameters('clusterNameSuffix'))]" 92 | }, 93 | "resources": [ 94 | { 95 | "name": "[variables('defaultStorageName')]", 96 | "type": "Microsoft.Storage/storageAccounts", 97 | "location": "[resourceGroup().location]", 98 | "apiVersion": "2015-06-15", 99 | "dependsOn": [ ], 100 | "tags": { 101 | "displayName": "[variables('defaultStorageName')]" 102 | }, 103 | "properties": { 104 | "accountType": "[variables('storageType')]" 105 | } 106 | }, 107 | { 108 | "name": "[variables('virtualNetworkName')]", 109 | "type": "Microsoft.Network/virtualNetworks", 110 | "location": "[resourceGroup().location]", 111 | "apiVersion": "2015-06-15", 112 | "dependsOn": [ ], 113 | "tags": { 114 | "displayName": "[variables('virtualNetworkName')]" 115 | }, 116 | "properties": { 117 | "addressSpace": { 118 | "addressPrefixes": [ 119 | "[variables('virtualNetworkPrefix')]" 120 | ] 121 | }, 122 | "subnets": [ 123 | { 124 | "name": "[variables('virtualNetworkDefaultSubnetName')]", 125 | "properties": { 126 | "addressPrefix": "[variables('virtualNetworkSubnet1Prefix')]" 127 | } 128 | } 129 | ] 130 | } 131 | }, 132 | { 133 | "name": "[variables('clusterName')]", 134 | "type": "Microsoft.HDInsight/clusters", 135 | "location": "[resourceGroup().location]", 136 | "apiVersion": "[variables('clusterApiVersion')]", 137 | "properties": { 138 | "clusterVersion": "[parameters('clusterVersion')]", 139 | "osType": "Linux", 140 | "clusterDefinition": { 141 | "kind": "[variables('clusterType')]", 142 | "configurations": { 143 | "gateway": { 144 | "restAuthCredential.isEnabled": true, 145 | "restAuthCredential.username": "[parameters('clusterLoginUserName')]", 146 | "restAuthCredential.password": "[parameters('clusterLoginPassword')]" 147 | } 148 | } 149 | }, 150 | "storageProfile": { 151 | "storageaccounts": [ 152 | { 153 | "name": "[concat(variables('defaultStorageName'),'.blob.core.windows.net')]", 154 | "isDefault": true, 155 | "container": "[variables('clusterName')]", 156 | "key": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('defaultStorageName')), variables('defaultApiVersion')).key1]" 157 | } 158 | ] 159 | }, 160 | "computeProfile": { 161 | "roles": [ 162 | { 163 | "name": "headnode", 164 | "targetInstanceCount": "2", 165 | "hardwareProfile": { 166 | "vmSize": "[parameters('headNodeSize')]" 167 | }, 168 | "osProfile": { 169 | "linuxOperatingSystemProfile": { 170 | "username": "[parameters('sshUserName')]", 171 | "password": "[parameters('sshPassword')]" 172 | } 173 | }, 174 | "virtualNetworkProfile": { 175 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 176 | "subnet": "[concat(resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName')), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 177 | } 178 | }, 179 | { 180 | "name": "workernode", 181 | "targetInstanceCount": "[parameters('clusterSize')]", 182 | "hardwareProfile": { 183 | "vmSize": "[parameters('workerNodeSize')]" 184 | }, 185 | "dataDisksGroups": [ 186 | { 187 | "disksPerNode": "[parameters('disksPerWorkerNode')]" 188 | } 189 | ], 190 | "osProfile": { 191 | "linuxOperatingSystemProfile": { 192 | "username": "[parameters('sshUserName')]", 193 | "password": "[parameters('sshPassword')]" 194 | } 195 | }, 196 | "virtualNetworkProfile": { 197 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 198 | "subnet": "[concat('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 199 | } 200 | }, 201 | { 202 | "name": "zookeepernode", 203 | "targetInstanceCount": 3, 204 | "hardwareProfile": { 205 | "vmSize": "[parameters('zookeeperNodeSize')]" 206 | }, 207 | "osProfile": { 208 | "linuxOperatingSystemProfile": { 209 | "username": "[parameters('sshUserName')]", 210 | "password": "[parameters('sshPassword')]" 211 | } 212 | }, 213 | "virtualNetworkProfile": { 214 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 215 | "subnet": "[concat('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 216 | } 217 | } 218 | ] 219 | } 220 | }, 221 | "dependsOn": [ 222 | "[variables('defaultStorageName')]", 223 | "[variables('virtualNetworkName')]" 224 | ] 225 | }, 226 | { 227 | "name": "[variables('hadoopClusterName')]", 228 | "type": "Microsoft.HDInsight/clusters", 229 | "location": "[resourceGroup().location]", 230 | "apiVersion": "[variables('clusterApiVersion')]", 231 | "properties": { 232 | "clusterVersion": "[parameters('clusterVersion')]", 233 | "osType": "Linux", 234 | "clusterDefinition": { 235 | "kind": "[variables('hadoopClusterType')]", 236 | "configurations": { 237 | "gateway": { 238 | "restAuthCredential.isEnabled": true, 239 | "restAuthCredential.username": "[parameters('clusterLoginUserName')]", 240 | "restAuthCredential.password": "[parameters('clusterLoginPassword')]" 241 | } 242 | } 243 | }, 244 | "storageProfile": { 245 | "storageaccounts": [ 246 | { 247 | "name": "[concat(variables('defaultStorageName'),'.blob.core.windows.net')]", 248 | "isDefault": true, 249 | "container": "[variables('hadoopClusterName')]", 250 | "key": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('defaultStorageName')), variables('defaultApiVersion')).key1]" 251 | } 252 | ] 253 | }, 254 | "computeProfile": { 255 | "roles": [ 256 | { 257 | "name": "headnode", 258 | "targetInstanceCount": "2", 259 | "hardwareProfile": { 260 | "vmSize": "[parameters('headNodeSize')]" 261 | }, 262 | "osProfile": { 263 | "linuxOperatingSystemProfile": { 264 | "username": "[parameters('sshUserName')]", 265 | "password": "[parameters('sshPassword')]" 266 | } 267 | }, 268 | "virtualNetworkProfile": { 269 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 270 | "subnet": "[concat(resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName')), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 271 | } 272 | }, 273 | { 274 | "name": "workernode", 275 | "targetInstanceCount": "[parameters('clusterSize')]", 276 | "hardwareProfile": { 277 | "vmSize": "[parameters('workerNodeSize')]" 278 | }, 279 | "osProfile": { 280 | "linuxOperatingSystemProfile": { 281 | "username": "[parameters('sshUserName')]", 282 | "password": "[parameters('sshPassword')]" 283 | } 284 | }, 285 | "virtualNetworkProfile": { 286 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 287 | "subnet": "[concat('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 288 | } 289 | } 290 | ] 291 | } 292 | }, 293 | "dependsOn": [ 294 | "[variables('defaultStorageName')]", 295 | "[variables('virtualNetworkName')]" 296 | ] 297 | } 298 | ], 299 | "outputs": { 300 | "cluster": { 301 | "type": "object", 302 | "value": "[reference(resourceId('Microsoft.HDInsight/clusters', variables('clusterName')))]" 303 | }, 304 | "hadoopCluster": { 305 | "type": "object", 306 | "value": "[reference(resourceId('Microsoft.HDInsight/clusters', variables('hadoopClusterName')))]" 307 | } 308 | } 309 | } -------------------------------------------------------------------------------- /src/arm/HDInsight4.0/hdinsight-kafka-2.1-spark-2.4-vnet/azuredeploy.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "clusterNameSuffix": { 6 | "type": "string", 7 | "metadata": { 8 | "description": "The cluster name suffix, cluster type is the prefix" 9 | } 10 | }, 11 | "clusterVersion": { 12 | "type": "string", 13 | "defaultValue": "4.0", 14 | "metadata": { 15 | "description": "The HDInsight version to deploy." 16 | } 17 | }, 18 | "headNodeSize": { 19 | "type": "string", 20 | "defaultValue": "Standard_D4_v2", 21 | "metadata": { 22 | "description": "The VM size of the head nodes." 23 | } 24 | }, 25 | "workerNodeSize": { 26 | "type": "string", 27 | "defaultValue": "Standard_D4_v2", 28 | "metadata": { 29 | "description": "The VM size of the worker nodes." 30 | } 31 | }, 32 | "zookeeperNodeSize": { 33 | "type": "string", 34 | "defaultValue": "Standard_D3_v2", 35 | "metadata": { 36 | "description": "The VM size of the zookeeper nodes." 37 | } 38 | }, 39 | "clusterSize": { 40 | "type": "int", 41 | "defaultValue": 4, 42 | "metadata": { 43 | "description": "The number of worker nodes in the cluster." 44 | } 45 | }, 46 | "disksPerWorkerNode": { 47 | "type": "int", 48 | "defaultValue": 8, 49 | "metadata": { 50 | "description": "The number of disks per worker node." 51 | } 52 | }, 53 | "clusterLoginUserName": { 54 | "type": "string", 55 | "defaultValue": "admin", 56 | "metadata": { 57 | "description": "These credentials can be used to submit jobs to the cluster and to log into cluster dashboards." 58 | } 59 | }, 60 | "clusterLoginPassword": { 61 | "type": "securestring", 62 | "metadata": { 63 | "description": "The password must be at least 10 characters in length and must contain at least one digit, one non-alphanumeric character, and one upper or lower case letter." 64 | } 65 | }, 66 | "sshUserName": { 67 | "type": "string", 68 | "metadata": { 69 | "description": "These credentials can be used to remotely access the cluster." 70 | } 71 | }, 72 | "sshPassword": { 73 | "type": "securestring", 74 | "metadata": { 75 | "description": "The password must be at least 10 characters in length and must contain at least one digit, one non-alphanumeric character, and one upper or lower case letter." 76 | } 77 | }, 78 | "kafkaVersion": { 79 | "type": "string", 80 | "defaultValue": "2.1", 81 | "metadata": { 82 | "description": "Kafka version to be installed in the cluster. This parameter only applies to Kafka cluster type in HDI 4.0 and will fail for other cluster types and other cluster versions." 83 | } 84 | }, 85 | "sparkVersion": { 86 | "type": "string", 87 | "defaultValue": "2.4", 88 | "metadata": { 89 | "description": "Spark version to be installed in the cluster. This parameter only applies to Spark cluster type in HDI 4.0 and will fail for other cluster types and other cluster versions." 90 | } 91 | } 92 | }, 93 | "variables": { 94 | "defaultApiVersion": "2015-06-15", 95 | "clusterApiVersion": "2015-03-01-preview", 96 | "defaultStorageName": "[concat('store', uniqueString(resourceGroup().id))]", 97 | "storageType": "Standard_LRS", 98 | "virtualNetworkName": "[concat('vnet', uniqueString(resourceGroup().id))]", 99 | "virtualNetworkPrefix": "10.0.0.0/16", 100 | "virtualNetworkDefaultSubnetName": "default", 101 | "virtualNetworkSubnet1Prefix": "10.0.0.0/20", 102 | "clusterType": "kafka", 103 | "clusterName": "[concat(variables('clusterType'), parameters('clusterNameSuffix'))]", 104 | "sparkClusterType": "spark", 105 | "sparkClusterName": "[concat(variables('sparkClusterType'), parameters('clusterNameSuffix'))]" 106 | }, 107 | "resources": [ 108 | { 109 | "name": "[variables('defaultStorageName')]", 110 | "type": "Microsoft.Storage/storageAccounts", 111 | "location": "[resourceGroup().location]", 112 | "apiVersion": "2015-06-15", 113 | "dependsOn": [ ], 114 | "tags": { 115 | "displayName": "[variables('defaultStorageName')]" 116 | }, 117 | "properties": { 118 | "accountType": "[variables('storageType')]" 119 | } 120 | }, 121 | { 122 | "name": "[variables('virtualNetworkName')]", 123 | "type": "Microsoft.Network/virtualNetworks", 124 | "location": "[resourceGroup().location]", 125 | "apiVersion": "2015-06-15", 126 | "dependsOn": [ ], 127 | "tags": { 128 | "displayName": "[variables('virtualNetworkName')]" 129 | }, 130 | "properties": { 131 | "addressSpace": { 132 | "addressPrefixes": [ 133 | "[variables('virtualNetworkPrefix')]" 134 | ] 135 | }, 136 | "subnets": [ 137 | { 138 | "name": "[variables('virtualNetworkDefaultSubnetName')]", 139 | "properties": { 140 | "addressPrefix": "[variables('virtualNetworkSubnet1Prefix')]" 141 | } 142 | } 143 | ] 144 | } 145 | }, 146 | { 147 | "name": "[variables('clusterName')]", 148 | "type": "Microsoft.HDInsight/clusters", 149 | "location": "[resourceGroup().location]", 150 | "apiVersion": "[variables('clusterApiVersion')]", 151 | "properties": { 152 | "clusterVersion": "[parameters('clusterVersion')]", 153 | "osType": "Linux", 154 | "clusterDefinition": { 155 | "kind": "[variables('clusterType')]", 156 | "componentVersion": { 157 | "Kafka": "[parameters('kafkaVersion')]" 158 | }, 159 | "configurations": { 160 | "gateway": { 161 | "restAuthCredential.isEnabled": true, 162 | "restAuthCredential.username": "[parameters('clusterLoginUserName')]", 163 | "restAuthCredential.password": "[parameters('clusterLoginPassword')]" 164 | } 165 | } 166 | }, 167 | "storageProfile": { 168 | "storageaccounts": [ 169 | { 170 | "name": "[concat(variables('defaultStorageName'),'.blob.core.windows.net')]", 171 | "isDefault": true, 172 | "container": "[variables('clusterName')]", 173 | "key": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('defaultStorageName')), variables('defaultApiVersion')).key1]" 174 | } 175 | ] 176 | }, 177 | "computeProfile": { 178 | "roles": [ 179 | { 180 | "name": "headnode", 181 | "targetInstanceCount": "2", 182 | "hardwareProfile": { 183 | "vmSize": "[parameters('headNodeSize')]" 184 | }, 185 | "osProfile": { 186 | "linuxOperatingSystemProfile": { 187 | "username": "[parameters('sshUserName')]", 188 | "password": "[parameters('sshPassword')]" 189 | } 190 | }, 191 | "virtualNetworkProfile": { 192 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 193 | "subnet": "[concat(resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName')), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 194 | } 195 | }, 196 | { 197 | "name": "workernode", 198 | "targetInstanceCount": "[parameters('clusterSize')]", 199 | "hardwareProfile": { 200 | "vmSize": "[parameters('workerNodeSize')]" 201 | }, 202 | "dataDisksGroups": [ 203 | { 204 | "disksPerNode": "[parameters('disksPerWorkerNode')]" 205 | } 206 | ], 207 | "osProfile": { 208 | "linuxOperatingSystemProfile": { 209 | "username": "[parameters('sshUserName')]", 210 | "password": "[parameters('sshPassword')]" 211 | } 212 | }, 213 | "virtualNetworkProfile": { 214 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 215 | "subnet": "[concat('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 216 | } 217 | }, 218 | { 219 | "name": "zookeepernode", 220 | "targetInstanceCount": 3, 221 | "hardwareProfile": { 222 | "vmSize": "[parameters('zookeeperNodeSize')]" 223 | }, 224 | "osProfile": { 225 | "linuxOperatingSystemProfile": { 226 | "username": "[parameters('sshUserName')]", 227 | "password": "[parameters('sshPassword')]" 228 | } 229 | }, 230 | "virtualNetworkProfile": { 231 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 232 | "subnet": "[concat('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 233 | } 234 | } 235 | ] 236 | } 237 | }, 238 | "dependsOn": [ 239 | "[variables('defaultStorageName')]", 240 | "[variables('virtualNetworkName')]" 241 | ] 242 | }, 243 | { 244 | "name": "[variables('sparkClusterName')]", 245 | "type": "Microsoft.HDInsight/clusters", 246 | "location": "[resourceGroup().location]", 247 | "apiVersion": "[variables('clusterApiVersion')]", 248 | "properties": { 249 | "clusterVersion": "[parameters('clusterVersion')]", 250 | "osType": "Linux", 251 | "clusterDefinition": { 252 | "kind": "[variables('sparkClusterType')]", 253 | "componentVersion": { 254 | "Spark": "[parameters('sparkVersion')]" 255 | }, 256 | "configurations": { 257 | "gateway": { 258 | "restAuthCredential.isEnabled": true, 259 | "restAuthCredential.username": "[parameters('clusterLoginUserName')]", 260 | "restAuthCredential.password": "[parameters('clusterLoginPassword')]" 261 | } 262 | } 263 | }, 264 | "storageProfile": { 265 | "storageaccounts": [ 266 | { 267 | "name": "[concat(variables('defaultStorageName'),'.blob.core.windows.net')]", 268 | "isDefault": true, 269 | "container": "[variables('sparkClusterName')]", 270 | "key": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('defaultStorageName')), variables('defaultApiVersion')).key1]" 271 | } 272 | ] 273 | }, 274 | "computeProfile": { 275 | "roles": [ 276 | { 277 | "name": "headnode", 278 | "targetInstanceCount": "2", 279 | "hardwareProfile": { 280 | "vmSize": "[parameters('headNodeSize')]" 281 | }, 282 | "osProfile": { 283 | "linuxOperatingSystemProfile": { 284 | "username": "[parameters('sshUserName')]", 285 | "password": "[parameters('sshPassword')]" 286 | } 287 | }, 288 | "virtualNetworkProfile": { 289 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 290 | "subnet": "[concat(resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName')), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 291 | } 292 | }, 293 | { 294 | "name": "workernode", 295 | "targetInstanceCount": "[parameters('clusterSize')]", 296 | "hardwareProfile": { 297 | "vmSize": "[parameters('workerNodeSize')]" 298 | }, 299 | "osProfile": { 300 | "linuxOperatingSystemProfile": { 301 | "username": "[parameters('sshUserName')]", 302 | "password": "[parameters('sshPassword')]" 303 | } 304 | }, 305 | "virtualNetworkProfile": { 306 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 307 | "subnet": "[concat('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 308 | } 309 | } 310 | ] 311 | } 312 | }, 313 | "dependsOn": [ 314 | "[variables('defaultStorageName')]", 315 | "[variables('virtualNetworkName')]" 316 | ] 317 | } 318 | ], 319 | "outputs": { 320 | "cluster": { 321 | "type": "object", 322 | "value": "[reference(resourceId('Microsoft.HDInsight/clusters', variables('clusterName')))]" 323 | }, 324 | "sparkCluster": { 325 | "type": "object", 326 | "value": "[reference(resourceId('Microsoft.HDInsight/clusters', variables('sparkClusterName')))]" 327 | } 328 | } 329 | } 330 | -------------------------------------------------------------------------------- /src/arm/HDInsight5.0/hdinsight-kafka-2.4-spark-3.1-vnet/azuredeploy.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "clusterNameSuffix": { 6 | "type": "string", 7 | "metadata": { 8 | "description": "The cluster name suffix, cluster type is the prefix" 9 | } 10 | }, 11 | "clusterVersion": { 12 | "type": "string", 13 | "defaultValue": "5.0", 14 | "metadata": { 15 | "description": "The HDInsight version to deploy." 16 | } 17 | }, 18 | "headNodeSize": { 19 | "type": "string", 20 | "defaultValue": "Standard_D4_v2", 21 | "metadata": { 22 | "description": "The VM size of the head nodes." 23 | } 24 | }, 25 | "workerNodeSize": { 26 | "type": "string", 27 | "defaultValue": "Standard_D4_v2", 28 | "metadata": { 29 | "description": "The VM size of the worker nodes." 30 | } 31 | }, 32 | "zookeeperNodeSize": { 33 | "type": "string", 34 | "defaultValue": "Standard_D3_v2", 35 | "metadata": { 36 | "description": "The VM size of the zookeeper nodes." 37 | } 38 | }, 39 | "clusterSize": { 40 | "type": "int", 41 | "defaultValue": 4, 42 | "metadata": { 43 | "description": "The number of worker nodes in the cluster." 44 | } 45 | }, 46 | "disksPerWorkerNode": { 47 | "type": "int", 48 | "defaultValue": 8, 49 | "metadata": { 50 | "description": "The number of disks per worker node." 51 | } 52 | }, 53 | "clusterLoginUserName": { 54 | "type": "string", 55 | "defaultValue": "admin", 56 | "metadata": { 57 | "description": "These credentials can be used to submit jobs to the cluster and to log into cluster dashboards." 58 | } 59 | }, 60 | "clusterLoginPassword": { 61 | "type": "securestring", 62 | "metadata": { 63 | "description": "The password must be at least 10 characters in length and must contain at least one digit, one non-alphanumeric character, and one upper or lower case letter." 64 | } 65 | }, 66 | "sshUserName": { 67 | "type": "string", 68 | "metadata": { 69 | "description": "These credentials can be used to remotely access the cluster." 70 | } 71 | }, 72 | "sshPassword": { 73 | "type": "securestring", 74 | "metadata": { 75 | "description": "The password must be at least 10 characters in length and must contain at least one digit, one non-alphanumeric character, and one upper or lower case letter." 76 | } 77 | }, 78 | "kafkaVersion": { 79 | "type": "string", 80 | "defaultValue": "2.4", 81 | "metadata": { 82 | "description": "Kafka version to be installed in the cluster. This parameter applies to Kafka cluster type in HDI 4.0/5.0 and will fail for other cluster types and other cluster versions." 83 | } 84 | }, 85 | "sparkVersion": { 86 | "type": "string", 87 | "defaultValue": "3.1", 88 | "metadata": { 89 | "description": "Spark version to be installed in the cluster. This parameter only applies to Spark cluster type in HDI 5.0 and will fail for other cluster types and other cluster versions." 90 | } 91 | } 92 | }, 93 | "variables": { 94 | "defaultApiVersion": "2015-06-15", 95 | "clusterApiVersion": "2015-03-01-preview", 96 | "defaultStorageName": "[concat('store', uniqueString(resourceGroup().id))]", 97 | "storageType": "Standard_LRS", 98 | "virtualNetworkName": "[concat('vnet', uniqueString(resourceGroup().id))]", 99 | "virtualNetworkPrefix": "10.0.0.0/16", 100 | "virtualNetworkDefaultSubnetName": "default", 101 | "virtualNetworkSubnet1Prefix": "10.0.0.0/20", 102 | "clusterType": "kafka", 103 | "clusterName": "[concat(variables('clusterType'), parameters('clusterNameSuffix'))]", 104 | "sparkClusterType": "spark", 105 | "sparkClusterName": "[concat(variables('sparkClusterType'), parameters('clusterNameSuffix'))]" 106 | }, 107 | "resources": [ 108 | { 109 | "name": "[variables('defaultStorageName')]", 110 | "type": "Microsoft.Storage/storageAccounts", 111 | "location": "[resourceGroup().location]", 112 | "apiVersion": "2015-06-15", 113 | "dependsOn": [ ], 114 | "tags": { 115 | "displayName": "[variables('defaultStorageName')]" 116 | }, 117 | "properties": { 118 | "accountType": "[variables('storageType')]" 119 | } 120 | }, 121 | { 122 | "name": "[variables('virtualNetworkName')]", 123 | "type": "Microsoft.Network/virtualNetworks", 124 | "location": "[resourceGroup().location]", 125 | "apiVersion": "2015-06-15", 126 | "dependsOn": [ ], 127 | "tags": { 128 | "displayName": "[variables('virtualNetworkName')]" 129 | }, 130 | "properties": { 131 | "addressSpace": { 132 | "addressPrefixes": [ 133 | "[variables('virtualNetworkPrefix')]" 134 | ] 135 | }, 136 | "subnets": [ 137 | { 138 | "name": "[variables('virtualNetworkDefaultSubnetName')]", 139 | "properties": { 140 | "addressPrefix": "[variables('virtualNetworkSubnet1Prefix')]" 141 | } 142 | } 143 | ] 144 | } 145 | }, 146 | { 147 | "name": "[variables('clusterName')]", 148 | "type": "Microsoft.HDInsight/clusters", 149 | "location": "[resourceGroup().location]", 150 | "apiVersion": "[variables('clusterApiVersion')]", 151 | "properties": { 152 | "clusterVersion": "[parameters('clusterVersion')]", 153 | "osType": "Linux", 154 | "clusterDefinition": { 155 | "kind": "[variables('clusterType')]", 156 | "componentVersion": { 157 | "Kafka": "[parameters('kafkaVersion')]" 158 | }, 159 | "configurations": { 160 | "gateway": { 161 | "restAuthCredential.isEnabled": true, 162 | "restAuthCredential.username": "[parameters('clusterLoginUserName')]", 163 | "restAuthCredential.password": "[parameters('clusterLoginPassword')]" 164 | } 165 | } 166 | }, 167 | "storageProfile": { 168 | "storageaccounts": [ 169 | { 170 | "name": "[concat(variables('defaultStorageName'),'.blob.core.windows.net')]", 171 | "isDefault": true, 172 | "container": "[variables('clusterName')]", 173 | "key": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('defaultStorageName')), variables('defaultApiVersion')).key1]" 174 | } 175 | ] 176 | }, 177 | "computeProfile": { 178 | "roles": [ 179 | { 180 | "name": "headnode", 181 | "targetInstanceCount": "2", 182 | "hardwareProfile": { 183 | "vmSize": "[parameters('headNodeSize')]" 184 | }, 185 | "osProfile": { 186 | "linuxOperatingSystemProfile": { 187 | "username": "[parameters('sshUserName')]", 188 | "password": "[parameters('sshPassword')]" 189 | } 190 | }, 191 | "virtualNetworkProfile": { 192 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 193 | "subnet": "[concat(resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName')), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 194 | } 195 | }, 196 | { 197 | "name": "workernode", 198 | "targetInstanceCount": "[parameters('clusterSize')]", 199 | "hardwareProfile": { 200 | "vmSize": "[parameters('workerNodeSize')]" 201 | }, 202 | "dataDisksGroups": [ 203 | { 204 | "disksPerNode": "[parameters('disksPerWorkerNode')]" 205 | } 206 | ], 207 | "osProfile": { 208 | "linuxOperatingSystemProfile": { 209 | "username": "[parameters('sshUserName')]", 210 | "password": "[parameters('sshPassword')]" 211 | } 212 | }, 213 | "virtualNetworkProfile": { 214 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 215 | "subnet": "[concat('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 216 | } 217 | }, 218 | { 219 | "name": "zookeepernode", 220 | "targetInstanceCount": 3, 221 | "hardwareProfile": { 222 | "vmSize": "[parameters('zookeeperNodeSize')]" 223 | }, 224 | "osProfile": { 225 | "linuxOperatingSystemProfile": { 226 | "username": "[parameters('sshUserName')]", 227 | "password": "[parameters('sshPassword')]" 228 | } 229 | }, 230 | "virtualNetworkProfile": { 231 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 232 | "subnet": "[concat('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 233 | } 234 | } 235 | ] 236 | } 237 | }, 238 | "dependsOn": [ 239 | "[variables('defaultStorageName')]", 240 | "[variables('virtualNetworkName')]" 241 | ] 242 | }, 243 | { 244 | "name": "[variables('sparkClusterName')]", 245 | "type": "Microsoft.HDInsight/clusters", 246 | "location": "[resourceGroup().location]", 247 | "apiVersion": "[variables('clusterApiVersion')]", 248 | "properties": { 249 | "clusterVersion": "[parameters('clusterVersion')]", 250 | "osType": "Linux", 251 | "clusterDefinition": { 252 | "kind": "[variables('sparkClusterType')]", 253 | "componentVersion": { 254 | "Spark": "[parameters('sparkVersion')]" 255 | }, 256 | "configurations": { 257 | "gateway": { 258 | "restAuthCredential.isEnabled": true, 259 | "restAuthCredential.username": "[parameters('clusterLoginUserName')]", 260 | "restAuthCredential.password": "[parameters('clusterLoginPassword')]" 261 | } 262 | } 263 | }, 264 | "storageProfile": { 265 | "storageaccounts": [ 266 | { 267 | "name": "[concat(variables('defaultStorageName'),'.blob.core.windows.net')]", 268 | "isDefault": true, 269 | "container": "[variables('sparkClusterName')]", 270 | "key": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('defaultStorageName')), variables('defaultApiVersion')).key1]" 271 | } 272 | ] 273 | }, 274 | "computeProfile": { 275 | "roles": [ 276 | { 277 | "name": "headnode", 278 | "targetInstanceCount": "2", 279 | "hardwareProfile": { 280 | "vmSize": "[parameters('headNodeSize')]" 281 | }, 282 | "osProfile": { 283 | "linuxOperatingSystemProfile": { 284 | "username": "[parameters('sshUserName')]", 285 | "password": "[parameters('sshPassword')]" 286 | } 287 | }, 288 | "virtualNetworkProfile": { 289 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 290 | "subnet": "[concat(resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName')), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 291 | } 292 | }, 293 | { 294 | "name": "workernode", 295 | "targetInstanceCount": "[parameters('clusterSize')]", 296 | "hardwareProfile": { 297 | "vmSize": "[parameters('workerNodeSize')]" 298 | }, 299 | "osProfile": { 300 | "linuxOperatingSystemProfile": { 301 | "username": "[parameters('sshUserName')]", 302 | "password": "[parameters('sshPassword')]" 303 | } 304 | }, 305 | "virtualNetworkProfile": { 306 | "id": "[resourceId('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'))]", 307 | "subnet": "[concat('Microsoft.Network/virtualNetworks', variables('virtualNetworkName'), '/subnets/', variables('virtualNetworkDefaultSubnetName'))]" 308 | } 309 | } 310 | ] 311 | } 312 | }, 313 | "dependsOn": [ 314 | "[variables('defaultStorageName')]", 315 | "[variables('virtualNetworkName')]" 316 | ] 317 | } 318 | ], 319 | "outputs": { 320 | "cluster": { 321 | "type": "object", 322 | "value": "[reference(resourceId('Microsoft.HDInsight/clusters', variables('clusterName')))]" 323 | }, 324 | "sparkCluster": { 325 | "type": "object", 326 | "value": "[reference(resourceId('Microsoft.HDInsight/clusters', variables('sparkClusterName')))]" 327 | } 328 | } 329 | } 330 | -------------------------------------------------------------------------------- /src/python/rebalance/test_rebalance_rackaware.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import random 3 | import json 4 | import rebalance_rackaware 5 | 6 | ''' 7 | To run a single unit test: 8 | python rebalance_rackaware_test.py RebalanceTests.test_reassignment_plan_HA_for_topic 9 | ''' 10 | 11 | # Tests 12 | class RebalanceTests(unittest.TestCase): 13 | brokers_info = { 14 | 'wn30-foobar': '1017', 15 | 'wn25-foobar': '1016', 16 | 'wn7-foobar': '1008', 17 | 'wn12-foobar': '1026', 18 | 'wn11-foobar': '1020', 19 | 'wn19-foobar': '1001', 20 | 'wn13-foobar': '1028', 21 | 'wn8-foobar': '1009', 22 | 'wn16-foobar': '1013', 23 | 'wn29-foobar': '1030', 24 | 'wn5-foobar': '1014', 25 | 'wn21-foobar': '1006' 26 | } 27 | 28 | topicInfo_lines = ['Topic:dummyTopic PartitionCount:13 ReplicationFactor:3 Configs:segment.bytes=104857600,cleanup.policy=compact,compression.type=uncompressed', ' Topic: dummyTopic Partition: 0 Leader: 1026 Replicas: 1026,1028,1014 Isr: 1026,1028,1014', ' Topic: dummyTopic Partition: 1 Leader: 1020 Replicas: 1020,1014,1017 Isr: 1020,1014,1017', ' Topic: dummyTopic Partition: 2 Leader: 1028 Replicas: 1028,1017,1013 Isr: 1028,1017,1013', ' Topic: dummyTopic Partition: 3 Leader: 1009 Replicas: 1009,1013,1009 Isr: 1009,1013,1009', ' Topic: dummyTopic Partition: 4 Leader: 1030 Replicas: 1030,1001,1009 Isr: 1030,1001,1009', ' Topic: dummyTopic Partition: 5 Leader: 1008 Replicas: 1008,1020,1016 Isr: 1008,1020,1016', ' Topic: dummyTopic Partition: 6 Leader: 1013 Replicas: 1013,1014,1008 Isr: 1013,1014,1008', ' Topic: dummyTopic Partition: 7 Leader: 1001 Replicas: 1001,1028,1016 Isr: 1001,1028,1016', ' Topic: dummyTopic Partition: 8 Leader: 1014 Replicas: 1014,1017,1008 Isr: 1014,1017,1008', ' Topic: dummyTopic Partition: 9 Leader: 1006 Replicas: 1006,1001,1026 Isr: 1006,1001,1026', ' Topic: dummyTopic Partition: 10 Leader: 1017 Replicas: 1017,1028,1016 Isr: 1017,1028,1016', ' Topic: dummyTopic Partition: 11 Leader: 1016 Replicas: 1016,1026,1017 Isr: 1016,1026,1017', ' Topic: dummyTopic Partition: 12 Leader: 1006 Replicas: 1006,1020,1016 Isr: 1006,1020,1016', ' Topic: dummyTopic Partition: 13 Leader: 1030 Replicas: 1030,1001,1026 Isr: 1030,1001,1026'] 29 | 30 | cluster_topo = ''' 31 | { 32 | "hostGroups": { 33 | "headnode": [ 34 | { 35 | "vmId": 0, 36 | "fqdn": "hn0-foobar", 37 | "state": "Succeeded", 38 | "faultDomain": 0, 39 | "updateDomain": 0, 40 | "availabilitySetId": "/subscriptions/63b34899-ad65-4508-bd67-d8d7efc28194/resourcegroups/rg0-d93ef9c75f8b46f6be278cc0fff8470fresourcegroup/providers/Microsoft.Compute/availabilitySets/headnode-0" 41 | } 42 | ], 43 | "workernode": [ 44 | { 45 | "vmId": 0, 46 | "fqdn": "wn30-foobar", 47 | "state": "Succeeded", 48 | "faultDomain": 0, 49 | "updateDomain": 0, 50 | "availabilitySetId": "/subscriptions/63b34899-ad65-4508-bd67-d8d7efc28194/resourcegroups/rg0-d93ef9c75f8b46f6be278cc0fff8470fresourcegroup/providers/Microsoft.Compute/availabilitySets/workernode-0" 51 | }, 52 | { 53 | "vmId": 1, 54 | "fqdn": "wn25-foobar", 55 | "state": "Succeeded", 56 | "faultDomain": 0, 57 | "updateDomain": 1, 58 | "availabilitySetId": "/subscriptions/63b34899-ad65-4508-bd67-d8d7efc28194/resourcegroups/rg0-d93ef9c75f8b46f6be278cc0fff8470fresourcegroup/providers/Microsoft.Compute/availabilitySets/workernode-0" 59 | }, 60 | { 61 | "vmId": 10, 62 | "fqdn": "wn7-foobar", 63 | "state": "Succeeded", 64 | "faultDomain": 0, 65 | "updateDomain": 2, 66 | "availabilitySetId": "/subscriptions/63b34899-ad65-4508-bd67-d8d7efc28194/resourcegroups/rg0-d93ef9c75f8b46f6be278cc0fff8470fresourcegroup/providers/Microsoft.Compute/availabilitySets/workernode-0" 67 | }, 68 | { 69 | "vmId": 11, 70 | "fqdn": "wn12-foobar", 71 | "state": "Succeeded", 72 | "faultDomain": 1, 73 | "updateDomain": 0, 74 | "availabilitySetId": "/subscriptions/63b34899-ad65-4508-bd67-d8d7efc28194/resourcegroups/rg0-d93ef9c75f8b46f6be278cc0fff8470fresourcegroup/providers/Microsoft.Compute/availabilitySets/workernode-0" 75 | }, 76 | { 77 | "vmId": 12, 78 | "fqdn": "wn11-foobar", 79 | "state": "Succeeded", 80 | "faultDomain": 1, 81 | "updateDomain": 1, 82 | "availabilitySetId": "/subscriptions/63b34899-ad65-4508-bd67-d8d7efc28194/resourcegroups/rg0-d93ef9c75f8b46f6be278cc0fff8470fresourcegroup/providers/Microsoft.Compute/availabilitySets/workernode-0" 83 | }, 84 | { 85 | "vmId": 2, 86 | "fqdn": "wn19-foobar", 87 | "state": "Succeeded", 88 | "faultDomain": 1, 89 | "updateDomain": 2, 90 | "availabilitySetId": "/subscriptions/63b34899-ad65-4508-bd67-d8d7efc28194/resourcegroups/rg0-d93ef9c75f8b46f6be278cc0fff8470fresourcegroup/providers/Microsoft.Compute/availabilitySets/workernode-0" 91 | }, 92 | { 93 | "vmId": 3, 94 | "fqdn": "wn13-foobar", 95 | "state": "Succeeded", 96 | "faultDomain": 2, 97 | "updateDomain": 0, 98 | "availabilitySetId": "/subscriptions/63b34899-ad65-4508-bd67-d8d7efc28194/resourcegroups/rg0-d93ef9c75f8b46f6be278cc0fff8470fresourcegroup/providers/Microsoft.Compute/availabilitySets/workernode-0" 99 | }, 100 | { 101 | "vmId": 4, 102 | "fqdn": "wn8-foobar", 103 | "state": "Succeeded", 104 | "faultDomain": 2, 105 | "updateDomain": 1, 106 | "availabilitySetId": "/subscriptions/63b34899-ad65-4508-bd67-d8d7efc28194/resourcegroups/rg0-d93ef9c75f8b46f6be278cc0fff8470fresourcegroup/providers/Microsoft.Compute/availabilitySets/workernode-0" 107 | }, 108 | { 109 | "vmId": 5, 110 | "fqdn": "wn16-foobar", 111 | "state": "Succeeded", 112 | "faultDomain": 2, 113 | "updateDomain": 2, 114 | "availabilitySetId": "/subscriptions/63b34899-ad65-4508-bd67-d8d7efc28194/resourcegroups/rg0-d93ef9c75f8b46f6be278cc0fff8470fresourcegroup/providers/Microsoft.Compute/availabilitySets/workernode-0" 115 | }, 116 | { 117 | "vmId": 6, 118 | "fqdn": "wn29-foobar", 119 | "state": "Succeeded", 120 | "faultDomain": 0, 121 | "updateDomain": 1, 122 | "availabilitySetId": "/subscriptions/63b34899-ad65-4508-bd67-d8d7efc28194/resourcegroups/rg0-d93ef9c75f8b46f6be278cc0fff8470fresourcegroup/providers/Microsoft.Compute/availabilitySets/workernode-0" 123 | }, 124 | { 125 | "vmId": 7, 126 | "fqdn": "wn5-foobar", 127 | "state": "Succeeded", 128 | "faultDomain": 0, 129 | "updateDomain": 0, 130 | "availabilitySetId": "/subscriptions/63b34899-ad65-4508-bd67-d8d7efc28194/resourcegroups/rg0-d93ef9c75f8b46f6be278cc0fff8470fresourcegroup/providers/Microsoft.Compute/availabilitySets/workernode-0" 131 | }, 132 | { 133 | "vmId": 8, 134 | "fqdn": "wn21-foobar", 135 | "state": "Succeeded", 136 | "faultDomain": 1, 137 | "updateDomain": 2, 138 | "availabilitySetId": "/subscriptions/63b34899-ad65-4508-bd67-d8d7efc28194/resourcegroups/rg0-d93ef9c75f8b46f6be278cc0fff8470fresourcegroup/providers/Microsoft.Compute/availabilitySets/workernode-0" 139 | } 140 | ], 141 | "zookeepernode": [ 142 | { 143 | "vmId": 0, 144 | "fqdn": "zk0-foobar", 145 | "state": "Succeeded", 146 | "faultDomain": 0, 147 | "updateDomain": 0, 148 | "availabilitySetId": "/subscriptions/63b34899-ad65-4508-bd67-d8d7efc28194/resourcegroups/rg0-d93ef9c75f8b46f6be278cc0fff8470fresourcegroup/providers/Microsoft.Compute/availabilitySets/zookeepernode-0" 149 | } 150 | ] 151 | } 152 | } 153 | ''' 154 | 155 | compute_storage_cost = False 156 | dummy_topic = "dummyTopic" 157 | 158 | def test_parse_topo_info(self): 159 | host_info = rebalance_rackaware.parse_topo_info(self.cluster_topo, self.brokers_info) 160 | broker_1020 = [element for element in host_info if int(element[rebalance_rackaware.BROKER_ID]) == 1020][0] 161 | self.assertEqual(broker_1020[rebalance_rackaware.VM_ID], 12) 162 | self.assertEqual(broker_1020[rebalance_rackaware.FQDN], 'wn11-foobar') 163 | self.assertEqual(broker_1020[rebalance_rackaware.RACK], 'FD1UD1') 164 | 165 | def test_get_partition_info(self): 166 | partitions_info = rebalance_rackaware.get_partition_info(self.topicInfo_lines, [], self.dummy_topic) 167 | partition_4 = [element for element in partitions_info if int(element[rebalance_rackaware.PARTITION]) == 4][0] 168 | self.assertEqual(partition_4[rebalance_rackaware.PARTITION], 4) 169 | self.assertEqual(partition_4[rebalance_rackaware.LEADER], 1030) 170 | self.assertEqual(set(partition_4[rebalance_rackaware.REPLICAS]), set([1030, 1001, 1009])) 171 | self.assertEqual(set(partition_4[rebalance_rackaware.ISR]), set([1030, 1001, 1009])) 172 | 173 | def test_generate_fd_list_ud_list(self): 174 | host_info = rebalance_rackaware.parse_topo_info(self.cluster_topo, self.brokers_info) 175 | fd_list, ud_list = rebalance_rackaware.generate_fd_list_ud_list(host_info) 176 | self.assertEqual(set(fd_list), set(['0','1','2'])) 177 | self.assertEqual(set(ud_list), set(['0','1','2'])) 178 | 179 | def test_generate_fd_ud_list(self): 180 | host_info = [{'vmId': 37, 'fqdn': 'wn25-foobar', 'updateDomain': '1', 'brokerId': '1016', 'faultDomain': '1', 'rack': 'FD1UD1'},{'vmId': 1, 'fqdn': 'wn25-foobar', 'updateDomain': '1', 'brokerId': '1016', 'faultDomain': '1', 'rack': 'FD1UD1'},{'vmId': 10, 'fqdn': 'wn7-foobar', 'updateDomain': '1', 'brokerId': '1008', 'faultDomain': '2', 'rack': 'FD2UD1'}] 181 | rgen = rebalance_rackaware.ReassignmentGenerator(host_info, self.dummy_topic, "partitionInfo", self.compute_storage_cost) 182 | fd_ud_list = rgen._generate_fd_ud_list() 183 | self.assertEqual(fd_ud_list, ['FD1UD1', 'FD2UD1']) 184 | 185 | # 3 FDs, 5 UDs 186 | def test_generate_alternated_fd_ud_list_test_case_3FDs_5UDs(self): 187 | (FDs, UDs) = 3,5 188 | fd_ud_list, fd_list, ud_list = self.generate_fd_ud_list(FDs,UDs) 189 | rgen = rebalance_rackaware.ReassignmentGenerator("hostInfo", self.dummy_topic, "partitionInfo", self.compute_storage_cost) 190 | alternated_list = rgen._generate_alternated_fd_ud_list(fd_ud_list, fd_list, ud_list) 191 | expected_list = ['FD0UD0', 'FD1UD1', 'FD2UD2', 'FD0UD3', 'FD1UD4', 'FD2UD0', 'FD0UD1', 'FD1UD2', 'FD2UD3', 'FD0UD4', 'FD1UD0', 'FD2UD1', 'FD0UD2', 'FD1UD3', 'FD2UD4'] 192 | self.assertEqual(len(alternated_list), len(fd_ud_list)) 193 | self.assertEqual(alternated_list, expected_list) 194 | 195 | #3 FDs, 2 UDs 196 | def test_generate_alternated_fd_ud_list_test_case_3FDs_2UDs(self): 197 | fd_ud_list = ['FD0UD0', 'FD0UD1', 'FD1UD0', 'FD1UD1', 'FD2UD0', 'FD2UD1'] 198 | fd_list = ['0', '1', '2'] 199 | ud_list = ['0', '1'] 200 | rgen = rebalance_rackaware.ReassignmentGenerator("hostInfo", self.dummy_topic, "partitionInfo", self.compute_storage_cost) 201 | alternated_list = rgen._generate_alternated_fd_ud_list(fd_ud_list, fd_list, ud_list) 202 | expected_list = ['FD0UD0', 'FD1UD1', 'FD2UD0', 'FD0UD1', 'FD1UD0', 'FD2UD1'] 203 | self.assertEqual(len(alternated_list), len(fd_ud_list)) 204 | self.assertEqual(alternated_list, expected_list) 205 | 206 | #3 FDs, 2 UDs with gaps 207 | def test_generate_alternated_fd_ud_list_test_case_3FDs_2UDs_with_gaps(self): 208 | fd_ud_list = ['FD0UD0', 'FD1UD1', 'FD2UD0', 'FD0UD1'] 209 | fd_list = ['0', '1', '2'] 210 | ud_list = ['0', '1'] 211 | rgen = rebalance_rackaware.ReassignmentGenerator("hostInfo", self.dummy_topic, "partitionInfo", self.compute_storage_cost) 212 | alternated_list = rgen._generate_alternated_fd_ud_list(fd_ud_list, fd_list, ud_list) 213 | expected_list = ['FD0UD0', 'FD1UD1', 'FD2UD0', 'FD0UD1'] 214 | self.assertEqual(len(alternated_list), len(fd_ud_list)) 215 | self.assertEqual(alternated_list, expected_list) 216 | 217 | #3 FDs, 3UDs 218 | def test_generate_alternated_fd_ud_list_test_case_3FDs_3UDs(self): 219 | (FDs, UDs) = 3,3 220 | fd_ud_list, fd_list, ud_list = self.generate_fd_ud_list(FDs,UDs) 221 | rgen = rebalance_rackaware.ReassignmentGenerator("hostInfo", self.dummy_topic, "partitionInfo", self.compute_storage_cost) 222 | alternated_list = rgen._generate_alternated_fd_ud_list(fd_ud_list, fd_list, ud_list) 223 | expected_list = ['FD0UD0', 'FD1UD1', 'FD2UD2', 'FD0UD1', 'FD1UD2', 'FD2UD0', 'FD0UD2', 'FD1UD0', 'FD2UD1'] 224 | self.assertEqual(len(alternated_list), len(fd_ud_list)) 225 | self.assertEqual(alternated_list, expected_list) 226 | 227 | #3 FDs, 3UDs with Gaps 228 | def test_generate_alternated_fd_ud_list_test_case_3FDs_3UDs_with_gaps(self): 229 | fd_ud_list = ['FD0UD1', 'FD0UD2', 'FD1UD0', 'FD1UD1', 'FD2UD0', 'FD2UD2'] 230 | fd_list = ['0', '1', '2'] 231 | ud_list = ['0', '1', '2'] 232 | rgen = rebalance_rackaware.ReassignmentGenerator("hostInfo", self.dummy_topic, "partitionInfo", self.compute_storage_cost) 233 | alternated_list = rgen._generate_alternated_fd_ud_list(fd_ud_list, fd_list, ud_list) 234 | expected_list = ['FD1UD1', 'FD2UD2', 'FD0UD1', 'FD2UD0', 'FD0UD2', 'FD1UD0'] 235 | self.assertEqual(len(alternated_list), len(fd_ud_list)) 236 | self.assertEqual(alternated_list, expected_list) 237 | 238 | #3 FDs, 6UDs (n, nm) 239 | def test_generate_alternated_fd_ud_list_test_case_3FDs_6UDs(self): 240 | (FDs, UDs) = 3,6 241 | fd_ud_list, fd_list, ud_list = self.generate_fd_ud_list(FDs,UDs) 242 | rgen = rebalance_rackaware.ReassignmentGenerator("hostInfo", self.dummy_topic, "partitionInfo", self.compute_storage_cost) 243 | alternated_list = rgen._generate_alternated_fd_ud_list(fd_ud_list, fd_list, ud_list) 244 | expected_list = ['FD0UD0', 'FD1UD1', 'FD2UD2', 'FD0UD1', 'FD1UD2', 'FD2UD3', 'FD0UD2', 'FD1UD3', 'FD2UD4', 'FD0UD3', 'FD1UD4', 'FD2UD5', 'FD0UD4', 'FD1UD5', 'FD2UD0', 'FD0UD5', 'FD1UD0', 'FD2UD1'] 245 | self.assertEqual(len(alternated_list), len(fd_ud_list)) 246 | self.assertEqual(alternated_list, expected_list) 247 | 248 | #4 FDs, 6UDs (n, nm) 249 | def test_generate_alternated_fd_ud_list_test_case_4FDs_6UDs(self): 250 | (FDs, UDs) = 4,6 251 | fd_ud_list, fd_list, ud_list = self.generate_fd_ud_list(FDs,UDs) 252 | rgen = rebalance_rackaware.ReassignmentGenerator("hostInfo", self.dummy_topic, "partitionInfo", self.compute_storage_cost) 253 | alternated_list = rgen._generate_alternated_fd_ud_list(fd_ud_list, fd_list, ud_list) 254 | expected_list = ['FD0UD0', 'FD1UD1', 'FD2UD2', 'FD3UD3', 'FD0UD1', 'FD1UD2', 'FD2UD3', 'FD3UD4', 'FD0UD2', 'FD1UD3', 'FD2UD4', 'FD3UD5', 'FD0UD3', 'FD1UD4', 'FD2UD5', 'FD3UD0', 'FD0UD4', 'FD1UD5', 'FD2UD0', 'FD3UD1', 'FD0UD5', 'FD1UD0', 'FD2UD1', 'FD3UD2'] 255 | self.assertEqual(len(alternated_list), len(fd_ud_list)) 256 | self.assertEqual(alternated_list, expected_list) 257 | 258 | def test_reassignment_plan_HA_for_topic(self): 259 | host_info = rebalance_rackaware.parse_topo_info(self.cluster_topo, self.brokers_info) 260 | brokers_replica_count = [] 261 | for host in host_info: 262 | b = { 263 | rebalance_rackaware.BROKER_ID : host[rebalance_rackaware.BROKER_ID], 264 | rebalance_rackaware.LEADERS : 0, 265 | rebalance_rackaware.FOLLOWERS: 0, 266 | } 267 | brokers_replica_count.append(b) 268 | partitions_info = rebalance_rackaware.get_partition_info(self.topicInfo_lines, [], self.dummy_topic) 269 | rgen = rebalance_rackaware.ReassignmentGenerator(host_info, self.dummy_topic, partitions_info, self.compute_storage_cost) 270 | fd_ud_list, fd_list, ud_list = self.generate_fd_ud_list(3,3) 271 | alternated_list = rgen._generate_alternated_fd_ud_list(fd_ud_list, fd_list, ud_list) 272 | reassignment_plan, balanced_partitions = rgen._generate_reassignment_plan_for_topic(3,0,alternated_list,3,3,brokers_replica_count, None) 273 | topic_balanced = rgen._verify_reassignment_plan(reassignment_plan,self.dummy_topic,3,3,3) 274 | self.assertTrue(topic_balanced) 275 | 276 | def test_check_if_partition_balanced(self): 277 | host_info = [{'vmId': 1, 'fqdn': 'wn01-foobar', 'updateDomain': '0', 'brokerId': '1016', 'faultDomain': '1', 'rack': 'FD1UD0'}, 278 | {'vmId': 2, 'fqdn': 'wn02-foobar', 'updateDomain': '2', 'brokerId': '1017', 'faultDomain': '0', 'rack': 'FD0UD2'}, 279 | {'vmId': 3, 'fqdn': 'wn77-foobar', 'updateDomain': '1', 'brokerId': '1008', 'faultDomain': '2', 'rack': 'FD2UD1'}, 280 | {'vmId': 4, 'fqdn': 'wn47-foobar', 'updateDomain': '2', 'brokerId': '1098', 'faultDomain': '2', 'rack': 'FD2UD2'}, 281 | {'vmId': 5, 'fqdn': 'wn56-foobar', 'updateDomain': '1', 'brokerId': '2', 'faultDomain': '2', 'rack': 'FD2UD1'}] 282 | rgen = rebalance_rackaware.ReassignmentGenerator(host_info, self.dummy_topic, "partitions", self.compute_storage_cost) 283 | 284 | partition_1 = {"partition": 1,"leader": 1016,"replicas": [1016, 1017, 1008],"isr": [1016, 1017, 1008]} 285 | partition_2 = {"partition": 1,"leader": 2,"replicas": [1016, 2, 1098],"isr": [1016]} 286 | partition_3 = {"partition": 1,"leader": 2,"replicas": [1016, 2],"isr": [1016]} 287 | partition_4 = {"partition": 1,"leader": 2,"replicas": [1016, 1017, 1008, 1098],"isr": [1016]} 288 | partition_5 = {"partition": 1,"leader": 2,"replicas": [1008, 1098],"isr": [1008]} 289 | partition_6 = {"partition": 1,"leader": 2,"replicas": [1016, 1017],"isr": [1008]} 290 | 291 | #Balanced Checks 292 | self.assertTrue(rgen._check_if_partition_balanced(partition_1,3,3,3,[],[])) 293 | self.assertTrue(rgen._check_if_partition_balanced(partition_3,2,3,3,[],[])) 294 | self.assertTrue(rgen._check_if_partition_balanced(partition_4,4,3,3,[],[])) 295 | 296 | #Imbalanced Checks 297 | self.assertFalse(rgen._check_if_partition_balanced(partition_2,3,3,3,[],[])) 298 | self.assertFalse(rgen._check_if_partition_balanced(partition_5,2,3,3,[],[])) 299 | self.assertFalse(rgen._check_if_partition_balanced(partition_6,3,3,3,[],[])) 300 | 301 | def get_broker_info(self, b_id, host_info): 302 | return [element for element in host_info if int(element[rebalance_rackaware.BROKER_ID]) == b_id][0] 303 | 304 | def generate_fd_ud_list(self, m, n): 305 | fd_ud_list, fd_list, ud_list = [], [], set() 306 | for i in range(0, m): 307 | fd_list.append(str(i)) 308 | for j in range(0, n): 309 | ud_list.add(str(j)) 310 | fd_ud_list.append('FD' + str(i) + 'UD' + str(j)) 311 | 312 | self.assertEqual(len(fd_ud_list), m*n) 313 | self.assertEqual(len(fd_list), m) 314 | self.assertEqual(len(ud_list), n) 315 | return fd_ud_list, fd_list, sorted(ud_list) 316 | 317 | def main(): 318 | unittest.main() 319 | 320 | if __name__ == '__main__': 321 | main() -------------------------------------------------------------------------------- /src/python/rebalance/rebalance_rackaware.py: -------------------------------------------------------------------------------- 1 | """ 2 | Rebalance Kafka partition replicas to achieve HA (Fault Domain/Update Domain awareness). The tool distributes 3 | replicas of partitions of a topic across brokers in a manner such that each replica is in a separate fault domain and 4 | update domain. The tool also distributes the leaders such that each broker has approximately the same number of 5 | leaders for partitions. Rebalance can be executed for one or more topics. 6 | PRE-REQS: 7 | ========= 8 | sudo apt-get install -y libffi-dev libssl-dev 9 | sudo pip install --upgrade requests[security] PyOpenSSL ndg-httpsclient pyasn1 kazoo retry pexpect 10 | RUNNING THE SCRIPT: 11 | =================== 12 | 1) Copy the script to /usr/hdp/current/kafka-broker/bin on your cluster 13 | 2) Run this script with sudo privilege due to permission issues on some python packages: sudo python rebalance_new.py 14 | """ 15 | 16 | import argparse 17 | import errno 18 | import json 19 | import logging 20 | import os 21 | import os.path 22 | import pexpect 23 | import random 24 | import re 25 | import requests 26 | import socket 27 | import subprocess 28 | import sys 29 | import tempfile 30 | import traceback 31 | from logging import StreamHandler 32 | from logging.handlers import RotatingFileHandler 33 | from logging.handlers import SysLogHandler 34 | from operator import itemgetter 35 | 36 | from hdinsight_common.AmbariHelper import AmbariHelper 37 | from kazoo.client import KazooClient 38 | from kazoo.client import KazooState 39 | from retry import retry 40 | 41 | # LOGGING 42 | 43 | logger = logging.getLogger(__name__) 44 | log_file = "rebalance_log" 45 | log_dir = "/var/log/kafka/" 46 | SIMPLE_FORMATTER = logging.Formatter('%(asctime)s - %(filename)s [%(process)d] %(name)s - %(levelname)s - %(message)s') 47 | SYSLOG_FORMAT_STRING = ' %(filename)s [%(process)d] - %(name)s - %(levelname)s - %(message)s' 48 | SYSLOG_FORMATTER = logging.Formatter(SYSLOG_FORMAT_STRING) 49 | MIN_LOG_LEVEL = logging.DEBUG 50 | MAX_RETRIES = 3 51 | RETRY_INTERVAL_DELAY = 1 52 | RETRY_INTERVAL_BACKOFF = 2 53 | 54 | '''Filters (lets through) all messages with level < LEVEL''' 55 | 56 | 57 | class LogFilter(logging.Filter): 58 | def __init__(self, level): 59 | self.level = level 60 | 61 | def filter(self, log_record): 62 | return log_record.levelno < self.level 63 | 64 | 65 | # LOG_LOCAL0 - belongs to hdinsight-agent 66 | # LOG_LOCAL1 - belongs to ambari-agent 67 | # LOG_LOCAL2 - belongs to syslog catch all 68 | # We want to log to LOG_LOCAL2 69 | def initialize_logger(logger, log_file, syslog_facility=SysLogHandler.LOG_LOCAL2): 70 | logger.setLevel(MIN_LOG_LEVEL) 71 | if not len(logger.handlers): 72 | add_console_handler(logger) 73 | add_file_handler(logger, log_file) 74 | add_syslog_handler(logger, syslog_facility) 75 | 76 | 77 | '''Given a logger, we attach a console handler that will log only error messages''' 78 | 79 | 80 | def add_console_handler(logger): 81 | stdout_handler = StreamHandler(sys.stdout) 82 | stderr_handler = StreamHandler(sys.stderr) 83 | log_filter = LogFilter(logging.WARNING) 84 | 85 | stdout_handler.addFilter(log_filter) 86 | stdout_handler.setLevel(logging.INFO) 87 | stdout_handler.setFormatter(SIMPLE_FORMATTER) 88 | 89 | stderr_handler.setLevel(max(MIN_LOG_LEVEL, logging.WARNING)) 90 | stderr_handler.setFormatter(SIMPLE_FORMATTER) 91 | 92 | logger.addHandler(stdout_handler) 93 | logger.addHandler(stderr_handler) 94 | 95 | 96 | '''Given a logger, we attach a rotating file handler that will log to the specified output file''' 97 | 98 | 99 | def add_file_handler(logger, log_file_name): 100 | if not log_file_name.endswith('.log'): 101 | log_file_name = log_file_name + '.log' 102 | 103 | try: 104 | os.makedirs(log_dir) 105 | except OSError as exc: 106 | if exc.errno == errno.EEXIST and os.path.isdir(log_dir): 107 | pass 108 | else: 109 | raise_error('Unable to create log dir: {0}'.format(log_dir)) 110 | 111 | log_file_path = os.path.join(log_dir, log_file_name) 112 | file_handler = RotatingFileHandler(filename=log_file_path, maxBytes=104857600, backupCount=100) 113 | file_handler.setLevel(logging.DEBUG) 114 | file_handler.setFormatter(SIMPLE_FORMATTER) 115 | 116 | logger.addHandler(file_handler) 117 | 118 | 119 | # add syslog handler if we are on linux. We need to add retry on this because /dev/log might not be created by 120 | # rsyslog yet 121 | @retry(exceptions=BaseException, tries=MAX_RETRIES, delay=RETRY_INTERVAL_DELAY, backoff=RETRY_INTERVAL_BACKOFF) 122 | def add_syslog_handler(logger, syslog_facility): 123 | try: 124 | mds_syslog_handler = SysLogHandler(address="/dev/log", 125 | facility=syslog_facility) 126 | mds_syslog_handler.setLevel(MIN_LOG_LEVEL) 127 | mds_syslog_handler.setFormatter(SYSLOG_FORMATTER) 128 | logger.addHandler(mds_syslog_handler) 129 | return True 130 | except Exception: 131 | raise_error('Exception occurred when adding syslog handler: ' + traceback.format_exc()) 132 | return False 133 | 134 | 135 | # Constants 136 | KAFKA_BROKER = "kafka-broker" 137 | ASSIGNMENT_JSON_FILE = "kafkaRebalancePlan.json" 138 | ZOOKEEPER_PORT = ":2181" 139 | BROKER_PORT = ":9092" 140 | ZOOKEEPER_HOSTS = None 141 | BROKERS_ID_PATH = "brokers/ids" 142 | KAFKA_HOME = "/usr/hdp/current/kafka-broker/" 143 | KAFKA_LIBS_PATH = KAFKA_HOME + "libs/" 144 | KAFKA_TOPICS_TOOL_PATH = KAFKA_HOME + "bin/kafka-topics.sh" 145 | KAFKA_REASSIGN_PARTITIONS_TOOL_PATH = KAFKA_HOME + "bin/kafka-reassign-partitions.sh" 146 | FQDN = "fqdn" 147 | BROKER_ID = "brokerId" 148 | FAULT_DOMAIN = "faultDomain" 149 | UPDATE_DOMAIN = "updateDomain" 150 | FAULT_DOMAIN_SHORT = "FD" 151 | UPDATE_DOMAIN_SHORT = "UD" 152 | RACK = "rack" 153 | VM_ID = "vmId" 154 | PARTITION = "partition" 155 | REPLICAS = "replicas" 156 | LEADER = "leader" 157 | LEADERS = "leaders" 158 | FOLLOWERS = "followers" 159 | TOPICS = "topics" 160 | ISR = "isr" 161 | FREE_DISK_SPACE = "fds" 162 | PARTITION_SIZE = "partition_size" 163 | ASSIGNED = "assigned" 164 | DEFAULT_REBALANCE_THROTTLE_RATE_BPS = "50000000" 165 | ALL_TOPICS_STRING = "all" 166 | 167 | ''' 168 | Get information of Zookeeper Hosts 169 | ''' 170 | 171 | 172 | def get_zookeeper_connect_string(): 173 | ah = AmbariHelper() 174 | hosts = ah.get_host_components() 175 | zkHosts = "" 176 | for item in hosts["items"]: 177 | if item["HostRoles"]["component_name"] == "ZOOKEEPER_SERVER": 178 | zkHosts += item["HostRoles"]["host_name"] 179 | zkHosts += ZOOKEEPER_PORT 180 | zkHosts += "," 181 | if len(zkHosts) > 2: 182 | return zkHosts[:-1] 183 | else: 184 | raise_error("Failed to get Zookeeper information from Ambari!") 185 | 186 | 187 | ''' 188 | Get information of Broker Hosts 189 | ''' 190 | 191 | 192 | def get_broker_connect_string(): 193 | ah = AmbariHelper() 194 | hosts = ah.get_host_components() 195 | brokerHosts = "" 196 | for item in hosts["items"]: 197 | if item["HostRoles"]["component_name"] == "KAFKA_BROKER": 198 | brokerHosts += item["HostRoles"]["host_name"] 199 | brokerHosts += BROKER_PORT 200 | brokerHosts += "," 201 | if len(brokerHosts) > 2: 202 | return brokerHosts[:-1] 203 | else: 204 | raise_error("Failed to get Broker information from Ambari!") 205 | 206 | 207 | ''' 208 | Returns a list of all topics in Kafka by executing the Kafka-topics tool 209 | ''' 210 | 211 | 212 | def get_topic_list(): 213 | try: 214 | kafka_version, hdp_version = get_kafka_hdp_version() 215 | if kafka_version >= '3.2.0': 216 | s = subprocess.check_output([ 217 | KAFKA_TOPICS_TOOL_PATH, 218 | "--bootstrap-server", 219 | get_broker_connect_string(), 220 | "--list" 221 | ]) 222 | else: 223 | s = subprocess.check_output([ 224 | KAFKA_TOPICS_TOOL_PATH, 225 | "--zookeeper", 226 | get_zookeeper_connect_string(), 227 | "--list" 228 | ]) 229 | 230 | if len(s) > 0: 231 | logger.info("List of all topics: %s", s.split()) 232 | return s.split() 233 | else: 234 | return [] 235 | except Exception: 236 | raise_error('Exception occurred when calling Kafka topics tool: ' + traceback.format_exc()) 237 | sys.exit() 238 | 239 | 240 | ''' 241 | Uses AmbariHelper from hdinsight-common to get the cluster manifest and parses it to get the cluster topology JSON object. 242 | ''' 243 | 244 | 245 | def get_cluster_topology_json(): 246 | ambariHelper = AmbariHelper() 247 | cluster_manifest = ambariHelper.get_cluster_manifest() 248 | settings = cluster_manifest.settings 249 | if "cluster_topology_json_url" in settings: 250 | json_url = settings["cluster_topology_json_url"] 251 | logger.info("Retrieved Cluster Topology JSON document.") 252 | logger.info("URL: %s", json_url) 253 | r = requests.get(json_url) 254 | topology_info = r.text 255 | logger.debug("Cluster Topology: %s", topology_info) 256 | 257 | return topology_info 258 | else: 259 | raise_error("Failed to get cluster_topology_json_url from cluster manifest.") 260 | 261 | 262 | ''' 263 | Parses the cluster topology JSON doc and returns Host information 264 | Returns: A list of dictionaries. 265 | host_info = [ 266 | { 267 | "vmId": 1 268 | "FD": 0 269 | "UD": 0 270 | "fqdn": 'wn0-k09v3' 271 | "brokerId": 1024, 272 | "rack": FD0UD0, 273 | "free_disk_space": 0 274 | }, 275 | ] 276 | ''' 277 | 278 | 279 | def parse_topo_info(cluster_topology_json, brokers_info, dead_hosts=None): 280 | logger.info("Parsing topology info to retrieve information about hosts.") 281 | workernode_info = json.loads(cluster_topology_json)["hostGroups"]["workernode"] 282 | host_info = [] 283 | for node in workernode_info: 284 | host = { 285 | VM_ID: node[VM_ID], 286 | FAULT_DOMAIN: str(node[FAULT_DOMAIN]), 287 | UPDATE_DOMAIN: str(node[UPDATE_DOMAIN]), 288 | FQDN: node[FQDN], 289 | BROKER_ID: brokers_info[node[FQDN]] if node[FQDN] in brokers_info else None, 290 | RACK: FAULT_DOMAIN_SHORT + str(node[FAULT_DOMAIN]) + UPDATE_DOMAIN_SHORT + str(node[UPDATE_DOMAIN]), 291 | FREE_DISK_SPACE: 0 292 | } 293 | if dead_hosts: 294 | if not node[FQDN] in dead_hosts: 295 | host_info.append(host) 296 | else: 297 | host_info.append(host) 298 | return host_info 299 | 300 | 301 | ''' 302 | Call the Kafka topic tool to get partition info about a topic. The return format is: 303 | [ 304 | 'Topic:dummyTopic PartitionCount:13 ReplicationFactor:3 Configs:segment.bytes=104857600,cleanup.policy=compact,compression.type=uncompressed', 305 | ' Topic: dummyTopic Partition: 0 Leader: 1026 Replicas: 1026,1028,1014 Isr: 1026,1028,1014', 306 | ' Topic: dummyTopic Partition: 1 Leader: 1020 Replicas: 1020,1014,1017 Isr: 1020,1014,1017', 307 | .. 308 | ] 309 | ''' 310 | 311 | 312 | def get_topic_info(topic): 313 | kafka_version, hdp_version = get_kafka_hdp_version() 314 | if kafka_version >= '3.2.0': 315 | topicInfo = subprocess.check_output([ 316 | KAFKA_TOPICS_TOOL_PATH, 317 | "--bootstrap-server", 318 | get_broker_connect_string(), 319 | "--describe", 320 | "--topic", 321 | topic 322 | ]) 323 | else: 324 | topicInfo = subprocess.check_output([ 325 | KAFKA_TOPICS_TOOL_PATH, 326 | "--zookeeper", 327 | get_zookeeper_connect_string(), 328 | "--describe", 329 | "--topic", 330 | topic 331 | ]) 332 | if topicInfo is None or len(topicInfo) == 0: 333 | raise_error("Failed to get Kafka partition info for topic " + topic) 334 | return topicInfo 335 | 336 | 337 | def get_replica_count_topic(topic): 338 | topicInfo = get_topic_info(topic) 339 | topicInfo_lines = topicInfo.split('\n') 340 | if len(topicInfo_lines) < 2: 341 | raise_error("Failed to parse Kafka topic info") 342 | 343 | summaryWithSpaceFixed = topicInfo_lines[0].replace(": ", ":") 344 | summary = summaryWithSpaceFixed.split() 345 | replica_count = int(summary[2].split(":")[1]) 346 | return replica_count 347 | 348 | 349 | ''' 350 | Parses through the output of the Kafka Topic tools and returns info about partitions for a given topic. 351 | Return format: partitions_info = [ 352 | { 353 | "partition": partition, 354 | "leader": leader, 355 | "replicas": replicas, 356 | "isr": isr 357 | }, 358 | ... 359 | ] 360 | ''' 361 | 362 | 363 | def get_partition_info(topic, topicInfo_lines, partition_sizes): 364 | logger.info("Retrieving partition information for topic: %s", topic) 365 | partitions_info = [] 366 | for i in range(1, len(topicInfo_lines)): 367 | if len(topicInfo_lines[i].strip()) == 0: 368 | continue 369 | partition = int(topicInfo_lines[i].split('Partition: ')[1].split()[0]) 370 | leader = int(topicInfo_lines[i].split('Leader: ')[1].split()[0]) 371 | replicas = map(int, topicInfo_lines[i].split('Replicas: ')[1].split()[0].split(',')) 372 | isr = map(int, topicInfo_lines[i].split('Isr: ')[1].split()[0].split(',')) 373 | partition_name = str(topic + "-" + str(partition)) 374 | partition_size = 0 375 | if partition_sizes: 376 | if partition_name in partition_sizes: 377 | partition_size = int(partition_sizes[partition_name]) 378 | partition_info = { 379 | PARTITION: partition, 380 | LEADER: leader, 381 | REPLICAS: replicas, 382 | ISR: isr, 383 | PARTITION_SIZE: partition_size, 384 | ASSIGNED: None 385 | } 386 | partitions_info.append(partition_info) 387 | 388 | # Return the list sorted by increasing partition size so that we rebalance the smaller partitions first 389 | return sorted(partitions_info, key=itemgetter(PARTITION_SIZE)) 390 | 391 | 392 | # Connect to Zookeeper 393 | @retry(exceptions=BaseException, tries=MAX_RETRIES, delay=RETRY_INTERVAL_DELAY, backoff=RETRY_INTERVAL_BACKOFF, 394 | logger=logger) 395 | def connect(zk_quorum): 396 | logger.info('Connecting to zookeeper quorum at: {0}'.format(zk_quorum)) 397 | zk = KazooClient(hosts=zk_quorum) 398 | zk.start() 399 | zk.add_listener(connection_lost) 400 | return zk 401 | 402 | 403 | def connection_lost(state): 404 | if state == KazooState.LOST or state == KazooState.SUSPENDED: 405 | raise RuntimeError("Fatal error lost connection to zookeeper.") 406 | 407 | 408 | ''' 409 | Get broker ID to Host mapping from zookeeper. 410 | Returns a dictionary: 411 | brokers_info = { 412 | 'wn30-foobar': '1017', 413 | 'wn25-foobar': '1016', 414 | 'wn7-foobar': '1008', 415 | .. 416 | } 417 | ''' 418 | 419 | 420 | def get_brokerhost_info(zookeeper_client): 421 | logger.info("Associating brokers to hosts...") 422 | zk_brokers_ids = zookeeper_client.get_children(BROKERS_ID_PATH) 423 | brokers_info = {} 424 | for zk_broker_id in zk_brokers_ids: 425 | zk_broker_id_data, stat = zookeeper_client.get('{0}/{1}'.format(BROKERS_ID_PATH, zk_broker_id)) 426 | zk_broker_info = json.loads(zk_broker_id_data) 427 | zk_broker_host = _get_fqdn(zk_broker_info['host']) 428 | brokers_info[zk_broker_host] = zk_broker_id 429 | return brokers_info 430 | 431 | 432 | def _valid_ipv4_address(host): 433 | sub_parts = host.split('.') 434 | if len(sub_parts) != 4: 435 | return False 436 | try: 437 | return all(0 <= int(part) < 256 for part in sub_parts) 438 | except ValueError: 439 | return False 440 | 441 | 442 | def _get_fqdn(zk_broker_host): 443 | if _valid_ipv4_address(zk_broker_host): 444 | zk_broker_host = socket.getfqdn(zk_broker_host) 445 | return zk_broker_host.split('.')[0] 446 | 447 | 448 | def generate_fd_list_ud_list(host_info): 449 | # Get set of UDs & FDs 450 | ud_set = set() 451 | for val in host_info: 452 | ud_set.add(val[UPDATE_DOMAIN]) 453 | ud_list = sorted(ud_set) 454 | 455 | fd_set = set() 456 | for val in host_info: 457 | fd_set.add(val[FAULT_DOMAIN]) 458 | fd_list = sorted(fd_set) 459 | 460 | return fd_list, ud_list 461 | 462 | 463 | def check_brokers_up(host_info): 464 | for host in host_info: 465 | if not host[BROKER_ID]: 466 | logger.error( 467 | "VM %s with FQDN: %s has no brokers assigned. Ensure that all brokers are up! It is not recommended " 468 | "to perform replica rebalance when brokers are down.", host[VM_ID], host[FQDN]) 469 | return 470 | return True 471 | 472 | 473 | ''' 474 | Determine the free space available on the brokers along with the sizes of the partitions hosted on them. 475 | Returns a dictionary of the format: {"topic-partition_number, size"} 476 | ''' 477 | 478 | 479 | def get_storage_info(host_info): 480 | logger.info("Querying partition sizes from all brokers... This operation can take a few minutes.") 481 | global_partition_sizes = {} 482 | for host in host_info: 483 | free_disk_space, partition_sizes = get_partition_sizes(host[FQDN]) 484 | host[FREE_DISK_SPACE] = int(free_disk_space) 485 | 486 | for i in range(0, len(partition_sizes)): 487 | partitions = partition_sizes[i].split(';') 488 | for p in partitions[1:-1]: 489 | p_name = str(p.split(',')[1].split('/')[2]) 490 | p_size = int(p.split(',')[0]) 491 | if p_name in global_partition_sizes and not (global_partition_sizes[p_name] is None): 492 | global_partition_sizes[p_name] = max(global_partition_sizes[p_name], p_size) 493 | else: 494 | global_partition_sizes[p_name] = p_size 495 | return global_partition_sizes 496 | 497 | 498 | ''' 499 | Generate a replica reassignment JSON file to be passed to the Kafka Replica reassignment tool. 500 | The method parses the cluster manifest to retrieve the topology information about hosts, including the fault & update domains. These are passed to the ReassignmentGenerator class which checks if each topic is already balanced and generates a reassignment plan if not. 501 | Return value (reassignment json) is of the format: 502 | {"partitions": 503 | [{"topic": "foo", 504 | "partition": 1, 505 | "replicas": [1,2,3] } 506 | ], 507 | "version":1 508 | } 509 | Sample Cluster Topology JSON: 510 | { 511 | "hostGroups": { 512 | "gateway": [ 513 | { 514 | ... 515 | }, 516 | . 517 | . 518 | ], 519 | "headnode": [ 520 | { 521 | ... 522 | }, 523 | . 524 | . 525 | ], 526 | "workernode": [ 527 | { 528 | "vmId": 0, 529 | "fqdn": "wn0-k09v3", 530 | "state": "Succeeded", 531 | "faultDomain": 0, 532 | "updateDomain": 2, 533 | "availabilitySetId": "/subscriptions/abe48551-c98b-4263-97b3-098a4c35bc08/resourcegroups/rg0-d373d1ab2fb94339ad55b18da21bb049resourcegroup/providers/Microsoft.Compute/availabilitySets/workernode-0" 534 | }, 535 | . 536 | . 537 | ], 538 | "zookeepernode": [ 539 | { 540 | ... 541 | } 542 | ] 543 | } 544 | } 545 | ''' 546 | 547 | 548 | def generate_reassignment_plan(plan_directory, topics, brokers_info, compute_storage_cost=False, dead_hosts=None, 549 | force_rebalance=False): 550 | logger.info("Starting tool execution...") 551 | ret = None 552 | # Retrieve Cluster topology 553 | cluster_topology_json = get_cluster_topology_json() 554 | # Parse JSON to retrieve information about hosts 555 | host_info = parse_topo_info(cluster_topology_json, brokers_info, dead_hosts) 556 | partitions_sizes = [] 557 | if compute_storage_cost: 558 | partitions_sizes = get_storage_info(host_info) 559 | logger.debug("Partition Sizes: %s", str(partitions_sizes)) 560 | logger.debug("\nHost Information: \n%s", host_info) 561 | logger.debug("\n\nHost Rack Information: \n%s\n", "\n".join( 562 | list(map(lambda datum: "Broker: {0} Rack: {1}".format(str(datum['brokerId']), datum['rack']), host_info)))) 563 | fd_list, ud_list = generate_fd_list_ud_list(host_info) 564 | fd_count, ud_count = len(fd_list), len(ud_list) 565 | 566 | logger.info("Checking if all brokers are up.") 567 | if check_brokers_up(host_info): 568 | 569 | # Keep track of number of replicas we assign to each broker. This count is across all topics 570 | brokers_replica_count = [] 571 | for host in host_info: 572 | b = { 573 | BROKER_ID: host[BROKER_ID], 574 | LEADERS: 0, 575 | FOLLOWERS: 0, 576 | } 577 | brokers_replica_count.append(b) 578 | 579 | # Keep track of already balanced partitions across topics. This is so that we can verify # of leaders across 580 | # brokers at the end. 581 | global_balanced_partitions = [] 582 | 583 | for topic in topics: 584 | logger.info("Getting topic information for Topic: %s", topic) 585 | # Get topic info using the Kafka topic tool 586 | topicInfo = get_topic_info(topic) 587 | replica_count_topic = get_replica_count_topic(topic) 588 | logger.info("Replica count for topic %s is %s", topic, replica_count_topic) 589 | logger.debug("Info for topic %s: \n%s", topic, topicInfo) 590 | topicInfo_lines = topicInfo.split('\n') 591 | if len(topicInfo_lines) < 2: 592 | raise_error("Failed to parse Kafka topic info for topic: %s", topic) 593 | 594 | partition_info = get_partition_info(topic, topicInfo_lines, partitions_sizes) 595 | logger.debug("Partition info for topic %s : \n%s", topic, partition_info) 596 | reassignment_generator = ReassignmentGenerator(host_info, topic, partition_info, compute_storage_cost) 597 | # Generate Rack alternated list 598 | fd_ud_list = reassignment_generator._generate_fd_ud_list() 599 | rack_alternated_list = reassignment_generator._generate_alternated_fd_ud_list(fd_ud_list, fd_list, ud_list) 600 | logger.debug("Generated Rack alternated list: %s", str(rack_alternated_list)) 601 | 602 | # Variables to keep track of which rack in the alternated list is the next one to be assigned a replica. 603 | rand_rack = random.randrange(0, len(rack_alternated_list)) 604 | logger.debug("Rand_rack: %s", str(rand_rack)) 605 | next_Leader = (int(len(ud_list)) * rand_rack) % len(rack_alternated_list) 606 | 607 | logger.debug("Start with position in Rack Alternated List: %s", str(next_Leader)) 608 | 609 | reassignment_plan, balanced_partitions_for_topic = reassignment_generator._generate_reassignment_plan_for_topic( 610 | replica_count_topic, next_Leader, rack_alternated_list, fd_count, ud_count, brokers_replica_count, 611 | force_rebalance) 612 | logger.debug("\nAlready balanced partitions for the topic %s are: \n%s", topic, 613 | balanced_partitions_for_topic) 614 | 615 | for partition in balanced_partitions_for_topic: 616 | global_balanced_partitions.append(partition) 617 | if reassignment_plan is not None: 618 | logger.info("\n\nReassignment plan generated for topic: %s\n%s\n", topic, reassignment_plan) 619 | if ret is not None: 620 | ret["partitions"] += reassignment_plan["partitions"] 621 | else: 622 | ret = reassignment_plan 623 | verify_plan = reassignment_generator._verify_reassignment_plan(reassignment_plan, topic, 624 | replica_count_topic, fd_count, ud_count) 625 | verify_leaders_distributed(host_info, ret, global_balanced_partitions) 626 | 627 | if ret is not None: 628 | verify_leader_count_balanced = verify_leaders_distributed(host_info, ret, global_balanced_partitions) 629 | 630 | # save the reassignment plan in ASSIGNMENT_JSON_FILE 631 | ret = json.dumps(ret) 632 | f = open(os.path.join(plan_directory, ASSIGNMENT_JSON_FILE), "w") 633 | f.write(ret) 634 | f.close() 635 | 636 | logger.info("\n\nComplete Reassignment plan for all topics! \n%s\n", ret) 637 | else: 638 | # remove contents from ASSIGNMENT_JSON_FILE 639 | open(os.path.join(plan_directory, ASSIGNMENT_JSON_FILE), "w").close() 640 | 641 | return ret 642 | 643 | 644 | class ReassignmentGenerator: 645 | def __init__(self, host_info, topic, partition_info, compute_storage_cost): 646 | self.host_info = host_info 647 | self.topic = topic 648 | self.partition_info = partition_info 649 | self.partitions_count = len(partition_info) 650 | self.compute_storage_cost = compute_storage_cost 651 | 652 | def _generate_fd_ud_list(self): 653 | # Get set of FD+UDs 654 | fd_ud_set = set() 655 | for val in self.host_info: 656 | fd_ud_set.add(val[RACK]) 657 | return sorted(fd_ud_set) 658 | 659 | def _get_fd_rack(self, rack): 660 | domains = re.findall(r'\d+', rack) 661 | return int(domains[0]) 662 | 663 | def _get_ud_rack(self, rack): 664 | domains = re.findall(r'\d+', rack) 665 | return int(domains[1]) 666 | 667 | def _gcd(self, a, b): 668 | while b: 669 | a, b = b, a % b 670 | return a 671 | 672 | '''Generates a list of alternated FD + UD combinations. List = [ (fd1,ud1) , (fd2,ud2), ... ] Example with 3 FDs 673 | and 3 UDs : ['FD0UD0', 'FD1UD1', 'FD2UD2', 'FD0UD1', 'FD1UD2', 'FD2UD0', 'FD0UD2', 'FD1UD0', 'FD2UD1'] ''' 674 | 675 | def _generate_alternated_fd_ud_list(self, fd_ud_list, fd_list, ud_list): 676 | alternated_list = [] 677 | 678 | # Find largest FD# & UD#. This is required because there could be gaps and we need to know the largest # to 679 | # compute the possible FD x UD matrix. Missing combinations of (FD,UD) in the VMs allocated are not added to 680 | # the final list. 681 | fd_length = max(map(int, fd_list)) + 1 682 | ud_length = max(map(int, ud_list)) + 1 683 | 684 | i = 0 685 | j = 0 686 | k = 1 687 | 688 | # Traverse matrix in diagonal slices 689 | while True: 690 | current_rack = FAULT_DOMAIN_SHORT + str(i % fd_length) + UPDATE_DOMAIN_SHORT + str(j % ud_length) 691 | if current_rack in fd_ud_list: 692 | if not current_rack in alternated_list: 693 | alternated_list.append(current_rack) 694 | # If FD + UD combo is already present in alternated_list, we are revisiting this the second time. Hence, 695 | # break out of the loop. 696 | else: 697 | break 698 | i += 1 699 | 700 | # If matrix inputs are of form (n,n) or (m,m), add a shift to UD index so that we get a different 701 | # diagonal slice. To get the next adjacent diagonal slice, we add a shift by ud_length - 1. 702 | if self._gcd(ud_length, fd_length) > 1 and i % fd_length == 0: 703 | j = k 704 | k += 1 705 | else: 706 | j += 1 707 | 708 | return alternated_list 709 | 710 | ''' 711 | Conditions required for a partition to be eligible for ReassignmentGenerator 712 | 1> MIN(len(ISR)) >= 1 713 | 2> Leader is in ISR 714 | 3> Leader is assigned (not -1) 715 | 4> Replicas are present 716 | 5> Number of replicas for partition should be equal to replica count for topic 717 | ''' 718 | 719 | def _is_partition_eligible_reassignment(self, partition, replica_count_topic): 720 | does_not_meet_criteria_msg = "Partition for topic does not meet criteria for rebalance. Skipping." 721 | partition[ASSIGNED] = False 722 | if not len(partition[ISR]) >= 1: 723 | logger.warning( 724 | "%s - Topic: %s, Partition: %s. Criteria not met: 'There should be at least one replica in the ISR'.", 725 | does_not_meet_criteria_msg, self.topic, partition[PARTITION]) 726 | return False 727 | if not partition[LEADER] in partition[ISR]: 728 | logger.warning("%s - Topic: %s, Partition: %s. Criteria not met: 'The leader should be in the ISR'.", 729 | does_not_meet_criteria_msg, self.topic, partition[PARTITION]) 730 | return False 731 | if not int(partition[LEADER]) != -1: 732 | logger.warning( 733 | "%s - Topic: %s, Partition: %s. Criteria not met: 'There should be an assigned leader. Leader cannot " 734 | "be -1'.", 735 | does_not_meet_criteria_msg, self.topic, partition[PARTITION]) 736 | return False 737 | if not partition[REPLICAS]: 738 | logger.warning("%s - Topic: %s, Partition: %s. Criteria not met: 'Replicas cannot be null'.", 739 | does_not_meet_criteria_msg, self.topic, partition[PARTITION]) 740 | return False 741 | partition[ASSIGNED] = True 742 | return True 743 | 744 | def _get_brokers_in_rack(self, rack): 745 | return [element for element in self.host_info if element[RACK] == rack] 746 | 747 | def _get_broker_info(self, b_id): 748 | retList = [element for element in self.host_info if int(element[BROKER_ID]) == b_id] 749 | if retList: 750 | return retList[0] 751 | else: 752 | logger.warn("Cannot retrieve host associated with broker with ID: %s", b_id) 753 | return None 754 | 755 | def _get_count_replicas_in_broker(self, broker_id, broker_replica_count): 756 | retList = [element for element in broker_replica_count if element[BROKER_ID] == broker_id] 757 | if retList: 758 | return retList[0] 759 | else: 760 | return None 761 | 762 | def _increment_count_replicas_in_broker(self, broker_id, broker_replica_count, type_of_count): 763 | retList = [element for element in broker_replica_count if element[BROKER_ID] == broker_id] 764 | if retList: 765 | e = retList[0] 766 | e[type_of_count] += 1 767 | else: 768 | logger.warning("%s not found in broker_replica_count!") 769 | 770 | def _get_weighted_count_replicas_in_rack(self, broker_replica_count, rack_alternated_list, rack_index, 771 | type_of_replica): 772 | count = 0 773 | brokers_in_rack = self._get_brokers_in_rack(rack_alternated_list[rack_index]) 774 | if brokers_in_rack: 775 | for broker in brokers_in_rack: 776 | count += self._get_count_replicas_in_broker(broker[BROKER_ID], broker_replica_count)[type_of_replica] 777 | return count / float(len(brokers_in_rack)) 778 | else: 779 | logger.error("No brokers were found for rack %s. Please verify brokers are up!", 780 | rack_alternated_list[rack_index]) 781 | return 782 | 783 | '''Determines the rack (FD+UD combination) for the replica. Once determined, there could be multiple brokers that 784 | meet the criteria. We choose the broker which has less number of replicas assigned to it. (distribute the load) ''' 785 | 786 | def _assign_replica_for_partition(self, rack_alternated_list, broker_replica_count, next_rack, type_of_replica): 787 | 788 | eligible_brokers = self._get_brokers_in_rack(rack_alternated_list[next_rack]) 789 | if eligible_brokers: 790 | new_broker = eligible_brokers[0] 791 | for broker in eligible_brokers: 792 | a = self._get_count_replicas_in_broker(broker[BROKER_ID], broker_replica_count)[type_of_replica] 793 | b = self._get_count_replicas_in_broker(new_broker[BROKER_ID], broker_replica_count)[type_of_replica] 794 | if a < b: 795 | new_broker = broker 796 | 797 | self._increment_count_replicas_in_broker(new_broker[BROKER_ID], broker_replica_count, type_of_replica) 798 | return new_broker[BROKER_ID] 799 | else: 800 | logger.error("No eligible brokers found for rack: %s", rack_alternated_list[next_rack]) 801 | return 802 | 803 | ''' 804 | This method reassigns the replicas for the given partition. The algorithm for assignment is as follows: 805 | 1> Iterate through the rack alternated list and look at sets of size replica_count. 806 | For 3 x 3: the list is: (0,0), (1,1), (2,2), (0,1), (1,2), (2,0), (0,2), (1,0), (2,1) 807 | In first iteration we look at: (0,0) (1,1) (2,2) if replica count is 3. 808 | Each of these represent racks for which there could be multiple brokers. 809 | 2> Determine which of the racks has the least number of leaders. 810 | 3> Assign this rack as the leader for the partition. 811 | 4> Determine all eligible brokers within this rack. Assign the broker with the least number of leaders within the rack as the leader for this partition. 812 | 5> Assign the remaining replicas to the 2 other racks in the set. These are follower replicas. 813 | This is to ensure we will not always get the same set of sequences. 814 | 6> Look at the next set of 3 Racks and repeat from 1> 815 | ''' 816 | 817 | def _scan_partition_for_reassignment(self, index, brokers_replica_count, rack_alternated_list, start_index, 818 | ud_count, replica_count_topic): 819 | reassignment = {"topic": self.topic, 820 | PARTITION: int(self.partition_info[index][PARTITION]), 821 | REPLICAS: [] 822 | } 823 | 824 | replica_count = int(replica_count_topic) 825 | rack_count = len(rack_alternated_list) 826 | 827 | ''' 828 | Re-assign replicas for the PARTITION. 829 | Replicas will be distributed across following racks: start_index, start_index + 1, ...., start_index + replica_count - 1. 830 | ''' 831 | # Determine which rack has fewest LEADERS 832 | current_min = sys.maxint 833 | relative_rack_index = 0 834 | for i in range(0, replica_count): 835 | leaders_in_current_rack = self._get_weighted_count_replicas_in_rack(brokers_replica_count, 836 | rack_alternated_list, 837 | (start_index + i) % rack_count, LEADERS) 838 | if leaders_in_current_rack < current_min: 839 | current_min = leaders_in_current_rack 840 | relative_rack_index = i 841 | rack_index = (start_index + relative_rack_index) % rack_count 842 | 843 | # Do the actual assignment of leader 844 | leader_broker_id = self._assign_replica_for_partition(rack_alternated_list, brokers_replica_count, rack_index, 845 | LEADERS) 846 | 847 | # Check if there is sufficient space on the broker, if not set the "ASSIGNED" property of partition to False 848 | # to indicate that it was not assigned 849 | if self.compute_storage_cost: 850 | logger.debug("Checking if there is sufficient disk space on broker.") 851 | host_for_broker = [element for element in self.host_info if element[BROKER_ID] == leader_broker_id][0] 852 | free_disk_space = host_for_broker[FREE_DISK_SPACE] 853 | if free_disk_space < self.partition_info[index][PARTITION_SIZE]: 854 | logger.warning( 855 | "Not sufficient disk space on elected leader: %s with broker ID: %s. Skipping rebalance for " 856 | "partition: %s", 857 | host_for_broker[FQDN], host_for_broker[BROKER_ID], self.partition_info[index][PARTITION]) 858 | self.partition_info[index][ASSIGNED] = False 859 | return None, start_index % rack_count 860 | else: 861 | # Since we are assigning the partition to the broker, reduce the available free space by the size of 862 | # the partition 863 | host_for_broker[FREE_DISK_SPACE] -= self.partition_info[index][PARTITION_SIZE] 864 | reassignment[REPLICAS].append(int(leader_broker_id)) 865 | 866 | # Assign replicas 867 | p_size = "N/A" 868 | for follower_index in range(0, replica_count): 869 | if follower_index != relative_rack_index: 870 | follower_broker_id = self._assign_replica_for_partition(rack_alternated_list, brokers_replica_count, 871 | (start_index + follower_index) % rack_count, 872 | FOLLOWERS) 873 | reassignment[REPLICAS].append(int(follower_broker_id)) 874 | if self.compute_storage_cost: 875 | host_for_broker = \ 876 | [element for element in self.host_info if element[BROKER_ID] == follower_broker_id][0] 877 | host_for_broker[FREE_DISK_SPACE] -= self.partition_info[index][PARTITION_SIZE] 878 | p_size = self.partition_info[index][PARTITION_SIZE] 879 | logger.debug("Topic: %s Reassigning Partition: %s of SIZE: %s from %s --> %s", self.topic, 880 | self.partition_info[index][PARTITION], p_size, self.partition_info[index][REPLICAS], 881 | reassignment[REPLICAS]) 882 | self.partition_info[index][ASSIGNED] = True 883 | 884 | start_index += min(ud_count, replica_count) 885 | return reassignment, start_index % rack_count 886 | 887 | ''' 888 | Iterate through all replicas of a topic to determine if it is balanced: 889 | 1) Add the UDs of the replicas to a list - fd_list 890 | 1) Add the UDs of the replicas to a list - fd_list 891 | 2) Verify that number of domains the replicas are in is equal min(#replicas, #domains). This ensures that all replicas are in separate UDs and separate FDs. 892 | ''' 893 | 894 | def _check_if_partition_balanced(self, partition, replica_count, fd_count, ud_count, brokers_replica_count, 895 | balanced_partitions): 896 | logger.debug("Checking if Partition %s is balanced.", partition) 897 | if len(partition[REPLICAS]) != replica_count: 898 | logger.warning( 899 | "The replica count for the partition is not the same as the replica count for the topic. Rebalance " 900 | "recommended.") 901 | return False 902 | 903 | fd_list, ud_list = [], [] 904 | for replica in partition[REPLICAS]: 905 | # Get the rack associated with the replica and add to list 906 | host = self._get_broker_info(int(replica)) 907 | # If host was removed from the rack the above will return null. In this case the partition is not balanced 908 | if not host: 909 | return False 910 | current_FD = host[FAULT_DOMAIN] 911 | current_UD = host[UPDATE_DOMAIN] 912 | fd_list.append(current_FD) 913 | ud_list.append(current_UD) 914 | 915 | if len(set(fd_list)) == min(fd_count, replica_count) and len(set(ud_list)) == min(ud_count, replica_count): 916 | if brokers_replica_count: 917 | # Update brokers_replica_count to keep track of number of leaders, followers across brokers 918 | self._increment_count_replicas_in_broker(str(partition[REPLICAS][0]), brokers_replica_count, LEADERS) 919 | for i in range(1, len(partition[REPLICAS])): 920 | self._increment_count_replicas_in_broker(str(partition[REPLICAS][i]), brokers_replica_count, 921 | FOLLOWERS) 922 | balanced_partitions.append(partition) 923 | logger.debug("Partition is balanced across available fault and update domains!") 924 | return True 925 | logger.debug("Partition needs to be balanced.") 926 | return False 927 | 928 | def _generate_reassignment_plan_for_topic(self, replica_count_topic, next_Leader, rack_alternated_list, fd_count, 929 | ud_count, brokers_replica_count, force_rebalance): 930 | ret = None 931 | reassignment = {"partitions": [], "version": 1} 932 | reassignment_changes = [] 933 | retained_assignment = {"partitions": []} 934 | balanced_partitions = [] 935 | 936 | # Check if #replicas is less than 3 if #FD==3/#FD==1 or #replica is less than 4 if #FD is 2 937 | if (fd_count == 1 and replica_count_topic < 3) or (fd_count == 2 and replica_count_topic < 4) or ( 938 | fd_count == 3 and replica_count_topic < 3): 939 | logger.warning(""" 940 | There are not as many upgrade/fault domains as the replica count for the topic %s.\n 941 | Replica Count: %s, Number of Fault Domains: %s, Number of Update Domains: %s.\n 942 | The recommendation is to have at least 3 replicas if number of fault domains in the region is 3, 943 | and 4 replicas if number of fault domains is 2. 944 | """, self.topic, replica_count_topic, fd_count, ud_count) 945 | if not force_rebalance: 946 | logger.error( 947 | "Rebalance with HA not possible! Skipping rebalance for the topic. If you like to rebalance " 948 | "regardless, please run the tool with -force flag.") 949 | return ret, balanced_partitions 950 | else: 951 | logger.info("Proceeding with generation of reassignment plan since -force flag was specified.") 952 | 953 | # Check if there is a valid number of replicas for the topic 954 | if replica_count_topic <= 1: 955 | logger.warning( 956 | "Invalid number of replicas for topic %s. Rebalance with HA guarantee not possible! The tool will try " 957 | "to do whats possible.", 958 | self.topic) 959 | return ret, balanced_partitions 960 | 961 | logger.info("Checking if Topic: %s needs to be re-balanced.", self.topic) 962 | # Keep track of numbers of replicas assigned to each broker 963 | 964 | # Iterate through all partitions and check whether they need to be re-balanced 965 | for i in range(0, len(self.partition_info)): 966 | if (not self._check_if_partition_balanced(self.partition_info[i], replica_count_topic, fd_count, ud_count, 967 | brokers_replica_count, balanced_partitions)): 968 | if self._is_partition_eligible_reassignment(self.partition_info[i], replica_count_topic): 969 | r, next_Leader = self._scan_partition_for_reassignment(i, brokers_replica_count, 970 | rack_alternated_list, next_Leader, ud_count, 971 | replica_count_topic) 972 | if r is not None: 973 | reassignment["partitions"].append(r) 974 | ret = reassignment 975 | reassignment_changes.append( 976 | "Partition {0} {1} --> {2}".format(int(self.partition_info[i][PARTITION]), 977 | self.partition_info[i][REPLICAS], r[REPLICAS])) 978 | else: 979 | # Partition is already balanced. Add the existing assignment to the rebalance plan. 980 | current_partition_assignment = { 981 | "topic": self.topic, 982 | PARTITION: self.partition_info[i][PARTITION], 983 | REPLICAS: self.partition_info[i][ISR] 984 | } 985 | retained_assignment["partitions"].append(current_partition_assignment) 986 | 987 | # If some partitions need to be rebalanced (ret is not None) then append the retained assignment to the 988 | # reassignment partitions and update ret 989 | if len(retained_assignment["partitions"]) > 0 and ret is not None: 990 | logger.info("Topic: %s Partitions Already Balanced: %s Partitions To Be Rebalanced: %s", self.topic, 991 | list(map(lambda datum: datum['partition'], retained_assignment["partitions"])), 992 | list(map(lambda datum: datum['partition'], ret["partitions"]))) 993 | for j in retained_assignment["partitions"]: 994 | reassignment["partitions"].append(j) 995 | ret = reassignment 996 | 997 | if len(reassignment_changes) > 0: 998 | logger.info( 999 | "\n\nReassignment changes: Topic: {0} \n{1}\n".format(self.topic, "\n".join(reassignment_changes))) 1000 | return ret, balanced_partitions 1001 | 1002 | ''' 1003 | Verifies that the reassignment plan generated for the topic guarantees high availability. 1004 | ''' 1005 | 1006 | def _verify_reassignment_plan(self, reassignment_plan, topic, replica_count, fd_count, ud_count, 1007 | brokers_replica_count=None, balanced_partitions=[]): 1008 | logger.info("Verifying that the rebalance plan generated meets conditions for HA.") 1009 | partitions_plan = reassignment_plan["partitions"] 1010 | for p in partitions_plan: 1011 | if not self._check_if_partition_balanced(p, replica_count, fd_count, ud_count, brokers_replica_count, 1012 | balanced_partitions): 1013 | logger.warning("Unable to generate reassignment plan that guarantees high availability for topic: %s", 1014 | topic) 1015 | return False 1016 | return True 1017 | 1018 | 1019 | def verify_leaders_distributed(host_info, reassignment_plan, balanced_partitions): 1020 | # Keep track of numbers of replicas assigned to each broker 1021 | brokers_replica_count = [] 1022 | for host in host_info: 1023 | b = { 1024 | BROKER_ID: host[BROKER_ID], 1025 | LEADERS: 0, 1026 | FOLLOWERS: 0, 1027 | RACK: host[RACK] 1028 | } 1029 | brokers_replica_count.append(b) 1030 | 1031 | partitions = reassignment_plan["partitions"] + balanced_partitions 1032 | 1033 | for p in partitions: 1034 | is_leader = True 1035 | for replica in p[REPLICAS]: 1036 | e = [element for element in brokers_replica_count if element[BROKER_ID] == str(replica)][0] 1037 | if is_leader: 1038 | e[LEADERS] += 1 1039 | is_leader = False 1040 | else: 1041 | e[FOLLOWERS] += 1 1042 | 1043 | logger.debug("Count of Replicas Across Brokers: " + str(brokers_replica_count)) 1044 | 1045 | 1046 | def reassign_verify(plan_directory): 1047 | kafka_version, hdp_version = get_kafka_hdp_version() 1048 | if kafka_version >= '3.2.0': 1049 | s = subprocess.check_output([ 1050 | KAFKA_REASSIGN_PARTITIONS_TOOL_PATH, 1051 | "--bootstrap-server", 1052 | get_broker_connect_string(), 1053 | "--reassignment-json-file", 1054 | os.path.join(plan_directory, ASSIGNMENT_JSON_FILE), 1055 | "--verify" 1056 | ]) 1057 | else: 1058 | s = subprocess.check_output([ 1059 | KAFKA_REASSIGN_PARTITIONS_TOOL_PATH, 1060 | "--zookeeper", 1061 | get_zookeeper_connect_string(), 1062 | "--reassignment-json-file", 1063 | os.path.join(plan_directory, ASSIGNMENT_JSON_FILE), 1064 | "--verify" 1065 | ]) 1066 | logger.info(s) 1067 | 1068 | 1069 | def reassign_exec(plan_directory, throttle_limit): 1070 | if os.stat(os.path.join(plan_directory, ASSIGNMENT_JSON_FILE)).st_size == 0: 1071 | logger.info("The reassignment plan is empty. The cluster is possibly already balanced.") 1072 | else: 1073 | kafka_version, hdp_version = get_kafka_hdp_version() 1074 | if kafka_version >= '3.2.0': 1075 | s = subprocess.check_output([ 1076 | KAFKA_REASSIGN_PARTITIONS_TOOL_PATH, 1077 | "--bootstrap-server", 1078 | get_broker_connect_string(), 1079 | "--reassignment-json-file", 1080 | os.path.join(plan_directory, ASSIGNMENT_JSON_FILE), 1081 | "--throttle", 1082 | throttle_limit if throttle_limit else DEFAULT_REBALANCE_THROTTLE_RATE_BPS, 1083 | "--execute" 1084 | ]) 1085 | expected_output = "Successfully started partition reassignments" 1086 | else: 1087 | s = subprocess.check_output([ 1088 | KAFKA_REASSIGN_PARTITIONS_TOOL_PATH, 1089 | "--zookeeper", 1090 | get_zookeeper_connect_string(), 1091 | "--reassignment-json-file", 1092 | os.path.join(plan_directory, ASSIGNMENT_JSON_FILE), 1093 | "--throttle", 1094 | throttle_limit if throttle_limit else DEFAULT_REBALANCE_THROTTLE_RATE_BPS, 1095 | "--execute" 1096 | ]) 1097 | expected_output = "Successfully started reassignment of partitions" 1098 | logger.info(s) 1099 | if expected_output not in s: 1100 | raise_error("Rebalance operation failed!") 1101 | 1102 | 1103 | ''' 1104 | Log Kafka and HDP Version 1105 | ''' 1106 | 1107 | 1108 | def get_kafka_hdp_version(): 1109 | p1 = subprocess.Popen(["find {0} -name \*kafka_\*".format(KAFKA_LIBS_PATH)], shell=True, stdout=subprocess.PIPE) 1110 | data = p1.stdout.readline() 1111 | assert p1.wait() == 0 1112 | data = data.split('\n')[0].split('/')[-1].split('-') 1113 | splits = data[1].split('.') 1114 | kafka_version = ".".join(splits[:3]) 1115 | 1116 | # Verify Kafka version is >= 0.8.1. The official partition reassignment tool is not stable for lower versions 1117 | if int(splits[0]) < 1: 1118 | if int(splits[1]) < 9: 1119 | if int(splits[2]) < 1: 1120 | logger.warning( 1121 | "The official Kafka Partition reassignment tool has known bugs for versions 0.8.0 and below, " 1122 | "and can render a topic unusable. Please see " 1123 | "https://cwiki.apache.org/confluence/display/KAFKA/Replication+tools for more info. The tool is " 1124 | "stable from version 0.8.1. It is highly discouraged to continue execution.") 1125 | 1126 | hdp_version = ".".join(splits[3:]) 1127 | return kafka_version, hdp_version 1128 | 1129 | 1130 | ''' 1131 | Queries service information from Ambari to get the Kafka log directories 1132 | ''' 1133 | 1134 | 1135 | def get_kafka_log_dirs(): 1136 | ah = AmbariHelper() 1137 | json_output = ah.query_url( 1138 | "clusters/" + ah.cluster_name() + "/configurations/service_config_versions?service_name.in(KAFKA)&is_current=true") 1139 | kafka_brokers_configs = \ 1140 | [element for element in json_output["items"][0]["configurations"] if element["type"] == KAFKA_BROKER][0] 1141 | return kafka_brokers_configs["properties"]["log.dirs"].split(',') 1142 | 1143 | 1144 | ''' 1145 | SSH'es to a host using the supplied credentials and executes a command. 1146 | Throws an exception if the command doesn't return 0. 1147 | bgrun: run command in the background 1148 | ''' 1149 | 1150 | 1151 | def ssh(host, cmd, user, password, timeout=30, bg_run=False): 1152 | fname = tempfile.mktemp() 1153 | fout = open(fname, 'w') 1154 | 1155 | options = '-q -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oPubkeyAuthentication=no' 1156 | if bg_run: 1157 | options += ' -f' 1158 | ssh_cmd = 'ssh %s@%s %s "%s"' % (user, host, options, cmd) 1159 | child = pexpect.spawn(ssh_cmd, timeout=timeout) 1160 | child.expect(['password: ']) 1161 | child.sendline(password) 1162 | child.logfile = fout 1163 | child.expect(pexpect.EOF) 1164 | child.close() 1165 | fout.close() 1166 | 1167 | fin = open(fname, 'r') 1168 | stdout = fin.read() 1169 | fin.close() 1170 | 1171 | if 0 != child.exitstatus: 1172 | raise Exception(stdout) 1173 | 1174 | return stdout 1175 | 1176 | 1177 | def get_partition_sizes(fqdn): 1178 | kafka_log_dirs = get_kafka_log_dirs() 1179 | disk_space_query = "df %s --output=avail | awk 'NR>1'" % kafka_log_dirs[0] 1180 | free_disk_space = ssh(fqdn, disk_space_query, user_name, password) 1181 | 1182 | partition_sizes = [] 1183 | for log_dir in kafka_log_dirs: 1184 | partition_sizes_query = "sudo du %s -d 1 | sort -nr | tr '\t' ',' | tr '\n' ';'" % log_dir 1185 | partition_sizes.append(ssh(fqdn, partition_sizes_query, user_name, password)) 1186 | return free_disk_space, partition_sizes 1187 | 1188 | 1189 | def raise_error(msg): 1190 | logger.error(msg) 1191 | raise Exception(msg) 1192 | 1193 | 1194 | def main(): 1195 | parser = argparse.ArgumentParser(description='Kafka Replica Reassignment Tool') 1196 | parser.add_argument('--topics', 1197 | help='Comma separated list of topics to reassign replicas. Use ALL|all to rebalance all topics', 1198 | type=str) 1199 | parser.add_argument('--execute', action='store_true', default=False, 1200 | help='whether or not to execute the reassignment plan') 1201 | parser.add_argument('--verify', action='store_true', default=False, 1202 | help='Execute rebalance of given plan and verify execution') 1203 | parser.add_argument('--force', action='store_true', default=False, 1204 | help='Force rebalance of all partitions in a topic, even if already balanaced.') 1205 | parser.add_argument('--throttle', help='Upper bound on bandwidth used to move replicas from machine to machine.') 1206 | parser.add_argument('--rebalancePlanDir', 1207 | help='Directory where the rebalance plan should be saved or retrieved from.') 1208 | parser.add_argument('--computeStorageCost', action='store_true', default=False, 1209 | help='Use this for a non-new cluster to use compute free disk space per broker and partition sizes to determine the best reassignment plan. ') 1210 | parser.add_argument('--deadhosts', help='Comma separated list of hosts which have been removed from the cluster', 1211 | type=str) 1212 | parser.add_argument('--username', help='Username for current user.') 1213 | parser.add_argument('--password', help='Password for current user.') 1214 | args = parser.parse_args() 1215 | parser.print_help() 1216 | 1217 | kafka_version, hdp_version = get_kafka_hdp_version() 1218 | logger.info("Kafka Version: %s", kafka_version) 1219 | logger.info("HDP Version: %s", hdp_version) 1220 | 1221 | topics = args.topics 1222 | compute_storage_cost = args.computeStorageCost 1223 | global user_name 1224 | user_name = args.username 1225 | global password 1226 | password = args.password 1227 | dead_hosts = None 1228 | force_rebalance = args.force 1229 | throttle_limit = args.throttle 1230 | 1231 | if args.deadhosts: 1232 | dead_hosts = [item for item in args.deadhosts.split(',')] 1233 | 1234 | # Create directory to store rebalance plan if the specified directory not exist. 1235 | if args.rebalancePlanDir: 1236 | try: 1237 | os.makedirs(args.rebalancePlanDir) 1238 | except OSError as exc: 1239 | if exc.errno == errno.EEXIST and os.path.isdir(args.rebalancePlanDir): 1240 | pass 1241 | else: 1242 | raise_error('Unable to create log dir: {0}'.format(args.rebalancePlanDir)) 1243 | else: 1244 | logger.info("Please specify path the directory where the rebalance plan should be saved/read from using " 1245 | "--rebalancePlanDir.") 1246 | sys.exit() 1247 | 1248 | if args.verify: 1249 | reassign_verify(args.rebalancePlanDir) 1250 | return 1251 | 1252 | if args.execute: 1253 | reassign_exec(args.rebalancePlanDir, throttle_limit) 1254 | return 1255 | 1256 | if topics is None: 1257 | logger.info("Please specify topics to rebalance using -topics. Use ALL to rebalance all topics.") 1258 | sys.exit() 1259 | 1260 | if topics.lower() == ALL_TOPICS_STRING.lower(): 1261 | topics = get_topic_list() 1262 | else: 1263 | topics = [item for item in topics.split(',')] 1264 | 1265 | logger.info("Following topics selected: %s", str(topics)) 1266 | 1267 | # Initialize Zookeeper Client 1268 | zookeeper_quorum = get_zookeeper_connect_string() 1269 | zookeeper_client = connect(zookeeper_quorum) 1270 | # Get broker Ids to Hosts mapping 1271 | brokers_info = get_brokerhost_info(zookeeper_client) 1272 | reassignment_plan = generate_reassignment_plan(args.rebalancePlanDir, topics, brokers_info, compute_storage_cost, 1273 | dead_hosts, force_rebalance) 1274 | 1275 | if reassignment_plan is None: 1276 | logger.info( 1277 | "No need to rebalance. Current Kafka replica assignment has High Availability OR minimum requirements for " 1278 | "rebalance not met. Check logs at %s for more info.", 1279 | str(log_dir) + str(log_file)) 1280 | return 1281 | else: 1282 | logger.info("This is the reassignment-json-file, saved as %s at the specified directory: %s", 1283 | ASSIGNMENT_JSON_FILE, args.rebalancePlanDir) 1284 | logger.info("Please re-run this tool with '-execute' to perform rebalance operation.") 1285 | 1286 | 1287 | if __name__ == "__main__": 1288 | initialize_logger(logger, log_file) 1289 | main() 1290 | --------------------------------------------------------------------------------