├── .gitignore ├── test ├── ansible.cfg ├── requirements.yml └── playbook.yml ├── templates ├── etc │ ├── defaults │ │ └── kafka.j2 │ ├── systemd │ │ └── system │ │ │ └── kafka.service.j2 │ └── init │ │ └── kafka.conf.j2 └── usr │ └── local │ └── kafka │ └── config │ ├── consumer.properties.j2 │ ├── log4j.properties.j2 │ ├── producer.properties.j2 │ └── server.properties.j2 ├── handlers └── main.yml ├── tasks ├── java.yml ├── system.yml ├── main.yml ├── limits.yml ├── check-env.yml ├── kafka-install.yml └── kafka-cfg.yml ├── meta └── main.yml ├── .travis.yml ├── LICENSE ├── Vagrantfile ├── README.md └── defaults └── main.yml /.gitignore: -------------------------------------------------------------------------------- 1 | # Vim temporary swap files. 2 | *.sw[opa] 3 | -------------------------------------------------------------------------------- /test/ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | roles_path = ../../ 3 | -------------------------------------------------------------------------------- /test/requirements.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - src: https://github.com/hpcloud-mon/ansible-zookeeper 4 | name: zookeeper 5 | -------------------------------------------------------------------------------- /templates/etc/defaults/kafka.j2: -------------------------------------------------------------------------------- 1 | KAFKA_HEAP_OPTS='{{ kafka_heap_opts }}' 2 | LOG_DIR='{{ kafka_log_dir }}' 3 | 4 | {% for key, value in kafka_environment.iteritems() %} 5 | {{key}}={{value}} 6 | {% endfor %} 7 | -------------------------------------------------------------------------------- /test/playbook.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - hosts: localhost 4 | vars: 5 | zookeeper_hosts: localhost:2181 6 | kafka_zookeeper_hosts: 7 | - localhost 8 | kafka_hosts: 9 | - localhost 10 | roles: 11 | - zookeeper 12 | - ansible-kafka 13 | -------------------------------------------------------------------------------- /handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "start kafka" 3 | service: name=kafka state=started 4 | sudo: yes 5 | 6 | - name: "stop kafka" 7 | service: name=kafka state=stopped 8 | sudo: yes 9 | 10 | - name: "restart kafka" 11 | service: name=kafka state=restarted 12 | sudo: yes 13 | 14 | -------------------------------------------------------------------------------- /templates/usr/local/kafka/config/consumer.properties.j2: -------------------------------------------------------------------------------- 1 | # consumer.properties.j2 2 | 3 | zookeeper.connect={{ kafka_zookeeper_connection_string }} 4 | 5 | # timeout in ms for connecting to zookeeper 6 | zookeeper.connection.timeout.ms=6000 7 | 8 | # consumer group id 9 | group.id=my-consumer-group 10 | -------------------------------------------------------------------------------- /tasks/java.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "Check if Java is installed" 3 | shell: command -v java 4 | register: check_java 5 | ignore_errors: True 6 | changed_when: False 7 | check_mode: False 8 | 9 | - name: "Install OpenJDK" 10 | apt: name="{{ kafka_java_version }}" state=present 11 | become: true 12 | when: check_java.rc != 0 13 | -------------------------------------------------------------------------------- /templates/etc/systemd/system/kafka.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kafka Daemon 3 | After=zookeeper.service 4 | 5 | [Service] 6 | Type=simple 7 | User={{ kafka_user }} 8 | Group={{ kafka_group }} 9 | LimitNOFILE={{ kafka_nofiles_limit }} 10 | Restart=on-failure 11 | EnvironmentFile=/etc/default/kafka 12 | ExecStart=/usr/local/kafka/bin/kafka-server-start.sh /etc/kafka/server.properties 13 | 14 | [Install] 15 | WantedBy=multi-user.target 16 | -------------------------------------------------------------------------------- /tasks/system.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "Create kafka system group" 3 | group: name={{ kafka_group }} system=yes 4 | sudo: yes 5 | 6 | - name: "Create kafka system user" 7 | user: name={{ kafka_user }} system=yes group={{ kafka_group }} 8 | sudo: yes 9 | 10 | - name: "Add kafka utilities to /etc/profile* so they'll be available in $PATH" 11 | lineinfile: dest=/etc/profile.d/Z99-kafka.sh regexp="^export PATH=.*$" line='export PATH="${PATH}:/usr/local/kafka/bin"' state=present create=yes 12 | sudo: yes 13 | 14 | -------------------------------------------------------------------------------- /meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | galaxy_info: 3 | author: Jay Taylor 4 | description: An Ansible role for installing and configuring a kafka cluster. 5 | company: Gigawatt 6 | license: BSD 7 | min_ansible_version: 2.2.3.0 8 | platforms: 9 | - name: Debian 10 | versions: 11 | - 7 12 | - name: Ubuntu 13 | versions: 14 | - trusty 15 | - xenial 16 | categories: 17 | - cloud 18 | - kafka 19 | - pubsub 20 | - queue 21 | - messaging 22 | - clustering 23 | - development 24 | dependencies: [] 25 | -------------------------------------------------------------------------------- /tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - include: check-env.yml 3 | tags: 4 | - kafka-install 5 | - kafka-cfg 6 | - check-env 7 | 8 | - include: java.yml 9 | when: kafka_java_enabled 10 | tags: 11 | - kafka-install 12 | - java 13 | 14 | - include: system.yml 15 | tags: 16 | - kafka-install 17 | - kafka-cfg 18 | 19 | - include: limits.yml 20 | tags: 21 | - kafka-cfg 22 | - limits 23 | 24 | - include: kafka-install.yml 25 | tags: 26 | - kafka-install 27 | 28 | - include: kafka-cfg.yml 29 | tags: 30 | - kafka-cfg 31 | -------------------------------------------------------------------------------- /tasks/limits.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "Increase soft nofile limit for kafka system user" 3 | lineinfile: dest=/etc/security/limits.conf regexp="^[ \t]*{{ kafka_user }} soft nofile .+[ \t]*$" line="kafka soft nofile {{ kafka_nofiles_limit }}" state=present create=yes 4 | sudo: yes 5 | 6 | - name: "Increase hard nofile limit for kafka system user" 7 | lineinfile: dest=/etc/security/limits.conf regexp="^[ \t]*{{ kafka_user }} hard nofile .+[ \t]*$" line="kafka hard nofile {{ kafka_nofiles_limit }}" state=present create=yes 8 | sudo: yes 9 | 10 | - name: "Add 'session required pam_limits.so' to /etc/pam.d/common-session" 11 | lineinfile: dest=/etc/pam.d/common-session regexp="^[ \t]*session required pam_limits\.so[ \t]*$" line="session required pam_limits.so" state=present create=yes 12 | sudo: yes 13 | 14 | -------------------------------------------------------------------------------- /tasks/check-env.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: "Check 'kafka_zookeeper_hosts' variable" 4 | fail: msg="Playbook execution aborted due to missing or empty required variable 'kafka_zookeeper_hosts'" 5 | when: kafka_zookeeper_hosts is not defined or kafka_zookeeper_hosts | length == 0 6 | 7 | - name: "Check 'kafka_hosts' variable" 8 | fail: msg="Playbook execution aborted due to missing or empty required variable 'kafka_hosts'" 9 | when: kafka_hosts is not defined or kafka_hosts == '' 10 | 11 | - name: "Check 'kafka_generate_broker_id' variable" 12 | fail: msg="Playbook execution aborted because when 'kafka_version' < '0.9.0.0' either 'kafka_broker_id' must be defined or 'kafka_generate_broker_id' enabled" 13 | when: > 14 | not kafka_generate_broker_id | bool and 15 | kafka_broker_id is not defined and 16 | kafka_version | version_compare('0.9.0.0', '<') 17 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | sudo: required 4 | dist: trusty 5 | 6 | language: python 7 | python: 2.7 8 | 9 | env: 10 | matrix: 11 | - ANSIBLE_VERSION=2.2.3.0 ANSIBLE_EXTRA_VARS="kafka_version=0.11.0.2" 12 | - ANSIBLE_VERSION=2.2.3.0 ANSIBLE_EXTRA_VARS="kafka_version=1.0.1 kafka_generate_broker_id=false" 13 | - ANSIBLE_VERSION=2.4.2.0 ANSIBLE_EXTRA_VARS="kafka_version=0.11.0.2" 14 | - ANSIBLE_VERSION=2.4.2.0 ANSIBLE_EXTRA_VARS="kafka_version=1.0.1" 15 | 16 | before_install: 17 | - sudo apt-get update -qq 18 | 19 | install: 20 | - pip install ansible==$ANSIBLE_VERSION 21 | 22 | script: 23 | - cd test 24 | - ansible-galaxy install -r requirements.yml 25 | - ansible-playbook -i "localhost," playbook.yml --extra-vars="${ANSIBLE_EXTRA_VARS}" --syntax-check 26 | - ansible-playbook -i "localhost," playbook.yml --extra-vars="${ANSIBLE_EXTRA_VARS}" --connection=local --sudo 27 | - > 28 | ansible-playbook -i "localhost," playbook.yml --extra-vars="${ANSIBLE_EXTRA_VARS}" --connection=local --sudo 29 | | grep -q 'changed=0.*failed=0' 30 | && (echo 'Idempotence test: pass' && exit 0) 31 | || (echo 'Idempotence test: fail' && exit 1) 32 | -------------------------------------------------------------------------------- /templates/etc/init/kafka.conf.j2: -------------------------------------------------------------------------------- 1 | #!upstart 2 | description "kafka" 3 | 4 | start on (local-filesystems and net-device-up IFACE!=lo) 5 | stop on runlevel [!12345] 6 | 7 | respawn 8 | 9 | limit nofile {{ kafka_nofiles_limit }} {{ kafka_nofiles_limit }} 10 | 11 | env USER={{ kafka_user }} 12 | env GROUP={{ kafka_group }} 13 | env LOG_DIR={{ kafka_log_dir }} 14 | env STDOUT={{ kafka_log_dir }}/kafka.log 15 | env STDERR={{ kafka_log_dir }}/kafka.err 16 | env PID=/var/run/kafka.pid 17 | 18 | script 19 | KAFKA_OPTS=$(tr '\n' ' ' < /etc/default/kafka) 20 | 21 | test -d "${LOG_DIR}" || mkdir -p "${LOG_DIR}" 22 | chown -R "${USER}:${GROUP}" "${LOG_DIR}" 23 | echo $$ > "${PID}" 24 | sudo su $USER --shell /bin/sh -c 'echo "start inititated, ulimit -n => $(ulimit -n)"'" 1>>${STDOUT} 2>>${STDERR}" 25 | # Rather than using setuid/setgid sudo is used because the pre-start task must run as root. 26 | exec sudo -H -u ${USER} -g ${GROUP} /bin/sh -c "${KAFKA_OPTS} /usr/local/kafka/bin/kafka-server-start.sh {{ kafka_conf_dir }}/server.properties 1>>${STDOUT} 2>>${STDERR}" 27 | end script 28 | 29 | post-stop script 30 | rm -f "${PID}" 31 | exec sudo --set-home --user="${USER}" --group="${GROUP}" /bin/sh -c "/usr/local/kafka/bin/kafka-server-stop.sh 1>>${STDOUT} 2>>${STDERR}" 32 | end script 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 by Jay Taylor. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without modification, 4 | are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, 7 | this list of conditions and the following disclaimer. 8 | 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | 3. The names of the contributors may not be used to endorse or promote 14 | products derived from this software without specific prior written 15 | permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 18 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 21 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 23 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 24 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 26 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /tasks/kafka-install.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "Download kafka binary package archive" 3 | get_url: url={{ kafka_url }} dest={{ kafka_bin_tmp }} timeout=600 4 | 5 | - name: "Download the corresponding cryptographic signature file" 6 | get_url: url={{ kafka_sig_url }} dest={{ kafka_sig_tmp }} timeout=600 7 | 8 | - name: "Download the kafka PGP keys" 9 | get_url: url={{ kafka_keys_url }} dest={{ kafka_keys_tmp }} timeout=600 10 | changed_when: False 11 | 12 | - name: "Import kafka PGP keys" 13 | shell: gpg --import {{ kafka_keys_tmp }} 14 | changed_when: False 15 | 16 | - name: "Verify kafka binary package archive authenticity" 17 | shell: gpg --verify {{ kafka_sig_tmp }} {{ kafka_bin_tmp }} 18 | changed_when: False 19 | 20 | - name: "Extract downloaded kafka archive" 21 | unarchive: copy=no creates=/usr/local/kafka_{{ kafka_scala_version }}-{{ kafka_version }} dest=/usr/local src={{ kafka_bin_tmp }} 22 | sudo: yes 23 | 24 | - name: "Link /usr/local/kafka to the right version" 25 | file: path=/usr/local/kafka state=link src=/usr/local/kafka_{{ kafka_scala_version }}-{{ kafka_version }} 26 | sudo: yes 27 | 28 | - name: "Create upstart config" 29 | template: dest=/etc/init/kafka.conf src=etc/init/kafka.conf.j2 mode=0644 owner=root group=root 30 | sudo: yes 31 | when: ansible_service_mgr != 'systemd' 32 | notify: 33 | - restart kafka 34 | 35 | - name: "Create systemd config" 36 | template: dest=/etc/systemd/system/kafka.service src=etc/systemd/system/kafka.service.j2 mode=0644 owner=root group=root 37 | sudo: yes 38 | when: ansible_service_mgr == 'systemd' 39 | notify: 40 | - restart kafka 41 | 42 | - name: "Reload systemctl daemon" 43 | command: systemctl daemon-reload 44 | sudo: yes 45 | when: ansible_service_mgr == 'systemd' 46 | 47 | - name: "Create kafka directories" 48 | file: path={{ item }} state=directory mode=0755 owner={{ kafka_user }} group={{ kafka_group }} 49 | sudo: yes 50 | with_items: 51 | - "{{ kafka_data_dir }}" 52 | - "{{ kafka_log_dir }}" 53 | 54 | - name: "Remove lost+found in the datadir" 55 | file: path="{{ kafka_data_dir }}/lost+found" state=absent 56 | sudo: yes 57 | 58 | - name: "Symlink kafka_conf_dir to /usr/local/kafka/config" 59 | file: src=/usr/local/kafka/config path={{ kafka_conf_dir }} state=link 60 | sudo: yes 61 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | # All Vagrant configuration is done below. The "2" in Vagrant.configure 5 | # configures the configuration version (we support older styles for 6 | # backwards compatibility). Please don't change it unless you know what 7 | # you're doing. 8 | Vagrant.configure(2) do |config| 9 | # The most common configuration options are documented and commented below. 10 | # For a complete reference, please see the online documentation at 11 | # https://docs.vagrantup.com. 12 | 13 | # Every Vagrant development environment requires a box. You can search for 14 | # boxes at https://atlas.hashicorp.com/search. 15 | config.vm.box = "ubuntu/trusty64" 16 | 17 | # Disable automatic box update checking. If you disable this, then 18 | # boxes will only be checked for updates when the user runs 19 | # `vagrant box outdated`. This is not recommended. 20 | # config.vm.box_check_update = false 21 | 22 | # Create a forwarded port mapping which allows access to a specific port 23 | # within the machine from a port on the host machine. In the example below, 24 | # accessing "localhost:8080" will access port 80 on the guest machine. 25 | # config.vm.network "forwarded_port", guest: 80, host: 8080 26 | 27 | # Create a private network, which allows host-only access to the machine 28 | # using a specific IP. 29 | # config.vm.network "private_network", ip: "192.168.33.10" 30 | 31 | # Create a public network, which generally matched to bridged network. 32 | # Bridged networks make the machine appear as another physical device on 33 | # your network. 34 | # config.vm.network "public_network" 35 | 36 | # Share an additional folder to the guest VM. The first argument is 37 | # the path on the host to the actual folder. The second argument is 38 | # the path on the guest to mount the folder. And the optional third 39 | # argument is a set of non-required options. 40 | # config.vm.synced_folder "../data", "/vagrant_data" 41 | 42 | # Provider-specific configuration so you can fine-tune various 43 | # backing providers for Vagrant. These expose provider-specific options. 44 | # Example for VirtualBox: 45 | # 46 | # config.vm.provider "virtualbox" do |vb| 47 | # # Display the VirtualBox GUI when booting the machine 48 | # vb.gui = true 49 | # 50 | # # Customize the amount of memory on the VM: 51 | # vb.memory = "1024" 52 | # end 53 | # 54 | # View the documentation for the provider you are using for more 55 | # information on available options. 56 | config.vm.provider "virtualbox" do |v| 57 | v.memory = 2048 58 | v.cpus = 2 59 | end 60 | 61 | # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies 62 | # such as FTP and Heroku are also available. See the documentation at 63 | # https://docs.vagrantup.com/v2/push/atlas.html for more information. 64 | # config.push.define "atlas" do |push| 65 | # push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" 66 | # end 67 | 68 | # Enable provisioning with a shell script. Additional provisioners such as 69 | # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the 70 | # documentation for more information about their specific syntax and use. 71 | # config.vm.provision "shell", inline: <<-SHELL 72 | # sudo apt-get update 73 | # sudo apt-get install -y apache2 74 | # SHELL 75 | config.vm.provision "shell", inline: <<-SHELL 76 | set -e 77 | sudo apt-get update -qq 78 | sudo apt-get install python-dev python-pip git -y 79 | sudo pip install ansible 80 | rm -rf /tmp/ansible-kafka 81 | cp -R /vagrant /tmp/ansible-kafka 82 | cd /tmp/ansible-kafka/test 83 | ANSIBLE_EXTRA_VARS="" 84 | ansible-galaxy install -r requirements.yml 85 | ansible-playbook -i "localhost," playbook.yml --extra-vars="${ANSIBLE_EXTRA_VARS}" --syntax-check 86 | ansible-playbook -i "localhost," playbook.yml --extra-vars="${ANSIBLE_EXTRA_VARS}" --connection=local --sudo 87 | SHELL 88 | end 89 | -------------------------------------------------------------------------------- /tasks/kafka-cfg.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "Generate kafka_server fact" 3 | set_fact: kafka_server="{{ kafka_server_defaults | combine(kafka_server) }}" 4 | 5 | - name: "Generate kafka_healtcheck_address fact" 6 | set_fact: kafka_healthcheck_address="{{ ((kafka_server.host_name | default('127.0.0.1')) == '') | ternary('127.0.0.1', kafka_server.host_name) }}" 7 | when: kafka_healthcheck_address == "" 8 | 9 | - name: "Generate kafka_producer fact" 10 | set_fact: kafka_producer="{{ kafka_producer_defaults | combine(kafka_producer) }}" 11 | 12 | - name: "Generate generic unique machine id integer" 13 | # NB: This uses a combination of root partition UUID + network interface MAC address. 14 | shell: ( test -r /etc/fstab && find /dev/disk/by-uuid/ -type l -printf "%f\n" | sort | head -1 | grep --ignore-case --only-matching --extended-regexp --max 1 '[0-9a-f]{3,}[0-9a-f-]+' | tr -d '-' || echo '0' ; ifconfig | grep --ignore-case --only-matching --extended-regexp '([0-9a-f]{2}:){5}[0-9a-f]{2}' | tr -d ':' | tr -d '\n') | python -c 'import sys; x, y = sys.stdin.read().split(chr(10))[0:2]; x = int(x, 16); y = int(y, 16); sys.stdout.write((str(x + y)[-9:])); sys.exit(1 if x == 0 and y == 0 else 0)' 15 | register: machineidinteger 16 | changed_when: False 17 | check_mode: False 18 | when: kafka_generate_broker_id | bool 19 | 20 | - name: "Use generated unique machine id integer as broker id" 21 | set_fact: kafka_broker_id={{ machineidinteger.stdout_lines[0] }} 22 | when: kafka_generate_broker_id | bool 23 | 24 | - name: "Raise reserved broker id range" 25 | set_fact: kafka_reserved_broker_max_id=1000000000 26 | when: kafka_generate_broker_id | bool and kafka_version | version_compare('0.9.0.0', '>=') 27 | 28 | - name: "Generate the zookeeper hosts connection string" 29 | set_fact: kafka_zookeeper_connection_string="{{ kafka_zookeeper_hosts | join(':' ~ kafka_zookeeper_port ~ ',') }}:{{ kafka_zookeeper_port }}" 30 | 31 | - name: "Generate the kafka hosts connection string" 32 | set_fact: kafka_connection_string="{{ kafka_hosts | join(':' ~ kafka_server.port ~ ',') }}:{{ kafka_server.port }}" 33 | 34 | - name: "Render and write out kafka configuration files" 35 | template: src=usr/local/kafka/config/{{ item }}.j2 dest="{{ kafka_conf_dir }}/{{ item }}" mode=0640 owner={{ kafka_user }} group={{ kafka_group }} 36 | sudo: yes 37 | with_items: 38 | - server.properties 39 | - producer.properties 40 | - consumer.properties 41 | - log4j.properties 42 | notify: 43 | - restart kafka 44 | 45 | - name: "Render and write out kafka user env file" 46 | template: src=etc/defaults/kafka.j2 dest="/etc/default/kafka" mode=0644 owner=root group=root 47 | sudo: yes 48 | notify: 49 | - restart kafka 50 | 51 | - name: "Enable kafka system service" 52 | service: name=kafka state=started enabled=yes 53 | sudo: yes 54 | tags: 55 | - kafka-install 56 | 57 | - name: "Link alternate logs directory and touch output files" 58 | shell: "( test ! -e /usr/local/kafka/logs || rm -rf /usr/local/kafka/logs ) && ln -s {{ kafka_log_dir }} /usr/local/kafka/logs && touch {{ kafka_log_dir }}/state-change.log {{ kafka_log_dir }}/kafkaServer.out && chown {{ kafka_user }}:{{ kafka_group }} {{ kafka_log_dir }}/state-change.log {{ kafka_log_dir }}/kafkaServer.out" 59 | changed_when: False 60 | sudo: yes 61 | 62 | - meta: flush_handlers 63 | 64 | - name: "Wait for kafka to come up and open it's server port" 65 | wait_for: port={{ kafka_server.port }} host={{ kafka_healthcheck_address }} connect_timeout={{ kafka_port_test_timeout_seconds }} timeout={{ kafka_port_test_timeout_seconds }} 66 | ignore_errors: yes 67 | register: healthcheck 68 | tags: 69 | - kafka-healthcheck 70 | 71 | - name: "Check kafka port test result" 72 | fail: msg="Kafka port not open on host={{ kafka_healthcheck_address }}, port={{ kafka_server.port }}" 73 | when: healthcheck.elapsed is defined and (healthcheck.elapsed >= kafka_port_test_timeout_seconds and kafka_port_test_timeout_seconds > 0) 74 | tags: 75 | - kafka-healthcheck 76 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Ansible Kafka 2 | 3 | [![Build Status](https://travis-ci.org/jaytaylor/ansible-kafka.svg?branch=master)](https://travis-ci.org/jaytaylor/ansible-kafka) 4 | [![Galaxy](https://img.shields.io/badge/galaxy-jaytaylor.kafka-blue.svg)](https://galaxy.ansible.com/list#/roles/4083) 5 | 6 | An ansible role to install and configure [kafka](https://kafka.apache.org/) distributed pub/sub messaging queue clusters. 7 | 8 | 9 | ## How to get it 10 | 11 | Add to your playbooks requirements.yml: 12 | 13 | - src: https://github.com/jaytaylor/ansible-kafka 14 | 15 | and then run: 16 | 17 | ansible-galaxy install -r requirements.yml --ignore-errors 18 | 19 | Or if you just want to grab this role and start using it: 20 | 21 | cd my-playbook-folder/roles 22 | git clone https://github.com/jaytaylor/ansible-kafka.git 23 | cd - 24 | 25 | ## How to use it 26 | 27 | Once the role is installed, configure it from your playbook file (e.g. playbook.yml). 28 | 29 | Example: 30 | 31 | ```yml 32 | --- 33 | - hosts: [all] 34 | roles: 35 | - { 36 | role: "ansible-kafka", 37 | kafka_hosts: "{{ groups.kafka | list }}", 38 | kafka_zookeeper_hosts: "{{ zookeeper_hosts | list }}" 39 | kafka_version: 0.11.0.2, # Kafka version override. 40 | kafka_scala_serverion: 2.10 # Scala version override. 41 | } 42 | ``` 43 | 44 | Where `kafka_hosts` and `zookeeper_hosts` are both defined variables 45 | (e.g. in group_vars/all), and contain the list of hosts to use. 46 | 47 | ## Important Note 48 | 49 | If you are using this role from the ansible-galaxy website, make sure you use "jaytaylor.kafka" for the role name (rather than "ansible-kafka"). 50 | 51 | ## Role variables 52 | 53 | - `kafka_hosts` - list of hosts in the cluster. 54 | - `kafka_zookeeper_hosts` - list of zookeeper hosts for the cluster. 55 | - `kafka_broker_id` - Integer uniquely identifying the broker, by default one will be generated for you either by this role or by kafka itself for versions >= 0.9. 56 | - `kafka_generate_broker_id` - Flag controlling whether to generate a broker id, defaults to `yes`. 57 | - `kafka_server_defaults` - Default Kafka server settings. This variable shoulnd't usually be changed. 58 | - `kafka_producer_defaults` - Default Kafka producer settings. This variable shouldn't usually be changed. 59 | - `kafka_server` - Allows to overwrite particular default server settings (from `kafka_server_defaults` variable). Values in this hash are combined with `kafka_server_defaults` variable. 60 | - `kafka_producer` - Allows to overwrite particular default producer settings (from `kafka_producer_defaults` variable). Values in this hash are combined with `kafka_server_defaults` variable. 61 | - `kafka_healthcheck_address` - If not defined, this will dynamicaly set to `kafka_server_defaults.host_name` or `kafka_server.host_name` if defined. Defaults to '127.0.0.1' 62 | - `kafka_java_version` - Java version to install. Defaults to "openjdk-7-jre-headless" 63 | 64 | ## Version notes 65 | 66 | Version 2.0.0 does not support Ansible versions lower than 2.2. 67 | 68 | Before version 2.0.0, `server` and `producer` Ansible variables were used to configure Kafka Server and Kafka Producer respecively, but 2.0.0 introduced better variable scoping and better variable overwritting: 69 | 70 | - `server` variable became `kafka_server`, so it is inside Kafka role "scope". It is a hash that is combined with `kafka_server_defaults` hash, overwriting the values defined in the former. This makes easier to overwrite just some particular setting, instead of rewritting the whole `server` variable hash jus tto change one setting. 71 | - `producer` variable became `kafka_producer`, and it behaves now the same way as `kafka_server` variable. Also `kafka_producer_defaults` hash has the default values for producer. 72 | 73 | Also: 74 | 75 | - `healthcheck_address` became `kafka_healthcheck_address`, which improves its scope. 76 | - `nofiles_limit` became `kafka_nofiles_limit` 77 | 78 | ## License 79 | 80 | BSD 81 | 82 | ## Author 83 | 84 | Jay Taylor 85 | 86 | [@jtaylor](https://twitter.com/jtaylor) - [jaytaylor.com](http://jaytaylor.com) - [github](https://github.com/jaytaylor) 87 | -------------------------------------------------------------------------------- /templates/usr/local/kafka/config/log4j.properties.j2: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | kafka.logs.dir={{ kafka_log_dir }} 17 | 18 | log4j.rootLogger={{ kafka_logger_root }} 19 | 20 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 21 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 22 | log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n 23 | 24 | log4j.appender.kafkaAppender=org.apache.log4j.RollingFileAppender 25 | log4j.appender.kafkaAppender.MaxFileSize={{ kafka_max_logfile_size }} 26 | log4j.appender.kafkaAppender.MaxBackupIndex={{ kafka_max_logbackup_idx }} 27 | log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log 28 | log4j.appender.kafkaAppender.DatePattern={{ kafka_log4j_date_pattern }} 29 | log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout 30 | log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 31 | 32 | log4j.appender.stateChangeAppender=org.apache.log4j.RollingFileAppender 33 | log4j.appender.stateChangeAppender.MaxFileSize={{ kafka_max_logfile_size }} 34 | log4j.appender.stateChangeAppender.MaxBackupIndex={{ kafka_max_logbackup_idx }} 35 | log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log 36 | log4j.appender.stateChangeAppender.DatePattern={{ kafka_log4j_date_pattern }} 37 | log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout 38 | log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 39 | 40 | # NB: Tracing requests results in large logs. 41 | log4j.appender.requestAppender=org.apache.log4j.RollingFileAppender 42 | log4j.appender.requestAppender.MaxFileSize={{ kafka_max_logfile_size }} 43 | log4j.appender.requestAppender.MaxBackupIndex={{ kafka_max_logbackup_idx }} 44 | log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log 45 | log4j.appender.requestAppender.DatePattern={{ kafka_log4j_date_pattern }} 46 | log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout 47 | log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 48 | 49 | log4j.appender.cleanerAppender=org.apache.log4j.RollingFileAppender 50 | log4j.appender.cleanerAppender.MaxFileSize={{ kafka_max_logfile_size }} 51 | log4j.appender.cleanerAppender.MaxBackupIndex={{ kafka_max_logbackup_idx }} 52 | log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log 53 | log4j.appender.cleanerAppender.DatePattern={{ kafka_log4j_date_pattern }} 54 | log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout 55 | log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 56 | 57 | log4j.appender.controllerAppender=org.apache.log4j.RollingFileAppender 58 | log4j.appender.controllerAppender.MaxFileSize={{ kafka_max_logfile_size }} 59 | log4j.appender.controllerAppender.MaxBackupIndex={{ kafka_max_logbackup_idx }} 60 | log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log 61 | log4j.appender.controllerAppender.DatePattern={{ kafka_log4j_date_pattern }} 62 | log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout 63 | log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 64 | 65 | # Turn on all our debugging info 66 | #log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG, kafkaAppender 67 | #log4j.logger.kafka.client.ClientUtils=DEBUG, kafkaAppender 68 | #log4j.logger.kafka.perf=DEBUG, kafkaAppender 69 | #log4j.logger.kafka.perf.ProducerPerformance$ProducerThread=DEBUG, kafkaAppender 70 | #log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG 71 | log4j.logger.kafka={{ kafka_logger_kafka }} 72 | 73 | log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender 74 | log4j.additivity.kafka.network.RequestChannel$=false 75 | 76 | #log4j.logger.kafka.network.Processor=TRACE, requestAppender 77 | #log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender 78 | #log4j.additivity.kafka.server.KafkaApis=false 79 | log4j.logger.kafka.request.logger={{ kafka_logger_kafka_request }} 80 | log4j.additivity.kafka.request.logger=false 81 | 82 | log4j.logger.kafka.controller={{ kafka_logger_kafka_controller }} 83 | log4j.additivity.kafka.controller=false 84 | 85 | log4j.logger.kafka.log.LogCleaner={{ kafka_logger_kafka_cleaner }} 86 | log4j.additivity.kafka.log.LogCleaner=false 87 | 88 | log4j.logger.state.change.logger={{ kafka_logger_state }} 89 | log4j.additivity.state.change.logger=false 90 | -------------------------------------------------------------------------------- /defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kafka_zookeeper_hosts: # <-- must be overridden further up the chain 3 | # e.g. [ 'srvr1', 'srvr2' ] 4 | kafka_zookeeper_port: 2181 5 | 6 | kafka_hosts: # <-- must be overridden further up the chain 7 | # e.g. [ 'srvr1', 'srvr2' ] 8 | 9 | kafka_version: "0.11.0.2" 10 | 11 | kafka_scala_version: "2.11" 12 | # NB: 2.11 is recommended at https://kafka.apache.org/downloads.html. 13 | 14 | kafka_java_enabled: true 15 | # set false if the role should not handle the java dependency 16 | kafka_java_version: "openjdk-7-jre-headless" 17 | 18 | # Use this selector to use a mirror for a "current" version or an "archived" one 19 | kafka_mirror_selector: "current" 20 | kafka_mirror: 21 | current: http://apache.mirrors.tds.net/kafka 22 | archive: https://archive.apache.org/dist/kafka 23 | kafka_url: "{{ kafka_mirror[kafka_mirror_selector] }}/{{ kafka_version }}/kafka_{{ kafka_scala_version }}-{{ kafka_version }}.tgz" 24 | kafka_bin_tmp: "/tmp/kafka_{{ kafka_scala_version }}-{{ kafka_version }}.tar.gz" 25 | 26 | kafka_sig_mirror_selector: "{{ kafka_mirror_selector }}" 27 | kafka_sig_mirror: 28 | current: https://dist.apache.org/repos/dist/release/kafka 29 | archive: https://archive.apache.org/dist/kafka/ 30 | kafka_sig_url: "{{ kafka_sig_mirror[kafka_sig_mirror_selector] }}/{{ kafka_version }}/kafka_{{ kafka_scala_version }}-{{ kafka_version }}.tgz.asc" 31 | kafka_keys_url: "http://kafka.apache.org/KEYS" 32 | kafka_keys_tmp: "/tmp/kafka_KEYS" 33 | kafka_sig_tmp: "/tmp/kafka_{{ kafka_scala_version }}-{{ kafka_version }}.tar.gz.asc" 34 | 35 | kafka_conf_dir: /etc/kafka 36 | kafka_data_dir: /var/kafka 37 | kafka_log_dir: /var/log/kafka 38 | 39 | kafka_user: kafka 40 | kafka_group: kafka 41 | 42 | kafka_max_logfile_size: 50MB 43 | kafka_max_logbackup_idx: 7 44 | kafka_logger_root: INFO, stdout 45 | kafka_logger_kafka: INFO, kafkaAppender 46 | kafka_logger_kafka_controller: TRACE, controllerAppender 47 | kafka_logger_kafka_request: WARN, requestAppender 48 | kafka_logger_kafka_cleaner: INFO, cleanerAppender 49 | kafka_logger_state: TRACE, stateChangeAppender 50 | 51 | kafka_heap_opts: "-Xmx1G -Xms1G" 52 | 53 | kafka_nofiles_limit: 50000 54 | 55 | kafka_port_test_timeout_seconds: 120 56 | 57 | kafka_generate_broker_id: yes 58 | 59 | kafka_log4j_date_pattern: "'.'yyyy-MM-dd" # Rollover at midnight each day. 60 | 61 | 62 | kafka_environment: {} 63 | 64 | kafka_healthcheck_address: "" 65 | 66 | kafka_server: {} 67 | 68 | kafka_server_defaults: 69 | # broker_id: <-- this is auto-set by hashing the machine-id during kafka-cfg step. 70 | # zookeeper_hosts: <-- this is auto-set by the global "kafka_zookeeper_hosts" setting. 71 | log_dirs: "{{ kafka_data_dir }}" 72 | port: 9092 73 | message_max_bytes: 74 | num_network_threads: 3 75 | num_io_threads: 8 76 | background_threads: 77 | queued_max_requests: 78 | host_name: localhost 79 | advertised_host_name: 80 | advertised_port: 81 | advertised_listeners: 82 | listeners: 83 | socket_send_buffer_bytes: 102400 84 | socket_receive_buffer_bytes: 102400 85 | socket_request_max_bytes: 104857600 86 | num_partitions: 1 87 | log_segment_bytes: 1073741824 88 | log_roll_ms: 89 | log_roll_hours: 90 | log_cleanup_policy: 91 | log_retention_ms: 92 | log_retention_minutes: 93 | log_retention_hours: 168 94 | log_retention_bytes: 1073741824 95 | log_retention_check_interval_ms: 300000 96 | log_cleaner_enable: false 97 | log_cleaner_threads: 98 | log_cleaner_io_max_bytes_per_second: 99 | log_cleaner_dedupe_buffer_size: 100 | log_cleaner_io_buffer_size: 101 | log_cleaner_io_buffer_load_factor: 102 | log_cleaner_backoff_ms: 103 | log_cleaner_min_cleanable_ratio: 104 | log_cleaner_delete_retention_ms: 105 | log_index_size_max_bytes: 106 | log_index_interval_bytes: 107 | log_flush_interval_messages: 1000 108 | log_flush_scheduler_interval_ms: 1000 109 | log_flush_interval_ms: 110 | log_delete_delay_ms: 111 | log_flush_offset_checkpoint_interval_ms: 112 | log_segment_delete_delay_ms: 113 | auto_create_topics_enable: 114 | controller_socket_timeout_ms: 115 | controller_message_queue_size: 116 | default_replication_factor: 117 | replica_lag_time_max_ms: 118 | replica_lag_max_messages: 119 | replica_socket_timeout_ms: 120 | replica_socket_receive_buffer_bytes: 121 | replica_fetch_max_bytes: 122 | replica_fetch_wait_max_ms: 123 | replica_fetch_min_bytes: 124 | num_replica_fetchers: 125 | replica_high_watermark_checkpoint_interval_ms: 126 | fetch_purgatory_purge_interval_requests: 127 | producer_purgatory_purge_interval_requests: 128 | zookeeper_session_timeout_ms: 129 | zookeeper_connection_timeout_ms: 6000 130 | zookeeper_sync_time_ms: 131 | controlled_shutdown_enable: "true" 132 | controlled_shutdown_max_retries: 3 133 | controlled_shutdown_retry_backoff_ms: 5000 134 | auto_leader_rebalance_enable: 135 | leader_imbalance_per_broker_percentage: 136 | leader_imbalance_check_interval_seconds: 137 | offset_metadata_max_bytes: 138 | max_connections_per_ip: 139 | max_connections_per_ip_overrides: 140 | connections_max_idle_ms: 141 | log_roll_jitter_ms: 142 | log_roll_jitter_hours: 143 | num_recovery_threads_per_data_dir: 1 144 | unclean_leader_election_enable: 145 | delete_topic_enable: 146 | offsets_topic_num_partitions: 147 | offsets_topic_retention_minutes: 148 | offsets_retention_check_interval_ms: 149 | offsets_topic_replication_factor: 150 | offsets_topic_segment_bytes: 151 | offsets_load_buffer_size: 152 | offsets_commit_required_acks: 153 | offsets_commit_timeout_ms: 154 | misc: {} 155 | 156 | kafka_producer: {} 157 | 158 | kafka_producer_defaults: 159 | # metadata_broker_list: <-- this is auto-set by the global "kafka_hosts" setting. 160 | request_required_acks: 1 161 | request_timeout_ms: 162 | type: sync 163 | serializer_class: kafka.serializer.DefaultEncoder 164 | partitioner_class: 165 | key_serializer_class: 166 | compression_codec: none 167 | compressed_topics: 168 | message_send_max_retries: 169 | retry_backoff_ms: 170 | topic_metadata_refresh_interval_ms: 171 | send_buffer_bytes: 172 | # async options. 173 | queue_buffering_max_ms: 174 | queue_buffering_max_messages: 175 | queue_enqueue_timeout_ms: 176 | batch_num_messages: 177 | -------------------------------------------------------------------------------- /templates/usr/local/kafka/config/producer.properties.j2: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # see kafka.producer.ProducerConfig for more details 16 | 17 | ############################# Producer Basics ############################# 18 | 19 | # list of brokers used for bootstrapping knowledge about the rest of the cluster 20 | # format: host1:port1,host2:port2 ... 21 | metadata.broker.list={{ kafka_connection_string }} 22 | 23 | {% if kafka_producer %} 24 | 25 | {% if kafka_producer.request_required_acks %} 26 | # This value controls when a produce request is considered completed. 27 | # Specifically, how many other brokers must have committed the data to their log 28 | # and acknowledged this to the leader? Typical values are: 29 | # 30 | # 0, which means that the producer never waits for an acknowledgement from the 31 | # broker (the same behavior as 0.7). This option provides the lowest 32 | # latency but the weakest durability guarantees (some data will be lost 33 | # when a server fails). 34 | # 1, which means that the producer gets an acknowledgement after the leader 35 | # replica has received the data. This option provides better durability as 36 | # the client waits until the server acknowledges the request as successful 37 | # (only messages that were written to the now-dead leader but not yet 38 | # replicated will be lost). 39 | # -1, The producer gets an acknowledgement after all in-sync replicas have 40 | # received the data. This option provides the greatest level of durability. 41 | # However, it does not completely eliminate the risk of message loss 42 | # because the number of in sync replicas may, in rare cases, shrink to 1. 43 | # If you want to ensure that some minimum number of replicas (typically a 44 | # majority) receive a write, then you must set the topic-level 45 | # min.insync.replicas setting. Please read the Replication section of the 46 | # design documentation for a more in-depth discussion. 47 | request.required.acks={{ kafka_producer.request_required_acks }} 48 | {% endif %} 49 | 50 | {% if kafka_producer.request_timeout_ms %} 51 | # The amount of time the broker will wait trying to meet the 52 | # request.required.acks requirement before sending back an error to the client. 53 | request.timeout.ms={{ kafka_producer.request_timeout_ms }} 54 | {% endif %} 55 | 56 | {% if kafka_producer.type %} 57 | # specifies whether the messages are sent asynchronously (async) or synchronously (sync) 58 | producer.type={{ kafka_producer.type }} 59 | {% endif %} 60 | 61 | {% if kafka_producer.serializer_class %} 62 | # message encoder 63 | serializer.class={{ kafka_producer.serializer_class }} 64 | {% endif %} 65 | 66 | {% if kafka_producer.key_serializer_class %} 67 | # The serializer class for keys (defaults to the same as for messages if nothing 68 | # is given). 69 | key.serializer.class={{ kafka_producer.key_serializer_class }} 70 | {% endif %} 71 | 72 | {% if kafka_producer.partitioner_class %} 73 | # name of the partitioner class for partitioning events; default partition spreads data randomly 74 | partitioner.class={{ kafka_producer.partitioner_class }} 75 | {% endif %} 76 | 77 | {% if kafka_producer.compression_codec %} 78 | # specify the compression codec for all data generated: none, gzip, snappy, lz4. 79 | # the old config values work as well: 0, 1, 2, 3 for none, gzip, snappy, lz4, respectively 80 | compression.codec={{ kafka_producer.compression_codec }} 81 | {% endif %} 82 | 83 | {% if kafka_producer.compressed_topics %} 84 | # allow topic level compression 85 | compressed.topics={{ kafka_producer.compressed_topics }} 86 | {% endif %} 87 | 88 | {% if kafka_producer.compressed_topics %} 89 | # This property will cause the producer to automatically retry a failed send 90 | # request. This property specifies the number of retries when such failures 91 | # occur. Note that setting a non-zero value here can lead to duplicates in the 92 | # case of network errors that cause a message to be sent but the acknowledgement 93 | # to be lost. 94 | message.send.max.retries={{ kafka_producer.message_send_max_retries }} 95 | {% endif %} 96 | 97 | {% if kafka_producer.retry_backoff_ms %} 98 | # Before each retry, the producer refreshes the metadata of relevant topics to 99 | # see if a new leader has been elected. Since leader election takes a bit of 100 | # time, this property specifies the amount of time that the producer waits 101 | # before refreshing the metadata. 102 | retry.backoff.ms={{ kafka_producer.retry_backoff_ms }} 103 | {% endif %} 104 | 105 | {% if kafka_producer.topic_metadata_refresh_interval_ms %} 106 | # The producer generally refreshes the topic metadata from brokers when there is 107 | # a failure (partition missing, leader not available...). It will also poll 108 | # regularly (default: every 10min so 600000ms). If you set this to a negative 109 | # value, metadata will only get refreshed on failure. If you set this to zero, 110 | # the metadata will get refreshed after each message sent (not recommended). 111 | # Important note: the refresh happen only AFTER the message is sent, so if the 112 | # producer never sends a message the metadata is never refreshed 113 | topic.metadata.refresh.interval.ms={{ kafka_producer.topic_metadata_refresh_interval_ms }} 114 | {% endif %} 115 | 116 | {% if kafka_producer.send_buffer_bytes %} 117 | # Socket write buffer size 118 | send.buffer.bytes={{ prodcuer.send_buffer_bytes }} 119 | {% endif %} 120 | 121 | ############################# Async Producer ############################# 122 | {% if kafka_producer.queue_buffering_max_ms %} 123 | # maximum time, in milliseconds, for buffering data on the producer queue 124 | queue.buffering.max.ms={{ kafka_producer.queue_buffering_max_ms }} 125 | {% endif %} 126 | 127 | {% if kafka_producer.queue_buffering_max_messages %} 128 | # the maximum size of the blocking queue for buffering on the producer 129 | queue.buffering.max.messages={{ kafka_producer.queue_buffering_max_messages }} 130 | {% endif %} 131 | 132 | {% if kafka_producer.queue_enqueue_timeout_ms %} 133 | # Timeout for event enqueue: 134 | # 0: events will be enqueued immediately or dropped if the queue is full 135 | # -ve: enqueue will block indefinitely if the queue is full 136 | # +ve: enqueue will block up to this many milliseconds if the queue is full 137 | queue.enqueue.timeout.ms={{ kafka_producer.queue_enqueue_timeout_ms }} 138 | {% endif %} 139 | 140 | {% if kafka_producer.batch_num_messages %} 141 | # the number of messages batched at the producer 142 | batch.num.messages={{ kafka_producer.batch_num_messages }} 143 | {% endif %} 144 | 145 | {% endif %} 146 | -------------------------------------------------------------------------------- /templates/usr/local/kafka/config/server.properties.j2: -------------------------------------------------------------------------------- 1 | # server.properties.j2 2 | 3 | # default: -1 4 | # Each broker is uniquely identified by a non-negative integer id. This id serves as the broker's "name" and allows the broker to be moved to a different host/port without confusing consumers. You can choose any number you like so long as it is unique. 5 | {% if kafka_broker_id is defined %} 6 | broker.id={{ kafka_broker_id }} 7 | {% endif %} 8 | 9 | {% if kafka_reserved_broker_max_id is defined %} 10 | # default: 1000 11 | # Max number that can be used for a broker.id 12 | reserved.broker.max.id={{ kafka_reserved_broker_max_id }} 13 | {% endif %} 14 | 15 | # default: null 16 | # Specifies the ZooKeeper connection string in the form hostname:port, where hostname and port are the host and port for a node in your ZooKeeper cluster. To allow connecting through other ZooKeeper nodes when that host is down you can also specify multiple hosts in the form hostname1:port1,hostname2:port2,hostname3:port3. ZooKeeper also allows you to add a "chroot" path which will make all kafka data for this cluster appear under a particular path. This is a way to setup multiple Kafka clusters or other applications on the same ZooKeeper cluster. To do this give a connection string in the form hostname1:port1,hostname2:port2,hostname3:port3/chroot/path which would put all this cluster's data under the path /chroot/path. Note that you must create this path yourself prior to starting the broker and consumers must use the same connection string. 17 | zookeeper.connect={{ kafka_zookeeper_connection_string }} 18 | 19 | {% if kafka_server %} 20 | 21 | {% if kafka_server.log_dirs %} 22 | # default: /tmp/kafka-logs 23 | # A comma-separated list of one or more directories in which Kafka data is stored. Each new partition that is created will be placed in the directory which currently has the fewest partitions. 24 | log.dirs={{ kafka_server.log_dirs }} 25 | {% endif %} 26 | 27 | {% if kafka_server.port %} 28 | # default: 9092 29 | # The port on which the server accepts client connections. 30 | # 31 | # Kafka 0.10.0 32 | # DEPRECATED: only used when `listeners` is not set. Use `listeners` instead. the port to listen and accept connections on 33 | port={{ kafka_server.port }} 34 | {% endif %} 35 | 36 | {% if kafka_server.message_max_bytes %} 37 | # default: 1000000 38 | # The maximum size of a message that the server can receive. It is important that this property be in sync with the maximum fetch size your consumers use or else an unruly producer will be able to publish messages too large for consumers to consume. 39 | message.max.bytes={{ kafka_server.message_max_bytes }} 40 | {% endif %} 41 | 42 | {% if kafka_server.num_network_threads %} 43 | # The number of network threads that the server uses for handling network requests. You probably don't need to change this. 44 | num.network.threads={{ kafka_server.num_network_threads }} 45 | {% endif %} 46 | 47 | {% if kafka_server.num_io_threads %} 48 | # The number of I/O threads that the server uses for executing requests. You should have at least as many threads as you have disks. 49 | num.io.threads={{ kafka_server.num_io_threads }} 50 | {% endif %} 51 | 52 | {% if kafka_server.background_threads %} 53 | # default: 10 54 | # The number of threads to use for various background processing tasks such as file deletion. You should not need to change this. 55 | background.threads={{ kafka_server.background_threads }} 56 | {% endif %} 57 | 58 | {% if kafka_server.queued_max_requests %} 59 | # default: 500 60 | # The number of requests that can be queued up for processing by the I/O threads before the network threads stop reading in new requests. 61 | queued.max.requests={{ kafka_server.queued_max_requests }} 62 | {% endif %} 63 | 64 | {% if kafka_server.listeners %} 65 | # default: null 66 | # Listener List - Comma-separated list of URIs we will listen on and their protocols. Specify hostname as 0.0.0.0 to bind to all interfaces. Leave hostname empty to bind to default interface. Examples of legal listener lists: PLAINTEXT://myhost:9092,TRACE://:9091 PLAINTEXT://0.0.0.0:9092, TRACE://localhost:9093 67 | listeners={{ kafka_server.listeners }} 68 | {% endif %} 69 | 70 | {% if kafka_server.advertised_listeners %} 71 | # default: null 72 | # Listeners to publish to ZooKeeper for clients to use, if different than the listeners above. In IaaS environments, this may need to be different from the interface to which the broker binds. If this is not set, the value for `listeners` will be used. 73 | advertised.listeners={{ kafka_server.advertised_listeners }} 74 | {% endif %} 75 | 76 | {% if kafka_server.host_name %} 77 | # default: null 78 | # Hostname of broker. If this is set, it will only bind to this address. If this is not set, it will bind to all interfaces, and publish one to ZK. 79 | # 80 | # Kafka 0.10.0 81 | # DEPRECATED: only used when `listeners` is not set. Use `listeners` instead. hostname of broker. If this is set, it will only bind to this address. If this is not set, it will bind to all interfaces 82 | host.name={{ kafka_server.host_name }} 83 | {% endif %} 84 | 85 | {% if kafka_server.advertised_host_name %} 86 | # default: null 87 | # If this is set this is the hostname that will be given out to producers, consumers, and other brokers to connect to. 88 | # 89 | # Kafka 0.10.0 90 | # DEPRECATED: only used when `advertised.listeners` or `listeners` are not set. Use `advertised.listeners` instead. Hostname to publish to ZooKeeper for clients to use. In IaaS environments, this may need to be different from the interface to which the broker binds. If this is not set, it will use the value for `host.name` if configured. Otherwise it will use the value returned from java.net.InetAddress.getCanonicalHostName(). 91 | advertised.host.name={{ kafka_server.advertised_host_name }} 92 | {% endif %} 93 | 94 | {% if kafka_server.advertised_port %} 95 | # default: null 96 | # The port to give out to producers, consumers, and other brokers to use in establishing connections. This only needs to be set if this port is different from the port the server should bind to. 97 | # 98 | # Kafka 0.10.0 99 | # DEPRECATED: only used when `advertised.listeners` or `listeners` are not set. Use `advertised.listeners` instead. The port to publish to ZooKeeper for clients to use. In IaaS environments, this may need to be different from the port to which the broker binds. If this is not set, it will publish the same port that the broker binds to. 100 | advertised.port={{ kafka_server.advertised_port }} 101 | {% endif %} 102 | 103 | {% if kafka_server.socket_send_buffer_bytes %} 104 | # default: 100 * 1024 105 | # The SO_SNDBUFF buffer the server prefers for socket connections. 106 | socket.send.buffer.bytes={{ kafka_server.socket_send_buffer_bytes }} 107 | {% endif %} 108 | 109 | {% if kafka_server.socket_receive_buffer_bytes %} 110 | # default: 100 * 1024 111 | # The SO_RCVBUFF buffer the server prefers for socket connections. 112 | socket.receive.buffer.bytes={{ kafka_server.socket_receive_buffer_bytes }} 113 | {% endif %} 114 | 115 | {% if kafka_server.socket_request_max_bytes %} 116 | # default: 100 * 1024 * 1024 117 | # The maximum request size the server will allow. This prevents the server from running out of memory and should be smaller than the Java heap size. 118 | socket.request.max.bytes={{ kafka_server.socket_request_max_bytes }} 119 | {% endif %} 120 | 121 | {% if kafka_server.num_partitions %} 122 | # The default number of partitions per topic if a partition count isn't given at topic creation time. 123 | num.partitions={{ kafka_server.num_partitions }} 124 | {% endif %} 125 | 126 | {% if kafka_server.log_segment_bytes %} 127 | # default: 1024 * 1024 * 1024 128 | # The log for a topic partition is stored as a directory of segment files. This setting controls the size to which a segment file will grow before a new segment is rolled over in the log. This setting can be overridden on a per-topic basis (see the per-topic configuration section). 129 | log.segment.bytes={{ kafka_server.log_segment_bytes }} 130 | {% endif %} 131 | 132 | {% if kafka_server.log_roll_ms %} 133 | # default: 24 * 7 hours 134 | # This setting will force Kafka to roll a new log segment even if the log.segment.bytes size has not been reached. This setting can be overridden on a per-topic basis (see the per-topic configuration section). 135 | log.roll.ms={{ kafka_server.log_roll_ms }} 136 | {% endif %} 137 | 138 | {% if kafka_server.log_roll_hours %} 139 | # default: 24 * 7 hours 140 | # This setting will force Kafka to roll a new log segment even if the log.segment.bytes size has not been reached. This setting can be overridden on a per-topic basis (see the per-topic configuration section). 141 | log.roll.hours={{ kafka_server.log_roll_hours }} 142 | {% endif %} 143 | 144 | {% if kafka_server.log_cleanup_policy %} 145 | # default: delete 146 | # This can take either the value delete or compact. If delete is set, log segments will be deleted when they reach the size or time limits set. If compact is set log compaction will be used to clean out obsolete records. This setting can be overridden on a per-topic basis (see the per-topic configuration section). 147 | log.cleanup.policy={{ kafka_server.log_cleanup_policy }} 148 | {% endif %} 149 | 150 | {% if kafka_server.log_retention_ms %} 151 | # default: 7 days 152 | # The amount of time to keep a log segment before it is deleted, i.e. the default data retention window for all topics. Note that if both log.retention.minutes and log.retention.bytes are both set we delete a segment when either limit is exceeded. This setting can be overridden on a per-topic basis (see the per-topic configuration section). 153 | log.retention.ms={{ kafka_server.log_retention_ms }} 154 | {% endif %} 155 | 156 | {% if kafka_server.log_retention_minutes %} 157 | # default: 7 days 158 | # The amount of time to keep a log segment before it is deleted, i.e. the default data retention window for all topics. Note that if both log.retention.minutes and log.retention.bytes are both set we delete a segment when either limit is exceeded. This setting can be overridden on a per-topic basis (see the per-topic configuration section). 159 | log.retention.minutes={{ kafka_server.log_retention_minutes }} 160 | {% endif %} 161 | 162 | {% if kafka_server.log_retention_hours %} 163 | # default: 7 days 164 | # The amount of time to keep a log segment before it is deleted, i.e. the default data retention window for all topics. Note that if both log.retention.minutes and log.retention.bytes are both set we delete a segment when either limit is exceeded. This setting can be overridden on a per-topic basis (see the per-topic configuration section). 165 | log.retention.hours={{ kafka_server.log_retention_hours }} 166 | {% endif %} 167 | 168 | {% if kafka_server.log_retention_bytes %} 169 | # default: -1 170 | # The amount of data to retain in the log for each topic-partitions. Note that this is the limit per-partition so multiply by the number of partitions to get the total data retained for the topic. Also note that if both log.retention.hours and log.retention.bytes are both set we delete a segment when either limit is exceeded. This setting can be overridden on a per-topic basis (see the per-topic configuration section). 171 | log.retention.bytes={{ kafka_server.log_retention_bytes }} 172 | {% endif %} 173 | 174 | {% if kafka_server.log_retention_check_interval_ms %} 175 | # default: 5 minutes 176 | # The period with which we check whether any log segment is eligible for deletion to meet the retention policies. 177 | log.retention.check.interval.ms={{ kafka_server.log_retention_check_interval_ms }} 178 | {% endif %} 179 | 180 | {% if kafka_server.log_cleaner_enable %} 181 | # default: false 182 | # This configuration must be set to true for log compaction to run. 183 | log.cleaner.enable={{ kafka_server.log_cleaner_enable }} 184 | {% endif %} 185 | 186 | {% if kafka_server.log_cleaner_threads %} 187 | # The number of threads to use for cleaning logs in log compaction. 188 | log.cleaner.threads={{ kafka_server.log_cleaner_threads }} 189 | {% endif %} 190 | 191 | {% if kafka_server.log_cleaner_io_max_bytes_per_second %} 192 | # default: Double.MaxValue 193 | # The maximum amount of I/O the log cleaner can do while performing log compaction. This setting allows setting a limit for the cleaner to avoid impacting live request serving. 194 | log.cleaner.io.max.bytes.per.second={{ kafka_server.log_cleaner_io_max_bytes_per_second }} 195 | {% endif %} 196 | 197 | {% if kafka_server.log_cleaner_dedupe_buffer_size %} 198 | # default: 500*1024*1024 199 | # The size of the buffer the log cleaner uses for indexing and deduplicating logs during cleaning. Larger is better provided you have sufficient memory. 200 | log.cleaner.dedupe.buffer.size={{ kafka_server.log_cleaner_dedupe_buffer_size }} 201 | {% endif %} 202 | 203 | {% if kafka_server.log_cleaner_io_buffer_size %} 204 | # default: 512*1024 205 | # The size of the I/O chunk used during log cleaning. You probably don't need to change this. 206 | log.cleaner.io.buffer.size={{ kafka_server.log_cleaner_io_buffer_size }} 207 | {% endif %} 208 | 209 | {% if kafka_server.log_cleaner_io_buffer_load_factor %} 210 | # default: 0.9 211 | # The load factor of the hash table used in log cleaning. You probably don't need to change this. 212 | log.cleaner.io.buffer.load.factor={{ kafka_server.log_cleaner_io_buffer_load_factor }} 213 | {% endif %} 214 | 215 | {% if kafka_server.log_cleaner_backoff_ms %} 216 | # default: 15000 217 | # The interval between checks to see if any logs need cleaning. 218 | log.cleaner.backoff.ms={{ kafka_server.log_cleaner_backoff_ms }} 219 | {% endif %} 220 | 221 | {% if kafka_server.log_cleaner_min_cleanable_ratio %} 222 | # default: 0.5 223 | # This configuration controls how frequently the log compactor will attempt to clean the log (assuming log compaction is enabled). By default we will avoid cleaning a log where more than 50% of the log has been compacted. This ratio bounds the maximum space wasted in the log by duplicates (at 50% at most 50% of the log could be duplicates). A higher ratio will mean fewer, more efficient cleanings but will mean more wasted space in the log. This setting can be overridden on a per-topic basis (see the per-topic configuration section). 224 | log.cleaner.min.cleanable.ratio={{ kafka_server.log_cleaner_min_cleanable_ratio }} 225 | {% endif %} 226 | 227 | {% if kafka_server.log_cleaner_delete_retention_ms %} 228 | # default: 1 day 229 | # The amount of time to retain delete tombstone markers for log compacted topics. This setting also gives a bound on the time in which a consumer must complete a read if they begin from offset 0 to ensure that they get a valid snapshot of the final stage (otherwise delete tombstones may be collected before they complete their scan). This setting can be overridden on a per-topic basis (see the per-topic configuration section). 230 | log.cleaner.delete.retention.ms={{ kafka_server.log_cleaner_delete_retention_ms }} 231 | {% endif %} 232 | 233 | {% if kafka_server.log_index_size_max_bytes %} 234 | # default: 10 * 1024 * 1024 235 | # The maximum size in bytes we allow for the offset index for each log segment. Note that we will always pre-allocate a sparse file with this much space and shrink it down when the log rolls. If the index fills up we will roll a new log segment even if we haven't reached the log.segment.bytes limit. This setting can be overridden on a per-topic basis (see the per-topic configuration section). 236 | log.index.size.max.bytes={{ kafka_server.log_index_size_max_bytes }} 237 | {% endif %} 238 | 239 | {% if kafka_server.log_index_interval_bytes %} 240 | # default: 4096 241 | # The byte interval at which we add an entry to the offset index. When executing a fetch request the server must do a linear scan for up to this many bytes to find the correct position in the log to begin and end the fetch. So setting this value to be larger will mean larger index files (and a bit more memory usage) but less scanning. However the server will never add more than one index entry per log append (even if more than log.index.interval worth of messages are appended). In general you probably don't need to mess with this value. 242 | log.index.interval.bytes={{ kafka_server.log_index_interval_bytes }} 243 | {% endif %} 244 | 245 | {% if kafka_server.log_flush_interval_messages %} 246 | # default: Long.MaxValue 247 | # The number of messages written to a log partition before we force an fsync on the log. Setting this lower will sync data to disk more often but will have a major impact on performance. We generally recommend that people make use of replication for durability rather than depending on single-server fsync, however this setting can be used to be extra certain. 248 | log.flush.interval.messages={{ kafka_server.log_flush_interval_messages }} 249 | {% endif %} 250 | 251 | {% if kafka_server.log_flush_scheduler_interval_ms %} 252 | # default: Long.MaxValue 253 | # The frequency in ms that the log flusher checks whether any log is eligible to be flushed to disk. 254 | log.flush.scheduler.interval.ms={{ kafka_server.log_flush_scheduler_interval_ms }} 255 | {% endif %} 256 | 257 | {% if kafka_server.log_flush_interval_ms %} 258 | # default: Long.MaxValue 259 | # The maximum time between fsync calls on the log. If used in conjuction with log.flush.interval.messages the log will be flushed when either criteria is met. 260 | log.flush.interval.ms={{ kafka_server.log_flush_interval_ms }} 261 | {% endif %} 262 | 263 | {% if kafka_server.log_delete_delay_ms %} 264 | # default: 60000 265 | # The period of time we hold log files around after they are removed from the in-memory segment index. This period of time allows any in-progress reads to complete uninterrupted without locking. You generally don't need to change this. 266 | log.delete.delay.ms={{ kafka_server.log_delete_delay_ms }} 267 | {% endif %} 268 | 269 | {% if kafka_server.log_flush_offset_checkpoint_interval_ms %} 270 | # default: 60000 271 | # The frequency with which we checkpoint the last flush point for logs for recovery. You should not need to change this. 272 | log.flush.offset.checkpoint.interval.ms={{ kafka_server.log_flush_offset_checkpoint_interval_ms }} 273 | {% endif %} 274 | 275 | {% if kafka_server.log_segment_delete_delay_ms %} 276 | # default: 60000 277 | # the amount of time to wait before deleting a file from the filesystem. 278 | log.segment.delete.delay.ms={{ kafka_server.log_segment_delete_delay_ms }} 279 | {% endif %} 280 | 281 | {% if kafka_server.auto_create_topics_enable %} 282 | # default: true 283 | # Enable auto creation of topic on the server. If this is set to true then attempts to produce data or fetch metadata for a non-existent topic will automatically create it with the default replication factor and number of partitions. 284 | auto.create.topics.enable={{ kafka_server.auto_create_topics_enable }} 285 | {% endif %} 286 | 287 | {% if kafka_server.controller_socket_timeout_ms %} 288 | # default: 30000 289 | # The socket timeout for commands from the partition management controller to the replicas. 290 | controller.socket.timeout.ms={{ kafka_server.controller_socket_timeout_ms }} 291 | {% endif %} 292 | 293 | {% if kafka_server.controller_message_queue_size %} 294 | # default: Int.MaxValue 295 | # The buffer size for controller-to-broker-channels 296 | controller.message.queue.size={{ kafka_server.controller_message_queue_size }} 297 | {% endif %} 298 | 299 | {% if kafka_server.default_replication_factor %} 300 | # The default replication factor for automatically created topics. 301 | default.replication.factor={{ kafka_server.default_replication_factor }} 302 | {% endif %} 303 | 304 | {% if kafka_server.replica_lag_time_max_ms %} 305 | # default: 10000 306 | # If a follower hasn't sent any fetch requests for this window of time, the leader will remove the follower from ISR (in-sync replicas) and treat it as dead. 307 | replica.lag.time.max.ms={{ kafka_server.replica_lag_time_max_ms }} 308 | {% endif %} 309 | 310 | {% if kafka_server.replica_lag_max_messages %} 311 | # default: 4000 312 | # If a replica falls more than this many messages behind the leader, the leader will remove the follower from ISR and treat it as dead. 313 | replica.lag.max.messages={{ kafka_server.replica_lag_max_messages }} 314 | {% endif %} 315 | 316 | {% if kafka_server.replica_socket_timeout_ms %} 317 | # default: 30 * 1000 318 | # The socket timeout for network requests to the leader for replicating data. 319 | replica.socket.timeout.ms={{ kafka_server.replica_socket_timeout_ms }} 320 | {% endif %} 321 | 322 | {% if kafka_server.replica_socket_receive_buffer_bytes %} 323 | # default: 64 * 1024 324 | # The socket receive buffer for network requests to the leader for replicating data. 325 | replica.socket.receive.buffer.bytes={{ kafka_server.replica_socket_receive_buffer_bytes }} 326 | {% endif %} 327 | 328 | {% if kafka_server.replica_fetch_max_bytes %} 329 | # default: 1024 * 1024 330 | # The number of byes of messages to attempt to fetch for each partition in the fetch requests the replicas send to the leader. 331 | replica.fetch.max.bytes={{ kafka_server.replica_fetch_max_bytes }} 332 | {% endif %} 333 | 334 | {% if kafka_server.replica_fetch_wait_max_ms %} 335 | # default: 500 336 | # The maximum amount of time to wait time for data to arrive on the leader in the fetch requests sent by the replicas to the leader. 337 | replica.fetch.wait.max.ms={{ kafka_server.replica_fetch_wait_max_ms }} 338 | {% endif %} 339 | 340 | {% if kafka_server.replica_fetch_min_bytes %} 341 | # Minimum bytes expected for each fetch response for the fetch requests from the replica to the leader. If not enough bytes, wait up to replica.fetch.wait.max.ms for this many bytes to arrive. 342 | replica.fetch.min.bytes={{ kafka_server.replica_fetch_min_bytes }} 343 | {% endif %} 344 | 345 | {% if kafka_server.num_replica_fetchers %} 346 | # Number of threads used to replicate messages from leaders. Increasing this value can increase the degree of I/O parallelism in the follower broker. 347 | num.replica.fetchers={{ kafka_server.num_replica_fetchers }} 348 | {% endif %} 349 | 350 | {% if kafka_server.replica_high_watermark_checkpoint_interval_ms %} 351 | # default: 5000 352 | # The frequency with which each replica saves its high watermark to disk to handle recovery. 353 | replica.high.watermark.checkpoint.interval.ms={{ kafka_server.replica_high_watermark_checkpoint_interval_ms }} 354 | {% endif %} 355 | 356 | {% if kafka_server.fetch_purgatory_purge_interval_requests %} 357 | # default: 1000 358 | # The purge interval (in number of requests) of the fetch request purgatory. 359 | fetch.purgatory.purge.interval.requests={{ kafka_server.fetch_purgatory_purge_interval_requests }} 360 | {% endif %} 361 | 362 | {% if kafka_server.producer_purgatory_purge_interval_requests %} 363 | # default: 1000 364 | # The purge interval (in number of requests) of the producer request purgatory. 365 | producer.purgatory.purge.interval.requests={{ kafka_server.producer_purgatory_purge_interval_requests }} 366 | {% endif %} 367 | 368 | {% if kafka_server.zookeeper_session_timeout_ms %} 369 | # default: 6000 370 | # ZooKeeper session timeout. If the server fails to heartbeat to ZooKeeper within this period of time it is considered dead. If you set this too low the server may be falsely considered dead; if you set it too high it may take too long to recognize a truly dead server. 371 | zookeeper.session.timeout.ms={{ kafka_server.zookeeper_session_timeout_ms }} 372 | {% endif %} 373 | 374 | {% if kafka_server.zookeeper_connection_timeout_ms %} 375 | # default: 6000 376 | # The maximum amount of time that the client waits to establish a connection to zookeeper. 377 | zookeeper.connection.timeout.ms={{ kafka_server.zookeeper_connection_timeout_ms }} 378 | {% endif %} 379 | 380 | {% if kafka_server.zookeeper_sync_time_ms %} 381 | # default: 2000 382 | # How far a ZK follower can be behind a ZK leader. 383 | zookeeper.sync.time.ms={{ kafka_server.zookeeper_sync_time_ms }} 384 | {% endif %} 385 | 386 | {% if kafka_server.controlled_shutdown_enable %} 387 | # default: true 388 | # Enable controlled shutdown of the broker. If enabled, the broker will move all leaders on it to some other brokers before shutting itself down. This reduces the unavailability window during shutdown. 389 | controlled.shutdown.enable={{ kafka_server.controlled_shutdown_enable }} 390 | {% endif %} 391 | 392 | {% if kafka_server.controlled_shutdown_max_retries %} 393 | # Number of retries to complete the controlled shutdown successfully before executing an unclean shutdown. 394 | controlled.shutdown.max.retries={{ kafka_server.controlled_shutdown_max_retries }} 395 | {% endif %} 396 | 397 | {% if kafka_server.controlled_shutdown_retry_backoff_ms %} 398 | # default: 5000 399 | # Backoff time between shutdown retries. 400 | controlled.shutdown.retry.backoff.ms={{ kafka_server.controlled_shutdown_retry_backoff_ms }} 401 | {% endif %} 402 | 403 | {% if kafka_server.auto_leader_rebalance_enable %} 404 | # default: true 405 | # If this is enabled the controller will automatically try to balance leadership for partitions among the brokers by periodically returning leadership to the "preferred" replica for each partition if it is available. 406 | auto.leader.rebalance.enable={{ kafka_server.auto_leader_rebalance_enable }} 407 | {% endif %} 408 | 409 | {% if kafka_server.leader_imbalance_per_broker_percentage %} 410 | # default: 10 411 | # The percentage of leader imbalance allowed per broker. The controller will rebalance leadership if this ratio goes above the configured value per broker. 412 | leader.imbalance.per.broker.percentage={{ kafka_server.leader_imbalance_per_broker_percentage }} 413 | {% endif %} 414 | 415 | {% if kafka_server.leader_imbalance_check_interval_seconds %} 416 | # default: 300 417 | # The frequency with which to check for leader imbalance. 418 | leader.imbalance.check.interval.seconds={{ kafka_server.leader_imbalance_check_interval_seconds }} 419 | {% endif %} 420 | 421 | {% if kafka_server.offset_metadata_max_bytes %} 422 | # default: 4096 423 | # The maximum amount of metadata to allow clients to save with their offsets. 424 | offset.metadata.max.bytes={{ kafka_server.offset_metadata_max_bytes }} 425 | {% endif %} 426 | 427 | {% if kafka_server.max_connections_per_ip %} 428 | # default: Int.MaxValue 429 | # The maximum number of connections that a broker allows from each ip address. 430 | max.connections.per.ip={{ kafka_server.max_connections_per_ip }} 431 | {% endif %} 432 | 433 | {% if kafka_server.max_connections_per_ip_overrides %} 434 | # Per-ip or hostname overrides to the default maximum number of connections. 435 | max.connections.per.ip.overrides={{ kafka_server.max_connections_per_ip_overrides }} 436 | {% endif %} 437 | 438 | {% if kafka_server.connections_max_idle_ms %} 439 | # default: 600000 440 | # Idle connections timeout: the server socket processor threads close the connections that idle more than this. 441 | connections.max.idle.ms={{ kafka_server.connections_max_idle_ms }} 442 | {% endif %} 443 | 444 | {% if kafka_server.log_roll_jitter_ms %} 445 | # The maximum jitter to subtract from logRollTimeMillis. 446 | log.roll.jitter.ms={{ kafka_server.log_roll_jitter_ms }} 447 | {% endif %} 448 | 449 | {% if kafka_server.log_roll_jitter_hours %} 450 | # The maximum jitter to subtract from logRollTimeMillis. 451 | log.roll.jitter.hours={{ kafka_server.log_roll_jitter_hours }} 452 | {% endif %} 453 | 454 | {% if kafka_server.num_recovery_threads_per_data_dir %} 455 | # The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. 456 | num.recovery.threads.per.data.dir={{ kafka_server.num_recovery_threads_per_data_dir }} 457 | {% endif %} 458 | 459 | {% if kafka_server.unclean_leader_election_enable %} 460 | # default: true 461 | # Indicates whether to enable replicas not in the ISR set to be elected as leader as a last resort, even though doing so may result in data loss. 462 | unclean.leader.election.enable={{ kafka_server.unclean_leader_election_enable }} 463 | {% endif %} 464 | 465 | {% if kafka_server.delete_topic_enable %} 466 | # default: false 467 | # Enable delete topic. 468 | delete.topic.enable={{ kafka_server.delete_topic_enable }} 469 | {% endif %} 470 | 471 | {% if kafka_server.offsets_topic_num_partitions %} 472 | # default: 50 473 | # The number of partitions for the offset commit topic. Since changing this after deployment is currently unsupported, we recommend using a higher setting for production (e.g., 100-200). 474 | offsets.topic.num.partitions={{ kafka_server.offsets_topic_num_partitions }} 475 | {% endif %} 476 | 477 | {% if kafka_server.offsets_topic_retention_minutes %} 478 | # default: 1440 479 | # Offsets that are older than this age will be marked for deletion. The actual purge will occur when the log cleaner compacts the offsets topic. 480 | offsets.topic.retention.minutes={{ kafka_server.offsets_topic_retention_minutes }} 481 | {% endif %} 482 | 483 | {% if kafka_server.offsets_retention_check_interval_ms %} 484 | # default: 600000 485 | # The frequency at which the offset manager checks for stale offsets. 486 | offsets.retention.check.interval.ms={{ kafka_server.offsets_retention_check_interval_ms }} 487 | {% endif %} 488 | 489 | {% if kafka_server.offsets_topic_replication_factor %} 490 | # The replication factor for the offset commit topic. A higher setting (e.g., three or four) is recommended in order to ensure higher availability. If the offsets topic is created when fewer brokers than the replication factor then the offsets topic will be created with fewer replicas. 491 | offsets.topic.replication.factor={{ kafka_server.offsets_topic_replication_factor }} 492 | {% endif %} 493 | 494 | {% if kafka_server.offsets_topic_segment_bytes %} 495 | # default: 104857600 496 | # Segment size for the offsets topic. Since it uses a compacted topic, this should be kept relatively low in order to facilitate faster log compaction and loads. 497 | offsets.topic.segment.bytes={{ kafka_server.offsets_topic_segment_bytes }} 498 | {% endif %} 499 | 500 | {% if kafka_server.offsets_load_buffer_size %} 501 | # default: 5242880 502 | # An offset load occurs when a broker becomes the offset manager for a set of consumer groups (i.e., when it becomes a leader for an offsets topic partition). This setting corresponds to the batch size (in bytes) to use when reading from the offsets segments when loading offsets into the offset manager's cache. 503 | offsets.load.buffer.size={{ kafka_server.offsets_load_buffer_size }} 504 | {% endif %} 505 | 506 | {% if kafka_server.offsets_commit_required_acks %} 507 | # default: -1 508 | # The number of acknowledgements that are required before the offset commit can be accepted. This is similar to the producer's acknowledgement setting. In general, the default should not be overridden. 509 | offsets.commit.required.acks={{ kafka_server.offsets_commit_required_acks }} 510 | {% endif %} 511 | 512 | {% if kafka_server.offsets_commit_timeout_ms %} 513 | # default: 5000 514 | # The offset commit will be delayed until this timeout or the required number of replicas have received the offset commit. This is similar to the producer request timeout. 515 | offsets.commit.timeout.ms={{ kafka_server.offsets_commit_timeout_ms }} 516 | {% endif %} 517 | 518 | {% endif %} 519 | 520 | {% for key, value in kafka_server.misc.iteritems() -%} 521 | {{key}}={{value}} 522 | {% endfor -%} 523 | --------------------------------------------------------------------------------