├── .ansible-lint ├── .gitattributes ├── .github └── workflows │ ├── linter.yaml │ └── molecule.yaml ├── .gitignore ├── .markdownlint.json ├── .prettierrc.yaml ├── .yamllint.yaml ├── LICENSE ├── README.md ├── defaults └── main │ ├── 001-kafka.yml │ └── 002-log4j.yml ├── handlers └── main.yaml ├── meta └── main.yml ├── molecule └── default │ ├── converge.yml │ ├── molecule.yml │ ├── prepare.yml │ ├── requirements.yml │ └── verify.yml ├── tasks └── main.yaml ├── templates ├── connect-distributed.properties.j2 ├── connect-standalone.properties.j2 ├── consumer.properties.j2 ├── kafka.initd.j2 ├── kafka.service.j2 ├── log4j.properties.j2 ├── producer.properties.j2 ├── server.properties.j2 └── zookeeper.properties.j2 └── vars ├── Debian.yml └── RedHat.yml /.ansible-lint: -------------------------------------------------------------------------------- 1 | exclude_paths: 2 | - ./molecule-venv/ 3 | - ./tests/roles/ 4 | 5 | mock_roles: 6 | - sleighzy.zookeeper 7 | 8 | skip_list: 9 | - '106' # Role name does not match ``^[a-z][a-z0-9_]+$`` pattern 10 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.yaml linguist-detectable 2 | *.yml linguist-detectable 3 | -------------------------------------------------------------------------------- /.github/workflows/linter.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Lint Code Base 3 | 4 | # 5 | # Documentation: 6 | # https://help.github.com/en/articles/workflow-syntax-for-github-actions 7 | # 8 | on: 9 | push: 10 | branches: [master] 11 | pull_request: 12 | branches: [master] 13 | workflow_dispatch: 14 | 15 | jobs: 16 | build: 17 | name: Linter 18 | runs-on: ubuntu-latest 19 | 20 | steps: 21 | - name: Checkout Code 22 | uses: actions/checkout@v4 23 | with: 24 | fetch-depth: 0 25 | 26 | - name: Lint Code Base 27 | uses: github/super-linter/slim@v4 28 | env: 29 | DEFAULT_BRANCH: master 30 | FILTER_REGEX_EXCLUDE: .*(tests/|Dockerfile.j2).* 31 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 32 | LINTER_RULES_PATH: / 33 | MARKDOWN_CONFIG_FILE: .markdownlint.json 34 | 35 | - name: Ansible Lint 36 | uses: ansible/ansible-lint@v6 37 | -------------------------------------------------------------------------------- /.github/workflows/molecule.yaml: -------------------------------------------------------------------------------- 1 | name: Molecule 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | - release/v* 8 | pull_request: 9 | branches: 10 | - master 11 | workflow_dispatch: 12 | 13 | defaults: 14 | run: 15 | working-directory: 'sleighzy.kafka' 16 | 17 | jobs: 18 | molecule: 19 | name: Molecule 20 | runs-on: ubuntu-latest 21 | 22 | steps: 23 | - name: Check out the codebase. 24 | uses: actions/checkout@v4 25 | with: 26 | path: 'sleighzy.kafka' 27 | 28 | - name: Set up Python 3. 29 | uses: actions/setup-python@v4 30 | with: 31 | python-version: '3.x' 32 | 33 | - name: Install test dependencies. 34 | run: pip3 install ansible docker "molecule-plugins[docker]" 35 | 36 | - name: Run Molecule tests. 37 | run: molecule test 38 | env: 39 | PY_COLORS: '1' 40 | ANSIBLE_FORCE_COLOR: '1' 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # General file ignores 2 | .DS_Store 3 | .vscode 4 | super-linter.log 5 | 6 | # Symlink to current repository to enable Ansible to find the role 7 | # using its expected full name for Molecule tests 8 | .cache/ 9 | 10 | # Python virtual env for Molecule testing 11 | molecule-venv/ 12 | -------------------------------------------------------------------------------- /.markdownlint.json: -------------------------------------------------------------------------------- 1 | { 2 | "line-length": { 3 | "code_blocks": false, 4 | "tables": false 5 | }, 6 | "list-marker-space": { 7 | "ol_single": 2, 8 | "ol_multi": 2 9 | }, 10 | "no-inline-html": { 11 | "allowed_elements": ["br"] 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /.prettierrc.yaml: -------------------------------------------------------------------------------- 1 | proseWrap: 'always' 2 | singleQuote: true 3 | trailingComma: 'all' 4 | overrides: 5 | - files: 6 | - '*.yml' 7 | - '*.yaml' 8 | options: 9 | proseWrap: 'never' 10 | -------------------------------------------------------------------------------- /.yamllint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Based on ansible-lint config 3 | extends: default 4 | 5 | yaml-files: 6 | - '*.yaml' 7 | - '*.yml' 8 | 9 | ignore: | 10 | molecule-venv/ 11 | tests/roles/ 12 | 13 | rules: 14 | braces: 15 | max-spaces-inside: 1 16 | level: error 17 | brackets: 18 | max-spaces-inside: 1 19 | level: error 20 | colons: 21 | max-spaces-after: -1 22 | level: error 23 | commas: 24 | max-spaces-after: -1 25 | level: error 26 | comments: disable 27 | comments-indentation: disable 28 | document-start: disable 29 | empty-lines: 30 | max: 3 31 | level: error 32 | hyphens: 33 | level: error 34 | indentation: disable 35 | key-duplicates: enable 36 | line-length: disable 37 | new-line-at-end-of-file: disable 38 | new-lines: 39 | type: unix 40 | trailing-spaces: disable 41 | truthy: disable 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Simon Leigh 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Apache Kafka 2 | 3 | ![Lint Code Base] ![Molecule] 4 | 5 | Ansible role to install and configure [Apache Kafka] 3.8 6 | 7 | [Apache Kafka] is a distributed event streaming platform using publish-subscribe 8 | topics. Applications and streaming components can produce and consume messages 9 | by subscribing to these topics. Kafka is extremely fast, handling megabytes of 10 | reads and writes per second from thousands of clients. Messages are persisted 11 | and replicated to prevent data loss. Data streams are partitioned and can be 12 | elastically scaled with no downtime. 13 | 14 | ## WARNING 15 | 16 | This Ansible role does not handle the migration process of upgrading from older 17 | versions of Kafka. Please ensure that you read the upgrade documentation and 18 | update the relevant configuration files before running this role. 19 | 20 | 21 | 22 | For example, depending on how you upgrade, the `server.properties` file may need 23 | the following properties added to reflect your current version prior to running 24 | this Ansible playbook: 25 | 26 | - `inter.broker.protocol.version` 27 | - `log.message.format.version` 28 | 29 | ## Supported Platforms 30 | 31 | - RedHat 6 32 | - RedHat 7 33 | - RedHat 8 34 | - Debian 10.x 35 | - Ubuntu 18.04.x 36 | - Ubuntu 20.04.x 37 | 38 | ## Requirements 39 | 40 | - [Apache ZooKeeper] 41 | - Java 8 (deprecated) / 11 / 17 42 | 43 | The below Apache ZooKeeper role from Ansible Galaxy can be used if one is 44 | needed. 45 | 46 | ```sh 47 | ansible-galaxy install sleighzy.zookeeper 48 | ``` 49 | 50 | Ansible 2.9.16 or 2.10.4 are the minimum required versions to workaround an 51 | issue with certain kernels that have broken the `systemd` status check. The 52 | error message "`Service is in unknown state`" will be output when attempting to 53 | start the service via the Ansible role and the task will fail. The service will 54 | start as expected if the `systemctl start` command is run on the physical host. 55 | See for more information. 56 | 57 | ## Role Variables 58 | 59 | | Variable | Default | Comments | 60 | | ---------------------------------------------- | ------------------------------------ | ---------------------------------------------------------------- | 61 | | kafka_download_base_url | | | 62 | | kafka_download_validate_certs | yes | | 63 | | kafka_version | 3.8.1 | | 64 | | kafka_scala_version | 2.13 | | 65 | | kafka_create_user_group | true | | 66 | | kafka_user | kafka | | 67 | | kafka_group | kafka | | 68 | | kafka_root_dir | /opt | | 69 | | kafka_dir | {{ kafka_root_dir }}/kafka | | 70 | | kafka_start | yes | | 71 | | kafka_restart | yes | | 72 | | kafka_log_dir | /var/log/kafka | | 73 | | kafka_broker_id | 0 | | 74 | | kafka_java_heap | -Xms1G -Xmx1G | | 75 | | kafka_background_threads | 10 | | 76 | | kafka_listeners | PLAINTEXT://:9092 | | 77 | | kafka_num_network_threads | 3 | | 78 | | kafka_num_io_threads | 8 | | 79 | | kafka_num_replica_fetchers | 1 | | 80 | | kafka_socket_send_buffer_bytes | 102400 | | 81 | | kafka_socket_receive_buffer_bytes | 102400 | | 82 | | kafka_socket_request_max_bytes | 104857600 | | 83 | | kafka_replica_socket_receive_buffer_bytes | 65536 | | 84 | | kafka_data_log_dirs | /var/lib/kafka/logs | | 85 | | kafka_num_partitions | 1 | | 86 | | kafka_num_recovery_threads_per_data_dir | 1 | | 87 | | kafka_log_cleaner_threads | 1 | | 88 | | kafka_offsets_topic_replication_factor | 1 | | 89 | | kafka_transaction_state_log_replication_factor | 1 | | 90 | | kafka_transaction_state_log_min_isr | 1 | | 91 | | kafka_log_retention_hours | 168 | | 92 | | kafka_log_segment_bytes | 1073741824 | | 93 | | kafka_log_retention_check_interval_ms | 300000 | | 94 | | kafka_auto_create_topics_enable | false | | 95 | | kafka_delete_topic_enable | true | | 96 | | kafka_default_replication_factor | 1 | | 97 | | kafka_group_initial_rebalance_delay_ms | 0 | | 98 | | kafka_zookeeper_connect | localhost:2181 | | 99 | | kafka_zookeeper_connection_timeout | 6000 | | 100 | | kafka_bootstrap_servers | localhost:9092 | | 101 | | kafka_consumer_group_id | kafka-consumer-group | | 102 | | kafka_server_config_params | | General dictionary that will be templated into server.properties | 103 | 104 | See [log4j.yml](./defaults/main/002-log4j.yml) for detailed 105 | log4j-related available variables. 106 | 107 | ## Starting and Stopping Kafka services using systemd 108 | 109 | - The Kafka service can be started via: `systemctl start kafka` 110 | - The Kafka service can be stopped via: `systemctl stop kafka` 111 | 112 | ## Starting and Stopping Kafka services using initd 113 | 114 | - The Kafka service can be started via: `service kafka start` 115 | - The Kafka service can be stopped via: `service kafka stop` 116 | 117 | ## Default Properties 118 | 119 | | Property | Value | 120 | | ------------------------------ | -------------------- | 121 | | ZooKeeper connection | localhost:2181 | 122 | | Kafka bootstrap servers | localhost:9092 | 123 | | Kafka consumer group ID | kafka-consumer-group | 124 | | Kafka broker ID | 0 | 125 | | Number of partitions | 1 | 126 | | Data log file retention period | 168 hours | 127 | | Enable auto topic creation | false | 128 | | Enable topic deletion | true | 129 | 130 | ### Ports 131 | 132 | | Port | Description | 133 | | ---- | ------------------- | 134 | | 9092 | Kafka listener port | 135 | 136 | ### Directories and Files 137 | 138 | | Directory / File | | 139 | | ------------------------------------------------------------ | --------------------------------------- | 140 | | Kafka installation directory (symlink to installed version) | `/opt/kafka` | 141 | | Kafka configuration directory (symlink to /opt/kafka/config) | `/etc/kafka` | 142 | | Directory to store data files | `/var/lib/kafka/logs` | 143 | | Directory to store logs files | `/var/log/kafka` | 144 | | Kafka service | `/usr/lib/systemd/system/kafka.service` | 145 | 146 | ## Example Playbook 147 | 148 | Add the below to a playbook to run those role against hosts belonging to the 149 | `kafka-nodes` group. 150 | 151 | ```yaml 152 | - hosts: kafka-nodes 153 | roles: 154 | - sleighzy.kafka 155 | ``` 156 | 157 | ## Linting 158 | 159 | Linting should be done using [ansible-lint]. 160 | 161 | ```sh 162 | pip3 install ansible-lint --user 163 | 164 | ansible-lint -c ./.ansible-lint . 165 | ``` 166 | 167 | ## Testing 168 | 169 | This module uses the [Ansible Molecule] testing framework. This test suite 170 | creates a Kafka and ZooKeeper cluster consisting of three nodes running within 171 | Docker containers. Each container runs a different OS to test the supported 172 | platforms for this Ansible role. 173 | 174 | As per the [Molecule Installation guide] this should be done using a virtual 175 | environment. The commands below will create a Python virtual environment and 176 | install Molecule including the Docker driver. 177 | 178 | ```sh 179 | $ python3 -m venv molecule-venv 180 | $ source molecule-venv/bin/activate 181 | (molecule-venv) $ pip3 install ansible docker "molecule-plugins[docker]" 182 | ``` 183 | 184 | Run playbook and tests. Linting errors need to be corrected before Molecule will 185 | execute any tests. This will run all tests and then destroy the Docker 186 | containers. 187 | 188 | ```sh 189 | molecule test 190 | ``` 191 | 192 | The below command can be used to run the playbook without the tests. This can be 193 | run multiple times when making changes to the role, and ensuring that operations 194 | are idempotent. 195 | 196 | ```sh 197 | molecule converge 198 | ``` 199 | 200 | The below commands can be used to just run the tests without tearing everything 201 | down. The command `molecule verify` can be repeated for each test run. 202 | 203 | ```sh 204 | molecule create 205 | molecule converge 206 | molecule verify 207 | ``` 208 | 209 | Tear down Molecule tests and Docker containers. 210 | 211 | ```sh 212 | molecule destroy 213 | ``` 214 | 215 | ## License 216 | 217 | ![MIT license] 218 | 219 | [ansible-lint]: https://docs.ansible.com/ansible-lint/ 220 | [ansible molecule]: https://molecule.readthedocs.io/ 221 | [apache kafka]: http://kafka.apache.org/ 222 | [apache zookeeper]: https://zookeeper.apache.org/ 223 | [lint code base]: 224 | https://github.com/sleighzy/ansible-kafka/workflows/Lint%20Code%20Base/badge.svg 225 | [molecule]: 226 | https://github.com/sleighzy/ansible-kafka/workflows/Molecule/badge.svg 227 | [molecule installation guide]: 228 | https://molecule.readthedocs.io/en/stable/installation.html 229 | [mit license]: https://img.shields.io/badge/License-MIT-blue.svg 230 | -------------------------------------------------------------------------------- /defaults/main/001-kafka.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # The Apache Kafka version to be downloaded and installed 3 | # kafka_download_base_url should be set to https://archive.apache.org/dist/kafka/ for older versions than the current 4 | kafka_download_base_url: https://downloads.apache.org/kafka 5 | kafka_download_validate_certs: yes 6 | kafka_version: 3.8.1 7 | kafka_scala_version: 2.13 8 | 9 | # The kafka user and group to create files/dirs with and for running the kafka service 10 | kafka_create_user_group: true 11 | kafka_user: kafka 12 | kafka_group: kafka 13 | 14 | kafka_root_dir: /opt 15 | kafka_dir: "{{ kafka_root_dir }}/kafka" 16 | 17 | # The application log folder (e.g: server.log) 18 | kafka_log_dir: /var/log/kafka 19 | 20 | # Start kafka after installation 21 | kafka_start: yes 22 | # Restart kafka on configuration change 23 | kafka_restart: yes 24 | 25 | ############################# Server ############################# 26 | 27 | # The id of the broker. This must be set to a unique integer for each broker. 28 | kafka_broker_id: 0 29 | 30 | # The Java heap size (memory) allocation (xmx, xms) 31 | kafka_java_heap: "-Xms1G -Xmx1G" 32 | 33 | # The number of threads to use for various background processing tasks 34 | kafka_background_threads: 10 35 | 36 | # The address the socket server listens on. It will get the value returned from 37 | # java.net.InetAddress.getCanonicalHostName() if not configured. 38 | # FORMAT: 39 | # listeners = security_protocol://host_name:port 40 | # EXAMPLE: 41 | # listeners = PLAINTEXT://your.host.name:9092 42 | #listeners=PLAINTEXT://:9092 43 | kafka_listeners: 44 | - "PLAINTEXT://:9092" 45 | 46 | # Hostname and port the broker will advertise to producers and consumers. If not set, 47 | # it uses the value for "listeners" if configured. Otherwise, it will use the value 48 | # returned from java.net.InetAddress.getCanonicalHostName(). 49 | #advertised.listeners=PLAINTEXT://your.host.name:9092 50 | # kafka_advertised_listeners: 51 | # - "SASL_SSL://:9094" 52 | # - "PLAINTEXT://:9092" 53 | 54 | # The number of threads handling network requests 55 | kafka_num_network_threads: 3 56 | # The number of threads that the server uses for processing requests, which may include disk I/O 57 | kafka_num_io_threads: 8 58 | # Specify the number of threads that are used to replicate messages from a source broker. Increasing this value can lead to increased parallelism in I/O operations in the broker. 59 | kafka_num_replica_fetchers: 1 60 | 61 | # The send buffer (SO_SNDBUF) used by the socket server 62 | kafka_socket_send_buffer_bytes: 102400 63 | # The receive buffer (SO_RCVBUF) used by the socket server 64 | kafka_socket_receive_buffer_bytes: 102400 65 | # The maximum size of a request that the socket server will accept (protection against OOM) 66 | kafka_socket_request_max_bytes: 104857600 67 | # The socket receive buffer for network requests 68 | kafka_replica_socket_receive_buffer_bytes: 65536 69 | 70 | # A comma separated list of directories under which to store data log files 71 | kafka_data_log_dirs: /var/lib/kafka/logs 72 | 73 | # The default number of log partitions per topic. More partitions allow greater 74 | # parallelism for consumption, but this will also result in more files across 75 | # the brokers. 76 | kafka_num_partitions: 1 77 | 78 | # The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. 79 | # This value is recommended to be increased for installations with data dirs located in RAID array. 80 | kafka_num_recovery_threads_per_data_dir: 1 81 | 82 | # The number of background threads to use for log cleaning 83 | kafka_log_cleaner_threads: 1 84 | 85 | # The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" 86 | # For anything other than development testing, a value greater than 1 is recommended for to ensure availability such as 3. 87 | kafka_offsets_topic_replication_factor: 1 88 | kafka_transaction_state_log_replication_factor: 1 89 | kafka_transaction_state_log_min_isr: 1 90 | 91 | # The minimum age of a log file to be eligible for deletion 92 | kafka_log_retention_hours: 168 93 | 94 | # The maximum size of a log segment file. When this size is reached a new log segment will be created. 95 | kafka_log_segment_bytes: 1073741824 96 | 97 | # The interval at which log segments are checked to see if they can be deleted according 98 | # to the retention policies 99 | kafka_log_retention_check_interval_ms: 300000 100 | 101 | # Enable auto creation of topic on the server 102 | kafka_auto_create_topics_enable: false 103 | 104 | # Enables delete topic. Delete topic through the admin tool will have no 105 | # effect if this config is turned off 106 | kafka_delete_topic_enable: true 107 | 108 | # Default replication factor for automatically created topics. 109 | kafka_default_replication_factor: 1 110 | 111 | kafka_group_initial_rebalance_delay_ms: 0 112 | 113 | ############################# Zookeeper ############################# 114 | 115 | # Zookeeper connection string (see zookeeper docs for details). 116 | # This is a comma separated host:port pairs, each corresponding to a zk 117 | # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". 118 | # You can also append an optional chroot string to the urls to specify the 119 | # root directory for all kafka znodes. 120 | kafka_zookeeper_connect: "localhost:2181" 121 | 122 | # the directory where the snapshot is stored. 123 | kafka_zookeeper_data_dir: /tmp/zookeeper 124 | # the port at which the clients will connect 125 | kafka_zookeeper_client_port: 2181 126 | # disable the per-ip limit on the number of connections since this is a non-production config 127 | kafka_zookeeper_max_client_cnxns: 0 128 | 129 | ############################# Timeout ############################# 130 | 131 | # Timeout in ms for connecting to zookeeper 132 | kafka_zookeeper_connection_timeout_ms: 6000 133 | 134 | # Offset commit will be delayed until all replicas for the offsets topic receive the commit or this timeout is reached. This is similar to the producer request timeout. 135 | kafka_offsets_commit_timeout_ms: 5000 136 | 137 | # Max wait time for each fetcher request issued by follower replicas. This value should always be less than the replica.lag.time.max.ms at all times to prevent frequent shrinking of ISR for low throughput topics 138 | kafka_replica_fetch_wait_max_ms: 500 139 | 140 | # The amount of time to sleep when there are no logs to clean 141 | kafka_log_cleaner_backoff_ms: 15000 142 | 143 | ########################### Kafka Connect ############################# 144 | 145 | kafka_connect_bootstrap_servers: "localhost:9092" 146 | kafka_connect_group_id: connect-cluster 147 | kafka_connect_key_converter: org.apache.kafka.connect.json.JsonConverter 148 | kafka_connect_value_converter: org.apache.kafka.connect.json.JsonConverter 149 | kafka_connect_key_converter_schemas_enable: true 150 | kafka_connect_value_converter_schemas_enable: true 151 | kafka_connect_internal_key_converter: org.apache.kafka.connect.json.JsonConverter 152 | kafka_connect_internal_value_converter: org.apache.kafka.connect.json.JsonConverter 153 | kafka_connect_internal_key_converter_schemas_enable: false 154 | kafka_connect_internal_value_converter_schemas_enable: false 155 | kafka_connect_offset_storage_replication_factor: 1 156 | kafka_connect_config_storage_replication_factor: 1 157 | kafka_connect_status_storage_replication_factor: 1 158 | kafka_connect_offset_flush_interval_ms: 10000 159 | kafka_connect_plugin_path: /usr/local/share/java,/usr/local/share/kafka/plugins,/opt/connectors 160 | 161 | kafka_connect_offset_storage_file_filename: /tmp/connect.offsets 162 | 163 | ############################# Producer ############################# 164 | 165 | kafka_producer_bootstrap_servers: "localhost:9092" 166 | kafka_producer_compression_type: none 167 | 168 | ############################# Consumer ############################# 169 | 170 | kafka_consumer_group_id: kafka-consumer-group 171 | kafka_consumer_bootstrap_servers: "localhost:9092" 172 | 173 | # Example authentication 174 | ############################# AUTHENTICATION ############################# 175 | # kafka_listener_auth: 176 | # - name: "listener.name.sasl_ssl.plain.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule" 177 | # username: "admin" 178 | # password: "admin-secret" 179 | # users: 180 | # - user: "test1" 181 | # pass: "test1-secret" 182 | # - user: "test2" 183 | # pass: "test2-secret" 184 | # 185 | # kafka_sasl_enabled_mechanisms: "PLAIN" 186 | # kafka_ssl: 187 | # cipher_suites: "TLS_DHE_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384" 188 | # truststore_location: "/etc/security/broker.keystore.jks" 189 | # truststore_password: "secret" 190 | # keystore_location: "/etc/security/broker.keystore.jks" 191 | # keystore_password: "secret" 192 | # key_password: "secret" 193 | # client_auth: "required" 194 | 195 | # Here you can define custom opts e.g. settings for JMX Exporter 196 | # kafka_opts: "" -------------------------------------------------------------------------------- /defaults/main/002-log4j.yml: -------------------------------------------------------------------------------- 1 | --- 2 | log4j_rootlogger: "INFO, stdout, kafkaAppender" 3 | 4 | log4j_appender_stdout: "org.apache.log4j.ConsoleAppender" 5 | log4j_appender_stdout_layout: "org.apache.log4j.PatternLayout" 6 | log4j_appender_stdout_layout_conversionpattern: "[%d] %p %m (%c)%n" 7 | 8 | log4j_appender_kafkaappender: "org.apache.log4j.DailyRollingFileAppender" 9 | log4j_appender_kafkaappender_datepattern: "'.'yyyy-MM-dd-HH" 10 | log4j_appender_kafkaappender_file: "${kafka.logs.dir}/server.log" 11 | log4j_appender_kafkaappender_layout: "org.apache.log4j.PatternLayout" 12 | log4j_appender_kafkaappender_layout_conversionpattern: "[%d] %p %m (%c)%n" 13 | 14 | log4j_appender_statechangeappender: "org.apache.log4j.DailyRollingFileAppender" 15 | log4j_appender_statechangeappender_datepattern: "'.'yyyy-MM-dd-HH" 16 | log4j_appender_statechangeappender_file: "${kafka.logs.dir}/state-change.log" 17 | log4j_appender_statechangeappender_layout: "org.apache.log4j.PatternLayout" 18 | log4j_appender_statechangeappender_layout_conversionpattern: "[%d] %p %m (%c)%n" 19 | 20 | log4j_appender_requestappender: "org.apache.log4j.DailyRollingFileAppender" 21 | log4j_appender_requestappender_datepattern: "'.'yyyy-MM-dd-HH" 22 | log4j_appender_requestappender_file: "${kafka.logs.dir}/kafka-request.log" 23 | log4j_appender_requestappender_layout: "org.apache.log4j.PatternLayout" 24 | log4j_appender_requestappender_layout_conversionpattern: "[%d] %p %m (%c)%n" 25 | 26 | log4j_appender_cleanerappender: "org.apache.log4j.DailyRollingFileAppender" 27 | log4j_appender_cleanerappender_datepattern: "'.'yyyy-MM-dd-HH" 28 | log4j_appender_cleanerappender_file: "${kafka.logs.dir}/log-cleaner.log" 29 | log4j_appender_cleanerappender_layout: "org.apache.log4j.PatternLayout" 30 | log4j_appender_cleanerappender_layout_conversionpattern: "[%d] %p %m (%c)%n" 31 | 32 | log4j_appender_controllerappender: "org.apache.log4j.DailyRollingFileAppender" 33 | log4j_appender_controllerappender_datepattern: "'.'yyyy-MM-dd-HH" 34 | log4j_appender_controllerappender_file: "${kafka.logs.dir}/controller.log" 35 | log4j_appender_controllerappender_layout: "org.apache.log4j.PatternLayout" 36 | log4j_appender_controllerappender_layout_conversionpattern: "[%d] %p %m (%c)%n" 37 | 38 | log4j_appender_authorizerappender: "org.apache.log4j.DailyRollingFileAppender" 39 | log4j_appender_authorizerappender_datepattern: "'.'yyyy-MM-dd-HH" 40 | log4j_appender_authorizerappender_file: "${kafka.logs.dir}/kafka-authorizer.log" 41 | log4j_appender_authorizerappender_layout: "org.apache.log4j.PatternLayout" 42 | log4j_appender_authorizerappender_layout_conversionpattern: "[%d] %p %m (%c)%n" 43 | 44 | # Change the line below to adjust ZK client logging 45 | log4j_logger_org_apache_zookeeper: "INFO" 46 | 47 | # Change the two lines below to adjust the general broker logging level (output to server.log and stdout) 48 | log4j_logger_kafka: "INFO" 49 | log4j_logger_org_apache_kafka: "INFO" 50 | 51 | # Change to DEBUG or TRACE to enable request logging 52 | log4j_logger_kafka_request_logger: "WARN, requestAppender" 53 | log4j_additivity_kafka_request_logger: false 54 | 55 | # Uncomment the lines below and change log4j_logger_kafka_network_requestchannel$ to TRACE for additional output 56 | # related to the handling of requests 57 | #log4j_logger_kafka_network_processor: TRACE, requestAppender 58 | #log4j_logger_kafka_server_kafkaapis: TRACE, requestAppender 59 | #log4j_additivity_kafka_server_kafkaapis: false 60 | log4j_logger_kafka_network_requestchannel: "WARN, requestAppender" 61 | log4j_additivity_kafka_network_requestchannel: false 62 | 63 | log4j_logger_kafka_controller: "TRACE, controllerAppender" 64 | log4j_additivity_kafka_controller: false 65 | 66 | log4j_logger_kafka_log_logcleaner: "INFO, cleanerAppender" 67 | log4j_additivity_kafka_log_logcleaner: false 68 | 69 | log4j_logger_state_change_logger: "INFO, stateChangeAppender" 70 | log4j_additivity_state_change_logger: false 71 | 72 | # Access denials are logged at INFO level, change to DEBUG to also log allowed accesses 73 | log4j_logger_kafka_authorizer_logger: "INFO, authorizerAppender" 74 | log4j_additivity_kafka_authorizer_logger: false 75 | -------------------------------------------------------------------------------- /handlers/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Reload initd 3 | ansible.builtin.command: initctl reload-configuration 4 | changed_when: false 5 | 6 | - name: Restart kafka service 7 | ansible.builtin.service: 8 | name: kafka 9 | state: restarted 10 | when: kafka_restart 11 | 12 | - name: Restart kafka systemd 13 | ansible.builtin.systemd: 14 | name: kafka 15 | state: restarted 16 | daemon_reload: yes 17 | when: kafka_restart 18 | -------------------------------------------------------------------------------- /meta/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | galaxy_info: 3 | author: Simon Leigh 4 | namespace: sleighzy 5 | role_name: kafka 6 | description: Apache Kafka installation for RHEL/CentOS and Debian/Ubuntu 7 | license: MIT 8 | min_ansible_version: 2.10.4 9 | platforms: 10 | - name: EL 11 | versions: 12 | - '6' 13 | - '7' 14 | - '8' 15 | - name: Debian 16 | versions: 17 | - buster 18 | - name: Ubuntu 19 | versions: 20 | - bionic 21 | - focal 22 | galaxy_tags: 23 | - clustering 24 | - kafka 25 | - messaging 26 | - pubsub 27 | - streaming 28 | 29 | dependencies: [] 30 | -------------------------------------------------------------------------------- /molecule/default/converge.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Converge 3 | hosts: all 4 | roles: 5 | - sleighzy.zookeeper 6 | 7 | tasks: 8 | - name: 'Test Kafka role' 9 | ansible.builtin.include_role: 10 | name: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}" 11 | -------------------------------------------------------------------------------- /molecule/default/molecule.yml: -------------------------------------------------------------------------------- 1 | --- 2 | dependency: 3 | name: galaxy 4 | options: 5 | ignore-certs: True 6 | ignore-errors: True 7 | role-file: molecule/default/requirements.yml 8 | requirements-file: molecule/default/requirements.yml 9 | driver: 10 | name: docker 11 | platforms: 12 | - name: server-1 13 | image: geerlingguy/docker-debian10-ansible:latest 14 | docker_networks: 15 | - name: kafka 16 | ipam_config: 17 | - subnet: '172.40.0.0/16' 18 | networks: 19 | - name: kafka 20 | ipv4_address: '172.40.10.1' 21 | etc_hosts: "{'server-2': '172.40.10.2', 'server-3': '172.40.10.3'}" 22 | pre_build_image: true 23 | privileged: true 24 | tmpfs: 25 | - /run 26 | - /tmp 27 | volumes: 28 | - /sys/fs/cgroup:/sys/fs/cgroup:rw 29 | cgroupns_mode: host 30 | capabilities: 31 | - SYS_ADMIN 32 | command: /lib/systemd/systemd 33 | groups: 34 | - kafka-nodes 35 | - zookeeper-nodes 36 | - name: server-2 37 | image: redhat/ubi9:latest 38 | networks: 39 | - name: kafka 40 | ipv4_address: '172.40.10.2' 41 | etc_hosts: "{'server-1': '172.40.10.1', 'server-3': '172.40.10.3'}" 42 | tmpfs: 43 | - /run 44 | - /tmp 45 | volumes: 46 | - /sys/fs/cgroup:/sys/fs/cgroup:rw 47 | cgroupns_mode: host 48 | command: '/usr/lib/systemd/systemd' 49 | pre_build_image: true 50 | capabilities: 51 | - SYS_ADMIN 52 | groups: 53 | - kafka-nodes 54 | - zookeeper-nodes 55 | - name: server-3 56 | image: redhat/ubi9:latest 57 | networks: 58 | - name: kafka 59 | ipv4_address: '172.40.10.3' 60 | etc_hosts: "{'server-1': '172.40.10.1', 'server-2': '172.40.10.2'}" 61 | privileged: true 62 | tmpfs: 63 | - /run 64 | - /tmp 65 | volumes: 66 | - /sys/fs/cgroup:/sys/fs/cgroup:rw 67 | cgroupns_mode: host 68 | command: '/usr/lib/systemd/systemd' 69 | pre_build_image: true 70 | capabilities: 71 | - SYS_ADMIN 72 | groups: 73 | - kafka-nodes 74 | - zookeeper-nodes 75 | provisioner: 76 | name: ansible 77 | inventory: 78 | group_vars: 79 | kafka_zookeeper_connect: 'server-1:2181,server-2:2181,server-3:2181/kafka' 80 | kafka_bootstrap_servers: 'server-1:9092,server-2:9092,server-3:9092' 81 | host_vars: 82 | server-1: 83 | zookeeper_id: 1 84 | kafka_broker_id: 0 85 | kafka_listener_hostname: server-1 86 | server-2: 87 | zookeeper_id: 2 88 | kafka_broker_id: 1 89 | kafka_listener_hostname: server-2 90 | kafka_download_validate_certs: no 91 | server-3: 92 | zookeeper_id: 3 93 | kafka_broker_id: 2 94 | kafka_listener_hostname: server-3 95 | verifier: 96 | name: ansible 97 | scenario: 98 | create_sequence: 99 | - dependency 100 | - create 101 | - prepare 102 | check_sequence: 103 | - dependency 104 | - cleanup 105 | - destroy 106 | - create 107 | - prepare 108 | - converge 109 | - check 110 | - destroy 111 | converge_sequence: 112 | - dependency 113 | - create 114 | - prepare 115 | - converge 116 | test_sequence: 117 | - destroy 118 | - dependency 119 | - syntax 120 | - create 121 | - prepare 122 | - converge 123 | - idempotence 124 | - verify 125 | - destroy 126 | -------------------------------------------------------------------------------- /molecule/default/prepare.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Prepare 3 | hosts: all 4 | gather_facts: true 5 | 6 | pre_tasks: 7 | - name: Install Java 8 (OpenJDK) on RedHat/CentOS 8 | ansible.builtin.yum: 9 | name: java-1.8.0-openjdk 10 | state: installed 11 | when: ansible_os_family == "RedHat" 12 | 13 | - name: Install Java 11 (OpenJDK) on Debian 14 | ansible.builtin.apt: 15 | name: openjdk-11-jdk 16 | state: present 17 | update_cache: yes 18 | when: ansible_os_family == "Debian" 19 | 20 | # The installation of this package into the container means 21 | # that the "ps" command is available for viewing the running process. 22 | # Installing this package however also prevents an issue whereby the 23 | # ZooKeeper service is constantly restarted by systemd which causes 24 | # the Molecule tests to fail as the service is not started correctly. 25 | - name: Install ps on Debian 26 | ansible.builtin.apt: 27 | name: procps 28 | state: present 29 | when: ansible_os_family == "Debian" 30 | 31 | - name: Install ps on RedHat/CentOS 32 | ansible.builtin.yum: 33 | name: procps 34 | state: present 35 | when: ansible_os_family == "RedHat" 36 | -------------------------------------------------------------------------------- /molecule/default/requirements.yml: -------------------------------------------------------------------------------- 1 | roles: 2 | - name: sleighzy.zookeeper 3 | 4 | collections: 5 | - community.docker 6 | -------------------------------------------------------------------------------- /molecule/default/verify.yml: -------------------------------------------------------------------------------- 1 | - name: Verify 2 | hosts: all 3 | 4 | tasks: 5 | - name: Get users 6 | ansible.builtin.getent: 7 | database: passwd 8 | 9 | - name: Get groups 10 | ansible.builtin.getent: 11 | database: group 12 | 13 | - name: Assert that user and group 'kafka' exists 14 | ansible.builtin.assert: 15 | that: 16 | - "'kafka' in getent_passwd" 17 | - "'kafka' in getent_group" 18 | 19 | - name: Register '/opt/kafka_2.13-3.8.1' installation directory status 20 | ansible.builtin.stat: 21 | path: '/opt/kafka_2.13-3.8.1' 22 | register: install_dir 23 | 24 | - name: Assert that '/opt/kafka_2.13-3.8.1' directory is created 25 | ansible.builtin.assert: 26 | that: 27 | - install_dir.stat.exists 28 | - install_dir.stat.isdir 29 | - install_dir.stat.pw_name == 'kafka' 30 | - install_dir.stat.gr_name == 'kafka' 31 | 32 | - name: Register '/opt/kafka' symlink directory status 33 | ansible.builtin.stat: 34 | path: '/opt/kafka' 35 | register: kafka_dir 36 | 37 | - name: Assert that '/opt/kafka' symlink is created 38 | ansible.builtin.assert: 39 | that: 40 | - kafka_dir.stat.exists 41 | - kafka_dir.stat.islnk 42 | - kafka_dir.stat.lnk_target == '/opt/kafka_2.13-3.8.1' 43 | 44 | - name: Register '/var/log/kafka' directory status 45 | ansible.builtin.stat: 46 | path: '/var/log/kafka' 47 | register: kafka_log_dir 48 | 49 | - name: Assert that '/var/log/kafka' directory is created 50 | ansible.builtin.assert: 51 | that: 52 | - kafka_log_dir.stat.exists 53 | - kafka_log_dir.stat.isdir 54 | - kafka_log_dir.stat.pw_name == 'kafka' 55 | - kafka_log_dir.stat.gr_name == 'kafka' 56 | 57 | - name: Register '/opt/kafka/logs' symlink directory status 58 | ansible.builtin.stat: 59 | path: '/opt/kafka/logs' 60 | register: application_logs_symlink 61 | 62 | - name: Assert that '/opt/kafka/logs' symlink is created 63 | ansible.builtin.assert: 64 | that: 65 | - application_logs_symlink.stat.exists 66 | - application_logs_symlink.stat.islnk 67 | - application_logs_symlink.stat.lnk_target == '/var/log/kafka' 68 | - application_logs_symlink.stat.pw_name == 'kafka' 69 | - application_logs_symlink.stat.gr_name == 'kafka' 70 | 71 | - name: Register '/etc/kafka' directory status 72 | ansible.builtin.stat: 73 | path: '/etc/kafka' 74 | register: config_dir 75 | 76 | - name: Assert that '/etc/kafka' directory is created 77 | ansible.builtin.assert: 78 | that: 79 | - config_dir.stat.exists 80 | - config_dir.stat.isdir 81 | - config_dir.stat.pw_name == 'kafka' 82 | - config_dir.stat.gr_name == 'kafka' 83 | 84 | - name: Populate service facts 85 | ansible.builtin.service_facts: 86 | 87 | - name: Assert that Kafka service is installed, running, and enabled 88 | ansible.builtin.assert: 89 | that: 90 | - "'kafka.service' in ansible_facts.services" 91 | - ansible_facts.services['kafka.service'].state == 'running' 92 | - ansible_facts.services['kafka.service'].status == 'enabled' 93 | -------------------------------------------------------------------------------- /tasks/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Load OS-specific variables 3 | ansible.builtin.include_vars: '{{ item }}' 4 | with_first_found: 5 | - ../vars/{{ ansible_os_family }}.yml 6 | - ../vars/{{ ansible_distribution_release }}.yml 7 | - ../vars/empty.yml 8 | tags: 9 | - always 10 | 11 | - name: Create kafka group 12 | ansible.builtin.group: 13 | name: '{{ kafka_group }}' 14 | state: present 15 | system: yes 16 | when: kafka_create_user_group | bool 17 | tags: 18 | - kafka_group 19 | 20 | - name: Create kafka user 21 | ansible.builtin.user: 22 | name: '{{ kafka_user }}' 23 | group: '{{ kafka_group }}' 24 | state: present 25 | createhome: no 26 | system: yes 27 | when: kafka_create_user_group | bool 28 | tags: 29 | - kafka_user 30 | 31 | - name: Check if Kafka has already been downloaded and unpacked 32 | ansible.builtin.stat: 33 | path: '{{ kafka_dir }}_{{ kafka_scala_version }}-{{ kafka_version }}' 34 | register: dir 35 | 36 | - name: Download Apache Kafka 37 | ansible.builtin.get_url: 38 | url: '{{ kafka_download_base_url }}/{{ kafka_version }}/kafka_{{ kafka_scala_version }}-{{ kafka_version }}.tgz' 39 | dest: /tmp 40 | validate_certs: '{{ kafka_download_validate_certs }}' 41 | mode: 0644 42 | when: not dir.stat.exists 43 | tags: 44 | - kafka_download 45 | 46 | - name: Unpack Apache Kafka 47 | ansible.builtin.unarchive: 48 | src: /tmp/kafka_{{ kafka_scala_version }}-{{ kafka_version }}.tgz 49 | dest: '{{ kafka_root_dir }}' 50 | copy: no 51 | group: '{{ kafka_group }}' 52 | owner: '{{ kafka_user }}' 53 | when: not dir.stat.exists 54 | tags: 55 | - kafka_unpack 56 | 57 | - name: Create symlink to kafka installation directory 58 | ansible.builtin.file: 59 | src: '{{ kafka_root_dir }}/kafka_{{ kafka_scala_version }}-{{ kafka_version }}' 60 | dest: '{{ kafka_dir }}' 61 | state: link 62 | group: '{{ kafka_group }}' 63 | owner: '{{ kafka_user }}' 64 | tags: 65 | - kafka_dirs 66 | 67 | - name: Create directory for kafka data log files 68 | ansible.builtin.file: 69 | path: '{{ item }}' 70 | state: directory 71 | group: '{{ kafka_group }}' 72 | owner: '{{ kafka_user }}' 73 | mode: 0755 74 | with_items: "{{ kafka_data_log_dirs.split(',') }}" 75 | tags: 76 | - kafka_dirs 77 | 78 | - name: Create directory for kafka application logs 79 | ansible.builtin.file: 80 | path: '{{ kafka_log_dir }}' 81 | state: directory 82 | group: '{{ kafka_group }}' 83 | owner: '{{ kafka_user }}' 84 | mode: 0755 85 | tags: 86 | - kafka_dirs 87 | 88 | - name: Register directory status for application log files 89 | ansible.builtin.stat: 90 | path: '{{ kafka_dir }}/logs' 91 | register: application_logs_dir 92 | tags: 93 | - kafka_dirs 94 | 95 | - name: Create symlink to application log directory 96 | ansible.builtin.file: 97 | src: '{{ kafka_log_dir }}' 98 | dest: '{{ kafka_dir }}/logs' 99 | state: link 100 | group: '{{ kafka_group }}' 101 | owner: '{{ kafka_user }}' 102 | mode: 0755 103 | follow: no 104 | when: not application_logs_dir.stat.exists 105 | tags: 106 | - kafka_dirs 107 | 108 | - name: Create directory for symlink to kafka configuration files 109 | ansible.builtin.file: 110 | path: /etc/kafka 111 | state: directory 112 | group: '{{ kafka_group }}' 113 | owner: '{{ kafka_user }}' 114 | mode: 0755 115 | tags: 116 | - kafka_dirs 117 | 118 | - name: Template configuration file to server.properties 119 | ansible.builtin.template: 120 | src: server.properties.j2 121 | dest: '{{ kafka_dir }}/config/server.properties' 122 | group: '{{ kafka_group }}' 123 | owner: '{{ kafka_user }}' 124 | mode: 0644 125 | notify: 126 | - Restart kafka service 127 | tags: 128 | - kafka_config 129 | 130 | - name: Template configuration file to zookeeper.properties 131 | ansible.builtin.template: 132 | src: zookeeper.properties.j2 133 | dest: '{{ kafka_dir }}/config/zookeeper.properties' 134 | group: '{{ kafka_group }}' 135 | owner: '{{ kafka_user }}' 136 | mode: 0644 137 | tags: 138 | - kafka_config 139 | 140 | - name: Template configuration file to connect-standalone.properties 141 | ansible.builtin.template: 142 | src: connect-standalone.properties.j2 143 | dest: '{{ kafka_dir }}/config/connect-standalone.properties' 144 | group: '{{ kafka_group }}' 145 | owner: '{{ kafka_user }}' 146 | mode: 0644 147 | tags: 148 | - kafka_config 149 | 150 | - name: Template configuration file to connect-distributed.properties 151 | ansible.builtin.template: 152 | src: connect-distributed.properties.j2 153 | dest: '{{ kafka_dir }}/config/connect-distributed.properties' 154 | group: '{{ kafka_group }}' 155 | owner: '{{ kafka_user }}' 156 | mode: 0644 157 | tags: 158 | - kafka_config 159 | 160 | - name: Template configuration file to producer.properties 161 | ansible.builtin.template: 162 | src: producer.properties.j2 163 | dest: '{{ kafka_dir }}/config/producer.properties' 164 | group: '{{ kafka_group }}' 165 | owner: '{{ kafka_user }}' 166 | mode: 0644 167 | tags: 168 | - kafka_config 169 | 170 | - name: Template configuration file to consumer.properties 171 | ansible.builtin.template: 172 | src: consumer.properties.j2 173 | dest: '{{ kafka_dir }}/config/consumer.properties' 174 | group: '{{ kafka_group }}' 175 | owner: '{{ kafka_user }}' 176 | mode: 0644 177 | tags: 178 | - kafka_config 179 | 180 | - name: Template configuration file to log4j.properties 181 | ansible.builtin.template: 182 | src: log4j.properties.j2 183 | dest: '{{ kafka_dir }}/config/log4j.properties' 184 | group: '{{ kafka_group }}' 185 | owner: '{{ kafka_user }}' 186 | mode: 0644 187 | tags: 188 | - kafka_config 189 | 190 | - name: Create symlink to kafka server properties file 191 | ansible.builtin.file: 192 | src: '{{ kafka_dir }}/config/server.properties' 193 | dest: /etc/kafka/server.properties 194 | state: link 195 | group: '{{ kafka_group }}' 196 | owner: '{{ kafka_user }}' 197 | tags: 198 | - kafka_config 199 | 200 | - name: Create symlink to kafka connect standalone properties file 201 | ansible.builtin.file: 202 | src: '{{ kafka_dir }}/config/connect-standalone.properties' 203 | dest: /etc/kafka/connect-standalone.properties 204 | state: link 205 | group: '{{ kafka_group }}' 206 | owner: '{{ kafka_user }}' 207 | tags: 208 | - kafka_config 209 | 210 | - name: Create symlink to kafka connect distributed properties file 211 | ansible.builtin.file: 212 | src: '{{ kafka_dir }}/config/connect-distributed.properties' 213 | dest: /etc/kafka/connect-distributed.properties 214 | state: link 215 | group: '{{ kafka_group }}' 216 | owner: '{{ kafka_user }}' 217 | tags: 218 | - kafka_config 219 | 220 | - name: Create symlink to kafka producer properties file 221 | ansible.builtin.file: 222 | src: '{{ kafka_dir }}/config/producer.properties' 223 | dest: /etc/kafka/producer.properties 224 | state: link 225 | group: '{{ kafka_group }}' 226 | owner: '{{ kafka_user }}' 227 | tags: 228 | - kafka_config 229 | 230 | - name: Create symlink to kafka consumer properties file 231 | ansible.builtin.file: 232 | src: '{{ kafka_dir }}/config/consumer.properties' 233 | dest: /etc/kafka/consumer.properties 234 | state: link 235 | group: '{{ kafka_group }}' 236 | owner: '{{ kafka_user }}' 237 | tags: 238 | - kafka_config 239 | 240 | - name: Create symlink to kafka log4j properties file 241 | ansible.builtin.file: 242 | src: '{{ kafka_dir }}/config/log4j.properties' 243 | dest: /etc/kafka/log4j.properties 244 | state: link 245 | group: '{{ kafka_group }}' 246 | owner: '{{ kafka_user }}' 247 | tags: 248 | - kafka_config 249 | 250 | - name: Template Kafka init.d service file 251 | ansible.builtin.template: 252 | src: kafka.initd.j2 253 | dest: /etc/init.d/kafka 254 | group: '{{ kafka_group }}' 255 | owner: '{{ kafka_user }}' 256 | mode: 0644 257 | notify: 258 | - Reload initd 259 | - Restart kafka service 260 | when: ansible_os_family == "RedHat" and ansible_distribution_major_version == '6' 261 | tags: 262 | - kafka_service 263 | 264 | - name: Template kafka systemd service 265 | ansible.builtin.template: 266 | src: kafka.service.j2 267 | dest: '{{ kafka_unit_path }}' 268 | group: '{{ kafka_group }}' 269 | owner: '{{ kafka_user }}' 270 | mode: 0644 271 | notify: 272 | - Restart kafka systemd 273 | when: ansible_os_family == "RedHat" and ansible_distribution_major_version > '6' or ansible_os_family == "Debian" 274 | tags: 275 | - kafka_service 276 | 277 | - name: Install and start the kafka service 278 | ansible.builtin.service: 279 | name: kafka 280 | state: started 281 | enabled: yes 282 | when: kafka_start 283 | tags: 284 | - kafka_service 285 | 286 | - name: Delete the kafka archive file 287 | ansible.builtin.file: 288 | path: /tmp/kafka_{{ kafka_scala_version }}-{{ kafka_version }}.tgz 289 | state: absent 290 | tags: 291 | - kafka_cleanup 292 | -------------------------------------------------------------------------------- /templates/connect-distributed.properties.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | ## 4 | # Licensed to the Apache Software Foundation (ASF) under one or more 5 | # contributor license agreements. See the NOTICE file distributed with 6 | # this work for additional information regarding copyright ownership. 7 | # The ASF licenses this file to You under the Apache License, Version 2.0 8 | # (the "License"); you may not use this file except in compliance with 9 | # the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | ## 19 | 20 | # This file contains some of the configurations for the Kafka Connect distributed worker. This file is intended 21 | # to be used with the examples, and some settings may differ from those used in a production system, especially 22 | # the `bootstrap.servers` and those specifying replication factors. 23 | 24 | # A list of host/port pairs to use for establishing the initial connection to the Kafka cluster. 25 | bootstrap.servers={{ kafka_connect_bootstrap_servers }} 26 | 27 | # unique name for the cluster, used in forming the Connect cluster group. Note that this must not conflict with consumer group IDs 28 | group.id={{ kafka_connect_group_id }} 29 | 30 | # The converters specify the format of data in Kafka and how to translate it into Connect data. Every Connect user will 31 | # need to configure these based on the format they want their data in when loaded from or stored into Kafka 32 | key.converter={{ kafka_connect_key_converter }} 33 | value.converter={{ kafka_connect_value_converter }} 34 | # Converter-specific settings can be passed in by prefixing the Converter's setting with the converter we want to apply 35 | # it to 36 | key.converter.schemas.enable={{ kafka_connect_key_converter_schemas_enable }} 37 | value.converter.schemas.enable={{ kafka_connect_value_converter_schemas_enable }} 38 | 39 | # The internal converter used for offsets, config, and status data is configurable and must be specified, but most users will 40 | # always want to use the built-in default. Offset, config, and status data is never visible outside of Kafka Connect in this format. 41 | internal.key.converter={{ kafka_connect_internal_key_converter }} 42 | internal.value.converter={{ kafka_connect_internal_value_converter }} 43 | internal.key.converter.schemas.enable={{ kafka_connect_internal_key_converter_schemas_enable }} 44 | internal.value.converter.schemas.enable={{ kafka_connect_internal_value_converter_schemas_enable }} 45 | 46 | # Topic to use for storing offsets. This topic should have many partitions and be replicated and compacted. 47 | # Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create 48 | # the topic before starting Kafka Connect if a specific topic configuration is needed. 49 | # Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value. 50 | # Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able 51 | # to run this example on a single-broker cluster and so here we instead set the replication factor to 1. 52 | offset.storage.topic=connect-offsets 53 | offset.storage.replication.factor={{ kafka_connect_offset_storage_replication_factor }} 54 | #offset.storage.partitions=25 55 | 56 | # Topic to use for storing connector and task configurations; note that this should be a single partition, highly replicated, 57 | # and compacted topic. Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create 58 | # the topic before starting Kafka Connect if a specific topic configuration is needed. 59 | # Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value. 60 | # Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able 61 | # to run this example on a single-broker cluster and so here we instead set the replication factor to 1. 62 | config.storage.topic=connect-configs 63 | config.storage.replication.factor={{ kafka_connect_config_storage_replication_factor }} 64 | 65 | # Topic to use for storing statuses. This topic can have multiple partitions and should be replicated and compacted. 66 | # Kafka Connect will attempt to create the topic automatically when needed, but you can always manually create 67 | # the topic before starting Kafka Connect if a specific topic configuration is needed. 68 | # Most users will want to use the built-in default replication factor of 3 or in some cases even specify a larger value. 69 | # Since this means there must be at least as many brokers as the maximum replication factor used, we'd like to be able 70 | # to run this example on a single-broker cluster and so here we instead set the replication factor to 1. 71 | status.storage.topic=connect-status 72 | status.storage.replication.factor={{ kafka_connect_status_storage_replication_factor }} 73 | #status.storage.partitions=5 74 | 75 | # Flush much faster than normal, which is useful for testing/debugging 76 | offset.flush.interval.ms={{ kafka_connect_offset_flush_interval_ms }} 77 | 78 | # These are provided to inform the user about the presence of the REST host and port configs 79 | # Hostname & Port for the REST API to listen on. If this is set, it will bind to the interface used to listen to requests. 80 | #rest.host.name= 81 | #rest.port=8083 82 | 83 | # The Hostname & Port that will be given out to other workers to connect to i.e. URLs that are routable from other servers. 84 | #rest.advertised.host.name= 85 | #rest.advertised.port= 86 | 87 | # Set to a list of filesystem paths separated by commas (,) to enable class loading isolation for plugins 88 | # (connectors, converters, transformations). The list should consist of top level directories that include 89 | # any combination of: 90 | # a) directories immediately containing jars with plugins and their dependencies 91 | # b) uber-jars with plugins and their dependencies 92 | # c) directories immediately containing the package directory structure of classes of plugins and their dependencies 93 | # Examples: 94 | # plugin.path=/usr/local/share/java,/usr/local/share/kafka/plugins,/opt/connectors, 95 | plugin.path={{ kafka_connect_plugin_path }} 96 | -------------------------------------------------------------------------------- /templates/connect-standalone.properties.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # These are defaults. This file just demonstrates how to override some settings. 19 | bootstrap.servers={{ kafka_connect_bootstrap_servers }} 20 | 21 | # The converters specify the format of data in Kafka and how to translate it into Connect data. Every Connect user will 22 | # need to configure these based on the format they want their data in when loaded from or stored into Kafka 23 | key.converter={{ kafka_connect_key_converter }} 24 | value.converter={{ kafka_connect_value_converter }} 25 | # Converter-specific settings can be passed in by prefixing the Converter's setting with the converter we want to apply 26 | # it to 27 | key.converter.schemas.enable={{ kafka_connect_key_converter_schemas_enable }} 28 | value.converter.schemas.enable={{ kafka_connect_value_converter_schemas_enable }} 29 | 30 | # The internal converter used for offsets and config data is configurable and must be specified, but most users will 31 | # always want to use the built-in default. Offset and config data is never visible outside of Kafka Connect in this format. 32 | internal.key.converter={{ kafka_connect_internal_key_converter }} 33 | internal.value.converter={{ kafka_connect_internal_value_converter }} 34 | internal.key.converter.schemas.enable={{ kafka_connect_internal_key_converter_schemas_enable }} 35 | internal.value.converter.schemas.enable={{ kafka_connect_internal_value_converter_schemas_enable }} 36 | 37 | offset.storage.file.filename={{ kafka_connect_offset_storage_file_filename }} 38 | # Flush much faster than normal, which is useful for testing/debugging 39 | offset.flush.interval.ms={{ kafka_connect_offset_flush_interval_ms }} 40 | 41 | # Set to a list of filesystem paths separated by commas (,) to enable class loading isolation for plugins 42 | # (connectors, converters, transformations). The list should consist of top level directories that include 43 | # any combination of: 44 | # a) directories immediately containing jars with plugins and their dependencies 45 | # b) uber-jars with plugins and their dependencies 46 | # c) directories immediately containing the package directory structure of classes of plugins and their dependencies 47 | # Note: symlinks will be followed to discover dependencies or plugins. 48 | # Examples: 49 | # plugin.path=/usr/local/share/java,/usr/local/share/kafka/plugins,/opt/connectors, 50 | plugin.path={{ kafka_connect_plugin_path }} 51 | -------------------------------------------------------------------------------- /templates/consumer.properties.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # see org.apache.kafka.clients.consumer.ConsumerConfig for more details 18 | 19 | # list of brokers used for bootstrapping knowledge about the rest of the cluster 20 | # format: host1:port1,host2:port2 ... 21 | bootstrap.servers={{ kafka_consumer_bootstrap_servers }} 22 | 23 | # consumer group id 24 | group.id={{ kafka_consumer_group_id }} 25 | 26 | # What to do when there is no initial offset in Kafka or if the current 27 | # offset does not exist any more on the server: latest, earliest, none 28 | #auto.offset.reset= 29 | -------------------------------------------------------------------------------- /templates/kafka.initd.j2: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # kafka Apache Kafka 4 | # 5 | # chkconfig: 345 20 80 6 | # description: Apache Kafka 7 | # processname: kafka 8 | 9 | # Source function library. 10 | # shellcheck disable=SC1091 11 | . /etc/init.d/functions 12 | 13 | RETVAL=0 14 | prog="kafka" 15 | LOCKFILE=/var/lock/subsys/$prog 16 | 17 | # Declare variables for Kafka 18 | # shellcheck disable=SC1083 19 | KAFKA_HOME_DIR={{ kafka_dir }} 20 | 21 | start() { 22 | echo -n "Starting $prog: " 23 | "$KAFKA_HOME_DIR/bin/kafka-start-server.sh" "$KAFKA_HOME_DIR/config/server.properties" 24 | RETVAL=$? 25 | [ $RETVAL -eq 0 ] && touch $LOCKFILE 26 | echo 27 | return $RETVAL 28 | } 29 | 30 | stop() { 31 | echo -n "Shutting down $prog: " 32 | # shellcheck disable=SC2015 33 | "$KAFKA_HOME_DIR/bin/kafka-stop-server.sh" && success || failure 34 | RETVAL=$? 35 | [ $RETVAL -eq 0 ] && rm -f $LOCKFILE 36 | echo 37 | return $RETVAL 38 | } 39 | 40 | status() { 41 | echo -n "Checking $prog status: " 42 | 43 | # TODO 44 | #RETVAL=$? 45 | return $RETVAL 46 | } 47 | 48 | case "$1" in 49 | start) 50 | start 51 | ;; 52 | stop) 53 | stop 54 | ;; 55 | status) 56 | status 57 | ;; 58 | restart) 59 | stop 60 | start 61 | ;; 62 | *) 63 | echo "Usage: $prog {start|stop|status|restart}" 64 | exit 1 65 | ;; 66 | esac 67 | exit $RETVAL 68 | -------------------------------------------------------------------------------- /templates/kafka.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Apache Kafka 3 | Documentation=http://kafka.apache.org/documentation.html 4 | Requires=network.target 5 | After=network.target 6 | 7 | [Service] 8 | Type=simple 9 | StandardOutput=null 10 | Environment="KAFKA_HEAP_OPTS={{ kafka_java_heap }}" 11 | Environment="LOG_DIR={{ kafka_log_dir }}" 12 | {% if kafka_opts is defined %} 13 | Environment="KAFKA_OPTS={{ kafka_opts }}" 14 | {% endif %} 15 | {% if kafka_extra_args is defined %} 16 | Environment="EXTRA_ARGS={{ kafka_extra_args }}" 17 | {% endif %} 18 | {% if kafka_jmx_port is defined %} 19 | Environment="JMX_PORT={{ kafka_jmx_port }}" 20 | {% endif %} 21 | ExecStart={{ kafka_dir }}/bin/kafka-server-start.sh /etc/kafka/server.properties 22 | ExecStop={{ kafka_dir }}/bin/kafka-server-stop.sh 23 | User={{ kafka_user }} 24 | Group={{ kafka_group }} 25 | Restart=on-failure 26 | LimitNOFILE=infinity 27 | SuccessExitStatus=143 28 | 29 | [Install] 30 | WantedBy=multi-user.target 31 | Alias=kafka.service 32 | -------------------------------------------------------------------------------- /templates/log4j.properties.j2: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Unspecified loggers and loggers with additivity=true output to server.log and stdout 17 | # Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise 18 | log4j.rootLogger={{ log4j_rootlogger }} 19 | 20 | log4j.appender.stdout={{ log4j_appender_stdout }} 21 | log4j.appender.stdout.layout={{ log4j_appender_stdout_layout }} 22 | log4j.appender.stdout.layout.ConversionPattern={{ log4j_appender_stdout_layout_conversionpattern }} 23 | 24 | log4j.appender.kafkaAppender={{ log4j_appender_kafkaappender }} 25 | log4j.appender.kafkaAppender.DatePattern={{ log4j_appender_kafkaappender_datepattern }} 26 | log4j.appender.kafkaAppender.File={{ log4j_appender_kafkaappender_file }} 27 | log4j.appender.kafkaAppender.layout={{ log4j_appender_kafkaappender_layout }} 28 | log4j.appender.kafkaAppender.layout.ConversionPattern={{ log4j_appender_kafkaappender_layout_conversionpattern }} 29 | 30 | log4j.appender.stateChangeAppender={{ log4j_appender_statechangeappender}} 31 | log4j.appender.stateChangeAppender.DatePattern={{ log4j_appender_statechangeappender_datepattern }} 32 | log4j.appender.stateChangeAppender.File={{ log4j_appender_statechangeappender_file }} 33 | log4j.appender.stateChangeAppender.layout={{ log4j_appender_statechangeappender_layout }} 34 | log4j.appender.stateChangeAppender.layout.ConversionPattern={{ log4j_appender_statechangeappender_layout_conversionpattern }} 35 | 36 | log4j.appender.requestAppender={{ log4j_appender_requestappender }} 37 | log4j.appender.requestAppender.DatePattern={{ log4j_appender_requestappender_datepattern }} 38 | log4j.appender.requestAppender.File={{ log4j_appender_requestappender_file }} 39 | log4j.appender.requestAppender.layout={{ log4j_appender_requestappender_layout }} 40 | log4j.appender.requestAppender.layout.ConversionPattern={{ log4j_appender_requestappender_layout_conversionpattern }} 41 | 42 | log4j.appender.cleanerAppender={{ log4j_appender_cleanerappender }} 43 | log4j.appender.cleanerAppender.DatePattern={{ log4j_appender_cleanerappender_datepattern }} 44 | log4j.appender.cleanerAppender.File={{ log4j_appender_cleanerappender_file }} 45 | log4j.appender.cleanerAppender.layout={{ log4j_appender_cleanerappender_layout }} 46 | log4j.appender.cleanerAppender.layout.ConversionPattern={{ log4j_appender_cleanerappender_layout_conversionpattern }} 47 | 48 | log4j.appender.controllerAppender={{ log4j_appender_controllerappender }} 49 | log4j.appender.controllerAppender.DatePattern={{ log4j_appender_controllerappender_datepattern }} 50 | log4j.appender.controllerAppender.File={{ log4j_appender_controllerappender_file }} 51 | log4j.appender.controllerAppender.layout={{ log4j_appender_controllerappender_layout }} 52 | log4j.appender.controllerAppender.layout.ConversionPattern={{ log4j_appender_controllerappender_layout_conversionpattern }} 53 | 54 | log4j.appender.authorizerAppender={{ log4j_appender_authorizerappender }} 55 | log4j.appender.authorizerAppender.DatePattern={{ log4j_appender_authorizerappender_datepattern }} 56 | log4j.appender.authorizerAppender.File={{ log4j_appender_authorizerappender_file }} 57 | log4j.appender.authorizerAppender.layout={{ log4j_appender_authorizerappender_layout }} 58 | log4j.appender.authorizerAppender.layout.ConversionPattern={{ log4j_appender_authorizerappender_layout_conversionpattern }} 59 | 60 | # Change the line below to adjust ZK client logging 61 | log4j.logger.org.apache.zookeeper={{ log4j_logger_org_apache_zookeeper }} 62 | 63 | # Change the two lines below to adjust the general broker logging level (output to server.log and stdout) 64 | log4j.logger.kafka={{ log4j_logger_kafka }} 65 | log4j.logger.org.apache.kafka={{ log4j_logger_org_apache_kafka }} 66 | 67 | # Change to DEBUG or TRACE to enable request logging 68 | log4j.logger.kafka.request.logger={{ log4j_logger_kafka_request_logger }} 69 | log4j.additivity.kafka.request.logger={{ log4j_additivity_kafka_request_logger }} 70 | 71 | # Uncomment the lines below and change log4j.logger.kafka.network.RequestChannel$ to TRACE for additional output 72 | # related to the handling of requests 73 | #log4j.logger.kafka.network.Processor=false 74 | #log4j.logger.kafka.server.KafkaApis=false 75 | #log4j.additivity.kafka.server.KafkaApis=false 76 | {% if log4j_logger_kafka_network_processor is defined %} 77 | log4j.logger.kafka.network.Processor={{ log4j_logger_kafka_network_processor }} 78 | {% endif %} 79 | {% if log4j_logger_kafka_server_kafkaapis is defined %} 80 | log4j.logger.kafka.server.KafkaApis={{ log4j_logger_kafka_server_kafkaapis }} 81 | {% endif %} 82 | {% if log4j_additivity_kafka_server_kafkaapis is defined %} 83 | log4j.additivity.kafka.server.KafkaApis={{ log4j_additivity_kafka_server_kafkaapis }} 84 | {% endif %} 85 | log4j.logger.kafka.network.RequestChannel$={{ log4j_logger_kafka_network_requestchannel }} 86 | log4j.additivity.kafka.network.RequestChannel$={{ log4j_additivity_kafka_network_requestchannel }} 87 | 88 | log4j.logger.kafka.controller={{ log4j_logger_kafka_controller }} 89 | log4j.additivity.kafka.controller={{ log4j_additivity_kafka_controller }} 90 | 91 | log4j.logger.kafka.log.LogCleaner={{ log4j_logger_kafka_log_logcleaner }} 92 | log4j.additivity.kafka.log.LogCleaner={{ log4j_additivity_kafka_log_logcleaner }} 93 | 94 | log4j.logger.state.change.logger={{ log4j_logger_state_change_logger }} 95 | log4j.additivity.state.change.logger={{ log4j_additivity_state_change_logger }} 96 | 97 | # Access denials are logged at INFO level, change to DEBUG to also log allowed accesses 98 | log4j.logger.kafka.authorizer.logger={{ log4j_logger_kafka_authorizer_logger }} 99 | log4j.additivity.kafka.authorizer.logger={{ log4j_additivity_kafka_authorizer_logger }} 100 | -------------------------------------------------------------------------------- /templates/producer.properties.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # see org.apache.kafka.clients.producer.ProducerConfig for more details 18 | 19 | ############################# Producer Basics ############################# 20 | 21 | # list of brokers used for bootstrapping knowledge about the rest of the cluster 22 | # format: host1:port1,host2:port2 ... 23 | bootstrap.servers={{ kafka_producer_bootstrap_servers }} 24 | 25 | # specify the compression codec for all data generated: none, gzip, snappy, lz4, zstd 26 | compression.type={{ kafka_producer_compression_type }} 27 | 28 | # name of the partitioner class for partitioning records; 29 | # The default uses "sticky" partitioning logic which spreads the load evenly between partitions, but improves throughput by attempting to fill the batches sent to each partition. 30 | #partitioner.class= 31 | 32 | # the maximum amount of time the client will wait for the response of a request 33 | #request.timeout.ms= 34 | 35 | # how long `KafkaProducer.send` and `KafkaProducer.partitionsFor` will block for 36 | #max.block.ms= 37 | 38 | # the producer will wait for up to the given delay to allow other records to be sent so that the sends can be batched together 39 | #linger.ms= 40 | 41 | # the maximum size of a request in bytes 42 | #max.request.size= 43 | 44 | # the default batch size in bytes when batching multiple records sent to a partition 45 | #batch.size= 46 | 47 | # the total bytes of memory the producer can use to buffer records waiting to be sent to the server 48 | #buffer.memory= 49 | -------------------------------------------------------------------------------- /templates/server.properties.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # This configuration file is intended for use in ZK-based mode, where Apache ZooKeeper is required. 19 | # See kafka.server.KafkaConfig for additional details and defaults 20 | 21 | ############################# Server Basics ############################# 22 | 23 | # The id of the broker. This must be set to a unique integer for each broker. 24 | broker.id={{ kafka_broker_id }} 25 | 26 | # Enable auto creation of topic on the server 27 | auto.create.topics.enable={{ kafka_auto_create_topics_enable }} 28 | 29 | # Enables delete topic. Delete topic through the admin tool will have no 30 | # effect if this config is turned off 31 | delete.topic.enable={{ kafka_delete_topic_enable }} 32 | 33 | # Default replication factor for automatically created topics. 34 | default.replication.factor={{ kafka_default_replication_factor }} 35 | 36 | # The number of threads to use for various background processing tasks 37 | background.threads={{ kafka_background_threads }} 38 | 39 | ############################# Socket Server Settings ############################# 40 | 41 | # The address the socket server listens on. If not configured, the host name will be equal to the value of 42 | # java.net.InetAddress.getCanonicalHostName(), with PLAINTEXT listener name, and port 9092. 43 | # FORMAT: 44 | # listeners = listener_name://host_name:port 45 | # EXAMPLE: 46 | # listeners = PLAINTEXT://your.host.name:9092 47 | #listeners=PLAINTEXT://:9092 48 | listeners={{ kafka_listeners | join(",")}} 49 | 50 | # Listener name, hostname and port the broker will advertise to clients. 51 | # If not set, it uses the value for "listeners". 52 | #advertised.listeners=PLAINTEXT://your.host.name:9092 53 | {% if kafka_advertised_listeners is defined %} 54 | advertised.listeners={{ kafka_advertised_listeners | join(",")}} 55 | {% endif %} 56 | 57 | # Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details 58 | #listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL 59 | 60 | # The number of threads that the server uses for receiving requests from the network and sending responses to the network 61 | num.network.threads={{ kafka_num_network_threads }} 62 | 63 | # The number of threads that the server uses for processing requests, which may include disk I/O 64 | num.io.threads={{ kafka_num_io_threads }} 65 | 66 | # Specify the number of threads that are used to replicate messages from a source broker. Increasing this value can lead to increased parallelism in I/O operations in the broker. 67 | num.replica.fetchers={{ kafka_num_replica_fetchers }} 68 | 69 | # The send buffer (SO_SNDBUF) used by the socket server 70 | socket.send.buffer.bytes={{ kafka_socket_send_buffer_bytes }} 71 | 72 | # The receive buffer (SO_RCVBUF) used by the socket server 73 | socket.receive.buffer.bytes={{ kafka_socket_receive_buffer_bytes }} 74 | 75 | # The maximum size of a request that the socket server will accept (protection against OOM) 76 | socket.request.max.bytes={{ kafka_socket_request_max_bytes }} 77 | 78 | # The socket receive buffer for network requests 79 | replica.socket.receive.buffer.bytes={{ kafka_replica_socket_receive_buffer_bytes }} 80 | 81 | ############################# Log Basics ############################# 82 | 83 | # A comma separated list of directories under which to store log files 84 | log.dirs={{ kafka_data_log_dirs }} 85 | 86 | # The default number of log partitions per topic. More partitions allow greater 87 | # parallelism for consumption, but this will also result in more files across 88 | # the brokers. 89 | num.partitions={{ kafka_num_partitions }} 90 | 91 | # The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. 92 | # This value is recommended to be increased for installations with data dirs located in RAID array. 93 | num.recovery.threads.per.data.dir={{ kafka_num_recovery_threads_per_data_dir }} 94 | 95 | # The number of background threads to use for log cleaning 96 | log.cleaner.threads={{ kafka_log_cleaner_threads }} 97 | 98 | ############################# Internal Topic Settings ############################# 99 | # The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" 100 | # For anything other than development testing, a value greater than 1 is recommended for to ensure availability such as 3. 101 | offsets.topic.replication.factor={{ kafka_offsets_topic_replication_factor }} 102 | transaction.state.log.replication.factor={{ kafka_transaction_state_log_replication_factor }} 103 | transaction.state.log.min.isr={{ kafka_transaction_state_log_min_isr }} 104 | 105 | ############################# Log Flush Policy ############################# 106 | 107 | # Messages are immediately written to the filesystem but by default we only fsync() to sync 108 | # the OS cache lazily. The following configurations control the flush of data to disk. 109 | # There are a few important trade-offs here: 110 | # 1. Durability: Unflushed data may be lost if you are not using replication. 111 | # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. 112 | # 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. 113 | # The settings below allow one to configure the flush policy to flush data after a period of time or 114 | # every N messages (or both). This can be done globally and overridden on a per-topic basis. 115 | 116 | # The number of messages to accept before forcing a flush of data to disk 117 | #log.flush.interval.messages=10000 118 | 119 | # The maximum amount of time a message can sit in a log before we force a flush 120 | #log.flush.interval.ms=1000 121 | 122 | ############################# Log Retention Policy ############################# 123 | 124 | # The following configurations control the disposal of log segments. The policy can 125 | # be set to delete segments after a period of time, or after a given size has accumulated. 126 | # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens 127 | # from the end of the log. 128 | 129 | # The minimum age of a log file to be eligible for deletion due to age 130 | log.retention.hours={{ kafka_log_retention_hours }} 131 | 132 | # A size-based retention policy for logs. Segments are pruned from the log unless the remaining 133 | # segments drop below log.retention.bytes. Functions independently of log.retention.hours. 134 | #log.retention.bytes=1073741824 135 | 136 | # The maximum size of a log segment file. When this size is reached a new log segment will be created. 137 | log.segment.bytes={{ kafka_log_segment_bytes }} 138 | 139 | # The interval at which log segments are checked to see if they can be deleted according 140 | # to the retention policies 141 | log.retention.check.interval.ms={{ kafka_log_retention_check_interval_ms }} 142 | 143 | ############################# Zookeeper ############################# 144 | 145 | # Zookeeper connection string (see zookeeper docs for details). 146 | # This is a comma separated host:port pairs, each corresponding to a zk 147 | # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". 148 | # You can also append an optional chroot string to the urls to specify the 149 | # root directory for all kafka znodes. 150 | zookeeper.connect={{ kafka_zookeeper_connect }} 151 | 152 | ############################# Timeout ############################# 153 | 154 | # Timeout in ms for connecting to zookeeper 155 | zookeeper.connection.timeout.ms={{ kafka_zookeeper_connection_timeout_ms }} 156 | # Offset commit will be delayed until all replicas for the offsets topic receive the commit or this timeout is reached. This is similar to the producer request timeout. 157 | offsets.commit.timeout.ms={{ kafka_offsets_commit_timeout_ms }} 158 | # Max wait time for each fetcher request issued by follower replicas. This value should always be less than the replica.lag.time.max.ms at all times to prevent frequent shrinking of ISR for low throughput topics 159 | replica.fetch.wait.max.ms={{ kafka_replica_fetch_wait_max_ms }} 160 | # The amount of time to sleep when there are no logs to clean 161 | log.cleaner.backoff.ms={{ kafka_log_cleaner_backoff_ms }} 162 | 163 | ############################# Group Coordinator Settings ############################# 164 | 165 | # The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. 166 | # The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. 167 | # The default value for this is 3 seconds. 168 | # We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. 169 | # However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. 170 | group.initial.rebalance.delay.ms={{ kafka_group_initial_rebalance_delay_ms }} 171 | 172 | {% if kafka_listener_auth is defined %} 173 | ############################# AUTHENTICATION ############################# 174 | {% for type in kafka_listener_auth %} 175 | {{ type.name }} required \ 176 | {% for user in type.users %} 177 | user_{{ user.user }}="{{ user.pass }}" \ 178 | {% endfor %} 179 | username="{{ type.username }}" \ 180 | password="{{ type.password }}"; 181 | {% endfor %} 182 | {% endif %} 183 | 184 | {% if kafka_sasl_enabled_mechanisms is defined %} 185 | sasl.enabled.mechanisms={{ kafka_sasl_enabled_mechanisms }} 186 | {% endif %} 187 | {% if kafka_ssl is defined %} 188 | ssl.cipher.suites={{ kafka_ssl.cipher_suites }} 189 | ssl.truststore.location={{ kafka_ssl.truststore_location }} 190 | ssl.truststore.password={{ kafka_ssl.truststore_password }} 191 | ssl.keystore.location={{ kafka_ssl.keystore_location }} 192 | ssl.keystore.password={{ kafka_ssl.keystore_password }} 193 | ssl.key.password={{ kafka_ssl.key_password }} 194 | ssl.client.auth={{ kafka_ssl.client_auth }} 195 | ssl.truststore.type={{ kafka_ssl.truststore_type | default('jks') }} 196 | ssl.keystore.type={{ kafka_ssl.keystore_type | default('jks') }} 197 | {% endif %} 198 | 199 | {% for key, value in (kafka_server_config_params | default({})).items() %} 200 | {{key}}={{value}} 201 | {% endfor %} -------------------------------------------------------------------------------- /templates/zookeeper.properties.j2: -------------------------------------------------------------------------------- 1 | # {{ ansible_managed }} 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # the directory where the snapshot is stored. 18 | dataDir={{ kafka_zookeeper_data_dir }} 19 | # the port at which the clients will connect 20 | clientPort={{ kafka_zookeeper_client_port }} 21 | # disable the per-ip limit on the number of connections since this is a non-production config 22 | maxClientCnxns={{ kafka_zookeeper_max_client_cnxns }} 23 | # Disable the adminserver by default to avoid port conflicts. 24 | # Set the port to something non-conflicting if choosing to enable this 25 | admin.enableServer=false 26 | # admin.serverPort=8080 27 | -------------------------------------------------------------------------------- /vars/Debian.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kafka_unit_path: /lib/systemd/system/kafka.service 3 | -------------------------------------------------------------------------------- /vars/RedHat.yml: -------------------------------------------------------------------------------- 1 | --- 2 | kafka_unit_path: /usr/lib/systemd/system/kafka.service 3 | --------------------------------------------------------------------------------