├── .circleci └── config.yml ├── .dockerignore ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── k8s ├── README.md ├── kafka-persistent.yaml ├── kafka-zk-persistent.yaml ├── kafka-zk.yaml ├── main.sh └── zk.yaml ├── kafka_download.sh ├── kafka_env.sh ├── kafka_server.sh ├── kafka_server_status.sh ├── kafka_setup.sh └── openshift ├── README.md ├── buildconfig.yaml ├── kafka-persistent.yaml ├── kafka-zk-persistent.yaml ├── kafka-zk.yaml └── main.sh /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | docker: 4 | docker: 5 | - image: docker 6 | steps: 7 | - setup_remote_docker 8 | - checkout 9 | - run: 10 | name: Docker login 11 | command: | 12 | docker login -u=${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} 13 | - run: 14 | name: Install essentials 15 | command: | 16 | apk add --no-cache --virtual .build-deps make gcc 17 | - run: 18 | name: build, test and push images 19 | command: | 20 | if [ "$CIRCLE_BRANCH" == "master" ]; then 21 | docker build -t ${DOCKER_USERNAME}/kafka:latest .; 22 | docker push ${DOCKER_USERNAME}/kafka:latest; 23 | else 24 | make docker-build docker-test docker-push; 25 | fi 26 | 27 | k8s: 28 | machine: 29 | image: ubuntu-1604:201903-01 30 | steps: 31 | - checkout 32 | - run: 33 | name: update pkgs 34 | command: sudo apt-get update && sudo apt-get install -f -y conntrack 35 | - run: 36 | name: install minikube and kubectl 37 | command: make minikube-install 38 | - run: 39 | name: run minikube 40 | command: make minikube-run 41 | - run: 42 | name: run tests 43 | command: make minikube-test-zk 44 | - run: 45 | name: clean resources 46 | command: make minikube-clean-resources 47 | - run: 48 | name: run tests with persistent storage 49 | command: make minikube-test-persistent 50 | 51 | openshift: 52 | machine: 53 | image: ubuntu-1604:201903-01 54 | steps: 55 | - checkout 56 | - run: 57 | name: install oc 58 | command: make oc-install 59 | - run: 60 | name: run local openshift cluster 61 | command: make oc-cluster-run 62 | - run: 63 | name: run tests 64 | command: make oc-test-zk 65 | - run: 66 | name: clean resources 67 | command: make oc-clean-resources 68 | - run: 69 | name: run tests with persistent storage 70 | command: make oc-test-persistent 71 | 72 | gh-tag-release: 73 | docker: 74 | - image: cibuilds/github:0.12.2 75 | steps: 76 | - checkout 77 | - run: 78 | name: Install essentials 79 | command: | 80 | apk add --no-cache --virtual .build-deps make gcc curl 81 | - run: 82 | name: Create a new tag 83 | command: | 84 | VERSION=v$(make version) 85 | git tag -f ${VERSION} 86 | git remote set-url origin https://${CIRCLE_PROJECT_USERNAME}:${GITHUB_TOKEN}@github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME} 87 | git push -f --tags 88 | - run: 89 | name: Create a tag release on github 90 | command: | 91 | VERSION=v$(make version) 92 | ghr -t ${GITHUB_TOKEN} -u ${CIRCLE_PROJECT_USERNAME} -r ${CIRCLE_PROJECT_REPONAME} -c ${CIRCLE_SHA1} -b "Kafka version ${VERSION}" -delete ${VERSION} ./k8s 93 | 94 | workflows: 95 | version: 2 96 | build: 97 | jobs: 98 | - docker: 99 | filters: 100 | branches: 101 | only: 102 | - /^\d+\.\d+-\d+\.\d+\.\d+$/ 103 | - master 104 | - k8s: 105 | requires: 106 | - docker 107 | filters: 108 | branches: 109 | only: 110 | - /^\d+\.\d+-\d+\.\d+\.\d+$/ 111 | - openshift: 112 | requires: 113 | - docker 114 | filters: 115 | branches: 116 | only: 117 | - /^\d+\.\d+-\d+\.\d+\.\d+$/ 118 | - gh-tag-release: 119 | requires: 120 | - k8s 121 | - openshift 122 | filters: 123 | branches: 124 | only: 125 | - /^\d+\.\d+-\d+\.\d+\.\d+$/ -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | *.* 2 | !README.md 3 | !Dockerfile 4 | !kafka_*.sh -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | !.gitignore 3 | !.circleci 4 | !.dockerignore 5 | 6 | minikube 7 | minishift 8 | kubectl 9 | oc 10 | 11 | *.gz 12 | *.zip 13 | *.tar 14 | *.tgz -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8-jre-alpine 2 | 3 | MAINTAINER Enrique Garcia 4 | 5 | ARG KAFKA_HOME=/opt/kafka 6 | ARG KAFKA_USER=kafka 7 | ARG KAFKA_GROUP=kafka 8 | ARG KAFKA_VERSION="2.5.0" 9 | ARG SCALA_VERSION="2.13" 10 | 11 | ENV KAFKA_HOME=${KAFKA_HOME} \ 12 | KAFKA_VERSION=${KAFKA_VERSION} \ 13 | SCALA_VERSION=${SCALA_VERSION} \ 14 | KAFKA_ZK_LOCAL=true \ 15 | KAFKA_REPLICAS=1 \ 16 | KAFKA_USER=$KAFKA_USER \ 17 | KAFKA_GROUP=$KAFKA_GROUP \ 18 | KAFKA_DATA_DIR=$KAFKA_HOME/data \ 19 | SERVER_port=9092 \ 20 | ZK_clientPort=2181 21 | 22 | # Required packages 23 | RUN apk add --update --no-cache \ 24 | bash tar gnupg openssl ca-certificates sudo 25 | 26 | # Download kafka distribution under KAFKA_HOME directory 27 | ADD kafka_download.sh /tmp/ 28 | 29 | RUN mkdir -p $KAFKA_HOME \ 30 | && chmod a+x /tmp/kafka_download.sh 31 | 32 | RUN /tmp/kafka_download.sh 33 | 34 | RUN rm -rf /tmp/kafka_download.sh \ 35 | && apk del gnupg 36 | 37 | # Add custom scripts and configure user 38 | ADD kafka_*.sh $KAFKA_HOME/bin/ 39 | 40 | RUN addgroup -S -g 1001 $KAFKA_GROUP \ 41 | && adduser -h $KAFKA_HOME -g "Kafka user" -u 1001 -D -S -G $KAFKA_GROUP $KAFKA_USER \ 42 | && chown -R $KAFKA_USER:$KAFKA_GROUP $KAFKA_HOME \ 43 | && chmod a+x $KAFKA_HOME/bin/kafka_*.sh \ 44 | && chmod -R a+w $KAFKA_HOME \ 45 | && ln -s $KAFKA_HOME/bin/kafka_*.sh /usr/bin \ 46 | && echo "${KAFKA_USER} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers 47 | 48 | USER $KAFKA_USER 49 | WORKDIR $KAFKA_HOME/bin/ 50 | 51 | EXPOSE $ZK_port $SERVER_port 52 | 53 | HEALTHCHECK --interval=10s --retries=10 CMD "kafka_server_status.sh" 54 | 55 | CMD kafka_server.sh start 56 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .DEFAULT_GOAL := help 2 | 3 | DOCKER_ORG ?= engapa 4 | DOCKER_IMAGE ?= kafka 5 | 6 | SCALA_VERSION ?= 2.13 7 | KAFKA_VERSION ?= 2.5.0 8 | 9 | 10 | .PHONY: help 11 | help: ## Show this help 12 | @grep -E '^[0-9a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 13 | 14 | .PHONY: clean 15 | clean: ## Clean docker containers and images 16 | @docker rm -f $$(docker ps -a -f "ancestor=$(DOCKER_ORG)/$(DOCKER_IMAGE):$(SCALA_VERSION)-$(KAFKA_VERSION)" --format '{{.Names}}') > /dev/null 2>&1 || true 17 | @docker rmi -f $(DOCKER_ORG)/$(DOCKER_IMAGE):$(SCALA_VERSION)-$(KAFKA_VERSION) > /dev/null 2>&1 || true 18 | 19 | .PHONY: docker-build 20 | docker-build: ## Build the docker image 21 | @docker build --no-cache \ 22 | -t $(DOCKER_ORG)/$(DOCKER_IMAGE):$(SCALA_VERSION)-$(KAFKA_VERSION) . 23 | 24 | .PHONY: docker-run 25 | docker-run: ## Create a docker container 26 | @docker run -d --name kafka $(DOCKER_ORG)/$(DOCKER_IMAGE):$(SCALA_VERSION)-$(KAFKA_VERSION) 27 | 28 | .PHONY: docker-test 29 | docker-test: docker-run ## Test for docker container 30 | @until [ "$$(docker ps --filter 'name=kafka' --filter 'health=healthy' --format '{{.Names}}')" == "kafka" ]; do \ 31 | sleep 10; \ 32 | (docker ps --filter 'name=kafka' --format '{{.Names}}' | grep kafka > /dev/null 2>&1) || exit $$?; \ 33 | echo "Checking healthy status of kafka ..."; \ 34 | done 35 | 36 | .PHONY: docker-push 37 | docker-push: ## Publish docker images 38 | @docker push $(DOCKER_ORG)/$(DOCKER_IMAGE):$(SCALA_VERSION)-$(KAFKA_VERSION) 39 | 40 | 41 | .PHONY: minikube-install 42 | minikube-install: ## Install minikube and kubectl 43 | @k8s/main.sh minikube-install 44 | @k8s/main.sh kubectl-install 45 | 46 | .PHONY: minikube-run 47 | minikube-run: ## Run minikube 48 | @k8s/main.sh minikube-run 49 | 50 | .PHONY: minikube-test-zk 51 | minikube-test-zk: ## Launch tests on minikube, within an internal zookeeper cluster 52 | @k8s/main.sh test-zk 53 | 54 | .PHONY: minikube-test-persistent 55 | minikube-test-persistent: ## Launch tests on minikube, within an external zookeeper cluster 56 | @k8s/main.sh test-persistent 57 | 58 | .PHONY: minikube-clean-resources 59 | minikube-clean-resources: ## Clean kafka and zookeeper respources 60 | @k8s/main.sh clean-resources 61 | 62 | .PHONY: minikube-delete 63 | minikube-delete: ## Remove minikube 64 | @k8s/main.sh minikube-delete 65 | 66 | .PHONY: oc-install 67 | oc-install: ## Install oc tools 68 | @openshift/main.sh oc-install 69 | 70 | .PHONY: oc-cluster-run 71 | oc-cluster-run: ## Run a cluster through oc command 72 | @openshift/main.sh oc-cluster-run 73 | 74 | .PHONY: oc-test-zk 75 | oc-test-zk: ## Launch tests on openshift, within an internal zookeeper cluster 76 | @openshift/main.sh test-zk 77 | 78 | .PHONY: oc-clean-resources 79 | oc-clean-resources: ## Clean kafka and zookeeper respources 80 | @openshift/main.sh clean-resources 81 | 82 | .PHONY: oc-test-persistent 83 | oc-test-persistent: ## Launch tests on openshift, within an external zookeeper cluster 84 | @openshift/main.sh test-persistent 85 | 86 | .PHONY: oc-cluster-delete 87 | oc-cluster-clean: ## Remove openshift cluster 88 | @openshift/main.sh oc-cluster-delete 89 | 90 | .PHONY: version 91 | version: ## Get version 92 | @echo "$(SCALA_VERSION)-$(KAFKA_VERSION)" 93 | 94 | ## TODO: helm, ksonnet for deploy on kubernetes -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kafka Docker Image 2 | [![Build status](https://circleci.com/gh/engapa/kafka-k8s-openshift/tree/master.svg?style=svg "Build status")](https://circleci.com/gh/engapa/kafka-k8s-openshift/tree/master) 3 | [![Docker Pulls](https://img.shields.io/docker/pulls/engapa/kafka.svg)](https://hub.docker.com/r/engapa/kafka/) 4 | [![Docker Layering](https://images.microbadger.com/badges/image/engapa/kafka.svg)](https://microbadger.com/images/engapa/kafka) 5 | [![Docker Version](https://images.microbadger.com/badges/version/engapa/kafka.svg)](https://microbadger.com/images/engapa/kafka) 6 | ![OSS](https://badges.frapsoft.com/os/v1/open-source.svg?v=103 "We love OpenSource") 7 | 8 | This project is meant to create an optimised docker image to run kafka containers as 'statefulset' into kubernetes/openshift. 9 | 10 | Obviously, the docker image can be used locally for testing or development purposes. 11 | 12 | ## Build and publish a kafka docker image 13 | 14 | To get a docker image ready with default values type: 15 | 16 | ```bash 17 | $ make clean-all docker-build docker-test docker-push 18 | ``` 19 | To get your own image: 20 | ```bash 21 | $ export KAFKA_HOME="/opt/kafka" 22 | $ export SCALA_VERSION="2.13" 23 | $ export KAFKA_VERSION="2.5.0" 24 | $ docker build --build-arg SCALA_VERSION=$SCALA_VERSION --build-arg KAFKA_VERSION=$KAFKA_VERSION --build-arg KAFKA_HOME=$KAFKA_HOME \ 25 | -t your-org/kafka:${SCALA_VERSION}-${KAFKA_VERSION} . 26 | ``` 27 | 28 | > NOTE: build-args are optional arguments if you want different values from default ones in the Dockerfile 29 | 30 | The built docker image will contain a kafka distribution (${SCALA_VERSION}-${KAFKA_VERSION}) under the directory $KAFKA_HOME. 31 | 32 | The provided scripts are: 33 | 34 | * **kafka_download.sh** : This script is used to download the suitable release. 35 | * **kafka_env.sh** : It purpose is load the default environments variables. 36 | * **kafka_setup.sh** : Configure kafka and zookeeper dynamically , based on [utils-docker project](https://github.com/engapa/utils-docker) 37 | * **kafka_server.sh** : A central script to manage kafka and optional zookeeper processes. 38 | * **kafka_server_status.sh** : Checks kafka server status. 39 | 40 | Public docker images are available [HERE](https://cloud.docker.com/repository/docker/engapa/kafka/tags) 41 | 42 | ## Getting started with a single docker container locally 43 | 44 | The example bellow shows you how to run an all-in-one docker kafka container (with zookeeper as internal sidecar): 45 | 46 | ```bash 47 | $ docker run -it -p 9092:9092 -p 2181:2181 \ 48 | -e "SETUP_DEBUG=true" \ 49 | -e "SERVER_advertised_listeners=PLAINTEXT://localhost:9092" \ 50 | -e "SERVER_listener_security_protocol_map=PLAINTEXT:PLAINTEXT" \ 51 | -e "SERVER_listeners=PLAINTEXT://0.0.0.0:9092" \ 52 | -h kafka engapa/kafka:${SCALA_VERSION}-${KAFKA_VERSION} 53 | 54 | Writing environment variables to file : 55 | 56 | PREFIX : SERVER_ 57 | DEST_FILE : /opt/kafka/config/server.properties 58 | EXCLUSIONS : 59 | CREATE_FILE : true 60 | OVERRIDE : true 61 | FROM_SEPARATOR : _ 62 | TO_SEPARATOR : . 63 | LOWER : true 64 | ....................................... 65 | 66 | [DEBUG] [2017-01-31_20:17:26] - [OVERRIDE] : SERVER_log_dirs --> log.dirs=/opt/kafka/logs 67 | [DEBUG] [2017-01-31_20:17:26] - [OVERRIDE] : SERVER_zookeeper_connect --> zookeeper.connect=localhost:2181 68 | [DEBUG] [2017-01-31_20:17:26] - [OVERRIDE] : SERVER_broker_id --> broker.id=-1 69 | ....................................... 70 | 71 | Writing environment variables to file : 72 | 73 | PREFIX : ZK_ 74 | DEST_FILE : /opt/kafka/config/zookeeper.properties 75 | EXCLUSIONS : 76 | CREATE_FILE : true 77 | OVERRIDE : true 78 | FROM_SEPARATOR : _ 79 | TO_SEPARATOR : . 80 | LOWER : false 81 | ....................................... 82 | 83 | [DEBUG] [2017-01-31_20:17:26] - [OVERRIDE] : ZK_dataDir --> dataDir=/opt/kafka/zookeeper/data 84 | [DEBUG] [2017-01-31_20:17:26] - [OVERRIDE] : ZK_clientPort --> clientPort=2181 85 | [DEBUG] [2017-01-31_20:17:26] - [ ADD ] : ZK_dataLogDir --> dataLogDir=/opt/kafka/zookeeper/data-log 86 | ... 87 | [2017-01-31 20:17:28,150] INFO Socket connection established to localhost/127.0.0.1:2181, initiating session (org.apache.zookeeper.ClientCnxn) 88 | [2017-01-31 20:17:28,308] INFO Session establishment complete on server localhost/127.0.0.1:2181, sessionid = 0x159f62cc8c00000, negotiated timeout = 6000 (org.apache.zookeeper.ClientCnxn) 89 | ... 90 | [2017-01-31 20:17:29,646] INFO Kafka version : 2.3.0 (org.apache.kafka.common.utils.AppInfoParser) 91 | [2017-01-31 20:17:29,646] INFO Kafka commitId : f10ef2720b03b247 (org.apache.kafka.common.utils.AppInfoParser) 92 | [2017-01-31 20:17:29,647] INFO [Kafka Server 1001], started (kafka.server.KafkaServer) 93 | ``` 94 | 95 | >NOTE: We've passed a SETUP_DEBUG environment variable (SETUP_DEBUG=true) to view the setup process details. 96 | 97 | ### Setting up 98 | 99 | Users can provide parameters to config files just adding environment variables with specific name patterns. 100 | 101 | This table collects the patterns of variable names which will are written in each file: 102 | 103 | PREFIX | FILE (${KAFKA_HOME}/config) | Example 104 | -----------|-----------------------------|----------------------------- 105 | SERVER_ | server.properties | SERVER_broker_id=1 --> broker.id=1 106 | LOG4J_ | log4j.properties | LOG4J_log4j_rootLogger=INFO, stdout--> log4j.rootLogger=INFO, stdout 107 | CONSUMER_ | consumer.properties| CONSUMER_zookeeper_connect=127.0.0.1:2181 --> zookeeper.connect=127.0.0.1:2181 108 | PRODUCER_ | producer.properties| PRODUCER_compression_type=none --> compression.type=none 109 | ZK_ | zookeeper.properties | ZK_maxClientCnxns=0 --> maxClientCnxns=0 110 | CONN_CONSOLE_SINK_ |connect-console-sink.properties | CONN_CONSOLE_SINK_tasks_max=1 --> tasks.max=1 111 | CONN_CONSOLE_SOURCE_ | connect-console-source.properties | CONN_CONSOLE_SOURCE_topic=connect-test --> topic=connect-test 112 | CONN_DISTRIB_ | connect-distributed.properties | CONN_DISTRIB_group_id=connect-cluster --> group.id=connect-cluster 113 | CONN_FILE_SINK_ | connect-file-sink.properties | CONN_FILE_SINK_connector_class=FileStreamSink --> connector.class=FileStreamSink 114 | CONN_FILE_SOURCE_ | connect-file-source.properties | CONN_FILE_SOURCE_tasks_max=1 --> tasks.max=1 115 | CONN_LOG4J_ | connect-log4j.properties | CONN_LOG4J_log4j_rootLogger=INFO, stdout --> log4j.rootLogger=INFO, stdout 116 | CONN_STANDALONE_ | connect-standalone.properties | CONN_STANDALONE_bootstrap_servers=localhost:9092 --> bootstrap.servers=localhost:9092 117 | TOOLS_LOG4J_ | tools-log4j.properties | TOOLS_LOG4J_log4j_appender_stderr_Target=System.err --> log4j.appender.stderr.Target=System.err 118 | 119 | So we can configure our kafka server via environment variables directly: 120 | 121 | ```bash 122 | $ docker run -it -d -e "LOG4J_log4j_rootLogger=DEBUG, stdout" -e "SERVER_log_retention_hours=24"\ 123 | engapa/kafka:${SCALA_VERSION}-${KAFKA_VERSION} 124 | ``` 125 | 126 | Also you may use `--env-file` option to load these variables from a file. 127 | 128 | And, of course, you could provide your own property files directly by option `-v` with the suitable properties files. 129 | 130 | The override option of kafka server is preserved and anybody can use it on this way: 131 | 132 | ```bash 133 | $ docker run -it \ 134 | -e "SETUP_DEBUG=true" \ 135 | -h kafka engapa/kafka:${SCALA_VERSION}-${KAFKA_VERSION} \ 136 | /bin/bash -c "kafka_server.sh start --override advertised.host.name=kafka" 137 | [2017-02-04 19:06:10,504] INFO KafkaConfig values: 138 | advertised.host.name = kafka 139 | ... 140 | [2017-02-04 19:06:11,693] INFO [Kafka Server 1001], started (kafka.server.KafkaServer) 141 | ``` 142 | 143 | #### Run local zookeeper 144 | 145 | By default a zookeeper process is started too, as we said previously. 146 | This behaviour is managed by the env variable KAFKA_ZK_LOCAL (defaults to "true"). 147 | 148 | #### External zookeeper 149 | 150 | If you want to deploy a kafka server without a local zookeeper then you should provide these env values: 151 | 152 | * KAFKA_ZK_LOCAL=false 153 | * SERVER_zookeeper_connect=\\[,\\] 154 | 155 | For instance: 156 | 157 | ```bash 158 | $ docker run -it \ 159 | -e "KAFKA_ZK_LOCAL=false" \ 160 | -e "SERVER_zookeeper_connect=zookeeperserver1:2181,zookeeperserver2:2181,zookeeperserver3:2181" \ 161 | engapa/kafka:${SCALA_VERSION}-${KAFKA_VERSION} 162 | ``` 163 | 164 | ## Kubernetes 165 | 166 | In [k8s directory](k8s) there are some examples and utilities for Kubernetes 167 | 168 | ## Openshift 169 | 170 | In [openshift directory](openshift) there are some resources for Openshift. 171 | 172 | ## Extra Dockers 173 | 174 | Another great kafka docker images can be found at: 175 | 176 | - https://hub.docker.com/r/spotify/kafka/ 177 | - https://hub.docker.com/r/wurstmeister/kafka 178 | 179 | ## Author 180 | 181 | Enrique Garcia **engapa@gmail.com** 182 | -------------------------------------------------------------------------------- /k8s/README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes resources 2 | 3 | Here we have some examples of resources that may be deployed on your kubernetes environment. 4 | 5 | Tests were done using the version 1.18 of kubernetes. 6 | 7 | ## Topologies 8 | 9 | We've got two ways to deploy a kafka cluster (and Ephemeral and Persistent modes according the storage type that you prefer): 10 | 11 | Users can choose how to connect to a zookeeper cluster by configuring these parameters: 12 | 13 | * KAFKA_ZK_LOCAL: set to 'true' value if an internal zookeeper process should be run. Change to 'false' if you have a reachable zookeeper cluster to connect to. 14 | * SERVER_zookeeper_connect=\. This property is required if `KAFKA_ZK_LOCAL=false` in other case the connection string will be auto-generated. 15 | 16 | The resource `kafka.yaml` can be launched with internal (`KAFKA_ZK_LOCAL=true`) or external (`KAFKA_ZK_LOCAL=false` and `SERVER_zookeeper_connect`) zookeeper. 17 | Both cases haven't persistent storage and would be appropriated for testing purposes. 18 | 19 | For production environments we recommend you to use resources with suffix `persistent` (`KAFKA_ZK_LOCAL=false` and `SERVER_zookeeper_connect`) or `zk-persistent` (`KAFKA_ZK_LOCAL=true`). 20 | In both cases we'll have persistent storage (even for the zookeeper process). 21 | 22 | ### Examples 23 | #### Ephemeral cluster with Zookeeper sidecar 24 | 25 | Optionally users can choose run an internal zookeeper cluster by configuring these parameters: 26 | 27 | * KAFKA_ZK_LOCAL=true 28 | * SERVER_zookeeper_connect: This property is not required, it will be auto-generated internally. 29 | 30 | ```bash 31 | $ kubectl create -f kafka.yaml 32 | ``` 33 | 34 | > NOTE: params between '[]' characters are optional. 35 | 36 | The number of nodes must be a valid quorum for zookeeper (1, 3, 5, ...). 37 | For example, if you want to have a quorum of 3 zookeeper nodes, then we'll have got 3 kafka brokers too. 38 | 39 | #### Persistent storage with external Zookeeper 40 | 41 | First of all, [deploy a zookeeper cluster](https://github.com/engapa/zookeeper-k8s-openshift). 42 | 43 | ```bash 44 | $ kubectl create -f kafka[-zk]-persistent.yaml 45 | ``` 46 | 47 | ## Local testing 48 | 49 | We recommend to use "minikube" in order to get quickly a ready kafka cluster. 50 | 51 | Install and setup you local kubernetes cluster: 52 | 53 | ```bash 54 | $ minikube get-k8s-versions 55 | $ minikube config get kubernetes-version 56 | ``` 57 | 58 | If no version is showed in last command this means that the latest stable version is being used. 59 | 60 | ```bash 61 | [$ minikube config set kubernetes-version ] 62 | $ minikube start 63 | $ kubectl create -f kafka.yaml 64 | $ minikube dashboard 65 | ``` 66 | 67 | ## Clean up 68 | 69 | To remove all resources related to one kafka cluster deployment launch this command: 70 | 71 | ```bash 72 | $ kubectl delete all,statefulset[,pvc] -l app= [-n |--all-namespaces] 73 | ``` 74 | where '' is the value of param NAME. Note that pvc resources are marked as optional in the command, 75 | it's up to you preserver or not the persistent volumes (by default when a pvc is deleted the persistent volume will be deleted as well). 76 | Type the namespace option if you are in a different namespace that resources are, and indicate --all-namespaces option if all namespaces should be considered. 77 | 78 | It's possible delete all resources created by using the template: 79 | with cluster created by template name: 80 | 81 | ```bash 82 | $ kubectl delete all,statefulset[,pvc] -l template=kafka[-zk][-persistent] [-n ] [--all-namespaces] 83 | ``` 84 | 85 | Also someone can remove all resources of type kafka, belong to all clusters and templates: 86 | 87 | ```bash 88 | $ kubectl delete all,statefulset[,pvc] -l component=kafka [-n ] [--all-namespaces] 89 | ``` 90 | 91 | And finally if you even want to remove the template: 92 | 93 | ```bash 94 | $ kubectl delete template kafka[-zk][-persistent] [-n ] [--all-namespaces] 95 | ``` 96 | -------------------------------------------------------------------------------- /k8s/kafka-persistent.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: kafka-persistent 5 | labels: 6 | app: kafka-persistent 7 | component: kafka 8 | annotations: 9 | service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" 10 | spec: 11 | ports: 12 | - port: 9092 13 | name: server 14 | clusterIP: None 15 | selector: 16 | app: kafka-persistent 17 | component: kafka 18 | --- 19 | apiVersion: apps/v1 20 | kind: StatefulSet 21 | metadata: 22 | name: kafka-persistent 23 | labels: 24 | app: kafka-persistent 25 | component: kafka 26 | # annotations: 27 | ## Use this annotation if you want allocate each pod on different node 28 | ## Note the number of nodes must be upper than REPLICAS parameter. 29 | # scheduler.alpha.kubernetes.io/affinity: > 30 | # { 31 | # "podAntiAffinity": { 32 | # "requiredDuringSchedulingIgnoredDuringExecution": [{ 33 | # "labelSelector": { 34 | # "matchExpressions": [{ 35 | # "key": "app", 36 | # "operator": "In", 37 | # "values": ["kafka"] 38 | # }] 39 | # }, 40 | # "topologyKey": "kubernetes.io/hostname" 41 | # }] 42 | # } 43 | # } 44 | spec: 45 | serviceName: kafka-persistent 46 | selector: 47 | matchLabels: 48 | app: kafka-persistent 49 | component: kafka 50 | replicas: 3 51 | podManagementPolicy: "Parallel" 52 | template: 53 | metadata: 54 | labels: 55 | app: kafka-persistent 56 | component: kafka 57 | spec: 58 | securityContext: 59 | runAsUser: 1001 60 | fsGroup: 1001 61 | containers: 62 | - name: kafka 63 | imagePullPolicy: IfNotPresent 64 | image: engapa/kafka:2.13-2.5.0 65 | resources: 66 | requests: 67 | memory: 512M 68 | cpu: 300m 69 | limits: 70 | memory: 512M 71 | cpu: 300m 72 | ports: 73 | - containerPort: 9092 74 | name: broker-port 75 | env: 76 | - name: KAFKA_REPLICAS 77 | value: "3" 78 | - name: KAFKA_ZK_LOCAL 79 | value: "false" 80 | - name: KAFKA_HEAP_OPTS 81 | value: "-Xmx250M -Xms250M" 82 | - name: SERVER_num_partitions 83 | value: "1" 84 | - name: SERVER_delete_topic_enable 85 | value: "true" 86 | - name: SERVER_log_retention_hours 87 | value: "2147483647" 88 | - name: SERVER_zookeeper_connect 89 | value: "zk-0.zk:2181,zk-1.zk:2181,zk-2.zk:2181" 90 | - name: SERVER_log_dirs 91 | value: "/opt/kafka/data/logs" 92 | - name: SERVER_zookeeper_connection_timeout_ms 93 | value: "300000" 94 | livenessProbe: 95 | exec: 96 | command: 97 | - kafka_server_status.sh 98 | initialDelaySeconds: 30 99 | timeoutSeconds: 5 100 | readinessProbe: 101 | exec: 102 | command: 103 | - kafka_server_status.sh 104 | initialDelaySeconds: 30 105 | timeoutSeconds: 5 106 | volumeMounts: 107 | - name: data 108 | mountPath: /opt/kafka/data 109 | volumeClaimTemplates: 110 | - metadata: 111 | name: data 112 | spec: 113 | accessModes: [ "ReadWriteOnce" ] 114 | resources: 115 | requests: 116 | storage: 1Gi 117 | -------------------------------------------------------------------------------- /k8s/kafka-zk-persistent.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: kafka-zk-persistent 6 | labels: 7 | app: kafka-zk-persistent 8 | component: kafka 9 | annotations: 10 | service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" 11 | spec: 12 | ports: 13 | - port: 9092 14 | name: server 15 | - port: 2181 16 | name: zkclient 17 | - port: 2888 18 | name: zkserver 19 | - port: 3888 20 | name: zkleader 21 | clusterIP: None 22 | selector: 23 | app: kafka-zk-persistent 24 | component: kafka 25 | --- 26 | apiVersion: apps/v1 27 | kind: StatefulSet 28 | metadata: 29 | name: kafka-zk-persistent 30 | labels: 31 | app: kafka-zk-persistent 32 | component: kafka 33 | spec: 34 | podManagementPolicy: "Parallel" 35 | serviceName: kafka 36 | selector: 37 | matchLabels: 38 | app: kafka-zk-persistent 39 | component: kafka 40 | replicas: 1 41 | template: 42 | metadata: 43 | labels: 44 | app: kafka-zk-persistent 45 | component: kafka 46 | # annotations: 47 | # # Use this annotation if you want allocate each pod on different node 48 | # # Note the number of nodes must be upper than REPLICAS parameter. 49 | # scheduler.alpha.kubernetes.io/affinity: > 50 | # { 51 | # "podAntiAffinity": { 52 | # "requiredDuringSchedulingIgnoredDuringExecution": [{ 53 | # "labelSelector": { 54 | # "matchExpressions": [{ 55 | # "key": "app", 56 | # "operator": "In", 57 | # "values": ["kafka"] 58 | # }] 59 | # }, 60 | # "topologyKey": "kubernetes.io/hostname" 61 | # }] 62 | # } 63 | # } 64 | spec: 65 | securityContext: 66 | runAsUser: 1001 67 | fsGroup: 1001 68 | containers: 69 | - name: kafka 70 | imagePullPolicy: IfNotPresent 71 | image: engapa/kafka:2.13-2.5.0 72 | resources: 73 | requests: 74 | memory: 1024M 75 | cpu: 1 76 | limits: 77 | memory: 1024M 78 | cpu: 1 79 | ports: 80 | - containerPort: 9092 81 | name: server 82 | - containerPort: 2181 83 | name: zkclient 84 | - containerPort: 2888 85 | name: zkserver 86 | - containerPort: 3888 87 | name: zkleader 88 | env: 89 | - name : KAFKA_REPLICAS 90 | value: "3" 91 | - name: KAFKA_ZK_LOCAL 92 | value: "true" 93 | - name: SERVER_num_partitions 94 | value: "1" 95 | - name: SERVER_delete_topic_enable 96 | value: "true" 97 | - name: SERVER_log_retention_hours 98 | value: "2147483647" 99 | - name: SERVER_zookeeper_connect 100 | value: "kafka-zk-0.kafka-zk:2181,kafka-zk-1.kafka-zk:2181,kafka-zk-2.kafka-zk:2181" 101 | - name: SERVER_log_dirs 102 | value: "/opt/kafka/data/logs" 103 | - name: SERVER_zookeeper_connection_timeout_ms 104 | value: "30000" 105 | # TODO: Resolve HostNotFound exception of zookeeper servers while kafka is starting up 106 | readinessProbe: 107 | exec: 108 | command: 109 | - kafka_server_status.sh 110 | initialDelaySeconds: 30 111 | timeoutSeconds: 5 112 | livenessProbe: 113 | exec: 114 | command: 115 | - kafka_server_status.sh 116 | initialDelaySeconds: 30 117 | timeoutSeconds: 5 118 | volumeMounts: 119 | - name: kafka-data 120 | mountPath: /opt/kafka/data 121 | - name: zk-data 122 | mountPath: /opt/kafka/zookeeper/data 123 | - name: zk-datalog 124 | mountPath: /opt/kafka/zookeeper/data-log 125 | volumeClaimTemplates: 126 | - metadata: 127 | name: kafka-data 128 | spec: 129 | accessModes: [ "ReadWriteOnce" ] 130 | resources: 131 | requests: 132 | storage: 1Gi 133 | - metadata: 134 | name: zk-data 135 | spec: 136 | accessModes: [ "ReadWriteOnce" ] 137 | resources: 138 | requests: 139 | storage: 1Gi 140 | - metadata: 141 | name: zk-datalog 142 | spec: 143 | accessModes: [ "ReadWriteOnce" ] 144 | resources: 145 | requests: 146 | storage: 1Gi 147 | -------------------------------------------------------------------------------- /k8s/kafka-zk.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: kafka-zk 5 | labels: 6 | component: kafka 7 | annotations: 8 | service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" 9 | spec: 10 | ports: 11 | - port: 9092 12 | name: server 13 | - port: 2181 14 | name: zkclient 15 | - port: 2888 16 | name: zkserver 17 | - port: 3888 18 | name: zkleader 19 | clusterIP: None 20 | selector: 21 | app: kafka-zk 22 | component: kafka 23 | --- 24 | apiVersion: apps/v1 25 | kind: StatefulSet 26 | metadata: 27 | name: kafka-zk 28 | labels: 29 | app: kafka-zk 30 | component: kafka 31 | spec: 32 | serviceName: kafka-zk 33 | selector: 34 | matchLabels: 35 | app: kafka-zk 36 | component: kafka 37 | replicas: 3 38 | podManagementPolicy: "Parallel" 39 | template: 40 | metadata: 41 | labels: 42 | app: kafka-zk 43 | component: kafka 44 | spec: 45 | securityContext: 46 | runAsUser: 1001 47 | fsGroup: 1001 48 | containers: 49 | - name: kafka 50 | imagePullPolicy: IfNotPresent 51 | image: engapa/kafka:2.13-2.5.0 52 | resources: 53 | requests: 54 | memory: 512M 55 | cpu: 300m 56 | limits: 57 | memory: 512M 58 | cpu: 300m 59 | ports: 60 | - containerPort: 9092 61 | name: broker-port 62 | - containerPort: 2181 63 | name: zkclient 64 | - containerPort: 2888 65 | name: zkserver 66 | - containerPort: 3888 67 | name: zkleader 68 | env: 69 | - name: KAFKA_REPLICAS 70 | value: "3" 71 | - name: KAFKA_ZK_LOCAL 72 | value: "true" 73 | - name: SERVER_zookeeper_connection_timeout_ms 74 | value: "300000" 75 | # livenessProbe: 76 | # exec: 77 | # command: 78 | # - kafka_server_status.sh 79 | # initialDelaySeconds: 60 80 | # timeoutSeconds: 60 81 | # readinessProbe: 82 | # exec: 83 | # command: 84 | # - kafka_server_status.sh 85 | # initialDelaySeconds: 60 86 | # timeoutSeconds: 60 -------------------------------------------------------------------------------- /k8s/main.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | MINIKUBE_VERSION=${MINIKUBE_VERSION:-"v1.9.2"} 6 | KUBE_VERSION=${KUBE_VERSION:-"v1.18.2"} 7 | 8 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 9 | 10 | DISTRO=$(uname -s | tr '[:upper:]' '[:lower:]') 11 | 12 | function kubectl-install() 13 | { 14 | 15 | if [[ "${KUBE_VERSION}" == 'latest' ]]; then 16 | KUBE_VERSION=$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt) 17 | fi 18 | 19 | # Download kubectl 20 | curl -L -o kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBE_VERSION}/bin/$DISTRO/amd64/kubectl 21 | chmod +x kubectl 22 | sudo mv kubectl /usr/local/bin/ 23 | mkdir -p ${HOME}/.kube 24 | touch ${HOME}/.kube/config 25 | 26 | } 27 | 28 | function minikube-install() 29 | { 30 | # Download minikube 31 | curl -L -o minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-$DISTRO-amd64 32 | chmod +x minikube 33 | sudo install minikube /usr/local/bin/ 34 | 35 | } 36 | 37 | function minikube-run() 38 | { 39 | 40 | export MINIKUBE_WANTUPDATENOTIFICATION=false 41 | export MINIKUBE_WANTREPORTERRORPROMPT=false 42 | export MINIKUBE_HOME=$HOME 43 | export CHANGE_MINIKUBE_NONE_USER=true 44 | export KUBECONFIG=$HOME/.kube/config 45 | 46 | sudo -E minikube start --driver=none --cpus 2 --memory 3062 --kubernetes-version=${KUBE_VERSION} 47 | 48 | # this for loop waits until kubectl can access the api server that Minikube has created 49 | for i in {1..150}; do # timeout for 5 minutes 50 | kubectl version &> /dev/null 51 | if [ $? -ne 1 ]; then 52 | break 53 | fi 54 | sleep 2 55 | done 56 | 57 | # Check kubernetes info 58 | kubectl cluster-info 59 | # RBAC 60 | kubectl create clusterrolebinding add-on-cluster-admin --clusterrole cluster-admin --serviceaccount=kube-system:default 61 | # Install Helm 62 | # curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash 63 | } 64 | 65 | # $1 : file 66 | # $2 : Number of replicas 67 | function check() 68 | { 69 | SLEEP_TIME=10 70 | MAX_ATTEMPTS=50 71 | ATTEMPTS=0 72 | READY_REPLICAS="0" 73 | REPLICAS=${2:-1} 74 | until [[ "$READY_REPLICAS" == "$REPLICAS" ]]; do 75 | sleep $SLEEP_TIME 76 | ATTEMPTS=`expr $ATTEMPTS + 1` 77 | if [[ $ATTEMPTS -gt $MAX_ATTEMPTS ]]; then 78 | echo "ERROR: Max number of attempts was reached (${MAX_ATTEMPTS})" 79 | exit 1 80 | fi 81 | READY_REPLICAS=$(kubectl get -f $1 -o jsonpath='{.items[?(@.kind=="StatefulSet")].status.readyReplicas}' 2>&1) 82 | echo "[${ATTEMPTS}/${MAX_ATTEMPTS}] - Ready replicas : ${READY_REPLICAS:-0}/$REPLICAS ... " 83 | done 84 | kubectl get all 85 | } 86 | 87 | function test-zk() 88 | { 89 | # Given 90 | file=$DIR/kafka-zk.yaml 91 | # When 92 | kubectl create -f $file 93 | # Then 94 | check $file 3 95 | 96 | } 97 | 98 | function test-persistent() 99 | { 100 | # Given 101 | # A zookeeper cluster is deployed previously with three replicas 102 | echo "Deploying zookeeper cluster ..." 103 | file_zk=$DIR/zk.yaml 104 | kubectl create -f $file_zk 105 | check $file_zk 1 106 | 107 | file=$DIR/kafka-persistent.yaml 108 | # When 109 | echo "Deploying kafka cluster with persistent storage ..." 110 | kubectl create -f $file 111 | # Then 112 | check $file 3 113 | 114 | kubectl get pvc,pv 115 | } 116 | 117 | function test-zk-persistent() 118 | { 119 | # Given 120 | file=$DIR/kafka-zk-persistent.yaml 121 | # When 122 | kubectl create -f $file 123 | # Then 124 | check $file 1 125 | 126 | kubectl get pvc,pv 127 | } 128 | 129 | function test-all() 130 | { 131 | test && kubectl delete -l component=kafka all && delete -l component=zk all 132 | test-persistent && kubectl delete -l component=kafka all,pv,pvc && kubectl delete -l component=zk all,pv,pvc 133 | } 134 | 135 | function clean-resources() 136 | { 137 | echo "Cleaning resources ...." 138 | kubectl delete -l component=kafka all,pv,pvc 139 | kubectl delete -l component=zk all,pv,pvc 140 | } 141 | 142 | function minikube-delete(){ 143 | 144 | echo "Deleting minikube cluster ...." 145 | minikube delete 146 | 147 | } 148 | function help() # Show a list of functions 149 | { 150 | declare -F -p | cut -d " " -f 3 151 | } 152 | 153 | if [[ "_$1" = "_" ]]; then 154 | help 155 | else 156 | "$@" 157 | fi 158 | -------------------------------------------------------------------------------- /k8s/zk.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: zk 5 | labels: 6 | zk-name: zk 7 | component: zk 8 | annotations: 9 | service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" 10 | spec: 11 | ports: 12 | - port: 2181 13 | name: zkclient 14 | - port: 2888 15 | name: zkserver 16 | - port: 3888 17 | name: zkleader 18 | clusterIP: None 19 | selector: 20 | zk-name: zk 21 | component: zk 22 | --- 23 | apiVersion: apps/v1 24 | kind: StatefulSet 25 | metadata: 26 | name: zk 27 | labels: 28 | zk-name: zk 29 | component: zk 30 | spec: 31 | serviceName: zk 32 | selector: 33 | matchLabels: 34 | zk-name: zk 35 | component: zk 36 | replicas: 1 37 | template: 38 | metadata: 39 | labels: 40 | zk-name: zk 41 | component: zk 42 | spec: 43 | securityContext: 44 | runAsUser: 1001 45 | fsGroup: 1001 46 | containers: 47 | - name: zk 48 | imagePullPolicy: IfNotPresent 49 | image: engapa/zookeeper:3.4.14 50 | resources: 51 | requests: 52 | memory: 512M 53 | cpu: 300m 54 | limits: 55 | memory: 512M 56 | cpu: 300m 57 | ports: 58 | - containerPort: 2181 59 | name: zkclient 60 | - containerPort: 2888 61 | name: zkserver 62 | - containerPort: 3888 63 | name: zkleader 64 | env: 65 | - name: ZOO_REPLICAS 66 | value: "1" 67 | - name: JAVA_ZK_JVMFLAG 68 | value: "\"-Xmx512M -Xms512M\"" 69 | readinessProbe: 70 | exec: 71 | command: 72 | - zk_status.sh 73 | initialDelaySeconds: 20 74 | timeoutSeconds: 10 75 | livenessProbe: 76 | exec: 77 | command: 78 | - zk_status.sh 79 | initialDelaySeconds: 20 80 | timeoutSeconds: 10 81 | -------------------------------------------------------------------------------- /kafka_download.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | function download_kafka_release () { 4 | 5 | KAFKA_VERSION=${1:-${KAFKA_VERSION}} 6 | SCALA_VERSION=${2:-${SCALA_VERSION}} 7 | KAFKA_HOME=${3:-${KAFKA_HOME}} 8 | 9 | if [[ -z $KAFKA_VERSION || -z $SCALA_VERSION || -z $KAFKA_HOME ]]; then 10 | echo 'KAFKA_VERSION, SCALA_VERSION and KAFKA_HOME are required values.' && exit 1; 11 | fi 12 | 13 | URL_PREFIX="https://dist.apache.org/repos/dist/release/kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}" 14 | 15 | wget -q -O /tmp/kafka.tgz "${URL_PREFIX}.tgz" 16 | wget -q -O /tmp/kafka.asc "${URL_PREFIX}.tgz.asc" 17 | 18 | wget -q -O /tmp/KEYS https://kafka.apache.org/KEYS 19 | gpg -q --import /tmp/KEYS 20 | 21 | gpg -q --batch --verify /tmp/kafka.asc /tmp/kafka.tgz 22 | tar -xzf /tmp/kafka.tgz --strip-components 1 -C $KAFKA_HOME 23 | 24 | rm -rf /tmp/kafka.{asc,tgz} \ 25 | /tmp/KEYS 26 | rm -rf $KAFKA_HOME/{NOTICE,LICENSE} \ 27 | $KAFKA_HOME/site-docs \ 28 | $KAFKA_HOME/bin/windows 29 | 30 | } 31 | 32 | function download_kafka_utils() { 33 | 34 | wget -q -O ${KAFKA_HOME}/bin/kafka_common_functions.sh https://raw.githubusercontent.com/engapa/utils-docker/master/common-functions.sh 35 | 36 | } 37 | 38 | download_kafka_release "$@" && download_kafka_utils -------------------------------------------------------------------------------- /kafka_env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | ### Default properties 4 | 5 | # For Zookeeper versions since 3.5, we really need "ruok" to check zookeeper cluster status, but for the future ... 6 | export ZK_4lw_commands_whitelist=* 7 | 8 | function zk_local_cluster() { 9 | 10 | # Required envs for replicated mode 11 | export ZK_tickTime=${ZK_tickTime:-2000} 12 | export ZK_initLimit=${ZK_initLimit:-5} 13 | export ZK_syncLimit=${ZK_syncLimit:-2} 14 | 15 | ZOO_SERVER_PORT=${ZOO_SERVER_PORT:-2888} 16 | ZOO_ELECTION_PORT=${ZOO_ELECTION_PORT:-3888} 17 | 18 | SERVER_zookeeper_connect='' 19 | 20 | for (( i=1; i<=$KAFKA_REPLICAS; i++ )); do 21 | export ZK_server_$i="$NAME-$((i-1)).$DOMAIN:$ZOO_SERVER_PORT:$ZOO_ELECTION_PORT" 22 | SERVER_zookeeper_connect=${SERVER_zookeeper_connect}"$NAME-$((i-1)).$DOMAIN:"${ZK_clientPort}"," 23 | done 24 | 25 | export SERVER_zookeeper_connect=${SERVER_zookeeper_connect::-1} 26 | 27 | } 28 | 29 | if [[ "x$KAFKA_ZK_LOCAL" == "xtrue" ]];then 30 | export ZK_dataDir=${ZK_dataDir:-$KAFKA_HOME/zookeeper/data} 31 | export ZK_dataLogDir=${ZK_dataLogDir:-$KAFKA_HOME/zookeeper/data-log} 32 | export ZK_clientPort=${ZK_clientPort:-2181} 33 | export SERVER_zookeeper_connect=${SERVER_zookeeper_connect:-"localhost:${ZK_clientPort}"} 34 | fi 35 | 36 | HOST=`hostname -s` 37 | DOMAIN=`hostname -d` 38 | 39 | if [ $KAFKA_REPLICAS -gt 1 ];then 40 | if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then 41 | NAME=${BASH_REMATCH[1]} 42 | ORD=${BASH_REMATCH[2]} 43 | export SERVER_broker_id=$((ORD+1)) 44 | if $KAFKA_ZK_LOCAL;then 45 | zk_local_cluster 46 | fi 47 | elif $KAFKA_ZK_LOCAL; then 48 | echo "Unable to create local Zookeeper. Name of host doesn't match with pattern: (.*)-([0-9]+). Consider Kubernetes StatefulSets." 49 | exit 1 50 | fi 51 | fi 52 | 53 | export SERVER_log_dirs=${SERVER_log_dirs:-$KAFKA_HOME/logs} -------------------------------------------------------------------------------- /kafka_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ### Default properties 4 | 5 | . kafka_env.sh 6 | 7 | KAFKA_PID_FILE=$KAFKA_HOME/kafka.pid 8 | ZOO_PID_FILE=$KAFKA_HOME/zookeeper.pid 9 | 10 | function start() { 11 | 12 | . kafka_setup.sh 13 | 14 | if $KAFKA_ZK_LOCAL;then 15 | 16 | mkdir -p ${ZK_dataDir} ${ZK_dataLogDir} 17 | echo "${SERVER_broker_id:-0}" > $ZK_dataDir/myid 18 | 19 | ZOO_LOG_DIR=${ZOO_LOG_DIR:-${KAFKA_HOME}/zookeeper} 20 | mkdir -p $ZOO_LOG_DIR 21 | 22 | KAFKA_HEAP_OPTS="${ZOO_HEAP_OPTS:-}" $KAFKA_HOME/bin/zookeeper-server-start.sh $KAFKA_CONF_DIR/zookeeper.properties > $ZOO_LOG_DIR/zookeeper.out 2>&1 < /dev/null & 23 | ZOO_PID=$! 24 | if [ $? -eq 0 ]; then 25 | echo -n $ZOO_PID > $ZOO_PID_FILE 26 | else 27 | echo "Failed to start zookeeper server" 28 | return 1 29 | fi 30 | # Is zookeeper ok 31 | COUNT_DOWN_ZOO=20 32 | 33 | until [[ $COUNT_DOWN_ZOO -lt 0 ]]; do 34 | if [[ 'imok' == $(echo ruok | nc 127.0.0.1 $ZK_clientPort) ]]; then 35 | echo "Zookeeper server is ready ! " 36 | break; 37 | fi 38 | if [[ $COUNT_DOWN_ZOO -eq 0 ]]; then 39 | echo "No Zookeeper server is ready" 40 | return 1 41 | fi 42 | echo "Waiting for local Zookeper server ..." 43 | COUNT_DOWN_ZOO=`expr $COUNT_DOWN_ZOO - 1` 44 | sleep 3 45 | done 46 | fi 47 | 48 | $KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_CONF_DIR/server.properties "$@" & 49 | KAFKA_PID=$! 50 | RET_CODE=$? 51 | if [[ $RET_CODE -eq 0 ]]; then 52 | echo -n $! > $KAFKA_PID_FILE 53 | else 54 | echo "Failed to start kafka server" 55 | return 1 56 | fi 57 | wait $KAFKA_PID 58 | 59 | } 60 | 61 | function stop() { 62 | 63 | $KAFKA_HOME/bin/kafka-server-stop.sh 64 | if [[ -f $KAFKA_PID_FILE ]]; then 65 | kill -s TERM $(cat "$KAFKA_PID_FILE") 66 | rm $KAFKA_PID_FILE 67 | fi 68 | 69 | sleep 5 70 | 71 | if $KAFKA_ZK_LOCAL;then 72 | $KAFKA_HOME/bin/zookeeper-server-stop.sh 73 | if [[ -f $ZOO_PID_FILE ]]; then 74 | kill -s TERM $(cat "$ZOO_PID_FILE") 75 | rm $ZOO_PID_FILE 76 | fi 77 | fi 78 | 79 | sleep 5 80 | COUNT_DOWN=9 81 | 82 | until [[ $COUNT_DOWN -lt 0 ]]; do 83 | KAFKA_JAVA_PIDS=$(ps ax | grep -i 'kafka' | grep java | grep -v grep | awk '{print $1}') 84 | if [[ -z $KAFKA_JAVA_PIDS ]]; then 85 | echo "It seems that there is not a kafka server running yet - OK, :-)" 86 | return 0 87 | elif [[ $COUNT_DOWN -gt 0 ]]; then 88 | echo "(${COUNT_DOWN}) Trying stop processes: ${KAFKA_JAVA_PIDS}" 89 | kill -s TERM $KAFKA_JAVA_PIDS 90 | else 91 | echo "(${COUNT_DOWN}) Killing processes: ${KAFKA_JAVA_PIDS}" 92 | kill -s KILL $KAFKA_JAVA_PIDS 93 | fi 94 | COUNT_DOWN=`expr $COUNT_DOWN - 1` 95 | sleep 3 96 | done 97 | 98 | return 1 99 | 100 | } 101 | 102 | trap "stop" SIGHUP SIGINT SIGTERM 103 | 104 | # Main options 105 | case "$1" in 106 | start) 107 | shift 108 | start "$@" 109 | exit $? 110 | ;; 111 | stop) 112 | stop 113 | exit $? 114 | ;; 115 | restart) 116 | shift 117 | stop 118 | sleep 3 119 | start "$@" 120 | exit $? 121 | ;; 122 | *) 123 | echo "Usage: $0 {start|stop|restart}" 124 | exit 1 125 | esac -------------------------------------------------------------------------------- /kafka_server_status.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ### Default properties 4 | 5 | timeout -t 30 $KAFKA_HOME/bin/kafka-topics.sh --bootstrap-server localhost:${SERVER_port} --list > /dev/null 2>&1 6 | if [[ $? -eq 0 ]]; then 7 | echo -e "\033[32mKafka server is running, :-)\033[0m" 8 | exit 0 9 | else 10 | echo -e "\033[31mKafka server is not running, :-(\033[0m" 11 | exit 1 12 | fi -------------------------------------------------------------------------------- /kafka_setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | ### Default properties 4 | 5 | export KAFKA_CONF_DIR=$KAFKA_HOME/config 6 | export SERVER_log_dirs=${SERVER_log_dirs:-"$KAFKA_DATA_DIR/logs"} 7 | 8 | 9 | . ${KAFKA_HOME}/bin/kafka_common_functions.sh 10 | 11 | DEBUG=${SETUP_DEBUG:-false} 12 | LOWER=${SETUP_LOWER:-false} 13 | 14 | # Server 15 | PREFIX=SERVER_ DEST_FILE=${KAFKA_CONF_DIR}/server.properties env_vars_in_file 16 | # Log4j 17 | PREFIX=LOG4J_ DEST_FILE=${KAFKA_CONF_DIR}/log4j.properties env_vars_in_file 18 | # Consumer 19 | PREFIX=CONSUMER_ DEST_FILE=${KAFKA_CONF_DIR}/consumer.properties env_vars_in_file 20 | # Producer 21 | PREFIX=PRODUCER_ DEST_FILE=${KAFKA_CONF_DIR}/producer.properties env_vars_in_file 22 | # Zookeeper 23 | PREFIX=ZK_ DEST_FILE=${KAFKA_CONF_DIR}/zookeeper.properties env_vars_in_file 24 | # Connect 25 | PREFIX=CONN_CONSOLE_SINK_ DEST_FILE=${KAFKA_CONF_DIR}/connect-console-sink.properties env_vars_in_file 26 | PREFIX=CONN_CONSOLE_SOURCE_ DEST_FILE=${KAFKA_CONF_DIR}/connect-console-source.properties env_vars_in_file 27 | PREFIX=CONN_DISTRIB_ DEST_FILE=${KAFKA_CONF_DIR}/connect-distributed.properties env_vars_in_file 28 | PREFIX=CONN_FILE_SINK_ DEST_FILE=${KAFKA_CONF_DIR}/connect-file-sink.properties env_vars_in_file 29 | PREFIX=CONN_FILE_SOURCE_ DEST_FILE=${KAFKA_CONF_DIR}/connect-file-source.properties env_vars_in_file 30 | PREFIX=CONN_LOG4J_ DEST_FILE=${KAFKA_CONF_DIR}/connect-log4j.properties env_vars_in_file 31 | PREFIX=CONN_STANDALONE_ DEST_FILE=${KAFKA_CONF_DIR}/connect-standalone.properties env_vars_in_file 32 | # Tools log4j 33 | PREFIX=TOOLS_LOG4J_ DEST_FILE=${KAFKA_CONF_DIR}/tools-log4j.properties env_vars_in_file 34 | 35 | # Ensure permission on possible mount volumes 36 | for dir in $KAFKA_DATA_DIR $KAFKA_HOME/zookeeper $SERVER_log_dirs; do 37 | if [[ ! -d $dir ]]; then 38 | echo "Creating directory $dir ..." 39 | mkdir -p $dir 40 | else 41 | echo "Ensuring permission for directory $dir ..." 42 | sudo chown -R $KAFKA_USER:$KAFKA_GROUP $dir 43 | fi 44 | done 45 | -------------------------------------------------------------------------------- /openshift/README.md: -------------------------------------------------------------------------------- 1 | # Kafka cluster 2 | 3 | Kafka cluster deployment. 4 | 5 | The resources here are templates for Openshift catalog. 6 | 7 | It isn't necessary to clone this repo, you can use the resources directly trough the URLs ("https://raw.githubusercontent.com/engapa/kafka-k8s-openshift/master/openshift/\". 8 | 9 | ## Requirements 10 | 11 | - [oc](https://github.com/openshift/origin/releases) (v3.11) 12 | - [minihift](https://github.com/minishift/minishift) (v1.34.2) 13 | 14 | ### DEV environment 15 | 16 | We'll use only opensource, that's 'openshift origin'. 17 | 18 | [Minishift](https://github.com/minishift/minishift) is the simplest way to get a local Openshift installation on our workstation. 19 | After install the command client check everything is alright to continue: 20 | 21 | ```bash 22 | $ minishift version 23 | minishift v1.34.2+83ebaab 24 | $ minishift start [options] 25 | ... 26 | $ minishift openshift version 27 | openshift v3.11.0+57f8760-31 28 | ``` 29 | >NOTE: minishift has configured the oc client correctly to connect to local Openshift cluster properly. 30 | 31 | ```bash 32 | oc version 33 | oc v3.11.0+0cbc58b 34 | kubernetes v1.11.0+d4cacc0 35 | features: Basic-Auth 36 | 37 | Server https://192.168.2.32:8443 38 | kubernetes v1.11.0+d4cacc0 39 | ``` 40 | 41 | Login with admin user and provide a password: 42 | 43 | ```bash 44 | $ oc login -u admin -p xxxxx 45 | ``` 46 | 47 | Create a new project: 48 | 49 | ```bash 50 | $ oc new-project test 51 | ``` 52 | 53 | You may use the Openshift dashboard (`minishift console`) if you prefer to do those steps through the web interface. 54 | 55 | > TRICK: Login as cluster admin: `oc login -u system:admin -n default`, 56 | change permissions of default scc `oc edit scc restricted` and change runAsUser.type value to RunAsAny. 57 | 58 | For local environment we'll use a non persistent deployments (kafka.yaml) 59 | 60 | ### PROD environment 61 | 62 | To connect to external cluster we need to know the URL to login with your credentials. 63 | 64 | For production environments we'll use zookeeper and kafka deployments with persistence. 65 | 66 | We recommend you to use zookeeper template **zk-persistent.yaml** at https://github.com/engapa/zookeeper-k8s-openshift/tree/master/openshift. 67 | 68 | This means that although pods are destroyed all data are safe under persistent volumes, and when pod are recreated the volumes will be attached again. 69 | 70 | The statefulset objects have an "antiaffinity" pod scheduler policy so pods will be allocated on separated nodes. 71 | It's required the same number of nodes that the max value of parameter `ZOO_REPLICAS` or `KAFKA_REPLICAS`. 72 | 73 | ## Building the image 74 | 75 | This is a recommended step, although you can always use the [public images at dockerhub](https://hub.docker.com/r/engapa/kafka) which are automatically uploaded with CI of this project. 76 | 77 | To build and save a docker image of kafka in your private Openshift registry just follow these instructions: 78 | 79 | 1 - Create an image builder and build the container image 80 | 81 | ```bash 82 | $ oc create -f buildconfig.yaml 83 | $ oc new-app kafka-builder -p GITHUB_REF="v2.13-2.5.0" -p IMAGE_STREAM_VERSION="2.13-2.3.0" 84 | ``` 85 | If you want to get an image from another git commit: 86 | 87 | ```bash 88 | $ oc start-build kafka-builder --commit=master 89 | ``` 90 | 91 | 2 - Check that image is ready: 92 | 93 | ```bash 94 | $ oc get is -l component=zk [-n project] 95 | NAME DOCKER REPO TAGS UPDATED 96 | kafka 172.30.1.1:5000/test/kafka 2.13-2.5.0 1 days ago 97 | ``` 98 | 99 | **NOTE**: If you want to use this local/private image from containers on other projects then use the "\/NAME" value as `SOURCE_IMAGE` parameter value, and use one value of "TAGS" as `KAFKA_VERSION` parameter value (e.g: test/kafka:2.13-2.5.0). 100 | 101 | ## Topologies 102 | 103 | We've got two ways to deploy a kafka cluster (and Ephemeral and Persistent modes according the storage type that you prefer): 104 | 105 | Users can choose how to connect to a zookeeper cluster by configuring these parameters: 106 | 107 | * KAFKA_ZK_LOCAL: set to 'true' value if an internal zookeeper process should be run. Change to 'false' if you have a reachable zookeeper cluster to connect to. 108 | * SERVER_zookeeper_connect=\. This property is required if `KAFKA_ZK_LOCAL=false` in other case the connection string will be auto-generated. 109 | 110 | The resource `kafka.yaml` can be launched with internal (`KAFKA_ZK_LOCAL=true`) or external zookeeper (`KAFKA_ZK_LOCAL=false` and `SERVER_zookeeper_connect`). 111 | Both cases haven't persistent storage and would be appropriated for testing purposes. 112 | 113 | For production environments we recommend you to use the template in file `kafka-persistent` (`KAFKA_ZK_LOCAL=false` and `SERVER_zookeeper_connect` with zookeeper services). 114 | In both cases we'll have persistent storage (even for the zookeeper process). 115 | 116 | ### Examples 117 | 118 | #### Ephemeral cluster with Zookeeper sidecar (DEV environment) 119 | 120 | Optionally users can choose run an internal zookeeper cluster by configuring these parameters: 121 | 122 | * KAFKA_ZK_LOCAL=true 123 | * SERVER_zookeeper_connect: This property is not required, it will be auto-generated internally. 124 | 125 | ```bash 126 | $ oc create -f kafka.yaml 127 | $ oc new-app kafka -p REPLICAS=1 -p ZK_LOCAL=true -p SOURCE_IMAGE=172.30.1.1:5000/test/kafka 128 | ``` 129 | 130 | The number of nodes must be a valid quorum for zookeeper (1, 3, 5, ...). 131 | For example, if you want to have a quorum of 3 zookeeper nodes, then we'll have got 3 kafka brokers too. 132 | 133 | #### Persistent storage with external Zookeeper (PROD environment) 134 | 135 | First of all, [deploy a zookeeper cluster](https://github.com/engapa/zookeeper-k8s-openshift). 136 | 137 | ```bash 138 | $ oc create -f kafka-persistent.yaml 139 | $ oc new-app kafka -p SERVER_zookeeper_connect= -p SOURCE_IMAGE=172.30.1.1:5000/test/kafka 140 | ``` 141 | 142 | ## Clean up 143 | 144 | To remove all resources related to one kafka cluster deployment launch this command: 145 | 146 | ```bash 147 | $ oc delete all,statefulset[,pvc] -l app= [-n |--all-namespaces] 148 | ``` 149 | where '\' is the value of param NAME. Note that pvc resources are marked as optional in the command, 150 | it's up to you preserver or not the persistent volumes (by default when a pvc is deleted the persistent volume will be deleted as well). 151 | Type the namespace option if you are in a different namespace that resources are, and indicate --all-namespaces option if all namespaces should be considered. 152 | 153 | It's possible delete all resources created from this template: 154 | 155 | ```bash 156 | $ oc delete all,statefulset[,pvc] -l template=kafka[-zk][-persistent] [-n ] [--all-namespaces] 157 | ``` 158 | 159 | Also someone can remove all resources of type kafka, belong to all clusters and templates: 160 | 161 | ```bash 162 | $ oc delete all,statefulset[,pvc] -l component=kafka [-n ] [--all-namespaces] 163 | ``` 164 | 165 | To remove the templates: 166 | 167 | ```bash 168 | $ oc delete template kafka-builder 169 | $ oc delete template kafka[-zk][-persistent] [-n ] [--all-namespaces] 170 | ``` 171 | -------------------------------------------------------------------------------- /openshift/buildconfig.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Template 3 | metadata: 4 | name: kafka-builder 5 | annotations: 6 | description: Kafka image build template 7 | openshift.io/display-name: Kafka Image Builder 8 | iconClass: icon-database 9 | tags: database,kafka 10 | labels: 11 | template: kafka-builder 12 | component: kafka 13 | 14 | parameters: 15 | - name: GITHUB_REPOSITORY 16 | value: "https://github.com/engapa/kafka-k8s-openshift" 17 | description: Source code respository (Github) 18 | required: true 19 | - name: GITHUB_REF 20 | description: Github source ref 21 | value: "master" 22 | required: true 23 | - name: GITHUB_HOOK_SECRET 24 | description: Secret to notify a change from Github. 25 | value: "secretito" 26 | required: false 27 | - name: IMAGE_STREAM_NAME 28 | description: 29 | value: "kafka" 30 | required: true 31 | - name: IMAGE_STREAM_VERSION 32 | description: 33 | value: "latest" 34 | required: true 35 | 36 | objects: 37 | - apiVersion: v1 38 | kind: ImageStream 39 | metadata: 40 | name: ${IMAGE_STREAM_NAME} 41 | spec: 42 | lookupPolicy: 43 | local: true 44 | 45 | - kind: BuildConfig 46 | apiVersion: v1 47 | metadata: 48 | name: kafka-builder 49 | spec: 50 | runPolicy: Serial 51 | triggers: 52 | - type: GitHub 53 | github: 54 | secret: ${GITHUB_HOOK_SECRET} 55 | - type: ConfigChange 56 | source: 57 | git: 58 | uri: ${GITHUB_REPOSITORY} 59 | ref: ${GITHUB_REF} 60 | strategy: 61 | type: Docker 62 | output: 63 | to: 64 | kind: ImageStreamTag 65 | name: "${IMAGE_STREAM_NAME}:${IMAGE_STREAM_VERSION}" -------------------------------------------------------------------------------- /openshift/kafka-persistent.yaml: -------------------------------------------------------------------------------- 1 | kind: Template 2 | apiVersion: v1 3 | metadata: 4 | name: kafka-persistent 5 | annotations: 6 | openshift.io/display-name: Kafka (Persistent) 7 | description: Create a Kafka cluster, with persistent storage. 8 | iconClass: icon-database 9 | tags: messaging,kafka 10 | labels: 11 | template: kafka-persistent 12 | component: kafka 13 | parameters: 14 | - name: NAME 15 | description: Name. 16 | required: true 17 | value: kafka-persistent 18 | - name: KAFKA_VERSION 19 | description: Kafka Version (Scala and kafka version). 20 | required: true 21 | value: "2.13-2.5.0" 22 | - name: SOURCE_IMAGE 23 | description: Container image source. 24 | value: kafka 25 | required: true 26 | - name: REPLICAS 27 | description: Number of replicas. 28 | required: true 29 | value: "3" 30 | - name: KAFKA_HEAP_OPTS 31 | description: Kafka JVM Heap options. Consider value of params RESOURCE_MEMORY_REQ and RESOURCE_MEMORY_LIMIT. 32 | required: true 33 | value: "-Xmx256M -Xms256M" 34 | - name: SERVER_NUM_PARTITIONS 35 | description: > 36 | The default number of log partitions per topic. 37 | More partitions allow greater 38 | parallelism for consumption, but this will also result in more files across 39 | the brokers. 40 | required: true 41 | value: "1" 42 | - name: SERVER_DELETE_TOPIC_ENABLE 43 | description: > 44 | Topic deletion enabled. 45 | Switch to enable topic deletion or not, default value is 'true' 46 | value: "true" 47 | - name: SERVER_LOG_RETENTION_HOURS 48 | description: > 49 | Log retention hours. 50 | The minimum age of a log file to be eligible for deletion. 51 | value: "2147483647" 52 | - name: SERVER_ZOOKEEPER_CONNECT 53 | description: > 54 | Zookeeper conection string, a list as URL with nodes separated by ','. 55 | value: "zk.myproject.svc:2181" 56 | required: true 57 | - name: SERVER_ZOOKEEPER_CONNECT_TIMEOUT 58 | description: > 59 | The max time that the client waits to establish a connection to zookeeper (ms). 60 | value: "6000" 61 | required: true 62 | - name: VOLUME_KAFKA_CAPACITY 63 | description: Kafka logs capacity. 64 | required: true 65 | value: "1Gi" 66 | - name: RESOURCE_MEMORY_REQ 67 | description: The memory resource request. 68 | value: "512M" 69 | - name: RESOURCE_MEMORY_LIMIT 70 | description: The limits for memory resource. 71 | value: "512M" 72 | - name: RESOURCE_CPU_REQ 73 | description: The CPU resource request. 74 | value: "300m" 75 | - name: RESOURCE_CPU_LIMIT 76 | description: The limits for CPU resource. 77 | value: "300m" 78 | 79 | objects: 80 | - apiVersion: v1 81 | kind: Service 82 | metadata: 83 | name: ${NAME} 84 | labels: 85 | app: ${NAME} 86 | component: kafka 87 | annotations: 88 | service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" 89 | spec: 90 | ports: 91 | - port: 9092 92 | name: server 93 | clusterIP: None 94 | selector: 95 | app: ${NAME} 96 | component: kafka 97 | - apiVersion: apps/v1 98 | kind: StatefulSet 99 | metadata: 100 | name: ${NAME} 101 | labels: 102 | app: ${NAME} 103 | component: kafka 104 | spec: 105 | podManagementPolicy: "Parallel" 106 | serviceName: ${NAME} 107 | selector: 108 | matchLabels: 109 | app: ${NAME} 110 | component: kafka 111 | replicas: ${REPLICAS} 112 | template: 113 | metadata: 114 | labels: 115 | app: ${NAME} 116 | component: kafka 117 | # annotations: 118 | # # Use this annotation if you want allocate each pod on different node 119 | # # Note the number of nodes must be upper than REPLICAS parameter. 120 | # scheduler.alpha.kubernetes.io/affinity: > 121 | # { 122 | # "podAntiAffinity": { 123 | # "requiredDuringSchedulingIgnoredDuringExecution": [{ 124 | # "labelSelector": { 125 | # "matchExpressions": [{ 126 | # "key": "app", 127 | # "operator": "In", 128 | # "values": ["${NAME}"] 129 | # }] 130 | # }, 131 | # "topologyKey": "kubernetes.io/hostname" 132 | # }] 133 | # } 134 | # } 135 | spec: 136 | securityContext: 137 | runAsUser: 1001 138 | fsGroup: 1001 139 | containers: 140 | - name: ${NAME} 141 | imagePullPolicy: IfNotPresent 142 | image: ${SOURCE_IMAGE}:${KAFKA_VERSION} 143 | resources: 144 | requests: 145 | memory: ${RESOURCE_MEMORY_REQ} 146 | cpu: ${RESOURCE_CPU_REQ} 147 | limits: 148 | memory: ${RESOURCE_MEMORY_LIMIT} 149 | cpu: ${RESOURCE_CPU_LIMIT} 150 | ports: 151 | - containerPort: 9092 152 | name: server 153 | env: 154 | - name : KAFKA_REPLICAS 155 | value: ${REPLICAS} 156 | - name: KAFKA_ZK_LOCAL 157 | value: "false" 158 | - name: KAFKA_HEAP_OPTS 159 | value: ${KAFKA_HEAP_OPTS} 160 | - name: SERVER_num_partitions 161 | value: ${SERVER_NUM_PARTITIONS} 162 | - name: SERVER_delete_topic_enable 163 | value: ${SERVER_DELETE_TOPIC_ENABLE} 164 | - name: SERVER_log_retention_hours 165 | value: ${SERVER_LOG_RETENTION_HOURS} 166 | - name: SERVER_zookeeper_connect 167 | value: ${SERVER_ZOOKEEPER_CONNECT} 168 | - name: SERVER_log_dirs 169 | value: "/opt/kafka/data/logs" 170 | - name: SERVER_zookeeper_connection_timeout_ms 171 | value: ${SERVER_ZOOKEEPER_CONNECT_TIMEOUT} 172 | readinessProbe: 173 | exec: 174 | command: 175 | - kafka_server_status.sh 176 | initialDelaySeconds: 30 177 | timeoutSeconds: 5 178 | livenessProbe: 179 | exec: 180 | command: 181 | - kafka_server_status.sh 182 | initialDelaySeconds: 30 183 | timeoutSeconds: 5 184 | volumeMounts: 185 | - name: kafka-data 186 | mountPath: /opt/kafka/data 187 | volumeClaimTemplates: 188 | - metadata: 189 | name: kafka-data 190 | spec: 191 | accessModes: [ "ReadWriteOnce" ] 192 | resources: 193 | requests: 194 | storage: ${VOLUME_KAFKA_CAPACITY} 195 | selector: 196 | component: kafka 197 | contents: data 198 | -------------------------------------------------------------------------------- /openshift/kafka-zk-persistent.yaml: -------------------------------------------------------------------------------- 1 | kind: Template 2 | apiVersion: v1 3 | metadata: 4 | name: kafka-zk-persistent 5 | annotations: 6 | openshift.io/display-name: Kafka [+Zookeeper] (Persistent) 7 | description: Deploy a Kafka cluster, and an optional zookeeper, with persistent storage. 8 | iconClass: icon-database 9 | tags: messaging,kafka 10 | labels: 11 | template: kafka-zk-persistent 12 | component: kafka 13 | parameters: 14 | - name: NAME 15 | description: Name. 16 | required: true 17 | value: kafka-zk-persistent 18 | - name: KAFKA_VERSION 19 | description: Kafka Version (Scala and kafka version). 20 | required: true 21 | value: "2.13-2.5.0" 22 | - name: SOURCE_IMAGE 23 | description: Container image source. 24 | value: kafka 25 | required: true 26 | - name: REPLICAS 27 | description: Number of replicas. 28 | required: true 29 | value: "1" 30 | - name: ZK_HEAP_OPTS 31 | description: Zookeeper JVM Heap options. Consider value of params RESOURCE_MEMORY_REQ, RESOURCE_MEMORY_LIMIT and KAFKA_HEAP_OPTS. 32 | required: true 33 | value: "-Xmx512M -Xms512M" 34 | - name: KAFKA_HEAP_OPTS 35 | description: Kafka JVM Heap options. Consider value of params RESOURCE_MEMORY_REQ, RESOURCE_MEMORY_LIMIT and ZK_HEAP_OPTS. 36 | required: true 37 | value: "-Xmx512M -Xms512M" 38 | - name: SERVER_NUM_PARTITIONS 39 | description: > 40 | The default number of log partitions per topic. 41 | More partitions allow greater 42 | parallelism for consumption, but this will also result in more files across 43 | the brokers. 44 | required: true 45 | value: "1" 46 | - name: SERVER_DELETE_TOPIC_ENABLE 47 | description: > 48 | Topic deletion enabled. 49 | Switch to enable topic deletion or not, default value is 'true' 50 | value: "true" 51 | - name: SERVER_LOG_RETENTION_HOURS 52 | description: > 53 | Log retention hours. 54 | The minimum age of a log file to be eligible for deletion. 55 | value: "2147483647" 56 | - name: SERVER_ZOOKEEPER_CONNECT_TIMEOUT 57 | description: > 58 | The max time that the client waits to establish a connection to zookeeper (ms). 59 | value: "6000" 60 | required: true 61 | - name: VOLUME_KAFKA_CAPACITY 62 | description: Kafka logs capacity. 63 | required: true 64 | value: "10Gi" 65 | - name: VOLUME_ZK_DATA_CAPACITY 66 | description: Zookeeper data capacity 67 | required: true 68 | value: "1Gi" 69 | - name: VOLUME_ZK_DATALOG_CAPACITY 70 | description: Zookeeper data log capacity 71 | required: true 72 | value: "1Gi" 73 | - name: RESOURCE_MEMORY_REQ 74 | description: The memory resource request. 75 | value: "1Gi" 76 | - name: RESOURCE_MEMORY_LIMIT 77 | description: The limits for memory resource. 78 | value: "1Gi" 79 | - name: RESOURCE_CPU_REQ 80 | description: The CPU resource request. 81 | value: "1" 82 | - name: RESOURCE_CPU_LIMIT 83 | description: The limits for CPU resource. 84 | value: "1" 85 | - name: LP_INITIAL_DELAY 86 | description: > 87 | LivenessProbe initial delay in seconds. 88 | value: "30" 89 | 90 | objects: 91 | - apiVersion: v1 92 | kind: Service 93 | metadata: 94 | name: ${NAME} 95 | labels: 96 | app: ${NAME} 97 | component: kafka 98 | annotations: 99 | service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" 100 | spec: 101 | ports: 102 | - port: 9092 103 | name: server 104 | - port: 2181 105 | name: zkclient 106 | - port: 2888 107 | name: zkserver 108 | - port: 3888 109 | name: zkleader 110 | clusterIP: None 111 | selector: 112 | app: ${NAME} 113 | component: kafka 114 | - apiVersion: apps/v1 115 | kind: StatefulSet 116 | metadata: 117 | name: ${NAME} 118 | labels: 119 | app: ${NAME} 120 | component: kafka 121 | spec: 122 | podManagementPolicy: "Parallel" 123 | serviceName: ${NAME} 124 | selector: 125 | matchLabels: 126 | app: ${NAME} 127 | component: kafka 128 | replicas: ${REPLICAS} 129 | template: 130 | metadata: 131 | labels: 132 | app: ${NAME} 133 | template: kafka-zk-persistent 134 | component: kafka 135 | # annotations: 136 | # # Use this annotation if you want allocate each pod on different node 137 | # # Note the number of nodes must be upper than REPLICAS parameter. 138 | # scheduler.alpha.kubernetes.io/affinity: > 139 | # { 140 | # "podAntiAffinity": { 141 | # "requiredDuringSchedulingIgnoredDuringExecution": [{ 142 | # "labelSelector": { 143 | # "matchExpressions": [{ 144 | # "key": "app", 145 | # "operator": "In", 146 | # "values": ["${NAME}"] 147 | # }] 148 | # }, 149 | # "topologyKey": "kubernetes.io/hostname" 150 | # }] 151 | # } 152 | # } 153 | spec: 154 | containers: 155 | - name: ${NAME} 156 | imagePullPolicy: IfNotPresent 157 | image: ${SOURCE_IMAGE}:${KAFKA_VERSION} 158 | resources: 159 | requests: 160 | memory: ${RESOURCE_MEMORY_REQ} 161 | cpu: ${RESOURCE_CPU_REQ} 162 | limits: 163 | memory: ${RESOURCE_MEMORY_LIMIT} 164 | cpu: ${RESOURCE_CPU_LIMIT} 165 | ports: 166 | - containerPort: 9092 167 | name: server 168 | - containerPort: 2181 169 | name: zkclient 170 | - containerPort: 2888 171 | name: zkserver 172 | - containerPort: 3888 173 | name: zkleader 174 | env: 175 | - name : KAFKA_REPLICAS 176 | value: ${REPLICAS} 177 | - name: KAFKA_ZK_LOCAL 178 | value: "true" 179 | - name : ZOO_HEAP_OPTS 180 | value: ${ZK_HEAP_OPTS} 181 | - name: KAFKA_HEAP_OPTS 182 | value: ${KAFKA_HEAP_OPTS} 183 | - name: SERVER_num_partitions 184 | value: ${SERVER_NUM_PARTITIONS} 185 | - name: SERVER_delete_topic_enable 186 | value: ${SERVER_DELETE_TOPIC_ENABLE} 187 | - name: SERVER_log_retention_hours 188 | value: ${SERVER_LOG_RETENTION_HOURS} 189 | - name: SERVER_zookeeper_connect 190 | value: "localhost:2181" 191 | - name: SERVER_log_dirs 192 | value: "/opt/kafka/data/logs" 193 | - name: SERVER_zookeeper_connection_timeout_ms 194 | value: ${SERVER_ZOOKEEPER_CONNECT_TIMEOUT} 195 | # TODO: Resolve HostNotFound exception of zookeeper servers while kafka is starting up 196 | #readinessProbe: 197 | # exec: 198 | # command: 199 | # - kafka_server_status.sh 200 | # initialDelaySeconds: 15 201 | # timeoutSeconds: 5 202 | livenessProbe: 203 | exec: 204 | command: 205 | - kafka_server_status.sh 206 | initialDelaySeconds: ${LP_INITIAL_DELAY} 207 | timeoutSeconds: 5 208 | securityContext: 209 | runAsUser: 1001 210 | fsGroup: 1001 211 | volumeMounts: 212 | - name: kafka-data 213 | mountPath: /opt/kafka/data 214 | - name: zk-data 215 | mountPath: /opt/kafka/zookeeper/data 216 | - name: zk-datalog 217 | mountPath: /opt/kafka/zookeeper/data-log 218 | volumeClaimTemplates: 219 | - metadata: 220 | name: kafka-data 221 | annotations: 222 | volume.alpha.kubernetes.io/storage-class: anything 223 | spec: 224 | accessModes: [ "ReadWriteOnce" ] 225 | resources: 226 | requests: 227 | storage: ${VOLUME_KAFKA_CAPACITY} 228 | - metadata: 229 | name: zk-data 230 | annotations: 231 | volume.alpha.kubernetes.io/storage-class: anything 232 | spec: 233 | accessModes: [ "ReadWriteOnce" ] 234 | resources: 235 | requests: 236 | storage: ${VOLUME_ZK_DATA_CAPACITY} 237 | - metadata: 238 | name: zk-datalog 239 | annotations: 240 | volume.alpha.kubernetes.io/storage-class: anything 241 | spec: 242 | accessModes: [ "ReadWriteOnce" ] 243 | resources: 244 | requests: 245 | storage: ${VOLUME_ZK_DATALOG_CAPACITY} 246 | -------------------------------------------------------------------------------- /openshift/kafka-zk.yaml: -------------------------------------------------------------------------------- 1 | kind: Template 2 | apiVersion: v1 3 | metadata: 4 | name: kafka-zk 5 | annotations: 6 | openshift.io/display-name: Kafka [+Zookeeper] (Ephemeral) 7 | description: Deploy a Kafka cluster, an optional zookeeper could be deployed too. 8 | iconClass: icon-database 9 | tags: messaging,kafka 10 | labels: 11 | template: kafka-zk 12 | component: kafka 13 | parameters: 14 | - name: NAME 15 | description: Name. 16 | required: true 17 | value: kafka-zk 18 | - name: KAFKA_VERSION 19 | description: Kafka Version (Scala and kafka version). 20 | required: true 21 | value: "2.13-2.5.0" 22 | - name: SOURCE_IMAGE 23 | description: Container image source. 24 | value: kafka 25 | required: true 26 | - name: REPLICAS 27 | description: Number of replicas. 28 | required: true 29 | value: "1" 30 | - name: ZK_HEAP_OPTS 31 | description: Zookeeper JVM Heap options. Consider value of params RESOURCE_MEMORY_REQ, RESOURCE_MEMORY_LIMIT and KAFKA_HEAP_OPTS. 32 | required: true 33 | value: "-Xmx512M -Xms512M" 34 | - name: KAFKA_HEAP_OPTS 35 | description: Kafka JVM Heap options. Consider value of params RESOURCE_MEMORY_REQ, RESOURCE_MEMORY_LIMIT and ZK_HEAP_OPTS. 36 | required: true 37 | value: "-Xmx512M -Xms512M" 38 | - name: SERVER_NUM_PARTITIONS 39 | description: > 40 | The default number of log partitions per topic. 41 | More partitions allow greater 42 | parallelism for consumption, but this will also result in more files across 43 | the brokers. 44 | required: true 45 | value: "1" 46 | - name: SERVER_DELETE_TOPIC_ENABLE 47 | description: > 48 | Topic deletion enabled. 49 | Switch to enable topic deletion or not, default value is 'false'. 50 | value: "false" 51 | - name: SERVER_LOG_RETENTION_HOURS 52 | description: > 53 | Log retention hours. 54 | The minimum age of a log file to be eligible for deletion. 55 | value: "40" 56 | - name: KAFKA_ZK_LOCAL 57 | description: > 58 | Use local zookeeper (*KAFKA_ZK_LOCAL). 59 | Set value to 'true' to start a local zookeeper process into the same container. 60 | required: true 61 | value: "true" 62 | - name: SERVER_ZOOKEEPER_CONNECT 63 | description: > 64 | Zookeeper connection list as URL, nodes separated by ','. 65 | This value takes effect when KAFKA_ZK_LOCAL is false, in other case this value will be auto-generated internally. 66 | value: "localhost:2181" 67 | - name: SERVER_ZOOKEEPER_CONNECT_TIMEOUT 68 | description: > 69 | The max time that the client waits to establish a connection to zookeeper (ms). 70 | value: "6000" 71 | required: true 72 | - name: RESOURCE_MEMORY_REQ 73 | description: The memory resource request. 74 | value: "512M" 75 | - name: RESOURCE_MEMORY_LIMIT 76 | description: The limits for memory resource. 77 | value: "512M" 78 | - name: RESOURCE_CPU_REQ 79 | description: The CPU resource request. 80 | value: "300m" 81 | - name: RESOURCE_CPU_LIMIT 82 | description: The limits for CPU resource. 83 | value: "300m" 84 | 85 | objects: 86 | - apiVersion: v1 87 | kind: Service 88 | metadata: 89 | name: ${NAME} 90 | labels: 91 | component: kafka 92 | annotations: 93 | service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" 94 | spec: 95 | ports: 96 | - port: 9092 97 | name: server 98 | - port: 2181 99 | name: zkclient 100 | - port: 2888 101 | name: zkserver 102 | - port: 3888 103 | name: zkleader 104 | clusterIP: None 105 | selector: 106 | app: ${NAME} 107 | component: kafka 108 | - apiVersion: apps/v1 109 | kind: StatefulSet 110 | metadata: 111 | name: ${NAME} 112 | labels: 113 | app: ${NAME} 114 | component: ${NAME} 115 | spec: 116 | podManagementPolicy: "Parallel" 117 | serviceName: ${NAME} 118 | selector: 119 | matchLabels: 120 | app: ${NAME} 121 | component: kafka 122 | replicas: ${REPLICAS} 123 | template: 124 | metadata: 125 | labels: 126 | app: ${NAME} 127 | template: kafka-zk 128 | component: kafka 129 | spec: 130 | securityContext: 131 | runAsUser: 1001 132 | fsGroup: 1001 133 | containers: 134 | - name: ${NAME} 135 | imagePullPolicy: IfNotPresent 136 | image: ${SOURCE_IMAGE}:${KAFKA_VERSION} 137 | resources: 138 | requests: 139 | memory: ${RESOURCE_MEMORY_REQ} 140 | cpu: ${RESOURCE_CPU_REQ} 141 | limits: 142 | memory: ${RESOURCE_MEMORY_LIMIT} 143 | cpu: ${RESOURCE_CPU_LIMIT} 144 | ports: 145 | - containerPort: 9092 146 | name: server 147 | - containerPort: 2181 148 | name: zkclient 149 | - containerPort: 2888 150 | name: zkserver 151 | - containerPort: 3888 152 | name: zkleader 153 | env: 154 | - name : KAFKA_REPLICAS 155 | value: ${REPLICAS} 156 | - name: KAFKA_ZK_LOCAL 157 | value: ${KAFKA_ZK_LOCAL} 158 | - name : ZOO_HEAP_OPTS 159 | value: ${ZK_HEAP_OPTS} 160 | - name: KAFKA_HEAP_OPTS 161 | value: ${KAFKA_HEAP_OPTS} 162 | - name: SERVER_num_partitions 163 | value: ${SERVER_NUM_PARTITIONS} 164 | - name: SERVER_delete_topic_enable 165 | value: ${SERVER_DELETE_TOPIC_ENABLE} 166 | - name: SERVER_log_retention_hours 167 | value: ${SERVER_LOG_RETENTION_HOURS} 168 | - name: SERVER_zookeeper_connect 169 | value: ${SERVER_ZOOKEEPER_CONNECT} 170 | - name: SERVER_zookeeper_connection_timeout_ms 171 | value: ${SERVER_ZOOKEEPER_CONNECT_TIMEOUT} 172 | readinessProbe: 173 | exec: 174 | command: 175 | - kafka_server_status.sh 176 | initialDelaySeconds: 30 177 | timeoutSeconds: 5 178 | livenessProbe: 179 | exec: 180 | command: 181 | - kafka_server_status.sh 182 | initialDelaySeconds: 30 183 | timeoutSeconds: 5 184 | -------------------------------------------------------------------------------- /openshift/main.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | SCALA_VERSION=${SCALA_VERSION:-"2.13"} 6 | KAFKA_VERSION=${KAFKA_VERSION:-"2.5.0"} 7 | KAFKA_IMAGE=${KAFKA_IMAGE:-"engapa/kafka:${SCALA_VERSION}-${KAFKA_VERSION}"} 8 | ZOO_VERSION='3.6.0' 9 | ZK_IMAGE="engapa/zookeeper:${ZOO_VERSION}" 10 | 11 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 12 | 13 | 14 | function oc-install() 15 | { 16 | # Download oc 17 | curl -LO https://github.com/openshift/origin/releases/download/v3.11.0/openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz 18 | tar -xvzf openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz 19 | mv openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit/oc ./oc 20 | rm -rf openshift-origin-client-tools* 21 | chmod a+x oc 22 | sudo mv oc /usr/local/bin/oc 23 | } 24 | 25 | function oc-cluster-run() 26 | { 27 | 28 | # Add internal insecure registry 29 | sudo sed -i 's#^ExecStart=.*#ExecStart=/usr/bin/dockerd --insecure-registry='172.30.0.0/16' -H fd://#' /lib/systemd/system/docker.service 30 | sudo systemctl daemon-reload 31 | sudo systemctl restart docker 32 | 33 | # Run openshift cluster 34 | oc cluster up --enable=[*] 35 | 36 | # Waiting for cluster 37 | for i in {1..150}; do # timeout for 5 minutes 38 | oc cluster status &> /dev/null 39 | if [[ $? -ne 1 ]]; then 40 | break 41 | fi 42 | sleep 2 43 | done 44 | 45 | oc login -u system:admin 46 | oc adm policy add-scc-to-group privileged system:serviceaccounts:myproject 47 | oc create -f $DIR/kafka-zk.yaml 48 | oc create -f $DIR/kafka-persistent.yaml 49 | 50 | } 51 | 52 | function build_local_image() 53 | { 54 | 55 | oc new-build --name kafka --strategy docker --binary --docker-image "openjdk:8-jre-alpine" 56 | oc start-build kafka --from-dir $DIR/.. --follow 57 | 58 | } 59 | 60 | # $1 : Number of replicas 61 | function check() 62 | { 63 | 64 | SLEEP_TIME=10 65 | MAX_ATTEMPTS=50 66 | ATTEMPTS=0 67 | READY_REPLICAS="0" 68 | REPLICAS=${1:-1} 69 | until [[ "$READY_REPLICAS" == "$REPLICAS" ]]; do 70 | sleep $SLEEP_TIME 71 | ATTEMPTS=`expr $ATTEMPTS + 1` 72 | if [[ $ATTEMPTS -gt $MAX_ATTEMPTS ]]; then 73 | echo "ERROR: Max number of attempts was reached (${MAX_ATTEMPTS})" 74 | exit 1 75 | fi 76 | READY_REPLICAS=$(oc get statefulset -l component=${2:-kafka} -o jsonpath='{.items[?(@.kind=="StatefulSet")].status.readyReplicas}' 2>&1) 77 | echo "[${ATTEMPTS}/${MAX_ATTEMPTS}] - Ready replicas : ${READY_REPLICAS:-0}/$REPLICAS ... " 78 | done 79 | oc get all 80 | } 81 | 82 | function test-zk() 83 | { 84 | # Given 85 | REPLICAS=${1:-1} 86 | # When 87 | oc new-app --template=kafka-zk -p REPLICAS=$REPLICAS -p SOURCE_IMAGE="engapa/kafka" 88 | # Then 89 | check $REPLICAS 90 | 91 | } 92 | 93 | function test-persistent() 94 | { 95 | # Given 96 | echo "Installing zookeeper cluster ..." 97 | oc create -f https://raw.githubusercontent.com/engapa/zookeeper-k8s-openshift/v$ZOO_VERSION/openshift/zk.yaml 98 | oc new-app --template=zk -p ZOO_REPLICAS=1 -p SOURCE_IMAGE="engapa/zookeeper" 99 | check 1 zk 100 | 101 | echo "Installing kafka cluster with persistent storage ..." 102 | REPLICAS=${1:-1} 103 | for i in $(seq 1 ${REPLICAS});do 104 | cat << PV | oc create -f - 105 | apiVersion: v1 106 | kind: PersistentVolume 107 | metadata: 108 | name: kafka-persistent-data-disk-$i 109 | contents: data 110 | labels: 111 | component: kafka 112 | spec: 113 | capacity: 114 | storage: 1Gi 115 | accessModes: 116 | - ReadWriteOnce 117 | hostPath: 118 | path: /tmp/oc/kafka-persistent-data-disk-$i 119 | PV 120 | done 121 | # When 122 | oc new-app --template=kafka-persistent -p REPLICAS=${REPLICAS} -p SOURCE_IMAGE="engapa/kafka" 123 | # Then 124 | check ${REPLICAS} 125 | oc get pv,pvc 126 | } 127 | 128 | function test-all() 129 | { 130 | REPLICAS=$1 131 | test $REPLICAS && oc delete -l component=kafka all 132 | test-persistent $REPLICAS && oc delete -l component=kafka all,pv,pvc 133 | } 134 | 135 | function clean-resources() 136 | { 137 | echo "Cleaning resources ...." 138 | oc delete -l component=kafka all,pv,pvc 139 | oc delete -l component=zk all,pv,pvc 140 | } 141 | 142 | function oc-cluster-delete() 143 | { 144 | echo "Stopping cluster ...." 145 | oc cluster down 146 | } 147 | 148 | function help() # Show a list of functions 149 | { 150 | declare -F -p | cut -d " " -f 3 151 | } 152 | 153 | if [[ "_$1" = "_" ]]; then 154 | help 155 | else 156 | "$@" 157 | fi 158 | --------------------------------------------------------------------------------