├── Dockerfile ├── README.md ├── config ├── kafka.jaas.tmpl ├── log4j.properties └── zookeeper.jaas.tmpl ├── scripts ├── configureKerberosClient.sh ├── create-kafka-topics.sh ├── start-all.sh ├── start-kafka.sh └── start-zookeeper.sh └── supervisor ├── initialize.ini ├── kafka.ini └── zookeeper.ini /Dockerfile: -------------------------------------------------------------------------------- 1 | # Kafka and Zookeeper 2 | 3 | FROM centos 4 | 5 | ENV SCALA_VERSION 2.12 6 | ENV KAFKA_VERSION 2.3.0 7 | ENV KAFKA_HOME /opt/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION" 8 | 9 | # Install Kafka, Zookeeper and other needed things 10 | RUN yum update -y && \ 11 | yum install -y epel-release wget nc net-tools openssl krb5-workstation krb5-libs java which && \ 12 | yum install -y python3-pip && \ 13 | pip3 install supervisor && \ 14 | wget -q \ 15 | http://apache.mirrors.spacedump.net/kafka/"$KAFKA_VERSION"/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz \ 16 | -O /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz && \ 17 | tar xfz /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz -C /opt && \ 18 | rm /tmp/kafka_"$SCALA_VERSION"-"$KAFKA_VERSION".tgz && \ 19 | yum clean all 20 | 21 | ADD scripts/start-all.sh /usr/bin/start-all.sh 22 | ADD scripts/start-kafka.sh /usr/bin/start-kafka.sh 23 | ADD scripts/start-zookeeper.sh /usr/bin/start-zookeeper.sh 24 | ADD scripts/create-kafka-topics.sh /usr/bin/create-kafka-topics.sh 25 | ADD scripts/configureKerberosClient.sh /usr/bin/configureKerberosClient.sh 26 | 27 | ADD config/log4j.properties config/zookeeper.jaas.tmpl config/kafka.jaas.tmpl "$KAFKA_HOME"/config/ 28 | 29 | RUN mkdir -p /tmp/zookeeper && \ 30 | mkdir -p /tmp/kafka-logs && \ 31 | mkdir -p /var/log/supervisor && \ 32 | mkdir -p /var/log/zookeeper && \ 33 | mkdir "$KAFKA_HOME"/logs && \ 34 | mkdir -p /var/private/ssl/ && \ 35 | chmod -R 777 /var/log/supervisor/ && \ 36 | chmod -R 777 /var/log/zookeeper/ && \ 37 | chmod -R 777 /var/run/ && \ 38 | chmod -R 777 "$KAFKA_HOME"/logs && \ 39 | chmod -R 777 "$KAFKA_HOME"/config && \ 40 | chmod -R 777 /tmp/zookeeper && \ 41 | chmod -R 777 /tmp/kafka-logs && \ 42 | chmod -R 777 /var/private/ssl 43 | 44 | # Supervisor config 45 | ADD supervisor/initialize.ini supervisor/kafka.ini supervisor/zookeeper.ini /etc/supervisord.d/ 46 | 47 | RUN echo_supervisord_conf | sed -e 's:;\[include\]:\[include\]:g' | sed -e 's:;files = relative/directory/\*.ini:files = /etc/supervisord.d/\*.ini:g' > /etc/supervisord.conf 48 | 49 | # 2181 is zookeeper, 9092-9099 is kafka (for different listeners like SSL, INTERNAL, PLAINTEXT etc.) 50 | EXPOSE 2181 9092 9093 9094 9095 9096 9097 9098 9099 51 | 52 | CMD ["supervisord", "-n"] 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Docker kafka plus zookeeper image 2 | 3 | ## Building the image 4 | ``` 5 | docker build -f Dockerfile -t docker-kafka . 6 | ``` 7 | 8 | ## Configuring the image 9 | 10 | Configuration of kafka can be changed/influenced by setting a set of environment variables 11 | 12 | 13 | 14 | |env var|default|options|description| 15 | | --- | --- | --- | --- | 16 | | ADVERTISED_LISTENERS | | | the listeners advertised to the outside world with associated listener name | 17 | | LISTENERS | | | the listeners being created by the broker with their associated name | 18 | | SECURITY_PROTOCOL_MAP | | | mapping from the listener names to security protocol | 19 | | SSL_CERT | | | Optional pem certificate | 20 | | SSL_KEY | | | Optional ssl private key | 21 | | SSL_DN | | | Optional subject to use for the generated certificate, e.g. CN=kafka.example.com,OU=data,O=example,L=Kris,S=Geus,C=NL | 22 | | SSL_PASSWORD | | | Optional password to use for the store and key otherwise will be automatically generated | 23 | | INTER_BROKER | | | the listener name the internal connections will use | 24 | | LOG\_RETENTION_HOURS | 168 | | Number of hours the messages are kept in the topic log | 25 | | LOG\_RETENTION_BYTES | 1073741824 | | Size of the topic logs at which pruning will take place| 26 | | NUM_PARTITIONS | 1 | | Default number of partitions for a topic | 27 | | AUTO\_CREATE_TOPICS | true | true, false | Automatically create a topic when messages are produced | 28 | | KAFKA\_CREATE_TOPICS | | | Comma separated list op topics to create. Topic can specify partitions, replication and cleanup.policy by appending a colon separated list of these configurations | 29 | 30 | 31 | 32 | ## Running the image 33 | To run the image with 2 topics, advertising an internal (9092) and external listener (443). This example is useful when you have some kind 34 | of edge termination for ssl externally or port mapping, e.g. port 443 external SSL --> port 9092 internal. To also make this example work if you don't have such ssl termination setup the portmapping 9092->9092 is added as an extra. 35 | 36 | ``` 37 | docker run --rm -p 2181:2181 -p 443:9092 -p 9092:9092 -p 9093:9093 \ 38 | --env ADVERTISED_LISTENERS=PLAINTEXT://kafka:443,INTERNAL://localhost:9093 \ 39 | --env LISTENERS=PLAINTEXT://0.0.0.0:9092,INTERNAL://0.0.0.0:9093 \ 40 | --env SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,INTERNAL:PLAINTEXT \ 41 | --env INTER_BROKER=INTERNAL \ 42 | --env KAFKA_CREATE_TOPICS="test:36:1,krisgeus:12:1:compact" \ 43 | --name kafka \ 44 | krisgeus/docker-kafka 45 | ``` 46 | 47 | ## Testing 48 | 49 | Testing if the image works can be done by using the kafka console producer/consumer 50 | since we named the image we need to add an entry to the `/etc/hosts` file to connect the name kafka to the local ip 127.0.0.1 51 | 52 | ``` 53 | 127.0.0.1 kafka 54 | ``` 55 | 56 | Now we can use that hostname (the same as the advertised host) to connect the producer and consumer 57 | 58 | ``` 59 | kafka-console-producer --broker-list kafka:9092 \ 60 | --topic test 61 | ``` 62 | 63 | Sending messages can be done by typeing the following in the console 64 | ``` 65 | message1 66 | message2 67 | ``` 68 | 69 | To consume this use the following in a separate terminal window 70 | 71 | ``` 72 | kafka-console-consumer --bootstrap-server kafka:9092 \ 73 | --topic test \ 74 | --from-beginning 75 | 76 | message1 77 | message2 78 | ``` 79 | 80 | __A compact topic needs a key so remember to start the producer with the mandatory parse.key and key.separator options!__ 81 | 82 | ``` 83 | kafka-console-producer --broker-list kafka:9092 \ 84 | --topic krisgeus \ 85 | --property "parse.key=true" \ 86 | --property "key.separator=:" 87 | ``` 88 | 89 | Sending messages can be done by typeing the following in the console 90 | ``` 91 | key1:value1 92 | key2:value2 93 | ``` 94 | 95 | To consume this use the following in a separate terminal window 96 | 97 | ``` 98 | kafka-console-consumer --bootstrap-server kafka:9092 \ 99 | --topic krisgeus \ 100 | --from-beginning 101 | 102 | value1 103 | value2 104 | ``` 105 | ## SSL key and cert example 106 | 107 | 1. generate a private key 108 | 109 | ``` 110 | openssl genrsa -des3 -passout pass:apachepass -out pass.key 2048 111 | openssl rsa -passin pass:apachepass -in pass.key -out ssl.key 112 | ``` 113 | 114 | 2. generate a cert 115 | 116 | ``` 117 | openssl req -new -key ssl.key -out fqdn.csr -subj "/C=/ST=/L=/O=/CN=domain” 118 | ``` 119 | 120 | 3. cert self-signing 121 | 122 | ``` 123 | openssl x509 -req -days 365 -in fqdn.csr -signkey ssl.key -out selfsign.crt 124 | ``` 125 | 126 | 4. passning a key and cert to the docker 127 | 128 | ``` 129 | --env SSL_CERT="`cat selfsign.crt`" \ 130 | --env SSL_KEY="`cat /ssl.key`" \ 131 | ``` 132 | 133 | 5. generate a client truststore 134 | 135 | ``` 136 | keytool -import -alias localhost -keystore client.truststore.jks -file selfsign.crt -noprompt --storepass secretpass --keypass secretpass 137 | ``` 138 | 139 | 6. kafka client config 140 | 141 | ``` 142 | security.protocol=SSL 143 | ssl.truststore.location=client.truststore.jks 144 | ssl.truststore.password=secretpass 145 | ``` 146 | 147 | ## Kerberos consuming example 148 | 149 | You can use the ticket cache or the keytab to configure 150 | 151 | ### TicketCache 152 | `kinit kafka/$(hostname -f) -kt /kafka.keytab` 153 | 154 | `vi /kafka-client.jaas` 155 | 156 | ``` 157 | KafkaClient { 158 | com.sun.security.auth.module.Krb5LoginModule required 159 | useTicketCache=true 160 | serviceName="kafka" 161 | useKeyTab=false; 162 | }; 163 | ``` 164 | 165 | ``` 166 | export KAFKA_OPTS="-Djava.security.auth.login.config=/kafka-client.jaas -Djava.security.krb5.conf=/etc/krb5.conf -Dsun.security.krb5.debug=false" 167 | 168 | ${KAFKA_HOME}/kafka-console-consumer.sh --bootstrap-server $(hostname):9092 --topic divolte --from-beginning --consumer-property security.protocol=SASL_PLAINTEXT 169 | ``` 170 | 171 | ### Keytab 172 | 173 | `vi /kafka-client-keytab.jaas` 174 | 175 | ``` 176 | KafkaClient { 177 | com.sun.security.auth.module.Krb5LoginModule required 178 | useKeyTab=true 179 | storeKey=true 180 | keyTab="/kafka.keytab" 181 | principal="kafka/divolte-kafka.divolte_divolte.io"; 182 | }; 183 | ``` 184 | 185 | ``` 186 | export KAFKA_OPTS="-Djava.security.auth.login.config=/kafka-client-keytab.jaas -Djava.security.krb5.conf=/etc/krb5.conf -Dsun.security.krb5.debug=false" 187 | 188 | ${KAFKA_HOME}/kafka-console-consumer.sh --bootstrap-server $(hostname):9092 --topic divolte --from-beginning --consumer-property security.protocol=SASL_PLAINTEXT 189 | ``` 190 | -------------------------------------------------------------------------------- /config/kafka.jaas.tmpl: -------------------------------------------------------------------------------- 1 | KafkaServer { 2 | com.sun.security.auth.module.Krb5LoginModule required 3 | refreshKrb5Config=true 4 | useKeyTab=true 5 | keyTab="/kafka.keytab" 6 | storeKey=true 7 | principal="kafka/HOSTNAME"; 8 | }; 9 | 10 | Client { 11 | com.sun.security.auth.module.Krb5LoginModule required 12 | refreshKrb5Config=true 13 | useKeyTab=true 14 | keyTab="/kafka.keytab" 15 | storeKey=true 16 | principal="kafka/HOSTNAME"; 17 | }; 18 | -------------------------------------------------------------------------------- /config/log4j.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | log4j.rootLogger=INFO, stdout 17 | 18 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 19 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 20 | log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n 21 | 22 | log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender 23 | log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH 24 | log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log 25 | log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout 26 | log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 27 | 28 | log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender 29 | log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH 30 | log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log 31 | log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout 32 | log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 33 | 34 | log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender 35 | log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH 36 | log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log 37 | log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout 38 | log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 39 | 40 | log4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender 41 | log4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd-HH 42 | log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log 43 | log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout 44 | log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 45 | 46 | log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender 47 | log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH 48 | log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log 49 | log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout 50 | log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 51 | 52 | log4j.appender.authorizerAppender=org.apache.log4j.DailyRollingFileAppender 53 | log4j.appender.authorizerAppender.DatePattern='.'yyyy-MM-dd-HH 54 | log4j.appender.authorizerAppender.File=${kafka.logs.dir}/kafka-authorizer.log 55 | log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout 56 | log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n 57 | 58 | # Turn on all our debugging info 59 | #log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG, kafkaAppender 60 | #log4j.logger.kafka.client.ClientUtils=DEBUG, kafkaAppender 61 | #log4j.logger.kafka.perf=DEBUG, kafkaAppender 62 | #log4j.logger.kafka.perf.ProducerPerformance$ProducerThread=DEBUG, kafkaAppender 63 | #log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG 64 | log4j.logger.kafka=INFO, kafkaAppender 65 | 66 | log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender 67 | log4j.additivity.kafka.network.RequestChannel$=false 68 | 69 | #log4j.logger.kafka.network.Processor=TRACE, requestAppender 70 | #log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender 71 | #log4j.additivity.kafka.server.KafkaApis=false 72 | log4j.logger.kafka.request.logger=WARN, requestAppender 73 | log4j.additivity.kafka.request.logger=false 74 | 75 | log4j.logger.kafka.controller=WARN, controllerAppender 76 | log4j.additivity.kafka.controller=false 77 | 78 | log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender 79 | log4j.additivity.kafka.log.LogCleaner=false 80 | 81 | log4j.logger.state.change.logger=WARN, stateChangeAppender 82 | log4j.additivity.state.change.logger=false 83 | 84 | #Change this to debug to get the actual audit log for authorizer. 85 | log4j.logger.kafka.authorizer.logger=WARN, authorizerAppender 86 | log4j.additivity.kafka.authorizer.logger=false -------------------------------------------------------------------------------- /config/zookeeper.jaas.tmpl: -------------------------------------------------------------------------------- 1 | Server { 2 | com.sun.security.auth.module.Krb5LoginModule required 3 | refreshKrb5Config=true 4 | useKeyTab=true 5 | keyTab="/zookeeper.keytab" 6 | storeKey=true 7 | principal="zookeeper/HOSTNAME"; 8 | }; -------------------------------------------------------------------------------- /scripts/configureKerberosClient.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "===================================================================================" 3 | echo "==== Kerberos Client ==============================================================" 4 | echo "===================================================================================" 5 | KADMIN_PRINCIPAL_FULL=$KADMIN_PRINCIPAL@$REALM 6 | 7 | echo "REALM: $REALM" 8 | echo "KADMIN_PRINCIPAL_FULL: $KADMIN_PRINCIPAL_FULL" 9 | echo "KADMIN_PASSWORD: $KADMIN_PASSWORD" 10 | echo "" 11 | 12 | function kadminCommand { 13 | kadmin -p $KADMIN_PRINCIPAL_FULL -w $KADMIN_PASSWORD -q "$1" 14 | } 15 | 16 | echo "===================================================================================" 17 | echo "==== /etc/krb5.conf ===============================================================" 18 | echo "===================================================================================" 19 | tee /etc/krb5.conf <&2 echo "KDC is unavailable - sleeping 1 sec" 39 | sleep 1 40 | done 41 | echo "KDC and Kadmin are operational" 42 | echo "" 43 | 44 | echo "===================================================================================" 45 | echo "==== Add kafka and zookeeper principals for host ${HOSTNAME} ======================" 46 | echo "===================================================================================" 47 | echo "Add zookeeper user" 48 | kadminCommand "addprinc -pw zookeeper zookeeper/$(hostname -f)@${REALM}" 49 | echo "Create zookeeper keytab" 50 | kadminCommand "xst -k /zookeeper.keytab zookeeper/$(hostname -f)" 51 | echo "Add kafka user" 52 | kadminCommand "addprinc -pw kafka kafka/$(hostname -f)@${REALM}" 53 | echo "Create kafka keytab" 54 | kadminCommand "xst -k /kafka.keytab kafka/$(hostname -f)" 55 | echo "" 56 | exit 0 57 | -------------------------------------------------------------------------------- /scripts/create-kafka-topics.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Run KafkaTopics to create the topics if they don't exist yet 4 | if [[ -n $KAFKA_CREATE_TOPICS ]]; then 5 | for TOPIC in ${KAFKA_CREATE_TOPICS//,/ }; do 6 | echo "creating topic: $TOPIC" 7 | TOPIC_CONFIG=(${TOPIC//:/ }) 8 | if [ ${TOPIC_CONFIG[3]} ]; then 9 | $KAFKA_HOME/bin/kafka-topics.sh --create --if-not-exists --zookeeper ${HOSTNAME}:2181 --replication-factor ${TOPIC_CONFIG[2]} --partitions ${TOPIC_CONFIG[1]} --topic "${TOPIC_CONFIG[0]}" --config cleanup.policy="${TOPIC_CONFIG[3]}" 10 | else 11 | $KAFKA_HOME/bin/kafka-topics.sh --create --if-not-exists --zookeeper ${HOSTNAME}:2181 --replication-factor ${TOPIC_CONFIG[2]} --partitions ${TOPIC_CONFIG[1]} --topic "${TOPIC_CONFIG[0]}" 12 | fi 13 | done 14 | fi 15 | 16 | exit 0 -------------------------------------------------------------------------------- /scripts/start-all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # The full path of the lock file to use. 4 | LOCKFILE="/tmp/startscript-lock" 5 | export JAVA_HOME=$(readlink -f $(which java) | sed -e 's/\/bin\/java//g') 6 | 7 | start(){ 8 | # Assert that there is no other Lambda instance, created with this script, running. 9 | [ -f $LOCKFILE ] && return 0 10 | 11 | # Create a lock file to prevent multiple instantiations. 12 | touch $LOCKFILE 13 | 14 | if [ ! -z "$ENABLE_KERBEROS" ]; then 15 | # Start Kerberos keytab creation 16 | echo "Starting keytab creation..." 17 | /usr/bin/configureKerberosClient.sh 18 | returnedValue=$? 19 | if [ $returnedValue -eq 0 ] 20 | then 21 | echo "Krb5 configuration has been started!" 22 | else 23 | echo "Krb5 configuration has failed to start with code $returnedValue." 24 | return $returnedValue 25 | fi 26 | fi 27 | 28 | # Start Zookeeper. 29 | echo "Starting Zookeeper..." 30 | supervisorctl start zookeeper 31 | # Wait for Zookeeper to start. 32 | while [ "$(supervisorctl status zookeeper | tr -s ' ' | cut -f2 -d' ')" == "STARTING" ] 33 | do 34 | sleep 10 35 | done 36 | zookeeper_status=$(supervisorctl status zookeeper | tr -s ' ' | cut -f2 -d' ') 37 | if [ "$zookeeper_status" != "RUNNING" ] 38 | then 39 | echo "Zookeeper has failed to start with code $zookeeper_status." 40 | else 41 | echo "Zookeeper has been started!" 42 | fi 43 | 44 | # Start Kafka on master node. 45 | echo "Starting kafka..." 46 | supervisorctl start kafka 47 | # Wait for Kafka to start. 48 | while [ "$(supervisorctl status kafka | tr -s ' ' | cut -f2 -d' ')" == "STARTING" ] 49 | do 50 | sleep 10 51 | done 52 | kafka_status=$(supervisorctl status kafka | tr -s ' ' | cut -f2 -d' ') 53 | if [ "$kafka_status" != "RUNNING" ] 54 | then 55 | echo "Kafka has failed to start with code $kafka_status." 56 | else 57 | echo "Kafka has been started!" 58 | fi 59 | 60 | # Start Topic creation 61 | echo "Starting topic creation..." 62 | # Wait for kafka broker to be available 63 | while [ $(echo dump | nc $(hostname -s) 2181 | grep brokers) == "" ] 64 | do 65 | echo "Waiting for Kafka Broker to be available in zookeeper" 66 | sleep 10 67 | done 68 | 69 | until echo exit | nc --send-only $(hostname -s) 9092; 70 | do 71 | echo "Waiting for Kafka Broker to be really available" 72 | sleep 10 73 | done 74 | 75 | /usr/bin/create-kafka-topics.sh >> $KAFKA_HOME/logs/create-kafka-topics.log 76 | returnedValue=$? 77 | if [ $returnedValue -eq 0 ] 78 | then 79 | echo "Topic creation has been done!" 80 | else 81 | echo "Topic creation has failed to execute with code $returnedValue." 82 | return $returnedValue 83 | fi 84 | 85 | # Don;t exit as long as kafka is running 86 | while [ "$(supervisorctl status kafka | tr -s ' ' | cut -f2 -d' ')" == "RUNNING" ] 87 | do 88 | sleep 300 89 | done 90 | 91 | return 0 92 | } 93 | 94 | stop(){ 95 | # Stop Kafka. 96 | supervisorctl stop kafka 97 | kafka_status=$(supervisorctl status kafka | tr -s ' ' | cut -f2 -d' ') 98 | if [ "$kafka_status" == "STOPPED" ] 99 | then 100 | echo "Kafka has been stopped!" 101 | else 102 | echo "Kafka has failed to stop with returned code $kafka_status" 103 | fi 104 | 105 | # Stop Zookeeper. 106 | supervisorctl stop zookeeper 107 | zookeeper_status=$(supervisorctl status zookeeper | tr -s ' ' | cut -f2 -d' ') 108 | if [ "$zookeeper_status" == "STOPPED" ] 109 | then 110 | echo "Zookeeper has been stopped!" 111 | else 112 | echo "Zookeeper has failed to stop with returned code $zookeeper_status" 113 | fi 114 | 115 | # Remove lock file. 116 | rm -f $LOCKFILE 117 | 118 | return 0 119 | } 120 | 121 | restart(){ 122 | stop 123 | start 124 | } 125 | 126 | RETVAL=0 127 | 128 | case "$1" in 129 | start) 130 | start 131 | ;; 132 | stop) 133 | stop 134 | ;; 135 | restart|reload|force-reload) 136 | restart 137 | ;; 138 | condrestart) 139 | [ -f $LOCKFILE ] && restart || : 140 | ;; 141 | status) 142 | # If the lock file exists, then the Lambda instance is running. 143 | [ -f $LOCKFILE ] && echo "Kafka startup instance is running." || echo "Kafka startup instance is not running." 144 | RETVAL=$? 145 | ;; 146 | *) 147 | echo "Usage: $0 {start|stop|status|restart|reload|force-reload|condrestart}" 148 | RETVAL=1 149 | esac 150 | 151 | exit $RETVAL 152 | 153 | 154 | -------------------------------------------------------------------------------- /scripts/start-kafka.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Optional ENV variables: 4 | # * ZK_CHROOT: the zookeeper chroot that's used by Kafka (without / prefix), e.g. "kafka" 5 | # * LOG_RETENTION_HOURS: the minimum age of a log file in hours to be eligible for deletion (default is 168, for 1 week) 6 | # * LOG_RETENTION_BYTES: configure the size at which segments are pruned from the log, (default is 1073741824, for 1GB) 7 | # * NUM_PARTITIONS: configure the default number of log partitions per topic 8 | # * ADVERTISED_LISTENERS: the listeners advertised to the outside world with associated listener name 9 | # * LISTENERS: the listeners being created by the broker with their associated name 10 | # * SECURITY_PROTOCOL_MAP: mapping from the listener names to security protocol 11 | # * INTER_BROKER: the listener name the internal connections will use 12 | 13 | # Allow specification of log retention policies 14 | 15 | # Configure kerberos for kafka 16 | echo "Make sure new config items are put at end of config file even if no newline is present as final character in the config" 17 | echo >> $KAFKA_HOME/config/server.properties 18 | 19 | if [ ! -z "$ENABLE_KERBEROS" ]; then 20 | echo "set SASL mechanism" 21 | if grep -r -q "^#\?sasl.enabled.mechanisms" $KAFKA_HOME/config/server.properties; then 22 | sed -r -i "s/#?(sasl.enabled.mechanisms)=(.*)/\1=GSSAPI/g" $KAFKA_HOME/config/server.properties 23 | else 24 | echo "sasl.enabled.mechanisms=GSSAPI" >> $KAFKA_HOME/config/server.properties 25 | fi 26 | 27 | echo "set Kerberos service name for kafka" 28 | if grep -r -q "^#\?sasl.kerberos.service.name" $KAFKA_HOME/config/server.properties; then 29 | sed -r -i "s/#?(sasl.kerberos.service.name)=(.*)/\1=kafka/g" $KAFKA_HOME/config/server.properties 30 | else 31 | echo "sasl.kerberos.service.name=kafka" >> $KAFKA_HOME/config/server.properties 32 | fi 33 | 34 | echo "create jaas config based on template" 35 | sed "s/HOSTNAME/$(hostname -f)/g" $KAFKA_HOME/config/kafka.jaas.tmpl > $KAFKA_HOME/config/kafka.jaas 36 | 37 | export KAFKA_OPTS="-Djava.security.auth.login.config=${KAFKA_HOME}/config/kafka.jaas -Djava.security.krb5.conf=/etc/krb5.conf -Dsun.security.krb5.debug=true" 38 | fi 39 | 40 | if [ ! -z "$LOG_RETENTION_HOURS" ]; then 41 | echo "log retention hours: $LOG_RETENTION_HOURS" 42 | sed -r -i "s/#?(log.retention.hours)=(.*)/\1=$LOG_RETENTION_HOURS/g" $KAFKA_HOME/config/server.properties 43 | fi 44 | if [ ! -z "$LOG_RETENTION_BYTES" ]; then 45 | echo "log retention bytes: $LOG_RETENTION_BYTES" 46 | sed -r -i "s/#?(log.retention.bytes)=(.*)/\1=$LOG_RETENTION_BYTES/g" $KAFKA_HOME/config/server.properties 47 | fi 48 | 49 | # Configure the default number of log partitions per topic 50 | if [ ! -z "$NUM_PARTITIONS" ]; then 51 | echo "default number of partition: $NUM_PARTITIONS" 52 | sed -r -i "s/#?(num.partitions)=(.*)/\1=$NUM_PARTITIONS/g" $KAFKA_HOME/config/server.properties 53 | fi 54 | 55 | # Enable/disable auto creation of topics 56 | if [ ! -z "$AUTO_CREATE_TOPICS" ]; then 57 | echo "auto.create.topics.enable: $AUTO_CREATE_TOPICS" 58 | if grep -r -q "^#\?auto.create.topics.enable" $KAFKA_HOME/config/server.properties; then 59 | sed -r -i "s/#?(auto.create.topics.enable)=(.*)/\1=$AUTO_CREATE_TOPICS/g" $KAFKA_HOME/config/server.properties 60 | else 61 | echo "auto.create.topics.enable=$AUTO_CREATE_TOPICS" >> $KAFKA_HOME/config/server.properties 62 | fi 63 | fi 64 | 65 | if [ ! -z "$ADVERTISED_LISTENERS" ]; then 66 | echo "advertised.listeners: ${ADVERTISED_LISTENERS}" 67 | if grep -r -q "^#\?advertised.listeners=" $KAFKA_HOME/config/server.properties; then 68 | # use | as a delimiter to make sure // does not confuse sed 69 | sed -r -i "s|^#?(advertised.listeners)=(.*)|\1=${ADVERTISED_LISTENERS}|g" $KAFKA_HOME/config/server.properties 70 | else 71 | echo "advertised.listeners=${ADVERTISED_LISTENERS}" >> $KAFKA_HOME/config/server.properties 72 | fi 73 | fi 74 | 75 | if [ ! -z "$LISTENERS" ]; then 76 | echo "listeners: ${LISTENERS}" 77 | if grep -r -q "^#\?listeners=" $KAFKA_HOME/config/server.properties; then 78 | # use | as a delimiter to make sure // does not confuse sed 79 | sed -r -i "s|^#?(listeners)=(.*)|\1=${LISTENERS}|g" $KAFKA_HOME/config/server.properties 80 | else 81 | echo "listeners=${LISTENERS}" >> $KAFKA_HOME/config/server.properties 82 | fi 83 | fi 84 | 85 | if [ ! -z "$SECURITY_PROTOCOL_MAP" ]; then 86 | echo "listener.security.protocol.map: ${SECURITY_PROTOCOL_MAP}" 87 | if grep -r -q "^#\?listener.security.protocol.map=" $KAFKA_HOME/config/server.properties; then 88 | sed -r -i "s/^#?(listener.security.protocol.map)=(.*)/\1=${SECURITY_PROTOCOL_MAP}/g" $KAFKA_HOME/config/server.properties 89 | else 90 | echo "listener.security.protocol.map=${SECURITY_PROTOCOL_MAP}" >> $KAFKA_HOME/config/server.properties 91 | fi 92 | fi 93 | 94 | if [ ! -z "$INTER_BROKER" ]; then 95 | echo "inter.broker.listener_name: ${INTER_BROKER}" 96 | if grep -r -q "^#\?inter.broker.listener.name=" $KAFKA_HOME/config/server.properties; then 97 | sed -r -i "s/^#?(inter.broker.listener.name)=(.*)/\1=${INTER_BROKER}/g" $KAFKA_HOME/config/server.properties 98 | else 99 | echo "inter.broker.listener.name=${INTER_BROKER}" >> $KAFKA_HOME/config/server.properties 100 | fi 101 | fi 102 | 103 | if echo "$SECURITY_PROTOCOL_MAP" | grep -q ":SSL"; then 104 | if [ ! -f /var/private/ssl/server.keystore.jks ]; then 105 | if [ -z "$SSL_PASSWORD" ]; then 106 | SSL_PASSWORD=`date +%s | sha256sum | base64 | head -c 32` 107 | fi 108 | if [ ! -z "$SSL_CERT" ]; then 109 | mkdir -p /var/private/ssl/server/ 110 | echo "${SSL_KEY}" >> /var/private/ssl/server/ssl.key 111 | echo "${SSL_CERT}" >> /var/private/ssl/server/cert.pem 112 | openssl pkcs12 -export -in /var/private/ssl/server/cert.pem -inkey /var/private/ssl/server/ssl.key -name localhost -password pass:${SSL_PASSWORD} -out /var/private/ssl/server/pkcs12.p12 113 | ${JAVA_HOME}/bin/keytool -importkeystore -deststorepass ${SSL_PASSWORD} -destkeypass ${SSL_PASSWORD} -destkeystore /var/private/ssl/server.keystore.jks -srckeystore /var/private/ssl/server/pkcs12.p12 -srcstoretype PKCS12 -srcstorepass ${SSL_PASSWORD} -alias localhost 114 | else 115 | ${JAVA_HOME}/bin/keytool -genkey -noprompt -alias localhost -dname "${SSL_DN}" -keystore /var/private/ssl/server.keystore.jks --storepass ${SSL_PASSWORD} --keypass ${SSL_PASSWORD} 116 | fi 117 | if grep -r -q "^#\?ssl.keystore.location=" $KAFKA_HOME/config/server.properties; then 118 | # use | as a delimiter to make sure // does not confuse sed 119 | sed -r -i "s|^#?(ssl.keystore.location)=(.*)|\1=/var/private/ssl/server.keystore.jks|g" $KAFKA_HOME/config/server.properties 120 | else 121 | echo "ssl.keystore.location=/var/private/ssl/server.keystore.jks" >> $KAFKA_HOME/config/server.properties 122 | fi 123 | if grep -r -q "^#\?ssl.keystore.password=" $KAFKA_HOME/config/server.properties; then 124 | # use | as a delimiter to make sure // does not confuse sed 125 | sed -r -i "s|^#?(ssl.keystore.password)=(.*)|\1=${SSL_PASSWORD}|g" $KAFKA_HOME/config/server.properties 126 | else 127 | echo "ssl.keystore.password=${SSL_PASSWORD}" >> $KAFKA_HOME/config/server.properties 128 | fi 129 | if grep -r -q "^#\?ssl.key.password=" $KAFKA_HOME/config/server.properties; then 130 | # use | as a delimiter to make sure // does not confuse sed 131 | sed -r -i "s|^#?(ssl.key.password)=(.*)|\1=${SSL_PASSWORD}|g" $KAFKA_HOME/config/server.properties 132 | else 133 | echo "ssl.key.password=${SSL_PASSWORD}" >> $KAFKA_HOME/config/server.properties 134 | fi 135 | fi 136 | fi 137 | 138 | if [ ! -z "$HOSTNAME" ]; then 139 | echo "zookeeper connect string to $HOSTNAME" 140 | if grep -r -q "^#\?zookeeper.connect=" $KAFKA_HOME/config/server.properties; then 141 | sed -r -i "s/^#?(zookeeper.connect)=(.*)/\1=${HOSTNAME}:2181/g" $KAFKA_HOME/config/server.properties 142 | else 143 | echo "zookeeper.connect=${HOSTNAME}:2181" >> $KAFKA_HOME/config/server.properties 144 | fi 145 | fi 146 | 147 | export EXTRA_ARGS="-name kafkaServer" # no -loggc to minimize logging 148 | $KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server.properties 149 | -------------------------------------------------------------------------------- /scripts/start-zookeeper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Configure kerberos for zookeeper 4 | if [ ! -z "$ENABLE_KERBEROS" ]; then 5 | echo "Make sure new config items are put at end of config file even if no newline is present as final character in the config" 6 | echo >> $KAFKA_HOME/config/zookeeper.properties 7 | 8 | echo "set authProvider.1 to SASLAuthenticationProvider" 9 | if grep -r -q "^#\?authProvider\.1" $KAFKA_HOME/config/zookeeper.properties; then 10 | sed -r -i "s/#?(authProvider\.1)=(.*)/\1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider/g" $KAFKA_HOME/config/zookeeper.properties 11 | else 12 | echo "authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider" >> $KAFKA_HOME/config/zookeeper.properties 13 | fi 14 | 15 | echo "set requireClientAuthScheme to sasl" 16 | if grep -r -q "^#\?requireClientAuthScheme" $KAFKA_HOME/config/zookeeper.properties; then 17 | sed -r -i "s/#?(requireClientAuthScheme)=(.*)/\1=sasl/g" $KAFKA_HOME/config/zookeeper.properties 18 | else 19 | echo "requireClientAuthScheme=sasl" >> $KAFKA_HOME/config/zookeeper.properties 20 | fi 21 | 22 | echo "set jaasLoginRenew period" 23 | if grep -r -q "^#\?jaasLoginRenew" $KAFKA_HOME/config/zookeeper.properties; then 24 | sed -r -i "s/#?(jaasLoginRenew)=(.*)/\1=3600000/g" $KAFKA_HOME/config/zookeeper.properties 25 | else 26 | echo "jaasLoginRenew=3600000" >> $KAFKA_HOME/config/zookeeper.properties 27 | fi 28 | 29 | echo "create jaas config based on template" 30 | sed "s/HOSTNAME/$(hostname -f)/g" $KAFKA_HOME/config/zookeeper.jaas.tmpl > $KAFKA_HOME/config/zookeeper.jaas 31 | 32 | export KAFKA_OPTS="-Djava.security.auth.login.config=${KAFKA_HOME}/config/zookeeper.jaas -Djava.security.krb5.conf=/etc/krb5.conf -Dsun.security.krb5.debug=true" 33 | fi 34 | 35 | # Run Zookeeper 36 | export EXTRA_ARGS="-name zookeeper" # no -loggc to minimize logging 37 | $KAFKA_HOME/bin/zookeeper-server-start.sh $KAFKA_HOME/config/zookeeper.properties -------------------------------------------------------------------------------- /supervisor/initialize.ini: -------------------------------------------------------------------------------- 1 | [program:initialize] 2 | command=/usr/bin/start-all.sh restart 3 | autostart=true 4 | autorestart=false -------------------------------------------------------------------------------- /supervisor/kafka.ini: -------------------------------------------------------------------------------- 1 | [program:kafka] 2 | command=/usr/bin/start-kafka.sh 3 | autostart=false 4 | autorestart=false -------------------------------------------------------------------------------- /supervisor/zookeeper.ini: -------------------------------------------------------------------------------- 1 | [program:zookeeper] 2 | command=/usr/bin/start-zookeeper.sh 3 | autostart=false 4 | autorestart=false 5 | stdout_logfile = /var/log/zookeeper/zookeeper.log 6 | stdout_logfile_backups = 0 7 | redirect_stderr = true 8 | --------------------------------------------------------------------------------