├── README.md ├── api ├── .dockerignore ├── Dockerfile ├── build.gradle └── src │ ├── main │ └── resources │ │ └── log4j.xml │ └── ratpack │ └── ratpack.groovy ├── build-images.sh ├── java ├── Dockerfile └── README.md ├── java8 └── Dockerfile ├── kafka ├── Dockerfile ├── server.properties.initial └── start-server.sh ├── shutdown.sh ├── startup.sh ├── zoo.cfg.initial └── zookeeper ├── Dockerfile ├── README.md └── start-server.sh /README.md: -------------------------------------------------------------------------------- 1 | docker-zookeeper 2 | ================ 3 | 4 | A 5 node zookeeper ensemble that runs in Docker 5 | 6 | Note: the wouterd/zookeeper image is in the docker index as an "automated build", so you don't have to build the 7 | Dockerfile in the zookeeper folder. 8 | 9 | Start ensemble by running: 10 | 11 | ./startup.sh 12 | 13 | From the root of the project. 14 | 15 | This will: 16 | 17 | - Create a config container 18 | - Create 5 zookeeper containers that wait for a config file to appear on the config container 19 | - Generate the configuration 20 | - Push the configuration into the config container 21 | - Start the ensemble 22 | -------------------------------------------------------------------------------- /api/.dockerignore: -------------------------------------------------------------------------------- 1 | .gradle 2 | .idea 3 | build/tmp 4 | build.gradle 5 | -------------------------------------------------------------------------------- /api/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM wouterd/java7 2 | 3 | MAINTAINER Wouter Danes 4 | 5 | ADD build/distributions/api.tar / 6 | 7 | ENTRYPOINT ["/api/bin/api"] 8 | 9 | EXPOSE 5050 10 | -------------------------------------------------------------------------------- /api/build.gradle: -------------------------------------------------------------------------------- 1 | buildscript { 2 | repositories { 3 | jcenter() 4 | } 5 | dependencies { 6 | classpath 'io.ratpack:ratpack-gradle:0.9.6' 7 | } 8 | } 9 | 10 | apply plugin: 'ratpack-groovy' 11 | apply plugin: 'idea' 12 | 13 | repositories { 14 | jcenter() 15 | } 16 | 17 | dependencies { 18 | compile group: 'org.apache.curator', name: 'curator-x-discovery', version: '2.6.0' 19 | compile group: 'org.apache.curator', name: 'curator-recipes', version: '2.6.0' 20 | compile group: 'ch.qos.logback', name: 'logback-classic', version: '1.1.2' 21 | 22 | testCompile 'org.spockframework:spock-core:0.7-groovy-2.0' 23 | testCompile group: 'org.hamcrest', name: 'hamcrest-all', version: '1.3' 24 | } 25 | 26 | run { 27 | systemProperty 'ratpack.port', '12345' 28 | } 29 | -------------------------------------------------------------------------------- /api/src/main/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /api/src/ratpack/ratpack.groovy: -------------------------------------------------------------------------------- 1 | import org.apache.curator.framework.CuratorFramework 2 | import org.apache.curator.framework.CuratorFrameworkFactory 3 | import org.apache.curator.retry.ExponentialBackoffRetry 4 | import org.apache.curator.x.discovery.ServiceDiscoveryBuilder 5 | import org.apache.curator.x.discovery.ServiceInstance 6 | import org.slf4j.LoggerFactory 7 | 8 | import static ratpack.groovy.Groovy.ratpack 9 | 10 | ratpack { 11 | bindings { 12 | def zkConnection = System.getenv()['ZOO'] 13 | if (!zkConnection) { 14 | LoggerFactory.getLogger('Service Initialization').error('Environment variable ZOO not set!') 15 | System.exit(-1) 16 | } 17 | def curator = CuratorFrameworkFactory.newClient(zkConnection, new ExponentialBackoffRetry(1000, 3)) 18 | curator.start() 19 | bind(CuratorFramework, curator) 20 | def discoveryBuilder = ServiceDiscoveryBuilder.builder(Void) 21 | def serviceDiscovery = discoveryBuilder.basePath('apis').client(curator).build() 22 | 23 | def addresses = NetworkInterface.getByName('eth0').inetAddresses 24 | def serviceIp = (addresses.find { it instanceof Inet4Address } as Inet4Address).hostAddress 25 | 26 | def serviceInstance = ServiceInstance.builder().address(serviceIp).name('myLittleApi').port(5050).build() 27 | 28 | serviceDiscovery.start() 29 | serviceDiscovery.registerService serviceInstance 30 | } 31 | handlers { 32 | get { 33 | def hostname = System.getenv()['HOSTNAME'] 34 | if (hostname) { 35 | response.headers.set('Via', hostname) 36 | } 37 | response.status 200 38 | response.send 'Hi there! Dude!' 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /build-images.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | docker build -t wouterd/java7 java/ 6 | 7 | docker build -t wouterd/kafka kafka/ 8 | 9 | docker build -t wouterd/zookeeper zookeeper/ 10 | 11 | gradle-1-12 -p api distTar 12 | 13 | docker build -t my-little-api api/ 14 | -------------------------------------------------------------------------------- /java/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:jessie 2 | 3 | MAINTAINER Wouter Danes 4 | 5 | RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" >> /etc/apt/sources.list && \ 6 | TMPNAME=$(tempfile) && \ 7 | apt-get update >> /dev/null 2> $TMPNAME && \ 8 | PGPKEY=`cat $TMPNAME | cut -d":" -f6 | cut -d" " -f3` && \ 9 | apt-key adv --keyserver keyserver.ubuntu.com --recv-keys $PGPKEY && \ 10 | rm $TMPNAME && \ 11 | apt-get update && \ 12 | echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ 13 | echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \ 14 | DEBIAN_FRONTEND=noninteractive apt-get install -y oracle-java7-installer oracle-java7-set-default 15 | 16 | #Clean up apt-repository 17 | RUN apt-get clean && \ 18 | rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 19 | -------------------------------------------------------------------------------- /java/README.md: -------------------------------------------------------------------------------- 1 | wouterd/java7 2 | ============= 3 | 4 | Docker image with java7 (oracle) based on debian:jessie 5 | -------------------------------------------------------------------------------- /java8/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:jessie 2 | 3 | MAINTAINER Wouter Danes 4 | 5 | RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu precise main" >> /etc/apt/sources.list && \ 6 | TMPNAME=$(tempfile) && \ 7 | apt-get update >> /dev/null 2> $TMPNAME && \ 8 | PGPKEY=`cat $TMPNAME | cut -d":" -f6 | cut -d" " -f3` && \ 9 | apt-key adv --keyserver keyserver.ubuntu.com --recv-keys $PGPKEY && \ 10 | rm $TMPNAME && \ 11 | apt-get update && \ 12 | echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ 13 | echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \ 14 | DEBIAN_FRONTEND=noninteractive apt-get install -y oracle-java8-installer oracle-java8-set-default 15 | 16 | #Clean up apt-repository 17 | RUN apt-get clean && \ 18 | rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 19 | -------------------------------------------------------------------------------- /kafka/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM wouterd/java7 2 | 3 | MAINTAINER Wouter Danes 4 | 5 | ADD http://ftp.nluug.nl/internet/apache/kafka/0.8.1.1/kafka_2.9.2-0.8.1.1.tgz /kafka.tgz 6 | 7 | RUN mkdir -p /kafka && \ 8 | cd /kafka && \ 9 | tar xvfz ../kafka.tgz --strip-components=1 && \ 10 | rm /kafka.tgz 11 | 12 | ADD start-server.sh / 13 | 14 | RUN chmod +x /start-server.sh 15 | 16 | ADD server.properties.initial /server.properties 17 | 18 | CMD ["/start-server.sh"] 19 | 20 | EXPOSE 9092 21 | -------------------------------------------------------------------------------- /kafka/server.properties.initial: -------------------------------------------------------------------------------- 1 | # The port the socket server listens on 2 | port=9092 3 | 4 | # The number of threads handling network requests 5 | num.network.threads=2 6 | 7 | # The number of threads doing disk I/O 8 | num.io.threads=8 9 | 10 | # The send buffer (SO_SNDBUF) used by the socket server 11 | socket.send.buffer.bytes=1048576 12 | 13 | # The receive buffer (SO_RCVBUF) used by the socket server 14 | socket.receive.buffer.bytes=1048576 15 | 16 | # The maximum size of a request that the socket server will accept (protection against OOM) 17 | socket.request.max.bytes=104857600 18 | 19 | ############################# Log Basics ############################# 20 | 21 | # A comma seperated list of directories under which to store log files 22 | log.dirs=/tmp/kafka-logs 23 | 24 | # The default number of log partitions per topic. More partitions allow greater 25 | # parallelism for consumption, but this will also result in more files across 26 | # the brokers. 27 | num.partitions=2 28 | 29 | ############################# Log Retention Policy ############################# 30 | 31 | # The following configurations control the disposal of log segments. The policy can 32 | # be set to delete segments after a period of time, or after a given size has accumulated. 33 | # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens 34 | # from the end of the log. 35 | 36 | # The minimum age of a log file to be eligible for deletion 37 | log.retention.hours=168 38 | 39 | # The maximum size of a log segment file. When this size is reached a new log segment will be created. 40 | log.segment.bytes=536870912 41 | 42 | # The interval at which log segments are checked to see if they can be deleted according 43 | # to the retention policies 44 | log.retention.check.interval.ms=60000 45 | 46 | # By default the log cleaner is disabled and the log retention policy will default to just delete segments after their retention expires. 47 | # If log.cleaner.enable=true is set the cleaner will be enabled and individual logs can then be marked for log compaction. 48 | log.cleaner.enable=false 49 | 50 | # Timeout in ms for connecting to zookeeper 51 | zookeeper.connection.timeout.ms=1000000 52 | 53 | # root directory for all kafka znodes. 54 | #zookeeper.connect=localhost:2181 55 | 56 | # The id of the broker. This must be set to a unique integer for each broker. 57 | #broker.id=0 58 | -------------------------------------------------------------------------------- /kafka/start-server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z ${BROKER_ID} ] ; then 4 | echo 'No BROKER_ID specified, please specify one between 1 and 255' 5 | exit -1 6 | fi 7 | 8 | if [ -z ${ZOO} ] ; then 9 | echo 'No Zookeeper connection string (ZOO) specified.' 10 | exit -1 11 | fi 12 | 13 | echo "broker.id=${BROKER_ID}"$'\n' >> /server.properties 14 | echo "zookeeper.connect=${ZOO}"$'\n' >> /server.properties 15 | 16 | /kafka/bin/kafka-server-start.sh /server.properties 17 | -------------------------------------------------------------------------------- /shutdown.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for i in {1..4} ; do 4 | docker rm -f kafka${i} 5 | done 6 | 7 | for i in {1..5} ; do 8 | docker rm -f zoo${i} 9 | done 10 | 11 | for i in {1..5} ; do 12 | docker rm -f api${i} 13 | done 14 | -------------------------------------------------------------------------------- /startup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | if [ ! ${nopull} ] ; then 6 | echo 'Pulling all images needed, just to be sure' 7 | docker pull busybox 8 | docker pull wouterd/zookeeper 9 | docker pull wouterd/kafka 10 | fi 11 | 12 | # Need a volume to read the config from 13 | conf_container=zoo1 14 | 15 | # Start the zookeeper containers 16 | for i in {1..5} ; do 17 | if [ "${i}" == "1" ] ; then 18 | docker run -d -v /zoo/conf --name "zoo${i}" -e ZOO_ID=${i} wouterd/zookeeper 19 | else 20 | docker run -d --volumes-from ${conf_container} --name "zoo${i}" -e ZOO_ID=${i} wouterd/zookeeper 21 | fi 22 | done 23 | 24 | config=$(cat zoo.cfg.initial) 25 | 26 | # Look up the zookeeper instance IPs and create the config file 27 | for i in {1..5} ; do 28 | container_name=zoo${i} 29 | container_ip=$(docker inspect --format '{{.NetworkSettings.IPAddress}}' ${container_name}) 30 | line="server.${i}=${container_ip}:2888:3888" 31 | config="${config}"$'\n'"${line}" 32 | done 33 | 34 | # Write the config to the config container 35 | echo "${config}" | docker run -i --rm --volumes-from ${conf_container} busybox sh -c 'cat > /zoo/conf/zoo.cfg' 36 | 37 | zoo_links='--link zoo1:zoo1 --link zoo2:zoo2 --link zoo3:zoo3 --link zoo4:zoo4 --link zoo5:zoo5' 38 | zoo_connect='zoo1:2181,zoo2:2181,zoo3:2181,zoo4:2181,zoo5:2181' 39 | 40 | # Start 4 Kafka's 41 | # for i in {1..4} ; do 42 | # docker run -d ${zoo_links} -e BROKER_ID=${i} -e ZOO=${zoo_connect} --name kafka${i} -h kafka${i} wouterd/kafka 43 | # done 44 | 45 | # Start 5 apis 46 | for i in {1..5} ; do 47 | docker run -d ${zoo_links} -e ZOO=${zoo_connect} --name api${i} my-little-api 48 | done 49 | -------------------------------------------------------------------------------- /zoo.cfg.initial: -------------------------------------------------------------------------------- 1 | tickTime=2000 2 | dataDir=/var/lib/zookeeper 3 | clientPort=2181 4 | initLimit=5 5 | syncLimit=2 6 | -------------------------------------------------------------------------------- /zookeeper/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM wouterd/java7 2 | 3 | MAINTAINER Wouter Danes 4 | 5 | ADD http://ftp.nluug.nl/internet/apache/zookeeper/stable/zookeeper-3.4.6.tar.gz /zookeeper.tar.gz 6 | 7 | RUN mkdir -p zoo && \ 8 | cd zoo && \ 9 | tar xvfz ../zookeeper.tar.gz --strip-components=1 && \ 10 | rm /zookeeper.tar.gz 11 | 12 | ADD start-server.sh / 13 | 14 | RUN chmod +x /start-server.sh 15 | 16 | CMD ["/start-server.sh"] 17 | 18 | EXPOSE 2888 3888 2181 19 | -------------------------------------------------------------------------------- /zookeeper/README.md: -------------------------------------------------------------------------------- 1 | A zookeeper image that exposes client and ensemble ports. It waits for a file at /conf/zoo.cfg to start the server. 2 | Ideal when you want to create an ensemble using Docker. See https://github.com/wouterd/docker-zookeeper. 3 | -------------------------------------------------------------------------------- /zookeeper/start-server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z ${ZOO_ID} ] ; then 4 | echo 'No ID specified, please specify one between 1 and 255' 5 | exit -1 6 | fi 7 | 8 | if [ ! -f /conf/zoo.cfg ] ; then 9 | echo 'Waiting for config file to appear...' 10 | while [ ! -f /zoo/conf/zoo.cfg ] ; do 11 | sleep 1 12 | done 13 | echo 'Config file found, starting server.' 14 | fi 15 | 16 | mkdir -p /var/lib/zookeeper 17 | 18 | echo "${ZOO_ID}" > /var/lib/zookeeper/myid 19 | 20 | /zoo/bin/zkServer.sh start-foreground 21 | --------------------------------------------------------------------------------