├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── bin ├── publish-node.sh ├── rabbitmq-reload ├── rabbitmq-start └── supervisord-wrapper.sh └── etc ├── confd ├── conf.d │ ├── hosts.toml │ └── rabbitmq.toml ├── confd.toml └── templates │ ├── hosts.dynamic.tmpl │ └── rabbitmq.config.tmpl └── supervisor ├── conf.d └── supervisord.conf └── supervisord.conf /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | 25 | # PyInstaller 26 | # Usually these files are written by a python script from a template 27 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 28 | *.manifest 29 | *.spec 30 | 31 | # Installer logs 32 | pip-log.txt 33 | pip-delete-this-directory.txt 34 | 35 | # Unit test / coverage reports 36 | htmlcov/ 37 | .tox/ 38 | .coverage 39 | .cache 40 | nosetests.xml 41 | coverage.xml 42 | 43 | # Translations 44 | *.mo 45 | *.pot 46 | 47 | # Django stuff: 48 | *.log 49 | 50 | # Sphinx documentation 51 | docs/_build/ 52 | 53 | # PyBuilder 54 | target/ 55 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:2.7-slim 2 | 3 | ENV ETCDCTL_VERSION v2.3.1 4 | ENV CONFD_VERSION 0.12.0-alpha3 5 | ENV DUMB_INIT_VERSION 1.2.0 6 | 7 | # Install Native dependencies 8 | RUN \ 9 | apt-get update && \ 10 | apt-get upgrade -y && \ 11 | # Curl Wget 12 | apt-get install -y curl wget && \ 13 | 14 | # Rabbitmq 15 | wget -qO - https://www.rabbitmq.com/rabbitmq-release-signing-key.asc | apt-key add - && \ 16 | echo "deb http://www.rabbitmq.com/debian/ testing main" > /etc/apt/sources.list.d/rabbitmq.list && \ 17 | apt-get update && \ 18 | apt-get install -y rabbitmq-server && \ 19 | 20 | rabbitmq-plugins enable rabbitmq_management && \ 21 | echo "[{rabbit, [{loopback_users, []}]}]." > /etc/rabbitmq/rabbitmq.config && \ 22 | sed --follow-symlinks \ 23 | -e 's/-rabbit error_logger.*/-rabbit error_logger tty \\/' \ 24 | -e 's/-rabbit sasl_error_logger.*/-rabbit sasl_error_logger tty \\/' \ 25 | -e 's/-sasl sasl_error_logger.*/-sasl sasl_error_logger tty \\/' \ 26 | -i /usr/lib/rabbitmq/bin/rabbitmq-server && \ 27 | 28 | # Etcd 29 | curl -L https://github.com/coreos/etcd/releases/download/$ETCDCTL_VERSION/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz -o /tmp/etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz && \ 30 | cd /tmp && gzip -dc etcd-$ETCDCTL_VERSION-linux-amd64.tar.gz | tar -xof - && \ 31 | cp -f /tmp/etcd-$ETCDCTL_VERSION-linux-amd64/etcdctl /usr/local/bin && \ 32 | 33 | # Supervisor 34 | pip install supervisor==3.3.1 supervisor-stdout && \ 35 | mkdir -p /var/log/supervisor && \ 36 | 37 | # Confd 38 | curl -L https://github.com/kelseyhightower/confd/releases/download/v$CONFD_VERSION/confd-${CONFD_VERSION}-linux-amd64 -o /usr/local/bin/confd && \ 39 | chmod 555 /usr/local/bin/confd && \ 40 | 41 | # Dumb Init 42 | wget -O /usr/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v${DUMB_INIT_VERSION}/dumb-init_${DUMB_INIT_VERSION}_amd64 && \ 43 | chmod +x /usr/bin/dumb-init && \ 44 | 45 | # Cleanup 46 | apt-get clean && \ 47 | rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /tmp/* 48 | 49 | #Supervisor Config 50 | ADD etc/supervisor /etc/supervisor 51 | RUN ln -sf /etc/supervisor/supervisord.conf /etc/supervisord.conf 52 | 53 | #Confd Defaults 54 | ADD etc/confd /etc/confd 55 | 56 | #Add custom scipts 57 | ADD bin /usr/local/bin 58 | RUN chmod -R +x /usr/local/bin 59 | 60 | # Define mount points. 61 | VOLUME ["/var/lib/rabbitmq"] 62 | 63 | EXPOSE 5672 44001 15672 25672 4369 64 | 65 | ENTRYPOINT ["/usr/bin/dumb-init", "/usr/local/bin/supervisord-wrapper.sh"] 66 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Totem 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rabbitmq-cluster [![](https://badge.imagelayers.io/totem/rabbitmq-cluster:develop.svg)](https://imagelayers.io/?images=totem/rabbitmq-cluster:develop 'Get your own badge on imagelayers.io') 2 | 3 | Rabbitmq Cluster using Docker. 4 | 5 | ## Status 6 | In Testing 7 | 8 | ## Requirements 9 | - Docker 1.2 + 10 | - Etcd 0.4.6 + 11 | 12 | ## Features 13 | - Automatic seed node detection using etcd (Seed node will initialize the settings for cluster). 14 | - rabbitmq.conf file for automatic cluster initialization using etcd discovery. 15 | - hosts file initialization using etcd discovery. 16 | - Centralized logging using syslog. 17 | - Random Erlang Cookie for the cluster. 18 | 19 | ## Creating cluster (different machines) 20 | Assuming that there are 2 machines (machine-1, machine-2), with data directory 21 | at /data (you may choose different path). You may deploy all nodes at once using command: 22 | 23 | ``` 24 | RABBITMQ_USER= 25 | RABBITMQ_PASSWORD= 26 | sudo docker run --rm -P -p 5672:5672 -p 25672:25672 -p 4369:4369 -p 44001:44001 --name node1 -e HOST_IP=$COREOS_PRIVATE_IPV4 -e RABBITMQ_USER=$RABBITMQ_USER -e RABBITMQ_PASSWORD=$RABBITMQ_PASSWORD -v /dev/log:/dev/log -v /data:/var/lib/rabbitmq totem/rabbitmq-cluster 27 | ``` 28 | 29 | where $COREOS_PRIVATE_IPV4 is the private IP address for the host. 30 | (If using ec2, ensure that machines can talk to each other on ports: 4369, 5672, 35197) 31 | 35197) 32 | 33 | ## Proxying Cluster using Yoda 34 | If you are using dynamic proxy like yoda (that supports tcp) for port 5672, you may 35 | need to add additional port binding in Yoda proxy. 36 | e.g.: -p 5673:5672 37 | 38 | To configure yoda with tcp listeners and sidekick discovery, see [Yoda Proxy](https://github.com/totem/yoda-proxy) 39 | and [Yoda Discover](https://github.com/totem/yoda-discover) 40 | 41 | -------------------------------------------------------------------------------- /bin/publish-node.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -le 2 | 3 | echo "Starting discover for rabbitmq nodes" 4 | 5 | ETCDCTL="etcdctl --peers $ETCD_URL" 6 | PUBLISH_NODE_TTL=${PUBLISH_NODE_TTL:-120} 7 | PUBLISH_NODE_POLL=${PUBLISH_NODE_POLL:-60s} 8 | 9 | NODE=$(cat /var/lib/rabbitmq/nodename) 10 | HOST_IP=${HOST_IP:-$(/sbin/ifconfig eth0 | grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}')} 11 | 12 | while supervisorctl status rabbitmq-server | grep 'RUNNING' 13 | do 14 | echo "Publishing $ETCD_RABBITMQ_BASE/rabbitmq/nodes/${NODE} ${HOST_IP} with ttl ${PUBLISH_NODE_TTL}" 15 | ${ETCDCTL} set --ttl ${PUBLISH_NODE_TTL} $ETCD_RABBITMQ_BASE/rabbitmq/nodes/${NODE} ${HOST_IP} 16 | sleep ${PUBLISH_NODE_POLL} 17 | done 18 | -------------------------------------------------------------------------------- /bin/rabbitmq-reload: -------------------------------------------------------------------------------- 1 | #!/bin/bash -el 2 | 3 | NODE=$(cat /var/lib/rabbitmq/nodename) 4 | ETCDCTL="etcdctl --peers $ETCD_URL" 5 | # Set the nodename 6 | export RABBITMQ_NODENAME=rabbit@${NODE} 7 | export RABBITMQ_USER=${RABBITMQ_USER:-rabbitmq} 8 | export RABBITMQ_PASSWORD=${RABBITMQ_PASSWORD:-rabbitmq} 9 | export RABBITMQ_CLUSTER_NAME=${RABBITMQ_CLUSTER_NAME:-totem} 10 | 11 | 12 | if [ "$($ETCDCTL get $ETCD_RABBITMQ_BASE/rabbitmq/initialized/$NODE)" == 'true' ]; then 13 | echo "Node is already initialized. Skipping reload." 14 | exit 0 15 | fi 16 | 17 | 18 | if [ "$($ETCDCTL get $ETCD_RABBITMQ_BASE/rabbitmq/seed)" == "$NODE" ]; then 19 | # Give some time time for inital startup. (Need better way to handle this.) 20 | sleep 15s 21 | rabbitmqctl delete_user guest || echo 'Skip deletion of guest account. Probably deleted before.' 22 | rabbitmqctl set_cluster_name $RABBITMQ_CLUSTER_NAME 23 | rabbitmqctl set_policy ha-all "^ha\." '{"ha-mode":"all", "ha-sync-mode":"automatic"}' 24 | rabbitmqctl add_user ${RABBITMQ_USER} ${RABBITMQ_PASSWORD} || echo 'Rabbitmq user already exists' 25 | rabbitmqctl set_permissions -p / ${RABBITMQ_USER} ".*" ".*" ".*" 26 | rabbitmqctl set_user_tags ${RABBITMQ_USER} administrator 27 | echo "Cluster Initialized." 28 | else 29 | if ! $ETCDCTL mk $ETCD_RABBITMQ_BASE/rabbitmq/initialized/$NODE false; then 30 | echo "Another process is already trying to initialize this node. Skipping initialization" 31 | exit 0; 32 | fi 33 | # Ensure that we are not publishing non-seed node yet. 34 | supervisorctl stop publish-node 35 | # Give some time time for startup. (Need better way to handle this.) 36 | sleep 15s 37 | supervisorctl start publish-node 38 | fi 39 | 40 | 41 | $ETCDCTL set $ETCD_RABBITMQ_BASE/rabbitmq/initialized/$NODE true 42 | echo "Node Initialized." 43 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /bin/rabbitmq-start: -------------------------------------------------------------------------------- 1 | #!/bin/bash -le 2 | 3 | ETCDCTL="etcdctl --peers $ETCD_URL" 4 | ulimit -n 1024 5 | 6 | NODE=$(cat /var/lib/rabbitmq/nodename) 7 | # Set the nodename 8 | export RABBITMQ_NODENAME=rabbit@${NODE} 9 | #export RABBITMQ_NODENAME=${RABBITMQ_NODENAME:-rabbit@$(hostname -f)} 10 | 11 | # call "rabbitmqctl stop" when exiting 12 | trap "{ echo Stopping rabbitmq; rabbitmqctl stop; exit 0; }" EXIT 13 | 14 | rabbitmq-server $@ -------------------------------------------------------------------------------- /bin/supervisord-wrapper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | export HOST_IP="${HOST_IP:-$(/sbin/ip route|awk '/default/ { print $3 }')}" 4 | export ETCD_URL="${ETCD_URL:-${HOST_IP}:4001}" 5 | export ETCD_RABBITMQ_BASE="${ETCD_RABBITMQ_BASE:-/totem}" 6 | export NODE_PREFIX="${NODE_PREFIX:-totem-rabbitmq}" 7 | export RABBITMQ_CLUSTER_NAME="${RABBITMQ_CLUSTER_NAME:-totem}" 8 | export LOG_IDENTIFIER="${LOG_IDENTIFIER:-rabbitmq-cluster}" 9 | 10 | ETCDCTL="etcdctl --peers $ETCD_URL" 11 | 12 | if ! $ETCDCTL cluster-health | grep 'cluster is healthy'; then 13 | echo "ERROR: Etcd cluster is not healthy. Supervisord-wrapper can not start. Command failed $ETCDCTL cluster-health." 14 | exit 1; 15 | fi 16 | 17 | 18 | # Check if nodename exists. If not create a new node 19 | if [ ! -f /var/lib/rabbitmq/nodename ]; then 20 | # Generate Persistent host file 21 | NODE=${NODE:-${NODE_PREFIX}-$(< /dev/urandom tr -dc A-Z-a-z-0-9 | head -c${1:-10};echo;)} 22 | echo $NODE > /var/lib/rabbitmq/nodename 23 | fi 24 | NODE=$(cat /var/lib/rabbitmq/nodename) 25 | 26 | echo "Modifying Host entries..." 27 | echo 127.0.0.1 $NODE >> /etc/hosts 28 | cat /etc/hosts /etc/confd/templates/hosts.dynamic.tmpl > /etc/confd/templates/hosts.tmpl 29 | 30 | echo "Modify confd settings (ETCD_URL, ETCD_RABBITMQ_BASE)" 31 | sed -i -e "s/127.0.0.1[:]4001/$ETCD_URL/g" -e "s|/totem|$ETCD_RABBITMQ_BASE|g" /etc/confd/confd.toml 32 | 33 | echo "Check/Create Erlang Cookie (For RabbitMq cluster)" 34 | 35 | $ETCDCTL mk $ETCD_RABBITMQ_BASE/rabbitmq/cookie $(< /dev/urandom tr -dc A-Z-a-z-0-9 | head -c${1:-32};echo;) || echo "Utilizing existing cookie..." 36 | ERLANG_COOKIE=$($ETCDCTL get $ETCD_RABBITMQ_BASE/rabbitmq/cookie) 37 | if [ -z $ERLANG_COOKIE ]; then 38 | echo "ERROR: Erlang cookie was found empty. Can not continue...." 39 | exit 10 40 | fi 41 | echo "$ERLANG_COOKIE" > /var/lib/rabbitmq/.erlang.cookie 42 | chmod 600 /var/lib/rabbitmq/.erlang.cookie 43 | 44 | 45 | echo "Changing owner for attached volume to rabbitmq" 46 | chown -R rabbitmq:rabbitmq /var/lib/rabbitmq 47 | 48 | echo "Updating environment file" 49 | cat <> /etc/environment 50 | ETCDCTL="$ETCDCTL" 51 | ETCD_URL="$ETCD_URL" 52 | ETCD_RABBITMQ_BASE=$ETCD_RABBITMQ_BASE 53 | NODE=$NODE 54 | RABBITMQ_NODENAME=rabbit@$NODE 55 | RABBITMQ_CLUSTER_NAME=${RABBITMQ_CLUSTER_NAME} 56 | LOG_IDENTIFIER=${LOG_IDENTIFIER} 57 | END 58 | 59 | echo "Registering shutdown hook prior to shutdown" 60 | shutdown () { 61 | set +e; 62 | echo Stopping supervisor; 63 | kill -s SIGTERM "$(cat /var/run/supervisord.pid)"; 64 | exit 0 65 | } 66 | trap 'shutdown' EXIT 67 | 68 | if ! $ETCDCTL mk $ETCD_RABBITMQ_BASE/rabbitmq/seed $NODE; then 69 | seed="$($ETCDCTL get $ETCD_RABBITMQ_BASE/rabbitmq/seed)" 70 | if [ "$seed" != "$NODE" ]; then 71 | while [ "$($ETCDCTL get $ETCD_RABBITMQ_BASE/rabbitmq/initialized/$seed)" != 'true' ]; do 72 | echo "Waiting for seed node initialization..." 73 | sleep 60s 74 | done 75 | fi 76 | fi 77 | 78 | if [ -f /var/lib/rabbitmq/reset ]; then 79 | # Remove the deprecated file as we no longer need it. 80 | rm /var/lib/rabbitmq/reset 81 | $ETCDCTL set $ETCD_RABBITMQ_BASE/rabbitmq/initialized/$NODE true 82 | fi 83 | 84 | if [ "$($ETCDCTL get $ETCD_RABBITMQ_BASE/rabbitmq/initialized/$NODE || echo 'false' )" != 'true' ]; then 85 | echo "Removing mnesia folder as the node is not initialized..." 86 | rm -rf /var/lib/rabbitmq/mnesia 87 | fi 88 | 89 | 90 | echo "Starting supervisord" 91 | supervisord -n 92 | -------------------------------------------------------------------------------- /etc/confd/conf.d/hosts.toml: -------------------------------------------------------------------------------- 1 | [template] 2 | src = "hosts.tmpl" 3 | dest = "/etc/hosts" 4 | uid = 0 5 | gid = 0 6 | mode = "0644" 7 | keys = [ 8 | "rabbitmq" 9 | ] -------------------------------------------------------------------------------- /etc/confd/conf.d/rabbitmq.toml: -------------------------------------------------------------------------------- 1 | [template] 2 | src = "rabbitmq.config.tmpl" 3 | dest = "/etc/rabbitmq/rabbitmq.config" 4 | uid = 0 5 | gid = 0 6 | mode = "0644" 7 | keys = [ 8 | "rabbitmq" 9 | ] 10 | reload_cmd = "/bin/bash -c '/usr/local/bin/rabbitmq-reload 2>&1 | logger -t $LOG_IDENTIFIER[$$]' " 11 | -------------------------------------------------------------------------------- /etc/confd/confd.toml: -------------------------------------------------------------------------------- 1 | confdir = "/etc/confd" 2 | interval = 30 3 | prefix = "/totem" 4 | nodes = [ 5 | "http://127.0.0.1:4001", 6 | ] 7 | quiet = true 8 | debug = false 9 | verbose = false 10 | -------------------------------------------------------------------------------- /etc/confd/templates/hosts.dynamic.tmpl: -------------------------------------------------------------------------------- 1 | # Dynamic Host Entries. This is concatenated with /etc/hosts to form hosts.toml 2 | {{ if printf "/rabbitmq/nodes" | ls }}{{ range printf "/rabbitmq/nodes/*" | gets }} 3 | {{ .Value }} {{ .Key | base }} 4 | {{ end }}{{ end }} -------------------------------------------------------------------------------- /etc/confd/templates/rabbitmq.config.tmpl: -------------------------------------------------------------------------------- 1 | [{rabbit, [ 2 | {cluster_partition_handling, pause_minority}, 3 | {loopback_users, []}, 4 | {cluster_nodes, {{"{"}}{{ if printf "/rabbitmq/nodes" | ls }}[{{ range $index, $node := printf "/rabbitmq/nodes/*" | gets }}{{if $index }},{{ end }}'rabbit@{{ $node.Key | base }}'{{ end }}]{{ else }}[]{{ end }}, disc{{ "}}" }}, 5 | {kernel, [ 6 | {inet_dist_listen_max, 44001}, 7 | {inet_dist_listen_min, 44001} 8 | ]} 9 | ]}]. 10 | -------------------------------------------------------------------------------- /etc/supervisor/conf.d/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | nodaemon=true 3 | 4 | [program:rabbitmq-server] 5 | command=/usr/local/bin/rabbitmq-start 6 | startsecs=5 7 | autostart=true 8 | autorestart=true 9 | stdout_events_enabled = true 10 | stderr_events_enabled = true 11 | 12 | [program:confd] 13 | command=/usr/local/bin/confd 14 | autorestart=true 15 | startsecs=5 16 | stdout_events_enabled = true 17 | stderr_events_enabled = true 18 | 19 | [program:publish-node] 20 | command=/usr/local/bin/publish-node.sh 21 | startsecs=5 22 | autostart=true 23 | autorestart=true 24 | startretries=1000 25 | stdout_events_enabled = true 26 | stderr_events_enabled = true 27 | 28 | [eventlistener:stdout] 29 | command = supervisor_stdout 30 | buffer_size = 100 31 | events = PROCESS_LOG 32 | result_handler = supervisor_stdout:event_handler 33 | -------------------------------------------------------------------------------- /etc/supervisor/supervisord.conf: -------------------------------------------------------------------------------- 1 | ; supervisor config file 2 | 3 | [inet_http_server] 4 | port = 127.0.0.1:10001 5 | 6 | [supervisord] 7 | logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log) 8 | pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid) 9 | childlogdir=/var/log/supervisor ; ('AUTO' child log dir, default $TEMP) 10 | 11 | ; the below section must remain in the config file for RPC 12 | ; (supervisorctl/web interface) to work, additional interfaces may be 13 | ; added by defining them in separate rpcinterface: sections 14 | [rpcinterface:supervisor] 15 | supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface 16 | 17 | [supervisorctl] 18 | serverurl=http://localhost:10001 ; use a unix:// URL for a unix socket 19 | 20 | ; The [include] section can just contain the "files" setting. This 21 | ; setting can list multiple files (separated by whitespace or 22 | ; newlines). It can also contain wildcards. The filenames are 23 | ; interpreted as relative to this file. Included files *cannot* 24 | ; include files themselves. 25 | 26 | [include] 27 | files = /etc/supervisor/conf.d/*.conf 28 | --------------------------------------------------------------------------------