├── .dockerignore ├── .editorconfig ├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── bin └── covertool ├── build.config ├── consul-test.json ├── deploy-container.sh ├── docker-compose.yml ├── erlang.mk ├── etc └── rabbit-test.config ├── examples ├── compose_consul_haproxy │ ├── README.md │ └── docker-compose.yml ├── compose_etcd2 │ ├── README.md │ ├── conf │ │ └── rabbitmq.config │ └── docker-compose.yml ├── compose_etcd2_haproxy │ ├── README.md │ ├── conf │ │ └── rabbitmq.config │ └── docker-compose.yml ├── k8s_minikube │ ├── README.md │ └── rabbitmq.yaml ├── k8s_rbac_statefulsets │ ├── README.md │ ├── rabbitmq-rbac.yaml │ ├── rabbitmq-service.yaml │ └── rabbitmq.yaml └── k8s_statefulsets │ ├── README.md │ ├── rabbitmq-service.yaml │ └── rabbitmq.yaml ├── include └── autocluster.hrl ├── rabbitmq-autocluster.iml ├── rabbitmq-components.mk ├── root ├── launch.sh ├── usr │ └── lib │ │ └── rabbitmq │ │ ├── .gitkeep │ │ └── etc │ │ └── rabbitmq │ │ └── rabbitmq.config └── var │ └── lib │ └── rabbitmq │ └── .erlang.cookie ├── src ├── autocluster.erl ├── autocluster_app.erl ├── autocluster_aws.erl ├── autocluster_backend.erl ├── autocluster_cleanup.erl ├── autocluster_config.erl ├── autocluster_consul.erl ├── autocluster_dns.erl ├── autocluster_etcd.erl ├── autocluster_httpc.erl ├── autocluster_k8s.erl ├── autocluster_log.erl ├── autocluster_periodic.erl ├── autocluster_sup.erl └── autocluster_util.erl └── test ├── etcd_SUITE.erl ├── etcd_lock_statem.erl ├── health_check_SUITE.erl ├── periodic_SUITE.erl ├── src ├── autocluster_all_tests.erl ├── autocluster_aws_tests.erl ├── autocluster_config_tests.erl ├── autocluster_consul_tests.erl ├── autocluster_dns_tests.erl ├── autocluster_etcd_tests.erl ├── autocluster_httpc_tests.erl ├── autocluster_k8s_tests.erl ├── autocluster_sup_tests.erl ├── autocluster_testing.erl ├── autocluster_tests.erl └── autocluster_util_tests.erl └── unit_test_SUITE.erl /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !root 3 | !plugins 4 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # top-most EditorConfig file 2 | root = true 3 | 4 | # Unix-style newlines with a newline ending every file 5 | [*] 6 | end_of_line = lf 7 | insert_final_newline = true 8 | 9 | [Makefile] 10 | indent_style = tab 11 | 12 | # 2 space indentation 13 | [{*.erl, *.hrl, *.md, *.rst}] 14 | indent_style = space 15 | indent_size = 2 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .erlang.mk 2 | .eunit 3 | deps 4 | *.o 5 | *.beam 6 | *.plt 7 | build 8 | dist 9 | erl_crash.dump 10 | ebin 11 | rel/example_project 12 | .concrete/DEV_MODE 13 | .rebar 14 | docker/rabbitmq_autocluster*ez 15 | .vagrant 16 | .idea 17 | cover 18 | plugins 19 | autocluster.d 20 | bin/rebar 21 | rebar.lock 22 | cobertura.xml 23 | *.tgz 24 | _build 25 | /apps/autocluster_aws/autocluster_aws.d 26 | /logs/ 27 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: erlang 2 | 3 | otp_release: 4 | # Primary target 5 | - 17.5 6 | # Secondary target - runing PropEr tests and checking compatibily with modern erlang 7 | - 19.1 8 | 9 | addons: 10 | apt: 11 | packages: 12 | - curl 13 | - xsltproc 14 | 15 | cache: 16 | apt: true 17 | directories: 18 | - $HOME/.cache/plt 19 | - $HOME/.cache/bin 20 | 21 | install: 22 | - export PATH=$PATH:$HOME/.local/bin 23 | 24 | before_script: 25 | - git checkout -B "${TRAVIS_TAG:-${TRAVIS_BRANCH}}" 26 | - | 27 | if [ ! -e $HOME/.cache/bin/etcd ]; then 28 | mkdir -p $HOME/.cache/bin 29 | (set -e 30 | cd $HOME/.cache/bin 31 | curl -L https://github.com/coreos/etcd/releases/download/v2.3.7/etcd-v2.3.7-linux-amd64.tar.gz -o etcd.tar.gz 32 | tar -xz --strip-components=1 -f etcd.tar.gz 33 | ) 34 | fi 35 | export USE_ETCD=$HOME/.cache/bin/etcd 36 | 37 | script: 38 | - make RABBITMQ_CURRENT_FETCH_URL=https://github.com/rabbitmq/rabbitmq-trick-erlang.mk-into-using-proper-url-for-deps 39 | - | 40 | set -exo pipefail 41 | function ensure_plt() { 42 | # We want this to happen after ct run, so `rabbit` dependency will be fetched. 43 | # Then we can add rabbit/rabbit_common to plt, thus getting 0 44 | # warnings after running dialyzer itself 45 | 46 | # Try to use cached plt if it's valid 47 | if [ -r $HOME/.cache/plt/.autocluster.plt ]; then 48 | cp -va $HOME/.cache/plt/.autocluster.plt . 49 | # Cached PLT can become invalid (i.e. reference some renamed/deleted .beam) 50 | dialyzer --check_plt --plt ./.autocluster.plt || rm -f ./.autocluster.plt 51 | fi 52 | 53 | # Build PLT from scratch if there is no cached copy 54 | if [ ! -f ./.autocluster.plt ]; then 55 | make plt || true # making plt produces some warnings which we don't care about 56 | mkdir -p $HOME/.cache/plt/ 57 | cp -va .autocluster.plt $HOME/.cache/plt/ 58 | fi 59 | } 60 | 61 | function upload_logs() { 62 | tar cjvf logs.tar.bz2 logs/ 63 | echo Uploading logs for further investigation to: 64 | curl -sST logs.tar.bz2 chunk.io 65 | } 66 | 67 | if [ "$TRAVIS_OTP_RELEASE" == "19.1" ]; then 68 | export PROPER_ONLY=true 69 | else 70 | export PROPER_ONLY= 71 | fi 72 | 73 | case "$PROPER_ONLY" in 74 | true) 75 | if ! make test-build ct IS_APP=1 CT_SUITES=etcd; then 76 | upload_logs 77 | exit 1 78 | fi 79 | ;; 80 | *) 81 | pip install --user codecov 82 | if ! make test-build ct COVER=true IS_APP=1; then 83 | upload_logs 84 | exit 1 85 | fi 86 | ensure_plt 87 | make dialyze 88 | ./bin/covertool -cover ct.coverdata -appname autocluster 89 | codecov -f coverage.xml -X search 90 | ;; 91 | esac 92 | set +exo pipefail 93 | 94 | before_deploy: 95 | - make clean 96 | - make dist 97 | - tar cvfz autocluster-${TRAVIS_TAG}.tgz plugins/autocluster*.ez plugins/rabbitmq_aws-*.ez 98 | - echo "TRAVIS_OTP_RELEASE = ${TRAVIS_OTP_RELEASE}" 99 | 100 | deploy: 101 | provider: releases 102 | api_key: 103 | secure: ktklMK+XMOteFt+m9NHhVqKkA1Wo8f9L/cJphUmBMgb3TS+4+vAU50yY8omIyprS8poc3mBWxjYD9p9xdeDnXY2tiFrLDKCWU/jbH3awD0uL6W0Di8BYAVOGhr2Jjjp6gi/B67wHtCtzEoSSNNfMMZ+RWf4GZjJ96NXOLhPRx4k= 104 | file: autocluster-${TRAVIS_TAG}.tgz 105 | skip_cleanup: true 106 | on: 107 | condition: "$TRAVIS_OTP_RELEASE = 17.5" 108 | tags: true 109 | repo: aweber/rabbitmq-autocluster 110 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.6 2 | 3 | # Version of RabbitMQ to install 4 | ENV RABBITMQ_VERSION=3.6.14 \ 5 | ERL_EPMD_PORT=4369 \ 6 | AUTOCLUSTER_VERSION=0.10.0 \ 7 | HOME=/var/lib/rabbitmq \ 8 | PATH=/usr/lib/rabbitmq/sbin:$PATH \ 9 | RABBITMQ_LOGS=- \ 10 | RABBITMQ_SASL_LOGS=- \ 11 | RABBITMQ_DIST_PORT=25672 \ 12 | RABBITMQ_SERVER_ERL_ARGS="+K true +A128 +P 1048576 -kernel inet_default_connect_options [{nodelay,true}]" \ 13 | RABBITMQ_MNESIA_DIR=/var/lib/rabbitmq/mnesia \ 14 | RABBITMQ_PID_FILE=/var/lib/rabbitmq/rabbitmq.pid \ 15 | RABBITMQ_PLUGINS_DIR=/usr/lib/rabbitmq/plugins \ 16 | RABBITMQ_PLUGINS_EXPAND_DIR=/var/lib/rabbitmq/plugins \ 17 | LANG=en_US.UTF-8 18 | 19 | RUN \ 20 | apk --update add \ 21 | coreutils curl xz "su-exec>=0.2" \ 22 | erlang erlang-asn1 erlang-crypto erlang-eldap erlang-erts erlang-inets erlang-mnesia \ 23 | erlang-os-mon erlang-public-key erlang-sasl erlang-ssl erlang-syntax-tools erlang-xmerl && \ 24 | curl -sL -o /tmp/rabbitmq-server-generic-unix-${RABBITMQ_VERSION}.tar.gz https://www.rabbitmq.com/releases/rabbitmq-server/v${RABBITMQ_VERSION}/rabbitmq-server-generic-unix-${RABBITMQ_VERSION}.tar.xz && \ 25 | cd /usr/lib/ && \ 26 | tar xf /tmp/rabbitmq-server-generic-unix-${RABBITMQ_VERSION}.tar.gz && \ 27 | rm /tmp/rabbitmq-server-generic-unix-${RABBITMQ_VERSION}.tar.gz && \ 28 | mv /usr/lib/rabbitmq_server-${RABBITMQ_VERSION} /usr/lib/rabbitmq && \ 29 | curl -sL -o /usr/lib/rabbitmq/plugins/autocluster-${AUTOCLUSTER_VERSION}.ez https://github.com/rabbitmq/rabbitmq-autocluster/releases/download/${AUTOCLUSTER_VERSION}/autocluster-${AUTOCLUSTER_VERSION}.ez && \ 30 | curl -sL -o /usr/lib/rabbitmq/plugins/rabbitmq_aws-${AUTOCLUSTER_VERSION}.ez https://github.com/rabbitmq/rabbitmq-autocluster/releases/download/${AUTOCLUSTER_VERSION}/rabbitmq_aws-${AUTOCLUSTER_VERSION}.ez && \ 31 | apk --purge del curl tar gzip xz 32 | 33 | COPY root/ / 34 | 35 | # Fetch the external plugins and setup RabbitMQ 36 | RUN \ 37 | adduser -D -u 1000 -h $HOME rabbitmq rabbitmq && \ 38 | cp /var/lib/rabbitmq/.erlang.cookie /root/ && \ 39 | chown rabbitmq /var/lib/rabbitmq/.erlang.cookie && \ 40 | chmod 0600 /var/lib/rabbitmq/.erlang.cookie /root/.erlang.cookie && \ 41 | chown -R rabbitmq /usr/lib/rabbitmq /var/lib/rabbitmq && sync && \ 42 | /usr/lib/rabbitmq/sbin/rabbitmq-plugins --offline enable \ 43 | rabbitmq_management \ 44 | rabbitmq_consistent_hash_exchange \ 45 | rabbitmq_federation \ 46 | rabbitmq_federation_management \ 47 | rabbitmq_mqtt \ 48 | rabbitmq_shovel \ 49 | rabbitmq_shovel_management \ 50 | rabbitmq_stomp \ 51 | rabbitmq_web_stomp \ 52 | autocluster 53 | 54 | VOLUME $HOME 55 | EXPOSE 4369 5671 5672 15672 25672 56 | ENTRYPOINT ["/launch.sh"] 57 | CMD ["rabbitmq-server"] 58 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014-2015 AWeber Communications 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | * Neither the name of the rabbitmq-autocluster plugin nor the names of its 13 | contributors may be used to endorse or promote products derived from this 14 | software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 17 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 | IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 20 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 23 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 24 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECT = autocluster 2 | PROJECT_DESCRIPTION = Forms RabbitMQ clusters using a variety of backends (AWS EC2, DNS, Consul, Kubernetes, etc) 3 | PROJECT_MOD = autocluster_app 4 | PROJECT_REGISTERED = autocluster_app autocluster_sup autocluster_cleanup 5 | 6 | define PROJECT_ENV 7 | [] 8 | endef 9 | 10 | DEPS = rabbit_common rabbitmq_aws 11 | 12 | TEST_DEPS += rabbit erlsh rabbitmq_ct_helpers meck 13 | IGNORE_DEPS += rabbitmq_java_client 14 | 15 | DEP_PLUGINS = rabbit_common/mk/rabbitmq-plugin.mk 16 | 17 | NO_AUTOPATCH += rabbitmq_aws 18 | 19 | dep_rabbitmq_aws = git https://github.com/rabbitmq/rabbitmq-aws.git stable 20 | 21 | # FIXME: Use erlang.mk patched for RabbitMQ, while waiting for PRs to be 22 | # reviewed and merged. 23 | 24 | ERLANG_MK_REPO = https://github.com/rabbitmq/erlang.mk.git 25 | ERLANG_MK_COMMIT = rabbitmq-tmp 26 | BUILD_DEPS+= rabbit ranch 27 | current_rmq_ref = stable 28 | 29 | include rabbitmq-components.mk 30 | include erlang.mk 31 | 32 | # -------------------------------------------------------------------- 33 | # Testing. 34 | # -------------------------------------------------------------------- 35 | 36 | plt: PLT_APPS += rabbit rabbit_common ranch mnesia ssl compiler crypto common_test inets sasl ssh test_server snmp xmerl observer runtime_tools tools debugger edoc syntax_tools et os_mon hipe public_key webtool wx asn1 otp_mibs 37 | 38 | TEST_CONFIG_FILE=$(CURDIR)/etc/rabbit-test 39 | 40 | WITH_BROKER_TEST_COMMANDS := autocluster_all_tests:run() 41 | -------------------------------------------------------------------------------- /bin/covertool: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/rabbitmq-autocluster/e9aef34aa7a50b78f8230db9d7be712bedb630b2/bin/covertool -------------------------------------------------------------------------------- /build.config: -------------------------------------------------------------------------------- 1 | # Do *not* comment or remove core modules 2 | # unless you know what you are doing. 3 | # 4 | # Feel free to comment plugins out however. 5 | 6 | # Core modules. 7 | core/core 8 | index/* 9 | core/index 10 | core/deps 11 | 12 | # Plugins that must run before Erlang code gets compiled. 13 | plugins/erlydtl 14 | plugins/protobuffs 15 | 16 | # Core modules, continued. 17 | core/erlc 18 | core/docs 19 | core/rel 20 | core/test 21 | core/compat 22 | 23 | # Plugins. 24 | plugins/asciidoc 25 | plugins/bootstrap 26 | plugins/c_src 27 | plugins/ci 28 | plugins/ct 29 | plugins/dialyzer 30 | plugins/edoc 31 | plugins/elvis 32 | plugins/escript 33 | # plugins/eunit 34 | plugins/relx 35 | plugins/shell 36 | plugins/triq 37 | plugins/xref 38 | 39 | # Plugins enhancing the functionality of other plugins. 40 | plugins/cover 41 | 42 | # Core modules which can use variables from plugins. 43 | core/deps-tools 44 | -------------------------------------------------------------------------------- /consul-test.json: -------------------------------------------------------------------------------- 1 | { 2 | "acl_datacenter": "test", 3 | "bootstrap": true, 4 | "data_dir": "/tmp/consul", 5 | "datacenter": "test", 6 | "server": true 7 | } 8 | -------------------------------------------------------------------------------- /deploy-container.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker build -t aweber/rabbitmq-autocluster:$1 . 3 | docker tag aweber/rabbitmq-autocluster:$1 aweber/rabbitmq-autocluster:latest 4 | docker push aweber/rabbitmq-autocluster:$1 5 | docker push aweber/rabbitmq-autocluster:latest 6 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | consul: 2 | image: gavinmroy/alpine-consul:0.5.2-0 3 | ports: 4 | - 8300 5 | - 8301 6 | - 8301/udp 7 | - 8302/udp 8 | - 8400 9 | - 8500 10 | 11 | etcd2: 12 | image: slintes/etcd2:latest 13 | ports: 14 | - 2379 15 | - 2380 16 | - 4001 17 | - 7001 18 | 19 | -------------------------------------------------------------------------------- /etc/rabbit-test.config: -------------------------------------------------------------------------------- 1 | [{rabbit, [ 2 | {loopback_users, []}, 3 | {log_levels, [{autocluster, debug}, {connection, info}]} 4 | ]}]. 5 | -------------------------------------------------------------------------------- /examples/compose_consul_haproxy/README.md: -------------------------------------------------------------------------------- 1 | This example shows how to create a dynamic RabbitMQ cluster using: 2 | 3 | 1. [Docker compose](https://docs.docker.com/compose/) 4 | 5 | 2. [Consul](https://www.consul.io) 6 | 7 | 3. [HA proxy](https://github.com/docker/dockercloud-haproxy) 8 | 9 | --- 10 | 11 | How to run: 12 | 13 | ``` 14 | docker-compose up 15 | ``` 16 | 17 | How to scale: 18 | 19 | ``` 20 | docker-compose scale rabbit=3 21 | ``` 22 | 23 | 24 | --- 25 | 26 | Check running status: 27 | 28 | - Consul Management: http://localhost:8500/ui/ 29 | - RabbitMQ Management: http://localhost:15672/#/ 30 | -------------------------------------------------------------------------------- /examples/compose_consul_haproxy/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | services: 3 | rabbit: 4 | environment: 5 | - TCP_PORTS=15672, 5672 6 | - AUTOCLUSTER_TYPE=consul 7 | - AUTOCLUSTER_DELAY=60 8 | - CONSUL_HOST=consul 9 | - CONSUL_SVC_ADDR_AUTO=true 10 | - AUTOCLUSTER_CLEANUP=true 11 | - CLEANUP_WARN_ONLY=false 12 | - CONSUL_DEREGISTER_AFTER=60 13 | networks: 14 | - back 15 | image: pivotalrabbitmq/rabbitmq-autocluster 16 | expose: 17 | - 15672 18 | - 5672 19 | - 5671 20 | - 15671 21 | tty: true 22 | command: sh -c "sleep 20; rabbitmq-server;" 23 | lb: 24 | image: dockercloud/haproxy 25 | environment: 26 | - MODE=tcp 27 | links: 28 | - rabbit 29 | volumes: 30 | - /var/run/docker.sock:/var/run/docker.sock 31 | ports: 32 | - 15672:15672 33 | - 5672:5672 34 | networks: 35 | - back 36 | consul1: 37 | image: "consul" 38 | container_name: "consul" 39 | hostname: "consul" 40 | ports: 41 | - "8400:8400" 42 | - "8500:8500" 43 | - "8600:53" 44 | networks: 45 | - back 46 | 47 | networks: 48 | back: 49 | -------------------------------------------------------------------------------- /examples/compose_etcd2/README.md: -------------------------------------------------------------------------------- 1 | This example shows how to create a static RabbitMQ cluster using: 2 | 3 | 1. [Docker compose](https://docs.docker.com/compose/) 4 | 5 | 2. [etcd2](https://github.com/coreos/etcd) as back-end 6 | 7 | --- 8 | 9 | It creates a cluster with 2 nodes and the UI enabled. 10 | 11 | You can customize the `rabbitmq.config` inside `conf/rabbitmq.config` 12 | 13 | 14 | How to run: 15 | ``` 16 | docker-compose up 17 | ``` 18 | 19 | 20 | --- 21 | 22 | Check running status: 23 | 24 | - RabbitMQ Management: http://localhost:15672/#/ 25 | 26 | 27 | -------------------------------------------------------------------------------- /examples/compose_etcd2/conf/rabbitmq.config: -------------------------------------------------------------------------------- 1 | [ { rabbit, [ 2 | { loopback_users, [ ] } ] }, 3 | 4 | {autocluster, 5 | [ 6 | {startup_delay, 15}, 7 | {backend, etcd}, 8 | {etcd_host, "etcd0"}, 9 | {etcd_port, 2379} 10 | ]} 11 | 12 | ]. 13 | -------------------------------------------------------------------------------- /examples/compose_etcd2/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | services: 3 | etcd0: 4 | image: quay.io/coreos/etcd:v2.1.1 5 | networks: 6 | - back 7 | ports: 8 | - "4001:4001" 9 | - 2380 10 | - 2379 11 | command: 12 | - --name=etcd0 13 | - --initial-cluster-token=etcd-cluster-1 14 | - --initial-cluster=etcd0=http://etcd0:2380 15 | - --initial-cluster-state=new 16 | - --initial-advertise-peer-urls=http://etcd0:2380 17 | - --listen-client-urls=http://0.0.0.0:2379,http://0.0.0.0:4001 18 | - --listen-peer-urls=http://0.0.0.0:2380 19 | - --advertise-client-urls=http://etcd0:2379 20 | rabbit_node_1: 21 | environment: 22 | - RABBITMQ_ERLANG_COOKIE='secret_cookie' 23 | - AUTOCLUSTER_TYPE=etcd 24 | - AUTOCLUSTER_DELAY=10 25 | - ETCD_HOST=etcd0 26 | networks: 27 | - back 28 | depends_on: 29 | - etcd0 30 | hostname: rabbit_node_1 31 | image: pivotalrabbitmq/rabbitmq-autocluster 32 | ports: 33 | - "15672:15672" 34 | - "5672:5672" 35 | tty: true 36 | volumes: 37 | - rabbit1:/var/lib/rabbitmq 38 | - ./conf/:/etc/rabbitmq/ 39 | rabbit_node_2: 40 | environment: 41 | - RABBITMQ_ERLANG_COOKIE='secret_cookie' 42 | - AUTOCLUSTER_TYPE=etcd 43 | - AUTOCLUSTER_DELAY=40 44 | - ETCD_HOST=etcd0 45 | networks: 46 | - back 47 | hostname: rabbit_node_2 48 | image: pivotalrabbitmq/rabbitmq-autocluster 49 | depends_on: 50 | - rabbit_node_1 51 | - etcd0 52 | ports: 53 | - "15673:15672" 54 | - "5673:5672" 55 | tty: true 56 | volumes: 57 | - rabbit2:/var/lib/rabbitmq 58 | - ./conf/:/etc/rabbitmq/ 59 | volumes: 60 | rabbit1: 61 | driver: local 62 | rabbit2: 63 | driver: local 64 | 65 | networks: 66 | back: 67 | -------------------------------------------------------------------------------- /examples/compose_etcd2_haproxy/README.md: -------------------------------------------------------------------------------- 1 | This example shows how to create a dynamic RabbitMQ cluster using: 2 | 3 | 4 | 1. [Docker compose](https://docs.docker.com/compose/) 5 | 6 | 2. [etcd2](https://github.com/coreos/etcd) as back-end 7 | 8 | 3. [HA proxy](https://github.com/docker/dockercloud-haproxy) 9 | 10 | --- 11 | 12 | You can customize the `rabbitmq.config` inside `conf/rabbitmq.config` 13 | 14 | 15 | How to run: 16 | ``` 17 | docker-compose up 18 | ``` 19 | 20 | How to scale: 21 | 22 | ``` 23 | docker-compose scale rabbit=3 24 | ``` 25 | 26 | --- 27 | 28 | Check running status: 29 | 30 | - RabbitMQ Management: http://localhost:15672/#/ 31 | 32 | -------------------------------------------------------------------------------- /examples/compose_etcd2_haproxy/conf/rabbitmq.config: -------------------------------------------------------------------------------- 1 | [ { rabbit, [ 2 | { loopback_users, [ ] } ] }, 3 | 4 | {autocluster, 5 | [ 6 | {cluster_cleanup, true}, 7 | {cleanup_warn_only, false}, 8 | {startup_delay, 15}, 9 | {backend, etcd}, 10 | {etcd_host, "etcd0"}, 11 | {etcd_port, 2379} 12 | ]} 13 | 14 | ]. 15 | -------------------------------------------------------------------------------- /examples/compose_etcd2_haproxy/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | services: 3 | rabbit: 4 | environment: 5 | - RABBITMQ_ERLANG_COOKIE='secret_cookie' 6 | - TCP_PORTS=15672, 5672 7 | - AUTOCLUSTER_TYPE=etcd 8 | - AUTOCLUSTER_DELAY=20 9 | - ETCD_HOST=etcd0 10 | - AUTOCLUSTER_CLEANUP=true 11 | - CLEANUP_WARN_ONLY=false 12 | networks: 13 | - back 14 | image: pivotalrabbitmq/rabbitmq-autocluster 15 | expose: 16 | - 15672 17 | - 5672 18 | - 5671 19 | - 15671 20 | tty: true 21 | volumes: 22 | - ./conf/:/etc/rabbitmq/ 23 | lb: 24 | image: dockercloud/haproxy 25 | environment: 26 | - MODE=tcp 27 | links: 28 | - rabbit 29 | volumes: 30 | - /var/run/docker.sock:/var/run/docker.sock 31 | ports: 32 | - 15672:15672 33 | - 5672:5672 34 | networks: 35 | - back 36 | etcd0: 37 | image: quay.io/coreos/etcd:v2.1.1 38 | networks: 39 | - back 40 | ports: 41 | - "4001:4001" 42 | - 2380 43 | - 2379 44 | command: 45 | - --name=etcd0 46 | - --initial-cluster-token=etcd-cluster-1 47 | - --initial-cluster=etcd0=http://etcd0:2380 48 | - --initial-cluster-state=new 49 | - --initial-advertise-peer-urls=http://etcd0:2380 50 | - --listen-client-urls=http://0.0.0.0:2379,http://0.0.0.0:4001 51 | - --listen-peer-urls=http://0.0.0.0:2380 52 | - --advertise-client-urls=http://etcd0:2379 53 | 54 | volumes: 55 | rabbit1: 56 | driver: local 57 | 58 | networks: 59 | back: 60 | -------------------------------------------------------------------------------- /examples/k8s_minikube/README.md: -------------------------------------------------------------------------------- 1 | 2 | Test RabbitMQ-Autoclsuter on K8s through Minikube 3 | 4 | 1. Install [`kubectl`](https://kubernetes.io/docs/tasks/kubectl/install/): 5 | ``` 6 | # OS X 7 | curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/darwin/amd64/kubectl 8 | 9 | # Linux 10 | curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl 11 | 12 | chmod +x ./kubectl 13 | sudo mv ./kubectl /usr/local/bin/kubectl 14 | ``` 15 | 2.Install [`Minikube`](https://github.com/kubernetes/minikube/releases): 16 | 17 | ``` 18 | OSX 19 | 20 | curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.17.1/minikube-darwin-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ 21 | 22 | Linux 23 | 24 | curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.17.1/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ 25 | ``` 26 | 27 | 3. Start `minikube` virtual machine: 28 | ``` 29 | minikube start --cpus=2 --memory=2040 --vm-driver=virtualbox 30 | ``` 31 | 32 | 4. Create a namespace only for RabbitMQ test: 33 | ``` 34 | kubectl create namespace test-rabbitmq 35 | ``` 36 | 37 | 5. Run the `etcd` image and expose it: 38 | ``` 39 | kubectl run etcd --image=microbox/etcd --port=4001 --namespace=test-rabbitmq -- --name etcd 40 | kubectl --namespace=test-rabbitmq expose deployment etcd 41 | ``` 42 | 43 | 6. Deploy the `YAML` file: 44 | 45 | ``` 46 | kubectl create -f examples/k8s_minikube/rabbitmq.yaml 47 | ``` 48 | 49 | 7. Check the cluster status: 50 | Wait few seconds....then 51 | 52 | ``` 53 | FIRST_POD=$(kubectl get pods --namespace test-rabbitmq -l 'app=rabbitmq' -o jsonpath='{.items[0].metadata.name }') 54 | kubectl exec --namespace=test-rabbitmq $FIRST_POD rabbitmqctl cluster_status 55 | ``` 56 | as result: 57 | ``` 58 | Cluster status of node 'rabbit@172.17.0.9' ... 59 | [{nodes,[{disc,['rabbit@172.17.0.7','rabbit@172.17.0.8', 60 | 'rabbit@172.17.0.9']}]}, 61 | {running_nodes,['rabbit@172.17.0.7','rabbit@172.17.0.8','rabbit@172.17.0.9']}, 62 | {cluster_name,<<"rabbit@rabbitmq-deployment-3409700153-b1bv7">>}, 63 | {partitions,[]}, 64 | {alarms,[{'rabbit@172.17.0.7',[]}, 65 | {'rabbit@172.17.0.8',[]}, 66 | {'rabbit@172.17.0.9',[]}]}] 67 | ``` 68 | 69 | 70 | 8. Expose the cluster using a load-balancer: 71 | 72 | ``` 73 | kubectl expose deployment rabbitmq-deployment --port 15672 --type=LoadBalancer --namespace=test-rabbitmq 74 | 75 | minikube service rabbitmq-deployment --namespace=test-rabbitmq 76 | 77 | ``` 78 | 79 | 9. Enable the K8s dashboard (Optional): 80 | ``` 81 | minikube dashboard 82 | ``` 83 | 84 | 10. Scale up the RabbitMQ cluster 85 | ``` 86 | kubectl scale deployment/rabbitmq-deployment --namespace=test-rabbitmq --replicas=6 87 | ``` 88 | 89 | Excute again: 90 | ``` 91 | kubectl exec --namespace=test-rabbitmq $FIRST_POD rabbitmqctl cluster_status 92 | ``` 93 | you should have 6 nodes: 94 | ``` 95 | Cluster status of node 'rabbit@172.17.0.9' ... 96 | [{nodes,[{disc,['rabbit@172.17.0.10','rabbit@172.17.0.11', 97 | 'rabbit@172.17.0.12','rabbit@172.17.0.7','rabbit@172.17.0.8', 98 | 'rabbit@172.17.0.9']}]}, 99 | {running_nodes,['rabbit@172.17.0.12','rabbit@172.17.0.11', 100 | 'rabbit@172.17.0.10','rabbit@172.17.0.7','rabbit@172.17.0.8', 101 | 'rabbit@172.17.0.9']}, 102 | {cluster_name,<<"rabbit@rabbitmq-deployment-3409700153-1rr4x">>}, 103 | {partitions,[]}, 104 | {alarms,[{'rabbit@172.17.0.12',[]}, 105 | {'rabbit@172.17.0.11',[]}, 106 | {'rabbit@172.17.0.10',[]}, 107 | {'rabbit@172.17.0.7',[]}, 108 | {'rabbit@172.17.0.8',[]}, 109 | {'rabbit@172.17.0.9',[]}]}] 110 | ``` 111 | -------------------------------------------------------------------------------- /examples/k8s_minikube/rabbitmq.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: rabbitmq-deployment 5 | namespace: test-rabbitmq 6 | spec: 7 | replicas: 3 8 | template: 9 | metadata: 10 | labels: 11 | app: rabbitmq 12 | spec: 13 | containers: 14 | - name: rabbitmq-autocluster 15 | image: pivotalrabbitmq/rabbitmq-autocluster 16 | imagePullPolicy: Always 17 | env: 18 | # For consupmption by rabbitmq-env.conf 19 | - name: MY_POD_IP 20 | valueFrom: 21 | fieldRef: 22 | fieldPath: status.podIP 23 | - name: RABBITMQ_USE_LONGNAME 24 | value: "true" 25 | - name: ERLANG_COOKIE 26 | value: "test" 27 | # - name: RABBITMQ_ERLANG_COOKIE 28 | # value: "secret_cookie" 29 | - name: RABBITMQ_NODENAME 30 | value: "rabbit@$(MY_POD_IP)" 31 | - name: AUTOCLUSTER_TYPE 32 | value: "etcd" 33 | - name: AUTOCLUSTER_DELAY 34 | value: "60" 35 | - name: ETCD_HOST 36 | value: "etcd" 37 | - name: AUTOCLUSTER_CLEANUP 38 | value: "true" 39 | - name: CLEANUP_WARN_ONLY 40 | value: "false" 41 | -------------------------------------------------------------------------------- /examples/k8s_rbac_statefulsets/README.md: -------------------------------------------------------------------------------- 1 | RabbitMQ-Autocluster on K8s [StatefulSet Controller](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) 2 | 3 | 1. Install [`kubectl`](https://kubernetes.io/docs/tasks/tools/install-kubectl/) 4 | 5 | 2. Install [`minikube`](https://kubernetes.io/docs/tasks/tools/install-minikube/) 6 | 7 | 3. Start `minikube` virtual machine: 8 | ``` 9 | $ minikube start --cpus=2 --memory=2040 --vm-driver=virtualbox 10 | ``` 11 | 12 | 4. Create a namespace only for RabbitMQ test: 13 | ``` 14 | $ kubectl create namespace test-rabbitmq 15 | ``` 16 | 17 | 5. As from Kubernetes 1.6 onwards, RBAC policies are enabled by default, it need deploy the RBAC `YAML` file, regards how to check whether RBAC is enabled or not, pls run `kubectl cluster-info dump | grep authorization-mode` 18 | ``` 19 | $ kubectl create -f examples/k8s_statefulsets/rabbitmq-rbac.yaml 20 | ``` 21 | 22 | 6. Deploy the service `YAML` file: 23 | ``` 24 | $ kubectl create -f examples/k8s_statefulsets/rabbitmq-service.yaml 25 | ``` 26 | 27 | 7. Deploy the RabbitMQ StatefulSet `YAML` file: 28 | ``` 29 | $ kubectl create -f examples/k8s_statefulsets/rabbitmq.yaml 30 | ``` 31 | 32 | 8. Check the cluster status: 33 | 34 | Wait few seconds....then 35 | 36 | ``` 37 | $ FIRST_POD=$(kubectl get pods --namespace test-rabbitmq -l 'app=rabbitmq' -o jsonpath='{.items[0].metadata.name }') 38 | kubectl exec --namespace=test-rabbitmq $FIRST_POD rabbitmqctl cluster_status 39 | ``` 40 | as result you should have: 41 | ``` 42 | Cluster status of node 'rabbit@172.17.0.2' 43 | [{nodes,[{disc,['rabbit@172.17.0.2','rabbit@172.17.0.4', 44 | 'rabbit@172.17.0.5']}]}, 45 | {running_nodes,['rabbit@172.17.0.5','rabbit@172.17.0.4','rabbit@172.17.0.2']}, 46 | {cluster_name,<<"rabbit@rabbitmq-0.rabbitmq.test-rabbitmq.svc.cluster.local">>}, 47 | {partitions,[]}, 48 | {alarms,[{'rabbit@172.17.0.5',[]}, 49 | {'rabbit@172.17.0.4',[]}, 50 | {'rabbit@172.17.0.2',[]}]}] 51 | ``` 52 | 53 | 9. Get your `minikube` ip: 54 | ``` 55 | $ minikube ip 56 | 192.168.99.104 57 | ``` 58 | 59 | 10. Ports: 60 | * `http://<>:31672` - Management UI 61 | * `amqp://guest:guest@<>:30672` - AMQP 62 | 63 | 11. Scaling: 64 | ``` 65 | $ kubectl scale statefulset/rabbitmq --namespace=test-rabbitmq --replicas=5 66 | ``` 67 | 68 | -------------------------------------------------------------------------------- /examples/k8s_rbac_statefulsets/rabbitmq-rbac.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: rabbitmq 6 | namespace: test-rabbitmq 7 | --- 8 | kind: Role 9 | apiVersion: rbac.authorization.k8s.io/v1beta1 10 | metadata: 11 | name: rabbitmq 12 | namespace: test-rabbitmq 13 | rules: 14 | - apiGroups: [""] 15 | resources: ["endpoints"] 16 | verbs: ["get"] 17 | --- 18 | kind: RoleBinding 19 | apiVersion: rbac.authorization.k8s.io/v1beta1 20 | metadata: 21 | name: rabbitmq 22 | namespace: test-rabbitmq 23 | roleRef: 24 | apiGroup: rbac.authorization.k8s.io 25 | kind: Role 26 | name: rabbitmq 27 | subjects: 28 | - kind: ServiceAccount 29 | name: rabbitmq 30 | 31 | -------------------------------------------------------------------------------- /examples/k8s_rbac_statefulsets/rabbitmq-service.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | namespace: test-rabbitmq 5 | name: rabbitmq 6 | labels: 7 | app: rabbitmq 8 | type: LoadBalancer 9 | spec: 10 | type: NodePort 11 | ports: 12 | - name: http 13 | protocol: TCP 14 | port: 15672 15 | targetPort: 15672 16 | nodePort: 31672 17 | - name: amqp 18 | protocol: TCP 19 | port: 5672 20 | targetPort: 5672 21 | nodePort: 30672 22 | selector: 23 | app: rabbitmq 24 | -------------------------------------------------------------------------------- /examples/k8s_rbac_statefulsets/rabbitmq.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1beta1 2 | kind: StatefulSet 3 | metadata: 4 | name: rabbitmq 5 | namespace: test-rabbitmq 6 | spec: 7 | serviceName: rabbitmq 8 | replicas: 3 9 | template: 10 | metadata: 11 | labels: 12 | app: rabbitmq 13 | spec: 14 | serviceAccountName: rabbitmq 15 | terminationGracePeriodSeconds: 10 16 | containers: 17 | - name: rabbitmq-autocluster 18 | image: pivotalrabbitmq/rabbitmq-autocluster 19 | ports: 20 | - name: http 21 | protocol: TCP 22 | containerPort: 15672 23 | - name: amqp 24 | protocol: TCP 25 | containerPort: 5672 26 | livenessProbe: 27 | exec: 28 | command: ["rabbitmqctl", "status"] 29 | initialDelaySeconds: 30 30 | timeoutSeconds: 5 31 | readinessProbe: 32 | exec: 33 | command: ["rabbitmqctl", "status"] 34 | initialDelaySeconds: 10 35 | timeoutSeconds: 5 36 | imagePullPolicy: Always 37 | env: 38 | - name: MY_POD_IP 39 | valueFrom: 40 | fieldRef: 41 | fieldPath: status.podIP 42 | - name: RABBITMQ_USE_LONGNAME 43 | value: "true" 44 | - name: RABBITMQ_NODENAME 45 | value: "rabbit@$(MY_POD_IP)" 46 | - name: AUTOCLUSTER_TYPE 47 | value: "k8s" 48 | - name: AUTOCLUSTER_DELAY 49 | value: "10" 50 | - name: K8S_ADDRESS_TYPE 51 | value: "ip" 52 | - name: AUTOCLUSTER_CLEANUP 53 | value: "true" 54 | - name: CLEANUP_WARN_ONLY 55 | value: "false" 56 | -------------------------------------------------------------------------------- /examples/k8s_statefulsets/README.md: -------------------------------------------------------------------------------- 1 | RabbitMQ-Autocluster on K8s [StatefulSet Controller](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) 2 | 3 | 1. Install [`kubectl`](https://kubernetes.io/docs/tasks/tools/install-kubectl/) 4 | 5 | 6 | 2. Install [`minikube`](https://kubernetes.io/docs/tasks/tools/install-minikube/) 7 | 8 | 9 | 3. Start `minikube` virtual machine: 10 | ``` 11 | $ minikube start --cpus=2 --memory=2040 --vm-driver=virtualbox 12 | ``` 13 | 14 | 4. Create a namespace only for RabbitMQ test: 15 | ``` 16 | $ kubectl create namespace test-rabbitmq 17 | ``` 18 | 19 | 5. Deploy the service `YAML` file: 20 | 21 | ``` 22 | $ kubectl create -f examples/k8s_statefulsets/rabbitmq-service.yaml 23 | ``` 24 | 6. Deploy the RabbitMQ StatefulSet `YAML` file: 25 | 26 | ``` 27 | $ kubectl create -f examples/k8s_statefulsets/rabbitmq.yaml 28 | ``` 29 | 7. Check the cluster status: 30 | 31 | Wait few seconds....then 32 | 33 | ``` 34 | $ FIRST_POD=$(kubectl get pods --namespace test-rabbitmq -l 'app=rabbitmq' -o jsonpath='{.items[0].metadata.name }') 35 | kubectl exec --namespace=test-rabbitmq $FIRST_POD rabbitmqctl cluster_status 36 | ``` 37 | as result you should have: 38 | ``` 39 | Cluster status of node 'rabbit@172.17.0.2' 40 | [{nodes,[{disc,['rabbit@172.17.0.2','rabbit@172.17.0.4', 41 | 'rabbit@172.17.0.5']}]}, 42 | {running_nodes,['rabbit@172.17.0.5','rabbit@172.17.0.4','rabbit@172.17.0.2']}, 43 | {cluster_name,<<"rabbit@rabbitmq-0.rabbitmq.test-rabbitmq.svc.cluster.local">>}, 44 | {partitions,[]}, 45 | {alarms,[{'rabbit@172.17.0.5',[]}, 46 | {'rabbit@172.17.0.4',[]}, 47 | {'rabbit@172.17.0.2',[]}]}] 48 | ``` 49 | 50 | 8. Get your `minikube` ip: 51 | ``` 52 | $ minikube ip 53 | 192.168.99.104 54 | ``` 55 | 9. Ports: 56 | * `http://<>:31672` - Management UI 57 | * `amqp://guest:guest@<>:30672` - AMQP 58 | 59 | 10. Scaling: 60 | ``` 61 | $ kubectl scale statefulset/rabbitmq --namespace=test-rabbitmq --replicas=5 62 | ``` 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /examples/k8s_statefulsets/rabbitmq-service.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | namespace: test-rabbitmq 5 | name: rabbitmq 6 | labels: 7 | app: rabbitmq 8 | type: LoadBalancer 9 | spec: 10 | type: NodePort 11 | ports: 12 | - name: http 13 | protocol: TCP 14 | port: 15672 15 | targetPort: 15672 16 | nodePort: 31672 17 | - name: amqp 18 | protocol: TCP 19 | port: 5672 20 | targetPort: 5672 21 | nodePort: 30672 22 | selector: 23 | app: rabbitmq 24 | -------------------------------------------------------------------------------- /examples/k8s_statefulsets/rabbitmq.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1beta1 2 | kind: StatefulSet 3 | metadata: 4 | name: rabbitmq 5 | namespace: test-rabbitmq 6 | spec: 7 | serviceName: rabbitmq 8 | replicas: 3 9 | template: 10 | metadata: 11 | labels: 12 | app: rabbitmq 13 | spec: 14 | terminationGracePeriodSeconds: 10 15 | containers: 16 | - name: rabbitmq-autocluster 17 | image: pivotalrabbitmq/rabbitmq-autocluster 18 | ports: 19 | - name: http 20 | protocol: TCP 21 | containerPort: 15672 22 | - name: amqp 23 | protocol: TCP 24 | containerPort: 5672 25 | livenessProbe: 26 | exec: 27 | command: ["rabbitmqctl", "status"] 28 | initialDelaySeconds: 30 29 | timeoutSeconds: 5 30 | readinessProbe: 31 | exec: 32 | command: ["rabbitmqctl", "status"] 33 | initialDelaySeconds: 10 34 | timeoutSeconds: 5 35 | imagePullPolicy: Always 36 | env: 37 | - name: MY_POD_IP 38 | valueFrom: 39 | fieldRef: 40 | fieldPath: status.podIP 41 | - name: RABBITMQ_USE_LONGNAME 42 | value: "true" 43 | - name: RABBITMQ_NODENAME 44 | value: "rabbit@$(MY_POD_IP)" 45 | - name: AUTOCLUSTER_TYPE 46 | value: "k8s" 47 | - name: AUTOCLUSTER_DELAY 48 | value: "10" 49 | - name: K8S_ADDRESS_TYPE 50 | value: "ip" 51 | - name: AUTOCLUSTER_CLEANUP 52 | value: "true" 53 | - name: CLEANUP_WARN_ONLY 54 | value: "false" 55 | -------------------------------------------------------------------------------- /include/autocluster.hrl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% @author Gavin M. Roy 3 | %% @copyright 2015-2016 AWeber Communications 4 | %% @end 5 | %%============================================================================== 6 | 7 | -record(config, {key, os, default, type, is_port}). 8 | 9 | %% Config Record key environment variable default type is port 10 | -define(CONFIG_MAP, 11 | [{config, backend, "AUTOCLUSTER_TYPE", unconfigured, atom, false}, %% General 12 | {config, autocluster_failure, "AUTOCLUSTER_FAILURE", ignore, atom, false}, 13 | {config, startup_delay, "AUTOCLUSTER_DELAY", 5, integer, false}, 14 | {config, lock_wait_time, "LOCK_WAIT_TIME", 300, integer, false}, 15 | {config, cluster_cleanup, "AUTOCLUSTER_CLEANUP", false, atom, false}, 16 | {config, autocluster_log_level, "AUTOCLUSTER_LOG_LEVEL", info, atom, false}, 17 | {config, cleanup_interval, "CLEANUP_INTERVAL", 60, integer, false}, 18 | {config, cleanup_warn_only, "CLEANUP_WARN_ONLY", true, atom, false}, 19 | {config, longname, "RABBITMQ_USE_LONGNAME", false, atom, false}, 20 | {config, node_name, "RABBITMQ_NODENAME", "rabbit", string, false}, 21 | {config, node_type, "RABBITMQ_NODE_TYPE", disc, atom, false}, 22 | {config, http_proxy, "HTTP_PROXY", "undefined", string, false}, 23 | {config, https_proxy, "HTTPS_PROXY", "undefined", string, false}, 24 | {config, proxy_exclusions, "PROXY_EXCLUSIONS", [], list, false}, 25 | 26 | {config, aws_autoscaling, "AWS_AUTOSCALING", false, atom, false}, %% AWS 27 | {config, aws_ec2_tags, "AWS_EC2_TAGS", [], proplist, false}, 28 | {config, aws_access_key, "AWS_ACCESS_KEY_ID", "undefined", string, false}, 29 | {config, aws_secret_key, "AWS_SECRET_ACCESS_KEY", "undefined", string, false}, 30 | {config, aws_ec2_region, "AWS_DEFAULT_REGION", "undefined", string, false}, 31 | {config, aws_use_private_ip, "AWS_USE_PRIVATE_IP", false, atom, false}, 32 | 33 | {config, cluster_name, "CLUSTER_NAME", "undefined", string, false}, %% Consul && etcd 34 | 35 | {config, consul_acl_token, "CONSUL_ACL_TOKEN", "undefined", string, false}, %% Consul 36 | {config, consul_include_nodes_with_warnings, "CONSUL_INCLUDE_NODES_WITH_WARNINGS", false, atom, false}, 37 | {config, consul_scheme, "CONSUL_SCHEME", "http", string, false}, 38 | {config, consul_host, "CONSUL_HOST", "localhost", string, false}, 39 | {config, consul_port, "CONSUL_PORT", 8500, integer, true}, 40 | {config, consul_lock_prefix, "CONSUL_LOCK_PREFIX", "rabbitmq", string, false}, 41 | {config, consul_domain, "CONSUL_DOMAIN", "consul", string, false}, 42 | {config, consul_svc, "CONSUL_SVC", "rabbitmq", string, false}, 43 | {config, consul_svc_addr, "CONSUL_SVC_ADDR", "undefined", string, false}, 44 | {config, consul_svc_addr_auto, "CONSUL_SVC_ADDR_AUTO", false, atom, false}, 45 | {config, consul_svc_addr_nic, "CONSUL_SVC_ADDR_NIC", "undefined", string, false}, 46 | {config, consul_svc_addr_nodename, "CONSUL_SVC_ADDR_NODENAME", false, atom, false}, 47 | {config, consul_svc_port, "CONSUL_SVC_PORT", 5672, integer, true}, 48 | {config, consul_svc_ttl, "CONSUL_SVC_TTL", 30, integer, false}, 49 | {config, consul_svc_tags, "CONSUL_SVC_TAGS", [], list, false}, 50 | {config, consul_deregister_after, "CONSUL_DEREGISTER_AFTER", "", integer, false}, %% consul deregister_critical_service_after 51 | {config, consul_use_longname, "CONSUL_USE_LONGNAME", false, atom, false}, 52 | 53 | {config, autocluster_host, "AUTOCLUSTER_HOST", "undefined", string, false}, %% DNS 54 | 55 | {config, k8s_scheme, "K8S_SCHEME", "https", string, false}, %% kubernetes 56 | {config, k8s_host, "K8S_HOST", "kubernetes.default.svc.cluster.local", 57 | string, false}, 58 | {config, k8s_port, "K8S_PORT", 443, integer, true}, 59 | {config, k8s_token_path, "K8S_TOKEN_PATH", "/var/run/secrets/kubernetes.io/serviceaccount/token", 60 | string, false}, 61 | {config, k8s_cert_path, "K8S_CERT_PATH", "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", 62 | string, false}, 63 | {config, k8s_namespace_path, "K8S_NAMESPACE_PATH", "/var/run/secrets/kubernetes.io/serviceaccount/namespace", 64 | string, false}, 65 | {config, k8s_service_name, "K8S_SERVICE_NAME", "rabbitmq", string, false}, 66 | {config, k8s_address_type, "K8S_ADDRESS_TYPE", "ip", string, false}, 67 | {config, k8s_hostname_suffix, "K8S_HOSTNAME_SUFFIX", "", string, false}, 68 | 69 | {config, etcd_scheme, "ETCD_SCHEME", "http", string, false}, %% etcd 70 | {config, etcd_host, "ETCD_HOST", "localhost", string, false}, 71 | {config, etcd_port, "ETCD_PORT", 2379, integer, true}, 72 | {config, etcd_prefix, "ETCD_PREFIX", "rabbitmq", string, false}, 73 | {config, etcd_node_ttl, "ETCD_NODE_TTL", 30, integer, false}]). 74 | 75 | -define(CONSUL_CHECK_NOTES, "rabbitmq-autocluster node check"). 76 | 77 | %%-------------------------------------------------------------------- 78 | %% @doc 79 | %% State that is passed around and updated by 80 | %% initialization steps. 81 | %% @end 82 | %%-------------------------------------------------------------------- 83 | -record(startup_state, {backend_name :: atom() 84 | , backend_module :: module() 85 | , best_node_to_join :: undefined | node() 86 | , startup_lock_data = undefined 87 | }). 88 | 89 | %%-------------------------------------------------------------------- 90 | %% @doc 91 | %% Extended node information used for choosing the preferred cluster member 92 | %% to join. 93 | %% @end 94 | %%-------------------------------------------------------------------- 95 | -record(candidate_seed_node, {name :: node() 96 | , uptime :: non_neg_integer() 97 | , alive :: boolean() 98 | , clustered_with :: [node()] 99 | , alive_cluster_nodes :: [node()] 100 | , partitioned_cluster_nodes :: [node()] 101 | , other_cluster_nodes :: [node()] 102 | }). 103 | -------------------------------------------------------------------------------- /rabbitmq-autocluster.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /root/launch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # add debug 6 | [[ -n "$DEBUG" ]] && set -x 7 | 8 | # allow the container to be started with `--user` 9 | if [ "$1" = 'rabbitmq-server' -a "$(id -u)" = '0' ]; then 10 | chown -R rabbitmq:rabbitmq ${HOME} 11 | exec su-exec rabbitmq "$0" "$@" 12 | fi 13 | 14 | exec "$@" 15 | -------------------------------------------------------------------------------- /root/usr/lib/rabbitmq/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rabbitmq/rabbitmq-autocluster/e9aef34aa7a50b78f8230db9d7be712bedb630b2/root/usr/lib/rabbitmq/.gitkeep -------------------------------------------------------------------------------- /root/usr/lib/rabbitmq/etc/rabbitmq/rabbitmq.config: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | rabbit, 4 | [ 5 | {loopback_users, []}, 6 | {cluster_partition_handling, autoheal}, 7 | {delegate_count, 64}, 8 | {fhc_read_buffering, false}, 9 | {fhc_write_buffering, false}, 10 | {heartbeat, 60}, 11 | {queue_index_embed_msgs_below, 0}, 12 | {queue_index_max_journal_entries, 8192}, 13 | {queue_master_locator, <<"min-masters">>}, 14 | {log_levels, [{autocluster, info}, 15 | {connection, error}, 16 | {channel, warning}, 17 | {federation, info}, 18 | {mirroring, info}]}, 19 | {vm_memory_high_watermark, 0.8} 20 | ] 21 | } 22 | ]. 23 | -------------------------------------------------------------------------------- /root/var/lib/rabbitmq/.erlang.cookie: -------------------------------------------------------------------------------- 1 | ALWEDHDBZTQYWTJGTXWV 2 | -------------------------------------------------------------------------------- /src/autocluster_app.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% @author Gavin M. Roy 3 | %% @copyright 2016 AWeber Communications 4 | %% @end 5 | %%============================================================================== 6 | -module(autocluster_app). 7 | 8 | -behaviour(application). 9 | 10 | %% Application callbacks 11 | -export([start/2, stop/1]). 12 | 13 | %%%=================================================================== 14 | %%% Application callbacks 15 | %%%=================================================================== 16 | 17 | %%-------------------------------------------------------------------- 18 | %% @private 19 | %% @doc 20 | %% This function is called whenever an application is started using 21 | %% application:start/[1,2], and should start the processes of the 22 | %% application. If the application is structured according to the OTP 23 | %% design principles as a supervision tree, this means starting the 24 | %% top supervisor of the tree. 25 | %% 26 | %% @end 27 | %%-------------------------------------------------------------------- 28 | -spec(start(StartType :: normal | {takeover, node()} | {failover, node()}, 29 | StartArgs :: term()) -> 30 | {ok, pid()} | 31 | {ok, pid(), State :: term()} | 32 | {error, Reason :: term()}). 33 | start(_StartType, _StartArgs) -> 34 | autocluster_sup:start_link(). 35 | 36 | %%-------------------------------------------------------------------- 37 | %% @private 38 | %% @doc 39 | %% This function is called whenever an application has stopped. It 40 | %% is intended to be the opposite of Module:start/2 and should do 41 | %% any necessary cleaning up. The return value is ignored. 42 | %% 43 | %% @end 44 | %%-------------------------------------------------------------------- 45 | -spec(stop(State :: term()) -> term()). 46 | stop(_State) -> 47 | ok. 48 | -------------------------------------------------------------------------------- /src/autocluster_aws.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% @author Gavin M. Roy 3 | %% @copyright 2016 AWeber Communications 4 | %% @end 5 | %%============================================================================== 6 | -module(autocluster_aws). 7 | 8 | -behavior(autocluster_backend). 9 | 10 | -export([nodelist/0, 11 | lock/1, 12 | unlock/1, 13 | register/0, 14 | unregister/0]). 15 | 16 | %% Export all for unit tests 17 | -ifdef(TEST). 18 | -compile(export_all). 19 | -endif. 20 | 21 | -include_lib("rabbitmq_aws/include/rabbitmq_aws.hrl"). 22 | 23 | -define(INSTANCE_ID_URL, 24 | "http://169.254.169.254/latest/meta-data/instance-id"). 25 | 26 | -type tags() :: [{string(), string()}]. 27 | -type filters() :: [{string(), string()}]. 28 | 29 | -spec nodelist() -> {ok, [node()]}|{error, Reason :: string()}. 30 | %% @doc Return the nodelist from the AWS API 31 | %% @end 32 | %% 33 | nodelist() -> 34 | {ok, _} = application:ensure_all_started(rabbitmq_aws), 35 | ok = maybe_set_region(autocluster_config:get(aws_ec2_region)), 36 | ok = maybe_set_credentials(autocluster_config:get(aws_access_key), 37 | autocluster_config:get(aws_secret_key)), 38 | case autocluster_config:get(aws_autoscaling) of 39 | true -> 40 | get_autoscaling_group_node_list(instance_id(), get_tags()); 41 | false -> 42 | get_node_list_from_tags(get_tags()) 43 | end. 44 | 45 | -spec lock(term()) -> not_supported. 46 | lock(_) -> 47 | not_supported. 48 | 49 | -spec unlock(term()) -> ok | {error, string()}. 50 | unlock(_) -> 51 | ok. 52 | 53 | -spec register() -> ok | {error, string()}. 54 | %% @doc This is not required for the AWS backend. 55 | %% @end 56 | %% 57 | register() -> 58 | ok. 59 | 60 | -spec unregister() -> ok | {error, string()}. 61 | %% @doc This is not required for the AWS backend. 62 | %% @end 63 | %% 64 | unregister() -> 65 | ok. 66 | 67 | -spec api_get_request(string(), string()) 68 | -> {ok, list()} | {error, Reason :: string()}. 69 | %% @private 70 | %% @doc Make a GET request to the AWS API 71 | %% @end 72 | %% 73 | api_get_request(Service, Path) -> 74 | case rabbitmq_aws:get(Service, Path) of 75 | {ok, {_Headers, Payload}} -> 76 | autocluster_log:debug("AWS request: ~s~nResponse: ~p~n", 77 | [Path, Payload]), 78 | {ok, Payload}; 79 | {error, {credentials, _}} -> {error, credentials}; 80 | {error, Message, _} -> {error, Message} 81 | end. 82 | 83 | -spec build_instance_list_qargs(Instances :: list(), Accum :: list()) -> list(). 84 | %% @private 85 | %% @doc Build the Query args for filtering instances by InstanceID. 86 | %% @end 87 | %% 88 | build_instance_list_qargs([], Accum) -> Accum; 89 | build_instance_list_qargs([H|T], Accum) -> 90 | Key = "InstanceId." ++ integer_to_list(length(Accum) + 1), 91 | build_instance_list_qargs(T, lists:append([{Key, H}], Accum)). 92 | 93 | 94 | -spec find_autoscaling_group(Instances :: list(), Instance :: string()) 95 | -> string() | error. 96 | %% @private 97 | %% @doc Attempt to find the Auto Scaling Group ID by finding the current 98 | %% instance in the list of instances returned by the autoscaling API 99 | %% endpoint. 100 | %% @end 101 | %% 102 | find_autoscaling_group([], _) -> error; 103 | find_autoscaling_group([H|T], Instance) -> 104 | case proplists:get_value("InstanceId", H) == Instance of 105 | true -> 106 | {ok, proplists:get_value("AutoScalingGroupName", H)}; 107 | false -> 108 | find_autoscaling_group(T, Instance) 109 | end. 110 | 111 | 112 | flatten_autoscaling_datastructure(Value) -> 113 | Response = proplists:get_value("DescribeAutoScalingInstancesResponse", Value), 114 | Result = proplists:get_value("DescribeAutoScalingInstancesResult", Response), 115 | Instances = proplists:get_value("AutoScalingInstances", Result), 116 | [Instance || {_, Instance} <- Instances]. 117 | 118 | get_next_token(Value) -> 119 | Response = proplists:get_value("DescribeAutoScalingInstancesResponse", Value), 120 | Result = proplists:get_value("DescribeAutoScalingInstancesResult", Response), 121 | NextToken = proplists:get_value("NextToken", Result), 122 | NextToken. 123 | 124 | get_all_autoscaling_instances(Accum) -> 125 | QArgs = [{"Action", "DescribeAutoScalingInstances"}, {"Version", "2011-01-01"}], 126 | fetch_all_autoscaling_instances(QArgs, Accum). 127 | 128 | get_all_autoscaling_instances(Accum, 'undefined') -> {ok, Accum}; 129 | get_all_autoscaling_instances(Accum, NextToken) -> 130 | QArgs = [{"Action", "DescribeAutoScalingInstances"}, {"Version", "2011-01-01"}, {"NextToken", NextToken}], 131 | fetch_all_autoscaling_instances(QArgs, Accum). 132 | 133 | fetch_all_autoscaling_instances(QArgs, Accum) -> 134 | Path = "/?" ++ rabbitmq_aws_urilib:build_query_string(QArgs), 135 | 136 | case api_get_request("autoscaling", Path) of 137 | {ok, Payload} -> 138 | Instances = flatten_autoscaling_datastructure(Payload), 139 | NextToken = get_next_token(Payload), 140 | get_all_autoscaling_instances(lists:append(Instances, Accum), NextToken); 141 | {error, Reason} = Error -> 142 | autocluster_log:error("Error fetching autoscaling group instance list: ~p", [Reason]), 143 | Error 144 | end. 145 | 146 | get_autoscaling_group_node_list(error, _) -> {error, instance_discovery}; 147 | get_autoscaling_group_node_list(Instance, Tag) -> 148 | case get_all_autoscaling_instances([]) of 149 | {ok, Instances} -> 150 | case find_autoscaling_group(Instances, Instance) of 151 | {ok, Group} -> 152 | autocluster_log:debug("Fetching autoscaling = Group: ~p", [Group]), 153 | Values = get_autoscaling_instances(Instances, Group, []), 154 | autocluster_log:debug("Fetching autoscaling = Instances: ~p", [Values]), 155 | Names = get_hostname_by_instance_ids(Values, Tag), 156 | autocluster_log:debug("Fetching autoscaling = DNS: ~p", [Names]), 157 | {ok, [autocluster_util:node_name(N) || N <- Names]}; 158 | error -> {error, autoscaling_group_not_found} 159 | end; 160 | error -> 161 | {error, describe_autoscaling_instances} 162 | end. 163 | 164 | 165 | get_autoscaling_instances([], _, Accum) -> Accum; 166 | get_autoscaling_instances([H|T], Group, Accum) -> 167 | GroupName = proplists:get_value("AutoScalingGroupName", H), 168 | case GroupName == Group of 169 | true -> 170 | Node = proplists:get_value("InstanceId", H), 171 | get_autoscaling_instances(T, Group, lists:append([Node], Accum)); 172 | false -> 173 | get_autoscaling_instances(T, Group, Accum) 174 | end. 175 | 176 | -spec get_node_list_from_tags(tags()) -> {error, atom()} | {ok, [node()]}. 177 | get_node_list_from_tags([]) -> 178 | {error, no_configured_tags}; 179 | get_node_list_from_tags(Tags) -> 180 | {ok, [autocluster_util:node_name(N) || N <- get_hostname_by_tags(Tags)]}. 181 | 182 | 183 | get_hostname_by_instance_ids(Instances, Tag) -> 184 | QArgs = build_instance_list_qargs(Instances, 185 | [{"Action", "DescribeInstances"}, 186 | {"Version", "2015-10-01"}]), 187 | QArgs2 = lists:keysort(1, maybe_add_tag_filters(Tag, QArgs, 1)), 188 | Path = "/?" ++ rabbitmq_aws_urilib:build_query_string(QArgs2), 189 | get_hostname_names(Path). 190 | 191 | 192 | get_hostname_by_tags(Tags) -> 193 | QArgs = [{"Action", "DescribeInstances"}, {"Version", "2015-10-01"}], 194 | QArgs2 = lists:keysort(1, maybe_add_tag_filters(Tags, QArgs, 1)), 195 | Path = "/?" ++ rabbitmq_aws_urilib:build_query_string(QArgs2), 196 | get_hostname_names(Path). 197 | 198 | 199 | get_hostname_name_from_reservation_set([], Accum) -> Accum; 200 | get_hostname_name_from_reservation_set([{"item", RI}|T], Accum) -> 201 | InstancesSet = proplists:get_value("instancesSet", RI), 202 | Item = proplists:get_value("item", InstancesSet), 203 | DNSName = proplists:get_value(select_hostname(), Item), 204 | if 205 | DNSName == [] -> get_hostname_name_from_reservation_set(T, Accum); 206 | true -> get_hostname_name_from_reservation_set(T, lists:append([DNSName], Accum)) 207 | end. 208 | 209 | get_hostname_names(Path) -> 210 | case api_get_request("ec2", Path) of 211 | {ok, Payload} -> 212 | Response = proplists:get_value("DescribeInstancesResponse", Payload), 213 | ReservationSet = proplists:get_value("reservationSet", Response), 214 | get_hostname_name_from_reservation_set(ReservationSet, []); 215 | {error, Reason} -> 216 | autocluster_log:error("Error fetching node list: ~p", [Reason]), 217 | error 218 | end. 219 | 220 | -spec get_tags() -> tags(). 221 | get_tags() -> 222 | Tags = autocluster_config:get(aws_ec2_tags), 223 | if 224 | Tags == "unused" -> [{"ignore", "me"}]; %% this is to trick dialyzer 225 | true -> Tags 226 | end. 227 | 228 | -spec select_hostname() -> string(). 229 | select_hostname() -> 230 | case autocluster_config:get(aws_use_private_ip) of 231 | true -> "privateIpAddress"; 232 | false -> "privateDnsName"; 233 | _ -> "privateDnsName" 234 | end. 235 | 236 | -spec instance_id() -> string() | error. 237 | %% @private 238 | %% @doc Return the local instance ID from the EC2 metadata service 239 | %% @end 240 | %% 241 | instance_id() -> 242 | case httpc:request(?INSTANCE_ID_URL) of 243 | {ok, {{_, 200, _}, _, Value}} -> Value; 244 | _ -> error 245 | end. 246 | 247 | -spec maybe_add_tag_filters(tags(), filters(), integer()) -> filters(). 248 | maybe_add_tag_filters([], QArgs, _) -> QArgs; 249 | maybe_add_tag_filters([{Key, Value}|T], QArgs, Num) -> 250 | maybe_add_tag_filters(T, lists:append([{"Filter." ++ integer_to_list(Num) ++ ".Name", "tag:" ++ Key}, 251 | {"Filter." ++ integer_to_list(Num) ++ ".Value.1", Value}], QArgs), Num+1). 252 | 253 | 254 | -spec maybe_set_credentials(AccessKey :: string(), 255 | SecretKey :: string()) -> ok. 256 | %% @private 257 | %% @doc Set the API credentials if they are set in configuration. 258 | %% @end 259 | %% 260 | maybe_set_credentials("undefined", _) -> ok; 261 | maybe_set_credentials(_, "undefined") -> ok; 262 | maybe_set_credentials(AccessKey, SecretKey) -> 263 | rabbitmq_aws:set_credentials(AccessKey, SecretKey). 264 | 265 | 266 | -spec maybe_set_region(Region :: string()) -> ok. 267 | %% @private 268 | %% @doc Set the region from the configuration value, if it was set. 269 | %% @end 270 | %% 271 | maybe_set_region("undefined") -> ok; 272 | maybe_set_region(Value) -> 273 | autocluster_log:debug("Setting region: ~p", [Value]), 274 | rabbitmq_aws:set_region(Value). 275 | -------------------------------------------------------------------------------- /src/autocluster_backend.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% @author Gavin M. Roy 3 | %% @copyright 2015-2016 AWeber Communications 4 | %% @end 5 | %%============================================================================== 6 | -module(autocluster_backend). 7 | 8 | -type lock_result() :: ok | {ok, LockData :: term()} | not_supported | {error, Reason :: string()}. 9 | -type either_ok_or_error() :: ok|{error, Reason :: string()}. 10 | -type either_value_or_error(Value) :: {ok, Value} | {error, Reason :: string()}. 11 | 12 | -export_type([lock_result/0, either_ok_or_error/0, either_value_or_error/1]). 13 | 14 | %% @doc 15 | %% Tries to acquire a lock in some external backend. 16 | %% The only argument is some piece of data that may be stored inside 17 | %% the lock to help with lock holder identification (if backend 18 | %% supports it). 19 | %% 20 | %% Time to acquire lock is limited by configuration setting 21 | %% LOCK_WAIT_TIME. 22 | %% 23 | %% Backends not supporting locking mechanism should return 24 | %% `not_supported` atom, in that case autocluster will fallback to 25 | %% random startup delay behaviour as a poor man substitute for 26 | %% locking. 27 | %% 28 | %% Can return some piece of data that should be passed to unlock 29 | %% (i.e. if some sort of compare-and-set operation is used in lock 30 | %% implementation). 31 | %% 32 | %% If there is a need to perform some periodic activity while the lock 33 | %% is held (i.e. update the lock TTL in backend), one of the functions 34 | %% from autocluster_periodic should be used. 35 | %% 36 | %% Idea is that single lock protects every important operation during startup: 37 | %% - Fetching nodelist from a backend 38 | %% - Joining a node to a cluster 39 | %% - Registering node in a backend 40 | %% This removes a lot of race conditions. 41 | %% 42 | %% @end 43 | -callback lock(string()) -> lock_result(). 44 | 45 | %% @doc 46 | %% 47 | %% Releases lock. The only argument is the resulting data from lock/1 48 | %% call. Should try to return an error when lock was unexpectedly 49 | %% stolen from us. 50 | %% 51 | %% @end 52 | -callback unlock(LockData :: term()) -> either_ok_or_error(). 53 | 54 | %% @doc 55 | %% 56 | %% Returns a list of nodes that were registered in a backend by 57 | %% the register/0. 58 | %% 59 | %% @end 60 | -callback nodelist() -> either_value_or_error([node()]). 61 | 62 | %% @doc 63 | %% 64 | %% Registers current node in a backend. 65 | %% 66 | %% Can be no-op for some backends. 67 | %% 68 | %% If there is a need to perform some periodic activity while the node 69 | %% is alive (i.e. update some TTL in backend), one of the functions 70 | %% from autocluster_periodic should be used. 71 | %% 72 | %% @end 73 | -callback register() -> either_ok_or_error(). 74 | 75 | %% @doc 76 | %% 77 | %% Removes current node from backend database. 78 | %% 79 | %% Can be no-op for some backends. 80 | %% 81 | %% @end 82 | -callback unregister() -> either_ok_or_error(). 83 | -------------------------------------------------------------------------------- /src/autocluster_cleanup.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% @author Gavin M. Roy 3 | %% @copyright 2016 AWeber Communications 4 | %% @end 5 | %%============================================================================== 6 | -module(autocluster_cleanup). 7 | 8 | -behaviour(gen_server). 9 | 10 | -export([start_link/0, 11 | check_cluster/0]). 12 | 13 | %% gen_server callbacks 14 | -export([init/1, 15 | handle_call/3, 16 | handle_cast/2, 17 | handle_info/2, 18 | terminate/2, 19 | code_change/3]). 20 | 21 | %% Export all for unit tests 22 | -ifdef(TEST). 23 | -compile(export_all). 24 | -endif. 25 | 26 | -record(state, {interval, warn_only, timer}). 27 | 28 | %%%=================================================================== 29 | %%% API 30 | %%%=================================================================== 31 | 32 | -spec(start_link() -> 33 | {ok, Pid :: pid()} | ignore | {error, Reason :: term()}). 34 | start_link() -> 35 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). 36 | 37 | 38 | -spec(check_cluster() ->ok). 39 | check_cluster() -> 40 | autocluster_log:debug("(cleanup) checking cluster"), 41 | ok = gen_server:call(?MODULE, check_cluster). 42 | 43 | 44 | %%%=================================================================== 45 | %%% gen_server callbacks 46 | %%%=================================================================== 47 | 48 | %%-------------------------------------------------------------------- 49 | %% @private 50 | %% @doc 51 | %% Initializes the server 52 | %% 53 | %% @spec init(Args) -> {ok, State} | 54 | %% {ok, State, Timeout} | 55 | %% ignore | 56 | %% {stop, Reason} 57 | %% @end 58 | %%-------------------------------------------------------------------- 59 | -spec(init(Args :: term()) -> 60 | {ok, State :: #state{}} | 61 | {ok, State :: #state{}, timeout() | hibernate} | 62 | {stop, Reason :: term()} | ignore). 63 | init([]) -> 64 | case autocluster_config:get(cluster_cleanup) of 65 | true -> 66 | Interval = autocluster_config:get(cleanup_interval), 67 | State = #state{interval = Interval, 68 | warn_only = autocluster_config:get(cleanup_warn_only), 69 | timer = apply_interval(Interval)}, 70 | autocluster_log:info("(cleanup) Timer started {~p,~p}", 71 | [State#state.interval, State#state.warn_only]), 72 | {ok, State}; 73 | _ -> ignore 74 | end. 75 | 76 | %%-------------------------------------------------------------------- 77 | %% @private 78 | %% @doc 79 | %% Handling call messages 80 | %% 81 | %% @end 82 | %%-------------------------------------------------------------------- 83 | -spec(handle_call(Request :: term(), From :: {pid(), Tag :: term()}, 84 | State :: #state{}) -> 85 | {reply, Reply :: term(), NewState :: #state{}} | 86 | {reply, Reply :: term(), NewState :: #state{}, timeout() | hibernate} | 87 | {noreply, NewState :: #state{}} | 88 | {noreply, NewState :: #state{}, timeout() | hibernate} | 89 | {stop, Reason :: term(), Reply :: term(), NewState :: #state{}} | 90 | {stop, Reason :: term(), NewState :: #state{}}). 91 | 92 | handle_call(check_cluster, _From, State) -> 93 | autocluster_log:debug("(cleanup) Checking for partitioned nodes."), 94 | maybe_cleanup(State), 95 | {reply, ok, State}; 96 | handle_call(_Request, _From, State) -> 97 | {reply, ok, State}. 98 | 99 | %%-------------------------------------------------------------------- 100 | %% @private 101 | %% @doc 102 | %% Handling cast messages 103 | %% 104 | %% @end 105 | %%-------------------------------------------------------------------- 106 | -spec(handle_cast(Request :: term(), State :: #state{}) -> 107 | {noreply, NewState :: #state{}} | 108 | {noreply, NewState :: #state{}, timeout() | hibernate} | 109 | {stop, Reason :: term(), NewState :: #state{}}). 110 | handle_cast(_Request, State) -> 111 | {noreply, State}. 112 | 113 | %%-------------------------------------------------------------------- 114 | %% @private 115 | %% @doc 116 | %% Handling all non call/cast messages 117 | %% 118 | %% @spec handle_info(Info, State) -> {noreply, State} | 119 | %% {noreply, State, Timeout} | 120 | %% {stop, Reason, State} 121 | %% @end 122 | %%-------------------------------------------------------------------- 123 | -spec(handle_info(Info :: timeout() | term(), State :: #state{}) -> 124 | {noreply, NewState :: #state{}} | 125 | {noreply, NewState :: #state{}, timeout() | hibernate} | 126 | {stop, Reason :: term(), NewState :: #state{}}). 127 | handle_info(_Info, State) -> 128 | {noreply, State}. 129 | 130 | %%-------------------------------------------------------------------- 131 | %% @private 132 | %% @doc 133 | %% This function is called by a gen_server when it is about to 134 | %% terminate. It should be the opposite of Module:init/1 and do any 135 | %% necessary cleaning up. When it returns, the gen_server terminates 136 | %% with Reason. The return value is ignored. 137 | %% 138 | %% @spec terminate(Reason, State) -> void() 139 | %% @end 140 | %%-------------------------------------------------------------------- 141 | -spec(terminate(Reason :: (normal | shutdown | {shutdown, term()} | term()), 142 | State :: #state{}) -> term()). 143 | terminate(_Reason, _State) -> 144 | ok. 145 | 146 | %%-------------------------------------------------------------------- 147 | %% @private 148 | %% @doc 149 | %% Convert process state when code is changed 150 | %% 151 | %% @spec code_change(OldVsn, State, Extra) -> {ok, NewState} 152 | %% @end 153 | %%-------------------------------------------------------------------- 154 | -spec(code_change(OldVsn :: term() | {down, term()}, State :: #state{}, 155 | Extra :: term()) -> 156 | {ok, NewState :: #state{}} | {error, Reason :: term()}). 157 | code_change(_OldVsn, State, _Extra) -> 158 | {ok, State}. 159 | 160 | %%%=================================================================== 161 | %%% Internal functions 162 | %%%=================================================================== 163 | 164 | %%-------------------------------------------------------------------- 165 | %% @private 166 | %% @doc Create the timer that will invoke a gen_server cast for this 167 | %% module invoking maybe_cleanup/1 every N seconds. 168 | %% @spec apply_interval(integer()) -> timer:tref() 169 | %% @end 170 | %%-------------------------------------------------------------------- 171 | -spec apply_interval(integer()) -> timer:tref(). 172 | apply_interval(Seconds) -> 173 | {ok, TRef} = timer:apply_interval(Seconds * 1000, autocluster_cleanup, 174 | check_cluster, []), 175 | TRef. 176 | 177 | %%-------------------------------------------------------------------- 178 | %% @private 179 | %% @doc Fetch the list of nodes from service discovery and all of the 180 | %% partitioned nodes in RabbitMQ, removing any node from the 181 | %% partitioned list that exists in the service discovery list. 182 | %% @spec maybe_cleanup(State :: #state{}) -> NewState :: #state{} 183 | %% @end 184 | %%-------------------------------------------------------------------- 185 | -spec maybe_cleanup(State :: #state{}) -> ok. 186 | maybe_cleanup(State) -> 187 | maybe_cleanup(State, unreachable_nodes()). 188 | 189 | %%-------------------------------------------------------------------- 190 | %% @private 191 | %% @doc Fetch the list of nodes from service discovery and all of the 192 | %% unreachable nodes in RabbitMQ, removing any node from the 193 | %% unreachable list that exists in the service discovery list. 194 | %% @spec maybe_cleanup(State :: #state{}, 195 | %% UnreachableNodes :: [node()]) -> ok 196 | %% @end 197 | %%-------------------------------------------------------------------- 198 | -spec maybe_cleanup(State :: #state{}, 199 | UnreachableNodes :: [node()]) -> ok. 200 | maybe_cleanup(_, []) -> 201 | autocluster_log:debug("(cleanup) No partitioned nodes found."); 202 | maybe_cleanup(State, UnreachableNodes) -> 203 | autocluster_log:debug("(cleanup) Unreachable RabbitMQ nodes ~p", 204 | [UnreachableNodes]), 205 | case lists:subtract(UnreachableNodes, service_discovery_nodes()) of 206 | [] -> 207 | autocluster_log:debug("(cleanup) No unreachable nodes found."), 208 | ok; 209 | Nodes -> 210 | autocluster_log:debug("(cleanup) unreachable nodes ~p", [Nodes]), 211 | maybe_remove_nodes(Nodes, State#state.warn_only) 212 | end. 213 | 214 | %%-------------------------------------------------------------------- 215 | %% @private 216 | %% @doc Iterate over the list of partitioned nodes, either logging the 217 | %% node that would be removed or actually removing it. 218 | %% @spec maybe_remove_nodes(PartitionedNodes :: [node()], 219 | %% WarnOnly :: true | false) -> ok 220 | %% @end 221 | %%-------------------------------------------------------------------- 222 | -spec maybe_remove_nodes(PartitionedNodes :: [node()], 223 | WarnOnly :: true | false) -> ok. 224 | maybe_remove_nodes([], _) -> ok; 225 | maybe_remove_nodes([Node|Nodes], true) -> 226 | autocluster_log:warning("(cleanup) ~p is unhealthy", [Node]), 227 | maybe_remove_nodes(Nodes, true); 228 | maybe_remove_nodes([Node|Nodes], false) -> 229 | autocluster_log:warning("(cleanup) removing ~p from cluster", [Node]), 230 | rabbit_mnesia:forget_cluster_node(Node, false), 231 | maybe_remove_nodes(Nodes, false). 232 | 233 | %%-------------------------------------------------------------------- 234 | %% @private 235 | %% @doc Return nodes in the RabbitMQ cluster that are unhealthy. 236 | %% @spec unreachable_nodes() -> [node()] 237 | %% @end 238 | %%-------------------------------------------------------------------- 239 | -spec unreachable_nodes() -> [node()]. 240 | unreachable_nodes() -> 241 | Status = rabbit_mnesia:status(), 242 | Nodes = proplists:get_value(nodes, Status, []), 243 | Running = proplists:get_value(running_nodes, Status, []), 244 | All = lists:merge(proplists:get_value(disc, Nodes, []), 245 | proplists:get_value(ram, Nodes, [])), 246 | lists:subtract(All, Running). 247 | 248 | 249 | %%-------------------------------------------------------------------- 250 | %% @private 251 | %% @doc Return the nodes that the service discovery backend knows about 252 | %% @spec service_discovery_nodes() -> [node()] 253 | %% @end 254 | %%-------------------------------------------------------------------- 255 | -spec service_discovery_nodes() -> [node()]. 256 | service_discovery_nodes() -> 257 | Module = autocluster_util:backend_module(), 258 | case Module:nodelist() of 259 | {ok, Nodes} -> 260 | autocluster_log:debug("(cleanup) ~p returned ~p", 261 | [Module, Nodes]), 262 | Nodes; 263 | {error, Reason} -> 264 | autocluster_log:debug("(cleanup) ~p returned error ~p", 265 | [Module, Reason]), 266 | [] 267 | end. 268 | -------------------------------------------------------------------------------- /src/autocluster_config.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% @author Gavin M. Roy 3 | %% @copyright 2015-2016 AWeber Communications 4 | %% @end 5 | %%============================================================================== 6 | -module(autocluster_config). 7 | 8 | -export([get/1]). 9 | 10 | -include("autocluster.hrl"). 11 | 12 | %% Export all for unit tests 13 | -ifdef(TEST). 14 | -compile(export_all). 15 | -endif. 16 | 17 | %%-------------------------------------------------------------------- 18 | %% @private 19 | %% @doc 20 | %% Fetch the specified config value, checking first for an OS 21 | %% environment variable, then checking application config. If neither 22 | %% are set, return the default value. 23 | %% @end 24 | %%-------------------------------------------------------------------- 25 | -spec get(Key :: atom()) -> atom() | integer() | string() | undefined. 26 | get(Key) -> 27 | maybe_get_value(Key, lists:keysearch(Key, #config.key, ?CONFIG_MAP)). 28 | 29 | 30 | %%-------------------------------------------------------------------- 31 | %% @private 32 | %% @doc 33 | %% Return the a value from the OS environment value, or the erlang 34 | %% application environment value if OS env var is not set, or the 35 | %% default value. 36 | %% @end 37 | %%-------------------------------------------------------------------- 38 | -spec getenv(OSKey :: string(), AppKey :: atom(), 39 | Default :: atom() | integer() | string()) 40 | -> atom() | integer() | string(). 41 | getenv(OSKey, AppKey, Default) -> 42 | case getenv(OSKey) of 43 | false -> application:get_env(autocluster, AppKey, Default); 44 | Value -> Value 45 | end. 46 | 47 | 48 | %%-------------------------------------------------------------------- 49 | %% @private 50 | %% @doc 51 | %% Attempt to return the variable from the operating system. If it 52 | %% fails, and the variable starts with ``RABBITMQ_``, chop off the 53 | %% ``RABBITMQ_`` prefix and attempt to get it from there. 54 | %% @end 55 | %%-------------------------------------------------------------------- 56 | -spec getenv(Key :: string()) -> string() | false. 57 | getenv(Key) -> 58 | process_getenv_value(Key, os:getenv(Key)). 59 | 60 | 61 | %%-------------------------------------------------------------------- 62 | %% @private 63 | %% @doc 64 | %% Evaluate the check to see if the requested key is in the 65 | %% configuration map. If not, return ``undefined`` otherwise attempt 66 | %% to get the configuration from the OS environment variable or the 67 | %% application environment variable. 68 | %% @end 69 | %%-------------------------------------------------------------------- 70 | -spec maybe_get_value(Key :: atom(), {value, #config{}} | false) 71 | -> atom() | integer() | string() | undefined. 72 | maybe_get_value(_, false) -> undefined; 73 | maybe_get_value(Key, {value, Config}) -> 74 | normalize(Config, getenv(Config#config.os, Key, Config#config.default)). 75 | 76 | 77 | %%-------------------------------------------------------------------- 78 | %% @private 79 | %% @doc 80 | %% Check the response of os:getenv/1 to see if it's false and if it is 81 | %% chain down to maybe_getenv_with_subkey/2 to see if the environment 82 | %% variable has a prefix of RABBITMQ_, potentially trying to get an 83 | %% environment variable without the prefix. 84 | %% @end 85 | %%-------------------------------------------------------------------- 86 | -spec process_getenv_value(Key :: string(), Value :: string() | false) 87 | -> string() | false. 88 | process_getenv_value(Key, false) -> 89 | maybe_getenv_with_subkey(Key, string:left(Key, 9)); 90 | process_getenv_value(_, Value) -> Value. 91 | 92 | 93 | %%-------------------------------------------------------------------- 94 | %% @private 95 | %% @doc 96 | %% Check to see if the OS environment variable starts with RABBITMQ_ 97 | %% and if so, try and fetch the value from an environment variable 98 | %% with the prefix removed. 99 | %% @end 100 | %%-------------------------------------------------------------------- 101 | -spec maybe_getenv_with_subkey(Key :: string(), Prefix :: string()) 102 | -> string() | false. 103 | maybe_getenv_with_subkey(Key, "RABBITMQ_") -> 104 | os:getenv(string:sub_string(Key, 10)); 105 | maybe_getenv_with_subkey(_, _) -> 106 | false. 107 | 108 | 109 | %%-------------------------------------------------------------------- 110 | %% @private 111 | %% @doc 112 | %% Return the normalized value in as the proper data type 113 | %% @end 114 | %%-------------------------------------------------------------------- 115 | -spec normalize(Map :: #config{}, 116 | Value :: atom() | boolean() | integer() | string()) -> 117 | atom() | integer() | string(). 118 | normalize(Config, Value) when Config#config.is_port =:= true -> 119 | autocluster_util:parse_port(Value); 120 | normalize(Config, Value) when Config#config.type =:= atom -> 121 | autocluster_util:as_atom(Value); 122 | normalize(Config, Value) when Config#config.type =:= integer -> 123 | autocluster_util:as_integer(Value); 124 | normalize(Config, Value) when Config#config.type =:= string -> 125 | autocluster_util:as_string(Value); 126 | normalize(Config, Value) when Config#config.type =:= proplist -> 127 | autocluster_util:as_proplist(Value); 128 | normalize(Config, Value) when Config#config.type =:= list -> 129 | autocluster_util:as_list(Value). 130 | -------------------------------------------------------------------------------- /src/autocluster_dns.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% @author Gavin M. Roy 3 | %% @copyright 2015-2016 AWeber Communications 4 | %% @end 5 | %%============================================================================== 6 | -module(autocluster_dns). 7 | 8 | -behavior(autocluster_backend). 9 | 10 | %% autocluster_backend methods 11 | -export([nodelist/0, 12 | lock/1, 13 | unlock/1, 14 | register/0, 15 | unregister/0]). 16 | 17 | %% Export all for unit tests 18 | -ifdef(TEST). 19 | -compile(export_all). 20 | -endif. 21 | 22 | -include("autocluster.hrl"). 23 | 24 | %% @spec nodelist() -> {ok, list()}|{error, Reason :: string()} 25 | %% @doc Return a list of nodes registered in Consul 26 | %% @end 27 | %% 28 | nodelist() -> {ok, [autocluster_util:node_name(N) || N <- build_node_list()]}. 29 | 30 | -spec lock(string()) -> not_supported. 31 | lock(_) -> 32 | not_supported. 33 | 34 | -spec unlock(term()) -> ok. 35 | unlock(_) -> 36 | ok. 37 | 38 | %% @spec register() -> ok|{error, Reason :: string()} 39 | %% @doc Stub, since this module does not update DNS 40 | %% @end 41 | %% 42 | register() -> ok. 43 | 44 | 45 | %% @spec unregister() -> ok|{error, Reason :: string()} 46 | %% @doc Stub, since this module does not update DNS 47 | %% @end 48 | %% 49 | unregister() -> ok. 50 | 51 | 52 | %% @spec build_node_list() -> list() 53 | %% @doc Return a list of nodes from DNS A RRs 54 | %% @end 55 | %% 56 | build_node_list() -> 57 | Name = autocluster_config:get(autocluster_host), 58 | Hosts = [extract_host(inet:gethostbyaddr(A)) || A <- inet_res:lookup(Name, in, a)], 59 | lists:filter(fun(E) -> E =/= error end, Hosts). 60 | 61 | 62 | %% @spec extract_host({ok, dns_msg()}) -> string() 63 | %% @doc Return a list of nodes from DNS A RRs 64 | %% @end 65 | %% 66 | extract_host({ok, {hostent, FQDN, _, _, _, _}}) -> 67 | case autocluster_config:get(longname) of 68 | true -> FQDN; 69 | false -> lists:nth(1, string:tokens(FQDN, ".")) 70 | end; 71 | extract_host(Error) -> 72 | autocluster_log:error("Error resolving host by address: ~p", [Error]), 73 | error. 74 | -------------------------------------------------------------------------------- /src/autocluster_etcd.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% @author Gavin M. Roy 3 | %% @copyright 2015-2016 AWeber Communications 4 | %% @end 5 | %%============================================================================== 6 | -module(autocluster_etcd). 7 | 8 | -behavior(autocluster_backend). 9 | 10 | %% autocluster_backend methods 11 | -export([nodelist/0, 12 | lock/1, 13 | unlock/1, 14 | register/0, 15 | unregister/0]). 16 | 17 | %% Private 18 | -export([lock_ttl_update_callback/1, node_key_update_callback/0]). 19 | 20 | %% Export all for unit tests 21 | -ifdef(TEST). 22 | -compile(export_all). 23 | -endif. 24 | 25 | -include("autocluster.hrl"). 26 | 27 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 28 | %% autocluster_backend methods 29 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 30 | 31 | %% @doc Return a list of nodes registered in etcd 32 | %% @end 33 | -spec nodelist() -> {ok, [node()]}|{error, Reason :: string()}. 34 | nodelist() -> 35 | case etcd_get(nodes_path(), [{recursive, true}]) of 36 | {ok, Nodes} -> 37 | NodeList = extract_nodes(Nodes), 38 | {ok, NodeList}; 39 | {error, "404"} -> 40 | {ok, []}; 41 | Error -> Error 42 | end. 43 | 44 | %% @doc Tries to acquire lock using compare-and-swap operation in 45 | %% etcd. If locking succeeds, starts periodic action to refresh TTL on 46 | %% the lock. 47 | %% @end. 48 | -spec lock(string()) -> ok | {error, string()}. 49 | lock(Who) -> 50 | Now = time_compat:erlang_system_time(seconds), 51 | EndTime = Now + autocluster_config:get(lock_wait_time), 52 | lock(Who ++ " - " ++ generate_unique_string(), Now, EndTime). 53 | 54 | %% @doc Stops lock TTL updater and removes lock from etcd. 'ok' is 55 | %% only returned when lock was successfull released (i.e. it wasn't 56 | %% stolen from us somehow). 57 | %% @end 58 | -spec unlock(term()) -> ok | {error, string()}. 59 | unlock(UniqueId) -> 60 | stop_lock_ttl_updater(UniqueId), 61 | case delete_etcd_lock_key(UniqueId) of 62 | {ok, _} -> 63 | ok; 64 | {error, _} = Err -> 65 | Err 66 | end. 67 | 68 | %% @doc Registers this node's key in etcd and start a periodic process 69 | %% to refresh TTL of that key. 70 | %% @end 71 | -spec register() -> ok|{error, Reason :: string()}. 72 | register() -> 73 | set_etcd_node_key(), 74 | start_node_key_updater(). 75 | 76 | %% @doc Remove this node's key from etcd 77 | %% @end 78 | -spec unregister() -> ok|{error, Reason :: string()}. 79 | unregister() -> 80 | stop_node_key_updater(), 81 | autocluster_log:info("Unregistering node with etcd"), 82 | case etcd_delete(node_path(), [{recursive, true}]) of 83 | {ok, _} -> ok; 84 | Error -> Error 85 | end. 86 | 87 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 88 | %% Helpers 89 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 90 | 91 | %% @doc 92 | %% Tries to acquire the lock. Will retry until the lock is finally 93 | %% granted or time is up. 94 | %% @end 95 | -spec lock(string(), pos_integer(), pos_integer()) -> ok | {error, string()}. 96 | lock(_, Now, EndTime) when EndTime < Now -> 97 | {error, "Acquiring the lock taking too long, bailing out"}; 98 | lock(UniqueId, _, EndTime) -> 99 | case try_insert_lock_key(UniqueId) of 100 | true -> 101 | start_lock_ttl_updater(UniqueId), 102 | {ok, UniqueId}; 103 | false -> 104 | wait_for_lock_release(), 105 | lock(UniqueId, time_compat:erlang_system_time(seconds), EndTime); 106 | {error, Reason} -> 107 | {error, lists:flatten(io_lib:format("Error while acquiring the lock, reason: ~p", [Reason]))} 108 | end. 109 | 110 | 111 | %% @doc Update etcd, setting a key for this node with a TTL of etcd_node_ttl 112 | %% @end 113 | -spec set_etcd_node_key() -> ok. 114 | set_etcd_node_key() -> 115 | Interval = autocluster_config:get(etcd_node_ttl), 116 | case etcd_put(node_path(), [{ttl, Interval}], [{value, enabled}]) of 117 | {ok, _} -> 118 | autocluster_log:debug("Updated node registration with etcd"); 119 | {error, Error} -> 120 | autocluster_log:debug("Failed to update node registration with etcd: ~s", [Error]) 121 | end, 122 | ok. 123 | 124 | %% @doc Part of etcd path that allows us to distinguish different 125 | %% cluster using the same etcd server. 126 | %% @end 127 | -spec cluster_name_path_part() -> string(). 128 | cluster_name_path_part() -> 129 | case autocluster_config:get(cluster_name) of 130 | "undefined" -> "default"; 131 | Value -> Value 132 | end. 133 | 134 | %% @doc Return a list of path segments that are the base path for all 135 | %% etcd keys related to current cluster. 136 | %% @end 137 | -spec base_path() -> [autocluster_httpc:path_component()]. 138 | base_path() -> 139 | [v2, keys, autocluster_config:get(etcd_prefix), cluster_name_path_part()]. 140 | 141 | %% @doc Returns etcd path under which nodes should be registered. 142 | %% @end 143 | -spec nodes_path() -> [autocluster_httpc:path_component()]. 144 | nodes_path() -> 145 | base_path() ++ [nodes]. 146 | 147 | %% @doc Returns etcd path under which current node should be registered 148 | %% @end 149 | -spec node_path() -> [autocluster_httpc:path_component()]. 150 | node_path() -> 151 | nodes_path() ++ [atom_to_list(node())]. 152 | 153 | %% @doc Returns etcd path for startup lock 154 | %% @end 155 | -spec startup_lock_path() -> [autocluster_httpc:path_component()]. 156 | startup_lock_path() -> 157 | base_path() ++ ["startup_lock"]. 158 | 159 | %% @doc Return the list of erlang nodes 160 | %% @end 161 | %% 162 | -spec extract_nodes(list(), list()) -> [node()]. 163 | extract_nodes([], Nodes) -> Nodes; 164 | extract_nodes([{struct, H}|T], Nodes) -> 165 | extract_nodes(T, lists:append(Nodes, [get_node_from_key(proplists:get_value(<<"key">>, H))])). 166 | 167 | %% @doc Return the list of erlang nodes 168 | %% @end 169 | %% 170 | -spec extract_nodes(list()) -> [node()]. 171 | extract_nodes([]) -> []; 172 | extract_nodes({struct, Nodes}) -> 173 | {struct, Dir} = proplists:get_value(<<"node">>, Nodes), 174 | case proplists:get_value(<<"nodes">>, Dir) of 175 | undefined -> []; 176 | Values -> extract_nodes(Values, []) 177 | end; 178 | extract_nodes(Miss) -> 179 | io:format("Unparsed: ~p~n", [Miss]), 180 | []. 181 | 182 | 183 | %% @doc Given an etcd key, return the erlang node name 184 | %% @end 185 | %% 186 | -spec get_node_from_key(binary()) -> node(). 187 | get_node_from_key(<<"/", V/binary>>) -> get_node_from_key(V); 188 | get_node_from_key(V) -> 189 | %% nodes path is /v2/keys///nodes 190 | %% etcd returns node keys as ///nodes/ 191 | %% We are mapping path components from "" up to "nodes", 192 | %% and discarding the same number of characters from the key returned by etcd. 193 | Path = string:concat(autocluster_httpc:build_path(lists:sublist(nodes_path(), 3, 3)), "/"), 194 | autocluster_util:node_name(string:substr(binary_to_list(V), length(Path))). 195 | 196 | %% @doc Generate random string. We are using it for compare-and-change 197 | %% operations in etcd. 198 | %% @end 199 | -spec generate_unique_string() -> string(). 200 | generate_unique_string() -> 201 | [ $a - 1 + rand_compat:uniform(26) || _ <- lists:seq(1, 32) ]. 202 | 203 | %% @doc Tries to acquire a lock in etcd. This can either succeed, fail 204 | %% because somebody else is holding the lock, or completely file due 205 | %% to some I/O error. 206 | %% @end 207 | -spec try_insert_lock_key(string()) -> boolean() | {error, term()}. 208 | try_insert_lock_key(UniqueId) -> 209 | Ttl = autocluster_config:get(etcd_node_ttl), 210 | case set_etcd_lock_key(UniqueId, Ttl) of 211 | {ok, _} -> 212 | true; 213 | %% Precondition failed 214 | {error, "412"} -> 215 | false; 216 | {error, _} = Err -> 217 | Err 218 | end. 219 | 220 | %% @doc Orders etcd to create startup lock key if it doesn't exist already. 221 | %% @end 222 | -spec set_etcd_lock_key(string(), non_neg_integer()) -> {ok, term()} | {error, string()}. 223 | set_etcd_lock_key(UniqueId, Ttl) -> 224 | etcd_put(startup_lock_path(), 225 | [{ttl, Ttl}, {'prevExist', "false"}], 226 | [{value, UniqueId}]). 227 | 228 | %% @doc Refresh startup lock TTL in etcd, but only if we are the holder of that lock. 229 | %% @end 230 | -spec refresh_etcd_lock_ttl(string(), non_neg_integer()) -> {ok, term()} | {error, string()}. 231 | refresh_etcd_lock_ttl(UniqueId, Ttl) -> 232 | etcd_put(startup_lock_path(), 233 | [], 234 | [{ttl, Ttl}, {'prevExist', true}, {'prevValue', UniqueId}, {refresh, true}]). 235 | 236 | %% @doc Delete startup lock in etcd, but only if we are the holder of that lock. 237 | %% @end 238 | -spec delete_etcd_lock_key(string()) -> {ok, term()} | {error, string()}. 239 | delete_etcd_lock_key(UniqueId) -> 240 | etcd_delete(startup_lock_path(), 241 | [{'prevExist', "true"}, {'prevValue', UniqueId}]). 242 | 243 | 244 | -spec etcd_delete(Path, Query) -> {ok, term()} | {error, string()} when 245 | Path :: [autocluster_httpc:path_component()], 246 | Query :: [autocluster_httpc:query_component()]. 247 | etcd_delete(Path, Query) -> 248 | autocluster_util:stringify_error( 249 | autocluster_httpc:delete(autocluster_config:get(etcd_scheme), 250 | autocluster_config:get(etcd_host), 251 | autocluster_config:get(etcd_port), 252 | Path, Query, "")). 253 | 254 | -spec etcd_get(Path, Query) -> {ok, term()} | {error, string()} when 255 | Path :: [autocluster_httpc:path_component()], 256 | Query :: [autocluster_httpc:query_component()]. 257 | etcd_get(Path, Query) -> 258 | autocluster_util:stringify_error( 259 | autocluster_httpc:get(autocluster_config:get(etcd_scheme), 260 | autocluster_config:get(etcd_host), 261 | autocluster_config:get(etcd_port), 262 | Path, Query)). 263 | 264 | -spec etcd_put(Path, Query, Body) -> {ok, term()} | {error, string()} when 265 | Path :: [autocluster_httpc:path_component()], 266 | Query :: [autocluster_httpc:query_component()], 267 | Body :: [autocluster_httpc:query_component()]. 268 | etcd_put(Path, Query, Body) -> 269 | autocluster_util:stringify_error( 270 | autocluster_httpc:put(autocluster_config:get(etcd_scheme), 271 | autocluster_config:get(etcd_host), 272 | autocluster_config:get(etcd_port), 273 | Path, Query, autocluster_httpc:build_query(Body))). 274 | 275 | -spec lock_ttl_update_callback(string()) -> string(). 276 | lock_ttl_update_callback(UniqueId) -> 277 | _ = refresh_etcd_lock_ttl(UniqueId, autocluster_config:get(etcd_node_ttl)), 278 | UniqueId. 279 | 280 | -spec start_lock_ttl_updater(string()) -> ok. 281 | start_lock_ttl_updater(UniqueId) -> 282 | Interval = autocluster_config:get(etcd_node_ttl), 283 | autocluster_log:debug("Starting startup lock refresher"), 284 | autocluster_periodic:start_delayed({autocluster_etcd_lock, UniqueId}, 285 | Interval * 500, 286 | {?MODULE, lock_ttl_update_callback, [UniqueId]}). 287 | 288 | -spec stop_lock_ttl_updater(string()) -> ok. 289 | stop_lock_ttl_updater(UniqueId) -> 290 | _ = autocluster_periodic:stop({autocluster_etcd_lock, UniqueId}), 291 | autocluster_log:debug("Stopped startup lock refresher"), 292 | ok. 293 | 294 | -spec wait_for_lock_release() -> ok. 295 | wait_for_lock_release() -> 296 | %% XXX Try to use etcd wait feature, but we somehow need to know 297 | %% the index from the last lock attempt operation. 298 | timer:sleep(1000). 299 | 300 | -spec node_key_update_callback() -> ok. 301 | node_key_update_callback() -> 302 | set_etcd_node_key(). 303 | 304 | -spec start_node_key_updater() -> ok. 305 | start_node_key_updater() -> 306 | Interval = autocluster_config:get(etcd_node_ttl), 307 | autocluster_log:debug("Starting etcd TTL node key updater"), 308 | autocluster_periodic:start_delayed(autocluster_etcd_node_key_updater, Interval * 500, 309 | {?MODULE, node_key_update_callback, []}). 310 | 311 | -spec stop_node_key_updater() -> ok. 312 | stop_node_key_updater() -> 313 | _ = autocluster_periodic:stop(autocluster_etcd_node_key_updater), 314 | ok. 315 | -------------------------------------------------------------------------------- /src/autocluster_httpc.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% @author Gavin M. Roy 3 | %% @copyright 2015-2016 AWeber Communications 4 | %% @end 5 | %%============================================================================== 6 | -module(autocluster_httpc). 7 | 8 | %% API 9 | -export([build_query/1, 10 | build_path/1, 11 | build_uri/5, 12 | delete/6, 13 | get/5, 14 | get/7, 15 | post/6, 16 | put/6, 17 | put/7]). 18 | 19 | %% Export all for unit tests 20 | -ifdef(TEST). 21 | -compile(export_all). 22 | -endif. 23 | 24 | 25 | -define(CONTENT_JSON, "application/json"). 26 | -define(CONTENT_URLENCODED, "application/x-www-form-urlencoded"). 27 | 28 | 29 | -type path_component() :: autocluster_util:stringifyable(). 30 | -type query_component() :: {autocluster_util:stringifyable(), autocluster_util:stringifyable()}. 31 | -export_type([path_component/0, query_component/0]). 32 | 33 | 34 | %% @public 35 | %% @doc Build the path from a list of segments 36 | %% @end 37 | %% 38 | -spec build_path([path_component()]) -> string(). 39 | build_path(Args) -> 40 | build_path(Args, []). 41 | 42 | 43 | %% @public 44 | %% @doc Build the path from a list of segments 45 | %% @end 46 | %% 47 | -spec build_path([path_component()], string()) -> string(). 48 | build_path([Part|Parts], Path) -> 49 | build_path(Parts, string:join([Path, percent_encode(Part)], "/")); 50 | build_path([], Path) -> Path. 51 | 52 | 53 | %% @public 54 | %% @doc Build the request URI 55 | %% @end 56 | %% 57 | -spec build_uri(Scheme, Host, Port, [path_component()], [query_component()]) -> string() when 58 | Scheme :: string(), 59 | Host :: string(), 60 | Port :: integer(). 61 | build_uri(Scheme, Host, Port, Path, QArgs) -> 62 | build_uri(string:join([Scheme, "://", Host, ":", autocluster_util:as_string(Port)], ""), Path, QArgs). 63 | 64 | 65 | %% @public 66 | %% @doc Build the requst URI for the given base URI, path and query args 67 | %% @end 68 | %% 69 | -spec build_uri(string(), [path_component()], [query_component()]) -> string(). 70 | build_uri(Base, Path, []) -> 71 | string:join([Base, build_path(Path)], ""); 72 | build_uri(Base, Path, QArgs) -> 73 | string:join([Base, string:join([build_path(Path), 74 | build_query(QArgs)], "?")], ""). 75 | 76 | 77 | %% @public 78 | %% @doc Build the query parameters string from a proplist 79 | %% @end 80 | %% 81 | -spec build_query([query_component()]) -> string(). 82 | build_query(Args) -> 83 | build_query(Args, []). 84 | 85 | 86 | %% @public 87 | %% @doc Build the query parameters string from a proplist 88 | %% @end 89 | %% 90 | -spec build_query([query_component()], [string()]) -> string(). 91 | build_query([{Key,Value}|Args], Parts) -> 92 | build_query(Args, lists:merge(Parts, [string:join([percent_encode(Key), 93 | percent_encode(Value)], "=")])); 94 | build_query([Key|Args], Parts) -> 95 | build_query(Args, lists:merge(Parts, [percent_encode(Key)])); 96 | build_query([], Parts) -> 97 | string:join(Parts, "&"). 98 | 99 | 100 | %% @public 101 | %% @doc Perform a HTTP GET request 102 | %% @end 103 | %% 104 | -spec get(Scheme, Host, Port, Path, Args) -> Result when 105 | Scheme :: string(), 106 | Host :: string(), 107 | Port :: integer(), 108 | Path :: [path_component()], 109 | Args :: [query_component()], 110 | Result :: {ok, term()} | {error, Reason::term()}. 111 | get(Scheme, Host, Port, Path, Args) -> 112 | get(Scheme, Host, Port, Path, Args, [], []). 113 | 114 | 115 | %% @public 116 | %% @doc Perform a HTTP GET request 117 | %% @end 118 | %% 119 | -spec get(Scheme, Host, Port, Path, Args, Headers, HttpOpts) -> Result when 120 | Scheme :: string(), 121 | Host :: string(), 122 | Port :: integer(), 123 | Path :: [path_component()], 124 | Args :: [query_component()], 125 | Headers :: [{string(), string()}], 126 | HttpOpts :: [{atom(), term()}], 127 | Result :: {ok, term()} | {error, Reason::term()}. 128 | get(Scheme, Host, Port, Path, Args, Headers, HttpOpts) -> 129 | URL = build_uri(Scheme, Host, Port, Path, Args), 130 | autocluster_log:debug("GET ~s", [URL]), 131 | Response = httpc:request(get, {URL, Headers}, HttpOpts, []), 132 | autocluster_log:debug("Response: [~p]", [Response]), 133 | parse_response(Response). 134 | 135 | %% @public 136 | %% @doc Perform a HTTP POST request 137 | %% @end 138 | %% 139 | -spec post(Scheme, Host, Port, Path, Args, Body) -> Result when 140 | Scheme :: string(), 141 | Host :: string(), 142 | Port :: integer(), 143 | Path :: [path_component()], 144 | Args :: [query_component()], 145 | Body :: string() | binary(), 146 | Result :: {ok, term()} | {error, Reason::term()}. 147 | post(Scheme, Host, Port, Path, Args, Body) -> 148 | URL = build_uri(Scheme, Host, Port, Path, Args), 149 | autocluster_log:debug("POST ~s [~p]", [URL, Body]), 150 | Response = httpc:request(post, {URL, [], ?CONTENT_JSON, Body}, [], []), 151 | autocluster_log:debug("Response: [~p]", [Response]), 152 | parse_response(Response). 153 | 154 | 155 | %% @public 156 | %% @spec put(Scheme, Host, Port, Path, Args, Body) -> Result 157 | %% @where Scheme = string(), 158 | %% Host = string(), 159 | %% Port = integer(), 160 | %% Path = string(), 161 | %% Args = proplist(), 162 | %% Body = string(), 163 | %% Result = {ok, mixed}|{error, Reason::string()} 164 | %% @doc Perform a HTTP PUT request 165 | %% @end 166 | %% 167 | put(Scheme, Host, Port, Path, Args, Body) -> 168 | URL = build_uri(Scheme, Host, Port, Path, Args), 169 | autocluster_log:debug("PUT ~s [~p]", [URL, Body]), 170 | Response = httpc:request(put, {URL, [], ?CONTENT_URLENCODED, Body}, [], []), 171 | autocluster_log:debug("Response: [~p]", [Response]), 172 | parse_response(Response). 173 | 174 | 175 | %% @public 176 | %% @spec put(Scheme, Host, Port, Path, Args, Headers, Body) -> Result 177 | %% @where Scheme = string(), 178 | %% Host = string(), 179 | %% Port = integer(), 180 | %% Path = string(), 181 | %% Args = proplist(), 182 | %% Headers = proplist(), 183 | %% Body = string(), 184 | %% Result = {ok, mixed}|{error, Reason::string()} 185 | %% @doc Perform a HTTP PUT request 186 | %% @end 187 | %% 188 | put(Scheme, Host, Port, Path, Args, Headers, Body) -> 189 | URL = build_uri(Scheme, Host, Port, Path, Args), 190 | autocluster_log:debug("PUT ~s [~p] [~p]", [URL, Headers, Body]), 191 | Response = httpc:request(put, {URL, Headers, ?CONTENT_URLENCODED, Body}, [], []), 192 | autocluster_log:debug("Response: [~p]", [Response]), 193 | parse_response(Response). 194 | 195 | 196 | %% @public 197 | %% @spec delete(Scheme, Host, Port, Path, Args, Body) -> Result 198 | %% @where Scheme = string(), 199 | %% Host = string(), 200 | %% Port = integer(), 201 | %% Path = string(), 202 | %% Args = proplist(), 203 | %% Body = string(), 204 | %% Result = {ok, mixed}|{error, Reason::string()} 205 | %% @doc Perform a HTTP DELETE request 206 | %% @end 207 | %% 208 | delete(Scheme, Host, Port, Path, Args, Body) -> 209 | URL = build_uri(Scheme, Host, Port, Path, Args), 210 | autocluster_log:debug("DELETE ~s [~p]", [URL, Body]), 211 | Response = httpc:request(delete, {URL, [], ?CONTENT_URLENCODED, Body}, [], []), 212 | autocluster_log:debug("Response: [~p]", [Response]), 213 | parse_response(Response). 214 | 215 | 216 | %% @private 217 | %% @spec decode_body(mixed) -> list() 218 | %% @doc Decode the response body and return a list 219 | %% @end 220 | %% 221 | decode_body(_, []) -> []; 222 | decode_body(?CONTENT_JSON, Body) -> 223 | case rabbit_misc:json_decode(autocluster_util:as_string(Body)) of 224 | {ok, Value} -> Value; 225 | error -> [] 226 | end. 227 | 228 | 229 | %% @private 230 | %% @spec parse_response(Response) -> {ok, string()} | {error, mixed} 231 | %% @where Response = {status_line(), headers(), Body} | {status_code(), Body} 232 | %% @doc Decode the response body and return a list 233 | %% @end 234 | %% 235 | parse_response({error, Reason}) -> 236 | autocluster_log:debug("HTTP Error ~p", [Reason]), 237 | {error, Reason}; 238 | 239 | parse_response({ok, 200, Body}) -> {ok, decode_body(?CONTENT_JSON, Body)}; 240 | parse_response({ok, 201, Body}) -> {ok, decode_body(?CONTENT_JSON, Body)}; 241 | parse_response({ok, 204, _}) -> {ok, []}; 242 | parse_response({ok, Code, Body}) -> 243 | autocluster_log:debug("HTTP Response (~p) ~s", [Code, Body]), 244 | {error, integer_to_list(Code)}; 245 | 246 | parse_response({ok, {{_,200,_},Headers,Body}}) -> 247 | {ok, decode_body(proplists:get_value("content-type", Headers, ?CONTENT_JSON), Body)}; 248 | parse_response({ok,{{_,201,_},Headers,Body}}) -> 249 | {ok, decode_body(proplists:get_value("content-type", Headers, ?CONTENT_JSON), Body)}; 250 | parse_response({ok,{{_,204,_},_,_}}) -> {ok, []}; 251 | parse_response({ok,{{_Vsn,Code,_Reason},_,Body}}) -> 252 | autocluster_log:debug("HTTP Response (~p) ~s", [Code, Body]), 253 | {error, integer_to_list(Code)}. 254 | 255 | 256 | %% @private 257 | %% @spec percent_encode(Value) -> string() 258 | %% @where 259 | %% Value = atom() or binary() or integer() or list() 260 | %% @doc Percent encode the query value, automatically 261 | %% converting atoms, binaries, or integers 262 | %% @end 263 | %% 264 | percent_encode(Value) -> 265 | http_uri:encode(autocluster_util:as_string(Value)). 266 | -------------------------------------------------------------------------------- /src/autocluster_k8s.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% @author Grzegorz Grasza 3 | %% @copyright 2016 Intel Corporation 4 | %% @end 5 | %%============================================================================== 6 | -module(autocluster_k8s). 7 | 8 | -behavior(autocluster_backend). 9 | 10 | %% autocluster_backend methods 11 | -export([nodelist/0, 12 | lock/1, 13 | unlock/1, 14 | register/0, 15 | unregister/0]). 16 | 17 | %% Export all for unit tests 18 | -ifdef(TEST). 19 | -compile(export_all). 20 | -endif. 21 | 22 | -include("autocluster.hrl"). 23 | 24 | 25 | %% @spec nodelist() -> {ok, list()}|{error, Reason :: string()} 26 | %% @doc Return a list of nodes registered in K8s 27 | %% @end 28 | %% 29 | nodelist() -> 30 | case make_request() of 31 | {ok, Response} -> 32 | Addresses = extract_node_list(Response), 33 | {ok, lists:map(fun node_name/1, Addresses)}; 34 | {error, Reason} -> 35 | autocluster_log:info( 36 | "Failed to get nodes from k8s - ~p", [Reason]), 37 | {error, Reason} 38 | end. 39 | 40 | 41 | -spec lock(string()) -> not_supported. 42 | lock(_) -> 43 | not_supported. 44 | 45 | -spec unlock(term()) -> ok. 46 | unlock(_) -> 47 | ok. 48 | 49 | %% @spec register() -> ok|{error, Reason :: string()} 50 | %% @doc Stub, since this module does not update DNS 51 | %% @end 52 | %% 53 | register() -> ok. 54 | 55 | 56 | %% @spec unregister() -> ok|{error, Reason :: string()} 57 | %% @doc Stub, since this module does not update DNS 58 | %% @end 59 | %% 60 | unregister() -> ok. 61 | 62 | 63 | %% @doc Perform a HTTP GET request to K8s 64 | %% @end 65 | %% 66 | -spec make_request() -> {ok, term()} | {error, term()}. 67 | make_request() -> 68 | {ok, Token} = file:read_file(autocluster_config:get(k8s_token_path)), 69 | Token1 = binary:replace(Token, <<"\n">>, <<>>), 70 | autocluster_httpc:get( 71 | autocluster_config:get(k8s_scheme), 72 | autocluster_config:get(k8s_host), 73 | autocluster_config:get(k8s_port), 74 | base_path(), 75 | [], 76 | [{"Authorization", "Bearer " ++ binary_to_list(Token1)}], 77 | [{ssl, [{cacertfile, autocluster_config:get(k8s_cert_path)}]}]). 78 | 79 | %% @spec node_name(k8s_endpoint) -> list() 80 | %% @doc Return a full rabbit node name, appending hostname suffix 81 | %% @end 82 | %% 83 | node_name(Address) -> 84 | autocluster_util:node_name( 85 | autocluster_util:as_string(Address) ++ autocluster_config:get(k8s_hostname_suffix)). 86 | 87 | 88 | %% @spec maybe_ready_address(k8s_subsets()) -> list() 89 | %% @doc Return a list of ready nodes 90 | %% SubSet can contain also "notReadyAddresses" 91 | %% @end 92 | %% 93 | maybe_ready_address(Subset) -> 94 | case proplists:get_value(<<"notReadyAddresses">>, Subset) of 95 | undefined -> ok; 96 | NotReadyAddresses -> 97 | Formatted = string:join([binary_to_list(get_address(X)) 98 | || {struct, X} <- NotReadyAddresses], ", "), 99 | autocluster_log:info("k8s endpoint listing returned nodes not yet ready: ~s", 100 | [Formatted]) 101 | end, 102 | case proplists:get_value(<<"addresses">>, Subset) of 103 | undefined -> []; 104 | Address -> Address 105 | end. 106 | 107 | %% @doc Return a list of nodes 108 | %% see http://kubernetes.io/docs/api-reference/v1/definitions/#_v1_endpoints 109 | %% @end 110 | %% 111 | -spec extract_node_list({struct, term()}) -> [binary()]. 112 | extract_node_list({struct, Response}) -> 113 | IpLists = [[get_address(Address) 114 | || {struct, Address} <- maybe_ready_address(Subset)] 115 | || {struct, Subset} <- proplists:get_value(<<"subsets">>, Response)], 116 | sets:to_list(sets:union(lists:map(fun sets:from_list/1, IpLists))). 117 | 118 | 119 | %% @doc Return a list of path segments that are the base path for k8s key actions 120 | %% @end 121 | %% 122 | -spec base_path() -> [autocluster_httpc:path_component()]. 123 | base_path() -> 124 | {ok, NameSpace} = file:read_file( 125 | autocluster_config:get(k8s_namespace_path)), 126 | NameSpace1 = binary:replace(NameSpace, <<"\n">>, <<>>), 127 | [api, v1, namespaces, NameSpace1, endpoints, 128 | autocluster_config:get(k8s_service_name)]. 129 | 130 | get_address(Address) -> 131 | proplists:get_value(list_to_binary(autocluster_config:get(k8s_address_type)), Address). 132 | -------------------------------------------------------------------------------- /src/autocluster_log.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% @author Gavin M. Roy 3 | %% @copyright 2015-2016 AWeber Communications 4 | %% @end 5 | %%============================================================================== 6 | -module(autocluster_log). 7 | 8 | %% API 9 | -export([set_level/1, 10 | debug/1, debug/2, 11 | info/1, info/2, 12 | error/1, error/2, 13 | warning/1, warning/2]). 14 | 15 | %% Export all for unit tests 16 | -ifdef(TEST). 17 | -compile(export_all). 18 | -endif. 19 | 20 | 21 | %%-------------------------------------------------------------------- 22 | %% @doc 23 | %% Log a debug message 24 | %% @end 25 | %%-------------------------------------------------------------------- 26 | -spec debug(Message :: string()) -> ok. 27 | debug(Message) 28 | -> log(debug, Message, []). 29 | 30 | 31 | %%-------------------------------------------------------------------- 32 | %% @doc 33 | %% Log a debug message with arguments 34 | %% @end 35 | %%-------------------------------------------------------------------- 36 | -spec debug(Message :: string(), Args :: list()) -> ok. 37 | debug(Message, Args) 38 | -> log(debug, Message, Args). 39 | 40 | 41 | %%-------------------------------------------------------------------- 42 | %% @doc 43 | %% Log an informational message 44 | %% @end 45 | %%-------------------------------------------------------------------- 46 | -spec info(Message :: string()) -> ok. 47 | info(Message) 48 | -> log(info, Message, []). 49 | 50 | 51 | %%-------------------------------------------------------------------- 52 | %% @doc 53 | %% Log an informational message with arguments 54 | %% @end 55 | %%-------------------------------------------------------------------- 56 | -spec info(Message :: string(), Args :: list()) -> ok. 57 | info(Message, Args) 58 | -> log(info, Message, Args). 59 | 60 | 61 | %%-------------------------------------------------------------------- 62 | %% @doc 63 | %% Log an error message 64 | %% @end 65 | %%-------------------------------------------------------------------- 66 | -spec error(Message :: string()) -> ok. 67 | error(Message) 68 | -> log(error, Message, []). 69 | 70 | 71 | %%-------------------------------------------------------------------- 72 | %% @doc 73 | %% Log an informational message with arguments 74 | %% @end 75 | %%-------------------------------------------------------------------- 76 | -spec error(Message :: string(), Args :: list()) -> ok. 77 | error(Message, Args) 78 | -> log(error, Message, Args). 79 | 80 | 81 | %%-------------------------------------------------------------------- 82 | %% @doc 83 | %% Log a warning message 84 | %% @end 85 | %%-------------------------------------------------------------------- 86 | -spec warning(Message :: string()) -> ok. 87 | warning(Message) 88 | -> log(error, Message, []). 89 | 90 | 91 | %%-------------------------------------------------------------------- 92 | %% @doc 93 | %% Log a warning message with arguments 94 | %% @end 95 | %%-------------------------------------------------------------------- 96 | -spec warning(Message :: string(), Args :: list()) -> ok. 97 | warning(Message, Args) 98 | -> log(error, Message, Args). 99 | 100 | 101 | %%-------------------------------------------------------------------- 102 | %% @doc 103 | %% Set the log level for the autocluster logger 104 | %% @end 105 | %%-------------------------------------------------------------------- 106 | -spec set_level(atom()) -> ok. 107 | set_level(Level) -> 108 | case application:get_env(rabbit, log_levels) of 109 | undefined -> 110 | set_log_levels([{connection, info}, {autocluster, Level}]); 111 | {ok, Levels} -> 112 | case proplists:get_value(autocluster, Levels) of 113 | undefined -> 114 | set_log_levels(lists:append(Levels, [{autocluster, Level}])); 115 | Level -> 116 | ok; 117 | _Current -> 118 | Temp = proplists:delete(autocluster, Levels), 119 | set_log_levels(lists:append(Temp, [{autocluster, Level}])) 120 | end 121 | end. 122 | 123 | 124 | %%-------------------------------------------------------------------- 125 | %% @private 126 | %% @doc 127 | %% Ensure all logged lines share the same format 128 | %% @end 129 | %%-------------------------------------------------------------------- 130 | -spec log(Level :: atom(), Message :: string(), Args :: list()) -> ok. 131 | log(Level, Message, Args) -> 132 | rabbit_log:log(autocluster, Level, 133 | string:join(["autocluster: ", Message, "~n"], ""), 134 | Args). 135 | 136 | 137 | %%-------------------------------------------------------------------- 138 | %% @private 139 | %% @doc 140 | %% Override the RabbitMQ configured logging levels 141 | %% @end 142 | %%-------------------------------------------------------------------- 143 | -spec set_log_levels(list()) -> ok. 144 | set_log_levels(Levels) -> 145 | application:set_env(rabbit, log_levels, Levels), 146 | debug("log level set to ~s", 147 | [proplists:get_value(autocluster, Levels)]), 148 | ok. 149 | -------------------------------------------------------------------------------- /src/autocluster_periodic.erl: -------------------------------------------------------------------------------- 1 | -module(autocluster_periodic). 2 | 3 | %% @doc 4 | %% Manages periodic activities in a centralized fashion. TRefs from 5 | %% {@link timer:apply_interval/4} are stored to ETS along with the 6 | %% provided identifier, so you can later stop the activity using only 7 | %% that identifier (withouth knowing timer ref). 8 | %% 9 | %% Initial plan was to go even further, and make this into supervisor 10 | %% with a bunch of dynamic gen_server childs - to improve visibility 11 | %% of this periodic processes through tools like {@link //observer}. 12 | %% But as everything happens during broker startup, we don't have any 13 | %% stably running application at that time. 14 | %% 15 | %% @end 16 | 17 | %% API 18 | -export([start_immediate/3 19 | ,start_delayed/3 20 | ,stop/1 21 | ,stop_all/0 22 | ]). 23 | 24 | -define(TABLE, ?MODULE). 25 | 26 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 27 | %% API 28 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 29 | 30 | %% @doc 31 | %% Adds periodic activity with a given id. MFA is invoked immediately 32 | %% after addition. 33 | %% @end 34 | -spec start_immediate(term(), pos_integer(), {module(), atom(), term()}) -> ok. 35 | start_immediate(Id, Interval, {M, F, A} = MFA) -> 36 | _ = (catch apply(M, F, A)), 37 | start_delayed(Id, Interval, MFA). 38 | 39 | %% @doc 40 | %% Adds periodic activity with a given id. MFA is invoked only after 41 | %% given interval of time has been passed. 42 | %% @end 43 | -spec start_delayed(term(), pos_integer(), {module(), atom(), term()}) -> ok. 44 | start_delayed(Id, Interval, {M, F, A}) -> 45 | ensure_ets_table(), 46 | {ok, TRef} = timer:apply_interval(Interval, M, F, A), 47 | true = ets:insert_new(?TABLE, {Id, TRef}), 48 | ok. 49 | 50 | %% @doc 51 | %% Stops periodic activity previously added with {@link 52 | %% start_immediate/3} or {@link start_delayed/3}. 53 | %% @end 54 | -spec stop(term()) -> ok | {error, atom()}. 55 | stop(Id) -> 56 | case ets:lookup(?TABLE, Id) of 57 | [{Id, TRef}] -> 58 | _ = timer:cancel(TRef), 59 | ets:delete(?TABLE, Id); 60 | [] -> 61 | {error, not_running} 62 | end. 63 | 64 | %% @doc 65 | %% Stops all periodic activities registered using this module. Useful 66 | %% only during testing. 67 | %% @end 68 | -spec stop_all() -> ok. 69 | stop_all() -> 70 | ensure_ets_table(), 71 | Timers = ets:match(?TABLE, '$1'), 72 | lists:foreach(fun([{_Id, TRef}]) -> 73 | timer:cancel(TRef) 74 | end, 75 | Timers), 76 | ets:delete_all_objects(?TABLE), 77 | ok. 78 | 79 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 80 | %% Helpers 81 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 82 | ensure_ets_table() -> 83 | case ets:info(?TABLE) of 84 | undefined -> 85 | _ = ets:new(?TABLE, [public, named_table]), 86 | ok; 87 | _ -> 88 | ok 89 | end, 90 | ok. 91 | -------------------------------------------------------------------------------- /src/autocluster_sup.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% @author Gavin M. Roy 3 | %% @copyright 2016 AWeber Communications 4 | %% @end 5 | %%============================================================================== 6 | -module(autocluster_sup). 7 | 8 | -behaviour(supervisor). 9 | 10 | %% API 11 | -export([start_link/0]). 12 | 13 | %% Supervisor callbacks 14 | -export([init/1]). 15 | 16 | -define(SERVER, ?MODULE). 17 | 18 | %%%=================================================================== 19 | %%% API functions 20 | %%%=================================================================== 21 | 22 | %%-------------------------------------------------------------------- 23 | %% @doc 24 | %% Conditionally start the supervisor based upon the cluster_cleanup 25 | %% flag. This is only time that the supervisor needs to be running. 26 | %% 27 | %% @end 28 | %%-------------------------------------------------------------------- 29 | -spec(start_link() -> 30 | {ok, Pid :: pid()} | ignore | {error, Reason :: term()}). 31 | start_link() -> 32 | supervisor:start_link({local, ?SERVER}, ?MODULE, []). 33 | 34 | %%%=================================================================== 35 | %%% Supervisor callbacks 36 | %%%=================================================================== 37 | 38 | %%-------------------------------------------------------------------- 39 | %% @private 40 | %% @doc 41 | %% Whenever a supervisor is started using supervisor:start_link/[2,3], 42 | %% this function is called by the new process to find out about 43 | %% restart strategy, maximum restart frequency and child 44 | %% specifications. 45 | %% 46 | %% @end 47 | %%-------------------------------------------------------------------- 48 | -spec init(Args :: term()) -> 49 | {ok, {{RestartStrategy :: supervisor:strategy(), 50 | MaxR :: non_neg_integer(), 51 | MaxT :: pos_integer()}, 52 | [ChildSpec :: supervisor:child_spec()]}}. 53 | init([]) -> 54 | Children = case autocluster_config:get(cluster_cleanup) of 55 | true -> 56 | [{autocluster_cleanup, {autocluster_cleanup, start_link, []}, 57 | permanent, 10000, worker, 58 | [autocluster_cleanup]}]; 59 | _ -> [] 60 | end, 61 | {ok, {{one_for_one, 3, 10}, Children}}. 62 | -------------------------------------------------------------------------------- /src/autocluster_util.erl: -------------------------------------------------------------------------------- 1 | %%============================================================================== 2 | %% @author Gavin M. Roy 3 | %% @copyright 2015-2016 AWeber Communications 4 | %% @end 5 | %%============================================================================== 6 | -module(autocluster_util). 7 | 8 | %% API 9 | -export([as_atom/1, 10 | as_integer/1, 11 | as_string/1, 12 | as_list/1, 13 | backend_module/0, 14 | nic_ipv4/1, 15 | node_hostname/1, 16 | node_name/1, 17 | parse_port/1, 18 | augment_nodelist/1, 19 | stringify_error/1, 20 | as_proplist/1 21 | ]). 22 | 23 | -include("autocluster.hrl"). 24 | 25 | %% Private exports for RPC 26 | -export([augmented_node_info/0]). 27 | 28 | %% Export all for unit tests 29 | -ifdef(TEST). 30 | -compile(export_all). 31 | -endif. 32 | 33 | -type ifopt() :: {flag,[atom()]} | {addr, inet:ip_address()} | 34 | {netmask,inet:ip_address()} | {broadaddr,inet:ip_address()} | 35 | {dstaddr,inet:ip_address()} | {hwaddr,[byte()]}. 36 | 37 | -type stringifyable() :: atom() | binary() | string() | integer(). 38 | -export_type([stringifyable/0]). 39 | 40 | %%-------------------------------------------------------------------- 41 | %% @doc 42 | %% Return the passed in value as an atom. 43 | %% @end 44 | %%-------------------------------------------------------------------- 45 | -spec as_atom(atom() | binary() | string()) -> atom(). 46 | as_atom(Value) when is_atom(Value) -> 47 | Value; 48 | as_atom(Value) when is_binary(Value) -> 49 | list_to_atom(binary_to_list(Value)); 50 | as_atom(Value) when is_list(Value) -> 51 | list_to_atom(Value); 52 | as_atom(Value) -> 53 | autocluster_log:error("Unexpected data type for atom value: ~p~n", 54 | [Value]), 55 | Value. 56 | 57 | 58 | %%-------------------------------------------------------------------- 59 | %% @doc 60 | %% Return the passed in value as an integer. 61 | %% @end 62 | %%-------------------------------------------------------------------- 63 | -spec as_integer(binary() | integer() | string()) -> integer(). 64 | as_integer([]) -> undefined; 65 | as_integer(Value) when is_binary(Value) -> 66 | list_to_integer(as_string(Value)); 67 | as_integer(Value) when is_list(Value) -> 68 | list_to_integer(Value); 69 | as_integer(Value) when is_integer(Value) -> 70 | Value; 71 | as_integer(Value) -> 72 | autocluster_log:error("Unexpected data type for integer value: ~p~n", 73 | [Value]), 74 | Value. 75 | 76 | 77 | %%-------------------------------------------------------------------- 78 | %% @doc 79 | %% Return the passed in value as a string. 80 | %% @end 81 | %%-------------------------------------------------------------------- 82 | -spec as_string(Value :: stringifyable()) 83 | -> string(). 84 | as_string([]) -> ""; 85 | as_string(Value) when is_atom(Value) -> 86 | as_string(atom_to_list(Value)); 87 | as_string(Value) when is_binary(Value) -> 88 | as_string(binary_to_list(Value)); 89 | as_string(Value) when is_integer(Value) -> 90 | as_string(integer_to_list(Value)); 91 | as_string(Value) when is_list(Value) -> 92 | lists:flatten(Value); 93 | as_string(Value) -> 94 | autocluster_log:error("Unexpected data type for list value: ~p~n", 95 | [Value]), 96 | Value. 97 | 98 | 99 | %%-------------------------------------------------------------------- 100 | %% @doc 101 | %% Return the passed in value as a list of strings. 102 | %% @end 103 | %%-------------------------------------------------------------------- 104 | -spec as_list(Value :: stringifyable() | list()) -> list(). 105 | as_list([]) -> []; 106 | as_list(Value) when is_atom(Value) ; is_integer(Value) ; is_binary(Value) -> 107 | [Value]; 108 | as_list(Value) when is_list(Value) -> 109 | case io_lib:printable_list(Value) or io_lib:printable_unicode_list(Value) of 110 | true -> [case string:to_float(S) of 111 | {Float, []} -> Float; 112 | _ -> case string:to_integer(S) of 113 | {Integer, []} -> Integer; 114 | _ -> string:strip(S) 115 | end 116 | end || S <- string:tokens(Value, ",")]; 117 | false -> Value 118 | end; 119 | as_list(Value) -> 120 | autocluster_log:error("Unexpected data type for list value: ~p~n", 121 | [Value]), 122 | Value. 123 | 124 | 125 | %%-------------------------------------------------------------------- 126 | %% @doc 127 | %% Return the module to use for node discovery. 128 | %% @end 129 | %%-------------------------------------------------------------------- 130 | -spec backend_module() -> module() | undefined. 131 | backend_module() -> 132 | backend_module(autocluster_config:get(backend)). 133 | 134 | 135 | %%-------------------------------------------------------------------- 136 | %% @private 137 | %% @doc 138 | %% Return the module to use for node discovery. 139 | %% @end 140 | %%-------------------------------------------------------------------- 141 | -spec backend_module(atom()) -> module() | undefined. 142 | backend_module(aws) -> autocluster_aws; 143 | backend_module(consul) -> autocluster_consul; 144 | backend_module(dns) -> autocluster_dns; 145 | backend_module(etcd) -> autocluster_etcd; 146 | backend_module(k8s) -> autocluster_k8s; 147 | backend_module(_) -> undefined. 148 | 149 | 150 | %%-------------------------------------------------------------------- 151 | %% @doc 152 | %% Return the IP address for the current node for the specified 153 | %% network interface controller. 154 | %% @end 155 | %%-------------------------------------------------------------------- 156 | -spec nic_ipv4(Device :: string()) 157 | -> {ok, string()} | {error, not_found}. 158 | nic_ipv4(Device) -> 159 | {ok, Interfaces} = inet:getifaddrs(), 160 | nic_ipv4(Device, Interfaces). 161 | 162 | 163 | %%-------------------------------------------------------------------- 164 | %% @doc 165 | %% Parse the interface out of the list of interfaces, returning the 166 | %% IPv4 address if found. 167 | %% @end 168 | %%-------------------------------------------------------------------- 169 | -spec nic_ipv4(Device :: string(), Interfaces :: list()) 170 | -> {ok, string()} | {error, not_found}. 171 | nic_ipv4(_, []) -> {error, not_found}; 172 | nic_ipv4(Device, [{Interface, Opts}|_]) when Interface =:= Device -> 173 | {ok, nic_ipv4_address(Opts)}; 174 | nic_ipv4(Device, [_|T]) -> 175 | nic_ipv4(Device,T). 176 | 177 | 178 | %%-------------------------------------------------------------------- 179 | %% @doc 180 | %% Return the formatted IPv4 address out of the list of addresses 181 | %% for the interface. 182 | %% @end 183 | %%-------------------------------------------------------------------- 184 | -spec nic_ipv4_address([ifopt()]) -> {ok, string()} | {error, not_found}. 185 | nic_ipv4_address([]) -> {error, not_found}; 186 | nic_ipv4_address([{addr, {A,B,C,D}}|_]) -> 187 | inet_parse:ntoa({A,B,C,D}); 188 | nic_ipv4_address([_|T]) -> 189 | nic_ipv4_address(T). 190 | 191 | 192 | %%-------------------------------------------------------------------- 193 | %% @doc 194 | %% Return the hostname for the current node (without the tuple) 195 | %% Hostname can be taken from network info or nodename. 196 | %% @end 197 | %%-------------------------------------------------------------------- 198 | -spec node_hostname(boolean()) -> string(). 199 | node_hostname(false = _FromNodename) -> 200 | {ok, Hostname} = inet:gethostname(), 201 | Hostname; 202 | node_hostname(true = _FromNodename) -> 203 | {match, [Hostname]} = re:run(atom_to_list(node()), "@(.*)", 204 | [{capture, [1], list}]), 205 | Hostname. 206 | 207 | %%-------------------------------------------------------------------- 208 | %% @doc 209 | %% Return the proper node name for clustering purposes 210 | %% @end 211 | %%-------------------------------------------------------------------- 212 | -spec node_name(Value :: atom() | binary() | string()) -> atom(). 213 | node_name(Value) when is_atom(Value) -> 214 | node_name(atom_to_list(Value)); 215 | node_name(Value) when is_binary(Value) -> 216 | node_name(binary_to_list(Value)); 217 | node_name(Value) when is_list(Value) -> 218 | case lists:member($@, Value) of 219 | true -> 220 | list_to_atom(Value); 221 | false -> 222 | list_to_atom(string:join([node_prefix(), 223 | node_name_parse(as_string(Value))], 224 | "@")) 225 | end. 226 | 227 | %%-------------------------------------------------------------------- 228 | %% @doc 229 | %% Parse the value passed into nodename, checking if it's an ip 230 | %% address. If so, return it. If not, then check to see if longname 231 | %% support is turned on. if so, return it. If not, extract the left 232 | %% most part of the name, delimited by periods. 233 | %% @end 234 | %%-------------------------------------------------------------------- 235 | -spec node_name_parse(Value :: string()) -> string(). 236 | node_name_parse(Value) -> 237 | case inet:parse_ipv4strict_address(Value) of 238 | {ok, _} -> 239 | Value; 240 | {error, einval} -> 241 | node_name_parse(autocluster_config:get(longname), Value) 242 | end. 243 | 244 | 245 | %%-------------------------------------------------------------------- 246 | %% @doc 247 | %% Continue the parsing logic from node_name_parse/1. This is where 248 | %% result of the IPv4 check is processed. 249 | %% @end 250 | %%-------------------------------------------------------------------- 251 | -spec node_name_parse(IsIPv4 :: true | false, Value :: string()) 252 | -> string(). 253 | node_name_parse(true, Value) -> Value; 254 | node_name_parse(false, Value) -> 255 | Parts = string:tokens(Value, "."), 256 | node_name_parse(length(Parts), Value, Parts). 257 | 258 | 259 | %%-------------------------------------------------------------------- 260 | %% @doc 261 | %% Properly deal with returning the hostname if it's made up of 262 | %% multiple segments like www.rabbitmq.com, returning www, or if it's 263 | %% only a single segment, return that. 264 | %% @end 265 | %%-------------------------------------------------------------------- 266 | -spec node_name_parse(Segments :: integer(), 267 | Value :: string(), 268 | Parts :: [string()]) 269 | -> string(). 270 | node_name_parse(1, Value, _) -> Value; 271 | node_name_parse(_, _, Parts) -> 272 | as_string(lists:nth(1, Parts)). 273 | 274 | 275 | %%-------------------------------------------------------------------- 276 | %% @doc 277 | %% Extract the "local part" of the ``RABBITMQ_NODENAME`` environment 278 | %% variable, if set, otherwise use the default node name value 279 | %% (rabbit). 280 | %% @end 281 | %%-------------------------------------------------------------------- 282 | -spec node_prefix() -> string(). 283 | node_prefix() -> 284 | Value = autocluster_config:get(node_name), 285 | lists:nth(1, string:tokens(Value, "@")). 286 | 287 | 288 | %%-------------------------------------------------------------------- 289 | %% @doc 290 | %% Returns the port, even if Docker linking overwrites a configuration 291 | %% value to be a URI instead of numeric value 292 | %% @end 293 | %%-------------------------------------------------------------------- 294 | -spec parse_port(Value :: integer() | string()) -> integer(). 295 | parse_port(Value) when is_list(Value) -> 296 | as_integer(lists:last(string:tokens(Value, ":"))); 297 | parse_port(Value) -> as_integer(Value). 298 | 299 | %%-------------------------------------------------------------------- 300 | %% @doc 301 | %% Filters dead nodes from node list, augments it with additional 302 | %% information - what other nodes it is clustered with, uptime and all 303 | %% other pieces of information necessary to choose the best node to 304 | %% join to. 305 | %% @end 306 | %%-------------------------------------------------------------------- 307 | -spec augment_nodelist([node()]) -> [#candidate_seed_node{}]. 308 | augment_nodelist(Nodes) -> 309 | %% TODO: this should not be hardcoded 310 | {Responses, UnreachableNodes} = rpc:multicall(Nodes, autocluster_util, augmented_node_info, [], 5000), 311 | autocluster_log:debug("Fetching node details. Unreachable nodes (or nodes that responded with an error): ~p", [UnreachableNodes]), 312 | autocluster_log:debug("Fetching node details. Responses: ~p", [Responses]), 313 | [A || A = #candidate_seed_node{} <- Responses]. 314 | 315 | %%-------------------------------------------------------------------- 316 | %% @doc 317 | %% Collects information for clustering decisions on the current 318 | %% node. Only called using RPC from augment_nodelist/1. 319 | %% @end 320 | %%-------------------------------------------------------------------- 321 | -spec augmented_node_info() -> #candidate_seed_node{}. 322 | augmented_node_info() -> 323 | Running = rabbit_mnesia:cluster_nodes(running), 324 | Partitioned = rabbit_node_monitor:partitions(), 325 | #candidate_seed_node{ 326 | name = node(), 327 | uptime = element(1, erlang:statistics(wall_clock)), 328 | alive = true, 329 | clustered_with = rabbit_mnesia:cluster_nodes(all), 330 | alive_cluster_nodes = Running -- Partitioned, 331 | partitioned_cluster_nodes = Partitioned, 332 | other_cluster_nodes = [] 333 | }. 334 | 335 | -spec stringify_error({ok, term()} | {error, term()}) -> {ok, term()} | {error, string()}. 336 | stringify_error({ok, _} = Res) -> 337 | Res; 338 | stringify_error({error, Str}) when is_list(Str) -> 339 | {error, Str}; 340 | stringify_error({error, Term}) -> 341 | {error, lists:flatten(io_lib:format("~p", [Term]))}. 342 | 343 | 344 | %%-------------------------------------------------------------------- 345 | %% @doc 346 | %% Returns a proplist from a JSON structure (environment variable) or 347 | %% the settings key (already a proplist). 348 | %% @end 349 | %%-------------------------------------------------------------------- 350 | -spec as_proplist(Value :: string() | [{string(), string()}]) -> 351 | [{string(), string()}]. 352 | as_proplist([Tuple | _] = Json) when is_tuple(Tuple) -> 353 | Json; 354 | as_proplist([]) -> 355 | []; 356 | as_proplist(List) when is_list(List) -> 357 | case rabbit_misc:json_decode(List) of 358 | {ok, {struct, _} = Json} -> 359 | [{binary_to_list(K), binary_to_list(V)} 360 | || {K, V} <- rabbit_misc:json_to_term(Json)]; 361 | _ -> 362 | autocluster_log:error("Unexpected data type for proplist value: ~p. JSON parser returned an error!~n", 363 | [List]), 364 | [] 365 | end; 366 | as_proplist(Value) -> 367 | autocluster_log:error("Unexpected data type for proplist value: ~p.~n", 368 | [Value]), 369 | []. 370 | -------------------------------------------------------------------------------- /test/etcd_SUITE.erl: -------------------------------------------------------------------------------- 1 | %% Integration tests for etcd-related code 2 | -module(etcd_SUITE). 3 | 4 | -include_lib("common_test/include/ct.hrl"). 5 | 6 | %% CT callbacks 7 | -export([all/0 8 | ,init_per_suite/1 9 | ,end_per_suite/1 10 | ,init_per_testcase/2 11 | ,end_per_testcase/2 12 | ]). 13 | 14 | %% Tests 15 | -export([simple_lock_unlock_sequence/1 16 | ,lock_survives_longer_than_its_ttl/1 17 | ,breaking_the_lock_causes_unlock_failure/1 18 | ,prop_etcd_locking_works_fine/1 19 | ]). 20 | 21 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 22 | %% CT callbacks 23 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 24 | all() -> 25 | HasEtcd = autocluster_testing:has_etcd(), 26 | ProperOnly = HasEtcd andalso os:getenv("PROPER_ONLY") =:= "true", 27 | case ProperOnly of 28 | true -> 29 | [prop_etcd_locking_works_fine]; 30 | false -> 31 | autocluster_testing:optionals(HasEtcd, 32 | [simple_lock_unlock_sequence 33 | ,lock_survives_longer_than_its_ttl 34 | ,breaking_the_lock_causes_unlock_failure 35 | ]) 36 | end. 37 | 38 | init_per_suite(Config) -> 39 | application:ensure_all_started(inets), 40 | autocluster_log:set_level(debug), 41 | {ok, Process, PortNumber} = autocluster_testing:start_etcd(?config(priv_dir, Config)), 42 | [{etcd_process, Process}, {etcd_port_num, PortNumber} | Config]. 43 | 44 | end_per_suite(Config) -> 45 | autocluster_testing:stop_etcd(?config(etcd_process, Config)), 46 | Config. 47 | 48 | init_per_testcase(_, Config) -> 49 | autocluster_testing:reset(), 50 | os:putenv("ETCD_PORT", integer_to_list(?config(etcd_port_num, Config))), 51 | autocluster_etcd:etcd_delete(autocluster_etcd:startup_lock_path(), []), 52 | autocluster_periodic:stop_all(), 53 | rabbit_ct_helpers:start_long_running_testsuite_monitor(Config). 54 | 55 | end_per_testcase(_, Config0) -> 56 | Config1 = rabbit_ct_helpers:stop_long_running_testsuite_monitor(Config0), 57 | autocluster_periodic:stop_all(), 58 | Config1. 59 | 60 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 61 | %% Test cases 62 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 63 | simple_lock_unlock_sequence(_Config) -> 64 | {ok, Data} = autocluster_etcd:lock("ct-test-lock"), 65 | ok = autocluster_etcd:unlock(Data), 66 | ok. 67 | 68 | lock_survives_longer_than_its_ttl(_Config) -> 69 | os:putenv("ETCD_NODE_TTL", "2"), 70 | {ok, Data} = autocluster_etcd:lock("ct-test-lock"), 71 | timer:sleep(3000), 72 | ok = autocluster_etcd:unlock(Data), 73 | ok. 74 | 75 | prop_etcd_locking_works_fine(_Config) -> 76 | rabbit_ct_proper_helpers:run_proper(fun etcd_lock_statem:prop_etcd_locking_works_fine/0, [], 30). 77 | 78 | breaking_the_lock_causes_unlock_failure(_Config) -> 79 | {ok, Data} = autocluster_etcd:lock("ct-test-lock"), 80 | autocluster_etcd:etcd_delete(autocluster_etcd:startup_lock_path(), []), 81 | {error, _} = autocluster_etcd:unlock(Data), 82 | ok. 83 | 84 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 85 | %% Helpers 86 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 87 | -------------------------------------------------------------------------------- /test/etcd_lock_statem.erl: -------------------------------------------------------------------------------- 1 | -module(etcd_lock_statem). 2 | 3 | -behaviour(proper_statem). 4 | -behaviour(gen_server). 5 | -compile([export_all]). 6 | 7 | 8 | -include_lib("proper/include/proper.hrl"). 9 | 10 | -define(SERVER, ?MODULE). 11 | 12 | %% properties for testing 13 | -export([prop_etcd_locking_works_fine/0]). 14 | 15 | %% proper_statem callbacks 16 | -export([command/1 17 | ,initial_state/0 18 | ,next_state/3 19 | ,precondition/2 20 | ,postcondition/3 21 | ]). 22 | 23 | %% commands that proper_statem will invoke 24 | -export([break_current_lock_command/0 25 | ,release_current_lock_command/0 26 | ,release_broken_lock_command/0 27 | ,start_new_contender_command/0 28 | ]). 29 | 30 | %% gen_server callbacks 31 | -export([init/1 32 | ,handle_call/3 33 | ,handle_cast/2 34 | ,handle_info/2 35 | ,terminate/2 36 | ,code_change/3 37 | ]). 38 | 39 | -define(MAX_CONTENDERS, 5). 40 | 41 | -type contender_id() :: string(). 42 | 43 | -record(state, {locked :: boolean() 44 | ,current_lock_holder :: undefined | contender_id() 45 | ,contender_count :: non_neg_integer() 46 | ,broken_lock_count :: non_neg_integer 47 | }). 48 | 49 | -record(server_state, {name_counter = 1 50 | ,lock_data = #{} 51 | ,current_lock_holder = undefined :: undefined | contender_id() 52 | ,current_lock_pid :: undefined | pid() 53 | ,current_lock_data :: undefined | {data, term()} 54 | ,broken_locks = #{} 55 | ,contenders = #{}}). 56 | 57 | 58 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 59 | %% Test cases 60 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 61 | prop_etcd_locking_works_fine() -> 62 | ?FORALL(Cmds, commands(?MODULE), 63 | ?TRAPEXIT( 64 | begin 65 | ct:log("~n================================================================================~nNew run", []), 66 | {ok, Pid} = gen_server:start_link({local, ?MODULE}, ?MODULE, [], []), 67 | {History, State, Result} = run_commands(?MODULE, Cmds), 68 | DynamicState = sys:get_state(Pid), 69 | gen_server:call(Pid, stop), 70 | ?WHENFAIL(ct:pal("Result: ~w~nState:~n~sDynamic:~n~s~s", 71 | [Result, dump_state(State), dump_server_state(DynamicState), dump_history(History, Cmds)]), 72 | aggregate(command_names(Cmds), Result =:= ok)) 73 | end)). 74 | 75 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 76 | %% proper_statem callbacks 77 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 78 | initial_state() -> 79 | #state{locked = false 80 | ,current_lock_holder = undefined 81 | ,contender_count = 0 82 | ,broken_lock_count = 0}. 83 | 84 | command(_State) -> 85 | oneof([{call, ?MODULE, break_current_lock_command, []}, 86 | {call, ?MODULE, release_current_lock_command, []}, 87 | {call, ?MODULE, release_broken_lock_command, []}, 88 | {call, ?MODULE, start_new_contender_command, []}]). 89 | 90 | %%---------------------------------------------------------------------- 91 | 92 | %% - Can't break lock if nobody holds it 93 | %% - Can't break lock if there is too many pending broken locks 94 | precondition(#state{locked = false}, {call, _, break_current_lock_command, _}) -> 95 | false; 96 | precondition(#state{broken_lock_count = Cnt}, {call, _, break_current_lock_command, _}) 97 | when Cnt > ?MAX_CONTENDERS -> 98 | false; 99 | 100 | %% can't release lock if nobody holds it 101 | precondition(#state{locked = false}, {call, _, release_current_lock_command, _}) -> 102 | false; 103 | 104 | %% No broken locks, nothing to attempt to release 105 | precondition(#state{broken_lock_count = 0}, {call, _, release_broken_lock_command, _}) -> 106 | false; 107 | 108 | %% Don't request new lock when there are a lot of other contenders. 109 | precondition(#state{contender_count = Cnt}, {call, _, start_new_contender_command, _}) 110 | when Cnt > ?MAX_CONTENDERS -> 111 | false; 112 | 113 | precondition(_, _) -> 114 | true. 115 | 116 | %%---------------------------------------------------------------------- 117 | next_state(S0, Res, {call, _, start_new_contender_command, []}) -> 118 | S1 = inc_contender_count(S0), 119 | maybe_promote_contender(S1, Res); 120 | next_state(S0, Res, {call, _, release_current_lock_command, []}) -> 121 | S1 = S0#state{locked = false, 122 | current_lock_holder = undefined}, 123 | maybe_promote_contender(S1, Res); 124 | next_state(S, Res, {call, _, break_current_lock_command, []}) -> 125 | S1 = S#state{broken_lock_count = S#state.broken_lock_count + 1, 126 | locked = false, 127 | current_lock_holder = undefined}, 128 | maybe_promote_contender(S1, Res); 129 | next_state(S, _Res, {call, _, release_broken_lock_command, []}) -> 130 | dec_broken_count(S). 131 | 132 | %%---------------------------------------------------------------------- 133 | %% {call, _, start_new_contender_command, _} 134 | 135 | %% Releasing lock (properly or by forcefully breaking it) should allow 136 | %% next contender to take the lock (if there is any). 137 | postcondition(#state{contender_count = Cnt} = State, {call, _, Command, _} = Call, Result) 138 | when Command =:= break_current_lock_command; Command =:= release_current_lock_command -> 139 | assert_common_invariants(next_state(State, Result, Call)), 140 | case {Cnt, Result} of 141 | {0, undefined} -> 142 | true; 143 | {0, _NotUndefined} -> 144 | error({somebody_got_lock, Result}); 145 | {_, undefined} -> 146 | error(nobody_got_lock); 147 | {_, _} -> 148 | true 149 | end; 150 | 151 | %% Starting new contender when there is no others should result in immediate locking 152 | postcondition(#state{contender_count = 0} = State, {call, _, start_new_contender_command, _} = Call, Result) -> 153 | assert_common_invariants(next_state(State, Result, Call)), 154 | case Result of 155 | undefined -> 156 | error(single_contender_didnt_get_lock); 157 | _ -> 158 | true 159 | end; 160 | 161 | postcondition(State, Call, Result) -> 162 | assert_common_invariants(next_state(State, Result, Call)). 163 | 164 | assert_common_invariants(State) -> 165 | assert_current_lock_holder_is_correct(State), 166 | assert_contender_count_is_correct(State), 167 | assert_broken_lock_count_is_correct(State). 168 | 169 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 170 | %% Commands that proper_statem will invoke - delegate to our gen_server 171 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 172 | start_new_contender_command() -> 173 | gen_server:call(?SERVER, start_new_contender). 174 | 175 | break_current_lock_command() -> 176 | gen_server:call(?SERVER, break_current_lock). 177 | 178 | release_current_lock_command() -> 179 | gen_server:call(?SERVER, release_current_lock). 180 | 181 | release_broken_lock_command() -> 182 | gen_server:call(?SERVER, release_broken_lock). 183 | 184 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 185 | %% gen_server for tracking external entities 186 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 187 | init([]) -> 188 | autocluster_periodic:stop_all(), 189 | timer:sleep(1000), 190 | autocluster_etcd:etcd_delete(autocluster_etcd:base_path(), [{recursive, true}]), 191 | ct:log("Started dynamic state server from a clean slate"), 192 | State = #server_state{}, 193 | {ok, State}. 194 | 195 | handle_call(start_new_contender, _From, State) -> 196 | ct:log("Got start_new_contender call in state~n~s", [dump_server_state(State)]), 197 | wait_and_reply(start_new_contender(State)); 198 | handle_call(release_current_lock, _From, State) -> 199 | ct:log("Got release_current_lock call in state~n~s", [dump_server_state(State)]), 200 | wait_and_reply(release_current_lock(State)); 201 | handle_call(release_broken_lock, _From, State) -> 202 | ct:log("Got release_broken_lock call in state~n~s", [dump_server_state(State)]), 203 | wait_and_reply(release_broken_lock(State)); 204 | handle_call(break_current_lock, _From, State) -> 205 | ct:log("Got break_current_lock call in state~n~s", [dump_server_state(State)]), 206 | wait_and_reply(break_current_lock(State)); 207 | handle_call(get_lock_holder, _From, #server_state{current_lock_holder = Current} = State) -> 208 | {reply, Current, State}; 209 | handle_call(get_contender_count, _From, #server_state{contenders = Contenders} = State) -> 210 | {reply, maps:size(Contenders), State}; 211 | handle_call(get_broken_lock_count, _From, #server_state{broken_locks = Broken} = State) -> 212 | {reply, maps:size(Broken), State}; 213 | handle_call(stop, _From, State) -> 214 | {stop, normal, ok, State}; 215 | handle_call(_Request, _From, State) -> 216 | Reply = ok, 217 | {reply, Reply, State}. 218 | 219 | handle_cast(_Request, State) -> 220 | {noreply, State}. 221 | 222 | handle_info(_Info, State) -> 223 | {noreply, State}. 224 | 225 | terminate(_Reason, #server_state{contenders = Contenders, broken_locks = Broken, current_lock_pid = CurrentPid}) -> 226 | process_flag(trap_exit, true), 227 | [ exit(Pid, kill) || {_, Pid} <- maps:to_list(Contenders) ], 228 | [ exit(Pid, kill) || {_, Pid} <- maps:to_list(Broken) ], 229 | CurrentPid =/= undefined andalso exit(CurrentPid, kill), 230 | process_flag(trap_exit, false), 231 | ok. 232 | 233 | code_change(_OldVsn, State, _Extra) -> 234 | {ok, State}. 235 | 236 | 237 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 238 | %% gen_server implementation helpers 239 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 240 | contender_loop(Name, Parent) -> 241 | {ok, UniqueId} = autocluster_etcd:lock(Name), 242 | Parent ! {lock_acquired, Name, UniqueId}, 243 | receive 244 | unlock_normal -> 245 | ok = autocluster_etcd:unlock(UniqueId); 246 | unlock_broken -> 247 | {error, _} = autocluster_etcd:unlock(UniqueId) 248 | end, 249 | Parent ! {done, Name}, 250 | ok. 251 | 252 | start_new_contender(#server_state{name_counter = NameCtr} = State0) -> 253 | Name = integer_to_list(NameCtr), 254 | State1 = inc_name_counter(State0), 255 | Parent = self(), 256 | ContederPid = spawn_link(fun() -> contender_loop(Name, Parent) end), 257 | add_contender(Name, ContederPid, State1). 258 | 259 | release_current_lock(#server_state{current_lock_pid = CurrentPid} = State) -> 260 | CurrentPid ! unlock_normal, 261 | State. 262 | 263 | release_broken_lock(#server_state{broken_locks = Broken} = State) -> 264 | BrokenList = maps:to_list(Broken), 265 | {_ChosenName, ChosenPid} = lists:nth(rand_compat:uniform(length(BrokenList)), BrokenList), 266 | ChosenPid ! unlock_broken, 267 | State. 268 | 269 | break_current_lock(#server_state{broken_locks = Broken0, 270 | current_lock_holder = Current, 271 | current_lock_data = {data, UniqueId}, 272 | current_lock_pid = CurrentPid} = State) -> 273 | stop_ttl_lock_updater(UniqueId), 274 | delete_etcd_lock_key(UniqueId), 275 | Broken1 = maps:put(Current, CurrentPid, Broken0), 276 | State#server_state{broken_locks = Broken1, 277 | current_lock_holder = undefined, 278 | current_lock_data = undefined, 279 | current_lock_pid = undefined}. 280 | 281 | wait_for_notifies(#server_state{contenders = Contenders, current_lock_holder = Current, broken_locks = Broken} = State0) -> 282 | receive 283 | {lock_acquired, Name, UniqueId} -> 284 | ct:log("Contender ~p received lock: ~p", [Name, UniqueId]), 285 | State1 = State0#server_state{current_lock_holder = Name, 286 | current_lock_pid = maps:get(Name, Contenders), 287 | current_lock_data = {data, UniqueId}, 288 | contenders = maps:remove(Name, Contenders)}, 289 | wait_for_notifies(State1); 290 | {done, Current} -> % current lock is released 291 | ct:log("Process ~p released lock", [Current]), 292 | State1 = State0#server_state{current_lock_holder = undefined, 293 | current_lock_pid = undefined, 294 | current_lock_data = undefined}, 295 | wait_for_notifies(State1); 296 | {done, Name} -> % broken lock process finished successfully 297 | ct:log("Process ~p received correct error while releasing broken lock", [Name]), 298 | State1 = State0#server_state{broken_locks = maps:remove(Name, Broken)}, 299 | wait_for_notifies(State1) 300 | after 301 | 1900 -> %% Slightly more than sleep in autocluster_etcd:wait_for_lock_release/0 302 | State0 303 | end. 304 | 305 | inc_name_counter(#server_state{name_counter = Ctr} = State) -> 306 | State#server_state{name_counter = Ctr + 1}. 307 | 308 | add_contender(Name, Pid, #server_state{contenders = Contenders0} = State) -> 309 | Contenders1 = maps:put(Name, Pid, Contenders0), 310 | State#server_state{contenders = Contenders1}. 311 | 312 | wait_and_reply(State0) -> 313 | State1 = wait_for_notifies(State0), 314 | std_reply(State1). 315 | 316 | std_reply(#server_state{current_lock_holder = Holder} = State) -> 317 | ct:log("Ready to send reply ~p", [Holder]), 318 | {reply, Holder, State}. 319 | 320 | stop_ttl_lock_updater(UniqueId) -> 321 | autocluster_periodic:stop({autocluster_etcd_lock, UniqueId}). 322 | 323 | delete_etcd_lock_key(UniqueId) -> 324 | autocluster_etcd:delete_etcd_lock_key(UniqueId). 325 | 326 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 327 | %% proper_statem implementation helpers 328 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 329 | maybe_promote_contender(#state{locked = false, contender_count = Cnt} = S, NewHolder) 330 | when Cnt > 0 -> 331 | S#state{current_lock_holder = NewHolder, locked = true, contender_count = S#state.contender_count - 1}; 332 | maybe_promote_contender(S, _LockHolder) -> 333 | S. 334 | 335 | dec_broken_count(#state{broken_lock_count = Cnt} = S) -> 336 | S#state{broken_lock_count = Cnt - 1}. 337 | 338 | 339 | inc_contender_count(#state{contender_count = Cnt} = S) -> 340 | S#state{contender_count = Cnt + 1}. 341 | 342 | assert_current_lock_holder_is_correct(#state{current_lock_holder = ModelCurrent}) -> 343 | case gen_server:call(?MODULE, get_lock_holder) of 344 | ModelCurrent -> 345 | true; 346 | RealCurrent -> 347 | error({model_current_is, ModelCurrent, when_it_should_be, RealCurrent}) 348 | end. 349 | 350 | assert_contender_count_is_correct(#state{contender_count = ModelCount}) -> 351 | case gen_server:call(?MODULE, get_contender_count) of 352 | ModelCount -> 353 | true; 354 | RealCount -> 355 | error({model_contender_count_is, ModelCount, when_it_should_be, RealCount}) 356 | end. 357 | 358 | assert_broken_lock_count_is_correct(#state{broken_lock_count = ModelCount}) -> 359 | case gen_server:call(?MODULE, get_broken_lock_count) of 360 | ModelCount -> 361 | true; 362 | RealCount -> 363 | error({model_broken_lock_count_is, ModelCount, when_it_should_be, RealCount}) 364 | end. 365 | 366 | dump_server_state(ServerState) -> 367 | format_record(ServerState, record_info(fields, server_state)). 368 | 369 | dump_state(State) -> 370 | format_record(State, record_info(fields, state)). 371 | 372 | format_record(Record, Names) -> 373 | Values = tl(tuple_to_list(Record)), 374 | lists:flatten(lists:zipwith(fun (K, V) -> 375 | io_lib:format(" ~s=~p~n", [K, V]) 376 | end, 377 | Names, Values)). 378 | 379 | dump_history(History, Commands) -> 380 | lists:flatten(lists:zipwith3(fun(SeqNo, {State, Result}, {_, _, {_, _, Cmd, _}}) -> 381 | io_lib:format("State ~p:~n~sCommand ~p -> ~p~n~n", 382 | [SeqNo, dump_state(State), Cmd, Result]) 383 | end, 384 | lists:seq(1, length(History)), 385 | History, 386 | lists:sublist(Commands, length(History)))). 387 | -------------------------------------------------------------------------------- /test/health_check_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(health_check_SUITE). 2 | 3 | -include_lib("common_test/include/ct.hrl"). 4 | 5 | %% common_test exports 6 | -export([all/0 7 | ,groups/0 8 | ,init_per_suite/1 9 | ,end_per_suite/1 10 | ,init_per_testcase/2 11 | ,end_per_testcase/2 12 | ]). 13 | 14 | %% test cases 15 | -export([cluster_is_assembled/1 16 | ,nodes_contiune_startup_with_ignore_failure_mode/1 17 | ,startup_lock_released_after_startup_failure/1 18 | ]). 19 | 20 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 21 | %% Common test callbacks 22 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 23 | all() -> 24 | autocluster_testing:optional(autocluster_testing:has_etcd(), 25 | {group, etcd_group}). 26 | 27 | groups() -> 28 | autocluster_testing:optional(autocluster_testing:has_etcd(), 29 | {etcd_group, [], [cluster_is_assembled, 30 | nodes_contiune_startup_with_ignore_failure_mode, 31 | startup_lock_released_after_startup_failure]}). 32 | 33 | init_per_suite(Config0) -> 34 | rabbit_ct_helpers:log_environment(), 35 | rabbit_ct_helpers:run_setup_steps(Config0). 36 | 37 | end_per_suite(Config) -> 38 | rabbit_ct_helpers:run_teardown_steps(Config). 39 | 40 | init_per_testcase(cluster_is_assembled, Config) -> 41 | start_3_node_cluster_with_etcd(Config, cluster_is_assembled); 42 | init_per_testcase(nodes_contiune_startup_with_ignore_failure_mode, Config) -> 43 | os:putenv("ETCD_PORT", "1"), 44 | os:putenv("AUTOCLUSTER_FAILURE", "ignore"), 45 | start_3_node_cluster_with_etcd(Config, nodes_contiune_startup_with_ignore_failure_mode); 46 | init_per_testcase(startup_lock_released_after_startup_failure, Config) -> 47 | os:putenv("AUTOCLUSTER_FAILURE", "ignore"), 48 | start_3_node_cluster_with_etcd(Config, startup_lock_released_after_startup_failure); 49 | init_per_testcase(_, Config) -> 50 | Config. 51 | 52 | end_per_testcase(cluster_is_assembled, Config) -> 53 | stop_cluster_and_etcd(Config); 54 | end_per_testcase(nodes_contiune_startup_with_ignore_failure_mode, Config) -> 55 | autocluster_testing:reset(), %% Cleanup os environment variables 56 | stop_cluster_and_etcd(Config); 57 | end_per_testcase(startup_lock_released_after_startup_failure, Config) -> 58 | autocluster_testing:reset(), %% Cleanup os environment variables 59 | stop_cluster_and_etcd(Config); 60 | end_per_testcase(_, Config) -> 61 | Config. 62 | 63 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 64 | %% Test cases 65 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 66 | cluster_is_assembled(Config) -> 67 | ExpectedNodes = lists:sort(rabbit_ct_broker_helpers:get_node_configs(Config, nodename)), 68 | lists:foreach(fun (Node) -> 69 | case lists:sort(rabbit_ct_broker_helpers:rpc(Config, Node, rabbit_mnesia, cluster_nodes, [running])) of 70 | ExpectedNodes -> 71 | ok; 72 | GotNodes -> 73 | exit({nodes_from, Node, {got, GotNodes}, {expected, ExpectedNodes}}) 74 | end 75 | end, ExpectedNodes), 76 | ok. 77 | 78 | nodes_contiune_startup_with_ignore_failure_mode(Config) -> 79 | assert_nodes_not_clustered(Config), 80 | ok. 81 | 82 | %% `rabbit_mnesia:join_cluster/2` is broken, but `start_app` will 83 | %% succeed because of `ignore` failure mode. But we need to be 84 | %% sure that startup lock was relesead even when there was an 85 | %% error. 86 | startup_lock_released_after_startup_failure(Config) -> 87 | AllNodes = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), 88 | lists:foreach(fun (Node) -> 89 | RPC = fun (M, F, A) -> 90 | rabbit_ct_broker_helpers:rpc(Config, Node, M, F, A) 91 | end, 92 | Ctl = fun (Args) -> 93 | {ok, _} = rabbit_ct_broker_helpers:rabbitmqctl(Config, Node, Args) 94 | end, 95 | Ctl(["stop_app"]), 96 | RPC(application, ensure_all_started, [meck]), 97 | RPC(rabbit_mnesia, reset, []), 98 | RPC(meck, new, [rabbit_mnesia, [passthrough, no_link]]), 99 | RPC(meck, expect, [rabbit_mnesia, join_cluster, [{['_', '_'], {error, borken_for_tests}}]]), 100 | Ctl(["start_app"]) 101 | end, 102 | AllNodes), 103 | assert_nodes_not_clustered(Config), 104 | 105 | RPC = fun (M, F, A) -> 106 | rabbit_ct_broker_helpers:rpc(Config, 1, M, F, A) 107 | end, 108 | Path = RPC(autocluster_etcd, startup_lock_path, []), 109 | case RPC(autocluster_etcd, etcd_get, [Path, []]) of 110 | {error, "404"} -> 111 | ok; 112 | Res -> 113 | exit({startup_lock_is_still_here, Res}) 114 | end, 115 | ok. 116 | 117 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 118 | %% Helpers 119 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 120 | generate_erlang_node_config(Config) -> 121 | ErlangNodeConfig = [{rabbit, [{dummy_param_for_comma, true} 122 | ,{cluster_partition_handling, ignore} 123 | ]}, 124 | {autocluster, [{dummy_param_for_comma, true} 125 | ,{autocluster_log_level, debug} 126 | ,{backend, etcd} 127 | ,{autocluster_failure, stop} 128 | ,{cleanup_interval, 10} 129 | ,{cluster_cleanup, true} 130 | ,{cleanup_warn_only, false} 131 | ,{etcd_scheme, http} 132 | ,{etcd_host, "localhost"} 133 | ,{etcd_port, ?config(etcd_port_num, Config)} 134 | ]}], 135 | rabbit_ct_helpers:merge_app_env(Config, ErlangNodeConfig). 136 | 137 | 138 | start_3_node_cluster_with_etcd(Config0, TestCase) -> 139 | NodeNames = [ atom_to_list(TestCase) ++ "-" ++ integer_to_list(N) || N <- lists:seq(1, 3)], 140 | {ok, Process, PortNumber} = autocluster_testing:start_etcd(?config(priv_dir, Config0)), 141 | Config1 = [{rmq_nodes_count, NodeNames} 142 | ,{rmq_nodes_clustered, false} 143 | ,{broker_with_plugins, true} 144 | ,{etcd_process, Process} 145 | ,{etcd_port_num, PortNumber} 146 | | Config0], 147 | rabbit_ct_helpers:run_steps(Config1, 148 | [fun generate_erlang_node_config/1] 149 | ++ rabbit_ct_broker_helpers:setup_steps()). 150 | 151 | stop_cluster_and_etcd(Config) -> 152 | autocluster_testing:stop_etcd(?config(etcd_process, Config)), 153 | rabbit_ct_helpers:run_steps(Config, 154 | rabbit_ct_broker_helpers:teardown_steps()). 155 | 156 | assert_nodes_not_clustered(Config) -> 157 | AllNodes = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), 158 | lists:foreach(fun (Node) -> 159 | case rabbit_ct_broker_helpers:rpc(Config, Node, rabbit_mnesia, cluster_nodes, [running]) of 160 | [Node] -> 161 | ok; 162 | Smth -> 163 | exit({strange_report, Node, Smth}) 164 | end 165 | end, AllNodes), 166 | ok. 167 | -------------------------------------------------------------------------------- /test/periodic_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(periodic_SUITE). 2 | 3 | -include_lib("common_test/include/ct.hrl"). 4 | 5 | %% CT callbacks 6 | -export([all/0 7 | ,init_per_testcase/2 8 | ,end_per_testcase/2 9 | ]). 10 | 11 | %% Test cases 12 | -export([immediate_is_immediate/1 13 | ,delayed_is_delayed/1 14 | ,callback_repeated/1 15 | ,stop_is_working/1 16 | ,errors_are_ignored/1 17 | ,multiple_timers_supported/1 18 | ]). 19 | 20 | %% Private 21 | -export([periodic_report_back/1 22 | ,first_error_then_report_back/1]). 23 | 24 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 25 | %% CT callbacks 26 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 27 | all() -> 28 | [immediate_is_immediate 29 | ,delayed_is_delayed 30 | ,callback_repeated 31 | ,stop_is_working 32 | ,errors_are_ignored 33 | ,multiple_timers_supported 34 | ]. 35 | 36 | init_per_testcase(_C, Config) -> 37 | autocluster_periodic:stop_all(), 38 | flush_messages(), 39 | Config. 40 | 41 | end_per_testcase(_C, Config) -> 42 | autocluster_periodic:stop_all(), 43 | Config. 44 | 45 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 46 | %% Test cases 47 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 48 | immediate_is_immediate(_Config) -> 49 | Id = make_ref(), 50 | autocluster_periodic:start_immediate(Id, 100000, {?MODULE, periodic_report_back, [{self(), Id}]}), 51 | message_during(Id, 1000), 52 | ok. 53 | 54 | delayed_is_delayed(_Config) -> 55 | Id = make_ref(), 56 | autocluster_periodic:start_delayed(Id, 2000, {?MODULE, periodic_report_back, [{self(), Id}]}), 57 | no_message_during(Id, 1000), 58 | message_during(Id, 2000), 59 | ok. 60 | 61 | callback_repeated(_Config) -> 62 | Id = make_ref(), 63 | autocluster_periodic:start_immediate(Id, 100, {?MODULE, periodic_report_back, [{self(), Id}]}), 64 | timer:sleep(1000), 65 | case count_acks(Id, 0) of 66 | Int when Int > 7, Int < 13 -> 67 | ok; 68 | Int -> 69 | error({wrong_number_of_acks, Int}) 70 | end, 71 | ok. 72 | 73 | stop_is_working (_Config) -> 74 | Id = make_ref(), 75 | autocluster_periodic:start_immediate(Id, 100, {?MODULE, periodic_report_back, [{self(), Id}]}), 76 | timer:sleep(1000), 77 | autocluster_periodic:stop(Id), 78 | _ = count_acks(Id, 0), 79 | 0 = count_acks(Id, 500), 80 | ok. 81 | 82 | errors_are_ignored(_Config) -> 83 | Id = make_ref(), 84 | reset_first_error(), 85 | autocluster_periodic:start_immediate(Id, 100, {?MODULE, first_error_then_report_back, [{self(), Id}]}), 86 | timer:sleep(1000), 87 | acks_within(Id, 0, 7, 13), 88 | ok. 89 | 90 | multiple_timers_supported(_Config) -> 91 | Id1 = make_ref(), 92 | autocluster_periodic:start_immediate(Id1, 100, {?MODULE, periodic_report_back, [{self(), Id1}]}), 93 | Id2 = make_ref(), 94 | autocluster_periodic:start_immediate(Id2, 100, {?MODULE, periodic_report_back, [{self(), Id2}]}), 95 | timer:sleep(1000), 96 | acks_within(Id1, 0, 7, 13), 97 | acks_within(Id2, 0, 7, 13), 98 | ok. 99 | 100 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 101 | %% Helpers 102 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 103 | periodic_report_back({Pid, Ref} = State) -> 104 | Pid ! Ref, 105 | State. 106 | 107 | reset_first_error() -> 108 | _ = (catch ets:delete(first_error_tab)), 109 | ets:new(first_error_tab, [public, named_table]). 110 | 111 | first_error_then_report_back({Pid, Ref} = State) -> 112 | case ets:match(first_error_tab, '$1') of 113 | [] -> 114 | ets:insert(first_error_tab, {its_me, mario}), 115 | error(i_accidentially_my_process); 116 | [_] -> 117 | Pid ! Ref, 118 | State 119 | end. 120 | 121 | no_message_during(Id, Timeout) -> 122 | receive 123 | Id -> error(too_early) 124 | after 125 | Timeout -> ok 126 | end. 127 | 128 | message_during(Id, Timeout) -> 129 | receive 130 | Id -> ok 131 | after 132 | Timeout -> error(delayed_callback_wasnt_invoked) 133 | end. 134 | 135 | count_acks(Id, Timeout) -> 136 | count_acks(Id, Timeout, 0). 137 | 138 | count_acks(Id, Timeout, Cnt) -> 139 | receive 140 | Id -> 141 | count_acks(Id, Timeout, Cnt + 1) 142 | after 143 | Timeout -> 144 | Cnt 145 | end. 146 | 147 | acks_within(Id, Timeout, LowerBound, UpperBound) -> 148 | case count_acks(Id, Timeout) of 149 | Int when Int > LowerBound, Int < UpperBound -> 150 | ok; 151 | Int -> 152 | error({wrong_number_of_acks, Int}) 153 | end. 154 | 155 | flush_messages() -> 156 | receive 157 | _ -> 158 | flush_messages() 159 | after 160 | 0 -> 161 | ok 162 | end. 163 | -------------------------------------------------------------------------------- /test/src/autocluster_all_tests.erl: -------------------------------------------------------------------------------- 1 | -module(autocluster_all_tests). 2 | 3 | -export([run/0]). 4 | 5 | -include_lib("eunit/include/eunit.hrl"). 6 | 7 | run() -> 8 | Result = {eunit:test(autocluster_aws_tests, [verbose]), 9 | eunit:test(autocluster_config_tests, [verbose]), 10 | eunit:test(autocluster_consul_tests, [verbose]), 11 | eunit:test(autocluster_dns_tests, [verbose]), 12 | eunit:test(autocluster_etcd_tests, [verbose]), 13 | eunit:test(autocluster_httpc_tests, [verbose]), 14 | eunit:test(autocluster_k8s_tests, [verbose]), 15 | eunit:test(autocluster_sup_tests, [verbose]), 16 | eunit:test(autocluster_util_tests, [verbose]), 17 | eunit:test(autocluster_boot_tests, [verbose])}, 18 | ?assertEqual({ok, ok, ok, ok, ok, ok, ok, ok, ok, ok}, Result). 19 | -------------------------------------------------------------------------------- /test/src/autocluster_aws_tests.erl: -------------------------------------------------------------------------------- 1 | -module(autocluster_aws_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | 5 | -include("autocluster.hrl"). 6 | 7 | %% 8 | maybe_add_tag_filters_test() -> 9 | autocluster_testing:reset(), 10 | Tags = [{"region", "us-west-2"}, {"service", "rabbitmq"}], 11 | Expecation = [{"Filter.2.Name", "tag:service"}, {"Filter.2.Value.1", "rabbitmq"}, 12 | {"Filter.1.Name", "tag:region"}, {"Filter.1.Value.1", "us-west-2"}], 13 | Result = autocluster_aws:maybe_add_tag_filters(Tags, [], 1), 14 | ?assertEqual(Expecation, Result). 15 | 16 | 17 | get_hostname_name_from_reservation_set_test_() -> 18 | { 19 | foreach, 20 | fun autocluster_testing:on_start/0, 21 | fun autocluster_testing:on_finish/1, 22 | [{"from private DNS", 23 | fun() -> 24 | Expectation = ["ip-10-0-16-31.eu-west-1.compute.internal", 25 | "ip-10-0-16-29.eu-west-1.compute.internal"], 26 | ?assertEqual(Expectation, 27 | autocluster_aws:get_hostname_name_from_reservation_set( 28 | reservation_set(), [])) 29 | end}, 30 | {"from private IP", 31 | fun() -> 32 | os:putenv("AWS_USE_PRIVATE_IP", "true"), 33 | Expectation = ["10.0.16.31", "10.0.16.29"], 34 | ?assertEqual(Expectation, 35 | autocluster_aws:get_hostname_name_from_reservation_set( 36 | reservation_set(), [])) 37 | end}] 38 | }. 39 | 40 | reservation_set() -> 41 | [{"item", [{"reservationId","r-006cfdbf8d04c5f01"}, 42 | {"ownerId","248536293561"}, 43 | {"groupSet",[]}, 44 | {"instancesSet", 45 | [{"item", 46 | [{"instanceId","i-0c6d048641f09cad2"}, 47 | {"imageId","ami-ef4c7989"}, 48 | {"instanceState", 49 | [{"code","16"},{"name","running"}]}, 50 | {"privateDnsName", 51 | "ip-10-0-16-29.eu-west-1.compute.internal"}, 52 | {"dnsName",[]}, 53 | {"instanceType","c4.large"}, 54 | {"launchTime","2017-04-07T12:05:10"}, 55 | {"subnetId","subnet-61ff660"}, 56 | {"vpcId","vpc-4fe1562b"}, 57 | {"privateIpAddress","10.0.16.29"}]}]}]}, 58 | {"item", [{"reservationId","r-006cfdbf8d04c5f01"}, 59 | {"ownerId","248536293561"}, 60 | {"groupSet",[]}, 61 | {"instancesSet", 62 | [{"item", 63 | [{"instanceId","i-1c6d048641f09cad2"}, 64 | {"imageId","ami-af4c7989"}, 65 | {"instanceState", 66 | [{"code","16"},{"name","running"}]}, 67 | {"privateDnsName", 68 | "ip-10-0-16-31.eu-west-1.compute.internal"}, 69 | {"dnsName",[]}, 70 | {"instanceType","c4.large"}, 71 | {"launchTime","2017-04-07T12:05:10"}, 72 | {"subnetId","subnet-61ff660"}, 73 | {"vpcId","vpc-4fe1562b"}, 74 | {"privateIpAddress","10.0.16.31"}]}]}]}]. 75 | -------------------------------------------------------------------------------- /test/src/autocluster_config_tests.erl: -------------------------------------------------------------------------------- 1 | -module(autocluster_config_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | 5 | 6 | get_test_() -> 7 | { 8 | foreach, 9 | fun autocluster_testing:on_start/0, 10 | fun autocluster_testing:on_finish/1, 11 | [ 12 | { 13 | "invalid value", 14 | fun() -> 15 | ?assertEqual(undefined, autocluster_config:get(invalid_config_key)) 16 | end 17 | } 18 | ] 19 | }. 20 | 21 | 22 | app_envvar_test_() -> 23 | { 24 | foreach, 25 | fun autocluster_testing:on_start/0, 26 | fun autocluster_testing:on_finish/1, 27 | [ 28 | { 29 | "app atom value", 30 | fun() -> 31 | application:set_env(autocluster, longname, true), 32 | ?assertEqual(true, autocluster_config:get(longname)) 33 | end 34 | }, 35 | { 36 | "app integer value", 37 | fun() -> 38 | application:set_env(autocluster, consul_port, 8502), 39 | ?assertEqual(8502, autocluster_config:get(consul_port)) 40 | end 41 | }, 42 | { 43 | "app string value", 44 | fun() -> 45 | application:set_env(autocluster, consul_svc, "rabbit"), 46 | ?assertEqual("rabbit", autocluster_config:get(consul_svc)) 47 | end 48 | }, 49 | { 50 | "app string value when binary", 51 | fun() -> 52 | application:set_env(autocluster, consul_svc, <<"rabbit">>), 53 | ?assertEqual("rabbit", autocluster_config:get(consul_svc)) 54 | end 55 | }, 56 | { 57 | "app list value when string", 58 | fun() -> 59 | application:set_env(autocluster, proxy_exclusions, "foo,42,42.5"), 60 | ?assertEqual(["foo", 42, 42.5], autocluster_config:get(proxy_exclusions)) 61 | end 62 | }, 63 | { 64 | "consul tags", 65 | fun() -> 66 | application:set_env(autocluster, consul_svc_tags, "urlprefix-:5672 proto=tcp, mq, mq server"), 67 | ?assertEqual(["urlprefix-:5672 proto=tcp","mq","mq server"], autocluster_config:get(consul_svc_tags)) 68 | end 69 | } 70 | ] 71 | }. 72 | 73 | 74 | os_envvar_test_() -> 75 | { 76 | foreach, 77 | fun autocluster_testing:on_start/0, 78 | fun autocluster_testing:on_finish/1, 79 | [ 80 | { 81 | "os atom value", 82 | fun() -> 83 | os:putenv("RABBITMQ_USE_LONGNAME", "true"), 84 | ?assertEqual(true, autocluster_config:get(longname)) 85 | end 86 | }, 87 | { 88 | "os integer value", 89 | fun() -> 90 | os:putenv("CONSUL_PORT", "8501"), 91 | ?assertEqual(8501, autocluster_config:get(consul_port)) 92 | end 93 | }, 94 | { 95 | "prefixed envvar", 96 | fun() -> 97 | os:putenv("RABBITMQ_USE_LONGNAME", "true"), 98 | os:putenv("USE_LONGNAME", "false"), 99 | ?assertEqual(true, autocluster_config:get(longname)) 100 | end 101 | }, 102 | { 103 | "no prefixed envvar", 104 | fun() -> 105 | os:putenv("USE_LONGNAME", "true"), 106 | ?assertEqual(true, autocluster_config:get(longname)) 107 | end 108 | }, 109 | { 110 | "docker changing CONSUL_PORT value", 111 | fun() -> 112 | os:putenv("CONSUL_PORT", "tcp://172.17.10.3:8501"), 113 | ?assertEqual(8501, autocluster_config:get(consul_port)) 114 | end 115 | }, 116 | { 117 | "aws tags", 118 | fun() -> 119 | os:putenv("AWS_EC2_TAGS", 120 | "{\"region\": \"us-west-2\",\"service\": \"rabbitmq\"}"), 121 | ?assertEqual([{"region", "us-west-2"}, {"service", "rabbitmq"}], 122 | autocluster_config:get(aws_ec2_tags)) 123 | end 124 | }, 125 | { 126 | "proxy exclusions", 127 | fun() -> 128 | os:putenv("PROXY_EXCLUSIONS", "foo,42,42.5"), 129 | ?assertEqual(["foo", 42, 42.5], autocluster_config:get(proxy_exclusions)) 130 | end 131 | }, 132 | { 133 | "consul tags set", 134 | fun() -> 135 | os:putenv("CONSUL_SVC_TAGS", "urlprefix-:5672 proto=tcp, mq, mq server"), 136 | ?assertEqual(["urlprefix-:5672 proto=tcp","mq","mq server"], autocluster_config:get(consul_svc_tags)) 137 | end 138 | }, 139 | { 140 | "consul tags unset", 141 | fun() -> 142 | ?assertEqual([], autocluster_config:get(consul_svc_tags)) 143 | end 144 | } 145 | ] 146 | }. 147 | -------------------------------------------------------------------------------- /test/src/autocluster_dns_tests.erl: -------------------------------------------------------------------------------- 1 | -module(autocluster_dns_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | 5 | -include("autocluster.hrl"). 6 | 7 | %% 8 | extract_host_long_test() -> 9 | autocluster_testing:reset(), 10 | os:putenv("RABBITMQ_USE_LONGNAME", "true"), 11 | Record = {ok,{hostent,"rabbit2.ec2-internal",[],inet,4,[{192,168,1,1}]}}, 12 | Expecation = "rabbit2.ec2-internal", 13 | ?assertEqual(Expecation, autocluster_dns:extract_host(Record)). 14 | 15 | extract_host_short_test() -> 16 | autocluster_testing:reset(), 17 | Record = {ok,{hostent,"rabbit2.service.ec2.consul",[],inet,4,[{192,168,1,3}]}}, 18 | Expecation = "rabbit2", 19 | ?assertEqual(Expecation, autocluster_dns:extract_host(Record)). 20 | -------------------------------------------------------------------------------- /test/src/autocluster_etcd_tests.erl: -------------------------------------------------------------------------------- 1 | -module(autocluster_etcd_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | 5 | -include("autocluster.hrl"). 6 | 7 | extract_nodes_test() -> 8 | Values = {struct, [ 9 | {<<"node">>, {struct, [ 10 | {<<"nodes">>, [ 11 | {struct, [{<<"key">>, <<"rabbitmq/default/nodes/foo">>}]}, 12 | {struct, [{<<"key">>, <<"rabbitmq/default/nodes/bar">>}]}, 13 | {struct, [{<<"key">>, <<"rabbitmq/default/nodes/hare@baz">>}]}]}]}}]}, 14 | Expectation = ['rabbit@foo', 'rabbit@bar', 'hare@baz'], 15 | ?assertEqual(Expectation, autocluster_etcd:extract_nodes(Values)). 16 | 17 | base_path_test() -> 18 | autocluster_testing:reset(), 19 | ?assertEqual([v2, keys, "rabbitmq", "default"], autocluster_etcd:base_path()). 20 | 21 | get_node_from_key_test() -> 22 | ?assertEqual('rabbit@foo', autocluster_etcd:get_node_from_key(<<"rabbitmq/default/nodes/foo">>)). 23 | 24 | get_node_from_key_full_name_test() -> 25 | ?assertEqual('hare@foo', autocluster_etcd:get_node_from_key(<<"rabbitmq/default/nodes/hare@foo">>)). 26 | 27 | get_node_from_key_leading_slash_test() -> 28 | ?assertEqual('rabbit@foo', autocluster_etcd:get_node_from_key(<<"/rabbitmq/default/nodes/foo">>)). 29 | 30 | node_path_test() -> 31 | autocluster_testing:reset(), 32 | Expectation = [v2, keys, "rabbitmq", "default", nodes, atom_to_list(node())], 33 | ?assertEqual(Expectation, autocluster_etcd:node_path()). 34 | 35 | nodelist_without_existing_directory_test_() -> 36 | EtcdNodesResponse = {struct,[{<<"action">>,<<"get">>}, 37 | {<<"node">>, 38 | {struct,[{<<"key">>,<<"/rabbitmq/default">>}, 39 | {<<"dir">>,true}, 40 | {<<"nodes">>, 41 | [{struct,[{<<"key">>,<<"/rabbitmq/default/docker-autocluster-4">>}, 42 | {<<"value">>,<<"enabled">>}, 43 | {<<"expiration">>, <<"2016-07-04T12:47:17.245647965Z">>}, 44 | {<<"ttl">>,23}, 45 | {<<"modifiedIndex">>,3976}, 46 | {<<"createdIndex">>,3976}]}]}]}}]}, 47 | autocluster_testing:with_mock( 48 | [autocluster_httpc], 49 | [{"etcd backend creates directory when it's missing", 50 | fun () -> 51 | meck:sequence(autocluster_httpc, get, 5, [{error, "404"}, EtcdNodesResponse]), 52 | meck:expect(autocluster_httpc, put, fun (_, _, _, _, _, _) -> {ok, ok} end), 53 | autocluster_etcd:nodelist(), 54 | ?assert(meck:validate(autocluster_httpc)) 55 | end}]). 56 | -------------------------------------------------------------------------------- /test/src/autocluster_httpc_tests.erl: -------------------------------------------------------------------------------- 1 | -module(autocluster_httpc_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | 5 | build_path_test() -> 6 | ?assertEqual("/foo/b%40r/baz", 7 | autocluster_httpc:build_path(["foo", "b@r", "baz"])). 8 | 9 | build_query_test() -> 10 | ?assertEqual("dc=pr%40duction&recurse", 11 | autocluster_httpc:build_query([{"dc", "pr@duction"}, "recurse"])). 12 | 13 | build_uri_args_test() -> 14 | ?assertEqual("https://127.0.0.1:8501/v1/agent/service/deregister/rabbitmq%3Aautocluster?tag=foo", 15 | autocluster_httpc:build_uri("https", "127.0.0.1", 8501, [v1, agent, service, deregister, "rabbitmq:autocluster"], [{tag, "foo"}])). 16 | 17 | build_uri_no_args_test() -> 18 | ?assertEqual("http://localhost:8502/v1/agent/service/deregister/rabbitmq-autocluster", 19 | autocluster_httpc:build_uri("http", "localhost", 8502, [v1, agent, service, deregister, "rabbitmq-autocluster"], [])). 20 | 21 | http_error_parsed_as_string_test_() -> 22 | autocluster_testing:with_mock( 23 | [httpc], 24 | [fun() -> 25 | meck:expect(httpc, request, fun(_, _, _, _) -> {ok, 404, <<"some junk">>} end), 26 | ?assertEqual({error, "404"}, autocluster_httpc:get("http", "localhost", 80, "/", [])) 27 | end]). 28 | -------------------------------------------------------------------------------- /test/src/autocluster_k8s_tests.erl: -------------------------------------------------------------------------------- 1 | -module(autocluster_k8s_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | 5 | -include("autocluster.hrl"). 6 | 7 | %% 8 | extract_node_list_long_test() -> 9 | autocluster_testing:reset(), 10 | {ok, Response} = 11 | rabbit_misc:json_decode( 12 | [<<"{\"name\": \"mysvc\",\n\"subsets\": [\n{\n\"addresses\": ">>, 13 | <<"[{\"ip\": \"10.10.1.1\"}, {\"ip\": \"10.10.2.2\"}],\n">>, 14 | <<"\"ports\": [{\"name\": \"a\", \"port\": 8675}, {\"name\": ">>, 15 | <<"\"b\", \"port\": 309}]\n},\n{\n\"addresses\": [{\"ip\": ">>, 16 | <<"\"10.10.3.3\"}],\n\"ports\": [{\"name\": \"a\", \"port\": 93}">>, 17 | <<",{\"name\": \"b\", \"port\": 76}]\n}]}">>]), 18 | Expectation = [<<"10.10.1.1">>, <<"10.10.2.2">>, <<"10.10.3.3">>], 19 | ?assertEqual(Expectation, autocluster_k8s:extract_node_list(Response)). 20 | 21 | extract_node_list_short_test() -> 22 | autocluster_testing:reset(), 23 | {ok, Response} = 24 | rabbit_misc:json_decode( 25 | [<<"{\"name\": \"mysvc\",\n\"subsets\": [\n{\n\"addresses\": ">>, 26 | <<"[{\"ip\": \"10.10.1.1\"}, {\"ip\": \"10.10.2.2\"}],\n">>, 27 | <<"\"ports\": [{\"name\": \"a\", \"port\": 8675}, {\"name\": ">>, 28 | <<"\"b\", \"port\": 309}]\n}]}">>]), 29 | Expectation = [<<"10.10.1.1">>, <<"10.10.2.2">>], 30 | ?assertEqual(Expectation, autocluster_k8s:extract_node_list(Response)). 31 | 32 | extract_node_list_hostname_short_test() -> 33 | autocluster_testing:reset(), 34 | os:putenv("K8S_ADDRESS_TYPE", "hostname"), 35 | {ok, Response} = 36 | rabbit_misc:json_decode( 37 | [<<"{\"name\": \"mysvc\",\n\"subsets\": [\n{\n\"addresses\": ">>, 38 | <<"[{\"ip\": \"10.10.1.1\", \"hostname\": \"rabbitmq-1\"}, ">>, 39 | <<"{\"ip\": \"10.10.2.2\", \"hostname\": \"rabbitmq-2\"}],\n">>, 40 | <<"\"ports\": [{\"name\": \"a\", \"port\": 8675}, {\"name\": ">>, 41 | <<"\"b\", \"port\": 309}]\n}]}">>]), 42 | Expectation = [<<"rabbitmq-1">>, <<"rabbitmq-2">>], 43 | ?assertEqual(Expectation, autocluster_k8s:extract_node_list(Response)). 44 | 45 | extract_node_list_real_test() -> 46 | autocluster_testing:reset(), 47 | {ok, Response} = 48 | rabbit_misc:json_decode( 49 | [<<"{\"kind\":\"Endpoints\",\"apiVersion\":\"v1\",\"metadata\":{\"name\":\"galera\",\"namespace\":\"default\",\"selfLink\":\"/api/v1/namespaces/default/endpoints/galera\",\"uid\":\"646f8305-3491-11e6-8c20-ecf4bbd91e6c\",\"resourceVersion\":\"17373568\",\"creationTimestamp\":\"2016-06-17T13:42:54Z\",\"labels\":{\"app\":\"mysqla\"}},\"subsets\":[{\"addresses\":[{\"ip\":\"10.1.29.8\",\"targetRef\":{\"kind\":\"Pod\",\"namespace\":\"default\",\"name\":\"mariadb-tco7k\",\"uid\":\"fb59cc71-558c-11e6-86e9-ecf4bbd91e6c\",\"resourceVersion\":\"13034802\"}},{\"ip\":\"10.1.47.2\",\"targetRef\":{\"kind\":\"Pod\",\"namespace\":\"default\",\"name\":\"mariadb-izgp8\",\"uid\":\"fb484ab3-558c-11e6-86e9-ecf4bbd91e6c\",\"resourceVersion\":\"13035747\"}},{\"ip\":\"10.1.47.3\",\"targetRef\":{\"kind\":\"Pod\",\"namespace\":\"default\",\"name\":\"mariadb-init-ffrsz\",\"uid\":\"fb12e1d3-558c-11e6-86e9-ecf4bbd91e6c\",\"resourceVersion\":\"13032722\"}},{\"ip\":\"10.1.94.2\",\"targetRef\":{\"kind\":\"Pod\",\"namespace\":\"default\",\"name\":\"mariadb-zcc0o\",\"uid\":\"fb31ce6e-558c-11e6-86e9-ecf4bbd91e6c\",\"resourceVersion\":\"13034771\"}}],\"ports\":[{\"name\":\"mysql\",\"port\":3306,\"protocol\":\"TCP\"}]}]}">>]), 50 | Expectation = [<<"10.1.94.2">>, <<"10.1.47.3">>, <<"10.1.47.2">>, 51 | <<"10.1.29.8">>], 52 | ?assertEqual(Expectation, autocluster_k8s:extract_node_list(Response)). 53 | 54 | extract_node_list_with_not_ready_addresses_test() -> 55 | autocluster_testing:reset(), 56 | {ok, Response} = 57 | rabbit_misc:json_decode( 58 | [<<"{\"kind\":\"Endpoints\",\"apiVersion\":\"v1\",\"metadata\":{\"name\":\"rabbitmq\",\"namespace\":\"test-rabbitmq\",\"selfLink\":\"\/api\/v1\/namespaces\/test-rabbitmq\/endpoints\/rabbitmq\",\"uid\":\"4ff733b8-3ad2-11e7-a40d-080027cbdcae\",\"resourceVersion\":\"170098\",\"creationTimestamp\":\"2017-05-17T07:27:41Z\",\"labels\":{\"app\":\"rabbitmq\",\"type\":\"LoadBalancer\"}},\"subsets\":[{\"notReadyAddresses\":[{\"ip\":\"172.17.0.2\",\"hostname\":\"rabbitmq-0\",\"nodeName\":\"minikube\",\"targetRef\":{\"kind\":\"Pod\",\"namespace\":\"test-rabbitmq\",\"name\":\"rabbitmq-0\",\"uid\":\"e980fe5a-3afd-11e7-a40d-080027cbdcae\",\"resourceVersion\":\"170044\"}},{\"ip\":\"172.17.0.4\",\"hostname\":\"rabbitmq-1\",\"nodeName\":\"minikube\",\"targetRef\":{\"kind\":\"Pod\",\"namespace\":\"test-rabbitmq\",\"name\":\"rabbitmq-1\",\"uid\":\"f6285603-3afd-11e7-a40d-080027cbdcae\",\"resourceVersion\":\"170071\"}},{\"ip\":\"172.17.0.5\",\"hostname\":\"rabbitmq-2\",\"nodeName\":\"minikube\",\"targetRef\":{\"kind\":\"Pod\",\"namespace\":\"test-rabbitmq\",\"name\":\"rabbitmq-2\",\"uid\":\"fd5a86dc-3afd-11e7-a40d-080027cbdcae\",\"resourceVersion\":\"170096\"}}],\"ports\":[{\"name\":\"amqp\",\"port\":5672,\"protocol\":\"TCP\"},{\"name\":\"http\",\"port\":15672,\"protocol\":\"TCP\"}]}]}">>]), 59 | Expectation = [], 60 | ?assertEqual(Expectation, autocluster_k8s:extract_node_list(Response)). 61 | 62 | 63 | 64 | node_name_empty_test() -> 65 | autocluster_testing:reset(), 66 | os:putenv("RABBITMQ_USE_LONGNAME", "true"), 67 | Expectation = 'rabbit@rabbitmq-0', 68 | ?assertEqual(Expectation, autocluster_k8s:node_name(<<"rabbitmq-0">>)). 69 | 70 | node_name_suffix_test() -> 71 | autocluster_testing:reset(), 72 | os:putenv("RABBITMQ_USE_LONGNAME", "true"), 73 | os:putenv("K8S_HOSTNAME_SUFFIX", ".rabbitmq.default.svc.cluster.local"), 74 | Expectation = 'rabbit@rabbitmq-0.rabbitmq.default.svc.cluster.local', 75 | ?assertEqual(Expectation, autocluster_k8s:node_name(<<"rabbitmq-0">>)). 76 | -------------------------------------------------------------------------------- /test/src/autocluster_sup_tests.erl: -------------------------------------------------------------------------------- 1 | -module(autocluster_sup_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | 5 | start_link_test_() -> 6 | { 7 | foreach, 8 | fun() -> 9 | meck:new(supervisor, [passthrough, unstick]) 10 | end, 11 | fun(_) -> 12 | meck:unload(supervisor) 13 | end, 14 | [ 15 | { 16 | "supervisor start_link", fun() -> 17 | meck:expect(supervisor, start_link, fun(_, _, _) -> {ok, test_result} end), 18 | ?assertEqual({ok, test_result}, autocluster_sup:start_link()), 19 | ?assert(meck:validate(supervisor)) 20 | end} 21 | ] 22 | }. 23 | 24 | init_test_() -> 25 | { 26 | foreach, 27 | fun autocluster_testing:on_start/0, 28 | fun autocluster_testing:on_finish/1, 29 | [ 30 | { 31 | "default", 32 | fun() -> 33 | Expectation = {ok, {{one_for_one, 3, 10}, []}}, 34 | ?assertEqual(Expectation, autocluster_sup:init([])) 35 | end 36 | }, 37 | { 38 | "cluster cleanup enabled", 39 | fun() -> 40 | os:putenv("AUTOCLUSTER_CLEANUP", "true"), 41 | Expectation = {ok, {{one_for_one, 3, 10}, 42 | [{autocluster_cleanup, 43 | {autocluster_cleanup, start_link, []}, 44 | permanent, 10000, worker, 45 | [autocluster_cleanup]}]}}, 46 | ?assertEqual(Expectation, autocluster_sup:init([])) 47 | end 48 | } 49 | ] 50 | }. 51 | -------------------------------------------------------------------------------- /test/src/autocluster_testing.erl: -------------------------------------------------------------------------------- 1 | -module(autocluster_testing). 2 | 3 | %% API 4 | -export([on_start/0, on_finish/1, reset/0, with_mock/2, with_mock_each/2, optional/2, optionals/2]). 5 | -export([has_etcd/0, start_etcd/1, stop_etcd/1]). 6 | 7 | -include("autocluster.hrl"). 8 | -include_lib("common_test/include/ct.hrl"). 9 | 10 | on_start() -> 11 | reset(), 12 | []. 13 | 14 | on_finish([]) -> 15 | reset(), 16 | ok; 17 | on_finish(Mocks) -> 18 | reset(), 19 | meck:unload(Mocks). 20 | 21 | maybe_reset_truncated_envvar(Key, "RABBITMQ_") -> 22 | os:unsetenv(string:sub_string(Key, 10)); 23 | maybe_reset_truncated_envvar(_, _) -> ok. 24 | 25 | reset() -> reset(?CONFIG_MAP). 26 | reset([]) -> ok; 27 | reset([H|T]) -> 28 | application:unset_env(autocluster, H#config.key), 29 | os:unsetenv(H#config.os), 30 | maybe_reset_truncated_envvar(H#config.os, string:left(H#config.os, 9)), 31 | reset(T). 32 | 33 | with_mock(ModulesToMock, InitiatorOrTests) -> 34 | with_mock_generator(setup, ModulesToMock, InitiatorOrTests). 35 | 36 | with_mock_each(ModulesToMock, InitiatorOrTests) -> 37 | with_mock_generator(foreach, ModulesToMock, InitiatorOrTests). 38 | 39 | with_mock_generator(FixtureType, ModulesToMock, InitiatorOrTests) -> 40 | { 41 | FixtureType, 42 | fun() -> 43 | autocluster_testing:reset(), 44 | lists:foreach(fun ({Mod, Opts}) when is_list(Opts) -> meck:new(Mod, Opts); 45 | ({Mod, Opt}) when is_atom(Opt) -> meck:new(Mod, [Opt]); 46 | (Mod) -> meck:new(Mod, []) 47 | end, ModulesToMock), 48 | lists:map(fun ({Mod, _}) -> Mod; 49 | (Mod) -> Mod 50 | end, ModulesToMock) 51 | end, 52 | fun on_finish/1, 53 | InitiatorOrTests 54 | }. 55 | 56 | -spec optional(boolean(), Val) -> [Val] when Val :: term(). 57 | optional(true, Value) -> 58 | [Value]; 59 | optional(false, _) -> 60 | []. 61 | 62 | -spec optionals(boolean(), [Val]) -> [Val] when Val :: term(). 63 | optionals(true, Vals) -> 64 | Vals; 65 | optionals(false, _) -> 66 | []. 67 | 68 | 69 | -spec has_etcd() -> boolean(). 70 | has_etcd() -> 71 | case os:getenv("USE_ETCD") of 72 | false -> 73 | false; 74 | _ -> 75 | true 76 | end. 77 | 78 | -spec start_etcd(file:name_all()) -> {ok, port(), 1..65535} | {error, term()}. 79 | start_etcd(PrivDir) -> 80 | Port = get_free_tcp_port(), 81 | PortStr = integer_to_list(Port), 82 | PeerPortStr = integer_to_list(get_free_tcp_port()), 83 | Fdlink = erlsh:fdlink_executable(), 84 | EtcdDataDir = filename:join(PrivDir, "etcd-" ++ PortStr ++ ".data"), 85 | {done, 0, _} = erlsh:run(["rm", "-rf", EtcdDataDir]), 86 | Process = open_port({spawn_executable, Fdlink}, 87 | [use_stdio, exit_status, 88 | {args, [find_etcd_executable(), 89 | "--data-dir", EtcdDataDir, 90 | "--listen-client-urls", "http://localhost:" ++ PortStr, 91 | "--advertise-client-urls", "http://localhost:" ++ PortStr, 92 | "--listen-peer-urls", "http://localhost:" ++ PeerPortStr 93 | ]}]), 94 | ok = wait_tcp_port(Port), 95 | LoopPid = spawn(fun () -> etcd_port_loop(Process) end), 96 | erlang:port_connect(Process, LoopPid), 97 | unlink(Process), 98 | {ok, Process, Port}. 99 | 100 | etcd_port_loop(_) -> 101 | Ref = make_ref(), 102 | receive 103 | Ref -> 104 | ok 105 | end. 106 | 107 | -spec stop_etcd(port()) -> ok. 108 | stop_etcd(_Port) -> 109 | ok. 110 | 111 | -spec get_free_tcp_port() -> 1..65535. 112 | get_free_tcp_port() -> 113 | {ok, LSock} = gen_tcp:listen(0, []), 114 | {ok, Port} = inet:port(LSock), 115 | ok = gen_tcp:close(LSock), 116 | Port. 117 | 118 | -spec find_etcd_executable() -> string(). 119 | find_etcd_executable() -> 120 | os:getenv("USE_ETCD"). 121 | 122 | -spec wait_tcp_port(inet:port_number()) -> ok | error. 123 | wait_tcp_port(Port) -> 124 | timer:sleep(1000), 125 | wait_tcp_port(Port, 10). 126 | 127 | -spec wait_tcp_port(inet:port_number(), non_neg_integer()) -> ok. 128 | wait_tcp_port(_, 0) -> 129 | error; 130 | wait_tcp_port(Port, TriesLeft) -> 131 | case gen_tcp:connect("localhost", Port, []) of 132 | {ok, Sock} -> 133 | ct:pal("Connected to ~p through ~p on try ~p", [Port, Sock, TriesLeft]), 134 | gen_tcp:close(Sock), 135 | ok; 136 | _ -> 137 | timer:sleep(1000), 138 | wait_tcp_port(Port, TriesLeft - 1) 139 | end. 140 | -------------------------------------------------------------------------------- /test/src/autocluster_tests.erl: -------------------------------------------------------------------------------- 1 | -module(autocluster_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | 5 | -include("autocluster.hrl"). 6 | 7 | -compile([export_all]). 8 | 9 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 10 | %% Tests 11 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 12 | run_steps_empty_list_test() -> 13 | ?assertEqual(ok, autocluster:run_steps([], #startup_state{})). 14 | 15 | run_steps_error_logged_test_() -> 16 | autocluster_testing:with_mock( 17 | [autocluster_log], 18 | fun() -> 19 | ErrStep = fun (_State) -> {error, "enough!"} end, 20 | meck:expect(autocluster_log, error, fun (_, _) -> ok end), 21 | meck:expect(autocluster_log, info, fun (_, _) -> ok end), 22 | ?assertEqual(ok, autocluster:run_steps([ErrStep], #startup_state{})), 23 | [{Fmt, Args}] = [{Fmt, Args} || {_Pid, {autocluster_log, error, [Fmt, Args]}, _Result} <- meck:history(autocluster_log)], 24 | ErrorMsg = iolist_to_binary(io_lib:format(Fmt, Args)), 25 | {match, _} = re:run(ErrorMsg, "enough!") 26 | end). 27 | 28 | run_steps_honors_failure_mode_test_()-> 29 | Cases = [{stop, {'EXIT', {error, "no more!"}}}, 30 | {ignore, ok}], 31 | ErrStep = fun (_State) -> {error, "no more!"} end, 32 | autocluster_testing:with_mock_each( 33 | [autocluster_log], 34 | lists:map(fun({FailureSetting, Expected}) -> 35 | {lists:flatten(io_lib:format("Error in failure mode '~p' results in '~p'", [FailureSetting, Expected])), 36 | fun() -> 37 | meck:expect(autocluster_log, error, fun (_, _) -> ok end), 38 | meck:expect(autocluster_log, info, fun (_, _) -> ok end), 39 | os:putenv("AUTOCLUSTER_FAILURE", atom_to_list(FailureSetting)), 40 | ?assertEqual(Expected, (catch autocluster:run_steps([ErrStep], #startup_state{}))) 41 | end} 42 | end, 43 | Cases)). 44 | 45 | run_steps_state_is_updated_test() -> 46 | Step1 = fun (#startup_state{} = S) -> {ok, S#startup_state{backend_name = from_step_1}} end, 47 | Step2 = fun (#startup_state{backend_name = from_step_1} = S) -> {ok, S} end, 48 | ?assertEqual(ok, autocluster:run_steps([Step1, Step2], #startup_state{})). 49 | 50 | initialize_backend_starts_required_apps_test_() -> 51 | autocluster_testing:with_mock( 52 | [{application, [unstick, passthrough]}], 53 | fun () -> 54 | meck:expect(application, ensure_all_started, fun (rabbitmq_aws) -> ok end), 55 | autocluster_testing:reset(), 56 | os:putenv("AUTOCLUSTER_TYPE", "aws"), 57 | {ok, _} = autocluster:initialize_backend(#startup_state{}), 58 | ?assert(meck:called(application, ensure_all_started, '_')) 59 | end). 60 | 61 | initialize_backend_with_proxy_test_() -> 62 | autocluster_testing:with_mock( 63 | [{application, [unstick, passthrough]}], 64 | fun () -> 65 | meck:expect(application, ensure_all_started, fun (rabbitmq_aws) -> ok end), 66 | autocluster_testing:reset(), 67 | os:putenv("AUTOCLUSTER_TYPE", "aws"), 68 | os:putenv("HTTP_PROXY", "proxy.eng.megacorp.com"), 69 | os:putenv("HTTPS_PROXY", "proxy.eng.megacorp.com"), 70 | os:putenv("PROXY_EXCLUSIONS", "0.0.0.0, 1.1.1.1"), 71 | inets:start(), 72 | {ok, _} = autocluster:initialize_backend(#startup_state{}), 73 | {ok, {{"proxy.eng.megacorp.com", 80}, ["0.0.0.0", "1.1.1.1"]}} = httpc:get_option(proxy), 74 | {ok, {{"proxy.eng.megacorp.com", 443}, ["0.0.0.0", "1.1.1.1"]}} = httpc:get_option(https_proxy), 75 | ?assert(meck:called(application, ensure_all_started, '_')), 76 | inets:stop() 77 | end). 78 | 79 | initialize_backend_update_required_fields_for_all_known_backends_test_() -> 80 | Cases = [{aws, autocluster_aws} 81 | ,{consul, autocluster_consul} 82 | ,{dns, autocluster_dns} 83 | ,{etcd, autocluster_etcd} 84 | ,{k8s, autocluster_k8s} 85 | ], 86 | autocluster_testing:with_mock_each( 87 | [{application, [unstick, passthrough]}], 88 | [{eunit_title("'~s' backend is known and corresponds to '~s' mod", [Backend, Mod]), 89 | fun () -> 90 | meck:expect(application, ensure_all_started, fun (_) -> ok end), 91 | autocluster_testing:reset(), 92 | os:putenv("AUTOCLUSTER_TYPE", atom_to_list(Backend)), 93 | {ok, UpdatedState} = autocluster:initialize_backend(#startup_state{}), 94 | case UpdatedState of 95 | #startup_state{backend_name = Backend, backend_module = Mod} -> 96 | ok; 97 | _ -> 98 | exit({unexpected_state, UpdatedState}) 99 | end, 100 | ok 101 | end} || {Backend, Mod} <- Cases]). 102 | 103 | initialize_backend_handle_error_test_() -> 104 | Cases = [unconfigured, some_unknown_backend], 105 | [ {eunit_title("Backend '~s' is treated as an error", [BackendOption]), 106 | fun () -> 107 | autocluster_testing:reset(), 108 | os:putenv("AUTOCLUSTER_TYPE", atom_to_list(BackendOption)), 109 | {error, _} = autocluster:initialize_backend(#startup_state{}) 110 | end} || BackendOption <- Cases]. 111 | 112 | 113 | 114 | acquire_startup_lock_store_lock_data_on_success_test_() -> 115 | autocluster_testing:with_mock( 116 | [autocluster_etcd], 117 | fun () -> 118 | State0 = #startup_state{backend_name = etcd, backend_module = autocluster_etcd}, 119 | meck:expect(autocluster_etcd, lock, fun (_) -> {ok, some_lock_data} end), 120 | State = autocluster:acquire_startup_lock(State0), 121 | {ok, #startup_state{startup_lock_data = some_lock_data}} = State 122 | end). 123 | 124 | acquire_startup_lock_delay_when_unsupported_test_() -> 125 | autocluster_testing:with_mock( 126 | [autocluster_etcd, {timer, [unstick, passthrough]}], 127 | fun () -> 128 | meck:expect(timer, sleep, fun(Int) when is_integer(Int) -> ok end), 129 | meck:expect(autocluster_etcd, lock, fun (_) -> not_supported end), 130 | State0 = #startup_state{backend_name = etcd, backend_module = autocluster_etcd}, 131 | autocluster:acquire_startup_lock(State0), 132 | ?assert(meck:called(timer, sleep, '_')), 133 | ok 134 | end). 135 | 136 | acquire_startup_lock_reports_error_test_() -> 137 | autocluster_testing:with_mock( 138 | [autocluster_etcd], 139 | fun () -> 140 | meck:expect(autocluster_etcd, lock, fun (_) -> {error, "borken"} end), 141 | State = #startup_state{backend_name = etcd, backend_module = autocluster_etcd}, 142 | {error, Err} = autocluster:acquire_startup_lock(State), 143 | {match, _} = re:run(Err, "borken"), 144 | ok 145 | end). 146 | 147 | find_best_node_to_join_updates_state_test_() -> 148 | autocluster_testing:with_mock( 149 | [autocluster_etcd, {autocluster_util, [passthrough]}], 150 | fun () -> 151 | os:putenv("AUTOCLUSTER_TYPE", "etcd"), 152 | meck:expect(autocluster_etcd, nodelist, fun () -> {ok, ['some-other-node@localhost']} end), 153 | meck:expect(autocluster_util, augment_nodelist, 154 | fun([Node]) -> 155 | [#candidate_seed_node{ 156 | name = Node, 157 | uptime = 0, 158 | alive = true, 159 | clustered_with = [], 160 | alive_cluster_nodes = [], 161 | partitioned_cluster_nodes = [], 162 | other_cluster_nodes = [] 163 | }] 164 | end), 165 | State0 = #startup_state{backend_name = etcd, backend_module = autocluster_etcd}, 166 | {ok, State1} = autocluster:find_best_node_to_join(State0), 167 | case State1 of 168 | #startup_state{best_node_to_join = 'some-other-node@localhost'} -> 169 | ok; 170 | _ -> 171 | exit({not_updated, State1#startup_state.best_node_to_join}) 172 | end, 173 | ok 174 | end). 175 | 176 | find_best_node_to_join_reports_error_test_() -> 177 | autocluster_testing:with_mock( 178 | [autocluster_etcd], 179 | fun () -> 180 | os:putenv("AUTOCLUSTER_TYPE", "etcd"), 181 | meck:expect(autocluster_etcd, nodelist, fun () -> {error, "Do not want"} end), 182 | State0 = #startup_state{backend_name = etcd, backend_module = autocluster_etcd}, 183 | case autocluster:find_best_node_to_join(State0) of 184 | {error, _} -> 185 | ok; 186 | Result -> 187 | exit({unexpected, Result}) 188 | end, 189 | ok 190 | end). 191 | 192 | maybe_cluster_handles_all_possible_join_cluster_result_test_() -> 193 | Cases = [{ok, ok}, 194 | {{ok, already_member}, ok}, 195 | {{error, {inconsistent_cluster, "some-reason"}}, error}], 196 | autocluster_testing:with_mock_each( 197 | [{application, [unstick, passthrough]}, 198 | {mnesia, [passthrough]}, 199 | rabbit_mnesia], 200 | [{eunit_title("Join result '~p' is handle correctly", [JoinResult]), 201 | fun () -> 202 | meck:expect(mnesia, stop, fun () -> ok end), 203 | meck:expect(application, stop, fun (rabbit) -> ok end), 204 | meck:expect(rabbit_mnesia, join_cluster, fun ('some-node@localhost', _) -> JoinResult end), 205 | State0 = #startup_state{backend_name = etcd, backend_module = autocluster_etcd, 206 | best_node_to_join = 'some-node@localhost'}, 207 | case autocluster:maybe_cluster(State0) of 208 | {Expect, _} -> 209 | ok; 210 | Got -> 211 | exit({not_good, Got}) 212 | end 213 | end} || {JoinResult, Expect} <- Cases]). 214 | 215 | register_with_backend_handles_success_test() -> 216 | autocluster_testing:with_mock( 217 | [autocluster_etcd], 218 | fun() -> 219 | State = #startup_state{backend_name = etcd, 220 | backend_module = autocluster_etcd}, 221 | meck:expect(autocluster_etcd, register, fun () -> ok end), 222 | {ok, State} = autocluster:register_with_backend(State), 223 | ?assert(meck:validate(autocluster_etcd)) 224 | end), 225 | ok. 226 | 227 | register_with_backend_handles_failure_test() -> 228 | autocluster_testing:with_mock( 229 | [autocluster_etcd], 230 | fun() -> 231 | State = #startup_state{backend_name = etcd, 232 | backend_module = autocluster_etcd}, 233 | meck:expect(autocluster_etcd, register, 234 | fun () -> {error, "something going on"} end), 235 | {error, _} = autocluster:register_with_backend(State), 236 | ?assert(meck:validate(autocluster_etcd)) 237 | end), 238 | ok. 239 | 240 | release_startup_lock_handles_success_test() -> 241 | autocluster_testing:with_mock( 242 | [autocluster_etcd], 243 | fun() -> 244 | State = #startup_state{backend_name = etcd, 245 | backend_module = autocluster_etcd}, 246 | meck:expect(autocluster_etcd, unlock, 247 | fun () -> ok end), 248 | {ok, State} = autocluster:release_startup_lock(State), 249 | ?assert(meck:validate(autocluster_etcd)) 250 | end), 251 | ok. 252 | 253 | release_startup_lock_handles_failure_test() -> 254 | autocluster_testing:with_mock( 255 | [autocluster_etcd], 256 | fun() -> 257 | State = #startup_state{backend_name = etcd, 258 | backend_module = autocluster_etcd}, 259 | meck:expect(autocluster_etcd, unlock, 260 | fun () -> {error, "STOLEN!!!"} end), 261 | {error, _} = autocluster:release_startup_lock(State), 262 | ?assert(meck:validate(autocluster_etcd)) 263 | end), 264 | ok. 265 | 266 | choose_best_node_empty_list_test() -> 267 | ?assertEqual(undefined, autocluster:choose_best_node([])). 268 | 269 | choose_best_node_only_self_test() -> 270 | ?assertEqual(undefined, autocluster:choose_best_node([#candidate_seed_node{name = node()}])). 271 | 272 | 273 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 274 | %% Tests with empty backend, RabbitMQ has to start even if the backend is empty 275 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 276 | 277 | initialize_backend_empty_test_() -> 278 | [ {eunit_title("Backend is empty", []), 279 | fun () -> 280 | autocluster_testing:reset(), 281 | {error, unconfigured} = autocluster:initialize_backend( 282 | autocluster:new_startup_state()) 283 | end} ]. 284 | 285 | 286 | 287 | acquire_startup_lock_backend_empty_test_() -> 288 | [ {eunit_title("Acquire startup lock with empty backend", []), 289 | fun () -> 290 | autocluster_testing:reset(), 291 | {error, unconfigured} = autocluster:acquire_startup_lock( 292 | autocluster:new_startup_state()) 293 | end} ]. 294 | 295 | release_startup_lock_backend_empty_test_() -> 296 | [ {eunit_title("Release startup lock with empty backend", []), 297 | fun () -> 298 | autocluster_testing:reset(), 299 | {error, unconfigured} = autocluster:release_startup_lock( 300 | autocluster:new_startup_state()) 301 | end} ]. 302 | 303 | 304 | register_with_backend_backend_empty_test_() -> 305 | [ {eunit_title("Register in backend with empty backend", []), 306 | fun () -> 307 | autocluster_testing:reset(), 308 | {error, unconfigured} = autocluster:register_with_backend( 309 | autocluster:new_startup_state()) 310 | end} ]. 311 | 312 | nodelist_backend_empty_test_() -> 313 | [ {eunit_title("Register in nodelist with empty backend", []), 314 | fun () -> 315 | autocluster_testing:reset(), 316 | {error, unconfigured} = autocluster:backend_nodelist( 317 | autocluster:new_startup_state()) 318 | end} ]. 319 | 320 | 321 | 322 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 323 | %% Helpers 324 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 325 | eunit_title(Fmt, Args) -> 326 | lists:flatten(io_lib:format(Fmt, Args)). 327 | -------------------------------------------------------------------------------- /test/src/autocluster_util_tests.erl: -------------------------------------------------------------------------------- 1 | -module(autocluster_util_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | 5 | -include("autocluster.hrl"). 6 | 7 | 8 | -define(INTERFACES, 9 | {ok, [{"lo0", 10 | [{flags,[up,loopback,running,multicast]}, 11 | {addr,{0,0,0,0,0,0,0,1}}, 12 | {netmask,{65535,65535,65535,65535,65535,65535,65535,65535}}, 13 | {addr,{127,0,0,1}}, 14 | {netmask,{255,0,0,0}}, 15 | {addr,{65152,0,0,0,0,0,0,1}}, 16 | {netmask,{65535,65535,65535,65535,0,0,0,0}}]}, 17 | {"en0", 18 | [{flags,[up,broadcast,running,multicast]}, 19 | {hwaddr,[100,200,100,200,100,128]}, 20 | {addr,{65540,0,0,0,47848,22271,65097,640}}, 21 | {netmask,{65535,65535,65535,65535,0,0,0,0}}, 22 | {addr,{10,1,1,128}}, 23 | {netmask,{255,255,0,0}}, 24 | {broadaddr,{10,229,255,255}}, 25 | {addr,{9760,152,49152,0,16,150,0,1449}}, 26 | {netmask,{65535,65535,65535,65535,65535,65535,0,0}}]}]}). 27 | 28 | 29 | as_atom_test_() -> 30 | { 31 | foreach, 32 | fun() -> 33 | autocluster_testing:reset(), 34 | meck:new(autocluster_log, []), 35 | [autocluster_log] 36 | end, 37 | fun autocluster_testing:on_finish/1, 38 | [ 39 | {"atom", fun() -> ?assertEqual(foo, autocluster_util:as_atom(foo)) end}, 40 | {"binary", fun() -> ?assertEqual(bar, autocluster_util:as_atom(<<"bar">>)) end}, 41 | {"string", fun() -> ?assertEqual(baz, autocluster_util:as_atom("baz")) end}, 42 | {"other", 43 | fun() -> 44 | meck:expect(autocluster_log, error, 45 | fun(_Message, Args) -> ?assertEqual([42], Args) end), 46 | ?assertEqual(42, autocluster_util:as_atom(42)), 47 | ?assert(meck:validate(autocluster_log)) 48 | end 49 | } 50 | ] 51 | }. 52 | 53 | 54 | as_integer_test_() -> 55 | { 56 | foreach, 57 | fun() -> 58 | autocluster_testing:reset(), 59 | meck:new(autocluster_log, []), 60 | [autocluster_log] 61 | end, 62 | fun autocluster_testing:on_finish/1, 63 | [ 64 | {"integer", fun() -> ?assertEqual(42, autocluster_util:as_integer(42)) end}, 65 | {"binary", fun() -> ?assertEqual(42, autocluster_util:as_integer(<<"42">>)) end}, 66 | {"string", fun() -> ?assertEqual(42, autocluster_util:as_integer("42")) end}, 67 | {"other", 68 | fun() -> 69 | meck:expect(autocluster_log, error, 70 | fun(_Message, Args) -> ?assertEqual(['42'], Args) end), 71 | ?assertEqual('42', autocluster_util:as_integer('42')), 72 | ?assert(meck:validate(autocluster_log)) 73 | end 74 | } 75 | ] 76 | }. 77 | 78 | 79 | as_string_test_() -> 80 | { 81 | foreach, 82 | fun() -> 83 | autocluster_testing:reset(), 84 | meck:new(autocluster_log, []), 85 | [autocluster_log] 86 | end, 87 | fun autocluster_testing:on_finish/1, 88 | [ 89 | {"atom", fun() -> ?assertEqual("42", autocluster_util:as_string('42')) end}, 90 | {"integer", fun() -> ?assertEqual("42", autocluster_util:as_string(42)) end}, 91 | {"binary", fun() -> ?assertEqual("42", autocluster_util:as_string(<<"42">>)) end}, 92 | {"string", fun() -> ?assertEqual("42", autocluster_util:as_string("42")) end}, 93 | {"other", 94 | fun() -> 95 | meck:expect(autocluster_log, error, 96 | fun(_Message, Args) -> ?assertEqual([#config{}], Args) end), 97 | ?assertEqual(#config{}, autocluster_util:as_string(#config{})), 98 | ?assert(meck:validate(autocluster_log)) 99 | end 100 | } 101 | ] 102 | }. 103 | 104 | 105 | as_list_test_() -> 106 | { 107 | foreach, 108 | fun() -> 109 | autocluster_testing:reset(), 110 | meck:new(autocluster_log, []), 111 | [autocluster_log] 112 | end, 113 | fun autocluster_testing:on_finish/1, 114 | [ 115 | {"integer", fun() -> ?assertEqual([42], autocluster_util:as_list(42)) end}, 116 | {"atom", fun() -> ?assertEqual(['42'], autocluster_util:as_list('42')) end}, 117 | {"binary", fun() -> ?assertEqual([<<"42">>], autocluster_util:as_list(<<"42">>)) end}, 118 | {"string", fun() -> ?assertEqual(["foo"], autocluster_util:as_list("foo")) end}, 119 | {"string-list", fun() -> ?assertEqual(["foo", 42, 42.5], autocluster_util:as_list("foo,42,42.5")) end}, 120 | {"other", 121 | fun() -> 122 | meck:expect(autocluster_log, error, 123 | fun(_Message, Args) -> ?assertEqual([#config{}], Args) end), 124 | ?assertEqual(#config{}, autocluster_util:as_list(#config{})), 125 | ?assert(meck:validate(autocluster_log)) 126 | end 127 | } 128 | ] 129 | }. 130 | 131 | 132 | backend_module_test_() -> 133 | { 134 | foreach, 135 | fun autocluster_testing:on_start/0, 136 | fun autocluster_testing:on_finish/1, 137 | [ 138 | { 139 | "aws", fun() -> 140 | os:putenv("AUTOCLUSTER_TYPE", "aws"), 141 | ?assertEqual(autocluster_aws, autocluster_util:backend_module()) 142 | end 143 | }, 144 | { 145 | "consul", fun() -> 146 | os:putenv("AUTOCLUSTER_TYPE", "consul"), 147 | ?assertEqual(autocluster_consul, autocluster_util:backend_module()) 148 | end 149 | }, 150 | { 151 | "dns", fun() -> 152 | os:putenv("AUTOCLUSTER_TYPE", "dns"), 153 | ?assertEqual(autocluster_dns, autocluster_util:backend_module()) 154 | end 155 | }, 156 | { 157 | "etcd", fun() -> 158 | os:putenv("AUTOCLUSTER_TYPE", "etcd"), 159 | ?assertEqual(autocluster_etcd, autocluster_util:backend_module()) 160 | end 161 | }, 162 | { 163 | "k8s", fun() -> 164 | os:putenv("AUTOCLUSTER_TYPE", "k8s"), 165 | ?assertEqual(autocluster_k8s, autocluster_util:backend_module()) 166 | end 167 | }, 168 | {"unconfigured", fun() -> 169 | ?assertEqual(undefined, autocluster_util:backend_module()) 170 | end} 171 | ] 172 | }. 173 | 174 | 175 | nic_ipaddr_test_() -> 176 | { 177 | foreach, 178 | fun() -> 179 | autocluster_testing:reset(), 180 | meck:new(inet, [unstick, passthrough]), 181 | [inet] 182 | end, 183 | fun autocluster_testing:on_finish/1, 184 | [ 185 | { 186 | "parsing datastructure", 187 | fun() -> 188 | meck:expect(inet, getifaddrs, fun() -> ?INTERFACES end), 189 | ?assertEqual({ok, "10.1.1.128"}, autocluster_util:nic_ipv4("en0")), 190 | ?assert(meck:validate(inet)) 191 | end 192 | }, 193 | { 194 | "nic not found", fun() -> 195 | meck:expect(inet, getifaddrs, fun() -> ?INTERFACES end), 196 | ?assertEqual({error, not_found}, autocluster_util:nic_ipv4("en1")), 197 | ?assert(meck:validate(inet)) 198 | end 199 | } 200 | ] 201 | }. 202 | 203 | 204 | node_hostname_test_() -> 205 | { 206 | foreach, 207 | fun() -> 208 | autocluster_testing:reset(), 209 | meck:new(inet, [unstick, passthrough]), 210 | [inet] 211 | end, 212 | fun autocluster_testing:on_finish/1, 213 | [ 214 | %% Hostname from nodename cannot be tested as mocking the 'erlang' module 215 | %% causes the test process to hang. 216 | { 217 | "hostname from network", fun() -> 218 | meck:expect(inet, gethostname, fun() -> {ok, "hal4500"} end), 219 | ?assertEqual("hal4500", autocluster_util:node_hostname(false)) 220 | end 221 | } 222 | ] 223 | }. 224 | 225 | 226 | node_name_test_() -> 227 | { 228 | foreach, 229 | fun() -> 230 | autocluster_testing:reset(), 231 | meck:new(autocluster_log, []), 232 | [autocluster_log] 233 | end, 234 | fun autocluster_testing:on_finish/1, 235 | [ 236 | {"sname", fun() -> ?assertEqual('rabbit@node1', autocluster_util:node_name("node1")) end}, 237 | {"sname from binary", fun() -> ?assertEqual('rabbit@node1', autocluster_util:node_name(<<"node1">>)) end}, 238 | {"sname from lname", fun() -> ?assertEqual('rabbit@node2', autocluster_util:node_name("node2.foo.bar")) end}, 239 | { 240 | "lname", fun() -> 241 | os:putenv("RABBITMQ_USE_LONGNAME", "true"), 242 | ?assertEqual('rabbit@node3.foo.bar', autocluster_util:node_name("node3.foo.bar")) 243 | end 244 | }, 245 | {"ipaddr", fun() -> ?assertEqual('rabbit@172.20.1.4', autocluster_util:node_name("172.20.1.4")) end} 246 | ] 247 | }. 248 | 249 | 250 | parse_port_test_() -> 251 | { 252 | foreach, 253 | fun autocluster_testing:on_start/0, 254 | fun autocluster_testing:on_finish/1, 255 | [ 256 | {"integer", fun() -> ?assertEqual(5672, autocluster_util:parse_port(5672)) end}, 257 | {"string", fun() -> ?assertEqual(5672, autocluster_util:parse_port("5672")) end}, 258 | {"uri", fun() -> ?assertEqual(5672, autocluster_util:parse_port("amqp://localhost:5672")) end} 259 | ] 260 | }. 261 | -------------------------------------------------------------------------------- /test/unit_test_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(unit_test_SUITE). 2 | 3 | -include_lib("common_test/include/ct.hrl"). 4 | 5 | -export([all/0]). 6 | 7 | -export([autocluster_config_tests/1 8 | ,autocluster_consul_tests/1 9 | ,autocluster_dns_tests/1 10 | ,autocluster_etcd_tests/1 11 | ,autocluster_httpc_tests/1 12 | ,autocluster_k8s_tests/1 13 | ,autocluster_sup_tests/1 14 | ,autocluster_util_tests/1 15 | ,autocluster_tests/1 16 | ]). 17 | 18 | 19 | all() -> 20 | [autocluster_config_tests 21 | ,autocluster_consul_tests 22 | ,autocluster_dns_tests 23 | ,autocluster_etcd_tests 24 | ,autocluster_httpc_tests 25 | ,autocluster_k8s_tests 26 | ,autocluster_sup_tests 27 | ,autocluster_util_tests 28 | ,autocluster_tests 29 | ]. 30 | 31 | autocluster_config_tests(_Config) -> 32 | ok = eunit:test(autocluster_config_tests, [verbose]). 33 | 34 | autocluster_consul_tests(_Config) -> 35 | ok = eunit:test(autocluster_consul_tests, [verbose]). 36 | 37 | autocluster_dns_tests(_Config) -> 38 | ok = eunit:test(autocluster_dns_tests, [verbose]). 39 | 40 | autocluster_etcd_tests(_Config) -> 41 | ok = eunit:test(autocluster_etcd_tests, [verbose]). 42 | 43 | autocluster_httpc_tests(_Config) -> 44 | ok = eunit:test(autocluster_httpc_tests, [verbose]). 45 | 46 | autocluster_k8s_tests(_Config) -> 47 | ok = eunit:test(autocluster_k8s_tests, [verbose]). 48 | 49 | autocluster_sup_tests(_Config) -> 50 | ok = eunit:test(autocluster_sup_tests, [verbose]). 51 | 52 | autocluster_util_tests(_Config) -> 53 | ok = eunit:test(autocluster_util_tests, [verbose]). 54 | 55 | autocluster_tests(_Config) -> 56 | ok = eunit:test(autocluster_tests, [verbose]). 57 | --------------------------------------------------------------------------------