├── .gitignore ├── .gitmodules ├── Dockerfile ├── LICENSE ├── README.md ├── bin └── consul-manage ├── etc ├── consul.hcl └── containerpilot.json5 ├── examples ├── compose │ └── docker-compose.yml ├── triton-multi-dc │ ├── docker-compose-multi-dc.yml.template │ └── setup-multi-dc.sh └── triton │ ├── docker-compose.yml │ └── setup.sh ├── makefile └── test ├── Dockerfile ├── compose.sh └── triton.sh /.gitignore: -------------------------------------------------------------------------------- 1 | _env* 2 | examples/triton-multi-dc/docker-compose-*.yml 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "test/triton-docker-cli"] 2 | path = test/triton-docker-cli 3 | url = https://github.com/joyent/triton-docker-cli 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.6 2 | 3 | # Alpine packages 4 | RUN apk --no-cache \ 5 | add \ 6 | curl \ 7 | bash \ 8 | ca-certificates 9 | 10 | # The Consul binary 11 | ENV CONSUL_VERSION=1.0.0 12 | RUN export CONSUL_CHECKSUM=585782e1fb25a2096e1776e2da206866b1d9e1f10b71317e682e03125f22f479 \ 13 | && export archive=consul_${CONSUL_VERSION}_linux_amd64.zip \ 14 | && curl -Lso /tmp/${archive} https://releases.hashicorp.com/consul/${CONSUL_VERSION}/${archive} \ 15 | && echo "${CONSUL_CHECKSUM} /tmp/${archive}" | sha256sum -c \ 16 | && cd /bin \ 17 | && unzip /tmp/${archive} \ 18 | && chmod +x /bin/consul \ 19 | && rm /tmp/${archive} 20 | 21 | # Add Containerpilot and set its configuration 22 | ENV CONTAINERPILOT_VER=3.6.0 23 | ENV CONTAINERPILOT=/etc/containerpilot.json5 24 | RUN export CONTAINERPILOT_CHECKSUM=1248784ff475e6fda69ebf7a2136adbfb902f74b \ 25 | && curl -Lso /tmp/containerpilot.tar.gz \ 26 | "https://github.com/joyent/containerpilot/releases/download/${CONTAINERPILOT_VER}/containerpilot-${CONTAINERPILOT_VER}.tar.gz" \ 27 | && echo "${CONTAINERPILOT_CHECKSUM} /tmp/containerpilot.tar.gz" | sha1sum -c \ 28 | && tar zxf /tmp/containerpilot.tar.gz -C /usr/local/bin \ 29 | && rm /tmp/containerpilot.tar.gz 30 | 31 | # configuration files and bootstrap scripts 32 | COPY etc/containerpilot.json5 /etc/ 33 | COPY etc/consul.hcl /etc/consul/ 34 | COPY bin/* /usr/local/bin/ 35 | 36 | # Put Consul data on a separate volume (via etc/consul.hcl) to avoid filesystem 37 | # performance issues with Docker image layers. Not necessary on Triton, but... 38 | VOLUME ["/data"] 39 | 40 | # We don't need to expose these ports in order for other containers on Triton 41 | # to reach this container in the default networking environment, but if we 42 | # leave this here then we get the ports as well-known environment variables 43 | # for purposes of linking. 44 | EXPOSE 8300 8301 8301/udp 8302 8302/udp 8400 8500 53 53/udp 45 | 46 | #ENV GOMAXPROCS 2 47 | ENV SHELL /bin/bash 48 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2015 Casey Bisson 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | 9 | This project is built on Jeff Lindsay's substantial foundation work as found in https://github.com/gliderlabs/docker-consul/tree/legacy. The license for that work is included below. 10 | 11 | 12 | 13 | Copyright (C) 2014 Jeff Lindsay 14 | 15 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 16 | 17 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 18 | 19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Consul with the Autopilot Pattern 2 | 3 | [Consul](http://www.consul.io/) in Docker, designed to be self-operating according to the autopilot pattern. This application demonstrates support for configuring the Consul raft so it can be used as a highly-available discovery catalog for other applications using the Autopilot pattern. 4 | 5 | [![DockerPulls](https://img.shields.io/docker/pulls/autopilotpattern/consul.svg)](https://registry.hub.docker.com/u/autopilotpattern/consul/) 6 | [![DockerStars](https://img.shields.io/docker/stars/autopilotpattern/consul.svg)](https://registry.hub.docker.com/u/autopilotpattern/consul/) 7 | 8 | ## Using Consul with ContainerPilot 9 | 10 | This design starts up all Consul instances with the `-bootstrap-expect` flag. This option tells Consul how many nodes we expect and automatically bootstraps when that many servers are available. We still need to tell Consul how to find the other nodes, and this is where [Triton Container Name Service (CNS)](https://docs.joyent.com/public-cloud/network/cns) and [ContainerPilot](https://joyent.com/containerpilot) come into play. 11 | 12 | The ContainerPilot configuration has a management script that is run on each health check interval. This management script runs `consul info` and gets the number of peers for this node. If the number of peers is not equal to 2 (there are 3 nodes in total but a node isn't its own peer), then the script will attempt to find another node via `consul join`. This command will use the Triton CNS name for the Consul service. Because each node is automatically added to the A Record for the CNS name when it starts, the nodes will all eventually find at least one other node and bootstrap the Consul raft. 13 | 14 | When run locally for testing, we don't have access to Triton CNS. The `local-compose.yml` file uses the v2 Compose API, which automatically creates a user-defined network and allows us to use Docker DNS for the service. 15 | 16 | ## Run it! 17 | 18 | 1. [Get a Joyent account](https://my.joyent.com/landing/signup/) and [add your SSH key](https://docs.joyent.com/public-cloud/getting-started). 19 | 1. Install the [Docker Toolbox](https://docs.docker.com/installation/mac/) (including `docker` and `docker-compose`) on your laptop or other environment, as well as the [Joyent Triton CLI](https://www.joyent.com/blog/introducing-the-triton-command-line-tool) (`triton` replaces our old `sdc-*` CLI tools). 20 | 21 | Check that everything is configured correctly by changing to the `examples/triton` directory and executing `./setup.sh`. This will check that your environment is setup correctly and will create an `_env` file that includes injecting an environment variable for a service name for Consul in Triton CNS. We'll use this CNS name to bootstrap the cluster. 22 | 23 | ```bash 24 | $ docker-compose up -d 25 | Creating consul_consul_1 26 | 27 | $ docker-compose scale consul=3 28 | Creating and starting consul_consul_2 ... 29 | Creating and starting consul_consul_3 ... 30 | 31 | $ docker-compose ps 32 | Name Command State Ports 33 | -------------------------------------------------------------------------------- 34 | consul_consul_1 /usr/local/bin/containerpilot... Up 53/tcp, 53/udp, 35 | 8300/tcp, 8301/tcp, 36 | 8301/udp, 8302/tcp, 37 | 8302/udp, 8400/tcp, 38 | 0.0.0.0:8500->8500/tcp 39 | consul_consul_2 /usr/local/bin/containerpilot... Up 53/tcp, 53/udp, 40 | 8300/tcp, 8301/tcp, 41 | 8301/udp, 8302/tcp, 42 | 8302/udp, 8400/tcp, 43 | 0.0.0.0:8500->8500/tcp 44 | consul_consul_3 /usr/local/bin/containerpilot... Up 53/tcp, 53/udp, 45 | 8300/tcp, 8301/tcp, 46 | 8301/udp, 8302/tcp, 47 | 8302/udp, 8400/tcp, 48 | 0.0.0.0:8500->8500/tcp 49 | 50 | $ docker exec -it consul_consul_3 consul info | grep num_peers 51 | num_peers = 2 52 | 53 | ``` 54 | 55 | ### Run it with more than one datacenter! 56 | 57 | Within the `examples/triton-multi-dc` directory, execute `./setup-multi-dc.sh`, providing as arguments Triton profiles which belong to the desired data centers. 58 | 59 | Since interacting with multiple data centers requires switching between Triton profiles it's easier to perform the following steps in separate terminals. It is possible to perform all the steps for a single data center and then change profiles. Additionally, setting `COMPOSE_PROJECT_NAME` to match the profile or data center will help distinguish nodes in Triton Portal and the `triton instance ls` listing. 60 | 61 | One `_env` and one `docker-compose-.yml` should be generated for each profile. Execute the following commands, once for each profile/datacenter, within `examples/triton-multi-dc`: 62 | 63 | ``` 64 | $ eval "$(TRITON_PROFILE= triton env -d)" 65 | 66 | # The following helps when executing docker-compose multiple times. Alternatively, pass the -f flag to each invocation of docker-compose. 67 | $ export COMPOSE_FILE=docker-compose-.yml 68 | 69 | # The following is not strictly necessary but helps to discern between clusters. Alternatively, pass the -p flag to each invocation of docker-compose. 70 | $ export COMPOSE_PROJECT_NAME= 71 | 72 | $ docker-compose up -d 73 | Creating _consul_1 ... done 74 | 75 | $ docker-compose scale consul=3 76 | ``` 77 | 78 | Note: the `cns.joyent.com` hostnames cannot be resolved from outside the datacenters. Change `cns.joyent.com` to `triton.zone` to access the web UI. 79 | 80 | ## Environment Variables 81 | 82 | - `CONSUL_DEV`: Enable development mode, allowing a node to self-elect as a cluster leader. Consul flag: [`-dev`](https://www.consul.io/docs/agent/options.html#_dev). 83 | - The following errors will occur if `CONSUL_DEV` is omitted and not enough Consul instances are deployed: 84 | ``` 85 | [ERR] agent: failed to sync remote state: No cluster leader 86 | [ERR] agent: failed to sync changes: No cluster leader 87 | [ERR] agent: Coordinate update error: No cluster leader 88 | ``` 89 | - `CONSUL_DATACENTER_NAME`: Explicitly set the name of the data center in which Consul is running. Consul flag: [`-datacenter`](https://www.consul.io/docs/agent/options.html#datacenter). 90 | - If this variable is specified it will be used as-is. 91 | - If not specified, automatic detection of the datacenter will be attempted. See [issue #23](https://github.com/autopilotpattern/consul/issues/23) for more details. 92 | - Consul's default of "dc1" will be used if none of the above apply. 93 | 94 | - `CONSUL_BIND_ADDR`: Explicitly set the corresponding Consul configuration. This value will be set to `0.0.0.0` if `CONSUL_BIND_ADDR` is not specified and `CONSUL_RETRY_JOIN_WAN` is provided. Be aware of the security implications of binding the server to a public address and consider setting up encryption or using a VPN to isolate WAN traffic from the public internet. 95 | - `CONSUL_SERF_LAN_BIND`: Explicitly set the corresponding Consul configuration. This value will be set to the server's private address automatically if not specified. Consul flag: [`-serf-lan-bind`](https://www.consul.io/docs/agent/options.html#serf_lan_bind). 96 | - `CONSUL_SERF_WAN_BIND`: Explicitly set the corresponding Consul configuration. This value will be set to the server's public address automatically if not specified. Consul flag: [`-serf-wan-bind`](https://www.consul.io/docs/agent/options.html#serf_wan_bind). 97 | - `CONSUL_ADVERTISE_ADDR`: Explicitly set the corresponding Consul configuration. This value will be set to the server's private address automatically if not specified. Consul flag: [`-advertise-addr`](https://www.consul.io/docs/agent/options.html#advertise_addr). 98 | - `CONSUL_ADVERTISE_ADDR_WAN`: Explicitly set the corresponding Consul configuration. This value will be set to the server's public address automatically if not specified. Consul flag: [`-advertise-addr-wan`](https://www.consul.io/docs/agent/options.html#advertise_addr_wan). 99 | 100 | - `CONSUL_RETRY_JOIN_WAN`: sets the remote datacenter addresses to join. Must be a valid HCL list (i.e. comma-separated quoted addresses). Consul flag: [`-retry-join-wan`](https://www.consul.io/docs/agent/options.html#retry_join_wan). 101 | - The following error will occur if `CONSUL_RETRY_JOIN_WAN` is provided but improperly formatted: 102 | ``` 103 | ==> Error parsing /etc/consul/consul.hcl: ... unexpected token while parsing list: IDENT 104 | ``` 105 | - Gossip over the WAN requires the following ports to be accessible between data centers, make sure that adequate firewall rules have been established for the following ports (this should happen automatically when using docker-compose with Triton): 106 | - `8300`: Server RPC port (TCP) 107 | - `8302`: Serf WAN gossip port (TCP + UDP) 108 | 109 | ## Using this in your own composition 110 | 111 | There are two ways to run Consul and both come into play when deploying ContainerPilot, a cluster of Consul servers and individual Consul client agents. 112 | 113 | ### Servers 114 | 115 | The Consul container created by this project provides a scalable Consul cluster. Use this cluster with any project that requires Consul and not just ContainerPilot/Autopilot applications. 116 | 117 | The following Consul service definition can be dropped into any Docker Compose file to run a Consul cluster alongside other application containers. 118 | 119 | ```yaml 120 | version: '2.1' 121 | 122 | services: 123 | 124 | consul: 125 | image: autopilotpattern/consul:1.0.0r43 126 | restart: always 127 | mem_limit: 128m 128 | ports: 129 | - 8500 130 | environment: 131 | - CONSUL=consul 132 | - LOG_LEVEL=info 133 | command: > 134 | /usr/local/bin/containerpilot 135 | ``` 136 | 137 | In our experience, including a Consul cluster within a project's `docker-compose.yml` can help developers understand and test how a service should be discovered and registered within a wider infrastructure context. 138 | 139 | ### Clients 140 | 141 | ContainerPilot utilizes Consul's [HTTP Agent API](https://www.consul.io/api/agent.html) for a handful of endpoints, such as `UpdateTTL`, `CheckRegister`, `ServiceRegister` and `ServiceDeregister`. Connecting ContainerPilot to Consul can be achieved by running Consul as a client to a cluster (mentioned above). It's easy to run this Consul client agent from ContainerPilot itself. 142 | 143 | The following snippet demonstrates how to achieve running Consul inside a container and connecting it to ContainerPilot by configuring a `consul-agent` job. 144 | 145 | ```json5 146 | { 147 | consul: 'localhost:8500', 148 | jobs: [ 149 | { 150 | name: "consul-agent", 151 | restarts: "unlimited", 152 | exec: [ 153 | "/usr/bin/consul", "agent", 154 | "-data-dir=/data", 155 | "-log-level=err", 156 | "-rejoin", 157 | "-retry-join", '{{ .CONSUL | default "consul" }}', 158 | "-retry-max", "10", 159 | "-retry-interval", "10s", 160 | ], 161 | health: { 162 | exec: "curl -so /dev/null http://localhost:8500", 163 | interval: 10, 164 | ttl: 25, 165 | } 166 | } 167 | ] 168 | } 169 | ``` 170 | 171 | Many application setups in the Autopilot Pattern library include a Consul agent process within each container to handle connecting ContainerPilot itself to a cluster of Consul servers. This helps performance of Consul features at the cost of more clients connected to the cluster. 172 | 173 | A more detailed example of a ContainerPilot configuration that uses a Consul agent co-process can be found in [autopilotpattern/nginx](https://github.com/autopilotpattern/nginx). 174 | 175 | ## Triton-specific availability advantages 176 | 177 | Some details about how Docker containers work on Triton have specific bearing on the durability and availability of this service: 178 | 179 | 1. Docker containers are first-order objects on Triton. They run on bare metal, and their overall availability is similar or better than what you expect of a virtual machine in other environments. 180 | 1. Docker containers on Triton preserve their IP and any data on disk when they reboot. 181 | 1. Linked containers in Docker Compose on Triton are distributed across multiple unique physical nodes for maximum availability in the case of node failures. 182 | 183 | ## Consul encryption 184 | 185 | Consul supports TLS encryption for RPC and symmetric pre-shared key encryption for its gossip protocol. Deploying these features requires managing these secrets, and a demonstration of how to do so can be found in the [Vault example](https://github.com/autopilotpattern/vault). 186 | 187 | ### Testing 188 | 189 | The `tests/` directory includes integration tests for both the Triton and Compose example stacks described above. Build the test runner by making sure you've pulled down the submodule with `git submodule update --init` and then `make build/tester`. 190 | 191 | Running `make test/triton` will run the tests in a container locally but targeting Triton Cloud. To run those tests you'll need a Triton Cloud account with your Triton command line profile set up. The test rig will use the value of the `TRITON_PROFILE` environment variable to determine what data center to target. The tests use your own credentials mounted from your Docker host (your laptop, for example), so if you have a passphrase on your ssh key you'll need to add `-it` to the arguments of the `test/triton` Make target. 192 | 193 | ## Credit where it's due 194 | 195 | This project builds on the fine examples set by [Jeff Lindsay](https://github.com/progrium)'s ([Glider Labs](https://github.com/gliderlabs)) [Consul in Docker](https://github.com/gliderlabs/docker-consul/tree/legacy) work. It also, obviously, wouldn't be possible without the outstanding work of the [Hashicorp team](https://hashicorp.com) that made [consul.io](https://www.consul.io). 196 | -------------------------------------------------------------------------------- /bin/consul-manage: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eo pipefail 3 | 4 | # 5 | # Update the -advertise address based on the interface that ContainerPilot has 6 | # been told to listen on. 7 | # 8 | preStart() { 9 | 10 | if [ -n "$CONSUL_DATACENTER_NAME" ]; then 11 | _log "Updating consul datacenter name (specified: '${CONSUL_DATACENTER_NAME}' )" 12 | sed -i "s/CONSUL_DATACENTER_NAME/${CONSUL_DATACENTER_NAME}/" /etc/consul/consul.hcl 13 | elif [ -f "/native/usr/sbin/mdata-get" ]; then 14 | DETECTED_DATACENTER_NAME=$(/native/usr/sbin/mdata-get sdc:datacenter_name) 15 | _log "Updating consul datacenter name (detected: '${DETECTED_DATACENTER_NAME}')" 16 | sed -i "s/CONSUL_DATACENTER_NAME/${DETECTED_DATACENTER_NAME}/" /etc/consul/consul.hcl 17 | else 18 | _log "Updating consul datacenter name (default: 'dc1')" 19 | sed -i "s/CONSUL_DATACENTER_NAME/dc1/" /etc/consul/consul.hcl 20 | fi 21 | 22 | if [ -n "$CONSUL_RETRY_JOIN_WAN" ]; then 23 | _log "Updating consul retry_join_wan field" 24 | sed -i '/^retry_join_wan/d' /etc/consul/consul.hcl 25 | echo "retry_join_wan = [${CONSUL_RETRY_JOIN_WAN}]" >> /etc/consul/consul.hcl 26 | 27 | # translate_wan_addrs allows us to reach remote nodes through their advertise_addr_wan 28 | sed -i '/^translate_wan_addrs/d' /etc/consul/consul.hcl 29 | _log "Updating consul translate_wan_addrs field" 30 | echo "translate_wan_addrs = true" >> /etc/consul/consul.hcl 31 | 32 | # only set bind_addr = 0.0.0.0 if none was specified explicitly with CONSUL_BIND_ADDR 33 | if [ -n "$CONSUL_BIND_ADDR" ]; then 34 | updateConfigFromEnvOrDefault 'bind_addr' 'CONSUL_BIND_ADDR' "$CONTAINERPILOT_CONSUL_IP" 35 | else 36 | sed -i '/^bind_addr/d' /etc/consul/consul.hcl 37 | _log "Updating consul field bind_addr to 0.0.0.0 CONSUL_BIND_ADDR was empty and CONSUL_RETRY_JOIN_WAN was not empty" 38 | echo "bind_addr = \"0.0.0.0\"" >> /etc/consul/consul.hcl 39 | fi 40 | else 41 | # if no WAN addresses were provided, set the bind_addr to the private address 42 | updateConfigFromEnvOrDefault 'bind_addr' 'CONSUL_BIND_ADDR' "$CONTAINERPILOT_CONSUL_IP" 43 | fi 44 | 45 | IP_ADDRESS=$(hostname -i) 46 | 47 | # the serf_lan_bind field was recently renamed to serf_wan 48 | # serf_lan tells nodes their address within the LAN 49 | updateConfigFromEnvOrDefault 'serf_lan' 'CONSUL_SERF_LAN_BIND' "$CONTAINERPILOT_CONSUL_IP" 50 | 51 | # the serf_wan_bind field was recently renamed to serf_wan 52 | # if this field is not set WAN joins will be refused since the bind address will differ 53 | # from the address used to reach the node 54 | updateConfigFromEnvOrDefault 'serf_wan' 'CONSUL_SERF_WAN_BIND' "$IP_ADDRESS" 55 | 56 | # advertise_addr tells nodes their private, routeable address 57 | updateConfigFromEnvOrDefault 'advertise_addr' 'CONSUL_ADVERTISE_ADDR' "$CONTAINERPILOT_CONSUL_IP" 58 | 59 | # advertise_addr_wan tells nodes their public address for WAN communication 60 | updateConfigFromEnvOrDefault 'advertise_addr_wan' 'CONSUL_ADVERTISE_ADDR_WAN' "$IP_ADDRESS" 61 | } 62 | 63 | # 64 | # Check if a member of a raft. If consul info returns an error we'll pipefail 65 | # and exit for a failed health check. 66 | # 67 | # If we have no peers then try to join the raft via the CNS svc record. Once a 68 | # node is connected to at least one other peer it'll get the rest of the raft 69 | # via the Consul LAN gossip. 70 | # 71 | # If we end up joining ourselves we just retry on the next health check until 72 | # we've got the whole cluster together. 73 | # 74 | health() { 75 | if [ $(consul info | awk '/num_peers/{print$3}') == 0 ]; then 76 | _log "No peers in raft" 77 | consul join ${CONSUL} 78 | fi 79 | } 80 | 81 | _log() { 82 | echo " $(date -u '+%Y-%m-%d %H:%M:%S') containerpilot: $@" 83 | } 84 | 85 | 86 | # 87 | # Defines $1 in the consul configuration as either an env or a default. 88 | # This basically behaves like ${!name_of_var} and ${var:-default} together 89 | # but separates the indirect reference from the default so it's more obvious 90 | # 91 | # Check if $2 is the name of a defined environment variable and use ${!2} to 92 | # reference it indirectly. 93 | # 94 | # If it is not defined, use $3 as the value 95 | # 96 | updateConfigFromEnvOrDefault() { 97 | _log "Updating consul field $1" 98 | sed -i "/^$1/d" /etc/consul/consul.hcl 99 | 100 | if [ -n "${!2}" ]; then 101 | echo "$1 = \"${!2}\"" >> /etc/consul/consul.hcl 102 | else 103 | echo "$1 = \"$3\"" >> /etc/consul/consul.hcl 104 | fi 105 | } 106 | 107 | # --------------------------------------------------- 108 | # parse arguments 109 | 110 | # Get function list 111 | funcs=($(declare -F -p | cut -d " " -f 3)) 112 | 113 | until 114 | if [ ! -z "$1" ]; then 115 | # check if the first arg is a function in this file, or use a default 116 | if [[ " ${funcs[@]} " =~ " $1 " ]]; then 117 | cmd=$1 118 | shift 1 119 | fi 120 | 121 | $cmd "$@" 122 | if [ $? == 127 ]; then 123 | help 124 | fi 125 | 126 | exit 127 | else 128 | health 129 | fi 130 | do 131 | echo 132 | done 133 | -------------------------------------------------------------------------------- /etc/consul.hcl: -------------------------------------------------------------------------------- 1 | bind_addr = "0.0.0.0" 2 | datacenter = "CONSUL_DATACENTER_NAME" 3 | data_dir = "/data" 4 | client_addr = "0.0.0.0" 5 | ports { 6 | dns = 53 7 | } 8 | recursors = ["8.8.8.8", "8.8.4.4"] 9 | disable_update_check = true 10 | -------------------------------------------------------------------------------- /etc/containerpilot.json5: -------------------------------------------------------------------------------- 1 | { 2 | consul: "127.0.0.1:8500", 3 | jobs: [ 4 | { 5 | name: "preStart", 6 | exec: ["/usr/local/bin/consul-manage", "preStart"], 7 | }, 8 | { 9 | name: "consul", 10 | port: 8500, 11 | {{ if .CONSUL_DEV }}exec: [ 12 | "/bin/consul", "agent", 13 | "-dev", 14 | "-config-dir=/etc/consul"], 15 | {{ else }}exec: [ 16 | "/bin/consul", "agent", 17 | "-server", 18 | "-bootstrap-expect", "3", 19 | "-config-dir=/etc/consul", 20 | "-ui"],{{ end }} 21 | when: { 22 | source: "preStart", 23 | once: "exitSuccess" 24 | }, 25 | health:{ 26 | exec: ["/usr/local/bin/consul-manage", "health"], 27 | interval: 10, 28 | ttl: 25 29 | } 30 | }, 31 | { 32 | name: "preStop", 33 | exec: ["consul", "leave"], 34 | when: { 35 | source: "consul", 36 | once: "stopping" 37 | } 38 | } 39 | ] 40 | } 41 | -------------------------------------------------------------------------------- /examples/compose/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | 3 | services: 4 | 5 | # Service definition for Consul cluster with a minimum of 3 nodes. 6 | # For local development we use Compose v2 so that we have an automatically 7 | # created user-defined network and internal DNS for the name "consul". 8 | # Nodes will use Docker DNS for the service (passed in via the CONSUL 9 | # env var) to find each other and bootstrap the cluster. 10 | # Note: Unless CONSUL_DEV is set, at least three instances are required for quorum. 11 | consul: 12 | image: autopilotpattern/consul:${TAG:-latest} 13 | restart: always 14 | mem_limit: 128m 15 | ports: 16 | - 8500 17 | environment: 18 | - CONSUL=consul 19 | - CONSUL_DATACENTER_NAME=dc1 20 | command: > 21 | /usr/local/bin/containerpilot 22 | -------------------------------------------------------------------------------- /examples/triton-multi-dc/docker-compose-multi-dc.yml.template: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | 3 | services: 4 | 5 | # Service definition for Consul cluster running in us-east-1. 6 | # Cloned by ../../setup-multi-datacenter.sh once per profile 7 | consul: 8 | image: autopilotpattern/consul:${TAG:-latest} 9 | labels: 10 | - triton.cns.services=consul 11 | - com.docker.swarm.affinities=["container!=~*consul*"] 12 | restart: always 13 | mem_limit: 128m 14 | ports: 15 | - 8300 # Server RPC port 16 | - "8302/tcp" # Serf WAN port 17 | - "8302/udp" # Serf WAN port 18 | - 8500 19 | env_file: 20 | - ENV_FILE_NAME 21 | network_mode: bridge 22 | command: > 23 | /usr/local/bin/containerpilot -------------------------------------------------------------------------------- /examples/triton-multi-dc/setup-multi-dc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e -o pipefail 3 | 4 | help() { 5 | echo 6 | echo 'Usage ./setup-multi-datacenter.sh [ [...]]' 7 | echo 8 | echo 'Generates one _env file and docker-compose.yml file per triton profile, each of which' 9 | echo 'is presumably associated with a different datacenter.' 10 | } 11 | 12 | if [ "$#" -lt 1 ]; then 13 | help 14 | exit 1 15 | fi 16 | 17 | # --------------------------------------------------- 18 | # Top-level commands 19 | 20 | # 21 | # Check for triton profile $1 and output _env file named $2 22 | # 23 | generate_env() { 24 | local triton_profile=$1 25 | local output_file=$2 26 | 27 | command -v docker >/dev/null 2>&1 || { 28 | echo 29 | tput rev # reverse 30 | tput bold # bold 31 | echo 'Docker is required, but does not appear to be installed.' 32 | tput sgr0 # clear 33 | echo 'See https://docs.joyent.com/public-cloud/api-access/docker' 34 | exit 1 35 | } 36 | command -v triton >/dev/null 2>&1 || { 37 | echo 38 | tput rev # reverse 39 | tput bold # bold 40 | echo 'Error! Joyent Triton CLI is required, but does not appear to be installed.' 41 | tput sgr0 # clear 42 | echo 'See https://www.joyent.com/blog/introducing-the-triton-command-line-tool' 43 | exit 1 44 | } 45 | 46 | # make sure Docker client is pointed to the same place as the Triton client 47 | local docker_user=$(docker info 2>&1 | awk -F": " '/SDCAccount:/{print $2}') 48 | local docker_dc=$(echo $DOCKER_HOST | awk -F"/" '{print $3}' | awk -F'.' '{print $1}') 49 | 50 | local triton_user=$(triton profile get $triton_profile | awk -F": " '/account:/{print $2}') 51 | local triton_dc=$(triton profile get $triton_profile | awk -F"/" '/url:/{print $3}' | awk -F'.' '{print $1}') 52 | local triton_account=$(TRITON_PROFILE=$triton_profile triton account get | awk -F": " '/id:/{print $2}') 53 | 54 | if [ ! "$docker_user" = "$triton_user" ] || [ ! "$docker_dc" = "$triton_dc" ]; then 55 | echo 56 | tput rev # reverse 57 | tput bold # bold 58 | echo 'Error! The Triton CLI configuration does not match the Docker CLI configuration.' 59 | tput sgr0 # clear 60 | echo 61 | echo "Docker user: ${docker_user}" 62 | echo "Triton user: ${triton_user}" 63 | echo "Docker data center: ${docker_dc}" 64 | echo "Triton data center: ${triton_dc}" 65 | exit 1 66 | fi 67 | 68 | local triton_cns_enabled=$(triton account get | awk -F": " '/cns/{print $2}') 69 | if [ ! "true" == "$triton_cns_enabled" ]; then 70 | echo 71 | tput rev # reverse 72 | tput bold # bold 73 | echo 'Error! Triton CNS is required and not enabled.' 74 | tput sgr0 # clear 75 | echo 76 | exit 1 77 | fi 78 | 79 | # setup environment file 80 | if [ ! -f "$output_file" ]; then 81 | echo '# Consul bootstrap via Triton CNS' >> $output_file 82 | echo CONSUL=consul.svc.${triton_account}.${triton_dc}.cns.joyent.com >> $output_file 83 | echo >> $output_file 84 | else 85 | echo "Existing _env file found at $1, exiting" 86 | exit 87 | fi 88 | } 89 | 90 | 91 | declare -a written 92 | declare -a consul_hostnames 93 | 94 | # check that we won't overwrite any _env files first 95 | if [ -f "_env" ]; then 96 | echo "Existing env file found, exiting: _env" 97 | fi 98 | 99 | # check the names of _env files we expect to generate 100 | for profile in "$@" 101 | do 102 | if [ -f "_env-$profile" ]; then 103 | echo "Existing env file found, exiting: _env-$profile" 104 | exit 2 105 | fi 106 | 107 | if [ -f "_env-$profile" ]; then 108 | echo "Existing env file found, exiting: _env-$profile" 109 | exit 3 110 | fi 111 | 112 | if [ -f "docker-compose-$profile.yml" ]; then 113 | echo "Existing docker-compose file found, exiting: docker-compose-$profile.yml" 114 | exit 4 115 | fi 116 | done 117 | 118 | # check that the docker-compose.yml template is in the right place 119 | if [ ! -f "docker-compose-multi-dc.yml.template" ]; then 120 | echo "Multi-datacenter docker-compose.yml template is missing!" 121 | exit 5 122 | fi 123 | 124 | echo "profiles: $@" 125 | 126 | # invoke ./setup.sh once per profile 127 | for profile in "$@" 128 | do 129 | echo "Temporarily switching profile: $profile" 130 | eval "$(TRITON_PROFILE=$profile triton env -d)" 131 | generate_env $profile "_env-$profile" 132 | 133 | unset CONSUL 134 | source "_env-$profile" 135 | 136 | consul_hostnames+=("\"${CONSUL//cns.joyent.com/triton.zone}\"") 137 | 138 | cp docker-compose-multi-dc.yml.template \ 139 | "docker-compose-$profile.yml" 140 | 141 | sed -i '' "s/ENV_FILE_NAME/_env-$profile/" "docker-compose-$profile.yml" 142 | 143 | written+=("_env-$profile") 144 | done 145 | 146 | 147 | # finalize _env and prepare docker-compose.yml files 148 | for profile in "$@" 149 | do 150 | # add the CONSUL_RETRY_JOIN_WAN addresses to each _env 151 | echo '# Consul multi-DC bootstrap via Triton CNS' >> _env-$profile 152 | echo "CONSUL_RETRY_JOIN_WAN=$(IFS=,; echo "${consul_hostnames[*]}")" >> _env-$profile 153 | 154 | cp docker-compose-multi-dc.yml.template \ 155 | "docker-compose-$profile.yml" 156 | 157 | sed -i '' "s/ENV_FILE_NAME/_env-$profile/" "docker-compose-$profile.yml" 158 | done 159 | 160 | echo "Wrote: ${written[@]}" 161 | -------------------------------------------------------------------------------- /examples/triton/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | 3 | services: 4 | 5 | # Service definition for Consul cluster with a minimum of 3 nodes. 6 | # Nodes will use Triton CNS for the service (passed in via the CONSUL 7 | # env var) to find each other and bootstrap the cluster. 8 | consul: 9 | image: autopilotpattern/consul:${TAG:-latest} 10 | labels: 11 | - triton.cns.services=consul 12 | - com.docker.swarm.affinities=["container!=~*consul*"] 13 | restart: always 14 | mem_limit: 128m 15 | ports: 16 | - 8500 17 | env_file: 18 | - _env 19 | network_mode: bridge 20 | command: > 21 | /usr/local/bin/containerpilot 22 | -------------------------------------------------------------------------------- /examples/triton/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e -o pipefail 3 | 4 | help() { 5 | echo 6 | echo 'Usage ./setup.sh' 7 | echo 8 | echo 'Checks that your Triton and Docker environment is sane and configures' 9 | echo 'an environment file to use.' 10 | } 11 | 12 | # populated by `check` function whenever we're using Triton 13 | TRITON_USER= 14 | TRITON_DC= 15 | TRITON_ACCOUNT= 16 | 17 | # --------------------------------------------------- 18 | # Top-level commands 19 | 20 | # Check for correct configuration and setup _env file 21 | check() { 22 | 23 | command -v docker >/dev/null 2>&1 || { 24 | echo 25 | tput rev # reverse 26 | tput bold # bold 27 | echo 'Docker is required, but does not appear to be installed.' 28 | tput sgr0 # clear 29 | echo 'See https://docs.joyent.com/public-cloud/api-access/docker' 30 | exit 1 31 | } 32 | command -v triton >/dev/null 2>&1 || { 33 | echo 34 | tput rev # reverse 35 | tput bold # bold 36 | echo 'Error! Joyent Triton CLI is required, but does not appear to be installed.' 37 | tput sgr0 # clear 38 | echo 'See https://www.joyent.com/blog/introducing-the-triton-command-line-tool' 39 | exit 1 40 | } 41 | 42 | # make sure Docker client is pointed to the same place as the Triton client 43 | local docker_user=$(docker info 2>&1 | awk -F": " '/SDCAccount:/{print $2}') 44 | local docker_dc=$(echo $DOCKER_HOST | awk -F"/" '{print $3}' | awk -F'.' '{print $1}') 45 | 46 | TRITON_USER=$(triton profile get | awk -F": " '/account:/{print $2}') 47 | TRITON_DC=$(triton profile get | awk -F"/" '/url:/{print $3}' | awk -F'.' '{print $1}') 48 | TRITON_ACCOUNT=$(triton account get | awk -F": " '/id:/{print $2}') 49 | if [ ! "$docker_user" = "$TRITON_USER" ] || [ ! "$docker_dc" = "$TRITON_DC" ]; then 50 | echo 51 | tput rev # reverse 52 | tput bold # bold 53 | echo 'Error! The Triton CLI configuration does not match the Docker CLI configuration.' 54 | tput sgr0 # clear 55 | echo 56 | echo "Docker user: ${docker_user}" 57 | echo "Triton user: ${TRITON_USER}" 58 | echo "Docker data center: ${docker_dc}" 59 | echo "Triton data center: ${TRITON_DC}" 60 | exit 1 61 | fi 62 | 63 | local triton_cns_enabled=$(triton account get | awk -F": " '/cns/{print $2}') 64 | if [ ! "true" == "$triton_cns_enabled" ]; then 65 | echo 66 | tput rev # reverse 67 | tput bold # bold 68 | echo 'Error! Triton CNS is required and not enabled.' 69 | tput sgr0 # clear 70 | echo 71 | exit 1 72 | fi 73 | 74 | # setup environment file 75 | if [ ! -f "_env" ]; then 76 | echo '# Consul bootstrap via Triton CNS' >> _env 77 | echo CONSUL=consul.svc.${TRITON_ACCOUNT}.${TRITON_DC}.cns.joyent.com >> _env 78 | echo >> _env 79 | else 80 | echo 'Existing _env file found, exiting' 81 | exit 82 | fi 83 | } 84 | 85 | # --------------------------------------------------- 86 | # parse arguments 87 | 88 | # Get function list 89 | funcs=($(declare -F -p | cut -d " " -f 3)) 90 | 91 | until 92 | if [ ! -z "$1" ]; then 93 | # check if the first arg is a function in this file, or use a default 94 | if [[ " ${funcs[@]} " =~ " $1 " ]]; then 95 | cmd=$1 96 | shift 1 97 | fi 98 | 99 | $cmd "$@" 100 | if [ $? == 127 ]; then 101 | help 102 | fi 103 | 104 | exit 105 | else 106 | check 107 | fi 108 | do 109 | echo 110 | done 111 | -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | # Makefile for shipping and testing the container image. 2 | 3 | MAKEFLAGS += --warn-undefined-variables 4 | .DEFAULT_GOAL := build 5 | .PHONY: * 6 | 7 | # we get these from CI environment if available, otherwise from git 8 | GIT_COMMIT ?= $(shell git rev-parse --short HEAD) 9 | GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) 10 | 11 | namespace ?= autopilotpattern 12 | tag := branch-$(shell basename $(GIT_BRANCH)) 13 | image := $(namespace)/consul 14 | testImage := $(namespace)/consul-testrunner 15 | 16 | ## Display this help message 17 | help: 18 | @awk '/^##.*$$/,/[a-zA-Z_-]+:/' $(MAKEFILE_LIST) | awk '!(NR%2){print $$0p}{p=$$0}' | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' | sort 19 | 20 | 21 | # ------------------------------------------------ 22 | # Container builds 23 | 24 | ## Builds the application container image locally 25 | build: build/tester 26 | docker build -t=$(image):$(tag) . 27 | 28 | ## Build the test running container 29 | build/tester: 30 | docker build -f test/Dockerfile -t=$(testImage):$(tag) . 31 | 32 | ## Push the current application container images to the Docker Hub 33 | push: 34 | docker push $(image):$(tag) 35 | docker push $(testImage):$(tag) 36 | 37 | ## Tag the current images as 'latest' and push them to the Docker Hub 38 | ship: 39 | docker tag $(image):$(tag) $(image):latest 40 | docker tag $(testImage):$(tag) $(testImage):latest 41 | docker tag $(image):$(tag) $(image):latest 42 | docker push $(image):$(tag) 43 | docker push $(image):latest 44 | 45 | 46 | # ------------------------------------------------ 47 | # Test running 48 | 49 | ## Pull the container images from the Docker Hub 50 | pull: 51 | docker pull $(image):$(tag) 52 | 53 | ## Run all integration tests 54 | test: test/compose test/triton 55 | 56 | ## Run the integration test runner against Compose locally. 57 | test/compose: 58 | docker run --rm \ 59 | -e TAG=$(tag) \ 60 | -e GIT_BRANCH=$(GIT_BRANCH) \ 61 | --network=bridge \ 62 | -v /var/run/docker.sock:/var/run/docker.sock \ 63 | -w /src \ 64 | $(testImage):$(tag) /src/compose.sh 65 | 66 | ## Run the integration test runner. Runs locally but targets Triton. 67 | test/triton: 68 | $(call check_var, TRITON_PROFILE, \ 69 | required to run integration tests on Triton.) 70 | docker run --rm \ 71 | -e TAG=$(tag) \ 72 | -e TRITON_PROFILE=$(TRITON_PROFILE) \ 73 | -e GIT_BRANCH=$(GIT_BRANCH) \ 74 | -v ~/.ssh:/root/.ssh:ro \ 75 | -v ~/.triton/profiles.d:/root/.triton/profiles.d:ro \ 76 | -w /src \ 77 | $(testImage):$(tag) /src/triton.sh 78 | 79 | # runs the integration test above but entirely within your local 80 | # development environment rather than the clean test rig 81 | test/triton/dev: 82 | ./test/triton.sh 83 | 84 | 85 | # ------------------------------------------------ 86 | # Multi-datacenter usage 87 | clean/triton-multi-dc: 88 | rm -rf examples/triton-multi-dc/_env* examples/triton-multi-dc/docker-compose-*.yml 89 | 90 | 91 | ## Print environment for build debugging 92 | debug: 93 | @echo GIT_COMMIT=$(GIT_COMMIT) 94 | @echo GIT_BRANCH=$(GIT_BRANCH) 95 | @echo namespace=$(namespace) 96 | @echo tag=$(tag) 97 | @echo image=$(image) 98 | @echo testImage=$(testImage) 99 | 100 | check_var = $(foreach 1,$1,$(__check_var)) 101 | __check_var = $(if $(value $1),,\ 102 | $(error Missing $1 $(if $(value 2),$(strip $2)))) 103 | -------------------------------------------------------------------------------- /test/Dockerfile: -------------------------------------------------------------------------------- 1 | # NOTE: this Dockerfile needs to be run from one-level up so that 2 | # we get the examples docker-compose.yml files. Use 'make build/tester' 3 | # in the makefile at the root of this repo and everything will work 4 | 5 | FROM alpine:3.6 6 | 7 | RUN apk update \ 8 | && apk add nodejs nodejs-npm python3 openssl bash curl docker 9 | RUN npm install -g triton json 10 | 11 | # the Compose package in the public releases doesn't work on Alpine 12 | RUN pip3 install docker-compose==1.10.0 13 | 14 | # install specific version of Docker and Compose client 15 | COPY test/triton-docker-cli/triton-docker /usr/local/bin/triton-docker 16 | RUN sed -i 's/1.9.0/1.10.0/' /usr/local/bin/triton-docker \ 17 | && ln -s /usr/local/bin/triton-docker /usr/local/bin/triton-compose \ 18 | && ln -s /usr/local/bin/triton-docker /usr/local/bin/triton-docker-install \ 19 | && /usr/local/bin/triton-docker-install \ 20 | && rm /usr/local/bin/triton-compose-helper \ 21 | && ln -s /usr/bin/docker-compose /usr/local/bin/triton-compose-helper 22 | 23 | 24 | # install test targets 25 | COPY examples/compose/docker-compose.yml /src/local-compose.yml 26 | COPY examples/triton/docker-compose.yml /src/docker-compose.yml 27 | 28 | # install test code 29 | COPY test/triton.sh /src/triton.sh 30 | COPY test/compose.sh /src/compose.sh 31 | COPY examples/triton/setup.sh /src/setup.sh 32 | -------------------------------------------------------------------------------- /test/compose.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | export GIT_BRANCH="${GIT_BRANCH:-$(git rev-parse --abbrev-ref HEAD)}" 5 | export TAG="${TAG:-branch-$(basename "$GIT_BRANCH")}" 6 | export COMPOSE_PROJECT="${COMPOSE_PROJECT_NAME:-consul}" 7 | export COMPOSE_FILE="${COMPOSE_FILE:-./local-compose.yml}" 8 | 9 | project="$COMPOSE_PROJECT" 10 | manifest="$COMPOSE_FILE" 11 | 12 | fail() { 13 | echo 14 | echo '------------------------------------------------' 15 | echo 'FAILED: dumping logs' 16 | echo '------------------------------------------------' 17 | docker-compose -p "$project" -f "$manifest" ps 18 | docker-compose -p "$project" -f "$manifest" logs 19 | echo '------------------------------------------------' 20 | echo 'FAILED' 21 | echo "$1" 22 | echo '------------------------------------------------' 23 | exit 1 24 | } 25 | 26 | pass() { 27 | teardown 28 | echo 29 | echo '------------------------------------------------' 30 | echo 'PASSED!' 31 | echo 32 | exit 0 33 | } 34 | 35 | function finish { 36 | result=$? 37 | if [ $result -ne 0 ]; then fail "unexpected error"; fi 38 | pass 39 | } 40 | trap finish EXIT 41 | 42 | 43 | 44 | # -------------------------------------------------------------------- 45 | # Helpers 46 | 47 | # asserts that 'count' Consul instances are running and marked as Up 48 | # by Docker. fails after the timeout. 49 | wait_for_containers() { 50 | local count timeout i got 51 | count="$1" 52 | timeout="${3:-60}" # default 60sec 53 | i=0 54 | echo "waiting for $count Consul containers to be Up..." 55 | while [ $i -lt "$timeout" ]; do 56 | got=$(docker-compose -p "$project" -f "$manifest" ps consul) 57 | got=$(echo "$got" | grep -c "Up") 58 | if [ "$got" -eq "$count" ]; then 59 | echo "$count instances reported Up in <= $i seconds" 60 | return 61 | fi 62 | i=$((i+1)) 63 | sleep 1 64 | done 65 | fail "$count instances did not report Up within $timeout seconds" 66 | } 67 | 68 | # asserts that the raft has become healthy with 'count' instances 69 | # and an elected leader. Queries Consul to determine the status 70 | # of the raft. Compares the status against a list of containers 71 | # and verifies that the leader is among those. 72 | wait_for_cluster() { 73 | local count timeout i got 74 | count="$1" 75 | timeout="${2:-60}" # default 60sec 76 | i=0 77 | echo "waiting for raft w/ $count instances to converge..." 78 | while [ $i -lt "$timeout" ]; do 79 | leader=$(docker exec -i "${project}_consul_1" \ 80 | curl -s "http://localhost:8500/v1/status/leader") 81 | leader=$(echo "$leader"| json) 82 | if [[ "$leader" != "" ]]; then 83 | peers=$(docker exec -i "${project}_consul_1" \ 84 | curl -s "http://localhost:8500/v1/status/peers") 85 | peers=$(echo "$peers"| json -a) 86 | peer_count=$(echo "$peers" | wc -l | tr -d ' ') 87 | if [ "$peer_count" -eq "$count" ]; then 88 | echo "$peers" | grep -q "$leader" 89 | if [ $? -eq 0 ]; then 90 | echo "raft converged in <= $i seconds w/ leader $leader" 91 | return 92 | fi 93 | fi 94 | fi 95 | i=$((i+1)) 96 | sleep 1 97 | done 98 | fail "raft w/ $count instances did not converge within $timeout seconds" 99 | } 100 | 101 | restart() { 102 | node="${project}_$1" 103 | docker restart "$node" 104 | } 105 | 106 | netsplit() { 107 | # it's a bit of a pain to netsplit this container without extra privileges, 108 | # or doing some non-portable stuff in the underlying VM, so instead we'll 109 | # pause the container which will cause its TTL to expire 110 | echo "netsplitting ${project}_$1" 111 | docker pause "${project}_$1" 112 | } 113 | 114 | heal() { 115 | echo "healing netsplit for ${project}_$1" 116 | docker unpause "${project}_$1" 117 | } 118 | 119 | run() { 120 | echo 121 | echo '* cleaning up previous test run' 122 | echo 123 | docker-compose -p "$project" -f "$manifest" stop 124 | docker-compose -p "$project" -f "$manifest" rm -f 125 | 126 | echo 127 | echo '* standing up initial test targets' 128 | echo 129 | docker-compose -p "$project" -f "$manifest" up -d 130 | } 131 | 132 | teardown() { 133 | echo 134 | echo '* tearing down containers' 135 | echo 136 | docker-compose -p "$project" -f "$manifest" stop 137 | docker-compose -p "$project" -f "$manifest" rm -f 138 | } 139 | 140 | scale() { 141 | count="$1" 142 | echo 143 | echo '* scaling up cluster' 144 | echo 145 | docker-compose -p "$project" -f "$manifest" scale consul="$count" 146 | } 147 | 148 | # -------------------------------------------------------------------- 149 | # Test sections 150 | 151 | test-rejoin-raft() { 152 | count="$1" 153 | echo 154 | echo '------------------------------------------------' 155 | echo "executing rejoin-raft test with $count nodes" 156 | echo '------------------------------------------------' 157 | run 158 | scale "$count" 159 | wait_for_containers "$count" 160 | wait_for_cluster "$count" 161 | 162 | restart "consul_1" 163 | wait_for_containers "$count" 164 | wait_for_cluster "$count" 165 | } 166 | 167 | test-graceful-leave() { 168 | echo 169 | echo '------------------------------------------------' 170 | echo "executing graceful-leave test with 5 nodes" 171 | echo '------------------------------------------------' 172 | run 173 | scale 5 174 | wait_for_containers 5 175 | wait_for_cluster 5 176 | 177 | echo 178 | echo '* writing key' 179 | docker exec -i "${project}_consul_1" \ 180 | curl --fail -s -o /dev/null -XPUT --data "hello" \ 181 | "http://localhost:8500/v1/kv/test_grace" 182 | 183 | echo 184 | echo '* gracefully stopping 3 nodes of cluster' 185 | docker stop "${project}_consul_3" 186 | docker stop "${project}_consul_4" 187 | docker stop "${project}_consul_5" 188 | wait_for_containers 2 189 | 190 | echo 191 | echo '* checking consistent read' 192 | consistent=$(docker exec -i "${project}_consul_1" \ 193 | curl -s "http://localhost:8500/v1/kv/test_grace?consistent") 194 | if [[ "$consistent" == "aGVsbG8=" ]]; then 195 | fail "got '${consistent}' back from query; should not have cluster leader after 3 nodes" 196 | fi 197 | 198 | echo '* checking stale read' 199 | stale=$(docker exec -i "${project}_consul_1" \ 200 | curl -s "http://localhost:8500/v1/kv/test_grace?stale") 201 | stale=$(echo "$stale" | json -a Value) 202 | # this value is "hello" base64 encoded 203 | if [[ "$stale" != "aGVsbG8=" ]]; then 204 | fail "got '${stale}' back from query; could not get stale key after 3 nodes gracefully exit" 205 | fi 206 | } 207 | 208 | test-quorum-consistency() { 209 | echo 210 | echo '------------------------------------------------' 211 | echo "executing quorum-consistency test with 5 nodes" 212 | echo '------------------------------------------------' 213 | run 214 | scale 5 215 | wait_for_containers 5 216 | wait_for_cluster 5 217 | 218 | echo 219 | echo '* writing key' 220 | docker exec "${project}_consul_1" \ 221 | curl --fail -s -o /dev/null -XPUT --data "hello" \ 222 | "http://localhost:8500/v1/kv/test_grace" 223 | 224 | echo 225 | echo '* netsplitting 3 nodes of cluster' 226 | netsplit "consul_3" 227 | netsplit "consul_4" 228 | netsplit "consul_5" 229 | 230 | echo 231 | echo '* checking consistent read' 232 | consistent=$(docker exec -i "${project}_consul_1" \ 233 | curl -s "http://localhost:8500/v1/kv/test_grace?consistent") 234 | if [[ "$consistent" == "aGVsbG8=" ]]; then 235 | fail "got '${consistent}' back from query; should not have cluster leader after 3 nodes" 236 | fi 237 | 238 | echo 239 | echo '* checking write to isolated node' 240 | set +e 241 | docker exec -i "${project}_consul_1" \ 242 | curl --fail -XPUT -d someval localhost:8500/kv/somekey 243 | result=$? 244 | set -e 245 | if [ "$result" -eq 0 ]; then 246 | fail 'was able to write to isolated node' 247 | fi 248 | 249 | echo '* checking stale read' 250 | stale=$(docker exec -i "${project}_consul_1" \ 251 | curl -s "http://localhost:8500/v1/kv/test_grace?stale") 252 | stale=$(echo "$stale" | json -a Value) 253 | # this value is "hello" base64 encoded 254 | if [[ "$stale" != "aGVsbG8=" ]]; then 255 | fail "got '${stale}' back from query; could not get stale key after 3 nodes netsplit" 256 | fi 257 | 258 | echo 259 | echo '* healing netsplit' 260 | heal "consul_3" 261 | heal "consul_4" 262 | heal "consul_5" 263 | wait_for_cluster 5 264 | 265 | echo '* checking consistent read' 266 | consistent=$(docker exec -i "${project}_consul_1" \ 267 | curl -s "http://localhost:8500/v1/kv/test_grace?consistent") 268 | consistent=$(echo "$consistent" | json -a Value) 269 | # this value is "hello" base64 encoded 270 | if [[ "$consistent" != "aGVsbG8=" ]]; then 271 | fail "got '${consistent}' back from query; could not get consistent key after recovery" 272 | fi 273 | } 274 | 275 | # -------------------------------------------------------------------- 276 | # Main loop 277 | 278 | test-rejoin-raft 3 279 | test-rejoin-raft 5 280 | test-graceful-leave 281 | test-quorum-consistency 282 | -------------------------------------------------------------------------------- /test/triton.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | export GIT_BRANCH="${GIT_BRANCH:-$(git rev-parse --abbrev-ref HEAD)}" 5 | export TAG="${TAG:-branch-$(basename "$GIT_BRANCH")}" 6 | export COMPOSE_PROJECT="${COMPOSE_PROJECT_NAME:-consul}" 7 | export COMPOSE_FILE="${COMPOSE_FILE:-./docker-compose.yml}" 8 | 9 | project="$COMPOSE_PROJECT" 10 | manifest="$COMPOSE_FILE" 11 | 12 | fail() { 13 | echo 14 | echo '------------------------------------------------' 15 | echo 'FAILED: dumping logs' 16 | echo '------------------------------------------------' 17 | triton-compose -p "$project" -f "$manifest" ps 18 | triton-compose -p "$project" -f "$manifest" logs 19 | echo '------------------------------------------------' 20 | echo 'FAILED' 21 | echo "$1" 22 | echo '------------------------------------------------' 23 | exit 1 24 | } 25 | 26 | pass() { 27 | teardown 28 | echo 29 | echo '------------------------------------------------' 30 | echo 'PASSED!' 31 | echo 32 | exit 0 33 | } 34 | 35 | function finish { 36 | result=$? 37 | if [ $result -ne 0 ]; then fail "unexpected error"; fi 38 | pass 39 | } 40 | trap finish EXIT 41 | 42 | 43 | 44 | # -------------------------------------------------------------------- 45 | # Helpers 46 | 47 | # asserts that 'count' Consul instances are running and marked as Up 48 | # by Triton. fails after the timeout. 49 | wait_for_containers() { 50 | local count timeout i got 51 | count="$1" 52 | timeout="${3:-60}" # default 60sec 53 | i=0 54 | echo "waiting for $count Consul containers to be Up..." 55 | while [ $i -lt "$timeout" ]; do 56 | got=$(triton-compose -p "$project" -f "$manifest" ps consul) 57 | got=$(echo "$got" | grep -c "Up") 58 | if [ "$got" -eq "$count" ]; then 59 | echo "$count instances reported Up in <= $i seconds" 60 | return 61 | fi 62 | i=$((i+1)) 63 | sleep 1 64 | done 65 | fail "$count instances did not report Up within $timeout seconds" 66 | } 67 | 68 | # asserts that the raft has become healthy with 'count' instances 69 | # and an elected leader. Queries Consul to determine the status 70 | # of the raft. Compares the status against a list of containers 71 | # and verifies that the leader is among those. 72 | wait_for_cluster() { 73 | local count timeout i got consul_ip 74 | count="$1" 75 | timeout="${2:-60}" # default 60sec 76 | i=0 77 | echo "waiting for raft w/ $count instances to converge..." 78 | consul_ip=$(triton ip "${project}_consul_1") 79 | while [ $i -lt "$timeout" ]; do 80 | leader=$(curl -s "http://${consul_ip}:8500/v1/status/leader") 81 | leader=$(echo "$leader" | json) 82 | if [[ "$leader" != "[]" ]]; then 83 | peers=$(curl -s "http://${consul_ip}:8500/v1/status/peers") 84 | peers=$(echo "$peers" | json -a) 85 | peer_count=$(echo "$peers" | wc -l | tr -d ' ') 86 | if [ "$peer_count" -eq "$count" ]; then 87 | echo "$peers" | grep -q "$leader" 88 | if [ $? -eq 0 ]; then 89 | echo "raft converged in <= $i seconds w/ leader $leader" 90 | return 91 | fi 92 | fi 93 | fi 94 | i=$((i+1)) 95 | sleep 1 96 | done 97 | fail "raft w/ $count instances did not converge within $timeout seconds" 98 | } 99 | 100 | restart() { 101 | node="${project}_$1" 102 | triton-docker restart "$node" 103 | } 104 | 105 | netsplit() { 106 | echo "netsplitting ${project}_$1" 107 | triton-docker exec "${project}_$1" ifconfig eth0 down 108 | } 109 | 110 | heal() { 111 | echo "healing netsplit for ${project}_$1" 112 | triton-docker exec "${project}_$1" ifconfig eth0 up 113 | } 114 | 115 | run() { 116 | echo 117 | echo '* cleaning up previous test run' 118 | echo 119 | triton-compose -p "$project" -f "$manifest" stop 120 | triton-compose -p "$project" -f "$manifest" rm -f 121 | 122 | echo 123 | echo '* standing up initial test targets' 124 | echo 125 | triton-compose -p "$project" -f "$manifest" up -d 126 | } 127 | 128 | teardown() { 129 | echo 130 | echo '* tearing down containers' 131 | echo 132 | triton-compose -p "$project" -f "$manifest" stop 133 | triton-compose -p "$project" -f "$manifest" rm -f 134 | } 135 | 136 | scale() { 137 | count="$1" 138 | echo 139 | echo '* scaling up cluster' 140 | echo 141 | triton-compose -p "$project" -f "$manifest" scale consul="$count" 142 | } 143 | 144 | # -------------------------------------------------------------------- 145 | # Test sections 146 | 147 | profile() { 148 | echo 149 | echo '------------------------------------------------' 150 | echo 'setting up profile for tests' 151 | echo '------------------------------------------------' 152 | echo 153 | export TRITON_PROFILE="${TRITON_PROFILE:-us-east-1}" 154 | set +e 155 | # if we're already set up for Docker this will fail noisily 156 | triton profile docker-setup -y "$TRITON_PROFILE" > /dev/null 2>&1 157 | set -e 158 | triton profile set-current "$TRITON_PROFILE" 159 | eval "$(triton env)" 160 | 161 | # print out for profile debugging 162 | env | grep DOCKER 163 | env | grep SDC 164 | env | grep TRITON 165 | 166 | bash /src/setup.sh 167 | } 168 | 169 | test-rejoin-raft() { 170 | count="$1" 171 | echo 172 | echo '------------------------------------------------' 173 | echo "executing rejoin-raft test with $count nodes" 174 | echo '------------------------------------------------' 175 | run 176 | scale "$count" 177 | wait_for_containers "$count" 178 | wait_for_cluster "$count" 179 | 180 | restart "consul_1" 181 | wait_for_containers "$count" 182 | wait_for_cluster "$count" 183 | } 184 | 185 | test-graceful-leave() { 186 | echo 187 | echo '------------------------------------------------' 188 | echo "executing graceful-leave test with 5 nodes" 189 | echo '------------------------------------------------' 190 | run 191 | scale 5 192 | wait_for_containers 5 193 | wait_for_cluster 5 194 | 195 | echo 196 | echo '* writing key' 197 | consul_ip=$(triton ip "${project}_consul_1") 198 | curl --fail -s -o /dev/null -XPUT --data "hello" \ 199 | "http://${consul_ip}:8500/v1/kv/test_grace" 200 | 201 | echo 202 | echo '* gracefully stopping 3 nodes of cluster' 203 | triton-docker stop "${project}_consul_3" 204 | triton-docker stop "${project}_consul_4" 205 | triton-docker stop "${project}_consul_5" 206 | wait_for_containers 2 207 | 208 | echo 209 | echo '* checking consistent read' 210 | consistent=$(curl -s "http://${consul_ip}:8500/v1/kv/test_grace?consistent") 211 | if [[ "$consistent" == "aGVsbG8=" ]]; then 212 | fail "got '${consistent}' back from query; should not have cluster leader after 3 nodes" 213 | fi 214 | 215 | echo '* checking stale read' 216 | stale=$(curl -s "http://${consul_ip}:8500/v1/kv/test_grace?stale") 217 | stale=$(echo "$stale" | json -a Value) 218 | # this value is "hello" base64 encoded 219 | if [[ "$stale" != "aGVsbG8=" ]]; then 220 | fail "got '${stale}' back from query; could not get stale key after 3 nodes gracefully exit" 221 | fi 222 | } 223 | 224 | test-quorum-consistency() { 225 | echo 226 | echo '------------------------------------------------' 227 | echo "executing quorum-consistency test with 5 nodes" 228 | echo '------------------------------------------------' 229 | run 230 | scale 5 231 | wait_for_containers 5 232 | wait_for_cluster 5 233 | 234 | echo 235 | echo '* writing key' 236 | consul_ip=$(triton ip "${project}_consul_1") 237 | curl --fail -s -o /dev/null -XPUT --data "hello" \ 238 | "http://${consul_ip}:8500/v1/kv/test_quorum" 239 | 240 | echo 241 | echo '* netsplitting 3 nodes of cluster' 242 | netsplit "consul_3" 243 | netsplit "consul_4" 244 | netsplit "consul_5" 245 | 246 | echo 247 | echo '* checking consistent read' 248 | consistent=$(curl -s "http://${consul_ip}:8500/v1/kv/test_quorum?consistent") 249 | if [[ "$consistent" == "aGVsbG8=" ]]; then 250 | fail "got '${consistent}' back from query; should not have cluster leader after 3 nodes" 251 | fi 252 | 253 | echo 254 | echo '* checking write to isolated node' 255 | set +e 256 | docker exec -it "${project}_consul_1" \ 257 | curl --fail -XPUT -d someval localhost:8500/kv/somekey 258 | result=$? 259 | set -e 260 | if [ "$result" -eq 0 ]; then 261 | fail 'was able to write to isolated node' 262 | fi 263 | 264 | echo '* checking stale read' 265 | stale=$(curl -s "http://${consul_ip}:8500/v1/kv/test_quorum?stale") 266 | stale=$(echo "$stale" | json -a Value) 267 | # this value is "hello" base64 encoded 268 | if [[ "$stale" != "aGVsbG8=" ]]; then 269 | fail "got '${stale}' back from query; could not get stale key after 3 nodes netsplit" 270 | fi 271 | 272 | echo 273 | echo '* healing netsplit' 274 | heal "consul_3" 275 | heal "consul_4" 276 | heal "consul_5" 277 | wait_for_cluster 5 278 | 279 | echo '* checking consistent read' 280 | consistent=$(curl -s "http://${consul_ip}:8500/v1/kv/test_quorum?consistent") 281 | consistent=$(echo "$consistent" | json -a Value) 282 | # this value is "hello" base64 encoded 283 | if [[ "$consistent" != "aGVsbG8=" ]]; then 284 | fail "got '${consistent}' back from query; could not get consistent key after recovery" 285 | fi 286 | } 287 | 288 | # -------------------------------------------------------------------- 289 | # Main loop 290 | 291 | profile 292 | test-rejoin-raft 3 293 | test-rejoin-raft 5 294 | test-graceful-leave 295 | test-quorum-consistency 296 | --------------------------------------------------------------------------------