├── .dockerignore ├── .gitmodules ├── etc ├── redis.conf.tmpl ├── sentinel.conf.tmpl └── containerpilot.json ├── test ├── tests.sh ├── Dockerfile └── tests.py ├── .gitignore ├── bin ├── redis-server-sentinel.sh └── manage.sh ├── examples ├── compose │ ├── README.md │ ├── docker-compose.yml │ └── docker-compose-with-web.yml ├── webserver │ ├── README.md │ ├── package.json │ ├── containerpilot.json │ ├── Dockerfile │ └── server.js └── triton │ ├── README.md │ ├── docker-compose.yml │ ├── docker-compose-with-web.yml │ └── setup.sh ├── Dockerfile ├── README.md ├── Makefile └── LICENSE /.dockerignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | **/.DS_Store 3 | _env* 4 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "test/testing"] 2 | path = test/testing 3 | url = git@github.com:autopilotpattern/testing.git 4 | -------------------------------------------------------------------------------- /etc/redis.conf.tmpl: -------------------------------------------------------------------------------- 1 | appendonly yes 2 | protected-mode no 3 | {{range service "redis"}} 4 | slaveof {{.Address}} 6379 5 | {{end}} 6 | -------------------------------------------------------------------------------- /test/tests.sh: -------------------------------------------------------------------------------- 1 | echo CONSUL=redis-consul.svc.${TRITON_ACCOUNT}.${TRITON_DC}.cns.joyent.com >> /src/triton/_env 2 | python3 tests.py 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # credentials 2 | _env* 3 | manta 4 | manta.pub 5 | 6 | # macos frustration 7 | .DS_Store 8 | 9 | node_modules 10 | -------------------------------------------------------------------------------- /bin/redis-server-sentinel.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -xe 2 | manage.sh onStart #|| exit $? 3 | if [[ $? != 0 ]]; then 4 | exit $? 5 | fi 6 | redis-sentinel /etc/sentinel.conf & 7 | echo $! > /var/run/sentinel.pid 8 | exec redis-server /etc/redis.conf $* 9 | -------------------------------------------------------------------------------- /examples/compose/README.md: -------------------------------------------------------------------------------- 1 | # Autopilot Pattern Redis on local Docker 2 | 3 | To launch redis locally (on Docker for Mac as an example): 4 | 5 | ```bash 6 | $ docker-compose -p redis up -d 7 | ``` 8 | 9 | To scale it: 10 | 11 | ```bash 12 | $ docker-compose -p redis scale redis=3 13 | ``` 14 | -------------------------------------------------------------------------------- /examples/webserver/README.md: -------------------------------------------------------------------------------- 1 | example webserver 2 | ================= 3 | 4 | A very simple Node.js [Autopilot 5 | pattern](https://www.joyent.com/blog/category:Autopilot+Pattern) application that serves the value of 6 | `test:key` from [Redis on Autopilot](https://github.com/autopilotpattern/redis). 7 | -------------------------------------------------------------------------------- /examples/webserver/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "redis-node-example", 3 | "version": "1.0.0", 4 | "description": "An example of accessing Redis on Autopilot from Node.js", 5 | "scripts": { 6 | "test": "echo \"Error: no test specified\" && exit 1" 7 | }, 8 | "author": "Jason Pincin ", 9 | "license": "MIT", 10 | "dependencies": { 11 | "ioredis": "~2.5.0", 12 | "piloted": "~2.1.0" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /etc/sentinel.conf.tmpl: -------------------------------------------------------------------------------- 1 | protected-mode no 2 | port 26379 3 | dir /tmp 4 | {{range service "redis"}} 5 | sentinel monitor mymaster {{.Address}} 6379 {{key_or_default "service/redis-sentinel/quorum" "2"}} 6 | {{else}} 7 | sentinel monitor mymaster {{env "MASTER_ADDRESS"}} 6379 {{key_or_default "service/redis-sentinel/quorum" "2"}} 8 | {{end}} 9 | sentinel failover-timeout mymaster {{key_or_default "service/redis-sentinel/failover-timeout" "30000"}} 10 | -------------------------------------------------------------------------------- /examples/webserver/containerpilot.json: -------------------------------------------------------------------------------- 1 | { 2 | "consul": "{{ .CONSUL }}:8500", 3 | "services": [ 4 | { 5 | "name": "webserver", 6 | "port": 8000, 7 | "health": "/usr/bin/curl -o /dev/null --fail -s http://localhost:8000/", 8 | "poll": 3, 9 | "ttl": 10 10 | } 11 | ], 12 | "backends": [ 13 | { "name": "redis-sentinel", "poll": 7, "onChange": "pkill -SIGHUP node" } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /test/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.4 2 | 3 | RUN apk update \ 4 | && apk add nodejs docker python3 5 | RUN npm install -g triton json 6 | 7 | # install dependencies 8 | RUN pip3 install \ 9 | docker-compose==1.10.0 \ 10 | python-Consul==0.4.7 \ 11 | IPy==0.83 12 | 13 | # install test targets 14 | COPY examples/compose/docker-compose.yml /src/compose/docker-compose.yml 15 | COPY examples/triton/docker-compose.yml /src/triton/docker-compose.yml 16 | 17 | # install tests 18 | COPY test/testing/testcases.py /src/testcases.py 19 | COPY test/tests.py /src/tests.py 20 | COPY test/tests.sh /src/tests.sh 21 | -------------------------------------------------------------------------------- /examples/compose/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | 3 | services: 4 | redis: 5 | image: autopilotpattern/redis:${TAG:-latest} 6 | mem_limit: 512m 7 | restart: always 8 | environment: 9 | - CONSUL=consul 10 | links: 11 | - consul:consul 12 | expose: 13 | - 6379 14 | - 26379 15 | network_mode: bridge 16 | 17 | consul: 18 | image: autopilotpattern/consul:0.7.2-r0.8 19 | command: > 20 | /usr/local/bin/containerpilot 21 | /bin/consul agent -server 22 | -bootstrap-expect 1 23 | -config-dir=/etc/consul 24 | -ui-dir /ui 25 | restart: always 26 | mem_limit: 128m 27 | expose: 28 | - 53 29 | - 8300 30 | - 8301 31 | - 8302 32 | - 8400 33 | - 8500 34 | network_mode: bridge 35 | -------------------------------------------------------------------------------- /examples/webserver/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:7.7.1 2 | 3 | # Install ContainerPilot 4 | ENV CONTAINERPILOT_VERSION 2.7.0 5 | RUN export CP_SHA1=687f7d83e031be7f497ffa94b234251270aee75b \ 6 | && curl -Lso /tmp/containerpilot.tar.gz \ 7 | "https://github.com/joyent/containerpilot/releases/download/${CONTAINERPILOT_VERSION}/containerpilot-${CONTAINERPILOT_VERSION}.tar.gz" \ 8 | && echo "${CP_SHA1} /tmp/containerpilot.tar.gz" | sha1sum -c \ 9 | && tar zxf /tmp/containerpilot.tar.gz -C /bin \ 10 | && rm /tmp/containerpilot.tar.gz 11 | 12 | # COPY ContainerPilot configuration 13 | COPY containerpilot.json /etc/ 14 | ENV CONTAINERPILOT=file:///etc/containerpilot.json 15 | 16 | # COPY node app 17 | COPY package.json /opt/example/ 18 | COPY server.js /opt/example/ 19 | WORKDIR /opt/example/ 20 | RUN npm install 21 | 22 | EXPOSE 8000 23 | CMD ["/bin/containerpilot", "node", "/opt/example/server.js"] 24 | -------------------------------------------------------------------------------- /examples/triton/README.md: -------------------------------------------------------------------------------- 1 | # Autopilot Pattern Redis on Triton 2 | 3 | 1. [Get a Joyent account](https://my.joyent.com/landing/signup/) and [add your SSH key](https://docs.joyent.com/public-cloud/getting-started). 4 | 2. Install [Docker](https://docs.docker.com/docker-for-mac/install/) on your laptop or other environment, as well as the [Joyent Triton CLI](https://www.joyent.com/blog/introducing-the-triton-command-line-tool). 5 | 3. [Configure Docker and Docker Compose for use with Joyent.](https://docs.joyent.com/public-cloud/api-access/docker) 6 | 7 | Check that everything is configured correctly by running the `setup.sh` script. This will check that your environment is setup correctly and create an `_env` file that includes environment variables with reasonable defaults. 8 | 9 | ```bash 10 | $ setup.sh 11 | $ vim _env 12 | ``` 13 | 14 | See the [README](../../README.md) for details on environment variables in `_env`. 15 | 16 | Start everything: 17 | 18 | ```bash 19 | docker-compose -p redis up -d 20 | ``` 21 | 22 | To scale it: 23 | 24 | ```bash 25 | $ docker-compose -p redis scale redis=3 26 | ``` 27 | -------------------------------------------------------------------------------- /examples/compose/docker-compose-with-web.yml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | 3 | services: 4 | redis: 5 | image: autopilotpattern/redis:${TAG:-latest} 6 | mem_limit: 512m 7 | restart: always 8 | environment: 9 | - CONSUL=consul 10 | links: 11 | - consul:consul 12 | expose: 13 | - 6379 14 | - 26379 15 | network_mode: bridge 16 | 17 | consul: 18 | image: autopilotpattern/consul:0.7.2-r0.8 19 | command: > 20 | /usr/local/bin/containerpilot 21 | /bin/consul agent -server 22 | -bootstrap-expect 1 23 | -config-dir=/etc/consul 24 | -ui-dir /ui 25 | restart: always 26 | mem_limit: 128m 27 | expose: 28 | - 53 29 | - 8300 30 | - 8301 31 | - 8302 32 | - 8400 33 | - 8500 34 | network_mode: bridge 35 | 36 | webserver: 37 | image: autopilotpattern/redis-example-webserver:1.0.0 38 | restart: always 39 | mem_limit: 128m 40 | environment: 41 | - CONSUL=consul 42 | links: 43 | - redis:redis 44 | - consul:consul 45 | ports: 46 | - 8000 47 | network_mode: bridge 48 | -------------------------------------------------------------------------------- /examples/webserver/server.js: -------------------------------------------------------------------------------- 1 | const http = require('http') 2 | const piloted = require('piloted') 3 | const Redis = require('ioredis') 4 | 5 | let redis 6 | 7 | main() 8 | async function main () { 9 | await piloted.config(require('/etc/containerpilot.json')) 10 | configureRedis() 11 | piloted.on('refresh', configureRedis) 12 | 13 | const server = http.createServer(onRequest) 14 | server.listen(8000) 15 | } 16 | 17 | async function onRequest (req, res) { 18 | try { 19 | const value = await redis.get('test:key') 20 | res.end(`test:key = ${value}`) 21 | console.log('%s 200 OK', new Date()) 22 | } 23 | catch (err) { 24 | res.statusCode = 500 25 | res.end(`An error occurred: ${err.message}`) 26 | console.log('%s 500 ERROR %s', new Date(), err.message) 27 | } 28 | } 29 | 30 | function configureRedis () { 31 | if (redis) redis.quit() 32 | 33 | const sentinels = piloted 34 | .serviceHosts('redis-sentinel') 35 | .map(s => Object.assign(s, { host: s.address })) 36 | 37 | redis = new Redis({ sentinels, name: 'mymaster' }) 38 | redis.once('connect', () => console.log('%s connected to redis', new Date())) 39 | } 40 | -------------------------------------------------------------------------------- /etc/containerpilot.json: -------------------------------------------------------------------------------- 1 | { 2 | "consul": "localhost:8500", 3 | "logging": { 4 | "level": "INFO", 5 | "format": "default", 6 | "output": "stdout" 7 | }, 8 | "preStart": [ "/usr/local/bin/manage.sh", "preStart" ], 9 | "preStop": [ "/usr/local/bin/manage.sh", "preStop" ], 10 | "services": [ 11 | { 12 | "name": "redis-replica", 13 | "port": 6379, 14 | "health": [ "/usr/local/bin/manage.sh", "health" ], 15 | "poll": 5, 16 | "ttl": 25 17 | }, 18 | { 19 | "name": "redis-sentinel", 20 | "port": 26379, 21 | "health": [ "/usr/local/bin/manage.sh", "healthSentinel" ], 22 | "poll": 5, 23 | "ttl": 25 24 | } 25 | ], 26 | "tasks": [ 27 | { 28 | "name": "backup", 29 | "command": ["/usr/local/bin/manage.sh", "backUpIfTime"], 30 | "frequency": "10m", 31 | "timeout": "5m" 32 | } 33 | ], 34 | "coprocesses": [ 35 | { 36 | "command": ["/usr/local/bin/consul", "agent", 37 | "-data-dir=/opt/consul/data", 38 | "-config-dir=/opt/consul/config", 39 | "-rejoin", 40 | "-retry-join", "{{ if .CONSUL }}{{ .CONSUL }}{{ else }}consul{{ end }}", 41 | "-retry-max", "10", 42 | "-retry-interval", "10s"], 43 | "restarts": "unlimited" 44 | } 45 | ] 46 | } 47 | -------------------------------------------------------------------------------- /examples/triton/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | 3 | services: 4 | redis: 5 | image: autopilotpattern/redis:${TAG:-latest} 6 | mem_limit: 4g 7 | # Joyent recommends setting instances to always restart on Triton 8 | restart: always 9 | labels: 10 | - triton.cns.services=redis 11 | # This label sets the CNS name, Triton's automatic DNS 12 | # Learn more at https://docs.joyent.com/public-cloud/network/cns 13 | - com.joyent.package=g4-general-4G 14 | # This label selects the proper Joyent resource package 15 | # https://www.joyent.com/blog/optimizing-docker-on-triton#ram-cpu-and-disk-resources-for-your-containers 16 | environment: 17 | - CONTAINERPILOT=file:///etc/containerpilot.json 18 | - affinity:com.docker.compose.service!=~redis 19 | # This helps distribute Redis instances throughout the data center 20 | # Learn more at https://www.joyent.com/blog/optimizing-docker-on-triton#controlling-container-placement 21 | env_file: _env 22 | network_mode: bridge 23 | ports: 24 | - 6379 25 | - 26379 26 | # These port delcarations should not be made for production. Without these declarations, Redis 27 | # will be available to other containers via private interfaces. With these declarations, Redis is 28 | # also accessible publicly. This will also result in a public redis CNS record being created, 29 | # in the triton.zone domain. 30 | 31 | # Consul acts as our service catalog and is used to coordinate global state among 32 | # our Redis containers 33 | consul: 34 | image: autopilotpattern/consul:0.7.2-r0.8 35 | command: > 36 | /usr/local/bin/containerpilot 37 | /bin/consul agent -server 38 | -bootstrap-expect 1 39 | -config-dir=/etc/consul 40 | -ui-dir /ui 41 | # Change "-bootstrap" to "-bootstrap-expect 3", then scale to 3 or more to 42 | # turn this into an HA Consul raft. 43 | restart: always 44 | mem_limit: 128m 45 | ports: 46 | - 8500 47 | # As above, this port delcarations should not be made for production. 48 | labels: 49 | - triton.cns.services=redis-consul 50 | network_mode: bridge 51 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM redis:3.2.8-alpine 2 | 3 | RUN apk add --no-cache curl jq openssl tar bash 4 | 5 | # Add ContainerPilot and set its configuration file path 6 | ENV CONTAINERPILOT_VER 2.7.0 7 | ENV CONTAINERPILOT file:///etc/containerpilot.json 8 | RUN export CONTAINERPILOT_CHECKSUM=687f7d83e031be7f497ffa94b234251270aee75b \ 9 | && curl -Lso /tmp/containerpilot.tar.gz \ 10 | "https://github.com/joyent/containerpilot/releases/download/${CONTAINERPILOT_VER}/containerpilot-${CONTAINERPILOT_VER}.tar.gz" \ 11 | && echo "${CONTAINERPILOT_CHECKSUM} /tmp/containerpilot.tar.gz" | sha1sum -c \ 12 | && tar zxf /tmp/containerpilot.tar.gz -C /usr/local/bin \ 13 | && rm /tmp/containerpilot.tar.gz 14 | 15 | ENV CONSUL_VER 0.7.2 16 | ENV CONSUL_SHA256 aa97f4e5a552d986b2a36d48fdc3a4a909463e7de5f726f3c5a89b8a1be74a58 17 | RUN curl -Lso /tmp/consul.zip "https://releases.hashicorp.com/consul/${CONSUL_VER}/consul_${CONSUL_VER}_linux_amd64.zip" \ 18 | && echo "${CONSUL_SHA256} /tmp/consul.zip" | sha256sum -c \ 19 | && unzip /tmp/consul -d /usr/local/bin \ 20 | && rm /tmp/consul.zip \ 21 | && mkdir -p /opt/consul/config 22 | 23 | ENV CONSUL_TEMPLATE_VER 0.15.0 24 | ENV CONSUL_TEMPLATE_SHA256 b7561158d2074c3c68ff62ae6fc1eafe8db250894043382fb31f0c78150c513a 25 | RUN curl -Lso /tmp/consul-template.zip "https://releases.hashicorp.com/consul-template/${CONSUL_TEMPLATE_VER}/consul-template_${CONSUL_TEMPLATE_VER}_linux_amd64.zip" \ 26 | && echo "${CONSUL_TEMPLATE_SHA256} /tmp/consul-template.zip" | sha256sum -c \ 27 | && unzip -d /usr/local/bin /tmp/consul-template.zip \ 28 | && rm /tmp/consul-template.zip 29 | 30 | ENV CONSUL_CLI_VER 0.3.1 31 | ENV CONSUL_CLI_SHA256 037150d3d689a0babf4ba64c898b4497546e2fffeb16354e25cef19867e763f1 32 | RUN curl -Lso /tmp/consul-cli.tgz "https://github.com/CiscoCloud/consul-cli/releases/download/v${CONSUL_CLI_VER}/consul-cli_${CONSUL_CLI_VER}_linux_amd64.tar.gz" \ 33 | && echo "${CONSUL_CLI_SHA256} /tmp/consul-cli.tgz" | sha256sum -c \ 34 | && tar zxf /tmp/consul-cli.tgz -C /usr/local/bin --strip-components 1 \ 35 | && rm /tmp/consul-cli.tgz 36 | 37 | COPY etc/* /etc/ 38 | COPY bin/* /usr/local/bin/ 39 | 40 | # override the parent entrypoint 41 | ENTRYPOINT [] 42 | 43 | CMD [ "containerpilot", \ 44 | "/usr/local/bin/redis-server-sentinel.sh" \ 45 | ] 46 | -------------------------------------------------------------------------------- /examples/triton/docker-compose-with-web.yml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | 3 | services: 4 | redis: 5 | image: autopilotpattern/redis:${TAG:-latest} 6 | mem_limit: 4g 7 | # Joyent recommends setting instances to always restart on Triton 8 | restart: always 9 | labels: 10 | - triton.cns.services=redis 11 | # This label sets the CNS name, Triton's automatic DNS 12 | # Learn more at https://docs.joyent.com/public-cloud/network/cns 13 | - com.joyent.package=g4-general-4G 14 | # This label selects the proper Joyent resource package 15 | # https://www.joyent.com/blog/optimizing-docker-on-triton#ram-cpu-and-disk-resources-for-your-containers 16 | environment: 17 | - CONTAINERPILOT=file:///etc/containerpilot.json 18 | - affinity:com.docker.compose.service!=~redis 19 | # This helps distribute Redis instances throughout the data center 20 | # Learn more at https://www.joyent.com/blog/optimizing-docker-on-triton#controlling-container-placement 21 | env_file: _env 22 | network_mode: bridge 23 | ports: 24 | - 6379 25 | - 26379 26 | # These port delcarations should not be made for production. Without these declarations, Redis 27 | # will be available to other containers via private interfaces. With these declarations, Redis is 28 | # also accessible publicly. This will also result in a public redis CNS record being created, 29 | # in the triton.zone domain. 30 | 31 | # Consul acts as our service catalog and is used to coordinate global state among 32 | # our Redis containers 33 | consul: 34 | image: autopilotpattern/consul:0.7.2-r0.8 35 | command: > 36 | /usr/local/bin/containerpilot 37 | /bin/consul agent -server 38 | -bootstrap-expect 1 39 | -config-dir=/etc/consul 40 | -ui-dir /ui 41 | # Change "-bootstrap" to "-bootstrap-expect 3", then scale to 3 or more to 42 | # turn this into an HA Consul raft. 43 | restart: always 44 | mem_limit: 128m 45 | ports: 46 | - 8500 47 | # As above, this port delcarations should not be made for production. 48 | labels: 49 | - triton.cns.services=redis-consul 50 | network_mode: bridge 51 | 52 | webserver: 53 | image: autopilotpattern/redis-example-webserver:1.0.0 54 | restart: always 55 | mem_limit: 128m 56 | ports: 57 | - 8000 58 | env_file: _env 59 | network_mode: bridge 60 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Autopilot Pattern Redis 2 | 3 | Redis designed for automated operation using the [Autopilot Pattern](http://autopilotpattern.io/). 4 | 5 | [![DockerPulls](https://img.shields.io/docker/pulls/autopilotpattern/redis.svg)](https://registry.hub.docker.com/u/autopilotpattern/redis/) 6 | [![DockerStars](https://img.shields.io/docker/stars/autopilotpattern/redis.svg)](https://registry.hub.docker.com/u/autopilotpattern/redis/) 7 | 8 | The initial work was sponsored by [Faithlife](https://faithlife.com/about). 9 | 10 | --- 11 | 12 | ## Architecture 13 | 14 | A running cluster includes the following components: 15 | 16 | - [Redis](http://redis.io/): we're using Redis 3.2. 17 | - [Redis Sentinel](http://redis.io/topics/sentinel): manage failover. 18 | - [ContainerPilot](https://www.joyent.com/containerpilot): included in our Redis containers to orchestrate bootstrap behavior and coordinate replication using keys and checks stored in Consul in the startup, `health`, and `backup` handlers. 19 | - [Consul](https://www.consul.io/): is our service catalog that works with ContainerPilot and helps coordinate service discovery, replication, and failover 20 | - [Manta](https://www.joyent.com/object-storage): the Joyent object store, for securely and durably storing our Redis backups. 21 | - `manage.sh`: a small bash script that ContainerPilot will call into to do the heavy lifting of bootstrapping Redis. 22 | 23 | When a new Redis node is started. 24 | 25 | 26 | ### Bootstrapping 27 | 28 | The `onStart` task is run as part of app start, not from ContainerPilot's `preStart` handler, 29 | because the Consul agent must be running, and ContainerPilot doesn't start coprocesses until 30 | after `preStart`. 31 | 32 | `onStart` performs the following: 33 | 34 | 1. Is this container configured as a replica? If yes: 35 | 1. Wait for the master to become healthy in the service registry. 36 | 1. If there is no healthy master, try to reconfigure as master and restart. 37 | 1. Is this container configured as the master? If yes: 38 | 1. Verify this node should still start as master. 39 | 1. If this node shouldn't be master, reconfigure as a replica and restart. 40 | 1. Write redis and sentinel configurations based on the master in the service registry or this node if there is no master. 41 | 1. Restore the last backup if one exists. 42 | 43 | ### Maintenance via `health` handler 44 | 45 | `health` performs the following: 46 | 47 | 1. Ping redis, verify the response. 48 | 1. Verify the service configuration (master or replica) matches redis's role (master or slave). Sentinel may have performed a failover and changed this node's role. If the role is changed, the service registry needs to be updated so any containers started in the future are configured correctly. If the service configuration and role do not match, reconfigure to match the current role. 49 | 50 | `healthSentinel` pings sentinel. 51 | 52 | ### Backups via `backup` task 53 | 54 | ContainerPilot calls the `backup` handler via a recurring task. The backup handler will: 55 | 56 | 1. Check the backup run TTL health check on the redis service. 57 | 1. If the TTL has expired: 58 | 1. Pass the check. 59 | 1. Create a backup. 60 | 1. Upload the backup to Manta. 61 | 62 | --- 63 | 64 | ## Running the cluster 65 | 66 | Starting a new cluster is easy once you have [your `_env` file set with the configuration details](#configuration), **just run `docker-compose up -d` and in a few moments you'll have a running Redis master**. Both the master and replicas are described as a single `docker-compose` service. During startup, [ContainerPilot](http://containerpilot.io) will ask Consul if an existing master has been created. If not, the node will initialize as a new master and all future nodes will self-configure replication with the master via Sentinel. 67 | 68 | **Run `docker-compose scale redis=3` to add replicas**. The replicas will automatically configure themselves to to replicate from the master and will register themselves in Consul as replicas once they're ready. There should be at least 3 nodes to have a quorum in case of a node failure. 69 | 70 | ### Configuration 71 | 72 | Pass these variables via an `_env` file. The included `examples/triton/setup.sh` can be used to test your Docker and Triton environment, and to encode the Manta SSH key in the `_env` file. 73 | 74 | - `MANTA_URL`: the full Manta endpoint URL. (ex. `https://us-east.manta.joyent.com`) 75 | - `MANTA_USER`: the Manta account name. 76 | - `MANTA_SUBUSER`: the Manta subuser account name, if any. 77 | - `MANTA_ROLE`: the Manta role name, if any. 78 | - `MANTA_KEY_ID`: the MD5-format ssh key id for the Manta account/subuser (ex. `1a:b8:30:2e:57:ce:59:1d:16:f6:19:97:f2:60:2b:3d`); the included `setup.sh` will encode this automatically 79 | - `MANTA_PRIVATE_KEY`: the private ssh key for the Manta account/subuser; the included `setup.sh` will encode this automatically 80 | - `MANTA_BUCKET`: the path on Manta where backups will be stored. (ex. `/myaccount/stor/manage`); the bucket must already exist and be writeable by the `MANTA_USER`/`MANTA_PRIVATE_KEY` 81 | 82 | These variables are optional but you most likely want them: 83 | 84 | - `LOG_LEVEL`: will set the logging level of the `manage.sh` script. Set to `DEBUG` for more logging. 85 | - `CONSUL` is the hostname for the Consul instance(s) to join. Defaults to `consul`. 86 | 87 | ### Where to store data 88 | 89 | This pattern automates the data management and makes container effectively stateless to the Docker daemon and schedulers. This is designed to maximize convenience and reliability by minimizing the external coordination needed to manage the database. The use of external volumes (`--volumes-from`, `-v`, etc.) is not recommended. 90 | 91 | On Triton, there's no need to use data volumes because the performance hit you normally take with overlay file systems in Linux doesn't happen with ZFS. 92 | 93 | ### Using an existing database 94 | 95 | If you start your Redis container instance with a data directory that already contains a database (specifically, a appendonly.aof file), the pre-existing database won't be changed in any way. 96 | -------------------------------------------------------------------------------- /examples/triton/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e -o pipefail 3 | 4 | help() { 5 | echo 6 | echo 'Usage ./setup.sh ~/path/to/MANTA_PRIVATE_KEY' 7 | echo 8 | echo 'Checks that your Triton and Docker environment is sane and configures' 9 | echo 'an environment file to use.' 10 | echo 11 | echo 'MANTA_PRIVATE_KEY is the filesystem path to an SSH private key' 12 | echo 'used to connect to Manta for the database backups.' 13 | echo 14 | echo 'Additional details must be configured in the _env file, but this script will properly' 15 | echo 'encode the SSH key details for use with this MySQL image.' 16 | echo 17 | } 18 | 19 | 20 | # populated by `check` function whenever we're using Triton 21 | TRITON_USER= 22 | TRITON_DC= 23 | TRITON_ACCOUNT= 24 | 25 | # --------------------------------------------------- 26 | # Top-level commands 27 | 28 | # Check for correct configuration and setup _env file 29 | envcheck() { 30 | 31 | if [ -z "$1" ]; then 32 | tput rev # reverse 33 | tput bold # bold 34 | echo 'Please provide a path to a SSH private key to access Manta.' 35 | tput sgr0 # clear 36 | 37 | help 38 | exit 1 39 | fi 40 | 41 | if [ ! -f "$1" ]; then 42 | tput rev # reverse 43 | tput bold # bold 44 | echo 'SSH private key for Manta is unreadable.' 45 | tput sgr0 # clear 46 | 47 | help 48 | exit 1 49 | fi 50 | 51 | # Assign args to named vars 52 | MANTA_PRIVATE_KEY_PATH=$1 53 | 54 | command -v docker >/dev/null 2>&1 || { 55 | echo 56 | tput rev # reverse 57 | tput bold # bold 58 | echo 'Docker is required, but does not appear to be installed.' 59 | tput sgr0 # clear 60 | echo 'See https://docs.joyent.com/public-cloud/api-access/docker' 61 | exit 1 62 | } 63 | command -v json >/dev/null 2>&1 || { 64 | echo 65 | tput rev # reverse 66 | tput bold # bold 67 | echo 'Error! JSON CLI tool is required, but does not appear to be installed.' 68 | tput sgr0 # clear 69 | echo 'See https://apidocs.joyent.com/cloudapi/#getting-started' 70 | exit 1 71 | } 72 | 73 | command -v triton >/dev/null 2>&1 || { 74 | echo 75 | tput rev # reverse 76 | tput bold # bold 77 | echo 'Error! Joyent Triton CLI is required, but does not appear to be installed.' 78 | tput sgr0 # clear 79 | echo 'See https://www.joyent.com/blog/introducing-the-triton-command-line-tool' 80 | exit 1 81 | } 82 | 83 | # make sure Docker client is pointed to the same place as the Triton client 84 | local docker_user=$(docker info 2>&1 | awk -F": " '/SDCAccount:/{print $2}') 85 | local docker_dc=$(echo $DOCKER_HOST | awk -F"/" '{print $3}' | awk -F'.' '{print $1}') 86 | TRITON_USER=$(triton profile get | awk -F": " '/account:/{print $2}') 87 | TRITON_DC=$(triton profile get | awk -F"/" '/url:/{print $3}' | awk -F'.' '{print $1}') 88 | TRITON_ACCOUNT=$(triton account get | awk -F": " '/id:/{print $2}') 89 | if [ ! "$docker_user" = "$TRITON_USER" ] || [ ! "$docker_dc" = "$TRITON_DC" ]; then 90 | echo 91 | tput rev # reverse 92 | tput bold # bold 93 | echo 'Error! The Triton CLI configuration does not match the Docker CLI configuration.' 94 | tput sgr0 # clear 95 | echo 96 | echo "Docker user: ${docker_user}" 97 | echo "Triton user: ${TRITON_USER}" 98 | echo "Docker data center: ${docker_dc}" 99 | echo "Triton data center: ${TRITON_DC}" 100 | exit 1 101 | fi 102 | 103 | local triton_cns_enabled=$(triton account get | awk -F": " '/cns/{print $2}') 104 | if [ ! "true" == "$triton_cns_enabled" ]; then 105 | echo 106 | tput rev # reverse 107 | tput bold # bold 108 | echo 'Error! Triton CNS is required and not enabled.' 109 | tput sgr0 # clear 110 | echo 111 | exit 1 112 | fi 113 | 114 | # setup environment file 115 | if [ ! -f "_env" ]; then 116 | echo '# Environment variables for backups to Manta' >> _env 117 | echo 'MANTA_BUCKET= # an existing Manta bucket' >> _env 118 | echo 'MANTA_USER= # a user with access to that bucket' >> _env 119 | echo 'MANTA_SUBUSER=' >> _env 120 | echo 'MANTA_ROLE=' >> _env 121 | echo 'MANTA_URL=https://us-east.manta.joyent.com' >> _env 122 | 123 | # MANTA_KEY_ID must be the md5 formatted key fingerprint. A SHA256 will result in errors. 124 | echo MANTA_KEY_ID=$(ssh-keygen -yl -E md5 -f ${MANTA_PRIVATE_KEY_PATH} | awk '{print substr($2,5)}') >> _env 125 | 126 | # munge the private key so that we can pass it into an env var sanely 127 | # and then unmunge it in our startup script 128 | echo MANTA_PRIVATE_KEY=$(cat ${MANTA_PRIVATE_KEY_PATH} | tr '\n' '#') >> _env 129 | echo >> _env 130 | 131 | echo '# Consul discovery via Triton CNS' >> _env 132 | echo CONSUL=redis-consul.svc.${TRITON_ACCOUNT}.${TRITON_DC}.cns.joyent.com >> _env 133 | echo >> _env 134 | 135 | echo 'Edit the _env file with your desired MANTA_* config' 136 | else 137 | echo 'Existing _env file found, exiting' 138 | exit 139 | fi 140 | } 141 | 142 | # runs unit tests inside a Docker container 143 | test() { 144 | docker run -it --rm \ 145 | -v $(pwd)/bin:/usr/local/bin \ 146 | -w /usr/local/bin \ 147 | my_mysql \ 148 | python test.py 149 | } 150 | 151 | # --------------------------------------------------- 152 | # parse arguments 153 | 154 | # Get function list 155 | funcs=($(declare -F -p | cut -d " " -f 3)) 156 | 157 | until 158 | if [ ! -z "$1" ]; then 159 | # check if the first arg is a function in this file, or use a default 160 | if [[ " ${funcs[@]} " =~ " $1 " ]]; then 161 | cmd=$1 162 | shift 1 163 | else 164 | cmd="envcheck" 165 | fi 166 | 167 | $cmd "$@" 168 | if [ $? == 127 ]; then 169 | help 170 | fi 171 | 172 | exit 173 | else 174 | help 175 | fi 176 | do 177 | echo 178 | done 179 | -------------------------------------------------------------------------------- /test/tests.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | import os 3 | from os.path import expanduser 4 | import random 5 | import re 6 | import string 7 | import subprocess 8 | import sys 9 | import time 10 | import unittest 11 | import uuid 12 | 13 | from testcases import AutopilotPatternTest, WaitTimeoutError, \ 14 | dump_environment_to_file 15 | import requests 16 | 17 | 18 | class RedisStackTest(AutopilotPatternTest): 19 | project_name = 'redis' 20 | 21 | def setUp(self): 22 | if 'COMPOSE_FILE' in os.environ and 'triton' in os.environ['COMPOSE_FILE']: 23 | account = os.environ['TRITON_ACCOUNT'] 24 | dc = os.environ['TRITON_DC'] 25 | self.consul_cns = 'redis-consul.svc.{}.{}.triton.zone'.format(account, dc) 26 | self.redis_cns = 'redis.svc.{}.{}.triton.zone'.format(account, dc) 27 | os.environ['CONSUL'] = self.consul_cns 28 | 29 | def test_redis(self): 30 | ############################################### 31 | # scale up 32 | ############################################### 33 | self.instrument(self.wait_for_containers, 34 | {'redis': 1, 'consul': 1}, timeout=300) 35 | self.instrument(self.wait_for_service, 'redis', count=1, timeout=300) 36 | self.instrument(self.wait_for_service, 'redis-sentinel', count=1, timeout=180) 37 | 38 | self.compose_scale('redis', 3) 39 | self.instrument(self.wait_for_containers, 40 | {'redis': 3, 'consul': 1}, timeout=300) 41 | self.instrument(self.wait_for_service, 'redis', count=1, timeout=180) 42 | self.instrument(self.wait_for_service, 'redis-replica', count=2, timeout=180) 43 | self.instrument(self.wait_for_service, 'redis-sentinel', count=3, timeout=180) 44 | 45 | ############################################### 46 | # manual fail over 47 | ############################################### 48 | master_container = self.get_service_instances_from_consul('redis')[0] 49 | master_ip = self.get_service_addresses_from_consul('redis')[0] 50 | 51 | # force redis sentinel to failover master to a new redis instance 52 | self.docker_exec(master_container, 'redis-cli -p 26379 sentinel failover mymaster') 53 | self.instrument(self.wait_for_failover_from, master_ip, timeout=120) 54 | 55 | ############################################### 56 | # validate replication 57 | ############################################### 58 | master_container = self.get_service_instances_from_consul('redis')[0] 59 | replica_containers = self.get_service_instances_from_consul('redis-replica') 60 | value = str(uuid.uuid4()) 61 | self.docker_exec(master_container, 'redis-cli set test:repl ' + value) 62 | for replica in replica_containers: 63 | self.instrument(self.wait_for_replicated_value, replica, 'test:repl', value) 64 | 65 | ############################################### 66 | # container destruction fail over 67 | ############################################### 68 | # TODO: kill the leader, verify failover 69 | # this test is failing due to sentinel never electing a new master when 70 | # the existign master container is stopped; more investigation required 71 | # self.docker_stop(master_container) 72 | # self.instrument(self.wait_for_service, 'redis', count=1, timeout=60) 73 | # self.instrument(self.wait_for_service, 'redis-replica', count=1, timeout=60) 74 | 75 | def wait_for_failover_from(self, from_ip, timeout=30): 76 | """ 77 | Waits for the IP address of the `redis` service in Consul to change 78 | from what we knew the IP address to be prior to failing over 79 | """ 80 | # verify "redis" address changes 81 | while timeout > 0: 82 | addresses = self.get_service_addresses_from_consul('redis') 83 | if (len(addresses) > 0 and addresses[0] != from_ip): 84 | new_master_ip = addresses[0] 85 | break 86 | time.sleep(1) 87 | timeout -= 1 88 | else: 89 | raise WaitTimeoutError("Timed out waiting for redis service to be updated in Consul.") 90 | 91 | # verify old master becoems a replica and new master is no longer a replica 92 | while timeout > 0: 93 | addresses = self.get_service_addresses_from_consul('redis-replica') 94 | if from_ip in addresses and new_master_ip not in addresses: 95 | break 96 | time.sleep(1) 97 | timeout -= 1 98 | else: 99 | raise WaitTimeoutError("Timed out waiting for redis service to be updated in Consul.") 100 | 101 | 102 | def wait_for_replicated_value(self, replica, key, value, timeout=30): 103 | """ 104 | Waits for the given key/value pair to be written to the given replica 105 | """ 106 | while timeout > 0: 107 | replicated_value = self.docker_exec(replica, 'redis-cli get test:repl').strip("\n") 108 | if (replicated_value == value): 109 | break 110 | time.sleep(1) 111 | timeout -= 1 112 | else: 113 | raise WaitTimeoutError("Timed out waiting for redis replica to receive replicated value.") 114 | 115 | def wait_for_containers(self, expected={}, timeout=30): 116 | """ 117 | Waits for all containers to be marked as 'Up' for all services. 118 | `expected` should be a dict of {"service_name": count}. 119 | TODO: lower this into the base class implementation. 120 | """ 121 | svc_regex = re.compile(r'^{}_(\w+)_\d+$'.format(self.project_name)) 122 | 123 | def get_service_name(container_name): 124 | return svc_regex.match(container_name).group(1) 125 | 126 | while timeout > 0: 127 | containers = self.compose_ps() 128 | found = defaultdict(int) 129 | states = [] 130 | for container in containers: 131 | service = get_service_name(container.name) 132 | found[service] = found[service] + 1 133 | states.append(container.state == 'Up') 134 | if all(states): 135 | if not expected or found == expected: 136 | break 137 | time.sleep(1) 138 | timeout -= 1 139 | else: 140 | raise WaitTimeoutError("Timed out waiting for containers to start.") 141 | 142 | if __name__ == "__main__": 143 | unittest.main() 144 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for shipping and testing the container image. 2 | 3 | MAKEFLAGS += --warn-undefined-variables 4 | .DEFAULT_GOAL := build 5 | .PHONY: * 6 | 7 | # we get these from CI environment if available, otherwise from git 8 | GIT_COMMIT ?= $(shell git rev-parse --short HEAD) 9 | GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) 10 | WORKSPACE ?= $(shell pwd) 11 | 12 | namespace ?= autopilotpattern 13 | tag := branch-$(shell basename $(GIT_BRANCH)) 14 | image := $(namespace)/redis 15 | testImage := $(namespace)/redis-testrunner 16 | 17 | dockerLocal := DOCKER_HOST= DOCKER_TLS_VERIFY= DOCKER_CERT_PATH= docker 18 | composeLocal := DOCKER_HOST= DOCKER_TLS_VERIFY= DOCKER_CERT_PATH= docker-compose 19 | 20 | ## Display this help message 21 | help: 22 | @awk '/^##.*$$/,/[a-zA-Z_-]+:/' $(MAKEFILE_LIST) | awk '!(NR%2){print $$0p}{p=$$0}' | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' | sort 23 | 24 | 25 | # ------------------------------------------------ 26 | # Container builds 27 | 28 | ## Builds the application container image locally 29 | build: test-runner 30 | $(dockerLocal) build -t=$(image):$(tag) . 31 | 32 | ## Build the test running container 33 | test-runner: 34 | $(dockerLocal) build -f test/Dockerfile -t=$(testImage):$(tag) . 35 | 36 | ## Push the current application container images to the Docker Hub 37 | push: 38 | $(dockerLocal) push $(image):$(tag) 39 | $(dockerLocal) push $(testImage):$(tag) 40 | 41 | ## Tag the current images as 'latest' 42 | tag: 43 | $(dockerLocal) tag $(testImage):$(tag) $(testImage):latest 44 | $(dockerLocal) tag $(image):$(tag) $(image):latest 45 | 46 | ## Push latest tag(s) to the Docker Hub 47 | ship: tag 48 | $(dockerLocal) push $(image):$(tag) 49 | $(dockerLocal) push $(image):latest 50 | 51 | 52 | # ------------------------------------------------ 53 | # Test running 54 | 55 | ## Pull the container images from the Docker Hub 56 | pull: 57 | docker pull $(image):$(tag) 58 | docker pull $(testImage):$(tag) 59 | 60 | $(DOCKER_CERT_PATH)/key.pub: 61 | ssh-keygen -y -f $(DOCKER_CERT_PATH)/key.pem > $(DOCKER_CERT_PATH)/key.pub 62 | 63 | # For Jenkins test runner only: make sure we have public keys available 64 | SDC_KEYS_VOL ?= -v $(DOCKER_CERT_PATH):$(DOCKER_CERT_PATH) 65 | keys: $(DOCKER_CERT_PATH)/key.pub 66 | 67 | run-local: 68 | cd examples/compose && TAG=$(tag) $(composeLocal) -p redis up -d 69 | 70 | stop-local: 71 | cd examples/compose && TAG=$(tag) $(composeLocal) -p redis stop || true 72 | cd examples/compose && TAG=$(tag) $(composeLocal) -p redis rm -f || true 73 | 74 | run: 75 | $(call check_var, TRITON_PROFILE \ 76 | required to run the example on Triton.) 77 | cd examples/triton && TAG=$(tag) docker-compose -p redis up -d 78 | 79 | stop: 80 | $(call check_var, TRITON_PROFILE \ 81 | required to run the example on Triton.) 82 | cd examples/compose && TAG=$(tag) docker-compose -p redis stop || true 83 | cd examples/compose && TAG=$(tag) docker-compose -p redis rm -f || true 84 | 85 | test-image: 86 | docker build -f test/Dockerfile . 87 | 88 | run-test-image-local: 89 | $(dockerLocal) run -it --rm \ 90 | -v /var/run/docker.sock:/var/run/docker.sock \ 91 | -e TAG=$(tag) \ 92 | -e COMPOSE_FILE=compose/docker-compose.yml \ 93 | -e COMPOSE_HTTP_TIMEOUT=300 \ 94 | -w /src \ 95 | `docker build -f test/Dockerfile . | tail -n 1 | awk '{print $$3}'` \ 96 | sh 97 | 98 | run-test-image: 99 | $(call check_var, TRITON_ACCOUNT TRITON_DC, \ 100 | required to run integration tests on Triton.) 101 | $(dockerLocal) run -it --rm \ 102 | -e TAG=$(tag) \ 103 | -e COMPOSE_FILE=triton/docker-compose.yml \ 104 | -e COMPOSE_HTTP_TIMEOUT=300 \ 105 | -e DOCKER_HOST=$(DOCKER_HOST) \ 106 | -e DOCKER_TLS_VERIFY=1 \ 107 | -e DOCKER_CERT_PATH=$(DOCKER_CERT_PATH) \ 108 | -e TRITON_ACCOUNT=$(TRITON_ACCOUNT) \ 109 | -e TRITON_DC=$(TRITON_DC) \ 110 | $(SDC_KEYS_VOL) -w /src \ 111 | $(testImage):$(tag) sh 112 | 113 | ## Run integration tests against local Docker daemon 114 | test-local: 115 | $(dockerLocal) run -it --rm \ 116 | -v /var/run/docker.sock:/var/run/docker.sock \ 117 | -e TAG=$(tag) \ 118 | -e COMPOSE_FILE=compose/docker-compose.yml \ 119 | -e COMPOSE_HTTP_TIMEOUT=300 \ 120 | -w /src \ 121 | `docker build -f test/Dockerfile . | tail -n 1 | awk '{print $$3}'` \ 122 | python3 tests.py 123 | 124 | ## Run the integration test runner locally but target Triton 125 | test: 126 | $(call check_var, TRITON_ACCOUNT TRITON_DC, \ 127 | required to run integration tests on Triton.) 128 | $(dockerLocal) run --rm \ 129 | -e TAG=$(tag) \ 130 | -e COMPOSE_FILE=triton/docker-compose.yml \ 131 | -e COMPOSE_HTTP_TIMEOUT=300 \ 132 | -e DOCKER_HOST=$(DOCKER_HOST) \ 133 | -e DOCKER_TLS_VERIFY=1 \ 134 | -e DOCKER_CERT_PATH=$(DOCKER_CERT_PATH) \ 135 | -e TRITON_ACCOUNT=$(TRITON_ACCOUNT) \ 136 | -e TRITON_DC=$(TRITON_DC) \ 137 | $(SDC_KEYS_VOL) -w /src \ 138 | $(testImage):$(tag) sh tests.sh 139 | 140 | ## Print environment for build debugging 141 | debug: 142 | @echo WORKSPACE=$(WORKSPACE) 143 | @echo GIT_COMMIT=$(GIT_COMMIT) 144 | @echo GIT_BRANCH=$(GIT_BRANCH) 145 | @echo namespace=$(namespace) 146 | @echo tag=$(tag) 147 | @echo image=$(image) 148 | @echo testImage=$(testImage) 149 | 150 | # Create backup user/policies (usage: make manta EMAIL=example@example.com PASSWORD=pwd) 151 | # ------------------------------------------------------- 152 | # Create user and policies for backups 153 | # Requires SDC_ACCOUNT to be set 154 | # usage: 155 | # make manta EMAIL=example@example.com PASSWORD=strongpassword 156 | # 157 | ## Create backup user and policies 158 | manta: 159 | $(call check_var, EMAIL PASSWORD SDC_ACCOUNT, \ 160 | Required to create a Manta login.) 161 | 162 | ssh-keygen -t rsa -b 4096 -C "${EMAIL}" -f manta 163 | sdc-user create --login=${MANTA_LOGIN} --password=${PASSWORD} --email=${EMAIL} 164 | sdc-user upload-key $(ssh-keygen -E md5 -lf ./manta | awk -F' ' '{gsub("MD5:","");{print $2}}') --name=${MANTA_LOGIN}-key ${MANTA_LOGIN} ./manta.pub 165 | sdc-policy create --name=${MANTA_POLICY} \ 166 | --rules='CAN getobject' \ 167 | --rules='CAN putobject' \ 168 | --rules='CAN putmetadata' \ 169 | --rules='CAN putsnaplink' \ 170 | --rules='CAN getdirectory' \ 171 | --rules='CAN putdirectory' 172 | sdc-role create --name=${MANTA_ROLE} \ 173 | --policies=${MANTA_POLICY} \ 174 | --members=${MANTA_LOGIN} 175 | mmkdir ${SDC_ACCOUNT}/stor/${MANTA_LOGIN} 176 | mchmod -- +triton_redis /${SDC_ACCOUNT}/stor/${MANTA_LOGIN} 177 | 178 | 179 | # ------------------------------------------------------- 180 | # helper functions for testing if variables are defined 181 | # 182 | check_var = $(foreach 1,$1,$(__check_var)) 183 | __check_var = $(if $(value $1),,\ 184 | $(error Missing $1 $(if $(value 2),$(strip $2)))) 185 | -------------------------------------------------------------------------------- /bin/manage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | CONSUL=localhost 4 | 5 | readonly lockPath=service/redis/locks/master 6 | readonly lastBackupKey=service/redis/last-backup 7 | 8 | consulCommand() { 9 | consul-cli --quiet --consul="${CONSUL}:8500" $* 10 | } 11 | 12 | preStart() { 13 | logDebug "preStart" 14 | 15 | if [[ -n ${CONSUL_LOCAL_CONFIG} ]]; then 16 | echo "$CONSUL_LOCAL_CONFIG" > "/opt/consul/config/local.json" 17 | fi 18 | } 19 | 20 | onStart() { 21 | logDebug "onStart" 22 | 23 | waitForLeader 24 | 25 | getRegisteredServiceName 26 | if [[ "${registeredServiceName}" == "redis-replica" ]]; then 27 | 28 | echo "Getting master address" 29 | 30 | if [[ "$(consulCommand catalog service "redis" | jq any)" == "true" ]]; then 31 | # only wait for a healthy service if there is one registered in the catalog 32 | local i 33 | for (( i = 0; i < ${MASTER_WAIT_TIMEOUT-60}; i++ )); do 34 | getServiceAddresses "redis" 35 | if [[ ${serviceAddresses} ]]; then 36 | break 37 | fi 38 | sleep 1 39 | done 40 | fi 41 | 42 | if [[ ! ${serviceAddresses} ]]; then 43 | echo "No healthy master, trying to set this node as master" 44 | 45 | logDebug "Locking ${lockPath}" 46 | local session=$(consulCommand kv lock "${lockPath}" --ttl=30s --lock-delay=5s) 47 | echo ${session} > /var/run/redis-master.sid 48 | 49 | getServiceAddresses "redis" 50 | if [[ ! ${serviceAddresses} ]]; then 51 | echo "Still no healthy master, setting this node as master" 52 | 53 | setRegisteredServiceName "redis" 54 | exit 2 55 | fi 56 | 57 | logDebug "Unlocking ${lockPath}" 58 | consulCommand kv unlock "${lockPath}" --session="$session" 59 | fi 60 | 61 | else 62 | 63 | local session=$(< /var/run/redis-master.sid) 64 | if [[ "$(consulCommand kv lock "${lockPath}" --ttl=30s --session="${session}")" != "${session}" ]]; then 65 | echo "This node is no longer the master" 66 | 67 | setRegisteredServiceName "redis-replica" 68 | exit 2 69 | fi 70 | 71 | fi 72 | 73 | if [[ ${serviceAddresses} ]]; then 74 | echo "Master is ${serviceAddresses}" 75 | else 76 | getNodeAddress 77 | echo "Master is ${nodeAddress} (this node)" 78 | export MASTER_ADDRESS=${nodeAddress} 79 | fi 80 | if [[ ! -f /etc/redis.conf ]] && [[ ! -f /etc/sentinel.conf ]]; then 81 | # don't overwrite sentinel.conf because Sentinel rewrites it with state configuration 82 | consul-template -consul=${CONSUL}:8500 -once -template=/etc/redis.conf.tmpl:/etc/redis.conf -template=/etc/sentinel.conf.tmpl:/etc/sentinel.conf 83 | if [[ $? != 0 ]]; then 84 | exit 1 85 | fi 86 | fi 87 | 88 | if [[ "$MANTA_PRIVATE_KEY" ]]; then 89 | echo "$MANTA_PRIVATE_KEY" | tr '#' '\n' > /tmp/mantakey.pem 90 | fi 91 | 92 | if [[ ! -f /data/appendonly.aof ]]; then 93 | # only restore from backup if no data exists 94 | if [[ -s /data/dump.rdb ]]; then 95 | loadBackupRdb 96 | else 97 | restoreFromBackup 98 | fi 99 | fi 100 | } 101 | 102 | health() { 103 | logDebug "health" 104 | 105 | redis-cli PING | grep PONG > /dev/null 106 | if [[ $? -ne 0 ]]; then 107 | echo "redis ping failed" 108 | exit 1 109 | fi 110 | 111 | getRedisInfo 112 | local role=${redisInfo[role]} 113 | getRegisteredServiceName 114 | logDebug "Role ${role}, service ${registeredServiceName}" 115 | 116 | if [[ "${registeredServiceName}" == "redis" ]] && [[ "${role}" != "master" ]]; then 117 | setRegisteredServiceName "redis-replica" 118 | elif [[ "${registeredServiceName}" == "redis-replica" ]] && [[ "${role}" != "slave" ]]; then 119 | setRegisteredServiceName "redis" 120 | elif [[ "${registeredServiceName}" == "redis" ]] && [[ -f /var/run/redis-master.sid ]]; then 121 | getNodeAddress 122 | getServiceAddresses "redis" 123 | if [[ "${nodeAddress}" == "${serviceAddresses}" ]]; then 124 | local session=$(< /var/run/redis-master.sid) 125 | 126 | logDebug "Unlocking ${lockPath}" 127 | consulCommand kv unlock "${lockPath}" --session="$session" 128 | 129 | rm /var/run/redis-master.sid 130 | fi 131 | fi 132 | } 133 | 134 | healthSentinel() { 135 | logDebug "healthSentinel" 136 | redis-cli -p 26379 PING | grep PONG > /dev/null 137 | if [[ $? -ne 0 ]]; then 138 | echo "sentinel ping failed" 139 | exit 1 140 | fi 141 | } 142 | 143 | preStop() { 144 | logDebug "preStop" 145 | 146 | local sentinels=$(redis-cli -p 26379 SENTINEL SENTINELS mymaster | awk '/^ip$/ { getline; print $0 }') 147 | logDebug "Sentinels to reset: ${sentinels}" 148 | if [[ -f /var/run/sentinel.pid ]]; then 149 | kill $(cat /var/run/sentinel.pid) 150 | rm /var/run/sentinel.pid 151 | fi 152 | 153 | for sentinel in ${sentinels} ; do 154 | echo "Resetting sentinel $sentinel" 155 | redis-cli -h "${sentinel}" -p 26379 SENTINEL RESET "*" 156 | done 157 | } 158 | 159 | backUpIfTime() { 160 | logDebug "backUpIfTime" 161 | 162 | local backupCheckName=redis-backup-run 163 | local status=$(consulCommand agent checks | jq -r ".\"${backupCheckName}\".Status") 164 | logDebug "status $status" 165 | if [[ "${status}" != "passing" ]]; then 166 | # TODO: pass the check after the backup? 167 | consulCommand check pass "${backupCheckName}" 168 | if [[ $? != 0 ]]; then 169 | consulCommand check register "${backupCheckName}" --ttl=${BACKUP_TTL-24h} || exit 1 170 | consulCommand check pass "${backupCheckName}" || exit 1 171 | fi 172 | 173 | saveBackup 174 | fi 175 | } 176 | 177 | saveBackup() { 178 | logDebug "saveBackup" 179 | 180 | echo "Saving backup" 181 | local prevLastSave=$(redis-cli LASTSAVE) 182 | redis-cli BGSAVE || (echo "BGSAVE failed" ; exit 1) 183 | 184 | local tries=0 185 | while true 186 | do 187 | logDebug -n "." 188 | tries=$((tries + 1)) 189 | local lastSave=$(redis-cli LASTSAVE) 190 | if [[ "${lastSave}" != "${prevLastSave}" ]]; then 191 | logDebug "" 192 | break 193 | elif [[ $tries -eq 60 ]]; then 194 | logDebug "" 195 | echo "Timeout waiting for backup" 196 | exit 1 197 | fi 198 | sleep 1 199 | done 200 | 201 | local backupFilename=dump-$(date -u +%Y%m%d%H%M%S -d @${lastSave}).rdb.gz 202 | gzip /data/dump.rdb -c > /data/${backupFilename} 203 | 204 | echo "Uploading ${backupFilename}" 205 | (manta ${MANTA_BUCKET}/${backupFilename} --upload-file /data/${backupFilename} -H 'content-type: application/gzip; type=file' --fail) || (echo "Backup upload failed" ; exit 1) 206 | 207 | (consulCommand kv write "${lastBackupKey}" "${backupFilename}") || (echo "Set last backup value failed" ; exit 1) 208 | 209 | # remove the backup files so they don't grow without limit 210 | rm ${backupFilename} 211 | } 212 | 213 | restoreFromBackup() { 214 | local backupFilename=$(consulCommand kv read --format=text "${lastBackupKey}") 215 | 216 | if [[ -n ${backupFilename} ]]; then 217 | echo "Downloading ${backupFilename}" 218 | manta ${MANTA_BUCKET}/${backupFilename} | gunzip > /data/dump.rdb 219 | if [[ ! -s /data/dump.rdb ]]; then 220 | echo "Backup download failed" 221 | exit 1 222 | fi 223 | 224 | loadBackupRdb 225 | fi 226 | } 227 | 228 | loadBackupRdb() { 229 | echo "Initializing from /data/dump.rdb" 230 | 231 | redis-server --appendonly no --protected-mode yes & 232 | local i 233 | for (( i = 0; i < 10; i++ )); do 234 | sleep 0.1 235 | redis-cli PING | grep PONG > /dev/null && break 236 | done 237 | 238 | redis-cli CONFIG SET appendonly yes | grep OK > /dev/null || exit 1 239 | 240 | for (( i = 0; i < 600; i++ )); do 241 | sleep 0.1 242 | getRedisInfo 243 | logDebug "aof_rewrite_in_progress ${redisInfo[aof_rewrite_in_progress]}" 244 | if [[ "${redisInfo[aof_rewrite_in_progress]}" == "0" ]]; then 245 | break 246 | fi 247 | done 248 | 249 | logDebug "Shutting down" 250 | redis-cli SHUTDOWN || exit 1 251 | 252 | wait 253 | } 254 | 255 | waitForLeader() { 256 | logDebug "Waiting for consul leader" 257 | local tries=0 258 | while true 259 | do 260 | logDebug "Waiting for consul leader" 261 | tries=$((tries + 1)) 262 | local leader=$(consulCommand --template="{{.}}" status leader) 263 | if [[ -n "$leader" ]]; then 264 | break 265 | elif [[ $tries -eq 60 ]]; then 266 | echo "No consul leader" 267 | exit 1 268 | fi 269 | sleep 1 270 | done 271 | } 272 | 273 | getServiceAddresses() { 274 | local serviceInfo=$(consulCommand health service --passing "$1") 275 | serviceAddresses=($(echo $serviceInfo | jq -r '.[].Service.Address')) 276 | logDebug "serviceAddresses $1 ${serviceAddresses[*]}" 277 | } 278 | 279 | getRegisteredServiceName() { 280 | registeredServiceName=$(jq -r '.services[0].name' /etc/containerpilot.json) 281 | } 282 | 283 | setRegisteredServiceName() { 284 | jq ".services[0].name = \"$1\"" /etc/containerpilot.json > /etc/containerpilot.json.new 285 | mv /etc/containerpilot.json.new /etc/containerpilot.json 286 | kill -HUP 1 287 | } 288 | 289 | declare -A redisInfo 290 | getRedisInfo() { 291 | eval $(redis-cli INFO | tr -d '\r' | egrep -v '^(#.*)?$' | sed -E 's/^([^:]*):(.*)$/redisInfo[\1]="\2"/') 292 | } 293 | 294 | manta() { 295 | local alg=rsa-sha256 296 | local keyId=/$MANTA_USER/keys/$MANTA_KEY_ID 297 | if [[ "${MANTA_SUBUSER}" != "" ]]; then 298 | keyId=/$MANTA_USER/$MANTA_SUBUSER/keys/$MANTA_KEY_ID 299 | fi 300 | local now=$(date -u "+%a, %d %h %Y %H:%M:%S GMT") 301 | local sig=$(echo "date:" $now | \ 302 | tr -d '\n' | \ 303 | openssl dgst -sha256 -sign /tmp/mantakey.pem | \ 304 | openssl enc -e -a | tr -d '\n') 305 | 306 | if [[ -z "$sig" ]]; then 307 | return 1 308 | fi 309 | 310 | curl -sS $MANTA_URL"$@" -H "date: $now" \ 311 | -H "Authorization: Signature keyId=\"$keyId\",algorithm=\"$alg\",signature=\"$sig\"" 312 | } 313 | 314 | getNodeAddress() { 315 | nodeAddress=$(ifconfig eth0 | awk '/inet addr/ {gsub("addr:", "", $2); print $2}') 316 | } 317 | 318 | logDebug() { 319 | if [[ "${LOG_LEVEL}" == "DEBUG" ]]; then 320 | echo "manage: $*" 321 | fi 322 | } 323 | 324 | help() { 325 | echo "Usage: ./manage.sh preStart => configure Consul agent" 326 | echo " ./manage.sh onStart => first-run configuration" 327 | echo " ./manage.sh health => health check Redis" 328 | echo " ./manage.sh healthSentinel => health check Sentinel" 329 | echo " ./manage.sh preStop => prepare for stop" 330 | echo " ./manage.sh backUpIfTime => save backup if it is time" 331 | echo " ./manage.sh saveBackup => save backup now" 332 | } 333 | 334 | until 335 | cmd=$1 336 | if [[ -z "$cmd" ]]; then 337 | help 338 | fi 339 | shift 1 340 | $cmd "$@" 341 | [ "$?" -ne 127 ] 342 | do 343 | help 344 | exit 345 | done 346 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Mozilla Public License, version 2.0 2 | 3 | 1. Definitions 4 | 5 | 1.1. "Contributor" 6 | 7 | means each individual or legal entity that creates, contributes to the 8 | creation of, or owns Covered Software. 9 | 10 | 1.2. "Contributor Version" 11 | 12 | means the combination of the Contributions of others (if any) used by a 13 | Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | 17 | means Covered Software of a particular Contributor. 18 | 19 | 1.4. "Covered Software" 20 | 21 | means Source Code Form to which the initial Contributor has attached the 22 | notice in Exhibit A, the Executable Form of such Source Code Form, and 23 | Modifications of such Source Code Form, in each case including portions 24 | thereof. 25 | 26 | 1.5. "Incompatible With Secondary Licenses" 27 | means 28 | 29 | a. that the initial Contributor has attached the notice described in 30 | Exhibit B to the Covered Software; or 31 | 32 | b. that the Covered Software was made available under the terms of 33 | version 1.1 or earlier of the License, but not also under the terms of 34 | a Secondary License. 35 | 36 | 1.6. "Executable Form" 37 | 38 | means any form of the work other than Source Code Form. 39 | 40 | 1.7. "Larger Work" 41 | 42 | means a work that combines Covered Software with other material, in a 43 | separate file or files, that is not Covered Software. 44 | 45 | 1.8. "License" 46 | 47 | means this document. 48 | 49 | 1.9. "Licensable" 50 | 51 | means having the right to grant, to the maximum extent possible, whether 52 | at the time of the initial grant or subsequently, any and all of the 53 | rights conveyed by this License. 54 | 55 | 1.10. "Modifications" 56 | 57 | means any of the following: 58 | 59 | a. any file in Source Code Form that results from an addition to, 60 | deletion from, or modification of the contents of Covered Software; or 61 | 62 | b. any new file in Source Code Form that contains any Covered Software. 63 | 64 | 1.11. "Patent Claims" of a Contributor 65 | 66 | means any patent claim(s), including without limitation, method, 67 | process, and apparatus claims, in any patent Licensable by such 68 | Contributor that would be infringed, but for the grant of the License, 69 | by the making, using, selling, offering for sale, having made, import, 70 | or transfer of either its Contributions or its Contributor Version. 71 | 72 | 1.12. "Secondary License" 73 | 74 | means either the GNU General Public License, Version 2.0, the GNU Lesser 75 | General Public License, Version 2.1, the GNU Affero General Public 76 | License, Version 3.0, or any later versions of those licenses. 77 | 78 | 1.13. "Source Code Form" 79 | 80 | means the form of the work preferred for making modifications. 81 | 82 | 1.14. "You" (or "Your") 83 | 84 | means an individual or a legal entity exercising rights under this 85 | License. For legal entities, "You" includes any entity that controls, is 86 | controlled by, or is under common control with You. For purposes of this 87 | definition, "control" means (a) the power, direct or indirect, to cause 88 | the direction or management of such entity, whether by contract or 89 | otherwise, or (b) ownership of more than fifty percent (50%) of the 90 | outstanding shares or beneficial ownership of such entity. 91 | 92 | 93 | 2. License Grants and Conditions 94 | 95 | 2.1. Grants 96 | 97 | Each Contributor hereby grants You a world-wide, royalty-free, 98 | non-exclusive license: 99 | 100 | a. under intellectual property rights (other than patent or trademark) 101 | Licensable by such Contributor to use, reproduce, make available, 102 | modify, display, perform, distribute, and otherwise exploit its 103 | Contributions, either on an unmodified basis, with Modifications, or 104 | as part of a Larger Work; and 105 | 106 | b. under Patent Claims of such Contributor to make, use, sell, offer for 107 | sale, have made, import, and otherwise transfer either its 108 | Contributions or its Contributor Version. 109 | 110 | 2.2. Effective Date 111 | 112 | The licenses granted in Section 2.1 with respect to any Contribution 113 | become effective for each Contribution on the date the Contributor first 114 | distributes such Contribution. 115 | 116 | 2.3. Limitations on Grant Scope 117 | 118 | The licenses granted in this Section 2 are the only rights granted under 119 | this License. No additional rights or licenses will be implied from the 120 | distribution or licensing of Covered Software under this License. 121 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 122 | Contributor: 123 | 124 | a. for any code that a Contributor has removed from Covered Software; or 125 | 126 | b. for infringements caused by: (i) Your and any other third party's 127 | modifications of Covered Software, or (ii) the combination of its 128 | Contributions with other software (except as part of its Contributor 129 | Version); or 130 | 131 | c. under Patent Claims infringed by Covered Software in the absence of 132 | its Contributions. 133 | 134 | This License does not grant any rights in the trademarks, service marks, 135 | or logos of any Contributor (except as may be necessary to comply with 136 | the notice requirements in Section 3.4). 137 | 138 | 2.4. Subsequent Licenses 139 | 140 | No Contributor makes additional grants as a result of Your choice to 141 | distribute the Covered Software under a subsequent version of this 142 | License (see Section 10.2) or under the terms of a Secondary License (if 143 | permitted under the terms of Section 3.3). 144 | 145 | 2.5. Representation 146 | 147 | Each Contributor represents that the Contributor believes its 148 | Contributions are its original creation(s) or it has sufficient rights to 149 | grant the rights to its Contributions conveyed by this License. 150 | 151 | 2.6. Fair Use 152 | 153 | This License is not intended to limit any rights You have under 154 | applicable copyright doctrines of fair use, fair dealing, or other 155 | equivalents. 156 | 157 | 2.7. Conditions 158 | 159 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in 160 | Section 2.1. 161 | 162 | 163 | 3. Responsibilities 164 | 165 | 3.1. Distribution of Source Form 166 | 167 | All distribution of Covered Software in Source Code Form, including any 168 | Modifications that You create or to which You contribute, must be under 169 | the terms of this License. You must inform recipients that the Source 170 | Code Form of the Covered Software is governed by the terms of this 171 | License, and how they can obtain a copy of this License. You may not 172 | attempt to alter or restrict the recipients' rights in the Source Code 173 | Form. 174 | 175 | 3.2. Distribution of Executable Form 176 | 177 | If You distribute Covered Software in Executable Form then: 178 | 179 | a. such Covered Software must also be made available in Source Code Form, 180 | as described in Section 3.1, and You must inform recipients of the 181 | Executable Form how they can obtain a copy of such Source Code Form by 182 | reasonable means in a timely manner, at a charge no more than the cost 183 | of distribution to the recipient; and 184 | 185 | b. You may distribute such Executable Form under the terms of this 186 | License, or sublicense it under different terms, provided that the 187 | license for the Executable Form does not attempt to limit or alter the 188 | recipients' rights in the Source Code Form under this License. 189 | 190 | 3.3. Distribution of a Larger Work 191 | 192 | You may create and distribute a Larger Work under terms of Your choice, 193 | provided that You also comply with the requirements of this License for 194 | the Covered Software. If the Larger Work is a combination of Covered 195 | Software with a work governed by one or more Secondary Licenses, and the 196 | Covered Software is not Incompatible With Secondary Licenses, this 197 | License permits You to additionally distribute such Covered Software 198 | under the terms of such Secondary License(s), so that the recipient of 199 | the Larger Work may, at their option, further distribute the Covered 200 | Software under the terms of either this License or such Secondary 201 | License(s). 202 | 203 | 3.4. Notices 204 | 205 | You may not remove or alter the substance of any license notices 206 | (including copyright notices, patent notices, disclaimers of warranty, or 207 | limitations of liability) contained within the Source Code Form of the 208 | Covered Software, except that You may alter any license notices to the 209 | extent required to remedy known factual inaccuracies. 210 | 211 | 3.5. Application of Additional Terms 212 | 213 | You may choose to offer, and to charge a fee for, warranty, support, 214 | indemnity or liability obligations to one or more recipients of Covered 215 | Software. However, You may do so only on Your own behalf, and not on 216 | behalf of any Contributor. You must make it absolutely clear that any 217 | such warranty, support, indemnity, or liability obligation is offered by 218 | You alone, and You hereby agree to indemnify every Contributor for any 219 | liability incurred by such Contributor as a result of warranty, support, 220 | indemnity or liability terms You offer. You may include additional 221 | disclaimers of warranty and limitations of liability specific to any 222 | jurisdiction. 223 | 224 | 4. Inability to Comply Due to Statute or Regulation 225 | 226 | If it is impossible for You to comply with any of the terms of this License 227 | with respect to some or all of the Covered Software due to statute, 228 | judicial order, or regulation then You must: (a) comply with the terms of 229 | this License to the maximum extent possible; and (b) describe the 230 | limitations and the code they affect. Such description must be placed in a 231 | text file included with all distributions of the Covered Software under 232 | this License. Except to the extent prohibited by statute or regulation, 233 | such description must be sufficiently detailed for a recipient of ordinary 234 | skill to be able to understand it. 235 | 236 | 5. Termination 237 | 238 | 5.1. The rights granted under this License will terminate automatically if You 239 | fail to comply with any of its terms. However, if You become compliant, 240 | then the rights granted under this License from a particular Contributor 241 | are reinstated (a) provisionally, unless and until such Contributor 242 | explicitly and finally terminates Your grants, and (b) on an ongoing 243 | basis, if such Contributor fails to notify You of the non-compliance by 244 | some reasonable means prior to 60 days after You have come back into 245 | compliance. Moreover, Your grants from a particular Contributor are 246 | reinstated on an ongoing basis if such Contributor notifies You of the 247 | non-compliance by some reasonable means, this is the first time You have 248 | received notice of non-compliance with this License from such 249 | Contributor, and You become compliant prior to 30 days after Your receipt 250 | of the notice. 251 | 252 | 5.2. If You initiate litigation against any entity by asserting a patent 253 | infringement claim (excluding declaratory judgment actions, 254 | counter-claims, and cross-claims) alleging that a Contributor Version 255 | directly or indirectly infringes any patent, then the rights granted to 256 | You by any and all Contributors for the Covered Software under Section 257 | 2.1 of this License shall terminate. 258 | 259 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user 260 | license agreements (excluding distributors and resellers) which have been 261 | validly granted by You or Your distributors under this License prior to 262 | termination shall survive termination. 263 | 264 | 6. Disclaimer of Warranty 265 | 266 | Covered Software is provided under this License on an "as is" basis, 267 | without warranty of any kind, either expressed, implied, or statutory, 268 | including, without limitation, warranties that the Covered Software is free 269 | of defects, merchantable, fit for a particular purpose or non-infringing. 270 | The entire risk as to the quality and performance of the Covered Software 271 | is with You. Should any Covered Software prove defective in any respect, 272 | You (not any Contributor) assume the cost of any necessary servicing, 273 | repair, or correction. This disclaimer of warranty constitutes an essential 274 | part of this License. No use of any Covered Software is authorized under 275 | this License except under this disclaimer. 276 | 277 | 7. Limitation of Liability 278 | 279 | Under no circumstances and under no legal theory, whether tort (including 280 | negligence), contract, or otherwise, shall any Contributor, or anyone who 281 | distributes Covered Software as permitted above, be liable to You for any 282 | direct, indirect, special, incidental, or consequential damages of any 283 | character including, without limitation, damages for lost profits, loss of 284 | goodwill, work stoppage, computer failure or malfunction, or any and all 285 | other commercial damages or losses, even if such party shall have been 286 | informed of the possibility of such damages. This limitation of liability 287 | shall not apply to liability for death or personal injury resulting from 288 | such party's negligence to the extent applicable law prohibits such 289 | limitation. Some jurisdictions do not allow the exclusion or limitation of 290 | incidental or consequential damages, so this exclusion and limitation may 291 | not apply to You. 292 | 293 | 8. Litigation 294 | 295 | Any litigation relating to this License may be brought only in the courts 296 | of a jurisdiction where the defendant maintains its principal place of 297 | business and such litigation shall be governed by laws of that 298 | jurisdiction, without reference to its conflict-of-law provisions. Nothing 299 | in this Section shall prevent a party's ability to bring cross-claims or 300 | counter-claims. 301 | 302 | 9. Miscellaneous 303 | 304 | This License represents the complete agreement concerning the subject 305 | matter hereof. If any provision of this License is held to be 306 | unenforceable, such provision shall be reformed only to the extent 307 | necessary to make it enforceable. Any law or regulation which provides that 308 | the language of a contract shall be construed against the drafter shall not 309 | be used to construe this License against a Contributor. 310 | 311 | 312 | 10. Versions of the License 313 | 314 | 10.1. New Versions 315 | 316 | Mozilla Foundation is the license steward. Except as provided in Section 317 | 10.3, no one other than the license steward has the right to modify or 318 | publish new versions of this License. Each version will be given a 319 | distinguishing version number. 320 | 321 | 10.2. Effect of New Versions 322 | 323 | You may distribute the Covered Software under the terms of the version 324 | of the License under which You originally received the Covered Software, 325 | or under the terms of any subsequent version published by the license 326 | steward. 327 | 328 | 10.3. Modified Versions 329 | 330 | If you create software not governed by this License, and you want to 331 | create a new license for such software, you may create and use a 332 | modified version of this License if you rename the license and remove 333 | any references to the name of the license steward (except to note that 334 | such modified license differs from this License). 335 | 336 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 337 | Licenses If You choose to distribute Source Code Form that is 338 | Incompatible With Secondary Licenses under the terms of this version of 339 | the License, the notice described in Exhibit B of this License must be 340 | attached. 341 | 342 | Exhibit A - Source Code Form License Notice 343 | 344 | This Source Code Form is subject to the 345 | terms of the Mozilla Public License, v. 346 | 2.0. If a copy of the MPL was not 347 | distributed with this file, You can 348 | obtain one at 349 | http://mozilla.org/MPL/2.0/. 350 | 351 | If it is not possible or desirable to put the notice in a particular file, 352 | then You may include the notice in a location (such as a LICENSE file in a 353 | relevant directory) where a recipient would be likely to look for such a 354 | notice. 355 | 356 | You may add additional accurate notices of copyright ownership. 357 | 358 | Exhibit B - "Incompatible With Secondary Licenses" Notice 359 | 360 | This Source Code Form is "Incompatible 361 | With Secondary Licenses", as defined by 362 | the Mozilla Public License, v. 2.0. 363 | 364 | --------------------------------------------------------------------------------