├── .editorconfig ├── .gitignore ├── Dockerfile ├── README.md ├── aws ├── README.md ├── docker-compose.yml ├── keepalived │ ├── keepalived-a.conf │ ├── keepalived-b.conf │ └── notify.sh ├── ubuntu-setup.sh └── webserver │ └── index.html ├── docker-compose.yml ├── haproxy └── haproxy.cfg ├── keepalived ├── notify.sh ├── proxy-a │ └── keepalived.conf └── proxy-b │ └── keepalived.conf └── web ├── server-a └── index.html └── server-b └── index.html /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 4 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | insert_final_newline = true 9 | 10 | [*.{yml,yaml}] 11 | indent_size = 2 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pem 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG ALPINE_TAG="3.10" 2 | FROM alpine:${ALPINE_TAG} 3 | RUN apk add --no-cache bind-tools curl nghttp2 openssl-dev bash netcat-openbsd ipvsadm 4 | ENTRYPOINT ["/bin/sh", "-c"] 5 | CMD ["echo runner"] 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Highly Available Load Balancing with Floating IP 2 | 3 | Using Docker, Keepalived and HAProxy with NGINX server as a web application. 4 | 5 | Note: Cloud environments require manual configuration for creating/attaching/detaching Floating IP. For AWS see `aws/keepalived/notify.sh`. 6 | 7 | ## Set-up 8 | 9 | ```sh 10 | # enable ip_vs 11 | sudo modprobe ip_vs 12 | 13 | # configure system 14 | # see: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/load_balancer_administration/s1-initial-setup-forwarding-vsa 15 | sudo sysctl -w net.ipv4.ip_forward=1 16 | sudo sysctl -w net.ipv4.ip_nonlocal_bind=1 17 | ``` 18 | 19 | ## Running 20 | 21 | ### Config 22 | 23 | ```yaml 24 | Virtual Server: 25 | IP: 192.168.0.150/24 26 | Port H2: 8080 27 | Port HTTP: 80 28 | Host 1: 29 | IP: 192.168.0.24/24 30 | Interface: enp2s0 31 | Instances: 32 | - keepalived-a 33 | - haproxy-a 34 | - haproxy-b 35 | - web-a 36 | - web-b 37 | Host 2: 38 | IP: 192.168.0.66/24 39 | Interface: wlp3s0 40 | Instances: 41 | - keepalived-b 42 | - haproxy-a 43 | - haproxy-b 44 | - web-a 45 | - web-b 46 | ``` 47 | 48 | ### Host 1 49 | 50 | ```sh 51 | docker-compose up -d keepalived-a haproxy-a haproxy-b web-a web-b 52 | ``` 53 | 54 | ### Host 2 55 | 56 | ```sh 57 | docker-compose up -d keepalived-b haproxy-a haproxy-b web-a web-b 58 | ``` 59 | 60 | ### Usage 61 | 62 | ```sh 63 | ############## 64 | # h2 protocol 65 | ############## 66 | curl 192.168.0.150:8080 --http2-prior-knowledge 67 | # Server B 68 | curl 192.168.0.150:8080 --http2-prior-knowledge 69 | # Server A 70 | 71 | ################ 72 | # http protocol 73 | ################ 74 | curl 192.168.0.150 75 | # Server B 76 | curl 192.168.0.150 77 | # Server A 78 | ``` 79 | 80 | ## Debugging 81 | 82 | ### Wireshark 83 | 84 | Look for `vrrp` packets. 85 | 86 | ### Getting docker's private ip address 87 | 88 | ```sh 89 | docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker-compose ps -q) 90 | ``` 91 | -------------------------------------------------------------------------------- /aws/README.md: -------------------------------------------------------------------------------- 1 | # Keepalived Proxy AWS 2 | 3 | 1. Create instances 4 | 2. Copy contents of this directory into them 5 | 3. Run `ubuntu-setup.sh` on them 6 | 4. Run `docker-compose up -d webserver` on one of them - remember IP address 7 | 5. Configure `keepalived/keepalived-*.conf` files. 8 | * `real_server` in virtual server should be one with running `webserver` docker-compose service 9 | * `unicast_src_ip` is always a IP address of a current keepalived host 10 | * in `unicast_peer` should be other instance's ip with another keepalived host 11 | 6. Run proper keepalived services on different host 12 | 7. `curl 10.0.0.100` on any of the keepalived hosts and try failover (e.g. run `docker-compose down` on one instance and check logs on another one) 13 | -------------------------------------------------------------------------------- /aws/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.6" 2 | 3 | services: 4 | webserver: 5 | image: "nginx:stable-alpine" 6 | volumes: 7 | - "./web/index.html:/usr/share/nginx/html/index.html:ro" 8 | ports: 9 | - 80:80 10 | 11 | keepalived-a: 12 | image: "osixia/keepalived:2.0.17" 13 | network_mode: host 14 | cap_add: 15 | - NET_ADMIN 16 | - NET_BROADCAST 17 | - NET_RAW 18 | environment: 19 | KEEPALIVED_COMMAND_LINE_ARGUMENTS: >- 20 | --log-detail 21 | volumes: 22 | - "./keepalived/keepalived-a.conf:/usr/local/etc/keepalived/keepalived.conf:ro" 23 | - "./keepalived/notify.sh:/container/service/keepalived/assets/notify.custom.sh:ro" 24 | 25 | keepalived-b: 26 | image: "osixia/keepalived:2.0.17" 27 | network_mode: host 28 | cap_add: 29 | - NET_ADMIN 30 | - NET_BROADCAST 31 | - NET_RAW 32 | environment: 33 | KEEPALIVED_COMMAND_LINE_ARGUMENTS: >- 34 | --log-detail 35 | volumes: 36 | - "./keepalived/keepalived-b.conf:/usr/local/etc/keepalived/keepalived.conf:ro" 37 | - "./keepalived/notify.sh:/container/service/keepalived/assets/notify.custom.sh:ro" 38 | -------------------------------------------------------------------------------- /aws/keepalived/keepalived-a.conf: -------------------------------------------------------------------------------- 1 | vrrp_instance VI_1 { 2 | 3 | # UNIQUE # 4 | state MASTER 5 | priority 150 6 | # UNIQUE # 7 | 8 | interface eth0 9 | advert_int 1 10 | virtual_router_id 51 11 | 12 | # my ip 13 | unicast_src_ip 10.0.0.15 14 | 15 | # peer ip 16 | unicast_peer { 17 | 10.0.0.139 18 | } 19 | 20 | virtual_ipaddress { 21 | 10.0.0.100/24 dev eth0 22 | } 23 | 24 | authentication { 25 | auth_type PASS 26 | auth_pass d0ck3r 27 | } 28 | 29 | notify "/container/service/keepalived/assets/notify.custom.sh" 30 | } 31 | 32 | virtual_server 10.0.0.100 80 { 33 | delay_loop 5 34 | lb_algo rr 35 | lb_kind NAT 36 | persistence_timeout 600 37 | protocol TCP 38 | 39 | real_server 10.0.0.84 80 { 40 | TCP_CHECK { 41 | connect_timeout 10 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /aws/keepalived/keepalived-b.conf: -------------------------------------------------------------------------------- 1 | vrrp_instance VI_1 { 2 | 3 | # UNIQUE # 4 | state BACKUP 5 | priority 100 6 | # UNIQUE # 7 | 8 | interface eth0 9 | advert_int 1 10 | virtual_router_id 51 11 | 12 | # my ip 13 | unicast_src_ip 10.0.0.139 14 | 15 | # peer ip 16 | unicast_peer { 17 | 10.0.0.15 18 | } 19 | 20 | virtual_ipaddress { 21 | 10.0.0.100/24 dev eth0 22 | } 23 | 24 | authentication { 25 | auth_type PASS 26 | auth_pass d0ck3r 27 | } 28 | 29 | notify "/container/service/keepalived/assets/notify.custom.sh" 30 | } 31 | 32 | virtual_server 10.0.0.100 80 { 33 | delay_loop 5 34 | lb_algo rr 35 | lb_kind NAT 36 | persistence_timeout 600 37 | protocol TCP 38 | 39 | real_server 10.0.0.84 80 { 40 | TCP_CHECK { 41 | connect_timeout 10 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /aws/keepalived/notify.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # for ANY state transition. 4 | # "notify" script is called AFTER the 5 | # notify_* script(s) and is executed 6 | # with 3 arguments provided by keepalived 7 | # (ie don't include parameters in the notify line). 8 | # arguments 9 | # $1 = "GROUP"|"INSTANCE" 10 | # $2 = name of group or instance 11 | # $3 = target state of transition 12 | # ("MASTER"|"BACKUP"|"FAULT") 13 | 14 | TYPE=$1 15 | NAME=$2 16 | STATE=$3 17 | 18 | case $STATE in 19 | "MASTER") echo "[$1 - $2] I'm the MASTER! Whup whup." > /proc/1/fd/1 20 | exit 0 21 | ;; 22 | "BACKUP") echo "[$1 - $2] Ok, i'm just a backup, great." > /proc/1/fd/1 23 | exit 0 24 | ;; 25 | "FAULT") echo "[$1 - $2] Fault, what ?" > /proc/1/fd/1 26 | exit 0 27 | ;; 28 | *) echo "[$1 - $2] Unknown state" > /proc/1/fd/1 29 | exit 1 30 | ;; 31 | esac 32 | 33 | ## TODO: use aws change eip script 34 | # source https://blog.rapid7.com/2014/12/03/keepalived-and-haproxy-in-aws-an-exploratory-guide/ 35 | 36 | # EIP=9.8.7.6 37 | # INSTANCE_ID=i-abcd1234 38 | 39 | # /usr/local/bin/aws ec2 disassociate-address --public-ip $EIP 40 | # /usr/local/bin/aws ec2 associate-address --public-ip $EIP --instance-id $INSTANCE_ID 41 | 42 | -------------------------------------------------------------------------------- /aws/ubuntu-setup.sh: -------------------------------------------------------------------------------- 1 | apt update 2 | apt -y install docker.io dnsutils tshark ipvsadm 3 | docker --version 4 | 5 | # install docker-compose 6 | curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose 7 | chmod +x /usr/local/bin/docker-compose 8 | ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose 9 | docker-compose --version 10 | 11 | # enable ip_vs 12 | modprobe ip_vs 13 | -------------------------------------------------------------------------------- /aws/webserver/index.html: -------------------------------------------------------------------------------- 1 | Web Server 2 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | 3 | x-haproxy-defaults: &haproxy-service 4 | image: "haproxy:2.0-alpine" 5 | volumes: 6 | - "./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro" 7 | 8 | x-keepalived-defaults: &keepalived-service 9 | image: "osixia/keepalived:2.0.19" 10 | cap_add: 11 | - NET_ADMIN 12 | - NET_BROADCAST 13 | - NET_RAW 14 | environment: 15 | KEEPALIVED_COMMAND_LINE_ARGUMENTS: >- 16 | --log-detail 17 | # -–dont-release-vrrp 18 | # -–dont-release-ipvs 19 | # --log-detail 20 | # --dump-conf 21 | 22 | networks: 23 | ha-stack: 24 | ipam: 25 | config: 26 | - subnet: 172.20.0.0/24 27 | 28 | services: 29 | runner: 30 | image: "runner:local" 31 | build: 32 | context: . 33 | networks: 34 | - ha-stack 35 | 36 | keepalived-a: 37 | <<: *keepalived-service 38 | network_mode: host 39 | volumes: 40 | - "./keepalived/proxy-a/keepalived.conf:/usr/local/etc/keepalived/keepalived.conf:ro" 41 | - "./keepalived/notify.sh:/container/service/keepalived/assets/notify.custom.sh:ro" 42 | 43 | keepalived-b: 44 | <<: *keepalived-service 45 | network_mode: host 46 | volumes: 47 | - "./keepalived/proxy-b/keepalived.conf:/usr/local/etc/keepalived/keepalived.conf:ro" 48 | - "./keepalived/notify.sh:/container/service/keepalived/assets/notify.custom.sh:ro" 49 | 50 | haproxy-a: 51 | <<: *haproxy-service 52 | networks: 53 | ha-stack: 54 | ipv4_address: 172.20.0.50 55 | aliases: 56 | - haproxy-a.ha.stack 57 | 58 | haproxy-b: 59 | <<: *haproxy-service 60 | networks: 61 | ha-stack: 62 | ipv4_address: 172.20.0.60 63 | aliases: 64 | - haproxy-b.ha.stack 65 | 66 | web-b: 67 | image: "nginx:stable-alpine" 68 | networks: 69 | ha-stack: 70 | aliases: 71 | - web-b.ha.stack 72 | volumes: 73 | - "./web/server-b/index.html:/usr/share/nginx/html/index.html:ro" 74 | 75 | web-a: 76 | image: "nginx:stable-alpine" 77 | networks: 78 | ha-stack: 79 | aliases: 80 | - web-a.ha.stack 81 | volumes: 82 | - "./web/server-a/index.html:/usr/share/nginx/html/index.html:ro" 83 | -------------------------------------------------------------------------------- /haproxy/haproxy.cfg: -------------------------------------------------------------------------------- 1 | global 2 | maxconn 10000 3 | log stdout format raw local0 4 | 5 | defaults 6 | log global 7 | mode http 8 | option httplog 9 | option dontlognull 10 | option http-use-htx 11 | timeout connect 1 12 | timeout client 5 13 | timeout server 5 14 | 15 | frontend web-in 16 | bind :80 17 | bind :8080 proto h2 18 | default_backend web-upstream 19 | 20 | backend web-upstream 21 | retries 2 22 | server web-a web-a.ha.stack:80 check 23 | server web-b web-b.ha.stack:80 check 24 | -------------------------------------------------------------------------------- /keepalived/notify.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # for ANY state transition. 4 | # "notify" script is called AFTER the 5 | # notify_* script(s) and is executed 6 | # with 3 arguments provided by keepalived 7 | # (ie don't include parameters in the notify line). 8 | # arguments 9 | # $1 = "GROUP"|"INSTANCE" 10 | # $2 = name of group or instance 11 | # $3 = target state of transition 12 | # ("MASTER"|"BACKUP"|"FAULT") 13 | 14 | TYPE=$1 15 | NAME=$2 16 | STATE=$3 17 | 18 | case $STATE in 19 | "MASTER") echo "[$1 - $2] I'm the MASTER! Whup whup." > /proc/1/fd/1 20 | exit 0 21 | ;; 22 | "BACKUP") echo "[$1 - $2] Ok, i'm just a backup, great." > /proc/1/fd/1 23 | exit 0 24 | ;; 25 | "FAULT") echo "[$1 - $2] Fault, what ?" > /proc/1/fd/1 26 | exit 0 27 | ;; 28 | *) echo "[$1 - $2] Unknown state" > /proc/1/fd/1 29 | exit 1 30 | ;; 31 | esac 32 | -------------------------------------------------------------------------------- /keepalived/proxy-a/keepalived.conf: -------------------------------------------------------------------------------- 1 | global_defs { 2 | # UNIQUE # 3 | router_id LVS_MAIN 4 | # UNIQUE # 5 | } 6 | 7 | vrrp_instance VI_1 { 8 | # UNIQUE # 9 | state MASTER 10 | priority 150 11 | # UNIQUE # 12 | 13 | advert_int 1 14 | virtual_router_id 51 15 | 16 | # CHANGE TO YOUR NEEDS # 17 | # real network interface 18 | interface enp2s0 19 | 20 | # my ip (on real network) 21 | unicast_src_ip 192.168.0.24/24 22 | 23 | # peer ip (on real network) 24 | unicast_peer { 25 | 192.168.0.66/24 26 | } 27 | # CHANGE TO YOUR NEEDS # 28 | 29 | virtual_ipaddress { 30 | 192.168.0.150/24 31 | } 32 | 33 | authentication { 34 | auth_type PASS 35 | auth_pass d0ck3r 36 | } 37 | 38 | notify "/container/service/keepalived/assets/notify.custom.sh" 39 | } 40 | 41 | virtual_server 192.168.0.150 80 { 42 | delay_loop 5 43 | lb_algo wlc 44 | lb_kind NAT 45 | persistence_timeout 600 46 | protocol TCP 47 | 48 | real_server 172.20.0.50 80 { 49 | weight 100 50 | TCP_CHECK { 51 | connect_timeout 10 52 | } 53 | } 54 | real_server 172.20.0.60 80 { 55 | weight 100 56 | TCP_CHECK { 57 | connect_timeout 10 58 | } 59 | } 60 | } 61 | 62 | virtual_server 192.168.0.150 8080 { 63 | delay_loop 5 64 | lb_algo wlc 65 | lb_kind NAT 66 | persistence_timeout 600 67 | protocol TCP 68 | 69 | real_server 172.20.0.50 8080 { 70 | weight 100 71 | TCP_CHECK { 72 | connect_timeout 10 73 | } 74 | } 75 | real_server 172.20.0.60 8080 { 76 | weight 100 77 | TCP_CHECK { 78 | connect_timeout 10 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /keepalived/proxy-b/keepalived.conf: -------------------------------------------------------------------------------- 1 | global_defs { 2 | # UNIQUE # 3 | router_id LVS_BCKP 4 | # UNIQUE # 5 | } 6 | 7 | vrrp_instance VI_1 { 8 | # UNIQUE # 9 | state BACKUP 10 | priority 100 11 | # UNIQUE # 12 | 13 | advert_int 1 14 | virtual_router_id 51 15 | 16 | # CHANGE TO YOUR NEEDS # 17 | # real network interface 18 | interface wlp3s0 19 | 20 | # my ip (on real network) 21 | unicast_src_ip 192.168.0.66/24 22 | 23 | # peer ip (on real network) 24 | unicast_peer { 25 | 192.168.0.24/24 26 | } 27 | # CHANGE TO YOUR NEEDS # 28 | 29 | virtual_ipaddress { 30 | 192.168.0.150/24 31 | } 32 | 33 | authentication { 34 | auth_type PASS 35 | auth_pass d0ck3r 36 | } 37 | 38 | notify "/container/service/keepalived/assets/notify.custom.sh" 39 | } 40 | 41 | virtual_server 192.168.0.150 80 { 42 | delay_loop 5 43 | lb_algo wlc 44 | lb_kind NAT 45 | persistence_timeout 600 46 | protocol TCP 47 | 48 | real_server 172.20.0.50 80 { 49 | weight 100 50 | TCP_CHECK { 51 | connect_timeout 10 52 | } 53 | } 54 | real_server 172.20.0.60 80 { 55 | weight 100 56 | TCP_CHECK { 57 | connect_timeout 10 58 | } 59 | } 60 | } 61 | 62 | virtual_server 192.168.0.150 8080 { 63 | delay_loop 5 64 | lb_algo wlc 65 | lb_kind NAT 66 | persistence_timeout 600 67 | protocol TCP 68 | 69 | real_server 172.20.0.50 8080 { 70 | weight 100 71 | TCP_CHECK { 72 | connect_timeout 10 73 | } 74 | } 75 | real_server 172.20.0.60 8080 { 76 | weight 100 77 | TCP_CHECK { 78 | connect_timeout 10 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /web/server-a/index.html: -------------------------------------------------------------------------------- 1 | Server A 2 | -------------------------------------------------------------------------------- /web/server-b/index.html: -------------------------------------------------------------------------------- 1 | Server B 2 | --------------------------------------------------------------------------------