├── LICENSE ├── README.md ├── haproxy └── haproxy-sample.cfg ├── puppet ├── mysql.pp └── xinetd.pp ├── scripts ├── xinetd-helper ├── xinetd-mysql └── xinetd-mysql-check-lag ├── tests └── xinetd-mysql-test.sh └── xinetd ├── mysqlchk_general └── xinetd.conf /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 GitHub 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mysql-haproxy-xinetd 2 | 3 | This repository contains sample config files and scripts used for setting up a context aware MySQL replication pool on HAProxy. 4 | 5 | A context aware pool is a pool of backend MySQL servers, such that each _tells_ HAproxy whether it should be included in the pool or not. 6 | 7 | HAProxy-wise, the pool is fixed, and MySQL server state changes are not reflected in its config. This setup is resilient to `service haproxy reload/restart`. 8 | 9 | A backend server is able to tell HAProxy: 10 | 11 | - I'm good to participate in a pool (`HTTP 200`) 12 | - I'm in bad state; don't send traffic my way (`HTTP 503`) 13 | - I'm in maintenance mode. No error on my side, but don't send traffic my way (`HTTP 404`) 14 | 15 | The servers respond to an explicit request (check) type sent by HAProxy, such as `/check-lag` or `/ignore-lag`. 16 | 17 | HAProxy uses a `main` backend pool and a `backup` backend pool, such that the `backup` pool is activated when the `main` pool runs out of capacity. HAProxy uses different check types for the two pools. 18 | -------------------------------------------------------------------------------- /haproxy/haproxy-sample.cfg: -------------------------------------------------------------------------------- 1 | global 2 | ... 3 | 4 | frontend mysql_ro 5 | <% (1..@nbproc.to_i).each do |proc| %> 6 | bind 0.0.0.0:3306 process <%= proc %> 7 | <% end %> 8 | mode tcp 9 | acl mysql_not_enough_capacity nbsrv(mysql_ro_main) lt 2 10 | use_backend mysql_ro_backup if mysql_not_enough_capacity 11 | default_backend mysql_ro_main 12 | maxconn 20000 13 | timeout client 86400000 14 | 15 | backend mysql_ro_main 16 | option httpchk GET /check-lag 17 | http-check disable-on-404 18 | balance roundrobin 19 | retries 1 20 | timeout connect 1000 21 | timeout check 300 22 | timeout server 86400000 23 | 24 | default-server port 9876 fall 2 inter 5000 rise 1 downinter 5000 on-marked-down shutdown-sessions weight 10 25 | server my-db-0001 my-db-0001.heliumcarbon.com:3306 check 26 | server my-db-0002 my-db-0002.heliumcarbon.com:3306 check 27 | server my-db-0003 my-db-0003.heliumcarbon.com:3306 check 28 | server my-db-0004 my-db-0004.heliumcarbon.com:3306 check 29 | server my-db-0005 my-db-0005.heliumcarbon.com:3306 check 30 | server my-db-0006 my-db-0006.heliumcarbon.com:3306 check 31 | 32 | backend mysql_ro_backup 33 | option httpchk GET /ignore-lag 34 | http-check disable-on-404 35 | balance roundrobin 36 | retries 1 37 | timeout connect 1000 38 | timeout check 300 39 | timeout server 86400000 40 | 41 | default-server port 9876 fall 2 inter 10000 rise 1 downinter 10000 on-marked-down shutdown-sessions weight 10 42 | server my-db-0001 my-db-0001.heliumcarbon.com:3306 check 43 | server my-db-0002 my-db-0002.heliumcarbon.com:3306 check 44 | server my-db-0003 my-db-0003.heliumcarbon.com:3306 check 45 | server my-db-0004 my-db-0004.heliumcarbon.com:3306 check 46 | server my-db-0005 my-db-0005.heliumcarbon.com:3306 check 47 | server my-db-0006 my-db-0006.heliumcarbon.com:3306 check 48 | 49 | listen monitoring 0.0.0.0:1234 50 | mode health 51 | option tcplog 52 | 53 | listen statsctl 54 | ... 55 | -------------------------------------------------------------------------------- /puppet/mysql.pp: -------------------------------------------------------------------------------- 1 | class mysql ( 2 | ) { 3 | 4 | ... 5 | 6 | class { '::xinetd': } 7 | 8 | ::services { 'mysqlchk_general': 9 | ensure => present, 10 | service_name => 'mysqlchk_general', 11 | port => 9876, 12 | protocol => 'tcp', 13 | } 14 | 15 | file { '/etc/xinetd.d/mysqlchk_general': 16 | ensure => present, 17 | mode => '0644', 18 | source => 'puppet:///modules/mysql/etc/xinetd.d/mysqlchk_general', 19 | notify => Service['xinetd'], 20 | } 21 | 22 | ... 23 | } 24 | -------------------------------------------------------------------------------- /puppet/xinetd.pp: -------------------------------------------------------------------------------- 1 | # 2 | # xinetd daemon 3 | # 4 | class xinetd ( 5 | $ensure = running 6 | ) { 7 | package { 'xinetd': 8 | ensure => present, 9 | } 10 | service { 'xinetd': 11 | ensure => $ensure, 12 | enable => true, 13 | hasrestart => true, 14 | restart => '/usr/sbin/service xinetd restart', 15 | hasstatus => true, 16 | status => '/usr/sbin/service xinetd status', 17 | require => Package['xinetd'], 18 | } 19 | file { '/etc/xinetd.conf': 20 | ensure => present, 21 | mode => '0644', 22 | source => 'puppet:///modules/xinetd/etc/xinetd.conf', 23 | notify => Service['xinetd'], 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /scripts/xinetd-helper: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #/ 3 | 4 | # echo HTTP response and exit 5 | http_response() { 6 | local http_code="$1" 7 | local message="$2" 8 | 9 | /bin/echo -en "${http_code}\r\n" 10 | /bin/echo -en "Content-Type: text/plain\r\n" 11 | /bin/echo -en "Content-Length: ${#message}\r\n" 12 | /bin/echo -en "\r\n" 13 | /bin/echo -en "${message}\r\n" 14 | /bin/echo -en "\r\n" 15 | exit 16 | } 17 | 18 | http_200() { 19 | http_response "HTTP/1.1 200 OK" "200; $1" 20 | } 21 | 22 | http_404() { 23 | http_response "HTTP/1.1 404 Not Found" "404; $1" 24 | } 25 | 26 | http_503() { 27 | http_response "HTTP/1.1 503 Service Unavailable" "503; $1" 28 | } 29 | -------------------------------------------------------------------------------- /scripts/xinetd-mysql: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # General purpose and selector script for the various xinetd-mysql commands. 4 | # This script assumes it is being called by a HTTP client, as is the case where 5 | # it is being called by HAProxy or curl. It reads the HTTP request line, and based 6 | # on the URI path, it invokes the specific xinetd-mysql script 7 | # 8 | # For manual tests, it is possible to provide with replacement-to-URI argument, e.g. 9 | # xinetd-mysql /ignore-lag 10 | # If an argument is given, the script trusts it and does not attempt to read from standard input 11 | # 12 | 13 | if [ $# -eq 1 ] ; then 14 | uri_path="$1" 15 | else 16 | read http_first_line 17 | uri_path=$(echo $http_first_line | cut -d" " -f2) 18 | fi 19 | 20 | if [ "$uri_path" == "/master" ] ; then 21 | . xinetd-mysql-master 22 | exit 23 | fi 24 | 25 | if [ "$uri_path" == "/check-lag" ] ; then 26 | . xinetd-mysql-check-lag 27 | exit 28 | fi 29 | 30 | if [ "$uri_path" == "/ignore-lag" ] ; then 31 | . xinetd-mysql-ignore-lag 32 | exit 33 | fi 34 | 35 | if [ "$uri_path" == "/ignore-lag-allow-backup" ] ; then 36 | . xinetd-mysql-ignore-lag-allow-backup 37 | exit 38 | fi 39 | 40 | # ... More types of checks 41 | 42 | # Default path is lag check 43 | if [ "$uri_path" == "/" ] ; then 44 | . xinetd-mysql-check-lag 45 | exit 46 | fi 47 | 48 | . xinetd-helper 49 | http_503 "Unexpected URI (${uri_path})" 50 | -------------------------------------------------------------------------------- /scripts/xinetd-mysql-check-lag: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # This script checks if a MySQL server should take part in a pool, based on its role and its lag 4 | # 5 | 6 | . xinetd-helper 7 | 8 | # manual override: 9 | if [ -f /etc/xinetd-mysql/mysql-force-error-ro-pool-hint ] ; then 10 | http_503 "Forced hard out of pool. $(cat /etc/xinetd-mysql/mysql-force-error-ro-pool-hint)" 11 | fi 12 | if [ -f /etc/xinetd-mysql/mysql-force-into-ro-pool-hint ] ; then 13 | http_200 "Forced into pool. $(cat /etc/xinetd-mysql/mysql-force-into-ro-pool-hint)" 14 | fi 15 | if [ -f /etc/xinetd-mysql/mysql-force-outside-ro-pool-hint ] ; then 16 | http_404 "Forced out of pool. $(cat /etc/xinetd-mysql/mysql-force-outside-ro-pool-hint)" 17 | fi 18 | 19 | # Begin with should-or-should-not even consider to be in pool checks: 20 | if [ -f /etc/xinetd-mysql/mysql-master-hint ] ; then 21 | http_404 "Master: disabled" 22 | fi 23 | 24 | if [ -f /etc/xinetd-mysql/mysql-delayed-hint ] ; then 25 | http_404 "Delayed server: disabled" 26 | fi 27 | 28 | 29 | # Proceed to lag specific checks 30 | absolute_lag=$(mysql -h 127.0.0.1 -u ... -p ... -e "select absolute_lag as 'xinetd-mysql-lag' from ...;" -ss 2>/dev/null) 31 | exit_code=$? 32 | 33 | acceptable_lag=5 34 | 35 | if [ "$exit_code" != "0" ] ; then 36 | http_503 "MySQL error" 37 | fi 38 | 39 | if [ $absolute_lag -gt $acceptable_lag ] ; then 40 | http_503 "MySQL lagging" 41 | fi 42 | 43 | if [ $absolute_lag -le $acceptable_lag ] ; then 44 | http_200 "Lag OK" 45 | fi 46 | 47 | http_503 "Unexpected error" 48 | -------------------------------------------------------------------------------- /tests/xinetd-mysql-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Simplified shell-based tests for the xinetd-mysql scripts. 4 | # 5 | 6 | set -e 7 | 8 | no_such_uri_output=$(xinetd-mysql "/no-such-uri") 9 | echo ${no_such_uri_output} | grep -q "Unexpected URI" || { 10 | echo "xinetd-mysql /no-such-uri failed" 11 | echo "output was:" 12 | echo "${no_such_uri_output}" 13 | exit 1 14 | } 15 | 16 | lag_output=$(xinetd-mysql "/check-lag") 17 | echo ${lag_output} | grep -q "503; MySQL error" || { 18 | # Actually attempted to connect to MySQL, of course there isn't one in the testing environment 19 | echo "xinetd-mysql /check-lag failed" 20 | echo "output was:" 21 | echo "${lag_output}" 22 | exit 1 23 | } 24 | 25 | master_output=$(xinetd-mysql "/master") 26 | echo ${master_output} | grep -q "404; Not a master" || { 27 | # Was looking for a /etc/xinetd-mysql/ master hint file. Of course it's not there in the testing environment 28 | echo "xinetd-mysql /master failed" 29 | echo "output was:" 30 | echo "${master_output}" 31 | exit 1 32 | } 33 | 34 | exit 0 35 | -------------------------------------------------------------------------------- /xinetd/mysqlchk_general: -------------------------------------------------------------------------------- 1 | # 2 | # /etc/xinetd.d/mysqlchk_general 3 | # 4 | service mysqlchk_general 5 | { 6 | flags = REUSE 7 | socket_type = stream 8 | protocol = tcp 9 | port = 9876 10 | wait = no 11 | user = nobody 12 | server = /path/to/scipts/xinetd-mysql 13 | log_on_failure += USERID 14 | disable = no 15 | } 16 | -------------------------------------------------------------------------------- /xinetd/xinetd.conf: -------------------------------------------------------------------------------- 1 | defaults 2 | { 3 | instances = 50 4 | log_on_failure = HOST 5 | cps = 100 1 6 | } 7 | 8 | includedir /etc/xinetd.d 9 | --------------------------------------------------------------------------------