├── Dockerfile ├── README.md ├── conf ├── 20-innodb.cnf ├── 30-replication.cnf ├── 90-galera.cnf ├── 95-mysqld-extra-opts.cnf └── mysqld_safe_syslog.cnf ├── init.sh ├── mariadb-cluster-setup-macvlan.yaml ├── mariadb-cluster-setup.yaml ├── my.cnf ├── nginx-galera.conf ├── swarm ├── README.md ├── galera-cluster-s.yaml ├── galera-cluster.env ├── galera-cluster.yaml └── nginx-galera.conf └── version /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:24.04 2 | MAINTAINER Wei Zhou 3 | 4 | ENV DEBIAN_FRONTEND noninteractive 5 | 6 | RUN apt update -qq && \ 7 | apt upgrade -y && \ 8 | apt install -y curl iproute2 net-tools iputils-ping apt-transport-https 9 | 10 | RUN groupadd -r mysql && useradd -r -g mysql mysql && \ 11 | mkdir -p /var/run/mysqld && \ 12 | chown mysql:mysql /var/run/mysqld 13 | 14 | RUN curl -LsS https://downloads.mariadb.com/MariaDB/mariadb_repo_setup | bash && \ 15 | apt update -qq && \ 16 | apt install -y mariadb-server mariadb-client mariadb-backup && \ 17 | apt install -y galera-arbitrator-4 galera-4 && \ 18 | rm -rf /var/lib/mysql && \ 19 | mkdir /var/lib/mysql 20 | 21 | COPY conf/ /etc/mysql/conf.d/ 22 | COPY my.cnf /etc/mysql/my.cnf 23 | COPY init.sh /init.sh 24 | 25 | EXPOSE 3306 4444 4567 4568 26 | 27 | ENTRYPOINT ["/init.sh"] 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MariaDB Galera Cluster in Docker Containers 2 | 3 | This repository maintains the Dockerfile and scripts to build MariaDB Galera Cluster in Docker Containers. 4 | 5 | ## What does it do 6 | 7 | Dockerfile : 8 | 9 | 1. Build from official docker container 10 | 2. Install mariadb-server mariadb-client mariadb-backup and dependencies 11 | 3. Copy files (conf/, my.cnf, init.sh) 12 | 4. Set /init.sh as entrypoint 13 | 14 | init.sh : 15 | 16 | 1. Modify mysql config file based on environment variables. 17 | 2. Check service in the cluster to see if this is a new cluster 18 | 3. Install mysql from scratch, if mysql data is not found 19 | 4. Updated safe_to_bootstrap to 1 in /var/lib/mysql/grastate.dat, if mysql data is found and this is a new cluster 20 | 5. Start mysqld with different arguments. 21 | 22 | ## Docker image 23 | The docker image can be found at https://hub.docker.com/repository/docker/ustcweizhou/mariadb-cluster 24 | 25 | ## Example (DBs use private IP and accessible via nginx) 26 | 27 | You can test with command: 28 | 29 | docker-compose -f mariadb-cluster-setup.yaml up -d 30 | 31 | If it does not work, please download latest docker-compose and try again. 32 | 33 | sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose 34 | sudo chmod +x /usr/local/bin/docker-compose 35 | 36 | When all containsers are running, there is a nginx container running with nginx-galera.conf to expose the mariadb cluster. 37 | 38 | # docker-compose -f mariadb-cluster-setup.yaml ps 39 | Name Command State Ports 40 | ------------------------------------------------------------------------------------------------------ 41 | mariadb-cluster_db01_1 /init.sh Up (healthy) 3306/tcp, 4444/tcp, 4567/tcp, 4568/tcp 42 | mariadb-cluster_db02_1 /init.sh Up 3306/tcp, 4444/tcp, 4567/tcp, 4568/tcp 43 | mariadb-cluster_db03_1 /init.sh Up 3306/tcp, 4444/tcp, 4567/tcp, 4568/tcp 44 | mariadb-cluster_dbvip_1 nginx -g daemon off; Up 0.0.0.0:13306->3306/tcp, 80/tcp 45 | 46 | # mysql -h 127.0.0.1 -P13306 -uroot -pcloudstack -e "show status where variable_name in ('wsrep_cluster_status', 'wsrep_incoming_addresses','wsrep_local_state_comment');" 47 | +---------------------------+----------------------------------------+ 48 | | Variable_name | Value | 49 | +---------------------------+----------------------------------------+ 50 | | wsrep_local_state_comment | Synced | 51 | | wsrep_incoming_addresses | 172.16.10.11,172.16.10.12,172.16.10.13 | 52 | | wsrep_cluster_status | Primary | 53 | +---------------------------+----------------------------------------+ 54 | 55 | ## Example (DBs use public IPs which are directly accessible) 56 | 57 | You can test with command: 58 | 59 | docker-compose -f mariadb-cluster-setup-macvlan.yaml up -d 60 | 61 | When all containsers are running, 62 | 63 | # docker-compose -f mariadb-cluster-setup-macvlan.yaml ps 64 | Name Command State Ports 65 | --------------------------------------------------------------- 66 | docker-mariadb-cluster_db01_1 /init.sh Up (healthy) 67 | docker-mariadb-cluster_db02_1 /init.sh Up (healthy) 68 | docker-mariadb-cluster_db03_1 /init.sh Up 69 | 70 | The IPs are accessible, except the container host 71 | 72 | # mysql -h 10.0.33.53 -P3306 -uroot -pcloudstack -e "show status where variable_name in ('wsrep_cluster_status', 'wsrep_incoming_addresses','wsrep_local_state_comment');" 73 | mysql: [Warning] Using a password on the command line interface can be insecure. 74 | +---------------------------+---------------------------------------------------+ 75 | | Variable_name | Value | 76 | +---------------------------+---------------------------------------------------+ 77 | | wsrep_local_state_comment | Synced | 78 | | wsrep_incoming_addresses | 10.0.33.53:3306,10.0.35.200:3306,10.0.33.212:3306 | 79 | | wsrep_cluster_status | Primary | 80 | +---------------------------+---------------------------------------------------+ 81 | 82 | To access the service from the container host, 83 | 84 | ip link add host-nic link eth0 type macvlan mode bridge 85 | ip addr add 10.0.33.6/20 dev host-nic (only if the host IP is in different range as container IP) 86 | ip link set host-nic up 87 | ip route add 10.0.33.53/32 dev host-nic 88 | 89 | ## Releases 90 | 91 | v1.0 Ubuntu 18.04 and Mariadb 10.4 92 | v1.1 Ubuntu 20.04 and Mariadb 10.5.8 93 | v1.2 Ubuntu 20.04 and Mariadb 10.5.12 94 | v1.3 Ubuntu 22.04 and Mariadb 11.3.2 95 | v1.4 Ubuntu 24.04 and Mariadb 11.7.2 96 | -------------------------------------------------------------------------------- /conf/20-innodb.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | # 3 | # * InnoDB 4 | # 5 | # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. 6 | # Read the manual for more InnoDB related options. There are many! 7 | # you can't just change log file size, it requires special procedure 8 | innodb_log_file_size = 50M 9 | innodb_log_buffer_size = 8M 10 | innodb_file_per_table = 1 11 | innodb_open_files = 400 12 | innodb_io_capacity = 400 13 | innodb_flush_method = O_DIRECT 14 | innodb_buffer_pool_size = 256M 15 | -------------------------------------------------------------------------------- /conf/30-replication.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | log_bin = /var/log/mysql/mariadb-bin 3 | sync_binlog = 0 4 | log_bin_index = /var/log/mysql/mariadb-bin.index 5 | expire_logs_days = 3 6 | max_binlog_size = 100M 7 | 8 | -------------------------------------------------------------------------------- /conf/90-galera.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | query_cache_size = 0 3 | binlog_format = ROW 4 | default_storage_engine = InnoDB 5 | innodb_autoinc_lock_mode = 2 6 | innodb_doublewrite = 1 7 | server_id = 1 8 | wsrep_gtid_mode = ON 9 | wsrep_gtid_domain_id = 0 10 | gtid_domain_id = 1 11 | wsrep_node_name = db01 12 | log_slave_updates = ON 13 | innodb_flush_log_at_trx_commit = 2 14 | wsrep_on = ON 15 | wsrep_provider_options = "gcache.size=512M" 16 | wsrep_cluster_address = 17 | wsrep_cluster_name = mariadb 18 | wsrep_sst_method = mariabackup 19 | wsrep_sst_auth = mariabackup:mypassword 20 | wsrep_provider = /usr/lib/galera/libgalera_smm.so 21 | wsrep_slave_threads = 192 22 | wsrep_node_address = 23 | wsrep_node_incoming_address = 24 | wsrep_node_name = 25 | -------------------------------------------------------------------------------- /conf/95-mysqld-extra-opts.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | innodb_rollback_on_timeout = 1 3 | innodb_lock_wait_timeout = 600 4 | 5 | -------------------------------------------------------------------------------- /conf/mysqld_safe_syslog.cnf: -------------------------------------------------------------------------------- 1 | [mysqld_safe] 2 | skip_log_error 3 | syslog 4 | 5 | -------------------------------------------------------------------------------- /init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | IP=$(hostname --ip-address | cut -d" " -f1) 5 | HOSTNAME=$(hostname -s) 6 | 7 | INIT_SQL="/init.sql" 8 | INIT_LOG="/init.log" 9 | ln -sf /proc/$$/fd/1 $INIT_LOG 10 | 11 | if [ -z "$CHECK_MAX_RETRIES" ];then 12 | CHECK_MAX_RETRIES=60 13 | fi 14 | 15 | if [ -z "$CHECK_INTERVAL" ];then 16 | CHECK_INTERVAL=1 17 | fi 18 | 19 | check_server_status() { 20 | local server=$1 21 | set +e 22 | local status=$(mysql -h $server -uroot -p${DB_ROOT_PASSWORD} -NB -e "SHOW STATUS WHERE Variable_name='wsrep_local_state_comment'") 23 | if [ $? -ne 0 ];then 24 | echo "Failed to connect database on $server" 25 | else 26 | status=$(echo $status | awk '{print $2}') 27 | if [ "$status" != "Synced" ];then 28 | echo "Database server $server is not Synced but $status" 29 | fi 30 | fi 31 | set -e 32 | } 33 | 34 | wait_for_server_to_be_up() { 35 | local server=$1 36 | local retry=$CHECK_MAX_RETRIES 37 | echo -n "====== Checking server $server status " >>$INIT_LOG 38 | while [ $retry -gt 0 ];do 39 | echo -n "." >>$INIT_LOG 40 | status=$(check_server_status $server) 41 | if [ -z "$status" ];then 42 | echo " Done ======" >>$INIT_LOG 43 | break 44 | fi 45 | let retry=retry-1 46 | sleep $CHECK_INTERVAL 47 | done 48 | if [ $retry -eq 0 ];then 49 | echo " timeout. Exiting ======" >>$INIT_LOG 50 | exit 1 51 | fi 52 | } 53 | mkdir -p /var/log/mysql 54 | chown -R mysql:mysql /var/lib/mysql /var/log/mysql 55 | 56 | if [ -z "$CLUSTER_ADDRESS" ];then 57 | echo "Missing CLUSTER_ADDRESS" >&2 58 | exit 1 59 | else 60 | sed -i "s|^wsrep_cluster_address =.*|wsrep_cluster_address = '${CLUSTER_ADDRESS}'|g" /etc/mysql/conf.d/90-galera.cnf 61 | sed -i "s|^wsrep_node_address =.*|wsrep_node_address = ${IP}|g" /etc/mysql/conf.d/90-galera.cnf 62 | sed -i "s|^wsrep_node_incoming_address =.*|wsrep_node_incoming_address = ${IP}|g" /etc/mysql/conf.d/90-galera.cnf 63 | sed -i "s|^wsrep_node_name =.*|wsrep_node_name = ${HOSTNAME}|g" /etc/mysql/conf.d/90-galera.cnf 64 | fi 65 | 66 | if [ -z "${DB_ROOT_PASSWORD}" ];then 67 | echo "Missing DB_ROOT_PASSWORD" >&2 68 | exit 1 69 | fi 70 | 71 | if [ -z "${DB_MARIABACKUP_PASSWORD}" ];then 72 | echo "Missing DB_MARIABACKUP_PASSWORD" >&2 73 | exit 1 74 | else 75 | sed -i "s|^wsrep_sst_auth =.*|wsrep_sst_auth = mariabackup:${DB_MARIABACKUP_PASSWORD}|g" /etc/mysql/conf.d/90-galera.cnf 76 | fi 77 | 78 | master_server= 79 | is_master_server=true 80 | new_cluster=true 81 | hosts=$(echo $CLUSTER_ADDRESS | rev | cut -d '/' -f1 | rev | tr "," " ") 82 | if [ ! -z "$hosts" ];then 83 | for host in $hosts;do 84 | if [ -z "$master_server" ];then 85 | master_server=$host # master server is the first server in CLUSTER_ADDRESS 86 | if [ "$master_server" != "$IP" ] && [ "$master_server" != "$HOSTNAME" ];then 87 | is_master_server=false 88 | fi 89 | fi 90 | status=$(check_server_status $host) 91 | if [ -z "$status" ];then 92 | new_cluster=false 93 | break 94 | fi 95 | done 96 | fi 97 | 98 | cmd="mysqld" 99 | if [ "$new_cluster" = "true" ];then 100 | if [ "$is_master_server" = "true" ];then 101 | cmd+=" --wsrep-new-cluster" 102 | else 103 | wait_for_server_to_be_up $master_server 104 | fi 105 | fi 106 | 107 | if [ ! -d "/var/lib/mysql/mysql" ];then 108 | echo "====== Installing mysql from scratch ======" >>$INIT_LOG 109 | mysql_install_db --datadir="/var/lib/mysql/" --user=mysql 110 | if [ "$new_cluster" = "true" ];then 111 | cmd+=" --init-file=$INIT_SQL" 112 | fi 113 | elif [ "$new_cluster" = "true" ] && [ -f "/var/lib/mysql/grastate.dat" ] && [ "$is_master_server" = "true" ];then 114 | echo "====== Starting existing mariadb cluster ======" >>$INIT_LOG 115 | sed -i "s|^safe_to_bootstrap: 0|safe_to_bootstrap: 1|g" /var/lib/mysql/grastate.dat 116 | echo "====== Updated safe_to_bootstrap to 1 in /var/lib/mysql/grastate.dat ======" >>$INIT_LOG 117 | fi 118 | 119 | if [ ! -f "$INIT_SQL" ];then 120 | cat >$INIT_SQL << EOF 121 | DELETE FROM mysql.global_priv WHERE NOT (Host = 'localhost' AND User = 'mariadb.sys'); 122 | CREATE USER 'root'@'%' IDENTIFIED BY "${DB_ROOT_PASSWORD}"; 123 | GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION; 124 | CREATE USER 'mariabackup'@'%' IDENTIFIED BY "${DB_MARIABACKUP_PASSWORD}"; 125 | GRANT RELOAD, PROCESS, LOCK TABLES, REPLICATION CLIENT ON *.* TO 'mariabackup'@'%'; 126 | FLUSH PRIVILEGES; 127 | EOF 128 | fi 129 | 130 | echo "====== Executing command: $cmd ======" >>$INIT_LOG 131 | $cmd 132 | -------------------------------------------------------------------------------- /mariadb-cluster-setup-macvlan.yaml: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | services: 3 | db01: 4 | image: ustcweizhou/mariadb-cluster:latest 5 | hostname: db01 6 | networks: 7 | bridge-mgmt: 8 | ipv4_address: 10.0.34.1 9 | volumes: 10 | - /docker/db01:/var/lib/mysql 11 | healthcheck: 12 | test: ["CMD", "mysqladmin", "-uroot", "-pcloudstack", "ping"] 13 | interval: 10s 14 | timeout: 10s 15 | retries: 60 16 | start_period: 10s 17 | environment: 18 | - NODE_NAME=db01 19 | - CLUSTER_ADDRESS=gcomm://db01,db02,db03 20 | - DB_ROOT_PASSWORD=cloudstack 21 | - DB_MARIABACKUP_PASSWORD=cloudstack 22 | 23 | db02: 24 | image: ustcweizhou/mariadb-cluster:latest 25 | hostname: db02 26 | depends_on: 27 | db01: 28 | condition: service_healthy 29 | networks: 30 | bridge-mgmt: 31 | ipv4_address: 10.0.34.2 32 | volumes: 33 | - /docker/db02:/var/lib/mysql 34 | healthcheck: 35 | test: ["CMD", "mysqladmin", "-uroot", "-pcloudstack", "ping"] 36 | interval: 10s 37 | timeout: 10s 38 | retries: 60 39 | start_period: 10s 40 | environment: 41 | - NODE_NAME=db02 42 | - CLUSTER_ADDRESS=gcomm://db01,db02,db03 43 | - DB_ROOT_PASSWORD=cloudstack 44 | - DB_MARIABACKUP_PASSWORD=cloudstack 45 | 46 | db03: 47 | image: ustcweizhou/mariadb-cluster:latest 48 | hostname: db03 49 | depends_on: 50 | db01: 51 | condition: service_healthy 52 | db02: 53 | condition: service_healthy 54 | networks: 55 | bridge-mgmt: 56 | ipv4_address: 10.0.34.3 57 | volumes: 58 | - /docker/db03:/var/lib/mysql 59 | environment: 60 | - NODE_NAME=db03 61 | - CLUSTER_ADDRESS=gcomm://db01,db02,db03 62 | - DB_ROOT_PASSWORD=cloudstack 63 | - DB_MARIABACKUP_PASSWORD=cloudstack 64 | 65 | networks: 66 | bridge-mgmt: 67 | driver: macvlan 68 | driver_opts: 69 | parent: eth0 70 | com.docker.network.bridge.name: "bridge-mgmt" 71 | ipam: 72 | config: 73 | - subnet: "10.0.32.0/20" 74 | gateway: "10.0.32.1" 75 | ip_range: "10.0.34.0/30" 76 | -------------------------------------------------------------------------------- /mariadb-cluster-setup.yaml: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | services: 3 | db01: 4 | image: ustcweizhou/mariadb-cluster:latest 5 | hostname: db01 6 | networks: 7 | br-db: 8 | ipv4_address: 172.16.10.11 9 | volumes: 10 | - /docker/db01:/var/lib/mysql 11 | healthcheck: 12 | test: ["CMD", "mysqladmin", "-uroot", "-pcloudstack", "ping"] 13 | interval: 10s 14 | timeout: 10s 15 | retries: 60 16 | start_period: 10s 17 | environment: 18 | - NODE_NAME=db01 19 | - CLUSTER_ADDRESS=gcomm://db01,db02,db03 20 | - DB_ROOT_PASSWORD=cloudstack 21 | - DB_MARIABACKUP_PASSWORD=cloudstack 22 | 23 | db02: 24 | image: ustcweizhou/mariadb-cluster:latest 25 | hostname: db02 26 | depends_on: 27 | db01: 28 | condition: service_healthy 29 | networks: 30 | br-db: 31 | ipv4_address: 172.16.10.12 32 | volumes: 33 | - /docker/db02:/var/lib/mysql 34 | healthcheck: 35 | test: ["CMD", "mysqladmin", "-uroot", "-pcloudstack", "ping"] 36 | interval: 10s 37 | timeout: 10s 38 | retries: 60 39 | start_period: 10s 40 | environment: 41 | - NODE_NAME=db02 42 | - CLUSTER_ADDRESS=gcomm://db01,db02,db03 43 | - DB_ROOT_PASSWORD=cloudstack 44 | - DB_MARIABACKUP_PASSWORD=cloudstack 45 | 46 | db03: 47 | image: ustcweizhou/mariadb-cluster:latest 48 | hostname: db03 49 | depends_on: 50 | db01: 51 | condition: service_healthy 52 | db02: 53 | condition: service_healthy 54 | networks: 55 | br-db: 56 | ipv4_address: 172.16.10.13 57 | volumes: 58 | - /docker/db03:/var/lib/mysql 59 | environment: 60 | - NODE_NAME=db03 61 | - CLUSTER_ADDRESS=gcomm://db01,db02,db03 62 | - DB_ROOT_PASSWORD=cloudstack 63 | - DB_MARIABACKUP_PASSWORD=cloudstack 64 | 65 | dbvip: 66 | image: nginx 67 | hostname: dbvip 68 | networks: 69 | br-db: 70 | ipv4_address: 172.16.10.14 71 | ports: 72 | - "13306:3306" 73 | volumes: 74 | - ./nginx-galera.conf:/etc/nginx/nginx.conf 75 | depends_on: 76 | db01: 77 | condition: service_healthy 78 | 79 | networks: 80 | br-db: 81 | driver: bridge 82 | driver_opts: 83 | com.docker.network.bridge.name: "br-db" 84 | ipam: 85 | driver: default 86 | config: 87 | - subnet: 172.16.10.0/24 88 | gateway: 172.16.10.254 89 | -------------------------------------------------------------------------------- /my.cnf: -------------------------------------------------------------------------------- 1 | # MariaDB database server configuration file. 2 | # 3 | # You can copy this file to one of: 4 | # - "/etc/mysql/my.cnf" to set global options, 5 | # - "~/.my.cnf" to set user-specific options. 6 | # 7 | # One can use all long options that the program supports. 8 | # Run program with --help to get a list of available options and with 9 | # --print-defaults to see which it would actually understand and use. 10 | # 11 | # For explanations see 12 | # http://dev.mysql.com/doc/mysql/en/server-system-variables.html 13 | 14 | # This will be passed to all mysql clients 15 | # It has been reported that passwords should be enclosed with ticks/quotes 16 | # escpecially if they contain "#" chars... 17 | # Remember to edit /etc/mysql/debian.cnf when changing the socket location. 18 | [client] 19 | port=3306 20 | socket = /var/run/mysqld/mysqld.sock 21 | 22 | # Here is entries for some specific programs 23 | # The following values assume you have at least 32M ram 24 | 25 | # This was formally known as [safe_mysqld]. Both versions are currently parsed. 26 | [mysqld_safe] 27 | socket = /var/run/mysqld/mysqld.sock 28 | nice = 0 29 | 30 | [mysqld] 31 | innodb_flush_log_at_trx_commit=1 32 | innodb_old_blocks_time=1000 33 | innodb_buffer_pool_size=128M 34 | sync_binlog=0 35 | query_cache_type=1 36 | innodb_log_file_size=48M 37 | # 38 | # * Basic Settings 39 | # 40 | user = mysql 41 | pid-file = /var/run/mysqld/mysqld.pid 42 | socket = /var/run/mysqld/mysqld.sock 43 | port=3306 44 | basedir = /usr 45 | datadir = /var/lib/mysql 46 | tmpdir = /tmp 47 | lc_messages_dir = /usr/share/mysql 48 | lc_messages = en_US 49 | skip-external-locking 50 | # 51 | # Instead of skip-networking the default is now to listen only on 52 | # localhost which is more compatible and is not less secure. 53 | #bind-address = 127.0.0.1 54 | # 55 | # * Fine Tuning 56 | # 57 | max_connections = 100 58 | connect_timeout = 5 59 | wait_timeout = 600 60 | max_allowed_packet=16M 61 | thread_cache_size = 128 62 | sort_buffer_size = 4M 63 | bulk_insert_buffer_size = 16M 64 | tmp_table_size = 32M 65 | max_heap_table_size = 32M 66 | # 67 | # * MyISAM 68 | # 69 | # This replaces the startup script and checks MyISAM tables if needed 70 | # the first time they are touched. On error, make copy and try a repair. 71 | myisam_recover_options = BACKUP 72 | key_buffer_size = 128M 73 | #open-files-limit = 2000 74 | table_open_cache = 400 75 | myisam_sort_buffer_size = 512M 76 | concurrent_insert = 2 77 | read_buffer_size = 2M 78 | read_rnd_buffer_size = 1M 79 | # 80 | # * Query Cache Configuration 81 | # 82 | # Cache only tiny result sets, so we can fit more in the query cache. 83 | query_cache_limit = 128K 84 | query_cache_size=16M 85 | # for more write intensive setups, set to DEMAND or OFF 86 | #query_cache_type = DEMAND 87 | # 88 | # * Logging and Replication 89 | # 90 | # Both location gets rotated by the cronjob. 91 | # Be aware that this log type is a performance killer. 92 | # As of 5.1 you can enable the log at runtime! 93 | #general_log_file = /var/log/mysql/mysql.log 94 | #general_log = 1 95 | # 96 | # Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf. 97 | # 98 | # we do want to know about network errors and such 99 | log_warnings = 2 100 | # 101 | # Enable the slow query log to see queries with especially long duration 102 | #slow_query_log[={0|1}] 103 | slow_query_log_file = /var/log/mysql/mariadb-slow.log 104 | long_query_time = 10 105 | #log_slow_rate_limit = 1000 106 | log_slow_verbosity = query_plan 107 | 108 | #log-queries-not-using-indexes 109 | #log_slow_admin_statements 110 | # 111 | # The following can be used as easy to replay backup logs or for replication. 112 | # note: if you are setting up a replication slave, see README.Debian about 113 | # other settings you may need to change. 114 | #server-id = 1 115 | #report_host = master1 116 | #auto_increment_increment = 2 117 | #auto_increment_offset = 1 118 | #log_bin = /var/log/mysql/mariadb-bin 119 | #log_bin_index = /var/log/mysql/mariadb-bin.index 120 | # not fab for performance, but safer 121 | #sync_binlog = 1 122 | expire_logs_days = 10 123 | max_binlog_size = 100M 124 | # slaves 125 | #relay_log = /var/log/mysql/relay-bin 126 | #relay_log_index = /var/log/mysql/relay-bin.index 127 | #relay_log_info_file = /var/log/mysql/relay-bin.info 128 | #log_slave_updates 129 | #read_only 130 | # 131 | # If applications support it, this stricter sql_mode prevents some 132 | # mistakes like inserting invalid dates etc. 133 | #sql_mode = NO_ENGINE_SUBSTITUTION,TRADITIONAL 134 | # 135 | # * InnoDB 136 | # 137 | # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. 138 | # Read the manual for more InnoDB related options. There are many! 139 | default_storage_engine = InnoDB 140 | innodb_buffer_pool_size = 256M 141 | innodb_log_buffer_size = 8M 142 | innodb_file_per_table = 1 143 | innodb_open_files = 400 144 | innodb_io_capacity = 400 145 | innodb_flush_method = O_DIRECT 146 | # 147 | # * Security Features 148 | # 149 | # Read the manual, too, if you want chroot! 150 | # chroot = /var/lib/mysql/ 151 | # 152 | # For generating SSL certificates I recommend the OpenSSL GUI "tinyca". 153 | # 154 | # ssl-ca=/etc/mysql/cacert.pem 155 | # ssl-cert=/etc/mysql/server-cert.pem 156 | # ssl-key=/etc/mysql/server-key.pem 157 | 158 | # 159 | # * Galera-related settings 160 | # 161 | [galera] 162 | # Mandatory settings 163 | #wsrep_on=ON 164 | #wsrep_provider= 165 | #wsrep_cluster_address= 166 | #binlog_format=row 167 | #default_storage_engine=InnoDB 168 | #innodb_autoinc_lock_mode=2 169 | # 170 | # Allow server to accept connections on all interfaces. 171 | # 172 | bind-address=0.0.0.0 173 | # 174 | # Optional setting 175 | #wsrep_slave_threads=1 176 | #innodb_flush_log_at_trx_commit=0 177 | 178 | [mysqldump] 179 | quick 180 | quote-names 181 | max_allowed_packet=16M 182 | 183 | [mysql] 184 | #no-auto-rehash # faster start of mysql but no tab completion 185 | 186 | [isamchk] 187 | key_buffer = 16M 188 | 189 | # 190 | # * IMPORTANT: Additional settings that can override those from this file! 191 | # The files must end with '.cnf', otherwise they'll be ignored. 192 | # 193 | !include /etc/mysql/mariadb.cnf 194 | !includedir /etc/mysql/conf.d/ 195 | -------------------------------------------------------------------------------- /nginx-galera.conf: -------------------------------------------------------------------------------- 1 | error_log stderr notice; 2 | worker_processes auto; 3 | events { 4 | multi_accept on; 5 | use epoll; 6 | worker_connections 1024; 7 | } 8 | stream { 9 | upstream mariadb { 10 | server 172.16.10.11:3306; 11 | server 172.16.10.12:3306 backup; 12 | server 172.16.10.13:3306 backup; 13 | } 14 | server { 15 | listen 0.0.0.0:3306; 16 | proxy_pass mariadb; 17 | proxy_timeout 10m; 18 | proxy_connect_timeout 1s; 19 | } 20 | } 21 | 22 | -------------------------------------------------------------------------------- /swarm/README.md: -------------------------------------------------------------------------------- 1 | # MariaDB Galera Cluster by Docker Swarm 2 | 3 | This describes how to set up mariadb galera cluster by docker swarm. There are two modes: 4 | - Single-server mode 5 | - Multiple-servers mode 6 | 7 | ## 1. Prerequisites 8 | 9 | ### Run Docker in swarm mode 10 | ``` 11 | https://docs.docker.com/engine/swarm/swarm-mode/ 12 | ``` 13 | 14 | ### Join nodes to swarm (Required by Multiple-servers mode) 15 | ``` 16 | https://docs.docker.com/engine/swarm/join-nodes/ 17 | ``` 18 | Add labels to swarm nodes 19 | ``` 20 | # docker node update --label-add role=infra01 node42 21 | # docker node update --label-add role=infra02 node43 22 | # docker node update --label-add role=infra03 node44 23 | ``` 24 | 25 | ## 2. Modify ENV file 26 | 27 | galera-cluster.env stores some database configurations. Data will be put to $DATA_DIR/$PROJECT/ folder on swarm nodes. 28 | 29 | ``` 30 | PROJECT=test 31 | DATA_DIR=/root/docker/ 32 | DB_ROOT_PASSWORD=cloudstack 33 | DB_MARIABACKUP_PASSWORD=cloudstack 34 | ``` 35 | 36 | ## 3. Set up a Mariadb Galera cluster 37 | 38 | ### Load global variables 39 | ``` 40 | export $(cat galera-cluster.env) >/dev/null 2>&1 41 | ``` 42 | 43 | ### Set up cluster in Single-server mode 44 | All services will be running on same server. 45 | ``` 46 | docker stack deploy --compose-file=galera-cluster-s.yaml galera 47 | ``` 48 | 49 | ### Set up cluster in Multiple-servers mode 50 | ``` 51 | docker stack deploy --compose-file=galera-cluster.yaml galera 52 | ``` 53 | Database services will be running on different swarm nodes. 54 | 55 | db01 -> server with label role=infra01 56 | db02 -> server with label role=infra02 57 | db03 -> server with label role=infra03 58 | 59 | ## Expectations 60 | 61 | ### docker services 62 | 63 | ``` 64 | # docker service ls 65 | ID NAME MODE REPLICAS IMAGE PORTS 66 | nvdvgc95h2tz galera_db01 replicated 1/1 ustcweizhou/mariadb-cluster:ubuntu18-10.4 67 | zmftqcxxe2mw galera_db02 replicated 1/1 ustcweizhou/mariadb-cluster:ubuntu18-10.4 68 | wr6gghkdrctd galera_db03 replicated 1/1 ustcweizhou/mariadb-cluster:ubuntu18-10.4 69 | ri5jr1caz58i galera_dbvip replicated 1/1 nginx:latest *:13306->3306/tcp 70 | ``` 71 | 72 | ### services are running on different swarm nodes 73 | ``` 74 | # docker service ps galera_db01 75 | ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS 76 | 10c4u0b6mkdk galera_db01.1 ustcweizhou/mariadb-cluster:ubuntu18-10.4 node42 Running Running 36 minutes ago 77 | 78 | # docker service ps galera_db02 79 | ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS 80 | y6k6wlf8lo5h galera_db02.1 ustcweizhou/mariadb-cluster:ubuntu18-10.4 node43 Running Running 36 minutes ago 81 | 82 | # docker service ps galera_db03 83 | ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS 84 | yax7cbo415ee galera_db03.1 ustcweizhou/mariadb-cluster:ubuntu18-10.4 node44 Running Running 36 minutes ago 85 | 86 | # docker service ps galera_dbvip 87 | ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS 88 | oi9cdv2vjjqu galera_dbvip.1 nginx:latest swarm-manager Running Running 36 minutes ago 89 | ``` 90 | 91 | ### MariaDB status 92 | ``` 93 | # mysql -h 127.0.0.1 -P13306 -uroot -pcloudstack -e "show status where variable_name in ('wsrep_cluster_status', 'wsrep_incoming_addresses','wsrep_local_state_comment');" 94 | mysql: [Warning] Using a password on the command line interface can be insecure. 95 | +---------------------------+-----------------------------------------+ 96 | | Variable_name | Value | 97 | +---------------------------+-----------------------------------------+ 98 | | wsrep_local_state_comment | Synced | 99 | | wsrep_incoming_addresses | 192.168.10.3,192.168.10.9,192.168.10.14 | 100 | | wsrep_cluster_status | Primary | 101 | +---------------------------+-----------------------------------------+ 102 | ``` 103 | 104 | -------------------------------------------------------------------------------- /swarm/galera-cluster-s.yaml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | services: 3 | db01: 4 | image: ustcweizhou/mariadb-cluster:ubuntu18-10.4 5 | hostname: db01 6 | networks: 7 | overlay-db: 8 | volumes: 9 | - ${DATA_DIR}/$PROJECT/db01:/var/lib/mysql 10 | environment: 11 | - NODE_NAME=db01 12 | - CLUSTER_ADDRESS=gcomm://db01,db02,db03 13 | - DB_ROOT_PASSWORD=${DB_ROOT_PASSWORD} 14 | - DB_MARIABACKUP_PASSWORD=${DB_MARIABACKUP_PASSWORD} 15 | 16 | db02: 17 | image: ustcweizhou/mariadb-cluster:ubuntu18-10.4 18 | hostname: db02 19 | networks: 20 | overlay-db: 21 | volumes: 22 | - ${DATA_DIR}/$PROJECT/db02:/var/lib/mysql 23 | environment: 24 | - NODE_NAME=db02 25 | - CLUSTER_ADDRESS=gcomm://db01,db02,db03 26 | - DB_ROOT_PASSWORD=${DB_ROOT_PASSWORD} 27 | - DB_MARIABACKUP_PASSWORD=${DB_MARIABACKUP_PASSWORD} 28 | 29 | db03: 30 | image: ustcweizhou/mariadb-cluster:ubuntu18-10.4 31 | hostname: db03 32 | networks: 33 | overlay-db: 34 | volumes: 35 | - ${DATA_DIR}/$PROJECT/db03:/var/lib/mysql 36 | environment: 37 | - NODE_NAME=db03 38 | - CLUSTER_ADDRESS=gcomm://db01,db02,db03 39 | - DB_ROOT_PASSWORD=${DB_ROOT_PASSWORD} 40 | - DB_MARIABACKUP_PASSWORD=${DB_MARIABACKUP_PASSWORD} 41 | 42 | dbvip: 43 | image: nginx 44 | hostname: dbvip 45 | networks: 46 | overlay-db: 47 | ports: 48 | - "13306:3306" 49 | configs: 50 | - source: nginx-galera 51 | target: /etc/nginx/nginx.conf 52 | deploy: 53 | restart_policy: 54 | condition: on-failure 55 | 56 | configs: 57 | nginx-galera: 58 | file: ./nginx-galera.conf 59 | 60 | networks: 61 | overlay-db: 62 | driver: overlay 63 | attachable: true 64 | ipam: 65 | driver: default 66 | config: 67 | - subnet: 192.168.10.0/24 68 | -------------------------------------------------------------------------------- /swarm/galera-cluster.env: -------------------------------------------------------------------------------- 1 | # Database configurations, data will be put to $DATA_DIR/$PROJECT/ folder 2 | PROJECT=test 3 | DATA_DIR=/root/docker/ 4 | DB_ROOT_PASSWORD=cloudstack 5 | DB_MARIABACKUP_PASSWORD=cloudstack 6 | -------------------------------------------------------------------------------- /swarm/galera-cluster.yaml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | services: 3 | db01: 4 | image: ustcweizhou/mariadb-cluster:ubuntu18-10.4 5 | hostname: db01 6 | networks: 7 | overlay-db: 8 | volumes: 9 | - ${DATA_DIR}/$PROJECT/db01:/var/lib/mysql 10 | environment: 11 | - NODE_NAME=db01 12 | - CLUSTER_ADDRESS=gcomm://db01,db02,db03 13 | - DB_ROOT_PASSWORD=${DB_ROOT_PASSWORD} 14 | - DB_MARIABACKUP_PASSWORD=${DB_MARIABACKUP_PASSWORD} 15 | deploy: 16 | placement: 17 | constraints: 18 | - "node.labels.role==infra01" 19 | 20 | db02: 21 | image: ustcweizhou/mariadb-cluster:ubuntu18-10.4 22 | hostname: db02 23 | networks: 24 | overlay-db: 25 | volumes: 26 | - ${DATA_DIR}/$PROJECT/db02:/var/lib/mysql 27 | environment: 28 | - NODE_NAME=db02 29 | - CLUSTER_ADDRESS=gcomm://db01,db02,db03 30 | - DB_ROOT_PASSWORD=${DB_ROOT_PASSWORD} 31 | - DB_MARIABACKUP_PASSWORD=${DB_MARIABACKUP_PASSWORD} 32 | deploy: 33 | placement: 34 | constraints: 35 | - "node.labels.role==infra02" 36 | 37 | db03: 38 | image: ustcweizhou/mariadb-cluster:ubuntu18-10.4 39 | hostname: db03 40 | networks: 41 | overlay-db: 42 | volumes: 43 | - ${DATA_DIR}/$PROJECT/db03:/var/lib/mysql 44 | environment: 45 | - NODE_NAME=db03 46 | - CLUSTER_ADDRESS=gcomm://db01,db02,db03 47 | - DB_ROOT_PASSWORD=${DB_ROOT_PASSWORD} 48 | - DB_MARIABACKUP_PASSWORD=${DB_MARIABACKUP_PASSWORD} 49 | deploy: 50 | placement: 51 | constraints: 52 | - "node.labels.role==infra03" 53 | 54 | dbvip: 55 | image: nginx 56 | hostname: dbvip 57 | networks: 58 | overlay-db: 59 | ports: 60 | - "13306:3306" 61 | configs: 62 | - source: nginx-galera 63 | target: /etc/nginx/nginx.conf 64 | deploy: 65 | restart_policy: 66 | condition: on-failure 67 | placement: 68 | constraints: 69 | - "node.role==manager" 70 | 71 | configs: 72 | nginx-galera: 73 | file: ./nginx-galera.conf 74 | 75 | networks: 76 | overlay-db: 77 | driver: overlay 78 | attachable: true 79 | ipam: 80 | driver: default 81 | config: 82 | - subnet: 192.168.10.0/24 83 | -------------------------------------------------------------------------------- /swarm/nginx-galera.conf: -------------------------------------------------------------------------------- 1 | error_log stderr notice; 2 | worker_processes auto; 3 | events { 4 | multi_accept on; 5 | use epoll; 6 | worker_connections 1024; 7 | } 8 | stream { 9 | upstream mariadb { 10 | server db01:3306; 11 | server db02:3306 backup; 12 | server db03:3306 backup; 13 | } 14 | server { 15 | listen 0.0.0.0:3306; 16 | proxy_pass mariadb; 17 | proxy_timeout 10m; 18 | proxy_connect_timeout 1s; 19 | } 20 | } 21 | 22 | -------------------------------------------------------------------------------- /version: -------------------------------------------------------------------------------- 1 | 1.4 2 | --------------------------------------------------------------------------------