├── .gitignore ├── Dockerfile ├── README.md ├── docker-entrypoint.sh ├── example ├── galera-pv-host.yaml ├── galera.yaml └── secrets-development.yaml └── galera ├── galera-recovery.sh ├── galera.cnf └── on-start.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # Build 2 | .gradle/ 3 | .settings/ 4 | .project 5 | .DS_Store 6 | 7 | # Derrived 8 | /data/ 9 | /build/ 10 | /logs/ 11 | /work/ 12 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mariadb:10.3 2 | 3 | RUN set -x && \ 4 | apt-get update && apt-get install -y --no-install-recommends ca-certificates wget && \ 5 | rm -rf /var/lib/apt/lists/* && \ 6 | \ 7 | wget -O /usr/local/bin/peer-finder https://storage.googleapis.com/kubernetes-release/pets/peer-finder && \ 8 | chmod +x /usr/local/bin/peer-finder && \ 9 | \ 10 | apt-get purge -y --auto-remove ca-certificates wget 11 | 12 | ADD ["galera/", "/opt/galera/"] 13 | 14 | RUN set -x && \ 15 | cd /opt/galera && chmod +x on-start.sh galera-recovery.sh 16 | 17 | ADD ["docker-entrypoint.sh", "/usr/local/bin/"] 18 | ENTRYPOINT ["docker-entrypoint.sh"] 19 | CMD ["mysqld"] 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MariaDB Galera on Kubernetes 2 | 3 | Example of Docker image of MariaDB Galera cluster to be used in Kubernetes StatefulSet 4 | definition. 5 | 6 | Based on official [MariaDB image][mariadb-image]. 7 | Uses [peer-finder.go][peer-finder] util from Kibernetes contrib. 8 | Depending on service peers updates `wsrep_*` settings in a Galera config file. 9 | 10 | ## Settings 11 | 12 | See: [MariaDB image][mariadb-image] documentation 13 | 14 | Additional variables: 15 | 16 | * `POD_NAMESPACE` - The namespace, e.g. `default` 17 | * `GALERA_CONF` - The location of galera config file, e.g. `/etc/mysql/conf.d/galera.cnf` 18 | * `GALERA_SERVICE` - The service name to lookup, e.g. `galera` 19 | 20 | [peer-finder]: https://github.com/kubernetes/contrib/tree/master/peer-finder 21 | [mariadb-image]: https://hub.docker.com/_/mariadb/ 22 | -------------------------------------------------------------------------------- /docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eo pipefail 3 | shopt -s nullglob 4 | 5 | if [ "$TRACE" = "1" ]; then 6 | set -x 7 | fi 8 | 9 | # if command starts with an option, prepend mysqld 10 | if [ "${1:0:1}" = '-' ]; then 11 | set -- mysqld "$@" 12 | fi 13 | 14 | # skip setup if they want an option that stops mysqld 15 | wantHelp= 16 | for arg; do 17 | case "$arg" in 18 | -'?'|--help|--print-defaults|-V|--version) 19 | wantHelp=1 20 | break 21 | ;; 22 | esac 23 | done 24 | 25 | # usage: file_env VAR [DEFAULT] 26 | # ie: file_env 'XYZ_DB_PASSWORD' 'example' 27 | # (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of 28 | # "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) 29 | file_env() { 30 | local var="$1" 31 | local fileVar="${var}_FILE" 32 | local def="${2:-}" 33 | if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then 34 | echo >&2 "error: both $var and $fileVar are set (but are exclusive)" 35 | exit 1 36 | fi 37 | local val="$def" 38 | if [ "${!var:-}" ]; then 39 | val="${!var}" 40 | elif [ "${!fileVar:-}" ]; then 41 | val="$(< "${!fileVar}")" 42 | fi 43 | export "$var"="$val" 44 | unset "$fileVar" 45 | } 46 | 47 | _check_config() { 48 | toRun=( "$@" --verbose --help --log-bin-index="$(mktemp -u)" ) 49 | if ! errors="$("${toRun[@]}" 2>&1 >/dev/null)"; then 50 | cat >&2 <<-EOM 51 | ERROR: mysqld failed while attempting to check config 52 | command was: "${toRun[*]}" 53 | $errors 54 | EOM 55 | exit 1 56 | fi 57 | } 58 | 59 | _datadir() { 60 | "$@" --verbose --help --log-bin-index="$(mktemp -u)" 2>/dev/null | awk '$1 == "datadir" { print $2; exit }' 61 | } 62 | 63 | # allow the container to be started with `--user` 64 | if [ "$1" = 'mysqld' -a -z "$wantHelp" -a "$(id -u)" = '0' ]; then 65 | _check_config "$@" 66 | export DATADIR="$(_datadir "$@")" 67 | mkdir -p "$DATADIR" 68 | 69 | # Run Galera auto-discovery on Kubernetes 70 | if hash peer-finder 2>/dev/null; then 71 | peer-finder -on-start=/opt/galera/on-start.sh -service="${GALERA_SERVICE:-galera}" 72 | fi 73 | 74 | chown -R mysql:mysql "$DATADIR" 75 | exec gosu mysql "$BASH_SOURCE" "$@" 76 | fi 77 | 78 | if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then 79 | # still need to check config, container may have started with --user 80 | _check_config "$@" 81 | 82 | # Run Galera auto-recovery 83 | if [ -f /var/lib/mysql/ibdata1 ]; then 84 | echo "Galera - Determining recovery position..." 85 | set +e 86 | start_pos_opt=$(/opt/galera/galera-recovery.sh "${@:2}") 87 | set -e 88 | if [ $? -eq 0 ]; then 89 | echo "Galera recovery position: $start_pos_opt" 90 | set -- "$@" $start_pos_opt 91 | else 92 | echo "FATAL - Galera recovery failed!" 93 | exit 1 94 | fi 95 | fi 96 | 97 | # Get config 98 | DATADIR="$(_datadir "$@")" 99 | 100 | if [ ! -d "$DATADIR/mysql" ]; then 101 | file_env 'MYSQL_ROOT_PASSWORD' 102 | if [ -z "$MYSQL_ROOT_PASSWORD" -a -z "$MYSQL_ALLOW_EMPTY_PASSWORD" -a -z "$MYSQL_RANDOM_ROOT_PASSWORD" ]; then 103 | echo >&2 'error: database is uninitialized and password option is not specified ' 104 | echo >&2 ' You need to specify one of MYSQL_ROOT_PASSWORD, MYSQL_ALLOW_EMPTY_PASSWORD and MYSQL_RANDOM_ROOT_PASSWORD' 105 | exit 1 106 | fi 107 | 108 | mkdir -p "$DATADIR" 109 | 110 | echo 'Initializing database' 111 | mysql_install_db --datadir="$DATADIR" --rpm 112 | echo 'Database initialized' 113 | 114 | "$@" --skip-networking --socket=/var/run/mysqld/mysqld.sock & 115 | pid="$!" 116 | 117 | mysql=( mysql --protocol=socket -uroot -hlocalhost --socket=/var/run/mysqld/mysqld.sock ) 118 | 119 | for i in {30..0}; do 120 | if echo 'SELECT 1' | "${mysql[@]}" &> /dev/null; then 121 | break 122 | fi 123 | echo 'MySQL init process in progress...' 124 | sleep 1 125 | done 126 | if [ "$i" = 0 ]; then 127 | echo >&2 'MySQL init process failed.' 128 | exit 1 129 | fi 130 | 131 | if [ -z "$MYSQL_INITDB_SKIP_TZINFO" ]; then 132 | # sed is for https://bugs.mysql.com/bug.php?id=20545 133 | mysql_tzinfo_to_sql /usr/share/zoneinfo | sed 's/Local time zone must be set--see zic manual page/FCTY/' | "${mysql[@]}" mysql 134 | fi 135 | 136 | if [ ! -z "$MYSQL_RANDOM_ROOT_PASSWORD" ]; then 137 | export MYSQL_ROOT_PASSWORD="$(pwgen -1 32)" 138 | echo "GENERATED ROOT PASSWORD: $MYSQL_ROOT_PASSWORD" 139 | fi 140 | "${mysql[@]}" <<-EOSQL 141 | -- What's done in this file shouldn't be replicated 142 | -- or products like mysql-fabric won't work 143 | SET @@SESSION.SQL_LOG_BIN=0; 144 | 145 | DELETE FROM mysql.user ; 146 | CREATE USER 'root'@'%' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' ; 147 | GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION ; 148 | DROP DATABASE IF EXISTS test ; 149 | FLUSH PRIVILEGES ; 150 | EOSQL 151 | 152 | if [ ! -z "$MYSQL_ROOT_PASSWORD" ]; then 153 | mysql+=( -p"${MYSQL_ROOT_PASSWORD}" ) 154 | fi 155 | 156 | file_env 'MYSQL_DATABASE' 157 | if [ "$MYSQL_DATABASE" ]; then 158 | echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" | "${mysql[@]}" 159 | mysql+=( "$MYSQL_DATABASE" ) 160 | fi 161 | 162 | file_env 'MYSQL_USER' 163 | file_env 'MYSQL_PASSWORD' 164 | if [ "$MYSQL_USER" -a "$MYSQL_PASSWORD" ]; then 165 | echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" | "${mysql[@]}" 166 | 167 | if [ "$MYSQL_DATABASE" ]; then 168 | echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" | "${mysql[@]}" 169 | fi 170 | 171 | echo 'FLUSH PRIVILEGES ;' | "${mysql[@]}" 172 | fi 173 | 174 | echo 175 | for f in /docker-entrypoint-initdb.d/*; do 176 | case "$f" in 177 | *.sh) echo "$0: running $f"; . "$f" ;; 178 | *.sql) echo "$0: running $f"; "${mysql[@]}" < "$f"; echo ;; 179 | *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${mysql[@]}"; echo ;; 180 | *) echo "$0: ignoring $f" ;; 181 | esac 182 | echo 183 | done 184 | 185 | if ! kill -s TERM "$pid" || ! wait "$pid"; then 186 | echo >&2 'MySQL init process failed.' 187 | exit 1 188 | fi 189 | 190 | echo 191 | echo 'MySQL init process done. Ready for start up.' 192 | echo 193 | fi 194 | fi 195 | 196 | exec "$@" -------------------------------------------------------------------------------- /example/galera-pv-host.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolume 3 | metadata: 4 | name: datadir-mysql-0 5 | spec: 6 | accessModes: 7 | - ReadWriteOnce 8 | capacity: 9 | storage: 2Gi 10 | hostPath: 11 | path: /data/datadir-mysql-0/ 12 | --- 13 | apiVersion: v1 14 | kind: PersistentVolume 15 | metadata: 16 | name: datadir-mysql-1 17 | spec: 18 | accessModes: 19 | - ReadWriteOnce 20 | capacity: 21 | storage: 2Gi 22 | hostPath: 23 | path: /data/datadir-mysql-1/ 24 | --- 25 | apiVersion: v1 26 | kind: PersistentVolume 27 | metadata: 28 | name: datadir-mysql-2 29 | spec: 30 | accessModes: 31 | - ReadWriteOnce 32 | capacity: 33 | storage: 2Gi 34 | hostPath: 35 | path: /data/datadir-mysql-2/ -------------------------------------------------------------------------------- /example/galera.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # MariaDB 10.1 Galera Cluster 3 | # 4 | apiVersion: v1 5 | kind: Service 6 | metadata: 7 | annotations: 8 | service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" 9 | name: galera 10 | labels: 11 | app: mysql 12 | spec: 13 | ports: 14 | - port: 3306 15 | name: mysql 16 | clusterIP: None 17 | selector: 18 | app: mysql 19 | --- 20 | apiVersion: v1 21 | kind: ConfigMap 22 | metadata: 23 | name: mysql-config-vol 24 | data: 25 | galera.cnf: | 26 | [galera] 27 | user = mysql 28 | bind-address = 0.0.0.0 29 | 30 | default_storage_engine = InnoDB 31 | binlog_format = ROW 32 | innodb_autoinc_lock_mode = 2 33 | innodb_flush_log_at_trx_commit = 0 34 | query_cache_size = 0 35 | query_cache_type = 0 36 | 37 | # MariaDB Galera settings 38 | wsrep_on=ON 39 | wsrep_provider=/usr/lib/galera/libgalera_smm.so 40 | wsrep_sst_method=rsync 41 | 42 | # Cluster settings (automatically updated) 43 | wsrep_cluster_address=gcomm:// 44 | wsrep_cluster_name=galera 45 | wsrep_node_address=127.0.0.1 46 | mariadb.cnf: | 47 | [client] 48 | default-character-set = utf8 49 | [mysqld] 50 | character-set-server = utf8 51 | collation-server = utf8_general_ci 52 | # InnoDB tuning 53 | innodb_log_file_size = 50M 54 | --- 55 | apiVersion: apps/v1beta1 56 | kind: StatefulSet 57 | metadata: 58 | name: mysql 59 | spec: 60 | serviceName: "galera" 61 | replicas: 3 62 | template: 63 | metadata: 64 | labels: 65 | app: mysql 66 | spec: 67 | initContainers: 68 | - name: copy-mariadb-config 69 | image: busybox 70 | command: ['sh', '-c', 'cp /configmap/* /etc/mysql/conf.d'] 71 | volumeMounts: 72 | - name: configmap 73 | mountPath: /configmap 74 | - name: config 75 | mountPath: /etc/mysql/conf.d 76 | containers: 77 | - name: mysql 78 | image: ausov/k8s-mariadb-cluster 79 | ports: 80 | - containerPort: 3306 81 | name: mysql 82 | - containerPort: 4444 83 | name: sst 84 | - containerPort: 4567 85 | name: replication 86 | - containerPort: 4568 87 | name: ist 88 | env: 89 | - name: POD_NAMESPACE 90 | valueFrom: 91 | fieldRef: 92 | apiVersion: v1 93 | fieldPath: metadata.namespace 94 | - name: MYSQL_ROOT_PASSWORD 95 | valueFrom: 96 | secretKeyRef: 97 | name: mysql 98 | key: password 99 | readinessProbe: 100 | exec: 101 | command: ["bash", "-c", "mysql -uroot -p\"${MYSQL_ROOT_PASSWORD}\" -e 'show databases;'"] 102 | initialDelaySeconds: 20 103 | timeoutSeconds: 5 104 | volumeMounts: 105 | - name: config 106 | mountPath: /etc/mysql/conf.d 107 | - name: datadir 108 | mountPath: /var/lib/mysql 109 | volumes: 110 | - name: config 111 | emptyDir: {} 112 | - name: configmap 113 | configMap: 114 | name: mysql-config-vol 115 | items: 116 | - path: "galera.cnf" 117 | key: galera.cnf 118 | - path: "mariadb.cnf" 119 | key: mariadb.cnf 120 | volumeClaimTemplates: 121 | - metadata: 122 | name: datadir 123 | spec: 124 | accessModes: [ "ReadWriteOnce" ] 125 | resources: 126 | requests: 127 | storage: 2Gi 128 | -------------------------------------------------------------------------------- /example/secrets-development.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: mysql 5 | type: Opaque 6 | data: 7 | # Root password (base64): changeit 8 | password: Y2hhbmdlaXQ= 9 | -------------------------------------------------------------------------------- /galera/galera-recovery.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | cmdline_args=$@ 4 | user=mysql 5 | euid=$(id -u) 6 | log_file=$(mktemp /tmp/wsrep_recovery.XXXXXX) 7 | start_pos='' 8 | start_pos_opt='' 9 | 10 | log() { 11 | local msg="galera-recovery.sh: $@" 12 | # Print all messages to stderr as we reserve stdout for printing 13 | # --wsrep-start-position=XXXX. 14 | echo "$msg" >&2 15 | } 16 | 17 | finish() { 18 | rm -f "$log_file" 19 | } 20 | 21 | trap finish EXIT 22 | 23 | wsrep_recover_position() { 24 | eval mysqld --user=$user \ 25 | --wsrep-recover \ 26 | --log-error="$log_file" 27 | if [ $? -ne 0 ]; then 28 | # Something went wrong, let us also print the error log so that it 29 | # shows up in systemctl status output as a hint to the user. 30 | log "Failed to start mysqld for wsrep recovery: '`cat $log_file`'" 31 | exit 1 32 | fi 33 | 34 | start_pos=$(sed -n 's/.*WSREP: Recovered position:\s*//p' $log_file) 35 | 36 | if [ -z $start_pos ]; then 37 | skipped="$(grep WSREP $log_file | grep 'skipping position recovery')" 38 | if [ -z "$skipped" ]; then 39 | log "==================================================" 40 | log "WSREP: Failed to recover position: '`cat $log_file`'" 41 | log "==================================================" 42 | exit 1 43 | else 44 | log "WSREP: Position recovery skipped." 45 | fi 46 | 47 | else 48 | log "Found WSREP position: $start_pos" 49 | 50 | # TODO Find a better solution to automatically restore after a full cluster crash 51 | # Force start even if some latest TX are lost before a crash 52 | # otherwise container just cannot start in K8s StatefulSet configuration 53 | sed -i 's/safe_to_bootstrap: 0/safe_to_bootstrap: 1/g' /var/lib/mysql/grastate.dat 54 | 55 | start_pos_opt="--wsrep_start_position=$start_pos" 56 | fi 57 | } 58 | 59 | if [ -n "$log_file" -a -f "$log_file" ]; then 60 | [ "$euid" = "0" ] && chown $user $log_file 61 | chmod 600 $log_file 62 | else 63 | log "WSREP: mktemp failed" 64 | fi 65 | 66 | if [ -f /var/lib/mysql/ibdata1 ]; then 67 | log "Attempting to recover GTID positon..." 68 | wsrep_recover_position 69 | else 70 | log "No ibdata1 found, starting a fresh node..." 71 | fi 72 | 73 | echo "$start_pos_opt" 74 | -------------------------------------------------------------------------------- /galera/galera.cnf: -------------------------------------------------------------------------------- 1 | [galera] 2 | user = mysql 3 | bind-address = 0.0.0.0 4 | 5 | default_storage_engine = InnoDB 6 | binlog_format = ROW 7 | innodb_autoinc_lock_mode = 2 8 | innodb_flush_log_at_trx_commit = 0 9 | query_cache_size = 0 10 | query_cache_type = 0 11 | 12 | # MariaDB Galera settings 13 | wsrep_on=ON 14 | wsrep_provider=/usr/lib/galera/libgalera_smm.so 15 | wsrep_sst_method=rsync 16 | 17 | # Cluster settings (automatically updated) 18 | wsrep_cluster_address=gcomm:// 19 | wsrep_cluster_name=galera 20 | wsrep_node_address=127.0.0.1 21 | -------------------------------------------------------------------------------- /galera/on-start.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | GALERA_CONF="${GALERA_CONF:-"/etc/mysql/conf.d/galera.cnf"}" 4 | 5 | if ! [ -f "${GALERA_CONF}" ]; then 6 | cp /opt/galera/galera.cnf "${GALERA_CONF}" 7 | fi 8 | 9 | function join { 10 | local IFS="$1"; shift; echo "$*"; 11 | } 12 | 13 | HOSTNAME=$(hostname) 14 | # Parse out cluster name, formatted as: petset_name-index 15 | IFS='-' read -ra ADDR <<< "$(hostname)" 16 | CLUSTER_NAME="${ADDR[0]}" 17 | 18 | while read -ra LINE; do 19 | if [[ "${LINE}" == *"${HOSTNAME}"* ]]; then 20 | MY_NAME=$LINE 21 | else 22 | PEERS=("${PEERS[@]}" $LINE) 23 | fi 24 | done 25 | 26 | if [ "${#PEERS[@]}" = 0 ]; then 27 | export WSREP_CLUSTER_ADDRESS="" 28 | else 29 | export WSREP_CLUSTER_ADDRESS=$(join , "${PEERS[@]}") 30 | fi 31 | sed -i -e "s|^wsrep_node_address[[:space:]]*=.*$|wsrep_node_address=${MY_NAME}|" "${GALERA_CONF}" 32 | sed -i -e "s|^wsrep_cluster_name[[:space:]]*=.*$|wsrep_cluster_name=${CLUSTER_NAME}|" "${GALERA_CONF}" 33 | sed -i -e "s|^wsrep_cluster_address[[:space:]]*=.*$|wsrep_cluster_address=gcomm://${WSREP_CLUSTER_ADDRESS}|" "${GALERA_CONF}" 34 | 35 | # don't need a restart, we're just writing the conf in case there's an 36 | # unexpected restart on the node. 37 | 38 | if [ -n "$WSREP_CLUSTER_ADDRESS" ]; then 39 | mkdir -p "$DATADIR/mysql" 40 | echo "*** [Galera] Joining cluster: $WSREP_CLUSTER_ADDRESS" 41 | else 42 | echo "*** [Galera] Starting new cluster!" 43 | fi 44 | --------------------------------------------------------------------------------