├── .gitignore ├── README.md ├── backup-restore ├── Dockerfile └── entrypoint.sh ├── backup-sync ├── Dockerfile └── entrypoint.sh ├── backup ├── Dockerfile └── entrypoint.sh ├── failover-restore ├── Dockerfile └── entrypoint.sh ├── init ├── Dockerfile └── entrypoint.sh └── view-recreation ├── Dockerfile └── entrypoint.sh /.gitignore: -------------------------------------------------------------------------------- 1 | .idea -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # clickhouse-tasks -------------------------------------------------------------------------------- /backup-restore/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM amazonlinux:latest 2 | 3 | RUN yum install aws-cli -y 4 | RUN yum install perl-libwww-perl -y 5 | ADD ./entrypoint.sh ./entrypoint.sh 6 | RUN chmod +x ./entrypoint.sh 7 | 8 | ENTRYPOINT ["./entrypoint.sh"] 9 | 10 | -------------------------------------------------------------------------------- /backup-restore/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | function clear_dir { 4 | read -r -a array <<< "$TABLES" 5 | for TABLE in "${array[@]}"; do 6 | rm -rf "${CLICKHOUSE_DATA_PATH}/${DATABASE}/${TABLE}/detached/"* 7 | done 8 | } 9 | 10 | function drop_partition { 11 | local TABLE=$1 12 | local PARTITION=$2 13 | local ADDRESS=$3 14 | 15 | echo "SQL: ALTER TABLE ${TABLE} DROP PARTITION '${PARTITION}'" 16 | echo "ALTER TABLE ${TABLE} DROP PARTITION '${PARTITION}'" | POST "http://${USERNAME}:${PASSWORD}@${ADDRESS}:8123/" 17 | } 18 | 19 | function truncate_table { 20 | local TABLE=$1 21 | 22 | echo "Truncating table ${TABLE}" 23 | 24 | for i in `seq 1 ${NUMBER_INSTANCES}`; do 25 | local ADDRESS="${INSTANCE_NAME}${i}.${ZONE_NAME}" 26 | local IS_LEADER=$(echo "select is_leader from system.replicas where table = '$TABLE'" | POST "http://${USERNAME}:${PASSWORD}@${ADDRESS}:8123/") 27 | if [[ ${IS_LEADER} = "1" ]]; then 28 | for CUR_YEAR in $(seq ${START_YEAR} ${YEAR}); do 29 | for CUR_MONTH in 01 02 03 04 05 06 07 08 09 10 11 12; do 30 | drop_partition ${TABLE} "${CUR_YEAR}${CUR_MONTH}" ${ADDRESS} 31 | done 32 | done 33 | fi 34 | done 35 | 36 | echo "Table truncated" 37 | } 38 | 39 | function load_file { 40 | local FROM=$1 41 | local TO=$2 42 | echo "Loading file from s3://${FROM} to ${TO}" 43 | aws s3 sync "s3://${FROM}" "${TO}" --quiet 44 | } 45 | 46 | function attach_partition { 47 | local TABLE=$1 48 | local PARTITION=$2 49 | local KEY=$3 50 | 51 | load_file "${S3_BACKUPS_BUCKET}/${PARTITION}/${KEY}/data/${DATABASE}/${TABLE}" "${CLICKHOUSE_DATA_PATH}/${DATABASE}/${TABLE}/detached" 52 | 53 | chown -R 105:106 "${CLICKHOUSE_DATA_PATH}/${DATABASE}/${TABLE}/detached" 54 | echo "SQL: ALTER TABLE ${TABLE} ATTACH PARTITION '${PARTITION}'" 55 | echo "ALTER TABLE ${TABLE} ATTACH PARTITION '${PARTITION}'" | POST "http://${USERNAME}:${PASSWORD}@localhost:8123/" 56 | } 57 | 58 | function main { 59 | if [[ ${EXECUTE_SCRIPT} == 0 ]]; then 60 | exit 61 | fi 62 | 63 | clear_dir 64 | 65 | read -r -a array <<< "$TABLES" 66 | for TABLE in "${array[@]}"; do 67 | truncate_table ${TABLE} 68 | done 69 | 70 | for CUR_YEAR in $(seq ${START_YEAR} ${YEAR}); do 71 | for CUR_MONTH in 01 02 03 04 05 06 07 08 09 10 11 12; do 72 | if [[ ${CUR_YEAR} = ${YEAR} && ${CUR_MONTH} = ${MONTH} ]]; then 73 | read -r -a array <<< "$TABLES" 74 | for TABLE in "${array[@]}"; do 75 | attach_partition ${TABLE} $(date -d "-1 day" +"%Y%m") $(date -d "-1 day" +"%d") 76 | done 77 | break 78 | fi 79 | 80 | read -r -a array <<< "$TABLES" 81 | for TABLE in "${array[@]}"; do 82 | attach_partition ${TABLE} "${CUR_YEAR}${CUR_MONTH}" "full" 83 | done 84 | done 85 | done 86 | } 87 | 88 | YEAR=$(date +"%Y") 89 | MONTH=$(date +"%m") 90 | DAY=$(date +"%d") 91 | 92 | main 93 | -------------------------------------------------------------------------------- /backup-sync/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM amazonlinux:latest 2 | 3 | RUN yum install aws-cli -y 4 | ADD ./entrypoint.sh ./entrypoint.sh 5 | RUN chmod +x ./entrypoint.sh 6 | 7 | ENTRYPOINT ["./entrypoint.sh"] 8 | 9 | -------------------------------------------------------------------------------- /backup-sync/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | if [[ ${RESYNC} == 0 ]]; then 3 | aws s3 sync "s3://${BACKUP_PROD}" "s3://${BACKUP_STAGING}" 4 | else 5 | aws s3 sync "s3://${BACKUP_PROD}" "s3://${BACKUP_STAGING}" --delete 6 | fi -------------------------------------------------------------------------------- /backup/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM amazonlinux:latest 2 | 3 | RUN yum install aws-cli -y 4 | RUN yum install perl-libwww-perl -y 5 | ADD ./entrypoint.sh ./entrypoint.sh 6 | RUN chmod +x ./entrypoint.sh 7 | 8 | ENTRYPOINT ["./entrypoint.sh"] 9 | 10 | -------------------------------------------------------------------------------- /backup/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | function upload_dir { 4 | local FROM=$1 5 | local TO=$2 6 | echo "Uploading dir from $FROM to $TO" 7 | aws s3 sync "${FROM}" "s3://${TO}" --quiet 8 | } 9 | 10 | function clear_dir { 11 | local KEY=$1 12 | echo "Clearing dir s3://${KEY}" 13 | aws s3 rm "s3://${KEY}" --recursive 14 | } 15 | 16 | function backup_partition { 17 | local PARTITION=$1 18 | local KEY=$2 19 | 20 | clear_dir "${S3_BACKUPS_BUCKET}/${PARTITION}/${KEY}" 21 | 22 | read -r -a array <<< "${TABLES}" 23 | for TABLE in "${array[@]}"; do 24 | echo "Backuping table ${TABLE}" 25 | echo "SQL: ALTER TABLE ${TABLE} FREEZE PARTITION '${PARTITION}'" 26 | echo "ALTER TABLE ${TABLE} FREEZE PARTITION '${PARTITION}'" | POST "http://${USERNAME}:${PASSWORD}@localhost:8123/" 27 | 28 | local CURRENT_NUMBER=$(cat ${BACKUP_PATH}/increment.txt) 29 | upload_dir "${BACKUP_PATH}/${CURRENT_NUMBER}" "${S3_BACKUPS_BUCKET}/${PARTITION}/${KEY}" 30 | done 31 | } 32 | 33 | function clear_backup_dir { 34 | echo "REMOVING ALL IN ${BACKUP_PATH}/*" 35 | rm -rf "${BACKUP_PATH}/*" 36 | } 37 | 38 | function main { 39 | local DAY=$(date +"%d") 40 | local YEAR=$(date +"%Y") 41 | local MONTH=$(date +"%m") 42 | echo "Current date: ${DAY}/${MONTH}/${YEAR}" 43 | 44 | if [[ ${FULL_BACKUP} == 0 ]]; then 45 | if [[ ${DAY} = "${DAY_TO_UPDATE_LAST_MONTH}" ]]; then 46 | backup_partition $(date -d "-1 month" +"%Y%m") "full" 47 | fi 48 | backup_partition $(date +"%Y%m") ${DAY} 49 | else 50 | for CUR_YEAR in $(seq ${START_YEAR} ${YEAR}); do 51 | for CUR_MONTH in 01 02 03 04 05 06 07 08 09 10 11 12; do 52 | if [[ ${CUR_YEAR} = ${YEAR} && ${CUR_MONTH} = ${MONTH} ]]; then 53 | backup_partition "${CUR_YEAR}${CUR_MONTH}" ${DAY} 54 | break 55 | fi 56 | backup_partition "${CUR_YEAR}${CUR_MONTH}" "full" 57 | done 58 | done 59 | fi 60 | 61 | clear_backup_dir 62 | } 63 | 64 | main 65 | 66 | -------------------------------------------------------------------------------- /failover-restore/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM amazonlinux:latest 2 | 3 | RUN yum install aws-cli -y 4 | RUN yum install curl -y 5 | ADD ./entrypoint.sh ./entrypoint.sh 6 | RUN chmod +x ./entrypoint.sh 7 | 8 | ENTRYPOINT ["./entrypoint.sh"] 9 | -------------------------------------------------------------------------------- /failover-restore/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo "Processing files from failover bucket" 3 | touch logs 4 | for file in $(aws s3 ls ${FAILOVER_BUCKET} | awk '{print $4}'); do 5 | aws s3 cp "s3://${FAILOVER_BUCKET}/${file}" "${file}" 6 | while IFS='' read -r SQL_STATEMENT || [[ -n "$SQL_STATEMENT" ]]; do 7 | response=$(curl --data "${SQL_STATEMENT}" --write-out %{http_code} --silent --output logs http://${USERNAME}:${PASSWORD}@localhost:8123) 8 | echo "RESULT:" 9 | cat logs 10 | if [[ ${response} != "200" ]]; then 11 | echo "ERROR: ${SQL_STATEMENT}" 12 | fi 13 | done < "${file}" 14 | aws s3 rm "s3://${FAILOVER_BUCKET}/${file}" 15 | done 16 | echo "Files processed" 17 | -------------------------------------------------------------------------------- /init/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM amazonlinux:latest 2 | 3 | RUN yum install aws-cli -y 4 | RUN yum install perl-libwww-perl -y 5 | RUN yum install wget -y 6 | RUN yum install java-1.8.0-openjdk -y 7 | RUN wget "https://archive.apache.org/dist/zookeeper/zookeeper-3.4.9/zookeeper-3.4.9.tar.gz" 8 | RUN yum install -y tar.x86_64 gzip gunzip 9 | RUN tar -xzf "zookeeper-3.4.9.tar.gz" 10 | RUN rm "zookeeper-3.4.9.tar.gz" 11 | ADD ./entrypoint.sh ./entrypoint.sh 12 | RUN chmod +x ./entrypoint.sh 13 | 14 | ENTRYPOINT ["./entrypoint.sh"] 15 | -------------------------------------------------------------------------------- /init/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | function load_file { 4 | local FROM=$1 5 | local TO=$2 6 | echo "Loading file from s3://${FROM} to ${TO}" 7 | aws s3 cp "s3://${FROM}" ${TO} 8 | } 9 | 10 | function update_conf { 11 | echo "Updating configs" 12 | load_file "${S3_CONFIGS_BUCKET}/${SERVER_CONFIG}" "${CONFIG_PATH}/config.xml" 13 | load_file "${S3_CONFIGS_BUCKET}/${USERS_CONFIG}" "${CONFIG_PATH}/users.xml" 14 | echo "Configs updated" 15 | } 16 | 17 | function delete_node_zk { 18 | local TABLE=$1 19 | local ID=$2 20 | echo "Deleting ${TABLE} from Zookeeper." 21 | for i in `seq 1 ${NUMBER_INSTANCES}`; 22 | do 23 | echo "Executing delete all on instance${i}" 24 | ./zookeeper-3.4.9/bin/zkCli.sh -server "${INSTANCE_NAME}${i}.${ZONE_NAME}" rmr "/clickhouse/tables/${TABLE}/replicas/${ID}" 25 | done 26 | echo "Deleted ${TABLE} from Zookeeper." 27 | } 28 | 29 | function get_scheme_from_cluster { 30 | local TABLE=$1 31 | local ID=$2 32 | 33 | echo "Getting ${TABLE} scheme from cluster." 34 | for i in `seq 1 ${NUMBER_INSTANCES}`; 35 | do 36 | echo "Getting scheme from node" 37 | STATEMENT=$(curl --data "SHOW CREATE TABLE ${TABLE}" "http://${USERNAME}:${PASSWORD}@${INSTANCE_NAME}${i}.${ZONE_NAME}:8123/") 38 | if [[ ${STATEMENT} != *"Exception"* ]]; then 39 | STATEMENT=$(echo ${STATEMENT} | sed s/"\\\'"/"'"/g) 40 | STATEMENT=$(echo ${STATEMENT} | sed s/"'${i}'"/"'${ID}'"/g) 41 | break 42 | else 43 | STATEMENT="" 44 | fi 45 | done 46 | echo "Got statement: ${STATEMENT}" 47 | } 48 | 49 | function main { 50 | local ID=$(cat ${ID_PATH}) 51 | update_conf 52 | 53 | echo "Node ID is ${ID}" 54 | 55 | echo "Starting registering" 56 | until $(curl --output /dev/null --silent --head --fail http://localhost:8123); do 57 | echo 'Waiting Clickhouse to deploy' 58 | sleep 5 59 | done 60 | echo "Initializing Clickhouse" 61 | 62 | read -r -a array <<< "$TABLES" 63 | for TABLE in "${array[@]}" 64 | do 65 | IS_EXISTS=$(curl --data "EXISTS ${TABLE}" "http://${USERNAME}:${PASSWORD}@localhost:8123/") 66 | echo "Is exists ${TABLE} = ${IS_EXISTS}" 67 | if [[ ${IS_EXISTS} = "0" ]]; then 68 | delete_node_zk ${TABLE} ${ID} 69 | STATEMENT="" 70 | get_scheme_from_cluster ${TABLE} ${ID} 71 | if [[ ${STATEMENT} != "" ]]; then 72 | echo "Executing statement got from cluster" 73 | curl --data "${STATEMENT}" "http://${USERNAME}:${PASSWORD}@localhost:8123/" 74 | echo "Statement got from cluster executed" 75 | else 76 | echo "Loading init sql for ${TABLE}" 77 | load_file "${S3_CONFIGS_BUCKET}/${TABLE}.sql" "${TABLE}.sql" 78 | STATEMENT=$(cat "${TABLE}.sql") 79 | STATEMENT=$(echo ${STATEMENT} | sed s/'${ID}'/"${ID}"/g) 80 | curl --data "${STATEMENT}" "http://${USERNAME}:${PASSWORD}@localhost:8123/" 81 | echo "Init sql executed" 82 | fi 83 | else 84 | echo "Table ${TABLE} already exists." 85 | fi 86 | done 87 | 88 | echo "Done initializing Clickhouse" 89 | } 90 | 91 | STATEMENT="" 92 | main -------------------------------------------------------------------------------- /view-recreation/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM amazonlinux:latest 2 | 3 | RUN yum install aws-cli -y 4 | RUN yum install perl-libwww-perl -y 5 | RUN yum install wget -y 6 | ADD ./entrypoint.sh ./entrypoint.sh 7 | RUN chmod +x ./entrypoint.sh 8 | 9 | ENTRYPOINT ["./entrypoint.sh"] 10 | 11 | -------------------------------------------------------------------------------- /view-recreation/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | function load_file { 4 | local FROM=$1 5 | local TO=$2 6 | 7 | echo "Loading file from s3://${FROM} to ${TO}" 8 | aws s3 cp "s3://${FROM}" ${TO} 9 | } 10 | 11 | function drop_table { 12 | local TABLE=$1 13 | local ADDRESS=$2 14 | 15 | echo "SQL: DROP TABLE ${TABLE}" 16 | curl --data "DROP TABLE ${TABLE}" "http://${USERNAME}:${PASSWORD}@${ADDRESS}:8123/" 17 | } 18 | 19 | function create_table { 20 | local TABLE=$1 21 | local ID=$2 22 | local ADDRESS=$3 23 | 24 | echo "Loading init sql for ${TABLE}" 25 | load_file "${S3_CONFIGS_BUCKET}/${TABLE}.sql" "${TABLE}.sql" 26 | STATEMENT=$(cat "${TABLE}.sql") 27 | STATEMENT=$(echo ${STATEMENT} | sed s/'${ID}'/"${ID}"/g) 28 | curl --data "${STATEMENT}" "http://${USERNAME}:${PASSWORD}@${ADDRESS}:8123/" 29 | echo "Init sql executed" 30 | } 31 | 32 | function main { 33 | for i in `seq 1 ${NUMBER_INSTANCES}`; do 34 | local ADDRESS="${INSTANCE_NAME}${i}.${ZONE_NAME}" 35 | read -r -a array <<< "$TABLES" 36 | for TABLE in "${array[@]}" 37 | do 38 | IS_EXISTS=$(curl --data "EXISTS ${TABLE}" "http://${USERNAME}:${PASSWORD}@${ADDRESS}:8123/") 39 | echo "Is exists ${TABLE} = ${IS_EXISTS}" 40 | if [[ ${IS_EXISTS} = "1" ]]; then 41 | drop_table ${TABLE} ${ADDRESS} 42 | fi 43 | create_table ${TABLE} ${i} ${ADDRESS} 44 | done 45 | done 46 | 47 | echo "Done recreating table in Clickhouse" 48 | } 49 | 50 | main 51 | --------------------------------------------------------------------------------