├── .gitattributes ├── .gitignore ├── README.md └── compress-data.sh /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Windows image file caches 2 | Thumbs.db 3 | ehthumbs.db 4 | 5 | # Folder config file 6 | Desktop.ini 7 | 8 | # Recycle Bin used on file shares 9 | $RECYCLE.BIN/ 10 | 11 | # Windows Installer files 12 | *.cab 13 | *.msi 14 | *.msm 15 | *.msp 16 | 17 | # Windows shortcuts 18 | *.lnk 19 | 20 | # ========================= 21 | # Operating System Files 22 | # ========================= 23 | 24 | # OSX 25 | # ========================= 26 | 27 | .DS_Store 28 | .AppleDouble 29 | .LSOverride 30 | 31 | # Thumbnails 32 | ._* 33 | 34 | # Files that might appear on external disk 35 | .Spotlight-V100 36 | .Trashes 37 | 38 | # Directories potentially created on remote AFP share 39 | .AppleDB 40 | .AppleDesktop 41 | Network Trash Folder 42 | Temporary Items 43 | .apdisk 44 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Description 2 | Bash script for compressing data, useful for backup archives. 3 | 4 | **Tested on RHEL (CentOS) 6!** 5 | 6 | ## Highlights 7 | 8 | 1. compresses the data into .tar.xz using 4 cores, level of compression set to 9 (can easy be changed) 9 | 2. stores previous archives in separate directory 10 | 3. flexible configuration 11 | 4. resources initial check (source, output, runtime directories read/write rights) 12 | 5. checks for lock file (prevents script execution if source is still created) 13 | 6. text formatting (bold, colored) 14 | 7. e-mail sending on success and or fail 15 | 8. outputs compression info (source size, archive size, compression ratio) 16 | 9. logging and debugging (outputs more info about process) 17 | 10. possibility to change std output (to console and log or log only) 18 | 19 | ## Roadmap 20 | 21 | 1. prompts (prevent accidental execution) 22 | 2. exclusions (list of file/directories) 23 | -------------------------------------------------------------------------------- /compress-data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################################################ 3 | ## Bash script for file or directory compression ## 4 | ################################################################################ 5 | ## 1. Configuration variables 6 | ## $srcName - Source, file or directory to compress. 7 | ## $srcDir - Directory where the source resides. 8 | ## $runDir - Runtime directory. Lock and log files reside here. 9 | ## $outDir - Directory where the archive will be placed. 10 | ## $archDir - Directory where the previous archives will be placed. 11 | ## $archName - Archive file name. 12 | ## [ example: "archive_$(date +"[%Y.%m.%d - %H:%M:%S]")" ] 13 | ## $logName - Log file name. 14 | ## $lockCheck - Check for lock file or not. 15 | ## If you have condition(s) in which creation of an archive is not 16 | ## recommended or fatal (e.g. while creating snapshot, directory must not 17 | ## be tempered with in any way) and the process which is running prior to 18 | ## archivation (rsnapshot, rsync, rdiff etc.), can create a lock file, 19 | ## you must switch this check ON and fill out a name for a lock file, 20 | ## to prevent integrity violation. 21 | ## [ default: true ] 22 | ## $lockName - Lock file name. 23 | ## $lockSleep - How much time to wait until next try (in milliseconds). 24 | ## [ default: 600 ] - 10 minutes. 25 | ## $lockWait - How many tries before script stop. 26 | ## [ default: 12 ] - Retry for 2 hours if $lockSleep = 600 27 | ## oldArchDaysBack - How much time (in days) into the past, search must look 28 | ## for old archive files inside the vault ($archDir) for deleting the oldest. 29 | ## [ default: 30 ] - One month. 30 | ## @WARNING! If you set $maxOldArch to say 10 and you run archivation process 31 | ## say every 24 hours, but your $oldArchDaysBack is set to 7, script will 32 | ## not be able to account for 3 oldest archive files, so your vault can 33 | ## grow more than 10! 34 | ## maxOldArch - How much old archive files must be kept inside the vault. 35 | ## [ default: 30 ] 36 | ## @WARNING! Do not set higher than $oldArchDaysBack if you run this script 37 | ## daily! 38 | ## $mailSendSucc - Enable mail sending on success. 39 | ## $mailSendFail - Enable mail sending on fail. 40 | ## $mailFileName - Mail file name. 41 | ## $mailTo - To which e-mail messages must be sent. 42 | ## $mailSubject - Subject of the message. 43 | ## $stdType - Sets output to console and or log file. 44 | ## [ default: "cl" ] { "cl" = console & log, "l" = log only } 45 | ## $txtFormat - Allow text formating (bold and colored text). 46 | ## [ default: true ] 47 | ## $debug - Enable debugging or not. 48 | ## [ default: true ] 49 | ## $timeStamp - Timestamp format. 50 | ## [ default: "[%Y.%m.%d - %H:%M:%S]" ] 51 | ## $compressor - Compressor and it's params. 52 | ## [ default: "xz -T 4 -9 -c" ] 53 | ## $archExt - Archive extension, depending on $compressor setting. 54 | ## [ default: ".tar.xz" ] 55 | ## 56 | ## 2. Helper functions 57 | ## msg() - Prints messages. 58 | ## [ example: msg "Archive file name is: %s\n" "$arcname" ] 59 | ## tf() - Formats text. 60 | ## tStamp() - Prints timestamp. 61 | ## [ example: "$(tStamp "[%Y.%m.%d - %H:%M:%S]")" ] 62 | ## [ fallback: "${timeStamp}" ] 63 | ## secToTime() - Turns seconds into readable time (e.g. 3h 45m 21s). 64 | ## sendMail() - Sends e-mails. 65 | ## ifDebug() - Checks if debuging is enabled. 66 | ## ifLock() - Checks if lock file check is enabled. 67 | ## succ() - Process success output. 68 | ## fail() - Process fail output. 69 | ## 3. Functions 70 | ## checkConf() - Validates configuration. 71 | ## checkLock() - Checks for lock file. 72 | ## checkForOldArch() - Checks old archive files. 73 | ## execComp() - Executes compression and assigns execution time to $execTime. 74 | ## size() - Gets object size values. 75 | ## @Takes two parameters, object and presentation format 76 | ## (h - human readable, b - bytes)! 77 | ## [ example: "$(size "$src" "b")" ] 78 | ## compRatio() - Gets compression ratio. 79 | ## @Takes two parameters, source and archive! 80 | ## [ example: "$(compRatio "$src" "$archive")" ] 81 | ## compInf() - Outputs compression info. 82 | ## [ example: compInf $src $archive ] 83 | ################################################################################ 84 | 85 | ## =========================== ## 86 | ## Set configuration variables ## 87 | ## =========================== ## 88 | 89 | srcName="test-file" 90 | srcDir="/tmp/" 91 | runDir="/tmp/run/" 92 | outDir="/tmp/out/" 93 | archDir="/tmp/vault/" 94 | archName="arch" 95 | logName=$archName 96 | lockCheck=true 97 | lockName=$archName 98 | lockSleep=600 99 | lockWait=12 100 | oldArchDaysBack=30 101 | maxOldArch=10 102 | mailSendSucc=false 103 | mailSendFail=true 104 | mailFileName=$archName 105 | mailTo="your@email.address" 106 | mailSubjectSucc="$0 says: Process has finished successfuly!" 107 | mailSubjectFail="$0 says: Warning! Process has failed!" 108 | stdType="cl" 109 | txtFormat=true 110 | debug=true 111 | ## Do not change this variables if you don't know how! 112 | timeStamp="[%Y.%m.%d - %T]" 113 | compressor="xz -T 4 -9 -c" 114 | archExt=".tar.xz" 115 | 116 | ## ======================================= ## 117 | ## WARNING! Do not modify past this point! ## 118 | ## ======================================= ## 119 | 120 | ## ---------------- ## 121 | ## Global variables ## 122 | ## ---------------- ## 123 | 124 | export TERM=xterm 125 | TIMEFORMAT="%E" 126 | 127 | runDir=${runDir%/} ; srcDir=${srcDir%/} ; srcName=${srcName%/} ## Deslashify. 128 | outDir=${outDir%/} ; archDir=${archDir%/} ## Deslashify. 129 | src="$srcDir/$srcName" ## Full path to source. 130 | archiveFN="$archName$archExt" ## Archive file name and extension. 131 | archive="$outDir/$archiveFN" ## Full path to archive. 132 | logFile="$runDir/$logName.log" ## Full path to log file. 133 | logFileF="$runDir/$logName.f.log" ## Full path to log file. 134 | lockFile="$runDir/$lockName.lock" ## Full path to lock file. 135 | mailFile="$runDir/$mailFileName.mail" ## Full path to mail file. 136 | execute=0 ## Allow execution. 137 | execTime="" ## Compression execution time. 138 | 139 | ## ------------- ## 140 | ## Initial setup ## 141 | ## ------------- ## 142 | 143 | exec 3>&1 1>>$logFile 2>&1 ## Modifies std output. 144 | 145 | ## ---------------- ## 146 | ## Helper functions ## 147 | ## ---------------- ## 148 | 149 | ## Prints messages. 150 | msg() { [[ $stdType == "cl" ]] && printf "$@" | tee /dev/fd/3 || printf "$@" ; } 151 | 152 | ## Text formating. 153 | tf() { 154 | if [[ $txtFormat == true ]] ; then 155 | res="" 156 | for ((i=2; i<=$#; i++)) ; do 157 | case "${!i}" in 158 | "bold" ) res="$res\e[1m" ;; 159 | "underline" ) res="$res\e[4m" ;; 160 | "reverse" ) res="$res\e[7m" ;; 161 | "red" ) res="$res\e[91m" ;; 162 | "green" ) res="$res\e[92m" ;; 163 | "yellow" ) res="$res\e[93m" ;; 164 | esac 165 | done 166 | echo -e "$res$1\e[0m" 167 | else 168 | echo "$1" 169 | fi 170 | } 171 | 172 | ## Sets timestamp. 173 | tStamp() { 174 | if [[ -n ${1} ]] && [[ ! -n ${2} ]] ; then 175 | date +"${1}" 176 | elif [[ -n ${1} ]] && [[ -n ${2} ]] ; then 177 | date -d "${2}" +"${1}" 178 | else 179 | date +"${timeStamp}" 180 | fi 181 | } 182 | 183 | ## Seconds to readable time. 184 | secToTime() { 185 | timeInSec=$1 186 | if [[ $timeInSec -ge 0 ]] && [[ $timeInSec -le 59 ]]; then 187 | echo "${timeInSec}s" 188 | elif [[ $timeInSec -ge 60 ]] && [[ $timeInSec -le 3599 ]]; then 189 | m=$(( timeInSec / 60 )) 190 | s=$(( timeInSec % 60 )) 191 | echo "${m}m ${s}s" 192 | elif [[ $timeInSec -ge 3600 ]] && [[ $timeInSec -le 86399 ]]; then 193 | h=$(( timeInSec / 3600 )) 194 | m=$(( (timeInSec % 3600) / 60 )) 195 | s=$(( (timeInSec % 3600) % 60 )) 196 | echo "${h}h ${m}m ${s}s" 197 | fi 198 | } 199 | 200 | ## Sends e-mail. 201 | sendMail() { 202 | if [[ ! -n $1 ]] && [[ $mailSendSucc == true ]] ; then 203 | echo "Process finished, no errors found!" > $mailFile 204 | mail -s "$mailSubjectSucc" $mailTo < $mailFile 205 | fi 206 | if [[ -n $1 ]] && [[ $mailSendFail == true ]] ; then 207 | echo "Process has failed at step: $1" > $mailFile 208 | mail -s "$mailSubjectFail" $mailTo < $mailFile 209 | fi 210 | } 211 | 212 | ## Checks debug status. 213 | ifDebug() { [[ $debug == true ]] ; } 214 | 215 | ## Checks lock status. 216 | ifLock() { [[ $lockCheck == true ]] ; } 217 | 218 | ## Outputs process success. 219 | succ() { msg "%s %s!\n\n" "$(tStamp)" "$(tf "SUCCESS" "bold" "green")" \ 220 | ; sendMail ; exit 0 ; } 221 | 222 | ## Outputs process fail. 223 | fail() { msg "%s %s at %s!\n\n" "$(tStamp)" "$(tf "FAIL" "bold" "red")" \ 224 | "$(tf "$1" "bold" "underline" "yellow")" ; sendMail "$1" ; exit 1 ; } 225 | 226 | ## --------- ## 227 | ## Functions ## 228 | ## --------- ## 229 | 230 | ## Checks configuration. 231 | checkConf() { 232 | 233 | ## Check file/directory permissions. 234 | checkPerm() { 235 | if [[ ! -n ${2} ]] ; then 236 | [[ -r ${1} ]] && [[ -w ${1} ]] && echo 1 || echo 0 ; 237 | else 238 | case "$2" in 239 | "f" ) [[ -f ${1} ]] && [[ -r ${1} ]] && [[ -w ${1} ]] && \ 240 | echo 1 || echo 0 ;; 241 | "d" ) [[ -d ${1} ]] && [[ -r ${1} ]] && [[ -w ${1} ]] && \ 242 | echo 1 || echo 0 ;; 243 | "fd" ) [[ -d ${1} ]] || [[ -f ${1} ]] && [[ -r ${1} ]] && \ 244 | [[ -w ${1} ]] && echo 1 || echo 0 ;; 245 | esac 246 | fi 247 | } 248 | 249 | ## Output file/directory status. 250 | status() { [[ $1 == 1 ]] && tf "OK" "bold" "green" || \ 251 | tf "NOT OK" "bold" "red" ; } 252 | 253 | ## Check source. 254 | pass=$(( pass + $(checkPerm "$src" "fd") )) 255 | ifDebug && msg " -> Source\t\t%s\tis set to: %s\n" \ 256 | "$(status "$(checkPerm "$src" "fd")")" "$(tf $src "bold")" 257 | 258 | ## Check runtime directory. 259 | pass=$(( pass + $(checkPerm "$runDir" "d") )) 260 | ifDebug && msg " -> Runtime directory %s\tis set to: %s\n" \ 261 | "$(status "$(checkPerm "$runDir" "d")")" "$(tf $runDir "bold")" 262 | 263 | ## Check output directory. 264 | pass=$(( pass + $(checkPerm "$outDir" "d") )) 265 | ifDebug && msg " -> Output directory %s\tis set to: %s\n" \ 266 | "$(status "$(checkPerm "$outDir" "d")")" "$(tf $outDir "bold")" 267 | 268 | ## Check archive directory. 269 | pass=$(( pass + $(checkPerm "$archDir" "d") )) 270 | ifDebug && msg " -> Archive directory %s\tis set to: %s\n" \ 271 | "$(status "$(checkPerm "$archDir" "d")")" "$(tf $archDir "bold")" 272 | 273 | ## Display rest of the config. 274 | ifDebug && msg " -> Archive\t\t\tis set to: %s\n" \ 275 | "$(tf $archiveFN "bold")" 276 | ifDebug && msg " -> Log file\t\t\tis set to: %s\n" \ 277 | "$(tf $logName "bold")" 278 | ifDebug && ifLock && msg " -> Lock file\t\t\tis set to: %s\n" \ 279 | "$(tf $lockName "bold")" 280 | ifDebug && msg " -> Timestamp\t\t\tis set to: %s\n" \ 281 | "$(tf "$timeStamp" "bold")" 282 | ifDebug && msg " -> Compressor\t\t\tis set to: %s\n" \ 283 | "$(tf "$compressor" "bold")" 284 | 285 | ## Validate config 286 | if [[ $pass == 4 ]] ; then 287 | ifDebug && msg " -> Configuration is %s\n" "$(status 1)" 288 | else 289 | msg " -> Configuration is %s, exiting...\n" "$(status 0)" 290 | fail "configuration check" 291 | fi 292 | } 293 | 294 | ## Checks for lock file. 295 | checkLock() { 296 | [[ -f ${lockFile} ]] && ifDebug && msg \ 297 | " -> Lock file %s, waiting iterations are set to: %s\n" \ 298 | "$(tf "is in place" "bold")" "$(tf "$lockWait" "bold")" 299 | for ((i=1; i<=lockWait; i++)) ; do 300 | if [[ -f ${lockFile} ]] ; then 301 | ifDebug && msg " -> waiting for %s (%s)\n" \ 302 | "$(tf "$(secToTime "$lockSleep")" "bold")" \ 303 | "$(tf "$i" "bold" "yellow")" ; sleep $lockSleep 304 | else 305 | i=1000 306 | fi 307 | if [[ $i == "$lockWait" ]] ; then 308 | msg " -> Lock file %s, exiting...\n" "$(tf "still in place" "bold")" 309 | fail "lock file check" 310 | fi 311 | done 312 | } 313 | 314 | ## Checks if the old archive exists, if it does, deletes it. 315 | checkForOldArch() { 316 | if [[ -f ${archive} ]] ; then 317 | 318 | oldArchCount() { 319 | res=$(printf "%s" "$(ls -afq $archDir | wc -l)") 320 | echo $(( res - 2 )) 321 | } 322 | oldestArch() { 323 | find $archDir -type f -mtime -$oldArchDaysBack -print0 \ 324 | | xargs -0 ls -tr | head -n 1 325 | } 326 | 327 | ## Move previous archive to the vault and rename it. 328 | ifDebug && msg \ 329 | " -> Previous archive file %s, created on %s in %s, moving...\n" \ 330 | "$(tf "exists" "bold" "yellow")" \ 331 | "$(tf "$(date -r $archive +"%Y.%m.%d")" "bold")" \ 332 | "$(tf "$(date -r $archive +"%R")" "bold")" 333 | prevArch="$archDir/$archName$(date -r $archive +"_%Y%m%d-%H%M%S")$archExt" 334 | mv -f "$archive" "$prevArch" || fail "moving archive to the vault" 335 | 336 | ## Check if previous archive were moved to the vault. 337 | if [[ -f ${prevArch} ]] && [[ ! -f ${archive} ]] ; then 338 | ifDebug && msg \ 339 | " -> Previous archive %s to the vault as: %s, proceeding...\n" \ 340 | "$(tf "moved" "bold" "green")" "$(tf "$prevArch" "bold")" 341 | execute=$(( execute + 1 )) 342 | else 343 | msg " -> %s %s previous archive to the vault, exiting...\n" \ 344 | "$(tf "WARNING!" "bold" "yellow")" "$(tf "Can't move" "bold")" 345 | fail "moving archive to the vault" 346 | fi 347 | 348 | ## Count old archive files inside the vault and delete the oldest. 349 | oldArchCount=$(oldArchCount) 350 | if [[ $oldArchCount -gt $maxOldArch ]] ; then 351 | ifDebug && msg " -> Number of old archives inside the vault is: %s\n" \ 352 | "$(tf "$(oldArchCount)" "bold")" 353 | for ((i=oldArchCount; i>maxOldArch; i--)) ; do 354 | oldestArch=$(oldestArch) 355 | rm -f "$oldestArch" || fail "delete oldest archive" 356 | if [[ ! -f ${oldestArch} ]] ; then 357 | ifDebug && msg " -> Oldest file (%s) %s, proceeding...\n" \ 358 | "$(tf "$oldestArch" "bold")" "$(tf "was deleted" "bold" "yellow")" 359 | else 360 | msg " -> %s %s the oldest archive (%s), exiting...\n" \ 361 | "$(tf "WARNING!" "bold" "yellow")" "$(tf "Can't delete" "bold")" \ 362 | "$(tf "$oldestArch" "bold")" ; fail "delete oldest archive" 363 | fi 364 | done 365 | fi 366 | 367 | ## Check if old archives were indeed deleted. 368 | ifDebug && msg " -> Number of old archives inside the vault is: %s\n" \ 369 | "$(tf "$(oldArchCount)" "bold")" 370 | oldArchCount=$(oldArchCount) 371 | if [[ $oldArchCount -gt 10 ]] ; then 372 | msg " -> %s %s the oldest archive(s), exiting...\n" \ 373 | "$(tf "WARNING!" "bold" "yellow")" "$(tf "Can't delete" "bold")" 374 | fail "delete oldest archive(s)" 375 | else 376 | execute=$(( execute + 1 )) 377 | fi 378 | 379 | else 380 | ifDebug && msg " -> Old archive %s, proceeding...\n" \ 381 | "$(tf "does not exist" "bold" "green")" ; execute=2 382 | fi 383 | } 384 | 385 | ## Executes compression. 386 | execComp() { 387 | ## Go to source directory. 388 | cd $srcDir || fail "cd to source directory" 389 | if [[ ${PWD} != "$srcDir" ]] ; then 390 | msg " -> %s Can't cd to source directory, exiting...\n%s" \ 391 | "$(tf "WARNING!" "bold" "yellow")" ; fail "cd to source directory" 392 | fi 393 | execTime=$( { time tar cf - $srcName | $compressor - > $archive ; } 2>&1 ) 394 | } 395 | 396 | ## Gets object size values. 397 | size() { 398 | [[ $2 == "b" ]] && du -bs "$1" | awk '{ print $1 }' 399 | [[ $2 == "h" ]] && du -hs "$1" | awk '{ print $1 }' 400 | } 401 | 402 | ## Rounds floating numbers. 403 | round() { printf %."$2"f "$(echo "(((10^$2)*$1)+0.5)/(10^$2)" | bc)" ; } 404 | 405 | ## Gets compression ratio. 406 | compRatio() { printf "%.*f" 2 "$(let res="$1/$2"; printf "%s" "$res")" ; } 407 | 408 | ## Outputs compression info. 409 | compInf() { 410 | msg "\t-> Source size: %s (%s bytes)\n" \ 411 | "$(tf "$(size "$1" "h")" "bold")" "$(tf "$(size "$1" "b")" "bold")" 412 | msg "\t-> Archive size: %s (%s bytes)\n" \ 413 | "$(tf "$(size "$2" "h")" "bold")" "$(tf "$(size "$2" "b")" "bold")" 414 | msg "\t-> Compression ratio: %s\n" \ 415 | "$(tf "$(compRatio "$(size "$1" "b")" "$(size "$2" "b")")" "bold")" 416 | } 417 | 418 | ## Begins compression process. 419 | compress() { 420 | if [[ $execute == 2 ]] ; then 421 | ifDebug && msg " -> Flag %s set, executing...\n" "$(tf "is" "bold" "green")" 422 | execComp 423 | 424 | execTime=$(round "$execTime" 0) 425 | execTime=$(secToTime "$execTime") 426 | 427 | ## Check if archive file created. 428 | if [[ -f ${archive} ]] ; then 429 | msg " -> Archive %s created in %s!\n" "$(tf $archive "bold")" \ 430 | "$(tf "$execTime" "bold")" ; compInf $src $archive ; succ 431 | else 432 | msg " -> %s Archive file was %s created, exiting...\n" \ 433 | "$(tf "WARNING!" "bold" "yellow")" "$(tf "not" "bold" "red")" 434 | fail "archive file creation" 435 | fi 436 | else 437 | msg " -> Flag %s set, exiting...\n" "$(tf "is not" "bold" "red")" 438 | fail "execution flag check" 439 | fi 440 | } 441 | 442 | ## ---------- ## 443 | ## Initialize ## 444 | ## ---------- ## 445 | msg "%s Initializing...\n" "$(tStamp)" 446 | checkConf ## Check configuration. 447 | ifLock && checkLock ## Check for lock file. 448 | checkForOldArch ## Check for old archive file. 449 | compress ## Begin compression. 450 | --------------------------------------------------------------------------------