├── common ├── asd.cron.hourly ├── zsh-completion ├── bash-completion ├── asd.conf └── anything-sync-daemon.in ├── init ├── asd-resync.timer ├── asd-resync.service ├── asd.service └── asd.upstart ├── .github └── workflows │ └── shellcheck.yml ├── README.md ├── LICENSE ├── INSTALL ├── Makefile └── doc └── asd.1 /common/asd.cron.hourly: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | /usr/bin/anything-sync-daemon resync &> /dev/null 3 | -------------------------------------------------------------------------------- /init/asd-resync.timer: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Timer for Anything-sync-daemon - 1Hour 3 | PartOf=asd-resync.service asd.service 4 | 5 | [Timer] 6 | OnUnitActiveSec=1h 7 | -------------------------------------------------------------------------------- /common/zsh-completion: -------------------------------------------------------------------------------- 1 | #compdef asd anything-sync-daemon 2 | 3 | _asd() { 4 | local -a options 5 | 6 | options=('p:Preview what asd will do/is doing and printout useful info' 7 | 'c:Clean (delete without prompting) ALL crashrecovery dirs for all sync targets') 8 | 9 | _describe 'options' options 10 | } 11 | 12 | _asd 13 | -------------------------------------------------------------------------------- /init/asd-resync.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Timed resync 3 | Documentation=man:asd(1) man:anything-sync-daemon(1) 4 | Documentation=https://wiki.archlinux.org/index.php/Anything-sync-daemon 5 | After=asd.service 6 | Wants=asd-resync.timer 7 | PartOf=asd.service 8 | 9 | [Service] 10 | Environment="DEBUG=1" 11 | ExecStart=/usr/bin/anything-sync-daemon resync 12 | 13 | [Install] 14 | WantedBy=default.target 15 | -------------------------------------------------------------------------------- /.github/workflows/shellcheck.yml: -------------------------------------------------------------------------------- 1 | name: Shellcheck 2 | 3 | on: [ push ] 4 | 5 | jobs: 6 | lint: 7 | # The type of runner that the job will run on 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v2 12 | - name: Static analysis using shellcheck 13 | run: | 14 | sudo apt-get install shellcheck -y 15 | shellcheck common/anything-sync-daemon.in -e SC1091 16 | -------------------------------------------------------------------------------- /init/asd.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Anything-sync-daemon 3 | Documentation=man:asd(1) man:anything-sync-daemon(1) 4 | Documentation=https://wiki.archlinux.org/index.php/Anything-sync-daemon 5 | Wants=local-fs.target 6 | Wants=asd-resync.service 7 | After=winbindd.service 8 | 9 | [Service] 10 | Type=oneshot 11 | RemainAfterExit=yes 12 | Environment="DEBUG=1" 13 | ExecStart=/usr/bin/anything-sync-daemon sync 14 | ExecStop=/usr/bin/anything-sync-daemon unsync 15 | 16 | [Install] 17 | WantedBy=multi-user.target 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Anything-sync-daemon 2 | Anything-sync-daemon (asd) is a tiny pseudo-daemon designed to manage user defined dirs in tmpfs and to periodically sync back to the physical disc (HDD/SSD). This is accomplished via a symlinking step and an innovative use of rsync to maintain back-up and synchronization between the two. One of the major design goals of asd is a completely transparent user experience. 3 | 4 | ## Documentation 5 | Consult the man page or the wiki page: https://wiki.archlinux.org/index.php/Anything-sync-daemon 6 | 7 | ## Installation from Source 8 | To build from source, see the included INSTALL text document. 9 | 10 | ## Installation from Distro Packages 11 | * ![logo](http://www.monitorix.org/imgs/archlinux.png "arch logo")Arch: in the [AUR](https://aur.archlinux.org/packages/anything-sync-daemon). 12 | -------------------------------------------------------------------------------- /common/bash-completion: -------------------------------------------------------------------------------- 1 | _asd() { 2 | local cur prev opts 3 | COMPREPLY="" 4 | 5 | #word we're currently working on completing 6 | cur="${COMP_WORDS[COMP_CWORD]}" 7 | 8 | #the previous word 9 | prev="${COMP_WORDS[COMP_CWORD-1]}" 10 | 11 | #the escaped spaces are needed for bash-complete to put a space after completion 12 | #otherwise we get stuck on the last letter of our completions 13 | opts="parse\ clean\ " 14 | 15 | 16 | case "${prev}" in 17 | "parse"|"p"|"Parse"|"P"|"Preview"|"preview"|"debug") 18 | printf "%b" "\nPreview what asd will do/is doing and print out useful info\n" 19 | printf "%b " $@ 20 | ;; 21 | "clean"|"Clean"|"c"|"C") 22 | printf "%b" "\nClean (delete without prompting) ALL crashrecovery dirs for all sync targets\n" 23 | printf "%b " $@ 24 | ;; 25 | "asd"|"anything-sync-daemon"|*) 26 | COMPREPLY=($(compgen -W "${opts}" -- ${cur}) );; 27 | esac 28 | 29 | 30 | } 31 | complete -F _asd asd 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013-2022 graysky 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | DEPENDENCIES 2 | bash >=4.0 3 | cron (only if using upstart) 4 | find 5 | rsync 6 | 7 | a supported init system 8 | -systemd 9 | -upstart 10 | 11 | WARNING 12 | To avoid data loss, it is HIGHLY recommended that users stop any running 13 | versions of this software before install updates. Internal changes are 14 | always possible from one release to another. 15 | 16 | BUILDING 17 | Setup the via a make. 18 | 19 | $ make 20 | 21 | Running a `make install-xxx` as root will distribute the files to the file- 22 | system. Most users will want to select from three options depending on target 23 | init system (do not run both)! 24 | 25 | # make install-systemd-all 26 | # make install-upstart-all 27 | 28 | As of v3.13, the Makefiles for systemd-native distros such as Arch, Exherbo, 29 | and Fedora, no longer install the deprecated cron script using a systemd 30 | timer instead. Users may override and install the deprecated cron script by 31 | running make with `install-with-cron` instead of `install` at this point. 32 | 33 | USE A DISTRO PROVIDED PACKAGE 34 | ARCH LINUX 35 | Arch users may build the package directly with the provided PKGBUILD: 36 | https://aur.archlinux.org/packages/anything-sync-daemon 37 | 38 | CHAKRA 39 | Chakra users may build the package directly with the provided PKGBUILD: 40 | http://chakraos.org/ccr/packages.php?ID=3750 41 | -------------------------------------------------------------------------------- /common/asd.conf: -------------------------------------------------------------------------------- 1 | # 2 | # /etc/asd.conf 3 | # 4 | # For documentation, refer to the asd man page 5 | 6 | ## WARNING Do NOT edit anything in this file while asd is running! 7 | ## To protect data from corruption, in the event that you do make an edit 8 | ## while asd is active, any changes made will be applied the next time 9 | ## you start-up asd. 10 | 11 | # Define the target(s) directories in the WHATTOSYNC array. 12 | # Do NOT define a file! These MUST be directories with an absolute path. 13 | # 14 | # Note that the target DIRECTORIES and all subdirs under them will be included. 15 | # In other words, this is recursive. 16 | # 17 | # Below is an example to whet your appetite. 18 | #WHATTOSYNC=('/srv/http' '/var/lib/monitorix' '/foo/bar') 19 | WHATTOSYNC=() 20 | 21 | # Define where data will reside in tmpfs. 22 | # This location must be mounted to tmpfs and MUST be writable and executable. 23 | # 24 | # If using bleachbit, do NOT invoke it with the '--clean system.tmp' switch or 25 | # you will remove a key dot file (.foo) from /tmp that asd needs to keep track 26 | # of sync status. 27 | # 28 | # Note that using a value of '/dev/shm' can cause problems with systemd's 29 | # NAMESPACE spawning only when users enable the overlayfs option. 30 | # 31 | #VOLATILE="/tmp" 32 | 33 | # ASD can break hardlinks present in the system, it has a safety check to ensure 34 | # that the synced directories don't have the presence of any hardlinks in them 35 | # by default, incase you want the directory to work standalone 36 | # and not affect any other hardlinks of the files present in the synced directory, 37 | # you can disable this safety check 38 | # ENABLE_HARDLINK_SAFETY_CHECK=1 39 | 40 | # Uncomment and set to yes to use an overlayfs instead of a full copy to reduce 41 | # the memory costs and to improve sync/unsync operations. 42 | # 43 | # You must modprobe either the 'overlayfs' or 'overlay' module prior to running asd if 44 | # you enable this option. Distros running the linux kernel version >=3.18.0 are likely 45 | # using the 'overlay' module while some distros shipping older kernels, notably Ubuntu 46 | # provide the older version of this technology which is provided in the 'overlayfs' 47 | # module not 'overlay' module. 48 | #USE_OVERLAYFS="no" 49 | 50 | # Uncomment and set to no to completely disable the crash recovery feature of asd. 51 | # 52 | # The default is to create crash recovery backups if the system is ungracefully 53 | # powered-down due to a kernel panic, hitting the reset switch, battery going 54 | # dead, etc. Some users keep very diligent backups and don't care to have this 55 | # feature enabled. 56 | #USE_BACKUPS="yes" 57 | 58 | # Uncomment and set to an integer that is the maximum number of crash recovery 59 | # snapshots to keep (the oldest ones are delete first). 60 | # 61 | # The default is to save the most recent 5 crash recovery snapshots. 62 | #BACKUP_LIMIT=5 63 | -------------------------------------------------------------------------------- /init/asd.upstart: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | ### BEGIN INIT INFO 3 | # Provides: anything-sync-daemon 4 | # Required-Start: $local_fs 5 | # Required-Stop: $local_fs 6 | # Default-Start: 2 3 4 5 7 | # Default-Stop: 0 1 6 8 | # Short-Description: Keeps user defined dirs in tmpfs. 9 | # Description: Anything-sync-daemon (asd) is a tiny pseudo-daemon designed to manage 10 | # user specified directories in tmpfs and to periodically sync them back 11 | # to the physical disc (HDD/SSD). 12 | ### END INIT INFO 13 | 14 | # PATH should only include /usr/* if it runs after the mountnfs.sh script 15 | PATH=/sbin:/usr/sbin:/bin:/usr/bin 16 | DESC="Anything-sync-daemon" 17 | NAME=anything-sync-daemon 18 | DAEMON=/usr/bin/$NAME 19 | LOCKFILE=/var/run/asd 20 | SCRIPTNAME=/etc/init.d/$NAME 21 | 22 | # Exit if the package is not installed 23 | [ -x "$DAEMON" ] || exit 0 24 | 25 | # Exit if rsync is not installed 26 | if [ ! -x "/usr/bin/rsync" ]; then 27 | log_failure_msg "Could not locate rsync." 28 | exit 1 29 | fi 30 | 31 | # Load the VERBOSE setting and other rcS variables 32 | . /lib/init/vars.sh 33 | 34 | # Define LSB log_* functions. 35 | # Depend on lsb-base (>= 3.2-14) to ensure that this file is present 36 | # and status_of_proc is working. 37 | . /lib/lsb/init-functions 38 | 39 | # 40 | # Function that starts the daemon/service 41 | # 42 | do_start() 43 | { 44 | [ -f $LOCKFILE ] && return 1 45 | $DAEMON sync 46 | RETVAL="$?" 47 | [ "$RETVAL" != 0 ] && return 2 48 | return 0 49 | } 50 | 51 | # 52 | # Function that stops the daemon/service 53 | # 54 | do_stop() 55 | { 56 | [ ! -f $LOCKFILE ] && return 1 57 | $DAEMON unsync 58 | RETVAL="$?" 59 | [ "$RETVAL" != 0 ] && return 2 60 | return 0 61 | } 62 | 63 | case "$1" in 64 | start) 65 | [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" 66 | do_start 67 | case "$?" in 68 | 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; 69 | 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; 70 | esac 71 | ;; 72 | stop) 73 | [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" 74 | do_stop 75 | case "$?" in 76 | 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; 77 | 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; 78 | esac 79 | ;; 80 | status) 81 | if [ -f $LOCKFILE ]; then 82 | log_success_msg "$NAME is running" 83 | exit 0 84 | else 85 | log_failure_msg "$NAME is not running" 86 | exit 1 87 | fi 88 | ;; 89 | restart|force-reload) 90 | # 91 | # If the "reload" option is implemented then remove the 92 | # 'force-reload' alias 93 | # 94 | log_daemon_msg "Restarting $DESC" "$NAME" 95 | do_stop 96 | case "$?" in 97 | 0|1) 98 | do_start 99 | case "$?" in 100 | 0) log_end_msg 0 ;; 101 | 1) log_end_msg 1 ;; # Old process is still running 102 | *) log_end_msg 1 ;; # Failed to start 103 | esac 104 | ;; 105 | *) 106 | # Failed to stop 107 | log_end_msg 1 108 | ;; 109 | esac 110 | ;; 111 | *) 112 | echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 113 | exit 3 114 | ;; 115 | esac 116 | 117 | : 118 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VERSION = 6.0.0 2 | PN = anything-sync-daemon 3 | 4 | PREFIX ?= /usr 5 | CONFDIR = /etc 6 | CRONDIR = /etc/cron.hourly 7 | INITDIR_SYSTEMD = /usr/lib/systemd/system 8 | INITDIR_UPSTART = /etc/init.d 9 | BINDIR = $(PREFIX)/bin 10 | DOCDIR = $(PREFIX)/share/doc/$(PN) 11 | MANDIR = $(PREFIX)/share/man/man1 12 | ZSHDIR = $(PREFIX)/share/zsh/site-functions 13 | BSHDIR = $(PREFIX)/share/bash-completion/completions 14 | 15 | # set to anything except 0 to enable manpage compression 16 | COMPRESS_MAN = 1 17 | 18 | RM = rm 19 | SED = sed 20 | INSTALL = install -p 21 | INSTALL_PROGRAM = $(INSTALL) -m755 22 | INSTALL_SCRIPT = $(INSTALL) -m755 23 | INSTALL_DATA = $(INSTALL) -m644 24 | INSTALL_DIR = $(INSTALL) -d 25 | 26 | Q = @ 27 | 28 | common/$(PN): Makefile common/$(PN).in 29 | $(Q)echo -e '\033[1;32mSetting version\033[0m' 30 | $(Q)$(SED) 's/@VERSION@/'$(VERSION)'/' common/$(PN).in > common/$(PN) 31 | 32 | help: install 33 | 34 | stop-asd: 35 | ifneq ($(PREFIX), /usr) 36 | sudo -E asd unsync 37 | endif 38 | 39 | disable-systemd: 40 | ifeq ($(PREFIX), /usr) 41 | systemctl stop asd asd-resync || /bin/true 42 | endif 43 | 44 | install-bin: stop-asd disable-systemd common/$(PN) 45 | $(Q)echo -e '\033[1;32mInstalling main script...\033[0m' 46 | $(INSTALL_DIR) "$(DESTDIR)$(BINDIR)" 47 | $(INSTALL_PROGRAM) common/$(PN) "$(DESTDIR)$(BINDIR)/$(PN)" 48 | ln -sf $(PN) "$(DESTDIR)$(BINDIR)/asd" 49 | cp -n common/asd.conf "$(DESTDIR)$(CONFDIR)/asd.conf" 50 | $(INSTALL_DIR) "$(DESTDIR)$(ZSHDIR)" 51 | $(INSTALL_DATA) common/zsh-completion "$(DESTDIR)/$(ZSHDIR)/_asd" 52 | $(INSTALL_DIR) "$(DESTDIR)$(BSHDIR)" 53 | $(INSTALL_DATA) common/bash-completion "$(DESTDIR)/$(BSHDIR)/asd" 54 | 55 | install-man: 56 | $(Q)echo -e '\033[1;32mInstalling manpage...\033[0m' 57 | $(INSTALL_DIR) "$(DESTDIR)$(MANDIR)" 58 | $(INSTALL_DATA) doc/asd.1 "$(DESTDIR)$(MANDIR)/asd.1" 59 | ifneq ($(COMPRESS_MAN),0) 60 | gzip -9 "$(DESTDIR)$(MANDIR)/asd.1" 61 | ln -sf asd.1.gz "$(DESTDIR)$(MANDIR)/$(PN).1.gz" 62 | else 63 | ln -sf asd.1 "$(DESTDIR)$(MANDIR)/$(PN).1" 64 | endif 65 | 66 | install-cron: 67 | $(Q)echo -e '\033[1;32mInstalling cronjob...\033[0m' 68 | $(INSTALL_DIR) "$(DESTDIR)$(CRONDIR)" 69 | $(INSTALL_SCRIPT) common/asd.cron.hourly "$(DESTDIR)$(CRONDIR)/asd-update" 70 | 71 | install-systemd: 72 | $(Q)echo -e '\033[1;32mInstalling systemd files...\033[0m' 73 | $(INSTALL_DIR) "$(DESTDIR)$(CONFDIR)" 74 | $(INSTALL_DIR) "$(DESTDIR)$(INITDIR_SYSTEMD)" 75 | $(INSTALL_DATA) init/asd.service "$(DESTDIR)$(INITDIR_SYSTEMD)/asd.service" 76 | $(INSTALL_DATA) init/asd-resync.service "$(DESTDIR)$(INITDIR_SYSTEMD)/asd-resync.service" 77 | $(INSTALL_DATA) init/asd-resync.timer "$(DESTDIR)$(INITDIR_SYSTEMD)/asd-resync.timer" 78 | 79 | install-upstart: 80 | $(Q)echo -e '\033[1;32mInstalling upstart files...\033[0m' 81 | $(INSTALL_DIR) "$(DESTDIR)$(CONFDIR)" 82 | $(INSTALL_DIR) "$(DESTDIR)$(INITDIR_UPSTART)" 83 | $(INSTALL_SCRIPT) init/asd.upstart "$(DESTDIR)$(INITDIR_UPSTART)/asd" 84 | 85 | 86 | install-systemd-all: install-bin install-man install-systemd 87 | 88 | install-upstart-all: install-bin install-man install-cron install-upstart 89 | 90 | install: 91 | $(Q)echo "run one of the following:" 92 | $(Q)echo " make install-systemd-all (systemd based systems)" 93 | $(Q)echo " make install-upstart-all (upstart based systems)" 94 | $(Q)echo 95 | $(Q)echo "or check out the Makefile for specific rules" 96 | 97 | uninstall-bin: 98 | $(RM) "$(DESTDIR)$(BINDIR)/$(PN)" 99 | $(RM) "$(DESTDIR)$(BINDIR)/asd" 100 | $(RM) "$(DESTDIR)/$(ZSHDIR)/_asd" 101 | $(RM) "$(DESTDIR)/$(BSHDIR)/asd" 102 | 103 | uninstall-man: 104 | $(RM) -f "$(DESTDIR)$(MANDIR)/$(PN).1.gz" 105 | $(RM) -f "$(DESTDIR)$(MANDIR)/asd.1.gz" 106 | $(RM) -f "$(DESTDIR)$(MANDIR)/$(PN).1" 107 | $(RM) -f "$(DESTDIR)$(MANDIR)/asd.1" 108 | 109 | uninstall-cron: 110 | $(RM) "$(DESTDIR)$(CRONDIR)/asd-update" 111 | 112 | uninstall-systemd: 113 | $(RM) "$(DESTDIR)$(CONFDIR)/asd.conf" 114 | $(RM) "$(DESTDIR)$(INITDIR_SYSTEMD)/asd.service" 115 | $(RM) "$(DESTDIR)$(INITDIR_SYSTEMD)/asd-resync.service" 116 | $(RM) "$(DESTDIR)$(INITDIR_SYSTEMD)/asd-resync.timer" 117 | 118 | uninstall-upstart: 119 | $(RM) "$(DESTDIR)$(CONFDIR)/asd.conf" 120 | $(RM) "$(DESTDIR)$(INITDIR_UPSTART)/asd" 121 | 122 | uninstall-systemd-all: uninstall-bin uninstall-man uninstall-systemd 123 | 124 | uninstall-upstart-all: uninstall-bin uninstall-man uninstall-cron uninstall-upstart 125 | 126 | uninstall: 127 | $(Q)echo "run one of the following:" 128 | $(Q)echo " make uninstall-systemd-all (systemd based systems)" 129 | $(Q)echo " make uninstall-upstart-all (upstart based systems)" 130 | $(Q)echo 131 | $(Q)echo "or check out the Makefile for specific rules" 132 | 133 | clean: 134 | $(RM) -f common/$(PN) 135 | 136 | .PHONY: help install-bin install-man install-cron install-systemd install-upstart install-systemd-all install-upstart-all install uninstall-bin uninstall-man uninstall-cron uninstall-systemd uninstall-upstart uninstall-systemd-all uninstall clean 137 | -------------------------------------------------------------------------------- /doc/asd.1: -------------------------------------------------------------------------------- 1 | .\" Text automatically generated by txt2man 2 | .TH anything-sync-daemon 1 "26 November 2016" "" "" 3 | .SH NAME 4 | \fBanything-sync-daemon \fP- Symlinks and syncs user specified dirs to RAM thus reducing HDD/SDD calls and speeding-up the system. 5 | \fB 6 | .SH DESCRIPTION 7 | Anything-sync-daemon (asd) is a tiny pseudo-daemon designed to manage user specified directories referred to as sync targets from here on out, in tmpfs and to periodically sync them back to the physical disc (HDD/SSD). This is accomplished via a bind mounting step and an innovative use of rsync to maintain synchronization between a tmpfs copy and media-bound backups. Additionally, asd features several crash-recovery features. 8 | .PP 9 | Design goals of asd: 10 | .RS 11 | .IP \(bu 3 12 | Completely transparent user experience. 13 | .IP \(bu 3 14 | Reduced wear to physical discs (particularly SSDs). 15 | .IP \(bu 3 16 | Speed. 17 | .RE 18 | .PP 19 | Since the sync targets is relocated into tmpfs (RAM disk), the corresponding onslaught of I/O associated with system usage of them is also redirected from the physical disc to RAM, thus reducing wear to the physical disc and also improving speed and responsiveness. 20 | .SH SETUP 21 | /etc/asd.conf contains all user managed settings. 22 | Optionally another file can be used by setting the ASDCONF environment variable. 23 | .PP 24 | NOTE: edits made to /etc/asd.conf while asd is running will be applied only after asd has been restarted from the init service. 25 | .RS 26 | .IP \(bu 3 27 | At a minimum, define the sync targets to be managed by asd in the WHATTOSYNC array. Syntax below. 28 | .IP \(bu 3 29 | Optionally uncomment and define the location of your distro's tmpfs* in the VOLATILE variable. 30 | .IP \(bu 3 31 | Optionally enable the use of overlayfs to improve sync speed even further and use a smaller memory footprint. Do this in the USE_OVERLAYFS variable. Note that this option requires your kernel to be configured to use either the 'overlay' or 'overlayfs' module. See the FAQ below for additional details on this feature. 32 | .IP \(bu 3 33 | Optionally disable the use of crash-recovery snapshots. Do this in the USE_BACKUPS variable. 34 | .IP \(bu 3 35 | Optionally define the number of crash-recovery snapshots to keep. Do this in the BACKUP_LIMIT variable. 36 | .RE 37 | .PP 38 | *Note that the default value of "/tmp" should work just fine for the VOLATILE setting. If using bleachbit, do NOT invoke it with the '\fB--clean\fP system.tmp' switch or you will remove a key dot file (.foo) from /tmp that asd needs to keep track of sync status. Also note that using a value of "/dev/shm" can cause problems with systemd's NAMESPACE spawning only when users enable the overlayfs option. 39 | .PP 40 | Example: 41 | .PP 42 | .nf 43 | .fam C 44 | WHATTOSYNC=('/var/lib/monitorix' '/srv/http' '/foo/bar') 45 | 46 | or 47 | 48 | WHATTOSYNC=( 49 | '/var/lib/monitorix' 50 | '/srv/http' 51 | '/foo/bar' 52 | ) 53 | 54 | .fam T 55 | .fi 56 | .SH RUNNING ASD 57 | .SS PREVIEW MODE 58 | The preview option can be called to show users exactly what asd will do/is doing based on the entries in /etc/asd.conf as well printout useful information such as dir size, paths, and if any recovery snapshots have been created. 59 | .PP 60 | .nf 61 | .fam C 62 | $ asd p 63 | 64 | Anything-sync-daemon on Arch Linux. 65 | 66 | Systemd service is currently active. 67 | Systemd resync service is currently active. 68 | Overlayfs v23 is currently active. 69 | 70 | Asd will manage the following per /run/asd.conf settings: 71 | 72 | owner/group id: root/0 73 | target to manage: /srv/http/serve 74 | sync target: /srv/http/.serve-backup_asd 75 | tmpfs target: /tmp/asd-root/srv/http/serve 76 | dir size: 21M 77 | overlayfs size: 15M 78 | recovery dirs: 2 <- delete with the c option 79 | dir path/size: /srv/http/.serve-backup_asd-crashrecovery-20141105_124948 (17M) 80 | dir path/size: /srv/http/.serve-backup_asd-crashrecovery-20150124_062311 (21M) 81 | 82 | owner/group id: facade/100 83 | target to manage: /home/facade/logs 84 | sync target: /home/facade/.logs-backup_asd 85 | tmpfs target: /tmp/asd-facadey/home/facade/logs 86 | dir size: 1.5M 87 | overlayfs size: 480K 88 | recovery dirs: none 89 | 90 | .fam T 91 | .fi 92 | .SS CLEAN MODE 93 | The clean mode will delete ALL recovery snapshots that have accumulated. Run this only if you are sure that you want to delete them. 94 | .PP 95 | Note that if a sync target is owned by root or another user, and if you call asd to clean, it will throw errors based on the permissions of your sync targets. 96 | .PP 97 | .nf 98 | .fam C 99 | $ asd c 100 | 101 | Anything-sync-daemon on Arch Linux. 102 | 103 | Deleting 2 crashrecovery dirs for sync target /srv/http/serve 104 | /srv/http/.serve-backup_asd-crashrecovery-20141105_124948 105 | /srv/http/.serve-backup_asd-crashrecovery-20150124_062311 106 | 107 | .fam T 108 | .fi 109 | .SS START AND STOP ASD FOR SYSTEMD USERS 110 | Both a systemd service file and timer are provided, and should be used to start or stop asd. 111 | .PP 112 | The role of the timer is update the tmpfs copies back to the disk. This occurs once per hour by default. The timer is started automatically with asd.service. 113 | .PP 114 | .nf 115 | .fam C 116 | # systemctl [option] asd 117 | 118 | .fam T 119 | .fi 120 | Available options: 121 | start 122 | stop 123 | enable 124 | disable 125 | .SS START AND STOP ASD FOR USERS OF OTHER INIT SYSTEMS 126 | For distros not using systemd, another init script should be used to manage the daemon. Examples are provided and are known to work with Upstart. 127 | .PP 128 | Note that for these init systems, the supplied cron script (installed to /etc/cron.hourly) will run the resync option to keep the tmpfs copies sync'ed. Of course, the target system must have cron installed and active for this to happen. 129 | .SH SUPPORTED DISTROS 130 | At this time, the following distros are officially supported but there is no reason to think that asd will not run on another distro: 131 | .IP \(bu 3 132 | Arch Linux 133 | .SH FAQ 134 | Q1: What is overlayfs mode? 135 | .PP 136 | A1: Overlayfs is a simple union file-system mainlined in the Linux kernel version 3.18.0. Starting with asd version 5.54, overlayfs can be used to reduce the memory footprint of asd's tmpfs space and to speed up sync and unsync operations. The magic is in how the overlay mount only writes out data that has changed rather than the entire sync target. See Example 1 below. The same recovery features asd uses in its default mode are also active when running in overlayfs mode. Overlayfs mode is enabled by uncommenting the USE_OVERLAYFS= in /etc/asd.conf followed by a restart of the daemon. 137 | .PP 138 | There are several versions of overlayfs available to the Linux kernel in production in various distros. Versions 22 and lower have a module called 'overlayfs' while newer versions (23 and higher) have a module called 'overlay' -- note the lack of the 'fs' in the newer version. Asd will automatically detect the overlayfs available to your kernel if it is configured to use one of them. 139 | .PP 140 | See the example in the PREVIEW MODE section above which shows a system using overlayfs to illustrate the memory savings that can be achieved. Note the "overlayfs size" report compared to the total "dir size" report for each sync target. Be aware that these numbers will change depending on just how much data is written to the sync target, but in common use cases, the overlayfs size will always be less than the dir size. 141 | .PP 142 | Q2: Why do I see directory ".foo-backup_asd" ".foo-backup_asd-old"? 143 | .PP 144 | A2: The way the backup process of asd works is that it creates a hard linked clone of the original directory; this is known as .foo-backup_asd-old. The other .foo-backup_asd is just a bind mount to the original directory link which is used to access the contents of the original directory for overlay purposes. 145 | .PP 146 | Q3: My system crashed and asd didn't sync back. What do I do? 147 | .PP 148 | A3: The "last good" backup of your sync targets is just fine still sitting happily on your filesystem. Upon restarting asd (on a reboot for example), a check is preformed to see if asd was exited in some corrupted state. If it is detected, asd will snapshot the "last good" backup before it rotates it back into place. Note that, since asd tries to decrease the disk usage, it never really "copies" the full contents of the directory and just uses the hardlinks to the previous files. And during the rsync step, it creates new files so that the previous hardlinks are untouched. So trying to modify the directory during the time asd is trying to backup can leave the directory in some corrupted state. 149 | .PP 150 | Q4: Where can I find this snapshot? 151 | .PP 152 | A4: You will find the snapshot in the same directory as the sync target and it will contain a date-time-stamp that corresponds to the time at which the recovery took place. For example, a /foo/bar snapshot will be /foo/.bar-backup_asd-crashrecovery-20141221_070112.tar.zstd -- of course, the date_time suffix will be different for you. 153 | .PP 154 | Q5: How can I restore the snapshot? 155 | .PP 156 | A5: Follow these steps: 157 | .RS 158 | .IP 1. 4 159 | Stop asd. 160 | .IP 2. 4 161 | Confirm that the directories created by asd is not present. If they are, asd did not stop correctly for other reasons. 162 | .IP 3. 4 163 | Move the "bad" copy of the sync taget to a backup (don't blindly delete anything). 164 | .IP 4. 4 165 | Untar the snapshot directory to the expected sync target. 166 | .RE 167 | .PP 168 | Example using /foo/bar: 169 | .RS 170 | .IP 1. 4 171 | cd /foo 172 | .IP 1. 4 173 | mv bar bar-bad 174 | .IP 2. 4 175 | tar -xvf .bar-backup_asd-crashrecovery-20141221_070112.tar.zstd 176 | .RE 177 | .PP 178 | At this point, check that everything is fine with the data on /foo/bar and, if all is well, it is safe to delete the snapshot. 179 | .PP 180 | Q6: Can asd delete the snapshots automatically? 181 | .PP 182 | A6: Yes, run asd with the "clean" switch to delete snapshots. 183 | .SH CONTRIBUTE 184 | Users wishing to contribute to this code, should fork and send a pull request. Source is freely available on the project page linked below. 185 | .SH BUGS 186 | Discover a bug? Please open an issue on the project page linked below. 187 | .RS 188 | .IP \(bu 3 189 | Currently, asd cannot handle open files on a sync target so if a hung process has something open there, it can be messy. 190 | .SH ONLINE 191 | .IP \(bu 3 192 | Project page: https://github.com/graysky2/anything-sync-daemon 193 | .IP \(bu 3 194 | Wiki page: https://wiki.archlinux.org/index.php/Anything-sync-daemon 195 | .SH AUTHOR 196 | graysky (graysky AT archlinux DOT us) 197 | .SH MAINTAINER 198 | Manorit Chawdhry (manorit2001@gmail.com) 199 | -------------------------------------------------------------------------------- /common/anything-sync-daemon.in: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # Anything-sync-daemon by graysky 5 | # Inspired by some code originally written by Colin Verot 6 | # 7 | 8 | # For debug, add DEBUG=1 asd 9 | debug() { 10 | if [[ -v $DEBUG && $DEBUG -eq 1 ]] 11 | then 12 | echo -e "$@" 13 | fi 14 | } 15 | 16 | set -e 17 | Error() { 18 | echo "Error occurred at $1" 19 | exit 1 20 | } 21 | trap 'Error $LINENO' ERR 22 | 23 | debug "checking flock" 24 | command -v flock >/dev/null 2>&1 || { 25 | echo "I require flock but it's not installed. Aborting." >&2 26 | exit 1; } 27 | debug "flock found" 28 | 29 | BLD="\e[01m" 30 | RED="\e[01;31m" 31 | GRN="\e[01;32m" 32 | BLU="\e[01;34m" 33 | NRM="\e[00m" 34 | VERS="@VERSION@" 35 | 36 | ASDCONF=${ASDCONF:-"/etc/asd.conf"} 37 | DAEMON_FILE=${DAEMON_FILE:-"/run/asd"} 38 | CRASH_RECOVERY_SUFFIX="crashrecovery" 39 | ASDNAME="asd" 40 | if [[ "$ASDCONF" != "/etc/asd.conf" ]]; then 41 | suffix=$(realpath "$ASDCONF" | sha256sum | cut -b -16) 42 | ASDNAME="$ASDNAME-$suffix" 43 | if [[ "$DAEMON_FILE" == "/run/asd" ]]; then 44 | DAEMON_FILE="/run/$ASDNAME" 45 | fi 46 | fi 47 | LOCKFILE="/run/$ASDNAME-lock" 48 | # shellcheck disable=SC2015 49 | [[ ${FLOCKER} != "$0" ]] && echo -e "${RED}Waiting for lock...${NRM}" && exec env FLOCKER="$0" flock --verbose -e "$LOCKFILE" "$0" "$*" || 50 | 51 | debug "${RED}ASDNAME:${NRM} $ASDNAME" 52 | debug "${BLU}Configuration file:${NRM} $ASDCONF" 53 | debug "${GRN}Daemon file:${NRM} $DAEMON_FILE" 54 | debug "${RED}Lockfile:${NRM} $LOCKFILE\n" 55 | 56 | # Setup check /etc/asd.conf 57 | debug "Checking for existence of $ASDCONF" 58 | if [[ -f "$ASDCONF" ]]; then 59 | debug "Checking if asd is already running" 60 | if [[ ! -f "$DAEMON_FILE" ]]; then 61 | debug "Daemon file not found: asd is not running" 62 | # do nothing if asd is currently running, otherwise 63 | # make sure only comments and variables/arrays are defined to prevent 64 | # problems 65 | if grep -Eqv "^$|^\s*['\")#]|^[^ ]*=[^;]*" "$ASDCONF"; then 66 | # found text that isn't a blank line, comment, or variable present so exit 67 | echo -e " ${RED}ERROR:${NRM}${BLD} Syntax error(s) detected in ${BLU}$ASDCONF${NRM}" 68 | echo -e "${NRM}${BLD}Line number: offending comment${NRM}" 69 | grep -Evn "^$|^\s*['\")#]|^[^ ]*=[^;]*" "$ASDCONF" 70 | exit 1 71 | fi 72 | else 73 | debug "Daemon file found: asd is running" 74 | fi 75 | # shellcheck source=./asd.conf 76 | . "$ASDCONF" 77 | else 78 | echo -e " ${BLD}Cannot find $ASDCONF so bailing.${NRM}" 79 | echo -e " ${BLD}Reinstall package to use anything-sync-daemon.${NRM}" 80 | exit 1 81 | fi 82 | 83 | # if asd is active, source the snapshot of /etc/asd.conf preferentially 84 | if [[ -f "${DAEMON_FILE}.conf" ]]; then 85 | debug "\nRestoring old asd.conf configurations" 86 | unset WHATTOSYNC USE_OVERLAYFS USE_BACKUPS VOLATILE 87 | ASDCONF="${DAEMON_FILE}.conf" 88 | # shellcheck source=./asd.conf 89 | . "$ASDCONF" 90 | fi 91 | 92 | debug "" # just for an additional enter 93 | 94 | # define default number of crash-recovery snapshots to save if the user did not 95 | # and check that it is an integer if user did define it 96 | if [[ -z "$BACKUP_LIMIT" ]]; then 97 | BACKUP_LIMIT=5 98 | else 99 | if [[ "$BACKUP_LIMIT" =~ ^[0-9]+$ ]]; then 100 | # correctly setup 101 | /bin/true 102 | else 103 | echo -e " ${RED}ERROR:${NRM}${BLD} Bad value for BACKUP_LIMIT detected!${NRM}" 104 | exit 1 105 | fi 106 | fi 107 | debug "Backup limit: $BACKUP_LIMIT" 108 | 109 | # saving current extended pattern matching setting 110 | # command returns non zero exit code if the option is unset hence 111 | # pipe with true 112 | previous_extglob_setting=$(shopt -p extglob || true) 113 | 114 | # ensuring pattern matching is enabled 115 | shopt -s extglob 116 | 117 | # removing any trailing slash(es) from the list of directories to sync 118 | WHATTOSYNC=("${WHATTOSYNC[@]%%+(/)}") 119 | VOLATILE="${VOLATILE%%+(/)}" 120 | 121 | # setting everything back 122 | $previous_extglob_setting 123 | unset previous_extglob_setting 124 | 125 | [[ -z "$VOLATILE" ]] && VOLATILE=/tmp 126 | debug "Volatile dir: $VOLATILE" 127 | 128 | # bail if $VOLATILE isn't tmpfs 129 | df -T "$VOLATILE" | grep -m 1 -q '\( tmpfs \|^/dev/zram\)' || { 130 | echo "$VOLATILE is not tmpfs/zram so running asd is pointless. Aborting." >&2 131 | exit 1; } 132 | 133 | [[ -z "$ENABLE_HARDLINK_SAFETY_CHECK" ]] && ENABLE_HARDLINK_SAFETY_CHECK=1 134 | debug "Hardlink safety check: $ENABLE_HARDLINK_SAFETY_CHECK" 135 | 136 | # simple function to determine user intent rather than using a null value 137 | case "${USE_OVERLAYFS,,}" in 138 | y|yes|true|t|on|1|enabled|enable|use) 139 | OLFS=1 140 | ;; 141 | *) 142 | OLFS=0 143 | ;; 144 | esac 145 | 146 | # since the default for this one is a yes, need to force a null value to yes 147 | [[ -z "${USE_BACKUPS,,}" ]] && USE_BACKUPS="yes" 148 | debug "Backups enabled: $USE_BACKUPS" 149 | 150 | case "${USE_BACKUPS,,}" in 151 | y|yes|true|t|on|1|enabled|enable|use) 152 | CRRE=1 153 | ;; 154 | *) 155 | CRRE=0 156 | ;; 157 | esac 158 | debug "Crash recovery enabled: $CRRE" 159 | 160 | # determine is we are using overlayfs (v22 and below) or overlay (v23 and above) 161 | # overlay FS v23 and later requires both an upper and a work directory, both on 162 | # the same filesystem, but not part of the same subtree. 163 | # 164 | # ubuntu 15.04 has both overlay and overlayfs so prefer version 23 165 | if [[ $OLFS -eq 1 ]]; then 166 | # first test if either module is manually loaded manually or hardcoded 167 | [[ $(grep -ciE "overlayfs$" /proc/filesystems) -eq 1 ]] && OLFSVER=22 168 | [[ $(grep -ciE "overlay$" /proc/filesystems) -eq 1 ]] && OLFSVER=23 169 | if [[ -z $OLFSVER ]]; then 170 | # since mount should call modprobe on invocation, check to see if either 171 | # module is in the tree using modinfo 172 | modinfo overlayfs &>/dev/null && OLFSVER=22 173 | modinfo overlay &>/dev/null && OLFSVER=23 174 | if [[ -z $OLFSVER ]]; then 175 | echo "overlayfs is not supported by your system, falling back to normal" 176 | OLFS=0 177 | fi 178 | fi 179 | fi 180 | debug "overlayfs enabled: $OLFS" 181 | debug "overlayfs version: $OLFSVER" 182 | 183 | # get distro name 184 | # first try os-release 185 | if [[ -f /etc/os-release ]]; then 186 | # shellcheck source=/dev/null 187 | source /etc/os-release 188 | if [[ -n "$PRETTY_NAME" ]]; then 189 | distro="$PRETTY_NAME" 190 | elif [[ -n "$NAME" ]]; then 191 | distro="$NAME" 192 | fi 193 | else 194 | # if not os-release try issue 195 | if [[ -n $(sed 's| \\.*$||' /etc/issue | head -n 1) ]]; then 196 | distro="$(sed 's| \\.*$||' /etc/issue | head -n 1)" 197 | else 198 | # fuck it 199 | distro= 200 | fi 201 | fi 202 | 203 | header() { 204 | [[ -z "$distro" ]] && echo -e "${BLD}Anything-sync-daemon v$VERS${NRM}" || 205 | echo -e "${BLD}Anything-sync-daemon v$VERS${NRM}${BLD} on $distro${NRM}" 206 | echo 207 | } 208 | 209 | dep_check() { 210 | # Function is used to ensure all dependencies are installed 211 | debug "\n${BLU}Checking dependencies${NRM}" 212 | debug "checking rsync" 213 | command -v rsync >/dev/null 2>&1 || { 214 | echo "I require rsync but it's not installed. Aborting." >&2 215 | exit 1; } 216 | debug "checking awk" 217 | command -v awk >/dev/null 2>&1 || { 218 | echo "I require awk but it's not installed. Aborting." >&2; exit 1; } 219 | debug "checking pv" 220 | command -v pv >/dev/null 2>&1 || { 221 | echo "I require pv but it's not installed. Aborting." >&2; exit 1; } 222 | debug "checking tar" 223 | command -v tar >/dev/null 2>&1 || { 224 | echo "I require tar but it's not installed. Aborting." >&2; exit 1; } 225 | debug "checking zstd" 226 | command -v zstd >/dev/null 2>&1 || { 227 | echo "I require zstd but it's not installed. Aborting." >&2; exit 1; } 228 | if [[ $OLFS -eq 1 ]]; then 229 | [[ $OLFSVER -ge 22 ]] || { 230 | echo -e " ${BLD}Your kernel requires either the ${BLU}overlay${NRM}${BLD} or ${BLU}overlayfs${NRM}${BLD} module to use${NRM}" 231 | echo -e " ${BLD}to use asd's in overlay mode. Cannot find either in your kernel so compile it in and${NRM}" 232 | echo -e " ${BLD}try again or remove the option from ${BLU}$ASDCONF${NRM}${BLD}. ${RED}Aborting!${NRM}" >&2; exit 1;} 233 | fi 234 | } 235 | 236 | config_check() { 237 | debug "\n${GRN}Checking configs${NRM}" 238 | # nothing to do if these are empty 239 | if [[ -z "${WHATTOSYNC[0]}" ]]; then 240 | echo -e " ${BLD}Must define at least one directory in ${NRM}${BLU}$ASDCONF${NRM}" 241 | exit 1 242 | fi 243 | 244 | # make sure the user defined real dirs 245 | for DIR in "${WHATTOSYNC[@]}"; do 246 | debug "DIR: $DIR" 247 | [[ ${DIR##*/} == .* ]] && BACKUP="${DIR%/*}/${DIR##*/}-backup_asd" || 248 | BACKUP="${DIR%/*}/.${DIR##*/}-backup_asd" 249 | [[ ${DIR##*/} == .* ]] && BACK_OLD="${DIR%/*}/${DIR##*/}-backup_asd-old" || 250 | BACK_OLD="${DIR%/*}/.${DIR##*/}-backup_asd-old" 251 | debug "BACKUP: $BACKUP" 252 | debug "BACK_OLD: $BACK_OLD" 253 | if [[ ! -d "$DIR" ]]; then 254 | if [[ ! -d "$BACK_OLD" ]]; then 255 | echo -e "${BLD}Bad entry in your WHATTOSYNC array detected:${NRM}" 256 | echo -e " ${BLD}${RED}$DIR${NRM}" 257 | echo -e "${BLD}Edit ${BLU}$ASDCONF${NRM}${BLD} correcting the mistake and try again.${NRM}" 258 | exit 1 259 | fi 260 | else 261 | # sanity check for hardlinks 262 | if [[ ! -d "$BACK_OLD" && $ENABLE_HARDLINK_SAFETY_CHECK -ne 0 && -n $(find "$DIR" -type f -links +1) ]]; then 263 | echo -e "$DIR:\n${RED} Presence of hardlinks found, asd might break them:${NRM}" 264 | exit 1 265 | else 266 | debug "No hardlinks found" 267 | fi 268 | fi 269 | done 270 | debug "Configs seem to be fine" 271 | } 272 | 273 | root_check() { 274 | # we call this to ensure that only the root user is calling the 275 | # function why care? both the sync and unsync functions require 276 | # root access to $DAEMON_FILE Running as unprivileged user will 277 | # fuck up the sync process resulting in unhappy users 278 | 279 | debug "\n${RED}Checking root permissions${NRM}" 280 | if [[ $EUID -ne 0 ]]; then 281 | echo -e " ${BLD}This function must be called as root!${NRM}" 1>&2 282 | exit 1 283 | fi 284 | debug "Have root permissions" 285 | } 286 | 287 | ungraceful_state_check() { 288 | # if the machine was ungracefully shutdown then the backup will be 289 | # on the filesystem and the link to tmpfs will be on the filesystem 290 | # but the contents will be empty we need to simply remove the link 291 | # and rotate the backup into place 292 | debug "\n${BLU}checking ungraceful state${NRM}" 293 | local DIR USER BACKUP TMP 294 | for DIR in "${WHATTOSYNC[@]}"; do 295 | # did user define a real dir 296 | # this is the hdd bound backup in case of power failure 297 | [[ ${DIR##*/} == .* ]] && BACKUP="${DIR%/*}/${DIR##*/}-backup_asd" || 298 | BACKUP="${DIR%/*}/.${DIR##*/}-backup_asd" 299 | [[ ${DIR##*/} == .* ]] && BACK_OLD="${DIR%/*}/${DIR##*/}-backup_asd-old" || 300 | BACK_OLD="${DIR%/*}/.${DIR##*/}-backup_asd-old" 301 | if [[ -d "$BACKUP" ]]; then 302 | USER=$(stat -c %U "$BACKUP") 303 | else 304 | USER=$(stat -c %U "$DIR") 305 | fi 306 | TMP="$VOLATILE/$ASDNAME-$USER$DIR" 307 | UPPER="$VOLATILE/$ASDNAME-$USER$DIR-rw" 308 | WORK="$VOLATILE/.$ASDNAME-$USER$DIR" 309 | debug "DIR: $DIR\nBACKUP: $BACKUP\nBACK_OLD: $BACK_OLD\nUSER: $USER\nTMP: $TMP" 310 | 311 | if [[ -e "$TMP"/.flagged || ! -d "$BACKUP" ]]; then 312 | debug "No ungraceful state detected" 313 | # all is well so continue 314 | continue 315 | else 316 | echo "Ungraceful state detected for $DIR so fixing" 317 | NOW=$(date +%Y%m%d_%H%M%S) 318 | 319 | debug "unmounting $DIR" 320 | mountpoint -q "$DIR" && umount -R -l "$DIR" 321 | debug "unmounting $BACKUP" 322 | mountpoint -q "$BACKUP" && umount -R -l "$BACKUP" && rm -rf "$BACKUP" && debug "removed $BACKUP dir" 323 | 324 | mountpoint -q "$TMP" && umount "$TMP" && rm -rf "$TMP" "$UPPER" "$WORK" && debug "unmounted overlay dirs" 325 | 326 | if [[ -d "$BACK_OLD" ]]; then 327 | if [[ $CRRE -eq 1 ]]; then 328 | debug "copying $BACK_OLD to $BACKUP-$CRASH_RECOVERY_SUFFIX-$NOW.tar.zstd" 329 | tar cf - -C "${BACK_OLD%/*}" "${BACK_OLD##*/}" | pv -s "$(du -sb "$BACK_OLD" | awk '{print $1}')" | zstd > "$BACKUP-$CRASH_RECOVERY_SUFFIX-$NOW.tar.zstd" 330 | fi 331 | rm -rf "$BACK_OLD" && debug "deleting the $BACK_OLD directory" 332 | fi 333 | fi 334 | done 335 | } 336 | 337 | cleanup() { 338 | debug "\n${BLU}Cleaning up crashrecoveries${NRM}" 339 | local DIR USER GROUP BACKUP TMP 340 | for DIR in "${WHATTOSYNC[@]}"; do 341 | # this is the hdd bound backup in case of power failure 342 | [[ ${DIR##*/} == .* ]] && BACKUP="${DIR%/*}/${DIR##*/}-backup_asd" || 343 | BACKUP="${DIR%/*}/.${DIR##*/}-backup_asd" 344 | USER=$(stat -c %U "$DIR") 345 | GROUP=$(id -g "$USER") 346 | TMP="$VOLATILE/$ASDNAME-$USER$DIR" 347 | UPPER="$VOLATILE/$ASDNAME-$USER$DIR-rw" 348 | WORK="$VOLATILE/.$ASDNAME-$USER$DIR" 349 | debug "DIR: $DIR\nUSER: $USER\nGROUP: $GROUP\nTMP: $TMP\nUPPER: $UPPER\nWORK: $WORK" 350 | 351 | mapfile -t CRASHArr < <(find "${BACKUP%/*}" -maxdepth 1 -name "${BACKUP##*/}-$CRASH_RECOVERY_SUFFIX-*" 2>/dev/null|sort -r) 352 | 353 | if [[ ${#CRASHArr[@]} -gt 0 ]]; then 354 | echo -e "${BLD}Deleting ${#CRASHArr[@]} crashrecovery dir(s) for sync target ${BLU}$DIR${NRM}" 355 | for backup in "${CRASHArr[@]}"; do 356 | echo -e "${BLD}${RED} $backup${NRM}" 357 | debug "removing $backup" 358 | rm -rf "$backup" 359 | done 360 | unset CRASHArr 361 | else 362 | echo -e "${BLD}Found no crashrecovery dirs for: ${BLU}$DIR${NRM}${BLD}${NRM}" 363 | fi 364 | echo 365 | done 366 | } 367 | 368 | enforce() { 369 | debug "\n${BLU}Enforcing number of backups${NRM}" 370 | local DIR BACKUP 371 | for DIR in "${WHATTOSYNC[@]}"; do 372 | # this is the hdd bound backup in case of power failure 373 | [[ ${DIR##*/} == .* ]] && BACKUP="${DIR%/*}/${DIR##*/}-backup_asd" || 374 | BACKUP="${DIR%/*}/.${DIR##*/}-backup_asd" 375 | debug "DIR: $DIR\nBACKUP: $BACKUP" 376 | mapfile -t CRASHArr < <(find "${BACKUP%/*}" -maxdepth 1 -name "${BACKUP##*/}-$CRASH_RECOVERY_SUFFIX-*" 2>/dev/null|sort -r) 377 | 378 | if [[ ${#CRASHArr[@]} -gt $BACKUP_LIMIT ]]; then 379 | debug "The backups are greater than $BACKUP_LIMIT" 380 | for remove in "${CRASHArr[@]:$BACKUP_LIMIT}"; do 381 | debug "removing $remove" 382 | rm -rf "$remove" 383 | done 384 | else 385 | debug "The backups are less than $BACKUP_LIMIT, nothing to do" 386 | fi 387 | unset CRASHArr 388 | done 389 | } 390 | 391 | do_sync() { 392 | debug "\n${GRN}Syncing files${NRM}" 393 | 394 | # make a snapshot of /etc/asd.conf and redefine its location to tmpfs while 395 | # asd is running to keep any edits made to the live /etc/asd.conf from 396 | # potentially orphaning the tmpfs copies thus preserving the data 397 | [[ ! -f "${DAEMON_FILE}.conf" ]] && cp "$ASDCONF" "${DAEMON_FILE}.conf" && debug "copied $ASDCONF to ${DAEMON_FILE}.conf" 398 | 399 | # sync to tmpfs and back again 400 | local DIR USER GROUP BACKUP TMP 401 | for DIR in "${WHATTOSYNC[@]}"; do 402 | # this is the hdd bound backup in case of power failure 403 | [[ ${DIR##*/} == .* ]] && BACKUP="${DIR%/*}/${DIR##*/}-backup_asd" || 404 | BACKUP="${DIR%/*}/.${DIR##*/}-backup_asd" 405 | [[ ${DIR##*/} == .* ]] && BACK_OLD="${DIR%/*}/${DIR##*/}-backup_asd-old" || 406 | BACK_OLD="${DIR%/*}/.${DIR##*/}-backup_asd-old" 407 | USER=$(stat -c %U "$DIR") 408 | GROUP=$(id -g "$USER") 409 | TMP="$VOLATILE/$ASDNAME-$USER$DIR" 410 | UPPER="$VOLATILE/$ASDNAME-$USER$DIR-rw" 411 | WORK="$VOLATILE/.$ASDNAME-$USER$DIR" 412 | debug "\nDIR: $DIR\nUSER: $USER\nGROUP: $GROUP\nTMP: $TMP\nUPPER: $UPPER\nWORK: $WORK\n" 413 | 414 | # make tmpfs container 415 | if [[ -d "$DIR" ]]; then 416 | # retain permissions on sync target 417 | PREFIXP=$(stat -c %a "$DIR") 418 | [[ -r "$TMP" ]] || install -dm"$PREFIXP" --owner="$USER" --group="$GROUP" "$TMP" 419 | [[ -r "$BACKUP" ]] || install -dm"$PREFIXP" --owner="$USER" --group="$GROUP" "$BACKUP" 420 | 421 | if [[ $OLFS -eq 1 ]]; then 422 | debug "ensuring overlay directories" 423 | if [[ $OLFSVER -eq 23 ]]; then 424 | [[ -r "$UPPER" ]] || install -dm"$PREFIXP" --owner="$USER" --group="$GROUP" "$UPPER" 425 | [[ -r "$WORK" ]] || install -dm"$PREFIXP" --owner="$USER" --group="$GROUP" "$WORK" 426 | elif [[ $OLFSVER -eq 22 ]]; then 427 | [[ -r "$UPPER" ]] || install -dm"$PREFIXP" --owner="$USER" --group="$GROUP" "$UPPER" 428 | fi 429 | fi 430 | 431 | # sync the tmpfs targets to the disc 432 | if [[ -e "$TMP"/.flagged ]]; then 433 | debug "Syncing $TMP and $BACKUP" 434 | # don't do inplace sync 435 | set -x 436 | rsync -aX --delete-after --exclude .flagged "$TMP/" "$BACKUP/" --info=progress2 437 | set +x 438 | else 439 | # backup target and link to tmpfs container 440 | debug "Bind mounting $DIR -> $BACKUP" 441 | mount --rbind --make-private -o noatime "$DIR" "$BACKUP" 442 | if [[ $CRRE -eq 1 ]];then 443 | debug "Creating new linked backup directory $BACK_OLD" 444 | tempfile=$(mktemp) 445 | 446 | set -x 447 | # this copies all the files 448 | find "$BACKUP" -type l -printf '%P\n' > "$tempfile" 449 | rm -rf "$BACK_OLD" && rsync -aX --no-links --link-dest="$DIR" "$DIR/" "$BACK_OLD/" --info=progress2 450 | 451 | # this is used to handle the symlinks 452 | rsync -aXl --files-from="$tempfile" "$DIR/" "$BACK_OLD/" --info=progress2 453 | 454 | set +x 455 | rm "$tempfile" 456 | fi 457 | 458 | # initial sync 459 | if [[ $OLFS -eq 1 ]]; then 460 | debug "Mounting overlay directory" 461 | if [[ $OLFSVER -eq 23 ]]; then 462 | mount -t overlay overlaid -olowerdir="$BACKUP",upperdir="$UPPER",workdir="$WORK" "$TMP" 463 | elif [[ $OLFSVER -eq 22 ]]; then 464 | mount -t overlayfs overlaid -olowerdir="$BACKUP",upperdir="$UPPER" "$TMP" 465 | fi 466 | else 467 | debug "Doing initial sync with $BACKUP and $TMP" 468 | set -x 469 | rsync -aXl --append "$BACKUP/" "$TMP/" --info=progress2 470 | set +x 471 | fi 472 | debug "bind mounting $TMP -> $DIR" 473 | mount --rbind --make-private -o noatime "$TMP" "$DIR" 474 | 475 | touch "$TMP"/.flagged 476 | fi 477 | fi 478 | done 479 | echo -e "${BLD}Sync successful${NRM}" 480 | 481 | debug "creating $DAEMON_FILE" 482 | touch "$DAEMON_FILE" 483 | } 484 | 485 | do_unsync() { 486 | debug "\n${RED}Unsyncing files${NRM}" 487 | 488 | local DIR USER BACKUP TMP 489 | for DIR in "${WHATTOSYNC[@]}"; do 490 | # this is the hdd bound backup in case of power failure 491 | [[ ${DIR##*/} == .* ]] && BACKUP="${DIR%/*}/${DIR##*/}-backup_asd" || 492 | BACKUP="${DIR%/*}/.${DIR##*/}-backup_asd" 493 | [[ ${DIR##*/} == .* ]] && BACK_OLD="${DIR%/*}/${DIR##*/}-backup_asd-old" || 494 | BACK_OLD="${DIR%/*}/.${DIR##*/}-backup_asd-old" 495 | USER=$(stat -c %U "$DIR") 496 | GROUP=$(id -g "$USER") 497 | TMP="$VOLATILE/$ASDNAME-$USER$DIR" 498 | UPPER="$VOLATILE/$ASDNAME-$USER$DIR-rw" 499 | WORK="$VOLATILE/.$ASDNAME-$USER$DIR" 500 | debug "DIR: $DIR\nUSER: $USER\nGROUP: $GROUP\nTMP: $TMP\nUPPER: $UPPER\nWORK: $WORK\n" 501 | 502 | # remove link and move data from tmpfs to disk 503 | if mountpoint -q "$DIR"; then 504 | # this assumes that the backup is always 505 | # updated so be sure to invoke a sync before an unsync 506 | debug "unmounting $DIR" 507 | umount -R -f -l "$DIR" 508 | debug "unmounting $BACKUP" 509 | umount -R -f -l "$BACKUP" && rm -rf "$BACKUP" && debug "removing $BACKUP" 510 | 511 | if [[ $OLFS -eq 1 ]] && mountpoint -q "$TMP"; then 512 | umount -l "$TMP" && debug "unmount $TMP" 513 | rm -rf "$TMP" "$UPPER" "$WORK" && debug "removing overlayfs folders" 514 | else 515 | [[ -d "$TMP" ]] && rm -rf "$TMP" && debug "removing $TMP" 516 | fi 517 | [[ $CRRE -eq 1 ]] && rm -rf "$BACK_OLD" && debug "removing $BACK_OLD" 518 | fi 519 | done 520 | 521 | # delete daemon file in the end, so that unsync can be run again 522 | # incase of some failure midway 523 | # since unsync also requires sync before, if any of the DIRS are unsynced during last run, 524 | # they will be synced again before unsyncing not leading to any breakage 525 | 526 | debug "Removing $DAEMON_FILE and ${DAEMON_FILE}.conf" 527 | rm -f "$DAEMON_FILE" "${DAEMON_FILE}.conf" 528 | 529 | echo -e "${BLD}Unsync successful${NRM}" 530 | } 531 | 532 | parse() { 533 | if [[ -f /usr/lib/systemd/system/asd.service ]]; then 534 | # running systemd 535 | asd_state=$(systemctl show -p ActiveState --value asd) 536 | resync_state=$(systemctl show -p ActiveState --value asd-resync.timer) 537 | [[ "$asd_state" = "active" ]] && asd_color="${GRN}" || asd_color="${RED}" 538 | [[ "$resync_state" = "active" ]] && resync_color="${GRN}" || resync_color="${RED}" 539 | echo -e " ${BLD}Systemd service is currently ${asd_color}$asd_state${NRM}${BLD}.${NRM}" 540 | echo -e " ${BLD}Systemd resync service is currently ${resync_color}$resync_state${NRM}${BLD}.${NRM}" 541 | else 542 | # using other init system + cron job for resync 543 | [[ -x /etc/cron.hourly/asd-update ]] && resync_state="present" || resync_state="not present" 544 | [[ "$resync_state" = "present" ]] && resync_color="${GRN}" || resync_color="${RED}" 545 | echo -e " ${BLD}Daemon pid file is $([[ -f $DAEMON_FILE ]] && 546 | echo -e "${GRN}"present"${NRM}""${BLD}" || echo -e "${RED}"not present"${NRM}""${BLD}").${NRM}" 547 | echo -e " ${BLD}Resync cronjob is ${resync_color}${resync_state}${NRM}${BLD}.${NRM}" 548 | fi 549 | [[ $OLFS -eq 1 ]] && 550 | echo -e "${BLD} Overlayfs v$OLFSVER is currently ${GRN}active${NRM}${BLD}.${NRM}" || 551 | echo -e "${BLD} Overlayfs technology is currently ${RED}inactive${NRM}${BLD}.${NRM}" 552 | echo 553 | echo -e "${BLD}Asd will manage the following per ${BLU}${ASDCONF}${NRM}${BLD} settings:${NRM}" 554 | echo 555 | local DIR USER GROUP BACKUP TMP 556 | for DIR in "${WHATTOSYNC[@]}"; do 557 | # this is the hdd bound backup in case of power failure 558 | [[ ${DIR##*/} == .* ]] && BACKUP="${DIR%/*}/${DIR##*/}-backup_asd" || 559 | BACKUP="${DIR%/*}/.${DIR##*/}-backup_asd" 560 | USER=$(stat -c %U "$DIR") 561 | GROUP=$(id -g "$USER") 562 | TMP="$VOLATILE/$ASDNAME-$USER$DIR" 563 | UPPER="$VOLATILE/$ASDNAME-$USER$DIR-rw" 564 | WORK="$VOLATILE/.$ASDNAME-$USER$DIR" 565 | 566 | # sync target dir size 567 | if [[ -d "$DIR" ]]; then 568 | echo -en " ${BLD}owner/group id:" 569 | echo -e "$(tput cr)""$(tput cuf 20)" "$USER/$GROUP${NRM}" 570 | echo -en " ${BLD}target to manage:" 571 | echo -e "$(tput cr)""$(tput cuf 20)" "${GRN}$DIR${NRM}" 572 | echo -en " ${BLD}sync target:" 573 | echo -e "$(tput cr)""$(tput cuf 20)" "${BLU}$BACKUP${NRM}" 574 | echo -en " ${BLD}tmpfs target:" 575 | echo -e "$(tput cr)""$(tput cuf 20)" "${RED}$TMP${NRM}" 576 | echo -en " ${BLD}dir size:" 577 | psize=$(du -Dh --max-depth=0 "$DIR" 2>/dev/null | awk '{ print $1 }') 578 | echo -e "$(tput cr)$(tput cuf 20) $psize${NRM}" 579 | if [[ $OLFS -eq 1 ]]; then 580 | rwsize=$(du -Dh --max-depth=0 "$UPPER" 2>/dev/null | awk '{ print $1 }') 581 | echo -en " ${BLD}overlayfs size:" 582 | echo -e "$(tput cr)$(tput cuf 20) $rwsize${NRM}" 583 | fi 584 | echo -en " ${BLD}recovery dirs:" 585 | mapfile -t CRASHArr < <(find "${BACKUP%/*}" -maxdepth 1 -name "${BACKUP##*/}-$CRASH_RECOVERY_SUFFIX-*" 2>/dev/null|sort -r) 586 | if [[ "${#CRASHArr[@]}" -eq 0 ]]; then 587 | echo -e "$(tput cr)$(tput cuf 20) none${NRM}" 588 | else 589 | echo -e "$(tput cr)$(tput cuf 20) ${RED}${#CRASHArr[@]}${NRM}${BLD} <- delete with the c option${NRM}" 590 | for backup in "${CRASHArr[@]}"; do 591 | psize=$(du -Dh --max-depth=0 "$backup" 2>/dev/null | awk '{ print $1 }') 592 | echo -en " ${BLD} dir path/size:" 593 | echo -e "$(tput cr)$(tput cuf 20) ${BLU}$backup ${NRM}${BLD}($psize)${NRM}" 594 | done 595 | fi 596 | unset CRASHArr 597 | echo 598 | fi 599 | done 600 | } 601 | 602 | case "$1" in 603 | p|P|Parse|parse|Preview|preview|debug) 604 | header 605 | dep_check 606 | config_check 607 | parse 608 | ;; 609 | c|C|clean|Clean) 610 | header 611 | dep_check 612 | config_check 613 | cleanup 614 | ;; 615 | sync) 616 | if [[ ! -f "$DAEMON_FILE" ]]; then 617 | root_check 618 | dep_check 619 | config_check 620 | ungraceful_state_check 621 | do_sync 622 | enforce 623 | else 624 | echo -e "${GRN}ASD is already running${NRM}" 625 | fi 626 | ;; 627 | resync) 628 | if [[ -f "$DAEMON_FILE" ]]; then 629 | root_check 630 | ungraceful_state_check 631 | do_sync 632 | else 633 | echo -e "${RED}ASD is not running${NRM}" 634 | fi 635 | ;; 636 | unsync) 637 | # make sure the daemon ran to setup the links 638 | if [[ -f "$DAEMON_FILE" ]]; then 639 | root_check 640 | ungraceful_state_check 641 | do_sync 642 | do_unsync 643 | else 644 | echo -e "${RED}ASD is not running${NRM}" 645 | fi 646 | ;; 647 | *) 648 | echo -e "${BLD}Anything-sync-daemon v$VERS${NRM}" 649 | echo 650 | echo -e " ${BLD}$0 ${NRM}${GRN}[option]${NRM}" 651 | echo -e " ${BLD} ${NRM}${GRN}preview${NRM}${BLD} Parse config file (${NRM}${BLU}${ASDCONF}${NRM}${BLD}) to see what will be managed.${NRM}" 652 | echo -e " ${BLD} ${NRM}${GRN}clean${NRM}${BLD} Clean (delete without prompting) ALL crashrecovery dirs.${NRM}" 653 | echo -e " ${BLD} ${NRM}${GRN}resync${NRM}${BLD} Synchronize the tmpfs and media bound copy. Must be run as root user.${NRM}" 654 | echo -e " ${BLD} ${NRM}${RED}sync${NRM}${BLD} Force a manual sync. Must be run as root user and NOT recommended.${NRM}" 655 | echo -e " ${BLD} ${NRM}${RED}unsync${NRM}${BLD} Force a manual unsync. Must be run as root user and NOT recommended.${NRM}" 656 | echo 657 | echo -e " ${BLD}It is ${RED}HIGHLY DISCOURAGED${NRM}${BLD} to directly call $0 to sync or to unsync.${NRM}" 658 | if [[ -f /usr/lib/systemd/system/asd.service ]]; then 659 | echo -e " ${BLD}Instead, use systemd to start/stop anything-sync-daemon.${NRM}" 660 | echo 661 | echo -e " ${BLD}systemctl ${NRM}${GRN}[option]${NRM}${BLD} asd asd-resync${NRM}" 662 | echo -e " ${BLD} ${NRM}${GRN}start${NRM}${BLD} Turn on daemon; make symlinks and actively manage targets in tmpfs.${NRM}" 663 | echo -e " ${BLD} ${NRM}${GRN}stop${NRM}${BLD} Turn off daemon; remove symlinks and rotate tmpfs data back to disc.${NRM}" 664 | echo -e " ${BLD} ${NRM}${GRN}enable${NRM}${BLD} Autostart daemon when system comes up.${NRM}" 665 | echo -e " ${BLD} ${NRM}${GRN}disable${NRM}${BLD} Remove daemon from the list of autostart daemons.${NRM}" 666 | elif [[ -f /etc/init.d/asd ]]; then 667 | echo -e " ${BLD}Instead, use the init system to start/stop anything-sync-daemon.${NRM}" 668 | echo 669 | echo -e " ${BLD}sudo service asd ${NRM}${GRN}[option]${NRM}${BLD} or /etc/init.d/asd ${NRM}${GRN}[option]${NRM}" 670 | echo -e " ${BLD} ${NRM}${GRN}start${NRM}${BLD} Turn on daemon; make symlinks and actively manage targets in tmpfs.${NRM}" 671 | echo -e " ${BLD} ${NRM}${GRN}stop${NRM}${BLD} Turn off daemon; remove symlinks and rotate tmpfs data back to disc.${NRM}" 672 | fi 673 | ;; 674 | esac 675 | exit 0 676 | 677 | # vim:set ts=2 sw=2 et: 678 | --------------------------------------------------------------------------------