33 | function program_version_number {
34 | grep -Em 1 -e '\$version *=.*v[0-9]+\.[0-9]+\.?[0-9]*.*;' $1 | \
35 | grep -Eom 1 -e 'v[0-9]+\.[0-9]+\.?[0-9]*'
36 | }
37 |
38 | echo "Tagging `latest_version_number`"
39 |
40 | if [[ -n `git diff` ]]; then
41 | echo "We have uncommitted changes."
42 | git status
43 | echo "Aborting."
44 | exit
45 | fi;
46 |
47 | if [[ `program_version_number ../monikop` != `latest_version_number` ]]; then
48 | echo "Version number mismatch between monikop and NEWS. Aborting."
49 | exit
50 | elif [[ `program_version_number ../pokinom` != `latest_version_number` ]]; then
51 | echo "Version number mismatch between pokinom and NEWS. Aborting."
52 | exit
53 | fi
54 |
55 | if ! git tag -a -m "`latest_NEWS_section`" `latest_version_number`; then
56 | echo "Setting tag `latest_version_number` failed. But maybe things are already in place."
57 | else
58 | echo "Tagging `latest_version_number` successful."
59 | fi
60 |
61 | if [[ `git describe $(latest_version_number)` != `latest_version_number` ]]; then
62 | echo "Tag `latest_version_number` missing. Aborting."
63 | exit
64 | fi
65 |
66 | exit
67 |
--------------------------------------------------------------------------------
/test/monikop.config.test.1:
--------------------------------------------------------------------------------
1 | # -*- perl -*-
2 | #############################################################
3 | # Monikop Settings
4 | #
5 | # Copy this file to monikop.config and adapt it to your needs.
6 | ##############################################################
7 | # Possible data sources, and by what directory name to represent them in
8 | # destination.
9 | # When the latter is not unique, care must be taken that all pathnames in the
10 | # respective sources are unique.
11 | %sources = (
12 | 'rsync://localhost:2000/test_01/data' => '',
13 | 'rsync://localhost:2000/test_02/data' => '',
14 | );
15 |
16 | # Possible mount points of data destinations.
17 | @usable_mount_points = (
18 | '/tmp/monikop-test/mnt/03',
19 | '/tmp/monikop-test/mnt/04',
20 | );
21 |
22 | # Common directory (under a mount point) to put new data in.
23 | # Must agree with Pokinom's setting.
24 | $path_under_mount_point =
25 | 'measuring_data'
26 | ;
27 |
28 | # Directories (under any mount point) of this name will be deleted
29 | # Must agree with Pokinom's setting.
30 | $path_under_mount_point_backed_up =
31 | 'backed_up'
32 | ;
33 |
34 | # Directory name (under a mount point) while being deleted.
35 | # Must agree with Pokinom's setting.
36 | $path_under_mount_point_being_deleted =
37 | 'being_deleted'
38 | ;
39 |
40 | # Path and file name prefix of rsync's raw logs:
41 | $rsync_log_prefix =
42 | '/tmp/monikop-test/log/log.'
43 | ;
44 |
45 | # Path and file name prefix of the list of successfully rsynced files:
46 | $finished_prefix =
47 | '/tmp/monikop-test/log/finished.'
48 | ;
49 |
50 | # How to suffix the name of the duplicate of a safe file:
51 | $safe_file_backup_suffix =
52 | '.bak'
53 | ;
54 |
55 | # How to suffix the name of an unfinished safe file:
56 | $safe_file_unfinished_suffix =
57 | '.unfinished'
58 | ;
59 |
60 | # What to do (shutdown) when F3 has been pressed:
61 | $key_f3_action =
62 | "sudo halt -p"
63 | ;
64 |
65 | # What to do (reboot) when F6 has been pressed:
66 | $key_f6_action =
67 | "sudo reboot"
68 | ;
69 |
70 | # Rsyncs time (in seconds) to wait for a response:
71 | $rsync_timeout =
72 | 30
73 | ;
74 |
75 | # Rsyncs directory (relative to destination) for partially transferred files.
76 | # Must agree with Pokinom's setting.
77 | $rsync_partial_dir_name =
78 | '.rsync_partial'
79 | ;
80 |
--------------------------------------------------------------------------------
/test/monikop.config.test.3:
--------------------------------------------------------------------------------
1 | # -*- perl -*-
2 | #############################################################
3 | # Monikop Settings
4 | #
5 | # Copy this file to monikop.config and adapt it to your needs.
6 | ##############################################################
7 | # Possible data sources, and by what directory name to represent them in
8 | # destination.
9 | # When the latter is not unique, care must be taken that all pathnames in the
10 | # respective sources are unique.
11 | %sources = (
12 | 'rsync://localhost:2000/test_01/data' => 'dir_01',
13 | 'rsync://localhost:2000/test_02/data' => 'dir_02',
14 | );
15 |
16 | # Possible mount points of data destinations.
17 | @usable_mount_points = (
18 | '/tmp/monikop-test/mnt/03',
19 | '/tmp/monikop-test/mnt/04',
20 | );
21 |
22 | # Common directory (under a mount point) to put new data in.
23 | # Must agree with Pokinom's setting.
24 | $path_under_mount_point =
25 | 'measuring_data'
26 | ;
27 |
28 | # Directories (under any mount point) of this name will be deleted
29 | # Must agree with Pokinom's setting.
30 | $path_under_mount_point_backed_up =
31 | 'backed_up'
32 | ;
33 |
34 | # Directory name (under a mount point) while being deleted.
35 | # Must agree with Pokinom's setting.
36 | $path_under_mount_point_being_deleted =
37 | 'being_deleted'
38 | ;
39 |
40 | # Path and file name prefix of rsync's raw logs:
41 | $rsync_log_prefix =
42 | '/tmp/monikop-test/log/log.'
43 | ;
44 |
45 | # Path and file name prefix of the list of successfully rsynced files:
46 | $finished_prefix =
47 | '/tmp/monikop-test/log/finished.'
48 | ;
49 |
50 | # How to suffix the name of the duplicate of a safe file:
51 | $safe_file_backup_suffix =
52 | '.bak'
53 | ;
54 |
55 | # How to suffix the name of an unfinished safe file:
56 | $safe_file_unfinished_suffix =
57 | '.unfinished'
58 | ;
59 |
60 | # What to do (shutdown) when F3 has been pressed:
61 | $key_f3_action =
62 | "sudo halt -p"
63 | ;
64 |
65 | # What to do (reboot) when F6 has been pressed:
66 | $key_f6_action =
67 | "sudo reboot"
68 | ;
69 |
70 | # Rsyncs time (in seconds) to wait for a response:
71 | $rsync_timeout =
72 | 30
73 | ;
74 |
75 | # Rsyncs directory (relative to destination) for partially transferred files.
76 | # Must agree with Pokinom's setting.
77 | $rsync_partial_dir_name =
78 | '.rsync_partial'
79 | ;
80 |
--------------------------------------------------------------------------------
/test/monikop.config.test.2:
--------------------------------------------------------------------------------
1 | # -*- perl -*-
2 | #############################################################
3 | # Monikop Settings
4 | #
5 | # Copy this file to monikop.config and adapt it to your needs.
6 | ##############################################################
7 | # Possible data sources, and by what directory name to represent them in
8 | # destination.
9 | # When the latter is not unique, care must be taken that all pathnames in the
10 | # respective sources are unique.
11 | %sources = (
12 | 'rsync://localhost:2000/test_01/data' => '',
13 | 'rsync://localhost:2000/test_02/data' => '',
14 | );
15 |
16 | # Possible mount points of data destinations. Must be unique.
17 | @usable_mount_points = (
18 | '/tmp/monikop-test/mnt/03',
19 | '/tmp/monikop-test/mnt/04',
20 | '/tmp/monikop-test/mnt/05',
21 | );
22 |
23 | # Common directory (under a mount point) to put new data in.
24 | # Must agree with Pokinom's setting.
25 | $path_under_mount_point =
26 | 'measuring_data'
27 | ;
28 |
29 | # Directories (under any mount point) of this name will be deleted
30 | # Must agree with Pokinom's setting.
31 | $path_under_mount_point_backed_up =
32 | 'backed_up'
33 | ;
34 |
35 | # Directory name (under a mount point) while being deleted.
36 | # Must agree with Pokinom's setting.
37 | $path_under_mount_point_being_deleted =
38 | 'being_deleted'
39 | ;
40 |
41 | # Path and file name prefix of rsync's raw logs:
42 | $rsync_log_prefix =
43 | '/tmp/monikop-test/log/log.'
44 | ;
45 |
46 | # Path and file name prefix of the list of successfully rsynced files:
47 | $finished_prefix =
48 | '/tmp/monikop-test/log/finished.'
49 | ;
50 |
51 | # How to suffix the name of the duplicate of a safe file:
52 | $safe_file_backup_suffix =
53 | '.bak'
54 | ;
55 |
56 | # How to suffix the name of an unfinished safe file:
57 | $safe_file_unfinished_suffix =
58 | '.unfinished'
59 | ;
60 |
61 | # What to do (shutdown) when F3 has been pressed:
62 | $key_f3_action =
63 | "sudo halt -p"
64 | ;
65 |
66 | # What to do (reboot) when F6 has been pressed:
67 | $key_f6_action =
68 | "sudo reboot"
69 | ;
70 |
71 | # Rsyncs time (in seconds) to wait for a response:
72 | $rsync_timeout =
73 | 30
74 | ;
75 |
76 | # Rsyncs directory (relative to destination) for partially transferred files.
77 | # Must agree with Pokinom's setting.
78 | $rsync_partial_dir_name =
79 | '.rsync_partial'
80 | ;
81 |
--------------------------------------------------------------------------------
/monikop.config.example:
--------------------------------------------------------------------------------
1 | # (This is -*- perl -*- code.)
2 | ##############################################################################
3 | # Monikop's configuration file.
4 | #
5 | # Copy this file to `monikop.config' and adapt it to your needs.
6 | ##############################################################################
7 |
8 | # Possible data Sources, and by what directory name to represent them in
9 | # Destination.
10 | # When the latter is not unique, care must be taken that all pathnames in the
11 | # respective Sources are unique, or files will overwrite each other in
12 | # unpredictable ways.
13 | %sources = (
14 | 'data_producer1::data' => 'p1_dir',
15 | 'data_producer2::data' => 'p2_dir',
16 | 'data_producer3::data' => '',
17 | 'data_producer4::data' => '',
18 | );
19 |
20 | # Possible mount points of the removable disks.
21 | @usable_mount_points = (
22 | '/media/disk_1',
23 | '/media/disk_2',
24 | '/media/disk_3',
25 | );
26 |
27 | # Common directory (under a mount point) to put new data in.
28 | # Must agree with Pokinom's setting.
29 | $path_under_mount_point =
30 | 'measuring_data'
31 | ;
32 |
33 | # Directories (under any mount point) of this name will be deleted by Monikop.
34 | # Must agree with Pokinom's setting.
35 | $path_under_mount_point_backed_up =
36 | 'backed_up'
37 | ;
38 |
39 | # Directory name (under a mount point) while being deleted.
40 | # Must agree with Pokinom's setting.
41 | $path_under_mount_point_being_deleted =
42 | 'being_deleted'
43 | ;
44 |
45 | # Path and file name prefix for rsync's raw logs:
46 | $rsync_log_prefix =
47 | '~/log/monikop/log.'
48 | ;
49 |
50 | # Path and file name prefix for the list of successfully rsynced files:
51 | $finished_prefix =
52 | '~/log/monikop/finished.'
53 | ;
54 |
55 | # Safe files are supposed to survive power cuts during write operations.
56 | # How to suffix the name of the duplicate of a safe file:
57 | $safe_file_backup_suffix =
58 | '.bak'
59 | ;
60 |
61 | # How to suffix the name of an unfinished safe file:
62 | $safe_file_unfinished_suffix =
63 | '.unfinished'
64 | ;
65 |
66 | # What to do (shutdown) when F3 has been pressed:
67 | #$key_f3_action =
68 | # "sudo halt -p"
69 | # ;
70 |
71 | # What to do (shutdown) when F3 has been pressed (on a systemd-based system):
72 | $key_f3_action =
73 | "systemctl poweroff"
74 | ;
75 |
76 | # What to do (reboot) when F6 has been pressed:
77 | #$key_f6_action =
78 | # "sudo reboot"
79 | # ;
80 |
81 | # What to do (reboot) when F6 has been pressed (on a systemd-based system):
82 | $key_f6_action =
83 | "systemctl reboot"
84 | ;
85 |
86 | # Rsync's time (in seconds) to wait for a response. This is roughly the time
87 | # Monikop needs to notice the disappearance of a Source. Must not be 0.
88 | $rsync_timeout =
89 | 30
90 | ;
91 |
92 | # Rsync's directory (relative to mount point of removable disk) for partially
93 | # transferred files.
94 | # Must agree with Pokinom's setting. Make sure your payload data does not
95 | # contain an equally-named directory.
96 | $rsync_partial_dir_name =
97 | '.rsync_partial'
98 | ;
99 |
--------------------------------------------------------------------------------
/doc/usage.muse:
--------------------------------------------------------------------------------
1 | #title Monikop (and Pokinom)
2 | #subtitle rsync between unconnected hosts
3 | #author Bert Burgemeister
4 |
5 | * Usage
6 |
7 | Both Monikop and Pokinom will create automatically any directories they need.
8 |
9 | ** Monikop
10 |
11 | Put removable disks into Monikop's host on Rover and switch it on. Immediately,
12 | Monikop starts pulling data from Sources it can reach. Monikop will
13 | notice additional Sources that become reachable later and will start
14 | pulling data there as well.
15 |
16 | For each Source, Monikop keeps starting over to see if there is new
17 | data. Only Monikop's shutdown or the disappearance of the data Source
18 | will end this cycle.
19 |
20 | One removable disk is sufficient for Monikop's correct
21 | operation, but if speed is important, putting in as many disks as
22 | there are data Sources may be beneficial as Monikop uses them in parallel.
23 |
24 | To end a session, press [F3] to shut down Monikop, and remove the
25 | disks. Monikop's display shows which disks are not yet used so you
26 | can avoid carrying empty disks around.
27 |
28 |
29 |
30 | ** Pokinom
31 |
32 | Put removable disks into Pokinom's host in office and switch it
33 | on. Immediately, Pokinom starts pushing data to Destination.
34 | Interrupting this by shutting down Pokinom early is not a problem as
35 | long as it is later given the opportunity to finish. Otherwise files,
36 | even those already copied to
37 | Destination, won't be deleted by Monikop from their removable disks
38 | during the next cycle.
39 |
40 | Press [F9] to toggle whether or not you want Pokinom to shut down
41 | when finished.
42 |
43 | File permissions in Destination's receiving directory must not be
44 | changed in a way that prevents the rsync server from modifying.
45 | Best practice is to move anything out of this directory prior to any
46 | processing.
47 |
48 | Pokinom needs a sufficient amount of free disk space on Destination; it
49 | must be rebooted once this temporarily hasn't been the case.
50 |
51 |
52 | ; TODO: data transferred, but to ignore (probably none)
53 |
54 |
55 | ; TODO:
56 | ; ** fsck-pokinom
57 | ; - fsck-pokinom: to be run by root
58 |
59 |
60 | ** Crash Recovery
61 |
62 | Removable disks may get lost before they reach Destination, or
63 | Destination may crash shortly after receiving fresh data. The
64 | following may help in these cases.
65 |
66 | *** Data Recovery from Source(s) on Rover
67 |
68 | On Monikop's host, stop Monikop and delete the log files whose
69 | directory and name prefix is set in [[installation#monikop.config][monikop.config]] by
70 | =$rsync_log_prefix= and =$finished_prefix=, and whose names resemble the
71 | Source they belong to.
72 |
73 | On next startup, Monikop will pull all data from this Source again.
74 |
75 | *** Data Loss on Destination: Recovery from Removable Disks
76 |
77 | Data on removable disks are deleted not until the disk is
78 | finished by Pokinom and re-inserted in
79 | Monikop. (Non-)deletability is expressed by directory names defined
80 | in both [[installation#monikop.config][monikop.config]] and
81 | [[installation#pokinom.config][pokinom.config]]:
82 | - =$path_under_mount_point= sets the name of a directory fresh data
83 | reside in on each removable disk. Once finished by Pokinom, it
84 | is renamed into the name set by
85 | - =$path_under_mount_point_backed_up=. You can simply rename it
86 | back and Pokinom will push its content to Destination again.
87 | If you don't, Monikop will rename it into the name set by
88 | - =$path_under_mount_point_being_deleted= as soon as it sees it,
89 | and start deleting it while a new =$path_under_mount_point= is
90 | created and filled with fresh data.
91 |
92 | *** Disk Failure
93 |
94 | Suppose the system reports disk error on (say) /dev/sdb1. What is it's
95 | label?
96 |
97 |
98 | ls -l /dev/disk/by-label
99 |
100 |
101 | shows the mapping.
102 |
103 |
104 | * Bugs
105 |
106 | - Monikop and Pokinom allow files on Sources to change at any time
107 | and will reflect such changes in Destination. As a downside,
108 | Monikop and Pokinom are unable to tell whether a file is transferred
109 | completely. You should be able to assert completeness of your
110 | files by other means.
111 | - Empty directories on Sources are being ignored.
112 | - Any directories on Sources whose names conflict with the setting
113 | =$rsync_partial_dir_name= in [[installation#monikop][monikop.config]] are being ignored.
114 | - Deletions on Sources won't propagate to Destination.
115 | - For user information on progress, both Monikop and Pokinom rely on
116 | Rsync's output which is not always reliable as to the total number
117 | of files.
118 | - During copying, occasionally obsolete versions of a file may
119 | temporarily appear on Destination. This can happen with files that
120 | have grown bigger after having been copied already.
121 | - Frequent power cuts (as opposed to normal shutdown operations) may
122 | compromise efficiency in terms of disk usage.
123 | - By running multiple instances of Rsync, Monikop puts considerable
124 | strain on the system. This may reveal previously unnoticed
125 | hardware faults.
126 |
--------------------------------------------------------------------------------
/doc/build-html.el:
--------------------------------------------------------------------------------
1 | ;;;; Make html files from .muse files in current dir, put them into ../html/
2 |
3 | ;; (color-theme-whateveryouwant)
4 | (require 'muse-mode)
5 | (require 'muse-html)
6 | (require 'muse-project)
7 | (setq muse-html-table-attributes " class=\"muse-table\" border=\"1\" cellpadding=\"5\"")
8 | (setq muse-colors-inline-image-method #'muse-colors-use-publishing-directory)
9 | (muse-derive-style "xhtml-plainandsimple" "xhtml1.1"
10 | :header "
11 | (muse-html-encoding)\"?>
12 |
13 |
14 |
15 | <lisp>(muse-publishing-directive \"title\")</lisp>
16 | (muse-publishing-directive \"title\")\" />
17 | (muse-publishing-directive \"author\")\" />
18 | (muse-publishing-directive \"author\")\" />
19 |
20 | muse-html-meta-http-equiv\" content=\"muse-html-meta-content-type\" />
21 |
22 | (let ((maintainer (muse-style-element :maintainer)))
23 | (when maintainer
24 | (concat \"\")))
25 |
26 | (muse-style-element :style-sheet muse-publishing-current-style)
27 |
28 |
29 |
30 |
34 |
38 |
39 |
40 | "
41 | :footer "
42 |
43 |
49 |
50 | "
51 | :style-sheet "")
161 |
162 | (setq muse-project-alist
163 | '(("Monikop and Pokinom"
164 | ("." :default "index")
165 | (:base
166 | "xhtml-plainandsimple"
167 | :path "../html"
168 | :exclude "\\(footer.muse\\)\\|\\(footer-right.muse\\)\\|\\(sidebar.muse\\)"))))
169 |
170 | (muse-project-publish "Monikop and Pokinom" t)
171 |
--------------------------------------------------------------------------------
/doc/fake-pokinom-screenshot.pl:
--------------------------------------------------------------------------------
1 | #! /usr/bin/perl
2 | #use strict;
3 | #use warnings;
4 | use File::Basename;
5 | use File::Rsync;
6 | use Thread 'async';
7 | use threads::shared;
8 | use Curses;
9 |
10 | my @pokinom_banner = (
11 | " _/_/_/ _/_/ _/ _/ _/_/_/ _/ _/ _/_/ _/ _/",
12 | " _/ _/ _/ _/ _/ _/ _/ _/_/ _/ _/ _/ _/_/ _/_/ ",
13 | " _/_/_/ _/ _/ _/_/ _/ _/ _/ _/ _/ _/ _/ _/ _/ ",
14 | " _/ _/ _/ _/ _/ _/ _/ _/_/ _/ _/ _/ _/ ",
15 | "_/ _/_/ _/ _/ _/_/_/ _/ _/ _/_/ _/ _/ ",
16 | );
17 |
18 | my $version = "v0.0.1";
19 |
20 | # Debug mode:
21 | # 0 = clean UI; 1 = lots of scrolling junk; anything else = both (pipe to file)
22 | my $debug = 0;
23 |
24 |
25 | sub act_on_keypress {
26 | my ($pressed_key) = @_;
27 | if ($pressed_key eq 267) { qx($shut_down_action); }
28 | elsif ($pressed_key eq 273) { # F9
29 | $shut_down_when_done = $shut_down_when_done ? 0 : 1; }
30 | }
31 |
32 | my %being_deleted_thread;
33 | my %rsync_worker_thread;
34 | my $display_thread;
35 |
36 | $ENV{USER} = $rsync_username if ($rsync_username);
37 | $ENV{RSYNC_PASSWORD} = $rsync_password if ($rsync_password);
38 |
39 | $SIG{TERM} = sub {
40 | $display_thread->kill('TERM')->join;
41 | die "Caught signal $_[0]";
42 | };
43 |
44 | # Preparations done; sleeves up!
45 |
46 | # Make sure we have dirs to put our logs in:
47 | ## map {
48 | ## my ($filename, $directory) = fileparse $_;
49 | ## qx(mkdir -p $directory);
50 | ## } ( $rsync_log_prefix, $interrupted_prefix );
51 | ##
52 | ## # Find usable (i.e. mounted) sources
53 | ## my @raw_mount_points = grep (s/\S+ on (.*) type .*/$1/, qx/mount/);
54 | ## chomp @raw_mount_points;
55 | ## my @sources = intersection @raw_mount_points, @usable_mount_points;
56 | ## debug_print "SOURCES:\n";
57 | ## debug_print @sources;
58 | @sources = (
59 | '/media/disk_1',
60 | '/media/disk_2',
61 | '/media/disk_3',
62 | '/media/disk_4',
63 | '/media/disk_5',
64 | '/media/disk_6',
65 | '/media/disk_7',
66 | );
67 |
68 | # Turn a path into a legal perl identifier:
69 | sub make_key_from_path {
70 | my $path = shift;
71 | ($path) =~ s/\/?(.*)\/?/$1/g;
72 | ($path) =~ s/\W/_/g;
73 | $path;
74 | }
75 |
76 | map {
77 | $source_roots{make_key_from_path $_} = $_
78 | } @sources;
79 |
80 | %speeds = (
81 | 'media_disk_1' => '15.20MB/s',
82 | 'media_disk_2' => '10.02MB/',
83 | 'media_disk_3' => '-',
84 | 'media_disk_4' => '242.73kB/s',
85 | 'media_disk_5' => '6.78MB/s',
86 | 'media_disk_6' => '-',
87 | 'media_disk_7' => '-',
88 | );
89 |
90 | %done = (
91 | 'media_disk_1' => 0,
92 | 'media_disk_2' => 0,
93 | 'media_disk_3' => 1,
94 | 'media_disk_4' => 0,
95 | 'media_disk_5' => 0,
96 | 'media_disk_6' => 0,
97 | 'media_disk_7' => 1,
98 | );
99 |
100 | %progress_ratios = (
101 | 'media_disk_1' => '951/2300',
102 | 'media_disk_2' => '217/352',
103 | 'media_disk_3' => 'Done',
104 | 'media_disk_4' => '16/223',
105 | 'media_disk_5' => '1854/1929',
106 | 'media_disk_6' => 'Wait',
107 | 'media_disk_7' => 'Done',
108 | );
109 |
110 |
111 | unless ($debug == 1) {
112 | # Talk to the user.
113 | $display_thread = async {
114 | $SIG{TERM} = sub {
115 | endwin(); # Leave a usable terminal.
116 | threads->exit()
117 | };
118 |
119 | my $redraw_window_count = 0;
120 | initscr();
121 | cbreak();
122 | noecho();
123 | curs_set(0);
124 | my $window_top = newwin(24 - 8, 79, 0, 0);
125 | my $window_center = newwin(5, 79, 24 - 8, 0);
126 | my $window_bottom = newwin(3, 79, 24 - 3, 0);
127 | $window_bottom->keypad(1);
128 | $window_bottom->nodelay(1);
129 | start_color;
130 | init_pair 1, COLOR_MAGENTA, COLOR_BLACK;
131 | init_pair 2, COLOR_RED, COLOR_BLACK;
132 | init_pair 3, COLOR_CYAN, COLOR_BLACK;
133 | init_pair 4, COLOR_YELLOW, COLOR_BLACK;
134 | my $MAGENTA = COLOR_PAIR(1);
135 | my $RED = COLOR_PAIR(2);
136 | my $CYAN = COLOR_PAIR(3);
137 | my $YELLOW = COLOR_PAIR(4);
138 | while (1) {
139 | $window_top->attron($CYAN);
140 | $window_top->box(0,0);
141 | $window_top->addstr(0, 30, " P r o g r e s s ");
142 | $window_top->attroff($CYAN);
143 | $window_top->addstr(15, 1, "$version");
144 | my $sources_format = "%-25s%-18s%-8s";
145 | $window_top->attron(A_BOLD);
146 | $window_top->addstr(1, 12,
147 | sprintf ($sources_format,
148 | "Source Medium", "Speed", "To Do"));
149 | $window_top->attroff(A_BOLD);
150 | my $line_number = 2;
151 | map {
152 | my $source = $_;
153 | $window_top->attron($CYAN);
154 | $window_top->attron($RED) if $done{$source};
155 | $window_top->
156 | addstr($line_number, 12,
157 | sprintf($sources_format,
158 | substr($source_roots{$source}, 0, 24),
159 | substr($speeds{$source}, 0, 17),
160 | substr($progress_ratios{$source}, -8, 8)));
161 | ++ $line_number;
162 | $window_top->addstr($line_number, 1,
163 | sprintf($sources_format, "", "", "", ""));
164 | $window_top->attroff($RED);
165 | $window_top->attroff($CYAN);
166 | } sort (keys %source_roots);
167 | $line_number = 0;
168 | map {
169 | $window_center->addstr($line_number, 2, $_);
170 | ++ $line_number;
171 | } @pokinom_banner;
172 | $window_center->move(0, 0);
173 |
174 | $window_bottom->box(0,0);
175 | $window_bottom->attron(A_BOLD);
176 | $window_bottom->
177 | addstr(1, 3,
178 | sprintf ("[F3]: Turn off now.%54s",
179 | $shut_down_when_done ? "Turning off when done. [F9]: Stay on."
180 | : "Staying on. [F9]: Turn off when done."));
181 | $window_bottom->attroff(A_BOLD);
182 |
183 | $window_top->noutrefresh();
184 | $window_bottom->noutrefresh();
185 | $window_center->noutrefresh(); # Last window gets the cursor.
186 | sleep 2;
187 | if (++ $redraw_window_count > 5) {
188 | $redraw_window_count = 0;
189 | redrawwin();
190 | }
191 | doupdate();
192 | act_on_keypress($window_bottom->getch());
193 | if (! grep(/0/, values %done) && $shut_down_when_done) {
194 | qx ($shut_down_action);
195 | }
196 | }
197 | endwin();
198 | };
199 | }
200 |
201 | sleep;
202 |
203 |
--------------------------------------------------------------------------------
/doc/fake-monikop-screenshot.pl:
--------------------------------------------------------------------------------
1 | #! /usr/bin/perl
2 | #use strict;
3 | #use warnings;
4 | use integer;
5 | use File::Rsync;
6 | use File::Basename;
7 | use Thread 'async';
8 | use threads::shared;
9 | use Curses;
10 |
11 | my @monikop_banner = (
12 | " _/ _/ _/_/ _/ _/ _/_/_/ _/ _/ _/_/ _/_/_/ ",
13 | " _/_/ _/_/ _/ _/ _/_/ _/ _/ _/ _/ _/ _/ _/ _/",
14 | " _/ _/ _/ _/ _/ _/ _/ _/ _/ _/_/ _/ _/ _/_/_/ ",
15 | " _/ _/ _/ _/ _/ _/_/ _/ _/ _/ _/ _/ _/ ",
16 | "_/ _/ _/_/ _/ _/ _/_/_/ _/ _/ _/_/ _/ ",
17 | );
18 |
19 | $version = "v0.0.1";
20 |
21 | # Debug mode:
22 | # 0 = clean UI; 1 = lots of scrolling junk; anything else = both (pipe to file)
23 | my $debug = 0;
24 | $debug = $ARGV[1] if $ARGV[1];
25 |
26 | # Where to read local configuration:
27 | my $monikop_config = '~/monikop/monikop.config';
28 | $monikop_config = $ARGV[0] if $ARGV[0];
29 |
30 | ########################################
31 | # Settings
32 | ########################################
33 | # Possible data sources, and by what directory name to represent them in
34 | # destination.
35 | # When the latter is not unique, care must be taken that all pathnames in the
36 | # respective sources are unique.
37 | my %sources = (
38 | 'data_producer1::data' => 'p1_dir',
39 | 'data_producer2::data' => 'p2_dir',
40 | 'data_producer3::data' => '',
41 | 'data_producer4::data' => '',
42 | 'data_producer5::data' => '',
43 | 'data_producer6::data' => '',
44 | 'data_producer7::data' => '',
45 | );
46 |
47 | # Places to store run-time information to share between threads:
48 | my %speeds :shared; # rsync output
49 | my %progress_ratios :shared; # rsync output
50 | my %destination_usages :shared; # i.e. used/unused
51 | my %destination_usage_ratios :shared;
52 | my %destination_source_is_writing_to :shared;
53 | my %reachable :shared;
54 |
55 | sub debug_print { if ($debug) { print @_; } };
56 |
57 | # Turn a path into a legal perl identifier:
58 | sub make_key_from_path {
59 | my $path = shift;
60 | ($path) =~ s/\/?(.*)\/?/$1/g;
61 | ($path) =~ s/\W/_/g;
62 | $path;
63 | }
64 |
65 | my %source_roots;
66 | map {
67 | $source_roots{make_key_from_path $_} = $_
68 | } keys %sources;
69 |
70 | my %source_dirs_in_destination;
71 | map {
72 | $source_dirs_in_destination{make_key_from_path $_} = $sources{$_}
73 | } keys %sources;
74 |
75 | sub act_on_keypress {
76 | my ($pressed_key) = @_;
77 | if ($pressed_key eq 267) { qx($key_f3_action) }
78 | elsif ($pressed_key eq 270) { qx($key_f6_action); }
79 | }
80 |
81 | %destination_source_is_writing_to = (
82 | make_key_from_path ('/data_producer1::data') => '/media/disk_2',
83 | make_key_from_path ('/data_producer2::data') => '/media/disk_4',
84 | make_key_from_path ('/data_producer3::data') => '/media/disk_7',
85 | make_key_from_path ('/data_producer4::data') => '/media/disk_5',
86 | make_key_from_path ('/data_producer5::data') => '/media/disk_8',
87 | make_key_from_path ('/data_producer6::data') => '/media/disk_3',
88 | );
89 |
90 | $SIG{TERM} = sub {
91 | $display_thread->kill('TERM')->join;
92 | die "Caught signal $_[0]";
93 | };
94 |
95 | @destination_roots = (
96 | '/media/disk_1',
97 | '/media/disk_2',
98 | '/media/disk_3',
99 | '/media/disk_4',
100 | '/media/disk_5',
101 | '/media/disk_6',
102 | '/media/disk_7',
103 | '/media/disk_8',
104 | );
105 |
106 | %destination_usage_ratios = (
107 | '/media/disk_1' => 38,
108 | '/media/disk_2' => 94,
109 | '/media/disk_3' => 10,
110 | '/media/disk_4' => 27,
111 | '/media/disk_5' => 6,
112 | '/media/disk_6' => 5,
113 | '/media/disk_7' => 99,
114 | '/media/disk_8' => 10,
115 | );
116 |
117 | %destination_usages = (
118 | '/media/disk_1' => 0,
119 | '/media/disk_2' => 1,
120 | '/media/disk_3' => 1,
121 | '/media/disk_4' => 1,
122 | '/media/disk_5' => 1,
123 | '/media/disk_6' => 0,
124 | '/media/disk_7' => 1,
125 | '/media/disk_8' => 1,
126 | );
127 |
128 | %reachable = (
129 | 'data_producer1__data' => 1,
130 | 'data_producer2__data' => 1,
131 | 'data_producer3__data' => 1,
132 | 'data_producer4__data' => 1,
133 | 'data_producer5__data' => 1,
134 | 'data_producer6__data' => 1,
135 | );
136 |
137 | %speeds = (
138 | 'data_producer1__data' => '23.30MB/s',
139 | 'data_producer2__data' => '31.53MB/s',
140 | 'data_producer3__data' => '-',
141 | 'data_producer4__data' => '243.81kB/s',
142 | 'data_producer5__data' => '39.19MB/s',
143 | 'data_producer6__data' => '23.30MB/s',
144 | );
145 |
146 | %progress_ratios = (
147 | 'data_producer1__data' => '951/2300',
148 | 'data_producer2__data' => '217/352',
149 | 'data_producer3__data' => '0',
150 | 'data_producer4__data' => '16/223',
151 | 'data_producer5__data' => '1854/1929',
152 | 'data_producer6__data' => '1773/1929',
153 | );
154 |
155 | unless ($debug == 1) {
156 | # Talk to the user.
157 | $display_thread = async {
158 | $SIG{TERM} = sub {
159 | endwin(); # Leave a usable terminal.
160 | threads->exit()
161 | };
162 |
163 | my $redraw_window_count = 0;
164 | initscr();
165 | cbreak();
166 | noecho();
167 | curs_set(0);
168 | my $window_left = newwin(24 -8, 29, 0, 0);
169 | my $window_right = newwin(24 -8, 50, 0, 29);
170 | my $window_center = newwin(5, 79, 24 -8, 0);
171 | my $window_bottom = newwin(3, 79, 24 -3, 0);
172 | $window_bottom->keypad(1);
173 | $window_bottom->nodelay(1);
174 | start_color;
175 | init_pair 1, COLOR_MAGENTA, COLOR_BLACK;
176 | init_pair 2, COLOR_RED, COLOR_BLACK;
177 | init_pair 3, COLOR_CYAN, COLOR_BLACK;
178 | init_pair 4, COLOR_YELLOW, COLOR_BLACK;
179 | my $MAGENTA = COLOR_PAIR(1);
180 | my $RED = COLOR_PAIR(2);
181 | my $CYAN = COLOR_PAIR(3);
182 | my $YELLOW = COLOR_PAIR(4);
183 |
184 | while (1) {
185 | $window_left->attron($CYAN);
186 | $window_left->box(0, 0);
187 | $window_left->addstr(0, 6, "Data Destinations");
188 | $window_left->attroff($CYAN);
189 | my $destinations_format = "%-18s%-6s%-3s";
190 | $window_left->attron(A_BOLD);
191 | $window_left->addstr(1, 1, sprintf($destinations_format,
192 | "Removable", "Fresh", "Usg"));
193 | $window_left->addstr(2, 1, sprintf($destinations_format,
194 | "Disk", "Data?", "%"));
195 | $window_left->attroff(A_BOLD);
196 | my $destination_usage;
197 | my $line_number = 3;
198 | map {
199 | if ($destination_usages{$_}) {
200 | $window_left->attron($RED);
201 | $destination_usage = "yes";
202 | } else {
203 | $window_left->attron($CYAN);
204 | $destination_usage = "no";
205 | }
206 | $window_left->
207 | addstr($line_number, 1,
208 | sprintf($destinations_format,
209 | substr($_, -17, 17),
210 | substr($destination_usage, -6, 6),
211 | substr($destination_usage_ratios{$_}
212 | ? $destination_usage_ratios{$_}
213 | : "?",
214 | -3, 3)));
215 | ++ $line_number;
216 | $window_left->attroff($RED);
217 | $window_left->attroff($CYAN);
218 | } sort @destination_roots;
219 |
220 | $window_right->attron($MAGENTA);
221 | $window_right->box(0,0);
222 | $window_right->addstr(0, 19, "Data Sources");
223 | $window_right->attroff($MAGENTA);
224 | my $sources_format = "%-15s%-11s%-9s%-13s";
225 | $window_right->attron(A_BOLD);
226 | $window_right->
227 | addstr(1, 1, sprintf ($sources_format,
228 | "Data", "", "Files", " Writing"));
229 | $window_right->
230 | addstr(2, 1, sprintf ($sources_format,
231 | "Source", "Speed", "To Copy", " To"));
232 | $window_right->attroff(A_BOLD);
233 | $line_number = 3;
234 | $window_right->attron($MAGENTA);
235 | map {
236 | my $source = $_;
237 | my $current_destination = '?';
238 | if (exists $destination_source_is_writing_to{$source}) {
239 | $current_destination =
240 | $destination_source_is_writing_to{$source};
241 | }
242 | if ($reachable{$source}) {
243 | $window_right->
244 | addstr($line_number, 1,
245 | sprintf($sources_format,
246 | substr($source_roots{$source}, 0, 14),
247 | substr($speeds{$source}, 0, 11),
248 | substr($progress_ratios{$source},
249 | -9, 9),
250 | substr($current_destination, -13, 13)));
251 | ++ $line_number;
252 | }
253 | $window_right->
254 | addstr($line_number, 1,
255 | sprintf($sources_format, "", "", "", ""));
256 | } sort (keys %source_roots);
257 | $window_right->attroff($MAGENTA);
258 |
259 | $line_number = 0;
260 | map {
261 | $window_center->addstr($line_number, 2, $_);
262 | ++ $line_number;
263 | } @monikop_banner;
264 | $window_center->addstr(4, 72, "$version");
265 | $window_center->move(0, 0);
266 |
267 | $window_bottom->box(0,0);
268 | $window_bottom->attron(A_BOLD);
269 | $window_bottom->addstr(1, 3, "[F3]: Turn off computer.");
270 | $window_bottom->addstr(1, 53, "[F6]: Restart computer.");
271 | $window_bottom->attroff(A_BOLD);
272 |
273 | $window_left->noutrefresh();
274 | $window_right->noutrefresh();
275 | $window_bottom->noutrefresh();
276 | $window_center->noutrefresh(); # Last window gets the cursor.
277 | act_on_keypress($window_bottom->getch());
278 | sleep 2;
279 | if (++ $redraw_window_count > 5) {
280 | $redraw_window_count = 0;
281 | redrawwin();
282 | }
283 | doupdate();
284 | }
285 | endwin();
286 | };
287 | }
288 |
289 | sleep;
290 |
--------------------------------------------------------------------------------
/doc/installation.muse:
--------------------------------------------------------------------------------
1 | #title Monikop (and Pokinom)
2 | #subtitle rsync between unconnected hosts
3 | #author Bert Burgemeister
4 |
5 |
6 |
7 | * Installation
8 |
9 | We assume Debian GNU/Linux here, but any distribution should
10 | work. Adapt installation instructions accordingly.
11 |
12 | Debian Packages needed:
13 |
14 | - to run Monikop (on Rover) or Pokinom (in office):
15 | rsync, mingetty, sudo, libcurses-perl, libfile-rsync-perl;
16 |
17 | - to install from a git repository: git-core;
18 |
19 | - to run the tests: bc, time.
20 |
21 | - Both Monikop and Pokinom run on text console; you don't need
22 | anything like Gnome, KDE or even X.
23 |
24 |
25 | ** Prepare Removable Disks
26 |
27 | Put sticker labels with disk names on your removable disks.
28 |
29 | *** File Systems
30 |
31 | Create labelled file systems on the removable disks. Example
32 | (suppose a removable disk with a sticker label "=disk_10=" on its case is attached to =/dev/sdg1=):
33 |
34 |
35 | # mke2fs -j -L disk_10 /dev/sdg1
36 |
37 |
38 | On both Monikop's and Pokinom's host, label the system root
39 | partition. If it were on =/dev/sda1/, that's e.g.=:
40 |
41 |
42 | # e2label /dev/sda1 root
43 |
44 |
45 | On both Monikop's and Pokinom's host, label the swap partition. If it
46 | happens to be on /dev/sda5, e.g.:
47 |
48 |
49 | # swapoff
50 |
51 | # mkswap -L swap /dev/sda5
52 |
53 | # swapon
54 |
55 |
56 | *** Mount Points
57 |
58 | On both Monikop's and Pokinom's host, create mount points, one for
59 | each removable disk:
60 |
61 |
62 | # mkdir -p /media/disk_{01,02,03,04...}
63 |
64 | # chmod a+rx /media/disk_{01,02,03,04...}
65 |
66 |
67 | #fstab
68 | In =/etc/fstab= on both Monikop's and Pokinom's host, make use of the disk labels:
69 |
70 |
71 | ## System partitions ###
72 | LABEL=root / ext3 defaults,errors=remount-ro 0 1
73 | LABEL=swap none swap sw 0 0
74 | ## Removable disks
75 | LABEL=disk_01 /media/disk_01 ext3 rw,user,auto 0 0
76 | LABEL=disk_02 /media/disk_02 ext3 rw,user,auto 0 0
77 | LABEL=disk_03 /media/disk_03 ext3 rw,user,auto 0 0
78 | LABEL=disk_04 /media/disk_04 ext3 rw,user,auto 0 0
79 | # etc.
80 |
81 |
82 | Put each removable disk in and make it writable; e.g.:
83 |
84 |
85 | # mount /media/disk_01
86 |
87 | # chmod a+rwx /media/disk_01
88 |
89 |
90 | *** Maintain Bootability
91 |
92 | On both Monikop's and Pokinom's host, make sure the operating system boots
93 | actually from its system disk rather than from some of the removable
94 | ones. Change =/boot/grub/menu.lst= where it says # kopt=root=...:
95 |
96 |
97 | ### BEGIN AUTOMAGIC KERNELS LIST
98 | ## lines between the AUTOMAGIC KERNELS LIST markers will be modified
99 | ## by the debian update-grub script except for the default options below
100 |
101 | ## DO NOT UNCOMMENT THEM, Just edit them to your needs
102 |
103 | ## ## Start Default Options ##
104 | ## default kernel options
105 | ## default kernel options for automagic boot options
106 | ## If you want special options for specific kernels use kopt_x_y_z
107 | ## where x.y.z is kernel version. Minor versions can be omitted.
108 | ## e.g. kopt=root=/dev/hda1 ro
109 | ## kopt_2_6_8=root=/dev/hdc1 ro
110 | ## kopt_2_6_8_2_686=root=/dev/hdc2 ro
111 | # kopt=root=/dev/disk/by-label/root noresume ro
112 |
113 |
114 | and call
115 |
116 |
117 | # update-grub
118 |
119 |
120 |
121 | #Configure_Monikop_and_Pokinom
122 | ** Configure Monikop and Pokinom
123 |
124 | Create a user on both Monikop's and Pokinom's machine. For
125 | description's sake, we assume they're called m-operator.
126 |
127 | Inside m-operator's home directory, [[download][get Monikop (and Pokinom)]];
128 | unpack the tarball:
129 |
130 |
131 | $ tar -xzf monikop-.tar.gz
132 |
133 | $ mv monikop- monikop
134 |
135 | $ cd monikop
136 |
137 |
138 | Copy =monikop.config.example= to =monikop.config= and
139 | =pokinom.config.example= to =pokinom.config, respectively,= and
140 | adapt them according to your needs. Both are perl code, so be careful
141 | and keep the punctuation in place.
142 |
143 | #monikop.config
144 | *** =monikop.config=
145 |
146 |
147 |
148 | For Monikop, change in [[installation#monikop.config][monikop.config]] at least:
149 |
150 | - =%sources=: Data producing Sources on Rover in one of the formats Rsync
151 | understands, together with a source-specific directory name where data
152 | of the respective Source goes. Those directory names can be equal for
153 | several Sources as long as all filenames in the payload are certain to be
154 | unique.
155 |
156 | - =@usable_mount_points=: Mount points (directories) you set up [[installation#fstab][earlier]] for your
157 | removable disks.
158 |
159 |
160 | #pokinom.config
161 |
162 | *** =pokinom.config=
163 |
164 |
165 |
166 | For Pokinom you should edit in [[installation#pokinom.config][pokinom.config]] at least:
167 |
168 | - =@usable_mount_points= (as with [[installation#monikop.config][monikop.config]])
169 | - =$destination=: Data destination in one of the formats Rsync
170 | understands; cf. setup of [[installation#Data_Destination][Data Destination]].
171 | - =$rsync_username=, =$rsync_password=: credentials of (and only
172 | known to) the Rsync server;
173 | cf. setup of [[installation#Data_Destination][Data Destination]].
174 |
175 |
176 | *** Automatic Program Start
177 |
178 | Append to
179 | =/home/m-operator/.profile= (create it if necessary):
180 |
181 |
182 | /home/m-operator/monikop/monikop
183 |
184 |
185 | or
186 |
187 |
188 | /home/m-operator/monikop/pokinom,
189 |
190 |
191 | respectively.
192 |
193 | If necessary, specify path to config file, e.g.
194 |
195 | /home/m-operator/monikop/monikop /home/m-operator/monikop/monikop.config
196 |
197 |
198 |
199 | *** Setup Sudo
200 |
201 | (Not necessary on a systemd-based system.)
202 |
203 | On both Monikop's and Pokinom's host authorise m-operator to shut down computer.
204 | Use =visudo= to change =/etc/sudoers=; add:
205 |
206 |
207 | m-operator ALL=(ALL) NOPASSWD: /sbin/halt -p
208 | m-operator ALL=(ALL) NOPASSWD: /sbin/reboot
209 |
210 |
211 |
212 | *** Automatic Login (under systemd)
213 |
214 | On both Monikop's and Pokinom's host, change the line in
215 | =/etc/inittab= that looks like
216 |
217 |
218 | 1:2345:respawn:/sbin/getty 38400 tty1
219 |
220 |
221 | into
222 |
223 |
224 | 1:2345:respawn:/sbin/mingetty --autologin m-operator --noclear tty1
225 |
226 |
227 | *** Automatic Login (under systemd)
228 |
229 | On both Monikop's and Pokinom's host, create the file (and the
230 | containing directory)
231 | /etc/systemd/system/getty@tty1.service.d/autologin.conf:
232 |
233 |
234 | [Service]
235 | ExecStart=
236 | ExecStart=-/sbin/agetty --autologin m-operator --noclear %I 38400 linux
237 |
238 |
239 |
240 | #Configure_Rsync_on_Sources
241 | ** Configure Rsync on Sources
242 |
243 | Install package rsync.
244 |
245 | Example for =/etc/rsyncd.conf=:
246 |
247 |
248 | pid file=/var/run/rsyncd.pid
249 | [data]
250 | path = /mnt/hdd_0
251 | use chroot = false
252 | lock file = /var/lock/rsyncd
253 | read only = yes
254 | list = yes
255 | transfer logging = false
256 |
257 |
258 | In =/etc/default/rsync=, change the line
259 |
260 |
261 | RSYNC_ENABLE = false
262 |
263 |
264 | to
265 |
266 |
267 | RSYNC_ENABLE = true
268 |
269 |
270 | Start rsync server:
271 |
272 | =# /etc/initd/rsync start=
273 |
274 | or reboot.
275 |
276 |
277 | On Windows, install Cygwin for [[https://cygwin.com/setup-x86.exe][x86]] or [[https://cygwin.com/setup-x86_64.exe][amd64]] including package rsync.
278 | Start Cygwin as Administrator.
279 |
280 | Inside Cygwin, edit /etc/rsyncd.conf:
281 |
282 |
283 | use chroot = false
284 | strict modes = false
285 | hosts allow = *
286 | logfile = rsyncd.log
287 | [data]
288 | # /cygdrive/e/log stands for E:\log
289 | path = /cygdrive/e/log
290 | read only = false
291 | transfer logging = false
292 |
293 |
294 | Configure rsync as a service:
295 |
296 |
297 | $ cygrunsrv --install "rsyncd" --path /usr/bin/rsync \
298 | --args "--daemon --no-detach" \
299 | --desc "Start rsync daemon for accepting incoming rsync connections" \
300 | --disp "Rsync Daemon" \
301 | --type auto
302 |
303 |
304 | Start the rsync service (or just reboot):
305 |
306 | $ net start rsyncd
307 |
308 |
309 | #Network_Setup
310 | ** Network Setup
311 |
312 | Depending on the amount of data to transfer, consider putting a
313 | dedicated NIC for each Source into Monikop's machine. In this case,
314 | you should provide for non-overlapping subnets. [[http://jodies.de/ipcalc][IP-Calculator]] may be
315 | helpful.
316 |
317 |
318 | *** Monikop
319 |
320 | **** Name the Sources
321 |
322 | #etc_hosts
323 | Example for =/etc/hosts=:
324 |
325 |
326 | 127.0.0.1 localhost
327 | 192.168.200.10 data-producer1
328 | 192.168.200.20 data-producer2
329 | 192.168.200.30 data-producer3
330 | 192.168.200.50 data-producer4
331 | 192.168.178.1 monikop
332 |
333 |
334 |
335 | **** Configure NICs
336 |
337 | Example for =/etc/network/interfaces=:
338 |
339 | # The loopback network interface
340 | auto lo
341 | iface lo inet loopback
342 |
343 | # Net of smaller Sources
344 | allow-hotplug eth1
345 | iface eth1 inet static
346 | address 192.168.178.1
347 | netmask 255.255.255.0
348 |
349 | # Dedicated NIC for data-producer1
350 | allow-hotplug eth2
351 | iface eth2 inet static
352 | address 192.168.200.9
353 | netmask 255.255.255.248
354 |
355 | # Dedicated NIC for data_producer2
356 | allow-hotplug eth3
357 | iface eth3 inet static
358 | address 192.168.200.19
359 | netmask 255.255.255.248
360 |
361 | # Dedicated NIC for data_producer3
362 | allow-hotplug eth4
363 | iface eth4 inet static
364 | address 192.168.200.29
365 | netmask 255.255.255.248
366 |
367 | # Dedicated NIC for data_producer4
368 | allow-hotplug eth5
369 | iface eth5 inet static
370 | address 192.168.200.49
371 | netmask 255.255.255.248
372 |
373 |
374 |
375 | *** Data Sources
376 |
377 | Use [[installation#etc_hosts][/etc/hosts]] as with Monikop. For Windows, it's =%SystemRoot%\system32\drivers\etc\hosts=.
378 |
379 |
380 | **** Source's NIC
381 |
382 | Example for =/etc/network/interfaces=:
383 |
384 |
385 | auto lo
386 | iface lo inet loopback
387 |
388 | # service (not relevant for Monikop)
389 | allow-hotplug eth0
390 | iface eth0 inet static
391 | address 192.168.178.2
392 | netmask 255.255.255.0
393 |
394 | # Monikop's dedicated NIC
395 | allow-hotplug eth1
396 | iface eth1 inet static
397 | address 192.168.200.10
398 | netmask 255.255.255.248
399 |
400 |
401 | For Windows, configure your network settings accordingly.
402 |
403 |
404 | *** Pokinom
405 |
406 | Pokinom's network settings don't need any special treatment. Just
407 | integrate it into the office LAN Destination is connected to.
408 |
409 |
410 | #Data_Destination
411 | ** Data Destination
412 |
413 | *** Rsync Server on Destination
414 |
415 | Install package rsync.
416 |
417 | Adapt =/etc/rsyncd.conf=, e.g.:
418 |
419 |
420 | gid = data_receiving_group
421 | use chroot = yes
422 | max connections = 0
423 | pid file = /var/run/rsyncd.pid
424 |
425 | [incoming]
426 | path = /mnt/./raid_0
427 | list = no
428 | comment = Pokinom only; requires authentication
429 | read only = no
430 | incoming chmod = g+r,g+w,g+X
431 | write only = yes
432 | # Pokinom's IP:
433 | hosts allow = 192.168.180.120
434 | auth users = m-operator
435 | secrets file = /etc/rsyncd.secrets
436 |
437 |
438 | =/etc/rsyncd.secrets= contains Rsync's credentials which must
439 | correspond to settings =$rsync_passwd= and =$rsync_username= in [[installation#pokinom.config][pokinom.config]]:
440 |
441 |
442 | m-operator:sEcReT
443 |
444 |
445 | =/etc/rsyncd.secrets= must not be world-readable.
446 |
447 | In =/etc/default/rsync=, change the line
448 |
449 |
450 | RSYNC_ENABLE = false
451 |
452 |
453 | to
454 |
455 |
456 | RSYNC_ENABLE = true
457 |
458 |
459 | Start rsync server:
460 |
461 |
462 | # /etc/initd/rsync start
463 |
464 |
465 | or reboot.
466 |
467 | With the above, rsync puts the payload it receives into
468 | =/mnt/raid_0/NEW_DATA/=. ("=NEW_DATA=" was set with
469 | =$destination= in [[installation#pokinom.config][pokinom.config]].)
470 |
471 | =NEW_DATA/= and everything inside belongs to user
472 | nobody and group data_receiving_group.
473 |
474 | If on Destination you can't do without Windows, install rsync under
475 | Cygwin as described [[installation#Configure_Rsync_on_Sources][above]].
476 |
477 | ; TODO: net topology for Monikop, for Pokinom
478 |
--------------------------------------------------------------------------------
/pokinom:
--------------------------------------------------------------------------------
1 | #! /usr/bin/perl
2 | use strict;
3 | use warnings;
4 | use File::Basename;
5 | use File::Rsync;
6 | use Thread 'async';
7 | use threads::shared;
8 | use Curses;
9 |
10 | my @pokinom_banner = (
11 | " _/_/_/ _/_/ _/ _/ _/_/_/ _/ _/ _/_/ _/ _/",
12 | " _/ _/ _/ _/ _/ _/ _/ _/_/ _/ _/ _/ _/_/ _/_/ ",
13 | " _/_/_/ _/ _/ _/_/ _/ _/ _/ _/ _/ _/ _/ _/ _/ ",
14 | " _/ _/ _/ _/ _/ _/ _/ _/_/ _/ _/ _/ _/ ",
15 | "_/ _/_/ _/ _/ _/_/_/ _/ _/ _/_/ _/ _/ ",
16 | );
17 |
18 | # Version number. Should agree with Pokinom's one.
19 | # Format: v<1>.<2>.<3> where
20 | # <3> = bug fix,
21 | # <2> = new feature,
22 | # <1> = incompatible change.
23 | my $version = 'v0.1.2';
24 |
25 | # Debug mode:
26 | # 0 = clean UI; 1 = lots of scrolling junk; anything else = both (pipe to file)
27 | my $debug = 0;
28 | $debug = $ARGV[1] if $ARGV[1];
29 |
30 | # Where to read local configuration:
31 | my $pokinom_config = '~/monikop/pokinom.config';
32 | if ($ARGV[0]) {
33 | $pokinom_config = $ARGV[0]
34 | }
35 |
36 | ########################################
37 | # Settings
38 | ########################################
39 | # Possible mount points.
40 | my @usable_mount_points;
41 |
42 | # Directory relative to a mount point where new data resides.
43 | # Must agree with Monikop's setting.
44 | my $path_under_mount_point;
45 |
46 | # Directories of this name will be deleted.
47 | # Must agree with Monikop's setting.
48 | my $path_under_mount_point_backed_up;
49 |
50 | # Directory name while being deleted by monikop.
51 | # Must agree with Monikop's setting.
52 | my $path_under_mount_point_being_deleted;
53 |
54 | # Data destination.
55 | my $destination;
56 |
57 | # Credentials of the remote rsync server. String, or 0 if not used.
58 | my $rsync_username;
59 | my $rsync_password;
60 |
61 | # Full path to rsync's raw log
62 | my $rsync_log_prefix;
63 |
64 | # Full path to a file to store list of rsync's incompletely
65 | # transferred files in:
66 | my $interrupted_prefix;
67 |
68 | # Shut down when finished? (default); 1 = yes; 2 = stay on.
69 | my $shut_down_when_done :shared;
70 |
71 | # How to turn off
72 | my $shut_down_action;
73 |
74 | # Rsync's directory (relative to destination) for partially transferred files.
75 | # Must agree with Monikop's setting.
76 | my $rsync_partial_dir_name;
77 |
78 | # Local changes to the above.
79 | eval `cat $pokinom_config`;
80 |
81 | # Places for running rsyncs to put their runtime info in
82 | my %speeds :shared;
83 | my %progress_ratios :shared;
84 | my %done :shared;
85 |
86 | sub debug_print { if ($debug) { print "\n"; print @_; } };
87 |
88 | # Return sorted intersection of arrays which are supposed to have unique
89 | # elements.
90 | sub intersection {
91 | my @intersection = ();
92 | my %count = ();
93 | my $element;
94 | foreach $element (@_) { $count{$element}++ }
95 | foreach $element (keys %count) {
96 | push @intersection, $element if $count{$element} > 1;
97 | }
98 | sort @intersection;
99 | }
100 |
101 | # Write @content to a file with name $filename.
102 | sub write_list {
103 | my ($filename, @content) = @_;
104 | open FILE, '>', $filename
105 | or die "[" . $$ . "] open $filename failed: $!\n";
106 | print FILE @content;
107 | close FILE;
108 | }
109 |
110 | my %source_roots;
111 | my %rsync_outfun;
112 | my %rsync;
113 |
114 | sub rsync_preparation_form {
115 | my ($source) = @_;
116 | $speeds{$source} = "-";
117 | join ( '',
118 | "\n",
119 | ########## Capture rsync's status messages for use by UI
120 | '$rsync_outfun{\'', $source, '\'} = sub {',
121 | ' my ($outline, $outputchannel) = @_ ; ',
122 | ' my ($speed) = $outline =~ /\d+\s+\d+%\s+(\S+)/; ',
123 | ' my ($progress_ratio) = $outline =~ /.+to-check=(\d+\/\d+)\)$/; ',
124 | ' if ($speed and $outputchannel eq \'out\') {',
125 | ' $speeds{\'', $source, '\'} = $speed;',
126 | ' } else {',
127 | ' $speeds{\'', $source, '\'} = "-";',
128 | ' };',
129 | ' if ($progress_ratio and $outputchannel eq \'out\') {',
130 | ' $progress_ratios{\'', $source, '\'} = $progress_ratio;',
131 | ' } ;',
132 | '};',
133 | "\n",
134 | ########## Run rsync
135 | '$rsync{\'', $source, '\'} = File::Rsync->new; ',
136 | ########## Return fodder for another eval
137 | '$rsync_exec_form{\'', $source, '\'} = sub {',
138 | ' \'$rsync{\\\'', $source, '\\\'}->exec(',
139 | ' {',
140 | ' src => \\\'', $source_roots{$source}, '/', $path_under_mount_point, '/\\\', ',
141 | ' dest => \\\'' . $destination . '/\\\', ',
142 | ' outfun => $rsync_outfun{\\\'', $source, '\\\'}, ',
143 | ' progress => 1, debug => 0, verbose => 0, ',
144 | ' filter => [\\\'merge,- ', $interrupted_prefix, $source, '\\\'], ',
145 | ' literal => [\\\'--recursive\\\', \\\'--times\\\', ',
146 | ' \\\'--partial-dir=', $rsync_partial_dir_name, '\\\', ',
147 | ' \\\'--update\\\', ',
148 | ' \\\'--prune-empty-dirs\\\', ',
149 | ' \\\'--log-file-format=%i %b %n\\\', ',
150 | ' , \\\'--log-file=', $rsync_log_prefix, $source, '\\\'] ',
151 | ' }',
152 | ' );\' ',
153 | '};',
154 | "\n",
155 | )};
156 |
157 | sub act_on_keypress {
158 | my ($pressed_key) = @_;
159 | if ($pressed_key eq 267) { qx($shut_down_action); }
160 | elsif ($pressed_key eq 273) { # F9
161 | $shut_down_when_done = $shut_down_when_done ? 0 : 1; }
162 | }
163 |
164 | my %being_deleted_thread;
165 | my %rsync_worker_thread;
166 | my $display_thread;
167 |
168 | $ENV{USER} = $rsync_username if ($rsync_username);
169 | $ENV{RSYNC_PASSWORD} = $rsync_password if ($rsync_password);
170 |
171 | $SIG{TERM} = sub {
172 | $display_thread->kill('TERM')->join;
173 | die "Caught signal $_[0]";
174 | };
175 |
176 | # Preparations done; sleeves up!
177 |
178 | # Make sure we have dirs to put our logs in:
179 | map {
180 | my ($filename, $directory) = fileparse $_;
181 | qx(mkdir -p $directory);
182 | } ( $rsync_log_prefix, $interrupted_prefix );
183 |
184 | # Find usable (i.e. mounted) sources
185 | my @raw_mount_points = grep (s/\S+ on (.*) type .*/$1/, qx/mount/);
186 | chomp @raw_mount_points;
187 | my @sources = intersection @raw_mount_points, @usable_mount_points;
188 | debug_print "SOURCES:\n";
189 | debug_print @sources;
190 |
191 | # Turn a path into a legal perl identifier:
192 | sub make_key_from_path {
193 | my $path = shift;
194 | ($path) =~ s/\/?(.*)\/?/$1/g;
195 | ($path) =~ s/\W/_/g;
196 | $path;
197 | }
198 |
199 | map {
200 | $source_roots{make_key_from_path $_} = $_
201 | } @sources;
202 |
203 | # Clean up sources if necessary:
204 | map {
205 | my $p_i_d = $source_roots{$_} . '/' . $path_under_mount_point;
206 | my $p_i_d_being_deleted =
207 | $source_roots{$_} . '/' . $path_under_mount_point_being_deleted;
208 | $being_deleted_thread{$_} =
209 | async { qx(rm -rf $p_i_d_being_deleted 2> /dev/null); };
210 | } keys %source_roots;
211 |
212 | # Wait for $destination if necessary:
213 | my $rsync_ping = File::Rsync->new;
214 | my $empty_directory = dirname($rsync_log_prefix) . "/empty_directory";
215 | qx(rm -rf $empty_directory; mkdir -p $empty_directory);
216 |
217 | while (1) {
218 | print "Waiting for $destination to become writable.\n";
219 | sleep 2;
220 | $rsync_ping->exec({ src => $empty_directory, dest => $destination});
221 | last if $? == 0;
222 | }
223 |
224 | my %rsync_exec_form;
225 | my $kludge; # Don't ask
226 |
227 | # Set up and start things per source_root, in parallel:
228 | map {
229 | $progress_ratios{$_} = "?"; # Initialize for UI
230 | $done{$_} = 0;
231 |
232 | debug_print 'rsync_preparation_form:' . rsync_preparation_form ($_). "\n";
233 | eval rsync_preparation_form $_;
234 | debug_print "EVAL RSYNC_PREPARATION_FORM $_: $@ \n";
235 |
236 | $rsync_worker_thread{$_} = async {
237 | my $rsync_log_name = $rsync_log_prefix . $_;
238 | my $complete_source =
239 | $source_roots{$_} . '/' . $path_under_mount_point;
240 | my $complete_source_backed_up =
241 | $source_roots{$_} . '/' . $path_under_mount_point_backed_up;
242 | my @interrupted =
243 | qx((cd $complete_source 2> /dev/null && find ./ -path *$rsync_partial_dir_name/*));
244 | # Write exclusion list: don't transfer files Monikop gave up upon.
245 | grep s/\.(.*\/)$rsync_partial_dir_name\/(.*)/$1$2/, @interrupted;
246 | write_list $interrupted_prefix . $_, @interrupted;
247 | debug_print "INTERRUPTED";
248 | debug_print @interrupted;
249 | $kludge = $rsync{$_};
250 | $kludge = $rsync_outfun{$_};
251 | if (-d $complete_source) {
252 | if (eval ($rsync_exec_form{$_}() )) {
253 | debug_print "EVAL RSYNC_EXEC_FORM in thread (successful) $complete_source: $@ \n";
254 | } else {
255 | $display_thread->kill('TERM')->join if $display_thread;
256 | # TODO: in case of overfull destination, warn nicer
257 | warn "EVAL RSYNC_EXEC_FORM in thread (failed) $complete_source: $@ \n";
258 | threads->exit();
259 | }
260 | }
261 | $progress_ratios{$_} = "Wait";
262 | $speeds{$_} = "-";
263 | };
264 | } keys %source_roots;
265 |
266 | # Repeat rsync runs, this time sequentially, in order to get the newest of a
267 | # file which may exist in multiple versions on different sources:
268 | my $rsync_worker_thread = async {
269 | sleep 4;
270 | map {
271 | $rsync_worker_thread{$_}->join;
272 | debug_print "JOINED $_\n";
273 | } keys %source_roots;
274 | map {
275 | $progress_ratios{$_} = "?"; # Initialize for UI
276 | $done{$_} = 0;
277 | my $rsync_log_name = $rsync_log_prefix . $_;
278 | eval rsync_preparation_form $_;
279 | debug_print "EVAL RSYNC_PREPARATION_FORM $_: $@ \n";
280 | my $complete_source =
281 | $source_roots{$_} . '/' . $path_under_mount_point;
282 | my $complete_source_backed_up =
283 | $source_roots{$_} . '/' . $path_under_mount_point_backed_up;
284 | my @interrupted =
285 | qx((cd $complete_source 2> /dev/null && find ./ -path *$rsync_partial_dir_name/*));
286 | # Write exclusion list: don't transfer files Monikop gave up upon.
287 | grep s/\.(.*\/)$rsync_partial_dir_name\/(.*)/$1$2/, @interrupted;
288 | write_list $interrupted_prefix . $_, @interrupted;
289 | if (-d $complete_source) {
290 | if (eval ($rsync_exec_form{$_}() )) {
291 | debug_print "EVAL RSYNC_EXEC_FORM sequential (successful) $complete_source: $@ \n";
292 | qx(mv $complete_source $complete_source_backed_up);
293 | } else {
294 | $display_thread->kill('TERM')->join if $display_thread;
295 | # TODO: in case of overfull destination, warn nicer
296 | warn "EVAL RSYNC_EXEC_FORM sequential (failed) $complete_source: $@ \n";
297 | threads->exit();
298 | }
299 | }
300 | $progress_ratios{$_} = "Done";
301 | $speeds{$_} = "-";
302 | $done{$_} = 1;
303 | unless ($debug) {
304 | unlink $rsync_log_name;
305 | unlink $interrupted_prefix . $_;
306 | }
307 | } keys %source_roots;
308 | };
309 |
310 | unless ($debug == 1) {
311 | # Talk to the user.
312 | $display_thread = async {
313 | $SIG{TERM} = sub {
314 | endwin(); # Leave a usable terminal.
315 | threads->exit()
316 | };
317 |
318 | my $redraw_window_count = 0;
319 | initscr();
320 | cbreak();
321 | noecho();
322 | curs_set(0);
323 | my $window_top = newwin(LINES() - 8, 79, 0, 0);
324 | my $window_center = newwin(5, 79, LINES() - 8, 0);
325 | my $window_bottom = newwin(3, 79, LINES() - 3, 0);
326 | $window_bottom->keypad(1);
327 | $window_bottom->nodelay(1);
328 | start_color;
329 | init_pair 1, COLOR_MAGENTA, COLOR_BLACK;
330 | init_pair 2, COLOR_RED, COLOR_BLACK;
331 | init_pair 3, COLOR_CYAN, COLOR_BLACK;
332 | init_pair 4, COLOR_YELLOW, COLOR_BLACK;
333 | my $MAGENTA = COLOR_PAIR(1);
334 | my $RED = COLOR_PAIR(2);
335 | my $CYAN = COLOR_PAIR(3);
336 | my $YELLOW = COLOR_PAIR(4);
337 | while (1) {
338 | $window_top->attron($CYAN);
339 | $window_top->box(0,0);
340 | $window_top->addstr(0, 30, " P r o g r e s s ");
341 | $window_top->attroff($CYAN);
342 | $window_top->addstr(LINES() - 9, 1, "$version");
343 | my $sources_format = "%-25s%-18s%-8s";
344 | $window_top->attron(A_BOLD);
345 | $window_top->addstr(1, 12,
346 | sprintf ($sources_format,
347 | "Source Medium", "Speed", "To Do"));
348 | $window_top->attroff(A_BOLD);
349 | my $line_number = 2;
350 | map {
351 | my $source = $_;
352 | $window_top->attron($CYAN);
353 | $window_top->attron($RED) if $done{$source};
354 | $window_top->
355 | addstr($line_number, 12,
356 | sprintf($sources_format,
357 | substr($source_roots{$source}, 0, 24),
358 | substr($speeds{$source}, 0, 17),
359 | substr($progress_ratios{$source}, -8, 8)));
360 | ++ $line_number;
361 | $window_top->addstr($line_number, 1,
362 | sprintf($sources_format, "", "", "", ""));
363 | $window_top->attroff($RED);
364 | $window_top->attroff($CYAN);
365 | } sort (keys %source_roots);
366 | $line_number = 0;
367 | map {
368 | $window_center->addstr($line_number, 2, $_);
369 | ++ $line_number;
370 | } @pokinom_banner;
371 | $window_center->move(0, 0);
372 |
373 | $window_bottom->box(0,0);
374 | $window_bottom->attron(A_BOLD);
375 | $window_bottom->
376 | addstr(1, 3,
377 | sprintf ("[F3]: Turn off now.%54s",
378 | $shut_down_when_done ? "Turning off when done. [F9]: Stay on."
379 | : "Staying on. [F9]: Turn off when done."));
380 | $window_bottom->attroff(A_BOLD);
381 |
382 | $window_top->noutrefresh();
383 | $window_bottom->noutrefresh();
384 | $window_center->noutrefresh(); # Last window gets the cursor.
385 | sleep 2;
386 | if (++ $redraw_window_count > 5) {
387 | $redraw_window_count = 0;
388 | redrawwin();
389 | }
390 | doupdate();
391 | act_on_keypress($window_bottom->getch());
392 | if (! grep(/0/, values %done) && $shut_down_when_done) {
393 | qx ($shut_down_action);
394 | }
395 | }
396 | endwin();
397 | };
398 | }
399 |
400 | sleep;
401 |
402 | # Tidy up. (Except we don't reach this.)
403 | map {
404 | $being_deleted_thread{$_}->join if $being_deleted_thread{$_};
405 | } keys %source_roots;
406 |
407 | $rsync_worker_thread->join if $rsync_worker_thread;
408 | $display_thread->join if $display_thread;
409 |
410 | __END__
411 |
412 |
--------------------------------------------------------------------------------
/test/test.sh:
--------------------------------------------------------------------------------
1 | #! /bin/bash
2 |
3 | # Caveats: - kills all killable instances of rsync, monikop, pokinom
4 | # - don't disturb test timing by putting too much (extra) load
5 | # on the machine
6 |
7 | killall --quiet rsync
8 |
9 | TESTDIR=/tmp/monikop-test
10 | DEV=$TESTDIR/dev
11 | MNT=$TESTDIR/mnt
12 | LOG=$TESTDIR/log
13 | RSYNC=$TESTDIR/rsync
14 | MONIKOP_1="../monikop ../test/monikop.config.test.1"
15 | MONIKOP_2="../monikop ../test/monikop.config.test.2"
16 | MONIKOP_3="../monikop ../test/monikop.config.test.3"
17 | POKINOM="../pokinom ../test/pokinom.config.test"
18 | TEST_COUNT=0
19 | FAIL_COUNT=0
20 | FAILED_TESTS=""
21 |
22 | function kill_rsyncd {
23 | kill `cat $TESTDIR/rsync/rsyncd.pid`
24 | }
25 |
26 | function start_rsyncd {
27 | kill_rsyncd 2> /dev/null
28 | rm -f $RSYNC/rsyncd.pid 2> /dev/null
29 | chmod o-rwx ../test/rsyncd.secrets.test
30 | rsync --daemon --config=../test/rsyncd.conf.test
31 | }
32 |
33 | # make_test_drive
34 | function make_test_drive {
35 | mkdir -p $MNT/$1
36 | dd if=/dev/zero of=$DEV/$1 bs=1024 count=$2 2> /dev/null
37 | /sbin/mkfs.ext3 -m 0 -Fq $DEV/$1
38 | if ! mount $MNT/$1 2> /dev/null; then
39 | echo "# Can't mount $DEV/$1 to $MNT/$1."
40 | echo "# Redo from start after adding the following line to your /etc/fstab:"
41 | echo
42 | echo " $DEV/$1 $MNT/$1 ext3 loop,user,noauto 0 0"
43 | echo
44 | return 1
45 | fi
46 | chmod a+w $MNT/$1
47 | }
48 |
49 | # make_test_file
50 | function make_test_file {
51 | mkdir -p `dirname "$1"`
52 | dd if=/dev/zero of="$1" bs=1024 count=$2 2> /dev/null
53 | echo "++++++++++++++++++++++++++$RANDOM***$3---$1" >> "$1"
54 | touch -t $3 $1
55 | }
56 |
57 | # find_and_compare ... :: ...
58 | function find_and_compare {
59 | ORIGIN_DIRS=$1; shift;
60 | until [[ $1 == "::" ]]; do
61 | ORIGIN_DIRS="$ORIGIN_DIRS $1"; shift;
62 | done
63 | shift
64 | COPY_DIRS=$@
65 | MISSING=""
66 | DIVERGING=""
67 | DIVERGING_MTIME=""
68 | RETURN_VALUE=0
69 | for ORIGIN_DIR in $ORIGIN_DIRS; do
70 | while read -r -d $'\0' ORIGIN_FILE; do
71 | ORIGIN_FILE_ESCAPED=${ORIGIN_FILE//\\/\\\\}
72 | ORIGIN_FILE_ESCAPED=${ORIGIN_FILE_ESCAPED//\[/\\[}
73 | for COPY_DIR in $COPY_DIRS; do
74 | FOUND=`find $COPY_DIR -path "$COPY_DIR/${ORIGIN_FILE_ESCAPED#$ORIGIN_DIR/}" -print0 2> /dev/null`
75 | if [[ -n "$FOUND" ]] ; then
76 | break
77 | fi
78 | done
79 | if [[ -z "$FOUND" ]] ; then
80 | MISSING="$MISSING $ORIGIN_FILE";
81 | elif ! cmp --quiet "$ORIGIN_FILE" "$FOUND"; then
82 | DIVERGING="$DIVERGING $ORIGIN_FILE"
83 | elif [[ `stat --printf="%Y" "$ORIGIN_FILE"` != `stat --printf="%Y" "$FOUND"` ]]; then
84 | DIVERGING_MTIME="$DIVERGING_MTIME $ORIGIN_FILE"
85 | fi
86 | done < <(find $ORIGIN_DIR -type f -print0 2> /dev/null)
87 | done
88 | if [[ -n $MISSING ]]; then
89 | RETURN_VALUE=1
90 | echo "MISSING: $MISSING"
91 | fi
92 | if [[ -n $DIVERGING ]]; then
93 | RETURN_VALUE=$((return_value + 2))
94 | echo "DIVERGING: $DIVERGING"
95 | fi
96 | if [[ -n $DIVERGING_MTIME ]]; then
97 | RETURN_VALUE=$((return_value + 4))
98 | echo "DIVERGING MTIME: $DIVERGING_MTIME"
99 | fi
100 | return $RETURN_VALUE
101 | }
102 |
103 | # run_test
104 | function run_test {
105 | sleep 4
106 | killall monikop pokinom 2> /dev/null
107 | sleep 2
108 | killall -KILL monikop pokinom 2> /dev/null
109 | sleep 2
110 | echo "RUNNING $2 [$3]"
111 | $2
112 | RETURN_VALUE=$?
113 | if [[ $1 != "ignore" ]]; then
114 | TEST_COUNT=$(( TEST_COUNT + 1 ))
115 | if [[ $RETURN_VALUE -ne $1 ]]; then
116 | FAIL_COUNT=$(( FAIL_COUNT + 1 ))
117 | FAILED_TESTS="$FAILED_TESTS$2($1? $RETURN_VALUE!) [$3]\n"
118 | echo "$2 should have returned $1 but returned $RETURN_VALUE instead."
119 | fi
120 | else
121 | echo "(DUMMY TEST, IGNORED)"
122 | fi
123 | sleep 2
124 | }
125 |
126 | # Create and mount test drives:
127 | umount $MNT/* #2> /dev/null
128 | rm -rf $DEV $MNT $LOG
129 | mkdir -p $DEV $MNT $RSYNC
130 |
131 | for i in 01 02 03 04; do
132 | make_test_drive $i 1024000
133 | if [[ $? == 1 ]]; then
134 | MOUNTING_PROBLEM=1
135 | fi
136 | done
137 | make_test_drive 05 3072000
138 | if [[ $? == 1 ]]; then
139 | MOUNTING_PROBLEM=1
140 | fi
141 | if [[ $MOUNTING_PROBLEM == 1 ]]; then exit; fi
142 |
143 | function fill_sources_with_big_files {
144 | for i in f1 f2 f3; do
145 | make_test_file $MNT/01/data/$i 250000 200703250845.33
146 | done
147 | for i in f10 f11 f12; do
148 | make_test_file $MNT/02/data/$i 250000 200703250845.33
149 | done
150 | for i in f4 f5 f6; do
151 | make_test_file $MNT/01/data/d1/$i 20000 200703250845.33
152 | make_test_file $MNT/01/data/d1/d2/$i 20000 200703250845.33
153 | done
154 | for i in f7 f8 f9; do
155 | make_test_file $MNT/02/data/d1/$i 20000 200703250845.33
156 | make_test_file $MNT/02/data/d1/d2/$i 20000 200703250845.33
157 | done
158 | }
159 |
160 | function fill_sources_with_hidden_files {
161 | for i in 01 02; do
162 | make_test_file $MNT/$i/data/.hidden_dir_$i/.hidden_file 20 200804250955.10
163 | done
164 | }
165 |
166 | function fill_sources_with_few_small_files {
167 | for i in 01 02; do
168 | for j in file_one file_two file_three; do
169 | make_test_file $MNT/$i/data/$j.$i 20 200004250955.10
170 | done
171 | done
172 | }
173 |
174 | function fill_destinations_with_few_small_files {
175 | for i in 03 04; do
176 | for j in file_one file_two file_three; do
177 | make_test_file $MNT/$i/measuring_data/$i/$j 20 200004250955.10
178 | done
179 | done
180 | }
181 |
182 | # Check how fast we are:
183 | fill_sources_with_big_files
184 | T1=`/usr/bin/time --format="%e" rsync --recursive --times $MNT/01/data/ $MNT/03/ 2>&1 &`
185 | T2=`/usr/bin/time --format="%e" rsync --recursive --times $MNT/02/data/ $MNT/04/ 2>&1 &`
186 | INTERRUPTION_TIME_0=`echo "($T1 + $T2) * 3" | bc`
187 | INTERRUPTION_TIME_1=`echo "($T1 + $T2) * .08" | bc`
188 | INTERRUPTION_TIME_2=`echo "($T1 + $T2) * .82" | bc`
189 | echo "One run of a testee takes about $INTERRUPTION_TIME_0 seconds."
190 | rm -rf $MNT/0{1,2,3,4}/*
191 |
192 | ######################################################################
193 | # Define tests:
194 | ######################################################################
195 |
196 | function test_monikop_simple {
197 | sleep 4
198 | $MONIKOP_1 & sleep $INTERRUPTION_TIME_0; /bin/kill -TERM $!
199 | sleep 2
200 | find_and_compare $MNT/0{1,2}/data :: $MNT/0{3,4}/measuring_data
201 | }
202 |
203 | function test_monikop_simple_late_sources {
204 | kill_rsyncd
205 | $MONIKOP_1 & sleep $INTERRUPTION_TIME_2;
206 | start_rsyncd; sleep $INTERRUPTION_TIME_0; /bin/kill -TERM $!
207 | sleep 2
208 | find_and_compare $MNT/0{1,2}/data :: $MNT/0{3,4}/measuring_data
209 | }
210 |
211 | function test_monikop_short {
212 | $MONIKOP_1 & sleep $INTERRUPTION_TIME_1; /bin/kill -TERM $!
213 | sleep 2
214 | find_and_compare $MNT/0{1,2}/data :: $MNT/0{3,4}/measuring_data
215 | }
216 |
217 | function test_monikop_short_2 {
218 | $MONIKOP_2 & sleep $INTERRUPTION_TIME_1; /bin/kill -TERM $!
219 | sleep 2
220 | find_and_compare $MNT/0{1,2}/data :: $MNT/0{3,4,5}/measuring_data
221 | }
222 |
223 | function test_monikop_short_kill_rsync_first {
224 | $MONIKOP_2 & sleep $INTERRUPTION_TIME_1; /usr/bin/killall -KILL rsync; sleep 1; /bin/kill -TERM $!
225 | sleep 2
226 | find_and_compare $MNT/0{1,2}/data :: $MNT/0{3,4,5}/measuring_data
227 | RETURN=$?
228 | start_rsyncd
229 | sleep 2
230 | return $RETURN
231 | }
232 |
233 | function test_monikop_short_cut_sources {
234 | $MONIKOP_2 & sleep $INTERRUPTION_TIME_1; kill_rsyncd; sleep 1; /bin/kill -TERM $!
235 | sleep 2
236 | find_and_compare $MNT/0{1,2}/data :: $MNT/0{3,4,5}/measuring_data
237 | RETURN=$?
238 | start_rsyncd
239 | sleep 2
240 | return $RETURN
241 | }
242 |
243 | function test_monikop_simple_2 {
244 | $MONIKOP_2 & sleep $INTERRUPTION_TIME_0; /bin/kill -TERM $!
245 | sleep 2
246 | find_and_compare $MNT/0{1,2}/data :: $MNT/0{3,4,5}/measuring_data
247 | }
248 |
249 | function test_monikop_simple_3 {
250 | $MONIKOP_3 & sleep $INTERRUPTION_TIME_0; /bin/kill -TERM $!
251 | sleep 2
252 | find_and_compare $MNT/0{1,2}/data :: $MNT/0{3,4}/measuring_data/dir_0{1,2}
253 | }
254 |
255 | function test_monikop_overflow {
256 | # Stuff one of the destinations a bit:
257 | make_test_file $MNT/03/stuffing 250000 199903250845
258 | $MONIKOP_1 & sleep $INTERRUPTION_TIME_0; /bin/kill -TERM $!
259 | sleep 2
260 | find_and_compare $MNT/0{1,2}/data :: $MNT/0{3,4}/measuring_data
261 | }
262 |
263 | function test_monikop_no_destination {
264 | # We test basically if there is something to kill.
265 | umount $MNT/{03,04}
266 | $MONIKOP_1 & sleep $INTERRUPTION_TIME_2; /bin/kill -TERM $!
267 | RETURN=$?
268 | mount $MNT/03
269 | mount $MNT/04
270 | return $RETURN
271 | }
272 |
273 | function test_monikop_no_source {
274 | # We test basically if there is something to kill.
275 | kill_rsyncd
276 | $MONIKOP_1 & sleep $INTERRUPTION_TIME_2; /bin/kill -TERM $!
277 | RETURN=$?
278 | start_rsyncd
279 | return $RETURN
280 | }
281 |
282 | function test_pokinom_clean_finish {
283 | $POKINOM & sleep $INTERRUPTION_TIME_0; /bin/kill -TERM $!
284 | sleep 2
285 | find_and_compare $MNT/0{1,2}/data :: $MNT/05/NEW_DATA
286 | }
287 |
288 | function test_pokinom_short {
289 | $POKINOM & sleep $INTERRUPTION_TIME_1; /bin/kill -TERM $!
290 | sleep 2
291 | find_and_compare $MNT/0{1,2}/data :: $MNT/05/NEW_DATA
292 | }
293 |
294 | function test_pokinom_late_destination {
295 | kill_rsyncd
296 | $POKINOM & sleep $INTERRUPTION_TIME_2; start_rsyncd; sleep $INTERRUPTION_TIME_0; /bin/kill -TERM $!
297 | sleep 2
298 | find_and_compare $MNT/0{1,2}/data :: $MNT/05/NEW_DATA
299 | }
300 |
301 | function test_dirs_backed_up {
302 | test -d $MNT/03/backed_up && test -d $MNT/04/backed_up
303 | }
304 |
305 | function test_monikop_deletes_being_deleted_dir {
306 | mkdir -p $MNT/0{3,4}/{being_deleted,backed_up}
307 | touch $MNT/0{3,4}/{being_deleted,backed_up}/some_file
308 | touch $MNT/0{3,4}/{being_deleted,backed_up}/.some_hidden_file
309 | $MONIKOP_1 & sleep $INTERRUPTION_TIME_2; /bin/kill -TERM $!
310 | test -d $MNT/03/being_deleted || test -d $MNT/04/being_deleted
311 | }
312 |
313 | function test_pokinom_deletes_being_deleted_dir {
314 | mkdir -p $MNT/0{3,4}/being_deleted
315 | touch $MNT/0{3,4}/being_deleted/some_file
316 | touch $MNT/0{3,4}/being_deleted/.some_hidden_file
317 | $POKINOM & sleep $INTERRUPTION_TIME_2; /bin/kill -TERM $!
318 | test -d $MNT/03/being_deleted || test -d $MNT/04/being_deleted
319 | }
320 |
321 | function test_pokinom_newer_files_win {
322 | fill_destinations_with_few_small_files
323 | $POKINOM & sleep $INTERRUPTION_TIME_2; /bin/kill -TERM $!
324 | for i in 03 04; do
325 | mv $MNT/$i/backed_up $MNT/$i/measuring_data
326 | touch $MNT/$i/measuring_data/$i/*
327 | done
328 | $POKINOM & sleep $INTERRUPTION_TIME_2; /bin/kill -TERM $!
329 | sleep 2
330 | find_and_compare $MNT/0{3,4}/backed_up :: $MNT/05/NEW_DATA
331 | }
332 |
333 | function test_pokinom_older_files_lose {
334 | fill_destinations_with_few_small_files
335 | $POKINOM & sleep $INTERRUPTION_TIME_2; /bin/kill -TERM $!
336 | for i in 03 04; do
337 | mv $MNT/$i/backed_up $MNT/$i/measuring_data
338 | done
339 | touch -t 198001011200.00 $MNT/03/measuring_data/03/file_one
340 | $POKINOM & sleep $INTERRUPTION_TIME_2; /bin/kill -TERM $!
341 | sleep 2
342 | find_and_compare $MNT/0{3,4}/backed_up :: $MNT/05/NEW_DATA
343 | }
344 |
345 | ######################################################################
346 | # Run the tests:
347 | ######################################################################
348 | start_rsyncd
349 |
350 | #########################
351 | ## Run tests: Monikop
352 | #########################
353 |
354 | fill_sources_with_big_files
355 |
356 | run_test 1 test_monikop_deletes_being_deleted_dir "Monikop deletes left-over directory named being_deleted."
357 |
358 | rm -rf $MNT/0{3,4}/* $LOG
359 |
360 | chmod a-w,a-x $MNT/0{3,4}
361 | run_test 1 test_monikop_simple "Unwritable destination"
362 | chmod a+w,a+x $MNT/0{3,4}
363 | run_test 0 test_monikop_simple "No-longer-unwritable destination"
364 |
365 | rm -rf $MNT/0{3,4}/* $LOG
366 |
367 | run_test 0 test_monikop_simple_3 "Source-specific directories on disks"
368 |
369 | rm -rf $MNT/0{3,4}/* $LOG
370 |
371 | run_test 0 test_monikop_simple_late_sources "Simple run, sources coming up late."
372 |
373 | mv $MNT/03/measuring_data $MNT/03/backed_up
374 | mv $MNT/04/measuring_data $MNT/04/backed_up
375 | rm -rf $LOG
376 |
377 | run_test 0 test_monikop_simple "Simple run, deletion."
378 |
379 | rm -rf $MNT/0{3,4}/* $LOG
380 |
381 | run_test 1 test_monikop_short "Interruption, finished.* or finished.*.bak deleted."
382 | rm -f $LOG/finished.rsync___localhost_2000_test_01_data $LOG/finished.rsync___localhost_2000_test_02_data.bak
383 | run_test 0 test_monikop_simple "Recovery after interruption, finished.* or finished.*.bak deleted."
384 |
385 | rm -rf $MNT/0{3,4}/* $LOG
386 |
387 | run_test 1 test_monikop_short "Interruption, finished.* and/or log.* deleted."
388 | rm -f $LOG/finished.rsync___localhost_2000_test_01_data $LOG/log.rsync___localhost_2000_test_01_data
389 | rm -f $LOG/rm log.rsync___localhost_2000_test_02_data
390 | run_test 0 test_monikop_simple "Recovery after interruption, finished.* and/or log.* deleted."
391 |
392 | rm -rf $MNT/0{3,4}/* $LOG
393 |
394 | run_test 1 test_monikop_short_2 "Repeated interruption (1)."
395 | run_test ignore test_monikop_short_2 "Repeated interruption (2) (No test, side effect only)."
396 | run_test 0 test_monikop_simple_2 "Repeated interruption (3)."
397 |
398 | mv $MNT/03/measuring_data $MNT/03/backed_up
399 | mv $MNT/04/measuring_data $MNT/04/backed_up
400 | mv $MNT/05/measuring_data $MNT/05/backed_up
401 | rm -rf $LOG
402 |
403 | run_test 1 test_monikop_short_2 "Repeated interruption, deletion (1)."
404 | run_test ignore test_monikop_short_2 "Repeated interruption, deletion (2) (No test, side effect only)."
405 | run_test 0 test_monikop_simple_2 "Repeated interruption, deletion (3)."
406 |
407 | rm -rf $MNT/0{3,4,5}/* $LOG
408 |
409 | run_test 0 test_monikop_no_destination "No destination available."
410 | run_test 0 test_monikop_no_source "No destination available."
411 |
412 | rm -rf $MNT/0{3,4}/* $LOG
413 |
414 | run_test 1 test_monikop_short_kill_rsync_first "Rsync killed."
415 | ps aux | grep rsync
416 | run_test 0 test_monikop_simple_2 "Rsync killed."
417 |
418 | rm -rf $MNT/0{3,4,5}/* $LOG
419 |
420 | run_test 1 test_monikop_short_cut_sources "Connection to source destroyed."
421 | run_test 0 test_monikop_simple_2 "Connection to source destroyed."
422 |
423 | rm -rf $MNT/0{3,4,5}/* $LOG
424 |
425 | fill_sources_with_few_small_files
426 |
427 | run_test 0 test_monikop_simple "Don't re-rsync after deletion of finished.* (Preparation #1)."
428 | rm -rf $MNT/{03,04}/*
429 | run_test 1 test_monikop_short "Don't re-rsync after deletion of finished.* (Preparation #2, fill finished.*)."
430 | rm -f $LOG/log.rsync___localhost_2000_test_*
431 | rm -f $LOG/finished.rsync___localhost_2000_test_*_data
432 | run_test 1 test_monikop_short "Don't re-rsync after deletion of finished.*"
433 | rm -rf $MNT/0{3,4}/* $LOG
434 | run_test 0 test_monikop_simple "Don't re-rsync after deletion of finished.*.bak (Preparation #1)."
435 | rm -rf $MNT/{03,04}/*
436 | run_test 1 test_monikop_short "Don't re-rsync after deletion of finished.*.bak (Preparation #2, fill finished.*)."
437 | rm -f $LOG/log.rsync___localhost_2000_test_*
438 | rm -f $LOG/finished.rsync___localhost_2000_test_*_data.bak
439 | run_test 1 test_monikop_short "Don't re-rsync after deletion of finished.*.bak."
440 |
441 | rm -rf $MNT/0{3,4}/* $LOG
442 |
443 | ##################################################
444 | # Run tests: Pokinom
445 | ##################################################
446 |
447 | run_test 1 test_pokinom_deletes_being_deleted_dir "Pokinom deletes left-over directory named being_deleted."
448 |
449 | rm -rf $MNT/0{3,4,5}/*
450 |
451 | run_test 0 test_pokinom_newer_files_win "Pokinom overwrites older files in Destination."
452 |
453 | run_test 4 test_pokinom_older_files_lose "Pokinom discards older files on removable disk."
454 |
455 | ##################################################
456 | # Run tests: Monikop and Pokinom together
457 | ##################################################
458 |
459 | rm -rf $MNT/0{1,2,3,4,5}/*
460 | fill_sources_with_hidden_files
461 |
462 | run_test 0 test_monikop_simple "Preparation for simple Pokinom test, hidden files."
463 | run_test 0 test_pokinom_clean_finish "Simple Pokinom test, hidden files."
464 | run_test 0 test_dirs_backed_up "Simple Pokinom test, hidden files."
465 | run_test 1 test_monikop_short "After test with hidden files, this one should do nothing but delete backed_up."
466 | run_test 1 test_dirs_backed_up "Deletion of backed_up with hidden files."
467 |
468 | rm -rf $MNT/0{1,2,3,4,5}/*
469 | fill_sources_with_big_files
470 |
471 | run_test 0 test_monikop_simple "Simple run in preparation for simple Pokinom test."
472 | run_test 0 test_pokinom_clean_finish "Simple Pokinom test."
473 | run_test 0 test_dirs_backed_up "Simple Pokinom test: directories renamed?."
474 |
475 | rm -rf $MNT/05/* $LOG
476 |
477 | run_test 0 test_monikop_simple "Preparation for Pokinom's destination overfull."
478 | # Stuff destination:
479 | make_test_file $MNT/05/stuffing 2000000 199903250845
480 | run_test 1 test_pokinom_clean_finish "Pokinom's destination overfull."
481 | rm $MNT/05/stuffing
482 | run_test 0 test_pokinom_clean_finish "Pokinom's destination no longer overfull: recovering."
483 |
484 | rm -rf $MNT/05/* $LOG
485 |
486 | run_test 0 test_monikop_simple "Simple run in preparation for Pokinom, late destination."
487 | run_test 0 test_pokinom_late_destination "Pokinom, late destination."
488 |
489 | rm -rf $MNT/05/* $LOG
490 |
491 | run_test 0 test_monikop_simple "Simple run in preparation for Pokinom stopped early."
492 | run_test 1 test_pokinom_short "Pokinom stopped early."
493 | run_test 0 test_monikop_simple "Simple run after Pokinom having been stopped early."
494 | run_test 0 test_pokinom_clean_finish "Simple run after Pokinom having been stopped early."
495 |
496 | rm -rf $MNT/05/* $LOG
497 |
498 | run_test 0 test_monikop_simple "Simple run in preparation for \"file grown too large\""
499 | rm $MNT/01/data/f3
500 | cat $MNT/01/data/f1 >> $MNT/01/data/f2
501 | run_test 2 test_monikop_simple "Repeated run, file grown too large (1)."
502 | run_test 2 test_pokinom_clean_finish "Repeated run, file grown too large (2)."
503 | run_test 1 test_monikop_simple "Repeated run, file grown too large (3)."
504 | run_test 0 test_pokinom_clean_finish "Repeated run, file grown too large (4)."
505 |
506 | rm -rf $MNT/05/* $LOG
507 |
508 | run_test 1 test_monikop_overflow "Initially, too little room on disks (1)."
509 | run_test 1 test_pokinom_clean_finish "Initially, too little room on disks (2)."
510 | run_test 1 test_monikop_overflow "Previously, too little room on disks (1)."
511 | run_test 0 test_pokinom_clean_finish "Previously, too little room on disks (2)."
512 |
513 | rm -rf $MNT/0{3,4,5}/* $LOG
514 |
515 | run_test 1 test_monikop_short "Unfinished by Monikop, then another full cycle."
516 | run_test ignore test_pokinom_clean_finish "Unfinished by Monikop, then another full cycle (Outcome unpredictable)."
517 | run_test ignore test_monikop_simple "Previously unfinished by Monikop, now another full cycle (Outcome unpredictable)."
518 | run_test 0 test_pokinom_clean_finish "Previously unfinished by Monikop, now another full cycle."
519 |
520 | rm -rf $MNT/0{1,2,3,4,5}/* $LOG
521 |
522 | make_test_file $MNT/01/data/d1/f1 10 200703250845.33
523 | make_test_file $MNT/01/data/d2/f3 10 200703250845.33
524 | make_test_file $MNT/01/data/f4 10 200703250845.33
525 | make_test_file $MNT/01/data/f5 10 200703250845.33
526 | make_test_file $MNT/01/data/f6 10 200703250845.33
527 | make_test_file $MNT/01/data/f7 10 200703250845.33
528 | make_test_file $MNT/01/data/f8 10 200703250845.33
529 | make_test_file $MNT/01/data/f9 10 200703250845.33
530 | make_test_file $MNT/01/data/f10 10 200703250845.33
531 | make_test_file $MNT/01/data/.f11 10 200703250845.33
532 | make_test_file $MNT/01/data/d3/d4/f4 10 200703250845.33
533 | mv $MNT/01/data/d1/f1 "$MNT/01/data/d1/Große Datei"
534 | mv $MNT/01/data/d1 "$MNT/01/data/Schönes Verzeichnis"
535 | mv $MNT/01/data/f4 "$MNT/01/data/[square brackets]"
536 | mv $MNT/01/data/f5 "$MNT/01/data/\`backquotes\`"
537 | mv $MNT/01/data/f6 "$MNT/01/data/'single quotes'"
538 | mv $MNT/01/data/f7 "$MNT/01/data/\"double quotes\""
539 | mv $MNT/01/data/f8 "$MNT/01/data/b\\a\\ckslashes"
540 | ## Won't work:
541 | #mv $MNT/01/data/f9 "`echo -e "$MNT/01/data/newlines\nin\nname"`";
542 | #mv $MNT/01/data/d2 $MNT/01/data/.rsync_partial
543 | #mv $MNT/01/data/d3/d4 $MNT/01/data/d3/.rsync_partial
544 | run_test 0 test_monikop_simple "Weird file names."
545 | run_test 0 test_pokinom_clean_finish "Weird file names."
546 | run_test 1 test_monikop_short "Weird file names, second run: nothing to do."
547 |
548 | ########################################
549 | # End of tests
550 | ########################################
551 | kill_rsyncd
552 |
553 | echo "TOTAL NUMBER OF TESTS: $TEST_COUNT"
554 | echo "NUMBER OF FAILED TESTS: $FAIL_COUNT"
555 | echo "FAILED TESTS:"
556 | echo -e "$FAILED_TESTS"
557 |
558 | exit $FAIL_COUNT
559 |
--------------------------------------------------------------------------------
/monikop:
--------------------------------------------------------------------------------
1 | #! /usr/bin/perl
2 | use strict;
3 | use warnings;
4 | use integer;
5 | use File::Rsync;
6 | use File::Basename;
7 | use Thread 'async';
8 | use threads::shared;
9 | use Curses;
10 |
11 | my @monikop_banner = (
12 | " _/ _/ _/_/ _/ _/ _/_/_/ _/ _/ _/_/ _/_/_/ ",
13 | " _/_/ _/_/ _/ _/ _/_/ _/ _/ _/ _/ _/ _/ _/ _/",
14 | " _/ _/ _/ _/ _/ _/ _/ _/ _/ _/_/ _/ _/ _/_/_/ ",
15 | " _/ _/ _/ _/ _/ _/_/ _/ _/ _/ _/ _/ _/ ",
16 | "_/ _/ _/_/ _/ _/ _/_/_/ _/ _/ _/_/ _/ ",
17 | );
18 |
19 | # Version number. Should agree with Pokinom's one.
20 | # Format: v<1>.<2>.<3> where
21 | # <3> = bug fix,
22 | # <2> = new feature,
23 | # <1> = incompatible change.
24 | my $version = 'v0.1.2';
25 |
26 | # Debug mode:
27 | # 0 = clean UI; 1 = lots of scrolling junk; anything else = both (pipe to file).
28 | my $debug = 0;
29 | $debug = $ARGV[1] if $ARGV[1];
30 |
31 | # Where to read local configuration:
32 | my $monikop_config = '~/monikop/monikop.config';
33 | $monikop_config = $ARGV[0] if $ARGV[0];
34 |
35 | ########################################
36 | # Settings
37 | ########################################
38 | # Possible data sources, and by what directory name to represent them in
39 | # destination.
40 | # When the latter is not unique, care must be taken that all pathnames in the
41 | # respective sources are unique.
42 | my %sources;
43 |
44 | # Possible mount points of data destinations. Must be unique.
45 | my @usable_mount_points;
46 |
47 | # Common directory (under a mount point) to put new data in.
48 | # Must agree with Pokinom's setting.
49 | my $path_under_mount_point;
50 |
51 | # Directories (under any mount point) of this name will be deleted
52 | # Must agree with Pokinom's setting.
53 | my $path_under_mount_point_backed_up;
54 |
55 | # Directory name (under a mount point) while being deleted.
56 | # Must agree with Pokinom's setting.
57 | my $path_under_mount_point_being_deleted;
58 |
59 | # Path and file name prefix of rsync's raw logs:
60 | my $rsync_log_prefix;
61 |
62 | # Path and file name prefix of the list of successfully rsynced files:
63 | my $finished_prefix;
64 |
65 | # How to suffix the name of the duplicate of a safe file:
66 | my $safe_file_backup_suffix;
67 |
68 | # How to suffix the name of an unfinished safe file:
69 | my $safe_file_unfinished_suffix;
70 |
71 | # What to do (shutdown) when F3 has been pressed:
72 | my $key_f3_action;
73 |
74 | # What to do (reboot) when F6 has been pressed:
75 | my $key_f6_action;
76 |
77 | # Rsync's time (in seconds) to wait for a response:
78 | my $rsync_timeout;
79 |
80 | # Rsync's directory (relative to destination) for partially transferred files.
81 | # Must agree with Pokinom's setting.
82 | my $rsync_partial_dir_name;
83 |
84 | # Put actual values into the above.
85 | eval `cat $monikop_config`;
86 |
87 | # Time in seconds before rsync is restarted and user information is
88 | # recalculated:
89 | my $coffee_break = 10;
90 |
91 | # Places to store run-time information to share between threads:
92 | my %speeds :shared; # rsync output
93 | my %progress_ratios :shared; # rsync output
94 | my %destination_usages :shared; # i.e. used/unused
95 | my %destination_usage_ratios :shared;
96 | my %destination_source_is_writing_to :shared;
97 | my %reachable :shared;
98 |
99 | my $kludge; # Don't ask
100 |
101 | sub debug_print { if ($debug) { print "\n"; print @_; } };
102 |
103 | # Return the hash referenced by argument, which is sorted if accessed as an
104 | # array:
105 | sub sort_hash {
106 | my %hash_table = @_;
107 | my @sorted_hash = ();
108 | foreach my $key (sort keys %hash_table) {
109 | push @sorted_hash, $key, $hash_table{$key};
110 | }
111 | @sorted_hash;
112 | }
113 |
114 | # Turn a path into a legal perl identifier:
115 | sub make_key_from_path {
116 | my $path = shift;
117 | ($path) =~ s/\/?(.*)\/?/$1/g;
118 | ($path) =~ s/\W/_/g;
119 | $path;
120 | }
121 |
122 | my %source_roots;
123 | map {
124 | $source_roots{make_key_from_path $_} = $_
125 | } keys %sources;
126 |
127 | my %source_dirs_in_destination;
128 | map {
129 | $source_dirs_in_destination{make_key_from_path $_} = $sources{$_}
130 | } keys %sources;
131 |
132 | # Crudely turn date string(s) into a number. Chronological order is preserved.
133 | sub normalize_date {
134 | my $date = join '', @_;
135 | $date =~ tr/ \/:-//d;
136 | $date;
137 | }
138 |
139 | # Return sorted intersection of arrays which are supposed to have unique
140 | # elements:
141 | sub intersection {
142 | my @intersection = ();
143 | my %count = ();
144 | my $element;
145 | foreach $element (@_) { $count{$element}++ }
146 | foreach $element (keys %count) {
147 | push @intersection, $element if $count{$element} > 1;
148 | }
149 | sort @intersection;
150 | }
151 |
152 | # Write @content to a file with name $filename or a name starting with
153 | # $filename and ending with $safe_file_backup_suffix. Leave at least one such
154 | # file, even if interrupted.
155 | sub safe_write {
156 | my ($filename, @content) = @_;
157 | my $filename_a = $filename;
158 | my $filename_b = $filename . $safe_file_backup_suffix;
159 | my $filename_unfinished = $filename . $safe_file_unfinished_suffix;
160 | local (*FILE_UNFINISHED);
161 | open FILE_UNFINISHED, '>', $filename_unfinished
162 | or die "[" . $$ . "] open $filename_unfinished failed: $!\n";
163 | print FILE_UNFINISHED @content;
164 | close FILE_UNFINISHED;
165 | qx(cp $filename_unfinished $filename_b);
166 | qx(mv $filename_unfinished $filename_a);
167 | }
168 |
169 | # Put contents of $filename into an array:
170 | sub read_list {
171 | my ($filename) = @_;
172 | local (*FILE);
173 | open FILE, '<', $filename
174 | or warn "[" . $$ . "] open $filename failed: $!\n";
175 | my @value = ;
176 | close FILE;
177 | @value;
178 | }
179 |
180 | # Read a file written by safe_write
181 | sub safe_read {
182 | my ($filename) = @_;
183 | my $filename_a = $filename;
184 | my $filename_b = $filename . $safe_file_backup_suffix;
185 | if (stat $filename_a) { $filename = $filename_a }
186 | elsif (stat $filename_b) { $filename = $filename_b }
187 | else { return () }
188 | debug_print "SAFE_READ: $filename";
189 | read_list $filename;
190 | }
191 |
192 | my @destination_roots;
193 | my %rsync_outfun;
194 | my %rsync;
195 | my %rsync_exec_form;
196 | my %rsync_dir;
197 | my %rsync_dir_exec_form;
198 | my %rsync_dir_err_form;
199 | my %rsync_worker_thread;
200 | my %being_deleted_thread;
201 | my $destinations_monitor_thread;
202 | my $display_thread;
203 |
204 | sub rsync_preparation_form {
205 | my ($source) = @_;
206 | $speeds{$source} = "-";
207 | join ( '',
208 | "\n",
209 | ########## Capture rsync's status messages for use by UI
210 | '$rsync_outfun{\'', $source, '\'} = sub {',
211 | ' my ($outline, $outputchannel) = @_ ; ',
212 | ' my ($speed) = $outline =~ /\d+\s+\d+%\s+(\S+)/; ',
213 | ' my ($progress_ratio) = ',
214 | ' $outline =~ /.+to-check=(\d+\/\d+)\)$/; ',
215 | ' if ($speed and $outputchannel eq \'out\') {',
216 | ' $speeds{\'', $source, '\'} = $speed;',
217 | ' } else {',
218 | ' $speeds{\'', $source, '\'} = "-";',
219 | ' };',
220 | ' if ($progress_ratio and $outputchannel eq \'out\') {',
221 | ' $progress_ratios{\'', $source, '\'} = $progress_ratio;',
222 | ' } ;',
223 | '};',
224 | "\n",
225 | ########## Run rsync: main worker
226 | '$rsync{\'', $source, '\'} = File::Rsync->new; ',
227 | ########## Return fodder for another eval
228 | '$rsync_exec_form{\'', $source, '\'} = sub {',
229 | ' my ($complete_destination) = @_;',
230 | ' \'$rsync{\\\'', $source, '\\\'}->exec(',
231 | ' {',
232 | ' src => \\\'', $source_roots{$source}, '/\\\', ',
233 | ' dest => \\\'\' . $complete_destination . \'/\\\', ',
234 | ' outfun => $rsync_outfun{\\\'', $source, '\\\'}, ',
235 | ' progress => 1, debug => 0, verbose => 0, ',
236 | ' filter => [\\\'merge,- ', $finished_prefix, $source,
237 | '\\\'], ',
238 | ' literal => [',
239 | ' \\\'--recursive\\\', \\\'--times\\\', ',
240 | ' \\\'--partial-dir=',
241 | $rsync_partial_dir_name, '\\\', ',
242 | ' \\\'--timeout=', $rsync_timeout, '\\\', ',
243 | ' \\\'--prune-empty-dirs\\\', ',
244 | ' \\\'--log-file-format=%i %b %l %M %n\\\', ',
245 | join (', ',
246 | map {
247 | '\\\'--compare-dest=' . $_ . '/'
248 | . $path_under_mount_point . '/'.
249 | $source_dirs_in_destination{$source}
250 | . '/\\\''
251 | }
252 | ( @destination_roots )),
253 | ' , \\\'--log-file=', $rsync_log_prefix, $source, '\\\'] ',
254 | ' }',
255 | ' );\' ',
256 | '};',
257 | "\n",
258 | ########## Run rsync: get directory from source
259 | '$rsync_dir{\'', $source, '\'} = File::Rsync->new; ',
260 | ########## Return fodder for another eval: dir
261 | '$rsync_dir_exec_form{\'', $source, '\'} = sub {',
262 | ' \'$rsync_dir{\\\'', $source, '\\\'}->list(',
263 | ' {',
264 | ' src => \\\'', $source_roots{$source}, '/\\\', ',
265 | ' literal => [ \\\'--recursive\\\', ',
266 | ' \\\'--no-human-readable\\\', ',
267 | ' \\\'--timeout=', $rsync_timeout, '\\\'] ',
268 | ' }',
269 | ' );\' ',
270 | '};',
271 | "\n",
272 | ########## Return fodder for another eval: error code from last rsync call
273 | '$rsync_dir_err_form{\'', $source, '\'} = sub {',
274 | ' \'$rsync_dir{\\\'', $source, '\\\'}->err();\' ',
275 | '}',
276 | "\n"
277 | )};
278 |
279 | sub act_on_keypress {
280 | my ($pressed_key) = @_;
281 | if ($pressed_key eq 267) { qx($key_f3_action) }
282 | elsif ($pressed_key eq 270) { qx($key_f6_action); }
283 | }
284 |
285 | # Run rsync for one $source, try all destinations:
286 | sub rsync_someplace {
287 | my ($source, @destinations) = @_;
288 | my $success;
289 | $kludge = $rsync{$source};
290 | $kludge = $rsync_outfun{$source};
291 | my $rsync_log_name = $rsync_log_prefix . $source;
292 | my $finished_name = $finished_prefix . $source;
293 | foreach (@destinations) {
294 | $destination_source_is_writing_to{$source} = $_;
295 | my $common_destination = $_ . '/' . $path_under_mount_point;
296 | my $complete_destination = $common_destination . '/'
297 | . $source_dirs_in_destination{$source};
298 | qx(mkdir -p $common_destination);
299 | if ($?) { die "Fatal: $common_destination is not writable."}
300 | if (eval ($rsync_exec_form{$source} ($complete_destination))) {
301 | debug_print "EVAL RSYNC_EXEC_FORM (successful) $source,\ $complete_destination: $@ \n";
302 | $success = 1;
303 | last; # unnecessary reruns would put empty
304 | # dirs into otherwise unused destinations
305 | } else {
306 | debug_print "EVAL RSYNC_EXEC_FORM (failed) $source, $complete_destination: $@ \n";
307 | $success = 0;
308 | }
309 | }
310 | $success;
311 | }
312 |
313 | $SIG{TERM} = sub {
314 | $display_thread->kill('TERM')->join;
315 | die "Caught signal $_[0]";
316 | };
317 |
318 |
319 | # Preparations done; sleeves up!
320 |
321 | # Make sure we have dirs to put our logs in:
322 | map {
323 | my ($filename, $directory) = fileparse $_;
324 | qx(mkdir -p $directory);
325 | } ( $rsync_log_prefix, $finished_prefix );
326 |
327 | # Find usable destinations:
328 | my @raw_mount_points = grep (s/\S+ on (.*) type .*/$1/, qx/mount/);
329 | chomp @raw_mount_points;
330 | @destination_roots = intersection @raw_mount_points, @usable_mount_points;
331 | debug_print "DESTINATION_ROOTS:\n";
332 | debug_print @destination_roots;
333 |
334 | # Clean up destinations:
335 | map {
336 | my $p_i_d = $_ . '/' . $path_under_mount_point;
337 | my $p_i_d_backed_up = $_ . '/' . $path_under_mount_point_backed_up;
338 | my $p_i_d_being_deleted = $_ . '/' . $path_under_mount_point_being_deleted;
339 | if (-d $p_i_d_backed_up and -d $p_i_d_being_deleted) {
340 | warn "[" . $$ . "] " .
341 | "Both $p_i_d_backed_up and $ p_i_d_being_deleted exist.\n" .
342 | "This does not normally happen.\n" .
343 | "I'm deleting $p_i_d_being_deleted. Be patient.\n";
344 | qx(rm -rf $p_i_d_being_deleted);
345 | }
346 | qx(mv -f $p_i_d_backed_up $p_i_d_being_deleted 2> /dev/null);
347 | $being_deleted_thread{$_} = async {
348 | $SIG{TERM} = sub { threads->exit() };
349 | qx(rm -rf $p_i_d_being_deleted); };
350 | } @destination_roots;
351 |
352 | if (scalar @destination_roots) {
353 | # Set up and start things per source_root:
354 | map {
355 | # rotate for crude load balancing:
356 | push (@destination_roots, shift (@destination_roots));
357 | $progress_ratios{$_} = "?"; # Initialize for UI
358 |
359 | debug_print 'rsync_preparation_form:' .
360 | rsync_preparation_form ($_). "\n";
361 | eval rsync_preparation_form $_;
362 | debug_print "EVAL RSYNC_PREPARATION_FORM $_: $@ \n";
363 |
364 | $rsync_worker_thread{$_} = async {
365 | $SIG{TERM} = sub { threads->exit() };
366 | my $rsync_log_name = $rsync_log_prefix . $_;
367 | my $finished_name = $finished_prefix . $_;
368 | while (1) {
369 | $kludge = $rsync_dir{$_};
370 | debug_print 'rsync_dir_exec_form $_:'.
371 | $rsync_dir_exec_form{$_} () . "\n";
372 | my @rsync_ls = eval $rsync_dir_exec_form{$_}();
373 | $reachable{$_} = eval $rsync_dir_err_form{$_}() ? 0 : 1;
374 | debug_print "REACHABLE: $reachable{$_}\n";
375 | if ($reachable{$_}) {
376 | my %old_finished = safe_read $finished_name;
377 | if (-f $rsync_log_name) {
378 | my @rsync_log = read_list $rsync_log_name;
379 | foreach (@rsync_log) {
380 | my ($file_length, $modification_time, $filename) =
381 | /[\d\/\s:\[\]]+ [>c\.][fd]\S{9} \d+ (\d+) ([\d\/:-]+) (.*)$/;
382 | if ($filename) {
383 | $old_finished{$filename . "\n"} =
384 | "### " . $modification_time . " " .
385 | $file_length . "\n";
386 | }
387 | }
388 | safe_write $finished_name, sort_hash %old_finished;
389 | unlink $rsync_log_name unless $debug;
390 | }
391 | my %finished = ();
392 | # Delete from %old_finished what has to be re-rsynced.
393 | foreach (@rsync_ls) {
394 | my ($ls_size, $ls_modification_date,
395 | $ls_modification_time, $ls_filename) =
396 | /[drwx-]+\s+(\d+) ([\d\/]+) ([\d:]+) (.*)/;
397 | if ($ls_filename &&
398 | exists $old_finished{$ls_filename . "\n"}) {
399 | my ($finished_modification_date, $finished_size) =
400 | $old_finished{$ls_filename . "\n"} =~
401 | /### (\S+) (\d+)$/;
402 | if ( ($finished_size eq $ls_size)
403 | && (normalize_date
404 | ($finished_modification_date)
405 | eq normalize_date
406 | ($ls_modification_date,
407 | $ls_modification_time)) )
408 | {
409 | $finished{$ls_filename . "\n"} =
410 | $old_finished{$ls_filename . "\n"};
411 | }
412 | }
413 | }
414 | safe_write $finished_name, %finished;
415 | if (rsync_someplace $_, @destination_roots) {
416 | $progress_ratios{$_} = '0'; # Clean staleness for UI
417 | }
418 | sleep $coffee_break;
419 | }
420 | }
421 | }
422 | } keys %source_roots;
423 | }
424 |
425 | # Provide some reassuring user information:
426 | $destinations_monitor_thread = async {
427 | $SIG{TERM} = sub { threads->exit() };
428 | while () {
429 | map {
430 | my $destination_root = $_;
431 | my $destination_usage = 0;
432 | map {
433 | my $source_root = $_;
434 | my $complete_destination = $destination_root . '/'
435 | . $path_under_mount_point . '/'
436 | . $source_dirs_in_destination{$source_root};
437 | my @dir = qx(ls -A $complete_destination/ 2> /dev/null);
438 | $destination_usage = 1 if scalar @dir; # 0 = no new data
439 | } keys %source_roots;
440 | $destination_usages{$destination_root} = $destination_usage;
441 | my @destination_usage_ratio =
442 | grep s/\S+\s+\S+\s+\S+\s+\S+\s+(\d*)%\s+\S+/$1/, qx(df -P $_);
443 | chomp @destination_usage_ratio;
444 | ($destination_usage_ratios{$_}) = @destination_usage_ratio;
445 | } @destination_roots;
446 | sleep $coffee_break;
447 | }
448 | };
449 |
450 | unless ($debug == 1) {
451 | # Talk to the user.
452 | $display_thread = async {
453 | $SIG{TERM} = sub {
454 | endwin(); # Leave a usable terminal.
455 | threads->exit()
456 | };
457 |
458 | my $redraw_window_count = 0;
459 | initscr();
460 | cbreak();
461 | noecho();
462 | curs_set(0);
463 | my $window_left = newwin(LINES() -8, 29, 0, 0);
464 | my $window_right = newwin(LINES() -8, 50, 0, 29);
465 | my $window_center = newwin(5, 79, LINES() -8, 0);
466 | my $window_bottom = newwin(3, 79, LINES() -3, 0);
467 | $window_bottom->keypad(1);
468 | $window_bottom->nodelay(1);
469 | start_color;
470 | init_pair 1, COLOR_MAGENTA, COLOR_BLACK;
471 | init_pair 2, COLOR_RED, COLOR_BLACK;
472 | init_pair 3, COLOR_CYAN, COLOR_BLACK;
473 | init_pair 4, COLOR_YELLOW, COLOR_BLACK;
474 | my $MAGENTA = COLOR_PAIR(1);
475 | my $RED = COLOR_PAIR(2);
476 | my $CYAN = COLOR_PAIR(3);
477 | my $YELLOW = COLOR_PAIR(4);
478 |
479 | while (1) {
480 | $window_left->attron($CYAN);
481 | $window_left->box(0, 0);
482 | $window_left->addstr(0, 6, "Data Destinations");
483 | $window_left->attroff($CYAN);
484 | my $destinations_format = "%-18s%-6s%-3s";
485 | $window_left->attron(A_BOLD);
486 | $window_left->addstr(1, 1, sprintf($destinations_format,
487 | "Removable", "Fresh", "Usg"));
488 | $window_left->addstr(2, 1, sprintf($destinations_format,
489 | "Disk", "Data?", "%"));
490 | $window_left->attroff(A_BOLD);
491 | my $destination_usage;
492 | my $line_number = 3;
493 | map {
494 | if ($destination_usages{$_}) {
495 | $window_left->attron($RED);
496 | $destination_usage = "yes";
497 | } else {
498 | $window_left->attron($CYAN);
499 | $destination_usage = "no";
500 | }
501 | $window_left->
502 | addstr($line_number, 1,
503 | sprintf($destinations_format,
504 | substr($_, -17, 17),
505 | substr($destination_usage, -6, 6),
506 | substr($destination_usage_ratios{$_}
507 | ? $destination_usage_ratios{$_}
508 | : "?",
509 | -3, 3)));
510 | ++ $line_number;
511 | $window_left->attroff($RED);
512 | $window_left->attroff($CYAN);
513 | } sort @destination_roots;
514 |
515 | $window_right->attron($MAGENTA);
516 | $window_right->box(0,0);
517 | $window_right->addstr(0, 19, "Data Sources");
518 | $window_right->attroff($MAGENTA);
519 | my $sources_format = "%-15s%-11s%-9s%-13s";
520 | $window_right->attron(A_BOLD);
521 | $window_right->
522 | addstr(1, 1, sprintf ($sources_format,
523 | "Data", "", "Files", " Writing"));
524 | $window_right->
525 | addstr(2, 1, sprintf ($sources_format,
526 | "Source", "Speed", "To Copy", " To"));
527 | $window_right->attroff(A_BOLD);
528 | $line_number = 3;
529 | $window_right->attron($MAGENTA);
530 | map {
531 | my $source = $_;
532 | my $current_destination = '?';
533 | my $progress_ratio = $progress_ratios{$source};
534 | if (length $progress_ratio > 9) {
535 | $progress_ratio = eval ("100*" . $progress_ratio) . "%";
536 | }
537 | if (exists $destination_source_is_writing_to{$source}) {
538 | $current_destination =
539 | $destination_source_is_writing_to{$source};
540 | }
541 | if ($reachable{$source}) {
542 | $window_right->
543 | addstr($line_number, 1,
544 | sprintf($sources_format,
545 | substr($source_roots{$source}, 0, 14),
546 | substr($speeds{$source}, 0, 11),
547 | substr($progress_ratio,
548 | -9, 9),
549 | substr($current_destination, -13, 13)));
550 | ++ $line_number;
551 | }
552 | $window_right->
553 | addstr($line_number, 1,
554 | sprintf($sources_format, "", "", "", ""));
555 | } sort (keys %source_roots);
556 | $window_right->attroff($MAGENTA);
557 |
558 | $line_number = 0;
559 | map {
560 | $window_center->addstr($line_number, 2, $_);
561 | ++ $line_number;
562 | } @monikop_banner;
563 | $window_center->addstr(4, 78 - length $version, "$version");
564 | $window_center->move(0, 0);
565 |
566 | $window_bottom->box(0,0);
567 | $window_bottom->attron(A_BOLD);
568 | $window_bottom->addstr(1, 3, "[F3]: Turn off computer.");
569 | $window_bottom->addstr(1, 53, "[F6]: Restart computer.");
570 | $window_bottom->attroff(A_BOLD);
571 |
572 | $window_left->noutrefresh();
573 | $window_right->noutrefresh();
574 | $window_bottom->noutrefresh();
575 | $window_center->noutrefresh(); # Last window gets the cursor.
576 | act_on_keypress($window_bottom->getch());
577 | sleep 2;
578 | if (++ $redraw_window_count > 5) {
579 | $redraw_window_count = 0;
580 | redrawwin();
581 | }
582 | doupdate();
583 | }
584 | endwin();
585 | };
586 | }
587 |
588 | sleep;
589 |
590 | # Tidy up. (Except we don't reach this.)
591 | map {
592 | $being_deleted_thread{$_}->join if $being_deleted_thread{$_};
593 | } @destination_roots;
594 |
595 | map {
596 | $rsync_worker_thread{$_}->join if $rsync_worker_thread{$_};
597 | } keys %source_roots;
598 |
599 | $destinations_monitor_thread->join if $destinations_monitor_thread;
600 |
601 | $display_thread->join if $display_thread;
602 |
603 | __END__
604 |
--------------------------------------------------------------------------------