├── backup-deleted.diff ├── backup-dir-dels.diff ├── catch_crash_signals.diff ├── checksum-reading.diff ├── checksum-updating.diff ├── clone-dest.diff ├── congestion.diff ├── date-only.diff ├── detect-renamed-lax.diff ├── detect-renamed.diff ├── direct-io.diff ├── downdate.diff ├── fileflags.diff ├── filter-attribute-mods.diff ├── ignore-case.diff ├── kerberos.diff ├── link-by-hash.diff ├── omit-dir-changes.diff ├── slow-down.diff ├── slp.diff ├── soften-links.diff ├── source-backup.diff ├── source-filter_dest-filter.diff ├── sparse-block.diff └── transliterate.diff /backup-deleted.diff: -------------------------------------------------------------------------------- 1 | This patches adds the --backup-deleted option, as proposed by Jonathan 2 | Kames in bug 7889. 3 | 4 | To use this patch, run these commands for a successful build: 5 | 6 | patch -p1 0 && fnamecmp_type == FNAMECMP_FNAME) { 19 | + if (inplace && make_backups > 1 && fnamecmp_type == FNAMECMP_FNAME) { 20 | if (!(backupptr = get_backup_name(fname))) 21 | goto cleanup; 22 | if (!(back_file = make_file(fname, NULL, NULL, 0, NO_FILTERS))) 23 | @@ -1882,7 +1882,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, 24 | goto notify_others; 25 | } 26 | 27 | - if (inplace && make_backups > 0 && fnamecmp_type == FNAMECMP_FNAME) { 28 | + if (inplace && make_backups > 1 && fnamecmp_type == FNAMECMP_FNAME) { 29 | if (!(backupptr = get_backup_name(fname))) { 30 | goto cleanup; 31 | } 32 | @@ -2002,7 +2002,7 @@ int atomic_create(struct file_struct *file, char *fname, const char *slnk, const 33 | skip_atomic = 0; 34 | 35 | if (del_for_flag) { 36 | - if (make_backups > 0 && !dir_in_the_way) { 37 | + if (make_backups > 1 && !dir_in_the_way) { 38 | if (!make_backup(fname, skip_atomic)) 39 | return 0; 40 | } else if (skip_atomic) { 41 | diff --git a/options.c b/options.c 42 | --- a/options.c 43 | +++ b/options.c 44 | @@ -776,7 +776,8 @@ static struct poptOption long_options[] = { 45 | {"no-i", 0, POPT_ARG_VAL, &itemize_changes, 0, 0, 0 }, 46 | {"bwlimit", 0, POPT_ARG_STRING, &bwlimit_arg, OPT_BWLIMIT, 0, 0 }, 47 | {"no-bwlimit", 0, POPT_ARG_VAL, &bwlimit, 0, 0, 0 }, 48 | - {"backup", 'b', POPT_ARG_VAL, &make_backups, 1, 0, 0 }, 49 | + {"backup", 'b', POPT_ARG_VAL, &make_backups, 2, 0, 0 }, 50 | + {"backup-deleted", 0, POPT_ARG_VAL, &make_backups, 1, 0, 0 }, 51 | {"no-backup", 0, POPT_ARG_VAL, &make_backups, 0, 0, 0 }, 52 | {"backup-dir", 0, POPT_ARG_STRING, &backup_dir, 0, 0, 0 }, 53 | {"suffix", 0, POPT_ARG_STRING, &backup_suffix, 0, 0, 0 }, 54 | @@ -2805,6 +2806,10 @@ void server_options(char **args, int *argc_p) 55 | args[ac++] = safe_arg("--compress-choice", compress_choice); 56 | 57 | if (am_sender) { 58 | + /* A remote sender just needs the above -b option. 59 | + * A remote receiver will override that with this option. */ 60 | + if (make_backups == 1) 61 | + args[ac++] = "--backup-deleted"; 62 | if (max_delete > 0) { 63 | if (asprintf(&arg, "--max-delete=%d", max_delete) < 0) 64 | goto oom; 65 | diff --git a/receiver.c b/receiver.c 66 | --- a/receiver.c 67 | +++ b/receiver.c 68 | @@ -427,7 +427,7 @@ static void handle_delayed_updates(char *local_name) 69 | struct file_struct *file = cur_flist->files[ndx]; 70 | fname = local_name ? local_name : f_name(file, NULL); 71 | if ((partialptr = partial_dir_fname(fname)) != NULL) { 72 | - if (make_backups > 0 && !make_backup(fname, False)) 73 | + if (make_backups > 1 && !make_backup(fname, False)) 74 | continue; 75 | if (DEBUG_GTE(RECV, 1)) { 76 | rprintf(FINFO, "renaming %s to %s\n", 77 | @@ -748,7 +748,7 @@ int recv_files(int f_in, int f_out, char *local_name) 78 | } else { 79 | /* Reminder: --inplace && --partial-dir are never 80 | * enabled at the same time. */ 81 | - if (inplace && make_backups > 0) { 82 | + if (inplace && make_backups > 1) { 83 | if (!(fnamecmp = get_backup_name(fname))) 84 | fnamecmp = fname; 85 | else 86 | diff --git a/rsync.1.md b/rsync.1.md 87 | --- a/rsync.1.md 88 | +++ b/rsync.1.md 89 | @@ -428,6 +428,7 @@ has its own detailed description later in this manpage. 90 | --relative, -R use relative path names 91 | --no-implied-dirs don't send implied dirs with --relative 92 | --backup, -b make backups (see --suffix & --backup-dir) 93 | +--backup-deleted make backups only of deleted files 94 | --backup-dir=DIR make backups into hierarchy based in DIR 95 | --suffix=SUFFIX backup suffix (default ~ w/o --backup-dir) 96 | --update, -u skip files that are newer on the receiver 97 | @@ -1006,6 +1007,13 @@ expand it. 98 | rules specify a trailing inclusion/exclusion of `*`, the auto-added rule 99 | would never be reached). 100 | 101 | +0. `--backup-deleted` 102 | + 103 | + With this option, deleted destination files are renamed, while modified 104 | + destination files are not. Otherwise, this option behaves the same as 105 | + [`--backup`](#opt), described above. Note that if [`--backup`](#opt) is 106 | + also specified, whichever option is specified last takes precedence. 107 | + 108 | 0. `--backup-dir=DIR` 109 | 110 | This implies the [`--backup`](#opt) option, and tells rsync to store all 111 | diff --git a/rsync.c b/rsync.c 112 | --- a/rsync.c 113 | +++ b/rsync.c 114 | @@ -733,7 +733,7 @@ int finish_transfer(const char *fname, const char *fnametmp, 115 | goto do_set_file_attrs; 116 | } 117 | 118 | - if (make_backups > 0 && overwriting_basis) { 119 | + if (make_backups > 1 && overwriting_basis) { 120 | int ok = make_backup(fname, False); 121 | if (!ok) 122 | exit_cleanup(RERR_FILEIO); 123 | -------------------------------------------------------------------------------- /backup-dir-dels.diff: -------------------------------------------------------------------------------- 1 | This patches creates two new command line options as follows: 2 | --backup-dir-dels=DIR 3 | --suffix-dels=SUFFIX 4 | 5 | The backup-dir-dels and suffix-dels options give the ability to store 6 | backup of removed files on the receiver in different directories or with 7 | different suffix than the backup of files that have been changed but that 8 | are still on the source drive. Both commands can be combined. 9 | 10 | The default behaviour if one or both of the options are not specified 11 | is the previous behaviour, both backups use the same directory or 12 | suffix. 13 | 14 | Marc St-Onge 15 | 16 | To use this patch, run these commands for a successful build: 17 | 18 | patch -p1 = backup_dir_remainder) { 84 | + if (stringjoin(rel, remainder, fname, suffix, NULL) >= remainder) { 85 | rprintf(FERROR, "backup filename too long\n"); 86 | *name = '\0'; 87 | return False; 88 | @@ -82,7 +89,7 @@ static BOOL copy_valid_path(const char *fname) 89 | return True; 90 | *b = '\0'; 91 | 92 | - val = validate_backup_dir(); 93 | + val = validate_backup_dir(buf); 94 | if (val == 0) 95 | break; 96 | if (val < 0) { 97 | @@ -98,9 +105,9 @@ static BOOL copy_valid_path(const char *fname) 98 | for ( ; b; name = b + 1, b = strchr(name, '/')) { 99 | *b = '\0'; 100 | 101 | - while (do_mkdir(backup_dir_buf, ACCESSPERMS) < 0) { 102 | + while (do_mkdir(buf, ACCESSPERMS) < 0) { 103 | if (errno == EEXIST) { 104 | - val = validate_backup_dir(); 105 | + val = validate_backup_dir(buf); 106 | if (val > 0) 107 | break; 108 | if (val == 0) 109 | @@ -134,7 +141,7 @@ static BOOL copy_valid_path(const char *fname) 110 | free_xattr(&sx); 111 | } 112 | #endif 113 | - set_file_attrs(backup_dir_buf, file, NULL, NULL, 0); 114 | + set_file_attrs(buf, file, NULL, NULL, 0); 115 | unmake_file(file); 116 | } 117 | 118 | @@ -156,7 +163,12 @@ static BOOL copy_valid_path(const char *fname) 119 | /* Make a complete pathname for backup file and verify any new path elements. */ 120 | char *get_backup_name(const char *fname) 121 | { 122 | + char *buf = deleting ? backup_dir_dels_buf : backup_dir_buf; 123 | + char *suffix = deleting ? backup_suffix_dels : backup_suffix; 124 | + 125 | if (backup_dir) { 126 | + int prefix_len = deleting ? backup_dir_dels_len : backup_dir_len; 127 | + unsigned int remainder = deleting ? backup_dir_dels_remainder : backup_dir_remainder; 128 | static int initialized = 0; 129 | if (!initialized) { 130 | int ret; 131 | @@ -170,14 +182,14 @@ char *get_backup_name(const char *fname) 132 | initialized = 1; 133 | } 134 | /* copy fname into backup_dir_buf while validating the dirs. */ 135 | - if (copy_valid_path(fname)) 136 | - return backup_dir_buf; 137 | + if (copy_valid_path(fname, buf, prefix_len, remainder, suffix)) 138 | + return buf; 139 | /* copy_valid_path() has printed an error message. */ 140 | return NULL; 141 | } 142 | 143 | - if (stringjoin(backup_dir_buf, MAXPATHLEN, fname, backup_suffix, NULL) < MAXPATHLEN) 144 | - return backup_dir_buf; 145 | + if (stringjoin(backup_dir_buf, MAXPATHLEN, fname, suffix, NULL) < MAXPATHLEN) 146 | + return buf; 147 | 148 | rprintf(FERROR, "backup filename too long\n"); 149 | return NULL; 150 | @@ -353,3 +365,13 @@ int make_backup(const char *fname, BOOL prefer_rename) 151 | rprintf(FINFO, "backed up %s to %s\n", fname, buf); 152 | return ret; 153 | } 154 | + 155 | +/* backup switch routine called only when backing-up removed file */ 156 | +int safe_delete(const char *fname) 157 | +{ 158 | + int ret; 159 | + deleting = 1; 160 | + ret = make_backup(fname, True); 161 | + deleting = 0; 162 | + return ret; 163 | +} 164 | diff --git a/delete.c b/delete.c 165 | --- a/delete.c 166 | +++ b/delete.c 167 | @@ -28,16 +28,23 @@ extern int max_delete; 168 | extern char *backup_dir; 169 | extern char *backup_suffix; 170 | extern int backup_suffix_len; 171 | +extern char *backup_dir_dels; 172 | +extern char *backup_suffix_dels; 173 | +extern int backup_suffix_dels_len; 174 | extern struct stats stats; 175 | 176 | int ignore_perishable = 0; 177 | int non_perishable_cnt = 0; 178 | int skipped_deletes = 0; 179 | 180 | +/* Function now compares both backup_suffix and backup_suffix_dels. */ 181 | static inline int is_backup_file(char *fn) 182 | { 183 | int k = strlen(fn) - backup_suffix_len; 184 | - return k > 0 && strcmp(fn+k, backup_suffix) == 0; 185 | + if (k > 0 && strcmp(fn+k, backup_suffix) == 0) 186 | + return 1; 187 | + k += backup_suffix_len - backup_suffix_dels_len; 188 | + return k > 0 && strcmp(fn+k, backup_suffix_dels) == 0; 189 | } 190 | 191 | /* The directory is about to be deleted: if DEL_RECURSE is given, delete all 192 | @@ -162,9 +169,9 @@ enum delret delete_item(char *fbuf, uint16 mode, uint16 flags) 193 | what = "rmdir"; 194 | ok = do_rmdir(fbuf) == 0; 195 | } else { 196 | - if (make_backups > 0 && !(flags & DEL_FOR_BACKUP) && (backup_dir || !is_backup_file(fbuf))) { 197 | + if (make_backups > 0 && !(flags & DEL_FOR_BACKUP) && (backup_dir_dels || !is_backup_file(fbuf))) { 198 | what = "make_backup"; 199 | - ok = make_backup(fbuf, True); 200 | + ok = safe_delete(fbuf); 201 | if (ok == 2) { 202 | what = "unlink"; 203 | ok = robust_unlink(fbuf) == 0; 204 | diff --git a/options.c b/options.c 205 | --- a/options.c 206 | +++ b/options.c 207 | @@ -166,10 +166,14 @@ int no_detach 208 | int write_batch = 0; 209 | int read_batch = 0; 210 | int backup_dir_len = 0; 211 | +int backup_dir_dels_len = 0; 212 | int backup_suffix_len; 213 | +int backup_suffix_dels_len; 214 | unsigned int backup_dir_remainder; 215 | +unsigned int backup_dir_dels_remainder; 216 | 217 | char *backup_suffix = NULL; 218 | +char *backup_suffix_dels = NULL; 219 | char *tmpdir = NULL; 220 | char *partial_dir = NULL; 221 | char *basis_dir[MAX_BASIS_DIRS+1]; 222 | @@ -182,7 +186,9 @@ char *password_file = NULL; 223 | char *early_input_file = NULL; 224 | char *rsync_path = RSYNC_PATH; 225 | char *backup_dir = NULL; 226 | +char *backup_dir_dels = NULL; 227 | char backup_dir_buf[MAXPATHLEN]; 228 | +char backup_dir_dels_buf[MAXPATHLEN]; 229 | char *sockopts = NULL; 230 | char *usermap = NULL; 231 | char *groupmap = NULL; 232 | @@ -780,7 +786,9 @@ static struct poptOption long_options[] = { 233 | {"backup-deleted", 0, POPT_ARG_VAL, &make_backups, 1, 0, 0 }, 234 | {"no-backup", 0, POPT_ARG_VAL, &make_backups, 0, 0, 0 }, 235 | {"backup-dir", 0, POPT_ARG_STRING, &backup_dir, 0, 0, 0 }, 236 | + {"backup-dir-dels", 0, POPT_ARG_STRING, &backup_dir_dels, 0, 0, 0 }, 237 | {"suffix", 0, POPT_ARG_STRING, &backup_suffix, 0, 0, 0 }, 238 | + {"suffix-dels", 0, POPT_ARG_STRING, &backup_suffix_dels, 0, 0, 0 }, 239 | {"list-only", 0, POPT_ARG_VAL, &list_only, 2, 0, 0 }, 240 | {"read-batch", 0, POPT_ARG_STRING, &batch_name, OPT_READ_BATCH, 0, 0 }, 241 | {"write-batch", 0, POPT_ARG_STRING, &batch_name, OPT_WRITE_BATCH, 0, 0 }, 242 | @@ -2253,6 +2261,8 @@ int parse_arguments(int *argc_p, const char ***argv_p) 243 | tmpdir = sanitize_path(NULL, tmpdir, NULL, 0, SP_DEFAULT); 244 | if (backup_dir) 245 | backup_dir = sanitize_path(NULL, backup_dir, NULL, 0, SP_DEFAULT); 246 | + if (backup_dir_dels) 247 | + backup_dir_dels = sanitize_path(NULL, backup_dir_dels, NULL, 0, SP_DEFAULT); 248 | } 249 | if (daemon_filter_list.head && !am_sender) { 250 | filter_rule_list *elp = &daemon_filter_list; 251 | @@ -2274,6 +2284,14 @@ int parse_arguments(int *argc_p, const char ***argv_p) 252 | if (check_filter(elp, FLOG, dir, 1) < 0) 253 | goto options_rejected; 254 | } 255 | + /* Clean backup_dir_dels same as for backup_dir */ 256 | + if (backup_dir_dels) { 257 | + if (!*backup_dir_dels) 258 | + goto options_rejected; 259 | + clean_fname(backup_dir_dels, 1); 260 | + if (check_filter(elp, FLOG, backup_dir_dels, 1) < 0) 261 | + goto options_rejected; 262 | + } 263 | } 264 | 265 | if (!backup_suffix) 266 | @@ -2285,6 +2303,20 @@ int parse_arguments(int *argc_p, const char ***argv_p) 267 | backup_suffix); 268 | goto cleanup; 269 | } 270 | + /* --suffix-dels defaults to --suffix, or empty for a client given an 271 | + * explicit --backup-dir-dels (just as --suffix defaults to empty when 272 | + * a --backup-dir is given). The second case does not apply to the 273 | + * server for consistency with server_options, which sends --suffix-dels 274 | + * to the server iff it differs from --suffix. */ 275 | + if (!backup_suffix_dels) 276 | + backup_suffix_dels = backup_dir_dels && !am_server ? "" : backup_suffix; 277 | + backup_suffix_dels_len = strlen(backup_suffix_dels); 278 | + if (strchr(backup_suffix_dels, '/') != NULL) { 279 | + snprintf(err_buf, sizeof err_buf, 280 | + "--suffix-dels cannot contain slashes: %s\n", 281 | + backup_suffix_dels); 282 | + return 0; 283 | + } 284 | if (backup_dir) { 285 | size_t len; 286 | make_backups = 1; /* --backup-dir implies --backup */ 287 | @@ -2321,6 +2353,34 @@ int parse_arguments(int *argc_p, const char ***argv_p) 288 | "P *%s", backup_suffix); 289 | parse_filter_str(&filter_list, backup_dir_buf, rule_template(0), 0); 290 | } 291 | + if (backup_dir_dels) { 292 | + backup_dir_dels_len = strlcpy(backup_dir_dels_buf, backup_dir_dels, sizeof backup_dir_dels_buf); 293 | + backup_dir_dels_remainder = sizeof backup_dir_dels_buf - backup_dir_dels_len; 294 | + if (backup_dir_dels_remainder < 32) { 295 | + snprintf(err_buf, sizeof err_buf, 296 | + "the --backup-dir-dels path is WAY too long.\n"); 297 | + return 0; 298 | + } 299 | + if (backup_dir_dels_buf[backup_dir_dels_len - 1] != '/') { 300 | + backup_dir_dels_buf[backup_dir_dels_len++] = '/'; 301 | + backup_dir_dels_buf[backup_dir_dels_len] = '\0'; 302 | + } 303 | + if (INFO_GTE(BACKUP, 1) && !am_sender) 304 | + rprintf(FINFO, "backup_dir_dels is %s\n", backup_dir_dels_buf); 305 | + } else if (backup_dir) { 306 | + backup_dir_dels = backup_dir; 307 | + backup_dir_dels_len = backup_dir_len; 308 | + backup_dir_dels_remainder = backup_dir_remainder; 309 | + strlcpy(backup_dir_dels_buf, backup_dir_buf, sizeof backup_dir_buf); 310 | + } else if (!backup_suffix_dels_len && (!am_server || !am_sender)) { 311 | + snprintf(err_buf, sizeof err_buf, 312 | + "--suffix-dels cannot be a null string without --backup-dir-dels\n"); 313 | + return 0; 314 | + } else if (make_backups && delete_mode && !delete_excluded && !am_server) { 315 | + snprintf(backup_dir_dels_buf, sizeof backup_dir_dels_buf, 316 | + "P *%s", backup_suffix_dels); 317 | + parse_filter_str(&filter_list, backup_dir_dels_buf, rule_template(0), 0); 318 | + } 319 | 320 | if (make_backups && !backup_dir) 321 | omit_dir_times = -1; /* Implied, so avoid -O to sender. */ 322 | @@ -2790,11 +2850,20 @@ void server_options(char **args, int *argc_p) 323 | args[ac++] = "--backup-dir"; 324 | args[ac++] = safe_arg("", backup_dir); 325 | } 326 | + if (backup_dir_dels && backup_dir_dels != backup_dir) { 327 | + args[ac++] = "--backup-dir-dels"; 328 | + args[ac++] = backup_dir_dels; 329 | + } 330 | 331 | /* Only send --suffix if it specifies a non-default value. */ 332 | if (strcmp(backup_suffix, backup_dir ? "" : BACKUP_SUFFIX) != 0) 333 | args[ac++] = safe_arg("--suffix", backup_suffix); 334 | 335 | + /* Only send --suffix-dels if it specifies a value different from the 336 | + * --suffix value, which would normally be used for deletions too. */ 337 | + if (strcmp(backup_suffix_dels, backup_suffix) != 0) 338 | + args[ac++] = safe_arg("--suffix-dels", backup_suffix_dels); 339 | + 340 | if (checksum_choice) 341 | args[ac++] = safe_arg("--checksum-choice", checksum_choice); 342 | 343 | diff --git a/rsync.1.md b/rsync.1.md 344 | --- a/rsync.1.md 345 | +++ b/rsync.1.md 346 | @@ -430,7 +430,9 @@ has its own detailed description later in this manpage. 347 | --backup, -b make backups (see --suffix & --backup-dir) 348 | --backup-deleted make backups only of deleted files 349 | --backup-dir=DIR make backups into hierarchy based in DIR 350 | +--backup-dir-dels=DIR backup removed files into hierarchy based in DIR 351 | --suffix=SUFFIX backup suffix (default ~ w/o --backup-dir) 352 | +--suffix-dels=SUFFIX set removed-files suffix (def. --suffix w/o b-d-d) 353 | --update, -u skip files that are newer on the receiver 354 | --inplace update destination files in-place 355 | --append append data onto shorter files 356 | @@ -1028,6 +1030,11 @@ expand it. 357 | daemon is the receiver, the backup dir cannot go outside the module's path 358 | hierarchy, so take extra care not to delete it or copy into it. 359 | 360 | +0. `--backup-dir-dels=DIR` 361 | + 362 | + Works like [`--backup-dir`](#opt) except for deleted files in conjunction 363 | + with the [`--backup-deleted`](#opt) option. 364 | + 365 | 0. `--suffix=SUFFIX` 366 | 367 | This option allows you to override the default backup suffix used with the 368 | -------------------------------------------------------------------------------- /catch_crash_signals.diff: -------------------------------------------------------------------------------- 1 | Igor Yu. Zhbanov wrote: 2 | > I am using rsync compiled with Cygwin on windows. 3 | > I must call rsync from the *.bat script (I don't want to use a bash on Windows) 4 | > and I have noticed that in the case when program compiled by Cygwin crashes 5 | > via segmentation fault and default SIGSEGV handler is called, then it 6 | > terminates process with exit status 0 as I see it from my *.bat script. 7 | > (But if I invoke a program from bash (compiled with Cygwin too) I will see 8 | > error code 139 as expected.) 9 | > 10 | > It is a Cygwin's problem, not an rsync's, but to use it on windows and 11 | > to distinguish situations when rsync crashes and when it exits normally, 12 | > I have written signal handler which terminates process with code 50. 13 | > You may use conventional 139. Also signal handler writes corresponding 14 | > message to log file. 15 | > 16 | > By the way. When I terminate rsync in daemon mode by pressing Control-C, 17 | > it writes an error to log. May be this is not an error but info or notice? 18 | 19 | I'm not sure I like this, but if you run into the cygwin problem, this might 20 | prove helpful. 21 | 22 | To use this patch, run these commands for a successful build: 23 | 24 | patch -p1 0) {} 125 | + int status; 126 | + while (waitpid(-1, &status, WNOHANG) > 0) { 127 | + if (WIFSIGNALED(status)) { 128 | + rprintf(FLOG, 129 | + "rsync error: (3) Child proccess has unexpectedly died with signal %d\n", 130 | + WTERMSIG(status)); 131 | + } else if (WIFEXITED(status) && WEXITSTATUS(status) == RERR_WECRASHED) { 132 | + rprintf(FLOG, 133 | + "rsync error: (3) Child proccess has CRASHED.\n"); 134 | + } 135 | + } 136 | #endif 137 | #ifndef HAVE_SIGACTION 138 | signal(SIGCHLD, sigchld_handler); 139 | -------------------------------------------------------------------------------- /checksum-updating.diff: -------------------------------------------------------------------------------- 1 | This builds on the checksum-reading patch and adds the ability to 2 | create and/or update the .rsyncsums files using extended mode args to 3 | the --sumfiles=MODE option and the "checksum files = MODE" daemon 4 | parameter. 5 | 6 | CAUTION: This patch is only lightly tested. If you're interested 7 | in using it, please help out. 8 | 9 | To use this patch, run these commands for a successful build: 10 | 11 | patch -p1 to_redo) 47 | + 48 | static struct csum_cache { 49 | struct file_list *flist; 50 | + const char *dirname; 51 | + int checksum_matches; 52 | + int checksum_updates; 53 | } *csum_cache = NULL; 54 | 55 | static struct file_list *flist_new(int flags, const char *msg); 56 | @@ -351,7 +360,79 @@ static void flist_done_allocating(struct file_list *flist) 57 | flist->pool_boundary = ptr; 58 | } 59 | 60 | -void reset_checksum_cache() 61 | +static void checksum_filename(int slot, const char *dirname, char *fbuf) 62 | +{ 63 | + if (dirname && *dirname) { 64 | + unsigned int len; 65 | + if (slot) { 66 | + len = strlcpy(fbuf, basis_dir[slot-1], MAXPATHLEN); 67 | + if (len >= MAXPATHLEN) 68 | + return; 69 | + } else 70 | + len = 0; 71 | + if (pathjoin(fbuf+len, MAXPATHLEN-len, dirname, RSYNCSUMS_FILE) >= MAXPATHLEN-len) 72 | + return; 73 | + } else 74 | + strlcpy(fbuf, RSYNCSUMS_FILE, MAXPATHLEN); 75 | +} 76 | + 77 | +static void write_checksums(int slot, struct file_list *flist, int whole_dir) 78 | +{ 79 | + int i; 80 | + FILE *out_fp; 81 | + char fbuf[MAXPATHLEN]; 82 | + int new_entries = csum_cache[slot].checksum_updates != 0; 83 | + int counts_match = flist->used == csum_cache[slot].checksum_matches; 84 | + int no_skipped = whole_dir && REGULAR_SKIPPED(flist) == 0; 85 | + const char *dirname = csum_cache[slot].dirname; 86 | + 87 | + flist_sort_and_clean(flist, 0); 88 | + 89 | + if (dry_run && !(checksum_files & CSF_AFFECT_DRYRUN)) 90 | + return; 91 | + 92 | + checksum_filename(slot, dirname, fbuf); 93 | + 94 | + if (flist->high - flist->low < 0 && no_skipped) { 95 | + unlink(fbuf); 96 | + return; 97 | + } 98 | + 99 | + if (!new_entries && (counts_match || !whole_dir)) 100 | + return; 101 | + 102 | + if (!(out_fp = fopen(fbuf, "w"))) 103 | + return; 104 | + 105 | + for (i = flist->low; i <= flist->high; i++) { 106 | + struct file_struct *file = flist->sorted[i]; 107 | + const char *cp = F_SUM(file); 108 | + const char *end = cp + flist_csum_len; 109 | + const char *alt_sum = file->basename + strlen(file->basename) + 1; 110 | + if (whole_dir && !(file->flags & FLAG_SUM_KEEP)) 111 | + continue; 112 | + if (file_sum_nni->num == CSUM_MD5) 113 | + fprintf(out_fp, "%s ", alt_sum); 114 | + if (file->flags & FLAG_SUM_MISSING) { 115 | + do { 116 | + fputs("==", out_fp); 117 | + } while (++cp != end); 118 | + } else { 119 | + do { 120 | + fprintf(out_fp, "%02x", (int)CVAL(cp, 0)); 121 | + } while (++cp != end); 122 | + } 123 | + if (file_sum_nni->num < CSUM_MD5) 124 | + fprintf(out_fp, " %s", alt_sum); 125 | + fprintf(out_fp, " %10.0f %10.0f %10lu %10lu %s\n", 126 | + (double)F_LENGTH(file), (double)file->modtime, 127 | + (long)F_CTIME(file), (long)F_INODE(file), file->basename); 128 | + } 129 | + 130 | + fclose(out_fp); 131 | +} 132 | + 133 | +void reset_checksum_cache(int whole_dir) 134 | { 135 | int slot, slots = am_sender ? 1 : basis_dir_cnt + 1; 136 | 137 | @@ -362,6 +443,9 @@ void reset_checksum_cache() 138 | struct file_list *flist = csum_cache[slot].flist; 139 | 140 | if (flist) { 141 | + if (checksum_files & CSF_UPDATE && flist->next) 142 | + write_checksums(slot, flist, whole_dir); 143 | + 144 | /* Reset the pool memory and empty the file-list array. */ 145 | pool_free_old(flist->file_pool, 146 | pool_boundary(flist->file_pool, 0)); 147 | @@ -372,6 +456,10 @@ void reset_checksum_cache() 148 | flist->low = 0; 149 | flist->high = -1; 150 | flist->next = NULL; 151 | + 152 | + csum_cache[slot].checksum_matches = 0; 153 | + csum_cache[slot].checksum_updates = 0; 154 | + REGULAR_SKIPPED(flist) = 0; 155 | } 156 | } 157 | 158 | @@ -379,7 +467,7 @@ void reset_checksum_cache() 159 | static int add_checksum(struct file_list *flist, const char *dirname, 160 | const char *basename, int basename_len, OFF_T file_length, 161 | time_t mtime, uint32 ctime, uint32 inode, 162 | - const char *sum) 163 | + const char *sum, const char *alt_sum, int flags) 164 | { 165 | struct file_struct *file; 166 | int alloc_len, extra_len; 167 | @@ -396,7 +484,7 @@ static int add_checksum(struct file_list *flist, const char *dirname, 168 | if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN)) 169 | extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN; 170 | #endif 171 | - alloc_len = FILE_STRUCT_LEN + extra_len + basename_len; 172 | + alloc_len = FILE_STRUCT_LEN + extra_len + basename_len + flist_csum_len*2 + 1; 173 | bp = pool_alloc(flist->file_pool, alloc_len, "add_checksum"); 174 | 175 | memset(bp, 0, extra_len + FILE_STRUCT_LEN); 176 | @@ -405,7 +493,14 @@ static int add_checksum(struct file_list *flist, const char *dirname, 177 | bp += FILE_STRUCT_LEN; 178 | 179 | memcpy(bp, basename, basename_len); 180 | + if (alt_sum) 181 | + strlcpy(bp+basename_len, alt_sum, flist_csum_len*2 + 1); 182 | + else { 183 | + memset(bp+basename_len, '=', flist_csum_len*2); 184 | + bp[basename_len+flist_csum_len*2] = '\0'; 185 | + } 186 | 187 | + file->flags = flags; 188 | file->mode = S_IFREG; 189 | file->modtime = mtime; 190 | file->len32 = (uint32)file_length; 191 | @@ -434,10 +529,11 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam 192 | char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN]; 193 | FILE *fp; 194 | char *cp; 195 | - int len, i; 196 | time_t mtime; 197 | + int len, i, flags; 198 | OFF_T file_length; 199 | uint32 ctime, inode; 200 | + const char *alt_sum = NULL; 201 | int dlen = dirname ? strlcpy(fbuf, dirname, sizeof fbuf) : 0; 202 | 203 | if (dlen >= (int)(sizeof fbuf - 1 - RSYNCSUMS_LEN)) 204 | @@ -458,7 +554,7 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam 205 | while (fgets(line, sizeof line, fp)) { 206 | cp = line; 207 | if (file_sum_nni->num == CSUM_MD5) { 208 | - char *alt_sum = cp; 209 | + alt_sum = cp; 210 | if (*cp == '=') 211 | while (*++cp == '=') {} 212 | else 213 | @@ -469,7 +565,14 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam 214 | } 215 | 216 | if (*cp == '=') { 217 | - continue; 218 | + for (i = 0; i < flist_csum_len*2; i++, cp++) { 219 | + if (*cp != '=') { 220 | + cp = ""; 221 | + break; 222 | + } 223 | + } 224 | + memset(sum, 0, flist_csum_len); 225 | + flags = FLAG_SUM_MISSING; 226 | } else { 227 | for (i = 0; i < flist_csum_len*2; i++, cp++) { 228 | int x; 229 | @@ -487,13 +590,14 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam 230 | else 231 | sum[i/2] = x << 4; 232 | } 233 | + flags = 0; 234 | } 235 | if (*cp != ' ') 236 | break; 237 | while (*++cp == ' ') {} 238 | 239 | if (file_sum_nni->num < CSUM_MD5) { 240 | - char *alt_sum = cp; 241 | + alt_sum = cp; 242 | if (*cp == '=') 243 | while (*++cp == '=') {} 244 | else 245 | @@ -543,24 +647,112 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam 246 | continue; 247 | 248 | strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen); 249 | + if (is_excluded(fbuf, 0, ALL_FILTERS)) { 250 | + flags |= FLAG_SUM_KEEP; 251 | + csum_cache[slot].checksum_matches++; 252 | + } 253 | 254 | add_checksum(flist, dirname, cp, len, file_length, 255 | mtime, ctime, inode, 256 | - sum); 257 | + sum, alt_sum, flags); 258 | } 259 | fclose(fp); 260 | 261 | flist_sort_and_clean(flist, CLEAN_KEEP_LAST); 262 | } 263 | 264 | +void set_cached_checksum(struct file_list *file_flist, struct file_struct *file) 265 | +{ 266 | + int j; 267 | + FILE *out_fp; 268 | + STRUCT_STAT st; 269 | + char fbuf[MAXPATHLEN]; 270 | + const char *fn = f_name(file, NULL); 271 | + struct file_list *flist = csum_cache[0].flist; 272 | + 273 | + if (dry_run && !(checksum_files & CSF_AFFECT_DRYRUN)) 274 | + return; 275 | + 276 | + if (stat(fn, &st) < 0) 277 | + return; 278 | + 279 | + checksum_filename(0, file->dirname, fbuf); 280 | + 281 | + if (file_flist != flist->next) { 282 | + const char *cp = F_SUM(file); 283 | + const char *end = cp + flist_csum_len; 284 | + 285 | + if (!(out_fp = fopen(fbuf, "a"))) 286 | + return; 287 | + 288 | + if (file_sum_nni->num == CSUM_MD5) { 289 | + for (j = 0; j < flist_csum_len; j++) 290 | + fputs("==", out_fp); 291 | + fputc(' ', out_fp); 292 | + } 293 | + do { 294 | + fprintf(out_fp, "%02x", (int)CVAL(cp, 0)); 295 | + } while (++cp != end); 296 | + if (file_sum_nni->num < CSUM_MD5) { 297 | + fputc(' ', out_fp); 298 | + for (j = 0; j < flist_csum_len; j++) 299 | + fputs("==", out_fp); 300 | + } 301 | + fprintf(out_fp, " %10.0f %10.0f %10lu %10lu %s\n", 302 | + (double)st.st_size, (double)st.st_mtime, 303 | + (long)(uint32)st.st_ctime, (long)(uint32)st.st_ino, 304 | + file->basename); 305 | + 306 | + fclose(out_fp); 307 | + return; 308 | + } 309 | + 310 | + if ((j = flist_find(flist, file)) >= 0) { 311 | + struct file_struct *fp = flist->sorted[j]; 312 | + int inc = 0; 313 | + if (F_LENGTH(fp) != st.st_size) { 314 | + fp->len32 = (uint32)st.st_size; 315 | + if (st.st_size > 0xFFFFFFFFu) { 316 | + OPT_EXTRA(fp, 0)->unum = (uint32)(st.st_size >> 32); 317 | + fp->flags |= FLAG_LENGTH64; 318 | + } else 319 | + fp->flags &= FLAG_LENGTH64; 320 | + inc = 1; 321 | + } 322 | + if (fp->modtime != st.st_mtime) { 323 | + fp->modtime = st.st_mtime; 324 | + inc = 1; 325 | + } 326 | + if (F_CTIME(fp) != (uint32)st.st_ctime) { 327 | + F_CTIME(fp) = (uint32)st.st_ctime; 328 | + inc = 1; 329 | + } 330 | + if (F_INODE(fp) != (uint32)st.st_ino) { 331 | + F_INODE(fp) = (uint32)st.st_ino; 332 | + inc = 1; 333 | + } 334 | + memcpy(F_SUM(fp), F_SUM(file), MAX_DIGEST_LEN); 335 | + csum_cache[0].checksum_updates += inc; 336 | + fp->flags &= ~FLAG_SUM_MISSING; 337 | + fp->flags |= FLAG_SUM_KEEP; 338 | + return; 339 | + } 340 | + 341 | + csum_cache[0].checksum_updates += 342 | + add_checksum(flist, file->dirname, file->basename, strlen(file->basename) + 1, 343 | + st.st_size, (uint32)st.st_mtime, (uint32)st.st_ctime, 344 | + st.st_ino, F_SUM(file), NULL, FLAG_SUM_KEEP); 345 | +} 346 | + 347 | void get_cached_checksum(int slot, const char *fname, struct file_struct *file, 348 | - STRUCT_STAT *stp, char *sum_buf) 349 | + int basename_len, STRUCT_STAT *stp, char *sum_buf) 350 | { 351 | struct file_list *flist = csum_cache[slot].flist; 352 | int j; 353 | 354 | if (!flist->next) { 355 | flist->next = cur_flist; /* next points from checksum flist to file flist */ 356 | + csum_cache[slot].dirname = file->dirname; 357 | read_checksums(slot, flist, file->dirname); 358 | } 359 | 360 | @@ -572,12 +764,31 @@ void get_cached_checksum(int slot, const char *fname, struct file_struct *file, 361 | && (checksum_files & CSF_LAX 362 | || (F_CTIME(fp) == (uint32)stp->st_ctime 363 | && F_INODE(fp) == (uint32)stp->st_ino))) { 364 | - memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN); 365 | + if (fp->flags & FLAG_SUM_MISSING) { 366 | + fp->flags &= ~FLAG_SUM_MISSING; 367 | + csum_cache[slot].checksum_updates++; 368 | + file_checksum(fname, stp, sum_buf); 369 | + memcpy(F_SUM(fp), sum_buf, MAX_DIGEST_LEN); 370 | + } else { 371 | + csum_cache[slot].checksum_matches++; 372 | + memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN); 373 | + } 374 | + fp->flags |= FLAG_SUM_KEEP; 375 | return; 376 | } 377 | + clear_file(fp); 378 | } 379 | 380 | file_checksum(fname, stp, sum_buf); 381 | + 382 | + if (checksum_files & CSF_UPDATE) { 383 | + if (basename_len < 0) 384 | + basename_len = strlen(file->basename) + 1; 385 | + csum_cache[slot].checksum_updates += 386 | + add_checksum(flist, file->dirname, file->basename, basename_len, 387 | + stp->st_size, stp->st_mtime, (uint32)stp->st_ctime, 388 | + (uint32)stp->st_ino, sum_buf, NULL, FLAG_SUM_KEEP); 389 | + } 390 | } 391 | 392 | /* Call this with EITHER (1) "file, NULL, 0" to chdir() to the file's 393 | @@ -1583,6 +1794,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, 394 | if (is_excluded(thisname, S_ISDIR(st.st_mode) != 0, filter_level)) { 395 | if (ignore_perishable) 396 | non_perishable_cnt++; 397 | + if (S_ISREG(st.st_mode)) 398 | + REGULAR_SKIPPED(flist)++; 399 | return NULL; 400 | } 401 | 402 | @@ -1629,13 +1842,13 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, 403 | lastdir[len] = '\0'; 404 | lastdir_len = len; 405 | if (checksum_files && am_sender && flist) 406 | - reset_checksum_cache(); 407 | + reset_checksum_cache(0); 408 | } 409 | } else { 410 | basename = thisname; 411 | if (checksum_files && am_sender && flist && lastdir_len == -2) { 412 | lastdir_len = -1; 413 | - reset_checksum_cache(); 414 | + reset_checksum_cache(0); 415 | } 416 | } 417 | basename_len = strlen(basename) + 1; /* count the '\0' */ 418 | @@ -1759,7 +1972,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, 419 | 420 | if (always_checksum && am_sender && S_ISREG(st.st_mode)) { 421 | if (flist && checksum_files) 422 | - get_cached_checksum(0, thisname, file, &st, tmp_sum); 423 | + get_cached_checksum(0, thisname, file, basename_len, &st, tmp_sum); 424 | else 425 | file_checksum(thisname, &st, tmp_sum); 426 | if (sender_keeps_checksum) 427 | @@ -2151,6 +2364,9 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len, 428 | 429 | closedir(d); 430 | 431 | + if (checksum_files & CSF_UPDATE && am_sender && f >= 0) 432 | + reset_checksum_cache(1); 433 | + 434 | if (f >= 0 && recurse && !divert_dirs) { 435 | int i, end = flist->used - 1; 436 | /* send_if_directory() bumps flist->used, so use "end". */ 437 | @@ -2816,6 +3032,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) 438 | rprintf(FINFO, "[%s] flist_eof=1\n", who_am_i()); 439 | } 440 | 441 | + if (checksum_files & CSF_UPDATE && flist_eof) 442 | + reset_checksum_cache(0); /* writes any last updates */ 443 | + 444 | return flist; 445 | } 446 | 447 | diff --git a/generator.c b/generator.c 448 | --- a/generator.c 449 | +++ b/generator.c 450 | @@ -113,6 +113,7 @@ static int dir_tweaking; 451 | static int symlink_timeset_failed_flags; 452 | static int need_retouch_dir_times; 453 | static int need_retouch_dir_perms; 454 | +static int started_whole_dir, upcoming_whole_dir; 455 | static const char *solo_file = NULL; 456 | 457 | /* Forward declarations. */ 458 | @@ -627,7 +628,7 @@ int quick_check_ok(enum filetype ftype, const char *fn, struct file_struct *file 459 | if (always_checksum > 0) { 460 | char sum[MAX_DIGEST_LEN]; 461 | if (checksum_files && slot >= 0) 462 | - get_cached_checksum(slot, fn, file, st, sum); 463 | + get_cached_checksum(slot, fn, file, -1, st, sum); 464 | else 465 | file_checksum(fn, st, sum); 466 | return memcmp(sum, F_SUM(file), flist_csum_len) == 0; 467 | @@ -1359,7 +1360,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, 468 | } 469 | } 470 | if (checksum_files) { 471 | - reset_checksum_cache(); 472 | + reset_checksum_cache(started_whole_dir); 473 | + started_whole_dir = upcoming_whole_dir; 474 | } 475 | need_new_dirscan = 0; 476 | } 477 | @@ -1547,6 +1549,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, 478 | else 479 | change_local_filter_dir(fname, strlen(fname), F_DEPTH(file)); 480 | } 481 | + upcoming_whole_dir = file->flags & FLAG_CONTENT_DIR && f_out != -1 ? 1 : 0; 482 | prior_dir_file = file; 483 | goto cleanup; 484 | } 485 | @@ -1819,6 +1822,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, 486 | handle_partial_dir(partialptr, PDIR_DELETE); 487 | } 488 | set_file_attrs(fname, file, &sx, NULL, maybe_ATTRS_REPORT | maybe_ATTRS_ACCURATE_TIME); 489 | + if (checksum_files & CSF_UPDATE) 490 | + set_cached_checksum(cur_flist, file); 491 | if (itemizing) 492 | itemize(fnamecmp, file, ndx, statret, &sx, 0, 0, NULL); 493 | #ifdef SUPPORT_HARD_LINKS 494 | @@ -2311,6 +2316,7 @@ void generate_files(int f_out, const char *local_name) 495 | } else 496 | change_local_filter_dir(fbuf, strlen(fbuf), F_DEPTH(fp)); 497 | } 498 | + upcoming_whole_dir = fp->flags & FLAG_CONTENT_DIR ? 1 : 0; 499 | } 500 | for (i = cur_flist->low; i <= cur_flist->high; i++) { 501 | struct file_struct *file = cur_flist->sorted[i]; 502 | @@ -2405,6 +2411,9 @@ void generate_files(int f_out, const char *local_name) 503 | wait_for_receiver(); 504 | } 505 | 506 | + if (checksum_files) 507 | + reset_checksum_cache(started_whole_dir); 508 | + 509 | info_levels[INFO_FLIST] = save_info_flist; 510 | info_levels[INFO_PROGRESS] = save_info_progress; 511 | 512 | diff --git a/io.c b/io.c 513 | --- a/io.c 514 | +++ b/io.c 515 | @@ -55,6 +55,7 @@ extern int read_batch; 516 | extern int compat_flags; 517 | extern int protect_args; 518 | extern int checksum_seed; 519 | +extern int checksum_files; 520 | extern int daemon_connection; 521 | extern int protocol_version; 522 | extern int remove_source_files; 523 | @@ -1117,6 +1118,9 @@ static void got_flist_entry_status(enum festatus status, int ndx) 524 | if (inc_recurse) 525 | flist->in_progress++; 526 | } 527 | + } else if (checksum_files & CSF_UPDATE) { 528 | + struct file_struct *file = flist->files[ndx - flist->ndx_start]; 529 | + set_cached_checksum(flist, file); 530 | } 531 | #endif 532 | break; 533 | diff --git a/loadparm.c b/loadparm.c 534 | --- a/loadparm.c 535 | +++ b/loadparm.c 536 | @@ -166,6 +166,10 @@ static struct enum_list enum_checksum_files[] = { 537 | { CSF_IGNORE_FILES, "none" }, 538 | { CSF_LAX_MODE, "lax" }, 539 | { CSF_STRICT_MODE, "strict" }, 540 | + { CSF_LAX_MODE|CSF_UPDATE, "+lax" }, 541 | + { CSF_STRICT_MODE|CSF_UPDATE, "+strict" }, 542 | + { CSF_LAX_MODE|CSF_UPDATE|CSF_AFFECT_DRYRUN, "++lax" }, 543 | + { CSF_STRICT_MODE|CSF_UPDATE|CSF_AFFECT_DRYRUN, "++strict" }, 544 | { -1, NULL } 545 | }; 546 | 547 | diff --git a/options.c b/options.c 548 | --- a/options.c 549 | +++ b/options.c 550 | @@ -1755,7 +1755,15 @@ int parse_arguments(int *argc_p, const char ***argv_p) 551 | 552 | case OPT_SUMFILES: 553 | arg = poptGetOptArg(pc); 554 | - checksum_files = 0; 555 | + if (*arg == '+') { 556 | + arg++; 557 | + checksum_files = CSF_UPDATE; 558 | + if (*arg == '+') { 559 | + arg++; 560 | + checksum_files |= CSF_AFFECT_DRYRUN; 561 | + } 562 | + } else 563 | + checksum_files = 0; 564 | if (strcmp(arg, "lax") == 0) 565 | checksum_files |= CSF_LAX_MODE; 566 | else if (strcmp(arg, "strict") == 0) 567 | diff --git a/receiver.c b/receiver.c 568 | --- a/receiver.c 569 | +++ b/receiver.c 570 | @@ -51,6 +51,7 @@ extern int sparse_files; 571 | extern int preallocate_files; 572 | extern int keep_partial; 573 | extern int checksum_seed; 574 | +extern int checksum_files; 575 | extern int whole_file; 576 | extern int inplace; 577 | extern int inplace_partial; 578 | @@ -440,7 +441,8 @@ static void handle_delayed_updates(char *local_name) 579 | "rename failed for %s (from %s)", 580 | full_fname(fname), partialptr); 581 | } else { 582 | - if (remove_source_files || (preserve_hard_links && F_IS_HLINKED(file))) 583 | + if (remove_source_files || checksum_files & CSF_UPDATE 584 | + || (preserve_hard_links && F_IS_HLINKED(file))) 585 | send_msg_success(fname, ndx); 586 | handle_partial_dir(partialptr, PDIR_DELETE); 587 | } 588 | @@ -926,7 +928,8 @@ int recv_files(int f_in, int f_out, char *local_name) 589 | case 2: 590 | break; 591 | case 1: 592 | - if (remove_source_files || inc_recurse || (preserve_hard_links && F_IS_HLINKED(file))) 593 | + if (remove_source_files || inc_recurse || checksum_files & CSF_UPDATE 594 | + || (preserve_hard_links && F_IS_HLINKED(file))) 595 | send_msg_success(fname, ndx); 596 | break; 597 | case 0: { 598 | diff --git a/rsync.1.md b/rsync.1.md 599 | --- a/rsync.1.md 600 | +++ b/rsync.1.md 601 | @@ -839,9 +839,13 @@ expand it. 602 | 603 | The MODE value is either "lax", for relaxed checking (which compares size 604 | and mtime), "strict" (which also compares ctime and inode), or "none" to 605 | - ignore any .rsyncsums files ("none" is the default). Rsync does not create 606 | - or update these files, but there is a perl script in the support directory 607 | - named "rsyncsums" that can be used for that. 608 | + ignore any .rsyncsums files ("none" is the default). 609 | + If you want rsync to create and/or update these files, specify a prefixed 610 | + plus ("+lax" or "+strict"). Adding a second prefixed '+' causes the 611 | + checksum-file updates to happen even when the transfer is in [`--dry-run`](#opt) 612 | + mode ("++lax" or "++strict"). There is also a perl script in the support 613 | + directory named "rsyncsums" that can be used to update the .rsyncsums 614 | + files. 615 | 616 | This option has no effect unless [`--checksum`](#opt) (`-c`) was also 617 | specified. It also only affects the current side of the transfer, so if 618 | diff --git a/rsync.h b/rsync.h 619 | --- a/rsync.h 620 | +++ b/rsync.h 621 | @@ -1117,6 +1117,8 @@ typedef struct { 622 | 623 | #define CSF_ENABLE (1<<1) 624 | #define CSF_LAX (1<<2) 625 | +#define CSF_UPDATE (1<<3) 626 | +#define CSF_AFFECT_DRYRUN (1<<4) 627 | 628 | #define CSF_IGNORE_FILES 0 629 | #define CSF_LAX_MODE (CSF_ENABLE|CSF_LAX) 630 | diff --git a/rsyncd.conf.5.md b/rsyncd.conf.5.md 631 | --- a/rsyncd.conf.5.md 632 | +++ b/rsyncd.conf.5.md 633 | @@ -458,13 +458,15 @@ in the values of parameters. See that section for details. 634 | This parameter tells rsync to make use of any cached checksum information 635 | it finds in per-directory .rsyncsums files when the current transfer is 636 | using the `--checksum` option. The value can be set to either "lax", 637 | - "strict", or "none". See the client's `--sumfiles` option for what these 638 | - choices do. 639 | + "strict", "+lax", "+strict", "++lax", "++strict", or +"none". See the 640 | + client's `--sumfiles` option for what these choices do. 641 | 642 | Note also that the client's command-line option, `--sumfiles`, has no 643 | effect on a daemon. A daemon will only access checksum files if this 644 | - config option tells it to. See also the `exclude` directive for a way to 645 | - hide the .rsyncsums files from the user. 646 | + config option tells it to. You can configure updating of the .rsyncsums 647 | + files even if the module itself is configured to be read-only. See also 648 | + the `exclude` directive for a way to hide the .rsyncsums files from the 649 | + user. 650 | 651 | 0. `read only` 652 | 653 | -------------------------------------------------------------------------------- /clone-dest.diff: -------------------------------------------------------------------------------- 1 | This patch adds the --clone-dest option that works link --link-dest 2 | but without requiring the metadata of the files to match in order 3 | to be able to share the file's data. 4 | 5 | NOTE: this patch is mostly untested because I don't currently have 6 | a btrfs mount to test it out on. I still need to make sure that a 7 | cloned file gets its destination attributes set correctly after the 8 | clone, for instance. 9 | 10 | To use this patch, run these commands for a successful build: 11 | 12 | patch -p1 mode) == 0) { 62 | + finish_transfer(fname, fname, cmpbuf, NULL, file, 1, 0); 63 | + } else { 64 | + rsyserr(FERROR_XFER, errno, "failed to clone %s to %s", cmpbuf, fname); 65 | + exit_cleanup(RERR_UNSUPPORTED); 66 | + } 67 | if (atimes_ndx) 68 | set_file_attrs(fname, file, sxp, NULL, 0); 69 | if (preserve_hard_links && F_IS_HLINKED(file)) 70 | @@ -1104,7 +1111,7 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx, 71 | 72 | if (match_level == 3) { 73 | #ifdef SUPPORT_HARD_LINKS 74 | - if (alt_dest_type == LINK_DEST 75 | + if ((alt_dest_type == LINK_DEST || alt_dest_type == CLONE_DEST) 76 | #ifndef CAN_HARDLINK_SYMLINK 77 | && !S_ISLNK(file->mode) 78 | #endif 79 | diff --git a/options.c b/options.c 80 | --- a/options.c 81 | +++ b/options.c 82 | @@ -582,7 +582,7 @@ enum {OPT_SERVER = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM, 83 | OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW, OPT_MIN_SIZE, OPT_CHMOD, 84 | OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE, 85 | OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG, OPT_BLOCK_SIZE, 86 | - OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT, OPT_STDERR, 87 | + OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT, OPT_STDERR, OPT_CLONE_DEST, 88 | OPT_OLD_COMPRESS, OPT_NEW_COMPRESS, OPT_NO_COMPRESS, OPT_OLD_ARGS, 89 | OPT_STOP_AFTER, OPT_STOP_AT, 90 | OPT_REFUSED_BASE = 9000}; 91 | @@ -743,6 +743,7 @@ static struct poptOption long_options[] = { 92 | {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 }, 93 | {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 }, 94 | {"link-dest", 0, POPT_ARG_STRING, 0, OPT_LINK_DEST, 0, 0 }, 95 | + {"clone-dest", 0, POPT_ARG_STRING, 0, OPT_CLONE_DEST, 0, 0 }, 96 | {"fuzzy", 'y', POPT_ARG_NONE, 0, 'y', 0, 0 }, 97 | {"no-fuzzy", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 }, 98 | {"no-y", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 }, 99 | @@ -1004,6 +1005,9 @@ static void set_refuse_options(void) 100 | #ifndef SUPPORT_HARD_LINKS 101 | parse_one_refuse_match(0, "link-dest", list_end); 102 | #endif 103 | +#ifndef FICLONE 104 | + parse_one_refuse_match(0, "clone-dest", list_end); 105 | +#endif 106 | #ifndef HAVE_MKTIME 107 | parse_one_refuse_match(0, "stop-at", list_end); 108 | #endif 109 | @@ -1333,6 +1337,8 @@ char *alt_dest_opt(int type) 110 | return "--copy-dest"; 111 | case LINK_DEST: 112 | return "--link-dest"; 113 | + case CLONE_DEST: 114 | + return "--clone-dest"; 115 | default: 116 | NOISY_DEATH("Unknown alt_dest_opt type"); 117 | } 118 | @@ -1714,6 +1720,10 @@ int parse_arguments(int *argc_p, const char ***argv_p) 119 | want_dest_type = LINK_DEST; 120 | goto set_dest_dir; 121 | 122 | + case OPT_CLONE_DEST: 123 | + want_dest_type = CLONE_DEST; 124 | + goto set_dest_dir; 125 | + 126 | case OPT_COPY_DEST: 127 | want_dest_type = COPY_DEST; 128 | goto set_dest_dir; 129 | diff --git a/rsync.1.md b/rsync.1.md 130 | --- a/rsync.1.md 131 | +++ b/rsync.1.md 132 | @@ -510,6 +510,7 @@ has its own detailed description later in this manpage. 133 | --compare-dest=DIR also compare destination files relative to DIR 134 | --copy-dest=DIR ... and include copies of unchanged files 135 | --link-dest=DIR hardlink to files in DIR when unchanged 136 | +--clone-dest=DIR clone (reflink) files from DIR when unchanged 137 | --compress, -z compress file data during the transfer 138 | --compress-choice=STR choose the compression algorithm (aka --zc) 139 | --compress-level=NUM explicitly set compression level (aka --zl) 140 | @@ -2720,6 +2721,18 @@ expand it. 141 | this bug by avoiding the `-o` option (or using `--no-o`) when sending to an 142 | old rsync. 143 | 144 | +0. `--clone-dest=DIR` 145 | + 146 | + This option behaves like [`--link-dest`](#opt), but unchanged files are 147 | + reflinked from _DIR_ to the destination directory. The files do not need 148 | + to match in attributes, as the data is cloned separately from the 149 | + attributes. 150 | + 151 | + If _DIR_ is a relative path, it is relative to the destination directory. 152 | + See also [`--compare-dest`](#opt) and [`--copy-dest`](#opt). 153 | + 154 | + All non-regular files are hard-linked (when possible). 155 | + 156 | 0. `--compress`, `-z` 157 | 158 | With this option, rsync compresses the file data as it is sent to the 159 | diff --git a/rsync.h b/rsync.h 160 | --- a/rsync.h 161 | +++ b/rsync.h 162 | @@ -175,6 +175,11 @@ 163 | #define COMPARE_DEST 1 164 | #define COPY_DEST 2 165 | #define LINK_DEST 3 166 | +#define CLONE_DEST 4 167 | + 168 | +#if !defined FICLONE && defined __linux__ 169 | +#define FICLONE _IOW(0x94, 9, int) 170 | +#endif 171 | 172 | #define MPLEX_BASE 7 173 | 174 | diff --git a/syscall.c b/syscall.c 175 | --- a/syscall.c 176 | +++ b/syscall.c 177 | @@ -146,6 +146,54 @@ int do_link(const char *old_path, const char *new_path) 178 | } 179 | #endif 180 | 181 | +int do_clone(const char *old_path, const char *new_path, mode_t mode) 182 | +{ 183 | +#ifdef FICLONE 184 | + int ifd, ofd, ret, save_errno; 185 | + 186 | + if (dry_run) return 0; 187 | + RETURN_ERROR_IF_RO_OR_LO; 188 | + 189 | + if ((ifd = do_open(old_path, O_RDONLY, 0)) < 0) { 190 | + save_errno = errno; 191 | + rsyserr(FERROR_XFER, errno, "open %s", full_fname(old_path)); 192 | + errno = save_errno; 193 | + return -1; 194 | + } 195 | + 196 | + if (robust_unlink(new_path) && errno != ENOENT) { 197 | + save_errno = errno; 198 | + rsyserr(FERROR_XFER, errno, "unlink %s", full_fname(new_path)); 199 | + close(ifd); 200 | + errno = save_errno; 201 | + return -1; 202 | + } 203 | + 204 | + mode &= INITACCESSPERMS; 205 | + if ((ofd = do_open(new_path, O_WRONLY | O_CREAT | O_TRUNC | O_EXCL, mode)) < 0) { 206 | + save_errno = errno; 207 | + rsyserr(FERROR_XFER, save_errno, "open %s", full_fname(new_path)); 208 | + close(ifd); 209 | + errno = save_errno; 210 | + return -1; 211 | + } 212 | + 213 | + ret = ioctl(ofd, FICLONE, ifd); 214 | + save_errno = errno; 215 | + close(ifd); 216 | + close(ofd); 217 | + if (ret < 0) 218 | + unlink(new_path); 219 | + errno = save_errno; 220 | + return ret; 221 | +#else 222 | + (void)old_path; 223 | + (void)new_path; 224 | + errno = ENOTSUP; 225 | + return -1; 226 | +#endif 227 | +} 228 | + 229 | int do_lchown(const char *path, uid_t owner, gid_t group) 230 | { 231 | if (dry_run) return 0; 232 | diff --git a/t_stub.c b/t_stub.c 233 | --- a/t_stub.c 234 | +++ b/t_stub.c 235 | @@ -38,6 +38,7 @@ size_t max_alloc = 0; /* max_alloc is needed when combined with util2.o */ 236 | char *partial_dir; 237 | char *module_dir; 238 | filter_rule_list daemon_filter_list; 239 | +short info_levels[COUNT_INFO], debug_levels[COUNT_DEBUG]; 240 | 241 | void rprintf(UNUSED(enum logcode code), const char *format, ...) 242 | { 243 | diff --git a/t_unsafe.c b/t_unsafe.c 244 | --- a/t_unsafe.c 245 | +++ b/t_unsafe.c 246 | @@ -28,7 +28,6 @@ int am_root = 0; 247 | int am_sender = 1; 248 | int read_only = 0; 249 | int list_only = 0; 250 | -short info_levels[COUNT_INFO], debug_levels[COUNT_DEBUG]; 251 | 252 | int 253 | main(int argc, char **argv) 254 | -------------------------------------------------------------------------------- /congestion.diff: -------------------------------------------------------------------------------- 1 | From: Dave Taht 2 | 3 | In the bufferbloat age, anything that can make for a kinder, gentler bulk 4 | transfer protocol seems desirable. 5 | 6 | This add support for user and server selectable congestion control algorithms. 7 | 8 | For example: 9 | --congestion-alg=lp # For the tcp-lp algorithm on the command line 10 | 11 | And diffserv support: 12 | --diffserv=8 for setting the CS1 bit 13 | 14 | Also available in rsync daemon modules: 15 | 16 | [mystuff] 17 | congestion alg = westwood # for a wireless connection 18 | diffserv = 8 19 | 20 | This could be improved by being able to specify a list of congestion algorithms 21 | to try, symbolic names for diffserv (CS1), a better name than 'congestion-alg', 22 | and some error checking. 23 | 24 | To use this patch, run these commands for a successful build: 25 | 26 | patch -p1 ai_addr, res->ai_addrlen) < 0) { 140 | if (connect_timeout < 0) 141 | @@ -440,6 +474,7 @@ static int *open_socket_in(int type, int port, const char *bind_addr, 142 | continue; 143 | } 144 | 145 | + set_special_sockopts(s); 146 | setsockopt(s, SOL_SOCKET, SO_REUSEADDR, 147 | (char *)&one, sizeof one); 148 | if (sockopts) 149 | -------------------------------------------------------------------------------- /date-only.diff: -------------------------------------------------------------------------------- 1 | Jeremy Bornstein wrote: 2 | 3 | I recently had the need to transfer files only with different mod 4 | dates (and to *not* transfer them based on file size differences). 5 | This is because I'm backing up files remotely on an untrusted machine, 6 | so I'm encrypting them with gpg before transfer. I discovered that 7 | rsync didn't already have a --date-only flag, so I added one and am 8 | enclosing the diffs in case you (as I hope) decide to include this 9 | option in future releases. 10 | 11 | To use this patch, run these commands for a successful build: 12 | 13 | patch -p1 st_size != F_LENGTH(file)) 31 | return 0; 32 | 33 | + if (date_only) 34 | + return !mtime_differs(st, file); 35 | + 36 | /* If always_checksum is set then we use the checksum instead 37 | * of the file mtime to determine whether to sync. */ 38 | if (always_checksum > 0) { 39 | diff --git a/options.c b/options.c 40 | --- a/options.c 41 | +++ b/options.c 42 | @@ -119,6 +119,7 @@ int safe_symlinks = 0; 43 | int copy_unsafe_links = 0; 44 | int munge_symlinks = 0; 45 | int size_only = 0; 46 | +int date_only = 0; 47 | int daemon_bwlimit = 0; 48 | int bwlimit = 0; 49 | int fuzzy_basis = 0; 50 | @@ -689,6 +690,7 @@ static struct poptOption long_options[] = { 51 | {"chmod", 0, POPT_ARG_STRING, 0, OPT_CHMOD, 0, 0 }, 52 | {"ignore-times", 'I', POPT_ARG_NONE, &ignore_times, 0, 0, 0 }, 53 | {"size-only", 0, POPT_ARG_NONE, &size_only, 0, 0, 0 }, 54 | + {"date-only", 0, POPT_ARG_NONE, &date_only, 0, 0, 0 }, 55 | {"one-file-system", 'x', POPT_ARG_NONE, 0, 'x', 0, 0 }, 56 | {"no-one-file-system",0, POPT_ARG_VAL, &one_file_system, 0, 0, 0 }, 57 | {"no-x", 0, POPT_ARG_VAL, &one_file_system, 0, 0, 0 }, 58 | @@ -2852,6 +2854,9 @@ void server_options(char **args, int *argc_p) 59 | else if (missing_args == 1 && !am_sender) 60 | args[ac++] = "--ignore-missing-args"; 61 | 62 | + if (date_only) 63 | + args[ac++] = "--date-only"; 64 | + 65 | if (modify_window_set && am_sender) { 66 | char *fmt = modify_window < 0 ? "-@%d" : "--modify-window=%d"; 67 | if (asprintf(&arg, fmt, modify_window) < 0) 68 | diff --git a/rsync.1.md b/rsync.1.md 69 | --- a/rsync.1.md 70 | +++ b/rsync.1.md 71 | @@ -504,6 +504,7 @@ has its own detailed description later in this manpage. 72 | --contimeout=SECONDS set daemon connection timeout in seconds 73 | --ignore-times, -I don't skip files that match size and time 74 | --size-only skip files that match in size 75 | +--date-only skip files that match in mod-time 76 | --modify-window=NUM, -@ set the accuracy for mod-time comparisons 77 | --temp-dir=DIR, -T create temporary files in directory DIR 78 | --fuzzy, -y find similar file for basis if no dest file 79 | @@ -776,6 +777,14 @@ expand it. 80 | after using another mirroring system which may not preserve timestamps 81 | exactly. 82 | 83 | +0. `--date-only` 84 | + 85 | + Normally rsync will skip any files that are already the same size and have 86 | + the same modification time-stamp. With the `--date-only` option, files will 87 | + be skipped if they have the same timestamp, regardless of size. This may be 88 | + useful when the remote files have passed through a size-changing filter, 89 | + e.g. for encryption. 90 | + 91 | 0. `--modify-window=NUM`, `-@` 92 | 93 | When comparing two timestamps, rsync treats the timestamps as being equal 94 | -------------------------------------------------------------------------------- /detect-renamed-lax.diff: -------------------------------------------------------------------------------- 1 | This patch adds the options --detect-renamed-lax and --detect-moved. 2 | These modify the --detect-renamed algorithm to adopt a matching file 3 | without verifying that the content is as expected. The former blindly 4 | accepts a file that matches in size and modified time. The latter 5 | requires that the filename also match (ignoring any renamed files). 6 | 7 | This patch is EXPERIMENTAL, though it did work correctly in my light 8 | testing. 9 | 10 | To use this patch, run these commands for a successful build: 11 | 12 | patch -p1 22 | 23 | based-on: patch/master/detect-renamed 24 | diff --git a/generator.c b/generator.c 25 | --- a/generator.c 26 | +++ b/generator.c 27 | @@ -467,7 +467,9 @@ static int fattr_find(struct file_struct *f, char *fname) 28 | continue; 29 | } 30 | } 31 | - ok_match = mid; 32 | + /* --detect-moved doesn't allow non-basename matches */ 33 | + if (detect_renamed != 3) 34 | + ok_match = mid; 35 | diff = u_strcmp(fmid->basename, f->basename); 36 | if (diff == 0) { 37 | good_match = mid; 38 | @@ -1991,6 +1993,21 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, 39 | fnamecmp = partialptr; 40 | fnamecmp_type = FNAMECMP_PARTIAL_DIR; 41 | statret = 0; 42 | + if (detect_renamed > 1 && quick_check_ok(FT_REG, fnamecmp, file, &sx.st)) { 43 | + /* Adopt the partial file. */ 44 | + finish_transfer(fname, fnamecmp, NULL, NULL, file, 1, 1); 45 | + handle_partial_dir(partialptr, PDIR_DELETE); 46 | + if (itemizing) 47 | + itemize(fnamecmp, file, ndx, -1, &sx, 48 | + ITEM_LOCAL_CHANGE, fnamecmp_type, NULL); 49 | +#ifdef SUPPORT_HARD_LINKS 50 | + if (preserve_hard_links && F_IS_HLINKED(file)) 51 | + finish_hard_link(file, fname, ndx, &sx.st, itemizing, code, -1); 52 | +#endif 53 | + if (remove_source_files == 1) 54 | + goto return_with_success; 55 | + goto cleanup; 56 | + } 57 | } 58 | 59 | if (!do_xfers) 60 | diff --git a/options.c b/options.c 61 | --- a/options.c 62 | +++ b/options.c 63 | @@ -744,7 +744,9 @@ static struct poptOption long_options[] = { 64 | {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 }, 65 | {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 }, 66 | {"link-dest", 0, POPT_ARG_STRING, 0, OPT_LINK_DEST, 0, 0 }, 67 | - {"detect-renamed", 0, POPT_ARG_NONE, &detect_renamed, 0, 0, 0 }, 68 | + {"detect-renamed", 0, POPT_ARG_VAL, &detect_renamed, 1, 0, 0 }, 69 | + {"detect-renamed-lax",0, POPT_ARG_VAL, &detect_renamed, 2, 0, 0 }, 70 | + {"detect-moved", 0, POPT_ARG_VAL, &detect_renamed, 3, 0, 0 }, 71 | {"fuzzy", 'y', POPT_ARG_NONE, 0, 'y', 0, 0 }, 72 | {"no-fuzzy", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 }, 73 | {"no-y", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 }, 74 | @@ -2838,8 +2840,14 @@ void server_options(char **args, int *argc_p) 75 | args[ac++] = "--super"; 76 | if (size_only) 77 | args[ac++] = "--size-only"; 78 | - if (detect_renamed) 79 | - args[ac++] = "--detect-renamed"; 80 | + if (detect_renamed) { 81 | + if (detect_renamed == 1) 82 | + args[ac++] = "--detect-renamed"; 83 | + else if (detect_renamed == 2) 84 | + args[ac++] = "--detect-renamed-lax"; 85 | + else 86 | + args[ac++] = "--detect-moved"; 87 | + } 88 | if (do_stats) 89 | args[ac++] = "--stats"; 90 | } else { 91 | diff --git a/rsync.1.md b/rsync.1.md 92 | --- a/rsync.1.md 93 | +++ b/rsync.1.md 94 | @@ -508,6 +508,8 @@ has its own detailed description later in this manpage. 95 | --temp-dir=DIR, -T create temporary files in directory DIR 96 | --fuzzy, -y find similar file for basis if no dest file 97 | --detect-renamed try to find renamed files to speed the xfer 98 | +--detect-renamed-lax ...& assume identical to src files (risky!) 99 | +--detect-moved ... only if basenames match (less risky) 100 | --compare-dest=DIR also compare destination files relative to DIR 101 | --copy-dest=DIR ... and include copies of unchanged files 102 | --link-dest=DIR hardlink to files in DIR when unchanged 103 | @@ -2652,6 +2654,20 @@ expand it. 104 | otential alternate-basis files will be removed as the transfer progresses. 105 | This option conflicts with [`--inplace`](#opt) and [`--append`](#opt). 106 | 107 | +0. `--detect-renamed-lax` 108 | + 109 | + This version of [`--detect-renamed`](#opt) makes rsync hard-link `dest/D` 110 | + to `dest/S` without verifying that `src/S` and `dest/S` have the same data. 111 | + This poses a significant risk of corrupting the destination by representing 112 | + a new source file by an unrelated destination file that coincidentally 113 | + passes the quick check with the source file. Use this option only if you 114 | + accept the risk and disk I/O is a bottleneck. 115 | + 116 | +0. `--detect-moved` 117 | + 118 | + A less risky variant of [`--detect-renamed-lax`](#opt) that only uses a 119 | + destination file that has the same basename as the new source file. 120 | + 121 | 0. `--compare-dest=DIR` 122 | 123 | This option instructs rsync to use _DIR_ on the destination machine as an 124 | -------------------------------------------------------------------------------- /direct-io.diff: -------------------------------------------------------------------------------- 1 | This patch adds the --direct-io option, which opens files with O_DIRECT. 2 | 3 | TODO: we probably need to make our I/O aligned on 512-byte boundaries. 4 | 5 | Written by: Dag Wieers 6 | 7 | To use this patch, run these commands for a successful build: 8 | 9 | patch -p1 20 | 21 | +extern int direct_io; 22 | extern int module_id; 23 | extern int local_server; 24 | extern int sanitize_paths; 25 | @@ -764,6 +765,8 @@ static struct poptOption long_options[] = { 26 | {"partial-dir", 0, POPT_ARG_STRING, &partial_dir, 0, 0, 0 }, 27 | {"delay-updates", 0, POPT_ARG_VAL, &delay_updates, 1, 0, 0 }, 28 | {"no-delay-updates", 0, POPT_ARG_VAL, &delay_updates, 0, 0, 0 }, 29 | + {"direct-io", 'n', POPT_ARG_VAL, &direct_io, 1, 0, 0 }, 30 | + {"no-direct-io", 0, POPT_ARG_VAL, &direct_io, 0, 0, 0 }, 31 | {"prune-empty-dirs",'m', POPT_ARG_VAL, &prune_empty_dirs, 1, 0, 0 }, 32 | {"no-prune-empty-dirs",0,POPT_ARG_VAL, &prune_empty_dirs, 0, 0, 0 }, 33 | {"no-m", 0, POPT_ARG_VAL, &prune_empty_dirs, 0, 0, 0 }, 34 | diff --git a/rsync.1.md b/rsync.1.md 35 | --- a/rsync.1.md 36 | +++ b/rsync.1.md 37 | @@ -495,6 +495,7 @@ has its own detailed description later in this manpage. 38 | --partial keep partially transferred files 39 | --partial-dir=DIR put a partially transferred file into DIR 40 | --delay-updates put all updated files into place at end 41 | +--direct-io don't use buffer cache for xfer file I/O 42 | --prune-empty-dirs, -m prune empty directory chains from file-list 43 | --numeric-ids don't map uid/gid values by user/group name 44 | --usermap=STRING custom username mapping 45 | @@ -3433,6 +3434,17 @@ expand it. 46 | update algorithm that is even more atomic (it uses [`--link-dest`](#opt) 47 | and a parallel hierarchy of files). 48 | 49 | +0. `--direct-io` 50 | + 51 | + This option opens files with a direct-I/O flag that makes the file I/O 52 | + avoid the buffer cache. The option only affects one side of the transfer 53 | + (unless the transfer is local). If you want it to affect both sides, use 54 | + the [`--remote-option`](#opt) (`-M`) option to specify it for the remote 55 | + side. For instance, this specifies it for both sides (via brace 56 | + expansion): 57 | + 58 | + > rsync -av {,-M}--direct-io /src/ host:/dest/ 59 | + 60 | 0. `--prune-empty-dirs`, `-m` 61 | 62 | This option tells the receiving rsync to get rid of empty directories from 63 | diff --git a/syscall.c b/syscall.c 64 | --- a/syscall.c 65 | +++ b/syscall.c 66 | @@ -44,6 +44,8 @@ extern int preserve_perms; 67 | extern int preserve_executability; 68 | extern int open_noatime; 69 | 70 | +int direct_io = 0; 71 | + 72 | #ifndef S_BLKSIZE 73 | # if defined hpux || defined __hpux__ || defined __hpux 74 | # define S_BLKSIZE 1024 75 | @@ -95,7 +97,12 @@ int do_symlink(const char *lnk, const char *path) 76 | * and write the lnk into it. */ 77 | if (am_root < 0) { 78 | int ok, len = strlen(lnk); 79 | - int fd = open(path, O_WRONLY|O_CREAT|O_TRUNC, S_IWUSR|S_IRUSR); 80 | + int flags = O_WRONLY|O_CREAT|O_TRUNC; 81 | + 82 | + if (direct_io) 83 | + flags |= O_DIRECT; 84 | + 85 | + int fd = open(path, flags, S_IWUSR|S_IRUSR); 86 | if (fd < 0) 87 | return -1; 88 | ok = write(fd, lnk, len) == len; 89 | @@ -224,6 +231,8 @@ int do_open(const char *pathname, int flags, mode_t mode) 90 | if (open_noatime) 91 | flags |= O_NOATIME; 92 | #endif 93 | + if (direct_io) 94 | + flags |= O_DIRECT; 95 | 96 | return open(pathname, flags | O_BINARY, mode); 97 | } 98 | @@ -684,6 +693,9 @@ int do_open_nofollow(const char *pathname, int flags) 99 | #endif 100 | } 101 | 102 | + if (direct_io) 103 | + flags |= O_DIRECT; 104 | + 105 | #ifdef O_NOFOLLOW 106 | fd = open(pathname, flags|O_NOFOLLOW); 107 | #else 108 | -------------------------------------------------------------------------------- /downdate.diff: -------------------------------------------------------------------------------- 1 | A patch from Stefan Müller to add the --downdate option, which works 2 | in the opposite manner as --update. 3 | 4 | To use this patch, run these commands for a successful build: 5 | 6 | patch -p1 0 && statret == 0 && file->modtime - sx.st.st_mtime >= modify_window) { 27 | + if (INFO_GTE(SKIP, 1)) 28 | + rprintf(FINFO, "%s is older\n", fname); 29 | + return; 30 | + } 31 | + 32 | fnamecmp_type = FNAMECMP_FNAME; 33 | 34 | if (statret == 0 && !(stype == FT_REG || (write_devices && stype == FT_DEVICE))) { 35 | @@ -2162,6 +2169,7 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo) 36 | ignore_existing = -ignore_existing; 37 | ignore_non_existing = -ignore_non_existing; 38 | update_only = -update_only; 39 | + downdate_only = -downdate_only; 40 | always_checksum = -always_checksum; 41 | size_only = -size_only; 42 | append_mode = -append_mode; 43 | @@ -2187,6 +2195,7 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo) 44 | ignore_existing = -ignore_existing; 45 | ignore_non_existing = -ignore_non_existing; 46 | update_only = -update_only; 47 | + downdate_only = -downdate_only; 48 | always_checksum = -always_checksum; 49 | size_only = -size_only; 50 | append_mode = -append_mode; 51 | diff --git a/options.c b/options.c 52 | --- a/options.c 53 | +++ b/options.c 54 | @@ -68,6 +68,7 @@ int omit_dir_times = 0; 55 | int omit_link_times = 0; 56 | int trust_sender = 0; 57 | int update_only = 0; 58 | +int downdate_only = 0; 59 | int open_noatime = 0; 60 | int cvs_exclude = 0; 61 | int dry_run = 0; 62 | @@ -693,6 +694,7 @@ static struct poptOption long_options[] = { 63 | {"no-one-file-system",0, POPT_ARG_VAL, &one_file_system, 0, 0, 0 }, 64 | {"no-x", 0, POPT_ARG_VAL, &one_file_system, 0, 0, 0 }, 65 | {"update", 'u', POPT_ARG_NONE, &update_only, 0, 0, 0 }, 66 | + {"downdate", 'w', POPT_ARG_NONE, &downdate_only, 0, 0, 0 }, 67 | {"existing", 0, POPT_ARG_NONE, &ignore_non_existing, 0, 0, 0 }, 68 | {"ignore-non-existing",0,POPT_ARG_NONE, &ignore_non_existing, 0, 0, 0 }, 69 | {"ignore-existing", 0, POPT_ARG_NONE, &ignore_existing, 0, 0, 0 }, 70 | diff --git a/rsync.1.md b/rsync.1.md 71 | --- a/rsync.1.md 72 | +++ b/rsync.1.md 73 | @@ -431,6 +431,7 @@ has its own detailed description later in this manpage. 74 | --backup-dir=DIR make backups into hierarchy based in DIR 75 | --suffix=SUFFIX backup suffix (default ~ w/o --backup-dir) 76 | --update, -u skip files that are newer on the receiver 77 | +--downdate, -w skip files that are older on the receiver 78 | --inplace update destination files in-place 79 | --append append data onto shorter files 80 | --append-verify --append w/old data in file checksum 81 | -------------------------------------------------------------------------------- /filter-attribute-mods.diff: -------------------------------------------------------------------------------- 1 | From: Matt McCutchen 2 | 3 | Implement the "m", "o", "g" include modifiers to tweak the permissions, 4 | owner, or group of matching files. 5 | 6 | To use this patch, run these commands for a successful build: 7 | 8 | patch -p1 ref_cnt++; 41 | + assert(chmod->ref_cnt != 0); /* Catch overflow. */ 42 | + return chmod; 43 | +} 44 | + 45 | +static void unref_filter_chmod(struct filter_chmod_struct *chmod) 46 | +{ 47 | + chmod->ref_cnt--; 48 | + if (chmod->ref_cnt == 0) { 49 | + free(chmod->modestr); 50 | + free_chmod_mode(chmod->modes); 51 | + free(chmod); 52 | + } 53 | +} 54 | + 55 | static void free_filter(filter_rule *ex) 56 | { 57 | + if (ex->rflags & FILTRULE_CHMOD) 58 | + unref_filter_chmod(ex->chmod); 59 | if (ex->rflags & FILTRULE_PERDIR_MERGE) 60 | teardown_mergelist(ex); 61 | free(ex->pattern); 62 | @@ -1006,7 +1028,9 @@ static void report_filter_result(enum logcode code, char const *name, 63 | 64 | /* This function is used to check if a file should be included/excluded 65 | * from the list of files based on its name and type etc. The value of 66 | - * filter_level is set to either SERVER_FILTERS or ALL_FILTERS. */ 67 | + * filter_level is set to either SERVER_FILTERS or ALL_FILTERS. 68 | + * "last_hit_filter_rule" will be set to the operative filter, or NULL if none. */ 69 | + 70 | int name_is_excluded(const char *fname, int name_flags, int filter_level) 71 | { 72 | if (daemon_filter_list.head && check_filter(&daemon_filter_list, FLOG, fname, name_flags) < 0) { 73 | @@ -1015,6 +1039,9 @@ int name_is_excluded(const char *fname, int name_flags, int filter_level) 74 | return 1; 75 | } 76 | 77 | + /* Don't leave a daemon include in last_hit_filter_rule. */ 78 | + last_hit_filter_rule = NULL; 79 | + 80 | if (filter_level != ALL_FILTERS) 81 | return 0; 82 | 83 | @@ -1034,7 +1061,8 @@ int check_server_filter(filter_rule_list *listp, enum logcode code, const char * 84 | } 85 | 86 | /* Return -1 if file "name" is defined to be excluded by the specified 87 | - * exclude list, 1 if it is included, and 0 if it was not matched. */ 88 | + * exclude list, 1 if it is included, and 0 if it was not matched. 89 | + * Sets last_hit_filter_rule to the filter that was hit, or NULL if none. */ 90 | int check_filter(filter_rule_list *listp, enum logcode code, 91 | const char *name, int name_flags) 92 | { 93 | @@ -1057,10 +1085,12 @@ int check_filter(filter_rule_list *listp, enum logcode code, 94 | } 95 | if (rule_matches(name, ent, name_flags)) { 96 | report_filter_result(code, name, ent, name_flags, listp->debug_type); 97 | + last_hit_filter_rule = ent; 98 | return ent->rflags & FILTRULE_INCLUDE ? 1 : -1; 99 | } 100 | } 101 | 102 | + last_hit_filter_rule = NULL; 103 | return 0; 104 | } 105 | 106 | @@ -1077,9 +1107,45 @@ static const uchar *rule_strcmp(const uchar *str, const char *rule, int rule_len 107 | return NULL; 108 | } 109 | 110 | +static char *grab_paren_value(const uchar **s_ptr) 111 | +{ 112 | + const uchar *start, *end; 113 | + int val_sz; 114 | + char *val; 115 | + 116 | + if ((*s_ptr)[1] != '(') 117 | + return NULL; 118 | + start = (*s_ptr) + 2; 119 | + 120 | + for (end = start; *end != ')'; end++) 121 | + if (!*end || *end == ' ' || *end == '_') 122 | + return NULL; 123 | + 124 | + val_sz = end - start + 1; 125 | + val = new_array(char, val_sz); 126 | + strlcpy(val, (const char *)start, val_sz); 127 | + *s_ptr = end; /* remember ++s in parse_rule_tok */ 128 | + return val; 129 | +} 130 | + 131 | +static struct filter_chmod_struct *make_chmod_struct(char *modestr) 132 | +{ 133 | + struct filter_chmod_struct *chmod; 134 | + struct chmod_mode_struct *modes = NULL; 135 | + 136 | + if (!parse_chmod(modestr, &modes)) 137 | + return NULL; 138 | + 139 | + chmod = new(struct filter_chmod_struct); 140 | + chmod->ref_cnt = 1; 141 | + chmod->modestr = modestr; 142 | + chmod->modes = modes; 143 | + return chmod; 144 | +} 145 | + 146 | #define FILTRULES_FROM_CONTAINER (FILTRULE_ABS_PATH | FILTRULE_INCLUDE \ 147 | | FILTRULE_DIRECTORY | FILTRULE_NEGATE \ 148 | - | FILTRULE_PERISHABLE) 149 | + | FILTRULE_PERISHABLE | FILTRULES_ATTRS) 150 | 151 | /* Gets the next include/exclude rule from *rulestr_ptr and advances 152 | * *rulestr_ptr to point beyond it. Stores the pattern's start (within 153 | @@ -1094,6 +1160,7 @@ static filter_rule *parse_rule_tok(const char **rulestr_ptr, 154 | const char **pat_ptr, unsigned int *pat_len_ptr) 155 | { 156 | const uchar *s = (const uchar *)*rulestr_ptr; 157 | + char *val; 158 | filter_rule *rule; 159 | unsigned int len; 160 | 161 | @@ -1112,6 +1179,12 @@ static filter_rule *parse_rule_tok(const char **rulestr_ptr, 162 | /* Inherit from the template. Don't inherit FILTRULES_SIDES; we check 163 | * that later. */ 164 | rule->rflags = template->rflags & FILTRULES_FROM_CONTAINER; 165 | + if (template->rflags & FILTRULE_CHMOD) 166 | + rule->chmod = ref_filter_chmod(template->chmod); 167 | + if (template->rflags & FILTRULE_FORCE_OWNER) 168 | + rule->force_uid = template->force_uid; 169 | + if (template->rflags & FILTRULE_FORCE_GROUP) 170 | + rule->force_gid = template->force_gid; 171 | 172 | /* Figure out what kind of a filter rule "s" is pointing at. Note 173 | * that if FILTRULE_NO_PREFIXES is set, the rule is either an include 174 | @@ -1258,11 +1331,63 @@ static filter_rule *parse_rule_tok(const char **rulestr_ptr, 175 | goto invalid; 176 | rule->rflags |= FILTRULE_EXCLUDE_SELF; 177 | break; 178 | + case 'g': { 179 | + gid_t gid; 180 | + 181 | + if (!(val = grab_paren_value(&s))) 182 | + goto invalid; 183 | + if (group_to_gid(val, &gid, True)) { 184 | + rule->rflags |= FILTRULE_FORCE_GROUP; 185 | + rule->force_gid = gid; 186 | + } else { 187 | + rprintf(FERROR, 188 | + "unknown group '%s' in filter rule: %s\n", 189 | + val, *rulestr_ptr); 190 | + exit_cleanup(RERR_SYNTAX); 191 | + } 192 | + free(val); 193 | + break; 194 | + } 195 | + case 'm': { 196 | + struct filter_chmod_struct *chmod; 197 | + 198 | + if (!(val = grab_paren_value(&s))) 199 | + goto invalid; 200 | + if ((chmod = make_chmod_struct(val))) { 201 | + if (rule->rflags & FILTRULE_CHMOD) 202 | + unref_filter_chmod(rule->chmod); 203 | + rule->rflags |= FILTRULE_CHMOD; 204 | + rule->chmod = chmod; 205 | + } else { 206 | + rprintf(FERROR, 207 | + "unparseable chmod string '%s' in filter rule: %s\n", 208 | + val, *rulestr_ptr); 209 | + exit_cleanup(RERR_SYNTAX); 210 | + } 211 | + break; 212 | + } 213 | case 'n': 214 | if (!(rule->rflags & FILTRULE_MERGE_FILE)) 215 | goto invalid; 216 | rule->rflags |= FILTRULE_NO_INHERIT; 217 | break; 218 | + case 'o': { 219 | + uid_t uid; 220 | + 221 | + if (!(val = grab_paren_value(&s))) 222 | + goto invalid; 223 | + if (user_to_uid(val, &uid, True)) { 224 | + rule->rflags |= FILTRULE_FORCE_OWNER; 225 | + rule->force_uid = uid; 226 | + } else { 227 | + rprintf(FERROR, 228 | + "unknown user '%s' in filter rule: %s\n", 229 | + val, *rulestr_ptr); 230 | + exit_cleanup(RERR_SYNTAX); 231 | + } 232 | + free(val); 233 | + break; 234 | + } 235 | case 'p': 236 | rule->rflags |= FILTRULE_PERISHABLE; 237 | break; 238 | @@ -1576,6 +1701,23 @@ char *get_rule_prefix(filter_rule *rule, const char *pat, int for_xfer, 239 | else if (am_sender) 240 | return NULL; 241 | } 242 | + if (rule->rflags & FILTRULES_ATTRS) { 243 | + if (!for_xfer || protocol_version >= 31) { 244 | + if (rule->rflags & FILTRULE_CHMOD) 245 | + if (!snappendf(&op, (buf + sizeof buf) - op, 246 | + "m(%s)", rule->chmod->modestr)) 247 | + return NULL; 248 | + if (rule->rflags & FILTRULE_FORCE_OWNER) 249 | + if (!snappendf(&op, (buf + sizeof buf) - op, 250 | + "o(%u)", (unsigned)rule->force_uid)) 251 | + return NULL; 252 | + if (rule->rflags & FILTRULE_FORCE_GROUP) 253 | + if (!snappendf(&op, (buf + sizeof buf) - op, 254 | + "g(%u)", (unsigned)rule->force_gid)) 255 | + return NULL; 256 | + } else if (!am_sender) 257 | + return NULL; 258 | + } 259 | if (op - buf > legal_len) 260 | return NULL; 261 | if (legal_len) 262 | diff --git a/flist.c b/flist.c 263 | --- a/flist.c 264 | +++ b/flist.c 265 | @@ -86,6 +86,7 @@ extern char curr_dir[MAXPATHLEN]; 266 | extern struct chmod_mode_struct *chmod_modes; 267 | 268 | extern filter_rule_list filter_list, implied_filter_list, daemon_filter_list; 269 | +extern filter_rule *last_hit_filter_rule; 270 | 271 | #ifdef ICONV_OPTION 272 | extern int filesfrom_convert; 273 | @@ -1259,7 +1260,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, 274 | } else if (readlink_stat(thisname, &st, linkname) != 0) { 275 | int save_errno = errno; 276 | /* See if file is excluded before reporting an error. */ 277 | - if (filter_level != NO_FILTERS 278 | + if (filter_level != NO_FILTERS && filter_level != ALL_FILTERS_NO_EXCLUDE 279 | && (is_excluded(thisname, 0, filter_level) 280 | || is_excluded(thisname, 1, filter_level))) { 281 | if (ignore_perishable && save_errno != ENOENT) 282 | @@ -1304,6 +1305,12 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, 283 | 284 | if (filter_level == NO_FILTERS) 285 | goto skip_filters; 286 | + if (filter_level == ALL_FILTERS_NO_EXCLUDE) { 287 | + /* Call only for the side effect of setting last_hit_filter_rule to 288 | + * any operative include filter, which might affect attributes. */ 289 | + is_excluded(thisname, S_ISDIR(st.st_mode) != 0, ALL_FILTERS); 290 | + goto skip_filters; 291 | + } 292 | 293 | if (S_ISDIR(st.st_mode)) { 294 | if (!xfer_dirs) { 295 | @@ -1536,12 +1543,23 @@ static struct file_struct *send_file_name(int f, struct file_list *flist, 296 | int flags, int filter_level) 297 | { 298 | struct file_struct *file; 299 | + BOOL can_tweak_mode; 300 | 301 | file = make_file(fname, flist, stp, flags, filter_level); 302 | if (!file) 303 | return NULL; 304 | 305 | - if (chmod_modes && !S_ISLNK(file->mode) && file->mode) 306 | + can_tweak_mode = !S_ISLNK(file->mode) && file->mode; 307 | + if ((filter_level == ALL_FILTERS || filter_level == ALL_FILTERS_NO_EXCLUDE) 308 | + && last_hit_filter_rule) { 309 | + if ((last_hit_filter_rule->rflags & FILTRULE_CHMOD) && can_tweak_mode) 310 | + file->mode = tweak_mode(file->mode, last_hit_filter_rule->chmod->modes); 311 | + if ((last_hit_filter_rule->rflags & FILTRULE_FORCE_OWNER) && uid_ndx) 312 | + F_OWNER(file) = last_hit_filter_rule->force_uid; 313 | + if ((last_hit_filter_rule->rflags & FILTRULE_FORCE_GROUP) && gid_ndx) 314 | + F_GROUP(file) = last_hit_filter_rule->force_gid; 315 | + } 316 | + if (chmod_modes && can_tweak_mode) 317 | file->mode = tweak_mode(file->mode, chmod_modes); 318 | 319 | if (f >= 0) { 320 | @@ -2443,7 +2461,7 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) 321 | struct file_struct *file; 322 | file = send_file_name(f, flist, fbuf, &st, 323 | FLAG_TOP_DIR | FLAG_CONTENT_DIR | flags, 324 | - NO_FILTERS); 325 | + ALL_FILTERS_NO_EXCLUDE); 326 | if (!file) 327 | continue; 328 | if (inc_recurse) { 329 | @@ -2457,7 +2475,7 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) 330 | } else 331 | send_if_directory(f, flist, file, fbuf, len, flags); 332 | } else 333 | - send_file_name(f, flist, fbuf, &st, flags, NO_FILTERS); 334 | + send_file_name(f, flist, fbuf, &st, flags, ALL_FILTERS_NO_EXCLUDE); 335 | } 336 | 337 | if (reenable_multiplex >= 0) 338 | diff --git a/rsync.1.md b/rsync.1.md 339 | --- a/rsync.1.md 340 | +++ b/rsync.1.md 341 | @@ -1513,7 +1513,9 @@ expand it. 342 | > --chmod=D2775,F664 343 | 344 | It is also legal to specify multiple `--chmod` options, as each additional 345 | - option is just appended to the list of changes to make. 346 | + option is just appended to the list of changes to make. To change 347 | + permissions of files matching a pattern, use an include filter with the `m` 348 | + modifier, which takes effect before any `--chmod` options. 349 | 350 | See the [`--perms`](#opt) and [`--executability`](#opt) options for how the 351 | resulting permission value can be applied to the files in the transfer. 352 | @@ -3033,6 +3035,10 @@ expand it. 353 | An older rsync client may need to use [`-s`](#opt) to avoid a complaint 354 | about wildcard characters, but a modern rsync handles this automatically. 355 | 356 | + To change ownership of files matching a pattern, use an include filter with 357 | + a `o` or `g` modifier, which take effect before uid/gid mapping and 358 | + therefore *can* be mixed with [`--usermap` & `--groupmap`](#opt--usermap). 359 | + 360 | 0. `--timeout=SECONDS` 361 | 362 | This option allows you to set a maximum I/O timeout in seconds. If no data 363 | @@ -4187,6 +4193,15 @@ The following modifiers are accepted after an include (+) or exclude (-) rule: 364 | like "CVS" and "`*.o`" are marked as perishable, and will not prevent a 365 | directory that was removed on the source from being deleted on the 366 | destination. 367 | +- An `m(CHMOD)` on an include rule tweaks the permissions of matching source 368 | + files in the same way as [`--chmod`](#opt). This happens before any tweaks 369 | + requested via [`--chmod`](#opt). 370 | +- An `o(USER)` on an include rule pretends that matching source files are owned 371 | + by `USER` (a name or numeric uid). This happens before any uid mapping by 372 | + name or [`--usermap`](#opt). 373 | +- A `g(GROUP)` on an include rule pretends that matching source files are owned 374 | + by `GROUP` (a name or numeric gid). This happens before any gid mapping by 375 | + name or [`--groupmap`](#opt--usermap). 376 | - An `x` indicates that a rule affects xattr names in xattr copy/delete 377 | operations (and is thus ignored when matching file/dir names). If no 378 | xattr-matching rules are specified, a default xattr filtering rule is used 379 | @@ -4244,6 +4259,12 @@ The following modifiers are accepted after a merge or dir-merge rule: 380 | rules in the file must not specify sides (via a modifier or a rule prefix 381 | such as `hide`). 382 | 383 | +The attribute-affecting modifiers `m`, `o`, and `g` work only in client filters 384 | +(not in daemon filters), and only the modifiers of the first matching rule are 385 | +applied. As an example, assuming [`--super`](#opt) is enabled, the rule 386 | +"`+o(root),g(root),m(go=) *~`" would ensure that all "backup" 387 | +files belong to root and are not accessible to anyone else. 388 | + 389 | Per-directory rules are inherited in all subdirectories of the directory where 390 | the merge-file was found unless the 'n' modifier was used. Each subdirectory's 391 | rules are prefixed to the inherited per-directory rules from its parents, which 392 | diff --git a/rsync.h b/rsync.h 393 | --- a/rsync.h 394 | +++ b/rsync.h 395 | @@ -181,6 +181,9 @@ 396 | #define NO_FILTERS 0 397 | #define SERVER_FILTERS 1 398 | #define ALL_FILTERS 2 399 | +/* Don't let the file be excluded, but check for a filter that might affect 400 | + * its attributes via FILTRULES_ATTRS. */ 401 | +#define ALL_FILTERS_NO_EXCLUDE 3 402 | 403 | #define XFLG_FATAL_ERRORS (1<<0) 404 | #define XFLG_OLD_PREFIXES (1<<1) 405 | @@ -982,6 +985,8 @@ struct map_struct { 406 | int status; /* first errno from read errors */ 407 | }; 408 | 409 | +struct chmod_mode_struct; 410 | + 411 | #define NAME_IS_FILE (0) /* filter name as a file */ 412 | #define NAME_IS_DIR (1<<0) /* filter name as a dir */ 413 | #define NAME_IS_XATTR (1<<2) /* filter name as an xattr */ 414 | @@ -1007,8 +1012,18 @@ struct map_struct { 415 | #define FILTRULE_CLEAR_LIST (1<<18)/* this item is the "!" token */ 416 | #define FILTRULE_PERISHABLE (1<<19)/* perishable if parent dir goes away */ 417 | #define FILTRULE_XATTR (1<<20)/* rule only applies to xattr names */ 418 | +#define FILTRULE_CHMOD (1<<21)/* chmod-tweak matching files */ 419 | +#define FILTRULE_FORCE_OWNER (1<<22)/* force owner of matching files */ 420 | +#define FILTRULE_FORCE_GROUP (1<<23)/* force group of matching files */ 421 | 422 | #define FILTRULES_SIDES (FILTRULE_SENDER_SIDE | FILTRULE_RECEIVER_SIDE) 423 | +#define FILTRULES_ATTRS (FILTRULE_CHMOD | FILTRULE_FORCE_OWNER | FILTRULE_FORCE_GROUP) 424 | + 425 | +struct filter_chmod_struct { 426 | + unsigned int ref_cnt; 427 | + char *modestr; 428 | + struct chmod_mode_struct *modes; 429 | +}; 430 | 431 | typedef struct filter_struct { 432 | struct filter_struct *next; 433 | @@ -1018,6 +1033,11 @@ typedef struct filter_struct { 434 | int slash_cnt; 435 | struct filter_list_struct *mergelist; 436 | } u; 437 | + /* TODO: Use an "extras" mechanism to avoid 438 | + * allocating this memory when we don't need it. */ 439 | + struct filter_chmod_struct *chmod; 440 | + uid_t force_uid; 441 | + gid_t force_gid; 442 | uchar elide; 443 | } filter_rule; 444 | 445 | diff --git a/util1.c b/util1.c 446 | --- a/util1.c 447 | +++ b/util1.c 448 | @@ -918,6 +918,25 @@ size_t stringjoin(char *dest, size_t destsize, ...) 449 | return ret; 450 | } 451 | 452 | +/* Append formatted text at *dest_ptr up to a maximum of sz (like snprintf). 453 | + * On success, advance *dest_ptr and return True; on overflow, return False. */ 454 | +BOOL snappendf(char **dest_ptr, size_t sz, const char *format, ...) 455 | +{ 456 | + va_list ap; 457 | + size_t len; 458 | + 459 | + va_start(ap, format); 460 | + len = vsnprintf(*dest_ptr, sz, format, ap); 461 | + va_end(ap); 462 | + 463 | + if (len >= sz) 464 | + return False; 465 | + else { 466 | + *dest_ptr += len; 467 | + return True; 468 | + } 469 | +} 470 | + 471 | int count_dir_elements(const char *p) 472 | { 473 | int cnt = 0, new_component = 1; 474 | -------------------------------------------------------------------------------- /ignore-case.diff: -------------------------------------------------------------------------------- 1 | This adds the --ignore-case option, which makes rsync compare filenames 2 | in a case-insensitive manner. 3 | 4 | To use this patch, run these commands for a successful build: 5 | 6 | patch -p1 = 0 ? f_name(dir_flist->files[dir_ndx], NULL) : empty_dir; 55 | - if (strcmp(cur_dir, d) != 0) { 56 | + int dir_differs = ignore_case ? strcasecmp(cur_dir, d) : strcmp(cur_dir, d); 57 | + if (dir_differs) { 58 | rprintf(FERROR, 59 | "ABORTING due to invalid path from sender: %s/%s\n", 60 | cur_dir, file->basename); 61 | @@ -3205,6 +3207,7 @@ int f_name_cmp(const struct file_struct *f1, const struct file_struct *f2) 62 | { 63 | int dif; 64 | const uchar *c1, *c2; 65 | + uchar ch1, ch2; 66 | enum fnc_state state1, state2; 67 | enum fnc_type type1, type2; 68 | enum fnc_type t_path = protocol_version >= 29 ? t_PATH : t_ITEM; 69 | @@ -3315,7 +3318,15 @@ int f_name_cmp(const struct file_struct *f1, const struct file_struct *f2) 70 | if (type1 != type2) 71 | return type1 == t_PATH ? 1 : -1; 72 | } 73 | - } while ((dif = (int)*c1++ - (int)*c2++) == 0); 74 | + ch1 = *c1++; 75 | + ch2 = *c2++; 76 | + if (ignore_case) { 77 | + if (isupper(ch1)) 78 | + ch1 = tolower(ch1); 79 | + if (isupper(ch2)) 80 | + ch2 = tolower(ch2); 81 | + } 82 | + } while ((dif = (int)ch1 - (int)ch2) == 0); 83 | 84 | return dif; 85 | } 86 | diff --git a/ifuncs.h b/ifuncs.h 87 | --- a/ifuncs.h 88 | +++ b/ifuncs.h 89 | @@ -110,3 +110,38 @@ static inline char *my_strdup(const char *str, const char *file, int line) 90 | memcpy(buf, str, len); 91 | return buf; 92 | } 93 | + 94 | +static inline int 95 | +strEQ(const char *s1, const char *s2) 96 | +{ 97 | + return strcmp(s1, s2) == 0; 98 | +} 99 | + 100 | +static inline int 101 | +strnEQ(const char *s1, const char *s2, size_t n) 102 | +{ 103 | + return strncmp(s1, s2, n) == 0; 104 | +} 105 | + 106 | +static inline int 107 | +ic_strEQ(const char *s1, const char *s2) 108 | +{ 109 | + extern int ignore_case; 110 | + if (ignore_case) 111 | + return strcasecmp(s1, s2) == 0; 112 | + return strcmp(s1, s2) == 0; 113 | +} 114 | + 115 | +static inline int 116 | +ic_strnEQ(const char *s1, const char *s2, size_t n) 117 | +{ 118 | + extern int ignore_case; 119 | + if (ignore_case) 120 | + return strncasecmp(s1, s2, n) == 0; 121 | + return strncmp(s1, s2, n) == 0; 122 | +} 123 | + 124 | +#define strNE(s1,s2) (!strEQ(s1,s2)) 125 | +#define strnNE(s1,s2,n) (!strnEQ(s1,s2,n)) 126 | +#define ic_strNE(s1,s2) (!ic_strEQ(s1,s2)) 127 | +#define ic_strnNE(s1,s2) (!ic_strnEQ(s1,s2,n)) 128 | diff --git a/lib/wildmatch.c b/lib/wildmatch.c 129 | --- a/lib/wildmatch.c 130 | +++ b/lib/wildmatch.c 131 | @@ -53,6 +53,8 @@ 132 | #define ISUPPER(c) (ISASCII(c) && isupper(c)) 133 | #define ISXDIGIT(c) (ISASCII(c) && isxdigit(c)) 134 | 135 | +extern int ignore_case; 136 | + 137 | #ifdef WILD_TEST_ITERATIONS 138 | int wildmatch_iteration_count; 139 | #endif 140 | @@ -72,6 +74,8 @@ static int dowild(const uchar *p, const uchar *text, const uchar*const *a) 141 | for ( ; (p_ch = *p) != '\0'; text++, p++) { 142 | int matched, special; 143 | uchar t_ch, prev_ch; 144 | + if (ignore_case && ISUPPER(p_ch)) 145 | + p_ch = tolower(p_ch); 146 | while ((t_ch = *text) == '\0') { 147 | if (*a == NULL) { 148 | if (p_ch != '*') 149 | @@ -237,12 +241,21 @@ static int dowild(const uchar *p, const uchar *text, const uchar*const *a) 150 | * of "text" and any strings in array "a". */ 151 | static int doliteral(const uchar *s, const uchar *text, const uchar*const *a) 152 | { 153 | + uchar s_ch, t_ch; 154 | for ( ; *s != '\0'; text++, s++) { 155 | while (*text == '\0') { 156 | if ((text = *a++) == NULL) 157 | return FALSE; 158 | } 159 | - if (*text != *s) 160 | + s_ch = *s; 161 | + t_ch = *text; 162 | + if (ignore_case) { 163 | + if (ISUPPER(s_ch)) 164 | + s_ch = tolower(s_ch); 165 | + if (ISUPPER(t_ch)) 166 | + t_ch = tolower(t_ch); 167 | + } 168 | + if (t_ch != s_ch) 169 | return FALSE; 170 | } 171 | 172 | @@ -288,10 +301,14 @@ static const uchar *trailing_N_elements(const uchar*const **a_ptr, int count) 173 | int wildmatch(const char *pattern, const char *text) 174 | { 175 | static const uchar *nomore[1]; /* A NULL pointer. */ 176 | + int ret; 177 | #ifdef WILD_TEST_ITERATIONS 178 | wildmatch_iteration_count = 0; 179 | #endif 180 | - return dowild((const uchar*)pattern, (const uchar*)text, nomore) == TRUE; 181 | + force_lower_case = ignore_case; 182 | + ret = dowild((const uchar*)pattern, (const uchar*)text, nomore) == TRUE; 183 | + force_lower_case = 0; 184 | + return ret; 185 | } 186 | 187 | /* Match the "pattern" against the forced-to-lower-case "text" string. */ 188 | @@ -331,12 +348,14 @@ int wildmatch_array(const char *pattern, const char*const *texts, int where) 189 | if (!text) 190 | return FALSE; 191 | 192 | + force_lower_case = ignore_case; 193 | + 194 | if ((matched = dowild(p, text, a)) != TRUE && where < 0 195 | && matched != ABORT_ALL) { 196 | while (1) { 197 | if (*text == '\0') { 198 | if ((text = (uchar*)*a++) == NULL) 199 | - return FALSE; 200 | + break; 201 | continue; 202 | } 203 | if (*text++ == '/' && (matched = dowild(p, text, a)) != FALSE 204 | @@ -344,6 +363,9 @@ int wildmatch_array(const char *pattern, const char*const *texts, int where) 205 | break; 206 | } 207 | } 208 | + 209 | + force_lower_case = 0; 210 | + 211 | return matched == TRUE; 212 | } 213 | 214 | diff --git a/options.c b/options.c 215 | --- a/options.c 216 | +++ b/options.c 217 | @@ -131,6 +131,7 @@ OFF_T max_size = -1; 218 | OFF_T min_size = -1; 219 | int ignore_errors = 0; 220 | int modify_window = 0; 221 | +int ignore_case = 0; 222 | int blocking_io = -1; 223 | int checksum_seed = 0; 224 | int inplace = 0; 225 | @@ -784,6 +785,8 @@ static struct poptOption long_options[] = { 226 | {"read-batch", 0, POPT_ARG_STRING, &batch_name, OPT_READ_BATCH, 0, 0 }, 227 | {"write-batch", 0, POPT_ARG_STRING, &batch_name, OPT_WRITE_BATCH, 0, 0 }, 228 | {"only-write-batch", 0, POPT_ARG_STRING, &batch_name, OPT_ONLY_WRITE_BATCH, 0, 0 }, 229 | + {"ignore-case", 0, POPT_ARG_VAL, &ignore_case, 1, 0, 0 }, 230 | + {"no-ignore-case", 0, POPT_ARG_VAL, &ignore_case, 0, 0, 0 }, 231 | {"files-from", 0, POPT_ARG_STRING, &files_from, 0, 0, 0 }, 232 | {"from0", '0', POPT_ARG_VAL, &eol_nulls, 1, 0, 0}, 233 | {"no-from0", 0, POPT_ARG_VAL, &eol_nulls, 0, 0, 0}, 234 | @@ -2865,6 +2868,9 @@ void server_options(char **args, int *argc_p) 235 | args[ac++] = arg; 236 | } 237 | 238 | + if (ignore_case) 239 | + args[ac++] = "--ignore-case"; 240 | + 241 | if (partial_dir && am_sender) { 242 | if (partial_dir != tmp_partialdir) { 243 | args[ac++] = "--partial-dir"; 244 | diff --git a/rsync.1.md b/rsync.1.md 245 | --- a/rsync.1.md 246 | +++ b/rsync.1.md 247 | @@ -528,6 +528,7 @@ has its own detailed description later in this manpage. 248 | --secluded-args, -s use the protocol to safely send the args 249 | --trust-sender trust the remote sender's file list 250 | --copy-as=USER[:GROUP] specify user & optional group for the copy 251 | +--ignore-case ignore case when comparing filenames 252 | --address=ADDRESS bind address for outgoing socket to daemon 253 | --port=PORT specify double-colon alternate port number 254 | --sockopts=OPTIONS specify custom TCP options 255 | @@ -2583,6 +2584,12 @@ expand it. 256 | 257 | > sudo rsync -aive lsh -M--copy-as=joe src/ lh:dest/ 258 | 259 | +0. `--ignore-case` 260 | + 261 | + This option tells rsync to ignore upper-/lower-case differences when 262 | + comparing filenames. This can avoid problems when sending files to a 263 | + filesystem that ignores these differences. 264 | + 265 | 0. `--temp-dir=DIR`, `-T` 266 | 267 | This option instructs rsync to use DIR as a scratch directory when creating 268 | diff --git a/t_stub.c b/t_stub.c 269 | --- a/t_stub.c 270 | +++ b/t_stub.c 271 | @@ -34,6 +34,7 @@ int preserve_perms = 0; 272 | int preserve_executability = 0; 273 | int omit_link_times = 0; 274 | int open_noatime = 0; 275 | +int ignore_case = 0; 276 | size_t max_alloc = 0; /* max_alloc is needed when combined with util2.o */ 277 | char *partial_dir; 278 | char *module_dir; 279 | diff --git a/wildtest.c b/wildtest.c 280 | --- a/wildtest.c 281 | +++ b/wildtest.c 282 | @@ -30,6 +30,7 @@ 283 | int fnmatch_errors = 0; 284 | #endif 285 | 286 | +int ignore_case = 0; 287 | int wildmatch_errors = 0; 288 | 289 | typedef char bool; 290 | -------------------------------------------------------------------------------- /kerberos.diff: -------------------------------------------------------------------------------- 1 | This patch adds a kerberos authentication method to daemon mode. 2 | 3 | NOTE: minimally munged to work with 3.1.1, but as yet untested! 4 | 5 | To use this patch, run these commands for a successful build: 6 | 7 | patch -p1 @])], 88 | + [], 89 | + [with_gssapi=check]) 90 | + 91 | +AH_TEMPLATE([GSSAPI_OPTION], 92 | +[Define if you want GSSAPI authentication. Specifing a value will set the search path.]) 93 | + 94 | +AS_IF([test "x$with_gssapi" != xno], 95 | + [AC_SEARCH_LIBS([gss_import_name], gss gssapi_krb5 , 96 | + [AC_CHECK_HEADERS(gssapi/gssapi_generic.h gssapi/gssapi.h) ] 97 | + [ AC_DEFINE([GSSAPI_OPTION], [1]) ] 98 | + , 99 | + [if test "x$with_gssapi" = xcheck; then 100 | + AC_MSG_FAILURE( 101 | + [--with-gssapi was given, but test for function failed]) 102 | + fi 103 | + ]) 104 | + ]) 105 | + 106 | +if test x"$enable_gssapi" != x"no"; then 107 | + AC_DEFINE(GSSAPI_OPTION, 1) 108 | +fi 109 | + 110 | AC_CACHE_CHECK([whether chown() modifies symlinks],rsync_cv_chown_modifies_symlink,[ 111 | AC_RUN_IFELSE([AC_LANG_SOURCE([[ 112 | #if HAVE_UNISTD_H 113 | diff --git a/daemon-parm.txt b/daemon-parm.txt 114 | --- a/daemon-parm.txt 115 | +++ b/daemon-parm.txt 116 | @@ -60,6 +60,7 @@ BOOL read_only True 117 | BOOL reverse_lookup True 118 | BOOL strict_modes True 119 | BOOL transfer_logging False 120 | +BOOL use_gssapi False 121 | BOOL write_only False 122 | 123 | BOOL3 munge_symlinks Unset 124 | diff --git a/gss-auth.c b/gss-auth.c 125 | new file mode 100644 126 | --- /dev/null 127 | +++ b/gss-auth.c 128 | @@ -0,0 +1,334 @@ 129 | +/* 130 | + * GSSAPI authentication. 131 | + * 132 | + * Copyright (C) 1998-2001 Andrew Tridgell 133 | + * Copyright (C) 2001-2002 Martin Pool 134 | + * Copyright (C) 2002-2008 Wayne Davison 135 | + * 136 | + * This program is free software; you can redistribute it and/or modify 137 | + * it under the terms of the GNU General Public License as published by 138 | + * the Free Software Foundation; either version 3 of the License, or 139 | + * (at your option) any later version. 140 | + * 141 | + * This program is distributed in the hope that it will be useful, 142 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of 143 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 144 | + * GNU General Public License for more details. 145 | + * 146 | + * You should have received a copy of the GNU General Public License along 147 | + * with this program; if not, visit the http://fsf.org website. 148 | + */ 149 | + 150 | +#include "rsync.h" 151 | + 152 | +#ifdef GSSAPI_OPTION 153 | + 154 | +#define RSYNC_GSS_SERVICE "host" 155 | + 156 | +struct init_context_data { 157 | + gss_cred_id_t initiator_cred_handle; 158 | + gss_ctx_id_t *context_handle; 159 | + gss_name_t target_name; 160 | + gss_OID mech_type; 161 | + OM_uint32 req_flags; 162 | + OM_uint32 time_req; 163 | + gss_channel_bindings_t input_chan_bindings; 164 | + gss_OID *actual_mech_type; 165 | + OM_uint32 *ret_flags; 166 | + OM_uint32 *time_rec; 167 | +}; 168 | + 169 | +struct accept_context_data { 170 | + gss_ctx_id_t *context_handle; 171 | + gss_cred_id_t acceptor_cred_handle; 172 | + gss_channel_bindings_t input_chan_bindings; 173 | + gss_name_t *src_name; 174 | + gss_OID *mech_type; 175 | + OM_uint32 *ret_flags; 176 | + OM_uint32 *time_rec; 177 | + gss_cred_id_t *delegated_cred_handle; 178 | +}; 179 | + 180 | +int auth_gss_client(int fd, const char *host) 181 | +{ 182 | + gss_ctx_id_t ctxt = GSS_C_NO_CONTEXT; 183 | + gss_name_t target_name = GSS_C_NO_NAME; 184 | + struct init_context_data cb_data; 185 | + char *buffer; 186 | + int status; 187 | + OM_uint32 min_stat; 188 | + 189 | + buffer = new_array(char, (strlen(host) + 2 + strlen(RSYNC_GSS_SERVICE))); 190 | + 191 | + sprintf(buffer, "%s@%s", RSYNC_GSS_SERVICE, host); 192 | + 193 | + import_gss_name(&target_name, buffer, GSS_C_NT_HOSTBASED_SERVICE); 194 | + free(buffer); 195 | + 196 | + cb_data.initiator_cred_handle = GSS_C_NO_CREDENTIAL; 197 | + cb_data.context_handle = &ctxt; 198 | + cb_data.target_name = target_name; 199 | + cb_data.mech_type = GSS_C_NO_OID; 200 | + cb_data.req_flags = GSS_C_MUTUAL_FLAG | GSS_C_REPLAY_FLAG; 201 | + cb_data.time_req = 0; 202 | + cb_data.input_chan_bindings = GSS_C_NO_CHANNEL_BINDINGS; 203 | + cb_data.actual_mech_type = NULL; 204 | + cb_data.ret_flags = NULL; 205 | + cb_data.time_rec = NULL; 206 | + 207 | + status = do_gss_dialog(fd, fd, 0, &cb_init_sec_context, (void *)&cb_data); 208 | + if (ctxt != GSS_C_NO_CONTEXT) 209 | + gss_delete_sec_context(&min_stat, &ctxt, GSS_C_NO_BUFFER); 210 | + free_gss_name(&target_name); 211 | + 212 | + return status; 213 | +} 214 | + 215 | +/* 216 | + * The call back function for a gss_init_sec_context dialog 217 | + */ 218 | +OM_uint32 cb_init_sec_context(OM_uint32 *min_statp, gss_buffer_t in_token, gss_buffer_t out_token, void *cb_data) 219 | +{ 220 | + struct init_context_data *context_data; 221 | + 222 | + context_data = (struct init_context_data *) cb_data; 223 | + return gss_init_sec_context(min_statp, context_data->initiator_cred_handle, context_data->context_handle, context_data->target_name, context_data->mech_type, context_data->req_flags, context_data->time_req, context_data->input_chan_bindings, in_token, context_data->actual_mech_type, out_token, context_data->ret_flags, context_data->time_rec); 224 | +} 225 | + 226 | +/* Possibly negotiate authentication with the client. Use "leader" to 227 | + * start off the auth if necessary. 228 | + * 229 | + * Return NULL if authentication failed. Return "" if anonymous access. 230 | + * Otherwise return username. 231 | + */ 232 | +char *auth_gss_server(int fd_in, int fd_out, int module, const char *host, const char *addr, const char *leader) 233 | +{ 234 | + struct accept_context_data cb_data; 235 | + gss_cred_id_t server_creds = GSS_C_NO_CREDENTIAL; 236 | + gss_ctx_id_t context = GSS_C_NO_CONTEXT; 237 | + OM_uint32 ret_flags; 238 | + char *users = lp_auth_users(module); 239 | + OM_uint32 maj_stat, min_stat; 240 | + gss_name_t server_name = GSS_C_NO_NAME; 241 | + gss_name_t client_name = GSS_C_NO_NAME; 242 | + gss_OID doid = GSS_C_NO_OID; 243 | + char *user = NULL; 244 | + 245 | + /* if no auth list then allow anyone in! */ 246 | + if (!users || !*users) 247 | + return ""; 248 | + 249 | + import_gss_name(&server_name, "host", GSS_C_NT_HOSTBASED_SERVICE); 250 | + 251 | + maj_stat = gss_acquire_cred(&min_stat, server_name, GSS_C_INDEFINITE, GSS_C_NULL_OID_SET, GSS_C_ACCEPT, &server_creds, NULL, NULL); 252 | + if (maj_stat != GSS_S_COMPLETE) { 253 | + error_gss(maj_stat, min_stat, "error acquiring credentials on module %s from %s (%s)", lp_name(module), host, addr); 254 | + return NULL; 255 | + } 256 | + 257 | + io_printf(fd_out, "%s\n", leader); 258 | + 259 | + cb_data.context_handle = &context; 260 | + cb_data.acceptor_cred_handle = server_creds; 261 | + cb_data.input_chan_bindings = GSS_C_NO_CHANNEL_BINDINGS; 262 | + cb_data.src_name = &client_name; 263 | + cb_data.mech_type = &doid; 264 | + cb_data.ret_flags = &ret_flags; 265 | + cb_data.time_rec = NULL; 266 | + cb_data.delegated_cred_handle = NULL; 267 | + 268 | + if (do_gss_dialog(fd_in, fd_out, -1, &cb_accept_sec_context, (void *)&cb_data) < 0) 269 | + return NULL; 270 | + 271 | + user = get_cn(client_name, doid); 272 | + 273 | + free_gss_name(&server_name); 274 | + free_gss_name(&client_name); 275 | + 276 | + return user; 277 | +} 278 | + 279 | +/* 280 | + * The call back function for a gss_accept_sec_context dialog 281 | + */ 282 | +OM_uint32 cb_accept_sec_context(OM_uint32 *min_statp, gss_buffer_t in_token, gss_buffer_t out_token, void *cb_data) 283 | +{ 284 | + struct accept_context_data *context_data; 285 | + 286 | + context_data = (struct accept_context_data *) cb_data; 287 | + return gss_accept_sec_context(min_statp, context_data->context_handle, context_data->acceptor_cred_handle, in_token, context_data->input_chan_bindings, context_data->src_name, context_data->mech_type, out_token, context_data->ret_flags, context_data->time_rec, context_data->delegated_cred_handle); 288 | +} 289 | + 290 | +void free_gss_buffer(gss_buffer_t gss_buffer) 291 | +{ 292 | + OM_uint32 maj_stat, min_stat; 293 | + 294 | + if (gss_buffer->length > 0) { 295 | + maj_stat = gss_release_buffer(&min_stat, gss_buffer); 296 | + if (maj_stat != GSS_S_COMPLETE) { 297 | + error_gss(maj_stat, min_stat, "can't release a buffer"); 298 | + } 299 | + } 300 | +} 301 | + 302 | +void free_gss_name(gss_name_t *gss_buffer) 303 | +{ 304 | + OM_uint32 maj_stat, min_stat; 305 | + 306 | + if (*gss_buffer != GSS_C_NO_NAME) { 307 | + maj_stat = gss_release_name(&min_stat, gss_buffer); 308 | + if (maj_stat != GSS_S_COMPLETE) { 309 | + error_gss(maj_stat, min_stat, "can't release a name"); 310 | + } 311 | + } 312 | +} 313 | + 314 | +void import_gss_name(gss_name_t *gss_name, const char *name, gss_OID type) 315 | +{ 316 | + gss_buffer_desc gssname; 317 | + OM_uint32 maj_stat, min_stat; 318 | + 319 | + gssname.value = strdup(name); 320 | + gssname.length = strlen(name) +1 ; 321 | + 322 | + maj_stat = gss_import_name(&min_stat, &gssname, type, gss_name); 323 | + 324 | + if (maj_stat != GSS_S_COMPLETE) 325 | + error_gss(maj_stat, min_stat, "can't resolve %s", name); 326 | + 327 | + free_gss_buffer(&gssname); 328 | +} 329 | + 330 | +char *export_name(const gss_name_t input_name) 331 | +{ 332 | + OM_uint32 maj_stat, min_stat; 333 | + gss_buffer_desc exported_name; 334 | + char *exported; 335 | + gss_OID name_oid; 336 | + 337 | + exported = NULL; 338 | + 339 | + maj_stat = gss_display_name(&min_stat, input_name, &exported_name, &name_oid); 340 | + if (maj_stat != GSS_S_COMPLETE) { 341 | + error_gss(maj_stat, min_stat, "can't get display name"); 342 | + return NULL; 343 | + } 344 | + 345 | + if (exported_name.length > 0) 346 | + exported = strdup(exported_name.value); 347 | + 348 | + free_gss_buffer(&exported_name); 349 | + 350 | + return exported; 351 | +} 352 | + 353 | +void error_gss(OM_uint32 major, OM_uint32 minor, const char *format, ...) 354 | +{ 355 | + OM_uint32 min_stat; 356 | + gss_buffer_desc gss_msg = GSS_C_EMPTY_BUFFER; 357 | + OM_uint32 msg_ctx; 358 | + va_list ap; 359 | + char message[BIGPATHBUFLEN]; 360 | + 361 | + va_start(ap, format); 362 | + vsnprintf(message, sizeof message, format, ap); 363 | + va_end(ap); 364 | + 365 | + msg_ctx = 0; 366 | + if (major != GSS_S_FAILURE) /* Don't print unspecified failure, the message is useless */ 367 | + do { 368 | + gss_display_status(&min_stat, major, GSS_C_GSS_CODE, GSS_C_NULL_OID, &msg_ctx, &gss_msg); 369 | + rprintf(FERROR, "GSS-API error: %s: %s\n", message, (char *) gss_msg.value); 370 | + free_gss_buffer(&gss_msg); 371 | + } while (msg_ctx != 0); 372 | + 373 | + if (minor != 0) { 374 | + do { 375 | + gss_display_status(&min_stat, minor, GSS_C_MECH_CODE, GSS_C_NULL_OID, &msg_ctx, &gss_msg); 376 | + rprintf(FERROR, "GSS-API error: %s: %s\n",message, (char *) gss_msg.value); 377 | + free_gss_buffer(&gss_msg); 378 | + } while (msg_ctx != 0); 379 | + } 380 | +} 381 | + 382 | +/* 383 | + * This function manage a gss dialog 384 | + * gss tokens are eaten by a call-back function and then send by this function. 385 | + * Argument to this function can be passed throught the cb_data argument 386 | + * When told to act as a server, it just begin to wait for a first token before beginning operation 387 | + * on it 388 | + */ 389 | +int do_gss_dialog(int fd_in, int fd_out, int isServer, OM_uint32 (*eat_token)(OM_uint32 *,gss_buffer_t, gss_buffer_t, void *), void *cb_data) 390 | +{ 391 | + OM_uint32 maj_stat, min_stat; 392 | + gss_buffer_desc in_token = GSS_C_EMPTY_BUFFER; 393 | + gss_buffer_desc out_token = GSS_C_EMPTY_BUFFER; 394 | + 395 | + if (isServer) 396 | + recv_gss_token(fd_in, &in_token); 397 | + 398 | + do { 399 | + maj_stat = (*eat_token)(&min_stat, &in_token, &out_token, cb_data); 400 | + free_gss_buffer(&in_token); 401 | + if (maj_stat != GSS_S_COMPLETE 402 | + && maj_stat != GSS_S_CONTINUE_NEEDED) { 403 | + error_gss(maj_stat, min_stat, "error during dialog"); 404 | + return -1; 405 | + } 406 | + 407 | + if (out_token.length != 0) { 408 | + send_gss_token(fd_out, &out_token); 409 | + } 410 | + free_gss_buffer(&out_token); 411 | + 412 | + if (maj_stat == GSS_S_CONTINUE_NEEDED) { 413 | + recv_gss_token(fd_in, &in_token); 414 | + } 415 | + } while (maj_stat == GSS_S_CONTINUE_NEEDED); 416 | + 417 | + return 0; 418 | +} 419 | + 420 | +char *get_cn(const gss_name_t input_name, const gss_OID mech_type) 421 | +{ 422 | + OM_uint32 maj_stat, min_stat; 423 | + gss_name_t output_name; 424 | + gss_buffer_desc exported_name; 425 | + char *cn; 426 | + 427 | + cn = NULL; 428 | + maj_stat = gss_canonicalize_name(&min_stat, input_name, mech_type, &output_name); 429 | + if (maj_stat != GSS_S_COMPLETE) { 430 | + error_gss(maj_stat, min_stat, "canonizing name"); 431 | + return NULL; 432 | + } 433 | + 434 | + maj_stat = gss_export_name(&min_stat, output_name, &exported_name); 435 | + if (maj_stat != GSS_S_COMPLETE) { 436 | + error_gss(maj_stat, min_stat, "canonizing name"); 437 | + return NULL; 438 | + } 439 | + if (exported_name.length > 0) 440 | + cn = strdup(exported_name.value); 441 | + 442 | + free_gss_name(&output_name); 443 | + free_gss_buffer(&exported_name); 444 | + 445 | + return cn; 446 | +} 447 | + 448 | +void send_gss_token(int fd, gss_buffer_t token) 449 | +{ 450 | + write_int(fd, token->length); 451 | + write_buf(fd, token->value, token->length); 452 | +} 453 | + 454 | +void recv_gss_token(int fd, gss_buffer_t token) 455 | +{ 456 | + token->length = read_int(fd); 457 | + if (token->length > 0) { 458 | + token->value = new_array(char, token->length); 459 | + read_buf(fd, token->value, token->length); 460 | + } 461 | +} 462 | +#endif /* GSSAPI_OPTION */ 463 | diff --git a/main.c b/main.c 464 | --- a/main.c 465 | +++ b/main.c 466 | @@ -1580,7 +1580,7 @@ static int start_client(int argc, char *argv[]) 467 | * remote shell command, we need to do the RSYNCD protocol first */ 468 | if (daemon_connection) { 469 | int tmpret; 470 | - tmpret = start_inband_exchange(f_in, f_out, shell_user, remote_argc, remote_argv); 471 | + tmpret = start_inband_exchange(f_in, f_out, shell_user, shell_machine, remote_argc, remote_argv); 472 | if (tmpret < 0) 473 | return tmpret; 474 | } 475 | diff --git a/rsync.h b/rsync.h 476 | --- a/rsync.h 477 | +++ b/rsync.h 478 | @@ -529,6 +529,15 @@ enum delret { 479 | #define iconv_t int 480 | #endif 481 | 482 | +#ifdef GSSAPI_OPTION 483 | +#ifdef HAVE_GSSAPI_GSSAPI_GENERIC_H 484 | +#include 485 | +#endif 486 | +#ifdef HAVE_GSSAPI_GSSAPI_H 487 | +#include 488 | +#endif 489 | +#endif 490 | + 491 | #include 492 | 493 | #include "lib/pool_alloc.h" 494 | -------------------------------------------------------------------------------- /link-by-hash.diff: -------------------------------------------------------------------------------- 1 | Jason M. Felice wrote: 2 | 3 | This patch adds the --link-by-hash=DIR option, which hard links received files 4 | in a link farm arranged by MD4 or MD5 file hash. The result is that the system 5 | will only store one copy of the unique contents of each file, regardless of the 6 | file's name. 7 | 8 | To use this patch, run these commands for a successful build: 9 | 10 | patch -p1 0) { 95 | rprintf(FLOG, "rsync allowed access on module %s from %s (%s)\n", 96 | name, host, addr); 97 | diff --git a/daemon-parm.txt b/daemon-parm.txt 98 | --- a/daemon-parm.txt 99 | +++ b/daemon-parm.txt 100 | @@ -29,6 +29,7 @@ STRING hosts_deny NULL 101 | STRING include NULL 102 | STRING include_from NULL 103 | STRING incoming_chmod NULL 104 | +STRING link_by_hash_dir NULL 105 | STRING lock_file DEFAULT_LOCK_FILE 106 | STRING log_file NULL 107 | STRING log_format "%o %h [%a] %m (%u) %f %l" 108 | diff --git a/hashlink.c b/hashlink.c 109 | new file mode 100644 110 | --- /dev/null 111 | +++ b/hashlink.c 112 | @@ -0,0 +1,92 @@ 113 | +/* 114 | + Copyright (C) Cronosys, LLC 2004 115 | + 116 | + This program is free software; you can redistribute it and/or modify 117 | + it under the terms of the GNU General Public License as published by 118 | + the Free Software Foundation; either version 2 of the License, or 119 | + (at your option) any later version. 120 | + 121 | + This program is distributed in the hope that it will be useful, 122 | + but WITHOUT ANY WARRANTY; without even the implied warranty of 123 | + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 124 | + GNU General Public License for more details. 125 | + 126 | + You should have received a copy of the GNU General Public License 127 | + along with this program; if not, write to the Free Software 128 | + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 129 | +*/ 130 | + 131 | +/* This file contains code used by the --link-by-hash option. */ 132 | + 133 | +#include "rsync.h" 134 | +#include "inums.h" 135 | + 136 | +extern int protocol_version; 137 | +extern char *link_by_hash_dir; 138 | +extern char sender_file_sum[MAX_DIGEST_LEN]; 139 | + 140 | +char link_by_hash_extra_sum[MAX_DIGEST_LEN]; /* Only used when md4 sums are in the transfer */ 141 | + 142 | +#ifdef HAVE_LINK 143 | + 144 | +/* This function is always called after a file is received, so the 145 | + * sender_file_sum buffer has whatever the last checksum was for the 146 | + * transferred file. */ 147 | +void link_by_hash(const char *fname, const char *fnametmp, struct file_struct *file) 148 | +{ 149 | + STRUCT_STAT st; 150 | + char *hashname, *last_slash, *num_str; 151 | + const char *hex; 152 | + int num = 0; 153 | + 154 | + /* We don't bother to hard-link 0-length files. */ 155 | + if (F_LENGTH(file) == 0) 156 | + return; 157 | + 158 | + hex = sum_as_hex(5, protocol_version >= 30 ? sender_file_sum : link_by_hash_extra_sum, 0); 159 | + if (asprintf(&hashname, "%s/%.3s/%.3s/%.3s/%s.%s.000000", 160 | + link_by_hash_dir, hex, hex+3, hex+6, hex+9, big_num(F_LENGTH(file))) < 0) 161 | + { 162 | + out_of_memory("make_hash_name"); 163 | + } 164 | + 165 | + last_slash = strrchr(hashname, '/'); 166 | + num_str = strrchr(last_slash, '.') + 1; 167 | + 168 | + while (1) { 169 | + if (num >= 999999) { /* Surely we'll never reach this... */ 170 | + if (DEBUG_GTE(HASHLINK, 1)) 171 | + rprintf(FINFO, "link-by-hash: giving up after \"%s\".\n", hashname); 172 | + goto cleanup; 173 | + } 174 | + if (num > 0 && DEBUG_GTE(HASHLINK, 1)) 175 | + rprintf(FINFO, "link-by-hash: max link count exceeded, starting new file \"%s\".\n", hashname); 176 | + 177 | + snprintf(num_str, 7, "%d", num++); 178 | + if (do_stat(hashname, &st) < 0) 179 | + break; 180 | + 181 | + if (do_link(hashname, fnametmp) < 0) { 182 | + if (errno == EMLINK) 183 | + continue; 184 | + rsyserr(FERROR, errno, "link \"%s\" -> \"%s\"", hashname, full_fname(fname)); 185 | + } else { 186 | + if (DEBUG_GTE(HASHLINK, 2)) 187 | + rprintf(FINFO, "link-by-hash (existing): \"%s\" -> %s\n", hashname, full_fname(fname)); 188 | + robust_rename(fnametmp, fname, NULL, 0644); 189 | + } 190 | + 191 | + goto cleanup; 192 | + } 193 | + 194 | + if (DEBUG_GTE(HASHLINK, 2)) 195 | + rprintf(FINFO, "link-by-hash (new): %s -> \"%s\"\n", full_fname(fname), hashname); 196 | + 197 | + if (do_link(fname, hashname) < 0 198 | + && (errno != ENOENT || make_path(hashname, MKP_DROP_NAME) < 0 || do_link(fname, hashname) < 0)) 199 | + rsyserr(FERROR, errno, "link \"%s\" -> \"%s\"", full_fname(fname), hashname); 200 | + 201 | + cleanup: 202 | + free(hashname); 203 | +} 204 | +#endif 205 | diff --git a/options.c b/options.c 206 | --- a/options.c 207 | +++ b/options.c 208 | @@ -173,6 +173,7 @@ char *backup_suffix = NULL; 209 | char *tmpdir = NULL; 210 | char *partial_dir = NULL; 211 | char *basis_dir[MAX_BASIS_DIRS+1]; 212 | +char *link_by_hash_dir = NULL; 213 | char *config_file = NULL; 214 | char *shell_cmd = NULL; 215 | char *logfile_name = NULL; 216 | @@ -231,7 +232,7 @@ static const char *debug_verbosity[] = { 217 | /*2*/ "BIND,CMD,CONNECT,DEL,DELTASUM,DUP,FILTER,FLIST,ICONV", 218 | /*3*/ "ACL,BACKUP,CONNECT2,DELTASUM2,DEL2,EXIT,FILTER2,FLIST2,FUZZY,GENR,OWN,RECV,SEND,TIME", 219 | /*4*/ "CMD2,DELTASUM3,DEL3,EXIT2,FLIST3,ICONV2,OWN2,PROTO,TIME2", 220 | - /*5*/ "CHDIR,DELTASUM4,FLIST4,FUZZY2,HASH,HLINK", 221 | + /*5*/ "CHDIR,DELTASUM4,FLIST4,FUZZY2,HASH,HASHLINK,HLINK", 222 | }; 223 | 224 | #define MAX_VERBOSITY ((int)(sizeof debug_verbosity / sizeof debug_verbosity[0]) - 1) 225 | @@ -302,6 +303,7 @@ static struct output_struct debug_words[COUNT_DEBUG+1] = { 226 | DEBUG_WORD(FUZZY, W_REC, "Debug fuzzy scoring (levels 1-2)"), 227 | DEBUG_WORD(GENR, W_REC, "Debug generator functions"), 228 | DEBUG_WORD(HASH, W_SND|W_REC, "Debug hashtable code"), 229 | + DEBUG_WORD(HASHLINK, W_REC, "Debug hashlink code (levels 1-2)"), 230 | DEBUG_WORD(HLINK, W_SND|W_REC, "Debug hard-link actions (levels 1-3)"), 231 | DEBUG_WORD(ICONV, W_CLI|W_SRV, "Debug iconv character conversions (levels 1-2)"), 232 | DEBUG_WORD(IO, W_CLI|W_SRV, "Debug I/O routines (levels 1-4)"), 233 | @@ -582,7 +584,7 @@ enum {OPT_SERVER = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM, 234 | OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW, OPT_MIN_SIZE, OPT_CHMOD, 235 | OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE, 236 | OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG, OPT_BLOCK_SIZE, 237 | - OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT, OPT_STDERR, 238 | + OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT, OPT_STDERR, OPT_LINK_BY_HASH, 239 | OPT_OLD_COMPRESS, OPT_NEW_COMPRESS, OPT_NO_COMPRESS, OPT_OLD_ARGS, 240 | OPT_STOP_AFTER, OPT_STOP_AT, 241 | OPT_REFUSED_BASE = 9000}; 242 | @@ -743,6 +745,7 @@ static struct poptOption long_options[] = { 243 | {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 }, 244 | {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 }, 245 | {"link-dest", 0, POPT_ARG_STRING, 0, OPT_LINK_DEST, 0, 0 }, 246 | + {"link-by-hash", 0, POPT_ARG_STRING, 0, OPT_LINK_BY_HASH, 0, 0}, 247 | {"fuzzy", 'y', POPT_ARG_NONE, 0, 'y', 0, 0 }, 248 | {"no-fuzzy", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 }, 249 | {"no-y", 0, POPT_ARG_VAL, &fuzzy_basis, 0, 0, 0 }, 250 | @@ -990,6 +993,9 @@ static void set_refuse_options(void) 251 | ref = cp + 1; 252 | } 253 | 254 | + if (*lp_link_by_hash_dir(module_id)) 255 | + parse_one_refuse_match(0, "link-by-hash", list_end); 256 | + 257 | if (am_daemon) { 258 | #ifdef ICONV_OPTION 259 | if (!*lp_charset(module_id)) 260 | @@ -1867,6 +1873,20 @@ int parse_arguments(int *argc_p, const char ***argv_p) 261 | goto cleanup; 262 | #endif 263 | 264 | + case OPT_LINK_BY_HASH: 265 | +#ifdef HAVE_LINK 266 | + arg = poptGetOptArg(pc); 267 | + if (sanitize_paths) 268 | + arg = sanitize_path(NULL, arg, NULL, 0, SP_DEFAULT); 269 | + link_by_hash_dir = (char *)arg; 270 | + break; 271 | +#else 272 | + snprintf(err_buf, sizeof err_buf, 273 | + "hard links are not supported on this %s\n", 274 | + am_server ? "server" : "client"); 275 | + return 0; 276 | +#endif 277 | + 278 | case OPT_STOP_AFTER: { 279 | long val; 280 | arg = poptGetOptArg(pc); 281 | @@ -2252,6 +2272,8 @@ int parse_arguments(int *argc_p, const char ***argv_p) 282 | tmpdir = sanitize_path(NULL, tmpdir, NULL, 0, SP_DEFAULT); 283 | if (backup_dir) 284 | backup_dir = sanitize_path(NULL, backup_dir, NULL, 0, SP_DEFAULT); 285 | + if (link_by_hash_dir) 286 | + link_by_hash_dir = sanitize_path(NULL, link_by_hash_dir, NULL, 0, SP_DEFAULT); 287 | } 288 | if (daemon_filter_list.head && !am_sender) { 289 | filter_rule_list *elp = &daemon_filter_list; 290 | @@ -2941,6 +2963,12 @@ void server_options(char **args, int *argc_p) 291 | args[ac++] = "--no-W"; 292 | } 293 | 294 | + if (link_by_hash_dir && am_sender) { 295 | + args[ac++] = "--link-by-hash"; 296 | + args[ac++] = link_by_hash_dir; 297 | + link_by_hash_dir = NULL; /* optimize sending-side checksums */ 298 | + } 299 | + 300 | if (files_from && (!am_sender || filesfrom_host)) { 301 | if (filesfrom_host) { 302 | args[ac++] = "--files-from"; 303 | diff --git a/rsync.1.md b/rsync.1.md 304 | --- a/rsync.1.md 305 | +++ b/rsync.1.md 306 | @@ -510,6 +510,7 @@ has its own detailed description later in this manpage. 307 | --compare-dest=DIR also compare destination files relative to DIR 308 | --copy-dest=DIR ... and include copies of unchanged files 309 | --link-dest=DIR hardlink to files in DIR when unchanged 310 | +--link-by-hash=DIR create hardlinks by hash into DIR 311 | --compress, -z compress file data during the transfer 312 | --compress-choice=STR choose the compression algorithm (aka --zc) 313 | --compress-level=NUM explicitly set compression level (aka --zl) 314 | @@ -2720,6 +2721,50 @@ expand it. 315 | this bug by avoiding the `-o` option (or using `--no-o`) when sending to an 316 | old rsync. 317 | 318 | +0. `--link-by-hash=DIR` 319 | + 320 | + This option hard links the destination files into _DIR_, a link farm 321 | + arranged by MD5 file hash. The result is that the system will only store 322 | + (usually) one copy of the unique contents of each file, regardless of the 323 | + file's name (it will use extra files if the links overflow the available 324 | + maximum). 325 | + 326 | + This patch does not take into account file permissions, extended 327 | + attributes, or ACLs when linking things together, so you should only use 328 | + this if you don't care about preserving those extra file attributes (or if 329 | + they are always the same for identical files). 330 | + 331 | + The _DIR_ is relative to the destination directory, so either specify a full 332 | + path to the hash hierarchy, or specify a relative path that puts the links 333 | + outside the destination (e.g. "../links"). 334 | + 335 | + Keep in mind that the hierarchy is never pruned, so if you need to reclaim 336 | + space, you should remove any files that have just one link (since they are 337 | + not linked into any destination dirs anymore): 338 | + 339 | + > find $DIR -links 1 -delete 340 | + 341 | + The link farm's directory hierarchy is determined by the file's (32-char) 342 | + MD5 hash and the file-length. The hash is split up into directory shards. 343 | + For example, if a file is 54321 bytes long, it could be stored like this: 344 | + 345 | + > $DIR/123/456/789/01234567890123456789012.54321.0 346 | + 347 | + Note that the directory layout in this patch was modified for version 348 | + 3.1.0, so anyone using an older version of this patch should move their 349 | + existing link hierarchy out of the way and then use the newer rsync to copy 350 | + the saved hierarchy into its new layout. Assuming that no files have 351 | + overflowed their link limits, this would work: 352 | + 353 | + > mv $DIR $DIR.old 354 | + > rsync -aiv --link-by-hash=$DIR $DIR.old/ $DIR.tmp/ 355 | + > rm -rf $DIR.tmp 356 | + > rm -rf $DIR.old 357 | + 358 | + If some of your files are at their link limit, you'd be better of using a 359 | + script to calculate the md5 sum of each file in the hierarchy and move it 360 | + to its new location. 361 | + 362 | 0. `--compress`, `-z` 363 | 364 | With this option, rsync compresses the file data as it is sent to the 365 | diff --git a/rsync.c b/rsync.c 366 | --- a/rsync.c 367 | +++ b/rsync.c 368 | @@ -52,6 +52,7 @@ extern int flist_eof; 369 | extern int file_old_total; 370 | extern int keep_dirlinks; 371 | extern int make_backups; 372 | +extern char *link_by_hash_dir; 373 | extern int sanitize_paths; 374 | extern struct file_list *cur_flist, *first_flist, *dir_flist; 375 | extern struct chmod_mode_struct *daemon_chmod_modes; 376 | @@ -760,6 +761,10 @@ int finish_transfer(const char *fname, const char *fnametmp, 377 | } 378 | if (ret == 0) { 379 | /* The file was moved into place (not copied), so it's done. */ 380 | +#ifdef HAVE_LINK 381 | + if (link_by_hash_dir) 382 | + link_by_hash(fname, fnametmp, file); 383 | +#endif 384 | return 1; 385 | } 386 | /* The file was copied, so tweak the perms of the copied file. If it 387 | diff --git a/rsync.h b/rsync.h 388 | --- a/rsync.h 389 | +++ b/rsync.h 390 | @@ -1446,7 +1446,8 @@ extern short info_levels[], debug_levels[]; 391 | #define DEBUG_FUZZY (DEBUG_FLIST+1) 392 | #define DEBUG_GENR (DEBUG_FUZZY+1) 393 | #define DEBUG_HASH (DEBUG_GENR+1) 394 | -#define DEBUG_HLINK (DEBUG_HASH+1) 395 | +#define DEBUG_HASHLINK (DEBUG_HASH+1) 396 | +#define DEBUG_HLINK (DEBUG_HASHLINK+1) 397 | #define DEBUG_ICONV (DEBUG_HLINK+1) 398 | #define DEBUG_IO (DEBUG_ICONV+1) 399 | #define DEBUG_NSTR (DEBUG_IO+1) 400 | diff --git a/rsyncd.conf.5.md b/rsyncd.conf.5.md 401 | --- a/rsyncd.conf.5.md 402 | +++ b/rsyncd.conf.5.md 403 | @@ -388,6 +388,23 @@ in the values of parameters. See that section for details. 404 | is 0, which means no limit. A negative value disables the module. See 405 | also the "[lock file](#)" parameter. 406 | 407 | +0. `link by hash dir` 408 | + 409 | + When the "link by hash dir" parameter is set to a non-empty string, 410 | + received files will be hard linked into **DIR**, a link farm arranged by 411 | + MD5 file hash. See the `--link-by-hash` option for a full explanation. 412 | + 413 | + The **DIR** must be accessible inside any chroot restrictions for the 414 | + module, but can exist outside the transfer location if there is an 415 | + inside-the-chroot path to the module (see "use chroot"). Note that a 416 | + user-specified option does not allow this outside-the-transfer-area 417 | + placement. 418 | + 419 | + If this parameter is set, it will disable the `--link-by-hash` command-line 420 | + option for copies into the module. 421 | + 422 | +The default is for this parameter to be unset. 423 | + 424 | 0. `log file` 425 | 426 | When the "log file" parameter is set to a non-empty string, the rsync 427 | -------------------------------------------------------------------------------- /omit-dir-changes.diff: -------------------------------------------------------------------------------- 1 | This patch from Antti Tapaninen added the --omit-dir-changes option, which 2 | tells rsync to not affect any attributes on the directories in the transfer. 3 | 4 | To use this patch, run these commands for a successful build: 5 | 6 | patch -p1 mode) ? !omit_dir_times 24 | : S_ISLNK(file->mode) ? !omit_link_times 25 | : 1; 26 | + int omit_uid_gid = omit_dir_changes && S_ISDIR(sxp->st.st_mode); 27 | 28 | if (S_ISREG(file->mode) && F_LENGTH(file) != sxp->st.st_size) 29 | iflags |= ITEM_REPORT_SIZE; 30 | @@ -543,9 +545,9 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre 31 | } else if (preserve_executability 32 | && ((sxp->st.st_mode & 0111 ? 1 : 0) ^ (file->mode & 0111 ? 1 : 0))) 33 | iflags |= ITEM_REPORT_PERMS; 34 | - if (uid_ndx && am_root && (uid_t)F_OWNER(file) != sxp->st.st_uid) 35 | + if (uid_ndx && !omit_uid_gid && am_root && (uid_t)F_OWNER(file) != sxp->st.st_uid) 36 | iflags |= ITEM_REPORT_OWNER; 37 | - if (gid_ndx && !(file->flags & FLAG_SKIP_GROUP) && sxp->st.st_gid != (gid_t)F_GROUP(file)) 38 | + if (gid_ndx && !omit_uid_gid && !(file->flags & FLAG_SKIP_GROUP) && sxp->st.st_gid != (gid_t)F_GROUP(file)) 39 | iflags |= ITEM_REPORT_GROUP; 40 | #ifdef SUPPORT_ACLS 41 | if (preserve_acls && !S_ISLNK(file->mode)) { 42 | @@ -1451,7 +1453,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, 43 | real_ret = statret; 44 | if (file->flags & FLAG_DIR_CREATED) 45 | statret = -1; 46 | - if (!preserve_perms) { /* See comment in non-dir code below. */ 47 | + if (!preserve_perms || omit_dir_changes) { /* See comment in non-dir code below. */ 48 | file->mode = dest_mode(file->mode, sx.st.st_mode, dflt_perms, statret == 0); 49 | } 50 | if (statret != 0 && basis_dir[0] != NULL) { 51 | diff --git a/options.c b/options.c 52 | --- a/options.c 53 | +++ b/options.c 54 | @@ -64,6 +64,7 @@ int preserve_gid = 0; 55 | int preserve_mtimes = 0; 56 | int preserve_atimes = 0; 57 | int preserve_crtimes = 0; 58 | +int omit_dir_changes = 0; 59 | int omit_dir_times = 0; 60 | int omit_link_times = 0; 61 | int trust_sender = 0; 62 | @@ -647,6 +648,7 @@ static struct poptOption long_options[] = { 63 | {"omit-link-times", 'J', POPT_ARG_VAL, &omit_link_times, 1, 0, 0 }, 64 | {"no-omit-link-times",0, POPT_ARG_VAL, &omit_link_times, 0, 0, 0 }, 65 | {"no-J", 0, POPT_ARG_VAL, &omit_link_times, 0, 0, 0 }, 66 | + {"omit-dir-changes", 0, POPT_ARG_NONE, &omit_dir_changes, 0, 0, 0 }, 67 | {"modify-window", '@', POPT_ARG_INT, &modify_window, OPT_MODIFY_WINDOW, 0, 0 }, 68 | {"super", 0, POPT_ARG_VAL, &am_root, 2, 0, 0 }, 69 | {"no-super", 0, POPT_ARG_VAL, &am_root, 0, 0, 0 }, 70 | @@ -2321,7 +2323,7 @@ int parse_arguments(int *argc_p, const char ***argv_p) 71 | parse_filter_str(&filter_list, backup_dir_buf, rule_template(0), 0); 72 | } 73 | 74 | - if (make_backups && !backup_dir) 75 | + if (omit_dir_changes || (make_backups && !backup_dir)) 76 | omit_dir_times = -1; /* Implied, so avoid -O to sender. */ 77 | 78 | if (stdout_format) { 79 | @@ -2837,6 +2839,8 @@ void server_options(char **args, int *argc_p) 80 | args[ac++] = "--size-only"; 81 | if (do_stats) 82 | args[ac++] = "--stats"; 83 | + if (omit_dir_changes) 84 | + args[ac++] = "--omit-dir-changes"; 85 | } else { 86 | if (skip_compress) 87 | args[ac++] = safe_arg("--skip-compress", skip_compress); 88 | diff --git a/rsync.1.md b/rsync.1.md 89 | --- a/rsync.1.md 90 | +++ b/rsync.1.md 91 | @@ -463,6 +463,7 @@ has its own detailed description later in this manpage. 92 | --crtimes, -N preserve create times (newness) 93 | --omit-dir-times, -O omit directories from --times 94 | --omit-link-times, -J omit symlinks from --times 95 | +--omit-dir-changes omit directories from any attribute changes 96 | --super receiver attempts super-user activities 97 | --fake-super store/recover privileged attrs using xattrs 98 | --sparse, -S turn sequences of nulls into sparse blocks 99 | @@ -1656,6 +1657,11 @@ expand it. 100 | This tells rsync to omit symlinks when it is preserving modification, 101 | access, and create times. 102 | 103 | +0. `--omit-dir-changes` 104 | + 105 | + This tells rsync to omit directories when applying any preserved attributes 106 | + (owner, group, times, permissions) to already existing directories. 107 | + 108 | 0. `--super` 109 | 110 | This tells the receiving side to attempt super-user activities even if the 111 | diff --git a/rsync.c b/rsync.c 112 | --- a/rsync.c 113 | +++ b/rsync.c 114 | @@ -35,6 +35,7 @@ extern int preserve_executability; 115 | extern int preserve_mtimes; 116 | extern int omit_dir_times; 117 | extern int omit_link_times; 118 | +extern int omit_dir_changes; 119 | extern int am_root; 120 | extern int am_server; 121 | extern int am_daemon; 122 | @@ -490,6 +491,7 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp, 123 | stat_x sx2; 124 | int change_uid, change_gid; 125 | mode_t new_mode = file->mode; 126 | + int omit_uid_gid; 127 | int inherit; 128 | 129 | if (!sxp) { 130 | @@ -520,9 +522,10 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp, 131 | get_acl(fname, sxp); 132 | #endif 133 | 134 | - change_uid = am_root && uid_ndx && sxp->st.st_uid != (uid_t)F_OWNER(file); 135 | - change_gid = gid_ndx && !(file->flags & FLAG_SKIP_GROUP) 136 | - && sxp->st.st_gid != (gid_t)F_GROUP(file); 137 | + omit_uid_gid = omit_dir_changes && S_ISDIR(sxp->st.st_mode); 138 | + change_uid = am_root && !omit_uid_gid && uid_ndx && sxp->st.st_uid != (uid_t)F_OWNER(file); 139 | + change_gid = gid_ndx && !omit_uid_gid && !(file->flags & FLAG_SKIP_GROUP) 140 | + && sxp->st.st_gid != (gid_t)F_GROUP(file); 141 | #ifndef CAN_CHOWN_SYMLINK 142 | if (S_ISLNK(sxp->st.st_mode)) { 143 | ; 144 | -------------------------------------------------------------------------------- /slow-down.diff: -------------------------------------------------------------------------------- 1 | This patch adds a --slow-down=USECs option that causes the sender to scan 2 | the filelist more slowly, and the generator to scan for deletions more 3 | slowly. It doesn't do anything to make anyone slow down during the normal 4 | transfer processing, though. 5 | 6 | The idea is to lessen rsync's impact on disk I/O. Unfortunately, there 7 | should really be a way to affect more of rsync's processing, perhaps by 8 | specifying a maximum disk I/O rate (and have that affect a maximum stat() 9 | rate or something like that). 10 | 11 | To use this patch, run these commands for a successful build: 12 | 13 | patch -p1 rsync somehost.mydomain.com:: 140 | 141 | +And, if Service Location Protocol is available, the following will list the 142 | +available rsync servers: 143 | + 144 | +> rsync rsync:// 145 | + 146 | +See the following section for even more usage details. 147 | + 148 | +One more thing, if Service Location Protocol is available, the following will 149 | +list the available rsync servers: 150 | + 151 | +> rsync rsync:// 152 | + 153 | ## COPYING TO A DIFFERENT NAME 154 | 155 | When you want to copy a directory to a different name, use a trailing slash on 156 | diff --git a/rsync.h b/rsync.h 157 | --- a/rsync.h 158 | +++ b/rsync.h 159 | @@ -234,6 +234,10 @@ 160 | #define SIGNIFICANT_ITEM_FLAGS (~(\ 161 | ITEM_BASIS_TYPE_FOLLOWS | ITEM_XNAME_FOLLOWS | ITEM_LOCAL_CHANGE)) 162 | 163 | +/* this is the minimum we'll use, irrespective of config setting */ 164 | +/* definitely don't set to less than about 30 seconds */ 165 | +#define SLP_MIN_TIMEOUT 120 166 | + 167 | #define CFN_KEEP_DOT_DIRS (1<<0) 168 | #define CFN_KEEP_TRAILING_SLASH (1<<1) 169 | #define CFN_DROP_TRAILING_DOT_DIR (1<<2) 170 | diff --git a/rsyncd.conf b/rsyncd.conf 171 | new file mode 100644 172 | --- /dev/null 173 | +++ b/rsyncd.conf 174 | @@ -0,0 +1 @@ 175 | +slp refresh = 300 176 | diff --git a/rsyncd.conf.5.md b/rsyncd.conf.5.md 177 | --- a/rsyncd.conf.5.md 178 | +++ b/rsyncd.conf.5.md 179 | @@ -120,6 +120,21 @@ parameters: 180 | You can override the default backlog value when the daemon listens for 181 | connections. It defaults to 5. 182 | 183 | +0. `use slp` 184 | + 185 | + You can enable Service Location Protocol support by enabling this global 186 | + parameter. The default is "false". 187 | + 188 | +0. `slp refresh` 189 | + 190 | + This parameter is used to determine how long service advertisements are 191 | + valid (measured in seconds), and is only applicable if you have Service 192 | + Location Protocol support compiled in. If this is not set or is set to 193 | + zero, then service advertisements never time out. If this is set to less 194 | + than 120 seconds, then 120 seconds is used. If it is set to more than 195 | + 65535, then 65535 is used (which is a limitation of SLP). Using 3600 196 | + (one hour) is a good number if you tend to change your configuration. 197 | + 198 | You may also include any [MODULE PARAMETERS](#) in the global part of the 199 | config file, in which case the supplied value will override the default for 200 | that parameter. 201 | @@ -1208,6 +1223,7 @@ A more sophisticated example would be: 202 | > max connections = 4 203 | > syslog facility = local5 204 | > pid file = /var/run/rsyncd.pid 205 | +> slp refresh = 3600 206 | > 207 | > [ftp] 208 | > path = /var/ftp/./pub 209 | diff --git a/socket.c b/socket.c 210 | --- a/socket.c 211 | +++ b/socket.c 212 | @@ -534,6 +534,16 @@ void start_accept_loop(int port, int (*fn)(int, int)) 213 | { 214 | fd_set deffds; 215 | int *sp, maxfd, i; 216 | +#ifdef HAVE_LIBSLP 217 | + time_t next_slp_refresh; 218 | + short slp_timeout = lp_use_slp() ? lp_slp_refresh() : 0; 219 | + if (slp_timeout) { 220 | + if (slp_timeout < SLP_MIN_TIMEOUT) 221 | + slp_timeout = SLP_MIN_TIMEOUT; 222 | + /* re-register before slp times out */ 223 | + slp_timeout -= 15; 224 | + } 225 | +#endif 226 | 227 | #ifdef HAVE_SIGACTION 228 | sigact.sa_flags = SA_NOCLDSTOP; 229 | @@ -561,14 +571,25 @@ void start_accept_loop(int port, int (*fn)(int, int)) 230 | maxfd = sp[i]; 231 | } 232 | 233 | +#ifdef HAVE_LIBSLP 234 | + next_slp_refresh = time(NULL) + slp_timeout; 235 | +#endif 236 | + 237 | /* now accept incoming connections - forking a new process 238 | * for each incoming connection */ 239 | while (1) { 240 | fd_set fds; 241 | pid_t pid; 242 | int fd; 243 | + int sel_ret; 244 | struct sockaddr_storage addr; 245 | socklen_t addrlen = sizeof addr; 246 | +#ifdef HAVE_LIBSLP 247 | + struct timeval slp_tv; 248 | + 249 | + slp_tv.tv_sec = 10; 250 | + slp_tv.tv_usec = 0; 251 | +#endif 252 | 253 | /* close log file before the potentially very long select so 254 | * file can be trimmed by another process instead of growing 255 | @@ -581,7 +602,18 @@ void start_accept_loop(int port, int (*fn)(int, int)) 256 | fds = deffds; 257 | #endif 258 | 259 | - if (select(maxfd + 1, &fds, NULL, NULL, NULL) < 1) 260 | +#ifdef HAVE_LIBSLP 261 | + sel_ret = select(maxfd + 1, &fds, NULL, NULL, 262 | + slp_timeout ? &slp_tv : NULL); 263 | + if (sel_ret == 0 && slp_timeout && time(NULL) > next_slp_refresh) { 264 | + rprintf(FINFO, "Service registration expired, refreshing it\n"); 265 | + register_services(); 266 | + next_slp_refresh = time(NULL) + slp_timeout; 267 | + } 268 | +#else 269 | + sel_ret = select(maxfd + 1, &fds, NULL, NULL, NULL); 270 | +#endif 271 | + if (sel_ret < 1) 272 | continue; 273 | 274 | for (i = 0, fd = -1; sp[i] >= 0; i++) { 275 | diff --git a/srvloc.c b/srvloc.c 276 | new file mode 100644 277 | --- /dev/null 278 | +++ b/srvloc.c 279 | @@ -0,0 +1,103 @@ 280 | +/* -*- c-file-style: "linux"; -*- 281 | + 282 | + Copyright (C) 2002 by Brad Hards 283 | + 284 | + This program is free software; you can redistribute it and/or modify 285 | + it under the terms of the GNU General Public License as published by 286 | + the Free Software Foundation; either version 2 of the License, or 287 | + (at your option) any later version. 288 | + 289 | + This program is distributed in the hope that it will be useful, 290 | + but WITHOUT ANY WARRANTY; without even the implied warranty of 291 | + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 292 | + GNU General Public License for more details. 293 | + 294 | + You should have received a copy of the GNU General Public License 295 | + along with this program; if not, write to the Free Software 296 | + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 297 | +*/ 298 | + 299 | +/* This file implements the service location functionality */ 300 | +/* Basically, it uses normal Service Location Protocol API */ 301 | + 302 | +/* It is really a cheap hack - just to show how it might work 303 | + in a real application. 304 | +*/ 305 | + 306 | +#include "rsync.h" 307 | + 308 | +#include 309 | +#include 310 | +#include 311 | + 312 | +/* This one just prints out the attributes */ 313 | +static SLPBoolean getAttrCallback(UNUSED(SLPHandle hslp), const char *attrlist, 314 | + SLPError errcode, UNUSED(void *cookie)) 315 | +{ 316 | + char *cleanstr; 317 | + 318 | + if (errcode == SLP_OK) { 319 | + if (!strcmp(attrlist, "(comment=)")) 320 | + rprintf(FINFO, "\t(No description)\n"); 321 | + else { 322 | + cleanstr = strrchr(attrlist, ')') ; 323 | + *cleanstr = ' '; /* remove last ')' */ 324 | + rprintf(FINFO, "\t%s\n", strchr(attrlist, '=') + 1); 325 | + } 326 | + } 327 | + return SLP_FALSE; 328 | +} 329 | + 330 | +static SLPBoolean getSLPSrvURLCallback(UNUSED(SLPHandle hslp), 331 | + const char *srvurl, UNUSED(unsigned short lifetime), 332 | + SLPError errcode, void *cookie) 333 | +{ 334 | + SLPError result; 335 | + SLPHandle attrhslp; 336 | + 337 | + if (errcode == SLP_OK) { 338 | + /* chop service: off the front */ 339 | + rprintf(FINFO, " %s ", (strchr(srvurl, ':') + 1)); 340 | + /* check for any attributes */ 341 | + if (SLPOpen("en", SLP_FALSE,&attrhslp) == SLP_OK) { 342 | + result = SLPFindAttrs(attrhslp, srvurl, 343 | + "", /* return all attributes */ 344 | + "", /* use configured scopes */ 345 | + getAttrCallback, NULL); 346 | + if (result != SLP_OK) { 347 | + rprintf(FERROR, "errorcode: %i\n",result); 348 | + } 349 | + SLPClose(attrhslp); 350 | + } 351 | + *(SLPError*)cookie = SLP_OK; 352 | + } else 353 | + *(SLPError*)cookie = errcode; 354 | + 355 | + /* Return SLP_TRUE because we want to be called again 356 | + * if more services were found. */ 357 | + 358 | + return SLP_TRUE; 359 | +} 360 | + 361 | +int print_service_list(void) 362 | +{ 363 | + SLPError err; 364 | + SLPError callbackerr; 365 | + SLPHandle hslp; 366 | + 367 | + err = SLPOpen("en",SLP_FALSE,&hslp); 368 | + if (err != SLP_OK) { 369 | + rprintf(FERROR, "Error opening slp handle %i\n", err); 370 | + return err; 371 | + } 372 | + 373 | + SLPFindSrvs(hslp, "rsync", 374 | + 0, /* use configured scopes */ 375 | + 0, /* no attr filter */ 376 | + getSLPSrvURLCallback, &callbackerr); 377 | + 378 | + /* Now that we're done using slp, close the slp handle */ 379 | + SLPClose(hslp); 380 | + 381 | + return 0; 382 | +} 383 | diff --git a/srvreg.c b/srvreg.c 384 | new file mode 100644 385 | --- /dev/null 386 | +++ b/srvreg.c 387 | @@ -0,0 +1,128 @@ 388 | +/* -*- c-file-style: "linux"; -*- 389 | + 390 | + Copyright (C) 2002 by Brad Hards 391 | + 392 | + This program is free software; you can redistribute it and/or modify 393 | + it under the terms of the GNU General Public License as published by 394 | + the Free Software Foundation; either version 2 of the License, or 395 | + (at your option) any later version. 396 | + 397 | + This program is distributed in the hope that it will be useful, 398 | + but WITHOUT ANY WARRANTY; without even the implied warranty of 399 | + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 400 | + GNU General Public License for more details. 401 | + 402 | + You should have received a copy of the GNU General Public License 403 | + along with this program; if not, write to the Free Software 404 | + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 405 | +*/ 406 | + 407 | +/* This file implements the service registration functionality */ 408 | + 409 | +/* Basically, it uses normal Service Location Protocol API */ 410 | + 411 | +#include "rsync.h" 412 | +#include "slp.h" 413 | +#include "netdb.h" 414 | + 415 | +extern int rsync_port; 416 | + 417 | +static void slp_callback(UNUSED(SLPHandle hslp), SLPError errcode, void *cookie) 418 | +{ 419 | + /* return the error code in the cookie */ 420 | + *(SLPError*)cookie = errcode; 421 | + 422 | + /* You could do something else here like print out 423 | + * the errcode, etc. Remember, as a general rule, 424 | + * do not try to do too much in a callback because 425 | + * it is being executed by the same thread that is 426 | + * reading slp packets from the wire. */ 427 | +} 428 | + 429 | +int register_services(void) 430 | +{ 431 | + SLPError err, callbackerr; 432 | + SLPHandle hslp; 433 | + int n; 434 | + int i; 435 | + char srv[120]; 436 | + char attr[120]; 437 | + char localhost[256]; 438 | + extern char *config_file; 439 | + short timeout; 440 | + struct addrinfo aih, *ai = 0; 441 | + 442 | + if (!lp_load(config_file, 0)) { 443 | + exit_cleanup(RERR_SYNTAX); 444 | + } 445 | + 446 | + n = lp_num_modules(); 447 | + 448 | + if (0 == lp_slp_refresh()) 449 | + timeout = SLP_LIFETIME_MAXIMUM; /* don't expire, ever */ 450 | + else if (SLP_MIN_TIMEOUT > lp_slp_refresh()) 451 | + timeout = SLP_MIN_TIMEOUT; /* use a reasonable minimum */ 452 | + else if (SLP_LIFETIME_MAXIMUM <= lp_slp_refresh()) 453 | + timeout = (SLP_LIFETIME_MAXIMUM - 1); /* as long as possible */ 454 | + else 455 | + timeout = lp_slp_refresh(); 456 | + 457 | + rprintf(FINFO, "rsyncd registering %d service%s with slpd for %d seconds:\n", n, ((n==1)? "":"s"), timeout); 458 | + err = SLPOpen("en",SLP_FALSE,&hslp); 459 | + if (err != SLP_OK) { 460 | + rprintf(FINFO, "Error opening slp handle %i\n",err); 461 | + return err; 462 | + } 463 | + if (gethostname(localhost, sizeof localhost)) { 464 | + rprintf(FINFO, "Could not get hostname: %s\n", strerror(errno)); 465 | + return err; 466 | + } 467 | + memset(&aih, 0, sizeof aih); 468 | + aih.ai_family = PF_UNSPEC; 469 | + aih.ai_flags = AI_CANONNAME; 470 | + if (0 != (err = getaddrinfo(localhost, 0, &aih, &ai)) || !ai) { 471 | + rprintf(FINFO, "Could not resolve hostname: %s\n", gai_strerror(err)); 472 | + return err; 473 | + } 474 | + /* Register each service with SLP */ 475 | + for (i = 0; i < n; i++) { 476 | + if (!lp_list(i)) 477 | + continue; 478 | + 479 | + snprintf(srv, sizeof srv, "service:rsync://%s:%d/%s", 480 | + ai->ai_canonname, 481 | + rsync_port, 482 | + lp_name(i)); 483 | + rprintf(FINFO, " %s\n", srv); 484 | + if (lp_comment(i)) { 485 | + snprintf(attr, sizeof attr, "(comment=%s)", 486 | + lp_comment(i)); 487 | + } 488 | + err = SLPReg(hslp, 489 | + srv, /* service to register */ 490 | + timeout, 491 | + 0, /* this is ignored */ 492 | + attr, /* attributes */ 493 | + SLP_TRUE, /* new registration - don't change this */ 494 | + slp_callback, /* callback */ 495 | + &callbackerr); 496 | + 497 | + /* err may contain an error code that occurred as the slp library 498 | + * _prepared_ to make the call. */ 499 | + if (err != SLP_OK || callbackerr != SLP_OK) 500 | + rprintf(FINFO, "Error registering service with slp %i\n", err); 501 | + 502 | + /* callbackerr may contain an error code (that was assigned through 503 | + * the callback cookie) that occurred as slp packets were sent on 504 | + * the wire. */ 505 | + if (callbackerr != SLP_OK) 506 | + rprintf(FINFO, "Error registering service with slp %i\n",callbackerr); 507 | + } 508 | + 509 | + /* Now that we're done using slp, close the slp handle */ 510 | + freeaddrinfo(ai); 511 | + SLPClose(hslp); 512 | + 513 | + /* refresh is done in main select loop */ 514 | + return 0; 515 | +} 516 | diff --git a/usage.c b/usage.c 517 | --- a/usage.c 518 | +++ b/usage.c 519 | @@ -138,6 +138,11 @@ static void print_info_flags(enum logcode f) 520 | #endif 521 | "crtimes", 522 | 523 | +#ifndef HAVE_LIBSLP 524 | + "no " 525 | +#endif 526 | + "SLP", 527 | + 528 | "*Optimizations", 529 | 530 | #ifndef USE_ROLL_SIMD 531 | -------------------------------------------------------------------------------- /soften-links.diff: -------------------------------------------------------------------------------- 1 | Marco d'Itri wrote: 2 | 3 | I run one of the debian mirrors, and I had to write this patch because 4 | my archive is split between more than one disk. Would you accept a more 5 | polished version of this patch for inclusion in rsync? 6 | 7 | To use this patch, run these commands for a successful build: 8 | 9 | patch -p1 11 | 12 | based-on: 6c8ca91c731b7bf2b081694bda85b7dadc2b7aff 13 | diff --git a/options.c b/options.c 14 | --- a/options.c 15 | +++ b/options.c 16 | @@ -34,6 +34,7 @@ extern filter_rule_list filter_list; 17 | extern filter_rule_list daemon_filter_list; 18 | 19 | int make_backups = 0; 20 | +int make_source_backups = 0; 21 | 22 | /** 23 | * If 1, send the whole file as literal data rather than trying to 24 | @@ -777,6 +778,7 @@ static struct poptOption long_options[] = { 25 | {"bwlimit", 0, POPT_ARG_STRING, &bwlimit_arg, OPT_BWLIMIT, 0, 0 }, 26 | {"no-bwlimit", 0, POPT_ARG_VAL, &bwlimit, 0, 0, 0 }, 27 | {"backup", 'b', POPT_ARG_VAL, &make_backups, 1, 0, 0 }, 28 | + {"source-backup", 0, POPT_ARG_NONE, &make_source_backups, 0, 0, 0}, 29 | {"no-backup", 0, POPT_ARG_VAL, &make_backups, 0, 0, 0 }, 30 | {"backup-dir", 0, POPT_ARG_STRING, &backup_dir, 0, 0, 0 }, 31 | {"suffix", 0, POPT_ARG_STRING, &backup_suffix, 0, 0, 0 }, 32 | @@ -2840,6 +2842,8 @@ void server_options(char **args, int *argc_p) 33 | } else { 34 | if (skip_compress) 35 | args[ac++] = safe_arg("--skip-compress", skip_compress); 36 | + if (make_source_backups) 37 | + args[ac++] = "--source-backup"; 38 | } 39 | 40 | if (max_alloc_arg && max_alloc != DEFAULT_MAX_ALLOC) 41 | diff --git a/rsync.1.md b/rsync.1.md 42 | --- a/rsync.1.md 43 | +++ b/rsync.1.md 44 | @@ -477,6 +477,7 @@ has its own detailed description later in this manpage. 45 | --existing skip creating new files on receiver 46 | --ignore-existing skip updating files that exist on receiver 47 | --remove-source-files sender removes synchronized files (non-dir) 48 | +--source-backup ... and backs up those files 49 | --del an alias for --delete-during 50 | --delete delete extraneous files from dest dirs 51 | --delete-before receiver deletes before xfer, not during 52 | @@ -1893,6 +1894,17 @@ expand it. 53 | not remove a file the receiver just verified, such as when the user 54 | accidentally makes the source and destination directory the same path. 55 | 56 | +0. `--source-backup` 57 | + 58 | + Makes the sender back up the source files it removes due to 59 | + [`--remove-source-files`](#opt). This option is independent of 60 | + [`--backup`](#opt) but uses the same [`--backup-dir`](#opt) and 61 | + [`--suffix`](#opt) settings, if any. With [`--backup-dir`](#opt), rsync 62 | + looks for each file's backup dir relative to the source argument the file 63 | + came from. Consequently, if the [`--backup-dir`](#opt) path is relative, 64 | + each source argument gets a separate backup dir at that path relative to 65 | + the argument. 66 | + 67 | 0. `--delete` 68 | 69 | This tells rsync to delete extraneous files from the receiving side (ones 70 | diff --git a/sender.c b/sender.c 71 | --- a/sender.c 72 | +++ b/sender.c 73 | @@ -44,6 +44,7 @@ extern int protocol_version; 74 | extern int remove_source_files; 75 | extern int updating_basis_file; 76 | extern int make_backups; 77 | +extern int make_source_backups; 78 | extern int inplace; 79 | extern int inplace_partial; 80 | extern int batch_fd; 81 | @@ -131,6 +132,7 @@ void successful_send(int ndx) 82 | struct file_struct *file; 83 | struct file_list *flist; 84 | STRUCT_STAT st; 85 | + int result; 86 | 87 | if (!remove_source_files) 88 | return; 89 | @@ -162,7 +164,11 @@ void successful_send(int ndx) 90 | return; 91 | } 92 | 93 | - if (do_unlink(fname) < 0) { 94 | + if (make_source_backups) 95 | + result = !make_backup(fname, True); 96 | + else 97 | + result = do_unlink(fname); 98 | + if (result < 0) { 99 | failed_op = "remove"; 100 | failed: 101 | if (errno == ENOENT) 102 | -------------------------------------------------------------------------------- /source-filter_dest-filter.diff: -------------------------------------------------------------------------------- 1 | CAUTION: This patch compiles, but is otherwise totally untested! 2 | 3 | This patch also implements --times-only. 4 | 5 | Implementation details for the --source-filter and -dest-filter options: 6 | 7 | - These options open a *HUGE* security hole in daemon mode unless they 8 | are refused in your rsyncd.conf! 9 | 10 | - Filtering disables rsync alogrithm. (This should be fixed.) 11 | 12 | - Source filter makes temporary files in /tmp. (Should be overridable.) 13 | 14 | - If source filter fails, data is send unfiltered. (Should be changed 15 | to abort.) 16 | 17 | - Failure of destination filter, causes data loss!!! (Should be changed 18 | to abort.) 19 | 20 | - If filter changes size of file, you should use --times-only option to 21 | prevent repeated transfers of unchanged files. 22 | 23 | - If the COMMAND contains single quotes, option-passing breaks. (Needs 24 | to be fixed.) 25 | 26 | To use this patch, run these commands for a successful build: 27 | 28 | patch -p1 st_size != F_LENGTH(file)) 50 | + if (!times_only && st->st_size != F_LENGTH(file)) 51 | return 0; 52 | 53 | /* If always_checksum is set then we use the checksum instead 54 | diff --git a/main.c b/main.c 55 | --- a/main.c 56 | +++ b/main.c 57 | @@ -191,7 +191,7 @@ int shell_exec(const char *cmd) 58 | } 59 | 60 | /* Wait for a process to exit, calling io_flush while waiting. */ 61 | -static void wait_process_with_flush(pid_t pid, int *exit_code_ptr) 62 | +void wait_process_with_flush(pid_t pid, int *exit_code_ptr) 63 | { 64 | pid_t waited_pid; 65 | int status; 66 | diff --git a/options.c b/options.c 67 | --- a/options.c 68 | +++ b/options.c 69 | @@ -119,6 +119,7 @@ int safe_symlinks = 0; 70 | int copy_unsafe_links = 0; 71 | int munge_symlinks = 0; 72 | int size_only = 0; 73 | +int times_only = 0; 74 | int daemon_bwlimit = 0; 75 | int bwlimit = 0; 76 | int fuzzy_basis = 0; 77 | @@ -179,6 +180,8 @@ char *logfile_name = NULL; 78 | char *logfile_format = NULL; 79 | char *stdout_format = NULL; 80 | char *password_file = NULL; 81 | +char *source_filter = NULL; 82 | +char *dest_filter = NULL; 83 | char *early_input_file = NULL; 84 | char *rsync_path = RSYNC_PATH; 85 | char *backup_dir = NULL; 86 | @@ -689,6 +692,7 @@ static struct poptOption long_options[] = { 87 | {"chmod", 0, POPT_ARG_STRING, 0, OPT_CHMOD, 0, 0 }, 88 | {"ignore-times", 'I', POPT_ARG_NONE, &ignore_times, 0, 0, 0 }, 89 | {"size-only", 0, POPT_ARG_NONE, &size_only, 0, 0, 0 }, 90 | + {"times-only", 0, POPT_ARG_NONE, ×_only , 0, 0, 0 }, 91 | {"one-file-system", 'x', POPT_ARG_NONE, 0, 'x', 0, 0 }, 92 | {"no-one-file-system",0, POPT_ARG_VAL, &one_file_system, 0, 0, 0 }, 93 | {"no-x", 0, POPT_ARG_VAL, &one_file_system, 0, 0, 0 }, 94 | @@ -829,6 +833,8 @@ static struct poptOption long_options[] = { 95 | {"early-input", 0, POPT_ARG_STRING, &early_input_file, 0, 0, 0 }, 96 | {"blocking-io", 0, POPT_ARG_VAL, &blocking_io, 1, 0, 0 }, 97 | {"no-blocking-io", 0, POPT_ARG_VAL, &blocking_io, 0, 0, 0 }, 98 | + {"source-filter", 0, POPT_ARG_STRING, &source_filter, 0, 0, 0 }, 99 | + {"dest-filter", 0, POPT_ARG_STRING, &dest_filter, 0, 0, 0 }, 100 | {"outbuf", 0, POPT_ARG_STRING, &outbuf_mode, 0, 0, 0 }, 101 | {"remote-option", 'M', POPT_ARG_STRING, 0, 'M', 0, 0 }, 102 | {"protocol", 0, POPT_ARG_INT, &protocol_version, 0, 0, 0 }, 103 | @@ -2444,6 +2450,16 @@ int parse_arguments(int *argc_p, const char ***argv_p) 104 | } 105 | } 106 | 107 | + if (source_filter || dest_filter) { 108 | + if (whole_file == 0) { 109 | + snprintf(err_buf, sizeof err_buf, 110 | + "--no-whole-file cannot be used with --%s-filter\n", 111 | + source_filter ? "source" : "dest"); 112 | + return 0; 113 | + } 114 | + whole_file = 1; 115 | + } 116 | + 117 | if (files_from) { 118 | char *h, *p; 119 | int q; 120 | @@ -2852,6 +2868,25 @@ void server_options(char **args, int *argc_p) 121 | else if (missing_args == 1 && !am_sender) 122 | args[ac++] = "--ignore-missing-args"; 123 | 124 | + if (times_only && am_sender) 125 | + args[ac++] = "--times-only"; 126 | + 127 | + if (source_filter && !am_sender) { 128 | + /* Need to single quote the arg to keep the remote shell 129 | + * from splitting it. FIXME: breaks if command has single quotes. */ 130 | + if (asprintf(&arg, "--source-filter='%s'", source_filter) < 0) 131 | + goto oom; 132 | + args[ac++] = arg; 133 | + } 134 | + 135 | + if (dest_filter && am_sender) { 136 | + /* Need to single quote the arg to keep the remote shell 137 | + * from splitting it. FIXME: breaks if command has single quotes. */ 138 | + if (asprintf(&arg, "--dest-filter='%s'", dest_filter) < 0) 139 | + goto oom; 140 | + args[ac++] = arg; 141 | + } 142 | + 143 | if (modify_window_set && am_sender) { 144 | char *fmt = modify_window < 0 ? "-@%d" : "--modify-window=%d"; 145 | if (asprintf(&arg, fmt, modify_window) < 0) 146 | diff --git a/pipe.c b/pipe.c 147 | --- a/pipe.c 148 | +++ b/pipe.c 149 | @@ -27,6 +27,7 @@ extern int am_server; 150 | extern int blocking_io; 151 | extern int filesfrom_fd; 152 | extern int munge_symlinks; 153 | +extern mode_t orig_umask; 154 | extern char *logfile_name; 155 | extern int remote_option_cnt; 156 | extern const char **remote_options; 157 | @@ -176,3 +177,77 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out, 158 | 159 | return pid; 160 | } 161 | + 162 | +pid_t run_filter(char *command[], int out, int *pipe_to_filter) 163 | +{ 164 | + pid_t pid; 165 | + int pipefds[2]; 166 | + 167 | + if (DEBUG_GTE(CMD, 1)) 168 | + print_child_argv("opening connection using:", command); 169 | + 170 | + if (pipe(pipefds) < 0) { 171 | + rsyserr(FERROR, errno, "pipe"); 172 | + exit_cleanup(RERR_IPC); 173 | + } 174 | + 175 | + pid = do_fork(); 176 | + if (pid == -1) { 177 | + rsyserr(FERROR, errno, "fork"); 178 | + exit_cleanup(RERR_IPC); 179 | + } 180 | + 181 | + if (pid == 0) { 182 | + if (dup2(pipefds[0], STDIN_FILENO) < 0 183 | + || close(pipefds[1]) < 0 184 | + || dup2(out, STDOUT_FILENO) < 0) { 185 | + rsyserr(FERROR, errno, "Failed dup/close"); 186 | + exit_cleanup(RERR_IPC); 187 | + } 188 | + umask(orig_umask); 189 | + set_blocking(STDIN_FILENO); 190 | + if (blocking_io) 191 | + set_blocking(STDOUT_FILENO); 192 | + execvp(command[0], command); 193 | + rsyserr(FERROR, errno, "Failed to exec %s", command[0]); 194 | + exit_cleanup(RERR_IPC); 195 | + } 196 | + 197 | + if (close(pipefds[0]) < 0) { 198 | + rsyserr(FERROR, errno, "Failed to close"); 199 | + exit_cleanup(RERR_IPC); 200 | + } 201 | + 202 | + *pipe_to_filter = pipefds[1]; 203 | + 204 | + return pid; 205 | +} 206 | + 207 | +pid_t run_filter_on_file(char *command[], int out, int in) 208 | +{ 209 | + pid_t pid; 210 | + 211 | + if (DEBUG_GTE(CMD, 1)) 212 | + print_child_argv("opening connection using:", command); 213 | + 214 | + pid = do_fork(); 215 | + if (pid == -1) { 216 | + rsyserr(FERROR, errno, "fork"); 217 | + exit_cleanup(RERR_IPC); 218 | + } 219 | + 220 | + if (pid == 0) { 221 | + if (dup2(in, STDIN_FILENO) < 0 222 | + || dup2(out, STDOUT_FILENO) < 0) { 223 | + rsyserr(FERROR, errno, "Failed to dup2"); 224 | + exit_cleanup(RERR_IPC); 225 | + } 226 | + if (blocking_io) 227 | + set_blocking(STDOUT_FILENO); 228 | + execvp(command[0], command); 229 | + rsyserr(FERROR, errno, "Failed to exec %s", command[0]); 230 | + exit_cleanup(RERR_IPC); 231 | + } 232 | + 233 | + return pid; 234 | +} 235 | diff --git a/receiver.c b/receiver.c 236 | --- a/receiver.c 237 | +++ b/receiver.c 238 | @@ -60,6 +60,7 @@ extern BOOL want_progress_now; 239 | extern mode_t orig_umask; 240 | extern struct stats stats; 241 | extern char *tmpdir; 242 | +extern char *dest_filter; 243 | extern char *partial_dir; 244 | extern char *basis_dir[MAX_BASIS_DIRS+1]; 245 | extern char sender_file_sum[MAX_DIGEST_LEN]; 246 | @@ -528,6 +529,7 @@ int recv_files(int f_in, int f_out, char *local_name) 247 | char *fnametmp, fnametmpbuf[MAXPATHLEN]; 248 | char *fnamecmp, *partialptr; 249 | char fnamecmpbuf[MAXPATHLEN]; 250 | + char *filter_argv[MAX_FILTER_ARGS + 1]; 251 | uchar fnamecmp_type; 252 | struct file_struct *file; 253 | int itemizing = am_server ? logfile_format_has_i : stdout_format_has_i; 254 | @@ -538,6 +540,7 @@ int recv_files(int f_in, int f_out, char *local_name) 255 | const char *parent_dirname = ""; 256 | #endif 257 | int ndx, recv_ok, one_inplace; 258 | + pid_t pid = 0; 259 | 260 | if (DEBUG_GTE(RECV, 1)) 261 | rprintf(FINFO, "recv_files(%d) starting\n", cur_flist->used); 262 | @@ -548,6 +551,23 @@ int recv_files(int f_in, int f_out, char *local_name) 263 | if (whole_file < 0) 264 | whole_file = 0; 265 | 266 | + if (dest_filter) { 267 | + char *p; 268 | + char *sep = " \t"; 269 | + int i; 270 | + for (p = strtok(dest_filter, sep), i = 0; 271 | + p && i < MAX_FILTER_ARGS; 272 | + p = strtok(0, sep)) 273 | + filter_argv[i++] = p; 274 | + filter_argv[i] = NULL; 275 | + if (p) { 276 | + rprintf(FERROR, 277 | + "Too many arguments to dest-filter (> %d)\n", 278 | + MAX_FILTER_ARGS); 279 | + exit_cleanup(RERR_SYNTAX); 280 | + } 281 | + } 282 | + 283 | progress_init(); 284 | 285 | while (1) { 286 | @@ -873,6 +893,9 @@ int recv_files(int f_in, int f_out, char *local_name) 287 | else if (!am_server && INFO_GTE(NAME, 1) && INFO_EQ(PROGRESS, 1)) 288 | rprintf(FINFO, "%s\n", fname); 289 | 290 | + if (dest_filter) 291 | + pid = run_filter(filter_argv, fd2, &fd2); 292 | + 293 | /* recv file data */ 294 | recv_ok = receive_data(f_in, fnamecmp, fd1, st.st_size, fname, fd2, file, inplace || one_inplace); 295 | 296 | @@ -888,6 +911,16 @@ int recv_files(int f_in, int f_out, char *local_name) 297 | exit_cleanup(RERR_FILEIO); 298 | } 299 | 300 | + if (dest_filter) { 301 | + int status; 302 | + wait_process_with_flush(pid, &status); 303 | + if (status != 0) { 304 | + rprintf(FERROR, "filter %s exited code: %d\n", 305 | + dest_filter, status); 306 | + continue; 307 | + } 308 | + } 309 | + 310 | if ((recv_ok && (!delay_updates || !partialptr)) || inplace) { 311 | if (partialptr == fname) 312 | partialptr = NULL; 313 | diff --git a/rsync.1.md b/rsync.1.md 314 | --- a/rsync.1.md 315 | +++ b/rsync.1.md 316 | @@ -504,6 +504,7 @@ has its own detailed description later in this manpage. 317 | --contimeout=SECONDS set daemon connection timeout in seconds 318 | --ignore-times, -I don't skip files that match size and time 319 | --size-only skip files that match in size 320 | +--times-only skip files that match in mod-time 321 | --modify-window=NUM, -@ set the accuracy for mod-time comparisons 322 | --temp-dir=DIR, -T create temporary files in directory DIR 323 | --fuzzy, -y find similar file for basis if no dest file 324 | @@ -553,6 +554,8 @@ has its own detailed description later in this manpage. 325 | --write-batch=FILE write a batched update to FILE 326 | --only-write-batch=FILE like --write-batch but w/o updating dest 327 | --read-batch=FILE read a batched update from FILE 328 | +--source-filter=COMMAND filter file through COMMAND at source 329 | +--dest-filter=COMMAND filter file through COMMAND at destination 330 | --protocol=NUM force an older protocol version to be used 331 | --iconv=CONVERT_SPEC request charset conversion of filenames 332 | --checksum-seed=NUM set block/file checksum seed (advanced) 333 | @@ -3713,6 +3716,36 @@ expand it. 334 | [`--write-batch`](#opt). If _FILE_ is `-`, the batch data will be read 335 | from standard input. See the "BATCH MODE" section for details. 336 | 337 | +0. `--source-filter=COMMAND` 338 | + 339 | + This option allows the user to specify a filter program that will be 340 | + applied to the contents of all transferred regular files before the data is 341 | + sent to destination. COMMAND will receive the data on its standard input 342 | + and it should write the filtered data to standard output. COMMAND should 343 | + exit non-zero if it cannot process the data or if it encounters an error 344 | + when writing the data to stdout. 345 | + 346 | + Example: `--source-filter="gzip -9"` will cause remote files to be 347 | + compressed. Use of `--source-filter` automatically enables 348 | + [`--whole-file`](#opt). If your filter does not output the same number of 349 | + bytes that it received on input, you should use `--times-only` to 350 | + disable size and content checks on subsequent rsync runs. 351 | + 352 | +0. `--dest-filter=COMMAND` 353 | + 354 | + This option allows you to specify a filter program that will be applied to 355 | + the contents of all transferred regular files before the data is written to 356 | + disk. COMMAND will receive the data on its standard input and it should 357 | + write the filtered data to standard output. COMMAND should exit non-zero 358 | + if it cannot process the data or if it encounters an error when writing the 359 | + data to stdout. 360 | + 361 | + Example: --dest-filter="gzip -9" will cause remote files to be compressed. 362 | + Use of --dest-filter automatically enables --whole-file. If your filter 363 | + does not output the same number of bytes that it received on input, you 364 | + should use --times-only to disable size and content checks on subsequent 365 | + rsync runs. 366 | + 367 | 0. `--protocol=NUM` 368 | 369 | Force an older protocol version to be used. This is useful for creating a 370 | diff --git a/rsync.h b/rsync.h 371 | --- a/rsync.h 372 | +++ b/rsync.h 373 | @@ -169,6 +169,7 @@ 374 | #define IOERR_DEL_LIMIT (1<<2) 375 | 376 | #define MAX_ARGS 1000 377 | +#define MAX_FILTER_ARGS 100 378 | #define MAX_BASIS_DIRS 20 379 | #define MAX_SERVER_ARGS (MAX_BASIS_DIRS*2 + 100) 380 | 381 | diff --git a/sender.c b/sender.c 382 | --- a/sender.c 383 | +++ b/sender.c 384 | @@ -21,6 +21,7 @@ 385 | 386 | #include "rsync.h" 387 | #include "inums.h" 388 | +#include "ifuncs.h" 389 | 390 | extern int do_xfers; 391 | extern int am_server; 392 | @@ -50,6 +51,7 @@ extern int batch_fd; 393 | extern int write_batch; 394 | extern int file_old_total; 395 | extern BOOL want_progress_now; 396 | +extern char *source_filter; 397 | extern struct stats stats; 398 | extern struct file_list *cur_flist, *first_flist, *dir_flist; 399 | extern char num_dev_ino_buf[4 + 8 + 8]; 400 | @@ -211,6 +213,26 @@ void send_files(int f_in, int f_out) 401 | int f_xfer = write_batch < 0 ? batch_fd : f_out; 402 | int save_io_error = io_error; 403 | int ndx, j; 404 | + char *filter_argv[MAX_FILTER_ARGS + 1]; 405 | + char *tmp = 0; 406 | + int unlink_tmp = 0; 407 | + 408 | + if (source_filter) { 409 | + char *p; 410 | + char *sep = " \t"; 411 | + int i; 412 | + for (p = strtok(source_filter, sep), i = 0; 413 | + p && i < MAX_FILTER_ARGS; 414 | + p = strtok(0, sep)) 415 | + filter_argv[i++] = p; 416 | + filter_argv[i] = NULL; 417 | + if (p) { 418 | + rprintf(FERROR, 419 | + "Too many arguments to source-filter (> %d)\n", 420 | + MAX_FILTER_ARGS); 421 | + exit_cleanup(RERR_SYNTAX); 422 | + } 423 | + } 424 | 425 | if (DEBUG_GTE(SEND, 1)) 426 | rprintf(FINFO, "send_files starting\n"); 427 | @@ -348,6 +370,7 @@ void send_files(int f_in, int f_out) 428 | exit_cleanup(RERR_PROTOCOL); 429 | } 430 | 431 | + unlink_tmp = 0; 432 | fd = do_open(fname, O_RDONLY, 0); 433 | if (fd == -1) { 434 | if (errno == ENOENT) { 435 | @@ -367,6 +390,33 @@ void send_files(int f_in, int f_out) 436 | continue; 437 | } 438 | 439 | + if (source_filter) { 440 | + int fd2; 441 | + char *tmpl = "/tmp/rsync-filtered_sourceXXXXXX"; 442 | + 443 | + tmp = strdup(tmpl); 444 | + fd2 = mkstemp(tmp); 445 | + if (fd2 == -1) { 446 | + rprintf(FERROR, "mkstemp %s failed: %s\n", 447 | + tmp, strerror(errno)); 448 | + } else { 449 | + int status; 450 | + pid_t pid = run_filter_on_file(filter_argv, fd2, fd); 451 | + close(fd); 452 | + close(fd2); 453 | + wait_process_with_flush(pid, &status); 454 | + if (status != 0) { 455 | + rprintf(FERROR, 456 | + "bypassing source filter %s; exited with code: %d\n", 457 | + source_filter, status); 458 | + fd = do_open(fname, O_RDONLY, 0); 459 | + } else { 460 | + fd = do_open(tmp, O_RDONLY, 0); 461 | + unlink_tmp = 1; 462 | + } 463 | + } 464 | + } 465 | + 466 | /* map the local file */ 467 | if (do_fstat(fd, &st) != 0) { 468 | io_error |= IOERR_GENERAL; 469 | @@ -437,6 +487,8 @@ void send_files(int f_in, int f_out) 470 | } 471 | } 472 | close(fd); 473 | + if (unlink_tmp) 474 | + unlink(tmp); 475 | 476 | free_sums(s); 477 | 478 | -------------------------------------------------------------------------------- /sparse-block.diff: -------------------------------------------------------------------------------- 1 | This patch adds the --sparse-block option. Andrea Righi writes: 2 | 3 | In some filesystems, typically optimized for large I/O throughputs (like 4 | IBM GPFS, IBM SAN FS, or distributed filesystems in general) a lot of 5 | lseek() operations can strongly impact on performances. In this cases it 6 | can be helpful to enlarge the block size used to handle sparse files 7 | directly from a command line parameter. 8 | 9 | For example, using a sparse write size of 32KB, I've been able to 10 | increase the transfer rate of an order of magnitude copying the output 11 | files of scientific applications from GPFS to GPFS or GPFS to SAN FS. 12 | 13 | -Andrea 14 | 15 | To use this patch, run these commands for a successful build: 16 | 17 | patch -p1 0) { 35 | int r1; 36 | if (sparse_files > 0) { 37 | - int len1 = MIN(len, SPARSE_WRITE_SIZE); 38 | + int len1 = MIN(len, sparse_files_block_size ? sparse_files_block_size : SPARSE_WRITE_SIZE); 39 | r1 = write_sparse(f, use_seek, offset, buf, len1); 40 | offset += r1; 41 | } else { 42 | diff --git a/options.c b/options.c 43 | --- a/options.c 44 | +++ b/options.c 45 | @@ -83,6 +83,7 @@ int remove_source_files = 0; 46 | int one_file_system = 0; 47 | int protocol_version = PROTOCOL_VERSION; 48 | int sparse_files = 0; 49 | +long sparse_files_block_size = 0; 50 | int preallocate_files = 0; 51 | int do_compression = 0; 52 | int do_compression_level = CLVL_NOT_SPECIFIED; 53 | @@ -702,6 +703,7 @@ static struct poptOption long_options[] = { 54 | {"sparse", 'S', POPT_ARG_VAL, &sparse_files, 1, 0, 0 }, 55 | {"no-sparse", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 }, 56 | {"no-S", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 }, 57 | + {"sparse-block", 0, POPT_ARG_LONG, &sparse_files_block_size, 0, 0, 0 }, 58 | {"preallocate", 0, POPT_ARG_NONE, &preallocate_files, 0, 0, 0}, 59 | {"inplace", 0, POPT_ARG_VAL, &inplace, 1, 0, 0 }, 60 | {"no-inplace", 0, POPT_ARG_VAL, &inplace, 0, 0, 0 }, 61 | @@ -2772,6 +2774,12 @@ void server_options(char **args, int *argc_p) 62 | args[ac++] = arg; 63 | } 64 | 65 | + if (sparse_files_block_size) { 66 | + if (asprintf(&arg, "--sparse-block=%lu", sparse_files_block_size) < 0) 67 | + goto oom; 68 | + args[ac++] = arg; 69 | + } 70 | + 71 | if (io_timeout) { 72 | if (asprintf(&arg, "--timeout=%d", io_timeout) < 0) 73 | goto oom; 74 | diff --git a/rsync.1.md b/rsync.1.md 75 | --- a/rsync.1.md 76 | +++ b/rsync.1.md 77 | @@ -466,6 +466,7 @@ has its own detailed description later in this manpage. 78 | --super receiver attempts super-user activities 79 | --fake-super store/recover privileged attrs using xattrs 80 | --sparse, -S turn sequences of nulls into sparse blocks 81 | +--sparse-block=SIZE set block size used to handle sparse files 82 | --preallocate allocate dest files before writing them 83 | --dry-run, -n perform a trial run with no changes made 84 | --whole-file, -W copy files whole (w/o delta-xfer algorithm) 85 | @@ -1730,6 +1731,18 @@ expand it. 86 | (as opposed to allocated sequences of null bytes) if the kernel version and 87 | filesystem type support creating holes in the allocated data. 88 | 89 | +0. `--sparse-block=SIZE` 90 | + 91 | + Change the block size used to handle sparse files to SIZE bytes. This 92 | + option only has an effect if the [`--sparse`](#opt) (`-S`) option was also 93 | + specified. The default block size used by rsync to detect a file hole is 94 | + 1024 bytes; when the receiver writes data to the destination file and 95 | + option [`--sparse`](#opt) is used, rsync checks every 1024-bytes chunk to 96 | + detect if they are actually filled with data or not. With certain 97 | + filesystems, optimized to receive data streams for example, enlarging this 98 | + block size can strongly increase performance. The option can be used to 99 | + tune this block size. 100 | + 101 | 0. `--dry-run`, `-n` 102 | 103 | This makes rsync perform a trial run that doesn't make any changes (and 104 | -------------------------------------------------------------------------------- /transliterate.diff: -------------------------------------------------------------------------------- 1 | This patch adds an option --tr=BAD/GOOD to transliterate filenames. It 2 | can be used to remove characters illegal on the destination filesystem. 3 | Jeff Weber expressed interest in this: 4 | 5 | http://lists.samba.org/archive/rsync/2007-October/018996.html 6 | 7 | To use this patch, run these commands for a successful build: 8 | 9 | patch -p1 = 30)) 134 | args[ac++] = "--no-implied-dirs"; 135 | 136 | + if (tr_opt) { 137 | + if (asprintf(&arg, "--tr=%s", tr_opt) < 0) 138 | + goto oom; 139 | + args[ac++] = arg; 140 | + } 141 | + 142 | if (write_devices && am_sender) 143 | args[ac++] = "--write-devices"; 144 | 145 | diff --git a/rsync.1.md b/rsync.1.md 146 | --- a/rsync.1.md 147 | +++ b/rsync.1.md 148 | @@ -555,6 +555,7 @@ has its own detailed description later in this manpage. 149 | --read-batch=FILE read a batched update from FILE 150 | --protocol=NUM force an older protocol version to be used 151 | --iconv=CONVERT_SPEC request charset conversion of filenames 152 | +--tr=BAD/GOOD transliterate filenames 153 | --checksum-seed=NUM set block/file checksum seed (advanced) 154 | --ipv4, -4 prefer IPv4 155 | --ipv6, -6 prefer IPv6 156 | @@ -3755,6 +3756,22 @@ expand it. 157 | free to specify just the local charset for a daemon transfer (e.g. 158 | `--iconv=utf8`). 159 | 160 | +0. `--tr=BAD/GOOD` 161 | + 162 | + Transliterates filenames on the receiver, after the iconv conversion (if 163 | + any). This can be used to remove characters illegal on the destination 164 | + filesystem. If you use this option, consider saving a "find . -ls" listing 165 | + of the source in the destination to help you determine the original 166 | + filenames in case of need. 167 | + 168 | + The argument consists of a string of characters to remove, optionally 169 | + followed by a slash and a string of corresponding characters with which to 170 | + replace them. The second string may be shorter, in which case any leftover 171 | + characters in the first string are simply deleted. For example, 172 | + `--tr=':\/!'` replaces colons with exclamation marks and deletes 173 | + backslashes. Slashes cannot be transliterated because it would cause 174 | + havoc. 175 | + 176 | 0. `--ipv4`, `-4` or `--ipv6`, `-6` 177 | 178 | Tells rsync to prefer IPv4/IPv6 when creating sockets or running ssh. This 179 | --------------------------------------------------------------------------------