├── libfs ├── .gitignore ├── README.md ├── Cargo.toml └── src │ ├── errors.rs │ ├── fallback.rs │ ├── lib.rs │ ├── common.rs │ └── linux.rs ├── .gitignore ├── tests ├── scripts │ ├── make-filesystems.sh │ └── test-linux.sh ├── util.rs └── linux.rs ├── libxcp ├── Cargo.toml ├── src │ ├── errors.rs │ ├── paths.rs │ ├── drivers │ │ ├── mod.rs │ │ ├── parfile.rs │ │ └── parblock.rs │ ├── feedback.rs │ ├── lib.rs │ ├── backup.rs │ ├── config.rs │ └── operations.rs └── README.md ├── completions ├── xcp.bash ├── xcp.zsh └── xcp.fish ├── Cargo.toml ├── src ├── progress.rs ├── main.rs └── options.rs ├── .github └── workflows │ └── tests.yml ├── README.md └── Cargo.lock /libfs/.gitignore: -------------------------------------------------------------------------------- 1 | Cargo.lock 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | **/*.rs.bk 3 | /.idea 4 | bacon.toml 5 | -------------------------------------------------------------------------------- /tests/scripts/make-filesystems.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sudo apt update 4 | 5 | sudo apt install -y zfsutils-linux xfsprogs \ 6 | btrfs-progs ntfs-3g dosfstools 7 | 8 | for fs in "$@"; do 9 | root=/fs/$fs 10 | img=$root.img 11 | 12 | echo >&2 "==== creating $fs in $root ====" 13 | 14 | sudo mkdir --parents $root 15 | sudo fallocate --length 2.5G $img 16 | 17 | case $fs in 18 | zfs) sudo zpool create -m $root test $img ;; 19 | ntfs) sudo mkfs.ntfs --fast --force $img ;; 20 | *) sudo mkfs.$fs $img ;; 21 | esac 22 | 23 | # zfs gets automounted 24 | if [[ $fs != zfs ]]; then 25 | if [[ $fs = fat ]]; then 26 | sudo mount -o uid=$(id -u) $img $root 27 | else 28 | sudo mount $img $root 29 | fi 30 | fi 31 | 32 | # fat mount point cannot be chowned 33 | # and is handled by the uid= option above 34 | if [[ $fs != fat ]]; then 35 | sudo chown $USER $root 36 | fi 37 | 38 | git clone . $root/src 39 | done 40 | 41 | findmnt --real 42 | -------------------------------------------------------------------------------- /libfs/README.md: -------------------------------------------------------------------------------- 1 | # libfs: Advanced file and fs operations 2 | 3 | `libfs` is a library of file and filesystem operations that is supplementary to 4 | [std::fs](https://doc.rust-lang.org/std/fs/). Current features: 5 | 6 | * High and mid-level functions for creating and copying sparse files. 7 | * Copying will use Linux 8 | [copy_file_range](https://man7.org/linux/man-pages/man2/copy_file_range.2.html) 9 | where possible, with fall-back to userspace. 10 | * Scanning and merging extent information on filesystems that support it. 11 | * File permission copying, including 12 | [xattrs](https://man7.org/linux/man-pages/man7/xattr.7.html). 13 | 14 | Some of the features are Linux specific, but most have fall-back alternative 15 | implementations for other Unix-like OSs. Further support is todo. 16 | 17 | `libfs` is part of the [xcp](https://crates.io/crates/xcp) project. 18 | 19 | [![Crates.io](https://img.shields.io/crates/v/xcp.svg?colorA=777777)](https://crates.io/crates/libfs) 20 | [![doc.rs](https://docs.rs/libfs/badge.svg)](https://docs.rs/libfs) 21 | ![Github Actions](https://github.com/tarka/xcp/actions/workflows/tests.yml/badge.svg) 22 | -------------------------------------------------------------------------------- /libxcp/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "libxcp" 3 | description = "`libxcp` is a high-level file-copy engine with support for multi-threading, fine-grained progress feedback, pluggable drivers, and `.gitignore` filters. `libxcp` provides the core functionality of `xcp`." 4 | version = "0.24.2" 5 | edition = "2024" 6 | rust-version = "1.88.0" 7 | 8 | authors = ["Steve Smith "] 9 | homepage = "https://github.com/tarka/xcp" 10 | repository = "https://github.com/tarka/xcp" 11 | readme = "README.md" 12 | 13 | keywords = ["coreutils", "cp", "files", "filesystem"] 14 | categories =["filesystem"] 15 | license = "GPL-3.0-only" 16 | 17 | [features] 18 | default = ["parblock", "use_linux"] 19 | parblock = [] 20 | use_linux = ["libfs/use_linux"] 21 | 22 | [dependencies] 23 | anyhow = "1.0.99" 24 | blocking-threadpool = "1.0.1" 25 | cfg-if = "1.0.3" 26 | crossbeam-channel = "0.5.15" 27 | ignore = "0.4.23" 28 | libfs = { version = "0.9.2", path = "../libfs" } 29 | log = "0.4.27" 30 | num_cpus = "1.17.0" 31 | regex = "1.11.2" 32 | thiserror = "2.0.16" 33 | walkdir = "2.5.0" 34 | 35 | [dev-dependencies] 36 | tempfile = "3.21.0" 37 | 38 | [lints.clippy] 39 | upper_case_acronyms = "allow" 40 | -------------------------------------------------------------------------------- /libfs/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "libfs" 3 | description = "`libfs` is a library of file and filesystem operations that is supplementary to `std::fs`" 4 | version = "0.9.2" 5 | edition = "2024" 6 | rust-version = "1.88.0" 7 | 8 | authors = ["Steve Smith "] 9 | homepage = "https://github.com/tarka/xcp/libfs" 10 | repository = "https://github.com/tarka/xcp/libfs" 11 | readme = "README.md" 12 | 13 | keywords = ["coreutils", "files", "filesystem", "sparse"] 14 | categories =["filesystem"] 15 | license = "GPL-3.0-only" 16 | 17 | [features] 18 | default = ["use_linux"] 19 | use_linux = [] 20 | # For CI; disable feature testing on filesystems that don't support 21 | # it. See .github/workflows/tests.yml 22 | test_no_acl = [] 23 | test_no_reflink = [] 24 | test_no_sparse = [] 25 | test_no_extents = [] 26 | test_no_sockets = [] 27 | 28 | [dependencies] 29 | cfg-if = "1.0.3" 30 | libc = "0.2.175" 31 | linux-raw-sys = { version = "0.10.0", features = ["ioctl"] } 32 | log = "0.4.27" 33 | rustix = { version = "1.0.8", features = ["fs"] } 34 | thiserror = "2.0.16" 35 | xattr = "1.5.1" 36 | 37 | [dev-dependencies] 38 | exacl = "0.12.0" 39 | tempfile = "3.21.0" 40 | 41 | [lints.clippy] 42 | upper_case_acronyms = "allow" 43 | -------------------------------------------------------------------------------- /libfs/src/errors.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | use std::path::PathBuf; 18 | 19 | #[derive(Debug, thiserror::Error)] 20 | pub enum Error { 21 | #[error("Invalid source: {0}")] 22 | InvalidSource(&'static str), 23 | 24 | #[error("Invalid path: {0}")] 25 | InvalidPath(PathBuf), 26 | 27 | #[error(transparent)] 28 | IOError(#[from] std::io::Error), 29 | 30 | #[error(transparent)] 31 | OSError(#[from] rustix::io::Errno), 32 | 33 | #[error("Unsupported operation; this function should never be called on this OS.")] 34 | UnsupportedOperation, 35 | } 36 | 37 | pub type Result = std::result::Result; 38 | -------------------------------------------------------------------------------- /libxcp/src/errors.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | //! Custom error types. 18 | 19 | use std::path::PathBuf; 20 | 21 | pub use anyhow::Result; 22 | 23 | #[derive(Debug, thiserror::Error)] 24 | pub enum XcpError { 25 | #[error("Error during copy: {0}")] 26 | CopyError(String), 27 | 28 | #[error("Destination Exists: {0}, {1}")] 29 | DestinationExists(&'static str, PathBuf), 30 | 31 | #[error("Early shutdown: {0}")] 32 | EarlyShutdown(&'static str), 33 | 34 | #[error("Invalid arguments: {0}")] 35 | InvalidArguments(String), 36 | 37 | #[error("Invalid destination: {0}")] 38 | InvalidDestination(&'static str), 39 | 40 | #[error("Invalid source: {0}")] 41 | InvalidSource(&'static str), 42 | 43 | #[error("Failed to reflink file and 'always' was specified: {0}")] 44 | ReflinkFailed(String), 45 | 46 | #[error("Unknown driver: {0}")] 47 | UnknownDriver(String), 48 | 49 | #[error("Unknown file-type: {0}")] 50 | UnknownFileType(PathBuf), 51 | 52 | #[error("Unsupported OS")] 53 | UnsupportedOS(&'static str), 54 | } 55 | -------------------------------------------------------------------------------- /libxcp/src/paths.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2024, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | use std::path::Path; 18 | use ignore::gitignore::{Gitignore, GitignoreBuilder}; 19 | use log::info; 20 | use walkdir::DirEntry; 21 | 22 | use crate::config::Config; 23 | use crate::errors::Result; 24 | 25 | /// Parse a git ignore file. 26 | pub fn parse_ignore(source: &Path, config: &Config) -> Result> { 27 | let gitignore = if config.gitignore { 28 | let gifile = source.join(".gitignore"); 29 | info!("Using .gitignore file {gifile:?}"); 30 | let mut builder = GitignoreBuilder::new(source); 31 | builder.add(&gifile); 32 | let ignore = builder.build()?; 33 | Some(ignore) 34 | } else { 35 | None 36 | }; 37 | Ok(gitignore) 38 | } 39 | 40 | /// Filter to return whether a given file should be ignored by a 41 | /// filter file. 42 | pub fn ignore_filter(entry: &DirEntry, ignore: &Option) -> bool { 43 | match ignore { 44 | None => true, 45 | Some(gi) => { 46 | let path = entry.path(); 47 | let m = gi.matched(path, path.is_dir()); 48 | !m.is_ignore() 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /completions/xcp.bash: -------------------------------------------------------------------------------- 1 | _xcp() { 2 | local cur prev words cword 3 | _init_completion || return 4 | 5 | # do not suggest options after -- 6 | local i 7 | for ((i = 1; i < cword; i++)); do 8 | if [[ ${words[$i]} == -- ]]; then 9 | _filedir 10 | return 11 | fi 12 | done 13 | 14 | local options=( 15 | -T 16 | -g 17 | -h 18 | -n 19 | -f 20 | -r 21 | -v 22 | -w 23 | -L 24 | "$(_parse_help "$1" -h)" # long options will be parsed from `--help` 25 | ) 26 | local units='B K M G' # in line with most completions prefer M to MB/MiB 27 | local drivers='parfile parblock' 28 | local reflink='auto always never' 29 | local backup='none numbered auto' 30 | 31 | case "$prev" in 32 | -h | --help) return ;; 33 | 34 | --block-size) 35 | if [[ -z $cur ]]; then 36 | COMPREPLY=(1M) # replace "nothing" with the default block size 37 | else 38 | local num="${cur%%[^0-9]*}" # suggest unit suffixes after numbers 39 | local unit="${cur##*[0-9]}" 40 | COMPREPLY=($(compgen -P "$num" -W "$units" -- "$unit")) 41 | fi 42 | return 43 | ;; 44 | 45 | --reflink) 46 | COMPREPLY=($(compgen -W "$reflink" -- "$cur")) 47 | return 48 | ;; 49 | 50 | --backup) 51 | COMPREPLY=($(compgen -W "$backup" -- "$cur")) 52 | return 53 | ;; 54 | 55 | --driver) 56 | COMPREPLY=($(compgen -W "$drivers" -- "$cur")) 57 | return 58 | ;; 59 | 60 | -w | --workers) 61 | COMPREPLY=($(compgen -W "{0..$(_ncpus)}" -- "$cur")) # 0 == auto 62 | return 63 | ;; 64 | esac 65 | 66 | if [[ $cur == -* ]]; then 67 | COMPREPLY=($(compgen -W "${options[*]}" -- "$cur")) 68 | return 69 | fi 70 | 71 | _filedir # suggest files if nothing else matched 72 | } && complete -F _xcp xcp 73 | 74 | # vim: sw=2 sts=2 et ai ft=bash 75 | # path: /usr/share/bash-completion/completions/xcp 76 | -------------------------------------------------------------------------------- /tests/scripts/test-linux.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | # chdir to source root 6 | cd "$(dirname "$0")"/../.. 7 | 8 | # get the name of the filesystem that contains the source code 9 | fs=$(df --output=fstype . | tail -n 1) 10 | 11 | # list features supported by all filesystems 12 | features=(use_linux "$@") 13 | 14 | # Some permissions tests don't work with root privs 15 | if [[ "$(id -u)" == "0" ]]; then 16 | features+=(test_no_root) 17 | fi 18 | 19 | # disable tests that will not work on this filesystem 20 | case "$fs" in 21 | xfs | btrfs | bcachefs) ;; 22 | 23 | ext4) 24 | features+=( 25 | test_no_reflink 26 | ) 27 | ;; 28 | 29 | ext[23]) 30 | features+=( 31 | test_no_extents 32 | test_no_reflink 33 | ) 34 | ;; 35 | 36 | f2fs) 37 | features+=( 38 | test_no_reflink 39 | ) 40 | ;; 41 | 42 | fuseblk) 43 | echo >&2 "WARNING: assuming ntfs" 44 | features+=( 45 | test_no_acl 46 | test_no_extents 47 | test_no_reflink 48 | test_no_sparse 49 | test_no_perms 50 | ) 51 | ;; 52 | 53 | vfat) 54 | features+=( 55 | test_no_acl 56 | test_no_extents 57 | test_no_reflink 58 | test_no_sockets 59 | test_no_sparse 60 | test_no_symlinks 61 | test_no_xattr 62 | test_no_perms 63 | ) 64 | ;; 65 | 66 | tmpfs) 67 | features+=( 68 | test_no_extents 69 | test_no_reflink 70 | test_no_sparse 71 | ) 72 | ;; 73 | 74 | zfs) 75 | features+=( 76 | test_no_acl 77 | test_no_extents 78 | test_no_reflink 79 | test_no_sparse 80 | ) 81 | ;; 82 | 83 | *) 84 | echo >&2 "WARNING: unknown filesystem $fs, advanced FS tests disabled." 85 | features+=( 86 | test_no_acl 87 | test_no_extents 88 | test_no_reflink 89 | test_no_sparse 90 | ) 91 | ;; 92 | esac 93 | 94 | echo >&2 "found filesystem $fs, using flags ${features[*]}" 95 | 96 | cargo test --workspace --release --locked --features "$( 97 | export IFS=, 98 | echo "${features[*]}" 99 | )" 100 | -------------------------------------------------------------------------------- /completions/xcp.zsh: -------------------------------------------------------------------------------- 1 | #compdef xcp 2 | 3 | typeset -A opt_args 4 | 5 | _xcp() { 6 | local -a args 7 | 8 | # short + long 9 | args+=( 10 | '(- *)'{-h,--help}'[Print help]' 11 | '*'{-v,--verbose}'[Increase verbosity (can be repeated)]' 12 | {-T,--no-target-directory}'[Overwrite target directory, do not create a subdirectory]' 13 | {-g,--glob}'[Expand (glob) filename patterns]' 14 | {-n,--no-clobber}'[Do not overwrite an existing file]' 15 | {-f,--force}'[Compatibility only option]' 16 | {-r,--recursive}'[Copy directories recursively]' 17 | {-w,--workers}'[Workers for recursive copies (0=auto)]:workers:_values workers {0..$(getconf _NPROCESSORS_ONLN)}' 18 | {-L,--dereference}'[Dereference symlinks in source]' 19 | {-o,--ownership}'[Copy ownship (user/group)]' 20 | ) 21 | 22 | # long 23 | args+=( 24 | --block-size'[Block size for file operations]: :_numbers -u bytes -d 1M size B K M G' 25 | --driver'[How to parallelise file operations]:driver:(( 26 | parfile\:"parallelise at the file level (default)" 27 | parblock\:"parallelise at the block level" 28 | ))' 29 | --reflink'[Whether and how to use reflinks]:reflink:(( 30 | auto\:"attempt to reflink and fallback to a copy (default)" 31 | always\:"return an error if it cannot reflink" 32 | never\:"always perform a full data copy" 33 | ))' 34 | --backup'[Whether to create backups of overwritten files]:backup:(( 35 | none\:"no backups (default)" 36 | numbered\:"follow the semantics of cp numbered backups" 37 | auto\:"create a numbered backup if previous backup exists" 38 | ))' 39 | --fsync'[Sync each file to disk after it is written]' 40 | --gitignore'[Use .gitignore if present]' 41 | --no-perms'[Do not copy file permissions]' 42 | --no-timestamps'[Do not copy file timestamps]' 43 | --no-progress'[Disable progress bar]' 44 | --target-directory'[Copy into a subdirectory of the target]: :_files -/' 45 | ) 46 | 47 | # positional 48 | args+=( 49 | '*:paths:_files' 50 | ) 51 | 52 | _arguments -s -S $args 53 | } 54 | 55 | _xcp "$@" 56 | 57 | # vim: sw=2 sts=2 et ai ft=zsh 58 | # path: /usr/share/zsh/site-functions/_xcp 59 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | ".", 4 | "libxcp", 5 | "libfs", 6 | ] 7 | default-members = [".", "libfs"] 8 | resolver = "2" 9 | 10 | [package] 11 | name = "xcp" 12 | description = "xcp is a (partial) clone of the Unix `cp` command, with more user-friendly feedback and some performance optimisations. See the README for features and limitations." 13 | version = "0.24.2" 14 | rust-version = "1.88.0" 15 | edition = "2024" 16 | 17 | authors = ["Steve Smith "] 18 | homepage = "https://github.com/tarka/xcp" 19 | repository = "https://github.com/tarka/xcp" 20 | readme = "README.md" 21 | 22 | keywords = ["coreutils", "cp", "files", "filesystem"] 23 | categories =["command-line-utilities"] 24 | license = "GPL-3.0-only" 25 | 26 | [features] 27 | default = ["parblock", "use_linux"] 28 | parblock = ["libxcp/parblock"] 29 | use_linux = ["libfs/use_linux", "libxcp/use_linux"] 30 | # For CI; disable feature testing on filesystems that don't support 31 | # it. See .github/workflows/tests.yml 32 | test_no_reflink = ["libfs/test_no_reflink"] 33 | test_no_sparse = ["libfs/test_no_sparse"] 34 | test_no_extents = ["libfs/test_no_extents"] 35 | test_no_sockets = ["libfs/test_no_sockets"] 36 | test_no_acl = ["libfs/test_no_acl"] 37 | test_no_xattr = [] 38 | test_no_symlinks = [] 39 | test_no_perms = [] 40 | test_no_root = [] 41 | test_run_root = [] 42 | test_run_expensive = [] 43 | 44 | [dependencies] 45 | anyhow = "1.0.99" 46 | crossbeam-channel = "0.5.15" 47 | clap = { version = "4.5.46", features = ["derive"] } 48 | glob = "0.3.3" 49 | ignore = "0.4.23" 50 | indicatif = "0.18.0" 51 | libfs = { version = "0.9.2", path = "libfs" } 52 | libxcp = { version = "0.24.2", path = "libxcp" } 53 | log = "0.4.27" 54 | num_cpus = "1.17.0" 55 | simplelog = "0.12.2" 56 | unbytify = "0.2.0" 57 | terminal_size = "0.4.3" 58 | 59 | [dev-dependencies] 60 | cfg-if = "1.0.3" 61 | fslock = "0.2.1" 62 | rand = "0.9.2" 63 | rand_distr = "0.5.1" 64 | rand_xorshift = "0.4.0" 65 | rustix = { version = "1.0.8", features = ["process"] } 66 | tempfile = "3.21.0" 67 | test-case = "3.3.1" 68 | uuid = { version = "1.18.0", features = ["v4"] } 69 | walkdir = "2.5.0" 70 | xattr = "1.5.1" 71 | 72 | [lints.clippy] 73 | upper_case_acronyms = "allow" 74 | -------------------------------------------------------------------------------- /completions/xcp.fish: -------------------------------------------------------------------------------- 1 | set -l drivers ' 2 | parfile\t"parallelise at the file level (default)" 3 | parblock\t"parallelise at the block level" 4 | ' 5 | 6 | set -l reflinks ' 7 | auto\t"attempt to reflink and fallback to a copy (default)" 8 | always\t"return an error if it cannot reflink" 9 | never\t"always perform a full data copy" 10 | ' 11 | 12 | set -l backup ' 13 | none\t"no backups (default)" 14 | numbered\t"follow the semantics of cp numbered backups" 15 | auto\t"create a numbered backup if previous backup exists" 16 | ' 17 | 18 | # short + long 19 | complete -c xcp -s T -l no-target-directory -d 'Overwrite target directory, do not create a subdirectory' 20 | complete -c xcp -s g -l glob -d 'Expand (glob) filename patterns' 21 | complete -c xcp -s h -l help -f -d 'Print help' 22 | complete -c xcp -s n -l no-clobber -d 'Do not overwrite an existing file' 23 | complete -c xcp -s f -l force -d 'Compatibility only option' 24 | complete -c xcp -s r -l recursive -d 'Copy directories recursively' 25 | complete -c xcp -s v -l verbose -d 'Increase verbosity (can be repeated)' 26 | complete -c xcp -s w -l workers -d 'Workers for recursive copies (0=auto)' -x -a '(seq 0 (getconf _NPROCESSORS_ONLN))' 27 | complete -c xcp -s L -l dereference -d 'Dereference symlinks in source' 28 | complete -c xcp -s o -l ownership -d 'Copy ownship (user/group)' 29 | 30 | # long 31 | complete -c xcp -l fsync -d 'Sync each file to disk after it is written' 32 | complete -c xcp -l target-directory -d 'Copy into a subdirectory of the target' 33 | complete -c xcp -l gitignore -d 'Use .gitignore if present' 34 | complete -c xcp -l no-perms -d 'Do not copy file permissions' 35 | complete -c xcp -l no-timestamps -d 'Do not copy file timestamps' 36 | complete -c xcp -l no-progress -d 'Disable progress bar' 37 | complete -c xcp -l block-size -d 'Block size for file operations' -x -a '(seq 1 16){B,K,M,G}' 38 | complete -c xcp -l driver -d 'Parallelise at the file or at the block level' -x -a "$drivers" 39 | complete -c xcp -l reflink -d 'Whether and how to use reflinks' -x -a "$reflinks" 40 | complete -c xcp -l backup -d 'Whether to create backups of overwritten files' -x -a "$backup" 41 | 42 | # docs: https://fishshell.com/docs/current/completions.html 43 | # path: /usr/share/fish/vendor_completions.d/xcp.fish 44 | # vim: sw=2 sts=2 et ai ft=fish 45 | -------------------------------------------------------------------------------- /libfs/src/fallback.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | use std::fs::File; 18 | use std::path::Path; 19 | 20 | use log::warn; 21 | 22 | use crate::Extent; 23 | use crate::common::{copy_bytes_uspace, copy_range_uspace}; 24 | use crate::errors::{Result, Error}; 25 | 26 | pub fn copy_file_bytes(infd: &File, outfd: &File, bytes: u64) -> Result { 27 | copy_bytes_uspace(infd, outfd, bytes as usize) 28 | } 29 | 30 | pub fn copy_file_offset(infd: &File, outfd: &File, bytes: u64, off: i64) -> Result { 31 | copy_range_uspace(infd, outfd, bytes as usize, off as usize) 32 | } 33 | 34 | // No sparse file handling by default, needs to be implemented 35 | // per-OS. This effectively disables the following operations. 36 | pub fn probably_sparse(_fd: &File) -> Result { 37 | Ok(false) 38 | } 39 | 40 | pub fn map_extents(_fd: &File) -> Result>> { 41 | // FIXME: Implement for *BSD with lseek? 42 | Ok(None) 43 | } 44 | 45 | pub fn next_sparse_segments(_infd: &File, _outfd: &File, _pos: u64) -> Result<(u64, u64)> { 46 | // FIXME: Implement for *BSD with lseek? 47 | Err(Error::UnsupportedOperation {}) 48 | } 49 | 50 | pub fn copy_sparse(infd: &File, outfd: &File) -> Result { 51 | let len = infd.metadata()?.len(); 52 | copy_file_bytes(&infd, &outfd, len) 53 | .map(|i| i as u64) 54 | } 55 | 56 | pub fn copy_node(src: &Path, _dest: &Path) -> Result<()> { 57 | // FreeBSD `cp` just warns about this, so do the same here. 58 | warn!("Socket copy not supported by this OS: {}", src.to_string_lossy()); 59 | Ok(()) 60 | } 61 | 62 | pub fn reflink(_infd: &File, _outfd: &File) -> Result { 63 | Ok(false) 64 | } 65 | -------------------------------------------------------------------------------- /libxcp/README.md: -------------------------------------------------------------------------------- 1 | # libxcp: High-level file-copy engine 2 | 3 | `libxcp` is a high-level file-copy engine. It has a support for multi-threading, 4 | fine-grained progress feedback, pluggable drivers, and `.gitignore` filters. 5 | `libxcp` is the core functionality of the [xcp](https://crates.io/crates/xcp) 6 | command-line utility. 7 | 8 | [![Crates.io](https://img.shields.io/crates/v/xcp.svg?colorA=777777)](https://crates.io/crates/libxcp) 9 | [![doc.rs](https://docs.rs/libxcp/badge.svg)](https://docs.rs/libxcp) 10 | ![Github Actions](https://github.com/tarka/xcp/actions/workflows/tests.yml/badge.svg) 11 | 12 | ### Features 13 | 14 | * On Linux it uses `copy_file_range` call to copy files. This is the most 15 | efficient method of file-copying under Linux; in particular it is 16 | filesystem-aware, and can massively speed-up copies on network mounts by 17 | performing the copy operations server-side. However, unlike `copy_file_range` 18 | sparse files are detected and handled appropriately. 19 | * Support for modern filesystem features such as [reflinks](https://btrfs.readthedocs.io/en/latest/Reflink.html). 20 | * Optimised for 'modern' systems (i.e. multiple cores, copious RAM, and 21 | solid-state disks, especially ones connected into the main system bus, 22 | e.g. NVMe). 23 | * Optional aggressive parallelism for systems with parallel IO. Quick 24 | experiments on a modern laptop suggest there may be benefits to parallel 25 | copies on NVMe disks. This is obviously highly system-dependent. 26 | * Switchable 'drivers' to facilitate experimenting with alternative strategies 27 | for copy optimisation. Currently 2 drivers are available: 28 | * 'parfile': the previous hard-coded xcp copy method, which parallelises 29 | tree-walking and per-file copying. This is the default. 30 | * 'parblock': An experimental driver that parallelises copying at the block 31 | level. This has the potential for performance improvements in some 32 | architectures, but increases complexity. Testing is welcome. 33 | * Non-Linux Unix-like OSs (OS X, *BSD) are supported via fall-back operation 34 | (although sparse-files are not yet supported in this case). 35 | * Optionally understands `.gitignore` files to limit the copied directories. 36 | 37 | ## Testing 38 | 39 | `libxcp` itself doesn't have many tests; the top-level `xcp` application however 40 | has a full functional test suite, including fuzzed stress-tests. This should be 41 | considered the test suite for now. 42 | -------------------------------------------------------------------------------- /src/progress.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | use crate::options::Opts; 18 | 19 | use libxcp::errors::Result; 20 | use terminal_size::Width; 21 | 22 | struct NoopBar; 23 | 24 | struct VisualBar { 25 | bar: indicatif::ProgressBar, 26 | } 27 | 28 | pub trait ProgressBar { 29 | #[allow(unused)] 30 | fn set_size(&self, size: u64); 31 | fn inc_size(&self, size: u64); 32 | fn inc(&self, size: u64); 33 | fn end(&self); 34 | } 35 | 36 | 37 | impl ProgressBar for NoopBar { 38 | fn set_size(&self, _size: u64) { 39 | } 40 | fn inc_size(&self, _size: u64) { 41 | } 42 | fn inc(&self, _size: u64) { 43 | } 44 | fn end(&self) { 45 | } 46 | } 47 | 48 | impl ProgressBar for VisualBar { 49 | fn set_size(&self, size: u64) { 50 | self.bar.set_length(size); 51 | } 52 | 53 | fn inc_size(&self, size: u64) { 54 | self.bar.inc_length(size); 55 | } 56 | 57 | fn inc(&self, size: u64) { 58 | self.bar.inc(size); 59 | } 60 | 61 | fn end(&self) { 62 | self.bar.finish(); 63 | } 64 | } 65 | 66 | impl VisualBar { 67 | fn new(size: u64) -> Result { 68 | let bar = indicatif::ProgressBar::new(size).with_style( 69 | indicatif::ProgressStyle::default_bar() 70 | .template( 71 | match terminal_size::terminal_size() { 72 | Some((Width(width), _)) if width < 160 => "[{wide_bar:.cyan/blue}]\n{bytes:>11} / {total_bytes:<11} | {percent:>3}% | {bytes_per_sec:^13} | {eta_precise} remaining", 73 | _ => "[{wide_bar:.cyan/blue}] {bytes:>11} / {total_bytes:<11} | {percent:>3}% | {bytes_per_sec:^13} | {eta_precise} remaining", 74 | } 75 | )? 76 | .progress_chars("#>-"), 77 | ); 78 | Ok(Self { bar }) 79 | } 80 | } 81 | 82 | pub fn create_bar(opts: &Opts, size: u64) -> Result> { 83 | if opts.no_progress { 84 | Ok(Box::new(NoopBar {})) 85 | } else { 86 | Ok(Box::new(VisualBar::new(size)?)) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /libxcp/src/drivers/mod.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018-2019, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | //! Support for pluggable copy drivers. 18 | //! 19 | //! Two drivers are currently supported: 20 | //! * `parfile`: Parallelise copying at the file level. This can improve 21 | //! speed on modern NVME devices, but can bottleneck on larger files. 22 | //! * `parblock`: Parallelise copying at the block level. Block-size is 23 | //! configurable. This can have better performance for large files, 24 | //! but has a higher overhead. 25 | //! 26 | //! Drivers are configured with the [Config] struct. A convenience 27 | //! function [load_driver()] is provided to load a dynamic-dispatched 28 | //! instance of each driver. 29 | //! 30 | //! # Example 31 | //! 32 | //! See the example in top-level module. 33 | 34 | pub mod parfile; 35 | #[cfg(feature = "parblock")] 36 | pub mod parblock; 37 | 38 | use std::path::{Path, PathBuf}; 39 | use std::result; 40 | use std::str::FromStr; 41 | use std::sync::Arc; 42 | 43 | use crate::config::Config; 44 | use crate::errors::{Result, XcpError}; 45 | use crate::feedback::StatusUpdater; 46 | 47 | /// The trait specifying driver operations; drivers should implement 48 | /// this. 49 | pub trait CopyDriver { 50 | /// Recursively copy a set of directories or files to a 51 | /// destination. `dest` can be a file if a single file is provided 52 | /// the source. `StatusUpdater.send()` will be called with 53 | /// `StatusUpdate` objects depending on the driver configuration. 54 | /// `copy()` itself will block until all work is complete, so 55 | /// should be run in a thread if real-time updates are required. 56 | fn copy(&self, sources: Vec, dest: &Path, stats: Arc) -> Result<()>; 57 | } 58 | 59 | /// An enum specifing the driver to use. This is just a helper for 60 | /// applications to use with [load_driver()]. [FromStr] is implemented 61 | /// to help with this. 62 | #[derive(Debug, Clone, Copy)] 63 | pub enum Drivers { 64 | ParFile, 65 | #[cfg(feature = "parblock")] 66 | ParBlock, 67 | } 68 | 69 | // String conversion helper as a convenience for command-line parsing. 70 | impl FromStr for Drivers { 71 | type Err = XcpError; 72 | 73 | fn from_str(s: &str) -> result::Result { 74 | match s.to_lowercase().as_str() { 75 | "parfile" => Ok(Drivers::ParFile), 76 | #[cfg(feature = "parblock")] 77 | "parblock" => Ok(Drivers::ParBlock), 78 | _ => Err(XcpError::UnknownDriver(s.to_owned())), 79 | } 80 | } 81 | } 82 | 83 | /// Load and configure the given driver. 84 | pub fn load_driver(driver: Drivers, config: &Arc) -> Result> { 85 | let driver_impl: Box = match driver { 86 | Drivers::ParFile => Box::new(parfile::Driver::new(config.clone())?), 87 | #[cfg(feature = "parblock")] 88 | Drivers::ParBlock => Box::new(parblock::Driver::new(config.clone())?), 89 | }; 90 | 91 | Ok(driver_impl) 92 | } 93 | -------------------------------------------------------------------------------- /libfs/src/lib.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | mod common; 18 | mod errors; 19 | 20 | use std::{fs, ops::Range}; 21 | 22 | use cfg_if::cfg_if; 23 | use rustix::fs::FileTypeExt; 24 | 25 | cfg_if! { 26 | if #[cfg(all(target_os = "linux", feature = "use_linux"))] { 27 | mod linux; 28 | use linux as backend; 29 | } else { 30 | mod fallback; 31 | use fallback as backend; 32 | } 33 | } 34 | pub use backend::{ 35 | copy_file_bytes, 36 | copy_file_offset, 37 | copy_node, 38 | copy_sparse, 39 | probably_sparse, 40 | next_sparse_segments, 41 | map_extents, 42 | reflink, 43 | }; 44 | pub use common::{ 45 | allocate_file, 46 | copy_file, 47 | copy_owner, 48 | copy_permissions, 49 | copy_timestamps, 50 | is_same_file, 51 | merge_extents, 52 | sync, 53 | }; 54 | pub use errors::Error; 55 | 56 | /// Flag whether the current OS support 57 | /// [xattrs](https://man7.org/linux/man-pages/man7/xattr.7.html). 58 | pub const XATTR_SUPPORTED: bool = { 59 | // NOTE: The xattr crate has a SUPPORTED_PLATFORM flag, however it 60 | // allows NetBSD, which fails for us, so we stick to platforms we've 61 | // tested. 62 | cfg_if! { 63 | if #[cfg(any(target_os = "linux", target_os = "freebsd"))] { 64 | true 65 | } else { 66 | false 67 | } 68 | } 69 | }; 70 | 71 | /// Enum mapping for various *nix file types. Mapped from 72 | /// [std::fs::FileType] and [rustix::fs::FileTypeExt]. 73 | #[derive(Debug)] 74 | pub enum FileType { 75 | File, 76 | Dir, 77 | Symlink, 78 | Socket, 79 | Fifo, 80 | Char, 81 | Block, 82 | Other 83 | } 84 | 85 | impl From for FileType { 86 | fn from(ft: fs::FileType) -> Self { 87 | if ft.is_dir() { 88 | FileType::Dir 89 | } else if ft.is_file() { 90 | FileType::File 91 | } else if ft.is_symlink() { 92 | FileType::Symlink 93 | } else if ft.is_socket() { 94 | FileType::Socket 95 | } else if ft.is_fifo() { 96 | FileType::Fifo 97 | } else if ft.is_char_device() { 98 | FileType::Char 99 | } else if ft.is_block_device() { 100 | FileType::Block 101 | } else { 102 | FileType::Other 103 | } 104 | } 105 | } 106 | 107 | /// Struct representing a file extent metadata. 108 | #[derive(Debug, PartialEq)] 109 | pub struct Extent { 110 | /// Extent logical start 111 | pub start: u64, 112 | /// Extent logical end 113 | pub end: u64, 114 | /// Whether extent is shared between multiple file. This generally 115 | /// only applies to reflinked files on filesystems that support 116 | /// CoW. 117 | pub shared: bool, 118 | } 119 | 120 | impl From for Range { 121 | fn from(e: Extent) -> Self { 122 | e.start..e.end 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /libxcp/src/feedback.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2024, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | //! Support for runtime feedback of copy progress. 18 | //! 19 | 20 | //! Users of `libxcp` can implement the [StatusUpdater] trait and pass 21 | //! an instance to the driver, usually using `load_driver()`. Two 22 | //! implementations are provided: 23 | //! 24 | //! * [NoopUpdater] 25 | //! * [ChannelUpdater] 26 | 27 | use std::sync::Arc; 28 | use std::sync::atomic::{AtomicU64, Ordering}; 29 | use crossbeam_channel as cbc; 30 | 31 | use crate::config::Config; 32 | use crate::errors::{Result, XcpError}; 33 | 34 | /// A struct representing an updated status. 35 | #[derive(Debug)] 36 | pub enum StatusUpdate { 37 | /// An update representing a successful copy of bytes between 38 | /// files. 39 | Copied(u64), 40 | /// An update representing that this number of bytes will need to be copied. 41 | Size(u64), 42 | /// An error during a copy operation. 43 | Error(XcpError) 44 | } 45 | 46 | pub trait StatusUpdater: Sync + Send { 47 | fn send(&self, update: StatusUpdate) -> Result<()>; 48 | } 49 | 50 | /// An implementation of [StatusUpdater] which will return 51 | /// [StatusUpdate] objects via a channel. On copy completion the 52 | /// channel will be closed, allowing the caller to iterator over 53 | /// returned updates. See the top-level module for an example of 54 | /// usage. 55 | pub struct ChannelUpdater { 56 | chan_tx: cbc::Sender, 57 | chan_rx: cbc::Receiver, 58 | config: Arc, 59 | sent: AtomicU64, 60 | } 61 | 62 | impl ChannelUpdater { 63 | /// Create a new ChannelUpdater, including the channels. 64 | pub fn new(config: &Arc) -> ChannelUpdater { 65 | let (chan_tx, chan_rx) = cbc::unbounded(); 66 | ChannelUpdater { 67 | chan_tx, 68 | chan_rx, 69 | config: config.clone(), 70 | sent: AtomicU64::new(0), 71 | } 72 | } 73 | 74 | /// Retrieve a clone of the receive end of the update 75 | /// channel. Note: As ChannelUpdater is consumed by the driver 76 | /// call you should call this before then; e.g: 77 | /// 78 | /// # use std::sync::Arc; 79 | /// use libxcp::config::Config; 80 | /// use libxcp::feedback::{ChannelUpdater, StatusUpdater}; 81 | /// 82 | /// let config = Arc::new(Config::default()); 83 | /// let updater = ChannelUpdater::new(&config); 84 | /// let stat_rx = updater.rx_channel(); 85 | /// let stats: Arc = Arc::new(updater); 86 | pub fn rx_channel(&self) -> cbc::Receiver { 87 | self.chan_rx.clone() 88 | } 89 | } 90 | 91 | impl StatusUpdater for ChannelUpdater { 92 | // Wrapper around channel-send that groups updates together 93 | fn send(&self, update: StatusUpdate) -> Result<()> { 94 | if let StatusUpdate::Copied(bytes) = update { 95 | // Avoid saturating the queue with small writes 96 | let bsize = self.config.block_size; 97 | let prev_written = self.sent.fetch_add(bytes, Ordering::Relaxed); 98 | if ((prev_written + bytes) / bsize) > (prev_written / bsize) { 99 | self.chan_tx.send(update)?; 100 | } 101 | } else { 102 | self.chan_tx.send(update)?; 103 | } 104 | Ok(()) 105 | } 106 | } 107 | 108 | /// A null updater for when no feedback is required. 109 | pub struct NoopUpdater; 110 | 111 | impl StatusUpdater for NoopUpdater { 112 | fn send(&self, _update: StatusUpdate) -> Result<()> { 113 | Ok(()) 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: [push] 4 | 5 | jobs: 6 | ubuntu: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v4 10 | 11 | - name: CI runtime info 12 | run: uname -a && cat /etc/os-release 13 | 14 | - name: Add necessary packages 15 | run: sudo apt-get update && sudo apt-get install -y libacl1-dev 16 | 17 | - name: Update Rust to latest 18 | run: ~/.cargo/bin/rustup update 19 | 20 | - name: Create filesystems 21 | # f2fs and exfat modules are in linux-modules-extra-azure 22 | # and cannot be installed reliably: 23 | # https://github.com/actions/runner-images/issues/7587 24 | run: tests/scripts/make-filesystems.sh ext2 ext4 xfs btrfs ntfs fat zfs 25 | 26 | - name: Run tests on ext2 27 | run: /fs/ext2/src/tests/scripts/test-linux.sh 28 | if: always() 29 | 30 | - name: Run tests on ext4 31 | run: /fs/ext4/src/tests/scripts/test-linux.sh 32 | if: always() 33 | 34 | - name: Run tests on XFS 35 | run: /fs/xfs/src/tests/scripts/test-linux.sh 36 | if: always() 37 | 38 | - name: Run tests on btrfs 39 | run: /fs/btrfs/src/tests/scripts/test-linux.sh 40 | if: always() 41 | 42 | - name: Run tests on ntfs 43 | run: /fs/ntfs/src/tests/scripts/test-linux.sh 44 | if: always() 45 | 46 | - name: Run tests on fat 47 | run: /fs/fat/src/tests/scripts/test-linux.sh 48 | if: always() 49 | 50 | - name: Run tests on ZFS 51 | run: /fs/zfs/src/tests/scripts/test-linux.sh 52 | if: always() 53 | 54 | root: 55 | runs-on: ubuntu-latest 56 | steps: 57 | - uses: actions/checkout@v4 58 | 59 | - name: Add necessary packages 60 | run: sudo apt-get update && sudo apt-get install -y libacl1-dev rustup 61 | 62 | - name: Install stable rust for root 63 | run: sudo rustup default stable 64 | 65 | - name: Run root tests 66 | run: sudo ./tests/scripts/test-linux.sh test_run_root 67 | 68 | expensive: 69 | runs-on: ubuntu-latest 70 | steps: 71 | - uses: actions/checkout@v4 72 | 73 | - name: Add necessary packages 74 | run: sudo apt-get update && sudo apt-get install -y libacl1-dev 75 | 76 | - name: Update Rust to latest 77 | run: ~/.cargo/bin/rustup update 78 | 79 | - name: Run expensive tests 80 | run: ./tests/scripts/test-linux.sh test_run_expensive 81 | 82 | macos: 83 | runs-on: macos-latest 84 | steps: 85 | - uses: actions/checkout@v4 86 | 87 | - name: Install Rust 88 | run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash /dev/stdin -y 89 | 90 | - name: Update Rust (installer may lag behind) 91 | run: ~/.cargo/bin/rustup update 92 | 93 | - name: Run all tests 94 | run: ~/.cargo/bin/cargo test --workspace --features=test_no_reflink,test_no_sockets,test_run_expensive 95 | 96 | freebsd: 97 | runs-on: ubuntu-latest 98 | steps: 99 | - uses: actions/checkout@v4 100 | 101 | - uses: vmactions/freebsd-vm@v1 102 | with: 103 | usesh: true 104 | prepare: | 105 | pkg install -y curl 106 | pw user add -n testing -m 107 | run: | 108 | su testing -c ' 109 | curl -sSf https://sh.rustup.rs | sh /dev/stdin -y \ 110 | && ~/.cargo/bin/cargo test --workspace --features=test_no_reflink,test_no_sockets \ 111 | && ~/.cargo/bin/cargo clean 112 | ' 113 | 114 | nightly: 115 | runs-on: ubuntu-latest 116 | steps: 117 | - uses: actions/checkout@v4 118 | 119 | - name: Add necessary packages 120 | run: sudo apt-get update && sudo apt-get install -y libacl1-dev 121 | 122 | - name: Install Rust nightly 123 | run: ~/.cargo/bin/rustup toolchain install nightly 124 | 125 | - name: Compile and test with nightly 126 | run: ~/.cargo/bin/cargo +nightly test --workspace --features=test_no_reflink 127 | 128 | msrv-check: 129 | runs-on: ubuntu-latest 130 | steps: 131 | - uses: actions/checkout@v4 132 | 133 | - name: Update Rust to latest 134 | run: ~/.cargo/bin/rustup update 135 | 136 | - name: Install MSRV checker 137 | run: ~/.cargo/bin/cargo install cargo-msrv 138 | 139 | - name: Check MSRV 140 | run: ~/.cargo/bin/cargo msrv verify 141 | -------------------------------------------------------------------------------- /libxcp/src/lib.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2024, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | //! `libxcp` is a high-level file-copy engine. It has a support for 18 | //! multi-threading, fine-grained progress feedback, pluggable 19 | //! drivers, and `.gitignore` filters. `libxcp` is the core 20 | //! functionality of the [xcp] command-line utility. 21 | //! 22 | //! # Usage example 23 | //! 24 | //! # use libxcp::errors::Result; 25 | //! # use std::path::PathBuf; 26 | //! # use std::sync::Arc; 27 | //! # use std::thread; 28 | //! # use tempfile::TempDir; 29 | //! use libxcp::config::Config; 30 | //! use libxcp::errors::XcpError; 31 | //! use libxcp::feedback::{ChannelUpdater, StatusUpdater, StatusUpdate}; 32 | //! use libxcp::drivers::{Drivers, load_driver}; 33 | //! # fn main() -> Result<()> { 34 | //! 35 | //! let sources = vec![PathBuf::from("src")]; 36 | //! let dest = TempDir::new()?; 37 | //! 38 | //! let config = Arc::new(Config::default()); 39 | //! let updater = ChannelUpdater::new(&config); 40 | //! // The ChannelUpdater is consumed by the driver (so it is properly closed 41 | //! // on completion). Retrieve our end of the connection before then. 42 | //! let stat_rx = updater.rx_channel(); 43 | //! let stats: Arc = Arc::new(updater); 44 | //! 45 | //! let driver = load_driver(Drivers::ParFile, &config)?; 46 | //! 47 | //! // As we want realtime updates via the ChannelUpdater the 48 | //! // copy operation should run in the background. 49 | //! let handle = thread::spawn(move || { 50 | //! driver.copy(sources, dest.path(), stats) 51 | //! }); 52 | //! 53 | //! // Gather the results as we go; our end of the channel has been 54 | //! // moved to the driver call and will end when drained. 55 | //! for stat in stat_rx { 56 | //! match stat { 57 | //! StatusUpdate::Copied(v) => { 58 | //! println!("Copied {} bytes", v); 59 | //! }, 60 | //! StatusUpdate::Size(v) => { 61 | //! println!("Size update: {}", v); 62 | //! }, 63 | //! StatusUpdate::Error(e) => { 64 | //! panic!("Error during copy: {}", e); 65 | //! } 66 | //! } 67 | //! } 68 | //! 69 | //! handle.join() 70 | //! .map_err(|_| XcpError::CopyError("Error during copy operation".to_string()))??; 71 | //! 72 | //! println!("Copy complete"); 73 | //! 74 | //! # Ok(()) 75 | //! # } 76 | //! 77 | //! [xcp]: https://crates.io/crates/xcp/ 78 | 79 | pub mod config; 80 | pub mod drivers; 81 | pub mod errors; 82 | pub mod feedback; 83 | 84 | // Internal 85 | mod backup; 86 | mod operations; 87 | mod paths; 88 | 89 | #[cfg(test)] 90 | #[allow(unused)] 91 | mod tests { 92 | use std::path::PathBuf; 93 | use std::sync::Arc; 94 | use std::thread; 95 | 96 | use tempfile::TempDir; 97 | 98 | use crate::errors::{Result, XcpError}; 99 | use crate::config::Config; 100 | use crate::feedback::{ChannelUpdater, StatusUpdater, StatusUpdate}; 101 | use crate::drivers::{Drivers, load_driver}; 102 | 103 | #[test] 104 | fn simple_usage_test() -> Result<()> { 105 | let sources = vec![PathBuf::from("src")]; 106 | let dest = TempDir::new()?; 107 | 108 | let config = Arc::new(Config::default()); 109 | let updater = ChannelUpdater::new(&config); 110 | let stat_rx = updater.rx_channel(); 111 | let stats: Arc = Arc::new(updater); 112 | 113 | let driver = load_driver(Drivers::ParFile, &config)?; 114 | 115 | let handle = thread::spawn(move || { 116 | driver.copy(sources, dest.path(), stats) 117 | }); 118 | 119 | // Gather the results as we go; our end of the channel has been 120 | // moved to the driver call and will end when drained. 121 | for stat in stat_rx { 122 | match stat { 123 | StatusUpdate::Copied(v) => { 124 | println!("Copied {v} bytes"); 125 | }, 126 | StatusUpdate::Size(v) => { 127 | println!("Size update: {v}"); 128 | }, 129 | StatusUpdate::Error(e) => { 130 | println!("Error during copy: {e}"); 131 | return Err(e.into()); 132 | } 133 | } 134 | } 135 | 136 | handle.join() 137 | .map_err(|_| XcpError::CopyError("Error during copy operation".to_string()))??; 138 | 139 | println!("Copy complete"); 140 | 141 | Ok(()) 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /libxcp/src/drivers/parfile.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | //! Parallelise copying at the file level. This can improve speed on 18 | //! modern NVME devices, but can bottleneck on larger files. 19 | 20 | use crossbeam_channel as cbc; 21 | use log::{debug, error, info}; 22 | use libfs::copy_node; 23 | use std::fs::remove_file; 24 | use std::os::unix::fs::symlink; 25 | use std::path::{Path, PathBuf}; 26 | use std::sync::Arc; 27 | use std::thread; 28 | 29 | use crate::config::Config; 30 | use crate::drivers::CopyDriver; 31 | use crate::errors::{Result, XcpError}; 32 | use crate::feedback::{StatusUpdate, StatusUpdater}; 33 | use crate::operations::{CopyHandle, Operation, tree_walker}; 34 | 35 | // ********************************************************************** // 36 | 37 | pub struct Driver { 38 | config: Arc, 39 | } 40 | 41 | impl Driver { 42 | pub fn new(config: Arc) -> Result { 43 | Ok(Self { 44 | config, 45 | }) 46 | } 47 | } 48 | 49 | impl CopyDriver for Driver { 50 | fn copy(&self, sources: Vec, dest: &Path, stats: Arc) -> Result<()> { 51 | let (work_tx, work_rx) = cbc::unbounded(); 52 | 53 | // Thread which walks the file tree and sends jobs to the 54 | // workers. The worker tx channel is moved to the walker so it is 55 | // closed, which will cause the workers to shutdown on completion. 56 | let walk_worker = { 57 | let sc = stats.clone(); 58 | let d = dest.to_path_buf(); 59 | let o = self.config.clone(); 60 | thread::spawn(move || tree_walker(sources, &d, &o, work_tx, sc)) 61 | }; 62 | 63 | // Worker threads. Will consume work and then shutdown once the 64 | // queue is closed by the walker. 65 | let nworkers = self.config.num_workers(); 66 | let mut joins = Vec::with_capacity(nworkers); 67 | for _ in 0..nworkers { 68 | let copy_worker = { 69 | let wrx = work_rx.clone(); 70 | let sc = stats.clone(); 71 | let conf = self.config.clone(); 72 | thread::spawn(move || copy_worker(wrx, &conf, sc)) 73 | }; 74 | joins.push(copy_worker); 75 | } 76 | 77 | walk_worker.join() 78 | .map_err(|_| XcpError::CopyError("Error walking copy tree".to_string()))??; 79 | for handle in joins { 80 | handle.join() 81 | .map_err(|_| XcpError::CopyError("Error during copy operation".to_string()))??; 82 | } 83 | 84 | Ok(()) 85 | } 86 | 87 | } 88 | 89 | // ********************************************************************** // 90 | 91 | fn copy_worker(work: cbc::Receiver, config: &Arc, updates: Arc) -> Result<()> { 92 | debug!("Starting copy worker {:?}", thread::current().id()); 93 | for op in work { 94 | debug!("Received operation {op:?}"); 95 | 96 | match op { 97 | Operation::Copy(from, to) => { 98 | info!("Worker[{:?}]: Copy {:?} -> {:?}", thread::current().id(), from, to); 99 | // copy_file() sends back its own updates, but we should 100 | // send back any errors as they may have occurred 101 | // before the copy started.. 102 | let r = CopyHandle::new(&from, &to, config) 103 | .and_then(|hdl| hdl.copy_file(&updates)); 104 | if let Err(e) = r { 105 | updates.send(StatusUpdate::Error(XcpError::CopyError(e.to_string())))?; 106 | error!("Error copying: {from:?} -> {to:?}; aborting."); 107 | return Err(e) 108 | } 109 | } 110 | 111 | Operation::Link(from, to) => { 112 | info!("Worker[{:?}]: Symlink {:?} -> {:?}", thread::current().id(), from, to); 113 | let _r = symlink(&from, &to); 114 | } 115 | 116 | Operation::Special(from, to) => { 117 | info!("Worker[{:?}]: Special file {:?} -> {:?}", thread::current().id(), from, to); 118 | if to.exists() { 119 | if config.no_clobber { 120 | return Err(XcpError::DestinationExists("Destination file exists and --no-clobber is set.", to).into()); 121 | } 122 | remove_file(&to)?; 123 | } 124 | copy_node(&from, &to)?; 125 | } 126 | 127 | } 128 | } 129 | debug!("Copy worker {:?} shutting down", thread::current().id()); 130 | Ok(()) 131 | } 132 | -------------------------------------------------------------------------------- /libxcp/src/backup.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | path::{Path, PathBuf}, 3 | env::current_dir, sync::OnceLock, fs::ReadDir, 4 | }; 5 | 6 | use regex::Regex; 7 | 8 | use crate::{errors::{Result, XcpError}, config::{Config, Backup}}; 9 | 10 | const BAK_PATTTERN: &str = r"^\~(\d+)\~$"; 11 | static BAK_REGEX: OnceLock = OnceLock::new(); 12 | 13 | fn get_regex() -> &'static Regex { 14 | // Fixed regex, so should never error. 15 | BAK_REGEX.get_or_init(|| Regex::new(BAK_PATTTERN).unwrap()) 16 | } 17 | 18 | pub(crate) fn get_backup_path(file: &Path) -> Result { 19 | let num = next_backup_num(file)?; 20 | let suffix = format!(".~{num}~"); 21 | // Messy but PathBuf has no concept of mulitiple extensions. 22 | let mut bstr = file.to_path_buf().into_os_string(); 23 | bstr.push(suffix); 24 | let backup = PathBuf::from(bstr); 25 | Ok(backup) 26 | } 27 | 28 | pub(crate) fn needs_backup(file: &Path, conf: &Config) -> Result { 29 | let need = match conf.backup { 30 | Backup::None => false, 31 | Backup::Auto if file.exists() => { 32 | has_backup(file)? 33 | } 34 | Backup::Numbered if file.exists() => true, 35 | _ => false, 36 | }; 37 | Ok(need) 38 | } 39 | 40 | fn ls_file_dir(file: &Path) -> Result { 41 | let cwd = current_dir()?; 42 | let ls_dir = file.parent() 43 | .map(|p| if p.as_os_str().is_empty() { 44 | &cwd 45 | } else { 46 | p 47 | }) 48 | .unwrap_or(&cwd) 49 | .read_dir()?; 50 | Ok(ls_dir) 51 | } 52 | 53 | fn filename(path: &Path) -> Result { 54 | let fname = path.file_name() 55 | .ok_or(XcpError::InvalidArguments(format!("Invalid path found: {path:?}")))? 56 | .to_string_lossy(); 57 | Ok(fname.to_string()) 58 | } 59 | 60 | fn has_backup(file: &Path) -> Result { 61 | let fname = filename(file)?; 62 | let exists = ls_file_dir(file)? 63 | .any(|der| if let Ok(de) = der { 64 | is_num_backup(&fname, &de.path()).is_some() 65 | } else { 66 | false 67 | }); 68 | Ok(exists) 69 | } 70 | 71 | fn next_backup_num(file: &Path) -> Result { 72 | let fname = filename(file)?; 73 | let current = ls_file_dir(file)? 74 | .filter_map(|der| is_num_backup(&fname, &der.ok()?.path())) 75 | .max() 76 | .unwrap_or(0); 77 | Ok(current + 1) 78 | } 79 | 80 | fn is_num_backup(base_file: &str, candidate: &Path) -> Option { 81 | let cname = candidate 82 | .file_name()? 83 | .to_str()?; 84 | if !cname.starts_with(base_file) { 85 | return None 86 | } 87 | let ext = candidate 88 | .extension()? 89 | .to_string_lossy(); 90 | let num = get_regex() 91 | .captures(&ext)? 92 | .get(1)? 93 | .as_str() 94 | .parse::() 95 | .ok()?; 96 | Some(num) 97 | } 98 | 99 | #[cfg(test)] 100 | mod tests { 101 | use super::*; 102 | use std::{path::PathBuf, fs::File}; 103 | use tempfile::TempDir; 104 | 105 | #[test] 106 | fn test_is_backup() { 107 | let cand = PathBuf::from("/some/path/file.txt.~123~"); 108 | 109 | let bnum = is_num_backup("file.txt", &cand); 110 | assert!(bnum.is_some()); 111 | assert_eq!(123, bnum.unwrap()); 112 | 113 | let bnum = is_num_backup("other_file.txt", &cand); 114 | assert!(bnum.is_none()); 115 | 116 | let bnum = is_num_backup("le.txt", &cand); 117 | assert!(bnum.is_none()); 118 | } 119 | 120 | #[test] 121 | fn test_backup_num_scan() -> Result<()> { 122 | let tdir = TempDir::new()?; 123 | let dir = tdir.path(); 124 | let base = dir.join("file.txt"); 125 | 126 | { 127 | File::create(&base)?; 128 | } 129 | let next = next_backup_num(&base)?; 130 | assert_eq!(1, next); 131 | 132 | { 133 | File::create(dir.join("file.txt.~123~"))?; 134 | } 135 | let next = next_backup_num(&base)?; 136 | assert_eq!(124, next); 137 | 138 | { 139 | File::create(dir.join("file.txt.~999~"))?; 140 | } 141 | let next = next_backup_num(&base)?; 142 | assert_eq!(1000, next); 143 | 144 | Ok(()) 145 | } 146 | 147 | #[test] 148 | fn test_gen_backup_path() -> Result<()> { 149 | let tdir = TempDir::new()?; 150 | let dir = tdir.path(); 151 | let base = dir.join("file.txt"); 152 | { 153 | File::create(&base)?; 154 | } 155 | 156 | let backup = get_backup_path(&base)?; 157 | let mut bs = base.into_os_string(); 158 | bs.push(".~1~"); 159 | assert_eq!(PathBuf::from(bs), backup); 160 | 161 | Ok(()) 162 | } 163 | 164 | #[test] 165 | fn test_needs_backup() -> Result<()> { 166 | let tdir = TempDir::new()?; 167 | let dir = tdir.path(); 168 | let base = dir.join("file.txt"); 169 | 170 | { 171 | File::create(&base)?; 172 | } 173 | assert!(!has_backup(&base)?); 174 | 175 | { 176 | File::create(dir.join("file.txt.~123~"))?; 177 | } 178 | assert!(has_backup(&base)?); 179 | 180 | { 181 | File::create(dir.join("file.txt.~999~"))?; 182 | } 183 | assert!(has_backup(&base)?); 184 | 185 | Ok(()) 186 | } 187 | 188 | } 189 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # xcp: An extended cp 2 | 3 | `xcp` is a (partial) clone of the Unix `cp` command. It is not intended as a 4 | full replacement, but as a companion utility with some more user-friendly 5 | feedback and some optimisations that make sense under certain tasks (see 6 | below). 7 | 8 | [![Crates.io](https://img.shields.io/crates/v/xcp.svg?colorA=777777)](https://crates.io/crates/xcp) 9 | ![Github Actions](https://github.com/tarka/xcp/actions/workflows/tests.yml/badge.svg) 10 | [![Packaging status](https://repology.org/badge/tiny-repos/xcp.svg)](https://repology.org/project/xcp/versions) 11 | 12 | *Warning*: `xcp` is currently beta-level software and almost certainly contains 13 | bugs and unexpected or inconsistent behaviour. It probably shouldn't be used for 14 | anything critical yet. 15 | 16 | Please note that there are some known issues with copying files from virtual 17 | filesystems (e.g. `/proc`, `/sys`). See [this LWN 18 | article](https://lwn.net/Articles/846403/) for an overview of some of the 19 | complexities of dealing with kernel-generated files. This is a common problem 20 | with file utilities which rely on random access; for example `rsync` has the 21 | same issue. 22 | 23 | ## Installation 24 | 25 | ### Cargo 26 | 27 | `xcp` can be installed directly from `crates.io` with: 28 | ``` 29 | cargo install xcp 30 | ``` 31 | 32 | ### Arch Linux 33 | 34 | [`xcp`](https://aur.archlinux.org/packages/xcp/) is available on the Arch Linux User Repository. If you use an AUR helper, you can execute a command such as this: 35 | ``` 36 | yay -S xcp 37 | ``` 38 | 39 | ### NetBSD 40 | [`xcp`](https://pkgsrc.se/sysutils/xcp) is available on NetBSD from the official repositories. To install it, simply run: 41 | ``` 42 | pkgin install xcp 43 | ``` 44 | 45 | ## Features and Anti-Features 46 | 47 | ### Features 48 | 49 | * Displays a progress-bar, both for directory and single file copies. This can 50 | be disabled with `--no-progress`. 51 | * On Linux it uses `copy_file_range` call to copy files. This is the most 52 | efficient method of file-copying under Linux; in particular it is 53 | filesystem-aware, and can massively speed-up copies on network mounts by 54 | performing the copy operations server-side. However, unlike `copy_file_range` 55 | sparse files are detected and handled appropriately. 56 | * Support for modern filesystem features such as [reflinks](https://btrfs.readthedocs.io/en/latest/Reflink.html). 57 | * Optimised for 'modern' systems (i.e. multiple cores, copious RAM, and 58 | solid-state disks, especially ones connected into the main system bus, 59 | e.g. NVMe). 60 | * Optional aggressive parallelism for systems with parallel IO. Quick 61 | experiments on a modern laptop suggest there may be benefits to parallel 62 | copies on NVMe disks. This is obviously highly system-dependent. 63 | * Switchable 'drivers' to facilitate experimenting with alternative strategies 64 | for copy optimisation. Currently 2 drivers are available: 65 | * 'parfile': the previous hard-coded xcp copy method, which parallelises 66 | tree-walking and per-file copying. This is the default. 67 | * 'parblock': An experimental driver that parallelises copying at the block 68 | level. This has the potential for performance improvements in some 69 | architectures, but increases complexity. Testing is welcome. 70 | * Non-Linux Unix-like OSs (OS X, *BSD) are supported via fall-back operation 71 | (although sparse-files are not yet supported in this case). 72 | * Optionally understands `.gitignore` files to limit the copied directories. 73 | * Optional native file-globbing. 74 | 75 | ### (Possible) future features 76 | 77 | * Conversion of files to sparse where appropriate, as with `cp`'s 78 | `--sparse=always` flag. 79 | * Aggressive sparseness detection with `lseek`. 80 | * On non-Linux OSs sparse-files are not currenty supported but could be added if 81 | supported by the OS. 82 | 83 | ### Differences with `cp` 84 | 85 | * Permissions, xattrs and ACLs are copied by default; this can be disabled with 86 | `--no-perms`. 87 | * Virtual file copies are not supported; for example `/proc` and `/sys` files. 88 | * Character files such as [sockets](https://man7.org/linux/man-pages/man7/unix.7.html) and 89 | [pipes](https://man7.org/linux/man-pages/man3/mkfifo.3.html) are copied as 90 | devices (i.e. via [mknod](https://man7.org/linux/man-pages/man2/mknod.2.html)) 91 | rather than copying their contents as a stream. 92 | * The `--reflink=never` option may silently perform a reflink operation 93 | regardless. This is due to the use of 94 | [copy_file_range](https://man7.org/linux/man-pages/man2/copy_file_range.2.html) 95 | which has no such override and may perform its own optimisations. 96 | * `cp` 'simple' backups are not supported, only numbered. 97 | * Some `cp` options are not available but may be added in the future. 98 | 99 | ## Performance 100 | 101 | Benchmarks are mostly meaningless, but the following are results from a laptop 102 | with an NVMe disk and in single-user mode. The target copy directory is a git 103 | checkout of the Firefox codebase, having been recently gc'd (i.e. a single 4.1GB 104 | pack file). `fstrim -va` and `echo 3 | sudo tee /proc/sys/vm/drop_caches` are 105 | run before each test run to minimise SSD allocation performance interference. 106 | 107 | Note: `xcp` is optimised for 'modern' systems with lots of RAM and solid-state 108 | disks. In particular it is likely to perform worse on spinning disks unless they 109 | are in highly parallel arrays. 110 | 111 | ### Local copy 112 | 113 | * Single 4.1GB file copy, with the kernel cache dropped each run: 114 | * `cp`: ~6.2s 115 | * `xcp`: ~4.2s 116 | * Single 4.1GB file copy, warmed cache (3 runs each): 117 | * `cp`: ~1.85s 118 | * `xcp`: ~1.7s 119 | * Directory copy, kernel cache dropped each run: 120 | * `cp`: ~48s 121 | * `xcp`: ~56s 122 | * Directory copy, warmed cache (3 runs each): 123 | * `cp`: ~6.9s 124 | * `xcp`: ~7.4s 125 | 126 | ### NFS copy 127 | 128 | `xcp` uses `copy_file_range`, which is filesystem aware. On NFSv4 this will result 129 | in the copy occurring server-side rather than transferring across the network. For 130 | large files this can be a significant win: 131 | 132 | * Single 4.1GB file on NFSv4 mount 133 | * `cp`: 6m18s 134 | * `xcp`: 0m37s 135 | -------------------------------------------------------------------------------- /libxcp/src/config.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2024, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | //! Driver configuration support. 18 | 19 | use std::result; 20 | use std::str::FromStr; 21 | 22 | use crate::errors::XcpError; 23 | 24 | /// Enum defining configuration options for handling 25 | /// [reflinks](https://btrfs.readthedocs.io/en/latest/Reflink.html). [FromStr] 26 | /// is supported. 27 | #[derive(Clone, Copy, Debug, Default, PartialEq)] 28 | pub enum Reflink { 29 | /// Attempt to reflink and fallback to a copy if it is not 30 | /// possible. 31 | #[default] 32 | Auto, 33 | /// Always attempt a reflink; return an error if not supported. 34 | Always, 35 | /// Always perform a full data copy. Note: when using Linux 36 | /// accelerated copy operations (the default when available) the 37 | /// kernel may choose to reflink rather than perform a fully copy 38 | /// regardless of this setting. 39 | Never, 40 | } 41 | 42 | // String conversion helper as a convenience for command-line parsing. 43 | impl FromStr for Reflink { 44 | type Err = XcpError; 45 | 46 | fn from_str(s: &str) -> result::Result { 47 | match s.to_lowercase().as_str() { 48 | "always" => Ok(Reflink::Always), 49 | "auto" => Ok(Reflink::Auto), 50 | "never" => Ok(Reflink::Never), 51 | _ => Err(XcpError::InvalidArguments(format!("Unexpected value for 'reflink': {s}"))), 52 | } 53 | } 54 | } 55 | 56 | /// Enum defining configuration options for handling backups of 57 | /// overwritten files. [FromStr] is supported. 58 | #[derive(Clone, Copy, Debug, PartialEq)] 59 | pub enum Backup { 60 | /// Do not create backups. 61 | None, 62 | /// Create a numbered backup if a previous backup exists. 63 | Auto, 64 | /// Create numbered backups. Numbered backups follow the semantics 65 | /// of `cp` numbered backups (e.g. `file.txt.~123~`). 66 | Numbered, 67 | } 68 | 69 | impl FromStr for Backup { 70 | type Err = XcpError; 71 | 72 | fn from_str(s: &str) -> result::Result { 73 | match s.to_lowercase().as_str() { 74 | "none" | "off" => Ok(Backup::None), 75 | "auto" => Ok(Backup::Auto), 76 | "numbered" => Ok(Backup::Numbered), 77 | _ => Err(XcpError::InvalidArguments(format!("Unexpected value for 'backup': {s}"))), 78 | } 79 | } 80 | } 81 | 82 | /// A structure defining the runtime options for copy-drivers. This 83 | /// would normally be passed to `load_driver()`. 84 | #[derive(Clone, Debug)] 85 | pub struct Config { 86 | /// Number of parallel workers. 0 means use the number of logical 87 | /// CPUs (the default). 88 | pub workers: usize, 89 | 90 | /// Block size for operations. Defaults to the full file size. Use 91 | /// a smaller value for finer-grained feedback. 92 | pub block_size: u64, 93 | 94 | /// Use .gitignore if present. 95 | /// 96 | /// NOTE: This is fairly basic at the moment, and only honours a 97 | /// .gitignore in the directory root for each source directory; 98 | /// global or sub-directory ignores are skipped. Default is 99 | /// `false`. 100 | pub gitignore: bool, 101 | 102 | /// Do not overwrite existing files. Default is `false`. 103 | pub no_clobber: bool, 104 | 105 | /// Do not copy the file permissions. Default is `false`. 106 | pub no_perms: bool, 107 | 108 | /// Do not copy the file permissions. Default is `false`. 109 | pub no_timestamps: bool, 110 | 111 | /// Copy ownership. 112 | /// 113 | /// Whether to copy ownship (user/group). This option requires 114 | /// root permissions or appropriate capabilities; if the attempt 115 | /// to copy ownership fails a warning is issued but the operation 116 | /// continues. 117 | pub ownership: bool, 118 | 119 | /// Dereference symlinks. Default is `false`. 120 | pub dereference: bool, 121 | 122 | /// Target should not be a directory. 123 | /// 124 | /// Analogous to cp's no-target-directory. Expected behavior is that when 125 | /// copying a directory to another directory, instead of creating a sub-folder 126 | /// in target, overwrite target. Default is 'false`. 127 | pub no_target_directory: bool, 128 | 129 | /// Sync each file to disk after writing. Default is `false`. 130 | pub fsync: bool, 131 | 132 | /// Reflink options. 133 | /// 134 | /// Whether and how to use reflinks. 'auto' (the default) will 135 | /// attempt to reflink and fallback to a copy if it is not 136 | /// possible, 'always' will return an error if it cannot reflink, 137 | /// and 'never' will always perform a full data copy. 138 | pub reflink: Reflink, 139 | 140 | /// Backup options 141 | /// 142 | /// Whether to create backups of overwritten files. Current 143 | /// options are `None` or 'Numbered'. Numbered backups follow the 144 | /// semantics of `cp` numbered backups 145 | /// (e.g. `file.txt.~123~`). Default is `None`. 146 | pub backup: Backup, 147 | } 148 | 149 | impl Config { 150 | pub(crate) fn num_workers(&self) -> usize { 151 | if self.workers == 0 { 152 | num_cpus::get() 153 | } else { 154 | self.workers 155 | } 156 | } 157 | } 158 | 159 | impl Default for Config { 160 | fn default() -> Self { 161 | Config { 162 | workers: num_cpus::get(), 163 | block_size: u64::MAX, 164 | gitignore: false, 165 | no_clobber: false, 166 | no_perms: false, 167 | no_timestamps: false, 168 | ownership: false, 169 | dereference: false, 170 | no_target_directory: false, 171 | fsync: false, 172 | reflink: Reflink::Auto, 173 | backup: Backup::None, 174 | } 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | mod options; 18 | mod progress; 19 | 20 | use std::path::PathBuf; 21 | use std::{result, thread}; 22 | use std::sync::Arc; 23 | 24 | use glob::{glob, Paths}; 25 | use libxcp::config::{Config, Reflink}; 26 | use libxcp::drivers::load_driver; 27 | use libxcp::errors::{Result, XcpError}; 28 | use libxcp::feedback::{ChannelUpdater, StatusUpdate, StatusUpdater}; 29 | use log::{error, info, warn}; 30 | 31 | use crate::options::Opts; 32 | 33 | fn init_logging(opts: &Opts) -> Result<()> { 34 | use simplelog::{ColorChoice, Config, SimpleLogger, TermLogger, TerminalMode}; 35 | 36 | TermLogger::init( 37 | opts.log_level(), 38 | Config::default(), 39 | TerminalMode::Mixed, 40 | ColorChoice::Auto, 41 | ).or_else( 42 | |_| SimpleLogger::init(opts.log_level(), Config::default()) 43 | )?; 44 | 45 | Ok(()) 46 | } 47 | 48 | // Expand a list of file-paths or glob-patterns into a list of concrete paths. 49 | // FIXME: This currently eats non-existent files that are not 50 | // globs. Should we convert empty glob results into errors? 51 | fn expand_globs(patterns: &[String]) -> Result> { 52 | let paths = patterns.iter() 53 | .map(|s| glob(s.as_str())) 54 | .collect::, _>>()? 55 | .iter_mut() 56 | // Force resolve each glob Paths iterator into a vector of the results... 57 | .map::, _>, _>(Iterator::collect) 58 | // And lift all the results up to the top. 59 | .collect::>, _>>()? 60 | .iter() 61 | .flat_map(ToOwned::to_owned) 62 | .collect::>(); 63 | 64 | Ok(paths) 65 | } 66 | 67 | fn expand_sources(source_list: &[String], opts: &Opts) -> Result> { 68 | if opts.glob { 69 | expand_globs(source_list) 70 | } else { 71 | let pb = source_list.iter() 72 | .map(PathBuf::from) 73 | .collect::>(); 74 | Ok(pb) 75 | } 76 | } 77 | 78 | fn opts_check(opts: &Opts) -> Result<()> { 79 | #[cfg(any(target_os = "linux", target_os = "android"))] 80 | if opts.reflink == Reflink::Never { 81 | warn!("--reflink=never is selected, however the Linux kernel may override this."); 82 | } 83 | 84 | if opts.no_clobber && opts.force { 85 | return Err(XcpError::InvalidArguments("--force and --noclobber cannot be set at the same time.".to_string()).into()); 86 | } 87 | Ok(()) 88 | } 89 | 90 | fn main() -> Result<()> { 91 | let opts = Opts::from_args()?; 92 | init_logging(&opts)?; 93 | opts_check(&opts)?; 94 | 95 | let (dest, source_patterns) = match opts.target_directory { 96 | Some(ref d) => { (d, opts.paths.as_slice()) } 97 | None => { 98 | opts.paths.split_last().ok_or(XcpError::InvalidArguments("Insufficient arguments".to_string()))? 99 | } 100 | }; 101 | let dest = PathBuf::from(dest); 102 | 103 | let sources = expand_sources(source_patterns, &opts)?; 104 | if sources.is_empty() { 105 | return Err(XcpError::InvalidSource("No source files found.").into()); 106 | } else if !dest.is_dir() { 107 | if sources.len() == 1 && sources[0].is_dir() && dest.exists() { 108 | return Err(XcpError::InvalidDestination("Cannot copy a directory to a file.").into()); 109 | } else if sources.len() > 1 { 110 | return Err(XcpError::InvalidDestination("Multiple sources and destination is not a directory.").into()); 111 | } 112 | } 113 | 114 | // Sanity-check all sources up-front 115 | for source in &sources { 116 | info!("Copying source {source:?} to {dest:?}"); 117 | if !source.exists() { 118 | return Err(XcpError::InvalidSource("Source does not exist.").into()); 119 | } 120 | 121 | if source.is_dir() && !opts.recursive { 122 | return Err(XcpError::InvalidSource("Source is directory and --recursive not specified.").into()); 123 | } 124 | if source == &dest { 125 | return Err(XcpError::InvalidSource("Cannot copy a directory into itself").into()); 126 | } 127 | 128 | let sourcedir = source 129 | .components() 130 | .next_back() 131 | .ok_or(XcpError::InvalidSource("Failed to find source directory name."))?; 132 | 133 | let target_base = if dest.exists() && dest.is_dir() && !opts.no_target_directory { 134 | dest.join(sourcedir) 135 | } else { 136 | dest.to_path_buf() 137 | }; 138 | 139 | if source == &target_base { 140 | return Err(XcpError::InvalidSource("Source is same as destination").into()); 141 | } 142 | } 143 | 144 | 145 | // ========== Start copy ============ 146 | 147 | let config = Arc::new(Config::from(&opts)); 148 | let driver = load_driver(opts.driver, &config)?; 149 | 150 | let updater = ChannelUpdater::new(&config); 151 | let stat_rx = updater.rx_channel(); 152 | let stats: Arc = Arc::new(updater); 153 | 154 | let handle = thread::spawn(move || -> Result<()> { 155 | driver.copy(sources, &dest, stats) 156 | }); 157 | 158 | 159 | // ========== Collect output and display ============ 160 | 161 | let pb = progress::create_bar(&opts, 0)?; 162 | 163 | // Gather the results as we go; our end of the channel has been 164 | // moved to the driver call and will end when drained. 165 | for stat in stat_rx { 166 | match stat { 167 | StatusUpdate::Copied(v) => pb.inc(v), 168 | StatusUpdate::Size(v) => pb.inc_size(v), 169 | StatusUpdate::Error(e) => { 170 | // FIXME: Optional continue? 171 | error!("Received error: {e}"); 172 | return Err(e.into()); 173 | } 174 | } 175 | } 176 | 177 | handle.join() 178 | .map_err(|_| XcpError::CopyError("Error during copy operation".to_string()))??; 179 | 180 | info!("Copy complete"); 181 | pb.end(); 182 | 183 | Ok(()) 184 | } 185 | -------------------------------------------------------------------------------- /src/options.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | use clap::{ArgAction, Parser}; 18 | 19 | use libxcp::config::{Backup, Config, Reflink}; 20 | use log::LevelFilter; 21 | use unbytify::unbytify; 22 | 23 | use libxcp::drivers::Drivers; 24 | use libxcp::errors::Result; 25 | 26 | #[derive(Clone, Debug, Parser)] 27 | #[command( 28 | name = "xcp", 29 | about = "A (partial) clone of the Unix `cp` command with progress and pluggable drivers.", 30 | version, 31 | )] 32 | pub struct Opts { 33 | /// Verbosity. 34 | /// 35 | /// Can be specified multiple times to increase logging. 36 | #[arg(short, long, action = ArgAction::Count)] 37 | pub verbose: u8, 38 | 39 | /// Copy directories recursively 40 | #[arg(short, long)] 41 | pub recursive: bool, 42 | 43 | /// Dereference symlinks in source 44 | /// 45 | /// Follow symlinks, possibly recursively, when copying source 46 | /// files. 47 | #[arg(short = 'L', long)] 48 | pub dereference: bool, 49 | 50 | /// Number of parallel workers. 51 | /// 52 | /// Default is 4; if the value is negative or 0 it uses the number 53 | /// of logical CPUs. 54 | #[arg(short, long, default_value = "4")] 55 | pub workers: usize, 56 | 57 | /// Block size for operations. 58 | /// 59 | /// Accepts standard size modifiers like "M" and "GB". Actual 60 | /// usage internally depends on the driver. 61 | #[arg(long, default_value = "1MB", value_parser=unbytify)] 62 | pub block_size: u64, 63 | 64 | /// Do not overwrite an existing file 65 | #[arg(short, long)] 66 | pub no_clobber: bool, 67 | 68 | /// Force (compatability only) 69 | /// 70 | /// Overwrite files; this is the default behaviour, this flag is 71 | /// for compatibility with `cp` only. See `--no-clobber` for the 72 | /// inverse flag. Using this in conjunction with `--no-clobber` 73 | /// will cause an error. 74 | #[arg(short = 'f', long = "force")] 75 | pub force: bool, 76 | 77 | /// Use .gitignore if present. 78 | /// 79 | /// NOTE: This is fairly basic at the moment, and only honours a 80 | /// .gitignore in the directory root for directory copies; global 81 | /// or sub-directory ignores are skipped. 82 | #[arg(long)] 83 | pub gitignore: bool, 84 | 85 | /// Expand file patterns. 86 | /// 87 | /// Glob (expand) filename patterns natively (note; the shell may still do its own expansion first) 88 | #[arg(short, long)] 89 | pub glob: bool, 90 | 91 | /// Disable progress bar. 92 | #[arg(long)] 93 | pub no_progress: bool, 94 | 95 | /// Do not copy the file permissions. 96 | #[arg(long)] 97 | pub no_perms: bool, 98 | 99 | /// Do not copy the file timestamps. 100 | #[arg(long)] 101 | pub no_timestamps: bool, 102 | 103 | /// Copy ownership. 104 | /// 105 | /// Whether to copy ownship (user/group). This option requires 106 | /// root permissions or appropriate capabilities; if the attempt 107 | /// to copy ownership fails a warning is issued but the operation 108 | /// continues. 109 | #[arg(short, long)] 110 | pub ownership: bool, 111 | 112 | /// Driver to use, defaults to 'file-parallel'. 113 | /// 114 | /// Currently there are 2; the default "parfile", which 115 | /// parallelises copies across workers at the file level, and an 116 | /// experimental "parblock" driver, which parellelises at the 117 | /// block level. See also '--block-size'. 118 | #[arg(long, default_value = "parfile")] 119 | pub driver: Drivers, 120 | 121 | /// Target should not be a directory. 122 | /// 123 | /// Analogous to cp's no-target-directory. Expected behavior is that when 124 | /// copying a directory to another directory, instead of creating a sub-folder 125 | /// in target, overwrite target. 126 | #[arg(short = 'T', long)] 127 | pub no_target_directory: bool, 128 | 129 | /// Copy into a subdirectory of the target 130 | #[arg(long)] 131 | pub target_directory: Option, 132 | 133 | /// Sync each file to disk after writing. 134 | #[arg(long)] 135 | pub fsync: bool, 136 | 137 | /// Reflink options. 138 | /// 139 | /// Whether and how to use reflinks. 'auto' (the default) will 140 | /// attempt to reflink and fallback to a copy if it is not 141 | /// possible, 'always' will return an error if it cannot reflink, 142 | /// and 'never' will always perform a full data copy. 143 | /// 144 | /// Note: when using Linux accelerated copy operations (the 145 | /// default when available) the kernel may choose to reflink 146 | /// rather than perform a fully copy regardless of this setting. 147 | #[arg(long, default_value = "auto")] 148 | pub reflink: Reflink, 149 | 150 | /// Backup options 151 | /// 152 | /// Whether to create backups of overwritten files. Current 153 | /// options are 'none'/'off', or 'numbered', or 'auto'. Numbered 154 | /// backups follow the semantics of `cp` numbered backups 155 | /// (e.g. `file.txt.~123~`). 'auto' will only create a numbered 156 | /// backup if a previous backups exists. Default is 'none'. 157 | #[arg(long, default_value = "none")] 158 | pub backup: Backup, 159 | 160 | /// Path list. 161 | /// 162 | /// Source and destination files, or multiple source(s) to a directory. 163 | pub paths: Vec, 164 | } 165 | 166 | impl Opts { 167 | pub fn from_args() -> Result { 168 | Ok(Opts::parse()) 169 | } 170 | 171 | pub fn log_level(&self) -> LevelFilter { 172 | match self.verbose { 173 | 0 => LevelFilter::Warn, 174 | 1 => LevelFilter::Info, 175 | 2 => LevelFilter::Debug, 176 | _ => LevelFilter::Trace, 177 | } 178 | } 179 | } 180 | 181 | impl From<&Opts> for Config { 182 | fn from(opts: &Opts) -> Self { 183 | Config { 184 | workers: if opts.workers == 0 { 185 | num_cpus::get() 186 | } else { 187 | opts.workers 188 | }, 189 | block_size: if opts.no_progress { 190 | usize::MAX as u64 191 | } else { 192 | opts.block_size 193 | }, 194 | gitignore: opts.gitignore, 195 | no_clobber: opts.no_clobber, 196 | no_perms: opts.no_perms, 197 | no_timestamps: opts.no_timestamps, 198 | ownership: opts.ownership, 199 | dereference: opts.dereference, 200 | no_target_directory: opts.no_target_directory, 201 | fsync: opts.fsync, 202 | reflink: opts.reflink, 203 | backup: opts.backup, 204 | } 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /libxcp/src/drivers/parblock.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | //! Parallelise copying at the block level. Block-size is 18 | //! configurable. This can have better performance for large files, 19 | //! but has a higher overhead. 20 | 21 | use std::cmp; 22 | use std::fs::remove_file; 23 | use std::ops::Range; 24 | use std::os::unix::fs::symlink; 25 | use std::path::{Path, PathBuf}; 26 | use std::sync::Arc; 27 | use std::thread; 28 | 29 | use cfg_if::cfg_if; 30 | use crossbeam_channel as cbc; 31 | use libfs::copy_node; 32 | use log::{error, info}; 33 | use blocking_threadpool::{Builder, ThreadPool}; 34 | 35 | use crate::config::Config; 36 | use crate::drivers::CopyDriver; 37 | use crate::errors::{Result, XcpError}; 38 | use crate::feedback::{StatusUpdate, StatusUpdater}; 39 | use crate::operations::{CopyHandle, Operation, tree_walker}; 40 | use libfs::{copy_file_offset, map_extents, merge_extents, probably_sparse}; 41 | 42 | // ********************************************************************** // 43 | 44 | const fn supported_platform() -> bool { 45 | cfg_if! { 46 | if #[cfg( 47 | any(target_os = "linux", 48 | target_os = "android", 49 | target_os = "freebsd", 50 | target_os = "netbsd", 51 | target_os = "dragonfly", 52 | target_os = "macos", 53 | ))] 54 | { 55 | true 56 | } else { 57 | false 58 | } 59 | } 60 | } 61 | 62 | 63 | pub struct Driver { 64 | config: Arc, 65 | } 66 | 67 | impl Driver { 68 | pub fn new(config: Arc) -> Result { 69 | if !supported_platform() { 70 | let msg = "The parblock driver is not currently supported on this OS."; 71 | error!("{msg}"); 72 | return Err(XcpError::UnsupportedOS(msg).into()); 73 | } 74 | 75 | Ok(Self { 76 | config, 77 | }) 78 | } 79 | } 80 | 81 | impl CopyDriver for Driver { 82 | fn copy(&self, sources: Vec, dest: &Path, stats: Arc) -> Result<()> { 83 | let (file_tx, file_rx) = cbc::unbounded::(); 84 | 85 | // Start (single) dispatch worker 86 | let dispatcher = { 87 | let q_config = self.config.clone(); 88 | let st = stats.clone(); 89 | thread::spawn(move || dispatch_worker(file_rx, &st, q_config)) 90 | }; 91 | 92 | // Thread which walks the file tree and sends jobs to the 93 | // workers. The worker tx channel is moved to the walker so it is 94 | // closed, which will cause the workers to shutdown on completion. 95 | let walk_worker = { 96 | let sc = stats.clone(); 97 | let d = dest.to_path_buf(); 98 | let c = self.config.clone(); 99 | thread::spawn(move || tree_walker(sources, &d, &c, file_tx, sc)) 100 | }; 101 | 102 | walk_worker.join() 103 | .map_err(|_| XcpError::CopyError("Error walking copy tree".to_string()))??; 104 | dispatcher.join() 105 | .map_err(|_| XcpError::CopyError("Error dispatching copy operation".to_string()))??; 106 | 107 | Ok(()) 108 | } 109 | } 110 | 111 | // ********************************************************************** // 112 | 113 | fn queue_file_range( 114 | handle: &Arc, 115 | range: Range, 116 | pool: &ThreadPool, 117 | status_channel: &Arc, 118 | ) -> Result { 119 | let len = range.end - range.start; 120 | let bsize = handle.config.block_size; 121 | let blocks = (len / bsize) + (if len % bsize > 0 { 1 } else { 0 }); 122 | 123 | for blkn in 0..blocks { 124 | let harc = handle.clone(); 125 | let stat_tx = status_channel.clone(); 126 | let bytes = cmp::min(len - (blkn * bsize), bsize); 127 | let off = range.start + (blkn * bsize); 128 | 129 | pool.execute(move || { 130 | let copy_result = copy_file_offset(&harc.infd, &harc.outfd, bytes, off as i64); 131 | let stat_result = match copy_result { 132 | Ok(bytes) => { 133 | stat_tx.send(StatusUpdate::Copied(bytes as u64)) 134 | } 135 | Err(e) => { 136 | error!("Error copying: aborting."); 137 | stat_tx.send(StatusUpdate::Error(XcpError::CopyError(e.to_string()))) 138 | } 139 | }; 140 | if let Err(e) = stat_result { 141 | let msg = format!("Failed to send status update message. This should not happen; aborting. Error: {e}"); 142 | error!("{msg}"); 143 | panic!("{}", msg); 144 | } 145 | }); 146 | } 147 | Ok(len) 148 | } 149 | 150 | fn queue_file_blocks( 151 | source: &Path, 152 | dest: &Path, 153 | pool: &ThreadPool, 154 | status_channel: &Arc, 155 | config: &Arc, 156 | ) -> Result { 157 | let handle = CopyHandle::new(source, dest, config)?; 158 | let len = handle.metadata.len(); 159 | 160 | if handle.try_reflink()? { 161 | info!("Reflinked, skipping rest of copy"); 162 | return Ok(len); 163 | } 164 | 165 | // Put the open files in an Arc, which we drop once work has been 166 | // queued. This will keep the files open until all work has been 167 | // consumed, then close them. (This may be overkill; opening the 168 | // files in the workers would also be valid.) 169 | let harc = Arc::new(handle); 170 | 171 | let queue_whole_file = || { 172 | queue_file_range(&harc, 0..len, pool, status_channel) 173 | }; 174 | 175 | if probably_sparse(&harc.infd)? { 176 | if let Some(extents) = map_extents(&harc.infd)? { 177 | let sparse_map = merge_extents(extents)?; 178 | let mut queued = 0; 179 | for ext in sparse_map { 180 | queued += queue_file_range(&harc, ext.into(), pool, status_channel)?; 181 | } 182 | Ok(queued) 183 | } else { 184 | queue_whole_file() 185 | } 186 | } else { 187 | queue_whole_file() 188 | } 189 | } 190 | 191 | // Dispatch worker; receives queued files and hands them to 192 | // queue_file_blocks() which splits them onto the copy-pool. 193 | fn dispatch_worker(file_q: cbc::Receiver, stats: &Arc, config: Arc) -> Result<()> { 194 | let nworkers = config.num_workers(); 195 | let copy_pool = Builder::new() 196 | .num_threads(nworkers) 197 | // Use bounded queue for backpressure; this limits open 198 | // files in-flight so we don't run out of file handles. 199 | // FIXME: Number is arbitrary ATM, we should be able to 200 | // calculate it from ulimits. 201 | .queue_len(128) 202 | .build(); 203 | for op in file_q { 204 | match op { 205 | Operation::Copy(from, to) => { 206 | info!("Dispatch[{:?}]: Copy {:?} -> {:?}", thread::current().id(), from, to); 207 | let r = queue_file_blocks(&from, &to, ©_pool, stats, &config); 208 | if let Err(e) = r { 209 | stats.send(StatusUpdate::Error(XcpError::CopyError(e.to_string())))?; 210 | error!("Dispatcher: Error copying {from:?} -> {to:?}."); 211 | return Err(e) 212 | } 213 | } 214 | 215 | // Inline the following operations as the should be near-instant. 216 | Operation::Link(from, to) => { 217 | info!("Dispatch[{:?}]: Symlink {:?} -> {:?}", thread::current().id(), from, to); 218 | let r = symlink(&from, &to); 219 | if let Err(e) = r { 220 | stats.send(StatusUpdate::Error(XcpError::CopyError(e.to_string())))?; 221 | error!("Error symlinking: {from:?} -> {to:?}; aborting."); 222 | return Err(e.into()) 223 | } 224 | } 225 | 226 | Operation::Special(from, to) => { 227 | info!("Dispatch[{:?}]: Special file {:?} -> {:?}", thread::current().id(), from, to); 228 | if to.exists() { 229 | if config.no_clobber { 230 | return Err(XcpError::DestinationExists("Destination file exists and --no-clobber is set.", to).into()); 231 | } 232 | remove_file(&to)?; 233 | } 234 | copy_node(&from, &to)?; 235 | } 236 | } 237 | } 238 | info!("Queuing complete"); 239 | 240 | copy_pool.join(); 241 | info!("Pool complete"); 242 | 243 | Ok(()) 244 | } 245 | -------------------------------------------------------------------------------- /libxcp/src/operations.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | use std::os::unix::fs::{chown, MetadataExt}; 18 | use std::{cmp, thread}; 19 | use std::fs::{self, canonicalize, create_dir_all, read_link, File, Metadata}; 20 | use std::path::{Path, PathBuf}; 21 | use std::sync::Arc; 22 | 23 | use crossbeam_channel as cbc; 24 | use libfs::{ 25 | allocate_file, copy_file_bytes, copy_owner, copy_permissions, copy_timestamps, next_sparse_segments, probably_sparse, reflink, sync, FileType 26 | }; 27 | use log::{debug, error, info, warn}; 28 | use walkdir::WalkDir; 29 | 30 | use crate::backup::{get_backup_path, needs_backup}; 31 | use crate::config::{Config, Reflink}; 32 | use crate::errors::{Result, XcpError}; 33 | use crate::feedback::{StatusUpdate, StatusUpdater}; 34 | use crate::paths::{parse_ignore, ignore_filter}; 35 | 36 | #[derive(Debug)] 37 | pub struct CopyHandle { 38 | pub infd: File, 39 | pub outfd: File, 40 | pub metadata: Metadata, 41 | pub config: Arc, 42 | } 43 | 44 | impl CopyHandle { 45 | pub fn new(from: &Path, to: &Path, config: &Arc) -> Result { 46 | let infd = File::open(from)?; 47 | let metadata = infd.metadata()?; 48 | 49 | if needs_backup(to, config)? { 50 | let backup = get_backup_path(to)?; 51 | info!("Backup: Rename {to:?} to {backup:?}"); 52 | fs::rename(to, backup)?; 53 | } 54 | 55 | let outfd = File::create(to)?; 56 | allocate_file(&outfd, metadata.len())?; 57 | 58 | let handle = CopyHandle { 59 | infd, 60 | outfd, 61 | metadata, 62 | config: config.clone(), 63 | }; 64 | 65 | Ok(handle) 66 | } 67 | 68 | /// Copy len bytes from wherever the descriptor cursors are set. 69 | fn copy_bytes(&self, len: u64, updates: &Arc) -> Result { 70 | let mut written = 0; 71 | while written < len { 72 | let bytes_to_copy = cmp::min(len - written, self.config.block_size); 73 | let bytes = copy_file_bytes(&self.infd, &self.outfd, bytes_to_copy)? as u64; 74 | written += bytes; 75 | updates.send(StatusUpdate::Copied(bytes))?; 76 | } 77 | 78 | Ok(written) 79 | } 80 | 81 | /// Wrapper around copy_bytes that looks for sparse blocks and skips them. 82 | fn copy_sparse(&self, updates: &Arc) -> Result { 83 | let len = self.metadata.len(); 84 | let mut pos = 0; 85 | 86 | while pos < len { 87 | let (next_data, next_hole) = next_sparse_segments(&self.infd, &self.outfd, pos)?; 88 | 89 | let _written = self.copy_bytes(next_hole - next_data, updates)?; 90 | pos = next_hole; 91 | } 92 | 93 | Ok(len) 94 | } 95 | 96 | pub fn try_reflink(&self) -> Result { 97 | match self.config.reflink { 98 | Reflink::Always | Reflink::Auto => { 99 | debug!("Attempting reflink from {:?}->{:?}", self.infd, self.outfd); 100 | let worked = reflink(&self.infd, &self.outfd)?; 101 | if worked { 102 | debug!("Reflink {:?} succeeded", self.outfd); 103 | Ok(true) 104 | } else if self.config.reflink == Reflink::Always { 105 | Err(XcpError::ReflinkFailed(format!("{:?}->{:?}", self.infd, self.outfd)).into()) 106 | } else { 107 | debug!("Failed to reflink, falling back to copy"); 108 | Ok(false) 109 | } 110 | } 111 | 112 | Reflink::Never => { 113 | Ok(false) 114 | } 115 | } 116 | } 117 | 118 | pub fn copy_file(&self, updates: &Arc) -> Result { 119 | if self.try_reflink()? { 120 | return Ok(self.metadata.len()); 121 | } 122 | let total = if probably_sparse(&self.infd)? { 123 | self.copy_sparse(updates)? 124 | } else { 125 | self.copy_bytes(self.metadata.len(), updates)? 126 | }; 127 | 128 | Ok(total) 129 | } 130 | 131 | fn finalise_copy(&self) -> Result<()> { 132 | if !self.config.no_perms { 133 | copy_permissions(&self.infd, &self.outfd)?; 134 | } 135 | if !self.config.no_timestamps { 136 | copy_timestamps(&self.infd, &self.outfd)?; 137 | } 138 | if self.config.ownership && copy_owner(&self.infd, &self.outfd).is_err() { 139 | warn!("Failed to copy file ownership: {:?}", self.infd); 140 | } 141 | if self.config.fsync { 142 | debug!("Syncing file {:?}", self.outfd); 143 | sync(&self.outfd)?; 144 | } 145 | Ok(()) 146 | } 147 | } 148 | 149 | impl Drop for CopyHandle { 150 | fn drop(&mut self) { 151 | // FIXME: Should we check for panicking() here? 152 | if let Err(e) = self.finalise_copy() { 153 | error!("Error during finalising copy operation {:?} -> {:?}: {}", self.infd, self.outfd, e); 154 | } 155 | } 156 | } 157 | 158 | #[derive(Debug)] 159 | pub enum Operation { 160 | Copy(PathBuf, PathBuf), 161 | Link(PathBuf, PathBuf), 162 | Special(PathBuf, PathBuf), 163 | } 164 | 165 | pub fn tree_walker( 166 | sources: Vec, 167 | dest: &Path, 168 | config: &Config, 169 | work_tx: cbc::Sender, 170 | stats: Arc, 171 | ) -> Result<()> { 172 | debug!("Starting walk worker {:?}", thread::current().id()); 173 | 174 | for source in sources { 175 | let sourcedir = source 176 | .components() 177 | .next_back() 178 | .ok_or(XcpError::InvalidSource("Failed to find source directory name."))?; 179 | 180 | let target_base = if dest.exists() && dest.is_dir() && !config.no_target_directory { 181 | dest.join(sourcedir) 182 | } else { 183 | dest.to_path_buf() 184 | }; 185 | debug!("Target base is {target_base:?}"); 186 | 187 | let gitignore = parse_ignore(&source, config)?; 188 | 189 | for entry in WalkDir::new(&source) 190 | .into_iter() 191 | .filter_entry(|e| ignore_filter(e, &gitignore)) 192 | { 193 | debug!("Got tree entry {entry:?}"); 194 | let epath = entry?.into_path(); 195 | let from = if config.dereference { 196 | let cpath = canonicalize(&epath)?; 197 | debug!("Dereferencing {epath:?} into {cpath:?}"); 198 | cpath 199 | } else { 200 | epath.clone() 201 | }; 202 | let meta = from.symlink_metadata()?; 203 | let path = epath.strip_prefix(&source)?; 204 | let target = if !empty_path(path) { 205 | target_base.join(path) 206 | } else { 207 | target_base.clone() 208 | }; 209 | 210 | if config.no_clobber && target.exists() { 211 | let msg = "Destination file exists and --no-clobber is set."; 212 | stats.send(StatusUpdate::Error( 213 | XcpError::DestinationExists(msg, target)))?; 214 | return Err(XcpError::EarlyShutdown(msg).into()); 215 | } 216 | 217 | let ft = FileType::from(meta.file_type()); 218 | match ft { 219 | FileType::File => { 220 | debug!("Send copy operation {from:?} to {target:?}"); 221 | stats.send(StatusUpdate::Size(meta.len()))?; 222 | work_tx.send(Operation::Copy(from, target))?; 223 | } 224 | 225 | FileType::Symlink => { 226 | let lfile = read_link(from)?; 227 | debug!("Send symlink operation {lfile:?} to {target:?}"); 228 | work_tx.send(Operation::Link(lfile, target))?; 229 | } 230 | 231 | FileType::Dir => { 232 | // Create dir tree immediately as we can't 233 | // guarantee a worker will action the creation 234 | // before a subsequent copy operation requires it. 235 | debug!("Creating target directory {target:?}"); 236 | if let Err(err) = create_dir_all(&target) { 237 | let msg = format!("Error creating target directory: {err}"); 238 | error!("{msg}"); 239 | return Err(XcpError::CopyError(msg).into()) 240 | } 241 | if config.ownership && 242 | let Err(e) = chown(&target, Some(meta.uid()), Some(meta.gid())) 243 | { 244 | warn!("Failed to copy directory ownership: {target:?}: {e}"); 245 | } 246 | } 247 | 248 | FileType::Socket | FileType::Char | FileType::Fifo => { 249 | debug!("Special file found: {from:?} to {target:?}"); 250 | work_tx.send(Operation::Special(from, target))?; 251 | } 252 | 253 | FileType::Block | FileType::Other => { 254 | error!("Unsupported filetype found: {target:?} -> {ft:?}"); 255 | return Err(XcpError::UnknownFileType(target).into()); 256 | } 257 | }; 258 | } 259 | } 260 | debug!("Walk-worker finished: {:?}", thread::current().id()); 261 | 262 | Ok(()) 263 | } 264 | 265 | fn empty_path(path: &Path) -> bool { 266 | *path == PathBuf::new() 267 | } 268 | -------------------------------------------------------------------------------- /tests/util.rs: -------------------------------------------------------------------------------- 1 | #![allow(unused)] 2 | /* 3 | * Copyright © 2018, Steve Smith 4 | * 5 | * This program is free software: you can redistribute it and/or 6 | * modify it under the terms of the GNU General Public License version 7 | * 3 as published by the Free Software Foundation. 8 | * 9 | * This program is distributed in the hope that it will be useful, but 10 | * WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | use anyhow::{self, Error}; 19 | use fslock::LockFile; 20 | use rand::{Rng, RngCore, SeedableRng, rng}; 21 | use rand_distr::{Alphanumeric, Pareto, Triangular, StandardUniform}; 22 | use rand_xorshift::XorShiftRng; 23 | use std::cmp; 24 | use std::env::current_dir; 25 | use std::fs::{create_dir_all, File, FileTimes}; 26 | use std::io::{BufRead, BufReader, Read, Seek, SeekFrom, Write}; 27 | use std::path::{Path, PathBuf}; 28 | use std::process::{Command, Output}; 29 | use std::result; 30 | use std::time::{Duration, SystemTime}; 31 | use tempfile::{tempdir_in, TempDir}; 32 | use uuid::Uuid; 33 | use walkdir::WalkDir; 34 | 35 | pub type TResult = result::Result<(), Error>; 36 | 37 | pub fn get_command() -> Result { 38 | let exe = env!("CARGO_BIN_EXE_xcp"); 39 | Ok(Command::new(exe)) 40 | } 41 | 42 | pub fn run(args: &[&str]) -> Result { 43 | let out = get_command()?.args(args).output()?; 44 | println!("STDOUT: {}", String::from_utf8_lossy(&out.stdout)); 45 | println!("STDERR: {}", String::from_utf8_lossy(&out.stderr)); 46 | Ok(out) 47 | } 48 | 49 | pub fn tempdir_rel() -> Result { 50 | // let uuid = Uuid::new_v4(); 51 | // let dir = PathBuf::from("target/").join(uuid.to_string()); 52 | // create_dir_all(&dir)?; 53 | // Ok(dir) 54 | Ok(tempdir_in(current_dir()?.join("target"))?) 55 | } 56 | 57 | pub fn create_file(path: &Path, text: &str) -> Result<(), Error> { 58 | let file = File::create(path)?; 59 | write!(&file, "{text}")?; 60 | Ok(()) 61 | } 62 | 63 | pub fn set_time_past(file: &Path) -> Result<(), Error> { 64 | let yr = Duration::from_secs(60 * 60 * 24 * 365); 65 | let past = SystemTime::now().checked_sub(yr).unwrap(); 66 | let ft = FileTimes::new() 67 | .set_modified(past); 68 | File::open(file)?.set_times(ft)?; 69 | Ok(()) 70 | } 71 | 72 | pub fn timestamps_same(from: &SystemTime, to: &SystemTime) -> bool { 73 | let from_s = from.duration_since(SystemTime::UNIX_EPOCH).unwrap().as_secs() as i64; 74 | let to_s = to.duration_since(SystemTime::UNIX_EPOCH).unwrap().as_secs() as i64; 75 | // 5s tolerance 76 | from_s.abs_diff(to_s) < 5 77 | } 78 | 79 | 80 | #[cfg(any(target_os = "linux", target_os = "android"))] 81 | #[allow(unused)] 82 | pub fn create_sparse(file: &Path, head: u64, tail: u64) -> Result { 83 | let data = "c00lc0d3"; 84 | let len = 4096u64 * 4096 + data.len() as u64 + tail; 85 | 86 | let out = Command::new("/usr/bin/truncate") 87 | .args(["-s", len.to_string().as_str(), file.to_str().unwrap()]) 88 | .output()?; 89 | assert!(out.status.success()); 90 | 91 | let mut fd = std::fs::OpenOptions::new() 92 | .write(true) 93 | .append(false) 94 | .open(file)?; 95 | 96 | fd.seek(SeekFrom::Start(head))?; 97 | write!(fd, "{data}")?; 98 | 99 | fd.seek(SeekFrom::Start(1024 * 4096))?; 100 | write!(fd, "{data}")?; 101 | 102 | fd.seek(SeekFrom::Start(4096 * 4096))?; 103 | write!(fd, "{data}")?; 104 | 105 | Ok(len) 106 | } 107 | 108 | #[allow(unused)] 109 | pub fn file_contains(path: &Path, text: &str) -> Result { 110 | let mut dest = File::open(path)?; 111 | let mut buf = String::new(); 112 | dest.read_to_string(&mut buf)?; 113 | 114 | Ok(buf == text) 115 | } 116 | 117 | pub fn files_match(a: &Path, b: &Path) -> bool { 118 | println!("Checking: {a:?}"); 119 | if a.metadata().unwrap().len() != b.metadata().unwrap().len() { 120 | return false; 121 | } 122 | let mut abr = BufReader::with_capacity(1024 * 1024, File::open(a).unwrap()); 123 | let mut bbr = BufReader::with_capacity(1024 * 1024, File::open(b).unwrap()); 124 | loop { 125 | let read = { 126 | let ab = abr.fill_buf().unwrap(); 127 | let bb = bbr.fill_buf().unwrap(); 128 | if ab != bb { 129 | return false; 130 | } 131 | if ab.is_empty() { 132 | return true; 133 | } 134 | ab.len() 135 | }; 136 | abr.consume(read); 137 | bbr.consume(read); 138 | } 139 | } 140 | 141 | #[test] 142 | fn test_hasher() -> TResult { 143 | { 144 | let dir = tempdir_rel()?; 145 | let a = dir.path().join("source.txt"); 146 | let b = dir.path().join("dest.txt"); 147 | let text = "sd;lkjfasl;kjfa;sldkfjaslkjfa;jsdlfkjsdlfkajl"; 148 | create_file(&a, text)?; 149 | create_file(&b, text)?; 150 | assert!(files_match(&a, &b)); 151 | } 152 | { 153 | let dir = tempdir_rel()?; 154 | let a = dir.path().join("source.txt"); 155 | let b = dir.path().join("dest.txt"); 156 | create_file(&a, "lskajdf;laksjdfl;askjdf;alksdj")?; 157 | create_file(&b, "29483793857398")?; 158 | assert!(!files_match(&a, &b)); 159 | } 160 | 161 | Ok(()) 162 | } 163 | 164 | #[cfg(any(target_os = "linux", target_os = "android"))] 165 | pub fn quickstat(file: &Path) -> Result<(i32, i32, i32), Error> { 166 | let out = Command::new("stat") 167 | .args(["--format", "%s %b %B", file.to_str().unwrap()]) 168 | .output()?; 169 | assert!(out.status.success()); 170 | 171 | let stdout = String::from_utf8(out.stdout)?; 172 | let stats = stdout 173 | .split_whitespace() 174 | .map(|s| s.parse::().unwrap()) 175 | .collect::>(); 176 | let (size, blocks, blksize) = (stats[0], stats[1], stats[2]); 177 | 178 | Ok((size, blocks, blksize)) 179 | } 180 | 181 | #[cfg(any(target_os = "linux", target_os = "android"))] 182 | pub fn probably_sparse(file: &Path) -> Result { 183 | let (size, blocks, blksize) = quickstat(file)?; 184 | Ok(blocks < size / blksize) 185 | } 186 | #[cfg(not(any(target_os = "linux", target_os = "android")))] 187 | pub fn probably_sparse(file: &Path) -> Result { 188 | Ok(false) 189 | } 190 | 191 | pub fn rand_data(len: usize) -> Vec { 192 | rng() 193 | .sample_iter(StandardUniform) 194 | .take(len) 195 | .collect() 196 | } 197 | 198 | const MAXDEPTH: u64 = 2; 199 | 200 | pub fn gen_file_name(rng: &mut dyn RngCore, len: u64) -> String { 201 | let r = rng 202 | .sample_iter(Alphanumeric) 203 | .take(len as usize) 204 | .collect::>(); 205 | String::from_utf8(r).unwrap() 206 | } 207 | 208 | pub fn gen_file(path: &Path, rng: &mut dyn RngCore, size: usize, sparse: bool) -> TResult { 209 | println!("Generating: {path:?}"); 210 | let mut fd = File::create(path)?; 211 | const BSIZE: usize = 4096; 212 | let mut buffer = [0; BSIZE]; 213 | let mut left = size; 214 | 215 | while left > 0 { 216 | let blen = cmp::min(left, BSIZE); 217 | let b = &mut buffer[..blen]; 218 | rng.fill(b); 219 | if sparse && b[0] % 3 == 0 { 220 | fd.seek(SeekFrom::Current(blen as i64))?; 221 | left -= blen; 222 | } else { 223 | left -= fd.write(b)?; 224 | } 225 | } 226 | 227 | Ok(()) 228 | } 229 | 230 | /// Recursive random file-tree generator. The distributions have been 231 | /// manually chosen to give a rough approximation of a working 232 | /// project, with most files in the 10's of Ks, and a few larger 233 | /// ones. With a seeded PRNG (see below) this will give a repeatable 234 | /// tree depending on the seed. 235 | pub fn gen_subtree(base: &Path, rng: &mut dyn RngCore, depth: u64, with_sparse: bool) -> TResult { 236 | create_dir_all(base)?; 237 | 238 | let dist0 = Triangular::new(0.0, 64.0, 64.0 / 5.0)?; 239 | let dist1 = Triangular::new(1.0, 64.0, 64.0 / 5.0)?; 240 | let distf = Pareto::new(50.0 * 1024.0, 1.0)?; 241 | 242 | let nfiles = rng.sample(dist0) as u64; 243 | for _ in 0..nfiles { 244 | let fnlen = rng.sample(dist1) as u64; 245 | let fsize = rng.sample(distf) as u64; 246 | let fname = gen_file_name(rng, fnlen); 247 | let path = base.join(fname); 248 | let sparse = with_sparse && nfiles % 3 == 0; 249 | gen_file(&path, rng, fsize as usize, sparse)?; 250 | } 251 | 252 | if depth < MAXDEPTH { 253 | let ndirs = rng.sample(dist1) as u64; 254 | for _ in 0..ndirs { 255 | let fnlen = rng.sample(dist1) as u64; 256 | let fname = gen_file_name(rng, fnlen); 257 | let path = base.join(fname); 258 | gen_subtree(&path, rng, depth + 1, with_sparse)?; 259 | } 260 | } 261 | 262 | Ok(()) 263 | } 264 | 265 | pub fn gen_global_filetree(with_sparse: bool) -> anyhow::Result { 266 | let path = PathBuf::from("target/generated_filetree"); 267 | let lockfile = path.with_extension("lock"); 268 | 269 | let mut lf = LockFile::open(&lockfile)?; 270 | lf.lock()?; 271 | if !path.exists() { 272 | gen_filetree(&path, 0, with_sparse)?; 273 | } 274 | lf.unlock(); 275 | 276 | Ok(path) 277 | } 278 | 279 | pub fn gen_filetree(base: &Path, seed: u64, with_sparse: bool) -> TResult { 280 | let mut rng = XorShiftRng::seed_from_u64(seed); 281 | gen_subtree(base, &mut rng, 0, with_sparse) 282 | } 283 | 284 | pub fn compare_trees(src: &Path, dest: &Path) -> TResult { 285 | let pref = src.components().count(); 286 | for entry in WalkDir::new(src) { 287 | let from = entry?.into_path(); 288 | let tail: PathBuf = from.components().skip(pref).collect(); 289 | let to = dest.join(tail); 290 | 291 | assert!(to.exists()); 292 | assert_eq!(from.is_dir(), to.is_dir()); 293 | assert_eq!( 294 | from.metadata()?.file_type().is_symlink(), 295 | to.metadata()?.file_type().is_symlink() 296 | ); 297 | 298 | if from.is_file() { 299 | assert_eq!(probably_sparse(&to)?, probably_sparse(&to)?); 300 | assert!(files_match(&from, &to)); 301 | // FIXME: Ideally we'd check sparse holes here, but 302 | // there's no guarantee they'll match exactly due to 303 | // low-level filesystem details (SEEK_HOLE behaviour, 304 | // tail-packing, compression, etc.) 305 | } 306 | } 307 | Ok(()) 308 | } 309 | -------------------------------------------------------------------------------- /tests/linux.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | mod util; 18 | 19 | #[cfg(all(target_os = "linux", feature = "use_linux"))] 20 | mod test { 21 | use std::{process::Command, fs::{File, OpenOptions}, io::SeekFrom}; 22 | use std::io::{Seek, Write}; 23 | use libfs::{map_extents, sync}; 24 | use test_case::test_case; 25 | 26 | use crate::util::*; 27 | 28 | #[cfg_attr(feature = "parblock", test_case("parblock"; "Test with parallel block driver"))] 29 | #[test_case("parfile"; "Test with parallel file driver")] 30 | #[cfg_attr(feature = "test_no_reflink", ignore = "No FS support")] 31 | fn file_copy_reflink_always(drv: &str) { 32 | let dir = tempdir_rel().unwrap(); 33 | let source_path = dir.path().join("source.bin"); 34 | let dest_path = dir.path().join("dest.bin"); 35 | let size = 128 * 1024; 36 | 37 | { 38 | let mut infd = File::create(&source_path).unwrap(); 39 | let data = rand_data(size); 40 | infd.write_all(&data).unwrap(); 41 | } 42 | 43 | { 44 | let infd = File::open(&source_path).unwrap(); 45 | let inext = map_extents(&infd).unwrap().unwrap(); 46 | // Single file, extent not shared. 47 | assert!(!inext[0].shared); 48 | } 49 | 50 | let out = run(&[ 51 | "--driver", drv, 52 | "--reflink=always", 53 | source_path.to_str().unwrap(), 54 | dest_path.to_str().unwrap(), 55 | ]) 56 | .unwrap(); 57 | 58 | // Should always work on CoW FS 59 | assert!(out.status.success()); 60 | assert!(files_match(&source_path, &dest_path)); 61 | 62 | { 63 | let infd = File::open(&source_path).unwrap(); 64 | let outfd = File::open(&dest_path).unwrap(); 65 | // Extents should be shared. 66 | let inext = map_extents(&infd).unwrap().unwrap(); 67 | let outext = map_extents(&outfd).unwrap().unwrap(); 68 | assert!(inext[0].shared); 69 | assert!(outext[0].shared); 70 | } 71 | 72 | { 73 | let mut outfd = OpenOptions::new() 74 | .create(false) 75 | .write(true) 76 | .read(true) 77 | .open(&dest_path).unwrap(); 78 | outfd.seek(SeekFrom::Start(0)).unwrap(); 79 | let data = rand_data(size); 80 | outfd.write_all(&data).unwrap(); 81 | // brtfs at least seems to need this to force CoW and 82 | // de-share the extents. 83 | sync(&outfd).unwrap(); 84 | } 85 | 86 | { 87 | let infd = File::open(&source_path).unwrap(); 88 | let outfd = File::open(&dest_path).unwrap(); 89 | // First extent should now be un-shared. 90 | let inext = map_extents(&infd).unwrap().unwrap(); 91 | let outext = map_extents(&outfd).unwrap().unwrap(); 92 | assert!(!inext[0].shared); 93 | assert!(!outext[0].shared); 94 | } 95 | 96 | } 97 | 98 | #[cfg_attr(feature = "parblock", test_case("parblock"; "Test with parallel block driver"))] 99 | #[test_case("parfile"; "Test with parallel file driver")] 100 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 101 | fn test_sparse(drv: &str) { 102 | use std::fs::read; 103 | 104 | let dir = tempdir_rel().unwrap(); 105 | let from = dir.path().join("sparse.bin"); 106 | let to = dir.path().join("target.bin"); 107 | 108 | let slen = create_sparse(&from, 0, 0).unwrap(); 109 | assert_eq!(slen, from.metadata().unwrap().len()); 110 | assert!(probably_sparse(&from).unwrap()); 111 | 112 | let out = run(&[ 113 | "--driver", 114 | drv, 115 | from.to_str().unwrap(), 116 | to.to_str().unwrap(), 117 | ]).unwrap(); 118 | assert!(out.status.success()); 119 | 120 | assert!(probably_sparse(&to).unwrap()); 121 | 122 | assert_eq!(quickstat(&from).unwrap(), quickstat(&to).unwrap()); 123 | 124 | let from_data = read(&from).unwrap(); 125 | let to_data = read(&to).unwrap(); 126 | assert_eq!(from_data, to_data); 127 | } 128 | 129 | #[cfg_attr(feature = "parblock", test_case("parblock"; "Test with parallel block driver"))] 130 | #[test_case("parfile"; "Test with parallel file driver")] 131 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 132 | fn test_sparse_leading_gap(drv: &str) { 133 | use std::fs::read; 134 | 135 | let dir = tempdir_rel().unwrap(); 136 | let from = dir.path().join("sparse.bin"); 137 | let to = dir.path().join("target.bin"); 138 | 139 | let slen = create_sparse(&from, 1024, 0).unwrap(); 140 | assert_eq!(slen, from.metadata().unwrap().len()); 141 | assert!(probably_sparse(&from).unwrap()); 142 | 143 | let out = run(&[ 144 | "--driver", 145 | drv, 146 | from.to_str().unwrap(), 147 | to.to_str().unwrap(), 148 | ]).unwrap(); 149 | 150 | assert!(out.status.success()); 151 | assert!(probably_sparse(&to).unwrap()); 152 | assert_eq!(quickstat(&from).unwrap(), quickstat(&to).unwrap()); 153 | 154 | let from_data = read(&from).unwrap(); 155 | let to_data = read(&to).unwrap(); 156 | assert_eq!(from_data, to_data); 157 | } 158 | 159 | #[cfg_attr(feature = "parblock", test_case("parblock"; "Test with parallel block driver"))] 160 | #[test_case("parfile"; "Test with parallel file driver")] 161 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 162 | fn test_sparse_trailng_gap(drv: &str) { 163 | use std::fs::read; 164 | 165 | let dir = tempdir_rel().unwrap(); 166 | let from = dir.path().join("sparse.bin"); 167 | let to = dir.path().join("target.bin"); 168 | 169 | let slen = create_sparse(&from, 1024, 1024).unwrap(); 170 | assert_eq!(slen, from.metadata().unwrap().len()); 171 | assert!(probably_sparse(&from).unwrap()); 172 | 173 | let out = run(&[ 174 | "--driver", 175 | drv, 176 | from.to_str().unwrap(), 177 | to.to_str().unwrap(), 178 | ]).unwrap(); 179 | assert!(out.status.success()); 180 | 181 | assert!(probably_sparse(&to).unwrap()); 182 | assert_eq!(quickstat(&from).unwrap(), quickstat(&to).unwrap()); 183 | 184 | let from_data = read(&from).unwrap(); 185 | let to_data = read(&to).unwrap(); 186 | assert_eq!(from_data, to_data); 187 | } 188 | 189 | #[cfg_attr(feature = "parblock", test_case("parblock"; "Test with parallel block driver"))] 190 | #[test_case("parfile"; "Test with parallel file driver")] 191 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 192 | fn test_sparse_single_overwrite(drv: &str) { 193 | use std::fs::read; 194 | 195 | let dir = tempdir_rel().unwrap(); 196 | let from = dir.path().join("sparse.bin"); 197 | let to = dir.path().join("target.bin"); 198 | 199 | let slen = create_sparse(&from, 1024, 1024).unwrap(); 200 | create_file(&to, "").unwrap(); 201 | assert_eq!(slen, from.metadata().unwrap().len()); 202 | assert!(probably_sparse(&from).unwrap()); 203 | 204 | let out = run(&[ 205 | "--driver", 206 | drv, 207 | from.to_str().unwrap(), 208 | to.to_str().unwrap(), 209 | ]).unwrap(); 210 | assert!(out.status.success()); 211 | assert!(probably_sparse(&to).unwrap()); 212 | assert_eq!(quickstat(&from).unwrap(), quickstat(&to).unwrap()); 213 | 214 | let from_data = read(&from).unwrap(); 215 | let to_data = read(&to).unwrap(); 216 | assert_eq!(from_data, to_data); 217 | } 218 | 219 | #[cfg_attr(feature = "parblock", test_case("parblock"; "Test with parallel block driver"))] 220 | #[test_case("parfile"; "Test with parallel file driver")] 221 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 222 | fn test_empty_sparse(drv: &str) { 223 | use std::fs::read; 224 | 225 | let dir = tempdir_rel().unwrap(); 226 | let from = dir.path().join("sparse.bin"); 227 | let to = dir.path().join("target.bin"); 228 | 229 | let out = Command::new("/usr/bin/truncate") 230 | .args(["-s", "1M", from.to_str().unwrap()]) 231 | .output().unwrap(); 232 | assert!(out.status.success()); 233 | assert_eq!(from.metadata().unwrap().len(), 1024 * 1024); 234 | 235 | let out = run(&[ 236 | "--driver", 237 | drv, 238 | from.to_str().unwrap(), 239 | to.to_str().unwrap(), 240 | ]).unwrap(); 241 | assert!(out.status.success()); 242 | assert_eq!(to.metadata().unwrap().len(), 1024 * 1024); 243 | 244 | assert!(probably_sparse(&to).unwrap()); 245 | assert_eq!(quickstat(&from).unwrap(), quickstat(&to).unwrap()); 246 | 247 | let from_data = read(&from).unwrap(); 248 | let to_data = read(&to).unwrap(); 249 | assert_eq!(from_data, to_data); 250 | } 251 | 252 | 253 | #[cfg_attr(feature = "parblock", test_case("parblock"; "Test with parallel block driver"))] 254 | #[test_case("parfile"; "Test with parallel file driver")] 255 | #[cfg_attr(not(feature = "test_run_expensive"), ignore = "Stress test")] 256 | fn copy_generated_tree_sparse(drv: &str) { 257 | // Spam some output to keep CI from timing-out (hopefully). 258 | println!("Generating file tree..."); 259 | let src = gen_global_filetree(false).unwrap(); 260 | 261 | let dir = tempdir_rel().unwrap(); 262 | let dest = dir.path().join("target"); 263 | 264 | println!("Running copy..."); 265 | let out = run(&[ 266 | "--driver", drv, 267 | "-r", 268 | "--no-progress", 269 | src.to_str().unwrap(), 270 | dest.to_str().unwrap(), 271 | ]).unwrap(); 272 | assert!(out.status.success()); 273 | 274 | println!("Compare trees..."); 275 | compare_trees(&src, &dest).unwrap(); 276 | } 277 | } 278 | -------------------------------------------------------------------------------- /libfs/src/common.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | 18 | use log::{debug, warn}; 19 | use rustix::fs::{fsync, ftruncate}; 20 | use rustix::io::{pread, pwrite}; 21 | use std::cmp; 22 | use std::fs::{File, FileTimes}; 23 | use std::io::{ErrorKind, Read, Write}; 24 | use std::os::unix::fs::{fchown, MetadataExt}; 25 | use std::path::Path; 26 | use xattr::FileExt; 27 | 28 | use crate::errors::{Result, Error}; 29 | use crate::{Extent, XATTR_SUPPORTED, copy_sparse, probably_sparse, copy_file_bytes}; 30 | 31 | fn copy_xattr(infd: &File, outfd: &File) -> Result<()> { 32 | // FIXME: Flag for xattr. 33 | if XATTR_SUPPORTED { 34 | debug!("Starting xattr copy..."); 35 | for attr in infd.list_xattr()? { 36 | if let Some(val) = infd.get_xattr(&attr)? { 37 | debug!("Copy xattr {attr:?}"); 38 | outfd.set_xattr(attr, val.as_slice())?; 39 | } 40 | } 41 | } 42 | Ok(()) 43 | } 44 | 45 | /// Copy file permissions. Will also copy 46 | /// [xattr](https://man7.org/linux/man-pages/man7/xattr.7.html)'s if 47 | /// possible. 48 | pub fn copy_permissions(infd: &File, outfd: &File) -> Result<()> { 49 | let xr = copy_xattr(infd, outfd); 50 | if let Err(e) = xr { 51 | // FIXME: We don't have a way of detecting if the 52 | // target FS supports XAttr, so assume any error is 53 | // "Unsupported" for now. 54 | warn!("Failed to copy xattrs from {infd:?}: {e}"); 55 | } 56 | 57 | // FIXME: ACLs, selinux, etc. 58 | 59 | let inmeta = infd.metadata()?; 60 | 61 | debug!("Performing permissions copy"); 62 | outfd.set_permissions(inmeta.permissions())?; 63 | 64 | Ok(()) 65 | } 66 | 67 | /// Copy file timestamps. 68 | pub fn copy_timestamps(infd: &File, outfd: &File) -> Result<()> { 69 | let inmeta = infd.metadata()?; 70 | 71 | debug!("Performing timestamp copy"); 72 | let ftime = FileTimes::new() 73 | .set_accessed(inmeta.accessed()?) 74 | .set_modified(inmeta.modified()?); 75 | outfd.set_times(ftime)?; 76 | 77 | Ok(()) 78 | } 79 | 80 | pub fn copy_owner(infd: &File, outfd: &File) -> Result<()> { 81 | let inmeta = infd.metadata()?; 82 | fchown(outfd, Some(inmeta.uid()), Some(inmeta.gid()))?; 83 | 84 | Ok(()) 85 | } 86 | 87 | pub(crate) fn read_bytes(fd: &File, buf: &mut [u8], off: usize) -> Result { 88 | Ok(pread(fd, buf, off as u64)?) 89 | } 90 | 91 | pub(crate) fn write_bytes(fd: &File, buf: &mut [u8], off: usize) -> Result { 92 | Ok(pwrite(fd, buf, off as u64)?) 93 | } 94 | 95 | /// Copy a block of bytes at an offset between files. Uses Posix pread/pwrite. 96 | pub(crate) fn copy_range_uspace(reader: &File, writer: &File, nbytes: usize, off: usize) -> Result { 97 | // FIXME: For larger buffers we should use a pre-allocated thread-local? 98 | let mut buf = vec![0; nbytes]; 99 | 100 | let mut written: usize = 0; 101 | while written < nbytes { 102 | let next = cmp::min(nbytes - written, nbytes); 103 | let noff = off + written; 104 | 105 | let rlen = match read_bytes(reader, &mut buf[..next], noff) { 106 | Ok(0) => return Err(Error::InvalidSource("Source file ended prematurely.")), 107 | Ok(len) => len, 108 | Err(e) => return Err(e), 109 | }; 110 | 111 | let _wlen = match write_bytes(writer, &mut buf[..rlen], noff) { 112 | Ok(len) if len < rlen => { 113 | return Err(Error::InvalidSource("Failed write to file.")) 114 | } 115 | Ok(len) => len, 116 | Err(e) => return Err(e), 117 | }; 118 | 119 | written += rlen; 120 | } 121 | Ok(written) 122 | } 123 | 124 | /// Slightly modified version of io::copy() that only copies a set amount of bytes. 125 | pub(crate) fn copy_bytes_uspace(mut reader: &File, mut writer: &File, nbytes: usize) -> Result { 126 | let mut buf = vec![0; nbytes]; 127 | 128 | let mut written = 0; 129 | while written < nbytes { 130 | let next = cmp::min(nbytes - written, nbytes); 131 | let len = match reader.read(&mut buf[..next]) { 132 | Ok(0) => return Err(Error::InvalidSource("Source file ended prematurely.")), 133 | Ok(len) => len, 134 | Err(ref e) if e.kind() == ErrorKind::Interrupted => continue, 135 | Err(e) => return Err(e.into()) 136 | }; 137 | writer.write_all(&buf[..len])?; 138 | written += len; 139 | } 140 | Ok(written) 141 | } 142 | 143 | /// Allocate file space on disk. Uses Posix ftruncate(). 144 | pub fn allocate_file(fd: &File, len: u64) -> Result<()> { 145 | Ok(ftruncate(fd, len)?) 146 | } 147 | 148 | /// Merge any contiguous extents in a list. See [merge_extents]. 149 | pub fn merge_extents(extents: Vec) -> Result> { 150 | let mut merged: Vec = vec![]; 151 | 152 | let mut prev: Option = None; 153 | for e in extents { 154 | match prev { 155 | Some(p) => { 156 | if e.start == p.end + 1 { 157 | // Current & prev are contiguous, merge & see what 158 | // comes next. 159 | prev = Some(Extent { 160 | start: p.start, 161 | end: e.end, 162 | shared: p.shared & e.shared, 163 | }); 164 | } else { 165 | merged.push(p); 166 | prev = Some(e); 167 | } 168 | } 169 | // First iter 170 | None => prev = Some(e), 171 | } 172 | } 173 | if let Some(p) = prev { 174 | merged.push(p); 175 | } 176 | 177 | Ok(merged) 178 | } 179 | 180 | 181 | /// Determine if two files are the same by examining their inodes. 182 | pub fn is_same_file(src: &Path, dest: &Path) -> Result { 183 | let sstat = src.metadata()?; 184 | let dstat = dest.metadata()?; 185 | let same = (sstat.ino() == dstat.ino()) 186 | && (sstat.dev() == dstat.dev()); 187 | 188 | Ok(same) 189 | } 190 | 191 | /// Copy a file. This differs from [std::fs::copy] in that it looks 192 | /// for sparse blocks and skips them. 193 | pub fn copy_file(from: &Path, to: &Path) -> Result { 194 | let infd = File::open(from)?; 195 | let len = infd.metadata()?.len(); 196 | 197 | let outfd = File::create(to)?; 198 | allocate_file(&outfd, len)?; 199 | 200 | let total = if probably_sparse(&infd)? { 201 | copy_sparse(&infd, &outfd)? 202 | } else { 203 | copy_file_bytes(&infd, &outfd, len)? as u64 204 | }; 205 | 206 | Ok(total) 207 | } 208 | 209 | /// Sync an open file to disk. Uses `fsync(2)`. 210 | pub fn sync(fd: &File) -> Result<()> { 211 | Ok(fsync(fd)?) 212 | } 213 | 214 | #[cfg(test)] 215 | mod tests { 216 | use super::*; 217 | use std::fs::read; 218 | use std::ops::Range; 219 | use tempfile::tempdir; 220 | 221 | impl From> for Extent { 222 | fn from(r: Range) -> Self { 223 | Extent { 224 | start: r.start, 225 | end: r.end, 226 | shared: false, 227 | } 228 | } 229 | } 230 | 231 | #[test] 232 | fn test_copy_bytes_uspace_large() { 233 | let dir = tempdir().unwrap(); 234 | let from = dir.path().join("from.bin"); 235 | let to = dir.path().join("to.bin"); 236 | let size = 128 * 1024; 237 | let data = "X".repeat(size); 238 | 239 | { 240 | let mut fd: File = File::create(&from).unwrap(); 241 | write!(fd, "{data}").unwrap(); 242 | } 243 | 244 | { 245 | let infd = File::open(&from).unwrap(); 246 | let outfd = File::create(&to).unwrap(); 247 | let written = copy_bytes_uspace(&infd, &outfd, size).unwrap(); 248 | 249 | assert_eq!(written, size); 250 | } 251 | 252 | assert_eq!(from.metadata().unwrap().len(), to.metadata().unwrap().len()); 253 | 254 | { 255 | let from_data = read(&from).unwrap(); 256 | let to_data = read(&to).unwrap(); 257 | assert_eq!(from_data, to_data); 258 | } 259 | } 260 | 261 | #[test] 262 | fn test_copy_range_uspace_large() { 263 | let dir = tempdir().unwrap(); 264 | let from = dir.path().join("from.bin"); 265 | let to = dir.path().join("to.bin"); 266 | let size = 128 * 1024; 267 | let data = "X".repeat(size); 268 | 269 | { 270 | let mut fd: File = File::create(&from).unwrap(); 271 | write!(fd, "{data}").unwrap(); 272 | } 273 | 274 | { 275 | let infd = File::open(&from).unwrap(); 276 | let outfd = File::create(&to).unwrap(); 277 | 278 | let blocksize = size / 4; 279 | let mut written = 0; 280 | 281 | for off in (0..4).rev() { 282 | written += copy_range_uspace(&infd, &outfd, blocksize, blocksize * off).unwrap(); 283 | } 284 | 285 | assert_eq!(written, size); 286 | } 287 | 288 | assert_eq!(from.metadata().unwrap().len(), to.metadata().unwrap().len()); 289 | 290 | { 291 | let from_data = read(&from).unwrap(); 292 | let to_data = read(&to).unwrap(); 293 | assert_eq!(from_data, to_data); 294 | } 295 | } 296 | 297 | #[test] 298 | fn test_extent_merge() -> Result<()> { 299 | assert_eq!(merge_extents(vec!())?, vec!()); 300 | assert_eq!(merge_extents( 301 | vec!((0..1).into()))?, 302 | vec!((0..1).into())); 303 | 304 | assert_eq!(merge_extents( 305 | vec!((0..1).into(), 306 | (10..20).into()))?, 307 | vec!((0..1).into(), 308 | (10..20).into())); 309 | assert_eq!(merge_extents( 310 | vec!((0..10).into(), 311 | (11..20).into()))?, 312 | vec!((0..20).into())); 313 | assert_eq!( 314 | merge_extents( 315 | vec!((0..5).into(), 316 | (11..20).into(), 317 | (21..30).into(), 318 | (40..50).into()))?, 319 | vec!((0..5).into(), 320 | (11..30).into(), 321 | (40..50).into()) 322 | ); 323 | assert_eq!( 324 | merge_extents(vec!((0..5).into(), 325 | (11..20).into(), 326 | (21..30).into(), 327 | (40..50).into(), 328 | (51..60).into()))?, 329 | vec!((0..5).into(), 330 | (11..30).into(), 331 | (40..60).into()) 332 | ); 333 | assert_eq!( 334 | merge_extents( 335 | vec!((0..10).into(), 336 | (11..20).into(), 337 | (21..30).into(), 338 | (31..50).into(), 339 | (51..60).into()))?, 340 | vec!((0..60).into()) 341 | ); 342 | Ok(()) 343 | } 344 | 345 | 346 | #[test] 347 | fn test_copy_file() -> Result<()> { 348 | let dir = tempdir()?; 349 | let from = dir.path().join("file.bin"); 350 | let len = 32 * 1024 * 1024; 351 | 352 | { 353 | let mut fd = File::create(&from)?; 354 | let data = "X".repeat(len); 355 | write!(fd, "{data}").unwrap(); 356 | } 357 | 358 | assert_eq!(len, from.metadata()?.len() as usize); 359 | 360 | let to = dir.path().join("sparse.copy.bin"); 361 | crate::copy_file(&from, &to)?; 362 | 363 | assert_eq!(len, to.metadata()?.len() as usize); 364 | 365 | Ok(()) 366 | } 367 | } 368 | -------------------------------------------------------------------------------- /libfs/src/linux.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018-2019, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | use std::{fs::File, path::Path}; 18 | use std::io; 19 | use std::os::unix::io::AsRawFd; 20 | use std::os::unix::prelude::PermissionsExt; 21 | 22 | use linux_raw_sys::ioctl::{FS_IOC_FIEMAP, FIEMAP_EXTENT_LAST, FICLONE, FIEMAP_EXTENT_SHARED}; 23 | use rustix::fs::CWD; 24 | use rustix::{fs::{copy_file_range, seek, mknodat, FileType, Mode, RawMode, SeekFrom}, io::Errno}; 25 | 26 | use crate::Extent; 27 | use crate::errors::Result; 28 | use crate::common::{copy_bytes_uspace, copy_range_uspace}; 29 | 30 | // Wrapper for copy_file_range(2) that checks for non-fatal errors due 31 | // to limitations of the syscall. 32 | fn try_copy_file_range( 33 | infd: &File, 34 | in_off: Option<&mut u64>, 35 | outfd: &File, 36 | out_off: Option<&mut u64>, 37 | bytes: u64, 38 | ) -> Option> { 39 | let cfr_ret = copy_file_range(infd, in_off, outfd, out_off, bytes as usize); 40 | 41 | match cfr_ret { 42 | Ok(retval) => { 43 | Some(Ok(retval)) 44 | }, 45 | Err(Errno::NOSYS) | Err(Errno::PERM) | Err(Errno::XDEV) => { 46 | None 47 | }, 48 | Err(errno) => { 49 | Some(Err(errno.into())) 50 | }, 51 | } 52 | } 53 | 54 | /// File copy operation that defers file offset tracking to the 55 | /// underlying call. On Linux this attempts to use 56 | /// [copy_file_range](https://man7.org/linux/man-pages/man2/copy_file_range.2.html) 57 | /// and falls back to user-space if that is not available. 58 | pub fn copy_file_bytes(infd: &File, outfd: &File, bytes: u64) -> Result { 59 | try_copy_file_range(infd, None, outfd, None, bytes) 60 | .unwrap_or_else(|| copy_bytes_uspace(infd, outfd, bytes as usize)) 61 | } 62 | 63 | /// File copy operation that that copies a block at offset`off`. On 64 | /// Linux this attempts to use 65 | /// [copy_file_range](https://man7.org/linux/man-pages/man2/copy_file_range.2.html) 66 | /// and falls back to user-space if that is not available. 67 | pub fn copy_file_offset(infd: &File, outfd: &File, bytes: u64, off: i64) -> Result { 68 | let mut off_in = off as u64; 69 | let mut off_out = off as u64; 70 | try_copy_file_range(infd, Some(&mut off_in), outfd, Some(&mut off_out), bytes) 71 | .unwrap_or_else(|| copy_range_uspace(infd, outfd, bytes as usize, off as usize)) 72 | } 73 | 74 | /// Guestimate if file is sparse; if it has less blocks that would be 75 | /// expected for its stated size. This is the same test used by 76 | /// coreutils `cp`. 77 | // FIXME: Should work on *BSD? 78 | pub fn probably_sparse(fd: &File) -> Result { 79 | use std::os::linux::fs::MetadataExt; 80 | const ST_NBLOCKSIZE: u64 = 512; 81 | let stat = fd.metadata()?; 82 | Ok(stat.st_blocks() < stat.st_size() / ST_NBLOCKSIZE) 83 | } 84 | 85 | #[derive(PartialEq, Debug)] 86 | enum SeekOff { 87 | Offset(u64), 88 | EOF, 89 | } 90 | 91 | fn lseek(fd: &File, from: SeekFrom) -> Result { 92 | match seek(fd, from) { 93 | Err(errno) if errno == Errno::NXIO => Ok(SeekOff::EOF), 94 | Err(err) => Err(err.into()), 95 | Ok(off) => Ok(SeekOff::Offset(off)), 96 | } 97 | } 98 | 99 | const FIEMAP_PAGE_SIZE: usize = 32; 100 | 101 | #[repr(C)] 102 | #[derive(Copy, Clone, Debug)] 103 | struct FiemapExtent { 104 | fe_logical: u64, // Logical offset in bytes for the start of the extent 105 | fe_physical: u64, // Physical offset in bytes for the start of the extent 106 | fe_length: u64, // Length in bytes for the extent 107 | fe_reserved64: [u64; 2], 108 | fe_flags: u32, // FIEMAP_EXTENT_* flags for this extent 109 | fe_reserved: [u32; 3], 110 | } 111 | impl FiemapExtent { 112 | fn new() -> FiemapExtent { 113 | FiemapExtent { 114 | fe_logical: 0, 115 | fe_physical: 0, 116 | fe_length: 0, 117 | fe_reserved64: [0; 2], 118 | fe_flags: 0, 119 | fe_reserved: [0; 3], 120 | } 121 | } 122 | } 123 | 124 | #[repr(C)] 125 | #[derive(Copy, Clone, Debug)] 126 | struct FiemapReq { 127 | fm_start: u64, // Logical offset (inclusive) at which to start mapping (in) 128 | fm_length: u64, // Logical length of mapping which userspace cares about (in) 129 | fm_flags: u32, // FIEMAP_FLAG_* flags for request (in/out) 130 | fm_mapped_extents: u32, // Number of extents that were mapped (out) 131 | fm_extent_count: u32, // Size of fm_extents array (in) 132 | fm_reserved: u32, 133 | fm_extents: [FiemapExtent; FIEMAP_PAGE_SIZE], // Array of mapped extents (out) 134 | } 135 | impl FiemapReq { 136 | fn new() -> FiemapReq { 137 | FiemapReq { 138 | fm_start: 0, 139 | fm_length: u64::MAX, 140 | fm_flags: 0, 141 | fm_mapped_extents: 0, 142 | fm_extent_count: FIEMAP_PAGE_SIZE as u32, 143 | fm_reserved: 0, 144 | fm_extents: [FiemapExtent::new(); FIEMAP_PAGE_SIZE], 145 | } 146 | } 147 | } 148 | 149 | fn fiemap(fd: &File, req: &mut FiemapReq) -> Result { 150 | // FIXME: Rustix has an IOCTL mini-framework but it's a little 151 | // tricky and is unsafe anyway. This is simpler for now. 152 | let req_ptr: *mut FiemapReq = req; 153 | if unsafe { libc::ioctl(fd.as_raw_fd(), FS_IOC_FIEMAP as u64, req_ptr) } != 0 { 154 | let oserr = io::Error::last_os_error(); 155 | if oserr.raw_os_error() == Some(libc::EOPNOTSUPP) { 156 | return Ok(false) 157 | } 158 | return Err(oserr.into()); 159 | } 160 | 161 | Ok(true) 162 | } 163 | 164 | /// Attempt to retrieve a map of the underlying allocated extents for 165 | /// a file. Will return [None] if the filesystem doesn't support 166 | /// extents. On Linux this is the raw list from 167 | /// [fiemap](https://docs.kernel.org/filesystems/fiemap.html). See 168 | /// [merge_extents](super::merge_extents) for a tool to merge contiguous extents. 169 | pub fn map_extents(fd: &File) -> Result>> { 170 | let mut req = FiemapReq::new(); 171 | let mut extents = Vec::with_capacity(FIEMAP_PAGE_SIZE); 172 | 173 | loop { 174 | if !fiemap(fd, &mut req)? { 175 | return Ok(None) 176 | } 177 | if req.fm_mapped_extents == 0 { 178 | break; 179 | } 180 | 181 | for i in 0..req.fm_mapped_extents as usize { 182 | let e = req.fm_extents[i]; 183 | let ext = Extent { 184 | start: e.fe_logical, 185 | end: e.fe_logical + e.fe_length, 186 | shared: e.fe_flags & FIEMAP_EXTENT_SHARED != 0, 187 | }; 188 | extents.push(ext); 189 | } 190 | 191 | let last = req.fm_extents[(req.fm_mapped_extents - 1) as usize]; 192 | if last.fe_flags & FIEMAP_EXTENT_LAST != 0 { 193 | break; 194 | } 195 | 196 | // Looks like we're going around again... 197 | req.fm_start = last.fe_logical + last.fe_length; 198 | } 199 | 200 | Ok(Some(extents)) 201 | } 202 | 203 | /// Search the file for the next non-sparse file section. Returns the 204 | /// start and end of the data segment. 205 | // FIXME: Should work on *BSD too? 206 | pub fn next_sparse_segments(infd: &File, outfd: &File, pos: u64) -> Result<(u64, u64)> { 207 | let next_data = match lseek(infd, SeekFrom::Data(pos))? { 208 | SeekOff::Offset(off) => off, 209 | SeekOff::EOF => infd.metadata()?.len(), 210 | }; 211 | let next_hole = match lseek(infd, SeekFrom::Hole(next_data))? { 212 | SeekOff::Offset(off) => off, 213 | SeekOff::EOF => infd.metadata()?.len(), 214 | }; 215 | 216 | lseek(infd, SeekFrom::Start(next_data))?; // FIXME: EOF (but shouldn't happen) 217 | lseek(outfd, SeekFrom::Start(next_data))?; 218 | 219 | Ok((next_data, next_hole)) 220 | } 221 | 222 | /// Copy data between files, looking for sparse blocks and skipping 223 | /// them. 224 | pub fn copy_sparse(infd: &File, outfd: &File) -> Result { 225 | let len = infd.metadata()?.len(); 226 | 227 | let mut pos = 0; 228 | while pos < len { 229 | let (next_data, next_hole) = next_sparse_segments(infd, outfd, pos)?; 230 | 231 | let _written = copy_file_bytes(infd, outfd, next_hole - next_data)?; 232 | pos = next_hole; 233 | } 234 | 235 | Ok(len) 236 | } 237 | 238 | /// Create a clone of a special file (unix socket, char-device, etc.) 239 | pub fn copy_node(src: &Path, dest: &Path) -> Result<()> { 240 | use std::os::unix::fs::MetadataExt; 241 | let meta = src.metadata()?; 242 | let rmode = RawMode::from(meta.permissions().mode()); 243 | let mode = Mode::from_raw_mode(rmode); 244 | let ftype = FileType::from_raw_mode(rmode); 245 | let dev = meta.dev(); 246 | 247 | mknodat(CWD, dest, ftype, mode, dev)?; 248 | Ok(()) 249 | } 250 | 251 | /// Reflink a file. This will reuse the underlying data on disk for 252 | /// the target file, utilising copy-on-write for any future 253 | /// updates. Only certain filesystems support this; if not supported 254 | /// the function returns `false`. 255 | pub fn reflink(infd: &File, outfd: &File) -> Result { 256 | if unsafe { libc::ioctl(outfd.as_raw_fd(), FICLONE as u64, infd.as_raw_fd()) } != 0 { 257 | let oserr = io::Error::last_os_error(); 258 | match oserr.raw_os_error() { 259 | Some(libc::EOPNOTSUPP) 260 | | Some(libc::EINVAL) 261 | | Some(libc::EXDEV) 262 | | Some(libc::ETXTBSY) => 263 | return Ok(false), 264 | _ => 265 | return Err(oserr.into()), 266 | } 267 | } 268 | Ok(true) 269 | } 270 | 271 | #[cfg(test)] 272 | #[allow(unused)] 273 | mod tests { 274 | use super::*; 275 | use crate::{allocate_file, copy_permissions}; 276 | use std::env::{current_dir, var}; 277 | use std::fs::{read, OpenOptions}; 278 | use std::io::{self, Seek, Write}; 279 | use std::iter; 280 | use std::os::unix::net::UnixListener; 281 | use std::path::PathBuf; 282 | use std::process::Command; 283 | use linux_raw_sys::ioctl::FIEMAP_EXTENT_SHARED; 284 | use log::warn; 285 | use rustix::fs::FileTypeExt; 286 | use tempfile::{tempdir_in, TempDir}; 287 | 288 | fn tempdir() -> Result { 289 | // Force into local dir as /tmp might be tmpfs, which doesn't 290 | // support all VFS options (notably fiemap). 291 | Ok(tempdir_in(current_dir()?.join("../target"))?) 292 | } 293 | 294 | #[test] 295 | #[cfg_attr(feature = "test_no_reflink", ignore = "No FS support")] 296 | fn test_reflink() -> Result<()> { 297 | let dir = tempdir()?; 298 | let from = dir.path().join("file.bin"); 299 | let to = dir.path().join("copy.bin"); 300 | let size = 128 * 1024; 301 | 302 | { 303 | let mut fd: File = File::create(&from)?; 304 | let data = "X".repeat(size); 305 | write!(fd, "{data}")?; 306 | } 307 | 308 | let from_fd = File::open(from)?; 309 | let to_fd = File::create(to)?; 310 | 311 | { 312 | let mut from_map = FiemapReq::new(); 313 | assert!(fiemap(&from_fd, &mut from_map)?); 314 | assert!(from_map.fm_mapped_extents > 0); 315 | // Un-refed file, no shared extents 316 | assert!(from_map.fm_extents[0].fe_flags & FIEMAP_EXTENT_SHARED == 0); 317 | } 318 | 319 | let worked = reflink(&from_fd, &to_fd)?; 320 | assert!(worked); 321 | 322 | { 323 | let mut from_map = FiemapReq::new(); 324 | assert!(fiemap(&from_fd, &mut from_map)?); 325 | assert!(from_map.fm_mapped_extents > 0); 326 | 327 | let mut to_map = FiemapReq::new(); 328 | assert!(fiemap(&to_fd, &mut to_map)?); 329 | assert!(to_map.fm_mapped_extents > 0); 330 | 331 | // Now both have shared extents 332 | assert_eq!(from_map.fm_mapped_extents, to_map.fm_mapped_extents); 333 | assert!(from_map.fm_extents[0].fe_flags & FIEMAP_EXTENT_SHARED != 0); 334 | assert!(to_map.fm_extents[0].fe_flags & FIEMAP_EXTENT_SHARED != 0); 335 | } 336 | 337 | Ok(()) 338 | } 339 | 340 | #[test] 341 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 342 | fn test_sparse_detection_small_data() -> Result<()> { 343 | assert!(!probably_sparse(&File::open("Cargo.toml")?)?); 344 | 345 | let dir = tempdir()?; 346 | let file = dir.path().join("sparse.bin"); 347 | let out = Command::new("/usr/bin/truncate") 348 | .args(["-s", "1M", file.to_str().unwrap()]) 349 | .output()?; 350 | assert!(out.status.success()); 351 | 352 | { 353 | let fd = File::open(&file)?; 354 | assert!(probably_sparse(&fd)?); 355 | } 356 | { 357 | let mut fd = OpenOptions::new().write(true).append(false).open(&file)?; 358 | write!(fd, "test")?; 359 | assert!(probably_sparse(&fd)?); 360 | } 361 | 362 | Ok(()) 363 | } 364 | 365 | #[test] 366 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 367 | fn test_sparse_detection_half() -> Result<()> { 368 | assert!(!probably_sparse(&File::open("Cargo.toml")?)?); 369 | 370 | let dir = tempdir()?; 371 | let file = dir.path().join("sparse.bin"); 372 | let out = Command::new("/usr/bin/truncate") 373 | .args(["-s", "1M", file.to_str().unwrap()]) 374 | .output()?; 375 | assert!(out.status.success()); 376 | { 377 | let mut fd = OpenOptions::new().write(true).append(false).open(&file)?; 378 | let s = "x".repeat(512*1024); 379 | fd.write_all(s.as_bytes())?; 380 | assert!(probably_sparse(&fd)?); 381 | } 382 | 383 | Ok(()) 384 | } 385 | 386 | #[test] 387 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 388 | fn test_copy_bytes_sparse() -> Result<()> { 389 | let dir = tempdir()?; 390 | let file = dir.path().join("sparse.bin"); 391 | let from = dir.path().join("from.txt"); 392 | let data = "test data"; 393 | 394 | { 395 | let mut fd = File::create(&from)?; 396 | write!(fd, "{data}")?; 397 | } 398 | 399 | let out = Command::new("/usr/bin/truncate") 400 | .args(["-s", "1M", file.to_str().unwrap()]) 401 | .output()?; 402 | assert!(out.status.success()); 403 | 404 | { 405 | let infd = File::open(&from)?; 406 | let outfd: File = OpenOptions::new().write(true).append(false).open(&file)?; 407 | copy_file_bytes(&infd, &outfd, data.len() as u64)?; 408 | } 409 | 410 | assert!(probably_sparse(&File::open(file)?)?); 411 | 412 | Ok(()) 413 | } 414 | 415 | #[test] 416 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 417 | fn test_sparse_copy_middle() -> Result<()> { 418 | let dir = tempdir()?; 419 | let file = dir.path().join("sparse.bin"); 420 | let from = dir.path().join("from.txt"); 421 | let data = "test data"; 422 | 423 | { 424 | let mut fd = File::create(&from)?; 425 | write!(fd, "{data}")?; 426 | } 427 | 428 | let out = Command::new("/usr/bin/truncate") 429 | .args(["-s", "1M", file.to_str().unwrap()]) 430 | .output()?; 431 | assert!(out.status.success()); 432 | 433 | let offset = 512 * 1024; 434 | { 435 | let infd = File::open(&from)?; 436 | let outfd: File = OpenOptions::new().write(true).append(false).open(&file)?; 437 | let mut off_in = 0; 438 | let mut off_out = offset as u64; 439 | let copied = copy_file_range( 440 | &infd, 441 | Some(&mut off_in), 442 | &outfd, 443 | Some(&mut off_out), 444 | data.len(), 445 | )?; 446 | assert_eq!(copied as usize, data.len()); 447 | } 448 | 449 | assert!(probably_sparse(&File::open(&file)?)?); 450 | 451 | let bytes = read(&file)?; 452 | 453 | assert!(bytes.len() == 1024 * 1024); 454 | assert!(bytes[offset] == b't'); 455 | assert!(bytes[offset + 1] == b'e'); 456 | assert!(bytes[offset + 2] == b's'); 457 | assert!(bytes[offset + 3] == b't'); 458 | assert!(bytes[offset + data.len()] == 0); 459 | 460 | Ok(()) 461 | } 462 | 463 | #[test] 464 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 465 | fn test_copy_range_middle() -> Result<()> { 466 | let dir = tempdir()?; 467 | let file = dir.path().join("sparse.bin"); 468 | let from = dir.path().join("from.txt"); 469 | let data = "test data"; 470 | let offset: usize = 512 * 1024; 471 | 472 | { 473 | let mut fd = File::create(&from)?; 474 | fd.seek(io::SeekFrom::Start(offset as u64))?; 475 | write!(fd, "{data}")?; 476 | } 477 | 478 | let out = Command::new("/usr/bin/truncate") 479 | .args(["-s", "1M", file.to_str().unwrap()]) 480 | .output()?; 481 | assert!(out.status.success()); 482 | 483 | { 484 | let infd = File::open(&from)?; 485 | let outfd: File = OpenOptions::new().write(true).append(false).open(&file)?; 486 | let copied = 487 | copy_file_offset(&infd, &outfd, data.len() as u64, offset as i64)?; 488 | assert_eq!(copied as usize, data.len()); 489 | } 490 | 491 | assert!(probably_sparse(&File::open(&file)?)?); 492 | 493 | let bytes = read(&file)?; 494 | assert_eq!(bytes.len(), 1024 * 1024); 495 | assert_eq!(bytes[offset], b't'); 496 | assert_eq!(bytes[offset + 1], b'e'); 497 | assert_eq!(bytes[offset + 2], b's'); 498 | assert_eq!(bytes[offset + 3], b't'); 499 | assert_eq!(bytes[offset + data.len()], 0); 500 | 501 | Ok(()) 502 | } 503 | 504 | #[test] 505 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 506 | fn test_lseek_data() -> Result<()> { 507 | let dir = tempdir()?; 508 | let file = dir.path().join("sparse.bin"); 509 | let from = dir.path().join("from.txt"); 510 | let data = "test data"; 511 | let offset = 512 * 1024; 512 | 513 | { 514 | let mut fd = File::create(&from)?; 515 | write!(fd, "{data}")?; 516 | } 517 | 518 | let out = Command::new("/usr/bin/truncate") 519 | .args(["-s", "1M", file.to_str().unwrap()]) 520 | .output()?; 521 | assert!(out.status.success()); 522 | { 523 | let infd = File::open(&from)?; 524 | let outfd: File = OpenOptions::new().write(true).append(false).open(&file)?; 525 | let mut off_in = 0; 526 | let mut off_out = offset; 527 | let copied = copy_file_range( 528 | &infd, 529 | Some(&mut off_in), 530 | &outfd, 531 | Some(&mut off_out), 532 | data.len(), 533 | )?; 534 | assert_eq!(copied as usize, data.len()); 535 | } 536 | 537 | assert!(probably_sparse(&File::open(&file)?)?); 538 | 539 | let off = lseek(&File::open(&file)?, SeekFrom::Data(0))?; 540 | assert_eq!(off, SeekOff::Offset(offset)); 541 | 542 | Ok(()) 543 | } 544 | 545 | #[test] 546 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 547 | fn test_sparse_rust_seek() -> Result<()> { 548 | let dir = tempdir()?; 549 | let file = dir.path().join("sparse.bin"); 550 | 551 | let data = "c00lc0d3"; 552 | 553 | { 554 | let mut fd = File::create(&file)?; 555 | write!(fd, "{data}")?; 556 | 557 | fd.seek(io::SeekFrom::Start(1024 * 4096))?; 558 | write!(fd, "{data}")?; 559 | 560 | fd.seek(io::SeekFrom::Start(4096 * 4096 - data.len() as u64))?; 561 | write!(fd, "{data}")?; 562 | } 563 | 564 | assert!(probably_sparse(&File::open(&file)?)?); 565 | 566 | let bytes = read(&file)?; 567 | assert!(bytes.len() == 4096 * 4096); 568 | 569 | let offset = 1024 * 4096; 570 | assert!(bytes[offset] == b'c'); 571 | assert!(bytes[offset + 1] == b'0'); 572 | assert!(bytes[offset + 2] == b'0'); 573 | assert!(bytes[offset + 3] == b'l'); 574 | assert!(bytes[offset + data.len()] == 0); 575 | 576 | Ok(()) 577 | } 578 | 579 | #[test] 580 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 581 | fn test_lseek_no_data() -> Result<()> { 582 | let dir = tempdir()?; 583 | let file = dir.path().join("sparse.bin"); 584 | 585 | let out = Command::new("/usr/bin/truncate") 586 | .args(["-s", "1M", file.to_str().unwrap()]) 587 | .output()?; 588 | assert!(out.status.success()); 589 | assert!(probably_sparse(&File::open(&file)?)?); 590 | 591 | let fd = File::open(&file)?; 592 | let off = lseek(&fd, SeekFrom::Data(0))?; 593 | assert!(off == SeekOff::EOF); 594 | 595 | Ok(()) 596 | } 597 | 598 | #[test] 599 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 600 | fn test_allocate_file_is_sparse() -> Result<()> { 601 | let dir = tempdir()?; 602 | let file = dir.path().join("sparse.bin"); 603 | let len = 32 * 1024 * 1024; 604 | 605 | { 606 | let fd = File::create(&file)?; 607 | allocate_file(&fd, len)?; 608 | } 609 | 610 | assert_eq!(len, file.metadata()?.len()); 611 | assert!(probably_sparse(&File::open(&file)?)?); 612 | 613 | Ok(()) 614 | } 615 | 616 | #[test] 617 | #[cfg_attr(feature = "test_no_extents", ignore = "No FS support")] 618 | fn test_empty_extent() -> Result<()> { 619 | let dir = tempdir()?; 620 | let file = dir.path().join("sparse.bin"); 621 | 622 | let out = Command::new("/usr/bin/truncate") 623 | .args(["-s", "1M", file.to_str().unwrap()]) 624 | .output()?; 625 | assert!(out.status.success()); 626 | 627 | let fd = File::open(file)?; 628 | 629 | let extents_p = map_extents(&fd)?; 630 | assert!(extents_p.is_some()); 631 | let extents = extents_p.unwrap(); 632 | assert_eq!(extents.len(), 0); 633 | 634 | Ok(()) 635 | } 636 | 637 | #[test] 638 | #[cfg_attr(feature = "test_no_extents", ignore = "No FS support")] 639 | fn test_extent_fetch() -> Result<()> { 640 | let dir = tempdir()?; 641 | let file = dir.path().join("sparse.bin"); 642 | let from = dir.path().join("from.txt"); 643 | let data = "test data"; 644 | 645 | { 646 | let mut fd = File::create(&from)?; 647 | write!(fd, "{data}")?; 648 | } 649 | 650 | let out = Command::new("/usr/bin/truncate") 651 | .args(["-s", "1M", file.to_str().unwrap()]) 652 | .output()?; 653 | assert!(out.status.success()); 654 | 655 | let offset = 512 * 1024; 656 | { 657 | let infd = File::open(&from)?; 658 | let outfd: File = OpenOptions::new().write(true).append(false).open(&file)?; 659 | let mut off_in = 0; 660 | let mut off_out = offset; 661 | let copied = copy_file_range( 662 | &infd, 663 | Some(&mut off_in), 664 | &outfd, 665 | Some(&mut off_out), 666 | data.len(), 667 | )?; 668 | assert_eq!(copied as usize, data.len()); 669 | } 670 | 671 | let fd = File::open(file)?; 672 | 673 | let extents_p = map_extents(&fd)?; 674 | assert!(extents_p.is_some()); 675 | let extents = extents_p.unwrap(); 676 | assert_eq!(extents.len(), 1); 677 | assert_eq!(extents[0].start, offset); 678 | assert_eq!(extents[0].end, offset + 4 * 1024); // FIXME: Assume 4k blocks 679 | assert!(!extents[0].shared); 680 | 681 | Ok(()) 682 | } 683 | 684 | #[test] 685 | #[cfg_attr(feature = "test_no_extents", ignore = "No FS support")] 686 | fn test_extent_fetch_many() -> Result<()> { 687 | let dir = tempdir()?; 688 | let file = dir.path().join("sparse.bin"); 689 | 690 | let out = Command::new("/usr/bin/truncate") 691 | .args(["-s", "1M", file.to_str().unwrap()]) 692 | .output()?; 693 | assert!(out.status.success()); 694 | 695 | let fsize = 1024 * 1024; 696 | // FIXME: Assumes 4k blocks 697 | let bsize = 4 * 1024; 698 | let block = iter::repeat_n(0xff_u8, bsize).collect::>(); 699 | 700 | let mut fd = OpenOptions::new().write(true).append(false).open(&file)?; 701 | // Skip every-other block 702 | for off in (0..fsize).step_by(bsize * 2) { 703 | lseek(&fd, SeekFrom::Start(off))?; 704 | fd.write_all(block.as_slice())?; 705 | } 706 | 707 | let extents_p = map_extents(&fd)?; 708 | assert!(extents_p.is_some()); 709 | let extents = extents_p.unwrap(); 710 | assert_eq!(extents.len(), fsize as usize / bsize / 2); 711 | 712 | Ok(()) 713 | } 714 | 715 | #[test] 716 | #[cfg_attr(feature = "test_no_extents", ignore = "No FS support")] 717 | fn test_extent_not_sparse() -> Result<()> { 718 | let dir = tempdir()?; 719 | let file = dir.path().join("file.bin"); 720 | let size = 128 * 1024; 721 | 722 | { 723 | let mut fd: File = File::create(&file)?; 724 | let data = "X".repeat(size); 725 | write!(fd, "{data}")?; 726 | } 727 | 728 | let fd = File::open(file)?; 729 | let extents_p = map_extents(&fd)?; 730 | assert!(extents_p.is_some()); 731 | let extents = extents_p.unwrap(); 732 | 733 | assert_eq!(1, extents.len()); 734 | assert_eq!(0u64, extents[0].start); 735 | assert_eq!(size as u64, extents[0].end); 736 | 737 | Ok(()) 738 | } 739 | 740 | #[test] 741 | #[cfg_attr(feature = "test_no_extents", ignore = "No FS support")] 742 | fn test_extent_unsupported_fs() -> Result<()> { 743 | let file = "/proc/cpuinfo"; 744 | let fd = File::open(file)?; 745 | let extents_p = map_extents(&fd)?; 746 | assert!(extents_p.is_none()); 747 | 748 | Ok(()) 749 | } 750 | 751 | #[test] 752 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 753 | fn test_copy_file_sparse() -> Result<()> { 754 | let dir = tempdir()?; 755 | let from = dir.path().join("sparse.bin"); 756 | let len = 32 * 1024 * 1024; 757 | 758 | { 759 | let fd = File::create(&from)?; 760 | allocate_file(&fd, len)?; 761 | } 762 | 763 | assert_eq!(len, from.metadata()?.len()); 764 | assert!(probably_sparse(&File::open(&from)?)?); 765 | 766 | let to = dir.path().join("sparse.copy.bin"); 767 | crate::copy_file(&from, &to)?; 768 | 769 | assert_eq!(len, to.metadata()?.len()); 770 | assert!(probably_sparse(&File::open(&to)?)?); 771 | 772 | Ok(()) 773 | } 774 | 775 | #[test] 776 | #[cfg_attr(feature = "test_no_sockets", ignore = "No FS support")] 777 | fn test_copy_socket() { 778 | let dir = tempdir().unwrap(); 779 | let from = dir.path().join("from.sock"); 780 | let to = dir.path().join("to.sock"); 781 | 782 | let _sock = UnixListener::bind(&from).unwrap(); 783 | assert!(from.metadata().unwrap().file_type().is_socket()); 784 | 785 | copy_node(&from, &to).unwrap(); 786 | 787 | assert!(to.exists()); 788 | assert!(to.metadata().unwrap().file_type().is_socket()); 789 | } 790 | 791 | 792 | #[test] 793 | #[cfg_attr(feature = "test_no_acl", ignore = "No FS support")] 794 | fn test_copy_acl() -> Result<()> { 795 | use exacl::{getfacl, AclEntry, Perm, setfacl}; 796 | 797 | let dir = tempdir()?; 798 | let from = dir.path().join("file.bin"); 799 | let to = dir.path().join("copy.bin"); 800 | let data = "X".repeat(1024); 801 | 802 | { 803 | let mut fd: File = File::create(&from)?; 804 | write!(fd, "{data}")?; 805 | 806 | let mut fd: File = File::create(&to)?; 807 | write!(fd, "{data}")?; 808 | } 809 | 810 | let acl = AclEntry::allow_user("mail", Perm::READ, None); 811 | 812 | let mut from_acl = getfacl(&from, None)?; 813 | from_acl.push(acl.clone()); 814 | setfacl(&[&from], &from_acl, None)?; 815 | 816 | { 817 | let from_fd: File = File::open(&from)?; 818 | let to_fd: File = File::open(&to)?; 819 | copy_permissions(&from_fd, &to_fd)?; 820 | } 821 | 822 | let to_acl = getfacl(&from, None)?; 823 | assert!(to_acl.contains(&acl)); 824 | 825 | Ok(()) 826 | } 827 | } 828 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 4 4 | 5 | [[package]] 6 | name = "aho-corasick" 7 | version = "1.1.3" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" 10 | dependencies = [ 11 | "memchr", 12 | ] 13 | 14 | [[package]] 15 | name = "anstream" 16 | version = "0.6.20" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192" 19 | dependencies = [ 20 | "anstyle", 21 | "anstyle-parse", 22 | "anstyle-query", 23 | "anstyle-wincon", 24 | "colorchoice", 25 | "is_terminal_polyfill", 26 | "utf8parse", 27 | ] 28 | 29 | [[package]] 30 | name = "anstyle" 31 | version = "1.0.11" 32 | source = "registry+https://github.com/rust-lang/crates.io-index" 33 | checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" 34 | 35 | [[package]] 36 | name = "anstyle-parse" 37 | version = "0.2.7" 38 | source = "registry+https://github.com/rust-lang/crates.io-index" 39 | checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" 40 | dependencies = [ 41 | "utf8parse", 42 | ] 43 | 44 | [[package]] 45 | name = "anstyle-query" 46 | version = "1.1.4" 47 | source = "registry+https://github.com/rust-lang/crates.io-index" 48 | checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" 49 | dependencies = [ 50 | "windows-sys 0.60.2", 51 | ] 52 | 53 | [[package]] 54 | name = "anstyle-wincon" 55 | version = "3.0.10" 56 | source = "registry+https://github.com/rust-lang/crates.io-index" 57 | checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" 58 | dependencies = [ 59 | "anstyle", 60 | "once_cell_polyfill", 61 | "windows-sys 0.60.2", 62 | ] 63 | 64 | [[package]] 65 | name = "anyhow" 66 | version = "1.0.99" 67 | source = "registry+https://github.com/rust-lang/crates.io-index" 68 | checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" 69 | 70 | [[package]] 71 | name = "autocfg" 72 | version = "1.4.0" 73 | source = "registry+https://github.com/rust-lang/crates.io-index" 74 | checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" 75 | 76 | [[package]] 77 | name = "bitflags" 78 | version = "2.9.2" 79 | source = "registry+https://github.com/rust-lang/crates.io-index" 80 | checksum = "6a65b545ab31d687cff52899d4890855fec459eb6afe0da6417b8a18da87aa29" 81 | 82 | [[package]] 83 | name = "blocking-threadpool" 84 | version = "1.0.1" 85 | source = "registry+https://github.com/rust-lang/crates.io-index" 86 | checksum = "c4ba4d6edfe07b0a4940ab5c05a7114155ffbe9d0c64df7a2e39cb002f879869" 87 | dependencies = [ 88 | "crossbeam-channel", 89 | "num_cpus", 90 | ] 91 | 92 | [[package]] 93 | name = "bstr" 94 | version = "1.10.0" 95 | source = "registry+https://github.com/rust-lang/crates.io-index" 96 | checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" 97 | dependencies = [ 98 | "memchr", 99 | "serde", 100 | ] 101 | 102 | [[package]] 103 | name = "bumpalo" 104 | version = "3.19.0" 105 | source = "registry+https://github.com/rust-lang/crates.io-index" 106 | checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" 107 | 108 | [[package]] 109 | name = "cfg-if" 110 | version = "1.0.3" 111 | source = "registry+https://github.com/rust-lang/crates.io-index" 112 | checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" 113 | 114 | [[package]] 115 | name = "clap" 116 | version = "4.5.46" 117 | source = "registry+https://github.com/rust-lang/crates.io-index" 118 | checksum = "2c5e4fcf9c21d2e544ca1ee9d8552de13019a42aa7dbf32747fa7aaf1df76e57" 119 | dependencies = [ 120 | "clap_builder", 121 | "clap_derive", 122 | ] 123 | 124 | [[package]] 125 | name = "clap_builder" 126 | version = "4.5.46" 127 | source = "registry+https://github.com/rust-lang/crates.io-index" 128 | checksum = "fecb53a0e6fcfb055f686001bc2e2592fa527efaf38dbe81a6a9563562e57d41" 129 | dependencies = [ 130 | "anstream", 131 | "anstyle", 132 | "clap_lex", 133 | "strsim", 134 | ] 135 | 136 | [[package]] 137 | name = "clap_derive" 138 | version = "4.5.45" 139 | source = "registry+https://github.com/rust-lang/crates.io-index" 140 | checksum = "14cb31bb0a7d536caef2639baa7fad459e15c3144efefa6dbd1c84562c4739f6" 141 | dependencies = [ 142 | "heck", 143 | "proc-macro2", 144 | "quote", 145 | "syn", 146 | ] 147 | 148 | [[package]] 149 | name = "clap_lex" 150 | version = "0.7.5" 151 | source = "registry+https://github.com/rust-lang/crates.io-index" 152 | checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" 153 | 154 | [[package]] 155 | name = "colorchoice" 156 | version = "1.0.4" 157 | source = "registry+https://github.com/rust-lang/crates.io-index" 158 | checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" 159 | 160 | [[package]] 161 | name = "console" 162 | version = "0.16.0" 163 | source = "registry+https://github.com/rust-lang/crates.io-index" 164 | checksum = "2e09ced7ebbccb63b4c65413d821f2e00ce54c5ca4514ddc6b3c892fdbcbc69d" 165 | dependencies = [ 166 | "encode_unicode", 167 | "libc", 168 | "once_cell", 169 | "unicode-width", 170 | "windows-sys 0.60.2", 171 | ] 172 | 173 | [[package]] 174 | name = "crossbeam-channel" 175 | version = "0.5.15" 176 | source = "registry+https://github.com/rust-lang/crates.io-index" 177 | checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" 178 | dependencies = [ 179 | "crossbeam-utils", 180 | ] 181 | 182 | [[package]] 183 | name = "crossbeam-deque" 184 | version = "0.8.5" 185 | source = "registry+https://github.com/rust-lang/crates.io-index" 186 | checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" 187 | dependencies = [ 188 | "crossbeam-epoch", 189 | "crossbeam-utils", 190 | ] 191 | 192 | [[package]] 193 | name = "crossbeam-epoch" 194 | version = "0.9.18" 195 | source = "registry+https://github.com/rust-lang/crates.io-index" 196 | checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" 197 | dependencies = [ 198 | "crossbeam-utils", 199 | ] 200 | 201 | [[package]] 202 | name = "crossbeam-utils" 203 | version = "0.8.21" 204 | source = "registry+https://github.com/rust-lang/crates.io-index" 205 | checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" 206 | 207 | [[package]] 208 | name = "deranged" 209 | version = "0.3.11" 210 | source = "registry+https://github.com/rust-lang/crates.io-index" 211 | checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" 212 | dependencies = [ 213 | "powerfmt", 214 | ] 215 | 216 | [[package]] 217 | name = "encode_unicode" 218 | version = "1.0.0" 219 | source = "registry+https://github.com/rust-lang/crates.io-index" 220 | checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" 221 | 222 | [[package]] 223 | name = "errno" 224 | version = "0.3.13" 225 | source = "registry+https://github.com/rust-lang/crates.io-index" 226 | checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" 227 | dependencies = [ 228 | "libc", 229 | "windows-sys 0.52.0", 230 | ] 231 | 232 | [[package]] 233 | name = "exacl" 234 | version = "0.12.0" 235 | source = "registry+https://github.com/rust-lang/crates.io-index" 236 | checksum = "22be12de19decddab85d09f251ec8363f060ccb22ec9c81bc157c0c8433946d8" 237 | dependencies = [ 238 | "bitflags", 239 | "log", 240 | "scopeguard", 241 | "uuid", 242 | ] 243 | 244 | [[package]] 245 | name = "fastrand" 246 | version = "2.3.0" 247 | source = "registry+https://github.com/rust-lang/crates.io-index" 248 | checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" 249 | 250 | [[package]] 251 | name = "fslock" 252 | version = "0.2.1" 253 | source = "registry+https://github.com/rust-lang/crates.io-index" 254 | checksum = "04412b8935272e3a9bae6f48c7bfff74c2911f60525404edfdd28e49884c3bfb" 255 | dependencies = [ 256 | "libc", 257 | "winapi", 258 | ] 259 | 260 | [[package]] 261 | name = "getrandom" 262 | version = "0.3.3" 263 | source = "registry+https://github.com/rust-lang/crates.io-index" 264 | checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" 265 | dependencies = [ 266 | "cfg-if", 267 | "libc", 268 | "r-efi", 269 | "wasi", 270 | ] 271 | 272 | [[package]] 273 | name = "glob" 274 | version = "0.3.3" 275 | source = "registry+https://github.com/rust-lang/crates.io-index" 276 | checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" 277 | 278 | [[package]] 279 | name = "globset" 280 | version = "0.4.15" 281 | source = "registry+https://github.com/rust-lang/crates.io-index" 282 | checksum = "15f1ce686646e7f1e19bf7d5533fe443a45dbfb990e00629110797578b42fb19" 283 | dependencies = [ 284 | "aho-corasick", 285 | "bstr", 286 | "log", 287 | "regex-automata", 288 | "regex-syntax", 289 | ] 290 | 291 | [[package]] 292 | name = "heck" 293 | version = "0.5.0" 294 | source = "registry+https://github.com/rust-lang/crates.io-index" 295 | checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" 296 | 297 | [[package]] 298 | name = "hermit-abi" 299 | version = "0.5.2" 300 | source = "registry+https://github.com/rust-lang/crates.io-index" 301 | checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" 302 | 303 | [[package]] 304 | name = "ignore" 305 | version = "0.4.23" 306 | source = "registry+https://github.com/rust-lang/crates.io-index" 307 | checksum = "6d89fd380afde86567dfba715db065673989d6253f42b88179abd3eae47bda4b" 308 | dependencies = [ 309 | "crossbeam-deque", 310 | "globset", 311 | "log", 312 | "memchr", 313 | "regex-automata", 314 | "same-file", 315 | "walkdir", 316 | "winapi-util", 317 | ] 318 | 319 | [[package]] 320 | name = "indicatif" 321 | version = "0.18.0" 322 | source = "registry+https://github.com/rust-lang/crates.io-index" 323 | checksum = "70a646d946d06bedbbc4cac4c218acf4bbf2d87757a784857025f4d447e4e1cd" 324 | dependencies = [ 325 | "console", 326 | "portable-atomic", 327 | "unicode-width", 328 | "unit-prefix", 329 | "web-time", 330 | ] 331 | 332 | [[package]] 333 | name = "is_terminal_polyfill" 334 | version = "1.70.1" 335 | source = "registry+https://github.com/rust-lang/crates.io-index" 336 | checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" 337 | 338 | [[package]] 339 | name = "itoa" 340 | version = "1.0.11" 341 | source = "registry+https://github.com/rust-lang/crates.io-index" 342 | checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" 343 | 344 | [[package]] 345 | name = "js-sys" 346 | version = "0.3.77" 347 | source = "registry+https://github.com/rust-lang/crates.io-index" 348 | checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" 349 | dependencies = [ 350 | "once_cell", 351 | "wasm-bindgen", 352 | ] 353 | 354 | [[package]] 355 | name = "lazy_static" 356 | version = "0.2.11" 357 | source = "registry+https://github.com/rust-lang/crates.io-index" 358 | checksum = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" 359 | 360 | [[package]] 361 | name = "libc" 362 | version = "0.2.175" 363 | source = "registry+https://github.com/rust-lang/crates.io-index" 364 | checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" 365 | 366 | [[package]] 367 | name = "libfs" 368 | version = "0.9.2" 369 | dependencies = [ 370 | "cfg-if", 371 | "exacl", 372 | "libc", 373 | "linux-raw-sys 0.10.0", 374 | "log", 375 | "rustix", 376 | "tempfile", 377 | "thiserror", 378 | "xattr", 379 | ] 380 | 381 | [[package]] 382 | name = "libm" 383 | version = "0.2.11" 384 | source = "registry+https://github.com/rust-lang/crates.io-index" 385 | checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" 386 | 387 | [[package]] 388 | name = "libxcp" 389 | version = "0.24.2" 390 | dependencies = [ 391 | "anyhow", 392 | "blocking-threadpool", 393 | "cfg-if", 394 | "crossbeam-channel", 395 | "ignore", 396 | "libfs", 397 | "log", 398 | "num_cpus", 399 | "regex", 400 | "tempfile", 401 | "thiserror", 402 | "walkdir", 403 | ] 404 | 405 | [[package]] 406 | name = "linux-raw-sys" 407 | version = "0.9.4" 408 | source = "registry+https://github.com/rust-lang/crates.io-index" 409 | checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" 410 | 411 | [[package]] 412 | name = "linux-raw-sys" 413 | version = "0.10.0" 414 | source = "registry+https://github.com/rust-lang/crates.io-index" 415 | checksum = "13d6a630ed4f43c11056af8768c4773df2c43bc780b6d8a46de345c17236c562" 416 | 417 | [[package]] 418 | name = "log" 419 | version = "0.4.27" 420 | source = "registry+https://github.com/rust-lang/crates.io-index" 421 | checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" 422 | 423 | [[package]] 424 | name = "memchr" 425 | version = "2.7.5" 426 | source = "registry+https://github.com/rust-lang/crates.io-index" 427 | checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" 428 | 429 | [[package]] 430 | name = "num-conv" 431 | version = "0.1.0" 432 | source = "registry+https://github.com/rust-lang/crates.io-index" 433 | checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" 434 | 435 | [[package]] 436 | name = "num-traits" 437 | version = "0.2.19" 438 | source = "registry+https://github.com/rust-lang/crates.io-index" 439 | checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" 440 | dependencies = [ 441 | "autocfg", 442 | "libm", 443 | ] 444 | 445 | [[package]] 446 | name = "num_cpus" 447 | version = "1.17.0" 448 | source = "registry+https://github.com/rust-lang/crates.io-index" 449 | checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" 450 | dependencies = [ 451 | "hermit-abi", 452 | "libc", 453 | ] 454 | 455 | [[package]] 456 | name = "num_threads" 457 | version = "0.1.7" 458 | source = "registry+https://github.com/rust-lang/crates.io-index" 459 | checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" 460 | dependencies = [ 461 | "libc", 462 | ] 463 | 464 | [[package]] 465 | name = "once_cell" 466 | version = "1.21.3" 467 | source = "registry+https://github.com/rust-lang/crates.io-index" 468 | checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" 469 | 470 | [[package]] 471 | name = "once_cell_polyfill" 472 | version = "1.70.1" 473 | source = "registry+https://github.com/rust-lang/crates.io-index" 474 | checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" 475 | 476 | [[package]] 477 | name = "portable-atomic" 478 | version = "1.11.1" 479 | source = "registry+https://github.com/rust-lang/crates.io-index" 480 | checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" 481 | 482 | [[package]] 483 | name = "powerfmt" 484 | version = "0.2.0" 485 | source = "registry+https://github.com/rust-lang/crates.io-index" 486 | checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" 487 | 488 | [[package]] 489 | name = "ppv-lite86" 490 | version = "0.2.21" 491 | source = "registry+https://github.com/rust-lang/crates.io-index" 492 | checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" 493 | dependencies = [ 494 | "zerocopy", 495 | ] 496 | 497 | [[package]] 498 | name = "proc-macro2" 499 | version = "1.0.101" 500 | source = "registry+https://github.com/rust-lang/crates.io-index" 501 | checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" 502 | dependencies = [ 503 | "unicode-ident", 504 | ] 505 | 506 | [[package]] 507 | name = "quote" 508 | version = "1.0.40" 509 | source = "registry+https://github.com/rust-lang/crates.io-index" 510 | checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" 511 | dependencies = [ 512 | "proc-macro2", 513 | ] 514 | 515 | [[package]] 516 | name = "r-efi" 517 | version = "5.3.0" 518 | source = "registry+https://github.com/rust-lang/crates.io-index" 519 | checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" 520 | 521 | [[package]] 522 | name = "rand" 523 | version = "0.9.2" 524 | source = "registry+https://github.com/rust-lang/crates.io-index" 525 | checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" 526 | dependencies = [ 527 | "rand_chacha", 528 | "rand_core", 529 | ] 530 | 531 | [[package]] 532 | name = "rand_chacha" 533 | version = "0.9.0" 534 | source = "registry+https://github.com/rust-lang/crates.io-index" 535 | checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" 536 | dependencies = [ 537 | "ppv-lite86", 538 | "rand_core", 539 | ] 540 | 541 | [[package]] 542 | name = "rand_core" 543 | version = "0.9.3" 544 | source = "registry+https://github.com/rust-lang/crates.io-index" 545 | checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" 546 | dependencies = [ 547 | "getrandom", 548 | ] 549 | 550 | [[package]] 551 | name = "rand_distr" 552 | version = "0.5.1" 553 | source = "registry+https://github.com/rust-lang/crates.io-index" 554 | checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" 555 | dependencies = [ 556 | "num-traits", 557 | "rand", 558 | ] 559 | 560 | [[package]] 561 | name = "rand_xorshift" 562 | version = "0.4.0" 563 | source = "registry+https://github.com/rust-lang/crates.io-index" 564 | checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" 565 | dependencies = [ 566 | "rand_core", 567 | ] 568 | 569 | [[package]] 570 | name = "regex" 571 | version = "1.11.2" 572 | source = "registry+https://github.com/rust-lang/crates.io-index" 573 | checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" 574 | dependencies = [ 575 | "aho-corasick", 576 | "memchr", 577 | "regex-automata", 578 | "regex-syntax", 579 | ] 580 | 581 | [[package]] 582 | name = "regex-automata" 583 | version = "0.4.10" 584 | source = "registry+https://github.com/rust-lang/crates.io-index" 585 | checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" 586 | dependencies = [ 587 | "aho-corasick", 588 | "memchr", 589 | "regex-syntax", 590 | ] 591 | 592 | [[package]] 593 | name = "regex-syntax" 594 | version = "0.8.6" 595 | source = "registry+https://github.com/rust-lang/crates.io-index" 596 | checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" 597 | 598 | [[package]] 599 | name = "rustix" 600 | version = "1.0.8" 601 | source = "registry+https://github.com/rust-lang/crates.io-index" 602 | checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" 603 | dependencies = [ 604 | "bitflags", 605 | "errno", 606 | "libc", 607 | "linux-raw-sys 0.9.4", 608 | "windows-sys 0.52.0", 609 | ] 610 | 611 | [[package]] 612 | name = "rustversion" 613 | version = "1.0.21" 614 | source = "registry+https://github.com/rust-lang/crates.io-index" 615 | checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" 616 | 617 | [[package]] 618 | name = "same-file" 619 | version = "1.0.6" 620 | source = "registry+https://github.com/rust-lang/crates.io-index" 621 | checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" 622 | dependencies = [ 623 | "winapi-util", 624 | ] 625 | 626 | [[package]] 627 | name = "scopeguard" 628 | version = "1.2.0" 629 | source = "registry+https://github.com/rust-lang/crates.io-index" 630 | checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" 631 | 632 | [[package]] 633 | name = "serde" 634 | version = "1.0.210" 635 | source = "registry+https://github.com/rust-lang/crates.io-index" 636 | checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" 637 | dependencies = [ 638 | "serde_derive", 639 | ] 640 | 641 | [[package]] 642 | name = "serde_derive" 643 | version = "1.0.210" 644 | source = "registry+https://github.com/rust-lang/crates.io-index" 645 | checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" 646 | dependencies = [ 647 | "proc-macro2", 648 | "quote", 649 | "syn", 650 | ] 651 | 652 | [[package]] 653 | name = "simplelog" 654 | version = "0.12.2" 655 | source = "registry+https://github.com/rust-lang/crates.io-index" 656 | checksum = "16257adbfaef1ee58b1363bdc0664c9b8e1e30aed86049635fb5f147d065a9c0" 657 | dependencies = [ 658 | "log", 659 | "termcolor", 660 | "time", 661 | ] 662 | 663 | [[package]] 664 | name = "strsim" 665 | version = "0.11.1" 666 | source = "registry+https://github.com/rust-lang/crates.io-index" 667 | checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" 668 | 669 | [[package]] 670 | name = "syn" 671 | version = "2.0.106" 672 | source = "registry+https://github.com/rust-lang/crates.io-index" 673 | checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" 674 | dependencies = [ 675 | "proc-macro2", 676 | "quote", 677 | "unicode-ident", 678 | ] 679 | 680 | [[package]] 681 | name = "tempfile" 682 | version = "3.21.0" 683 | source = "registry+https://github.com/rust-lang/crates.io-index" 684 | checksum = "15b61f8f20e3a6f7e0649d825294eaf317edce30f82cf6026e7e4cb9222a7d1e" 685 | dependencies = [ 686 | "fastrand", 687 | "getrandom", 688 | "once_cell", 689 | "rustix", 690 | "windows-sys 0.52.0", 691 | ] 692 | 693 | [[package]] 694 | name = "termcolor" 695 | version = "1.4.1" 696 | source = "registry+https://github.com/rust-lang/crates.io-index" 697 | checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" 698 | dependencies = [ 699 | "winapi-util", 700 | ] 701 | 702 | [[package]] 703 | name = "terminal_size" 704 | version = "0.4.3" 705 | source = "registry+https://github.com/rust-lang/crates.io-index" 706 | checksum = "60b8cb979cb11c32ce1603f8137b22262a9d131aaa5c37b5678025f22b8becd0" 707 | dependencies = [ 708 | "rustix", 709 | "windows-sys 0.60.2", 710 | ] 711 | 712 | [[package]] 713 | name = "test-case" 714 | version = "3.3.1" 715 | source = "registry+https://github.com/rust-lang/crates.io-index" 716 | checksum = "eb2550dd13afcd286853192af8601920d959b14c401fcece38071d53bf0768a8" 717 | dependencies = [ 718 | "test-case-macros", 719 | ] 720 | 721 | [[package]] 722 | name = "test-case-core" 723 | version = "3.3.1" 724 | source = "registry+https://github.com/rust-lang/crates.io-index" 725 | checksum = "adcb7fd841cd518e279be3d5a3eb0636409487998a4aff22f3de87b81e88384f" 726 | dependencies = [ 727 | "cfg-if", 728 | "proc-macro2", 729 | "quote", 730 | "syn", 731 | ] 732 | 733 | [[package]] 734 | name = "test-case-macros" 735 | version = "3.3.1" 736 | source = "registry+https://github.com/rust-lang/crates.io-index" 737 | checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" 738 | dependencies = [ 739 | "proc-macro2", 740 | "quote", 741 | "syn", 742 | "test-case-core", 743 | ] 744 | 745 | [[package]] 746 | name = "thiserror" 747 | version = "2.0.16" 748 | source = "registry+https://github.com/rust-lang/crates.io-index" 749 | checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" 750 | dependencies = [ 751 | "thiserror-impl", 752 | ] 753 | 754 | [[package]] 755 | name = "thiserror-impl" 756 | version = "2.0.16" 757 | source = "registry+https://github.com/rust-lang/crates.io-index" 758 | checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" 759 | dependencies = [ 760 | "proc-macro2", 761 | "quote", 762 | "syn", 763 | ] 764 | 765 | [[package]] 766 | name = "time" 767 | version = "0.3.36" 768 | source = "registry+https://github.com/rust-lang/crates.io-index" 769 | checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" 770 | dependencies = [ 771 | "deranged", 772 | "itoa", 773 | "libc", 774 | "num-conv", 775 | "num_threads", 776 | "powerfmt", 777 | "serde", 778 | "time-core", 779 | "time-macros", 780 | ] 781 | 782 | [[package]] 783 | name = "time-core" 784 | version = "0.1.2" 785 | source = "registry+https://github.com/rust-lang/crates.io-index" 786 | checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" 787 | 788 | [[package]] 789 | name = "time-macros" 790 | version = "0.2.18" 791 | source = "registry+https://github.com/rust-lang/crates.io-index" 792 | checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" 793 | dependencies = [ 794 | "num-conv", 795 | "time-core", 796 | ] 797 | 798 | [[package]] 799 | name = "unbytify" 800 | version = "0.2.0" 801 | source = "registry+https://github.com/rust-lang/crates.io-index" 802 | checksum = "61f431354fd60c251d35ccc3d3ecf14e5f37e52ce807f6436f394fb3f0fc9869" 803 | dependencies = [ 804 | "lazy_static", 805 | ] 806 | 807 | [[package]] 808 | name = "unicode-ident" 809 | version = "1.0.18" 810 | source = "registry+https://github.com/rust-lang/crates.io-index" 811 | checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" 812 | 813 | [[package]] 814 | name = "unicode-width" 815 | version = "0.2.1" 816 | source = "registry+https://github.com/rust-lang/crates.io-index" 817 | checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" 818 | 819 | [[package]] 820 | name = "unit-prefix" 821 | version = "0.5.1" 822 | source = "registry+https://github.com/rust-lang/crates.io-index" 823 | checksum = "323402cff2dd658f39ca17c789b502021b3f18707c91cdf22e3838e1b4023817" 824 | 825 | [[package]] 826 | name = "utf8parse" 827 | version = "0.2.2" 828 | source = "registry+https://github.com/rust-lang/crates.io-index" 829 | checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" 830 | 831 | [[package]] 832 | name = "uuid" 833 | version = "1.18.0" 834 | source = "registry+https://github.com/rust-lang/crates.io-index" 835 | checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be" 836 | dependencies = [ 837 | "getrandom", 838 | "js-sys", 839 | "wasm-bindgen", 840 | ] 841 | 842 | [[package]] 843 | name = "walkdir" 844 | version = "2.5.0" 845 | source = "registry+https://github.com/rust-lang/crates.io-index" 846 | checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" 847 | dependencies = [ 848 | "same-file", 849 | "winapi-util", 850 | ] 851 | 852 | [[package]] 853 | name = "wasi" 854 | version = "0.14.2+wasi-0.2.4" 855 | source = "registry+https://github.com/rust-lang/crates.io-index" 856 | checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" 857 | dependencies = [ 858 | "wit-bindgen-rt", 859 | ] 860 | 861 | [[package]] 862 | name = "wasm-bindgen" 863 | version = "0.2.100" 864 | source = "registry+https://github.com/rust-lang/crates.io-index" 865 | checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" 866 | dependencies = [ 867 | "cfg-if", 868 | "once_cell", 869 | "rustversion", 870 | "wasm-bindgen-macro", 871 | ] 872 | 873 | [[package]] 874 | name = "wasm-bindgen-backend" 875 | version = "0.2.100" 876 | source = "registry+https://github.com/rust-lang/crates.io-index" 877 | checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" 878 | dependencies = [ 879 | "bumpalo", 880 | "log", 881 | "proc-macro2", 882 | "quote", 883 | "syn", 884 | "wasm-bindgen-shared", 885 | ] 886 | 887 | [[package]] 888 | name = "wasm-bindgen-macro" 889 | version = "0.2.100" 890 | source = "registry+https://github.com/rust-lang/crates.io-index" 891 | checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" 892 | dependencies = [ 893 | "quote", 894 | "wasm-bindgen-macro-support", 895 | ] 896 | 897 | [[package]] 898 | name = "wasm-bindgen-macro-support" 899 | version = "0.2.100" 900 | source = "registry+https://github.com/rust-lang/crates.io-index" 901 | checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" 902 | dependencies = [ 903 | "proc-macro2", 904 | "quote", 905 | "syn", 906 | "wasm-bindgen-backend", 907 | "wasm-bindgen-shared", 908 | ] 909 | 910 | [[package]] 911 | name = "wasm-bindgen-shared" 912 | version = "0.2.100" 913 | source = "registry+https://github.com/rust-lang/crates.io-index" 914 | checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" 915 | dependencies = [ 916 | "unicode-ident", 917 | ] 918 | 919 | [[package]] 920 | name = "web-time" 921 | version = "1.1.0" 922 | source = "registry+https://github.com/rust-lang/crates.io-index" 923 | checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" 924 | dependencies = [ 925 | "js-sys", 926 | "wasm-bindgen", 927 | ] 928 | 929 | [[package]] 930 | name = "winapi" 931 | version = "0.3.9" 932 | source = "registry+https://github.com/rust-lang/crates.io-index" 933 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 934 | dependencies = [ 935 | "winapi-i686-pc-windows-gnu", 936 | "winapi-x86_64-pc-windows-gnu", 937 | ] 938 | 939 | [[package]] 940 | name = "winapi-i686-pc-windows-gnu" 941 | version = "0.4.0" 942 | source = "registry+https://github.com/rust-lang/crates.io-index" 943 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 944 | 945 | [[package]] 946 | name = "winapi-util" 947 | version = "0.1.8" 948 | source = "registry+https://github.com/rust-lang/crates.io-index" 949 | checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" 950 | dependencies = [ 951 | "windows-sys 0.52.0", 952 | ] 953 | 954 | [[package]] 955 | name = "winapi-x86_64-pc-windows-gnu" 956 | version = "0.4.0" 957 | source = "registry+https://github.com/rust-lang/crates.io-index" 958 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 959 | 960 | [[package]] 961 | name = "windows-link" 962 | version = "0.1.3" 963 | source = "registry+https://github.com/rust-lang/crates.io-index" 964 | checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" 965 | 966 | [[package]] 967 | name = "windows-sys" 968 | version = "0.52.0" 969 | source = "registry+https://github.com/rust-lang/crates.io-index" 970 | checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" 971 | dependencies = [ 972 | "windows-targets 0.52.6", 973 | ] 974 | 975 | [[package]] 976 | name = "windows-sys" 977 | version = "0.60.2" 978 | source = "registry+https://github.com/rust-lang/crates.io-index" 979 | checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" 980 | dependencies = [ 981 | "windows-targets 0.53.3", 982 | ] 983 | 984 | [[package]] 985 | name = "windows-targets" 986 | version = "0.52.6" 987 | source = "registry+https://github.com/rust-lang/crates.io-index" 988 | checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" 989 | dependencies = [ 990 | "windows_aarch64_gnullvm 0.52.6", 991 | "windows_aarch64_msvc 0.52.6", 992 | "windows_i686_gnu 0.52.6", 993 | "windows_i686_gnullvm 0.52.6", 994 | "windows_i686_msvc 0.52.6", 995 | "windows_x86_64_gnu 0.52.6", 996 | "windows_x86_64_gnullvm 0.52.6", 997 | "windows_x86_64_msvc 0.52.6", 998 | ] 999 | 1000 | [[package]] 1001 | name = "windows-targets" 1002 | version = "0.53.3" 1003 | source = "registry+https://github.com/rust-lang/crates.io-index" 1004 | checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" 1005 | dependencies = [ 1006 | "windows-link", 1007 | "windows_aarch64_gnullvm 0.53.0", 1008 | "windows_aarch64_msvc 0.53.0", 1009 | "windows_i686_gnu 0.53.0", 1010 | "windows_i686_gnullvm 0.53.0", 1011 | "windows_i686_msvc 0.53.0", 1012 | "windows_x86_64_gnu 0.53.0", 1013 | "windows_x86_64_gnullvm 0.53.0", 1014 | "windows_x86_64_msvc 0.53.0", 1015 | ] 1016 | 1017 | [[package]] 1018 | name = "windows_aarch64_gnullvm" 1019 | version = "0.52.6" 1020 | source = "registry+https://github.com/rust-lang/crates.io-index" 1021 | checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" 1022 | 1023 | [[package]] 1024 | name = "windows_aarch64_gnullvm" 1025 | version = "0.53.0" 1026 | source = "registry+https://github.com/rust-lang/crates.io-index" 1027 | checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" 1028 | 1029 | [[package]] 1030 | name = "windows_aarch64_msvc" 1031 | version = "0.52.6" 1032 | source = "registry+https://github.com/rust-lang/crates.io-index" 1033 | checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" 1034 | 1035 | [[package]] 1036 | name = "windows_aarch64_msvc" 1037 | version = "0.53.0" 1038 | source = "registry+https://github.com/rust-lang/crates.io-index" 1039 | checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" 1040 | 1041 | [[package]] 1042 | name = "windows_i686_gnu" 1043 | version = "0.52.6" 1044 | source = "registry+https://github.com/rust-lang/crates.io-index" 1045 | checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" 1046 | 1047 | [[package]] 1048 | name = "windows_i686_gnu" 1049 | version = "0.53.0" 1050 | source = "registry+https://github.com/rust-lang/crates.io-index" 1051 | checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" 1052 | 1053 | [[package]] 1054 | name = "windows_i686_gnullvm" 1055 | version = "0.52.6" 1056 | source = "registry+https://github.com/rust-lang/crates.io-index" 1057 | checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" 1058 | 1059 | [[package]] 1060 | name = "windows_i686_gnullvm" 1061 | version = "0.53.0" 1062 | source = "registry+https://github.com/rust-lang/crates.io-index" 1063 | checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" 1064 | 1065 | [[package]] 1066 | name = "windows_i686_msvc" 1067 | version = "0.52.6" 1068 | source = "registry+https://github.com/rust-lang/crates.io-index" 1069 | checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" 1070 | 1071 | [[package]] 1072 | name = "windows_i686_msvc" 1073 | version = "0.53.0" 1074 | source = "registry+https://github.com/rust-lang/crates.io-index" 1075 | checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" 1076 | 1077 | [[package]] 1078 | name = "windows_x86_64_gnu" 1079 | version = "0.52.6" 1080 | source = "registry+https://github.com/rust-lang/crates.io-index" 1081 | checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" 1082 | 1083 | [[package]] 1084 | name = "windows_x86_64_gnu" 1085 | version = "0.53.0" 1086 | source = "registry+https://github.com/rust-lang/crates.io-index" 1087 | checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" 1088 | 1089 | [[package]] 1090 | name = "windows_x86_64_gnullvm" 1091 | version = "0.52.6" 1092 | source = "registry+https://github.com/rust-lang/crates.io-index" 1093 | checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" 1094 | 1095 | [[package]] 1096 | name = "windows_x86_64_gnullvm" 1097 | version = "0.53.0" 1098 | source = "registry+https://github.com/rust-lang/crates.io-index" 1099 | checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" 1100 | 1101 | [[package]] 1102 | name = "windows_x86_64_msvc" 1103 | version = "0.52.6" 1104 | source = "registry+https://github.com/rust-lang/crates.io-index" 1105 | checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" 1106 | 1107 | [[package]] 1108 | name = "windows_x86_64_msvc" 1109 | version = "0.53.0" 1110 | source = "registry+https://github.com/rust-lang/crates.io-index" 1111 | checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" 1112 | 1113 | [[package]] 1114 | name = "wit-bindgen-rt" 1115 | version = "0.39.0" 1116 | source = "registry+https://github.com/rust-lang/crates.io-index" 1117 | checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" 1118 | dependencies = [ 1119 | "bitflags", 1120 | ] 1121 | 1122 | [[package]] 1123 | name = "xattr" 1124 | version = "1.5.1" 1125 | source = "registry+https://github.com/rust-lang/crates.io-index" 1126 | checksum = "af3a19837351dc82ba89f8a125e22a3c475f05aba604acc023d62b2739ae2909" 1127 | dependencies = [ 1128 | "libc", 1129 | "rustix", 1130 | ] 1131 | 1132 | [[package]] 1133 | name = "xcp" 1134 | version = "0.24.2" 1135 | dependencies = [ 1136 | "anyhow", 1137 | "cfg-if", 1138 | "clap", 1139 | "crossbeam-channel", 1140 | "fslock", 1141 | "glob", 1142 | "ignore", 1143 | "indicatif", 1144 | "libfs", 1145 | "libxcp", 1146 | "log", 1147 | "num_cpus", 1148 | "rand", 1149 | "rand_distr", 1150 | "rand_xorshift", 1151 | "rustix", 1152 | "simplelog", 1153 | "tempfile", 1154 | "terminal_size", 1155 | "test-case", 1156 | "unbytify", 1157 | "uuid", 1158 | "walkdir", 1159 | "xattr", 1160 | ] 1161 | 1162 | [[package]] 1163 | name = "zerocopy" 1164 | version = "0.8.26" 1165 | source = "registry+https://github.com/rust-lang/crates.io-index" 1166 | checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" 1167 | dependencies = [ 1168 | "zerocopy-derive", 1169 | ] 1170 | 1171 | [[package]] 1172 | name = "zerocopy-derive" 1173 | version = "0.8.26" 1174 | source = "registry+https://github.com/rust-lang/crates.io-index" 1175 | checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" 1176 | dependencies = [ 1177 | "proc-macro2", 1178 | "quote", 1179 | "syn", 1180 | ] 1181 | --------------------------------------------------------------------------------