├── .circleci └── config.yml ├── .github └── workflows │ └── tests.yml ├── .gitignore ├── COPYING ├── Cargo.lock ├── Cargo.toml ├── README.md ├── completions ├── xcp.bash ├── xcp.fish └── xcp.zsh ├── libfs ├── .gitignore ├── Cargo.toml ├── README.md └── src │ ├── common.rs │ ├── errors.rs │ ├── fallback.rs │ ├── lib.rs │ └── linux.rs ├── libxcp ├── Cargo.toml ├── README.md └── src │ ├── backup.rs │ ├── config.rs │ ├── drivers │ ├── mod.rs │ ├── parblock.rs │ └── parfile.rs │ ├── errors.rs │ ├── feedback.rs │ ├── lib.rs │ ├── operations.rs │ └── paths.rs ├── src ├── main.rs ├── options.rs └── progress.rs └── tests ├── common.rs ├── linux.rs ├── scripts ├── make-filesystems.sh └── test-linux.sh └── util.rs /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | executors: 3 | rust-executor: 4 | docker: 5 | - image: cimg/rust:1.82.0 6 | 7 | jobs: 8 | test: 9 | executor: rust-executor 10 | steps: 11 | - checkout 12 | - run: 13 | name: Install packages 14 | command: sudo apt-get update && sudo apt-get install -y libacl1-dev 15 | - run: 16 | name: Run tests 17 | command: cargo test --color never 18 | - run: 19 | name: Run expensive tests 20 | command: cargo test --color never -- --ignored --nocapture 21 | 22 | workflows: 23 | rust-test-workflow: 24 | jobs: 25 | - test 26 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: [push] 4 | 5 | jobs: 6 | ubuntu: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v4 10 | 11 | - name: CI runtime info 12 | run: uname -a && cat /etc/os-release 13 | 14 | - name: Add necessary packages 15 | run: sudo apt-get update && sudo apt-get install -y libacl1-dev 16 | 17 | - name: Update Rust to latest 18 | run: ~/.cargo/bin/rustup update 19 | 20 | - name: Create filesystems 21 | # f2fs and exfat modules are in linux-modules-extra-azure 22 | # and cannot be installed reliably: 23 | # https://github.com/actions/runner-images/issues/7587 24 | run: tests/scripts/make-filesystems.sh ext2 ext4 xfs btrfs ntfs fat zfs 25 | 26 | - name: Run tests on ext2 27 | run: /fs/ext2/src/tests/scripts/test-linux.sh 28 | if: always() 29 | 30 | - name: Run tests on ext4 31 | run: /fs/ext4/src/tests/scripts/test-linux.sh 32 | if: always() 33 | 34 | - name: Run tests on XFS 35 | run: /fs/xfs/src/tests/scripts/test-linux.sh 36 | if: always() 37 | 38 | - name: Run tests on btrfs 39 | run: /fs/btrfs/src/tests/scripts/test-linux.sh 40 | if: always() 41 | 42 | - name: Run tests on ntfs 43 | run: /fs/ntfs/src/tests/scripts/test-linux.sh 44 | if: always() 45 | 46 | - name: Run tests on fat 47 | run: /fs/fat/src/tests/scripts/test-linux.sh 48 | if: always() 49 | 50 | - name: Run tests on ZFS 51 | run: /fs/zfs/src/tests/scripts/test-linux.sh 52 | if: always() 53 | 54 | expensive: 55 | runs-on: ubuntu-latest 56 | steps: 57 | - uses: actions/checkout@v4 58 | 59 | - name: Add necessary packages 60 | run: sudo apt-get update && sudo apt-get install -y libacl1-dev 61 | 62 | - name: Update Rust to latest 63 | run: ~/.cargo/bin/rustup update 64 | 65 | - name: Run expensive tests 66 | run: ./tests/scripts/test-linux.sh test_run_expensive 67 | 68 | macos: 69 | runs-on: macos-latest 70 | steps: 71 | - uses: actions/checkout@v4 72 | 73 | - name: Install Rust 74 | run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash /dev/stdin -y 75 | 76 | - name: Update Rust (installer may lag behind) 77 | run: ~/.cargo/bin/rustup update 78 | 79 | - name: Run all tests 80 | run: ~/.cargo/bin/cargo test --workspace --features=test_no_reflink,test_no_sockets,test_run_expensive 81 | 82 | freebsd: 83 | runs-on: ubuntu-latest 84 | steps: 85 | - uses: actions/checkout@v4 86 | 87 | - uses: vmactions/freebsd-vm@v1 88 | with: 89 | usesh: true 90 | prepare: | 91 | pkg install -y curl 92 | pw user add -n testing -m 93 | run: | 94 | su testing -c ' 95 | curl -sSf https://sh.rustup.rs | sh /dev/stdin -y \ 96 | && ~/.cargo/bin/cargo test --workspace --features=test_no_reflink,test_no_sockets \ 97 | && ~/.cargo/bin/cargo clean 98 | ' 99 | 100 | nightly: 101 | runs-on: ubuntu-latest 102 | steps: 103 | - uses: actions/checkout@v4 104 | 105 | - name: Add necessary packages 106 | run: sudo apt-get update && sudo apt-get install -y libacl1-dev 107 | 108 | - name: Install Rust nightly 109 | run: ~/.cargo/bin/rustup toolchain install nightly 110 | 111 | - name: Compile and test with nightly 112 | run: ~/.cargo/bin/cargo +nightly test --workspace --features=test_no_reflink 113 | 114 | msrv-check: 115 | runs-on: ubuntu-latest 116 | steps: 117 | - uses: actions/checkout@v4 118 | 119 | - name: Update Rust to latest 120 | run: ~/.cargo/bin/rustup update 121 | 122 | - name: Install MSRV checker 123 | run: ~/.cargo/bin/cargo install cargo-msrv 124 | 125 | - name: Check MSRV 126 | run: ~/.cargo/bin/cargo msrv verify 127 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | **/*.rs.bk 3 | /.idea 4 | bacon.toml 5 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "aho-corasick" 7 | version = "1.1.3" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" 10 | dependencies = [ 11 | "memchr", 12 | ] 13 | 14 | [[package]] 15 | name = "anstream" 16 | version = "0.6.18" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" 19 | dependencies = [ 20 | "anstyle", 21 | "anstyle-parse", 22 | "anstyle-query", 23 | "anstyle-wincon", 24 | "colorchoice", 25 | "is_terminal_polyfill", 26 | "utf8parse", 27 | ] 28 | 29 | [[package]] 30 | name = "anstyle" 31 | version = "1.0.10" 32 | source = "registry+https://github.com/rust-lang/crates.io-index" 33 | checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" 34 | 35 | [[package]] 36 | name = "anstyle-parse" 37 | version = "0.2.6" 38 | source = "registry+https://github.com/rust-lang/crates.io-index" 39 | checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" 40 | dependencies = [ 41 | "utf8parse", 42 | ] 43 | 44 | [[package]] 45 | name = "anstyle-query" 46 | version = "1.1.2" 47 | source = "registry+https://github.com/rust-lang/crates.io-index" 48 | checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" 49 | dependencies = [ 50 | "windows-sys 0.59.0", 51 | ] 52 | 53 | [[package]] 54 | name = "anstyle-wincon" 55 | version = "3.0.6" 56 | source = "registry+https://github.com/rust-lang/crates.io-index" 57 | checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" 58 | dependencies = [ 59 | "anstyle", 60 | "windows-sys 0.59.0", 61 | ] 62 | 63 | [[package]] 64 | name = "anyhow" 65 | version = "1.0.97" 66 | source = "registry+https://github.com/rust-lang/crates.io-index" 67 | checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" 68 | 69 | [[package]] 70 | name = "autocfg" 71 | version = "1.4.0" 72 | source = "registry+https://github.com/rust-lang/crates.io-index" 73 | checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" 74 | 75 | [[package]] 76 | name = "bitflags" 77 | version = "2.9.0" 78 | source = "registry+https://github.com/rust-lang/crates.io-index" 79 | checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" 80 | 81 | [[package]] 82 | name = "blocking-threadpool" 83 | version = "1.0.1" 84 | source = "registry+https://github.com/rust-lang/crates.io-index" 85 | checksum = "c4ba4d6edfe07b0a4940ab5c05a7114155ffbe9d0c64df7a2e39cb002f879869" 86 | dependencies = [ 87 | "crossbeam-channel", 88 | "num_cpus", 89 | ] 90 | 91 | [[package]] 92 | name = "bstr" 93 | version = "1.10.0" 94 | source = "registry+https://github.com/rust-lang/crates.io-index" 95 | checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" 96 | dependencies = [ 97 | "memchr", 98 | "serde", 99 | ] 100 | 101 | [[package]] 102 | name = "bumpalo" 103 | version = "3.17.0" 104 | source = "registry+https://github.com/rust-lang/crates.io-index" 105 | checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" 106 | 107 | [[package]] 108 | name = "cfg-if" 109 | version = "1.0.0" 110 | source = "registry+https://github.com/rust-lang/crates.io-index" 111 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 112 | 113 | [[package]] 114 | name = "clap" 115 | version = "4.5.35" 116 | source = "registry+https://github.com/rust-lang/crates.io-index" 117 | checksum = "d8aa86934b44c19c50f87cc2790e19f54f7a67aedb64101c2e1a2e5ecfb73944" 118 | dependencies = [ 119 | "clap_builder", 120 | "clap_derive", 121 | ] 122 | 123 | [[package]] 124 | name = "clap_builder" 125 | version = "4.5.35" 126 | source = "registry+https://github.com/rust-lang/crates.io-index" 127 | checksum = "2414dbb2dd0695280da6ea9261e327479e9d37b0630f6b53ba2a11c60c679fd9" 128 | dependencies = [ 129 | "anstream", 130 | "anstyle", 131 | "clap_lex", 132 | "strsim", 133 | ] 134 | 135 | [[package]] 136 | name = "clap_derive" 137 | version = "4.5.32" 138 | source = "registry+https://github.com/rust-lang/crates.io-index" 139 | checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" 140 | dependencies = [ 141 | "heck", 142 | "proc-macro2", 143 | "quote", 144 | "syn", 145 | ] 146 | 147 | [[package]] 148 | name = "clap_lex" 149 | version = "0.7.4" 150 | source = "registry+https://github.com/rust-lang/crates.io-index" 151 | checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" 152 | 153 | [[package]] 154 | name = "colorchoice" 155 | version = "1.0.3" 156 | source = "registry+https://github.com/rust-lang/crates.io-index" 157 | checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" 158 | 159 | [[package]] 160 | name = "console" 161 | version = "0.15.11" 162 | source = "registry+https://github.com/rust-lang/crates.io-index" 163 | checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" 164 | dependencies = [ 165 | "encode_unicode", 166 | "libc", 167 | "once_cell", 168 | "unicode-width", 169 | "windows-sys 0.59.0", 170 | ] 171 | 172 | [[package]] 173 | name = "crossbeam-channel" 174 | version = "0.5.15" 175 | source = "registry+https://github.com/rust-lang/crates.io-index" 176 | checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" 177 | dependencies = [ 178 | "crossbeam-utils", 179 | ] 180 | 181 | [[package]] 182 | name = "crossbeam-deque" 183 | version = "0.8.5" 184 | source = "registry+https://github.com/rust-lang/crates.io-index" 185 | checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" 186 | dependencies = [ 187 | "crossbeam-epoch", 188 | "crossbeam-utils", 189 | ] 190 | 191 | [[package]] 192 | name = "crossbeam-epoch" 193 | version = "0.9.18" 194 | source = "registry+https://github.com/rust-lang/crates.io-index" 195 | checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" 196 | dependencies = [ 197 | "crossbeam-utils", 198 | ] 199 | 200 | [[package]] 201 | name = "crossbeam-utils" 202 | version = "0.8.21" 203 | source = "registry+https://github.com/rust-lang/crates.io-index" 204 | checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" 205 | 206 | [[package]] 207 | name = "deranged" 208 | version = "0.3.11" 209 | source = "registry+https://github.com/rust-lang/crates.io-index" 210 | checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" 211 | dependencies = [ 212 | "powerfmt", 213 | ] 214 | 215 | [[package]] 216 | name = "encode_unicode" 217 | version = "1.0.0" 218 | source = "registry+https://github.com/rust-lang/crates.io-index" 219 | checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" 220 | 221 | [[package]] 222 | name = "errno" 223 | version = "0.3.10" 224 | source = "registry+https://github.com/rust-lang/crates.io-index" 225 | checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" 226 | dependencies = [ 227 | "libc", 228 | "windows-sys 0.52.0", 229 | ] 230 | 231 | [[package]] 232 | name = "exacl" 233 | version = "0.12.0" 234 | source = "registry+https://github.com/rust-lang/crates.io-index" 235 | checksum = "22be12de19decddab85d09f251ec8363f060ccb22ec9c81bc157c0c8433946d8" 236 | dependencies = [ 237 | "bitflags", 238 | "log", 239 | "scopeguard", 240 | "uuid", 241 | ] 242 | 243 | [[package]] 244 | name = "fastrand" 245 | version = "2.3.0" 246 | source = "registry+https://github.com/rust-lang/crates.io-index" 247 | checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" 248 | 249 | [[package]] 250 | name = "fslock" 251 | version = "0.2.1" 252 | source = "registry+https://github.com/rust-lang/crates.io-index" 253 | checksum = "04412b8935272e3a9bae6f48c7bfff74c2911f60525404edfdd28e49884c3bfb" 254 | dependencies = [ 255 | "libc", 256 | "winapi", 257 | ] 258 | 259 | [[package]] 260 | name = "getrandom" 261 | version = "0.3.2" 262 | source = "registry+https://github.com/rust-lang/crates.io-index" 263 | checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" 264 | dependencies = [ 265 | "cfg-if", 266 | "libc", 267 | "r-efi", 268 | "wasi", 269 | ] 270 | 271 | [[package]] 272 | name = "glob" 273 | version = "0.3.2" 274 | source = "registry+https://github.com/rust-lang/crates.io-index" 275 | checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" 276 | 277 | [[package]] 278 | name = "globset" 279 | version = "0.4.15" 280 | source = "registry+https://github.com/rust-lang/crates.io-index" 281 | checksum = "15f1ce686646e7f1e19bf7d5533fe443a45dbfb990e00629110797578b42fb19" 282 | dependencies = [ 283 | "aho-corasick", 284 | "bstr", 285 | "log", 286 | "regex-automata", 287 | "regex-syntax", 288 | ] 289 | 290 | [[package]] 291 | name = "heck" 292 | version = "0.5.0" 293 | source = "registry+https://github.com/rust-lang/crates.io-index" 294 | checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" 295 | 296 | [[package]] 297 | name = "hermit-abi" 298 | version = "0.3.9" 299 | source = "registry+https://github.com/rust-lang/crates.io-index" 300 | checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" 301 | 302 | [[package]] 303 | name = "ignore" 304 | version = "0.4.23" 305 | source = "registry+https://github.com/rust-lang/crates.io-index" 306 | checksum = "6d89fd380afde86567dfba715db065673989d6253f42b88179abd3eae47bda4b" 307 | dependencies = [ 308 | "crossbeam-deque", 309 | "globset", 310 | "log", 311 | "memchr", 312 | "regex-automata", 313 | "same-file", 314 | "walkdir", 315 | "winapi-util", 316 | ] 317 | 318 | [[package]] 319 | name = "indicatif" 320 | version = "0.17.11" 321 | source = "registry+https://github.com/rust-lang/crates.io-index" 322 | checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" 323 | dependencies = [ 324 | "console", 325 | "number_prefix", 326 | "portable-atomic", 327 | "unicode-width", 328 | "web-time", 329 | ] 330 | 331 | [[package]] 332 | name = "is_terminal_polyfill" 333 | version = "1.70.1" 334 | source = "registry+https://github.com/rust-lang/crates.io-index" 335 | checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" 336 | 337 | [[package]] 338 | name = "itoa" 339 | version = "1.0.11" 340 | source = "registry+https://github.com/rust-lang/crates.io-index" 341 | checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" 342 | 343 | [[package]] 344 | name = "js-sys" 345 | version = "0.3.72" 346 | source = "registry+https://github.com/rust-lang/crates.io-index" 347 | checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" 348 | dependencies = [ 349 | "wasm-bindgen", 350 | ] 351 | 352 | [[package]] 353 | name = "lazy_static" 354 | version = "0.2.11" 355 | source = "registry+https://github.com/rust-lang/crates.io-index" 356 | checksum = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" 357 | 358 | [[package]] 359 | name = "libc" 360 | version = "0.2.171" 361 | source = "registry+https://github.com/rust-lang/crates.io-index" 362 | checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" 363 | 364 | [[package]] 365 | name = "libfs" 366 | version = "0.9.1" 367 | dependencies = [ 368 | "cfg-if", 369 | "exacl", 370 | "libc", 371 | "linux-raw-sys", 372 | "log", 373 | "rustix", 374 | "tempfile", 375 | "thiserror", 376 | "xattr", 377 | ] 378 | 379 | [[package]] 380 | name = "libm" 381 | version = "0.2.11" 382 | source = "registry+https://github.com/rust-lang/crates.io-index" 383 | checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" 384 | 385 | [[package]] 386 | name = "libxcp" 387 | version = "0.24.1" 388 | dependencies = [ 389 | "anyhow", 390 | "blocking-threadpool", 391 | "cfg-if", 392 | "crossbeam-channel", 393 | "ignore", 394 | "libfs", 395 | "log", 396 | "num_cpus", 397 | "regex", 398 | "tempfile", 399 | "thiserror", 400 | "walkdir", 401 | ] 402 | 403 | [[package]] 404 | name = "linux-raw-sys" 405 | version = "0.9.4" 406 | source = "registry+https://github.com/rust-lang/crates.io-index" 407 | checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" 408 | 409 | [[package]] 410 | name = "log" 411 | version = "0.4.27" 412 | source = "registry+https://github.com/rust-lang/crates.io-index" 413 | checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" 414 | 415 | [[package]] 416 | name = "memchr" 417 | version = "2.7.4" 418 | source = "registry+https://github.com/rust-lang/crates.io-index" 419 | checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" 420 | 421 | [[package]] 422 | name = "num-conv" 423 | version = "0.1.0" 424 | source = "registry+https://github.com/rust-lang/crates.io-index" 425 | checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" 426 | 427 | [[package]] 428 | name = "num-traits" 429 | version = "0.2.19" 430 | source = "registry+https://github.com/rust-lang/crates.io-index" 431 | checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" 432 | dependencies = [ 433 | "autocfg", 434 | "libm", 435 | ] 436 | 437 | [[package]] 438 | name = "num_cpus" 439 | version = "1.16.0" 440 | source = "registry+https://github.com/rust-lang/crates.io-index" 441 | checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" 442 | dependencies = [ 443 | "hermit-abi", 444 | "libc", 445 | ] 446 | 447 | [[package]] 448 | name = "num_threads" 449 | version = "0.1.7" 450 | source = "registry+https://github.com/rust-lang/crates.io-index" 451 | checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" 452 | dependencies = [ 453 | "libc", 454 | ] 455 | 456 | [[package]] 457 | name = "number_prefix" 458 | version = "0.4.0" 459 | source = "registry+https://github.com/rust-lang/crates.io-index" 460 | checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" 461 | 462 | [[package]] 463 | name = "once_cell" 464 | version = "1.21.1" 465 | source = "registry+https://github.com/rust-lang/crates.io-index" 466 | checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc" 467 | 468 | [[package]] 469 | name = "portable-atomic" 470 | version = "1.11.0" 471 | source = "registry+https://github.com/rust-lang/crates.io-index" 472 | checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" 473 | 474 | [[package]] 475 | name = "powerfmt" 476 | version = "0.2.0" 477 | source = "registry+https://github.com/rust-lang/crates.io-index" 478 | checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" 479 | 480 | [[package]] 481 | name = "ppv-lite86" 482 | version = "0.2.21" 483 | source = "registry+https://github.com/rust-lang/crates.io-index" 484 | checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" 485 | dependencies = [ 486 | "zerocopy", 487 | ] 488 | 489 | [[package]] 490 | name = "proc-macro2" 491 | version = "1.0.94" 492 | source = "registry+https://github.com/rust-lang/crates.io-index" 493 | checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" 494 | dependencies = [ 495 | "unicode-ident", 496 | ] 497 | 498 | [[package]] 499 | name = "quote" 500 | version = "1.0.40" 501 | source = "registry+https://github.com/rust-lang/crates.io-index" 502 | checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" 503 | dependencies = [ 504 | "proc-macro2", 505 | ] 506 | 507 | [[package]] 508 | name = "r-efi" 509 | version = "5.2.0" 510 | source = "registry+https://github.com/rust-lang/crates.io-index" 511 | checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" 512 | 513 | [[package]] 514 | name = "rand" 515 | version = "0.9.0" 516 | source = "registry+https://github.com/rust-lang/crates.io-index" 517 | checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" 518 | dependencies = [ 519 | "rand_chacha", 520 | "rand_core", 521 | "zerocopy", 522 | ] 523 | 524 | [[package]] 525 | name = "rand_chacha" 526 | version = "0.9.0" 527 | source = "registry+https://github.com/rust-lang/crates.io-index" 528 | checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" 529 | dependencies = [ 530 | "ppv-lite86", 531 | "rand_core", 532 | ] 533 | 534 | [[package]] 535 | name = "rand_core" 536 | version = "0.9.3" 537 | source = "registry+https://github.com/rust-lang/crates.io-index" 538 | checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" 539 | dependencies = [ 540 | "getrandom", 541 | ] 542 | 543 | [[package]] 544 | name = "rand_distr" 545 | version = "0.5.1" 546 | source = "registry+https://github.com/rust-lang/crates.io-index" 547 | checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" 548 | dependencies = [ 549 | "num-traits", 550 | "rand", 551 | ] 552 | 553 | [[package]] 554 | name = "rand_xorshift" 555 | version = "0.4.0" 556 | source = "registry+https://github.com/rust-lang/crates.io-index" 557 | checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" 558 | dependencies = [ 559 | "rand_core", 560 | ] 561 | 562 | [[package]] 563 | name = "regex" 564 | version = "1.11.1" 565 | source = "registry+https://github.com/rust-lang/crates.io-index" 566 | checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" 567 | dependencies = [ 568 | "aho-corasick", 569 | "memchr", 570 | "regex-automata", 571 | "regex-syntax", 572 | ] 573 | 574 | [[package]] 575 | name = "regex-automata" 576 | version = "0.4.8" 577 | source = "registry+https://github.com/rust-lang/crates.io-index" 578 | checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" 579 | dependencies = [ 580 | "aho-corasick", 581 | "memchr", 582 | "regex-syntax", 583 | ] 584 | 585 | [[package]] 586 | name = "regex-syntax" 587 | version = "0.8.5" 588 | source = "registry+https://github.com/rust-lang/crates.io-index" 589 | checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" 590 | 591 | [[package]] 592 | name = "rustix" 593 | version = "1.0.5" 594 | source = "registry+https://github.com/rust-lang/crates.io-index" 595 | checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" 596 | dependencies = [ 597 | "bitflags", 598 | "errno", 599 | "libc", 600 | "linux-raw-sys", 601 | "windows-sys 0.52.0", 602 | ] 603 | 604 | [[package]] 605 | name = "rustversion" 606 | version = "1.0.20" 607 | source = "registry+https://github.com/rust-lang/crates.io-index" 608 | checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" 609 | 610 | [[package]] 611 | name = "same-file" 612 | version = "1.0.6" 613 | source = "registry+https://github.com/rust-lang/crates.io-index" 614 | checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" 615 | dependencies = [ 616 | "winapi-util", 617 | ] 618 | 619 | [[package]] 620 | name = "scopeguard" 621 | version = "1.2.0" 622 | source = "registry+https://github.com/rust-lang/crates.io-index" 623 | checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" 624 | 625 | [[package]] 626 | name = "serde" 627 | version = "1.0.210" 628 | source = "registry+https://github.com/rust-lang/crates.io-index" 629 | checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" 630 | dependencies = [ 631 | "serde_derive", 632 | ] 633 | 634 | [[package]] 635 | name = "serde_derive" 636 | version = "1.0.210" 637 | source = "registry+https://github.com/rust-lang/crates.io-index" 638 | checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" 639 | dependencies = [ 640 | "proc-macro2", 641 | "quote", 642 | "syn", 643 | ] 644 | 645 | [[package]] 646 | name = "simplelog" 647 | version = "0.12.2" 648 | source = "registry+https://github.com/rust-lang/crates.io-index" 649 | checksum = "16257adbfaef1ee58b1363bdc0664c9b8e1e30aed86049635fb5f147d065a9c0" 650 | dependencies = [ 651 | "log", 652 | "termcolor", 653 | "time", 654 | ] 655 | 656 | [[package]] 657 | name = "strsim" 658 | version = "0.11.1" 659 | source = "registry+https://github.com/rust-lang/crates.io-index" 660 | checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" 661 | 662 | [[package]] 663 | name = "syn" 664 | version = "2.0.100" 665 | source = "registry+https://github.com/rust-lang/crates.io-index" 666 | checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" 667 | dependencies = [ 668 | "proc-macro2", 669 | "quote", 670 | "unicode-ident", 671 | ] 672 | 673 | [[package]] 674 | name = "tempfile" 675 | version = "3.19.1" 676 | source = "registry+https://github.com/rust-lang/crates.io-index" 677 | checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" 678 | dependencies = [ 679 | "fastrand", 680 | "getrandom", 681 | "once_cell", 682 | "rustix", 683 | "windows-sys 0.52.0", 684 | ] 685 | 686 | [[package]] 687 | name = "termcolor" 688 | version = "1.4.1" 689 | source = "registry+https://github.com/rust-lang/crates.io-index" 690 | checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" 691 | dependencies = [ 692 | "winapi-util", 693 | ] 694 | 695 | [[package]] 696 | name = "terminal_size" 697 | version = "0.4.2" 698 | source = "registry+https://github.com/rust-lang/crates.io-index" 699 | checksum = "45c6481c4829e4cc63825e62c49186a34538b7b2750b73b266581ffb612fb5ed" 700 | dependencies = [ 701 | "rustix", 702 | "windows-sys 0.59.0", 703 | ] 704 | 705 | [[package]] 706 | name = "test-case" 707 | version = "3.3.1" 708 | source = "registry+https://github.com/rust-lang/crates.io-index" 709 | checksum = "eb2550dd13afcd286853192af8601920d959b14c401fcece38071d53bf0768a8" 710 | dependencies = [ 711 | "test-case-macros", 712 | ] 713 | 714 | [[package]] 715 | name = "test-case-core" 716 | version = "3.3.1" 717 | source = "registry+https://github.com/rust-lang/crates.io-index" 718 | checksum = "adcb7fd841cd518e279be3d5a3eb0636409487998a4aff22f3de87b81e88384f" 719 | dependencies = [ 720 | "cfg-if", 721 | "proc-macro2", 722 | "quote", 723 | "syn", 724 | ] 725 | 726 | [[package]] 727 | name = "test-case-macros" 728 | version = "3.3.1" 729 | source = "registry+https://github.com/rust-lang/crates.io-index" 730 | checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" 731 | dependencies = [ 732 | "proc-macro2", 733 | "quote", 734 | "syn", 735 | "test-case-core", 736 | ] 737 | 738 | [[package]] 739 | name = "thiserror" 740 | version = "2.0.12" 741 | source = "registry+https://github.com/rust-lang/crates.io-index" 742 | checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" 743 | dependencies = [ 744 | "thiserror-impl", 745 | ] 746 | 747 | [[package]] 748 | name = "thiserror-impl" 749 | version = "2.0.12" 750 | source = "registry+https://github.com/rust-lang/crates.io-index" 751 | checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" 752 | dependencies = [ 753 | "proc-macro2", 754 | "quote", 755 | "syn", 756 | ] 757 | 758 | [[package]] 759 | name = "time" 760 | version = "0.3.36" 761 | source = "registry+https://github.com/rust-lang/crates.io-index" 762 | checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" 763 | dependencies = [ 764 | "deranged", 765 | "itoa", 766 | "libc", 767 | "num-conv", 768 | "num_threads", 769 | "powerfmt", 770 | "serde", 771 | "time-core", 772 | "time-macros", 773 | ] 774 | 775 | [[package]] 776 | name = "time-core" 777 | version = "0.1.2" 778 | source = "registry+https://github.com/rust-lang/crates.io-index" 779 | checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" 780 | 781 | [[package]] 782 | name = "time-macros" 783 | version = "0.2.18" 784 | source = "registry+https://github.com/rust-lang/crates.io-index" 785 | checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" 786 | dependencies = [ 787 | "num-conv", 788 | "time-core", 789 | ] 790 | 791 | [[package]] 792 | name = "unbytify" 793 | version = "0.2.0" 794 | source = "registry+https://github.com/rust-lang/crates.io-index" 795 | checksum = "61f431354fd60c251d35ccc3d3ecf14e5f37e52ce807f6436f394fb3f0fc9869" 796 | dependencies = [ 797 | "lazy_static", 798 | ] 799 | 800 | [[package]] 801 | name = "unicode-ident" 802 | version = "1.0.18" 803 | source = "registry+https://github.com/rust-lang/crates.io-index" 804 | checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" 805 | 806 | [[package]] 807 | name = "unicode-width" 808 | version = "0.2.0" 809 | source = "registry+https://github.com/rust-lang/crates.io-index" 810 | checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" 811 | 812 | [[package]] 813 | name = "utf8parse" 814 | version = "0.2.2" 815 | source = "registry+https://github.com/rust-lang/crates.io-index" 816 | checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" 817 | 818 | [[package]] 819 | name = "uuid" 820 | version = "1.16.0" 821 | source = "registry+https://github.com/rust-lang/crates.io-index" 822 | checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" 823 | dependencies = [ 824 | "getrandom", 825 | ] 826 | 827 | [[package]] 828 | name = "walkdir" 829 | version = "2.5.0" 830 | source = "registry+https://github.com/rust-lang/crates.io-index" 831 | checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" 832 | dependencies = [ 833 | "same-file", 834 | "winapi-util", 835 | ] 836 | 837 | [[package]] 838 | name = "wasi" 839 | version = "0.14.2+wasi-0.2.4" 840 | source = "registry+https://github.com/rust-lang/crates.io-index" 841 | checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" 842 | dependencies = [ 843 | "wit-bindgen-rt", 844 | ] 845 | 846 | [[package]] 847 | name = "wasm-bindgen" 848 | version = "0.2.100" 849 | source = "registry+https://github.com/rust-lang/crates.io-index" 850 | checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" 851 | dependencies = [ 852 | "cfg-if", 853 | "once_cell", 854 | "rustversion", 855 | "wasm-bindgen-macro", 856 | ] 857 | 858 | [[package]] 859 | name = "wasm-bindgen-backend" 860 | version = "0.2.100" 861 | source = "registry+https://github.com/rust-lang/crates.io-index" 862 | checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" 863 | dependencies = [ 864 | "bumpalo", 865 | "log", 866 | "proc-macro2", 867 | "quote", 868 | "syn", 869 | "wasm-bindgen-shared", 870 | ] 871 | 872 | [[package]] 873 | name = "wasm-bindgen-macro" 874 | version = "0.2.100" 875 | source = "registry+https://github.com/rust-lang/crates.io-index" 876 | checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" 877 | dependencies = [ 878 | "quote", 879 | "wasm-bindgen-macro-support", 880 | ] 881 | 882 | [[package]] 883 | name = "wasm-bindgen-macro-support" 884 | version = "0.2.100" 885 | source = "registry+https://github.com/rust-lang/crates.io-index" 886 | checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" 887 | dependencies = [ 888 | "proc-macro2", 889 | "quote", 890 | "syn", 891 | "wasm-bindgen-backend", 892 | "wasm-bindgen-shared", 893 | ] 894 | 895 | [[package]] 896 | name = "wasm-bindgen-shared" 897 | version = "0.2.100" 898 | source = "registry+https://github.com/rust-lang/crates.io-index" 899 | checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" 900 | dependencies = [ 901 | "unicode-ident", 902 | ] 903 | 904 | [[package]] 905 | name = "web-time" 906 | version = "1.1.0" 907 | source = "registry+https://github.com/rust-lang/crates.io-index" 908 | checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" 909 | dependencies = [ 910 | "js-sys", 911 | "wasm-bindgen", 912 | ] 913 | 914 | [[package]] 915 | name = "winapi" 916 | version = "0.3.9" 917 | source = "registry+https://github.com/rust-lang/crates.io-index" 918 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 919 | dependencies = [ 920 | "winapi-i686-pc-windows-gnu", 921 | "winapi-x86_64-pc-windows-gnu", 922 | ] 923 | 924 | [[package]] 925 | name = "winapi-i686-pc-windows-gnu" 926 | version = "0.4.0" 927 | source = "registry+https://github.com/rust-lang/crates.io-index" 928 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 929 | 930 | [[package]] 931 | name = "winapi-util" 932 | version = "0.1.8" 933 | source = "registry+https://github.com/rust-lang/crates.io-index" 934 | checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" 935 | dependencies = [ 936 | "windows-sys 0.52.0", 937 | ] 938 | 939 | [[package]] 940 | name = "winapi-x86_64-pc-windows-gnu" 941 | version = "0.4.0" 942 | source = "registry+https://github.com/rust-lang/crates.io-index" 943 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 944 | 945 | [[package]] 946 | name = "windows-sys" 947 | version = "0.52.0" 948 | source = "registry+https://github.com/rust-lang/crates.io-index" 949 | checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" 950 | dependencies = [ 951 | "windows-targets", 952 | ] 953 | 954 | [[package]] 955 | name = "windows-sys" 956 | version = "0.59.0" 957 | source = "registry+https://github.com/rust-lang/crates.io-index" 958 | checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" 959 | dependencies = [ 960 | "windows-targets", 961 | ] 962 | 963 | [[package]] 964 | name = "windows-targets" 965 | version = "0.52.6" 966 | source = "registry+https://github.com/rust-lang/crates.io-index" 967 | checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" 968 | dependencies = [ 969 | "windows_aarch64_gnullvm", 970 | "windows_aarch64_msvc", 971 | "windows_i686_gnu", 972 | "windows_i686_gnullvm", 973 | "windows_i686_msvc", 974 | "windows_x86_64_gnu", 975 | "windows_x86_64_gnullvm", 976 | "windows_x86_64_msvc", 977 | ] 978 | 979 | [[package]] 980 | name = "windows_aarch64_gnullvm" 981 | version = "0.52.6" 982 | source = "registry+https://github.com/rust-lang/crates.io-index" 983 | checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" 984 | 985 | [[package]] 986 | name = "windows_aarch64_msvc" 987 | version = "0.52.6" 988 | source = "registry+https://github.com/rust-lang/crates.io-index" 989 | checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" 990 | 991 | [[package]] 992 | name = "windows_i686_gnu" 993 | version = "0.52.6" 994 | source = "registry+https://github.com/rust-lang/crates.io-index" 995 | checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" 996 | 997 | [[package]] 998 | name = "windows_i686_gnullvm" 999 | version = "0.52.6" 1000 | source = "registry+https://github.com/rust-lang/crates.io-index" 1001 | checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" 1002 | 1003 | [[package]] 1004 | name = "windows_i686_msvc" 1005 | version = "0.52.6" 1006 | source = "registry+https://github.com/rust-lang/crates.io-index" 1007 | checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" 1008 | 1009 | [[package]] 1010 | name = "windows_x86_64_gnu" 1011 | version = "0.52.6" 1012 | source = "registry+https://github.com/rust-lang/crates.io-index" 1013 | checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" 1014 | 1015 | [[package]] 1016 | name = "windows_x86_64_gnullvm" 1017 | version = "0.52.6" 1018 | source = "registry+https://github.com/rust-lang/crates.io-index" 1019 | checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" 1020 | 1021 | [[package]] 1022 | name = "windows_x86_64_msvc" 1023 | version = "0.52.6" 1024 | source = "registry+https://github.com/rust-lang/crates.io-index" 1025 | checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" 1026 | 1027 | [[package]] 1028 | name = "wit-bindgen-rt" 1029 | version = "0.39.0" 1030 | source = "registry+https://github.com/rust-lang/crates.io-index" 1031 | checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" 1032 | dependencies = [ 1033 | "bitflags", 1034 | ] 1035 | 1036 | [[package]] 1037 | name = "xattr" 1038 | version = "1.5.0" 1039 | source = "registry+https://github.com/rust-lang/crates.io-index" 1040 | checksum = "0d65cbf2f12c15564212d48f4e3dfb87923d25d611f2aed18f4cb23f0413d89e" 1041 | dependencies = [ 1042 | "libc", 1043 | "rustix", 1044 | ] 1045 | 1046 | [[package]] 1047 | name = "xcp" 1048 | version = "0.24.1" 1049 | dependencies = [ 1050 | "anyhow", 1051 | "cfg-if", 1052 | "clap", 1053 | "crossbeam-channel", 1054 | "fslock", 1055 | "glob", 1056 | "ignore", 1057 | "indicatif", 1058 | "libfs", 1059 | "libxcp", 1060 | "log", 1061 | "num_cpus", 1062 | "rand", 1063 | "rand_distr", 1064 | "rand_xorshift", 1065 | "rustix", 1066 | "simplelog", 1067 | "tempfile", 1068 | "terminal_size", 1069 | "test-case", 1070 | "unbytify", 1071 | "uuid", 1072 | "walkdir", 1073 | "xattr", 1074 | ] 1075 | 1076 | [[package]] 1077 | name = "zerocopy" 1078 | version = "0.8.24" 1079 | source = "registry+https://github.com/rust-lang/crates.io-index" 1080 | checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" 1081 | dependencies = [ 1082 | "zerocopy-derive", 1083 | ] 1084 | 1085 | [[package]] 1086 | name = "zerocopy-derive" 1087 | version = "0.8.24" 1088 | source = "registry+https://github.com/rust-lang/crates.io-index" 1089 | checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" 1090 | dependencies = [ 1091 | "proc-macro2", 1092 | "quote", 1093 | "syn", 1094 | ] 1095 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | ".", 4 | "libxcp", 5 | "libfs", 6 | ] 7 | default-members = [".", "libfs"] 8 | resolver = "2" 9 | 10 | [package] 11 | name = "xcp" 12 | description = "xcp is a (partial) clone of the Unix `cp` command, with more user-friendly feedback and some performance optimisations. See the README for features and limitations." 13 | version = "0.24.1" 14 | edition = "2021" 15 | rust-version = "1.82.0" 16 | 17 | authors = ["Steve Smith "] 18 | homepage = "https://github.com/tarka/xcp" 19 | repository = "https://github.com/tarka/xcp" 20 | readme = "README.md" 21 | 22 | keywords = ["coreutils", "cp", "files", "filesystem"] 23 | categories =["command-line-utilities"] 24 | license = "GPL-3.0-only" 25 | 26 | [features] 27 | default = ["parblock", "use_linux"] 28 | parblock = ["libxcp/parblock"] 29 | use_linux = ["libfs/use_linux", "libxcp/use_linux"] 30 | # For CI; disable feature testing on filesystems that don't support 31 | # it. See .github/workflows/tests.yml 32 | test_no_reflink = ["libfs/test_no_reflink"] 33 | test_no_sparse = ["libfs/test_no_sparse"] 34 | test_no_extents = ["libfs/test_no_extents"] 35 | test_no_sockets = ["libfs/test_no_sockets"] 36 | test_no_acl = ["libfs/test_no_acl"] 37 | test_no_xattr = [] 38 | test_no_symlinks = [] 39 | test_no_perms = [] 40 | test_run_expensive = [] 41 | 42 | [dependencies] 43 | anyhow = "1.0.97" 44 | crossbeam-channel = "0.5.15" 45 | clap = { version = "4.5.35", features = ["derive"] } 46 | glob = "0.3.2" 47 | ignore = "0.4.23" 48 | indicatif = "0.17.11" 49 | libfs = { version = "0.9.1", path = "libfs" } 50 | libxcp = { version = "0.24.1", path = "libxcp" } 51 | log = "0.4.27" 52 | num_cpus = "1.16.0" 53 | simplelog = "0.12.2" 54 | unbytify = "0.2.0" 55 | terminal_size = "0.4.2" 56 | 57 | [dev-dependencies] 58 | cfg-if = "1.0.0" 59 | fslock = "0.2.1" 60 | rand = "0.9.0" 61 | rand_distr = "0.5.1" 62 | rand_xorshift = "0.4.0" 63 | rustix = "1.0.5" 64 | tempfile = "3.19.1" 65 | test-case = "3.3.1" 66 | uuid = { version = "1.16.0", features = ["v4"] } 67 | walkdir = "2.5.0" 68 | xattr = "1.5.0" 69 | 70 | [lints.clippy] 71 | upper_case_acronyms = "allow" 72 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # xcp: An extended cp 2 | 3 | `xcp` is a (partial) clone of the Unix `cp` command. It is not intended as a 4 | full replacement, but as a companion utility with some more user-friendly 5 | feedback and some optimisations that make sense under certain tasks (see 6 | below). 7 | 8 | [![Crates.io](https://img.shields.io/crates/v/xcp.svg?colorA=777777)](https://crates.io/crates/xcp) 9 | ![Github Actions](https://github.com/tarka/xcp/actions/workflows/tests.yml/badge.svg) 10 | [![CircleCI](https://circleci.com/gh/tarka/xcp.svg?style=shield)](https://circleci.com/gh/tarka/xcp) 11 | [![Packaging status](https://repology.org/badge/tiny-repos/xcp.svg)](https://repology.org/project/xcp/versions) 12 | 13 | *Warning*: `xcp` is currently beta-level software and almost certainly contains 14 | bugs and unexpected or inconsistent behaviour. It probably shouldn't be used for 15 | anything critical yet. 16 | 17 | Please note that there are some known issues with copying files from virtual 18 | filesystems (e.g. `/proc`, `/sys`). See [this LWN 19 | article](https://lwn.net/Articles/846403/) for an overview of some of the 20 | complexities of dealing with kernel-generated files. This is a common problem 21 | with file utilities which rely on random access; for example `rsync` has the 22 | same issue. 23 | 24 | ## Installation 25 | 26 | ### Cargo 27 | 28 | `xcp` can be installed directly from `crates.io` with: 29 | ``` 30 | cargo install xcp 31 | ``` 32 | 33 | ### Arch Linux 34 | 35 | [`xcp`](https://aur.archlinux.org/packages/xcp/) is available on the Arch Linux User Repository. If you use an AUR helper, you can execute a command such as this: 36 | ``` 37 | yay -S xcp 38 | ``` 39 | 40 | ### NetBSD 41 | [`xcp`](https://pkgsrc.se/sysutils/xcp) is available on NetBSD from the official repositories. To install it, simply run: 42 | ``` 43 | pkgin install xcp 44 | ``` 45 | 46 | ## Features and Anti-Features 47 | 48 | ### Features 49 | 50 | * Displays a progress-bar, both for directory and single file copies. This can 51 | be disabled with `--no-progress`. 52 | * On Linux it uses `copy_file_range` call to copy files. This is the most 53 | efficient method of file-copying under Linux; in particular it is 54 | filesystem-aware, and can massively speed-up copies on network mounts by 55 | performing the copy operations server-side. However, unlike `copy_file_range` 56 | sparse files are detected and handled appropriately. 57 | * Support for modern filesystem features such as [reflinks](https://btrfs.readthedocs.io/en/latest/Reflink.html). 58 | * Optimised for 'modern' systems (i.e. multiple cores, copious RAM, and 59 | solid-state disks, especially ones connected into the main system bus, 60 | e.g. NVMe). 61 | * Optional aggressive parallelism for systems with parallel IO. Quick 62 | experiments on a modern laptop suggest there may be benefits to parallel 63 | copies on NVMe disks. This is obviously highly system-dependent. 64 | * Switchable 'drivers' to facilitate experimenting with alternative strategies 65 | for copy optimisation. Currently 2 drivers are available: 66 | * 'parfile': the previous hard-coded xcp copy method, which parallelises 67 | tree-walking and per-file copying. This is the default. 68 | * 'parblock': An experimental driver that parallelises copying at the block 69 | level. This has the potential for performance improvements in some 70 | architectures, but increases complexity. Testing is welcome. 71 | * Non-Linux Unix-like OSs (OS X, *BSD) are supported via fall-back operation 72 | (although sparse-files are not yet supported in this case). 73 | * Optionally understands `.gitignore` files to limit the copied directories. 74 | * Optional native file-globbing. 75 | 76 | ### (Possible) future features 77 | 78 | * Conversion of files to sparse where appropriate, as with `cp`'s 79 | `--sparse=always` flag. 80 | * Aggressive sparseness detection with `lseek`. 81 | * On non-Linux OSs sparse-files are not currenty supported but could be added if 82 | supported by the OS. 83 | 84 | ### Differences with `cp` 85 | 86 | * Permissions, xattrs and ACLs are copied by default; this can be disabled with 87 | `--no-perms`. 88 | * Virtual file copies are not supported; for example `/proc` and `/sys` files. 89 | * Character files such as [sockets](https://man7.org/linux/man-pages/man7/unix.7.html) and 90 | [pipes](https://man7.org/linux/man-pages/man3/mkfifo.3.html) are copied as 91 | devices (i.e. via [mknod](https://man7.org/linux/man-pages/man2/mknod.2.html)) 92 | rather than copying their contents as a stream. 93 | * The `--reflink=never` option may silently perform a reflink operation 94 | regardless. This is due to the use of 95 | [copy_file_range](https://man7.org/linux/man-pages/man2/copy_file_range.2.html) 96 | which has no such override and may perform its own optimisations. 97 | * `cp` 'simple' backups are not supported, only numbered. 98 | * Some `cp` options are not available but may be added in the future. 99 | 100 | ## Performance 101 | 102 | Benchmarks are mostly meaningless, but the following are results from a laptop 103 | with an NVMe disk and in single-user mode. The target copy directory is a git 104 | checkout of the Firefox codebase, having been recently gc'd (i.e. a single 4.1GB 105 | pack file). `fstrim -va` and `echo 3 | sudo tee /proc/sys/vm/drop_caches` are 106 | run before each test run to minimise SSD allocation performance interference. 107 | 108 | Note: `xcp` is optimised for 'modern' systems with lots of RAM and solid-state 109 | disks. In particular it is likely to perform worse on spinning disks unless they 110 | are in highly parallel arrays. 111 | 112 | ### Local copy 113 | 114 | * Single 4.1GB file copy, with the kernel cache dropped each run: 115 | * `cp`: ~6.2s 116 | * `xcp`: ~4.2s 117 | * Single 4.1GB file copy, warmed cache (3 runs each): 118 | * `cp`: ~1.85s 119 | * `xcp`: ~1.7s 120 | * Directory copy, kernel cache dropped each run: 121 | * `cp`: ~48s 122 | * `xcp`: ~56s 123 | * Directory copy, warmed cache (3 runs each): 124 | * `cp`: ~6.9s 125 | * `xcp`: ~7.4s 126 | 127 | ### NFS copy 128 | 129 | `xcp` uses `copy_file_range`, which is filesystem aware. On NFSv4 this will result 130 | in the copy occurring server-side rather than transferring across the network. For 131 | large files this can be a significant win: 132 | 133 | * Single 4.1GB file on NFSv4 mount 134 | * `cp`: 6m18s 135 | * `xcp`: 0m37s 136 | -------------------------------------------------------------------------------- /completions/xcp.bash: -------------------------------------------------------------------------------- 1 | _xcp() { 2 | local cur prev words cword 3 | _init_completion || return 4 | 5 | # do not suggest options after -- 6 | local i 7 | for ((i = 1; i < cword; i++)); do 8 | if [[ ${words[$i]} == -- ]]; then 9 | _filedir 10 | return 11 | fi 12 | done 13 | 14 | local options=( 15 | -T 16 | -g 17 | -h 18 | -n 19 | -f 20 | -r 21 | -v 22 | -w 23 | -L 24 | "$(_parse_help "$1" -h)" # long options will be parsed from `--help` 25 | ) 26 | local units='B K M G' # in line with most completions prefer M to MB/MiB 27 | local drivers='parfile parblock' 28 | local reflink='auto always never' 29 | local backup='none numbered auto' 30 | 31 | case "$prev" in 32 | -h | --help) return ;; 33 | 34 | --block-size) 35 | if [[ -z $cur ]]; then 36 | COMPREPLY=(1M) # replace "nothing" with the default block size 37 | else 38 | local num="${cur%%[^0-9]*}" # suggest unit suffixes after numbers 39 | local unit="${cur##*[0-9]}" 40 | COMPREPLY=($(compgen -P "$num" -W "$units" -- "$unit")) 41 | fi 42 | return 43 | ;; 44 | 45 | --reflink) 46 | COMPREPLY=($(compgen -W "$reflink" -- "$cur")) 47 | return 48 | ;; 49 | 50 | --backup) 51 | COMPREPLY=($(compgen -W "$backup" -- "$cur")) 52 | return 53 | ;; 54 | 55 | --driver) 56 | COMPREPLY=($(compgen -W "$drivers" -- "$cur")) 57 | return 58 | ;; 59 | 60 | -w | --workers) 61 | COMPREPLY=($(compgen -W "{0..$(_ncpus)}" -- "$cur")) # 0 == auto 62 | return 63 | ;; 64 | esac 65 | 66 | if [[ $cur == -* ]]; then 67 | COMPREPLY=($(compgen -W "${options[*]}" -- "$cur")) 68 | return 69 | fi 70 | 71 | _filedir # suggest files if nothing else matched 72 | } && complete -F _xcp xcp 73 | 74 | # vim: sw=2 sts=2 et ai ft=bash 75 | # path: /usr/share/bash-completion/completions/xcp 76 | -------------------------------------------------------------------------------- /completions/xcp.fish: -------------------------------------------------------------------------------- 1 | set -l drivers ' 2 | parfile\t"parallelise at the file level (default)" 3 | parblock\t"parallelise at the block level" 4 | ' 5 | 6 | set -l reflinks ' 7 | auto\t"attempt to reflink and fallback to a copy (default)" 8 | always\t"return an error if it cannot reflink" 9 | never\t"always perform a full data copy" 10 | ' 11 | 12 | set -l backup ' 13 | none\t"no backups (default)" 14 | numbered\t"follow the semantics of cp numbered backups" 15 | auto\t"create a numbered backup if previous backup exists" 16 | ' 17 | 18 | # short + long 19 | complete -c xcp -s T -l no-target-directory -d 'Overwrite target directory, do not create a subdirectory' 20 | complete -c xcp -s g -l glob -d 'Expand (glob) filename patterns' 21 | complete -c xcp -s h -l help -f -d 'Print help' 22 | complete -c xcp -s n -l no-clobber -d 'Do not overwrite an existing file' 23 | complete -c xcp -s f -l force -d 'Compatibility only option' 24 | complete -c xcp -s r -l recursive -d 'Copy directories recursively' 25 | complete -c xcp -s v -l verbose -d 'Increase verbosity (can be repeated)' 26 | complete -c xcp -s w -l workers -d 'Workers for recursive copies (0=auto)' -x -a '(seq 0 (getconf _NPROCESSORS_ONLN))' 27 | complete -c xcp -s L -l dereference -d 'Dereference symlinks in source' 28 | complete -c xcp -s o -l ownership -d 'Copy ownship (user/group)' 29 | 30 | # long 31 | complete -c xcp -l fsync -d 'Sync each file to disk after it is written' 32 | complete -c xcp -l target-directory -d 'Copy into a subdirectory of the target' 33 | complete -c xcp -l gitignore -d 'Use .gitignore if present' 34 | complete -c xcp -l no-perms -d 'Do not copy file permissions' 35 | complete -c xcp -l no-timestamps -d 'Do not copy file timestamps' 36 | complete -c xcp -l no-progress -d 'Disable progress bar' 37 | complete -c xcp -l block-size -d 'Block size for file operations' -x -a '(seq 1 16){B,K,M,G}' 38 | complete -c xcp -l driver -d 'Parallelise at the file or at the block level' -x -a "$drivers" 39 | complete -c xcp -l reflink -d 'Whether and how to use reflinks' -x -a "$reflinks" 40 | complete -c xcp -l backup -d 'Whether to create backups of overwritten files' -x -a "$backup" 41 | 42 | # docs: https://fishshell.com/docs/current/completions.html 43 | # path: /usr/share/fish/vendor_completions.d/xcp.fish 44 | # vim: sw=2 sts=2 et ai ft=fish 45 | -------------------------------------------------------------------------------- /completions/xcp.zsh: -------------------------------------------------------------------------------- 1 | #compdef xcp 2 | 3 | typeset -A opt_args 4 | 5 | _xcp() { 6 | local -a args 7 | 8 | # short + long 9 | args+=( 10 | '(- *)'{-h,--help}'[Print help]' 11 | '*'{-v,--verbose}'[Increase verbosity (can be repeated)]' 12 | {-T,--no-target-directory}'[Overwrite target directory, do not create a subdirectory]' 13 | {-g,--glob}'[Expand (glob) filename patterns]' 14 | {-n,--no-clobber}'[Do not overwrite an existing file]' 15 | {-f,--force}'[Compatibility only option]' 16 | {-r,--recursive}'[Copy directories recursively]' 17 | {-w,--workers}'[Workers for recursive copies (0=auto)]:workers:_values workers {0..$(getconf _NPROCESSORS_ONLN)}' 18 | {-L,--dereference}'[Dereference symlinks in source]' 19 | {-o,--ownership}'[Copy ownship (user/group)]' 20 | ) 21 | 22 | # long 23 | args+=( 24 | --block-size'[Block size for file operations]: :_numbers -u bytes -d 1M size B K M G' 25 | --driver'[How to parallelise file operations]:driver:(( 26 | parfile\:"parallelise at the file level (default)" 27 | parblock\:"parallelise at the block level" 28 | ))' 29 | --reflink'[Whether and how to use reflinks]:reflink:(( 30 | auto\:"attempt to reflink and fallback to a copy (default)" 31 | always\:"return an error if it cannot reflink" 32 | never\:"always perform a full data copy" 33 | ))' 34 | --backup'[Whether to create backups of overwritten files]:backup:(( 35 | none\:"no backups (default)" 36 | numbered\:"follow the semantics of cp numbered backups" 37 | auto\:"create a numbered backup if previous backup exists" 38 | ))' 39 | --fsync'[Sync each file to disk after it is written]' 40 | --gitignore'[Use .gitignore if present]' 41 | --no-perms'[Do not copy file permissions]' 42 | --no-timestamps'[Do not copy file timestamps]' 43 | --no-progress'[Disable progress bar]' 44 | --target-directory'[Copy into a subdirectory of the target]: :_files -/' 45 | ) 46 | 47 | # positional 48 | args+=( 49 | '*:paths:_files' 50 | ) 51 | 52 | _arguments -s -S $args 53 | } 54 | 55 | _xcp "$@" 56 | 57 | # vim: sw=2 sts=2 et ai ft=zsh 58 | # path: /usr/share/zsh/site-functions/_xcp 59 | -------------------------------------------------------------------------------- /libfs/.gitignore: -------------------------------------------------------------------------------- 1 | Cargo.lock 2 | -------------------------------------------------------------------------------- /libfs/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "libfs" 3 | description = "`libfs` is a library of file and filesystem operations that is supplementary to `std::fs`" 4 | version = "0.9.1" 5 | edition = "2021" 6 | rust-version = "1.82.0" 7 | 8 | authors = ["Steve Smith "] 9 | homepage = "https://github.com/tarka/xcp/libfs" 10 | repository = "https://github.com/tarka/xcp/libfs" 11 | readme = "README.md" 12 | 13 | keywords = ["coreutils", "files", "filesystem", "sparse"] 14 | categories =["filesystem"] 15 | license = "GPL-3.0-only" 16 | 17 | [features] 18 | default = ["use_linux"] 19 | use_linux = [] 20 | # For CI; disable feature testing on filesystems that don't support 21 | # it. See .github/workflows/tests.yml 22 | test_no_acl = [] 23 | test_no_reflink = [] 24 | test_no_sparse = [] 25 | test_no_extents = [] 26 | test_no_sockets = [] 27 | 28 | [dependencies] 29 | cfg-if = "1.0.0" 30 | libc = "0.2.171" 31 | linux-raw-sys = { version = "0.9.4", features = ["ioctl"] } 32 | log = "0.4.27" 33 | rustix = { version = "1.0.5", features = ["fs"] } 34 | thiserror = "2.0.12" 35 | xattr = "1.5.0" 36 | 37 | [dev-dependencies] 38 | exacl = "0.12.0" 39 | tempfile = "3.19.1" 40 | 41 | [lints.clippy] 42 | upper_case_acronyms = "allow" 43 | -------------------------------------------------------------------------------- /libfs/README.md: -------------------------------------------------------------------------------- 1 | # libfs: Advanced file and fs operations 2 | 3 | `libfs` is a library of file and filesystem operations that is supplementary to 4 | [std::fs](https://doc.rust-lang.org/std/fs/). Current features: 5 | 6 | * High and mid-level functions for creating and copying sparse files. 7 | * Copying will use Linux 8 | [copy_file_range](https://man7.org/linux/man-pages/man2/copy_file_range.2.html) 9 | where possible, with fall-back to userspace. 10 | * Scanning and merging extent information on filesystems that support it. 11 | * File permission copying, including 12 | [xattrs](https://man7.org/linux/man-pages/man7/xattr.7.html). 13 | 14 | Some of the features are Linux specific, but most have fall-back alternative 15 | implementations for other Unix-like OSs. Further support is todo. 16 | 17 | `libfs` is part of the [xcp](https://crates.io/crates/xcp) project. 18 | 19 | [![Crates.io](https://img.shields.io/crates/v/xcp.svg?colorA=777777)](https://crates.io/crates/libfs) 20 | [![doc.rs](https://docs.rs/libfs/badge.svg)](https://docs.rs/libfs) 21 | ![Github Actions](https://github.com/tarka/xcp/actions/workflows/tests.yml/badge.svg) 22 | [![CircleCI](https://circleci.com/gh/tarka/xcp.svg?style=shield)](https://circleci.com/gh/tarka/xcp) 23 | -------------------------------------------------------------------------------- /libfs/src/common.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | 18 | use log::{debug, warn}; 19 | use rustix::fs::{fsync, ftruncate}; 20 | use rustix::io::{pread, pwrite}; 21 | use std::cmp; 22 | use std::fs::{File, FileTimes}; 23 | use std::io::{ErrorKind, Read, Write}; 24 | use std::os::unix::fs::{fchown, MetadataExt}; 25 | use std::path::Path; 26 | use xattr::FileExt; 27 | 28 | use crate::errors::{Result, Error}; 29 | use crate::{Extent, XATTR_SUPPORTED, copy_sparse, probably_sparse, copy_file_bytes}; 30 | 31 | fn copy_xattr(infd: &File, outfd: &File) -> Result<()> { 32 | // FIXME: Flag for xattr. 33 | if XATTR_SUPPORTED { 34 | debug!("Starting xattr copy..."); 35 | for attr in infd.list_xattr()? { 36 | if let Some(val) = infd.get_xattr(&attr)? { 37 | debug!("Copy xattr {:?}", attr); 38 | outfd.set_xattr(attr, val.as_slice())?; 39 | } 40 | } 41 | } 42 | Ok(()) 43 | } 44 | 45 | /// Copy file permissions. Will also copy 46 | /// [xattr](https://man7.org/linux/man-pages/man7/xattr.7.html)'s if 47 | /// possible. 48 | pub fn copy_permissions(infd: &File, outfd: &File) -> Result<()> { 49 | let xr = copy_xattr(infd, outfd); 50 | if let Err(e) = xr { 51 | // FIXME: We don't have a way of detecting if the 52 | // target FS supports XAttr, so assume any error is 53 | // "Unsupported" for now. 54 | warn!("Failed to copy xattrs from {:?}: {}", infd, e); 55 | } 56 | 57 | // FIXME: ACLs, selinux, etc. 58 | 59 | let inmeta = infd.metadata()?; 60 | 61 | debug!("Performing permissions copy"); 62 | outfd.set_permissions(inmeta.permissions())?; 63 | 64 | Ok(()) 65 | } 66 | 67 | /// Copy file timestamps. 68 | pub fn copy_timestamps(infd: &File, outfd: &File) -> Result<()> { 69 | let inmeta = infd.metadata()?; 70 | 71 | debug!("Performing timestamp copy"); 72 | let ftime = FileTimes::new() 73 | .set_accessed(inmeta.accessed()?) 74 | .set_modified(inmeta.modified()?); 75 | outfd.set_times(ftime)?; 76 | 77 | Ok(()) 78 | } 79 | 80 | pub fn copy_owner(infd: &File, outfd: &File) -> Result<()> { 81 | let inmeta = infd.metadata()?; 82 | fchown(outfd, Some(inmeta.uid()), Some(inmeta.gid()))?; 83 | 84 | Ok(()) 85 | } 86 | 87 | pub(crate) fn read_bytes(fd: &File, buf: &mut [u8], off: usize) -> Result { 88 | Ok(pread(fd, buf, off as u64)?) 89 | } 90 | 91 | pub(crate) fn write_bytes(fd: &File, buf: &mut [u8], off: usize) -> Result { 92 | Ok(pwrite(fd, buf, off as u64)?) 93 | } 94 | 95 | /// Copy a block of bytes at an offset between files. Uses Posix pread/pwrite. 96 | pub(crate) fn copy_range_uspace(reader: &File, writer: &File, nbytes: usize, off: usize) -> Result { 97 | // FIXME: For larger buffers we should use a pre-allocated thread-local? 98 | let mut buf = vec![0; nbytes]; 99 | 100 | let mut written: usize = 0; 101 | while written < nbytes { 102 | let next = cmp::min(nbytes - written, nbytes); 103 | let noff = off + written; 104 | 105 | let rlen = match read_bytes(reader, &mut buf[..next], noff) { 106 | Ok(0) => return Err(Error::InvalidSource("Source file ended prematurely.")), 107 | Ok(len) => len, 108 | Err(e) => return Err(e), 109 | }; 110 | 111 | let _wlen = match write_bytes(writer, &mut buf[..rlen], noff) { 112 | Ok(len) if len < rlen => { 113 | return Err(Error::InvalidSource("Failed write to file.")) 114 | } 115 | Ok(len) => len, 116 | Err(e) => return Err(e), 117 | }; 118 | 119 | written += rlen; 120 | } 121 | Ok(written) 122 | } 123 | 124 | /// Slightly modified version of io::copy() that only copies a set amount of bytes. 125 | pub(crate) fn copy_bytes_uspace(mut reader: &File, mut writer: &File, nbytes: usize) -> Result { 126 | let mut buf = vec![0; nbytes]; 127 | 128 | let mut written = 0; 129 | while written < nbytes { 130 | let next = cmp::min(nbytes - written, nbytes); 131 | let len = match reader.read(&mut buf[..next]) { 132 | Ok(0) => return Err(Error::InvalidSource("Source file ended prematurely.")), 133 | Ok(len) => len, 134 | Err(ref e) if e.kind() == ErrorKind::Interrupted => continue, 135 | Err(e) => return Err(e.into()) 136 | }; 137 | writer.write_all(&buf[..len])?; 138 | written += len; 139 | } 140 | Ok(written) 141 | } 142 | 143 | /// Allocate file space on disk. Uses Posix ftruncate(). 144 | pub fn allocate_file(fd: &File, len: u64) -> Result<()> { 145 | Ok(ftruncate(fd, len)?) 146 | } 147 | 148 | /// Merge any contiguous extents in a list. See [merge_extents]. 149 | pub fn merge_extents(extents: Vec) -> Result> { 150 | let mut merged: Vec = vec![]; 151 | 152 | let mut prev: Option = None; 153 | for e in extents { 154 | match prev { 155 | Some(p) => { 156 | if e.start == p.end + 1 { 157 | // Current & prev are contiguous, merge & see what 158 | // comes next. 159 | prev = Some(Extent { 160 | start: p.start, 161 | end: e.end, 162 | shared: p.shared & e.shared, 163 | }); 164 | } else { 165 | merged.push(p); 166 | prev = Some(e); 167 | } 168 | } 169 | // First iter 170 | None => prev = Some(e), 171 | } 172 | } 173 | if let Some(p) = prev { 174 | merged.push(p); 175 | } 176 | 177 | Ok(merged) 178 | } 179 | 180 | 181 | /// Determine if two files are the same by examining their inodes. 182 | pub fn is_same_file(src: &Path, dest: &Path) -> Result { 183 | let sstat = src.metadata()?; 184 | let dstat = dest.metadata()?; 185 | let same = (sstat.ino() == dstat.ino()) 186 | && (sstat.dev() == dstat.dev()); 187 | 188 | Ok(same) 189 | } 190 | 191 | /// Copy a file. This differs from [std::fs::copy] in that it looks 192 | /// for sparse blocks and skips them. 193 | pub fn copy_file(from: &Path, to: &Path) -> Result { 194 | let infd = File::open(from)?; 195 | let len = infd.metadata()?.len(); 196 | 197 | let outfd = File::create(to)?; 198 | allocate_file(&outfd, len)?; 199 | 200 | let total = if probably_sparse(&infd)? { 201 | copy_sparse(&infd, &outfd)? 202 | } else { 203 | copy_file_bytes(&infd, &outfd, len)? as u64 204 | }; 205 | 206 | Ok(total) 207 | } 208 | 209 | /// Sync an open file to disk. Uses `fsync(2)`. 210 | pub fn sync(fd: &File) -> Result<()> { 211 | Ok(fsync(fd)?) 212 | } 213 | 214 | #[cfg(test)] 215 | mod tests { 216 | use super::*; 217 | use std::fs::read; 218 | use std::ops::Range; 219 | use tempfile::tempdir; 220 | 221 | impl From> for Extent { 222 | fn from(r: Range) -> Self { 223 | Extent { 224 | start: r.start, 225 | end: r.end, 226 | shared: false, 227 | } 228 | } 229 | } 230 | 231 | #[test] 232 | fn test_copy_bytes_uspace_large() { 233 | let dir = tempdir().unwrap(); 234 | let from = dir.path().join("from.bin"); 235 | let to = dir.path().join("to.bin"); 236 | let size = 128 * 1024; 237 | let data = "X".repeat(size); 238 | 239 | { 240 | let mut fd: File = File::create(&from).unwrap(); 241 | write!(fd, "{}", data).unwrap(); 242 | } 243 | 244 | { 245 | let infd = File::open(&from).unwrap(); 246 | let outfd = File::create(&to).unwrap(); 247 | let written = copy_bytes_uspace(&infd, &outfd, size).unwrap(); 248 | 249 | assert_eq!(written, size); 250 | } 251 | 252 | assert_eq!(from.metadata().unwrap().len(), to.metadata().unwrap().len()); 253 | 254 | { 255 | let from_data = read(&from).unwrap(); 256 | let to_data = read(&to).unwrap(); 257 | assert_eq!(from_data, to_data); 258 | } 259 | } 260 | 261 | #[test] 262 | fn test_copy_range_uspace_large() { 263 | let dir = tempdir().unwrap(); 264 | let from = dir.path().join("from.bin"); 265 | let to = dir.path().join("to.bin"); 266 | let size = 128 * 1024; 267 | let data = "X".repeat(size); 268 | 269 | { 270 | let mut fd: File = File::create(&from).unwrap(); 271 | write!(fd, "{}", data).unwrap(); 272 | } 273 | 274 | { 275 | let infd = File::open(&from).unwrap(); 276 | let outfd = File::create(&to).unwrap(); 277 | 278 | let blocksize = size / 4; 279 | let mut written = 0; 280 | 281 | for off in (0..4).rev() { 282 | written += copy_range_uspace(&infd, &outfd, blocksize, blocksize * off).unwrap(); 283 | } 284 | 285 | assert_eq!(written, size); 286 | } 287 | 288 | assert_eq!(from.metadata().unwrap().len(), to.metadata().unwrap().len()); 289 | 290 | { 291 | let from_data = read(&from).unwrap(); 292 | let to_data = read(&to).unwrap(); 293 | assert_eq!(from_data, to_data); 294 | } 295 | } 296 | 297 | #[test] 298 | fn test_extent_merge() -> Result<()> { 299 | assert_eq!(merge_extents(vec!())?, vec!()); 300 | assert_eq!(merge_extents( 301 | vec!((0..1).into()))?, 302 | vec!((0..1).into())); 303 | 304 | assert_eq!(merge_extents( 305 | vec!((0..1).into(), 306 | (10..20).into()))?, 307 | vec!((0..1).into(), 308 | (10..20).into())); 309 | assert_eq!(merge_extents( 310 | vec!((0..10).into(), 311 | (11..20).into()))?, 312 | vec!((0..20).into())); 313 | assert_eq!( 314 | merge_extents( 315 | vec!((0..5).into(), 316 | (11..20).into(), 317 | (21..30).into(), 318 | (40..50).into()))?, 319 | vec!((0..5).into(), 320 | (11..30).into(), 321 | (40..50).into()) 322 | ); 323 | assert_eq!( 324 | merge_extents(vec!((0..5).into(), 325 | (11..20).into(), 326 | (21..30).into(), 327 | (40..50).into(), 328 | (51..60).into()))?, 329 | vec!((0..5).into(), 330 | (11..30).into(), 331 | (40..60).into()) 332 | ); 333 | assert_eq!( 334 | merge_extents( 335 | vec!((0..10).into(), 336 | (11..20).into(), 337 | (21..30).into(), 338 | (31..50).into(), 339 | (51..60).into()))?, 340 | vec!((0..60).into()) 341 | ); 342 | Ok(()) 343 | } 344 | 345 | 346 | #[test] 347 | fn test_copy_file() -> Result<()> { 348 | let dir = tempdir()?; 349 | let from = dir.path().join("file.bin"); 350 | let len = 32 * 1024 * 1024; 351 | 352 | { 353 | let mut fd = File::create(&from)?; 354 | let data = "X".repeat(len); 355 | write!(fd, "{}", data).unwrap(); 356 | } 357 | 358 | assert_eq!(len, from.metadata()?.len() as usize); 359 | 360 | let to = dir.path().join("sparse.copy.bin"); 361 | crate::copy_file(&from, &to)?; 362 | 363 | assert_eq!(len, to.metadata()?.len() as usize); 364 | 365 | Ok(()) 366 | } 367 | } 368 | -------------------------------------------------------------------------------- /libfs/src/errors.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | use std::path::PathBuf; 18 | 19 | #[derive(Debug, thiserror::Error)] 20 | pub enum Error { 21 | #[error("Invalid source: {0}")] 22 | InvalidSource(&'static str), 23 | 24 | #[error("Invalid path: {0}")] 25 | InvalidPath(PathBuf), 26 | 27 | #[error(transparent)] 28 | IOError(#[from] std::io::Error), 29 | 30 | #[error(transparent)] 31 | OSError(#[from] rustix::io::Errno), 32 | 33 | #[error("Unsupported operation; this function should never be called on this OS.")] 34 | UnsupportedOperation, 35 | } 36 | 37 | pub type Result = std::result::Result; 38 | -------------------------------------------------------------------------------- /libfs/src/fallback.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | use std::fs::File; 18 | use std::path::Path; 19 | 20 | use log::warn; 21 | 22 | use crate::Extent; 23 | use crate::common::{copy_bytes_uspace, copy_range_uspace}; 24 | use crate::errors::{Result, Error}; 25 | 26 | pub fn copy_file_bytes(infd: &File, outfd: &File, bytes: u64) -> Result { 27 | copy_bytes_uspace(infd, outfd, bytes as usize) 28 | } 29 | 30 | pub fn copy_file_offset(infd: &File, outfd: &File, bytes: u64, off: i64) -> Result { 31 | copy_range_uspace(infd, outfd, bytes as usize, off as usize) 32 | } 33 | 34 | // No sparse file handling by default, needs to be implemented 35 | // per-OS. This effectively disables the following operations. 36 | pub fn probably_sparse(_fd: &File) -> Result { 37 | Ok(false) 38 | } 39 | 40 | pub fn map_extents(_fd: &File) -> Result>> { 41 | // FIXME: Implement for *BSD with lseek? 42 | Ok(None) 43 | } 44 | 45 | pub fn next_sparse_segments(_infd: &File, _outfd: &File, _pos: u64) -> Result<(u64, u64)> { 46 | // FIXME: Implement for *BSD with lseek? 47 | Err(Error::UnsupportedOperation {}) 48 | } 49 | 50 | pub fn copy_sparse(infd: &File, outfd: &File) -> Result { 51 | let len = infd.metadata()?.len(); 52 | copy_file_bytes(&infd, &outfd, len) 53 | .map(|i| i as u64) 54 | } 55 | 56 | pub fn copy_node(src: &Path, _dest: &Path) -> Result<()> { 57 | // FreeBSD `cp` just warns about this, so do the same here. 58 | warn!("Socket copy not supported by this OS: {}", src.to_string_lossy()); 59 | Ok(()) 60 | } 61 | 62 | pub fn reflink(_infd: &File, _outfd: &File) -> Result { 63 | Ok(false) 64 | } 65 | -------------------------------------------------------------------------------- /libfs/src/lib.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | mod common; 18 | mod errors; 19 | 20 | use std::{fs, ops::Range}; 21 | 22 | use cfg_if::cfg_if; 23 | use rustix::fs::FileTypeExt; 24 | 25 | cfg_if! { 26 | if #[cfg(all(target_os = "linux", feature = "use_linux"))] { 27 | mod linux; 28 | use linux as backend; 29 | } else { 30 | mod fallback; 31 | use fallback as backend; 32 | } 33 | } 34 | pub use backend::{ 35 | copy_file_bytes, 36 | copy_file_offset, 37 | copy_node, 38 | copy_sparse, 39 | probably_sparse, 40 | next_sparse_segments, 41 | map_extents, 42 | reflink, 43 | }; 44 | pub use common::{ 45 | allocate_file, 46 | copy_file, 47 | copy_owner, 48 | copy_permissions, 49 | copy_timestamps, 50 | is_same_file, 51 | merge_extents, 52 | sync, 53 | }; 54 | pub use errors::Error; 55 | 56 | /// Flag whether the current OS support 57 | /// [xattrs](https://man7.org/linux/man-pages/man7/xattr.7.html). 58 | pub const XATTR_SUPPORTED: bool = { 59 | // NOTE: The xattr crate has a SUPPORTED_PLATFORM flag, however it 60 | // allows NetBSD, which fails for us, so we stick to platforms we've 61 | // tested. 62 | cfg_if! { 63 | if #[cfg(any(target_os = "linux", target_os = "freebsd"))] { 64 | true 65 | } else { 66 | false 67 | } 68 | } 69 | }; 70 | 71 | /// Enum mapping for various *nix file types. Mapped from 72 | /// [std::fs::FileType] and [rustix::fs::FileTypeExt]. 73 | #[derive(Debug)] 74 | pub enum FileType { 75 | File, 76 | Dir, 77 | Symlink, 78 | Socket, 79 | Fifo, 80 | Char, 81 | Block, 82 | Other 83 | } 84 | 85 | impl From for FileType { 86 | fn from(ft: fs::FileType) -> Self { 87 | if ft.is_dir() { 88 | FileType::Dir 89 | } else if ft.is_file() { 90 | FileType::File 91 | } else if ft.is_symlink() { 92 | FileType::Symlink 93 | } else if ft.is_socket() { 94 | FileType::Socket 95 | } else if ft.is_fifo() { 96 | FileType::Fifo 97 | } else if ft.is_char_device() { 98 | FileType::Char 99 | } else if ft.is_block_device() { 100 | FileType::Block 101 | } else { 102 | FileType::Other 103 | } 104 | } 105 | } 106 | 107 | /// Struct representing a file extent metadata. 108 | #[derive(Debug, PartialEq)] 109 | pub struct Extent { 110 | /// Extent logical start 111 | pub start: u64, 112 | /// Extent logical end 113 | pub end: u64, 114 | /// Whether extent is shared between multiple file. This generally 115 | /// only applies to reflinked files on filesystems that support 116 | /// CoW. 117 | pub shared: bool, 118 | } 119 | 120 | impl From for Range { 121 | fn from(e: Extent) -> Self { 122 | e.start..e.end 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /libfs/src/linux.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018-2019, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | use std::{fs::File, path::Path}; 18 | use std::io; 19 | use std::os::unix::io::AsRawFd; 20 | use std::os::unix::prelude::PermissionsExt; 21 | 22 | use linux_raw_sys::ioctl::{FS_IOC_FIEMAP, FIEMAP_EXTENT_LAST, FICLONE, FIEMAP_EXTENT_SHARED}; 23 | use rustix::fs::CWD; 24 | use rustix::{fs::{copy_file_range, seek, mknodat, FileType, Mode, RawMode, SeekFrom}, io::Errno}; 25 | 26 | use crate::Extent; 27 | use crate::errors::Result; 28 | use crate::common::{copy_bytes_uspace, copy_range_uspace}; 29 | 30 | // Wrapper for copy_file_range(2) that checks for non-fatal errors due 31 | // to limitations of the syscall. 32 | fn try_copy_file_range( 33 | infd: &File, 34 | in_off: Option<&mut u64>, 35 | outfd: &File, 36 | out_off: Option<&mut u64>, 37 | bytes: u64, 38 | ) -> Option> { 39 | let cfr_ret = copy_file_range(infd, in_off, outfd, out_off, bytes as usize); 40 | 41 | match cfr_ret { 42 | Ok(retval) => { 43 | Some(Ok(retval)) 44 | }, 45 | Err(Errno::NOSYS) | Err(Errno::PERM) | Err(Errno::XDEV) => { 46 | None 47 | }, 48 | Err(errno) => { 49 | Some(Err(errno.into())) 50 | }, 51 | } 52 | } 53 | 54 | /// File copy operation that defers file offset tracking to the 55 | /// underlying call. On Linux this attempts to use 56 | /// [copy_file_range](https://man7.org/linux/man-pages/man2/copy_file_range.2.html) 57 | /// and falls back to user-space if that is not available. 58 | pub fn copy_file_bytes(infd: &File, outfd: &File, bytes: u64) -> Result { 59 | try_copy_file_range(infd, None, outfd, None, bytes) 60 | .unwrap_or_else(|| copy_bytes_uspace(infd, outfd, bytes as usize)) 61 | } 62 | 63 | /// File copy operation that that copies a block at offset`off`. On 64 | /// Linux this attempts to use 65 | /// [copy_file_range](https://man7.org/linux/man-pages/man2/copy_file_range.2.html) 66 | /// and falls back to user-space if that is not available. 67 | pub fn copy_file_offset(infd: &File, outfd: &File, bytes: u64, off: i64) -> Result { 68 | let mut off_in = off as u64; 69 | let mut off_out = off as u64; 70 | try_copy_file_range(infd, Some(&mut off_in), outfd, Some(&mut off_out), bytes) 71 | .unwrap_or_else(|| copy_range_uspace(infd, outfd, bytes as usize, off as usize)) 72 | } 73 | 74 | /// Guestimate if file is sparse; if it has less blocks that would be 75 | /// expected for its stated size. This is the same test used by 76 | /// coreutils `cp`. 77 | // FIXME: Should work on *BSD? 78 | pub fn probably_sparse(fd: &File) -> Result { 79 | use std::os::linux::fs::MetadataExt; 80 | const ST_NBLOCKSIZE: u64 = 512; 81 | let stat = fd.metadata()?; 82 | Ok(stat.st_blocks() < stat.st_size() / ST_NBLOCKSIZE) 83 | } 84 | 85 | #[derive(PartialEq, Debug)] 86 | enum SeekOff { 87 | Offset(u64), 88 | EOF, 89 | } 90 | 91 | fn lseek(fd: &File, from: SeekFrom) -> Result { 92 | match seek(fd, from) { 93 | Err(errno) if errno == Errno::NXIO => Ok(SeekOff::EOF), 94 | Err(err) => Err(err.into()), 95 | Ok(off) => Ok(SeekOff::Offset(off)), 96 | } 97 | } 98 | 99 | const FIEMAP_PAGE_SIZE: usize = 32; 100 | 101 | #[repr(C)] 102 | #[derive(Copy, Clone, Debug)] 103 | struct FiemapExtent { 104 | fe_logical: u64, // Logical offset in bytes for the start of the extent 105 | fe_physical: u64, // Physical offset in bytes for the start of the extent 106 | fe_length: u64, // Length in bytes for the extent 107 | fe_reserved64: [u64; 2], 108 | fe_flags: u32, // FIEMAP_EXTENT_* flags for this extent 109 | fe_reserved: [u32; 3], 110 | } 111 | impl FiemapExtent { 112 | fn new() -> FiemapExtent { 113 | FiemapExtent { 114 | fe_logical: 0, 115 | fe_physical: 0, 116 | fe_length: 0, 117 | fe_reserved64: [0; 2], 118 | fe_flags: 0, 119 | fe_reserved: [0; 3], 120 | } 121 | } 122 | } 123 | 124 | #[repr(C)] 125 | #[derive(Copy, Clone, Debug)] 126 | struct FiemapReq { 127 | fm_start: u64, // Logical offset (inclusive) at which to start mapping (in) 128 | fm_length: u64, // Logical length of mapping which userspace cares about (in) 129 | fm_flags: u32, // FIEMAP_FLAG_* flags for request (in/out) 130 | fm_mapped_extents: u32, // Number of extents that were mapped (out) 131 | fm_extent_count: u32, // Size of fm_extents array (in) 132 | fm_reserved: u32, 133 | fm_extents: [FiemapExtent; FIEMAP_PAGE_SIZE], // Array of mapped extents (out) 134 | } 135 | impl FiemapReq { 136 | fn new() -> FiemapReq { 137 | FiemapReq { 138 | fm_start: 0, 139 | fm_length: u64::MAX, 140 | fm_flags: 0, 141 | fm_mapped_extents: 0, 142 | fm_extent_count: FIEMAP_PAGE_SIZE as u32, 143 | fm_reserved: 0, 144 | fm_extents: [FiemapExtent::new(); FIEMAP_PAGE_SIZE], 145 | } 146 | } 147 | } 148 | 149 | fn fiemap(fd: &File, req: &mut FiemapReq) -> Result { 150 | // FIXME: Rustix has an IOCTL mini-framework but it's a little 151 | // tricky and is unsafe anyway. This is simpler for now. 152 | let req_ptr: *mut FiemapReq = req; 153 | if unsafe { libc::ioctl(fd.as_raw_fd(), FS_IOC_FIEMAP as u64, req_ptr) } != 0 { 154 | let oserr = io::Error::last_os_error(); 155 | if oserr.raw_os_error() == Some(libc::EOPNOTSUPP) { 156 | return Ok(false) 157 | } 158 | return Err(oserr.into()); 159 | } 160 | 161 | Ok(true) 162 | } 163 | 164 | /// Attempt to retrieve a map of the underlying allocated extents for 165 | /// a file. Will return [None] if the filesystem doesn't support 166 | /// extents. On Linux this is the raw list from 167 | /// [fiemap](https://docs.kernel.org/filesystems/fiemap.html). See 168 | /// [merge_extents](super::merge_extents) for a tool to merge contiguous extents. 169 | pub fn map_extents(fd: &File) -> Result>> { 170 | let mut req = FiemapReq::new(); 171 | let mut extents = Vec::with_capacity(FIEMAP_PAGE_SIZE); 172 | 173 | loop { 174 | if !fiemap(fd, &mut req)? { 175 | return Ok(None) 176 | } 177 | if req.fm_mapped_extents == 0 { 178 | break; 179 | } 180 | 181 | for i in 0..req.fm_mapped_extents as usize { 182 | let e = req.fm_extents[i]; 183 | let ext = Extent { 184 | start: e.fe_logical, 185 | end: e.fe_logical + e.fe_length, 186 | shared: e.fe_flags & FIEMAP_EXTENT_SHARED != 0, 187 | }; 188 | extents.push(ext); 189 | } 190 | 191 | let last = req.fm_extents[(req.fm_mapped_extents - 1) as usize]; 192 | if last.fe_flags & FIEMAP_EXTENT_LAST != 0 { 193 | break; 194 | } 195 | 196 | // Looks like we're going around again... 197 | req.fm_start = last.fe_logical + last.fe_length; 198 | } 199 | 200 | Ok(Some(extents)) 201 | } 202 | 203 | /// Search the file for the next non-sparse file section. Returns the 204 | /// start and end of the data segment. 205 | // FIXME: Should work on *BSD too? 206 | pub fn next_sparse_segments(infd: &File, outfd: &File, pos: u64) -> Result<(u64, u64)> { 207 | let next_data = match lseek(infd, SeekFrom::Data(pos))? { 208 | SeekOff::Offset(off) => off, 209 | SeekOff::EOF => infd.metadata()?.len(), 210 | }; 211 | let next_hole = match lseek(infd, SeekFrom::Hole(next_data))? { 212 | SeekOff::Offset(off) => off, 213 | SeekOff::EOF => infd.metadata()?.len(), 214 | }; 215 | 216 | lseek(infd, SeekFrom::Start(next_data))?; // FIXME: EOF (but shouldn't happen) 217 | lseek(outfd, SeekFrom::Start(next_data))?; 218 | 219 | Ok((next_data, next_hole)) 220 | } 221 | 222 | /// Copy data between files, looking for sparse blocks and skipping 223 | /// them. 224 | pub fn copy_sparse(infd: &File, outfd: &File) -> Result { 225 | let len = infd.metadata()?.len(); 226 | 227 | let mut pos = 0; 228 | while pos < len { 229 | let (next_data, next_hole) = next_sparse_segments(infd, outfd, pos)?; 230 | 231 | let _written = copy_file_bytes(infd, outfd, next_hole - next_data)?; 232 | pos = next_hole; 233 | } 234 | 235 | Ok(len) 236 | } 237 | 238 | /// Create a clone of a special file (unix socket, char-device, etc.) 239 | pub fn copy_node(src: &Path, dest: &Path) -> Result<()> { 240 | use std::os::unix::fs::MetadataExt; 241 | let meta = src.metadata()?; 242 | let rmode = RawMode::from(meta.permissions().mode()); 243 | let mode = Mode::from_raw_mode(rmode); 244 | let ftype = FileType::from_raw_mode(rmode); 245 | let dev = meta.dev(); 246 | 247 | mknodat(CWD, dest, ftype, mode, dev)?; 248 | Ok(()) 249 | } 250 | 251 | /// Reflink a file. This will reuse the underlying data on disk for 252 | /// the target file, utilising copy-on-write for any future 253 | /// updates. Only certain filesystems support this; if not supported 254 | /// the function returns `false`. 255 | pub fn reflink(infd: &File, outfd: &File) -> Result { 256 | if unsafe { libc::ioctl(outfd.as_raw_fd(), FICLONE as u64, infd.as_raw_fd()) } != 0 { 257 | let oserr = io::Error::last_os_error(); 258 | match oserr.raw_os_error() { 259 | Some(libc::EOPNOTSUPP) 260 | | Some(libc::EINVAL) 261 | | Some(libc::EXDEV) 262 | | Some(libc::ETXTBSY) => 263 | return Ok(false), 264 | _ => 265 | return Err(oserr.into()), 266 | } 267 | } 268 | Ok(true) 269 | } 270 | 271 | #[cfg(test)] 272 | #[allow(unused)] 273 | mod tests { 274 | use super::*; 275 | use crate::{allocate_file, copy_permissions}; 276 | use std::env::{current_dir, var}; 277 | use std::fs::{read, OpenOptions}; 278 | use std::io::{self, Seek, Write}; 279 | use std::iter; 280 | use std::os::unix::net::UnixListener; 281 | use std::path::PathBuf; 282 | use std::process::Command; 283 | use linux_raw_sys::ioctl::FIEMAP_EXTENT_SHARED; 284 | use log::warn; 285 | use rustix::fs::FileTypeExt; 286 | use tempfile::{tempdir_in, TempDir}; 287 | 288 | fn tempdir() -> Result { 289 | // Force into local dir as /tmp might be tmpfs, which doesn't 290 | // support all VFS options (notably fiemap). 291 | Ok(tempdir_in(current_dir()?.join("../target"))?) 292 | } 293 | 294 | #[test] 295 | #[cfg_attr(feature = "test_no_reflink", ignore = "No FS support")] 296 | fn test_reflink() -> Result<()> { 297 | let dir = tempdir()?; 298 | let from = dir.path().join("file.bin"); 299 | let to = dir.path().join("copy.bin"); 300 | let size = 128 * 1024; 301 | 302 | { 303 | let mut fd: File = File::create(&from)?; 304 | let data = "X".repeat(size); 305 | write!(fd, "{}", data)?; 306 | } 307 | 308 | let from_fd = File::open(from)?; 309 | let to_fd = File::create(to)?; 310 | 311 | { 312 | let mut from_map = FiemapReq::new(); 313 | assert!(fiemap(&from_fd, &mut from_map)?); 314 | assert!(from_map.fm_mapped_extents > 0); 315 | // Un-refed file, no shared extents 316 | assert!(from_map.fm_extents[0].fe_flags & FIEMAP_EXTENT_SHARED == 0); 317 | } 318 | 319 | let worked = reflink(&from_fd, &to_fd)?; 320 | assert!(worked); 321 | 322 | { 323 | let mut from_map = FiemapReq::new(); 324 | assert!(fiemap(&from_fd, &mut from_map)?); 325 | assert!(from_map.fm_mapped_extents > 0); 326 | 327 | let mut to_map = FiemapReq::new(); 328 | assert!(fiemap(&to_fd, &mut to_map)?); 329 | assert!(to_map.fm_mapped_extents > 0); 330 | 331 | // Now both have shared extents 332 | assert_eq!(from_map.fm_mapped_extents, to_map.fm_mapped_extents); 333 | assert!(from_map.fm_extents[0].fe_flags & FIEMAP_EXTENT_SHARED != 0); 334 | assert!(to_map.fm_extents[0].fe_flags & FIEMAP_EXTENT_SHARED != 0); 335 | } 336 | 337 | Ok(()) 338 | } 339 | 340 | #[test] 341 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 342 | fn test_sparse_detection_small_data() -> Result<()> { 343 | assert!(!probably_sparse(&File::open("Cargo.toml")?)?); 344 | 345 | let dir = tempdir()?; 346 | let file = dir.path().join("sparse.bin"); 347 | let out = Command::new("/usr/bin/truncate") 348 | .args(["-s", "1M", file.to_str().unwrap()]) 349 | .output()?; 350 | assert!(out.status.success()); 351 | 352 | { 353 | let fd = File::open(&file)?; 354 | assert!(probably_sparse(&fd)?); 355 | } 356 | { 357 | let mut fd = OpenOptions::new().write(true).append(false).open(&file)?; 358 | write!(fd, "test")?; 359 | assert!(probably_sparse(&fd)?); 360 | } 361 | 362 | Ok(()) 363 | } 364 | 365 | #[test] 366 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 367 | fn test_sparse_detection_half() -> Result<()> { 368 | assert!(!probably_sparse(&File::open("Cargo.toml")?)?); 369 | 370 | let dir = tempdir()?; 371 | let file = dir.path().join("sparse.bin"); 372 | let out = Command::new("/usr/bin/truncate") 373 | .args(["-s", "1M", file.to_str().unwrap()]) 374 | .output()?; 375 | assert!(out.status.success()); 376 | { 377 | let mut fd = OpenOptions::new().write(true).append(false).open(&file)?; 378 | let s = "x".repeat(512*1024); 379 | fd.write_all(s.as_bytes())?; 380 | assert!(probably_sparse(&fd)?); 381 | } 382 | 383 | Ok(()) 384 | } 385 | 386 | #[test] 387 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 388 | fn test_copy_bytes_sparse() -> Result<()> { 389 | let dir = tempdir()?; 390 | let file = dir.path().join("sparse.bin"); 391 | let from = dir.path().join("from.txt"); 392 | let data = "test data"; 393 | 394 | { 395 | let mut fd = File::create(&from)?; 396 | write!(fd, "{}", data)?; 397 | } 398 | 399 | let out = Command::new("/usr/bin/truncate") 400 | .args(["-s", "1M", file.to_str().unwrap()]) 401 | .output()?; 402 | assert!(out.status.success()); 403 | 404 | { 405 | let infd = File::open(&from)?; 406 | let outfd: File = OpenOptions::new().write(true).append(false).open(&file)?; 407 | copy_file_bytes(&infd, &outfd, data.len() as u64)?; 408 | } 409 | 410 | assert!(probably_sparse(&File::open(file)?)?); 411 | 412 | Ok(()) 413 | } 414 | 415 | #[test] 416 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 417 | fn test_sparse_copy_middle() -> Result<()> { 418 | let dir = tempdir()?; 419 | let file = dir.path().join("sparse.bin"); 420 | let from = dir.path().join("from.txt"); 421 | let data = "test data"; 422 | 423 | { 424 | let mut fd = File::create(&from)?; 425 | write!(fd, "{}", data)?; 426 | } 427 | 428 | let out = Command::new("/usr/bin/truncate") 429 | .args(["-s", "1M", file.to_str().unwrap()]) 430 | .output()?; 431 | assert!(out.status.success()); 432 | 433 | let offset = 512 * 1024; 434 | { 435 | let infd = File::open(&from)?; 436 | let outfd: File = OpenOptions::new().write(true).append(false).open(&file)?; 437 | let mut off_in = 0; 438 | let mut off_out = offset as u64; 439 | let copied = copy_file_range( 440 | &infd, 441 | Some(&mut off_in), 442 | &outfd, 443 | Some(&mut off_out), 444 | data.len(), 445 | )?; 446 | assert_eq!(copied as usize, data.len()); 447 | } 448 | 449 | assert!(probably_sparse(&File::open(&file)?)?); 450 | 451 | let bytes = read(&file)?; 452 | 453 | assert!(bytes.len() == 1024 * 1024); 454 | assert!(bytes[offset] == b't'); 455 | assert!(bytes[offset + 1] == b'e'); 456 | assert!(bytes[offset + 2] == b's'); 457 | assert!(bytes[offset + 3] == b't'); 458 | assert!(bytes[offset + data.len()] == 0); 459 | 460 | Ok(()) 461 | } 462 | 463 | #[test] 464 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 465 | fn test_copy_range_middle() -> Result<()> { 466 | let dir = tempdir()?; 467 | let file = dir.path().join("sparse.bin"); 468 | let from = dir.path().join("from.txt"); 469 | let data = "test data"; 470 | let offset: usize = 512 * 1024; 471 | 472 | { 473 | let mut fd = File::create(&from)?; 474 | fd.seek(io::SeekFrom::Start(offset as u64))?; 475 | write!(fd, "{}", data)?; 476 | } 477 | 478 | let out = Command::new("/usr/bin/truncate") 479 | .args(["-s", "1M", file.to_str().unwrap()]) 480 | .output()?; 481 | assert!(out.status.success()); 482 | 483 | { 484 | let infd = File::open(&from)?; 485 | let outfd: File = OpenOptions::new().write(true).append(false).open(&file)?; 486 | let copied = 487 | copy_file_offset(&infd, &outfd, data.len() as u64, offset as i64)?; 488 | assert_eq!(copied as usize, data.len()); 489 | } 490 | 491 | assert!(probably_sparse(&File::open(&file)?)?); 492 | 493 | let bytes = read(&file)?; 494 | assert_eq!(bytes.len(), 1024 * 1024); 495 | assert_eq!(bytes[offset], b't'); 496 | assert_eq!(bytes[offset + 1], b'e'); 497 | assert_eq!(bytes[offset + 2], b's'); 498 | assert_eq!(bytes[offset + 3], b't'); 499 | assert_eq!(bytes[offset + data.len()], 0); 500 | 501 | Ok(()) 502 | } 503 | 504 | #[test] 505 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 506 | fn test_lseek_data() -> Result<()> { 507 | let dir = tempdir()?; 508 | let file = dir.path().join("sparse.bin"); 509 | let from = dir.path().join("from.txt"); 510 | let data = "test data"; 511 | let offset = 512 * 1024; 512 | 513 | { 514 | let mut fd = File::create(&from)?; 515 | write!(fd, "{}", data)?; 516 | } 517 | 518 | let out = Command::new("/usr/bin/truncate") 519 | .args(["-s", "1M", file.to_str().unwrap()]) 520 | .output()?; 521 | assert!(out.status.success()); 522 | { 523 | let infd = File::open(&from)?; 524 | let outfd: File = OpenOptions::new().write(true).append(false).open(&file)?; 525 | let mut off_in = 0; 526 | let mut off_out = offset; 527 | let copied = copy_file_range( 528 | &infd, 529 | Some(&mut off_in), 530 | &outfd, 531 | Some(&mut off_out), 532 | data.len(), 533 | )?; 534 | assert_eq!(copied as usize, data.len()); 535 | } 536 | 537 | assert!(probably_sparse(&File::open(&file)?)?); 538 | 539 | let off = lseek(&File::open(&file)?, SeekFrom::Data(0))?; 540 | assert_eq!(off, SeekOff::Offset(offset)); 541 | 542 | Ok(()) 543 | } 544 | 545 | #[test] 546 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 547 | fn test_sparse_rust_seek() -> Result<()> { 548 | let dir = tempdir()?; 549 | let file = dir.path().join("sparse.bin"); 550 | 551 | let data = "c00lc0d3"; 552 | 553 | { 554 | let mut fd = File::create(&file)?; 555 | write!(fd, "{}", data)?; 556 | 557 | fd.seek(io::SeekFrom::Start(1024 * 4096))?; 558 | write!(fd, "{}", data)?; 559 | 560 | fd.seek(io::SeekFrom::Start(4096 * 4096 - data.len() as u64))?; 561 | write!(fd, "{}", data)?; 562 | } 563 | 564 | assert!(probably_sparse(&File::open(&file)?)?); 565 | 566 | let bytes = read(&file)?; 567 | assert!(bytes.len() == 4096 * 4096); 568 | 569 | let offset = 1024 * 4096; 570 | assert!(bytes[offset] == b'c'); 571 | assert!(bytes[offset + 1] == b'0'); 572 | assert!(bytes[offset + 2] == b'0'); 573 | assert!(bytes[offset + 3] == b'l'); 574 | assert!(bytes[offset + data.len()] == 0); 575 | 576 | Ok(()) 577 | } 578 | 579 | #[test] 580 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 581 | fn test_lseek_no_data() -> Result<()> { 582 | let dir = tempdir()?; 583 | let file = dir.path().join("sparse.bin"); 584 | 585 | let out = Command::new("/usr/bin/truncate") 586 | .args(["-s", "1M", file.to_str().unwrap()]) 587 | .output()?; 588 | assert!(out.status.success()); 589 | assert!(probably_sparse(&File::open(&file)?)?); 590 | 591 | let fd = File::open(&file)?; 592 | let off = lseek(&fd, SeekFrom::Data(0))?; 593 | assert!(off == SeekOff::EOF); 594 | 595 | Ok(()) 596 | } 597 | 598 | #[test] 599 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 600 | fn test_allocate_file_is_sparse() -> Result<()> { 601 | let dir = tempdir()?; 602 | let file = dir.path().join("sparse.bin"); 603 | let len = 32 * 1024 * 1024; 604 | 605 | { 606 | let fd = File::create(&file)?; 607 | allocate_file(&fd, len)?; 608 | } 609 | 610 | assert_eq!(len, file.metadata()?.len()); 611 | assert!(probably_sparse(&File::open(&file)?)?); 612 | 613 | Ok(()) 614 | } 615 | 616 | #[test] 617 | #[cfg_attr(feature = "test_no_extents", ignore = "No FS support")] 618 | fn test_empty_extent() -> Result<()> { 619 | let dir = tempdir()?; 620 | let file = dir.path().join("sparse.bin"); 621 | 622 | let out = Command::new("/usr/bin/truncate") 623 | .args(["-s", "1M", file.to_str().unwrap()]) 624 | .output()?; 625 | assert!(out.status.success()); 626 | 627 | let fd = File::open(file)?; 628 | 629 | let extents_p = map_extents(&fd)?; 630 | assert!(extents_p.is_some()); 631 | let extents = extents_p.unwrap(); 632 | assert_eq!(extents.len(), 0); 633 | 634 | Ok(()) 635 | } 636 | 637 | #[test] 638 | #[cfg_attr(feature = "test_no_extents", ignore = "No FS support")] 639 | fn test_extent_fetch() -> Result<()> { 640 | let dir = tempdir()?; 641 | let file = dir.path().join("sparse.bin"); 642 | let from = dir.path().join("from.txt"); 643 | let data = "test data"; 644 | 645 | { 646 | let mut fd = File::create(&from)?; 647 | write!(fd, "{}", data)?; 648 | } 649 | 650 | let out = Command::new("/usr/bin/truncate") 651 | .args(["-s", "1M", file.to_str().unwrap()]) 652 | .output()?; 653 | assert!(out.status.success()); 654 | 655 | let offset = 512 * 1024; 656 | { 657 | let infd = File::open(&from)?; 658 | let outfd: File = OpenOptions::new().write(true).append(false).open(&file)?; 659 | let mut off_in = 0; 660 | let mut off_out = offset; 661 | let copied = copy_file_range( 662 | &infd, 663 | Some(&mut off_in), 664 | &outfd, 665 | Some(&mut off_out), 666 | data.len(), 667 | )?; 668 | assert_eq!(copied as usize, data.len()); 669 | } 670 | 671 | let fd = File::open(file)?; 672 | 673 | let extents_p = map_extents(&fd)?; 674 | assert!(extents_p.is_some()); 675 | let extents = extents_p.unwrap(); 676 | assert_eq!(extents.len(), 1); 677 | assert_eq!(extents[0].start, offset); 678 | assert_eq!(extents[0].end, offset + 4 * 1024); // FIXME: Assume 4k blocks 679 | assert!(!extents[0].shared); 680 | 681 | Ok(()) 682 | } 683 | 684 | #[test] 685 | #[cfg_attr(feature = "test_no_extents", ignore = "No FS support")] 686 | fn test_extent_fetch_many() -> Result<()> { 687 | let dir = tempdir()?; 688 | let file = dir.path().join("sparse.bin"); 689 | 690 | let out = Command::new("/usr/bin/truncate") 691 | .args(["-s", "1M", file.to_str().unwrap()]) 692 | .output()?; 693 | assert!(out.status.success()); 694 | 695 | let fsize = 1024 * 1024; 696 | // FIXME: Assumes 4k blocks 697 | let bsize = 4 * 1024; 698 | let block = iter::repeat_n(0xff_u8, bsize).collect::>(); 699 | 700 | let mut fd = OpenOptions::new().write(true).append(false).open(&file)?; 701 | // Skip every-other block 702 | for off in (0..fsize).step_by(bsize * 2) { 703 | lseek(&fd, SeekFrom::Start(off))?; 704 | fd.write_all(block.as_slice())?; 705 | } 706 | 707 | let extents_p = map_extents(&fd)?; 708 | assert!(extents_p.is_some()); 709 | let extents = extents_p.unwrap(); 710 | assert_eq!(extents.len(), fsize as usize / bsize / 2); 711 | 712 | Ok(()) 713 | } 714 | 715 | #[test] 716 | #[cfg_attr(feature = "test_no_extents", ignore = "No FS support")] 717 | fn test_extent_not_sparse() -> Result<()> { 718 | let dir = tempdir()?; 719 | let file = dir.path().join("file.bin"); 720 | let size = 128 * 1024; 721 | 722 | { 723 | let mut fd: File = File::create(&file)?; 724 | let data = "X".repeat(size); 725 | write!(fd, "{}", data)?; 726 | } 727 | 728 | let fd = File::open(file)?; 729 | let extents_p = map_extents(&fd)?; 730 | assert!(extents_p.is_some()); 731 | let extents = extents_p.unwrap(); 732 | 733 | assert_eq!(1, extents.len()); 734 | assert_eq!(0u64, extents[0].start); 735 | assert_eq!(size as u64, extents[0].end); 736 | 737 | Ok(()) 738 | } 739 | 740 | #[test] 741 | #[cfg_attr(feature = "test_no_extents", ignore = "No FS support")] 742 | fn test_extent_unsupported_fs() -> Result<()> { 743 | let file = "/proc/cpuinfo"; 744 | let fd = File::open(file)?; 745 | let extents_p = map_extents(&fd)?; 746 | assert!(extents_p.is_none()); 747 | 748 | Ok(()) 749 | } 750 | 751 | #[test] 752 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 753 | fn test_copy_file_sparse() -> Result<()> { 754 | let dir = tempdir()?; 755 | let from = dir.path().join("sparse.bin"); 756 | let len = 32 * 1024 * 1024; 757 | 758 | { 759 | let fd = File::create(&from)?; 760 | allocate_file(&fd, len)?; 761 | } 762 | 763 | assert_eq!(len, from.metadata()?.len()); 764 | assert!(probably_sparse(&File::open(&from)?)?); 765 | 766 | let to = dir.path().join("sparse.copy.bin"); 767 | crate::copy_file(&from, &to)?; 768 | 769 | assert_eq!(len, to.metadata()?.len()); 770 | assert!(probably_sparse(&File::open(&to)?)?); 771 | 772 | Ok(()) 773 | } 774 | 775 | #[test] 776 | #[cfg_attr(feature = "test_no_sockets", ignore = "No FS support")] 777 | fn test_copy_socket() { 778 | let dir = tempdir().unwrap(); 779 | let from = dir.path().join("from.sock"); 780 | let to = dir.path().join("to.sock"); 781 | 782 | let _sock = UnixListener::bind(&from).unwrap(); 783 | assert!(from.metadata().unwrap().file_type().is_socket()); 784 | 785 | copy_node(&from, &to).unwrap(); 786 | 787 | assert!(to.exists()); 788 | assert!(to.metadata().unwrap().file_type().is_socket()); 789 | } 790 | 791 | 792 | #[test] 793 | #[cfg_attr(feature = "test_no_acl", ignore = "No FS support")] 794 | fn test_copy_acl() -> Result<()> { 795 | use exacl::{getfacl, AclEntry, Perm, setfacl}; 796 | 797 | let dir = tempdir()?; 798 | let from = dir.path().join("file.bin"); 799 | let to = dir.path().join("copy.bin"); 800 | let data = "X".repeat(1024); 801 | 802 | { 803 | let mut fd: File = File::create(&from)?; 804 | write!(fd, "{}", data)?; 805 | 806 | let mut fd: File = File::create(&to)?; 807 | write!(fd, "{}", data)?; 808 | } 809 | 810 | let acl = AclEntry::allow_user("mail", Perm::READ, None); 811 | 812 | let mut from_acl = getfacl(&from, None)?; 813 | from_acl.push(acl.clone()); 814 | setfacl(&[&from], &from_acl, None)?; 815 | 816 | { 817 | let from_fd: File = File::open(&from)?; 818 | let to_fd: File = File::open(&to)?; 819 | copy_permissions(&from_fd, &to_fd)?; 820 | } 821 | 822 | let to_acl = getfacl(&from, None)?; 823 | assert!(to_acl.contains(&acl)); 824 | 825 | Ok(()) 826 | } 827 | } 828 | -------------------------------------------------------------------------------- /libxcp/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "libxcp" 3 | description = "`libxcp` is a high-level file-copy engine with support for multi-threading, fine-grained progress feedback, pluggable drivers, and `.gitignore` filters. `libxcp` provides the core functionality of `xcp`." 4 | version = "0.24.1" 5 | edition = "2021" 6 | rust-version = "1.82.0" 7 | 8 | authors = ["Steve Smith "] 9 | homepage = "https://github.com/tarka/xcp" 10 | repository = "https://github.com/tarka/xcp" 11 | readme = "README.md" 12 | 13 | keywords = ["coreutils", "cp", "files", "filesystem"] 14 | categories =["filesystem"] 15 | license = "GPL-3.0-only" 16 | 17 | [features] 18 | default = ["parblock", "use_linux"] 19 | parblock = [] 20 | use_linux = ["libfs/use_linux"] 21 | 22 | [dependencies] 23 | anyhow = "1.0.97" 24 | blocking-threadpool = "1.0.1" 25 | cfg-if = "1.0.0" 26 | crossbeam-channel = "0.5.15" 27 | ignore = "0.4.23" 28 | libfs = { version = "0.9.1", path = "../libfs" } 29 | log = "0.4.27" 30 | num_cpus = "1.16.0" 31 | regex = "1.11.1" 32 | thiserror = "2.0.12" 33 | walkdir = "2.5.0" 34 | 35 | [dev-dependencies] 36 | tempfile = "3.19.1" 37 | 38 | [lints.clippy] 39 | upper_case_acronyms = "allow" 40 | -------------------------------------------------------------------------------- /libxcp/README.md: -------------------------------------------------------------------------------- 1 | # libxcp: High-level file-copy engine 2 | 3 | `libxcp` is a high-level file-copy engine. It has a support for multi-threading, 4 | fine-grained progress feedback, pluggable drivers, and `.gitignore` filters. 5 | `libxcp` is the core functionality of the [xcp](https://crates.io/crates/xcp) 6 | command-line utility. 7 | 8 | [![Crates.io](https://img.shields.io/crates/v/xcp.svg?colorA=777777)](https://crates.io/crates/libxcp) 9 | [![doc.rs](https://docs.rs/libxcp/badge.svg)](https://docs.rs/libxcp) 10 | ![Github Actions](https://github.com/tarka/xcp/actions/workflows/tests.yml/badge.svg) 11 | [![CircleCI](https://circleci.com/gh/tarka/xcp.svg?style=shield)](https://circleci.com/gh/tarka/xcp) 12 | 13 | ### Features 14 | 15 | * On Linux it uses `copy_file_range` call to copy files. This is the most 16 | efficient method of file-copying under Linux; in particular it is 17 | filesystem-aware, and can massively speed-up copies on network mounts by 18 | performing the copy operations server-side. However, unlike `copy_file_range` 19 | sparse files are detected and handled appropriately. 20 | * Support for modern filesystem features such as [reflinks](https://btrfs.readthedocs.io/en/latest/Reflink.html). 21 | * Optimised for 'modern' systems (i.e. multiple cores, copious RAM, and 22 | solid-state disks, especially ones connected into the main system bus, 23 | e.g. NVMe). 24 | * Optional aggressive parallelism for systems with parallel IO. Quick 25 | experiments on a modern laptop suggest there may be benefits to parallel 26 | copies on NVMe disks. This is obviously highly system-dependent. 27 | * Switchable 'drivers' to facilitate experimenting with alternative strategies 28 | for copy optimisation. Currently 2 drivers are available: 29 | * 'parfile': the previous hard-coded xcp copy method, which parallelises 30 | tree-walking and per-file copying. This is the default. 31 | * 'parblock': An experimental driver that parallelises copying at the block 32 | level. This has the potential for performance improvements in some 33 | architectures, but increases complexity. Testing is welcome. 34 | * Non-Linux Unix-like OSs (OS X, *BSD) are supported via fall-back operation 35 | (although sparse-files are not yet supported in this case). 36 | * Optionally understands `.gitignore` files to limit the copied directories. 37 | 38 | ## Testing 39 | 40 | `libxcp` itself doesn't have many tests; the top-level `xcp` application however 41 | has a full functional test suite, including fuzzed stress-tests. This should be 42 | considered the test suite for now. 43 | -------------------------------------------------------------------------------- /libxcp/src/backup.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | path::{Path, PathBuf}, 3 | env::current_dir, sync::OnceLock, fs::ReadDir, 4 | }; 5 | 6 | use regex::Regex; 7 | 8 | use crate::{errors::{Result, XcpError}, config::{Config, Backup}}; 9 | 10 | const BAK_PATTTERN: &str = r"^\~(\d+)\~$"; 11 | static BAK_REGEX: OnceLock = OnceLock::new(); 12 | 13 | fn get_regex() -> &'static Regex { 14 | // Fixed regex, so should never error. 15 | BAK_REGEX.get_or_init(|| Regex::new(BAK_PATTTERN).unwrap()) 16 | } 17 | 18 | pub(crate) fn get_backup_path(file: &Path) -> Result { 19 | let num = next_backup_num(file)?; 20 | let suffix = format!(".~{}~", num); 21 | // Messy but PathBuf has no concept of mulitiple extensions. 22 | let mut bstr = file.to_path_buf().into_os_string(); 23 | bstr.push(suffix); 24 | let backup = PathBuf::from(bstr); 25 | Ok(backup) 26 | } 27 | 28 | pub(crate) fn needs_backup(file: &Path, conf: &Config) -> Result { 29 | let need = match conf.backup { 30 | Backup::None => false, 31 | Backup::Auto if file.exists() => { 32 | has_backup(file)? 33 | } 34 | Backup::Numbered if file.exists() => true, 35 | _ => false, 36 | }; 37 | Ok(need) 38 | } 39 | 40 | fn ls_file_dir(file: &Path) -> Result { 41 | let cwd = current_dir()?; 42 | let ls_dir = file.parent() 43 | .map(|p| if p.as_os_str().is_empty() { 44 | &cwd 45 | } else { 46 | p 47 | }) 48 | .unwrap_or(&cwd) 49 | .read_dir()?; 50 | Ok(ls_dir) 51 | } 52 | 53 | fn filename(path: &Path) -> Result { 54 | let fname = path.file_name() 55 | .ok_or(XcpError::InvalidArguments(format!("Invalid path found: {:?}", path)))? 56 | .to_string_lossy(); 57 | Ok(fname.to_string()) 58 | } 59 | 60 | fn has_backup(file: &Path) -> Result { 61 | let fname = filename(file)?; 62 | let exists = ls_file_dir(file)? 63 | .any(|der| if let Ok(de) = der { 64 | is_num_backup(&fname, &de.path()).is_some() 65 | } else { 66 | false 67 | }); 68 | Ok(exists) 69 | } 70 | 71 | fn next_backup_num(file: &Path) -> Result { 72 | let fname = filename(file)?; 73 | let current = ls_file_dir(file)? 74 | .filter_map(|der| is_num_backup(&fname, &der.ok()?.path())) 75 | .max() 76 | .unwrap_or(0); 77 | Ok(current + 1) 78 | } 79 | 80 | fn is_num_backup(base_file: &str, candidate: &Path) -> Option { 81 | let cname = candidate 82 | .file_name()? 83 | .to_str()?; 84 | if !cname.starts_with(base_file) { 85 | return None 86 | } 87 | let ext = candidate 88 | .extension()? 89 | .to_string_lossy(); 90 | let num = get_regex() 91 | .captures(&ext)? 92 | .get(1)? 93 | .as_str() 94 | .parse::() 95 | .ok()?; 96 | Some(num) 97 | } 98 | 99 | #[cfg(test)] 100 | mod tests { 101 | use super::*; 102 | use std::{path::PathBuf, fs::File}; 103 | use tempfile::TempDir; 104 | 105 | #[test] 106 | fn test_is_backup() { 107 | let cand = PathBuf::from("/some/path/file.txt.~123~"); 108 | 109 | let bnum = is_num_backup("file.txt", &cand); 110 | assert!(bnum.is_some()); 111 | assert_eq!(123, bnum.unwrap()); 112 | 113 | let bnum = is_num_backup("other_file.txt", &cand); 114 | assert!(bnum.is_none()); 115 | 116 | let bnum = is_num_backup("le.txt", &cand); 117 | assert!(bnum.is_none()); 118 | } 119 | 120 | #[test] 121 | fn test_backup_num_scan() -> Result<()> { 122 | let tdir = TempDir::new()?; 123 | let dir = tdir.path(); 124 | let base = dir.join("file.txt"); 125 | 126 | { 127 | File::create(&base)?; 128 | } 129 | let next = next_backup_num(&base)?; 130 | assert_eq!(1, next); 131 | 132 | { 133 | File::create(dir.join("file.txt.~123~"))?; 134 | } 135 | let next = next_backup_num(&base)?; 136 | assert_eq!(124, next); 137 | 138 | { 139 | File::create(dir.join("file.txt.~999~"))?; 140 | } 141 | let next = next_backup_num(&base)?; 142 | assert_eq!(1000, next); 143 | 144 | Ok(()) 145 | } 146 | 147 | #[test] 148 | fn test_gen_backup_path() -> Result<()> { 149 | let tdir = TempDir::new()?; 150 | let dir = tdir.path(); 151 | let base = dir.join("file.txt"); 152 | { 153 | File::create(&base)?; 154 | } 155 | 156 | let backup = get_backup_path(&base)?; 157 | let mut bs = base.into_os_string(); 158 | bs.push(".~1~"); 159 | assert_eq!(PathBuf::from(bs), backup); 160 | 161 | Ok(()) 162 | } 163 | 164 | #[test] 165 | fn test_needs_backup() -> Result<()> { 166 | let tdir = TempDir::new()?; 167 | let dir = tdir.path(); 168 | let base = dir.join("file.txt"); 169 | 170 | { 171 | File::create(&base)?; 172 | } 173 | assert!(!has_backup(&base)?); 174 | 175 | { 176 | File::create(dir.join("file.txt.~123~"))?; 177 | } 178 | assert!(has_backup(&base)?); 179 | 180 | { 181 | File::create(dir.join("file.txt.~999~"))?; 182 | } 183 | assert!(has_backup(&base)?); 184 | 185 | Ok(()) 186 | } 187 | 188 | } 189 | -------------------------------------------------------------------------------- /libxcp/src/config.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2024, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | //! Driver configuration support. 18 | 19 | use std::result; 20 | use std::str::FromStr; 21 | 22 | use crate::errors::XcpError; 23 | 24 | /// Enum defining configuration options for handling 25 | /// [reflinks](https://btrfs.readthedocs.io/en/latest/Reflink.html). [FromStr] 26 | /// is supported. 27 | #[derive(Clone, Copy, Debug, Default, PartialEq)] 28 | pub enum Reflink { 29 | /// Attempt to reflink and fallback to a copy if it is not 30 | /// possible. 31 | #[default] 32 | Auto, 33 | /// Always attempt a reflink; return an error if not supported. 34 | Always, 35 | /// Always perform a full data copy. Note: when using Linux 36 | /// accelerated copy operations (the default when available) the 37 | /// kernel may choose to reflink rather than perform a fully copy 38 | /// regardless of this setting. 39 | Never, 40 | } 41 | 42 | // String conversion helper as a convenience for command-line parsing. 43 | impl FromStr for Reflink { 44 | type Err = XcpError; 45 | 46 | fn from_str(s: &str) -> result::Result { 47 | match s.to_lowercase().as_str() { 48 | "always" => Ok(Reflink::Always), 49 | "auto" => Ok(Reflink::Auto), 50 | "never" => Ok(Reflink::Never), 51 | _ => Err(XcpError::InvalidArguments(format!("Unexpected value for 'reflink': {}", s))), 52 | } 53 | } 54 | } 55 | 56 | /// Enum defining configuration options for handling backups of 57 | /// overwritten files. [FromStr] is supported. 58 | #[derive(Clone, Copy, Debug, PartialEq)] 59 | pub enum Backup { 60 | /// Do not create backups. 61 | None, 62 | /// Create a numbered backup if a previous backup exists. 63 | Auto, 64 | /// Create numbered backups. Numbered backups follow the semantics 65 | /// of `cp` numbered backups (e.g. `file.txt.~123~`). 66 | Numbered, 67 | } 68 | 69 | impl FromStr for Backup { 70 | type Err = XcpError; 71 | 72 | fn from_str(s: &str) -> result::Result { 73 | match s.to_lowercase().as_str() { 74 | "none" | "off" => Ok(Backup::None), 75 | "auto" => Ok(Backup::Auto), 76 | "numbered" => Ok(Backup::Numbered), 77 | _ => Err(XcpError::InvalidArguments(format!("Unexpected value for 'backup': {}", s))), 78 | } 79 | } 80 | } 81 | 82 | /// A structure defining the runtime options for copy-drivers. This 83 | /// would normally be passed to `load_driver()`. 84 | #[derive(Clone, Debug)] 85 | pub struct Config { 86 | /// Number of parallel workers. 0 means use the number of logical 87 | /// CPUs (the default). 88 | pub workers: usize, 89 | 90 | /// Block size for operations. Defaults to the full file size. Use 91 | /// a smaller value for finer-grained feedback. 92 | pub block_size: u64, 93 | 94 | /// Use .gitignore if present. 95 | /// 96 | /// NOTE: This is fairly basic at the moment, and only honours a 97 | /// .gitignore in the directory root for each source directory; 98 | /// global or sub-directory ignores are skipped. Default is 99 | /// `false`. 100 | pub gitignore: bool, 101 | 102 | /// Do not overwrite existing files. Default is `false`. 103 | pub no_clobber: bool, 104 | 105 | /// Do not copy the file permissions. Default is `false`. 106 | pub no_perms: bool, 107 | 108 | /// Do not copy the file permissions. Default is `false`. 109 | pub no_timestamps: bool, 110 | 111 | /// Copy ownership. 112 | /// 113 | /// Whether to copy ownship (user/group). This option requires 114 | /// root permissions or appropriate capabilities; if the attempt 115 | /// to copy ownership fails a warning is issued but the operation 116 | /// continues. 117 | pub ownership: bool, 118 | 119 | /// Dereference symlinks. Default is `false`. 120 | pub dereference: bool, 121 | 122 | /// Target should not be a directory. 123 | /// 124 | /// Analogous to cp's no-target-directory. Expected behavior is that when 125 | /// copying a directory to another directory, instead of creating a sub-folder 126 | /// in target, overwrite target. Default is 'false`. 127 | pub no_target_directory: bool, 128 | 129 | /// Sync each file to disk after writing. Default is `false`. 130 | pub fsync: bool, 131 | 132 | /// Reflink options. 133 | /// 134 | /// Whether and how to use reflinks. 'auto' (the default) will 135 | /// attempt to reflink and fallback to a copy if it is not 136 | /// possible, 'always' will return an error if it cannot reflink, 137 | /// and 'never' will always perform a full data copy. 138 | pub reflink: Reflink, 139 | 140 | /// Backup options 141 | /// 142 | /// Whether to create backups of overwritten files. Current 143 | /// options are `None` or 'Numbered'. Numbered backups follow the 144 | /// semantics of `cp` numbered backups 145 | /// (e.g. `file.txt.~123~`). Default is `None`. 146 | pub backup: Backup, 147 | } 148 | 149 | impl Config { 150 | pub(crate) fn num_workers(&self) -> usize { 151 | if self.workers == 0 { 152 | num_cpus::get() 153 | } else { 154 | self.workers 155 | } 156 | } 157 | } 158 | 159 | impl Default for Config { 160 | fn default() -> Self { 161 | Config { 162 | workers: num_cpus::get(), 163 | block_size: u64::MAX, 164 | gitignore: false, 165 | no_clobber: false, 166 | no_perms: false, 167 | no_timestamps: false, 168 | ownership: false, 169 | dereference: false, 170 | no_target_directory: false, 171 | fsync: false, 172 | reflink: Reflink::Auto, 173 | backup: Backup::None, 174 | } 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /libxcp/src/drivers/mod.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018-2019, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | //! Support for pluggable copy drivers. 18 | //! 19 | //! Two drivers are currently supported: 20 | //! * `parfile`: Parallelise copying at the file level. This can improve 21 | //! speed on modern NVME devices, but can bottleneck on larger files. 22 | //! * `parblock`: Parallelise copying at the block level. Block-size is 23 | //! configurable. This can have better performance for large files, 24 | //! but has a higher overhead. 25 | //! 26 | //! Drivers are configured with the [Config] struct. A convenience 27 | //! function [load_driver()] is provided to load a dynamic-dispatched 28 | //! instance of each driver. 29 | //! 30 | //! # Example 31 | //! 32 | //! See the example in top-level module. 33 | 34 | pub mod parfile; 35 | #[cfg(feature = "parblock")] 36 | pub mod parblock; 37 | 38 | use std::path::{Path, PathBuf}; 39 | use std::result; 40 | use std::str::FromStr; 41 | use std::sync::Arc; 42 | 43 | use crate::config::Config; 44 | use crate::errors::{Result, XcpError}; 45 | use crate::feedback::StatusUpdater; 46 | 47 | /// The trait specifying driver operations; drivers should implement 48 | /// this. 49 | pub trait CopyDriver { 50 | /// Recursively copy a set of directories or files to a 51 | /// destination. `dest` can be a file if a single file is provided 52 | /// the source. `StatusUpdater.send()` will be called with 53 | /// `StatusUpdate` objects depending on the driver configuration. 54 | /// `copy()` itself will block until all work is complete, so 55 | /// should be run in a thread if real-time updates are required. 56 | fn copy(&self, sources: Vec, dest: &Path, stats: Arc) -> Result<()>; 57 | } 58 | 59 | /// An enum specifing the driver to use. This is just a helper for 60 | /// applications to use with [load_driver()]. [FromStr] is implemented 61 | /// to help with this. 62 | #[derive(Debug, Clone, Copy)] 63 | pub enum Drivers { 64 | ParFile, 65 | #[cfg(feature = "parblock")] 66 | ParBlock, 67 | } 68 | 69 | // String conversion helper as a convenience for command-line parsing. 70 | impl FromStr for Drivers { 71 | type Err = XcpError; 72 | 73 | fn from_str(s: &str) -> result::Result { 74 | match s.to_lowercase().as_str() { 75 | "parfile" => Ok(Drivers::ParFile), 76 | #[cfg(feature = "parblock")] 77 | "parblock" => Ok(Drivers::ParBlock), 78 | _ => Err(XcpError::UnknownDriver(s.to_owned())), 79 | } 80 | } 81 | } 82 | 83 | /// Load and configure the given driver. 84 | pub fn load_driver(driver: Drivers, config: &Arc) -> Result> { 85 | let driver_impl: Box = match driver { 86 | Drivers::ParFile => Box::new(parfile::Driver::new(config.clone())?), 87 | #[cfg(feature = "parblock")] 88 | Drivers::ParBlock => Box::new(parblock::Driver::new(config.clone())?), 89 | }; 90 | 91 | Ok(driver_impl) 92 | } 93 | -------------------------------------------------------------------------------- /libxcp/src/drivers/parblock.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | //! Parallelise copying at the block level. Block-size is 18 | //! configurable. This can have better performance for large files, 19 | //! but has a higher overhead. 20 | 21 | use std::cmp; 22 | use std::fs::remove_file; 23 | use std::ops::Range; 24 | use std::os::unix::fs::symlink; 25 | use std::path::{Path, PathBuf}; 26 | use std::sync::Arc; 27 | use std::thread; 28 | 29 | use cfg_if::cfg_if; 30 | use crossbeam_channel as cbc; 31 | use libfs::copy_node; 32 | use log::{error, info}; 33 | use blocking_threadpool::{Builder, ThreadPool}; 34 | 35 | use crate::config::Config; 36 | use crate::drivers::CopyDriver; 37 | use crate::errors::{Result, XcpError}; 38 | use crate::feedback::{StatusUpdate, StatusUpdater}; 39 | use crate::operations::{CopyHandle, Operation, tree_walker}; 40 | use libfs::{copy_file_offset, map_extents, merge_extents, probably_sparse}; 41 | 42 | // ********************************************************************** // 43 | 44 | const fn supported_platform() -> bool { 45 | cfg_if! { 46 | if #[cfg( 47 | any(target_os = "linux", 48 | target_os = "android", 49 | target_os = "freebsd", 50 | target_os = "netbsd", 51 | target_os = "dragonfly", 52 | target_os = "macos", 53 | ))] 54 | { 55 | true 56 | } else { 57 | false 58 | } 59 | } 60 | } 61 | 62 | 63 | pub struct Driver { 64 | config: Arc, 65 | } 66 | 67 | impl Driver { 68 | pub fn new(config: Arc) -> Result { 69 | if !supported_platform() { 70 | let msg = "The parblock driver is not currently supported on this OS."; 71 | error!("{}", msg); 72 | return Err(XcpError::UnsupportedOS(msg).into()); 73 | } 74 | 75 | Ok(Self { 76 | config, 77 | }) 78 | } 79 | } 80 | 81 | impl CopyDriver for Driver { 82 | fn copy(&self, sources: Vec, dest: &Path, stats: Arc) -> Result<()> { 83 | let (file_tx, file_rx) = cbc::unbounded::(); 84 | 85 | // Start (single) dispatch worker 86 | let dispatcher = { 87 | let q_config = self.config.clone(); 88 | let st = stats.clone(); 89 | thread::spawn(move || dispatch_worker(file_rx, &st, q_config)) 90 | }; 91 | 92 | // Thread which walks the file tree and sends jobs to the 93 | // workers. The worker tx channel is moved to the walker so it is 94 | // closed, which will cause the workers to shutdown on completion. 95 | let walk_worker = { 96 | let sc = stats.clone(); 97 | let d = dest.to_path_buf(); 98 | let c = self.config.clone(); 99 | thread::spawn(move || tree_walker(sources, &d, &c, file_tx, sc)) 100 | }; 101 | 102 | walk_worker.join() 103 | .map_err(|_| XcpError::CopyError("Error walking copy tree".to_string()))??; 104 | dispatcher.join() 105 | .map_err(|_| XcpError::CopyError("Error dispatching copy operation".to_string()))??; 106 | 107 | Ok(()) 108 | } 109 | } 110 | 111 | // ********************************************************************** // 112 | 113 | fn queue_file_range( 114 | handle: &Arc, 115 | range: Range, 116 | pool: &ThreadPool, 117 | status_channel: &Arc, 118 | ) -> Result { 119 | let len = range.end - range.start; 120 | let bsize = handle.config.block_size; 121 | let blocks = (len / bsize) + (if len % bsize > 0 { 1 } else { 0 }); 122 | 123 | for blkn in 0..blocks { 124 | let harc = handle.clone(); 125 | let stat_tx = status_channel.clone(); 126 | let bytes = cmp::min(len - (blkn * bsize), bsize); 127 | let off = range.start + (blkn * bsize); 128 | 129 | pool.execute(move || { 130 | let copy_result = copy_file_offset(&harc.infd, &harc.outfd, bytes, off as i64); 131 | let stat_result = match copy_result { 132 | Ok(bytes) => { 133 | stat_tx.send(StatusUpdate::Copied(bytes as u64)) 134 | } 135 | Err(e) => { 136 | error!("Error copying: aborting."); 137 | stat_tx.send(StatusUpdate::Error(XcpError::CopyError(e.to_string()))) 138 | } 139 | }; 140 | if let Err(e) = stat_result { 141 | let msg = format!("Failed to send status update message. This should not happen; aborting. Error: {}", e); 142 | error!("{}", msg); 143 | panic!("{}", msg); 144 | } 145 | }); 146 | } 147 | Ok(len) 148 | } 149 | 150 | fn queue_file_blocks( 151 | source: &Path, 152 | dest: &Path, 153 | pool: &ThreadPool, 154 | status_channel: &Arc, 155 | config: &Arc, 156 | ) -> Result { 157 | let handle = CopyHandle::new(source, dest, config)?; 158 | let len = handle.metadata.len(); 159 | 160 | if handle.try_reflink()? { 161 | info!("Reflinked, skipping rest of copy"); 162 | return Ok(len); 163 | } 164 | 165 | // Put the open files in an Arc, which we drop once work has been 166 | // queued. This will keep the files open until all work has been 167 | // consumed, then close them. (This may be overkill; opening the 168 | // files in the workers would also be valid.) 169 | let harc = Arc::new(handle); 170 | 171 | let queue_whole_file = || { 172 | queue_file_range(&harc, 0..len, pool, status_channel) 173 | }; 174 | 175 | if probably_sparse(&harc.infd)? { 176 | if let Some(extents) = map_extents(&harc.infd)? { 177 | let sparse_map = merge_extents(extents)?; 178 | let mut queued = 0; 179 | for ext in sparse_map { 180 | queued += queue_file_range(&harc, ext.into(), pool, status_channel)?; 181 | } 182 | Ok(queued) 183 | } else { 184 | queue_whole_file() 185 | } 186 | } else { 187 | queue_whole_file() 188 | } 189 | } 190 | 191 | // Dispatch worker; receives queued files and hands them to 192 | // queue_file_blocks() which splits them onto the copy-pool. 193 | fn dispatch_worker(file_q: cbc::Receiver, stats: &Arc, config: Arc) -> Result<()> { 194 | let nworkers = config.num_workers(); 195 | let copy_pool = Builder::new() 196 | .num_threads(nworkers) 197 | // Use bounded queue for backpressure; this limits open 198 | // files in-flight so we don't run out of file handles. 199 | // FIXME: Number is arbitrary ATM, we should be able to 200 | // calculate it from ulimits. 201 | .queue_len(128) 202 | .build(); 203 | for op in file_q { 204 | match op { 205 | Operation::Copy(from, to) => { 206 | info!("Dispatch[{:?}]: Copy {:?} -> {:?}", thread::current().id(), from, to); 207 | let r = queue_file_blocks(&from, &to, ©_pool, stats, &config); 208 | if let Err(e) = r { 209 | stats.send(StatusUpdate::Error(XcpError::CopyError(e.to_string())))?; 210 | error!("Dispatcher: Error copying {:?} -> {:?}.", from, to); 211 | return Err(e) 212 | } 213 | } 214 | 215 | // Inline the following operations as the should be near-instant. 216 | Operation::Link(from, to) => { 217 | info!("Dispatch[{:?}]: Symlink {:?} -> {:?}", thread::current().id(), from, to); 218 | let r = symlink(&from, &to); 219 | if let Err(e) = r { 220 | stats.send(StatusUpdate::Error(XcpError::CopyError(e.to_string())))?; 221 | error!("Error symlinking: {:?} -> {:?}; aborting.", from, to); 222 | return Err(e.into()) 223 | } 224 | } 225 | 226 | Operation::Special(from, to) => { 227 | info!("Dispatch[{:?}]: Special file {:?} -> {:?}", thread::current().id(), from, to); 228 | if to.exists() { 229 | if config.no_clobber { 230 | return Err(XcpError::DestinationExists("Destination file exists and --no-clobber is set.", to).into()); 231 | } 232 | remove_file(&to)?; 233 | } 234 | copy_node(&from, &to)?; 235 | } 236 | } 237 | } 238 | info!("Queuing complete"); 239 | 240 | copy_pool.join(); 241 | info!("Pool complete"); 242 | 243 | Ok(()) 244 | } 245 | -------------------------------------------------------------------------------- /libxcp/src/drivers/parfile.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | //! Parallelise copying at the file level. This can improve speed on 18 | //! modern NVME devices, but can bottleneck on larger files. 19 | 20 | use crossbeam_channel as cbc; 21 | use log::{debug, error, info}; 22 | use libfs::copy_node; 23 | use std::fs::remove_file; 24 | use std::os::unix::fs::symlink; 25 | use std::path::{Path, PathBuf}; 26 | use std::sync::Arc; 27 | use std::thread; 28 | 29 | use crate::config::Config; 30 | use crate::drivers::CopyDriver; 31 | use crate::errors::{Result, XcpError}; 32 | use crate::feedback::{StatusUpdate, StatusUpdater}; 33 | use crate::operations::{CopyHandle, Operation, tree_walker}; 34 | 35 | // ********************************************************************** // 36 | 37 | pub struct Driver { 38 | config: Arc, 39 | } 40 | 41 | impl Driver { 42 | pub fn new(config: Arc) -> Result { 43 | Ok(Self { 44 | config, 45 | }) 46 | } 47 | } 48 | 49 | impl CopyDriver for Driver { 50 | fn copy(&self, sources: Vec, dest: &Path, stats: Arc) -> Result<()> { 51 | let (work_tx, work_rx) = cbc::unbounded(); 52 | 53 | // Thread which walks the file tree and sends jobs to the 54 | // workers. The worker tx channel is moved to the walker so it is 55 | // closed, which will cause the workers to shutdown on completion. 56 | let walk_worker = { 57 | let sc = stats.clone(); 58 | let d = dest.to_path_buf(); 59 | let o = self.config.clone(); 60 | thread::spawn(move || tree_walker(sources, &d, &o, work_tx, sc)) 61 | }; 62 | 63 | // Worker threads. Will consume work and then shutdown once the 64 | // queue is closed by the walker. 65 | let nworkers = self.config.num_workers(); 66 | let mut joins = Vec::with_capacity(nworkers); 67 | for _ in 0..nworkers { 68 | let copy_worker = { 69 | let wrx = work_rx.clone(); 70 | let sc = stats.clone(); 71 | let conf = self.config.clone(); 72 | thread::spawn(move || copy_worker(wrx, &conf, sc)) 73 | }; 74 | joins.push(copy_worker); 75 | } 76 | 77 | walk_worker.join() 78 | .map_err(|_| XcpError::CopyError("Error walking copy tree".to_string()))??; 79 | for handle in joins { 80 | handle.join() 81 | .map_err(|_| XcpError::CopyError("Error during copy operation".to_string()))??; 82 | } 83 | 84 | Ok(()) 85 | } 86 | 87 | } 88 | 89 | // ********************************************************************** // 90 | 91 | fn copy_worker(work: cbc::Receiver, config: &Arc, updates: Arc) -> Result<()> { 92 | debug!("Starting copy worker {:?}", thread::current().id()); 93 | for op in work { 94 | debug!("Received operation {:?}", op); 95 | 96 | match op { 97 | Operation::Copy(from, to) => { 98 | info!("Worker[{:?}]: Copy {:?} -> {:?}", thread::current().id(), from, to); 99 | // copy_file() sends back its own updates, but we should 100 | // send back any errors as they may have occurred 101 | // before the copy started.. 102 | let r = CopyHandle::new(&from, &to, config) 103 | .and_then(|hdl| hdl.copy_file(&updates)); 104 | if let Err(e) = r { 105 | updates.send(StatusUpdate::Error(XcpError::CopyError(e.to_string())))?; 106 | error!("Error copying: {:?} -> {:?}; aborting.", from, to); 107 | return Err(e) 108 | } 109 | } 110 | 111 | Operation::Link(from, to) => { 112 | info!("Worker[{:?}]: Symlink {:?} -> {:?}", thread::current().id(), from, to); 113 | let _r = symlink(&from, &to); 114 | } 115 | 116 | Operation::Special(from, to) => { 117 | info!("Worker[{:?}]: Special file {:?} -> {:?}", thread::current().id(), from, to); 118 | if to.exists() { 119 | if config.no_clobber { 120 | return Err(XcpError::DestinationExists("Destination file exists and --no-clobber is set.", to).into()); 121 | } 122 | remove_file(&to)?; 123 | } 124 | copy_node(&from, &to)?; 125 | } 126 | 127 | } 128 | } 129 | debug!("Copy worker {:?} shutting down", thread::current().id()); 130 | Ok(()) 131 | } 132 | -------------------------------------------------------------------------------- /libxcp/src/errors.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | //! Custom error types. 18 | 19 | use std::path::PathBuf; 20 | 21 | pub use anyhow::Result; 22 | 23 | #[derive(Debug, thiserror::Error)] 24 | pub enum XcpError { 25 | #[error("Error during copy: {0}")] 26 | CopyError(String), 27 | 28 | #[error("Destination Exists: {0}, {1}")] 29 | DestinationExists(&'static str, PathBuf), 30 | 31 | #[error("Early shutdown: {0}")] 32 | EarlyShutdown(&'static str), 33 | 34 | #[error("Invalid arguments: {0}")] 35 | InvalidArguments(String), 36 | 37 | #[error("Invalid destination: {0}")] 38 | InvalidDestination(&'static str), 39 | 40 | #[error("Invalid source: {0}")] 41 | InvalidSource(&'static str), 42 | 43 | #[error("Failed to reflink file and 'always' was specified: {0}")] 44 | ReflinkFailed(String), 45 | 46 | #[error("Unknown driver: {0}")] 47 | UnknownDriver(String), 48 | 49 | #[error("Unknown file-type: {0}")] 50 | UnknownFileType(PathBuf), 51 | 52 | #[error("Unsupported OS")] 53 | UnsupportedOS(&'static str), 54 | } 55 | -------------------------------------------------------------------------------- /libxcp/src/feedback.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2024, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | //! Support for runtime feedback of copy progress. 18 | //! 19 | 20 | //! Users of `libxcp` can implement the [StatusUpdater] trait and pass 21 | //! an instance to the driver, usually using `load_driver()`. Two 22 | //! implementations are provided: 23 | //! 24 | //! * [NoopUpdater] 25 | //! * [ChannelUpdater] 26 | 27 | use std::sync::Arc; 28 | use std::sync::atomic::{AtomicU64, Ordering}; 29 | use crossbeam_channel as cbc; 30 | 31 | use crate::config::Config; 32 | use crate::errors::{Result, XcpError}; 33 | 34 | /// A struct representing an updated status. 35 | #[derive(Debug)] 36 | pub enum StatusUpdate { 37 | /// An update representing a successful copy of bytes between 38 | /// files. 39 | Copied(u64), 40 | /// An update representing that this number of bytes will need to be copied. 41 | Size(u64), 42 | /// An error during a copy operation. 43 | Error(XcpError) 44 | } 45 | 46 | pub trait StatusUpdater: Sync + Send { 47 | fn send(&self, update: StatusUpdate) -> Result<()>; 48 | } 49 | 50 | /// An implementation of [StatusUpdater] which will return 51 | /// [StatusUpdate] objects via a channel. On copy completion the 52 | /// channel will be closed, allowing the caller to iterator over 53 | /// returned updates. See the top-level module for an example of 54 | /// usage. 55 | pub struct ChannelUpdater { 56 | chan_tx: cbc::Sender, 57 | chan_rx: cbc::Receiver, 58 | config: Arc, 59 | sent: AtomicU64, 60 | } 61 | 62 | impl ChannelUpdater { 63 | /// Create a new ChannelUpdater, including the channels. 64 | pub fn new(config: &Arc) -> ChannelUpdater { 65 | let (chan_tx, chan_rx) = cbc::unbounded(); 66 | ChannelUpdater { 67 | chan_tx, 68 | chan_rx, 69 | config: config.clone(), 70 | sent: AtomicU64::new(0), 71 | } 72 | } 73 | 74 | /// Retrieve a clone of the receive end of the update 75 | /// channel. Note: As ChannelUpdater is consumed by the driver 76 | /// call you should call this before then; e.g: 77 | /// 78 | /// # use std::sync::Arc; 79 | /// use libxcp::config::Config; 80 | /// use libxcp::feedback::{ChannelUpdater, StatusUpdater}; 81 | /// 82 | /// let config = Arc::new(Config::default()); 83 | /// let updater = ChannelUpdater::new(&config); 84 | /// let stat_rx = updater.rx_channel(); 85 | /// let stats: Arc = Arc::new(updater); 86 | pub fn rx_channel(&self) -> cbc::Receiver { 87 | self.chan_rx.clone() 88 | } 89 | } 90 | 91 | impl StatusUpdater for ChannelUpdater { 92 | // Wrapper around channel-send that groups updates together 93 | fn send(&self, update: StatusUpdate) -> Result<()> { 94 | if let StatusUpdate::Copied(bytes) = update { 95 | // Avoid saturating the queue with small writes 96 | let bsize = self.config.block_size; 97 | let prev_written = self.sent.fetch_add(bytes, Ordering::Relaxed); 98 | if ((prev_written + bytes) / bsize) > (prev_written / bsize) { 99 | self.chan_tx.send(update)?; 100 | } 101 | } else { 102 | self.chan_tx.send(update)?; 103 | } 104 | Ok(()) 105 | } 106 | } 107 | 108 | /// A null updater for when no feedback is required. 109 | pub struct NoopUpdater; 110 | 111 | impl StatusUpdater for NoopUpdater { 112 | fn send(&self, _update: StatusUpdate) -> Result<()> { 113 | Ok(()) 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /libxcp/src/lib.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2024, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | //! `libxcp` is a high-level file-copy engine. It has a support for 18 | //! multi-threading, fine-grained progress feedback, pluggable 19 | //! drivers, and `.gitignore` filters. `libxcp` is the core 20 | //! functionality of the [xcp] command-line utility. 21 | //! 22 | //! # Usage example 23 | //! 24 | //! # use libxcp::errors::Result; 25 | //! # use std::path::PathBuf; 26 | //! # use std::sync::Arc; 27 | //! # use std::thread; 28 | //! # use tempfile::TempDir; 29 | //! use libxcp::config::Config; 30 | //! use libxcp::errors::XcpError; 31 | //! use libxcp::feedback::{ChannelUpdater, StatusUpdater, StatusUpdate}; 32 | //! use libxcp::drivers::{Drivers, load_driver}; 33 | //! # fn main() -> Result<()> { 34 | //! 35 | //! let sources = vec![PathBuf::from("src")]; 36 | //! let dest = TempDir::new()?; 37 | //! 38 | //! let config = Arc::new(Config::default()); 39 | //! let updater = ChannelUpdater::new(&config); 40 | //! // The ChannelUpdater is consumed by the driver (so it is properly closed 41 | //! // on completion). Retrieve our end of the connection before then. 42 | //! let stat_rx = updater.rx_channel(); 43 | //! let stats: Arc = Arc::new(updater); 44 | //! 45 | //! let driver = load_driver(Drivers::ParFile, &config)?; 46 | //! 47 | //! // As we want realtime updates via the ChannelUpdater the 48 | //! // copy operation should run in the background. 49 | //! let handle = thread::spawn(move || { 50 | //! driver.copy(sources, dest.path(), stats) 51 | //! }); 52 | //! 53 | //! // Gather the results as we go; our end of the channel has been 54 | //! // moved to the driver call and will end when drained. 55 | //! for stat in stat_rx { 56 | //! match stat { 57 | //! StatusUpdate::Copied(v) => { 58 | //! println!("Copied {} bytes", v); 59 | //! }, 60 | //! StatusUpdate::Size(v) => { 61 | //! println!("Size update: {}", v); 62 | //! }, 63 | //! StatusUpdate::Error(e) => { 64 | //! panic!("Error during copy: {}", e); 65 | //! } 66 | //! } 67 | //! } 68 | //! 69 | //! handle.join() 70 | //! .map_err(|_| XcpError::CopyError("Error during copy operation".to_string()))??; 71 | //! 72 | //! println!("Copy complete"); 73 | //! 74 | //! # Ok(()) 75 | //! # } 76 | //! 77 | //! [xcp]: https://crates.io/crates/xcp/ 78 | 79 | pub mod config; 80 | pub mod drivers; 81 | pub mod errors; 82 | pub mod feedback; 83 | 84 | // Internal 85 | mod backup; 86 | mod operations; 87 | mod paths; 88 | 89 | #[cfg(test)] 90 | #[allow(unused)] 91 | mod tests { 92 | use std::path::PathBuf; 93 | use std::sync::Arc; 94 | use std::thread; 95 | 96 | use tempfile::TempDir; 97 | 98 | use crate::errors::{Result, XcpError}; 99 | use crate::config::Config; 100 | use crate::feedback::{ChannelUpdater, StatusUpdater, StatusUpdate}; 101 | use crate::drivers::{Drivers, load_driver}; 102 | 103 | #[test] 104 | fn simple_usage_test() -> Result<()> { 105 | let sources = vec![PathBuf::from("src")]; 106 | let dest = TempDir::new()?; 107 | 108 | let config = Arc::new(Config::default()); 109 | let updater = ChannelUpdater::new(&config); 110 | let stat_rx = updater.rx_channel(); 111 | let stats: Arc = Arc::new(updater); 112 | 113 | let driver = load_driver(Drivers::ParFile, &config)?; 114 | 115 | let handle = thread::spawn(move || { 116 | driver.copy(sources, dest.path(), stats) 117 | }); 118 | 119 | // Gather the results as we go; our end of the channel has been 120 | // moved to the driver call and will end when drained. 121 | for stat in stat_rx { 122 | match stat { 123 | StatusUpdate::Copied(v) => { 124 | println!("Copied {} bytes", v); 125 | }, 126 | StatusUpdate::Size(v) => { 127 | println!("Size update: {}", v); 128 | }, 129 | StatusUpdate::Error(e) => { 130 | println!("Error during copy: {}", e); 131 | return Err(e.into()); 132 | } 133 | } 134 | } 135 | 136 | handle.join() 137 | .map_err(|_| XcpError::CopyError("Error during copy operation".to_string()))??; 138 | 139 | println!("Copy complete"); 140 | 141 | Ok(()) 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /libxcp/src/operations.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | use std::{cmp, thread}; 18 | use std::fs::{self, canonicalize, create_dir_all, read_link, File, Metadata}; 19 | use std::path::{Path, PathBuf}; 20 | use std::sync::Arc; 21 | 22 | use crossbeam_channel as cbc; 23 | use libfs::{ 24 | allocate_file, copy_file_bytes, copy_owner, copy_permissions, copy_timestamps, next_sparse_segments, probably_sparse, reflink, sync, FileType 25 | }; 26 | use log::{debug, error, info, warn}; 27 | use walkdir::WalkDir; 28 | 29 | use crate::backup::{get_backup_path, needs_backup}; 30 | use crate::config::{Config, Reflink}; 31 | use crate::errors::{Result, XcpError}; 32 | use crate::feedback::{StatusUpdate, StatusUpdater}; 33 | use crate::paths::{parse_ignore, ignore_filter}; 34 | 35 | #[derive(Debug)] 36 | pub struct CopyHandle { 37 | pub infd: File, 38 | pub outfd: File, 39 | pub metadata: Metadata, 40 | pub config: Arc, 41 | } 42 | 43 | impl CopyHandle { 44 | pub fn new(from: &Path, to: &Path, config: &Arc) -> Result { 45 | let infd = File::open(from)?; 46 | let metadata = infd.metadata()?; 47 | 48 | if needs_backup(to, config)? { 49 | let backup = get_backup_path(to)?; 50 | info!("Backup: Rename {:?} to {:?}", to, backup); 51 | fs::rename(to, backup)?; 52 | } 53 | 54 | let outfd = File::create(to)?; 55 | allocate_file(&outfd, metadata.len())?; 56 | 57 | let handle = CopyHandle { 58 | infd, 59 | outfd, 60 | metadata, 61 | config: config.clone(), 62 | }; 63 | 64 | Ok(handle) 65 | } 66 | 67 | /// Copy len bytes from wherever the descriptor cursors are set. 68 | fn copy_bytes(&self, len: u64, updates: &Arc) -> Result { 69 | let mut written = 0; 70 | while written < len { 71 | let bytes_to_copy = cmp::min(len - written, self.config.block_size); 72 | let bytes = copy_file_bytes(&self.infd, &self.outfd, bytes_to_copy)? as u64; 73 | written += bytes; 74 | updates.send(StatusUpdate::Copied(bytes))?; 75 | } 76 | 77 | Ok(written) 78 | } 79 | 80 | /// Wrapper around copy_bytes that looks for sparse blocks and skips them. 81 | fn copy_sparse(&self, updates: &Arc) -> Result { 82 | let len = self.metadata.len(); 83 | let mut pos = 0; 84 | 85 | while pos < len { 86 | let (next_data, next_hole) = next_sparse_segments(&self.infd, &self.outfd, pos)?; 87 | 88 | let _written = self.copy_bytes(next_hole - next_data, updates)?; 89 | pos = next_hole; 90 | } 91 | 92 | Ok(len) 93 | } 94 | 95 | pub fn try_reflink(&self) -> Result { 96 | match self.config.reflink { 97 | Reflink::Always | Reflink::Auto => { 98 | debug!("Attempting reflink from {:?}->{:?}", self.infd, self.outfd); 99 | let worked = reflink(&self.infd, &self.outfd)?; 100 | if worked { 101 | debug!("Reflink {:?} succeeded", self.outfd); 102 | Ok(true) 103 | } else if self.config.reflink == Reflink::Always { 104 | Err(XcpError::ReflinkFailed(format!("{:?}->{:?}", self.infd, self.outfd)).into()) 105 | } else { 106 | debug!("Failed to reflink, falling back to copy"); 107 | Ok(false) 108 | } 109 | } 110 | 111 | Reflink::Never => { 112 | Ok(false) 113 | } 114 | } 115 | } 116 | 117 | pub fn copy_file(&self, updates: &Arc) -> Result { 118 | if self.try_reflink()? { 119 | return Ok(self.metadata.len()); 120 | } 121 | let total = if probably_sparse(&self.infd)? { 122 | self.copy_sparse(updates)? 123 | } else { 124 | self.copy_bytes(self.metadata.len(), updates)? 125 | }; 126 | 127 | Ok(total) 128 | } 129 | 130 | fn finalise_copy(&self) -> Result<()> { 131 | if !self.config.no_perms { 132 | copy_permissions(&self.infd, &self.outfd)?; 133 | } 134 | if !self.config.no_timestamps { 135 | copy_timestamps(&self.infd, &self.outfd)?; 136 | } 137 | if self.config.ownership && copy_owner(&self.infd, &self.outfd).is_err() { 138 | warn!("Failed to copy file ownership: {:?}", self.infd); 139 | } 140 | if self.config.fsync { 141 | debug!("Syncing file {:?}", self.outfd); 142 | sync(&self.outfd)?; 143 | } 144 | Ok(()) 145 | } 146 | } 147 | 148 | impl Drop for CopyHandle { 149 | fn drop(&mut self) { 150 | // FIXME: Should we check for panicking() here? 151 | if let Err(e) = self.finalise_copy() { 152 | error!("Error during finalising copy operation {:?} -> {:?}: {}", self.infd, self.outfd, e); 153 | } 154 | } 155 | } 156 | 157 | #[derive(Debug)] 158 | pub enum Operation { 159 | Copy(PathBuf, PathBuf), 160 | Link(PathBuf, PathBuf), 161 | Special(PathBuf, PathBuf), 162 | } 163 | 164 | pub fn tree_walker( 165 | sources: Vec, 166 | dest: &Path, 167 | config: &Config, 168 | work_tx: cbc::Sender, 169 | stats: Arc, 170 | ) -> Result<()> { 171 | debug!("Starting walk worker {:?}", thread::current().id()); 172 | 173 | for source in sources { 174 | let sourcedir = source 175 | .components() 176 | .next_back() 177 | .ok_or(XcpError::InvalidSource("Failed to find source directory name."))?; 178 | 179 | let target_base = if dest.exists() && dest.is_dir() && !config.no_target_directory { 180 | dest.join(sourcedir) 181 | } else { 182 | dest.to_path_buf() 183 | }; 184 | debug!("Target base is {:?}", target_base); 185 | 186 | let gitignore = parse_ignore(&source, config)?; 187 | 188 | for entry in WalkDir::new(&source) 189 | .into_iter() 190 | .filter_entry(|e| ignore_filter(e, &gitignore)) 191 | { 192 | debug!("Got tree entry {:?}", entry); 193 | let epath = entry?.into_path(); 194 | let from = if config.dereference { 195 | let cpath = canonicalize(&epath)?; 196 | debug!("Dereferencing {:?} into {:?}", epath, cpath); 197 | cpath 198 | } else { 199 | epath.clone() 200 | }; 201 | let meta = from.symlink_metadata()?; 202 | let path = epath.strip_prefix(&source)?; 203 | let target = if !empty_path(path) { 204 | target_base.join(path) 205 | } else { 206 | target_base.clone() 207 | }; 208 | 209 | if config.no_clobber && target.exists() { 210 | let msg = "Destination file exists and --no-clobber is set."; 211 | stats.send(StatusUpdate::Error( 212 | XcpError::DestinationExists(msg, target)))?; 213 | return Err(XcpError::EarlyShutdown(msg).into()); 214 | } 215 | 216 | let ft = FileType::from(meta.file_type()); 217 | match ft { 218 | FileType::File => { 219 | debug!("Send copy operation {:?} to {:?}", from, target); 220 | stats.send(StatusUpdate::Size(meta.len()))?; 221 | work_tx.send(Operation::Copy(from, target))?; 222 | } 223 | 224 | FileType::Symlink => { 225 | let lfile = read_link(from)?; 226 | debug!("Send symlink operation {:?} to {:?}", lfile, target); 227 | work_tx.send(Operation::Link(lfile, target))?; 228 | } 229 | 230 | FileType::Dir => { 231 | // Create dir tree immediately as we can't 232 | // guarantee a worker will action the creation 233 | // before a subsequent copy operation requires it. 234 | debug!("Creating target directory {:?}", target); 235 | if let Err(err) = create_dir_all(&target) { 236 | let msg = format!("Error creating target directory: {}", err); 237 | error!("{msg}"); 238 | return Err(XcpError::CopyError(msg).into()) 239 | } 240 | } 241 | 242 | FileType::Socket | FileType::Char | FileType::Fifo => { 243 | debug!("Special file found: {:?} to {:?}", from, target); 244 | work_tx.send(Operation::Special(from, target))?; 245 | } 246 | 247 | FileType::Block | FileType::Other => { 248 | error!("Unsupported filetype found: {:?} -> {:?}", target, ft); 249 | return Err(XcpError::UnknownFileType(target).into()); 250 | } 251 | }; 252 | } 253 | } 254 | debug!("Walk-worker finished: {:?}", thread::current().id()); 255 | 256 | Ok(()) 257 | } 258 | 259 | fn empty_path(path: &Path) -> bool { 260 | *path == PathBuf::new() 261 | } 262 | -------------------------------------------------------------------------------- /libxcp/src/paths.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2024, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | use std::path::Path; 18 | use ignore::gitignore::{Gitignore, GitignoreBuilder}; 19 | use log::info; 20 | use walkdir::DirEntry; 21 | 22 | use crate::config::Config; 23 | use crate::errors::Result; 24 | 25 | /// Parse a git ignore file. 26 | pub fn parse_ignore(source: &Path, config: &Config) -> Result> { 27 | let gitignore = if config.gitignore { 28 | let gifile = source.join(".gitignore"); 29 | info!("Using .gitignore file {:?}", gifile); 30 | let mut builder = GitignoreBuilder::new(source); 31 | builder.add(&gifile); 32 | let ignore = builder.build()?; 33 | Some(ignore) 34 | } else { 35 | None 36 | }; 37 | Ok(gitignore) 38 | } 39 | 40 | /// Filter to return whether a given file should be ignored by a 41 | /// filter file. 42 | pub fn ignore_filter(entry: &DirEntry, ignore: &Option) -> bool { 43 | match ignore { 44 | None => true, 45 | Some(gi) => { 46 | let path = entry.path(); 47 | let m = gi.matched(path, path.is_dir()); 48 | !m.is_ignore() 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | mod options; 18 | mod progress; 19 | 20 | use std::path::PathBuf; 21 | use std::{result, thread}; 22 | use std::sync::Arc; 23 | 24 | use glob::{glob, Paths}; 25 | use libxcp::config::{Config, Reflink}; 26 | use libxcp::drivers::load_driver; 27 | use libxcp::errors::{Result, XcpError}; 28 | use libxcp::feedback::{ChannelUpdater, StatusUpdate, StatusUpdater}; 29 | use log::{error, info, warn}; 30 | 31 | use crate::options::Opts; 32 | 33 | fn init_logging(opts: &Opts) -> Result<()> { 34 | use simplelog::{ColorChoice, Config, SimpleLogger, TermLogger, TerminalMode}; 35 | 36 | TermLogger::init( 37 | opts.log_level(), 38 | Config::default(), 39 | TerminalMode::Mixed, 40 | ColorChoice::Auto, 41 | ).or_else( 42 | |_| SimpleLogger::init(opts.log_level(), Config::default()) 43 | )?; 44 | 45 | Ok(()) 46 | } 47 | 48 | // Expand a list of file-paths or glob-patterns into a list of concrete paths. 49 | // FIXME: This currently eats non-existent files that are not 50 | // globs. Should we convert empty glob results into errors? 51 | fn expand_globs(patterns: &[String]) -> Result> { 52 | let paths = patterns.iter() 53 | .map(|s| glob(s.as_str())) 54 | .collect::, _>>()? 55 | .iter_mut() 56 | // Force resolve each glob Paths iterator into a vector of the results... 57 | .map::, _>, _>(Iterator::collect) 58 | // And lift all the results up to the top. 59 | .collect::>, _>>()? 60 | .iter() 61 | .flat_map(ToOwned::to_owned) 62 | .collect::>(); 63 | 64 | Ok(paths) 65 | } 66 | 67 | fn expand_sources(source_list: &[String], opts: &Opts) -> Result> { 68 | if opts.glob { 69 | expand_globs(source_list) 70 | } else { 71 | let pb = source_list.iter() 72 | .map(PathBuf::from) 73 | .collect::>(); 74 | Ok(pb) 75 | } 76 | } 77 | 78 | fn opts_check(opts: &Opts) -> Result<()> { 79 | #[cfg(any(target_os = "linux", target_os = "android"))] 80 | if opts.reflink == Reflink::Never { 81 | warn!("--reflink=never is selected, however the Linux kernel may override this."); 82 | } 83 | 84 | if opts.no_clobber && opts.force { 85 | return Err(XcpError::InvalidArguments("--force and --noclobber cannot be set at the same time.".to_string()).into()); 86 | } 87 | Ok(()) 88 | } 89 | 90 | fn main() -> Result<()> { 91 | let opts = Opts::from_args()?; 92 | init_logging(&opts)?; 93 | opts_check(&opts)?; 94 | 95 | let (dest, source_patterns) = match opts.target_directory { 96 | Some(ref d) => { (d, opts.paths.as_slice()) } 97 | None => { 98 | opts.paths.split_last().ok_or(XcpError::InvalidArguments("Insufficient arguments".to_string()))? 99 | } 100 | }; 101 | let dest = PathBuf::from(dest); 102 | 103 | let sources = expand_sources(source_patterns, &opts)?; 104 | if sources.is_empty() { 105 | return Err(XcpError::InvalidSource("No source files found.").into()); 106 | } else if !dest.is_dir() { 107 | if sources.len() == 1 && sources[0].is_dir() && dest.exists() { 108 | return Err(XcpError::InvalidDestination("Cannot copy a directory to a file.").into()); 109 | } else if sources.len() > 1 { 110 | return Err(XcpError::InvalidDestination("Multiple sources and destination is not a directory.").into()); 111 | } 112 | } 113 | 114 | // Sanity-check all sources up-front 115 | for source in &sources { 116 | info!("Copying source {:?} to {:?}", source, dest); 117 | if !source.exists() { 118 | return Err(XcpError::InvalidSource("Source does not exist.").into()); 119 | } 120 | 121 | if source.is_dir() && !opts.recursive { 122 | return Err(XcpError::InvalidSource("Source is directory and --recursive not specified.").into()); 123 | } 124 | if source == &dest { 125 | return Err(XcpError::InvalidSource("Cannot copy a directory into itself").into()); 126 | } 127 | 128 | let sourcedir = source 129 | .components() 130 | .next_back() 131 | .ok_or(XcpError::InvalidSource("Failed to find source directory name."))?; 132 | 133 | let target_base = if dest.exists() && dest.is_dir() && !opts.no_target_directory { 134 | dest.join(sourcedir) 135 | } else { 136 | dest.to_path_buf() 137 | }; 138 | 139 | if source == &target_base { 140 | return Err(XcpError::InvalidSource("Source is same as destination").into()); 141 | } 142 | } 143 | 144 | 145 | // ========== Start copy ============ 146 | 147 | let config = Arc::new(Config::from(&opts)); 148 | let driver = load_driver(opts.driver, &config)?; 149 | 150 | let updater = ChannelUpdater::new(&config); 151 | let stat_rx = updater.rx_channel(); 152 | let stats: Arc = Arc::new(updater); 153 | 154 | let handle = thread::spawn(move || -> Result<()> { 155 | driver.copy(sources, &dest, stats) 156 | }); 157 | 158 | 159 | // ========== Collect output and display ============ 160 | 161 | let pb = progress::create_bar(&opts, 0)?; 162 | 163 | // Gather the results as we go; our end of the channel has been 164 | // moved to the driver call and will end when drained. 165 | for stat in stat_rx { 166 | match stat { 167 | StatusUpdate::Copied(v) => pb.inc(v), 168 | StatusUpdate::Size(v) => pb.inc_size(v), 169 | StatusUpdate::Error(e) => { 170 | // FIXME: Optional continue? 171 | error!("Received error: {}", e); 172 | return Err(e.into()); 173 | } 174 | } 175 | } 176 | 177 | handle.join() 178 | .map_err(|_| XcpError::CopyError("Error during copy operation".to_string()))??; 179 | 180 | info!("Copy complete"); 181 | pb.end(); 182 | 183 | Ok(()) 184 | } 185 | -------------------------------------------------------------------------------- /src/options.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | use clap::{ArgAction, Parser}; 18 | 19 | use libxcp::config::{Backup, Config, Reflink}; 20 | use log::LevelFilter; 21 | use unbytify::unbytify; 22 | 23 | use libxcp::drivers::Drivers; 24 | use libxcp::errors::Result; 25 | 26 | #[derive(Clone, Debug, Parser)] 27 | #[command( 28 | name = "xcp", 29 | about = "A (partial) clone of the Unix `cp` command with progress and pluggable drivers.", 30 | version, 31 | )] 32 | pub struct Opts { 33 | /// Verbosity. 34 | /// 35 | /// Can be specified multiple times to increase logging. 36 | #[arg(short, long, action = ArgAction::Count)] 37 | pub verbose: u8, 38 | 39 | /// Copy directories recursively 40 | #[arg(short, long)] 41 | pub recursive: bool, 42 | 43 | /// Dereference symlinks in source 44 | /// 45 | /// Follow symlinks, possibly recursively, when copying source 46 | /// files. 47 | #[arg(short = 'L', long)] 48 | pub dereference: bool, 49 | 50 | /// Number of parallel workers. 51 | /// 52 | /// Default is 4; if the value is negative or 0 it uses the number 53 | /// of logical CPUs. 54 | #[arg(short, long, default_value = "4")] 55 | pub workers: usize, 56 | 57 | /// Block size for operations. 58 | /// 59 | /// Accepts standard size modifiers like "M" and "GB". Actual 60 | /// usage internally depends on the driver. 61 | #[arg(long, default_value = "1MB", value_parser=unbytify)] 62 | pub block_size: u64, 63 | 64 | /// Do not overwrite an existing file 65 | #[arg(short, long)] 66 | pub no_clobber: bool, 67 | 68 | /// Force (compatability only) 69 | /// 70 | /// Overwrite files; this is the default behaviour, this flag is 71 | /// for compatibility with `cp` only. See `--no-clobber` for the 72 | /// inverse flag. Using this in conjunction with `--no-clobber` 73 | /// will cause an error. 74 | #[arg(short = 'f', long = "force")] 75 | pub force: bool, 76 | 77 | /// Use .gitignore if present. 78 | /// 79 | /// NOTE: This is fairly basic at the moment, and only honours a 80 | /// .gitignore in the directory root for directory copies; global 81 | /// or sub-directory ignores are skipped. 82 | #[arg(long)] 83 | pub gitignore: bool, 84 | 85 | /// Expand file patterns. 86 | /// 87 | /// Glob (expand) filename patterns natively (note; the shell may still do its own expansion first) 88 | #[arg(short, long)] 89 | pub glob: bool, 90 | 91 | /// Disable progress bar. 92 | #[arg(long)] 93 | pub no_progress: bool, 94 | 95 | /// Do not copy the file permissions. 96 | #[arg(long)] 97 | pub no_perms: bool, 98 | 99 | /// Do not copy the file timestamps. 100 | #[arg(long)] 101 | pub no_timestamps: bool, 102 | 103 | /// Copy ownership. 104 | /// 105 | /// Whether to copy ownship (user/group). This option requires 106 | /// root permissions or appropriate capabilities; if the attempt 107 | /// to copy ownership fails a warning is issued but the operation 108 | /// continues. 109 | #[arg(short, long)] 110 | pub ownership: bool, 111 | 112 | /// Driver to use, defaults to 'file-parallel'. 113 | /// 114 | /// Currently there are 2; the default "parfile", which 115 | /// parallelises copies across workers at the file level, and an 116 | /// experimental "parblock" driver, which parellelises at the 117 | /// block level. See also '--block-size'. 118 | #[arg(long, default_value = "parfile")] 119 | pub driver: Drivers, 120 | 121 | /// Target should not be a directory. 122 | /// 123 | /// Analogous to cp's no-target-directory. Expected behavior is that when 124 | /// copying a directory to another directory, instead of creating a sub-folder 125 | /// in target, overwrite target. 126 | #[arg(short = 'T', long)] 127 | pub no_target_directory: bool, 128 | 129 | /// Copy into a subdirectory of the target 130 | #[arg(long)] 131 | pub target_directory: Option, 132 | 133 | /// Sync each file to disk after writing. 134 | #[arg(long)] 135 | pub fsync: bool, 136 | 137 | /// Reflink options. 138 | /// 139 | /// Whether and how to use reflinks. 'auto' (the default) will 140 | /// attempt to reflink and fallback to a copy if it is not 141 | /// possible, 'always' will return an error if it cannot reflink, 142 | /// and 'never' will always perform a full data copy. 143 | /// 144 | /// Note: when using Linux accelerated copy operations (the 145 | /// default when available) the kernel may choose to reflink 146 | /// rather than perform a fully copy regardless of this setting. 147 | #[arg(long, default_value = "auto")] 148 | pub reflink: Reflink, 149 | 150 | /// Backup options 151 | /// 152 | /// Whether to create backups of overwritten files. Current 153 | /// options are 'none'/'off', or 'numbered', or 'auto'. Numbered 154 | /// backups follow the semantics of `cp` numbered backups 155 | /// (e.g. `file.txt.~123~`). 'auto' will only create a numbered 156 | /// backup if a previous backups exists. Default is 'none'. 157 | #[arg(long, default_value = "none")] 158 | pub backup: Backup, 159 | 160 | /// Path list. 161 | /// 162 | /// Source and destination files, or multiple source(s) to a directory. 163 | pub paths: Vec, 164 | } 165 | 166 | impl Opts { 167 | pub fn from_args() -> Result { 168 | Ok(Opts::parse()) 169 | } 170 | 171 | pub fn log_level(&self) -> LevelFilter { 172 | match self.verbose { 173 | 0 => LevelFilter::Warn, 174 | 1 => LevelFilter::Info, 175 | 2 => LevelFilter::Debug, 176 | _ => LevelFilter::Trace, 177 | } 178 | } 179 | } 180 | 181 | impl From<&Opts> for Config { 182 | fn from(opts: &Opts) -> Self { 183 | Config { 184 | workers: if opts.workers == 0 { 185 | num_cpus::get() 186 | } else { 187 | opts.workers 188 | }, 189 | block_size: if opts.no_progress { 190 | usize::MAX as u64 191 | } else { 192 | opts.block_size 193 | }, 194 | gitignore: opts.gitignore, 195 | no_clobber: opts.no_clobber, 196 | no_perms: opts.no_perms, 197 | no_timestamps: opts.no_timestamps, 198 | ownership: opts.ownership, 199 | dereference: opts.dereference, 200 | no_target_directory: opts.no_target_directory, 201 | fsync: opts.fsync, 202 | reflink: opts.reflink, 203 | backup: opts.backup, 204 | } 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /src/progress.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | use crate::options::Opts; 18 | 19 | use libxcp::errors::Result; 20 | use terminal_size::Width; 21 | 22 | struct NoopBar; 23 | 24 | struct VisualBar { 25 | bar: indicatif::ProgressBar, 26 | } 27 | 28 | pub trait ProgressBar { 29 | #[allow(unused)] 30 | fn set_size(&self, size: u64); 31 | fn inc_size(&self, size: u64); 32 | fn inc(&self, size: u64); 33 | fn end(&self); 34 | } 35 | 36 | 37 | impl ProgressBar for NoopBar { 38 | fn set_size(&self, _size: u64) { 39 | } 40 | fn inc_size(&self, _size: u64) { 41 | } 42 | fn inc(&self, _size: u64) { 43 | } 44 | fn end(&self) { 45 | } 46 | } 47 | 48 | impl ProgressBar for VisualBar { 49 | fn set_size(&self, size: u64) { 50 | self.bar.set_length(size); 51 | } 52 | 53 | fn inc_size(&self, size: u64) { 54 | self.bar.inc_length(size); 55 | } 56 | 57 | fn inc(&self, size: u64) { 58 | self.bar.inc(size); 59 | } 60 | 61 | fn end(&self) { 62 | self.bar.finish(); 63 | } 64 | } 65 | 66 | impl VisualBar { 67 | fn new(size: u64) -> Result { 68 | let bar = indicatif::ProgressBar::new(size).with_style( 69 | indicatif::ProgressStyle::default_bar() 70 | .template( 71 | match terminal_size::terminal_size() { 72 | Some((Width(width), _)) if width < 160 => "[{wide_bar:.cyan/blue}]\n{bytes:>11} / {total_bytes:<11} | {percent:>3}% | {bytes_per_sec:^13} | {eta_precise} remaining", 73 | _ => "[{wide_bar:.cyan/blue}] {bytes:>11} / {total_bytes:<11} | {percent:>3}% | {bytes_per_sec:^13} | {eta_precise} remaining", 74 | } 75 | )? 76 | .progress_chars("#>-"), 77 | ); 78 | Ok(Self { bar }) 79 | } 80 | } 81 | 82 | pub fn create_bar(opts: &Opts, size: u64) -> Result> { 83 | if opts.no_progress { 84 | Ok(Box::new(NoopBar {})) 85 | } else { 86 | Ok(Box::new(VisualBar::new(size)?)) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /tests/linux.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018, Steve Smith 3 | * 4 | * This program is free software: you can redistribute it and/or 5 | * modify it under the terms of the GNU General Public License version 6 | * 3 as published by the Free Software Foundation. 7 | * 8 | * This program is distributed in the hope that it will be useful, but 9 | * WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 11 | * General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | mod util; 18 | 19 | #[cfg(all(target_os = "linux", feature = "use_linux"))] 20 | mod test { 21 | use std::{process::Command, fs::{File, OpenOptions}, io::SeekFrom}; 22 | use std::io::{Seek, Write}; 23 | use libfs::{map_extents, sync}; 24 | use test_case::test_case; 25 | 26 | use crate::util::*; 27 | 28 | #[cfg_attr(feature = "parblock", test_case("parblock"; "Test with parallel block driver"))] 29 | #[test_case("parfile"; "Test with parallel file driver")] 30 | #[cfg_attr(feature = "test_no_reflink", ignore = "No FS support")] 31 | fn file_copy_reflink_always(drv: &str) { 32 | let dir = tempdir_rel().unwrap(); 33 | let source_path = dir.path().join("source.bin"); 34 | let dest_path = dir.path().join("dest.bin"); 35 | let size = 128 * 1024; 36 | 37 | { 38 | let mut infd = File::create(&source_path).unwrap(); 39 | let data = rand_data(size); 40 | infd.write_all(&data).unwrap(); 41 | } 42 | 43 | { 44 | let infd = File::open(&source_path).unwrap(); 45 | let inext = map_extents(&infd).unwrap().unwrap(); 46 | // Single file, extent not shared. 47 | assert_eq!(false, inext[0].shared); 48 | } 49 | 50 | let out = run(&[ 51 | "--driver", drv, 52 | "--reflink=always", 53 | source_path.to_str().unwrap(), 54 | dest_path.to_str().unwrap(), 55 | ]) 56 | .unwrap(); 57 | 58 | // Should always work on CoW FS 59 | assert!(out.status.success()); 60 | assert!(files_match(&source_path, &dest_path)); 61 | 62 | { 63 | let infd = File::open(&source_path).unwrap(); 64 | let outfd = File::open(&dest_path).unwrap(); 65 | // Extents should be shared. 66 | let inext = map_extents(&infd).unwrap().unwrap(); 67 | let outext = map_extents(&outfd).unwrap().unwrap(); 68 | assert_eq!(true, inext[0].shared); 69 | assert_eq!(true, outext[0].shared); 70 | } 71 | 72 | { 73 | let mut outfd = OpenOptions::new() 74 | .create(false) 75 | .write(true) 76 | .read(true) 77 | .open(&dest_path).unwrap(); 78 | outfd.seek(SeekFrom::Start(0)).unwrap(); 79 | let data = rand_data(size); 80 | outfd.write_all(&data).unwrap(); 81 | // brtfs at least seems to need this to force CoW and 82 | // de-share the extents. 83 | sync(&outfd).unwrap(); 84 | } 85 | 86 | { 87 | let infd = File::open(&source_path).unwrap(); 88 | let outfd = File::open(&dest_path).unwrap(); 89 | // First extent should now be un-shared. 90 | let inext = map_extents(&infd).unwrap().unwrap(); 91 | let outext = map_extents(&outfd).unwrap().unwrap(); 92 | assert_eq!(false, inext[0].shared); 93 | assert_eq!(false, outext[0].shared); 94 | } 95 | 96 | } 97 | 98 | #[cfg_attr(feature = "parblock", test_case("parblock"; "Test with parallel block driver"))] 99 | #[test_case("parfile"; "Test with parallel file driver")] 100 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 101 | fn test_sparse(drv: &str) { 102 | use std::fs::read; 103 | 104 | let dir = tempdir_rel().unwrap(); 105 | let from = dir.path().join("sparse.bin"); 106 | let to = dir.path().join("target.bin"); 107 | 108 | let slen = create_sparse(&from, 0, 0).unwrap(); 109 | assert_eq!(slen, from.metadata().unwrap().len()); 110 | assert!(probably_sparse(&from).unwrap()); 111 | 112 | let out = run(&[ 113 | "--driver", 114 | drv, 115 | from.to_str().unwrap(), 116 | to.to_str().unwrap(), 117 | ]).unwrap(); 118 | assert!(out.status.success()); 119 | 120 | assert!(probably_sparse(&to).unwrap()); 121 | 122 | assert_eq!(quickstat(&from).unwrap(), quickstat(&to).unwrap()); 123 | 124 | let from_data = read(&from).unwrap(); 125 | let to_data = read(&to).unwrap(); 126 | assert_eq!(from_data, to_data); 127 | } 128 | 129 | #[cfg_attr(feature = "parblock", test_case("parblock"; "Test with parallel block driver"))] 130 | #[test_case("parfile"; "Test with parallel file driver")] 131 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 132 | fn test_sparse_leading_gap(drv: &str) { 133 | use std::fs::read; 134 | 135 | let dir = tempdir_rel().unwrap(); 136 | let from = dir.path().join("sparse.bin"); 137 | let to = dir.path().join("target.bin"); 138 | 139 | let slen = create_sparse(&from, 1024, 0).unwrap(); 140 | assert_eq!(slen, from.metadata().unwrap().len()); 141 | assert!(probably_sparse(&from).unwrap()); 142 | 143 | let out = run(&[ 144 | "--driver", 145 | drv, 146 | from.to_str().unwrap(), 147 | to.to_str().unwrap(), 148 | ]).unwrap(); 149 | 150 | assert!(out.status.success()); 151 | assert!(probably_sparse(&to).unwrap()); 152 | assert_eq!(quickstat(&from).unwrap(), quickstat(&to).unwrap()); 153 | 154 | let from_data = read(&from).unwrap(); 155 | let to_data = read(&to).unwrap(); 156 | assert_eq!(from_data, to_data); 157 | } 158 | 159 | #[cfg_attr(feature = "parblock", test_case("parblock"; "Test with parallel block driver"))] 160 | #[test_case("parfile"; "Test with parallel file driver")] 161 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 162 | fn test_sparse_trailng_gap(drv: &str) { 163 | use std::fs::read; 164 | 165 | let dir = tempdir_rel().unwrap(); 166 | let from = dir.path().join("sparse.bin"); 167 | let to = dir.path().join("target.bin"); 168 | 169 | let slen = create_sparse(&from, 1024, 1024).unwrap(); 170 | assert_eq!(slen, from.metadata().unwrap().len()); 171 | assert!(probably_sparse(&from).unwrap()); 172 | 173 | let out = run(&[ 174 | "--driver", 175 | drv, 176 | from.to_str().unwrap(), 177 | to.to_str().unwrap(), 178 | ]).unwrap(); 179 | assert!(out.status.success()); 180 | 181 | assert!(probably_sparse(&to).unwrap()); 182 | assert_eq!(quickstat(&from).unwrap(), quickstat(&to).unwrap()); 183 | 184 | let from_data = read(&from).unwrap(); 185 | let to_data = read(&to).unwrap(); 186 | assert_eq!(from_data, to_data); 187 | } 188 | 189 | #[cfg_attr(feature = "parblock", test_case("parblock"; "Test with parallel block driver"))] 190 | #[test_case("parfile"; "Test with parallel file driver")] 191 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 192 | fn test_sparse_single_overwrite(drv: &str) { 193 | use std::fs::read; 194 | 195 | let dir = tempdir_rel().unwrap(); 196 | let from = dir.path().join("sparse.bin"); 197 | let to = dir.path().join("target.bin"); 198 | 199 | let slen = create_sparse(&from, 1024, 1024).unwrap(); 200 | create_file(&to, "").unwrap(); 201 | assert_eq!(slen, from.metadata().unwrap().len()); 202 | assert!(probably_sparse(&from).unwrap()); 203 | 204 | let out = run(&[ 205 | "--driver", 206 | drv, 207 | from.to_str().unwrap(), 208 | to.to_str().unwrap(), 209 | ]).unwrap(); 210 | assert!(out.status.success()); 211 | assert!(probably_sparse(&to).unwrap()); 212 | assert_eq!(quickstat(&from).unwrap(), quickstat(&to).unwrap()); 213 | 214 | let from_data = read(&from).unwrap(); 215 | let to_data = read(&to).unwrap(); 216 | assert_eq!(from_data, to_data); 217 | } 218 | 219 | #[cfg_attr(feature = "parblock", test_case("parblock"; "Test with parallel block driver"))] 220 | #[test_case("parfile"; "Test with parallel file driver")] 221 | #[cfg_attr(feature = "test_no_sparse", ignore = "No FS support")] 222 | fn test_empty_sparse(drv: &str) { 223 | use std::fs::read; 224 | 225 | let dir = tempdir_rel().unwrap(); 226 | let from = dir.path().join("sparse.bin"); 227 | let to = dir.path().join("target.bin"); 228 | 229 | let out = Command::new("/usr/bin/truncate") 230 | .args(["-s", "1M", from.to_str().unwrap()]) 231 | .output().unwrap(); 232 | assert!(out.status.success()); 233 | assert_eq!(from.metadata().unwrap().len(), 1024 * 1024); 234 | 235 | let out = run(&[ 236 | "--driver", 237 | drv, 238 | from.to_str().unwrap(), 239 | to.to_str().unwrap(), 240 | ]).unwrap(); 241 | assert!(out.status.success()); 242 | assert_eq!(to.metadata().unwrap().len(), 1024 * 1024); 243 | 244 | assert!(probably_sparse(&to).unwrap()); 245 | assert_eq!(quickstat(&from).unwrap(), quickstat(&to).unwrap()); 246 | 247 | let from_data = read(&from).unwrap(); 248 | let to_data = read(&to).unwrap(); 249 | assert_eq!(from_data, to_data); 250 | } 251 | 252 | 253 | #[cfg_attr(feature = "parblock", test_case("parblock"; "Test with parallel block driver"))] 254 | #[test_case("parfile"; "Test with parallel file driver")] 255 | #[cfg_attr(not(feature = "test_run_expensive"), ignore = "Stress test")] 256 | fn copy_generated_tree_sparse(drv: &str) { 257 | // Spam some output to keep CI from timing-out (hopefully). 258 | println!("Generating file tree..."); 259 | let src = gen_global_filetree(false).unwrap(); 260 | 261 | let dir = tempdir_rel().unwrap(); 262 | let dest = dir.path().join("target"); 263 | 264 | println!("Running copy..."); 265 | let out = run(&[ 266 | "--driver", drv, 267 | "-r", 268 | "--no-progress", 269 | src.to_str().unwrap(), 270 | dest.to_str().unwrap(), 271 | ]).unwrap(); 272 | assert!(out.status.success()); 273 | 274 | println!("Compare trees..."); 275 | compare_trees(&src, &dest).unwrap(); 276 | } 277 | } 278 | -------------------------------------------------------------------------------- /tests/scripts/make-filesystems.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sudo apt update 4 | 5 | sudo apt install -y zfsutils-linux xfsprogs \ 6 | btrfs-progs ntfs-3g dosfstools 7 | 8 | for fs in "$@"; do 9 | root=/fs/$fs 10 | img=$root.img 11 | 12 | echo >&2 "==== creating $fs in $root ====" 13 | 14 | sudo mkdir --parents $root 15 | sudo fallocate --length 2.5G $img 16 | 17 | case $fs in 18 | zfs) sudo zpool create -m $root test $img ;; 19 | ntfs) sudo mkfs.ntfs --fast --force $img ;; 20 | *) sudo mkfs.$fs $img ;; 21 | esac 22 | 23 | # zfs gets automounted 24 | if [[ $fs != zfs ]]; then 25 | if [[ $fs = fat ]]; then 26 | sudo mount -o uid=$(id -u) $img $root 27 | else 28 | sudo mount $img $root 29 | fi 30 | fi 31 | 32 | # fat mount point cannot be chowned 33 | # and is handled by the uid= option above 34 | if [[ $fs != fat ]]; then 35 | sudo chown $USER $root 36 | fi 37 | 38 | git clone . $root/src 39 | done 40 | 41 | findmnt --real 42 | -------------------------------------------------------------------------------- /tests/scripts/test-linux.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | # chdir to source root 6 | cd "$(dirname "$0")"/../.. 7 | 8 | # get the name of the filesystem that contains the source code 9 | fs=$(df --output=fstype . | tail -n 1) 10 | 11 | # list features supported by all filesystems 12 | features=(use_linux "$@") 13 | 14 | # disable tests that will not work on this filesystem 15 | case "$fs" in 16 | xfs | btrfs | bcachefs) ;; 17 | 18 | ext4) 19 | features+=( 20 | test_no_reflink 21 | ) 22 | ;; 23 | 24 | ext[23]) 25 | features+=( 26 | test_no_extents 27 | test_no_reflink 28 | ) 29 | ;; 30 | 31 | f2fs) 32 | features+=( 33 | test_no_reflink 34 | ) 35 | ;; 36 | 37 | fuseblk) 38 | echo >&2 "WARNING: assuming ntfs" 39 | features+=( 40 | test_no_acl 41 | test_no_extents 42 | test_no_reflink 43 | test_no_sparse 44 | test_no_perms 45 | ) 46 | ;; 47 | 48 | vfat) 49 | features+=( 50 | test_no_acl 51 | test_no_extents 52 | test_no_reflink 53 | test_no_sockets 54 | test_no_sparse 55 | test_no_symlinks 56 | test_no_xattr 57 | test_no_perms 58 | ) 59 | ;; 60 | 61 | tmpfs) 62 | features+=( 63 | test_no_extents 64 | test_no_reflink 65 | test_no_sparse 66 | ) 67 | ;; 68 | 69 | zfs) 70 | features+=( 71 | test_no_acl 72 | test_no_extents 73 | test_no_reflink 74 | test_no_sparse 75 | ) 76 | ;; 77 | 78 | *) 79 | echo >&2 "WARNING: unknown filesystem $fs, advanced FS tests disabled." 80 | features+=( 81 | test_no_acl 82 | test_no_extents 83 | test_no_reflink 84 | test_no_sparse 85 | ) 86 | ;; 87 | esac 88 | 89 | echo >&2 "found filesystem $fs, using flags ${features[*]}" 90 | 91 | cargo test --workspace --release --locked --features "$( 92 | export IFS=, 93 | echo "${features[*]}" 94 | )" 95 | -------------------------------------------------------------------------------- /tests/util.rs: -------------------------------------------------------------------------------- 1 | #![allow(unused)] 2 | /* 3 | * Copyright © 2018, Steve Smith 4 | * 5 | * This program is free software: you can redistribute it and/or 6 | * modify it under the terms of the GNU General Public License version 7 | * 3 as published by the Free Software Foundation. 8 | * 9 | * This program is distributed in the hope that it will be useful, but 10 | * WITHOUT ANY WARRANTY; without even the implied warranty of 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | * General Public License for more details. 13 | * 14 | * You should have received a copy of the GNU General Public License 15 | * along with this program. If not, see . 16 | */ 17 | 18 | use anyhow::{self, Error}; 19 | use fslock::LockFile; 20 | use rand::{Rng, RngCore, SeedableRng, rng}; 21 | use rand_distr::{Alphanumeric, Pareto, Triangular, StandardUniform}; 22 | use rand_xorshift::XorShiftRng; 23 | use std::cmp; 24 | use std::env::current_dir; 25 | use std::fs::{create_dir_all, File, FileTimes}; 26 | use std::io::{BufRead, BufReader, Read, Seek, SeekFrom, Write}; 27 | use std::path::{Path, PathBuf}; 28 | use std::process::{Command, Output}; 29 | use std::result; 30 | use std::time::{Duration, SystemTime}; 31 | use tempfile::{tempdir_in, TempDir}; 32 | use uuid::Uuid; 33 | use walkdir::WalkDir; 34 | 35 | pub type TResult = result::Result<(), Error>; 36 | 37 | pub fn get_command() -> Result { 38 | let exe = env!("CARGO_BIN_EXE_xcp"); 39 | Ok(Command::new(exe)) 40 | } 41 | 42 | pub fn run(args: &[&str]) -> Result { 43 | let out = get_command()?.args(args).output()?; 44 | println!("STDOUT: {}", String::from_utf8_lossy(&out.stdout)); 45 | println!("STDERR: {}", String::from_utf8_lossy(&out.stderr)); 46 | Ok(out) 47 | } 48 | 49 | pub fn tempdir_rel() -> Result { 50 | // let uuid = Uuid::new_v4(); 51 | // let dir = PathBuf::from("target/").join(uuid.to_string()); 52 | // create_dir_all(&dir)?; 53 | // Ok(dir) 54 | Ok(tempdir_in(current_dir()?.join("target"))?) 55 | } 56 | 57 | pub fn create_file(path: &Path, text: &str) -> Result<(), Error> { 58 | let file = File::create(path)?; 59 | write!(&file, "{}", text)?; 60 | Ok(()) 61 | } 62 | 63 | pub fn set_time_past(file: &Path) -> Result<(), Error> { 64 | let yr = Duration::from_secs(60 * 60 * 24 * 365); 65 | let past = SystemTime::now().checked_sub(yr).unwrap(); 66 | let ft = FileTimes::new() 67 | .set_modified(past); 68 | File::open(file)?.set_times(ft)?; 69 | Ok(()) 70 | } 71 | 72 | pub fn timestamps_same(from: &SystemTime, to: &SystemTime) -> bool { 73 | let from_s = from.duration_since(SystemTime::UNIX_EPOCH).unwrap().as_secs() as i64; 74 | let to_s = to.duration_since(SystemTime::UNIX_EPOCH).unwrap().as_secs() as i64; 75 | // 5s tolerance 76 | from_s.abs_diff(to_s) < 5 77 | } 78 | 79 | 80 | #[cfg(any(target_os = "linux", target_os = "android"))] 81 | #[allow(unused)] 82 | pub fn create_sparse(file: &Path, head: u64, tail: u64) -> Result { 83 | let data = "c00lc0d3"; 84 | let len = 4096u64 * 4096 + data.len() as u64 + tail; 85 | 86 | let out = Command::new("/usr/bin/truncate") 87 | .args(["-s", len.to_string().as_str(), file.to_str().unwrap()]) 88 | .output()?; 89 | assert!(out.status.success()); 90 | 91 | let mut fd = std::fs::OpenOptions::new() 92 | .write(true) 93 | .append(false) 94 | .open(file)?; 95 | 96 | fd.seek(SeekFrom::Start(head))?; 97 | write!(fd, "{}", data)?; 98 | 99 | fd.seek(SeekFrom::Start(1024 * 4096))?; 100 | write!(fd, "{}", data)?; 101 | 102 | fd.seek(SeekFrom::Start(4096 * 4096))?; 103 | write!(fd, "{}", data)?; 104 | 105 | Ok(len) 106 | } 107 | 108 | #[allow(unused)] 109 | pub fn file_contains(path: &Path, text: &str) -> Result { 110 | let mut dest = File::open(path)?; 111 | let mut buf = String::new(); 112 | dest.read_to_string(&mut buf)?; 113 | 114 | Ok(buf == text) 115 | } 116 | 117 | pub fn files_match(a: &Path, b: &Path) -> bool { 118 | println!("Checking: {:?}", a); 119 | if a.metadata().unwrap().len() != b.metadata().unwrap().len() { 120 | return false; 121 | } 122 | let mut abr = BufReader::with_capacity(1024 * 1024, File::open(a).unwrap()); 123 | let mut bbr = BufReader::with_capacity(1024 * 1024, File::open(b).unwrap()); 124 | loop { 125 | let read = { 126 | let ab = abr.fill_buf().unwrap(); 127 | let bb = bbr.fill_buf().unwrap(); 128 | if ab != bb { 129 | return false; 130 | } 131 | if ab.is_empty() { 132 | return true; 133 | } 134 | ab.len() 135 | }; 136 | abr.consume(read); 137 | bbr.consume(read); 138 | } 139 | } 140 | 141 | #[test] 142 | fn test_hasher() -> TResult { 143 | { 144 | let dir = tempdir_rel()?; 145 | let a = dir.path().join("source.txt"); 146 | let b = dir.path().join("dest.txt"); 147 | let text = "sd;lkjfasl;kjfa;sldkfjaslkjfa;jsdlfkjsdlfkajl"; 148 | create_file(&a, text)?; 149 | create_file(&b, text)?; 150 | assert!(files_match(&a, &b)); 151 | } 152 | { 153 | let dir = tempdir_rel()?; 154 | let a = dir.path().join("source.txt"); 155 | let b = dir.path().join("dest.txt"); 156 | create_file(&a, "lskajdf;laksjdfl;askjdf;alksdj")?; 157 | create_file(&b, "29483793857398")?; 158 | assert!(!files_match(&a, &b)); 159 | } 160 | 161 | Ok(()) 162 | } 163 | 164 | #[cfg(any(target_os = "linux", target_os = "android"))] 165 | pub fn quickstat(file: &Path) -> Result<(i32, i32, i32), Error> { 166 | let out = Command::new("stat") 167 | .args(["--format", "%s %b %B", file.to_str().unwrap()]) 168 | .output()?; 169 | assert!(out.status.success()); 170 | 171 | let stdout = String::from_utf8(out.stdout)?; 172 | let stats = stdout 173 | .split_whitespace() 174 | .map(|s| s.parse::().unwrap()) 175 | .collect::>(); 176 | let (size, blocks, blksize) = (stats[0], stats[1], stats[2]); 177 | 178 | Ok((size, blocks, blksize)) 179 | } 180 | 181 | #[cfg(any(target_os = "linux", target_os = "android"))] 182 | pub fn probably_sparse(file: &Path) -> Result { 183 | let (size, blocks, blksize) = quickstat(file)?; 184 | Ok(blocks < size / blksize) 185 | } 186 | #[cfg(not(any(target_os = "linux", target_os = "android")))] 187 | pub fn probably_sparse(file: &Path) -> Result { 188 | Ok(false) 189 | } 190 | 191 | pub fn rand_data(len: usize) -> Vec { 192 | rng() 193 | .sample_iter(StandardUniform) 194 | .take(len) 195 | .collect() 196 | } 197 | 198 | const MAXDEPTH: u64 = 2; 199 | 200 | pub fn gen_file_name(rng: &mut dyn RngCore, len: u64) -> String { 201 | let r = rng 202 | .sample_iter(Alphanumeric) 203 | .take(len as usize) 204 | .collect::>(); 205 | String::from_utf8(r).unwrap() 206 | } 207 | 208 | pub fn gen_file(path: &Path, rng: &mut dyn RngCore, size: usize, sparse: bool) -> TResult { 209 | println!("Generating: {:?}", path); 210 | let mut fd = File::create(path)?; 211 | const BSIZE: usize = 4096; 212 | let mut buffer = [0; BSIZE]; 213 | let mut left = size; 214 | 215 | while left > 0 { 216 | let blen = cmp::min(left, BSIZE); 217 | let b = &mut buffer[..blen]; 218 | rng.fill(b); 219 | if sparse && b[0] % 3 == 0 { 220 | fd.seek(SeekFrom::Current(blen as i64))?; 221 | left -= blen; 222 | } else { 223 | left -= fd.write(b)?; 224 | } 225 | } 226 | 227 | Ok(()) 228 | } 229 | 230 | /// Recursive random file-tree generator. The distributions have been 231 | /// manually chosen to give a rough approximation of a working 232 | /// project, with most files in the 10's of Ks, and a few larger 233 | /// ones. With a seeded PRNG (see below) this will give a repeatable 234 | /// tree depending on the seed. 235 | pub fn gen_subtree(base: &Path, rng: &mut dyn RngCore, depth: u64, with_sparse: bool) -> TResult { 236 | create_dir_all(base)?; 237 | 238 | let dist0 = Triangular::new(0.0, 64.0, 64.0 / 5.0)?; 239 | let dist1 = Triangular::new(1.0, 64.0, 64.0 / 5.0)?; 240 | let distf = Pareto::new(50.0 * 1024.0, 1.0)?; 241 | 242 | let nfiles = rng.sample(dist0) as u64; 243 | for _ in 0..nfiles { 244 | let fnlen = rng.sample(dist1) as u64; 245 | let fsize = rng.sample(distf) as u64; 246 | let fname = gen_file_name(rng, fnlen); 247 | let path = base.join(fname); 248 | let sparse = with_sparse && nfiles % 3 == 0; 249 | gen_file(&path, rng, fsize as usize, sparse)?; 250 | } 251 | 252 | if depth < MAXDEPTH { 253 | let ndirs = rng.sample(dist1) as u64; 254 | for _ in 0..ndirs { 255 | let fnlen = rng.sample(dist1) as u64; 256 | let fname = gen_file_name(rng, fnlen); 257 | let path = base.join(fname); 258 | gen_subtree(&path, rng, depth + 1, with_sparse)?; 259 | } 260 | } 261 | 262 | Ok(()) 263 | } 264 | 265 | pub fn gen_global_filetree(with_sparse: bool) -> anyhow::Result { 266 | let path = PathBuf::from("target/generated_filetree"); 267 | let lockfile = path.with_extension("lock"); 268 | 269 | let mut lf = LockFile::open(&lockfile)?; 270 | lf.lock()?; 271 | if !path.exists() { 272 | gen_filetree(&path, 0, with_sparse)?; 273 | } 274 | lf.unlock(); 275 | 276 | Ok(path) 277 | } 278 | 279 | pub fn gen_filetree(base: &Path, seed: u64, with_sparse: bool) -> TResult { 280 | let mut rng = XorShiftRng::seed_from_u64(seed); 281 | gen_subtree(base, &mut rng, 0, with_sparse) 282 | } 283 | 284 | pub fn compare_trees(src: &Path, dest: &Path) -> TResult { 285 | let pref = src.components().count(); 286 | for entry in WalkDir::new(src) { 287 | let from = entry?.into_path(); 288 | let tail: PathBuf = from.components().skip(pref).collect(); 289 | let to = dest.join(tail); 290 | 291 | assert!(to.exists()); 292 | assert_eq!(from.is_dir(), to.is_dir()); 293 | assert_eq!( 294 | from.metadata()?.file_type().is_symlink(), 295 | to.metadata()?.file_type().is_symlink() 296 | ); 297 | 298 | if from.is_file() { 299 | assert_eq!(probably_sparse(&to)?, probably_sparse(&to)?); 300 | assert!(files_match(&from, &to)); 301 | // FIXME: Ideally we'd check sparse holes here, but 302 | // there's no guarantee they'll match exactly due to 303 | // low-level filesystem details (SEEK_HOLE behaviour, 304 | // tail-packing, compression, etc.) 305 | } 306 | } 307 | Ok(()) 308 | } 309 | --------------------------------------------------------------------------------