├── .github └── workflows │ └── test.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── LICENSE.txt ├── README.md ├── rustfmt.toml └── src ├── index.rs ├── lib.rs ├── main.rs ├── streaming_iterator.rs └── sync ├── fs.rs ├── locations.rs ├── mod.rs ├── ssh ├── mod.rs └── proto.rs └── utils.rs /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | - push 5 | - pull_request 6 | 7 | jobs: 8 | build: 9 | strategy: 10 | matrix: 11 | os: [ubuntu-latest] 12 | rust-version: [1.46.0, stable, nightly] 13 | include: 14 | - os: macos-latest 15 | rust-version: stable 16 | fail-fast: false 17 | runs-on: ${{ matrix.os }} 18 | steps: 19 | - uses: actions/checkout@v2 20 | - name: Set up Rust ${{ matrix.rust-version }} 21 | uses: actions-rs/toolchain@v1 22 | with: 23 | toolchain: ${{ matrix.rust-version }} 24 | override: true 25 | - name: Install dependencies 26 | if: matrix.os == 'ubuntu-latest' 27 | run: | 28 | sudo apt-get update -qq 29 | sudo apt-get install -yy libsqlite3-dev 30 | - name: Build 31 | run: cargo build --verbose 32 | - name: Run tests 33 | run: cargo test --verbose 34 | - name: Build doc 35 | run: cargo doc 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Rust 2 | target 3 | 4 | # Vi 5 | .*.swp 6 | 7 | # Emacs 8 | \#*# 9 | 10 | # OS files 11 | .DS_Store 12 | desktop.ini 13 | 14 | # Archives 15 | *.tar 16 | *.tar.gz 17 | *.tar.bz2 18 | *.zip 19 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | [[package]] 4 | name = "ansi_term" 5 | version = "0.11.0" 6 | source = "registry+https://github.com/rust-lang/crates.io-index" 7 | checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" 8 | dependencies = [ 9 | "winapi", 10 | ] 11 | 12 | [[package]] 13 | name = "atty" 14 | version = "0.2.14" 15 | source = "registry+https://github.com/rust-lang/crates.io-index" 16 | checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" 17 | dependencies = [ 18 | "hermit-abi", 19 | "libc", 20 | "winapi", 21 | ] 22 | 23 | [[package]] 24 | name = "autocfg" 25 | version = "1.0.1" 26 | source = "registry+https://github.com/rust-lang/crates.io-index" 27 | checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" 28 | 29 | [[package]] 30 | name = "bitflags" 31 | version = "1.3.2" 32 | source = "registry+https://github.com/rust-lang/crates.io-index" 33 | checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 34 | 35 | [[package]] 36 | name = "bytes" 37 | version = "1.1.0" 38 | source = "registry+https://github.com/rust-lang/crates.io-index" 39 | checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" 40 | 41 | [[package]] 42 | name = "cdchunking" 43 | version = "0.2.1" 44 | source = "registry+https://github.com/rust-lang/crates.io-index" 45 | checksum = "22a387449903c8ce0a92df1e70c4e0762e1ae0254b69f7f6ff1d8b46f634c2e5" 46 | 47 | [[package]] 48 | name = "cfg-if" 49 | version = "1.0.0" 50 | source = "registry+https://github.com/rust-lang/crates.io-index" 51 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 52 | 53 | [[package]] 54 | name = "chrono" 55 | version = "0.4.19" 56 | source = "registry+https://github.com/rust-lang/crates.io-index" 57 | checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" 58 | dependencies = [ 59 | "libc", 60 | "num-integer", 61 | "num-traits", 62 | "time", 63 | "winapi", 64 | ] 65 | 66 | [[package]] 67 | name = "clap" 68 | version = "2.33.3" 69 | source = "registry+https://github.com/rust-lang/crates.io-index" 70 | checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" 71 | dependencies = [ 72 | "ansi_term", 73 | "atty", 74 | "bitflags", 75 | "strsim", 76 | "textwrap", 77 | "unicode-width", 78 | "vec_map", 79 | ] 80 | 81 | [[package]] 82 | name = "env_logger" 83 | version = "0.7.1" 84 | source = "registry+https://github.com/rust-lang/crates.io-index" 85 | checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" 86 | dependencies = [ 87 | "atty", 88 | "humantime", 89 | "log", 90 | "termcolor", 91 | ] 92 | 93 | [[package]] 94 | name = "futures" 95 | version = "0.3.17" 96 | source = "registry+https://github.com/rust-lang/crates.io-index" 97 | checksum = "a12aa0eb539080d55c3f2d45a67c3b58b6b0773c1a3ca2dfec66d58c97fd66ca" 98 | dependencies = [ 99 | "futures-channel", 100 | "futures-core", 101 | "futures-executor", 102 | "futures-io", 103 | "futures-sink", 104 | "futures-task", 105 | "futures-util", 106 | ] 107 | 108 | [[package]] 109 | name = "futures-channel" 110 | version = "0.3.17" 111 | source = "registry+https://github.com/rust-lang/crates.io-index" 112 | checksum = "5da6ba8c3bb3c165d3c7319fc1cc8304facf1fb8db99c5de877183c08a273888" 113 | dependencies = [ 114 | "futures-core", 115 | "futures-sink", 116 | ] 117 | 118 | [[package]] 119 | name = "futures-core" 120 | version = "0.3.17" 121 | source = "registry+https://github.com/rust-lang/crates.io-index" 122 | checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d" 123 | 124 | [[package]] 125 | name = "futures-executor" 126 | version = "0.3.17" 127 | source = "registry+https://github.com/rust-lang/crates.io-index" 128 | checksum = "45025be030969d763025784f7f355043dc6bc74093e4ecc5000ca4dc50d8745c" 129 | dependencies = [ 130 | "futures-core", 131 | "futures-task", 132 | "futures-util", 133 | ] 134 | 135 | [[package]] 136 | name = "futures-io" 137 | version = "0.3.17" 138 | source = "registry+https://github.com/rust-lang/crates.io-index" 139 | checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377" 140 | 141 | [[package]] 142 | name = "futures-macro" 143 | version = "0.3.17" 144 | source = "registry+https://github.com/rust-lang/crates.io-index" 145 | checksum = "18e4a4b95cea4b4ccbcf1c5675ca7c4ee4e9e75eb79944d07defde18068f79bb" 146 | dependencies = [ 147 | "autocfg", 148 | "proc-macro-hack", 149 | "proc-macro2", 150 | "quote", 151 | "syn", 152 | ] 153 | 154 | [[package]] 155 | name = "futures-sink" 156 | version = "0.3.17" 157 | source = "registry+https://github.com/rust-lang/crates.io-index" 158 | checksum = "36ea153c13024fe480590b3e3d4cad89a0cfacecc24577b68f86c6ced9c2bc11" 159 | 160 | [[package]] 161 | name = "futures-task" 162 | version = "0.3.17" 163 | source = "registry+https://github.com/rust-lang/crates.io-index" 164 | checksum = "1d3d00f4eddb73e498a54394f228cd55853bdf059259e8e7bc6e69d408892e99" 165 | 166 | [[package]] 167 | name = "futures-util" 168 | version = "0.3.17" 169 | source = "registry+https://github.com/rust-lang/crates.io-index" 170 | checksum = "36568465210a3a6ee45e1f165136d68671471a501e632e9a98d96872222b5481" 171 | dependencies = [ 172 | "autocfg", 173 | "futures-channel", 174 | "futures-core", 175 | "futures-io", 176 | "futures-macro", 177 | "futures-sink", 178 | "futures-task", 179 | "memchr", 180 | "pin-project-lite", 181 | "pin-utils", 182 | "proc-macro-hack", 183 | "proc-macro-nested", 184 | "slab", 185 | ] 186 | 187 | [[package]] 188 | name = "getrandom" 189 | version = "0.2.3" 190 | source = "registry+https://github.com/rust-lang/crates.io-index" 191 | checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" 192 | dependencies = [ 193 | "cfg-if", 194 | "libc", 195 | "wasi", 196 | ] 197 | 198 | [[package]] 199 | name = "hermit-abi" 200 | version = "0.1.19" 201 | source = "registry+https://github.com/rust-lang/crates.io-index" 202 | checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" 203 | dependencies = [ 204 | "libc", 205 | ] 206 | 207 | [[package]] 208 | name = "humantime" 209 | version = "1.3.0" 210 | source = "registry+https://github.com/rust-lang/crates.io-index" 211 | checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" 212 | dependencies = [ 213 | "quick-error", 214 | ] 215 | 216 | [[package]] 217 | name = "libc" 218 | version = "0.2.101" 219 | source = "registry+https://github.com/rust-lang/crates.io-index" 220 | checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21" 221 | 222 | [[package]] 223 | name = "libsqlite3-sys" 224 | version = "0.11.1" 225 | source = "registry+https://github.com/rust-lang/crates.io-index" 226 | checksum = "3567bc1a0c84e2c0d71eeb4a1f08451babf7843babd733158777d9c686dad9f3" 227 | dependencies = [ 228 | "pkg-config", 229 | "vcpkg", 230 | ] 231 | 232 | [[package]] 233 | name = "linked-hash-map" 234 | version = "0.5.4" 235 | source = "registry+https://github.com/rust-lang/crates.io-index" 236 | checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" 237 | 238 | [[package]] 239 | name = "log" 240 | version = "0.4.14" 241 | source = "registry+https://github.com/rust-lang/crates.io-index" 242 | checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" 243 | dependencies = [ 244 | "cfg-if", 245 | ] 246 | 247 | [[package]] 248 | name = "lru-cache" 249 | version = "0.1.2" 250 | source = "registry+https://github.com/rust-lang/crates.io-index" 251 | checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" 252 | dependencies = [ 253 | "linked-hash-map", 254 | ] 255 | 256 | [[package]] 257 | name = "memchr" 258 | version = "2.4.1" 259 | source = "registry+https://github.com/rust-lang/crates.io-index" 260 | checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" 261 | 262 | [[package]] 263 | name = "mio" 264 | version = "0.7.13" 265 | source = "registry+https://github.com/rust-lang/crates.io-index" 266 | checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16" 267 | dependencies = [ 268 | "libc", 269 | "log", 270 | "miow", 271 | "ntapi", 272 | "winapi", 273 | ] 274 | 275 | [[package]] 276 | name = "miow" 277 | version = "0.3.7" 278 | source = "registry+https://github.com/rust-lang/crates.io-index" 279 | checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" 280 | dependencies = [ 281 | "winapi", 282 | ] 283 | 284 | [[package]] 285 | name = "ntapi" 286 | version = "0.3.6" 287 | source = "registry+https://github.com/rust-lang/crates.io-index" 288 | checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" 289 | dependencies = [ 290 | "winapi", 291 | ] 292 | 293 | [[package]] 294 | name = "num-integer" 295 | version = "0.1.44" 296 | source = "registry+https://github.com/rust-lang/crates.io-index" 297 | checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" 298 | dependencies = [ 299 | "autocfg", 300 | "num-traits", 301 | ] 302 | 303 | [[package]] 304 | name = "num-traits" 305 | version = "0.2.14" 306 | source = "registry+https://github.com/rust-lang/crates.io-index" 307 | checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" 308 | dependencies = [ 309 | "autocfg", 310 | ] 311 | 312 | [[package]] 313 | name = "once_cell" 314 | version = "1.8.0" 315 | source = "registry+https://github.com/rust-lang/crates.io-index" 316 | checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" 317 | 318 | [[package]] 319 | name = "pin-project-lite" 320 | version = "0.2.7" 321 | source = "registry+https://github.com/rust-lang/crates.io-index" 322 | checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" 323 | 324 | [[package]] 325 | name = "pin-utils" 326 | version = "0.1.0" 327 | source = "registry+https://github.com/rust-lang/crates.io-index" 328 | checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" 329 | 330 | [[package]] 331 | name = "pkg-config" 332 | version = "0.3.19" 333 | source = "registry+https://github.com/rust-lang/crates.io-index" 334 | checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" 335 | 336 | [[package]] 337 | name = "ppv-lite86" 338 | version = "0.2.10" 339 | source = "registry+https://github.com/rust-lang/crates.io-index" 340 | checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" 341 | 342 | [[package]] 343 | name = "proc-macro-hack" 344 | version = "0.5.19" 345 | source = "registry+https://github.com/rust-lang/crates.io-index" 346 | checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" 347 | 348 | [[package]] 349 | name = "proc-macro-nested" 350 | version = "0.1.7" 351 | source = "registry+https://github.com/rust-lang/crates.io-index" 352 | checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" 353 | 354 | [[package]] 355 | name = "proc-macro2" 356 | version = "1.0.29" 357 | source = "registry+https://github.com/rust-lang/crates.io-index" 358 | checksum = "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d" 359 | dependencies = [ 360 | "unicode-xid", 361 | ] 362 | 363 | [[package]] 364 | name = "quick-error" 365 | version = "1.2.3" 366 | source = "registry+https://github.com/rust-lang/crates.io-index" 367 | checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" 368 | 369 | [[package]] 370 | name = "quote" 371 | version = "1.0.9" 372 | source = "registry+https://github.com/rust-lang/crates.io-index" 373 | checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" 374 | dependencies = [ 375 | "proc-macro2", 376 | ] 377 | 378 | [[package]] 379 | name = "rand" 380 | version = "0.8.4" 381 | source = "registry+https://github.com/rust-lang/crates.io-index" 382 | checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" 383 | dependencies = [ 384 | "libc", 385 | "rand_chacha", 386 | "rand_core", 387 | "rand_hc", 388 | ] 389 | 390 | [[package]] 391 | name = "rand_chacha" 392 | version = "0.3.1" 393 | source = "registry+https://github.com/rust-lang/crates.io-index" 394 | checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" 395 | dependencies = [ 396 | "ppv-lite86", 397 | "rand_core", 398 | ] 399 | 400 | [[package]] 401 | name = "rand_core" 402 | version = "0.6.3" 403 | source = "registry+https://github.com/rust-lang/crates.io-index" 404 | checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" 405 | dependencies = [ 406 | "getrandom", 407 | ] 408 | 409 | [[package]] 410 | name = "rand_hc" 411 | version = "0.3.1" 412 | source = "registry+https://github.com/rust-lang/crates.io-index" 413 | checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" 414 | dependencies = [ 415 | "rand_core", 416 | ] 417 | 418 | [[package]] 419 | name = "redox_syscall" 420 | version = "0.2.10" 421 | source = "registry+https://github.com/rust-lang/crates.io-index" 422 | checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" 423 | dependencies = [ 424 | "bitflags", 425 | ] 426 | 427 | [[package]] 428 | name = "remove_dir_all" 429 | version = "0.5.3" 430 | source = "registry+https://github.com/rust-lang/crates.io-index" 431 | checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" 432 | dependencies = [ 433 | "winapi", 434 | ] 435 | 436 | [[package]] 437 | name = "rusqlite" 438 | version = "0.16.0" 439 | source = "registry+https://github.com/rust-lang/crates.io-index" 440 | checksum = "6381ddfe91dbb659b4b132168da15985bc84162378cf4fcdc4eb99c857d063e2" 441 | dependencies = [ 442 | "bitflags", 443 | "chrono", 444 | "libsqlite3-sys", 445 | "lru-cache", 446 | "time", 447 | ] 448 | 449 | [[package]] 450 | name = "sha1" 451 | version = "0.6.0" 452 | source = "registry+https://github.com/rust-lang/crates.io-index" 453 | checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" 454 | 455 | [[package]] 456 | name = "signal-hook-registry" 457 | version = "1.4.0" 458 | source = "registry+https://github.com/rust-lang/crates.io-index" 459 | checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" 460 | dependencies = [ 461 | "libc", 462 | ] 463 | 464 | [[package]] 465 | name = "slab" 466 | version = "0.4.4" 467 | source = "registry+https://github.com/rust-lang/crates.io-index" 468 | checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590" 469 | 470 | [[package]] 471 | name = "strsim" 472 | version = "0.8.0" 473 | source = "registry+https://github.com/rust-lang/crates.io-index" 474 | checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" 475 | 476 | [[package]] 477 | name = "syn" 478 | version = "1.0.76" 479 | source = "registry+https://github.com/rust-lang/crates.io-index" 480 | checksum = "c6f107db402c2c2055242dbf4d2af0e69197202e9faacbef9571bbe47f5a1b84" 481 | dependencies = [ 482 | "proc-macro2", 483 | "quote", 484 | "unicode-xid", 485 | ] 486 | 487 | [[package]] 488 | name = "syncfast" 489 | version = "0.2.0" 490 | dependencies = [ 491 | "cdchunking", 492 | "chrono", 493 | "clap", 494 | "env_logger", 495 | "futures", 496 | "log", 497 | "rusqlite", 498 | "sha1", 499 | "tempfile", 500 | "tokio", 501 | ] 502 | 503 | [[package]] 504 | name = "tempfile" 505 | version = "3.2.0" 506 | source = "registry+https://github.com/rust-lang/crates.io-index" 507 | checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" 508 | dependencies = [ 509 | "cfg-if", 510 | "libc", 511 | "rand", 512 | "redox_syscall", 513 | "remove_dir_all", 514 | "winapi", 515 | ] 516 | 517 | [[package]] 518 | name = "termcolor" 519 | version = "1.1.2" 520 | source = "registry+https://github.com/rust-lang/crates.io-index" 521 | checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" 522 | dependencies = [ 523 | "winapi-util", 524 | ] 525 | 526 | [[package]] 527 | name = "textwrap" 528 | version = "0.11.0" 529 | source = "registry+https://github.com/rust-lang/crates.io-index" 530 | checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" 531 | dependencies = [ 532 | "unicode-width", 533 | ] 534 | 535 | [[package]] 536 | name = "time" 537 | version = "0.1.43" 538 | source = "registry+https://github.com/rust-lang/crates.io-index" 539 | checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" 540 | dependencies = [ 541 | "libc", 542 | "winapi", 543 | ] 544 | 545 | [[package]] 546 | name = "tokio" 547 | version = "1.11.0" 548 | source = "registry+https://github.com/rust-lang/crates.io-index" 549 | checksum = "b4efe6fc2395938c8155973d7be49fe8d03a843726e285e100a8a383cc0154ce" 550 | dependencies = [ 551 | "autocfg", 552 | "bytes", 553 | "libc", 554 | "memchr", 555 | "mio", 556 | "once_cell", 557 | "pin-project-lite", 558 | "signal-hook-registry", 559 | "winapi", 560 | ] 561 | 562 | [[package]] 563 | name = "unicode-width" 564 | version = "0.1.8" 565 | source = "registry+https://github.com/rust-lang/crates.io-index" 566 | checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" 567 | 568 | [[package]] 569 | name = "unicode-xid" 570 | version = "0.2.2" 571 | source = "registry+https://github.com/rust-lang/crates.io-index" 572 | checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" 573 | 574 | [[package]] 575 | name = "vcpkg" 576 | version = "0.2.15" 577 | source = "registry+https://github.com/rust-lang/crates.io-index" 578 | checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" 579 | 580 | [[package]] 581 | name = "vec_map" 582 | version = "0.8.2" 583 | source = "registry+https://github.com/rust-lang/crates.io-index" 584 | checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" 585 | 586 | [[package]] 587 | name = "wasi" 588 | version = "0.10.2+wasi-snapshot-preview1" 589 | source = "registry+https://github.com/rust-lang/crates.io-index" 590 | checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" 591 | 592 | [[package]] 593 | name = "winapi" 594 | version = "0.3.9" 595 | source = "registry+https://github.com/rust-lang/crates.io-index" 596 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 597 | dependencies = [ 598 | "winapi-i686-pc-windows-gnu", 599 | "winapi-x86_64-pc-windows-gnu", 600 | ] 601 | 602 | [[package]] 603 | name = "winapi-i686-pc-windows-gnu" 604 | version = "0.4.0" 605 | source = "registry+https://github.com/rust-lang/crates.io-index" 606 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 607 | 608 | [[package]] 609 | name = "winapi-util" 610 | version = "0.1.5" 611 | source = "registry+https://github.com/rust-lang/crates.io-index" 612 | checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" 613 | dependencies = [ 614 | "winapi", 615 | ] 616 | 617 | [[package]] 618 | name = "winapi-x86_64-pc-windows-gnu" 619 | version = "0.4.0" 620 | source = "registry+https://github.com/rust-lang/crates.io-index" 621 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 622 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "syncfast" 3 | version = "0.2.0" 4 | edition = "2018" 5 | rust-version = "1.46" 6 | authors = ["Remi Rampin "] 7 | description = "rsync/rdiff/zsync clone" 8 | repository = "https://github.com/remram44/syncfast" 9 | documentation = "https://docs.rs/syncfast/" 10 | license = "BSD-3-Clause" 11 | 12 | [[bin]] 13 | name = "syncfast" 14 | path = "src/main.rs" 15 | 16 | [dependencies] 17 | cdchunking = "0.2" 18 | chrono = "0.4" 19 | clap = "2" 20 | futures = "0.3" 21 | env_logger = { version = "0.7", default-features = false, features = ["termcolor", "atty", "humantime"] } 22 | log = "0.4" 23 | rusqlite = { version = "0.16", features = ["chrono"] } 24 | sha1 = "0.6" 25 | tokio = { version = "1.11", features = ["io-std", "io-util", "process", "rt"] } 26 | 27 | [dev-dependencies] 28 | tempfile = "3" 29 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (C) 2019, Remi Rampin 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of the copyright holder nor the names of its contributors 15 | may be used to endorse or promote products derived from this software 16 | without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://github.com/remram44/syncfast/workflows/Test/badge.svg)](https://github.com/remram44/syncfast/actions) 2 | [![Crates.io](https://img.shields.io/crates/v/syncfast.svg)](https://crates.io/crates/syncfast) 3 | [![Documentation](https://docs.rs/syncfast/badge.svg)](https://docs.rs/syncfast) 4 | [![License](https://img.shields.io/crates/l/syncfast.svg)](https://github.com/remram44/syncfast/blob/master/LICENSE.txt) 5 | 6 | What is this? 7 | ============= 8 | 9 | This is an rsync clone written in the [Rust](https://www.rust-lang.org/) programming language. It is intended to provide the functionality of rsync, rdiff, and zsync in one single program, as well as some additions such as caching file signatures to make repeated synchronizations faster. It will also provide a library, allowing to use the functionality in your own programs. 10 | 11 | Current status 12 | ============== 13 | 14 | Core functionality is there. You can index and sync local folders, and sync over SSH. 15 | 16 | The next step is implementing syncing over HTTP, and syncing "offline" (diff/patch). 17 | 18 | How to use 19 | ========== 20 | 21 | ``` 22 | $ syncfast sync some/folder ssh://othermachine/home/folder 23 | ``` 24 | 25 | Notes 26 | ===== 27 | 28 | The rsync algorithm: https://rsync.samba.org/tech_report/ 29 | How rsync works: https://rsync.samba.org/how-rsync-works.html 30 | 31 | zsync: http://zsync.moria.org.uk/ 32 | 33 | Compression crate: https://crates.io/crates/flate2 34 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | max_width = 79 2 | spaces_around_ranges = true 3 | error_on_line_overflow = true 4 | -------------------------------------------------------------------------------- /src/index.rs: -------------------------------------------------------------------------------- 1 | use cdchunking::{ChunkInput, Chunker, ZPAQ}; 2 | use log::{debug, info, warn}; 3 | use rusqlite; 4 | use rusqlite::Connection; 5 | use rusqlite::types::ToSql; 6 | use sha1::Sha1; 7 | use std::fs::File; 8 | use std::path::{Path, PathBuf}; 9 | 10 | use crate::{Error, HashDigest, temp_name}; 11 | 12 | const SCHEMA: &str = " 13 | CREATE TABLE files( 14 | file_id INTEGER NOT NULL PRIMARY KEY, 15 | name VARCHAR(512) NOT NULL, 16 | modified DATETIME NOT NULL, 17 | size INTEGER NULL, 18 | blocks_hash VARCHAR(40) NULL, 19 | temporary BOOLEAN NOT NULL 20 | ); 21 | CREATE INDEX idx_files_name ON files(name); 22 | 23 | CREATE TABLE blocks( 24 | file_id INTEGER NOT NULL, 25 | hash VARCHAR(40) NOT NULL, 26 | offset INTEGER NOT NULL, 27 | size INTEGER NOT NULL, 28 | present BOOLEAN NOT NULL, 29 | PRIMARY KEY(file_id, offset) 30 | ); 31 | CREATE INDEX idx_blocks_file_id ON blocks(file_id); 32 | CREATE INDEX idx_blocks_hash ON blocks(hash); 33 | CREATE INDEX idx_blocks_offset ON blocks(file_id, offset); 34 | CREATE INDEX idx_blocks_present ON blocks(file_id, present); 35 | 36 | PRAGMA application_id=0x51367457; 37 | PRAGMA user_version=0x00000000; 38 | "; 39 | 40 | pub const ZPAQ_BITS: usize = 13; // 13 bits = 8 KiB block average 41 | pub const MAX_BLOCK_SIZE: usize = 1 << 15; // 32 KiB 42 | 43 | /// Index of files and blocks 44 | pub struct Index { 45 | db: Connection, 46 | in_transaction: bool, 47 | } 48 | 49 | impl Index { 50 | /// Open an index from a file 51 | pub fn open(filename: &Path) -> Result { 52 | let exists = filename.exists(); 53 | let db = Connection::open(filename)?; 54 | if !exists { 55 | warn!("Database doesn't exist, creating tables..."); 56 | db.execute_batch(SCHEMA)?; 57 | } 58 | Ok(Index { db, in_transaction: false }) 59 | } 60 | 61 | /// Open an in-memory index 62 | pub fn open_in_memory() -> Result { 63 | let db = Connection::open_in_memory()?; 64 | db.execute_batch(SCHEMA)?; 65 | Ok(Index { db, in_transaction: false }) 66 | } 67 | 68 | pub fn begin(&mut self) -> Result<(), Error> { 69 | if !self.in_transaction { 70 | self.db.execute_batch("BEGIN IMMEDIATE;")?; 71 | self.in_transaction = true; 72 | } 73 | Ok(()) 74 | } 75 | 76 | /// Try to find a block in the indexed files 77 | pub fn get_block( 78 | &self, 79 | hash: &HashDigest, 80 | ) -> Result, Error> { 81 | let mut stmt = self.db.prepare( 82 | " 83 | SELECT files.name, blocks.offset, blocks.size 84 | FROM blocks 85 | INNER JOIN files ON blocks.file_id = files.file_id 86 | WHERE blocks.hash = ? AND blocks.present = 1; 87 | ", 88 | )?; 89 | let mut rows = stmt.query(&[hash as &dyn ToSql])?; 90 | if let Some(row) = rows.next() { 91 | let row = row?; 92 | let path: String = row.get(0); 93 | let path: PathBuf = path.into(); 94 | let offset: i64 = row.get(1); 95 | let offset = offset as usize; 96 | let size: i64 = row.get(2); 97 | let size = size as usize; 98 | Ok(Some((path, offset, size))) 99 | } else { 100 | Ok(None) 101 | } 102 | } 103 | 104 | /// Try to get a file from its name 105 | pub fn get_file( 106 | &self, 107 | name: &Path, 108 | ) -> Result, HashDigest)>, Error> { 109 | let mut stmt = self.db.prepare( 110 | " 111 | SELECT file_id, modified, blocks_hash 112 | FROM files 113 | WHERE name = ? AND temporary = 0; 114 | ", 115 | )?; 116 | let mut rows = stmt.query(&[name.to_str().ok_or(Error::BadFilenameEncoding)?])?; 117 | if let Some(row) = rows.next() { 118 | let row = row?; 119 | let file_id = row.get(0); 120 | let modified = row.get(1); 121 | let blocks_hash = row.get(2); 122 | Ok(Some((file_id, modified, blocks_hash))) 123 | } else { 124 | Ok(None) 125 | } 126 | } 127 | 128 | /// Try to get a temporary file from its name 129 | pub fn get_temp_file( 130 | &self, 131 | name: &Path, 132 | ) -> Result)>, Error> { 133 | let name = temp_name(&name)?; 134 | let mut stmt = self.db.prepare( 135 | " 136 | SELECT file_id, modified 137 | FROM files 138 | WHERE name = ? AND temporary = 1; 139 | ", 140 | )?; 141 | let mut rows = stmt.query(&[name.to_str().ok_or(Error::BadFilenameEncoding)?])?; 142 | if let Some(row) = rows.next() { 143 | let row = row?; 144 | let file_id = row.get(0); 145 | let modified = row.get(1); 146 | Ok(Some((file_id, modified))) 147 | } else { 148 | Ok(None) 149 | } 150 | } 151 | 152 | /// Get the name of a file from its ID 153 | pub fn get_file_name(&self, file_id: u32) -> Result, Error> { 154 | let mut stmt = self.db.prepare( 155 | " 156 | SELECT name 157 | FROM files 158 | WHERE file_id = ?; 159 | ", 160 | )?; 161 | let mut rows = stmt.query(&[file_id])?; 162 | if let Some(row) = rows.next() { 163 | let row = row?; 164 | let path: String = row.get(0); 165 | Ok(Some(path.into())) 166 | } else { 167 | Ok(None) 168 | } 169 | } 170 | 171 | /// Add a file to the index 172 | /// 173 | /// This returns a tuple `(file_id, up_to_date)` where `file_id` can be 174 | /// used to insert blocks, and `up_to_date` indicates whether the file's 175 | /// modification date has changed and it should be re-indexed. 176 | pub fn add_file( 177 | &mut self, 178 | name: &Path, 179 | modified: chrono::DateTime, 180 | ) -> Result<(u32, bool), Error> { 181 | self.begin()?; 182 | if let Some((file_id, old_modified, _)) = self.get_file(name)? { 183 | if old_modified != modified { 184 | info!("Resetting file {:?}, modified", name); 185 | // Delete blocks 186 | self.db.execute( 187 | " 188 | DELETE FROM blocks WHERE file_id = ?; 189 | ", 190 | &[&file_id], 191 | )?; 192 | // Update modification time 193 | self.db.execute( 194 | " 195 | UPDATE files 196 | SET modified = ?, size = NULL, blocks_hash = NULL, temporary = 0 197 | WHERE file_id = ?; 198 | ", 199 | &[&modified as &dyn ToSql, &file_id], 200 | )?; 201 | Ok((file_id, false)) 202 | } else { 203 | debug!("File {:?} up to date", name); 204 | Ok((file_id, true)) 205 | } 206 | } else { 207 | info!("Inserting new file {:?}", name); 208 | self.db.execute( 209 | " 210 | INSERT INTO files(name, modified, temporary) 211 | VALUES(?, ?, 0); 212 | ", 213 | &[&name.to_str().ok_or(Error::BadFilenameEncoding)? as &dyn ToSql, &modified], 214 | )?; 215 | let file_id = self.db.last_insert_rowid(); 216 | Ok((file_id as u32, false)) 217 | } 218 | } 219 | 220 | /// Replace file in the index 221 | /// 222 | /// This is like add_file but will always replace an existing file. 223 | pub fn add_file_overwrite( 224 | &mut self, 225 | name: &Path, 226 | modified: chrono::DateTime, 227 | ) -> Result { 228 | self.begin()?; 229 | if let Some((file_id, _, _)) = self.get_file(name)? { 230 | info!("Resetting file {:?}", name); 231 | // Delete blocks 232 | self.db.execute( 233 | " 234 | DELETE FROM blocks WHERE file_id = ?; 235 | ", 236 | &[&file_id], 237 | )?; 238 | // Update modification time 239 | self.db.execute( 240 | " 241 | UPDATE files 242 | SET modified = ?, size = NULL, blocks_hash = NULL, temporary = 0 243 | WHERE file_id = ?; 244 | ", 245 | &[&modified as &dyn ToSql, &file_id], 246 | )?; 247 | Ok(file_id) 248 | } else { 249 | info!("Inserting new file {:?}", name); 250 | self.db.execute( 251 | " 252 | INSERT INTO files(name, modified, temporary) 253 | VALUES(?, ?, 0); 254 | ", 255 | &[&name.to_str().ok_or(Error::BadFilenameEncoding)? as &dyn ToSql, &modified], 256 | )?; 257 | let file_id = self.db.last_insert_rowid(); 258 | Ok(file_id as u32) 259 | } 260 | } 261 | 262 | pub fn add_temp_file( 263 | &mut self, 264 | name: &Path, 265 | ) -> Result { 266 | self.begin()?; 267 | let modified: chrono::DateTime = chrono::Utc::now(); 268 | let name = temp_name(name)?; 269 | if let Some((file_id, _, _)) = self.get_file(&name)? { 270 | info!("Resetting file {:?}", name); 271 | // Delete blocks 272 | self.db.execute( 273 | " 274 | DELETE FROM blocks WHERE file_id = ?; 275 | ", 276 | &[&file_id], 277 | )?; 278 | // Update modification time 279 | self.db.execute( 280 | " 281 | UPDATE files 282 | SET modified = ?, size = NULL, blocks_hash = NULL, temporary = 1 283 | WHERE file_id = ?; 284 | ", 285 | &[&modified as &dyn ToSql, &file_id], 286 | )?; 287 | Ok(file_id) 288 | } else { 289 | info!("Inserting new file {:?}", name); 290 | self.db.execute( 291 | " 292 | INSERT INTO files(name, modified, temporary) 293 | VALUES(?, ?, 1); 294 | ", 295 | &[&name.to_str().ok_or(Error::BadFilenameEncoding)? as &dyn ToSql, &modified], 296 | )?; 297 | let file_id = self.db.last_insert_rowid(); 298 | Ok(file_id as u32) 299 | } 300 | } 301 | 302 | /// Remove a file and all its blocks from the index 303 | pub fn remove_file(&mut self, file_id: u32) -> Result<(), Error> { 304 | self.begin()?; 305 | self.db.execute( 306 | " 307 | DELETE FROM blocks WHERE file_id = ?; 308 | ", 309 | &[&file_id], 310 | )?; 311 | self.db.execute( 312 | " 313 | DELETE FROM files WHERE file_id = ?; 314 | ", 315 | &[&file_id], 316 | )?; 317 | Ok(()) 318 | } 319 | 320 | /// Move a file, possibly over another 321 | pub fn move_temp_file_into_place( 322 | &mut self, 323 | file_id: u32, 324 | destination: &Path, 325 | ) -> Result<(), Error> { 326 | self.begin()?; 327 | let destination = destination.to_str().ok_or(Error::BadFilenameEncoding)?; 328 | 329 | // Delete old file 330 | self.db.execute( 331 | " 332 | DELETE FROM blocks WHERE file_id = ( 333 | SELECT file_id 334 | FROM files 335 | WHERE name = ? 336 | ); 337 | ", 338 | &[destination], 339 | )?; 340 | self.db.execute( 341 | " 342 | DELETE FROM files WHERE name = ?; 343 | ", 344 | &[destination], 345 | )?; 346 | 347 | // Move new file, clear temporary flag 348 | self.db.execute( 349 | " 350 | UPDATE files SET name = ?, temporary = 0 351 | WHERE file_id = ?; 352 | ", 353 | &[&destination as &dyn ToSql, &file_id], 354 | )?; 355 | Ok(()) 356 | } 357 | 358 | /// Get a list of all the files in the index 359 | pub fn list_files( 360 | &self, 361 | ) -> Result, usize, HashDigest)>, Error> 362 | { 363 | let mut stmt = self.db.prepare( 364 | " 365 | SELECT file_id, name, modified, size, blocks_hash 366 | FROM files 367 | WHERE temporary = 0; 368 | ", 369 | )?; 370 | let mut rows = stmt.query(rusqlite::NO_PARAMS)?; 371 | let mut results = Vec::new(); 372 | loop { 373 | match rows.next() { 374 | Some(Ok(row)) => { 375 | let path: String = row.get(1); 376 | let size: Option = row.get(3); 377 | results.push((row.get(0), path.into(), row.get(2), size.unwrap_or(0) as usize, row.get(4))) 378 | } 379 | Some(Err(e)) => return Err(e.into()), 380 | None => break, 381 | } 382 | } 383 | Ok(results) 384 | } 385 | 386 | /// Add a block to the index 387 | pub fn add_block( 388 | &mut self, 389 | hash: &HashDigest, 390 | file_id: u32, 391 | offset: usize, 392 | size: usize, 393 | ) -> Result<(), Error> { 394 | self.begin()?; 395 | self.db.execute( 396 | " 397 | INSERT INTO blocks(hash, file_id, offset, size, present) 398 | VALUES(?, ?, ?, ?, 1); 399 | ", 400 | &[ 401 | &hash as &dyn ToSql, 402 | &file_id, 403 | &(offset as i64), 404 | &(size as i64), 405 | ], 406 | )?; 407 | Ok(()) 408 | } 409 | 410 | /// Add a block to the index, that we haven't yet received 411 | pub fn add_missing_block( 412 | &mut self, 413 | hash: &HashDigest, 414 | file_id: u32, 415 | offset: usize, 416 | size: usize, 417 | ) -> Result<(), Error> { 418 | self.begin()?; 419 | self.db.execute( 420 | " 421 | INSERT INTO blocks(hash, file_id, offset, size, present) 422 | VALUES(?, ?, ?, ?, 0); 423 | ", 424 | &[ 425 | &hash as &dyn ToSql, 426 | &file_id, 427 | &(offset as i64), 428 | &(size as i64), 429 | ], 430 | )?; 431 | Ok(()) 432 | } 433 | 434 | pub fn set_file_size_and_compute_blocks_hash( 435 | &mut self, 436 | file_id: u32, 437 | size: usize, 438 | ) -> Result<(), Error> { 439 | self.begin()?; 440 | 441 | let blocks_hash = self.compute_blocks_hash(file_id)?; 442 | self.db.execute( 443 | " 444 | UPDATE files 445 | SET size = ?, blocks_hash = ? 446 | WHERE file_id = ?; 447 | ", 448 | &[&(size as i64) as &dyn ToSql, &blocks_hash, &file_id], 449 | )?; 450 | Ok(()) 451 | } 452 | 453 | /// Get a list of all the blocks in a specific file 454 | pub fn list_file_blocks( 455 | &self, 456 | file_id: u32, 457 | ) -> Result, Error> { 458 | let mut stmt = self.db.prepare( 459 | " 460 | SELECT hash, offset, size 461 | FROM blocks 462 | WHERE file_id = ?; 463 | ", 464 | )?; 465 | let mut rows = stmt.query(&[file_id])?; 466 | let mut results = Vec::new(); 467 | loop { 468 | match rows.next() { 469 | Some(Ok(row)) => { 470 | let offset: i64 = row.get(1); 471 | let size: i64 = row.get(2); 472 | results.push((row.get(0), offset as usize, size as usize)) 473 | } 474 | Some(Err(e)) => return Err(e.into()), 475 | None => break, 476 | } 477 | } 478 | Ok(results) 479 | } 480 | 481 | /// Get a list of temporary files 482 | pub fn list_temp_files(&self) -> Result, Error> { 483 | let mut stmt = self.db.prepare( 484 | " 485 | SELECT name 486 | FROM files 487 | WHERE temporary = 1; 488 | ", 489 | )?; 490 | let mut rows = stmt.query(rusqlite::NO_PARAMS)?; 491 | let mut results = Vec::new(); 492 | loop { 493 | match rows.next() { 494 | Some(Ok(row)) => { 495 | let name: String = row.get(0); 496 | results.push(name.into()); 497 | } 498 | Some(Err(e)) => return Err(e.into()), 499 | None => break, 500 | } 501 | } 502 | Ok(results) 503 | } 504 | 505 | pub fn check_temp_files(&self) -> Result, Error> { 506 | let mut stmt = self.db.prepare( 507 | " 508 | SELECT 509 | file_id, name, 510 | EXISTS ( 511 | SELECT hash FROM blocks 512 | WHERE blocks.file_id = files.file_id 513 | AND present = 0 514 | ) AS missing 515 | FROM files 516 | WHERE temporary = 1; 517 | ", 518 | )?; 519 | let mut rows = stmt.query(rusqlite::NO_PARAMS)?; 520 | let mut results = Vec::new(); 521 | loop { 522 | match rows.next() { 523 | Some(Ok(row)) => { 524 | let file_id = row.get(0); 525 | let name: String = row.get(1); 526 | let missing_blocks = row.get(2); 527 | results.push((file_id, name.into(), missing_blocks)); 528 | } 529 | Some(Err(e)) => return Err(e.into()), 530 | None => break, 531 | } 532 | } 533 | Ok(results) 534 | } 535 | 536 | /// Get a list of blocks that are referenced by files but not present 537 | pub fn list_missing_blocks(&self) -> Result, Error> { 538 | let mut stmt = self.db.prepare( 539 | " 540 | SELECT hash 541 | FROM blocks 542 | WHERE present = 0; 543 | ", 544 | )?; 545 | let mut rows = stmt.query(rusqlite::NO_PARAMS)?; 546 | let mut results = Vec::new(); 547 | loop { 548 | match rows.next() { 549 | Some(Ok(row)) => { 550 | let hash: HashDigest = row.get(0); 551 | results.push(hash); 552 | }, 553 | Some(Err(e)) => return Err(e.into()), 554 | None => break, 555 | } 556 | } 557 | Ok(results) 558 | } 559 | 560 | /// Get all locations where a block is to be found 561 | pub fn list_block_locations( 562 | &self, 563 | hash: &HashDigest, 564 | ) -> Result, Error> { 565 | let mut stmt = self.db.prepare( 566 | " 567 | SELECT files.file_id, files.name, blocks.offset, blocks.size 568 | FROM blocks 569 | INNER JOIN files ON files.file_id = blocks.file_id 570 | WHERE hash = ?; 571 | ", 572 | )?; 573 | let mut rows = stmt.query(&[hash])?; 574 | let mut results = Vec::new(); 575 | loop { 576 | match rows.next() { 577 | Some(Ok(row)) => { 578 | let file_id = row.get(0); 579 | let name: String = row.get(1); 580 | let offset: i64 = row.get(2); 581 | let size: i64 = row.get(3); 582 | results.push((file_id, name.into(), offset as usize, size as usize)); 583 | } 584 | Some(Err(e)) => return Err(e.into()), 585 | None => break, 586 | } 587 | } 588 | Ok(results) 589 | } 590 | 591 | pub fn mark_block_present( 592 | &mut self, 593 | file_id: u32, 594 | hash: &HashDigest, 595 | offset: usize, 596 | ) -> Result<(), Error> { 597 | self.begin()?; 598 | self.db.execute( 599 | " 600 | UPDATE blocks 601 | SET present = 1 602 | WHERE file_id = ? AND hash = ? AND offset = ?; 603 | ", 604 | &[&file_id as &dyn ToSql, &hash, &(offset as i64)], 605 | )?; 606 | Ok(()) 607 | } 608 | 609 | /// Cut up a file into blocks and add them to the index 610 | pub fn index_file( 611 | &mut self, 612 | path: &Path, 613 | name: &Path, 614 | ) -> Result<(), Error> { 615 | let file = File::open(path)?; 616 | let (file_id, up_to_date) = self.add_file( 617 | name, 618 | file.metadata()?.modified()?.into(), 619 | )?; 620 | if !up_to_date { 621 | // Use ZPAQ to cut the stream into blocks 622 | let chunker = Chunker::new( 623 | ZPAQ::new(ZPAQ_BITS), // 13 bits = 8 KiB block average 624 | ).max_size(MAX_BLOCK_SIZE); 625 | let mut chunk_iterator = chunker.stream(file); 626 | let mut start_offset = 0; 627 | let mut offset = 0; 628 | let mut sha1 = Sha1::new(); 629 | while let Some(chunk) = chunk_iterator.read() { 630 | match chunk? { 631 | ChunkInput::Data(d) => { 632 | sha1.update(d); 633 | offset += d.len(); 634 | } 635 | ChunkInput::End => { 636 | let digest = HashDigest(sha1.digest().bytes()); 637 | let size = offset - start_offset; 638 | debug!( 639 | "Adding block, offset={}, size={}, sha1={}", 640 | start_offset, size, digest, 641 | ); 642 | self.add_block(&digest, file_id, start_offset, size)?; 643 | start_offset = offset; 644 | sha1.reset(); 645 | } 646 | } 647 | } 648 | 649 | // Set blocks_hash 650 | let blocks_digest = self.compute_blocks_hash(file_id)?; 651 | self.db.execute( 652 | " 653 | UPDATE files SET blocks_hash = ? WHERE file_id = ?; 654 | ", 655 | &[&blocks_digest as &dyn ToSql, &file_id], 656 | )?; 657 | } 658 | Ok(()) 659 | } 660 | 661 | fn compute_blocks_hash(&self, file_id: u32) -> Result { 662 | let mut sha1 = Sha1::new(); 663 | let mut stmt = self.db.prepare( 664 | " 665 | SELECT hash 666 | FROM blocks 667 | WHERE file_id = ?; 668 | ", 669 | )?; 670 | let mut rows = stmt.query(&[file_id])?; 671 | loop { 672 | match rows.next() { 673 | Some(Ok(row)) => { 674 | let digest: HashDigest = row.get(0); 675 | sha1.update(&digest.0); 676 | } 677 | Some(Err(e)) => return Err(e.into()), 678 | None => break, 679 | } 680 | } 681 | Ok(HashDigest(sha1.digest().bytes())) 682 | } 683 | 684 | /// Index files and directories recursively 685 | pub fn index_path(&mut self, path: &Path) -> Result<(), Error> { 686 | self.index_path_rec(path, Path::new("")) 687 | } 688 | 689 | fn index_path_rec( 690 | &mut self, 691 | root: &Path, 692 | rel: &Path, 693 | ) -> Result<(), Error> { 694 | let path = root.join(rel); 695 | if path.is_dir() { 696 | info!("Indexing directory {:?} ({:?})", rel, path); 697 | for entry in path.read_dir()? { 698 | if let Ok(entry) = entry { 699 | if entry.file_name() == ".syncfast.idx" { 700 | continue; 701 | } 702 | self.index_path_rec(root, &rel.join(entry.file_name()))?; 703 | } 704 | } 705 | Ok(()) 706 | } else { 707 | let rel = if rel.starts_with(".") { 708 | rel.strip_prefix(".").unwrap() 709 | } else { 710 | rel 711 | }; 712 | info!("Indexing file {:?} ({:?})", rel, path); 713 | self.index_file(&path, &rel) 714 | } 715 | } 716 | 717 | /// List all files and remove those that don't exist on disk 718 | pub fn remove_missing_files(&mut self, path: &Path) -> Result<(), Error> { 719 | for (file_id, file_path, _modified, _size, _blocks_hash) in self.list_files()? { 720 | if !path.join(&file_path).is_file() { 721 | info!("Removing missing file {:?}", file_path); 722 | self.remove_file(file_id)?; 723 | } 724 | } 725 | Ok(()) 726 | } 727 | 728 | /// Commit the transaction 729 | pub fn commit(&mut self) -> Result<(), rusqlite::Error> { 730 | if self.in_transaction { 731 | self.db.execute_batch("COMMIT")?; 732 | self.in_transaction = false; 733 | } 734 | Ok(()) 735 | } 736 | } 737 | 738 | #[cfg(test)] 739 | mod tests { 740 | use std::io::Write; 741 | use std::path::Path; 742 | use tempfile::NamedTempFile; 743 | 744 | use crate::HashDigest; 745 | use super::{Index, MAX_BLOCK_SIZE}; 746 | 747 | #[test] 748 | fn test() { 749 | let mut file = NamedTempFile::new().expect("tempfile"); 750 | for i in 0 .. 2000 { 751 | writeln!(file, "Line {}", i + 1).expect("tempfile"); 752 | } 753 | for _ in 0 .. 2000 { 754 | writeln!(file, "Test content").expect("tempfile"); 755 | } 756 | file.flush().expect("tempfile"); 757 | let name = Path::new("dir/name").to_path_buf(); 758 | let mut index = Index::open_in_memory().expect("db"); 759 | index.index_file(file.path(), &name).expect("index"); 760 | index.commit().expect("db"); 761 | assert!(index 762 | .get_block(&HashDigest(*b"12345678901234567890")) 763 | .expect("get") 764 | .is_none()); 765 | let block1 = index 766 | .get_block(&HashDigest( 767 | *b"\xfb\x5e\xf7\xeb\xad\xd8\x2c\x80\x85\xc5\ 768 | \xff\x63\x82\x36\x22\xba\xe0\xe2\x63\xf6", 769 | )) 770 | .expect("get"); 771 | assert_eq!(block1, Some((name.clone(), 0, 11579)),); 772 | let block2 = index 773 | .get_block(&HashDigest( 774 | *b"\x57\x0d\x8b\x30\xfc\xfd\x58\x5e\x41\x27\ 775 | \xb5\x61\xf5\xec\xd3\x76\xff\x4d\x01\x01", 776 | )) 777 | .expect("get"); 778 | assert_eq!(block2, Some((name.clone(), 11579, 32768)),); 779 | let block3 = index 780 | .get_block(&HashDigest( 781 | *b"\xb9\xa8\xc2\x64\x1a\xf2\xcf\x8f\xd8\xf3\ 782 | \x6a\x24\x56\xa3\xea\xa9\x5c\x02\x91\x27", 783 | )) 784 | .expect("get"); 785 | assert_eq!(block3, Some((name.clone(), 44347, 546)),); 786 | assert_eq!(block3.unwrap().1 - block2.unwrap().1, MAX_BLOCK_SIZE); 787 | let file1 = index.get_file(&name).expect("db").expect("get_file"); 788 | assert_eq!(file1.0, 1); 789 | assert_eq!(file1.2, HashDigest( 790 | *b"\x84\xC2\x5D\x78\xED\xCD\xB6\x76\x31\x63\ 791 | \x9C\x43\x60\x4C\xF0\x14\x95\x64\xF0\x44", 792 | )); 793 | } 794 | } 795 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Rsync-like library/program. 2 | //! 3 | //! syncfast is intended to provide the functionality of rsync ("live" transfer 4 | //! of files/directories over SSH), rdiff (creation of binary patches between 5 | //! files, for later application), and zsync (efficient synchronization of 6 | //! files or file trees from a central "dumb" HTTP server). It also has some 7 | //! additions such as caching file signatures to make repeated synchronizations 8 | //! faster. 9 | 10 | mod index; 11 | mod streaming_iterator; 12 | pub mod sync; 13 | 14 | use rusqlite::types::{FromSql, FromSqlError, ToSql, ToSqlOutput}; 15 | use std::ffi::OsString; 16 | use std::fmt; 17 | use std::io::Write; 18 | use std::path::{Path, PathBuf}; 19 | 20 | pub use index::Index; 21 | 22 | /// General error type for this library 23 | #[derive(Debug)] 24 | pub enum Error { 25 | Io(std::io::Error), 26 | Sqlite(rusqlite::Error), 27 | Protocol(Box), 28 | Sync(String), 29 | UnsupportedForLocation(&'static str), 30 | BadFilenameEncoding, 31 | } 32 | 33 | impl fmt::Display for Error { 34 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 35 | match self { 36 | Error::Sqlite(e) => write!(f, "SQLite error: {}", e), 37 | Error::Io(e) => write!(f, "I/O error: {}", e), 38 | Error::Protocol(e) => write!(f, "{}", e), 39 | Error::Sync(e) => write!(f, "{}", e), 40 | Error::UnsupportedForLocation(e) => write!(f, "{}", e), 41 | Error::BadFilenameEncoding => write!(f, "Bad filename encoding"), 42 | } 43 | } 44 | } 45 | 46 | impl std::error::Error for Error { 47 | fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { 48 | use std::ops::Deref; 49 | match *self { 50 | Error::Sqlite(ref e) => Some(e), 51 | Error::Io(ref e) => Some(e), 52 | Error::Protocol(ref e) => Some(e.deref()), 53 | Error::Sync(..) => None, 54 | Error::UnsupportedForLocation(..) => None, 55 | Error::BadFilenameEncoding => None, 56 | } 57 | } 58 | } 59 | 60 | impl From for Error { 61 | fn from(e: rusqlite::Error) -> Error { 62 | Error::Sqlite(e) 63 | } 64 | } 65 | 66 | impl From for Error { 67 | fn from(e: std::io::Error) -> Error { 68 | Error::Io(e) 69 | } 70 | } 71 | 72 | pub const HASH_DIGEST_LEN: usize = 20; 73 | 74 | /// Type for the hashes 75 | #[derive(Clone, Debug, PartialEq, Eq, Hash)] 76 | pub struct HashDigest([u8; HASH_DIGEST_LEN]); 77 | 78 | impl ToSql for HashDigest { 79 | fn to_sql(&self) -> Result { 80 | // Write the hash to buffer on the stack, we know the size 81 | let mut buffer = Vec::with_capacity(40); 82 | for byte in &self.0 { 83 | write!(&mut buffer, "{:02x}", byte).unwrap(); 84 | } 85 | // Hexadecimal chars are ASCII, cast to string 86 | let string = String::from_utf8(buffer).unwrap(); 87 | 88 | Ok(ToSqlOutput::from(string)) 89 | } 90 | } 91 | 92 | #[derive(Debug)] 93 | enum InvalidHashDigest { 94 | WrongSize, 95 | InvalidChar, 96 | } 97 | 98 | impl fmt::Display for InvalidHashDigest { 99 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 100 | match self { 101 | InvalidHashDigest::WrongSize => { 102 | write!(f, "Invalid hash: wrong size") 103 | } 104 | InvalidHashDigest::InvalidChar => { 105 | write!(f, "Invalid hash: invalid character") 106 | } 107 | } 108 | } 109 | } 110 | 111 | impl std::error::Error for InvalidHashDigest {} 112 | 113 | impl FromSql for HashDigest { 114 | fn column_result( 115 | value: rusqlite::types::ValueRef, 116 | ) -> Result { 117 | value.as_str().and_then(|s| { 118 | if s.len() != 40 { 119 | Err(FromSqlError::Other(Box::new( 120 | InvalidHashDigest::WrongSize, 121 | ))) 122 | } else { 123 | let mut bytes = [0u8; HASH_DIGEST_LEN]; 124 | for (i, byte) in (&mut bytes).iter_mut().enumerate() { 125 | *byte = u8::from_str_radix(&s[i * 2 .. i * 2 + 2], 16) 126 | .map_err(|_| { 127 | FromSqlError::Other(Box::new( 128 | InvalidHashDigest::InvalidChar, 129 | )) 130 | })?; 131 | } 132 | Ok(HashDigest(bytes)) 133 | } 134 | }) 135 | } 136 | } 137 | 138 | impl fmt::Display for HashDigest { 139 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 140 | for byte in &self.0 { 141 | write!(f, "{:02x}", byte)?; 142 | } 143 | Ok(()) 144 | } 145 | } 146 | 147 | fn temp_name(name: &Path) -> Result { 148 | let mut temp_path = PathBuf::new(); 149 | if let Some(parent) = name.parent() { 150 | temp_path.push(parent); 151 | } 152 | let mut temp_name: OsString = ".syncfast_tmp_".into(); 153 | temp_name.push(name.file_name().ok_or( 154 | std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid path"), 155 | )?); 156 | temp_path.push(temp_name); 157 | Ok(temp_path) 158 | } 159 | 160 | fn untemp_name(name: &Path) -> Result { 161 | let mut temp_path = PathBuf::new(); 162 | if let Some(parent) = name.parent() { 163 | temp_path.push(parent); 164 | } 165 | let temp_name = name.file_name().ok_or( 166 | std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid path"), 167 | )?; 168 | let temp_name = temp_name.to_str().ok_or(Error::BadFilenameEncoding)?; 169 | let stripped_name = temp_name.strip_prefix(".syncfast_tmp_").ok_or( 170 | std::io::Error::new(std::io::ErrorKind::InvalidInput, "Not a temporary path"), 171 | )?; 172 | temp_path.push(stripped_name); 173 | Ok(temp_path) 174 | } 175 | 176 | #[cfg(test)] 177 | mod tests { 178 | use rusqlite::types::{FromSql, ToSql, ToSqlOutput, Value, ValueRef}; 179 | use sha1::Sha1; 180 | use std::path::Path; 181 | 182 | use super::{HashDigest, temp_name}; 183 | 184 | #[test] 185 | fn test_hash_tosql() { 186 | let mut sha1 = Sha1::new(); 187 | sha1.update(b"test"); 188 | let digest = HashDigest(sha1.digest().bytes()); 189 | assert_eq!( 190 | digest.to_sql().unwrap(), 191 | ToSqlOutput::Owned(Value::Text( 192 | "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3".into() 193 | )), 194 | ); 195 | } 196 | 197 | #[test] 198 | fn test_hash_fromsql() { 199 | let mut sha1 = Sha1::new(); 200 | sha1.update(b"test"); 201 | let digest = HashDigest(sha1.digest().bytes()); 202 | 203 | let hash = ::column_result(ValueRef::Text( 204 | "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", 205 | )); 206 | assert_eq!(hash.unwrap(), digest); 207 | } 208 | 209 | #[test] 210 | fn test_temp_name() { 211 | assert_eq!(temp_name(Path::new("file")).unwrap(), Path::new(".syncfast_tmp_file")); 212 | assert_eq!(temp_name(Path::new("dir/file")).unwrap(), Path::new("dir/.syncfast_tmp_file")); 213 | } 214 | } 215 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | extern crate clap; 2 | extern crate env_logger; 3 | extern crate syncfast; 4 | 5 | use clap::{App, Arg, SubCommand}; 6 | use std::env; 7 | use std::path::Path; 8 | 9 | use syncfast::{Error, Index}; 10 | use syncfast::sync::do_sync; 11 | use syncfast::sync::locations::Location; 12 | use syncfast::sync::ssh::{stdio_destination, stdio_source}; 13 | 14 | /// Command-line entrypoint 15 | fn main() { 16 | // Parse command line 17 | let cli = App::new("syncfast") 18 | .bin_name("syncfast") 19 | .version(env!("CARGO_PKG_VERSION")) 20 | .author(env!("CARGO_PKG_AUTHORS")) 21 | .about(env!("CARGO_PKG_DESCRIPTION")) 22 | .arg( 23 | Arg::with_name("verbose") 24 | .short("v") 25 | .help("Augment verbosity (print more details)") 26 | .multiple(true), 27 | ) 28 | .subcommand( 29 | SubCommand::with_name("index") 30 | .about("Index a file or directory") 31 | .arg( 32 | Arg::with_name("path") 33 | .required(true) 34 | .takes_value(true), 35 | ) 36 | .arg( 37 | Arg::with_name("index-file") 38 | .short("x") 39 | .takes_value(true) 40 | .default_value(".syncfast.idx"), 41 | ), 42 | ) 43 | .subcommand( 44 | SubCommand::with_name("sync") 45 | .about("Copy files") 46 | .arg( 47 | Arg::with_name("source") 48 | .required(true) 49 | .takes_value(true), 50 | ) 51 | .arg( 52 | Arg::with_name("destination") 53 | .required(true) 54 | .takes_value(true), 55 | ), 56 | ) 57 | .subcommand( 58 | SubCommand::with_name("remote-recv") 59 | .about( 60 | "Internal - process started on the remote to receive \ 61 | files. Expects stdin and stdout to be connected to the \ 62 | sender process", 63 | ) 64 | .arg( 65 | Arg::with_name("destination") 66 | .required(true) 67 | .takes_value(true), 68 | ), 69 | ) 70 | .subcommand( 71 | SubCommand::with_name("remote-send") 72 | .about( 73 | "Internal - process started on the remote to send \ 74 | files. Expects stdin and stdout to be connected to \ 75 | the receiver process", 76 | ) 77 | .arg( 78 | Arg::with_name("source") 79 | .required(true) 80 | .takes_value(true), 81 | ), 82 | ); 83 | 84 | let mut cli = cli; 85 | let matches = match cli.get_matches_from_safe_borrow(env::args_os()) { 86 | Ok(m) => m, 87 | Err(e) => { 88 | e.exit(); 89 | } 90 | }; 91 | 92 | // Set up logging 93 | { 94 | let level = match matches.occurrences_of("verbose") { 95 | 0 => log::LevelFilter::Warn, 96 | 1 => log::LevelFilter::Info, 97 | 2 => log::LevelFilter::Debug, 98 | _ => log::LevelFilter::Trace, 99 | }; 100 | let mut logger_builder = env_logger::builder(); 101 | logger_builder.filter(None, level); 102 | if let Ok(val) = env::var("SYNCFAST_LOG") { 103 | logger_builder.parse_filters(&val); 104 | } 105 | if let Ok(val) = env::var("SYNCFAST_LOG_STYLE") { 106 | logger_builder.parse_write_style(&val); 107 | } 108 | logger_builder.init(); 109 | } 110 | 111 | let res = match matches.subcommand_name() { 112 | Some("index") => || -> Result<(), Error> { 113 | let s_matches = matches.subcommand_matches("index").unwrap(); 114 | let path = Path::new(s_matches.value_of_os("path").unwrap()); 115 | 116 | let mut index = match s_matches.value_of_os("index-file") { 117 | Some(p) => Index::open(Path::new(p))?, 118 | None => { 119 | Index::open(&path.join(".syncfast.idx"))? 120 | }, 121 | }; 122 | index.index_path(path)?; 123 | index.remove_missing_files(path)?; 124 | index.commit()?; 125 | 126 | Ok(()) 127 | }(), 128 | Some("sync") => { 129 | let s_matches = matches.subcommand_matches("sync").unwrap(); 130 | let source = s_matches.value_of_os("source").unwrap(); 131 | let dest = s_matches.value_of_os("destination").unwrap(); 132 | 133 | let source = match source.to_str().and_then(Location::parse) { 134 | Some(s) => s, 135 | None => { 136 | eprintln!("Invalid source"); 137 | std::process::exit(2); 138 | } 139 | }; 140 | let dest = match dest.to_str().and_then(Location::parse) { 141 | Some(Location::Http(_)) => { 142 | eprintln!("Can't write to HTTP destination, only read"); 143 | std::process::exit(2); 144 | } 145 | Some(s) => s, 146 | None => { 147 | eprintln!("Invalid destination"); 148 | std::process::exit(2); 149 | } 150 | }; 151 | 152 | let runtime = tokio::runtime::Builder::new_current_thread() 153 | .enable_all() 154 | .build() 155 | .unwrap(); 156 | runtime.block_on(async move { 157 | let source: syncfast::sync::Source = 158 | match source.open_source() { 159 | Ok(o) => o, 160 | Err(e) => { 161 | eprintln!("Failed to open source: {}", e); 162 | std::process::exit(1); 163 | } 164 | }; 165 | let destination: syncfast::sync::Destination = 166 | match dest.open_destination() { 167 | Ok(o) => o, 168 | Err(e) => { 169 | eprintln!("Failed to open destination: {}", e); 170 | std::process::exit(1); 171 | } 172 | }; 173 | do_sync(source, destination).await 174 | }) 175 | } 176 | Some("remote-send") => { 177 | let s_matches = matches.subcommand_matches("remote-send").unwrap(); 178 | let source = s_matches.value_of_os("source").unwrap(); 179 | 180 | let source = match source.to_str().and_then(Location::parse) { 181 | Some(s) => s, 182 | None => { 183 | eprintln!("Invalid source"); 184 | std::process::exit(2); 185 | } 186 | }; 187 | 188 | let runtime = tokio::runtime::Builder::new_current_thread() 189 | .enable_all() 190 | .build() 191 | .unwrap(); 192 | runtime.block_on(async move { 193 | let source: syncfast::sync::Source = 194 | match source.open_source() { 195 | Ok(o) => o, 196 | Err(e) => { 197 | eprintln!("Failed to open source: {}", e); 198 | std::process::exit(1); 199 | } 200 | }; 201 | let destination: syncfast::sync::Destination = 202 | stdio_destination(); 203 | do_sync(source, destination).await 204 | }) 205 | } 206 | Some("remote-recv") => { 207 | let s_matches = matches.subcommand_matches("remote-recv").unwrap(); 208 | let destination = s_matches.value_of_os("destination").unwrap(); 209 | 210 | let destination = match destination.to_str().and_then(Location::parse) { 211 | Some(s) => s, 212 | None => { 213 | eprintln!("Invalid source"); 214 | std::process::exit(2); 215 | } 216 | }; 217 | 218 | let runtime = tokio::runtime::Builder::new_current_thread() 219 | .enable_all() 220 | .build() 221 | .unwrap(); 222 | runtime.block_on(async move { 223 | let source: syncfast::sync::Source = 224 | stdio_source(); 225 | let destination: syncfast::sync::Destination = 226 | match destination.open_destination() { 227 | Ok(o) => o, 228 | Err(e) => { 229 | eprintln!("Failed to open destination: {}", e); 230 | std::process::exit(1); 231 | } 232 | }; 233 | do_sync(source, destination).await 234 | }) 235 | } 236 | _ => { 237 | cli.print_help().expect("Can't print help"); 238 | std::process::exit(2); 239 | } 240 | }; 241 | 242 | if let Err(e) = res { 243 | eprintln!("{}", e); 244 | std::process::exit(1); 245 | } 246 | } 247 | -------------------------------------------------------------------------------- /src/streaming_iterator.rs: -------------------------------------------------------------------------------- 1 | pub trait StreamingIterator<'a> { 2 | type Item: 'a; 3 | 4 | fn next(&'a mut self) -> Option; 5 | } 6 | -------------------------------------------------------------------------------- /src/sync/fs.rs: -------------------------------------------------------------------------------- 1 | //! Synchronization from and to local files. 2 | 3 | use cdchunking::{Chunker, ZPAQ}; 4 | use futures::channel::mpsc::{Receiver, channel}; 5 | use futures::sink::SinkExt; 6 | use futures::stream::StreamExt; 7 | use log::{log_enabled, debug, info}; 8 | use log::Level::Debug; 9 | use std::cell::RefCell; 10 | use std::collections::VecDeque; 11 | use std::ffi::OsString; 12 | use std::fs::{File, OpenOptions}; 13 | use std::future::Future; 14 | use std::io::{Seek, SeekFrom, Write}; 15 | use std::ops::DerefMut; 16 | use std::path::{Path, PathBuf}; 17 | use std::pin::Pin; 18 | use std::rc::Rc; 19 | use std::string::FromUtf8Error; 20 | 21 | use crate::{Error, HashDigest, temp_name, untemp_name}; 22 | use crate::index::{MAX_BLOCK_SIZE, ZPAQ_BITS, Index}; 23 | use crate::sync::{Destination, DestinationEvent, Source, SourceEvent}; 24 | use crate::sync::utils::{Condition, ConditionFuture, move_file}; 25 | 26 | fn read_block(path: &Path, offset: usize) -> Result, Error> { 27 | let mut file = File::open(path)?; 28 | file.seek(SeekFrom::Start(offset as u64))?; 29 | let chunker = Chunker::new( 30 | ZPAQ::new(ZPAQ_BITS), 31 | ).max_size(MAX_BLOCK_SIZE); 32 | let block = chunker.whole_chunks(file).next() 33 | .unwrap_or(Err( 34 | std::io::Error::new( 35 | std::io::ErrorKind::InvalidData, 36 | "No such chunk in file", 37 | ), 38 | ))?; 39 | Ok(block) 40 | } 41 | 42 | fn write_block( 43 | name: &Path, 44 | offset: usize, 45 | block: &[u8], 46 | ) -> Result<(), Error> { 47 | let mut file = OpenOptions::new().write(true).create(true).open(name)?; 48 | file.seek(SeekFrom::Start(offset as u64))?; 49 | file.write_all(block)?; 50 | Ok(()) 51 | } 52 | 53 | pub fn fs_source(root_dir: PathBuf) -> Result { 54 | info!("Indexing source into {:?}...", root_dir.join(".syncfast.idx")); 55 | let mut index = Index::open(&root_dir.join(".syncfast.idx"))?; 56 | index.index_path(&root_dir)?; 57 | index.remove_missing_files(&root_dir)?; 58 | index.commit()?; 59 | 60 | // The source can't handle multiple input events, so we just implement 61 | // a Stream, and use a channel for the Sink 62 | debug!("FsSource: state=ListFiles"); 63 | let (sender, receiver) = channel(1); 64 | Ok(Source { 65 | // Stream generating events using FsSourceFrom::stream 66 | stream: futures::stream::unfold( 67 | Box::pin(FsSourceFrom { 68 | index, 69 | root_dir, 70 | receiver, 71 | state: FsSourceState::ListFiles(None), 72 | }), 73 | FsSourceFrom::stream, 74 | ).boxed_local(), 75 | // Simple Sink feeding the channel for the Stream to read 76 | sink: Box::pin(futures::sink::unfold((), move |(), event: DestinationEvent| { 77 | let mut sender = sender.clone(); 78 | async move { 79 | sender.send(event).await.map_err(|_| Error::Io(std::io::Error::new(std::io::ErrorKind::BrokenPipe, "FsSource channel is closed"))) 80 | } 81 | })), 82 | }) 83 | } 84 | 85 | enum FsSourceState { 86 | ListFiles(Option, usize, HashDigest)>>), 87 | Respond, 88 | ListBlocks(VecDeque<(HashDigest, usize)>), 89 | Done, 90 | } 91 | 92 | struct FsSourceFrom { 93 | index: Index, 94 | root_dir: PathBuf, 95 | receiver: Receiver, 96 | state: FsSourceState, 97 | } 98 | 99 | impl FsSourceFrom { 100 | fn project<'b>(self: &'b mut Pin>) -> (&'b mut Index, &'b Path, Pin<&'b mut Receiver>, &'b mut FsSourceState) { 101 | unsafe { // Required for pin projection 102 | let s = self.as_mut().get_unchecked_mut(); 103 | ( 104 | &mut s.index, 105 | &mut s.root_dir, 106 | Pin::new_unchecked(&mut s.receiver), 107 | &mut s.state, 108 | ) 109 | } 110 | } 111 | 112 | fn stream(mut stream: Pin>) -> impl Future, Pin>)>> { 113 | async { 114 | let (index, root_dir, mut receiver, state) = stream.project(); 115 | 116 | macro_rules! err { 117 | ($e:expr) => { 118 | Some((Err($e), stream)) 119 | } 120 | } 121 | // FIXME: Replace by try_block when supported by Rust 122 | macro_rules! try_ { 123 | ($v:expr) => { 124 | match $v { 125 | Ok(r) => r, 126 | Err(e) => return err!(e), 127 | } 128 | } 129 | } 130 | 131 | match *state { 132 | // Send files list 133 | FsSourceState::ListFiles(ref mut list) => { 134 | // If we don't have data, fetch from database 135 | if list.is_none() { 136 | // FIXME: Don't get all files at once, iterate 137 | let files = try_!(index.list_files()); 138 | let mut new_list = VecDeque::with_capacity(files.len()); 139 | for (_file_id, path, _modified, size, blocks_hash) in files { 140 | let path = path 141 | .into_os_string() 142 | .into_string(); 143 | let path = try_!(path.map_err(|_: OsString| Error::BadFilenameEncoding)); 144 | let path = path.into_bytes(); 145 | new_list.push_back((path, size as usize, blocks_hash)); 146 | } 147 | debug!("FsSource: preparing to send {} files", new_list.len()); 148 | *list = Some(new_list); 149 | } 150 | let list = list.as_mut().unwrap(); 151 | match list.pop_front() { 152 | Some((path, size, blocks_hash)) => { 153 | if log_enabled!(Debug) { 154 | debug!("FsSource: send FileEntry({})", String::from_utf8_lossy(&path)); 155 | } 156 | Some((Ok(SourceEvent::FileEntry(path, size, blocks_hash)), stream)) 157 | } 158 | None => { 159 | debug!("FsSource: state=Respond"); 160 | *state = FsSourceState::Respond; 161 | debug!("FsSource: send EndFiles"); 162 | Some((Ok(SourceEvent::EndFiles), stream)) 163 | } 164 | } 165 | } 166 | // Files are sent, respond to requests 167 | FsSourceState::Respond => { 168 | let req = match receiver.as_mut().next().await { 169 | None => { 170 | debug!("FsSource: got end of input"); 171 | return None; 172 | } 173 | Some(e) => e, 174 | }; 175 | debug!("FsSource: recv {:?}", req); 176 | match req { 177 | DestinationEvent::GetFile(path) => { 178 | let path_str = try_!( 179 | String::from_utf8(path) 180 | .map_err(|_: FromUtf8Error| Error::BadFilenameEncoding) 181 | ); 182 | let (file_id, _modified, _blocks_hash) = match try_!(index.get_file(Path::new(&path_str))) { 183 | Some(t) => t, 184 | None => return err!(Error::Sync("Requested file is unknown".to_owned())), 185 | }; 186 | debug!("FsSource: file_id={}", file_id); 187 | // FIXME: Don't get all blocks at once, iterate 188 | let blocks = try_!(index.list_file_blocks(file_id)); 189 | let mut new_blocks = VecDeque::with_capacity(blocks.len()); 190 | for (hash, _offset, size) in blocks { 191 | new_blocks.push_back((hash, size)); 192 | } 193 | debug!("FsSource: state=ListBlocks"); 194 | debug!("FsSource: preparing to send {} blocks", new_blocks.len()); 195 | *state = FsSourceState::ListBlocks(new_blocks); 196 | debug!("FsSource: send FileStart"); 197 | Some((Ok(SourceEvent::FileStart(path_str.into_bytes())), stream)) 198 | } 199 | DestinationEvent::GetBlock(hash) => { 200 | let (path, offset, _size) = match try_!(index.get_block(&hash)) { 201 | Some(t) => t, 202 | None => return err!(Error::Sync("Requested block is unknown".to_owned())), 203 | }; 204 | debug!("FsSource: found block in {:?} offset {}", path, offset); 205 | let data = try_!(read_block(&root_dir.join(&path), offset)); 206 | debug!("FsSource: send BlockData"); 207 | Some((Ok(SourceEvent::BlockData(hash, data)), stream)) 208 | } 209 | DestinationEvent::Complete => { 210 | *state = FsSourceState::Done; 211 | debug!("FsSource: state=Done"); 212 | None 213 | } 214 | } 215 | } 216 | // List blocks 217 | FsSourceState::ListBlocks(ref mut list) => { 218 | match list.pop_front() { 219 | Some((hash, size)) => { 220 | debug!("FsSource: send FileBlock"); 221 | Some((Ok(SourceEvent::FileBlock(hash, size)), stream)) 222 | } 223 | None => { 224 | debug!("FsSource: out of blocks"); 225 | debug!("FsSource: state=Respond"); 226 | *state = FsSourceState::Respond; 227 | debug!("FsSource: send FileEnd"); 228 | Some((Ok(SourceEvent::FileEnd), stream)) 229 | } 230 | } 231 | } 232 | // Stream is done 233 | FsSourceState::Done => None, 234 | } 235 | } 236 | } 237 | } 238 | 239 | pub fn fs_destination(root_dir: PathBuf) -> Result { 240 | info!( 241 | "Indexing destination into {:?}...", 242 | root_dir.join(".syncfast.idx") 243 | ); 244 | std::fs::create_dir_all(&root_dir)?; 245 | let mut index = Index::open(&root_dir.join(".syncfast.idx"))?; 246 | index.index_path(&root_dir)?; 247 | index.remove_missing_files(&root_dir)?; 248 | index.commit()?; 249 | 250 | // The destination has to handle input while producing output (for 251 | // example getting BlockData while sending GetBlock), so it has both a 252 | // custom Stream and Sink implementations 253 | // State changes are triggered by Sink 254 | let destination = Rc::new(RefCell::new(FsDestinationInner { 255 | index, 256 | root_dir, 257 | state: FsDestinationState::FilesList { cond: Default::default() }, 258 | })); 259 | debug!("FsDestination: state=FilesList"); 260 | Ok(Destination { 261 | // Stream generating events using FsDestination::stream 262 | stream: futures::stream::unfold( 263 | destination.clone(), 264 | FsDestinationInner::stream, 265 | ).boxed_local(), 266 | // Sink handling events using FsDestination::sink 267 | sink: Box::pin(futures::sink::unfold( 268 | destination, 269 | FsDestinationInner::sink, 270 | )), 271 | }) 272 | } 273 | 274 | struct FsDestinationInner { 275 | index: Index, 276 | root_dir: PathBuf, 277 | state: FsDestinationState, 278 | } 279 | 280 | enum FsDestinationState { 281 | FilesList { 282 | /// Sink indicates state change (`SourceEvent::EndFiles`) 283 | cond: Condition, 284 | }, 285 | GetFiles { 286 | /// List of files to request the blocks of 287 | files_to_request: VecDeque>, 288 | /// Number of files to receive 289 | files_to_receive: usize, 290 | /// Sink indicates state change (got `SourceEvent::FileEnd` and no more files_to_request) 291 | cond: Condition, 292 | /// file_id and offset for the blocks we're receiving (from previous FileStart) 293 | file_blocks_id: Option<(u32, usize)>, 294 | }, 295 | GetBlocks { 296 | /// List of blocks to request, None if we've sent `DestinationEvent::Complete` 297 | blocks_to_request: Option>, 298 | /// Number of blocks to receive 299 | blocks_to_receive: usize, 300 | }, 301 | } 302 | 303 | impl FsDestinationInner { 304 | fn stream(inner: Rc>) -> impl Future, Rc>)>> { 305 | async move { 306 | loop { 307 | // This works around borrow issue: have to do stuff after inner.borrow_mut() ends 308 | enum WhatToDo { 309 | Wait(ConditionFuture), 310 | Return(DestinationEvent), 311 | } 312 | let what_to_do = match inner.borrow_mut().state { 313 | // Receive files list 314 | FsDestinationState::FilesList { ref mut cond } => { 315 | // Nothing to produce, wait for state change 316 | WhatToDo::Wait(cond.wait()) 317 | } 318 | // Request blocks for files 319 | FsDestinationState::GetFiles { ref mut files_to_request, ref mut cond, .. } => { 320 | match files_to_request.pop_front() { 321 | Some(name) => { 322 | if log_enabled!(Debug) { 323 | debug!("FsDestination::stream: send GetFile({:?})", String::from_utf8_lossy(&name)); 324 | } 325 | WhatToDo::Return(DestinationEvent::GetFile(name)) 326 | } 327 | None => { 328 | debug!("FsDestination::stream: no more files, waiting..."); 329 | WhatToDo::Wait(cond.wait()) 330 | } 331 | } 332 | } 333 | // Request block data 334 | FsDestinationState::GetBlocks { ref mut blocks_to_request, .. } => { 335 | match blocks_to_request { 336 | Some(ref mut l) => match l.pop_front() { 337 | Some(hash) => { 338 | debug!("FsDestination::stream: send GetBlock({})", hash); 339 | WhatToDo::Return(DestinationEvent::GetBlock(hash)) 340 | } 341 | None => { 342 | debug!("FsDestination::stream: no more blocks, send Complete"); 343 | *blocks_to_request = None; 344 | WhatToDo::Return(DestinationEvent::Complete) 345 | } 346 | } 347 | None => { 348 | debug!("FsDestination::stream: done"); 349 | return None; 350 | } 351 | } 352 | } 353 | }; 354 | match what_to_do { 355 | WhatToDo::Wait(cond) => cond.await, 356 | WhatToDo::Return(r) => return Some((Ok(r), inner)), 357 | } 358 | } 359 | } 360 | } 361 | 362 | fn sink(inner: Rc>, event: SourceEvent) -> impl Future>, Error>> { 363 | async move { 364 | { 365 | let mut inner_: std::cell::RefMut = inner.borrow_mut(); 366 | let inner_: &mut FsDestinationInner = inner_.deref_mut(); 367 | 368 | // Can't mutably borrow more than once 369 | let mut new_state: Option = None; 370 | let state = &mut inner_.state; 371 | let index = &mut inner_.index; 372 | let root_dir = &inner_.root_dir; 373 | 374 | debug!("FsDestination::sink: recv {:?}", event); 375 | 376 | match state { 377 | // Receive files list 378 | FsDestinationState::FilesList { ref mut cond } => { 379 | match event { 380 | SourceEvent::FileEntry(path, _size, blocks_hash) => { 381 | let path: PathBuf = String::from_utf8(path) 382 | .map_err(|_: FromUtf8Error| Error::BadFilenameEncoding)? 383 | .into(); 384 | let file = inner_.index.get_file(&path)?; 385 | let add = match file { 386 | Some((_file_id, _modified, recorded_blocks_hash)) => { 387 | if blocks_hash == recorded_blocks_hash { 388 | debug!("FsDestination::sink: file's blocks_hash matches"); 389 | false // File is up to date, do nothing 390 | } else { 391 | debug!("FsDestination::sink: file exists but blocks_hash differs"); 392 | true 393 | } 394 | } 395 | None => { 396 | debug!("FsDestination::sink: file doesn't exist"); 397 | true 398 | } 399 | }; 400 | if add { 401 | // Create temporary file 402 | inner_.index.add_temp_file(&path)?; 403 | let temp_path = inner_.root_dir.join(temp_name(&path)?); 404 | debug!("FsDestination::sink: creating temp file {:?}", temp_path); 405 | if let Some(parent) = temp_path.parent() { 406 | std::fs::create_dir_all(parent)?; 407 | } 408 | OpenOptions::new() 409 | .write(true) 410 | .truncate(true) 411 | .create(true) 412 | .open(temp_path)?; 413 | } 414 | } 415 | SourceEvent::EndFiles => { 416 | // FIXME: Don't get all files at once, iterate 417 | let mut files_to_request = VecDeque::new(); 418 | for name in index.list_temp_files()? { 419 | let name = untemp_name(&name)?; 420 | let name = name 421 | .into_os_string() 422 | .into_string() 423 | .map_err(|_: OsString| Error::BadFilenameEncoding)? 424 | .into_bytes(); 425 | files_to_request.push_back(name); 426 | } 427 | if !files_to_request.is_empty() { 428 | let files_to_receive = files_to_request.len(); 429 | debug!("FsDestination::sink: state=GetFiles({} files)", files_to_receive); 430 | new_state = Some(FsDestinationState::GetFiles { 431 | files_to_request, 432 | files_to_receive, 433 | cond: Default::default(), 434 | file_blocks_id: None, 435 | }); 436 | } else { 437 | debug!("FsDestination::sink: state=GetBlocks(0 blocks)"); 438 | new_state = Some(FsDestinationState::GetBlocks { 439 | blocks_to_request: Some(VecDeque::new()), 440 | blocks_to_receive: 0, 441 | }); 442 | } 443 | cond.set(); 444 | } 445 | _ => return Err(Error::Sync("Unexpected message from source".to_owned())), 446 | } 447 | } 448 | // Receive blocks for files 449 | FsDestinationState::GetFiles { ref mut cond, ref mut file_blocks_id, ref mut files_to_receive, .. } => { 450 | *file_blocks_id = match (*file_blocks_id, event) { 451 | (None, SourceEvent::FileStart(path)) => { 452 | let path: PathBuf = String::from_utf8(path) 453 | .map_err(|_: FromUtf8Error| Error::BadFilenameEncoding)? 454 | .into(); 455 | let (file_id, _modified) = index.get_temp_file(&path)? 456 | .ok_or(Error::Sync(format!("Unknown file {:?}", path)))?; 457 | Some((file_id, 0)) 458 | } 459 | // FIXME: Don't need to capture all of them by ref, 460 | // but necessary for Rust 1.45 461 | (Some((file_id, offset)), SourceEvent::FileBlock(ref hash, ref size)) => { 462 | // See if we have this block, to copy it right now 463 | match index.get_block(&hash)? { 464 | Some((from_path, from_offset, _from_size)) => { 465 | let path = index.get_file_name(file_id)?; 466 | let path = path.ok_or(std::io::Error::new(std::io::ErrorKind::NotFound, "File gone from index during sync"))?; 467 | debug!("FsDestination::sink: Copying block from {:?} offset {:?}", from_path, from_offset); 468 | let block = read_block(&root_dir.join(&from_path), from_offset)?; 469 | write_block(&root_dir.join(&path), offset, &block)?; 470 | index.add_block(&hash, file_id, offset, *size)?; 471 | } 472 | None => { 473 | debug!("FsDestination::sink: Don't know that block"); 474 | index.add_missing_block(&hash, file_id, offset, *size)?; 475 | } 476 | } 477 | Some((file_id, offset + size)) 478 | } 479 | (Some((file_id, offset)), SourceEvent::FileEnd) => { 480 | index.set_file_size_and_compute_blocks_hash(file_id, offset)?; 481 | *files_to_receive -= 1; 482 | debug!("FsDestination::sink: {} files left to receive", *files_to_receive); 483 | if *files_to_receive == 0 { 484 | // FIXME: Don't get all files at once, iterate 485 | let mut blocks_to_request = VecDeque::new(); 486 | for hash in index.list_missing_blocks()? { 487 | blocks_to_request.push_back(hash); 488 | } 489 | let blocks_to_receive = blocks_to_request.len(); 490 | debug!("FsDestination::sink: state=GetBlocks({} blocks)", blocks_to_receive); 491 | new_state = Some(FsDestinationState::GetBlocks { 492 | blocks_to_request: Some(blocks_to_request), 493 | blocks_to_receive, 494 | }); 495 | cond.set(); 496 | } 497 | None 498 | } 499 | _ => return Err(Error::Sync("Unexpected message from source".to_owned())), 500 | } 501 | } 502 | // Receiving block data 503 | FsDestinationState::GetBlocks { ref mut blocks_to_receive, .. } => { 504 | match event { 505 | SourceEvent::BlockData(hash, data) => { 506 | for (file_id, name, offset, _size) in index.list_block_locations(&hash)? { 507 | debug!("FsDestination::sink: writing block to {:?} offset {}", name, offset); 508 | write_block(&root_dir.join(&name), offset, &data)?; 509 | index.mark_block_present(file_id, &hash, offset)?; 510 | } 511 | *blocks_to_receive -= 1; 512 | debug!("FsDestination::sink: {} blocks left to receive", *blocks_to_receive); 513 | if *blocks_to_receive == 0 { 514 | Self::finish(root_dir, index)?; 515 | } 516 | } 517 | _ => return Err(Error::Sync("Unexpected message from source".to_owned())), 518 | } 519 | } 520 | } 521 | if let Some(s) = new_state { 522 | *state = s; 523 | } 524 | } 525 | Ok(inner) 526 | } 527 | } 528 | 529 | fn finish(root_dir: &Path, index: &mut Index) -> Result<(), Error> { 530 | for (file_id, name, missing_blocks) in index.check_temp_files()? { 531 | if missing_blocks { 532 | return Err(Error::Sync( 533 | format!("Missing blocks in file {:?}", name), 534 | )); 535 | } 536 | 537 | let final_name = untemp_name(&name)?; 538 | debug!("FsDestination: moving {:?} to {:?}", name, final_name); 539 | 540 | // Rename temporary file into destination 541 | move_file(&root_dir.join(name), &root_dir.join(&final_name))?; 542 | 543 | // Update index 544 | index.move_temp_file_into_place(file_id, &final_name)?; 545 | } 546 | index.commit()?; 547 | Ok(()) 548 | } 549 | } 550 | -------------------------------------------------------------------------------- /src/sync/locations.rs: -------------------------------------------------------------------------------- 1 | //! File locations that we can sync from/to. 2 | 3 | use std::path::PathBuf; 4 | 5 | use crate::Error; 6 | use crate::sync::{Destination, Source}; 7 | use crate::sync::fs::{fs_destination, fs_source}; 8 | use crate::sync::ssh::{ssh_destination, ssh_source}; 9 | 10 | /// SSH remote path, with user and host 11 | #[derive(Clone, Debug, PartialEq, Eq, Hash)] 12 | pub struct SshLocation { 13 | /// Optional user name. If omitted, local user will be used. 14 | pub user: Option, 15 | /// Remote host name 16 | pub host: String, 17 | /// Path on the remote machine (may be relative to home) 18 | pub path: String, 19 | } 20 | 21 | /// A location, possible remote, that can be specified by the user 22 | #[derive(Clone, Debug, PartialEq, Eq, Hash)] 23 | pub enum Location { 24 | /// A path on the local machine 25 | Local(PathBuf), 26 | /// Remote directory accessible via SSH 27 | Ssh(SshLocation), 28 | /// Remote HTTP server 29 | Http(String), 30 | } 31 | 32 | impl Location { 33 | /// Parse a string into a location 34 | pub fn parse(s: &str) -> Option { 35 | if s.starts_with("http://") || s.starts_with("https://") { 36 | Some(Location::Http(s.into())) 37 | } else if s.starts_with("ssh://") { 38 | let idx_slash = match s[6 ..].find('/') { 39 | Some(i) => i + 6, 40 | None => return None, 41 | }; 42 | let (user, host) = match s[6 ..].find('@') { 43 | Some(idx_at) if idx_at + 6 < idx_slash => { 44 | let idx_at = idx_at + 6; 45 | (Some(&s[6 .. idx_at]), &s[idx_at + 1 .. idx_slash]) 46 | } 47 | _ => (None, &s[6 .. idx_slash]), 48 | }; 49 | let path = &s[idx_slash ..]; 50 | 51 | Some(Location::Ssh(SshLocation { 52 | user: user.map(Into::into), 53 | host: host.into(), 54 | path: path.into(), 55 | })) 56 | } else if s.starts_with("file:///") { 57 | // FIXME: Unquote path? 58 | Some(Location::Local(s[7 ..].into())) 59 | } else { 60 | // Return None if starts with [a-z]+:/ 61 | for (i, c) in s.char_indices() { 62 | if c == ':' { 63 | if i > 0 && &s[i + 1 .. i + 2] == "/" { 64 | return None; 65 | } 66 | } else if !c.is_ascii_alphabetic() { 67 | break; 68 | } 69 | } 70 | 71 | Some(Location::Local(s.into())) 72 | } 73 | } 74 | 75 | /// Create a `Destination` to sync to this location 76 | pub fn open_destination(&self) -> Result { 77 | let w: Destination = match self { 78 | Location::Local(path) => fs_destination(path.to_owned())?, 79 | Location::Ssh(ssh) => ssh_destination(ssh)?, 80 | Location::Http(_url) => { 81 | // Shouldn't happen, caught in main.rs 82 | return Err(Error::UnsupportedForLocation("Can't write to HTTP location")); 83 | } 84 | }; 85 | Ok(w) 86 | } 87 | 88 | /// Create a `Source` to sync from this location 89 | pub fn open_source(&self) -> Result { 90 | let w: Source = match self { 91 | Location::Local(path) => fs_source(path.to_owned())?, 92 | Location::Ssh(ssh) => ssh_source(ssh)?, 93 | Location::Http(_url) => unimplemented!(), // TODO: HTTP 94 | }; 95 | Ok(w) 96 | } 97 | } 98 | 99 | #[cfg(test)] 100 | mod tests { 101 | use super::{Location, SshLocation}; 102 | 103 | #[test] 104 | fn test_parse() { 105 | assert_eq!( 106 | Location::parse("http://example.org/"), 107 | Some(Location::Http("http://example.org/".into())), 108 | ); 109 | assert_eq!( 110 | Location::parse("some/local/path"), 111 | Some(Location::Local("some/local/path".into())), 112 | ); 113 | assert_eq!(Location::parse("scheme:/local/path"), None); 114 | assert_eq!( 115 | Location::parse("not-scheme://local/path"), 116 | Some(Location::Local("not-scheme://local/path".into())), 117 | ); 118 | assert_eq!( 119 | Location::parse("notscheme:local/path"), 120 | Some(Location::Local("notscheme:local/path".into())), 121 | ); 122 | assert_eq!( 123 | Location::parse("file:///home/ubuntu/file"), 124 | Some(Location::Local("/home/ubuntu/file".into())), 125 | ); 126 | assert_eq!(Location::parse("file://file"), None); 127 | assert_eq!( 128 | Location::parse("ssh://user@host/path"), 129 | Some(Location::Ssh(SshLocation { 130 | user: Some("user".into()), 131 | host: "host".into(), 132 | path: "/path".into(), 133 | })), 134 | ); 135 | assert_eq!( 136 | Location::parse("ssh://host/"), 137 | Some(Location::Ssh(SshLocation { 138 | user: None, 139 | host: "host".into(), 140 | path: "/".into(), 141 | })), 142 | ); 143 | assert_eq!(Location::parse("ssh://host"), None); 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /src/sync/mod.rs: -------------------------------------------------------------------------------- 1 | //! This module contains the transfer protocol handlers. 2 | 3 | pub mod fs; 4 | pub mod locations; 5 | pub mod ssh; 6 | mod utils; 7 | 8 | use log::info; 9 | use futures::join; 10 | use futures::sink::Sink; 11 | use futures::stream::{LocalBoxStream, StreamExt}; 12 | use std::pin::Pin; 13 | 14 | use crate::{Error, HashDigest}; 15 | 16 | pub enum SourceEvent { 17 | FileEntry(Vec, usize, HashDigest), 18 | EndFiles, 19 | FileStart(Vec), 20 | FileBlock(HashDigest, usize), 21 | FileEnd, 22 | BlockData(HashDigest, Vec), 23 | } 24 | 25 | impl std::fmt::Debug for SourceEvent { 26 | fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { 27 | match self { 28 | &SourceEvent::FileEntry(ref path, size, ref hash) => write!( 29 | f, 30 | "FileEntry({}, {}, {})", 31 | String::from_utf8_lossy(&path), 32 | size, 33 | hash, 34 | ), 35 | &SourceEvent::EndFiles => write!(f, "EndFiles"), 36 | &SourceEvent::FileStart(ref path) => write!( 37 | f, 38 | "FileStart({})", 39 | String::from_utf8_lossy(&path), 40 | ), 41 | &SourceEvent::FileBlock(ref hash, size) => write!( 42 | f, 43 | "FileBlock({}, {})", 44 | hash, 45 | size, 46 | ), 47 | &SourceEvent::FileEnd => write!(f, "FileEnd"), 48 | &SourceEvent::BlockData(ref hash, ref data) => write!( 49 | f, 50 | "BlockData({}, <{} bytes>)", 51 | hash, 52 | data.len(), 53 | ), 54 | } 55 | } 56 | } 57 | 58 | pub enum DestinationEvent { 59 | GetFile(Vec), 60 | GetBlock(HashDigest), 61 | Complete, 62 | } 63 | 64 | impl std::fmt::Debug for DestinationEvent { 65 | fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { 66 | match self { 67 | &DestinationEvent::GetFile(ref path) => write!( 68 | f, 69 | "GetFile({})", 70 | String::from_utf8_lossy(&path), 71 | ), 72 | &DestinationEvent::GetBlock(ref hash) => write!(f, "GetBlock({})", hash), 73 | &DestinationEvent::Complete => write!(f, "Complete"), 74 | } 75 | } 76 | } 77 | 78 | /// The source, representing where the files are coming from. 79 | /// 80 | /// This is relative to a single process, e.g. the sending side has a source 81 | /// that reads from files, and the receiving side has a source that reads from 82 | /// the network. 83 | pub struct Source { 84 | stream: LocalBoxStream<'static, Result>, 85 | sink: Pin>>, 86 | } 87 | 88 | /// The destination, representing where the files are being sent. 89 | /// 90 | /// This is relative to a single process, e.g. the sending side has a 91 | /// destination encapsulating some network protocol, and the receiving side has 92 | /// a destination that actually updates files. 93 | pub struct Destination { 94 | stream: LocalBoxStream<'static, Result>, 95 | sink: Pin>>, 96 | } 97 | 98 | pub async fn do_sync( 99 | source: Source, 100 | destination: Destination, 101 | ) -> Result<(), Error> { 102 | info!("Starting sync..."); 103 | let Source { stream: source_from, sink: source_to } = source; 104 | let Destination { stream: destination_from, sink: destination_to } = destination; 105 | info!("Streams opened"); 106 | 107 | // Concurrently forward streams into sinks 108 | let (r1, r2) = join!( 109 | source_from.forward(destination_to), 110 | destination_from.forward(source_to), 111 | ); 112 | r1?; 113 | r2?; 114 | info!("Sync complete"); 115 | 116 | Ok(()) 117 | } 118 | -------------------------------------------------------------------------------- /src/sync/ssh/mod.rs: -------------------------------------------------------------------------------- 1 | mod proto; 2 | 3 | use futures::stream::StreamExt; 4 | use log::{debug, info}; 5 | use std::collections::VecDeque; 6 | use std::convert::{TryFrom, TryInto}; 7 | use std::fmt::Debug; 8 | use std::future::Future; 9 | use std::pin::Pin; 10 | use std::process::Stdio; 11 | use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, stdin, stdout}; 12 | use tokio::process::{Child, Command}; 13 | 14 | use crate::Error; 15 | use crate::streaming_iterator::StreamingIterator; 16 | use crate::sync::{Destination, Source}; 17 | use crate::sync::locations::SshLocation; 18 | use crate::sync::ssh::proto::{OwnedMessage, Parser, write_message}; 19 | 20 | fn shell_escape(input: &str) -> String { 21 | let mut result = String::new(); 22 | result.push('"'); 23 | for c in input.chars() { 24 | if c == '\\' || c == '"' { 25 | result.push('\\'); 26 | } 27 | result.push(c); 28 | } 29 | result.push('"'); 30 | result 31 | } 32 | 33 | // First we define the SshStream and SshSink structs, which can read and write 34 | // messages to/from a process. 35 | // Then we implement SshSource and SshDestination, which run `remote-send` and 36 | // `remote-recv` and use SshStream and SshSink to do all the messaging. 37 | 38 | struct SshStream { 39 | stdout: R, 40 | parser: Parser, 41 | messages: VecDeque, 42 | } 43 | 44 | impl SshStream { 45 | fn new(stdout: R) -> SshStream { 46 | SshStream { 47 | stdout, 48 | parser: Default::default(), 49 | messages: VecDeque::new(), 50 | } 51 | } 52 | 53 | fn project<'b>(self: &'b mut Pin>>) -> (&'b mut R, &'b mut Parser, &'b mut VecDeque) where R: 'b { 54 | unsafe { 55 | let s = self.as_mut().get_unchecked_mut(); 56 | (&mut s.stdout, &mut s.parser, &mut s.messages) 57 | } 58 | } 59 | 60 | fn stream + Debug>(mut arg: Pin>>) -> impl Future, Pin>>)>> { 61 | async move { 62 | let (mut stream, parser, messages) = arg.project(); 63 | 64 | macro_rules! err { 65 | ($e:expr) => { 66 | Some((Err($e.into()), arg)) 67 | } 68 | } 69 | // FIXME: Replace by try_block when supported by Rust 70 | macro_rules! try_ { 71 | ($v:expr) => { 72 | match $v { 73 | Ok(r) => r, 74 | Err(e) => return err!(e), 75 | } 76 | } 77 | } 78 | 79 | let mut end = false; 80 | while messages.is_empty() && !end { 81 | // FIXME: Store the iterator instead of a vector of values, 82 | // however this makes it self-referential... 83 | let (mut iterator, end_) = try_!(parser.read_async(&mut stream).await); 84 | end = end_; 85 | loop { 86 | match iterator.next() { 87 | Some(Ok(msg)) => messages.push_back(msg.into()), 88 | Some(Err(e)) => return err!(e), 89 | None => break, 90 | } 91 | } 92 | } 93 | match messages.pop_front() { 94 | Some(msg) => { 95 | let event = match msg.try_into() { 96 | Ok(e) => e, 97 | Err(()) => return err!(Error::Protocol(Box::new( 98 | proto::Error("Message is not valid for this mode"), 99 | ))), 100 | }; 101 | debug!("ssh: recv {:?}", event); 102 | Some((Ok(event), arg)) 103 | } 104 | None => return None, 105 | } 106 | } 107 | } 108 | } 109 | 110 | struct SshSink { 111 | stdin: W, 112 | buffer: Vec, 113 | } 114 | 115 | impl SshSink { 116 | fn new(stdin: W) -> SshSink< W> { 117 | SshSink { 118 | stdin, 119 | buffer: Vec::new(), 120 | } 121 | } 122 | 123 | fn project<'a>(self: &'a mut Pin>>) -> (&'a mut W, &'a mut Vec) { 124 | unsafe { 125 | let s = self.as_mut().get_unchecked_mut(); 126 | (&mut s.stdin, &mut s.buffer) 127 | } 128 | } 129 | 130 | fn sink + Debug>(mut arg: Pin>>, event: T) -> impl Future>>, Error>> { 131 | async move { 132 | let (sink, mut buffer) = arg.project(); 133 | 134 | debug!("ssh: send {:?}", event); 135 | write_message(&event.into(), &mut buffer)?; 136 | //eprintln!("send \"{}\"", String::from_utf8_lossy(&buffer)); 137 | sink.write_all(buffer).await?; 138 | sink.flush().await?; 139 | buffer.clear(); 140 | Ok(arg) 141 | } 142 | } 143 | } 144 | 145 | pub fn ssh_source(loc: &SshLocation) -> Result { 146 | let SshLocation { user, host, path } = loc; 147 | let connection_arg = match user { 148 | Some(user) => { 149 | info!("Setting up source {}@{}:{}", user, host, path); 150 | format!("{}@{}", user, host) 151 | } 152 | None => { 153 | info!("Setting up source {}:{}", host, path); 154 | host.to_owned() 155 | } 156 | }; 157 | let escaped_path = shell_escape(path); 158 | debug!( 159 | "Running command: ssh {} syncfast remote-send {}", 160 | connection_arg, escaped_path, 161 | ); 162 | let process: Child = Command::new("ssh") 163 | .arg(connection_arg) 164 | .arg("syncfast") 165 | .arg("remote-send") 166 | .arg(escaped_path) 167 | .stdin(Stdio::piped()) 168 | .stdout(Stdio::piped()) 169 | .stderr(Stdio::inherit()) 170 | .spawn()?; 171 | 172 | Ok(Source { 173 | stream: futures::stream::unfold( 174 | Box::pin(SshStream::new(process.stdout.unwrap())), 175 | SshStream::stream, 176 | ).boxed_local(), 177 | sink: Box::pin(futures::sink::unfold( 178 | Box::pin(SshSink::new(process.stdin.unwrap())), 179 | SshSink::sink, 180 | )), 181 | }) 182 | } 183 | 184 | pub fn ssh_destination(loc: &SshLocation) -> Result { 185 | let SshLocation { user, host, path } = loc; 186 | let connection_arg = match user { 187 | Some(user) => { 188 | info!("Setting up destination {}@{}:{}", user, host, path); 189 | format!("{}@{}", user, host) 190 | } 191 | None => { 192 | info!("Setting up destination {}:{}", host, path); 193 | host.to_owned() 194 | } 195 | }; 196 | let escaped_path = shell_escape(path); 197 | debug!( 198 | "Running command: ssh {} syncfast remote-recv {}", 199 | connection_arg, escaped_path, 200 | ); 201 | let process: Child = Command::new("ssh") 202 | .arg(connection_arg) 203 | .arg("syncfast") 204 | .arg("remote-recv") 205 | .arg(escaped_path) 206 | .stdin(Stdio::piped()) 207 | .stdout(Stdio::piped()) 208 | .stderr(Stdio::inherit()) 209 | .spawn()?; 210 | 211 | Ok(Destination { 212 | stream: futures::stream::unfold( 213 | Box::pin(SshStream::new(process.stdout.unwrap())), 214 | SshStream::stream, 215 | ).boxed_local(), 216 | sink: Box::pin(futures::sink::unfold( 217 | Box::pin(SshSink::new(process.stdin.unwrap())), 218 | SshSink::sink, 219 | )), 220 | }) 221 | } 222 | 223 | pub fn stdio_source() -> Source { 224 | Source { 225 | stream: futures::stream::unfold( 226 | Box::pin(SshStream::new(stdin())), 227 | SshStream::stream, 228 | ).boxed_local(), 229 | sink: Box::pin(futures::sink::unfold( 230 | Box::pin(SshSink::new(stdout())), 231 | SshSink::sink, 232 | )), 233 | } 234 | } 235 | 236 | pub fn stdio_destination() -> Destination { 237 | Destination { 238 | stream: futures::stream::unfold( 239 | Box::pin(SshStream::new(stdin())), 240 | SshStream::stream, 241 | ).boxed_local(), 242 | sink: Box::pin(futures::sink::unfold( 243 | Box::pin(SshSink::new(stdout())), 244 | SshSink::sink, 245 | )), 246 | } 247 | } 248 | -------------------------------------------------------------------------------- /src/sync/ssh/proto.rs: -------------------------------------------------------------------------------- 1 | use log::warn; 2 | use std::convert::{TryFrom, TryInto}; 3 | use std::io::Write; 4 | use std::ops::Deref; 5 | use tokio::io::{AsyncRead, AsyncReadExt}; 6 | 7 | use crate::HashDigest; 8 | use crate::HASH_DIGEST_LEN; 9 | use crate::streaming_iterator::StreamingIterator; 10 | use crate::sync::{DestinationEvent, SourceEvent}; 11 | 12 | #[derive(Debug)] 13 | pub struct Error(pub &'static str); 14 | 15 | impl std::fmt::Display for Error { 16 | fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { 17 | self.0.fmt(f) 18 | } 19 | } 20 | 21 | impl std::error::Error for Error {} 22 | 23 | impl Into for Error { 24 | fn into(self) -> crate::Error { 25 | crate::Error::Protocol(Box::new(self)) 26 | } 27 | } 28 | 29 | #[derive(Debug, PartialEq)] 30 | pub enum Message<'a> { 31 | FileEntry(&'a [u8], usize, HashDigest), 32 | EndFiles, 33 | GetFile(&'a [u8]), 34 | FileStart(&'a [u8]), 35 | FileBlock(HashDigest, usize), 36 | FileEnd, 37 | GetBlock(HashDigest), 38 | BlockData(HashDigest, &'a [u8]), 39 | Complete, 40 | } 41 | 42 | #[derive(Debug, PartialEq)] 43 | pub enum OwnedMessage { 44 | FileEntry(Vec, usize, HashDigest), 45 | EndFiles, 46 | GetFile(Vec), 47 | FileStart(Vec), 48 | FileBlock(HashDigest, usize), 49 | FileEnd, 50 | GetBlock(HashDigest), 51 | BlockData(HashDigest, Vec), 52 | Complete, 53 | } 54 | 55 | impl<'a> From> for OwnedMessage { 56 | fn from(msg: Message<'a>) -> OwnedMessage { 57 | match msg { 58 | Message::FileEntry(name, size, digest) => OwnedMessage::FileEntry(name.to_owned(), size, digest), 59 | Message::EndFiles => OwnedMessage::EndFiles, 60 | Message::GetFile(name) => OwnedMessage::GetFile(name.to_owned()), 61 | Message::FileStart(name) => OwnedMessage::FileStart(name.to_owned()), 62 | Message::FileBlock(digest, size) => OwnedMessage::FileBlock(digest, size), 63 | Message::FileEnd => OwnedMessage::FileEnd, 64 | Message::GetBlock(digest) => OwnedMessage::GetBlock(digest), 65 | Message::BlockData(digest, data) => OwnedMessage::BlockData(digest, data.to_owned()), 66 | Message::Complete => OwnedMessage::Complete, 67 | } 68 | } 69 | } 70 | 71 | impl<'a> From<&'a OwnedMessage> for Message<'a> { 72 | fn from(msg: &'a OwnedMessage) -> Message<'a> { 73 | match msg { 74 | &OwnedMessage::FileEntry(ref name, size, ref digest) => Message::FileEntry(name, size, digest.clone()), 75 | &OwnedMessage::EndFiles => Message::EndFiles, 76 | &OwnedMessage::GetFile(ref name) => Message::GetFile(name), 77 | &OwnedMessage::FileStart(ref name) => Message::FileStart(name), 78 | &OwnedMessage::FileBlock(ref digest, size) => Message::FileBlock(digest.clone(), size), 79 | &OwnedMessage::FileEnd => Message::FileEnd, 80 | &OwnedMessage::GetBlock(ref digest) => Message::GetBlock(digest.clone()), 81 | &OwnedMessage::BlockData(ref digest, ref data) => Message::BlockData(digest.clone(), data), 82 | &OwnedMessage::Complete => Message::Complete, 83 | } 84 | } 85 | } 86 | 87 | impl From for OwnedMessage { 88 | fn from(event: SourceEvent) -> OwnedMessage { 89 | match event { 90 | SourceEvent::FileEntry(name, size, hash) => OwnedMessage::FileEntry(name, size, hash), 91 | SourceEvent::EndFiles => OwnedMessage::EndFiles, 92 | SourceEvent::FileStart(name) => OwnedMessage::FileStart(name), 93 | SourceEvent::FileBlock(hash, size) => OwnedMessage::FileBlock(hash, size), 94 | SourceEvent::FileEnd => OwnedMessage::FileEnd, 95 | SourceEvent::BlockData(hash, data) => OwnedMessage::BlockData(hash, data), 96 | } 97 | } 98 | } 99 | 100 | impl From for OwnedMessage { 101 | fn from(event: DestinationEvent) -> OwnedMessage { 102 | match event { 103 | DestinationEvent::GetFile(name) => OwnedMessage::GetFile(name), 104 | DestinationEvent::GetBlock(digest) => OwnedMessage::GetBlock(digest), 105 | DestinationEvent::Complete => OwnedMessage::Complete, 106 | } 107 | } 108 | } 109 | 110 | impl TryFrom for SourceEvent { 111 | type Error = (); 112 | 113 | fn try_from(message: OwnedMessage) -> Result { 114 | Ok(match message { 115 | OwnedMessage::FileEntry(name, size, hash) => SourceEvent::FileEntry(name, size, hash), 116 | OwnedMessage::EndFiles => SourceEvent::EndFiles, 117 | OwnedMessage::FileStart(name) => SourceEvent::FileStart(name), 118 | OwnedMessage::FileBlock(hash, size) => SourceEvent::FileBlock(hash, size), 119 | OwnedMessage::FileEnd => SourceEvent::FileEnd, 120 | OwnedMessage::BlockData(hash, data) => SourceEvent::BlockData(hash, data), 121 | _ => return Err(()), 122 | }) 123 | } 124 | } 125 | 126 | impl TryFrom for DestinationEvent { 127 | type Error = (); 128 | 129 | fn try_from(message: OwnedMessage) -> Result { 130 | Ok(match message { 131 | OwnedMessage::GetFile(name) => DestinationEvent::GetFile(name), 132 | OwnedMessage::GetBlock(digest) => DestinationEvent::GetBlock(digest), 133 | OwnedMessage::Complete => DestinationEvent::Complete, 134 | _ => return Err(()), 135 | }) 136 | } 137 | } 138 | 139 | pub fn write_message<'a, M: Into>, W: Write>(message: M, mut writer: W) -> std::io::Result<()> { 140 | let message = message.into(); 141 | match message { 142 | Message::FileEntry(name, size, digest) => { 143 | writer.write_all(b"FILE_ENTRY\n")?; 144 | writer.write_all(name)?; 145 | write!(writer, "\n{}\n", size)?; 146 | writer.write_all(&digest.0)?; 147 | writer.write_all(b"\n")?; 148 | } 149 | Message::EndFiles => { 150 | writer.write_all(b"END_FILES\n")?; 151 | } 152 | Message::GetFile(name) => { 153 | writer.write_all(b"GET_FILE\n")?; 154 | writer.write_all(name)?; 155 | writer.write_all(b"\n")?; 156 | } 157 | Message::FileStart(name) => { 158 | writer.write_all(b"FILE_START\n")?; 159 | writer.write_all(name)?; 160 | writer.write_all(b"\n")?; 161 | } 162 | Message::FileBlock(digest, size) => { 163 | writer.write_all(b"FILE_BLOCK\n")?; 164 | writer.write_all(&digest.0)?; 165 | write!(writer, "\n{}\n", size)?; 166 | } 167 | Message::FileEnd => { 168 | writer.write_all(b"FILE_END\n")?; 169 | } 170 | Message::GetBlock(digest) => { 171 | writer.write_all(b"GET_BLOCK\n")?; 172 | writer.write_all(&digest.0)?; 173 | writer.write_all(b"\n")?; 174 | } 175 | Message::BlockData(digest, data) => { 176 | writer.write_all(b"BLOCK_DATA\n")?; 177 | writer.write_all(&digest.0)?; 178 | write!(writer, "\n{}\n", data.len())?; 179 | writer.write_all(data)?; 180 | writer.write_all(b"\n")?; 181 | } 182 | Message::Complete => { 183 | writer.write_all(b"COMPLETE\n")?; 184 | } 185 | } 186 | Ok(()) 187 | } 188 | 189 | #[derive(Default)] 190 | pub struct Parser { 191 | buffer: Vec, 192 | pos: usize, 193 | } 194 | 195 | use std::future::Future; 196 | 197 | impl Parser { 198 | pub fn receive<'a, E, F>(&'a mut self, func: F) -> Result, E> 199 | where 200 | F: FnOnce(&mut Vec) -> Result<(), E> 201 | { 202 | self.buffer.drain(..self.pos); 203 | self.pos = 0; 204 | 205 | func(&mut self.buffer)?; 206 | Ok(Messages { 207 | buffer: &mut self.buffer, 208 | pos: &mut self.pos, 209 | }) 210 | } 211 | 212 | pub fn read_async<'a, R: AsyncRead + Unpin>( 213 | &'a mut self, 214 | mut reader: R, 215 | ) -> impl Future, bool), std::io::Error>> { 216 | async move { 217 | self.buffer.drain(..self.pos); 218 | self.pos = 0; 219 | 220 | let read_len = reader.read_buf(&mut self.buffer).await?; 221 | let end = read_len == 0; 222 | Ok((Messages { 223 | buffer: &mut self.buffer, 224 | pos: &mut self.pos, 225 | }, end)) 226 | } 227 | } 228 | 229 | pub fn parse<'a>(&'a mut self, input: &[u8]) -> Messages<'a> { 230 | self.buffer.drain(..self.pos); 231 | self.pos = 0; 232 | self.buffer.extend_from_slice(input); 233 | Messages { 234 | buffer: &mut self.buffer, 235 | pos: &mut self.pos, 236 | } 237 | } 238 | } 239 | 240 | pub struct Messages<'a> { 241 | buffer: &'a mut Vec, 242 | pos: &'a mut usize, 243 | } 244 | 245 | const COMMAND_MAX: usize = 20; 246 | const FILENAME_MAX: usize = 100; 247 | const SIZE_MAX: usize = 15; 248 | 249 | struct View<'a, T> { 250 | slice: &'a [T], 251 | pos: usize, 252 | } 253 | 254 | impl<'a, T> View<'a, T> { 255 | fn new(slice: &'a [T]) -> View<'a, T> { 256 | View { 257 | slice, 258 | pos: 0, 259 | } 260 | } 261 | 262 | fn advance(&mut self, offset: usize) -> &'a [T] { 263 | assert!(self.pos + offset <= self.slice.len()); 264 | let ret = &self.slice[self.pos..self.pos + offset]; 265 | self.pos += offset; 266 | ret 267 | } 268 | } 269 | 270 | impl<'a> View<'a, u8> { 271 | fn read_line( 272 | &mut self, 273 | max_size: usize, 274 | error: E, 275 | ) -> Result, E> { 276 | match self.slice[self.pos..self.slice.len().min(self.pos + max_size + 1)].iter().position(|&b| b == b'\n') { 277 | Some(s) => { 278 | let line = &self.slice[self.pos..self.pos + s]; 279 | self.advance(s + 1); 280 | Ok(Some(line)) 281 | } 282 | None => { 283 | if self.len() >= max_size { 284 | Err(error) 285 | } else { 286 | Ok(None) 287 | } 288 | } 289 | } 290 | } 291 | 292 | fn read_exact( 293 | &mut self, 294 | size: usize, 295 | error: E, 296 | ) -> Result, E> { 297 | if self.slice[self.pos..].len() >= size + 1 { 298 | if self.slice[self.pos + size] == b'\n' { 299 | let value = &self.slice[self.pos..self.pos + size]; 300 | self.advance(size + 1); 301 | Ok(Some(value)) 302 | } else { 303 | Err(error) 304 | } 305 | } else { 306 | Ok(None) 307 | } 308 | } 309 | } 310 | 311 | impl<'a, T> Deref for View<'a, T> { 312 | type Target = [T]; 313 | 314 | fn deref(&self) -> &[T] { 315 | &self.slice[self.pos..] 316 | } 317 | } 318 | 319 | impl<'a, 'b: 'a> StreamingIterator<'a> for Messages<'b> { 320 | type Item = Result, Error>; 321 | 322 | fn next(&'a mut self) -> Option, Error>> { 323 | //eprintln!("recv \"{}\"", String::from_utf8_lossy(&self.buffer[*self.pos..])); 324 | let mut buffer = View::new(&self.buffer[*self.pos..]); 325 | if buffer.len() == 0 { 326 | return None; 327 | } 328 | // Read command 329 | let command = match buffer.read_line(COMMAND_MAX, Error("Unterminated command")) { 330 | Err(e) => { 331 | warn!("ERROR: \"{}\"", String::from_utf8_lossy(&self.buffer[*self.pos..])); 332 | return Some(Err(e)); 333 | } 334 | Ok(Some(s)) => s, 335 | Ok(None) => return None, 336 | }; 337 | let ret = if command == b"FILE_ENTRY" { 338 | // Read filename 339 | let filename = match buffer.read_line(FILENAME_MAX, Error("Unterminated filename")) { 340 | Err(e) => return Some(Err(e)), 341 | Ok(Some(s)) => s, 342 | Ok(None) => return None, 343 | }; 344 | // Read size 345 | let size = match buffer.read_line(SIZE_MAX, Error("Unterminated size")) { 346 | Err(e) => return Some(Err(e)), 347 | Ok(Some(s)) => s, 348 | Ok(None) => return None, 349 | }; 350 | let size: Option<&str> = std::str::from_utf8(size).ok(); 351 | let size: Option = size.and_then(|s| s.parse().ok()); 352 | let size = match size { 353 | Some(s) => s, 354 | None => return Some(Err(Error("Invalid file size"))), 355 | }; 356 | // Read digest 357 | let digest = match buffer.read_exact(HASH_DIGEST_LEN, Error("Unterminated digest")) { 358 | Err(e) => return Some(Err(e)), 359 | Ok(Some(s)) => s, 360 | Ok(None) => return None, 361 | }; 362 | let digest = HashDigest(digest.try_into().unwrap()); 363 | // Success 364 | Message::FileEntry(filename, size, digest) 365 | } else if command == b"END_FILES" { 366 | Message::EndFiles 367 | } else if command == b"GET_FILE" { 368 | // Read filename 369 | let filename = match buffer.read_line(FILENAME_MAX, Error("Unterminated filename")) { 370 | Err(e) => return Some(Err(e)), 371 | Ok(Some(s)) => s, 372 | Ok(None) => return None, 373 | }; 374 | // Success 375 | Message::GetFile(filename) 376 | } else if command == b"FILE_START" { 377 | // Read filename 378 | let filename = match buffer.read_line(FILENAME_MAX, Error("Unterminated filename")) { 379 | Err(e) => return Some(Err(e)), 380 | Ok(Some(s)) => s, 381 | Ok(None) => return None, 382 | }; 383 | // Success 384 | Message::FileStart(filename) 385 | } else if command == b"FILE_BLOCK" { 386 | // Read digest 387 | let digest = match buffer.read_exact(HASH_DIGEST_LEN, Error("Unterminated digest")) { 388 | Err(e) => return Some(Err(e)), 389 | Ok(Some(s)) => s, 390 | Ok(None) => return None, 391 | }; 392 | let digest = HashDigest(digest.try_into().unwrap()); 393 | // Read size 394 | let size = match buffer.read_line(SIZE_MAX, Error("Unterminated size")) { 395 | Err(e) => return Some(Err(e)), 396 | Ok(Some(s)) => s, 397 | Ok(None) => return None, 398 | }; 399 | let size: Option<&str> = std::str::from_utf8(size).ok(); 400 | let size: Option = size.and_then(|s| s.parse().ok()); 401 | let size = match size { 402 | Some(s) => s, 403 | None => return Some(Err(Error("Invalid block size"))), 404 | }; 405 | // Success 406 | Message::FileBlock(digest, size) 407 | } else if command == b"FILE_END" { 408 | Message::FileEnd 409 | } else if command == b"GET_BLOCK" { 410 | // Read digest 411 | let digest = match buffer.read_exact(HASH_DIGEST_LEN, Error("Unterminated digest")) { 412 | Err(e) => return Some(Err(e)), 413 | Ok(Some(s)) => s, 414 | Ok(None) => return None, 415 | }; 416 | let digest = HashDigest(digest.try_into().unwrap()); 417 | // Success 418 | Message::GetBlock(digest) 419 | } else if command == b"BLOCK_DATA" { 420 | // Read digest 421 | let digest = match buffer.read_exact(HASH_DIGEST_LEN, Error("Unterminated digest")) { 422 | Err(e) => return Some(Err(e)), 423 | Ok(Some(s)) => s, 424 | Ok(None) => return None, 425 | }; 426 | let digest = HashDigest(digest.try_into().unwrap()); 427 | // Read data length 428 | let size = match buffer.read_line(SIZE_MAX, Error("Unterminated length")) { 429 | Err(e) => return Some(Err(e)), 430 | Ok(Some(s)) => s, 431 | Ok(None) => return None, 432 | }; 433 | let size: Option<&str> = std::str::from_utf8(size).ok(); 434 | let size: Option = size.and_then(|s| s.parse().ok()); 435 | let size = match size { 436 | Some(s) => s, 437 | None => return Some(Err(Error("Invalid block length"))), 438 | }; 439 | // Read data 440 | let data = match buffer.read_exact(size, Error("Invalid data end byte")) { 441 | Err(e) => return Some(Err(e)), 442 | Ok(Some(s)) => s, 443 | Ok(None) => return None, 444 | }; 445 | // Success 446 | Message::BlockData(digest, data) 447 | } else if command == b"COMPLETE" { 448 | Message::Complete 449 | } else { 450 | warn!("Unknown command: {:?}", command); 451 | return Some(Err(Error("Unknown command"))); 452 | }; 453 | 454 | *self.pos += buffer.pos; 455 | Some(Ok(ret)) 456 | } 457 | } 458 | 459 | #[cfg(test)] 460 | mod tests { 461 | use super::{OwnedMessage, Parser, Message, Messages, write_message}; 462 | use crate::HashDigest; 463 | use crate::streaming_iterator::StreamingIterator; 464 | 465 | fn compare<'a>(mut iterator: Messages<'a>, expected: &[Message<'static>]) { 466 | let mut expected = expected.iter(); 467 | loop { 468 | match (iterator.next(), expected.next()) { 469 | (None, None) => break, 470 | (Some(msg), Some(e)) => { 471 | let msg = msg.unwrap(); 472 | assert_eq!(&msg, e); 473 | } 474 | (Some(msg), None) => { 475 | let msg = msg.unwrap(); 476 | panic!("More messages than expected: {:?}", msg); 477 | } 478 | (None, Some(e)) => panic!("Fewer messages than expected: {:?}", e), 479 | } 480 | } 481 | } 482 | 483 | #[test] 484 | fn test_parse() { 485 | let inputs: &[&[u8]] = &[ 486 | b"FILE_ENTR", 487 | b"Y", 488 | b"\n", 489 | b"filename\n12", 490 | b"\n12345678901234567890\nCOMPLETE", 491 | b"\n", 492 | ]; 493 | let expected: &[&[Message<'static>]] = &[ 494 | &[], 495 | &[], 496 | &[], 497 | &[], 498 | &[Message::FileEntry( 499 | b"filename", 12, HashDigest(*b"12345678901234567890"), 500 | )], 501 | &[Message::Complete], 502 | ]; 503 | let mut parser: Parser = Default::default(); 504 | for (bytes, expected_messages) in inputs.iter().zip(expected) { 505 | compare( 506 | parser.receive::<(), _>(|buf| { buf.extend_from_slice(bytes); Ok(()) }).unwrap(), 507 | expected_messages, 508 | ); 509 | } 510 | } 511 | 512 | #[test] 513 | fn test_write() { 514 | let mut output = Vec::new(); 515 | write_message( 516 | Message::FileEntry(b"filename", 12, HashDigest(*b"12345678901234567890")), 517 | &mut output, 518 | ).unwrap(); 519 | write_message( 520 | &OwnedMessage::EndFiles, 521 | &mut output, 522 | ).unwrap(); 523 | // FIXME: Casts to &[u8] required for Rust < 1.47 524 | assert_eq!( 525 | &output as &[u8], 526 | b"FILE_ENTRY\nfilename\n12\n12345678901234567890\nEND_FILES\n" as &[u8], 527 | ); 528 | } 529 | } 530 | -------------------------------------------------------------------------------- /src/sync/utils.rs: -------------------------------------------------------------------------------- 1 | use futures::future::{FutureExt, Map}; 2 | use futures::channel::oneshot::{Canceled, Receiver, Sender, channel}; 3 | use std::path::Path; 4 | 5 | pub struct Condition { 6 | sender: Option>, 7 | receiver: Option>, 8 | } 9 | 10 | impl Default for Condition { 11 | fn default() -> Self { 12 | let (sender, receiver) = channel(); 13 | Condition { sender: Some(sender), receiver: Some(receiver) } 14 | } 15 | } 16 | 17 | pub type ConditionFuture = Map, fn(Result<(), Canceled>)>; 18 | 19 | impl Condition { 20 | pub fn set(&mut self) { 21 | let sender = self.sender.take().expect("Condition::set() called twice"); 22 | sender.send(()).expect("Condition::set()"); 23 | } 24 | 25 | pub fn wait(&mut self) -> ConditionFuture { 26 | fn error(r: Result<(), Canceled>) { 27 | r.expect("Condition::wait()"); 28 | } 29 | self.receiver.take().expect("Condition::wait() called twice").map(error) 30 | } 31 | } 32 | 33 | pub fn move_file(from: &Path, to: &Path) -> std::io::Result<()> { 34 | match std::fs::rename(from, to) { 35 | Ok(()) => Ok(()), 36 | Err(_) => { 37 | // Try copying then removing source file 38 | std::fs::copy(from, to)?; 39 | match std::fs::remove_file(from) { 40 | Ok(()) => Ok(()), 41 | Err(e) => { 42 | std::fs::remove_file(to).ok(); // Ignore this result on purpose 43 | Err(e) 44 | } 45 | } 46 | } 47 | } 48 | } 49 | --------------------------------------------------------------------------------