├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── assets └── concurrency-diagram.png ├── cspell.json ├── examples └── demo.gif └── src ├── downloader.rs ├── http_utils.rs ├── main.rs ├── progress_reporter.rs ├── resource.rs └── shared_types.rs /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "addr2line" 7 | version = "0.20.0" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" 10 | dependencies = [ 11 | "gimli", 12 | ] 13 | 14 | [[package]] 15 | name = "adler" 16 | version = "1.0.2" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" 19 | 20 | [[package]] 21 | name = "anstream" 22 | version = "0.3.2" 23 | source = "registry+https://github.com/rust-lang/crates.io-index" 24 | checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" 25 | dependencies = [ 26 | "anstyle", 27 | "anstyle-parse", 28 | "anstyle-query", 29 | "anstyle-wincon", 30 | "colorchoice", 31 | "is-terminal", 32 | "utf8parse", 33 | ] 34 | 35 | [[package]] 36 | name = "anstyle" 37 | version = "1.0.1" 38 | source = "registry+https://github.com/rust-lang/crates.io-index" 39 | checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" 40 | 41 | [[package]] 42 | name = "anstyle-parse" 43 | version = "0.2.1" 44 | source = "registry+https://github.com/rust-lang/crates.io-index" 45 | checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" 46 | dependencies = [ 47 | "utf8parse", 48 | ] 49 | 50 | [[package]] 51 | name = "anstyle-query" 52 | version = "1.0.0" 53 | source = "registry+https://github.com/rust-lang/crates.io-index" 54 | checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" 55 | dependencies = [ 56 | "windows-sys 0.48.0", 57 | ] 58 | 59 | [[package]] 60 | name = "anstyle-wincon" 61 | version = "1.0.1" 62 | source = "registry+https://github.com/rust-lang/crates.io-index" 63 | checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" 64 | dependencies = [ 65 | "anstyle", 66 | "windows-sys 0.48.0", 67 | ] 68 | 69 | [[package]] 70 | name = "async-channel" 71 | version = "1.9.0" 72 | source = "registry+https://github.com/rust-lang/crates.io-index" 73 | checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" 74 | dependencies = [ 75 | "concurrent-queue", 76 | "event-listener", 77 | "futures-core", 78 | ] 79 | 80 | [[package]] 81 | name = "async-recursion" 82 | version = "1.0.4" 83 | source = "registry+https://github.com/rust-lang/crates.io-index" 84 | checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" 85 | dependencies = [ 86 | "proc-macro2", 87 | "quote", 88 | "syn", 89 | ] 90 | 91 | [[package]] 92 | name = "async-stream" 93 | version = "0.3.5" 94 | source = "registry+https://github.com/rust-lang/crates.io-index" 95 | checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" 96 | dependencies = [ 97 | "async-stream-impl", 98 | "futures-core", 99 | "pin-project-lite", 100 | ] 101 | 102 | [[package]] 103 | name = "async-stream-impl" 104 | version = "0.3.5" 105 | source = "registry+https://github.com/rust-lang/crates.io-index" 106 | checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" 107 | dependencies = [ 108 | "proc-macro2", 109 | "quote", 110 | "syn", 111 | ] 112 | 113 | [[package]] 114 | name = "autocfg" 115 | version = "1.1.0" 116 | source = "registry+https://github.com/rust-lang/crates.io-index" 117 | checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" 118 | 119 | [[package]] 120 | name = "backtrace" 121 | version = "0.3.68" 122 | source = "registry+https://github.com/rust-lang/crates.io-index" 123 | checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" 124 | dependencies = [ 125 | "addr2line", 126 | "cc", 127 | "cfg-if", 128 | "libc", 129 | "miniz_oxide", 130 | "object", 131 | "rustc-demangle", 132 | ] 133 | 134 | [[package]] 135 | name = "base64" 136 | version = "0.21.2" 137 | source = "registry+https://github.com/rust-lang/crates.io-index" 138 | checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" 139 | 140 | [[package]] 141 | name = "bitflags" 142 | version = "1.3.2" 143 | source = "registry+https://github.com/rust-lang/crates.io-index" 144 | checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 145 | 146 | [[package]] 147 | name = "bitflags" 148 | version = "2.3.3" 149 | source = "registry+https://github.com/rust-lang/crates.io-index" 150 | checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" 151 | 152 | [[package]] 153 | name = "bumpalo" 154 | version = "3.13.0" 155 | source = "registry+https://github.com/rust-lang/crates.io-index" 156 | checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" 157 | 158 | [[package]] 159 | name = "bytes" 160 | version = "1.4.0" 161 | source = "registry+https://github.com/rust-lang/crates.io-index" 162 | checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" 163 | 164 | [[package]] 165 | name = "cc" 166 | version = "1.0.79" 167 | source = "registry+https://github.com/rust-lang/crates.io-index" 168 | checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" 169 | 170 | [[package]] 171 | name = "cfg-if" 172 | version = "1.0.0" 173 | source = "registry+https://github.com/rust-lang/crates.io-index" 174 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 175 | 176 | [[package]] 177 | name = "circular-buffer" 178 | version = "0.1.1" 179 | source = "registry+https://github.com/rust-lang/crates.io-index" 180 | checksum = "5887fa83bbf76a5cf15915add06c187cfc9d00f2ae9aa1f33d108a02ba7af345" 181 | 182 | [[package]] 183 | name = "clap" 184 | version = "4.3.19" 185 | source = "registry+https://github.com/rust-lang/crates.io-index" 186 | checksum = "5fd304a20bff958a57f04c4e96a2e7594cc4490a0e809cbd48bb6437edaa452d" 187 | dependencies = [ 188 | "clap_builder", 189 | "clap_derive", 190 | "once_cell", 191 | ] 192 | 193 | [[package]] 194 | name = "clap_builder" 195 | version = "4.3.19" 196 | source = "registry+https://github.com/rust-lang/crates.io-index" 197 | checksum = "01c6a3f08f1fe5662a35cfe393aec09c4df95f60ee93b7556505260f75eee9e1" 198 | dependencies = [ 199 | "anstream", 200 | "anstyle", 201 | "clap_lex", 202 | "strsim", 203 | ] 204 | 205 | [[package]] 206 | name = "clap_derive" 207 | version = "4.3.12" 208 | source = "registry+https://github.com/rust-lang/crates.io-index" 209 | checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" 210 | dependencies = [ 211 | "heck", 212 | "proc-macro2", 213 | "quote", 214 | "syn", 215 | ] 216 | 217 | [[package]] 218 | name = "clap_lex" 219 | version = "0.5.0" 220 | source = "registry+https://github.com/rust-lang/crates.io-index" 221 | checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" 222 | 223 | [[package]] 224 | name = "colorchoice" 225 | version = "1.0.0" 226 | source = "registry+https://github.com/rust-lang/crates.io-index" 227 | checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" 228 | 229 | [[package]] 230 | name = "concurrent-queue" 231 | version = "2.2.0" 232 | source = "registry+https://github.com/rust-lang/crates.io-index" 233 | checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" 234 | dependencies = [ 235 | "crossbeam-utils", 236 | ] 237 | 238 | [[package]] 239 | name = "console" 240 | version = "0.15.7" 241 | source = "registry+https://github.com/rust-lang/crates.io-index" 242 | checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" 243 | dependencies = [ 244 | "encode_unicode", 245 | "lazy_static", 246 | "libc", 247 | "unicode-width", 248 | "windows-sys 0.45.0", 249 | ] 250 | 251 | [[package]] 252 | name = "core-foundation" 253 | version = "0.9.3" 254 | source = "registry+https://github.com/rust-lang/crates.io-index" 255 | checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" 256 | dependencies = [ 257 | "core-foundation-sys", 258 | "libc", 259 | ] 260 | 261 | [[package]] 262 | name = "core-foundation-sys" 263 | version = "0.8.4" 264 | source = "registry+https://github.com/rust-lang/crates.io-index" 265 | checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" 266 | 267 | [[package]] 268 | name = "crossbeam-utils" 269 | version = "0.8.16" 270 | source = "registry+https://github.com/rust-lang/crates.io-index" 271 | checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" 272 | dependencies = [ 273 | "cfg-if", 274 | ] 275 | 276 | [[package]] 277 | name = "deranged" 278 | version = "0.3.6" 279 | source = "registry+https://github.com/rust-lang/crates.io-index" 280 | checksum = "8810e7e2cf385b1e9b50d68264908ec367ba642c96d02edfe61c39e88e2a3c01" 281 | 282 | [[package]] 283 | name = "dlrs" 284 | version = "0.1.2" 285 | dependencies = [ 286 | "async-channel", 287 | "async-recursion", 288 | "async-stream", 289 | "bytes", 290 | "circular-buffer", 291 | "clap", 292 | "futures", 293 | "if_chain", 294 | "indicatif", 295 | "indicatif-log-bridge", 296 | "log", 297 | "reqwest", 298 | "simplelog", 299 | "thiserror", 300 | "tokio", 301 | "url", 302 | ] 303 | 304 | [[package]] 305 | name = "encode_unicode" 306 | version = "0.3.6" 307 | source = "registry+https://github.com/rust-lang/crates.io-index" 308 | checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" 309 | 310 | [[package]] 311 | name = "encoding_rs" 312 | version = "0.8.32" 313 | source = "registry+https://github.com/rust-lang/crates.io-index" 314 | checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" 315 | dependencies = [ 316 | "cfg-if", 317 | ] 318 | 319 | [[package]] 320 | name = "errno" 321 | version = "0.3.1" 322 | source = "registry+https://github.com/rust-lang/crates.io-index" 323 | checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" 324 | dependencies = [ 325 | "errno-dragonfly", 326 | "libc", 327 | "windows-sys 0.48.0", 328 | ] 329 | 330 | [[package]] 331 | name = "errno-dragonfly" 332 | version = "0.1.2" 333 | source = "registry+https://github.com/rust-lang/crates.io-index" 334 | checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" 335 | dependencies = [ 336 | "cc", 337 | "libc", 338 | ] 339 | 340 | [[package]] 341 | name = "event-listener" 342 | version = "2.5.3" 343 | source = "registry+https://github.com/rust-lang/crates.io-index" 344 | checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" 345 | 346 | [[package]] 347 | name = "fastrand" 348 | version = "2.0.0" 349 | source = "registry+https://github.com/rust-lang/crates.io-index" 350 | checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" 351 | 352 | [[package]] 353 | name = "fnv" 354 | version = "1.0.7" 355 | source = "registry+https://github.com/rust-lang/crates.io-index" 356 | checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" 357 | 358 | [[package]] 359 | name = "foreign-types" 360 | version = "0.3.2" 361 | source = "registry+https://github.com/rust-lang/crates.io-index" 362 | checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" 363 | dependencies = [ 364 | "foreign-types-shared", 365 | ] 366 | 367 | [[package]] 368 | name = "foreign-types-shared" 369 | version = "0.1.1" 370 | source = "registry+https://github.com/rust-lang/crates.io-index" 371 | checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" 372 | 373 | [[package]] 374 | name = "form_urlencoded" 375 | version = "1.2.0" 376 | source = "registry+https://github.com/rust-lang/crates.io-index" 377 | checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" 378 | dependencies = [ 379 | "percent-encoding", 380 | ] 381 | 382 | [[package]] 383 | name = "futures" 384 | version = "0.3.28" 385 | source = "registry+https://github.com/rust-lang/crates.io-index" 386 | checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" 387 | dependencies = [ 388 | "futures-channel", 389 | "futures-core", 390 | "futures-executor", 391 | "futures-io", 392 | "futures-sink", 393 | "futures-task", 394 | "futures-util", 395 | ] 396 | 397 | [[package]] 398 | name = "futures-channel" 399 | version = "0.3.28" 400 | source = "registry+https://github.com/rust-lang/crates.io-index" 401 | checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" 402 | dependencies = [ 403 | "futures-core", 404 | "futures-sink", 405 | ] 406 | 407 | [[package]] 408 | name = "futures-core" 409 | version = "0.3.28" 410 | source = "registry+https://github.com/rust-lang/crates.io-index" 411 | checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" 412 | 413 | [[package]] 414 | name = "futures-executor" 415 | version = "0.3.28" 416 | source = "registry+https://github.com/rust-lang/crates.io-index" 417 | checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" 418 | dependencies = [ 419 | "futures-core", 420 | "futures-task", 421 | "futures-util", 422 | ] 423 | 424 | [[package]] 425 | name = "futures-io" 426 | version = "0.3.28" 427 | source = "registry+https://github.com/rust-lang/crates.io-index" 428 | checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" 429 | 430 | [[package]] 431 | name = "futures-macro" 432 | version = "0.3.28" 433 | source = "registry+https://github.com/rust-lang/crates.io-index" 434 | checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" 435 | dependencies = [ 436 | "proc-macro2", 437 | "quote", 438 | "syn", 439 | ] 440 | 441 | [[package]] 442 | name = "futures-sink" 443 | version = "0.3.28" 444 | source = "registry+https://github.com/rust-lang/crates.io-index" 445 | checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" 446 | 447 | [[package]] 448 | name = "futures-task" 449 | version = "0.3.28" 450 | source = "registry+https://github.com/rust-lang/crates.io-index" 451 | checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" 452 | 453 | [[package]] 454 | name = "futures-util" 455 | version = "0.3.28" 456 | source = "registry+https://github.com/rust-lang/crates.io-index" 457 | checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" 458 | dependencies = [ 459 | "futures-channel", 460 | "futures-core", 461 | "futures-io", 462 | "futures-macro", 463 | "futures-sink", 464 | "futures-task", 465 | "memchr", 466 | "pin-project-lite", 467 | "pin-utils", 468 | "slab", 469 | ] 470 | 471 | [[package]] 472 | name = "gimli" 473 | version = "0.27.3" 474 | source = "registry+https://github.com/rust-lang/crates.io-index" 475 | checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" 476 | 477 | [[package]] 478 | name = "h2" 479 | version = "0.3.20" 480 | source = "registry+https://github.com/rust-lang/crates.io-index" 481 | checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" 482 | dependencies = [ 483 | "bytes", 484 | "fnv", 485 | "futures-core", 486 | "futures-sink", 487 | "futures-util", 488 | "http", 489 | "indexmap", 490 | "slab", 491 | "tokio", 492 | "tokio-util", 493 | "tracing", 494 | ] 495 | 496 | [[package]] 497 | name = "hashbrown" 498 | version = "0.12.3" 499 | source = "registry+https://github.com/rust-lang/crates.io-index" 500 | checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" 501 | 502 | [[package]] 503 | name = "heck" 504 | version = "0.4.1" 505 | source = "registry+https://github.com/rust-lang/crates.io-index" 506 | checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" 507 | 508 | [[package]] 509 | name = "hermit-abi" 510 | version = "0.3.2" 511 | source = "registry+https://github.com/rust-lang/crates.io-index" 512 | checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" 513 | 514 | [[package]] 515 | name = "http" 516 | version = "0.2.9" 517 | source = "registry+https://github.com/rust-lang/crates.io-index" 518 | checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" 519 | dependencies = [ 520 | "bytes", 521 | "fnv", 522 | "itoa", 523 | ] 524 | 525 | [[package]] 526 | name = "http-body" 527 | version = "0.4.5" 528 | source = "registry+https://github.com/rust-lang/crates.io-index" 529 | checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" 530 | dependencies = [ 531 | "bytes", 532 | "http", 533 | "pin-project-lite", 534 | ] 535 | 536 | [[package]] 537 | name = "httparse" 538 | version = "1.8.0" 539 | source = "registry+https://github.com/rust-lang/crates.io-index" 540 | checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" 541 | 542 | [[package]] 543 | name = "httpdate" 544 | version = "1.0.2" 545 | source = "registry+https://github.com/rust-lang/crates.io-index" 546 | checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" 547 | 548 | [[package]] 549 | name = "hyper" 550 | version = "0.14.27" 551 | source = "registry+https://github.com/rust-lang/crates.io-index" 552 | checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" 553 | dependencies = [ 554 | "bytes", 555 | "futures-channel", 556 | "futures-core", 557 | "futures-util", 558 | "h2", 559 | "http", 560 | "http-body", 561 | "httparse", 562 | "httpdate", 563 | "itoa", 564 | "pin-project-lite", 565 | "socket2", 566 | "tokio", 567 | "tower-service", 568 | "tracing", 569 | "want", 570 | ] 571 | 572 | [[package]] 573 | name = "hyper-tls" 574 | version = "0.5.0" 575 | source = "registry+https://github.com/rust-lang/crates.io-index" 576 | checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" 577 | dependencies = [ 578 | "bytes", 579 | "hyper", 580 | "native-tls", 581 | "tokio", 582 | "tokio-native-tls", 583 | ] 584 | 585 | [[package]] 586 | name = "idna" 587 | version = "0.4.0" 588 | source = "registry+https://github.com/rust-lang/crates.io-index" 589 | checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" 590 | dependencies = [ 591 | "unicode-bidi", 592 | "unicode-normalization", 593 | ] 594 | 595 | [[package]] 596 | name = "if_chain" 597 | version = "1.0.2" 598 | source = "registry+https://github.com/rust-lang/crates.io-index" 599 | checksum = "cb56e1aa765b4b4f3aadfab769793b7087bb03a4ea4920644a6d238e2df5b9ed" 600 | 601 | [[package]] 602 | name = "indexmap" 603 | version = "1.9.3" 604 | source = "registry+https://github.com/rust-lang/crates.io-index" 605 | checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" 606 | dependencies = [ 607 | "autocfg", 608 | "hashbrown", 609 | ] 610 | 611 | [[package]] 612 | name = "indicatif" 613 | version = "0.17.5" 614 | source = "registry+https://github.com/rust-lang/crates.io-index" 615 | checksum = "8ff8cc23a7393a397ed1d7f56e6365cba772aba9f9912ab968b03043c395d057" 616 | dependencies = [ 617 | "console", 618 | "instant", 619 | "number_prefix", 620 | "portable-atomic", 621 | "unicode-width", 622 | ] 623 | 624 | [[package]] 625 | name = "indicatif-log-bridge" 626 | version = "0.2.1" 627 | source = "registry+https://github.com/rust-lang/crates.io-index" 628 | checksum = "20b43dee3b17c8e1bd4d8691b1122d57c1710d121a717d80c4dd13d6f566b954" 629 | dependencies = [ 630 | "indicatif", 631 | "log", 632 | ] 633 | 634 | [[package]] 635 | name = "instant" 636 | version = "0.1.12" 637 | source = "registry+https://github.com/rust-lang/crates.io-index" 638 | checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" 639 | dependencies = [ 640 | "cfg-if", 641 | ] 642 | 643 | [[package]] 644 | name = "ipnet" 645 | version = "2.8.0" 646 | source = "registry+https://github.com/rust-lang/crates.io-index" 647 | checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" 648 | 649 | [[package]] 650 | name = "is-terminal" 651 | version = "0.4.9" 652 | source = "registry+https://github.com/rust-lang/crates.io-index" 653 | checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" 654 | dependencies = [ 655 | "hermit-abi", 656 | "rustix", 657 | "windows-sys 0.48.0", 658 | ] 659 | 660 | [[package]] 661 | name = "itoa" 662 | version = "1.0.9" 663 | source = "registry+https://github.com/rust-lang/crates.io-index" 664 | checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" 665 | 666 | [[package]] 667 | name = "js-sys" 668 | version = "0.3.64" 669 | source = "registry+https://github.com/rust-lang/crates.io-index" 670 | checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" 671 | dependencies = [ 672 | "wasm-bindgen", 673 | ] 674 | 675 | [[package]] 676 | name = "lazy_static" 677 | version = "1.4.0" 678 | source = "registry+https://github.com/rust-lang/crates.io-index" 679 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" 680 | 681 | [[package]] 682 | name = "libc" 683 | version = "0.2.147" 684 | source = "registry+https://github.com/rust-lang/crates.io-index" 685 | checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" 686 | 687 | [[package]] 688 | name = "linux-raw-sys" 689 | version = "0.4.3" 690 | source = "registry+https://github.com/rust-lang/crates.io-index" 691 | checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" 692 | 693 | [[package]] 694 | name = "lock_api" 695 | version = "0.4.10" 696 | source = "registry+https://github.com/rust-lang/crates.io-index" 697 | checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" 698 | dependencies = [ 699 | "autocfg", 700 | "scopeguard", 701 | ] 702 | 703 | [[package]] 704 | name = "log" 705 | version = "0.4.19" 706 | source = "registry+https://github.com/rust-lang/crates.io-index" 707 | checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" 708 | 709 | [[package]] 710 | name = "memchr" 711 | version = "2.5.0" 712 | source = "registry+https://github.com/rust-lang/crates.io-index" 713 | checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" 714 | 715 | [[package]] 716 | name = "mime" 717 | version = "0.3.17" 718 | source = "registry+https://github.com/rust-lang/crates.io-index" 719 | checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" 720 | 721 | [[package]] 722 | name = "miniz_oxide" 723 | version = "0.7.1" 724 | source = "registry+https://github.com/rust-lang/crates.io-index" 725 | checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" 726 | dependencies = [ 727 | "adler", 728 | ] 729 | 730 | [[package]] 731 | name = "mio" 732 | version = "0.8.8" 733 | source = "registry+https://github.com/rust-lang/crates.io-index" 734 | checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" 735 | dependencies = [ 736 | "libc", 737 | "wasi", 738 | "windows-sys 0.48.0", 739 | ] 740 | 741 | [[package]] 742 | name = "native-tls" 743 | version = "0.2.11" 744 | source = "registry+https://github.com/rust-lang/crates.io-index" 745 | checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" 746 | dependencies = [ 747 | "lazy_static", 748 | "libc", 749 | "log", 750 | "openssl", 751 | "openssl-probe", 752 | "openssl-sys", 753 | "schannel", 754 | "security-framework", 755 | "security-framework-sys", 756 | "tempfile", 757 | ] 758 | 759 | [[package]] 760 | name = "num_cpus" 761 | version = "1.16.0" 762 | source = "registry+https://github.com/rust-lang/crates.io-index" 763 | checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" 764 | dependencies = [ 765 | "hermit-abi", 766 | "libc", 767 | ] 768 | 769 | [[package]] 770 | name = "num_threads" 771 | version = "0.1.6" 772 | source = "registry+https://github.com/rust-lang/crates.io-index" 773 | checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" 774 | dependencies = [ 775 | "libc", 776 | ] 777 | 778 | [[package]] 779 | name = "number_prefix" 780 | version = "0.4.0" 781 | source = "registry+https://github.com/rust-lang/crates.io-index" 782 | checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" 783 | 784 | [[package]] 785 | name = "object" 786 | version = "0.31.1" 787 | source = "registry+https://github.com/rust-lang/crates.io-index" 788 | checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" 789 | dependencies = [ 790 | "memchr", 791 | ] 792 | 793 | [[package]] 794 | name = "once_cell" 795 | version = "1.18.0" 796 | source = "registry+https://github.com/rust-lang/crates.io-index" 797 | checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" 798 | 799 | [[package]] 800 | name = "openssl" 801 | version = "0.10.55" 802 | source = "registry+https://github.com/rust-lang/crates.io-index" 803 | checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" 804 | dependencies = [ 805 | "bitflags 1.3.2", 806 | "cfg-if", 807 | "foreign-types", 808 | "libc", 809 | "once_cell", 810 | "openssl-macros", 811 | "openssl-sys", 812 | ] 813 | 814 | [[package]] 815 | name = "openssl-macros" 816 | version = "0.1.1" 817 | source = "registry+https://github.com/rust-lang/crates.io-index" 818 | checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" 819 | dependencies = [ 820 | "proc-macro2", 821 | "quote", 822 | "syn", 823 | ] 824 | 825 | [[package]] 826 | name = "openssl-probe" 827 | version = "0.1.5" 828 | source = "registry+https://github.com/rust-lang/crates.io-index" 829 | checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" 830 | 831 | [[package]] 832 | name = "openssl-sys" 833 | version = "0.9.90" 834 | source = "registry+https://github.com/rust-lang/crates.io-index" 835 | checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" 836 | dependencies = [ 837 | "cc", 838 | "libc", 839 | "pkg-config", 840 | "vcpkg", 841 | ] 842 | 843 | [[package]] 844 | name = "paris" 845 | version = "1.5.15" 846 | source = "registry+https://github.com/rust-lang/crates.io-index" 847 | checksum = "8fecab3723493c7851f292cb060f3ee1c42f19b8d749345d0d7eaf3fd19aa62d" 848 | 849 | [[package]] 850 | name = "parking_lot" 851 | version = "0.12.1" 852 | source = "registry+https://github.com/rust-lang/crates.io-index" 853 | checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" 854 | dependencies = [ 855 | "lock_api", 856 | "parking_lot_core", 857 | ] 858 | 859 | [[package]] 860 | name = "parking_lot_core" 861 | version = "0.9.8" 862 | source = "registry+https://github.com/rust-lang/crates.io-index" 863 | checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" 864 | dependencies = [ 865 | "cfg-if", 866 | "libc", 867 | "redox_syscall", 868 | "smallvec", 869 | "windows-targets 0.48.1", 870 | ] 871 | 872 | [[package]] 873 | name = "percent-encoding" 874 | version = "2.3.0" 875 | source = "registry+https://github.com/rust-lang/crates.io-index" 876 | checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" 877 | 878 | [[package]] 879 | name = "pin-project-lite" 880 | version = "0.2.10" 881 | source = "registry+https://github.com/rust-lang/crates.io-index" 882 | checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" 883 | 884 | [[package]] 885 | name = "pin-utils" 886 | version = "0.1.0" 887 | source = "registry+https://github.com/rust-lang/crates.io-index" 888 | checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" 889 | 890 | [[package]] 891 | name = "pkg-config" 892 | version = "0.3.27" 893 | source = "registry+https://github.com/rust-lang/crates.io-index" 894 | checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" 895 | 896 | [[package]] 897 | name = "portable-atomic" 898 | version = "1.4.2" 899 | source = "registry+https://github.com/rust-lang/crates.io-index" 900 | checksum = "f32154ba0af3a075eefa1eda8bb414ee928f62303a54ea85b8d6638ff1a6ee9e" 901 | 902 | [[package]] 903 | name = "proc-macro2" 904 | version = "1.0.66" 905 | source = "registry+https://github.com/rust-lang/crates.io-index" 906 | checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" 907 | dependencies = [ 908 | "unicode-ident", 909 | ] 910 | 911 | [[package]] 912 | name = "quote" 913 | version = "1.0.32" 914 | source = "registry+https://github.com/rust-lang/crates.io-index" 915 | checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" 916 | dependencies = [ 917 | "proc-macro2", 918 | ] 919 | 920 | [[package]] 921 | name = "redox_syscall" 922 | version = "0.3.5" 923 | source = "registry+https://github.com/rust-lang/crates.io-index" 924 | checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" 925 | dependencies = [ 926 | "bitflags 1.3.2", 927 | ] 928 | 929 | [[package]] 930 | name = "reqwest" 931 | version = "0.11.18" 932 | source = "registry+https://github.com/rust-lang/crates.io-index" 933 | checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" 934 | dependencies = [ 935 | "base64", 936 | "bytes", 937 | "encoding_rs", 938 | "futures-core", 939 | "futures-util", 940 | "h2", 941 | "http", 942 | "http-body", 943 | "hyper", 944 | "hyper-tls", 945 | "ipnet", 946 | "js-sys", 947 | "log", 948 | "mime", 949 | "native-tls", 950 | "once_cell", 951 | "percent-encoding", 952 | "pin-project-lite", 953 | "serde", 954 | "serde_json", 955 | "serde_urlencoded", 956 | "tokio", 957 | "tokio-native-tls", 958 | "tokio-util", 959 | "tower-service", 960 | "url", 961 | "wasm-bindgen", 962 | "wasm-bindgen-futures", 963 | "wasm-streams", 964 | "web-sys", 965 | "winreg", 966 | ] 967 | 968 | [[package]] 969 | name = "rustc-demangle" 970 | version = "0.1.23" 971 | source = "registry+https://github.com/rust-lang/crates.io-index" 972 | checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" 973 | 974 | [[package]] 975 | name = "rustix" 976 | version = "0.38.4" 977 | source = "registry+https://github.com/rust-lang/crates.io-index" 978 | checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" 979 | dependencies = [ 980 | "bitflags 2.3.3", 981 | "errno", 982 | "libc", 983 | "linux-raw-sys", 984 | "windows-sys 0.48.0", 985 | ] 986 | 987 | [[package]] 988 | name = "ryu" 989 | version = "1.0.15" 990 | source = "registry+https://github.com/rust-lang/crates.io-index" 991 | checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" 992 | 993 | [[package]] 994 | name = "schannel" 995 | version = "0.1.22" 996 | source = "registry+https://github.com/rust-lang/crates.io-index" 997 | checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" 998 | dependencies = [ 999 | "windows-sys 0.48.0", 1000 | ] 1001 | 1002 | [[package]] 1003 | name = "scopeguard" 1004 | version = "1.2.0" 1005 | source = "registry+https://github.com/rust-lang/crates.io-index" 1006 | checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" 1007 | 1008 | [[package]] 1009 | name = "security-framework" 1010 | version = "2.9.2" 1011 | source = "registry+https://github.com/rust-lang/crates.io-index" 1012 | checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" 1013 | dependencies = [ 1014 | "bitflags 1.3.2", 1015 | "core-foundation", 1016 | "core-foundation-sys", 1017 | "libc", 1018 | "security-framework-sys", 1019 | ] 1020 | 1021 | [[package]] 1022 | name = "security-framework-sys" 1023 | version = "2.9.1" 1024 | source = "registry+https://github.com/rust-lang/crates.io-index" 1025 | checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" 1026 | dependencies = [ 1027 | "core-foundation-sys", 1028 | "libc", 1029 | ] 1030 | 1031 | [[package]] 1032 | name = "serde" 1033 | version = "1.0.177" 1034 | source = "registry+https://github.com/rust-lang/crates.io-index" 1035 | checksum = "63ba2516aa6bf82e0b19ca8b50019d52df58455d3cf9bdaf6315225fdd0c560a" 1036 | 1037 | [[package]] 1038 | name = "serde_json" 1039 | version = "1.0.104" 1040 | source = "registry+https://github.com/rust-lang/crates.io-index" 1041 | checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" 1042 | dependencies = [ 1043 | "itoa", 1044 | "ryu", 1045 | "serde", 1046 | ] 1047 | 1048 | [[package]] 1049 | name = "serde_urlencoded" 1050 | version = "0.7.1" 1051 | source = "registry+https://github.com/rust-lang/crates.io-index" 1052 | checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" 1053 | dependencies = [ 1054 | "form_urlencoded", 1055 | "itoa", 1056 | "ryu", 1057 | "serde", 1058 | ] 1059 | 1060 | [[package]] 1061 | name = "signal-hook-registry" 1062 | version = "1.4.1" 1063 | source = "registry+https://github.com/rust-lang/crates.io-index" 1064 | checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" 1065 | dependencies = [ 1066 | "libc", 1067 | ] 1068 | 1069 | [[package]] 1070 | name = "simplelog" 1071 | version = "0.12.1" 1072 | source = "registry+https://github.com/rust-lang/crates.io-index" 1073 | checksum = "acee08041c5de3d5048c8b3f6f13fafb3026b24ba43c6a695a0c76179b844369" 1074 | dependencies = [ 1075 | "log", 1076 | "paris", 1077 | "termcolor", 1078 | "time", 1079 | ] 1080 | 1081 | [[package]] 1082 | name = "slab" 1083 | version = "0.4.8" 1084 | source = "registry+https://github.com/rust-lang/crates.io-index" 1085 | checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" 1086 | dependencies = [ 1087 | "autocfg", 1088 | ] 1089 | 1090 | [[package]] 1091 | name = "smallvec" 1092 | version = "1.11.0" 1093 | source = "registry+https://github.com/rust-lang/crates.io-index" 1094 | checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" 1095 | 1096 | [[package]] 1097 | name = "socket2" 1098 | version = "0.4.9" 1099 | source = "registry+https://github.com/rust-lang/crates.io-index" 1100 | checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" 1101 | dependencies = [ 1102 | "libc", 1103 | "winapi", 1104 | ] 1105 | 1106 | [[package]] 1107 | name = "strsim" 1108 | version = "0.10.0" 1109 | source = "registry+https://github.com/rust-lang/crates.io-index" 1110 | checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" 1111 | 1112 | [[package]] 1113 | name = "syn" 1114 | version = "2.0.27" 1115 | source = "registry+https://github.com/rust-lang/crates.io-index" 1116 | checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0" 1117 | dependencies = [ 1118 | "proc-macro2", 1119 | "quote", 1120 | "unicode-ident", 1121 | ] 1122 | 1123 | [[package]] 1124 | name = "tempfile" 1125 | version = "3.7.0" 1126 | source = "registry+https://github.com/rust-lang/crates.io-index" 1127 | checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998" 1128 | dependencies = [ 1129 | "cfg-if", 1130 | "fastrand", 1131 | "redox_syscall", 1132 | "rustix", 1133 | "windows-sys 0.48.0", 1134 | ] 1135 | 1136 | [[package]] 1137 | name = "termcolor" 1138 | version = "1.1.3" 1139 | source = "registry+https://github.com/rust-lang/crates.io-index" 1140 | checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" 1141 | dependencies = [ 1142 | "winapi-util", 1143 | ] 1144 | 1145 | [[package]] 1146 | name = "thiserror" 1147 | version = "1.0.44" 1148 | source = "registry+https://github.com/rust-lang/crates.io-index" 1149 | checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" 1150 | dependencies = [ 1151 | "thiserror-impl", 1152 | ] 1153 | 1154 | [[package]] 1155 | name = "thiserror-impl" 1156 | version = "1.0.44" 1157 | source = "registry+https://github.com/rust-lang/crates.io-index" 1158 | checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" 1159 | dependencies = [ 1160 | "proc-macro2", 1161 | "quote", 1162 | "syn", 1163 | ] 1164 | 1165 | [[package]] 1166 | name = "time" 1167 | version = "0.3.24" 1168 | source = "registry+https://github.com/rust-lang/crates.io-index" 1169 | checksum = "b79eabcd964882a646b3584543ccabeae7869e9ac32a46f6f22b7a5bd405308b" 1170 | dependencies = [ 1171 | "deranged", 1172 | "itoa", 1173 | "libc", 1174 | "num_threads", 1175 | "serde", 1176 | "time-core", 1177 | "time-macros", 1178 | ] 1179 | 1180 | [[package]] 1181 | name = "time-core" 1182 | version = "0.1.1" 1183 | source = "registry+https://github.com/rust-lang/crates.io-index" 1184 | checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" 1185 | 1186 | [[package]] 1187 | name = "time-macros" 1188 | version = "0.2.11" 1189 | source = "registry+https://github.com/rust-lang/crates.io-index" 1190 | checksum = "eb71511c991639bb078fd5bf97757e03914361c48100d52878b8e52b46fb92cd" 1191 | dependencies = [ 1192 | "time-core", 1193 | ] 1194 | 1195 | [[package]] 1196 | name = "tinyvec" 1197 | version = "1.6.0" 1198 | source = "registry+https://github.com/rust-lang/crates.io-index" 1199 | checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" 1200 | dependencies = [ 1201 | "tinyvec_macros", 1202 | ] 1203 | 1204 | [[package]] 1205 | name = "tinyvec_macros" 1206 | version = "0.1.1" 1207 | source = "registry+https://github.com/rust-lang/crates.io-index" 1208 | checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" 1209 | 1210 | [[package]] 1211 | name = "tokio" 1212 | version = "1.29.1" 1213 | source = "registry+https://github.com/rust-lang/crates.io-index" 1214 | checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" 1215 | dependencies = [ 1216 | "autocfg", 1217 | "backtrace", 1218 | "bytes", 1219 | "libc", 1220 | "mio", 1221 | "num_cpus", 1222 | "parking_lot", 1223 | "pin-project-lite", 1224 | "signal-hook-registry", 1225 | "socket2", 1226 | "tokio-macros", 1227 | "windows-sys 0.48.0", 1228 | ] 1229 | 1230 | [[package]] 1231 | name = "tokio-macros" 1232 | version = "2.1.0" 1233 | source = "registry+https://github.com/rust-lang/crates.io-index" 1234 | checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" 1235 | dependencies = [ 1236 | "proc-macro2", 1237 | "quote", 1238 | "syn", 1239 | ] 1240 | 1241 | [[package]] 1242 | name = "tokio-native-tls" 1243 | version = "0.3.1" 1244 | source = "registry+https://github.com/rust-lang/crates.io-index" 1245 | checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" 1246 | dependencies = [ 1247 | "native-tls", 1248 | "tokio", 1249 | ] 1250 | 1251 | [[package]] 1252 | name = "tokio-util" 1253 | version = "0.7.8" 1254 | source = "registry+https://github.com/rust-lang/crates.io-index" 1255 | checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" 1256 | dependencies = [ 1257 | "bytes", 1258 | "futures-core", 1259 | "futures-sink", 1260 | "pin-project-lite", 1261 | "tokio", 1262 | "tracing", 1263 | ] 1264 | 1265 | [[package]] 1266 | name = "tower-service" 1267 | version = "0.3.2" 1268 | source = "registry+https://github.com/rust-lang/crates.io-index" 1269 | checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" 1270 | 1271 | [[package]] 1272 | name = "tracing" 1273 | version = "0.1.37" 1274 | source = "registry+https://github.com/rust-lang/crates.io-index" 1275 | checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" 1276 | dependencies = [ 1277 | "cfg-if", 1278 | "pin-project-lite", 1279 | "tracing-core", 1280 | ] 1281 | 1282 | [[package]] 1283 | name = "tracing-core" 1284 | version = "0.1.31" 1285 | source = "registry+https://github.com/rust-lang/crates.io-index" 1286 | checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" 1287 | dependencies = [ 1288 | "once_cell", 1289 | ] 1290 | 1291 | [[package]] 1292 | name = "try-lock" 1293 | version = "0.2.4" 1294 | source = "registry+https://github.com/rust-lang/crates.io-index" 1295 | checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" 1296 | 1297 | [[package]] 1298 | name = "unicode-bidi" 1299 | version = "0.3.13" 1300 | source = "registry+https://github.com/rust-lang/crates.io-index" 1301 | checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" 1302 | 1303 | [[package]] 1304 | name = "unicode-ident" 1305 | version = "1.0.11" 1306 | source = "registry+https://github.com/rust-lang/crates.io-index" 1307 | checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" 1308 | 1309 | [[package]] 1310 | name = "unicode-normalization" 1311 | version = "0.1.22" 1312 | source = "registry+https://github.com/rust-lang/crates.io-index" 1313 | checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" 1314 | dependencies = [ 1315 | "tinyvec", 1316 | ] 1317 | 1318 | [[package]] 1319 | name = "unicode-width" 1320 | version = "0.1.10" 1321 | source = "registry+https://github.com/rust-lang/crates.io-index" 1322 | checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" 1323 | 1324 | [[package]] 1325 | name = "url" 1326 | version = "2.4.0" 1327 | source = "registry+https://github.com/rust-lang/crates.io-index" 1328 | checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" 1329 | dependencies = [ 1330 | "form_urlencoded", 1331 | "idna", 1332 | "percent-encoding", 1333 | ] 1334 | 1335 | [[package]] 1336 | name = "utf8parse" 1337 | version = "0.2.1" 1338 | source = "registry+https://github.com/rust-lang/crates.io-index" 1339 | checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" 1340 | 1341 | [[package]] 1342 | name = "vcpkg" 1343 | version = "0.2.15" 1344 | source = "registry+https://github.com/rust-lang/crates.io-index" 1345 | checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" 1346 | 1347 | [[package]] 1348 | name = "want" 1349 | version = "0.3.1" 1350 | source = "registry+https://github.com/rust-lang/crates.io-index" 1351 | checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" 1352 | dependencies = [ 1353 | "try-lock", 1354 | ] 1355 | 1356 | [[package]] 1357 | name = "wasi" 1358 | version = "0.11.0+wasi-snapshot-preview1" 1359 | source = "registry+https://github.com/rust-lang/crates.io-index" 1360 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" 1361 | 1362 | [[package]] 1363 | name = "wasm-bindgen" 1364 | version = "0.2.87" 1365 | source = "registry+https://github.com/rust-lang/crates.io-index" 1366 | checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" 1367 | dependencies = [ 1368 | "cfg-if", 1369 | "wasm-bindgen-macro", 1370 | ] 1371 | 1372 | [[package]] 1373 | name = "wasm-bindgen-backend" 1374 | version = "0.2.87" 1375 | source = "registry+https://github.com/rust-lang/crates.io-index" 1376 | checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" 1377 | dependencies = [ 1378 | "bumpalo", 1379 | "log", 1380 | "once_cell", 1381 | "proc-macro2", 1382 | "quote", 1383 | "syn", 1384 | "wasm-bindgen-shared", 1385 | ] 1386 | 1387 | [[package]] 1388 | name = "wasm-bindgen-futures" 1389 | version = "0.4.37" 1390 | source = "registry+https://github.com/rust-lang/crates.io-index" 1391 | checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" 1392 | dependencies = [ 1393 | "cfg-if", 1394 | "js-sys", 1395 | "wasm-bindgen", 1396 | "web-sys", 1397 | ] 1398 | 1399 | [[package]] 1400 | name = "wasm-bindgen-macro" 1401 | version = "0.2.87" 1402 | source = "registry+https://github.com/rust-lang/crates.io-index" 1403 | checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" 1404 | dependencies = [ 1405 | "quote", 1406 | "wasm-bindgen-macro-support", 1407 | ] 1408 | 1409 | [[package]] 1410 | name = "wasm-bindgen-macro-support" 1411 | version = "0.2.87" 1412 | source = "registry+https://github.com/rust-lang/crates.io-index" 1413 | checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" 1414 | dependencies = [ 1415 | "proc-macro2", 1416 | "quote", 1417 | "syn", 1418 | "wasm-bindgen-backend", 1419 | "wasm-bindgen-shared", 1420 | ] 1421 | 1422 | [[package]] 1423 | name = "wasm-bindgen-shared" 1424 | version = "0.2.87" 1425 | source = "registry+https://github.com/rust-lang/crates.io-index" 1426 | checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" 1427 | 1428 | [[package]] 1429 | name = "wasm-streams" 1430 | version = "0.2.3" 1431 | source = "registry+https://github.com/rust-lang/crates.io-index" 1432 | checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" 1433 | dependencies = [ 1434 | "futures-util", 1435 | "js-sys", 1436 | "wasm-bindgen", 1437 | "wasm-bindgen-futures", 1438 | "web-sys", 1439 | ] 1440 | 1441 | [[package]] 1442 | name = "web-sys" 1443 | version = "0.3.64" 1444 | source = "registry+https://github.com/rust-lang/crates.io-index" 1445 | checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" 1446 | dependencies = [ 1447 | "js-sys", 1448 | "wasm-bindgen", 1449 | ] 1450 | 1451 | [[package]] 1452 | name = "winapi" 1453 | version = "0.3.9" 1454 | source = "registry+https://github.com/rust-lang/crates.io-index" 1455 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 1456 | dependencies = [ 1457 | "winapi-i686-pc-windows-gnu", 1458 | "winapi-x86_64-pc-windows-gnu", 1459 | ] 1460 | 1461 | [[package]] 1462 | name = "winapi-i686-pc-windows-gnu" 1463 | version = "0.4.0" 1464 | source = "registry+https://github.com/rust-lang/crates.io-index" 1465 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 1466 | 1467 | [[package]] 1468 | name = "winapi-util" 1469 | version = "0.1.5" 1470 | source = "registry+https://github.com/rust-lang/crates.io-index" 1471 | checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" 1472 | dependencies = [ 1473 | "winapi", 1474 | ] 1475 | 1476 | [[package]] 1477 | name = "winapi-x86_64-pc-windows-gnu" 1478 | version = "0.4.0" 1479 | source = "registry+https://github.com/rust-lang/crates.io-index" 1480 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 1481 | 1482 | [[package]] 1483 | name = "windows-sys" 1484 | version = "0.45.0" 1485 | source = "registry+https://github.com/rust-lang/crates.io-index" 1486 | checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" 1487 | dependencies = [ 1488 | "windows-targets 0.42.2", 1489 | ] 1490 | 1491 | [[package]] 1492 | name = "windows-sys" 1493 | version = "0.48.0" 1494 | source = "registry+https://github.com/rust-lang/crates.io-index" 1495 | checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" 1496 | dependencies = [ 1497 | "windows-targets 0.48.1", 1498 | ] 1499 | 1500 | [[package]] 1501 | name = "windows-targets" 1502 | version = "0.42.2" 1503 | source = "registry+https://github.com/rust-lang/crates.io-index" 1504 | checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" 1505 | dependencies = [ 1506 | "windows_aarch64_gnullvm 0.42.2", 1507 | "windows_aarch64_msvc 0.42.2", 1508 | "windows_i686_gnu 0.42.2", 1509 | "windows_i686_msvc 0.42.2", 1510 | "windows_x86_64_gnu 0.42.2", 1511 | "windows_x86_64_gnullvm 0.42.2", 1512 | "windows_x86_64_msvc 0.42.2", 1513 | ] 1514 | 1515 | [[package]] 1516 | name = "windows-targets" 1517 | version = "0.48.1" 1518 | source = "registry+https://github.com/rust-lang/crates.io-index" 1519 | checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" 1520 | dependencies = [ 1521 | "windows_aarch64_gnullvm 0.48.0", 1522 | "windows_aarch64_msvc 0.48.0", 1523 | "windows_i686_gnu 0.48.0", 1524 | "windows_i686_msvc 0.48.0", 1525 | "windows_x86_64_gnu 0.48.0", 1526 | "windows_x86_64_gnullvm 0.48.0", 1527 | "windows_x86_64_msvc 0.48.0", 1528 | ] 1529 | 1530 | [[package]] 1531 | name = "windows_aarch64_gnullvm" 1532 | version = "0.42.2" 1533 | source = "registry+https://github.com/rust-lang/crates.io-index" 1534 | checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" 1535 | 1536 | [[package]] 1537 | name = "windows_aarch64_gnullvm" 1538 | version = "0.48.0" 1539 | source = "registry+https://github.com/rust-lang/crates.io-index" 1540 | checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" 1541 | 1542 | [[package]] 1543 | name = "windows_aarch64_msvc" 1544 | version = "0.42.2" 1545 | source = "registry+https://github.com/rust-lang/crates.io-index" 1546 | checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" 1547 | 1548 | [[package]] 1549 | name = "windows_aarch64_msvc" 1550 | version = "0.48.0" 1551 | source = "registry+https://github.com/rust-lang/crates.io-index" 1552 | checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" 1553 | 1554 | [[package]] 1555 | name = "windows_i686_gnu" 1556 | version = "0.42.2" 1557 | source = "registry+https://github.com/rust-lang/crates.io-index" 1558 | checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" 1559 | 1560 | [[package]] 1561 | name = "windows_i686_gnu" 1562 | version = "0.48.0" 1563 | source = "registry+https://github.com/rust-lang/crates.io-index" 1564 | checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" 1565 | 1566 | [[package]] 1567 | name = "windows_i686_msvc" 1568 | version = "0.42.2" 1569 | source = "registry+https://github.com/rust-lang/crates.io-index" 1570 | checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" 1571 | 1572 | [[package]] 1573 | name = "windows_i686_msvc" 1574 | version = "0.48.0" 1575 | source = "registry+https://github.com/rust-lang/crates.io-index" 1576 | checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" 1577 | 1578 | [[package]] 1579 | name = "windows_x86_64_gnu" 1580 | version = "0.42.2" 1581 | source = "registry+https://github.com/rust-lang/crates.io-index" 1582 | checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" 1583 | 1584 | [[package]] 1585 | name = "windows_x86_64_gnu" 1586 | version = "0.48.0" 1587 | source = "registry+https://github.com/rust-lang/crates.io-index" 1588 | checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" 1589 | 1590 | [[package]] 1591 | name = "windows_x86_64_gnullvm" 1592 | version = "0.42.2" 1593 | source = "registry+https://github.com/rust-lang/crates.io-index" 1594 | checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" 1595 | 1596 | [[package]] 1597 | name = "windows_x86_64_gnullvm" 1598 | version = "0.48.0" 1599 | source = "registry+https://github.com/rust-lang/crates.io-index" 1600 | checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" 1601 | 1602 | [[package]] 1603 | name = "windows_x86_64_msvc" 1604 | version = "0.42.2" 1605 | source = "registry+https://github.com/rust-lang/crates.io-index" 1606 | checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" 1607 | 1608 | [[package]] 1609 | name = "windows_x86_64_msvc" 1610 | version = "0.48.0" 1611 | source = "registry+https://github.com/rust-lang/crates.io-index" 1612 | checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" 1613 | 1614 | [[package]] 1615 | name = "winreg" 1616 | version = "0.10.1" 1617 | source = "registry+https://github.com/rust-lang/crates.io-index" 1618 | checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" 1619 | dependencies = [ 1620 | "winapi", 1621 | ] 1622 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "dlrs" 3 | description = "Multi-protocol CLI download accelerator" 4 | license = "MIT" 5 | keywords = ["cli", "async", "download-tool"] 6 | version = "0.1.2" 7 | edition = "2021" 8 | readme = "README.md" 9 | 10 | [dependencies] 11 | async-channel = "1.9.0" 12 | async-recursion = "1.0.4" 13 | async-stream = "0.3.5" 14 | bytes = "1.4.0" 15 | circular-buffer = "0.1.1" 16 | clap = { version = "4.3.19", features = ["derive"] } 17 | futures = "0.3.28" 18 | if_chain = "1.0.2" 19 | indicatif = "0.17.5" 20 | indicatif-log-bridge = "0.2.1" 21 | log = "0.4.19" 22 | reqwest = { version = "0.11.18", features = ["blocking", "stream"] } 23 | simplelog = { version = "0.12.1", features = ["paris"] } 24 | thiserror = "1.0.44" 25 | tokio = { version = "1.29.1", features = ["full"] } 26 | url = "2.4.0" 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) [year] [fullname] 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `dlrs` 2 | 3 | `dlrs` is a multi-protocol download tool for the command line. Written in Rust, dlrs 4 | splits your downloads to speed them up where possible. 5 | 6 | ![dlrs demo](./examples/demo.gif) 7 | 8 | ## Features 9 | 10 | - [x] **Split downloads:** dlrs can split your download into multiple segments that are then pulled in parallel, 11 | dramatically speeding them up for hosts that limit download speeds per connection. This is the same technique 12 | used by tools such as IDM and XDM. 13 | - [ ] **Use multiple hosts:** Downloading a file hosted by multiple speed-limiting hosts? dlrs can download the same file 14 | from multiple sources in parallel! 15 | - [ ] **Multiple protocols:** dlrs supports HTTP/HTTPS and FTP/SFTP. 16 | - [x] **Lightweight:** Written in Rust, dlrs comes packaged in a single binary at \_\_ MB. 17 | - [ ] **Remote Control (_planned_ )** RPC interface to control a dlrs daemon. Can be used for remote automation and building 18 | graphical interfaces on top of dlrs. 19 | 20 | ## Installation 21 | 22 | While automated builds are planned, `dlrs` can be installed with [cargo][cargo] on any machine: 23 | 24 | ```shell 25 | cargo install dlrs 26 | ``` 27 | 28 | ## Usage Examples 29 | 30 | - `dlrs -s 2 https://example.org/some-linux.iso` Download file with 2 splits. 31 | 32 | 33 | [cargo]: https://doc.rust-lang.org/cargo/ 34 | 35 | ## How it works 36 | 37 | The diagram describes most working parts of `dlrs`. It's close to what you'll find on reading the code. 38 | 39 | ![dlrs concurrency diagram](./assets/concurrency-diagram.png) 40 | 41 | ## License 42 | 43 | The project is [licensed](./LICENSE) under the MIT license. 44 | 45 | ## Contributing 46 | 47 | Both issues and pull requests are accepted. I welcome contributors to take a dig at making my terrible code less 48 | terrible. 🐙 49 | -------------------------------------------------------------------------------- /assets/concurrency-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ditsuke/dlrs/25945ef03008594827aa383464abcec09f5723af/assets/concurrency-diagram.png -------------------------------------------------------------------------------- /cspell.json: -------------------------------------------------------------------------------- 1 | {"words":["dlrs","reqwest","thiserror","simplelog","indicatif","keepalive"],"language":"en","version":"0.2","flagWords":[]} 2 | -------------------------------------------------------------------------------- /examples/demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ditsuke/dlrs/25945ef03008594827aa383464abcec09f5723af/examples/demo.gif -------------------------------------------------------------------------------- /src/downloader.rs: -------------------------------------------------------------------------------- 1 | use std::cmp; 2 | use std::error::Error; 3 | use std::io::SeekFrom; 4 | 5 | use bytes::Bytes; 6 | 7 | use futures::{future, TryStreamExt}; 8 | 9 | use indicatif::MultiProgress; 10 | 11 | 12 | use thiserror::Error; 13 | use tokio::fs::{File, OpenOptions}; 14 | use tokio::io::{AsyncSeekExt, AsyncWriteExt}; 15 | 16 | use tokio::{sync::mpsc, task::JoinHandle}; 17 | use url::Url; 18 | 19 | 20 | use crate::progress_reporter::spawn_progress_reporter; 21 | use crate::resource::{ResourceError}; 22 | 23 | use crate::resource::ResourceHandle; 24 | use crate::shared_types::{ByteCount, ChunkRange}; 25 | 26 | const MB_TO_BYTES: u32 = 1024 * 1024; 27 | const CHUNK_SIZE: u32 = 2 * MB_TO_BYTES; 28 | 29 | type ChunkBoundaries = Option; 30 | type FileChunk = (Bytes, ChunkBoundaries); 31 | type ChunkSpec = (ResourceHandle, ChunkBoundaries); 32 | enum DownloadUpdate { 33 | Chunk(Result), 34 | Complete, 35 | } 36 | 37 | pub(crate) struct DownloadPreferences { 38 | pub(crate) url: Url, 39 | pub(crate) preferred_splits: u8, 40 | pub(crate) output: Option, 41 | } 42 | 43 | pub(crate) async fn start_download( 44 | specs: DownloadPreferences, 45 | multi: MultiProgress, 46 | ) -> Result<(), Box> { 47 | let handle = ResourceHandle::try_from(&specs.url)?; 48 | 49 | let resource_specs = handle.get_specs().await?; 50 | debug!("File to download has specs: {:?}", resource_specs); 51 | 52 | let output_filename = match specs.output { 53 | Some(filename) => filename, 54 | None => { 55 | if let Some(filename) = resource_specs.inferred_filename { 56 | filename 57 | } else { 58 | "download".into() // TODO: come up with a better default way 59 | } 60 | } 61 | }; 62 | debug!("Output filename: {}", output_filename); 63 | 64 | let file_size = resource_specs.size; 65 | let chunk_count = match (file_size, resource_specs.supports_splits) { 66 | (Some(size), true) => Some((size as f32 / CHUNK_SIZE as f32).ceil() as u32), 67 | _ => None, 68 | }; 69 | debug!("File size: {:?}, chunk count: {chunk_count:?}", file_size); 70 | let chunk_bounds = match chunk_count { 71 | Some(count) => (0..count) 72 | .map(|i| { 73 | let start = i * CHUNK_SIZE; 74 | let end = cmp::min(start + CHUNK_SIZE, file_size.unwrap_or(0)); 75 | Some(ChunkRange { start, end }) 76 | }) 77 | .collect::>(), 78 | None => vec![None], 79 | }; 80 | 81 | let download_worker_count = if resource_specs.supports_splits { 82 | specs.preferred_splits 83 | } else { 84 | 1 85 | }; 86 | debug!("downloading with {download_worker_count} workers"); 87 | 88 | let (s_chunks, r_update) = mpsc::channel::(download_worker_count as usize); 89 | let (s_processing_q, r_processing_q) = async_channel::unbounded::(); 90 | let (s_progress, r_progress) = mpsc::channel::(download_worker_count as usize); 91 | 92 | let mut handles = vec![]; 93 | 94 | // writer/processor 95 | let output_file = OpenOptions::new() 96 | .create(true) 97 | .read(true) 98 | .write(true) 99 | .open(output_filename) 100 | .await?; 101 | 102 | // TODO: we'd have to leave this untouched if we're resuming a download 103 | output_file.set_len(file_size.unwrap_or(0) as u64).await?; 104 | let writer = Writer { 105 | output_file, 106 | chunk_count, 107 | r_chunks: r_update, 108 | s_processing_q: s_processing_q.clone(), 109 | }; 110 | handles.push(writer.spawn()); 111 | 112 | // download workers 113 | let download_worker = DownloadWorker { 114 | r_processing_q: r_processing_q.clone(), 115 | s_chunks: s_chunks.clone(), 116 | s_progress: s_progress.clone(), 117 | single_chunk_dl: matches!(chunk_count, Some(1) | None), 118 | }; 119 | for _ in 0..download_worker_count { 120 | let download_worker = download_worker.clone(); 121 | handles.push(download_worker.spawn()); 122 | } 123 | 124 | spawn_progress_reporter(file_size, r_progress, multi); 125 | 126 | // Send splits to download queue 127 | future::join_all( 128 | chunk_bounds 129 | .iter() 130 | .map(|bound| s_processing_q.send((handle.clone(), *bound))) 131 | .collect::>(), 132 | ) 133 | .await; 134 | 135 | futures::future::join_all(handles).await; 136 | debug!("exiting download function"); 137 | Ok(()) 138 | } 139 | 140 | #[derive(Clone, Debug)] 141 | struct DownloadWorker { 142 | r_processing_q: async_channel::Receiver, 143 | s_chunks: mpsc::Sender, 144 | s_progress: mpsc::Sender, 145 | single_chunk_dl: bool, 146 | } 147 | 148 | impl DownloadWorker { 149 | fn spawn(self) -> JoinHandle<()> { 150 | tokio::spawn(async move { 151 | while let Ok((handle, bounds)) = self.r_processing_q.recv().await { 152 | let r = handle 153 | .stream_range(bounds, self.s_progress.clone()) 154 | .await 155 | .try_for_each(|chunk| async { 156 | debug!("sending chunk for bounds {bounds:?} to writer"); 157 | self.s_chunks 158 | .send(DownloadUpdate::Chunk(Ok((chunk, bounds)))) 159 | .await 160 | .expect("failed to send file chunk to writer"); 161 | Ok(()) 162 | }) 163 | .await; 164 | if let Err(e) = r { 165 | self.s_chunks 166 | .send(DownloadUpdate::Chunk(Err((e, (handle, bounds))))) 167 | .await 168 | .expect("failed to send error to writer"); 169 | } 170 | 171 | if self.single_chunk_dl { 172 | self.s_chunks.send(DownloadUpdate::Complete).await.ok(); 173 | } 174 | } 175 | }) 176 | } 177 | } 178 | 179 | macro_rules! windup_writer { 180 | ($file:ident, $s_processing:ident, $r_chunks:ident) => { 181 | tokio::time::sleep(std::time::Duration::from_millis(100)).await; 182 | $file.shutdown().await.expect("shutdown failed"); 183 | $s_processing.close(); 184 | $r_chunks.close(); 185 | break; 186 | }; 187 | } 188 | 189 | #[derive(Debug, Clone, Copy)] 190 | enum ChunkState { 191 | Done, 192 | Pending, 193 | } 194 | 195 | struct Writer { 196 | output_file: File, 197 | chunk_count: Option, 198 | r_chunks: mpsc::Receiver, 199 | s_processing_q: async_channel::Sender, 200 | } 201 | 202 | impl Writer { 203 | fn spawn(self) -> JoinHandle<()> { 204 | let chunk_count = self.chunk_count; 205 | let s_processing_q = self.s_processing_q; 206 | let mut r_chunks = self.r_chunks; 207 | let mut output_file = self.output_file; 208 | let mut chunk_states = chunk_count.map(|count| vec![ChunkState::Pending; count as usize]); 209 | 210 | tokio::spawn(async move { 211 | let mut chunks_received = 0; 212 | while let Some(chunk) = r_chunks.recv().await { 213 | match chunk { 214 | DownloadUpdate::Chunk(Ok(chunk)) => { 215 | write_file_chunk(&mut output_file, &chunk) 216 | .await 217 | .expect("write failed"); 218 | let (_, chunk_range) = chunk; 219 | let chunk_index = if let Some(range) = chunk_range { 220 | range.start / CHUNK_SIZE 221 | } else { 222 | 0 223 | } as usize; 224 | if let Some(ref mut states) = chunk_states { 225 | states[chunk_index] = ChunkState::Done; 226 | } 227 | 228 | chunks_received += 1; 229 | let we_are_done = match chunk_count { 230 | Some(count) => chunks_received == count, 231 | None => false, 232 | }; 233 | if we_are_done { 234 | debug!("download complete (all chunks received)"); 235 | windup_writer!(output_file, s_processing_q, r_chunks); 236 | } 237 | } 238 | 239 | DownloadUpdate::Chunk(Err((e, chunk_spec))) => { 240 | error!("Error downloading chunk: {}. Retrying...", e); 241 | s_processing_q 242 | .clone() 243 | .send(chunk_spec) 244 | .await 245 | .expect("send failed"); 246 | } 247 | 248 | DownloadUpdate::Complete => { 249 | debug!("download complete (completion signal received)"); 250 | windup_writer!(output_file, s_processing_q, r_chunks); 251 | } 252 | } 253 | } 254 | }) 255 | } 256 | } 257 | 258 | #[derive(Error, Debug)] 259 | enum WriteChunkError {} 260 | async fn write_file_chunk(file: &mut File, chunk: &FileChunk) -> Result<(), WriteChunkError> { 261 | if let Some(range) = &chunk.1 { 262 | file.seek(SeekFrom::Start(range.start as u64)) 263 | .await 264 | .expect("seek failed"); 265 | } 266 | file.write_all(chunk.0.as_ref()) 267 | .await 268 | .expect("write failed"); 269 | file.flush().await.expect("flush failed"); 270 | Ok(()) 271 | } 272 | -------------------------------------------------------------------------------- /src/http_utils.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | 3 | use bytes::Bytes; 4 | use futures::{Stream, TryStreamExt}; 5 | use reqwest::{header::HeaderMap, Client}; 6 | use thiserror::Error; 7 | use url::Url; 8 | 9 | use crate::shared_types::ChunkRange; 10 | 11 | #[async_recursion::async_recursion] 12 | pub(crate) async fn get_headers_follow_redirects( 13 | client: &Client, 14 | url: &Url, 15 | ) -> Result<(HeaderMap, Url), Box> { 16 | let headers = client 17 | .head(url.as_str()) 18 | .send() 19 | .await? 20 | .error_for_status()? 21 | .headers() 22 | .to_owned(); 23 | if headers.get("Location").is_some() { 24 | debug!("Redirecting to {:?}", headers.get("Location").unwrap()); 25 | let new_url = headers.get("Location").unwrap().to_str().unwrap(); 26 | let new_url = Url::parse(new_url)?; 27 | get_headers_follow_redirects(client, &new_url).await?; 28 | } 29 | Ok((headers, url.clone())) 30 | } 31 | 32 | #[derive(Error, Debug)] 33 | pub(crate) enum GetError { 34 | #[error("Failed to GET resource: {0}")] 35 | Http(#[from] reqwest::Error), 36 | #[error("Resource handle does not support partial content (RANGE)")] 37 | NoPartialContentSupportWhenceRequested, 38 | } 39 | 40 | pub(crate) async fn get_stream( 41 | client: &Client, 42 | url: &Url, 43 | range: Option, 44 | ) -> Result>, GetError> { 45 | let mut headers = reqwest::header::HeaderMap::new(); 46 | if let Some(range) = range { 47 | headers.insert( 48 | reqwest::header::RANGE, 49 | format!("bytes={}-{}", range.start, range.end) 50 | .parse() 51 | .unwrap(), 52 | ); 53 | } 54 | 55 | let response = client 56 | .get(url.to_owned()) 57 | .headers(headers) 58 | .send() 59 | .await? 60 | .error_for_status()?; 61 | 62 | if range.is_some() && response.status() != reqwest::StatusCode::PARTIAL_CONTENT { 63 | return Err(GetError::NoPartialContentSupportWhenceRequested); 64 | } 65 | 66 | Ok(response.bytes_stream().map_err(GetError::from)) 67 | } 68 | 69 | pub(crate) fn get_file_name_from_headers(headers: &HeaderMap) -> Option { 70 | debug!("headers are {headers:?}"); 71 | headers 72 | .get("Content-Disposition")? 73 | .to_str() 74 | .ok()? 75 | .split(';') 76 | .map(|s| s.trim()) 77 | .find(|s| s.starts_with("filename=")) 78 | .map(|s| s.replace("filename=", "")) 79 | } 80 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | extern crate log; 3 | extern crate simplelog; 4 | 5 | mod downloader; 6 | mod http_utils; 7 | mod progress_reporter; 8 | mod resource; 9 | mod shared_types; 10 | 11 | use std::error::Error; 12 | 13 | use clap::Parser; 14 | use downloader::{start_download, DownloadPreferences}; 15 | 16 | use indicatif::MultiProgress; 17 | use indicatif_log_bridge::LogWrapper; 18 | use simplelog::TermLogger; 19 | use url::Url; 20 | 21 | #[derive(Parser, Debug)] 22 | #[command(author, version, about, long_about = None)] 23 | struct CliArgs { 24 | #[arg(short, long, default_value = "1")] 25 | splits: u8, 26 | 27 | #[arg(index = 1)] 28 | url: String, 29 | 30 | #[arg(short, long, default_value = None)] 31 | output: Option, 32 | } 33 | 34 | #[tokio::main] 35 | async fn main() -> Result<(), Box> { 36 | let multi_progress = MultiProgress::new(); 37 | let logger = TermLogger::new( 38 | simplelog::LevelFilter::Error, 39 | simplelog::Config::default(), 40 | simplelog::TerminalMode::Mixed, 41 | simplelog::ColorChoice::Auto, 42 | ); 43 | 44 | LogWrapper::new(multi_progress.clone(), logger) 45 | .try_init() 46 | .expect("failed to init global logger"); 47 | 48 | let args = CliArgs::parse(); 49 | 50 | let url = Url::parse(args.url.as_ref())?; 51 | let preferred_splits = args.splits; 52 | 53 | let preferences = DownloadPreferences { 54 | url, 55 | preferred_splits, 56 | output: args.output, 57 | }; 58 | 59 | start_download(preferences, multi_progress).await?; 60 | 61 | debug!("Download complete"); 62 | log::logger().flush(); 63 | Ok(()) 64 | } 65 | -------------------------------------------------------------------------------- /src/progress_reporter.rs: -------------------------------------------------------------------------------- 1 | use std::sync::Arc; 2 | use std::time::Duration; 3 | 4 | use circular_buffer::CircularBuffer; 5 | 6 | use indicatif::{MultiProgress, ProgressBar, ProgressState, ProgressStyle}; 7 | 8 | use tokio::sync::RwLock; 9 | use tokio::time::Instant; 10 | use tokio::{sync::mpsc, task::JoinHandle}; 11 | 12 | use crate::shared_types::ByteCount; 13 | 14 | pub(crate) struct ProgressReporter { 15 | rx_progress: mpsc::Receiver, 16 | total_size: Option, 17 | multi_progress: MultiProgress, 18 | } 19 | 20 | impl ProgressReporter { 21 | pub(crate) fn new( 22 | rx_progress: mpsc::Receiver, 23 | total_size: Option, 24 | multi_progress: MultiProgress, 25 | ) -> Self { 26 | Self { 27 | rx_progress, 28 | total_size, 29 | multi_progress, 30 | } 31 | } 32 | 33 | pub(crate) fn spawn(self) -> JoinHandle<()> { 34 | spawn_progress_reporter(self.total_size, self.rx_progress, self.multi_progress) 35 | } 36 | } 37 | 38 | pub(crate) fn spawn_progress_reporter( 39 | total_size: Option, 40 | mut rx_progress: mpsc::Receiver, 41 | multi: MultiProgress, 42 | ) -> JoinHandle<()> { 43 | tokio::spawn(async move { 44 | let mut progress = 0; 45 | type ProgressPoint = (ByteCount, Instant); 46 | let progress_q = Arc::new(RwLock::new(CircularBuffer::<50, ProgressPoint>::new())); 47 | let pb = total_size.map_or_else(ProgressBar::new_spinner, |size| { 48 | ProgressBar::new(size as u64) 49 | }); 50 | let pb = multi.add(pb); 51 | pb.set_style(ProgressStyle::with_template("{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({eta}) ({msg})") 52 | .unwrap() 53 | .with_key("eta", |state: &ProgressState, w: &mut dyn std::fmt::Write| write!(w, "{:.1}s", state.eta().as_secs_f64()).unwrap()) 54 | .progress_chars("#>-")); 55 | 56 | // Spawn a task to update the speed every 500ms 57 | { 58 | let progress_q = progress_q.clone(); 59 | let pb = pb.clone(); 60 | const UPDATE_INTERVAL: Duration = Duration::from_millis(500); 61 | tokio::spawn(async move { 62 | loop { 63 | tokio::time::sleep(UPDATE_INTERVAL).await; 64 | let q = progress_q.read().await; 65 | let back = q.back(); 66 | let front = q.front(); 67 | if let ( 68 | Some((latest_byte, latest_instant)), 69 | Some((oldest_byte, oldest_instant)), 70 | ) = (back, front) 71 | { 72 | if latest_byte == oldest_byte { 73 | continue; 74 | } 75 | let speed = (latest_byte - oldest_byte) as f64 76 | / latest_instant.duration_since(*oldest_instant).as_secs_f64(); 77 | let (unit, speed) = if speed > 1024.0 { 78 | ("MB/s", speed / (1024.0 * 1024.0)) 79 | } else { 80 | ("kB/s", speed / 1024.0) 81 | }; 82 | pb.set_message(format!("{:.1} {}", speed, unit)); 83 | } 84 | } 85 | }); 86 | } 87 | 88 | while let Some(chunk_size) = rx_progress.recv().await { 89 | progress += chunk_size; 90 | pb.set_position(progress); 91 | let mut q = progress_q.write().await; 92 | q.push_back((progress, Instant::now())); 93 | } 94 | let speed = progress as f64 / pb.elapsed().as_secs_f64(); 95 | let (unit, speed) = if speed > 1024.0 { 96 | ("MB/s", speed / (1024.0 * 1024.0)) 97 | } else { 98 | ("kB/s", speed / 1024.0) 99 | }; 100 | pb.finish_with_message(format!("({:.1} {})", speed, unit)); 101 | println!(""); 102 | }) 103 | } 104 | -------------------------------------------------------------------------------- /src/resource.rs: -------------------------------------------------------------------------------- 1 | use std::error::Error; 2 | use std::time::Duration; 3 | 4 | use async_stream::try_stream; 5 | use bytes::{Bytes, BytesMut}; 6 | 7 | use futures::{Stream, StreamExt}; 8 | 9 | use thiserror::Error; 10 | 11 | use tokio::sync::mpsc; 12 | use url::Url; 13 | 14 | use crate::http_utils; 15 | use crate::shared_types::{ByteCount, ChunkRange}; 16 | 17 | #[derive(Error, Debug)] 18 | pub(crate) enum ReadError { 19 | #[error("HTTP error: {0}")] 20 | Http(reqwest::Error), 21 | #[error("FTP error: {0}")] 22 | _Ftp(String), 23 | } 24 | 25 | #[derive(Error, Debug)] 26 | pub(crate) enum ResourceError { 27 | #[error("Resource read error: {0}")] 28 | ReadError(ReadError), 29 | #[error("Handle does not support partial content")] 30 | NoPartialContentSupport, 31 | } 32 | 33 | impl From for ResourceError { 34 | fn from(e: http_utils::GetError) -> Self { 35 | match e { 36 | http_utils::GetError::Http(e) => ResourceError::ReadError(ReadError::Http(e)), 37 | http_utils::GetError::NoPartialContentSupportWhenceRequested => { 38 | ResourceError::NoPartialContentSupport 39 | } 40 | } 41 | } 42 | } 43 | 44 | #[derive(Debug, Clone)] 45 | pub(crate) enum ResourceHandle { 46 | Ftp(Url), 47 | Http { url: Url, client: reqwest::Client }, 48 | } 49 | 50 | #[derive(Error, Debug)] 51 | pub(crate) enum HandleCreationError { 52 | #[error("Unsupported protocol: {protocol}")] 53 | UnsupportedProtocol { protocol: String }, 54 | #[error("Http client error: {0}")] 55 | Http(#[from] reqwest::Error), 56 | } 57 | 58 | impl TryFrom<&Url> for ResourceHandle { 59 | type Error = HandleCreationError; 60 | fn try_from(url: &Url) -> Result { 61 | match url.scheme() { 62 | "http" | "https" => Ok(ResourceHandle::Http { 63 | url: url.to_owned(), 64 | client: reqwest::Client::builder() 65 | .tcp_keepalive(Some(Duration::from_secs(2))) 66 | .build()?, 67 | }), 68 | "ftp" => Ok(ResourceHandle::Ftp(url.to_owned())), 69 | scheme => Err(HandleCreationError::UnsupportedProtocol { 70 | protocol: scheme.into(), 71 | }), 72 | } 73 | } 74 | } 75 | 76 | #[derive(Debug)] 77 | pub(crate) struct ResourceSpec { 78 | pub(crate) url: Url, 79 | pub(crate) size: Option, 80 | pub(crate) supports_splits: bool, 81 | pub(crate) inferred_filename: Option, 82 | } 83 | 84 | impl ResourceHandle { 85 | pub(crate) async fn get_specs(&self) -> Result> { 86 | match self { 87 | ResourceHandle::Ftp(_url) => { 88 | todo!() 89 | } 90 | ResourceHandle::Http { client, url } => { 91 | let (headers, url) = http_utils::get_headers_follow_redirects(client, url).await?; 92 | debug!("headers: {:?}", headers); 93 | 94 | let supports_splits = headers 95 | .get("Accept-Ranges") 96 | .map(|v| { 97 | v.to_str() 98 | .map_err(|e| { 99 | format!("failed to map Accept-Ranges from {url} to a string: {e}") 100 | }) 101 | .unwrap() 102 | == "bytes" 103 | }) 104 | .unwrap_or(false); 105 | 106 | let mut size = headers 107 | .get("Content-Length") 108 | .map(|v| v.to_str().unwrap().parse::().unwrap()); 109 | if size == Some(0) { 110 | size = None; 111 | } 112 | 113 | let inferred_filename = http_utils::get_file_name_from_headers(&headers) 114 | .unwrap_or_else(|| url.path_segments().unwrap().last().unwrap().to_owned()); 115 | let inferred_filename = if inferred_filename.is_empty() { 116 | None 117 | } else { 118 | Some(inferred_filename) 119 | }; 120 | 121 | Ok(ResourceSpec { 122 | url: url.clone(), 123 | size, 124 | supports_splits, 125 | inferred_filename, 126 | }) 127 | } 128 | } 129 | } 130 | 131 | pub(crate) async fn stream_range( 132 | &self, 133 | range: Option, 134 | s_progress: mpsc::Sender, 135 | ) -> impl Stream> { 136 | match self { 137 | ResourceHandle::Http { client, url } => { 138 | let (client, url) = (client.clone(), url.clone()); 139 | try_stream! { 140 | let mut stream = http_utils::get_stream(&client, &url, range).await?; 141 | 142 | // HACK: this feels so wrong on so many levels. 143 | // While I can't exactly figure out a "good" way to do this, 144 | // The solution should be verification of chunks and whether 145 | // they're done or not on the end of the writer. 146 | // OKAY, idea: the writer maintains an array of of chunks, 147 | // keeping track Chunk::ToBeWritten, along with the start 148 | // and end of the chunk. When a chunk is written, it's 149 | // marked as Chunk::Written only if the entire chunk was 150 | // written. If the chunk was only partially written, it's 151 | // Chunk::ToBeWritten bytecount is updated to reflect the 152 | // remaining bytes to be written. Once the chunk is 153 | // completely written we can update the chunks_written 154 | // counter and move on to the next chunk, thus allowing 155 | // it to figure out the termination condition on its own too. 156 | let buffer_capacity = if let Some(range) = range { 157 | (range.end - range.start + 1) as usize 158 | } else { 159 | 1000 // TODO: WTF`` 160 | }; 161 | let mut buffer = BytesMut::with_capacity(buffer_capacity); 162 | let mut cumulative = 0; 163 | 164 | while let Some(sub_chunk) = stream.next().await { 165 | let sub_chunk = sub_chunk?; 166 | cumulative += sub_chunk.len(); 167 | s_progress.try_send(sub_chunk.len() as ByteCount).ok(); 168 | buffer.extend_from_slice(&sub_chunk); 169 | if cumulative >= buffer_capacity { 170 | yield buffer.freeze(); 171 | cumulative = 0; 172 | buffer = BytesMut::with_capacity(buffer_capacity); 173 | } 174 | } 175 | 176 | if !buffer.is_empty() { 177 | yield buffer.freeze(); 178 | } 179 | } 180 | } 181 | _ => todo!(), 182 | } 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /src/shared_types.rs: -------------------------------------------------------------------------------- 1 | #[derive(Clone, Copy, Debug)] 2 | pub(crate) struct ChunkRange { 3 | pub(crate) start: u32, 4 | pub(crate) end: u32, 5 | } 6 | 7 | pub(crate) type ByteCount = u64; 8 | --------------------------------------------------------------------------------