├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── README.md └── src ├── client.rs ├── lib.rs └── main.rs /.gitignore: -------------------------------------------------------------------------------- 1 | /target -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | [[package]] 4 | name = "aho-corasick" 5 | version = "0.7.18" 6 | source = "registry+https://github.com/rust-lang/crates.io-index" 7 | checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" 8 | dependencies = [ 9 | "memchr", 10 | ] 11 | 12 | [[package]] 13 | name = "ansi_term" 14 | version = "0.11.0" 15 | source = "registry+https://github.com/rust-lang/crates.io-index" 16 | checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" 17 | dependencies = [ 18 | "winapi", 19 | ] 20 | 21 | [[package]] 22 | name = "anyhow" 23 | version = "1.0.41" 24 | source = "registry+https://github.com/rust-lang/crates.io-index" 25 | checksum = "15af2628f6890fe2609a3b91bef4c83450512802e59489f9c1cb1fa5df064a61" 26 | 27 | [[package]] 28 | name = "async-trait" 29 | version = "0.1.50" 30 | source = "registry+https://github.com/rust-lang/crates.io-index" 31 | checksum = "0b98e84bbb4cbcdd97da190ba0c58a1bb0de2c1fdf67d159e192ed766aeca722" 32 | dependencies = [ 33 | "proc-macro2", 34 | "quote", 35 | "syn", 36 | ] 37 | 38 | [[package]] 39 | name = "atty" 40 | version = "0.2.14" 41 | source = "registry+https://github.com/rust-lang/crates.io-index" 42 | checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" 43 | dependencies = [ 44 | "hermit-abi", 45 | "libc", 46 | "winapi", 47 | ] 48 | 49 | [[package]] 50 | name = "autocfg" 51 | version = "1.0.1" 52 | source = "registry+https://github.com/rust-lang/crates.io-index" 53 | checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" 54 | 55 | [[package]] 56 | name = "bitflags" 57 | version = "1.2.1" 58 | source = "registry+https://github.com/rust-lang/crates.io-index" 59 | checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" 60 | 61 | [[package]] 62 | name = "bytefmt" 63 | version = "0.1.7" 64 | source = "registry+https://github.com/rust-lang/crates.io-index" 65 | checksum = "590b1af059a21c47d4da7cd11f05e08b1992b58b5b4acf2a5e10d7e53aed3d74" 66 | dependencies = [ 67 | "regex", 68 | ] 69 | 70 | [[package]] 71 | name = "bytes" 72 | version = "1.0.1" 73 | source = "registry+https://github.com/rust-lang/crates.io-index" 74 | checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" 75 | 76 | [[package]] 77 | name = "cfg-if" 78 | version = "0.1.10" 79 | source = "registry+https://github.com/rust-lang/crates.io-index" 80 | checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" 81 | 82 | [[package]] 83 | name = "cfg-if" 84 | version = "1.0.0" 85 | source = "registry+https://github.com/rust-lang/crates.io-index" 86 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 87 | 88 | [[package]] 89 | name = "chooch" 90 | version = "0.2.0" 91 | dependencies = [ 92 | "anyhow", 93 | "bytefmt", 94 | "bytes", 95 | "dynamic-pool", 96 | "futures", 97 | "hyper", 98 | "indicatif", 99 | "rand", 100 | "structopt", 101 | "tokio", 102 | "trust-dns-resolver", 103 | ] 104 | 105 | [[package]] 106 | name = "clap" 107 | version = "2.33.3" 108 | source = "registry+https://github.com/rust-lang/crates.io-index" 109 | checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" 110 | dependencies = [ 111 | "ansi_term", 112 | "atty", 113 | "bitflags", 114 | "strsim", 115 | "textwrap", 116 | "unicode-width", 117 | "vec_map", 118 | ] 119 | 120 | [[package]] 121 | name = "console" 122 | version = "0.14.1" 123 | source = "registry+https://github.com/rust-lang/crates.io-index" 124 | checksum = "3993e6445baa160675931ec041a5e03ca84b9c6e32a056150d3aa2bdda0a1f45" 125 | dependencies = [ 126 | "encode_unicode", 127 | "lazy_static", 128 | "libc", 129 | "terminal_size", 130 | "winapi", 131 | ] 132 | 133 | [[package]] 134 | name = "crossbeam-queue" 135 | version = "0.2.3" 136 | source = "registry+https://github.com/rust-lang/crates.io-index" 137 | checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" 138 | dependencies = [ 139 | "cfg-if 0.1.10", 140 | "crossbeam-utils", 141 | "maybe-uninit", 142 | ] 143 | 144 | [[package]] 145 | name = "crossbeam-utils" 146 | version = "0.7.2" 147 | source = "registry+https://github.com/rust-lang/crates.io-index" 148 | checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" 149 | dependencies = [ 150 | "autocfg", 151 | "cfg-if 0.1.10", 152 | "lazy_static", 153 | ] 154 | 155 | [[package]] 156 | name = "data-encoding" 157 | version = "2.3.2" 158 | source = "registry+https://github.com/rust-lang/crates.io-index" 159 | checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" 160 | 161 | [[package]] 162 | name = "dynamic-pool" 163 | version = "0.2.2" 164 | source = "registry+https://github.com/rust-lang/crates.io-index" 165 | checksum = "1474fbe84c305c3ff13c3f37b10de450fa56105ec8ea41bf5de1b578016d211f" 166 | dependencies = [ 167 | "crossbeam-queue", 168 | ] 169 | 170 | [[package]] 171 | name = "encode_unicode" 172 | version = "0.3.6" 173 | source = "registry+https://github.com/rust-lang/crates.io-index" 174 | checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" 175 | 176 | [[package]] 177 | name = "enum-as-inner" 178 | version = "0.3.3" 179 | source = "registry+https://github.com/rust-lang/crates.io-index" 180 | checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" 181 | dependencies = [ 182 | "heck", 183 | "proc-macro2", 184 | "quote", 185 | "syn", 186 | ] 187 | 188 | [[package]] 189 | name = "fnv" 190 | version = "1.0.7" 191 | source = "registry+https://github.com/rust-lang/crates.io-index" 192 | checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" 193 | 194 | [[package]] 195 | name = "form_urlencoded" 196 | version = "1.0.1" 197 | source = "registry+https://github.com/rust-lang/crates.io-index" 198 | checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" 199 | dependencies = [ 200 | "matches", 201 | "percent-encoding", 202 | ] 203 | 204 | [[package]] 205 | name = "futures" 206 | version = "0.3.15" 207 | source = "registry+https://github.com/rust-lang/crates.io-index" 208 | checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27" 209 | dependencies = [ 210 | "futures-channel", 211 | "futures-core", 212 | "futures-executor", 213 | "futures-io", 214 | "futures-sink", 215 | "futures-task", 216 | "futures-util", 217 | ] 218 | 219 | [[package]] 220 | name = "futures-channel" 221 | version = "0.3.15" 222 | source = "registry+https://github.com/rust-lang/crates.io-index" 223 | checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2" 224 | dependencies = [ 225 | "futures-core", 226 | "futures-sink", 227 | ] 228 | 229 | [[package]] 230 | name = "futures-core" 231 | version = "0.3.15" 232 | source = "registry+https://github.com/rust-lang/crates.io-index" 233 | checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1" 234 | 235 | [[package]] 236 | name = "futures-executor" 237 | version = "0.3.15" 238 | source = "registry+https://github.com/rust-lang/crates.io-index" 239 | checksum = "badaa6a909fac9e7236d0620a2f57f7664640c56575b71a7552fbd68deafab79" 240 | dependencies = [ 241 | "futures-core", 242 | "futures-task", 243 | "futures-util", 244 | ] 245 | 246 | [[package]] 247 | name = "futures-io" 248 | version = "0.3.15" 249 | source = "registry+https://github.com/rust-lang/crates.io-index" 250 | checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1" 251 | 252 | [[package]] 253 | name = "futures-macro" 254 | version = "0.3.15" 255 | source = "registry+https://github.com/rust-lang/crates.io-index" 256 | checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121" 257 | dependencies = [ 258 | "autocfg", 259 | "proc-macro-hack", 260 | "proc-macro2", 261 | "quote", 262 | "syn", 263 | ] 264 | 265 | [[package]] 266 | name = "futures-sink" 267 | version = "0.3.15" 268 | source = "registry+https://github.com/rust-lang/crates.io-index" 269 | checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282" 270 | 271 | [[package]] 272 | name = "futures-task" 273 | version = "0.3.15" 274 | source = "registry+https://github.com/rust-lang/crates.io-index" 275 | checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae" 276 | 277 | [[package]] 278 | name = "futures-util" 279 | version = "0.3.15" 280 | source = "registry+https://github.com/rust-lang/crates.io-index" 281 | checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967" 282 | dependencies = [ 283 | "autocfg", 284 | "futures-channel", 285 | "futures-core", 286 | "futures-io", 287 | "futures-macro", 288 | "futures-sink", 289 | "futures-task", 290 | "memchr", 291 | "pin-project-lite", 292 | "pin-utils", 293 | "proc-macro-hack", 294 | "proc-macro-nested", 295 | "slab", 296 | ] 297 | 298 | [[package]] 299 | name = "getrandom" 300 | version = "0.2.3" 301 | source = "registry+https://github.com/rust-lang/crates.io-index" 302 | checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" 303 | dependencies = [ 304 | "cfg-if 1.0.0", 305 | "libc", 306 | "wasi", 307 | ] 308 | 309 | [[package]] 310 | name = "heck" 311 | version = "0.3.3" 312 | source = "registry+https://github.com/rust-lang/crates.io-index" 313 | checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" 314 | dependencies = [ 315 | "unicode-segmentation", 316 | ] 317 | 318 | [[package]] 319 | name = "hermit-abi" 320 | version = "0.1.19" 321 | source = "registry+https://github.com/rust-lang/crates.io-index" 322 | checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" 323 | dependencies = [ 324 | "libc", 325 | ] 326 | 327 | [[package]] 328 | name = "hostname" 329 | version = "0.3.1" 330 | source = "registry+https://github.com/rust-lang/crates.io-index" 331 | checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" 332 | dependencies = [ 333 | "libc", 334 | "match_cfg", 335 | "winapi", 336 | ] 337 | 338 | [[package]] 339 | name = "http" 340 | version = "0.2.4" 341 | source = "registry+https://github.com/rust-lang/crates.io-index" 342 | checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11" 343 | dependencies = [ 344 | "bytes", 345 | "fnv", 346 | "itoa", 347 | ] 348 | 349 | [[package]] 350 | name = "http-body" 351 | version = "0.4.2" 352 | source = "registry+https://github.com/rust-lang/crates.io-index" 353 | checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" 354 | dependencies = [ 355 | "bytes", 356 | "http", 357 | "pin-project-lite", 358 | ] 359 | 360 | [[package]] 361 | name = "httparse" 362 | version = "1.4.1" 363 | source = "registry+https://github.com/rust-lang/crates.io-index" 364 | checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68" 365 | 366 | [[package]] 367 | name = "httpdate" 368 | version = "1.0.1" 369 | source = "registry+https://github.com/rust-lang/crates.io-index" 370 | checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" 371 | 372 | [[package]] 373 | name = "hyper" 374 | version = "0.14.9" 375 | source = "registry+https://github.com/rust-lang/crates.io-index" 376 | checksum = "07d6baa1b441335f3ce5098ac421fb6547c46dda735ca1bc6d0153c838f9dd83" 377 | dependencies = [ 378 | "bytes", 379 | "futures-channel", 380 | "futures-core", 381 | "futures-util", 382 | "http", 383 | "http-body", 384 | "httparse", 385 | "httpdate", 386 | "itoa", 387 | "pin-project-lite", 388 | "socket2 0.4.0", 389 | "tokio", 390 | "tower-service", 391 | "tracing", 392 | "want", 393 | ] 394 | 395 | [[package]] 396 | name = "idna" 397 | version = "0.2.3" 398 | source = "registry+https://github.com/rust-lang/crates.io-index" 399 | checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" 400 | dependencies = [ 401 | "matches", 402 | "unicode-bidi", 403 | "unicode-normalization", 404 | ] 405 | 406 | [[package]] 407 | name = "indicatif" 408 | version = "0.16.2" 409 | source = "registry+https://github.com/rust-lang/crates.io-index" 410 | checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b" 411 | dependencies = [ 412 | "console", 413 | "lazy_static", 414 | "number_prefix", 415 | "regex", 416 | ] 417 | 418 | [[package]] 419 | name = "instant" 420 | version = "0.1.9" 421 | source = "registry+https://github.com/rust-lang/crates.io-index" 422 | checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" 423 | dependencies = [ 424 | "cfg-if 1.0.0", 425 | ] 426 | 427 | [[package]] 428 | name = "ipconfig" 429 | version = "0.2.2" 430 | source = "registry+https://github.com/rust-lang/crates.io-index" 431 | checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" 432 | dependencies = [ 433 | "socket2 0.3.19", 434 | "widestring", 435 | "winapi", 436 | "winreg", 437 | ] 438 | 439 | [[package]] 440 | name = "ipnet" 441 | version = "2.3.1" 442 | source = "registry+https://github.com/rust-lang/crates.io-index" 443 | checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" 444 | 445 | [[package]] 446 | name = "itoa" 447 | version = "0.4.7" 448 | source = "registry+https://github.com/rust-lang/crates.io-index" 449 | checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" 450 | 451 | [[package]] 452 | name = "lazy_static" 453 | version = "1.4.0" 454 | source = "registry+https://github.com/rust-lang/crates.io-index" 455 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" 456 | 457 | [[package]] 458 | name = "libc" 459 | version = "0.2.97" 460 | source = "registry+https://github.com/rust-lang/crates.io-index" 461 | checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6" 462 | 463 | [[package]] 464 | name = "linked-hash-map" 465 | version = "0.5.4" 466 | source = "registry+https://github.com/rust-lang/crates.io-index" 467 | checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" 468 | 469 | [[package]] 470 | name = "lock_api" 471 | version = "0.4.4" 472 | source = "registry+https://github.com/rust-lang/crates.io-index" 473 | checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb" 474 | dependencies = [ 475 | "scopeguard", 476 | ] 477 | 478 | [[package]] 479 | name = "log" 480 | version = "0.4.14" 481 | source = "registry+https://github.com/rust-lang/crates.io-index" 482 | checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" 483 | dependencies = [ 484 | "cfg-if 1.0.0", 485 | ] 486 | 487 | [[package]] 488 | name = "lru-cache" 489 | version = "0.1.2" 490 | source = "registry+https://github.com/rust-lang/crates.io-index" 491 | checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" 492 | dependencies = [ 493 | "linked-hash-map", 494 | ] 495 | 496 | [[package]] 497 | name = "match_cfg" 498 | version = "0.1.0" 499 | source = "registry+https://github.com/rust-lang/crates.io-index" 500 | checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" 501 | 502 | [[package]] 503 | name = "matches" 504 | version = "0.1.8" 505 | source = "registry+https://github.com/rust-lang/crates.io-index" 506 | checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" 507 | 508 | [[package]] 509 | name = "maybe-uninit" 510 | version = "2.0.0" 511 | source = "registry+https://github.com/rust-lang/crates.io-index" 512 | checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" 513 | 514 | [[package]] 515 | name = "memchr" 516 | version = "2.4.0" 517 | source = "registry+https://github.com/rust-lang/crates.io-index" 518 | checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" 519 | 520 | [[package]] 521 | name = "mio" 522 | version = "0.7.13" 523 | source = "registry+https://github.com/rust-lang/crates.io-index" 524 | checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16" 525 | dependencies = [ 526 | "libc", 527 | "log", 528 | "miow", 529 | "ntapi", 530 | "winapi", 531 | ] 532 | 533 | [[package]] 534 | name = "miow" 535 | version = "0.3.7" 536 | source = "registry+https://github.com/rust-lang/crates.io-index" 537 | checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" 538 | dependencies = [ 539 | "winapi", 540 | ] 541 | 542 | [[package]] 543 | name = "ntapi" 544 | version = "0.3.6" 545 | source = "registry+https://github.com/rust-lang/crates.io-index" 546 | checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" 547 | dependencies = [ 548 | "winapi", 549 | ] 550 | 551 | [[package]] 552 | name = "num_cpus" 553 | version = "1.13.0" 554 | source = "registry+https://github.com/rust-lang/crates.io-index" 555 | checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" 556 | dependencies = [ 557 | "hermit-abi", 558 | "libc", 559 | ] 560 | 561 | [[package]] 562 | name = "number_prefix" 563 | version = "0.4.0" 564 | source = "registry+https://github.com/rust-lang/crates.io-index" 565 | checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" 566 | 567 | [[package]] 568 | name = "once_cell" 569 | version = "1.8.0" 570 | source = "registry+https://github.com/rust-lang/crates.io-index" 571 | checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" 572 | 573 | [[package]] 574 | name = "parking_lot" 575 | version = "0.11.1" 576 | source = "registry+https://github.com/rust-lang/crates.io-index" 577 | checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" 578 | dependencies = [ 579 | "instant", 580 | "lock_api", 581 | "parking_lot_core", 582 | ] 583 | 584 | [[package]] 585 | name = "parking_lot_core" 586 | version = "0.8.3" 587 | source = "registry+https://github.com/rust-lang/crates.io-index" 588 | checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" 589 | dependencies = [ 590 | "cfg-if 1.0.0", 591 | "instant", 592 | "libc", 593 | "redox_syscall", 594 | "smallvec", 595 | "winapi", 596 | ] 597 | 598 | [[package]] 599 | name = "percent-encoding" 600 | version = "2.1.0" 601 | source = "registry+https://github.com/rust-lang/crates.io-index" 602 | checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" 603 | 604 | [[package]] 605 | name = "pin-project-lite" 606 | version = "0.2.7" 607 | source = "registry+https://github.com/rust-lang/crates.io-index" 608 | checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" 609 | 610 | [[package]] 611 | name = "pin-utils" 612 | version = "0.1.0" 613 | source = "registry+https://github.com/rust-lang/crates.io-index" 614 | checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" 615 | 616 | [[package]] 617 | name = "ppv-lite86" 618 | version = "0.2.10" 619 | source = "registry+https://github.com/rust-lang/crates.io-index" 620 | checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" 621 | 622 | [[package]] 623 | name = "proc-macro-error" 624 | version = "1.0.4" 625 | source = "registry+https://github.com/rust-lang/crates.io-index" 626 | checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" 627 | dependencies = [ 628 | "proc-macro-error-attr", 629 | "proc-macro2", 630 | "quote", 631 | "syn", 632 | "version_check", 633 | ] 634 | 635 | [[package]] 636 | name = "proc-macro-error-attr" 637 | version = "1.0.4" 638 | source = "registry+https://github.com/rust-lang/crates.io-index" 639 | checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" 640 | dependencies = [ 641 | "proc-macro2", 642 | "quote", 643 | "version_check", 644 | ] 645 | 646 | [[package]] 647 | name = "proc-macro-hack" 648 | version = "0.5.19" 649 | source = "registry+https://github.com/rust-lang/crates.io-index" 650 | checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" 651 | 652 | [[package]] 653 | name = "proc-macro-nested" 654 | version = "0.1.7" 655 | source = "registry+https://github.com/rust-lang/crates.io-index" 656 | checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" 657 | 658 | [[package]] 659 | name = "proc-macro2" 660 | version = "1.0.27" 661 | source = "registry+https://github.com/rust-lang/crates.io-index" 662 | checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038" 663 | dependencies = [ 664 | "unicode-xid", 665 | ] 666 | 667 | [[package]] 668 | name = "quick-error" 669 | version = "1.2.3" 670 | source = "registry+https://github.com/rust-lang/crates.io-index" 671 | checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" 672 | 673 | [[package]] 674 | name = "quote" 675 | version = "1.0.9" 676 | source = "registry+https://github.com/rust-lang/crates.io-index" 677 | checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" 678 | dependencies = [ 679 | "proc-macro2", 680 | ] 681 | 682 | [[package]] 683 | name = "rand" 684 | version = "0.8.4" 685 | source = "registry+https://github.com/rust-lang/crates.io-index" 686 | checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" 687 | dependencies = [ 688 | "libc", 689 | "rand_chacha", 690 | "rand_core", 691 | "rand_hc", 692 | ] 693 | 694 | [[package]] 695 | name = "rand_chacha" 696 | version = "0.3.1" 697 | source = "registry+https://github.com/rust-lang/crates.io-index" 698 | checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" 699 | dependencies = [ 700 | "ppv-lite86", 701 | "rand_core", 702 | ] 703 | 704 | [[package]] 705 | name = "rand_core" 706 | version = "0.6.3" 707 | source = "registry+https://github.com/rust-lang/crates.io-index" 708 | checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" 709 | dependencies = [ 710 | "getrandom", 711 | ] 712 | 713 | [[package]] 714 | name = "rand_hc" 715 | version = "0.3.1" 716 | source = "registry+https://github.com/rust-lang/crates.io-index" 717 | checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" 718 | dependencies = [ 719 | "rand_core", 720 | ] 721 | 722 | [[package]] 723 | name = "redox_syscall" 724 | version = "0.2.9" 725 | source = "registry+https://github.com/rust-lang/crates.io-index" 726 | checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee" 727 | dependencies = [ 728 | "bitflags", 729 | ] 730 | 731 | [[package]] 732 | name = "regex" 733 | version = "1.5.4" 734 | source = "registry+https://github.com/rust-lang/crates.io-index" 735 | checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" 736 | dependencies = [ 737 | "aho-corasick", 738 | "memchr", 739 | "regex-syntax", 740 | ] 741 | 742 | [[package]] 743 | name = "regex-syntax" 744 | version = "0.6.25" 745 | source = "registry+https://github.com/rust-lang/crates.io-index" 746 | checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" 747 | 748 | [[package]] 749 | name = "resolv-conf" 750 | version = "0.7.0" 751 | source = "registry+https://github.com/rust-lang/crates.io-index" 752 | checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" 753 | dependencies = [ 754 | "hostname", 755 | "quick-error", 756 | ] 757 | 758 | [[package]] 759 | name = "scopeguard" 760 | version = "1.1.0" 761 | source = "registry+https://github.com/rust-lang/crates.io-index" 762 | checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" 763 | 764 | [[package]] 765 | name = "signal-hook-registry" 766 | version = "1.4.0" 767 | source = "registry+https://github.com/rust-lang/crates.io-index" 768 | checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" 769 | dependencies = [ 770 | "libc", 771 | ] 772 | 773 | [[package]] 774 | name = "slab" 775 | version = "0.4.3" 776 | source = "registry+https://github.com/rust-lang/crates.io-index" 777 | checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527" 778 | 779 | [[package]] 780 | name = "smallvec" 781 | version = "1.6.1" 782 | source = "registry+https://github.com/rust-lang/crates.io-index" 783 | checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" 784 | 785 | [[package]] 786 | name = "socket2" 787 | version = "0.3.19" 788 | source = "registry+https://github.com/rust-lang/crates.io-index" 789 | checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" 790 | dependencies = [ 791 | "cfg-if 1.0.0", 792 | "libc", 793 | "winapi", 794 | ] 795 | 796 | [[package]] 797 | name = "socket2" 798 | version = "0.4.0" 799 | source = "registry+https://github.com/rust-lang/crates.io-index" 800 | checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" 801 | dependencies = [ 802 | "libc", 803 | "winapi", 804 | ] 805 | 806 | [[package]] 807 | name = "strsim" 808 | version = "0.8.0" 809 | source = "registry+https://github.com/rust-lang/crates.io-index" 810 | checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" 811 | 812 | [[package]] 813 | name = "structopt" 814 | version = "0.3.21" 815 | source = "registry+https://github.com/rust-lang/crates.io-index" 816 | checksum = "5277acd7ee46e63e5168a80734c9f6ee81b1367a7d8772a2d765df2a3705d28c" 817 | dependencies = [ 818 | "clap", 819 | "lazy_static", 820 | "structopt-derive", 821 | ] 822 | 823 | [[package]] 824 | name = "structopt-derive" 825 | version = "0.4.14" 826 | source = "registry+https://github.com/rust-lang/crates.io-index" 827 | checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90" 828 | dependencies = [ 829 | "heck", 830 | "proc-macro-error", 831 | "proc-macro2", 832 | "quote", 833 | "syn", 834 | ] 835 | 836 | [[package]] 837 | name = "syn" 838 | version = "1.0.73" 839 | source = "registry+https://github.com/rust-lang/crates.io-index" 840 | checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7" 841 | dependencies = [ 842 | "proc-macro2", 843 | "quote", 844 | "unicode-xid", 845 | ] 846 | 847 | [[package]] 848 | name = "terminal_size" 849 | version = "0.1.17" 850 | source = "registry+https://github.com/rust-lang/crates.io-index" 851 | checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" 852 | dependencies = [ 853 | "libc", 854 | "winapi", 855 | ] 856 | 857 | [[package]] 858 | name = "textwrap" 859 | version = "0.11.0" 860 | source = "registry+https://github.com/rust-lang/crates.io-index" 861 | checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" 862 | dependencies = [ 863 | "unicode-width", 864 | ] 865 | 866 | [[package]] 867 | name = "thiserror" 868 | version = "1.0.26" 869 | source = "registry+https://github.com/rust-lang/crates.io-index" 870 | checksum = "93119e4feac1cbe6c798c34d3a53ea0026b0b1de6a120deef895137c0529bfe2" 871 | dependencies = [ 872 | "thiserror-impl", 873 | ] 874 | 875 | [[package]] 876 | name = "thiserror-impl" 877 | version = "1.0.26" 878 | source = "registry+https://github.com/rust-lang/crates.io-index" 879 | checksum = "060d69a0afe7796bf42e9e2ff91f5ee691fb15c53d38b4b62a9a53eb23164745" 880 | dependencies = [ 881 | "proc-macro2", 882 | "quote", 883 | "syn", 884 | ] 885 | 886 | [[package]] 887 | name = "tinyvec" 888 | version = "1.2.0" 889 | source = "registry+https://github.com/rust-lang/crates.io-index" 890 | checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342" 891 | dependencies = [ 892 | "tinyvec_macros", 893 | ] 894 | 895 | [[package]] 896 | name = "tinyvec_macros" 897 | version = "0.1.0" 898 | source = "registry+https://github.com/rust-lang/crates.io-index" 899 | checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" 900 | 901 | [[package]] 902 | name = "tokio" 903 | version = "1.7.1" 904 | source = "registry+https://github.com/rust-lang/crates.io-index" 905 | checksum = "5fb2ed024293bb19f7a5dc54fe83bf86532a44c12a2bb8ba40d64a4509395ca2" 906 | dependencies = [ 907 | "autocfg", 908 | "bytes", 909 | "libc", 910 | "memchr", 911 | "mio", 912 | "num_cpus", 913 | "once_cell", 914 | "parking_lot", 915 | "pin-project-lite", 916 | "signal-hook-registry", 917 | "tokio-macros", 918 | "winapi", 919 | ] 920 | 921 | [[package]] 922 | name = "tokio-macros" 923 | version = "1.2.0" 924 | source = "registry+https://github.com/rust-lang/crates.io-index" 925 | checksum = "c49e3df43841dafb86046472506755d8501c5615673955f6aa17181125d13c37" 926 | dependencies = [ 927 | "proc-macro2", 928 | "quote", 929 | "syn", 930 | ] 931 | 932 | [[package]] 933 | name = "tower-service" 934 | version = "0.3.1" 935 | source = "registry+https://github.com/rust-lang/crates.io-index" 936 | checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" 937 | 938 | [[package]] 939 | name = "tracing" 940 | version = "0.1.26" 941 | source = "registry+https://github.com/rust-lang/crates.io-index" 942 | checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" 943 | dependencies = [ 944 | "cfg-if 1.0.0", 945 | "pin-project-lite", 946 | "tracing-core", 947 | ] 948 | 949 | [[package]] 950 | name = "tracing-core" 951 | version = "0.1.18" 952 | source = "registry+https://github.com/rust-lang/crates.io-index" 953 | checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052" 954 | dependencies = [ 955 | "lazy_static", 956 | ] 957 | 958 | [[package]] 959 | name = "trust-dns-proto" 960 | version = "0.20.3" 961 | source = "registry+https://github.com/rust-lang/crates.io-index" 962 | checksum = "ad0d7f5db438199a6e2609debe3f69f808d074e0a2888ee0bccb45fe234d03f4" 963 | dependencies = [ 964 | "async-trait", 965 | "cfg-if 1.0.0", 966 | "data-encoding", 967 | "enum-as-inner", 968 | "futures-channel", 969 | "futures-io", 970 | "futures-util", 971 | "idna", 972 | "ipnet", 973 | "lazy_static", 974 | "log", 975 | "rand", 976 | "smallvec", 977 | "thiserror", 978 | "tinyvec", 979 | "tokio", 980 | "url", 981 | ] 982 | 983 | [[package]] 984 | name = "trust-dns-resolver" 985 | version = "0.20.3" 986 | source = "registry+https://github.com/rust-lang/crates.io-index" 987 | checksum = "f6ad17b608a64bd0735e67bde16b0636f8aa8591f831a25d18443ed00a699770" 988 | dependencies = [ 989 | "cfg-if 1.0.0", 990 | "futures-util", 991 | "ipconfig", 992 | "lazy_static", 993 | "log", 994 | "lru-cache", 995 | "parking_lot", 996 | "resolv-conf", 997 | "smallvec", 998 | "thiserror", 999 | "tokio", 1000 | "trust-dns-proto", 1001 | ] 1002 | 1003 | [[package]] 1004 | name = "try-lock" 1005 | version = "0.2.3" 1006 | source = "registry+https://github.com/rust-lang/crates.io-index" 1007 | checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" 1008 | 1009 | [[package]] 1010 | name = "unicode-bidi" 1011 | version = "0.3.5" 1012 | source = "registry+https://github.com/rust-lang/crates.io-index" 1013 | checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0" 1014 | dependencies = [ 1015 | "matches", 1016 | ] 1017 | 1018 | [[package]] 1019 | name = "unicode-normalization" 1020 | version = "0.1.19" 1021 | source = "registry+https://github.com/rust-lang/crates.io-index" 1022 | checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" 1023 | dependencies = [ 1024 | "tinyvec", 1025 | ] 1026 | 1027 | [[package]] 1028 | name = "unicode-segmentation" 1029 | version = "1.7.1" 1030 | source = "registry+https://github.com/rust-lang/crates.io-index" 1031 | checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" 1032 | 1033 | [[package]] 1034 | name = "unicode-width" 1035 | version = "0.1.8" 1036 | source = "registry+https://github.com/rust-lang/crates.io-index" 1037 | checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" 1038 | 1039 | [[package]] 1040 | name = "unicode-xid" 1041 | version = "0.2.2" 1042 | source = "registry+https://github.com/rust-lang/crates.io-index" 1043 | checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" 1044 | 1045 | [[package]] 1046 | name = "url" 1047 | version = "2.2.2" 1048 | source = "registry+https://github.com/rust-lang/crates.io-index" 1049 | checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" 1050 | dependencies = [ 1051 | "form_urlencoded", 1052 | "idna", 1053 | "matches", 1054 | "percent-encoding", 1055 | ] 1056 | 1057 | [[package]] 1058 | name = "vec_map" 1059 | version = "0.8.2" 1060 | source = "registry+https://github.com/rust-lang/crates.io-index" 1061 | checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" 1062 | 1063 | [[package]] 1064 | name = "version_check" 1065 | version = "0.9.3" 1066 | source = "registry+https://github.com/rust-lang/crates.io-index" 1067 | checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" 1068 | 1069 | [[package]] 1070 | name = "want" 1071 | version = "0.3.0" 1072 | source = "registry+https://github.com/rust-lang/crates.io-index" 1073 | checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" 1074 | dependencies = [ 1075 | "log", 1076 | "try-lock", 1077 | ] 1078 | 1079 | [[package]] 1080 | name = "wasi" 1081 | version = "0.10.2+wasi-snapshot-preview1" 1082 | source = "registry+https://github.com/rust-lang/crates.io-index" 1083 | checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" 1084 | 1085 | [[package]] 1086 | name = "widestring" 1087 | version = "0.4.3" 1088 | source = "registry+https://github.com/rust-lang/crates.io-index" 1089 | checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" 1090 | 1091 | [[package]] 1092 | name = "winapi" 1093 | version = "0.3.9" 1094 | source = "registry+https://github.com/rust-lang/crates.io-index" 1095 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 1096 | dependencies = [ 1097 | "winapi-i686-pc-windows-gnu", 1098 | "winapi-x86_64-pc-windows-gnu", 1099 | ] 1100 | 1101 | [[package]] 1102 | name = "winapi-i686-pc-windows-gnu" 1103 | version = "0.4.0" 1104 | source = "registry+https://github.com/rust-lang/crates.io-index" 1105 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 1106 | 1107 | [[package]] 1108 | name = "winapi-x86_64-pc-windows-gnu" 1109 | version = "0.4.0" 1110 | source = "registry+https://github.com/rust-lang/crates.io-index" 1111 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 1112 | 1113 | [[package]] 1114 | name = "winreg" 1115 | version = "0.6.2" 1116 | source = "registry+https://github.com/rust-lang/crates.io-index" 1117 | checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" 1118 | dependencies = [ 1119 | "winapi", 1120 | ] 1121 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "chooch" 3 | version = "0.2.0" 4 | authors = ["Spencer Sharkey "] 5 | edition = "2018" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | anyhow = "1" 11 | bytefmt = "0.1.7" 12 | bytes = "1.0.1" 13 | dynamic-pool = "0.2" 14 | futures = "0.3" 15 | indicatif = "0.16" 16 | rand = "0.8" 17 | hyper = { version = "0.14", features = ["runtime", "client", "http1"] } 18 | structopt = "0.3" 19 | tokio = { version = "1", features = ["full"] } 20 | trust-dns-resolver = "0.20" -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # chooch - An Amazing Project 2 | 3 | Downloads files faster than wget/curl (in theory) using multiple connections. Chooch recycles the slowest connection and re-connects with a new and unique source port. 4 | 5 | In environments where LACP or ECMP (hash-based load balancing across multiple physical interfaces) is implemented, a naive multi-stream downloader may not utilize all the bond interfaces if they re-use the same static connections. Chooch aims to solve this by recycling slow connections and ensuring new connections are made with a new unique source port. Doing this adds entropy to the bond's hashing algorithm which may improve total throughput if the connections are evenly distributed across the it's physical interfaces. 6 | 7 | In the future, Chooch may be expanded to support multiple source IPs, or could include a utility to pre-determine the best source port set to use for a given machine/network configuration. However, this optimization would require the download server to be reachable using a static address and port, as the source IP is a common facet used in bond hashing algorithms. 8 | 9 | ### Features 10 | 11 | - Uses multiple connections in parallel to download quicker 12 | - Terminates the slowest connection and re-connects with a new client port. 13 | - Pre-allocates the file on-disk 14 | 15 | ### Usage 16 | 17 | ```bash 18 | USAGE: 19 | chooch [FLAGS] [OPTIONS] 20 | 21 | FLAGS: 22 | -f, --force-overwrite Overwrites existing output file if it already exists 23 | -h, --help Prints help information 24 | -s, --skip-prealloc Skips the pre-allocation of the target file 25 | -V, --version Prints version information 26 | 27 | OPTIONS: 28 | -i, --bind-ip Sets the IP address used to make outgoing connections. 29 | -c, --chunk-size [default: 32MB] 30 | -w, --workers [default: 6] 31 | 32 | ARGS: 33 | The URL you wish to download 34 | The output destination for this download 35 | ``` 36 | 37 | ### Todo 38 | 39 | - TLS Support 40 | - Handle requests without content-length for unlimited streaming 41 | - More options for HTTP requests (headers, method, etc) 42 | 43 | PRs welcome :) 44 | -------------------------------------------------------------------------------- /src/client.rs: -------------------------------------------------------------------------------- 1 | use std::net::{IpAddr, SocketAddr}; 2 | use std::sync::atomic::{AtomicU16, Ordering}; 3 | use std::task::Poll; 4 | 5 | use futures::future::BoxFuture; 6 | use hyper::http::request; 7 | use hyper::service::Service; 8 | use hyper::{Request, Uri}; 9 | use tokio::net::{TcpSocket, TcpStream}; 10 | use trust_dns_resolver::config::{ResolverConfig, ResolverOpts}; 11 | use trust_dns_resolver::TokioAsyncResolver; 12 | 13 | static NEXT_PORT: AtomicU16 = AtomicU16::new(1024); 14 | pub fn next_port() -> u16 { 15 | let cur = NEXT_PORT.load(Ordering::SeqCst); 16 | if cur >= u16::MAX { 17 | NEXT_PORT.swap(1024, Ordering::SeqCst) 18 | } else { 19 | NEXT_PORT.swap(cur + 1, Ordering::SeqCst) 20 | } 21 | } 22 | 23 | static APP_USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"),); 24 | 25 | #[derive(Clone)] 26 | pub struct UniquePortClient { 27 | inner: hyper::client::Client, 28 | } 29 | 30 | impl UniquePortClient { 31 | pub fn new(bind_ip: Option) -> Self { 32 | Self { 33 | inner: hyper::Client::builder().build::<_, hyper::Body>(ChoochConnector { bind_ip }), 34 | } 35 | } 36 | 37 | pub fn new_request(uri: Uri) -> request::Builder { 38 | Request::builder() 39 | .header("User-Agent", APP_USER_AGENT) 40 | .uri(uri) 41 | } 42 | } 43 | 44 | impl std::ops::Deref for UniquePortClient { 45 | type Target = hyper::client::Client; 46 | 47 | fn deref(&self) -> &Self::Target { 48 | &self.inner 49 | } 50 | } 51 | 52 | impl std::ops::DerefMut for UniquePortClient { 53 | fn deref_mut(&mut self) -> &mut Self::Target { 54 | &mut self.inner 55 | } 56 | } 57 | 58 | #[derive(Clone, Default)] 59 | pub struct ChoochConnector { 60 | bind_ip: Option, 61 | } 62 | 63 | impl Service for ChoochConnector { 64 | type Response = TcpStream; 65 | type Error = anyhow::Error; 66 | type Future = BoxFuture<'static, std::result::Result>; 67 | 68 | fn poll_ready( 69 | &mut self, 70 | _cx: &mut std::task::Context<'_>, 71 | ) -> std::task::Poll> { 72 | Poll::Ready(Ok(())) 73 | } 74 | 75 | fn call(&mut self, req: Uri) -> Self::Future { 76 | let bind_ip = self.bind_ip.clone(); 77 | Box::pin(async move { 78 | let host = req 79 | .host() 80 | .ok_or_else(|| anyhow::anyhow!("missing host from uri"))?; 81 | 82 | let ip = match host.parse().ok() { 83 | Some(ip) => ip, 84 | None => { 85 | let resolver = TokioAsyncResolver::tokio( 86 | ResolverConfig::default(), 87 | ResolverOpts::default(), 88 | )?; 89 | let response = resolver.lookup_ip(host).await?; 90 | response 91 | .into_iter() 92 | .next() 93 | .ok_or_else(|| anyhow::anyhow!(format!("error resolving host {}", host)))? 94 | } 95 | }; 96 | 97 | let (bind_ip, socket) = match ip { 98 | IpAddr::V4(_) => ( 99 | bind_ip.unwrap_or_else(|| "0.0.0.0".parse().unwrap()), 100 | TcpSocket::new_v4()?, 101 | ), 102 | IpAddr::V6(_) => ( 103 | bind_ip.unwrap_or_else(|| "[::]".parse().unwrap()), 104 | TcpSocket::new_v6()?, 105 | ), 106 | }; 107 | 108 | loop { 109 | // try to bind to the next available incremental port 110 | let next_port = next_port(); 111 | if socket 112 | .bind(match ip { 113 | IpAddr::V4(_) => SocketAddr::new(bind_ip, next_port), 114 | IpAddr::V6(_) => SocketAddr::new(bind_ip, next_port), 115 | }) 116 | .is_ok() 117 | { 118 | break; 119 | } 120 | } 121 | 122 | let target_socket_addr = SocketAddr::new(ip, req.port_u16().unwrap_or(80)); 123 | Ok(socket.connect(target_socket_addr).await?) 124 | }) 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | mod client; 2 | 3 | use std::{ 4 | collections::BTreeSet, 5 | net::IpAddr, 6 | ops::Range, 7 | sync::{Arc, Mutex}, 8 | time::{Duration, Instant}, 9 | }; 10 | 11 | use bytes::Bytes; 12 | use dynamic_pool::{DynamicPool, DynamicReset}; 13 | use futures::{FutureExt, Stream, StreamExt}; 14 | use hyper::{Body, Uri}; 15 | 16 | use crate::client::UniquePortClient; 17 | 18 | #[derive(Clone, Debug)] 19 | pub struct Config { 20 | pub url: Uri, 21 | pub concurrency: usize, 22 | pub chunk_size_bytes: usize, 23 | pub bind_ip: Option, 24 | } 25 | 26 | #[derive(Clone)] 27 | pub struct Choocher { 28 | worker_pool: DynamicPool, 29 | config: Config, 30 | slowest_worker_killer: Arc>, 31 | } 32 | 33 | impl Choocher { 34 | /// Creates a new Choocher instance given the URL, uses an optimal default configuration. 35 | pub fn new(url: Uri, chunk_size: usize, worker_count: usize, bind_ip: Option) -> Self { 36 | Self::new_with_config(Config { 37 | url, 38 | concurrency: worker_count, 39 | chunk_size_bytes: chunk_size, 40 | bind_ip, 41 | }) 42 | } 43 | 44 | // Creates a new Choocher instance, allowing the caller to specify a detailed config. 45 | pub fn new_with_config(config: Config) -> Self { 46 | let concurrency = config.concurrency; 47 | let bind_ip = config.bind_ip; 48 | Self { 49 | worker_pool: DynamicPool::new(0, concurrency, move || { 50 | ChoocherWorker::new(bind_ip.clone()) 51 | }), 52 | config, 53 | slowest_worker_killer: Arc::new(Mutex::new(ChoocherWorkerKiller::new(concurrency))), 54 | } 55 | } 56 | 57 | pub async fn chunks(self) -> anyhow::Result<(usize, impl Stream)> { 58 | let content_length = self.content_length().await? as usize; 59 | let chunks = self.chunks_for_content_length(content_length); 60 | let url = self.config.url.clone(); 61 | let pool = self.worker_pool.clone(); 62 | let slowest_worker_killer = self.slowest_worker_killer.clone(); 63 | let chunk_stream = futures::stream::iter(chunks) 64 | .map(move |chunk| { 65 | let url = url.clone(); 66 | let worker = pool.take(); 67 | let download_duration_tracker = slowest_worker_killer.clone(); 68 | 69 | // Spawn the download in its own task, so that we just have to poll the task completion, and can 70 | // actually use many cores for the job. 71 | let handle = tokio::spawn(async move { 72 | loop { 73 | let start = Instant::now(); 74 | 75 | if let Ok(bytes) = worker.fetch_chunk(url.clone(), chunk.clone()).await { 76 | let elapsed = start.elapsed(); 77 | 78 | // If we are the slowest download, we'll detach the client, forcing the pool to create a 79 | // new client in the future. 80 | if download_duration_tracker 81 | .lock() 82 | .unwrap() 83 | .should_discard_client(elapsed) 84 | { 85 | worker.detach(); 86 | } 87 | return bytes; 88 | } 89 | } 90 | }); 91 | 92 | handle.map(|x| x.unwrap()) 93 | }) 94 | .buffered(self.config.concurrency); 95 | 96 | Ok((content_length, chunk_stream)) 97 | } 98 | 99 | fn chunks_for_content_length(&self, content_length: usize) -> Vec> { 100 | let mut chunks = 101 | Vec::with_capacity((1 + (content_length / self.config.chunk_size_bytes)) as _); 102 | let mut last_end = 0; 103 | 104 | while last_end < content_length { 105 | let start = last_end; 106 | last_end = (start + self.config.chunk_size_bytes).min(content_length); 107 | chunks.push(start..last_end - 1); 108 | } 109 | 110 | chunks 111 | } 112 | 113 | /// Gets the content length from the configured URL using the HEAD method. 114 | async fn content_length(&self) -> anyhow::Result { 115 | let worker = self.worker_pool.take(); 116 | let client = worker.client(); 117 | let req = UniquePortClient::new_request(self.config.url.clone()) 118 | .method("HEAD") 119 | .body(Body::empty())?; 120 | let res = client.request(req).await?; 121 | if !res.status().is_success() { 122 | return Err(anyhow::anyhow!(format!("error status {}", res.status()))); 123 | } 124 | let header = res 125 | .headers() 126 | .get("content-length") 127 | .expect("expected content-length header"); 128 | Ok(header.to_str()?.parse()?) 129 | } 130 | } 131 | 132 | struct ChoocherWorker { 133 | client: UniquePortClient, 134 | } 135 | 136 | impl ChoocherWorker { 137 | fn new(bind_ip: Option) -> Self { 138 | Self { 139 | client: UniquePortClient::new(bind_ip), 140 | } 141 | } 142 | 143 | fn client(&self) -> &UniquePortClient { 144 | &self.client 145 | } 146 | 147 | async fn fetch_chunk(&self, url: Uri, range: Range) -> anyhow::Result { 148 | let req = UniquePortClient::new_request(url) 149 | .header("Range", format!("bytes={}-{}", range.start, range.end)) 150 | .body(Body::empty())?; 151 | let res = self.client.request(req).await?; 152 | if !res.status().is_success() { 153 | return Err(anyhow::anyhow!(format!("error status {}", res.status()))); 154 | } 155 | Ok(hyper::body::to_bytes(res.into_body()).await?) 156 | } 157 | } 158 | 159 | impl DynamicReset for ChoocherWorker { 160 | fn reset(&mut self) { 161 | // nothing to do here. 162 | } 163 | } 164 | 165 | struct ChoocherWorkerKiller { 166 | target_concurrency: usize, 167 | timings: BTreeSet, 168 | } 169 | 170 | impl ChoocherWorkerKiller { 171 | fn new(target_concurrency: usize) -> Self { 172 | assert!(target_concurrency > 0); 173 | Self { 174 | target_concurrency, 175 | timings: BTreeSet::new(), 176 | } 177 | } 178 | 179 | fn should_discard_client(&mut self, download_time: Duration) -> bool { 180 | // We don't have enough samples yet. 181 | if self.timings.len() < self.target_concurrency { 182 | self.timings.insert(download_time); 183 | return false; 184 | } 185 | 186 | let slowest_download_time = self 187 | .timings 188 | .iter() 189 | .rev() 190 | .next() 191 | .expect("invariant: should have some timings.") 192 | .clone(); 193 | 194 | self.timings.remove(&slowest_download_time); 195 | self.timings.insert(download_time); 196 | 197 | // We'll track the last N many chunk downloads, and reject clients if they exceed the slowest download time repeatedly. 198 | download_time > slowest_download_time 199 | } 200 | } 201 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use std::{net::IpAddr, path::PathBuf}; 2 | 3 | use chooch::Choocher; 4 | 5 | use futures::StreamExt; 6 | use hyper::Uri; 7 | use indicatif::{ProgressBar, ProgressStyle}; 8 | use rand::Rng; 9 | use structopt::StructOpt; 10 | use tokio::io::AsyncWriteExt; 11 | 12 | fn parse_bytes(src: &str) -> Result { 13 | bytefmt::parse(src).map(|n| n as usize) 14 | } 15 | 16 | #[derive(Debug, StructOpt)] 17 | #[structopt( 18 | name = "chooch", 19 | about = "Downloads files over HTTP using multiple streams" 20 | )] 21 | struct Opt { 22 | #[structopt(name = "url", help = "The URL you wish to download")] 23 | url: Uri, 24 | #[structopt(name = "output", help = "The output destination for this download")] 25 | output: PathBuf, 26 | #[structopt(long = "chunk-size", short, default_value = "32MB", parse(try_from_str = parse_bytes))] 27 | chunk_size: usize, 28 | #[structopt(long = "workers", short, default_value = "6")] 29 | worker_count: usize, 30 | #[structopt( 31 | long = "force-overwrite", 32 | short = "f", 33 | help = "Overwrites existing output file if it already exists" 34 | )] 35 | force_overwrite: bool, 36 | #[structopt( 37 | long = "skip-prealloc", 38 | short = "s", 39 | help = "Skips the pre-allocation of the target file" 40 | )] 41 | skip_prealloc: bool, 42 | #[structopt( 43 | long = "bind-ip", 44 | short = "ip", 45 | help = "Sets the IP address used to make outgoing connections." 46 | )] 47 | bind_ip: Option, 48 | #[structopt( 49 | long = "no-interactive", 50 | short = "n", 51 | help = "Turns off the progress bar in favor of logging messages to stdout." 52 | )] 53 | no_interactive: bool, 54 | } 55 | 56 | #[tokio::main] 57 | async fn main() -> anyhow::Result<()> { 58 | let opt = Opt::from_args(); 59 | 60 | let choocher = Choocher::new(opt.url, opt.chunk_size, opt.worker_count, opt.bind_ip); 61 | let (content_length, mut chunks) = choocher.chunks().await?; 62 | 63 | let real_path = opt.output; 64 | let tmp_path = create_temp_path(&real_path, opt.force_overwrite)?; 65 | println!("final path: {}", &real_path.to_str().unwrap()); 66 | println!("temp path: {}", &tmp_path.to_string_lossy()); 67 | 68 | let mut output_file = tokio::fs::OpenOptions::new() 69 | .write(true) 70 | .create_new(true) 71 | .open(&tmp_path) 72 | .await?; 73 | 74 | let skip_prealloc = opt.skip_prealloc; 75 | let no_interactive = opt.no_interactive; 76 | let tmp_path_filename = tmp_path 77 | .clone() 78 | .file_name() 79 | .unwrap() 80 | .to_string_lossy() 81 | .to_string(); 82 | let task_res = tokio::spawn(async move { 83 | if !skip_prealloc { 84 | println!( 85 | "preallocating file ({})", 86 | bytefmt::format(content_length as _) 87 | ); 88 | output_file.set_len(content_length as _).await?; 89 | } 90 | 91 | let mut bytes_written = 0; 92 | let bar = setup_progress_bar(content_length as u64); 93 | { 94 | while let Some(chunk) = chunks.next().await { 95 | output_file.write_all(&chunk).await?; 96 | bar.inc(chunk.len() as _); 97 | bytes_written += chunk.len(); 98 | if no_interactive { 99 | println!( 100 | "{} downloaded {}% ({})", 101 | tmp_path_filename, 102 | f64::trunc((bytes_written as f64 / content_length as f64) * 100.0), 103 | bytefmt::format(bytes_written as _) 104 | ); 105 | } 106 | } 107 | output_file.flush().await?; 108 | } 109 | 110 | bar.finish(); 111 | Ok(bytes_written) 112 | }); 113 | 114 | let exit_signal = tokio::signal::ctrl_c(); 115 | let res = async move { 116 | loop { 117 | tokio::select! { 118 | _ = exit_signal => { 119 | return Err(anyhow::anyhow!("user-terminated via signal")); 120 | } 121 | res = task_res => { 122 | return res? 123 | } 124 | } 125 | } 126 | }; 127 | 128 | match res.await { 129 | Ok(bytes_written) => { 130 | println!("done! renaming to final destination..."); 131 | tokio::fs::rename(&tmp_path, &real_path).await?; 132 | println!( 133 | "{} bytes written to {}", 134 | bytes_written, 135 | real_path.to_string_lossy() 136 | ); 137 | Ok(()) 138 | } 139 | Err(e) => { 140 | println!("something went wrong. removing temp file..."); 141 | tokio::fs::remove_file(&tmp_path).await.unwrap(); 142 | Err(e) 143 | } 144 | } 145 | } 146 | 147 | fn setup_progress_bar(length: u64) -> ProgressBar { 148 | let bar = ProgressBar::new_spinner(); 149 | bar.set_length(length); 150 | bar.set_style(ProgressStyle::default_spinner().template("[{elapsed_precise}] {bar:40.cyan/blue} {bytes:>7}/{total_bytes:7} ({bytes_per_sec}, eta: {eta_precise})")); 151 | bar 152 | } 153 | 154 | fn create_temp_path(real_path: &PathBuf, overwrite: bool) -> anyhow::Result { 155 | if real_path.exists() { 156 | if overwrite { 157 | println!( 158 | "warning: {} already exists, ovewriting...", 159 | &real_path.to_string_lossy() 160 | ) 161 | } else { 162 | return Err(anyhow::anyhow!( 163 | "cannot overwrite destination file, use --force to force overwrite" 164 | )); 165 | } 166 | } 167 | let mut tmp_path = real_path.clone(); 168 | tmp_path.set_file_name(format!( 169 | ".{}.choochdl~{}", 170 | real_path.file_name().unwrap().to_str().unwrap(), 171 | rand::thread_rng() 172 | .sample_iter(&rand::distributions::Alphanumeric) 173 | .take(6) 174 | .map(char::from) 175 | .collect::() 176 | )); 177 | Ok(tmp_path) 178 | } 179 | --------------------------------------------------------------------------------