├── .gitignore ├── Cargo.toml ├── LICENSE ├── README.md ├── fetcher ├── Cargo.toml ├── README.md ├── iso.ron ├── sample.ron └── src │ ├── inputs.rs │ ├── interactive.rs │ ├── machine.rs │ └── main.rs └── src ├── checksum.rs ├── checksum_system.rs ├── concatenator.rs ├── get.rs ├── get_many.rs ├── iface.rs ├── lib.rs ├── range.rs ├── source.rs ├── time.rs └── utils.rs /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea/ 2 | /target/ 3 | **/*.rs.bk 4 | Cargo.lock 5 | cache/ 6 | w 7 | .history 8 | .vscode -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "async-fetcher" 3 | version = "0.11.0" 4 | repository = "https://github.com/pop-os/async-fetcher" 5 | authors = ["Michael Aaron Murphy "] 6 | description = "Asynchronous multi-connection multi-download resumable HTTP file fetching state machine" 7 | keywords = ["async", "file", "fetch", "download", "parallel"] 8 | categories = [ 9 | "asynchronous", 10 | "network-programming", 11 | "web-programming::http-client", 12 | ] 13 | license = "MPL-2.0" 14 | readme = "README.md" 15 | edition = "2021" 16 | 17 | [workspace] 18 | members = ["fetcher"] 19 | 20 | [dependencies] 21 | derive_setters = "0.1.6" 22 | derive-new = "0.6.0" 23 | digest = "0.10.7" 24 | filetime = "0.2.23" 25 | futures = "0.3.30" 26 | hex = "0.4.3" 27 | httpdate = "1.0.3" 28 | log = "0.4.21" 29 | md-5 = "0.10.6" 30 | numtoa = "0.2.4" 31 | remem = "0.1.0" 32 | sha2 = "0.10.8" 33 | thiserror = "1.0.60" 34 | http = "1.1.0" 35 | ifaces = "0.1.0" 36 | async-shutdown = "0.2.2" 37 | reqwest = { version = "0.12.4", features = ["stream"] } 38 | 39 | [dependencies.serde] 40 | version = "1.0.201" 41 | features = ["derive"] 42 | 43 | [dependencies.tokio] 44 | version = "1.37.0" 45 | features = ["fs", "io-util", "rt", "sync", "time"] 46 | 47 | [features] 48 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Asynchronous File Fetcher 2 | 3 | This library provides an async service that can fetch multiple files concurrently, with multiple concurrent connections per file. 4 | 5 | If the process is terminated, the downloads can be resumed. Once fetched, checksums can be validated in parallel. 6 | 7 | The HTTP client used by the fetcher is `reqwest`. 8 | 9 | ## License 10 | 11 | Licensed under the [Mozilla Public License 2.0](https://choosealicense.com/licenses/mpl-2.0/). Permissions of this copyleft license are conditioned on making available source code of licensed files and modifications of those files under the same license (or in certain cases, one of the GNU licenses). Copyright and license notices must be preserved. Contributors provide an express grant of patent rights. However, a larger work using the licensed work may be distributed under different terms and without source code for files added in the larger work. 12 | 13 | ### Contribution 14 | 15 | Any contribution intentionally submitted for inclusion in the work by you shall be licensed under the Mozilla Public License 2.0 (MPL-2.0). It is required to add a boilerplate copyright notice to the top of each file: 16 | 17 | ```rs 18 | // Copyright {year} {person OR org} <{email}> 19 | // SPDX-License-Identifier: MPL-2.0 20 | ``` 21 | -------------------------------------------------------------------------------- /fetcher/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fetcher" 3 | version = "0.10.1" 4 | authors = ["Michael Aaron Murphy "] 5 | edition = "2018" 6 | license = "MPL-2.0" 7 | 8 | [dependencies] 9 | async-fetcher = { path = "../" } 10 | atty = "0.2.14" 11 | better-panic = "0.3.0" 12 | fomat-macros = "0.3.2" 13 | futures = "0.3.30" 14 | pbr = "1.1.1" 15 | ron = "0.8.1" 16 | serde = { version = "1.0.201", features = ["derive"] } 17 | thiserror = "1.0.60" 18 | bytes = "1.6.0" 19 | tokio-stream = "0.1.15" 20 | async-shutdown = "0.2.2" 21 | once_cell = "1.19.0" 22 | 23 | [dependencies.tokio-util] 24 | version = "0.7.11" 25 | features = ["codec"] 26 | 27 | [dependencies.tokio] 28 | version = "1.37.0" 29 | features = ["full"] 30 | -------------------------------------------------------------------------------- /fetcher/README.md: -------------------------------------------------------------------------------- 1 | # Pop Fetcher 2 | 3 | A command-line utility for massively-concurrent and efficient caching of files from the Internet. 4 | 5 | ## Features 6 | 7 | ### Fully asynchronous 8 | 9 | Designed from the ground up around an asynchronous paradigm 10 | 11 | - Concurrently fetches multiple files 12 | - Concurrently fetches multiple parts of a file 13 | - Concurrently fetches from multiple mirrors 14 | - Performs checksum validation in a thread pool 15 | 16 | ### Configurable 17 | 18 | Fetches a wealth of command-line arguments for configuring the behavior of the fetcher. Options defined via the command-line argument are applied to all sources given to the fetcher. Per-source configuration options are also supported when supplying inputs through standard input. 19 | 20 | ### IPC-driven design 21 | 22 | All sources are given to the fetcher through standard input. Each source is defined in [RON syntax](). Frames are denoted by a newline beginning with the `)` character. [An example input can be found here](./sample.ron). Additionally, if the standard output is not a TTY, all events will be written in RON syntax, where frames are delimited by newlines. 23 | 24 | ### Progress bars 25 | 26 | When run interactively (standard output is not a TTY), a progress bar is displayed for each file being actively fetched. 27 | 28 | ### Checksum validation 29 | 30 | Each input source may optionally define a checksum, which will be verified after fetching the file. If the checksum is not a match, the file which was fetched will be deleted. The following algorithms are currently supported: 31 | 32 | - MD5 33 | - SHA256 34 | 35 | ### Only fetch what you need 36 | 37 | Checks if previously-fetched files need to be fetched again 38 | 39 | - Compare the modified time stamp in the HTTP header with the local file 40 | - Compare the content length in the HTTP header with the file size 41 | 42 | Files which are partially-downloaded will also resume from where they left off. 43 | -------------------------------------------------------------------------------- /fetcher/iso.ron: -------------------------------------------------------------------------------- 1 | ( 2 | urls: ["https://pop-iso.sfo2.cdn.digitaloceanspaces.com/21.10/amd64/nvidia/6/pop-os_21.10_amd64_nvidia_6.iso"], 3 | dest: "cache/pop-os_21.10_amd64_nvidia_6.iso", 4 | sum: Some(Sha256("7684e9b5ced3397b549d805c2a8b76182abb5edd1f6c1583410728fcad95d19c")), 5 | ) -------------------------------------------------------------------------------- /fetcher/sample.ron: -------------------------------------------------------------------------------- 1 | ( 2 | urls: ["http://archive.ubuntu.com/ubuntu/pool/universe/r/rustc/rustc_1.53.0+dfsg1+llvm.orig.tar.xz"], 3 | dest: "cache/rustc_1.53.0.tar.xz", 4 | sum: Some(Sha256("1126c387579544bf8b3c239a51ed24f00404ed34373d58658a4fa1b0ae4b90db")) 5 | ) 6 | ( 7 | urls: ["http://archive.ubuntu.com/ubuntu/pool/universe/r/rustc/rustc_1.53.0+dfsg1+llvm.orig.tar.xz"], 8 | dest: "cache/rustc_1.53.0v2.tar.xz", 9 | sum: Some(Sha256("1126c387579544bf8b3c239a51ed24f00404ed34373d58658a4fa1b0ae4b90db")) 10 | ) -------------------------------------------------------------------------------- /fetcher/src/inputs.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021-2022 System76 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | use crate::{Checksum, SumStrBuf}; 5 | 6 | use async_fetcher::Source; 7 | use bytes::BytesMut; 8 | use futures::prelude::*; 9 | use serde::Deserialize; 10 | use std::sync::Arc; 11 | use std::{convert::TryFrom, io, path::PathBuf}; 12 | use tokio::fs::File; 13 | use tokio_util::codec::{Decoder, FramedRead}; 14 | 15 | #[derive(Debug, Error)] 16 | pub enum InputError { 17 | #[error("decoder error")] 18 | Decoder { 19 | input: Box, 20 | source: ron::de::Error, 21 | }, 22 | #[error("read error")] 23 | Read(#[from] io::Error), 24 | } 25 | 26 | #[derive(Default)] 27 | struct Inputs; 28 | 29 | impl Decoder for Inputs { 30 | type Error = InputError; 31 | type Item = Input; 32 | 33 | fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { 34 | let mut read = 0; 35 | 36 | for line in src.as_mut().split(|&byte| byte == b'\n') { 37 | read += line.len() + 1; 38 | 39 | if line.is_empty() { 40 | continue; 41 | } 42 | 43 | if line[0] == b')' { 44 | let value = ron::de::from_bytes::(&src[..read - 1]); 45 | 46 | eprintln!("({})", String::from_utf8_lossy(&src[..read - 1])); 47 | 48 | let remaining = src.len() + 1 - read; 49 | src.as_mut().copy_within(read - 1.., 0); 50 | src.truncate(remaining); 51 | 52 | return value.map(Some).map_err(|source| InputError::Decoder { 53 | input: String::from_utf8_lossy(src).into_owned().into(), 54 | source: source.into(), 55 | }); 56 | } 57 | } 58 | 59 | Ok(None) 60 | } 61 | } 62 | 63 | #[derive(Deserialize)] 64 | struct Input { 65 | urls: Vec>, 66 | dest: String, 67 | part: Option, 68 | sum: Option, 69 | } 70 | 71 | pub fn stream(input: File) -> impl Stream>)> + Send + Unpin { 72 | FramedRead::new(input, Inputs::default()) 73 | .filter_map(|result| async move { 74 | match result { 75 | Ok(input) => { 76 | let mut source = 77 | Source::new(Arc::from(input.urls), Arc::from(PathBuf::from(input.dest))); 78 | 79 | source.set_part(input.part.map(PathBuf::from).map(Arc::from)); 80 | 81 | let sum = match input.sum { 82 | Some(sum) => match Checksum::try_from(sum.as_ref()) { 83 | Ok(sum) => Some(sum), 84 | Err(why) => { 85 | eprintln!("invalid checksum: {}", why); 86 | None 87 | } 88 | }, 89 | None => None, 90 | }; 91 | 92 | Some((source, Arc::new(sum))) 93 | } 94 | Err(InputError::Read(why)) => { 95 | epintln!("read error: "(why)); 96 | None 97 | } 98 | Err(InputError::Decoder { input, source }) => { 99 | epintln!( 100 | "parsing error: " (source) "\n" 101 | " caused by input: " (input) 102 | ); 103 | 104 | None 105 | } 106 | } 107 | }) 108 | .boxed() 109 | } 110 | -------------------------------------------------------------------------------- /fetcher/src/interactive.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021-2022 System76 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | use crate::execute; 5 | 6 | use async_fetcher::{checksum_stream, Checksum, FetchEvent}; 7 | use futures::prelude::*; 8 | use pbr::{MultiBar, Pipe, ProgressBar, Units}; 9 | use std::{ 10 | collections::HashMap, 11 | path::Path, 12 | sync::{ 13 | atomic::{AtomicBool, Ordering}, 14 | Arc, 15 | }, 16 | time::Duration, 17 | }; 18 | use tokio::sync::mpsc; 19 | 20 | pub async fn run( 21 | etx: mpsc::UnboundedSender<(Arc, Arc>, FetchEvent)>, 22 | mut erx: mpsc::UnboundedReceiver<(Arc, Arc>, FetchEvent)>, 23 | ) { 24 | let complete = Arc::new(AtomicBool::new(false)); 25 | let progress = Arc::new(MultiBar::new()); 26 | 27 | let fetcher = { 28 | let complete = Arc::clone(&complete); 29 | let progress = Arc::clone(&progress); 30 | 31 | async move { 32 | let mut state = HashMap::, ProgressBar>::new(); 33 | 34 | while let Some((dest, _checksum, event)) = erx.recv().await { 35 | match event { 36 | FetchEvent::Progress(written) => { 37 | if let Some(bar) = state.get_mut(&dest) { 38 | bar.add(written as u64); 39 | } 40 | } 41 | 42 | FetchEvent::ContentLength(total) => { 43 | state 44 | .entry(dest.clone()) 45 | .and_modify(|bar| { 46 | bar.set(total); 47 | }) 48 | .or_insert_with(|| { 49 | let mut bar = progress.create_bar(total); 50 | bar.set_units(Units::Bytes); 51 | bar.message(&fomat!((dest.display()) ": ")); 52 | bar 53 | }); 54 | } 55 | 56 | FetchEvent::Fetched => { 57 | if let Some(mut bar) = state.remove(&dest) { 58 | bar.finish_print(&fomat!("Fetched "(dest.display()))); 59 | } 60 | } 61 | 62 | FetchEvent::Retrying => { 63 | if let Some(mut bar) = state.remove(&dest) { 64 | bar.finish_print(&fomat!("Retrying "(dest.display()))); 65 | } 66 | } 67 | 68 | _ => (), 69 | } 70 | } 71 | 72 | complete.store(true, Ordering::SeqCst); 73 | } 74 | }; 75 | 76 | let (fetch_tx, mut fetch_rx) = mpsc::channel::<(Arc, _)>(1); 77 | let fetch_results = async move { 78 | while let Some((dest, result)) = fetch_rx.recv().await { 79 | match result { 80 | Ok(false) => epintln!((dest.display()) " was successfully fetched"), 81 | Ok(true) => epintln!((dest.display()) " is now validating"), 82 | Err(why) => epintln!((dest.display()) " failed: " [why]), 83 | } 84 | } 85 | }; 86 | 87 | let (sum_tx, sum_rx) = mpsc::channel::<(Arc, _)>(1); 88 | let sum_results = async move { 89 | let mut stream = checksum_stream(tokio_stream::wrappers::ReceiverStream::new(sum_rx)) 90 | // Limiting up to 32 parallel tasks at a time. 91 | .buffer_unordered(32); 92 | 93 | while let Some((dest, result)) = stream.next().await { 94 | match result { 95 | Ok(()) => epintln!((dest.display()) " was successfully validated"), 96 | Err(why) => epintln!((dest.display()) " failed to validate: " [why]), 97 | } 98 | } 99 | }; 100 | 101 | let events = async move { 102 | join!( 103 | fetcher, 104 | fetch_results, 105 | sum_results, 106 | execute(etx, fetch_tx, sum_tx).boxed_local(), 107 | ) 108 | }; 109 | 110 | let progress_ticker = async { 111 | while !complete.load(Ordering::SeqCst) { 112 | eprintln!("update"); 113 | let _ = progress.listen(); 114 | tokio::time::sleep(Duration::from_millis(1000)).await; 115 | } 116 | }; 117 | 118 | join!(events, progress_ticker); 119 | 120 | let _ = progress.listen(); 121 | } 122 | -------------------------------------------------------------------------------- /fetcher/src/machine.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021-2022 System76 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | use crate::execute; 5 | 6 | use async_fetcher::{checksum_stream, Checksum, FetchEvent}; 7 | use futures::prelude::*; 8 | use serde::{Deserialize, Serialize}; 9 | use std::{ 10 | collections::HashMap, 11 | io::{self, Write}, 12 | path::Path, 13 | sync::Arc, 14 | }; 15 | use tokio::sync::mpsc; 16 | 17 | pub async fn run( 18 | etx: mpsc::UnboundedSender<(Arc, Arc>, FetchEvent)>, 19 | mut erx: mpsc::UnboundedReceiver<(Arc, Arc>, FetchEvent)>, 20 | ) { 21 | let (events_tx, mut events_rx) = mpsc::channel(1); 22 | 23 | // Handles all callback events from the fetcher 24 | let events_tx_ = events_tx.clone(); 25 | let fetch_events = async move { 26 | let mut state = HashMap::, (u64, u64)>::new(); 27 | while let Some((dest, _checksum, event)) = erx.recv().await { 28 | let event = match event { 29 | FetchEvent::Progress(written) => { 30 | if let Some(progress) = state.get_mut(&dest) { 31 | progress.0 += written as u64; 32 | Output( 33 | fomat!((dest.display())), 34 | OutputEvent::Progress(progress.0, progress.1), 35 | ) 36 | } else { 37 | continue; 38 | } 39 | } 40 | 41 | FetchEvent::ContentLength(length) => { 42 | state 43 | .entry(dest.clone()) 44 | .and_modify(|bar| bar.1 = length) 45 | .or_insert((0, length)); 46 | 47 | Output(fomat!((dest.display())), OutputEvent::Length(length)) 48 | } 49 | 50 | FetchEvent::Fetching => { 51 | state.insert(dest.clone(), (0, 0)); 52 | Output(fomat!((dest.display())), OutputEvent::Fetching) 53 | } 54 | 55 | FetchEvent::Fetched => { 56 | state.remove(&dest); 57 | Output(fomat!((dest.display())), OutputEvent::Fetched) 58 | } 59 | 60 | FetchEvent::Retrying => { 61 | state.remove(&dest); 62 | Output(fomat!((dest.display())), OutputEvent::Retrying) 63 | } 64 | }; 65 | 66 | if events_tx_.send(event).await.is_err() { 67 | break; 68 | } 69 | } 70 | }; 71 | 72 | // Handles all results from the fetcher. 73 | let events_tx_ = events_tx.clone(); 74 | let (fetch_tx, mut fetch_rx) = mpsc::channel::<(Arc, _)>(1); 75 | let fetch_results = async move { 76 | while let Some((dest, result)) = fetch_rx.recv().await { 77 | let event = match result { 78 | Ok(false) => None, 79 | Ok(true) => Some(Output(fomat!((dest.display())), OutputEvent::Validating)), 80 | Err(why) => { 81 | epintln!((dest.display()) " failed to validate: " [why]); 82 | 83 | Some(Output(fomat!((dest.display())), OutputEvent::Failed)) 84 | } 85 | }; 86 | 87 | if let Some(event) = event { 88 | if events_tx_.send(event).await.is_err() { 89 | break; 90 | } 91 | } 92 | } 93 | }; 94 | 95 | // Handles all results from checksum operations. 96 | let (sum_tx, sum_rx) = mpsc::channel::<(Arc, _)>(1); 97 | let sum_results = async move { 98 | let mut stream = checksum_stream(tokio_stream::wrappers::ReceiverStream::new(sum_rx)) 99 | // Limiting up to 32 parallel tasks at a time. 100 | .buffer_unordered(32); 101 | 102 | while let Some((dest, result)) = stream.next().await { 103 | let event = match result { 104 | Ok(()) => Output(fomat!((dest.display())), OutputEvent::Validated), 105 | 106 | Err(why) => { 107 | epintln!((dest.display()) " failed to validate: " [why]); 108 | 109 | Output(fomat!((dest.display())), OutputEvent::Invalid) 110 | } 111 | }; 112 | 113 | if events_tx.send(event).await.is_err() { 114 | break; 115 | } 116 | } 117 | }; 118 | 119 | // Centrally writes all events to standard out. 120 | let stdout_writer = async move { 121 | let output = io::stdout(); 122 | let mut output = output.lock(); 123 | 124 | while let Some(event) = events_rx.recv().await { 125 | match ron::ser::to_string(&event) { 126 | Ok(vector) => { 127 | let res1 = output.write_all(vector.as_bytes()); 128 | let res2 = output.write_all(b"\n"); 129 | 130 | if let Err(why) = res1.and(res2) { 131 | epintln!("failed to write serialized string to stdout: "(why)); 132 | return; 133 | } 134 | } 135 | Err(why) => { 136 | epintln!("failed to serialize: "(why)); 137 | } 138 | } 139 | } 140 | }; 141 | 142 | let _ = join!( 143 | stdout_writer, 144 | fetch_results, 145 | sum_results, 146 | fetch_events, 147 | execute(etx, fetch_tx, sum_tx).boxed_local() 148 | ); 149 | } 150 | 151 | #[derive(Deserialize, Serialize)] 152 | pub struct Output(String, OutputEvent); 153 | 154 | #[derive(Deserialize, Serialize)] 155 | pub enum OutputEvent { 156 | AlreadyFetched, 157 | Failed, 158 | Fetched, 159 | Fetching, 160 | Invalid, 161 | Length(u64), 162 | Progress(u64, u64), 163 | Retrying, 164 | Validated, 165 | Validating, 166 | } 167 | -------------------------------------------------------------------------------- /fetcher/src/main.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021-2022 System76 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | #[macro_use] 5 | extern crate fomat_macros; 6 | #[macro_use] 7 | extern crate futures; 8 | #[macro_use] 9 | extern crate thiserror; 10 | 11 | mod inputs; 12 | mod interactive; 13 | mod machine; 14 | 15 | use async_fetcher::{Error as FetchError, *}; 16 | use async_shutdown::Shutdown; 17 | use futures::prelude::*; 18 | use once_cell::sync::OnceCell; 19 | use std::{ 20 | io, 21 | os::unix::io::{AsRawFd, FromRawFd}, 22 | path::Path, 23 | sync::Arc, 24 | time::Duration, 25 | }; 26 | use tokio::fs::File; 27 | use tokio::sync::mpsc; 28 | 29 | fn shutdown_handle() -> &'static Shutdown { 30 | static SHUTDOWN: OnceCell = OnceCell::new(); 31 | SHUTDOWN.get_or_init(Shutdown::new) 32 | } 33 | 34 | #[tokio::main] 35 | async fn main() { 36 | better_panic::install(); 37 | 38 | let (tx, rx) = mpsc::unbounded_channel::<(Arc, Arc>, FetchEvent)>(); 39 | 40 | if atty::is(atty::Stream::Stdout) { 41 | interactive::run(tx, rx).await 42 | } else { 43 | machine::run(tx, rx).await 44 | } 45 | 46 | shutdown_handle().wait_shutdown_complete().await; 47 | } 48 | 49 | async fn execute( 50 | etx: mpsc::UnboundedSender<(Arc, Arc>, FetchEvent)>, 51 | result_sender: mpsc::Sender<(Arc, Result)>, 52 | checksum_sender: mpsc::Sender<(Arc, Checksum)>, 53 | ) { 54 | let stdin = io::stdin(); 55 | let stdin = stdin.lock(); 56 | 57 | let input_stream = inputs::stream(unsafe { File::from_raw_fd(stdin.as_raw_fd()) }); 58 | 59 | fetcher_stream(etx, result_sender, checksum_sender, input_stream).await 60 | } 61 | 62 | /// The fetcher, which will be used to create futures for fetching files. 63 | async fn fetcher_stream< 64 | S: Unpin + Send + Stream>)> + 'static, 65 | >( 66 | event_sender: mpsc::UnboundedSender<(Arc, Arc>, FetchEvent)>, 67 | result_sender: mpsc::Sender<(Arc, Result)>, 68 | checksum_sender: mpsc::Sender<(Arc, Checksum)>, 69 | sources: S, 70 | ) { 71 | let fetcher = Fetcher::default() 72 | // Add a handle to the shutdown mechanism used by this application. 73 | .shutdown(crate::shutdown_handle().clone()) 74 | // Fetch each file in parts, using up to 2 concurrent connections per file 75 | .connections_per_file(2) 76 | // Pass in the event sender which events will be sent to 77 | .events(event_sender) 78 | // Configure a timeout to bail when a connection stalls for too long 79 | .timeout(Duration::from_secs(15)) 80 | // Finalize the fetcher so that it can perform fetch tasks. 81 | .build() 82 | // Build a stream that will perform fetches when polled. 83 | .stream_from(sources, 4); 84 | 85 | futures::pin_mut!(fetcher); 86 | 87 | while let Some((dest, checksum, result)) = fetcher.next().await { 88 | match result { 89 | Ok(()) => { 90 | let _ = result_sender.send((dest.clone(), Ok(true))); 91 | if let Some(checksum) = checksum.as_ref() { 92 | let _ = checksum_sender.send((dest, checksum.clone())).await; 93 | } 94 | } 95 | Err(why) => { 96 | let _ = result_sender.send((dest, Err(why))).await; 97 | } 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/checksum.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 System76 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | use digest::{generic_array::GenericArray, Digest, OutputSizeUser}; 5 | use hex::FromHex; 6 | use md5::Md5; 7 | use serde::Deserialize; 8 | use sha2::Sha256; 9 | use std::{convert::TryFrom, io}; 10 | 11 | /// A checksum of a `Source` as a fixed-sized byte array. 12 | #[derive(Debug, Clone)] 13 | pub enum Checksum { 14 | Md5(GenericArray::OutputSize>), 15 | Sha256(GenericArray::OutputSize>), 16 | } 17 | 18 | /// An error that can occur from a failed checksum validation. 19 | #[derive(Debug, Error)] 20 | pub enum ChecksumError { 21 | #[error("expected {}, found {}", _0, _1)] 22 | Invalid(String, String), 23 | #[error("I/O error encountered while reading from reader")] 24 | IO(#[from] io::Error), 25 | } 26 | 27 | /// The `&str` representation of a `Checksum`. 28 | pub enum SumStr<'a> { 29 | Md5(&'a str), 30 | Sha256(&'a str), 31 | } 32 | 33 | /// The `String` representation of a `Checksum`. 34 | #[derive(Deserialize)] 35 | pub enum SumStrBuf { 36 | Md5(String), 37 | Sha256(String), 38 | } 39 | 40 | impl SumStrBuf { 41 | pub fn as_ref(&self) -> SumStr { 42 | match self { 43 | SumStrBuf::Md5(string) => SumStr::Md5(string.as_str()), 44 | SumStrBuf::Sha256(string) => SumStr::Sha256(string.as_str()), 45 | } 46 | } 47 | } 48 | 49 | impl<'a> TryFrom> for Checksum { 50 | type Error = hex::FromHexError; 51 | 52 | fn try_from(input: SumStr) -> Result { 53 | match input { 54 | SumStr::Md5(sum) => <[u8; 16]>::from_hex(sum) 55 | .map(GenericArray::from) 56 | .map(Checksum::Md5), 57 | SumStr::Sha256(sum) => <[u8; 32]>::from_hex(sum) 58 | .map(GenericArray::from) 59 | .map(Checksum::Sha256), 60 | } 61 | } 62 | } 63 | 64 | impl Checksum { 65 | pub fn validate( 66 | &self, 67 | reader: F, 68 | buffer: &mut [u8], 69 | ) -> Result<(), ChecksumError> { 70 | match self { 71 | Checksum::Md5(sum) => checksum::(reader, buffer, sum), 72 | Checksum::Sha256(sum) => checksum::(reader, buffer, sum), 73 | } 74 | } 75 | } 76 | 77 | pub(crate) fn checksum( 78 | reader: F, 79 | buffer: &mut [u8], 80 | expected: &GenericArray, 81 | ) -> Result<(), ChecksumError> { 82 | let result = generate_checksum::(reader, buffer).map_err(ChecksumError::IO)?; 83 | 84 | if result == *expected { 85 | Ok(()) 86 | } else { 87 | Err(ChecksumError::Invalid( 88 | hex::encode(expected), 89 | hex::encode(result), 90 | )) 91 | } 92 | } 93 | 94 | pub(crate) fn generate_checksum( 95 | mut reader: F, 96 | buffer: &mut [u8], 97 | ) -> io::Result> { 98 | let mut hasher = D::new(); 99 | let mut read; 100 | 101 | loop { 102 | read = reader.read(buffer)?; 103 | 104 | if read == 0 { 105 | return Ok(hasher.finalize()); 106 | } 107 | 108 | hasher.update(&buffer[..read]); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/checksum_system.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021-2022 System76 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | use crate::checksum::{Checksum, ChecksumError}; 5 | use futures::prelude::*; 6 | use remem::Pool; 7 | use std::{path::Path, sync::Arc}; 8 | 9 | /// Generates a stream of futures that validate checksums. 10 | /// 11 | /// The caller can choose to distribute these futures across a thread pool. 12 | /// 13 | /// ``` 14 | /// let mut stream = checksum_stream(checksums).map(tokio::spawn).buffered(8); 15 | /// while let Some((path, result)) = stream.next().await { 16 | /// eprintln!("{:?} checksum result: {:?}", path, result); 17 | /// } 18 | /// ``` 19 | pub fn checksum_stream, Checksum)> + Send + Unpin + 'static>( 20 | inputs: I, 21 | ) -> impl Stream, Result<(), ChecksumError>)>> { 22 | let buffer_pool = Pool::new(|| Box::new([0u8; 8 * 1024])); 23 | 24 | inputs.map(move |(dest, checksum)| { 25 | let pool = buffer_pool.clone(); 26 | 27 | async { 28 | tokio::task::spawn_blocking(move || { 29 | let buf = &mut **pool.get(); 30 | let result = validate_checksum(buf, &dest, &checksum); 31 | (dest, result) 32 | }) 33 | .await 34 | .unwrap() 35 | } 36 | }) 37 | } 38 | 39 | /// Validates the checksum of a single file 40 | pub fn validate_checksum( 41 | buf: &mut [u8], 42 | dest: &Path, 43 | checksum: &Checksum, 44 | ) -> Result<(), ChecksumError> { 45 | let error = match std::fs::File::open(&*dest) { 46 | Ok(file) => match checksum.validate(file, buf) { 47 | Ok(()) => return Ok(()), 48 | Err(why) => why, 49 | }, 50 | Err(why) => ChecksumError::from(why), 51 | }; 52 | 53 | let _ = std::fs::remove_file(&*dest); 54 | Err(error) 55 | } 56 | -------------------------------------------------------------------------------- /src/concatenator.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021-2022 System76 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | use crate::Error; 5 | 6 | use async_shutdown::ShutdownManager; 7 | use futures::{Stream, StreamExt}; 8 | use std::fs::{self, File}; 9 | use std::io::copy; 10 | use std::{path::Path, sync::Arc}; 11 | 12 | /// Accepts a stream of future file `parts` and concatenates them into the `dest` file. 13 | pub async fn concatenator( 14 | mut dest: File, 15 | mut parts: P, 16 | _path: Arc, 17 | shutdown: ShutdownManager<()>, 18 | ) -> Result<(), Error> 19 | where 20 | P: Stream, File), Error>> + Send + Unpin, 21 | { 22 | let main = async move { 23 | let _token = match shutdown.delay_shutdown_token() { 24 | Ok(token) => token, 25 | Err(_) => return Err(Error::Canceled), 26 | }; 27 | 28 | let task = async { 29 | while let Some(task_result) = parts.next().await { 30 | crate::utils::shutdown_check(&shutdown)?; 31 | 32 | let (source, mut source_file) = task_result?; 33 | concatenate(&mut dest, source, &mut source_file)?; 34 | } 35 | 36 | Ok(()) 37 | }; 38 | 39 | let result = task.await; 40 | 41 | result 42 | }; 43 | 44 | tokio::task::spawn_blocking(|| futures::executor::block_on(main)) 45 | .await 46 | .unwrap() 47 | } 48 | 49 | /// Concatenates a part into a file. 50 | fn concatenate( 51 | concatenated_file: &mut File, 52 | part_path: Arc, 53 | part_file: &mut File, 54 | ) -> Result<(), Error> { 55 | copy(part_file, concatenated_file).map_err(Error::Concatenate)?; 56 | 57 | if let Err(why) = fs::remove_file(&*part_path) { 58 | error!("failed to remove part file ({:?}): {}", part_path, why); 59 | } 60 | 61 | Ok(()) 62 | } 63 | -------------------------------------------------------------------------------- /src/get.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021-2022 System76 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | use super::*; 5 | use std::fs::File; 6 | use std::io::{Seek, SeekFrom, Write}; 7 | use std::path::Path; 8 | use std::sync::atomic::{AtomicU16, Ordering}; 9 | use std::sync::Arc; 10 | use std::time::Instant; 11 | 12 | pub(crate) struct FetchLocation { 13 | pub(crate) file: std::fs::File, 14 | pub(crate) dest: Arc, 15 | } 16 | 17 | impl FetchLocation { 18 | pub async fn create(dest: Arc, append: bool) -> Result { 19 | let mut builder = std::fs::OpenOptions::new(); 20 | 21 | builder.create(true).write(true).read(true); 22 | 23 | if append { 24 | builder.append(true); 25 | } else { 26 | builder.truncate(true); 27 | } 28 | 29 | let file = builder.open(&dest).map_err(Error::FileCreate)?; 30 | 31 | Ok(Self { file, dest }) 32 | } 33 | } 34 | pub(crate) async fn get( 35 | fetcher: Arc>, 36 | request: RequestBuilder, 37 | file: FetchLocation, 38 | final_destination: Arc, 39 | extra: Arc, 40 | attempts: Arc, 41 | ) -> Result<(Arc, File), crate::Error> { 42 | crate::utils::shutdown_check(&fetcher.shutdown)?; 43 | 44 | let shutdown = fetcher.shutdown.clone(); 45 | let FetchLocation { file, dest } = file; 46 | 47 | let main = async move { 48 | let _token = match shutdown.delay_shutdown_token() { 49 | Ok(token) => token, 50 | Err(_) => return Err(Error::Canceled), 51 | }; 52 | 53 | match &fetcher.client { 54 | Client::Reqwest(client) => 55 | { 56 | #[allow(irrefutable_let_patterns)] 57 | if let RequestBuilder::Reqwest(request) = request { 58 | let request = request.build().expect("failed to build request"); 59 | 60 | let req = async { client.execute(request).await.map_err(Error::from) }; 61 | 62 | let initial_response = 63 | crate::utils::timed_interrupt(Duration::from_secs(10), req).await?; 64 | 65 | if initial_response.status() == StatusCode::NOT_MODIFIED { 66 | return Ok::<_, crate::Error>((dest, file)); 67 | } 68 | 69 | let response = validate_reqwest(initial_response)? 70 | .bytes_stream() 71 | .map_err(|e| futures::io::Error::new(futures::io::ErrorKind::Other, e)) 72 | .into_async_read(); 73 | 74 | fetch_loop( 75 | fetcher, 76 | file, 77 | dest, 78 | final_destination, 79 | extra, 80 | attempts, 81 | shutdown, 82 | response, 83 | ) 84 | .await 85 | } else { 86 | Err(crate::Error::InvalidGetRequestBuilder) 87 | } 88 | } 89 | } 90 | }; 91 | 92 | tokio::task::spawn_blocking(|| futures::executor::block_on(main)) 93 | .await 94 | .unwrap() 95 | } 96 | 97 | #[allow(clippy::too_many_arguments)] 98 | async fn fetch_loop( 99 | fetcher: Arc>, 100 | mut file: File, 101 | dest: Arc, 102 | final_destination: Arc, 103 | extra: Arc, 104 | attempts: Arc, 105 | shutdown: ShutdownManager<()>, 106 | mut response: Response, 107 | ) -> Result<(Arc, File), crate::Error> { 108 | let mut read_total = 0; 109 | 110 | let mut now = Instant::now(); 111 | 112 | let update_progress = |progress: usize| { 113 | fetcher.send(|| { 114 | ( 115 | final_destination.clone(), 116 | extra.clone(), 117 | FetchEvent::Progress(progress as u64), 118 | ) 119 | }); 120 | }; 121 | 122 | let mut buffer = vec![0u8; 8192]; 123 | 124 | let fetch_loop = async { 125 | loop { 126 | if shutdown.is_shutdown_triggered() || shutdown.is_shutdown_completed() { 127 | return Err(Error::Canceled); 128 | } 129 | 130 | let bytes_read = async { response.read(&mut buffer).await.map_err(Error::Read) }; 131 | 132 | let read = match fetcher.timeout { 133 | Some(timeout) => crate::utils::timed_interrupt(timeout, bytes_read).await, 134 | None => crate::utils::network_interrupt(bytes_read).await, 135 | }?; 136 | 137 | if read == 0 { 138 | break; 139 | } 140 | 141 | read_total += read; 142 | 143 | file.write_all(&buffer[..read]).map_err(Error::Write)?; 144 | 145 | if now.elapsed().as_millis() as u64 > fetcher.progress_interval { 146 | update_progress(read_total); 147 | 148 | now = Instant::now(); 149 | read_total = 0; 150 | } 151 | 152 | attempts.store(0, Ordering::SeqCst); 153 | } 154 | 155 | Ok(()) 156 | }; 157 | 158 | let fetch_result = fetch_loop.await; 159 | let seek_result = file.seek(SeekFrom::Start(0)).map_err(Error::Write); 160 | 161 | let result = fetch_result.and(seek_result); 162 | 163 | if result.is_ok() && read_total != 0 { 164 | update_progress(read_total); 165 | } 166 | 167 | result.map(|_| (dest, file)) 168 | } 169 | -------------------------------------------------------------------------------- /src/get_many.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021-2022 System76 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | use crate::get::FetchLocation; 5 | use crate::*; 6 | use std::sync::atomic::AtomicU16; 7 | 8 | #[allow(clippy::too_many_arguments)] 9 | pub async fn get_many( 10 | fetcher: Arc>, 11 | to: Arc, 12 | uris: Arc<[Box]>, 13 | offset: u64, 14 | length: u64, 15 | modified: Option, 16 | extra: Arc, 17 | attempts: Arc, 18 | ) -> Result<(), Error> { 19 | let shutdown = fetcher.shutdown.clone(); 20 | let parent = to.parent().ok_or(Error::Parentless)?.to_owned(); 21 | let filename = to.file_name().ok_or(Error::Nameless)?.to_owned(); 22 | 23 | let mut buf = [0u8; 20]; 24 | 25 | let FetchLocation { file, .. } = FetchLocation::create(to.clone(), offset != 0).await?; 26 | 27 | let concurrent_fetches = fetcher.connections_per_file as usize; 28 | 29 | let to_ = to.clone(); 30 | let parts = 31 | stream::iter(range::generate(length, fetcher.max_part_size.into(), offset).enumerate()) 32 | // Generate a future for fetching each part that a range describes. 33 | .map(move |(partn, (range_start, range_end))| { 34 | let uri = uris[partn % uris.len()].clone(); 35 | 36 | let part_path = { 37 | let mut new_filename = filename.to_os_string(); 38 | new_filename.push(&[".part", partn.numtoa_str(10, &mut buf)].concat()); 39 | parent.join(new_filename) 40 | }; 41 | 42 | if part_path.exists() { 43 | let _ = std::fs::remove_file(&part_path); 44 | } 45 | 46 | let fetcher = fetcher.clone(); 47 | let to = to_.clone(); 48 | let extra = extra.clone(); 49 | let attempts = attempts.clone(); 50 | 51 | let builder = match &fetcher.client { 52 | Client::Reqwest(client) => RequestBuilder::Reqwest(client.get(&*uri)), 53 | }; 54 | 55 | async move { 56 | let range = range::to_string(range_start, Some(range_end)); 57 | let part_path: Arc = Arc::from(part_path); 58 | 59 | let request = match builder { 60 | RequestBuilder::Reqwest(inner) => { 61 | RequestBuilder::Reqwest(inner.header("range", range.as_str())) 62 | } 63 | }; 64 | 65 | crate::get( 66 | fetcher.clone(), 67 | request, 68 | FetchLocation::create(part_path.clone(), false).await?, 69 | to.clone(), 70 | extra.clone(), 71 | attempts.clone(), 72 | ) 73 | .await 74 | } 75 | }) 76 | // Ensure that only this many connections are happenning concurrently at a 77 | // time 78 | .buffered(concurrent_fetches); 79 | 80 | let _shutdown_token = shutdown.delay_shutdown_token(); 81 | 82 | concatenator(file, parts, to.clone(), shutdown.clone()).await?; 83 | 84 | if let Some(modified) = modified { 85 | crate::time::update_modified(&to, modified)?; 86 | } 87 | 88 | Ok(()) 89 | } 90 | -------------------------------------------------------------------------------- /src/iface.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | collections::hash_map::DefaultHasher, 3 | future::Future, 4 | hash::{Hash, Hasher}, 5 | }; 6 | 7 | /// Get the current state of network connections as a hash. 8 | pub fn state() -> u64 { 9 | let mut hash = DefaultHasher::new(); 10 | 11 | if let Ok(ifaces) = ifaces::Interface::get_all() { 12 | for iface in ifaces { 13 | iface.addr.hash(&mut hash); 14 | } 15 | } 16 | 17 | hash.finish() 18 | } 19 | 20 | /// Future which exits when the network state has changed. 21 | pub async fn watch_change() { 22 | let current = state(); 23 | 24 | loop { 25 | tokio::time::sleep(std::time::Duration::from_secs(1)).await; 26 | let new = state(); 27 | if new != current { 28 | break; 29 | } 30 | } 31 | } 32 | 33 | /// Re-attempts a request on network changes. 34 | pub async fn reconnect_on_change(func: Func, cont: Retry) -> Res 35 | where 36 | Func: Fn() -> Fut, 37 | Fut: Future, 38 | Retry: Fn() -> Cont, 39 | Cont: Future>, 40 | { 41 | loop { 42 | let changed = watch_change(); 43 | let future = func(); 44 | 45 | futures::pin_mut!(future); 46 | futures::pin_mut!(changed); 47 | 48 | use futures::future::Either; 49 | 50 | match futures::future::select(future, changed).await { 51 | Either::Left((result, _)) => break result, 52 | Either::Right(_) => { 53 | if let Some(result) = cont().await { 54 | break result; 55 | } 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021-2022 System76 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | //! Asynchronously fetch files from HTTP servers 5 | //! 6 | //! - Concurrently fetch multiple files at the same time. 7 | //! - Define alternative mirrors for the source of a file. 8 | //! - Use multiple concurrent connections per file. 9 | //! - Use mirrors for concurrent connections. 10 | //! - Resume a download which has been interrupted. 11 | //! - Progress events for fetches 12 | //! 13 | //! ``` 14 | //! let (events_tx, events_rx) = tokio::sync::mpsc::unbounded_channel(); 15 | //! 16 | //! let shutdown = async_shutdown::ShutdownManager::new(); 17 | //! 18 | //! let results_stream = Fetcher::default() 19 | //! // Define a max number of ranged connections per file. 20 | //! .connections_per_file(4) 21 | //! // Max size of a connection's part, concatenated on completion. 22 | //! .max_part_size(4 * 1024 * 1024) 23 | //! // The channel for sending progress notifications. 24 | //! .events(events_tx) 25 | //! // Maximum number of retry attempts. 26 | //! .retries(3) 27 | //! // Cancels the fetching process when a shutdown is triggered. 28 | //! .shutdown(shutdown) 29 | //! // How long to wait before aborting a download that hasn't progressed. 30 | //! .timeout(Duration::from_secs(15)) 31 | //! // Finalize the struct into an `Arc` for use with fetching. 32 | //! .build() 33 | //! // Take a stream of `Source` inputs and generate a stream of fetches. 34 | //! // Spawns 35 | //! .stream_from(input_stream, 4) 36 | //! ``` 37 | 38 | #[macro_use] 39 | extern crate derive_new; 40 | #[macro_use] 41 | extern crate derive_setters; 42 | #[macro_use] 43 | extern crate log; 44 | #[macro_use] 45 | extern crate thiserror; 46 | 47 | pub mod iface; 48 | 49 | mod checksum; 50 | mod checksum_system; 51 | mod concatenator; 52 | mod get; 53 | mod get_many; 54 | mod range; 55 | mod source; 56 | mod time; 57 | mod utils; 58 | 59 | pub use self::checksum::*; 60 | pub use self::checksum_system::*; 61 | pub use self::concatenator::*; 62 | pub use self::source::*; 63 | 64 | use self::get::{get, FetchLocation}; 65 | use self::get_many::get_many; 66 | use self::time::{date_as_timestamp, update_modified}; 67 | use async_shutdown::ShutdownManager; 68 | use futures::{ 69 | prelude::*, 70 | stream::{self, StreamExt}, 71 | }; 72 | 73 | use http::StatusCode; 74 | use httpdate::HttpDate; 75 | use numtoa::NumToA; 76 | use reqwest::redirect::Policy; 77 | use reqwest::{ 78 | Client as ReqwestClient, RequestBuilder as ReqwestBuilder, Response as ReqwestResponse, 79 | }; 80 | 81 | use std::sync::atomic::Ordering; 82 | use std::{ 83 | fmt::Debug, 84 | io, 85 | path::Path, 86 | pin::Pin, 87 | sync::{atomic::AtomicU16, Arc}, 88 | time::{Duration, UNIX_EPOCH}, 89 | }; 90 | use tokio::fs; 91 | use tokio::sync::mpsc; 92 | 93 | /// The result of a fetched task from a stream of input sources. 94 | pub type AsyncFetchOutput = (Arc, Arc, Result<(), Error>); 95 | 96 | /// A channel for sending `FetchEvent`s to. 97 | pub type EventSender = mpsc::UnboundedSender<(Arc, Data, FetchEvent)>; 98 | 99 | /// An error from the asynchronous file fetcher. 100 | #[derive(Debug, Error)] 101 | pub enum Error { 102 | #[error("task was canceled")] 103 | Canceled, 104 | #[error("http client error")] 105 | ReqwestClient(#[source] reqwest::Error), 106 | #[error("unable to concatenate fetched parts")] 107 | Concatenate(#[source] io::Error), 108 | #[error("unable to create file")] 109 | FileCreate(#[source] io::Error), 110 | #[error("unable to set timestamp on {:?}", _0)] 111 | FileTime(Arc, #[source] io::Error), 112 | #[error("content length is an invalid range")] 113 | InvalidRange(#[source] io::Error), 114 | #[error("unable to remove file with bad metadata")] 115 | MetadataRemove(#[source] io::Error), 116 | #[error("destination has no file name")] 117 | Nameless, 118 | #[error("network connection was interrupted while fetching")] 119 | NetworkChanged, 120 | #[error("unable to open fetched part")] 121 | OpenPart(Arc, #[source] io::Error), 122 | #[error("destination lacks parent")] 123 | Parentless, 124 | #[error("connection timed out")] 125 | TimedOut, 126 | #[error("error writing to file")] 127 | Write(#[source] io::Error), 128 | #[error("network input error")] 129 | Read(#[source] io::Error), 130 | #[error("failed to rename partial to destination")] 131 | Rename(#[source] io::Error), 132 | #[error("server responded with an error: {}", _0)] 133 | Status(StatusCode), 134 | #[error("internal tokio join handle error")] 135 | TokioSpawn(#[source] tokio::task::JoinError), 136 | #[error("the request builder did not match the client used")] 137 | InvalidGetRequestBuilder, 138 | } 139 | 140 | impl From for Error { 141 | fn from(e: reqwest::Error) -> Self { 142 | Self::ReqwestClient(e) 143 | } 144 | } 145 | 146 | /// Events which are submitted by the fetcher. 147 | #[derive(Debug)] 148 | pub enum FetchEvent { 149 | /// States that we know the length of the file being fetched. 150 | ContentLength(u64), 151 | /// Notifies that the file has been fetched. 152 | Fetched, 153 | /// Notifies that a file is being fetched. 154 | Fetching, 155 | /// Reports the amount of bytes that have been read for a file. 156 | Progress(u64), 157 | /// Notification that a fetch is being re-attempted. 158 | Retrying, 159 | } 160 | 161 | /// An asynchronous file fetcher for clients fetching files. 162 | /// 163 | /// The futures generated by the fetcher are compatible with single and multi-threaded 164 | /// runtimes, allowing you to choose between the runtime that works best for your 165 | /// application. A single-threaded runtime is generally recommended for fetching files, 166 | /// as your network connection is unlikely to be faster than a single CPU core. 167 | #[derive(new, Setters)] 168 | pub struct Fetcher { 169 | /// Creates an instance of a client. The caller can decide if the instance 170 | /// is shared or unique. 171 | #[setters(skip)] 172 | client: Client, 173 | 174 | /// The number of concurrent connections to sustain per file being fetched. 175 | /// # Note 176 | /// Defaults to 1 connection 177 | #[new(value = "1")] 178 | connections_per_file: u16, 179 | 180 | /// Configure the delay between file requests. 181 | /// # Note 182 | /// Defaults to no delay 183 | #[new(value = "0")] 184 | delay_between_requests: u64, 185 | 186 | /// The number of attempts to make when a request fails. 187 | /// # Note 188 | /// Defaults to 3 retries. 189 | #[new(value = "3")] 190 | retries: u16, 191 | 192 | /// The maximum size of a part file when downloading in parts. 193 | /// # Note 194 | /// Defaults to 2 MiB. 195 | #[new(value = "2 * 1024 * 1024")] 196 | max_part_size: u32, 197 | 198 | /// Time in ms between progress messages 199 | /// # Note 200 | /// Defaults to 500. 201 | #[new(value = "500")] 202 | progress_interval: u64, 203 | 204 | /// The time to wait between chunks before giving up. 205 | #[new(default)] 206 | #[setters(strip_option)] 207 | timeout: Option, 208 | 209 | /// Holds a sender for submitting events to. 210 | #[new(default)] 211 | #[setters(into)] 212 | #[setters(strip_option)] 213 | events: Option>>>, 214 | 215 | /// Utilized to know when to shut down the fetching process. 216 | #[new(value = "ShutdownManager::new()")] 217 | shutdown: ShutdownManager<()>, 218 | } 219 | 220 | /// The underlying Client used for the Fetcher 221 | pub enum Client { 222 | Reqwest(ReqwestClient), 223 | } 224 | 225 | pub(crate) enum RequestBuilder { 226 | Reqwest(ReqwestBuilder), 227 | } 228 | 229 | impl Default for Fetcher { 230 | fn default() -> Self { 231 | let client = ReqwestClient::builder() 232 | // Keep a TCP connection alive for up to 90s 233 | .tcp_keepalive(Duration::from_secs(90)) 234 | // Follow up to 10 redirect links 235 | .redirect(Policy::limited(10)) 236 | // Allow the server to be eager about sending packets 237 | .tcp_nodelay(true) 238 | // Cache DNS records for 24 hours 239 | // .dns_cache(Duration::from_secs(60 * 60 * 24)) 240 | .build() 241 | .expect("failed to create HTTP Client"); 242 | 243 | Self::new(Client::Reqwest(client)) 244 | } 245 | } 246 | 247 | impl Fetcher { 248 | /// Finalizes the fetcher to prepare it for fetch tasks. 249 | pub fn build(self) -> Arc { 250 | Arc::new(self) 251 | } 252 | 253 | /// Given an input stream of source fetches, returns an output stream of fetch results. 254 | /// 255 | /// Spawns up to `concurrent` + `1` number of concurrent async tasks on the runtime. 256 | /// One task for managing the fetch tasks, and one task per fetch request. 257 | pub fn stream_from( 258 | self: Arc, 259 | inputs: impl Stream)> + Send + 'static, 260 | concurrent: usize, 261 | ) -> Pin> + Send + 'static>> { 262 | let shutdown = self.shutdown.clone(); 263 | let cancel_trigger = shutdown.wait_shutdown_triggered(); 264 | // Takes input requests and converts them into a stream of fetch requests. 265 | let stream = inputs 266 | .map(move |(Source { dest, urls, part }, extra)| { 267 | let fetcher = self.clone(); 268 | async move { 269 | if fetcher.delay_between_requests != 0 { 270 | let delay = Duration::from_millis(fetcher.delay_between_requests); 271 | tokio::time::sleep(delay).await; 272 | } 273 | 274 | tokio::spawn(async move { 275 | let _token = match fetcher.shutdown.delay_shutdown_token() { 276 | Ok(token) => token, 277 | Err(_) => return (dest, extra, Err(Error::Canceled)), 278 | }; 279 | 280 | let task = async { 281 | match part { 282 | Some(part) => { 283 | match fetcher.request(urls, part.clone(), extra.clone()).await { 284 | Ok(()) => { 285 | fs::rename(&*part, &*dest).await.map_err(Error::Rename) 286 | } 287 | Err(why) => Err(why), 288 | } 289 | } 290 | None => fetcher.request(urls, dest.clone(), extra.clone()).await, 291 | } 292 | }; 293 | 294 | let result = task.await; 295 | 296 | (dest, extra, result) 297 | }) 298 | .await 299 | .unwrap() 300 | } 301 | }) 302 | .buffer_unordered(concurrent) 303 | .take_until(cancel_trigger); 304 | 305 | Box::pin(stream) 306 | } 307 | 308 | /// Request a file from one or more URIs. 309 | /// 310 | /// At least one URI must be provided as a source for the file. Each additional URI 311 | /// serves as a mirror for failover and load-balancing purposes. 312 | pub async fn request( 313 | self: Arc, 314 | uris: Arc<[Box]>, 315 | to: Arc, 316 | extra: Arc, 317 | ) -> Result<(), Error> { 318 | self.send(|| (to.clone(), extra.clone(), FetchEvent::Fetching)); 319 | 320 | remove_parts(&to).await; 321 | 322 | let attempts = Arc::new(AtomicU16::new(0)); 323 | 324 | let fetch = || async { 325 | loop { 326 | let task = self.clone().inner_request( 327 | &self.client, 328 | uris.clone(), 329 | to.clone(), 330 | extra.clone(), 331 | attempts.clone(), 332 | ); 333 | 334 | let result = task.await; 335 | 336 | if let Err(Error::NetworkChanged) | Err(Error::TimedOut) = result { 337 | let mut attempts = 5; 338 | while attempts != 0 { 339 | tokio::time::sleep(Duration::from_secs(3)).await; 340 | 341 | match &self.client { 342 | Client::Reqwest(client) => { 343 | let future = head_reqwest(client, &uris[0]); 344 | let net_check = 345 | crate::utils::timed_interrupt(Duration::from_secs(3), future); 346 | 347 | if net_check.await.is_ok() { 348 | tokio::time::sleep(Duration::from_secs(3)).await; 349 | break; 350 | } 351 | } 352 | }; 353 | 354 | attempts -= 1; 355 | } 356 | 357 | self.send(|| (to.clone(), extra.clone(), FetchEvent::Retrying)); 358 | remove_parts(&to).await; 359 | tokio::time::sleep(Duration::from_secs(3)).await; 360 | 361 | continue; 362 | } 363 | 364 | return result; 365 | } 366 | }; 367 | 368 | let task = async { 369 | let mut attempted = false; 370 | loop { 371 | if attempted { 372 | self.send(|| (to.clone(), extra.clone(), FetchEvent::Retrying)); 373 | } 374 | 375 | attempted = true; 376 | remove_parts(&to).await; 377 | 378 | let error = match fetch().await { 379 | Ok(()) => return Ok(()), 380 | Err(error) => error, 381 | }; 382 | 383 | if let Error::Canceled = error { 384 | return Err(error); 385 | } 386 | 387 | tokio::time::sleep(Duration::from_secs(3)).await; 388 | 389 | // Uncondtionally retry connection errors. 390 | if let Error::ReqwestClient(ref error) = error { 391 | use std::error::Error; 392 | if let Some(source) = error.source() { 393 | if let Some(error) = source.downcast_ref::() { 394 | if error.is_connect() || error.is_request() { 395 | error!("retrying due to connection error: {}", error); 396 | continue; 397 | } 398 | } 399 | } 400 | } 401 | 402 | error!("retrying after error encountered: {}", error); 403 | 404 | if attempts.fetch_add(1, Ordering::SeqCst) > self.retries { 405 | return Err(error); 406 | } 407 | } 408 | }; 409 | 410 | let result = task.await; 411 | 412 | remove_parts(&to).await; 413 | 414 | match result { 415 | Ok(()) => { 416 | self.send(|| (to.clone(), extra.clone(), FetchEvent::Fetched)); 417 | 418 | Ok(()) 419 | } 420 | Err(why) => Err(why), 421 | } 422 | } 423 | 424 | async fn inner_request( 425 | self: Arc, 426 | client: &Client, 427 | uris: Arc<[Box]>, 428 | to: Arc, 429 | extra: Arc, 430 | attempts: Arc, 431 | ) -> Result<(), Error> { 432 | let mut length = None; 433 | let mut modified = None; 434 | let mut resume = 0; 435 | 436 | match client { 437 | Client::Reqwest(client) => { 438 | let head_response = head_reqwest(client, &*uris[0]).await?; 439 | 440 | if let Some(response) = head_response.as_ref() { 441 | length = response 442 | .headers() 443 | .get(reqwest::header::CONTENT_LENGTH) 444 | .and_then(|value| value.to_str().ok()) 445 | .and_then(|value| value.parse().ok()); 446 | modified = response.last_modified(); 447 | } 448 | } 449 | } 450 | 451 | // If the file already exists, validate that it is the same. 452 | if to.exists() { 453 | if let (Some(length), Some(last_modified)) = (length, modified) { 454 | match fs::metadata(to.as_ref()).await { 455 | Ok(metadata) => { 456 | let modified = metadata.modified().map_err(Error::Write)?; 457 | let ts = modified 458 | .duration_since(UNIX_EPOCH) 459 | .expect("time went backwards"); 460 | 461 | if metadata.len() == length { 462 | if ts.as_secs() == date_as_timestamp(last_modified) { 463 | info!("already fetched {}", to.display()); 464 | return Ok(()); 465 | } else { 466 | error!("removing file with outdated timestamp: {:?}", to); 467 | let _ = fs::remove_file(to.as_ref()) 468 | .await 469 | .map_err(Error::MetadataRemove)?; 470 | } 471 | } else { 472 | resume = metadata.len(); 473 | } 474 | } 475 | Err(why) => { 476 | error!("failed to fetch metadata of {:?}: {}", to, why); 477 | fs::remove_file(to.as_ref()) 478 | .await 479 | .map_err(Error::MetadataRemove)?; 480 | } 481 | } 482 | } 483 | } 484 | 485 | // If set, this will use multiple connections to download a file in parts. 486 | if self.connections_per_file > 1 { 487 | if let Some(length) = length { 488 | if supports_range(client, &*uris[0], resume, Some(length)).await? { 489 | self.send(|| (to.clone(), extra.clone(), FetchEvent::ContentLength(length))); 490 | 491 | if resume != 0 { 492 | self.send(|| (to.clone(), extra.clone(), FetchEvent::Progress(resume))); 493 | } 494 | 495 | let result = get_many( 496 | self.clone(), 497 | to.clone(), 498 | uris, 499 | resume, 500 | length, 501 | modified, 502 | extra, 503 | attempts.clone(), 504 | ) 505 | .await; 506 | 507 | if let Err(why) = result { 508 | return Err(why); 509 | } 510 | 511 | if let Some(modified) = modified { 512 | update_modified(&to, modified)?; 513 | } 514 | 515 | return Ok(()); 516 | } 517 | } 518 | } 519 | 520 | if let Some(length) = length { 521 | self.send(|| (to.clone(), extra.clone(), FetchEvent::ContentLength(length))); 522 | 523 | if resume > length { 524 | resume = 0; 525 | } 526 | } 527 | 528 | let mut request = match client { 529 | Client::Reqwest(client) => RequestBuilder::Reqwest(client.get(&*uris[0])), 530 | }; 531 | 532 | if resume != 0 { 533 | if let Ok(true) = supports_range(client, &*uris[0], resume, length).await { 534 | match request { 535 | RequestBuilder::Reqwest(inner) => { 536 | request = RequestBuilder::Reqwest( 537 | inner.header("Range", range::to_string(resume, length)), 538 | ); 539 | } 540 | } 541 | self.send(|| (to.clone(), extra.clone(), FetchEvent::Progress(resume))); 542 | } else { 543 | resume = 0; 544 | } 545 | } 546 | 547 | let path = match crate::get( 548 | self.clone(), 549 | request, 550 | FetchLocation::create(to.clone(), resume != 0).await?, 551 | to.clone(), 552 | extra.clone(), 553 | attempts.clone(), 554 | ) 555 | .await 556 | { 557 | Ok((path, _)) => path, 558 | Err(Error::Status(StatusCode::NOT_MODIFIED)) => to, 559 | 560 | // Server does not support if-modified-since 561 | Err(Error::Status(StatusCode::NOT_IMPLEMENTED)) => { 562 | let request = match client { 563 | Client::Reqwest(client) => RequestBuilder::Reqwest(client.get(&*uris[0])), 564 | }; 565 | 566 | let (path, _) = crate::get( 567 | self.clone(), 568 | request, 569 | FetchLocation::create(to.clone(), resume != 0).await?, 570 | to.clone(), 571 | extra.clone(), 572 | attempts, 573 | ) 574 | .await?; 575 | 576 | path 577 | } 578 | 579 | Err(why) => return Err(why), 580 | }; 581 | 582 | if let Some(modified) = modified { 583 | update_modified(&path, modified)?; 584 | } 585 | 586 | Ok(()) 587 | } 588 | 589 | fn send(&self, event: impl FnOnce() -> (Arc, Arc, FetchEvent)) { 590 | if let Some(sender) = self.events.as_ref() { 591 | let _ = sender.send(event()); 592 | } 593 | } 594 | } 595 | 596 | async fn head_reqwest(client: &ReqwestClient, uri: &str) -> Result, Error> { 597 | let request = client.head(uri).build().unwrap(); 598 | 599 | match validate_reqwest(client.execute(request).await?).map(Some) { 600 | result @ Ok(_) => result, 601 | Err(Error::Status(StatusCode::NOT_MODIFIED)) 602 | | Err(Error::Status(StatusCode::NOT_IMPLEMENTED)) => Ok(None), 603 | Err(other) => Err(other), 604 | } 605 | } 606 | 607 | async fn supports_range( 608 | client: &Client, 609 | uri: &str, 610 | resume: u64, 611 | length: Option, 612 | ) -> Result { 613 | match client { 614 | Client::Reqwest(client) => { 615 | let request = client 616 | .head(uri) 617 | .header("Range", range::to_string(resume, length).as_str()) 618 | .build() 619 | .unwrap(); 620 | 621 | let response = client.execute(request).await?; 622 | 623 | if response.status() == StatusCode::PARTIAL_CONTENT { 624 | if let Some(header) = response.headers().get("Content-Range") { 625 | if let Ok(header) = header.to_str() { 626 | if header.starts_with(&format!("bytes {}-", resume)) { 627 | return Ok(true); 628 | } 629 | } 630 | } 631 | 632 | Ok(false) 633 | } else { 634 | validate_reqwest(response).map(|_| false) 635 | } 636 | } 637 | } 638 | } 639 | 640 | fn validate_reqwest(response: ReqwestResponse) -> Result { 641 | let status = response.status(); 642 | 643 | if status.is_informational() || status.is_success() { 644 | Ok(response) 645 | } else { 646 | Err(Error::Status(status)) 647 | } 648 | } 649 | 650 | trait ResponseExt { 651 | fn content_length(&self) -> Option; 652 | fn last_modified(&self) -> Option; 653 | } 654 | 655 | impl ResponseExt for ReqwestResponse { 656 | fn content_length(&self) -> Option { 657 | let header = self.headers().get("content-length")?; 658 | header.to_str().ok()?.parse::().ok() 659 | } 660 | 661 | fn last_modified(&self) -> Option { 662 | let header = self.headers().get("last-modified")?; 663 | httpdate::parse_http_date(header.to_str().ok()?) 664 | .ok() 665 | .map(HttpDate::from) 666 | } 667 | } 668 | 669 | /// Cleans up after a process that may have been aborted. 670 | async fn remove_parts(to: &Path) { 671 | let original_filename = match to.file_name().and_then(|x| x.to_str()) { 672 | Some(name) => name, 673 | None => return, 674 | }; 675 | 676 | if let Some(parent) = to.parent() { 677 | if let Ok(mut dir) = tokio::fs::read_dir(parent).await { 678 | while let Ok(Some(entry)) = dir.next_entry().await { 679 | if let Some(entry_name) = entry.file_name().to_str() { 680 | if let Some(potential_part) = entry_name.strip_prefix(original_filename) { 681 | if potential_part.starts_with(".part") { 682 | let path = entry.path(); 683 | let _ = tokio::fs::remove_file(path).await; 684 | } 685 | } 686 | } 687 | } 688 | } 689 | } 690 | } 691 | -------------------------------------------------------------------------------- /src/range.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021-2022 System76 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | use numtoa::NumToA; 5 | 6 | pub fn generate( 7 | mut length: u64, 8 | max_part_size: u64, 9 | mut offset: u64, 10 | ) -> impl Iterator + Send + 'static { 11 | length -= offset; 12 | 13 | std::iter::from_fn(move || { 14 | if length == 0 { 15 | return None; 16 | } 17 | 18 | let next; 19 | if length > max_part_size { 20 | next = (offset, offset + max_part_size - 1); 21 | offset += max_part_size; 22 | length -= max_part_size 23 | } else { 24 | next = (offset, offset + length - 1); 25 | length = 0; 26 | } 27 | 28 | Some(next) 29 | }) 30 | } 31 | 32 | pub(crate) fn to_string(from: u64, to: Option) -> String { 33 | let mut from_a = [0u8; 20]; 34 | let mut to_a = [0u8; 20]; 35 | [ 36 | "bytes=", 37 | from.numtoa_str(10, &mut from_a), 38 | "-", 39 | if let Some(to) = to { 40 | to.numtoa_str(10, &mut to_a) 41 | } else { 42 | "" 43 | }, 44 | ] 45 | .concat() 46 | } 47 | -------------------------------------------------------------------------------- /src/source.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021-2022 System76 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | use std::path::Path; 5 | use std::sync::Arc; 6 | 7 | /// Information about a source being fetched. 8 | #[derive(Debug)] 9 | pub struct Source { 10 | /// URLs whereby the file can be found. 11 | pub urls: Arc<[Box]>, 12 | 13 | /// Where the file shall ultimately be fetched to. 14 | pub dest: Arc, 15 | 16 | /// Where partial files should be stored. 17 | pub part: Option>, 18 | } 19 | 20 | impl Source { 21 | pub fn builder(dest: Arc, url: Box) -> SourceBuilder { 22 | SourceBuilder::new(dest, url) 23 | } 24 | 25 | pub fn new(urls: Arc<[Box]>, dest: Arc) -> Self { 26 | Self { 27 | urls, 28 | dest, 29 | part: None, 30 | } 31 | } 32 | 33 | /// Sets the partial destination of a source. 34 | pub fn set_part(&mut self, part: Option>) { 35 | self.part = part; 36 | } 37 | } 38 | 39 | /// Constructs a `Source`. 40 | pub struct SourceBuilder { 41 | urls: Vec>, 42 | dest: Arc, 43 | part: Option>, 44 | } 45 | 46 | impl SourceBuilder { 47 | pub fn new(dest: Arc, url: Box) -> Self { 48 | SourceBuilder { 49 | dest, 50 | urls: vec![url], 51 | part: None, 52 | } 53 | } 54 | 55 | /// A mirror where the source can be located. 56 | pub fn append_url(mut self, url: Box) -> Self { 57 | self.urls.push(url); 58 | self 59 | } 60 | 61 | /// A partial destination for a source. 62 | pub fn partial(mut self, part: Arc) -> Self { 63 | self.part = Some(part); 64 | self 65 | } 66 | 67 | pub fn build(self) -> Source { 68 | Source { 69 | urls: Arc::from(self.urls), 70 | dest: self.dest, 71 | part: self.part, 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/time.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021-2022 System76 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | use crate::Error; 5 | use filetime::FileTime; 6 | use httpdate::HttpDate; 7 | use std::path::Path; 8 | use std::sync::Arc; 9 | use std::time::{SystemTime, UNIX_EPOCH}; 10 | 11 | pub fn date_as_timestamp(date: HttpDate) -> u64 { 12 | SystemTime::from(date) 13 | .duration_since(UNIX_EPOCH) 14 | .expect("time backwards") 15 | .as_secs() 16 | } 17 | 18 | pub fn update_modified(to: &Arc, modified: HttpDate) -> Result<(), Error> { 19 | let filetime = FileTime::from_unix_time(date_as_timestamp(modified) as i64, 0); 20 | filetime::set_file_times(&to, filetime, filetime) 21 | .map_err(|why| Error::FileTime(to.clone(), why)) 22 | } 23 | -------------------------------------------------------------------------------- /src/utils.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021-2022 System76 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | use crate::Error; 5 | use futures::future::select; 6 | use std::future::Future; 7 | use std::time::Duration; 8 | 9 | pub async fn timed_interrupt(duration: Duration, future: F) -> Result 10 | where 11 | F: Future>, 12 | { 13 | run_timed(duration, network_interrupt(future)).await 14 | } 15 | 16 | pub async fn network_interrupt( 17 | future: impl Future>, 18 | ) -> Result { 19 | let ifaces_changed = async { 20 | crate::iface::watch_change().await; 21 | Err(Error::NetworkChanged) 22 | }; 23 | 24 | futures::pin_mut!(ifaces_changed); 25 | futures::pin_mut!(future); 26 | 27 | select(ifaces_changed, future).await.factor_first().0 28 | } 29 | 30 | pub async fn run_timed(duration: Duration, future: F) -> Result 31 | where 32 | F: Future>, 33 | { 34 | let timeout = async move { 35 | tokio::time::sleep(duration).await; 36 | Err(Error::TimedOut) 37 | }; 38 | 39 | futures::pin_mut!(future); 40 | futures::pin_mut!(timeout); 41 | 42 | select(timeout, future).await.factor_first().0 43 | } 44 | 45 | pub fn shutdown_check(shutdown: &async_shutdown::ShutdownManager<()>) -> Result<(), crate::Error> { 46 | if shutdown.is_shutdown_triggered() || shutdown.is_shutdown_completed() { 47 | return Err(Error::Canceled); 48 | } 49 | 50 | Ok(()) 51 | } 52 | --------------------------------------------------------------------------------