├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── config.yml │ └── feature_request.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── audit.yaml │ ├── cd.yaml │ └── ci.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── .reuse └── dep5 ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Cargo.toml ├── LICENSES ├── Apache-2.0.txt ├── LGPL-3.0-or-later.txt └── MIT.txt ├── README.md ├── cspell.json ├── examples ├── download.rs └── tui_basic.rs └── src ├── backend.rs ├── download.rs ├── downloader.rs ├── lib.rs ├── progress.rs └── verify.rs /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | # cSpell: ignore IssueHunt 4 | 5 | github: [hunger] 6 | patreon: # Replace with a single Patreon username 7 | open_collective: # Replace with a single Open Collective username 8 | ko_fi: # Replace with a single Ko-fi username 9 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 10 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 11 | liberapay: # Replace with a single Liberapay username 12 | issuehunt: # Replace with a single IssueHunt username 13 | otechie: # Replace with a single Otechie username 14 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 15 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | ______________________________________________________________________ 2 | 3 | name: Bug report 4 | about: Create a report to help us improve 5 | title: '' 6 | labels: bug 7 | assignees: '' 8 | 9 | ______________________________________________________________________ 10 | 11 | ## Bug description 12 | 13 | 14 | 15 | ## To Reproduce 16 | 17 | 24 | 25 | ## Expected behavior 26 | 27 | 28 | 29 | ## Screenshots 30 | 31 | 32 | 33 | ## Environment 34 | 35 | 36 | 37 | - OS: \[e.g. iOS\] 38 | - Browser \[e.g. chrome, safari\] 39 | - Version \[e.g. 22\] 40 | 41 | ## Additional context 42 | 43 | 44 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | ______________________________________________________________________ 2 | 3 | name: Feature request 4 | about: Suggest an idea for this project 5 | title: '' 6 | labels: enhancement 7 | assignees: '' 8 | 9 | ______________________________________________________________________ 10 | 11 | ## Problem 12 | 13 | 17 | 18 | ## Solution 19 | 20 | 21 | 22 | ## Alternatives 23 | 24 | 25 | 26 | ## Additional context 27 | 28 | 29 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 11 | -------------------------------------------------------------------------------- /.github/workflows/audit.yaml: -------------------------------------------------------------------------------- 1 | name: Security audit 2 | 3 | on: 4 | schedule: 5 | # Runs at 00:00 UTC everyday 6 | - cron: '0 0 * * *' 7 | push: 8 | paths: 9 | - '**/Cargo.toml' 10 | - '**/Cargo.lock' 11 | pull_request: 12 | 13 | jobs: 14 | audit: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Checkout repository 18 | uses: actions/checkout@v2 19 | - uses: actions-rs/audit-check@v1 20 | with: 21 | token: ${{ secrets.GITHUB_TOKEN }} 22 | -------------------------------------------------------------------------------- /.github/workflows/cd.yaml: -------------------------------------------------------------------------------- 1 | name: Continuous Deployment 2 | 3 | on: 4 | push: 5 | tags: 6 | - '[0-9]+.[0-9]+.[0-9]+' 7 | 8 | jobs: 9 | publish-cargo: 10 | name: Publishing to Cargo 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@master 14 | - uses: actions-rs/toolchain@v1 15 | with: 16 | toolchain: stable 17 | profile: minimal 18 | override: true 19 | - uses: actions-rs/cargo@v1 20 | with: 21 | command: publish 22 | args: --token ${{ secrets.CARGO_API_KEY }} --allow-dirty 23 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | name: Continuous Integration 3 | 4 | jobs: 5 | 6 | test: 7 | name: Test Suite 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Checkout repository 11 | uses: actions/checkout@v2 12 | - name: Install Rust 13 | uses: actions-rs/toolchain@v1 14 | with: 15 | toolchain: stable 16 | profile: minimal 17 | override: true 18 | - uses: actions-rs/cargo@v1 19 | with: 20 | command: test 21 | 22 | rustfmt: 23 | name: Rustfmt 24 | runs-on: ubuntu-latest 25 | steps: 26 | - name: Checkout repository 27 | uses: actions/checkout@v2 28 | - name: Install Rust 29 | uses: actions-rs/toolchain@v1 30 | with: 31 | toolchain: stable 32 | profile: minimal 33 | override: true 34 | components: rustfmt 35 | - name: Check formatting 36 | uses: actions-rs/cargo@v1 37 | with: 38 | command: fmt 39 | args: --all -- --check 40 | 41 | clippy: 42 | name: Clippy 43 | runs-on: ubuntu-latest 44 | steps: 45 | - name: Checkout repository 46 | uses: actions/checkout@v2 47 | - name: Install Rust 48 | uses: actions-rs/toolchain@v1 49 | with: 50 | toolchain: stable 51 | profile: minimal 52 | override: true 53 | components: clippy 54 | - name: Clippy Check 55 | uses: actions-rs/cargo@v1 56 | with: 57 | command: clippy 58 | args: -- -D warnings 59 | 60 | coverage: 61 | name: Code coverage 62 | runs-on: ubuntu-latest 63 | steps: 64 | - name: Checkout repository 65 | uses: actions/checkout@v2 66 | - name: Install stable toolchain 67 | uses: actions-rs/toolchain@v1 68 | with: 69 | toolchain: stable 70 | profile: minimal 71 | override: true 72 | - name: Run cargo-tarpaulin 73 | uses: actions-rs/tarpaulin@v0.1 74 | with: 75 | args: '--ignore-tests --out Lcov' 76 | - name: Upload to Coveralls 77 | # upload only if push 78 | if: ${{ github.event_name == 'push' }} 79 | uses: coverallsapp/github-action@master 80 | with: 81 | github-token: ${{ secrets.GITHUB_TOKEN }} 82 | path-to-lcov: './lcov.info' 83 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | **/*.rs.bk # backup files from rustfmt 3 | Cargo.lock 4 | .cargo-ok 5 | 6 | *.patch 7 | *.diff 8 | 9 | .vscode 10 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.4.0 4 | hooks: 5 | - id: check-added-large-files 6 | - id: check-case-conflict 7 | - id: check-executables-have-shebangs 8 | - id: check-json 9 | - id: check-merge-conflict 10 | - id: check-symlinks 11 | - id: check-toml 12 | - id: check-vcs-permalinks 13 | - id: check-xml 14 | - id: check-yaml 15 | - id: destroyed-symlinks 16 | - id: end-of-file-fixer 17 | - id: fix-byte-order-marker 18 | - id: trailing-whitespace 19 | - repo: https://github.com/psf/black 20 | rev: 23.9.1 21 | hooks: 22 | - id: black 23 | - repo: https://github.com/doublify/pre-commit-rust 24 | rev: v1.0 25 | hooks: 26 | - id: cargo-check 27 | - id: clippy 28 | - id: fmt 29 | - repo: https://github.com/streetsidesoftware/cspell-cli 30 | rev: v7.3.0 31 | hooks: 32 | - id: cspell 33 | - repo: https://github.com/redwarp/optimize-png-hooks 34 | rev: v1.2.2 35 | hooks: 36 | - id: optimize-png 37 | - repo: https://github.com/executablebooks/mdformat 38 | rev: 0.7.17 39 | hooks: 40 | - id: mdformat 41 | additional_dependencies: 42 | - mdformat-beautysh 43 | - mdformat-black 44 | - mdformat-config 45 | - mdformat-gfm 46 | - mdformat-rustfmt 47 | - mdformat-tables 48 | - mdformat-web 49 | - repo: https://github.com/sirosen/check-jsonschema 50 | rev: 0.26.3 51 | hooks: 52 | - id: check-github-workflows 53 | - id: check-github-actions 54 | - repo: https://github.com/fsfe/reuse-tool 55 | rev: v2.1.0 56 | hooks: 57 | - id: reuse 58 | 59 | ci: 60 | skip: 61 | - cargo-check 62 | - clippy 63 | - fmt 64 | - check-github-actions 65 | - check-github-workflows 66 | -------------------------------------------------------------------------------- /.reuse/dep5: -------------------------------------------------------------------------------- 1 | Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: downloader 3 | Upstream-Contact: Tobias Hunger 4 | Source: https://github.com/hunger/downloader 5 | 6 | Files: *.md 7 | Copyright: 2020 Tobias Hunger 8 | License: LGPL-3.0-or-later 9 | 10 | Files: .github/* .gitignore 11 | Source: https://github.com/rust-github/template 12 | Copyright: Github User Id @MarcoIeni and others 13 | License: MIT OR Apache-2.0 14 | 15 | Files: .pre-commit-config.yaml cspell.json 16 | Copyright: 2020 Tobias Hunger 17 | License: LGPL-3.0-or-later 18 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | This project adheres to the Rust Code of Conduct, which [can be found online](https://www.rust-lang.org/conduct.html). 4 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contribution guidelines 2 | 3 | First off, thank you for considering contributing to downloader. 4 | 5 | If your contribution is not straightforward, please first discuss the change you 6 | wish to make by creating a new issue before making the change. 7 | 8 | ## Reporting issues 9 | 10 | Before reporting an issue on the 11 | [issue tracker](https://github.com/MyUsername/downloader/issues), 12 | please check that it has not already been reported by searching for some related 13 | keywords. 14 | 15 | ## Pull requests 16 | 17 | Try to do one pull request per change. 18 | 19 | ## Developing 20 | 21 | ### Set up 22 | 23 | This is no different than other Rust projects. 24 | 25 | ```shell 26 | git clone https://github.com/MyUsername/downloader 27 | cd downloader 28 | cargo build 29 | ``` 30 | 31 | ### Useful Commands 32 | 33 | - Build and run release version: 34 | 35 | ```shell 36 | cargo build --release && cargo run --release 37 | ``` 38 | 39 | - Run Clippy: 40 | 41 | ```shell 42 | cargo clippy --all 43 | ``` 44 | 45 | - Run all tests: 46 | 47 | ```shell 48 | cargo test --all 49 | ``` 50 | 51 | - Check to see if there are code formatting issues 52 | 53 | ```shell 54 | cargo fmt --all -- --check 55 | ``` 56 | 57 | - Format the code in the project 58 | 59 | ```shell 60 | cargo fmt --all 61 | ``` 62 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: LGPL-3.0-or-later 2 | # Copyright (C) 2021 Tobias Hunger 3 | 4 | [package] 5 | name = "downloader" 6 | version = "0.2.7" 7 | authors = ["Tobias Hunger "] 8 | edition = "2018" 9 | description = "A simple way to download things via HTTP/HTTPS" 10 | repository = "https://github.com/hunger/downloader" 11 | license = "LGPL-3.0-or-later" 12 | keywords = [ "http", "https", "download" ] 13 | categories = [ "web-programming::http-client" ] 14 | 15 | [features] 16 | default = [ "default-tls" ] 17 | 18 | tui = [ "indicatif" ] 19 | verify = [ "digest" ] 20 | 21 | # Pass down features to reqwest: 22 | default-tls = ["reqwest/default-tls"] 23 | rustls-tls = ["reqwest/rustls-tls"] 24 | 25 | [dependencies] 26 | futures = { version = "0.3" } 27 | reqwest = { version = "0.12", default-features = false } 28 | rand = { version = "0.8" } 29 | thiserror = { version = "1.0" } 30 | tokio = { version = "1.23", features = [ "rt-multi-thread", "time" ] } 31 | 32 | digest = { version = "0.10.1", optional = true } 33 | indicatif = { version = "0.17.2", optional = true } 34 | 35 | [dev-dependencies] 36 | sha3 = "0.10.0" # used in examples 37 | -------------------------------------------------------------------------------- /LICENSES/Apache-2.0.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. 10 | 11 | "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. 12 | 13 | "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. 14 | 15 | "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. 16 | 17 | "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. 18 | 19 | "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. 20 | 21 | "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). 22 | 23 | "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. 24 | 25 | "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." 26 | 27 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 28 | 29 | 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 30 | 31 | 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 32 | 33 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: 34 | 35 | (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and 36 | 37 | (b) You must cause any modified files to carry prominent notices stating that You changed the files; and 38 | 39 | (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 40 | 41 | (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. 42 | 43 | You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 44 | 45 | 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 46 | 47 | 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 48 | 49 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 50 | 51 | 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 52 | 53 | 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. 54 | 55 | END OF TERMS AND CONDITIONS 56 | 57 | APPENDIX: How to apply the Apache License to your work. 58 | 59 | To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. 60 | 61 | Copyright [yyyy] [name of copyright owner] 62 | 63 | Licensed under the Apache License, Version 2.0 (the "License"); 64 | you may not use this file except in compliance with the License. 65 | You may obtain a copy of the License at 66 | 67 | http://www.apache.org/licenses/LICENSE-2.0 68 | 69 | Unless required by applicable law or agreed to in writing, software 70 | distributed under the License is distributed on an "AS IS" BASIS, 71 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 72 | See the License for the specific language governing permissions and 73 | limitations under the License. 74 | -------------------------------------------------------------------------------- /LICENSES/LGPL-3.0-or-later.txt: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | 6 | Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. 7 | 8 | This version of the GNU Lesser General Public License incorporates the terms and conditions of version 3 of the GNU General Public License, supplemented by the additional permissions listed below. 9 | 10 | 0. Additional Definitions. 11 | 12 | As used herein, "this License" refers to version 3 of the GNU Lesser General Public License, and the "GNU GPL" refers to version 3 of the GNU General Public License. 13 | 14 | "The Library" refers to a covered work governed by this License, other than an Application or a Combined Work as defined below. 15 | 16 | An "Application" is any work that makes use of an interface provided by the Library, but which is not otherwise based on the Library. Defining a subclass of a class defined by the Library is deemed a mode of using an interface provided by the Library. 17 | 18 | A "Combined Work" is a work produced by combining or linking an Application with the Library. The particular version of the Library with which the Combined Work was made is also called the "Linked Version". 19 | 20 | The "Minimal Corresponding Source" for a Combined Work means the Corresponding Source for the Combined Work, excluding any source code for portions of the Combined Work that, considered in isolation, are based on the Application, and not on the Linked Version. 21 | 22 | The "Corresponding Application Code" for a Combined Work means the object code and/or source code for the Application, including any data and utility programs needed for reproducing the Combined Work from the Application, but excluding the System Libraries of the Combined Work. 23 | 24 | 1. Exception to Section 3 of the GNU GPL. 25 | You may convey a covered work under sections 3 and 4 of this License without being bound by section 3 of the GNU GPL. 26 | 27 | 2. Conveying Modified Versions. 28 | If you modify a copy of the Library, and, in your modifications, a facility refers to a function or data to be supplied by an Application that uses the facility (other than as an argument passed when the facility is invoked), then you may convey a copy of the modified version: 29 | 30 | a) under this License, provided that you make a good faith effort to ensure that, in the event an Application does not supply the function or data, the facility still operates, and performs whatever part of its purpose remains meaningful, or 31 | 32 | b) under the GNU GPL, with none of the additional permissions of this License applicable to that copy. 33 | 34 | 3. Object Code Incorporating Material from Library Header Files. 35 | The object code form of an Application may incorporate material from a header file that is part of the Library. You may convey such object code under terms of your choice, provided that, if the incorporated material is not limited to numerical parameters, data structure layouts and accessors, or small macros, inline functions and templates (ten or fewer lines in length), you do both of the following: 36 | 37 | a) Give prominent notice with each copy of the object code that the Library is used in it and that the Library and its use are covered by this License. 38 | 39 | b) Accompany the object code with a copy of the GNU GPL and this license document. 40 | 41 | 4. Combined Works. 42 | You may convey a Combined Work under terms of your choice that, taken together, effectively do not restrict modification of the portions of the Library contained in the Combined Work and reverse engineering for debugging such modifications, if you also do each of the following: 43 | 44 | a) Give prominent notice with each copy of the Combined Work that the Library is used in it and that the Library and its use are covered by this License. 45 | 46 | b) Accompany the Combined Work with a copy of the GNU GPL and this license document. 47 | 48 | c) For a Combined Work that displays copyright notices during execution, include the copyright notice for the Library among these notices, as well as a reference directing the user to the copies of the GNU GPL and this license document. 49 | 50 | d) Do one of the following: 51 | 52 | 0) Convey the Minimal Corresponding Source under the terms of this License, and the Corresponding Application Code in a form suitable for, and under terms that permit, the user to recombine or relink the Application with a modified version of the Linked Version to produce a modified Combined Work, in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source. 53 | 54 | 1) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (a) uses at run time a copy of the Library already present on the user's computer system, and (b) will operate properly with a modified version of the Library that is interface-compatible with the Linked Version. 55 | 56 | e) Provide Installation Information, but only if you would otherwise be required to provide such information under section 6 of the GNU GPL, and only to the extent that such information is necessary to install and execute a modified version of the Combined Work produced by recombining or relinking the Application with a modified version of the Linked Version. (If you use option 4d0, the Installation Information must accompany the Minimal Corresponding Source and Corresponding Application Code. If you use option 4d1, you must provide the Installation Information in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source.) 57 | 58 | 5. Combined Libraries. 59 | You may place library facilities that are a work based on the Library side by side in a single library together with other library facilities that are not Applications and are not covered by this License, and convey such a combined library under terms of your choice, if you do both of the following: 60 | 61 | a) Accompany the combined library with a copy of the same work based on the Library, uncombined with any other library facilities, conveyed under the terms of this License. 62 | 63 | b) Give prominent notice with the combined library that part of it is a work based on the Library, and explaining where to find the accompanying uncombined form of the same work. 64 | 65 | 6. Revised Versions of the GNU Lesser General Public License. 66 | The Free Software Foundation may publish revised and/or new versions of the GNU Lesser General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. 67 | 68 | Each version is given a distinguishing version number. If the Library as you received it specifies that a certain numbered version of the GNU Lesser General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that published version or of any later version published by the Free Software Foundation. If the Library as you received it does not specify a version number of the GNU Lesser General Public License, you may choose any version of the GNU Lesser General Public License ever published by the Free Software Foundation. 69 | 70 | If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall 71 | apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library. 72 | -------------------------------------------------------------------------------- /LICENSES/MIT.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # downloader 2 | 3 | [![Crates.io](https://img.shields.io/crates/v/downloader.svg)](https://crates.io/crates/downloader) 4 | [![Docs.rs](https://docs.rs/downloader/badge.svg)](https://docs.rs/downloader) 5 | [![CI](https://github.com/hunger/downloader/workflows/Continuous%20Integration/badge.svg)](https://github.com/hunger/downloader/actions) 6 | [![Coverage Status](https://coveralls.io/repos/github/hunger/downloader/badge.svg?branch=main)](https://coveralls.io/github/hunger/downloader?branch=main) 7 | 8 | `downloader` is a crate to help with easily downloading of files from the 9 | internet. It takes a simple simple and straightforward approach using a url 10 | builder and fetcher. 11 | 12 | It supports system proxy configuration, parallel downloads of different files, 13 | validation of downloads via a callback, as well as files mirroring across different 14 | machines. 15 | 16 | Callbacks to provide progress information are supported as well. 17 | 18 | ## Usage 19 | 20 | ### Installation via Cargo 21 | 22 | Add the following line into your `Cargo.toml` file to make `downloader` a 23 | `[dependency]` of your crate: 24 | 25 | `downloader = ""` 26 | 27 | Alternatively you can run `cargo add downloader`. See crates.io for the latest 28 | version of the package. 29 | 30 | ### Example 31 | 32 | ```rust 33 | use downloader::downloader::Builder; 34 | use downloader::Download; 35 | use std::path::Path; 36 | use std::time::Duration; 37 | 38 | fn main() { 39 | let image = Download::new("https://example.com/example.png"); 40 | // other downloads... 41 | // image.urls.push("https://example.com/example2.png"); 42 | 43 | let mut dl = Builder::default() 44 | .connect_timeout(Duration::from_secs(4)) 45 | .download_folder(Path::new("../res")) // or any arbitrary path 46 | .parallel_requests(8) 47 | .build() 48 | .unwrap(); 49 | 50 | let response = dl.download(&[image]).unwrap(); // other error handling 51 | 52 | response.iter().for_each(|v| match v { 53 | Ok(v) => println!("Downloaded: {:?}", v), 54 | Err(e) => println!("Error: {:?}", e), 55 | }) 56 | } 57 | ``` 58 | 59 | ### Features 60 | 61 | - `tui` feature uses `indicatif` crate to provide a text ui for downloads 62 | - `verify` feature enables (optional) verification of downloads using sha3 hashes 63 | 64 | ## License 65 | 66 | Licensed under the GNU Lesser General Public License, Version 3.0 or later 67 | ([LICENSE-LGPLv3](LICENSE-LGPLv3.md) or ) 68 | 69 | ## Contribution 70 | 71 | Unless you explicitly state otherwise, any contribution intentionally submitted 72 | for inclusion in the work by you, shall be licensed as LGPLv3 or later, without 73 | any additional terms or conditions. 74 | 75 | See [CONTRIBUTING.md](CONTRIBUTING.md). 76 | -------------------------------------------------------------------------------- /cspell.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2", 3 | "language": "en", 4 | "languageSettings": [ 5 | { 6 | "languageId": "rust", 7 | "words": [ 8 | "reqwest", 9 | "thiserror" 10 | ] 11 | } 12 | ], 13 | "words": [ 14 | "clippy", 15 | "indicatif", 16 | "rustfmt" 17 | ], 18 | "ignorePaths": [ 19 | "cspell.json", 20 | ".pre-commit-config.yaml", 21 | ".github/workflows/*.yaml", 22 | "Cargo.toml", 23 | ".reuse" 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /examples/download.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: LGPL-3.0-or-later 2 | // Copyright (C) 2020 Tobias Hunger 3 | 4 | // Setup warnings/errors: 5 | #![forbid(unsafe_code)] 6 | #![deny(bare_trait_objects, unused_doc_comments, unused_import_braces)] 7 | // Clippy: 8 | #![warn(clippy::all, clippy::nursery, clippy::pedantic)] 9 | #![allow(clippy::non_ascii_literal)] 10 | 11 | use std::env::temp_dir; 12 | 13 | use downloader::Downloader; 14 | 15 | // Define a custom progress reporter: 16 | struct SimpleReporterPrivate { 17 | last_update: std::time::Instant, 18 | max_progress: Option, 19 | message: String, 20 | } 21 | struct SimpleReporter { 22 | private: std::sync::Mutex>, 23 | } 24 | 25 | impl SimpleReporter { 26 | #[cfg(not(feature = "tui"))] 27 | fn create() -> std::sync::Arc { 28 | std::sync::Arc::new(Self { 29 | private: std::sync::Mutex::new(None), 30 | }) 31 | } 32 | } 33 | 34 | impl downloader::progress::Reporter for SimpleReporter { 35 | fn setup(&self, max_progress: Option, message: &str) { 36 | let private = SimpleReporterPrivate { 37 | last_update: std::time::Instant::now(), 38 | max_progress, 39 | message: message.to_owned(), 40 | }; 41 | 42 | let mut guard = self.private.lock().unwrap(); 43 | *guard = Some(private); 44 | } 45 | 46 | fn progress(&self, current: u64) { 47 | if let Some(p) = self.private.lock().unwrap().as_mut() { 48 | let max_bytes = p 49 | .max_progress 50 | .map_or_else(|| "{unknown}".to_owned(), |bytes| format!("{bytes:?}")); 51 | if p.last_update.elapsed().as_millis() >= 1000 { 52 | println!( 53 | "test file: {} of {} bytes. [{}]", 54 | current, max_bytes, p.message 55 | ); 56 | p.last_update = std::time::Instant::now(); 57 | } 58 | } 59 | } 60 | 61 | fn set_message(&self, message: &str) { 62 | println!("test file: Message changed to: {message}"); 63 | } 64 | 65 | fn done(&self) { 66 | _ = self.private.lock().unwrap().take(); 67 | println!("test file: [DONE]"); 68 | } 69 | } 70 | 71 | fn main() { 72 | let mut downloader = Downloader::builder() 73 | .download_folder(&temp_dir()) 74 | .parallel_requests(1) 75 | .build() 76 | .unwrap(); 77 | 78 | let dl = downloader::Download::new("https://ash-speed.hetzner.com/100MB.bin"); 79 | 80 | #[cfg(not(feature = "tui"))] 81 | let dl = dl.progress(SimpleReporter::create()); 82 | 83 | #[cfg(feature = "verify")] 84 | let dl = { 85 | use downloader::verify; 86 | fn decode_hex(s: &str) -> Result, std::num::ParseIntError> { 87 | (0..s.len()) 88 | .step_by(2) 89 | .map(|i| u8::from_str_radix(&s[i..i + 2], 16)) 90 | .collect() 91 | } 92 | dl.verify(verify::with_digest::( 93 | decode_hex("2197e485d463ac2b868e87f0d4547b4223ff5220a0694af2593cbe7c796f7fd6").unwrap(), 94 | )) 95 | }; 96 | 97 | let result = downloader.download(&[dl]).unwrap(); 98 | 99 | for r in result { 100 | match r { 101 | Err(e) => println!("Error: {e}"), 102 | Ok(s) => println!("Success: {}", &s), 103 | }; 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /examples/tui_basic.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: LGPL-3.0-or-later 2 | // Copyright (C) 2020 Tobias Hunger 3 | // Copyright (C) 2021 Phoenix IR 4 | 5 | // Setup warnings/errors: 6 | #![forbid(unsafe_code)] 7 | #![deny(bare_trait_objects, unused_doc_comments, unused_import_braces)] 8 | // Clippy: 9 | #![warn(clippy::all, clippy::nursery, clippy::pedantic)] 10 | #![allow(clippy::non_ascii_literal)] 11 | 12 | use downloader::Downloader; 13 | use std::env::temp_dir; 14 | 15 | // Run example with: cargo run --example tui_basic --features tui 16 | fn main() { 17 | let mut downloader = Downloader::builder() 18 | .download_folder(&temp_dir()) 19 | .parallel_requests(1) 20 | .build() 21 | .unwrap(); 22 | 23 | // Download with an explicit filename 24 | let dl = downloader::Download::new("https://example.org/") 25 | .file_name(std::path::Path::new("example.html")); 26 | 27 | // Download with an inferred filename 28 | let dl2 = downloader::Download::new( 29 | "https://cdimage.debian.org/debian-cd/12.8.0/i386/iso-cd/debian-12.8.0-i386-netinst.iso", 30 | ); 31 | 32 | let result = downloader.download(&[dl, dl2]).unwrap(); 33 | 34 | for r in result { 35 | match r { 36 | Err(e) => print!("Error occurred! {e}"), 37 | Ok(s) => print!("Success: {}", &s), 38 | }; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/backend.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: LGPL-3.0-or-later 2 | // Copyright (C) 2020 Tobias Hunger 3 | 4 | //! The actual download code 5 | 6 | use crate::{Download, DownloadSummary, Error, Result, Verification}; 7 | 8 | use futures::stream::{self, StreamExt}; 9 | use rand::seq::SliceRandom; 10 | 11 | use std::io::{Seek, SeekFrom, Write}; 12 | 13 | fn select_url(urls: &[String]) -> String { 14 | assert!(!urls.is_empty()); 15 | urls.choose(&mut rand::thread_rng()).unwrap().clone() 16 | } 17 | 18 | async fn download_url( 19 | client: reqwest::Client, 20 | url: String, 21 | writer: &mut std::io::BufWriter, 22 | progress: &mut crate::Progress, 23 | message: &str, 24 | ) -> u16 { 25 | if let Ok(mut response) = client.get(&url).send().await { 26 | let total = response.content_length(); 27 | let mut current: u64 = 0; 28 | writer.seek(SeekFrom::Start(current)).unwrap_or(0); 29 | 30 | progress.setup(total, message); 31 | 32 | while let Some(bytes) = response.chunk().await.unwrap_or(None) { 33 | if writer.write_all(&bytes).is_err() {} 34 | 35 | current += bytes.len() as u64; 36 | progress.progress(current); 37 | } 38 | 39 | let result = response.status().as_u16(); 40 | progress.set_message(&format!("{message} - {result}")); 41 | result 42 | } else { 43 | reqwest::StatusCode::BAD_REQUEST.as_u16() 44 | } 45 | } 46 | 47 | async fn verify_download( 48 | path: std::path::PathBuf, 49 | verify_callback: crate::Verify, 50 | progress: crate::Progress, 51 | message: &str, 52 | ) -> Verification { 53 | let p = progress.clone(); 54 | let result = 55 | tokio::task::spawn_blocking(move || verify_callback(path, &move |c: u64| p.progress(c))) 56 | .await 57 | .unwrap_or(crate::Verification::NotVerified); 58 | progress.set_message(&format!( 59 | "{} - {}", 60 | message, 61 | match result { 62 | Verification::NotVerified => "not verified", 63 | Verification::Failed => "FAILED", 64 | Verification::Ok => "Ok", 65 | } 66 | )); 67 | progress.done(); 68 | result 69 | } 70 | 71 | async fn download( 72 | client: reqwest::Client, 73 | mut download: Download, 74 | retries: u16, 75 | ) -> Result { 76 | let mut summary = DownloadSummary { 77 | status: Vec::new(), 78 | file_name: std::mem::take(&mut download.file_name), 79 | verified: Verification::NotVerified, 80 | }; 81 | 82 | let mut urls = std::mem::take(&mut download.urls); 83 | assert!(!urls.is_empty()); 84 | 85 | let mut progress = download.progress.expect("This has been set!").clone(); 86 | let mut message = String::new(); 87 | 88 | let mut download_successful = false; 89 | 90 | if let Ok(file) = std::fs::OpenOptions::new() 91 | .create_new(true) 92 | .write(true) 93 | .open(&summary.file_name) 94 | { 95 | let mut writer = std::io::BufWriter::new(file); 96 | 97 | for retry in 1..=retries { 98 | let url = select_url(&urls); 99 | 100 | message = format!( 101 | "{} {}/{}", 102 | &summary 103 | .file_name 104 | .file_name() 105 | .unwrap_or_else(|| std::ffi::OsStr::new("")) 106 | .to_string_lossy(), 107 | retry, 108 | retries, 109 | ); 110 | 111 | let s = reqwest::StatusCode::from_u16( 112 | download_url( 113 | client.clone(), 114 | url.clone(), 115 | &mut writer, 116 | &mut progress, 117 | &message, 118 | ) 119 | .await, 120 | ) 121 | .unwrap_or(reqwest::StatusCode::BAD_REQUEST); 122 | 123 | summary.status.push((url.clone(), s.as_u16())); 124 | 125 | if s.is_server_error() { 126 | urls = urls 127 | .iter() 128 | .filter_map(|u| if u == &url { Some(u.clone()) } else { None }) 129 | .collect(); 130 | if urls.is_empty() { 131 | break; 132 | } 133 | } 134 | 135 | if s.is_success() { 136 | download_successful = true; 137 | break; 138 | } 139 | } 140 | } 141 | 142 | if !download_successful { 143 | return Err(Error::Download(summary)); 144 | } 145 | 146 | summary.verified = verify_download( 147 | summary.file_name.clone(), 148 | std::mem::replace(&mut download.verify_callback, crate::verify::noop()), 149 | progress.clone(), 150 | &message, 151 | ) 152 | .await; 153 | if summary.verified == Verification::Failed { 154 | return Err(Error::Verification(summary)); 155 | } 156 | 157 | Ok(summary) 158 | } 159 | 160 | /// Run the provided list of `downloads`, using the provided `client` 161 | pub(crate) fn run( 162 | client: &mut reqwest::Client, 163 | downloads: Vec, 164 | retries: u16, 165 | parallel_requests: u16, 166 | ) -> Vec> { 167 | let rt = tokio::runtime::Runtime::new().unwrap(); 168 | let cl = client.clone(); 169 | 170 | let result = rt.spawn(async move { 171 | stream::iter(downloads) 172 | .map(move |d| download(cl.clone(), d, retries)) 173 | .buffer_unordered(parallel_requests as usize) 174 | .collect::>>() 175 | .await 176 | }); 177 | 178 | rt.block_on(result).unwrap() 179 | } 180 | 181 | pub(crate) async fn async_run( 182 | client: &mut reqwest::Client, 183 | downloads: Vec, 184 | retries: u16, 185 | parallel_requests: u16, 186 | ) -> Vec> { 187 | let cl = client.clone(); 188 | 189 | let result = tokio::spawn(async move { 190 | stream::iter(downloads) 191 | .map(move |d| download(cl.clone(), d, retries)) 192 | .buffer_unordered(parallel_requests as usize) 193 | .collect::>>() 194 | .await 195 | }) 196 | .await; 197 | 198 | result.unwrap() 199 | } 200 | -------------------------------------------------------------------------------- /src/download.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: LGPL-3.0-or-later 2 | // Copyright (C) 2020 Tobias Hunger 3 | 4 | //! The `Download` struct is used to describe a file that is 5 | //! supposed to get downloaded. 6 | 7 | // ---------------------------------------------------------------------- 8 | // - Download: 9 | // ---------------------------------------------------------------------- 10 | 11 | /// A `Download`. 12 | pub struct Download { 13 | /// A list of URLs that this file can be retrieved from. `downloader` will pick 14 | /// the download URL from this list at random. 15 | pub urls: Vec, 16 | /// A progress `Reporter` to report the download process with. 17 | pub progress: Option, 18 | /// The file name to be used for the downloaded file. 19 | pub file_name: std::path::PathBuf, 20 | /// A callback used to verify the download with. 21 | pub verify_callback: crate::Verify, 22 | } 23 | 24 | fn file_name_from_url(url: &str) -> std::path::PathBuf { 25 | if url.is_empty() { 26 | return std::path::PathBuf::new(); 27 | } 28 | let Ok(url) = reqwest::Url::parse(url) else { 29 | return std::path::PathBuf::new(); 30 | }; 31 | 32 | url.path_segments() 33 | .map_or_else(std::path::PathBuf::new, |f| { 34 | std::path::PathBuf::from(f.last().unwrap_or("")) 35 | }) 36 | } 37 | 38 | impl Download { 39 | /// Create a new `Download` with a single download `url` 40 | #[must_use] 41 | pub fn new(url: &str) -> Self { 42 | Self { 43 | urls: vec![url.to_owned()], 44 | progress: None, 45 | file_name: file_name_from_url(url), 46 | verify_callback: crate::verify::noop(), 47 | } 48 | } 49 | 50 | /// Create a new `Download` based on a list of mirror urls. 51 | #[must_use] 52 | pub fn new_mirrored(urls: &[&str]) -> Self { 53 | let urls: Vec = urls.iter().map(|s| String::from(*s)).collect(); 54 | let url = urls.first().unwrap_or(&String::new()).clone(); 55 | 56 | Self { 57 | urls, 58 | progress: None, 59 | file_name: file_name_from_url(&url), 60 | verify_callback: crate::verify::noop(), 61 | } 62 | } 63 | 64 | /// Set the name of the downloaded file. This filename can be absolute or 65 | /// relative to the `download_folder` defined in the `Downloader`. 66 | /// 67 | /// Default is the file name on the server side (if available) 68 | #[must_use] 69 | pub fn file_name(mut self, path: &std::path::Path) -> Self { 70 | self.file_name = path.to_path_buf(); 71 | self 72 | } 73 | 74 | /// Register handling of progress information 75 | /// 76 | /// Defaults to not printing any progress information. 77 | #[must_use] 78 | pub fn progress(mut self, progress: crate::Progress) -> Self { 79 | self.progress = Some(progress); 80 | self 81 | } 82 | 83 | /// Register a callback to verify a download 84 | /// 85 | /// Default is to assume the file was downloaded correctly. 86 | #[must_use] 87 | pub fn verify(mut self, func: crate::Verify) -> Self { 88 | self.verify_callback = func; 89 | self 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/downloader.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: LGPL-3.0-or-later 2 | // Copyright (C) 2020 Tobias Hunger 3 | 4 | //! The `Downloader` that holds all the logic to manage the `Downloads` 5 | 6 | use crate::{Download, DownloadSummary, Error, Result}; 7 | 8 | use crate::progress::Factory; 9 | 10 | // ---------------------------------------------------------------------- 11 | // - Helper: 12 | // ---------------------------------------------------------------------- 13 | 14 | fn validate_downloads( 15 | downloads: &[Download], 16 | download_folder: &std::path::Path, 17 | factory: &dyn Factory, 18 | ) -> Result> { 19 | let mut known_urls = std::collections::HashSet::new(); 20 | let mut known_download_paths = std::collections::HashSet::new(); 21 | 22 | let mut result = Vec::with_capacity(downloads.len()); 23 | 24 | for d in downloads { 25 | if d.urls.is_empty() { 26 | return Err(Error::DownloadDefinition(String::from( 27 | "No URL found to download.", 28 | ))); 29 | } 30 | 31 | for u in &d.urls { 32 | if !known_urls.insert(u) { 33 | return Err(Error::DownloadDefinition(format!( 34 | "Download URL \"{u}\" is used more than once.", 35 | ))); 36 | } 37 | } 38 | 39 | let urls = d.urls.clone(); 40 | 41 | if d.file_name.to_string_lossy().is_empty() { 42 | return Err(Error::DownloadDefinition(String::from( 43 | "No download file name was provided.", 44 | ))); 45 | } 46 | 47 | let file_name = download_folder.join(&d.file_name); 48 | if d.file_name.to_string_lossy().is_empty() { 49 | return Err(Error::DownloadDefinition(String::from( 50 | "Failed to get full download path.", 51 | ))); 52 | } 53 | 54 | if !known_download_paths.insert(&d.file_name) { 55 | return Err(Error::DownloadDefinition(format!( 56 | "Download file name \"{}\" is used more than once.", 57 | d.file_name.to_string_lossy(), 58 | ))); 59 | } 60 | 61 | let progress = if d.progress.is_none() { 62 | factory.create_reporter() 63 | } else { 64 | d.progress.as_ref().expect("Was Some just now...").clone() 65 | }; 66 | 67 | result.push(Download { 68 | urls, 69 | file_name, 70 | progress: Some(progress), 71 | verify_callback: d.verify_callback.clone(), 72 | }); 73 | } 74 | 75 | Ok(result) 76 | } 77 | 78 | // ---------------------------------------------------------------------- 79 | // - Downloader: 80 | // ---------------------------------------------------------------------- 81 | 82 | /// This is the main entry point: You need to have a `Downloader` and then can call 83 | /// `download` on that, passing in a list of `Download` objects. 84 | pub struct Downloader { 85 | client: reqwest::Client, 86 | parallel_requests: u16, 87 | retries: u16, 88 | download_folder: std::path::PathBuf, 89 | } 90 | 91 | impl Downloader { 92 | /// Create a `Builder` for `Downloader` to allow for fine-grained configuration. 93 | #[must_use] 94 | pub fn builder() -> Builder { 95 | Builder::default() 96 | } 97 | 98 | /// Start the download 99 | /// 100 | /// # Errors 101 | /// `Error::DownloadDefinition` if the download is detected to be broken in some way. 102 | pub fn download(&mut self, downloads: &[Download]) -> Result>> { 103 | #[cfg(feature = "tui")] 104 | let factory = crate::progress::Tui::default(); 105 | #[cfg(not(feature = "tui"))] 106 | let factory = crate::progress::Noop::default(); 107 | 108 | let to_process = validate_downloads(downloads, &self.download_folder, &factory)?; 109 | if to_process.is_empty() { 110 | return Ok(Vec::new()); 111 | } 112 | 113 | Ok(crate::backend::run( 114 | &mut self.client, 115 | to_process, 116 | self.retries, 117 | self.parallel_requests, 118 | )) 119 | } 120 | 121 | /// Start the download asyncroniously 122 | /// 123 | /// # Errors 124 | /// `Error::DownloadDefinition` if the download is detected to be broken in some way. 125 | pub async fn async_download( 126 | &mut self, 127 | downloads: &[Download], 128 | ) -> Result>> { 129 | #[cfg(feature = "tui")] 130 | let factory = crate::progress::Tui::default(); 131 | #[cfg(not(feature = "tui"))] 132 | let factory = crate::progress::Noop::default(); 133 | 134 | let to_process = validate_downloads(downloads, &self.download_folder, &factory)?; 135 | if to_process.is_empty() { 136 | return Ok(Vec::new()); 137 | } 138 | 139 | let result = crate::backend::async_run( 140 | &mut self.client, 141 | to_process, 142 | self.retries, 143 | self.parallel_requests, 144 | ) 145 | .await; 146 | 147 | Ok(result) 148 | } 149 | } 150 | 151 | // ---------------------------------------------------------------------- 152 | // - Builder: 153 | // ---------------------------------------------------------------------- 154 | 155 | /// A builder for a `Downloader` 156 | pub struct Builder { 157 | user_agent: String, 158 | connect_timeout: std::time::Duration, 159 | timeout: std::time::Duration, 160 | parallel_requests: u16, 161 | retries: u16, 162 | download_folder: std::path::PathBuf, 163 | } 164 | 165 | impl Builder { 166 | /// Set the user agent to be used. 167 | /// 168 | /// A default value will be used if none is set. 169 | pub fn user_agent(&mut self, user_agent: &str) -> &mut Self { 170 | self.user_agent = user_agent.into(); 171 | self 172 | } 173 | 174 | /// Set the connection timeout. 175 | /// 176 | /// The default is 30s. 177 | pub fn connect_timeout(&mut self, timeout: std::time::Duration) -> &mut Self { 178 | self.connect_timeout = timeout; 179 | self 180 | } 181 | 182 | /// Set the timeout. 183 | /// 184 | /// The default is 5min. 185 | pub fn timeout(&mut self, timeout: std::time::Duration) -> &mut Self { 186 | self.timeout = timeout; 187 | self 188 | } 189 | 190 | /// Set the number of parallel requests. 191 | /// 192 | /// The default is 32. 193 | pub fn parallel_requests(&mut self, count: u16) -> &mut Self { 194 | self.parallel_requests = count; 195 | self 196 | } 197 | 198 | /// Set the number of retries. 199 | /// 200 | /// The default is 3. 201 | pub fn retries(&mut self, count: u16) -> &mut Self { 202 | self.retries = count; 203 | self 204 | } 205 | 206 | /// Set the folder to download into. 207 | /// 208 | /// The default is unset and a value is required. 209 | pub fn download_folder(&mut self, folder: &std::path::Path) -> &mut Self { 210 | self.download_folder = folder.to_path_buf(); 211 | self 212 | } 213 | 214 | /// Construct a new `reqwest::Client` configured with settings from the `Builder` 215 | /// 216 | /// # Errors 217 | /// * `Error::Setup`, when setup fails 218 | fn build_client(&self) -> crate::Result { 219 | reqwest::Client::builder() 220 | .user_agent(self.user_agent.clone()) 221 | .connect_timeout(self.connect_timeout) 222 | .timeout(self.timeout) 223 | .build() 224 | .map_err(|e| Error::Setup(format!("Failed to set up backend: {e}"))) 225 | } 226 | 227 | /// Build a downloader with a specified `reqwest::Client` 228 | /// 229 | /// # Errors 230 | /// * `Error::Setup`, when setup fails 231 | pub fn build_with_client(&mut self, client: reqwest::Client) -> crate::Result { 232 | let download_folder = &self.download_folder; 233 | if download_folder.to_string_lossy().is_empty() { 234 | return Err(crate::Error::Setup( 235 | "Required \"download_folder\" was not set.".into(), 236 | )); 237 | } 238 | if !download_folder.is_dir() { 239 | return Err(Error::Setup(format!( 240 | "Required \"download_folder\" with value \"{}\" is not a folder.", 241 | download_folder.to_string_lossy() 242 | ))); 243 | } 244 | 245 | Ok(Downloader { 246 | client, 247 | parallel_requests: self.parallel_requests, 248 | retries: self.retries, 249 | download_folder: download_folder.clone(), 250 | }) 251 | } 252 | 253 | /// Build a downloader. 254 | /// 255 | /// # Errors 256 | /// * `Error::Setup`, when setup fails 257 | pub fn build(&mut self) -> crate::Result { 258 | let client = self.build_client()?; 259 | self.build_with_client(client) 260 | } 261 | } 262 | 263 | impl Default for Builder { 264 | fn default() -> Self { 265 | let download_folder = 266 | std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from("")); 267 | let download_folder = if download_folder.to_string_lossy().is_empty() { 268 | std::path::PathBuf::from( 269 | std::env::var_os("HOME").unwrap_or_else(|| std::ffi::OsString::from("/")), 270 | ) 271 | } else { 272 | download_folder 273 | }; 274 | 275 | Self { 276 | user_agent: format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")), 277 | connect_timeout: std::time::Duration::from_secs(30), 278 | timeout: std::time::Duration::from_secs(300), 279 | parallel_requests: 32, 280 | retries: 3, 281 | download_folder, 282 | } 283 | } 284 | } 285 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: LGPL-3.0-or-later 2 | // Copyright (C) 2020 Tobias Hunger 3 | 4 | //! This crate provides a simple way to download files via HTTP/HTTPS. 5 | //! 6 | //! It tries to make it very simple to just specify a couple of 7 | //! URLs and then go and download all of the files. 8 | //! 9 | //! It supports system proxy configuration, parallel downloads of different files, 10 | //! validation of downloads via a callback, as well as files being mirrored on 11 | //! different machines. 12 | //! 13 | //! Callbacks to provide progress information are supported as well. 14 | 15 | // Setup warnings/errors: 16 | #![forbid(unsafe_code)] 17 | #![deny( 18 | bare_trait_objects, 19 | unused_doc_comments, 20 | unused_import_braces, 21 | missing_docs 22 | )] 23 | // Clippy: 24 | #![warn(clippy::all, clippy::nursery, clippy::pedantic)] 25 | #![allow(clippy::non_ascii_literal)] 26 | 27 | pub mod backend; 28 | pub mod download; 29 | pub mod downloader; 30 | pub mod progress; 31 | pub mod verify; 32 | 33 | pub use crate::download::Download; 34 | pub use crate::downloader::Downloader; 35 | pub use crate::progress::Progress; 36 | pub use crate::verify::{SimpleProgress, Verification, Verify}; 37 | 38 | // ---------------------------------------------------------------------- 39 | // - Error: 40 | // ---------------------------------------------------------------------- 41 | 42 | /// Possible `Error`s that can occurred during normal operation. 43 | #[derive(thiserror::Error, Debug)] 44 | pub enum Error { 45 | /// The Setup is incomplete or bogus. 46 | #[error("Setup error: {0}")] 47 | Setup(String), 48 | /// A Definition of a `Download` is incomplete 49 | #[error("Download definition: {0}")] 50 | DownloadDefinition(String), 51 | /// Writing into a file failed during download. 52 | #[error("File creation failed: {0}")] 53 | File(DownloadSummary), 54 | /// A download failed 55 | #[error("Download failed for {0}")] 56 | Download(DownloadSummary), 57 | /// Download file verification failed. 58 | #[error("Verification failed for {0}")] 59 | Verification(DownloadSummary), 60 | } 61 | 62 | /// `Result` type for the `gng_shared` library 63 | pub type Result = std::result::Result; 64 | 65 | // ---------------------------------------------------------------------- 66 | // - DownloadSummary: 67 | // ---------------------------------------------------------------------- 68 | 69 | /// The result of a `Download` 70 | pub struct DownloadSummary { 71 | /// A list of attempted downloads with URL and status code. 72 | pub status: Vec<(String, u16)>, 73 | /// The path this URL has been downloaded to. 74 | pub file_name: std::path::PathBuf, 75 | /// File verification status 76 | pub verified: Verification, 77 | } 78 | 79 | fn to_fmt(f: &mut std::fmt::Formatter<'_>, summary: &DownloadSummary) -> std::fmt::Result { 80 | writeln!( 81 | f, 82 | "{}: (verification: {}):", 83 | summary.file_name.to_string_lossy(), 84 | match summary.verified { 85 | Verification::NotVerified => "unverified", 86 | Verification::Failed => "FAILED", 87 | Verification::Ok => "Ok", 88 | }, 89 | )?; 90 | for i in 0..summary.status.len() { 91 | writeln!( 92 | f, 93 | " {}: {} with status {}", 94 | i + 1, 95 | summary.status[i].0, 96 | summary.status[i].1 97 | )?; 98 | } 99 | Ok(()) 100 | } 101 | 102 | impl std::fmt::Display for DownloadSummary { 103 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 104 | to_fmt(f, self) 105 | } 106 | } 107 | 108 | impl std::fmt::Debug for DownloadSummary { 109 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 110 | to_fmt(f, self) 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /src/progress.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: LGPL-3.0-or-later 2 | // Copyright (C) 2020 Tobias Hunger 3 | 4 | //! Progress reporting code 5 | 6 | // ---------------------------------------------------------------------- 7 | // - Types: 8 | // ---------------------------------------------------------------------- 9 | 10 | /// A Progress reporter to use for the `Download` 11 | pub type Progress = std::sync::Arc; 12 | 13 | // ---------------------------------------------------------------------- 14 | // - Traits: 15 | // ---------------------------------------------------------------------- 16 | 17 | /// An interface for `ProgressReporter`s 18 | pub trait Reporter: Send + Sync { 19 | /// Setup a TUI element for the next progress 20 | fn setup(&self, max_progress: Option, message: &str); 21 | /// Report progress 22 | fn progress(&self, current: u64); 23 | /// Report progress 24 | fn set_message(&self, message: &str); 25 | /// Finish up after progress reporting is done 26 | fn done(&self); 27 | } 28 | 29 | /// A `Factory` used to create `Reporter` when a Download does 30 | /// not come with a defined way to report progress already. 31 | pub trait Factory { 32 | /// Create an `Reporter` 33 | fn create_reporter(&self) -> crate::Progress; 34 | } 35 | 36 | // ---------------------------------------------------------------------- 37 | // - Noop: 38 | // ---------------------------------------------------------------------- 39 | 40 | /// Do not print anything 41 | #[derive(Default)] 42 | pub struct Noop {} 43 | 44 | impl Reporter for Noop { 45 | fn setup(&self, _: Option, _: &str) {} 46 | 47 | fn progress(&self, _: u64) {} 48 | 49 | fn set_message(&self, _: &str) {} 50 | 51 | fn done(&self) {} 52 | } 53 | 54 | impl Noop { 55 | /// Create a `Noop` `Reporter`. 56 | #[must_use] 57 | pub fn create() -> crate::Progress { 58 | std::sync::Arc::new(Self {}) 59 | } 60 | } 61 | 62 | impl Factory for Noop { 63 | fn create_reporter(&self) -> crate::Progress { 64 | Self::create() 65 | } 66 | } 67 | 68 | // ---------------------------------------------------------------------- 69 | // - TUI: 70 | // ---------------------------------------------------------------------- 71 | 72 | #[cfg(feature = "tui")] 73 | mod tui { 74 | /// Manage multiple progress reporters combined into one set of progress bars 75 | pub struct Tui { 76 | progress_group: indicatif::MultiProgress, 77 | } 78 | 79 | impl Default for Tui { 80 | fn default() -> Self { 81 | Self { 82 | progress_group: indicatif::MultiProgress::with_draw_target( 83 | indicatif::ProgressDrawTarget::stderr_with_hz(4), 84 | ), 85 | } 86 | } 87 | } 88 | 89 | impl super::Factory for Tui { 90 | /// Create a `Reporter` connected to this set of UI primitives. 91 | fn create_reporter(&self) -> crate::Progress { 92 | std::sync::Arc::new(TuiBar { 93 | progress_bar: std::sync::Mutex::new( 94 | self.progress_group.add(indicatif::ProgressBar::new(1)), 95 | ), 96 | }) 97 | } 98 | } 99 | 100 | struct TuiBar { 101 | progress_bar: std::sync::Mutex, 102 | } 103 | 104 | impl super::Reporter for TuiBar { 105 | fn setup(&self, max_progress: Option, message: &str) { 106 | let lock = self.progress_bar.lock().unwrap(); 107 | if let Some(t) = max_progress { 108 | lock.set_length(t); 109 | lock.set_style(indicatif::ProgressStyle::default_bar() 110 | .template("[{bar:20.cyan/blue}] {bytes}/{total_bytes} ({bytes_per_sec}, {eta}) - {msg}") 111 | .unwrap() 112 | .progress_chars("#- ")); 113 | lock.set_message(String::from(message)); 114 | lock.reset_eta(); 115 | } else { 116 | lock.set_style( 117 | indicatif::ProgressStyle::default_spinner() 118 | // For more spinners check out the cli-spinners project: 119 | // https://github.com/sindresorhus/cli-spinners/blob/master/spinners.json 120 | .tick_strings(&[ 121 | "▹▹▹▹▹", 122 | "▸▹▹▹▹", 123 | "▹▸▹▹▹", 124 | "▹▹▸▹▹", 125 | "▹▹▹▸▹", 126 | "▹▹▹▹▸", 127 | "▪▪▪▪▪", 128 | ]) 129 | .template("{spinner:.blue} {msg}") 130 | .unwrap() 131 | ); 132 | lock.set_message(String::from(message)); 133 | }; 134 | } 135 | 136 | fn progress(&self, current: u64) { 137 | let lock = self.progress_bar.lock().unwrap(); 138 | lock.set_position(current); 139 | } 140 | 141 | fn set_message(&self, message: &str) { 142 | let lock = self.progress_bar.lock().unwrap(); 143 | lock.set_message(String::from(message)); 144 | } 145 | 146 | fn done(&self) { 147 | let lock = self.progress_bar.lock().unwrap(); 148 | lock.finish(); 149 | } 150 | } 151 | } 152 | 153 | #[cfg(feature = "tui")] 154 | pub use tui::Tui; 155 | -------------------------------------------------------------------------------- /src/verify.rs: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: LGPL-3.0-or-later 2 | // Copyright (C) 2020 Tobias Hunger 3 | 4 | //! Verification callbacks 5 | 6 | // cSpell: ignore hasher 7 | 8 | // ---------------------------------------------------------------------- 9 | // - Types: 10 | // ---------------------------------------------------------------------- 11 | 12 | /// A simplified progress callback passed to `Verify`. It only takes a progress 13 | /// value, which is relative to the file length in bytes. 14 | pub type SimpleProgress = dyn Fn(u64) + Sync; 15 | 16 | /// A callback to used to verify the download. 17 | pub type Verify = 18 | std::sync::Arc Verification + Send + Sync>; 19 | 20 | /// The possible states of file verification 21 | #[derive(Debug, Eq, PartialEq)] 22 | pub enum Verification { 23 | /// The file has not been verified at all. 24 | NotVerified, 25 | /// The file failed the verification process. 26 | Failed, 27 | /// The file passed the verification process. 28 | Ok, 29 | } 30 | 31 | impl std::fmt::Display for Verification { 32 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 33 | write!( 34 | f, 35 | "{}", 36 | match &self { 37 | Self::NotVerified => "not verified", 38 | Self::Failed => "FAILED", 39 | Self::Ok => "Ok", 40 | } 41 | ) 42 | } 43 | } 44 | 45 | // ---------------------------------------------------------------------- 46 | // - Noop: 47 | // ---------------------------------------------------------------------- 48 | 49 | /// Do nothing to verify the download 50 | #[must_use] 51 | pub fn noop() -> crate::Verify { 52 | std::sync::Arc::new(|_: std::path::PathBuf, _: &crate::SimpleProgress| { 53 | Verification::NotVerified 54 | }) 55 | } 56 | 57 | // ---------------------------------------------------------------------- 58 | // - SHA3: 59 | // ---------------------------------------------------------------------- 60 | 61 | /// Make sure the downloaded file matches a provided hash using a provided Digest function 62 | #[cfg(feature = "verify")] 63 | #[must_use] 64 | pub fn with_digest(hash: Vec) -> crate::Verify { 65 | use std::io::Read; 66 | 67 | std::sync::Arc::new( 68 | move |path: std::path::PathBuf, cb: &crate::SimpleProgress| { 69 | let mut hasher = D::new(); 70 | 71 | if let Ok(file) = std::fs::OpenOptions::new().read(true).open(&path) { 72 | let mut reader = std::io::BufReader::with_capacity(1024 * 1024, file); 73 | let mut current = 0; 74 | 75 | let mut buffer = [0_u8; 1024 * 1024]; 76 | while let Ok(n) = reader.read(&mut buffer[..]) { 77 | if n == 0 { 78 | break; 79 | } 80 | 81 | hasher.update(&buffer[..n]); 82 | 83 | cb(current); 84 | current += n as u64; 85 | } 86 | 87 | let result = hasher.finalize(); 88 | 89 | if result.len() != hash.len() { 90 | return Verification::Failed; 91 | } 92 | for i in 0..result.len() { 93 | if result[i] != hash[i] { 94 | return Verification::Failed; 95 | } 96 | } 97 | return Verification::Ok; 98 | } 99 | 100 | Verification::Failed 101 | }, 102 | ) 103 | } 104 | --------------------------------------------------------------------------------