├── .config ├── spellcheck.toml └── trauma.dic ├── .editorconfig ├── .github ├── .kodiak.toml ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── feature_request.md │ └── help_request.md ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml ├── labels.yml ├── stale.yml └── workflows │ └── ci-rust.yml ├── .gitignore ├── .markdownlint.yml ├── CHANGELOG.md ├── Cargo.toml ├── LICENSE ├── README.md ├── SLSA.md ├── assets ├── fading-bars.png └── pip-style.png ├── examples ├── simple.rs ├── with-authentication.rs ├── with-path-as-filename.rs ├── with-range-error-handling.rs ├── with-report.rs ├── with-resume.rs └── with-style.rs ├── justfile └── src ├── download.rs ├── downloader.rs └── lib.rs /.config/spellcheck.toml: -------------------------------------------------------------------------------- 1 | [Hunspell] 2 | lang = "en_US" 3 | search_dirs = ["."] 4 | extra_dictionaries = ["trauma.dic"] 5 | skip_os_lookups = true 6 | use_builtin = true 7 | 8 | [Reflow] 9 | max_line_length = 80 10 | -------------------------------------------------------------------------------- /.config/trauma.dic: -------------------------------------------------------------------------------- 1 | 15 2 | CLI 3 | DLM 4 | Duma 5 | MAnager 6 | PR 7 | PRs 8 | Reqwest 9 | Siwi 10 | Tokio 11 | Zou 12 | ci 13 | downloader 14 | indicatif 15 | io 16 | resumable 17 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | indent_size = 2 7 | indent_style = space 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.py] 12 | indent_size = 4 13 | 14 | [Makefile] 15 | indent_style = tab 16 | -------------------------------------------------------------------------------- /.github/.kodiak.toml: -------------------------------------------------------------------------------- 1 | version = 1 2 | 3 | [merge] 4 | blacklist_title_regex = "^WIP:.*" 5 | blacklist_labels = ["do-not-merge"] 6 | delete_branch_on_merge = true 7 | method = "squash" 8 | prioritize_ready_to_merge = true 9 | require_automerge_label = false 10 | 11 | [update] 12 | always = true 13 | require_automerge_label = false 14 | 15 | [approve] 16 | auto_approve_usernames = ["dependabot"] 17 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | labels: "kind/bug" 5 | --- 6 | 7 | # Bug Report 8 | 9 | 10 | 11 | ## Current Behavior 12 | 13 | 14 | 15 | ## Expected Behavior 16 | 17 | 20 | 21 | ## Possible Solution 22 | 23 | 27 | 28 | ## Steps to Reproduce 29 | 30 | 34 | 35 | 1. 36 | 2. 37 | 3. 38 | 39 | ## Context 40 | 41 | 46 | 47 | ## Your Environment 48 | 49 | 75 | 76 | ```bash 77 | (replace the example bellow with the script output) 78 | Last commit: 79 | 583bc87 Fix configuration issue 80 | Browser(s): 81 | Mozilla Firefox 60.0 82 | Google Chrome 66.0.3359.139 83 | System Version: macOS 10.13.4 (17E202) 84 | ``` 85 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | labels: "kind/feature" 5 | --- 6 | 7 | # Feature request 8 | 9 | 10 | 11 | ## Current Behavior 12 | 13 | 14 | 15 | ## Expected Behavior 16 | 17 | 19 | 20 | ## Possible Solution 21 | 22 | 26 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/help_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Help request 3 | about: Ask for help 4 | labels: feedback/question 5 | --- 6 | 7 | # Help request 8 | 9 | 10 | 11 | ## Problem 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # Pull Request 2 | 3 | ## Types of changes 4 | 5 | 9 | 10 | - Bug fix (non-breaking change which fixes an issue) 11 | - New feature (non-breaking change which adds functionality) 12 | - Breaking change (fix or feature that would cause existing functionality to 13 | change) 14 | - Code cleanup / Refactoring 15 | - Documentation 16 | - Infrastructure and automation 17 | 18 | ## Description 19 | 20 | 24 | 25 | 29 | 30 | 37 | 38 | ## Checklist 39 | 40 | 45 | 46 | - [] I have updated the documentation accordingly 47 | - [] I have updated the Changelog (if applicable) 48 | 49 | 53 | 54 | Fixes: PeopleForBikes/PeopleForBikes.github.io# 55 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Maintain Rust dependencies. 4 | - package-ecosystem: cargo 5 | directory: "/" 6 | schedule: 7 | interval: weekly 8 | 9 | # Maintain dependencies for GitHub Actions. 10 | - package-ecosystem: "github-actions" 11 | directory: "/" 12 | schedule: 13 | interval: "weekly" 14 | -------------------------------------------------------------------------------- /.github/labels.yml: -------------------------------------------------------------------------------- 1 | --- 2 | labels: 3 | # Kinds. 4 | - name: "kind/bug" 5 | color: "#D73A4A" 6 | description: "Something isn't working" 7 | - name: "kind/design" 8 | color: "#8000FF" 9 | description: "Visual aspect" 10 | - name: "kind/docs" 11 | color: "#000BE0" 12 | description: "Document the project" 13 | - name: "kind/enhancement" 14 | color: "#A2EEEF" 15 | description: "Improve an existing feature" 16 | - name: "kind/optimization" 17 | color: "#A2EEEF" 18 | description: "Optimize an existing feature" 19 | - name: "kind/feature" 20 | color: "#A2EEEF" 21 | description: "New feature request" 22 | - name: "kind/infrastructure" 23 | color: "#387499" 24 | description: "Issue regarding the local or cloud infrastructure setup" 25 | - name: "kind/automation" 26 | color: "#387499" 27 | description: "Automate your tasks" 28 | - name: "kind/security" 29 | color: "#993333" 30 | description: "CVEs and other security flaws" 31 | 32 | # Feedback. 33 | - name: "feedback/question" 34 | color: "#D876E3" 35 | description: "Further information is requested" 36 | - name: "feedback/discussion" 37 | color: "#D876E3" 38 | description: "Discuss a specific aspect of the project" 39 | 40 | # T-Shirt sizes. 41 | - name: "size/XS" 42 | color: "#00FF00" 43 | description: "Extra small (0-9 lines of changes)" 44 | - name: "size/S" 45 | color: "#CCFF66" 46 | description: "Small (10-29 lines of changes)" 47 | - name: "size/M" 48 | color: "#FFFF00" 49 | description: "Medium (30-99 lines of changes)" 50 | - name: "size/L" 51 | color: "#FF9933" 52 | description: "Large (100-499 lines of changes)" 53 | - name: "size/XL" 54 | color: "#B60205" 55 | description: "Extra large (500-999 lines of changes)" 56 | - name: "size/XXL" 57 | color: "#8B0000" 58 | description: "Extra Extra Large (1000+ lines of changes)" 59 | 60 | # Experience. 61 | - name: "exp/beginner" 62 | color: "#CBE4CE" 63 | description: "Good for newcomers" 64 | - name: "exp/intermediate" 65 | color: "#CBE4CE" 66 | description: "Show off your skills" 67 | - name: "exp/expert" 68 | color: "#CBE4CE" 69 | description: "I have nothing else to teach you" 70 | 71 | # Statuses. 72 | - name: "status/claimed" 73 | color: "#FBCA04" 74 | description: "Assigned" 75 | - name: "status/help wanted" 76 | color: "#008672" 77 | description: "Could use an extra brain" 78 | - name: "status/more info needed" 79 | color: "#008672" 80 | description: "Needs clarification" 81 | - name: "status/invalid" 82 | color: "#D2DAE1" 83 | description: "No further triaging" 84 | - name: "status/wontfix" 85 | color: "#D2DAE1" 86 | description: "Fix is too controversial or do not want to implement it" 87 | - name: "status/duplicate" 88 | color: "#D2DAE1" 89 | description: "This issue or pull request already exists" 90 | - name: "status/review-carefully!" 91 | color: "#B60205" 92 | description: "Requires extra attention during review" 93 | 94 | # Environment. 95 | - name: "environment/dev" 96 | color: "#F7FBFF" 97 | description: "Developer environment" 98 | - name: "environment/prod" 99 | color: "#F7FBFF" 100 | description: "Production environment" 101 | 102 | # Bots. 103 | - name: "dependencies" 104 | color: "#0366d6" 105 | description: "Pull requests that update a dependency file" 106 | - name: "do-not-merge" 107 | color: "#DC143C" 108 | description: "Prevents PRs with this label to be merged" 109 | -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | # Number of days of inactivity before a PR becomes stale. 2 | daysUntilStale: 21 3 | # Number of days of inactivity before a stale PR is closed. 4 | daysUntilClose: 7 5 | 6 | # Limit to only `pulls`. 7 | only: pulls 8 | 9 | # Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable. 10 | exemptLabels: 11 | - no-mergify 12 | - do-not-merge 13 | 14 | # Comment to post when marking an issue as stale. Set to `false` to disable. 15 | markComment: > 16 | This pull request has been automatically marked as stale because it has not had 17 | recent activity. 18 | 19 | It will be closed in 7 days if no further activity occurs. 20 | 21 | Thank you for your contribution! 22 | 23 | # Comment to post when closing a stale issue. Set to `false` to disable. 24 | closeComment: > 25 | This pull request has been automatically closed because there has 26 | been no activity for 28 days. 27 | 28 | Please feel free to reopen it (or open a new one) if the proposed 29 | change is still appropriate. 30 | 31 | Thank you for your contribution! 32 | -------------------------------------------------------------------------------- /.github/workflows/ci-rust.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | pull_request: 5 | types: 6 | - opened 7 | - synchronize 8 | - reopened 9 | push: 10 | branches: 11 | - main 12 | tags: 13 | - "[0-9]+.[0-9]+.[0-9]+" 14 | 15 | jobs: 16 | test: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v4 20 | - uses: dtolnay/rust-toolchain@stable 21 | - uses: Swatinem/rust-cache@v2 22 | - name: Run the unit/doc tests 23 | run: cargo test 24 | - name: Test examples 25 | run: | 26 | EXAMPLES="simple 27 | with-report 28 | with-style 29 | with-resume 30 | " 31 | for example in $EXAMPLES; do 32 | echo "==== Testing $example =======================================" 33 | cargo run -q --example "$example" 34 | echo "=============================================================" 35 | done 36 | 37 | lint: 38 | runs-on: ubuntu-latest 39 | steps: 40 | - uses: actions/checkout@v4 41 | - uses: extractions/setup-just@v3 42 | - uses: dtolnay/rust-toolchain@stable 43 | with: 44 | components: clippy, rustfmt 45 | - uses: Swatinem/rust-cache@v2 46 | - run: cargo fmt --all -- --check 47 | - run: cargo check 48 | - run: cargo clippy -- -D warnings 49 | - name: Lint markdown 50 | run: just lint-md 51 | 52 | build: 53 | runs-on: ubuntu-latest 54 | steps: 55 | - uses: actions/checkout@v4 56 | - uses: dtolnay/rust-toolchain@stable 57 | - uses: Swatinem/rust-cache@v2 58 | - run: cargo build 59 | 60 | # Disabling this for now since it does not really add a lot of value but does 61 | # add a lot of CI time. 62 | # attest: 63 | # needs: 64 | # - build 65 | # - lint 66 | # - test 67 | # runs-on: ubuntu-latest 68 | # if: startsWith(github.ref, 'refs/tags/') 69 | # steps: 70 | # - uses: actions/checkout@v4 71 | # - uses: dtolnay/rust-toolchain 72 | # with: 73 | # profile: minimal 74 | # toolchain: stable 75 | # override: true 76 | # - uses: Swatinem/rust-cache@v2 77 | 78 | # # Generate attestation. 79 | # - uses: actions-rs/cargo@v1 80 | # with: 81 | # command: package 82 | # - run: | 83 | # mkdir -p dist 84 | # cp target/package/*.crate dist/ 85 | # - name: Generate provenance 86 | # uses: slsa-framework/github-actions-demo@v0.1 87 | # with: 88 | # artifact_path: dist/ 89 | # output_path: trauma.${{ github.sha }}.att 90 | 91 | # # Sign the attestation. 92 | # - uses: sigstore/cosign-installer@main 93 | # - run: | 94 | # export COSIGN_PASSWORD="${{ secrets.COSIGN_PASSWORD }}" 95 | # echo -n "${{ secrets.COSIGN_KEY }}" > /tmp/cosign.key 96 | # cosign sign-blob --key /tmp/cosign.key trauma.${{ github.sha }}.att > trauma.${{ github.sha }}.sig 97 | 98 | # # Upload attestation files. 99 | # - name: Upload provenance 100 | # uses: actions/upload-artifact@v3 101 | # with: 102 | # path: | 103 | # trauma.${{ github.sha }}.att 104 | # trauma.${{ github.sha }}.sig 105 | 106 | release: 107 | needs: 108 | - build 109 | - lint 110 | - test 111 | runs-on: ubuntu-latest 112 | if: startsWith(github.ref, 'refs/tags/') 113 | steps: 114 | - uses: actions/checkout@v4 115 | - uses: dtolnay/rust-toolchain@stable 116 | - uses: Swatinem/rust-cache@v2 117 | - uses: actions/download-artifact@v4 118 | - name: Get Changelog Entry 119 | id: changelog_reader 120 | uses: mindsers/changelog-reader-action@v2 121 | - name: Publish the release 122 | uses: softprops/action-gh-release@v2 123 | env: 124 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 125 | with: 126 | body: ${{ steps.changelog_reader.outputs.changes }} 127 | - run: cargo publish 128 | env: 129 | CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} 130 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | **/*.rs.bk 3 | /output/ 4 | /target 5 | /target/ 6 | Cargo.lock 7 | examples/miniserve/ 8 | -------------------------------------------------------------------------------- /.markdownlint.yml: -------------------------------------------------------------------------------- 1 | # Disable some built-in rules. 2 | headings: 3 | siblings_only: true 4 | line-length: 5 | tables: false 6 | code_blocks: false 7 | single-h1: false 8 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to 7 | [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 8 | 9 | ## [Unreleased] 10 | 11 | ## [2.2.6] - 2024-11-14 12 | 13 | ### Fixed 14 | 15 | - Updated the `reqwest-*` libraries. [#113] 16 | 17 | [#113]: https://github.com/rgreinho/trauma/pull/113 18 | [2.2.6]: https://github.com/rgreinho/trauma/releases/tag/2.2.6 19 | 20 | ## [2.2.5] 21 | 22 | ### Fixed 23 | 24 | - Handled 416 status when a file is already fully downloaded. [#86] 25 | 26 | [#86]: https://github.com/rgreinho/trauma/pull/86 27 | [2.2.5]: https://github.com/rgreinho/trauma/releases/tag/2.2.5 28 | 29 | ## [2.2.4] 30 | 31 | ### Fixed 32 | 33 | - Fixed a silent failure when a download filename contains a path. [#73] 34 | 35 | [#73]: https://github.com/rgreinho/trauma/pull/73 36 | [2.2.4]: https://github.com/rgreinho/trauma/releases/tag/2.2.4 37 | 38 | ## [2.2.3] - 2023-05-14 39 | 40 | ### Fixed 41 | 42 | - Ensured that custom headers are used for all the downloader requests, for 43 | instance, propagating authentication headers to check whether a download can 44 | be resumed. [#62] 45 | 46 | [#62]: https://github.com/rgreinho/trauma/pull/62 47 | [2.2.3]: https://github.com/rgreinho/trauma/releases/tag/2.2.3 48 | 49 | ## [2.2.2] - 2023-04-26 50 | 51 | ### Fixed 52 | 53 | - Fixed the total size which was incorrectly reported in the progress bar. [#59] 54 | 55 | [#59]: https://github.com/rgreinho/trauma/pull/59 56 | [2.2.2]: https://github.com/rgreinho/trauma/releases/tag/2.2.2 57 | 58 | ## [2.2.1] - 2023-03-06 59 | 60 | ### Fixed 61 | 62 | - Handled the case when the server does not send a `content-length` header, 63 | causing downloads to be skipped. [#56] 64 | 65 | [#56]: https://github.com/rgreinho/trauma/pull/56 66 | [2.2.1]: https://github.com/rgreinho/trauma/releases/tag/2.2.1 67 | 68 | ## [2.2.0] - 2022-01-21 69 | 70 | ### Added 71 | 72 | - Added the ability to provide custom HTTP headers to the downloader. [#53] 73 | 74 | [#53]: https://github.com/rgreinho/trauma/pull/53 75 | [2.2.0]: https://github.com/rgreinho/trauma/releases/tag/2.2.0 76 | 77 | ## [2.1.1] - 2022-11-19 78 | 79 | ### Fixed 80 | 81 | - Fixed a bug preventing the progress bars to be hidden. [#45] 82 | 83 | ### Changed 84 | 85 | - Upgraded [indicatif] from 0.17.0-rc.10 to 0.17.2. [#45] 86 | 87 | [#45]: https://github.com/rgreinho/trauma/pull/45 88 | [2.1.1]: https://github.com/rgreinho/trauma/releases/tag/2.1.1 89 | 90 | ## [2.1.0] - 2022-09-10 91 | 92 | ### Added 93 | 94 | - Added the ability to use a proxy. [#33] 95 | 96 | ### Fixed 97 | 98 | - Fixed the filename parsing when constructing from URL. [#33] 99 | 100 | [#33]: https://github.com/rgreinho/trauma/pull/33 101 | [2.1.0]: https://github.com/rgreinho/trauma/releases/tag/2.1.0 102 | 103 | ## [2.0.0] - 2022-04-21 104 | 105 | ### Added 106 | 107 | - Added the ability to resume downloads. [#26] 108 | 109 | ### Changed 110 | 111 | - Removed the `skip_existing` option. [#26] 112 | 113 | ### Fixed 114 | 115 | - Fixed a bug preventing the progress bars to be disabled. [#29] 116 | 117 | [#26]: https://github.com/rgreinho/trauma/pull/26 118 | [#29]: https://github.com/rgreinho/trauma/pull/29 119 | [2.0.0]: https://github.com/rgreinho/trauma/releases/tag/2.0.0 120 | 121 | ## [1.1.0] - 2022-04-15 122 | 123 | ### Added 124 | 125 | - Added ability to skip a download if a file with the same name exists at the 126 | destination. [#16] 127 | - Added ability to customize the progress bars [#24] 128 | - Customize the format 129 | - Customize the progression style 130 | - Leave them on the screen or clear them upon completion 131 | - Hide any or both of them 132 | - Add preconfigured styles 133 | 134 | [#16]: https://github.com/rgreinho/trauma/pull/16 135 | [#24]: https://github.com/rgreinho/trauma/pull/24 136 | [1.1.0]: https://github.com/rgreinho/trauma/releases/tag/1.1.0 137 | 138 | ## [1.0.0] - 2022-03-29 139 | 140 | Initial version with the following feature set: 141 | 142 | - Library only 143 | - HTTP(S) downloads 144 | - Download files via providing a list of URLs 145 | - Ability to rename downloaded files 146 | - Ability to configure the download manager 147 | - Download directory 148 | - Maximum simultaneous requests 149 | - Number of retries 150 | - Asynchronous w/ [Tokio] 151 | - Progress bar w/ [trauma] 152 | - Display the individual progress 153 | - Display the total progress 154 | 155 | [1.0.0]: https://github.com/rgreinho/trauma/releases/tag/1.0.0 156 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "trauma" 3 | version = "2.2.6" 4 | edition = "2021" 5 | license = "MIT" 6 | description = "Simplify and prettify HTTP downloads" 7 | homepage = "https://github.com/rgreinho/trauma" 8 | repository = "https://github.com/rgreinho/trauma" 9 | readme = "README.md" 10 | categories = ["concurrency"] 11 | keywords = ["http", "download", "async", "tokio", "indicatif"] 12 | 13 | [dependencies] 14 | form_urlencoded = "1.1.0" 15 | futures = "0.3.25" 16 | indicatif = "0.17.3" 17 | reqwest = { version = "0.12.4", features = ["stream", "socks"] } 18 | reqwest-middleware = "0.4.0" 19 | reqwest-retry = "0.7.0" 20 | reqwest-tracing = { version = "0.5", features = ["opentelemetry_0_22"] } 21 | task-local-extensions = "0.1.3" 22 | thiserror = "2.0.3" 23 | tokio = { version = "1", features = ["macros", "rt-multi-thread"] } 24 | tracing = "0.1" 25 | tracing-opentelemetry = "0.30" 26 | tracing-subscriber = "0.3" 27 | 28 | [dev-dependencies] 29 | color-eyre = "0.6.1" 30 | comfy-table = "7.0.0" 31 | console = "0.15" 32 | opentelemetry = "0.29.0" 33 | rand = "0.9.0" 34 | tracing-subscriber = { version = "0.3", features = ["env-filter"] } 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Rémy Greinhofer 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Trauma 2 | 3 | [![Crates.io](https://img.shields.io/crates/v/trauma.svg)](https://crates.io/crates/trauma) 4 | [![Documentation](https://docs.rs/trauma/badge.svg)](https://docs.rs/trauma/) 5 | [![ci](https://github.com/rgreinho/trauma/actions/workflows/ci-rust.yml/badge.svg)](https://github.com/rgreinho/trauma/actions/workflows/ci-rust.yml) 6 | 7 | Tokio Rust Asynchronous Universal download MAnager 8 | 9 | ## Description 10 | 11 | Trauma is a library simplifying and prettifying HTTP(s) downloads. The downloads 12 | are executed asynchronously and progress bars are drawn on the screen to help 13 | monitoring the process. 14 | 15 | ![screenshot](assets/pip-style.png) 16 | 17 | ### Features 18 | 19 | - Library only 20 | - HTTP(S) downloads 21 | - Support download via proxies 22 | - Download files via providing a list of URLs 23 | - Ability to rename downloaded files 24 | - Ability to configure the download manager 25 | - Download directory 26 | - Maximum simultaneous requests 27 | - Number of retries 28 | - Resume downloads (if supported by the remote server) 29 | - Custom HTTP Headers 30 | - Asynchronous w/ [Tokio] 31 | - Progress bar w/ [indicatif] 32 | - Display the individual progress 33 | - Display the total progress 34 | - Ability to customize the progress bars 35 | - Customize the format 36 | - Customize the progression style 37 | - Leave them on the screen or clear them upon completion 38 | - Hide any or both of them 39 | - Add pre-configured styles 40 | 41 | ## Usage 42 | 43 | Add this to your `Cargo.toml`: 44 | 45 | ```toml 46 | [dependencies] 47 | trauma = "2" 48 | ``` 49 | 50 | ## Quick start 51 | 52 | ```rust 53 | use std::path::PathBuf; 54 | use trauma::{download::Download, downloader::DownloaderBuilder, Error}; 55 | 56 | #[tokio::main] 57 | async fn main() -> Result<(), Error> { 58 | let reqwest_rs = "https://github.com/seanmonstar/reqwest/archive/refs/tags/v0.11.9.zip"; 59 | let downloads = vec![Download::try_from(reqwest_rs).unwrap()]; 60 | let downloader = DownloaderBuilder::new() 61 | .directory(PathBuf::from("output")) 62 | .build(); 63 | downloader.download(&downloads).await; 64 | Ok(()) 65 | } 66 | ``` 67 | 68 | More examples can be found in the [examples](examples) folder. They are well 69 | commented and will guide you through the different features of this library. 70 | 71 | ## Why another download manager 72 | 73 | Before starting this project, I spent some time searching the internet, trying 74 | not to reinvent the wheel. And I did find a bunch of interesting existing 75 | projects! 76 | 77 | However they are almost all abandoned: 78 | 79 | - DLM: 80 | - Active, but is just a binary/CLI tool 81 | - Snatch: 82 | - Inactive since Sept '17 83 | - Recommend switching to [Zou] 84 | - Zou: 85 | - Inactive since Oct '17 86 | - Duma: 87 | - Inactive since Nov '20 88 | - Siwi: 89 | - Inactive since Mar '21 90 | - Downloader: 91 | - Dying project 92 | - No answers to issues/PRs 93 | - Only automated updates are being merged 94 | - No release since Feb '21 95 | 96 | As a result, I decided to write `trauma`. 97 | 98 | [indicatif]: https://github.com/console-rs/indicatif 99 | [tokio]: https://tokio.rs/ 100 | [zou]: https://github.com/k0pernicus/zou 101 | -------------------------------------------------------------------------------- /SLSA.md: -------------------------------------------------------------------------------- 1 | # SLSA 2 | 3 | [SLSA] is a framework to guide developers and help them secure their software 4 | supply chain. 5 | 6 | In this document we are describing how we incrementally reach [SLSA] level 4, 7 | and explain why we meet each criteria. 8 | 9 | ## SLSA 1 10 | 11 | SLSA level 1 does not do much, but helps us get on track to build a resilient 12 | system. We only have 2 requirements to satisfy. 13 | 14 | ### ✅ [Build - Scripted build] 15 | 16 | > All build steps were fully defined in some sort of “build script”. The only 17 | > manual command, if any, was to invoke the build script. 18 | 19 | We use a GitHub Workflow to execute all the CI steps. 20 | 21 | ### ✅ [Provenance - Available] 22 | 23 | > The provenance is available to the consumer in a format that the consumer 24 | > accepts. The format SHOULD be in-toto SLSA Provenance, but another format MAY 25 | > be used if both producer and consumer agree and it meets all the other 26 | > requirements. 27 | 28 | We provide provenance using the SLSA [github-actions-demo] and attach it to the 29 | GitHub Release. 30 | 31 | ### Things to notice 32 | 33 | - The attestation will not be uploaded to crates.io, but will live in GitHub 34 | release. This is not ideal and will complexify the process to verify it. 35 | Ideally all should be managed by `cargo.` 36 | - The attestation does not mention which commit was used to build the crate. 37 | - Following [cosign]'s convention, we named the attestation 38 | `trauma.${{ github.sha }}.att` 39 | 40 | ## SLSA 2 41 | 42 | We keep adding to the security here, by requiring automated CI systems and 43 | introducing signing. 44 | 45 | ### ✅ [Source - Version controlled] 46 | 47 | > Every change to the source is tracked in a version control system that meets 48 | > the following requirements: 49 | > 50 | > - [Change history] There exists a record of the history of changes that went 51 | > into the revision. Each change must contain: the identities of the uploader 52 | > and reviewers (if any), timestamps of the reviews (if any) and submission, 53 | > the change description/justification, the content of the change, and the 54 | > parent revisions. 55 | > - [Immutable reference] There exists a way to indefinitely reference this 56 | > particular, immutable revision. In git, this is the {repo URL + 57 | > branch/tag/ref + commit ID}. 58 | 59 | We are using `git` as our VCS, which meets these requirements, so we're good 60 | here ✅. 61 | 62 | ### ✅ [Build - Build service] 63 | 64 | > All build steps ran using some build service, not on a developer’s 65 | > workstation. 66 | 67 | We use GitHub Actions as the automated build system. 68 | 69 | ### ✅ [Provenance - Authenticated] 70 | 71 | > The provenance’s authenticity and integrity can be verified by the consumer. 72 | > This SHOULD be through a digital signature from a private key accessible only 73 | > to the service generating the provenance. 74 | 75 | For this requirement, we are using [cosign] to sign the provenance attestation. 76 | The attestation signature will be attached to the GitHub release, next to the 77 | attestation itself. 78 | 79 | Here is the trauma public key `trauma.pub`: 80 | 81 | ```pgp 82 | -----BEGIN PUBLIC KEY----- 83 | MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9REilH9skU+RaK9Pcs3PhAKOSsjH 84 | hSt+Gu73ChvvFNZd5u9LDU7BpdJUG5vKBVMaHnOlPY1Az/dC2DxqQa5iRQ== 85 | -----END PUBLIC KEY----- 86 | ``` 87 | 88 | The signature could be verified using cosign as well: 89 | 90 | ```bash 91 | $ cosign verify-blob --key trauma.pub --signature trauma.sig trauma.att 92 | Verified OK 93 | ``` 94 | 95 | And we're good here. 96 | 97 | ### ❌ [Provenance - Service generated] 98 | 99 | > The data in the provenance MUST be obtained from the build service (either 100 | > because the generator is the build service or because the provenance generator 101 | > reads the data directly from the build service). 102 | > 103 | > Regular users of the service MUST NOT be able to inject or alter the contents, 104 | > except as noted below. 105 | > 106 | > The following provenance fields MAY be generated by the user-controlled build 107 | > steps: 108 | > 109 | > - The output artifact hash from Identifies Artifact. Reasoning: This only 110 | > allows a “bad” build to falsely claim that it produced a “good” artifact. 111 | > This is not a > security problem because the consumer MUST accept only 112 | > “good” builds and reject “bad” builds. 113 | > - The “reproducible” boolean and justification from Reproducible. 114 | 115 | We are generating the provenance through a custom action as a step in our GitHub 116 | Workflow. It is not a service offered by GitHub Actions itself. Therefore we do 117 | not meet the criteria. 118 | 119 | ### Things to notice 120 | 121 | - It is best practice to add a password to the signing key, but it was not 122 | required to be SLSA 2 compliant. 123 | - Could probably further improve the verification workflow by using 124 | [rekor]/[fulcio]. 125 | - Using the GitHub OIDC provider would also be a nice addition, but this feature 126 | is still marked as experimental at the time of writing. See the [openid 127 | signing] reference for more information. 128 | 129 | ## SLSA 3 130 | 131 | Now we are getting serious! 132 | 133 | ### ✅ [Source - Verified history] 134 | 135 | > Every change in the revision’s history has at least one strongly authenticated 136 | > actor identity (author, uploader, reviewer, etc.) and timestamp. It must be 137 | > clear which identities were verified, and those identities must use two-step 138 | > verification or similar. (Exceptions noted below.) 139 | 140 | Here are the elements allowing us to match this requirement: 141 | 142 | - We are using GitHub. 143 | - We can identify the author of a pull-request. 144 | - A GitHub account is required to submit a pull-request. 145 | - This repository is not in an organization, BUT 2FA is enabled. 146 | - This repository requires pull-request to add changes and the main branch has 147 | been protected. 148 | 149 | So we should be in compliance here. 150 | 151 | ### ✅ [Source - Retained indefinitely] (18 months for SLSA 3) 152 | 153 | > The revision and its change history are preserved indefinitely and cannot be 154 | > deleted, except when subject to an established and transparent policy for 155 | > obliteration, such as a legal or policy requirement. 156 | > 157 | > [Immutable history] It must not be possible for persons to delete or modify 158 | > the history, even with multi-party approval, except by trusted platform admins 159 | > with two-party approval following the obliterate policy. 160 | 161 | Since we are using GitHub we think we are in compliance with this item. Besides 162 | `git` does not allow to modify a commit without altering the child commits. 163 | 164 | ### ✅ [Build - Build as code] 165 | 166 | > The build definition and configuration is defined in text files, stored in a 167 | > version control system, and is executed by the build service. 168 | 169 | Our build operations are defined in the our GitHub workflow `ci-rust.yml`. 170 | 171 | ### ❓ [Build - Ephemeral environment] 172 | 173 | > The build service ensured that the build steps ran in an ephemeral 174 | > environment, such as a container or VM, provisioned solely for this build, and 175 | > not reused from a prior build. 176 | 177 | [github hosted runners] are ephemeral VMs. Therefore each job will be run in a 178 | different VM, **but not each step**. So if we follow the definition strictly, we 179 | are not in compliance here. But since our jobs are isolated from each others, we 180 | may still be good. 181 | 182 | ### ❓ [Build - Isolated] 183 | 184 | > The build service ensured that the build steps ran in an isolated environment 185 | > free of influence from other build instances, whether prior or concurrent. 186 | > 187 | > - It MUST NOT be possible for a build to access any secrets of the build 188 | > service, such as the provenance signing key. 189 | > - It MUST NOT be possible for two builds that overlap in time to influence one 190 | > another. 191 | > - It MUST NOT be possible for one build to persist or influence the build 192 | > environment of a subsequent build. 193 | > - Build caches, if used, MUST be purely content-addressable to prevent 194 | > tampering. 195 | 196 | We think we are in compliance here, but the secrets are usable to each step of 197 | the workflow, even though the contant cannot be viewed direclty. 198 | 199 | GitHub actions lets admin share secrets either at the organization level, either 200 | at the repository level. This part will require more digging to see if the way 201 | GitHub Actions is implemented is compliant or not. 202 | 203 | ### ❌ [Provenance - Non-falsifiable] 204 | 205 | > Provenance cannot be falsified by the build service’s users. 206 | > 207 | > NOTE: This requirement is a stricter version of Service Generated. 208 | > 209 | > - The provenance signing key MUST be stored in a secure key management system 210 | > accessible only to the build service account. 211 | > - The provenance signing key MUST NOT be accessible to the environment running 212 | > the user-defined build steps. 213 | > - Every field in the provenance MUST be generated or verified by the build 214 | > service in a trusted control plane. 215 | > - The user-controlled build steps MUST NOT be able to inject or alter the 216 | > contents, except as noted below. 217 | 218 | We clearly don't meet this criteria since our signing key is stored as a git 219 | secret, therefore accessible anywhere throughout the workflow. 220 | 221 | ### Things to notice 222 | 223 | - Although we can identify the authors of a pull-request, commit signing is not 224 | required. 225 | - Some definitions could use some clarification. 226 | - There is no standard naming and each CI system uses different words to mean 227 | the same thing, or the same words to mean different things. 228 | 229 | | GitHub | Tekton | 230 | | -------- | -------- | 231 | | Workflow | Pipeline | 232 | | Job | Task | 233 | | Step | Step | 234 | 235 | ## SLSA 4 236 | 237 | Since we believe we are SLSA 3 compliant, let's continue this excercise and see 238 | if we can reach level 4. 239 | 240 | ### ❌ [Source - Two-person reviewed] 241 | 242 | > Every change in the revision’s history was agreed to by two trusted persons 243 | > prior to submission, and both of these trusted persons were strongly 244 | > authenticated. 245 | 246 | We are failing this requirement since the repository only has 1 maintainer. 247 | 248 | ### ✅ [Build - Parameterless] 249 | 250 | > The build output cannot be affected by user parameters other than the build 251 | > entry point and the top-level source location. In other words, the build is 252 | > fully defined through the build script and nothing else. 253 | 254 | We are using Github Actions in this manner. The only parameters are information 255 | coming from the source to build itself (commit sha, repository name, etc.) 256 | 257 | ### ❌ [Build - Hermetic] 258 | 259 | > All transitive build steps, sources, and dependencies were fully declared up 260 | > front with immutable references, and the build steps ran with no network 261 | > access. 262 | 263 | All the dependencies are strongly specified in the `Cargo.lock` file. 264 | 265 | However the dependencies are being fetched from crates.io every time the build 266 | runs (even though we are using caching), so we may be failing this requirement. 267 | 268 | A way to satisfy this requirement would be to use `cargo vendor` to vendor all 269 | dependencies locally. 270 | 271 | ### ❓ [Build - Reproducible] (best effort) 272 | 273 | > Re-running the build steps with identical input artifacts results in 274 | > bit-for-bit identical output. Builds that cannot meet this MUST provide a 275 | > justification why the build cannot be made reproducible. 276 | 277 | We think we are doing good on this one, mainly dur to the use of the lock file, 278 | but this item would require deeper investigation. 279 | 280 | ### ❌ [Provenance - Dependencies complete] 281 | 282 | > Provenance records all build dependencies that were available while running 283 | > the build steps. This includes the initial state of the machine, VM, or 284 | > container of the build worker. 285 | 286 | We may provide an SBOM, but that does not seem to be enough to satisfy this 287 | criteria. 288 | 289 | ### 🚫 [Common - Security] 290 | 291 | > The system meets some TBD baseline security standard to prevent compromise. 292 | > (Patching, vulnerability scanning, user isolation, transport security, secure 293 | > boot, machine identity, etc. Perhaps NIST 800-53 or a subset thereof.) 294 | 295 | This criteria seems to only apply does services which are deployed somewhere, 296 | therefore not apply here. 297 | 298 | ### 🚫 [Common - Access] 299 | 300 | > All physical and remote access must be rare, logged, and gated behind 301 | > multi-party approval. 302 | 303 | This criteria seems to only apply does services which are deployed somewhere, 304 | therefore not apply here. 305 | 306 | ### ❌ [Common - Superusers] 307 | 308 | > Only a small number of platform admins may override the guarantees listed 309 | > here. Doing so MUST require approval of a second platform admin. 310 | 311 | Since we have one admin, we're also failing this criteria. But they are talking 312 | about a platform admin, so it may no apply here. 313 | 314 | ### Things to notice 315 | 316 | - A lot of projects/tools/libraries only have one maintainer. Therefore it rules 317 | them out from being SLSA 4 compliant right away. 318 | - Such a high level of details in the provenance seems ideal, but it looks like 319 | such a tool does not exist yet. 320 | - The criteria in the "Common" category seem to only applied to deployed 321 | services. 322 | 323 | [build - build as code]: https://slsa.dev/spec/v0.1/requirements#build-as-code 324 | [build - isolated]: https://slsa.dev/spec/v0.1/requirements#isolated 325 | [build - build service]: https://slsa.dev/spec/v0.1/requirements#build-service 326 | [build - ephemeral environment]: 327 | https://slsa.dev/spec/v0.1/requirements#ephemeral-environment 328 | [build - hermetic]: https://slsa.dev/spec/v0.1/requirements#hermetic 329 | [build - reproducible]: https://slsa.dev/spec/v0.1/requirements#reproducible 330 | [build - parameterless]: https://slsa.dev/spec/v0.1/requirements#parameterless 331 | [build - scripted build]: https://slsa.dev/spec/v0.1/requirements#scripted-build 332 | [common - security]: https://slsa.dev/spec/v0.1/requirements#security 333 | [common - access]: https://slsa.dev/spec/v0.1/requirements#access 334 | [common - superusers]: https://slsa.dev/spec/v0.1/requirements#superusers 335 | [cosign]: https://github.com/sigstore/cosign 336 | [fulcio]: https://github.com/sigstore/fulcio 337 | [github-actions-demo]: https://github.com/slsa-framework/github-actions-demo 338 | [github hosted runners]: 339 | https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners 340 | [openid signing]: https://docs.sigstore.dev/cosign/openid_signing 341 | [provenance - authenticated]: 342 | https://slsa.dev/spec/v0.1/requirements#authenticated 343 | [provenance - available]: https://slsa.dev/spec/v0.1/requirements#available 344 | [provenance - dependencies complete]: 345 | https://slsa.dev/spec/v0.1/requirements#dependencies-complete 346 | [provenance - non-falsifiable]: 347 | https://slsa.dev/spec/v0.1/requirements#non-falsifiable 348 | [provenance - service generated]: 349 | https://slsa.dev/spec/v0.1/requirements#service-generated 350 | [rekor]: https://github.com/sigstore/rekor 351 | [slsa]: https://slsa.dev/ 352 | [source - retained indefinitely]: 353 | https://slsa.dev/spec/v0.1/requirements#retained-indefinitely 354 | [source - two-person reviewed]: 355 | https://slsa.dev/spec/v0.1/requirements#two-person-reviewed 356 | [source - version controlled]: 357 | https://slsa.dev/spec/v0.1/requirements#version-controlled 358 | [source - verified history]: 359 | https://slsa.dev/spec/v0.1/requirements#verified-history 360 | -------------------------------------------------------------------------------- /assets/fading-bars.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rgreinho/trauma/36fb3968388afb2b292dfa9a6a411b334272a788/assets/fading-bars.png -------------------------------------------------------------------------------- /assets/pip-style.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rgreinho/trauma/36fb3968388afb2b292dfa9a6a411b334272a788/assets/pip-style.png -------------------------------------------------------------------------------- /examples/simple.rs: -------------------------------------------------------------------------------- 1 | //! Simple download example. 2 | //! 3 | //! Run with 4 | //! 5 | //! ```not_rust 6 | //! cargo run -q --example simple 7 | //! ``` 8 | 9 | use std::path::PathBuf; 10 | use trauma::{download::Download, downloader::DownloaderBuilder, Error}; 11 | 12 | #[tokio::main] 13 | async fn main() -> Result<(), Error> { 14 | let reqwest_rs = "https://github.com/seanmonstar/reqwest/archive/refs/tags/v0.11.9.zip"; 15 | let downloads = vec![Download::try_from(reqwest_rs).unwrap()]; 16 | let downloader = DownloaderBuilder::new() 17 | .directory(PathBuf::from("output")) 18 | .build(); 19 | downloader.download(&downloads).await; 20 | Ok(()) 21 | } 22 | -------------------------------------------------------------------------------- /examples/with-authentication.rs: -------------------------------------------------------------------------------- 1 | //! Download a file with authentication example. 2 | //! 3 | //! Setup for this example: 4 | //! 5 | //! From the root of the project: 6 | //! ```not_rust 7 | //! mkdir -p examples/miniserve 8 | //! cd examples/miniserve 9 | //! curl -sLO https://cdimage.debian.org/debian-cd/current/arm64/iso-cd/debian-11.7.0-arm64-netinst.iso 10 | //! miniserve --auth trauma:test . 11 | //! ``` 12 | //! 13 | //! Then from another terminal: 14 | //! 15 | //! ```not_rust 16 | //! cargo run -q --example simple 17 | //! ``` 18 | //! 19 | //! The value for the authentication header can be generated from the command line: 20 | //! ```not_rust 21 | //! echo -n "trauma:test"|base64 22 | //! ``` 23 | //! Or from a website like https://www.debugbear.com/basic-auth-header-generator. 24 | //! 25 | //! Miniserve is a utility written in rust to serve files over HTTP: 26 | //! https://github.com/svenstaro/miniserve 27 | //! 28 | 29 | use reqwest::header::{self, HeaderValue}; 30 | use std::path::PathBuf; 31 | use trauma::{download::Download, downloader::DownloaderBuilder, Error}; 32 | 33 | #[tokio::main] 34 | async fn main() -> Result<(), Error> { 35 | let reqwest_rs = "http://localhost:8080/debian-11.7.0-arm64-netinst.iso"; 36 | let downloads = vec![Download::try_from(reqwest_rs).unwrap()]; 37 | let auth = HeaderValue::from_str("Basic dHJhdW1hOnRlc3Q=").expect("Invalid auth"); 38 | let downloader = DownloaderBuilder::new() 39 | .directory(PathBuf::from("output")) 40 | .header(header::AUTHORIZATION, auth) 41 | .build(); 42 | let summaries = downloader.download(&downloads).await; 43 | let summary = summaries.first().unwrap(); 44 | println!("{:?}", summary.status()); 45 | Ok(()) 46 | } 47 | -------------------------------------------------------------------------------- /examples/with-path-as-filename.rs: -------------------------------------------------------------------------------- 1 | //! Download example using a target filename with a path. 2 | //! 3 | //! Run with 4 | //! 5 | //! ```not_rust 6 | //! cargo run -q --example with-path-as-filename 7 | //! ``` 8 | use trauma::{download::Download, downloader::DownloaderBuilder, Error}; 9 | 10 | use reqwest::Url; 11 | 12 | #[tokio::main] 13 | async fn main() -> Result<(), Error> { 14 | let reqwest_rs = "https://github.com/seanmonstar/reqwest/archive/refs/tags/v0.11.9.zip"; 15 | let downloads = vec![Download { 16 | url: Url::parse(reqwest_rs).unwrap(), 17 | filename: "output/test_dir/reqwest.zip".to_string(), 18 | }]; 19 | let downloader = DownloaderBuilder::new().build(); 20 | downloader.download(&downloads).await; 21 | Ok(()) 22 | } 23 | -------------------------------------------------------------------------------- /examples/with-range-error-handling.rs: -------------------------------------------------------------------------------- 1 | //! Range error handling example. 2 | //! 3 | //! Setup for this example: 4 | //! 5 | //! From the root of the project: 6 | //! ```not_rust 7 | //! mkdir -p examples/miniserve 8 | //! cd examples/miniserve 9 | //! curl -sLO http://212.183.159.230/5MB.zip 10 | //! miniserve . 11 | //! ``` 12 | //! Run with: 13 | //! 14 | //! ```not_rust 15 | //! cargo run -q --example range-error-handling 16 | //! ``` 17 | //! 18 | //! Miniserve is a utility written in rust to serve files over HTTP: 19 | //! https://github.com/svenstaro/miniserve 20 | 21 | use std::path::PathBuf; 22 | use trauma::{download::Download, downloader::DownloaderBuilder, Error}; 23 | 24 | #[tokio::main] 25 | async fn main() -> Result<(), Error> { 26 | let five_mb = "http://localhost:8080/5MB.zip"; 27 | let downloads = vec![Download::try_from(five_mb).unwrap()]; 28 | let downloader = DownloaderBuilder::new() 29 | .directory(PathBuf::from("output")) 30 | .build(); 31 | let summary = downloader.download(&downloads).await; 32 | dbg!(summary); 33 | Ok(()) 34 | } 35 | -------------------------------------------------------------------------------- /examples/with-report.rs: -------------------------------------------------------------------------------- 1 | //! Download files and show the report using comfy-table. 2 | //! 3 | //! Run with: 4 | //! 5 | //! ```not_rust 6 | //! cargo run -q --example with-report 7 | //! ``` 8 | 9 | use color_eyre::{eyre::Report, Result}; 10 | use comfy_table::{Row, Table}; 11 | use std::path::PathBuf; 12 | use trauma::{ 13 | download::{Download, Status, Summary}, 14 | downloader::DownloaderBuilder, 15 | }; 16 | 17 | #[tokio::main] 18 | async fn main() -> Result<(), Report> { 19 | // Setup the application. 20 | color_eyre::install()?; 21 | 22 | // Download the file(s). 23 | let reqwest_rs = "https://github.com/seanmonstar/reqwest/archive/refs/tags/v0.11.9.zip"; 24 | let fake = format!("{}.fake", &reqwest_rs); 25 | let downloads = vec![ 26 | Download::try_from(reqwest_rs).unwrap(), 27 | Download::try_from(reqwest_rs).unwrap(), 28 | Download::try_from(fake.as_str()).unwrap(), 29 | ]; 30 | let downloader = DownloaderBuilder::new() 31 | .directory(PathBuf::from("output")) 32 | .build(); 33 | let summaries = downloader.download(&downloads).await; 34 | 35 | // Display results. 36 | display_summary(&summaries); 37 | 38 | Ok(()) 39 | } 40 | 41 | fn display_summary(summaries: &[Summary]) { 42 | let mut table = Table::new(); 43 | let header = Row::from(vec!["File", "Size", "Status", "Error"]); 44 | table.set_header(header); 45 | summaries.iter().for_each(|s| { 46 | let mut error = String::new(); 47 | let status = match s.status() { 48 | Status::Success => String::from("✅"), 49 | Status::Fail(s) => { 50 | error = s.to_string(); 51 | error.truncate(50); 52 | if error.len() <= 50 { 53 | error.push_str("..."); 54 | } 55 | String::from("❌") 56 | } 57 | Status::NotStarted => String::from("🔜"), 58 | Status::Skipped(s) => { 59 | error = s.to_string(); 60 | String::from("⏭️") 61 | } 62 | }; 63 | table.add_row(vec![ 64 | &s.download().filename, 65 | &s.size().to_string(), 66 | &status, 67 | &error, 68 | ]); 69 | }); 70 | println!("{table}"); 71 | } 72 | -------------------------------------------------------------------------------- /examples/with-resume.rs: -------------------------------------------------------------------------------- 1 | //! Showcases the resume feature. 2 | //! 3 | //! Run with 4 | //! 5 | //! ```not_rust 6 | //! cargo run -q --example with-resume 7 | //! ``` 8 | 9 | use color_eyre::{ 10 | eyre::{eyre, Report}, 11 | Result, 12 | }; 13 | use futures::stream::StreamExt; 14 | use rand::Rng; 15 | use reqwest::{ 16 | header::{ACCEPT_RANGES, RANGE}, 17 | Url, 18 | }; 19 | use std::{fs, path::PathBuf}; 20 | use tokio::{fs::File, io::AsyncWriteExt}; 21 | use tracing::debug; 22 | use trauma::{download::Download, downloader::DownloaderBuilder}; 23 | 24 | #[tokio::main] 25 | async fn main() -> Result<(), Report> { 26 | // Setup the application. 27 | color_eyre::install()?; 28 | 29 | // Setup logging. 30 | tracing_subscriber::fmt::fmt() 31 | .with_env_filter("with_resume=debug,trauma=debug") 32 | .init(); 33 | 34 | // Prepare the download. 35 | let avatar = Url::parse("https://avatars.githubusercontent.com/u/6969134?v=4").unwrap(); 36 | let output = PathBuf::from("output/avatar.jpg"); 37 | fs::create_dir_all(output.parent().unwrap())?; 38 | 39 | // Make sure the server accepts range requests. 40 | let res = reqwest::Client::new() 41 | .head(&avatar.to_string()) 42 | .send() 43 | .await?; 44 | let headers = res.headers(); 45 | let resumable = match headers.get(ACCEPT_RANGES) { 46 | None => false, 47 | Some(x) if x == "none" => false, 48 | Some(_) => true, 49 | }; 50 | tracing::debug!("Is the file resumable: {:?}", &resumable); 51 | 52 | // We must ensure that the download is resumable to prove our point. 53 | assert!(resumable); 54 | 55 | // Request a random amount of data to simulate a previously failed download. 56 | let mut rng = rand::thread_rng(); 57 | let random_bytes: u8 = rng.gen(); 58 | let res = reqwest::Client::new() 59 | .get(&avatar.to_string()) 60 | .header(RANGE, format!("bytes=0-{}", random_bytes)) 61 | .send() 62 | .await?; 63 | 64 | // Retrieve the bits. 65 | let mut stream = res.bytes_stream(); 66 | let mut file = File::create(&output).await?; 67 | while let Some(item) = stream.next().await { 68 | file.write_all_buf(&mut item?).await?; 69 | } 70 | debug!("Retrieved {} bytes.", random_bytes); 71 | 72 | // Download the rest of the bits with the [`Downloader`]. 73 | let dl = Download::new( 74 | &avatar, 75 | output 76 | .file_name() 77 | .and_then(|n| n.to_str()) 78 | .ok_or(eyre!("invalid path terminator"))?, 79 | ); 80 | let downloads = vec![dl]; 81 | 82 | // Hidding the progress bar because of the logging. 83 | let downloader = DownloaderBuilder::hidden() 84 | .directory(output.parent().unwrap().to_path_buf()) 85 | .build(); 86 | downloader.download(&downloads).await; 87 | 88 | Ok(()) 89 | } 90 | -------------------------------------------------------------------------------- /examples/with-style.rs: -------------------------------------------------------------------------------- 1 | //! Changes the style of the downloader. 2 | //! 3 | //! Run with: 4 | //! 5 | //! ```not_rust 6 | //! cargo run -q --example with-style 7 | //! ``` 8 | 9 | use console::style; 10 | use std::path::PathBuf; 11 | use trauma::{ 12 | download::Download, 13 | downloader::{DownloaderBuilder, ProgressBarOpts, StyleOptions}, 14 | Error, 15 | }; 16 | 17 | #[tokio::main] 18 | async fn main() -> Result<(), Error> { 19 | let debian_net_install = 20 | "https://cdimage.debian.org/debian-cd/current/arm64/iso-cd/debian-11.7.0-arm64-netinst.iso"; 21 | let downloads = vec![Download::try_from(debian_net_install).unwrap()]; 22 | let style_opts = StyleOptions::new( 23 | // The main bar uses a predifined template and progression characters set. 24 | ProgressBarOpts::new( 25 | Some(ProgressBarOpts::TEMPLATE_BAR_WITH_POSITION.into()), 26 | Some(ProgressBarOpts::CHARS_FINE.into()), 27 | true, 28 | false, 29 | ), 30 | // The child bar defines a custom template and a custom progression 31 | // character set using unicode block characters. 32 | // Other examples or symbols can easily be found online, for instance at: 33 | // - https://changaco.oy.lc/unicode-progress-bars/ 34 | // - https://emojistock.com/circle-symbols/ 35 | ProgressBarOpts::new( 36 | Some(format!( 37 | "{{bar:40.cyan/blue}} {{percent:>2.magenta}}{} ● {{eta_precise:.blue}}", 38 | style("%").magenta(), 39 | )), 40 | Some("●◕◑◔○".into()), 41 | true, 42 | false, 43 | ), 44 | ); 45 | 46 | // Predefined styles can also be used. 47 | // let mut style_opts = StyleOptions::default(); 48 | // style_opts.set_child(ProgressBarOpts::with_pip_style()); 49 | 50 | let downloader = DownloaderBuilder::new() 51 | .directory(PathBuf::from("output")) 52 | .style_options(style_opts) 53 | .build(); 54 | downloader.download(&downloads).await; 55 | Ok(()) 56 | } 57 | -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | # Meta task running all the linters at once. 2 | lint: lint-md lint-spellcheck 3 | 4 | # Lint markown files. 5 | lint-md: 6 | npx --yes markdownlint-cli2 "**/*.md" "#target" 7 | 8 | # Spell check the source code. 9 | lint-spellcheck: 10 | @cargo spellcheck --version >/dev/null 2>&1|| cargo install --locked cargo-spellcheck 11 | cargo spellcheck check -m 1 12 | 13 | # Meta tasks running all formatters at once. 14 | fmt: fmt-md fmt-just 15 | 16 | # Format the jusfile. 17 | fmt-just: 18 | just --fmt --unstable 19 | 20 | # Format markdown files. 21 | fmt-md: 22 | npx --yes prettier --write --prose-wrap always **/*.md 23 | -------------------------------------------------------------------------------- /src/download.rs: -------------------------------------------------------------------------------- 1 | //! Represents a file to be downloaded. 2 | 3 | use crate::Error; 4 | use reqwest::{ 5 | header::{ACCEPT_RANGES, CONTENT_LENGTH}, 6 | StatusCode, Url, 7 | }; 8 | use reqwest_middleware::ClientWithMiddleware; 9 | use std::convert::TryFrom; 10 | 11 | /// Represents a file to be downloaded. 12 | #[derive(Debug, Clone)] 13 | pub struct Download { 14 | /// URL of the file to download. 15 | pub url: Url, 16 | /// File name used to save the file on disk. 17 | pub filename: String, 18 | } 19 | 20 | impl Download { 21 | /// Creates a new [`Download`]. 22 | /// 23 | /// When using the [`Download::try_from`] method, the file name is 24 | /// automatically extracted from the URL. 25 | /// 26 | /// ## Example 27 | /// 28 | /// The following calls are equivalent, minus some extra URL validations 29 | /// performed by `try_from`: 30 | /// 31 | /// ```no_run 32 | /// # use color_eyre::{eyre::Report, Result}; 33 | /// use trauma::download::Download; 34 | /// use reqwest::Url; 35 | /// 36 | /// # fn main() -> Result<(), Report> { 37 | /// Download::try_from("https://example.com/file-0.1.2.zip")?; 38 | /// Download::new(&Url::parse("https://example.com/file-0.1.2.zip")?, "file-0.1.2.zip"); 39 | /// # Ok(()) 40 | /// # } 41 | /// ``` 42 | pub fn new(url: &Url, filename: &str) -> Self { 43 | Self { 44 | url: url.clone(), 45 | filename: String::from(filename), 46 | } 47 | } 48 | 49 | /// Check whether the download is resumable. 50 | pub async fn is_resumable( 51 | &self, 52 | client: &ClientWithMiddleware, 53 | ) -> Result { 54 | let res = client.head(self.url.clone()).send().await?; 55 | let headers = res.headers(); 56 | match headers.get(ACCEPT_RANGES) { 57 | None => Ok(false), 58 | Some(x) if x == "none" => Ok(false), 59 | Some(_) => Ok(true), 60 | } 61 | } 62 | 63 | /// Retrieve the content_length of the download. 64 | /// 65 | /// Returns None if the "content-length" header is missing or if its value 66 | /// is not a u64. 67 | pub async fn content_length( 68 | &self, 69 | client: &ClientWithMiddleware, 70 | ) -> Result, reqwest_middleware::Error> { 71 | let res = client.head(self.url.clone()).send().await?; 72 | let headers = res.headers(); 73 | match headers.get(CONTENT_LENGTH) { 74 | None => Ok(None), 75 | Some(header_value) => match header_value.to_str() { 76 | Ok(v) => match v.to_string().parse::() { 77 | Ok(v) => Ok(Some(v)), 78 | Err(_) => Ok(None), 79 | }, 80 | Err(_) => Ok(None), 81 | }, 82 | } 83 | } 84 | } 85 | 86 | impl TryFrom<&Url> for Download { 87 | type Error = crate::Error; 88 | 89 | fn try_from(value: &Url) -> Result { 90 | value 91 | .path_segments() 92 | .ok_or_else(|| { 93 | Error::InvalidUrl(format!( 94 | "the url \"{}\" does not contain a valid path", 95 | value 96 | )) 97 | })? 98 | .last() 99 | .map(String::from) 100 | .map(|filename| Download { 101 | url: value.clone(), 102 | filename: form_urlencoded::parse(filename.as_bytes()) 103 | .map(|(key, val)| [key, val].concat()) 104 | .collect(), 105 | }) 106 | .ok_or_else(|| { 107 | Error::InvalidUrl(format!("the url \"{}\" does not contain a filename", value)) 108 | }) 109 | } 110 | } 111 | 112 | impl TryFrom<&str> for Download { 113 | type Error = crate::Error; 114 | 115 | fn try_from(value: &str) -> Result { 116 | Url::parse(value) 117 | .map_err(|e| { 118 | Error::InvalidUrl(format!("the url \"{}\" cannot be parsed: {}", value, e)) 119 | }) 120 | .and_then(|u| Download::try_from(&u)) 121 | } 122 | } 123 | 124 | #[derive(Debug, Clone, PartialEq, Eq)] 125 | pub enum Status { 126 | Fail(String), 127 | NotStarted, 128 | Skipped(String), 129 | Success, 130 | } 131 | /// Represents a [`Download`] summary. 132 | #[derive(Debug, Clone)] 133 | pub struct Summary { 134 | /// Downloaded items. 135 | download: Download, 136 | /// HTTP status code. 137 | statuscode: StatusCode, 138 | /// Download size in bytes. 139 | size: u64, 140 | /// Status. 141 | status: Status, 142 | /// Resumable. 143 | resumable: bool, 144 | } 145 | 146 | impl Summary { 147 | /// Create a new [`Download`] [`Summary`]. 148 | pub fn new(download: Download, statuscode: StatusCode, size: u64, resumable: bool) -> Self { 149 | Self { 150 | download, 151 | statuscode, 152 | size, 153 | status: Status::NotStarted, 154 | resumable, 155 | } 156 | } 157 | 158 | /// Attach a status to a [`Download`] [`Summary`]. 159 | pub fn with_status(self, status: Status) -> Self { 160 | Self { status, ..self } 161 | } 162 | 163 | /// Get the summary's status. 164 | pub fn statuscode(&self) -> StatusCode { 165 | self.statuscode 166 | } 167 | 168 | /// Get the summary's size. 169 | pub fn size(&self) -> u64 { 170 | self.size 171 | } 172 | 173 | /// Get a reference to the summary's download. 174 | pub fn download(&self) -> &Download { 175 | &self.download 176 | } 177 | 178 | /// Get a reference to the summary's status. 179 | pub fn status(&self) -> &Status { 180 | &self.status 181 | } 182 | 183 | pub fn fail(self, msg: impl std::fmt::Display) -> Self { 184 | Self { 185 | status: Status::Fail(format!("{}", msg)), 186 | ..self 187 | } 188 | } 189 | 190 | /// Set the summary's resumable. 191 | pub fn set_resumable(&mut self, resumable: bool) { 192 | self.resumable = resumable; 193 | } 194 | 195 | /// Get the summary's resumable. 196 | #[must_use] 197 | pub fn resumable(&self) -> bool { 198 | self.resumable 199 | } 200 | } 201 | 202 | #[cfg(test)] 203 | mod test { 204 | use super::*; 205 | 206 | const DOMAIN: &str = "http://domain.com/file.zip"; 207 | 208 | #[test] 209 | fn test_try_from_url() { 210 | let u = Url::parse(DOMAIN).unwrap(); 211 | let d = Download::try_from(&u).unwrap(); 212 | assert_eq!(d.filename, "file.zip") 213 | } 214 | 215 | #[test] 216 | fn test_try_from_string() { 217 | let d = Download::try_from(DOMAIN).unwrap(); 218 | assert_eq!(d.filename, "file.zip") 219 | } 220 | } 221 | -------------------------------------------------------------------------------- /src/downloader.rs: -------------------------------------------------------------------------------- 1 | //! Represents the download controller. 2 | 3 | use crate::download::{Download, Status, Summary}; 4 | use futures::stream::{self, StreamExt}; 5 | use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle}; 6 | use reqwest::{ 7 | header::{HeaderMap, HeaderValue, IntoHeaderName, RANGE}, 8 | StatusCode, 9 | }; 10 | use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; 11 | use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; 12 | use reqwest_tracing::TracingMiddleware; 13 | use std::{fs, path::PathBuf, sync::Arc}; 14 | use tokio::{fs::OpenOptions, io::AsyncWriteExt}; 15 | use tracing::debug; 16 | 17 | pub struct TimeTrace; 18 | 19 | /// Represents the download controller. 20 | /// 21 | /// A downloader can be created via its builder: 22 | /// 23 | /// ```rust 24 | /// # fn main() { 25 | /// use trauma::downloader::DownloaderBuilder; 26 | /// 27 | /// let d = DownloaderBuilder::new().build(); 28 | /// # } 29 | /// ``` 30 | #[derive(Debug, Clone)] 31 | pub struct Downloader { 32 | /// Directory where to store the downloaded files. 33 | directory: PathBuf, 34 | /// Number of retries per downloaded file. 35 | retries: u32, 36 | /// Number of maximum concurrent downloads. 37 | concurrent_downloads: usize, 38 | /// Downloader style options. 39 | style_options: StyleOptions, 40 | /// Resume the download if necessary and possible. 41 | resumable: bool, 42 | /// Custom HTTP headers. 43 | headers: Option, 44 | } 45 | 46 | impl Downloader { 47 | const DEFAULT_RETRIES: u32 = 3; 48 | const DEFAULT_CONCURRENT_DOWNLOADS: usize = 32; 49 | 50 | /// Starts the downloads. 51 | pub async fn download(&self, downloads: &[Download]) -> Vec { 52 | self.download_inner(downloads, None).await 53 | } 54 | 55 | /// Starts the downloads with proxy. 56 | pub async fn download_with_proxy( 57 | &self, 58 | downloads: &[Download], 59 | proxy: reqwest::Proxy, 60 | ) -> Vec { 61 | self.download_inner(downloads, Some(proxy)).await 62 | } 63 | 64 | /// Starts the downloads. 65 | pub async fn download_inner( 66 | &self, 67 | downloads: &[Download], 68 | proxy: Option, 69 | ) -> Vec { 70 | // Prepare the HTTP client. 71 | let retry_policy = ExponentialBackoff::builder().build_with_max_retries(self.retries); 72 | 73 | let mut inner_client_builder = reqwest::Client::builder(); 74 | if let Some(proxy) = proxy { 75 | inner_client_builder = inner_client_builder.proxy(proxy); 76 | } 77 | if let Some(headers) = &self.headers { 78 | inner_client_builder = inner_client_builder.default_headers(headers.clone()); 79 | } 80 | 81 | let inner_client = inner_client_builder.build().unwrap(); 82 | 83 | let client = ClientBuilder::new(inner_client) 84 | // Trace HTTP requests. See the tracing crate to make use of these traces. 85 | .with(TracingMiddleware::default()) 86 | // Retry failed requests. 87 | .with(RetryTransientMiddleware::new_with_policy(retry_policy)) 88 | .build(); 89 | 90 | // Prepare the progress bar. 91 | let multi = match self.style_options.clone().is_enabled() { 92 | true => Arc::new(MultiProgress::new()), 93 | false => Arc::new(MultiProgress::with_draw_target(ProgressDrawTarget::hidden())), 94 | }; 95 | let main = Arc::new( 96 | multi.add( 97 | self.style_options 98 | .main 99 | .clone() 100 | .to_progress_bar(downloads.len() as u64), 101 | ), 102 | ); 103 | main.tick(); 104 | 105 | // Download the files asynchronously. 106 | let summaries = stream::iter(downloads) 107 | .map(|d| self.fetch(&client, d, multi.clone(), main.clone())) 108 | .buffer_unordered(self.concurrent_downloads) 109 | .collect::>() 110 | .await; 111 | 112 | // Finish the progress bar. 113 | if self.style_options.main.clear { 114 | main.finish_and_clear(); 115 | } else { 116 | main.finish(); 117 | } 118 | 119 | // Return the download summaries. 120 | summaries 121 | } 122 | 123 | /// Fetches the files and write them to disk. 124 | async fn fetch( 125 | &self, 126 | client: &ClientWithMiddleware, 127 | download: &Download, 128 | multi: Arc, 129 | main: Arc, 130 | ) -> Summary { 131 | // Create a download summary. 132 | let mut size_on_disk: u64 = 0; 133 | let mut can_resume = false; 134 | let output = self.directory.join(&download.filename); 135 | let mut summary = Summary::new( 136 | download.clone(), 137 | StatusCode::BAD_REQUEST, 138 | size_on_disk, 139 | can_resume, 140 | ); 141 | let mut content_length: Option = None; 142 | 143 | // If resumable is turned on... 144 | if self.resumable { 145 | can_resume = match download.is_resumable(client).await { 146 | Ok(r) => r, 147 | Err(e) => { 148 | return summary.fail(e); 149 | } 150 | }; 151 | 152 | // Check if there is a file on disk already. 153 | if can_resume && output.exists() { 154 | debug!("A file with the same name already exists at the destination."); 155 | // If so, check file length to know where to restart the download from. 156 | size_on_disk = match output.metadata() { 157 | Ok(m) => m.len(), 158 | Err(e) => { 159 | return summary.fail(e); 160 | } 161 | }; 162 | 163 | // Retrieve the download size from the header if possible. 164 | content_length = match download.content_length(client).await { 165 | Ok(l) => l, 166 | Err(e) => { 167 | return summary.fail(e); 168 | } 169 | }; 170 | } 171 | 172 | // Update the summary accordingly. 173 | summary.set_resumable(can_resume); 174 | } 175 | 176 | // If resumable is turned on... 177 | // Request the file. 178 | debug!("Fetching {}", &download.url); 179 | let mut req = client.get(download.url.clone()); 180 | if self.resumable && can_resume { 181 | req = req.header(RANGE, format!("bytes={}-", size_on_disk)); 182 | } 183 | 184 | // Add extra headers if needed. 185 | if let Some(ref h) = self.headers { 186 | req = req.headers(h.to_owned()); 187 | } 188 | 189 | // Ensure there was no error while sending the request. 190 | let res = match req.send().await { 191 | Ok(res) => res, 192 | Err(e) => { 193 | return summary.fail(e); 194 | } 195 | }; 196 | 197 | // Check wether or not we need to download the file. 198 | if let Some(content_length) = content_length { 199 | if content_length == size_on_disk { 200 | return summary.with_status(Status::Skipped( 201 | "the file was already fully downloaded".into(), 202 | )); 203 | } 204 | } 205 | 206 | // Check the status for errors. 207 | match res.error_for_status_ref() { 208 | Ok(_res) => (), 209 | Err(e) => return summary.fail(e), 210 | }; 211 | 212 | // Update the summary with the collected details. 213 | let size = content_length.unwrap_or_default() + size_on_disk; 214 | let status = res.status(); 215 | summary = Summary::new(download.clone(), status, size, can_resume); 216 | 217 | // If there is nothing else to download for this file, we can return. 218 | if size_on_disk > 0 && size == size_on_disk { 219 | return summary.with_status(Status::Skipped( 220 | "the file was already fully downloaded".into(), 221 | )); 222 | } 223 | 224 | // Create the progress bar. 225 | // If the download is being resumed, the progress bar position is 226 | // updated to start where the download stopped before. 227 | let pb = multi.add( 228 | self.style_options 229 | .child 230 | .clone() 231 | .to_progress_bar(size) 232 | .with_position(size_on_disk), 233 | ); 234 | 235 | // Prepare the destination directory/file. 236 | let output_dir = output.parent().unwrap_or(&output); 237 | debug!("Creating destination directory {:?}", output_dir); 238 | match fs::create_dir_all(output_dir) { 239 | Ok(_res) => (), 240 | Err(e) => { 241 | return summary.fail(e); 242 | } 243 | }; 244 | 245 | debug!("Creating destination file {:?}", &output); 246 | let mut file = match OpenOptions::new() 247 | .create(true) 248 | .write(true) 249 | .append(can_resume) 250 | .open(output) 251 | .await 252 | { 253 | Ok(file) => file, 254 | Err(e) => { 255 | return summary.fail(e); 256 | } 257 | }; 258 | 259 | let mut final_size = size_on_disk; 260 | 261 | // Download the file chunk by chunk. 262 | debug!("Retrieving chunks..."); 263 | let mut stream = res.bytes_stream(); 264 | while let Some(item) = stream.next().await { 265 | // Retrieve chunk. 266 | let mut chunk = match item { 267 | Ok(chunk) => chunk, 268 | Err(e) => { 269 | return summary.fail(e); 270 | } 271 | }; 272 | let chunk_size = chunk.len() as u64; 273 | final_size += chunk_size; 274 | pb.inc(chunk_size); 275 | 276 | // Write the chunk to disk. 277 | match file.write_all_buf(&mut chunk).await { 278 | Ok(_res) => (), 279 | Err(e) => { 280 | return summary.fail(e); 281 | } 282 | }; 283 | } 284 | 285 | // Finish the progress bar once complete, and optionally remove it. 286 | if self.style_options.child.clear { 287 | pb.finish_and_clear(); 288 | } else { 289 | pb.finish(); 290 | } 291 | 292 | // Advance the main progress bar. 293 | main.inc(1); 294 | 295 | // Create a new summary with the real download size 296 | let summary = Summary::new(download.clone(), status, final_size, can_resume); 297 | // Return the download summary. 298 | summary.with_status(Status::Success) 299 | } 300 | } 301 | 302 | /// A builder used to create a [`Downloader`]. 303 | /// 304 | /// ```rust 305 | /// # fn main() { 306 | /// use trauma::downloader::DownloaderBuilder; 307 | /// 308 | /// let d = DownloaderBuilder::new().retries(5).directory("downloads".into()).build(); 309 | /// # } 310 | /// ``` 311 | pub struct DownloaderBuilder(Downloader); 312 | 313 | impl DownloaderBuilder { 314 | /// Creates a builder with the default options. 315 | pub fn new() -> Self { 316 | DownloaderBuilder::default() 317 | } 318 | 319 | /// Convenience function to hide the progress bars. 320 | pub fn hidden() -> Self { 321 | let d = DownloaderBuilder::default(); 322 | d.style_options(StyleOptions::new( 323 | ProgressBarOpts::hidden(), 324 | ProgressBarOpts::hidden(), 325 | )) 326 | } 327 | 328 | /// Sets the directory where to store the [`Download`]s. 329 | pub fn directory(mut self, directory: PathBuf) -> Self { 330 | self.0.directory = directory; 331 | self 332 | } 333 | 334 | /// Set the number of retries per [`Download`]. 335 | pub fn retries(mut self, retries: u32) -> Self { 336 | self.0.retries = retries; 337 | self 338 | } 339 | 340 | /// Set the number of concurrent [`Download`]s. 341 | pub fn concurrent_downloads(mut self, concurrent_downloads: usize) -> Self { 342 | self.0.concurrent_downloads = concurrent_downloads; 343 | self 344 | } 345 | 346 | /// Set the downloader style options. 347 | pub fn style_options(mut self, style_options: StyleOptions) -> Self { 348 | self.0.style_options = style_options; 349 | self 350 | } 351 | 352 | fn new_header(&self) -> HeaderMap { 353 | match self.0.headers { 354 | Some(ref h) => h.to_owned(), 355 | _ => HeaderMap::new(), 356 | } 357 | } 358 | 359 | /// Add the http headers. 360 | /// 361 | /// You need to pass in a `HeaderMap`, not a `HeaderName`. 362 | /// `HeaderMap` is a set of http headers. 363 | /// 364 | /// You can call `.headers()` multiple times and all `HeaderMap` will be merged into a single one. 365 | /// 366 | /// # Example 367 | /// 368 | /// ``` 369 | /// use reqwest::header::{self, HeaderValue, HeaderMap}; 370 | /// use trauma::downloader::DownloaderBuilder; 371 | /// 372 | /// let ua = HeaderValue::from_str("curl/7.87").expect("Invalid UA"); 373 | /// 374 | /// let builder = DownloaderBuilder::new() 375 | /// .headers(HeaderMap::from_iter([(header::USER_AGENT, ua)])) 376 | /// .build(); 377 | /// ``` 378 | /// 379 | /// See also [`header()`]. 380 | /// 381 | /// [`header()`]: DownloaderBuilder::header 382 | pub fn headers(mut self, headers: HeaderMap) -> Self { 383 | let mut new = self.new_header(); 384 | new.extend(headers); 385 | 386 | self.0.headers = Some(new); 387 | self 388 | } 389 | 390 | /// Add the http header 391 | /// 392 | /// # Example 393 | /// 394 | /// You can use the `.header()` chain to add multiple headers 395 | /// 396 | /// ``` 397 | /// use reqwest::header::{self, HeaderValue}; 398 | /// use trauma::downloader::DownloaderBuilder; 399 | /// 400 | /// const FIREFOX_UA: &str = 401 | /// "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/109.0"; 402 | /// 403 | /// let ua = HeaderValue::from_str(FIREFOX_UA).expect("Invalid UA"); 404 | /// let auth = HeaderValue::from_str("Basic aGk6MTIzNDU2Cg==").expect("Invalid auth"); 405 | /// 406 | /// let builder = DownloaderBuilder::new() 407 | /// .header(header::USER_AGENT, ua) 408 | /// .header(header::AUTHORIZATION, auth) 409 | /// .build(); 410 | /// ``` 411 | /// 412 | /// If you need to pass in a `HeaderMap`, instead of calling `.header()` multiple times. 413 | /// See also [`headers()`]. 414 | /// 415 | /// [`headers()`]: DownloaderBuilder::headers 416 | pub fn header(mut self, name: K, value: HeaderValue) -> Self { 417 | let mut new = self.new_header(); 418 | 419 | new.insert(name, value); 420 | 421 | self.0.headers = Some(new); 422 | self 423 | } 424 | 425 | /// Create the [`Downloader`] with the specified options. 426 | pub fn build(self) -> Downloader { 427 | Downloader { 428 | directory: self.0.directory, 429 | retries: self.0.retries, 430 | concurrent_downloads: self.0.concurrent_downloads, 431 | style_options: self.0.style_options, 432 | resumable: self.0.resumable, 433 | headers: self.0.headers, 434 | } 435 | } 436 | } 437 | 438 | impl Default for DownloaderBuilder { 439 | fn default() -> Self { 440 | Self(Downloader { 441 | directory: std::env::current_dir().unwrap_or_default(), 442 | retries: Downloader::DEFAULT_RETRIES, 443 | concurrent_downloads: Downloader::DEFAULT_CONCURRENT_DOWNLOADS, 444 | style_options: StyleOptions::default(), 445 | resumable: true, 446 | headers: None, 447 | }) 448 | } 449 | } 450 | 451 | /// Define the [`Downloader`] options. 452 | /// 453 | /// By default, the main progress bar will stay on the screen upon completion, 454 | /// but the child ones will be cleared once complete. 455 | #[derive(Debug, Clone)] 456 | pub struct StyleOptions { 457 | /// Style options for the main progress bar. 458 | main: ProgressBarOpts, 459 | /// Style options for the child progress bar(s). 460 | child: ProgressBarOpts, 461 | } 462 | 463 | impl Default for StyleOptions { 464 | fn default() -> Self { 465 | Self { 466 | main: ProgressBarOpts { 467 | template: Some(ProgressBarOpts::TEMPLATE_BAR_WITH_POSITION.into()), 468 | progress_chars: Some(ProgressBarOpts::CHARS_FINE.into()), 469 | enabled: true, 470 | clear: false, 471 | }, 472 | child: ProgressBarOpts::with_pip_style(), 473 | } 474 | } 475 | } 476 | 477 | impl StyleOptions { 478 | /// Create new [`Downloader`] [`StyleOptions`]. 479 | pub fn new(main: ProgressBarOpts, child: ProgressBarOpts) -> Self { 480 | Self { main, child } 481 | } 482 | 483 | /// Set the options for the main progress bar. 484 | pub fn set_main(&mut self, main: ProgressBarOpts) { 485 | self.main = main; 486 | } 487 | 488 | /// Set the options for the child progress bar. 489 | pub fn set_child(&mut self, child: ProgressBarOpts) { 490 | self.child = child; 491 | } 492 | 493 | /// Return `false` if neither the main nor the child bar is enabled. 494 | pub fn is_enabled(self) -> bool { 495 | self.main.enabled || self.child.enabled 496 | } 497 | } 498 | 499 | /// Define the options for a progress bar. 500 | #[derive(Debug, Clone)] 501 | pub struct ProgressBarOpts { 502 | /// Progress bar template string. 503 | template: Option, 504 | /// Progression characters set. 505 | /// 506 | /// There must be at least 3 characters for the following states: 507 | /// "filled", "current", and "to do". 508 | progress_chars: Option, 509 | /// Enable or disable the progress bar. 510 | enabled: bool, 511 | /// Clear the progress bar once completed. 512 | clear: bool, 513 | } 514 | 515 | impl Default for ProgressBarOpts { 516 | fn default() -> Self { 517 | Self { 518 | template: None, 519 | progress_chars: None, 520 | enabled: true, 521 | clear: true, 522 | } 523 | } 524 | } 525 | 526 | impl ProgressBarOpts { 527 | /// Template representing the bar and its position. 528 | /// 529 | ///`███████████████████████████████████████ 11/12 (99%) eta 00:00:02` 530 | pub const TEMPLATE_BAR_WITH_POSITION: &'static str = 531 | "{bar:40.blue} {pos:>}/{len} ({percent}%) eta {eta_precise:.blue}"; 532 | /// Template which looks like the Python package installer pip. 533 | /// 534 | /// `━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 211.23 KiB/211.23 KiB 1008.31 KiB/s eta 0s` 535 | pub const TEMPLATE_PIP: &'static str = 536 | "{bar:40.green/black} {bytes:>11.green}/{total_bytes:<11.green} {bytes_per_sec:>13.red} eta {eta:.blue}"; 537 | /// Use increasing quarter blocks as progress characters: `"█▛▌▖ "`. 538 | pub const CHARS_BLOCKY: &'static str = "█▛▌▖ "; 539 | /// Use fade-in blocks as progress characters: `"█▓▒░ "`. 540 | pub const CHARS_FADE_IN: &'static str = "█▓▒░ "; 541 | /// Use fine blocks as progress characters: `"█▉▊▋▌▍▎▏ "`. 542 | pub const CHARS_FINE: &'static str = "█▉▊▋▌▍▎▏ "; 543 | /// Use a line as progress characters: `"━╾─"`. 544 | pub const CHARS_LINE: &'static str = "━╾╴─"; 545 | /// Use rough blocks as progress characters: `"█ "`. 546 | pub const CHARS_ROUGH: &'static str = "█ "; 547 | /// Use increasing height blocks as progress characters: `"█▇▆▅▄▃▂▁ "`. 548 | pub const CHARS_VERTICAL: &'static str = "█▇▆▅▄▃▂▁ "; 549 | 550 | /// Create a new [`ProgressBarOpts`]. 551 | pub fn new( 552 | template: Option, 553 | progress_chars: Option, 554 | enabled: bool, 555 | clear: bool, 556 | ) -> Self { 557 | Self { 558 | template, 559 | progress_chars, 560 | enabled, 561 | clear, 562 | } 563 | } 564 | 565 | /// Create a [`ProgressStyle`] based on the provided options. 566 | pub fn to_progress_style(self) -> ProgressStyle { 567 | let mut style = ProgressStyle::default_bar(); 568 | if let Some(template) = self.template { 569 | style = style.template(&template).unwrap(); 570 | } 571 | if let Some(progress_chars) = self.progress_chars { 572 | style = style.progress_chars(&progress_chars); 573 | } 574 | style 575 | } 576 | 577 | /// Create a [`ProgressBar`] based on the provided options. 578 | pub fn to_progress_bar(self, len: u64) -> ProgressBar { 579 | // Return a hidden Progress bar if we disabled it. 580 | if !self.enabled { 581 | return ProgressBar::hidden(); 582 | } 583 | 584 | // Otherwise returns a ProgressBar with the style. 585 | let style = self.to_progress_style(); 586 | ProgressBar::new(len).with_style(style) 587 | } 588 | 589 | /// Create a new [`ProgressBarOpts`] which looks like Python pip. 590 | pub fn with_pip_style() -> Self { 591 | Self { 592 | template: Some(ProgressBarOpts::TEMPLATE_PIP.into()), 593 | progress_chars: Some(ProgressBarOpts::CHARS_LINE.into()), 594 | enabled: true, 595 | clear: true, 596 | } 597 | } 598 | 599 | /// Set to `true` to clear the progress bar upon completion. 600 | pub fn set_clear(&mut self, clear: bool) { 601 | self.clear = clear; 602 | } 603 | 604 | /// Create a new [`ProgressBarOpts`] which hides the progress bars. 605 | pub fn hidden() -> Self { 606 | Self { 607 | enabled: false, 608 | ..ProgressBarOpts::default() 609 | } 610 | } 611 | } 612 | 613 | #[cfg(test)] 614 | mod test { 615 | use super::*; 616 | 617 | #[test] 618 | fn test_builder_defaults() { 619 | let d = DownloaderBuilder::new().build(); 620 | assert_eq!(d.retries, Downloader::DEFAULT_RETRIES); 621 | assert_eq!( 622 | d.concurrent_downloads, 623 | Downloader::DEFAULT_CONCURRENT_DOWNLOADS 624 | ); 625 | } 626 | } 627 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Trauma is crate aiming at providing a simple way to download files 2 | //! asynchronously via HTTP(S). 3 | 4 | pub mod download; 5 | pub mod downloader; 6 | 7 | use std::io; 8 | use thiserror::Error; 9 | 10 | /// Errors that can happen when using Trauma. 11 | #[derive(Error, Debug)] 12 | pub enum Error { 13 | /// Error from an underlying system. 14 | #[error("Internal error: {0}")] 15 | Internal(String), 16 | /// Error from the underlying URL parser or the expected URL format. 17 | #[error("Invalid URL: {0}")] 18 | InvalidUrl(String), 19 | /// I/O Error. 20 | #[error("I/O error")] 21 | IOError { 22 | #[from] 23 | source: io::Error, 24 | }, 25 | /// Error from the Reqwest library. 26 | #[error("Reqwest Error")] 27 | Reqwest { 28 | #[from] 29 | source: reqwest::Error, 30 | }, 31 | } 32 | --------------------------------------------------------------------------------