├── .github └── workflows │ ├── ci.yml │ └── release.yml ├── .gitignore ├── CHANGELOG.md ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── ci ├── before_deploy.bash ├── before_deploy.ps1 ├── before_install.bash ├── benchmark.bash ├── benchmark.nginx.conf └── script.bash ├── rustfmt.toml └── src ├── app ├── core │ ├── bt.rs │ ├── http.rs │ ├── m3u8 │ │ ├── common.rs │ │ ├── ffmpeg.rs │ │ ├── m3u8.rs │ │ └── mod.rs │ └── mod.rs ├── mod.rs ├── receive │ ├── http_receiver.rs │ ├── m3u8_receiver.rs │ └── mod.rs ├── record │ ├── bytearray_recorder.rs │ ├── common.rs │ ├── mod.rs │ └── range_recorder.rs ├── show │ ├── bt_show.rs │ ├── common.rs │ ├── http_show.rs │ ├── m3u8_show.rs │ └── mod.rs └── status │ ├── mod.rs │ └── rate_status.rs ├── arguments ├── clap_cli.rs ├── cmd_args.rs └── mod.rs ├── common ├── buf.rs ├── bytes │ ├── bytes.rs │ ├── bytes_type.rs │ └── mod.rs ├── character.rs ├── colors.rs ├── crypto.rs ├── errors.rs ├── file.rs ├── liberal.rs ├── list.rs ├── mod.rs ├── net │ ├── mod.rs │ └── net.rs ├── range.rs ├── size.rs ├── tasks.rs ├── terminal.rs ├── time.rs └── uri.rs ├── config └── mod.rs ├── features ├── args.rs ├── mod.rs ├── running.rs └── stack.rs ├── lib.rs └── main.rs /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Build & Test 2 | 3 | on: [push, pull_request] 4 | 5 | env: 6 | CARGO_TERM_COLOR: always 7 | 8 | jobs: 9 | test: 10 | name: Test 11 | strategy: 12 | matrix: 13 | build: 14 | - linux 15 | - windows 16 | - mac 17 | - mac-aarch64 18 | include: 19 | - build: linux 20 | os: ubuntu-latest 21 | rust: "stable" 22 | - build: windows 23 | os: windows-latest 24 | rust: "stable" 25 | - build: mac 26 | os: macos-latest 27 | rust: "stable" 28 | - build: mac-aarch64 29 | os: macos-latest 30 | rust: "stable" 31 | target: aarch64-apple-darwin 32 | 33 | runs-on: ${{ matrix.os }} 34 | 35 | steps: 36 | - uses: actions/checkout@v3 37 | 38 | - if: contains(matrix.target, 'x86') 39 | uses: ilammy/setup-nasm@v1 40 | 41 | - name: Install Rust 42 | uses: actions-rs/toolchain@v1 43 | with: 44 | toolchain: ${{ matrix.rust }} 45 | profile: minimal 46 | override: true 47 | 48 | - name: Install aarch64 toolchain 49 | if: matrix.target == 'aarch64-apple-darwin' 50 | run: rustup target add aarch64-apple-darwin 51 | 52 | - uses: Swatinem/rust-cache@v1 53 | 54 | - name: Install dependencies for aws-lc-rs 55 | shell: bash 56 | run: | 57 | if [ "${{ matrix.os }}" = "windows-latest" ]; then 58 | echo 'LIBCLANG_PATH="C:\Program Files\LLVM\bin"' >> $GITHUB_ENV 59 | echo 'AWS_LC_SYS_PREBUILT_NASM=1' >> $GITHUB_ENV 60 | cargo install --force --locked bindgen-cli 61 | elif [ "${{ matrix.os }}" = "ubuntu-latest" ]; then 62 | sudo apt-get install -y build-essential cmake golang libclang1 libclang-dev 63 | cargo install --force --locked bindgen-cli 64 | fi 65 | 66 | - name: Build 67 | if: matrix.target != 'aarch64-apple-darwin' 68 | run: cargo build --verbose 69 | 70 | - name: Build aarch64 71 | if: matrix.target == 'aarch64-apple-darwin' 72 | run: cargo build --target=aarch64-apple-darwin --verbose 73 | 74 | - name: Tests 75 | if: matrix.os == 'ubuntu-latest' 76 | run: | 77 | cargo test --verbose 78 | bash ci/script.bash 79 | 80 | - name: Benchmark 81 | if: matrix.os == 'ubuntu-latest' 82 | run: | 83 | bash ci/benchmark.bash 84 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | # Only do the release on x.y.z tags. 4 | on: 5 | push: 6 | tags: 7 | - "[0-9]+.[0-9]+.[0-9]+" 8 | 9 | # We need this to be able to create releases. 10 | permissions: 11 | contents: write 12 | 13 | jobs: 14 | # The create-release job runs purely to initialize the GitHub release itself, 15 | # and names the release after the `x.y.z` tag that was pushed. It's separate 16 | # from building the release so that we only create the release once. 17 | create-release: 18 | name: create-release 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@v4 22 | - name: Get the release version from the tag 23 | if: env.VERSION == '' 24 | run: echo "VERSION=${{ github.ref_name }}" >> $GITHUB_ENV 25 | - name: Show the version 26 | run: | 27 | echo "version is: $VERSION" 28 | - name: Check that tag version and Cargo.toml version are the same 29 | shell: bash 30 | run: | 31 | if ! grep -q "version = \"$VERSION\"" Cargo.toml; then 32 | echo "version does not match Cargo.toml" >&2 33 | exit 1 34 | fi 35 | - name: Create GitHub release 36 | env: 37 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 38 | run: gh release create $VERSION --draft --verify-tag --title $VERSION 39 | outputs: 40 | version: ${{ env.VERSION }} 41 | 42 | build-release: 43 | name: build-release 44 | needs: ["create-release"] 45 | runs-on: ${{ matrix.os }} 46 | env: 47 | # For some builds, we use cross to test on 32-bit and big-endian 48 | # systems. 49 | CARGO: cargo 50 | # When CARGO is set to CROSS, this is set to `--target matrix.target`. 51 | TARGET_FLAGS: 52 | # When CARGO is set to CROSS, TARGET_DIR includes matrix.target. 53 | TARGET_DIR: ./target 54 | # Bump this as appropriate. We pin to a version to make sure CI 55 | # continues to work as cross releases in the past have broken things 56 | # in subtle ways. 57 | CROSS_VERSION: v0.2.5 58 | # Emit backtraces on panics. 59 | RUST_BACKTRACE: 1 60 | strategy: 61 | fail-fast: false 62 | matrix: 63 | include: 64 | - build: linux-x86_64-musl 65 | os: ubuntu-latest 66 | rust: stable 67 | target: x86_64-unknown-linux-musl 68 | strip: x86_64-linux-musl-strip 69 | use-cross: true 70 | 71 | - build: linux-x86_64-gnu 72 | os: ubuntu-latest 73 | rust: stable 74 | target: x86_64-unknown-linux-gnu 75 | strip: strip 76 | use-cross: true 77 | 78 | - build: stable-aarch64-musl 79 | os: ubuntu-latest 80 | rust: stable 81 | target: aarch64-unknown-linux-musl 82 | strip: aarch64-linux-musl-strip 83 | use-cross: true 84 | qemu: qemu-aarch64 85 | 86 | - build: stable-aarch64-gnu 87 | os: ubuntu-latest 88 | rust: stable 89 | target: aarch64-unknown-linux-gnu 90 | strip: aarch64-linux-gnu-strip 91 | use-cross: true 92 | qemu: qemu-aarch64 93 | 94 | - build: macos-x86_64 95 | os: macos-latest 96 | rust: stable 97 | target: x86_64-apple-darwin 98 | 99 | - build: macos-aarch64 100 | os: macos-latest 101 | rust: stable 102 | use-cross: true 103 | target: aarch64-apple-darwin 104 | 105 | - build: win-msvc 106 | os: windows-latest 107 | rust: stable 108 | target: x86_64-pc-windows-msvc 109 | 110 | - build: win-gnu 111 | os: windows-latest 112 | rust: stable-x86_64-gnu 113 | target: x86_64-pc-windows-gnu 114 | 115 | - build: win-msvc-aarch64 116 | os: windows-latest 117 | rust: stable 118 | target: aarch64-pc-windows-msvc 119 | use-cross: true 120 | steps: 121 | - name: Checkout repository 122 | uses: actions/checkout@v4 123 | 124 | - if: contains(matrix.target, 'x86') 125 | uses: ilammy/setup-nasm@v1 126 | 127 | - name: Install packages (Ubuntu) 128 | if: matrix.os == 'ubuntu-latest' 129 | shell: bash 130 | run: | 131 | ci/before_install.bash 132 | 133 | - name: Install Rust 134 | uses: dtolnay/rust-toolchain@master 135 | with: 136 | toolchain: ${{ matrix.rust }} 137 | target: ${{ matrix.target }} 138 | 139 | - name: Use Cross 140 | if: matrix.use-cross 141 | uses: taiki-e/install-action@v2 142 | with: 143 | tool: cross 144 | 145 | - name: Set CARGO 146 | if: matrix.use-cross 147 | shell: bash 148 | run: | 149 | echo "CARGO=cross" >> $GITHUB_ENV 150 | 151 | - name: Set target variables 152 | shell: bash 153 | run: | 154 | echo "TARGET_FLAGS=--target ${{ matrix.target }}" >> $GITHUB_ENV 155 | echo "TARGET_DIR=./target/${{ matrix.target }}" >> $GITHUB_ENV 156 | 157 | - name: Show command used for Cargo 158 | shell: bash 159 | run: | 160 | echo "cargo command is: ${{ env.CARGO }}" 161 | echo "target flag is: ${{ env.TARGET_FLAGS }}" 162 | echo "target dir is: ${{ env.TARGET_DIR }}" 163 | 164 | - name: Install dependencies for aws-lc-rs 165 | shell: bash 166 | run: | 167 | if [ "${{ matrix.os }}" = "windows-latest" ]; then 168 | echo 'LIBCLANG_PATH="C:\Program Files\LLVM\bin"' >> $GITHUB_ENV 169 | echo 'AWS_LC_SYS_PREBUILT_NASM=1' >> $GITHUB_ENV 170 | cargo install --force --locked bindgen-cli 171 | elif [ "${{ matrix.os }}" = "ubuntu-latest" ]; then 172 | sudo apt-get install -y build-essential cmake golang libclang1 libclang-dev 173 | cargo install --force --locked bindgen-cli 174 | fi 175 | 176 | - name: Build release binary 177 | shell: bash 178 | run: | 179 | ${{ env.CARGO }} build --verbose --release ${{ env.TARGET_FLAGS }} 180 | if [ "${{ matrix.os }}" = "windows-latest" ]; then 181 | bin="target/${{ matrix.target }}/release/ag.exe" 182 | else 183 | bin="target/${{ matrix.target }}/release/ag" 184 | fi 185 | echo "BIN=$bin" >> $GITHUB_ENV 186 | 187 | - name: Strip release binary (macos) 188 | if: matrix.os == 'macos-latest' 189 | shell: bash 190 | run: strip "$BIN" 191 | 192 | - name: Strip release binary (cross) 193 | if: env.CARGO == 'cross' && matrix.os == 'ubuntu-latest' 194 | shell: bash 195 | run: | 196 | docker run --rm -v \ 197 | "$PWD/target:/target:Z" \ 198 | "rustembedded/cross:${{ matrix.target }}" \ 199 | "${{ matrix.strip }}" \ 200 | "/target/${{ matrix.target }}/release/ag" 201 | 202 | - name: Determine archive name 203 | shell: bash 204 | run: | 205 | version="${{ needs.create-release.outputs.version }}" 206 | echo "ARCHIVE=aget-rs-$version-${{ matrix.target }}" >> $GITHUB_ENV 207 | 208 | - name: Creating directory for archive 209 | shell: bash 210 | run: | 211 | mkdir -p "$ARCHIVE"/ 212 | cp "$BIN" "$ARCHIVE"/ 213 | cp {README.md,LICENSE-APACHE,LICENSE-MIT} "$ARCHIVE"/ 214 | 215 | - name: Build archive (Windows) 216 | if: matrix.os == 'windows-latest' 217 | shell: bash 218 | run: | 219 | 7z a "$ARCHIVE.zip" "$ARCHIVE" 220 | certutil -hashfile "$ARCHIVE.zip" SHA256 > "$ARCHIVE.zip.sha256" 221 | echo "ASSET=$ARCHIVE.zip" >> $GITHUB_ENV 222 | echo "ASSET_SUM=$ARCHIVE.zip.sha256" >> $GITHUB_ENV 223 | 224 | - name: Build archive (Unix) 225 | if: matrix.os != 'windows-latest' 226 | shell: bash 227 | run: | 228 | tar czf "$ARCHIVE.tar.gz" "$ARCHIVE" 229 | shasum -a 256 "$ARCHIVE.tar.gz" > "$ARCHIVE.tar.gz.sha256" 230 | echo "ASSET=$ARCHIVE.tar.gz" >> $GITHUB_ENV 231 | echo "ASSET_SUM=$ARCHIVE.tar.gz.sha256" >> $GITHUB_ENV 232 | 233 | - name: Upload release archive 234 | env: 235 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 236 | shell: bash 237 | run: | 238 | version="${{ needs.create-release.outputs.version }}" 239 | gh release upload "$version" ${{ env.ASSET }} ${{ env.ASSET_SUM }} 240 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 0.6.3 - 2024-12-26 4 | 5 | ### Update 6 | 7 | - Update dependencies and rqbit api 8 | 9 | ### Fixed 10 | 11 | - Fix the error initializing persistent DHT 12 | 13 | ## 0.6.2 - 2024-10-25 14 | 15 | ### Added 16 | 17 | - Add `--bt-peer-connect-timeout`, `--bt-peer-read-write-timeout` and `--bt-peer-keep-alive-interval` options. 18 | 19 | ## 0.6.1 - 2024-10-16 20 | 21 | ### Fixed 22 | 23 | - Fix overflow when progress bar length is small 24 | 25 | ## 0.6.0 - 2024-10-14 26 | 27 | ### Added 28 | 29 | Support torrent and magnet link 30 | 31 | ## 0.5.1 - 2024-01-11 32 | 33 | ### Added 34 | 35 | - Add option `--insecure` to skip to verify the server's TLS certificate 36 | 37 | ## 0.5.0 - 2023-12-15 38 | 39 | ### Changed 40 | 41 | - Use `reqwest` to instead of `awc` 42 | 43 | ### Updated 44 | 45 | - Support proxy 46 | 47 | Use `--proxy` option or set global proxy environment variables 48 | 49 | ## 0.4.1 - 2022-04-20 50 | 51 | ### Added 52 | 53 | - Use `tracing` to log. 54 | 55 | ## 0.4.0 - 2022-04-19 56 | 57 | ### Updated 58 | 59 | - Update dependencies. 60 | 61 | ### Changed 62 | 63 | - No dependency on `OpenSSL`, using `rustls`. 64 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "aget-rs" 3 | version = "0.6.3" 4 | authors = ["PeterDing "] 5 | homepage = "https://github.com/PeterDing/aget-rs" 6 | description = "Aget-rs - Fast Asynchronous Downloader with Rust 🦀" 7 | license = "MIT/Apache-2.0" 8 | readme = "README.md" 9 | categories = ["command-line-utilities", "asynchronous", "download"] 10 | keywords = ["download", "network", "asynchronous", "tool"] 11 | edition = "2021" 12 | 13 | [lib] 14 | name = "aget" 15 | path = "src/lib.rs" 16 | 17 | [[bin]] 18 | name = "ag" 19 | path = "src/main.rs" 20 | 21 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 22 | 23 | [dependencies] 24 | # for async 25 | futures = "0.3" 26 | actix-rt = "2.10" 27 | 28 | # for http 29 | http = "1.1" 30 | url = "2.5" 31 | reqwest = { version = "0.12", features = [ 32 | "rustls-tls", 33 | "stream", 34 | "gzip", 35 | "brotli", 36 | "deflate", 37 | ], default-features = false } 38 | 39 | # for errors 40 | thiserror = "2.0" 41 | 42 | # for crypto 43 | aes = "0.8" 44 | cbc = { version = "0.1", features = ["alloc", "block-padding"] } 45 | 46 | # utilities 47 | term_size = "0.3" 48 | ansi_term = "0.12" 49 | percent-encoding = "2" 50 | bytes = "1" 51 | clap = { version = "4", features = ["derive", "cargo"] } 52 | toml = "0.8" 53 | serde = { version = "1.0", features = ["derive"] } 54 | dirs = "5.0" 55 | 56 | # for m3u8 57 | m3u8-rs = "6" 58 | 59 | # for torrent 60 | librqbit = { git = "https://github.com/ikatson/rqbit.git", rev = "b9f949c", features = [ 61 | "rust-tls", 62 | ], default-features = false } 63 | 64 | # for tracing 65 | tracing = "0.1" 66 | tracing-subscriber = { version = "0.3", features = [ 67 | "default", 68 | "time", 69 | "local-time", 70 | "registry", 71 | ] } 72 | tracing-appender = "0.2" 73 | time = { version = "0.3", features = ["formatting", "macros"] } 74 | 75 | [dev-dependencies] 76 | rand = "0.8" 77 | 78 | [profile.release] 79 | opt-level = 3 80 | debug = "none" 81 | strip = "symbols" 82 | debug-assertions = false 83 | overflow-checks = false 84 | lto = "fat" 85 | panic = "abort" 86 | incremental = false 87 | codegen-units = 1 88 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright (c) 2022 Peter Ding 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2022 Peter Ding 2 | 3 | Permission is hereby granted, free of charge, to any 4 | person obtaining a copy of this software and associated 5 | documentation files (the "Software"), to deal in the 6 | Software without restriction, including without 7 | limitation the rights to use, copy, modify, merge, 8 | publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software 10 | is furnished to do so, subject to the following 11 | conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions 15 | of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 18 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 19 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 20 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 21 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 22 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 24 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | DEALINGS IN THE SOFTWARE. 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

Aget-rs - Fast Asynchronous Downloader with Rust 🦀

2 | 3 | [![CI](https://github.com/PeterDing/aget-rs/actions/workflows/ci.yml/badge.svg)](https://github.com/PeterDing/aget-rs/actions/workflows/ci.yml) 4 | 5 | `aget-rs` is a fast asynchronous downloader written in Rust. 6 | It requests a resource with a number of concurrent asynchronous request in a single thread. 7 | 8 | Especially, the concurrent amount can be any positive number as your wish. 9 | 10 | `aget-rs` supports to download a **HTTP/S** link, a **M3U8** video link, a **Torrent** and a **magnet** link. 11 | 12 | ## Installation 13 | 14 | You can download the last release from https://github.com/PeterDing/aget-rs/releases 15 | 16 | ## Benchmark 17 | 18 | We conside that there is a file to download. This file has 10MB. 19 | The server which hosts the file has been set a limit rate to 100KB/s, but no connecting count limit. 20 | 21 | It will be easy to calculate the total costing time when we use 1, 10, 100 connections to request the file. 22 | 23 | In the benchmark test, we use `nginx` to simulate the environment where a limit rate is 100KB/s for downloading. 24 | 25 | Following is the results of using `curl` and `aget-rs`. (For more details, you can find at [here](ci/benchmark.bash)) 26 | 27 | - One connection using `curl` 28 | 29 | ``` 30 | time curl http://localhost:9010/abc 31 | % Total % Received % Xferd Average Speed Time Time Time Current 32 | Dload Upload Total Spent Left Speed 33 | 100 10.0M 100 10.0M 0 0 100k 0 0:01:42 0:01:42 --:--:-- 103k 34 | real 1m42.147s 35 | user 0m0.021s 36 | sys 0m0.035s 37 | ``` 38 | 39 | **time cost: 102s** 40 | 41 | - 10 connections using `aget-rs` 42 | 43 | ``` 44 | time ag http://localhost:9010/abc -s 10 -k 1m 45 | File: abc 46 | Length: 10.0M (10485760) 47 | 10.0M/10.0M 100.00% NaNG/s eta: 0s [==================================>] 48 | real 0m10.016s 49 | user 0m0.040s 50 | sys 0m0.020s 51 | ``` 52 | 53 | **time cost: 10s, 10 times faster than curl** 54 | 55 | - 100 connections using `aget-rs` 56 | 57 | ``` 58 | time ag http://localhost:9010/abc -s 100 -k 103k 59 | File: abc 60 | Length: 10.0M (10485760) 61 | 10.0M/10.0M 100.00% NaNG/s eta: 0s [==================================>] 62 | real 0m2.016s 63 | user 0m0.087s 64 | sys 0m0.029s 65 | ``` 66 | 67 | **time cost: 2s, 50 times faster than curl** 68 | 69 | ## Usage 70 | 71 | - Request a resource with default configuration 72 | 73 | The default concurrent amount is `10` and chunk length is `1m`. 74 | 75 | ```shell 76 | ag http://cdimage.ubuntu.com/ubuntu/releases/18.10/release/ubuntu-18.10-server-amd64.iso 77 | ``` 78 | 79 | - Set concurrent amount and chunk length 80 | 81 | Use `-s` or `--concurrency` to set the number of concurrent request. 82 | Use `-k` or `--chunk-size` to set the chunk length of each request. 83 | `--chunk-size` takes a literal size description, example `1k` for one Kilobyte, 84 | `2m` for two Megabyte, `1g` for Gigabyte. 85 | 86 | ```shell 87 | ag "url of resource" -s 20 -k 1m 88 | ``` 89 | 90 | - Set a path for output 91 | 92 | Use `-o` or `--out` to set the path. 93 | If the argument is not gave, we take the last part of the url' path as the path. 94 | 95 | ```shell 96 | ag "url of resource" -o /path/to/file 97 | ``` 98 | 99 | When download a torrent or magnet link, the path is the output directory. 100 | 101 | - Set request headers 102 | 103 | Use `-H` to set headers. 104 | 105 | ```shell 106 | ag "url of resource" -H "Cookie: key=value" -H "Accept: */*" 107 | ``` 108 | 109 | - Set request method and data 110 | 111 | Use `-X` or `--method` to set method for http, example, `GET`, `POST`. 112 | The default method is `GET`. 113 | With a data, using `-d` or `--data`, example, `a=b` 114 | 115 | ```shell 116 | ag "url of resource" -d "a=b" 117 | ``` 118 | 119 | - Download a torrent or magnet link 120 | 121 | **Warning**: The `/path/to/outdir` directory below command must NOT exist. It will be created automatically. 122 | 123 | ```shell 124 | ag "magnet:..." -o /path/to/outdir 125 | ag "/path/to/torrent" -o /path/to/outdir 126 | ag "http://example.com/some.torrent" -o /path/to/outdir 127 | ``` 128 | 129 | Use `--bt-file-regex` to only download files matching it in the torrent. 130 | 131 | ```shell 132 | ag "magnet:..." -o /path/to/outdir --bt-file-regex ".*\.mp4" 133 | ``` 134 | 135 | Use `--seed` to seed the torrent after downloaded. 136 | 137 | ```shell 138 | ag "magnet:..." -o /path/to/outdir --seed 139 | ``` 140 | 141 | Use `--bt-trackers` to specify trackers with comma as delimiter. 142 | 143 | ```shell 144 | ag "magnet:..." -o /path/to/outdir --bt-trackers "udp://tracker.opentrackr.org:1337/announce,udp://opentracker.io:6969/announce" 145 | ``` 146 | 147 | ## Options 148 | 149 | ``` 150 | Usage: ag [OPTIONS] 151 | 152 | Arguments: 153 | 154 | 155 | Options: 156 | -m, --method Request method, e.g. GET, POST [default: GET] 157 | -H, --header
Request headers, e.g. -H "User-Agent: aget" 158 | -d, --data Request with POST method with the data, e.g. -d "a=b" 159 | --insecure Skip to verify the server's TLS certificate 160 | -s, --concurrency The number of concurrency request [default: 10] 161 | -k, --chunk-size The number ofinterval length of each concurrent request [default: '50m'] 162 | -t, --timeout Timeout(seconds) of request [default: 60] 163 | --dns-timeout DNS Timeout(seconds) of request [default: 10] 164 | --retries The maximum times of retring [default: 5] 165 | --retry-wait The seconds between retries [default: 0] 166 | --proxy [protocol://]host[:port] Use this proxy 167 | --type Task type, auto/http/m3u8/bt [default: auto] 168 | --bt-file-regex A regex to only download files matching it in the torrent 169 | --seed Seed the torrent 170 | --bt-trackers Trackers for the torrent, e.g. --bt-trackers "udp://tracker.opentrackr.org:1337/announce 171 | ,udp://opentracker.io:6969/announce" 172 | --bt-peer-connect-timeout 173 | Peer connect timeout in seconds. [default: 10] 174 | --bt-peer-read-write-timeout 175 | Peer read/write timeout in seconds. [default: 10] 176 | --bt-peer-keep-alive-interval 177 | Peer keep-alive interval in seconds. [default: 120] 178 | --debug Debug output. Print all trackback for debugging 179 | --quiet Quiet mode. Don't show progress bar and task information. But still show the error information 180 | -o, --out The path of output for the request e.g. -o "/path/to/file" 181 | -h, --help Print help 182 | -V, --version Print version 183 | ``` 184 | 185 | ## Configuration 186 | 187 | Aget can be configured by a configuration file. The file locates at `~/.config/aget/config`. 188 | Following options can be set. Aget uses these options as the defaults for each command. 189 | 190 | ``` 191 | headers = [["key", "value"], ...] 192 | concurrency = ... 193 | chunk_size = "..." 194 | timeout = ... 195 | dns_timeout = ... 196 | retries = ... 197 | retry_wait = ... 198 | ``` 199 | 200 | If the file does not exist, aget will use the default configuration. 201 | 202 | ```toml 203 | headers = [["user-agent", "aget/version"]] 204 | concurrency = 10 205 | chunk_size = "50m" 206 | timeout = 60 207 | dns_timeout = 10 208 | retries = 5 209 | retry_wait = 0 210 | ``` 211 | -------------------------------------------------------------------------------- /ci/before_deploy.bash: -------------------------------------------------------------------------------- 1 | build() { 2 | echo "-: build release" 3 | 4 | cargo build --target "$TARGET" --release --verbose 5 | 6 | echo "-: build release, done" 7 | } 8 | 9 | pack() { 10 | echo "-: pack binary" 11 | 12 | local tempdir 13 | local out_dir 14 | local package_name 15 | local gcc_prefix 16 | 17 | tempdir=$(mktemp -d 2>/dev/null || mktemp -d -t tmp) 18 | out_dir=$(pwd) 19 | package_name="$PROJECT_NAME-$TRAVIS_TAG-$TARGET" 20 | 21 | gcc_prefix="" 22 | 23 | # create a "staging" directory 24 | mkdir "$tempdir/$package_name" 25 | 26 | # copying the main binary 27 | cp "target/$TARGET/release/$BIN_NAME" "$tempdir/$package_name/" 28 | "${gcc_prefix}"strip "$tempdir/$package_name/$BIN_NAME" 29 | 30 | # readme and license 31 | cp README.md "$tempdir/$package_name" 32 | cp LICENSE-MIT "$tempdir/$package_name" 33 | cp LICENSE-APACHE "$tempdir/$package_name" 34 | 35 | # archiving 36 | pushd "$tempdir" 37 | tar czf "$out_dir/$package_name.tar.gz" "$package_name"/* 38 | popd 39 | rm -r "$tempdir" 40 | 41 | echo "-: pack binary, done" 42 | } 43 | 44 | make_deb() { 45 | echo "-: make deb" 46 | 47 | local tempdir 48 | local architecture 49 | local version 50 | local dpkgname 51 | local conflictname 52 | local gcc_prefix 53 | 54 | case $TARGET in 55 | x86_64*) 56 | architecture=amd64 57 | gcc_prefix="" 58 | ;; 59 | i686*) 60 | architecture=i386 61 | gcc_prefix="" 62 | ;; 63 | *) 64 | echo "make_deb: skipping target '${TARGET}'" >&2 65 | return 0 66 | ;; 67 | esac 68 | version=${TRAVIS_TAG#v} 69 | if [[ $TARGET = *musl* ]]; then 70 | dpkgname=$PROJECT_NAME-musl 71 | conflictname=$PROJECT_NAME 72 | else 73 | dpkgname=$PROJECT_NAME 74 | conflictname=$PROJECT_NAME-musl 75 | fi 76 | 77 | tempdir=$(mktemp -d 2>/dev/null || mktemp -d -t tmp) 78 | 79 | # copy the main binary 80 | install -Dm755 "target/$TARGET/release/$BIN_NAME" "$tempdir/usr/bin/$BIN_NAME" 81 | "${gcc_prefix}"strip "$tempdir/usr/bin/$BIN_NAME" 82 | 83 | # readme and license 84 | install -Dm644 README.md "$tempdir/usr/share/doc/$PROJECT_NAME/README.md" 85 | install -Dm644 LICENSE-MIT "$tempdir/usr/share/doc/$PROJECT_NAME/LICENSE-MIT" 86 | install -Dm644 LICENSE-APACHE "$tempdir/usr/share/doc/$PROJECT_NAME/LICENSE-APACHE" 87 | 88 | # Control file 89 | mkdir "$tempdir/DEBIAN" 90 | cat > "$tempdir/DEBIAN/control" < 96 | Architecture: $architecture 97 | Provides: $PROJECT_NAME 98 | Conflicts: $conflictname 99 | Description: Aget-rs - Fast Asynchronous Downloader. 100 | EOF 101 | 102 | fakeroot dpkg-deb --build "$tempdir" "${dpkgname}_${version}_${architecture}.deb" 103 | 104 | echo "-: make deb, done" 105 | } 106 | 107 | 108 | main() { 109 | build 110 | pack 111 | if [[ $TARGET = *linux* ]]; then 112 | make_deb 113 | fi 114 | } 115 | 116 | echo "-: before_deploy.bash" 117 | main 118 | -------------------------------------------------------------------------------- /ci/before_deploy.ps1: -------------------------------------------------------------------------------- 1 | # This script takes care of packaging the build artifacts that will go in the 2 | # release zipfile 3 | 4 | $SRC_DIR = $PWD.Path 5 | $STAGE = [System.Guid]::NewGuid().ToString() 6 | 7 | Set-Location $ENV:Temp 8 | New-Item -Type Directory -Name $STAGE 9 | Set-Location $STAGE 10 | 11 | $ZIP = "$SRC_DIR\$($Env:CRATE_NAME)-$($Env:APPVEYOR_REPO_TAG_NAME)-$($Env:TARGET).zip" 12 | 13 | Copy-Item "$SRC_DIR\target\release\ag.exe" '.\' 14 | 15 | # readme and license 16 | Copy-Item $SRC_DIR\README.md '.\' 17 | Copy-Item $SRC_DIR\LICENSE-MIT '.\' 18 | Copy-Item $SRC_DIR\LICENSE-APACHE '.\' 19 | 20 | 7z a "$ZIP" * 21 | 22 | Push-AppveyorArtifact "$ZIP" 23 | 24 | Set-Location .. 25 | Remove-Item $STAGE -Force -Recurse 26 | Set-Location $SRC_DIR 27 | -------------------------------------------------------------------------------- /ci/before_install.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if ! command -V sudo; then 4 | apt-get update 5 | apt-get install -y --no-install-recommends sudo 6 | fi 7 | sudo apt-get update 8 | sudo apt-get install -y --no-install-recommends \ 9 | zsh xz-utils liblz4-tool musl-tools brotli zstd \ 10 | build-essential openssl libssl-dev pkg-config 11 | 12 | # needed to build deb packages 13 | sudo apt-get install -y --no-install-recommends fakeroot 14 | -------------------------------------------------------------------------------- /ci/benchmark.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | echo "-: benchmark.bash" 6 | 7 | if [[ $RUNNER_OS != Linux ]]; then 8 | exit 0 9 | fi 10 | 11 | 12 | echo "-: install nginx" 13 | sudo apt-get install -y nginx curl 14 | 15 | echo "-: start nginx" 16 | sudo nginx -c "$(pwd)/ci/benchmark.nginx.conf" 17 | 18 | echo "-: make test file" 19 | sudo mkdir -p /data 20 | # size: 10m 21 | sudo dd if=/dev/zero of=file.txt count=10240 bs=1024 22 | sudo mv file.txt /data/abc 23 | 24 | echo "-: benchmark test begins" 25 | 26 | echo "-: Request with one connection" 27 | # 10240k / 100k/s = 102s = 1m42s 28 | time curl http://localhost:9010/abc > abc 29 | rm abc 30 | 31 | echo "-: Request with 10 connections" 32 | # 10240k / (10 * 100k/s) = 10s, each interval is 1m 33 | time target/debug/ag http://localhost:9010/abc -s 10 -k 1m 34 | rm abc 35 | 36 | echo "-: Request with 100 connections" 37 | # 10240k / (100 * 100k/s) = 1s, each interval is 102.4k 38 | time target/debug/ag http://localhost:9010/abc -s 100 -k 103k 39 | rm abc 40 | 41 | echo "-: benchmark test ends" 42 | -------------------------------------------------------------------------------- /ci/benchmark.nginx.conf: -------------------------------------------------------------------------------- 1 | worker_processes auto; 2 | pid /run/nginx.pid; 3 | 4 | events { 5 | worker_connections 768; 6 | # multi_accept on; 7 | } 8 | 9 | http { 10 | server { 11 | listen 9010 default_server; 12 | 13 | location / { 14 | limit_rate 100k; 15 | root /data; 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /ci/script.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | TEST_AIM_URI="https://github.com/mpv-player/mpv/archive/49d6a1e77d3dba5cf30a2cb9cd05dc6dd6407fb2.zip" 6 | TEST_AIM_MD5="703cd17daf71138b90622d52fe6fb6a5" 7 | TEST_AIM_NAME="49d6a1e77d3dba5cf30a2cb9cd05dc6dd6407fb2.zip" 8 | 9 | RUST_BACKTRACE=1 cargo run -- $TEST_AIM_URI --debug 10 | 11 | if [ "$RUNNER_OS" = "Linux" ] 12 | then 13 | md5="$(cat $TEST_AIM_NAME | md5sum | cut -f1 -d' ')" 14 | elif [ "$RUNNER_OS" = "macOS" ] 15 | then 16 | md5="$(cat $TEST_AIM_NAME | md5 | cut -f1 -d' ')" 17 | fi 18 | 19 | if [ "$md5" != "$TEST_AIM_MD5" ]; then exit 1; fi 20 | 21 | rm $TEST_AIM_NAME 22 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | imports_granularity = "Crate" 2 | max_width = 120 3 | -------------------------------------------------------------------------------- /src/app/core/bt.rs: -------------------------------------------------------------------------------- 1 | use std::{path::PathBuf, sync::Arc, time::Duration}; 2 | 3 | use librqbit::{ 4 | api::TorrentIdOrHash, dht::PersistentDhtConfig, AddTorrent, AddTorrentOptions, AddTorrentResponse, Api, 5 | PeerConnectionOptions, Session, SessionOptions, SessionPersistenceConfig, 6 | }; 7 | use url::Url; 8 | 9 | use crate::{ 10 | app::show::bt_show::BtShower, 11 | common::errors::{Error, Result}, 12 | features::{args::Args, running::Runnable}, 13 | }; 14 | 15 | pub struct BtHandler { 16 | torrent_or_magnet: Url, 17 | output: PathBuf, 18 | file_regex: Option, 19 | seed: bool, 20 | trackers: Option>, 21 | peer_connect_timeout: Option, 22 | peer_read_write_timeout: Option, 23 | peer_keep_alive_interval: Option, 24 | } 25 | 26 | impl BtHandler { 27 | pub fn new(args: &(impl Args + std::fmt::Debug)) -> BtHandler { 28 | tracing::debug!("BtHandler::new"); 29 | 30 | BtHandler { 31 | torrent_or_magnet: args.url(), 32 | output: args.output(), 33 | file_regex: args.bt_file_regex(), 34 | seed: args.seed(), 35 | trackers: args.bt_trackers(), 36 | peer_connect_timeout: args.bt_peer_connect_timeout(), 37 | peer_read_write_timeout: args.bt_peer_read_write_timeout(), 38 | peer_keep_alive_interval: args.bt_peer_keep_alive_interval(), 39 | } 40 | } 41 | 42 | async fn start(self) -> Result<()> { 43 | tracing::debug!("BtHandler::start"); 44 | 45 | let output_dir = &self.output; 46 | let persistence_dir = output_dir 47 | .join("..") 48 | .join(output_dir.file_name().unwrap().to_string_lossy().to_string() + ".bt.aget"); 49 | let dht_config_filename = persistence_dir.join("dht.json"); 50 | 51 | // 0. Check whether task is completed 52 | tracing::debug!("BtHandler: check whether task is completed"); 53 | if output_dir.exists() && !persistence_dir.exists() { 54 | return Ok(()); 55 | } 56 | 57 | // 1. Create session 58 | tracing::debug!("BtHandler: create session"); 59 | let sopts = SessionOptions { 60 | disable_dht: false, 61 | disable_dht_persistence: false, 62 | dht_config: Some(PersistentDhtConfig { 63 | config_filename: Some(dht_config_filename), 64 | ..Default::default() 65 | }), 66 | peer_id: None, 67 | peer_opts: Some(PeerConnectionOptions { 68 | connect_timeout: self.peer_connect_timeout.map(Duration::from_secs), 69 | read_write_timeout: self.peer_read_write_timeout.map(Duration::from_secs), 70 | keep_alive_interval: self.peer_keep_alive_interval.map(Duration::from_secs), 71 | }), 72 | fastresume: true, 73 | persistence: Some(SessionPersistenceConfig::Json { 74 | folder: Some(persistence_dir.clone()), 75 | }), 76 | ..Default::default() 77 | }; 78 | let session = Session::new_with_opts(output_dir.to_owned(), sopts) 79 | .await 80 | .map_err(|err| Error::BitTorrentError(err.to_string()))?; 81 | 82 | // 2. Create shower 83 | tracing::debug!("BtHandler: create shower"); 84 | let stats_watcher = StatsWatcher { 85 | session: session.clone(), 86 | forever: self.seed, 87 | }; 88 | let stats_watcher_join_handler = actix_rt::spawn(stats_watcher.watch()); 89 | 90 | // 3. Add torrent or magnet 91 | tracing::debug!("BtHandler: add torrent or magnet"); 92 | let topts = Some(AddTorrentOptions { 93 | only_files_regex: self.file_regex.clone(), 94 | trackers: self.trackers.clone(), 95 | ..Default::default() 96 | }); 97 | let response = session 98 | .add_torrent( 99 | AddTorrent::from_cli_argument(&self.torrent_or_magnet.as_str()) 100 | .map_err(|err| Error::BitTorrentError(err.to_string()))?, 101 | topts, 102 | ) 103 | .await 104 | .map_err(|err| Error::BitTorrentError(err.to_string()))?; 105 | 106 | match response { 107 | AddTorrentResponse::AlreadyManaged(id, handle) => { 108 | tracing::debug!("Torrent {} is already managed", id); 109 | handle 110 | .wait_until_completed() 111 | .await 112 | .map_err(|err| Error::BitTorrentError(err.to_string()))?; 113 | } 114 | AddTorrentResponse::Added(id, handle) => { 115 | tracing::debug!("Torrent {} is added", id); 116 | handle 117 | .wait_until_completed() 118 | .await 119 | .map_err(|err| Error::BitTorrentError(err.to_string()))?; 120 | } 121 | _ => { 122 | unreachable!() 123 | } 124 | } 125 | 126 | // 4. Start seeding 127 | if self.seed { 128 | tracing::debug!("BtHandler: start seeding"); 129 | println!("\nSeeding..."); 130 | } 131 | while self.seed { 132 | actix_rt::time::sleep(Duration::from_secs(1)).await; 133 | } 134 | 135 | // 5. Exit shower 136 | tracing::debug!("BtHandler: exit shower"); 137 | stats_watcher_join_handler.await.unwrap(); 138 | 139 | // 6. Remove persistence folder 140 | tracing::debug!("BtHandler: remove persistence folder"); 141 | std::fs::remove_dir_all(persistence_dir)?; 142 | 143 | Ok(()) 144 | } 145 | } 146 | 147 | impl Runnable for BtHandler { 148 | fn run(self) -> Result<()> { 149 | let sys = actix_rt::System::new(); 150 | sys.block_on(self.start()) 151 | } 152 | } 153 | 154 | struct StatsWatcher { 155 | session: Arc, 156 | forever: bool, 157 | } 158 | 159 | impl StatsWatcher { 160 | async fn watch(self) { 161 | let mut shower = BtShower::new(); 162 | 163 | let tid = TorrentIdOrHash::Id(0); 164 | let api = Api::new(self.session.clone(), None); 165 | 166 | let torrent_details = loop { 167 | if let Ok(torrent_details) = api.api_torrent_details(tid) { 168 | break torrent_details; 169 | } 170 | 171 | actix_rt::time::sleep(Duration::from_secs(1)).await; 172 | }; 173 | 174 | if self.forever { 175 | shower.print_msg("Seed the torrent. Press Ctrl+C to exit").unwrap(); 176 | } 177 | 178 | shower 179 | .print_name(torrent_details.name.as_deref().unwrap_or("unknown")) 180 | .expect("failed to print name"); 181 | 182 | let torrent_files: Vec<_> = torrent_details.files.unwrap_or_default(); 183 | let files: Vec<_> = torrent_files 184 | .iter() 185 | .map(|file| (file.name.as_str(), file.length, file.included)) 186 | .collect(); 187 | 188 | shower.print_files(&files[..]).unwrap(); 189 | 190 | let mut completed_idx: Vec = vec![false; files.len()]; 191 | 192 | loop { 193 | let stats = api.api_stats_v1(tid).expect("failed to get stats"); 194 | 195 | if let Some(live) = stats.live { 196 | let completed = stats.progress_bytes; 197 | let total = stats.total_bytes; 198 | let down_rate = live.download_speed.mbps * 1e6; 199 | let up_rate = live.upload_speed.mbps * 1e6; 200 | let uploaded = live.snapshot.uploaded_bytes; 201 | 202 | let eta = { 203 | let remains = total - completed; 204 | // rate > 1.0 for overflow 205 | if remains > 0 && down_rate > 1.0 { 206 | let eta = (remains as f64 / down_rate) as u64; 207 | // eta is large than 99 days, return 0 208 | if eta > 99 * 24 * 60 * 60 { 209 | 0 210 | } else { 211 | eta 212 | } 213 | } else { 214 | 0 215 | } 216 | }; 217 | 218 | let peer_stats = live.snapshot.peer_stats; 219 | let live_peers = peer_stats.live; 220 | let queued_peers = peer_stats.queued; 221 | 222 | shower 223 | .print_status( 224 | completed, 225 | total, 226 | eta, 227 | down_rate, 228 | up_rate, 229 | uploaded, 230 | live_peers, 231 | queued_peers, 232 | ) 233 | .unwrap(); 234 | 235 | files.iter().enumerate().for_each(|(i, (filename, length, included))| { 236 | if *included && !completed_idx[i] { 237 | let completed_size = stats.file_progress[i]; 238 | if completed_size == *length { 239 | shower.print_completed_file(filename).unwrap(); 240 | completed_idx[i] = true; 241 | } 242 | } 243 | }) 244 | } 245 | 246 | if !self.forever && stats.finished { 247 | break; 248 | } 249 | 250 | actix_rt::time::sleep(Duration::from_secs(1)).await; 251 | } 252 | } 253 | } 254 | -------------------------------------------------------------------------------- /src/app/core/http.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | fmt, 3 | path::PathBuf, 4 | sync::{Arc, Mutex}, 5 | time::Duration, 6 | }; 7 | 8 | use futures::{ 9 | channel::mpsc::{channel, Sender}, 10 | pin_mut, select, SinkExt, StreamExt, 11 | }; 12 | 13 | use crate::{ 14 | app::{ 15 | receive::http_receiver::HttpReceiver, 16 | record::{common::RECORDER_FILE_SUFFIX, range_recorder::RangeRecorder}, 17 | }, 18 | common::{ 19 | bytes::bytes_type::Bytes, 20 | errors::{Error, Result}, 21 | file::File, 22 | net::{ 23 | net::{build_http_client, redirect_and_contentlength, request}, 24 | ContentLengthValue, HttpClient, Method, Url, 25 | }, 26 | range::{split_pair, RangePair, SharedRangList}, 27 | time::interval_stream, 28 | }, 29 | features::{args::Args, running::Runnable, stack::StackLike}, 30 | }; 31 | 32 | /// Http task handler 33 | pub struct HttpHandler<'a> { 34 | output: PathBuf, 35 | method: Method, 36 | url: Url, 37 | headers: Vec<(&'a str, &'a str)>, 38 | data: Option<&'a str>, 39 | concurrency: u64, 40 | chunk_size: u64, 41 | proxy: Option<&'a str>, 42 | timeout: Duration, 43 | client: HttpClient, 44 | } 45 | 46 | impl<'a> std::fmt::Debug for HttpHandler<'a> { 47 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 48 | write!( 49 | f, 50 | "HttpHandler{{ method: {}, url: {}, headers: {:?}, data: {:?}, concurrency: {}, proxy: {:?} }}", 51 | self.method, self.url, self.headers, self.data, self.concurrency, self.proxy 52 | ) 53 | } 54 | } 55 | 56 | impl<'a> HttpHandler<'a> { 57 | pub fn new(args: &(impl Args + std::fmt::Debug)) -> Result { 58 | let headers = args.headers(); 59 | let timeout = args.timeout(); 60 | let dns_timeout = args.dns_timeout(); 61 | let keep_alive = args.keep_alive(); 62 | let skip_verify_tls_cert = args.skip_verify_tls_cert(); 63 | let proxy = args.proxy(); 64 | 65 | let client = build_http_client(&headers, timeout, dns_timeout, keep_alive, skip_verify_tls_cert, proxy)?; 66 | 67 | tracing::debug!("HttpHandler::new"); 68 | 69 | Ok(HttpHandler { 70 | output: args.output(), 71 | method: args.method(), 72 | url: args.url(), 73 | headers, 74 | data: args.data(), 75 | concurrency: args.concurrency(), 76 | chunk_size: args.chunk_size(), 77 | proxy, 78 | timeout, 79 | client, 80 | }) 81 | } 82 | 83 | async fn start(mut self) -> Result<()> { 84 | tracing::debug!("HttpHandler::start"); 85 | 86 | // 0. Check whether task is completed 87 | tracing::debug!("HttpHandler: check whether task is completed"); 88 | let mut rangerecorder = RangeRecorder::new(&*(self.output.to_string_lossy() + RECORDER_FILE_SUFFIX))?; 89 | if self.output.exists() && !rangerecorder.exists() { 90 | return Ok(()); 91 | } 92 | 93 | // 1. redirect and get content_length 94 | tracing::debug!("HttpHandler: redirect and content_length start"); 95 | let (url, cl) = redirect_and_contentlength( 96 | &self.client, 97 | self.method.clone(), 98 | self.url.clone(), 99 | self.data.map(|v| v.to_string()), 100 | ) 101 | .await?; 102 | tracing::debug!("HttpHandler: redirect to: {}", url); 103 | tracing::debug!("HttpHandler: content_length: {:?}", cl); 104 | 105 | self.url = url; 106 | 107 | let content_length = { 108 | match cl { 109 | ContentLengthValue::DirectLength(l) => l, 110 | ContentLengthValue::RangeLength(l) => l, 111 | _ => 0, 112 | } 113 | }; 114 | 115 | // 2. Compare recorded content length with the above one 116 | tracing::debug!("HttpHandler: compare recorded content length"); 117 | let mut direct = true; 118 | if let ContentLengthValue::RangeLength(cl) = cl { 119 | if self.output.exists() { 120 | if rangerecorder.exists() { 121 | rangerecorder.open()?; 122 | } else { 123 | // Task is completed 124 | return Ok(()); 125 | } 126 | } else { 127 | // Init rangerecorder 128 | rangerecorder.remove().unwrap_or(()); // Missing error 129 | rangerecorder.open()?; 130 | } 131 | 132 | let pre_cl = rangerecorder.total()?; 133 | 134 | // Inital rangerecorder 135 | if pre_cl == 0 && pre_cl != cl { 136 | rangerecorder.write_total(cl)?; 137 | direct = false; 138 | } 139 | // Content is empty 140 | else if pre_cl == 0 && pre_cl == cl { 141 | File::new(&self.output, true)?.open()?; 142 | rangerecorder.remove()?; 143 | return Ok(()); 144 | } 145 | // Content length is not consistent 146 | else if pre_cl != 0 && pre_cl != cl { 147 | return Err(Error::ContentLengthIsNotConsistent); 148 | } 149 | // Rewrite statistic status 150 | else if pre_cl != 0 && pre_cl == cl { 151 | rangerecorder.rewrite()?; 152 | direct = false; 153 | } 154 | } 155 | 156 | // 3. Create channel 157 | let (sender, receiver) = channel::<(RangePair, Bytes)>(self.concurrency as usize + 10); 158 | let runtime_error: Arc>> = Arc::new(Mutex::new(None)); 159 | 160 | // 4. Dispatch Task 161 | tracing::debug!("HttpHandler: dispatch task: direct: {}", direct); 162 | if direct { 163 | // We need a new `HttpClient` which has unlimited life time for `DirectRequestTask` 164 | let mut task = DirectRequestTask::new( 165 | self.client.clone(), 166 | self.method.clone(), 167 | self.url.clone(), 168 | self.data.map(|v| v.to_string()), 169 | sender.clone(), 170 | ); 171 | let runtime_error_clone = runtime_error.clone(); 172 | actix_rt::spawn(async move { 173 | if let Err(err) = task.start().await { 174 | if runtime_error_clone.lock().unwrap().is_none() { 175 | *runtime_error_clone.lock().unwrap() = Some(err); 176 | } 177 | } 178 | }); 179 | } else { 180 | // Make range pairs stack 181 | let mut stack = vec![]; 182 | let gaps = rangerecorder.gaps()?; 183 | for gap in gaps.iter() { 184 | let mut list = split_pair(gap, self.chunk_size); 185 | stack.append(&mut list); 186 | } 187 | stack.reverse(); 188 | let stack = SharedRangList::new(stack); 189 | // let stack = SharedRangList::new(rangerecorder.gaps()?); 190 | tracing::debug!("HttpHandler: range stack length: {}", stack.len()); 191 | 192 | let concurrency = std::cmp::min(stack.len() as u64, self.concurrency); 193 | for i in 1..concurrency + 1 { 194 | let mut task = RangeRequestTask::new( 195 | self.client.clone(), 196 | self.method.clone(), 197 | self.url.clone(), 198 | self.data.map(|v| v.to_string()), 199 | stack.clone(), 200 | sender.clone(), 201 | i, 202 | self.timeout, 203 | ); 204 | let runtime_error_clone = runtime_error.clone(); 205 | actix_rt::spawn(async move { 206 | if let Err(err) = task.start().await { 207 | if runtime_error_clone.lock().unwrap().is_none() { 208 | *runtime_error_clone.lock().unwrap() = Some(err); 209 | } 210 | } 211 | }); 212 | } 213 | } 214 | drop(sender); // Remove the reference and let `Task` to handle it 215 | 216 | // 5. Create receiver 217 | tracing::debug!("HttpHandler: create receiver"); 218 | let mut httpreceiver = HttpReceiver::new(&self.output, direct, content_length)?; 219 | httpreceiver.start(receiver).await?; 220 | 221 | if let Some(err) = runtime_error.lock().unwrap().take() { 222 | return Err(err); 223 | } 224 | 225 | // 6. Task succeeds. Remove rangerecorder file 226 | rangerecorder.remove().unwrap_or(()); // Missing error 227 | Ok(()) 228 | } 229 | } 230 | 231 | impl<'a> Runnable for HttpHandler<'a> { 232 | fn run(self) -> Result<()> { 233 | let sys = actix_rt::System::new(); 234 | sys.block_on(self.start()) 235 | } 236 | } 237 | 238 | /// Directly request the resource without range header 239 | struct DirectRequestTask { 240 | client: HttpClient, 241 | method: Method, 242 | url: Url, 243 | data: Option, 244 | sender: Sender<(RangePair, Bytes)>, 245 | } 246 | 247 | impl DirectRequestTask { 248 | #[tracing::instrument(skip(client, sender))] 249 | fn new( 250 | client: HttpClient, 251 | method: Method, 252 | url: Url, 253 | data: Option, 254 | sender: Sender<(RangePair, Bytes)>, 255 | ) -> DirectRequestTask { 256 | DirectRequestTask { 257 | client, 258 | method, 259 | url, 260 | data, 261 | sender, 262 | } 263 | } 264 | 265 | #[tracing::instrument(skip(self))] 266 | async fn start(&mut self) -> Result<()> { 267 | let resp = request( 268 | &self.client, 269 | self.method.clone(), 270 | self.url.clone(), 271 | self.data.clone(), 272 | None, 273 | ) 274 | .await; 275 | if let Err(err) = resp { 276 | tracing::error!("DirectRequestTask request error: {:?}", err); 277 | return Err(err); 278 | } 279 | let resp = resp.unwrap(); 280 | let mut stream = resp.bytes_stream(); 281 | 282 | let mut offset = 0u64; 283 | while let Some(item) = stream.next().await { 284 | match item { 285 | Ok(chunk) => { 286 | let len = chunk.len(); 287 | if len == 0 { 288 | continue; 289 | } 290 | 291 | let pair = RangePair::new(offset, offset + len as u64 - 1); // The pair is a closed interval 292 | self.sender.send((pair, chunk)).await.unwrap(); 293 | offset += len as u64; 294 | } 295 | Err(err) => { 296 | tracing::error!("DirectRequestTask read error: {:?}", err); 297 | break; 298 | } 299 | } 300 | } 301 | 302 | Ok(()) 303 | } 304 | } 305 | 306 | /// Request the resource with a range header which is in the `SharedRangList` 307 | struct RangeRequestTask { 308 | client: HttpClient, 309 | method: Method, 310 | url: Url, 311 | data: Option, 312 | stack: SharedRangList, 313 | sender: Sender<(RangePair, Bytes)>, 314 | id: u64, 315 | timeout: Duration, 316 | } 317 | 318 | impl RangeRequestTask { 319 | #[tracing::instrument(skip(client, sender))] 320 | fn new( 321 | client: HttpClient, 322 | method: Method, 323 | url: Url, 324 | data: Option, 325 | stack: SharedRangList, 326 | sender: Sender<(RangePair, Bytes)>, 327 | id: u64, 328 | timeout: Duration, 329 | ) -> RangeRequestTask { 330 | RangeRequestTask { 331 | client, 332 | method, 333 | url, 334 | data, 335 | stack, 336 | sender, 337 | id, 338 | timeout, 339 | } 340 | } 341 | 342 | #[tracing::instrument(skip(self))] 343 | async fn start(&mut self) -> Result<()> { 344 | tracing::debug!("Fire RangeRequestTask: {}", self.id); 345 | while let Some(pair) = self.stack.pop() { 346 | match self.req(pair).await { 347 | // Exit whole process when `Error::InnerError` is returned 348 | Err(Error::InnerError(msg)) => { 349 | tracing::error!("RangeRequestTask {}: InnerError: {}", self.id, msg); 350 | actix_rt::System::current().stop(); 351 | } 352 | Err(err @ Error::Timeout) => { 353 | tracing::debug!("RangeRequestTask timeout: {}", err); // Missing Timeout at runtime 354 | } 355 | // Return other response errors 356 | Err(err) => { 357 | tracing::debug!("RangeRequestTask {}: error: {}", self.id, err); 358 | return Err(err); 359 | } 360 | _ => {} 361 | } 362 | } 363 | Ok(()) 364 | } 365 | 366 | async fn req(&mut self, pair: RangePair) -> Result<()> { 367 | let resp = request( 368 | &self.client, 369 | self.method.clone(), 370 | self.url.clone(), 371 | self.data.clone(), 372 | Some(pair), 373 | ) 374 | .await; 375 | 376 | if let Err(err) = resp { 377 | self.stack.push(pair); 378 | return Err(err); 379 | } 380 | let resp = resp.unwrap(); 381 | 382 | let RangePair { begin, end } = pair; 383 | let length = pair.length(); 384 | let mut count = 0u64; 385 | let mut offset = begin; 386 | 387 | let stream = resp.bytes_stream().fuse(); 388 | 389 | // Set timeout for reading 390 | let tick = interval_stream(self.timeout).fuse(); 391 | 392 | pin_mut!(stream, tick); 393 | let mut fire = false; 394 | loop { 395 | select! { 396 | item = stream.next() => { 397 | if let Some(item) = item { 398 | match item { 399 | Ok(chunk) => { 400 | let len = chunk.len(); 401 | if len == 0 { 402 | continue; 403 | } 404 | 405 | // The pair is a closed interval 406 | let pr = RangePair::new(offset, offset + len as u64 - 1); 407 | if let Err(err) = self.sender.send((pr, chunk)).await { 408 | let pr = RangePair::new(offset, end); 409 | self.stack.push(pr); 410 | return Err(Error::InnerError(format!( 411 | "Error at `http::RangeRequestTask`: Sender error: {:?}", 412 | err 413 | ))); 414 | } 415 | offset += len as u64; 416 | count += len as u64; 417 | } 418 | Err(err) => { 419 | let pr = RangePair::new(offset, end); 420 | self.stack.push(pr); 421 | return Err(err.into()); 422 | } 423 | } 424 | } else { 425 | break; 426 | } 427 | } 428 | _ = tick.next() => { 429 | if fire { 430 | let pr = RangePair::new(offset, end); 431 | self.stack.push(pr); 432 | return Err(Error::Timeout); 433 | } else { 434 | fire = true; 435 | } 436 | } 437 | } 438 | } 439 | 440 | // Check range length whether is equal to the length of all received chunk 441 | if count != length { 442 | let pr = RangePair::new(offset, end); 443 | self.stack.push(pr); 444 | Err(Error::UncompletedRead) 445 | } else { 446 | Ok(()) 447 | } 448 | } 449 | } 450 | -------------------------------------------------------------------------------- /src/app/core/m3u8/common.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use m3u8_rs::{parse_playlist_res, Key, Playlist}; 4 | 5 | use crate::common::{ 6 | bytes::bytes::{decode_hex, u32_to_u8x4}, 7 | errors::{Error, Result}, 8 | list::SharedVec, 9 | net::{ 10 | net::{join_url, redirect, request}, 11 | HttpClient, Method, Url, 12 | }, 13 | }; 14 | 15 | #[derive(Debug, Clone)] 16 | pub struct M3u8Segment { 17 | pub index: u64, 18 | pub method: Method, 19 | pub url: Url, 20 | pub data: Option, 21 | pub key: Option<[u8; 16]>, 22 | pub iv: Option<[u8; 16]>, 23 | } 24 | 25 | pub type M3u8SegmentList = Vec; 26 | 27 | pub type SharedM3u8SegmentList = SharedVec; 28 | 29 | pub async fn get_m3u8(client: &HttpClient, method: Method, url: Url, data: Option) -> Result { 30 | // url -> (key, iv) 31 | let mut keymap: HashMap = HashMap::new(); 32 | let mut urls = vec![url]; 33 | let mut list = vec![]; 34 | 35 | let mut idx = 0; 36 | 37 | while let Some(url) = urls.pop() { 38 | tracing::debug!("m3u8 url: {}", url); 39 | let u = redirect(client, method.clone(), url.clone(), data.clone()).await?; 40 | 41 | if u != url { 42 | tracing::debug!("m3u8 redirect to: {}", u); 43 | urls.push(u.clone()); 44 | continue; 45 | } 46 | 47 | let base_url = u.clone(); 48 | 49 | // Read m3u8 content 50 | let resp = request(client, method.clone(), u.clone(), data.clone(), None).await?; 51 | let cn = resp.bytes().await?; 52 | let mut cn = cn.to_vec(); 53 | 54 | // Adding "\n" for the case when response content has not "\n" at end. 55 | cn.extend(b"\n"); 56 | 57 | // Parse m3u8 content 58 | let parsed = parse_playlist_res(cn.as_ref()); 59 | match parsed { 60 | Ok(Playlist::MasterPlaylist(mut pl)) => { 61 | pl.variants.reverse(); 62 | for variant in &pl.variants { 63 | let url = join_url(&base_url, &variant.uri)?; 64 | urls.push(url); 65 | } 66 | } 67 | Ok(Playlist::MediaPlaylist(pl)) => { 68 | let mut index = pl.media_sequence as u64; 69 | let mut key_m: Option = None; 70 | for segment in &pl.segments { 71 | let seg_url = join_url(&base_url, &segment.uri)?; 72 | 73 | // In `pl.segment`, the same key will not repeat, if previous key appears. 74 | let segment_key = if segment.key.is_none() && key_m.is_some() { 75 | &key_m 76 | } else { 77 | key_m = segment.key.clone(); 78 | &segment.key 79 | }; 80 | 81 | let (key, iv) = if let Some(key) = segment_key { 82 | let iv = if let Some(iv) = &key.iv { 83 | let mut i = [0; 16]; 84 | let buf = decode_hex(&iv[2..])?; 85 | i.clone_from_slice(&buf[..]); 86 | i 87 | } else { 88 | let mut iv = [0; 16]; 89 | let index_bin = u32_to_u8x4(index as u32); 90 | iv[12..].clone_from_slice(&index_bin); 91 | iv 92 | }; 93 | if let Some(url) = &key.uri { 94 | let key_url = join_url(&base_url, url)?; 95 | if let Some(k) = keymap.get(&key_url) { 96 | (Some(*k), Some(iv)) 97 | } else { 98 | let k = get_key(client, Method::GET, key_url.clone()).await?; 99 | keymap.insert(key_url.clone(), k); 100 | tracing::debug!( 101 | "Get key: {}, iv: {}", 102 | unsafe { std::str::from_utf8_unchecked(&k) }, 103 | unsafe { std::str::from_utf8_unchecked(&iv) } 104 | ); 105 | (Some(k), Some(iv)) 106 | } 107 | } else { 108 | (None, None) 109 | } 110 | } else { 111 | (None, None) 112 | }; 113 | 114 | list.push(M3u8Segment { 115 | index: idx, 116 | method: Method::GET, 117 | url: seg_url.clone(), 118 | data: None, 119 | key, 120 | iv, 121 | }); 122 | index += 1; 123 | idx += 1; 124 | } 125 | } 126 | Err(_) => return Err(Error::M3U8ParseFail), 127 | } 128 | } 129 | Ok(list) 130 | } 131 | 132 | async fn get_key(client: &HttpClient, method: Method, url: Url) -> Result<[u8; 16]> { 133 | let resp = request(client, method.clone(), url.clone(), None, None).await?; 134 | let cn = resp.bytes().await?; 135 | let mut buf = [0; 16]; 136 | buf[..].clone_from_slice(&cn); 137 | Ok(buf) 138 | } 139 | -------------------------------------------------------------------------------- /src/app/core/m3u8/ffmpeg.rs: -------------------------------------------------------------------------------- 1 | use std::{path::PathBuf, process::Command}; 2 | 3 | use crate::common::errors::Result; 4 | 5 | pub(crate) struct FFmpegExecutor; 6 | 7 | impl FFmpegExecutor { 8 | fn fix_m3u8(file_path: &PathBuf) -> Result<()> { 9 | // ffmpeg -y -loglevel repeat+info -i file:{file_path} -map 0 -dn -ignore_unknown -c copy -f {ext} -bsf:a aac_adtstoasc -movflags +faststart file:{file_path}.temp.{ext} 10 | let mut cmd = Command::new("ffmpeg"); 11 | 12 | cmd.arg("-y") 13 | .arg("-loglevel") 14 | .arg("repeat+info") 15 | .arg("-i") 16 | .arg(format!("file:{}", file_path.display())) 17 | .arg("-map") 18 | .arg("0") 19 | .arg("-dn") 20 | .arg("-ignore_unknown") 21 | .arg("-c") 22 | .arg("copy") 23 | .arg("-bsf:a") 24 | .arg("aac_adtstoasc") 25 | .arg("-movflags") 26 | .arg("+faststart") 27 | .arg(format!("file:{}.temp.mp4", file_path.display())); 28 | 29 | // if file_path.ends_with(".mp4") { 30 | // cmd.arg("-f").arg("mp4") 31 | // } 32 | Ok(()) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/app/core/m3u8/m3u8.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | cell::Cell, 3 | path::PathBuf, 4 | rc::Rc, 5 | sync::{Arc, Mutex}, 6 | time::Duration, 7 | }; 8 | 9 | use futures::{ 10 | channel::mpsc::{channel, Sender}, 11 | pin_mut, select, SinkExt, StreamExt, 12 | }; 13 | 14 | use crate::{ 15 | app::{ 16 | core::m3u8::common::{get_m3u8, M3u8Segment, SharedM3u8SegmentList}, 17 | receive::m3u8_receiver::M3u8Receiver, 18 | record::{bytearray_recorder::ByteArrayRecorder, common::RECORDER_FILE_SUFFIX}, 19 | }, 20 | common::{ 21 | bytes::bytes_type::Bytes, 22 | crypto::decrypt_aes128, 23 | errors::{Error, Result}, 24 | net::{ 25 | net::{build_http_client, request}, 26 | HttpClient, Method, Url, 27 | }, 28 | time::interval_stream, 29 | }, 30 | features::{args::Args, running::Runnable, stack::StackLike}, 31 | }; 32 | 33 | /// M3u8 task handler 34 | pub struct M3u8Handler<'a> { 35 | output: PathBuf, 36 | method: Method, 37 | url: Url, 38 | data: Option<&'a str>, 39 | concurrency: u64, 40 | timeout: Duration, 41 | client: HttpClient, 42 | } 43 | 44 | impl<'a> M3u8Handler<'a> { 45 | pub fn new(args: &impl Args) -> Result { 46 | let headers = args.headers(); 47 | let timeout = args.timeout(); 48 | let dns_timeout = args.dns_timeout(); 49 | let keep_alive = args.keep_alive(); 50 | let skip_verify_tls_cert = args.skip_verify_tls_cert(); 51 | let proxy = args.proxy(); 52 | 53 | let client = build_http_client(&headers, timeout, dns_timeout, keep_alive, skip_verify_tls_cert, proxy)?; 54 | 55 | tracing::debug!("M3u8Handler::new"); 56 | 57 | Ok(M3u8Handler { 58 | output: args.output(), 59 | method: args.method(), 60 | url: args.url(), 61 | data: args.data(), 62 | concurrency: args.concurrency(), 63 | timeout, 64 | client, 65 | }) 66 | } 67 | 68 | async fn start(self) -> Result<()> { 69 | tracing::debug!("M3u8Handler::start"); 70 | 71 | // 0. Check whether task is completed 72 | tracing::debug!("M3u8Handler: check whether task is completed"); 73 | let mut bytearrayrecorder = ByteArrayRecorder::new(&*(self.output.to_string_lossy() + RECORDER_FILE_SUFFIX))?; 74 | if self.output.exists() && !bytearrayrecorder.exists() { 75 | return Ok(()); 76 | } 77 | 78 | // 1. Get m3u8 info 79 | tracing::debug!("M3u8Handler: get m3u8"); 80 | let mut ls = get_m3u8( 81 | &self.client, 82 | self.method.clone(), 83 | self.url.clone(), 84 | self.data.map(|v| v.to_string()), 85 | ) 86 | .await?; 87 | ls.reverse(); 88 | 89 | // 2. Check recorder status 90 | if bytearrayrecorder.exists() { 91 | bytearrayrecorder.open()?; 92 | let total = bytearrayrecorder.index(0)?; 93 | if total != ls.len() as u64 { 94 | return Err(Error::PartsAreNotConsistent); 95 | } else { 96 | let index = bytearrayrecorder.index(1)?; 97 | ls.truncate((total - index) as usize); 98 | } 99 | } else { 100 | bytearrayrecorder.open()?; 101 | // Write total 102 | bytearrayrecorder.write(0, ls.len() as u64)?; 103 | } 104 | 105 | // Use atomic u64 to control the order of sending segment content 106 | let index = ls.last().unwrap().index; 107 | let sharedindex = Rc::new(Cell::new(index)); 108 | let stack = SharedM3u8SegmentList::new(ls); 109 | tracing::debug!("M3u8Handler: segments: {}", stack.len()); 110 | 111 | // 3. Create channel 112 | let (sender, receiver) = channel::<(u64, Bytes)>(self.concurrency as usize + 10); 113 | let runtime_error: Arc>> = Arc::new(Mutex::new(None)); 114 | 115 | // 4. Spawn request task 116 | let concurrency = std::cmp::min(stack.len() as u64, self.concurrency); 117 | for i in 1..concurrency + 1 { 118 | let mut task = RequestTask::new( 119 | self.client.clone(), 120 | stack.clone(), 121 | sender.clone(), 122 | i, 123 | sharedindex.clone(), 124 | self.timeout, 125 | ); 126 | let runtime_error_clone = runtime_error.clone(); 127 | actix_rt::spawn(async move { 128 | if let Err(err) = task.start().await { 129 | if runtime_error_clone.lock().unwrap().is_none() { 130 | *runtime_error_clone.lock().unwrap() = Some(err); 131 | } 132 | } 133 | }); 134 | } 135 | drop(sender); // Remove the reference and let `Task` to handle it 136 | 137 | // 5. Create receiver 138 | tracing::debug!("M3u8Handler: create receiver"); 139 | let mut m3u8receiver = M3u8Receiver::new(&self.output)?; 140 | m3u8receiver.start(receiver).await?; 141 | 142 | if let Some(err) = runtime_error.lock().unwrap().take() { 143 | return Err(err); 144 | } 145 | 146 | // 6. Fixup output file 147 | 148 | // 7. Task succeeds. Remove `ByteArrayRecorder` file 149 | bytearrayrecorder.remove().unwrap_or(()); // Missing error 150 | Ok(()) 151 | } 152 | } 153 | 154 | impl<'a> Runnable for M3u8Handler<'a> { 155 | fn run(self) -> Result<()> { 156 | let sys = actix_rt::System::new(); 157 | sys.block_on(self.start()) 158 | } 159 | } 160 | 161 | /// Request the resource with a range header which is in the `SharedRangList` 162 | struct RequestTask { 163 | client: HttpClient, 164 | stack: SharedM3u8SegmentList, 165 | sender: Sender<(u64, Bytes)>, 166 | id: u64, 167 | shared_index: Rc>, 168 | timeout: Duration, 169 | } 170 | 171 | impl RequestTask { 172 | fn new( 173 | client: HttpClient, 174 | stack: SharedM3u8SegmentList, 175 | sender: Sender<(u64, Bytes)>, 176 | id: u64, 177 | sharedindex: Rc>, 178 | timeout: Duration, 179 | ) -> RequestTask { 180 | RequestTask { 181 | client, 182 | stack, 183 | sender, 184 | id, 185 | shared_index: sharedindex, 186 | timeout, 187 | } 188 | } 189 | 190 | async fn start(&mut self) -> Result<()> { 191 | tracing::debug!("Fire RequestTask: {}", self.id); 192 | while let Some(segment) = self.stack.pop() { 193 | loop { 194 | match self.req(segment.clone()).await { 195 | // Exit whole process when `Error::InnerError` is returned 196 | Err(Error::InnerError(msg)) => { 197 | tracing::error!("RequestTask {}: InnerError: {}", self.id, msg); 198 | actix_rt::System::current().stop(); 199 | } 200 | Err(err @ Error::Timeout) => { 201 | tracing::debug!("RequestTask timeout: {:?}", err); // Missing Timeout at runtime 202 | } 203 | Err(err) => { 204 | tracing::debug!("RequestTask {}: error: {:?}", self.id, err); 205 | return Err(err); 206 | } 207 | _ => break, 208 | } 209 | } 210 | } 211 | Ok(()) 212 | } 213 | 214 | async fn req(&mut self, segment: M3u8Segment) -> Result<()> { 215 | let resp = request( 216 | &self.client, 217 | segment.method.clone(), 218 | segment.url.clone(), 219 | segment.data.clone(), 220 | None, 221 | ) 222 | .await?; 223 | 224 | let index = segment.index; 225 | 226 | // !!! resp.body().await can be overflow 227 | let mut buf: Vec = vec![]; 228 | 229 | let stream = resp.bytes_stream().fuse(); 230 | 231 | // Set timeout for reading 232 | let tick = interval_stream(self.timeout).fuse(); 233 | 234 | pin_mut!(stream, tick); 235 | let mut fire = false; 236 | loop { 237 | select! { 238 | item = stream.next() => { 239 | if let Some(item) = item { 240 | match item { 241 | Ok(chunk) => { 242 | buf.extend(chunk); 243 | } 244 | Err(err) => return Err(err.into()), 245 | } 246 | } else { 247 | break; 248 | } 249 | } 250 | _ = tick.next() => { 251 | if fire { 252 | return Err(Error::Timeout); 253 | } else { 254 | fire = true; 255 | } 256 | } 257 | } 258 | } 259 | 260 | // Decrypt ase128 encoded 261 | let de = if let (Some(key), Some(iv)) = (segment.key, segment.iv) { 262 | decrypt_aes128(&key[..], &iv[..], buf.as_ref())? 263 | } else { 264 | buf.to_vec() 265 | }; 266 | 267 | loop { 268 | if self.shared_index.get() == index { 269 | if let Err(err) = self.sender.send((index, Bytes::from(de))).await { 270 | return Err(Error::InnerError(format!( 271 | "Error at `http::RequestTask`: Sender error: {:?}", 272 | err 273 | ))); 274 | } 275 | self.shared_index.set(index + 1); 276 | return Ok(()); 277 | } else { 278 | actix_rt::time::sleep(Duration::from_millis(500)).await; 279 | } 280 | } 281 | } 282 | } 283 | -------------------------------------------------------------------------------- /src/app/core/m3u8/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod common; 2 | pub mod m3u8; 3 | 4 | pub use m3u8::M3u8Handler; 5 | -------------------------------------------------------------------------------- /src/app/core/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod bt; 2 | pub mod http; 3 | pub mod m3u8; 4 | -------------------------------------------------------------------------------- /src/app/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod core; 2 | pub mod receive; 3 | pub mod record; 4 | pub mod show; 5 | pub mod status; 6 | -------------------------------------------------------------------------------- /src/app/receive/http_receiver.rs: -------------------------------------------------------------------------------- 1 | use std::{io::SeekFrom, path::Path, time::Duration}; 2 | 3 | use futures::{channel::mpsc::Receiver, pin_mut, select, StreamExt}; 4 | 5 | use crate::{ 6 | app::{ 7 | record::{common::RECORDER_FILE_SUFFIX, range_recorder::RangeRecorder}, 8 | show::http_show::HttpShower, 9 | status::rate_status::RateStatus, 10 | }, 11 | common::{bytes::bytes_type::Bytes, errors::Result, file::File, range::RangePair, time::interval_stream}, 12 | }; 13 | 14 | pub struct HttpReceiver { 15 | output: File, 16 | rangerecorder: Option, 17 | ratestatus: RateStatus, 18 | shower: HttpShower, 19 | // Total content length of the uri 20 | total: u64, 21 | } 22 | 23 | impl HttpReceiver { 24 | pub fn new>(output: P, direct: bool, content_length: u64) -> Result { 25 | let mut outputfile = File::new(&output, true)?; 26 | outputfile.open()?; 27 | 28 | let (rangerecorder, total, completed) = if direct { 29 | (None, content_length, 0) 30 | } else { 31 | let mut rangerecorder = RangeRecorder::new(&*(output.as_ref().to_string_lossy() + RECORDER_FILE_SUFFIX))?; 32 | rangerecorder.open()?; 33 | let total = rangerecorder.total()?; 34 | let completed = rangerecorder.count()?; 35 | (Some(rangerecorder), total, completed) 36 | }; 37 | 38 | let mut ratestatus = RateStatus::new(); 39 | ratestatus.set_total(completed); 40 | 41 | Ok(HttpReceiver { 42 | output: outputfile, 43 | rangerecorder, 44 | ratestatus, 45 | shower: HttpShower::new(), 46 | // receiver, 47 | total, 48 | }) 49 | } 50 | 51 | fn show_infos(&mut self) -> Result<()> { 52 | if self.rangerecorder.is_none() { 53 | self.shower.print_msg("Server doesn't support range request.")?; 54 | } 55 | 56 | let file_name = &self.output.file_name().unwrap_or("[No Name]"); 57 | let total = self.total; 58 | self.shower.print_file(file_name)?; 59 | self.shower.print_total(total)?; 60 | // self.shower.print_concurrency(concurrency)?; 61 | self.show_status()?; 62 | Ok(()) 63 | } 64 | 65 | fn show_status(&mut self) -> Result<()> { 66 | let total = self.total; 67 | let completed = self.ratestatus.total(); 68 | let rate = self.ratestatus.rate(); 69 | 70 | let eta = if self.rangerecorder.is_some() || self.total != 0 { 71 | let remains = total - completed; 72 | // rate > 1.0 for overflow 73 | if remains > 0 && rate > 1.0 { 74 | let eta = (remains as f64 / rate) as u64; 75 | // eta is large than 99 days, return 0 76 | if eta > 99 * 24 * 60 * 60 { 77 | 0 78 | } else { 79 | eta 80 | } 81 | } else { 82 | 0 83 | } 84 | } else { 85 | 0 86 | }; 87 | 88 | self.shower.print_status(completed, total, rate, eta)?; 89 | self.ratestatus.clean(); 90 | Ok(()) 91 | } 92 | 93 | fn record_pair(&mut self, pair: RangePair) -> Result<()> { 94 | if let Some(ref mut rangerecorder) = self.rangerecorder { 95 | rangerecorder.write_pair(pair)?; 96 | } 97 | Ok(()) 98 | } 99 | pub async fn start(&mut self, receiver: Receiver<(RangePair, Bytes)>) -> Result<()> { 100 | self.show_infos()?; 101 | 102 | let receiver = receiver.fuse(); 103 | let tick = interval_stream(Duration::from_secs(2)).fuse(); 104 | 105 | pin_mut!(receiver, tick); 106 | 107 | loop { 108 | select! { 109 | item = receiver.next() => { 110 | if let Some((pair, chunk)) = item { 111 | self.output.write(&chunk[..], Some(SeekFrom::Start(pair.begin)))?; 112 | self.record_pair(pair)?; 113 | self.ratestatus.add(pair.length()); 114 | } else { 115 | break; 116 | } 117 | }, 118 | _ = tick.next() => { 119 | self.show_status()?; 120 | }, 121 | } 122 | } 123 | self.show_status()?; 124 | Ok(()) 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /src/app/receive/m3u8_receiver.rs: -------------------------------------------------------------------------------- 1 | use std::{io::SeekFrom, path::Path, time::Duration}; 2 | 3 | use futures::{channel::mpsc::Receiver, pin_mut, select, StreamExt}; 4 | 5 | use crate::{ 6 | app::{ 7 | record::{bytearray_recorder::ByteArrayRecorder, common::RECORDER_FILE_SUFFIX}, 8 | show::m3u8_show::M3u8Shower, 9 | status::rate_status::RateStatus, 10 | }, 11 | common::{bytes::bytes_type::Bytes, errors::Result, file::File, time::interval_stream}, 12 | }; 13 | 14 | pub struct M3u8Receiver { 15 | output: File, 16 | bytearrayrecorder: ByteArrayRecorder, 17 | ratestatus: RateStatus, 18 | shower: M3u8Shower, 19 | total: u64, 20 | completed: u64, 21 | seek: u64, 22 | } 23 | 24 | impl M3u8Receiver { 25 | pub fn new>(output: P) -> Result { 26 | let mut outputfile = File::new(&output, true)?; 27 | outputfile.open()?; 28 | 29 | // Record 3 variables in a `ByteArrayRecorder`: 30 | // [0-index, total segment number][1-index, completed segment number][2-index, seek offset] 31 | let mut bytearrayrecorder = 32 | ByteArrayRecorder::new(&*(output.as_ref().to_string_lossy() + RECORDER_FILE_SUFFIX))?; 33 | bytearrayrecorder.open()?; 34 | let total = bytearrayrecorder.index(0)?; 35 | let completed = bytearrayrecorder.index(1)?; 36 | let seek = bytearrayrecorder.index(2)?; 37 | 38 | Ok(M3u8Receiver { 39 | output: outputfile, 40 | bytearrayrecorder, 41 | ratestatus: RateStatus::new(), 42 | shower: M3u8Shower::new(), 43 | total, 44 | completed, 45 | seek, 46 | }) 47 | } 48 | 49 | fn show_infos(&mut self) -> Result<()> { 50 | let file_name = &self.output.file_name().unwrap_or("[No Name]"); 51 | let total = self.total; 52 | self.shower.print_file(file_name)?; 53 | self.shower.print_total(total)?; 54 | self.show_status()?; 55 | Ok(()) 56 | } 57 | 58 | fn show_status(&mut self) -> Result<()> { 59 | let total = self.total; 60 | let completed = self.completed; 61 | let rate = self.ratestatus.rate(); 62 | let length = self.seek; 63 | 64 | self.shower.print_status(completed, total, length, rate)?; 65 | self.ratestatus.clean(); 66 | Ok(()) 67 | } 68 | 69 | pub async fn start(&mut self, receiver: Receiver<(u64, Bytes)>) -> Result<()> { 70 | self.show_infos()?; 71 | 72 | let receiver = receiver.fuse(); 73 | let tick = interval_stream(Duration::from_secs(2)).fuse(); 74 | 75 | pin_mut!(receiver, tick); 76 | 77 | loop { 78 | select! { 79 | item = receiver.next() => { 80 | if let Some((index, chunk)) = item { 81 | let len = chunk.len() as u64; 82 | 83 | // Write chunk to file 84 | self.output.write(&chunk[..], Some(SeekFrom::Start(self.seek)))?; 85 | 86 | // Record info 87 | self.bytearrayrecorder.write(1, index + 1)?; // Write completed 88 | self.bytearrayrecorder.write(2, self.seek + len)?; // Write seek offset 89 | self.completed = index + 1; 90 | self.seek += len ; 91 | 92 | // Update rate 93 | self.ratestatus.add(len); 94 | } else { 95 | break; 96 | } 97 | }, 98 | _ = tick.next() => { 99 | self.show_status()?; 100 | }, 101 | } 102 | } 103 | self.show_status()?; 104 | Ok(()) 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/app/receive/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod http_receiver; 2 | pub mod m3u8_receiver; 3 | -------------------------------------------------------------------------------- /src/app/record/bytearray_recorder.rs: -------------------------------------------------------------------------------- 1 | use std::{io::SeekFrom, path::Path}; 2 | 3 | use crate::common::{ 4 | bytes::bytes::{u64_to_u8x8, u8x8_to_u64}, 5 | errors::Result, 6 | file::File, 7 | }; 8 | 9 | /// Byte array recorder 10 | /// 11 | /// `ByteArrayRecorder` struct records a list u64/ numbers. 12 | /// All information is stored at a local file. 13 | /// 14 | /// [8bit][8bit][8bit] 15 | /// `total` is given by user, presenting as the real total number of items of a list. 16 | pub struct ByteArrayRecorder { 17 | inner: File, 18 | } 19 | 20 | impl ByteArrayRecorder { 21 | pub fn new>(path: P) -> Result { 22 | let inner = File::new(path, true)?; 23 | Ok(ByteArrayRecorder { inner }) 24 | } 25 | 26 | pub fn open(&mut self) -> Result<&mut Self> { 27 | self.inner.open()?; 28 | Ok(self) 29 | } 30 | 31 | pub fn file_name(&self) -> Option<&str> { 32 | self.inner.file_name() 33 | } 34 | 35 | pub fn exists(&self) -> bool { 36 | self.inner.exists() 37 | } 38 | 39 | /// Delete the inner file 40 | pub fn remove(&self) -> Result<()> { 41 | self.inner.remove() 42 | } 43 | 44 | /// Read the index-th number 45 | pub fn index(&mut self, index: u64) -> Result { 46 | let mut buf: [u8; 8] = [0; 8]; 47 | self.inner.read(&mut buf, Some(SeekFrom::Start(index * 8)))?; 48 | Ok(u8x8_to_u64(&buf)) 49 | } 50 | 51 | pub fn write(&mut self, index: u64, num: u64) -> Result<()> { 52 | let buf = u64_to_u8x8(num); 53 | self.inner.write(&buf, Some(SeekFrom::Start(index * 8)))?; 54 | Ok(()) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/app/record/common.rs: -------------------------------------------------------------------------------- 1 | pub const RECORDER_FILE_SUFFIX: &str = ".rc.aget"; 2 | -------------------------------------------------------------------------------- /src/app/record/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod bytearray_recorder; 2 | pub mod common; 3 | pub mod range_recorder; 4 | -------------------------------------------------------------------------------- /src/app/record/range_recorder.rs: -------------------------------------------------------------------------------- 1 | use std::{cmp::max, io::SeekFrom, path::Path}; 2 | 3 | use crate::common::{ 4 | bytes::bytes::{u64_to_u8x8, u8x8_to_u64}, 5 | errors::Result, 6 | file::File, 7 | range::{RangeList, RangePair}, 8 | }; 9 | 10 | /// Range recorder 11 | /// 12 | /// This struct records pairs which are `common::range::RangePair`. 13 | /// All information is stored at a local file. 14 | /// 15 | /// [total 8bit][ [begin1 8bit,end1 8bit] [begin2 8bit,end2 8bit] ... ] 16 | /// `total` position is not sum_i{end_i - begin_i + 1}. It is given by 17 | /// user, presenting as the real total number. 18 | pub struct RangeRecorder { 19 | inner: File, 20 | } 21 | 22 | impl RangeRecorder { 23 | pub fn new>(path: P) -> Result { 24 | let inner = File::new(path, true)?; 25 | Ok(RangeRecorder { inner }) 26 | } 27 | 28 | pub fn open(&mut self) -> Result<&mut Self> { 29 | self.inner.open()?; 30 | Ok(self) 31 | } 32 | 33 | pub fn file_name(&self) -> Option<&str> { 34 | self.inner.file_name() 35 | } 36 | 37 | pub fn exists(&self) -> bool { 38 | self.inner.exists() 39 | } 40 | 41 | /// Delete the inner file 42 | pub fn remove(&self) -> Result<()> { 43 | self.inner.remove() 44 | } 45 | 46 | /// Get downloading file's content length stored in the aget file 47 | pub fn total(&mut self) -> Result { 48 | let mut buf: [u8; 8] = [0; 8]; 49 | self.inner.read(&mut buf, Some(SeekFrom::Start(0)))?; 50 | let cl = u8x8_to_u64(&buf); 51 | Ok(cl) 52 | } 53 | 54 | /// Count the length of total pairs 55 | pub fn count(&mut self) -> Result { 56 | let pairs = self.pairs()?; 57 | if pairs.is_empty() { 58 | Ok(0) 59 | } else { 60 | Ok(pairs.iter().map(RangePair::length).sum()) 61 | } 62 | } 63 | 64 | /// Recorded pairs 65 | pub fn pairs(&mut self) -> Result { 66 | let mut pairs: Vec<(u64, u64)> = Vec::new(); 67 | 68 | let mut buf: [u8; 16] = [0; 16]; 69 | self.inner.seek(SeekFrom::Start(8))?; 70 | loop { 71 | let s = self.inner.read(&mut buf, None)?; 72 | if s != 16 { 73 | break; 74 | } 75 | 76 | let mut raw = [0; 8]; 77 | raw.clone_from_slice(&buf[..8]); 78 | let begin = u8x8_to_u64(&raw); 79 | raw.clone_from_slice(&buf[8..]); 80 | let end = u8x8_to_u64(&raw); 81 | 82 | assert!( 83 | begin <= end, 84 | "Bug: `begin > end` in an pair of {}. : {} > {}", 85 | self.file_name().unwrap_or(""), 86 | begin, 87 | end 88 | ); 89 | 90 | pairs.push((begin, end)); 91 | } 92 | 93 | pairs.sort_unstable(); 94 | 95 | // merge pairs 96 | let mut merged_pairs: Vec<(u64, u64)> = Vec::new(); 97 | if !pairs.is_empty() { 98 | merged_pairs.push(pairs[0]); 99 | } 100 | for (begin, end) in pairs.iter() { 101 | let (pre_start, pre_end) = *merged_pairs.last().unwrap(); 102 | 103 | // case 1 104 | // ---------- 105 | // ----------- 106 | if pre_end + 1 < *begin { 107 | merged_pairs.push((*begin, *end)); 108 | // case 2 109 | // ----------------- 110 | // ---------- 111 | // -------- 112 | // ------ 113 | } else { 114 | let n_start = pre_start; 115 | let n_end = max(pre_end, *end); 116 | merged_pairs.pop(); 117 | merged_pairs.push((n_start, n_end)); 118 | } 119 | } 120 | 121 | Ok(merged_pairs 122 | .iter() 123 | .map(|(begin, end)| RangePair::new(*begin, *end)) 124 | .collect::()) 125 | } 126 | 127 | /// Get gaps between all pairs 128 | /// Each of gap is a closed interval 129 | pub fn gaps(&mut self) -> Result { 130 | let mut pairs = self.pairs()?; 131 | let total = self.total()?; 132 | pairs.push(RangePair::new(total, total)); 133 | 134 | // find gaps 135 | let mut gaps: RangeList = Vec::new(); 136 | // find first chunk 137 | let RangePair { begin, .. } = pairs[0]; 138 | if begin > 0 { 139 | gaps.push(RangePair::new(0, begin - 1)); 140 | } 141 | 142 | for (index, RangePair { end, .. }) in pairs.iter().enumerate() { 143 | if let Some(RangePair { begin: next_start, .. }) = pairs.get(index + 1) { 144 | if end + 1 < *next_start { 145 | gaps.push(RangePair::new(end + 1, next_start - 1)); 146 | } 147 | } 148 | } 149 | 150 | Ok(gaps) 151 | } 152 | 153 | pub fn write_total(&mut self, total: u64) -> Result<()> { 154 | let buf = u64_to_u8x8(total); 155 | self.inner.write(&buf, Some(SeekFrom::Start(0)))?; 156 | Ok(()) 157 | } 158 | 159 | pub fn write_pair(&mut self, pair: RangePair) -> Result<()> { 160 | let begin = u64_to_u8x8(pair.begin); 161 | let end = u64_to_u8x8(pair.end); 162 | let buf = [begin, end].concat(); 163 | self.inner.write(&buf, Some(SeekFrom::End(0)))?; 164 | Ok(()) 165 | } 166 | 167 | // Merge completed pairs and rewrite the aget file 168 | pub fn rewrite(&mut self) -> Result<()> { 169 | let total = self.total()?; 170 | let pairs = self.pairs()?; 171 | 172 | let mut buf: Vec = Vec::new(); 173 | buf.extend(&u64_to_u8x8(total)); 174 | for pair in pairs.iter() { 175 | buf.extend(&u64_to_u8x8(pair.begin)); 176 | buf.extend(&u64_to_u8x8(pair.end)); 177 | } 178 | 179 | // Clean all content of the file and set its length to zero 180 | self.inner.set_len(0)?; 181 | 182 | // Write new data 183 | self.inner.write(buf.as_slice(), Some(SeekFrom::Start(0)))?; 184 | 185 | Ok(()) 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /src/app/show/bt_show.rs: -------------------------------------------------------------------------------- 1 | use std::io::{stdout, Stdout, Write}; 2 | 3 | use crate::{ 4 | app::show::common::du_bars, 5 | common::{ 6 | colors::{Black, Blue, Cyan, Green, Red, Yellow, RGB}, 7 | errors::Result, 8 | liberal::ToDate, 9 | size::HumanReadable, 10 | terminal::terminal_width, 11 | }, 12 | }; 13 | 14 | pub struct BtShower { 15 | stdout: Stdout, 16 | } 17 | 18 | impl BtShower { 19 | pub fn new() -> BtShower { 20 | BtShower { stdout: stdout() } 21 | } 22 | 23 | pub fn print_msg(&mut self, msg: &str) -> Result<()> { 24 | writeln!(&mut self.stdout, "\n {}", Yellow.italic().paint(msg))?; 25 | Ok(()) 26 | } 27 | 28 | pub fn print_name(&mut self, name: &str) -> Result<()> { 29 | writeln!(&mut self.stdout, "\n{}: {}", Green.bold().paint("Torrent Name"), name,)?; 30 | Ok(()) 31 | } 32 | 33 | pub fn print_files(&mut self, files: &[(&str, u64, bool)]) -> Result<()> { 34 | for (filename, length, included) in files.iter() { 35 | writeln!( 36 | &mut self.stdout, 37 | "{} {}: {} ({})", 38 | if *included { 39 | Green.bold().paint("✓") 40 | } else { 41 | Red.bold().paint("✘") 42 | }, 43 | Blue.bold().paint("File"), 44 | filename, 45 | length.human_readable(), 46 | )?; 47 | } 48 | Ok(()) 49 | } 50 | 51 | pub fn print_status( 52 | &mut self, 53 | completed: u64, 54 | total: u64, 55 | eta: u64, 56 | down_rate: f64, 57 | up_rate: f64, 58 | uploaded: u64, 59 | live: usize, 60 | queued: usize, 61 | ) -> Result<()> { 62 | let percent = if total != 0 { 63 | completed as f64 / total as f64 64 | } else { 65 | 0.0 66 | }; 67 | 68 | let completed_str = completed.human_readable(); 69 | let total_str = total.human_readable(); 70 | let percent_str = format!("{:.2}", percent * 100.0); 71 | let down_rate_str = down_rate.human_readable(); 72 | let up_rate_str = up_rate.human_readable(); 73 | let uploaded_str = uploaded.human_readable(); 74 | let eta_str = eta.date(); 75 | 76 | // maximum info length is 41 e.g. 77 | // 571.9M/5.8G 9.63% ↓1.8M/s ↑192.1K/s(63.7M) eta: 49m peers: 22/102 78 | let info = format!( 79 | "{completed}/{total} {percent}% ↓{down_rate}/s ↑{up_rate}/s({uploaded}) eta: {eta} peers: {live}/{queued}", 80 | completed = completed_str, 81 | total = total_str, 82 | percent = percent_str, 83 | down_rate = down_rate_str, 84 | up_rate = up_rate_str, 85 | uploaded = uploaded_str, 86 | eta = eta_str, 87 | ); 88 | 89 | // set default info length 90 | let info_length = 80; 91 | let mut miss = info_length - info.len(); 92 | 93 | let terminal_width = terminal_width(); 94 | let bar_length = if terminal_width > info_length as u64 + 3 { 95 | terminal_width - info_length as u64 - 3 96 | } else { 97 | miss = 0; 98 | 0 99 | }; 100 | 101 | let (bar_done_str, bar_undone_str) = if total != 0 { 102 | let bar_done_length = (bar_length as f64 * percent) as u64; 103 | let bar_undone_length = bar_length - bar_done_length; 104 | du_bars(bar_done_length as usize, bar_undone_length as usize) 105 | } else { 106 | (" ".repeat(bar_length as usize), "".to_owned()) 107 | }; 108 | 109 | write!( 110 | &mut self.stdout, 111 | "\r{completed}/{total} {percent}% ↓{down_rate}/s ↑{up_rate}/s({uploaded}) eta: {eta} peers: {live}/{queued}{miss} {bar_done}{bar_undone} ", 112 | completed = Red.bold().paint(completed_str), 113 | total = Green.bold().paint(total_str), 114 | percent = Yellow.bold().paint(percent_str), 115 | down_rate = Blue.bold().paint(down_rate_str), 116 | up_rate = RGB(0x66, 0x00, 0xcc).bold().paint(up_rate_str), 117 | uploaded = uploaded_str, 118 | eta = Cyan.bold().paint(eta_str), 119 | miss = " ".repeat(miss), 120 | bar_done = if total != 0 { 121 | Red.bold().paint(bar_done_str).to_string() 122 | } else { 123 | bar_done_str 124 | }, 125 | bar_undone = if total != 0 { 126 | Black.bold().paint(bar_undone_str).to_string() 127 | } else { 128 | bar_undone_str 129 | } 130 | )?; 131 | self.stdout.flush()?; 132 | Ok(()) 133 | } 134 | 135 | pub fn print_completed_file(&mut self, name: &str) -> Result<()> { 136 | writeln!(&mut self.stdout, "\n{}: {}", Green.italic().paint("Completed"), name,)?; 137 | self.stdout.flush()?; 138 | Ok(()) 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /src/app/show/common.rs: -------------------------------------------------------------------------------- 1 | #[cfg(target_os = "windows")] 2 | pub fn bars() -> (&'static str, &'static str, &'static str) { 3 | // bar, bar_right, bar_left 4 | ("▓", "░", " ") 5 | } 6 | 7 | #[cfg(not(target_os = "windows"))] 8 | pub fn bars() -> (&'static str, &'static str, &'static str) { 9 | // bar, bar_right, bar_left 10 | ("━", "╸", "╺") 11 | } 12 | 13 | /// Return done and undone bars' string 14 | pub fn du_bars(bar_done_length: usize, bar_undone_length: usize) -> (String, String) { 15 | let (bar, bar_right, bar_left) = bars(); 16 | 17 | let bar_done_str = if bar_done_length > 0 { 18 | if bar_undone_length > 0 { 19 | bar.repeat((bar_done_length - 1) as usize) + bar_right 20 | } else { 21 | // Remove bar_right when bar_undone_length is zero 22 | bar.repeat(bar_done_length as usize) 23 | } 24 | } else { 25 | "".to_owned() 26 | }; 27 | 28 | let bar_undone_str = if bar_undone_length > 0 { 29 | bar_left.to_owned() + &bar.repeat(bar_undone_length as usize - 1) 30 | } else { 31 | "".to_owned() 32 | }; 33 | 34 | (bar_done_str, bar_undone_str) 35 | } 36 | -------------------------------------------------------------------------------- /src/app/show/http_show.rs: -------------------------------------------------------------------------------- 1 | use std::io::{stdout, Stdout, Write}; 2 | 3 | use crate::{ 4 | app::show::common::du_bars, 5 | common::{ 6 | colors::{Black, Blue, Cyan, Green, Red, Yellow}, 7 | errors::Result, 8 | liberal::ToDate, 9 | size::HumanReadable, 10 | terminal::terminal_width, 11 | }, 12 | }; 13 | 14 | pub struct HttpShower { 15 | stdout: Stdout, 16 | } 17 | 18 | impl HttpShower { 19 | pub fn new() -> HttpShower { 20 | HttpShower { stdout: stdout() } 21 | } 22 | 23 | pub fn print_msg(&mut self, msg: &str) -> Result<()> { 24 | writeln!(&mut self.stdout, "\n {}", Yellow.italic().paint(msg))?; 25 | Ok(()) 26 | } 27 | 28 | pub fn print_file(&mut self, path: &str) -> Result<()> { 29 | writeln!(&mut self.stdout, "\n{}: {}", Green.bold().paint("File"), path,)?; 30 | Ok(()) 31 | } 32 | 33 | pub fn print_total(&mut self, total: u64) -> Result<()> { 34 | writeln!( 35 | &mut self.stdout, 36 | "{}: {} ({})", 37 | Blue.bold().paint("Length"), 38 | total.human_readable(), 39 | total, 40 | )?; 41 | Ok(()) 42 | } 43 | 44 | pub fn print_concurrency(&mut self, concurrency: u64) -> Result<()> { 45 | writeln!( 46 | &mut self.stdout, 47 | "{}: {}\n", 48 | Yellow.bold().paint("concurrency"), 49 | concurrency, 50 | )?; 51 | Ok(()) 52 | } 53 | 54 | pub fn print_status(&mut self, completed: u64, total: u64, rate: f64, eta: u64) -> Result<()> { 55 | let percent = if total != 0 { 56 | completed as f64 / total as f64 57 | } else { 58 | 0.0 59 | }; 60 | 61 | let completed_str = completed.human_readable(); 62 | let total_str = total.human_readable(); 63 | let percent_str = format!("{:.2}", percent * 100.0); 64 | let rate_str = rate.human_readable(); 65 | let eta_str = eta.date(); 66 | 67 | // maximum info length is 41 e.g. 68 | // 1001.3k/1021.9m 97.98% 1003.1B/s eta: 12s 69 | let info = format!( 70 | "{completed}/{total} {percent}% {rate}/s eta: {eta}", 71 | completed = completed_str, 72 | total = total_str, 73 | percent = percent_str, 74 | rate = rate_str, 75 | eta = eta_str, 76 | ); 77 | 78 | // set default info length 79 | let info_length = 41; 80 | let mut miss = info_length - info.len(); 81 | 82 | let terminal_width = terminal_width(); 83 | let bar_length = if terminal_width > info_length as u64 + 3 { 84 | terminal_width - info_length as u64 - 3 85 | } else { 86 | miss = 0; 87 | 0 88 | }; 89 | 90 | let (bar_done_str, bar_undone_str) = if total != 0 { 91 | let bar_done_length = (bar_length as f64 * percent) as u64; 92 | let bar_undone_length = bar_length - bar_done_length; 93 | du_bars(bar_done_length as usize, bar_undone_length as usize) 94 | } else { 95 | (" ".repeat(bar_length as usize), "".to_owned()) 96 | }; 97 | 98 | write!( 99 | &mut self.stdout, 100 | "\r{completed}/{total} {percent}% {rate}/s eta: {eta}{miss} {bar_done}{bar_undone} ", 101 | completed = Red.bold().paint(completed_str), 102 | total = Green.bold().paint(total_str), 103 | percent = Yellow.bold().paint(percent_str), 104 | rate = Blue.bold().paint(rate_str), 105 | eta = Cyan.bold().paint(eta_str), 106 | miss = " ".repeat(miss), 107 | bar_done = if total != 0 { 108 | Red.bold().paint(bar_done_str).to_string() 109 | } else { 110 | bar_done_str 111 | }, 112 | bar_undone = if total != 0 { 113 | Black.bold().paint(bar_undone_str).to_string() 114 | } else { 115 | bar_undone_str 116 | } 117 | )?; 118 | 119 | self.stdout.flush()?; 120 | 121 | Ok(()) 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/app/show/m3u8_show.rs: -------------------------------------------------------------------------------- 1 | use std::io::{stdout, Stdout, Write}; 2 | 3 | use crate::{ 4 | app::show::common::du_bars, 5 | common::{ 6 | colors::{Black, Blue, Green, Red, Yellow}, 7 | errors::Result, 8 | size::HumanReadable, 9 | terminal::terminal_width, 10 | }, 11 | }; 12 | 13 | pub struct M3u8Shower { 14 | stdout: Stdout, 15 | } 16 | 17 | impl M3u8Shower { 18 | pub fn new() -> M3u8Shower { 19 | M3u8Shower { stdout: stdout() } 20 | } 21 | 22 | pub fn print_msg(&mut self, msg: &str) -> Result<()> { 23 | writeln!(&mut self.stdout, "\n {}", Yellow.italic().paint(msg))?; 24 | Ok(()) 25 | } 26 | 27 | pub fn print_file(&mut self, path: &str) -> Result<()> { 28 | writeln!( 29 | &mut self.stdout, 30 | // "\n {}: {}", 31 | "\n{}: {}", 32 | Green.bold().paint("File"), 33 | path, 34 | )?; 35 | Ok(()) 36 | } 37 | 38 | pub fn print_total(&mut self, total: u64) -> Result<()> { 39 | writeln!(&mut self.stdout, "{}: {}", Blue.bold().paint("Segments"), total,)?; 40 | Ok(()) 41 | } 42 | 43 | pub fn print_concurrency(&mut self, concurrency: u64) -> Result<()> { 44 | writeln!( 45 | &mut self.stdout, 46 | "{}: {}\n", 47 | Yellow.bold().paint("concurrency"), 48 | concurrency, 49 | )?; 50 | Ok(()) 51 | } 52 | 53 | pub fn print_status(&mut self, completed: u64, total: u64, length: u64, rate: f64) -> Result<()> { 54 | let percent = completed as f64 / total as f64; 55 | 56 | let completed_str = completed.to_string(); 57 | let total_str = total.to_string(); 58 | let length_str = length.human_readable(); 59 | let percent_str = format!("{:.2}", percent * 100.0); 60 | let rate_str = rate.human_readable(); 61 | 62 | // maximum info length is `completed_str.len()` + `total_str.len()` + 26 63 | // e.g. 64 | // 100/1021 97.98% 10m 1003.1B/s eta: 12s 65 | let info = format!( 66 | "{completed}/{total} {length} {percent}% {rate}/s", 67 | completed = completed_str, 68 | total = total_str, 69 | length = length_str, 70 | percent = percent_str, 71 | rate = rate_str, 72 | ); 73 | 74 | // set default info length 75 | let info_length = total_str.len() * 2 + 26; 76 | let mut miss = info_length - info.len(); 77 | 78 | let terminal_width = terminal_width(); 79 | let bar_length = if terminal_width > info_length as u64 + 3 { 80 | terminal_width - info_length as u64 - 3 81 | } else { 82 | miss = 0; 83 | 0 84 | }; 85 | 86 | let bar_done_length = (bar_length as f64 * percent) as u64; 87 | let bar_undone_length = bar_length - bar_done_length; 88 | let (bar_done_str, bar_undone_str) = du_bars(bar_done_length as usize, bar_undone_length as usize); 89 | 90 | write!( 91 | &mut self.stdout, 92 | "\r{completed}/{total} {length} {percent}% {rate}/s{miss} {bar_done}{bar_undone} ", 93 | completed = Red.bold().paint(completed_str), 94 | total = Green.bold().paint(total_str), 95 | length = Red.bold().paint(length_str), 96 | percent = Yellow.bold().paint(percent_str), 97 | rate = Blue.bold().paint(rate_str), 98 | miss = " ".repeat(miss), 99 | bar_done = Red.bold().paint(bar_done_str), 100 | bar_undone = Black.bold().paint(bar_undone_str), 101 | )?; 102 | 103 | self.stdout.flush()?; 104 | 105 | Ok(()) 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/app/show/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod bt_show; 2 | pub mod common; 3 | pub mod http_show; 4 | pub mod m3u8_show; 5 | -------------------------------------------------------------------------------- /src/app/status/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod rate_status; 2 | -------------------------------------------------------------------------------- /src/app/status/rate_status.rs: -------------------------------------------------------------------------------- 1 | use std::time::Instant; 2 | 3 | /// `RateStatus` records the rate of adding number 4 | pub struct RateStatus { 5 | /// Total number 6 | total: u64, 7 | 8 | /// The number at an one tick interval 9 | count: u64, 10 | 11 | /// The interval of one tick 12 | tick: Instant, 13 | } 14 | 15 | impl RateStatus { 16 | pub fn new() -> RateStatus { 17 | RateStatus::default() 18 | } 19 | 20 | pub fn total(&self) -> u64 { 21 | self.total 22 | } 23 | 24 | pub fn set_total(&mut self, total: u64) { 25 | self.total = total; 26 | } 27 | 28 | pub fn count(&self) -> u64 { 29 | self.count 30 | } 31 | 32 | pub fn rate(&self) -> f64 { 33 | let interval = self.tick.elapsed().as_secs_f64(); 34 | self.count as f64 / interval 35 | } 36 | 37 | pub fn add(&mut self, incr: u64) { 38 | self.total += incr; 39 | self.count += incr; 40 | } 41 | 42 | pub fn reset(&mut self) { 43 | self.total = 0; 44 | self.count = 0; 45 | self.tick = Instant::now(); 46 | } 47 | 48 | pub fn clean(&mut self) { 49 | self.count = 0; 50 | self.tick = Instant::now(); 51 | } 52 | } 53 | 54 | impl Default for RateStatus { 55 | fn default() -> RateStatus { 56 | RateStatus { 57 | total: 0, 58 | count: 0, 59 | tick: Instant::now(), 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/arguments/clap_cli.rs: -------------------------------------------------------------------------------- 1 | use clap::Parser; 2 | 3 | #[derive(Parser, Debug)] 4 | #[clap(author, version, about, long_about = None)] 5 | pub struct AgetCli { 6 | pub url: String, 7 | 8 | #[clap( 9 | short, 10 | long, 11 | default_value_t = String::from("GET"), 12 | help = "Request method, e.g. GET, POST" 13 | )] 14 | pub method: String, 15 | 16 | #[clap(short = 'H', long, help = r#"Request headers, e.g. -H "User-Agent: aget""#)] 17 | pub header: Option>, 18 | 19 | #[clap(short, long, help = r#"Request with POST method with the data, e.g. -d "a=b""#)] 20 | pub data: Option, 21 | 22 | #[clap(long, help = "Skip to verify the server's TLS certificate")] 23 | pub insecure: bool, 24 | 25 | #[clap(short = 's', long, help = "The number of concurrency request [default: 10]")] 26 | pub concurrency: Option, 27 | 28 | #[clap( 29 | short = 'k', 30 | long, 31 | help = "The number ofinterval length of each concurrent request [default: '50m']" 32 | )] 33 | pub chunk_size: Option, 34 | 35 | #[clap(short, long, help = "Timeout(seconds) of request [default: 60]")] 36 | pub timeout: Option, 37 | 38 | #[clap(long, help = "DNS Timeout(seconds) of request [default: 10]")] 39 | pub dns_timeout: Option, 40 | 41 | #[clap(long, help = "The maximum times of retring [default: 5]")] 42 | pub retries: Option, 43 | 44 | #[clap(long, help = "The seconds between retries [default: 0]")] 45 | pub retry_wait: Option, 46 | 47 | #[clap(long = "proxy", name = "PROXY", help = "[protocol://]host[:port] Use this proxy")] 48 | pub proxy: Option, 49 | 50 | #[clap( 51 | long = "type", 52 | name = "TYPE", 53 | default_value = "auto", 54 | help = "Task type, auto/http/m3u8/bt" 55 | )] 56 | pub tp: String, 57 | 58 | #[clap(long, help = "A regex to only download files matching it in the torrent")] 59 | pub bt_file_regex: Option, 60 | 61 | #[clap(long, help = "Seed the torrent")] 62 | pub seed: bool, 63 | 64 | #[clap( 65 | long, 66 | value_delimiter = ',', 67 | help = "Trackers for the torrent, e.g. --bt-trackers \"udp://tracker.opentrackr.org:1337/announce 68 | ,udp://opentracker.io:6969/announce\"" 69 | )] 70 | pub bt_trackers: Option>, 71 | 72 | #[clap(long, help = "Peer connect timeout in seconds. [default: 10]")] 73 | pub bt_peer_connect_timeout: Option, 74 | 75 | #[clap(long, help = "Peer read/write timeout in seconds. [default: 10]")] 76 | pub bt_peer_read_write_timeout: Option, 77 | 78 | #[clap(long, help = "Peer keep-alive interval in seconds. [default: 120]")] 79 | pub bt_peer_keep_alive_interval: Option, 80 | 81 | #[clap(long, help = "Debug output. Print all trackback for debugging")] 82 | pub debug: bool, 83 | 84 | #[clap( 85 | long, 86 | help = "Quiet mode. Don't show progress bar and task information. But still show the error information" 87 | )] 88 | pub quiet: bool, 89 | 90 | #[clap(short, long, help = r#"The path of output for the request e.g. -o "/path/to/file""#)] 91 | pub out: Option, 92 | } 93 | -------------------------------------------------------------------------------- /src/arguments/cmd_args.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | fmt, 3 | path::{Path, PathBuf}, 4 | time::Duration, 5 | }; 6 | 7 | use clap::Parser; 8 | 9 | #[cfg(windows)] 10 | use ansi_term::enable_ansi_support; 11 | 12 | use percent_encoding::percent_decode; 13 | 14 | use crate::{ 15 | arguments::clap_cli::AgetCli, 16 | common::{ 17 | character::escape_nonascii, 18 | errors::Error, 19 | liberal::ParseLiteralNumber, 20 | net::{net::parse_headers, Method, Url}, 21 | tasks::TaskType, 22 | }, 23 | config::Config, 24 | features::args::Args, 25 | }; 26 | 27 | const DEFAULT_HEADERS: [(&str, &str); 1] = [( 28 | "user-agent", 29 | concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")), 30 | )]; 31 | 32 | pub struct CmdArgs { 33 | cli: AgetCli, 34 | config: Config, 35 | } 36 | 37 | impl CmdArgs { 38 | pub fn new() -> CmdArgs { 39 | #[cfg(windows)] 40 | let _ = enable_ansi_support(); 41 | 42 | CmdArgs { 43 | cli: AgetCli::parse(), 44 | config: Config::new(), 45 | } 46 | } 47 | } 48 | 49 | impl Args for CmdArgs { 50 | /// Path of output 51 | fn output(&self) -> PathBuf { 52 | if let Some(path) = self.cli.out.clone() { 53 | PathBuf::from(path) 54 | } else { 55 | let url = self.url(); 56 | let path = Path::new(url.path()); 57 | if let Some(file_name) = path.file_name() { 58 | PathBuf::from( 59 | percent_decode(file_name.to_str().unwrap().as_bytes()) 60 | .decode_utf8() 61 | .unwrap() 62 | .to_string(), 63 | ) 64 | } else { 65 | panic!("{:?}", Error::NoFilename); 66 | } 67 | } 68 | } 69 | 70 | /// Request method for http 71 | fn method(&self) -> Method { 72 | if self.cli.data.is_some() { 73 | return Method::POST; 74 | } 75 | match self.cli.method.to_uppercase().as_str() { 76 | "GET" => Method::GET, 77 | "POST" => Method::POST, 78 | _ => panic!("{:?}", Error::UnsupportedMethod(self.cli.method.to_string())), 79 | } 80 | } 81 | 82 | /// The url of a task 83 | fn url(&self) -> Url { 84 | escape_nonascii(&self.cli.url).parse().expect("URL is unvalidable") 85 | } 86 | 87 | /// The data for http post request 88 | fn data(&self) -> Option<&str> { 89 | self.cli.data.as_deref() 90 | } 91 | 92 | /// Request headers 93 | fn headers(&self) -> Vec<(&str, &str)> { 94 | let mut headers = if let Some(ref headers) = self.cli.header { 95 | let v = parse_headers(headers.iter().map(|h| h.as_str())).unwrap(); 96 | v.into_iter().collect::>() 97 | } else { 98 | vec![] 99 | }; 100 | 101 | if let Some(config_headers) = &self.config.headers { 102 | for (uk, uv) in config_headers.iter() { 103 | let mut has = false; 104 | for (k, _) in headers.iter() { 105 | if k.to_lowercase() == *uk { 106 | has = true; 107 | break; 108 | } 109 | } 110 | if !has { 111 | headers.push((uk, uv)); 112 | } 113 | } 114 | } 115 | 116 | // Add default headers 117 | for (dk, dv) in DEFAULT_HEADERS { 118 | let mut has = false; 119 | for (k, _) in headers.iter() { 120 | if k.to_lowercase() == *dk { 121 | has = true; 122 | break; 123 | } 124 | } 125 | if !has { 126 | headers.push((dk, dv)); 127 | } 128 | } 129 | headers 130 | } 131 | 132 | /// Set proxy througth arg or environment variable 133 | /// 134 | /// The environment variables can be: 135 | /// http_proxy [protocol://][:port] 136 | /// Sets the proxy server to use for HTTP. 137 | /// 138 | /// HTTPS_PROXY [protocol://][:port] 139 | /// Sets the proxy server to use for HTTPS. 140 | /// 141 | /// ALL_PROXY [protocol://][:port] 142 | /// Sets the proxy server to use if no protocol-specific proxy is set. 143 | /// 144 | /// Protocols: 145 | /// http:// 146 | /// an HTTP proxy 147 | /// https:// 148 | /// as HTTPS proxy 149 | /// socks4:// 150 | /// socks4a:// 151 | /// socks5:// 152 | /// socks5h:// 153 | /// as SOCKS proxy 154 | fn proxy(&self) -> Option<&str> { 155 | self.cli.proxy.as_deref() 156 | } 157 | 158 | /// Set request timeout 159 | /// 160 | /// Request timeout is the total time before a response must be received. 161 | /// Default value is 5 seconds. 162 | fn timeout(&self) -> Duration { 163 | let timeout = match self.cli.timeout { 164 | Some(timeout) => timeout, 165 | None => match self.task_type() { 166 | TaskType::HTTP => self.config.timeout.unwrap_or(60), 167 | TaskType::M3U8 => self.config.timeout.unwrap_or(30), 168 | TaskType::BT => self.config.timeout.unwrap_or(60), 169 | }, 170 | }; 171 | 172 | Duration::from_secs(timeout) 173 | } 174 | 175 | fn dns_timeout(&self) -> Duration { 176 | Duration::from_secs(self.cli.dns_timeout.unwrap_or(10)) 177 | } 178 | 179 | fn keep_alive(&self) -> Duration { 180 | match self.task_type() { 181 | TaskType::HTTP => Duration::from_secs(60), 182 | TaskType::M3U8 => Duration::from_secs(10), 183 | TaskType::BT => Duration::from_secs(60), 184 | } 185 | } 186 | 187 | fn lifetime(&self) -> Duration { 188 | match self.task_type() { 189 | TaskType::HTTP => Duration::from_secs(0), 190 | TaskType::M3U8 => Duration::from_secs(0), 191 | TaskType::BT => Duration::from_secs(0), 192 | } 193 | } 194 | 195 | /// Always return `true` 196 | fn disable_redirects(&self) -> bool { 197 | true 198 | } 199 | 200 | /// Skip to verify the server's TLS certificate 201 | fn skip_verify_tls_cert(&self) -> bool { 202 | return self.cli.insecure; 203 | } 204 | 205 | /// The number of concurrency 206 | fn concurrency(&self) -> u64 { 207 | self.cli 208 | .concurrency 209 | .unwrap_or_else(|| self.config.concurrency.unwrap_or(10)) 210 | } 211 | 212 | /// The chunk size of each concurrency for http task 213 | fn chunk_size(&self) -> u64 { 214 | self.cli 215 | .chunk_size 216 | .as_deref() 217 | .map(|i| i.literal_number().unwrap()) 218 | .unwrap_or_else(|| { 219 | self.config 220 | .chunk_size 221 | .as_ref() 222 | .map(|i| i.as_str().literal_number().unwrap()) 223 | .unwrap_or(1024 * 1024 * 50) 224 | }) // 50m 225 | } 226 | 227 | /// The number of retry of a task, default is 5 228 | fn retries(&self) -> u64 { 229 | self.cli.retries.unwrap_or_else(|| self.config.retries.unwrap_or(5)) 230 | } 231 | 232 | /// The internal of each retry, default is zero 233 | fn retry_wait(&self) -> u64 { 234 | self.cli 235 | .retry_wait 236 | .unwrap_or_else(|| self.config.retry_wait.unwrap_or(0)) 237 | } 238 | 239 | /// Task type 240 | fn task_type(&self) -> TaskType { 241 | match self.cli.tp.as_str() { 242 | "auto" => { 243 | let url = self.url(); 244 | if url.scheme() == "magnet" { 245 | TaskType::BT 246 | } else if url.path().to_lowercase().ends_with(".torrent") { 247 | TaskType::BT 248 | } else if url.path().to_lowercase().ends_with(".m3u8") { 249 | TaskType::M3U8 250 | } else if url.scheme().starts_with("http") { 251 | TaskType::HTTP 252 | } else { 253 | panic!("{:?}", Error::UnsupportedTask(self.cli.tp.clone())) 254 | } 255 | } 256 | "http" => TaskType::HTTP, 257 | "m3u8" => TaskType::M3U8, 258 | "bt" => TaskType::BT, 259 | _ => panic!("{:?}", Error::UnsupportedTask(self.cli.tp.clone())), 260 | } 261 | } 262 | 263 | /// A regex to only download files matching it in the torrent 264 | fn bt_file_regex(&self) -> Option { 265 | self.cli.bt_file_regex.to_owned() 266 | } 267 | 268 | /// Seed the torrent 269 | fn seed(&self) -> bool { 270 | self.cli.seed 271 | } 272 | 273 | /// Trackers for the torrent 274 | fn bt_trackers(&self) -> Option> { 275 | self.cli.bt_trackers.to_owned() 276 | } 277 | 278 | /// Peer connect timeout 279 | fn bt_peer_connect_timeout(&self) -> Option { 280 | self.cli.bt_peer_connect_timeout 281 | } 282 | 283 | /// Peer read/write timeout 284 | fn bt_peer_read_write_timeout(&self) -> Option { 285 | self.cli.bt_peer_read_write_timeout 286 | } 287 | 288 | /// Peer keep alive interval 289 | fn bt_peer_keep_alive_interval(&self) -> Option { 290 | self.cli.bt_peer_keep_alive_interval 291 | } 292 | 293 | /// To debug mode, if it returns true 294 | fn debug(&self) -> bool { 295 | self.cli.debug 296 | } 297 | 298 | /// To quiet mode, if it return true 299 | fn quiet(&self) -> bool { 300 | self.cli.quiet 301 | } 302 | } 303 | 304 | impl fmt::Debug for CmdArgs { 305 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 306 | f.debug_struct("CmdArgs") 307 | .field("output", &self.output()) 308 | .field("method", &self.method()) 309 | .field("url", &self.url()) 310 | .field("data", &self.data()) 311 | .field("headers", &self.headers()) 312 | .field("proxy", &self.proxy()) 313 | .field("timeout", &self.timeout()) 314 | .field("dns_timeout", &self.dns_timeout()) 315 | .field("keep_alive", &self.keep_alive()) 316 | .field("lifetime", &self.lifetime()) 317 | .field("disable_redirects", &self.disable_redirects()) 318 | .field("concurrency", &self.concurrency()) 319 | .field("chunk_size", &self.chunk_size()) 320 | .field("retries", &self.retries()) 321 | .field("retry_wait", &self.retry_wait()) 322 | .field("task_type", &self.task_type()) 323 | .field("bt_file_regex", &self.bt_file_regex()) 324 | .field("seed", &self.seed()) 325 | .field("bt_trackers", &self.bt_trackers()) 326 | .field("bt_peer_connect_timeout", &self.bt_peer_connect_timeout()) 327 | .field("bt_peer_read_write_timeout", &self.bt_peer_read_write_timeout()) 328 | .field("bt_peer_keep_alive_interval", &self.bt_peer_keep_alive_interval()) 329 | .field("debug", &self.debug()) 330 | .field("quiet", &self.quiet()) 331 | .finish() 332 | } 333 | } 334 | -------------------------------------------------------------------------------- /src/arguments/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod clap_cli; 2 | pub mod cmd_args; 3 | -------------------------------------------------------------------------------- /src/common/buf.rs: -------------------------------------------------------------------------------- 1 | /// Default buffer size, 16k 2 | pub const SIZE: usize = 16 * 1024; 3 | 4 | /// Maximum buffer size, 100M 5 | pub const MAX_SIZE: usize = 100 * 1024 * 1024; 6 | -------------------------------------------------------------------------------- /src/common/bytes/bytes.rs: -------------------------------------------------------------------------------- 1 | use std::num::ParseIntError; 2 | 3 | use crate::common::errors::Result; 4 | 5 | /// Create an integer value from its representation as a byte array in big endian. 6 | pub fn u8x8_to_u64(u8x8: &[u8; 8]) -> u64 { 7 | u64::from_be_bytes(*u8x8) 8 | } 9 | 10 | /// Return the memory representation of this integer as a byte array in big-endian (network) byte 11 | /// order. 12 | pub fn u64_to_u8x8(u: u64) -> [u8; 8] { 13 | u.to_be_bytes() 14 | } 15 | 16 | pub fn u32_to_u8x4(u: u32) -> [u8; 4] { 17 | u.to_be_bytes() 18 | } 19 | 20 | pub fn decode_hex(s: &str) -> Result, ParseIntError> { 21 | (0..s.len()) 22 | .step_by(2) 23 | .map(|i| u8::from_str_radix(&s[i..i + 2], 16)) 24 | .collect() 25 | } 26 | -------------------------------------------------------------------------------- /src/common/bytes/bytes_type.rs: -------------------------------------------------------------------------------- 1 | pub use bytes::{Buf, BufMut, Bytes, BytesMut}; 2 | -------------------------------------------------------------------------------- /src/common/bytes/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod bytes; 2 | pub mod bytes_type; 3 | -------------------------------------------------------------------------------- /src/common/character.rs: -------------------------------------------------------------------------------- 1 | use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS}; 2 | 3 | const FRAGMENT: &AsciiSet = &CONTROLS.add(b' ').remove(b'?').remove(b'/').remove(b':').remove(b'='); 4 | 5 | pub fn escape_nonascii(target: &str) -> String { 6 | utf8_percent_encode(target, FRAGMENT).to_string() 7 | } 8 | 9 | #[cfg(test)] 10 | mod tests { 11 | use super::escape_nonascii; 12 | 13 | #[test] 14 | fn test_escape_nonascii() { 15 | let s = ":ss/s 来；】/ 【【 ? 是的 & 水电费=45 进来看"; 16 | println!("{}", s); 17 | println!("{}", escape_nonascii(s)); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/common/colors.rs: -------------------------------------------------------------------------------- 1 | pub use ansi_term::Colour::*; 2 | -------------------------------------------------------------------------------- /src/common/crypto.rs: -------------------------------------------------------------------------------- 1 | use aes::cipher::{block_padding::Pkcs7, BlockDecryptMut, BlockEncryptMut, KeyIvInit}; 2 | 3 | use crate::common::errors::Result; 4 | 5 | type Aes128CbcDec = cbc::Decryptor; 6 | type Aes128CbcEnc = cbc::Encryptor; 7 | 8 | pub fn encrypt_aes128(key: &[u8], iv: &[u8], buf: &[u8]) -> Vec { 9 | let cipher = Aes128CbcEnc::new(key.into(), iv.into()); 10 | cipher.encrypt_padded_vec_mut::(buf) 11 | } 12 | 13 | pub fn decrypt_aes128(key: &[u8], iv: &[u8], buf: &[u8]) -> Result> { 14 | let cipher = Aes128CbcDec::new(key.into(), iv.into()); 15 | Ok(cipher.decrypt_padded_vec_mut::(buf)?) 16 | } 17 | -------------------------------------------------------------------------------- /src/common/errors.rs: -------------------------------------------------------------------------------- 1 | use std::{io::Error as IoError, num, result}; 2 | 3 | use thiserror::Error as ThisError; 4 | 5 | use url::ParseError as UrlParseError; 6 | 7 | use aes::cipher::block_padding::UnpadError; 8 | 9 | pub type Result = result::Result; 10 | 11 | #[derive(Debug, ThisError)] 12 | pub enum Error { 13 | // For Arguments 14 | #[error("Output path is invalid: {0}")] 15 | InvalidPath(String), 16 | #[error("Uri is invalid: {0}")] 17 | InvaildUri(#[from] http::uri::InvalidUri), 18 | #[error("Header is invalid: {0}")] 19 | InvalidHeader(String), 20 | #[error("No filename.")] 21 | NoFilename, 22 | #[error("Directory is not found")] 23 | NotFoundDirectory, 24 | #[error("The file already exists.")] 25 | FileExists, 26 | #[error("The path is a directory.")] 27 | PathIsDirectory, 28 | #[error("Can't parse string as number: {0}")] 29 | IsNotNumber(#[from] num::ParseIntError), 30 | #[error("Io Error: {0}")] 31 | Io(#[from] IoError), 32 | #[error("{0} task is not supported")] 33 | UnsupportedTask(String), 34 | 35 | // For IO 36 | #[error("IO: Unexpected EOF")] 37 | UnexpectedEof, 38 | 39 | #[error("Procedure timeout")] 40 | Timeout, 41 | 42 | // For Network 43 | #[error("Request error: {0}")] 44 | RequestError(#[from] reqwest::Error), 45 | #[error("Network error: {0}")] 46 | NetError(String), 47 | #[error("Uncompleted Read")] 48 | UncompletedRead, 49 | #[error("{0} is unsupported")] 50 | UnsupportedMethod(String), 51 | #[error("header is invalid: {0}")] 52 | HeaderParseError(String), 53 | #[error("header is invalid: {0}")] 54 | UrlParseError(#[from] UrlParseError), 55 | #[error("BUG: {0}")] 56 | Bug(String), 57 | #[error("The two content lengths are not equal between the response and the aget file.")] 58 | ContentLengthIsNotConsistent, 59 | 60 | // For m3u8 61 | #[error("Fail to parse m3u8 file.")] 62 | M3U8ParseFail, 63 | #[error("The two m3u8 parts are not equal between the response and the aget file.")] 64 | PartsAreNotConsistent, 65 | 66 | // For torrent 67 | #[error("BitTorrent session error: {0}")] 68 | BitTorrentError(String), 69 | 70 | #[error("An internal error: {0}")] 71 | InnerError(String), 72 | #[error("Content does not has length")] 73 | NoContentLength, 74 | #[error("header is invalid: {0}")] 75 | InvaildHeader(String), 76 | #[error("response status code is: {0}")] 77 | Unsuccess(u16), 78 | #[error("Redirect to: {0}")] 79 | Redirect(String), 80 | #[error("No Location for redirection: {0}")] 81 | NoLocation(String), 82 | #[error("Fail to decrypt aes128 data: {0}")] 83 | AES128DecryptFail(UnpadError), 84 | } 85 | 86 | impl From for Error { 87 | fn from(err: http::header::ToStrError) -> Error { 88 | Error::NetError(format!("{}", err)) 89 | } 90 | } 91 | 92 | impl From for Error { 93 | fn from(err: http::Error) -> Error { 94 | Error::NetError(format!("{}", err)) 95 | } 96 | } 97 | 98 | impl From for Error { 99 | fn from(err: UnpadError) -> Error { 100 | Error::AES128DecryptFail(err) 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/common/file.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | fs::{create_dir_all, metadata, remove_file, File as StdFile, OpenOptions}, 3 | io::{Read, Seek, SeekFrom, Write}, 4 | path::{Path, PathBuf}, 5 | }; 6 | 7 | use crate::common::errors::{Error, Result}; 8 | 9 | /// File can be readed or writen only by opened. 10 | pub struct File { 11 | path: PathBuf, 12 | file: Option, 13 | readable: bool, 14 | } 15 | 16 | impl File { 17 | pub fn new>(path: P, readable: bool) -> Result { 18 | let path = path.as_ref().to_path_buf(); 19 | if path.is_dir() { 20 | return Err(Error::InvalidPath(format!("{:?}", path))); 21 | } 22 | 23 | Ok(File { 24 | path, 25 | file: None, 26 | readable, 27 | }) 28 | } 29 | 30 | /// Create the dir if it does not exists 31 | fn create_dir>(&self, dir: P) -> Result<()> { 32 | if !dir.as_ref().exists() { 33 | Ok(create_dir_all(dir)?) 34 | } else { 35 | Ok(()) 36 | } 37 | } 38 | 39 | /// Create or open the file 40 | pub fn open(&mut self) -> Result<&mut Self> { 41 | if let Some(dir) = self.path.parent() { 42 | self.create_dir(dir)?; 43 | } 44 | let file = OpenOptions::new() 45 | .read(self.readable) 46 | .write(true) 47 | .truncate(false) 48 | .create(true) 49 | .open(self.path.as_path())?; 50 | self.file = Some(file); 51 | Ok(self) 52 | } 53 | 54 | pub fn exists(&self) -> bool { 55 | self.path.as_path().exists() 56 | } 57 | 58 | pub fn file_name(&self) -> Option<&str> { 59 | if let Some(n) = self.path.as_path().file_name() { 60 | n.to_str() 61 | } else { 62 | None 63 | } 64 | } 65 | 66 | pub fn file(&mut self) -> Result<&mut StdFile> { 67 | if let Some(ref mut file) = self.file { 68 | Ok(file) 69 | } else { 70 | Err(Error::Bug("`store::File::file` must be opened".to_string())) 71 | } 72 | } 73 | 74 | pub fn size(&self) -> u64 { 75 | if let Ok(md) = metadata(&self.path) { 76 | md.len() 77 | } else { 78 | 0 79 | } 80 | } 81 | 82 | pub fn write(&mut self, buf: &[u8], seek: Option) -> Result { 83 | if let Some(seek) = seek { 84 | self.seek(seek)?; 85 | } 86 | Ok(self.file()?.write(buf)?) 87 | } 88 | 89 | pub fn read(&mut self, buf: &mut [u8], seek: Option) -> Result { 90 | if let Some(seek) = seek { 91 | self.seek(seek)?; 92 | } 93 | Ok(self.file()?.read(buf)?) 94 | } 95 | 96 | pub fn seek(&mut self, seek: SeekFrom) -> Result { 97 | Ok(self.file()?.seek(seek)?) 98 | } 99 | 100 | pub fn set_len(&mut self, size: u64) -> Result<()> { 101 | Ok(self.file()?.set_len(size)?) 102 | } 103 | 104 | pub fn remove(&self) -> Result<()> { 105 | Ok(remove_file(self.path.as_path())?) 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/common/liberal.rs: -------------------------------------------------------------------------------- 1 | use crate::common::errors::{Error, Result}; 2 | 3 | const SIZES: [&str; 5] = ["B", "K", "M", "G", "T"]; 4 | 5 | /// Convert liberal number to u64 6 | /// e.g. 7 | /// 100k -> 100 * 1024 8 | pub trait ParseLiteralNumber { 9 | fn literal_number(&self) -> Result; 10 | } 11 | 12 | impl ParseLiteralNumber for &str { 13 | fn literal_number(&self) -> Result { 14 | let (num, unit) = self.split_at(self.len() - 1); 15 | if unit.parse::().is_err() { 16 | let mut num = num.parse::()?; 17 | for s in &SIZES { 18 | if s == &unit.to_uppercase() { 19 | return Ok(num); 20 | } else { 21 | num *= 1024; 22 | } 23 | } 24 | Ok(num) 25 | } else { 26 | let num = self.parse::()?; 27 | Ok(num) 28 | } 29 | } 30 | } 31 | 32 | /// Convert seconds to date format 33 | pub trait ToDate { 34 | fn date(&self) -> String; 35 | } 36 | 37 | impl ToDate for u64 { 38 | fn date(&self) -> String { 39 | let mut num = *self as f64; 40 | if num < 60.0 { 41 | return format!("{:.0}s", num); 42 | } 43 | num /= 60.0; 44 | if num < 60.0 { 45 | return format!("{:.0}m", num); 46 | } 47 | num /= 60.0; 48 | if num < 24.0 { 49 | return format!("{:.0}h", num); 50 | } 51 | num /= 24.0; 52 | return format!("{:.0}d", num); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/common/list.rs: -------------------------------------------------------------------------------- 1 | use std::{cell::RefCell, rc::Rc}; 2 | 3 | use crate::features::stack::StackLike; 4 | 5 | #[derive(Debug, Clone)] 6 | pub struct SharedVec { 7 | inner: Rc>>, 8 | } 9 | 10 | impl SharedVec { 11 | pub fn new(list: Vec) -> SharedVec { 12 | SharedVec { 13 | inner: Rc::new(RefCell::new(list)), 14 | } 15 | } 16 | } 17 | 18 | impl StackLike for SharedVec { 19 | fn push(&mut self, item: T) { 20 | self.inner.borrow_mut().push(item) 21 | } 22 | 23 | fn pop(&mut self) -> Option { 24 | self.inner.borrow_mut().pop() 25 | } 26 | 27 | fn len(&self) -> usize { 28 | self.inner.borrow().len() 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/common/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod buf; 2 | pub mod bytes; 3 | pub mod character; 4 | pub mod colors; 5 | pub mod crypto; 6 | pub mod errors; 7 | pub mod file; 8 | pub mod liberal; 9 | pub mod list; 10 | pub mod net; 11 | pub mod range; 12 | pub mod size; 13 | pub mod tasks; 14 | pub mod terminal; 15 | pub mod time; 16 | pub mod uri; 17 | -------------------------------------------------------------------------------- /src/common/net/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod net; 2 | 3 | pub use http::Uri; 4 | pub use reqwest::{ 5 | header::{HeaderMap, HeaderName}, 6 | Client as HttpClient, Method, Proxy, Request, Response, 7 | }; 8 | pub use url::Url; 9 | 10 | #[derive(Debug)] 11 | pub enum ContentLengthValue { 12 | RangeLength(u64), 13 | DirectLength(u64), 14 | NoLength, 15 | } 16 | -------------------------------------------------------------------------------- /src/common/net/net.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use crate::common::{ 4 | errors::{Error, Result}, 5 | net::{ContentLengthValue, HeaderMap, HeaderName, HttpClient, Method, Proxy, Response, Url}, 6 | range::RangePair, 7 | }; 8 | 9 | pub fn parse_header(raw: &str) -> Result<(&str, &str), Error> { 10 | if let Some(index) = raw.find(": ") { 11 | return Ok((&raw[..index], &raw[index + 2..])); 12 | } 13 | if let Some(index) = raw.find(':') { 14 | return Ok((&raw[..index], &raw[index + 1..])); 15 | } 16 | Err(Error::InvalidHeader(raw.to_string())) 17 | } 18 | 19 | pub fn parse_headers<'a, I: IntoIterator>(raws: I) -> Result, Error> { 20 | let mut headers = vec![]; 21 | for raw in raws { 22 | let pair = parse_header(raw)?; 23 | headers.push(pair); 24 | } 25 | Ok(headers) 26 | } 27 | 28 | /// Builder a http client of curl 29 | pub fn build_http_client( 30 | headers: &[(&str, &str)], 31 | timeout: Duration, 32 | dns_timeout: Duration, 33 | keep_alive: Duration, 34 | skip_verify_tls_cert: bool, 35 | proxy: Option<&str>, 36 | ) -> Result { 37 | let mut default_headers = HeaderMap::new(); 38 | headers.iter().for_each(|(k, v)| { 39 | default_headers.insert(k.parse::().unwrap(), v.parse().unwrap()); 40 | }); 41 | if !default_headers.contains_key("accept") { 42 | default_headers.insert("accept", "*/*".parse().unwrap()); 43 | } 44 | 45 | let mut client = HttpClient::builder() 46 | .timeout(timeout) 47 | .connect_timeout(dns_timeout) 48 | .tcp_keepalive(keep_alive) 49 | .default_headers(default_headers); 50 | 51 | if skip_verify_tls_cert { 52 | client = client.danger_accept_invalid_certs(true); 53 | } 54 | 55 | if let Some(url) = proxy { 56 | client = client.proxy(Proxy::all(url)?); 57 | } 58 | Ok(client.build()?) 59 | } 60 | 61 | /// Check whether the response is success 62 | /// Check if status is within 200-299. 63 | pub fn is_success(resp: &reqwest::Response) -> Result<(), Error> { 64 | let status = resp.status(); 65 | if !status.is_success() { 66 | Err(Error::Unsuccess(status.as_u16())) 67 | } else { 68 | Ok(()) 69 | } 70 | } 71 | 72 | /// Send a request with a range header, returning the final url 73 | pub async fn redirect(client: &HttpClient, method: Method, url: Url, data: Option) -> Result { 74 | let mut req = client.request(method.clone(), url.clone()).header("range", "bytes=0-1"); 75 | 76 | if let Some(d) = data { 77 | req = req.body(d); 78 | }; 79 | 80 | let resp = req.send().await?; 81 | is_success(&resp)?; // Return unsuccess code 82 | 83 | Ok(resp.url().clone()) 84 | } 85 | 86 | /// Get the content length of the resource 87 | pub async fn redirect_and_contentlength( 88 | client: &HttpClient, 89 | method: Method, 90 | url: Url, 91 | data: Option, 92 | ) -> Result<(Url, ContentLengthValue)> { 93 | let mut req = client.request(method.clone(), url.clone()).header("range", "bytes=0-1"); 94 | if let Some(d) = data.clone() { 95 | req = req.body(d); 96 | } 97 | 98 | let resp = req.send().await?; 99 | is_success(&resp)?; 100 | 101 | let url = resp.url().clone(); 102 | 103 | let status_code = resp.status(); 104 | if status_code.as_u16() == 206 { 105 | let cl_str = resp.headers().get("content-range").unwrap().to_str().unwrap(); 106 | let index = cl_str.find('/').unwrap(); 107 | let length = cl_str[index + 1..].parse::()?; 108 | return Ok((url, ContentLengthValue::RangeLength(length))); 109 | } else { 110 | let content_length = resp.content_length(); 111 | if let Some(length) = content_length { 112 | return Ok((url, ContentLengthValue::DirectLength(length))); 113 | } else { 114 | return Ok((url, ContentLengthValue::NoLength)); 115 | } 116 | } 117 | } 118 | 119 | /// Send a request 120 | pub async fn request( 121 | client: &HttpClient, 122 | method: Method, 123 | url: Url, 124 | data: Option, 125 | range: Option, 126 | ) -> Result { 127 | let mut req = client.request(method, url); 128 | if let Some(RangePair { begin, end }) = range { 129 | req = req.header("range", format!("bytes={}-{}", begin, end)); 130 | } else { 131 | req = req.header("range", "bytes=0-"); 132 | } 133 | if let Some(d) = data.clone() { 134 | req = req.body(d); 135 | } 136 | 137 | let resp = req.send().await?; 138 | is_success(&resp)?; 139 | return Ok(resp); 140 | } 141 | 142 | pub fn join_url(base_url: &Url, url: &str) -> Result { 143 | let new_url: Url = if !url.to_lowercase().starts_with("http") { 144 | let base_url = Url::parse(&format!("{}", base_url))?; 145 | base_url.join(url)?.as_str().parse()? 146 | } else { 147 | url.parse()? 148 | }; 149 | Ok(new_url) 150 | } 151 | -------------------------------------------------------------------------------- /src/common/range.rs: -------------------------------------------------------------------------------- 1 | use std::{cell::RefCell, rc::Rc}; 2 | 3 | use crate::features::stack::StackLike; 4 | 5 | #[derive(Debug, Clone, Copy)] 6 | pub struct RangePair { 7 | pub begin: u64, 8 | pub end: u64, 9 | } 10 | 11 | impl RangePair { 12 | pub fn new(begin: u64, end: u64) -> RangePair { 13 | assert!(begin <= end, "`RangePair::new`: begin > end: {} > {}", begin, end); 14 | RangePair { begin, end } 15 | } 16 | 17 | // The length of a `RangePair` is the closed interval length 18 | pub fn length(&self) -> u64 { 19 | self.end - self.begin + 1 20 | } 21 | } 22 | 23 | pub type RangeList = Vec; 24 | 25 | #[derive(Debug, Clone)] 26 | pub struct SharedRangList { 27 | inner: Rc>, 28 | } 29 | 30 | impl SharedRangList { 31 | pub fn new(rangelist: RangeList) -> SharedRangList { 32 | SharedRangList { 33 | inner: Rc::new(RefCell::new(rangelist)), 34 | } 35 | } 36 | } 37 | 38 | impl StackLike for SharedRangList { 39 | fn push(&mut self, pair: RangePair) { 40 | self.inner.borrow_mut().push(pair) 41 | } 42 | 43 | fn pop(&mut self) -> Option { 44 | self.inner.borrow_mut().pop() 45 | } 46 | 47 | fn len(&self) -> usize { 48 | self.inner.borrow().len() 49 | } 50 | } 51 | 52 | /// Split a close `RangePair` to many piece of pairs that each of their size is equal to 53 | /// `chunk_size`, but the last piece size can be less then `chunk_size`. 54 | pub fn split_pair(pair: &RangePair, chunk_size: u64) -> RangeList { 55 | let mut stack = Vec::new(); 56 | 57 | let mut begin = pair.begin; 58 | let interval_end = pair.end; 59 | 60 | while begin + chunk_size - 1 <= interval_end { 61 | let end = begin + chunk_size - 1; 62 | stack.push(RangePair::new(begin, end)); 63 | begin += chunk_size; 64 | } 65 | 66 | if begin <= interval_end { 67 | stack.push(RangePair::new(begin, interval_end)); 68 | } 69 | 70 | stack 71 | } 72 | -------------------------------------------------------------------------------- /src/common/size.rs: -------------------------------------------------------------------------------- 1 | /// Here, we handle about size. 2 | 3 | const SIZES: [&str; 5] = ["B", "K", "M", "G", "T"]; 4 | 5 | /// Convert number to human-readable 6 | pub trait HumanReadable { 7 | fn human_readable(&self) -> String; 8 | } 9 | 10 | impl HumanReadable for u64 { 11 | fn human_readable(&self) -> String { 12 | let mut num = *self as f64; 13 | for s in &SIZES { 14 | if num < 1024.0 { 15 | return format!("{:.1}{}", num, s); 16 | } 17 | num /= 1024.0; 18 | } 19 | return format!("{:.1}{}", num, SIZES[SIZES.len() - 1]); 20 | } 21 | } 22 | 23 | impl HumanReadable for f64 { 24 | fn human_readable(&self) -> String { 25 | let mut num = *self; 26 | for s in &SIZES { 27 | if num < 1024.0 { 28 | return format!("{:.1}{}", num, s); 29 | } 30 | num /= 1024.0; 31 | } 32 | return format!("{:.1}{}", num, SIZES[SIZES.len() - 1]); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/common/tasks.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug, Clone)] 2 | pub enum TaskType { 3 | HTTP, 4 | M3U8, 5 | BT, 6 | } 7 | -------------------------------------------------------------------------------- /src/common/terminal.rs: -------------------------------------------------------------------------------- 1 | use term_size::dimensions; 2 | 3 | const MIN_TERMINAL_WIDTH: u64 = 60; 4 | 5 | pub fn terminal_width() -> u64 { 6 | if let Some((width, _)) = dimensions() { 7 | width as u64 8 | } else { 9 | // for envrionment in which atty is not available, 10 | // example, at ci of osx 11 | MIN_TERMINAL_WIDTH 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/common/time.rs: -------------------------------------------------------------------------------- 1 | use std::time::Duration; 2 | 3 | use actix_rt::time::sleep; 4 | use futures::{ 5 | future::ready, 6 | stream::{repeat, Stream}, 7 | FutureExt, StreamExt, 8 | }; 9 | 10 | /// Interval Stream 11 | pub fn interval_stream(timeout: Duration) -> impl Stream { 12 | repeat(()).then(move |_| sleep(timeout).then(|_| ready(()))) 13 | } 14 | -------------------------------------------------------------------------------- /src/common/uri.rs: -------------------------------------------------------------------------------- 1 | use std::path::Path; 2 | 3 | use crate::common::{ 4 | errors::{Error, Result}, 5 | net::Uri, 6 | }; 7 | 8 | /// Use the last of components of uri as a file name 9 | pub trait UriFileName { 10 | fn file_name(&self) -> Result<&str>; 11 | } 12 | 13 | impl UriFileName for Uri { 14 | fn file_name(&self) -> Result<&str> { 15 | let path = Path::new(self.path()); 16 | if let Some(file_name) = path.file_name() { 17 | Ok(file_name.to_str().unwrap()) 18 | } else { 19 | Err(Error::NoFilename) 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/config/mod.rs: -------------------------------------------------------------------------------- 1 | use std::{fs, io::Read}; 2 | 3 | use serde::Deserialize; 4 | 5 | #[derive(Deserialize, Default)] 6 | pub struct Config { 7 | pub(crate) headers: Option>, 8 | pub(crate) concurrency: Option, 9 | pub(crate) chunk_size: Option, 10 | pub(crate) timeout: Option, 11 | pub(crate) dns_timeout: Option, 12 | pub(crate) retries: Option, 13 | pub(crate) retry_wait: Option, 14 | } 15 | 16 | impl Config { 17 | pub fn new() -> Config { 18 | if let Some(path) = dirs::home_dir() { 19 | let config_dir = path.join(".config").join("aget"); 20 | if config_dir.is_dir() { 21 | let config_path = config_dir.join("config"); 22 | if config_path.exists() && config_path.is_file() { 23 | let mut fl = fs::File::open(&config_path) 24 | .expect(&format!("Can't open configuration file: {:?}", config_path)); 25 | let mut cn = String::new(); 26 | fl.read_to_string(&mut cn).unwrap(); 27 | let config: Config = toml::from_str(&cn).unwrap(); 28 | return config; 29 | } 30 | } 31 | } 32 | Config::default() 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/features/args.rs: -------------------------------------------------------------------------------- 1 | use std::{path::PathBuf, time::Duration}; 2 | 3 | use crate::common::{ 4 | net::{Method, Url}, 5 | tasks::TaskType, 6 | }; 7 | 8 | /// This a arg which gives parameters for apps 9 | pub trait Args { 10 | /// Path of output 11 | fn output(&self) -> PathBuf; 12 | 13 | /// Request method for http 14 | fn method(&self) -> Method; 15 | 16 | /// The uri of a task 17 | fn url(&self) -> Url; 18 | 19 | /// The data for http post request 20 | fn data(&self) -> Option<&str>; 21 | 22 | /// Request headers 23 | fn headers(&self) -> Vec<(&str, &str)>; 24 | 25 | /// Proxy: http, https, socks4, socks5 26 | fn proxy(&self) -> Option<&str>; 27 | 28 | /// The maximum time the request is allowed to take. 29 | fn timeout(&self) -> Duration; 30 | 31 | /// Connection timeout 32 | /// 33 | /// i.e. max time to connect to remote host including dns name resolution. 34 | /// Set to 1 second by default. 35 | fn dns_timeout(&self) -> Duration; 36 | 37 | /// Set keep-alive period for opened connection. 38 | /// 39 | /// Keep-alive period is the period between connection usage. If 40 | /// the delay between repeated usages of the same connection 41 | /// exceeds this period, the connection is closed. 42 | /// Default keep-alive period is 15 seconds. 43 | fn keep_alive(&self) -> Duration; 44 | 45 | /// Set max lifetime period for connection. 46 | /// 47 | /// Connection lifetime is max lifetime of any opened connection 48 | /// until it is closed regardless of keep-alive period. 49 | /// Default lifetime period is 75 seconds. 50 | fn lifetime(&self) -> Duration; 51 | 52 | /// Always return `true` 53 | fn disable_redirects(&self) -> bool; 54 | 55 | /// Skip to verify the server's TLS certificate 56 | fn skip_verify_tls_cert(&self) -> bool; 57 | 58 | /// The number of concurrency 59 | fn concurrency(&self) -> u64; 60 | 61 | /// The chunk size of each concurrency for http task 62 | fn chunk_size(&self) -> u64; 63 | 64 | /// The number of retry of a task 65 | fn retries(&self) -> u64; 66 | 67 | /// The internal of each retry 68 | fn retry_wait(&self) -> u64; 69 | 70 | /// Task type 71 | fn task_type(&self) -> TaskType; 72 | 73 | /// A regex to only download files matching it in the torrent 74 | fn bt_file_regex(&self) -> Option; 75 | 76 | /// Seed the torrent 77 | fn seed(&self) -> bool; 78 | 79 | /// Trackers for the torrent 80 | fn bt_trackers(&self) -> Option>; 81 | 82 | /// Peer connect timeout 83 | fn bt_peer_connect_timeout(&self) -> Option; 84 | 85 | /// Peer read/write timeout 86 | fn bt_peer_read_write_timeout(&self) -> Option; 87 | 88 | /// Peer keep alive interval 89 | fn bt_peer_keep_alive_interval(&self) -> Option; 90 | 91 | /// To debug mode, if it returns true 92 | fn debug(&self) -> bool; 93 | 94 | /// To quiet mode, if it return true 95 | fn quiet(&self) -> bool; 96 | } 97 | -------------------------------------------------------------------------------- /src/features/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod args; 2 | pub mod running; 3 | pub mod stack; 4 | -------------------------------------------------------------------------------- /src/features/running.rs: -------------------------------------------------------------------------------- 1 | use crate::common::errors::Result; 2 | 3 | pub trait Runnable { 4 | fn run(self) -> Result<()>; 5 | } 6 | -------------------------------------------------------------------------------- /src/features/stack.rs: -------------------------------------------------------------------------------- 1 | pub trait StackLike { 2 | fn push(&mut self, item: T); 3 | 4 | fn pop(&mut self) -> Option; 5 | 6 | fn len(&self) -> usize; 7 | } 8 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod app; 2 | pub mod arguments; 3 | pub mod common; 4 | pub mod config; 5 | pub mod features; 6 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | 3 | use std::{process::exit, str::FromStr, thread, time::Duration}; 4 | 5 | use time::{macros::format_description, UtcOffset}; 6 | use tracing_subscriber::fmt::time::OffsetTime; 7 | 8 | use aget::{ 9 | app::core::{bt::BtHandler, http::HttpHandler, m3u8::M3u8Handler}, 10 | arguments::cmd_args::CmdArgs, 11 | common::{errors::Error, tasks::TaskType}, 12 | features::{args::Args, running::Runnable}, 13 | }; 14 | 15 | fn main() { 16 | let cmdargs = CmdArgs::new(); 17 | let log_level = if cmdargs.debug() { "debug" } else { "error" }; 18 | 19 | let app_name = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")).to_string(); 20 | 21 | let (non_blocking, _guard) = tracing_appender::non_blocking(std::io::stdout()); 22 | let local_time = OffsetTime::new( 23 | UtcOffset::from_hms(8, 0, 0).unwrap(), 24 | format_description!("[year]-[month]-[day] [hour]:[minute]:[second].[subsecond digits:2]"), 25 | ); 26 | 27 | let log_level = tracing::Level::from_str(log_level).unwrap(); 28 | tracing_subscriber::fmt() 29 | .with_writer(non_blocking) 30 | .with_max_level(log_level) 31 | .with_timer(local_time) 32 | .init(); 33 | 34 | tracing::debug!("===== Aget-rs {}: begin =====", app_name); 35 | tracing::debug!("Args: {:?}", cmdargs); 36 | 37 | let tasktype = cmdargs.task_type(); 38 | for i in 0..cmdargs.retries() + 1 { 39 | if i != 0 { 40 | println!("Retry {}", i); 41 | } 42 | 43 | let result = match tasktype { 44 | TaskType::HTTP => { 45 | let httphandler = HttpHandler::new(&cmdargs).unwrap(); 46 | httphandler.run() 47 | } 48 | TaskType::M3U8 => { 49 | let m3u8handler = M3u8Handler::new(&cmdargs).unwrap(); 50 | m3u8handler.run() 51 | } 52 | TaskType::BT => { 53 | let bthandler = BtHandler::new(&cmdargs); 54 | bthandler.run() 55 | } 56 | }; 57 | 58 | if let Err(err) = result { 59 | tracing::error!("Error: {:?}", err); 60 | 61 | // if error is "error initializing persistent DHT", remove dht.json 62 | if let Error::BitTorrentError(msg) = err { 63 | if msg == "error initializing persistent DHT" { 64 | let output_dir = cmdargs.output(); 65 | let dht_file = output_dir 66 | .join("..") 67 | .join(output_dir.file_name().unwrap().to_string_lossy().to_string() + ".bt.aget") 68 | .join("dht.json"); 69 | if dht_file.exists() { 70 | std::fs::remove_file(dht_file).unwrap(); 71 | } 72 | } 73 | } 74 | 75 | // Retry 76 | let retrywait = cmdargs.retry_wait(); 77 | thread::sleep(Duration::from_secs(retrywait)); 78 | continue; 79 | } else { 80 | // Success 81 | return; 82 | } 83 | } 84 | 85 | // All retries fail 86 | exit(1); 87 | } 88 | --------------------------------------------------------------------------------