├── .github ├── ISSUE_TEMPLATE │ ├── bug-report.yml │ └── config.yml └── workflows │ ├── CI.yml │ ├── audit.yml │ └── test.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── examples └── upload_to_s3.py ├── pyproject.toml └── src └── lib.rs /.github/ISSUE_TEMPLATE/bug-report.yml: -------------------------------------------------------------------------------- 1 | name: "\U0001F41B Bug Report" 2 | description: Submit a bug report to help us improve hf_transfer 3 | body: 4 | - type: textarea 5 | id: system-info 6 | attributes: 7 | label: System Info 8 | description: Please make sure the bug you are describing is not a general 9 | usability thing. This library is provided *as-is*, not as a general 10 | usability tool. Issues need to concern the actual result of the downloaded 11 | or uploaded files. 12 | placeholder: hf_transfer version, platform, python version, huggingface_hub version... 13 | validations: 14 | required: true 15 | - type: textarea 16 | id: reproduction 17 | validations: 18 | required: true 19 | attributes: 20 | label: Reproduction 21 | description: | 22 | Please provide a code sample that reproduces the problem you ran into. It can be a Colab link or just a code snippet. 23 | If you have code snippets, error messages, stack traces please provide them here as well. 24 | Important! Use code tags to correctly format your code. See https://help.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks#syntax-highlighting 25 | Do not use screenshots, as they are hard to read and (more importantly) don't allow others to copy-and-paste your code. 26 | 27 | placeholder: | 28 | Steps to reproduce the behavior: 29 | 30 | 1. 31 | 2. 32 | 3. 33 | 34 | 35 | - type: textarea 36 | id: expected-behavior 37 | validations: 38 | required: true 39 | attributes: 40 | label: Expected behavior 41 | description: "A clear and concise description of what you would expect to happen." 42 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | version: 2.1 3 | -------------------------------------------------------------------------------- /.github/workflows/CI.yml: -------------------------------------------------------------------------------- 1 | # This file is autogenerated by maturin v1.7.8 2 | # To update, run 3 | # 4 | # maturin generate-ci github 5 | # 6 | name: CI 7 | 8 | on: 9 | push: 10 | branches: 11 | - main 12 | - master 13 | tags: 14 | - '*' 15 | pull_request: 16 | workflow_dispatch: 17 | 18 | permissions: 19 | contents: read 20 | 21 | jobs: 22 | linux: 23 | runs-on: ${{ matrix.platform.runner }} 24 | strategy: 25 | matrix: 26 | platform: 27 | - runner: ubuntu-22.04 28 | target: x86_64 29 | - runner: ubuntu-22.04 30 | target: x86 31 | - runner: ubuntu-22.04 32 | target: aarch64 33 | - runner: ubuntu-22.04 34 | target: armv7 35 | # - runner: ubuntu-22.04 36 | # target: s390x 37 | - runner: ubuntu-22.04 38 | target: ppc64le 39 | steps: 40 | - uses: actions/checkout@v4 41 | - uses: actions/setup-python@v5 42 | with: 43 | python-version: 3.x 44 | - name: Build wheels 45 | uses: PyO3/maturin-action@v1 46 | with: 47 | target: ${{ matrix.platform.target }} 48 | args: --release --out dist 49 | sccache: 'true' 50 | manylinux: auto 51 | before-script-linux: | 52 | if command -v apt-get &> /dev/null; then 53 | apt-get update && apt-get install libssl-dev libatomic-ops-dev -y 54 | elif command -v yum &> /dev/null; then 55 | yum install openssl-devel devtoolset-10-libatomic-devel perl-IPC-Cmd -y 56 | else 57 | echo "Neither apt-get nor yum is installed. Please install a package manager." 58 | exit 1 59 | fi 60 | - name: Build free-threaded wheels 61 | uses: PyO3/maturin-action@v1 62 | with: 63 | target: ${{ matrix.platform.target }} 64 | args: --release --out dist -i python3.13t 65 | sccache: 'true' 66 | manylinux: auto 67 | before-script-linux: | 68 | if command -v apt-get &> /dev/null; then 69 | apt-get update && apt-get install libssl-dev libatomic-ops-dev -y 70 | elif command -v yum &> /dev/null; then 71 | yum install openssl-devel devtoolset-10-libatomic-devel perl-IPC-Cmd -y 72 | else 73 | echo "Neither apt-get nor yum is installed. Please install a package manager." 74 | exit 1 75 | fi 76 | - name: Upload wheels 77 | uses: actions/upload-artifact@v4 78 | with: 79 | name: wheels-linux-${{ matrix.platform.target }} 80 | path: dist 81 | 82 | musllinux: 83 | runs-on: ${{ matrix.platform.runner }} 84 | strategy: 85 | matrix: 86 | platform: 87 | - runner: ubuntu-22.04 88 | target: x86_64 89 | - runner: ubuntu-22.04 90 | target: x86 91 | - runner: ubuntu-22.04 92 | target: aarch64 93 | - runner: ubuntu-22.04 94 | target: armv7 95 | steps: 96 | - uses: actions/checkout@v4 97 | - uses: actions/setup-python@v5 98 | with: 99 | python-version: 3.x 100 | - name: Build wheels 101 | uses: PyO3/maturin-action@v1 102 | with: 103 | target: ${{ matrix.platform.target }} 104 | args: --release --out dist 105 | sccache: 'true' 106 | manylinux: musllinux_1_2 107 | - name: Build free-threaded wheels 108 | uses: PyO3/maturin-action@v1 109 | with: 110 | target: ${{ matrix.platform.target }} 111 | args: --release --out dist -i python3.13t 112 | sccache: 'true' 113 | manylinux: musllinux_1_2 114 | - name: Upload wheels 115 | uses: actions/upload-artifact@v4 116 | with: 117 | name: wheels-musllinux-${{ matrix.platform.target }} 118 | path: dist 119 | 120 | windows: 121 | runs-on: ${{ matrix.platform.runner }} 122 | strategy: 123 | matrix: 124 | platform: 125 | - runner: windows-latest 126 | target: x64 127 | - runner: windows-latest 128 | target: x86 129 | steps: 130 | - uses: actions/checkout@v4 131 | - uses: actions/setup-python@v5 132 | with: 133 | python-version: 3.x 134 | architecture: ${{ matrix.platform.target }} 135 | - name: Build wheels 136 | uses: PyO3/maturin-action@v1 137 | with: 138 | target: ${{ matrix.platform.target }} 139 | args: --release --out dist 140 | sccache: 'true' 141 | # - name: Build free-threaded wheels 142 | # uses: PyO3/maturin-action@v1 143 | # with: 144 | # target: ${{ matrix.platform.target }} 145 | # args: --release --out dist -i python3.13t 146 | # sccache: 'true' 147 | - name: Upload wheels 148 | uses: actions/upload-artifact@v4 149 | with: 150 | name: wheels-windows-${{ matrix.platform.target }} 151 | path: dist 152 | 153 | macos: 154 | runs-on: ${{ matrix.platform.runner }} 155 | strategy: 156 | matrix: 157 | platform: 158 | - runner: macos-13 159 | target: x86_64 160 | - runner: macos-14 161 | target: aarch64 162 | steps: 163 | - uses: actions/checkout@v4 164 | - uses: actions/setup-python@v5 165 | with: 166 | python-version: 3.x 167 | - name: Build wheels 168 | uses: PyO3/maturin-action@v1 169 | with: 170 | target: ${{ matrix.platform.target }} 171 | args: --release --out dist 172 | sccache: 'true' 173 | - name: Build free-threaded wheels 174 | uses: PyO3/maturin-action@v1 175 | with: 176 | target: ${{ matrix.platform.target }} 177 | args: --release --out dist -i python3.13t 178 | sccache: 'true' 179 | - name: Upload wheels 180 | uses: actions/upload-artifact@v4 181 | with: 182 | name: wheels-macos-${{ matrix.platform.target }} 183 | path: dist 184 | 185 | sdist: 186 | runs-on: ubuntu-latest 187 | steps: 188 | - uses: actions/checkout@v4 189 | - name: Build sdist 190 | uses: PyO3/maturin-action@v1 191 | with: 192 | command: sdist 193 | args: --out dist 194 | - name: Upload sdist 195 | uses: actions/upload-artifact@v4 196 | with: 197 | name: wheels-sdist 198 | path: dist 199 | 200 | release: 201 | name: Release 202 | runs-on: ubuntu-latest 203 | if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} 204 | needs: [linux, musllinux, windows, macos, sdist] 205 | permissions: 206 | # Use to sign the release artifacts 207 | id-token: write 208 | # Used to upload release artifacts 209 | contents: write 210 | # Used to generate artifact attestation 211 | attestations: write 212 | steps: 213 | - uses: actions/download-artifact@v4 214 | - name: Generate artifact attestation 215 | uses: actions/attest-build-provenance@v1 216 | with: 217 | subject-path: 'wheels-*/*' 218 | - name: Publish to PyPI 219 | if: ${{ startsWith(github.ref, 'refs/tags/') }} 220 | uses: PyO3/maturin-action@v1 221 | env: 222 | MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} 223 | with: 224 | command: upload 225 | args: --non-interactive --skip-existing wheels-*/* 226 | -------------------------------------------------------------------------------- /.github/workflows/audit.yml: -------------------------------------------------------------------------------- 1 | name: "Audit Dependencies" 2 | on: 3 | push: 4 | paths: 5 | # Run if workflow changes 6 | - '.github/workflows/audit.yml' 7 | # Run on changed dependencies 8 | - '**/Cargo.toml' 9 | - '**/Cargo.lock' 10 | # Run if the configuration file changes 11 | - '**/audit.toml' 12 | # Rerun periodicly to pick up new advisories 13 | schedule: 14 | - cron: '0 0 * * *' 15 | # Run manually 16 | workflow_dispatch: 17 | 18 | jobs: 19 | audit: 20 | runs-on: ubuntu-latest 21 | permissions: 22 | contents: read 23 | issues: write 24 | steps: 25 | - uses: actions/checkout@v4 26 | - uses: actions-rust-lang/audit@v1 27 | name: Audit Rust Dependencies 28 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - master 8 | tags: 9 | - '*' 10 | pull_request: 11 | workflow_dispatch: 12 | 13 | permissions: 14 | contents: read 15 | 16 | jobs: 17 | tests: 18 | runs-on: ubuntu-latest 19 | steps: 20 | - uses: actions/checkout@v4 21 | - uses: actions/checkout@v4 22 | with: 23 | repository: huggingface/huggingface_hub 24 | path: huggingface_hub 25 | - uses: actions/setup-python@v5 26 | with: 27 | python-version: '3.10' 28 | - name: Create venv 29 | run: python3 -m venv .venv 30 | - name: Build wheels 31 | uses: PyO3/maturin-action@v1 32 | with: 33 | command: develop 34 | sccache: 'true' 35 | - name: Install huggingface_hub dependencies 36 | run: | 37 | source .venv/bin/activate 38 | python3 -m pip install -e 'huggingface_hub[testing]' 39 | - name: Run tests 40 | run: | 41 | source .venv/bin/activate 42 | HF_HUB_ENABLE_HF_TRANSFER=1 pytest huggingface_hub/tests/test_file_download.py 43 | HF_HUB_ENABLE_HF_TRANSFER=1 pytest huggingface_hub/tests/test_hf_api.py -k 'test_upload_lfs_file_multipart' 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | .pytest_cache/ 6 | *.py[cod] 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | .venv/ 14 | env/ 15 | bin/ 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | include/ 26 | man/ 27 | venv/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | pip-selfcheck.json 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | 45 | # Translations 46 | *.mo 47 | 48 | # Mr Developer 49 | .mr.developer.cfg 50 | .project 51 | .pydevproject 52 | 53 | # Rope 54 | .ropeproject 55 | 56 | # Django stuff: 57 | *.log 58 | *.pot 59 | 60 | .DS_Store 61 | 62 | # Sphinx documentation 63 | docs/_build/ 64 | 65 | # PyCharm 66 | .idea/ 67 | 68 | # VSCode 69 | .vscode/ 70 | 71 | # Pyenv 72 | .python-version -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "addr2line" 7 | version = "0.24.2" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" 10 | dependencies = [ 11 | "gimli", 12 | ] 13 | 14 | [[package]] 15 | name = "adler2" 16 | version = "2.0.0" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" 19 | 20 | [[package]] 21 | name = "atomic-waker" 22 | version = "1.1.2" 23 | source = "registry+https://github.com/rust-lang/crates.io-index" 24 | checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" 25 | 26 | [[package]] 27 | name = "autocfg" 28 | version = "1.4.0" 29 | source = "registry+https://github.com/rust-lang/crates.io-index" 30 | checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" 31 | 32 | [[package]] 33 | name = "backtrace" 34 | version = "0.3.74" 35 | source = "registry+https://github.com/rust-lang/crates.io-index" 36 | checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" 37 | dependencies = [ 38 | "addr2line", 39 | "cfg-if", 40 | "libc", 41 | "miniz_oxide", 42 | "object", 43 | "rustc-demangle", 44 | "windows-targets", 45 | ] 46 | 47 | [[package]] 48 | name = "base64" 49 | version = "0.22.1" 50 | source = "registry+https://github.com/rust-lang/crates.io-index" 51 | checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" 52 | 53 | [[package]] 54 | name = "bitflags" 55 | version = "2.6.0" 56 | source = "registry+https://github.com/rust-lang/crates.io-index" 57 | checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" 58 | 59 | [[package]] 60 | name = "bumpalo" 61 | version = "3.16.0" 62 | source = "registry+https://github.com/rust-lang/crates.io-index" 63 | checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" 64 | 65 | [[package]] 66 | name = "byteorder" 67 | version = "1.5.0" 68 | source = "registry+https://github.com/rust-lang/crates.io-index" 69 | checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" 70 | 71 | [[package]] 72 | name = "bytes" 73 | version = "1.9.0" 74 | source = "registry+https://github.com/rust-lang/crates.io-index" 75 | checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" 76 | 77 | [[package]] 78 | name = "cc" 79 | version = "1.2.5" 80 | source = "registry+https://github.com/rust-lang/crates.io-index" 81 | checksum = "c31a0499c1dc64f458ad13872de75c0eb7e3fdb0e67964610c914b034fc5956e" 82 | dependencies = [ 83 | "shlex", 84 | ] 85 | 86 | [[package]] 87 | name = "cfg-if" 88 | version = "1.0.0" 89 | source = "registry+https://github.com/rust-lang/crates.io-index" 90 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 91 | 92 | [[package]] 93 | name = "core-foundation" 94 | version = "0.9.4" 95 | source = "registry+https://github.com/rust-lang/crates.io-index" 96 | checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" 97 | dependencies = [ 98 | "core-foundation-sys", 99 | "libc", 100 | ] 101 | 102 | [[package]] 103 | name = "core-foundation-sys" 104 | version = "0.8.7" 105 | source = "registry+https://github.com/rust-lang/crates.io-index" 106 | checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" 107 | 108 | [[package]] 109 | name = "displaydoc" 110 | version = "0.2.5" 111 | source = "registry+https://github.com/rust-lang/crates.io-index" 112 | checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" 113 | dependencies = [ 114 | "proc-macro2", 115 | "quote", 116 | "syn", 117 | ] 118 | 119 | [[package]] 120 | name = "encoding_rs" 121 | version = "0.8.35" 122 | source = "registry+https://github.com/rust-lang/crates.io-index" 123 | checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" 124 | dependencies = [ 125 | "cfg-if", 126 | ] 127 | 128 | [[package]] 129 | name = "equivalent" 130 | version = "1.0.1" 131 | source = "registry+https://github.com/rust-lang/crates.io-index" 132 | checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" 133 | 134 | [[package]] 135 | name = "errno" 136 | version = "0.3.10" 137 | source = "registry+https://github.com/rust-lang/crates.io-index" 138 | checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" 139 | dependencies = [ 140 | "libc", 141 | "windows-sys 0.59.0", 142 | ] 143 | 144 | [[package]] 145 | name = "fastrand" 146 | version = "2.3.0" 147 | source = "registry+https://github.com/rust-lang/crates.io-index" 148 | checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" 149 | 150 | [[package]] 151 | name = "fnv" 152 | version = "1.0.7" 153 | source = "registry+https://github.com/rust-lang/crates.io-index" 154 | checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" 155 | 156 | [[package]] 157 | name = "foreign-types" 158 | version = "0.3.2" 159 | source = "registry+https://github.com/rust-lang/crates.io-index" 160 | checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" 161 | dependencies = [ 162 | "foreign-types-shared", 163 | ] 164 | 165 | [[package]] 166 | name = "foreign-types-shared" 167 | version = "0.1.1" 168 | source = "registry+https://github.com/rust-lang/crates.io-index" 169 | checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" 170 | 171 | [[package]] 172 | name = "form_urlencoded" 173 | version = "1.2.1" 174 | source = "registry+https://github.com/rust-lang/crates.io-index" 175 | checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" 176 | dependencies = [ 177 | "percent-encoding", 178 | ] 179 | 180 | [[package]] 181 | name = "futures" 182 | version = "0.3.31" 183 | source = "registry+https://github.com/rust-lang/crates.io-index" 184 | checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" 185 | dependencies = [ 186 | "futures-channel", 187 | "futures-core", 188 | "futures-executor", 189 | "futures-io", 190 | "futures-sink", 191 | "futures-task", 192 | "futures-util", 193 | ] 194 | 195 | [[package]] 196 | name = "futures-channel" 197 | version = "0.3.31" 198 | source = "registry+https://github.com/rust-lang/crates.io-index" 199 | checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" 200 | dependencies = [ 201 | "futures-core", 202 | "futures-sink", 203 | ] 204 | 205 | [[package]] 206 | name = "futures-core" 207 | version = "0.3.31" 208 | source = "registry+https://github.com/rust-lang/crates.io-index" 209 | checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" 210 | 211 | [[package]] 212 | name = "futures-executor" 213 | version = "0.3.31" 214 | source = "registry+https://github.com/rust-lang/crates.io-index" 215 | checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" 216 | dependencies = [ 217 | "futures-core", 218 | "futures-task", 219 | "futures-util", 220 | ] 221 | 222 | [[package]] 223 | name = "futures-io" 224 | version = "0.3.31" 225 | source = "registry+https://github.com/rust-lang/crates.io-index" 226 | checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" 227 | 228 | [[package]] 229 | name = "futures-macro" 230 | version = "0.3.31" 231 | source = "registry+https://github.com/rust-lang/crates.io-index" 232 | checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" 233 | dependencies = [ 234 | "proc-macro2", 235 | "quote", 236 | "syn", 237 | ] 238 | 239 | [[package]] 240 | name = "futures-sink" 241 | version = "0.3.31" 242 | source = "registry+https://github.com/rust-lang/crates.io-index" 243 | checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" 244 | 245 | [[package]] 246 | name = "futures-task" 247 | version = "0.3.31" 248 | source = "registry+https://github.com/rust-lang/crates.io-index" 249 | checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" 250 | 251 | [[package]] 252 | name = "futures-util" 253 | version = "0.3.31" 254 | source = "registry+https://github.com/rust-lang/crates.io-index" 255 | checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" 256 | dependencies = [ 257 | "futures-channel", 258 | "futures-core", 259 | "futures-io", 260 | "futures-macro", 261 | "futures-sink", 262 | "futures-task", 263 | "memchr", 264 | "pin-project-lite", 265 | "pin-utils", 266 | "slab", 267 | ] 268 | 269 | [[package]] 270 | name = "getrandom" 271 | version = "0.2.15" 272 | source = "registry+https://github.com/rust-lang/crates.io-index" 273 | checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" 274 | dependencies = [ 275 | "cfg-if", 276 | "libc", 277 | "wasi", 278 | ] 279 | 280 | [[package]] 281 | name = "gimli" 282 | version = "0.31.1" 283 | source = "registry+https://github.com/rust-lang/crates.io-index" 284 | checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" 285 | 286 | [[package]] 287 | name = "h2" 288 | version = "0.4.7" 289 | source = "registry+https://github.com/rust-lang/crates.io-index" 290 | checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" 291 | dependencies = [ 292 | "atomic-waker", 293 | "bytes", 294 | "fnv", 295 | "futures-core", 296 | "futures-sink", 297 | "http", 298 | "indexmap", 299 | "slab", 300 | "tokio", 301 | "tokio-util", 302 | "tracing", 303 | ] 304 | 305 | [[package]] 306 | name = "hashbrown" 307 | version = "0.15.2" 308 | source = "registry+https://github.com/rust-lang/crates.io-index" 309 | checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" 310 | 311 | [[package]] 312 | name = "heck" 313 | version = "0.5.0" 314 | source = "registry+https://github.com/rust-lang/crates.io-index" 315 | checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" 316 | 317 | [[package]] 318 | name = "hf_transfer" 319 | version = "0.1.9-dev0" 320 | dependencies = [ 321 | "futures", 322 | "openssl", 323 | "pyo3", 324 | "rand", 325 | "reqwest", 326 | "tokio", 327 | "tokio-util", 328 | ] 329 | 330 | [[package]] 331 | name = "http" 332 | version = "1.2.0" 333 | source = "registry+https://github.com/rust-lang/crates.io-index" 334 | checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" 335 | dependencies = [ 336 | "bytes", 337 | "fnv", 338 | "itoa", 339 | ] 340 | 341 | [[package]] 342 | name = "http-body" 343 | version = "1.0.1" 344 | source = "registry+https://github.com/rust-lang/crates.io-index" 345 | checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" 346 | dependencies = [ 347 | "bytes", 348 | "http", 349 | ] 350 | 351 | [[package]] 352 | name = "http-body-util" 353 | version = "0.1.2" 354 | source = "registry+https://github.com/rust-lang/crates.io-index" 355 | checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" 356 | dependencies = [ 357 | "bytes", 358 | "futures-util", 359 | "http", 360 | "http-body", 361 | "pin-project-lite", 362 | ] 363 | 364 | [[package]] 365 | name = "httparse" 366 | version = "1.9.5" 367 | source = "registry+https://github.com/rust-lang/crates.io-index" 368 | checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" 369 | 370 | [[package]] 371 | name = "hyper" 372 | version = "1.5.2" 373 | source = "registry+https://github.com/rust-lang/crates.io-index" 374 | checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" 375 | dependencies = [ 376 | "bytes", 377 | "futures-channel", 378 | "futures-util", 379 | "h2", 380 | "http", 381 | "http-body", 382 | "httparse", 383 | "itoa", 384 | "pin-project-lite", 385 | "smallvec", 386 | "tokio", 387 | "want", 388 | ] 389 | 390 | [[package]] 391 | name = "hyper-rustls" 392 | version = "0.27.5" 393 | source = "registry+https://github.com/rust-lang/crates.io-index" 394 | checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" 395 | dependencies = [ 396 | "futures-util", 397 | "http", 398 | "hyper", 399 | "hyper-util", 400 | "rustls", 401 | "rustls-pki-types", 402 | "tokio", 403 | "tokio-rustls", 404 | "tower-service", 405 | ] 406 | 407 | [[package]] 408 | name = "hyper-tls" 409 | version = "0.6.0" 410 | source = "registry+https://github.com/rust-lang/crates.io-index" 411 | checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" 412 | dependencies = [ 413 | "bytes", 414 | "http-body-util", 415 | "hyper", 416 | "hyper-util", 417 | "native-tls", 418 | "tokio", 419 | "tokio-native-tls", 420 | "tower-service", 421 | ] 422 | 423 | [[package]] 424 | name = "hyper-util" 425 | version = "0.1.10" 426 | source = "registry+https://github.com/rust-lang/crates.io-index" 427 | checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" 428 | dependencies = [ 429 | "bytes", 430 | "futures-channel", 431 | "futures-util", 432 | "http", 433 | "http-body", 434 | "hyper", 435 | "pin-project-lite", 436 | "socket2", 437 | "tokio", 438 | "tower-service", 439 | "tracing", 440 | ] 441 | 442 | [[package]] 443 | name = "icu_collections" 444 | version = "1.5.0" 445 | source = "registry+https://github.com/rust-lang/crates.io-index" 446 | checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" 447 | dependencies = [ 448 | "displaydoc", 449 | "yoke", 450 | "zerofrom", 451 | "zerovec", 452 | ] 453 | 454 | [[package]] 455 | name = "icu_locid" 456 | version = "1.5.0" 457 | source = "registry+https://github.com/rust-lang/crates.io-index" 458 | checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" 459 | dependencies = [ 460 | "displaydoc", 461 | "litemap", 462 | "tinystr", 463 | "writeable", 464 | "zerovec", 465 | ] 466 | 467 | [[package]] 468 | name = "icu_locid_transform" 469 | version = "1.5.0" 470 | source = "registry+https://github.com/rust-lang/crates.io-index" 471 | checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" 472 | dependencies = [ 473 | "displaydoc", 474 | "icu_locid", 475 | "icu_locid_transform_data", 476 | "icu_provider", 477 | "tinystr", 478 | "zerovec", 479 | ] 480 | 481 | [[package]] 482 | name = "icu_locid_transform_data" 483 | version = "1.5.0" 484 | source = "registry+https://github.com/rust-lang/crates.io-index" 485 | checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" 486 | 487 | [[package]] 488 | name = "icu_normalizer" 489 | version = "1.5.0" 490 | source = "registry+https://github.com/rust-lang/crates.io-index" 491 | checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" 492 | dependencies = [ 493 | "displaydoc", 494 | "icu_collections", 495 | "icu_normalizer_data", 496 | "icu_properties", 497 | "icu_provider", 498 | "smallvec", 499 | "utf16_iter", 500 | "utf8_iter", 501 | "write16", 502 | "zerovec", 503 | ] 504 | 505 | [[package]] 506 | name = "icu_normalizer_data" 507 | version = "1.5.0" 508 | source = "registry+https://github.com/rust-lang/crates.io-index" 509 | checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" 510 | 511 | [[package]] 512 | name = "icu_properties" 513 | version = "1.5.1" 514 | source = "registry+https://github.com/rust-lang/crates.io-index" 515 | checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" 516 | dependencies = [ 517 | "displaydoc", 518 | "icu_collections", 519 | "icu_locid_transform", 520 | "icu_properties_data", 521 | "icu_provider", 522 | "tinystr", 523 | "zerovec", 524 | ] 525 | 526 | [[package]] 527 | name = "icu_properties_data" 528 | version = "1.5.0" 529 | source = "registry+https://github.com/rust-lang/crates.io-index" 530 | checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" 531 | 532 | [[package]] 533 | name = "icu_provider" 534 | version = "1.5.0" 535 | source = "registry+https://github.com/rust-lang/crates.io-index" 536 | checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" 537 | dependencies = [ 538 | "displaydoc", 539 | "icu_locid", 540 | "icu_provider_macros", 541 | "stable_deref_trait", 542 | "tinystr", 543 | "writeable", 544 | "yoke", 545 | "zerofrom", 546 | "zerovec", 547 | ] 548 | 549 | [[package]] 550 | name = "icu_provider_macros" 551 | version = "1.5.0" 552 | source = "registry+https://github.com/rust-lang/crates.io-index" 553 | checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" 554 | dependencies = [ 555 | "proc-macro2", 556 | "quote", 557 | "syn", 558 | ] 559 | 560 | [[package]] 561 | name = "idna" 562 | version = "1.0.3" 563 | source = "registry+https://github.com/rust-lang/crates.io-index" 564 | checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" 565 | dependencies = [ 566 | "idna_adapter", 567 | "smallvec", 568 | "utf8_iter", 569 | ] 570 | 571 | [[package]] 572 | name = "idna_adapter" 573 | version = "1.2.0" 574 | source = "registry+https://github.com/rust-lang/crates.io-index" 575 | checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" 576 | dependencies = [ 577 | "icu_normalizer", 578 | "icu_properties", 579 | ] 580 | 581 | [[package]] 582 | name = "indexmap" 583 | version = "2.7.0" 584 | source = "registry+https://github.com/rust-lang/crates.io-index" 585 | checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" 586 | dependencies = [ 587 | "equivalent", 588 | "hashbrown", 589 | ] 590 | 591 | [[package]] 592 | name = "indoc" 593 | version = "2.0.5" 594 | source = "registry+https://github.com/rust-lang/crates.io-index" 595 | checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" 596 | 597 | [[package]] 598 | name = "ipnet" 599 | version = "2.10.1" 600 | source = "registry+https://github.com/rust-lang/crates.io-index" 601 | checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" 602 | 603 | [[package]] 604 | name = "itoa" 605 | version = "1.0.14" 606 | source = "registry+https://github.com/rust-lang/crates.io-index" 607 | checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" 608 | 609 | [[package]] 610 | name = "js-sys" 611 | version = "0.3.76" 612 | source = "registry+https://github.com/rust-lang/crates.io-index" 613 | checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" 614 | dependencies = [ 615 | "once_cell", 616 | "wasm-bindgen", 617 | ] 618 | 619 | [[package]] 620 | name = "libc" 621 | version = "0.2.169" 622 | source = "registry+https://github.com/rust-lang/crates.io-index" 623 | checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" 624 | 625 | [[package]] 626 | name = "linux-raw-sys" 627 | version = "0.4.14" 628 | source = "registry+https://github.com/rust-lang/crates.io-index" 629 | checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" 630 | 631 | [[package]] 632 | name = "litemap" 633 | version = "0.7.4" 634 | source = "registry+https://github.com/rust-lang/crates.io-index" 635 | checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" 636 | 637 | [[package]] 638 | name = "log" 639 | version = "0.4.22" 640 | source = "registry+https://github.com/rust-lang/crates.io-index" 641 | checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" 642 | 643 | [[package]] 644 | name = "memchr" 645 | version = "2.7.4" 646 | source = "registry+https://github.com/rust-lang/crates.io-index" 647 | checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" 648 | 649 | [[package]] 650 | name = "memoffset" 651 | version = "0.9.1" 652 | source = "registry+https://github.com/rust-lang/crates.io-index" 653 | checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" 654 | dependencies = [ 655 | "autocfg", 656 | ] 657 | 658 | [[package]] 659 | name = "mime" 660 | version = "0.3.17" 661 | source = "registry+https://github.com/rust-lang/crates.io-index" 662 | checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" 663 | 664 | [[package]] 665 | name = "miniz_oxide" 666 | version = "0.8.2" 667 | source = "registry+https://github.com/rust-lang/crates.io-index" 668 | checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" 669 | dependencies = [ 670 | "adler2", 671 | ] 672 | 673 | [[package]] 674 | name = "mio" 675 | version = "1.0.3" 676 | source = "registry+https://github.com/rust-lang/crates.io-index" 677 | checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" 678 | dependencies = [ 679 | "libc", 680 | "wasi", 681 | "windows-sys 0.52.0", 682 | ] 683 | 684 | [[package]] 685 | name = "native-tls" 686 | version = "0.2.12" 687 | source = "registry+https://github.com/rust-lang/crates.io-index" 688 | checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" 689 | dependencies = [ 690 | "libc", 691 | "log", 692 | "openssl", 693 | "openssl-probe", 694 | "openssl-sys", 695 | "schannel", 696 | "security-framework", 697 | "security-framework-sys", 698 | "tempfile", 699 | ] 700 | 701 | [[package]] 702 | name = "object" 703 | version = "0.36.7" 704 | source = "registry+https://github.com/rust-lang/crates.io-index" 705 | checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" 706 | dependencies = [ 707 | "memchr", 708 | ] 709 | 710 | [[package]] 711 | name = "once_cell" 712 | version = "1.20.2" 713 | source = "registry+https://github.com/rust-lang/crates.io-index" 714 | checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" 715 | 716 | [[package]] 717 | name = "openssl" 718 | version = "0.10.68" 719 | source = "registry+https://github.com/rust-lang/crates.io-index" 720 | checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" 721 | dependencies = [ 722 | "bitflags", 723 | "cfg-if", 724 | "foreign-types", 725 | "libc", 726 | "once_cell", 727 | "openssl-macros", 728 | "openssl-sys", 729 | ] 730 | 731 | [[package]] 732 | name = "openssl-macros" 733 | version = "0.1.1" 734 | source = "registry+https://github.com/rust-lang/crates.io-index" 735 | checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" 736 | dependencies = [ 737 | "proc-macro2", 738 | "quote", 739 | "syn", 740 | ] 741 | 742 | [[package]] 743 | name = "openssl-probe" 744 | version = "0.1.5" 745 | source = "registry+https://github.com/rust-lang/crates.io-index" 746 | checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" 747 | 748 | [[package]] 749 | name = "openssl-src" 750 | version = "300.4.1+3.4.0" 751 | source = "registry+https://github.com/rust-lang/crates.io-index" 752 | checksum = "faa4eac4138c62414b5622d1b31c5c304f34b406b013c079c2bbc652fdd6678c" 753 | dependencies = [ 754 | "cc", 755 | ] 756 | 757 | [[package]] 758 | name = "openssl-sys" 759 | version = "0.9.104" 760 | source = "registry+https://github.com/rust-lang/crates.io-index" 761 | checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" 762 | dependencies = [ 763 | "cc", 764 | "libc", 765 | "openssl-src", 766 | "pkg-config", 767 | "vcpkg", 768 | ] 769 | 770 | [[package]] 771 | name = "percent-encoding" 772 | version = "2.3.1" 773 | source = "registry+https://github.com/rust-lang/crates.io-index" 774 | checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" 775 | 776 | [[package]] 777 | name = "pin-project-lite" 778 | version = "0.2.15" 779 | source = "registry+https://github.com/rust-lang/crates.io-index" 780 | checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" 781 | 782 | [[package]] 783 | name = "pin-utils" 784 | version = "0.1.0" 785 | source = "registry+https://github.com/rust-lang/crates.io-index" 786 | checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" 787 | 788 | [[package]] 789 | name = "pkg-config" 790 | version = "0.3.31" 791 | source = "registry+https://github.com/rust-lang/crates.io-index" 792 | checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" 793 | 794 | [[package]] 795 | name = "portable-atomic" 796 | version = "1.10.0" 797 | source = "registry+https://github.com/rust-lang/crates.io-index" 798 | checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" 799 | 800 | [[package]] 801 | name = "ppv-lite86" 802 | version = "0.2.20" 803 | source = "registry+https://github.com/rust-lang/crates.io-index" 804 | checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" 805 | dependencies = [ 806 | "zerocopy", 807 | ] 808 | 809 | [[package]] 810 | name = "proc-macro2" 811 | version = "1.0.92" 812 | source = "registry+https://github.com/rust-lang/crates.io-index" 813 | checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" 814 | dependencies = [ 815 | "unicode-ident", 816 | ] 817 | 818 | [[package]] 819 | name = "pyo3" 820 | version = "0.23.3" 821 | source = "registry+https://github.com/rust-lang/crates.io-index" 822 | checksum = "e484fd2c8b4cb67ab05a318f1fd6fa8f199fcc30819f08f07d200809dba26c15" 823 | dependencies = [ 824 | "cfg-if", 825 | "indoc", 826 | "libc", 827 | "memoffset", 828 | "once_cell", 829 | "portable-atomic", 830 | "pyo3-build-config", 831 | "pyo3-ffi", 832 | "pyo3-macros", 833 | "unindent", 834 | ] 835 | 836 | [[package]] 837 | name = "pyo3-build-config" 838 | version = "0.23.3" 839 | source = "registry+https://github.com/rust-lang/crates.io-index" 840 | checksum = "dc0e0469a84f208e20044b98965e1561028180219e35352a2afaf2b942beff3b" 841 | dependencies = [ 842 | "once_cell", 843 | "target-lexicon", 844 | ] 845 | 846 | [[package]] 847 | name = "pyo3-ffi" 848 | version = "0.23.3" 849 | source = "registry+https://github.com/rust-lang/crates.io-index" 850 | checksum = "eb1547a7f9966f6f1a0f0227564a9945fe36b90da5a93b3933fc3dc03fae372d" 851 | dependencies = [ 852 | "libc", 853 | "pyo3-build-config", 854 | ] 855 | 856 | [[package]] 857 | name = "pyo3-macros" 858 | version = "0.23.3" 859 | source = "registry+https://github.com/rust-lang/crates.io-index" 860 | checksum = "fdb6da8ec6fa5cedd1626c886fc8749bdcbb09424a86461eb8cdf096b7c33257" 861 | dependencies = [ 862 | "proc-macro2", 863 | "pyo3-macros-backend", 864 | "quote", 865 | "syn", 866 | ] 867 | 868 | [[package]] 869 | name = "pyo3-macros-backend" 870 | version = "0.23.3" 871 | source = "registry+https://github.com/rust-lang/crates.io-index" 872 | checksum = "38a385202ff5a92791168b1136afae5059d3ac118457bb7bc304c197c2d33e7d" 873 | dependencies = [ 874 | "heck", 875 | "proc-macro2", 876 | "pyo3-build-config", 877 | "quote", 878 | "syn", 879 | ] 880 | 881 | [[package]] 882 | name = "quote" 883 | version = "1.0.37" 884 | source = "registry+https://github.com/rust-lang/crates.io-index" 885 | checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" 886 | dependencies = [ 887 | "proc-macro2", 888 | ] 889 | 890 | [[package]] 891 | name = "rand" 892 | version = "0.8.5" 893 | source = "registry+https://github.com/rust-lang/crates.io-index" 894 | checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" 895 | dependencies = [ 896 | "libc", 897 | "rand_chacha", 898 | "rand_core", 899 | ] 900 | 901 | [[package]] 902 | name = "rand_chacha" 903 | version = "0.3.1" 904 | source = "registry+https://github.com/rust-lang/crates.io-index" 905 | checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" 906 | dependencies = [ 907 | "ppv-lite86", 908 | "rand_core", 909 | ] 910 | 911 | [[package]] 912 | name = "rand_core" 913 | version = "0.6.4" 914 | source = "registry+https://github.com/rust-lang/crates.io-index" 915 | checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" 916 | dependencies = [ 917 | "getrandom", 918 | ] 919 | 920 | [[package]] 921 | name = "reqwest" 922 | version = "0.12.9" 923 | source = "registry+https://github.com/rust-lang/crates.io-index" 924 | checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" 925 | dependencies = [ 926 | "base64", 927 | "bytes", 928 | "encoding_rs", 929 | "futures-core", 930 | "futures-util", 931 | "h2", 932 | "http", 933 | "http-body", 934 | "http-body-util", 935 | "hyper", 936 | "hyper-rustls", 937 | "hyper-tls", 938 | "hyper-util", 939 | "ipnet", 940 | "js-sys", 941 | "log", 942 | "mime", 943 | "native-tls", 944 | "once_cell", 945 | "percent-encoding", 946 | "pin-project-lite", 947 | "rustls-pemfile", 948 | "serde", 949 | "serde_json", 950 | "serde_urlencoded", 951 | "sync_wrapper", 952 | "system-configuration", 953 | "tokio", 954 | "tokio-native-tls", 955 | "tokio-util", 956 | "tower-service", 957 | "url", 958 | "wasm-bindgen", 959 | "wasm-bindgen-futures", 960 | "wasm-streams", 961 | "web-sys", 962 | "windows-registry", 963 | ] 964 | 965 | [[package]] 966 | name = "ring" 967 | version = "0.17.8" 968 | source = "registry+https://github.com/rust-lang/crates.io-index" 969 | checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" 970 | dependencies = [ 971 | "cc", 972 | "cfg-if", 973 | "getrandom", 974 | "libc", 975 | "spin", 976 | "untrusted", 977 | "windows-sys 0.52.0", 978 | ] 979 | 980 | [[package]] 981 | name = "rustc-demangle" 982 | version = "0.1.24" 983 | source = "registry+https://github.com/rust-lang/crates.io-index" 984 | checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" 985 | 986 | [[package]] 987 | name = "rustix" 988 | version = "0.38.42" 989 | source = "registry+https://github.com/rust-lang/crates.io-index" 990 | checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" 991 | dependencies = [ 992 | "bitflags", 993 | "errno", 994 | "libc", 995 | "linux-raw-sys", 996 | "windows-sys 0.59.0", 997 | ] 998 | 999 | [[package]] 1000 | name = "rustls" 1001 | version = "0.23.20" 1002 | source = "registry+https://github.com/rust-lang/crates.io-index" 1003 | checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" 1004 | dependencies = [ 1005 | "once_cell", 1006 | "rustls-pki-types", 1007 | "rustls-webpki", 1008 | "subtle", 1009 | "zeroize", 1010 | ] 1011 | 1012 | [[package]] 1013 | name = "rustls-pemfile" 1014 | version = "2.2.0" 1015 | source = "registry+https://github.com/rust-lang/crates.io-index" 1016 | checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" 1017 | dependencies = [ 1018 | "rustls-pki-types", 1019 | ] 1020 | 1021 | [[package]] 1022 | name = "rustls-pki-types" 1023 | version = "1.10.1" 1024 | source = "registry+https://github.com/rust-lang/crates.io-index" 1025 | checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" 1026 | 1027 | [[package]] 1028 | name = "rustls-webpki" 1029 | version = "0.102.8" 1030 | source = "registry+https://github.com/rust-lang/crates.io-index" 1031 | checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" 1032 | dependencies = [ 1033 | "ring", 1034 | "rustls-pki-types", 1035 | "untrusted", 1036 | ] 1037 | 1038 | [[package]] 1039 | name = "ryu" 1040 | version = "1.0.18" 1041 | source = "registry+https://github.com/rust-lang/crates.io-index" 1042 | checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" 1043 | 1044 | [[package]] 1045 | name = "schannel" 1046 | version = "0.1.27" 1047 | source = "registry+https://github.com/rust-lang/crates.io-index" 1048 | checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" 1049 | dependencies = [ 1050 | "windows-sys 0.59.0", 1051 | ] 1052 | 1053 | [[package]] 1054 | name = "security-framework" 1055 | version = "2.11.1" 1056 | source = "registry+https://github.com/rust-lang/crates.io-index" 1057 | checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" 1058 | dependencies = [ 1059 | "bitflags", 1060 | "core-foundation", 1061 | "core-foundation-sys", 1062 | "libc", 1063 | "security-framework-sys", 1064 | ] 1065 | 1066 | [[package]] 1067 | name = "security-framework-sys" 1068 | version = "2.13.0" 1069 | source = "registry+https://github.com/rust-lang/crates.io-index" 1070 | checksum = "1863fd3768cd83c56a7f60faa4dc0d403f1b6df0a38c3c25f44b7894e45370d5" 1071 | dependencies = [ 1072 | "core-foundation-sys", 1073 | "libc", 1074 | ] 1075 | 1076 | [[package]] 1077 | name = "serde" 1078 | version = "1.0.216" 1079 | source = "registry+https://github.com/rust-lang/crates.io-index" 1080 | checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" 1081 | dependencies = [ 1082 | "serde_derive", 1083 | ] 1084 | 1085 | [[package]] 1086 | name = "serde_derive" 1087 | version = "1.0.216" 1088 | source = "registry+https://github.com/rust-lang/crates.io-index" 1089 | checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" 1090 | dependencies = [ 1091 | "proc-macro2", 1092 | "quote", 1093 | "syn", 1094 | ] 1095 | 1096 | [[package]] 1097 | name = "serde_json" 1098 | version = "1.0.134" 1099 | source = "registry+https://github.com/rust-lang/crates.io-index" 1100 | checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" 1101 | dependencies = [ 1102 | "itoa", 1103 | "memchr", 1104 | "ryu", 1105 | "serde", 1106 | ] 1107 | 1108 | [[package]] 1109 | name = "serde_urlencoded" 1110 | version = "0.7.1" 1111 | source = "registry+https://github.com/rust-lang/crates.io-index" 1112 | checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" 1113 | dependencies = [ 1114 | "form_urlencoded", 1115 | "itoa", 1116 | "ryu", 1117 | "serde", 1118 | ] 1119 | 1120 | [[package]] 1121 | name = "shlex" 1122 | version = "1.3.0" 1123 | source = "registry+https://github.com/rust-lang/crates.io-index" 1124 | checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" 1125 | 1126 | [[package]] 1127 | name = "slab" 1128 | version = "0.4.9" 1129 | source = "registry+https://github.com/rust-lang/crates.io-index" 1130 | checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" 1131 | dependencies = [ 1132 | "autocfg", 1133 | ] 1134 | 1135 | [[package]] 1136 | name = "smallvec" 1137 | version = "1.13.2" 1138 | source = "registry+https://github.com/rust-lang/crates.io-index" 1139 | checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" 1140 | 1141 | [[package]] 1142 | name = "socket2" 1143 | version = "0.5.8" 1144 | source = "registry+https://github.com/rust-lang/crates.io-index" 1145 | checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" 1146 | dependencies = [ 1147 | "libc", 1148 | "windows-sys 0.52.0", 1149 | ] 1150 | 1151 | [[package]] 1152 | name = "spin" 1153 | version = "0.9.8" 1154 | source = "registry+https://github.com/rust-lang/crates.io-index" 1155 | checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" 1156 | 1157 | [[package]] 1158 | name = "stable_deref_trait" 1159 | version = "1.2.0" 1160 | source = "registry+https://github.com/rust-lang/crates.io-index" 1161 | checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" 1162 | 1163 | [[package]] 1164 | name = "subtle" 1165 | version = "2.6.1" 1166 | source = "registry+https://github.com/rust-lang/crates.io-index" 1167 | checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" 1168 | 1169 | [[package]] 1170 | name = "syn" 1171 | version = "2.0.91" 1172 | source = "registry+https://github.com/rust-lang/crates.io-index" 1173 | checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" 1174 | dependencies = [ 1175 | "proc-macro2", 1176 | "quote", 1177 | "unicode-ident", 1178 | ] 1179 | 1180 | [[package]] 1181 | name = "sync_wrapper" 1182 | version = "1.0.2" 1183 | source = "registry+https://github.com/rust-lang/crates.io-index" 1184 | checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" 1185 | dependencies = [ 1186 | "futures-core", 1187 | ] 1188 | 1189 | [[package]] 1190 | name = "synstructure" 1191 | version = "0.13.1" 1192 | source = "registry+https://github.com/rust-lang/crates.io-index" 1193 | checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" 1194 | dependencies = [ 1195 | "proc-macro2", 1196 | "quote", 1197 | "syn", 1198 | ] 1199 | 1200 | [[package]] 1201 | name = "system-configuration" 1202 | version = "0.6.1" 1203 | source = "registry+https://github.com/rust-lang/crates.io-index" 1204 | checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" 1205 | dependencies = [ 1206 | "bitflags", 1207 | "core-foundation", 1208 | "system-configuration-sys", 1209 | ] 1210 | 1211 | [[package]] 1212 | name = "system-configuration-sys" 1213 | version = "0.6.0" 1214 | source = "registry+https://github.com/rust-lang/crates.io-index" 1215 | checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" 1216 | dependencies = [ 1217 | "core-foundation-sys", 1218 | "libc", 1219 | ] 1220 | 1221 | [[package]] 1222 | name = "target-lexicon" 1223 | version = "0.12.16" 1224 | source = "registry+https://github.com/rust-lang/crates.io-index" 1225 | checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" 1226 | 1227 | [[package]] 1228 | name = "tempfile" 1229 | version = "3.14.0" 1230 | source = "registry+https://github.com/rust-lang/crates.io-index" 1231 | checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" 1232 | dependencies = [ 1233 | "cfg-if", 1234 | "fastrand", 1235 | "once_cell", 1236 | "rustix", 1237 | "windows-sys 0.59.0", 1238 | ] 1239 | 1240 | [[package]] 1241 | name = "tinystr" 1242 | version = "0.7.6" 1243 | source = "registry+https://github.com/rust-lang/crates.io-index" 1244 | checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" 1245 | dependencies = [ 1246 | "displaydoc", 1247 | "zerovec", 1248 | ] 1249 | 1250 | [[package]] 1251 | name = "tokio" 1252 | version = "1.42.0" 1253 | source = "registry+https://github.com/rust-lang/crates.io-index" 1254 | checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" 1255 | dependencies = [ 1256 | "backtrace", 1257 | "bytes", 1258 | "libc", 1259 | "mio", 1260 | "pin-project-lite", 1261 | "socket2", 1262 | "windows-sys 0.52.0", 1263 | ] 1264 | 1265 | [[package]] 1266 | name = "tokio-native-tls" 1267 | version = "0.3.1" 1268 | source = "registry+https://github.com/rust-lang/crates.io-index" 1269 | checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" 1270 | dependencies = [ 1271 | "native-tls", 1272 | "tokio", 1273 | ] 1274 | 1275 | [[package]] 1276 | name = "tokio-rustls" 1277 | version = "0.26.1" 1278 | source = "registry+https://github.com/rust-lang/crates.io-index" 1279 | checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" 1280 | dependencies = [ 1281 | "rustls", 1282 | "tokio", 1283 | ] 1284 | 1285 | [[package]] 1286 | name = "tokio-util" 1287 | version = "0.7.13" 1288 | source = "registry+https://github.com/rust-lang/crates.io-index" 1289 | checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" 1290 | dependencies = [ 1291 | "bytes", 1292 | "futures-core", 1293 | "futures-sink", 1294 | "pin-project-lite", 1295 | "tokio", 1296 | ] 1297 | 1298 | [[package]] 1299 | name = "tower-service" 1300 | version = "0.3.3" 1301 | source = "registry+https://github.com/rust-lang/crates.io-index" 1302 | checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" 1303 | 1304 | [[package]] 1305 | name = "tracing" 1306 | version = "0.1.41" 1307 | source = "registry+https://github.com/rust-lang/crates.io-index" 1308 | checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" 1309 | dependencies = [ 1310 | "pin-project-lite", 1311 | "tracing-core", 1312 | ] 1313 | 1314 | [[package]] 1315 | name = "tracing-core" 1316 | version = "0.1.33" 1317 | source = "registry+https://github.com/rust-lang/crates.io-index" 1318 | checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" 1319 | dependencies = [ 1320 | "once_cell", 1321 | ] 1322 | 1323 | [[package]] 1324 | name = "try-lock" 1325 | version = "0.2.5" 1326 | source = "registry+https://github.com/rust-lang/crates.io-index" 1327 | checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" 1328 | 1329 | [[package]] 1330 | name = "unicode-ident" 1331 | version = "1.0.14" 1332 | source = "registry+https://github.com/rust-lang/crates.io-index" 1333 | checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" 1334 | 1335 | [[package]] 1336 | name = "unindent" 1337 | version = "0.2.3" 1338 | source = "registry+https://github.com/rust-lang/crates.io-index" 1339 | checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" 1340 | 1341 | [[package]] 1342 | name = "untrusted" 1343 | version = "0.9.0" 1344 | source = "registry+https://github.com/rust-lang/crates.io-index" 1345 | checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" 1346 | 1347 | [[package]] 1348 | name = "url" 1349 | version = "2.5.4" 1350 | source = "registry+https://github.com/rust-lang/crates.io-index" 1351 | checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" 1352 | dependencies = [ 1353 | "form_urlencoded", 1354 | "idna", 1355 | "percent-encoding", 1356 | ] 1357 | 1358 | [[package]] 1359 | name = "utf16_iter" 1360 | version = "1.0.5" 1361 | source = "registry+https://github.com/rust-lang/crates.io-index" 1362 | checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" 1363 | 1364 | [[package]] 1365 | name = "utf8_iter" 1366 | version = "1.0.4" 1367 | source = "registry+https://github.com/rust-lang/crates.io-index" 1368 | checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" 1369 | 1370 | [[package]] 1371 | name = "vcpkg" 1372 | version = "0.2.15" 1373 | source = "registry+https://github.com/rust-lang/crates.io-index" 1374 | checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" 1375 | 1376 | [[package]] 1377 | name = "want" 1378 | version = "0.3.1" 1379 | source = "registry+https://github.com/rust-lang/crates.io-index" 1380 | checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" 1381 | dependencies = [ 1382 | "try-lock", 1383 | ] 1384 | 1385 | [[package]] 1386 | name = "wasi" 1387 | version = "0.11.0+wasi-snapshot-preview1" 1388 | source = "registry+https://github.com/rust-lang/crates.io-index" 1389 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" 1390 | 1391 | [[package]] 1392 | name = "wasm-bindgen" 1393 | version = "0.2.99" 1394 | source = "registry+https://github.com/rust-lang/crates.io-index" 1395 | checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" 1396 | dependencies = [ 1397 | "cfg-if", 1398 | "once_cell", 1399 | "wasm-bindgen-macro", 1400 | ] 1401 | 1402 | [[package]] 1403 | name = "wasm-bindgen-backend" 1404 | version = "0.2.99" 1405 | source = "registry+https://github.com/rust-lang/crates.io-index" 1406 | checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" 1407 | dependencies = [ 1408 | "bumpalo", 1409 | "log", 1410 | "proc-macro2", 1411 | "quote", 1412 | "syn", 1413 | "wasm-bindgen-shared", 1414 | ] 1415 | 1416 | [[package]] 1417 | name = "wasm-bindgen-futures" 1418 | version = "0.4.49" 1419 | source = "registry+https://github.com/rust-lang/crates.io-index" 1420 | checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" 1421 | dependencies = [ 1422 | "cfg-if", 1423 | "js-sys", 1424 | "once_cell", 1425 | "wasm-bindgen", 1426 | "web-sys", 1427 | ] 1428 | 1429 | [[package]] 1430 | name = "wasm-bindgen-macro" 1431 | version = "0.2.99" 1432 | source = "registry+https://github.com/rust-lang/crates.io-index" 1433 | checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" 1434 | dependencies = [ 1435 | "quote", 1436 | "wasm-bindgen-macro-support", 1437 | ] 1438 | 1439 | [[package]] 1440 | name = "wasm-bindgen-macro-support" 1441 | version = "0.2.99" 1442 | source = "registry+https://github.com/rust-lang/crates.io-index" 1443 | checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" 1444 | dependencies = [ 1445 | "proc-macro2", 1446 | "quote", 1447 | "syn", 1448 | "wasm-bindgen-backend", 1449 | "wasm-bindgen-shared", 1450 | ] 1451 | 1452 | [[package]] 1453 | name = "wasm-bindgen-shared" 1454 | version = "0.2.99" 1455 | source = "registry+https://github.com/rust-lang/crates.io-index" 1456 | checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" 1457 | 1458 | [[package]] 1459 | name = "wasm-streams" 1460 | version = "0.4.2" 1461 | source = "registry+https://github.com/rust-lang/crates.io-index" 1462 | checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" 1463 | dependencies = [ 1464 | "futures-util", 1465 | "js-sys", 1466 | "wasm-bindgen", 1467 | "wasm-bindgen-futures", 1468 | "web-sys", 1469 | ] 1470 | 1471 | [[package]] 1472 | name = "web-sys" 1473 | version = "0.3.76" 1474 | source = "registry+https://github.com/rust-lang/crates.io-index" 1475 | checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" 1476 | dependencies = [ 1477 | "js-sys", 1478 | "wasm-bindgen", 1479 | ] 1480 | 1481 | [[package]] 1482 | name = "windows-registry" 1483 | version = "0.2.0" 1484 | source = "registry+https://github.com/rust-lang/crates.io-index" 1485 | checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" 1486 | dependencies = [ 1487 | "windows-result", 1488 | "windows-strings", 1489 | "windows-targets", 1490 | ] 1491 | 1492 | [[package]] 1493 | name = "windows-result" 1494 | version = "0.2.0" 1495 | source = "registry+https://github.com/rust-lang/crates.io-index" 1496 | checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" 1497 | dependencies = [ 1498 | "windows-targets", 1499 | ] 1500 | 1501 | [[package]] 1502 | name = "windows-strings" 1503 | version = "0.1.0" 1504 | source = "registry+https://github.com/rust-lang/crates.io-index" 1505 | checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" 1506 | dependencies = [ 1507 | "windows-result", 1508 | "windows-targets", 1509 | ] 1510 | 1511 | [[package]] 1512 | name = "windows-sys" 1513 | version = "0.52.0" 1514 | source = "registry+https://github.com/rust-lang/crates.io-index" 1515 | checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" 1516 | dependencies = [ 1517 | "windows-targets", 1518 | ] 1519 | 1520 | [[package]] 1521 | name = "windows-sys" 1522 | version = "0.59.0" 1523 | source = "registry+https://github.com/rust-lang/crates.io-index" 1524 | checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" 1525 | dependencies = [ 1526 | "windows-targets", 1527 | ] 1528 | 1529 | [[package]] 1530 | name = "windows-targets" 1531 | version = "0.52.6" 1532 | source = "registry+https://github.com/rust-lang/crates.io-index" 1533 | checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" 1534 | dependencies = [ 1535 | "windows_aarch64_gnullvm", 1536 | "windows_aarch64_msvc", 1537 | "windows_i686_gnu", 1538 | "windows_i686_gnullvm", 1539 | "windows_i686_msvc", 1540 | "windows_x86_64_gnu", 1541 | "windows_x86_64_gnullvm", 1542 | "windows_x86_64_msvc", 1543 | ] 1544 | 1545 | [[package]] 1546 | name = "windows_aarch64_gnullvm" 1547 | version = "0.52.6" 1548 | source = "registry+https://github.com/rust-lang/crates.io-index" 1549 | checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" 1550 | 1551 | [[package]] 1552 | name = "windows_aarch64_msvc" 1553 | version = "0.52.6" 1554 | source = "registry+https://github.com/rust-lang/crates.io-index" 1555 | checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" 1556 | 1557 | [[package]] 1558 | name = "windows_i686_gnu" 1559 | version = "0.52.6" 1560 | source = "registry+https://github.com/rust-lang/crates.io-index" 1561 | checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" 1562 | 1563 | [[package]] 1564 | name = "windows_i686_gnullvm" 1565 | version = "0.52.6" 1566 | source = "registry+https://github.com/rust-lang/crates.io-index" 1567 | checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" 1568 | 1569 | [[package]] 1570 | name = "windows_i686_msvc" 1571 | version = "0.52.6" 1572 | source = "registry+https://github.com/rust-lang/crates.io-index" 1573 | checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" 1574 | 1575 | [[package]] 1576 | name = "windows_x86_64_gnu" 1577 | version = "0.52.6" 1578 | source = "registry+https://github.com/rust-lang/crates.io-index" 1579 | checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" 1580 | 1581 | [[package]] 1582 | name = "windows_x86_64_gnullvm" 1583 | version = "0.52.6" 1584 | source = "registry+https://github.com/rust-lang/crates.io-index" 1585 | checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" 1586 | 1587 | [[package]] 1588 | name = "windows_x86_64_msvc" 1589 | version = "0.52.6" 1590 | source = "registry+https://github.com/rust-lang/crates.io-index" 1591 | checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" 1592 | 1593 | [[package]] 1594 | name = "write16" 1595 | version = "1.0.0" 1596 | source = "registry+https://github.com/rust-lang/crates.io-index" 1597 | checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" 1598 | 1599 | [[package]] 1600 | name = "writeable" 1601 | version = "0.5.5" 1602 | source = "registry+https://github.com/rust-lang/crates.io-index" 1603 | checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" 1604 | 1605 | [[package]] 1606 | name = "yoke" 1607 | version = "0.7.5" 1608 | source = "registry+https://github.com/rust-lang/crates.io-index" 1609 | checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" 1610 | dependencies = [ 1611 | "serde", 1612 | "stable_deref_trait", 1613 | "yoke-derive", 1614 | "zerofrom", 1615 | ] 1616 | 1617 | [[package]] 1618 | name = "yoke-derive" 1619 | version = "0.7.5" 1620 | source = "registry+https://github.com/rust-lang/crates.io-index" 1621 | checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" 1622 | dependencies = [ 1623 | "proc-macro2", 1624 | "quote", 1625 | "syn", 1626 | "synstructure", 1627 | ] 1628 | 1629 | [[package]] 1630 | name = "zerocopy" 1631 | version = "0.7.35" 1632 | source = "registry+https://github.com/rust-lang/crates.io-index" 1633 | checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" 1634 | dependencies = [ 1635 | "byteorder", 1636 | "zerocopy-derive", 1637 | ] 1638 | 1639 | [[package]] 1640 | name = "zerocopy-derive" 1641 | version = "0.7.35" 1642 | source = "registry+https://github.com/rust-lang/crates.io-index" 1643 | checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" 1644 | dependencies = [ 1645 | "proc-macro2", 1646 | "quote", 1647 | "syn", 1648 | ] 1649 | 1650 | [[package]] 1651 | name = "zerofrom" 1652 | version = "0.1.5" 1653 | source = "registry+https://github.com/rust-lang/crates.io-index" 1654 | checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" 1655 | dependencies = [ 1656 | "zerofrom-derive", 1657 | ] 1658 | 1659 | [[package]] 1660 | name = "zerofrom-derive" 1661 | version = "0.1.5" 1662 | source = "registry+https://github.com/rust-lang/crates.io-index" 1663 | checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" 1664 | dependencies = [ 1665 | "proc-macro2", 1666 | "quote", 1667 | "syn", 1668 | "synstructure", 1669 | ] 1670 | 1671 | [[package]] 1672 | name = "zeroize" 1673 | version = "1.8.1" 1674 | source = "registry+https://github.com/rust-lang/crates.io-index" 1675 | checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" 1676 | 1677 | [[package]] 1678 | name = "zerovec" 1679 | version = "0.10.4" 1680 | source = "registry+https://github.com/rust-lang/crates.io-index" 1681 | checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" 1682 | dependencies = [ 1683 | "yoke", 1684 | "zerofrom", 1685 | "zerovec-derive", 1686 | ] 1687 | 1688 | [[package]] 1689 | name = "zerovec-derive" 1690 | version = "0.10.3" 1691 | source = "registry+https://github.com/rust-lang/crates.io-index" 1692 | checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" 1693 | dependencies = [ 1694 | "proc-macro2", 1695 | "quote", 1696 | "syn", 1697 | ] 1698 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "hf_transfer" 3 | version = "0.1.9-dev0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | [lib] 8 | name = "hf_transfer" 9 | crate-type = ["cdylib"] 10 | 11 | [dependencies] 12 | futures = "0.3" 13 | openssl = { version = "0.10", features = ["vendored"] } 14 | pyo3 = { version = "0.23", features = ["extension-module", "abi3-py38"] } 15 | rand = "0.8.5" 16 | reqwest = { version = "0.12", features = ["stream"] } 17 | tokio = { version = "1.42", features = ["rt", "rt-multi-thread", "fs"] } 18 | tokio-util = { version = "0.7", features = ["codec"] } 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2023 HuggingFace Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HF Transfer 2 | 3 | Speed up file transfers with the Hub. 4 | 5 | # DISCLAIMER 6 | 7 | This library is a power user tool, to go beyond `~500MB/s` on very high bandwidth 8 | network, where Python cannot cap out the available bandwidth. 9 | 10 | This is *not* meant to be a general usability tool. 11 | It purposefully lacks progressbars and comes generally as-is. 12 | 13 | Please file issues *only* if there's an issue on the underlying downloaded file. 14 | 15 | ## Contributing 16 | 17 | ```sh 18 | python3 -m venv ~/.venv/hf_transfer 19 | source ~/.venv/hf_transfer/bin/activate 20 | pip install maturin 21 | maturin develop 22 | ``` 23 | 24 | ### `huggingface_hub` 25 | 26 | If you are working on changes with `huggingface_hub` 27 | 28 | ```sh 29 | git clone git@github.com:huggingface/huggingface_hub.git 30 | # git clone https://github.com/huggingface/huggingface_hub.git 31 | 32 | cd huggingface_hub 33 | python3 -m pip install -e ".[quality]" 34 | ``` 35 | 36 | You can use the following test script: 37 | 38 | ```py 39 | import os 40 | 41 | # os.environ["HF_ENDPOINT"] = "http://localhost:5564" 42 | os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" 43 | 44 | from huggingface_hub import HfApi, logging 45 | 46 | logging.set_verbosity_debug() 47 | hf = HfApi() 48 | hf.upload_file(path_or_fileobj="/path/to/my/repo/some_file", path_in_repo="some_file", repo_id="my/repo", repo_type="model") 49 | ``` 50 | 51 | -------------------------------------------------------------------------------- /examples/upload_to_s3.py: -------------------------------------------------------------------------------- 1 | # pip install boto3 hf_transfer 2 | 3 | import boto3 4 | from hf_transfer import multipart_upload 5 | from math import ceil 6 | import os 7 | from time import time 8 | 9 | # 10 MiB 10 | CHUNK_SIZE = 10_485_760 11 | 12 | s3 = boto3.client("s3") 13 | 14 | bucket = "test-hf-transfer-multi-part-upload" 15 | bucket_key = "some_file" 16 | 17 | upload = s3.create_multipart_upload( 18 | ACL="bucket-owner-full-control", 19 | Bucket=bucket, 20 | Key=bucket_key, 21 | ) 22 | upload_id = upload["UploadId"] 23 | print("created multipart upload") 24 | 25 | file_name = "some_file" 26 | file_size = os.stat(file_name).st_size 27 | 28 | urls = [] 29 | nb_parts = ceil(file_size / CHUNK_SIZE) 30 | for part_number in range(1, nb_parts + 1): 31 | params = { 32 | "Bucket": bucket, 33 | "Key": bucket_key, 34 | "PartNumber": part_number, 35 | "UploadId": upload_id, 36 | } 37 | urls.append( 38 | s3.generate_presigned_url( 39 | ClientMethod="upload_part", Params=params, ExpiresIn=86400 40 | ) 41 | ) 42 | print("prepared parts urls") 43 | 44 | print("uploading parts...") 45 | start = time() 46 | responses = multipart_upload( 47 | file_path=file_name, 48 | parts_urls=urls, 49 | chunk_size=CHUNK_SIZE, 50 | max_files=64, 51 | parallel_failures=63, 52 | max_retries=5, 53 | ) 54 | print(f"uploaded parts in {time() - start}") 55 | 56 | etag_with_parts = [] 57 | for part_number, header in enumerate(responses): 58 | etag = header.get("etag") 59 | etag_with_parts.append({"ETag": etag, "PartNumber": part_number + 1}) 60 | 61 | parts = {"Parts": etag_with_parts} 62 | 63 | s3.complete_multipart_upload( 64 | Bucket=bucket, Key=bucket_key, MultipartUpload=parts, UploadId=upload_id 65 | ) 66 | print("upload complete") 67 | 68 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["maturin>=1.4,<2"] 3 | build-backend = "maturin" 4 | 5 | [project] 6 | name = "hf_transfer" 7 | description = "Speed up file transfers with the Hugging Face Hub." 8 | readme = "README.md" 9 | license = {file = "LICENSE"} 10 | requires-python = ">=3.7" 11 | classifiers = [ 12 | "Programming Language :: Rust", 13 | "Programming Language :: Python :: Implementation :: CPython", 14 | "Programming Language :: Python :: Implementation :: PyPy", 15 | ] 16 | dynamic = ["version"] 17 | 18 | [project.urls] 19 | Issues = "https://github.com/huggingface/hf_transfer/issues" 20 | Repository = "https://github.com/huggingface/hf_transfer.git" 21 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | use futures::stream::FuturesUnordered; 2 | use futures::StreamExt; 3 | use pyo3::exceptions::PyException; 4 | use pyo3::prelude::*; 5 | use pyo3::types::PyAny; 6 | use rand::{thread_rng, Rng}; 7 | use reqwest::header::{ 8 | HeaderMap, HeaderName, HeaderValue, ToStrError, AUTHORIZATION, CONTENT_LENGTH, CONTENT_RANGE, 9 | RANGE, 10 | }; 11 | use reqwest::Url; 12 | use std::collections::HashMap; 13 | use std::fmt::Display; 14 | use std::fs::remove_file; 15 | use std::io::SeekFrom; 16 | use std::path::Path; 17 | use std::sync::Arc; 18 | use std::time::Duration; 19 | use tokio::fs::OpenOptions; 20 | use tokio::io::AsyncWriteExt; 21 | use tokio::io::{AsyncReadExt, AsyncSeekExt}; 22 | use tokio::sync::Semaphore; 23 | use tokio::time::sleep; 24 | use tokio_util::codec::{BytesCodec, FramedRead}; 25 | 26 | const BASE_WAIT_TIME: usize = 300; 27 | const MAX_WAIT_TIME: usize = 10_000; 28 | 29 | /// max_files: Number of open file handles, which determines the maximum number of parallel downloads 30 | /// parallel_failures: Number of maximum failures of different chunks in parallel (cannot exceed max_files) 31 | /// max_retries: Number of maximum attempts per chunk. (Retries are exponentially backed off + jitter) 32 | /// 33 | /// The number of threads can be tuned by the environment variable `TOKIO_WORKER_THREADS` as documented in 34 | /// https://docs.rs/tokio/latest/tokio/runtime/struct.Builder.html#method.worker_threads 35 | #[allow(clippy::too_many_arguments)] 36 | #[pyfunction] 37 | #[pyo3(signature = (url, filename, max_files, chunk_size, parallel_failures=0, max_retries=0, headers=None, callback=None))] 38 | fn download( 39 | url: String, 40 | filename: String, 41 | max_files: usize, 42 | chunk_size: usize, 43 | parallel_failures: usize, 44 | max_retries: usize, 45 | headers: Option>, 46 | callback: Option>, 47 | ) -> PyResult<()> { 48 | if parallel_failures > max_files { 49 | return Err(PyException::new_err( 50 | "Error parallel_failures cannot be > max_files".to_string(), 51 | )); 52 | } 53 | if (parallel_failures == 0) != (max_retries == 0) { 54 | return Err(PyException::new_err( 55 | "For retry mechanism you need to set both `parallel_failures` and `max_retries`" 56 | .to_string(), 57 | )); 58 | } 59 | tokio::runtime::Builder::new_multi_thread() 60 | .enable_all() 61 | .build()? 62 | .block_on(async { 63 | download_async( 64 | url, 65 | filename.clone(), 66 | max_files, 67 | chunk_size, 68 | parallel_failures, 69 | max_retries, 70 | headers, 71 | callback, 72 | ) 73 | .await 74 | }) 75 | .map_err(|err| { 76 | let path = Path::new(&filename); 77 | if path.exists() { 78 | match remove_file(filename) { 79 | Ok(_) => err, 80 | Err(err) => { 81 | PyException::new_err(format!("Error while removing corrupted file: {err}")) 82 | } 83 | } 84 | } else { 85 | err 86 | } 87 | }) 88 | } 89 | 90 | /// parts_urls: Dictionary consisting of part numbers as keys and the associated url as values 91 | /// completion_url: The url that should be called when the upload is finished 92 | /// max_files: Number of open file handles, which determines the maximum number of parallel uploads 93 | /// parallel_failures: Number of maximum failures of different chunks in parallel (cannot exceed max_files) 94 | /// max_retries: Number of maximum attempts per chunk. (Retries are exponentially backed off + jitter) 95 | /// 96 | /// The number of threads can be tuned by the environment variable `TOKIO_WORKER_THREADS` as documented in 97 | /// https://docs.rs/tokio/latest/tokio/runtime/struct.Builder.html#method.worker_threads 98 | /// 99 | /// See https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html for more information 100 | /// on the multipart upload 101 | #[allow(clippy::too_many_arguments)] 102 | #[pyfunction] 103 | #[pyo3(signature = (file_path, parts_urls, chunk_size, max_files, parallel_failures=0, max_retries=0, callback=None))] 104 | fn multipart_upload( 105 | file_path: String, 106 | parts_urls: Vec, 107 | chunk_size: u64, 108 | max_files: usize, 109 | parallel_failures: usize, 110 | max_retries: usize, 111 | callback: Option>, 112 | ) -> PyResult>> { 113 | if parallel_failures > max_files { 114 | return Err(PyException::new_err( 115 | "Error parallel_failures cannot be > max_files".to_string(), 116 | )); 117 | } 118 | if (parallel_failures == 0) != (max_retries == 0) { 119 | return Err(PyException::new_err( 120 | "For retry mechanism you need to set both `parallel_failures` and `max_retries`" 121 | .to_string(), 122 | )); 123 | } 124 | 125 | tokio::runtime::Builder::new_multi_thread() 126 | .enable_all() 127 | .build()? 128 | .block_on(async { 129 | upload_async( 130 | file_path, 131 | parts_urls, 132 | chunk_size, 133 | max_files, 134 | parallel_failures, 135 | max_retries, 136 | callback, 137 | ) 138 | .await 139 | }) 140 | } 141 | 142 | fn jitter() -> usize { 143 | thread_rng().gen_range(0..=500) 144 | } 145 | 146 | pub fn exponential_backoff(base_wait_time: usize, n: usize, max: usize) -> usize { 147 | (base_wait_time + n.pow(2) + jitter()).min(max) 148 | } 149 | 150 | #[allow(clippy::too_many_arguments)] 151 | async fn download_async( 152 | url: String, 153 | filename: String, 154 | max_files: usize, 155 | chunk_size: usize, 156 | parallel_failures: usize, 157 | max_retries: usize, 158 | input_headers: Option>, 159 | callback: Option>, 160 | ) -> PyResult<()> { 161 | let client = reqwest::Client::builder() 162 | // https://github.com/hyperium/hyper/issues/2136#issuecomment-589488526 163 | .http2_keep_alive_timeout(Duration::from_secs(15)) 164 | .build() 165 | .unwrap(); 166 | 167 | let mut headers = HeaderMap::new(); 168 | let mut auth_token = None; 169 | if let Some(input_headers) = input_headers { 170 | headers.reserve(input_headers.len()); 171 | for (k, v) in input_headers { 172 | let name: HeaderName = k 173 | .try_into() 174 | .map_err(|err| PyException::new_err(format!("Invalid header: {err}")))?; 175 | let value: HeaderValue = AsRef::::as_ref(&v) 176 | .try_into() 177 | .map_err(|err| PyException::new_err(format!("Invalid header value: {err}")))?; 178 | if name == AUTHORIZATION { 179 | auth_token = Some(value); 180 | } else { 181 | headers.insert(name, value); 182 | } 183 | } 184 | }; 185 | 186 | let response = if let Some(token) = auth_token.as_ref() { 187 | client.get(&url).header(AUTHORIZATION, token) 188 | } else { 189 | client.get(&url) 190 | } 191 | .headers(headers.clone()) 192 | .header(RANGE, "bytes=0-0") 193 | .send() 194 | .await 195 | .map_err(|err| PyException::new_err(format!("Error while downloading: {err}")))? 196 | .error_for_status() 197 | .map_err(|err| PyException::new_err(err.to_string()))?; 198 | 199 | // Only call the final redirect URL to avoid overloading the Hub with requests and also 200 | // altering the download count 201 | let redirected_url = response.url(); 202 | if Url::parse(&url) 203 | .map_err(|err| PyException::new_err(format!("failed to parse url: {err}")))? 204 | .host() 205 | == redirected_url.host() 206 | { 207 | if let Some(token) = auth_token { 208 | headers.insert(AUTHORIZATION, token); 209 | } 210 | } 211 | 212 | let content_range = response 213 | .headers() 214 | .get(CONTENT_RANGE) 215 | .ok_or(PyException::new_err("No content length"))? 216 | .to_str() 217 | .map_err(|err| PyException::new_err(format!("Error while downloading: {err}")))?; 218 | 219 | let size: Vec<&str> = content_range.split('/').collect(); 220 | // Content-Range: bytes 0-0/702517648 221 | // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range 222 | let length: usize = size 223 | .last() 224 | .ok_or(PyException::new_err( 225 | "Error while downloading: No size was detected", 226 | ))? 227 | .parse() 228 | .map_err(|err| PyException::new_err(format!("Error while downloading: {err}")))?; 229 | 230 | let mut handles = FuturesUnordered::new(); 231 | let semaphore = Arc::new(Semaphore::new(max_files)); 232 | let parallel_failures_semaphore = Arc::new(Semaphore::new(parallel_failures)); 233 | 234 | for start in (0..length).step_by(chunk_size) { 235 | let url = redirected_url.to_string(); 236 | let filename = filename.clone(); 237 | let client = client.clone(); 238 | let headers = headers.clone(); 239 | 240 | let stop = std::cmp::min(start + chunk_size - 1, length); 241 | let semaphore = semaphore.clone(); 242 | let parallel_failures_semaphore = parallel_failures_semaphore.clone(); 243 | handles.push(tokio::spawn(async move { 244 | let permit = semaphore 245 | .acquire_owned() 246 | .await 247 | .map_err(|err| PyException::new_err(format!("Error while downloading: {err}")))?; 248 | let mut chunk = download_chunk(&client, &url, &filename, start, stop, headers.clone()).await; 249 | let mut i = 0; 250 | if parallel_failures > 0 { 251 | while let Err(dlerr) = chunk { 252 | if i >= max_retries { 253 | return Err(PyException::new_err(format!( 254 | "Failed after too many retries ({max_retries}): {dlerr}" 255 | ))); 256 | } 257 | let parallel_failure_permit = parallel_failures_semaphore.clone().try_acquire_owned().map_err(|err| { 258 | PyException::new_err(format!( 259 | "Failed too many failures in parallel ({parallel_failures}): {dlerr} ({err})" 260 | )) 261 | })?; 262 | 263 | let wait_time = exponential_backoff(BASE_WAIT_TIME, i, MAX_WAIT_TIME); 264 | sleep(Duration::from_millis(wait_time as u64)).await; 265 | 266 | chunk = download_chunk(&client, &url, &filename, start, stop, headers.clone()).await; 267 | i += 1; 268 | drop(parallel_failure_permit); 269 | } 270 | } 271 | drop(permit); 272 | chunk.map_err(|e| PyException::new_err(format!("Downloading error {e}"))).and(Ok(stop - start)) 273 | })); 274 | } 275 | 276 | // Output the chained result 277 | while let Some(result) = handles.next().await { 278 | match result { 279 | Ok(Ok(size)) => { 280 | if let Some(ref callback) = callback { 281 | callback.call((size,), None)?; 282 | } 283 | } 284 | Ok(Err(py_err)) => { 285 | return Err(py_err); 286 | } 287 | Err(err) => { 288 | return Err(PyException::new_err(format!( 289 | "Error while downloading: {err}" 290 | ))); 291 | } 292 | } 293 | } 294 | Ok(()) 295 | } 296 | 297 | #[derive(Debug)] 298 | enum Error { 299 | Io(std::io::Error), 300 | Request(reqwest::Error), 301 | ToStrError(ToStrError), 302 | } 303 | 304 | impl From for Error { 305 | fn from(value: std::io::Error) -> Self { 306 | Self::Io(value) 307 | } 308 | } 309 | 310 | impl From for Error { 311 | fn from(value: reqwest::Error) -> Self { 312 | Self::Request(value) 313 | } 314 | } 315 | 316 | impl From for Error { 317 | fn from(value: ToStrError) -> Self { 318 | Self::ToStrError(value) 319 | } 320 | } 321 | 322 | impl Display for Error { 323 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 324 | match self { 325 | Self::Io(io) => write!(f, "Io: {io}"), 326 | Self::Request(req) => write!(f, "Request: {req}"), 327 | Self::ToStrError(req) => write!(f, "Response non ascii: {req}"), 328 | } 329 | } 330 | } 331 | 332 | impl std::error::Error for Error {} 333 | 334 | async fn download_chunk( 335 | client: &reqwest::Client, 336 | url: &str, 337 | filename: &str, 338 | start: usize, 339 | stop: usize, 340 | headers: HeaderMap, 341 | ) -> Result<(), Error> { 342 | // Process each socket concurrently. 343 | let range = format!("bytes={start}-{stop}"); 344 | let mut file = OpenOptions::new() 345 | .write(true) 346 | .truncate(false) 347 | .create(true) 348 | .open(filename) 349 | .await?; 350 | file.seek(SeekFrom::Start(start as u64)).await?; 351 | let response = client 352 | .get(url) 353 | .headers(headers) 354 | .header(RANGE, range) 355 | .send() 356 | .await? 357 | .error_for_status()?; 358 | let content = response.bytes().await?; 359 | file.write_all(&content).await?; 360 | Ok(()) 361 | } 362 | 363 | #[allow(clippy::too_many_arguments)] 364 | async fn upload_async( 365 | file_path: String, 366 | parts_urls: Vec, 367 | chunk_size: u64, 368 | max_files: usize, 369 | parallel_failures: usize, 370 | max_retries: usize, 371 | callback: Option>, 372 | ) -> PyResult>> { 373 | let client = reqwest::Client::new(); 374 | 375 | let mut handles = FuturesUnordered::new(); 376 | let semaphore = Arc::new(Semaphore::new(max_files)); 377 | let parallel_failures_semaphore = Arc::new(Semaphore::new(parallel_failures)); 378 | 379 | for (part_number, part_url) in parts_urls.iter().enumerate() { 380 | let url = part_url.to_string(); 381 | let path = file_path.to_owned(); 382 | let client = client.clone(); 383 | 384 | let start = (part_number as u64) * chunk_size; 385 | let semaphore = semaphore.clone(); 386 | let parallel_failures_semaphore = parallel_failures_semaphore.clone(); 387 | handles.push(tokio::spawn(async move { 388 | let permit = semaphore 389 | .clone() 390 | .acquire_owned() 391 | .await 392 | .map_err(|err| PyException::new_err(format!("Error acquiring semaphore: {err}")))?; 393 | let mut chunk = upload_chunk(&client, &url, &path, start, chunk_size).await; 394 | let mut i = 0; 395 | if parallel_failures > 0 { 396 | while let Err(ul_err) = chunk { 397 | if i >= max_retries { 398 | return Err(PyException::new_err(format!( 399 | "Failed after too many retries ({max_retries}): {ul_err}" 400 | ))); 401 | } 402 | 403 | let parallel_failure_permit = parallel_failures_semaphore.clone().try_acquire_owned().map_err(|err| { 404 | PyException::new_err(format!( 405 | "Failed too many failures in parallel ({parallel_failures}): {ul_err} ({err})" 406 | )) 407 | })?; 408 | 409 | let wait_time = exponential_backoff(BASE_WAIT_TIME, i, MAX_WAIT_TIME); 410 | sleep(Duration::from_millis(wait_time as u64)).await; 411 | 412 | chunk = upload_chunk(&client, &url, &path, start, chunk_size).await; 413 | i += 1; 414 | drop(parallel_failure_permit); 415 | } 416 | } 417 | drop(permit); 418 | chunk.map_err(|e|{ 419 | match e { 420 | Error::Io(io) => PyException::new_err(format!("Io error {io}")), 421 | Error::Request(req) => PyException::new_err(format!("Error while sending chunk {req}")), 422 | Error::ToStrError(req) => PyException::new_err(format!("Response header contains non ASCII chars: {req}")), 423 | } 424 | }).map(|chunk| (part_number, chunk, chunk_size)) 425 | })); 426 | } 427 | 428 | let mut results: Vec> = vec![HashMap::default(); parts_urls.len()]; 429 | 430 | while let Some(result) = handles.next().await { 431 | match result { 432 | Ok(Ok((part_number, headers, size))) => { 433 | if let Some(ref callback) = callback { 434 | callback.call((size,), None)?; 435 | } 436 | results[part_number] = headers; 437 | } 438 | Ok(Err(py_err)) => { 439 | return Err(py_err); 440 | } 441 | Err(err) => { 442 | return Err(PyException::new_err(format!( 443 | "Error occurred while uploading: {err}" 444 | ))); 445 | } 446 | } 447 | } 448 | 449 | Ok(results) 450 | } 451 | 452 | async fn upload_chunk( 453 | client: &reqwest::Client, 454 | url: &str, 455 | path: &str, 456 | start: u64, 457 | chunk_size: u64, 458 | ) -> Result, Error> { 459 | let mut options = OpenOptions::new(); 460 | let mut file = options.read(true).open(path).await?; 461 | let file_size = file.metadata().await?.len(); 462 | let bytes_transferred = std::cmp::min(file_size - start, chunk_size); 463 | 464 | file.seek(SeekFrom::Start(start)).await?; 465 | let chunk = file.take(chunk_size); 466 | 467 | let response = client 468 | .put(url) 469 | .header(CONTENT_LENGTH, bytes_transferred) 470 | .body(reqwest::Body::wrap_stream(FramedRead::new( 471 | chunk, 472 | BytesCodec::new(), 473 | ))) 474 | .send() 475 | .await?; 476 | let response = response.error_for_status()?; 477 | let mut headers = HashMap::new(); 478 | for (name, value) in response.headers().into_iter() { 479 | headers.insert(name.to_string(), value.to_str()?.to_owned()); 480 | } 481 | Ok(headers) 482 | } 483 | 484 | /// A Python module implemented in Rust. 485 | #[pymodule] 486 | fn hf_transfer(module: &Bound<'_, PyModule>) -> PyResult<()> { 487 | module.add_function(wrap_pyfunction!(download, module)?)?; 488 | module.add_function(wrap_pyfunction!(multipart_upload, module)?)?; 489 | module.add("__version__", env!("CARGO_PKG_VERSION"))?; 490 | Ok(()) 491 | } 492 | --------------------------------------------------------------------------------