├── .dockerignore ├── .envrc ├── .gitignore ├── static ├── fonts │ ├── FiraSans-Light.woff │ ├── WorkSans-Medium.ttf │ ├── FiraSans-Medium.woff │ ├── FiraSans-Regular.woff │ └── Inconsolata-Regular.ttf ├── css │ ├── panamax.css │ ├── rustup.css │ └── normalize.css └── js │ └── panamax.js ├── src ├── progress_bar.rs ├── ARCHITECTURE.md ├── mirror.default.toml ├── main.rs ├── crates_index.rs ├── download.rs ├── serve.rs ├── mirror.rs ├── crates.rs ├── verify.rs └── rustup.rs ├── Dockerfile ├── LICENSE-MIT ├── Cargo.toml ├── nginx.sample.conf ├── .github └── workflows │ ├── container-test.yml │ └── main.yml ├── flake.nix ├── templates └── index.html ├── flake.lock ├── README.md └── LICENSE-APACHE /.dockerignore: -------------------------------------------------------------------------------- 1 | target -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | use flake 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | **/*.rs.bk 3 | .idea/ 4 | .direnv/ 5 | -------------------------------------------------------------------------------- /static/fonts/FiraSans-Light.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panamax-rs/panamax/HEAD/static/fonts/FiraSans-Light.woff -------------------------------------------------------------------------------- /static/fonts/WorkSans-Medium.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panamax-rs/panamax/HEAD/static/fonts/WorkSans-Medium.ttf -------------------------------------------------------------------------------- /static/fonts/FiraSans-Medium.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panamax-rs/panamax/HEAD/static/fonts/FiraSans-Medium.woff -------------------------------------------------------------------------------- /static/fonts/FiraSans-Regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panamax-rs/panamax/HEAD/static/fonts/FiraSans-Regular.woff -------------------------------------------------------------------------------- /static/fonts/Inconsolata-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panamax-rs/panamax/HEAD/static/fonts/Inconsolata-Regular.ttf -------------------------------------------------------------------------------- /static/css/panamax.css: -------------------------------------------------------------------------------- 1 | p:not(#pitch) { 2 | margin-bottom: auto !important; 3 | } 4 | 5 | .instructions { 6 | padding-bottom: 2em; 7 | } 8 | 9 | .instructions>div { 10 | width: 50rem !important; 11 | } 12 | 13 | .instructions>div>pre { 14 | text-align: left !important; 15 | height: auto !important; 16 | overflow-x: auto !important; 17 | } 18 | 19 | .instructions>select { 20 | width: auto; 21 | font-size: initial; 22 | margin-top: 0.5rem; 23 | } -------------------------------------------------------------------------------- /src/progress_bar.rs: -------------------------------------------------------------------------------- 1 | use console::{pad_str, style}; 2 | 3 | pub fn current_step_prefix(step: usize, steps: usize) -> String { 4 | style(format!("[{step}/{steps}]")).bold().to_string() 5 | } 6 | 7 | pub fn padded_prefix_message(step: usize, steps: usize, msg: &str) -> String { 8 | pad_str( 9 | &format!("{} {}...", current_step_prefix(step, steps), msg), 10 | 34, 11 | console::Alignment::Left, 12 | None, 13 | ) 14 | .to_string() 15 | } 16 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rust:latest AS builder 2 | 3 | WORKDIR /app 4 | 5 | #ADD --chown=rust:rust . /app/ 6 | ADD . /app/ 7 | 8 | ARG CARGO_BUILD_EXTRA=" " 9 | RUN cargo build --release ${CARGO_BUILD_EXTRA} 10 | 11 | FROM debian:latest 12 | 13 | COPY --from=builder /app/target/release/panamax /usr/local/bin 14 | 15 | RUN apt update \ 16 | && apt install -y \ 17 | ca-certificates \ 18 | git \ 19 | libssl3 \ 20 | && git config --global --add safe.directory '*' 21 | 22 | ENTRYPOINT [ "/usr/local/bin/panamax" ] 23 | CMD ["--help"] 24 | -------------------------------------------------------------------------------- /static/js/panamax.js: -------------------------------------------------------------------------------- 1 | function rustup_text_unix(host, platform) { 2 | return `wget ${host}/rustup/dist/${platform}/rustup-init 3 | chmod +x rustup-init 4 | ./rustup-init` 5 | } 6 | 7 | function rustup_text_win(host, platform) { 8 | return `Download rustup-init.exe here: 9 | ${host}/rustup/dist/${platform}/rustup-init.exe` 10 | } 11 | 12 | function platform_change() { 13 | let rustup_text = document.getElementById("rustup-text"); 14 | let rustup_platform = document.getElementById("rustup-selected-platform"); 15 | let host = document.getElementById("panamax-host").textContent; 16 | let platform = rustup_platform.options[rustup_platform.selectedIndex].text; 17 | let is_exe = rustup_platform.options[rustup_platform.selectedIndex].value; 18 | if (is_exe === "true") { 19 | rustup_text.innerHTML = rustup_text_win(host, platform); 20 | } else { 21 | rustup_text.innerHTML = rustup_text_unix(host, platform); 22 | } 23 | } -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019 k3d3 2 | 3 | Permission is hereby granted, free of charge, to any 4 | person obtaining a copy of this software and associated 5 | documentation files (the "Software"), to deal in the 6 | Software without restriction, including without 7 | limitation the rights to use, copy, modify, merge, 8 | publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software 10 | is furnished to do so, subject to the following 11 | conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions 15 | of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 18 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 19 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 20 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 21 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 22 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 24 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | DEALINGS IN THE SOFTWARE. 26 | 27 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "panamax" 3 | version = "1.0.14" 4 | authors = ["k3d3 "] 5 | description = "Mirror rustup and crates.io repositories, for offline Rust and Cargo usage." 6 | license = "MIT/Apache-2.0" 7 | readme = "README.md" 8 | homepage = "https://github.com/panamax-rs/panamax" 9 | repository = "https://github.com/panamax-rs/panamax" 10 | edition = "2021" 11 | 12 | [dependencies] 13 | reqwest = { version = "0.11", features = ["blocking"] } 14 | indicatif = "0.17" 15 | clap = { version = "4.1", features = ["derive"] } 16 | serde = { version = "1.0", features = ["derive"] } 17 | console = "0.15" 18 | log = "0.4" 19 | env_logger = "0.10" 20 | sha2 = "0.10" 21 | url = "2.2" 22 | glob = "0.3" 23 | git2 = "0.16" 24 | serde_json = "1.0" 25 | thiserror = "1.0" 26 | tokio = { version = "1.25", features = ["full"] } 27 | warp = { version = "0.3", features = ["tls"] } 28 | askama = "0.11" 29 | askama_warp = "0.12" 30 | include_dir = "0.7" 31 | bytes = "1.1" 32 | tokio-stream = "0.1" 33 | tokio-util = { version = "0.7", features = ["codec"] } 34 | futures-util = "0.3" 35 | futures = "0.3" 36 | walkdir = "2.3" 37 | toml_edit = {version = "0.14", features = ["easy"] } 38 | 39 | [features] 40 | default = [] 41 | dev_reduced_crates = [] 42 | vendored-openssl = ["reqwest/native-tls-vendored"] 43 | -------------------------------------------------------------------------------- /nginx.sample.conf: -------------------------------------------------------------------------------- 1 | # In order for this server to work, you need to install some dependencies: 2 | # apt install nginx git fcgiwrap 3 | # 4 | # Also, ensure fcgiwrap is running as a daemon. On Ubuntu, this should happen automatically. 5 | server { 6 | listen 80 default_server; 7 | listen [::]:80 default_server; 8 | 9 | # Replace this with the path to your mirror directory. 10 | root /path/to/my-mirror; 11 | 12 | # Replace this with the domain name you're serving the mirror from. 13 | server_name panamax.internal; 14 | 15 | location / { 16 | autoindex on; 17 | } 18 | 19 | location ~ /crates.io-index(/.*) { 20 | # Replace this path with the path to crates.io-index in your mirror directory. 21 | fastcgi_param GIT_PROJECT_ROOT /path/to/my-mirror/crates.io-index; 22 | include fastcgi_params; 23 | fastcgi_pass unix:/var/run/fcgiwrap.socket; 24 | fastcgi_param SCRIPT_FILENAME /usr/lib/git-core/git-http-backend; 25 | fastcgi_param GIT_HTTP_EXPORT_ALL ""; 26 | fastcgi_param PATH_INFO $1; 27 | } 28 | 29 | # Rewrite the download URLs to match the proper crates location. 30 | rewrite "^/crates/([^/])/([^/]+)$" "/crates/1/$1/$2" last; 31 | rewrite "^/crates/([^/]{2})/([^/]+)$" "/crates/2/$1/$2" last; 32 | rewrite "^/crates/([^/])([^/]{2})/([^/]+)$" "/crates/3/$1/$1$2/$3" last; 33 | rewrite "^/crates/([^/]{2})([^/]{2})([^/]*)/([^/]+)$" "/crates/$1/$2/$1$2$3/$4" last; 34 | } 35 | -------------------------------------------------------------------------------- /.github/workflows/container-test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: ci 3 | 4 | on: 5 | pull_request: 6 | push: 7 | branches: 8 | - master 9 | 10 | env: 11 | CONTAINER_NAME: panamax-rs/panamax 12 | 13 | jobs: 14 | test: 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - name: Checkout 19 | uses: actions/checkout@v2 20 | 21 | - name: Test building image 22 | run: docker build -t ${{ env.CONTAINER_NAME }}:test . 23 | 24 | - name: Run the built image 25 | run: docker run --rm ${{ env.CONTAINER_NAME }}:test 26 | 27 | release: 28 | name: Release 29 | runs-on: ubuntu-latest 30 | needs: test 31 | if: github.ref == 'refs/heads/master' 32 | 33 | steps: 34 | - uses: actions/checkout@v2 35 | - uses: docker/setup-qemu-action@v1 36 | - uses: docker/setup-buildx-action@v1 37 | 38 | - name: Login to DockerHub 39 | uses: docker/login-action@v2 40 | with: 41 | username: ${{ secrets.DOCKERHUB_USERNAME }} 42 | password: ${{ secrets.DOCKERHUB_TOKEN }} 43 | 44 | - name: Build and export to Docker 45 | uses: docker/build-push-action@v3 46 | with: 47 | context: . 48 | load: true 49 | tags: ${{ env.TEST_TAG }} 50 | 51 | - name: Build and push 52 | uses: docker/build-push-action@v3 53 | with: 54 | context: . 55 | platforms: linux/amd64,linux/arm64 56 | push: false 57 | tags: ${{ env.CONTAINER_NAME }}:latest 58 | ... 59 | -------------------------------------------------------------------------------- /src/ARCHITECTURE.md: -------------------------------------------------------------------------------- 1 | # Architecture 2 | 3 | Panamax's main functionality is split up into two main components: Crates, and Rustup. Both of these pieces share the same functionality to download files. 4 | 5 | Additionally when downloading files, a shared progress bar is used. 6 | 7 | Finally, a mirror.toml file is used to configure everything. 8 | 9 | # Starting Points 10 | 11 | The two main commands, `init` and `sync`, are handled in the `init()` and `sync()` commands in mirror.rs. 12 | 13 | ## Main Components 14 | 15 | ### Crates 16 | 17 | The crates component is split up into two files: `crates_index.rs` for handling the crates.io-index git repository, and `crates.rs` for the crates files themselves. 18 | 19 | ### Rustup 20 | 21 | The rustup component is covered in `rustup.rs`. This includes functionality to download the rustup-init files, as well as the libraries and components required for the various Rust versions. 22 | 23 | ## Shared Components 24 | 25 | ### Download 26 | 27 | All details related to downloading (or more specifically, HTTP downloading) is covered in `download.rs`. This includes functionality such as retrying on failed downloads. 28 | 29 | ### Progress Bar 30 | 31 | When a mirror is downloading or updating, a progress bar is displayed. This file includes some common features of all progress bars within Panamax. This is covered in `progress_bar.rs`. 32 | 33 | ### Mirror Configuration 34 | 35 | All details related to configuration file management is handled in `mirror.rs`. Serde is used to parse the `mirror.toml` file, with the root being the `Mirror` struct. -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | description = "Rust dev environment for Panamax"; 3 | 4 | inputs = { 5 | nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; 6 | rust-overlay.url = "github:oxalica/rust-overlay"; 7 | }; 8 | 9 | outputs = { self, nixpkgs, rust-overlay }: 10 | let 11 | overlays = [ 12 | rust-overlay.overlays.default 13 | (final: prev: { 14 | rustToolchain = 15 | let 16 | rust = prev.rust-bin; 17 | in 18 | if builtins.pathExists ./rust-toolchain.toml then 19 | rust.fromRustupToolchainFile ./rust-toolchain.toml 20 | else if builtins.pathExists ./rust-toolchain then 21 | rust.fromRustupToolchainFile ./rust-toolchain 22 | else 23 | rust.stable.latest.default.override { 24 | extensions = [ "rust-src" ]; 25 | targets = [ "x86_64-unknown-linux-gnu" "wasm32-unknown-unknown" ]; 26 | }; 27 | }) 28 | ]; 29 | supportedSystems = [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]; 30 | forEachSupportedSystem = f: nixpkgs.lib.genAttrs supportedSystems (system: f { 31 | pkgs = import nixpkgs { inherit overlays system; }; 32 | }); 33 | in 34 | { 35 | devShells = forEachSupportedSystem ({ pkgs }: { 36 | default = pkgs.mkShell { 37 | packages = with pkgs; [ 38 | rustToolchain 39 | openssl 40 | pkg-config 41 | libgit2 42 | cargo-deny 43 | cargo-edit 44 | cargo-watch 45 | rust-analyzer 46 | ]; 47 | }; 48 | }); 49 | }; 50 | } 51 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | 3 | name: CI 4 | 5 | jobs: 6 | check: 7 | name: Build 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | - uses: actions-rs/toolchain@v1 12 | with: 13 | profile: minimal 14 | toolchain: stable 15 | override: true 16 | - uses: actions/cache@v2 17 | with: 18 | path: | 19 | ~/.cargo/registry 20 | ~/.cargo/git 21 | target 22 | key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} 23 | - uses: actions-rs/cargo@v1 24 | with: 25 | command: build 26 | args: --all 27 | 28 | fmt: 29 | name: Rustfmt 30 | runs-on: ubuntu-latest 31 | steps: 32 | - uses: actions/checkout@v2 33 | - uses: actions-rs/toolchain@v1 34 | with: 35 | profile: minimal 36 | toolchain: stable 37 | override: true 38 | - run: rustup component add rustfmt 39 | - uses: actions-rs/cargo@v1 40 | with: 41 | command: fmt 42 | args: --all -- --check 43 | 44 | clippy: 45 | name: Clippy 46 | runs-on: ubuntu-latest 47 | steps: 48 | - uses: actions/checkout@v2 49 | - uses: actions-rs/toolchain@v1 50 | with: 51 | profile: minimal 52 | toolchain: stable 53 | override: true 54 | - run: rustup component add clippy 55 | - uses: actions/cache@v2 56 | with: 57 | path: | 58 | ~/.cargo/registry 59 | ~/.cargo/git 60 | target 61 | key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} 62 | - uses: actions-rs/cargo@v1 63 | with: 64 | command: clippy 65 | args: -- -D warnings 66 | -------------------------------------------------------------------------------- /templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Panamax 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 |

12 | Panamax is a mirror for
13 | rustup and crates.io. 14 |

15 | 16 |
17 |

First, configure rustup for Panamax:

18 |
19 | 
echo "export RUSTUP_DIST_SERVER={{ host }}" >> ~/.bashrc
20 | echo "export RUSTUP_UPDATE_ROOT={{ host }}/rustup" >> ~/.bashrc
21 | 
22 | source ~/.bashrc
23 |
24 |

Then, configure cargo for Panamax:

25 |
26 | 
mkdir -p ~/.cargo
27 | 
28 | cat <<EOT > ~/.cargo/config
29 | [source.panamax]
30 | registry = "{{ host }}/git/crates.io-index"
31 | [source.panamax-sparse]
32 | registry = "sparse+{{ host }}/index/"
33 | 
34 | [source.crates-io]
35 | # To use sparse index, change "panamax" to "panamax-sparse".
36 | replace-with = "panamax"
37 | EOT
38 |
39 |

Finally, run rustup-init, and you're done!

40 | 43 | 44 |
45 | 
wget {{ host }}/rustup/dist/(replace with selected platform)/rustup-init
46 | chmod +x rustup-init
47 | ./rustup-init
48 |
49 |
50 | 51 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "flake-utils": { 4 | "inputs": { 5 | "systems": "systems" 6 | }, 7 | "locked": { 8 | "lastModified": 1705309234, 9 | "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", 10 | "owner": "numtide", 11 | "repo": "flake-utils", 12 | "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", 13 | "type": "github" 14 | }, 15 | "original": { 16 | "owner": "numtide", 17 | "repo": "flake-utils", 18 | "type": "github" 19 | } 20 | }, 21 | "nixpkgs": { 22 | "locked": { 23 | "lastModified": 1717459389, 24 | "narHash": "sha256-I8/plBsua4/NZ5bKgj+z7/ThiWuud1YFwLsn1QQ5PgE=", 25 | "owner": "NixOS", 26 | "repo": "nixpkgs", 27 | "rev": "3b01abcc24846ae49957b30f4345bab4b3f1d14b", 28 | "type": "github" 29 | }, 30 | "original": { 31 | "owner": "NixOS", 32 | "ref": "nixpkgs-unstable", 33 | "repo": "nixpkgs", 34 | "type": "github" 35 | } 36 | }, 37 | "nixpkgs_2": { 38 | "locked": { 39 | "lastModified": 1706487304, 40 | "narHash": "sha256-LE8lVX28MV2jWJsidW13D2qrHU/RUUONendL2Q/WlJg=", 41 | "owner": "NixOS", 42 | "repo": "nixpkgs", 43 | "rev": "90f456026d284c22b3e3497be980b2e47d0b28ac", 44 | "type": "github" 45 | }, 46 | "original": { 47 | "owner": "NixOS", 48 | "ref": "nixpkgs-unstable", 49 | "repo": "nixpkgs", 50 | "type": "github" 51 | } 52 | }, 53 | "root": { 54 | "inputs": { 55 | "nixpkgs": "nixpkgs", 56 | "rust-overlay": "rust-overlay" 57 | } 58 | }, 59 | "rust-overlay": { 60 | "inputs": { 61 | "flake-utils": "flake-utils", 62 | "nixpkgs": "nixpkgs_2" 63 | }, 64 | "locked": { 65 | "lastModified": 1717553884, 66 | "narHash": "sha256-+t3XaYEvlMo5BUJ/6C6RZcEfBTWFVUdMHpNoqUU+pSE=", 67 | "owner": "oxalica", 68 | "repo": "rust-overlay", 69 | "rev": "8795c817dfab19243a33387a16c98d2df4075bb3", 70 | "type": "github" 71 | }, 72 | "original": { 73 | "owner": "oxalica", 74 | "repo": "rust-overlay", 75 | "type": "github" 76 | } 77 | }, 78 | "systems": { 79 | "locked": { 80 | "lastModified": 1681028828, 81 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 82 | "owner": "nix-systems", 83 | "repo": "default", 84 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 85 | "type": "github" 86 | }, 87 | "original": { 88 | "owner": "nix-systems", 89 | "repo": "default", 90 | "type": "github" 91 | } 92 | } 93 | }, 94 | "root": "root", 95 | "version": 7 96 | } 97 | -------------------------------------------------------------------------------- /src/mirror.default.toml: -------------------------------------------------------------------------------- 1 | # This is a Panamax mirror. It is a self-contained directory made to be easily copied 2 | # to an offline network or machine via rsync, USB, or another method. 3 | 4 | # When offline, Panamax also includes a "serve" command that can be used to serve 5 | # rustup and cargo clients from the mirror. This will also give setup instructions 6 | # on the homepage. 7 | 8 | [mirror] 9 | # Global mirror settings. 10 | 11 | 12 | # Number of download retries before giving up. 13 | retries = 5 14 | 15 | 16 | # Contact information for the user agent. 17 | # This is entirely optional, and is not required for the crates.io CDN. 18 | # You may want to set this if you are mirroring from somewhere else. 19 | # contact = "your@email.com" 20 | 21 | 22 | [rustup] 23 | # These are the configuration parameters for the rustup half of the mirror. 24 | # This will download the rustup-init files, as well as all components needed 25 | # to run Rust on a machine. 26 | 27 | 28 | # Perform rustup synchronization. Set this to false if you only want to mirror crates.io. 29 | sync = true 30 | 31 | 32 | # Whether to mirror XZ archives. These archives are more efficiently compressed 33 | # than the GZ archives, and rustup uses them by default. 34 | download_xz = true 35 | # Whether to mirror GZ archives, for further backwards compatibility with rustup. 36 | download_gz = false 37 | 38 | 39 | # Number of downloads that can be ran in parallel. 40 | download_threads = 16 41 | 42 | 43 | # Where to download rustup files from. 44 | source = "https://static.rust-lang.org" 45 | 46 | 47 | # How many historical versions of Rust to keep. 48 | # Setting these to 1 will keep only the latest version. 49 | # Setting these to 2 or higher will keep the latest version, as well as historical versions. 50 | # Setting these to 0 will stop Panamax from downloading the release entirely. 51 | # Removing the line will keep all release versions. 52 | keep_latest_stables = 1 53 | keep_latest_betas = 1 54 | keep_latest_nightlies = 1 55 | 56 | 57 | # Pinned versions of Rust to download and keep alongside latest stable/beta/nightly 58 | # Version specifiers should be in the rustup toolchain format: 59 | # 60 | # [-][-] 61 | # 62 | # = stable|beta|nightly|| 63 | # = YYYY-MM-DD 64 | # = 65 | # 66 | # e.g. valid versions could be "1.42", "1.42.0", and "nightly-2014-12-18" 67 | # Uncomment the following lines to pin extra rust versions: 68 | 69 | #pinned_rust_versions = [ 70 | # "1.42" 71 | #] 72 | 73 | 74 | # UNIX platforms to include in the mirror 75 | # Uncomment the following lines to limit which platforms get downloaded. 76 | # This affects both rustup-inits and components. 77 | 78 | # platforms_unix = [ 79 | # "arm-unknown-linux-gnueabi", 80 | # "x86_64-unknown-linux-gnu", 81 | # "x86_64-unknown-linux-musl", 82 | # ] 83 | 84 | 85 | # Windows platforms to include in the mirror 86 | # Uncomment the following lines to limit which platforms get downloaded. 87 | # This affects both rustup-inits and components. 88 | 89 | # platforms_windows = [ 90 | # "x86_64-pc-windows-gnu", 91 | # "x86_64-pc-windows-msvc", 92 | # ] 93 | 94 | 95 | # Whether to download the rustc-dev component. 96 | # This component isn't always needed, so setting this to false can save lots of space. 97 | download_dev = false 98 | 99 | 100 | [crates] 101 | # These are the configuration parameters for the crates.io half of the mirror. 102 | # This will download the crates.io-index, as well as the crates themselves. 103 | # Once downloaded, it will then (optionally) rewrite the config.json to point to your mirror. 104 | 105 | 106 | # Perform crates synchronization. Set this to false if you only want to mirror rustup. 107 | sync = true 108 | 109 | 110 | # Number of downloads that can be ran in parallel. 111 | download_threads = 64 112 | 113 | 114 | # Where to download the crates from. 115 | # The default, "https://crates.io/api/v1/crates", will actually instead use the corresponding 116 | # url at https://static.crates.io in order to avoid a redirect and rate limiting 117 | source = "https://crates.io/api/v1/crates" 118 | 119 | 120 | # Where to clone the crates.io-index repository from. 121 | source_index = "https://github.com/rust-lang/crates.io-index" 122 | 123 | 124 | # URL where this mirror's crates directory can be accessed from. 125 | # Used for rewriting crates.io-index's config.json. 126 | # Remove this parameter to perform no rewriting. 127 | # If removed, the `panamax rewrite` command can be used later. 128 | base_url = "http://panamax.internal/crates" 129 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | #![forbid(unsafe_code)] 2 | use clap::Parser; 3 | use std::{net::IpAddr, path::PathBuf}; 4 | 5 | mod crates; 6 | mod crates_index; 7 | mod download; 8 | mod mirror; 9 | mod progress_bar; 10 | mod rustup; 11 | mod serve; 12 | mod verify; 13 | 14 | /// Mirror rustup and crates.io repositories, for offline Rust and cargo usage. 15 | #[derive(Debug, Parser)] 16 | enum Panamax { 17 | /// Create a new mirror directory. 18 | Init { 19 | #[arg(value_parser)] 20 | path: PathBuf, 21 | 22 | /// set [rustup] sync = false 23 | #[arg(long)] 24 | ignore_rustup: bool, 25 | }, 26 | 27 | /// Update an existing mirror directory. 28 | Sync { 29 | /// Mirror directory. 30 | #[arg(value_parser)] 31 | path: PathBuf, 32 | 33 | /// cargo-vendor directory. 34 | #[arg(long)] 35 | vendor_path: Option, 36 | 37 | /// cargo-lock file. 38 | #[arg(long = "cargo-lock")] 39 | cargo_lock_filepath: Option, 40 | 41 | #[arg(long)] 42 | skip_rustup: bool, 43 | }, 44 | 45 | /// Rewrite the config.json within crates.io-index. 46 | /// 47 | /// This can be used if rewriting config.json is 48 | /// required to be an extra step after syncing. 49 | #[command(name = "rewrite")] 50 | Rewrite { 51 | /// Mirror directory. 52 | #[arg(value_parser)] 53 | path: PathBuf, 54 | 55 | /// Base URL used for rewriting. Overrides value in mirror.toml. 56 | #[arg(short, long)] 57 | base_url: Option, 58 | }, 59 | 60 | /// Serve a mirror directory. 61 | #[command(name = "serve")] 62 | Serve { 63 | /// Mirror directory. 64 | #[arg(value_parser)] 65 | path: PathBuf, 66 | 67 | /// IP address to listen on. Defaults to listening on everything. 68 | #[arg(short, long)] 69 | listen: Option, 70 | 71 | /// Port to listen on. 72 | /// Defaults to 8080, or 8443 if TLS certificate provided. 73 | #[arg(short, long)] 74 | port: Option, 75 | 76 | /// Path to a TLS certificate file. This enables TLS. 77 | /// Also requires key_path. 78 | #[arg(long)] 79 | cert_path: Option, 80 | 81 | /// Path to a TLS key file. 82 | /// Also requires cert_path. 83 | #[arg(long)] 84 | key_path: Option, 85 | }, 86 | 87 | /// List platforms currently available. 88 | /// 89 | /// This is useful for finding what can be used for 90 | /// limiting platforms in mirror.toml. 91 | #[command(name = "list-platforms")] 92 | ListPlatforms { 93 | #[arg(long, default_value = "https://static.rust-lang.org")] 94 | source: String, 95 | 96 | #[arg(long, default_value = "nightly")] 97 | channel: String, 98 | }, 99 | 100 | /// Verify coherence between local mirror and local crates.io-index. 101 | /// If any missing crate is found, ask to user before downloading by default. 102 | #[command(name = "verify", alias = "check")] 103 | Verify { 104 | /// Mirror directory. 105 | #[arg(value_parser)] 106 | path: PathBuf, 107 | 108 | /// Dry run, i.e. no change will be made to the mirror. 109 | /// Missing crates are just printed to stdout, not downloaded. 110 | #[arg(long)] 111 | dry_run: bool, 112 | 113 | /// Assume yes from user. 114 | /// Ignored if dry-run is supplied. 115 | #[arg(long)] 116 | assume_yes: bool, 117 | 118 | /// cargo-vendor directory. 119 | #[arg(value_parser)] 120 | vendor_path: Option, 121 | 122 | /// cargo-lock file. 123 | #[arg(long = "cargo-lock")] 124 | cargo_lock_filepath: Option, 125 | }, 126 | } 127 | 128 | #[tokio::main] 129 | async fn main() { 130 | env_logger::init(); 131 | let opt = Panamax::parse(); 132 | match opt { 133 | Panamax::Init { 134 | path, 135 | ignore_rustup, 136 | } => mirror::init(&path, ignore_rustup), 137 | Panamax::Sync { 138 | path, 139 | vendor_path, 140 | cargo_lock_filepath, 141 | skip_rustup, 142 | } => mirror::sync(&path, vendor_path, cargo_lock_filepath, skip_rustup).await, 143 | Panamax::Rewrite { path, base_url } => mirror::rewrite(&path, base_url), 144 | Panamax::Serve { 145 | path, 146 | listen, 147 | port, 148 | cert_path, 149 | key_path, 150 | } => mirror::serve(path, listen, port, cert_path, key_path).await, 151 | Panamax::ListPlatforms { source, channel } => mirror::list_platforms(source, channel).await, 152 | Panamax::Verify { 153 | path, 154 | dry_run, 155 | assume_yes, 156 | vendor_path, 157 | cargo_lock_filepath, 158 | } => mirror::verify(path, dry_run, assume_yes, vendor_path, cargo_lock_filepath).await, 159 | } 160 | .unwrap_or_else(|e| { 161 | eprintln!("Panamax command failed! {e}"); 162 | std::process::exit(1); 163 | }); 164 | } 165 | -------------------------------------------------------------------------------- /static/css/rustup.css: -------------------------------------------------------------------------------- 1 | /*! rustup.css | Apache 2.0 + MIT License | https://github.com/rust-lang/rustup/blob/master/www/rustup.css */ 2 | 3 | @font-face { 4 | font-family: 'Fira Sans'; 5 | font-style: normal; 6 | font-weight: 300; 7 | src: local('Fira Sans Light'), url("fonts/FiraSans-Light.woff") format('woff'); 8 | } 9 | 10 | @font-face { 11 | font-family: 'Fira Sans'; 12 | font-style: normal; 13 | font-weight: 400; 14 | src: local('Fira Sans'), url("fonts/FiraSans-Regular.woff") format('woff'); 15 | } 16 | 17 | @font-face { 18 | font-family: 'Fira Sans'; 19 | font-style: normal; 20 | font-weight: 500; 21 | src: local('Fira Sans Medium'), url("fonts/FiraSans-Medium.woff") format('woff'); 22 | } 23 | 24 | @font-face { 25 | font-family: 'Work Sans'; 26 | font-style: normal; 27 | font-weight: 500; 28 | src: local('Work Sans Medium'), url("fonts/WorkSans-Medium.ttf") format('ttf'); 29 | } 30 | 31 | @font-face { 32 | font-family: 'Inconsolata'; 33 | font-style: normal; 34 | font-weight: 400; 35 | src: local('Inconsolata Regular'), url("fonts/Inconsolata-Regular.ttf") format('ttf'); 36 | } 37 | 38 | body { 39 | margin-top: 2em; 40 | background-color: white; 41 | color: #515151; 42 | font-family: "Fira Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; 43 | font-weight: 300; 44 | font-size: 25px; 45 | } 46 | 47 | pre { 48 | font-family: Inconsolata, Menlo, Monaco, Consolas, "Courier New", monospace; 49 | font-weight: 400; 50 | } 51 | 52 | body#idx #pitch>a { 53 | font-weight: 500; 54 | line-height: 2em; 55 | } 56 | 57 | a { 58 | color: #428bca; 59 | text-decoration: none; 60 | } 61 | 62 | a:hover { 63 | color: rgb(42, 100, 150); 64 | } 65 | 66 | body#idx>* { 67 | margin-left: auto; 68 | margin-right: auto; 69 | text-align: center; 70 | width: 35em; 71 | } 72 | 73 | body#idx>#pitch { 74 | width: 35rem; 75 | } 76 | 77 | #pitch em { 78 | font-style: normal; 79 | font-weight: 400; 80 | } 81 | 82 | body#idx p { 83 | margin-top: 2em; 84 | margin-bottom: 2em; 85 | } 86 | 87 | body#idx p.other-platforms-help { 88 | font-size: 0.6em; 89 | } 90 | 91 | .instructions { 92 | background-color: rgb(250, 250, 250); 93 | margin-left: auto; 94 | margin-right: auto; 95 | text-align: center; 96 | border-radius: 3px; 97 | border: 1px solid rgb(204, 204, 204); 98 | box-shadow: 0px 1px 4px 0px rgb(204, 204, 204); 99 | } 100 | 101 | .instructions>* { 102 | width: 40rem; 103 | margin-left: auto; 104 | margin-right: auto; 105 | } 106 | 107 | hr { 108 | margin-top: 2em; 109 | margin-bottom: 2em; 110 | } 111 | 112 | #platform-instructions-unix>div>pre, 113 | #platform-instructions-win32>div>pre, 114 | #platform-instructions-win64>div>pre, 115 | #platform-instructions-default>div>div>pre, 116 | #platform-instructions-unknown>div>div>pre { 117 | background-color: #515151; 118 | color: white; 119 | margin-left: auto; 120 | margin-right: auto; 121 | padding: 1rem; 122 | width: 45rem; 123 | text-align: center; 124 | border-radius: 3px; 125 | box-shadow: inset 0px 0px 20px 0px #333333; 126 | overflow-x: scroll; 127 | font-size: 0.6em; 128 | height: 27px; 129 | } 130 | 131 | #platform-instructions-unix div.copy-container, 132 | #platform-instructions-win32 div.copy-container, 133 | #platform-instructions-win64 div.copy-container, 134 | #platform-instructions-default div.copy-container, 135 | #platform-instructions-unknown div.copy-container { 136 | display: flex; 137 | align-items: center; 138 | } 139 | 140 | #platform-instructions-unix button.copy-button, 141 | #platform-instructions-win32 button.copy-button, 142 | #platform-instructions-win64 button.copy-button, 143 | #platform-instructions-default button.copy-button, 144 | #platform-instructions-unknown button.copy-button { 145 | height: 60px; 146 | margin: 1px; 147 | padding-right: 5px; 148 | border-radius: 3px; 149 | } 150 | 151 | #platform-instructions-unix div.copy-icon, 152 | #platform-instructions-win32 div.copy-icon, 153 | #platform-instructions-win64 div.copy-icon, 154 | #platform-instructions-default div.copy-icon, 155 | #platform-instructions-unknown div.copy-icon { 156 | margin-top: 12px; 157 | } 158 | 159 | #platform-instructions-unix div.copy-button-text, 160 | #platform-instructions-win32 div.copy-button-text, 161 | #platform-instructions-win64 div.copy-button-text, 162 | #platform-instructions-default div.copy-button-text, 163 | #platform-instructions-unknown div.copy-button-text { 164 | font-size: 10px; 165 | color: green; 166 | width: 41px; 167 | height: 15px; 168 | } 169 | 170 | #platform-instructions-win32 a.windows-download, 171 | #platform-instructions-win64 a.windows-download, 172 | #platform-instructions-default a.windows-download, 173 | #platform-instructions-unknown a.windows-download { 174 | display: block; 175 | padding-top: 0.4rem; 176 | padding-bottom: 0.6rem; 177 | font-family: "Work Sans", "Fira Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; 178 | font-weight: 500; 179 | letter-spacing: 0.1rem; 180 | } 181 | 182 | 183 | /* This is the box that prints navigator.platform, navigator.appVersion values */ 184 | 185 | #platform-instructions-unknown>div { 186 | font-size: 16px; 187 | line-height: 2rem; 188 | } 189 | 190 | #about { 191 | font-size: 16px; 192 | line-height: 2em; 193 | } 194 | 195 | #about>img { 196 | width: 30px; 197 | height: 30px; 198 | transform: translateY(11px); 199 | } 200 | 201 | #platform-button { 202 | background-color: #515151; 203 | color: white; 204 | margin-left: auto; 205 | margin-right: auto; 206 | padding: 1em; 207 | } 208 | 209 | .display-none { 210 | display: none; 211 | } 212 | 213 | .display-block { 214 | display: block; 215 | } 216 | 217 | .display-inline { 218 | display: inline; 219 | } -------------------------------------------------------------------------------- /src/crates_index.rs: -------------------------------------------------------------------------------- 1 | use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; 2 | use serde::Serialize; 3 | use std::{io, num::TryFromIntError, path::Path, time::Duration}; 4 | 5 | use git2::{ 6 | build::{CheckoutBuilder, RepoBuilder}, 7 | FetchOptions, RemoteCallbacks, Repository, Signature, 8 | }; 9 | use thiserror::Error; 10 | 11 | use crate::mirror::ConfigCrates; 12 | use crate::progress_bar::padded_prefix_message; 13 | 14 | #[derive(Error, Debug)] 15 | pub enum IndexSyncError { 16 | #[error("IO error: {0}")] 17 | Io(#[from] io::Error), 18 | 19 | #[error("JSON serialization error: {0}")] 20 | SerializeError(#[from] serde_json::Error), 21 | 22 | #[error("Git error: {0}")] 23 | GitError(#[from] git2::Error), 24 | 25 | #[error("Number conversion error: {0}")] 26 | IntegerConversionError(#[from] TryFromIntError), 27 | } 28 | 29 | #[derive(Debug, Serialize)] 30 | struct ConfigJson { 31 | dl: String, 32 | api: String, 33 | } 34 | 35 | /// Synchronize the crates.io-index repository. 36 | /// 37 | /// `mirror_path`: Root path to the mirror directory. 38 | /// 39 | /// `crates`: The crates section of the `mirror.toml` config file. 40 | pub fn sync_crates_repo(mirror_path: &Path, crates: &ConfigCrates) -> Result<(), IndexSyncError> { 41 | let repo_path = mirror_path.join("crates.io-index"); 42 | 43 | let prefix = padded_prefix_message(1, 3, "Fetching crates.io-index"); 44 | let pb = ProgressBar::new_spinner() 45 | .with_style( 46 | ProgressStyle::default_bar() 47 | .template("{prefix} {wide_bar} {spinner} [{elapsed_precise}]") 48 | .expect("template is correct") 49 | .progress_chars(" "), 50 | ) 51 | .with_finish(ProgressFinish::AndLeave) 52 | .with_prefix(prefix); 53 | // Enable the steady tick, so the transfer progress callback isn't spending its time 54 | // updating the progress bar. 55 | pb.enable_steady_tick(Duration::from_millis(10)); 56 | 57 | // Libgit2 has callbacks that allow us to update the progress bar 58 | // as the git download progresses. 59 | // FIXME: Enabling progress updates causes checkout times to balloon. 60 | let remote_callbacks = RemoteCallbacks::new(); 61 | /* 62 | remote_callbacks.transfer_progress(|p| { 63 | if p.received_objects() == p.total_objects() { 64 | pb.set_length(p.total_deltas() as u64); 65 | pb.set_position(p.indexed_deltas() as u64); 66 | } else { 67 | pb.set_length(p.total_objects() as u64); 68 | pb.set_position(p.indexed_objects() as u64); 69 | } 70 | 71 | true 72 | }); 73 | */ 74 | 75 | let mut proxy_opts = git2::ProxyOptions::new(); 76 | proxy_opts.auto(); 77 | 78 | let mut fetch_opts = FetchOptions::new(); 79 | fetch_opts.remote_callbacks(remote_callbacks); 80 | fetch_opts.proxy_options(proxy_opts); 81 | 82 | if !repo_path.join(".git").exists() { 83 | clone_repository(fetch_opts, &crates.source_index, &repo_path)?; 84 | // Remove master in order to ensure full scan is performed 85 | let repo = Repository::open(&repo_path)?; 86 | repo.find_reference("refs/heads/master")?.delete()?; 87 | } else { 88 | // Get (fetch) the branch's latest remote "master" commit 89 | let repo = Repository::open(&repo_path)?; 90 | let mut remote = repo.find_remote("origin")?; 91 | remote.fetch(&["master"], Some(&mut fetch_opts), None)?; 92 | } 93 | 94 | Ok(()) 95 | } 96 | 97 | /// Update the config.json file within crates-io.index. 98 | pub fn update_crates_config( 99 | mirror_path: &Path, 100 | crates: &ConfigCrates, 101 | ) -> Result<(), IndexSyncError> { 102 | let repo_path = mirror_path.join("crates.io-index"); 103 | 104 | if let Some(base_url) = &crates.base_url { 105 | rewrite_config_json(&repo_path, base_url)?; 106 | } 107 | 108 | Ok(()) 109 | } 110 | 111 | /// Perform a git fast-forward on the repository. This will destroy any local changes that have 112 | /// been made to the repo, and will make the local master identical to the remote master. 113 | pub fn fast_forward(repo_path: &Path) -> Result<(), IndexSyncError> { 114 | let repo = Repository::open(repo_path)?; 115 | 116 | let fetch_head = repo.find_reference("refs/remotes/origin/master")?; 117 | let fetch_commit = repo.reference_to_annotated_commit(&fetch_head)?; 118 | 119 | // Force fast-forward on master 120 | let refname = "refs/heads/master"; 121 | match repo.find_reference(refname) { 122 | Ok(mut r) => { 123 | r.set_target(fetch_commit.id(), "Performing fast-forward")?; 124 | } 125 | Err(_) => { 126 | // Remote branch doesn't exist, so use commit directly 127 | repo.reference(refname, fetch_commit.id(), true, "Performing fast-forward")?; 128 | } 129 | } 130 | 131 | // Set the "HEAD" reference to our new master commit. 132 | repo.set_head(refname)?; 133 | 134 | // Checkout the repo directory (so the files are actually created on disk). 135 | repo.checkout_head(Some( 136 | CheckoutBuilder::default().allow_conflicts(true).force(), 137 | ))?; 138 | 139 | Ok(()) 140 | } 141 | 142 | /// Clone a repository from scratch. This assumes the path does not exist. 143 | fn clone_repository( 144 | fetch_opts: FetchOptions, 145 | source_index: &str, 146 | repo_path: &Path, 147 | ) -> Result<(), IndexSyncError> { 148 | let mut repo_builder = RepoBuilder::new(); 149 | repo_builder.fetch_options(fetch_opts); 150 | repo_builder.clone(source_index, repo_path)?; 151 | Ok(()) 152 | } 153 | 154 | /// Fast-forward master, then rewrite the crates.io-index config.json. 155 | pub fn rewrite_config_json(repo_path: &Path, base_url: &str) -> Result<(), IndexSyncError> { 156 | let repo = Repository::open(repo_path)?; 157 | let refname = "refs/heads/master"; 158 | let signature = Signature::now("Panamax", "panamax@panamax")?; 159 | 160 | eprintln!("{}", padded_prefix_message(3, 3, "Syncing config")); 161 | 162 | let mut index = repo.index()?; 163 | 164 | let crate_path = format!( 165 | "{}/{}", 166 | base_url, "{prefix}/{crate}/{version}/{crate}-{version}.crate" 167 | ); 168 | 169 | // Create the new config.json. 170 | let config_json = ConfigJson { 171 | dl: crate_path, 172 | api: base_url.to_string(), 173 | }; 174 | let contents = serde_json::to_vec_pretty(&config_json)?; 175 | std::fs::write(repo_path.join("config.json"), contents)?; 176 | 177 | // Add config.json into the working index. 178 | // (a.k.a. "git add") 179 | index.add_path(Path::new("config.json"))?; 180 | let oid = index.write_tree()?; 181 | index.write()?; 182 | 183 | // Get the master commit's tree. 184 | let master = repo.find_reference(refname)?; 185 | let parent_commit = master.peel_to_commit()?; 186 | let tree = repo.find_tree(oid)?; 187 | 188 | // Commit this change to the repository. 189 | repo.commit( 190 | Some(refname), 191 | &signature, 192 | &signature, 193 | "Rewrite config.json", 194 | &tree, 195 | &[&parent_commit], 196 | )?; 197 | 198 | Ok(()) 199 | } 200 | -------------------------------------------------------------------------------- /src/download.rs: -------------------------------------------------------------------------------- 1 | use reqwest::header::{HeaderValue, USER_AGENT}; 2 | use reqwest::Client; 3 | use sha2::{Digest, Sha256}; 4 | use std::fs::File; 5 | use std::io::Write; 6 | use std::path::{Path, PathBuf}; 7 | use std::{fs, io}; 8 | use thiserror::Error; 9 | use tokio::io::AsyncReadExt; 10 | 11 | #[derive(Error, Debug)] 12 | pub enum DownloadError { 13 | #[error("IO error: {0}")] 14 | Io(#[from] io::Error), 15 | #[error("HTTP download error: {0}")] 16 | Download(#[from] reqwest::Error), 17 | #[error("Got bad crate: {0}")] 18 | BadCrate(String), 19 | #[error("Mismatched hash - expected '{expected}', got '{actual}'")] 20 | MismatchedHash { expected: String, actual: String }, 21 | #[error("HTTP not found. Status: {status}, URL: {url}, data: {data}")] 22 | NotFound { 23 | status: u16, 24 | url: String, 25 | data: String, 26 | }, 27 | } 28 | 29 | /// Download a URL and return it as a string. 30 | pub async fn download_string( 31 | from: &str, 32 | user_agent: &HeaderValue, 33 | ) -> Result { 34 | let client = Client::new(); 35 | 36 | Ok(client 37 | .get(from) 38 | .header(USER_AGENT, user_agent) 39 | .send() 40 | .await? 41 | .text() 42 | .await?) 43 | } 44 | 45 | /// Append a string to a path. 46 | pub fn append_to_path(path: &Path, suffix: &str) -> PathBuf { 47 | let mut new_path = path.as_os_str().to_os_string(); 48 | new_path.push(suffix); 49 | PathBuf::from(new_path) 50 | } 51 | 52 | /// Write a string to a file, creating directories if needed. 53 | pub fn write_file_create_dir(path: &Path, contents: &str) -> Result<(), DownloadError> { 54 | let mut res = fs::write(path, contents); 55 | 56 | if let Err(e) = &res { 57 | if e.kind() == io::ErrorKind::NotFound { 58 | if let Some(parent) = path.parent() { 59 | fs::create_dir_all(parent)?; 60 | } 61 | res = fs::write(path, contents); 62 | } 63 | } 64 | 65 | Ok(res?) 66 | } 67 | 68 | /// Create a file, creating directories if needed. 69 | pub fn create_file_create_dir(path: &Path) -> Result { 70 | let mut file_res = File::create(path); 71 | if let Err(e) = &file_res { 72 | if e.kind() == io::ErrorKind::NotFound { 73 | if let Some(parent) = path.parent() { 74 | fs::create_dir_all(parent)?; 75 | } 76 | file_res = File::create(path); 77 | } 78 | } 79 | 80 | Ok(file_res?) 81 | } 82 | 83 | pub fn move_if_exists(from: &Path, to: &Path) -> Result<(), DownloadError> { 84 | if from.exists() { 85 | fs::rename(from, to)?; 86 | } 87 | Ok(()) 88 | } 89 | 90 | pub fn move_if_exists_with_sha256(from: &Path, to: &Path) -> Result<(), DownloadError> { 91 | let sha256_from_path = append_to_path(from, ".sha256"); 92 | let sha256_to_path = append_to_path(to, ".sha256"); 93 | move_if_exists(&sha256_from_path, &sha256_to_path)?; 94 | move_if_exists(from, to)?; 95 | Ok(()) 96 | } 97 | 98 | /// Copy a file and its .sha256, creating `to`'s directory if it doesn't exist. 99 | /// Fails if the source .sha256 does not exist. 100 | pub fn copy_file_create_dir_with_sha256(from: &Path, to: &Path) -> Result<(), DownloadError> { 101 | let sha256_from_path = append_to_path(from, ".sha256"); 102 | let sha256_to_path = append_to_path(to, ".sha256"); 103 | copy_file_create_dir(&sha256_from_path, &sha256_to_path)?; 104 | copy_file_create_dir(from, to)?; 105 | Ok(()) 106 | } 107 | 108 | /// Copy a file, creating `to`'s directory if it doesn't exist. 109 | pub fn copy_file_create_dir(from: &Path, to: &Path) -> Result<(), DownloadError> { 110 | if to.exists() { 111 | return Ok(()); 112 | } 113 | if let Some(parent) = to.parent() { 114 | if !parent.exists() { 115 | fs::create_dir_all(parent)?; 116 | } 117 | } 118 | 119 | fs::copy(from, to)?; 120 | Ok(()) 121 | } 122 | 123 | async fn one_download( 124 | client: &Client, 125 | url: &str, 126 | path: &Path, 127 | hash: Option<&str>, 128 | user_agent: &HeaderValue, 129 | ) -> Result<(), DownloadError> { 130 | let mut http_res = client 131 | .get(url) 132 | .header(USER_AGENT, user_agent) 133 | .send() 134 | .await?; 135 | let part_path = append_to_path(path, ".part"); 136 | let mut sha256 = Sha256::new(); 137 | { 138 | let mut f = create_file_create_dir(&part_path)?; 139 | let status = http_res.status(); 140 | if status == 403 || status == 404 { 141 | let forbidden_path = append_to_path(path, ".notfound"); 142 | let text = http_res.text().await?; 143 | fs::write( 144 | forbidden_path, 145 | format!("Server returned {}: {}", status, &text), 146 | )?; 147 | return Err(DownloadError::NotFound { 148 | status: status.as_u16(), 149 | url: url.to_string(), 150 | data: text, 151 | }); 152 | } 153 | 154 | while let Some(chunk) = http_res.chunk().await? { 155 | if hash.is_some() { 156 | sha256.update(&chunk); 157 | } 158 | f.write_all(&chunk)?; 159 | } 160 | } 161 | 162 | let f_hash = format!("{:x}", sha256.finalize()); 163 | 164 | if let Some(h) = hash { 165 | if f_hash == h { 166 | move_if_exists(&part_path, path)?; 167 | Ok(()) 168 | } else { 169 | let badsha_path = append_to_path(path, ".badsha256"); 170 | fs::write(badsha_path, &f_hash)?; 171 | Err(DownloadError::MismatchedHash { 172 | expected: h.to_string(), 173 | actual: f_hash, 174 | }) 175 | } 176 | } else { 177 | fs::rename(part_path, path)?; 178 | Ok(()) 179 | } 180 | } 181 | 182 | /// Download file, verifying its hash, and retrying if needed 183 | pub async fn download( 184 | client: &Client, 185 | url: &str, 186 | path: &Path, 187 | hash: Option<&str>, 188 | retries: usize, 189 | force_download: bool, 190 | user_agent: &HeaderValue, 191 | ) -> Result<(), DownloadError> { 192 | if path.exists() && !force_download { 193 | if let Some(h) = hash { 194 | // Verify SHA-256 hash on the filesystem. 195 | let mut file = tokio::fs::File::open(path).await?; 196 | let mut buf = [0u8; 4096]; 197 | let mut sha256 = Sha256::new(); 198 | 199 | loop { 200 | let n = file.read(&mut buf).await?; 201 | if n == 0 { 202 | break; 203 | } 204 | 205 | sha256.update(&buf[..n]); 206 | } 207 | 208 | let f_hash = format!("{:x}", sha256.finalize()); 209 | if h == f_hash { 210 | // Calculated hash matches specified hash. 211 | return Ok(()); 212 | } 213 | } else { 214 | return Ok(()); 215 | } 216 | } 217 | 218 | let mut res = Ok(()); 219 | for _ in 0..=retries { 220 | res = match one_download(client, url, path, hash, user_agent).await { 221 | Ok(_) => break, 222 | Err(e) => Err(e), 223 | } 224 | } 225 | 226 | res 227 | } 228 | 229 | /// Download file and associated .sha256 file, verifying the hash, and retrying if needed 230 | pub async fn download_with_sha256_file( 231 | client: &Client, 232 | url: &str, 233 | path: &Path, 234 | retries: usize, 235 | force_download: bool, 236 | user_agent: &HeaderValue, 237 | ) -> Result<(), DownloadError> { 238 | let sha256_url = format!("{url}.sha256"); 239 | let sha256_data = download_string(&sha256_url, user_agent).await?; 240 | 241 | let sha256_hash = &sha256_data[..64]; 242 | download( 243 | client, 244 | url, 245 | path, 246 | Some(sha256_hash), 247 | retries, 248 | force_download, 249 | user_agent, 250 | ) 251 | .await?; 252 | 253 | let sha256_path = append_to_path(path, ".sha256"); 254 | write_file_create_dir(&sha256_path, &sha256_data)?; 255 | 256 | Ok(()) 257 | } 258 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Panamax 2 | 3 | [![crates.io](https://img.shields.io/crates/v/panamax.svg)](https://crates.io/crates/panamax) 4 | ![Docker Image Version (latest semver)](https://img.shields.io/docker/v/panamaxrs/panamax?label=docker&sort=semver) 5 | 6 | ![image](https://user-images.githubusercontent.com/1247908/132435079-b703bd5d-c139-4a73-818f-51746353b3ea.png) 7 | 8 | Panamax is a tool to mirror the Rust and crates.io repositories, for offline usage of `rustup` and `cargo`. 9 | 10 | ## Installation 11 | 12 | Panamax is itself available on crates.io, and can be installed via: 13 | 14 | ``` 15 | $ cargo install --locked panamax 16 | ``` 17 | 18 | Alternatively, you can clone this repository and `cargo build` or `cargo run` within it. 19 | 20 | ## Usage 21 | 22 | ## Docker 23 | 24 | Panamax is available as a docker image, so you can run: 25 | 26 | ``` 27 | $ docker run --rm -it -v /path/to/mirror/:/mirror --user $(id -u) panamaxrs/panamax init /mirror 28 | (Modify /path/to/mirror/mirror.toml as needed) 29 | $ docker run --rm -it -v /path/to/mirror/:/mirror --user $(id -u) panamaxrs/panamax sync /mirror 30 | (Once synced, serve the mirror) 31 | $ docker run --rm -it -v /path/to/mirror/:/mirror --user $(id -u) -p8080:8080 panamaxrs/panamax serve /mirror 32 | ``` 33 | 34 | Alternatively, you can run panamax in a bare-metal environment like below. 35 | 36 | ### Init 37 | 38 | In Panamax, mirrors consist of self-contained directories. To create a mirror directory `my-mirror`: 39 | 40 | ``` 41 | $ panamax init my-mirror 42 | Successfully created mirror base at `my-mirror`. 43 | Make any desired changes to my-mirror/mirror.toml, then run panamax sync my-mirror. 44 | ``` 45 | 46 | There will now be a `my-mirror` directory in your current directory. 47 | 48 | ### Modify mirror.toml 49 | 50 | Within the directory, you'll find a `mirror.toml` file. This file contains the full configuration of the mirror, and while it has sane defaults, you should ensure the values are set to what you want. 51 | 52 | The other important parameter to set is the `base_url` within the `[crates]` section. After `cargo` fetches the index, it will try to use this URL to actually download the crates. It's important this value is accurate, or `cargo` may not work with the mirror. 53 | 54 | You can modify `mirror.toml` at any point in time, even after the mirror is synchronized. 55 | 56 | ### Sync 57 | 58 | Once you have made the changes to `mirror.toml`, it is time to synchronize your mirror! 59 | 60 | ``` 61 | $ panamax sync my-mirror 62 | Syncing Rustup repositories... 63 | [1/5] Syncing rustup-init files... ██████████████████████████████████████████████████████████████ 27/27 [00:00:06] 64 | [2/5] Syncing latest stable... ████████████████████████████████████████████████████████████ 602/602 [00:09:02] 65 | [3/5] Syncing latest beta... ████████████████████████████████████████████████████████████ 524/524 [00:07:29] 66 | [4/5] Syncing latest nightly... ████████████████████████████████████████████████████████████ 546/546 [00:08:56] 67 | [5/5] Cleaning old files... ████████████████████████████████████████████████████████████ 546/546 [00:00:00] 68 | Syncing Rustup repositories complete! 69 | Syncing Crates repositories... 70 | [1/3] Fetching crates.io-index... ██████████████████████████████████████████████████████████ 1615/1615 [00:00:02] 71 | [2/3] Syncing crates files... ██████████████████████████████████████████████████████████ 6357/6357 [00:00:05] 72 | [3/3] Syncing index and config... 73 | Syncing Crates repositories complete! 74 | Sync complete. 75 | ``` 76 | 77 | Once this is step completes (without download errors), you will now have a full, synchronized copy of all the files needed to use `rustup` and `cargo` to their full potential! 78 | 79 | This directory can now be copied to a USB or rsync'd somewhere else, or even used in place - perfect for long plane trips! 80 | 81 | Additionally, this mirror can continually by synchronized in the future - one recommendation is to run this command in a cronjob once each night, to keep the mirror reasonably up to date. 82 | 83 | ### Sync Select Dependencies 84 | Optionally, panamax can be told to only grab crates needed to build a singular project. 85 | `cargo vendor` is used to create a folder with all needed dependencies, 86 | then a panamax command can parse the created directory and only grab those crates and versions. 87 | ``` 88 | # Only grab crates needed for panamax, as an example 89 | $ cargo vendor 90 | $ panamax sync my-mirror vendor 91 | ``` 92 | 93 | ## Server 94 | 95 | Panamax provides a warp-based HTTP(S) server that can handle serving a Rust mirror fast and at scale. This is the recommended way to serve the mirror. 96 | 97 | ``` 98 | $ panamax serve my-mirror 99 | Running HTTP on [::]:8080 100 | ``` 101 | 102 | The server's index page provides all the instructions needed on how to set up a Rust client that uses this mirror. 103 | 104 | If you would prefer having these instructions elsewhere, the rest of this README will describe the setup process in more detail. 105 | 106 | Additionally, if you would prefer hosting a server with nginx, there is a sample nginx configuration in the repository, at `nginx.sample.conf`. 107 | 108 | ## Configuring `rustup` and `cargo` 109 | 110 | Once you have a mirror server set up and running, it's time to tell your Rust components to use it. 111 | 112 | ### Setting environment variables 113 | 114 | In order to ensure `rustup` knows where to look for the Rust components, we need to set some environment variables. Assuming the mirror is hosted at http://panamax.internal/: 115 | 116 | ``` 117 | export RUSTUP_DIST_SERVER=http://panamax.internal 118 | export RUSTUP_UPDATE_ROOT=http://panamax.internal/rustup 119 | ``` 120 | 121 | These need to be set whenever `rustup` is used, so these should be added to your `.bashrc` file (or equivalent). 122 | 123 | ### Installing `rustup` 124 | 125 | If you already have `rustup` installed, this step isn't necessary, however if you don't have access to https://rustup.rs, the mirror also contains the `rustup-init` files needed to install `rustup`. 126 | 127 | Assuming the mirror is hosted at http://panamax.internal/, you will find the `rustup-init` files at http://panamax.internal/rustup/dist/. The `rustup-init` file you want depends on your architecture. Assuming you're running desktop Linux on a 64-bit machine: 128 | 129 | ``` 130 | wget http://panamax.internal/rustup/dist/x86_64-unknown-linux-gnu/rustup-init 131 | chmod +x rustup-init 132 | ./rustup-init 133 | ``` 134 | 135 | This will let you install `rustup` the similarly following the steps from https://rustup.rs. This will also let you use `rustup` to keep your Rust installation updated in the future. 136 | 137 | ### Configuring `cargo` 138 | 139 | `Cargo` also needs to be configured to point to the mirror. This can be done by adding the following lines to `~/.cargo/config` (creating the file if it doesn't exist): 140 | 141 | ``` 142 | [source.my-mirror] 143 | registry = "http://panamax.internal/crates.io-index" 144 | [source.crates-io] 145 | replace-with = "my-mirror" 146 | ``` 147 | 148 | `Cargo` should now be pointing to the correct location to use the mirror. 149 | 150 | ### Testing configuration 151 | 152 | You've now set up a Rust mirror! In order to make sure everything is set up properly, you can run a simple test: 153 | 154 | ``` 155 | $ cargo install ripgrep 156 | ``` 157 | 158 | This will install the grep-like `rg` tool (which is a great tool - props to burntsushi!). If `cargo` successfully downloads and builds everything, you have yourself a working mirror. Congratulations! 159 | 160 | 161 | ### Proxies 162 | 163 | If you need to run Panamax through a proxy, you will need to set your configuration options in two places. 164 | 165 | First, you'll need to set the environment variable `http_proxy` to something like `https://your.proxy:1234` (which can be http or https). 166 | 167 | Second, you'll need to set an http proxy in your `~/.gitconfig`, like so: 168 | 169 | ``` 170 | [http] 171 | proxy = https://your.proxy:1234 172 | ``` 173 | 174 | With these two parameters set, Panamax should work through an HTTP proxy. 175 | 176 | ## License 177 | 178 | Licensed under the terms of the MIT license and the Apache License (Version 2.0) 179 | 180 | See [LICENSE-MIT](LICENSE-MIT) and [LICENSE-APACHE](LICENSE-APACHE) for details. 181 | 182 | ### Contribution 183 | 184 | Unless you explicitly state otherwise, any contribution intentionally submitted 185 | for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any 186 | additional terms or conditions. 187 | -------------------------------------------------------------------------------- /static/css/normalize.css: -------------------------------------------------------------------------------- 1 | /*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css */ 2 | 3 | 4 | /* Document 5 | ========================================================================== */ 6 | 7 | 8 | /** 9 | * 1. Correct the line height in all browsers. 10 | * 2. Prevent adjustments of font size after orientation changes in iOS. 11 | */ 12 | 13 | html { 14 | line-height: 1.15; 15 | /* 1 */ 16 | -webkit-text-size-adjust: 100%; 17 | /* 2 */ 18 | } 19 | 20 | 21 | /* Sections 22 | ========================================================================== */ 23 | 24 | 25 | /** 26 | * Remove the margin in all browsers. 27 | */ 28 | 29 | body { 30 | margin: 0; 31 | } 32 | 33 | 34 | /** 35 | * Render the `main` element consistently in IE. 36 | */ 37 | 38 | main { 39 | display: block; 40 | } 41 | 42 | 43 | /** 44 | * Correct the font size and margin on `h1` elements within `section` and 45 | * `article` contexts in Chrome, Firefox, and Safari. 46 | */ 47 | 48 | h1 { 49 | font-size: 2em; 50 | margin: 0.67em 0; 51 | } 52 | 53 | 54 | /* Grouping content 55 | ========================================================================== */ 56 | 57 | 58 | /** 59 | * 1. Add the correct box sizing in Firefox. 60 | * 2. Show the overflow in Edge and IE. 61 | */ 62 | 63 | hr { 64 | box-sizing: content-box; 65 | /* 1 */ 66 | height: 0; 67 | /* 1 */ 68 | overflow: visible; 69 | /* 2 */ 70 | } 71 | 72 | 73 | /** 74 | * 1. Correct the inheritance and scaling of font size in all browsers. 75 | * 2. Correct the odd `em` font sizing in all browsers. 76 | */ 77 | 78 | pre { 79 | font-family: monospace, monospace; 80 | /* 1 */ 81 | font-size: 1em; 82 | /* 2 */ 83 | } 84 | 85 | 86 | /* Text-level semantics 87 | ========================================================================== */ 88 | 89 | 90 | /** 91 | * Remove the gray background on active links in IE 10. 92 | */ 93 | 94 | a { 95 | background-color: transparent; 96 | } 97 | 98 | 99 | /** 100 | * 1. Remove the bottom border in Chrome 57- 101 | * 2. Add the correct text decoration in Chrome, Edge, IE, Opera, and Safari. 102 | */ 103 | 104 | abbr[title] { 105 | border-bottom: none; 106 | /* 1 */ 107 | text-decoration: underline; 108 | /* 2 */ 109 | text-decoration: underline dotted; 110 | /* 2 */ 111 | } 112 | 113 | 114 | /** 115 | * Add the correct font weight in Chrome, Edge, and Safari. 116 | */ 117 | 118 | b, 119 | strong { 120 | font-weight: bolder; 121 | } 122 | 123 | 124 | /** 125 | * 1. Correct the inheritance and scaling of font size in all browsers. 126 | * 2. Correct the odd `em` font sizing in all browsers. 127 | */ 128 | 129 | code, 130 | kbd, 131 | samp { 132 | font-family: monospace, monospace; 133 | /* 1 */ 134 | font-size: 1em; 135 | /* 2 */ 136 | } 137 | 138 | 139 | /** 140 | * Add the correct font size in all browsers. 141 | */ 142 | 143 | small { 144 | font-size: 80%; 145 | } 146 | 147 | 148 | /** 149 | * Prevent `sub` and `sup` elements from affecting the line height in 150 | * all browsers. 151 | */ 152 | 153 | sub, 154 | sup { 155 | font-size: 75%; 156 | line-height: 0; 157 | position: relative; 158 | vertical-align: baseline; 159 | } 160 | 161 | sub { 162 | bottom: -0.25em; 163 | } 164 | 165 | sup { 166 | top: -0.5em; 167 | } 168 | 169 | 170 | /* Embedded content 171 | ========================================================================== */ 172 | 173 | 174 | /** 175 | * Remove the border on images inside links in IE 10. 176 | */ 177 | 178 | img { 179 | border-style: none; 180 | } 181 | 182 | 183 | /* Forms 184 | ========================================================================== */ 185 | 186 | 187 | /** 188 | * 1. Change the font styles in all browsers. 189 | * 2. Remove the margin in Firefox and Safari. 190 | */ 191 | 192 | button, 193 | input, 194 | optgroup, 195 | select, 196 | textarea { 197 | font-family: inherit; 198 | /* 1 */ 199 | font-size: 100%; 200 | /* 1 */ 201 | line-height: 1.15; 202 | /* 1 */ 203 | margin: 0; 204 | /* 2 */ 205 | } 206 | 207 | 208 | /** 209 | * Show the overflow in IE. 210 | * 1. Show the overflow in Edge. 211 | */ 212 | 213 | button, 214 | input { 215 | /* 1 */ 216 | overflow: visible; 217 | } 218 | 219 | 220 | /** 221 | * Remove the inheritance of text transform in Edge, Firefox, and IE. 222 | * 1. Remove the inheritance of text transform in Firefox. 223 | */ 224 | 225 | button, 226 | select { 227 | /* 1 */ 228 | text-transform: none; 229 | } 230 | 231 | 232 | /** 233 | * Correct the inability to style clickable types in iOS and Safari. 234 | */ 235 | 236 | button, 237 | [type="button"], 238 | [type="reset"], 239 | [type="submit"] { 240 | -webkit-appearance: button; 241 | } 242 | 243 | 244 | /** 245 | * Remove the inner border and padding in Firefox. 246 | */ 247 | 248 | button::-moz-focus-inner, 249 | [type="button"]::-moz-focus-inner, 250 | [type="reset"]::-moz-focus-inner, 251 | [type="submit"]::-moz-focus-inner { 252 | border-style: none; 253 | padding: 0; 254 | } 255 | 256 | 257 | /** 258 | * Restore the focus styles unset by the previous rule. 259 | */ 260 | 261 | button:-moz-focusring, 262 | [type="button"]:-moz-focusring, 263 | [type="reset"]:-moz-focusring, 264 | [type="submit"]:-moz-focusring { 265 | outline: 1px dotted ButtonText; 266 | } 267 | 268 | 269 | /** 270 | * Correct the padding in Firefox. 271 | */ 272 | 273 | fieldset { 274 | padding: 0.35em 0.75em 0.625em; 275 | } 276 | 277 | 278 | /** 279 | * 1. Correct the text wrapping in Edge and IE. 280 | * 2. Correct the color inheritance from `fieldset` elements in IE. 281 | * 3. Remove the padding so developers are not caught out when they zero out 282 | * `fieldset` elements in all browsers. 283 | */ 284 | 285 | legend { 286 | box-sizing: border-box; 287 | /* 1 */ 288 | color: inherit; 289 | /* 2 */ 290 | display: table; 291 | /* 1 */ 292 | max-width: 100%; 293 | /* 1 */ 294 | padding: 0; 295 | /* 3 */ 296 | white-space: normal; 297 | /* 1 */ 298 | } 299 | 300 | 301 | /** 302 | * Add the correct vertical alignment in Chrome, Firefox, and Opera. 303 | */ 304 | 305 | progress { 306 | vertical-align: baseline; 307 | } 308 | 309 | 310 | /** 311 | * Remove the default vertical scrollbar in IE 10+. 312 | */ 313 | 314 | textarea { 315 | overflow: auto; 316 | } 317 | 318 | 319 | /** 320 | * 1. Add the correct box sizing in IE 10. 321 | * 2. Remove the padding in IE 10. 322 | */ 323 | 324 | [type="checkbox"], 325 | [type="radio"] { 326 | box-sizing: border-box; 327 | /* 1 */ 328 | padding: 0; 329 | /* 2 */ 330 | } 331 | 332 | 333 | /** 334 | * Correct the cursor style of increment and decrement buttons in Chrome. 335 | */ 336 | 337 | [type="number"]::-webkit-inner-spin-button, 338 | [type="number"]::-webkit-outer-spin-button { 339 | height: auto; 340 | } 341 | 342 | 343 | /** 344 | * 1. Correct the odd appearance in Chrome and Safari. 345 | * 2. Correct the outline style in Safari. 346 | */ 347 | 348 | [type="search"] { 349 | -webkit-appearance: textfield; 350 | /* 1 */ 351 | outline-offset: -2px; 352 | /* 2 */ 353 | } 354 | 355 | 356 | /** 357 | * Remove the inner padding in Chrome and Safari on macOS. 358 | */ 359 | 360 | [type="search"]::-webkit-search-decoration { 361 | -webkit-appearance: none; 362 | } 363 | 364 | 365 | /** 366 | * 1. Correct the inability to style clickable types in iOS and Safari. 367 | * 2. Change font properties to `inherit` in Safari. 368 | */ 369 | 370 | ::-webkit-file-upload-button { 371 | -webkit-appearance: button; 372 | /* 1 */ 373 | font: inherit; 374 | /* 2 */ 375 | } 376 | 377 | 378 | /* Interactive 379 | ========================================================================== */ 380 | 381 | 382 | /* 383 | * Add the correct display in Edge, IE 10+, and Firefox. 384 | */ 385 | 386 | details { 387 | display: block; 388 | } 389 | 390 | 391 | /* 392 | * Add the correct display in all browsers. 393 | */ 394 | 395 | summary { 396 | display: list-item; 397 | } 398 | 399 | 400 | /* Misc 401 | ========================================================================== */ 402 | 403 | 404 | /** 405 | * Add the correct display in IE 10+. 406 | */ 407 | 408 | template { 409 | display: none; 410 | } 411 | 412 | 413 | /** 414 | * Add the correct display in IE 10. 415 | */ 416 | 417 | [hidden] { 418 | display: none; 419 | } -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /src/serve.rs: -------------------------------------------------------------------------------- 1 | use std::{collections::HashMap, io, net::SocketAddr, path::PathBuf, process::Stdio}; 2 | 3 | use askama::Template; 4 | use bytes::BytesMut; 5 | use futures_util::stream::TryStreamExt; 6 | use include_dir::{include_dir, Dir}; 7 | use thiserror::Error; 8 | use tokio::{ 9 | fs::File, 10 | io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}, 11 | process::{ChildStdout, Command}, 12 | }; 13 | use tokio_stream::StreamExt; 14 | use tokio_util::codec::{BytesCodec, FramedRead}; 15 | use warp::{ 16 | host::Authority, 17 | http, 18 | hyper::{body::Sender, Body, Response}, 19 | path::Tail, 20 | reject::Reject, 21 | Filter, Rejection, Stream, 22 | }; 23 | 24 | use crate::crates::get_crate_path; 25 | 26 | pub struct TlsConfig { 27 | pub cert_path: PathBuf, 28 | pub key_path: PathBuf, 29 | } 30 | 31 | #[derive(PartialEq, Eq, PartialOrd, Ord)] 32 | pub struct Platform { 33 | is_exe: bool, 34 | platform_triple: String, 35 | } 36 | 37 | #[derive(Template)] 38 | #[template(path = "index.html")] 39 | struct IndexTemplate { 40 | platforms: Vec, 41 | host: String, 42 | } 43 | 44 | const STATIC_DIR: Dir = include_dir!("static"); 45 | 46 | #[derive(Error, Debug)] 47 | pub enum ServeError { 48 | #[error("IO error: {0}")] 49 | Io(#[from] io::Error), 50 | #[error("Hyper error: {0}")] 51 | Hyper(#[from] warp::hyper::Error), 52 | #[error("Warp HTTP error: {0}")] 53 | Warp(#[from] warp::http::Error), 54 | #[error("{0}")] 55 | Other(String), 56 | } 57 | 58 | impl Reject for ServeError {} 59 | 60 | pub async fn serve(path: PathBuf, socket_addr: SocketAddr, tls_paths: Option) { 61 | let index_path = path.clone(); 62 | let is_tls = tls_paths.is_some(); 63 | 64 | // Handle the homepage 65 | let index = warp::path::end().and(warp::host::optional()).and_then( 66 | move |authority: Option| { 67 | let mirror_path = index_path.clone(); 68 | let protocol = if is_tls { "https://" } else { "http://" }; 69 | async move { 70 | get_rustup_platforms(mirror_path) 71 | .await 72 | .map(|platforms| IndexTemplate { 73 | platforms, 74 | host: authority 75 | .map(|a| format!("{}{}", protocol, a.as_str())) 76 | .unwrap_or_else(|| "http://panamax.internal".to_string()), 77 | }) 78 | .map_err(|_| { 79 | warp::reject::custom(ServeError::Other( 80 | "Could not retrieve rustup platforms.".to_string(), 81 | )) 82 | }) 83 | } 84 | }, 85 | ); 86 | 87 | // Handle all files baked into the binary with include_dir, at /static 88 | let static_dir = 89 | warp::path::path("static") 90 | .and(warp::path::tail()) 91 | .and_then(|path: Tail| async move { 92 | STATIC_DIR 93 | .get_file(path.as_str()) 94 | .ok_or_else(warp::reject::not_found) 95 | .map(|f| f.contents().to_vec()) 96 | }); 97 | 98 | let dist_dir = warp::path::path("dist").and(warp::fs::dir(path.join("dist"))); 99 | let rustup_dir = warp::path::path("rustup").and(warp::fs::dir(path.join("rustup"))); 100 | 101 | // Handle crates requests in the format of "/crates/ripgrep/0.1.0/download" 102 | // This format is the default for cargo, and will be used if an external process rewrites config.json in crates.io-index 103 | let crates_mirror_path = path.clone(); 104 | let crates_dir_native_format = warp::path!("crates" / String / String / "download").and_then( 105 | move |name: String, version: String| { 106 | let mirror_path = crates_mirror_path.clone(); 107 | async move { get_crate_file(mirror_path, &name, &version).await } 108 | }, 109 | ); 110 | 111 | // Handle crates requests in the format of either : 112 | // - "/crates/1/u/0.2.0/u-0.2.0.crate" 113 | // - "/crates/2/bm/0.11.0/bm-0.11.0.crate" 114 | // - "/crates/3/c/cde/0.1.1/cde-0.1.1.crate" 115 | // - "/crates/se/rd/serde/1.0.130/serde-1.0.130.crate" 116 | // This format is used by Panamax, and/or is used if config.json contains "/crates/{prefix}/{crate}/{version}/{crate}-{version}.crate" 117 | let crates_mirror_path_2 = path.clone(); 118 | let crates_dir_condensed_format_1 = warp::path!("crates" / "1" / String / String / String) 119 | .map(|name: String, version: String, crate_file: String| (name, version, crate_file)) 120 | .untuple_one(); 121 | let crates_dir_condensed_format_2 = warp::path!("crates" / "2" / String / String / String) 122 | .map(|name: String, version: String, crate_file: String| (name, version, crate_file)) 123 | .untuple_one(); 124 | let crates_dir_condensed_format_3 = 125 | warp::path!("crates" / "3" / String / String / String / String) 126 | .map( 127 | |_: String, name: String, version: String, crate_file: String| { 128 | (name, version, crate_file) 129 | }, 130 | ) 131 | .untuple_one(); 132 | let crates_dir_condensed_format_full = 133 | warp::path!("crates" / String / String / String / String / String) 134 | .map( 135 | |_: String, _: String, name: String, version: String, crate_file: String| { 136 | (name, version, crate_file) 137 | }, 138 | ) 139 | .untuple_one(); 140 | 141 | let crates_dir_condensed_format = crates_dir_condensed_format_1 142 | .or(crates_dir_condensed_format_2) 143 | .unify() 144 | .or(crates_dir_condensed_format_3) 145 | .unify() 146 | .or(crates_dir_condensed_format_full) 147 | .unify() 148 | .and_then(move |name: String, version: String, crate_file: String| { 149 | let mirror_path = crates_mirror_path_2.clone(); 150 | async move { 151 | if !crate_file.ends_with(".crate") || !crate_file.starts_with(&name) { 152 | return Err(warp::reject::not_found()); 153 | } 154 | get_crate_file(mirror_path, &name, &version).await 155 | } 156 | }); 157 | 158 | // Handle git client requests to /git/crates.io-index 159 | let path_for_git = path.clone(); 160 | let git = warp::path("git") 161 | .and(warp::path("crates.io-index")) 162 | .and(warp::path::tail()) 163 | .and(warp::method()) 164 | .and(warp::header::optional::("Content-Type")) 165 | .and(warp::addr::remote()) 166 | .and(warp::body::stream()) 167 | .and(warp::query::raw().or_else(|_| async { Ok::<(String,), Rejection>((String::new(),)) })) 168 | .and_then( 169 | move |path_tail, method, content_type, remote, body, query| { 170 | let mirror_path = path_for_git.clone(); 171 | async move { 172 | handle_git( 173 | mirror_path, 174 | path_tail, 175 | method, 176 | content_type, 177 | remote, 178 | body, 179 | query, 180 | ) 181 | .await 182 | } 183 | }, 184 | ); 185 | 186 | // Handle sparse index requests at /index/ 187 | let sparse_index = warp::path("index").and(warp::fs::dir(path.join("crates.io-index"))); 188 | 189 | let routes = index 190 | .or(static_dir) 191 | .or(dist_dir) 192 | .or(rustup_dir) 193 | .or(crates_dir_native_format) 194 | .or(crates_dir_condensed_format) 195 | .or(sparse_index) 196 | .or(git); 197 | 198 | match tls_paths { 199 | Some(TlsConfig { 200 | cert_path, 201 | key_path, 202 | }) => { 203 | println!("Running TLS on {socket_addr}"); 204 | warp::serve(routes) 205 | .tls() 206 | .cert_path(cert_path) 207 | .key_path(key_path) 208 | .run(socket_addr) 209 | .await; 210 | } 211 | None => { 212 | println!("Running HTTP on {socket_addr}"); 213 | warp::serve(routes).run(socket_addr).await; 214 | } 215 | } 216 | } 217 | 218 | /// Get all rustup platforms available on the mirror. 219 | async fn get_rustup_platforms(path: PathBuf) -> io::Result> { 220 | let rustup_path = path.join("rustup/dist"); 221 | 222 | let mut output = vec![]; 223 | 224 | // Look at the rustup/dist directory for all rustup-init and rustup-init.exe files. 225 | // Also return if the rustup-init file is a .exe or not. 226 | if let Ok(mut rd) = tokio::fs::read_dir(rustup_path).await { 227 | while let Some(entry) = rd.next_entry().await? { 228 | if entry.metadata().await?.is_dir() { 229 | if let Some(name) = entry.file_name().to_str() { 230 | let platform_triple = name.to_string(); 231 | if entry.path().join("rustup-init").exists() { 232 | output.push(Platform { 233 | is_exe: false, 234 | platform_triple, 235 | }); 236 | } else if entry.path().join("rustup-init.exe").exists() { 237 | output.push(Platform { 238 | is_exe: true, 239 | platform_triple, 240 | }); 241 | } 242 | } 243 | } 244 | } 245 | } 246 | 247 | // Sort by name, keeping non-exe versions at the top. 248 | output.sort(); 249 | 250 | Ok(output) 251 | } 252 | 253 | /// Return a crate file as an HTTP response. 254 | async fn get_crate_file( 255 | mirror_path: PathBuf, 256 | name: &str, 257 | version: &str, 258 | ) -> Result, Rejection> { 259 | let full_path = 260 | get_crate_path(&mirror_path, name, version).ok_or_else(warp::reject::not_found)?; 261 | 262 | let file = File::open(full_path) 263 | .await 264 | .map_err(|_| warp::reject::not_found())?; 265 | let meta = file 266 | .metadata() 267 | .await 268 | .map_err(|_| warp::reject::not_found())?; 269 | let stream = FramedRead::new(file, BytesCodec::new()).map_ok(BytesMut::freeze); 270 | 271 | let body = Body::wrap_stream(stream); 272 | 273 | let mut resp = Response::new(body); 274 | resp.headers_mut() 275 | .insert(http::header::CONTENT_LENGTH, meta.len().into()); 276 | 277 | Ok(resp) 278 | } 279 | 280 | /// Handle a request from a git client. 281 | async fn handle_git( 282 | mirror_path: PathBuf, 283 | path_tail: Tail, 284 | method: http::Method, 285 | content_type: Option, 286 | remote: Option, 287 | mut body: S, 288 | query: String, 289 | ) -> Result, Rejection> 290 | where 291 | S: Stream> + Send + Unpin + 'static, 292 | B: bytes::Buf + Sized, 293 | { 294 | let remote = remote 295 | .map(|r| r.ip().to_string()) 296 | .unwrap_or_else(|| "127.0.0.1".to_string()); 297 | 298 | // Run "git http-backend" 299 | let mut cmd = Command::new("git"); 300 | cmd.arg("http-backend"); 301 | 302 | // Clear environment variables, and set needed variables 303 | // See: https://git-scm.com/docs/git-http-backend 304 | cmd.env_clear(); 305 | cmd.env("GIT_PROJECT_ROOT", mirror_path); 306 | cmd.env( 307 | "PATH_INFO", 308 | format!("/crates.io-index/{}", path_tail.as_str()), 309 | ); 310 | cmd.env("REQUEST_METHOD", method.as_str()); 311 | cmd.env("QUERY_STRING", query); 312 | cmd.env("REMOTE_USER", ""); 313 | cmd.env("REMOTE_ADDR", remote); 314 | if let Some(content_type) = content_type { 315 | cmd.env("CONTENT_TYPE", content_type); 316 | } 317 | cmd.env("GIT_HTTP_EXPORT_ALL", "true"); 318 | cmd.stderr(Stdio::inherit()); 319 | cmd.stdout(Stdio::piped()); 320 | cmd.stdin(Stdio::piped()); 321 | 322 | let p = cmd.spawn().map_err(ServeError::from)?; 323 | 324 | // Handle sending git client body to http-backend, if any 325 | let mut git_input = p.stdin.expect("Process should always have stdin"); 326 | while let Some(Ok(mut buf)) = body.next().await { 327 | git_input 328 | .write_all_buf(&mut buf) 329 | .await 330 | .map_err(ServeError::from)?; 331 | } 332 | 333 | // Collect headers from git CGI output 334 | let mut git_output = BufReader::new(p.stdout.expect("Process should always have stdout")); 335 | let mut headers = HashMap::new(); 336 | loop { 337 | let mut line = String::new(); 338 | git_output 339 | .read_line(&mut line) 340 | .await 341 | .map_err(ServeError::from)?; 342 | 343 | let line = line.trim_end(); 344 | if line.is_empty() { 345 | break; 346 | } 347 | 348 | if let Some((key, value)) = line.split_once(": ") { 349 | headers.insert(key.to_string(), value.to_string()); 350 | } 351 | } 352 | 353 | // Add headers to response (except for Status, which is the "200 OK" line) 354 | let mut resp = Response::builder(); 355 | for (key, val) in headers { 356 | if key == "Status" { 357 | resp = resp.status(&val.as_bytes()[..3]); 358 | } else { 359 | resp = resp.header(&key, val); 360 | } 361 | } 362 | 363 | // Create channel, so data can be streamed without being fully loaded 364 | // into memory. Requires a separate future to be spawned. 365 | let (sender, body) = Body::channel(); 366 | tokio::spawn(send_git(sender, git_output)); 367 | 368 | let resp = resp.body(body).map_err(ServeError::from)?; 369 | Ok(resp) 370 | } 371 | 372 | /// Send data from git CGI process to hyper Sender, until there is no more 373 | /// data left. 374 | async fn send_git( 375 | mut sender: Sender, 376 | mut git_output: BufReader, 377 | ) -> Result<(), ServeError> { 378 | loop { 379 | let mut bytes_out = BytesMut::new(); 380 | git_output.read_buf(&mut bytes_out).await?; 381 | if bytes_out.is_empty() { 382 | return Ok(()); 383 | } 384 | sender.send_data(bytes_out.freeze()).await?; 385 | } 386 | } 387 | -------------------------------------------------------------------------------- /src/mirror.rs: -------------------------------------------------------------------------------- 1 | use std::net::{IpAddr, SocketAddr}; 2 | use std::path::{Path, PathBuf}; 3 | use std::{fs, io}; 4 | 5 | use console::style; 6 | use reqwest::header::HeaderValue; 7 | use serde::{Deserialize, Serialize}; 8 | use thiserror::Error; 9 | 10 | use crate::crates::is_new_crates_format; 11 | use crate::crates_index::rewrite_config_json; 12 | 13 | use crate::rustup::download_platform_list; 14 | use crate::serve::TlsConfig; 15 | use crate::verify; 16 | 17 | #[derive(Error, Debug)] 18 | pub enum MirrorError { 19 | #[error("IO error: {0}")] 20 | Io(#[from] io::Error), 21 | 22 | #[error("Git error: {0}")] 23 | Git(#[from] git2::Error), 24 | 25 | #[error("TOML deserialization error: {0:?}")] 26 | Parse(#[from] toml_edit::de::Error), 27 | 28 | #[error("Config file error: {0}")] 29 | Config(String), 30 | 31 | #[error("Command line error: {0}")] 32 | CmdLine(String), 33 | 34 | #[error("Download error: {0}")] 35 | DownloadError(#[from] crate::download::DownloadError), 36 | 37 | #[error("Toml error: {0}")] 38 | Serialize(#[from] toml_edit::TomlError), 39 | } 40 | 41 | #[derive(Serialize, Deserialize, Debug)] 42 | pub struct ConfigMirror { 43 | pub retries: usize, 44 | pub contact: Option, 45 | } 46 | 47 | #[derive(Serialize, Deserialize, Debug)] 48 | pub struct ConfigRustup { 49 | pub sync: bool, 50 | pub download_threads: usize, 51 | pub source: String, 52 | pub download_dev: Option, 53 | pub download_gz: Option, 54 | pub download_xz: Option, 55 | pub platforms_unix: Option>, 56 | pub platforms_windows: Option>, 57 | pub keep_latest_stables: Option, 58 | pub keep_latest_betas: Option, 59 | pub keep_latest_nightlies: Option, 60 | pub pinned_rust_versions: Option>, 61 | } 62 | 63 | #[derive(Serialize, Deserialize, Debug)] 64 | pub struct ConfigCrates { 65 | pub sync: bool, 66 | pub download_threads: usize, 67 | pub source: String, 68 | pub source_index: String, 69 | pub use_new_crates_format: Option, 70 | pub base_url: Option, 71 | } 72 | 73 | #[derive(Serialize, Deserialize, Debug)] 74 | pub struct Config { 75 | pub mirror: ConfigMirror, 76 | pub rustup: Option, 77 | pub crates: Option, 78 | } 79 | 80 | pub fn create_mirror_directories(path: &Path, ignore_rustup: bool) -> Result<(), io::Error> { 81 | if !ignore_rustup { 82 | // Rustup directories 83 | fs::create_dir_all(path.join("rustup/dist"))?; 84 | fs::create_dir_all(path.join("dist"))?; 85 | } 86 | 87 | // Crates directories 88 | fs::create_dir_all(path.join("crates.io-index"))?; 89 | fs::create_dir_all(path.join("crates"))?; 90 | Ok(()) 91 | } 92 | 93 | pub fn create_mirror_toml(path: &Path, ignore_rustup: bool) -> Result { 94 | if path.join("mirror.toml").exists() { 95 | return Ok(false); 96 | } 97 | 98 | // Read the defautlt toml, edit if required, using toml_edit to keep format 99 | let config = include_str!("mirror.default.toml"); 100 | let mut config = config.parse::()?; 101 | 102 | if ignore_rustup { 103 | config["rustup"]["sync"] = toml_edit::value(false); 104 | } 105 | 106 | let path = path.join("mirror.toml"); 107 | let bytes = config.to_string(); 108 | fs::write(path, bytes)?; 109 | 110 | Ok(true) 111 | } 112 | 113 | pub fn load_mirror_toml(path: &Path) -> Result { 114 | Ok(toml_edit::easy::from_str(&fs::read_to_string( 115 | path.join("mirror.toml"), 116 | )?)?) 117 | } 118 | 119 | pub fn init(path: &Path, ignore_rustup: bool) -> Result<(), MirrorError> { 120 | create_mirror_directories(path, ignore_rustup)?; 121 | if create_mirror_toml(path, ignore_rustup)? { 122 | eprintln!("Successfully created mirror base at `{}`.", path.display()); 123 | } else { 124 | eprintln!("Mirror base already exists at `{}`.", path.display()); 125 | } 126 | eprintln!( 127 | "Make any desired changes to {}/mirror.toml, then run panamax sync {}.", 128 | path.display(), 129 | path.display() 130 | ); 131 | 132 | Ok(()) 133 | } 134 | 135 | pub fn default_user_agent() -> String { 136 | format!("Panamax/{}", env!("CARGO_PKG_VERSION")) 137 | } 138 | 139 | pub async fn sync( 140 | path: &Path, 141 | vendor_path: Option, 142 | cargo_lock_filepath: Option, 143 | skip_rustup: bool, 144 | ) -> Result<(), MirrorError> { 145 | if !path.join("mirror.toml").exists() { 146 | eprintln!( 147 | "Mirror base not found! Run panamax init {} first.", 148 | path.display() 149 | ); 150 | return Ok(()); 151 | } 152 | let mirror = load_mirror_toml(path)?; 153 | 154 | // Fail if use_new_crates_format is not true, and old format is detected. 155 | // If use_new_crates_format is true and new format is detected, warn the user. 156 | // If use_new_crates_format is true, ignore the format and assume it's new. 157 | if let Some(crates) = &mirror.crates { 158 | if crates.sync && !is_new_crates_format(&path.join("crates"))? { 159 | eprintln!("Your crates directory is using the old 0.2 format, however"); 160 | eprintln!("Panamax 0.3+ has deprecated this format for a new one."); 161 | eprintln!("Please delete crates/ from your mirror directory to continue."); 162 | return Ok(()); 163 | } 164 | } 165 | 166 | // Handle the contact information 167 | let user_agent_str = if let Some(ref contact) = mirror.mirror.contact { 168 | if contact != "your@email.com" { 169 | format!("Panamax/{} ({})", env!("CARGO_PKG_VERSION"), contact) 170 | } else { 171 | default_user_agent() 172 | } 173 | } else { 174 | default_user_agent() 175 | }; 176 | 177 | // Set the user agent with contact information. 178 | let user_agent = match HeaderValue::from_str(&user_agent_str) { 179 | Ok(h) => h, 180 | Err(e) => { 181 | eprintln!("Your contact information contains invalid characters!"); 182 | eprintln!("It's recommended to use a URL or email address as contact information."); 183 | eprintln!("{e:?}"); 184 | return Ok(()); 185 | } 186 | }; 187 | 188 | if let Some(rustup) = mirror.rustup { 189 | if rustup.sync && !skip_rustup { 190 | crate::rustup::sync(path, &mirror.mirror, &rustup, &user_agent).await?; 191 | } else { 192 | eprintln!("Rustup sync is disabled, skipping..."); 193 | } 194 | } else { 195 | eprintln!("Rustup section missing, skipping..."); 196 | } 197 | 198 | if let Some(crates) = mirror.crates { 199 | if crates.sync { 200 | sync_crates( 201 | path, 202 | vendor_path, 203 | cargo_lock_filepath, 204 | &mirror.mirror, 205 | &crates, 206 | &user_agent, 207 | ) 208 | .await; 209 | } else { 210 | eprintln!("Crates sync is disabled, skipping..."); 211 | } 212 | } else { 213 | eprintln!("Crates section missing, skipping..."); 214 | } 215 | 216 | eprintln!("Sync complete."); 217 | 218 | Ok(()) 219 | } 220 | 221 | /// Rewrite the config.toml only. 222 | /// 223 | /// Note that this will also fast-forward the repository 224 | /// from origin/master, to keep a clean slate. 225 | pub fn rewrite(path: &Path, base_url: Option) -> Result<(), MirrorError> { 226 | if !path.join("mirror.toml").exists() { 227 | eprintln!( 228 | "Mirror base not found! Run panamax init {} first.", 229 | path.display() 230 | ); 231 | return Ok(()); 232 | } 233 | let mirror = load_mirror_toml(path)?; 234 | 235 | if let Some(crates) = mirror.crates { 236 | if let Some(base_url) = base_url.as_deref().or(crates.base_url.as_deref()) { 237 | if let Err(e) = rewrite_config_json(&path.join("crates.io-index"), base_url) { 238 | eprintln!("Updating crates.io-index config failed: {e:?}"); 239 | } 240 | } else { 241 | eprintln!("No base_url was provided."); 242 | eprintln!( 243 | "This needs to be provided by command line or in the mirror.toml to continue." 244 | ) 245 | } 246 | } else { 247 | eprintln!("Crates section missing in mirror.toml."); 248 | } 249 | 250 | Ok(()) 251 | } 252 | 253 | /// Synchronize and handle the crates.io-index repository. 254 | pub async fn sync_crates( 255 | path: &Path, 256 | vendor_path: Option, 257 | cargo_lock_filepath: Option, 258 | mirror: &ConfigMirror, 259 | crates: &ConfigCrates, 260 | user_agent: &HeaderValue, 261 | ) { 262 | eprintln!("{}", style("Syncing Crates repositories...").bold()); 263 | 264 | if let Err(e) = crate::crates_index::sync_crates_repo(path, crates) { 265 | eprintln!("Downloading crates.io-index repository failed: {e:?}"); 266 | eprintln!("You will need to sync again to finish this download."); 267 | return; 268 | } 269 | 270 | if let Err(e) = crate::crates::sync_crates_files( 271 | path, 272 | vendor_path, 273 | cargo_lock_filepath, 274 | mirror, 275 | crates, 276 | user_agent, 277 | ) 278 | .await 279 | { 280 | eprintln!("Downloading crates failed: {e:?}"); 281 | eprintln!("You will need to sync again to finish this download."); 282 | return; 283 | } 284 | 285 | if let Err(e) = crate::crates_index::update_crates_config(path, crates) { 286 | eprintln!("Updating crates.io-index config failed: {e:?}"); 287 | eprintln!("You will need to sync again to finish this download."); 288 | } 289 | 290 | eprintln!("{}", style("Syncing Crates repositories complete!").bold()); 291 | } 292 | 293 | pub async fn serve( 294 | path: PathBuf, 295 | listen: Option, 296 | port: Option, 297 | cert_path: Option, 298 | key_path: Option, 299 | ) -> Result<(), MirrorError> { 300 | let listen = listen.unwrap_or_else(|| { 301 | "::".parse() 302 | .expect(":: IPv6 address should never fail to parse") 303 | }); 304 | let port = port.unwrap_or_else(|| if cert_path.is_some() { 8443 } else { 8080 }); 305 | let socket_addr = SocketAddr::new(listen, port); 306 | 307 | match (cert_path, key_path) { 308 | (Some(cert_path), Some(key_path)) => { 309 | crate::serve::serve( 310 | path, 311 | socket_addr, 312 | Some(TlsConfig { 313 | cert_path, 314 | key_path, 315 | }), 316 | ) 317 | .await 318 | } 319 | (None, None) => crate::serve::serve(path, socket_addr, None).await, 320 | (Some(_), None) => { 321 | return Err(MirrorError::CmdLine( 322 | "cert_path set but key_path not set.".to_string(), 323 | )) 324 | } 325 | (None, Some(_)) => { 326 | return Err(MirrorError::CmdLine( 327 | "key_path set but cert_path not set.".to_string(), 328 | )) 329 | } 330 | }; 331 | 332 | Ok(()) 333 | } 334 | 335 | /// Print out a list of all platforms. 336 | pub(crate) async fn list_platforms(source: String, channel: String) -> Result<(), MirrorError> { 337 | let targets = download_platform_list(source.as_str(), channel.as_str()).await?; 338 | 339 | println!("All currently available platforms for the {channel} channel:"); 340 | for t in targets { 341 | println!(" {t}"); 342 | } 343 | 344 | Ok(()) 345 | } 346 | 347 | /// Verify coherence between local mirror and local crates.io-index. 348 | /// This function is bale to fix mirror by downloading missing crates. 349 | /// Users can alter the actual downloaded file at run time. 350 | pub(crate) async fn verify( 351 | path: PathBuf, 352 | dry_run: bool, 353 | assume_yes: bool, 354 | vendor_path: Option, 355 | cargo_lock_filepath: Option, 356 | ) -> Result<(), MirrorError> { 357 | if !path.join("mirror.toml").exists() { 358 | eprintln!( 359 | "Mirror base not found! Run panamax init {} first.", 360 | path.display() 361 | ); 362 | return Ok(()); 363 | } 364 | let config = load_mirror_toml(&path)?; 365 | 366 | // Fail if use_new_crates_format is not true, and old format is detected. 367 | // If use_new_crates_format is true and new format is detected, warn the user. 368 | // If use_new_crates_format is true, ignore the format and assume it's new. 369 | if let Some(config) = &config.crates { 370 | if config.sync && !is_new_crates_format(&path.join("crates"))? { 371 | eprintln!("Your crates directory is using the old 0.2 format, however"); 372 | eprintln!("Panamax 0.3+ has deprecated this format for a new one."); 373 | eprintln!("Please delete crates/ from your mirror directory to continue."); 374 | return Ok(()); 375 | } 376 | } 377 | 378 | eprintln!("{}", style("Verifying mirror state...").bold()); 379 | 380 | // Getting crates.sync config state 381 | let crates_config = config.crates.as_ref(); 382 | let sync = crates_config.map_or(false, |crate_config| crate_config.sync); 383 | 384 | // Determining number of steps 385 | let steps = if dry_run || !sync { 1 } else { 2 }; 386 | let mut current_step = 1; 387 | 388 | if let Some(mut missing_crates) = verify::verify_mirror( 389 | path.clone(), 390 | &mut current_step, 391 | steps, 392 | vendor_path, 393 | cargo_lock_filepath, 394 | ) 395 | .await? 396 | { 397 | if dry_run || !sync { 398 | if !sync { 399 | eprintln!("Crates sync is disabled, only printing missing crates..."); 400 | } 401 | missing_crates.iter().for_each(|c| { 402 | println!("Missing crate: {} - version {}", c.get_name(), c.get_vers()); 403 | }); 404 | return Ok(()); 405 | } 406 | 407 | // Safe to unwrap here 408 | let crates_config = crates_config.unwrap(); 409 | 410 | debug_assert_ne!(steps, current_step); 411 | 412 | // Ask users to choose whether to filter missing crates to download or not 413 | if !assume_yes { 414 | missing_crates = verify::handle_user_input(missing_crates).await?; 415 | } 416 | 417 | let mirror_config = &config.mirror; 418 | 419 | // Downloading missing crates 420 | verify::fix_mirror( 421 | mirror_config, 422 | crates_config, 423 | path, 424 | missing_crates, 425 | &mut current_step, 426 | steps, 427 | ) 428 | .await?; 429 | } 430 | 431 | Ok(()) 432 | } 433 | -------------------------------------------------------------------------------- /src/crates.rs: -------------------------------------------------------------------------------- 1 | use crate::crates_index::{fast_forward, IndexSyncError}; 2 | use crate::download::{download, DownloadError}; 3 | use crate::mirror::{ConfigCrates, ConfigMirror}; 4 | use crate::progress_bar::padded_prefix_message; 5 | use futures::StreamExt; 6 | use git2::Repository; 7 | use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; 8 | use reqwest::header::HeaderValue; 9 | use reqwest::Client; 10 | use serde::{Deserialize, Serialize}; 11 | use std::ffi::OsStr; 12 | use std::fs::read_dir; 13 | use std::path::{Path, PathBuf}; 14 | use std::time::Duration; 15 | use std::{ 16 | fs, 17 | io::{self, BufRead, Cursor}, 18 | }; 19 | use thiserror::Error; 20 | 21 | #[derive(Error, Debug)] 22 | pub enum SyncError { 23 | #[error("IO error: {0}")] 24 | Io(#[from] io::Error), 25 | 26 | #[error("Download error: {0}")] 27 | Download(#[from] DownloadError), 28 | 29 | #[error("JSON serialization error: {0}")] 30 | SerializeError(#[from] serde_json::Error), 31 | 32 | #[error("Git error: {0}")] 33 | GitError(#[from] git2::Error), 34 | 35 | #[error("Index syncing error: {0}")] 36 | IndexSync(#[from] IndexSyncError), 37 | } 38 | /// One entry found in a crates.io-index file. 39 | /// These files are formatted as lines of JSON. 40 | #[derive(Debug, Clone, Serialize, Deserialize)] 41 | pub struct CrateEntry { 42 | name: String, 43 | vers: String, 44 | cksum: Option, 45 | yanked: Option, 46 | } 47 | 48 | impl CrateEntry { 49 | pub(crate) fn get_name(&self) -> &str { 50 | self.name.as_str() 51 | } 52 | 53 | pub(crate) fn get_vers(&self) -> &str { 54 | self.vers.as_str() 55 | } 56 | } 57 | 58 | /// Download one single crate file. 59 | pub async fn sync_one_crate_entry( 60 | client: &Client, 61 | path: &Path, 62 | source: Option<&str>, 63 | retries: usize, 64 | crate_entry: &CrateEntry, 65 | user_agent: &HeaderValue, 66 | ) -> Result<(), DownloadError> { 67 | // If source is "https://crates.io/api/v1/crates" (the default, and thus a None here) 68 | // download straight from the static.crates.io CDN, to avoid bogging down crates.io itself 69 | // or affecting its statistics, and avoiding an extra redirect for each crate. 70 | let url = if let Some(source) = source { 71 | format!( 72 | "{}/{}/{}/download", 73 | source, crate_entry.name, crate_entry.vers 74 | ) 75 | } else { 76 | format!( 77 | "https://static.crates.io/crates/{}/{}-{}.crate", 78 | crate_entry.name, crate_entry.name, crate_entry.vers 79 | ) 80 | }; 81 | 82 | let file_path = get_crate_path(path, &crate_entry.name, &crate_entry.vers) 83 | .ok_or_else(|| DownloadError::BadCrate(crate_entry.name.clone()))?; 84 | 85 | download( 86 | client, 87 | &url[..], 88 | &file_path, 89 | crate_entry.cksum.as_deref(), 90 | retries, 91 | false, 92 | user_agent, 93 | ) 94 | .await 95 | } 96 | 97 | /// Synchronize the crate files themselves, using the index for a list of files. 98 | // TODO: There are still many unwraps in the foreach sections. This needs to be fixed. 99 | pub async fn sync_crates_files( 100 | path: &Path, 101 | vendor_path: Option, 102 | cargo_lock_filepath: Option, 103 | mirror: &ConfigMirror, 104 | crates: &ConfigCrates, 105 | user_agent: &HeaderValue, 106 | ) -> Result<(), SyncError> { 107 | let is_crate_whitelist_only = vendor_path.is_some() || cargo_lock_filepath.is_some(); 108 | 109 | // if a vendor_path, parse the filepath for Cargo.toml files for each crate, filling vendors 110 | let mut mirror_entries = vec![]; 111 | vendor_path_to_mirror_entries(&mut mirror_entries, vendor_path.as_ref()); 112 | // gather crates from Cargo.lock if supplied 113 | cargo_lock_to_mirror_entries(&mut mirror_entries, cargo_lock_filepath.as_ref()); 114 | 115 | let prefix = padded_prefix_message(2, 3, "Syncing crates files"); 116 | 117 | // For now, assume successful crates.io-index download 118 | let repo_path = path.join("crates.io-index"); 119 | let repo = Repository::open(&repo_path)?; 120 | 121 | // Set the crates.io URL, or None if default 122 | let crates_source = if crates.source == "https://crates.io/api/v1/crates" { 123 | None 124 | } else { 125 | Some(crates.source.as_str()) 126 | }; 127 | 128 | // Find Reference for origin/master 129 | let origin_master = repo.find_reference("refs/remotes/origin/master")?; 130 | let origin_master_tree = origin_master.peel_to_tree()?; 131 | 132 | let master = repo.find_reference("refs/heads/master").ok(); 133 | let master_tree = master.as_ref().and_then(|m| m.peel_to_tree().ok()); 134 | 135 | // Diff between master and origin/master (i.e. everything since the last fetch) 136 | let diff = repo.diff_tree_to_tree(master_tree.as_ref(), Some(&origin_master_tree), None)?; 137 | 138 | let mut changed_crates = Vec::new(); 139 | let mut removed_crates = Vec::new(); 140 | 141 | let pb = ProgressBar::new_spinner() 142 | .with_style( 143 | ProgressStyle::default_bar() 144 | .template("{prefix} {wide_bar} {spinner} [{elapsed_precise}]") 145 | .expect("template is correct") 146 | .progress_chars(" "), 147 | ) 148 | .with_finish(ProgressFinish::AndLeave) 149 | .with_prefix(prefix.clone()); 150 | pb.enable_steady_tick(Duration::from_millis(10)); 151 | 152 | // Figure out which crates we need to update/remove. 153 | diff.foreach( 154 | &mut |delta, _| { 155 | let df = delta.new_file(); 156 | let p = df.path().unwrap(); 157 | if p == Path::new("config.json") { 158 | return true; 159 | } 160 | if p.starts_with(".github/") { 161 | return true; 162 | } 163 | 164 | // DEV: if dev_reduced_crates is enabled, only download crates that start with z. 165 | // Keep this code in here, because it's helpful for development and debugging. 166 | #[cfg(feature = "dev_reduced_crates")] 167 | { 168 | // Get file name, try-convert to string, check if starts_with z, unwrap, or false if None 169 | if !p 170 | .file_name() 171 | .and_then(|x| x.to_str()) 172 | .map(|x| x.starts_with('z')) 173 | .unwrap_or(false) 174 | { 175 | return true; 176 | } 177 | } 178 | 179 | // Get the data for this crate file 180 | let oid = df.id(); 181 | if oid.is_zero() { 182 | // The crate was removed, continue to next crate. 183 | // Note that this does not include yanked crates. 184 | removed_crates.push(p.to_path_buf()); 185 | return true; 186 | } 187 | let blob = repo.find_blob(oid).unwrap(); 188 | let data = blob.content(); 189 | 190 | // Download one crate for each of the versions in the crate file 191 | for line in Cursor::new(data).lines() { 192 | let line = line.unwrap(); 193 | let c = match serde_json::from_str::(&line) { 194 | Ok(c) => { 195 | // if vendor_path, check for matching crate name/version 196 | if is_crate_whitelist_only { 197 | if mirror_entries 198 | .iter() 199 | .any(|a| a.name == c.name && a.vers == c.vers) 200 | { 201 | c 202 | } else { 203 | continue; 204 | } 205 | } else { 206 | c 207 | } 208 | } 209 | Err(_) => { 210 | continue; 211 | } 212 | }; 213 | 214 | changed_crates.push(c); 215 | } 216 | 217 | true 218 | }, 219 | None, 220 | None, 221 | None, 222 | ) 223 | .unwrap(); 224 | 225 | pb.finish_and_clear(); 226 | let pb = ProgressBar::new(changed_crates.len() as u64) 227 | .with_style( 228 | ProgressStyle::default_bar() 229 | .template( 230 | "{prefix} {wide_bar} {pos}/{len} [{elapsed_precise} / {duration_precise}]", 231 | ) 232 | .expect("template is correct") 233 | .progress_chars("█▉▊▋▌▍▎▏ "), 234 | ) 235 | .with_finish(ProgressFinish::AndLeave) 236 | .with_prefix(prefix); 237 | pb.enable_steady_tick(Duration::from_millis(10)); 238 | 239 | let client = Client::new(); 240 | 241 | // Dirty hack: 242 | // Since we can't rely on diff tree because these crates are manually set 243 | // we force them to always update. 244 | if is_crate_whitelist_only { 245 | changed_crates.append(&mut mirror_entries); 246 | } 247 | 248 | let tasks = futures::stream::iter(changed_crates.into_iter()) 249 | .map(|c| { 250 | let client = client.clone(); 251 | // Duplicate variables used in the async closure. 252 | let path = path.to_owned(); 253 | let mirror_retries = mirror.retries; 254 | let crates_source = crates_source.map(|s| s.to_string()); 255 | let user_agent = user_agent.to_owned(); 256 | let pb = pb.clone(); 257 | 258 | tokio::spawn(async move { 259 | let out = sync_one_crate_entry( 260 | &client, 261 | &path, 262 | crates_source.as_deref(), 263 | mirror_retries, 264 | &c, 265 | &user_agent, 266 | ) 267 | .await; 268 | 269 | pb.inc(1); 270 | 271 | out 272 | }) 273 | }) 274 | .buffer_unordered(crates.download_threads) 275 | .collect::>() 276 | .await; 277 | 278 | for t in tasks { 279 | let res = t.unwrap(); 280 | match res { 281 | Ok(()) 282 | | Err(DownloadError::NotFound { 283 | status: _, 284 | url: _, 285 | data: _, 286 | }) 287 | | Err(DownloadError::MismatchedHash { 288 | expected: _, 289 | actual: _, 290 | }) => {} 291 | 292 | Err(e) => { 293 | eprintln!("Downloading failed: {e:?}"); 294 | } 295 | } 296 | } 297 | 298 | // Delete any removed crates 299 | for rc in removed_crates { 300 | // Try to remove the file, but ignore it if it doesn't exist 301 | let _ = fs::remove_file(repo_path.join(rc)); 302 | } 303 | 304 | // Set master to origin/master. 305 | // 306 | // Note that this means config.json changes will have to be rewritten on every sync. 307 | fast_forward(&repo_path)?; 308 | 309 | Ok(()) 310 | } 311 | 312 | /// Detect if the crates directory is using the old format. 313 | pub fn is_new_crates_format(path: &Path) -> Result { 314 | if !path.exists() { 315 | // Path doesn't exist, so we can start with a clean slate. 316 | return Ok(true); 317 | } 318 | 319 | for crate_dir in read_dir(path)? { 320 | let crate_dir = crate_dir?; 321 | if !crate_dir.file_type()?.is_dir() { 322 | // Ignore any files in the directory. Only look at other directories. 323 | continue; 324 | } 325 | 326 | let dir_name = crate_dir 327 | .file_name() 328 | .into_string() 329 | .map_err(|_| io::ErrorKind::Other)?; 330 | match dir_name.as_str() { 331 | // 1-letter crate names cannot be numbers, so this must be new format. 332 | "1" | "2" | "3" => continue, 333 | // 2-letter directories are used for crates longer than 3 characters. 334 | x if x.len() == 2 => continue, 335 | // Unrecognized directory found, might be crate in old format. 336 | _ => { 337 | return Ok(false); 338 | } 339 | }; 340 | } 341 | 342 | Ok(true) 343 | } 344 | 345 | pub fn get_crate_path( 346 | mirror_path: &Path, 347 | crate_name: &str, 348 | crate_version: &str, 349 | ) -> Option { 350 | let crate_path = match crate_name.len() { 351 | 1 => PathBuf::from("1"), 352 | 2 => PathBuf::from("2"), 353 | 3 => { 354 | let first_char = crate_name.get(0..1)?; 355 | PathBuf::from("3").join(first_char) 356 | } 357 | n if n >= 4 => { 358 | let first_two = crate_name.get(0..2)?; 359 | let second_two = crate_name.get(2..4)?; 360 | [first_two, second_two].iter().collect() 361 | } 362 | _ => return None, 363 | }; 364 | 365 | Some( 366 | mirror_path 367 | .join("crates") 368 | .join(crate_path) 369 | .join(crate_name) 370 | .join(crate_version) 371 | .join(format!("{crate_name}-{crate_version}.crate")), 372 | ) 373 | } 374 | 375 | pub(crate) fn vendor_path_to_mirror_entries( 376 | mirror_entries: &mut Vec, 377 | vendor_path: Option<&PathBuf>, 378 | ) { 379 | if let Some(vendor_path) = &vendor_path { 380 | use walkdir::WalkDir; 381 | for entry in WalkDir::new(vendor_path.as_path()) 382 | .min_depth(1) 383 | .max_depth(2) 384 | { 385 | let path = entry.as_ref().unwrap().path(); 386 | if path.file_name() == Some(OsStr::new("Cargo.toml")) { 387 | let s = fs::read_to_string(entry.unwrap().path()).unwrap(); 388 | let crate_toml = s.parse::().unwrap(); 389 | if let toml_edit::easy::Value::Table(crate_f) = crate_toml { 390 | let name = crate_f["package"]["name"].to_string().replace('\"', ""); 391 | let version = crate_f["package"]["version"].to_string().replace('\"', ""); 392 | mirror_entries.push(CrateEntry { 393 | name, 394 | vers: version, 395 | cksum: None, 396 | yanked: None, 397 | }); 398 | } 399 | } 400 | } 401 | } 402 | } 403 | 404 | pub(crate) fn cargo_lock_to_mirror_entries( 405 | mirror_entries: &mut Vec, 406 | cargo_lock_filepath: Option<&PathBuf>, 407 | ) { 408 | if let Some(cargo_lock_filepath) = &cargo_lock_filepath { 409 | if cargo_lock_filepath.is_file() { 410 | let s = fs::read_to_string(cargo_lock_filepath).unwrap(); 411 | let cargo_lock = s.parse::().unwrap(); 412 | if let toml_edit::easy::Value::Table(global) = cargo_lock { 413 | let packages_array = &global["package"]; 414 | 415 | if let toml_edit::easy::Value::Array(packages) = packages_array { 416 | packages.iter().for_each(|package| { 417 | if let toml_edit::easy::Value::Table(package) = package { 418 | // filter out non crates-io crates 419 | if let Some(source) = package.get("source") { 420 | let source = source.to_string().replace('\"', ""); 421 | if source.contains( 422 | "registry+https://github.com/rust-lang/crates.io-index", 423 | ) { 424 | let name = package["name"].to_string().replace('\"', ""); 425 | let version = package["version"].to_string().replace('\"', ""); 426 | let checksum = 427 | package["checksum"].to_string().replace('\"', ""); 428 | mirror_entries.push(CrateEntry { 429 | name, 430 | vers: version, 431 | cksum: Some(checksum), 432 | yanked: None, 433 | }); 434 | } 435 | } 436 | } 437 | }); 438 | } 439 | } 440 | } else { 441 | eprintln!("{:?} is not a Cargo.lock!", cargo_lock_filepath); 442 | } 443 | } 444 | } 445 | -------------------------------------------------------------------------------- /src/verify.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | cmp::Ordering, 3 | convert::Infallible, 4 | io::{BufRead, Cursor, Write}, 5 | ops::RangeInclusive, 6 | path::{Path, PathBuf}, 7 | str::FromStr, 8 | time::Duration, 9 | }; 10 | 11 | use console::style; 12 | use futures::StreamExt; 13 | use git2::Repository; 14 | use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; 15 | use reqwest::Client; 16 | use warp::http::HeaderValue; 17 | 18 | use crate::{ 19 | crates::{ 20 | cargo_lock_to_mirror_entries, get_crate_path, sync_one_crate_entry, 21 | vendor_path_to_mirror_entries, CrateEntry, 22 | }, 23 | download::DownloadError, 24 | mirror::{default_user_agent, ConfigCrates, ConfigMirror, MirrorError}, 25 | progress_bar::padded_prefix_message, 26 | }; 27 | 28 | /// 29 | /// This variable is here to avoid to have false positive regarding crates.io [issue#1593](https://github.com/rust-lang/crates.io/issues/1593). 30 | /// 31 | static CRATES_403: [(&str, &str); 23] = [ 32 | ("glib-2-0-sys", "0.0.1"), 33 | ("glib-2-0-sys", "0.0.2"), 34 | ("glib-2-0-sys", "0.0.3"), 35 | ("glib-2-0-sys", "0.0.4"), 36 | ("glib-2-0-sys", "0.0.5"), 37 | ("glib-2-0-sys", "0.0.6"), 38 | ("glib-2-0-sys", "0.0.7"), 39 | ("glib-2-0-sys", "0.0.8"), 40 | ("glib-2-0-sys", "0.1.0"), 41 | ("glib-2-0-sys", "0.1.1"), 42 | ("glib-2-0-sys", "0.1.2"), 43 | ("glib-2-0-sys", "0.2.0"), 44 | ("gobject-2-0-sys", "0.0.2"), 45 | ("gobject-2-0-sys", "0.0.3"), 46 | ("gobject-2-0-sys", "0.0.2"), 47 | ("gobject-2-0-sys", "0.0.4"), 48 | ("gobject-2-0-sys", "0.0.5"), 49 | ("gobject-2-0-sys", "0.0.6"), 50 | ("gobject-2-0-sys", "0.0.7"), 51 | ("gobject-2-0-sys", "0.0.8"), 52 | ("gobject-2-0-sys", "0.0.9"), 53 | ("gobject-2-0-sys", "0.1.0"), 54 | ("gobject-2-0-sys", "0.2.0"), 55 | ]; 56 | 57 | /// Type used to represent user's input which will be used to indexed a `Vec` 58 | #[derive(Debug, PartialEq, Eq)] 59 | enum Input { 60 | Range(RangeInclusive), 61 | Vec(Vec), 62 | Usize(usize), 63 | Ignore, 64 | } 65 | 66 | impl Input { 67 | // Check if value is safe to useas an index for a given `Vec`'s length 68 | fn check(&self, length: usize) -> bool { 69 | match self { 70 | Input::Range(range) => *range.end() < length, 71 | Input::Usize(u) => *u < length, 72 | Input::Vec(v) => v.iter().all(|u| *u < length), 73 | Input::Ignore => false, 74 | } 75 | } 76 | } 77 | 78 | impl FromStr for Input { 79 | // `Infaillible` because if we can not parse input, `Self::Ignore` will be returned 80 | type Err = Infallible; 81 | 82 | /// Directly handling user input. 83 | /// All `0`s are ignored so that remove one in each cases can be safely done. 84 | fn from_str(s: &str) -> Result { 85 | // If s is empty or if s contains spaces and dashes, ignore it 86 | if s.is_empty() || (s.contains(' ') && s.contains('-')) { 87 | Ok(Self::Ignore) 88 | } else if s.contains(' ') { 89 | // Parsing as a `Vec`, ignoring `0` 90 | let mut result: Vec = s 91 | .split(' ') 92 | .filter_map(|s| match s.parse() { 93 | Ok(u) if u != 0 => Some(u), 94 | _ => None, 95 | }) 96 | .collect(); 97 | if result.len() == 1 { 98 | // If only one element, return it as a `usize` minus one 99 | Ok(Self::Usize(result[0] - 1)) 100 | } else if !result.is_empty() { 101 | // Sorting the `Vec` and remove one at each `usize` 102 | result.sort_unstable(); 103 | result.iter_mut().for_each(|u| *u -= 1); 104 | Ok(Self::Vec(result)) 105 | } else { 106 | Ok(Self::Ignore) 107 | } 108 | } else if s.contains('-') { 109 | // Parsing as a `Vec`, ignoring `0` 110 | let bounds: Vec = s 111 | .split('-') 112 | .filter_map(|s| match s.parse::() { 113 | Ok(u) if u != 0 => Some(u), 114 | _ => None, 115 | }) 116 | .collect(); 117 | if bounds.len() == 2 { 118 | // If we have exactly two elements 119 | let start = bounds[0] - 1; 120 | let end = bounds[1] - 1; 121 | match start.cmp(&end) { 122 | // x < y => x..=y 123 | Ordering::Less => Ok(Self::Range(RangeInclusive::new(start, end))), 124 | // x == y => x 125 | Ordering::Equal => Ok(Self::Usize(start)), 126 | Ordering::Greater => Ok(Self::Ignore), 127 | } 128 | } else { 129 | Ok(Self::Ignore) 130 | } 131 | } else { 132 | // Trying to parse it as a single `usize` different from `0`, otherwise we ignore it 133 | s.parse::().map_or(Ok(Self::Ignore), |u| { 134 | if u == 0 { 135 | Ok(Self::Ignore) 136 | } else { 137 | // Removing one 138 | Ok(Self::Usize(u - 1)) 139 | } 140 | }) 141 | } 142 | } 143 | } 144 | 145 | pub(crate) async fn verify_mirror( 146 | path: std::path::PathBuf, 147 | current_step: &mut usize, 148 | steps: usize, 149 | vendor_path: Option, 150 | cargo_lock_filepath: Option, 151 | ) -> Result>, MirrorError> { 152 | // Checking existence of local index 153 | let repo_path = path.join("crates.io-index"); 154 | 155 | if !repo_path.join(".git").exists() { 156 | eprintln!("No index repository found in {}.", repo_path.display()) 157 | } 158 | 159 | let prefix = padded_prefix_message( 160 | *current_step, 161 | steps, 162 | "Comparing local crates.io and mirror coherence", 163 | ); 164 | 165 | let pb = ProgressBar::new_spinner() 166 | .with_style( 167 | ProgressStyle::default_bar() 168 | .template("{prefix} {wide_bar} {spinner} [{elapsed_precise}]") 169 | .expect("Something went wrong with the template.") 170 | .progress_chars(" "), 171 | ) 172 | .with_prefix(prefix) 173 | .with_finish(ProgressFinish::AndLeave); 174 | pb.enable_steady_tick(Duration::from_millis(10)); 175 | 176 | // Getting diff tree from local crates.io repository. 177 | let repo = Repository::open(repo_path)?; 178 | let master = repo.find_reference("refs/heads/master")?; 179 | let master_tree = master.peel_to_tree()?; 180 | let diff = repo.diff_tree_to_tree(None, Some(&master_tree), None)?; 181 | 182 | let mut missing_crates = Vec::new(); 183 | 184 | let is_crate_whitelist_only = vendor_path.is_some() || cargo_lock_filepath.is_some(); 185 | // if a vendor_path, parse the filepath for Cargo.toml files for each crate, filling vendors 186 | let mut mirror_entries = vec![]; 187 | vendor_path_to_mirror_entries(&mut mirror_entries, vendor_path.as_ref()); 188 | cargo_lock_to_mirror_entries(&mut mirror_entries, cargo_lock_filepath.as_ref()); 189 | 190 | diff.foreach( 191 | &mut |delta, _| { 192 | let df = delta.new_file(); 193 | let p = df.path().unwrap(); 194 | if p == Path::new("config.json") { 195 | return true; 196 | } 197 | if p.starts_with(".github/") { 198 | return true; 199 | } 200 | 201 | let oid = df.id(); 202 | if oid.is_zero() { 203 | return true; 204 | } 205 | let blob = repo.find_blob(oid).unwrap(); 206 | let data = blob.content(); 207 | 208 | // Iterating over each line of a JSON file from local crates.io repository 209 | for line in Cursor::new(data).lines() { 210 | let line = line.unwrap(); 211 | let crate_entry: CrateEntry = match serde_json::from_str(&line) { 212 | Ok(c) => c, 213 | Err(_) => { 214 | continue; 215 | } 216 | }; 217 | 218 | // Checking only whitelisted crates if supplied 219 | if is_crate_whitelist_only 220 | && !mirror_entries.iter().any(|it| { 221 | it.get_name() == crate_entry.get_name() 222 | && it.get_vers() == crate_entry.get_vers() 223 | }) 224 | { 225 | continue; 226 | } 227 | 228 | // Building crates local path. 229 | let file_path = 230 | get_crate_path(&path, crate_entry.get_name(), crate_entry.get_vers()).unwrap(); 231 | 232 | // Checking if crate is missing. 233 | if !CRATES_403 234 | .iter() 235 | .any(|it| it.0 == crate_entry.get_name() && it.1 == crate_entry.get_vers()) 236 | && !file_path.exists() 237 | { 238 | missing_crates.push(crate_entry); 239 | } 240 | } 241 | 242 | true 243 | }, 244 | None, 245 | None, 246 | None, 247 | )?; 248 | 249 | pb.finish(); 250 | *current_step += 1; 251 | 252 | if !missing_crates.is_empty() { 253 | return Ok(Some(missing_crates)); 254 | } 255 | 256 | eprintln!("{}", style("Verification successful.").bold()); 257 | 258 | Ok(None) 259 | } 260 | 261 | /// This method is giving choice to users whether to filter some crates or not before downloading. 262 | pub(crate) async fn handle_user_input( 263 | mut missing_crates: Vec, 264 | ) -> Result, MirrorError> { 265 | println!("Found {} missing crates:", missing_crates.len()); 266 | missing_crates.iter().enumerate().for_each(|(i, c)| { 267 | println!( 268 | " {}: {} - version {}", 269 | // Adding one to index here to start presenting to users from `1..=missing_crates.len()` 270 | style((i + 1).to_string()).bold(), 271 | c.get_name(), 272 | c.get_vers() 273 | ); 274 | }); 275 | println!("{}", 276 | style("Missing crates to download (e.g.: '1 2 3' or '1-3') [Leave empty for downloading all of them]:").bold() 277 | ); 278 | std::io::stdout().flush()?; 279 | let mut input = String::new(); 280 | match std::io::stdin().read_line(&mut input)? { 281 | // Handling EOF 282 | 0 => Ok(missing_crates), 283 | _ => { 284 | // Popping '\n' 285 | input.pop(); 286 | // Safe to unwrap here 287 | let input = input.parse::().unwrap(); 288 | if input.check(missing_crates.len()) { 289 | // Input is not respecting `Vec` bounds, ignoring request 290 | Ok(Vec::new()) 291 | } else { 292 | match input { 293 | Input::Ignore => Ok(missing_crates), 294 | Input::Range(range) => { 295 | range.into_iter().for_each(|u| { 296 | missing_crates.remove(u); 297 | }); 298 | Ok(missing_crates) 299 | } 300 | Input::Usize(u) => Ok(vec![missing_crates.remove(u)]), 301 | Input::Vec(v) => { 302 | v.into_iter().for_each(|u| { 303 | missing_crates.remove(u); 304 | }); 305 | Ok(missing_crates) 306 | } 307 | } 308 | } 309 | } 310 | } 311 | } 312 | 313 | /// This method is cactually fixing mirror by downloading missing crates. 314 | pub(crate) async fn fix_mirror( 315 | mirror_config: &ConfigMirror, 316 | crates_config: &ConfigCrates, 317 | path: PathBuf, 318 | crates_to_fetch: Vec, 319 | current_step: &mut usize, 320 | steps: usize, 321 | ) -> Result<(), MirrorError> { 322 | let prefix = padded_prefix_message(*current_step, steps, "Repairing mirror"); 323 | 324 | let pb = ProgressBar::new(crates_to_fetch.len() as u64) 325 | .with_style( 326 | ProgressStyle::default_bar() 327 | .template( 328 | "{prefix} {wide_bar} {pos}/{len} [{elapsed_precise} / {duration_precise}]", 329 | ) 330 | .expect("Something went wrong with the template.") 331 | .progress_chars("█▉▊▋▌▍▎▏ "), 332 | ) 333 | .with_prefix(prefix) 334 | .with_finish(ProgressFinish::AndLeave); 335 | pb.enable_steady_tick(Duration::from_millis(10)); 336 | 337 | // Getting crates' source from config 338 | let crates_source = if crates_config.source != "https://crates.io/api/v1/crates" { 339 | Some(crates_config.source.as_str()) 340 | } else { 341 | None 342 | }; 343 | 344 | // Handle the contact information 345 | let user_agent_str = 346 | mirror_config 347 | .contact 348 | .as_ref() 349 | .map_or_else(default_user_agent, |contact| { 350 | if contact != "your@email.com" { 351 | format!("Panamax/{} ({})", env!("CARGO_PKG_VERSION"), contact) 352 | } else { 353 | default_user_agent() 354 | } 355 | }); 356 | 357 | // Set the user agent with contact information. 358 | let user_agent = match HeaderValue::from_str(&user_agent_str) { 359 | Ok(h) => h, 360 | Err(e) => { 361 | eprintln!("Your contact information contains invalid characters!"); 362 | eprintln!("It's recommended to use a URL or email address as contact information."); 363 | eprintln!("{e:?}"); 364 | return Ok(()); 365 | } 366 | }; 367 | 368 | let client = Client::new(); 369 | 370 | // This code is copied from `crates::sync_crates_files` and could be mutualised in a future commit. 371 | // For example in a function within module crates (e.g. `crates::build_and_run_tasks`) 372 | let tasks = futures::stream::iter(crates_to_fetch.into_iter()) 373 | .map(|c| { 374 | // Duplicate variables used in the async closure. 375 | let client = client.clone(); 376 | let path = path.clone(); 377 | let mirror_retries = mirror_config.retries; 378 | let crates_source = crates_source.map(|s| s.to_string()); 379 | let user_agent = user_agent.to_owned(); 380 | let pb = pb.clone(); 381 | 382 | tokio::spawn(async move { 383 | let out = sync_one_crate_entry( 384 | &client, 385 | &path, 386 | crates_source.as_deref(), 387 | mirror_retries, 388 | &c, 389 | &user_agent, 390 | ) 391 | .await; 392 | 393 | pb.inc(1); 394 | 395 | out 396 | }) 397 | }) 398 | .buffer_unordered(crates_config.download_threads) 399 | .collect::>() 400 | .await; 401 | 402 | for t in tasks { 403 | let res = t.unwrap(); 404 | match res { 405 | Ok(()) 406 | | Err(DownloadError::NotFound { 407 | status: _, 408 | url: _, 409 | data: _, 410 | }) 411 | | Err(DownloadError::MismatchedHash { 412 | expected: _, 413 | actual: _, 414 | }) => {} 415 | 416 | Err(e) => { 417 | eprintln!("Downloading failed: {e:?}"); 418 | } 419 | } 420 | } 421 | 422 | pb.finish_and_clear(); 423 | *current_step += 1; 424 | Ok(()) 425 | } 426 | 427 | #[cfg(test)] 428 | mod test { 429 | 430 | mod input { 431 | use crate::verify::Input; 432 | 433 | #[test] 434 | fn true_range() { 435 | let input = "1-5".to_string(); 436 | let expected_result = Input::Range(0usize..=4); 437 | let result = input.parse::().unwrap(); 438 | assert_eq!(expected_result, result); 439 | } 440 | 441 | #[test] 442 | fn false_range_true_usize() { 443 | let input = "1-1".to_string(); 444 | let expected_result = Input::Usize(0); 445 | let result = input.parse::().unwrap(); 446 | assert_eq!(expected_result, result); 447 | } 448 | 449 | #[test] 450 | fn garbage_range_1() { 451 | let input = "foo-bar".to_string(); 452 | let expected_result = Input::Ignore; 453 | let result = input.parse::().unwrap(); 454 | assert_eq!(expected_result, result); 455 | } 456 | 457 | #[test] 458 | fn garbage_range_2() { 459 | let input = "1-5 7".to_string(); 460 | let expected_result = Input::Ignore; 461 | let result = input.parse::().unwrap(); 462 | assert_eq!(expected_result, result); 463 | } 464 | 465 | #[test] 466 | fn garbage_range_3() { 467 | let input = "1-foo".to_string(); 468 | let expected_result = Input::Ignore; 469 | let result = input.parse::().unwrap(); 470 | assert_eq!(expected_result, result); 471 | } 472 | 473 | #[test] 474 | fn garbage_range_4() { 475 | let input = "5-1".to_string(); 476 | let expected_result = Input::Ignore; 477 | let result = input.parse::().unwrap(); 478 | assert_eq!(expected_result, result); 479 | } 480 | 481 | #[test] 482 | fn garbage_range_5() { 483 | let input = "0-2".to_string(); 484 | let expected_result = Input::Ignore; 485 | let result = input.parse::().unwrap(); 486 | assert_eq!(expected_result, result); 487 | } 488 | 489 | #[test] 490 | fn garbage_range_6() { 491 | let input = "0-0".to_string(); 492 | let expected_result = Input::Ignore; 493 | let result = input.parse::().unwrap(); 494 | assert_eq!(expected_result, result); 495 | } 496 | 497 | #[test] 498 | fn true_vec() { 499 | let input = "1 2 5 9".to_string(); 500 | let expected_result = Input::Vec(vec![0, 1, 4, 8]); 501 | let result = input.parse::().unwrap(); 502 | assert_eq!(expected_result, result); 503 | } 504 | 505 | #[test] 506 | fn true_vec_shuffled() { 507 | let input = "6 4 8 2".to_string(); 508 | let expected_result = Input::Vec(vec![1, 3, 5, 7]); 509 | let result = input.parse::().unwrap(); 510 | assert_eq!(expected_result, result); 511 | } 512 | 513 | #[test] 514 | fn garbage_vec() { 515 | let input = "foo bar fubar".to_string(); 516 | let expected_result = Input::Ignore; 517 | let result = input.parse::().unwrap(); 518 | assert_eq!(expected_result, result); 519 | } 520 | 521 | #[test] 522 | fn some_garbage_vec_1() { 523 | let input = "1 bar 6".to_string(); 524 | let expected_result = Input::Vec(vec![0, 5]); 525 | let result = input.parse::().unwrap(); 526 | assert_eq!(expected_result, result); 527 | } 528 | 529 | #[test] 530 | fn some_garbage_vec_2() { 531 | let input = "0 2 6".to_string(); 532 | let expected_result = Input::Vec(vec![1, 5]); 533 | let result = input.parse::().unwrap(); 534 | assert_eq!(expected_result, result); 535 | } 536 | 537 | #[test] 538 | fn some_garbage_vec_3() { 539 | let input = "4 0 2".to_string(); 540 | let expected_result = Input::Vec(vec![1, 3]); 541 | let result = input.parse::().unwrap(); 542 | assert_eq!(expected_result, result); 543 | } 544 | 545 | #[test] 546 | fn true_usize() { 547 | let input = "42".to_string(); 548 | let expected_result = Input::Usize(41); 549 | let result = input.parse::().unwrap(); 550 | assert_eq!(expected_result, result); 551 | } 552 | 553 | #[test] 554 | fn garbage_usize_1() { 555 | let input = "foo".to_string(); 556 | let expected_result = Input::Ignore; 557 | let result = input.parse::().unwrap(); 558 | assert_eq!(expected_result, result); 559 | } 560 | 561 | #[test] 562 | fn garbage_usize_2() { 563 | let input = "0".to_string(); 564 | let expected_result = Input::Ignore; 565 | let result = input.parse::().unwrap(); 566 | assert_eq!(expected_result, result); 567 | } 568 | 569 | #[test] 570 | fn full_garbage() { 571 | let input = "1-3 42".to_string(); 572 | let expected_result = Input::Ignore; 573 | let result = input.parse::().unwrap(); 574 | assert_eq!(expected_result, result); 575 | } 576 | } 577 | } 578 | -------------------------------------------------------------------------------- /src/rustup.rs: -------------------------------------------------------------------------------- 1 | use crate::download::{ 2 | append_to_path, copy_file_create_dir_with_sha256, download, download_string, 3 | download_with_sha256_file, move_if_exists, move_if_exists_with_sha256, write_file_create_dir, 4 | DownloadError, 5 | }; 6 | use crate::mirror::{ConfigMirror, ConfigRustup, MirrorError}; 7 | use crate::progress_bar::{current_step_prefix, padded_prefix_message}; 8 | use console::style; 9 | use futures::StreamExt; 10 | use indicatif::{ProgressBar, ProgressFinish, ProgressStyle}; 11 | use reqwest::header::HeaderValue; 12 | use reqwest::Client; 13 | use serde::{Deserialize, Serialize}; 14 | use std::collections::{HashMap, HashSet}; 15 | use std::path::{Path, PathBuf}; 16 | use std::time::Duration; 17 | use std::{fs, io}; 18 | use thiserror::Error; 19 | use tokio::task::JoinError; 20 | 21 | // The allowed platforms to validate the configuration 22 | // Note: These platforms should match the list on https://rust-lang.github.io/rustup/installation/other.html 23 | 24 | /// Windows platforms (platforms where rustup-init has a .exe extension) 25 | static PLATFORMS_WINDOWS: &[&str] = &[ 26 | "i586-pc-windows-msvc", 27 | "i686-pc-windows-gnu", 28 | "i686-pc-windows-msvc", 29 | "x86_64-pc-windows-gnu", 30 | "x86_64-pc-windows-msvc", 31 | ]; 32 | 33 | #[derive(Error, Debug)] 34 | pub enum SyncError { 35 | #[error("IO error: {0}")] 36 | Io(#[from] io::Error), 37 | 38 | #[error("Download error: {0}")] 39 | Download(#[from] DownloadError), 40 | 41 | #[error("TOML deserialization error: {0}")] 42 | Parse(#[from] toml_edit::de::Error), 43 | 44 | #[error("TOML serialization error: {0}")] 45 | Serialize(#[from] toml_edit::ser::Error), 46 | 47 | #[error("Path prefix strip error: {0}")] 48 | StripPrefix(#[from] std::path::StripPrefixError), 49 | 50 | #[error("Failed {count} downloads")] 51 | FailedDownloads { count: usize }, 52 | } 53 | 54 | #[derive(Deserialize, Debug)] 55 | pub struct TargetUrls { 56 | pub url: String, 57 | pub hash: String, 58 | pub xz_url: String, 59 | pub xz_hash: String, 60 | } 61 | 62 | #[derive(Deserialize, Debug)] 63 | pub struct Target { 64 | pub available: bool, 65 | 66 | #[serde(flatten)] 67 | pub target_urls: Option, 68 | } 69 | 70 | #[derive(Deserialize, Debug)] 71 | pub struct Pkg { 72 | pub version: String, 73 | pub target: HashMap, 74 | } 75 | 76 | #[derive(Deserialize, Debug)] 77 | pub struct Channel { 78 | #[serde(alias = "manifest-version")] 79 | pub manifest_version: String, 80 | pub date: String, 81 | pub pkg: HashMap, 82 | } 83 | 84 | #[derive(Deserialize, Debug)] 85 | struct Release { 86 | version: String, 87 | } 88 | 89 | #[derive(Deserialize, Debug)] 90 | pub struct Platforms { 91 | unix: Vec, 92 | windows: Vec, 93 | } 94 | 95 | impl Platforms { 96 | // &String instead of &str is required due to vec.contains not performing proper inference 97 | // here. See: 98 | // https://stackoverflow.com/questions/48985924/why-does-a-str-not-coerce-to-a-string-when-using-veccontains 99 | // https://github.com/rust-lang/rust/issues/42671 100 | #[allow(clippy::ptr_arg)] 101 | pub fn contains(&self, platform: &String) -> bool { 102 | self.unix.contains(platform) || self.windows.contains(platform) 103 | } 104 | 105 | pub fn len(&self) -> usize { 106 | self.unix.len() + self.windows.len() 107 | } 108 | } 109 | 110 | pub async fn download_platform_list( 111 | source: &str, 112 | channel: &str, 113 | ) -> Result, MirrorError> { 114 | let channel_url = format!("{source}/dist/channel-rust-{channel}.toml"); 115 | let user_agent = HeaderValue::from_str(&format!("Panamax/{}", env!("CARGO_PKG_VERSION"))) 116 | .expect("Hardcoded user agent string should never fail."); 117 | let channel_str = download_string(&channel_url, &user_agent).await?; 118 | let channel_data: Channel = toml_edit::easy::from_str(&channel_str)?; 119 | 120 | let mut targets = HashSet::new(); 121 | 122 | for (_, pkg) in channel_data.pkg { 123 | for (target, _) in pkg.target { 124 | if target == "*" { 125 | continue; 126 | } 127 | targets.insert(target); 128 | } 129 | } 130 | 131 | let mut targets: Vec = targets.into_iter().collect(); 132 | targets.sort(); 133 | 134 | Ok(targets) 135 | } 136 | 137 | pub async fn get_platforms(rustup: &ConfigRustup) -> Result { 138 | let all = download_platform_list(&rustup.source, "nightly").await?; 139 | 140 | let unix = match &rustup.platforms_unix { 141 | Some(p) => p.clone(), 142 | None => all 143 | .iter() 144 | .filter(|x| !PLATFORMS_WINDOWS.contains(&x.as_str())) 145 | .map(|x| x.to_string()) 146 | .collect(), 147 | }; 148 | 149 | let windows = match &rustup.platforms_windows { 150 | Some(p) => p.clone(), 151 | None => PLATFORMS_WINDOWS.iter().map(|x| x.to_string()).collect(), 152 | }; 153 | Ok(Platforms { unix, windows }) 154 | } 155 | 156 | /// Synchronize one rustup-init file. 157 | #[allow(clippy::too_many_arguments)] 158 | pub async fn sync_one_init( 159 | client: &Client, 160 | path: &Path, 161 | source: &str, 162 | platform: &str, 163 | is_exe: bool, 164 | rustup_version: &str, 165 | retries: usize, 166 | user_agent: &HeaderValue, 167 | ) -> Result<(), DownloadError> { 168 | let local_path = path 169 | .join("rustup") 170 | .join("archive") 171 | .join(rustup_version) 172 | .join(platform) 173 | .join(if is_exe { 174 | "rustup-init.exe" 175 | } else { 176 | "rustup-init" 177 | }); 178 | 179 | let archive_path = path.join("rustup/dist").join(platform).join(if is_exe { 180 | "rustup-init.exe" 181 | } else { 182 | "rustup-init" 183 | }); 184 | 185 | let source_url = if is_exe { 186 | format!("{source}/rustup/dist/{platform}/rustup-init.exe") 187 | } else { 188 | format!("{source}/rustup/dist/{platform}/rustup-init") 189 | }; 190 | 191 | download_with_sha256_file(client, &source_url, &local_path, retries, false, user_agent).await?; 192 | copy_file_create_dir_with_sha256(&local_path, &archive_path)?; 193 | 194 | Ok(()) 195 | } 196 | 197 | fn panamax_progress_bar(size: usize, prefix: String) -> ProgressBar { 198 | ProgressBar::new(size as u64) 199 | .with_style( 200 | ProgressStyle::default_bar() 201 | .template( 202 | "{prefix} {wide_bar} {pos}/{len} [{elapsed_precise} / {duration_precise}]", 203 | ) 204 | .expect("template is correct") 205 | .progress_chars("█▉▊▋▌▍▎▏ "), 206 | ) 207 | .with_finish(ProgressFinish::AndLeave) 208 | .with_prefix(prefix) 209 | } 210 | 211 | #[allow(clippy::too_many_arguments)] 212 | async fn create_sync_tasks( 213 | platforms: &[String], 214 | is_exe: bool, 215 | rustup_version: &str, 216 | path: &Path, 217 | source: &str, 218 | retries: usize, 219 | user_agent: &HeaderValue, 220 | threads: usize, 221 | pb: &ProgressBar, 222 | ) -> Vec, JoinError>> { 223 | let client = Client::new(); 224 | futures::stream::iter(platforms.iter()) 225 | .map(|platform| { 226 | let client = client.clone(); 227 | let rustup_version = rustup_version.to_string(); 228 | let path = path.to_path_buf(); 229 | let source = source.to_string(); 230 | let user_agent = user_agent.clone(); 231 | let platform = platform.clone(); 232 | let pb = pb.clone(); 233 | 234 | tokio::spawn(async move { 235 | let out = sync_one_init( 236 | &client, 237 | &path, 238 | &source, 239 | platform.as_str(), 240 | is_exe, 241 | &rustup_version, 242 | retries, 243 | &user_agent, 244 | ) 245 | .await; 246 | 247 | pb.inc(1); 248 | 249 | out 250 | }) 251 | }) 252 | .buffer_unordered(threads) 253 | .collect::>>() 254 | .await 255 | } 256 | 257 | /// Synchronize all rustup-init files. 258 | pub async fn sync_rustup_init( 259 | path: &Path, 260 | threads: usize, 261 | source: &str, 262 | prefix: String, 263 | retries: usize, 264 | user_agent: &HeaderValue, 265 | platforms: &Platforms, 266 | ) -> Result<(), SyncError> { 267 | let mut errors_occurred = 0usize; 268 | 269 | let client = Client::new(); 270 | 271 | // Download rustup release file 272 | let release_url = format!("{source}/rustup/release-stable.toml"); 273 | let release_path = path.join("rustup/release-stable.toml"); 274 | let release_part_path = append_to_path(&release_path, ".part"); 275 | 276 | download( 277 | &client, 278 | &release_url, 279 | &release_part_path, 280 | None, 281 | retries, 282 | false, 283 | user_agent, 284 | ) 285 | .await?; 286 | 287 | let rustup_version = get_rustup_version(&release_part_path)?; 288 | 289 | move_if_exists(&release_part_path, &release_path)?; 290 | 291 | let pb = panamax_progress_bar(platforms.len(), prefix); 292 | pb.enable_steady_tick(Duration::from_millis(10)); 293 | 294 | let unix_tasks = create_sync_tasks( 295 | &platforms.unix, 296 | false, 297 | &rustup_version, 298 | path, 299 | source, 300 | retries, 301 | user_agent, 302 | threads, 303 | &pb, 304 | ) 305 | .await; 306 | 307 | let win_tasks = create_sync_tasks( 308 | &platforms.windows, 309 | true, 310 | &rustup_version, 311 | path, 312 | source, 313 | retries, 314 | user_agent, 315 | threads, 316 | &pb, 317 | ) 318 | .await; 319 | 320 | for res in unix_tasks.into_iter().chain(win_tasks) { 321 | // Unwrap the join result. 322 | let res = res.unwrap(); 323 | 324 | if let Err(e) = res { 325 | match e { 326 | DownloadError::NotFound { .. } => {} 327 | _ => { 328 | errors_occurred += 1; 329 | eprintln!("Download failed: {e:?}"); 330 | } 331 | } 332 | } 333 | } 334 | 335 | if errors_occurred == 0 { 336 | Ok(()) 337 | } else { 338 | Err(SyncError::FailedDownloads { 339 | count: errors_occurred, 340 | }) 341 | } 342 | } 343 | 344 | /// Get the rustup file downloads, in pairs of URLs and sha256 hashes. 345 | pub fn rustup_download_list( 346 | path: &Path, 347 | download_dev: bool, 348 | download_gz: bool, 349 | download_xz: bool, 350 | platforms: &Platforms, 351 | ) -> Result<(String, Vec<(String, String)>), SyncError> { 352 | let channel_str = fs::read_to_string(path).map_err(DownloadError::Io)?; 353 | let channel: Channel = toml_edit::easy::from_str(&channel_str)?; 354 | 355 | Ok(( 356 | channel.date, 357 | channel 358 | .pkg 359 | .into_iter() 360 | .filter(|(pkg_name, _)| download_dev || pkg_name != "rustc-dev") 361 | .flat_map(|(_, pkg)| { 362 | pkg.target 363 | .into_iter() 364 | .filter( 365 | |(name, _)| platforms.contains(name) || name == "*", // The * platform contains rust-src, always download 366 | ) 367 | .flat_map(|(_, target)| -> Vec<(String, String)> { 368 | target 369 | .target_urls 370 | .map(|urls| { 371 | let mut v = Vec::new(); 372 | if download_gz { 373 | v.push((urls.url, urls.hash)); 374 | } 375 | if download_xz { 376 | v.push((urls.xz_url, urls.xz_hash)); 377 | } 378 | 379 | v 380 | }) 381 | .into_iter() 382 | .flatten() 383 | .map(|(url, hash)| { 384 | (url.split('/').collect::>()[3..].join("/"), hash) 385 | }) 386 | .collect() 387 | }) 388 | }) 389 | .collect(), 390 | )) 391 | } 392 | 393 | pub async fn sync_one_rustup_target( 394 | client: &Client, 395 | path: &Path, 396 | source: &str, 397 | url: &str, 398 | hash: &str, 399 | retries: usize, 400 | user_agent: &HeaderValue, 401 | ) -> Result<(), DownloadError> { 402 | // Chop off the source portion of the URL, to mimic the rest of the path 403 | //let target_url = path.join(url[source.len()..].trim_start_matches("/")); 404 | let target_url = format!("{source}/{url}"); 405 | let target_path: PathBuf = std::iter::once(path.to_owned()) 406 | .chain(url.split('/').map(PathBuf::from)) 407 | .collect(); 408 | 409 | download( 410 | client, 411 | &target_url, 412 | &target_path, 413 | Some(hash), 414 | retries, 415 | false, 416 | user_agent, 417 | ) 418 | .await 419 | } 420 | 421 | #[derive(Debug, Serialize, Deserialize)] 422 | pub struct ChannelHistoryFile { 423 | pub versions: HashMap>, 424 | } 425 | 426 | pub fn latest_dates_from_channel_history( 427 | channel_history: &ChannelHistoryFile, 428 | versions: usize, 429 | ) -> Vec { 430 | let mut dates: Vec = channel_history 431 | .versions 432 | .keys() 433 | .map(|x| x.to_string()) 434 | .collect(); 435 | dates.sort(); 436 | dates.reverse(); 437 | dates.truncate(versions); 438 | dates 439 | } 440 | 441 | pub fn clean_old_files( 442 | path: &Path, 443 | keep_stables: Option, 444 | keep_betas: Option, 445 | keep_nightlies: Option, 446 | pinned_rust_versions: Option<&Vec>, 447 | prefix: String, 448 | ) -> Result<(), SyncError> { 449 | let versions = [ 450 | ("stable", keep_stables), 451 | ("beta", keep_betas), 452 | ("nightly", keep_nightlies), 453 | ]; 454 | 455 | // Handle all of stable/beta/nightly 456 | let mut files_to_keep: HashSet = HashSet::new(); 457 | for (channel, keep_version) in versions { 458 | if let Some(s) = keep_version { 459 | let mut history = match get_channel_history(path, channel) { 460 | Ok(c) => c, 461 | Err(_) => continue, 462 | }; 463 | let latest_dates = latest_dates_from_channel_history(&history, s); 464 | for date in latest_dates { 465 | if let Some(t) = history.versions.get_mut(&date) { 466 | t.iter().for_each(|t| { 467 | // Convert the path to a PathBuf. 468 | let path: PathBuf = t.split('/').collect(); 469 | files_to_keep.insert(path); 470 | }); 471 | } 472 | } 473 | } 474 | } 475 | 476 | if let Some(pinned_versions) = pinned_rust_versions { 477 | for version in pinned_versions { 478 | let mut pinned = match get_channel_history(path, version) { 479 | Ok(c) => c, 480 | Err(_) => continue, 481 | }; 482 | let latest_dates = latest_dates_from_channel_history(&pinned, 1); 483 | for date in latest_dates { 484 | if let Some(t) = pinned.versions.get_mut(&date) { 485 | t.iter().for_each(|t| { 486 | // Convert the path to a PathBuf. 487 | let path: PathBuf = t.split('/').collect(); 488 | 489 | files_to_keep.insert(path); 490 | }); 491 | } 492 | } 493 | } 494 | } 495 | 496 | let dist_path = path.join("dist"); 497 | let mut files_to_delete = Vec::new(); 498 | 499 | for dir in fs::read_dir(dist_path)? { 500 | let dir = dir?.path(); 501 | let dir = dir.as_path(); 502 | if dir.is_dir() { 503 | for full_path in fs::read_dir(dir)? { 504 | let full_path = full_path?.path(); 505 | let file_path = full_path.strip_prefix(path)?; 506 | 507 | if !files_to_keep.contains(file_path) { 508 | files_to_delete.push(file_path.to_owned()); 509 | } 510 | } 511 | 512 | // Remove directory if empty. 513 | if dir.read_dir()?.next().is_none() { 514 | fs::remove_dir(dir)?; 515 | } 516 | } 517 | } 518 | 519 | // Progress bar! 520 | let pb = panamax_progress_bar(files_to_delete.len(), prefix); 521 | 522 | for f in files_to_delete { 523 | if let Err(e) = fs::remove_file(path.join(&f)) { 524 | eprintln!("Could not remove file {}: {:?}", f.to_string_lossy(), e); 525 | } 526 | pb.inc(1); 527 | } 528 | 529 | Ok(()) 530 | } 531 | 532 | pub fn get_channel_history(path: &Path, channel: &str) -> Result { 533 | let channel_history_path = path.join(format!("mirror-{channel}-history.toml")); 534 | let ch_data = fs::read_to_string(channel_history_path)?; 535 | Ok(toml_edit::easy::from_str(&ch_data)?) 536 | } 537 | 538 | pub fn add_to_channel_history( 539 | path: &Path, 540 | channel: &str, 541 | date: &str, 542 | files: &[(String, String)], 543 | extra_files: &[String], 544 | ) -> Result<(), SyncError> { 545 | let mut channel_history = match get_channel_history(path, channel) { 546 | Ok(c) => c, 547 | Err(SyncError::Io(_)) => ChannelHistoryFile { 548 | versions: HashMap::new(), 549 | }, 550 | Err(e) => return Err(e), 551 | }; 552 | 553 | let files = files.iter().map(|(f, _)| f.to_string()); 554 | let extra_files = extra_files.iter().map(|ef| ef.to_string()); 555 | 556 | let files = files.chain(extra_files).collect(); 557 | 558 | channel_history.versions.insert(date.to_string(), files); 559 | 560 | let ch_data = toml_edit::ser::to_string(&channel_history)?; 561 | 562 | let channel_history_path = path.join(format!("mirror-{channel}-history.toml")); 563 | write_file_create_dir(&channel_history_path, &ch_data)?; 564 | 565 | Ok(()) 566 | } 567 | 568 | /// Get the current rustup version from release-stable.toml. 569 | pub fn get_rustup_version(path: &Path) -> Result { 570 | let release_data: Release = toml_edit::easy::from_str(&fs::read_to_string(path)?)?; 571 | Ok(release_data.version) 572 | } 573 | 574 | /// Synchronize a rustup channel (stable, beta, or nightly). 575 | #[allow(clippy::too_many_arguments)] 576 | pub async fn sync_rustup_channel( 577 | path: &Path, 578 | source: &str, 579 | threads: usize, 580 | prefix: String, 581 | channel: &str, 582 | retries: usize, 583 | user_agent: &HeaderValue, 584 | download_dev: bool, 585 | download_gz: bool, 586 | download_xz: bool, 587 | platforms: &Platforms, 588 | ) -> Result<(), SyncError> { 589 | // Download channel file 590 | let (channel_url, channel_path, extra_files) = 591 | if let Some(inner_channel) = channel.strip_prefix("nightly-") { 592 | let url = format!("{source}/dist/{inner_channel}/channel-rust-nightly.toml"); 593 | let path_chunk = format!("dist/{inner_channel}/channel-rust-nightly.toml"); 594 | let path = path.join(&path_chunk); 595 | // Make sure the cleanup step doesn't delete the channel toml 596 | let extra_files = vec![path_chunk.clone(), format!("{path_chunk}.sha256")]; 597 | (url, path, extra_files) 598 | } else { 599 | let url = format!("{source}/dist/channel-rust-{channel}.toml"); 600 | let path = path.join(format!("dist/channel-rust-{channel}.toml")); 601 | (url, path, Vec::new()) 602 | }; 603 | let channel_part_path = append_to_path(&channel_path, ".part"); 604 | let client = Client::new(); 605 | download_with_sha256_file( 606 | &client, 607 | &channel_url, 608 | &channel_part_path, 609 | retries, 610 | true, 611 | user_agent, 612 | ) 613 | .await?; 614 | 615 | // Open toml file, find all files to download 616 | let (date, files) = rustup_download_list( 617 | &channel_part_path, 618 | download_dev, 619 | download_gz, 620 | download_xz, 621 | platforms, 622 | )?; 623 | move_if_exists_with_sha256(&channel_part_path, &channel_path)?; 624 | 625 | let pb = panamax_progress_bar(files.len(), prefix); 626 | pb.enable_steady_tick(Duration::from_millis(10)); 627 | 628 | let mut errors_occurred = 0usize; 629 | 630 | let tasks = futures::stream::iter(files.iter()) 631 | .map(|(url, hash)| { 632 | // Clone the variables that will be moved into the tokio task. 633 | let client = client.clone(); 634 | let path = path.to_path_buf(); 635 | let source = source.to_string(); 636 | let user_agent = user_agent.clone(); 637 | let url = url.clone(); 638 | let hash = hash.clone(); 639 | let pb = pb.clone(); 640 | 641 | tokio::spawn(async move { 642 | let out = sync_one_rustup_target( 643 | &client, 644 | &path, 645 | &source, 646 | &url, 647 | &hash, 648 | retries, 649 | &user_agent, 650 | ) 651 | .await; 652 | 653 | pb.inc(1); 654 | 655 | out 656 | }) 657 | }) 658 | .buffer_unordered(threads) 659 | .collect::>() 660 | .await; 661 | 662 | for res in tasks { 663 | // Unwrap the join result. 664 | let res = res.unwrap(); 665 | 666 | if let Err(e) = res { 667 | match e { 668 | DownloadError::NotFound { .. } => {} 669 | _ => { 670 | errors_occurred += 1; 671 | eprintln!("Download failed: {e:?}"); 672 | } 673 | } 674 | } 675 | } 676 | 677 | if errors_occurred == 0 { 678 | // Write channel history file 679 | add_to_channel_history(path, channel, &date, &files, &extra_files)?; 680 | Ok(()) 681 | } else { 682 | Err(SyncError::FailedDownloads { 683 | count: errors_occurred, 684 | }) 685 | } 686 | } 687 | 688 | /// Synchronize rustup. 689 | pub async fn sync( 690 | path: &Path, 691 | mirror: &ConfigMirror, 692 | rustup: &ConfigRustup, 693 | user_agent: &HeaderValue, 694 | ) -> Result<(), MirrorError> { 695 | let platforms = get_platforms(rustup).await?; 696 | // Default to not downloading rustc-dev 697 | let download_dev = rustup.download_dev.unwrap_or(false); 698 | 699 | let download_gz = rustup.download_gz.unwrap_or(false); 700 | let download_xz = rustup.download_xz.unwrap_or(true); 701 | 702 | let num_pinned_versions = rustup.pinned_rust_versions.as_ref().map_or(0, |v| v.len()); 703 | let num_steps = 1 + // sync rustup-init 704 | 1 + 1 + 1 + // sync latest stable, beta, nightly 705 | num_pinned_versions + // sync pinned rust versions 706 | 1; // clean old files 707 | let mut step = 0; 708 | 709 | eprintln!("{}", style("Syncing Rustup repositories...").bold()); 710 | 711 | // Mirror rustup-init 712 | step += 1; 713 | let prefix = padded_prefix_message(step, num_steps, "Syncing rustup-init files"); 714 | if let Err(e) = sync_rustup_init( 715 | path, 716 | rustup.download_threads, 717 | &rustup.source, 718 | prefix, 719 | mirror.retries, 720 | user_agent, 721 | &platforms, 722 | ) 723 | .await 724 | { 725 | eprintln!("Downloading rustup init files failed: {e:?}"); 726 | eprintln!("You will need to sync again to finish this download."); 727 | } 728 | 729 | let mut failures = false; 730 | 731 | // Mirror stable 732 | step += 1; 733 | if rustup.keep_latest_stables != Some(0) { 734 | let prefix = padded_prefix_message(step, num_steps, "Syncing latest stable"); 735 | if let Err(e) = sync_rustup_channel( 736 | path, 737 | &rustup.source, 738 | rustup.download_threads, 739 | prefix, 740 | "stable", 741 | mirror.retries, 742 | user_agent, 743 | download_dev, 744 | download_gz, 745 | download_xz, 746 | &platforms, 747 | ) 748 | .await 749 | { 750 | failures = true; 751 | eprintln!("Downloading stable release failed: {e:?}"); 752 | eprintln!("You will need to sync again to finish this download."); 753 | } 754 | } else { 755 | eprintln!( 756 | "{} Skipping syncing stable.", 757 | current_step_prefix(step, num_steps) 758 | ); 759 | } 760 | 761 | // Mirror beta 762 | step += 1; 763 | if rustup.keep_latest_betas != Some(0) { 764 | let prefix = padded_prefix_message(step, num_steps, "Syncing latest beta"); 765 | if let Err(e) = sync_rustup_channel( 766 | path, 767 | &rustup.source, 768 | rustup.download_threads, 769 | prefix, 770 | "beta", 771 | mirror.retries, 772 | user_agent, 773 | download_dev, 774 | download_gz, 775 | download_xz, 776 | &platforms, 777 | ) 778 | .await 779 | { 780 | failures = true; 781 | eprintln!("Downloading beta release failed: {e:?}"); 782 | eprintln!("You will need to sync again to finish this download."); 783 | } 784 | } else { 785 | eprintln!( 786 | "{} Skipping syncing beta.", 787 | current_step_prefix(step, num_steps) 788 | ); 789 | } 790 | 791 | // Mirror nightly 792 | step += 1; 793 | if rustup.keep_latest_nightlies != Some(0) { 794 | let prefix = padded_prefix_message(step, num_steps, "Syncing latest nightly"); 795 | if let Err(e) = sync_rustup_channel( 796 | path, 797 | &rustup.source, 798 | rustup.download_threads, 799 | prefix, 800 | "nightly", 801 | mirror.retries, 802 | user_agent, 803 | download_dev, 804 | download_gz, 805 | download_xz, 806 | &platforms, 807 | ) 808 | .await 809 | { 810 | failures = true; 811 | eprintln!("Downloading nightly release failed: {e:?}"); 812 | eprintln!("You will need to sync again to finish this download."); 813 | } 814 | } else { 815 | eprintln!( 816 | "{} Skipping syncing nightly.", 817 | current_step_prefix(step, num_steps) 818 | ); 819 | } 820 | 821 | // Mirror pinned rust versions 822 | if let Some(pinned_versions) = &rustup.pinned_rust_versions { 823 | for version in pinned_versions { 824 | step += 1; 825 | let prefix = 826 | padded_prefix_message(step, num_steps, &format!("Syncing pinned rust {version}")); 827 | if let Err(e) = sync_rustup_channel( 828 | path, 829 | &rustup.source, 830 | rustup.download_threads, 831 | prefix, 832 | version, 833 | mirror.retries, 834 | user_agent, 835 | download_dev, 836 | download_gz, 837 | download_xz, 838 | &platforms, 839 | ) 840 | .await 841 | { 842 | failures = true; 843 | if let SyncError::Download(DownloadError::NotFound { .. }) = e { 844 | eprintln!( 845 | "{} Pinned rust version {} could not be found.", 846 | current_step_prefix(step, num_steps), 847 | version 848 | ); 849 | return Err(MirrorError::Config(format!( 850 | "Pinned rust version {version} could not be found" 851 | ))); 852 | } else { 853 | eprintln!("Downloading pinned rust {version} failed: {e:?}"); 854 | eprintln!("You will need to sync again to finish this download."); 855 | } 856 | } 857 | } 858 | } 859 | 860 | // If all succeeds, clean files 861 | step += 1; 862 | if rustup.keep_latest_stables.is_none() 863 | && rustup.keep_latest_betas.is_none() 864 | && rustup.keep_latest_nightlies.is_none() 865 | { 866 | eprintln!( 867 | "{} Skipping cleaning files.", 868 | current_step_prefix(step, num_steps) 869 | ); 870 | } else if failures { 871 | eprintln!( 872 | "{} Skipping cleaning files due to download failures.", 873 | current_step_prefix(step, num_steps) 874 | ); 875 | } else { 876 | let prefix = padded_prefix_message(step, num_steps, "Cleaning old files"); 877 | if let Err(e) = clean_old_files( 878 | path, 879 | rustup.keep_latest_stables, 880 | rustup.keep_latest_betas, 881 | rustup.keep_latest_nightlies, 882 | rustup.pinned_rust_versions.as_ref(), 883 | prefix, 884 | ) { 885 | eprintln!("Cleaning old files failed: {e:?}"); 886 | eprintln!("You may need to sync again to clean these files."); 887 | } 888 | } 889 | 890 | eprintln!("{}", style("Syncing Rustup repositories complete!").bold()); 891 | 892 | Ok(()) 893 | } 894 | --------------------------------------------------------------------------------