├── .github └── workflows │ └── ci.yaml ├── .gitignore ├── CODEOWNERS ├── Cargo.toml ├── LICENSE ├── README.md ├── REMOTE_SETUP.MD └── src ├── main.rs └── patches.rs /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: Build and audit 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | types: [opened, synchronize, reopened, ready_for_review] 9 | 10 | jobs: 11 | quick_check: 12 | strategy: 13 | matrix: 14 | os: ["ubuntu-latest"] 15 | runs-on: ${{ matrix.os }} 16 | steps: 17 | - name: Install Rust nightly toolchain 18 | uses: actions-rs/toolchain@b2417cde72dcf67f306c0ae8e0828a81bf0b189f # v1.0.7 19 | with: 20 | profile: minimal 21 | toolchain: stable 22 | override: true 23 | 24 | - name: Cache Dependencies & Build Outputs 25 | uses: actions/cache@v4.2.3 26 | with: 27 | path: | 28 | ~/.cargo/registry 29 | ~/.cargo/git 30 | target 31 | key: ${{ runner.os }}-${{ matrix.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} 32 | 33 | - name: Install cargo-audit 34 | uses: baptiste0928/cargo-install@bf6758885262d0e6f61089a9d8c8790d3ac3368f #v1.3.0 35 | with: 36 | crate: cargo-audit 37 | 38 | - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 39 | 40 | - name: Cargo build 41 | uses: actions-rs/cargo@ae10961054e4aa8b4aa7dffede299aaf087aa33b # v1.0.3 42 | with: 43 | command: build 44 | args: --release 45 | 46 | - name: Cargo audit 47 | uses: actions-rs/cargo@ae10961054e4aa8b4aa7dffede299aaf087aa33b # v1.0.3 48 | with: 49 | command: audit 50 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | /target/ 3 | **/*.rs.bk 4 | ### Rust template 5 | # Generated by Cargo 6 | # will have compiled files and executables 7 | 8 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 9 | # More information here http://doc.crates.io/guide.html#cargotoml-vs-cargolock 10 | Cargo.lock 11 | 12 | # These are backup files generated by rustfmt 13 | ### JetBrains template 14 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 15 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 16 | 17 | # User-specific stuff: 18 | .idea/**/workspace.xml 19 | .idea/**/tasks.xml 20 | .idea/dictionaries 21 | 22 | # Sensitive or high-churn files: 23 | .idea/**/dataSources/ 24 | .idea/**/dataSources.ids 25 | .idea/**/dataSources.xml 26 | .idea/**/dataSources.local.xml 27 | .idea/**/sqlDataSources.xml 28 | .idea/**/dynamic.xml 29 | .idea/**/uiDesigner.xml 30 | 31 | # Gradle: 32 | .idea/**/gradle.xml 33 | .idea/**/libraries 34 | 35 | # CMake 36 | cmake-build-debug/ 37 | 38 | # Mongo Explorer plugin: 39 | .idea/**/mongoSettings.xml 40 | 41 | ## File-based project format: 42 | *.iws 43 | 44 | 45 | # mpeltonen/sbt-idea plugin 46 | .idea_modules/ 47 | 48 | # JIRA plugin 49 | atlassian-ide-plugin.xml 50 | 51 | # Cursive Clojure plugin 52 | .idea/replstate.xml 53 | 54 | # Crashlytics plugin (for Android Studio and IntelliJ) 55 | com_crashlytics_export_strings.xml 56 | crashlytics.properties 57 | crashlytics-build.properties 58 | fabric.properties 59 | 60 | # project file 61 | cargo-remote.iml 62 | 63 | # test config file 64 | .cargo-remote.toml 65 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Lists some code owners. 2 | # 3 | # A codeowner just oversees some part of the codebase. If an owned file is changed then the 4 | # corresponding codeowner receives a review request. An approval of the codeowner might be 5 | # required for merging a PR (depends on repository settings). 6 | # 7 | # For details about syntax, see: 8 | # https://help.github.com/en/articles/about-code-owners 9 | # But here are some important notes: 10 | # 11 | # - Glob syntax is git-like, e.g. `/core` means the core directory in the root, unlike `core` 12 | # which can be everywhere. 13 | # - Multiple owners are supported. 14 | # - Either handle (e.g, @github_user or @github/team) or email can be used. Keep in mind, 15 | # that handles might work better because they are more recognizable on GitHub, 16 | # eyou can use them for mentioning unlike an email. 17 | # - The latest matching rule, if multiple, takes precedence. 18 | 19 | # CI 20 | /.github/ @paritytech/ci 21 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cargo-remote" 3 | description = "Cargo subcommand to build rust projects remotely" 4 | keywords = ["remote", "build"] 5 | categories = ["command-line-utilities", "development-tools::build-utils", "development-tools::cargo-plugins"] 6 | maintenance = { status = "experimental" } 7 | version = "0.1.99" 8 | authors = ["Sebastian Geisler "] 9 | license = "MIT" 10 | repository = "https://github.com/sgeisler/cargo-remote" 11 | edition = "2018" 12 | 13 | [dependencies] 14 | structopt = "0.2.18" 15 | cargo_metadata = "0.14.2" 16 | log = "0.4.1" 17 | simple_logger = "1.3.0" 18 | toml = "0.5.1" 19 | xdg = "2.1.0" 20 | toml_edit = "0.14.3" 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Sebastian Geisler 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cargo Remote 2 | 3 | ***Use with caution, I didn't test this software well and it is a really hacky 4 | (at least for now). If you want to test it please create a VM or at least a separate 5 | user on your build host*** 6 | 7 | ## Why I built it 8 | 9 | One big annoyance when working on rust projects on my notebook are the compile 10 | times. Since I'm using rust nightly for some of my projects I have to recompile 11 | rather often. Currently there seem to be no good remote-build integrations for 12 | rust, so I decided to build one my own. 13 | 14 | ## Planned capabilities 15 | 16 | This first version is very simple (could have been a bash script), but I intend to 17 | enhance it to a point where it detects compatibility between local and remote 18 | versions, allows (nearly) all cargo commands and maybe even load distribution 19 | over multiple machines. 20 | 21 | ## Usage 22 | 23 | For now only `cargo remote [FLAGS] [OPTIONS] ` works: it copies the 24 | current project to a temporary directory (`~/remote-builds/`) on 25 | the remote server, calls `cargo ` remotely and optionally (`-c`) copies 26 | back the resulting target folder. This assumes that server and client are running 27 | the same rust version and have the same processor architecture. On the client `ssh` 28 | and `rsync` need to be installed. 29 | 30 | If you want to pass remote flags you have to end the options/flags section using 31 | `--`. E.g. to build in release mode and copy back the result use: 32 | 33 | ```bash 34 | cargo remote -c -- build --release 35 | ``` 36 | 37 | ### Configuration 38 | 39 | You can place a config file called `.cargo-remote.toml` in the same directory as your 40 | `Cargo.toml` or at `~/.config/cargo-remote/cargo-remote.toml`. There you can define a 41 | default remote build host and user. It can be overridden by the `-r` flag. 42 | 43 | Example config file: 44 | 45 | ```toml 46 | remote = "builds@myserver" 47 | ``` 48 | 49 | ### Flags and options 50 | 51 | ```bash 52 | USAGE: 53 | cargo remote [FLAGS] [OPTIONS] [remote options]... 54 | 55 | FLAGS: 56 | -c, --copy-back Transfer the target folder back to the local machine 57 | --help Prints help information 58 | -h, --transfer-hidden Transfer hidden files and directories to the build server 59 | -V, --version Prints version information 60 | 61 | OPTIONS: 62 | -b, --build-env Set remote environment variables. RUST_BACKTRACE, CC, LIB, etc. [default: 63 | RUST_BACKTRACE=1] 64 | -e, --env Environment profile. default_value = /etc/profile [default: /etc/profile] 65 | --manifest-path Path to the manifest to execute [default: Cargo.toml] 66 | -r, --remote Remote ssh build server 67 | -d, --rustup-default Rustup default (stable|beta|nightly) [default: stable] 68 | 69 | ARGS: 70 | cargo command that will be executed remotely 71 | ... cargo options and flags that will be applied remotely 72 | 73 | ``` 74 | 75 | ## How to install 76 | 77 | ```bash 78 | git clone https://github.com/paritytech/cargo-remote 79 | cargo install --path cargo-remote/ -f 80 | ``` 81 | 82 | ### MacOS Problems 83 | It was reported that the `rsync` version shipped with MacOS doesn't support the progress flag and thus fails when 84 | `cargo-remote` tries to use it. You can install a newer version by running 85 | ```bash 86 | brew install rsync 87 | ``` 88 | See also [#10](https://github.com/sgeisler/cargo-remote/issues/10). 89 | 90 | ### SSH Configuration Suggestion 91 | Establishing a new SSH connection for every remote call takes time. It is recommended to reuse existing SSH sessions by enabling SSH multiplexing. Add an entry like the following to your SSH config file (`~/.ssh/config`): 92 | 93 | ```sshconfig 94 | Host your-remote-build-machine 95 | HostName your.remote.server 96 | User yourusername 97 | Port p 98 | IdentityFile ~/.ssh/your_private_key 99 | IdentitiesOnly yes 100 | ControlMaster auto 101 | ControlPath ~/.ssh/control:%C 102 | ControlPersist 600 103 | ``` 104 | -------------------------------------------------------------------------------- /REMOTE_SETUP.MD: -------------------------------------------------------------------------------- 1 | # Server side setup 2 | 3 | ## 1 step 4 | 5 | ```Bash 6 | nano /etc/profile 7 | export PATH=/usr/local/cargo/bin:$PATH 8 | export RUSTUP_HOME=/usr/local/rustup 9 | export SCCACHE_IDLE_TIMEOUT=0 10 | export SCCACHE_REDIS=redis://127.0.0.1/0 11 | export RUSTC_WRAPPER=sccache 12 | ``` 13 | 14 | ## 2 install rust 15 | 16 | ```Bash 17 | wget https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init 18 | chmod +x rustup-init; \ 19 | ./rustup-init -y --no-modify-path --default-toolchain stable; \ 20 | rm rustup-init; \ 21 | chmod -R a+w+r /usr/local/cargo/; \ 22 | chmod -R a+w+r /usr/local/cargo/; \ 23 | rustup install nightly beta; \ 24 | rustup target add wasm32-unknown-unknown --toolchain nightly; \ 25 | cargo install cargo-audit --force; \ 26 | cargo install sccache --features redis --force; \ 27 | cargo install --git https://github.com/alexcrichton/wasm-gc --force ; 28 | ``` 29 | 30 | ## 3 redis 31 | 32 | ```Bash 33 | protected mode no 34 | 35 | maxmemory 50gb 36 | maxmemory-policy allkeys-lru 37 | service redis restart 38 | ``` 39 | 40 | 48 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use std::ffi::OsStr; 2 | use std::path::PathBuf; 3 | use std::process::{exit, Command, Stdio}; 4 | use structopt::StructOpt; 5 | use toml::Value; 6 | 7 | use log::{debug, error, warn}; 8 | 9 | const PROGRESS_FLAG: &str = "--info=progress2"; 10 | 11 | mod patches; 12 | 13 | #[derive(StructOpt, Debug)] 14 | #[structopt(name = "cargo-remote", bin_name = "cargo")] 15 | enum Opts { 16 | #[structopt(name = "remote")] 17 | Remote { 18 | #[structopt(short = "r", long = "remote", help = "Remote ssh build server")] 19 | remote: Option, 20 | 21 | #[structopt( 22 | short = "b", 23 | long = "build-env", 24 | help = "Set remote environment variables. RUST_BACKTRACE, CC, LIB, etc. ", 25 | default_value = "RUST_BACKTRACE=1" 26 | )] 27 | build_env: String, 28 | 29 | #[structopt( 30 | short = "d", 31 | long = "rustup-default", 32 | help = "Rustup default (stable|beta|nightly)", 33 | default_value = "stable" 34 | )] 35 | rustup_default: String, 36 | 37 | #[structopt( 38 | short = "e", 39 | long = "env", 40 | help = "Environment profile. default_value = /etc/profile", 41 | default_value = "/etc/profile" 42 | )] 43 | env: String, 44 | 45 | #[structopt( 46 | short = "c", 47 | long = "copy-back", 48 | help = "Transfer the target folder or specific file from that folder back to the local machine" 49 | )] 50 | copy_back: Option>, 51 | 52 | #[structopt( 53 | long = "no-copy-lock", 54 | help = "don't transfer the Cargo.lock file back to the local machine" 55 | )] 56 | no_copy_lock: bool, 57 | 58 | #[structopt( 59 | long = "manifest-path", 60 | help = "Path to the manifest to execute", 61 | default_value = "Cargo.toml", 62 | parse(from_os_str) 63 | )] 64 | manifest_path: PathBuf, 65 | 66 | #[structopt( 67 | short = "h", 68 | long = "transfer-hidden", 69 | help = "Transfer hidden files and directories to the build server" 70 | )] 71 | hidden: bool, 72 | 73 | #[structopt(help = "cargo command that will be executed remotely")] 74 | command: String, 75 | 76 | #[structopt( 77 | help = "cargo options and flags that will be applied remotely", 78 | name = "remote options" 79 | )] 80 | options: Vec, 81 | 82 | #[structopt(help = "ignore patches", long = "ignore-patches")] 83 | ignore_patches: bool, 84 | }, 85 | } 86 | 87 | /// Tries to parse the file [`config_path`]. Logs warnings and returns [`None`] if errors occur 88 | /// during reading or parsing, [`Some(Value)`] otherwise. 89 | fn config_from_file(config_path: &PathBuf) -> Option { 90 | let config_file = std::fs::read_to_string(config_path) 91 | .map_err(|e| { 92 | warn!( 93 | "Can't parse config file '{}' (error: {})", 94 | config_path.display(), 95 | e 96 | ); 97 | }) 98 | .ok()?; 99 | 100 | let value = config_file 101 | .parse::() 102 | .map_err(|e| { 103 | warn!( 104 | "Can't parse config file '{}' (error: {})", 105 | config_path.display(), 106 | e 107 | ); 108 | }) 109 | .ok()?; 110 | 111 | Some(value) 112 | } 113 | 114 | fn main() { 115 | simple_logger::SimpleLogger::new() 116 | .with_level(log::LevelFilter::Error) 117 | .env() 118 | .init() 119 | .unwrap(); 120 | 121 | let Opts::Remote { 122 | remote, 123 | build_env, 124 | rustup_default, 125 | env, 126 | copy_back, 127 | no_copy_lock, 128 | manifest_path, 129 | hidden, 130 | command, 131 | options, 132 | ignore_patches, 133 | } = Opts::from_args(); 134 | 135 | let mut metadata_cmd = cargo_metadata::MetadataCommand::new(); 136 | metadata_cmd.manifest_path(manifest_path).no_deps(); 137 | 138 | let project_metadata = match metadata_cmd.exec() { 139 | Ok(m) => m, 140 | Err(cargo_metadata::Error::CargoMetadata { stderr }) => { 141 | error!("Cargo Metadata execution failed:\n{}", stderr); 142 | exit(1) 143 | } 144 | Err(e) => { 145 | error!("Cargo Metadata failed:\n{:?}", e); 146 | exit(1) 147 | } 148 | }; 149 | let project_dir = project_metadata.workspace_root.clone().into_std_path_buf(); 150 | debug!("Project dir: {:?}", project_dir); 151 | 152 | let mut manifest_path = project_dir.clone(); 153 | manifest_path.push("Cargo.toml"); 154 | log::info!("Manifest_path: {:?}", manifest_path); 155 | 156 | let project_name = project_metadata 157 | .packages 158 | .iter() 159 | .find(|p| p.manifest_path == manifest_path) 160 | .map_or_else( 161 | || { 162 | debug!("No metadata found. Setting the remote dir name like the local. Or use --manifest_path for execute"); 163 | project_dir.file_name().unwrap() 164 | }, 165 | |p| OsStr::new(&p.name), 166 | ); 167 | 168 | let build_path_folder = "~/remote-builds/"; 169 | let build_path = format!("{}/{}/", build_path_folder, project_name.to_string_lossy()); 170 | 171 | debug!("Project name: {:?}", project_name); 172 | let configs = vec![ 173 | config_from_file(&project_dir.join(".cargo-remote.toml")), 174 | xdg::BaseDirectories::with_prefix("cargo-remote") 175 | .ok() 176 | .and_then(|base| base.find_config_file("cargo-remote.toml")) 177 | .and_then(|p| config_from_file(&p)), 178 | ]; 179 | 180 | // TODO: move Opts::Remote fields into own type and implement complete_from_config(&mut self, config: &Value) 181 | let build_server = remote 182 | .or_else(|| { 183 | configs 184 | .into_iter() 185 | .flat_map(|config| config.and_then(|c| c["remote"].as_str().map(String::from))) 186 | .next() 187 | }) 188 | .unwrap_or_else(|| { 189 | error!("No remote build server was defined (use config file or --remote flag)"); 190 | exit(-3); 191 | }); 192 | 193 | debug!("Transferring sources to build server."); 194 | // transfer project to build server 195 | copy_to_remote( 196 | &format!("{}/", project_dir.display()), 197 | &format!("{}:{}", build_server, build_path), 198 | hidden, 199 | ) 200 | .unwrap_or_else(|e| { 201 | error!("Failed to transfer project to build server (error: {})", e); 202 | exit(-4); 203 | }); 204 | 205 | if !ignore_patches { 206 | patches::handle_patches(&build_path, &build_server, manifest_path, hidden).unwrap_or_else( 207 | |err| { 208 | log::error!("Could not transfer patched workspaces to remote: {}", err); 209 | }, 210 | ); 211 | } else { 212 | log::debug!("Potential patches will be ignored due to command line flag."); 213 | } 214 | 215 | debug!("Build ENV: {:?}", build_env); 216 | debug!("Environment profile: {:?}", env); 217 | debug!("Build path: {:?}", build_path); 218 | let build_command = format!( 219 | "source {}; rustup default {}; cd {}; {} cargo {} {}", 220 | env, 221 | rustup_default, 222 | build_path, 223 | build_env, 224 | command, 225 | options.join(" ") 226 | ); 227 | 228 | debug!("Starting build process."); 229 | let output = Command::new("ssh") 230 | .arg("-t") 231 | .arg(&build_server) 232 | .arg(build_command) 233 | .stdout(Stdio::inherit()) 234 | .stderr(Stdio::inherit()) 235 | .stdin(Stdio::inherit()) 236 | .output() 237 | .unwrap_or_else(|e| { 238 | error!("Failed to run cargo command remotely (error: {})", e); 239 | exit(-5); 240 | }); 241 | 242 | if let Some(file_name) = copy_back { 243 | debug!("Transferring artifacts back to client."); 244 | let file_name = file_name.unwrap_or_else(String::new); 245 | Command::new("rsync") 246 | .arg("--links") 247 | .arg("--recursive") 248 | .arg("--quiet") 249 | .arg("--delete") 250 | .arg("--compress") 251 | .arg(PROGRESS_FLAG) 252 | .arg(format!( 253 | "{}:{}target/{}", 254 | build_server, build_path, file_name 255 | )) 256 | .arg(format!("{}/target/{}", project_dir.display(), file_name)) 257 | .stdout(Stdio::inherit()) 258 | .stderr(Stdio::inherit()) 259 | .stdin(Stdio::inherit()) 260 | .output() 261 | .unwrap_or_else(|e| { 262 | error!( 263 | "Failed to transfer target back to local machine (error: {})", 264 | e 265 | ); 266 | exit(-6); 267 | }); 268 | } 269 | 270 | if !no_copy_lock { 271 | debug!("Transferring Cargo.lock file back to client."); 272 | Command::new("rsync") 273 | .arg("--links") 274 | .arg("--recursive") 275 | .arg("--quiet") 276 | .arg("--delete") 277 | .arg("--compress") 278 | .arg(PROGRESS_FLAG) 279 | .arg(format!("{}:{}/Cargo.lock", build_server, build_path)) 280 | .arg(format!("{}/Cargo.lock", project_dir.display())) 281 | .stdout(Stdio::inherit()) 282 | .stderr(Stdio::inherit()) 283 | .stdin(Stdio::inherit()) 284 | .output() 285 | .unwrap_or_else(|e| { 286 | error!( 287 | "Failed to transfer Cargo.lock back to local machine (error: {})", 288 | e 289 | ); 290 | exit(-7); 291 | }); 292 | } 293 | 294 | if !output.status.success() { 295 | exit(output.status.code().unwrap_or(1)) 296 | } 297 | } 298 | 299 | pub fn copy_to_remote( 300 | local_dir: &str, 301 | remote_dir: &str, 302 | hidden: bool, 303 | ) -> Result { 304 | let mut rsync_to = Command::new("rsync"); 305 | rsync_to 306 | .arg("--links") 307 | .arg("--recursive") 308 | .arg("--times") 309 | .arg("--quiet") 310 | .arg("--delete") 311 | .arg("--compress") 312 | .arg(PROGRESS_FLAG) 313 | .arg("--exclude") 314 | .arg("target"); 315 | 316 | if !hidden { 317 | rsync_to.arg("--exclude").arg(".*"); 318 | } 319 | 320 | rsync_to 321 | .arg("--rsync-path") 322 | .arg("mkdir -p remote-builds && rsync") 323 | .arg(local_dir) 324 | .arg(remote_dir) 325 | .stdout(Stdio::inherit()) 326 | .stderr(Stdio::inherit()) 327 | .stdin(Stdio::inherit()) 328 | .output() 329 | } 330 | -------------------------------------------------------------------------------- /src/patches.rs: -------------------------------------------------------------------------------- 1 | use crate::copy_to_remote; 2 | use std::ffi::OsString; 3 | use std::io::Write; 4 | use std::path::PathBuf; 5 | use std::process::{Command, Stdio}; 6 | use toml_edit::{Document, InlineTable}; 7 | 8 | /// Handle patched dependencies in a Cargo.toml file. 9 | /// Adjustments are only needed when patches point to local files. 10 | /// Steps: 11 | /// 1. Read Cargo.toml of project 12 | /// 2. Extract list of patches 13 | /// 3. For each patched crate, check if there is a path given. If not, ignore. 14 | /// 4. Find the workspace of the patched crate via `cargo locate-project --workspace` 15 | /// 5. Add workspace to the list of projects that need to be copied 16 | /// 6. Copy folders via rsync 17 | pub fn handle_patches( 18 | build_path: &String, 19 | build_server: &String, 20 | manifest_path: PathBuf, 21 | copy_hidden_files: bool, 22 | ) -> Result<(), String> { 23 | let cargo_file_content = std::fs::read_to_string(&manifest_path).map_err(|err| { 24 | format!( 25 | "Unable to read cargo manifest at {}: {:?}", 26 | manifest_path.display(), 27 | err 28 | ) 29 | })?; 30 | 31 | let maybe_patches = 32 | extract_patched_crates_and_adjust_toml(cargo_file_content, |p| locate_workspace_folder(p))?; 33 | 34 | if let Some((patched_cargo_doc, project_list)) = maybe_patches { 35 | copy_patches_to_remote( 36 | &build_path, 37 | &build_server, 38 | patched_cargo_doc, 39 | project_list, 40 | copy_hidden_files, 41 | )?; 42 | } 43 | Ok(()) 44 | } 45 | 46 | fn locate_workspace_folder(mut crate_path: PathBuf) -> Result { 47 | crate_path.push("Cargo.toml"); 48 | let metadata_cmd = cargo_metadata::MetadataCommand::new() 49 | .manifest_path(&crate_path) 50 | .no_deps() 51 | .exec() 52 | .map_err(|err| { 53 | format!( 54 | "Unable to call cargo metadata on path {}: {:?}", 55 | crate_path.display(), 56 | err 57 | ) 58 | })?; 59 | 60 | Ok(metadata_cmd.workspace_root.into_std_path_buf()) 61 | } 62 | 63 | #[derive(Debug, Clone)] 64 | struct PatchProject { 65 | pub name: OsString, 66 | pub local_path: PathBuf, 67 | pub remote_path: PathBuf, 68 | } 69 | 70 | impl PatchProject { 71 | pub fn new(name: OsString, path: PathBuf, remote_path: PathBuf) -> Self { 72 | PatchProject { 73 | name, 74 | local_path: path, 75 | remote_path, 76 | } 77 | } 78 | } 79 | 80 | fn extract_patched_crates_and_adjust_toml Result>( 81 | manifest_content: String, 82 | locate_workspace: F, 83 | ) -> Result)>, String> { 84 | let mut manifest = manifest_content.parse::().map_err(|err| { 85 | format!( 86 | "Unable to parse Cargo.toml: {:?} content: {}", 87 | err, manifest_content 88 | ) 89 | })?; 90 | let mut workspaces_to_copy: Vec = Vec::new(); 91 | 92 | // A list of inline tables like 93 | // { path = "/some/path" } 94 | let patched_paths: Option> = 95 | manifest["patch"].as_table_mut().map(|patch| { 96 | patch 97 | .iter_mut() 98 | .filter_map(|(_, crate_table)| crate_table.as_table_mut()) 99 | .flat_map(|crate_table| { 100 | crate_table 101 | .iter_mut() 102 | .filter_map(|(_, patch_table)| patch_table.as_inline_table_mut()) 103 | }) 104 | .collect() 105 | }); 106 | 107 | let patched_paths = if let Some(p) = patched_paths { 108 | p 109 | } else { 110 | log::debug!("No patches in project."); 111 | return Ok(None); 112 | }; 113 | 114 | for inline_crate_table in patched_paths { 115 | // We only act if there is a path given for a crate 116 | if let Some(path) = inline_crate_table.get("path") { 117 | let path = PathBuf::from(path.as_str().ok_or("Unable to get path from toml Value")?); 118 | 119 | // Check if the current crate is located in a subfolder of a workspace we 120 | // already know. 121 | let known_workspace = workspaces_to_copy 122 | .iter() 123 | .find(|known_target| path.starts_with(&known_target.local_path)); 124 | match known_workspace { 125 | None => { 126 | // Project is unknown and needs to be copied 127 | let workspace_folder_path = locate_workspace(path.clone()).map_err(|err| { 128 | format!( 129 | "Can not determine workspace path for project at {}: {}", 130 | &path.display(), 131 | err 132 | ) 133 | })?; 134 | let workspace_folder_name = workspace_folder_path 135 | .file_name() 136 | .ok_or("Unable to get file name from workspace folder.")? 137 | .to_owned(); 138 | 139 | let mut remote_folder = PathBuf::from("../"); 140 | remote_folder.push(workspace_folder_name.clone()); 141 | 142 | log::debug!( 143 | "Found referenced project '{}', will copy to '{}'", 144 | &workspace_folder_path.display(), 145 | &remote_folder.display() 146 | ); 147 | 148 | // Add workspace to the list so it will be rsynced to the remote server 149 | workspaces_to_copy.push(PatchProject::new( 150 | workspace_folder_name, 151 | workspace_folder_path.clone(), 152 | remote_folder.clone(), 153 | )); 154 | 155 | // Build a new path for the crate relative to the workspace folder 156 | remote_folder.push(path.strip_prefix(workspace_folder_path).map_err( 157 | |err| format!("Unable to construct remote folder path: {}", err), 158 | )?); 159 | 160 | inline_crate_table.insert( 161 | "path", 162 | toml_edit::Value::from(remote_folder.to_str().unwrap()), 163 | ); 164 | } 165 | 166 | Some(patch_target) => { 167 | let mut new_path = patch_target.remote_path.clone(); 168 | new_path.push(path.strip_prefix(&patch_target.local_path).map_err(|err| { 169 | format!("Unable to construct remote folder path: {}", err) 170 | })?); 171 | 172 | inline_crate_table.insert( 173 | "path", 174 | toml_edit::Value::from( 175 | new_path.to_str().ok_or("Unable to modify path in toml.")?, 176 | ), 177 | ); 178 | } 179 | } 180 | } 181 | } 182 | Ok(Some((manifest, workspaces_to_copy))) 183 | } 184 | 185 | fn copy_patches_to_remote( 186 | build_path: &String, 187 | build_server: &String, 188 | patched_cargo_doc: Document, 189 | projects_to_copy: Vec, 190 | copy_hidden_files: bool, 191 | ) -> Result<(), String> { 192 | for patch_operation in projects_to_copy.iter() { 193 | let local_proj_path = format!("{}/", patch_operation.local_path.display()); 194 | let remote_proj_path = format!( 195 | "{}:remote-builds/{}", 196 | build_server, 197 | patch_operation.name.to_string_lossy() 198 | ); 199 | log::debug!( 200 | "Copying workspace {:?} from {} to {}.", 201 | patch_operation.name, 202 | &local_proj_path, 203 | &remote_proj_path 204 | ); 205 | // transfer project to build server 206 | copy_to_remote(&local_proj_path, &remote_proj_path, copy_hidden_files).map_err(|err| { 207 | format!( 208 | "Failed to transfer project {} to build server (error: {})", 209 | local_proj_path, err 210 | ) 211 | })?; 212 | } 213 | 214 | let remote_toml_path = format!("{}/Cargo.toml", build_path); 215 | log::debug!("Writing adjusted Cargo.toml to {}.", &remote_toml_path); 216 | let mut child = Command::new("ssh") 217 | .args(&[build_server, "-T", "cat > ", &remote_toml_path]) 218 | .stdin(Stdio::piped()) 219 | .spawn() 220 | .unwrap(); 221 | 222 | child 223 | .stdin 224 | .take() 225 | .unwrap() 226 | .write_all(patched_cargo_doc.to_string().as_bytes()) 227 | .map_err(|err| format!("Unable to copy patched Cargo.toml to remote: {}", err))?; 228 | 229 | child 230 | .wait_with_output() 231 | .map_err(|err| format!("Unable to copy patched Cargo.toml to remote: {}", err))?; 232 | Ok(()) 233 | } 234 | 235 | #[cfg(test)] 236 | mod tests { 237 | use std::path::PathBuf; 238 | 239 | use crate::patches::extract_patched_crates_and_adjust_toml; 240 | 241 | #[test] 242 | fn simple_modification_replaces_path() { 243 | let input = r#" 244 | "hello" = 'toml!' 245 | [patch.a] 246 | a-crate = { path = "/some/prefix/a/src/a-crate" } 247 | a-other-crate = { path = "/some/prefix/a/src/subfolder/a-other-crate" } 248 | git-patched-crate = { git = "https://some-url/test/test" } 249 | a-crate-different-folder = { path = "/some/prefix/a-2/src/subfolder/a-crate-different-folder" } 250 | [patch.b] 251 | b-crate = { path = "/some/prefix/b/src/b-crate" } 252 | b-other-crate = { path = "/some/prefix/b/src/subfolder/b-other-crate" } 253 | git-patched-crate = { git = "https://some-url/test/test" } 254 | "# 255 | .to_string(); 256 | let expect = r#" 257 | "hello" = 'toml!' 258 | [patch.a] 259 | a-crate = { path = "../a/src/a-crate" } 260 | a-other-crate = { path = "../a/src/subfolder/a-other-crate" } 261 | git-patched-crate = { git = "https://some-url/test/test" } 262 | a-crate-different-folder = { path = "../a-2/src/subfolder/a-crate-different-folder" } 263 | [patch.b] 264 | b-crate = { path = "../b/src/b-crate" } 265 | b-other-crate = { path = "../b/src/subfolder/b-other-crate" } 266 | git-patched-crate = { git = "https://some-url/test/test" } 267 | "# 268 | .to_string(); 269 | 270 | let result = extract_patched_crates_and_adjust_toml(input, |p| { 271 | if p.starts_with("/some/prefix/a") { 272 | return Ok(PathBuf::from("/some/prefix/a")); 273 | } else if p.starts_with("/some/prefix/a-2") { 274 | return Ok(PathBuf::from("/some/prefix/a-2")); 275 | } else if p.starts_with("/some/prefix/b") { 276 | return Ok(PathBuf::from("/some/prefix/b")); 277 | } 278 | Err("Invalid Path".to_string()) 279 | }) 280 | .expect("Toml patching failed") 281 | .unwrap(); 282 | assert_eq!(result.0.to_string(), expect); 283 | } 284 | } 285 | --------------------------------------------------------------------------------