├── .gitignore ├── .gitmodules ├── Cargo.toml ├── README.md ├── profdata ├── Cargo.toml ├── build.rs └── src │ ├── lib.rs │ └── profdata.cpp ├── profiler-rt ├── Cargo.toml ├── build.rs └── src │ ├── compat.c │ └── lib.rs └── src └── main.rs /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "compiler-rt"] 2 | path = profiler-rt/compiler-rt 3 | url = https://github.com/llvm-mirror/compiler-rt.git 4 | shallow = true 5 | [submodule "llvm"] 6 | path = profdata/llvm 7 | url = https://github.com/rust-lang/llvm.git 8 | shallow = true 9 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = ["profiler-rt", "profdata"] 3 | 4 | [package] 5 | name = "cargo-pgo" 6 | version = "0.1.0" 7 | authors = ["Vadim Chugunov "] 8 | 9 | [features] 10 | default = ["standalone"] 11 | standalone = ["profiler-rt", "profdata"] 12 | 13 | [[bin]] 14 | name = "cargo-pgo" 15 | path = "src/main.rs" 16 | 17 | [dependencies] 18 | profiler-rt = { path = "profiler-rt", optional = true } 19 | profdata = { path = "profdata", optional = true } 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Profile-Guided Optimization workflow for Cargo 2 | 3 | ## Setup 4 | 5 | - `git clone https://github.com/vadimcn/cargo-pgo.git`, 6 | - `cd cargo-pgo`, 7 | - `git submodule update --init` - this may take a while as LLVM is one of the upstream 8 | dependencies. Fortunately, only a small part of it needs to be built. 9 | - `cargo build --release`, 10 | - Add `cargo-pgo/target/release` to your PATH. 11 | 12 | ## Usage 13 | 14 | ### Remove any old profiling data 15 | ``` 16 | cargo pgo clean 17 | ``` 18 | 19 | ### Instrument your binary for profiling 20 | ``` 21 | cargo pgo instr build 22 | ``` 23 | 24 | This will spawn a normal Cargo build (with some extra flags passed to rustc via RUSTFLAGS), so all 25 | the usual `cargo build` flags do apply. 26 | Note that cargo-pgo will automatically add the `--release` flag, since there's little reason to 27 | PGO-optimize debug builds. 28 | 29 | ### Run training scenarios 30 | ``` 31 | cargo pgo instr run 32 | cargo pgo instr run 33 | ... 34 | ``` 35 | You can also use `cargo pgo instr test` or `cargo pgo instr bench`. 36 | Each execution will create a new raw profile file under `target/release/pgo`. 37 | 38 | ### Merge profiles 39 | Before using generated profiles, they must be first merged into an 'indexed' format: 40 | ``` 41 | cargo pgo merge 42 | ``` 43 | The output will be saved in `target/release/pgo/merged.profdata`. 44 | 45 | ### Build optimized binary 46 | ``` 47 | cargo pgo opt build 48 | ``` 49 | 50 | ### Run optimized binary 51 | ``` 52 | cargo pgo opt run|test|bench 53 | ``` 54 | 55 | ("Why not just '`cargo run`'?": Cargo keeps track of the flags it had passed 56 | to rustc last time, and automatically rebuilds the target if they change. Thus, `cargo run` 57 | would first revert the binary back to non-optimized state, which probably isn't what you want.) 58 | 59 | ### Do it in less steps 60 | Cargo automatically (re)builds stale binaries before running them, so you may skip both of the 61 | build steps above and jump straight to running. In addition to that, `cargo pgo opt ...` commands 62 | will automatically merge raw profiles if needed. 63 | All of the above steps may be condensed to just two commands: 64 | ``` 65 | cargo pgo instr run 66 | cargo pgo opt run 67 | ``` 68 | -------------------------------------------------------------------------------- /profdata/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "profdata" 3 | version = "0.1.0" 4 | authors = ["Vadim Chugunov "] 5 | license = "MIT" 6 | readme = "README.md" 7 | keywords = ["pgo", "cargo", "profile"] 8 | 9 | build = "build.rs" 10 | 11 | [build-dependencies] 12 | cmake = "^0.1" 13 | gcc = "^0.3" 14 | -------------------------------------------------------------------------------- /profdata/build.rs: -------------------------------------------------------------------------------- 1 | extern crate cmake; 2 | extern crate gcc; 3 | use std::path::PathBuf; 4 | use std::env; 5 | 6 | fn main() { 7 | cmake::Config::new("llvm") 8 | .define("LLVM_ENABLE_ZLIB", "OFF") 9 | .define("LLVM_INCLUDE_TESTS", "OFF") 10 | .profile("Release") 11 | .build_target("llvm-profdata") 12 | .build(); 13 | 14 | let out_dir = PathBuf::from(&env::var("OUT_DIR").unwrap()); 15 | gcc::Config::new() 16 | .file("src/profdata.cpp") 17 | .cpp(true) 18 | .flag("-std=c++11") 19 | .flag("-fno-exceptions") 20 | .flag("-fno-rtti") 21 | .define("NDEBUG", None) 22 | .opt_level(2) 23 | .include("llvm/include") 24 | .include(out_dir.join("build/include")) 25 | .compile("libprofdataimpl.a"); 26 | 27 | println!("cargo:rustc-link-search={}", out_dir.join("build/lib").to_str().unwrap()); 28 | println!("cargo:rustc-link-lib=LLVMProfileData"); 29 | println!("cargo:rustc-link-lib=LLVMCore"); 30 | println!("cargo:rustc-link-lib=LLVMSupport"); 31 | println!("cargo:rustc-link-lib=curses"); 32 | } 33 | -------------------------------------------------------------------------------- /profdata/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(improper_ctypes)] 2 | 3 | extern "C" { 4 | fn merge_instr_profiles_impl(inputs: &[&str], output: &str) -> bool; 5 | } 6 | 7 | pub fn merge_instr_profiles(inputs: &[&str], output: &str) -> bool { 8 | unsafe { 9 | merge_instr_profiles_impl(inputs, output) 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /profdata/src/profdata.cpp: -------------------------------------------------------------------------------- 1 | // Adapted from llvm-profdata 2 | 3 | #include "llvm/ADT/SmallSet.h" 4 | #include "llvm/ADT/SmallVector.h" 5 | #include "llvm/ADT/StringRef.h" 6 | #include "llvm/IR/LLVMContext.h" 7 | #include "llvm/ProfileData/InstrProfReader.h" 8 | #include "llvm/ProfileData/InstrProfWriter.h" 9 | #include "llvm/ProfileData/ProfileCommon.h" 10 | #include "llvm/ProfileData/SampleProfReader.h" 11 | #include "llvm/ProfileData/SampleProfWriter.h" 12 | #include "llvm/Support/CommandLine.h" 13 | #include "llvm/Support/Errc.h" 14 | #include "llvm/Support/FileSystem.h" 15 | #include "llvm/Support/Format.h" 16 | #include "llvm/Support/ManagedStatic.h" 17 | #include "llvm/Support/MemoryBuffer.h" 18 | #include "llvm/Support/Path.h" 19 | #include "llvm/Support/PrettyStackTrace.h" 20 | #include "llvm/Support/Signals.h" 21 | #include "llvm/Support/raw_ostream.h" 22 | #include 23 | 24 | using namespace llvm; 25 | 26 | static void exitWithError(const Twine &Message, StringRef Whence = "", 27 | StringRef Hint = "") { 28 | errs() << "error: "; 29 | if (!Whence.empty()) 30 | errs() << Whence << ": "; 31 | errs() << Message << "\n"; 32 | if (!Hint.empty()) 33 | errs() << Hint << "\n"; 34 | } 35 | 36 | static void exitWithError(Error E, StringRef Whence = "") { 37 | if (E.isA()) { 38 | handleAllErrors(std::move(E), [&](const InstrProfError &IPE) { 39 | instrprof_error instrError = IPE.get(); 40 | StringRef Hint = ""; 41 | if (instrError == instrprof_error::unrecognized_format) { 42 | // Hint for common error of forgetting -sample for sample profiles. 43 | Hint = "Perhaps you forgot to use the -sample option?"; 44 | } 45 | exitWithError(IPE.message(), Whence, Hint); 46 | }); 47 | } 48 | 49 | exitWithError(toString(std::move(E)), Whence); 50 | } 51 | 52 | static void exitWithErrorCode(std::error_code EC, StringRef Whence = "") { 53 | exitWithError(EC.message(), Whence); 54 | } 55 | 56 | static void handleMergeWriterError(Error E, StringRef WhenceFile = "", 57 | StringRef WhenceFunction = "", 58 | bool ShowHint = true) { 59 | if (!WhenceFile.empty()) 60 | errs() << WhenceFile << ": "; 61 | if (!WhenceFunction.empty()) 62 | errs() << WhenceFunction << ": "; 63 | 64 | auto IPE = instrprof_error::success; 65 | E = handleErrors(std::move(E), 66 | [&IPE](std::unique_ptr E) -> Error { 67 | IPE = E->get(); 68 | return Error(std::move(E)); 69 | }); 70 | errs() << toString(std::move(E)) << "\n"; 71 | 72 | if (ShowHint) { 73 | StringRef Hint = ""; 74 | if (IPE != instrprof_error::success) { 75 | switch (IPE) { 76 | case instrprof_error::hash_mismatch: 77 | case instrprof_error::count_mismatch: 78 | case instrprof_error::value_site_count_mismatch: 79 | Hint = "Make sure that all profile data to be merged is generated " 80 | "from the same binary."; 81 | break; 82 | default: 83 | break; 84 | } 85 | } 86 | 87 | if (!Hint.empty()) 88 | errs() << Hint << "\n"; 89 | } 90 | } 91 | 92 | template 93 | struct Slice { 94 | T* data; 95 | uintptr_t len; 96 | }; 97 | 98 | extern "C" 99 | bool merge_instr_profiles_impl(Slice> inputs, Slice output) { 100 | 101 | llvm_shutdown_obj shutdown; 102 | 103 | StringRef OutputFileName(output.data, output.len); 104 | std::error_code EC; 105 | raw_fd_ostream Output(OutputFileName, EC, sys::fs::F_None); 106 | if (EC) { 107 | exitWithErrorCode(EC, OutputFileName); 108 | return false; 109 | } 110 | 111 | InstrProfWriter Writer(false); 112 | SmallSet WriterErrorCodes; 113 | 114 | for (int i = 0; i < inputs.len; ++i) { 115 | StringRef InputFileName(inputs.data[i].data, inputs.data[i].len); 116 | auto ReaderOrErr = InstrProfReader::create(InputFileName); 117 | if (Error E = ReaderOrErr.takeError()) { 118 | exitWithError(std::move(E), InputFileName); 119 | return false; 120 | } 121 | 122 | auto Reader = std::move(ReaderOrErr.get()); 123 | bool IsIRProfile = Reader->isIRLevelProfile(); 124 | if (Writer.setIsIRLevelProfile(IsIRProfile)) { 125 | exitWithError("Merge IR generated profile with Clang generated profile."); 126 | return false; 127 | } 128 | 129 | for (auto &I : *Reader) { 130 | if (Error E = Writer.addRecord(std::move(I), 1)) { 131 | // Only show hint the first time an error occurs. 132 | instrprof_error IPE = InstrProfError::take(std::move(E)); 133 | bool firstTime = WriterErrorCodes.insert(IPE).second; 134 | handleMergeWriterError(make_error(IPE), InputFileName, 135 | I.Name, firstTime); 136 | } 137 | } 138 | if (Reader->hasError()) { 139 | exitWithError(Reader->getError(), InputFileName); 140 | return false; 141 | } 142 | } 143 | 144 | Writer.write(Output); 145 | return true; 146 | } 147 | -------------------------------------------------------------------------------- /profiler-rt/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "profiler-rt" 3 | version = "0.1.0" 4 | authors = ["Vadim Chugunov "] 5 | build = "build.rs" 6 | 7 | [build-dependencies] 8 | gcc = "^0.3" 9 | -------------------------------------------------------------------------------- /profiler-rt/build.rs: -------------------------------------------------------------------------------- 1 | extern crate gcc; 2 | use std::env; 3 | use std::fs; 4 | use std::path::PathBuf; 5 | 6 | fn main() { 7 | let dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()) 8 | .join("compiler-rt/lib/profile"); 9 | gcc::Config::new() 10 | .file(dir.join("GCDAProfiling.c")) 11 | .file(dir.join("InstrProfiling.c")) 12 | .file(dir.join("InstrProfilingValue.c")) 13 | .file(dir.join("InstrProfilingBuffer.c")) 14 | .file(dir.join("InstrProfilingFile.c")) 15 | .file(dir.join("InstrProfilingMerge.c")) 16 | .file(dir.join("InstrProfilingMergeFile.c")) 17 | .file(dir.join("InstrProfilingWriter.c")) 18 | .file(dir.join("InstrProfilingPlatformDarwin.c")) 19 | .file(dir.join("InstrProfilingPlatformLinux.c")) 20 | .file(dir.join("InstrProfilingPlatformOther.c")) 21 | .file(dir.join("InstrProfilingUtil.c")) 22 | .file(dir.join("InstrProfilingRuntime.cc")) 23 | .file(dir.join("../../../src/compat.c")) 24 | .opt_level(2) 25 | .include(dir) 26 | .compile("libprofiler-rt.a"); 27 | 28 | let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); 29 | let from = out_dir.join("libprofiler-rt.a"); 30 | let to = out_dir.join("..").join("..").join("..").join("libprofiler-rt.a"); 31 | fs::copy(from, to).unwrap(); 32 | } 33 | -------------------------------------------------------------------------------- /profiler-rt/src/compat.c: -------------------------------------------------------------------------------- 1 | void __llvm_profile_set_filename(const char *Name); 2 | void __llvm_profile_register_write_file_atexit(); 3 | void __llvm_profile_initialize_file(); 4 | 5 | void __llvm_profile_override_default_filename(const char *Name) { 6 | __llvm_profile_register_write_file_atexit(); 7 | __llvm_profile_set_filename(Name); 8 | __llvm_profile_initialize_file(); 9 | } 10 | -------------------------------------------------------------------------------- /profiler-rt/src/lib.rs: -------------------------------------------------------------------------------- 1 | 2 | extern "C" { 3 | fn __llvm_profile_register_write_file_atexit(); 4 | fn __llvm_profile_initialize_file(); 5 | } 6 | 7 | pub fn initialize() { 8 | unsafe { 9 | __llvm_profile_register_write_file_atexit(); 10 | __llvm_profile_initialize_file(); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use std::process::Command; 2 | use std::env; 3 | use std::fs; 4 | 5 | fn main() { 6 | let mut args = env::args(); 7 | args.next().expect("program name"); 8 | match args.next() { 9 | Some(ref s) if s == "pgo" => { 10 | match args.next() { 11 | Some(ref s) if s == "instr" => { 12 | match args.next() { 13 | Some(ref s) if s == "build" => instrumented("build", true, args), 14 | Some(ref s) if s == "rustc" => instrumented("rustc", true, args), 15 | Some(ref s) if s == "run" => instrumented("run", true, args), 16 | Some(ref s) if s == "test" => instrumented("test", true, args), 17 | Some(ref s) if s == "bench" => instrumented("bench", false, args), 18 | Some(ref s) => invalid_arg(s), 19 | _ => show_usage(), 20 | } 21 | } 22 | Some(ref s) if s == "opt" => { 23 | match args.next() { 24 | Some(ref s) if s == "build" => optimized("build", true, args), 25 | Some(ref s) if s == "rustc" => optimized("rustc", true, args), 26 | Some(ref s) if s == "run" => optimized("run", true, args), 27 | Some(ref s) if s == "test" => optimized("test", true, args), 28 | Some(ref s) if s == "bench" => optimized("bench", false, args), 29 | Some(ref s) => invalid_arg(s), 30 | _ => show_usage(), 31 | } 32 | } 33 | Some(ref s) if s == "merge" => { merge_profiles(); } 34 | Some(ref s) if s == "clean" => clean(), 35 | Some(ref s) => invalid_arg(s), 36 | _ => show_usage(), 37 | } 38 | } 39 | Some(ref s) => invalid_arg(s), 40 | _ => show_usage(), 41 | } 42 | } 43 | 44 | fn show_usage() { 45 | println!("cargo-pgo v{} {}", 46 | option_env!("CARGO_PKG_VERSION").unwrap_or("?"), 47 | if cfg!(feature = "standalone") { "(standalone)" } else { "" }); 48 | 49 | println!("Usage: cargo pgo ...\ 50 | \nCommands:\ 51 | \n instr build|rustc ... - build an instrumented binary\ 52 | \n instr run|test|bench ... - run the instrumented binary while recording profiling data\ 53 | \n merge - merge raw profiling data\ 54 | \n opt build|rustc ... - merge raw profiling data, then build an optimized binary\ 55 | \n opt run|test|bench ... - run the optimized binary\ 56 | \n clean - remove recorded profiling data"); 57 | } 58 | 59 | fn invalid_arg(arg: &str) { 60 | println!("Unexpected argument: {}\n", arg); 61 | show_usage(); 62 | } 63 | 64 | fn get_clang_target_arch() -> &'static str { 65 | if cfg!(target_arch = "x86_64") { "x86_64" } 66 | else if cfg!(target_arch = "x86") { "i386" } 67 | else if cfg!(target_arch = "aarch64") { "aarch64" } 68 | else if cfg!(target_arch = "arm") { "armhf" } 69 | else if cfg!(target_arch = "mips") { "mips" } 70 | else { unimplemented!() } 71 | } 72 | 73 | fn instrumented(subcommand: &str, release_flag: bool, args: env::Args) { 74 | let mut args: Vec = args.collect(); 75 | if release_flag { 76 | args.insert(0, "--release".to_string()); 77 | } 78 | let old_rustflags = env::var("RUSTFLAGS").unwrap_or(String::new()); 79 | 80 | let profiler_rt_lib = if cfg!(feature = "standalone") { 81 | "profiler-rt".to_string() 82 | } else { 83 | format!("clang_rt.profile-{0}", get_clang_target_arch()) 84 | }; 85 | let rustflags = format!("{0} --cfg=profiling \ 86 | -Cllvm-args=-profile-generate \ 87 | -Cllvm-args=-profile-generate-file=target/release/pgo/%p.profraw \ 88 | -Lnative={1} -Clink-args=-l{2}", 89 | old_rustflags, 90 | env::current_exe().unwrap().parent().unwrap().to_str().unwrap(), 91 | profiler_rt_lib); 92 | 93 | let mut child = Command::new("cargo") 94 | .arg(subcommand) 95 | .args(&args) 96 | .env("RUSTFLAGS", rustflags) 97 | .spawn().unwrap_or_else(|e| panic!("{}", e)); 98 | child.wait().unwrap_or_else(|e| panic!("{}", e)); 99 | } 100 | 101 | fn optimized(subcommand: &str, release_flag: bool, args: env::Args) { 102 | let mut args = args.collect::>(); 103 | if release_flag { 104 | args.insert(0, "--release".to_string()); 105 | } 106 | if !merge_profiles() { 107 | println!("Warning: no profiling data was found - this build will not be PGO-optimized."); 108 | } 109 | let old_rustflags = env::var("RUSTFLAGS").unwrap_or(String::new()); 110 | let rustflags = format!("{} -Cllvm-args=-profile-use=target/release/pgo/merged.profdata", 111 | old_rustflags); 112 | let mut child = Command::new("cargo") 113 | .arg(subcommand) 114 | .args(&args) 115 | .env("RUSTFLAGS", rustflags) 116 | .spawn().unwrap_or_else(|e| panic!("{}", e)); 117 | child.wait().unwrap_or_else(|e| panic!("{}", e)); 118 | } 119 | 120 | // Get all target/release/pgo/*.profraw files 121 | fn gather_raw_profiles() -> Vec { 122 | let dir = match fs::read_dir("target/release/pgo") { 123 | Ok(dir) => dir, 124 | Err(_) => return vec![], 125 | }; 126 | let mut raw_profiles = Vec::new(); 127 | let mut found_empty = false; 128 | for entry in dir { 129 | if let Ok(entry) = entry { 130 | if let Some(ext) = entry.path().extension() { 131 | if ext == "profraw" { 132 | if let Ok(metadata) = entry.metadata() { 133 | if metadata.len() > 0 { 134 | raw_profiles.push(entry.path()); 135 | } else { 136 | found_empty = true; 137 | } 138 | } 139 | } 140 | } 141 | } 142 | } 143 | if found_empty { 144 | println!("Warhing: empty profiling data files were found - some training runs may have crashed."); 145 | } 146 | raw_profiles 147 | } 148 | 149 | #[cfg(not(feature="standalone"))] 150 | // Use external tool 151 | fn merge_profiles() -> bool { 152 | let raw_profiles = gather_raw_profiles(); 153 | if raw_profiles.len() == 0 { 154 | return false; 155 | } 156 | let mut child = Command::new("llvm-profdata") 157 | .arg("merge") 158 | .args(&raw_profiles) 159 | .arg("--output").arg("target/release/pgo/merged.profdata") 160 | .spawn().unwrap_or_else(|e| panic!("{}", e)); 161 | let exit_status = child.wait().unwrap_or_else(|e| panic!("{}", e)); 162 | return exit_status.code() == Some(0); 163 | } 164 | 165 | #[cfg(feature="standalone")] 166 | // Use the built-in profile merger 167 | fn merge_profiles() -> bool { 168 | extern crate profdata; 169 | 170 | let raw_profiles = gather_raw_profiles(); 171 | if raw_profiles.len() == 0 { 172 | return false; 173 | } 174 | let inputs: Vec<&str> = raw_profiles.iter().map(|p| p.to_str().unwrap()).collect(); 175 | if !profdata::merge_instr_profiles(&inputs, "target/release/pgo/merged.profdata") { 176 | return false; 177 | } 178 | return true; 179 | } 180 | 181 | fn clean() { 182 | let _ = fs::remove_dir_all("target/release/pgo"); 183 | } 184 | --------------------------------------------------------------------------------