├── .gitignore ├── tests ├── manual │ ├── clang-cl │ │ ├── .gitignore │ │ ├── test.h │ │ ├── README.md │ │ ├── test.c │ │ └── build.ninja │ ├── .gitignore │ ├── prints │ │ ├── README.md │ │ ├── prints.bat │ │ ├── prints.sh │ │ ├── build.ninja │ │ └── build-win.ninja │ └── fdtest │ │ └── fdtest.py ├── e2e_test.rs ├── e2e │ ├── directories.rs │ ├── validations.rs │ ├── discovered.rs │ ├── bindings.rs │ ├── mod.rs │ ├── missing.rs │ ├── regen.rs │ └── basic.rs └── snapshot │ └── README.md ├── doc ├── build1.png ├── build2.png ├── build3.png ├── build4.png ├── build5.png ├── comparison.md └── development.md ├── benches ├── .gitignore ├── parse.rs └── canon.rs ├── .vscode ├── settings.json └── launch.json ├── git-pre-commit ├── src ├── main.rs ├── process.rs ├── lib.rs ├── signal.rs ├── progress.rs ├── densemap.rs ├── intern.rs ├── smallmap.rs ├── progress_dumb.rs ├── terminal.rs ├── trace.rs ├── hash.rs ├── eval.rs ├── scanner.rs ├── depfile.rs ├── canon.rs ├── process_posix.rs ├── run.rs ├── task.rs ├── process_win.rs ├── db.rs ├── load.rs ├── progress_fancy.rs └── graph.rs ├── dprint.json ├── .github └── workflows │ ├── ci.yml │ └── fmt.yml ├── Cargo.toml ├── README.md ├── LICENSE └── Cargo.lock /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /scratch 3 | -------------------------------------------------------------------------------- /tests/manual/clang-cl/.gitignore: -------------------------------------------------------------------------------- 1 | test.exe 2 | -------------------------------------------------------------------------------- /tests/manual/clang-cl/test.h: -------------------------------------------------------------------------------- 1 | #define X 3 2 | -------------------------------------------------------------------------------- /tests/manual/.gitignore: -------------------------------------------------------------------------------- 1 | .n2_db 2 | .ninja_log 3 | -------------------------------------------------------------------------------- /doc/build1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/evmar/n2/HEAD/doc/build1.png -------------------------------------------------------------------------------- /doc/build2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/evmar/n2/HEAD/doc/build2.png -------------------------------------------------------------------------------- /doc/build3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/evmar/n2/HEAD/doc/build3.png -------------------------------------------------------------------------------- /doc/build4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/evmar/n2/HEAD/doc/build4.png -------------------------------------------------------------------------------- /doc/build5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/evmar/n2/HEAD/doc/build5.png -------------------------------------------------------------------------------- /tests/manual/clang-cl/README.md: -------------------------------------------------------------------------------- 1 | Invokes clang-cl to sanity check the /showIncludes handling. 2 | -------------------------------------------------------------------------------- /benches/.gitignore: -------------------------------------------------------------------------------- 1 | # You can drop a build.ninja into this directory to benchmark it. 2 | /build.ninja 3 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.rulers": [ 3 | 80 4 | ], 5 | "editor.detectIndentation": false, 6 | } -------------------------------------------------------------------------------- /tests/manual/prints/README.md: -------------------------------------------------------------------------------- 1 | This manual test runs multiple subcommands that write to stdout, to verify 2 | interleaving output. 3 | -------------------------------------------------------------------------------- /tests/e2e_test.rs: -------------------------------------------------------------------------------- 1 | //! Integration test. Runs n2 binary against a temp directory. 2 | 3 | // All the tests live in the e2e/ subdir. 4 | mod e2e; 5 | -------------------------------------------------------------------------------- /tests/manual/clang-cl/test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "test.h" 3 | 4 | int main() { 5 | printf("hello, world: %d\n", X); 6 | return 0; 7 | } 8 | -------------------------------------------------------------------------------- /tests/manual/clang-cl/build.ninja: -------------------------------------------------------------------------------- 1 | rule clang-cl 2 | command = clang-cl /showIncludes $in 3 | deps = msvc 4 | 5 | build test.exe: clang-cl test.c 6 | default test.exe 7 | -------------------------------------------------------------------------------- /git-pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | diff=$(cargo fmt -- --check) 4 | result=$? 5 | 6 | if [[ ${result} -ne 0 ]] ; then 7 | echo 'run `cargo fmt`' 8 | exit 1 9 | fi 10 | -------------------------------------------------------------------------------- /tests/manual/prints/prints.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | echo %1: 1 3 | timeout /T 1 /NOBREAK > nul 4 | echo %1: 2 5 | timeout /T 1 /NOBREAK > nul 6 | echo %1: 3 7 | timeout /T 1 /NOBREAK > nul 8 | -------------------------------------------------------------------------------- /tests/manual/prints/prints.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | for i in $(seq 3); do 4 | echo $1: $i 5 | # Uncomment me to verify which fds are open: 6 | lsof -a -p $$ 7 | sleep 1 8 | done 9 | -------------------------------------------------------------------------------- /tests/manual/prints/build.ninja: -------------------------------------------------------------------------------- 1 | rule printy 2 | command = ./prints.sh $out 3 | 4 | build print1: printy 5 | build print2: printy 6 | build print3: printy 7 | 8 | build out: phony print1 print2 print3 9 | default out 10 | -------------------------------------------------------------------------------- /tests/manual/prints/build-win.ninja: -------------------------------------------------------------------------------- 1 | rule printy 2 | command = prints.bat $out 3 | 4 | build print1: printy 5 | build print2: printy 6 | build print3: printy 7 | 8 | build out: phony print1 print2 print3 9 | default out 10 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let exit_code = match n2::run::run() { 3 | Ok(code) => code, 4 | Err(err) => { 5 | println!("n2: error: {}", err); 6 | 1 7 | } 8 | }; 9 | if exit_code != 0 { 10 | std::process::exit(exit_code); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /dprint.json: -------------------------------------------------------------------------------- 1 | { 2 | "markdown": { 3 | "textWrap": "always" 4 | }, 5 | "includes": [ 6 | "**/*.{md}", 7 | "**/*.{toml}" 8 | ], 9 | "excludes": [], 10 | "plugins": [ 11 | "https://plugins.dprint.dev/markdown-0.15.3.wasm", 12 | "https://plugins.dprint.dev/toml-0.5.4.wasm" 13 | ] 14 | } -------------------------------------------------------------------------------- /src/process.rs: -------------------------------------------------------------------------------- 1 | //! Exposes process::run_command, a wrapper around platform-native process execution. 2 | 3 | #[cfg(unix)] 4 | pub use crate::process_posix::run_command; 5 | #[cfg(windows)] 6 | pub use crate::process_win::run_command; 7 | 8 | #[cfg(target_arch = "wasm32")] 9 | fn run_command( 10 | cmdline: &str, 11 | mut output_cb: impl FnMut(&[u8]), 12 | ) -> anyhow::Result<(Termination, Vec)> { 13 | anyhow::bail!("wasm cannot run commands"); 14 | } 15 | 16 | #[derive(Debug, PartialEq)] 17 | pub enum Termination { 18 | Success, 19 | Interrupted, 20 | Failure, 21 | } 22 | -------------------------------------------------------------------------------- /tests/manual/fdtest/fdtest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test from https://github.com/evmar/n2/issues/14 3 | Generates a build.ninja with a ton of parallel commands to see if we leak fds. 4 | """ 5 | 6 | import textwrap 7 | def write(f): 8 | f.write(textwrap.dedent('''\ 9 | rule b 10 | command = sleep 300; touch $out 11 | ''')) 12 | for i in range(1000): 13 | f.write(f'build foo{i}: b\n') 14 | # n2 needs an explicit default target: 15 | f.write('default') 16 | for i in range(1000): 17 | f.write(f' foo{i}') 18 | f.write('\n') 19 | with open('build.ninja', 'w', encoding='utf-8') as f: 20 | write(f) 21 | 22 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | env: 10 | CARGO_TERM_COLOR: always 11 | 12 | jobs: 13 | test: 14 | strategy: 15 | matrix: 16 | os: [ubuntu-latest, windows-latest] 17 | runs-on: ${{ matrix.os }} 18 | env: 19 | RUST_BACKTRACE: 1 20 | 21 | steps: 22 | - uses: actions/checkout@v4 23 | #- run: rustup toolchain install stable --profile minimal 24 | - uses: Swatinem/rust-cache@v2 25 | - name: Check formatting 26 | run: cargo fmt -- --check 27 | - name: Build 28 | run: cargo build --verbose -F crlf 29 | - name: Run tests 30 | run: cargo test --verbose -F crlf 31 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod canon; 2 | mod db; 3 | mod densemap; 4 | mod depfile; 5 | mod eval; 6 | mod graph; 7 | mod hash; 8 | pub mod load; 9 | pub mod parse; 10 | mod process; 11 | #[cfg(unix)] 12 | mod process_posix; 13 | #[cfg(windows)] 14 | mod process_win; 15 | mod progress; 16 | mod progress_dumb; 17 | mod progress_fancy; 18 | pub mod run; 19 | pub mod scanner; 20 | mod signal; 21 | mod smallmap; 22 | mod task; 23 | mod terminal; 24 | mod trace; 25 | mod work; 26 | 27 | #[cfg(feature = "jemalloc")] 28 | #[cfg(not(any(miri, windows, target_arch = "wasm32")))] 29 | use jemallocator::Jemalloc; 30 | 31 | #[cfg(feature = "jemalloc")] 32 | #[cfg(not(any(miri, windows, target_arch = "wasm32")))] 33 | #[global_allocator] 34 | static GLOBAL: Jemalloc = Jemalloc; 35 | -------------------------------------------------------------------------------- /.github/workflows/fmt.yml: -------------------------------------------------------------------------------- 1 | # Check formatting of toml/md files using dprint. 2 | # Rust format checking is done in the CI workflow. 3 | 4 | name: fmt 5 | 6 | on: 7 | push: 8 | branches: [main] 9 | pull_request: 10 | branches: [main] 11 | 12 | jobs: 13 | fmt: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v4 17 | 18 | - name: Cache dprint 19 | uses: actions/cache@v4 20 | with: 21 | path: | 22 | ~/.dprint 23 | ~/.cache/dprint 24 | key: dprint 25 | 26 | - name: Install dprint 27 | run: | 28 | if [ ! -f $HOME/.dprint/bin/dprint ]; then 29 | curl -fsSL https://dprint.dev/install.sh | sh 30 | fi 31 | echo $HOME/.dprint/bin >> $GITHUB_PATH 32 | 33 | - run: dprint check 34 | -------------------------------------------------------------------------------- /tests/e2e/directories.rs: -------------------------------------------------------------------------------- 1 | use crate::e2e::*; 2 | 3 | #[cfg(unix)] 4 | #[test] 5 | fn dep_on_current_directory() -> anyhow::Result<()> { 6 | let space = TestSpace::new()?; 7 | space.write( 8 | "build.ninja", 9 | " 10 | rule list_files 11 | command = ls $in > $out 12 | 13 | build out: list_files . 14 | ", 15 | )?; 16 | space.write("foo", "")?; 17 | 18 | let out = space.run_expect(&mut n2_command(vec!["-d", "explain", "out"]))?; 19 | assert_output_contains(&out, "ran 1 task"); 20 | assert_eq!(space.read("out")?, b"build.ninja\nfoo\nout\n"); 21 | 22 | let out = space.run_expect(&mut n2_command(vec!["-d", "explain", "out"]))?; 23 | assert_output_contains(&out, "no work to do"); 24 | 25 | // Expect: writing a file modifies the current directory's mtime, triggering a build. 26 | space.write("foo2", "")?; 27 | let out = space.run_expect(&mut n2_command(vec!["-d", "explain", "out"]))?; 28 | assert_output_contains(&out, "ran 1 task"); 29 | assert_eq!(space.read("out")?, b"build.ninja\nfoo\nfoo2\nout\n"); 30 | 31 | Ok(()) 32 | } 33 | -------------------------------------------------------------------------------- /src/signal.rs: -------------------------------------------------------------------------------- 1 | //! Unix signal handling (SIGINT). 2 | //! 3 | //! We let the first SIGINT reach child processes, which ought to build-fail 4 | //! and let the parent properly print that progress. This also lets us still 5 | //! write out pending debug traces, too. 6 | 7 | use std::sync::atomic::AtomicBool; 8 | 9 | static INTERRUPTED: AtomicBool = AtomicBool::new(false); 10 | 11 | #[cfg(unix)] 12 | extern "C" fn sigint_handler(_sig: libc::c_int) { 13 | INTERRUPTED.store(true, std::sync::atomic::Ordering::Relaxed); 14 | // SA_RESETHAND should clear the handler. 15 | } 16 | 17 | #[cfg(unix)] 18 | pub fn register_sigint() { 19 | // Safety: registering a signal handler is libc unsafe code. 20 | unsafe { 21 | let mut sa: libc::sigaction = std::mem::zeroed(); 22 | sa.sa_sigaction = sigint_handler as libc::sighandler_t; 23 | sa.sa_flags = libc::SA_RESETHAND; 24 | #[cfg(not(miri))] 25 | libc::sigaction(libc::SIGINT, &sa, std::ptr::null_mut()); 26 | } 27 | } 28 | 29 | pub fn was_interrupted() -> bool { 30 | INTERRUPTED.load(std::sync::atomic::Ordering::Relaxed) 31 | } 32 | -------------------------------------------------------------------------------- /tests/snapshot/README.md: -------------------------------------------------------------------------------- 1 | # Snapshot tests 2 | 3 | This directory is for real-world outputs from tools that generate Ninja files. 4 | 5 | Because these outputs are large, they aren't checked in to the Ninja tree. 6 | Instead for now it's a random zip file on Google drive at 7 | 8 | https://drive.google.com/file/d/1dlNAaf0XRXjV6UPkNV88JFuOBGJLdyGB/view?usp=sharing 9 | 10 | It expects to be unzipped directly into this directory: 11 | 12 | ``` 13 | $ unzip snapshot.zip 14 | ``` 15 | 16 | ## Test data in the zip 17 | 18 | ### llvm-cmake 19 | 20 | `llvm-cmake/` contains LLVM build files generated by CMake. 21 | 22 | https://llvm.org/docs/GettingStarted.html (note they have a CMake-specific page 23 | that has instructions that don't work(?!)) 24 | 25 | ``` 26 | $ cmake -G Ninja -B build -S llvm -DCMAKE_BUILD_TYPE=Release 27 | ``` 28 | 29 | ### llvm-gn 30 | 31 | `llvm-gn/` contains LLVM build files generated by the GN build system. 32 | 33 | Read llvm/utils/gn/README.rst in LLVM checkout for more, but in brief. 34 | 35 | ``` 36 | $ llvm/utils/gn/get.py 37 | $ llvm/utils/gn/gn.py gen out/gn 38 | ``` 39 | 40 | ## Reminder to self how I made it 41 | 42 | ``` 43 | $ zip -r snapshot.zip .gitignore llvm-* 44 | ``` 45 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "lldb", 9 | "request": "launch", 10 | "name": "Debug executable 'n2'", 11 | "cargo": { 12 | "args": [ 13 | "build", 14 | "--bin=n2", 15 | "--package=n2" 16 | ], 17 | "filter": { 18 | "name": "n2", 19 | "kind": "bin" 20 | } 21 | }, 22 | "args": [ 23 | "-C", 24 | "../projects/ninja" 25 | ], 26 | "cwd": "${workspaceFolder}" 27 | }, 28 | { 29 | "type": "lldb", 30 | "request": "launch", 31 | "name": "Debug unit tests in executable 'n2'", 32 | "cargo": { 33 | "args": [ 34 | "test", 35 | "--no-run", 36 | "--bin=n2", 37 | "--package=n2" 38 | ], 39 | "filter": { 40 | "name": "n2", 41 | "kind": "bin" 42 | } 43 | }, 44 | "args": [], 45 | "cwd": "${workspaceFolder}" 46 | } 47 | ] 48 | } -------------------------------------------------------------------------------- /src/progress.rs: -------------------------------------------------------------------------------- 1 | //! Build progress tracking and reporting, for the purpose of display to the 2 | //! user. 3 | 4 | use crate::{graph::Build, graph::BuildId, task::TaskResult, work::StateCounts}; 5 | 6 | /// Compute the message to display on the console for a given build. 7 | pub fn build_message(build: &Build) -> &str { 8 | build 9 | .desc 10 | .as_ref() 11 | .filter(|desc| !desc.is_empty()) 12 | .unwrap_or_else(|| build.cmdline.as_ref().unwrap()) 13 | } 14 | 15 | /// Trait for build progress notifications. 16 | pub trait Progress { 17 | /// Called as individual build tasks progress through build states. 18 | fn update(&self, counts: &StateCounts); 19 | 20 | /// Called when a task starts. 21 | fn task_started(&self, id: BuildId, build: &Build); 22 | 23 | /// Called when a task's last line of output changes. 24 | fn task_output(&self, id: BuildId, line: Vec); 25 | 26 | /// Called when a task completes. 27 | fn task_finished(&self, id: BuildId, build: &Build, result: &TaskResult); 28 | 29 | /// Log a line of output without corrupting the progress display. 30 | /// This line is persisted beyond further progress updates. For example, 31 | /// used when a task fails; we want the final output to show that failed 32 | /// task's output even if we do more work after it fails. 33 | fn log(&self, msg: &str); 34 | } 35 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "n2" 3 | version = "0.1.0" 4 | categories = ["development-tools", "development-tools::build-utils"] 5 | edition = "2021" 6 | exclude = [".github/*", ".vscode/*"] 7 | homepage = "https://github.com/evmar/n2" 8 | keywords = ["ninja", "build"] 9 | license = "Apache-2.0" 10 | readme = "README.md" 11 | repository = "https://github.com/evmar/n2" 12 | # https://github.com/evmar/n2/issues/74 13 | # Note: if we bump this, may need to bump .github/workflows/ci.yml version too. 14 | rust-version = "1.81.0" 15 | description = "a ninja compatible build system" 16 | 17 | [dependencies] 18 | anyhow = "1.0" 19 | lexopt = "0.3.0" 20 | libc = "0.2" 21 | rustc-hash = "1.1.0" 22 | 23 | [target.'cfg(windows)'.dependencies.windows-sys] 24 | version = "0.48" 25 | features = [ 26 | "Win32_Foundation", 27 | "Win32_Security", 28 | "Win32_System_Console", 29 | "Win32_System_Diagnostics_Debug", 30 | "Win32_System_Pipes", 31 | "Win32_System_Threading", 32 | ] 33 | 34 | [target.'cfg(not(any(windows, target_arch = "wasm32")))'.dependencies] 35 | jemallocator = { version = "0.5", optional = true } 36 | 37 | [dev-dependencies] 38 | divan = "0.1.16" 39 | tempfile = "3.6.0" 40 | 41 | [profile.release] 42 | debug = true 43 | lto = true 44 | 45 | [[bench]] 46 | name = "parse" 47 | harness = false 48 | 49 | [[bench]] 50 | name = "canon" 51 | harness = false 52 | 53 | [features] 54 | default = ["jemalloc"] 55 | jemalloc = ["jemallocator"] 56 | crlf = [] 57 | -------------------------------------------------------------------------------- /doc/comparison.md: -------------------------------------------------------------------------------- 1 | # Feature comparison with Ninja 2 | 3 | n2 is intended to be able to build any project that Ninja can load. Here is a 4 | comparison of things n2 does worse and better than Ninja. 5 | 6 | ## Improvements 7 | 8 | Here are some things n2 improves over Ninja: 9 | 10 | - Builds are more incremental: n2 starts running tasks as soon as an out of date 11 | one is found, rather than gathering all the out of date tasks before executing 12 | as Ninja does. 13 | - Fancier status output, modeled after Bazel. 14 | [Here's a small demo](https://asciinema.org/a/F2E7a6nX4feoSSWVI4oFAm21T). 15 | - `-d trace` generates a performance trace that can be visualized by Chrome's 16 | `about:tracing` or alternatives (speedscope, perfetto). 17 | 18 | ### Extensions 19 | 20 | Some extra build variables are available only in n2: 21 | 22 | - `hide_progress`: build edges with this flag will not show the last line of 23 | output in the fancy progress output. 24 | - `hide_success`: build edges with this flag will not show the complete command 25 | output when the command completes successfully. 26 | 27 | ## Missing 28 | 29 | - Windows is incomplete. 30 | - Ninja has special handling of backslashed paths that 31 | [n2 doesn't yet follow](https://github.com/evmar/n2/issues/42). 32 | - Dynamic dependencies. 33 | - `console` pool. n2 currently just treats `console` as an ordinary pool of 34 | depth 1, and only shows console output after the task completes. In practice 35 | this means commands that print progress when run currently show nothing until 36 | they're complete. 37 | - `subninja` is only partially implemented. 38 | 39 | ### Missing flags 40 | 41 | - `-l`, load average throttling 42 | - `-n`, dry run 43 | 44 | #### Missing subcommands 45 | 46 | Most of `-d` (debugging), `-t` (tools). 47 | 48 | No `-w` (warnings). 49 | -------------------------------------------------------------------------------- /src/densemap.rs: -------------------------------------------------------------------------------- 1 | //! A map of dense integer key to value. 2 | 3 | use std::marker::PhantomData; 4 | 5 | pub trait Index: From { 6 | fn index(&self) -> usize; 7 | } 8 | 9 | /// A map of a dense integer key to value, implemented as a vector. 10 | /// Effectively wraps Vec to provided typed keys. 11 | pub struct DenseMap { 12 | vec: Vec, 13 | key_type: std::marker::PhantomData, 14 | } 15 | 16 | impl Default for DenseMap { 17 | fn default() -> Self { 18 | DenseMap { 19 | vec: Vec::default(), 20 | key_type: PhantomData, 21 | } 22 | } 23 | } 24 | 25 | impl std::ops::Index for DenseMap { 26 | type Output = V; 27 | 28 | fn index(&self, k: K) -> &Self::Output { 29 | &self.vec[k.index()] 30 | } 31 | } 32 | 33 | impl std::ops::IndexMut for DenseMap { 34 | fn index_mut(&mut self, k: K) -> &mut Self::Output { 35 | &mut self.vec[k.index()] 36 | } 37 | } 38 | 39 | impl DenseMap { 40 | pub fn lookup(&self, k: K) -> Option<&V> { 41 | self.vec.get(k.index()) 42 | } 43 | 44 | pub fn next_id(&self) -> K { 45 | K::from(self.vec.len()) 46 | } 47 | 48 | pub fn push(&mut self, val: V) -> K { 49 | let id = self.next_id(); 50 | self.vec.push(val); 51 | id 52 | } 53 | } 54 | 55 | impl DenseMap { 56 | pub fn new_sized(n: K, default: V) -> Self { 57 | let mut m = Self::default(); 58 | m.vec.resize(n.index(), default); 59 | m 60 | } 61 | 62 | pub fn set_grow(&mut self, k: K, v: V, default: V) { 63 | if k.index() >= self.vec.len() { 64 | self.vec.resize(k.index() + 1, default); 65 | } 66 | self.vec[k.index()] = v 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /doc/development.md: -------------------------------------------------------------------------------- 1 | ## Git hook 2 | 3 | On a new checkout, run this to install the formatting check hook: 4 | 5 | ``` 6 | $ ln -s ../../git-pre-commit .git/hooks/pre-commit 7 | ``` 8 | 9 | ## Profiling 10 | 11 | ### gperftools 12 | 13 | I played with a few profilers, but I think the gperftools profiler turned out to 14 | be significantly better than the others. To install: 15 | 16 | ``` 17 | $ apt install libgoogle-perftools-dev 18 | $ go install github.com/google/pprof@latest 19 | ``` 20 | 21 | To use: 22 | 23 | ``` 24 | [possibly modify main.rs to make the app do more work than normal] 25 | $ LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libprofiler.so CPUPROFILE=p ./target/release/n2 ... 26 | $ pprof -http=:8080 ./target/release/n2 p 27 | ``` 28 | 29 | The web server it brings up shows an interactive graph, top functions, annotated 30 | code, disassembly... 31 | 32 | ### Mac 33 | 34 | ``` 35 | $ cargo instruments --release --template time --bin n2 -- -C ~/projects/llvm-project-16.0.0.src/build clang-format 36 | ``` 37 | 38 | TODO: notes on this vs `cargo flamegraph`. 39 | 40 | ### Windows 41 | 42 | It appears `perf` profiling of Rust under WSL2 is not a thing(?). 43 | 44 | ## Benchmarking 45 | 46 | ### Simple 47 | 48 | This benchmarks end-to-end n2 load time, by asking to build a nonexistent 49 | target: 50 | 51 | 1. `cargo install hyperfine` 52 | 2. `$ hyperfine -i -- './target/release/n2 -C ~/llvm-project/llvm/utils/gn/out/ xxx'` 53 | 54 | ### Micro 55 | 56 | There are microbenchmarks in the `benches/` directory. Run them with: 57 | 58 | ``` 59 | $ cargo bench 60 | ``` 61 | 62 | If there is a `build.ninja` in the `benches/` directory, the parsing benchmark 63 | will load it. For example, you can copy the `build.ninja` generated by the LLVM 64 | CMake build system (66mb on my system!). 65 | 66 | To run just the parsing benchmark: 67 | 68 | ``` 69 | $ cargo bench --bench parse -- parse 70 | ``` 71 | 72 | When iterating on benchmarks, it can help build time to disable `lto` in release 73 | mode by commenting out the `lto =` line in `Cargo.toml`. (On my system, `lto` is 74 | worth ~13% of parsing performance.) 75 | 76 | Read the test output in `target/criterion/report/index.html`. 77 | -------------------------------------------------------------------------------- /benches/parse.rs: -------------------------------------------------------------------------------- 1 | use divan::Bencher; 2 | use std::io::Write; 3 | use std::path::PathBuf; 4 | 5 | fn generate_build_ninja(statement_count: usize) -> Vec { 6 | let mut buf: Vec = Vec::new(); 7 | write!(buf, "rule cc\n command = touch $out",).unwrap(); 8 | for i in 0..statement_count { 9 | write!( 10 | buf, 11 | "build $out/foo/bar{}.o: cc $src/long/file/name{}.cc 12 | depfile = $out/foo/bar{}.o.d\n", 13 | i, i, i 14 | ) 15 | .unwrap(); 16 | } 17 | buf 18 | } 19 | 20 | mod parser { 21 | use super::*; 22 | use n2::parse::Parser; 23 | 24 | #[divan::bench] 25 | fn synthetic(bencher: Bencher) { 26 | let mut input = generate_build_ninja(1000); 27 | input.push(0); 28 | 29 | bencher.bench_local(|| { 30 | let mut parser = Parser::new(&input); 31 | while let Some(_) = parser.read().unwrap() {} 32 | }); 33 | } 34 | 35 | // This can take a while to run (~100ms per sample), so reduce total count. 36 | #[divan::bench(sample_size = 3, max_time = 1)] 37 | fn file(bencher: Bencher) { 38 | let input = match n2::scanner::read_file_with_nul("benches/build.ninja".as_ref()) { 39 | Ok(input) => input, 40 | Err(err) => { 41 | eprintln!("failed to read benches/build.ninja: {}", err); 42 | eprintln!("will skip benchmarking with real data"); 43 | return; 44 | } 45 | }; 46 | bencher.bench_local(|| { 47 | let mut parser = n2::parse::Parser::new(&input); 48 | while let Some(_) = parser.read().unwrap() {} 49 | }); 50 | } 51 | } 52 | 53 | #[divan::bench] 54 | fn load_synthetic(bencher: Bencher) { 55 | let mut input = generate_build_ninja(1000); 56 | input.push(0); 57 | bencher.bench_local(|| { 58 | let mut loader = n2::load::Loader::new(); 59 | let mut parser = n2::parse::Parser::new(&input); 60 | 61 | loader 62 | .parse_with_parser(&mut parser, PathBuf::from(""), &[]) 63 | .unwrap(); 64 | }); 65 | } 66 | 67 | fn main() { 68 | divan::main(); 69 | } 70 | -------------------------------------------------------------------------------- /benches/canon.rs: -------------------------------------------------------------------------------- 1 | use std::hint::black_box; 2 | 3 | use divan::Bencher; 4 | 5 | mod paths { 6 | pub const NOOP: &str = "examples/OrcV2Examples/OrcV2CBindingsVeryLazy/\ 7 | CMakeFiles/OrcV2CBindingsVeryLazy.dir/OrcV2CBindingsVeryLazy.c.o"; 8 | pub const PARENTS: &str = "examples/../OrcV2Examples/OrcV2CBindingsVeryLazy/../../../\ 9 | CMakeFiles/OrcV2CBindingsVeryLazy.dir/OrcV2CBindingsVeryLazy.c.o"; 10 | pub const ONE_DOT: &str = "examples/./OrcV2Examples/./OrcV2CBindingsVeryLazy/\ 11 | CMakeFiles/OrcV2CBindingsVeryLazy.dir/././OrcV2CBindingsVeryLazy.c.o"; 12 | pub const TWO_DOTS_IN_COMPONENT: &str = "examples/OrcV2Examples/OrcV2CBindingsVeryLazy/\ 13 | ..CMakeFiles/OrcV2CBindingsVeryLazy.dir/../OrcV2CBindingsVeryLazy.c.o"; 14 | } 15 | 16 | macro_rules! cases { 17 | () => { 18 | #[divan::bench] 19 | pub fn noop(b: Bencher) { 20 | run(b, paths::NOOP) 21 | } 22 | 23 | #[divan::bench] 24 | pub fn with_parents(b: Bencher) { 25 | run(b, paths::PARENTS) 26 | } 27 | 28 | #[divan::bench] 29 | pub fn with_one_dot(b: Bencher) { 30 | run(b, paths::ONE_DOT) 31 | } 32 | 33 | #[divan::bench] 34 | pub fn with_two_dots_in_component(b: Bencher) { 35 | run(b, paths::TWO_DOTS_IN_COMPONENT) 36 | } 37 | }; 38 | } 39 | 40 | mod inplace { 41 | use super::*; 42 | 43 | fn run(b: Bencher, path: &str) { 44 | b.with_inputs(|| path.to_string()).bench_values(|path| { 45 | let mut path = black_box(path); 46 | n2::canon::canonicalize_path(&mut path); 47 | // Return the String buffer, so that the deallocation is not benchmarked. 48 | black_box(path) 49 | }) 50 | } 51 | 52 | cases! {} 53 | } 54 | 55 | pub mod allocating { 56 | use super::*; 57 | 58 | fn run(b: Bencher, path: &str) { 59 | b.bench(|| { 60 | // Return the String buffer, so that the deallocation is not benchmarked. 61 | black_box(n2::canon::to_owned_canon_path(black_box(path))) 62 | }); 63 | } 64 | 65 | cases! {} 66 | } 67 | 68 | use divan::main; 69 | -------------------------------------------------------------------------------- /src/intern.rs: -------------------------------------------------------------------------------- 1 | //! String interning. 2 | //! Not currently used in n2. 3 | 4 | use hashbrown::raw::RawTable; 5 | use std::hash::Hasher; 6 | 7 | #[derive(Debug, Copy, Clone, Eq, PartialEq)] 8 | pub struct Symbol(usize); 9 | 10 | struct EndTab { 11 | data: Vec, 12 | ends: Vec, 13 | } 14 | 15 | impl EndTab { 16 | fn add(&mut self, s: &[u8]) -> Symbol { 17 | self.data.extend_from_slice(s); 18 | let sym = Symbol(self.ends.len()); 19 | self.ends.push(self.data.len()); 20 | sym 21 | } 22 | fn get(&self, sym: Symbol) -> &[u8] { 23 | let start = if sym.0 > 0 { self.ends[sym.0 - 1] } else { 0 }; 24 | let end = self.ends[sym.0]; 25 | &self.data[start..end] 26 | } 27 | } 28 | 29 | pub struct Intern { 30 | lookup: RawTable, 31 | endtab: EndTab, 32 | } 33 | 34 | fn hash_bytes(s: &[u8]) -> u64 { 35 | let mut hasher = ahash::AHasher::default(); 36 | hasher.write(s); 37 | hasher.finish() 38 | } 39 | 40 | impl Intern { 41 | pub fn new() -> Intern { 42 | Intern { 43 | lookup: RawTable::new(), 44 | endtab: EndTab { 45 | data: Vec::new(), 46 | ends: Vec::new(), 47 | }, 48 | } 49 | } 50 | 51 | pub fn add<'a>(&mut self, s: &'a [u8]) -> Symbol { 52 | let hash = hash_bytes(s); 53 | if let Some(sym) = self 54 | .lookup 55 | .get(hash, |sym: &Symbol| s == self.endtab.get(*sym)) 56 | { 57 | return *sym; 58 | } 59 | let sym = self.endtab.add(s); 60 | 61 | let endtab = &self.endtab; 62 | self.lookup 63 | .insert(hash, sym, |sym: &Symbol| hash_bytes(endtab.get(*sym))); 64 | sym 65 | } 66 | 67 | pub fn get(&self, sym: Symbol) -> &[u8] { 68 | self.endtab.get(sym) 69 | } 70 | } 71 | 72 | #[cfg(test)] 73 | mod tests { 74 | use super::*; 75 | 76 | #[test] 77 | fn user() { 78 | let mut i = Intern::new(); 79 | let hi = i.add("hi".as_bytes()); 80 | let yo = i.add("yo".as_bytes()); 81 | let hi2 = i.add("hi".as_bytes()); 82 | assert_eq!(hi, hi2); 83 | assert_ne!(hi, yo); 84 | 85 | assert_eq!(i.get(hi), "hi".as_bytes()); 86 | assert_eq!(i.get(yo), "yo".as_bytes()); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/smallmap.rs: -------------------------------------------------------------------------------- 1 | //! A map-like object for maps with few entries. 2 | //! TODO: this may not be needed at all, but the code used this pattern in a 3 | //! few places so I figured I may as well name it. 4 | 5 | use std::{borrow::Borrow, fmt::Debug}; 6 | 7 | /// A map-like object implemented as a list of pairs, for cases where the 8 | /// number of entries in the map is small. 9 | pub struct SmallMap(Vec<(K, V)>); 10 | 11 | impl Default for SmallMap { 12 | fn default() -> Self { 13 | SmallMap(Vec::default()) 14 | } 15 | } 16 | 17 | impl SmallMap { 18 | pub fn insert(&mut self, k: K, v: V) { 19 | for (ik, iv) in self.0.iter_mut() { 20 | if *ik == k { 21 | *iv = v; 22 | return; 23 | } 24 | } 25 | self.0.push((k, v)); 26 | } 27 | 28 | pub fn get(&self, q: &Q) -> Option<&V> 29 | where 30 | K: Borrow, 31 | Q: PartialEq + ?Sized, 32 | { 33 | for (k, v) in self.0.iter() { 34 | if k.borrow() == q { 35 | return Some(v); 36 | } 37 | } 38 | None 39 | } 40 | 41 | pub fn iter(&self) -> std::slice::Iter<'_, (K, V)> { 42 | self.0.iter() 43 | } 44 | 45 | pub fn iter_mut(&mut self) -> std::slice::IterMut<'_, (K, V)> { 46 | self.0.iter_mut() 47 | } 48 | 49 | pub fn into_iter(self) -> std::vec::IntoIter<(K, V)> { 50 | self.0.into_iter() 51 | } 52 | 53 | pub fn values(&self) -> impl Iterator + '_ { 54 | self.0.iter().map(|x| &x.1) 55 | } 56 | } 57 | 58 | impl std::convert::From<[(K, V); N]> for SmallMap { 59 | fn from(value: [(K, V); N]) -> Self { 60 | let mut result = SmallMap::default(); 61 | for (k, v) in value { 62 | result.insert(k, v); 63 | } 64 | result 65 | } 66 | } 67 | 68 | impl Debug for SmallMap { 69 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 70 | self.0.fmt(f) 71 | } 72 | } 73 | 74 | // Only for tests because it is order-sensitive 75 | #[cfg(test)] 76 | impl PartialEq for SmallMap { 77 | fn eq(&self, other: &Self) -> bool { 78 | return self.0 == other.0; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/progress_dumb.rs: -------------------------------------------------------------------------------- 1 | //! Build progress reporting for a "dumb" console, without any overprinting. 2 | 3 | use crate::progress::{build_message, Progress}; 4 | use crate::{ 5 | graph::Build, graph::BuildId, process::Termination, task::TaskResult, work::StateCounts, 6 | }; 7 | use std::cell::Cell; 8 | use std::io::Write; 9 | 10 | /// Progress implementation for "dumb" console, without any overprinting. 11 | #[derive(Default)] 12 | pub struct DumbConsoleProgress { 13 | /// Whether to print command lines of started programs. 14 | verbose: bool, 15 | 16 | /// The id of the last command printed, used to avoid printing it twice 17 | /// when we have two updates from the same command in a row. 18 | last_started: Cell>, 19 | } 20 | 21 | impl DumbConsoleProgress { 22 | pub fn new(verbose: bool) -> Self { 23 | Self { 24 | verbose, 25 | last_started: Default::default(), 26 | } 27 | } 28 | } 29 | 30 | impl Progress for DumbConsoleProgress { 31 | fn update(&self, _counts: &StateCounts) { 32 | // ignore 33 | } 34 | 35 | fn task_started(&self, id: BuildId, build: &Build) { 36 | self.log(if self.verbose { 37 | build.cmdline.as_ref().unwrap() 38 | } else { 39 | build_message(build) 40 | }); 41 | self.last_started.set(Some(id)); 42 | } 43 | 44 | fn task_output(&self, _id: BuildId, _line: Vec) { 45 | // ignore 46 | } 47 | 48 | fn task_finished(&self, id: BuildId, build: &Build, result: &TaskResult) { 49 | let mut hide_output = result.output.is_empty(); 50 | match result.termination { 51 | Termination::Success => { 52 | if result.output.is_empty() || self.last_started.get() == Some(id) { 53 | // Output is empty, or we just printed the command, don't print it again. 54 | } else { 55 | self.log(build_message(build)) 56 | } 57 | if build.hide_success { 58 | hide_output = true; 59 | } 60 | } 61 | Termination::Interrupted => self.log(&format!("interrupted: {}", build_message(build))), 62 | Termination::Failure => self.log(&format!("failed: {}", build_message(build))), 63 | }; 64 | if !hide_output { 65 | std::io::stdout().write_all(&result.output).unwrap(); 66 | } 67 | } 68 | 69 | fn log(&self, msg: &str) { 70 | println!("{}", msg); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/terminal.rs: -------------------------------------------------------------------------------- 1 | #[cfg(unix)] 2 | mod unix { 3 | pub fn use_fancy() -> bool { 4 | unsafe { 5 | libc::isatty(/* stdout */ 1) == 1 6 | } 7 | } 8 | 9 | pub fn get_cols() -> Option { 10 | if cfg!(miri) { 11 | return None; 12 | } 13 | unsafe { 14 | let mut winsize = std::mem::zeroed::(); 15 | if libc::ioctl(0, libc::TIOCGWINSZ, &mut winsize) < 0 { 16 | return None; 17 | } 18 | if winsize.ws_col < 10 { 19 | // https://github.com/evmar/n2/issues/63: ignore too-narrow widths 20 | return None; 21 | } 22 | Some(winsize.ws_col as usize) 23 | } 24 | } 25 | } 26 | 27 | #[cfg(unix)] 28 | pub use unix::*; 29 | 30 | #[cfg(windows)] 31 | mod windows { 32 | use windows_sys::Win32::{Foundation::*, System::Console::*}; 33 | 34 | pub fn use_fancy() -> bool { 35 | unsafe { 36 | let handle = GetStdHandle(STD_OUTPUT_HANDLE); 37 | let mut mode = 0; 38 | // Note: GetConsoleMode itself fails when not attached to a console. 39 | let ok = GetConsoleMode(handle, &mut mode) != 0; 40 | if ok { 41 | // Enable terminal processing so we can overwrite previous content. 42 | // Ignore errors. 43 | _ = SetConsoleMode(handle, mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING); 44 | } 45 | ok 46 | } 47 | } 48 | 49 | pub fn get_cols() -> Option { 50 | unsafe { 51 | let console = GetStdHandle(STD_OUTPUT_HANDLE); 52 | if console == INVALID_HANDLE_VALUE { 53 | return None; 54 | } 55 | let mut csbi = ::std::mem::zeroed::(); 56 | if GetConsoleScreenBufferInfo(console, &mut csbi) == 0 { 57 | return None; 58 | } 59 | if csbi.dwSize.X < 10 { 60 | // https://github.com/evmar/n2/issues/63: ignore too-narrow widths 61 | return None; 62 | } 63 | Some(csbi.dwSize.X as usize) 64 | } 65 | } 66 | } 67 | 68 | #[cfg(windows)] 69 | pub use windows::*; 70 | 71 | #[cfg(target_arch = "wasm32")] 72 | mod wasm { 73 | pub fn use_fancy() -> bool { 74 | false 75 | } 76 | 77 | pub fn get_cols() -> Option { 78 | None 79 | } 80 | } 81 | 82 | #[cfg(target_arch = "wasm32")] 83 | pub use wasm::*; 84 | -------------------------------------------------------------------------------- /tests/e2e/validations.rs: -------------------------------------------------------------------------------- 1 | //! Tests for the 'validations' feature, which are build edges marked with |@. 2 | 3 | use crate::e2e::*; 4 | 5 | #[test] 6 | fn basic_validation() -> anyhow::Result<()> { 7 | let space = TestSpace::new()?; 8 | space.write( 9 | "build.ninja", 10 | &[ 11 | TOUCH_RULE, 12 | "build my_validation: touch", 13 | "build out: touch |@ my_validation", 14 | "", 15 | ] 16 | .join("\n"), 17 | )?; 18 | space.run_expect(&mut n2_command(vec!["out"]))?; 19 | assert!(space.read("out").is_ok()); 20 | assert!(space.read("my_validation").is_ok()); 21 | Ok(()) 22 | } 23 | 24 | #[cfg(unix)] 25 | #[test] 26 | fn build_starts_before_validation_finishes() -> anyhow::Result<()> { 27 | // When a given target has a validation, that validation is part of the 28 | // overall build. But despite there being a build edge, the target shouldn't 29 | // wait for the validation. 30 | // To verify this, we make the validation command internally wait for the 31 | // target, effectively reversing the dependency order at runtime. 32 | let space = TestSpace::new()?; 33 | space.write( 34 | "build.ninja", 35 | " 36 | # Waits for the file $wait_for to exist, then touches $out. 37 | rule build_slow 38 | command = until [ -f $wait_for ]; do sleep 0.1; done; touch $out 39 | 40 | rule build_fast 41 | command = touch $out 42 | 43 | build out: build_fast regular_input |@ validation_input 44 | build regular_input: build_fast 45 | build validation_input: build_slow 46 | wait_for = out 47 | ", 48 | )?; 49 | space.run_expect(&mut n2_command(vec!["out"]))?; 50 | Ok(()) 51 | } 52 | 53 | #[cfg(unix)] 54 | #[test] 55 | fn build_fails_when_validation_fails() -> anyhow::Result<()> { 56 | let space = TestSpace::new()?; 57 | space.write( 58 | "build.ninja", 59 | " 60 | rule touch 61 | command = touch $out 62 | 63 | rule fail 64 | command = exit 1 65 | 66 | build out: touch |@ validation_input 67 | build validation_input: fail 68 | ", 69 | )?; 70 | let output = space.run(&mut n2_command(vec!["out"]))?; 71 | assert!(!output.status.success()); 72 | Ok(()) 73 | } 74 | 75 | #[test] 76 | fn validation_inputs_break_cycles() -> anyhow::Result<()> { 77 | let space = TestSpace::new()?; 78 | space.write( 79 | "build.ninja", 80 | &[ 81 | TOUCH_RULE, 82 | "build out: touch |@ validation_input", 83 | "build validation_input: touch out", 84 | "", 85 | ] 86 | .join("\n"), 87 | )?; 88 | space.run_expect(&mut n2_command(vec!["out"]))?; 89 | Ok(()) 90 | } 91 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # n2, an alternative ninja implementation 2 | 3 | ![CI status](https://github.com/evmar/n2/actions/workflows/ci.yml/badge.svg) 4 | 5 | n2 (pronounced "into") implements enough of [Ninja](https://ninja-build.org/) to 6 | successfully build some projects that build with Ninja. Compared to Ninja, n2 7 | missing some features but is faster to build and has a better UI; see 8 | [a more detailed comparison](doc/comparison.md). 9 | 10 | > [Here's a small demo](https://asciinema.org/a/F2E7a6nX4feoSSWVI4oFAm21T) of n2 11 | > building some of Clang. 12 | 13 | ## Install 14 | 15 | ``` 16 | $ cargo install --locked --git https://github.com/evmar/n2 17 | # (installs into ~/.cargo/bin/) 18 | # On Windows, add `--features crlf` to support files with CRLF linefeeds -- 19 | # costs 10% in file parse time. 20 | 21 | $ n2 -C some/build/dir some-target 22 | ``` 23 | 24 | ### Using with CMake 25 | 26 | When CMake generates Ninja files it attempts run a program named `ninja` with 27 | some particular Ninja behaviors. In particular, it attempts to inform Ninja/n2 28 | that its generated build files are up to date so that the build system doesn't 29 | attempt to rebuild them. 30 | 31 | n2 can emulate the expected CMake behavior when invoked as `ninja`. To do this 32 | you create a symlink named `ninja` somewhere in your `$PATH`, such that CMake 33 | can discover it. 34 | 35 | - UNIX: `ln -s path/to/n2 ninja` 36 | - Windows(cmd): `mklink ninja.exe path\to\n2` 37 | - Windows(PowerShell): `New-Item -Type Symlink ninja.exe -Target path\to\n2` 38 | 39 | > **Warning**\ 40 | > If you don't have Ninja installed at all, you must install such a symlink 41 | > because CMake attempts to invoke `ninja` itself! 42 | 43 | ## The console output 44 | 45 | While building, n2 displays build progress like this: 46 | 47 | ``` 48 | [=========================--------- ] 2772/4459 done, 8/930 running 49 | Building foo/bar (2s) 50 | Building foo/baz 51 | ``` 52 | 53 | The progress bar always covers all build steps needed for the targets, 54 | regardless of whether they need to be executed or not. 55 | 56 | The bar shows three categories of state: 57 | 58 | - **Done:** The `=` signs show the build steps that are already up to date. 59 | - **In progress:** The `-` signs show steps that are in-progress; if you had 60 | enough CPUs they would all be executing. The `8/930 running` after shows that 61 | n2 is currently executing 8 of the 930 available steps. 62 | - **Unknown:** The remaining empty space indicates steps whose status is yet to 63 | be known, as they depend on the in progress steps. For example, if an 64 | intermediate step doesn't write its outputs n2 may not need to execute the 65 | dependent steps. 66 | 67 | The lines below the progress bar show some build steps that are currrently 68 | running, along with how long they've been running if it has been a while. Their 69 | text is controlled by the input `build.ninja` file. 70 | 71 | ## More reading 72 | 73 | I wrote n2 to 74 | [explore some alternative ideas](http://neugierig.org/software/blog/2022/03/n2.html) 75 | I had around how to structure a build system. In a very real sense the 76 | exploration is more important than the actual software itself, so you can view 77 | the design notes as one of the primary artifacts of this. 78 | 79 | - [Design notes](doc/design_notes.md). 80 | - [Development tips](doc/development.md). 81 | - [Comparison with Ninja / missing features](doc/comparison.md). 82 | -------------------------------------------------------------------------------- /src/trace.rs: -------------------------------------------------------------------------------- 1 | //! Chrome trace output. 2 | 3 | use std::fs::File; 4 | use std::io::{BufWriter, Write}; 5 | use std::time::Instant; 6 | 7 | static mut TRACE: Option = None; 8 | 9 | pub struct Trace { 10 | start: Instant, 11 | w: BufWriter, 12 | count: usize, 13 | } 14 | 15 | impl Trace { 16 | fn new(path: &str) -> std::io::Result { 17 | let mut w = BufWriter::new(File::create(path)?); 18 | writeln!(w, "[")?; 19 | Ok(Trace { 20 | start: Instant::now(), 21 | w, 22 | count: 0, 23 | }) 24 | } 25 | 26 | fn write_event_prefix(&mut self, name: &str, ts: Instant) { 27 | if self.count > 0 { 28 | write!(self.w, ",").unwrap(); 29 | } 30 | self.count += 1; 31 | write!( 32 | self.w, 33 | "{{\"pid\":0, \"name\":{:?}, \"ts\":{}, ", 34 | name, 35 | ts.duration_since(self.start).as_micros(), 36 | ) 37 | .unwrap(); 38 | } 39 | 40 | pub fn write_complete(&mut self, name: &str, tid: usize, start: Instant, end: Instant) { 41 | self.write_event_prefix(name, start); 42 | writeln!( 43 | self.w, 44 | "\"tid\": {}, \"ph\":\"X\", \"dur\":{}}}", 45 | tid, 46 | end.duration_since(start).as_micros() 47 | ) 48 | .unwrap(); 49 | } 50 | 51 | /* 52 | These functions were useful when developing, but are currently unused. 53 | 54 | pub fn write_instant(&mut self, name: &str) { 55 | self.write_event_prefix(name, Instant::now()); 56 | writeln!(self.w, "\"ph\":\"i\"}}").unwrap(); 57 | } 58 | 59 | pub fn write_counts<'a>( 60 | &mut self, 61 | name: &str, 62 | counts: impl Iterator, 63 | ) { 64 | self.write_event_prefix(name, Instant::now()); 65 | write!(self.w, "\"ph\":\"C\", \"args\":{{").unwrap(); 66 | for (i, (name, count)) in counts.enumerate() { 67 | if i > 0 { 68 | write!(self.w, ",").unwrap(); 69 | } 70 | write!(self.w, "\"{}\":{}", name, count).unwrap(); 71 | } 72 | writeln!(self.w, "}}}}").unwrap(); 73 | } 74 | */ 75 | 76 | fn close(&mut self) { 77 | self.write_complete("main", 0, self.start, Instant::now()); 78 | writeln!(self.w, "]").unwrap(); 79 | self.w.flush().unwrap(); 80 | } 81 | } 82 | 83 | pub fn open(path: &str) -> std::io::Result<()> { 84 | let trace = Trace::new(path)?; 85 | // Safety: accessing global mut, not threadsafe. 86 | unsafe { 87 | TRACE = Some(trace); 88 | } 89 | Ok(()) 90 | } 91 | 92 | pub fn enabled() -> bool { 93 | // Safety: accessing global mut, not threadsafe. 94 | unsafe { matches!(TRACE, Some(_)) } 95 | } 96 | 97 | pub fn write_complete(name: &str, tid: usize, start: Instant, end: Instant) { 98 | // Safety: accessing global mut, not threadsafe. 99 | unsafe { 100 | if let Some(ref mut t) = TRACE { 101 | t.write_complete(name, tid, start, end); 102 | } 103 | } 104 | } 105 | 106 | pub fn scope(name: &'static str, f: impl FnOnce() -> T) -> T { 107 | let start = Instant::now(); 108 | let result = f(); 109 | let end = Instant::now(); 110 | write_complete(name, 0, start, end); 111 | result 112 | } 113 | 114 | pub fn close() { 115 | // Safety: accessing global mut, not threadsafe. 116 | unsafe { 117 | if let Some(ref mut t) = TRACE { 118 | t.close() 119 | } 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /tests/e2e/discovered.rs: -------------------------------------------------------------------------------- 1 | use crate::e2e::*; 2 | 3 | #[cfg(unix)] 4 | const GENDEP_RULE: &str = " 5 | rule gendep 6 | description = gendep $out 7 | command = echo \"$dep_content\" > $out.d && touch $out 8 | depfile = $out.d 9 | "; 10 | 11 | #[cfg(windows)] 12 | const GENDEP_RULE: &str = " 13 | rule gendep 14 | description = gendep $out 15 | command = cmd /c echo $dep_content > $out.d && type nul > $out 16 | depfile = $out.d 17 | "; 18 | 19 | /// depfile contains invalid syntax. 20 | #[test] 21 | fn bad_depfile() -> anyhow::Result<()> { 22 | let space = TestSpace::new()?; 23 | space.write( 24 | "build.ninja", 25 | &[ 26 | GENDEP_RULE, 27 | " 28 | build out: gendep 29 | dep_content = garbage text 30 | ", 31 | "", 32 | ] 33 | .join("\n"), 34 | )?; 35 | 36 | let out = space.run(&mut n2_command(vec!["out"]))?; 37 | assert_output_contains(&out, "parse error:"); 38 | Ok(()) 39 | } 40 | 41 | /// depfile contains reference to existing order-only dep. 42 | #[test] 43 | fn discover_existing_dep() -> anyhow::Result<()> { 44 | let space = TestSpace::new()?; 45 | space.write( 46 | "build.ninja", 47 | &[ 48 | GENDEP_RULE, 49 | TOUCH_RULE, 50 | "build in: touch", 51 | " 52 | build out: gendep || in 53 | dep_content = out: in 54 | ", 55 | "", 56 | ] 57 | .join("\n"), 58 | )?; 59 | 60 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 61 | assert_output_contains(&out, "ran 2 tasks"); 62 | 63 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 64 | assert_output_contains(&out, "no work"); 65 | 66 | // Even though out only has an order-only dep on 'in' in the build file, 67 | // we still should rebuild it due to the discovered dep on 'in'. 68 | space.write("in", "x")?; 69 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 70 | assert_output_contains(&out, "gendep out"); 71 | 72 | Ok(()) 73 | } 74 | 75 | #[cfg(unix)] 76 | #[test] 77 | fn multi_output_depfile() -> anyhow::Result<()> { 78 | let space = TestSpace::new()?; 79 | space.write( 80 | "build.ninja", 81 | " 82 | rule myrule 83 | command = echo \"out: foo\" > out.d && echo \"out2: foo2\" >> out.d && echo >> out.d && echo >> out.d && touch out out2 84 | depfile = out.d 85 | 86 | build out out2: myrule 87 | ", 88 | )?; 89 | space.write("foo", "")?; 90 | space.write("foo2", "")?; 91 | 92 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 93 | assert_output_contains(&out, "ran 1 task"); 94 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 95 | assert_output_contains(&out, "no work"); 96 | space.write("foo", "x")?; 97 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 98 | assert_output_contains(&out, "ran 1 task"); 99 | space.write("foo2", "x")?; 100 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 101 | assert_output_contains(&out, "ran 1 task"); 102 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 103 | assert_output_contains(&out, "no work"); 104 | Ok(()) 105 | } 106 | 107 | #[cfg(unix)] 108 | #[test] 109 | fn escaped_newline_in_depfile() -> anyhow::Result<()> { 110 | let space = TestSpace::new()?; 111 | space.write( 112 | "build.ninja", 113 | " 114 | rule myrule 115 | command = echo \"out: foo \\\\\" > out.d && echo \" foo2\" >> out.d && touch out 116 | depfile = out.d 117 | 118 | build out: myrule 119 | ", 120 | )?; 121 | space.write("foo", "")?; 122 | space.write("foo2", "")?; 123 | 124 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 125 | assert_output_contains(&out, "ran 1 task"); 126 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 127 | assert_output_contains(&out, "no work"); 128 | space.write("foo", "x")?; 129 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 130 | assert_output_contains(&out, "ran 1 task"); 131 | space.write("foo2", "x")?; 132 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 133 | assert_output_contains(&out, "ran 1 task"); 134 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 135 | assert_output_contains(&out, "no work"); 136 | Ok(()) 137 | } 138 | -------------------------------------------------------------------------------- /tests/e2e/bindings.rs: -------------------------------------------------------------------------------- 1 | //! Tests for behavior around variable bindings. 2 | 3 | use super::*; 4 | 5 | // Repro for issue #83. 6 | #[cfg(unix)] 7 | #[test] 8 | fn eval_twice() -> anyhow::Result<()> { 9 | let space = TestSpace::new()?; 10 | space.write( 11 | "build.ninja", 12 | &[ 13 | TOUCH_RULE, 14 | " 15 | var = 123 16 | rule custom 17 | command = $cmd $var 18 | build out: custom 19 | cmd = echo $var hello 20 | ", 21 | ] 22 | .join("\n"), 23 | )?; 24 | 25 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 26 | assert_output_contains(&out, "echo 123 hello 123"); 27 | Ok(()) 28 | } 29 | 30 | #[test] 31 | fn bad_rule_variable() -> anyhow::Result<()> { 32 | let space = TestSpace::new()?; 33 | space.write( 34 | "build.ninja", 35 | " 36 | rule my_rule 37 | command = touch $out 38 | my_var = foo 39 | 40 | build out: my_rule 41 | ", 42 | )?; 43 | 44 | let out = space.run(&mut n2_command(vec!["out"]))?; 45 | assert_output_contains(&out, "unexpected variable \"my_var\""); 46 | Ok(()) 47 | } 48 | 49 | #[cfg(unix)] 50 | #[test] 51 | fn deps_evaluate_build_bindings() -> anyhow::Result<()> { 52 | let space = TestSpace::new()?; 53 | space.write( 54 | "build.ninja", 55 | " 56 | rule touch 57 | command = touch $out 58 | rule copy 59 | command = cp $in $out 60 | build foo: copy ${my_dep} 61 | my_dep = bar 62 | build bar: touch 63 | ", 64 | )?; 65 | space.run_expect(&mut n2_command(vec!["foo"]))?; 66 | space.read("foo")?; 67 | Ok(()) 68 | } 69 | 70 | #[cfg(unix)] 71 | #[test] 72 | fn looks_up_values_from_build() -> anyhow::Result<()> { 73 | let space = TestSpace::new()?; 74 | space.write( 75 | "build.ninja", 76 | " 77 | rule copy_rspfile 78 | command = cp $out.rsp $out 79 | rspfile = $out.rsp 80 | 81 | build foo: copy_rspfile 82 | rspfile_content = Hello, world! 83 | ", 84 | )?; 85 | space.run_expect(&mut n2_command(vec!["foo"]))?; 86 | assert_eq!(space.read("foo")?, b"Hello, world!"); 87 | Ok(()) 88 | } 89 | 90 | #[cfg(unix)] 91 | #[test] 92 | fn build_bindings_arent_recursive() -> anyhow::Result<()> { 93 | let space = TestSpace::new()?; 94 | space.write( 95 | "build.ninja", 96 | " 97 | rule write_file 98 | command = echo $my_var > $out 99 | 100 | build foo: write_file 101 | my_var = Hello,$my_var_2 world! 102 | my_var_2 = my_var_2_value 103 | ", 104 | )?; 105 | space.run_expect(&mut n2_command(vec!["foo"]))?; 106 | assert_eq!(space.read("foo")?, b"Hello, world!\n"); 107 | Ok(()) 108 | } 109 | 110 | #[cfg(unix)] 111 | #[test] 112 | fn empty_variable_binding() -> anyhow::Result<()> { 113 | let space = TestSpace::new()?; 114 | space.write( 115 | "build.ninja", 116 | " 117 | empty_var = 118 | 119 | rule write_file 120 | command = echo $my_var > $out 121 | 122 | build foo: write_file 123 | my_var = Hello,$empty_var world! 124 | ", 125 | )?; 126 | space.run_expect(&mut n2_command(vec!["foo"]))?; 127 | assert_eq!(space.read("foo")?, b"Hello, world!\n"); 128 | Ok(()) 129 | } 130 | 131 | #[cfg(unix)] 132 | #[test] 133 | fn empty_build_variable() -> anyhow::Result<()> { 134 | let space = TestSpace::new()?; 135 | space.write( 136 | "build.ninja", 137 | " 138 | rule write_file 139 | command = echo $my_var > $out 140 | 141 | build foo: write_file 142 | empty = 143 | my_var = Hello, world! 144 | ", 145 | )?; 146 | space.run_expect(&mut n2_command(vec!["foo"]))?; 147 | assert_eq!(space.read("foo")?, b"Hello, world!\n"); 148 | Ok(()) 149 | } 150 | 151 | // Test for https://github.com/evmar/n2/issues/145: a variable in one file is visible 152 | // in an included file. 153 | #[test] 154 | fn across_files() -> anyhow::Result<()> { 155 | let space = TestSpace::new()?; 156 | space.write("world.txt", "")?; 157 | space.write( 158 | "build.ninja", 159 | &[ 160 | ECHO_RULE, 161 | " 162 | ext = txt 163 | include other.ninja 164 | ", 165 | ] 166 | .join("\n"), 167 | )?; 168 | space.write( 169 | "other.ninja", 170 | " 171 | build hello: echo world.$ext 172 | text = what a beautiful day 173 | ", 174 | )?; 175 | 176 | let out = space.run_expect(&mut n2_command(vec!["hello"]))?; 177 | assert_output_contains(&out, "what a beautiful day"); 178 | 179 | Ok(()) 180 | } 181 | -------------------------------------------------------------------------------- /tests/e2e/mod.rs: -------------------------------------------------------------------------------- 1 | //! Support code for e2e tests, which run n2 as a binary. 2 | 3 | mod basic; 4 | mod bindings; 5 | mod directories; 6 | mod discovered; 7 | mod missing; 8 | mod regen; 9 | mod validations; 10 | 11 | use anyhow::anyhow; 12 | 13 | pub fn n2_binary() -> std::path::PathBuf { 14 | std::env::current_exe() 15 | .expect("test binary path") 16 | .parent() 17 | .expect("test binary directory") 18 | .parent() 19 | .expect("binary directory") 20 | .join("n2") 21 | } 22 | 23 | pub fn n2_command(args: Vec<&str>) -> std::process::Command { 24 | let mut cmd = std::process::Command::new(n2_binary()); 25 | cmd.args(args); 26 | cmd 27 | } 28 | 29 | fn print_output(out: &std::process::Output) { 30 | // Gross: use print! instead of writing to stdout so Rust test 31 | // framework can capture it. 32 | print!("{}", std::str::from_utf8(&out.stdout).unwrap()); 33 | print!("{}", std::str::from_utf8(&out.stderr).unwrap()); 34 | } 35 | 36 | pub fn assert_output_contains(out: &std::process::Output, text: &str) { 37 | let out = std::str::from_utf8(&out.stdout).unwrap(); 38 | if !out.contains(text) { 39 | panic!( 40 | "assertion failed; expected output to contain {:?} but got:\n{}", 41 | text, out 42 | ); 43 | } 44 | } 45 | 46 | pub fn assert_output_not_contains(out: &std::process::Output, text: &str) { 47 | let out = std::str::from_utf8(&out.stdout).unwrap(); 48 | if out.contains(text) { 49 | panic!( 50 | "assertion failed; expected output to not contain {:?} but got:\n{}", 51 | text, out 52 | ); 53 | } 54 | } 55 | 56 | /// Manages a temporary directory for invoking n2. 57 | pub struct TestSpace { 58 | dir: tempfile::TempDir, 59 | } 60 | impl TestSpace { 61 | pub fn new() -> anyhow::Result { 62 | let dir = tempfile::tempdir()?; 63 | Ok(TestSpace { dir }) 64 | } 65 | 66 | /// Write a file into the working space. 67 | pub fn write(&self, path: &str, content: &str) -> std::io::Result<()> { 68 | std::fs::write(self.dir.path().join(path), content) 69 | } 70 | 71 | /// Read a file from the working space. 72 | pub fn read(&self, path: &str) -> anyhow::Result> { 73 | let path = self.dir.path().join(path); 74 | std::fs::read(&path).map_err(|err| anyhow!("read {}: {}", path.display(), err)) 75 | } 76 | 77 | pub fn metadata(&self, path: &str) -> std::io::Result { 78 | std::fs::metadata(self.dir.path().join(path)) 79 | } 80 | 81 | pub fn sub_mtime(&self, path: &str, dur: std::time::Duration) -> anyhow::Result<()> { 82 | let path = self.dir.path().join(path); 83 | let t = std::time::SystemTime::now() - dur; 84 | let f = std::fs::File::options().write(true).open(path)?; 85 | f.set_modified(t)?; 86 | Ok(()) 87 | } 88 | 89 | /// Invoke n2, returning process output. 90 | pub fn run(&self, cmd: &mut std::process::Command) -> std::io::Result { 91 | cmd.current_dir(self.dir.path()).output() 92 | } 93 | 94 | /// Like run, but also print output if the build failed. 95 | pub fn run_expect( 96 | &self, 97 | cmd: &mut std::process::Command, 98 | ) -> anyhow::Result { 99 | let out = self.run(cmd)?; 100 | if !out.status.success() { 101 | print_output(&out); 102 | anyhow::bail!("build failed, status {}", out.status); 103 | } 104 | Ok(out) 105 | } 106 | 107 | /// Persist the temp dir locally and abort the test. Debugging helper. 108 | #[allow(dead_code)] 109 | pub fn eject(self) -> ! { 110 | panic!("ejected at {:?}", self.dir.into_path()); 111 | } 112 | } 113 | 114 | // Ensure TOUCH_RULE has the same description and number of lines of text 115 | // on Windows/non-Windows to make tests agnostic to platform. 116 | 117 | #[cfg(unix)] 118 | pub const TOUCH_RULE: &str = " 119 | rule touch 120 | command = touch $out 121 | description = touch $out 122 | "; 123 | 124 | #[cfg(windows)] 125 | pub const TOUCH_RULE: &str = " 126 | rule touch 127 | command = cmd /c type nul > $out 128 | description = touch $out 129 | "; 130 | 131 | #[cfg(unix)] 132 | pub const ECHO_RULE: &str = " 133 | rule echo 134 | command = echo $text 135 | description = echo $out 136 | "; 137 | 138 | #[cfg(windows)] 139 | pub const ECHO_RULE: &str = " 140 | rule echo 141 | command = cmd /c echo $text 142 | description = echo $out 143 | "; 144 | -------------------------------------------------------------------------------- /tests/e2e/missing.rs: -------------------------------------------------------------------------------- 1 | //! Tests for behavior around missing files. 2 | 3 | use super::*; 4 | 5 | #[test] 6 | fn missing_input() -> anyhow::Result<()> { 7 | let space = TestSpace::new()?; 8 | space.write( 9 | "build.ninja", 10 | &[TOUCH_RULE, "build out: touch in", ""].join("\n"), 11 | )?; 12 | 13 | let out = space.run(&mut n2_command(vec!["out"]))?; 14 | assert_output_contains(&out, "input in missing"); 15 | 16 | Ok(()) 17 | } 18 | 19 | #[test] 20 | fn missing_generated() -> anyhow::Result<()> { 21 | let space = TestSpace::new()?; 22 | space.write( 23 | "build.ninja", 24 | &[ 25 | TOUCH_RULE, 26 | ECHO_RULE, 27 | "build mid: echo", // never writes output 28 | "build out: touch mid", // uses never-written output 29 | "", 30 | ] 31 | .join("\n"), 32 | )?; 33 | 34 | // https://github.com/evmar/n2/issues/69 35 | 36 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 37 | assert_output_contains(&out, "echo mid"); 38 | assert_output_contains(&out, "touch out"); 39 | 40 | Ok(()) 41 | } 42 | 43 | #[test] 44 | fn missing_phony() -> anyhow::Result<()> { 45 | let space = TestSpace::new()?; 46 | space.write( 47 | "build.ninja", 48 | &[ 49 | TOUCH_RULE, 50 | "build order_only: phony", // never writes output 51 | "build out: touch || order_only", // uses never-written output 52 | "", 53 | ] 54 | .join("\n"), 55 | )?; 56 | 57 | // https://github.com/evmar/n2/issues/69 58 | 59 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 60 | assert_output_contains(&out, "touch out"); 61 | 62 | Ok(()) 63 | } 64 | 65 | // Ensure we don't regress on 66 | // https://github.com/ninja-build/ninja/issues/1779 67 | // I can't remember the specific code CMake generates that relies on this; 68 | // I wonder if we can tighten the behavior at all. 69 | #[test] 70 | fn missing_phony_input() -> anyhow::Result<()> { 71 | let space = TestSpace::new()?; 72 | space.write( 73 | "build.ninja", 74 | &[TOUCH_RULE, "build out: phony || no_such_file", ""].join("\n"), 75 | )?; 76 | 77 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 78 | assert_output_contains(&out, "no work to do"); 79 | 80 | Ok(()) 81 | } 82 | 83 | #[test] 84 | fn phony_missing_file() -> anyhow::Result<()> { 85 | // https://ninja-build.org/manual.html#_the_literal_phony_literal_rule 86 | // build foo: phony 87 | // means "don't fail the build if foo doesn't exist, even in inputs" 88 | 89 | let space = TestSpace::new()?; 90 | space.write( 91 | "build.ninja", 92 | &[ 93 | TOUCH_RULE, 94 | "build out: touch | phony_file", 95 | "build phony_file: phony", 96 | "", 97 | ] 98 | .join("\n"), 99 | )?; 100 | 101 | // Expect the first build to generate some state... 102 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 103 | assert_output_contains(&out, "ran 1 task"); 104 | // ...but a second one should be up to date. 105 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 106 | // BUG: this should be 107 | // assert_output_contains(&out, "no work to do"); 108 | // but it is currently 109 | assert_output_contains(&out, "ran 1 task"); 110 | 111 | Ok(()) 112 | } 113 | 114 | #[test] 115 | fn phony_existing_file() -> anyhow::Result<()> { 116 | // Like phony_missing_file, but the file exists on disk. 117 | // https://github.com/evmar/n2/issues/40 118 | // CMake uses a phony rule targeted at a real file as a way of marking 119 | // "don't fail the build if this file is missing", but it had the consequence 120 | // of confusing our dirty-checking logic. 121 | 122 | let space = TestSpace::new()?; 123 | space.write( 124 | "build.ninja", 125 | &[ 126 | TOUCH_RULE, 127 | "build out: touch | phony_file", 128 | "build phony_file: phony", 129 | "", 130 | ] 131 | .join("\n"), 132 | )?; 133 | 134 | // Despite being a target of a phony rule, the file exists on disk. 135 | space.write("phony_file", "")?; 136 | 137 | // Expect the first build to generate some state... 138 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 139 | assert_output_contains(&out, "ran 1 task"); 140 | // ...but a second one should be up to date (#40 was that this ran again). 141 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 142 | assert_output_contains(&out, "no work to do"); 143 | 144 | Ok(()) 145 | } 146 | -------------------------------------------------------------------------------- /src/hash.rs: -------------------------------------------------------------------------------- 1 | //! A single hash over input attributes is recorded and used to determine when 2 | //! those inputs change. 3 | //! 4 | //! See "Manifests instead of mtime order" in 5 | //! https://neugierig.org/software/blog/2022/03/n2.html 6 | 7 | use crate::graph::{Build, FileId, FileState, GraphFiles, MTime, RspFile}; 8 | use std::{ 9 | collections::hash_map::DefaultHasher, 10 | fmt::Write, 11 | hash::{Hash, Hasher}, 12 | time::SystemTime, 13 | }; 14 | 15 | /// Hash value used to identify a given instance of a Build's execution; 16 | /// compared to verify whether a Build is up to date. 17 | #[derive(Debug, Copy, Clone, Eq, PartialEq)] 18 | pub struct BuildHash(pub u64); 19 | 20 | /// A trait for computing a build's manifest. Indirected as a trait so we can 21 | /// implement it a second time for "-d explain" debug purposes. 22 | trait Manifest { 23 | /// Write a list of files+mtimes. desc is used only for "-d explain" output. 24 | fn write_files( 25 | &mut self, 26 | desc: &str, 27 | files: &GraphFiles, 28 | file_state: &FileState, 29 | ids: &[FileId], 30 | ); 31 | fn write_rsp(&mut self, rspfile: &RspFile); 32 | fn write_cmdline(&mut self, cmdline: &str); 33 | } 34 | 35 | fn get_fileid_status<'a>( 36 | files: &'a GraphFiles, 37 | file_state: &FileState, 38 | id: FileId, 39 | ) -> (&'a str, SystemTime) { 40 | let name = &files.by_id[id].name; 41 | let mtime = file_state 42 | .get(id) 43 | .unwrap_or_else(|| panic!("no state for {:?}", name)); 44 | let mtime = match mtime { 45 | MTime::Stamp(mtime) => mtime, 46 | MTime::Missing => panic!("missing file: {:?}", name), 47 | }; 48 | (name.as_str(), mtime) 49 | } 50 | 51 | /// The BuildHasher used during normal builds, designed to not serialize too much. 52 | #[derive(Default)] 53 | struct TerseHash(DefaultHasher); 54 | 55 | const UNIT_SEPARATOR: u8 = 0x1F; 56 | 57 | impl TerseHash { 58 | fn write_string(&mut self, string: &str) { 59 | string.hash(&mut self.0); 60 | } 61 | 62 | fn write_separator(&mut self) { 63 | self.0.write_u8(UNIT_SEPARATOR); 64 | } 65 | 66 | fn finish(&mut self) -> BuildHash { 67 | BuildHash(self.0.finish()) 68 | } 69 | } 70 | 71 | impl Manifest for TerseHash { 72 | fn write_files<'a>( 73 | &mut self, 74 | _desc: &str, 75 | files: &GraphFiles, 76 | file_state: &FileState, 77 | ids: &[FileId], 78 | ) { 79 | for &id in ids { 80 | let (name, mtime) = get_fileid_status(files, file_state, id); 81 | self.write_string(name); 82 | mtime.hash(&mut self.0); 83 | } 84 | self.write_separator(); 85 | } 86 | 87 | fn write_cmdline(&mut self, cmdline: &str) { 88 | self.write_string(cmdline); 89 | self.write_separator(); 90 | } 91 | 92 | fn write_rsp(&mut self, rspfile: &RspFile) { 93 | rspfile.hash(&mut self.0); 94 | } 95 | } 96 | 97 | fn build_manifest( 98 | manifest: &mut M, 99 | files: &GraphFiles, 100 | file_state: &FileState, 101 | build: &Build, 102 | ) { 103 | manifest.write_files("in", files, file_state, build.dirtying_ins()); 104 | manifest.write_files("discovered", files, file_state, build.discovered_ins()); 105 | manifest.write_cmdline(build.cmdline.as_deref().unwrap_or("")); 106 | if let Some(rspfile) = &build.rspfile { 107 | manifest.write_rsp(rspfile); 108 | } 109 | manifest.write_files("out", files, file_state, build.outs()); 110 | } 111 | 112 | // Hashes the inputs of a build to compute a signature. 113 | // Prerequisite: all referenced files have already been stat()ed and are present. 114 | // (It doesn't make sense to hash a build with missing files, because it's out 115 | // of date regardless of the state of the other files.) 116 | pub fn hash_build(files: &GraphFiles, file_state: &FileState, build: &Build) -> BuildHash { 117 | let mut hasher = TerseHash::default(); 118 | build_manifest(&mut hasher, files, file_state, build); 119 | hasher.finish() 120 | } 121 | 122 | /// A BuildHasher that records human-readable text for "-d explain" debugging. 123 | #[derive(Default)] 124 | struct ExplainHash { 125 | text: String, 126 | } 127 | 128 | impl Manifest for ExplainHash { 129 | fn write_files<'a>( 130 | &mut self, 131 | desc: &str, 132 | files: &GraphFiles, 133 | file_state: &FileState, 134 | ids: &[FileId], 135 | ) { 136 | writeln!(&mut self.text, "{desc}:").unwrap(); 137 | for &id in ids { 138 | let (name, mtime) = get_fileid_status(files, file_state, id); 139 | let millis = mtime 140 | .duration_since(SystemTime::UNIX_EPOCH) 141 | .unwrap() 142 | .as_millis(); 143 | writeln!(&mut self.text, " {millis} {name}").unwrap(); 144 | } 145 | } 146 | 147 | fn write_rsp(&mut self, rspfile: &RspFile) { 148 | writeln!(&mut self.text, "rspfile path: {}", rspfile.path.display()).unwrap(); 149 | 150 | let mut h = DefaultHasher::new(); 151 | h.write(rspfile.content.as_bytes()); 152 | writeln!(&mut self.text, "rspfile hash: {:x}", h.finish()).unwrap(); 153 | } 154 | 155 | fn write_cmdline(&mut self, cmdline: &str) { 156 | writeln!(&mut self.text, "cmdline: {}", cmdline).unwrap(); 157 | } 158 | } 159 | 160 | /// Logs human-readable state of all the inputs used for hashing a given build. 161 | /// Used for "-d explain" debugging output. 162 | pub fn explain_hash_build(files: &GraphFiles, file_state: &FileState, build: &Build) -> String { 163 | let mut explainer = ExplainHash::default(); 164 | build_manifest(&mut explainer, files, file_state, build); 165 | explainer.text 166 | } 167 | -------------------------------------------------------------------------------- /tests/e2e/regen.rs: -------------------------------------------------------------------------------- 1 | //! Tests around regenerating the build.ninja file. 2 | 3 | use crate::e2e::*; 4 | 5 | #[cfg(unix)] 6 | #[test] 7 | fn generate_build_file() -> anyhow::Result<()> { 8 | // Run a project where a build rule generates the build.ninja. 9 | let space = TestSpace::new()?; 10 | space.write( 11 | "gen.sh", 12 | " 13 | echo 'regenerating build.ninja' 14 | cat >build.ninja < anyhow::Result<()> { 45 | // When we attempt to build build.ninja and it already up to date, 46 | // we attempt to reuse some build state. 47 | // Ensure a dependency shared by build.ninja and the desired target, 48 | // which itself has a build rule (here, phony) doesn't wedge the build. 49 | let space = TestSpace::new()?; 50 | let build_ninja = " 51 | rule regen 52 | command = cp build.ninja.in build.ninja 53 | description = regenerating 54 | generator = 1 55 | build build.ninja: regen | build.ninja.in sharedinput 56 | rule touch 57 | command = touch out 58 | build out: touch | sharedinput 59 | 60 | build sharedinput: phony 61 | "; 62 | space.write("build.ninja.in", build_ninja)?; 63 | space.write("build.ninja", build_ninja)?; 64 | // If this 'sharedinput' file doesn't exist, ninja will die after looping 65 | // 100 times(!). 66 | space.write("sharedinput", "")?; 67 | 68 | // Run: expect to regenerate because we don't know how the file was made. 69 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 70 | assert_output_contains(&out, "regenerating"); 71 | assert_output_contains(&out, "ran 2 tasks"); 72 | 73 | // Run: everything should be up to date. 74 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 75 | assert_output_not_contains(&out, "regenerating build.ninja"); 76 | assert_output_contains(&out, "no work"); 77 | 78 | Ok(()) 79 | } 80 | 81 | #[cfg(unix)] 82 | #[test] 83 | fn generate_specified_build_file() -> anyhow::Result<()> { 84 | // Run a project where a build rule generates specified_build.ninja. 85 | let space = TestSpace::new()?; 86 | space.write( 87 | "gen.sh", 88 | " 89 | echo 'regenerating specified_build.ninja' 90 | cat >specified_build.ninja < anyhow::Result<()> { 121 | // Run a project where a build rule generates the build.ninja but it fails. 122 | let space = TestSpace::new()?; 123 | space.write( 124 | "build.ninja", 125 | &[ 126 | TOUCH_RULE, 127 | "build out: touch", 128 | " 129 | rule regen 130 | command = sh ./gen.sh 131 | generator = 1", 132 | "build build.ninja: regen gen.sh", 133 | "", 134 | ] 135 | .join("\n"), 136 | )?; 137 | space.write("gen.sh", "exit 1")?; 138 | 139 | // Run: regenerate and fail. 140 | let out = space.run(&mut n2_command(vec!["out"]))?; 141 | assert_output_contains(&out, "failed:"); 142 | 143 | Ok(()) 144 | } 145 | 146 | /// Use "-t restat" to mark the build.ninja up to date ahead of time. 147 | #[cfg(unix)] // TODO: this ought to work on Windows, hrm. 148 | #[test] 149 | fn restat() -> anyhow::Result<()> { 150 | let space = TestSpace::new()?; 151 | space.write( 152 | "build.ninja", 153 | &[TOUCH_RULE, "build build.ninja: touch in", ""].join("\n"), 154 | )?; 155 | space.write("in", "")?; 156 | 157 | let out = space.run_expect(&mut n2_command(vec![ 158 | "-d", 159 | "ninja_compat", 160 | "-t", 161 | "restat", 162 | "build.ninja", 163 | "path_that_does_not_exist", // ninja doesn't check path existence 164 | ]))?; 165 | assert_output_not_contains(&out, "touch build.ninja"); 166 | 167 | // Building the build file should do nothing, because restat marked it up to date. 168 | let out = space.run_expect(&mut n2_command(vec!["build.ninja"]))?; 169 | assert_output_not_contains(&out, "touch build.ninja"); 170 | 171 | // But modifying the input should cause it to be up to date. 172 | space.write("in", "")?; 173 | let out = space.run_expect(&mut n2_command(vec!["build.ninja"]))?; 174 | assert_output_contains(&out, "touch build.ninja"); 175 | 176 | Ok(()) 177 | } 178 | -------------------------------------------------------------------------------- /src/eval.rs: -------------------------------------------------------------------------------- 1 | //! Represents parsed Ninja strings with embedded variable references, e.g. 2 | //! `c++ $in -o $out`, and mechanisms for expanding those into plain strings. 3 | 4 | use rustc_hash::FxHashMap; 5 | 6 | use crate::smallmap::SmallMap; 7 | use std::borrow::Borrow; 8 | use std::borrow::Cow; 9 | 10 | /// An environment providing a mapping of variable name to variable value. 11 | /// This represents one "frame" of evaluation context, a given EvalString may 12 | /// need multiple environments in order to be fully expanded. 13 | pub trait Env { 14 | fn get_var(&self, var: &str) -> Option>>; 15 | } 16 | 17 | /// One token within an EvalString, either literal text or a variable reference. 18 | #[derive(Debug, Clone, PartialEq)] 19 | pub enum EvalPart> { 20 | Literal(T), 21 | VarRef(T), 22 | } 23 | 24 | /// A parsed but unexpanded variable-reference string, e.g. "cc $in -o $out". 25 | /// This is generic to support EvalString<&str>, which is used for immediately- 26 | /// expanded evals, like top-level bindings, and EvalString, which is 27 | /// used for delayed evals like in `rule` blocks. 28 | #[derive(Debug, PartialEq)] 29 | pub struct EvalString>(Vec>); 30 | impl> EvalString { 31 | pub fn new(parts: Vec>) -> Self { 32 | EvalString(parts) 33 | } 34 | 35 | fn evaluate_inner(&self, result: &mut String, envs: &[&dyn Env]) { 36 | for part in &self.0 { 37 | match part { 38 | EvalPart::Literal(s) => result.push_str(s.as_ref()), 39 | EvalPart::VarRef(v) => { 40 | for (i, env) in envs.iter().enumerate() { 41 | if let Some(v) = env.get_var(v.as_ref()) { 42 | v.evaluate_inner(result, &envs[i + 1..]); 43 | break; 44 | } 45 | } 46 | } 47 | } 48 | } 49 | } 50 | 51 | fn calc_evaluated_length(&self, envs: &[&dyn Env]) -> usize { 52 | self.0 53 | .iter() 54 | .map(|part| match part { 55 | EvalPart::Literal(s) => s.as_ref().len(), 56 | EvalPart::VarRef(v) => { 57 | for (i, env) in envs.iter().enumerate() { 58 | if let Some(v) = env.get_var(v.as_ref()) { 59 | return v.calc_evaluated_length(&envs[i + 1..]); 60 | } 61 | } 62 | 0 63 | } 64 | }) 65 | .sum() 66 | } 67 | 68 | /// evalulate turns the EvalString into a regular String, looking up the 69 | /// values of variable references in the provided Envs. It will look up 70 | /// its variables in the earliest Env that has them, and then those lookups 71 | /// will be recursively expanded starting from the env after the one that 72 | /// had the first successful lookup. 73 | pub fn evaluate(&self, envs: &[&dyn Env]) -> String { 74 | let mut result = String::new(); 75 | result.reserve(self.calc_evaluated_length(envs)); 76 | self.evaluate_inner(&mut result, envs); 77 | result 78 | } 79 | } 80 | 81 | impl EvalString<&str> { 82 | pub fn into_owned(self) -> EvalString { 83 | EvalString( 84 | self.0 85 | .into_iter() 86 | .map(|part| match part { 87 | EvalPart::Literal(s) => EvalPart::Literal(s.to_owned()), 88 | EvalPart::VarRef(s) => EvalPart::VarRef(s.to_owned()), 89 | }) 90 | .collect(), 91 | ) 92 | } 93 | } 94 | 95 | impl EvalString { 96 | pub fn as_cow(&self) -> EvalString> { 97 | EvalString( 98 | self.0 99 | .iter() 100 | .map(|part| match part { 101 | EvalPart::Literal(s) => EvalPart::Literal(Cow::Borrowed(s.as_ref())), 102 | EvalPart::VarRef(s) => EvalPart::VarRef(Cow::Borrowed(s.as_ref())), 103 | }) 104 | .collect(), 105 | ) 106 | } 107 | } 108 | 109 | impl EvalString<&str> { 110 | pub fn as_cow(&self) -> EvalString> { 111 | EvalString( 112 | self.0 113 | .iter() 114 | .map(|part| match part { 115 | EvalPart::Literal(s) => EvalPart::Literal(Cow::Borrowed(*s)), 116 | EvalPart::VarRef(s) => EvalPart::VarRef(Cow::Borrowed(*s)), 117 | }) 118 | .collect(), 119 | ) 120 | } 121 | } 122 | 123 | /// A single scope's worth of variable definitions. 124 | #[derive(Debug, Default)] 125 | pub struct Vars<'text>(FxHashMap<&'text str, String>); 126 | 127 | impl<'text> Vars<'text> { 128 | pub fn insert(&mut self, key: &'text str, val: String) { 129 | self.0.insert(key, val); 130 | } 131 | 132 | pub fn get(&self, key: &str) -> Option<&String> { 133 | self.0.get(key) 134 | } 135 | 136 | pub fn get_all(&self) -> &FxHashMap<&'text str, String> { 137 | &self.0 138 | } 139 | } 140 | 141 | impl<'a> Env for Vars<'a> { 142 | fn get_var(&self, var: &str) -> Option>> { 143 | Some(EvalString::new(vec![EvalPart::Literal( 144 | std::borrow::Cow::Borrowed(self.get(var)?), 145 | )])) 146 | } 147 | } 148 | 149 | impl + PartialEq> Env for SmallMap> { 150 | fn get_var(&self, var: &str) -> Option>> { 151 | Some(self.get(var)?.as_cow()) 152 | } 153 | } 154 | 155 | impl + PartialEq> Env for SmallMap> { 156 | fn get_var(&self, var: &str) -> Option>> { 157 | Some(self.get(var)?.as_cow()) 158 | } 159 | } 160 | 161 | impl Env for SmallMap<&str, String> { 162 | fn get_var(&self, var: &str) -> Option>> { 163 | Some(EvalString::new(vec![EvalPart::Literal( 164 | std::borrow::Cow::Borrowed(self.get(var)?), 165 | )])) 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /src/scanner.rs: -------------------------------------------------------------------------------- 1 | //! Scans an input string (source file) character by character. 2 | 3 | use std::{io::Read, path::Path}; 4 | 5 | #[derive(Debug)] 6 | pub struct ParseError { 7 | msg: String, 8 | ofs: usize, 9 | } 10 | pub type ParseResult = Result; 11 | 12 | pub struct Scanner<'a> { 13 | buf: &'a [u8], 14 | pub ofs: usize, 15 | pub line: usize, 16 | } 17 | 18 | impl<'a> Scanner<'a> { 19 | pub fn new(buf: &'a [u8]) -> Self { 20 | if !buf.ends_with(b"\0") { 21 | panic!("Scanner requires nul-terminated buf"); 22 | } 23 | Scanner { 24 | buf, 25 | ofs: 0, 26 | line: 1, 27 | } 28 | } 29 | 30 | pub fn slice(&self, start: usize, end: usize) -> &'a str { 31 | unsafe { std::str::from_utf8_unchecked(self.buf.get_unchecked(start..end)) } 32 | } 33 | 34 | /// Assert the current position points at a \r\n pair. 35 | /// Used to skip over \r\n pairs in the input. 36 | #[cfg(feature = "crlf")] 37 | #[track_caller] 38 | fn assert_crlf(&self) { 39 | assert!(self.ofs < self.buf.len() - 2); 40 | assert!(self.buf[self.ofs] == b'\r'); 41 | assert!(self.buf[self.ofs + 1] == b'\n'); 42 | } 43 | 44 | fn get(&self) -> char { 45 | unsafe { *self.buf.get_unchecked(self.ofs) as char } 46 | } 47 | 48 | pub fn peek(&self) -> char { 49 | let c = self.get(); 50 | #[cfg(feature = "crlf")] 51 | if c == '\r' { 52 | self.assert_crlf(); 53 | return '\n'; 54 | } 55 | c 56 | } 57 | 58 | pub fn next(&mut self) { 59 | self.read(); 60 | } 61 | 62 | pub fn back(&mut self) { 63 | if self.ofs == 0 { 64 | panic!("back at start") 65 | } 66 | self.ofs -= 1; 67 | if self.get() == '\n' { 68 | if self.ofs > 0 && self.buf[self.ofs - 1] == b'\r' { 69 | self.ofs -= 1; 70 | } 71 | self.line -= 1; 72 | } 73 | } 74 | 75 | pub fn read(&mut self) -> char { 76 | #[allow(unused_mut)] 77 | let mut c = self.get(); 78 | #[cfg(feature = "crlf")] 79 | if c == '\r' { 80 | self.assert_crlf(); 81 | self.ofs += 1; 82 | c = '\n'; 83 | } 84 | if c == '\n' { 85 | self.line += 1; 86 | } 87 | if self.ofs == self.buf.len() { 88 | panic!("scanned past end") 89 | } 90 | self.ofs += 1; 91 | c 92 | } 93 | 94 | pub fn skip(&mut self, ch: char) -> bool { 95 | if self.read() != ch { 96 | self.back(); 97 | return false; 98 | } 99 | true 100 | } 101 | 102 | pub fn skip_spaces(&mut self) { 103 | while self.skip(' ') {} 104 | } 105 | 106 | pub fn expect(&mut self, ch: char) -> ParseResult<()> { 107 | let r = self.read(); 108 | if r != ch { 109 | self.back(); 110 | return self.parse_error(format!("expected {:?}, got {:?}", ch, r)); 111 | } 112 | Ok(()) 113 | } 114 | 115 | pub fn parse_error>(&self, msg: S) -> ParseResult { 116 | Err(ParseError { 117 | msg: msg.into(), 118 | ofs: self.ofs, 119 | }) 120 | } 121 | 122 | pub fn format_parse_error(&self, filename: &Path, err: ParseError) -> String { 123 | let mut ofs = 0; 124 | let lines = self.buf.split(|&c| c == b'\n'); 125 | for (line_number, line) in lines.enumerate() { 126 | if ofs + line.len() >= err.ofs { 127 | let mut msg = "parse error: ".to_string(); 128 | msg.push_str(&err.msg); 129 | msg.push('\n'); 130 | 131 | let prefix = format!("{}:{}: ", filename.display(), line_number + 1); 132 | msg.push_str(&prefix); 133 | 134 | let mut context = unsafe { std::str::from_utf8_unchecked(line) }; 135 | let mut col = err.ofs - ofs; 136 | if col > 40 { 137 | // Trim beginning of line to fit it on screen. 138 | msg.push_str("..."); 139 | context = &context[col - 20..]; 140 | col = 3 + 20; 141 | } 142 | if context.len() > 40 { 143 | context = &context[0..40]; 144 | msg.push_str(context); 145 | msg.push_str("..."); 146 | } else { 147 | msg.push_str(context); 148 | } 149 | msg.push('\n'); 150 | 151 | msg.push_str(&" ".repeat(prefix.len() + col)); 152 | msg.push_str("^\n"); 153 | return msg; 154 | } 155 | ofs += line.len() + 1; 156 | } 157 | panic!("invalid offset when formatting error") 158 | } 159 | } 160 | 161 | /// Scanner wants its input buffer to end in a trailing nul. 162 | /// This function is like std::fs::read() but appends a nul, efficiently. 163 | pub fn read_file_with_nul(path: &Path) -> std::io::Result> { 164 | // Using std::fs::read() to read the file and then pushing a nul on the end 165 | // causes us to allocate a buffer the size of the file, then grow it to push 166 | // the nul, copying the entire file(!). So instead create a buffer of the 167 | // right size up front. 168 | let mut file = std::fs::File::open(path)?; 169 | let size = file.metadata()?.len() as usize; 170 | let mut bytes = Vec::with_capacity(size + 1); 171 | unsafe { 172 | bytes.set_len(size); 173 | } 174 | file.read_exact(&mut bytes[..size])?; 175 | bytes.push(0); 176 | Ok(bytes) 177 | } 178 | 179 | #[cfg(test)] 180 | mod tests { 181 | use super::*; 182 | 183 | #[test] 184 | fn scanner() { 185 | let buf = b"1\n12\n\0"; 186 | let mut s = Scanner::new(buf); 187 | assert_eq!(s.peek(), '1'); 188 | s.next(); 189 | assert_eq!(s.read(), '\n'); 190 | assert_eq!(s.line, 2); 191 | assert_eq!(s.peek(), '1'); 192 | 193 | s.back(); 194 | assert_eq!(s.line, 1); 195 | assert_eq!(s.read(), '\n'); 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /src/depfile.rs: -------------------------------------------------------------------------------- 1 | //! Parsing of Makefile syntax as found in `.d` files emitted by C compilers. 2 | 3 | use crate::{ 4 | scanner::{ParseResult, Scanner}, 5 | smallmap::SmallMap, 6 | }; 7 | 8 | /// Skip spaces and backslashed newlines. 9 | fn skip_spaces(scanner: &mut Scanner) -> ParseResult<()> { 10 | loop { 11 | match scanner.read() { 12 | ' ' => {} 13 | '\\' => match scanner.read() { 14 | '\n' => {} 15 | _ => return scanner.parse_error("invalid backslash escape"), 16 | }, 17 | _ => { 18 | scanner.back(); 19 | break; 20 | } 21 | } 22 | } 23 | Ok(()) 24 | } 25 | 26 | /// Read one path from the input scanner. 27 | /// Note: treats colon as a valid character in a path because of Windows-style 28 | /// paths, but this means that the inital `output: ...` path will include the 29 | /// trailing colon. 30 | fn read_path<'a>(scanner: &mut Scanner<'a>) -> ParseResult> { 31 | skip_spaces(scanner)?; 32 | let start = scanner.ofs; 33 | loop { 34 | match scanner.read() { 35 | '\0' | ' ' | '\n' => { 36 | scanner.back(); 37 | break; 38 | } 39 | '\\' => { 40 | if scanner.peek() == '\n' { 41 | scanner.back(); 42 | break; 43 | } 44 | } 45 | _ => {} 46 | } 47 | } 48 | let end = scanner.ofs; 49 | if end == start { 50 | return Ok(None); 51 | } 52 | Ok(Some(scanner.slice(start, end))) 53 | } 54 | 55 | /// Parse a `.d` file into `Deps`. 56 | pub fn parse<'a>(scanner: &mut Scanner<'a>) -> ParseResult>> { 57 | let mut result = SmallMap::default(); 58 | loop { 59 | while matches!(scanner.peek(), ' ' | '\n') { 60 | scanner.next(); 61 | } 62 | let target = match read_path(scanner)? { 63 | None => break, 64 | Some(o) => o, 65 | }; 66 | scanner.skip_spaces(); 67 | let target = match target.strip_suffix(':') { 68 | None => { 69 | scanner.expect(':')?; 70 | target 71 | } 72 | Some(target) => target, 73 | }; 74 | let mut deps = Vec::new(); 75 | while let Some(p) = read_path(scanner)? { 76 | deps.push(p); 77 | } 78 | result.insert(target, deps); 79 | } 80 | scanner.expect('\0')?; 81 | 82 | Ok(result) 83 | } 84 | 85 | #[cfg(test)] 86 | mod tests { 87 | use super::*; 88 | use std::path::Path; 89 | 90 | fn try_parse(buf: &mut Vec) -> Result>, String> { 91 | buf.push(0); 92 | let mut scanner = Scanner::new(buf); 93 | parse(&mut scanner).map_err(|err| scanner.format_parse_error(Path::new("test"), err)) 94 | } 95 | 96 | fn must_parse(buf: &mut Vec) -> SmallMap<&str, Vec<&str>> { 97 | match try_parse(buf) { 98 | Err(err) => { 99 | println!("{}", err); 100 | panic!("failed parse"); 101 | } 102 | Ok(d) => d, 103 | } 104 | } 105 | 106 | fn test_for_crlf(input: &str, test: fn(String)) { 107 | test(input.to_string()); 108 | if cfg!(feature = "crlf") { 109 | let crlf = input.replace('\n', "\r\n"); 110 | test(crlf); 111 | } 112 | } 113 | 114 | #[test] 115 | fn test_parse_simple() { 116 | test_for_crlf( 117 | "build/browse.o: src/browse.cc src/browse.h build/browse_py.h\n", 118 | |text| { 119 | let mut file = text.into_bytes(); 120 | let deps = must_parse(&mut file); 121 | assert_eq!( 122 | deps, 123 | SmallMap::from([( 124 | "build/browse.o", 125 | vec!["src/browse.cc", "src/browse.h", "build/browse_py.h",] 126 | )]) 127 | ); 128 | }, 129 | ); 130 | } 131 | 132 | #[test] 133 | fn test_parse_space_suffix() { 134 | test_for_crlf("build/browse.o: src/browse.cc \n", |text| { 135 | let mut file = text.into_bytes(); 136 | let deps = must_parse(&mut file); 137 | assert_eq!( 138 | deps, 139 | SmallMap::from([("build/browse.o", vec!["src/browse.cc",])]) 140 | ); 141 | }); 142 | } 143 | 144 | #[test] 145 | fn test_parse_multiline() { 146 | test_for_crlf( 147 | "build/browse.o: src/browse.cc\\\n build/browse_py.h", 148 | |text| { 149 | let mut file = text.into_bytes(); 150 | let deps = must_parse(&mut file); 151 | assert_eq!( 152 | deps, 153 | SmallMap::from([( 154 | "build/browse.o", 155 | vec!["src/browse.cc", "build/browse_py.h",] 156 | )]) 157 | ); 158 | }, 159 | ); 160 | } 161 | 162 | #[test] 163 | fn test_parse_without_final_newline() { 164 | let mut file = b"build/browse.o: src/browse.cc".to_vec(); 165 | let deps = must_parse(&mut file); 166 | assert_eq!( 167 | deps, 168 | SmallMap::from([("build/browse.o", vec!["src/browse.cc",])]) 169 | ); 170 | } 171 | 172 | #[test] 173 | fn test_parse_spaces_before_colon() { 174 | let mut file = b"build/browse.o : src/browse.cc".to_vec(); 175 | let deps = must_parse(&mut file); 176 | assert_eq!( 177 | deps, 178 | SmallMap::from([("build/browse.o", vec!["src/browse.cc",])]) 179 | ); 180 | } 181 | 182 | #[test] 183 | fn test_parse_windows_dep_path() { 184 | let mut file = b"odd/path.o: C:/odd\\path.c".to_vec(); 185 | let deps = must_parse(&mut file); 186 | assert_eq!( 187 | deps, 188 | SmallMap::from([("odd/path.o", vec!["C:/odd\\path.c",])]) 189 | ); 190 | } 191 | 192 | #[test] 193 | fn test_parse_multiple_targets() { 194 | let mut file = b" 195 | out/a.o: src/a.c \\ 196 | src/b.c 197 | 198 | out/b.o : 199 | " 200 | .to_vec(); 201 | let deps = must_parse(&mut file); 202 | assert_eq!( 203 | deps, 204 | SmallMap::from([ 205 | ("out/a.o", vec!["src/a.c", "src/b.c",]), 206 | ("out/b.o", vec![]) 207 | ]) 208 | ); 209 | } 210 | 211 | #[test] 212 | fn test_parse_missing_colon() { 213 | let mut file = b"foo bar".to_vec(); 214 | let err = try_parse(&mut file).unwrap_err(); 215 | assert!( 216 | err.starts_with("parse error: expected ':'"), 217 | "expected parse error, got {:?}", 218 | err 219 | ); 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /src/canon.rs: -------------------------------------------------------------------------------- 1 | //! Path canonicalization. 2 | 3 | use std::hint::assert_unchecked; 4 | use std::mem::MaybeUninit; 5 | 6 | /// An on-stack stack of values. 7 | /// Used for tracking locations of parent components within a path. 8 | struct StackStack { 9 | n: usize, 10 | vals: [MaybeUninit; CAPACITY], 11 | } 12 | 13 | impl StackStack { 14 | fn new() -> Self { 15 | StackStack { 16 | n: 0, 17 | vals: [MaybeUninit::uninit(); CAPACITY], 18 | } 19 | } 20 | 21 | fn push(&mut self, val: T) { 22 | if self.n >= self.vals.len() { 23 | panic!("too many path components"); 24 | } 25 | self.vals[self.n].write(val); 26 | self.n += 1; 27 | } 28 | 29 | fn pop(&mut self) -> Option { 30 | if self.n > 0 { 31 | self.n -= 1; 32 | // Safety: we only access vals[i] after setting it. 33 | Some(unsafe { self.vals[self.n].assume_init() }) 34 | } else { 35 | None 36 | } 37 | } 38 | } 39 | 40 | /// Lexically canonicalize a path, removing redundant components. 41 | /// Does not access the disk, but only simplifies things like 42 | /// "foo/./bar" => "foo/bar". 43 | /// These paths can show up due to variable expansion in particular. 44 | pub fn canonicalize_path(path: &mut String) { 45 | assert!(!path.is_empty()); 46 | let mut components = StackStack::::new(); 47 | 48 | // Safety: we will modify the string by removing some ASCII characters in place 49 | // and shifting other contents left to fill the gaps, 50 | // so if it was valid UTF-8, it will remain that way. 51 | let data = unsafe { path.as_mut_vec() }; 52 | // Invariant: dst <= src <= data.len() 53 | let mut dst = 0; 54 | let mut src = 0; 55 | 56 | if let Some(b'/' | b'\\') = data.get(src) { 57 | src += 1; 58 | dst += 1; 59 | }; 60 | 61 | // One iteration per path component. 62 | while let Some(¤t) = data.get(src) { 63 | // Peek ahead for special path components: "/", ".", and "..". 64 | match current { 65 | b'/' | b'\\' => { 66 | src += 1; 67 | continue; 68 | } 69 | b'.' => { 70 | let Some(&next) = data.get(src + 1) else { 71 | break; // Trailing '.', trim. 72 | }; 73 | match next { 74 | b'/' | b'\\' => { 75 | // "./", skip. 76 | src += 2; 77 | continue; 78 | } 79 | // ".." 80 | b'.' => match data.get(src + 2) { 81 | None | Some(b'/' | b'\\') => { 82 | // ".." component, try to back up. 83 | if let Some(ofs) = components.pop() { 84 | dst = ofs; 85 | } else { 86 | // Safety: our invariant is dst <= src and we are inside a branch, 87 | // where even src + 2 < data.len() 88 | unsafe { assert_unchecked(dst < data.len()) }; 89 | data[dst] = b'.'; 90 | dst += 1; 91 | // Safety: see above 92 | unsafe { assert_unchecked(dst < data.len()) }; 93 | data[dst] = b'.'; 94 | dst += 1; 95 | if let Some(sep) = data.get(src + 2) { 96 | // Safety: see above 97 | unsafe { assert_unchecked(dst < data.len()) }; 98 | data[dst] = *sep; 99 | dst += 1; 100 | } 101 | } 102 | src += 3; 103 | continue; 104 | } 105 | _ => { 106 | // Component that happens to start with "..". 107 | // Handle as an ordinary component. 108 | } 109 | }, 110 | _ => {} 111 | } 112 | } 113 | _ => {} 114 | } 115 | 116 | // Mark this point as a possible target to pop to. 117 | components.push(dst); 118 | 119 | // Copy one path component, including trailing '/'. 120 | let stop = match data[src..].iter().position(|c| matches!(c, b'/' | b'\\')) { 121 | Some(pos) => src + pos + 1, 122 | None => data.len(), 123 | }; 124 | // Safety: dst <= src is out invariant, src <= stop <= data.len() by construction 125 | unsafe { assert_unchecked(dst <= src && src <= stop && stop <= data.len()) }; 126 | data.copy_within(src..stop, dst); 127 | dst += stop - src; 128 | src = stop; 129 | } 130 | 131 | if dst == 0 { 132 | data[0] = b'.'; 133 | dst = 1; 134 | } 135 | // Safety: dst <= src <= len 136 | unsafe { data.set_len(dst) }; 137 | } 138 | 139 | #[must_use = "this methods returns the canonicalized version; if possible, prefer `canonicalize_path`"] 140 | pub fn to_owned_canon_path(path: impl Into) -> String { 141 | let mut path = path.into(); 142 | canonicalize_path(&mut path); 143 | path 144 | } 145 | 146 | #[cfg(test)] 147 | mod tests { 148 | use super::*; 149 | 150 | // Assert that canon path equals expected path with different path separators 151 | #[track_caller] 152 | fn assert_canon_path_eq(left: &str, right: &str) { 153 | assert_eq!(to_owned_canon_path(left), right); 154 | assert_eq!( 155 | to_owned_canon_path(left.replace('/', "\\")), 156 | right.replace('/', "\\") 157 | ); 158 | } 159 | 160 | #[test] 161 | fn noop() { 162 | assert_canon_path_eq("foo", "foo"); 163 | 164 | assert_canon_path_eq("foo/bar", "foo/bar"); 165 | } 166 | 167 | #[test] 168 | fn dot() { 169 | assert_canon_path_eq("./foo", "foo"); 170 | assert_canon_path_eq("foo/.", "foo/"); 171 | assert_canon_path_eq("foo/./bar", "foo/bar"); 172 | assert_canon_path_eq("./", "."); 173 | assert_canon_path_eq("./.", "."); 174 | assert_canon_path_eq("././", "."); 175 | assert_canon_path_eq("././.", "."); 176 | assert_canon_path_eq(".", "."); 177 | } 178 | 179 | #[test] 180 | fn not_dot() { 181 | assert_canon_path_eq("t/.hidden", "t/.hidden"); 182 | assert_canon_path_eq("t/.._lib.c.o", "t/.._lib.c.o"); 183 | } 184 | 185 | #[test] 186 | fn slash() { 187 | assert_canon_path_eq("/foo", "/foo"); 188 | assert_canon_path_eq("foo//bar", "foo/bar"); 189 | } 190 | 191 | #[test] 192 | fn parent() { 193 | assert_canon_path_eq("foo/../bar", "bar"); 194 | 195 | assert_canon_path_eq("/foo/../bar", "/bar"); 196 | assert_canon_path_eq("../foo", "../foo"); 197 | assert_canon_path_eq("../foo/../bar", "../bar"); 198 | assert_canon_path_eq("../../bar", "../../bar"); 199 | assert_canon_path_eq("./../foo", "../foo"); 200 | assert_canon_path_eq("foo/..", "."); 201 | assert_canon_path_eq("foo/../", "."); 202 | assert_canon_path_eq("foo/../../", "../"); 203 | assert_canon_path_eq("foo/../../bar", "../bar"); 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /src/process_posix.rs: -------------------------------------------------------------------------------- 1 | //! Implements run_command on posix using posix_spawn. 2 | //! See run_command comments for why. 3 | 4 | use crate::process::Termination; 5 | use std::io::{Error, Read}; 6 | use std::os::fd::FromRawFd; 7 | use std::os::unix::process::ExitStatusExt; 8 | 9 | // https://github.com/rust-lang/libc/issues/2520 10 | // libc crate doesn't expose the 'environ' pointer. 11 | extern "C" { 12 | static environ: *const *mut libc::c_char; 13 | } 14 | 15 | fn check_posix_spawn(func: &str, ret: libc::c_int) -> anyhow::Result<()> { 16 | if ret != 0 { 17 | let err_str = unsafe { std::ffi::CStr::from_ptr(libc::strerror(ret)) }; 18 | anyhow::bail!("{}: {}", func, err_str.to_str().unwrap()); 19 | } 20 | Ok(()) 21 | } 22 | 23 | fn check_ret_errno(func: &str, ret: libc::c_int) -> anyhow::Result<()> { 24 | if ret < 0 { 25 | let errno = Error::last_os_error().raw_os_error().unwrap(); 26 | let err_str = unsafe { std::ffi::CStr::from_ptr(libc::strerror(errno)) }; 27 | anyhow::bail!("{}: {}", func, err_str.to_str().unwrap()); 28 | } 29 | Ok(()) 30 | } 31 | 32 | /// Wraps libc::posix_spawnattr_t, in particular to implement Drop. 33 | struct PosixSpawnAttr(libc::posix_spawnattr_t); 34 | 35 | impl PosixSpawnAttr { 36 | fn new() -> anyhow::Result { 37 | unsafe { 38 | let mut attr: libc::posix_spawnattr_t = std::mem::zeroed(); 39 | check_posix_spawn( 40 | "posix_spawnattr_init", 41 | libc::posix_spawnattr_init(&mut attr), 42 | )?; 43 | Ok(Self(attr)) 44 | } 45 | } 46 | 47 | fn as_ptr(&mut self) -> *mut libc::posix_spawnattr_t { 48 | &mut self.0 49 | } 50 | 51 | fn setflags(&mut self, flags: libc::c_short) -> anyhow::Result<()> { 52 | unsafe { 53 | check_posix_spawn( 54 | "posix_spawnattr_setflags", 55 | libc::posix_spawnattr_setflags(self.as_ptr(), flags), 56 | ) 57 | } 58 | } 59 | } 60 | 61 | impl Drop for PosixSpawnAttr { 62 | fn drop(&mut self) { 63 | unsafe { 64 | libc::posix_spawnattr_destroy(self.as_ptr()); 65 | } 66 | } 67 | } 68 | 69 | /// Wraps libc::posix_spawn_file_actions_t, in particular to implement Drop. 70 | struct PosixSpawnFileActions(libc::posix_spawn_file_actions_t); 71 | 72 | impl PosixSpawnFileActions { 73 | fn new() -> anyhow::Result { 74 | unsafe { 75 | let mut actions: libc::posix_spawn_file_actions_t = std::mem::zeroed(); 76 | check_posix_spawn( 77 | "posix_spawn_file_actions_init", 78 | libc::posix_spawn_file_actions_init(&mut actions), 79 | )?; 80 | Ok(Self(actions)) 81 | } 82 | } 83 | 84 | fn as_ptr(&mut self) -> *mut libc::posix_spawn_file_actions_t { 85 | &mut self.0 86 | } 87 | 88 | fn addopen( 89 | &mut self, 90 | fd: i32, 91 | path: &std::ffi::CStr, 92 | oflag: i32, 93 | mode: libc::mode_t, 94 | ) -> anyhow::Result<()> { 95 | unsafe { 96 | check_posix_spawn( 97 | "posix_spawn_file_actions_addopen", 98 | libc::posix_spawn_file_actions_addopen( 99 | self.as_ptr(), 100 | fd, 101 | path.as_ptr(), 102 | oflag, 103 | mode, 104 | ), 105 | ) 106 | } 107 | } 108 | 109 | fn adddup2(&mut self, fd: i32, newfd: i32) -> anyhow::Result<()> { 110 | unsafe { 111 | check_posix_spawn( 112 | "posix_spawn_file_actions_adddup2", 113 | libc::posix_spawn_file_actions_adddup2(self.as_ptr(), fd, newfd), 114 | ) 115 | } 116 | } 117 | 118 | fn addclose(&mut self, fd: i32) -> anyhow::Result<()> { 119 | unsafe { 120 | check_posix_spawn( 121 | "posix_spawn_file_actions_addclose", 122 | libc::posix_spawn_file_actions_addclose(self.as_ptr(), fd), 123 | ) 124 | } 125 | } 126 | } 127 | 128 | impl Drop for PosixSpawnFileActions { 129 | fn drop(&mut self) { 130 | unsafe { libc::posix_spawn_file_actions_destroy(&mut self.0) }; 131 | } 132 | } 133 | 134 | /// Create an anonymous pipe as in libc::pipe(), but using pipe2() when available 135 | /// to set CLOEXEC flag. 136 | fn pipe2() -> anyhow::Result<[libc::c_int; 2]> { 137 | // Compare to: https://doc.rust-lang.org/src/std/sys/unix/pipe.rs.html 138 | unsafe { 139 | let mut pipe: [libc::c_int; 2] = std::mem::zeroed(); 140 | 141 | // Mac: specially handled below with POSIX_SPAWN_CLOEXEC_DEFAULT 142 | #[cfg(target_os = "macos")] 143 | check_ret_errno("pipe", libc::pipe(pipe.as_mut_ptr()))?; 144 | 145 | // Assume all non-Mac have pipe2; we can refine this on user feedback. 146 | #[cfg(all(unix, not(target_os = "macos")))] 147 | check_ret_errno("pipe", libc::pipe2(pipe.as_mut_ptr(), libc::O_CLOEXEC))?; 148 | 149 | Ok(pipe) 150 | } 151 | } 152 | 153 | pub fn run_command(cmdline: &str, mut output_cb: impl FnMut(&[u8])) -> anyhow::Result { 154 | // Spawn the subprocess using posix_spawn with output redirected to the pipe. 155 | // We don't use Rust's process spawning because of issue #14 and because 156 | // we want to feed both stdout and stderr into the same pipe, which cannot 157 | // be done with the existing std::process API. 158 | let (pid, mut pipe) = unsafe { 159 | let pipe = pipe2()?; 160 | 161 | let mut attr = PosixSpawnAttr::new()?; 162 | 163 | // Apple-specific extension: close any open fds. 164 | #[cfg(target_os = "macos")] 165 | attr.setflags(libc::POSIX_SPAWN_CLOEXEC_DEFAULT as _)?; 166 | 167 | let mut actions = PosixSpawnFileActions::new()?; 168 | // open /dev/null over stdin 169 | actions.addopen(0, c"/dev/null", libc::O_RDONLY, 0)?; 170 | // stdout/stderr => pipe 171 | actions.adddup2(pipe[1], 1)?; 172 | actions.adddup2(pipe[1], 2)?; 173 | // close pipe in child 174 | actions.addclose(pipe[0])?; 175 | actions.addclose(pipe[1])?; 176 | 177 | let mut pid: libc::pid_t = 0; 178 | let path = c"/bin/sh"; 179 | let cmdline_nul = std::ffi::CString::new(cmdline).unwrap(); 180 | let argv: [*const libc::c_char; 4] = [ 181 | path.as_ptr(), 182 | c"-c".as_ptr(), 183 | cmdline_nul.as_ptr(), 184 | std::ptr::null(), 185 | ]; 186 | 187 | check_posix_spawn( 188 | "posix_spawn", 189 | libc::posix_spawn( 190 | &mut pid, 191 | path.as_ptr(), 192 | actions.as_ptr(), 193 | attr.as_ptr(), 194 | // posix_spawn wants mutable argv: 195 | // https://stackoverflow.com/questions/50596439/can-string-literals-be-passed-in-posix-spawns-argv 196 | argv.as_ptr() as *const *mut _, 197 | environ, 198 | ), 199 | )?; 200 | 201 | check_ret_errno("close", libc::close(pipe[1]))?; 202 | 203 | (pid, std::fs::File::from_raw_fd(pipe[0])) 204 | }; 205 | 206 | let mut buf: [u8; 4 << 10] = [0; 4 << 10]; 207 | loop { 208 | let n = pipe.read(&mut buf)?; 209 | if n == 0 { 210 | break; 211 | } 212 | output_cb(&buf[0..n]); 213 | } 214 | drop(pipe); 215 | 216 | let status = unsafe { 217 | let mut status: i32 = 0; 218 | check_ret_errno("waitpid", libc::waitpid(pid, &mut status, 0))?; 219 | std::process::ExitStatus::from_raw(status) 220 | }; 221 | 222 | let termination = if status.success() { 223 | Termination::Success 224 | } else if let Some(sig) = status.signal() { 225 | match sig { 226 | libc::SIGINT => { 227 | output_cb("interrupted".as_bytes()); 228 | Termination::Interrupted 229 | } 230 | _ => { 231 | output_cb(format!("signal {}", sig).as_bytes()); 232 | Termination::Failure 233 | } 234 | } 235 | } else { 236 | Termination::Failure 237 | }; 238 | 239 | Ok(termination) 240 | } 241 | -------------------------------------------------------------------------------- /src/run.rs: -------------------------------------------------------------------------------- 1 | //! Command line argument parsing and initial build invocation. 2 | 3 | use crate::{ 4 | load, progress::Progress, progress_dumb::DumbConsoleProgress, 5 | progress_fancy::FancyConsoleProgress, terminal, trace, work, 6 | }; 7 | use anyhow::anyhow; 8 | 9 | /// Arguments to start a build, after parsing all the command line etc. 10 | #[derive(Default)] 11 | struct BuildArgs { 12 | fake_ninja_compat: bool, 13 | options: work::Options, 14 | build_filename: Option, 15 | targets: Vec, 16 | verbose: bool, 17 | } 18 | 19 | /// Returns the number of completed tasks on a successful build. 20 | fn build(args: BuildArgs) -> anyhow::Result> { 21 | let (dumb_console, fancy_console); 22 | let progress: &dyn Progress = if terminal::use_fancy() { 23 | fancy_console = FancyConsoleProgress::new(args.verbose); 24 | &fancy_console 25 | } else { 26 | dumb_console = DumbConsoleProgress::new(args.verbose); 27 | &dumb_console 28 | }; 29 | 30 | let build_filename = args.build_filename.as_deref().unwrap_or("build.ninja"); 31 | let mut state = trace::scope("load::read", || load::read(build_filename))?; 32 | let mut work = work::Work::new( 33 | state.graph, 34 | state.hashes, 35 | state.db, 36 | &args.options, 37 | progress, 38 | state.pools, 39 | ); 40 | 41 | let mut tasks_run = 0; 42 | 43 | // Attempt to rebuild build.ninja. 44 | let build_file_target = work.lookup(&build_filename); 45 | if let Some(target) = build_file_target { 46 | work.want_file(target)?; 47 | if !trace::scope("work.run", || work.run())? { 48 | return Ok(None); 49 | } 50 | if work.tasks_run == 0 { 51 | // build.ninja already up to date. 52 | // TODO: this logic is not right in the case where a build has 53 | // a step that doesn't touch build.ninja. We should instead 54 | // verify the specific FileId was updated. 55 | } else { 56 | // Regenerated build.ninja; start over. 57 | tasks_run = work.tasks_run; 58 | state = trace::scope("load::read", || load::read(&build_filename))?; 59 | work = work::Work::new( 60 | state.graph, 61 | state.hashes, 62 | state.db, 63 | &args.options, 64 | progress, 65 | state.pools, 66 | ); 67 | } 68 | } 69 | 70 | if !args.targets.is_empty() { 71 | for name in &args.targets { 72 | let Some(target) = work.lookup(name) else { 73 | if args.options.adopt { 74 | // cmake invokes -t restat with paths that don't exist 75 | // https://github.com/evmar/n2/issues/142 76 | continue; 77 | } 78 | return Err(anyhow::anyhow!("unknown path requested: {:?}", name)); 79 | }; 80 | if Some(target) == build_file_target { 81 | // Already built above. 82 | continue; 83 | } 84 | work.want_file(target)?; 85 | } 86 | } else if !state.default.is_empty() { 87 | for target in state.default { 88 | work.want_file(target)?; 89 | } 90 | } else { 91 | work.want_every_file(build_file_target)?; 92 | } 93 | 94 | if !trace::scope("work.run", || work.run())? { 95 | return Ok(None); 96 | } 97 | // Include any tasks from initial build in final count of steps. 98 | Ok(Some(tasks_run + work.tasks_run)) 99 | } 100 | 101 | fn default_parallelism() -> anyhow::Result { 102 | // Ninja uses available processors + a constant, but I don't think the 103 | // difference matters too much. 104 | let par = std::thread::available_parallelism()?; 105 | Ok(usize::from(par)) 106 | } 107 | 108 | /// Run a tool as specified by the `-t` flag`. 109 | fn subtool(args: &mut BuildArgs, tool: &str) -> anyhow::Result> { 110 | match tool { 111 | "list" => { 112 | println!("subcommands:"); 113 | println!( 114 | " (none yet, but see README if you're looking here trying to get CMake to work)" 115 | ); 116 | return Ok(Some(1)); 117 | } 118 | "recompact" if args.fake_ninja_compat => { 119 | // CMake unconditionally invokes this tool, yuck. 120 | return Ok(Some(0)); // do nothing 121 | } 122 | "restat" if args.fake_ninja_compat => { 123 | // CMake invokes this after generating build files; mark build 124 | // targets as up to date by running the build with "adopt" flag 125 | // on. 126 | args.options.adopt = true; 127 | } 128 | _ => { 129 | anyhow::bail!("unknown -t {:?}, use -t list to list", tool); 130 | } 131 | } 132 | Ok(None) 133 | } 134 | 135 | /// Run a debug tool as specified by the `-d` flag. 136 | fn debugtool(args: &mut BuildArgs, tool: &str) -> anyhow::Result> { 137 | match tool { 138 | "list" => { 139 | println!("debug tools:"); 140 | println!(" ninja_compat enable ninja quirks compatibility mode"); 141 | println!(" explain print why each target is considered out of date"); 142 | println!(" trace generate json performance trace"); 143 | return Ok(Some(1)); 144 | } 145 | 146 | "ninja_compat" => args.fake_ninja_compat = true, 147 | "explain" => args.options.explain = true, 148 | "trace" => trace::open("trace.json")?, 149 | 150 | _ => anyhow::bail!("unknown -d {:?}, use -d list to list", tool), 151 | } 152 | Ok(None) 153 | } 154 | 155 | fn parse_args() -> anyhow::Result> { 156 | let mut args = BuildArgs::default(); 157 | args.fake_ninja_compat = std::path::Path::new(&std::env::args().next().unwrap()) 158 | .file_name() 159 | .unwrap() 160 | == std::ffi::OsStr::new(&format!("ninja{}", std::env::consts::EXE_SUFFIX)); 161 | 162 | use lexopt::prelude::*; 163 | let mut parser = lexopt::Parser::from_env(); 164 | while let Some(arg) = parser.next()? { 165 | match arg { 166 | Short('h') | Long("help") => { 167 | println!( 168 | "n2: a ninja-compatible build tool 169 | usage: n2 [options] [targets...] 170 | 171 | options: 172 | -C dir chdir before running 173 | -f file input build file [default: build.ninja] 174 | -j N parallelism [default: use system thread count] 175 | -k N keep going until at least N failures [default: 1] 176 | -v print executed command lines 177 | 178 | -t tool tools (`-t list` to list) 179 | -d tool debugging tools (use `-d list` to list) 180 | " 181 | ); 182 | return Ok(Err(0)); 183 | } 184 | 185 | Short('C') => { 186 | let dir = parser.value()?; 187 | std::env::set_current_dir(&dir) 188 | .map_err(|err| anyhow!("chdir {:?}: {}", dir, err))?; 189 | } 190 | 191 | Short('f') => args.build_filename = Some(parser.value()?.to_string_lossy().into()), 192 | Short('t') => { 193 | if let Some(exit) = subtool(&mut args, &*parser.value()?.to_string_lossy())? { 194 | return Ok(Err(exit)); 195 | } 196 | } 197 | Short('d') => { 198 | if let Some(exit) = debugtool(&mut args, &*parser.value()?.to_string_lossy())? { 199 | return Ok(Err(exit)); 200 | } 201 | } 202 | Short('j') => args.options.parallelism = parser.value()?.parse()?, 203 | Short('k') => args.options.failures_left = Some(parser.value()?.parse()?), 204 | Short('v') => args.verbose = true, 205 | 206 | Long("version") => { 207 | if args.fake_ninja_compat { 208 | // CMake requires a particular Ninja version. 209 | println!("1.10.2"); 210 | } else { 211 | println!("{}", env!("CARGO_PKG_VERSION")); 212 | } 213 | return Ok(Err(0)); 214 | } 215 | 216 | Value(arg) => args.targets.push(arg.to_string_lossy().into()), 217 | 218 | _ => anyhow::bail!("{}", arg.unexpected()), 219 | } 220 | } 221 | 222 | if args.options.parallelism == 0 { 223 | args.options.parallelism = default_parallelism()?; 224 | } 225 | 226 | Ok(Ok(args)) 227 | } 228 | 229 | fn run_impl() -> anyhow::Result { 230 | let args = match parse_args()? { 231 | Ok(args) => args, 232 | Err(exit) => return Ok(exit), 233 | }; 234 | 235 | match build(args)? { 236 | None => { 237 | // Don't print any summary, the failing task is enough info. 238 | return Ok(1); 239 | } 240 | Some(0) => { 241 | // Special case: don't print numbers when no work done. 242 | println!("n2: no work to do"); 243 | } 244 | Some(n) => { 245 | println!( 246 | "n2: ran {} task{}, now up to date", 247 | n, 248 | if n == 1 { "" } else { "s" } 249 | ); 250 | } 251 | } 252 | 253 | Ok(0) 254 | } 255 | 256 | pub fn run() -> anyhow::Result { 257 | let res = run_impl(); 258 | trace::close(); 259 | res 260 | } 261 | -------------------------------------------------------------------------------- /tests/e2e/basic.rs: -------------------------------------------------------------------------------- 1 | use crate::e2e::*; 2 | 3 | #[test] 4 | fn empty_file() -> anyhow::Result<()> { 5 | let space = TestSpace::new()?; 6 | space.write("build.ninja", "")?; 7 | let out = space.run(&mut n2_command(vec![]))?; 8 | assert_eq!(std::str::from_utf8(&out.stdout)?, "n2: no work to do\n"); 9 | Ok(()) 10 | } 11 | 12 | #[test] 13 | fn basic_build() -> anyhow::Result<()> { 14 | let space = TestSpace::new()?; 15 | space.write( 16 | "build.ninja", 17 | &[TOUCH_RULE, "build out: touch in", ""].join("\n"), 18 | )?; 19 | space.write("in", "")?; 20 | space.run_expect(&mut n2_command(vec!["out"]))?; 21 | assert!(space.read("out").is_ok()); 22 | 23 | Ok(()) 24 | } 25 | 26 | #[test] 27 | fn create_subdir() -> anyhow::Result<()> { 28 | // Run a build rule that needs a subdir to be automatically created. 29 | let space = TestSpace::new()?; 30 | space.write( 31 | "build.ninja", 32 | &[TOUCH_RULE, "build subdir/out: touch in", ""].join("\n"), 33 | )?; 34 | space.write("in", "")?; 35 | space.run_expect(&mut n2_command(vec!["subdir/out"]))?; 36 | assert!(space.read("subdir/out").is_ok()); 37 | 38 | Ok(()) 39 | } 40 | 41 | #[cfg(unix)] 42 | #[test] 43 | fn generate_rsp_file() -> anyhow::Result<()> { 44 | let space = TestSpace::new()?; 45 | space.write( 46 | "build.ninja", 47 | " 48 | rule cat 49 | command = cat ${out}.rsp > ${out} 50 | rspfile = ${out}.rsp 51 | rspfile_content = 1 $in 2 $in_newline 3 52 | 53 | rule litter 54 | command = cat make/me/${out}.rsp > ${out} 55 | rspfile = make/me/${out}.rsp 56 | rspfile_content = random stuff 57 | 58 | rule touch 59 | command = touch $out 60 | 61 | build main: cat foo bar baz in 62 | build foo: litter bar 63 | build bar: touch baz 64 | build baz: touch in 65 | ", 66 | )?; 67 | space.write("in", "go!")?; 68 | 69 | let _ = space.run_expect(&mut n2_command(vec!["main"]))?; 70 | 71 | // The 'main' and 'foo' targets copy the contents of their rsp file to their 72 | // output. 73 | let main_rsp = space.read("main").unwrap(); 74 | assert_eq!(main_rsp, b"1 foo bar baz in 2 foo\nbar\nbaz\nin 3"); 75 | let foo_rsp = space.read("foo").unwrap(); 76 | assert_eq!(foo_rsp, b"random stuff"); 77 | 78 | // The 'make/me' directory was created when writing an rsp file. 79 | // It should still be there. 80 | let meta = space.metadata("make/me").unwrap(); 81 | assert!(meta.is_dir()); 82 | 83 | // Run again: everything should be up to date. 84 | let out = space.run_expect(&mut n2_command(vec!["main"]))?; 85 | assert_output_contains(&out, "no work"); 86 | 87 | Ok(()) 88 | } 89 | 90 | /// Run a task that prints something, and verify it shows up. 91 | #[cfg(unix)] 92 | #[test] 93 | fn spam_output() -> anyhow::Result<()> { 94 | let space = TestSpace::new()?; 95 | space.write( 96 | "build.ninja", 97 | " 98 | rule quiet 99 | description = quiet $out 100 | command = touch $out 101 | rule spam 102 | description = spam $out 103 | command = echo greetz from $out && touch $out 104 | build a: quiet 105 | build b: spam a 106 | build c: quiet b 107 | ", 108 | )?; 109 | let out = space.run_expect(&mut n2_command(vec!["c"]))?; 110 | assert_output_contains( 111 | &out, 112 | "quiet a 113 | spam b 114 | greetz from b 115 | quiet c 116 | ", 117 | ); 118 | Ok(()) 119 | } 120 | 121 | #[test] 122 | fn specify_build_file() -> anyhow::Result<()> { 123 | let space = TestSpace::new()?; 124 | space.write( 125 | "build_specified.ninja", 126 | &[TOUCH_RULE, "build out: touch in", ""].join("\n"), 127 | )?; 128 | space.write("in", "")?; 129 | space.run_expect(&mut n2_command(vec!["-f", "build_specified.ninja", "out"]))?; 130 | assert!(space.read("out").is_ok()); 131 | 132 | Ok(()) 133 | } 134 | 135 | /// Regression test for https://github.com/evmar/n2/issues/44 136 | /// and https://github.com/evmar/n2/issues/46 . 137 | /// Build with the same output listed multiple times. 138 | #[test] 139 | fn repeated_out() -> anyhow::Result<()> { 140 | let space = TestSpace::new()?; 141 | space.write( 142 | "build.ninja", 143 | &[ 144 | TOUCH_RULE, 145 | "build dup dup: touch in", 146 | "build out: touch dup", 147 | "", 148 | ] 149 | .join("\n"), 150 | )?; 151 | space.write("in", "")?; 152 | space.write("dup", "")?; 153 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 154 | assert_output_contains(&out, "is repeated in output list"); 155 | 156 | Ok(()) 157 | } 158 | 159 | /// Regression test for https://github.com/evmar/n2/issues/55 160 | /// UTF-8 filename. 161 | #[cfg(unix)] 162 | #[test] 163 | fn utf8_filename() -> anyhow::Result<()> { 164 | let space = TestSpace::new()?; 165 | space.write( 166 | "build.ninja", 167 | &[ 168 | " 169 | rule echo 170 | description = unicode variable: $in 171 | command = echo unicode command line: $in && touch $out 172 | ", 173 | "build out: echo reykjavík.md", 174 | "", 175 | ] 176 | .join("\n"), 177 | )?; 178 | space.write("reykjavík.md", "")?; 179 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 180 | assert_output_contains(&out, "unicode variable: reykjavík.md"); 181 | assert_output_contains(&out, "unicode command line: reykjavík.md"); 182 | 183 | Ok(()) 184 | } 185 | 186 | #[test] 187 | fn explain() -> anyhow::Result<()> { 188 | let space = TestSpace::new()?; 189 | space.write( 190 | "build.ninja", 191 | &[TOUCH_RULE, "build out: touch in", ""].join("\n"), 192 | )?; 193 | space.write("in", "")?; 194 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 195 | assert_output_contains(&out, "up to date"); 196 | 197 | space.write("in", "x")?; 198 | let out = space.run_expect(&mut n2_command(vec!["-d", "explain", "out"]))?; 199 | // The main "explain" log line: 200 | assert_output_contains(&out, "explain: build.ninja:6: manifest changed"); 201 | // The dump of the file manifest after includes mtimes that we don't want 202 | // to be sensitive to, so just look for some bits we know show up there. 203 | assert_output_contains(&out, "discovered:"); 204 | 205 | Ok(()) 206 | } 207 | 208 | /// Meson generates a build step that writes to one of its inputs. 209 | #[test] 210 | fn write_to_input() -> anyhow::Result<()> { 211 | #[cfg(unix)] 212 | let touch_input_command = "touch out in"; 213 | #[cfg(windows)] 214 | let touch_input_command = "cmd /c type nul > in && cmd /c type nul > out"; 215 | let touch_input_rule = format!( 216 | " 217 | rule touch_in 218 | description = touch out+in 219 | command = {} 220 | ", 221 | touch_input_command 222 | ); 223 | 224 | let space = TestSpace::new()?; 225 | space.write( 226 | "build.ninja", 227 | &[&touch_input_rule, "build out: touch_in in", ""].join("\n"), 228 | )?; 229 | space.write("in", "")?; 230 | space.sub_mtime("in", std::time::Duration::from_secs(1))?; 231 | 232 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 233 | assert_output_contains(&out, "ran 1 task"); 234 | 235 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 236 | assert_output_contains(&out, "no work to do"); 237 | 238 | Ok(()) 239 | } 240 | 241 | #[test] 242 | fn showincludes() -> anyhow::Result<()> { 243 | let space = TestSpace::new()?; 244 | space.write( 245 | "build.ninja", 246 | &[ 247 | ECHO_RULE, 248 | " 249 | build out: echo 250 | text = Note: including file: foo 251 | deps = msvc 252 | ", 253 | ] 254 | .join("\n"), 255 | )?; 256 | space.write("foo", "")?; 257 | 258 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 259 | assert_output_contains(&out, "ran 1 task"); 260 | 261 | space.write("foo", "")?; 262 | let out = space.run_expect(&mut n2_command(vec!["out"]))?; 263 | assert_output_contains(&out, "ran 1 task"); 264 | 265 | Ok(()) 266 | } 267 | 268 | // Repro for issue #84: phony depending on phony. 269 | #[test] 270 | fn phony_depends() -> anyhow::Result<()> { 271 | let space = TestSpace::new()?; 272 | space.write( 273 | "build.ninja", 274 | &[ 275 | TOUCH_RULE, 276 | " 277 | build out1: touch 278 | build out2: phony out1 279 | build out3: phony out2 280 | ", 281 | ] 282 | .join("\n"), 283 | )?; 284 | space.run_expect(&mut n2_command(vec!["out3"]))?; 285 | space.read("out1")?; 286 | Ok(()) 287 | } 288 | 289 | // builddir controls where .n2_db is written. 290 | #[test] 291 | fn builddir() -> anyhow::Result<()> { 292 | let space = TestSpace::new()?; 293 | space.write( 294 | "build.ninja", 295 | &[ 296 | "builddir = foo", 297 | TOUCH_RULE, 298 | "build $builddir/bar: touch", 299 | "", 300 | ] 301 | .join("\n"), 302 | )?; 303 | space.run_expect(&mut n2_command(vec!["foo/bar"]))?; 304 | space.read("foo/.n2_db")?; 305 | Ok(()) 306 | } 307 | 308 | /// Verify the error message when a command doesn't exist. 309 | #[test] 310 | fn missing_command() -> anyhow::Result<()> { 311 | let space = TestSpace::new()?; 312 | space.write( 313 | "build.ninja", 314 | &[ 315 | "rule nope", 316 | " command = n2_no_such_command", 317 | "build out: nope", 318 | "", 319 | ] 320 | .join("\n"), 321 | )?; 322 | let out = space.run(&mut n2_command(vec!["out"]))?; 323 | 324 | if cfg!(windows) { 325 | assert_output_contains(&out, "The system cannot find the file specified."); 326 | } else { 327 | // Note on my local shell it prints "command not found" but the GitHub CI 328 | // /bin/sh prints "not found", so just look for that substring. 329 | assert_output_contains(&out, "not found"); 330 | } 331 | Ok(()) 332 | } 333 | -------------------------------------------------------------------------------- /src/task.rs: -------------------------------------------------------------------------------- 1 | //! Runs build tasks, potentially in parallel. 2 | //! Unaware of the build graph, pools, etc.; just command execution. 3 | //! 4 | //! We use one thread per subprocess. This differs from Ninja which goes to 5 | //! some effort to use ppoll-like behavior. Because the threads are mostly 6 | //! blocked in IO I don't expect this to be too costly in terms of CPU, but it's 7 | //! worth considering how much RAM it costs. On the positive side, the logic 8 | //! is significantly simpler than Ninja and we get free behaviors like parallel 9 | //! parsing of depfiles. 10 | 11 | use crate::{ 12 | depfile, 13 | graph::{Build, BuildId, RspFile}, 14 | process, 15 | scanner::{self, Scanner}, 16 | }; 17 | use anyhow::{anyhow, bail}; 18 | use std::path::{Path, PathBuf}; 19 | use std::sync::mpsc; 20 | use std::time::Instant; 21 | 22 | pub struct FinishedTask { 23 | /// A (faked) "thread id", used to put different finished builds in different 24 | /// tracks in a performance trace. 25 | pub tid: usize, 26 | pub buildid: BuildId, 27 | pub span: (Instant, Instant), 28 | pub result: TaskResult, 29 | } 30 | 31 | /// The result of running a build step. 32 | pub struct TaskResult { 33 | pub termination: process::Termination, 34 | /// Console output. 35 | pub output: Vec, 36 | pub discovered_deps: Option>, 37 | } 38 | 39 | /// Reads dependencies from a .d file path. 40 | fn read_depfile(path: &Path) -> anyhow::Result> { 41 | let bytes = match scanner::read_file_with_nul(path) { 42 | Ok(b) => b, 43 | // See discussion of missing depfiles in #80. 44 | // TODO(#99): warn or error in this circumstance? 45 | Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), 46 | Err(e) => bail!("read {}: {}", path.display(), e), 47 | }; 48 | 49 | let mut scanner = Scanner::new(&bytes); 50 | let parsed_deps = depfile::parse(&mut scanner) 51 | .map_err(|err| anyhow!(scanner.format_parse_error(path, err)))?; 52 | // TODO verify deps refers to correct output 53 | let deps: Vec = parsed_deps 54 | .values() 55 | .flat_map(|x| x.iter()) 56 | .map(|&dep| dep.to_owned()) 57 | .collect(); 58 | Ok(deps) 59 | } 60 | 61 | fn write_rspfile(rspfile: &RspFile) -> anyhow::Result<()> { 62 | if let Some(parent) = rspfile.path.parent() { 63 | std::fs::create_dir_all(parent)?; 64 | } 65 | std::fs::write(&rspfile.path, &rspfile.content)?; 66 | Ok(()) 67 | } 68 | 69 | /// Parse some subcommand output to extract "Note: including file:" lines as 70 | /// emitted by MSVC/clang-cl. 71 | fn extract_showincludes(output: Vec) -> (Vec, Vec) { 72 | let mut filtered_output = Vec::new(); 73 | let mut includes = Vec::new(); 74 | for line in output.split(|&c| c == b'\n') { 75 | if let Some(include) = line.strip_prefix(b"Note: including file: ") { 76 | let start = include.iter().position(|&c| c != b' ').unwrap_or(0); 77 | let end = if include.ends_with(&[b'\r']) { 78 | include.len() - 1 79 | } else { 80 | include.len() 81 | }; 82 | let include = &include[start..end]; 83 | includes.push(unsafe { String::from_utf8_unchecked(include.to_vec()) }); 84 | } else { 85 | if !filtered_output.is_empty() { 86 | filtered_output.push(b'\n'); 87 | } 88 | filtered_output.extend_from_slice(line); 89 | } 90 | } 91 | (includes, filtered_output) 92 | } 93 | 94 | /// Find the span of the last line of text in buf, ignoring trailing empty 95 | /// lines. 96 | fn find_last_line(buf: &[u8]) -> &[u8] { 97 | fn is_nl(c: u8) -> bool { 98 | c == b'\r' || c == b'\n' 99 | } 100 | 101 | let end = match buf.iter().rposition(|&c| !is_nl(c)) { 102 | Some(pos) => pos + 1, 103 | None => buf.len(), 104 | }; 105 | let start = match buf[..end].iter().rposition(|&c| is_nl(c)) { 106 | Some(pos) => pos + 1, 107 | None => 0, 108 | }; 109 | &buf[start..end] 110 | } 111 | 112 | /// Executes a build task as a subprocess. 113 | /// Returns an Err() if we failed outside of the process itself. 114 | /// This is run as a separate thread from the main n2 process and will block 115 | /// on the subprocess, so any additional per-subprocess work we can do belongs 116 | /// here. 117 | fn run_task( 118 | cmdline: &str, 119 | depfile: Option<&Path>, 120 | parse_showincludes: bool, 121 | rspfile: Option<&RspFile>, 122 | mut last_line_cb: impl FnMut(&[u8]), 123 | ) -> anyhow::Result { 124 | if let Some(rspfile) = rspfile { 125 | write_rspfile(rspfile)?; 126 | } 127 | 128 | let mut output = Vec::new(); 129 | let termination = process::run_command(cmdline, |buf| { 130 | output.extend_from_slice(buf); 131 | last_line_cb(find_last_line(&output)); 132 | })?; 133 | 134 | let mut discovered_deps = None; 135 | if parse_showincludes { 136 | // Remove /showIncludes lines from output, regardless of success/fail. 137 | let (includes, filtered) = extract_showincludes(output); 138 | output = filtered; 139 | discovered_deps = Some(includes); 140 | } 141 | if termination == process::Termination::Success { 142 | if let Some(depfile) = depfile { 143 | discovered_deps = Some(read_depfile(depfile)?); 144 | } 145 | } 146 | Ok(TaskResult { 147 | termination, 148 | output, 149 | discovered_deps, 150 | }) 151 | } 152 | 153 | /// Tracks faked "thread ids" -- integers assigned to build tasks to track 154 | /// parallelism in perf trace output. 155 | #[derive(Default)] 156 | struct ThreadIds { 157 | /// An entry is true when claimed, false or nonexistent otherwise. 158 | slots: Vec, 159 | } 160 | impl ThreadIds { 161 | fn claim(&mut self) -> usize { 162 | match self.slots.iter().position(|&used| !used) { 163 | Some(idx) => { 164 | self.slots[idx] = true; 165 | idx 166 | } 167 | None => { 168 | let idx = self.slots.len(); 169 | self.slots.push(true); 170 | idx 171 | } 172 | } 173 | } 174 | 175 | fn release(&mut self, slot: usize) { 176 | self.slots[slot] = false; 177 | } 178 | } 179 | 180 | enum Message { 181 | Output((BuildId, Vec)), 182 | Done(FinishedTask), 183 | } 184 | 185 | pub struct Runner { 186 | tx: mpsc::Sender, 187 | rx: mpsc::Receiver, 188 | pub running: usize, 189 | tids: ThreadIds, 190 | parallelism: usize, 191 | } 192 | 193 | impl Runner { 194 | pub fn new(parallelism: usize) -> Self { 195 | let (tx, rx) = mpsc::channel(); 196 | Runner { 197 | tx, 198 | rx, 199 | running: 0, 200 | tids: ThreadIds::default(), 201 | parallelism, 202 | } 203 | } 204 | 205 | pub fn can_start_more(&self) -> bool { 206 | self.running < self.parallelism 207 | } 208 | 209 | pub fn is_running(&self) -> bool { 210 | self.running > 0 211 | } 212 | 213 | pub fn start(&mut self, id: BuildId, build: &Build) { 214 | let cmdline = build.cmdline.clone().unwrap(); 215 | let depfile = build.depfile.clone().map(PathBuf::from); 216 | let rspfile = build.rspfile.clone(); 217 | let parse_showincludes = build.parse_showincludes; 218 | let hide_progress = build.hide_progress; 219 | 220 | let tid = self.tids.claim(); 221 | let tx = self.tx.clone(); 222 | std::thread::spawn(move || { 223 | let start = Instant::now(); 224 | let result = run_task( 225 | &cmdline, 226 | depfile.as_deref(), 227 | parse_showincludes, 228 | rspfile.as_ref(), 229 | |line| { 230 | if !hide_progress { 231 | let _ = tx.send(Message::Output((id, line.to_owned()))); 232 | } 233 | }, 234 | ) 235 | .unwrap_or_else(|err| TaskResult { 236 | termination: process::Termination::Failure, 237 | output: format!("{}\n", err).into_bytes(), 238 | discovered_deps: None, 239 | }); 240 | let finish = Instant::now(); 241 | 242 | let task = FinishedTask { 243 | tid, 244 | buildid: id, 245 | span: (start, finish), 246 | result, 247 | }; 248 | // The send will only fail if the receiver disappeared, e.g. due to shutting down. 249 | let _ = tx.send(Message::Done(task)); 250 | }); 251 | self.running += 1; 252 | } 253 | 254 | /// Wait for a build to complete. May block for a long time. 255 | pub fn wait(&mut self, mut output: impl FnMut(BuildId, Vec)) -> FinishedTask { 256 | loop { 257 | match self.rx.recv().unwrap() { 258 | Message::Output((bid, line)) => output(bid, line), 259 | Message::Done(task) => { 260 | self.tids.release(task.tid); 261 | self.running -= 1; 262 | return task; 263 | } 264 | } 265 | } 266 | } 267 | } 268 | 269 | #[cfg(test)] 270 | mod tests { 271 | use super::*; 272 | 273 | #[test] 274 | fn show_includes() { 275 | let (includes, output) = extract_showincludes( 276 | b"some text 277 | Note: including file: a 278 | other text 279 | Note: including file: b\r 280 | more text 281 | " 282 | .to_vec(), 283 | ); 284 | assert_eq!(includes, &["a", "b"]); 285 | assert_eq!( 286 | output, 287 | b"some text 288 | other text 289 | more text 290 | " 291 | ); 292 | } 293 | 294 | #[test] 295 | fn find_last() { 296 | assert_eq!(find_last_line(b""), b""); 297 | assert_eq!(find_last_line(b"\n"), b""); 298 | 299 | assert_eq!(find_last_line(b"hello"), b"hello"); 300 | assert_eq!(find_last_line(b"hello\n"), b"hello"); 301 | 302 | assert_eq!(find_last_line(b"hello\nt"), b"t"); 303 | assert_eq!(find_last_line(b"hello\nt\n"), b"t"); 304 | 305 | assert_eq!(find_last_line(b"hello\n\n"), b"hello"); 306 | assert_eq!(find_last_line(b"hello\nt\n\n"), b"t"); 307 | } 308 | 309 | #[test] 310 | fn missing_depfile_allowed() { 311 | let deps = read_depfile(Path::new("/missing/dep/file")).unwrap(); 312 | assert_eq!(deps.len(), 0); 313 | } 314 | } 315 | -------------------------------------------------------------------------------- /src/process_win.rs: -------------------------------------------------------------------------------- 1 | //! Implements run_command on Windows using native Windows calls. 2 | //! See run_command comments for why. 3 | 4 | use crate::process::Termination; 5 | use std::ffi::c_void; 6 | use std::io::Read; 7 | use std::os::windows::io::{FromRawHandle, OwnedHandle}; 8 | use std::os::windows::prelude::AsRawHandle; 9 | use std::pin::{pin, Pin}; 10 | use windows_sys::Win32::{ 11 | Foundation::*, 12 | Security::SECURITY_ATTRIBUTES, 13 | System::{Console::*, Diagnostics::Debug::*, Pipes::CreatePipe, Threading::*}, 14 | }; 15 | 16 | fn get_error_string(err: u32) -> String { 17 | let mut buf: [u8; 1024] = [0; 1024]; 18 | let len = unsafe { 19 | FormatMessageA( 20 | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, 21 | std::ptr::null(), 22 | err, 23 | 0x0000_0400, // MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT) 24 | buf.as_mut_ptr(), 25 | buf.len() as u32, 26 | std::ptr::null(), 27 | ) 28 | }; 29 | if len == 0 { 30 | panic!("FormatMessageA on error failed: {}", err); 31 | } 32 | std::str::from_utf8(&buf[..len as usize]) 33 | .unwrap() 34 | .trim_end() 35 | .to_owned() 36 | } 37 | 38 | /// Construct an error from GetLastError(). 39 | fn windows_error(func: &str) -> anyhow::Error { 40 | let err = unsafe { GetLastError() }; 41 | anyhow::anyhow!("{}: {}", func, get_error_string(err)) 42 | } 43 | /// Return an Err from the current function with GetLastError info in it. 44 | macro_rules! win_bail { 45 | ($func:ident) => { 46 | return Err(windows_error(stringify!($func))); 47 | }; 48 | } 49 | 50 | /// Wrapper for PROCESS_INFORMATION that cleans up on Drop. 51 | struct ProcessInformation(PROCESS_INFORMATION); 52 | 53 | impl ProcessInformation { 54 | fn new() -> Self { 55 | Self(unsafe { std::mem::zeroed() }) 56 | } 57 | fn as_mut_ptr(&mut self) -> *mut PROCESS_INFORMATION { 58 | &mut self.0 59 | } 60 | } 61 | 62 | impl std::ops::Deref for ProcessInformation { 63 | type Target = PROCESS_INFORMATION; 64 | 65 | fn deref(&self) -> &Self::Target { 66 | &self.0 67 | } 68 | } 69 | impl std::ops::DerefMut for ProcessInformation { 70 | fn deref_mut(&mut self) -> &mut Self::Target { 71 | &mut self.0 72 | } 73 | } 74 | impl Drop for ProcessInformation { 75 | fn drop(&mut self) { 76 | unsafe { 77 | if self.hProcess != 0 { 78 | CloseHandle(self.hProcess); 79 | } 80 | if self.hThread != 0 { 81 | CloseHandle(self.hThread); 82 | } 83 | } 84 | } 85 | } 86 | 87 | /// Wrapper for PROC_THREAD_ATTRIBUTE_LIST. 88 | /// Per MSDN: attribute values "must persist until the attribute list is 89 | /// destroyed using the DeleteProcThreadAttributeList function", which is 90 | /// captured by the 'a lifetime. 91 | struct ProcThreadAttributeList<'a> { 92 | /// The PROC_THREAD_ATTRIBUTE_LIST; this is a type whose size we discover at runtime. 93 | raw: Box<[u8]>, 94 | /// The inherit_handles pointer. 95 | _marker: std::marker::PhantomData<&'a [HANDLE]>, 96 | } 97 | impl<'a> ProcThreadAttributeList<'a> { 98 | fn new(count: usize) -> anyhow::Result { 99 | unsafe { 100 | let mut size = 0; 101 | if InitializeProcThreadAttributeList(std::ptr::null_mut(), count as u32, 0, &mut size) 102 | == 0 103 | { 104 | if GetLastError() != ERROR_INSUFFICIENT_BUFFER { 105 | win_bail!(InitializeProcThreadAttributeList); 106 | } 107 | } 108 | 109 | let mut buf = vec![0u8; size].into_boxed_slice(); 110 | if InitializeProcThreadAttributeList( 111 | buf.as_mut_ptr() as LPPROC_THREAD_ATTRIBUTE_LIST, 112 | count as u32, 113 | 0, 114 | &mut size, 115 | ) == 0 116 | { 117 | win_bail!(InitializeProcThreadAttributeList); 118 | } 119 | Ok(Self { 120 | raw: buf, 121 | _marker: std::marker::PhantomData, 122 | }) 123 | } 124 | } 125 | 126 | /// Mark some handles as to be inherited. 127 | fn inherit_handles(&mut self, handles: Pin<&'a [HANDLE]>) -> anyhow::Result<()> { 128 | unsafe { 129 | if UpdateProcThreadAttribute( 130 | self.as_mut_ptr(), 131 | 0, 132 | PROC_THREAD_ATTRIBUTE_HANDLE_LIST as usize, 133 | handles.as_ptr() as *const c_void, 134 | handles.len() * std::mem::size_of::(), 135 | std::ptr::null_mut(), 136 | std::ptr::null_mut(), 137 | ) == 0 138 | { 139 | win_bail!(UpdateProcThreadAttribute); 140 | } 141 | } 142 | Ok(()) 143 | } 144 | 145 | fn as_mut_ptr(&mut self) -> LPPROC_THREAD_ATTRIBUTE_LIST { 146 | self.raw.as_mut_ptr() as LPPROC_THREAD_ATTRIBUTE_LIST 147 | } 148 | } 149 | 150 | impl<'a> Drop for ProcThreadAttributeList<'a> { 151 | fn drop(&mut self) { 152 | unsafe { DeleteProcThreadAttributeList(self.as_mut_ptr()) }; 153 | } 154 | } 155 | 156 | pub fn run_command(cmdline: &str, mut output_cb: impl FnMut(&[u8])) -> anyhow::Result { 157 | // Don't want to run `cmd /c` since that limits cmd line length to 8192 bytes. 158 | // std::process::Command can't take a string and pass it through to CreateProcess unchanged, 159 | // so call that ourselves. 160 | // https://github.com/rust-lang/rust/issues/38227 161 | 162 | let (pipe_read, pipe_write) = unsafe { 163 | let mut pipe_read: HANDLE = 0; 164 | let mut pipe_write: HANDLE = 0; 165 | let mut attrs = std::mem::zeroed::(); 166 | attrs.nLength = std::mem::size_of::() as u32; 167 | attrs.bInheritHandle = TRUE; 168 | if CreatePipe( 169 | &mut pipe_read, 170 | &mut pipe_write, 171 | &mut attrs, 172 | /* use default buffer size */ 0, 173 | ) == 0 174 | { 175 | win_bail!(CreatePipe); 176 | } 177 | ( 178 | OwnedHandle::from_raw_handle(pipe_read as *mut c_void), 179 | OwnedHandle::from_raw_handle(pipe_write as *mut c_void), 180 | ) 181 | }; 182 | 183 | let process_info = unsafe { 184 | // TODO: Set this to just 0 for console pool jobs. 185 | let process_flags = CREATE_NEW_PROCESS_GROUP | EXTENDED_STARTUPINFO_PRESENT; 186 | 187 | let mut startup_info = std::mem::zeroed::(); 188 | startup_info.StartupInfo.cb = std::mem::size_of::() as u32; 189 | startup_info.StartupInfo.dwFlags = STARTF_USESTDHANDLES; 190 | startup_info.StartupInfo.hStdInput = GetStdHandle(STD_INPUT_HANDLE); 191 | let raw_pipe_write = pipe_write.as_raw_handle() as isize; 192 | startup_info.StartupInfo.hStdOutput = raw_pipe_write; 193 | startup_info.StartupInfo.hStdError = raw_pipe_write; 194 | 195 | // Safely inherit in/out handles. 196 | // https://devblogs.microsoft.com/oldnewthing/20111216-00/?p=8873 197 | let handles = pin!([startup_info.StartupInfo.hStdInput, raw_pipe_write]); 198 | let mut attrs = ProcThreadAttributeList::new(1)?; 199 | attrs.inherit_handles(handles)?; 200 | startup_info.lpAttributeList = attrs.as_mut_ptr(); 201 | 202 | let mut process_info = ProcessInformation::new(); 203 | 204 | let mut cmdline_nul: Vec = String::from(cmdline).into_bytes(); 205 | cmdline_nul.push(0); 206 | 207 | if CreateProcessA( 208 | std::ptr::null_mut(), 209 | cmdline_nul.as_mut_ptr(), 210 | std::ptr::null_mut(), 211 | std::ptr::null_mut(), 212 | /*inherit handles = */ TRUE, 213 | process_flags, 214 | std::ptr::null_mut(), 215 | std::ptr::null_mut(), 216 | &mut startup_info.StartupInfo, 217 | process_info.as_mut_ptr(), 218 | ) == 0 219 | { 220 | let err = GetLastError(); 221 | if err == ERROR_INVALID_PARAMETER { 222 | if cmdline.is_empty() { 223 | anyhow::bail!("CreateProcess failed: command is empty"); 224 | } 225 | if let Some(first_char) = cmdline.bytes().nth(0) { 226 | if first_char == b' ' || first_char == b'\t' { 227 | anyhow::bail!("CreateProcess failed: command has leading whitespace"); 228 | } 229 | } 230 | } 231 | win_bail!(CreateProcessA); 232 | } 233 | drop(pipe_write); 234 | 235 | process_info 236 | }; 237 | 238 | let mut pipe = std::fs::File::from(pipe_read); 239 | let mut buf: [u8; 4 << 10] = [0; 4 << 10]; 240 | loop { 241 | let n = pipe.read(&mut buf)?; 242 | if n == 0 { 243 | break; 244 | } 245 | output_cb(&buf[0..n]); 246 | } 247 | 248 | let exit_code = unsafe { 249 | if WaitForSingleObject(process_info.hProcess, INFINITE) != 0 { 250 | win_bail!(WaitForSingleObject); 251 | } 252 | 253 | let mut exit_code: u32 = 0; 254 | if GetExitCodeProcess(process_info.hProcess, &mut exit_code) == 0 { 255 | win_bail!(GetExitCodeProcess); 256 | } 257 | 258 | exit_code 259 | }; 260 | 261 | let termination = match exit_code { 262 | 0 => Termination::Success, 263 | 0xC000013A => Termination::Interrupted, 264 | _ => Termination::Failure, 265 | }; 266 | 267 | Ok(termination) 268 | } 269 | 270 | #[cfg(test)] 271 | mod tests { 272 | use super::*; 273 | 274 | /// Simple command that is expected to succeed. 275 | #[test] 276 | fn run_echo() -> anyhow::Result<()> { 277 | let mut output = Vec::new(); 278 | run_command("cmd /c echo hello", |buf| output.extend_from_slice(buf))?; 279 | assert_eq!(output, b"hello\r\n"); 280 | Ok(()) 281 | } 282 | 283 | /// Expect empty command to be specially handled in errors. 284 | #[test] 285 | fn empty_command() -> anyhow::Result<()> { 286 | let mut output = Vec::new(); 287 | let err = 288 | run_command("", |buf| output.extend_from_slice(buf)).expect_err("expected failure"); 289 | assert!(err.to_string().contains("command is empty")); 290 | Ok(()) 291 | } 292 | 293 | /// Expect leading whitespace to be specially handled in errors. 294 | #[test] 295 | fn initial_space() -> anyhow::Result<()> { 296 | let mut output = Vec::new(); 297 | let err = run_command(" cmd /c echo hello", |buf| output.extend_from_slice(buf)) 298 | .expect_err("expected failure"); 299 | assert!(err.to_string().contains("command has leading whitespace")); 300 | Ok(()) 301 | } 302 | } 303 | -------------------------------------------------------------------------------- /src/db.rs: -------------------------------------------------------------------------------- 1 | //! The n2 database stores information about previous builds for determining 2 | //! which files are up to date. 3 | 4 | use crate::{ 5 | densemap, densemap::DenseMap, graph::BuildId, graph::FileId, graph::Graph, graph::Hashes, 6 | hash::BuildHash, 7 | }; 8 | use anyhow::{anyhow, bail}; 9 | use std::collections::HashMap; 10 | use std::fs::File; 11 | use std::io::BufReader; 12 | use std::io::Read; 13 | use std::io::Write; 14 | use std::path::Path; 15 | 16 | const VERSION: u32 = 1; 17 | 18 | /// Files are identified by integers that are stable across n2 executions. 19 | #[derive(Debug, Clone, Copy)] 20 | pub struct Id(u32); 21 | impl densemap::Index for Id { 22 | fn index(&self) -> usize { 23 | self.0 as usize 24 | } 25 | } 26 | impl From for Id { 27 | fn from(u: usize) -> Id { 28 | Id(u as u32) 29 | } 30 | } 31 | 32 | /// The loaded state of a database, as needed to make updates to the stored 33 | /// state. Other state is directly loaded into the build graph. 34 | #[derive(Default)] 35 | pub struct IdMap { 36 | /// Maps db::Id to FileId. 37 | fileids: DenseMap, 38 | /// Maps FileId to db::Id. 39 | db_ids: HashMap, 40 | } 41 | 42 | /// RecordWriter buffers writes into a Vec. 43 | /// We attempt to write a full record per underlying finish() to lessen the chance of writing partial records. 44 | #[derive(Default)] 45 | struct RecordWriter(Vec); 46 | 47 | impl RecordWriter { 48 | fn write(&mut self, buf: &[u8]) { 49 | self.0.extend_from_slice(buf); 50 | } 51 | 52 | fn write_u16(&mut self, n: u16) { 53 | self.write(&n.to_le_bytes()); 54 | } 55 | 56 | fn write_u24(&mut self, n: u32) { 57 | self.write(&n.to_le_bytes()[..3]); 58 | } 59 | 60 | fn write_u64(&mut self, n: u64) { 61 | self.write(&n.to_le_bytes()); 62 | } 63 | 64 | fn write_str(&mut self, s: &str) { 65 | self.write_u16(s.len() as u16); 66 | self.write(s.as_bytes()); 67 | } 68 | 69 | fn write_id(&mut self, id: Id) { 70 | if id.0 > (1 << 24) { 71 | panic!("too many fileids"); 72 | } 73 | self.write_u24(id.0); 74 | } 75 | 76 | fn finish(&self, w: &mut impl Write) -> std::io::Result<()> { 77 | w.write_all(&self.0) 78 | } 79 | } 80 | 81 | /// An opened database, ready for writes. 82 | pub struct Writer { 83 | ids: IdMap, 84 | w: File, 85 | } 86 | 87 | impl Writer { 88 | fn create(path: &Path) -> std::io::Result { 89 | let f = std::fs::File::create(path)?; 90 | let mut w = Self::from_opened(IdMap::default(), f); 91 | w.write_signature()?; 92 | Ok(w) 93 | } 94 | 95 | fn from_opened(ids: IdMap, w: File) -> Self { 96 | Writer { ids, w } 97 | } 98 | 99 | fn write_signature(&mut self) -> std::io::Result<()> { 100 | self.w.write_all("n2db".as_bytes())?; 101 | self.w.write_all(&u32::to_le_bytes(VERSION)) 102 | } 103 | 104 | fn write_path(&mut self, name: &str) -> std::io::Result<()> { 105 | if name.len() >= 0b1000_0000_0000_0000 { 106 | panic!("filename too long"); 107 | } 108 | let mut w = RecordWriter::default(); 109 | w.write_str(&name); 110 | w.finish(&mut self.w) 111 | } 112 | 113 | fn ensure_id(&mut self, graph: &Graph, fileid: FileId) -> std::io::Result { 114 | let id = match self.ids.db_ids.get(&fileid) { 115 | Some(&id) => id, 116 | None => { 117 | let id = self.ids.fileids.push(fileid); 118 | self.ids.db_ids.insert(fileid, id); 119 | self.write_path(&graph.file(fileid).name)?; 120 | id 121 | } 122 | }; 123 | Ok(id) 124 | } 125 | 126 | pub fn write_build( 127 | &mut self, 128 | graph: &Graph, 129 | id: BuildId, 130 | hash: BuildHash, 131 | ) -> std::io::Result<()> { 132 | let build = &graph.builds[id]; 133 | let mut w = RecordWriter::default(); 134 | let outs = build.outs(); 135 | let mark = (outs.len() as u16) | 0b1000_0000_0000_0000; 136 | w.write_u16(mark); 137 | for &out in outs { 138 | let id = self.ensure_id(graph, out)?; 139 | w.write_id(id); 140 | } 141 | 142 | let deps = build.discovered_ins(); 143 | w.write_u16(deps.len() as u16); 144 | for &dep in deps { 145 | let id = self.ensure_id(graph, dep)?; 146 | w.write_id(id); 147 | } 148 | 149 | w.write_u64(hash.0); 150 | w.finish(&mut self.w) 151 | } 152 | } 153 | 154 | struct Reader<'a> { 155 | r: BufReader<&'a mut File>, 156 | ids: IdMap, 157 | graph: &'a mut Graph, 158 | hashes: &'a mut Hashes, 159 | } 160 | 161 | impl<'a> Reader<'a> { 162 | fn read_u16(&mut self) -> std::io::Result { 163 | let mut buf: [u8; 2] = [0; 2]; 164 | self.r.read_exact(&mut buf[..])?; 165 | Ok(u16::from_le_bytes(buf)) 166 | } 167 | 168 | fn read_u24(&mut self) -> std::io::Result { 169 | let mut buf: [u8; 4] = [0; 4]; 170 | self.r.read_exact(&mut buf[..3])?; 171 | Ok(u32::from_le_bytes(buf)) 172 | } 173 | 174 | fn read_u64(&mut self) -> std::io::Result { 175 | let mut buf: [u8; 8] = [0; 8]; 176 | self.r.read_exact(&mut buf)?; 177 | Ok(u64::from_le_bytes(buf)) 178 | } 179 | 180 | fn read_id(&mut self) -> std::io::Result { 181 | self.read_u24().map(Id) 182 | } 183 | 184 | fn read_str(&mut self, len: usize) -> std::io::Result { 185 | let mut buf = vec![0; len]; 186 | self.r.read_exact(buf.as_mut_slice())?; 187 | Ok(unsafe { String::from_utf8_unchecked(buf) }) 188 | } 189 | 190 | fn read_path(&mut self, len: usize) -> std::io::Result<()> { 191 | let name = self.read_str(len)?; 192 | // No canonicalization needed, paths were written canonicalized. 193 | let fileid = self.graph.files.id_from_canonical(name); 194 | let dbid = self.ids.fileids.push(fileid); 195 | self.ids.db_ids.insert(fileid, dbid); 196 | Ok(()) 197 | } 198 | 199 | fn read_build(&mut self, len: usize) -> std::io::Result<()> { 200 | // This record logs a build. We expect all the outputs to be 201 | // outputs of the same build id; if not, that means the graph has 202 | // changed since this log, in which case we just ignore it. 203 | // 204 | // It's possible we log a build that generates files A B, then 205 | // change the build file such that it only generates file A; this 206 | // logic will still attach the old dependencies to A, but it 207 | // shouldn't matter because the changed command line will cause us 208 | // to rebuild A regardless, and these dependencies are only used 209 | // to affect dirty checking, not build order. 210 | 211 | let mut unique_bid = None; 212 | let mut obsolete = false; 213 | for _ in 0..len { 214 | let fileid = self.read_id()?; 215 | if obsolete { 216 | // Even though we know we don't want this record, we must 217 | // keep reading to parse through it. 218 | continue; 219 | } 220 | match self.graph.file(self.ids.fileids[fileid]).input { 221 | None => { 222 | obsolete = true; 223 | } 224 | Some(bid) => { 225 | match unique_bid { 226 | None => unique_bid = Some(bid), 227 | Some(unique_bid) if unique_bid == bid => { 228 | // Ok, matches the existing id. 229 | } 230 | Some(_) => { 231 | // Mismatch. 232 | unique_bid = None; 233 | obsolete = true; 234 | } 235 | } 236 | } 237 | } 238 | } 239 | 240 | let len = self.read_u16()?; 241 | let mut deps = Vec::new(); 242 | for _ in 0..len { 243 | let id = self.read_id()?; 244 | deps.push(self.ids.fileids[id]); 245 | } 246 | 247 | let hash = BuildHash(self.read_u64()?); 248 | 249 | // unique_bid is set here if this record is valid. 250 | if let Some(id) = unique_bid { 251 | // Common case: only one associated build. 252 | self.graph.builds[id].set_discovered_ins(deps); 253 | self.hashes.set(id, hash); 254 | } 255 | Ok(()) 256 | } 257 | 258 | fn read_signature(&mut self) -> anyhow::Result<()> { 259 | let mut buf: [u8; 4] = [0; 4]; 260 | self.r.read_exact(&mut buf[..])?; 261 | if buf.as_slice() != "n2db".as_bytes() { 262 | bail!("invalid db signature"); 263 | } 264 | self.r.read_exact(&mut buf[..])?; 265 | let version = u32::from_le_bytes(buf); 266 | if version != VERSION { 267 | bail!("db version mismatch: got {version}, expected {VERSION}; TODO: db upgrades etc"); 268 | } 269 | Ok(()) 270 | } 271 | 272 | fn read_file(&mut self) -> anyhow::Result<()> { 273 | self.read_signature()?; 274 | loop { 275 | let mut len = match self.read_u16() { 276 | Ok(r) => r, 277 | Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => break, 278 | Err(err) => bail!(err), 279 | }; 280 | let mask = 0b1000_0000_0000_0000; 281 | if len & mask == 0 { 282 | self.read_path(len as usize)?; 283 | } else { 284 | len &= !mask; 285 | self.read_build(len as usize)?; 286 | } 287 | } 288 | Ok(()) 289 | } 290 | 291 | /// Reads an on-disk database, loading its state into the provided Graph/Hashes. 292 | fn read(f: &mut File, graph: &mut Graph, hashes: &mut Hashes) -> anyhow::Result { 293 | let mut r = Reader { 294 | r: std::io::BufReader::new(f), 295 | ids: IdMap::default(), 296 | graph, 297 | hashes, 298 | }; 299 | r.read_file()?; 300 | 301 | Ok(r.ids) 302 | } 303 | } 304 | 305 | /// Opens or creates an on-disk database, loading its state into the provided Graph. 306 | pub fn open(path: &Path, graph: &mut Graph, hashes: &mut Hashes) -> anyhow::Result { 307 | match std::fs::OpenOptions::new() 308 | .read(true) 309 | .append(true) 310 | .open(path) 311 | { 312 | Ok(mut f) => { 313 | let ids = Reader::read(&mut f, graph, hashes)?; 314 | Ok(Writer::from_opened(ids, f)) 315 | } 316 | Err(err) if err.kind() == std::io::ErrorKind::NotFound => { 317 | let w = Writer::create(path)?; 318 | Ok(w) 319 | } 320 | Err(err) => Err(anyhow!(err)), 321 | } 322 | } 323 | -------------------------------------------------------------------------------- /src/load.rs: -------------------------------------------------------------------------------- 1 | //! Graph loading: runs .ninja parsing and constructs the build graph from it. 2 | 3 | use crate::{ 4 | canon::{canonicalize_path, to_owned_canon_path}, 5 | db, 6 | eval::{self, EvalPart, EvalString}, 7 | graph::{self, FileId, RspFile}, 8 | parse::{self, Statement}, 9 | scanner, 10 | smallmap::SmallMap, 11 | trace, 12 | }; 13 | use anyhow::{anyhow, bail}; 14 | use std::collections::HashMap; 15 | use std::path::PathBuf; 16 | use std::{borrow::Cow, path::Path}; 17 | 18 | /// A variable lookup environment for magic $in/$out variables. 19 | struct BuildImplicitVars<'a> { 20 | graph: &'a graph::Graph, 21 | build: &'a graph::Build, 22 | } 23 | impl<'a> BuildImplicitVars<'a> { 24 | fn file_list(&self, ids: &[FileId], sep: char) -> String { 25 | let mut out = String::new(); 26 | for &id in ids { 27 | if !out.is_empty() { 28 | out.push(sep); 29 | } 30 | out.push_str(&self.graph.file(id).name); 31 | } 32 | out 33 | } 34 | } 35 | impl<'a> eval::Env for BuildImplicitVars<'a> { 36 | fn get_var(&self, var: &str) -> Option>> { 37 | let string_to_evalstring = 38 | |s: String| Some(EvalString::new(vec![EvalPart::Literal(Cow::Owned(s))])); 39 | match var { 40 | "in" => string_to_evalstring(self.file_list(self.build.explicit_ins(), ' ')), 41 | "in_newline" => string_to_evalstring(self.file_list(self.build.explicit_ins(), '\n')), 42 | "out" => string_to_evalstring(self.file_list(self.build.explicit_outs(), ' ')), 43 | "out_newline" => string_to_evalstring(self.file_list(self.build.explicit_outs(), '\n')), 44 | _ => None, 45 | } 46 | } 47 | } 48 | 49 | /// Internal state used while loading. 50 | #[derive(Default)] 51 | pub struct Loader { 52 | pub graph: graph::Graph, 53 | default: Vec, 54 | /// rule name -> list of (key, val) 55 | rules: HashMap>>, 56 | pools: SmallMap, 57 | builddir: Option, 58 | } 59 | 60 | impl Loader { 61 | pub fn new() -> Self { 62 | let mut loader = Loader::default(); 63 | 64 | loader.rules.insert("phony".to_owned(), SmallMap::default()); 65 | 66 | loader 67 | } 68 | 69 | /// Convert a path string to a FileId. 70 | fn path(&mut self, mut path: String) -> FileId { 71 | // Perf: this is called while parsing build.ninja files. We go to 72 | // some effort to avoid allocating in the common case of a path that 73 | // refers to a file that is already known. 74 | canonicalize_path(&mut path); 75 | self.graph.files.id_from_canonical(path) 76 | } 77 | 78 | fn evaluate_path(&mut self, path: EvalString<&str>, envs: &[&dyn eval::Env]) -> FileId { 79 | self.path(path.evaluate(envs)) 80 | } 81 | 82 | fn evaluate_paths( 83 | &mut self, 84 | paths: Vec>, 85 | envs: &[&dyn eval::Env], 86 | ) -> Vec { 87 | paths 88 | .into_iter() 89 | .map(|path| self.evaluate_path(path, envs)) 90 | .collect() 91 | } 92 | 93 | fn add_build( 94 | &mut self, 95 | filename: std::rc::Rc, 96 | env: &eval::Vars, 97 | b: parse::Build, 98 | ) -> anyhow::Result<()> { 99 | let ins = graph::BuildIns { 100 | ids: self.evaluate_paths(b.ins, &[&b.vars, env]), 101 | explicit: b.explicit_ins, 102 | implicit: b.implicit_ins, 103 | order_only: b.order_only_ins, 104 | // validation is implied by the other counts 105 | }; 106 | let outs = graph::BuildOuts { 107 | ids: self.evaluate_paths(b.outs, &[&b.vars, env]), 108 | explicit: b.explicit_outs, 109 | }; 110 | let mut build = graph::Build::new( 111 | graph::FileLoc { 112 | filename, 113 | line: b.line, 114 | }, 115 | ins, 116 | outs, 117 | ); 118 | 119 | let rule = match self.rules.get(b.rule) { 120 | Some(r) => r, 121 | None => bail!("unknown rule {:?}", b.rule), 122 | }; 123 | 124 | let implicit_vars = BuildImplicitVars { 125 | graph: &self.graph, 126 | build: &build, 127 | }; 128 | 129 | // temp variable in order to not move all of b into the closure 130 | let build_vars = &b.vars; 131 | let lookup = |key: &str| -> Option { 132 | // Look up `key = ...` binding in build and rule block. 133 | // See "Variable scope" in the design notes. 134 | Some(match build_vars.get(key) { 135 | Some(val) => val.evaluate(&[env]), 136 | None => rule.get(key)?.evaluate(&[&implicit_vars, build_vars, env]), 137 | }) 138 | }; 139 | 140 | let cmdline = lookup("command"); 141 | let desc = lookup("description"); 142 | let depfile = lookup("depfile"); 143 | let parse_showincludes = match lookup("deps").as_deref() { 144 | None => false, 145 | Some("gcc") => false, 146 | Some("msvc") => true, 147 | Some(other) => bail!("invalid deps attribute {:?}", other), 148 | }; 149 | let pool = lookup("pool"); 150 | 151 | let rspfile_path = lookup("rspfile"); 152 | let rspfile_content = lookup("rspfile_content"); 153 | let rspfile = match (rspfile_path, rspfile_content) { 154 | (None, None) => None, 155 | (Some(path), Some(content)) => Some(RspFile { 156 | path: std::path::PathBuf::from(path), 157 | content, 158 | }), 159 | _ => bail!("rspfile and rspfile_content need to be both specified"), 160 | }; 161 | let hide_success = lookup("hide_success").is_some(); 162 | let hide_progress = lookup("hide_progress").is_some(); 163 | 164 | build.cmdline = cmdline; 165 | build.desc = desc; 166 | build.depfile = depfile; 167 | build.parse_showincludes = parse_showincludes; 168 | build.rspfile = rspfile; 169 | build.pool = pool; 170 | build.hide_success = hide_success; 171 | build.hide_progress = hide_progress; 172 | 173 | self.graph.add_build(build) 174 | } 175 | 176 | pub fn read_file_by_id(&self, id: FileId) -> anyhow::Result<(PathBuf, Vec)> { 177 | let path = self.graph.file(id).path().to_path_buf(); 178 | 179 | match trace::scope("read file", || scanner::read_file_with_nul(&path)) { 180 | Ok(b) => Ok((path, b)), 181 | Err(e) => bail!("read {}: {}", path.display(), e), 182 | } 183 | } 184 | 185 | pub fn parse_with_parser( 186 | &mut self, 187 | parser: &mut parse::Parser, 188 | path: PathBuf, 189 | envs: &[&dyn eval::Env], 190 | ) -> anyhow::Result<()> { 191 | let filename = std::rc::Rc::new(path); 192 | 193 | loop { 194 | let stmt = match parser 195 | .read() 196 | .map_err(|err| anyhow!(parser.format_parse_error(&filename, err)))? 197 | { 198 | None => break, 199 | Some(s) => s, 200 | }; 201 | 202 | match stmt { 203 | Statement::Include(in_path) | Statement::Subninja(in_path) => { 204 | let id = self.evaluate_path(in_path, &[&parser.vars]); 205 | let (path, bytes) = self.read_file_by_id(id)?; 206 | let bytes = std::rc::Rc::new(bytes); 207 | let mut sub_parser = parse::Parser::new(&bytes); 208 | 209 | sub_parser.inherit(&parser); 210 | self.parse_with_parser(&mut sub_parser, path, envs)?; 211 | } 212 | 213 | Statement::Default(defaults) => { 214 | let evaluated = self.evaluate_paths(defaults, &[&parser.vars]); 215 | self.default.extend(evaluated); 216 | } 217 | 218 | Statement::Rule(rule) => { 219 | let mut vars: SmallMap> = SmallMap::default(); 220 | for (name, val) in rule.vars.into_iter() { 221 | // TODO: We should not need to call .into_owned() here 222 | // if we keep the contents of all included files in 223 | // memory. 224 | vars.insert(name.to_owned(), val.into_owned()); 225 | } 226 | self.rules.insert(rule.name.to_owned(), vars); 227 | } 228 | 229 | Statement::Build(build) => self.add_build(filename.clone(), &parser.vars, build)?, 230 | 231 | Statement::Pool(pool) => { 232 | self.pools.insert(pool.name.to_string(), pool.depth); 233 | } 234 | }; 235 | } 236 | 237 | self.builddir = parser.vars.get("builddir").cloned(); 238 | Ok(()) 239 | } 240 | } 241 | 242 | /// State loaded by read(). 243 | pub struct State { 244 | pub graph: graph::Graph, 245 | pub db: db::Writer, 246 | pub hashes: graph::Hashes, 247 | pub default: Vec, 248 | pub pools: SmallMap, 249 | } 250 | 251 | /// Load build.ninja/.n2_db and return the loaded build graph and state. 252 | pub fn read(build_filename: &str) -> anyhow::Result { 253 | let mut loader = Loader::new(); 254 | trace::scope("loader.read_file", || { 255 | let id = loader 256 | .graph 257 | .files 258 | .id_from_canonical(to_owned_canon_path(build_filename)); 259 | let (path, bytes) = loader.read_file_by_id(id)?; 260 | let mut parser = parse::Parser::new(&bytes); 261 | 262 | loader.parse_with_parser(&mut parser, path, &[]) 263 | })?; 264 | 265 | let mut hashes = graph::Hashes::default(); 266 | let db = trace::scope("db::open", || { 267 | let mut db_path = PathBuf::from(".n2_db"); 268 | if let Some(builddir) = &loader.builddir { 269 | db_path = Path::new(&builddir).join(db_path); 270 | if let Some(parent) = db_path.parent() { 271 | std::fs::create_dir_all(parent)?; 272 | } 273 | }; 274 | db::open(&db_path, &mut loader.graph, &mut hashes) 275 | }) 276 | .map_err(|err| anyhow!("load .n2_db: {}", err))?; 277 | 278 | Ok(State { 279 | graph: loader.graph, 280 | db, 281 | hashes, 282 | default: loader.default, 283 | pools: loader.pools, 284 | }) 285 | } 286 | 287 | /// Parse a single file's content. 288 | #[cfg(test)] 289 | pub fn parse(name: &str, mut content: Vec) -> anyhow::Result { 290 | content.push(0); 291 | let mut loader = Loader::new(); 292 | 293 | trace::scope("loader.read_file", || { 294 | let mut parser = parse::Parser::new(&content); 295 | loader.parse_with_parser(&mut parser, PathBuf::from(name), &[]) 296 | })?; 297 | 298 | Ok(loader.graph) 299 | } 300 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "anstyle" 7 | version = "1.0.10" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" 10 | 11 | [[package]] 12 | name = "anyhow" 13 | version = "1.0.95" 14 | source = "registry+https://github.com/rust-lang/crates.io-index" 15 | checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" 16 | 17 | [[package]] 18 | name = "bitflags" 19 | version = "2.6.0" 20 | source = "registry+https://github.com/rust-lang/crates.io-index" 21 | checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" 22 | 23 | [[package]] 24 | name = "cc" 25 | version = "1.2.7" 26 | source = "registry+https://github.com/rust-lang/crates.io-index" 27 | checksum = "a012a0df96dd6d06ba9a1b29d6402d1a5d77c6befd2566afdc26e10603dc93d7" 28 | dependencies = [ 29 | "shlex", 30 | ] 31 | 32 | [[package]] 33 | name = "cfg-if" 34 | version = "1.0.0" 35 | source = "registry+https://github.com/rust-lang/crates.io-index" 36 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 37 | 38 | [[package]] 39 | name = "clap" 40 | version = "4.5.26" 41 | source = "registry+https://github.com/rust-lang/crates.io-index" 42 | checksum = "a8eb5e908ef3a6efbe1ed62520fb7287959888c88485abe072543190ecc66783" 43 | dependencies = [ 44 | "clap_builder", 45 | ] 46 | 47 | [[package]] 48 | name = "clap_builder" 49 | version = "4.5.26" 50 | source = "registry+https://github.com/rust-lang/crates.io-index" 51 | checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" 52 | dependencies = [ 53 | "anstyle", 54 | "clap_lex", 55 | "terminal_size", 56 | ] 57 | 58 | [[package]] 59 | name = "clap_lex" 60 | version = "0.7.4" 61 | source = "registry+https://github.com/rust-lang/crates.io-index" 62 | checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" 63 | 64 | [[package]] 65 | name = "condtype" 66 | version = "1.3.0" 67 | source = "registry+https://github.com/rust-lang/crates.io-index" 68 | checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af" 69 | 70 | [[package]] 71 | name = "divan" 72 | version = "0.1.17" 73 | source = "registry+https://github.com/rust-lang/crates.io-index" 74 | checksum = "e0583193020b29b03682d8d33bb53a5b0f50df6daacece12ca99b904cfdcb8c4" 75 | dependencies = [ 76 | "cfg-if", 77 | "clap", 78 | "condtype", 79 | "divan-macros", 80 | "libc", 81 | "regex-lite", 82 | ] 83 | 84 | [[package]] 85 | name = "divan-macros" 86 | version = "0.1.17" 87 | source = "registry+https://github.com/rust-lang/crates.io-index" 88 | checksum = "8dc51d98e636f5e3b0759a39257458b22619cac7e96d932da6eeb052891bb67c" 89 | dependencies = [ 90 | "proc-macro2", 91 | "quote", 92 | "syn", 93 | ] 94 | 95 | [[package]] 96 | name = "errno" 97 | version = "0.3.10" 98 | source = "registry+https://github.com/rust-lang/crates.io-index" 99 | checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" 100 | dependencies = [ 101 | "libc", 102 | "windows-sys 0.59.0", 103 | ] 104 | 105 | [[package]] 106 | name = "fastrand" 107 | version = "2.3.0" 108 | source = "registry+https://github.com/rust-lang/crates.io-index" 109 | checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" 110 | 111 | [[package]] 112 | name = "getrandom" 113 | version = "0.2.15" 114 | source = "registry+https://github.com/rust-lang/crates.io-index" 115 | checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" 116 | dependencies = [ 117 | "cfg-if", 118 | "libc", 119 | "wasi", 120 | ] 121 | 122 | [[package]] 123 | name = "jemalloc-sys" 124 | version = "0.5.4+5.3.0-patched" 125 | source = "registry+https://github.com/rust-lang/crates.io-index" 126 | checksum = "ac6c1946e1cea1788cbfde01c993b52a10e2da07f4bac608228d1bed20bfebf2" 127 | dependencies = [ 128 | "cc", 129 | "libc", 130 | ] 131 | 132 | [[package]] 133 | name = "jemallocator" 134 | version = "0.5.4" 135 | source = "registry+https://github.com/rust-lang/crates.io-index" 136 | checksum = "a0de374a9f8e63150e6f5e8a60cc14c668226d7a347d8aee1a45766e3c4dd3bc" 137 | dependencies = [ 138 | "jemalloc-sys", 139 | "libc", 140 | ] 141 | 142 | [[package]] 143 | name = "lexopt" 144 | version = "0.3.0" 145 | source = "registry+https://github.com/rust-lang/crates.io-index" 146 | checksum = "baff4b617f7df3d896f97fe922b64817f6cd9a756bb81d40f8883f2f66dcb401" 147 | 148 | [[package]] 149 | name = "libc" 150 | version = "0.2.169" 151 | source = "registry+https://github.com/rust-lang/crates.io-index" 152 | checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" 153 | 154 | [[package]] 155 | name = "linux-raw-sys" 156 | version = "0.4.15" 157 | source = "registry+https://github.com/rust-lang/crates.io-index" 158 | checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" 159 | 160 | [[package]] 161 | name = "n2" 162 | version = "0.1.0" 163 | dependencies = [ 164 | "anyhow", 165 | "divan", 166 | "jemallocator", 167 | "lexopt", 168 | "libc", 169 | "rustc-hash", 170 | "tempfile", 171 | "windows-sys 0.48.0", 172 | ] 173 | 174 | [[package]] 175 | name = "once_cell" 176 | version = "1.20.2" 177 | source = "registry+https://github.com/rust-lang/crates.io-index" 178 | checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" 179 | 180 | [[package]] 181 | name = "proc-macro2" 182 | version = "1.0.93" 183 | source = "registry+https://github.com/rust-lang/crates.io-index" 184 | checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" 185 | dependencies = [ 186 | "unicode-ident", 187 | ] 188 | 189 | [[package]] 190 | name = "quote" 191 | version = "1.0.38" 192 | source = "registry+https://github.com/rust-lang/crates.io-index" 193 | checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" 194 | dependencies = [ 195 | "proc-macro2", 196 | ] 197 | 198 | [[package]] 199 | name = "regex-lite" 200 | version = "0.1.6" 201 | source = "registry+https://github.com/rust-lang/crates.io-index" 202 | checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" 203 | 204 | [[package]] 205 | name = "rustc-hash" 206 | version = "1.1.0" 207 | source = "registry+https://github.com/rust-lang/crates.io-index" 208 | checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" 209 | 210 | [[package]] 211 | name = "rustix" 212 | version = "0.38.43" 213 | source = "registry+https://github.com/rust-lang/crates.io-index" 214 | checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" 215 | dependencies = [ 216 | "bitflags", 217 | "errno", 218 | "libc", 219 | "linux-raw-sys", 220 | "windows-sys 0.59.0", 221 | ] 222 | 223 | [[package]] 224 | name = "shlex" 225 | version = "1.3.0" 226 | source = "registry+https://github.com/rust-lang/crates.io-index" 227 | checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" 228 | 229 | [[package]] 230 | name = "syn" 231 | version = "2.0.96" 232 | source = "registry+https://github.com/rust-lang/crates.io-index" 233 | checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" 234 | dependencies = [ 235 | "proc-macro2", 236 | "quote", 237 | "unicode-ident", 238 | ] 239 | 240 | [[package]] 241 | name = "tempfile" 242 | version = "3.15.0" 243 | source = "registry+https://github.com/rust-lang/crates.io-index" 244 | checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" 245 | dependencies = [ 246 | "cfg-if", 247 | "fastrand", 248 | "getrandom", 249 | "once_cell", 250 | "rustix", 251 | "windows-sys 0.59.0", 252 | ] 253 | 254 | [[package]] 255 | name = "terminal_size" 256 | version = "0.4.1" 257 | source = "registry+https://github.com/rust-lang/crates.io-index" 258 | checksum = "5352447f921fda68cf61b4101566c0bdb5104eff6804d0678e5227580ab6a4e9" 259 | dependencies = [ 260 | "rustix", 261 | "windows-sys 0.59.0", 262 | ] 263 | 264 | [[package]] 265 | name = "unicode-ident" 266 | version = "1.0.14" 267 | source = "registry+https://github.com/rust-lang/crates.io-index" 268 | checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" 269 | 270 | [[package]] 271 | name = "wasi" 272 | version = "0.11.0+wasi-snapshot-preview1" 273 | source = "registry+https://github.com/rust-lang/crates.io-index" 274 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" 275 | 276 | [[package]] 277 | name = "windows-sys" 278 | version = "0.48.0" 279 | source = "registry+https://github.com/rust-lang/crates.io-index" 280 | checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" 281 | dependencies = [ 282 | "windows-targets 0.48.5", 283 | ] 284 | 285 | [[package]] 286 | name = "windows-sys" 287 | version = "0.59.0" 288 | source = "registry+https://github.com/rust-lang/crates.io-index" 289 | checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" 290 | dependencies = [ 291 | "windows-targets 0.52.6", 292 | ] 293 | 294 | [[package]] 295 | name = "windows-targets" 296 | version = "0.48.5" 297 | source = "registry+https://github.com/rust-lang/crates.io-index" 298 | checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" 299 | dependencies = [ 300 | "windows_aarch64_gnullvm 0.48.5", 301 | "windows_aarch64_msvc 0.48.5", 302 | "windows_i686_gnu 0.48.5", 303 | "windows_i686_msvc 0.48.5", 304 | "windows_x86_64_gnu 0.48.5", 305 | "windows_x86_64_gnullvm 0.48.5", 306 | "windows_x86_64_msvc 0.48.5", 307 | ] 308 | 309 | [[package]] 310 | name = "windows-targets" 311 | version = "0.52.6" 312 | source = "registry+https://github.com/rust-lang/crates.io-index" 313 | checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" 314 | dependencies = [ 315 | "windows_aarch64_gnullvm 0.52.6", 316 | "windows_aarch64_msvc 0.52.6", 317 | "windows_i686_gnu 0.52.6", 318 | "windows_i686_gnullvm", 319 | "windows_i686_msvc 0.52.6", 320 | "windows_x86_64_gnu 0.52.6", 321 | "windows_x86_64_gnullvm 0.52.6", 322 | "windows_x86_64_msvc 0.52.6", 323 | ] 324 | 325 | [[package]] 326 | name = "windows_aarch64_gnullvm" 327 | version = "0.48.5" 328 | source = "registry+https://github.com/rust-lang/crates.io-index" 329 | checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" 330 | 331 | [[package]] 332 | name = "windows_aarch64_gnullvm" 333 | version = "0.52.6" 334 | source = "registry+https://github.com/rust-lang/crates.io-index" 335 | checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" 336 | 337 | [[package]] 338 | name = "windows_aarch64_msvc" 339 | version = "0.48.5" 340 | source = "registry+https://github.com/rust-lang/crates.io-index" 341 | checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" 342 | 343 | [[package]] 344 | name = "windows_aarch64_msvc" 345 | version = "0.52.6" 346 | source = "registry+https://github.com/rust-lang/crates.io-index" 347 | checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" 348 | 349 | [[package]] 350 | name = "windows_i686_gnu" 351 | version = "0.48.5" 352 | source = "registry+https://github.com/rust-lang/crates.io-index" 353 | checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" 354 | 355 | [[package]] 356 | name = "windows_i686_gnu" 357 | version = "0.52.6" 358 | source = "registry+https://github.com/rust-lang/crates.io-index" 359 | checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" 360 | 361 | [[package]] 362 | name = "windows_i686_gnullvm" 363 | version = "0.52.6" 364 | source = "registry+https://github.com/rust-lang/crates.io-index" 365 | checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" 366 | 367 | [[package]] 368 | name = "windows_i686_msvc" 369 | version = "0.48.5" 370 | source = "registry+https://github.com/rust-lang/crates.io-index" 371 | checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" 372 | 373 | [[package]] 374 | name = "windows_i686_msvc" 375 | version = "0.52.6" 376 | source = "registry+https://github.com/rust-lang/crates.io-index" 377 | checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" 378 | 379 | [[package]] 380 | name = "windows_x86_64_gnu" 381 | version = "0.48.5" 382 | source = "registry+https://github.com/rust-lang/crates.io-index" 383 | checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" 384 | 385 | [[package]] 386 | name = "windows_x86_64_gnu" 387 | version = "0.52.6" 388 | source = "registry+https://github.com/rust-lang/crates.io-index" 389 | checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" 390 | 391 | [[package]] 392 | name = "windows_x86_64_gnullvm" 393 | version = "0.48.5" 394 | source = "registry+https://github.com/rust-lang/crates.io-index" 395 | checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" 396 | 397 | [[package]] 398 | name = "windows_x86_64_gnullvm" 399 | version = "0.52.6" 400 | source = "registry+https://github.com/rust-lang/crates.io-index" 401 | checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" 402 | 403 | [[package]] 404 | name = "windows_x86_64_msvc" 405 | version = "0.48.5" 406 | source = "registry+https://github.com/rust-lang/crates.io-index" 407 | checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" 408 | 409 | [[package]] 410 | name = "windows_x86_64_msvc" 411 | version = "0.52.6" 412 | source = "registry+https://github.com/rust-lang/crates.io-index" 413 | checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" 414 | -------------------------------------------------------------------------------- /src/progress_fancy.rs: -------------------------------------------------------------------------------- 1 | //! Build progress reporting for a "fancy" console, with progress bar etc. 2 | 3 | use crate::progress::{build_message, Progress}; 4 | use crate::{ 5 | graph::Build, graph::BuildId, process::Termination, task::TaskResult, terminal, 6 | work::BuildState, work::StateCounts, 7 | }; 8 | use std::collections::VecDeque; 9 | use std::io::Write; 10 | use std::sync::Arc; 11 | use std::sync::Condvar; 12 | use std::sync::Mutex; 13 | use std::time::Duration; 14 | use std::time::Instant; 15 | 16 | /// Currently running build task, as tracked for progress updates. 17 | struct Task { 18 | id: BuildId, 19 | /// When the task started running. 20 | start: Instant, 21 | /// Build status message for the task. 22 | message: String, 23 | /// Last line of output from the task. 24 | last_line: Option, 25 | } 26 | 27 | /// Progress implementation for "fancy" console, with progress bar etc. 28 | /// Each time it prints, it clears from the cursor to the end of the console, 29 | /// prints the status text, and then moves moves the cursor back up to the 30 | /// start position. This means on errors etc. we can clear any status by 31 | /// clearing the console too. 32 | pub struct FancyConsoleProgress { 33 | state: Arc>, 34 | thread: Option>, 35 | } 36 | 37 | /// Screen updates happen after this duration passes, to reduce the amount 38 | /// of printing in the case of rapid updates. This helps with terminal flicker. 39 | const UPDATE_DELAY: Duration = std::time::Duration::from_millis(50); 40 | 41 | /// If there are no updates for this duration, the progress will print anyway. 42 | /// This lets the progress show ticking timers for long-running tasks so things 43 | /// do not appear hung. 44 | const TIMEOUT_DELAY: Duration = std::time::Duration::from_millis(500); 45 | 46 | impl FancyConsoleProgress { 47 | pub fn new(verbose: bool) -> Self { 48 | let dirty_cond = Arc::new(Condvar::new()); 49 | let state = Arc::new(Mutex::new(FancyState { 50 | done: false, 51 | pending: Vec::new(), 52 | dirty: false, 53 | dirty_cond: dirty_cond.clone(), 54 | counts: StateCounts::default(), 55 | tasks: VecDeque::new(), 56 | verbose, 57 | })); 58 | 59 | // Thread to debounce status updates -- waits a bit, then prints after 60 | // any dirty state. 61 | let thread = std::thread::spawn({ 62 | let state_lock = state.clone(); 63 | move || loop { 64 | // Wait to be notified of a display update or timeout. 65 | { 66 | let (state, _) = dirty_cond 67 | .wait_timeout_while( 68 | state_lock.lock().unwrap(), 69 | TIMEOUT_DELAY - UPDATE_DELAY, 70 | |state| !state.done && !state.dirty, 71 | ) 72 | .unwrap(); 73 | if state.done { 74 | std::io::stdout().write_all(&state.pending).unwrap(); 75 | break; 76 | } 77 | } 78 | 79 | // Delay a little bit in case more display updates come in. 80 | // We know .dirty will only ever be cleared below, so we 81 | // can drop the lock here while we sleep. 82 | std::thread::sleep(UPDATE_DELAY); 83 | 84 | state_lock.lock().unwrap().print_progress(); 85 | } 86 | }); 87 | 88 | FancyConsoleProgress { 89 | state, 90 | thread: Some(thread), 91 | } 92 | } 93 | } 94 | 95 | impl Progress for FancyConsoleProgress { 96 | fn update(&self, counts: &StateCounts) { 97 | self.state.lock().unwrap().update(counts); 98 | } 99 | 100 | fn task_started(&self, id: BuildId, build: &Build) { 101 | self.state.lock().unwrap().task_started(id, build); 102 | } 103 | 104 | fn task_output(&self, id: BuildId, line: Vec) { 105 | self.state.lock().unwrap().task_output(id, line); 106 | } 107 | 108 | fn task_finished(&self, id: BuildId, build: &Build, result: &TaskResult) { 109 | self.state.lock().unwrap().task_finished(id, build, result); 110 | } 111 | 112 | fn log(&self, msg: &str) { 113 | self.state.lock().unwrap().log(msg); 114 | } 115 | } 116 | 117 | impl Drop for FancyConsoleProgress { 118 | fn drop(&mut self) { 119 | self.state.lock().unwrap().cleanup(); 120 | self.thread.take().unwrap().join().unwrap(); 121 | } 122 | } 123 | 124 | struct FancyState { 125 | done: bool, 126 | 127 | /// Text to print on the next update. 128 | /// Typically starts with the "clear any existing progress bar" sequence. 129 | pending: Vec, 130 | 131 | /// True when there is new progress to display. 132 | /// When set, will notify dirty_cond. 133 | dirty: bool, 134 | dirty_cond: Arc, 135 | 136 | /// Counts of tasks in each state. TODO: pass this as function args? 137 | counts: StateCounts, 138 | /// Build tasks that are currently executing. 139 | /// Pushed to as tasks are started, so it's always in order of age. 140 | tasks: VecDeque, 141 | /// Whether to print command lines of started programs. 142 | verbose: bool, 143 | } 144 | 145 | impl FancyState { 146 | fn dirty(&mut self) { 147 | self.dirty = true; 148 | self.dirty_cond.notify_one(); 149 | } 150 | 151 | fn update(&mut self, counts: &StateCounts) { 152 | self.counts = counts.clone(); 153 | self.dirty(); 154 | } 155 | 156 | fn task_started(&mut self, id: BuildId, build: &Build) { 157 | if self.verbose { 158 | write!(&mut self.pending, "{}\n", build.cmdline.as_ref().unwrap()).ok(); 159 | } 160 | let message = build_message(build); 161 | self.tasks.push_back(Task { 162 | id, 163 | start: Instant::now(), 164 | message: message.to_string(), 165 | last_line: None, 166 | }); 167 | self.dirty(); 168 | } 169 | 170 | fn task_output(&mut self, id: BuildId, line: Vec) { 171 | let task = self.tasks.iter_mut().find(|t| t.id == id).unwrap(); 172 | task.last_line = Some(String::from_utf8_lossy(&line).into_owned()); 173 | self.dirty(); 174 | } 175 | 176 | fn task_finished(&mut self, id: BuildId, build: &Build, result: &TaskResult) { 177 | self.tasks 178 | .remove(self.tasks.iter().position(|t| t.id == id).unwrap()); 179 | 180 | // Show task name, status, and output. 181 | let buf = &mut self.pending; 182 | match result.termination { 183 | Termination::Success if result.output.is_empty() || build.hide_success => { 184 | // Common case: don't show anything. 185 | return; 186 | } 187 | Termination::Success => write!(buf, "{}\n", build_message(build)).ok(), 188 | Termination::Interrupted => write!(buf, "interrupted: {}\n", build_message(build)).ok(), 189 | Termination::Failure => write!(buf, "failed: {}\n", build_message(build)).ok(), 190 | }; 191 | buf.extend_from_slice(&result.output); 192 | if !result.output.ends_with(b"\n") { 193 | buf.push(b'\n'); 194 | } 195 | 196 | self.dirty(); 197 | } 198 | 199 | fn log(&mut self, msg: &str) { 200 | self.pending.extend_from_slice(msg.as_bytes()); 201 | self.pending.push(b'\n'); 202 | self.dirty(); 203 | } 204 | 205 | fn cleanup(&mut self) { 206 | self.done = true; 207 | self.dirty(); // let thread print final time 208 | } 209 | 210 | fn print_progress(&mut self) { 211 | let failed = self.counts.get(BuildState::Failed); 212 | let mut buf: &mut Vec = &mut self.pending; 213 | write!( 214 | &mut buf, 215 | "[{}] {}/{} done, ", 216 | progress_bar(&self.counts, 40), 217 | self.counts.get(BuildState::Done) + failed, 218 | self.counts.total() 219 | ) 220 | .ok(); 221 | if failed > 0 { 222 | write!(&mut buf, "{} failed, ", failed).ok(); 223 | } 224 | write!( 225 | &mut buf, 226 | "{}/{} running\n", 227 | self.tasks.len(), 228 | self.counts.get(BuildState::Queued) 229 | + self.counts.get(BuildState::Running) 230 | + self.counts.get(BuildState::Ready), 231 | ) 232 | .ok(); 233 | let mut lines = 1; 234 | 235 | let max_cols = terminal::get_cols().unwrap_or(80); 236 | let max_tasks = 8; 237 | let now = Instant::now(); 238 | for task in self.tasks.iter().take(max_tasks) { 239 | let delta = now.duration_since(task.start).as_secs() as usize; 240 | write!( 241 | &mut buf, 242 | "{}\n", 243 | task_message(&task.message, delta, max_cols) 244 | ) 245 | .ok(); 246 | lines += 1; 247 | if let Some(line) = &task.last_line { 248 | let max_len = max_cols - 2; 249 | write!(&mut buf, " {}\n", truncate(line, max_len)).ok(); 250 | lines += 1; 251 | } 252 | } 253 | 254 | if self.tasks.len() > max_tasks { 255 | let remaining = self.tasks.len() - max_tasks; 256 | write!(&mut buf, "...and {} more\n", remaining).ok(); 257 | lines += 1; 258 | } 259 | 260 | // Move cursor up to the first printed line, for overprinting. 261 | write!(&mut buf, "\x1b[{}A", lines).ok(); 262 | std::io::stdout().write_all(&buf).unwrap(); 263 | 264 | // Set up buf for next print. 265 | // If the user hit ctl-c, it may have printed something on the line. 266 | // So \r to go to first column first, then clear anything below. 267 | buf.clear(); 268 | buf.extend_from_slice(b"\r\x1b[J"); 269 | 270 | self.dirty = false; 271 | } 272 | } 273 | 274 | /// Format a task's status message to optionally include how long it has been running 275 | /// and also to fit within a maximum number of terminal columns. 276 | fn task_message(message: &str, seconds: usize, max_cols: usize) -> String { 277 | let time_note = if seconds > 2 { 278 | format!(" ({}s)", seconds) 279 | } else { 280 | "".into() 281 | }; 282 | let mut out = message.to_owned(); 283 | if out.len() + time_note.len() >= max_cols { 284 | out.truncate(max_cols - time_note.len() - 3); 285 | out.push_str("..."); 286 | } 287 | out.push_str(&time_note); 288 | out 289 | } 290 | 291 | fn truncate(s: &str, mut max: usize) -> &str { 292 | if max >= s.len() { 293 | return s; 294 | } 295 | while !s.is_char_boundary(max) { 296 | max -= 1; 297 | } 298 | &s[..max] 299 | } 300 | 301 | /// Render a StateCounts as an ASCII progress bar. 302 | fn progress_bar(counts: &StateCounts, bar_size: usize) -> String { 303 | let mut bar = String::with_capacity(bar_size); 304 | let mut sum: usize = 0; 305 | let total = counts.total(); 306 | if total == 0 { 307 | return " ".repeat(bar_size); 308 | } 309 | for (count, ch) in [ 310 | ( 311 | counts.get(BuildState::Done) + counts.get(BuildState::Failed), 312 | '=', 313 | ), 314 | ( 315 | counts.get(BuildState::Queued) 316 | + counts.get(BuildState::Running) 317 | + counts.get(BuildState::Ready), 318 | '-', 319 | ), 320 | (counts.get(BuildState::Want), ' '), 321 | ] { 322 | sum += count; 323 | let mut target_size = sum * bar_size / total; 324 | if count > 0 && target_size == bar.len() && target_size < bar_size { 325 | // Special case: for non-zero count, ensure we always get at least 326 | // one tick. 327 | target_size += 1; 328 | } 329 | while bar.len() < target_size { 330 | bar.push(ch); 331 | } 332 | } 333 | bar 334 | } 335 | 336 | #[cfg(test)] 337 | mod tests { 338 | use super::*; 339 | 340 | #[test] 341 | fn progress_bar_rendering() { 342 | let mut counts = StateCounts::default(); 343 | 344 | // Don't crash if we show progress before having any tasks. 345 | assert_eq!(progress_bar(&counts, 10), " "); 346 | 347 | counts.add(BuildState::Want, 100); 348 | assert_eq!(progress_bar(&counts, 10), " "); 349 | 350 | // Half want -> ready. 351 | counts.add(BuildState::Want, -50); 352 | counts.add(BuildState::Ready, 50); 353 | assert_eq!(progress_bar(&counts, 10), "----- "); 354 | 355 | // One ready -> done. 356 | counts.add(BuildState::Ready, -1); 357 | counts.add(BuildState::Done, 1); 358 | assert_eq!(progress_bar(&counts, 10), "=---- "); 359 | 360 | // All but one want -> ready. 361 | counts.add(BuildState::Want, -49); 362 | counts.add(BuildState::Ready, 49); 363 | assert_eq!(progress_bar(&counts, 10), "=-------- "); 364 | 365 | // All want -> ready. 366 | counts.add(BuildState::Want, -1); 367 | counts.add(BuildState::Ready, 1); 368 | assert_eq!(progress_bar(&counts, 10), "=---------"); 369 | } 370 | 371 | #[test] 372 | fn task_rendering() { 373 | assert_eq!(task_message("building foo.o", 0, 80), "building foo.o"); 374 | assert_eq!(task_message("building foo.o", 0, 10), "buildin..."); 375 | assert_eq!(task_message("building foo.o", 0, 5), "bu..."); 376 | } 377 | 378 | #[test] 379 | fn task_rendering_with_time() { 380 | assert_eq!(task_message("building foo.o", 5, 80), "building foo.o (5s)"); 381 | assert_eq!(task_message("building foo.o", 5, 10), "bu... (5s)"); 382 | } 383 | 384 | #[test] 385 | fn truncate_utf8() { 386 | let text = "utf8 progress bar: ━━━━━━━━━━━━"; 387 | for len in 10..text.len() { 388 | // test passes if this doesn't panic 389 | truncate(text, len); 390 | } 391 | } 392 | } 393 | -------------------------------------------------------------------------------- /src/graph.rs: -------------------------------------------------------------------------------- 1 | //! The build graph, a graph between files and commands. 2 | 3 | use rustc_hash::FxHashMap; 4 | 5 | use crate::{ 6 | densemap::{self, DenseMap}, 7 | hash::BuildHash, 8 | }; 9 | use std::collections::{hash_map::Entry, HashMap}; 10 | use std::path::{Path, PathBuf}; 11 | use std::time::SystemTime; 12 | 13 | /// Id for File nodes in the Graph. 14 | #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] 15 | pub struct FileId(u32); 16 | impl densemap::Index for FileId { 17 | fn index(&self) -> usize { 18 | self.0 as usize 19 | } 20 | } 21 | impl From for FileId { 22 | fn from(u: usize) -> FileId { 23 | FileId(u as u32) 24 | } 25 | } 26 | 27 | /// Id for Build nodes in the Graph. 28 | #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] 29 | pub struct BuildId(u32); 30 | impl densemap::Index for BuildId { 31 | fn index(&self) -> usize { 32 | self.0 as usize 33 | } 34 | } 35 | impl From for BuildId { 36 | fn from(u: usize) -> BuildId { 37 | BuildId(u as u32) 38 | } 39 | } 40 | 41 | /// A single file referenced as part of a build. 42 | #[derive(Debug)] 43 | pub struct File { 44 | /// Canonical path to the file. 45 | pub name: String, 46 | /// The Build that generates this file, if any. 47 | pub input: Option, 48 | /// The Builds that depend on this file as an input. 49 | pub dependents: Vec, 50 | } 51 | 52 | impl File { 53 | pub fn path(&self) -> &Path { 54 | Path::new(&self.name) 55 | } 56 | } 57 | 58 | /// A textual location within a build.ninja file, used in error messages. 59 | #[derive(Debug)] 60 | pub struct FileLoc { 61 | pub filename: std::rc::Rc, 62 | pub line: usize, 63 | } 64 | impl std::fmt::Display for FileLoc { 65 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { 66 | write!(f, "{}:{}", self.filename.display(), self.line) 67 | } 68 | } 69 | 70 | #[derive(Debug, Clone, Hash)] 71 | pub struct RspFile { 72 | pub path: std::path::PathBuf, 73 | pub content: String, 74 | } 75 | 76 | /// Input files to a Build. 77 | pub struct BuildIns { 78 | /// Internally we stuff explicit/implicit/order-only ins all into one Vec. 79 | /// This is mostly to simplify some of the iteration and is a little more 80 | /// memory efficient than three separate Vecs, but it is kept internal to 81 | /// Build and only exposed via methods on Build. 82 | pub ids: Vec, 83 | pub explicit: usize, 84 | pub implicit: usize, 85 | pub order_only: usize, 86 | // validation count implied by other counts. 87 | // pub validation: usize, 88 | } 89 | 90 | /// Output files from a Build. 91 | pub struct BuildOuts { 92 | /// Similar to ins, we keep both explicit and implicit outs in one Vec. 93 | pub ids: Vec, 94 | pub explicit: usize, 95 | } 96 | 97 | impl BuildOuts { 98 | /// CMake seems to generate build files with the same output mentioned 99 | /// multiple times in the outputs list. Given that Ninja accepts these, 100 | /// this function removes duplicates from the output list. 101 | pub fn remove_duplicates(&mut self) { 102 | let mut ids = Vec::new(); 103 | for (i, &id) in self.ids.iter().enumerate() { 104 | if self.ids[0..i].iter().any(|&prev| prev == id) { 105 | // Skip over duplicate. 106 | if i < self.explicit { 107 | self.explicit -= 1; 108 | } 109 | continue; 110 | } 111 | ids.push(id); 112 | } 113 | self.ids = ids; 114 | } 115 | } 116 | 117 | #[cfg(test)] 118 | mod tests { 119 | fn fileids(ids: Vec) -> Vec { 120 | ids.into_iter().map(FileId::from).collect() 121 | } 122 | 123 | use super::*; 124 | #[test] 125 | fn remove_dups_explicit() { 126 | let mut outs = BuildOuts { 127 | ids: fileids(vec![1, 1, 2]), 128 | explicit: 2, 129 | }; 130 | outs.remove_duplicates(); 131 | assert_eq!(outs.ids, fileids(vec![1, 2])); 132 | assert_eq!(outs.explicit, 1); 133 | } 134 | 135 | #[test] 136 | fn remove_dups_implicit() { 137 | let mut outs = BuildOuts { 138 | ids: fileids(vec![1, 2, 1]), 139 | explicit: 2, 140 | }; 141 | outs.remove_duplicates(); 142 | assert_eq!(outs.ids, fileids(vec![1, 2])); 143 | assert_eq!(outs.explicit, 2); 144 | } 145 | } 146 | 147 | /// A single build action, generating File outputs from File inputs with a command. 148 | pub struct Build { 149 | /// Source location this Build was declared. 150 | pub location: FileLoc, 151 | 152 | /// User-provided description of the build step. 153 | pub desc: Option, 154 | 155 | /// Command line to run. Absent for phony builds. 156 | pub cmdline: Option, 157 | 158 | /// Path to generated `.d` file, if any. 159 | pub depfile: Option, 160 | 161 | /// If true, extract "/showIncludes" lines from output. 162 | pub parse_showincludes: bool, 163 | 164 | // Struct that contains the path to the rsp file and its contents, if any. 165 | pub rspfile: Option, 166 | 167 | /// Pool to execute this build in, if any. 168 | pub pool: Option, 169 | 170 | pub ins: BuildIns, 171 | 172 | /// Additional inputs discovered from a previous build. 173 | discovered_ins: Vec, 174 | 175 | /// Output files. 176 | pub outs: BuildOuts, 177 | 178 | /// True if output of command should be hidden on successful completion. 179 | pub hide_success: bool, 180 | /// True if last line of output should not be shown in status. 181 | pub hide_progress: bool, 182 | } 183 | impl Build { 184 | pub fn new(loc: FileLoc, ins: BuildIns, outs: BuildOuts) -> Self { 185 | Build { 186 | location: loc, 187 | desc: None, 188 | cmdline: None, 189 | depfile: None, 190 | parse_showincludes: false, 191 | rspfile: None, 192 | pool: None, 193 | ins, 194 | discovered_ins: Vec::new(), 195 | outs, 196 | hide_success: false, 197 | hide_progress: false, 198 | } 199 | } 200 | 201 | /// Input paths that appear in `$in`. 202 | pub fn explicit_ins(&self) -> &[FileId] { 203 | &self.ins.ids[0..self.ins.explicit] 204 | } 205 | 206 | /// Input paths that, if changed, invalidate the output. 207 | /// Note this omits discovered_ins, which also invalidate the output. 208 | pub fn dirtying_ins(&self) -> &[FileId] { 209 | &self.ins.ids[0..(self.ins.explicit + self.ins.implicit)] 210 | } 211 | 212 | /// Inputs that are needed before building. 213 | /// Distinct from dirtying_ins in that it includes order-only dependencies. 214 | /// Note that we don't order on discovered_ins, because they're not allowed to 215 | /// affect build order. 216 | pub fn ordering_ins(&self) -> &[FileId] { 217 | &self.ins.ids[0..(self.ins.order_only + self.ins.explicit + self.ins.implicit)] 218 | } 219 | 220 | /// Inputs that are needed before validating information. 221 | /// Validation inputs will be built whenever this Build is built, but this Build will not 222 | /// wait for them to complete before running. The validation inputs can fail to build, which 223 | /// will cause the overall build to fail. 224 | pub fn validation_ins(&self) -> &[FileId] { 225 | &self.ins.ids[(self.ins.order_only + self.ins.explicit + self.ins.implicit)..] 226 | } 227 | 228 | pub fn set_discovered_ins(&mut self, deps: Vec) { 229 | self.discovered_ins = deps; 230 | } 231 | 232 | /// Input paths that were discovered after building, for use in the next build. 233 | pub fn discovered_ins(&self) -> &[FileId] { 234 | &self.discovered_ins 235 | } 236 | 237 | /// Output paths that appear in `$out`. 238 | pub fn explicit_outs(&self) -> &[FileId] { 239 | &self.outs.ids[0..self.outs.explicit] 240 | } 241 | 242 | /// Output paths that are updated when the build runs. 243 | pub fn outs(&self) -> &[FileId] { 244 | &self.outs.ids 245 | } 246 | } 247 | 248 | /// The build graph: owns Files/Builds and maps FileIds/BuildIds to them. 249 | #[derive(Default)] 250 | pub struct Graph { 251 | pub builds: DenseMap, 252 | pub files: GraphFiles, 253 | } 254 | 255 | /// Files identified by FileId, as well as mapping string filenames to them. 256 | /// Split from Graph for lifetime reasons. 257 | #[derive(Default)] 258 | pub struct GraphFiles { 259 | pub by_id: DenseMap, 260 | by_name: FxHashMap, 261 | } 262 | 263 | impl Graph { 264 | /// Look up a file by its FileId. 265 | pub fn file(&self, id: FileId) -> &File { 266 | &self.files.by_id[id] 267 | } 268 | 269 | /// Add a new Build, generating a BuildId for it. 270 | pub fn add_build(&mut self, mut build: Build) -> anyhow::Result<()> { 271 | let new_id = self.builds.next_id(); 272 | for &id in &build.ins.ids { 273 | self.files.by_id[id].dependents.push(new_id); 274 | } 275 | let mut fixup_dups = false; 276 | for &id in &build.outs.ids { 277 | let f = &mut self.files.by_id[id]; 278 | match f.input { 279 | Some(prev) if prev == new_id => { 280 | fixup_dups = true; 281 | println!( 282 | "n2: warn: {}: {:?} is repeated in output list", 283 | build.location, f.name, 284 | ); 285 | } 286 | Some(prev) => { 287 | anyhow::bail!( 288 | "{}: {:?} is already an output at {}", 289 | build.location, 290 | f.name, 291 | self.builds[prev].location 292 | ); 293 | } 294 | None => f.input = Some(new_id), 295 | } 296 | } 297 | if fixup_dups { 298 | build.outs.remove_duplicates(); 299 | } 300 | self.builds.push(build); 301 | Ok(()) 302 | } 303 | } 304 | 305 | impl GraphFiles { 306 | /// Look up a file by its name. Name must have been canonicalized already. 307 | pub fn lookup(&self, file: &str) -> Option { 308 | self.by_name.get(file).copied() 309 | } 310 | 311 | /// Look up a file by its name, adding it if not already present. 312 | /// Name must have been canonicalized already. Only accepting an owned 313 | /// string allows us to avoid a string copy and a hashmap lookup when we 314 | /// need to create a new id, but would also be possible to create a version 315 | /// of this function that accepts string references that is more optimized 316 | /// for the case where the entry already exists. But so far, all of our 317 | /// usages of this function have an owned string easily accessible anyways. 318 | pub fn id_from_canonical(&mut self, file: String) -> FileId { 319 | // TODO: so many string copies :< 320 | match self.by_name.entry(file) { 321 | Entry::Occupied(o) => *o.get(), 322 | Entry::Vacant(v) => { 323 | let id = self.by_id.push(File { 324 | name: v.key().clone(), 325 | input: None, 326 | dependents: Vec::new(), 327 | }); 328 | v.insert(id); 329 | id 330 | } 331 | } 332 | } 333 | 334 | pub fn all_ids(&self) -> impl Iterator { 335 | (0..self.by_id.next_id().0).map(|id| FileId(id)) 336 | } 337 | } 338 | 339 | /// MTime info gathered for a file. This also models "file is absent". 340 | /// It's not using an Option<> just because it makes the code using it easier 341 | /// to follow. 342 | #[derive(Copy, Clone, Debug, PartialEq)] 343 | pub enum MTime { 344 | Missing, 345 | Stamp(SystemTime), 346 | } 347 | 348 | /// stat() an on-disk path, producing its MTime. 349 | pub fn stat(path: &Path) -> std::io::Result { 350 | // TODO: On Windows, use FindFirstFileEx()/FindNextFile() to get timestamps per 351 | // directory, for better stat perf. 352 | Ok(match std::fs::metadata(path) { 353 | Ok(meta) => MTime::Stamp(meta.modified().unwrap()), 354 | Err(err) => { 355 | if err.kind() == std::io::ErrorKind::NotFound { 356 | MTime::Missing 357 | } else { 358 | return Err(err); 359 | } 360 | } 361 | }) 362 | } 363 | 364 | /// Gathered state of on-disk files. 365 | /// Due to discovered deps this map may grow after graph initialization. 366 | pub struct FileState(DenseMap>); 367 | 368 | impl FileState { 369 | pub fn new(graph: &Graph) -> Self { 370 | FileState(DenseMap::new_sized(graph.files.by_id.next_id(), None)) 371 | } 372 | 373 | pub fn get(&self, id: FileId) -> Option { 374 | self.0.lookup(id).copied().unwrap_or(None) 375 | } 376 | 377 | pub fn stat(&mut self, id: FileId, path: &Path) -> anyhow::Result { 378 | let mtime = stat(path).map_err(|err| anyhow::anyhow!("stat {:?}: {}", path, err))?; 379 | self.0.set_grow(id, Some(mtime), None); 380 | Ok(mtime) 381 | } 382 | } 383 | 384 | #[derive(Default)] 385 | pub struct Hashes(HashMap); 386 | 387 | impl Hashes { 388 | pub fn set(&mut self, id: BuildId, hash: BuildHash) { 389 | self.0.insert(id, hash); 390 | } 391 | 392 | pub fn get(&self, id: BuildId) -> Option { 393 | self.0.get(&id).copied() 394 | } 395 | } 396 | 397 | #[test] 398 | fn stat_mtime_resolution() { 399 | use std::time::Duration; 400 | 401 | let temp_dir = tempfile::tempdir().unwrap(); 402 | let filename = temp_dir.path().join("dummy"); 403 | 404 | // Write once and stat. 405 | std::fs::write(&filename, "foo").unwrap(); 406 | let mtime1 = match stat(&filename).unwrap() { 407 | MTime::Stamp(mtime) => mtime, 408 | _ => panic!("File not found: {}", filename.display()), 409 | }; 410 | 411 | // Sleep for a short interval. 412 | std::thread::sleep(std::time::Duration::from_millis(10)); 413 | 414 | // Write twice and stat. 415 | std::fs::write(&filename, "foo").unwrap(); 416 | let mtime2 = match stat(&filename).unwrap() { 417 | MTime::Stamp(mtime) => mtime, 418 | _ => panic!("File not found: {}", filename.display()), 419 | }; 420 | 421 | let diff = mtime2.duration_since(mtime1).unwrap(); 422 | assert!(diff > Duration::ZERO); 423 | assert!(diff < Duration::from_millis(100)); 424 | } 425 | --------------------------------------------------------------------------------