├── .buildbot.sh ├── .buildbot_dockerfile_debian ├── .github └── workflows │ └── sdci.yml ├── .gitignore ├── CHANGES.md ├── COPYRIGHT ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── deny.toml ├── examples ├── fm_options │ ├── lang_tests │ │ ├── echo.py │ │ ├── nondeterministic.py │ │ └── simple.py │ └── run_tests.rs └── rust_lang_tester │ ├── lang_tests │ ├── custom_cla.rs │ ├── custom_env.rs │ ├── echo.rs │ ├── echo_multiline.rs │ ├── exit_code.rs │ ├── ignore.rs │ ├── ignore2.rs │ ├── nested │ │ └── test.rs │ ├── no_main.rs │ ├── not_ignore.rs │ ├── sig_caught.rs │ ├── unknown_var.rs │ └── unused_var.rs │ └── run_tests.rs ├── lang_tests └── rerun │ ├── lang_tests │ ├── rerun_status.py │ ├── rerun_stderr.py │ └── rerun_stdout.py │ └── main.rs └── src ├── lib.rs ├── parser.rs └── tester.rs /.buildbot.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | set -e 4 | 5 | export CARGO_HOME="`pwd`/.cargo" 6 | export RUSTUP_HOME="`pwd`/.rustup" 7 | 8 | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rustup.sh 9 | sh rustup.sh --default-host x86_64-unknown-linux-gnu --default-toolchain stable -y --no-modify-path 10 | 11 | export PATH=`pwd`/.cargo/bin/:$PATH 12 | 13 | cargo fmt --all -- --check 14 | cargo test 15 | cargo run --example fm_options 16 | cargo run --example rust_lang_tester 17 | 18 | which cargo-deny | cargo install cargo-deny 19 | cargo-deny check license 20 | -------------------------------------------------------------------------------- /.buildbot_dockerfile_debian: -------------------------------------------------------------------------------- 1 | FROM debian:bullseye 2 | ARG CI_UID 3 | RUN useradd -m -u ${CI_UID} ci 4 | RUN apt-get update && \ 5 | apt-get -y install build-essential curl git procps python3 6 | WORKDIR /ci 7 | RUN chown ${CI_UID}:${CI_UID} . 8 | COPY --chown=${CI_UID}:${CI_UID} . . 9 | CMD sh -x .buildbot.sh 10 | -------------------------------------------------------------------------------- /.github/workflows/sdci.yml: -------------------------------------------------------------------------------- 1 | on: 2 | pull_request: 3 | merge_group: 4 | 5 | # This is required to silence emails about the workflow having no jobs. 6 | # We simply define a dummy job that does nothing much. 7 | jobs: 8 | dummy: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - run: /usr/bin/true 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | **/*.rs.bk 3 | Cargo.lock -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | # lang_tester 0.9.0 (2024-08-30) 2 | 3 | ## Breaking change 4 | 5 | * Update the [fm](https://crates.io/crates/fm) dependency, which has breaking 6 | changes, as well as [several new 7 | features](https://github.com/softdevteam/fm/blob/master/CHANGES.md#fm-040-2024-08-30). 8 | 9 | 10 | # lang_tester 0.8.2 (2024-07-18) 11 | 12 | * Sort test failures by name rather than the previous arbitrary order. 13 | 14 | 15 | # lang_tester 0.8.1 (2024-03-20) 16 | 17 | * Use newer version of fm that provides more comprehensive output by default. 18 | 19 | * Document and enforce that `ignore-if` cmds are run in `CARGO_MANIFEST_DIR`. 20 | 21 | 22 | # lang_tester 0.8.0 (2024-01-31) 23 | 24 | ## Breaking change 25 | 26 | * Remove `ignored` and add `ignore-if`. The latter runs an arbitrary shell 27 | command which, if it returns zero, causes the test to be ignored. This allows 28 | much more flexibility than the overly simplistic "always ignore this test" of 29 | `ignored`. Tests with `ignored: ` can be changed to `ignore-if: true` 30 | followed (or preceded) by a comment `# ` (assuming `comment_prefix` 31 | is set: see below). 32 | 33 | ## Non-breaking change 34 | 35 | * Allow comments in tests with a user-configurable prefix. By default no 36 | comment prefix is set. You can set one with `comment_prefix("...")`. For 37 | example `LangTester::new().comment_prefix("#")` causes lines in tests 38 | starting with `#` to be entirely ignored by lang_tester. 39 | 40 | 41 | # lang_tester 0.7.6 (2024-01-22) 42 | 43 | * `test_file_filter` is deprecated in favour of `test_path_filter`. The latter 44 | doesn't pre-filter non-files, making it more flexible. A simple way of moving 45 | from `test_file_filter` to `test_path_filter` is to change 46 | `test_file_filter(|p| ...)` to `test_path_fiter(|p| p.is_file() & ...)`. 47 | 48 | 49 | # lang_tester 0.7.5 (2023-11-03) 50 | 51 | * Make the library documentation (rather than the README) the source of 52 | documentation truth. 53 | 54 | 55 | # lang_tester 0.7.4 (2023-09-21) 56 | 57 | * Allow test filtering on the full test name (e.g. `lang_tests::a::b::c`) 58 | rather than just the leaf (e.g `c`). 59 | 60 | 61 | # lang_tester 0.7.3 (2023-04-06) 62 | 63 | * Add support for rerun-if-{status, stderr, stdout}. 64 | 65 | 66 | # lang_tester 0.7.2 (2021-12-07) 67 | 68 | * Fix poll() loop, so the full output of a subcommand is now read properly. 69 | 70 | 71 | # lang_tester 0.7.1 (2021-12-07) 72 | 73 | * Show a test as failing if `FMBuilder` throws an error. 74 | 75 | 76 | # lang_tester 0.7.0 (2021-06-18) 77 | 78 | ## Breaking changes 79 | 80 | * The `extra-args` key has been renamed to `exec-arg` to reflect the fact that 81 | each key is a single argument. 82 | 83 | ## Other changes 84 | 85 | * The `env-var` key has been added. This allows environment variables to be set 86 | on a per-test basis e.g.: 87 | 88 | ``` 89 | Compiler: 90 | env-var: DEBUG=1 91 | stdout: xyz 92 | ``` 93 | 94 | 95 | # lang_tester 0.6.2 (2021-05-24) 96 | 97 | * Fix file descriptor race for tests that contain stdin data: files were closed 98 | twice, which could lead to an active (reused) file descriptor being closed 99 | incorrectly. 100 | 101 | * Documentation fixes. 102 | 103 | 104 | # lang_tester 0.6.1 (2021-04-30) 105 | 106 | * Fix test file filtering. 107 | 108 | 109 | # lang_tester 0.6.0 (2021-04-30) 110 | 111 | * If a function passed by the user to the user (e.g. to `test_extract`) 112 | `panic`s, `lang_tester` now considers that a test failure and reports it to 113 | the user. Because this uses `catch_unwind` underneath, the functions passed 114 | to `lang_tester` must now be `RefUnwindSafe`. 115 | 116 | 117 | # lang_tester 0.5.0 (2021-01-27) 118 | 119 | * The `test_extract` function signature has changed from: 120 | ``` 121 | Fn(&str) -> Option + Send + Sync, 122 | ``` 123 | to: 124 | ``` 125 | Fn(&Path) -> String + Send + Sync, 126 | ``` 127 | 128 | In other words, users now have to both: 129 | 130 | 1. read the contents of a path themselves (but it doesn't necessarily have 131 | to be the path passed to the function!), 132 | 2. and return a `String` rather than an `Option`. 133 | 134 | In practise, most `test_extract` functions can be changed from (roughly): 135 | ``` 136 | test_extract(|s| { s.lines() ... }) 137 | ``` 138 | to: 139 | ``` 140 | test_extract(|p| { std::fs::read_to_string(p).lines() }) 141 | ``` 142 | 143 | 144 | # lang_tester 0.4.0 (2020-11-26) 145 | 146 | * Update to fm 0.2.0. This changes the interface exposed by the `fm_options` 147 | function. See the [`fm` 148 | changes](https://github.com/softdevteam/fm/blob/master/CHANGES.md) for more 149 | information. 150 | 151 | 152 | # lang_tester 0.3.13 (2020-11-09) 153 | 154 | * Silence some Clippy warnings and fix documentation inconsistencies. 155 | 156 | 157 | # lang_tester 0.3.12 (2020-07-13) 158 | 159 | * Failed stderr/stdout tests now use fm to show the offending line and up to 3 160 | lines of surrounding context. This makes it much easier to understand why a 161 | stderr/test failed. 162 | 163 | 164 | # lang_tester 0.3.11 (2020-07-09) 165 | 166 | * Remove the built-in fuzzy matcher and use the [`fm` 167 | library](https://crates.io/crates/fm) instead. This should be entirely 168 | backwards compatible in its default state. Users who want non-default `fm` 169 | options can use the new `fm_options` function in `LangTester`. 170 | 171 | * Add a `stdin` key to allow users to specify stdin input which should be 172 | passed to a sub-command. 173 | 174 | * Lines are no longer stripped of their leading or trailing whitespace allowing 175 | tests to be whitespace sensitive if required. Since matching in `fm` defaults 176 | to ignoring leading and trailing whitespace, the previous behaviour is 177 | preserved unless users explicitly tell `fm` to match whitespace. 178 | 179 | 180 | # lang_tester 0.3.10 (2020-06-04) 181 | 182 | * Print out the name of tests inside nested directories rather than flattening 183 | them all such that they appear to be the top-level directory. If you have 184 | tests `a/x` and `b/x` these are pretty printed as `a::x` and `b::x` 185 | respectively (whereas before they were pretty printed as simply `x`, meaning 186 | that you could not tell which had succeeded / failed). 187 | 188 | 189 | # lang_tester 0.3.9 (2020-05-18) 190 | 191 | * Add `test_threads` function which allows you to specify the number of test 192 | threads programatically. 193 | 194 | * Move from the deprecated `tempdir` to the maintained `tempfile` crate. 195 | 196 | 197 | # lang_tester 0.3.8 (2019-12-24) 198 | 199 | * Fix bug on OS X where input from sub-processes blocked forever. 200 | 201 | 202 | # lang_tester 0.3.7 (2019-11-26) 203 | 204 | * Add support for ignorable tests. A test command `ignore:` is interpreted as 205 | causing that entire test file to be ignored. As with `cargo test`, such tests 206 | can be run with the `--ignored` switch. 207 | 208 | * Fix a bug whereby the number of ignored tests was incorrectly reported. 209 | 210 | 211 | # lang_tester 0.3.6 (2019-11-21) 212 | 213 | * License as dual Apache-2.0/MIT (instead of a more complex, and little 214 | understood, triple license of Apache-2.0/MIT/UPL-1.0). 215 | 216 | 217 | # lang_tester 0.3.5 (2019-11-15) 218 | 219 | * Add support for programs which terminated due to a signal. Users can now 220 | specify `status: signal` to indicate that a test should exit due to a signal: 221 | on platforms which do not support this (e.g. Windows), such tests are 222 | ignored. Similarly, if a program was terminated due to a signal then, on 223 | Unix, the user is informed of that after test failure. 224 | 225 | 226 | # lang_tester 0.3.4 (2019-10-30) 227 | 228 | * Add support for `--nocapture` to better emulate `cargo test`. As with `cargo 229 | test`, if you're running more than one test then `--nocapture` is generally 230 | best paired with `--test-threads=1` to avoid confusing, multiplexed output to 231 | the terminal. 232 | 233 | * Be clearer that tests can have defaults: notably commands default to `status: 234 | success` unless overridden. 235 | 236 | 237 | # lang_tester 0.3.3 (2019-10-24) 238 | 239 | * Individual tests can now add extra arguments to an invoked command with the 240 | `extra-args` field. 241 | 242 | * Ensure that, if a command in a chain fails, the whole chain of commands 243 | fails. This means that if, for example, compilation of command C fails, we do 244 | not try and run C anyway (which can end up doing confusing things like 245 | running an old version of C). 246 | 247 | 248 | # lang_tester 0.3.2 (2019-07-31) 249 | 250 | * Fixed bug where potentially multi-line keys with empty values were not always 251 | parsed correctly. 252 | 253 | 254 | # lang_tester 0.3.1 (2019-06-04) 255 | 256 | * Add support for running a defined number of parallel processes, using the 257 | `cargo test`-ish option `--test-threads=n`. For example, to run tests 258 | sequentially, specify `--test-threads=1`. 259 | 260 | * Warn users if a given test has run unexpectedly long (currently every 261 | multiple of 60 seconds). This is often a sign that a test has entered an 262 | infinite loop. 263 | 264 | * Use better terminology in the documentation. Previously "test" was used to 265 | mean a number of subtly different things which was rather confusing. Now 266 | test files contain test data. Test data contains test commands. Test commands 267 | contain sub-tests. 268 | 269 | * Stop testing a given test file on the first failed sub-test. Previously only 270 | a test command which exited unsuccessfully caused a test file to be 271 | considered as failed, causing the source of errors to sometimes be missed. 272 | 273 | 274 | # lang_tester 0.3.0 (2019-05-29) 275 | 276 | ## Breaking changes 277 | 278 | * The `test_extract` and `test_cmds` functions must now satisfy the `Sync` 279 | trait. This is a breaking change, albeit one that nearly all such functions 280 | already satisfied. 281 | 282 | ## Major changes 283 | 284 | * When a test fails, report to the user both the parts of the test that failed 285 | and the parts that weren't specified. For example, if a test merely checks 286 | that a command runs successfully, we now report stdout and stderr output to 287 | the user, so that they can better understand what happened. 288 | 289 | ## Minor changes 290 | 291 | * Fatal errors (e.g. an inability to run a command, or an error in the way a 292 | user has specified a test, such as a syntax error) now cause the process to 293 | exit (whereas before they merely caused the thread erroring to panic, leading 294 | to errors being lost in the noise). 295 | 296 | 297 | # lang_tester 0.2.0 (2019-05-21) 298 | 299 | * Accept cargo-ish command-line parameters. In particular, this lets users run 300 | a subset of tests e.g. " ab cd" only runs tests with "ab" or "cd" 301 | in their name. If you don't want `lang_tester` to look at your command-line 302 | arguments, set `use_cmdline_args(false)` (the default is `true`). 303 | 304 | * Run tests in parallel (one per CPU core). Depending on the size of your 305 | machine and the size of your test suite, this can be a significant 306 | performance improvement. 307 | 308 | * The `status` field can now take integer exit codes. i.e. if you specify 309 | `status: 7` then the exit code of the binary being run will be checked to see 310 | if it is 7. 311 | 312 | 313 | # lang_tester 0.1.0 (2019-05-16) 314 | 315 | First stable release. 316 | -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | Except as otherwise noted (below and/or in individual files), this project is 2 | licensed under the Apache License, Version 2.0 3 | or the MIT license 4 | , at your option. 5 | 6 | Copyright is retained by contributors and/or the organisations they 7 | represent(ed) -- this project does not require copyright assignment. Please see 8 | version control history for a full list of contributors. Note that some files 9 | may include explicit copyright and/or licensing notices. 10 | 11 | The following contributors wish to explicitly make it known that the copyright 12 | of their contributions is retained by an organisation: 13 | 14 | Edd Barrett : copyright retained by 15 | King's College London 16 | Jacob Hughes : copyright retained by 17 | King's College London 18 | Laurence Tratt : copyright retained by 19 | King's College London 20 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "lang_tester" 3 | description = "Concise language testing framework for compilers and VMs" 4 | repository = "https://github.com/softdevteam/lang_tester/" 5 | version = "0.9.0" 6 | authors = ["Laurence Tratt "] 7 | readme = "README.md" 8 | license = "Apache-2.0/MIT" 9 | categories = ["development-tools"] 10 | edition = "2018" 11 | 12 | [[example]] 13 | name = "rust_lang_tester" 14 | path = "examples/rust_lang_tester/run_tests.rs" 15 | 16 | [[example]] 17 | name = "fm_options" 18 | path = "examples/fm_options/run_tests.rs" 19 | 20 | [[test]] 21 | name = "lang_tests" 22 | path = "lang_tests/rerun/main.rs" 23 | harness = false 24 | 25 | [dependencies] 26 | fm = "0.4.0" 27 | getopts = "0.2" 28 | libc = "0.2" 29 | num_cpus = "1.15" 30 | termcolor = "1" 31 | threadpool = "1.7" 32 | wait-timeout = "0.2" 33 | walkdir = "2" 34 | 35 | [dev-dependencies] 36 | regex = "1.4" 37 | tempfile = "3" 38 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use 2 | this file except in compliance with the License. You may obtain a copy of the 3 | License at 4 | 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed 8 | under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 9 | CONDITIONS OF ANY KIND, either express or implied. See the License for the 10 | specific language governing permissions and limitations under the License. 11 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Permission is hereby granted, free of charge, to any person obtaining a copy of 2 | this software and associated documentation files (the "Software"), to deal in 3 | the Software without restriction, including without limitation the rights to 4 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 5 | of the Software, and to permit persons to whom the Software is furnished to do 6 | so, subject to the following conditions: 7 | 8 | The above copyright notice and this permission notice shall be included in all 9 | copies or substantial portions of the Software. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 13 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 14 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 15 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 16 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 17 | SOFTWARE. 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # lang_tester 2 | 3 | This crate provides a simple language testing framework designed to help when 4 | you are testing things like compilers and virtual machines. It allows users to 5 | express simple tests for process success/failure and for stderr/stdout, including 6 | embedding those tests directly in the source file. It is loosely based on the 7 | [`compiletest_rs`](https://crates.io/crates/compiletest_rs) crate, but is much 8 | simpler (and hence sometimes less powerful), and designed to be used for 9 | testing non-Rust languages too. 10 | 11 | For example, a Rust language tester, loosely in the spirit of 12 | [`compiletest_rs`](https://crates.io/crates/compiletest_rs), looks as follows: 13 | 14 | ```rust 15 | use std::{env, fs::read_to_string, path::PathBuf, process::Command}; 16 | 17 | use lang_tester::LangTester; 18 | use tempfile::TempDir; 19 | 20 | fn main() { 21 | // We use rustc to compile files into a binary: we store those binary files 22 | // into `tempdir`. This may not be necessary for other languages. 23 | let tempdir = TempDir::new().unwrap(); 24 | LangTester::new() 25 | .test_dir("examples/rust_lang_tester/lang_tests") 26 | // Only use files named `*.rs` as test files. 27 | .test_path_filter(|p| p.extension().and_then(|x| x.to_str()) == Some("rs")) 28 | // Treat lines beginning with "#" inside a test as comments. 29 | .comment_prefix("#") 30 | // Extract the first sequence of commented line(s) as the tests. 31 | .test_extract(|p| { 32 | read_to_string(p) 33 | .unwrap() 34 | .lines() 35 | // Skip non-commented lines at the start of the file. 36 | .skip_while(|l| !l.starts_with("//")) 37 | // Extract consecutive commented lines. 38 | .take_while(|l| l.starts_with("//")) 39 | .map(|l| &l[COMMENT_PREFIX.len()..]) 40 | .collect::>() 41 | .join("\n") 42 | }) 43 | // We have two test commands: 44 | // * `Compiler`: runs rustc. 45 | // * `Run-time`: if rustc does not error, and the `Compiler` tests 46 | // succeed, then the output binary is run. 47 | .test_cmds(move |p| { 48 | // Test command 1: Compile `x.rs` into `tempdir/x`. 49 | let mut exe = PathBuf::new(); 50 | exe.push(&tempdir); 51 | exe.push(p.file_stem().unwrap()); 52 | let mut compiler = Command::new("rustc"); 53 | compiler.args(&["-o", exe.to_str().unwrap(), p.to_str().unwrap()]); 54 | // Test command 2: run `tempdir/x`. 55 | let runtime = Command::new(exe); 56 | vec![("Compiler", compiler), ("Run-time", runtime)] 57 | }) 58 | .run(); 59 | } 60 | ``` 61 | 62 | This defines a lang tester that uses all `*.rs` files in a given directory as 63 | test files, running two test commands against them: `Compiler` (i.e. `rustc`); 64 | and `Run-time` (the compiled binary). 65 | 66 | Users can then write test files such as the following: 67 | 68 | ```rust 69 | // Compiler: 70 | // stderr: 71 | // warning: unused variable: `x` 72 | // ...unused_var.rs:12:9 73 | // ... 74 | // 75 | // Run-time: 76 | // stdout: Hello world 77 | fn main() { 78 | let x = 0; 79 | println!("Hello world"); 80 | } 81 | ``` 82 | 83 | The above file contains 4 meaningful tests, two specified by the user and 84 | two implied by defaults: the `Compiler` should succeed (e.g. return a `0` exit 85 | code when run on Unix), and its `stderr` output should warn about an unused 86 | variable on line 12; and the resulting binary should succeed produce `Hello 87 | world` on `stdout`. 88 | 89 | 90 | ## Integration with Cargo. 91 | 92 | Tests created with lang_tester can be used as part of an existing test suite and 93 | can be run with the `cargo test` command. For example, if the Rust source file 94 | that runs your lang tests is `lang_tests/run.rs` then add the following to your 95 | Cargo.toml: 96 | 97 | ``` 98 | [[test]] 99 | name = "lang_tests" 100 | path = "lang_tests/run.rs" 101 | harness = false 102 | ``` 103 | -------------------------------------------------------------------------------- /deny.toml: -------------------------------------------------------------------------------- 1 | [licenses] 2 | confidence-threshold = 1.0 3 | allow = [ 4 | "Apache-2.0", 5 | "MIT", 6 | "Unlicense" 7 | ] 8 | -------------------------------------------------------------------------------- /examples/fm_options/lang_tests/echo.py: -------------------------------------------------------------------------------- 1 | # VM: 2 | # status: success 3 | # stdin: 4 | # a 5 | # b 6 | # a 7 | # stdout: 8 | # $1 9 | # b 10 | # $1 11 | 12 | import sys 13 | 14 | for l in sys.stdin: 15 | sys.stdout.write(l) 16 | -------------------------------------------------------------------------------- /examples/fm_options/lang_tests/nondeterministic.py: -------------------------------------------------------------------------------- 1 | # VM: 2 | # status: success 3 | # stdout: 4 | # $1 5 | # b 6 | # $1 7 | 8 | import random 9 | 10 | ALPHABET = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", 11 | "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"] 12 | 13 | x = random.choice(ALPHABET) 14 | print(x) 15 | print("b") 16 | print(x) 17 | -------------------------------------------------------------------------------- /examples/fm_options/lang_tests/simple.py: -------------------------------------------------------------------------------- 1 | # VM: 2 | # status: success 3 | # stdout: 4 | # $1 5 | # b 6 | # $1 7 | 8 | print("a") 9 | print("b") 10 | print("a") 11 | -------------------------------------------------------------------------------- /examples/fm_options/run_tests.rs: -------------------------------------------------------------------------------- 1 | //! This is an example lang_tester that shows how to set options using the `fm` fuzzy matcher 2 | //! library, using Python files as an example. 3 | 4 | use std::{fs::read_to_string, process::Command}; 5 | 6 | use lang_tester::LangTester; 7 | use regex::Regex; 8 | 9 | fn main() { 10 | LangTester::new() 11 | .test_dir("examples/fm_options/lang_tests") 12 | .test_path_filter(|p| p.extension().and_then(|x| x.to_str()) == Some("py")) 13 | .test_extract(|p| { 14 | read_to_string(p) 15 | .unwrap() 16 | .lines() 17 | // Skip non-commented lines at the start of the file. 18 | .skip_while(|l| !l.starts_with("#")) 19 | // Extract consecutive commented lines. 20 | .take_while(|l| l.starts_with("#")) 21 | .map(|l| &l[2..]) 22 | .collect::>() 23 | .join("\n") 24 | }) 25 | .fm_options(|_, _, fmb| { 26 | let ptn_re = Regex::new(r"\$.+?\b").unwrap(); 27 | let text_re = Regex::new(r".+?\b").unwrap(); 28 | fmb.name_matcher(ptn_re, text_re).trim_whitespace(false) 29 | }) 30 | .test_cmds(move |p| { 31 | let mut vm = Command::new("python3"); 32 | vm.args(&[p.to_str().unwrap()]); 33 | vec![("VM", vm)] 34 | }) 35 | .run(); 36 | } 37 | -------------------------------------------------------------------------------- /examples/rust_lang_tester/lang_tests/custom_cla.rs: -------------------------------------------------------------------------------- 1 | // Run-time: 2 | // exec-arg: 1 3 | // exec-arg: 2 3 4 | 5 | use std::env; 6 | 7 | fn main() { 8 | println!("{:?}", env::args()); 9 | let arg1 = env::args() 10 | .nth(1) 11 | .expect("no arg 1 passed") 12 | .parse::() 13 | .expect("arg 1 should be numeric"); 14 | 15 | let arg2 = env::args() 16 | .nth(2) 17 | .unwrap(); 18 | assert_eq!(arg2, "2 3"); 19 | } 20 | -------------------------------------------------------------------------------- /examples/rust_lang_tester/lang_tests/custom_env.rs: -------------------------------------------------------------------------------- 1 | // Run-time: 2 | // env-var: XYZ=123 3 | // env-var: XYZ=456 4 | // env-var: ABC=789 012 5 | 6 | use std::env; 7 | 8 | fn main() { 9 | assert_eq!(env::var("XYZ").unwrap(), "456".to_owned()); 10 | assert_eq!(env::var("ABC").unwrap(), "789 012"); 11 | } 12 | -------------------------------------------------------------------------------- /examples/rust_lang_tester/lang_tests/echo.rs: -------------------------------------------------------------------------------- 1 | // Run-time: 2 | // status: success 3 | // stdin: abc 4 | // stdout: Hello abc 5 | 6 | use std::io::{Read, stdin}; 7 | 8 | fn main() { 9 | let mut buf = String::new(); 10 | stdin().read_to_string(&mut buf).unwrap(); 11 | println!("Hello {}", buf); 12 | } 13 | -------------------------------------------------------------------------------- /examples/rust_lang_tester/lang_tests/echo_multiline.rs: -------------------------------------------------------------------------------- 1 | // Run-time: 2 | // status: success 3 | // stdin: 4 | // a 5 | // b 6 | // c 7 | // stdout: 8 | // Hello a 9 | // b 10 | // c 11 | 12 | use std::io::{Read, stdin}; 13 | 14 | fn main() { 15 | let mut buf = String::new(); 16 | stdin().read_to_string(&mut buf).unwrap(); 17 | println!("Hello {}", buf); 18 | } 19 | -------------------------------------------------------------------------------- /examples/rust_lang_tester/lang_tests/exit_code.rs: -------------------------------------------------------------------------------- 1 | // Run-time: 2 | // status: 7 3 | 4 | use std::process; 5 | 6 | fn main() { 7 | process::exit(7); 8 | } 9 | -------------------------------------------------------------------------------- /examples/rust_lang_tester/lang_tests/ignore.rs: -------------------------------------------------------------------------------- 1 | // # Always ignore this test 2 | // ignore-if: echo 123 | grep 2 3 | // Compiler: 4 | // status: success 5 | 6 | fn main() { 7 | panic!("Shouldn't happen."); 8 | } 9 | -------------------------------------------------------------------------------- /examples/rust_lang_tester/lang_tests/ignore2.rs: -------------------------------------------------------------------------------- 1 | // # Always ignore this test 2 | // ignore-if: cat examples/rust_lang_tester/lang_tests/ignore.rs 3 | // Compiler: 4 | // status: success 5 | 6 | fn main() { 7 | panic!("Shouldn't happen."); 8 | } 9 | -------------------------------------------------------------------------------- /examples/rust_lang_tester/lang_tests/nested/test.rs: -------------------------------------------------------------------------------- 1 | // Compiler: 2 | // status: success 3 | // 4 | // Run-time: 5 | // status: success 6 | // stdout: nested test 7 | 8 | fn main() { 9 | println!("nested test"); 10 | } 11 | -------------------------------------------------------------------------------- /examples/rust_lang_tester/lang_tests/no_main.rs: -------------------------------------------------------------------------------- 1 | // Compiler: 2 | // status: error 3 | // stderr: 4 | // error[E0601]: `main` function not found in crate `no_main` 5 | // ... 6 | // error: aborting due to 1 previous error 7 | // 8 | // For more information about this error, try `rustc --explain E0601`. 9 | -------------------------------------------------------------------------------- /examples/rust_lang_tester/lang_tests/not_ignore.rs: -------------------------------------------------------------------------------- 1 | // # Never ignore this test. 2 | // ignore-if: echo 123 | grep 4 3 | // Run-time: 4 | // stdout: 5 | // # an ignored comment 6 | // check 7 | 8 | fn main() { 9 | println!("check"); 10 | } 11 | -------------------------------------------------------------------------------- /examples/rust_lang_tester/lang_tests/sig_caught.rs: -------------------------------------------------------------------------------- 1 | // Run-time: 2 | // status: signal 3 | 4 | use std::process; 5 | 6 | fn main() { 7 | unsafe { 8 | let ptr = std::ptr::null::(); 9 | *ptr + 1; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /examples/rust_lang_tester/lang_tests/unknown_var.rs: -------------------------------------------------------------------------------- 1 | // Compiler: 2 | // status: error 3 | // stderr: 4 | // error[E0425]: cannot find value `x` in this scope 5 | // ...unknown_var.rs:9:20 6 | // ... 7 | 8 | fn main() { 9 | println!("{}", x); 10 | } 11 | -------------------------------------------------------------------------------- /examples/rust_lang_tester/lang_tests/unused_var.rs: -------------------------------------------------------------------------------- 1 | // Compiler: 2 | // stderr: 3 | // warning: unused variable: `x` 4 | // ...unused_var.rs:11:9 5 | // ... 6 | // 7 | // Run-time: 8 | // stdout: Hello world 9 | 10 | fn main() { 11 | let x = 0; 12 | println!("Hello world"); 13 | } 14 | -------------------------------------------------------------------------------- /examples/rust_lang_tester/run_tests.rs: -------------------------------------------------------------------------------- 1 | //! This is an example lang_tester for Rust: it is a simplified version of the test framework that 2 | //! rustc uses. In essence, the first sequence of commented line(s) (note that other lines e.g. 3 | //! `#![feature(...)]` lines and the like are skipped) in the file describe the test. 4 | //! 5 | //! See the test files in `lang_tests/` for example. 6 | 7 | use std::{fs::read_to_string, path::PathBuf, process::Command}; 8 | 9 | use lang_tester::LangTester; 10 | use tempfile::TempDir; 11 | 12 | static COMMENT_PREFIX: &str = "//"; 13 | 14 | fn main() { 15 | // We use rustc to compile files into a binary: we store those binary files into `tempdir`. 16 | // This may not be necessary for other languages. 17 | let tempdir = TempDir::new().unwrap(); 18 | LangTester::new() 19 | .test_dir("examples/rust_lang_tester/lang_tests") 20 | // Only use files named `*.rs` as test files. 21 | .test_path_filter(|p| p.extension().and_then(|x| x.to_str()) == Some("rs")) 22 | // Treat lines beginning with "#" as comments. 23 | .comment_prefix("#") 24 | // Extract the first sequence of commented line(s) as the tests. 25 | .test_extract(|p| { 26 | read_to_string(p) 27 | .unwrap() 28 | .lines() 29 | // Skip non-commented lines at the start of the file. 30 | .skip_while(|l| !l.starts_with(COMMENT_PREFIX)) 31 | // Extract consecutive commented lines. 32 | .take_while(|l| l.starts_with(COMMENT_PREFIX)) 33 | .map(|l| &l[COMMENT_PREFIX.len()..]) 34 | .collect::>() 35 | .join("\n") 36 | }) 37 | // We have two test commands: 38 | // * `Compiler`: runs rustc. 39 | // * `Run-time`: if rustc does not error, and the `Compiler` tests succeed, then the 40 | // output binary is run. 41 | .test_cmds(move |p| { 42 | // Test command 1: Compile `x.rs` into `tempdir/x`. 43 | let mut exe = PathBuf::new(); 44 | exe.push(&tempdir); 45 | exe.push(p.file_stem().unwrap()); 46 | let mut compiler = Command::new("rustc"); 47 | compiler.args(&["-o", exe.to_str().unwrap(), p.to_str().unwrap()]); 48 | // Test command 2: run `tempdir/x`. 49 | let runtime = Command::new(exe); 50 | vec![("Compiler", compiler), ("Run-time", runtime)] 51 | }) 52 | .run(); 53 | } 54 | -------------------------------------------------------------------------------- /lang_tests/rerun/lang_tests/rerun_status.py: -------------------------------------------------------------------------------- 1 | # VM: 2 | # status: success 3 | # rerun-if-status: 42 4 | 5 | import os, sys 6 | 7 | cookie = os.path.join(os.environ["CARGO_TARGET_TMPDIR"], "rerun_status_cookie") 8 | i = 0 9 | if os.path.exists(cookie): 10 | i = int(open(cookie, "r").read().strip()) + 1 11 | if i == 5: 12 | sys.exit(0) 13 | 14 | open(cookie, "w").write(str(i)) 15 | sys.exit(42) 16 | -------------------------------------------------------------------------------- /lang_tests/rerun/lang_tests/rerun_stderr.py: -------------------------------------------------------------------------------- 1 | # VM: 2 | # status: success 3 | # stderr: a 4 | # rerun-if-stderr: b 5 | 6 | import os, sys 7 | 8 | cookie = os.path.join(os.environ["CARGO_TARGET_TMPDIR"], "rerun_stderr_cookie") 9 | i = 0 10 | if os.path.exists(cookie): 11 | i = int(open(cookie, "r").read().strip()) + 1 12 | if i == 5: 13 | sys.stderr.write("a") 14 | sys.exit(0) 15 | 16 | open(cookie, "w").write(str(i)) 17 | sys.stderr.write("b") 18 | -------------------------------------------------------------------------------- /lang_tests/rerun/lang_tests/rerun_stdout.py: -------------------------------------------------------------------------------- 1 | # VM: 2 | # status: success 3 | # stdout: a 4 | # rerun-if-stdout: b 5 | 6 | import os, sys 7 | 8 | cookie = os.path.join(os.environ["CARGO_TARGET_TMPDIR"], "rerun_stdout_cookie") 9 | i = 0 10 | if os.path.exists(cookie): 11 | i = int(open(cookie, "r").read().strip()) + 1 12 | if i == 5: 13 | sys.stdout.write("a") 14 | sys.exit(0) 15 | 16 | open(cookie, "w").write(str(i)) 17 | sys.stdout.write("b") 18 | -------------------------------------------------------------------------------- /lang_tests/rerun/main.rs: -------------------------------------------------------------------------------- 1 | //! This is an example lang_tester that shows how to use the "rerun_if" commands. 2 | 3 | use std::{ 4 | env, 5 | fs::{read_to_string, remove_file}, 6 | path::PathBuf, 7 | process::Command, 8 | }; 9 | 10 | /// In order that we can meaningfully have the same test behave differently from run to run, we 11 | /// have to leave cookies behind in target/tmp, which we must clear before starting a test run. 12 | /// This is horrible, but without doing something stateful, we can't test anything. 13 | const COOKIES: &[&str] = &[ 14 | "rerun_status_cookie", 15 | "rerun_stderr_cookie", 16 | "rerun_stdout_cookie", 17 | ]; 18 | 19 | use lang_tester::LangTester; 20 | use regex::Regex; 21 | 22 | fn main() { 23 | env::set_var("CARGO_TARGET_TMPDIR", env!("CARGO_TARGET_TMPDIR")); 24 | 25 | for cookie in COOKIES { 26 | let mut p = PathBuf::new(); 27 | p.push(env::var("CARGO_TARGET_TMPDIR").unwrap()); 28 | p.push(cookie); 29 | if p.exists() { 30 | remove_file(p).unwrap(); 31 | } 32 | } 33 | 34 | LangTester::new() 35 | .rerun_at_most(5) 36 | .test_dir("lang_tests/rerun/") 37 | .test_path_filter(|p| p.extension().and_then(|x| x.to_str()) == Some("py")) 38 | .test_extract(|p| { 39 | read_to_string(p) 40 | .unwrap() 41 | .lines() 42 | // Skip non-commented lines at the start of the file. 43 | .skip_while(|l| !l.starts_with("#")) 44 | // Extract consecutive commented lines. 45 | .take_while(|l| l.starts_with("#")) 46 | .map(|l| &l[2..]) 47 | .collect::>() 48 | .join("\n") 49 | }) 50 | .fm_options(|_, _, fmb| { 51 | let ptn_re = Regex::new(r"\$.+?\b").unwrap(); 52 | let text_re = Regex::new(r".+?\b").unwrap(); 53 | fmb.name_matcher(ptn_re, text_re).trim_whitespace(false) 54 | }) 55 | .test_cmds(move |p| { 56 | let mut vm = Command::new("python3"); 57 | vm.args(&[p.to_str().unwrap()]); 58 | vec![("VM", vm)] 59 | }) 60 | .run(); 61 | } 62 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! This crate provides a simple language testing framework designed to help when you are testing 2 | //! things like compilers and virtual machines. It allows users to express simple tests for process 3 | //! success/failure and for stderr/stdout, including embedding those tests directly in the source 4 | //! file. It is loosely based on the [`compiletest_rs`](https://crates.io/crates/compiletest_rs) 5 | //! crate, but is much simpler (and hence sometimes less powerful), and designed to be used for 6 | //! testing non-Rust languages too. 7 | //! 8 | //! For example, a Rust language tester, loosely in the spirit of 9 | //! [`compiletest_rs`](https://crates.io/crates/compiletest_rs), looks as follows: 10 | //! 11 | //! ```rust,ignore 12 | //! use std::{env, fs::read_to_string, path::PathBuf, process::Command}; 13 | //! 14 | //! use lang_tester::LangTester; 15 | //! use tempfile::TempDir; 16 | //! 17 | //! fn main() { 18 | //! // We use rustc to compile files into a binary: we store those binary files 19 | //! // into `tempdir`. This may not be necessary for other languages. 20 | //! let tempdir = TempDir::new().unwrap(); 21 | //! LangTester::new() 22 | //! .test_dir("examples/rust_lang_tester/lang_tests") 23 | //! // Only use files named `*.rs` as test files. 24 | //! .test_path_filter(|p| p.extension().and_then(|x| x.to_str()) == Some("rs")) 25 | //! // Treat lines beginning with "#" inside a test as comments. 26 | //! .comment_prefix("#") 27 | //! // Extract the first sequence of commented line(s) as the tests. 28 | //! .test_extract(|p| { 29 | //! read_to_string(p) 30 | //! .unwrap() 31 | //! .lines() 32 | //! // Skip non-commented lines at the start of the file. 33 | //! .skip_while(|l| !l.starts_with("//")) 34 | //! // Extract consecutive commented lines. 35 | //! .take_while(|l| l.starts_with("//")) 36 | //! .map(|l| &l[COMMENT_PREFIX.len()..]) 37 | //! .collect::>() 38 | //! .join("\n") 39 | //! }) 40 | //! // We have two test commands: 41 | //! // * `Compiler`: runs rustc. 42 | //! // * `Run-time`: if rustc does not error, and the `Compiler` tests 43 | //! // succeed, then the output binary is run. 44 | //! .test_cmds(move |p| { 45 | //! // Test command 1: Compile `x.rs` into `tempdir/x`. 46 | //! let mut exe = PathBuf::new(); 47 | //! exe.push(&tempdir); 48 | //! exe.push(p.file_stem().unwrap()); 49 | //! let mut compiler = Command::new("rustc"); 50 | //! compiler.args(&["-o", exe.to_str().unwrap(), p.to_str().unwrap()]); 51 | //! // Test command 2: run `tempdir/x`. 52 | //! let runtime = Command::new(exe); 53 | //! vec![("Compiler", compiler), ("Run-time", runtime)] 54 | //! }) 55 | //! .run(); 56 | //! } 57 | //! ``` 58 | //! 59 | //! This defines a lang tester that uses all `*.rs` files in a given directory as test files, 60 | //! running two test commands against them: `Compiler` (i.e. `rustc`); and `Run-time` (the compiled 61 | //! binary). 62 | //! 63 | //! Users can then write test files such as the following: 64 | //! 65 | //! ```rust,ignore 66 | //! // Compiler: 67 | //! // stderr: 68 | //! // warning: unused variable: `x` 69 | //! // ...unused_var.rs:12:9 70 | //! // ... 71 | //! // 72 | //! // Run-time: 73 | //! // stdout: Hello world 74 | //! fn main() { 75 | //! let x = 0; 76 | //! println!("Hello world"); 77 | //! } 78 | //! ``` 79 | //! 80 | //! `lang_tester` is entirely ignorant of the language being tested, leaving it entirely to the 81 | //! user to determine what the test data in/for a file is. In this case, since we are embedding the 82 | //! test data as a Rust comment at the start of the file, the `test_extract` function we specified 83 | //! returns the following string: 84 | //! 85 | //! ```text 86 | //! Compiler: 87 | //! stderr: 88 | //! warning: unused variable: `x` 89 | //! ...unused_var.rs:12:9 90 | //! ... 91 | //! 92 | //! Run-time: 93 | //! stdout: Hello world 94 | //! ``` 95 | //! 96 | //! Test data is specified with a two-level indentation syntax: the outer most level of indentation 97 | //! defines a test command (multiple command names can be specified, as in the above); the inner 98 | //! most level of indentation defines alterations to the general command or sub-tests. Multi-line 99 | //! values are stripped of their common indentation, such that: 100 | //! 101 | //! ```text 102 | //! x: 103 | //! a 104 | //! b 105 | //! c 106 | //! ``` 107 | //! 108 | //! defines a test command `x` with a value `a\n b\nc`. Trailing whitespace is preserved. 109 | //! 110 | //! String matching is performed by the [fm crate](https://crates.io/crates/fm), which provides 111 | //! support for `...` operators and so on. Unless `lang_tester` is explicitly instructed otherwise, 112 | //! it uses `fm`'s defaults. In particular, even though `lang_tester` preserves (some) leading and 113 | //! (all) trailing whitespace, `fm` ignores leading and trailing whitespace by default (though this 114 | //! can be changed). 115 | //! 116 | //! Each test command must define at least one sub-test: 117 | //! 118 | //! * `status: >`, where `success` and `error` map to platform 119 | //! specific notions of a command completing successfully or unsuccessfully respectively. 120 | //! `signal` checks for termination due to a signal on Unix platforms; on non-Unix platforms, 121 | //! the test will be ignored. `` is a signed integer checking for a specific exit code on 122 | //! platforms that support it. If not specified, defaults to `success`. 123 | //! * `stderr: []`, `stdout: []` match `` against a command's `stderr` or 124 | //! `stdout`. The special string `...` can be used as a simple wildcard: if a line consists 125 | //! solely of `...`, it means "match zero or more lines"; if a line begins with `...`, it means 126 | //! "match the remainder of the line only"; if a line ends with `...`, it means "match the 127 | //! start of the line only". A line may start and end with `...`. Note that `stderr`/`stdout` 128 | //! matches ignore leading/trailing whitespace and newlines, but are case sensitive. If not 129 | //! specified, defaults to `...` (i.e. match anything). Note that the empty string matches only 130 | //! the empty string so e.g. `stderr:` on its own means that a command's `stderr` muct not 131 | //! contain any output. 132 | //! 133 | //! Test commands can alter the general command by specifying zero or more of the following: 134 | //! 135 | //! * `env-var: =` will set (or override if it is already present) the environment 136 | //! variable `` to the value ``. `env-var` can be specified multiple times, each 137 | //! setting an additional (or overriding an existing) environment variable. 138 | //! * `exec-arg: ` specifies a string which will be passed as an additional command-line 139 | //! argument to the command (in addition to those specified by the `test_cmds` function). 140 | //! Multiple `exec-arg`s can be specified, each adding an additional command-line argument. 141 | //! * `stdin: ` specifies text to be passed to the command's `stdin`. If the command 142 | //! exits without consuming all of ``, an error will be raised. Note, though, that 143 | //! operating system file buffers can mean that the command *appears* to have consumed all of 144 | //! `` without it actually having done so. 145 | //! 146 | //! Test commands can specify that a test should be rerun if one of the following (optional) is 147 | //! specified and it matches the test's output: 148 | //! 149 | //! * `rerun-if-status` follows the same format as the `status`. 150 | //! * `rerun-if-stderr` and `rerun-if-stdout` follow the same format as `stderr` and `stdout`. 151 | //! 152 | //! These can be useful if tests are subject to intermittent errors (e.g. network failure) that 153 | //! should not be considered as a failure of the test itself. Test commands are rerun at most *n* 154 | //! times, which by default is specified as 3. If no `rerun-if-` is specified, then the first time 155 | //! a test fails, it will be reported to the user. 156 | //! 157 | //! The above file thus contains 4 meaningful tests, two specified by the user and two implied by 158 | //! defaults: the `Compiler` should succeed (e.g. return a `0` exit code when run on Unix), and 159 | //! its `stderr` output should warn about an unused variable on line 12; and the resulting binary 160 | //! should succeed produce `Hello world` on `stdout`. 161 | //! 162 | //! A file's tests can be ignored entirely with: 163 | //! 164 | //! * `ignore-if: ` defines a shell command that will be run to determine whether to ignore 165 | //! this test or not. If `` returns 0 the test will be ignored, otherwise it will be run. 166 | //! `` will have its directory set to `CARGO_MANIFEST_DIR`. 167 | //! 168 | //! `lang_tester`'s output is deliberately similar to Rust's normal testing output. Running the 169 | //! example `rust_lang_tester` in this crate produces the following output: 170 | //! 171 | //! ```text 172 | //! $ cargo run --example=rust_lang_tester 173 | //! Compiling lang_tester v0.1.0 (/home/ltratt/scratch/softdev/lang_tester) 174 | //! Finished dev [unoptimized + debuginfo] target(s) in 3.49s 175 | //! Running `target/debug/examples/rust_lang_tester` 176 | //! 177 | //! running 4 tests 178 | //! test lang_tests::no_main ... ok 179 | //! test lang_tests::unknown_var ... ok 180 | //! test lang_tests::unused_var ... ok 181 | //! test lang_tests::exit_code ... ok 182 | //! 183 | //! test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out 184 | //! ``` 185 | //! 186 | //! If you want to run a subset of tests, you can specify simple filters which use substring match 187 | //! to run a subset of tests: 188 | //! 189 | //! ```text 190 | //! $ cargo run --example=rust_lang_tester var 191 | //! Compiling lang_tester v0.1.0 (/home/ltratt/scratch/softdev/lang_tester) 192 | //! Finished dev [unoptimized + debuginfo] target(s) in 3.37s 193 | //! Running `target/debug/examples/rust_lang_tester var` 194 | //! 195 | //! running 2 tests 196 | //! test lang_tests::unknown_var ... ok 197 | //! test lang_tests::unused_var ... ok 198 | //! 199 | //! test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 2 filtered out 200 | //! ``` 201 | //! 202 | //! ## Integration with Cargo. 203 | //! 204 | //! Tests created with lang_tester can be used as part of an existing test suite and can be run 205 | //! with the `cargo test` command. For example, if the Rust source file that runs your lang tests 206 | //! is `lang_tests/run.rs` then add the following to your Cargo.toml: 207 | //! 208 | //! ```text 209 | //! [[test]] 210 | //! name = "lang_tests" 211 | //! path = "lang_tests/run_tests.rs" 212 | //! harness = false 213 | //! ``` 214 | 215 | #![allow(clippy::needless_doctest_main)] 216 | #![allow(clippy::new_without_default)] 217 | #![allow(clippy::redundant_closure)] 218 | #![allow(clippy::type_complexity)] 219 | 220 | mod parser; 221 | mod tester; 222 | 223 | pub use tester::LangTester; 224 | 225 | pub(crate) fn fatal(msg: &str) -> ! { 226 | eprintln!("\nFatal exception:\n {}", msg); 227 | std::process::exit(1); 228 | } 229 | -------------------------------------------------------------------------------- /src/parser.rs: -------------------------------------------------------------------------------- 1 | use std::collections::hash_map::{Entry, HashMap}; 2 | 3 | use crate::{ 4 | fatal, 5 | tester::{Status, TestCmd, Tests}, 6 | }; 7 | 8 | /// Parse test data into a set of `Test`s. 9 | pub(crate) fn parse_tests<'a>(comment_prefix: Option<&str>, test_str: &'a str) -> Tests<'a> { 10 | let lines = test_str.lines().collect::>(); 11 | let mut tests = HashMap::new(); 12 | let mut line_off = 0; 13 | let mut ignore_if = None; 14 | while line_off < lines.len() { 15 | let indent = indent_level(&lines, line_off); 16 | if indent == lines[line_off].len() { 17 | line_off += 1; 18 | continue; 19 | } 20 | if let Some(cp) = comment_prefix { 21 | if lines[line_off][indent..].starts_with(cp) { 22 | line_off += 1; 23 | continue; 24 | } 25 | } 26 | let (test_name, val) = key_val(&lines, line_off, indent); 27 | if test_name == "ignore-if" { 28 | if ignore_if.is_some() { 29 | fatal(&format!( 30 | "'ignore-if' is specified more than once, line {}.", 31 | line_off 32 | )) 33 | } 34 | ignore_if = Some(val.into()); 35 | line_off += 1; 36 | continue; 37 | } 38 | if !val.is_empty() { 39 | fatal(&format!( 40 | "Test name '{}' can't have a value on line {}.", 41 | test_name, line_off 42 | )); 43 | } 44 | match tests.entry(test_name.to_lowercase()) { 45 | Entry::Occupied(_) => fatal(&format!( 46 | "Command name '{}' is specified more than once, line {}.", 47 | test_name, line_off 48 | )), 49 | Entry::Vacant(e) => { 50 | line_off += 1; 51 | let mut testcmd = TestCmd::default(); 52 | while line_off < lines.len() { 53 | let sub_indent = indent_level(&lines, line_off); 54 | if sub_indent == lines[line_off].len() { 55 | line_off += 1; 56 | continue; 57 | } 58 | if sub_indent == indent { 59 | break; 60 | } 61 | let (end_line_off, key, val) = 62 | key_multiline_val(comment_prefix, &lines, line_off, sub_indent); 63 | line_off = end_line_off; 64 | match key { 65 | "env-var" => { 66 | let val_str = val.join("\n"); 67 | match val_str.find('=') { 68 | Some(i) => { 69 | let key = val_str[..i].trim().to_owned(); 70 | let var = val_str[i + 1..].trim().to_owned(); 71 | testcmd.env.insert(key, var); 72 | } 73 | None => { 74 | fatal(&format!( 75 | "'{}' is not in the format '=' on line {}", 76 | val_str, line_off 77 | )); 78 | } 79 | } 80 | } 81 | "exec-arg" => { 82 | let val_str = val.join("\n"); 83 | testcmd.args.push(val_str); 84 | } 85 | "status" | "rerun-if-status" => { 86 | let val_str = val.join("\n"); 87 | let status = match val_str.to_lowercase().as_str() { 88 | "success" => Status::Success, 89 | "error" => Status::Error, 90 | "signal" => Status::Signal, 91 | x => { 92 | if let Ok(i) = x.parse::() { 93 | Status::Int(i) 94 | } else { 95 | fatal(&format!( 96 | "Unknown status '{}' on line {}", 97 | val_str, line_off 98 | )); 99 | } 100 | } 101 | }; 102 | match key { 103 | "status" => { 104 | testcmd.status = status; 105 | } 106 | "rerun-if-status" => { 107 | testcmd.rerun_if_status = Some(status); 108 | } 109 | _ => { 110 | unreachable!(); 111 | } 112 | } 113 | } 114 | "stdin" => { 115 | testcmd.stdin = Some(val.join("\n")); 116 | } 117 | "stderr" => { 118 | testcmd.stderr = val; 119 | } 120 | "stdout" => { 121 | testcmd.stdout = val; 122 | } 123 | "rerun-if-stderr" => { 124 | testcmd.rerun_if_stderr = Some(val); 125 | } 126 | "rerun-if-stdout" => { 127 | testcmd.rerun_if_stdout = Some(val); 128 | } 129 | _ => fatal(&format!("Unknown key '{}' on line {}.", key, line_off)), 130 | } 131 | } 132 | e.insert(testcmd); 133 | } 134 | } 135 | } 136 | Tests { ignore_if, tests } 137 | } 138 | 139 | fn indent_level(lines: &[&str], line_off: usize) -> usize { 140 | lines[line_off] 141 | .chars() 142 | .take_while(|c| c.is_whitespace()) 143 | .count() 144 | } 145 | 146 | /// Turn a line such as `key: val` into its separate components. 147 | fn key_val<'a>(lines: &[&'a str], line_off: usize, indent: usize) -> (&'a str, &'a str) { 148 | let line = lines[line_off]; 149 | let key_len = line[indent..] 150 | .chars() 151 | .take_while(|c| !(c.is_whitespace() || c == &':')) 152 | .count(); 153 | let key = &line[indent..indent + key_len]; 154 | let mut content_start = indent + key_len; 155 | content_start += line[content_start..] 156 | .chars() 157 | .take_while(|c| c.is_whitespace()) 158 | .count(); 159 | match line[content_start..].chars().next() { 160 | Some(':') => content_start += ':'.len_utf8(), 161 | _ => fatal(&format!( 162 | "Invalid key terminator at line {}.\n {}", 163 | line_off, line 164 | )), 165 | } 166 | content_start += line[content_start..] 167 | .chars() 168 | .take_while(|c| c.is_whitespace()) 169 | .count(); 170 | (key, &line[content_start..]) 171 | } 172 | 173 | /// Turn one more lines of the format `key: val` (where `val` may spread over many lines) into its 174 | /// separate components. 175 | fn key_multiline_val<'a>( 176 | comment_prefix: Option<&str>, 177 | lines: &[&'a str], 178 | mut line_off: usize, 179 | indent: usize, 180 | ) -> (usize, &'a str, Vec<&'a str>) { 181 | let (key, first_line_val) = key_val(lines, line_off, indent); 182 | line_off += 1; 183 | let mut val = vec![first_line_val]; 184 | if line_off < lines.len() { 185 | let sub_indent = indent_level(lines, line_off); 186 | while line_off < lines.len() { 187 | let cur_indent = indent_level(lines, line_off); 188 | if cur_indent == lines[line_off].len() { 189 | val.push(""); 190 | line_off += 1; 191 | continue; 192 | } 193 | if cur_indent <= indent { 194 | break; 195 | } 196 | if let Some(cp) = comment_prefix { 197 | if lines[line_off][sub_indent..].starts_with(cp) { 198 | line_off += 1; 199 | continue; 200 | } 201 | } 202 | val.push(&lines[line_off][sub_indent..]); 203 | line_off += 1; 204 | } 205 | } 206 | // Remove trailing empty strings 207 | while !val.is_empty() && val[val.len() - 1].is_empty() { 208 | val.pop(); 209 | } 210 | // Remove leading empty strings 211 | while !val.is_empty() && val[0].is_empty() { 212 | val.remove(0); 213 | } 214 | 215 | (line_off, key, val) 216 | } 217 | 218 | #[cfg(test)] 219 | mod test { 220 | use super::*; 221 | 222 | #[test] 223 | fn test_key_multiline() { 224 | assert_eq!(key_multiline_val(None, &["x:", ""], 0, 0), (2, "x", vec![])); 225 | assert_eq!( 226 | key_multiline_val(None, &["x: y", " z", "a"], 0, 0), 227 | (2, "x", vec!["y", "z"]) 228 | ); 229 | assert_eq!( 230 | key_multiline_val(None, &["x:", " z", "a"], 0, 0), 231 | (2, "x", vec!["z"]) 232 | ); 233 | assert_eq!( 234 | key_multiline_val(None, &["x:", " z ", " a ", " ", "b"], 0, 0), 235 | (4, "x", vec!["z ", "a "]) 236 | ); 237 | assert_eq!( 238 | key_multiline_val(None, &["x:", " z ", " a ", " ", " b"], 0, 0), 239 | (5, "x", vec!["z ", " a ", "", "b"]) 240 | ); 241 | assert_eq!( 242 | key_multiline_val( 243 | Some("#"), 244 | &["x:", " z ", " a ", " # c2", " ", " b"], 245 | 0, 246 | 0 247 | ), 248 | (6, "x", vec!["z ", " a ", "", "b"]) 249 | ); 250 | } 251 | } 252 | -------------------------------------------------------------------------------- /src/tester.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | cmp::max, 3 | collections::{hash_map::HashMap, HashSet}, 4 | convert::TryFrom, 5 | env, 6 | fs::canonicalize, 7 | io::{self, Read, Write}, 8 | os::{ 9 | raw::c_int, 10 | unix::{io::AsRawFd, process::ExitStatusExt}, 11 | }, 12 | panic::{catch_unwind, RefUnwindSafe}, 13 | path::{Path, PathBuf, MAIN_SEPARATOR}, 14 | process::{self, Command, ExitStatus}, 15 | str, 16 | sync::{ 17 | atomic::{AtomicUsize, Ordering}, 18 | Arc, Mutex, 19 | }, 20 | thread::sleep, 21 | time::{Duration, Instant}, 22 | }; 23 | 24 | use fm::{FMBuilder, FMatchError}; 25 | use getopts::Options; 26 | use libc::{ 27 | close, fcntl, poll, pollfd, F_GETFL, F_SETFL, O_NONBLOCK, POLLERR, POLLHUP, POLLIN, POLLNVAL, 28 | POLLOUT, 29 | }; 30 | use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; 31 | use threadpool::ThreadPool; 32 | use walkdir::WalkDir; 33 | 34 | use crate::{fatal, parser::parse_tests}; 35 | 36 | /// The size of the (stack allocated) buffer use to read stderr/stdout from a child process. 37 | const READBUF: usize = 1024 * 4; // bytes 38 | /// Print a warning to the user every multiple of `TIMEOUT` seconds that a child process has run 39 | /// without completing. 40 | const TIMEOUT: u64 = 60; // seconds 41 | /// The time that we should initially wait() for a child process to exit. This should be a very 42 | /// small value, as most child processes will exit almost immediately. 43 | const INITIAL_WAIT_TIMEOUT: u64 = 10000; // nanoseconds 44 | /// The maximum time we should wait() between checking if a child process has exited. 45 | const MAX_WAIT_TIMEOUT: u64 = 250_000_000; // nanoseconds 46 | // 47 | /// The default maximum number of times to rerun a command if it fails and a rerun-if-* matches. 48 | const DEFAULT_RERUN_AT_MOST: u64 = 3; 49 | 50 | pub struct LangTester { 51 | use_cmdline_args: bool, 52 | test_path_filter: Option bool + RefUnwindSafe>>, 53 | cmdline_filters: Option>, 54 | inner: Arc, 55 | } 56 | 57 | /// This is the information shared across test threads and which needs to be hidden behind an 58 | /// `Arc`. 59 | struct LangTesterPooler { 60 | test_dir: Option, 61 | test_threads: usize, 62 | ignored: bool, 63 | nocapture: bool, 64 | comment_prefix: Option, 65 | test_extract: Option String + RefUnwindSafe + Send + Sync>>, 66 | fm_options: Option< 67 | Box< 68 | dyn for<'a> Fn(&'a Path, TestStream, FMBuilder<'a>) -> FMBuilder<'a> 69 | + RefUnwindSafe 70 | + Send 71 | + Sync, 72 | >, 73 | >, 74 | test_cmds: Option Vec<(&str, Command)> + RefUnwindSafe + Send + Sync>>, 75 | rerun_at_most: u64, 76 | } 77 | 78 | /// Specify a given test stream. 79 | #[derive(Clone, Copy)] 80 | pub enum TestStream { 81 | Stderr, 82 | Stdout, 83 | } 84 | 85 | impl LangTester { 86 | /// Create a new `LangTester` with default options. Note that, at a minimum, you need to call 87 | /// [`test_dir`](#method.test_dir), [`test_extract`](#method.test_extract), and 88 | /// [`test_cmds`](#method.test_cmds). 89 | pub fn new() -> Self { 90 | LangTester { 91 | test_path_filter: None, 92 | use_cmdline_args: true, 93 | cmdline_filters: None, 94 | inner: Arc::new(LangTesterPooler { 95 | test_dir: None, 96 | ignored: false, 97 | nocapture: false, 98 | comment_prefix: None, 99 | test_threads: num_cpus::get(), 100 | fm_options: None, 101 | test_extract: None, 102 | test_cmds: None, 103 | rerun_at_most: DEFAULT_RERUN_AT_MOST, 104 | }), 105 | } 106 | } 107 | 108 | /// Specify the directory where test files are contained. Note that this directory will be 109 | /// searched recursively (i.e. subdirectories and their contents will also be considered as 110 | /// potential test files). 111 | pub fn test_dir(&mut self, test_dir: &str) -> &mut Self { 112 | let inner = Arc::get_mut(&mut self.inner).unwrap(); 113 | inner.test_dir = Some(canonicalize(test_dir).unwrap()); 114 | self 115 | } 116 | 117 | /// Specify the number of simultaneous running test cases. Defaults to using 118 | /// all available CPUs. 119 | pub fn test_threads(&mut self, test_threads: usize) -> &mut Self { 120 | let inner = Arc::get_mut(&mut self.inner).unwrap(); 121 | inner.test_threads = test_threads; 122 | self 123 | } 124 | 125 | /// Specify the maximum number of times to rerun a test if it fails and a rerun-if-* matches. 126 | /// Defaults to 3. 127 | pub fn rerun_at_most(&mut self, rerun_at_most: u64) -> &mut Self { 128 | let inner = Arc::get_mut(&mut self.inner).unwrap(); 129 | inner.rerun_at_most = rerun_at_most; 130 | self 131 | } 132 | 133 | /// If set, defines what lines will be treated as comments if, ignoring the current level of 134 | /// indentation, they begin with `comment_prefix`. 135 | /// 136 | /// This option defaults to `None`. 137 | pub fn comment_prefix>(&mut self, comment_prefix: S) -> &mut Self { 138 | let inner = Arc::get_mut(&mut self.inner).unwrap(); 139 | inner.comment_prefix = Some(comment_prefix.as_ref().to_owned()); 140 | self 141 | } 142 | 143 | /// If `test_file_filter` is specified, only files for which it returns `true` will be 144 | /// considered tests. A common use of this is to filter files based on filename extensions 145 | /// e.g.: 146 | /// 147 | /// ```rust,ignore 148 | /// LangTester::new() 149 | /// ... 150 | /// .test_file_filter(|p| p.extension().unwrap().to_str().unwrap() == "rs") 151 | /// ... 152 | /// ``` 153 | /// 154 | /// Note that `lang_tester` recursively searches directories for files. 155 | #[deprecated( 156 | since = "0.7.5", 157 | note = "Convert `test_file_filter(|p| ...)` to `test_path_fiter(|p| p.is_file() & ...)`" 158 | )] 159 | pub fn test_file_filter(&mut self, test_path_filter: F) -> &mut Self 160 | where 161 | F: 'static + Fn(&Path) -> bool + RefUnwindSafe, 162 | { 163 | self.test_path_filter = Some(Box::new(move |p| p.is_file() && test_path_filter(p))); 164 | self 165 | } 166 | 167 | /// If `test_path_filter` is specified, only paths for which it returns `true` will be 168 | /// considered tests. A common use of this is to filter tests based on filename extensions 169 | /// e.g.: 170 | /// 171 | /// ```rust,ignore 172 | /// LangTester::new() 173 | /// ... 174 | /// .test_path_filter(|p| p.extension().and_then(|x| x.to_str()) == Some("rs")) 175 | /// ... 176 | /// ``` 177 | /// 178 | /// Note that `lang_tester` recursively searches directories. 179 | pub fn test_path_filter(&mut self, test_path_filter: F) -> &mut Self 180 | where 181 | F: 'static + Fn(&Path) -> bool + RefUnwindSafe, 182 | { 183 | self.test_path_filter = Some(Box::new(test_path_filter)); 184 | self 185 | } 186 | 187 | /// Specify a function which can extract the test data for `lang_tester` from a test file path, 188 | /// returning it as a `String`. Note that the test data does not have to be extracted from the 189 | /// `Path` passed to the function -- it can come from any source. 190 | /// 191 | /// How the test data is extracted from the test file is entirely up to the user, though a 192 | /// common convention is to store the test data in a comment at the beginning of the test file. 193 | /// For example, for Rust code one could use a function along the lines of the following: 194 | /// 195 | /// ```rust,ignore 196 | /// LangTester::new() 197 | /// ... 198 | /// .test_extract(|p| { 199 | /// std::fs::read_to_string(p) 200 | /// .unwrap() 201 | /// .lines() 202 | /// // Skip non-commented lines at the start of the file. 203 | /// .skip_while(|l| !l.starts_with("//")) 204 | /// // Extract consecutive commented lines. 205 | /// .take_while(|l| l.starts_with("//")) 206 | /// .map(|l| &l[2..]) 207 | /// .collect::>() 208 | /// .join("\n") 209 | /// }) 210 | /// ... 211 | /// ``` 212 | pub fn test_extract(&mut self, test_extract: F) -> &mut Self 213 | where 214 | F: 'static + Fn(&Path) -> String + RefUnwindSafe + Send + Sync, 215 | { 216 | Arc::get_mut(&mut self.inner).unwrap().test_extract = Some(Box::new(test_extract)); 217 | self 218 | } 219 | 220 | /// Specify a function which sets options for the [`fm`](https://crates.io/crates/fm) library. 221 | /// `fm` is used for the fuzzy matching in `lang_tester`. This function can be used to override 222 | /// `fm`'s defaults for a given test file (passed as `Path`) and a given testing stream (stderr 223 | /// or stdout, passed as `TestStream`) when executing a command for that file: it is passed a 224 | /// [`FMBuilder`](https://docs.rs/fm/*/fm/struct.FMBuilder.html) and must return a `FMBuilder`. 225 | /// For example, to make use of `fm`'s "name matcher" option such that all instances of `$1` 226 | /// must match the same value (without precisely specifying what that value is) one could use 227 | /// the following: 228 | /// 229 | /// ```rust,ignore 230 | /// LangTester::new() 231 | /// ... 232 | /// .fm_options(|_, _, fmb| { 233 | /// let ptn_re = Regex::new(r"\$.+?\b").unwrap(); 234 | /// let text_re = Regex::new(r".+?\b").unwrap(); 235 | /// fmb.name_matcher(ptn_re, text_re) 236 | /// }) 237 | /// ``` 238 | pub fn fm_options(&mut self, fm_options: F) -> &mut Self 239 | where 240 | F: 'static 241 | + for<'a> Fn(&'a Path, TestStream, FMBuilder<'a>) -> FMBuilder<'a> 242 | + RefUnwindSafe 243 | + Send 244 | + Sync, 245 | { 246 | Arc::get_mut(&mut self.inner).unwrap().fm_options = Some(Box::new(fm_options)); 247 | self 248 | } 249 | 250 | /// Specify a function which takes a `Path` to a test file and returns a vector containing 1 or 251 | /// more (`name`, <[`Command`](https://doc.rust-lang.org/std/process/struct.Command.html)>) 252 | /// pairs. The commands will be executed in order on the test file: for each executed command, 253 | /// test commands starting with `` will be checked. For example, if your pipeline 254 | /// requires separate compilation and linking, you might specify something along the lines of 255 | /// the following: 256 | /// 257 | /// ```rust,ignore 258 | /// let tempdir = ...; // A `Path` to a temporary directory. 259 | /// LangTester::new() 260 | /// ... 261 | /// .test_cmds(|p| { 262 | /// let mut exe = PathBuf::new(); 263 | /// exe.push(&tempdir); 264 | /// exe.push(p.file_stem().unwrap()); 265 | /// let mut compiler = Command::new("rustc"); 266 | /// compiler.args(&["-o", exe.to_str().unwrap(), p.to_str().unwrap()]); 267 | /// let runtime = Command::new(exe); 268 | /// vec![("Compiler", compiler), ("Run-time", runtime)] 269 | /// }) 270 | /// ... 271 | /// ``` 272 | /// 273 | /// and then have test data such as: 274 | /// 275 | /// ```text 276 | /// Compiler: 277 | /// status: success 278 | /// stderr: 279 | /// stdout: 280 | /// 281 | /// Run-time: 282 | /// status: failure 283 | /// stderr: 284 | /// ... 285 | /// Error at line 10 286 | /// ... 287 | /// ``` 288 | pub fn test_cmds(&mut self, test_cmds: F) -> &mut Self 289 | where 290 | F: 'static + Fn(&Path) -> Vec<(&str, Command)> + RefUnwindSafe + Send + Sync, 291 | { 292 | Arc::get_mut(&mut self.inner).unwrap().test_cmds = Some(Box::new(test_cmds)); 293 | self 294 | } 295 | 296 | /// If set to `true`, this reads arguments from `std::env::args()` and interprets them in the 297 | /// same way as normal cargo test files. For example if you have tests "ab" and "cd" but only 298 | /// want to run the latter: 299 | /// 300 | /// ```sh 301 | /// $ c 302 | /// ``` 303 | /// 304 | /// As this suggests, a simple substring search is used to decide which tests to run. 305 | /// 306 | /// You can get help on `lang_tester`'s options: 307 | /// 308 | /// ```sh 309 | /// $ --help 310 | /// ``` 311 | /// 312 | /// This option defaults to `true`. 313 | pub fn use_cmdline_args(&mut self, use_cmdline_args: bool) -> &mut Self { 314 | self.use_cmdline_args = use_cmdline_args; 315 | self 316 | } 317 | 318 | /// Make sure the user has specified the minimum set of things we need from them. 319 | fn validate(&self) { 320 | if self.inner.test_dir.is_none() { 321 | fatal("test_dir must be specified."); 322 | } 323 | if self.inner.test_extract.is_none() { 324 | fatal("test_extract must be specified."); 325 | } 326 | if self.inner.test_cmds.is_none() { 327 | fatal("test_cmds must be specified."); 328 | } 329 | } 330 | 331 | /// Enumerate all the test files we need to check, along with the number of files filtered out 332 | /// (e.g. if you have tests `a, b, c` and the user does something like `cargo test b`, 2 tests 333 | /// (`a` and `c`) will be filtered out. The `PathBuf`s returned are guaranteed to be fully 334 | /// canonicalised. 335 | fn test_files( 336 | &self, 337 | failures: Arc>>, 338 | ) -> (Vec, usize) { 339 | let mut num_filtered = 0; 340 | let paths = WalkDir::new(self.inner.test_dir.as_ref().unwrap()) 341 | .into_iter() 342 | .filter_map(|x| x.ok()) 343 | .map(|x| canonicalize(x.into_path()).unwrap()) 344 | // Filter out non-test files 345 | .filter(|x| match self.test_path_filter.as_ref() { 346 | Some(f) => match catch_unwind(|| f(x)) { 347 | Ok(b) => b, 348 | Err(_) => { 349 | let failure = TestFailure { 350 | status: None, 351 | stdin_remaining: 0, 352 | stderr: None, 353 | stderr_match: None, 354 | stdout: None, 355 | stdout_match: None, 356 | }; 357 | failures 358 | .lock() 359 | .unwrap() 360 | .push((x.to_str().unwrap().to_owned(), failure)); 361 | false 362 | } 363 | }, 364 | None => true, 365 | }) 366 | // If the user has named one or more tests on the command-line, run only those, 367 | // filtering out the rest (counting them as ignored). 368 | .filter(|x| { 369 | let test_fname = format!( 370 | "lang_tests::{}", 371 | test_fname(self.inner.test_dir.as_deref().unwrap(), x.as_path(),) 372 | ); 373 | match self.cmdline_filters.as_ref() { 374 | Some(fs) => { 375 | debug_assert!(self.use_cmdline_args); 376 | for f in fs { 377 | if test_fname.contains(f) { 378 | return true; 379 | } 380 | } 381 | num_filtered += 1; 382 | false 383 | } 384 | None => true, 385 | } 386 | }) 387 | .collect(); 388 | (paths, num_filtered) 389 | } 390 | 391 | /// Run all the lang tests. 392 | pub fn run(&mut self) { 393 | self.validate(); 394 | if self.use_cmdline_args { 395 | let args: Vec = env::args().collect(); 396 | let matches = Options::new() 397 | .optflag("h", "help", "") 398 | .optflag("", "ignored", "Run only ignored tests") 399 | .optflag( 400 | "", 401 | "nocapture", 402 | "Pass command stderr/stdout through to the terminal", 403 | ) 404 | .optopt( 405 | "", 406 | "test-threads", 407 | "Number of threads used for running tests in parallel", 408 | "n_threads", 409 | ) 410 | .parse(&args[1..]) 411 | .unwrap_or_else(|_| usage()); 412 | if matches.opt_present("h") { 413 | usage(); 414 | } 415 | if matches.opt_present("ignored") { 416 | Arc::get_mut(&mut self.inner).unwrap().ignored = true; 417 | } 418 | if matches.opt_present("nocapture") { 419 | Arc::get_mut(&mut self.inner).unwrap().nocapture = true; 420 | } 421 | if let Some(s) = matches.opt_str("test-threads") { 422 | let test_threads = s.parse::().unwrap_or_else(|_| usage()); 423 | if test_threads == 0 { 424 | fatal("Must specify more than 0 threads."); 425 | } 426 | Arc::get_mut(&mut self.inner).unwrap().test_threads = test_threads; 427 | } 428 | if !matches.free.is_empty() { 429 | self.cmdline_filters = Some(matches.free); 430 | } 431 | } 432 | let failures = Arc::new(Mutex::new(Vec::new())); 433 | let (test_files, num_filtered) = self.test_files(Arc::clone(&failures)); 434 | let test_files_len = test_files.len(); 435 | let num_ignored = if failures.lock().unwrap().is_empty() { 436 | eprint!("\nrunning {} tests", test_files.len()); 437 | test_file(test_files, Arc::clone(&self.inner), Arc::clone(&failures)) 438 | } else { 439 | 0 440 | }; 441 | 442 | let mut failures = Mutex::into_inner(Arc::try_unwrap(failures).unwrap()).unwrap(); 443 | failures.sort_by_key(|x| x.0.to_lowercase()); 444 | self.pp_failures( 445 | &failures, 446 | max(test_files_len, failures.len()), 447 | num_ignored, 448 | num_filtered, 449 | ); 450 | 451 | if !failures.is_empty() { 452 | process::exit(1); 453 | } 454 | } 455 | 456 | /// Pretty print any failures to `stderr`. 457 | fn pp_failures( 458 | &self, 459 | failures: &[(String, TestFailure)], 460 | test_files_len: usize, 461 | num_ignored: usize, 462 | num_filtered: usize, 463 | ) { 464 | if !failures.is_empty() { 465 | eprintln!("\n\nfailures:"); 466 | for (test_fname, test) in failures { 467 | if let Some(ref status) = test.status { 468 | eprintln!("\n---- lang_tests::{} status ----\n{}", test_fname, status); 469 | } 470 | if test.stdin_remaining != 0 { 471 | eprintln!( 472 | "\n---- lang_tests::{} stdin ----\n{} bytes of stdin were not consumed", 473 | test_fname, test.stdin_remaining 474 | ); 475 | } 476 | if let Some(ref stderr) = test.stderr { 477 | eprintln!("\n---- lang_tests::{} stderr ----\n", test_fname); 478 | if let Some(ref stderr_match) = test.stderr_match { 479 | eprint!("{}", stderr_match); 480 | } else { 481 | eprintln!("{}", stderr); 482 | } 483 | } 484 | if let Some(ref stdout) = test.stdout { 485 | eprintln!("\n---- lang_tests::{} stdout ----\n", test_fname); 486 | if let Some(ref stdout_match) = test.stdout_match { 487 | eprint!("{}", stdout_match); 488 | } else { 489 | eprintln!("{}", stdout); 490 | } 491 | } 492 | } 493 | eprint!("failures:"); 494 | for (test_fname, _) in failures { 495 | eprint!("\n lang_tests::{}", test_fname); 496 | } 497 | } 498 | 499 | eprint!("\n\ntest result: "); 500 | if failures.is_empty() { 501 | write_with_colour("ok", Color::Green); 502 | } else { 503 | write_with_colour("FAILED", Color::Red); 504 | } 505 | eprintln!( 506 | ". {} passed; {} failed; {} ignored; 0 measured; {} filtered out\n", 507 | test_files_len - failures.len(), 508 | failures.len(), 509 | num_ignored, 510 | num_filtered 511 | ); 512 | } 513 | } 514 | 515 | /// The status of an executed command. 516 | #[derive(Clone, Debug, PartialEq, Eq)] 517 | pub(crate) enum Status { 518 | /// The command exited successfully (by whatever definition of "successful" the running 519 | /// platform uses). 520 | Success, 521 | /// The command did not execute successfully (by whatever definition of "not successful" the 522 | /// running platform uses). 523 | Error, 524 | /// The command terminated due to a signal. This option may not be available on all 525 | /// platforms. 526 | Signal, 527 | /// The command exited with a precise exit code. This option may not be available on all 528 | /// platforms. 529 | Int(i32), 530 | } 531 | 532 | /// A user `TestCmd`. 533 | #[derive(Clone, Debug)] 534 | pub(crate) struct TestCmd<'a> { 535 | pub status: Status, 536 | pub stdin: Option, 537 | pub stderr: Vec<&'a str>, 538 | pub stdout: Vec<&'a str>, 539 | /// A list of custom command line arguments which should be passed when 540 | /// executing the test command. 541 | pub args: Vec, 542 | pub env: HashMap, 543 | pub rerun_if_status: Option, 544 | pub rerun_if_stderr: Option>, 545 | pub rerun_if_stdout: Option>, 546 | } 547 | 548 | impl TestCmd<'_> { 549 | pub fn default() -> Self { 550 | Self { 551 | status: Status::Success, 552 | stdin: None, 553 | stderr: vec!["..."], 554 | stdout: vec!["..."], 555 | args: Vec::new(), 556 | env: HashMap::new(), 557 | rerun_if_status: None, 558 | rerun_if_stderr: None, 559 | rerun_if_stdout: None, 560 | } 561 | } 562 | } 563 | 564 | /// A collection of tests. 565 | pub(crate) struct Tests<'a> { 566 | pub ignore_if: Option, 567 | pub tests: HashMap>, 568 | } 569 | 570 | /// If one or more parts of a `TestCmd` fail, the parts that fail are set to `Some(...)` in an 571 | /// instance of this struct. 572 | #[derive(Debug)] 573 | struct TestFailure { 574 | status: Option, 575 | stdin_remaining: usize, 576 | stderr: Option, 577 | stderr_match: Option, 578 | stdout: Option, 579 | stdout_match: Option, 580 | } 581 | 582 | fn write_with_colour(s: &str, colour: Color) { 583 | let mut stderr = StandardStream::stderr(ColorChoice::Always); 584 | stderr.set_color(ColorSpec::new().set_fg(Some(colour))).ok(); 585 | io::stderr().write_all(s.as_bytes()).ok(); 586 | stderr.reset().ok(); 587 | } 588 | 589 | fn write_ignored(test_name: &str, message: &str, inner: Arc) { 590 | // Grab a lock on stderr so that we can avoid the possibility of lines blurring 591 | // together in confusing ways. 592 | let stderr = StandardStream::stderr(ColorChoice::Always); 593 | let mut handle = stderr.lock(); 594 | if inner.test_threads > 1 { 595 | handle 596 | .write_all(format!("\ntest lang_tests::{} ... ", test_name).as_bytes()) 597 | .ok(); 598 | } 599 | handle 600 | .set_color(ColorSpec::new().set_fg(Some(Color::Yellow))) 601 | .ok(); 602 | handle.write_all(b"ignored").ok(); 603 | handle.reset().ok(); 604 | if !message.is_empty() { 605 | handle.write_all(format!(" ({})", message).as_bytes()).ok(); 606 | } 607 | } 608 | 609 | fn usage() -> ! { 610 | eprintln!("Usage: [--ignored] [--nocapture] [--test-threads=] [] [... ]"); 611 | process::exit(1); 612 | } 613 | 614 | /// Check for the case where the user has a test called `X` but `test_cmds` doesn't have a command 615 | /// with a matching name. This is almost certainly a bug, in the sense that the test can never, 616 | /// ever fire. 617 | fn check_names(cmd_pairs: &[(String, Command)], tests: &HashMap) { 618 | let cmd_names = cmd_pairs.iter().map(|x| &x.0).collect::>(); 619 | let test_names = tests.keys().collect::>(); 620 | let diff = test_names 621 | .difference(&cmd_names) 622 | .map(|x| x.as_str()) 623 | .collect::>(); 624 | if !diff.is_empty() { 625 | fatal(&format!( 626 | "Command name(s) '{}' in tests are not found in the actual commands.", 627 | diff.join(", ") 628 | )); 629 | } 630 | } 631 | 632 | /// Run every test in `test_files`, returning a tuple `(failures, num_ignored)`. 633 | fn test_file( 634 | test_files: Vec, 635 | inner: Arc, 636 | failures: Arc>>, 637 | ) -> usize { 638 | let num_ignored = Arc::new(AtomicUsize::new(0)); 639 | let pool = ThreadPool::new(inner.test_threads); 640 | for p in test_files { 641 | let test_fname = test_fname(inner.test_dir.as_ref().unwrap(), &p); 642 | 643 | let num_ignored = num_ignored.clone(); 644 | let failures = failures.clone(); 645 | let inner = inner.clone(); 646 | pool.execute(move || { 647 | if inner.test_threads == 1 { 648 | eprint!("\ntest lang_test::{} ... ", test_fname); 649 | } 650 | let test_extract = inner.test_extract.as_ref().unwrap(); 651 | match catch_unwind(|| test_extract(p.as_path())) { 652 | Ok(test_str) => { 653 | if test_str.is_empty() { 654 | write_ignored(test_fname.as_str(), "test string is empty", inner); 655 | num_ignored.fetch_add(1, Ordering::Relaxed); 656 | return; 657 | } 658 | 659 | let tests = parse_tests(inner.comment_prefix.as_deref(), &test_str); 660 | let ignore = if let Some(ignore_if) = tests.ignore_if { 661 | Command::new(env::var("SHELL").unwrap_or_else(|_| "/bin/sh".to_owned())) 662 | .args(["-c", &ignore_if]) 663 | .current_dir(env::var("CARGO_MANIFEST_DIR").unwrap()) 664 | .stdin(process::Stdio::piped()) 665 | .stderr(process::Stdio::piped()) 666 | .stdout(process::Stdio::piped()) 667 | .status() 668 | .unwrap_or_else(|_| { 669 | fatal(&format!("Couldn't run ignore-if '{ignore_if}'")) 670 | }) 671 | .success() 672 | } else { 673 | false 674 | }; 675 | if (inner.ignored && !ignore) || (!inner.ignored && ignore) { 676 | write_ignored(test_fname.as_str(), "", inner); 677 | num_ignored.fetch_add(1, Ordering::Relaxed); 678 | return; 679 | } 680 | 681 | if run_tests(Arc::clone(&inner), tests.tests, p, test_fname, failures) { 682 | num_ignored.fetch_add(1, Ordering::Relaxed); 683 | } 684 | } 685 | Err(_) => { 686 | let failure = TestFailure { 687 | status: None, 688 | stdin_remaining: 0, 689 | stderr: None, 690 | stderr_match: None, 691 | stdout: None, 692 | stdout_match: None, 693 | }; 694 | failures.lock().unwrap().push((test_fname, failure)); 695 | } 696 | }; 697 | }); 698 | } 699 | pool.join(); 700 | 701 | Arc::try_unwrap(num_ignored).unwrap().into_inner() 702 | } 703 | 704 | /// Convert a test file name to a user-friendly test name (e.g. "lang_tests/a/b.x" might become 705 | /// "a::b.x"). 706 | fn test_fname(test_dir_path: &Path, test_fpath: &Path) -> String { 707 | if let Some(test_fpath) = test_fpath.as_os_str().to_str() { 708 | if let Some(testdir_path) = test_dir_path.as_os_str().to_str() { 709 | if test_fpath.starts_with(testdir_path) { 710 | return test_fpath[testdir_path.len() + MAIN_SEPARATOR.len_utf8()..] 711 | .to_owned() 712 | .replace(MAIN_SEPARATOR, "::"); 713 | } 714 | } 715 | } 716 | 717 | test_fpath.file_stem().unwrap().to_str().unwrap().to_owned() 718 | } 719 | 720 | /// Run the tests for `path`. 721 | fn run_tests( 722 | inner: Arc, 723 | tests: HashMap, 724 | path: PathBuf, 725 | test_fname: String, 726 | failures: Arc>>, 727 | ) -> bool { 728 | if !cfg!(unix) && tests.values().any(|t| t.status == Status::Signal) { 729 | write_ignored( 730 | test_fname.as_str(), 731 | "signal termination not supported on this platform", 732 | inner, 733 | ); 734 | return true; 735 | } 736 | 737 | let mut failure = TestFailure { 738 | status: None, 739 | stdin_remaining: 0, 740 | stderr: None, 741 | stderr_match: None, 742 | stdout: None, 743 | stdout_match: None, 744 | }; 745 | 746 | let test_cmds = inner.test_cmds.as_ref().unwrap(); 747 | let cmd_pairs = match catch_unwind(|| test_cmds(path.as_path())) { 748 | Ok(x) => x 749 | .into_iter() 750 | .map(|(test_name, cmd)| (test_name.to_lowercase(), cmd)) 751 | .collect::>(), 752 | Err(_) => { 753 | failures.lock().unwrap().push((test_fname, failure)); 754 | return false; 755 | } 756 | }; 757 | check_names(&cmd_pairs, &tests); 758 | 759 | 'a: for (cmd_name, mut cmd) in cmd_pairs { 760 | let default_test = TestCmd::default(); 761 | let test = tests.get(&cmd_name).unwrap_or(&default_test); 762 | cmd.args(&test.args); 763 | cmd.envs(&test.env); 764 | let mut rerun = 0; 765 | loop { 766 | rerun += 1; 767 | let (status, stdin_remaining, stderr, stdout) = 768 | run_cmd(inner.clone(), &test_fname, &mut cmd, test); 769 | 770 | let mut meant_to_error = false; 771 | 772 | // Give the user the option of setting options for the fuzzy matchers. 773 | let stderr_str = test.stderr.join("\n"); 774 | let mut stderr_fmb = FMBuilder::new(&stderr_str).unwrap(); 775 | let stdout_str = test.stdout.join("\n"); 776 | let mut stdout_fmb = FMBuilder::new(&stdout_str).unwrap(); 777 | 778 | let rerun_if_stderr_str = test.rerun_if_stderr.as_ref().unwrap_or(&vec![]).join("\n"); 779 | let mut rerun_if_stderr_fmb = FMBuilder::new(&rerun_if_stderr_str).unwrap(); 780 | let rerun_if_stdout_str = test.rerun_if_stdout.as_ref().unwrap_or(&vec![]).join("\n"); 781 | let mut rerun_if_stdout_fmb = FMBuilder::new(&rerun_if_stdout_str).unwrap(); 782 | if let Some(ref fm_options) = inner.fm_options { 783 | match catch_unwind(|| { 784 | ( 785 | fm_options(path.as_path(), TestStream::Stderr, stderr_fmb), 786 | fm_options(path.as_path(), TestStream::Stdout, stdout_fmb), 787 | fm_options(path.as_path(), TestStream::Stderr, rerun_if_stderr_fmb), 788 | fm_options(path.as_path(), TestStream::Stdout, rerun_if_stdout_fmb), 789 | ) 790 | }) { 791 | Ok((a, b, c, d)) => { 792 | stderr_fmb = a; 793 | stdout_fmb = b; 794 | rerun_if_stderr_fmb = c; 795 | rerun_if_stdout_fmb = d; 796 | } 797 | Err(_) => { 798 | failures.lock().unwrap().push((test_fname, failure)); 799 | return false; 800 | } 801 | } 802 | } 803 | 804 | let match_stderr = match stderr_fmb.build() { 805 | Ok(x) => x.matches(&stderr), 806 | Err(e) => { 807 | failure.stderr = Some(format!("FM error: {}", e)); 808 | break 'a; 809 | } 810 | }; 811 | let match_stdout = match stdout_fmb.build() { 812 | Ok(x) => x.matches(&stdout), 813 | Err(e) => { 814 | failure.stdout = Some(format!("FM error: {}", e)); 815 | break 'a; 816 | } 817 | }; 818 | 819 | // First, check whether the tests passed. 820 | let pass_status = match test.status { 821 | Status::Success => status.success(), 822 | Status::Error => { 823 | meant_to_error = true; 824 | !status.success() 825 | } 826 | Status::Signal => status.signal().is_some(), 827 | Status::Int(i) => status.code() == Some(i), 828 | }; 829 | 830 | // Second, if a test failed, we want to print out everything which didn't match 831 | // successfully (i.e. if the stderr test failed, print that out; but, equally, if 832 | // stderr wasn't specified as a test, print it out, because the user can't 833 | // otherwise know what it contains). 834 | if !(pass_status 835 | && stdin_remaining == 0 836 | && match_stderr.is_ok() 837 | && match_stdout.is_ok()) 838 | { 839 | if rerun <= inner.rerun_at_most { 840 | if let Some(rerun_if_status) = &test.rerun_if_status { 841 | let rerun = match rerun_if_status { 842 | Status::Success => status.success(), 843 | Status::Error => !status.success(), 844 | Status::Signal => status.signal().is_some(), 845 | Status::Int(i) => status.code() == Some(*i), 846 | }; 847 | if rerun { 848 | continue; 849 | } 850 | } 851 | if test.rerun_if_stderr.is_some() { 852 | match rerun_if_stderr_fmb.build() { 853 | Ok(x) if x.matches(&stderr).is_ok() => continue, 854 | Ok(_) => {} 855 | Err(e) => { 856 | failure.stderr = Some(format!("FM error: {}", e)); 857 | break 'a; 858 | } 859 | } 860 | } 861 | if test.rerun_if_stdout.is_some() { 862 | match rerun_if_stdout_fmb.build() { 863 | Ok(x) if x.matches(&stdout).is_ok() => continue, 864 | Ok(_) => {} 865 | Err(e) => { 866 | failure.stdout = Some(format!("FM error: {}", e)); 867 | break 'a; 868 | } 869 | } 870 | } 871 | } 872 | 873 | match test.status { 874 | Status::Success | Status::Error => { 875 | if status.success() { 876 | failure.status = Some("Success".to_owned()); 877 | } else if status.code().is_none() { 878 | failure.status = Some(format!( 879 | "Exited due to signal: {}", 880 | status.signal().unwrap() 881 | )); 882 | } else { 883 | failure.status = Some("Error".to_owned()); 884 | } 885 | } 886 | Status::Signal => { 887 | failure.status = Some("Exit was not due to signal".to_owned()); 888 | } 889 | Status::Int(_) => { 890 | failure.status = 891 | Some(status.code().map(|x| x.to_string()).unwrap_or_else(|| { 892 | format!("Exited due to signal: {}", status.signal().unwrap()) 893 | })) 894 | } 895 | } 896 | 897 | if match_stderr.is_err() || failure.stderr.is_none() { 898 | failure.stderr = Some(stderr); 899 | } 900 | if let Err(e) = match_stderr { 901 | failure.stderr_match = Some(e); 902 | } 903 | 904 | if match_stdout.is_err() || failure.stdout.is_none() { 905 | failure.stdout = Some(stdout); 906 | } 907 | if let Err(e) = match_stdout { 908 | failure.stdout_match = Some(e); 909 | } 910 | 911 | failure.stdin_remaining = stdin_remaining; 912 | 913 | // If a sub-test failed, bail out immediately, otherwise subsequent sub-tests 914 | // will overwrite the failure output! 915 | break 'a; 916 | } 917 | 918 | // If a command failed, and we weren't expecting it to, bail out immediately. 919 | if !status.success() && meant_to_error { 920 | break 'a; 921 | } 922 | break; 923 | } 924 | } 925 | 926 | { 927 | // Grab a lock on stderr so that we can avoid the possibility of lines blurring 928 | // together in confusing ways. 929 | let stderr = StandardStream::stderr(ColorChoice::Always); 930 | let mut handle = stderr.lock(); 931 | if inner.test_threads > 1 { 932 | handle 933 | .write_all(format!("\ntest lang_tests::{} ... ", test_fname).as_bytes()) 934 | .ok(); 935 | } 936 | match failure { 937 | TestFailure { 938 | status: None, 939 | stdin_remaining: 0, 940 | stderr: None, 941 | stderr_match: None, 942 | stdout: None, 943 | stdout_match: None, 944 | } => { 945 | handle 946 | .set_color(ColorSpec::new().set_fg(Some(Color::Green))) 947 | .ok(); 948 | handle.write_all(b"ok").ok(); 949 | } 950 | _ => { 951 | let mut failures = failures.lock().unwrap(); 952 | failures.push((test_fname, failure)); 953 | handle 954 | .set_color(ColorSpec::new().set_fg(Some(Color::Red))) 955 | .ok(); 956 | handle.write_all(b"FAILED").ok(); 957 | } 958 | } 959 | handle.reset().ok(); 960 | } 961 | 962 | false 963 | } 964 | 965 | fn run_cmd( 966 | inner: Arc, 967 | test_fname: &str, 968 | cmd: &mut Command, 969 | test: &TestCmd, 970 | ) -> (ExitStatus, usize, String, String) { 971 | // The basic sequence here is: 972 | // 1) Spawn the command 973 | // 2) Read everything from stderr & stdout until they are both disconnected 974 | // 3) wait() for the command to finish 975 | 976 | let mut child = cmd 977 | .stdin(process::Stdio::piped()) 978 | .stderr(process::Stdio::piped()) 979 | .stdout(process::Stdio::piped()) 980 | .spawn() 981 | .unwrap_or_else(|_| fatal(&format!("Couldn't run command {:?}.", cmd))); 982 | 983 | let mut stdin = child.stdin.take().unwrap(); 984 | let mut stderr = child.stderr.take().unwrap(); 985 | let mut stdout = child.stdout.take().unwrap(); 986 | 987 | let stdin_fd = stdin.as_raw_fd(); 988 | let stderr_fd = stderr.as_raw_fd(); 989 | let stdout_fd = stdout.as_raw_fd(); 990 | if let Err(e) = set_nonblock(stdin_fd) 991 | .and_then(|_| set_nonblock(stderr_fd)) 992 | .and_then(|_| set_nonblock(stdout_fd)) 993 | { 994 | fatal(&format!( 995 | "Couldn't set stdin and/or stderr and/or stdout to be non-blocking: {e:}" 996 | )); 997 | } 998 | 999 | const POLL_STDIN: usize = 0; 1000 | const POLL_STDERR: usize = 1; 1001 | const POLL_STDOUT: usize = 2; 1002 | 1003 | let mut cap_stderr = String::new(); 1004 | let mut cap_stdout = String::new(); 1005 | let mut stdin_off = 0; 1006 | let mut buf = [0; READBUF]; 1007 | let start = Instant::now(); 1008 | let mut last_warning = Instant::now(); 1009 | let mut next_warning = last_warning 1010 | .checked_add(Duration::from_secs(TIMEOUT)) 1011 | .unwrap(); 1012 | 1013 | // Has this file reached EOF and thus been closed? 1014 | const STATUS_EOF: u8 = 1; 1015 | // Has this file hit an error and thus been closed? Note that EOF and ERR 1016 | // are mutually exclusive. 1017 | const STATUS_ERR: u8 = 2; 1018 | 1019 | let mut statuses: [u8; 3] = [0, 0, 0]; 1020 | if test.stdin.is_none() { 1021 | unsafe { 1022 | close(stdin_fd); 1023 | } 1024 | statuses[POLL_STDIN] = STATUS_EOF; 1025 | } 1026 | loop { 1027 | // Are all files successfully closed? 1028 | if statuses[POLL_STDIN] == STATUS_EOF 1029 | && statuses[POLL_STDERR] == STATUS_EOF 1030 | && statuses[POLL_STDOUT] == STATUS_EOF 1031 | { 1032 | // If there's still stuff in the buffer to write out, we've failed. 1033 | if let Some(stdin_str) = &test.stdin { 1034 | if stdin_off < stdin_str.len() { 1035 | fatal(&format!("{} failed to consume all of stdin", test_fname)); 1036 | } 1037 | } 1038 | break; 1039 | } 1040 | 1041 | // Is at least one file in an error state and the other files are 1042 | // closed? 1043 | if statuses[POLL_STDIN] & (STATUS_EOF | STATUS_ERR) != 0 1044 | && statuses[POLL_STDERR] & (STATUS_EOF | STATUS_ERR) != 0 1045 | && statuses[POLL_STDOUT] & (STATUS_EOF | STATUS_ERR) != 0 1046 | { 1047 | fatal(&format!( 1048 | "{} has left one of stdin/stderr/stdout in an error condition", 1049 | test_fname 1050 | )); 1051 | } 1052 | 1053 | let mut pollfds = [ 1054 | pollfd { 1055 | fd: stdin_fd, 1056 | events: POLLOUT, 1057 | revents: 0, 1058 | }, 1059 | pollfd { 1060 | fd: stderr_fd, 1061 | events: POLLIN, 1062 | revents: 0, 1063 | }, 1064 | pollfd { 1065 | fd: stdout_fd, 1066 | events: POLLIN, 1067 | revents: 0, 1068 | }, 1069 | ]; 1070 | 1071 | if statuses[POLL_STDIN] & (STATUS_EOF | STATUS_ERR) != 0 { 1072 | // If the child process won't accept further input, there's no 1073 | // point polling it. 1074 | pollfds[POLL_STDIN].fd = -1; 1075 | } else if let Some(stdin_str) = &test.stdin { 1076 | if stdin_off == stdin_str.len() { 1077 | // There's nothing to write to the child's stdin, but we'd still 1078 | // like to check whether it is closed or has suffered an error, 1079 | // so we don't want to set the fd to -1. 1080 | pollfds[POLL_STDIN].events = 0; 1081 | } 1082 | } 1083 | 1084 | if statuses[POLL_STDERR] & (STATUS_EOF | STATUS_ERR) != 0 { 1085 | // If the child's stderr cannot produce further output, there's no 1086 | // point polling it. 1087 | pollfds[POLL_STDERR].fd = -1; 1088 | } 1089 | 1090 | if statuses[POLL_STDOUT] & (STATUS_EOF | STATUS_ERR) != 0 { 1091 | // If the child's stdout cannot produce further output, there's no 1092 | // point polling it. 1093 | pollfds[POLL_STDOUT].fd = -1; 1094 | } 1095 | 1096 | let timeout = i32::try_from( 1097 | next_warning 1098 | .checked_duration_since(Instant::now()) 1099 | .map(|d| d.as_millis()) 1100 | .unwrap_or(1000), 1101 | ) 1102 | .unwrap_or(1000); 1103 | if unsafe { poll((&mut pollfds) as *mut _ as *mut pollfd, 3, timeout) } != -1 { 1104 | assert_eq!(pollfds[POLL_STDIN].revents & POLLNVAL, 0); 1105 | if pollfds[POLL_STDIN].revents & POLLERR != 0 { 1106 | assert!(test.stdin.is_some()); 1107 | statuses[POLL_STDIN] = STATUS_ERR; 1108 | unsafe { 1109 | close(stdin_fd); 1110 | } 1111 | } else if pollfds[POLL_STDIN].revents & POLLOUT != 0 { 1112 | let stdin_str = test.stdin.as_ref().unwrap(); 1113 | match stdin.write(&stdin_str.as_bytes()[stdin_off..]) { 1114 | Ok(i) => stdin_off += i, 1115 | Err(e) => { 1116 | if e.kind() != io::ErrorKind::Interrupted { 1117 | unsafe { 1118 | close(stdin_fd); 1119 | } 1120 | statuses[POLL_STDIN] = STATUS_ERR; 1121 | } 1122 | } 1123 | } 1124 | debug_assert!(stdin_off <= stdin_str.len()); 1125 | if stdin_off == stdin_str.len() { 1126 | // We've fully written to the child's stdin. We close the child's stdin 1127 | // explicitly otherwise some child processes will hang, waiting for more input 1128 | // to be received. 1129 | unsafe { 1130 | close(stdin_fd); 1131 | } 1132 | statuses[POLL_STDIN] = STATUS_EOF; 1133 | } 1134 | } else if pollfds[POLL_STDIN].revents & POLLHUP != 0 { 1135 | // POSiX specifies that POLLOUT and POLLHUP are mutually exclusive. 1136 | unsafe { 1137 | close(stdin_fd); 1138 | } 1139 | statuses[POLL_STDIN] = STATUS_EOF; 1140 | } 1141 | 1142 | assert_eq!(pollfds[POLL_STDERR].revents & POLLNVAL, 0); 1143 | if pollfds[POLL_STDERR].revents & POLLERR != 0 { 1144 | unsafe { 1145 | close(stderr_fd); 1146 | } 1147 | statuses[POLL_STDERR] = STATUS_ERR; 1148 | } else { 1149 | if pollfds[POLL_STDERR].revents & POLLIN != 0 { 1150 | loop { 1151 | match stderr.read(&mut buf) { 1152 | Ok(i) => { 1153 | if i == 0 { 1154 | // We'll pick up POLLHUP on the next poll() 1155 | break; 1156 | } 1157 | let utf8 = str::from_utf8(&buf[..i]).unwrap_or_else(|_| { 1158 | fatal(&format!( 1159 | "Can't convert stderr from '{:?}' into UTF-8", 1160 | cmd 1161 | )) 1162 | }); 1163 | cap_stderr.push_str(utf8); 1164 | if inner.nocapture { 1165 | eprint!("{}", utf8); 1166 | } 1167 | } 1168 | Err(e) if e.kind() == io::ErrorKind::WouldBlock => break, 1169 | Err(e) if e.kind() == io::ErrorKind::Interrupted => (), 1170 | Err(e) => { 1171 | fatal(&format!("{}: failed to read stderr: {e:}", test_fname)) 1172 | } 1173 | } 1174 | } 1175 | } 1176 | if pollfds[POLL_STDERR].revents & POLLHUP != 0 { 1177 | // Note that POLLIN and POLLHUP are not mutually exclusive. 1178 | unsafe { 1179 | close(stderr_fd); 1180 | } 1181 | statuses[POLL_STDERR] = STATUS_EOF; 1182 | } 1183 | } 1184 | 1185 | assert_eq!(pollfds[POLL_STDOUT].revents & POLLNVAL, 0); 1186 | if pollfds[POLL_STDOUT].revents & POLLERR != 0 { 1187 | unsafe { 1188 | close(stdout_fd); 1189 | } 1190 | statuses[POLL_STDOUT] = STATUS_ERR; 1191 | } else { 1192 | if pollfds[POLL_STDOUT].revents & POLLIN != 0 { 1193 | loop { 1194 | match stdout.read(&mut buf) { 1195 | Ok(i) => { 1196 | if i == 0 { 1197 | // We'll pick up POLLHUP on the next poll() 1198 | break; 1199 | } 1200 | let utf8 = str::from_utf8(&buf[..i]).unwrap_or_else(|_| { 1201 | fatal(&format!( 1202 | "Can't convert stdout from '{:?}' into UTF-8", 1203 | cmd 1204 | )) 1205 | }); 1206 | cap_stdout.push_str(utf8); 1207 | if inner.nocapture { 1208 | eprint!("{}", utf8); 1209 | } 1210 | } 1211 | Err(e) if e.kind() == io::ErrorKind::WouldBlock => break, 1212 | Err(e) if e.kind() == io::ErrorKind::Interrupted => (), 1213 | Err(e) => { 1214 | fatal(&format!("{}: failed to read stdout: {e:}", test_fname)) 1215 | } 1216 | } 1217 | } 1218 | } 1219 | if pollfds[POLL_STDOUT].revents & POLLHUP != 0 { 1220 | // Note that POLLIN and POLLHUP are not mutually exclusive. 1221 | if statuses[POLL_STDOUT] & STATUS_EOF == 0 { 1222 | unsafe { 1223 | close(stdout_fd); 1224 | } 1225 | statuses[POLL_STDOUT] = STATUS_EOF; 1226 | } 1227 | } 1228 | } 1229 | } 1230 | 1231 | if Instant::now() >= next_warning { 1232 | let running_for = ((Instant::now() - start).as_secs() / TIMEOUT) * TIMEOUT; 1233 | if inner.test_threads == 1 { 1234 | eprint!("running for over {} seconds... ", running_for); 1235 | } else { 1236 | eprintln!( 1237 | "\nlang_tests::{} ... has been running for over {} seconds", 1238 | test_fname, running_for 1239 | ); 1240 | } 1241 | last_warning = next_warning; 1242 | next_warning = last_warning 1243 | .checked_add(Duration::from_secs(TIMEOUT)) 1244 | .unwrap(); 1245 | } 1246 | } 1247 | if statuses[POLL_STDIN] != 0 { 1248 | std::mem::forget(stdin); 1249 | } 1250 | if statuses[POLL_STDERR] != 0 { 1251 | std::mem::forget(stderr); 1252 | } 1253 | if statuses[POLL_STDOUT] != 0 { 1254 | std::mem::forget(stdout); 1255 | } 1256 | 1257 | let status = { 1258 | // We have no idea how long it will take the child process to exit. In practise, the mere 1259 | // act of yielding (via sleep) for a ridiculously short period of time will often be enough 1260 | // for the child process to exit. So we use an exponentially increasing timeout with a very 1261 | // short initial period so that, in the common case, we don't waste time waiting for 1262 | // something that's almost certainly already occurred. 1263 | let mut wait_timeout = INITIAL_WAIT_TIMEOUT; 1264 | loop { 1265 | match child.try_wait() { 1266 | Ok(Some(s)) => break s, 1267 | Ok(None) => (), 1268 | Err(e) => fatal(&format!("{:?} did not exit correctly: {:?}", cmd, e)), 1269 | } 1270 | 1271 | if Instant::now() >= next_warning { 1272 | let running_for = ((Instant::now() - start).as_secs() / TIMEOUT) * TIMEOUT; 1273 | if inner.test_threads == 1 { 1274 | eprint!("running for over {} seconds... ", running_for); 1275 | } else { 1276 | eprintln!( 1277 | "\nlang_tests::{} ... has been running for over {} seconds", 1278 | test_fname, running_for 1279 | ); 1280 | } 1281 | last_warning = next_warning; 1282 | next_warning = last_warning 1283 | .checked_add(Duration::from_secs(TIMEOUT)) 1284 | .unwrap(); 1285 | } 1286 | sleep(Duration::from_nanos(wait_timeout)); 1287 | wait_timeout *= 2; 1288 | if wait_timeout > MAX_WAIT_TIMEOUT { 1289 | wait_timeout = MAX_WAIT_TIMEOUT; 1290 | } 1291 | } 1292 | }; 1293 | 1294 | let stdin_remaining = if let Some(stdin_str) = &test.stdin { 1295 | stdin_str.len() - stdin_off 1296 | } else { 1297 | 0 1298 | }; 1299 | (status, stdin_remaining, cap_stderr, cap_stdout) 1300 | } 1301 | 1302 | fn set_nonblock(fd: c_int) -> Result<(), io::Error> { 1303 | let flags = unsafe { fcntl(fd, F_GETFL) }; 1304 | if flags == -1 || unsafe { fcntl(fd, F_SETFL, flags | O_NONBLOCK) } == -1 { 1305 | return Err(io::Error::last_os_error()); 1306 | } 1307 | 1308 | Ok(()) 1309 | } 1310 | --------------------------------------------------------------------------------