├── .gitignore ├── .travis.yml ├── Cargo.toml ├── LICENSE ├── README.md ├── examples ├── barebones.rs └── cli.rs ├── integration-tests ├── basic_regex.sh ├── check-next-simple.txt ├── constants │ ├── file.txt │ └── tempfile.txt ├── multiple-adjacent-checks.sh ├── multiple-checks-possibly-confusing-order.sh ├── multiple-checks.sh ├── named-regex.sh ├── single-check.sh └── xfail-expected-failure.sh ├── src ├── config.rs ├── config │ └── clap.rs ├── errors.rs ├── event_handler.rs ├── event_handler │ └── default.rs ├── lib.rs ├── main.rs ├── model.rs ├── parse.rs ├── run │ ├── find_files.rs │ ├── legacy_test_evaluator.rs │ ├── mod.rs │ ├── test_evaluator.rs │ └── test_evaluator │ │ ├── state.rs │ │ └── state_tests.rs ├── util.rs └── vars │ ├── mod.rs │ └── resolve.rs └── tests └── integration_tests.rs /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: rust 2 | 3 | rust: 4 | - stable 5 | - nightly 6 | - beta 7 | 8 | script: 9 | - cargo test --all 10 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "lit" 3 | version = "1.0.4" 4 | authors = ["Dylan McKay "] 5 | edition = "2018" 6 | 7 | description = "Integrated testing tool, inspired by LLVM's 'lit' testing script" 8 | documentation = "https://docs.rs/lit" 9 | repository = "https://github.com/dylanmckay/lit" 10 | 11 | license = "MIT" 12 | 13 | keywords = ["testing"] 14 | 15 | [features] 16 | default = ["clap"] 17 | 18 | [dependencies] 19 | clap = { version = "2.33", optional = true } 20 | error-chain = "0.12" 21 | itertools = "0.9" 22 | lazy_static = "1.4" 23 | log = "0.4" 24 | regex = "1.3" 25 | tempfile = "3.1" 26 | term = "0.6" 27 | walkdir = "2.3" 28 | 29 | [dev-dependencies] 30 | pretty_env_logger = "0.4" 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright (c) 2016 Dylan McKay 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 5 | 6 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 7 | 8 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # lit 2 | 3 | [![Crates.io](https://img.shields.io/crates/v/lit.svg)](https://crates.io/crates/lit) 4 | [![Build Status](https://travis-ci.org/dylanmckay/lit.svg?branch=master)](https://travis-ci.org/dylanmckay/lit) 5 | [![license](https://img.shields.io/github/license/dylanmckay/lit.svg)]() 6 | 7 | An integrated testing tool, similar to LLVM's integrated testing tool (`llvm-lit`) but in library form. 8 | 9 | [Rust API Documentation](https://docs.rs/lit) 10 | 11 | ## Usage 12 | 13 | Point `lit` at a directory containing test files and it will execute the commands and run the checks 14 | contained within the tests. 15 | 16 | Any plain-text based file format that supports comments can be used as a test file, provided the comment 17 | character has been added to lit's source code. 18 | 19 | All testing is done based on text comparisons with the output of various command line tools executed 20 | on each test file. The `CHECK` directives inside each test file validate that the command line tool 21 | contains the expected text. 22 | 23 | ### Testing a bash script 24 | 25 | Here is an example test file, it is a bash script. Assertions are added 26 | that ensure the bash script outputs the correct text to stdout/stderr. 27 | ```bash 28 | # RUN: sh -ea @file 29 | 30 | # CHECK: hello world 31 | echo hello world 32 | 33 | # CHECK: number 1 34 | # CHECK: number 2 35 | # CHECK: number 3 36 | # CHECK: number 4 37 | for i in $(seq 1 4); do echo number $i; done 38 | ``` 39 | 40 | ### Testing a C/C++ program 41 | 42 | Here is an example C/C++ test file. Assertions are added 43 | that ensure the C compiler name mangles the main method as a particular way. 44 | ```c 45 | // RUN: gcc @file -S 46 | // 47 | // Note: '-S' compiles this C program to plain-text assembly. 48 | 49 | // This next line verifies that there is an assembly label 50 | // with a name mangled underscore prefix. 51 | // CHECK: _main: 52 | int main() { 53 | return 0; 54 | } 55 | ``` 56 | 57 | ### The `RUN` directive 58 | 59 | This directive runs an executable, almost always operating on the test file as the source file. 60 | 61 | ``` 62 | RUN: 63 | ``` 64 | 65 | Each `RUN` directive runs the same test file in different conditions. 66 | 67 | ### The `CHECK` directive 68 | 69 | This directive is used to assert that the output of the `RUN` command 70 | contains a specific string. 71 | 72 | ``` 73 | CHECK: 74 | ``` 75 | 76 | If the substring is not found, then the test immediately fails. 77 | 78 | ## Variables 79 | 80 | Variables can be used in directives by `@`. The variable is substituted in-place with 81 | the value of the variable at the time of the test. 82 | 83 | ## Default variables available to tests 84 | 85 | These variables can be used by tests in directives. 86 | 87 | | Name (`*` = wildcard) | Description | Substituted value | 88 | |-------------------------|--------------|---------------------------------------------| 89 | | `@file` | | The path the the test file being executed. | 90 | | `@*tempfile*` | Any variable containing the text `tempfile` | A temporary file path. Subsequent uses of the same tempfile variable will give the same path. It is possible to use multiple tempfiles in one test by giving them separate names, like `@first_tempfile` and `@second_tempfile` | 91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /examples/barebones.rs: -------------------------------------------------------------------------------- 1 | extern crate lit; 2 | 3 | use std::env::consts; 4 | 5 | fn main() { 6 | lit::run::tests(lit::event_handler::Default::default(), |config| { 7 | config.add_search_path("test/"); 8 | config.add_extension("cpp"); 9 | 10 | config.constants.insert("arch".to_owned(), consts::ARCH.to_owned()); 11 | config.constants.insert("os".to_owned(), consts::OS.to_owned()); 12 | }).unwrap() 13 | } 14 | -------------------------------------------------------------------------------- /examples/cli.rs: -------------------------------------------------------------------------------- 1 | extern crate lit; 2 | extern crate clap; 3 | 4 | use clap::{App, Arg}; 5 | use std::env::consts; 6 | 7 | fn main() { 8 | let app = App::new("Example of a testing tool CLI frontend") 9 | .version(env!("CARGO_PKG_VERSION")) 10 | .author(env!("CARGO_PKG_AUTHORS")) 11 | .about(env!("CARGO_PKG_DESCRIPTION")) 12 | .arg(Arg::with_name("v") 13 | .short("v") 14 | .multiple(true) 15 | .help("Sets the level of verbosity")); 16 | 17 | let app = lit::config::clap::mount_inside_app(app, true); 18 | 19 | let matches = app.get_matches(); 20 | 21 | println!("Verbose: {}", matches.is_present("v")); 22 | 23 | lit::run::tests(lit::event_handler::Default::default(), |config| { 24 | config.add_search_path("integration-tests/"); 25 | config.add_extension("txt"); 26 | 27 | config.constants.insert("arch".to_owned(), consts::ARCH.to_owned()); 28 | config.constants.insert("os".to_owned(), consts::OS.to_owned()); 29 | 30 | lit::config::clap::parse_arguments(&matches, config); 31 | }).unwrap() 32 | } 33 | -------------------------------------------------------------------------------- /integration-tests/basic_regex.sh: -------------------------------------------------------------------------------- 1 | # RUN: sh @file 2 | 3 | # CHECK: ldi r[[\d+]], [[\d+]] 4 | echo ldi r2, 1 5 | 6 | -------------------------------------------------------------------------------- /integration-tests/check-next-simple.txt: -------------------------------------------------------------------------------- 1 | ; RUN: cat @file 2 | 3 | ; Prints the test itself, to verify strings within itself. 4 | 5 | ; CHECK: hello 6 | ; CHECK: world 7 | 8 | 9 | -------------------------------------------------------------------------------- /integration-tests/constants/file.txt: -------------------------------------------------------------------------------- 1 | ; RUN: cat @file 2 | 3 | foo 4 | 5 | -------------------------------------------------------------------------------- /integration-tests/constants/tempfile.txt: -------------------------------------------------------------------------------- 1 | ; RUN: echo "hello world once" > @first_tempfile && cat @first_tempfile 2 | ; RUN: echo "hello world twice" > @second_tempfile && cat @second_tempfile 3 | ; RUN: echo "hello world thrice" > @tempfiles_for_days && cat @tempfiles_for_days 4 | 5 | ; CHECK: hello world 6 | 7 | -------------------------------------------------------------------------------- /integration-tests/multiple-adjacent-checks.sh: -------------------------------------------------------------------------------- 1 | # RUN: cat @file 2 | 3 | # CHECK: hello world 4 | echo hello world 5 | # CHECK: this is me 6 | echo this is me 7 | 8 | # CHECK: life should be 9 | echo life should be 10 | # CHECK: fun for everyone 11 | echo fun for everyone 12 | 13 | 14 | -------------------------------------------------------------------------------- /integration-tests/multiple-checks-possibly-confusing-order.sh: -------------------------------------------------------------------------------- 1 | # RUN: cat @file 2 | 3 | echo but this text here is good 4 | 5 | # CHECK: hello there 6 | echo hello there 7 | 8 | # CHECK: but this text here is good 9 | echo but this text here is good 10 | 11 | -------------------------------------------------------------------------------- /integration-tests/multiple-checks.sh: -------------------------------------------------------------------------------- 1 | # RUN: sh @file 2 | 3 | # CHECK: fizz 1 4 | # CHECK: fizz 2 5 | # CHECK: fizz 100 6 | 7 | echo "warning: something is pretty tetchy, bro" 1>&2 8 | echo "warning: ah nah nevermind" 1>&2 9 | 10 | for i in $(seq 1 100); do 11 | echo fizz $i 12 | echo 13 | done 14 | -------------------------------------------------------------------------------- /integration-tests/named-regex.sh: -------------------------------------------------------------------------------- 1 | # RUN: sh @file 2 | 3 | # CHECK: hello [[name:\w+]] 4 | echo hello bob 5 | 6 | # CHECK: goodbye $$name 7 | echo goodbye bob 8 | 9 | 10 | -------------------------------------------------------------------------------- /integration-tests/single-check.sh: -------------------------------------------------------------------------------- 1 | # RUN: cat @file 2 | 3 | # CHECK: hello 4 | echo hello 5 | 6 | 7 | -------------------------------------------------------------------------------- /integration-tests/xfail-expected-failure.sh: -------------------------------------------------------------------------------- 1 | # RUN: sh @file 2 | # XFAIL: 3 | 4 | # CHECK: hello world 5 | echo hello cruel world 6 | 7 | -------------------------------------------------------------------------------- /src/config.rs: -------------------------------------------------------------------------------- 1 | //! The type that stores testing configuration. 2 | //! 3 | //! Use the code in this module to tune testing behaviour. 4 | 5 | #[cfg(feature = "clap")] pub mod clap; 6 | 7 | use std::path::{Path, PathBuf}; 8 | use std::collections::HashMap; 9 | use std::fmt; 10 | use tempfile::NamedTempFile; 11 | 12 | const DEFAULT_MAX_OUTPUT_CONTEXT_LINE_COUNT: usize = 10; 13 | 14 | /// The configuration of the test runner. 15 | #[derive(Clone, Debug)] 16 | pub struct Config 17 | { 18 | /// A list of file extensions which contain tests. 19 | pub supported_file_extensions: Vec, 20 | /// Paths to tests or folders containing tests. 21 | pub test_paths: Vec, 22 | /// Constants that tests can refer to via `@` syntax. 23 | pub constants: HashMap, 24 | /// A function which used to dynamically lookup variables. 25 | /// 26 | /// The default variable lookup can be found at `Config::DEFAULT_VARIABLE_LOOKUP`. 27 | /// 28 | /// In your own custom variable lookups, most of the time you will want to 29 | /// include a fallback call to `Config::DEFAULT_VARIABLE_LOOKUP`. 30 | pub variable_lookup: VariableLookup, 31 | /// Whether temporary files generated by the tests should be 32 | /// cleaned up, where possible. 33 | /// 34 | /// This includes temporary files created by using `@tempfile` 35 | /// variables. 36 | pub cleanup_temporary_files: bool, 37 | /// Export all generated test artifacts to the specified directory. 38 | pub save_artifacts_to_directory: Option, 39 | /// Whether verbose information about resolved variables should be printed to stderr. 40 | pub dump_variable_resolution: bool, 41 | /// If set, debug output should be truncated to this many number of 42 | /// context lines. 43 | pub truncate_output_context_to_number_of_lines: Option, 44 | /// A list of extra directory paths that should be included in the `$PATH` when 45 | /// executing processes specified inside the tests. 46 | pub extra_executable_search_paths: Vec, 47 | /// Whether messages on the standard error streams emitted during test runs 48 | /// should always be shown. 49 | pub always_show_stderr: bool, 50 | /// Which shell to use (defaults to 'bash'). 51 | pub shell: String, 52 | } 53 | 54 | /// A function which can dynamically define newly used variables in a test. 55 | #[derive(Clone)] 56 | pub struct VariableLookup(fn(&str) -> Option); 57 | 58 | impl Config 59 | { 60 | /// The default variable lookup function. 61 | /// 62 | /// The supported variables are: 63 | /// 64 | /// * Any variable containing the string `"tempfile"` 65 | /// * Each distinct variable will be resolved to a distinct temporary file path. 66 | pub const DEFAULT_VARIABLE_LOOKUP: VariableLookup = VariableLookup(|v| { 67 | if v.contains("tempfile") { 68 | let temp_file = NamedTempFile::new().expect("failed to create a temporary file"); 69 | Some(temp_file.into_temp_path().to_str().expect("temp file path is not utf-8").to_owned()) 70 | } else { 71 | None 72 | } 73 | }); 74 | 75 | /// Marks a file extension as supported by the runner. 76 | /// 77 | /// We only attempt to run tests for files within the extension 78 | /// whitelist. 79 | pub fn add_extension(&mut self, ext: S) where S: AsRef { 80 | self.supported_file_extensions.push(ext.as_ref().to_owned()) 81 | } 82 | 83 | /// Marks multiple file extensions as supported by the running. 84 | pub fn add_extensions(&mut self, extensions: &[&str]) { 85 | self.supported_file_extensions.extend(extensions.iter().map(|s| s.to_string())); 86 | } 87 | 88 | /// Adds a search path to the test runner. 89 | /// 90 | /// We will recurse through the path to find tests. 91 | pub fn add_search_path

(&mut self, path: P) where P: Into { 92 | self.test_paths.push(PathBuf::from(path.into()).canonicalize().unwrap()); 93 | } 94 | 95 | /// Adds an extra executable directory to the OS `$PATH` when executing tests. 96 | pub fn add_executable_search_path

(&mut self, path: P) where P: AsRef { 97 | self.extra_executable_search_paths.push(path.as_ref().to_owned()) 98 | } 99 | 100 | /// Gets an iterator over all test search directories. 101 | pub fn test_search_directories(&self) -> impl Iterator { 102 | self.test_paths.iter().filter(|p| { 103 | println!("test path file name: {:?}", p.file_name()); 104 | p.is_dir() 105 | }).map(PathBuf::as_ref) 106 | } 107 | 108 | /// Checks if a given extension will have tests run on it 109 | pub fn is_extension_supported(&self, extension: &str) -> bool { 110 | self.supported_file_extensions.iter(). 111 | find(|ext| &ext[..] == extension).is_some() 112 | } 113 | 114 | /// Looks up a variable. 115 | pub fn lookup_variable<'a>(&self, 116 | name: &str, 117 | variables: &'a mut HashMap) 118 | -> &'a str { 119 | if !variables.contains_key(name) { 120 | match self.variable_lookup.0(name) { 121 | Some(initial_value) => { 122 | variables.insert(name.to_owned(), initial_value.clone()); 123 | }, 124 | None => (), 125 | } 126 | } 127 | 128 | variables.get(name).expect(&format!("no variable with the name '{}' exists", name)) 129 | } 130 | } 131 | 132 | impl Default for Config 133 | { 134 | fn default() -> Self { 135 | let mut extra_executable_search_paths = Vec::new(); 136 | 137 | // Always inject the current directory of the executable into the PATH so 138 | // that lit can be used manually inside the test if desired. 139 | if let Ok(current_exe) = std::env::current_exe() { 140 | if let Some(parent) = current_exe.parent() { 141 | extra_executable_search_paths.push(parent.to_owned()); 142 | } 143 | } 144 | 145 | Config { 146 | supported_file_extensions: Vec::new(), 147 | test_paths: Vec::new(), 148 | constants: HashMap::new(), 149 | variable_lookup: Config::DEFAULT_VARIABLE_LOOKUP, 150 | cleanup_temporary_files: true, 151 | save_artifacts_to_directory: None, 152 | dump_variable_resolution: false, 153 | always_show_stderr: false, 154 | truncate_output_context_to_number_of_lines: Some(DEFAULT_MAX_OUTPUT_CONTEXT_LINE_COUNT), 155 | extra_executable_search_paths, 156 | shell: "bash".to_string(), 157 | } 158 | } 159 | } 160 | 161 | impl fmt::Debug for VariableLookup { 162 | fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { 163 | "".fmt(fmt) 164 | } 165 | } 166 | 167 | #[cfg(test)] 168 | mod test { 169 | use super::*; 170 | 171 | #[test] 172 | fn lookup_variable_works_correctly() { 173 | let config = Config { 174 | variable_lookup: VariableLookup(|v| { 175 | if v.contains("tempfile") { Some(format!("/tmp/temp-{}", v.as_bytes().as_ptr() as usize)) } else { None } 176 | }), 177 | constants: vec![("name".to_owned(), "bob".to_owned())].into_iter().collect(), 178 | ..Config::default() 179 | }; 180 | let mut variables = config.constants.clone(); 181 | 182 | // Can lookup constants 183 | assert_eq!("bob", config.lookup_variable("name", &mut variables), 184 | "cannot lookup constants by name"); 185 | let first_temp = config.lookup_variable("first_tempfile", &mut variables).to_owned(); 186 | let second_temp = config.lookup_variable("second_tempfile", &mut variables).to_owned(); 187 | 188 | assert!(first_temp != second_temp, 189 | "different temporary paths should be different"); 190 | 191 | assert_eq!(first_temp, 192 | config.lookup_variable("first_tempfile", &mut variables), 193 | "first temp has changed its value"); 194 | 195 | assert_eq!(second_temp, 196 | config.lookup_variable("second_tempfile", &mut variables), 197 | "second temp has changed its value"); 198 | } 199 | } 200 | 201 | -------------------------------------------------------------------------------- /src/config/clap.rs: -------------------------------------------------------------------------------- 1 | //! Routines for exposing a command line interface via the `clap` crate. 2 | //! 3 | //! These routines can be used to update `Config` objects with automatic CLI arguments. 4 | 5 | use crate::Config; 6 | use clap::{App, Arg, ArgMatches, SubCommand}; 7 | use std::{io::Write, path::Path}; 8 | 9 | /// The set of available debug parameters. 10 | const DEBUG_OPTION_VALUES: &'static [(&'static str, fn(&mut Config))] = &[ 11 | ("variable-resolution", |config: &mut Config| { 12 | config.dump_variable_resolution = true; 13 | }), 14 | ]; 15 | 16 | const SHOW_OPTION_VALUES: &'static [(&'static str, fn(&Config, &mut dyn Write) -> std::io::Result<()>)] = &[ 17 | ("test-file-paths", |config, writer| { 18 | let test_file_paths = crate::run::find_files::with_config(config).unwrap(); 19 | for test_file_path in test_file_paths { 20 | writeln!(writer, "{}", test_file_path.absolute.display())?; 21 | } 22 | 23 | Ok(()) 24 | 25 | }), 26 | ("lit-config", |config, writer| { 27 | writeln!(writer, "{:#?}", config) 28 | }), 29 | ]; 30 | 31 | const MULTIPLY_TRUNCATION_LINES_BY_THIS_AT_EACH_VERBOSITY_LEVEL: usize = 4; 32 | 33 | lazy_static! { 34 | static ref DEBUG_OPTION_HELP: String = { 35 | let debug_option_vals = DEBUG_OPTION_VALUES.iter().map(|d| d.0).collect::>(); 36 | let debug_option_vals = debug_option_vals.join(", "); 37 | 38 | format!("Enable debug output. Possible debugging flags are: {}.", debug_option_vals) 39 | }; 40 | 41 | static ref SHOW_SUBCOMMAND_WHAT_OPTION_HELP: String = { 42 | let show_option_vals = SHOW_OPTION_VALUES.iter().map(|d| format!(" - {}", d.0)).collect::>(); 43 | let show_option_vals = show_option_vals.join("\n"); 44 | 45 | format!("Show only a specific value. Possible values are:\n{}\nIf this value is not specified, all values are shown", show_option_vals) 46 | }; 47 | } 48 | 49 | 50 | /// Mounts extra arguments that can be used to fine-tune testing 51 | /// into a `clap` CLI application. 52 | pub fn mount_inside_app<'a, 'b>( 53 | app: App<'a, 'b>, 54 | test_paths_as_positional_arguments: bool, 55 | ) -> App<'a, 'b> { 56 | let app = app 57 | .arg(Arg::with_name("supported-file-extension") 58 | .long("add-file-extension") 59 | .takes_value(true) 60 | .value_name("EXT") 61 | .multiple(true) 62 | .help("Adds a file extension to the test search list. Extensions can be specified either with or without a leading period")) 63 | .arg(Arg::with_name("constant") 64 | .long("define-constant") 65 | .short("c") 66 | .takes_value(true) 67 | .value_name("NAME>==' 68 | .multiple(true) 69 | .help("Sets a constant, accessible in the test via '@")) 70 | .arg(Arg::with_name("show-context-lines") 71 | .long("show-context-lines") 72 | .short("C") 73 | .takes_value(true) 74 | .value_name("NUMBER OF CONTEXT LINES") 75 | .help("Sets the number of output lines to be displayed when showing failure context. Set to '-1' to disable truncation.")) 76 | .arg(Arg::with_name("always-show-stderr") 77 | .long("always-show-stderr") 78 | .help("Always echo the stderr streams emitted by programs under test. By default this is only done if the program exits with an error code. Stderr is also always printed when verbose mode is on.")) 79 | .arg(Arg::with_name("keep-tempfiles") 80 | .long("keep-tempfiles") 81 | .help("Disables automatic deletion of tempfiles generated during the test run")) 82 | .arg(Arg::with_name("save-artifacts-to") 83 | .long("save-artifacts-to") 84 | .short("O") 85 | .takes_value(true) 86 | .value_name("DIRECTORY") 87 | .help("Exports all program outputs, temporary files, and logs, to a directory at the specified path. Will create the directory if it does not yet exist.")) 88 | .arg(Arg::with_name("verbose") 89 | .long("verbose") 90 | .short("v") 91 | .multiple(true) 92 | .help("Increase the level of verbosity in the output. Pass '-vv' for maximum verbosity")) 93 | .arg(Arg::with_name("debug-all") 94 | .long("debug-all") 95 | .short("g") 96 | .help("Turn on all debugging flags")) 97 | .arg(Arg::with_name("debug") 98 | .long("debug") 99 | .takes_value(true) 100 | .value_name("FLAG") 101 | .multiple(true) 102 | .help(&DEBUG_OPTION_HELP[..])) 103 | .subcommand(SubCommand::with_name("show") 104 | .about("Shows information about the test suite, without running tests") 105 | .arg(Arg::with_name("what") 106 | .takes_value(true) 107 | .value_name("WHAT") 108 | .help(&SHOW_SUBCOMMAND_WHAT_OPTION_HELP))); 109 | 110 | // Test paths argument 111 | let test_paths_arg = { 112 | let mut arg = Arg::with_name("add-tests") 113 | // .long("add-tests") 114 | .takes_value(true) 115 | .value_name("PATH TO TEST OR TESTS") 116 | .multiple(true) 117 | .help("Adds a path to the test search pathset. If the path refers to a directory, it will be recursed, if it refers to a file, it will be treated as a test file"); 118 | 119 | // If positional arguments are disabled, add this as a longhand option anyway. 120 | if !test_paths_as_positional_arguments { 121 | arg = arg.long("add-tests"); 122 | } 123 | 124 | arg 125 | }; 126 | 127 | let app = app 128 | .arg(test_paths_arg); 129 | 130 | app 131 | } 132 | 133 | /// Parses command line arguments from `clap` into a destination `Config` object. 134 | pub fn parse_arguments(matches: &ArgMatches, 135 | destination_config: &mut Config) { 136 | if let Some(extensions) = matches.values_of("supported-file-extension") { 137 | for extension in extensions { 138 | destination_config.add_extension(extension); 139 | } 140 | } 141 | 142 | if let Some(test_paths) = matches.values_of("add-tests") { 143 | for test_path in test_paths { 144 | destination_config.add_search_path(test_path); 145 | } 146 | } 147 | 148 | if let Some(constant_define_strs) = matches.values_of("constant") { 149 | for constant_define_str in constant_define_strs { 150 | let constant_definition: ConstantDefinition = match constant_define_str.parse() { 151 | Ok(c) => c, 152 | Err(e) => panic!("could not parse constant definition: {}", e), 153 | }; 154 | 155 | destination_config.constants.insert(constant_definition.name, constant_definition.value); 156 | } 157 | } 158 | 159 | if matches.is_present("keep-tempfiles") { 160 | destination_config.cleanup_temporary_files = false; 161 | } 162 | 163 | if let Some(artifacts_path) = matches.value_of("save-artifacts-to") { 164 | destination_config.save_artifacts_to_directory = Some(Path::new(artifacts_path).to_owned()); 165 | } 166 | 167 | // Parse verbosity. 168 | { 169 | let verbosity_level = matches.occurrences_of("verbose"); 170 | 171 | if verbosity_level > 2 { 172 | warning(format!("the current verbosity level of '{}' specified is redundant, the maximum verbosity is '-vv' (corresponding to verbosity level 2)", verbosity_level)); 173 | } 174 | 175 | if verbosity_level > 0 { 176 | if let Some(truncation) = destination_config.truncate_output_context_to_number_of_lines { 177 | destination_config.truncate_output_context_to_number_of_lines = Some(truncation * MULTIPLY_TRUNCATION_LINES_BY_THIS_AT_EACH_VERBOSITY_LEVEL * (verbosity_level as usize)); 178 | } 179 | 180 | if verbosity_level >= 1 { 181 | destination_config.always_show_stderr = true; 182 | } 183 | 184 | if verbosity_level >= 2 { 185 | destination_config.dump_variable_resolution = true; 186 | } 187 | } 188 | } 189 | 190 | if matches.is_present("always-show-stderr") { 191 | destination_config.always_show_stderr = true; 192 | } 193 | 194 | if let Some(debug_flags) = matches.values_of("debug") { 195 | for debug_flag in debug_flags { 196 | let apply_fn = DEBUG_OPTION_VALUES.iter().find(|(k, _)| k == &debug_flag.trim()).map(|d| d.1); 197 | 198 | match apply_fn { 199 | Some(func) => func(destination_config), 200 | None => panic!("no debugging flag named '{}'", debug_flag), 201 | } 202 | } 203 | } 204 | 205 | if matches.is_present("debug-all") { 206 | for (_, debug_flag_fn) in DEBUG_OPTION_VALUES { 207 | debug_flag_fn(destination_config); 208 | } 209 | } 210 | 211 | if let Some(cli_show_context_lines) = matches.value_of("show-context-lines") { 212 | match cli_show_context_lines.parse::() { 213 | Ok(-1) => { 214 | destination_config.truncate_output_context_to_number_of_lines = None; 215 | }, 216 | Ok(lines) if lines < 0 => fatal_error(format!("invalid number of context lines: '{}' - must be a positive integer, or '-1' to disable truncation", cli_show_context_lines)), 217 | Ok(lines) => { 218 | destination_config.truncate_output_context_to_number_of_lines = Some(lines as usize); 219 | }, 220 | Err(_) => fatal_error(format!("invalid number of context lines: '{}' - must be a positive integer, or '-1' to disable truncation", cli_show_context_lines)), 221 | } 222 | } 223 | 224 | // NOTE: should process subcommands at the very end 225 | if let Some(matches) = matches.subcommand_matches("show") { 226 | let what_fns: Vec<_> = match matches.value_of("what") { 227 | Some(what) => { 228 | match SHOW_OPTION_VALUES.iter().find(|(name, _)| *name == what) { 229 | Some((name, what_fn)) => vec![(name, what_fn)], 230 | None => { 231 | fatal_error(format!("error: unknown show value: '{}'", what)); 232 | }, 233 | } 234 | }, 235 | None => { 236 | SHOW_OPTION_VALUES.iter().map(|(name, f)| (name, f)).collect() 237 | }, 238 | }; 239 | 240 | let writer = &mut std::io::stdout(); 241 | 242 | let show_labels = what_fns.len() > 1; 243 | for (label, what_fn) in what_fns { 244 | if show_labels { 245 | writeln!(writer, "=================================================================").unwrap(); 246 | writeln!(writer, "{}:", label).unwrap(); 247 | writeln!(writer, "=================================================================").unwrap(); 248 | writeln!(writer, "").unwrap(); 249 | } 250 | 251 | what_fn(&destination_config, writer).unwrap(); 252 | 253 | if show_labels { 254 | writeln!(writer, "").unwrap(); 255 | } 256 | } 257 | 258 | // No tests should be ran when running this subcommand. 259 | std::process::exit(0); 260 | } 261 | } 262 | 263 | #[derive(Clone, Debug, PartialEq, Eq)] 264 | struct ConstantDefinition { 265 | pub name: String, 266 | pub value: String, 267 | } 268 | 269 | impl std::str::FromStr for ConstantDefinition { 270 | type Err = String; 271 | 272 | fn from_str(s: &str) -> Result { 273 | if s.chars().filter(|&c| c == '=').count() != 1 { 274 | return Err(format!("constant definition must have exactly one equals sign but got '{}", s)) 275 | } 276 | if s.len() < 3 { 277 | return Err(format!("constant definitions must include both a and a , separated by equals")); 278 | } 279 | 280 | let (name, value) = s.split_at(s.find('=').unwrap()); 281 | let value = &value[1..]; // trim equals 282 | let (name, value) = (name.trim().to_owned(), value.trim().to_owned()); 283 | 284 | Ok(ConstantDefinition { name, value }) 285 | } 286 | } 287 | 288 | fn fatal_error(msg: impl AsRef) -> ! { 289 | eprintln!("error: {}", msg.as_ref()); 290 | std::process::exit(1); 291 | } 292 | 293 | fn warning(msg: impl AsRef) { 294 | eprintln!("warning: {}", msg.as_ref()); 295 | } 296 | -------------------------------------------------------------------------------- /src/errors.rs: -------------------------------------------------------------------------------- 1 | error_chain! { 2 | types { 3 | Error, ErrorKind, ResultExt; 4 | } 5 | 6 | foreign_links { 7 | Io(::std::io::Error); 8 | } 9 | 10 | errors { 11 | InvalidToolchainName(t: String) { 12 | description("invalid toolchain name") 13 | display("invalid toolchain name: '{}'", t) 14 | } 15 | } 16 | } 17 | 18 | -------------------------------------------------------------------------------- /src/event_handler.rs: -------------------------------------------------------------------------------- 1 | //! Logic for showing testing events to the user - the UI logic. 2 | //! 3 | //! All "UI" logic is driven through the `EventHandler` trait. 4 | 5 | pub use self::default::EventHandler as Default; 6 | 7 | use crate::{Config, model::{TestResult}}; 8 | 9 | mod default; 10 | 11 | /// An object which listens to events that occur during a test suite run. 12 | pub trait EventHandler { 13 | /// Called to notify before the test suite has started. 14 | fn on_test_suite_started(&mut self, suite_details: &TestSuiteDetails, config: &Config); 15 | 16 | /// Called to notify when the entire test suite has finished execution. 17 | fn on_test_suite_finished(&mut self, passed: bool, config: &Config); 18 | 19 | /// Called to notify when a test has been executed. 20 | fn on_test_finished(&mut self, result: TestResult, config: &Config); 21 | 22 | /// Called to notify about a nonfatal warning. 23 | fn note_warning(&mut self, message: &str); 24 | } 25 | 26 | /// Stores details about the test suite. 27 | #[derive(Clone, Debug, PartialEq, Eq)] 28 | pub struct TestSuiteDetails { 29 | /// The number of test files in the suite. 30 | pub number_of_test_files: usize, 31 | } 32 | 33 | -------------------------------------------------------------------------------- /src/event_handler/default.rs: -------------------------------------------------------------------------------- 1 | use crate::{util, Config, model::*}; 2 | 3 | use itertools::Itertools; 4 | use std::io; 5 | use std::io::prelude::*; 6 | use term; 7 | 8 | /// The default event handler, logging to stdout/stderr. 9 | pub struct EventHandler { 10 | test_results: Vec, 11 | } 12 | 13 | impl EventHandler { 14 | /// Creates a new default event handler. 15 | pub fn new() -> Self { 16 | EventHandler { test_results: Vec::new() } 17 | } 18 | } 19 | 20 | impl std::default::Default for EventHandler { 21 | fn default() -> Self { 22 | EventHandler::new() 23 | } 24 | } 25 | 26 | impl super::EventHandler for EventHandler { 27 | fn on_test_suite_started(&mut self, suite_details: &super::TestSuiteDetails, _: &Config) { 28 | print::reset_colors(); // our white might not match initial console white. we should be consistent. 29 | 30 | print::line(); 31 | print::horizontal_rule(); 32 | print::textln(format!("Running tests ({} files)", suite_details.number_of_test_files)); 33 | print::horizontal_rule(); 34 | print::line(); 35 | } 36 | 37 | fn on_test_suite_finished(&mut self, passed: bool, config: &Config) { 38 | // Sort the test results so that they will be consecutive. 39 | // This is required for itertools group_by used before to work properly. 40 | self.test_results.sort_by_key(|r| r.overall_result.human_label_pluralized()); 41 | 42 | print::line(); 43 | print::textln("finished running tests"); 44 | print::test_suite_status_message(passed, false, &self.test_results); 45 | print::line(); 46 | print::horizontal_rule(); 47 | print::horizontal_rule(); 48 | print::line(); 49 | 50 | if !passed { 51 | let failed_results = self.test_results.iter().filter(|r| r.overall_result.is_erroneous()).collect::>(); 52 | 53 | print::line(); 54 | print::textln_colored(format!("Failing tests ({}/{}):", failed_results.len(), self.test_results.len()), print::YELLOW); 55 | print::line(); 56 | 57 | for failed_test_result in failed_results.iter() { 58 | print::with(" ", print::StdStream::Err, print::RED); // indent the errors. 59 | self::result(failed_test_result, false, config); 60 | } 61 | } 62 | 63 | print::test_suite_status_message(passed, true, &self.test_results); 64 | 65 | // 'cargo test' will use the color we last emitted if we don't do this. 66 | print::reset_colors(); 67 | } 68 | 69 | fn on_test_finished(&mut self, result: TestResult, config: &Config) { 70 | self::result(&result, true, config); 71 | 72 | self.test_results.push(result); 73 | } 74 | 75 | fn note_warning(&mut self, message: &str) { 76 | print::warning(message); 77 | } 78 | } 79 | 80 | pub fn result(result: &TestResult, verbose: bool, config: &Config) { 81 | match result.overall_result { 82 | TestResultKind::Pass => { 83 | print::success(format!("PASS :: {}", result.path.relative.display())); 84 | }, 85 | TestResultKind::UnexpectedPass => { 86 | print::failure(format!("UNEXPECTED PASS :: {}", result.path.relative.display())); 87 | }, 88 | TestResultKind::Skip => { 89 | print::line(); 90 | print::warning(format!( 91 | "SKIP :: {} (test does not contain any test commands, perhaps you meant to add a 'CHECK'?)", 92 | result.path.relative.display())); 93 | print::line(); 94 | }, 95 | TestResultKind::Error { ref message } => { 96 | if verbose { print::line(); } 97 | 98 | print::error(format!("ERROR :: {}", result.path.relative.display())); 99 | 100 | if verbose { 101 | print::textln(message); 102 | 103 | print::line(); 104 | } 105 | } 106 | TestResultKind::Fail { ref reason, ref hint } => { 107 | if verbose { print::line(); } 108 | 109 | print::failure(format!("FAIL :: {}", result.path.relative.display())); 110 | 111 | // FIXME: improve formatting 112 | 113 | if verbose { 114 | print::line(); 115 | print::text("test failed: "); 116 | print::textln_colored(reason.human_summary(), print::RED); 117 | print::line(); 118 | print::textln(reason.human_detail_message(config)); 119 | 120 | if let Some(hint_text) = hint { 121 | print::textln(format!("hint: {}", hint_text)); 122 | } 123 | 124 | print::line(); 125 | } 126 | }, 127 | TestResultKind::ExpectedFailure { .. } => { 128 | print::warning(format!("XFAIL :: {}", result.path.relative.display())); 129 | }, 130 | TestResultKind::EmptyTest { .. } => { 131 | print::error(format!("EMPTY TEST :: {}", result.path.relative.display())); 132 | }, 133 | } 134 | 135 | if verbose && (result.overall_result.is_erroneous() || config.always_show_stderr) { 136 | for individual_run_result in result.individual_run_results.iter() { 137 | let (_, _, command_line, output) = individual_run_result; 138 | 139 | let formatted_stderr = crate::model::format_test_output("stderr", &output.stderr, 1, util::TruncateDirection::Bottom, config); 140 | if !output.stderr.is_empty() { 141 | print::textln(format!("NOTE: the program '{}' emitted text on standard error:", command_line)); 142 | print::line(); 143 | print::textln(formatted_stderr); 144 | print::line(); 145 | } 146 | } 147 | } 148 | } 149 | 150 | mod print { 151 | pub use term::color::*; 152 | use super::*; 153 | 154 | #[derive(Copy, Clone)] 155 | pub enum StdStream { Out, Err } 156 | 157 | pub fn line() { 158 | with("\n", 159 | StdStream::Out, 160 | term::color::WHITE); 161 | } 162 | 163 | pub fn horizontal_rule() { 164 | with("=================================================================\n", 165 | StdStream::Out, 166 | term::color::WHITE); 167 | } 168 | 169 | pub fn textln(msg: S) 170 | where S: Into { 171 | text(format!("{}\n", msg.into())) 172 | } 173 | 174 | pub fn text(msg: S) 175 | where S: Into { 176 | with(format!("{}", msg.into()), 177 | StdStream::Out, 178 | term::color::WHITE); 179 | } 180 | 181 | 182 | pub fn textln_colored(msg: S, color: u32) 183 | where S: Into { 184 | with(format!("{}\n", msg.into()), 185 | StdStream::Out, 186 | color); 187 | } 188 | 189 | 190 | pub fn success(msg: S) 191 | where S: Into { 192 | with(format!("{}\n", msg.into()), 193 | StdStream::Out, 194 | term::color::GREEN); 195 | } 196 | 197 | pub fn warning(msg: S) 198 | where S: Into { 199 | with(format!("{}\n", msg.into()), 200 | StdStream::Err, 201 | term::color::YELLOW); 202 | } 203 | 204 | pub fn error(msg: S) 205 | where S: Into { 206 | with(format!("{}\n", msg.into()), 207 | StdStream::Err, 208 | term::color::RED); 209 | } 210 | 211 | pub fn failure(msg: S) 212 | where S: Into { 213 | with(format!("{}\n", msg.into()), 214 | StdStream::Err, 215 | term::color::MAGENTA); 216 | } 217 | 218 | pub fn test_suite_status_message(passed: bool, verbose: bool, test_results: &[TestResult]) { 219 | if verbose { 220 | self::line(); 221 | self::horizontal_rule(); 222 | } 223 | 224 | if verbose { 225 | self::textln("Suite Status:"); 226 | self::line(); 227 | 228 | for (result_label, corresponding_results) in &test_results.iter().group_by(|r| r.overall_result.human_label_pluralized()) { 229 | self::textln(format!(" {}: {}", result_label, corresponding_results.count())); 230 | } 231 | 232 | self::line(); 233 | self::horizontal_rule(); 234 | self::line(); 235 | } 236 | 237 | match passed { 238 | true => self::success("all tests succeeded"), 239 | false => self::error("error: tests failed"), 240 | } 241 | } 242 | 243 | pub fn with(msg: S, 244 | stream: StdStream, 245 | color: term::color::Color) 246 | where S: Into { 247 | set_color(Some(msg), stream, color); 248 | reset_colors(); 249 | } 250 | 251 | pub fn set_color(msg: Option, 252 | stream: StdStream, 253 | color: term::color::Color) 254 | where S: Into { 255 | 256 | 257 | match stream { 258 | StdStream::Out => { 259 | let stdout_term_color = term::stdout().and_then(|mut t| if let Ok(()) = t.fg(color) { Some(t) } else { None }); 260 | 261 | if let Some(mut color_term) = stdout_term_color { 262 | if let Some(msg) = msg { 263 | write!(color_term, "{}", msg.into()).unwrap(); 264 | } 265 | } else { 266 | if let Some(msg) = msg { 267 | write!(io::stdout(), "{}", msg.into()).unwrap(); 268 | } 269 | } 270 | }, 271 | StdStream::Err => { 272 | let stderr_term_color = term::stderr().and_then(|mut t| if let Ok(()) = t.fg(color) { Some(t) } else { None }); 273 | 274 | if let Some(mut color_term) = stderr_term_color { 275 | if let Some(msg) = msg { 276 | write!(color_term, "{}", msg.into()).unwrap(); 277 | } 278 | } else { 279 | if let Some(msg) = msg { 280 | write!(io::stderr(), "{}", msg.into()).unwrap(); 281 | } 282 | } 283 | }, 284 | } 285 | } 286 | 287 | pub fn reset_colors() { 288 | for stream in [StdStream::Out, StdStream::Err].iter().cloned() { 289 | set_color::(None, stream, term::color::WHITE); 290 | } 291 | } 292 | } 293 | 294 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | //! A reusable testing tool, inspired by LLVM's `lit` tool. 2 | //! 3 | //! `lit` standing for _LLVM Integrated Test_. 4 | //! 5 | //! This crate contains both a reusable library for creating test tools and 6 | //! an executable with generalized command line interface for manual usage. 7 | 8 | pub use self::config::Config; 9 | 10 | pub use self::errors::*; 11 | pub use self::vars::{Variables, VariablesExt}; 12 | 13 | // The file extensions used by the integration tests for this repository. 14 | #[doc(hidden)] 15 | pub const INTEGRATION_TEST_FILE_EXTENSIONS: &'static [&'static str] = &[ 16 | "txt", "sh", 17 | ]; 18 | 19 | pub mod config; 20 | mod errors; 21 | pub mod event_handler; 22 | mod model; 23 | mod parse; 24 | pub mod run; 25 | mod util; 26 | mod vars; 27 | 28 | #[macro_use] 29 | extern crate error_chain; 30 | #[macro_use] 31 | extern crate lazy_static; 32 | #[macro_use] 33 | extern crate log; 34 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | extern crate lit; 2 | extern crate clap; 3 | 4 | use clap::{App, ArgMatches}; 5 | use std::env::consts; 6 | 7 | fn parse_cmdline() -> ArgMatches<'static> { 8 | let app = App::new("LLVM-lit inspired generic testing tool") 9 | .version(env!("CARGO_PKG_VERSION")) 10 | .author(env!("CARGO_PKG_AUTHORS")) 11 | .about(env!("CARGO_PKG_DESCRIPTION")); 12 | let app = lit::config::clap::mount_inside_app(app, true); 13 | 14 | let matches = app.get_matches(); 15 | matches 16 | } 17 | 18 | fn main() { 19 | let arg_matches = parse_cmdline(); 20 | 21 | lit::run::tests(lit::event_handler::Default::default(), |config| { 22 | config.add_search_path("integration-tests/"); 23 | for ext in lit::INTEGRATION_TEST_FILE_EXTENSIONS { 24 | config.add_extension(ext); 25 | } 26 | 27 | config.constants.insert("arch".to_owned(), consts::ARCH.to_owned()); 28 | config.constants.insert("os".to_owned(), consts::OS.to_owned()); 29 | 30 | lit::config::clap::parse_arguments(&arg_matches, config); 31 | }).unwrap() 32 | } 33 | -------------------------------------------------------------------------------- /src/model.rs: -------------------------------------------------------------------------------- 1 | use crate::{run, util, Config, Variables}; 2 | use std::{fmt, path::PathBuf}; 3 | use std::fmt::Write; 4 | 5 | /// A tool invocation. 6 | #[derive(Clone,Debug,PartialEq,Eq)] 7 | pub struct Invocation 8 | { 9 | /// The original command string. 10 | pub original_command: String, 11 | } 12 | 13 | // TODO: rename to TestFile 14 | #[derive(Clone, Debug, PartialEq, Eq)] 15 | pub struct TestFile 16 | { 17 | pub path: TestFilePath, 18 | pub commands: Vec, 19 | } 20 | 21 | #[derive(Clone, Debug, PartialEq, Eq)] 22 | pub struct TestFilePath { 23 | /// The on-disk path to the test file. 24 | pub absolute: PathBuf, 25 | pub relative: PathBuf, 26 | } 27 | 28 | #[derive(Clone,Debug,PartialEq,Eq)] 29 | pub struct Command 30 | { 31 | pub line_number: u32, 32 | pub kind: CommandKind, 33 | } 34 | 35 | #[derive(Clone,Debug)] 36 | pub enum CommandKind 37 | { 38 | /// Run an external tool. 39 | Run(Invocation), 40 | /// Verify that the output text matches an expression. 41 | Check(TextPattern), 42 | /// Verify that the very next output line matches an expression. 43 | CheckNext(TextPattern), 44 | /// Mark the test as supposed to fail. 45 | XFail, 46 | } 47 | 48 | #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] 49 | pub struct TextPattern { 50 | pub components: Vec, 51 | } 52 | 53 | /// A component in a text pattern. 54 | #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] 55 | pub enum PatternComponent { 56 | Text(String), 57 | Variable(String), 58 | Regex(String), 59 | NamedRegex { name: String, regex: String }, 60 | } 61 | 62 | #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] 63 | #[must_use] 64 | pub enum TestResultKind 65 | { 66 | /// Test passed successfully. 67 | Pass, 68 | /// Test passed but it was declared with `XFAIL`. 69 | UnexpectedPass, 70 | /// An error occurred whilst running the test. 71 | Error { message: String }, 72 | /// The test failed. 73 | Fail { 74 | reason: TestFailReason, 75 | hint: Option, 76 | }, 77 | /// The test was expected to fail and it did. 78 | ExpectedFailure { 79 | actual_reason: TestFailReason, 80 | }, 81 | EmptyTest, 82 | /// The test was skipped. 83 | Skip, 84 | } 85 | 86 | #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] 87 | pub enum TestFailReason { 88 | UnsuccessfulExecution { 89 | program_command_line: String, 90 | exit_status: i32, 91 | }, 92 | CheckFailed(CheckFailureInfo), 93 | } 94 | 95 | impl TestFailReason { 96 | pub fn human_summary(&self) -> &'static str { 97 | match *self { 98 | TestFailReason::UnsuccessfulExecution { .. } => { 99 | "unsuccessful program execution whilst running test" 100 | }, 101 | TestFailReason::CheckFailed(..) => { 102 | "test checked for text that did not exist in the output" 103 | }, 104 | } 105 | } 106 | 107 | pub fn human_detail_message(&self, config: &Config) -> String { 108 | match *self { 109 | TestFailReason::UnsuccessfulExecution { ref program_command_line, exit_status } => { 110 | format!("command '{}' exited with code '{}'", program_command_line, exit_status) 111 | }, 112 | TestFailReason::CheckFailed(ref check_failure_info) => { 113 | let mut buf = String::new(); 114 | writeln!(&mut buf, "expected text '{}' but that was not found", check_failure_info.expected_pattern).unwrap(); 115 | writeln!(&mut buf).unwrap(); 116 | 117 | // Write the successfully checked output. 118 | writeln!(&mut buf, "{}", format_test_output("successfully checked output", 119 | check_failure_info.successfully_checked_text(), 1, util::TruncateDirection::Top, 120 | config)).unwrap(); 121 | 122 | writeln!(&mut buf).unwrap(); 123 | 124 | // Write the remaining unchecked output. 125 | writeln!(&mut buf, "{}", format_test_output("remaining unchecked output", 126 | check_failure_info.remaining_text(), 127 | check_failure_info.successfully_checked_upto_line_number(), util::TruncateDirection::Bottom, 128 | config)).unwrap(); 129 | 130 | buf 131 | }, 132 | } 133 | } 134 | } 135 | 136 | pub(crate) fn format_test_output( 137 | output_label: &str, 138 | unformatted_output: &str, 139 | output_base_line_number: usize, 140 | truncate_direction: util::TruncateDirection, 141 | config: &Config) -> String { 142 | let mut formatted_output = util::decorate_with_line_numbers(unformatted_output, output_base_line_number); 143 | 144 | if let Some(max_line_count) = config.truncate_output_context_to_number_of_lines { 145 | formatted_output = util::truncate_to_max_lines(&formatted_output, max_line_count, truncate_direction); 146 | } 147 | let formatted_output = util::indent(&formatted_output, 1); 148 | 149 | format!("<{}>:\n\n{}\n", output_label, formatted_output, output_label) 150 | } 151 | 152 | /// Information about a failed check in a test. 153 | #[derive(Clone, PartialEq, Eq, PartialOrd, Ord)] 154 | pub struct CheckFailureInfo { 155 | pub complete_output_text: String, 156 | pub successfully_checked_until_byte_index: usize, 157 | pub expected_pattern: TextPattern, 158 | } 159 | 160 | /// Results from executing a test. 161 | #[derive(Debug)] 162 | pub struct TestResult 163 | { 164 | /// A path to the test. 165 | pub path: TestFilePath, 166 | /// The kind of result. 167 | pub overall_result: TestResultKind, 168 | pub individual_run_results: Vec<(TestResultKind, Invocation, run::CommandLine, ProgramOutput)>, 169 | } 170 | 171 | #[derive(Clone, Debug, PartialEq, Eq)] 172 | pub struct ProgramOutput { 173 | pub stdout: String, 174 | pub stderr: String, 175 | } 176 | 177 | 178 | #[derive(Debug)] 179 | pub struct Results 180 | { 181 | pub test_results: Vec, 182 | } 183 | 184 | impl PartialEq for CommandKind { 185 | fn eq(&self, other: &CommandKind) -> bool { 186 | match *self { 187 | CommandKind::Run(ref a) => if let CommandKind::Run(ref b) = *other { a == b } else { false }, 188 | CommandKind::Check(ref a) => if let CommandKind::Check(ref b) = *other { a.to_string() == b.to_string() } else { false }, 189 | CommandKind::CheckNext(ref a) => if let CommandKind::CheckNext(ref b) = *other { a.to_string() == b.to_string() } else { false }, 190 | CommandKind::XFail => *other == CommandKind::XFail, 191 | } 192 | } 193 | } 194 | 195 | impl Eq for CommandKind { } 196 | 197 | impl fmt::Display for TextPattern { 198 | fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { 199 | for component in self.components.iter() { 200 | match *component { 201 | PatternComponent::Text(ref text) => write!(fmt, "{}", text)?, 202 | PatternComponent::Variable(ref name) => write!(fmt, "$${}", name)?, 203 | PatternComponent::Regex(ref regex) => write!(fmt, "[[{}]]", regex)?, 204 | PatternComponent::NamedRegex { ref name, ref regex } => write!(fmt, "[[{}:{}]]", name, regex)?, 205 | } 206 | } 207 | 208 | Ok(()) 209 | } 210 | } 211 | 212 | impl Command 213 | { 214 | pub fn new(kind: CommandKind, line_number: u32) -> Self { 215 | Command { kind, line_number } 216 | } 217 | } 218 | 219 | impl TestResultKind { 220 | /// Checks if the result is considered an error. 221 | pub fn is_erroneous(&self) -> bool { 222 | use self::TestResultKind::*; 223 | 224 | match *self { 225 | UnexpectedPass | Error { .. } | Fail { .. } => true, 226 | Pass | Skip | ExpectedFailure { .. } | EmptyTest => false, 227 | } 228 | } 229 | 230 | pub fn unwrap(&self) { 231 | if self.is_erroneous() { 232 | panic!("error whilst running test: {:?}", self); 233 | } 234 | } 235 | 236 | pub fn human_label_pluralized(&self) -> &'static str { 237 | use self::TestResultKind::*; 238 | 239 | match *self { 240 | Pass => "Passes", 241 | UnexpectedPass => "Unexpected passes", 242 | Error { .. } => "Errors", 243 | Fail { .. } => "Test failures", 244 | ExpectedFailure { .. } => "Expected failures", 245 | EmptyTest { .. } => "Empty tests", 246 | Skip => "Skipped tests", 247 | } 248 | } 249 | } 250 | 251 | impl CheckFailureInfo { 252 | /// Gets the slice containing the portion of successfully checked text. 253 | pub fn successfully_checked_text(&self) -> &str { 254 | let byte_subslice = &self.complete_output_text.as_bytes()[0..self.successfully_checked_until_byte_index]; 255 | convert_bytes_to_str(byte_subslice) 256 | } 257 | 258 | /// Gets the slice containing the portion of unchecked, remaining text. 259 | pub fn remaining_text(&self) -> &str { 260 | let byte_subslice = &self.complete_output_text.as_bytes()[self.successfully_checked_until_byte_index..]; 261 | convert_bytes_to_str(byte_subslice) 262 | } 263 | 264 | pub fn successfully_checked_upto_line_number(&self) -> usize { 265 | self.successfully_checked_text().lines().count() + 1 266 | } 267 | } 268 | 269 | impl TestFile 270 | { 271 | /// Extra test-specific variables. 272 | pub fn variables(&self) -> Variables { 273 | let mut v = Variables::new(); 274 | v.insert("file".to_owned(), self.path.absolute.to_str().unwrap().to_owned()); 275 | v 276 | } 277 | 278 | /// Gets an iterator over all `RUN` commands in the test file. 279 | pub fn run_command_invocations(&self) -> impl Iterator { 280 | self.commands.iter().filter_map(|c| match c.kind { 281 | CommandKind::Run(ref invocation) => Some(invocation), 282 | _ => None, 283 | }) 284 | } 285 | 286 | /// Is this test expected to fail. 287 | pub fn is_expected_failure(&self) -> bool { 288 | self.commands.iter().any(|c| if let CommandKind::XFail = c.kind { true } else { false }) 289 | } 290 | } 291 | 292 | /// Build a text pattern from a single component. 293 | impl From for TextPattern { 294 | fn from(component: PatternComponent) -> Self { 295 | TextPattern { components: vec![component] } 296 | } 297 | } 298 | 299 | impl std::fmt::Debug for CheckFailureInfo { 300 | fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { 301 | #[derive(Debug)] 302 | struct CheckFailureInfo<'a> { 303 | expected_pattern: &'a TextPattern, 304 | successfully_checked_text: PrintStrTruncate<'a>, 305 | remaining_text: PrintStrTruncate<'a>, 306 | } 307 | 308 | const TRUNCATE_MIN: usize = 70; 309 | const TRUNCATE_MARKER: &'static str = "..."; 310 | struct PrintStrTruncate<'a>(&'a str); 311 | impl<'a> std::fmt::Debug for PrintStrTruncate<'a> { 312 | fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { 313 | if self.0.len() <= TRUNCATE_MIN { 314 | std::fmt::Debug::fmt(self.0, fmt) 315 | } else { 316 | let substr = &self.0[0..TRUNCATE_MIN]; 317 | substr.fmt(fmt)?; 318 | std::fmt::Display::fmt(TRUNCATE_MARKER, fmt) 319 | } 320 | } 321 | } 322 | 323 | CheckFailureInfo { 324 | expected_pattern: &self.expected_pattern, 325 | remaining_text: PrintStrTruncate(self.remaining_text()), 326 | successfully_checked_text: PrintStrTruncate(self.successfully_checked_text()), 327 | }.fmt(fmt) 328 | } 329 | } 330 | 331 | fn convert_bytes_to_str(bytes: &[u8]) -> &str { 332 | std::str::from_utf8(bytes).expect("invalid UTF-8 in output stream") 333 | } 334 | 335 | impl ProgramOutput { 336 | pub fn empty() -> Self { 337 | ProgramOutput { stdout: String::new(), stderr: String::new() } 338 | } 339 | } 340 | 341 | -------------------------------------------------------------------------------- /src/parse.rs: -------------------------------------------------------------------------------- 1 | use crate::model::*; 2 | 3 | use regex::Regex; 4 | use std::mem; 5 | 6 | lazy_static! { 7 | static ref DIRECTIVE_REGEX: Regex = Regex::new("([A-Z-]+):(.*)").unwrap(); 8 | static ref IDENTIFIER_REGEX: Regex = Regex::new("^[a-zA-Z_][a-zA-Z0-9_]*$").unwrap(); 9 | } 10 | 11 | /// Parses a test file 12 | pub fn test_file(path: TestFilePath, chars: I) -> Result 13 | where I: Iterator { 14 | let mut commands = Vec::new(); 15 | let test_body: String = chars.collect(); 16 | 17 | for (line_idx, line) in test_body.lines().enumerate() { 18 | let line_number = line_idx + 1; 19 | 20 | match self::possible_command(line, line_number as _) { 21 | Some(Ok(command)) => commands.push(command), 22 | Some(Err(e)) => { 23 | return Err(format!( 24 | "could not parse command: {}", e) 25 | ); 26 | }, 27 | None => continue, 28 | } 29 | } 30 | 31 | Ok(TestFile { 32 | path, 33 | commands: commands, 34 | }) 35 | } 36 | 37 | 38 | /// Parses a tool invocation. 39 | /// 40 | /// It is generatlly in the format: 41 | /// 42 | /// ``` bash 43 | /// [arg1] [arg2] ... 44 | /// ``` 45 | pub fn invocation<'a,I>(words: I) -> Result 46 | where I: Iterator { 47 | let parts: Vec<_> = words.collect(); 48 | let original_command = parts.join(" "); 49 | 50 | Ok(Invocation { original_command }) 51 | } 52 | 53 | pub fn text_pattern(s: &str) -> TextPattern { 54 | let mut components: Vec = vec![]; 55 | let mut chars = s.chars().peekable(); 56 | 57 | let mut current_text = vec![]; 58 | 59 | loop { 60 | let complete_text = |current_text: &mut Vec, components: &mut Vec| { 61 | let text = mem::replace(current_text, Vec::new()) 62 | .into_iter().collect(); 63 | components.push(PatternComponent::Text(text)); 64 | }; 65 | 66 | match (chars.next(), chars.peek().cloned()) { 67 | // Variable. 68 | (Some('$'), Some('$')) => { 69 | complete_text(&mut current_text, &mut components); 70 | chars.next(); // Eat second '$'. 71 | 72 | let name: String = chars.clone() 73 | .take_while(|c| c.is_alphanumeric()) 74 | .collect(); 75 | chars.nth(name.len() - 1); // Skip the variable name. 76 | components.push(PatternComponent::Variable(name)); 77 | }, 78 | // Named or unnamed regex. 79 | (Some('['), Some('[')) => { 80 | complete_text(&mut current_text, &mut components); 81 | chars.next(); // Eat second `[` 82 | 83 | let mut current_regex = vec![]; 84 | let mut bracket_level = 0i32; 85 | loop { 86 | match (chars.next(), chars.peek().cloned()) { 87 | (Some(']'), Some(']')) if bracket_level == 0=> { 88 | chars.next(); // Eat second `]`. 89 | break; 90 | }, 91 | (Some(c), _) => { 92 | match c { 93 | '[' => bracket_level += 1, 94 | ']' => bracket_level -= 1, 95 | _ => (), 96 | } 97 | 98 | current_regex.push(c); 99 | }, 100 | (None, _) => { 101 | break; 102 | }, 103 | } 104 | } 105 | 106 | let regex: String = current_regex.into_iter().collect(); 107 | 108 | let first_colon_idx = regex.chars().position(|c| c == ':'); 109 | let (name, regex): (Option<&str>, &str) = match first_colon_idx { 110 | Some(first_colon_idx) => { 111 | let substr = ®ex[0..first_colon_idx]; 112 | 113 | if IDENTIFIER_REGEX.is_match(&substr) { 114 | (Some(substr), ®ex[first_colon_idx+1..]) 115 | } else { 116 | (None, ®ex) 117 | } 118 | }, 119 | None => (None, ®ex), 120 | }; 121 | 122 | match name { 123 | Some(name) => components.push(PatternComponent::NamedRegex { name: name.to_owned(), regex: regex.to_owned() }), 124 | None => components.push(PatternComponent::Regex(regex.to_owned())), 125 | } 126 | 127 | }, 128 | (Some(c), _) => { 129 | current_text.push(c); 130 | }, 131 | (None, _) => { 132 | complete_text(&mut current_text, &mut components); 133 | break; 134 | } 135 | } 136 | } 137 | 138 | TextPattern { components: components } 139 | } 140 | 141 | /// Parses a possible command, if a string defines one. 142 | /// 143 | /// Returns `None` if no command is specified. 144 | pub fn possible_command(string: &str, line: u32) 145 | -> Option> { 146 | if !DIRECTIVE_REGEX.is_match(string) { return None; } 147 | 148 | let captures = DIRECTIVE_REGEX.captures(string).unwrap(); 149 | let command_str = captures.get(1).unwrap().as_str().trim(); 150 | let after_command_str = captures.get(2).unwrap().as_str().trim(); 151 | 152 | match command_str { 153 | // FIXME: better message if we have 'RUN :' 154 | "RUN" => { 155 | let inner_words = after_command_str.split_whitespace(); 156 | let invocation = match self::invocation(inner_words) { 157 | Ok(i) => i, 158 | Err(e) => return Some(Err(e)), 159 | }; 160 | 161 | Some(Ok(Command::new(CommandKind::Run(invocation), line))) 162 | }, 163 | "CHECK" => { 164 | let text_pattern = self::text_pattern(after_command_str); 165 | Some(Ok(Command::new(CommandKind::Check(text_pattern), line))) 166 | }, 167 | "CHECK-NEXT" => { 168 | let text_pattern = self::text_pattern(after_command_str); 169 | Some(Ok(Command::new(CommandKind::CheckNext(text_pattern), line))) 170 | }, 171 | "XFAIL" => { 172 | Some(Ok(Command::new(CommandKind::XFail, line))) 173 | }, 174 | _ => { 175 | Some(Err(format!("command '{}' not known", command_str))) 176 | }, 177 | } 178 | } 179 | 180 | #[cfg(tes)] 181 | mod test { 182 | use super::*; 183 | use std::collections::HashMap; 184 | 185 | #[test] 186 | fn parses_single_text() { 187 | assert_eq!(text_pattern("hello world"), 188 | "hello world"); 189 | } 190 | 191 | #[test] 192 | fn correctly_escapes_text() { 193 | assert_eq!(text_pattern("hello()").as_str(), 194 | "hello\\(\\)"); 195 | } 196 | 197 | #[test] 198 | fn correctly_picks_up_single_regex() { 199 | assert_eq!(text_pattern("[[\\d]]").as_str(), 200 | "\\d"); 201 | } 202 | 203 | #[test] 204 | fn correctly_picks_up_regex_between_text() { 205 | assert_eq!(text_pattern("1[[\\d]]3").as_str(), 206 | "1\\d3"); 207 | } 208 | 209 | #[test] 210 | fn correctly_picks_up_named_regex() { 211 | assert_eq!(text_pattern("[[num:\\d]]").as_str(), 212 | "(?P\\d)"); 213 | } 214 | } 215 | 216 | -------------------------------------------------------------------------------- /src/run/find_files.rs: -------------------------------------------------------------------------------- 1 | //! Functions for retrieving lists of files from disk. 2 | 3 | use crate::{Config, model::TestFilePath}; 4 | 5 | use std; 6 | use std::path::Path; 7 | use walkdir::WalkDir; 8 | 9 | /// Recursively finds tests for the given paths. 10 | pub fn with_config(config: &Config) -> Result, String> { 11 | let mut absolute_paths = Vec::new(); 12 | 13 | for path in config.test_paths.iter() { 14 | let path_str = path.display().to_string(); 15 | 16 | let test_paths = in_path(&path_str, config)?; 17 | absolute_paths.extend(test_paths.into_iter().map(|p| Path::new(&p).to_owned())); 18 | } 19 | 20 | let test_paths = absolute_paths.into_iter().map(|absolute_path| { 21 | let absolute_path = std::fs::canonicalize(absolute_path).unwrap(); 22 | let relative_path = relative_path::compute(&absolute_path, config).expect("could not compute relative path"); 23 | 24 | TestFilePath { absolute: absolute_path, relative: relative_path } 25 | }).collect(); 26 | 27 | Ok(test_paths) 28 | } 29 | 30 | pub fn in_path(path: &str, 31 | config: &Config) 32 | -> Result,String> { 33 | let metadata = match std::fs::metadata(path) { 34 | Ok(meta) => meta, 35 | Err(e) => return Err(format!("failed to open '{}': {}", 36 | path, e)), 37 | }; 38 | 39 | if metadata.is_dir() { 40 | tests_in_dir(path, config) 41 | } else { 42 | Ok(vec![path.to_owned()]) 43 | } 44 | } 45 | 46 | mod relative_path { 47 | use crate::Config; 48 | use std::path::{Path, PathBuf}; 49 | 50 | pub fn compute(test_absolute_path: &Path, config: &Config) 51 | -> Option { 52 | let mut take_path_relative_to_dir = None; 53 | 54 | if take_path_relative_to_dir.is_none() { 55 | if let Some(least_specific_parent_test_search_directory_path) = 56 | least_specific_parent_test_search_directory_path(test_absolute_path, config) { 57 | take_path_relative_to_dir = Some(least_specific_parent_test_search_directory_path); 58 | } 59 | } 60 | 61 | if take_path_relative_to_dir.is_none() { 62 | if let Some(most_common_test_path_ancestor) = 63 | most_common_test_path_ancestor(test_absolute_path, config) { 64 | take_path_relative_to_dir = Some(most_common_test_path_ancestor); 65 | } 66 | } 67 | 68 | take_path_relative_to_dir.map(|relative_to| { 69 | test_absolute_path.strip_prefix(relative_to).expect("relative path computation failed: not a prefix").to_owned() 70 | }) 71 | } 72 | 73 | /// Attempt to find the most specific prefix directory from the test search paths in the config. 74 | fn least_specific_parent_test_search_directory_path(test_absolute_path: &Path, config: &Config) 75 | -> Option { 76 | // N.B. we iterate over the test paths here. We don't check for the directory's actual 77 | // existence on the filesystem. This makes testing easier, but also: test paths can only 78 | // be strict prefixes/supersets of other test paths if they ARE directories. 79 | let matching_parent_test_search_directories = config.test_paths.iter() 80 | .filter(|possible_dir_path| test_absolute_path.starts_with(possible_dir_path)); 81 | 82 | let least_specific_matching_test_search_directory = matching_parent_test_search_directories.min_by_key(|p| p.components().count()); 83 | 84 | if let Some(least_specific_matching_test_search_directory) = least_specific_matching_test_search_directory { 85 | Some(least_specific_matching_test_search_directory.to_owned()) 86 | } else { 87 | None 88 | } 89 | } 90 | 91 | /// Otherwise, find the most common path from all the test file paths. 92 | /// 93 | /// NOTE: this will return `None` in several cases, such as if there is only one test path, 94 | /// or On windows in the case where there are tests located on several different device drives. 95 | fn most_common_test_path_ancestor(test_absolute_path: &Path, config: &Config) 96 | -> Option { 97 | // different disk drives at the same time. 98 | { 99 | let initial_current_path_containing_everything_so_far = test_absolute_path.parent().unwrap(); 100 | let mut current_path_containing_everything_so_far = initial_current_path_containing_everything_so_far; 101 | 102 | for test_path in config.test_paths.iter() { 103 | if !test_path.starts_with(current_path_containing_everything_so_far) { 104 | let common_ancestor = test_path.ancestors().find(|p| current_path_containing_everything_so_far.starts_with(p)); 105 | 106 | if let Some(common_ancestor) = common_ancestor { 107 | // The common ancestor path may be empty if the files are on different 108 | // devices. 109 | if common_ancestor.file_name().is_some() { 110 | current_path_containing_everything_so_far = common_ancestor; 111 | } 112 | } else { 113 | // N.B. we only ever expect no common ancestor on Windows 114 | // where paths may be on different devices. This should be uncommon. 115 | // We cannot use this logic to compute the relative path in this scenario. 116 | } 117 | } 118 | } 119 | 120 | if current_path_containing_everything_so_far != initial_current_path_containing_everything_so_far { 121 | Some(current_path_containing_everything_so_far.to_owned()) 122 | } else { 123 | None // no common prefix path could be calculated from the test paths 124 | } 125 | 126 | } 127 | } 128 | 129 | #[cfg(test)] 130 | mod test { 131 | use crate::Config; 132 | use std::path::Path; 133 | 134 | #[test] 135 | fn test_compute() { 136 | let config = Config { 137 | test_paths: [ 138 | "/home/foo/projects/cool-project/tests/", 139 | "/home/foo/projects/cool-project/tests/run-pass/", 140 | "/home/foo/projects/cool-project/tests/run-fail/", 141 | ].iter().map(|p| Path::new(p).to_owned()).collect(), 142 | ..Config::default() 143 | }; 144 | 145 | assert_eq!(super::compute( 146 | &Path::new("/home/foo/projects/cool-project/tests/run-pass/test1.txt"), &config), 147 | Some(Path::new("run-pass/test1.txt").to_owned())); 148 | } 149 | 150 | #[test] 151 | fn test_least_specific_parent_test_search_directory_path_when_all_test_paths_are_directories() { 152 | let config = Config { 153 | test_paths: [ 154 | "/home/foo/projects/cool-project/tests/", 155 | "/home/foo/projects/cool-project/tests/run-pass/", 156 | "/home/foo/projects/cool-project/tests/run-fail/", 157 | ].iter().map(|p| Path::new(p).to_owned()).collect(), 158 | ..Config::default() 159 | }; 160 | 161 | assert_eq!(super::least_specific_parent_test_search_directory_path( 162 | &Path::new("/home/foo/projects/cool-project/tests/run-pass/test1.txt"), &config), 163 | Some(Path::new("/home/foo/projects/cool-project/tests/").to_owned())); 164 | } 165 | 166 | #[test] 167 | fn test_least_specific_parent_test_search_directory_path_when_one_test_path_directory() { 168 | let config = Config { 169 | test_paths: [ 170 | "/home/foo/projects/cool-project/tests/", 171 | ].iter().map(|p| Path::new(p).to_owned()).collect(), 172 | ..Config::default() 173 | }; 174 | 175 | assert_eq!(super::least_specific_parent_test_search_directory_path( 176 | &Path::new("/home/foo/projects/cool-project/tests/run-pass/test1.txt"), &config), 177 | Some(Path::new("/home/foo/projects/cool-project/tests/").to_owned())); 178 | } 179 | 180 | #[test] 181 | fn test_most_common_test_path_ancestor_when_all_paths_are_absolute() { 182 | let config = Config { 183 | test_paths: [ 184 | "/home/foo/projects/cool-project/tests/run-pass/test1.txt", 185 | "/home/foo/projects/cool-project/tests/run-pass/test2.txt", 186 | "/home/foo/projects/cool-project/tests/run-fail/test3.txt", 187 | ].iter().map(|p| Path::new(p).to_owned()).collect(), 188 | ..Config::default() 189 | }; 190 | 191 | assert_eq!(super::most_common_test_path_ancestor( 192 | &Path::new("/home/foo/projects/cool-project/tests/run-pass/test1.txt"), &config), 193 | Some(Path::new("/home/foo/projects/cool-project/tests").to_owned())); 194 | } 195 | 196 | 197 | #[test] 198 | fn test_most_common_test_path_ancestor_when_all_paths_absolute_on_different_drives() { 199 | let config = Config { 200 | test_paths: [ 201 | "C:/tests/run-pass/test1.txt", 202 | "C:/tests/run-pass/test2.txt", 203 | "Z:/tests/run-fail/test3.txt", 204 | "Z:/tests/run-fail/test4.txt", 205 | ].iter().map(|p| Path::new(p).to_owned()).collect(), 206 | ..Config::default() 207 | }; 208 | 209 | assert_eq!(super::most_common_test_path_ancestor( 210 | &Path::new("C:/tests/run-pass/test2.txt"), &config), 211 | None); 212 | } 213 | } 214 | } 215 | 216 | fn tests_in_dir(path: &str, 217 | config: &Config) -> Result,String> { 218 | let tests = files_in_dir(path)?.into_iter() 219 | .filter(|f| { 220 | let path = std::path::Path::new(f); 221 | path.extension().map(|ext| config.is_extension_supported(ext.to_str().unwrap())).unwrap_or(false) 222 | }) 223 | .collect(); 224 | Ok(tests) 225 | } 226 | 227 | fn files_in_dir(path: &str) -> Result,String> { 228 | let mut dir_tests = Vec::new(); 229 | 230 | for entry in WalkDir::new(path) { 231 | let entry = entry.unwrap(); 232 | 233 | // don't go into an infinite loop 234 | if entry.path().to_str().unwrap() == path { 235 | continue; 236 | } 237 | 238 | if entry.metadata().unwrap().is_file() { 239 | dir_tests.push(entry.path().to_str().unwrap().to_owned()); 240 | } 241 | } 242 | 243 | Ok(dir_tests) 244 | } 245 | 246 | -------------------------------------------------------------------------------- /src/run/legacy_test_evaluator.rs: -------------------------------------------------------------------------------- 1 | use crate::Config; 2 | use std::collections::HashMap; 3 | use std::{env, fs, process}; 4 | use regex::Regex; 5 | use crate::model::*; 6 | use crate::{parse, vars}; 7 | 8 | use std; 9 | 10 | pub struct TestEvaluator 11 | { 12 | pub invocation: Invocation, 13 | } 14 | 15 | struct Checker 16 | { 17 | lines: Lines, 18 | variables: HashMap, 19 | } 20 | 21 | /// Iterator over a set of lines. 22 | struct Lines { 23 | lines: Vec, 24 | current: usize, 25 | } 26 | 27 | impl TestEvaluator 28 | { 29 | pub fn new(invocation: Invocation) -> Self { 30 | TestEvaluator { invocation: invocation } 31 | } 32 | 33 | pub fn execute_tests(self, test_file: &TestFile, config: &Config) -> TestResultKind { 34 | let mut cmd = self.build_command(test_file, config); 35 | 36 | let output = match cmd.output() { 37 | Ok(o) => o, 38 | Err(e) => match e.kind() { 39 | std::io::ErrorKind::NotFound => { 40 | return TestResultKind::Error( 41 | format!("shell '{}' does not exist", &config.shell).into(), 42 | ); 43 | }, 44 | _ => return TestResultKind::Error(e.into()), 45 | }, 46 | }; 47 | 48 | if !output.status.success() { 49 | let stderr = String::from_utf8(output.stderr).unwrap(); 50 | 51 | return TestResultKind::Fail { 52 | message: format!( 53 | "exited with code {}", output.status.code().unwrap()), 54 | reason: unimplemented!(), 55 | }; 56 | } 57 | 58 | let stdout = String::from_utf8(output.stdout).unwrap(); 59 | 60 | let stdout_lines: Vec<_> = stdout.lines().map(|l| l.trim().to_owned()).collect(); 61 | let stdout: String = stdout_lines.join("\n"); 62 | 63 | Checker::new(stdout).run(config, &test_file) 64 | } 65 | 66 | fn build_command(&self, 67 | test_file: &TestFile, 68 | config: &Config) -> process::Command { 69 | let mut variables = config.constants.clone(); 70 | variables.extend(test_file.variables()); 71 | 72 | let command_line: String = vars::resolve::invocation(&self.invocation, &config, &mut variables); 73 | 74 | let mut cmd = process::Command::new(&config.shell); 75 | cmd.args(&["-c", &command_line]); 76 | 77 | if let Ok(current_exe) = env::current_exe() { 78 | if let Some(parent) = current_exe.parent() { 79 | let current_path = env::var("PATH").unwrap_or(String::new()); 80 | cmd.env("PATH", format!("{}:{}", parent.to_str().unwrap(), current_path)); 81 | } 82 | } 83 | 84 | cmd 85 | } 86 | } 87 | 88 | impl Checker 89 | { 90 | fn new(stdout: String) -> Self { 91 | Checker { 92 | lines: stdout.into(), 93 | variables: HashMap::new(), 94 | } 95 | } 96 | 97 | fn run(&mut self, config: &Config, test_file: &TestFile) -> TestResultKind { 98 | let mut expect_test_pass = true; 99 | let result = self.run_expecting_pass(config, test_file, &mut expect_test_pass); 100 | 101 | if expect_test_pass { 102 | result 103 | } else { // expected failure 104 | match result { 105 | TestResultKind::Pass => TestResultKind::UnexpectedPass, 106 | TestResultKind::Error(_) | 107 | TestResultKind::Fail { .. } => TestResultKind::ExpectedFailure, 108 | TestResultKind::Skip => TestResultKind::Skip, 109 | TestResultKind::UnexpectedPass | 110 | TestResultKind::ExpectedFailure => unreachable!(), 111 | } 112 | } 113 | } 114 | 115 | fn run_expecting_pass(&mut self, 116 | config: &Config, 117 | test_file: &TestFile, 118 | expect_test_pass: &mut bool) -> TestResultKind { 119 | for command in test_file.commands.iter() { 120 | match command.kind { 121 | // Some tests can be marked as expected failures. 122 | CommandKind::XFail => *expect_test_pass = false, 123 | CommandKind::Run(..) => (), 124 | CommandKind::Check(ref text_pattern) => { 125 | let regex = vars::resolve::text_pattern(&text_pattern, config, &mut self.variables); 126 | 127 | let beginning_line = self.lines.peek().unwrap_or_else(|| "".to_owned()); 128 | let matched_line = self.lines.find(|l| regex.is_match(l)); 129 | 130 | if let Some(matched_line) = matched_line { 131 | self.process_captures(®ex, &matched_line); 132 | } else { 133 | let message = format_check_error(test_file, 134 | command, 135 | &format!("could not find match: '{}'", text_pattern), 136 | &beginning_line); 137 | return TestResultKind::Fail { message, reason: unimplemented!() }; 138 | } 139 | }, 140 | CommandKind::CheckNext(ref text_pattern) => { 141 | let regex = vars::resolve::text_pattern(&text_pattern, config, &mut self.variables); 142 | 143 | if let Some(next_line) = self.lines.next() { 144 | if regex.is_match(&next_line) { 145 | self.process_captures(®ex, &next_line); 146 | } else { 147 | let message = format_check_error(test_file, 148 | command, 149 | &format!("could not find pattern: '{}'", text_pattern), 150 | &next_line); 151 | 152 | return TestResultKind::Fail { message, reason: unimplemented!() }; 153 | } 154 | } else { 155 | return TestResultKind::Fail { 156 | message: format!("check-next reached the end of file unexpectedly"), 157 | reason: unimplemented!(), 158 | }; 159 | } 160 | }, 161 | } 162 | } 163 | 164 | // N.B. This currently only runs for successful 165 | // test runs. Perhaps it should run for all? 166 | if config.cleanup_temporary_files { 167 | let tempfiles = self.variables.iter() 168 | .filter(|(k,_)| k.contains("tempfile")) 169 | .map(|(_,v)| v); 170 | 171 | for tempfile in tempfiles { 172 | // Ignore errors, these are tempfiles, they go away anyway. 173 | fs::remove_file(tempfile).ok(); 174 | } 175 | } 176 | 177 | TestResultKind::Pass 178 | } 179 | 180 | pub fn process_captures(&mut self, regex: &Regex, line: &str) { 181 | // We shouldn't be calling this function if it didn't match. 182 | debug_assert_eq!(regex.is_match(line), true); 183 | let captures = if let Some(captures) = regex.captures(line) { 184 | captures 185 | } else { 186 | return; 187 | }; 188 | 189 | for capture_name in regex.capture_names() { 190 | // we only care about named captures. 191 | if let Some(name) = capture_name { 192 | let captured_value = captures.name(name).unwrap(); 193 | 194 | self.variables.insert(name.to_owned(), captured_value.as_str().to_owned()); 195 | } 196 | } 197 | } 198 | } 199 | 200 | impl Lines { 201 | pub fn new(lines: Vec) -> Self { 202 | Lines { lines: lines, current: 0 } 203 | } 204 | 205 | fn peek(&self) -> Option<::Item> { 206 | self.next_index().map(|idx| self.lines[idx].clone()) 207 | } 208 | 209 | fn next_index(&self) -> Option { 210 | if self.current > self.lines.len() { return None; } 211 | 212 | self.lines[self.current..].iter() 213 | .position(|l| parse::possible_command(l, 0).is_none()) 214 | .map(|offset| self.current + offset) 215 | } 216 | } 217 | 218 | impl Iterator for Lines 219 | { 220 | type Item = String; 221 | 222 | fn next(&mut self) -> Option { 223 | if let Some(next_index) = self.next_index() { 224 | self.current = next_index + 1; 225 | Some(self.lines[next_index].clone()) 226 | } else { 227 | None 228 | } 229 | } 230 | } 231 | 232 | impl From for Lines 233 | { 234 | fn from(s: String) -> Lines { 235 | Lines::new(s.split("\n").map(ToOwned::to_owned).collect()) 236 | } 237 | } 238 | 239 | fn format_check_error(test_file: &TestFile, 240 | command: &Command, 241 | msg: &str, 242 | next_line: &str) -> String { 243 | self::format_error(test_file, command, msg, next_line) 244 | } 245 | 246 | fn format_error(test_file: &TestFile, 247 | command: &Command, 248 | msg: &str, 249 | next_line: &str) -> String { 250 | format!("{}:{}: {}\nnext line: '{}'", test_file.path.display(), command.line_number, msg, next_line) 251 | } 252 | 253 | #[cfg(test)] 254 | mod test { 255 | use super::*; 256 | 257 | fn lines(s: &str) -> Vec { 258 | let lines: Lines = s.to_owned().into(); 259 | lines.collect() 260 | } 261 | 262 | #[test] 263 | fn trivial_lines_works_correctly() { 264 | assert_eq!(lines("hello\nworld\nfoo"), &["hello", "world", "foo"]); 265 | } 266 | 267 | #[test] 268 | fn lines_ignores_commands() { 269 | assert_eq!(lines("; RUN: cat %file\nhello\n; CHECK: foo\nfoo"), 270 | &["hello", "foo"]); 271 | } 272 | 273 | #[test] 274 | fn lines_can_peek() { 275 | let mut lines: Lines = "hello\nworld\nfoo".to_owned().into(); 276 | assert_eq!(lines.next(), Some("hello".to_owned())); 277 | assert_eq!(lines.peek(), Some("world".to_owned())); 278 | assert_eq!(lines.next(), Some("world".to_owned())); 279 | assert_eq!(lines.peek(), Some("foo".to_owned())); 280 | assert_eq!(lines.next(), Some("foo".to_owned())); 281 | } 282 | } 283 | 284 | -------------------------------------------------------------------------------- /src/run/mod.rs: -------------------------------------------------------------------------------- 1 | //! Routines for running tests. 2 | 3 | pub(crate) mod find_files; 4 | mod test_evaluator; 5 | 6 | pub use self::test_evaluator::CommandLine; 7 | 8 | use crate::{Config, event_handler::{EventHandler, TestSuiteDetails}}; 9 | use crate::model::*; 10 | 11 | /// Runs all tests according to a given config. 12 | /// 13 | /// Return `Ok` if all tests pass, and `Err` otherwise. 14 | /// 15 | /// # Parameters 16 | /// 17 | /// * `config_fn` is a function which sets up the test config. 18 | /// * `event_handler` is an object which presents the user interface to the user. 19 | /// 20 | pub fn tests( 21 | mut event_handler: impl EventHandler, 22 | config_fn: F, 23 | ) -> Result<(), ()> 24 | where F: Fn(&mut Config) { 25 | let mut config = Config::default(); 26 | config_fn(&mut config); 27 | 28 | // Used for storing artifacts generated during testing. 29 | let artifact_config = save_artifacts::Config { 30 | artifacts_dir: config.save_artifacts_to_directory.clone(), 31 | }; 32 | 33 | if config.test_paths.is_empty() { 34 | util::abort("no test paths given to lit") 35 | } 36 | 37 | let test_paths = match find_files::with_config(&config) { 38 | Ok(paths) => paths, 39 | Err(e) => util::abort(format!("could not find test files: {}", e)), 40 | }; 41 | 42 | if test_paths.is_empty() { 43 | event_handler.note_warning("could not find any tests"); 44 | return Err(()); 45 | } 46 | 47 | let test_suite_details = TestSuiteDetails { 48 | number_of_test_files: test_paths.len(), 49 | }; 50 | 51 | event_handler.on_test_suite_started(&test_suite_details, &config); 52 | 53 | let mut has_failure = false; 54 | for test_file_path in test_paths { 55 | let test_file = util::parse_test(test_file_path).unwrap(); 56 | let is_successful = self::single_file(&test_file, &mut event_handler, &config, &artifact_config); 57 | 58 | if !is_successful { has_failure = true; } 59 | } 60 | let is_successful = !has_failure; 61 | 62 | event_handler.on_test_suite_finished(is_successful, &config); 63 | save_artifacts::suite_status(is_successful, &artifact_config); 64 | 65 | if !has_failure { Ok(()) } else { Err(()) } 66 | } 67 | 68 | /// Executes a single, parsed test file. 69 | /// 70 | /// Returns `true` if all the tests in the file succeeded. 71 | fn single_file( 72 | test_file: &TestFile, 73 | event_handler: &mut dyn EventHandler, 74 | config: &Config, 75 | artifact_config: &save_artifacts::Config, 76 | ) -> bool { 77 | let test_results = test_evaluator::execute_tests(test_file, config); 78 | 79 | // The overall result is failure if there are any failures, otherwise it is a pass. 80 | let overall_result = test_results.iter().map(|(r, _, _, _)| r).filter(|r| match *r { 81 | TestResultKind::Pass { .. } => false, 82 | _ => true, 83 | }).next().cloned().unwrap_or(TestResultKind::Pass); 84 | 85 | let result = TestResult { 86 | path: test_file.path.clone(), 87 | overall_result, 88 | individual_run_results: test_results.into_iter().map(|(a, b, c, d)| (a, b.clone(), c, d)).collect(), 89 | }; 90 | 91 | save_artifacts::run_results(&result, test_file, artifact_config); 92 | 93 | let is_erroneous = result.overall_result.is_erroneous(); 94 | 95 | event_handler.on_test_finished(result, config); 96 | 97 | !is_erroneous 98 | } 99 | 100 | mod util 101 | { 102 | use crate::model::*; 103 | use crate::parse; 104 | 105 | use std::{io::Read, path::Path}; 106 | use std; 107 | 108 | pub fn parse_test(path: TestFilePath) -> Result { 109 | let mut text = String::new(); 110 | open_file(&path.absolute).read_to_string(&mut text).unwrap(); 111 | parse::test_file(path, text.chars()) 112 | } 113 | 114 | fn open_file(path: &Path) -> std::fs::File { 115 | match std::fs::File::open(path) { 116 | Ok(f) => f, 117 | Err(e) => abort(format!("could not open {}: {}", 118 | path.display(), e.to_string())), 119 | } 120 | } 121 | pub fn abort(msg: S) -> ! 122 | where S: Into { 123 | eprintln!("error: {}", msg.into()); 124 | 125 | std::process::exit(1); 126 | } 127 | } 128 | 129 | mod save_artifacts { 130 | use super::CommandLine; 131 | use crate::model::*; 132 | use std::path::{Path, PathBuf}; 133 | use std::fs; 134 | 135 | const SUITE_STATUS_PATH: &'static str = "suite-status.txt"; 136 | 137 | #[derive(Clone, Debug)] 138 | pub struct Config { 139 | pub artifacts_dir: Option, 140 | } 141 | 142 | pub fn suite_status(is_successful: bool, config: &Config) { 143 | save(&Path::new(SUITE_STATUS_PATH), config, || { 144 | if is_successful { 145 | "successful\n" 146 | } else { 147 | "failed\n" 148 | } 149 | }); 150 | } 151 | 152 | pub fn run_results(test_result: &TestResult, test_file: &TestFile, artifact_config: &Config) { 153 | let only_one_run_command = test_result.individual_run_results.len() == 1; 154 | 155 | for (i, (result_kind, _, command_line, output)) in test_result.individual_run_results.iter().enumerate() { 156 | let run_number = if only_one_run_command { None } else { Some(i + 1) }; 157 | self::individual_run_result(run_number, result_kind, command_line, output, test_file, artifact_config); 158 | } 159 | } 160 | 161 | pub fn individual_run_result(run_number: Option, result_kind: &TestResultKind, command_line: &CommandLine, output: &ProgramOutput, test_file: &TestFile, config: &Config) { 162 | let test_file_extension = test_file.path.absolute.extension().and_then(|s| s.to_str()).unwrap_or("txt"); 163 | 164 | let dir_run_result = match run_number { 165 | Some(run_number) => test_file.path.relative.join(format!("run-command-{}", run_number)), 166 | None => test_file.path.relative.clone(), 167 | }; 168 | 169 | save(&dir_run_result.join("result.txt"), config, || { 170 | format!("{:#?}\n", result_kind) 171 | }); 172 | 173 | save(&dir_run_result.join("stdout.txt"), config, || &output.stdout[..]); 174 | save(&dir_run_result.join("stderr.txt"), config, || &output.stderr[..]); 175 | save(&dir_run_result.join("command-line.txt"), config, || format!("{}\n", command_line.0)); 176 | 177 | save(&dir_run_result.join(&format!("copy-of-test-case.{}", test_file_extension)), config, || std::fs::read(&test_file.path.absolute).unwrap()); 178 | 179 | create_symlink(&test_file.path.absolute, &dir_run_result.join(&format!("symlink-to-test-case.{}", test_file_extension)), config) 180 | } 181 | 182 | fn save(relative_path: &Path, config: &Config, render: impl FnOnce() -> C ) 183 | where C: AsRef<[u8]> { 184 | if let Some(artifacts_dir) = config.artifacts_dir.as_ref() { 185 | let absolute_path = artifacts_dir.join(relative_path); 186 | let parent_directory = absolute_path.parent().unwrap(); 187 | 188 | let file_content = render(); 189 | 190 | fs::create_dir_all(parent_directory).unwrap(); 191 | fs::write(absolute_path, file_content).unwrap(); 192 | } 193 | } 194 | 195 | /// Creates a symlink, unless symlinks are not supported in this environment. 196 | fn create_symlink(src: &Path, relative_dst: &Path, config: &Config) { 197 | #[cfg(unix)] 198 | fn create_symlink_impl(src: &Path, dst: &Path) -> std::io::Result<()> { std::os::unix::fs::symlink(src, dst) } 199 | #[cfg(not(unix))] 200 | fn create_symlink_impl(_: &Path, _: &Path) -> std::io::Result<()> { Ok(()) } 201 | 202 | if let Some(artifacts_dir) = config.artifacts_dir.as_ref() { 203 | let dst = artifacts_dir.join(relative_dst); 204 | 205 | if dst.exists() { 206 | fs::remove_file(&dst).unwrap(); // Remove the symlink. 207 | } 208 | create_symlink_impl(src, &dst).unwrap(); 209 | } 210 | 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /src/run/test_evaluator.rs: -------------------------------------------------------------------------------- 1 | use crate::{ 2 | model::{CommandKind, Invocation, TestFile, TestResultKind, TestFailReason, ProgramOutput}, 3 | Config, 4 | vars, 5 | VariablesExt, 6 | }; 7 | use self::state::TestRunState; 8 | use std::{collections::HashMap, env, fs, process}; 9 | 10 | mod state; 11 | #[cfg(test)] mod state_tests; 12 | 13 | /// Responsible for evaluating specific tests and collecting 14 | /// the results. 15 | #[derive(Clone)] 16 | pub struct TestEvaluator 17 | { 18 | pub invocation: Invocation, 19 | } 20 | 21 | pub fn execute_tests<'test>(test_file: &'test TestFile, config: &Config) -> Vec<(TestResultKind, &'test Invocation, CommandLine, ProgramOutput)> { 22 | test_file.run_command_invocations().map(|invocation| { 23 | let initial_variables = { 24 | let mut vars = HashMap::new(); 25 | vars.extend(config.constants.clone()); 26 | vars.extend(test_file.variables()); 27 | vars 28 | }; 29 | 30 | let mut test_run_state = TestRunState::new(initial_variables); 31 | let (command, command_line) = self::build_command(invocation, test_file, config); 32 | 33 | let (program_output, execution_result) = self::collect_output(command, command_line.clone(), config); 34 | 35 | test_run_state.append_program_output(&program_output.stdout); 36 | test_run_state.append_program_stderr(&program_output.stderr); 37 | 38 | if execution_result.is_erroneous() { 39 | return (execution_result, invocation, command_line, program_output); 40 | } 41 | 42 | let overall_test_result_kind = run_test_checks(&mut test_run_state, test_file, config); 43 | (overall_test_result_kind, invocation, command_line, program_output) 44 | }).collect() 45 | } 46 | 47 | fn run_test_checks( 48 | test_run_state: &mut TestRunState, 49 | test_file: &TestFile, 50 | config: &Config, 51 | ) -> TestResultKind { 52 | let mut check_result = TestResultKind::EmptyTest; 53 | 54 | for command in test_file.commands.iter() { 55 | let test_result = match command.kind { 56 | CommandKind::Run(..) | // RUN commands are already handled above, in the loop. 57 | CommandKind::XFail => { // XFAIL commands are handled separately too. 58 | TestResultKind::Pass 59 | }, 60 | CommandKind::Check(ref text_pattern) => test_run_state.check(text_pattern, config), 61 | CommandKind::CheckNext(ref text_pattern) => test_run_state.check_next(text_pattern, config), 62 | }; 63 | 64 | if config.cleanup_temporary_files { 65 | let tempfile_paths = test_run_state.variables().tempfile_paths(); 66 | 67 | for tempfile in tempfile_paths { 68 | // Ignore errors, these are tempfiles, they go away anyway. 69 | fs::remove_file(tempfile).ok(); 70 | } 71 | } 72 | 73 | 74 | // Early return for failures. 75 | if test_result.is_erroneous() { 76 | check_result = test_result; 77 | break; 78 | } else { 79 | check_result = TestResultKind::Pass; 80 | } 81 | } 82 | 83 | match check_result { 84 | TestResultKind::Fail { reason, hint } => { 85 | if test_file.is_expected_failure() { 86 | TestResultKind::ExpectedFailure { actual_reason: reason } 87 | } else { 88 | TestResultKind::Fail { reason, hint} 89 | } 90 | }, 91 | r => r, 92 | } 93 | } 94 | 95 | fn collect_output( 96 | mut command: process::Command, 97 | command_line: CommandLine, 98 | config: &Config, 99 | ) -> (ProgramOutput, TestResultKind) { 100 | let mut test_result_kind = TestResultKind::Pass; 101 | 102 | let output = match command.output() { 103 | Ok(o) => o, 104 | Err(e) => { 105 | let error_message = match e.kind() { 106 | std::io::ErrorKind::NotFound => format!("shell '{}' does not exist", &config.shell).into(), 107 | _ => e.to_string(), 108 | }; 109 | 110 | return (ProgramOutput::empty(), TestResultKind::Error { message: error_message }); 111 | }, 112 | }; 113 | 114 | let program_output = ProgramOutput { 115 | stdout: String::from_utf8_lossy(&output.stdout).into_owned(), 116 | stderr: String::from_utf8_lossy(&output.stderr).into_owned(), 117 | }; 118 | 119 | if !output.status.success() { 120 | test_result_kind = TestResultKind::Fail { 121 | reason: TestFailReason::UnsuccessfulExecution { 122 | exit_status: output.status.code().unwrap_or_else(|| if output.status.success() { 0 } else { 1 }), 123 | program_command_line: command_line.0, 124 | }, 125 | hint: None, 126 | }; 127 | } 128 | 129 | (program_output, test_result_kind) 130 | } 131 | 132 | #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] 133 | pub struct CommandLine(pub String); 134 | 135 | /// Builds a command that can be used to execute the process behind a `RUN` directive. 136 | fn build_command(invocation: &Invocation, 137 | test_file: &TestFile, 138 | config: &Config) -> (process::Command, CommandLine) { 139 | let mut variables = config.constants.clone(); 140 | variables.extend(test_file.variables()); 141 | 142 | let command_line: String = vars::resolve::invocation(invocation, &config, &mut variables); 143 | 144 | let mut cmd = process::Command::new(&config.shell); 145 | cmd.args(&["-c", &command_line]); 146 | 147 | if !config.extra_executable_search_paths.is_empty() { 148 | let os_path_separator = if cfg!(windows) { ";" } else { ":" }; 149 | 150 | let current_path = env::var("PATH").unwrap_or(String::new()); 151 | let paths_to_inject = config.extra_executable_search_paths.iter().map(|p| p.display().to_string()).collect::>(); 152 | let os_path_to_inject = format!("{}{}{}", paths_to_inject.join(os_path_separator), os_path_separator, current_path); 153 | 154 | cmd.env("PATH", os_path_to_inject); 155 | } 156 | 157 | (cmd, CommandLine(command_line)) 158 | } 159 | 160 | impl std::fmt::Display for CommandLine { 161 | fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { 162 | self.0.fmt(fmt) 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /src/run/test_evaluator/state.rs: -------------------------------------------------------------------------------- 1 | //! The test evaluator implementation, independent of external 2 | //! resources like OS processes. 3 | 4 | use crate::{ 5 | Config, Variables, 6 | model::{self, TestResultKind, TestFailReason, TextPattern}, 7 | vars, 8 | }; 9 | use std::collections::HashMap; 10 | use regex::Regex; 11 | 12 | /// Byte-index relative to entire stream. 13 | #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] 14 | struct AbsoluteByteIndex(pub usize); 15 | 16 | /// Byte-index relative to start of unprocessed stream. 17 | #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] 18 | struct RelativeByteIndex(pub usize); 19 | 20 | /// The byte range of a matched pattern. 21 | #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] 22 | struct MatchedRange { 23 | start: RelativeByteIndex, 24 | end: RelativeByteIndex, 25 | } 26 | 27 | /// Responsible for storing the state of execution for a single `RUN` execution. 28 | #[derive(Debug)] 29 | pub struct TestRunState { 30 | /// All output bytes emitted by the program. 31 | complete_output_stream: String, 32 | /// The current position in the stream at which all prior output has been 33 | /// successfully checked by the test script. 34 | current_stream_byte_position: AbsoluteByteIndex, 35 | /// The stderr portion of the command output. This does not get used by `CHECK`s. 36 | complete_stderr: String, 37 | /// A list of available variables to the test script. 38 | variables: HashMap, 39 | } 40 | 41 | impl TestRunState { 42 | pub fn new(initial_variables: HashMap) -> Self { 43 | TestRunState { 44 | complete_output_stream: String::new(), 45 | current_stream_byte_position: AbsoluteByteIndex(0), 46 | complete_stderr: String::new(), 47 | variables: initial_variables, 48 | } 49 | } 50 | 51 | /// Appends output from the inner program. 52 | pub fn append_program_output(&mut self, output: &str) { 53 | self.complete_output_stream.extend(output.chars()) 54 | } 55 | 56 | /// Appends stderr output. 57 | pub fn append_program_stderr(&mut self, stderr: &str) { 58 | self.complete_stderr.extend(stderr.chars()) 59 | } 60 | 61 | /// Verifies that a text pattern appears subsequently in the stream. 62 | pub fn check( 63 | &mut self, 64 | text_pattern: &TextPattern, 65 | config: &Config) -> TestResultKind { 66 | self.check_extended(text_pattern, false, config) 67 | } 68 | 69 | /// Verifies that the very-next non-whitespace line matches a text pattern. 70 | pub fn check_next( 71 | &mut self, 72 | text_pattern: &TextPattern, 73 | config: &Config) -> TestResultKind { 74 | self.check_extended(text_pattern, true, config) 75 | } 76 | 77 | fn check_extended( 78 | &mut self, 79 | text_pattern: &TextPattern, 80 | require_on_next_line: bool, 81 | config: &Config) -> TestResultKind { 82 | 83 | self.eat_whitespace(); 84 | 85 | let next_relative_matched_range = self.next_unprocessed_byte_index_of(text_pattern, config); 86 | 87 | match next_relative_matched_range { 88 | Some(matched_range) => { 89 | // Logic for the CHECK-NEXT directive. 90 | if require_on_next_line { 91 | match self.unprocessed_output_stream().find("\n") { 92 | Some(index_of_first_new_line_byte) => { 93 | if matched_range.start.0 >= index_of_first_new_line_byte { 94 | return TestResultKind::Fail { 95 | reason: TestFailReason::CheckFailed(model::CheckFailureInfo { 96 | complete_output_text: self.complete_output_stream.clone(), 97 | successfully_checked_until_byte_index: self.current_stream_byte_position.0, 98 | expected_pattern: text_pattern.clone(), 99 | }), 100 | hint: Some(format!("found a match for '{}', but it does not appear on the next line, as required by the CHECK-NEXT directive", text_pattern)), 101 | }; 102 | } 103 | }, 104 | None => (), // we are on the last line, no need to verify that explicitly. 105 | } 106 | } 107 | 108 | self.current_stream_byte_position += matched_range.end; 109 | 110 | // No other checks should run against the partial line. 111 | self.eat_until_end_of_line(); 112 | 113 | TestResultKind::Pass 114 | }, 115 | None => { 116 | model::TestResultKind::Fail { 117 | reason: model::TestFailReason::CheckFailed(model::CheckFailureInfo { 118 | complete_output_text: self.complete_output_stream.clone(), 119 | successfully_checked_until_byte_index: self.current_stream_byte_position.0, 120 | expected_pattern: text_pattern.clone(), 121 | }), 122 | hint: None, 123 | } 124 | }, 125 | } 126 | } 127 | 128 | pub fn unprocessed_output_bytes(&self) -> &[u8] { 129 | &self.complete_output_stream.as_bytes()[self.current_stream_byte_position.0..] 130 | } 131 | 132 | /// Gets all of the non-consumed inner program bytes. 133 | pub fn unprocessed_output_stream(&self) -> &str { 134 | convert_bytes_to_str(self.unprocessed_output_bytes()) 135 | } 136 | 137 | /// Gets all variables in scope. 138 | pub fn variables(&self) -> &Variables { &self.variables } 139 | 140 | fn eat_whitespace(&mut self) { 141 | if self.unprocessed_output_stream().chars().next().map(char::is_whitespace).unwrap_or(false) { 142 | let first_nonwhitespace_offset = self.unprocessed_output_stream().chars().take_while(|c| c.is_whitespace()).map(char::len_utf8).sum(); 143 | let first_nonwhitespace_offset = RelativeByteIndex(first_nonwhitespace_offset); 144 | 145 | match first_nonwhitespace_offset { 146 | // if there are no non-whitespace characters, then there cannot be a match. 147 | RelativeByteIndex(0) => self.set_position_eof(), 148 | relative_index => self.current_stream_byte_position += relative_index, 149 | } 150 | } 151 | } 152 | 153 | /// Eats all characters until the end of the current line. 154 | fn eat_until_end_of_line(&mut self) { 155 | let unprocessed = self.unprocessed_output_stream(); 156 | 157 | match unprocessed.find("\n").map(RelativeByteIndex) { 158 | Some(new_line_index) => { 159 | self.current_stream_byte_position += RelativeByteIndex(new_line_index.0 + 1); 160 | }, 161 | None => self.set_position_eof(), // no more new lines in file. 162 | } 163 | } 164 | 165 | /// Gets the index of the next occurrence of the given text pattern. 166 | /// 167 | /// N.B. Does not advance the unprocessed stream pointer. This only takes a mutable 168 | /// reference because of the need to resolve the internal test variable list. 169 | fn next_unprocessed_byte_index_of(&mut self, text_pattern: &TextPattern, config: &Config) 170 | -> Option { 171 | let regex = vars::resolve::text_pattern(text_pattern, config, &mut self.variables); 172 | let output_str = self.unprocessed_output_stream(); 173 | 174 | debug!("converting expected text pattern to regex: {:?}", regex); 175 | 176 | match regex.find(output_str) { 177 | Some(regex_match) => { 178 | let matched_range = MatchedRange { 179 | start: RelativeByteIndex(regex_match.start()), 180 | end: RelativeByteIndex(regex_match.end()), 181 | }; 182 | 183 | let new_variables = process_captures(®ex, regex_match.as_str()); 184 | self.variables.extend(new_variables); 185 | 186 | Some(matched_range) 187 | }, 188 | None => None, 189 | } 190 | } 191 | 192 | fn set_position_eof(&mut self) { 193 | let output_bytes = self.complete_output_stream.as_bytes(); 194 | self.current_stream_byte_position = AbsoluteByteIndex(output_bytes.len()); 195 | } 196 | } 197 | 198 | impl std::ops::AddAssign for AbsoluteByteIndex { 199 | fn add_assign(&mut self, relative: RelativeByteIndex) { 200 | self.0 += relative.0; 201 | } 202 | } 203 | 204 | fn convert_bytes_to_str(bytes: &[u8]) -> &str { 205 | std::str::from_utf8(bytes).expect("invalid UTF-8 in output stream") 206 | } 207 | 208 | /// Returns all named capture groups from regexes as variables. 209 | fn process_captures( 210 | regex: &Regex, 211 | matched_text: &str) 212 | -> HashMap { 213 | // We shouldn't be calling this function if it didn't match. 214 | debug_assert_eq!(regex.is_match(matched_text), true); 215 | 216 | let captures = if let Some(captures) = regex.captures(matched_text) { 217 | captures 218 | } else { 219 | return HashMap::new(); 220 | }; 221 | 222 | let mut variables = HashMap::new(); 223 | 224 | for capture_name in regex.capture_names() { 225 | // we only care about named captures. 226 | if let Some(name) = capture_name { 227 | let captured_value = captures.name(name).unwrap(); 228 | 229 | variables.insert(name.to_owned(), captured_value.as_str().to_owned()); 230 | } 231 | } 232 | 233 | variables 234 | } 235 | -------------------------------------------------------------------------------- /src/run/test_evaluator/state_tests.rs: -------------------------------------------------------------------------------- 1 | //! Tests for the test evaluator state logic. 2 | 3 | use crate::{ 4 | Config, 5 | model::{self, TestFailReason}, 6 | }; 7 | use super::*; 8 | 9 | const EMOJI_SMILEY: char = '\u{1F600}'; 10 | const EMOJI_JOY: char = '\u{1F602}'; 11 | 12 | fn fixture_program_prints_whitespace_emoji_and_hello_world() -> TestRunState { 13 | let mut test_state = TestRunState::new(HashMap::new()); 14 | test_state.append_program_output(&format!(" \n{}\nhello \nworld", EMOJI_SMILEY)); 15 | test_state 16 | } 17 | 18 | // Stress-test for byte<->char conversion logic. 19 | fn fixture_program_prints_unicode_emoji() -> TestRunState { 20 | let mut test_state = TestRunState::new(HashMap::new()); 21 | test_state.append_program_output(&format!(" {}\n {} smiles.\n\t{}\njoy{}.", EMOJI_SMILEY, EMOJI_SMILEY, EMOJI_JOY, EMOJI_SMILEY)); 22 | test_state 23 | } 24 | 25 | // Prints the periodic table in order, useful of testing line constraints. 26 | fn fixture_program_prints_periodic_table_in_order() -> TestRunState { 27 | const ELEMENTS: &'static [&'static str] = &[ 28 | "Hydrogen", "Helium", "Lithium", "Beryllium", "Boron", "Carbon", 29 | "Nitrogen", "Oxygen", "Fluorine", "Neon", "Sodium", "Magnesium", 30 | ]; 31 | 32 | let mut test_state = TestRunState::new(HashMap::new()); 33 | test_state.append_program_output(&ELEMENTS.join(", is an element.\n")); 34 | test_state 35 | } 36 | 37 | #[test] 38 | fn check_next_works_standalone_in_very_basic_scenario() { 39 | let mut test_state = fixture_program_prints_whitespace_emoji_and_hello_world(); 40 | let config = Config::default(); 41 | 42 | assert!(test_state.unprocessed_output_stream().starts_with(" ")); 43 | 44 | test_state.check_next(&model::PatternComponent::Text(EMOJI_SMILEY.to_string()).into(), &config).unwrap(); 45 | assert_eq!(test_state.unprocessed_output_stream(), "hello \nworld"); 46 | 47 | let res = test_state.check_next(&model::PatternComponent::Text("world".to_owned()).into(), &config); 48 | match res { 49 | TestResultKind::Fail { reason, hint } => { 50 | match reason { 51 | TestFailReason::CheckFailed(..) => { 52 | assert_eq!(test_state.unprocessed_output_stream(), "hello \nworld", 53 | "errors should not consume any of the underlying stream"); 54 | assert_eq!(hint, Some("found a match for \'world\', but it does not appear on the next line, as required by the CHECK-NEXT directive".to_owned())); 55 | }, 56 | r => panic!("unexpected test failure reason: {:?}", r), 57 | } 58 | }, 59 | _ => panic!("unexpected failure reason: {:?}", res), 60 | } 61 | 62 | test_state.check_next(&model::PatternComponent::Text("hello".to_owned()).into(), &config).unwrap(); 63 | assert_eq!(test_state.unprocessed_output_stream(), "world"); 64 | } 65 | 66 | #[test] 67 | fn check_next_can_handle_multibyte_unicode_chars() { 68 | let mut test_state = fixture_program_prints_unicode_emoji(); 69 | let config = Config::default(); 70 | 71 | assert!(test_state.unprocessed_output_stream().starts_with(" ")); 72 | 73 | // Consume first smiley emoji 74 | test_state.check_next(&model::PatternComponent::Text(EMOJI_SMILEY.to_string()).into(), &config).unwrap(); 75 | assert!(test_state.unprocessed_output_stream().starts_with(&format!(" {} smiles.\n", EMOJI_SMILEY))); 76 | 77 | // Consume next identical smiley. 78 | test_state.check_next(&model::PatternComponent::Text(EMOJI_SMILEY.to_string()).into(), &config).unwrap(); 79 | assert!(test_state.unprocessed_output_stream().starts_with("\t")); 80 | 81 | // Consume the joy emoji. 82 | test_state.check_next(&model::PatternComponent::Text(EMOJI_JOY.to_string()).into(), &config).unwrap(); 83 | assert_eq!(test_state.unprocessed_output_stream(), format!("joy{}.", EMOJI_SMILEY)); 84 | 85 | // Consume the last smiley and terminating full stop. 86 | test_state.check_next(&model::PatternComponent::Text(format!("{}.", EMOJI_SMILEY)).into(), &config).unwrap(); 87 | assert_eq!(test_state.unprocessed_output_stream(), ""); 88 | } 89 | 90 | #[test] 91 | fn check_next_rejects_matches_not_on_next_line() { 92 | let mut test_state = fixture_program_prints_periodic_table_in_order(); 93 | let config = Config::default(); 94 | 95 | assert!(test_state.unprocessed_output_stream().starts_with("Hydrogen, is an element.\nHelium, is an element.\n")); 96 | 97 | test_state.check_next(&model::PatternComponent::Text("Hydrogen".to_owned()).into(), &config).unwrap(); 98 | assert!(test_state.unprocessed_output_stream().starts_with("Helium")); 99 | 100 | // Attempt to read ahead of next line, expect failure. 101 | let res = test_state.check_next(&model::PatternComponent::Text("Lithium".to_owned()).into(), &config); 102 | match res { 103 | TestResultKind::Fail { reason, hint } => { 104 | match reason { 105 | TestFailReason::CheckFailed(..) => { 106 | assert!(test_state.unprocessed_output_stream().starts_with("Helium"), 107 | "errors should not consume any of the underlying stream"); 108 | assert_eq!(hint, Some("found a match for \'Lithium\', but it does not appear on the next line, as required by the CHECK-NEXT directive".to_owned())); 109 | }, 110 | r => panic!("unexpected test failure reason: {:?}", r), 111 | } 112 | }, 113 | _ => panic!("unexpected failure reason: {:?}", res), 114 | } 115 | } 116 | 117 | #[test] 118 | fn check_with_nonexistent_regex_produces_failure() { 119 | let mut test_state = fixture_program_prints_periodic_table_in_order(); 120 | let config = Config::default(); 121 | 122 | assert!(test_state.unprocessed_output_stream().starts_with("Hydrogen, is an element.\nHelium, is an element.\n")); 123 | 124 | test_state.check(&model::PatternComponent::Text("Helium".to_owned()).into(), &config).unwrap(); 125 | 126 | let res = test_state.check(&model::PatternComponent::Text("nonexistent".to_owned()).into(), &config); 127 | 128 | // Validate that a nonexistent regex triggers a failure. 129 | if let TestResultKind::Fail { reason, hint } = res { 130 | match reason { 131 | TestFailReason::CheckFailed(failure_info) => { 132 | assert!(failure_info.successfully_checked_text().ends_with("Helium, is an element.\n")); 133 | assert!(failure_info.remaining_text().starts_with("Lithium, is an element.\n")); 134 | assert_eq!(hint, None); 135 | }, 136 | r => panic!("unexpected failure reason: {:?}", r), 137 | } 138 | } else { 139 | panic!("expected the pattern to fail: {:?}", res); 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /src/util.rs: -------------------------------------------------------------------------------- 1 | //! Utility functions for internal use. 2 | 3 | const DEFAULT_INDENT_ATOM: &'static str = " "; 4 | const TRUNCATED_TEXT_MARKER: &'static str = "... (truncated)"; 5 | 6 | #[derive(Copy, Clone, Debug, PartialEq, Eq)] 7 | pub enum TruncateDirection { Top, Bottom } 8 | 9 | /// Indents a piece of text. 10 | pub fn indent(text: &str, level: usize) -> String { 11 | indent_ext(text, level, DEFAULT_INDENT_ATOM) 12 | } 13 | 14 | pub fn indent_ext(text: &str, level: usize, indentation_atom: &str) -> String { 15 | let indent = (0..level).into_iter().map(|_| indentation_atom).collect::>().join(""); 16 | text.lines().map(|l| format!("{}{}", indent, l.trim())).collect::>().join("\n") + "\n" 17 | } 18 | 19 | pub fn decorate_with_line_numbers(text: &str, starts_from_line_number: usize) -> String { 20 | let max_line_num_digits = (starts_from_line_number + text.lines().count()).to_string().len(); 21 | 22 | text.lines().enumerate().map(|(relative_lineno, line)| { 23 | let line_number_str = (starts_from_line_number + relative_lineno).to_string(); 24 | let number_of_pad_chars = max_line_num_digits - line_number_str.len(); 25 | let horizontal_padding_str = (0..number_of_pad_chars).into_iter().map(|_| " ").collect::(); 26 | 27 | format!("{}{}| {}", line_number_str, horizontal_padding_str, line) 28 | }).collect::>().join("\n") 29 | } 30 | 31 | pub fn truncate_to_max_lines( 32 | text: &str, 33 | max_line_count: usize, 34 | truncate_direction: TruncateDirection) -> String { 35 | let lines = text.lines().collect::>(); 36 | 37 | let is_truncated = lines.len() > max_line_count; 38 | 39 | let truncated_lines: Vec<_> = match truncate_direction { 40 | TruncateDirection::Bottom => lines.into_iter().take(max_line_count).collect(), 41 | TruncateDirection::Top => lines.into_iter().rev().take(max_line_count).rev().collect(), 42 | }; 43 | 44 | let truncated_text = truncated_lines.join("\n"); 45 | 46 | if is_truncated { 47 | match truncate_direction { 48 | TruncateDirection::Bottom => truncated_text.to_owned() + "\n\n" + TRUNCATED_TEXT_MARKER, 49 | TruncateDirection::Top => TRUNCATED_TEXT_MARKER.to_string() + "\n\n" + &truncated_text[..], 50 | } 51 | } else { 52 | truncated_text // the text was not actually truncated 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/vars/mod.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::path::{Path, PathBuf}; 3 | 4 | pub mod resolve; 5 | 6 | pub type Variables = HashMap; 7 | 8 | pub trait VariablesExt { 9 | fn as_map(&self) -> &HashMap; 10 | 11 | /// Gets a list of tempfile paths in the variable list. 12 | fn tempfile_paths(&self) -> Vec { 13 | self.as_map().iter() 14 | .filter(|(k,_)| k.contains("tempfile")) 15 | .map(|(_,v)| Path::new(v).to_owned()) 16 | .collect() 17 | } 18 | } 19 | 20 | impl VariablesExt for Variables { 21 | fn as_map(&self) -> &Self { self } 22 | } 23 | 24 | -------------------------------------------------------------------------------- /src/vars/resolve.rs: -------------------------------------------------------------------------------- 1 | //! Utilities for resolving/substituting variables within different types. 2 | 3 | use crate::model::*; 4 | use crate::vars::Variables; 5 | use crate::Config; 6 | 7 | use regex::Regex; 8 | 9 | lazy_static! { 10 | static ref CONSTANT_REGEX: Regex = Regex::new("@([_a-zA-Z]+)").unwrap(); 11 | } 12 | 13 | /// A span representing where a constant name resides in a string. 14 | #[derive(Debug)] 15 | struct ConstantSpan { 16 | /// The name of the constant. 17 | name: String, 18 | /// The index of the first character. 19 | start: usize, 20 | /// The index of the last character. 21 | end: usize, 22 | } 23 | 24 | pub fn text_pattern(pattern: &TextPattern, config: &Config, 25 | variables: &mut Variables) -> Regex { 26 | let regex_parts: Vec<_> = pattern.components.iter().map(|comp| match *comp { 27 | PatternComponent::Text(ref text) => regex::escape(text), 28 | PatternComponent::Variable(ref name) => { 29 | // FIXME: proper error handling. 30 | let value = config.lookup_variable(name, variables); 31 | 32 | let var_resolution_log = format!("resolving '@{}' to '{}' in {:?}", name, value, pattern); 33 | debug!("{}", var_resolution_log); 34 | 35 | if config.dump_variable_resolution { 36 | eprintln!("[info] {}", var_resolution_log); 37 | } 38 | 39 | value.to_owned() 40 | }, 41 | PatternComponent::Regex(ref regex) => regex.clone(), 42 | PatternComponent::NamedRegex { ref name, ref regex } => format!("(?P<{}>{})", name, regex), 43 | }).collect(); 44 | Regex::new(®ex_parts.join("")).expect("generated invalid line match regex") 45 | } 46 | 47 | pub fn invocation(invocation: &Invocation, 48 | config: &Config, 49 | constants: &mut Variables) -> String { 50 | let mut command_line = String::new(); 51 | 52 | let _cmd: String = invocation.original_command.clone(); 53 | let mut constant_spans = CONSTANT_REGEX.find_iter(&_cmd).map(|mat| { 54 | let name = mat.as_str()[1..].to_owned(); // Skip the '@' character. 55 | 56 | ConstantSpan { 57 | name: name, 58 | start: mat.start(), 59 | end: mat.end(), 60 | } 61 | }); 62 | 63 | let mut index = 0; 64 | loop { 65 | if let Some(next_span) = constant_spans.next() { 66 | assert!(index <= next_span.start, "went too far"); 67 | 68 | let value = config.lookup_variable(&next_span.name, constants); 69 | 70 | let var_resolution_log = format!("resolving '@{}' to '{}' in {:?}", next_span.name, value, _cmd); 71 | debug!("{}", var_resolution_log); 72 | 73 | if config.dump_variable_resolution { 74 | eprintln!("[info] {}", var_resolution_log); 75 | } 76 | 77 | // Check if there is some text between us and the regex. 78 | if next_span.start != index { 79 | let part = &invocation.original_command[index..next_span.start]; 80 | 81 | command_line += part; 82 | index += part.len(); 83 | } 84 | 85 | assert_eq!(index, next_span.start, "we should be up to the regex"); 86 | command_line += &value; 87 | index += next_span.name.len() + 1; // Skip the `@` and the name. 88 | } else { 89 | // Almost finished, just copy over the rest of the text. 90 | command_line += &invocation.original_command[index..]; 91 | break; 92 | } 93 | } 94 | 95 | command_line 96 | } 97 | 98 | #[cfg(test)] 99 | mod test { 100 | use std::collections::HashMap; 101 | 102 | lazy_static! { 103 | static ref VARIABLES: HashMap = { 104 | let mut v = HashMap::new(); 105 | v.insert("po".to_owned(), "polonium".to_owned()); 106 | v.insert("name".to_owned(), "bob".to_owned()); 107 | v 108 | }; 109 | } 110 | 111 | mod text_pattern { 112 | use super::*; 113 | use crate::{parse, vars}; 114 | use crate::Config; 115 | 116 | fn resolve(s: &str) -> String { 117 | let text_pattern = parse::text_pattern(s); 118 | vars::resolve::text_pattern(&text_pattern, &Config::default(), &mut VARIABLES.clone()).as_str().to_owned() 119 | } 120 | 121 | #[test] 122 | fn correctly_picks_up_single_variable() { 123 | assert_eq!(resolve("$$po").as_str(), 124 | "polonium"); 125 | } 126 | 127 | #[test] 128 | fn correctly_picks_up_variable_between_junk() { 129 | assert_eq!(resolve("[[[a-z]]]$$po foo").as_str(), 130 | "[a-z]polonium foo"); 131 | } 132 | 133 | #[test] 134 | fn correctly_picks_up_variable_at_end() { 135 | assert_eq!(resolve("goodbye $$name").as_str(), 136 | "goodbye bob"); 137 | } 138 | } 139 | 140 | mod invocation { 141 | use crate::{parse, vars, Config}; 142 | use std::collections::HashMap; 143 | 144 | lazy_static! { 145 | static ref BASIC_CONSTANTS: HashMap = { 146 | let mut m = HashMap::new(); 147 | m.insert("cc".to_owned(), "clang++".to_owned()); 148 | m 149 | }; 150 | } 151 | 152 | fn resolve(s: &str, consts: &mut HashMap) -> String { 153 | let invocation = parse::invocation(s.split_whitespace()).unwrap(); 154 | vars::resolve::invocation(&invocation, &Config::default(), consts) 155 | } 156 | 157 | #[test] 158 | fn no_constants_is_nop() { 159 | assert_eq!(resolve("hello world", &mut BASIC_CONSTANTS.clone()), "hello world"); 160 | } 161 | 162 | #[test] 163 | fn only_const() { 164 | assert_eq!(resolve("@cc", &mut BASIC_CONSTANTS.clone()), "clang++"); 165 | } 166 | 167 | #[test] 168 | fn junk_then_const() { 169 | assert_eq!(resolve("foo bar! @cc", &mut BASIC_CONSTANTS.clone()), "foo bar! clang++"); 170 | } 171 | 172 | #[test] 173 | fn junk_then_const_then_junk() { 174 | assert_eq!(resolve("hello @cc world", &mut BASIC_CONSTANTS.clone()), "hello clang++ world"); 175 | } 176 | } 177 | } 178 | 179 | -------------------------------------------------------------------------------- /tests/integration_tests.rs: -------------------------------------------------------------------------------- 1 | use lit::run; 2 | 3 | const CRATE_PATH: &'static str = env!("CARGO_MANIFEST_DIR"); 4 | 5 | /// Runs all of the integration tests in the top-level directory 6 | /// of the repository. 7 | #[test] 8 | fn integration_tests() { 9 | pretty_env_logger::init(); 10 | 11 | run::tests(lit::event_handler::Default::default(), |config| { 12 | config.add_search_path(format!("{}/integration-tests", CRATE_PATH)); 13 | for ext in lit::INTEGRATION_TEST_FILE_EXTENSIONS { 14 | config.add_extension(ext); 15 | } 16 | }).expect("unit test(s) failed"); 17 | 18 | // Now run the tests again but use a custom shell instead. 19 | run::tests(lit::event_handler::Default::default(), |config| { 20 | config.add_search_path(format!("{}/integration-tests", CRATE_PATH)); 21 | for ext in lit::INTEGRATION_TEST_FILE_EXTENSIONS { 22 | config.add_extension(ext); 23 | } 24 | 25 | config.shell = "sh".to_string(); 26 | }).expect("unit test(s) failed"); 27 | } 28 | --------------------------------------------------------------------------------