├── .gitignore ├── deno.json ├── .rustfmt.toml ├── rust-toolchain.toml ├── Cargo.toml ├── src ├── collection │ ├── strategies │ │ ├── helpers.rs │ │ ├── mod.rs │ │ ├── file_test_mapper.rs │ │ ├── test_per_file.rs │ │ └── test_per_directory.rs │ └── mod.rs ├── lib.rs ├── runner.rs └── reporter.rs ├── .github └── workflows │ ├── release.yml │ └── ci.yml ├── LICENSE ├── README.md └── Cargo.lock /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | /target 3 | -------------------------------------------------------------------------------- /deno.json: -------------------------------------------------------------------------------- 1 | { 2 | "exclude": [ 3 | "target" 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /.rustfmt.toml: -------------------------------------------------------------------------------- 1 | max_width = 80 2 | tab_spaces = 2 3 | edition = "2021" 4 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "1.91.1" 3 | components = ["clippy", "rustfmt"] 4 | profile = "minimal" 5 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "file_test_runner" 3 | version = "0.11.1" 4 | edition = "2024" 5 | description = "File-based test runner for running tests found in files." 6 | authors = ["the Deno authors"] 7 | license = "MIT" 8 | repository = "https://github.com/denoland/file_test_runner" 9 | 10 | [dependencies] 11 | anyhow = "1.0.82" 12 | crossbeam-channel = "0.5.12" 13 | deno_terminal = "0.2.0" 14 | parking_lot = "0.12.1" 15 | rayon = "1.11.0" 16 | regex = "1.11.1" 17 | thiserror = "2" 18 | -------------------------------------------------------------------------------- /src/collection/strategies/helpers.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | use std::path::Path; 4 | 5 | use crate::PathedIoError; 6 | 7 | pub(crate) fn read_dir_entries( 8 | dir_path: &Path, 9 | ) -> Result, PathedIoError> { 10 | let mut entries = std::fs::read_dir(dir_path) 11 | .map_err(|err| PathedIoError::new(dir_path, err))? 12 | .collect::, _>>() 13 | .map_err(|err| PathedIoError::new(dir_path, err))?; 14 | entries.retain(|e| { 15 | !e.file_name().to_string_lossy().starts_with('.') 16 | && !e.file_name().eq_ignore_ascii_case("readme.md") 17 | }); 18 | entries.sort_by_key(|a| a.file_name()); 19 | Ok(entries) 20 | } 21 | 22 | pub(crate) fn append_to_category_name( 23 | category_name: &str, 24 | new_part: &str, 25 | ) -> String { 26 | format!("{}::{}", category_name, new_part) 27 | } 28 | -------------------------------------------------------------------------------- /src/collection/strategies/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | use std::path::Path; 4 | 5 | mod file_test_mapper; 6 | mod helpers; 7 | mod test_per_directory; 8 | mod test_per_file; 9 | 10 | pub use file_test_mapper::*; 11 | pub use test_per_directory::*; 12 | pub use test_per_file::*; 13 | 14 | use crate::collection::CollectTestsError; 15 | use crate::collection::CollectedTestCategory; 16 | 17 | /// Strategy for collecting tests. 18 | pub trait TestCollectionStrategy { 19 | /// Return a list of tests found in the provided base path. 20 | /// 21 | /// Collected tests may return optional data. This might be useful 22 | /// in scenarios where you want to collect multiple tests within 23 | /// a file using the `file_test_runner::collection::strategies::FileTestMapperStrategy`. 24 | fn collect_tests( 25 | &self, 26 | base: &Path, 27 | ) -> Result, CollectTestsError>; 28 | } 29 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | releaseKind: 7 | description: "Kind of release" 8 | type: choice 9 | options: 10 | - patch 11 | - minor 12 | required: true 13 | 14 | jobs: 15 | rust: 16 | name: release 17 | runs-on: ubuntu-latest 18 | timeout-minutes: 30 19 | 20 | steps: 21 | - name: Clone repository 22 | uses: actions/checkout@v6 23 | with: 24 | token: ${{ secrets.DENOBOT_PAT }} 25 | 26 | - uses: denoland/setup-deno@v2 27 | - uses: dsherret/rust-toolchain-file@v1 28 | 29 | - name: Tag and release 30 | env: 31 | GITHUB_TOKEN: ${{ secrets.DENOBOT_PAT }} 32 | GH_WORKFLOW_ACTOR: ${{ github.actor }} 33 | run: | 34 | git config user.email "denobot@users.noreply.github.com" 35 | git config user.name "denobot" 36 | deno run -A https://raw.githubusercontent.com/denoland/automation/0.20.0/tasks/publish_release.ts --${{github.event.inputs.releaseKind}} 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018-2025 the Deno authors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | pull_request: 5 | branches: [main] 6 | push: 7 | branches: [main] 8 | tags: 9 | - "*" 10 | workflow_dispatch: 11 | 12 | jobs: 13 | rust: 14 | name: file_test_runner-ubuntu-latest-release 15 | runs-on: ubuntu-latest 16 | timeout-minutes: 30 17 | 18 | env: 19 | CARGO_INCREMENTAL: 0 20 | GH_ACTIONS: 1 21 | RUST_BACKTRACE: full 22 | RUSTFLAGS: -D warnings 23 | 24 | steps: 25 | - name: Clone repository 26 | uses: actions/checkout@v6 27 | 28 | - uses: denoland/setup-deno@v2 29 | - uses: dsherret/rust-toolchain-file@v1 30 | 31 | - uses: Swatinem/rust-cache@v2 32 | with: 33 | save-if: ${{ github.ref == 'refs/heads/main' }} 34 | 35 | - name: Format 36 | run: | 37 | cargo fmt --all -- --check 38 | deno fmt --check 39 | 40 | - name: Lint 41 | run: cargo clippy --all-targets --all-features 42 | 43 | - name: Test 44 | run: cargo test --all-targets --all-features 45 | 46 | - name: Publish 47 | if: | 48 | github.repository == 'denoland/file_test_runner' && 49 | startsWith(github.ref, 'refs/tags/') 50 | env: 51 | CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} 52 | run: | 53 | cargo publish 54 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | pub mod collection; 4 | pub mod reporter; 5 | mod runner; 6 | 7 | use collection::CollectedTest; 8 | pub use runner::*; 9 | 10 | use std::path::Path; 11 | use std::path::PathBuf; 12 | 13 | use collection::CollectOptions; 14 | use collection::collect_tests_or_exit; 15 | use thiserror::Error; 16 | 17 | #[derive(Debug, Error)] 18 | #[error("{:#} ({})", err, path.display())] 19 | pub struct PathedIoError { 20 | path: PathBuf, 21 | err: std::io::Error, 22 | } 23 | 24 | impl PathedIoError { 25 | pub fn new(path: &Path, err: std::io::Error) -> Self { 26 | Self { 27 | path: path.to_path_buf(), 28 | err, 29 | } 30 | } 31 | } 32 | 33 | /// Helper function to collect and run the tests. 34 | pub fn collect_and_run_tests( 35 | collect_options: CollectOptions, 36 | run_options: RunOptions, 37 | run_test: impl (Fn(&CollectedTest) -> TestResult) + Send + Sync + 'static, 38 | ) { 39 | let category = collect_tests_or_exit(collect_options); 40 | run_tests(&category, run_options, run_test) 41 | } 42 | 43 | /// Gets if a `--no-capture` or `--nocapture` flag was provided to the cli args. 44 | pub static NO_CAPTURE: std::sync::LazyLock = 45 | std::sync::LazyLock::new(|| { 46 | std::env::args().any(|arg| arg == "--no-capture" || arg == "--nocapture") 47 | }); 48 | -------------------------------------------------------------------------------- /src/collection/strategies/file_test_mapper.rs: -------------------------------------------------------------------------------- 1 | use std::path::Path; 2 | 3 | use crate::collection::CollectTestsError; 4 | use crate::collection::CollectedCategoryOrTest; 5 | use crate::collection::CollectedTest; 6 | use crate::collection::CollectedTestCategory; 7 | 8 | use super::TestCollectionStrategy; 9 | 10 | /// Maps collected tests into categories or other tests. 11 | /// 12 | /// This is useful if you want to read a file, extract out all the tests, 13 | /// then map the file into a category of tests. 14 | #[derive(Debug, Clone)] 15 | pub struct FileTestMapperStrategy< 16 | TData: Clone + Send + 'static, 17 | TMapper: Fn( 18 | CollectedTest<()>, 19 | ) -> Result, CollectTestsError>, 20 | TBaseStrategy: TestCollectionStrategy<()>, 21 | > { 22 | /// Base strategy to use for collecting files. 23 | pub base_strategy: TBaseStrategy, 24 | /// Map function to map tests to a category or another test. 25 | pub map: TMapper, 26 | } 27 | 28 | impl< 29 | TData: Clone + Send + 'static, 30 | TMapper: Fn( 31 | CollectedTest<()>, 32 | ) -> Result, CollectTestsError>, 33 | TBaseStrategy: TestCollectionStrategy<()>, 34 | > FileTestMapperStrategy 35 | { 36 | fn map_category( 37 | &self, 38 | category: CollectedTestCategory<()>, 39 | ) -> Result, CollectTestsError> { 40 | let mut new_children = Vec::with_capacity(category.children.len()); 41 | for child in category.children { 42 | match child { 43 | CollectedCategoryOrTest::Category(c) => { 44 | new_children 45 | .push(CollectedCategoryOrTest::Category(self.map_category(c)?)); 46 | } 47 | CollectedCategoryOrTest::Test(t) => { 48 | new_children.push((self.map)(t)?); 49 | } 50 | } 51 | } 52 | Ok(CollectedTestCategory { 53 | name: category.name, 54 | path: category.path, 55 | children: new_children, 56 | }) 57 | } 58 | } 59 | 60 | impl< 61 | TData: Clone + Send + 'static, 62 | TMapper: Fn( 63 | CollectedTest<()>, 64 | ) -> Result, CollectTestsError>, 65 | TBaseStrategy: TestCollectionStrategy<()>, 66 | > TestCollectionStrategy 67 | for FileTestMapperStrategy 68 | { 69 | fn collect_tests( 70 | &self, 71 | base: &Path, 72 | ) -> Result, CollectTestsError> { 73 | let category = self.base_strategy.collect_tests(base)?; 74 | self.map_category(category) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # file_test_runner 2 | 3 | File-based test runner for running tests found in files via `cargo test`. 4 | 5 | This does two main steps: 6 | 7 | 1. Collects all files from a specified directory using a provided strategy 8 | (`file_test_runner::collect_tests`). 9 | 1. Runs all the files as tests with a custom test runner 10 | (`file_test_runner::run_tests`). 11 | 12 | The files it collects may be in any format. It's up to you to decide how they 13 | should be structured. 14 | 15 | ## Examples 16 | 17 | - https://github.com/denoland/deno_doc/blob/main/tests/specs_test.rs 18 | - https://github.com/denoland/deno_graph/blob/main/tests/specs_test.rs 19 | - https://github.com/denoland/deno/tree/main/tests/specs 20 | 21 | ## Setup 22 | 23 | 1. Add a `[[test]]` section to your Cargo.toml: 24 | 25 | ```toml 26 | [[test]] 27 | name = "specs" 28 | path = "tests/spec_test.rs" 29 | harness = false 30 | ``` 31 | 32 | 2. Add a `tests/spec_test.rs` file to run the tests with a main function: 33 | 34 | ```rs 35 | use std::panic::AssertUnwindSafe; 36 | 37 | use file_test_runner::collect_and_run_tests; 38 | use file_test_runner::collection::CollectedTest; 39 | use file_test_runner::collection::CollectOptions; 40 | use file_test_runner::collection::strategies::TestPerFileCollectionStrategy; 41 | use file_test_runner::RunOptions; 42 | use file_test_runner::TestResult; 43 | 44 | fn main() { 45 | collect_and_run_tests( 46 | CollectOptions { 47 | base: "tests/specs".into(), 48 | strategy: Box::new(TestPerFileCollectionStrategy { 49 | file_pattern: None 50 | }), 51 | filter_override: None, 52 | }, 53 | // the run options provide a way to set the reporter or parallelism 54 | RunOptions::default(), 55 | // custom function to run the test... 56 | |test| { 57 | // * do something like this 58 | // * or do some checks yourself and return a value like TestResult::Passed 59 | // * or use `TestResult::from_maybe_panic_or_result` to combine both of the above 60 | TestResult::from_maybe_panic(AssertUnwindSafe(|| { 61 | run_test(test); 62 | })) 63 | } 64 | ) 65 | } 66 | 67 | // The `test` object only contains the test name and 68 | // the path to the file on the file system which you can 69 | // then use to determine how to run your test 70 | fn run_test(test: &CollectedTest) { 71 | // Properties: 72 | // * `test.name` - Fully resolved name of the test. 73 | // * `test.path` - Path to the test file this test is associated with. 74 | // * `test.data` - Data associated with the test that may have been set 75 | // by the collection strategy. 76 | 77 | // helper function to get the text 78 | let file_text = test.read_to_string().unwrap(); 79 | 80 | // now you may do whatever with the file text and 81 | // assert it using assert_eq! or whatever 82 | } 83 | ``` 84 | 85 | 3. Add some files to the `tests/specs` directory or within sub directories of 86 | that directory. 87 | 88 | 4. Run `cargo test` to run the tests. Filtering should work OOTB. 89 | -------------------------------------------------------------------------------- /src/collection/strategies/test_per_file.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | use std::path::Path; 4 | 5 | use regex::Regex; 6 | 7 | use crate::PathedIoError; 8 | use crate::collection::CollectTestsError; 9 | use crate::collection::CollectedCategoryOrTest; 10 | use crate::collection::CollectedTest; 11 | use crate::collection::CollectedTestCategory; 12 | 13 | use super::TestCollectionStrategy; 14 | use super::helpers::append_to_category_name; 15 | use super::helpers::read_dir_entries; 16 | 17 | /// All the files in every sub directory will be traversed 18 | /// to find tests that match the pattern. 19 | /// 20 | /// Provide `None` to match all files. 21 | /// 22 | /// Note: This ignores readme.md files and hidden directories 23 | /// starting with a period. 24 | #[derive(Debug, Clone, Default)] 25 | pub struct TestPerFileCollectionStrategy { 26 | /// The regular expression to match. 27 | pub file_pattern: Option, 28 | } 29 | 30 | impl TestCollectionStrategy<()> for TestPerFileCollectionStrategy { 31 | fn collect_tests( 32 | &self, 33 | base: &Path, 34 | ) -> Result, CollectTestsError> { 35 | fn collect_test_per_file( 36 | category_name: &str, 37 | dir_path: &Path, 38 | pattern: Option<&Regex>, 39 | ) -> Result>, CollectTestsError> { 40 | let mut tests = vec![]; 41 | 42 | for entry in read_dir_entries(dir_path)? { 43 | let path = entry.path(); 44 | let file_type = entry 45 | .file_type() 46 | .map_err(|err| PathedIoError::new(&path, err))?; 47 | if file_type.is_dir() { 48 | let category_name = append_to_category_name( 49 | category_name, 50 | &path.file_name().unwrap().to_string_lossy(), 51 | ); 52 | let children = collect_test_per_file(&category_name, &path, pattern)?; 53 | if !children.is_empty() { 54 | tests.push(CollectedCategoryOrTest::Category( 55 | CollectedTestCategory { 56 | name: category_name, 57 | path, 58 | children, 59 | }, 60 | )); 61 | } 62 | } else if file_type.is_file() { 63 | if let Some(pattern) = pattern 64 | && !pattern.is_match(&path.to_string_lossy()) 65 | { 66 | continue; 67 | } 68 | let test = CollectedTest { 69 | name: append_to_category_name( 70 | category_name, 71 | &path.file_stem().unwrap().to_string_lossy(), 72 | ), 73 | path, 74 | line_and_column: None, 75 | data: (), 76 | }; 77 | tests.push(CollectedCategoryOrTest::Test(test)); 78 | } 79 | } 80 | 81 | Ok(tests) 82 | } 83 | 84 | let pattern = match self.file_pattern.as_ref() { 85 | Some(pattern) => Some(Regex::new(pattern).map_err(anyhow::Error::from)?), 86 | None => None, 87 | }; 88 | let category_name = base.file_name().unwrap().to_string_lossy(); 89 | let children = 90 | collect_test_per_file(&category_name, base, pattern.as_ref())?; 91 | Ok(CollectedTestCategory { 92 | name: category_name.to_string(), 93 | path: base.to_path_buf(), 94 | children, 95 | }) 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/collection/strategies/test_per_directory.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | use std::path::Path; 4 | 5 | use crate::PathedIoError; 6 | use crate::collection::CollectTestsError; 7 | use crate::collection::CollectedCategoryOrTest; 8 | use crate::collection::CollectedTest; 9 | use crate::collection::CollectedTestCategory; 10 | 11 | use super::TestCollectionStrategy; 12 | use super::helpers::append_to_category_name; 13 | use super::helpers::read_dir_entries; 14 | 15 | /// Recursively searches directories finding the provided 16 | /// filename. If a directory sub tree does not contain the file 17 | /// then an error is raised. Once a matching test file is found 18 | /// in a directory, traversing will stop. 19 | /// 20 | /// Note: This ignores hidden directories starting with a period. 21 | #[derive(Debug, Clone)] 22 | pub struct TestPerDirectoryCollectionStrategy { 23 | /// The file name to search for in each directory. 24 | /// 25 | /// Example: `__test__.jsonc` 26 | pub file_name: String, 27 | } 28 | 29 | impl TestCollectionStrategy<()> for TestPerDirectoryCollectionStrategy { 30 | fn collect_tests( 31 | &self, 32 | base: &Path, 33 | ) -> Result, CollectTestsError> { 34 | fn collect_test_per_directory( 35 | category_name: &str, 36 | dir_path: &Path, 37 | dir_test_file_name: &str, 38 | ) -> Result>, CollectTestsError> { 39 | let mut tests = vec![]; 40 | 41 | let mut found_dir = false; 42 | let mut is_dir_empty = true; 43 | for entry in read_dir_entries(dir_path)? { 44 | is_dir_empty = false; 45 | let path = entry.path(); 46 | let file_type = entry 47 | .file_type() 48 | .map_err(|err| PathedIoError::new(&path, err))?; 49 | if file_type.is_dir() { 50 | found_dir = true; 51 | let test_file_path = path.join(dir_test_file_name); 52 | if test_file_path.exists() { 53 | let test = CollectedTest { 54 | name: append_to_category_name( 55 | category_name, 56 | &path.file_name().unwrap().to_string_lossy(), 57 | ), 58 | path: test_file_path, 59 | line_and_column: None, 60 | data: (), 61 | }; 62 | tests.push(CollectedCategoryOrTest::Test(test)); 63 | } else { 64 | let category_name = append_to_category_name( 65 | category_name, 66 | &path.file_name().unwrap().to_string_lossy(), 67 | ); 68 | let children = collect_test_per_directory( 69 | &category_name, 70 | &path, 71 | dir_test_file_name, 72 | )?; 73 | if !children.is_empty() { 74 | tests.push(CollectedCategoryOrTest::Category( 75 | CollectedTestCategory { 76 | name: category_name, 77 | path, 78 | children, 79 | }, 80 | )); 81 | } 82 | } 83 | } 84 | } 85 | 86 | // Error when the directory file can't be found in order to catch people 87 | // accidentally not naming the test file correctly 88 | // (ex. `__test__.json` instead of `__test__.jsonc` in Deno's case) 89 | if !found_dir && !is_dir_empty { 90 | return Err(anyhow::anyhow!("Could not find '{}' in directory tree '{}'. Perhaps the file is named incorrectly?", dir_test_file_name, dir_path.display()).into()); 91 | } 92 | 93 | Ok(tests) 94 | } 95 | 96 | let category_name = base.file_name().unwrap().to_string_lossy(); 97 | let children = 98 | collect_test_per_directory(&category_name, base, &self.file_name)?; 99 | Ok(CollectedTestCategory { 100 | name: category_name.to_string(), 101 | path: base.to_path_buf(), 102 | children, 103 | }) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 4 4 | 5 | [[package]] 6 | name = "aho-corasick" 7 | version = "1.1.3" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" 10 | dependencies = [ 11 | "memchr", 12 | ] 13 | 14 | [[package]] 15 | name = "anyhow" 16 | version = "1.0.93" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" 19 | 20 | [[package]] 21 | name = "autocfg" 22 | version = "1.4.0" 23 | source = "registry+https://github.com/rust-lang/crates.io-index" 24 | checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" 25 | 26 | [[package]] 27 | name = "bitflags" 28 | version = "2.6.0" 29 | source = "registry+https://github.com/rust-lang/crates.io-index" 30 | checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" 31 | 32 | [[package]] 33 | name = "cfg-if" 34 | version = "1.0.0" 35 | source = "registry+https://github.com/rust-lang/crates.io-index" 36 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 37 | 38 | [[package]] 39 | name = "crossbeam-channel" 40 | version = "0.5.13" 41 | source = "registry+https://github.com/rust-lang/crates.io-index" 42 | checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" 43 | dependencies = [ 44 | "crossbeam-utils", 45 | ] 46 | 47 | [[package]] 48 | name = "crossbeam-deque" 49 | version = "0.8.6" 50 | source = "registry+https://github.com/rust-lang/crates.io-index" 51 | checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" 52 | dependencies = [ 53 | "crossbeam-epoch", 54 | "crossbeam-utils", 55 | ] 56 | 57 | [[package]] 58 | name = "crossbeam-epoch" 59 | version = "0.9.18" 60 | source = "registry+https://github.com/rust-lang/crates.io-index" 61 | checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" 62 | dependencies = [ 63 | "crossbeam-utils", 64 | ] 65 | 66 | [[package]] 67 | name = "crossbeam-utils" 68 | version = "0.8.20" 69 | source = "registry+https://github.com/rust-lang/crates.io-index" 70 | checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" 71 | 72 | [[package]] 73 | name = "deno_terminal" 74 | version = "0.2.0" 75 | source = "registry+https://github.com/rust-lang/crates.io-index" 76 | checksum = "daef12499e89ee99e51ad6000a91f600d3937fb028ad4918af76810c5bc9e0d5" 77 | dependencies = [ 78 | "once_cell", 79 | "termcolor", 80 | ] 81 | 82 | [[package]] 83 | name = "either" 84 | version = "1.15.0" 85 | source = "registry+https://github.com/rust-lang/crates.io-index" 86 | checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" 87 | 88 | [[package]] 89 | name = "file_test_runner" 90 | version = "0.11.1" 91 | dependencies = [ 92 | "anyhow", 93 | "crossbeam-channel", 94 | "deno_terminal", 95 | "parking_lot", 96 | "rayon", 97 | "regex", 98 | "thiserror", 99 | ] 100 | 101 | [[package]] 102 | name = "libc" 103 | version = "0.2.167" 104 | source = "registry+https://github.com/rust-lang/crates.io-index" 105 | checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" 106 | 107 | [[package]] 108 | name = "lock_api" 109 | version = "0.4.12" 110 | source = "registry+https://github.com/rust-lang/crates.io-index" 111 | checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" 112 | dependencies = [ 113 | "autocfg", 114 | "scopeguard", 115 | ] 116 | 117 | [[package]] 118 | name = "memchr" 119 | version = "2.7.4" 120 | source = "registry+https://github.com/rust-lang/crates.io-index" 121 | checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" 122 | 123 | [[package]] 124 | name = "once_cell" 125 | version = "1.20.2" 126 | source = "registry+https://github.com/rust-lang/crates.io-index" 127 | checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" 128 | 129 | [[package]] 130 | name = "parking_lot" 131 | version = "0.12.3" 132 | source = "registry+https://github.com/rust-lang/crates.io-index" 133 | checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" 134 | dependencies = [ 135 | "lock_api", 136 | "parking_lot_core", 137 | ] 138 | 139 | [[package]] 140 | name = "parking_lot_core" 141 | version = "0.9.10" 142 | source = "registry+https://github.com/rust-lang/crates.io-index" 143 | checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" 144 | dependencies = [ 145 | "cfg-if", 146 | "libc", 147 | "redox_syscall", 148 | "smallvec", 149 | "windows-targets", 150 | ] 151 | 152 | [[package]] 153 | name = "proc-macro2" 154 | version = "1.0.92" 155 | source = "registry+https://github.com/rust-lang/crates.io-index" 156 | checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" 157 | dependencies = [ 158 | "unicode-ident", 159 | ] 160 | 161 | [[package]] 162 | name = "quote" 163 | version = "1.0.37" 164 | source = "registry+https://github.com/rust-lang/crates.io-index" 165 | checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" 166 | dependencies = [ 167 | "proc-macro2", 168 | ] 169 | 170 | [[package]] 171 | name = "rayon" 172 | version = "1.11.0" 173 | source = "registry+https://github.com/rust-lang/crates.io-index" 174 | checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" 175 | dependencies = [ 176 | "either", 177 | "rayon-core", 178 | ] 179 | 180 | [[package]] 181 | name = "rayon-core" 182 | version = "1.13.0" 183 | source = "registry+https://github.com/rust-lang/crates.io-index" 184 | checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" 185 | dependencies = [ 186 | "crossbeam-deque", 187 | "crossbeam-utils", 188 | ] 189 | 190 | [[package]] 191 | name = "redox_syscall" 192 | version = "0.5.7" 193 | source = "registry+https://github.com/rust-lang/crates.io-index" 194 | checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" 195 | dependencies = [ 196 | "bitflags", 197 | ] 198 | 199 | [[package]] 200 | name = "regex" 201 | version = "1.11.1" 202 | source = "registry+https://github.com/rust-lang/crates.io-index" 203 | checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" 204 | dependencies = [ 205 | "aho-corasick", 206 | "memchr", 207 | "regex-automata", 208 | "regex-syntax", 209 | ] 210 | 211 | [[package]] 212 | name = "regex-automata" 213 | version = "0.4.9" 214 | source = "registry+https://github.com/rust-lang/crates.io-index" 215 | checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" 216 | dependencies = [ 217 | "aho-corasick", 218 | "memchr", 219 | "regex-syntax", 220 | ] 221 | 222 | [[package]] 223 | name = "regex-syntax" 224 | version = "0.8.5" 225 | source = "registry+https://github.com/rust-lang/crates.io-index" 226 | checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" 227 | 228 | [[package]] 229 | name = "scopeguard" 230 | version = "1.2.0" 231 | source = "registry+https://github.com/rust-lang/crates.io-index" 232 | checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" 233 | 234 | [[package]] 235 | name = "smallvec" 236 | version = "1.13.2" 237 | source = "registry+https://github.com/rust-lang/crates.io-index" 238 | checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" 239 | 240 | [[package]] 241 | name = "syn" 242 | version = "2.0.90" 243 | source = "registry+https://github.com/rust-lang/crates.io-index" 244 | checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" 245 | dependencies = [ 246 | "proc-macro2", 247 | "quote", 248 | "unicode-ident", 249 | ] 250 | 251 | [[package]] 252 | name = "termcolor" 253 | version = "1.4.1" 254 | source = "registry+https://github.com/rust-lang/crates.io-index" 255 | checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" 256 | dependencies = [ 257 | "winapi-util", 258 | ] 259 | 260 | [[package]] 261 | name = "thiserror" 262 | version = "2.0.4" 263 | source = "registry+https://github.com/rust-lang/crates.io-index" 264 | checksum = "2f49a1853cf82743e3b7950f77e0f4d622ca36cf4317cba00c767838bac8d490" 265 | dependencies = [ 266 | "thiserror-impl", 267 | ] 268 | 269 | [[package]] 270 | name = "thiserror-impl" 271 | version = "2.0.4" 272 | source = "registry+https://github.com/rust-lang/crates.io-index" 273 | checksum = "8381894bb3efe0c4acac3ded651301ceee58a15d47c2e34885ed1908ad667061" 274 | dependencies = [ 275 | "proc-macro2", 276 | "quote", 277 | "syn", 278 | ] 279 | 280 | [[package]] 281 | name = "unicode-ident" 282 | version = "1.0.14" 283 | source = "registry+https://github.com/rust-lang/crates.io-index" 284 | checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" 285 | 286 | [[package]] 287 | name = "winapi-util" 288 | version = "0.1.9" 289 | source = "registry+https://github.com/rust-lang/crates.io-index" 290 | checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" 291 | dependencies = [ 292 | "windows-sys", 293 | ] 294 | 295 | [[package]] 296 | name = "windows-sys" 297 | version = "0.59.0" 298 | source = "registry+https://github.com/rust-lang/crates.io-index" 299 | checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" 300 | dependencies = [ 301 | "windows-targets", 302 | ] 303 | 304 | [[package]] 305 | name = "windows-targets" 306 | version = "0.52.6" 307 | source = "registry+https://github.com/rust-lang/crates.io-index" 308 | checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" 309 | dependencies = [ 310 | "windows_aarch64_gnullvm", 311 | "windows_aarch64_msvc", 312 | "windows_i686_gnu", 313 | "windows_i686_gnullvm", 314 | "windows_i686_msvc", 315 | "windows_x86_64_gnu", 316 | "windows_x86_64_gnullvm", 317 | "windows_x86_64_msvc", 318 | ] 319 | 320 | [[package]] 321 | name = "windows_aarch64_gnullvm" 322 | version = "0.52.6" 323 | source = "registry+https://github.com/rust-lang/crates.io-index" 324 | checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" 325 | 326 | [[package]] 327 | name = "windows_aarch64_msvc" 328 | version = "0.52.6" 329 | source = "registry+https://github.com/rust-lang/crates.io-index" 330 | checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" 331 | 332 | [[package]] 333 | name = "windows_i686_gnu" 334 | version = "0.52.6" 335 | source = "registry+https://github.com/rust-lang/crates.io-index" 336 | checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" 337 | 338 | [[package]] 339 | name = "windows_i686_gnullvm" 340 | version = "0.52.6" 341 | source = "registry+https://github.com/rust-lang/crates.io-index" 342 | checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" 343 | 344 | [[package]] 345 | name = "windows_i686_msvc" 346 | version = "0.52.6" 347 | source = "registry+https://github.com/rust-lang/crates.io-index" 348 | checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" 349 | 350 | [[package]] 351 | name = "windows_x86_64_gnu" 352 | version = "0.52.6" 353 | source = "registry+https://github.com/rust-lang/crates.io-index" 354 | checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" 355 | 356 | [[package]] 357 | name = "windows_x86_64_gnullvm" 358 | version = "0.52.6" 359 | source = "registry+https://github.com/rust-lang/crates.io-index" 360 | checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" 361 | 362 | [[package]] 363 | name = "windows_x86_64_msvc" 364 | version = "0.52.6" 365 | source = "registry+https://github.com/rust-lang/crates.io-index" 366 | checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" 367 | -------------------------------------------------------------------------------- /src/runner.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | use std::cell::RefCell; 4 | use std::num::NonZeroUsize; 5 | use std::sync::Arc; 6 | use std::time::Duration; 7 | use std::time::Instant; 8 | 9 | use parking_lot::Mutex; 10 | use rayon::ThreadPool; 11 | 12 | use crate::NO_CAPTURE; 13 | use crate::collection::CollectedCategoryOrTest; 14 | use crate::collection::CollectedTest; 15 | use crate::collection::CollectedTestCategory; 16 | use crate::reporter::LogReporter; 17 | use crate::reporter::Reporter; 18 | use crate::reporter::ReporterContext; 19 | use crate::reporter::ReporterFailure; 20 | 21 | type RunTestFunc = 22 | Arc) -> TestResult) + Send + Sync>; 23 | 24 | struct Context { 25 | failures: Vec>, 26 | parallelism: NonZeroUsize, 27 | run_test: RunTestFunc, 28 | reporter: Arc>, 29 | pool: ThreadPool, 30 | } 31 | 32 | static GLOBAL_PANIC_HOOK_COUNT: Mutex = Mutex::new(0); 33 | 34 | type PanicHook = Box; 35 | 36 | thread_local! { 37 | static LOCAL_PANIC_HOOK: RefCell> = RefCell::new(None); 38 | } 39 | 40 | #[derive(Debug, Clone)] 41 | pub struct SubTestResult { 42 | pub name: String, 43 | pub result: TestResult, 44 | } 45 | 46 | #[must_use] 47 | #[derive(Debug, Clone)] 48 | pub enum TestResult { 49 | /// Test passed. 50 | Passed { 51 | /// Optional duration to report. 52 | duration: Option, 53 | }, 54 | /// Test was ignored. 55 | Ignored, 56 | /// Test failed, returning the captured output of the test. 57 | Failed { 58 | /// Optional duration to report. 59 | duration: Option, 60 | /// Test failure output that should be shown to the user. 61 | output: Vec, 62 | }, 63 | /// Multiple sub tests were run. 64 | SubTests { 65 | /// Optional duration to report. 66 | duration: Option, 67 | sub_tests: Vec, 68 | }, 69 | } 70 | 71 | impl TestResult { 72 | pub fn duration(&self) -> Option { 73 | match self { 74 | TestResult::Passed { duration } => *duration, 75 | TestResult::Ignored => None, 76 | TestResult::Failed { duration, .. } => *duration, 77 | TestResult::SubTests { duration, .. } => *duration, 78 | } 79 | } 80 | 81 | pub fn with_duration(self, duration: Duration) -> Self { 82 | match self { 83 | TestResult::Passed { duration: _ } => TestResult::Passed { 84 | duration: Some(duration), 85 | }, 86 | TestResult::Ignored => TestResult::Ignored, 87 | TestResult::Failed { 88 | duration: _, 89 | output, 90 | } => TestResult::Failed { 91 | duration: Some(duration), 92 | output, 93 | }, 94 | TestResult::SubTests { 95 | duration: _, 96 | sub_tests, 97 | } => TestResult::SubTests { 98 | duration: Some(duration), 99 | sub_tests, 100 | }, 101 | } 102 | } 103 | 104 | pub fn is_failed(&self) -> bool { 105 | match self { 106 | TestResult::Passed { .. } | TestResult::Ignored => false, 107 | TestResult::Failed { .. } => true, 108 | TestResult::SubTests { sub_tests, .. } => { 109 | sub_tests.iter().any(|s| s.result.is_failed()) 110 | } 111 | } 112 | } 113 | 114 | /// Allows using a closure that may panic, capturing the panic message and 115 | /// returning it as a TestResult::Failed. 116 | /// 117 | /// Ensure the code is unwind safe and use with `AssertUnwindSafe(|| { /* test code */ })`. 118 | pub fn from_maybe_panic( 119 | func: impl FnOnce() + std::panic::UnwindSafe, 120 | ) -> Self { 121 | Self::from_maybe_panic_or_result(|| { 122 | func(); 123 | TestResult::Passed { duration: None } 124 | }) 125 | } 126 | 127 | /// Allows using a closure that may panic, capturing the panic message and 128 | /// returning it as a TestResult::Failed. If a panic does not occur, uses 129 | /// the returned TestResult. 130 | /// 131 | /// Ensure the code is unwind safe and use with `AssertUnwindSafe(|| { /* test code */ })`. 132 | pub fn from_maybe_panic_or_result( 133 | func: impl FnOnce() -> TestResult + std::panic::UnwindSafe, 134 | ) -> Self { 135 | // increment the panic hook 136 | { 137 | let mut hook_count = GLOBAL_PANIC_HOOK_COUNT.lock(); 138 | if *hook_count == 0 { 139 | let _ = std::panic::take_hook(); 140 | std::panic::set_hook(Box::new(|info| { 141 | LOCAL_PANIC_HOOK.with(|hook| { 142 | if let Some(hook) = &*hook.borrow() { 143 | hook(info); 144 | } 145 | }); 146 | })); 147 | } 148 | *hook_count += 1; 149 | drop(hook_count); // explicit for clarity, drop after setting the hook 150 | } 151 | 152 | let panic_message = Arc::new(Mutex::new(Vec::::new())); 153 | 154 | let previous_panic_hook = LOCAL_PANIC_HOOK.with(|hook| { 155 | let panic_message = panic_message.clone(); 156 | hook.borrow_mut().replace(Box::new(move |info| { 157 | let backtrace = capture_backtrace(); 158 | panic_message.lock().extend( 159 | format!( 160 | "{}{}", 161 | info, 162 | backtrace 163 | .map(|trace| format!("\n{}", trace)) 164 | .unwrap_or_default() 165 | ) 166 | .into_bytes(), 167 | ); 168 | })) 169 | }); 170 | 171 | let result = std::panic::catch_unwind(func); 172 | 173 | // restore or clear the local panic hook 174 | LOCAL_PANIC_HOOK.with(|hook| { 175 | *hook.borrow_mut() = previous_panic_hook; 176 | }); 177 | 178 | // decrement the global panic hook 179 | { 180 | let mut hook_count = GLOBAL_PANIC_HOOK_COUNT.lock(); 181 | *hook_count -= 1; 182 | if *hook_count == 0 { 183 | let _ = std::panic::take_hook(); 184 | } 185 | drop(hook_count); // explicit for clarity, drop after taking the hook 186 | } 187 | 188 | result.unwrap_or_else(|_| TestResult::Failed { 189 | duration: None, 190 | output: panic_message.lock().clone(), 191 | }) 192 | } 193 | } 194 | 195 | fn capture_backtrace() -> Option { 196 | let backtrace = std::backtrace::Backtrace::capture(); 197 | if backtrace.status() != std::backtrace::BacktraceStatus::Captured { 198 | return None; 199 | } 200 | let text = format!("{}", backtrace); 201 | // strip the code in this crate from the start of the backtrace 202 | let lines = text.lines().collect::>(); 203 | let last_position = lines 204 | .iter() 205 | .position(|line| line.contains("core::panicking::panic_fmt")); 206 | Some(match last_position { 207 | Some(position) => lines[position + 2..].join("\n"), 208 | None => text, 209 | }) 210 | } 211 | 212 | #[derive(Clone)] 213 | pub struct RunOptions { 214 | pub parallelism: NonZeroUsize, 215 | pub reporter: Arc>, 216 | } 217 | 218 | impl Default for RunOptions { 219 | fn default() -> Self { 220 | Self { 221 | parallelism: RunOptions::default_parallelism(), 222 | reporter: Arc::new(LogReporter::default()), 223 | } 224 | } 225 | } 226 | 227 | impl RunOptions<()> { 228 | pub fn default_parallelism() -> NonZeroUsize { 229 | NonZeroUsize::new(if *NO_CAPTURE { 230 | 1 231 | } else { 232 | std::cmp::max( 233 | 1, 234 | std::env::var("FILE_TEST_RUNNER_PARALLELISM") 235 | .ok() 236 | .and_then(|v| v.parse().ok()) 237 | .unwrap_or_else(|| { 238 | std::thread::available_parallelism() 239 | .map(|v| v.get()) 240 | .unwrap_or(2) 241 | - 1 242 | }), 243 | ) 244 | }) 245 | .unwrap() 246 | } 247 | } 248 | 249 | pub fn run_tests( 250 | category: &CollectedTestCategory, 251 | options: RunOptions, 252 | run_test: impl (Fn(&CollectedTest) -> TestResult) + Send + Sync + 'static, 253 | ) { 254 | let total_tests = category.test_count(); 255 | if total_tests == 0 { 256 | return; // no tests to run because they were filtered out 257 | } 258 | 259 | let run_test = Arc::new(run_test); 260 | let max_parallelism = options.parallelism; 261 | 262 | // Create a rayon thread pool 263 | let pool = rayon::ThreadPoolBuilder::new() 264 | // +1 is one thread that drives tests into the pool of receivers 265 | .num_threads(max_parallelism.get() + 1) 266 | .build() 267 | .expect("Failed to create thread pool"); 268 | 269 | let mut context = Context { 270 | failures: Vec::new(), 271 | run_test, 272 | parallelism: options.parallelism, 273 | reporter: options.reporter, 274 | pool, 275 | }; 276 | run_category(category, &mut context); 277 | 278 | context 279 | .reporter 280 | .report_failures(&context.failures, total_tests); 281 | if !context.failures.is_empty() { 282 | panic!("{} failed of {}", context.failures.len(), total_tests); 283 | } 284 | } 285 | 286 | fn run_category( 287 | category: &CollectedTestCategory, 288 | context: &mut Context, 289 | ) { 290 | let mut tests = Vec::new(); 291 | let mut categories = Vec::new(); 292 | for child in &category.children { 293 | match child { 294 | CollectedCategoryOrTest::Category(c) => { 295 | categories.push(c); 296 | } 297 | CollectedCategoryOrTest::Test(t) => { 298 | tests.push(t.clone()); 299 | } 300 | } 301 | } 302 | 303 | if !tests.is_empty() { 304 | run_tests_for_category(category, tests, context); 305 | } 306 | 307 | for category in categories { 308 | run_category(category, context); 309 | } 310 | } 311 | 312 | fn run_tests_for_category( 313 | category: &CollectedTestCategory, 314 | tests: Vec>, 315 | context: &mut Context, 316 | ) { 317 | enum SendMessage { 318 | Start { 319 | test: CollectedTest, 320 | }, 321 | Result { 322 | test: CollectedTest, 323 | duration: Duration, 324 | result: TestResult, 325 | }, 326 | } 327 | 328 | if tests.is_empty() { 329 | return; // ignore empty categories if they exist for some reason 330 | } 331 | 332 | let reporter = &context.reporter; 333 | let max_parallelism = context.parallelism.get(); 334 | let reporter_context = ReporterContext { 335 | is_parallel: max_parallelism > 1, 336 | }; 337 | reporter.report_category_start(category, &reporter_context); 338 | 339 | let receive_receiver = { 340 | let (receiver_sender, receive_receiver) = 341 | crossbeam_channel::unbounded::>(); 342 | let (send_sender, send_receiver) = 343 | crossbeam_channel::bounded::>(max_parallelism); 344 | for _ in 0..max_parallelism { 345 | let send_receiver = send_receiver.clone(); 346 | let sender = receiver_sender.clone(); 347 | let run_test = context.run_test.clone(); 348 | context.pool.spawn(move || { 349 | let run_test = &run_test; 350 | while let Ok(test) = send_receiver.recv() { 351 | let start = Instant::now(); 352 | // it's more deterministic to send this back to the main thread 353 | // for when the parallelism is 1 354 | _ = sender.send(SendMessage::Start { test: test.clone() }); 355 | let result = (run_test)(&test); 356 | if sender 357 | .send(SendMessage::Result { 358 | test, 359 | duration: start.elapsed(), 360 | result, 361 | }) 362 | .is_err() 363 | { 364 | return; 365 | } 366 | } 367 | }); 368 | } 369 | 370 | context.pool.spawn(move || { 371 | for test in tests { 372 | if send_sender.send(test).is_err() { 373 | return; // receiver dropped due to fail fast 374 | } 375 | } 376 | }); 377 | 378 | receive_receiver 379 | }; 380 | 381 | while let Ok(message) = receive_receiver.recv() { 382 | match message { 383 | SendMessage::Start { test } => { 384 | reporter.report_test_start(&test, &reporter_context) 385 | } 386 | SendMessage::Result { 387 | test, 388 | duration, 389 | result, 390 | } => { 391 | reporter.report_test_end(&test, duration, &result, &reporter_context); 392 | let is_failure = result.is_failed(); 393 | let failure_output = collect_failure_output(result); 394 | if is_failure { 395 | context.failures.push(ReporterFailure { 396 | test, 397 | output: failure_output, 398 | }); 399 | } 400 | } 401 | } 402 | } 403 | 404 | reporter.report_category_end(category, &reporter_context); 405 | } 406 | 407 | fn collect_failure_output(result: TestResult) -> Vec { 408 | fn output_sub_tests( 409 | sub_tests: &[SubTestResult], 410 | failure_output: &mut Vec, 411 | ) { 412 | for sub_test in sub_tests { 413 | match &sub_test.result { 414 | TestResult::Passed { .. } | TestResult::Ignored => {} 415 | TestResult::Failed { output, .. } => { 416 | if !failure_output.is_empty() { 417 | failure_output.push(b'\n'); 418 | } 419 | failure_output.extend(output); 420 | } 421 | TestResult::SubTests { sub_tests, .. } => { 422 | if !sub_tests.is_empty() { 423 | output_sub_tests(sub_tests, failure_output); 424 | } 425 | } 426 | } 427 | } 428 | } 429 | 430 | let mut failure_output = Vec::new(); 431 | match result { 432 | TestResult::Passed { .. } | TestResult::Ignored => {} 433 | TestResult::Failed { output, .. } => { 434 | failure_output = output; 435 | } 436 | TestResult::SubTests { sub_tests, .. } => { 437 | output_sub_tests(&sub_tests, &mut failure_output); 438 | } 439 | } 440 | 441 | failure_output 442 | } 443 | 444 | #[cfg(test)] 445 | mod test { 446 | use super::*; 447 | 448 | #[test] 449 | fn test_collect_failure_output_failed() { 450 | let failure_output = collect_failure_output(super::TestResult::Failed { 451 | duration: None, 452 | output: b"error".to_vec(), 453 | }); 454 | assert_eq!(failure_output, b"error"); 455 | } 456 | 457 | #[test] 458 | fn test_collect_failure_output_sub_tests() { 459 | let failure_output = collect_failure_output(super::TestResult::SubTests { 460 | duration: None, 461 | sub_tests: vec![ 462 | super::SubTestResult { 463 | name: "step1".to_string(), 464 | result: super::TestResult::Passed { duration: None }, 465 | }, 466 | super::SubTestResult { 467 | name: "step2".to_string(), 468 | result: super::TestResult::Failed { 469 | duration: None, 470 | output: b"error1".to_vec(), 471 | }, 472 | }, 473 | super::SubTestResult { 474 | name: "step3".to_string(), 475 | result: super::TestResult::Failed { 476 | duration: None, 477 | output: b"error2".to_vec(), 478 | }, 479 | }, 480 | super::SubTestResult { 481 | name: "step4".to_string(), 482 | result: super::TestResult::SubTests { 483 | duration: None, 484 | sub_tests: vec![ 485 | super::SubTestResult { 486 | name: "sub-step1".to_string(), 487 | result: super::TestResult::Passed { duration: None }, 488 | }, 489 | super::SubTestResult { 490 | name: "sub-step2".to_string(), 491 | result: super::TestResult::Failed { 492 | duration: None, 493 | output: b"error3".to_vec(), 494 | }, 495 | }, 496 | ], 497 | }, 498 | }, 499 | ], 500 | }); 501 | 502 | assert_eq!( 503 | String::from_utf8(failure_output).unwrap(), 504 | "error1\nerror2\nerror3" 505 | ); 506 | } 507 | } 508 | -------------------------------------------------------------------------------- /src/reporter.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | use std::collections::HashMap; 4 | use std::sync::Arc; 5 | use std::sync::mpsc::RecvTimeoutError; 6 | use std::sync::mpsc::channel; 7 | use std::time::Duration; 8 | use std::time::Instant; 9 | 10 | use deno_terminal::colors; 11 | use parking_lot::Mutex; 12 | 13 | use crate::NO_CAPTURE; 14 | use crate::SubTestResult; 15 | use crate::TestResult; 16 | use crate::collection::CollectedTest; 17 | use crate::collection::CollectedTestCategory; 18 | 19 | #[derive(Clone)] 20 | pub struct ReporterContext { 21 | pub is_parallel: bool, 22 | } 23 | 24 | pub struct ReporterFailure { 25 | pub test: CollectedTest, 26 | pub output: Vec, 27 | } 28 | 29 | pub trait Reporter: Send + Sync { 30 | fn report_category_start( 31 | &self, 32 | category: &CollectedTestCategory, 33 | context: &ReporterContext, 34 | ); 35 | fn report_category_end( 36 | &self, 37 | category: &CollectedTestCategory, 38 | context: &ReporterContext, 39 | ); 40 | fn report_test_start( 41 | &self, 42 | test: &CollectedTest, 43 | context: &ReporterContext, 44 | ); 45 | fn report_test_end( 46 | &self, 47 | test: &CollectedTest, 48 | duration: Duration, 49 | result: &TestResult, 50 | context: &ReporterContext, 51 | ); 52 | fn report_failures( 53 | &self, 54 | failures: &[ReporterFailure], 55 | total_tests: usize, 56 | ); 57 | } 58 | 59 | #[derive(Debug)] 60 | pub struct LogReporter { 61 | pending_tests: Arc>>, 62 | _tx: std::sync::mpsc::Sender<()>, 63 | } 64 | 65 | impl Default for LogReporter { 66 | fn default() -> Self { 67 | let (tx, rx) = channel(); 68 | let pending_tests: Arc>> = 69 | Default::default(); 70 | std::thread::spawn({ 71 | let pending_tests = pending_tests.clone(); 72 | move || { 73 | loop { 74 | match rx.recv_timeout(Duration::from_millis(1_000)) { 75 | Err(RecvTimeoutError::Timeout) => { 76 | let mut tests_to_alert = Vec::new(); 77 | { 78 | let mut data = pending_tests.lock(); 79 | data.retain(|test_name, instant| { 80 | if instant.elapsed().as_secs() > 60 { 81 | tests_to_alert.push(test_name.clone()); 82 | false 83 | } else { 84 | true 85 | } 86 | }); 87 | } 88 | let stderr = &mut std::io::stderr(); 89 | for test_name in tests_to_alert { 90 | let _ = LogReporter::write_report_long_running_test( 91 | stderr, &test_name, 92 | ); 93 | } 94 | } 95 | _ => { 96 | return; 97 | } 98 | } 99 | } 100 | } 101 | }); 102 | Self { 103 | pending_tests, 104 | _tx: tx, 105 | } 106 | } 107 | } 108 | 109 | impl LogReporter { 110 | pub fn write_report_category_start( 111 | writer: &mut W, 112 | category: &CollectedTestCategory, 113 | ) -> std::io::Result<()> { 114 | writeln!(writer)?; 115 | writeln!( 116 | writer, 117 | " {} {}", 118 | colors::green_bold("Running"), 119 | category.name 120 | )?; 121 | writeln!(writer)?; 122 | Ok(()) 123 | } 124 | 125 | pub fn write_report_test_start( 126 | writer: &mut W, 127 | test: &CollectedTest, 128 | context: &ReporterContext, 129 | ) -> std::io::Result<()> { 130 | if !context.is_parallel { 131 | if *NO_CAPTURE { 132 | writeln!(writer, "test {} ...", test.name)?; 133 | } else { 134 | write!(writer, "test {} ... ", test.name)?; 135 | } 136 | } 137 | Ok(()) 138 | } 139 | 140 | pub fn write_report_test_end( 141 | writer: &mut W, 142 | test: &CollectedTest, 143 | duration: Duration, 144 | result: &TestResult, 145 | context: &ReporterContext, 146 | ) -> std::io::Result<()> { 147 | if context.is_parallel { 148 | write!(writer, "test {} ... ", test.name)?; 149 | } 150 | Self::write_end_test_message(writer, result, duration)?; 151 | Ok(()) 152 | } 153 | 154 | pub fn write_end_test_message( 155 | writer: &mut W, 156 | result: &TestResult, 157 | duration: Duration, 158 | ) -> std::io::Result<()> { 159 | fn output_sub_tests( 160 | writer: &mut W, 161 | indent: &str, 162 | sub_tests: &[SubTestResult], 163 | ) -> std::io::Result<()> { 164 | for sub_test in sub_tests { 165 | let duration_display = sub_test 166 | .result 167 | .duration() 168 | .map(|d| format!(" {}", format_duration(d))) 169 | .unwrap_or_default(); 170 | match &sub_test.result { 171 | TestResult::Passed { .. } => { 172 | writeln!( 173 | writer, 174 | "{}{} {}{}", 175 | indent, 176 | sub_test.name, 177 | colors::green_bold("ok"), 178 | duration_display, 179 | )?; 180 | } 181 | TestResult::Ignored => { 182 | writeln!( 183 | writer, 184 | "{}{} {}{}", 185 | indent, 186 | sub_test.name, 187 | colors::gray("ignored"), 188 | duration_display, 189 | )?; 190 | } 191 | TestResult::Failed { .. } => { 192 | writeln!( 193 | writer, 194 | "{}{} {}{}", 195 | indent, 196 | sub_test.name, 197 | colors::red_bold("fail"), 198 | duration_display, 199 | )?; 200 | } 201 | TestResult::SubTests { sub_tests, .. } => { 202 | writeln!( 203 | writer, 204 | "{}{}{}", 205 | indent, sub_test.name, duration_display 206 | )?; 207 | if sub_tests.is_empty() { 208 | writeln!( 209 | writer, 210 | "{} {}", 211 | indent, 212 | colors::gray("") 213 | )?; 214 | } else { 215 | output_sub_tests(writer, &format!("{} ", indent), sub_tests)?; 216 | } 217 | } 218 | } 219 | } 220 | Ok(()) 221 | } 222 | 223 | let duration_display = 224 | format_duration(result.duration().unwrap_or(duration)); 225 | match result { 226 | TestResult::Passed { .. } => { 227 | writeln!(writer, "{} {}", colors::green_bold("ok"), duration_display)?; 228 | } 229 | TestResult::Ignored => { 230 | writeln!(writer, "{}", colors::gray("ignored"))?; 231 | } 232 | TestResult::Failed { .. } => { 233 | writeln!(writer, "{} {}", colors::red_bold("fail"), duration_display)?; 234 | } 235 | TestResult::SubTests { sub_tests, .. } => { 236 | writeln!(writer, "{}", duration_display)?; 237 | output_sub_tests(writer, " ", sub_tests)?; 238 | } 239 | } 240 | 241 | Ok(()) 242 | } 243 | 244 | pub fn write_report_long_running_test( 245 | writer: &mut W, 246 | test_name: &str, 247 | ) -> std::io::Result<()> { 248 | writeln!( 249 | writer, 250 | "test {} has been running for more than 60 seconds", 251 | test_name, 252 | )?; 253 | Ok(()) 254 | } 255 | 256 | pub fn write_report_failures( 257 | writer: &mut W, 258 | failures: &[ReporterFailure], 259 | total_tests: usize, 260 | ) -> std::io::Result<()> { 261 | writeln!(writer)?; 262 | if !failures.is_empty() { 263 | writeln!(writer, "failures:")?; 264 | writeln!(writer)?; 265 | for failure in failures { 266 | writeln!(writer, "---- {} ----", failure.test.name)?; 267 | writeln!(writer, "{}", String::from_utf8_lossy(&failure.output))?; 268 | if let Some(line_and_column) = failure.test.line_and_column { 269 | writeln!( 270 | writer, 271 | "Test file: {}:{}:{}", 272 | failure.test.path.display(), 273 | line_and_column.0 + 1, 274 | line_and_column.1 + 1 275 | )?; 276 | } else { 277 | writeln!(writer, "Test file: {}", failure.test.path.display())?; 278 | } 279 | writeln!(writer)?; 280 | } 281 | writeln!(writer, "failed tests:")?; 282 | for failure in failures { 283 | writeln!(writer, " {}", failure.test.name)?; 284 | } 285 | } else { 286 | writeln!(writer, "{} tests passed", total_tests)?; 287 | } 288 | writeln!(writer)?; 289 | Ok(()) 290 | } 291 | } 292 | 293 | impl Reporter for LogReporter { 294 | fn report_category_start( 295 | &self, 296 | category: &CollectedTestCategory, 297 | _context: &ReporterContext, 298 | ) { 299 | let _ = LogReporter::write_report_category_start( 300 | &mut std::io::stderr(), 301 | category, 302 | ); 303 | } 304 | 305 | fn report_category_end( 306 | &self, 307 | _category: &CollectedTestCategory, 308 | _context: &ReporterContext, 309 | ) { 310 | } 311 | 312 | fn report_test_start( 313 | &self, 314 | test: &CollectedTest, 315 | context: &ReporterContext, 316 | ) { 317 | self 318 | .pending_tests 319 | .lock() 320 | .insert(test.name.clone(), Instant::now()); 321 | let _ = LogReporter::write_report_test_start( 322 | &mut std::io::stderr(), 323 | test, 324 | context, 325 | ); 326 | } 327 | 328 | fn report_test_end( 329 | &self, 330 | test: &CollectedTest, 331 | duration: Duration, 332 | result: &TestResult, 333 | context: &ReporterContext, 334 | ) { 335 | self.pending_tests.lock().remove(&test.name); 336 | let _ = LogReporter::write_report_test_end( 337 | &mut std::io::stderr(), 338 | test, 339 | duration, 340 | result, 341 | context, 342 | ); 343 | } 344 | 345 | fn report_failures( 346 | &self, 347 | failures: &[ReporterFailure], 348 | total_tests: usize, 349 | ) { 350 | let _ = LogReporter::write_report_failures( 351 | &mut std::io::stderr(), 352 | failures, 353 | total_tests, 354 | ); 355 | } 356 | } 357 | 358 | fn format_duration(duration: Duration) -> colors::Style { 359 | colors::gray(format!("({}ms)", duration.as_millis())) 360 | } 361 | 362 | /// A reporter that aggregates multiple reporters and reports to all of them. 363 | pub struct AggregateReporter { 364 | reporters: Vec>>, 365 | } 366 | 367 | impl AggregateReporter { 368 | pub fn new(reporters: Vec>>) -> Self { 369 | Self { reporters } 370 | } 371 | } 372 | 373 | impl Reporter for AggregateReporter { 374 | fn report_category_start( 375 | &self, 376 | category: &CollectedTestCategory, 377 | context: &ReporterContext, 378 | ) { 379 | for reporter in &self.reporters { 380 | reporter.report_category_start(category, context); 381 | } 382 | } 383 | 384 | fn report_category_end( 385 | &self, 386 | category: &CollectedTestCategory, 387 | context: &ReporterContext, 388 | ) { 389 | for reporter in &self.reporters { 390 | reporter.report_category_end(category, context); 391 | } 392 | } 393 | 394 | fn report_test_start( 395 | &self, 396 | test: &CollectedTest, 397 | context: &ReporterContext, 398 | ) { 399 | for reporter in &self.reporters { 400 | reporter.report_test_start(test, context); 401 | } 402 | } 403 | 404 | fn report_test_end( 405 | &self, 406 | test: &CollectedTest, 407 | duration: Duration, 408 | result: &TestResult, 409 | context: &ReporterContext, 410 | ) { 411 | for reporter in &self.reporters { 412 | reporter.report_test_end(test, duration, result, context); 413 | } 414 | } 415 | 416 | fn report_failures( 417 | &self, 418 | failures: &[ReporterFailure], 419 | total_tests: usize, 420 | ) { 421 | for reporter in &self.reporters { 422 | reporter.report_failures(failures, total_tests); 423 | } 424 | } 425 | } 426 | 427 | #[cfg(test)] 428 | mod test { 429 | use deno_terminal::colors; 430 | 431 | use super::*; 432 | 433 | fn build_end_test_message( 434 | result: &TestResult, 435 | duration: std::time::Duration, 436 | ) -> String { 437 | let mut output = Vec::new(); 438 | LogReporter::write_end_test_message(&mut output, result, duration).unwrap(); 439 | String::from_utf8(output).unwrap() 440 | } 441 | 442 | #[test] 443 | fn test_build_end_test_message_passed() { 444 | assert_eq!( 445 | build_end_test_message( 446 | &super::TestResult::Passed { duration: None }, 447 | std::time::Duration::from_millis(100), 448 | ), 449 | format!("{} {}\n", colors::green_bold("ok"), colors::gray("(100ms)")) 450 | ); 451 | } 452 | 453 | #[test] 454 | fn test_build_end_test_message_failed() { 455 | let message = build_end_test_message( 456 | &super::TestResult::Failed { 457 | output: b"error".to_vec(), 458 | duration: None, 459 | }, 460 | std::time::Duration::from_millis(100), 461 | ); 462 | assert_eq!( 463 | message, 464 | format!("{} {}\n", colors::red_bold("fail"), colors::gray("(100ms)")) 465 | ); 466 | } 467 | 468 | #[test] 469 | fn test_build_end_test_message_ignored() { 470 | assert_eq!( 471 | build_end_test_message( 472 | &super::TestResult::Ignored, 473 | std::time::Duration::from_millis(10), 474 | ), 475 | format!("{}\n", colors::gray("ignored")) 476 | ); 477 | } 478 | 479 | #[test] 480 | fn test_build_end_test_message_sub_tests() { 481 | let message = build_end_test_message( 482 | &super::TestResult::SubTests { 483 | duration: None, 484 | sub_tests: vec![ 485 | super::SubTestResult { 486 | name: "step1".to_string(), 487 | result: super::TestResult::Passed { 488 | duration: Some(Duration::from_millis(20)), 489 | }, 490 | }, 491 | super::SubTestResult { 492 | name: "step2".to_string(), 493 | result: super::TestResult::Failed { 494 | duration: None, 495 | output: b"error1".to_vec(), 496 | }, 497 | }, 498 | super::SubTestResult { 499 | name: "step3".to_string(), 500 | result: super::TestResult::Failed { 501 | duration: Some(Duration::from_millis(200)), 502 | output: b"error2".to_vec(), 503 | }, 504 | }, 505 | super::SubTestResult { 506 | name: "step4".to_string(), 507 | result: super::TestResult::SubTests { 508 | duration: None, 509 | sub_tests: vec![ 510 | super::SubTestResult { 511 | name: "sub-step1".to_string(), 512 | result: super::TestResult::Passed { duration: None }, 513 | }, 514 | super::SubTestResult { 515 | name: "sub-step2".to_string(), 516 | result: super::TestResult::Failed { 517 | duration: None, 518 | output: b"error3".to_vec(), 519 | }, 520 | }, 521 | ], 522 | }, 523 | }, 524 | ], 525 | }, 526 | std::time::Duration::from_millis(10), 527 | ); 528 | 529 | assert_eq!( 530 | message, 531 | format!( 532 | "{}\n step1 {} {}\n step2 {}\n step3 {} {}\n step4\n sub-step1 {}\n sub-step2 {}\n", 533 | colors::gray("(10ms)"), 534 | colors::green_bold("ok"), 535 | colors::gray("(20ms)"), 536 | colors::red_bold("fail"), 537 | colors::red_bold("fail"), 538 | colors::gray("(200ms)"), 539 | colors::green_bold("ok"), 540 | colors::red_bold("fail"), 541 | ) 542 | ); 543 | } 544 | } 545 | -------------------------------------------------------------------------------- /src/collection/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | use std::path::PathBuf; 4 | 5 | use deno_terminal::colors; 6 | use thiserror::Error; 7 | 8 | use crate::PathedIoError; 9 | 10 | use self::strategies::TestCollectionStrategy; 11 | 12 | pub mod strategies; 13 | 14 | #[derive(Debug, Clone)] 15 | pub enum CollectedCategoryOrTest { 16 | Category(CollectedTestCategory), 17 | Test(CollectedTest), 18 | } 19 | 20 | #[derive(Debug, Clone)] 21 | pub struct CollectedTestCategory { 22 | /// Fully resolved name of the test category. 23 | pub name: String, 24 | /// Path to the test category. May be a file or directory 25 | /// depending on how the test strategy collects tests. 26 | pub path: PathBuf, 27 | /// Children of the category. 28 | pub children: Vec>, 29 | } 30 | 31 | impl CollectedTestCategory { 32 | pub fn test_count(&self) -> usize { 33 | self 34 | .children 35 | .iter() 36 | .map(|child| match child { 37 | CollectedCategoryOrTest::Category(c) => c.test_count(), 38 | CollectedCategoryOrTest::Test(_) => 1, 39 | }) 40 | .sum() 41 | } 42 | 43 | pub fn filter_children(&mut self, filter: &str) { 44 | self.children.retain_mut(|mut child| match &mut child { 45 | CollectedCategoryOrTest::Category(c) => { 46 | c.filter_children(filter); 47 | !c.is_empty() 48 | } 49 | CollectedCategoryOrTest::Test(t) => t.name.contains(filter), 50 | }); 51 | } 52 | 53 | pub fn is_empty(&self) -> bool { 54 | for child in &self.children { 55 | match child { 56 | CollectedCategoryOrTest::Category(category) => { 57 | if !category.is_empty() { 58 | return false; 59 | } 60 | } 61 | CollectedCategoryOrTest::Test(_) => { 62 | return false; 63 | } 64 | } 65 | } 66 | 67 | true 68 | } 69 | 70 | /// Flattens all nested categories and returns a new category containing only tests as direct children. 71 | /// All subcategories are removed and their tests are moved to the top level. 72 | pub fn into_flat_category(self) -> Self { 73 | let mut flattened_tests = Vec::new(); 74 | 75 | fn collect_tests( 76 | children: Vec>, 77 | output: &mut Vec>, 78 | ) { 79 | for child in children { 80 | match child { 81 | CollectedCategoryOrTest::Category(category) => { 82 | collect_tests(category.children, output); 83 | } 84 | CollectedCategoryOrTest::Test(test) => { 85 | output.push(CollectedCategoryOrTest::Test(test)); 86 | } 87 | } 88 | } 89 | } 90 | 91 | collect_tests(self.children, &mut flattened_tests); 92 | 93 | CollectedTestCategory { 94 | name: self.name, 95 | path: self.path, 96 | children: flattened_tests, 97 | } 98 | } 99 | 100 | /// Splits this category into two separate categories based on a predicate. 101 | /// The first category contains tests matching the predicate, the second contains those that don't. 102 | /// Both categories preserve the same name and path as the original. 103 | pub fn partition(self, predicate: F) -> (Self, Self) 104 | where 105 | F: Fn(&CollectedTest) -> bool + Copy, 106 | { 107 | let mut matching_children = Vec::new(); 108 | let mut non_matching_children = Vec::new(); 109 | 110 | for child in self.children { 111 | match child { 112 | CollectedCategoryOrTest::Category(category) => { 113 | let (matching_cat, non_matching_cat) = category.partition(predicate); 114 | if !matching_cat.is_empty() { 115 | matching_children 116 | .push(CollectedCategoryOrTest::Category(matching_cat)); 117 | } 118 | if !non_matching_cat.is_empty() { 119 | non_matching_children 120 | .push(CollectedCategoryOrTest::Category(non_matching_cat)); 121 | } 122 | } 123 | CollectedCategoryOrTest::Test(test) => { 124 | if predicate(&test) { 125 | matching_children.push(CollectedCategoryOrTest::Test(test)); 126 | } else { 127 | non_matching_children.push(CollectedCategoryOrTest::Test(test)); 128 | } 129 | } 130 | } 131 | } 132 | 133 | let matching = CollectedTestCategory { 134 | name: self.name.clone(), 135 | path: self.path.clone(), 136 | children: matching_children, 137 | }; 138 | 139 | let non_matching = CollectedTestCategory { 140 | name: self.name, 141 | path: self.path, 142 | children: non_matching_children, 143 | }; 144 | 145 | (matching, non_matching) 146 | } 147 | } 148 | 149 | #[derive(Debug, Clone)] 150 | pub struct CollectedTest { 151 | /// Fully resolved name of the test. 152 | pub name: String, 153 | /// Path to the test file. 154 | pub path: PathBuf, 155 | /// Zero-indexed line and column of the test in the file. 156 | pub line_and_column: Option<(u32, u32)>, 157 | /// Data associated with the test that may have been 158 | /// set by the collection strategy. 159 | pub data: T, 160 | } 161 | 162 | impl CollectedTest { 163 | /// Helper to read the test file to a string. 164 | pub fn read_to_string(&self) -> Result { 165 | std::fs::read_to_string(&self.path) 166 | .map_err(|err| PathedIoError::new(&self.path, err)) 167 | } 168 | } 169 | 170 | pub struct CollectOptions { 171 | /// Base path to start from when searching for tests. 172 | pub base: PathBuf, 173 | /// Strategy to use for collecting tests. 174 | pub strategy: Box>, 175 | /// Override the filter provided on the command line. 176 | /// 177 | /// Generally, just provide `None` here. 178 | pub filter_override: Option, 179 | } 180 | 181 | /// Collect all the tests or exit if there are any errors. 182 | pub fn collect_tests_or_exit( 183 | options: CollectOptions, 184 | ) -> CollectedTestCategory { 185 | match collect_tests(options) { 186 | Ok(category) => category, 187 | Err(err) => { 188 | eprintln!("{}: {}", colors::red_bold("error"), err); 189 | std::process::exit(1); 190 | } 191 | } 192 | } 193 | 194 | #[derive(Debug, Error)] 195 | pub enum CollectTestsError { 196 | #[error(transparent)] 197 | InvalidTestName(#[from] InvalidTestNameError), 198 | #[error(transparent)] 199 | Io(#[from] PathedIoError), 200 | #[error("No tests found")] 201 | NoTestsFound, 202 | #[error(transparent)] 203 | Other(#[from] anyhow::Error), 204 | } 205 | 206 | pub fn collect_tests( 207 | options: CollectOptions, 208 | ) -> Result, CollectTestsError> { 209 | let mut category = options.strategy.collect_tests(&options.base)?; 210 | 211 | // error when no tests are found before filtering 212 | if category.is_empty() { 213 | return Err(CollectTestsError::NoTestsFound); 214 | } 215 | 216 | // ensure all test names are valid 217 | ensure_valid_test_names(&category)?; 218 | 219 | // filter 220 | let maybe_filter = options.filter_override.or_else(parse_cli_arg_filter); 221 | if let Some(filter) = &maybe_filter { 222 | category.filter_children(filter); 223 | } 224 | 225 | Ok(category) 226 | } 227 | 228 | fn ensure_valid_test_names( 229 | category: &CollectedTestCategory, 230 | ) -> Result<(), InvalidTestNameError> { 231 | for child in &category.children { 232 | match child { 233 | CollectedCategoryOrTest::Category(category) => { 234 | ensure_valid_test_names(category)?; 235 | } 236 | CollectedCategoryOrTest::Test(test) => { 237 | // only support characters that work with filtering with `cargo test` 238 | if !test 239 | .name 240 | .chars() 241 | .all(|c| c.is_alphanumeric() || matches!(c, '_' | ':')) 242 | { 243 | return Err(InvalidTestNameError(test.name.clone())); 244 | } 245 | } 246 | } 247 | } 248 | Ok(()) 249 | } 250 | 251 | #[derive(Debug, Error)] 252 | #[error( 253 | "Invalid test name ({0}). Use only alphanumeric and underscore characters so tests can be filtered via the command line." 254 | )] 255 | pub struct InvalidTestNameError(String); 256 | 257 | /// Parses the filter from the CLI args. This can be used 258 | /// with `category.filter_children(filter)`. 259 | pub fn parse_cli_arg_filter() -> Option { 260 | std::env::args() 261 | .nth(1) 262 | .filter(|s| !s.starts_with('-') && !s.is_empty()) 263 | } 264 | 265 | #[cfg(test)] 266 | mod tests { 267 | use super::*; 268 | 269 | #[test] 270 | fn test_partition() { 271 | // Create a test category with nested structure 272 | let category = CollectedTestCategory { 273 | name: "root".to_string(), 274 | path: PathBuf::from("/root"), 275 | children: vec![ 276 | CollectedCategoryOrTest::Test(CollectedTest { 277 | name: "test_foo".to_string(), 278 | path: PathBuf::from("/root/foo.rs"), 279 | line_and_column: None, 280 | data: (), 281 | }), 282 | CollectedCategoryOrTest::Test(CollectedTest { 283 | name: "test_bar".to_string(), 284 | path: PathBuf::from("/root/bar.rs"), 285 | line_and_column: None, 286 | data: (), 287 | }), 288 | CollectedCategoryOrTest::Category(CollectedTestCategory { 289 | name: "nested".to_string(), 290 | path: PathBuf::from("/root/nested"), 291 | children: vec![ 292 | CollectedCategoryOrTest::Test(CollectedTest { 293 | name: "test_baz".to_string(), 294 | path: PathBuf::from("/root/nested/baz.rs"), 295 | line_and_column: None, 296 | data: (), 297 | }), 298 | CollectedCategoryOrTest::Test(CollectedTest { 299 | name: "test_qux".to_string(), 300 | path: PathBuf::from("/root/nested/qux.rs"), 301 | line_and_column: None, 302 | data: (), 303 | }), 304 | ], 305 | }), 306 | ], 307 | }; 308 | 309 | // Partition based on whether name contains "ba" 310 | let (matching, non_matching) = 311 | category.partition(|test| test.name.contains("ba")); 312 | 313 | // Check matching category 314 | assert_eq!(matching.name, "root"); 315 | assert_eq!(matching.path, PathBuf::from("/root")); 316 | assert_eq!(matching.test_count(), 2); 317 | 318 | // Check that matching contains test_bar and nested/test_baz 319 | assert_eq!(matching.children.len(), 2); 320 | match &matching.children[0] { 321 | CollectedCategoryOrTest::Test(test) => assert_eq!(test.name, "test_bar"), 322 | _ => panic!("Expected test"), 323 | } 324 | match &matching.children[1] { 325 | CollectedCategoryOrTest::Category(cat) => { 326 | assert_eq!(cat.name, "nested"); 327 | assert_eq!(cat.children.len(), 1); 328 | match &cat.children[0] { 329 | CollectedCategoryOrTest::Test(test) => { 330 | assert_eq!(test.name, "test_baz") 331 | } 332 | _ => panic!("Expected test"), 333 | } 334 | } 335 | _ => panic!("Expected category"), 336 | } 337 | 338 | // Check non-matching category 339 | assert_eq!(non_matching.name, "root"); 340 | assert_eq!(non_matching.path, PathBuf::from("/root")); 341 | assert_eq!(non_matching.test_count(), 2); 342 | 343 | // Check that non-matching contains test_foo and nested/test_qux 344 | assert_eq!(non_matching.children.len(), 2); 345 | match &non_matching.children[0] { 346 | CollectedCategoryOrTest::Test(test) => assert_eq!(test.name, "test_foo"), 347 | _ => panic!("Expected test"), 348 | } 349 | match &non_matching.children[1] { 350 | CollectedCategoryOrTest::Category(cat) => { 351 | assert_eq!(cat.name, "nested"); 352 | assert_eq!(cat.children.len(), 1); 353 | match &cat.children[0] { 354 | CollectedCategoryOrTest::Test(test) => { 355 | assert_eq!(test.name, "test_qux") 356 | } 357 | _ => panic!("Expected test"), 358 | } 359 | } 360 | _ => panic!("Expected category"), 361 | } 362 | } 363 | 364 | #[test] 365 | fn test_partition_empty_categories_filtered() { 366 | // Create a category where all tests in a nested category match 367 | let category = CollectedTestCategory { 368 | name: "root".to_string(), 369 | path: PathBuf::from("/root"), 370 | children: vec![ 371 | CollectedCategoryOrTest::Test(CollectedTest { 372 | name: "test_match".to_string(), 373 | path: PathBuf::from("/root/match.rs"), 374 | line_and_column: None, 375 | data: (), 376 | }), 377 | CollectedCategoryOrTest::Category(CollectedTestCategory { 378 | name: "nested".to_string(), 379 | path: PathBuf::from("/root/nested"), 380 | children: vec![CollectedCategoryOrTest::Test(CollectedTest { 381 | name: "test_match2".to_string(), 382 | path: PathBuf::from("/root/nested/match2.rs"), 383 | line_and_column: None, 384 | data: (), 385 | })], 386 | }), 387 | ], 388 | }; 389 | 390 | let (matching, non_matching) = 391 | category.partition(|test| test.name.contains("match")); 392 | 393 | // All tests match, so matching should have everything 394 | assert_eq!(matching.test_count(), 2); 395 | assert_eq!(matching.children.len(), 2); 396 | 397 | // Non-matching should be empty (no children, and nested category filtered out) 398 | assert_eq!(non_matching.test_count(), 0); 399 | assert_eq!(non_matching.children.len(), 0); 400 | assert!(non_matching.is_empty()); 401 | } 402 | 403 | #[test] 404 | fn test_into_flat_category() { 405 | // Create a nested category structure 406 | let category = CollectedTestCategory { 407 | name: "root".to_string(), 408 | path: PathBuf::from("/root"), 409 | children: vec![ 410 | CollectedCategoryOrTest::Test(CollectedTest { 411 | name: "test_1".to_string(), 412 | path: PathBuf::from("/root/test1.rs"), 413 | line_and_column: None, 414 | data: (), 415 | }), 416 | CollectedCategoryOrTest::Category(CollectedTestCategory { 417 | name: "nested1".to_string(), 418 | path: PathBuf::from("/root/nested1"), 419 | children: vec![ 420 | CollectedCategoryOrTest::Test(CollectedTest { 421 | name: "test_2".to_string(), 422 | path: PathBuf::from("/root/nested1/test2.rs"), 423 | line_and_column: None, 424 | data: (), 425 | }), 426 | CollectedCategoryOrTest::Category(CollectedTestCategory { 427 | name: "deeply_nested".to_string(), 428 | path: PathBuf::from("/root/nested1/deeply"), 429 | children: vec![CollectedCategoryOrTest::Test(CollectedTest { 430 | name: "test_3".to_string(), 431 | path: PathBuf::from("/root/nested1/deeply/test3.rs"), 432 | line_and_column: None, 433 | data: (), 434 | })], 435 | }), 436 | ], 437 | }), 438 | CollectedCategoryOrTest::Category(CollectedTestCategory { 439 | name: "nested2".to_string(), 440 | path: PathBuf::from("/root/nested2"), 441 | children: vec![CollectedCategoryOrTest::Test(CollectedTest { 442 | name: "test_4".to_string(), 443 | path: PathBuf::from("/root/nested2/test4.rs"), 444 | line_and_column: None, 445 | data: (), 446 | })], 447 | }), 448 | ], 449 | }; 450 | 451 | let flattened = category.into_flat_category(); 452 | 453 | // Should preserve root name and path 454 | assert_eq!(flattened.name, "root"); 455 | assert_eq!(flattened.path, PathBuf::from("/root")); 456 | 457 | // Should have 4 direct children, all tests 458 | assert_eq!(flattened.children.len(), 4); 459 | assert_eq!(flattened.test_count(), 4); 460 | 461 | // All children should be tests, no categories 462 | for child in &flattened.children { 463 | assert!(matches!(child, CollectedCategoryOrTest::Test(_))); 464 | } 465 | 466 | // Verify test names are preserved 467 | let test_names: Vec = flattened 468 | .children 469 | .iter() 470 | .filter_map(|child| match child { 471 | CollectedCategoryOrTest::Test(test) => Some(test.name.clone()), 472 | _ => None, 473 | }) 474 | .collect(); 475 | 476 | assert_eq!(test_names.len(), 4); 477 | assert!(test_names.contains(&"test_1".to_string())); 478 | assert!(test_names.contains(&"test_2".to_string())); 479 | assert!(test_names.contains(&"test_3".to_string())); 480 | assert!(test_names.contains(&"test_4".to_string())); 481 | } 482 | } 483 | --------------------------------------------------------------------------------