├── .github
    └── workflows
    │   └── zig.yml
├── LICENSE
├── README.md
├── build.zig
├── build.zig.zon
└── src
    ├── example_tests.zig
    └── main.zig


/.github/workflows/zig.yml:
--------------------------------------------------------------------------------
 1 | name: CI
 2 | 
 3 | on:
 4 |   push:
 5 |     branches: [ main ]
 6 |   pull_request:
 7 |     branches: [ main ]
 8 | 
 9 | jobs:
10 |   build:
11 |     name: Build on ${{ matrix.os }} with Zig ${{ matrix.zig_version }}
12 |     runs-on: ${{ matrix.os }}
13 |     strategy:
14 |       fail-fast: false
15 |       matrix:
16 |         os: 
17 |           - ubuntu-latest
18 |           - macos-latest
19 |           - windows-latest
20 |           - ubuntu-24.04-arm
21 |         zig_version: [ "0.14.0-dev.3187+d4c85079c", "master" ]
22 | 
23 |     steps:      
24 |       - name: Check out repository
25 |         uses: actions/checkout@v4
26 | 
27 |       - name: Install Zig
28 |         uses: mlugg/setup-zig@v1
29 |         with:
30 |           version: ${{ matrix.zig_version }}
31 | 
32 |       - name: Build project
33 |         run: zig build
34 | 
35 |       - name: Run tests
36 |         run: zig build test --summary all
37 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright 2022-2024 cryptocode@zolo.io
 2 | 
 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of
 4 | this software and associated documentation files (the "Software"), to deal in
 5 | the Software without restriction, including without limitation the rights to
 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 7 | the Software, and to permit persons to whom the Software is furnished to do so,
 8 | subject to the following conditions:
 9 | 
10 | The above copyright notice and this permission notice shall be included in all
11 | copies or substantial portions of the Software.
12 | 
13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
15 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
16 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
17 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
18 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
19 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | <img align="right" width="320" src="https://user-images.githubusercontent.com/34946442/159163318-432052e3-69c7-4598-aaac-74d54f67c8b4.png">
  2 | 
  3 | Marble is a [metamorphic testing](https://en.wikipedia.org/wiki/Metamorphic_testing) library for Zig.
  4 | 
  5 | This library tracks Zig master and was last tested on `0.14.0-dev.3187+d4c85079c`
  6 | 
  7 | Metamorphic testing is a powerful technique that provides additional test coverage by applying a number of transformations to test input, and then checking if certain relations still hold between the outputs. Marble will automatically run through all possible combinations of these transformations.
  8 | 
  9 | Here's a [great introduction by](https://www.cockroachlabs.com/blog/metamorphic-testing-the-database/) Cockroach Labs. I highly recommend reading before using this library.
 10 | 
 11 | The repository contains a few [test examples](https://github.com/cryptocode/marble/blob/main/src/example_tests.zig)
 12 | 
 13 | ## Resources
 14 | * [Hillel Wayne's blog post on Metamorphic Testing (highly recommended)](https://www.hillelwayne.com/post/metamorphic-testing/)
 15 | * [Test your Machine Learning Algorithm with Metamorphic Testing](https://medium.com/trustableai/testing-ai-with-metamorphic-testing-61d690001f5c)
 16 | * [Original paper by T.Y. Chen et al](https://www.cse.ust.hk/~scc/publ/CS98-01-metamorphictesting.pdf)
 17 | * [Case study T.Y. Chen et al](http://grise.upm.es/rearviewmirror/conferencias/jiisic04/Papers/25.pdf)
 18 | * [Metamorphic Testing and Beyond T.Y. Chen et al](https://www.cs.hku.hk/data/techreps/document/TR-2003-06.pdf)
 19 | * [Survey on Metamorphic Testing](http://www.cs.ecu.edu/reu/reufiles/read/metamorphicTesting-16.pdf)
 20 | * [Performance Metamorphic Testing](http://www.lsi.us.es/~jtroya/publications/NIER17_at_ICSE17.pdf)
 21 | * [Experiences from Three Fuzzer Tools](https://johnwickerson.github.io/papers/dreamingup_MET21.pdf)
 22 | * [Monarch, a similar library for Rust](https://github.com/zmitchell/monarch/blob/master/src/runner.rs)
 23 | 
 24 | ## Building
 25 | 
 26 | To build and run test examples:
 27 | 
 28 | ```bash
 29 | zig build
 30 | zig build test
 31 | ```
 32 | 
 33 | ## Importing the library
 34 | Add Marble as a Zig package in your build file, or simply import it directly after vendoring/adding a submodule:
 35 | 
 36 | ```zig
 37 | const marble = @import("marble/main.zig");
 38 | ```
 39 | 
 40 | ## Writing tests
 41 | 
 42 | A metamorphic Zig test looks something like this:
 43 | 
 44 | ```zig
 45 | const SinusTest = struct {
 46 |     const tolerance = std.math.epsilon(f64) * 20;
 47 | 
 48 |     /// This test has a single value, but you could also design the test to take an
 49 |     /// array as input. The transformations, check and execute functions would then
 50 |     /// loop through them all. Alternatively, the test can be run multiple times
 51 |     /// with different inputs.
 52 |     value: f64,
 53 | 
 54 |     /// The mathematical property "sin(x) = sin(π − x)" must hold
 55 |     pub fn transformPi(self: *SinusTest) void {
 56 |         self.value = std.math.pi - self.value;
 57 |     }
 58 | 
 59 |     /// Adding half the epsilon must still cause the relation to hold given the tolerance
 60 |     pub fn transformEpsilon(self: *SinusTest) void {
 61 |         self.value = self.value + std.math.epsilon(f64) / 2.0;
 62 |     }
 63 | 
 64 |     /// A metamorphic relation is a relation between outputs in different executions.
 65 |     /// This relation must hold after every execution of transformation combinations.
 66 |     pub fn check(_: *SinusTest, original_output: f64, transformed_output: f64) bool {
 67 |         return std.math.approxEqAbs(f64, original_output, transformed_output, tolerance);
 68 |     }
 69 | 
 70 |     /// Called initially to compute the baseline output, and after every transformation combination
 71 |     pub fn execute(self: *SinusTest) f64 {
 72 |         return std.math.sin(self.value);
 73 |     }
 74 | };
 75 | 
 76 | test "sinus" {
 77 |     var i: f64 = 1;
 78 |     while (i < 100) : (i += 1) {
 79 |         var t = SinusTest{ .value = i };
 80 |         try std.testing.expect(try marble.run(SinusTest, &t, .{}));
 81 |     }
 82 | }
 83 | ```
 84 | 
 85 | You will get compile time errors if the requirements for a metamorphic test are not met.
 86 | 
 87 | In short, you must provide a `value` field, a `check` function, an `execute` function and one or more `transform...` functions.
 88 | 
 89 | ### Writing transformations
 90 | Add one or more functions starting with `transform...`
 91 | 
 92 | Marble will execute all combinations of the transformation functions. After every
 93 | combination, `execute` is called followed by `check`.
 94 | 
 95 | Transformations should change the `value` property - Marble will remember what it was originally. The transformations must be such that `check`
 96 | succeeds. That is, the relations between the inital output and the transformed output must still hold.
 97 | 
 98 | ### Checking if relations still hold
 99 | You must provide a `check` function to see if one or more relations hold, and return true if so. If false is returned, the test fails with a print-out of the current transformation-combination.
100 | 
101 | Relation checks may be conditional; check out the tests for examples on how this works.
102 | 
103 | ### Executing
104 | You must provide an `execute` function that computes a result based on the current value. The simplest form will simply return the current value, but you can
105 | do any arbitrary operation here. This function is called before any transformations to form a baseline. This baseline is passed as the first argument to `check`
106 | 
107 | ### Optional before/after calls
108 | 
109 | Before and after the test, and every combination, `before(...)` and `after(...)` is called if present. This is useful to reset state, initialize test cases, and perform clean-up.
110 | 
111 | ### What happens during a test run?
112 | 
113 | Using the example above, the following pseudocode runs will be performed:
114 | 
115 | ```
116 | baseline = execute()
117 | 
118 | // First combination
119 | transformPi()
120 | out = execute()
121 | check(baseline, out)
122 | 
123 | // Second combination
124 | transformEpsilon()
125 | out = execute()
126 | check(baseline, out)
127 | 
128 | // Third combination
129 | transformPi()
130 | transformEpsilon()
131 | out = execute()
132 | check(baseline, out)
133 | ```
134 | 
135 | ### Configuring runs
136 | 
137 | The `run` function takes a `RunConfiguration`:
138 | 
139 | ```zig
140 | /// If set to true, only run each transformation once separately
141 | skip_combinations: bool = false,
142 | 
143 | /// If true, print detailed information during the run
144 | verbose: bool = false,
145 | ```
146 | 
147 | ### Error reporting
148 | 
149 | If a test fails, the current combination being executed is printed. For instance, the following tells us that the combination of `transformAdditionalTerm` and `transformCase` caused the metamorphic relation to fail:
150 | 
151 | ```
152 | Test [2/2] test "query"... Test case failed with transformation(s):
153 |   >> transformAdditionalTerm
154 |   >> transformCase
155 | ```
156 | 
157 | ### Terminology
158 | 
159 | * Source test case output: The output produced by `execute()` on the initial input. This is also known as the baseline.
160 | * Derived test case output: The output produced by `execute()` after applying a specific combination of transformations.
161 | * Metamorphic relation: A property that must hold when considering a source test case and a derived test case.
162 | 


--------------------------------------------------------------------------------
/build.zig:
--------------------------------------------------------------------------------
 1 | const std = @import("std");
 2 | 
 3 | pub fn build(b: *std.Build) void {
 4 |     const target = b.standardTargetOptions(.{});
 5 |     const optimize = b.standardOptimizeOption(.{});
 6 | 
 7 |     const marble_mod = b.addModule("marble", .{
 8 |         .root_source_file = b.path("src/main.zig"),
 9 |     });
10 | 
11 |     const lib = b.addStaticLibrary(.{
12 |         .name = "marble",
13 |         .root_source_file = b.path("src/main.zig"),
14 |         .target = target,
15 |         .optimize = optimize,
16 |     });
17 |     b.installArtifact(lib);
18 | 
19 |     const tests = b.addTest(.{
20 |         .name = "example_tests",
21 |         .root_source_file = b.path("src/example_tests.zig"),
22 |         .target = target,
23 |         .optimize = optimize,
24 |     });
25 |     tests.root_module.addImport("marble", marble_mod);
26 | 
27 |     const example_tests = b.addRunArtifact(tests);
28 |     const test_step = b.step("test", "Run library tests");
29 |     test_step.dependOn(&example_tests.step);
30 | }
31 | 


--------------------------------------------------------------------------------
/build.zig.zon:
--------------------------------------------------------------------------------
 1 | .{
 2 |     .name = .marble,
 3 |     .version = "0.1.0",
 4 |     .paths = .{
 5 |         "build.zig",
 6 |         "build.zig.zon",
 7 |         "src/example_tests.zig",
 8 |         "src/main.zig",
 9 |         "LICENSE",
10 |         "README.md",
11 |     },
12 |     .fingerprint = 0xdf0f25ff70e782ed,
13 | }
14 | 


--------------------------------------------------------------------------------
/src/example_tests.zig:
--------------------------------------------------------------------------------
  1 | const std = @import("std");
  2 | const marble = @import("marble");
  3 | 
  4 | const SinusTest = struct {
  5 |     const tolerance = std.math.floatEps(f64) * 20;
  6 | 
  7 |     /// This test has a single value, but you could also design the test to take an
  8 |     /// array as input. The transformations, check and execute functions would then
  9 |     /// loop through them all. Alternatively, the test can be run multiple times
 10 |     /// with different inputs.
 11 |     value: f64,
 12 | 
 13 |     /// The mathematical property "sin(x) = sin(π − x)" must hold
 14 |     pub fn transformPi(self: *SinusTest) void {
 15 |         // If you flip this to + instead of - you'll observe how metamorphic tests fail.
 16 |         self.value = std.math.pi - self.value;
 17 |     }
 18 | 
 19 |     /// Adding half the epsilon must still cause the relation to hold given the tolerance
 20 |     pub fn transformEpsilon(self: *SinusTest) void {
 21 |         self.value = self.value + std.math.floatEps(f64) / 2.0;
 22 |     }
 23 | 
 24 |     /// A metamorphic relation is a relation between outputs in different executions.
 25 |     /// This relation must hold after every execution of transformation combinations.
 26 |     pub fn check(_: *SinusTest, original_output: f64, transformed_output: f64) bool {
 27 |         return std.math.approxEqAbs(f64, original_output, transformed_output, tolerance);
 28 |     }
 29 | 
 30 |     /// Called initially to compute the baseline output, and after every transformation combination
 31 |     pub fn execute(self: *SinusTest) f64 {
 32 |         return std.math.sin(self.value);
 33 |     }
 34 | };
 35 | 
 36 | test "sinus" {
 37 |     var i: f64 = 1;
 38 |     while (i < 100) : (i += 1) {
 39 |         var t = SinusTest{ .value = i };
 40 |         try std.testing.expect(try marble.run(SinusTest, &t, .{}));
 41 |     }
 42 | }
 43 | 
 44 | /// Input to a query
 45 | const Query = struct {
 46 |     term: []const u8 = "test",
 47 |     ascending: bool = false,
 48 |     page_size: usize = 50,
 49 | };
 50 | 
 51 | // This is an example of a "conditional relations" test where the metamorphic relationship
 52 | // depends on which transformations are applied in the current combination.
 53 | const QueryTest = struct {
 54 |     value: Query,
 55 |     additional_term: bool = false,
 56 | 
 57 |     /// Reset the additional_term flag before each combination. It will be
 58 |     /// flipped on for combinations including `transformAdditionalTerm`.
 59 |     pub fn before(self: *QueryTest, phase: marble.Phase) void {
 60 |         if (phase == .Combination) self.additional_term = false;
 61 |     }
 62 | 
 63 |     /// Sorting shouldn't affect total count
 64 |     pub fn transformSort(self: *QueryTest) void {
 65 |         self.value.ascending = true;
 66 |     }
 67 | 
 68 |     /// Page count shouldn't affect total count
 69 |     pub fn transformPageCount(self: *QueryTest) void {
 70 |         self.value.page_size = 25;
 71 |     }
 72 | 
 73 |     /// Our search engine is case insensitive
 74 |     pub fn transformCase(self: *QueryTest) void {
 75 |         self.value.term = "TEST";
 76 |     }
 77 | 
 78 |     /// Another term reduces the number of hits
 79 |     pub fn transformAdditionalTerm(self: *QueryTest) void {
 80 |         self.additional_term = true;
 81 |         self.value.term = "test another";
 82 |     }
 83 | 
 84 |     /// Number of total hits shouldn't change when changing sort order, page count and casing.
 85 |     /// However, we do expect multiple search terms to reduce the number of hits.
 86 |     /// These are two metamorphic relations, one of which is checked conditionally.
 87 |     pub fn check(self: *QueryTest, untransformed_hits: usize, hits_after_transformations: usize) bool {
 88 |         if (self.additional_term) return untransformed_hits >= hits_after_transformations;
 89 |         return untransformed_hits == hits_after_transformations;
 90 |     }
 91 | 
 92 |     /// Execute the query, returning the total number of hits. A real-world test could do a mocked REST call.
 93 |     pub fn execute(self: *QueryTest) usize {
 94 |         // Emulate fewer hits when additional search terms are added
 95 |         return if (self.additional_term) 50 else 100;
 96 |     }
 97 | };
 98 | 
 99 | test "query" {
100 |     var query_test = QueryTest{ .value = .{} };
101 |     try std.testing.expect(try marble.run(QueryTest, &query_test, .{ .skip_combinations = false, .verbose = false }));
102 | }
103 | 
104 | /// Test some metamorphic relations of binary search
105 | /// MT relations courtesy of @jacobdweightman
106 | const BinarySearchTest = struct {
107 |     const S = struct {
108 |         fn order(context: usize, rhs: usize) std.math.Order {
109 |             return std.math.order(context, rhs);
110 |         }
111 |     };
112 | 
113 |     /// The value is the binary search result qindex
114 |     value: ?usize = undefined,
115 |     arr: []const usize = undefined,
116 |     testing_accidental_insert: bool = undefined,
117 | 
118 |     pub fn before(self: *BinarySearchTest, phase: marble.Phase) void {
119 |         if (phase == .Combination) {
120 |             self.testing_accidental_insert = false;
121 |         }
122 |     }
123 | 
124 |     /// Test that basic relations hold:
125 |     ///   if x = A[k], then binarySearch(x, A) = k
126 |     pub fn transformSimple(self: *BinarySearchTest) void {
127 |         const x = self.arr[self.value.?];
128 |         self.value = std.sort.binarySearch(usize, self.arr, x, S.order);
129 |     }
130 | 
131 |     // This transform will catch an error where the value being searched for is
132 |     // accidentally being inserted into the array:
133 |     //   if A[k-1] < x < A[k+1] and x != A[k], then binarySearch(x, A) = -1
134 |     pub fn transformAccidentalInsert(self: *BinarySearchTest) void {
135 |         self.testing_accidental_insert = true;
136 |         if (self.value.? == 0 or self.value.? + 1 >= self.arr.len) return;
137 |         var x = self.arr[self.value.? - 1] + 1;
138 |         if (x == self.arr[self.value.?]) x += 1;
139 |         if (x >= self.arr[self.value.? + 1]) return;
140 |         self.value = std.sort.binarySearch(usize, self.arr, x, S.order);
141 |     }
142 | 
143 |     /// Test binary search array splitting correctness:
144 |     //    if x = A[k], then binarySearch(A[k-1], A) = k-1 and binarySearch(A[k+1], A) = k + 1
145 |     pub fn transformSplitting(self: *BinarySearchTest) void {
146 |         const x = self.arr[self.value.?];
147 |         self.value = std.sort.binarySearch(usize, self.arr, x, S.order);
148 |     }
149 | 
150 |     pub fn check(self: *BinarySearchTest, org: ?usize, new: ?usize) bool {
151 |         return (new == null and self.testing_accidental_insert) or org.? == new.?;
152 |     }
153 | 
154 |     pub fn execute(self: *BinarySearchTest) ?usize {
155 |         return self.value;
156 |     }
157 | };
158 | 
159 | test "std.sort.binarySearch" {
160 |     const array: []const usize = &.{ 4, 6, 10, 15, 18, 25, 40 };
161 |     var i: usize = 0;
162 |     while (i < array.len) : (i += 1) {
163 |         var bs_test = BinarySearchTest{ .value = i, .arr = array };
164 |         try std.testing.expect(try marble.run(BinarySearchTest, &bs_test, .{ .skip_combinations = true }));
165 |     }
166 | }
167 | 


--------------------------------------------------------------------------------
/src/main.zig:
--------------------------------------------------------------------------------
  1 | //! Marble is a Metamorphic Testing library for Zig
  2 | //! https://github.com/cryptocode/marble
  3 | 
  4 | const std = @import("std");
  5 | 
  6 | /// Generate an (n take r) list of transformer indices
  7 | /// Number of combinations returned is n! / (r! (n-r)!)
  8 | fn generateCombinations(n: usize, r: usize, allocator: std.mem.Allocator) !std.ArrayList(std.ArrayList(usize)) {
  9 |     var combinations = std.ArrayList(std.ArrayList(usize)).init(allocator);
 10 |     var combination = std.ArrayList(usize).init(allocator);
 11 | 
 12 |     // Start with the smallest lexicographic combination
 13 |     {
 14 |         var i: usize = 0;
 15 |         while (i < r) : (i += 1) {
 16 |             try combination.append(i);
 17 |         }
 18 |     }
 19 | 
 20 |     while (combination.items[r - 1] < n) {
 21 |         try combinations.append(try combination.clone());
 22 | 
 23 |         // Next combination in lexicographic order
 24 |         var k = r - 1;
 25 |         while (k != 0 and combination.items[k] == n - r + k) {
 26 |             k -= 1;
 27 |         }
 28 |         combination.items[k] += 1;
 29 | 
 30 |         var j: usize = k + 1;
 31 |         while (j < r) : (j += 1) {
 32 |             combination.items[j] = combination.items[j - 1] + 1;
 33 |         }
 34 |     }
 35 | 
 36 |     return combinations;
 37 | }
 38 | 
 39 | /// Generate the combinations for every n = 0..count-1 and r = 1..count
 40 | fn generateAllCombinations(transformation_count: usize, allocator: std.mem.Allocator) !std.ArrayList(std.ArrayList(usize)) {
 41 |     var res = std.ArrayList(std.ArrayList(usize)).init(allocator);
 42 |     var i: usize = 1;
 43 |     while (i <= transformation_count) : (i += 1) {
 44 |         try res.appendSlice((try generateCombinations(transformation_count, i, allocator)).items[0..]);
 45 |     }
 46 |     return res;
 47 | }
 48 | 
 49 | /// Phase indicator for the `before` and `after` functions
 50 | pub const Phase = enum {
 51 |     /// Before or after a test `run`
 52 |     Test,
 53 |     /// Before or after a transformation combination
 54 |     Combination,
 55 | };
 56 | 
 57 | /// Returns the type representing a discovered transformer function
 58 | fn Transformer(comptime TestType: type) type {
 59 |     return struct {
 60 |         function: *const fn (*TestType) void,
 61 |         name: []const u8,
 62 |     };
 63 | }
 64 | 
 65 | /// A metamorphic test-case is expected to have a number of functions whose name
 66 | /// starts with "transform". Combinations of these functions will be executed
 67 | /// during test runs.
 68 | fn findTransformers(comptime T: type) []const Transformer(T) {
 69 |     const functions = @typeInfo(T).@"struct".decls;
 70 |     var transformers: []const Transformer(T) = &[_]Transformer(T){};
 71 |     inline for (functions) |f| {
 72 |         if (std.mem.startsWith(u8, f.name, "transform")) {
 73 |             transformers = transformers ++ &[_]Transformer(T){.{
 74 |                 .function = @field(T, f.name),
 75 |                 .name = f.name,
 76 |             }};
 77 |         }
 78 |     }
 79 |     return transformers;
 80 | }
 81 | 
 82 | /// Configuration of a test run
 83 | pub const RunConfiguration = struct {
 84 |     /// If set to true, only run each transformation once separately
 85 |     skip_combinations: bool = false,
 86 |     /// If true, print detailed information during the run
 87 |     verbose: bool = false,
 88 | };
 89 | 
 90 | /// Run a testcase, returns true if all succeed
 91 | pub fn run(comptime T: type, testcase: *T, config: RunConfiguration) !bool {
 92 |     if (config.verbose) std.debug.print("\n", .{});
 93 |     const metamorphicTest = comptime findTransformers(T);
 94 |     if (@hasDecl(T, "before")) testcase.before(Phase.Test);
 95 | 
 96 |     const initial_value = testcase.value;
 97 | 
 98 |     // Execute on the initial value. The result is used as the baseline to check if a relation
 99 |     // holds after transformations.
100 |     const org_output = testcase.execute();
101 | 
102 |     var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
103 |     defer arena.deinit();
104 |     var combinations = try generateAllCombinations(metamorphicTest.len, arena.allocator());
105 |     for (combinations.items) |combination| {
106 |         if (combination.items.len > 1 and config.skip_combinations) {
107 |             if (config.verbose) std.debug.print("Skipping transformation combinations\n", .{});
108 |             break;
109 |         }
110 | 
111 |         // Reset to initial value for each transformer combination
112 |         testcase.value = initial_value;
113 | 
114 |         // The before-function is free to update the inital value set above
115 |         if (@hasDecl(T, "before")) testcase.before(Phase.Combination);
116 | 
117 |         if (config.verbose) std.debug.print(">> Combination\n", .{});
118 | 
119 |         // Run through all value transformations
120 |         for (combination.items) |transformer_index| {
121 |             const tr = metamorphicTest[transformer_index];
122 |             if (config.verbose) std.debug.print("  >> {s}\n", .{tr.name});
123 | 
124 |             @call(.auto, tr.function, .{testcase});
125 |         }
126 | 
127 |         // Execute
128 |         const transformed_output = testcase.execute();
129 | 
130 |         // Check if relation still holds
131 |         if (!testcase.check(org_output, transformed_output)) {
132 |             std.debug.print("Test case failed with transformation(s):\n", .{});
133 |             for (combination.items) |transformer_index| {
134 |                 const tr = metamorphicTest[transformer_index];
135 |                 std.debug.print("  >> {s}\n", .{tr.name});
136 |             }
137 | 
138 |             return false;
139 |         }
140 | 
141 |         if (@hasDecl(T, "after")) testcase.after(Phase.Combination);
142 |     }
143 | 
144 |     if (@hasDecl(T, "after")) testcase.after(Phase.Test);
145 | 
146 |     for (combinations.items) |*combination| {
147 |         combination.deinit();
148 |     }
149 |     combinations.deinit();
150 |     return true;
151 | }
152 | 


--------------------------------------------------------------------------------