├── requirements.txt
├── .gitignore
├── compile_and_run.sh
├── utils
    ├── matam_types.py
    ├── loading_bar.py
    ├── config.py
    ├── matam_parsing.py
    └── matam_html.py
├── example
    └── custom_tests.json
├── .github
    └── workflows
    │   └── main.yml
├── README.md
└── run_tests.py


/requirements.txt:
--------------------------------------------------------------------------------
1 | pyinstaller


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /dist/
2 | .idea/
3 | build/
4 | **.spec


--------------------------------------------------------------------------------
/compile_and_run.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | 
3 | g++ -DNDEBUG -std=c++17 -Wall -pedantic-errors -Werror -g -o mtm_blockchain *.cpp
4 | python3 ./tests/run_tests.py ./mtm_blockchain ./tests/custom_tests.json


--------------------------------------------------------------------------------
/utils/matam_types.py:
--------------------------------------------------------------------------------
 1 | import sys
 2 | if sys.version_info < (3, 10):
 3 |     sys.exit("Python %s.%s or later is required.\n" % (3, 10))
 4 | else:
 5 |     from typing import TypedDict, TypeAlias, List
 6 | 
 7 | TestTemplates: TypeAlias = dict[str, str]
 8 | TestParams: TypeAlias = dict[str, str]
 9 | 
10 | class TestParamRange(TypedDict):
11 |     first: int
12 |     last: int
13 | 
14 | 
15 | class TestCase(TypedDict):
16 |     name: str
17 |     template: str
18 |     params: TestParams
19 |     output_file: str
20 |     expected_output_file: str
21 |     expected_output_is_substring: bool | None
22 |     run_leaks: bool | None
23 |     params_range: TestParamRange | List[str] | None
24 | 
25 | 
26 | class TestFile(TypedDict):
27 |     templates: TestTemplates
28 |     tests: list[TestCase]
29 | 
30 | 
31 | class Summary(TypedDict):
32 |     title: str
33 |     actual: str | None
34 |     expected: str | None
35 |     error: str | None
36 |     diff_html: str | None
37 | 
38 | 
39 | class TestResult(TypedDict):
40 |     name: str
41 |     summary: Summary
42 |     passed: bool
43 |     command: str | None
44 | 
45 | 


--------------------------------------------------------------------------------
/utils/loading_bar.py:
--------------------------------------------------------------------------------
 1 | # Taken from https://stackoverflow.com/a/34325723
 2 | 
 3 | 
 4 | # Print iterations progress
 5 | def print_progress_bar(iteration, total, prefix='', suffix='', decimals=1, length=100, fill='█',
 6 |                        print_end="\r"):
 7 |     """
 8 |     Call in a loop to create terminal progress bar
 9 |     @params:
10 |         iteration   - Required  : current iteration (Int)
11 |         total       - Required  : total iterations (Int)
12 |         prefix      - Optional  : prefix string (Str)
13 |         suffix      - Optional  : suffix string (Str)
14 |         decimals    - Optional  : positive number of decimals in percent complete (Int)
15 |         length      - Optional  : character length of bar (Int)
16 |         fill        - Optional  : bar fill character (Str)
17 |         printEnd    - Optional  : end character (e.g. "\r", "\r\n") (Str)
18 |     """
19 |     percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
20 |     filled_length = int(length * iteration // total)
21 |     bar = fill * filled_length + '-' * (length - filled_length)
22 |     print(f'\r{prefix} |{bar}| {percent}% {suffix} ({iteration}/{total})', end=print_end,
23 |           flush=True)
24 |     # Print New Line on Complete
25 |     if iteration == total:
26 |         print(flush=True)
27 | 


--------------------------------------------------------------------------------
/example/custom_tests.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "templates": {
 3 |     "verify": "verify :::source::: :::target::: > :::out:::",
 4 |     "hash": "hash :::source::: :::target:::",
 5 |     "compress": "compress :::source::: :::target:::",
 6 |     "format": "format :::source::: :::target:::"
 7 |   },
 8 |   "tests": [
 9 |     {
10 |       "name": "Verify Example Test",
11 |       "template": "verify",
12 |       "params": {
13 |         "source": "./tests/verify.source",
14 |         "target": "./tests/verify.target",
15 |         "out": "./tests/verify.out"
16 |       },
17 |       "output_file": "./tests/verify.out",
18 |       "expected_output_file": "./tests/verify.expected"
19 |     },
20 |     {
21 |       "name": "Hash Example Test",
22 |       "template": "hash",
23 |       "params": {
24 |         "source": "./tests/hash.source",
25 |         "target": "./tests/hash.target.out"
26 |       },
27 |       "output_file": "./tests/hash.target.out",
28 |       "expected_output_file": "./tests/hash.target.expected"
29 |     },
30 |     {
31 |       "name": "Compress Example Test",
32 |       "template": "compress",
33 |       "params": {
34 |         "source": "./tests/compress.source",
35 |         "target": "./tests/compress.target.out"
36 |       },
37 |       "output_file": "./tests/compress.target.out",
38 |       "expected_output_file": "./tests/compress.target.expected"
39 |     },
40 |     {
41 |       "name": "Format Example Test",
42 |       "template": "format",
43 |       "params": {
44 |         "source": "./tests/format.source",
45 |         "target": "./tests/format.target.out"
46 |       },
47 |       "output_file": "./tests/format.target.out",
48 |       "expected_output_file": "./tests/format.target.expected"
49 |     }
50 |   ]
51 | }


--------------------------------------------------------------------------------
/utils/config.py:
--------------------------------------------------------------------------------
 1 | from platform import system
 2 | from os import environ
 3 | 
 4 | IS_MAC_OS = system() == 'Darwin'
 5 | 
 6 | LEAKS_CHECKER_NAME = 'leaks' if IS_MAC_OS else 'Valgrind'
 7 | LEAKS_CHECKER_COMMAND = 'export MallocStackLogging=1 && leaks --atExit --' \
 8 |     if IS_MAC_OS else 'valgrind --leak-check=full'
 9 | 
10 | NO_LEAKS_FOUND_TEXT = '0 leaks for 0 total leaked bytes.' if IS_MAC_OS else 'no leaks are possible'
11 | 
12 | 
13 | # Define constants
14 | STDOUT = 0
15 | STDERR = 1
16 | 
17 | TESTS_JSON_FILE_INDEX = 1
18 | EXECUTABLE_INDEX = 2
19 | EXPECTED_ARGS_AMOUNT = 3
20 | 
21 | HTML_COLORED_NEWLINE = '<span style="background-color: orange;">\\n</span><br/>'
22 | HTML_COLORED_WHITESPACE = '<span style="background-color: #BE5103;">&nbsp;</span>'  # burnt orange
23 | NORMAL_HTML_NEWLINE = '<br/>'
24 | 
25 | TEST_NAME = 'name'
26 | TEMPLATE_NAME = 'template'
27 | PARAMS = 'params'
28 | OUTPUT_FILE = 'output_file'
29 | EXPECTED_OUTPUT_FILE = 'expected_output_file'
30 | EXPECTED_OUTPUT_IS_SUBSTR = 'expected_output_is_substring'
31 | TEMP_REPORT = 'test_results_current.html'
32 | FINAL_REPORT = 'test_results.html'
33 | 
34 | TIMEOUT = int(environ.get('MATAM_TESTER_TEST_TIMEOUT', '1'))  # 1 second
35 | VALGRIND_TIMEOUT = int(environ.get('MATAM_TESTER_VALGRIND_TIMEOUT', '2'))  # 2 seconds
36 | 
37 | COMPARISON_TRIM_END_SPACES = int(environ.get('MATAM_TESTER_TRIMR_SPACES', '0'))
38 | COMPARISON_IGNORE_BLANK_LINES = int(environ.get('MATAM_TESTER_IGNORE_EMPTY_LINES', '0'))
39 | 
40 | RUN_MULTI_THREAD = int(environ.get('MATAM_TESTER_RUN_MULTI_THREADED', '0')) == 1
41 | EXPORT_TEMP_REPORT = int(environ.get('MATAM_TESTER_EXPORT_TEMP_REPORT', '0')) == 1
42 | 
43 | USE_OLD_DIFF_STYLE = int(environ.get('MATAM_TESTER_USE_OLD_DIFF_STYLE', '0')) == 1


--------------------------------------------------------------------------------
/.github/workflows/main.yml:
--------------------------------------------------------------------------------
 1 | name: Build and Release
 2 | 
 3 | on:
 4 |   push:
 5 |     tags:
 6 |       - '*'
 7 | 
 8 | env:
 9 |   GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}
10 | 
11 | jobs:
12 |   build:
13 |     runs-on: ${{ matrix.os }}
14 |     strategy:
15 |       matrix:
16 |         os: [ windows-latest, ubuntu-latest, macos-latest ]
17 |         python-version: [3.x]
18 |     steps:
19 |       - name: Checkout repository
20 |         uses: actions/checkout@v4
21 |       - name: Set up Python ${{ matrix.python-version }}
22 |         uses: actions/setup-python@v5
23 |         with:
24 |           python-version: ${{ matrix.python-version }}
25 | 
26 |       - name: Install dependencies
27 |         run: |
28 |           python -m pip install --upgrade pip
29 |           pip install -r requirements.txt
30 | 
31 |       - name: Compile Python code to executable
32 |         run: pyinstaller --paths=$(python -c "import site; print(site.getsitepackages())") --onefile run_tests.py -n student_test --distpath ./LocalGradescope
33 | 
34 |       - name: Upload artifact
35 |         uses: actions/upload-artifact@v4
36 |         with:
37 |           name: ${{ runner.os }} Local Gradescope
38 |           path: dist
39 |       - name: Zip up output
40 |         if: runner.os == 'Windows'
41 |         run: |
42 |           cd LocalGradescope
43 |           Compress-Archive -Path . -DestinationPath ../LocalGradescope-${{ runner.os }}-${{ github.ref_name }}.zip
44 |       - name: Zip up output
45 |         if: runner.os != 'Windows'
46 |         run: |
47 |           cd LocalGradescope
48 |           zip -r ../LocalGradescope-${{ runner.os }}-${{ github.ref_name }}.zip *
49 |       - name: Release
50 |         uses: softprops/action-gh-release@v2
51 |         with:
52 |           token: ${{ env.GIT_ACCESS_TOKEN }}
53 |           files: LocalGradescope-${{ runner.os }}-${{ github.ref_name }}.zip
54 | 


--------------------------------------------------------------------------------
/utils/matam_parsing.py:
--------------------------------------------------------------------------------
  1 | import sys
  2 | 
  3 | from utils.config import IS_MAC_OS, EXPECTED_OUTPUT_FILE, EXPECTED_OUTPUT_IS_SUBSTR, NORMAL_HTML_NEWLINE, \
  4 |     LEAKS_CHECKER_NAME
  5 | from utils.matam_types import Summary, TestCase, TestParamRange, TestParams
  6 | 
  7 | if sys.version_info < (3, 10):
  8 |     sys.exit("Python %s.%s or later is required.\n" % (3, 10))
  9 | else:
 10 |     from typing import List, Any, Iterable
 11 | 
 12 | def normalize_newlines(txt: str) -> str:
 13 |     return txt.replace('\r\n', '\n').replace('\r', '\n')
 14 | 
 15 | 
 16 | def test_exception_to_error_text(exception: Exception) -> str:
 17 |     if exception.stderr:
 18 |         return str(exception.stderr)
 19 |     if exception.stdout:
 20 |         return str(exception.stdout)
 21 |     return str(exception)
 22 | 
 23 | 
 24 | def remove_error_pipes_from_command(command: str) -> str:
 25 |     """
 26 |     Avoid pipe of stderr to output file to allow leaks test to work as normal
 27 |     :param command:
 28 |     :return:
 29 |     """
 30 |     command_without_err_pipe: str = command
 31 |     # macOs uses leaks which uses stdout, thus we need to undo the piping of it
 32 |     if IS_MAC_OS:
 33 |         index_of_out_pipe = command_without_err_pipe.rfind('>')
 34 |         if index_of_out_pipe != -1:
 35 |             command_without_err_pipe = command_without_err_pipe[:index_of_out_pipe]
 36 |         index_of_err_out_pipe = command_without_err_pipe.rfind('&>')
 37 |         if index_of_err_out_pipe != -1:
 38 |             command_without_err_pipe = command_without_err_pipe[:index_of_err_out_pipe]
 39 |         index_of_err_pipe = command_without_err_pipe.rfind('2>')
 40 |         if index_of_err_pipe != -1:
 41 |             command_without_err_pipe = command_without_err_pipe[:index_of_err_pipe]
 42 |         # If stderr is piped to stdout, remove said piping
 43 |         return command_without_err_pipe.replace('&>1', '>').replace('2>&1', '')
 44 |     else:
 45 |         index_of_err_pipe = command_without_err_pipe.rfind('2>')
 46 |         index_of_out_pipe = command_without_err_pipe.rfind('>')
 47 | 
 48 |         if index_of_err_pipe != -1:
 49 |             if index_of_out_pipe > index_of_err_pipe:
 50 |                 command_without_err_pipe = command_without_err_pipe[
 51 |                                            :index_of_err_pipe] + command_without_err_pipe[
 52 |                                                                  index_of_out_pipe:]
 53 |             else:
 54 |                 command_without_err_pipe = command_without_err_pipe[:index_of_err_pipe]
 55 | 
 56 |         # If stderr is piped to stdout, remove said piping
 57 |         return command_without_err_pipe.replace('&>1', '>').replace('2>&1', '')
 58 | 
 59 | 
 60 | def summarize_failed_test(test_name: str, expected_output: str, actual_output: str, diff_html: str) -> Summary:
 61 |     return Summary(
 62 |         title=f"{test_name} - Failed!",
 63 |         expected=expected_output,
 64 |         actual=actual_output,
 65 |         error=None,
 66 |         diff_html=diff_html
 67 |     )
 68 | 
 69 | 
 70 | def summarize_failed_test_due_to_exception(test_name: str, expected_output: str,
 71 |                                            exception: str) -> Summary:
 72 |     return Summary(
 73 |         title=f"{test_name} - Failed due to an error in the tester!",
 74 |         expected=expected_output,
 75 |         error=exception
 76 |     )
 77 | 
 78 | 
 79 | def summarize_failed_to_check_for_leaks(test_name: str, exception: str) -> Summary:
 80 |     return Summary(
 81 |         title=f'{test_name} has leaks!{NORMAL_HTML_NEWLINE}Failed due to an error raised by {LEAKS_CHECKER_NAME}!',
 82 |         error=exception
 83 |     )
 84 | 
 85 | 
 86 | def parse_test_placeholders(field: str, ranged_value: Any) -> str:
 87 |     return field.replace(':::placeholder:::', str(ranged_value))
 88 | 
 89 | 
 90 | def parse_ranged_tests(tests: List[TestCase]) -> List[TestCase]:
 91 |     for index, test in enumerate(tests):
 92 |         test_range: TestParamRange | List[str] | None = test.get('params_range', None)
 93 |         if test_range:
 94 |             ranged_values: Iterable[Any]
 95 |             if type(test_range) == dict:
 96 |                 ranged_values = range(test_range['first'], test_range['last'] + 1)
 97 |             # In this case it will be a list of strs
 98 |             else:
 99 |                 ranged_values = test_range
100 |             for range_item in ranged_values:
101 |                 parsed_params: TestParams = dict()
102 |                 for name, value in test['params'].items():
103 |                     parsed_value = parse_test_placeholders(value, range_item)
104 |                     parsed_params[name] = parsed_value
105 | 
106 |                 parsed_test = {
107 |                     'name': parse_test_placeholders(test['name'], range_item),
108 |                     'template': test['template'],
109 |                     'params': parsed_params,
110 |                     'output_file': parse_test_placeholders(test['output_file'], range_item),
111 |                     EXPECTED_OUTPUT_FILE: parse_test_placeholders(test[EXPECTED_OUTPUT_FILE], range_item),
112 |                     'run_leaks': test.get('run_leaks', None),
113 |                     EXPECTED_OUTPUT_IS_SUBSTR: test.get(EXPECTED_OUTPUT_IS_SUBSTR, False)
114 |                 }
115 | 
116 |                 tests.append(parsed_test)
117 | 
118 |     # Remove unparsed tests
119 |     return [test for test in tests if 'params_range' not in test]


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # MatamGenericTester
  2 | 
  3 | # Configuration
  4 | - templates
  5 | Test command templates, in which parameters are spedified by being wrapped in ':::' (on both sides)
  6 | each template has a name as a key, and the test command template as its value.
  7 | example:
  8 | `
  9 | "templates": {
 10 |     "some_cmd_template": "foo :::param1::: :::param2::: > :::param3:::",
 11 |     "another_cmd_template": "bar :::a_param::: :::another_param::: > :::lol_param:::"
 12 |   }
 13 | `
 14 | - tests
 15 |   A lish of tests to be ran by the tester, each one has its own configuration
 16 | 
 17 |  ## Test Object configuration:
 18 |  - name
 19 |    Test name, will be used by the report generated by the tester
 20 |  - template
 21 |    Name of the command template to be used for the test
 22 |  - params
 23 |    Key-Value pairs of test params, that will be used (will replace the placeholders in the template)
 24 |    Key is param name (without the ':::' wrappers) and value is param value.
 25 |  - output_file
 26 |    Path For the program to output the test's result
 27 |  - expected_output_file
 28 |    Path for a file containing the expected output of the test
 29 | # Optional Test Object Config:
 30 |   - expected_output_is_substring:
 31 |     Boolean. If set to true, tester will consider test as successful if the test's output contains the expected output, instead of checking if they are outright the same
 32 |   - run_leaks
 33 |     Boolean, default is true.
 34 |     If set to false, will not run leak analysis on the test.
 35 |   - params_range
 36 |     Can be used to generate multiple test objects based on the same structure.
 37 |     Useful in case where user wants to create multiple tests that only vary by output/input file's name, but otherwise use the same templates, params, etc.
 38 |     Put the string ':::placeholder:::' in each place where the genreated test should get a generated value.
 39 |     Placeholder can be used in fields: 'name', 'output_file', 'expected_output_file' and parameters' values.
 40 |     Can be a dictionary (with int values) or a list of strings:
 41 |     ### Dictionary:
 42 |     -- first: lower numerical value
 43 |     -- last: upper numerical value
 44 |     This option is useful to generate multiple tests varying only by a number, where said number is a range starting with 'first' and ending with 'last'
 45 |     Example:
 46 |     `
 47 |     [{
 48 |       "name": "Example Test :::placeholder:::",
 49 |       "template": "game",
 50 |       "params": {
 51 |         "events": "example:::placeholder:::.events",
 52 |         "players": "example:::placeholder:::.players",
 53 |         "out": "example:::placeholder:::.out"
 54 |       },
 55 |       "params_range": {
 56 |         "first": 1,
 57 |         "last": 3
 58 |       },
 59 |       "run_leaks": true,
 60 |       "output_file": "example:::placeholder:::.out",
 61 |       "expected_output_file": "example:::placeholder:::.expected"
 62 |     }]
 63 |     `
 64 |     What will be generated:
 65 |     `
 66 |     [
 67 |     {
 68 |       "name": "Example Test 1",
 69 |       "template": "game",
 70 |       "params": {
 71 |         "events": "example1.events",
 72 |         "players": "example1.players",
 73 |         "out": "example1.out"
 74 |       },
 75 |       "run_leaks": true,
 76 |       "output_file": "example1.out",
 77 |       "expected_output_file": "example1.expected"
 78 |     },
 79 |     {
 80 |       "name": "Example Test 2",
 81 |       "template": "game",
 82 |       "params": {
 83 |         "events": "example2.events",
 84 |         "players": "example2.players",
 85 |         "out": "example2.out"
 86 |       },
 87 |       "run_leaks": true,
 88 |       "output_file": "example2.out",
 89 |       "expected_output_file": "example2.expected"
 90 |     },
 91 |     {
 92 |       "name": "Example Test 3",
 93 |       "template": "game",
 94 |       "params": {
 95 |         "events": "example3.events",
 96 |         "players": "example3.players",
 97 |         "out": "example3.out"
 98 |       },
 99 |       "run_leaks": true,
100 |       "output_file": "example3.out",
101 |       "expected_output_file": "example3.expected"
102 |     },
103 |     ]
104 |     `
105 |     
106 |     
107 |     ### List:
108 |     Each string in the list is used to replace the placeholder in one of the tests.
109 |     Example:
110 |     `
111 |     [{
112 |       "name": ":::placeholder:::",
113 |       "template": "game",
114 |       "params": {
115 |         "events": ":::placeholder:::.events",
116 |         "players": ":::placeholder:::.players",
117 |         "out": ":::placeholder:::.out"
118 |       },
119 |       "params_range": [
120 |         "test4-event-error-first",
121 |         "test5-generic-game",
122 |       ],
123 |       "run_leaks": true,
124 |       "output_file": ":::placeholder:::.out",
125 |       "expected_output_file": ":::placeholder:::.expected"
126 |     }]
127 |     `
128 |     What will be generated:
129 |     `
130 |     [{
131 |       "name": "test4-event-error-first",
132 |       "template": "game",
133 |       "params": {
134 |         "events": "test4-event-error-first.events",
135 |         "players": "test4-event-error-first.players",
136 |         "out": "test4-event-error-first.out"
137 |       },
138 |       "run_leaks": true,
139 |       "output_file": "test4-event-error-first.out",
140 |       "expected_output_file": "test4-event-error-first.expected"
141 |     },
142 |     {
143 |       "name": "test5-generic-game",
144 |       "template": "game",
145 |       "params": {
146 |         "events": "test5-generic-game.events",
147 |         "players": "test5-generic-game.players",
148 |         "out": "test5-generic-game.out"
149 |       },
150 |       "run_leaks": true,
151 |       "output_file": "test5-generic-game.out",
152 |       "expected_output_file": "test5-generic-game.expected"
153 |     }
154 |     ]
155 |     `
156 | 
157 |     ## Environment Variables
158 |     - MATAM_TESTER_TEST_TIMEOUT
159 |       Time (in seconds) until each test should be killed. Default is 1.
160 |     - MATAM_TESTER_VALGRIND_TIMEOUT
161 |       Time (in seconds) until each leak test should be killed. Default is 2.
162 |     - MATAM_TESTER_TRIMR_SPACES
163 |       Should ignore whitespaces in the start/end of lines when comparing results. Default is 0.
164 |     - MATAM_TESTER_IGNORE_EMPTY_LINES
165 |       Should ignore empty lines when comparing results. Default is 0.
166 |     - MATAM_TESTER_RUN_MULTI_THREADED
167 |       Should run tests using multiple threads. Default is 0.
168 |     - MATAM_TESTER_EXPORT_TEMP_REPORT
169 |       Should create a temporary report while before all tests are done, that is updated after every test. Useful when all tests combined take a long time to run. Can only be used when running in single thread mode.
170 |       Default is 0.
171 | 


--------------------------------------------------------------------------------
/utils/matam_html.py:
--------------------------------------------------------------------------------
  1 | from utils.config import NORMAL_HTML_NEWLINE, HTML_COLORED_NEWLINE, HTML_COLORED_WHITESPACE, USE_OLD_DIFF_STYLE
  2 | from utils.matam_types import Summary, TestResult
  3 | from os import getcwd, chdir
  4 | if not USE_OLD_DIFF_STYLE:
  5 |     import html
  6 |     import difflib
  7 | 
  8 | 
  9 | def simple_html_format(text: str) -> str:
 10 |     return text.replace('<', '&lt;').replace('>', '&gt;').replace('\n', '<br/>')
 11 | 
 12 | 
 13 | def format_test_string_for_html(field: str, field_title: str) -> str:
 14 |     less_than = '<'
 15 |     greater_than = '>'
 16 |     field = field.replace(less_than, '&lt;').replace(greater_than, '&gt;')
 17 |     field = field.replace(' \n', f'{HTML_COLORED_WHITESPACE}\n')
 18 |     field = field.replace('\n\n', f'{HTML_COLORED_NEWLINE}{NORMAL_HTML_NEWLINE}')
 19 |     if field.endswith('\n'):
 20 |         field = field[:-1] + HTML_COLORED_NEWLINE
 21 |     if field.endswith(' '):
 22 |         field = field[:-1] + HTML_COLORED_WHITESPACE
 23 |     field = field.replace('\n', NORMAL_HTML_NEWLINE)
 24 |     return f'<div class="grid-child-element"><p>{field_title}</p><p>{field}</p></div>'
 25 | 
 26 | 
 27 | def format_summary_for_html(summary: Summary) -> str:
 28 |     report = f'<p>{summary.get("title")}</p>'
 29 | 
 30 |     # If there’s a pre-rendered diff HTML, include it directly (no escaping)
 31 |     diff_html: str = summary.get("diff_html")
 32 |     error: str = summary.get("error")
 33 |     if not USE_OLD_DIFF_STYLE and diff_html is not None and error is None:
 34 |         report += f'<div class="diff-wrapper">{diff_html}</div>'
 35 |         return report  # Skip the standard grid in this case
 36 | 
 37 |     report += '<div class="grid-container-element">'
 38 |     expected: str = summary.get("expected")
 39 |     if expected is not None:
 40 |         report += format_test_string_for_html(expected, 'Expected Output:')
 41 | 
 42 |     if error is not None:
 43 |         report += format_test_string_for_html(error, 'Error:')
 44 |     actual: str = summary.get("actual")
 45 |     if actual is not None:
 46 |         report += format_test_string_for_html(actual, 'Actual Output:')
 47 |     report += '</div>'
 48 |     return report
 49 | 
 50 | 
 51 | def generate_summary_html_content(results: list[TestResult], amount_failed: int) -> str:
 52 |     html = '''
 53 | <!DOCTYPE html>
 54 | <html>
 55 | <head>
 56 |     <meta charset="utf-8">
 57 | </head>
 58 | <body>
 59 | 
 60 | 
 61 |     '''
 62 |     html += f'<h2><span style="color:red;">{amount_failed} Failed</span> out of {len(results)}</h2>'
 63 |     for result in results:
 64 |         command_element: str = f"<p>Test Command:</p><code>{simple_html_format(result['command'])}</code>" \
 65 |             if result.get('command', None) else ''
 66 |         html += f'''
 67 |         <button type="button" class="collapsible" style="color:{'green' if result['passed'] else 'red'}">
 68 |         {result['name']}</button>
 69 | <div class="content">
 70 |   {command_element}
 71 |   <p>{format_summary_for_html(result.get('summary'))}</p>
 72 | </div>
 73 | '''
 74 | 
 75 |     html += '''
 76 | </body>
 77 | </html>'''
 78 | 
 79 |     # Style for highlighting invisibles
 80 |     html += '''
 81 |         <style>
 82 |         .diff-container {
 83 |             display: grid;
 84 |             grid-template-columns: 1fr 1fr;
 85 |             gap: 10px;
 86 |             font-family: monospace;
 87 |             font-size: 14px;
 88 |             white-space: pre-wrap;
 89 |             border: 1px solid #ddd;
 90 |             border-radius: 6px;
 91 |             overflow-x: scroll;
 92 |             margin-bottom: 1em;
 93 |         }
 94 |         .diff-column {
 95 |             padding: 6px;
 96 |         }
 97 |         .diff-header {
 98 |             background: #f0f0f0;
 99 |             font-weight: bold;
100 |             padding: 4px;
101 |             text-align: center;
102 |             border-bottom: 1px solid #ddd;
103 |         }
104 |         .line {
105 |             padding: 1px 4px;
106 |         }
107 |         .line.keep { background: none; }
108 |         .line.added { background: #bfe4ca; color: #22863a; }
109 |         .line.removed { background: #ffeef0; color: #cb2431; }
110 |         .line.empty { background: #f9f9f9; color: #aaa; }
111 |         </style>
112 |         '''
113 | 
114 |     html += '''
115 | <script>
116 | var coll = document.getElementsByClassName("collapsible");
117 | var i;
118 | 
119 | for (i = 0; i < coll.length; i++) {
120 |   coll[i].addEventListener("click", function() {
121 |     this.classList.toggle("active");
122 |     var content = this.nextElementSibling;
123 |     if (content.style.display === "block") {
124 |       content.style.display = "none";
125 |     } else {
126 |       content.style.display = "block";
127 |     }
128 |   });
129 | }
130 | </script>
131 | <style>
132 | .collapsible {
133 |   background-color: #eee;
134 |   color: #444;
135 |   cursor: pointer;
136 |   padding: 18px;
137 |   width: 100%;
138 |   border: none;
139 |   text-align: left;
140 |   outline: none;
141 |   font-size: 15px;
142 | }
143 | 
144 | /* Add a background color to the button if it is clicked on (add the .active class with JS), and when you move the mouse over it (hover) */
145 | .active, .collapsible:hover {
146 |   background-color: #ccc;
147 | }
148 | 
149 | /* Style the collapsible content. Note: hidden by default */
150 | .content {
151 |   padding: 0 18px;
152 |   display: none;
153 |   overflow: hidden;
154 |   background-color: #f1f1f1;
155 | }
156 | .collapsible:after {
157 |   content: '\\02795'; /* Unicode character for "plus" sign (+) */
158 |   font-size: 13px;
159 |   color: white;
160 |   float: right;
161 |   margin-left: 5px;
162 | }
163 | 
164 | .active:after {
165 |   content: "\\2796"; /* Unicode character for "minus" sign (-) */
166 | }
167 | </style>
168 | <style>
169 | .content {
170 |   padding: 0 18px;
171 |   background-color: white;
172 |   max-height: 0;
173 |   overflow: hidden;
174 |   transition: max-height 0.2s ease-out;
175 | }
176 | </style>
177 | <style>
178 | .grid-container-element { 
179 |     display: grid; 
180 |     grid-template-columns: 1fr 1fr; 
181 |     grid-gap: 20px; 
182 |     width: 80%; 
183 | } 
184 | .grid-child-element { 
185 |     margin: 10px; 
186 |     border: 1px solid red; 
187 | }
188 | </style>
189 | 
190 | <script>
191 | var coll = document.getElementsByClassName("collapsible");
192 | var i;
193 | 
194 | for (i = 0; i < coll.length; i++) {
195 |   coll[i].addEventListener("click", function() {
196 |     this.classList.toggle("active");
197 |     var content = this.nextElementSibling;
198 |     if (content.style.maxHeight){
199 |       content.style.maxHeight = null;
200 |     } else {
201 |       content.style.maxHeight = content.scrollHeight + "px";
202 |     }
203 |   });
204 | }
205 | </script>
206 |     '''
207 |     return html
208 | 
209 | def create_html_report(html: str, html_name: str) -> None:
210 |     try:
211 |         with open(html_name, "w", encoding='utf-8') as file:
212 |             file.write(html)
213 |     except Exception as e:
214 |         print('Could not create html report. Report content:')
215 |         print(html)
216 |         raise e
217 | 
218 | 
219 | def create_html_report_from_results(results: list[TestResult], initial_workdir: str, html_name: str) -> None:
220 |     amount_failed: int = 0
221 |     for t in results:
222 |         if t.get('passed', False) is False:
223 |             amount_failed += 1
224 | 
225 |     html: str = generate_summary_html_content(results, amount_failed)
226 |     curr_workdir: str = getcwd()
227 |     chdir(initial_workdir)
228 |     create_html_report(html, html_name)
229 |     chdir(curr_workdir)
230 | 
231 | 
232 | def _mark_invisibles(s: str) -> str:
233 |     s = html.escape(s)
234 |     s = (
235 |         s.replace(" ", '·')
236 |         .replace("\t", '→')
237 |         .replace("\r", '␍')
238 |         .replace("\n", '⏎')
239 |     )
240 |     return s
241 | 
242 | def generate_side_by_side_diff(expected_output: str, actual_output: str, test_name: str) -> str | None:
243 |     """
244 |     Generate a clean, side-by-side HTML diff view (Expected | Actual)
245 |     with visible whitespace, tabs, CR/LF, and colored differences.
246 |     """
247 |     if USE_OLD_DIFF_STYLE:
248 |         return None
249 | 
250 |     # Keep newline characters so we can highlight missing/extra ones
251 |     expected_lines = expected_output.splitlines(keepends=True)
252 |     actual_lines = actual_output.splitlines(keepends=True)
253 | 
254 |     differ = difflib.Differ()
255 |     diff = list(differ.compare(expected_lines, actual_lines))
256 | 
257 |     left_column = []
258 |     right_column = []
259 | 
260 |     for line in diff:
261 |         tag = line[:2]
262 |         content = _mark_invisibles(line[2:])
263 | 
264 |         if tag == "  ":  # unchanged
265 |             left_column.append(f"<div class='line keep'>{content}</div>")
266 |             right_column.append(f"<div class='line keep'>{content}</div>")
267 |         elif tag == "- ":  # only in expected
268 |             left_column.append(f"<div class='line removed'>{content}</div>")
269 |             right_column.append("<div class='line empty'></div>")
270 |         elif tag == "+ ":  # only in actual
271 |             left_column.append("<div class='line empty'></div>")
272 |             right_column.append(f"<div class='line added'>{content}</div>")
273 |         # ignoring diff hints: "? "
274 | 
275 |     # Balance both columns’ height
276 |     max_len = max(len(left_column), len(right_column))
277 |     while len(left_column) < max_len:
278 |         left_column.append("<div class='line empty'></div>")
279 |     while len(right_column) < max_len:
280 |         right_column.append("<div class='line empty'></div>")
281 | 
282 |     html_diff = f"""
283 | 
284 | 
285 |     <div class="diff-container">
286 |         <div class="diff-column">
287 |             <div class="diff-header">Expected</div>
288 |             {''.join(left_column)}
289 |         </div>
290 |         <div class="diff-column">
291 |             <div class="diff-header">Actual</div>
292 |             {''.join(right_column)}
293 |         </div>
294 |     </div>
295 |     """
296 | 
297 |     return f"<h3>{html.escape(test_name)}</h3>{html_diff}"
298 | 


--------------------------------------------------------------------------------
/run_tests.py:
--------------------------------------------------------------------------------
  1 | import sys
  2 | from os import getcwd, chdir, linesep
  3 | from os.path import dirname, join, normpath, isfile, isdir
  4 | import subprocess
  5 | import json
  6 | 
  7 | from multiprocessing.dummy import Pool as ThreadPool
  8 | 
  9 | from utils.config import RUN_MULTI_THREAD, FINAL_REPORT, EXECUTABLE_INDEX, TESTS_JSON_FILE_INDEX, \
 10 |     EXPECTED_ARGS_AMOUNT, \
 11 |     TIMEOUT, COMPARISON_IGNORE_BLANK_LINES, COMPARISON_TRIM_END_SPACES, VALGRIND_TIMEOUT, STDERR, \
 12 |     STDOUT, \
 13 |     LEAKS_CHECKER_NAME, NO_LEAKS_FOUND_TEXT, TEMPLATE_NAME, PARAMS, TEST_NAME, EXPECTED_OUTPUT_FILE, \
 14 |     EXPECTED_OUTPUT_IS_SUBSTR, OUTPUT_FILE, EXPORT_TEMP_REPORT, LEAKS_CHECKER_COMMAND, TEMP_REPORT
 15 | from utils.loading_bar import print_progress_bar
 16 | from utils.matam_html import create_html_report_from_results, generate_side_by_side_diff
 17 | from utils.matam_parsing import summarize_failed_test_due_to_exception, \
 18 |     test_exception_to_error_text, \
 19 |     normalize_newlines, summarize_failed_test, summarize_failed_to_check_for_leaks, \
 20 |     remove_error_pipes_from_command, \
 21 |     parse_ranged_tests
 22 | from utils.matam_types import TestResult, TestFile, Summary, TestCase, TestTemplates
 23 | 
 24 | if sys.version_info < (3, 10):
 25 |     sys.exit("Python %s.%s or later is required.\n" % (3, 10))
 26 | else:
 27 |     from typing import get_type_hints
 28 | 
 29 | 
 30 | def execute_test(command: str, relative_workdir: str, name: str, expected_output: str,
 31 |                  output_path: str,
 32 |                  results: list[TestResult], expected_is_substr: bool = False) -> None:
 33 |     try:
 34 |         with subprocess.Popen(command, shell=True, cwd=getcwd()) as proc:
 35 |             try:
 36 |                 proc.communicate(timeout=TIMEOUT)
 37 |             except subprocess.TimeoutExpired as e:
 38 |                 proc.kill()
 39 |                 results.append({
 40 |                     'name': name,
 41 |                     'summary': summarize_failed_test_due_to_exception(name, expected_output,
 42 |                                                                       test_exception_to_error_text(
 43 |                                                                           e)),
 44 |                     'passed': False,
 45 |                     'command': f'export TESTER_TMP_PWD=$(pwd) && cd {relative_workdir} && {command} && cd $TESTER_TMP_PWD && unset TESTER_TMP_PWD'
 46 |                 })
 47 |                 return
 48 |     except subprocess.CalledProcessError as e:
 49 |         results.append({
 50 |             'name': name,
 51 |             'summary': summarize_failed_test_due_to_exception(name, expected_output,
 52 |                                                               test_exception_to_error_text(e)),
 53 |             'passed': False,
 54 |             'command': f'export TESTER_TMP_PWD=$(pwd) && cd {relative_workdir} && {command} && cd $TESTER_TMP_PWD && unset TESTER_TMP_PWD'
 55 |         })
 56 |         return
 57 |     except subprocess.TimeoutExpired as e:
 58 |         results.append({
 59 |             'name': name,
 60 |             'summary': summarize_failed_test_due_to_exception(name, expected_output,
 61 |                                                               test_exception_to_error_text(e)),
 62 |             'passed': False,
 63 |             'command': f'export TESTER_TMP_PWD=$(pwd) && cd {relative_workdir} && {command} && cd $TESTER_TMP_PWD && unset TESTER_TMP_PWD'
 64 |         })
 65 |         return
 66 |     except Exception as e:
 67 |         results.append({
 68 |             'name': name,
 69 |             'summary': summarize_failed_test_due_to_exception(name, expected_output, str(e)),
 70 |             'passed': False,
 71 |             'command': f'export TESTER_TMP_PWD=$(pwd) && cd {relative_workdir} && {command} && cd $TESTER_TMP_PWD && unset TESTER_TMP_PWD'
 72 |         })
 73 |         return
 74 | 
 75 |     try:
 76 |         # norm path makes sure the path is formatted correctly
 77 |         with open(normpath(output_path), "r", encoding='utf-8') as file:
 78 |             actual_output = normalize_newlines(file.read())
 79 |     except UnicodeDecodeError as e:
 80 |         results.append({
 81 |             'name': name,
 82 |             'summary': summarize_failed_test_due_to_exception(name, expected_output,
 83 |                                                               f'Test printed invalid output. Exception: {str(e)}'),
 84 |             'passed': False,
 85 |             'command': f'export TESTER_TMP_PWD=$(pwd) && cd {relative_workdir} && {command} && cd $TESTER_TMP_PWD && unset TESTER_TMP_PWD'
 86 |         })
 87 |         return
 88 |     except FileNotFoundError as e:
 89 |         results.append({
 90 |             'name': name,
 91 |             'summary': summarize_failed_test_due_to_exception(name, expected_output,
 92 |                                                               f'Test failed to provide output. Exception: {str(e)}'),
 93 |             'passed': False,
 94 |             'command': f'export TESTER_TMP_PWD=$(pwd) && cd {relative_workdir} && {command} && cd $TESTER_TMP_PWD && unset TESTER_TMP_PWD'
 95 |         })
 96 |         return
 97 | 
 98 |     # Remove blank lines
 99 |     if COMPARISON_IGNORE_BLANK_LINES != 0:
100 |         actual_output = linesep.join([s for s in actual_output.splitlines() if s])
101 |         expected_output = linesep.join([s for s in expected_output.splitlines() if s])
102 | 
103 |     # Trim spaces from end of lines
104 |     if COMPARISON_TRIM_END_SPACES != 0:
105 |         actual_output = linesep.join([s.rstrip() for s in actual_output.splitlines()])
106 |         expected_output = linesep.join([s.rstrip() for s in expected_output.splitlines()])
107 | 
108 |     compare_result: bool = actual_output == expected_output
109 |     if expected_is_substr:
110 |         compare_result = expected_output in actual_output
111 | 
112 |     if compare_result:
113 |         results.append({
114 |             'name': name,
115 |             'summary': Summary(title=f"\n{name} - Passed!\n"),
116 |             'passed': True
117 |         })
118 |     else:
119 |         diff_html = generate_side_by_side_diff(expected_output, actual_output, name)
120 |         results.append({
121 |             'name': name,
122 |             'summary': summarize_failed_test(name, expected_output, actual_output, diff_html),
123 |             'passed': False,
124 |             'command': f'export TESTER_TMP_PWD=$(pwd) && cd {relative_workdir} && {command} && cd $TESTER_TMP_PWD && unset TESTER_TMP_PWD'
125 |         })
126 | 
127 | 
128 | def execute_memory_leaks_test(command: str, relative_workdir: str, name: str,
129 |                               results: list[TestResult]) -> None:
130 |     try:
131 |         proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True,
132 |                                 cwd=getcwd())
133 |         try:
134 |             proc_result = proc.communicate(timeout=VALGRIND_TIMEOUT)
135 |             result = proc_result[STDERR] if proc_result[STDERR] else proc_result[STDOUT]
136 |         except subprocess.TimeoutExpired:
137 |             proc.kill()
138 |             result = proc.communicate()
139 |             # Try using stderr or fallback to stdout
140 |             result = result[STDERR] if result[STDERR] else result[STDOUT]
141 |         try:
142 |             actual_output = normalize_newlines(result.decode('utf-8'))
143 |         except UnicodeDecodeError:
144 |             actual_output = normalize_newlines(result.decode('windows-1252'))
145 | 
146 |     except subprocess.CalledProcessError as e:
147 |         results.append({
148 |             'name': f'{name} - {LEAKS_CHECKER_NAME}',
149 |             'summary': summarize_failed_to_check_for_leaks(name, test_exception_to_error_text(e)),
150 |             'passed': False,
151 |             'command': f'export TESTER_TMP_PWD=$(pwd) && cd {relative_workdir} && {command} && cd $TESTER_TMP_PWD && unset TESTER_TMP_PWD'
152 |         })
153 |         return
154 |     except subprocess.TimeoutExpired as e:
155 |         results.append({
156 |             'name': f'{name} - {LEAKS_CHECKER_NAME}',
157 |             'summary': summarize_failed_to_check_for_leaks(name, test_exception_to_error_text(e)),
158 |             'passed': False,
159 |             'command': f'export TESTER_TMP_PWD=$(pwd) && cd {relative_workdir} && {command} && cd $TESTER_TMP_PWD && unset TESTER_TMP_PWD'
160 |         })
161 |         return
162 |     except Exception as e:
163 |         results.append({
164 |             'name': f'{name} - {LEAKS_CHECKER_NAME}',
165 |             'summary': summarize_failed_to_check_for_leaks(name, str(e)),
166 |             'passed': False,
167 |             'command': f'export TESTER_TMP_PWD=$(pwd) && cd {relative_workdir} && {command} && cd $TESTER_TMP_PWD && unset TESTER_TMP_PWD'
168 |         })
169 | 
170 |         return
171 | 
172 |     if NO_LEAKS_FOUND_TEXT in actual_output:
173 |         results.append({
174 |             'name': f'{name} - {LEAKS_CHECKER_NAME}',
175 |             'summary': Summary(title=f"\n{name} - no Leaks!\n"),
176 |             'passed': True
177 |         })
178 |         return
179 |     else:
180 |         results.append({
181 |             'name': f'{name} - {LEAKS_CHECKER_NAME}',
182 |             'summary': summarize_failed_to_check_for_leaks(name, actual_output),
183 |             'passed': False,
184 |             'command': f'export TESTER_TMP_PWD=$(pwd) && cd {relative_workdir} && {command} && cd $TESTER_TMP_PWD && unset TESTER_TMP_PWD'
185 |         })
186 | 
187 | 
188 | def run_test(executable_path: str, relative_workdir: str, initial_workdir: str, test: TestCase,
189 |              templates: TestTemplates,
190 |              results: list[TestResult], total_tests: int) -> None:
191 |     for key, key_type in get_type_hints(TestCase).items():
192 |         if key == 'params_range':
193 |             continue
194 | 
195 |         # If key is missing and None is not a valid type for said key
196 |         if key not in test and not isinstance(None, key_type.__args__):
197 |             name = test.get("name", "<missing>")
198 |             results.append({
199 |                 'name': name,
200 |                 'summary': Summary(
201 |                     title=f"\nTest \"{name}\": \"{key}\" missing from test object\n"),
202 |                 'passed': False
203 |             })
204 |             print_progress_bar(len(results), total_tests, prefix='Progress:', suffix='Complete',
205 |                                length=50)
206 |             return
207 | 
208 |     args: str = templates[test[TEMPLATE_NAME]]
209 |     for param_name, param_value in test[PARAMS].items():
210 |         args = args.replace(f':::{param_name}:::', param_value)
211 | 
212 |     name: str = test[TEST_NAME]
213 |     expected_output_path = test.get(EXPECTED_OUTPUT_FILE, None)
214 |     expected_is_substr: bool = test.get(EXPECTED_OUTPUT_IS_SUBSTR, False)
215 |     # norm path makes sure the path is formatted correctly
216 |     with open(normpath(expected_output_path), "r", encoding='utf-8') as file:
217 |         expected_output = normalize_newlines(file.read())
218 | 
219 |     output_path = test[OUTPUT_FILE]
220 |     test_command: str = f'{executable_path} {args}'
221 | 
222 |     execute_test(test_command, relative_workdir, name,
223 |                  expected_output, output_path, results, expected_is_substr=expected_is_substr)
224 |     if test.get("run_leaks") is not False:
225 |         command_without_err_pipes: str = remove_error_pipes_from_command(test_command)
226 |         leaks_check_command: str = f'{LEAKS_CHECKER_COMMAND} {command_without_err_pipes}'
227 |         execute_memory_leaks_test(leaks_check_command, relative_workdir, name, results)
228 |     # Advancing progress bar
229 |     print_progress_bar(len(results), total_tests, prefix='Progress:', suffix='Complete', length=50)
230 |     if EXPORT_TEMP_REPORT and not RUN_MULTI_THREAD:
231 |         create_html_report_from_results(results, initial_workdir, TEMP_REPORT)
232 | 
233 | 
234 | def get_tests_data_from_json(tests_file_path: str) -> TestFile:
235 |     try:
236 |         with open(tests_file_path, "r", encoding='utf-8') as file:
237 |             json_data = json.load(file)
238 |             return json_data
239 |     except (IOError, json.JSONDecodeError) as e:
240 |         print(f"Error reading tests JSON file: {e}")
241 |         raise e
242 |     except Exception as e:
243 |         print(f"Unexpected error reading tests JSON file: {e}")
244 |         raise e
245 | 
246 | 
247 | def main():
248 |     # Expect 3 at least args: script name, json path, executable path (may comprise multiple args if command is complex)
249 |     if len(sys.argv) < EXPECTED_ARGS_AMOUNT:
250 |         print(
251 |             f"Bad Usage of local tester, make sure executable path and json test file's path are passed properly." +
252 |             f" Total args passed: {len(sys.argv)}"
253 |         )
254 |         return
255 | 
256 |     initial_workdir = getcwd()
257 | 
258 |     # If EXECUTABLE_INDEX is a file, wrap it in ' so it works even with spaces in path
259 |     exec_path = normpath(join(initial_workdir, sys.argv[EXECUTABLE_INDEX]))
260 |     if isfile(exec_path):
261 |         executable = f"'{exec_path}'"
262 |     else:
263 |         executable = exec_path
264 | 
265 |     # Build executable. May include multiple inputs, any input that comes beginning in EXECUTABLE_INDEX
266 |     if len(sys.argv) > EXPECTED_ARGS_AMOUNT:
267 |         for i in range(EXECUTABLE_INDEX + 1, len(sys.argv)):
268 |             # norm path makes sure the path is formatted correctly
269 |             # If arg is a file or a dir, wrap it in ' in case it contains a space
270 |             curr_arg = normpath(join(initial_workdir, sys.argv[i]))
271 |             if isfile(curr_arg) or isdir(curr_arg):
272 |                 curr_arg = f"'{curr_arg}'"
273 |             executable += ' ' + curr_arg
274 |     tests_file_path = normpath(join(initial_workdir, sys.argv[TESTS_JSON_FILE_INDEX]))
275 | 
276 |     workdir = dirname(tests_file_path)
277 |     relative_workdir = dirname(sys.argv[TESTS_JSON_FILE_INDEX])
278 |     chdir(workdir)
279 | 
280 |     tests_data: TestFile = get_tests_data_from_json(tests_file_path)
281 |     tests_data['tests'] = parse_ranged_tests(tests_data['tests'])
282 |     results: list[TestResult] = []
283 | 
284 |     print("Running tests, please wait", end="", flush=True)
285 |     fn_args = []
286 | 
287 |     total_tests: int = 0
288 |     for test in tests_data['tests']:
289 |         # Default for run_leaks is true (for example, if not specified/defined)
290 |         if test.get('run_leaks', True):
291 |             total_tests += 2
292 |         else:
293 |             total_tests += 1
294 | 
295 |     print_progress_bar(0, total_tests, prefix='Progress:', suffix='Complete', length=50)
296 |     for test in tests_data['tests']:
297 |         fn_args.append(
298 |             (executable, relative_workdir, initial_workdir, test, tests_data['templates'], results,
299 |              total_tests)
300 |         )
301 | 
302 |     if RUN_MULTI_THREAD:
303 |         # none to use cpu count
304 |         pool = ThreadPool(None)
305 | 
306 |         pool.starmap(run_test, fn_args)
307 | 
308 |         pool.close()
309 |         pool.join()
310 |     else:
311 |         for args in fn_args:
312 |             run_test(*args)
313 | 
314 |     # Print new line to avoid console starting on same line as the loading bar
315 |     print("\n", end="", flush=True)
316 |     create_html_report_from_results(results, initial_workdir, FINAL_REPORT)
317 |     chdir(initial_workdir)
318 | 
319 | 
320 | if __name__ == "__main__":
321 |     main()
322 | 


--------------------------------------------------------------------------------