├── .gitignore ├── LICENSE ├── README.rst ├── bin └── testcode.py ├── docs ├── .static │ └── dummy_file ├── .templates │ └── dummy_file ├── Makefile ├── conf.py ├── configuration_files.rst ├── index.rst ├── installation.rst ├── jobconfig.rst ├── testcode.py.rst ├── userconfig.rst └── verification.rst └── lib └── testcode2 ├── __init__.py ├── _functools_dummy.py ├── ansi.py ├── compatibility.py ├── config.py ├── dir_lock.py ├── exceptions.py ├── queues.py ├── util.py ├── validation.py └── vcs.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *.swp 3 | *.pyc 4 | docs/.build 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 by James Spencer. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, 7 | this list of conditions and the following disclaimer. 8 | 2. Redistributions in binary form must reproduce the above copyright notice, 9 | this list of conditions and the following disclaimer in the documentation 10 | and/or other materials provided with the distribution. 11 | 3. The name of the author may not be used to endorse or promote products 12 | derived from this software without specific prior written permission. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE AUTHOR ''AS IS'' AND ANY EXPRESS OR IMPLIED 15 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 16 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 17 | EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 18 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 19 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 20 | OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 21 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 22 | OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 23 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | testcode 2 | ======== 3 | 4 | testcode is a python module for testing for regression errors in numerical 5 | (principally scientific) software. Essentially testcode runs a set of 6 | calculations, and compares the output data to that generated by a previous 7 | calculation (which is regarded to be "correct"). It is designed to be 8 | lightweight and highly portable. 9 | 10 | testcode can run a set of tests and check the calculated data is within a the 11 | desired tolerance of results contained in previous output (using an internal 12 | data extraction engine, a user-supplied data extraction program or 13 | a user-supplied verification program). The programs to be tested can be run in 14 | serial and in parallel and tests can be run in either locally or submitted to 15 | a compute cluster running a queueing system such as PBS. Previous tests can be 16 | compared and diffed against other tests or benchmarks. 17 | 18 | Documentation 19 | ------------- 20 | 21 | Full documentation can be found in the ``docs/`` subdirectory and in the 22 | appropriate docstrings. Documentation can be compiled using `sphinx 23 | `_. 24 | 25 | Documentation can also be viewed at `readthedocs 26 | `_. 27 | 28 | Author 29 | ------ 30 | 31 | James Spencer, Imperial College London. 32 | 33 | Contributions and suggestions from: 34 | 35 | Keith Refson, Science and Technology Facilities Council. 36 | 37 | Shawn Chin, Science and Technology Facilities Council. 38 | 39 | LICENSE 40 | ------- 41 | 42 | Modified BSD license; see LICENSE for more details. 43 | 44 | See also 45 | -------- 46 | 47 | `testcode_buildbot.py `_: a custom buildbot BuildStep for running testcode by Shawn Chin. 48 | -------------------------------------------------------------------------------- /bin/testcode.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | '''testcode [options] [action1 [action2...]] 3 | 4 | testcode is a simple framework for comparing output from (principally numeric) 5 | programs to previous output to reveal regression errors or miscompilation. 6 | 7 | Run a set of actions on a set of tests. 8 | 9 | Available actions: 10 | compare compare set of test outputs from a previous testcode 11 | run against the benchmark outputs. 12 | diff diff set of test outputs from a previous testcode 13 | run against the benchmark outputs. 14 | make-benchmarks create a new set of benchmarks and update the userconfig 15 | file with the new benchmark id. Also forces the tests 16 | to be run unless the 'compare' action is also given. 17 | recheck compare a set of test outputs and rerun failed tests. 18 | run run a set of tests and compare against the benchmark 19 | outputs. Default action. 20 | tidy Remove files from previous testcode runs from the test 21 | directories. 22 | 23 | Requires two configuration files, jobconfig and userconfig. See testcode 24 | documentation for further details.''' 25 | 26 | # copyright: (c) 2012 James Spencer 27 | # license: modified BSD; see LICENSE for more details 28 | 29 | import glob 30 | import optparse 31 | import os 32 | import re 33 | import subprocess 34 | import sys 35 | import threading 36 | import time 37 | 38 | try: 39 | import testcode2 40 | except ImportError: 41 | # try to find testcode2 assuming it is being run directly from the source 42 | # layout. 43 | SCRIPT_DIR = os.path.abspath(os.path.dirname(sys.argv[0])) 44 | TESTCODE2_LIB = os.path.join(SCRIPT_DIR, '../lib/') 45 | sys.path.extend([TESTCODE2_LIB]) 46 | import testcode2 47 | 48 | import testcode2.config 49 | import testcode2.util 50 | import testcode2.compatibility 51 | import testcode2.exceptions 52 | import testcode2.validation 53 | 54 | #--- testcode initialisation --- 55 | 56 | def init_tests(userconfig, jobconfig, test_id, reuse_id, executables=None, 57 | categories=None, nprocs=-1, benchmark=None, userconfig_options=None, 58 | jobconfig_options=None): 59 | '''Initialise tests from the configuration files and command-line options. 60 | 61 | userconfig, executables, test_id and userconfig_options are passed to 62 | testcode2.config.userconfig. 63 | 64 | jobconfig and jobconfig_options are passed to testcode2.config.parse_jobconfig. 65 | 66 | categories is passed to testcode2.config.select_tests. 67 | 68 | test_id is used to set the test identifier. If test_id is null and reused_id 69 | is true, then the identifier is set to that of the last tests ran by testcode 70 | otherwise a unique identifier based upon the date is used. 71 | 72 | nprocs is the number of processors each test is run on. If negative, the 73 | defaults in the configuration files are used. 74 | 75 | benchmark is the benchmark id labelling the set of benchmarks to compare the 76 | tests too. If None, the default in userconfig is used. 77 | 78 | Returns: 79 | 80 | user_options: dictionary containing user options specified in userconfig. 81 | test_programs: dict of the test programs defined in userconfig. 82 | tests: list of selected tests. 83 | ''' 84 | 85 | config_exists = os.path.exists(userconfig) and os.path.exists(jobconfig) 86 | 87 | try: 88 | (user_options, test_programs) = testcode2.config.parse_userconfig( 89 | userconfig, executables, test_id, userconfig_options) 90 | except testcode2.exceptions.TestCodeError: 91 | err = str(sys.exc_info()[1]) 92 | if not config_exists: 93 | err += (' Please run from a directory containing (or specify) the ' 94 | 'userconfig file. Use ``--help`` to see available options.') 95 | raise testcode2.exceptions.TestCodeError(err) 96 | 97 | # Set benchmark if required. 98 | if benchmark: 99 | for key in test_programs: 100 | test_programs[key].benchmark = [benchmark] 101 | 102 | try: 103 | (tests, test_categories) = testcode2.config.parse_jobconfig( 104 | jobconfig, user_options, test_programs, jobconfig_options) 105 | except testcode2.exceptions.TestCodeError: 106 | err = str(sys.exc_info()[1]) 107 | if not config_exists: 108 | err += (' Please run from a directory containing (or specify) the ' 109 | 'jobconfig file. Use ``--help`` to see available options.') 110 | raise testcode2.exceptions.TestCodeError(err) 111 | 112 | # Set number of processors... 113 | if nprocs >= 0: 114 | for test in tests: 115 | test.nprocs = nprocs 116 | if test.nprocs < test.min_nprocs: 117 | test.nprocs = test.min_nprocs 118 | if test.nprocs > test.max_nprocs: 119 | test.nprocs = test.max_nprocs 120 | 121 | # parse selected job categories from command line 122 | # Remove those tests which weren't run most recently if comparing. 123 | if categories: 124 | tests = testcode2.config.select_tests(tests, test_categories, 125 | categories, os.path.abspath(os.path.dirname(userconfig))) 126 | 127 | # Sort by path (as that's how they appear in the user's directory). 128 | tests.sort(key=lambda test: test.path) 129 | 130 | if not test_id: 131 | test_id = testcode2.config.get_unique_test_id(tests, reuse_id, 132 | user_options['date_fmt']) 133 | for key in test_programs: 134 | test_programs[key].test_id = test_id 135 | 136 | return (user_options, test_programs, tests) 137 | 138 | #--- create command line interface --- 139 | 140 | def parse_cmdline_args(args): 141 | '''Parse command line arguments. 142 | 143 | args: list of supplied arguments. 144 | 145 | Returns: 146 | 147 | options: object returned by optparse containing the options. 148 | actions: list of testcode2 actions to run. 149 | ''' 150 | 151 | # Curse not being able to use argparse in order to support python <= 2.7! 152 | parser = optparse.OptionParser(usage=__doc__) 153 | 154 | allowed_actions = ['compare', 'run', 'diff', 'tidy', 'make-benchmarks', 155 | 'recheck'] 156 | 157 | parser.add_option('-b', '--benchmark', help='Set the file ID of the ' 158 | 'benchmark files. Default: specified in the [user] section of the ' 159 | 'userconfig file.') 160 | parser.add_option('-c', '--category', action='append', default=[], 161 | help='Select the category/group of tests. Can be specified ' 162 | 'multiple times. Default: use the _default_ category if run is an ' 163 | 'action unless make-benchmarks is an action. All other cases use ' 164 | 'the _all_ category by default. The _default_ category contains ' 165 | 'all tests unless otherwise set in the jobconfig file.') 166 | parser.add_option('-e', '--executable', action='append', default=[], 167 | help='Set the executable(s) to be used to run the tests. Can be' 168 | ' a path or name of an option in the userconfig file, in which' 169 | ' case all test programs are set to use that value, or in the' 170 | ' format program_name=value, which affects only the specified' 171 | ' program.') 172 | parser.add_option('-f', '--first-run', action='store_true', default=False, 173 | dest='first_run', help='Run tests that were not were not run in ' 174 | 'the previous testcode run. Only relevant to the recheck action. ' 175 | 'Default: %default.') 176 | parser.add_option('-i', '--insert', action='store_true', default=False, 177 | help='Insert the new benchmark into the existing list of benchmarks' 178 | ' in userconfig rather than overwriting it. Only relevant to the' 179 | ' make-benchmarks action. Default: %default.') 180 | parser.add_option('--jobconfig', default='jobconfig', help='Set path to the' 181 | ' job configuration file. Default: %default.') 182 | parser.add_option('--job-option', action='append', dest='job_option', 183 | default=[], nargs=3, help='Override/add setting to jobconfig. ' 184 | 'Takes three arguments. Format: section_name option_name value. ' 185 | 'Default: none.') 186 | parser.add_option('--older-than', type='int', dest='older_than', default=14, 187 | help='Set the age (in days) of files to remove. Only relevant to ' 188 | 'the tidy action. Default: %default days.') 189 | parser.add_option('-p', '--processors', type='int', default=-1, 190 | dest='nprocs', help='Set the number of processors to run each test ' 191 | 'on. Default: use settings in configuration files.') 192 | parser.add_option('-q', '--quiet', action='store_const', const=0, 193 | dest='verbose', default=1, help='Print only minimal output. ' 194 | 'Default: False.') 195 | parser.add_option('-s', '--submit', dest='queue_system', default=None, 196 | help='Submit tests to a queueing system of the specified type. ' 197 | 'Only PBS system is currently implemented. Default: %default.') 198 | parser.add_option('-t', '--test-id', dest='test_id', help='Set the file ID ' 199 | 'of the test outputs. Default: unique filename based upon date ' 200 | 'if running tests and most recent test_id if comparing tests.') 201 | parser.add_option('--total-processors', type='int', default=-1, 202 | dest='tot_nprocs', help='Set the total number of processors to use ' 203 | 'to run tests concurrently. Relevant only to the run option. ' 204 | 'Default: run all tests concurrently run if --submit is used; run ' 205 | 'tests sequentially otherwise.') 206 | parser.add_option('--userconfig', default='userconfig', help='Set path to ' 207 | 'the user configuration file. Default: %default.') 208 | parser.add_option('--user-option', action='append', dest='user_option', 209 | default=[], nargs=3, help='Override/add setting to userconfig. ' 210 | 'Takes three arguments. Format: section_name option_name value. ' 211 | 'Default: none.') 212 | parser.add_option('-v', '--verbose', default=1, action="count", 213 | dest='verbose', help='Increase verbosity of output. Can be ' 214 | 'specified multiple times.') 215 | 216 | (options, args) = parser.parse_args(args) 217 | 218 | # Default action. 219 | if not args or ('make-benchmarks' in args and 'compare' not in args 220 | and 'run' not in args and 'recheck' not in args): 221 | # Run tests by default if no action provided. 222 | # Run tests before creating benchmark by default. 223 | args.append('run') 224 | 225 | # Default category. 226 | if not options.category: 227 | # We quietly filter out tests which weren't run last when diffing 228 | # or comparing. 229 | options.category = ['_all_'] 230 | if 'run' in args and 'make-benchmarks' not in args: 231 | options.category = ['_default_'] 232 | 233 | test_args = (arg not in allowed_actions for arg in args) 234 | if testcode2.compatibility.compat_any(test_args): 235 | print('At least one action is not understood: %s.' % (' '.join(args))) 236 | parser.print_usage() 237 | sys.exit(1) 238 | 239 | # Parse executable option to form dictionary in format expected by 240 | # parse_userconfig. 241 | exe = {} 242 | for item in options.executable: 243 | words = item.split('=') 244 | if len(words) == 1: 245 | # setting executable for all programs (unless otherwise specified) 246 | exe['_tc_all'] = words[0] 247 | else: 248 | # format: program_name=executable 249 | exe[words[0]] = words[1] 250 | options.executable = exe 251 | 252 | # Set FILESTEM if test_id refers to a benchmark file or the benchmark 253 | # refers to a test_id. 254 | filestem = testcode2.FILESTEM.copy() 255 | if options.benchmark and options.benchmark[:2] == 't:': 256 | filestem['benchmark'] = testcode2.FILESTEM['test'] 257 | options.benchmark = options.benchmark[2:] 258 | if options.test_id and options.test_id[:2] == 'b:': 259 | filestem['test'] = testcode2.FILESTEM['benchmark'] 260 | options.test_id = options.test_id[2:] 261 | if filestem['test'] != testcode2.FILESTEM['test'] and 'run' in args: 262 | print('Not allowed to set test filename to be a benchmark filename ' 263 | 'when running calculations.') 264 | sys.exit(1) 265 | testcode2.FILESTEM = filestem.copy() 266 | 267 | # Convert job-options and user-options to dict of dicsts format. 268 | for item in ['user_option', 'job_option']: 269 | uj_opt = getattr(options, item) 270 | opt = dict( (section, {}) for section in 271 | testcode2.compatibility.compat_set(opt[0] for opt in uj_opt) ) 272 | for (section, option, value) in uj_opt: 273 | opt[section][option] = value 274 | setattr(options, item, opt) 275 | 276 | return (options, args) 277 | 278 | #--- actions --- 279 | 280 | def run_tests(tests, verbose=1, cluster_queue=None, tot_nprocs=0): 281 | '''Run tests. 282 | 283 | tests: list of tests. 284 | verbose: level of verbosity in output. 285 | cluster_queue: name of cluster system to use. If None, tests are run locally. 286 | Currently only PBS is implemented. 287 | tot_nprocs: total number of processors available to run tests on. As many 288 | tests (in a LIFO fashion from the tests list) are run at the same time as 289 | possible without using more processors than this value. If less than 1 and 290 | cluster_queue is specified, then all tests are submitted to the cluster at 291 | the same time. If less than one and cluster_queue is not set, then 292 | tot_nprocs is ignored and the tests are run sequentially (default). 293 | ''' 294 | def run_test_worker(semaphore, semaphore_lock, tests, *run_test_args): 295 | '''Launch a test after waiting until resources are available to run it. 296 | 297 | semaphore: threading.Semaphore object containing the number of cores/processors 298 | which can be used concurrently to run tests. 299 | semaphore.lock: threading.Lock object used to restrict acquiring the semaphore 300 | to one thread at a time. 301 | tests: list of (serialized) tests to run in this thread. 302 | run_test_args: arguments to pass to test.run_test method. 303 | ''' 304 | 305 | # Ensure that only one test attempts to register resources with the 306 | # semaphore at a time. This restricts running the tests to a LIFO 307 | # fashion which is not perfect (we don't attempt to backfill with 308 | # smaller tests, for example) but is a reasonable and (most 309 | # importantly) simple first-order approach. 310 | for test in tests: 311 | semaphore_lock.acquire() 312 | # test.nprocs is <1 when program is run in serial. 313 | nprocs_used = max(1, test.nprocs) 314 | for i in range(nprocs_used): 315 | semaphore.acquire() 316 | semaphore_lock.release() 317 | 318 | test.run_test(*run_test_args) 319 | 320 | for i in range(nprocs_used): 321 | semaphore.release() 322 | 323 | # Check executables actually exist... 324 | compat = testcode2.compatibility 325 | executables = [test.test_program.exe for test in tests] 326 | executables = compat.compat_set(executables) 327 | for exe in executables: 328 | mswin = sys.platform.startswith('win') or sys.platform.startswith('cyg') 329 | # The test is not reliable if there's an unholy combination of windows 330 | # and cygwin being used to run the program. We've already warned the 331 | # user (in config.set_program_name) that we struggled to find the 332 | # executable. 333 | if not os.path.exists(exe) and not mswin: 334 | err = 'Executable does not exist: %s.' % (exe) 335 | raise testcode2.exceptions.TestCodeError(err) 336 | 337 | if tot_nprocs <= 0 and cluster_queue: 338 | # Running on cluster. Default to submitting all tests at once. 339 | tot_nprocs = sum(test.nprocs for test in tests) 340 | 341 | if tot_nprocs > 0: 342 | # Allow at most tot_nprocs cores to be used at once by tests. 343 | max_test_nprocs = max(test.nprocs for test in tests) 344 | if max_test_nprocs > tot_nprocs: 345 | err = ('Number of available cores less than the number required by ' 346 | 'the largest test: at least %d needed, %d available.' 347 | % (max_test_nprocs, tot_nprocs)) 348 | raise testcode2.exceptions.TestCodeError(err) 349 | 350 | # Need to serialize tests that run in the same directory with wildcard 351 | # patterns in the output file--otherwise we can't figure out which 352 | # output file belongs to which test. We might be able to for some 353 | # wildcards, but let's err on the side of caution. 354 | wildcards = re.compile('.*(\*|\?|\[.*\]).*') 355 | serialized_tests = [] 356 | test_store = {} 357 | for test in tests: 358 | if test.output and wildcards.match(test.output): 359 | if test.path in test_store: 360 | test_store[test.path].append(test) 361 | else: 362 | test_store[test.path] = [test] 363 | else: 364 | serialized_tests.append([test]) 365 | for (key, stests) in test_store.items(): 366 | if (len(stests) > 1) and verbose > 2: 367 | print('Warning: cannot run tests in %s concurrently.' % stests[0].path) 368 | serialized_tests += test_store.values() 369 | 370 | semaphore = threading.BoundedSemaphore(tot_nprocs) 371 | slock = threading.Lock() 372 | jobs = [threading.Thread( 373 | target=run_test_worker, 374 | args=(semaphore, slock, test, verbose, cluster_queue, 375 | os.getcwd()) 376 | ) 377 | for test in serialized_tests] 378 | for job in jobs: 379 | # daemonise so thread terminates when master dies 380 | try: 381 | job.setDaemon(True) 382 | except AttributeError: 383 | job.daemon = True 384 | job.start() 385 | 386 | # We avoid .join() which is blocking making it unresponsive to TERM 387 | while threading.activeCount() > 1: 388 | time.sleep(0.5) 389 | else: 390 | # run straight through, one at a time 391 | for test in tests: 392 | test.run_test(verbose, cluster_queue, os.getcwd()) 393 | 394 | 395 | def compare_tests(tests, verbose=1): 396 | '''Compare tests. 397 | 398 | tests: list of tests. 399 | verbose: level of verbosity in output. 400 | 401 | Returns: 402 | 403 | number of tests not checked due to test output file not existing. 404 | ''' 405 | 406 | not_checked = 0 407 | 408 | for test in tests: 409 | for (inp, args) in test.inputs_args: 410 | test_file = testcode2.util.testcode_filename( 411 | testcode2.FILESTEM['test'], 412 | test.test_program.test_id, inp, args 413 | ) 414 | test_file = os.path.join(test.path, test_file) 415 | if os.path.exists(test_file): 416 | test.verify_job(inp, args, verbose, os.getcwd()) 417 | else: 418 | if verbose > 0 and verbose <= 2: 419 | info_line = testcode2.util.info_line(test.path, inp, args, os.getcwd()) 420 | print('%sNot checked.' % info_line) 421 | if verbose > 1: 422 | print('Skipping comparison. ' 423 | 'Test file does not exist: %s.\n' % test_file) 424 | not_checked += 1 425 | 426 | return not_checked 427 | 428 | def recheck_tests(tests, verbose=1, cluster_queue=None, tot_nprocs=0, 429 | first_run=False): 430 | '''Check tests and re-run any failed/skipped tests. 431 | 432 | tests: list of tests. 433 | verbose: level of verbosity in output. 434 | cluster_queue: name of cluster system to use. If None, tests are run locally. 435 | Currently only PBS is implemented. 436 | tot_nprocs: total number of processors available to run tests on. As many 437 | tests (in a LIFO fashion from the tests list) are run at the same time as 438 | possible without using more processors than this value. If less than 1 and 439 | cluster_queue is specified, then all tests are submitted to the cluster at 440 | the same time. If less than one and cluster_queue is not set, then 441 | tot_nprocs is ignored and the tests are run sequentially (default). 442 | first_run: if true, run tests that were not run in the previous invocation. 443 | 444 | Returns: 445 | 446 | not_checked: number of tests not checked due to missing test output. 447 | ''' 448 | 449 | if verbose == 0: 450 | sep = ' ' 451 | else: 452 | sep = '\n\n' 453 | 454 | sys.stdout.write('Comparing tests to benchmarks:'+sep) 455 | 456 | not_checked = compare_tests(tests, verbose) 457 | end_status(tests, not_checked, verbose, False) 458 | 459 | rerun_tests = [] 460 | skip = testcode2.validation.Status(name='skipped') 461 | for test in tests: 462 | stat = test.get_status() 463 | if sum(stat[key] for key in ('failed', 'unknown')) != 0: 464 | # Tests which failed or are unknown should be rerun. 465 | rerun_tests.append(test) 466 | elif stat['ran'] == 0 and first_run: 467 | # Or if they were never run and want to be run... 468 | rerun_tests.append(test) 469 | elif stat['ran'] != 0: 470 | # mark tests as skipped using an internal API (naughty!) 471 | for inp_arg in test.inputs_args: 472 | test._update_status(skip, inp_arg) 473 | 474 | if verbose > 0: 475 | print('') 476 | if rerun_tests: 477 | sys.stdout.write('Rerunning failed tests:'+sep) 478 | run_tests(rerun_tests, verbose, cluster_queue, tot_nprocs) 479 | 480 | return not_checked 481 | 482 | def diff_tests(tests, diff_program, verbose=1): 483 | '''Diff tests. 484 | 485 | tests: list of tests. 486 | diff_program: diff program to use. 487 | verbose: level of verbosity in output. 488 | ''' 489 | 490 | for test in tests: 491 | cwd = os.getcwd() 492 | os.chdir(test.path) 493 | for (inp, args) in test.inputs_args: 494 | have_benchmark = True 495 | try: 496 | benchmark = test.test_program.select_benchmark_file( 497 | test.path, inp, args 498 | ) 499 | except testcode2.exceptions.TestCodeError: 500 | err = sys.exc_info()[1] 501 | have_benchmark = False 502 | test_file = testcode2.util.testcode_filename( 503 | testcode2.FILESTEM['test'], 504 | test.test_program.test_id, inp, args 505 | ) 506 | if not os.path.exists(test_file): 507 | if verbose > 0: 508 | print('Skipping diff with %s in %s: %s does not exist.' 509 | % (benchmark, test.path, test_file)) 510 | elif not have_benchmark: 511 | if verbose > 0: 512 | print('Skipping diff with %s. %s' % (test.path, err)) 513 | else: 514 | if verbose > 0: 515 | print('Diffing %s and %s in %s.' % 516 | (benchmark, test_file, test.path)) 517 | diff_cmd = '%s %s %s' % (diff_program, benchmark, test_file) 518 | diff_popen = subprocess.Popen(diff_cmd, shell=True) 519 | diff_popen.wait() 520 | os.chdir(cwd) 521 | 522 | def tidy_tests(tests, ndays): 523 | '''Tidy up test directories. 524 | 525 | tests: list of tests. 526 | ndays: test files older than ndays are deleted. 527 | ''' 528 | 529 | epoch_time = time.time() - 86400*ndays 530 | 531 | test_globs = ['test.out*','test.err*'] 532 | 533 | print( 534 | 'Delete all %s files older than %s days from each job directory?' 535 | % (' '.join(test_globs), ndays) 536 | ) 537 | ans = '' 538 | while ans != 'y' and ans != 'n': 539 | ans = testcode2.compatibility.compat_input('Confirm [y/n]: ') 540 | 541 | if ans == 'n': 542 | print('No files deleted.') 543 | else: 544 | for test in tests: 545 | cwd = os.getcwd() 546 | os.chdir(test.path) 547 | if test.submit_template: 548 | file_globs = test_globs + [test.submit_template] 549 | else: 550 | file_globs = test_globs 551 | for file_glob in file_globs: 552 | for test_file in glob.glob(file_glob): 553 | if os.stat(test_file)[-2] < epoch_time: 554 | os.remove(test_file) 555 | os.chdir(cwd) 556 | 557 | def make_benchmarks(test_programs, tests, userconfig, copy_files_since, 558 | insert_id=False): 559 | '''Make a new set of benchmarks. 560 | 561 | test_programs: dictionary of test programs. 562 | tests: list of tests. 563 | userconfig: path to the userconfig file. This is updated with the new benchmark id. 564 | copy_files_since: files produced since the timestamp (in seconds since the 565 | epoch) are copied to the testcode_data subdirectory in each test. 566 | insert_id: insert the new benchmark id into the existing list of benchmark ids in 567 | userconfig if True, otherwise overwrite the existing benchmark ids with the 568 | new benchmark id (default). 569 | ''' 570 | 571 | # All tests passed? 572 | statuses = [test.get_status() for test in tests] 573 | npassed = sum(status['passed'] for status in statuses) 574 | nran = sum(status['ran'] for status in statuses) 575 | if npassed != nran: 576 | ans = '' 577 | print('Not all tests passed.') 578 | while ans != 'y' and ans != 'n': 579 | ans = testcode2.compatibility.compat_input( 580 | 'Create new benchmarks? [y/n] ') 581 | if ans != 'y': 582 | return None 583 | 584 | # Get vcs info. 585 | vcs = {} 586 | for (key, program) in test_programs.items(): 587 | if program.vcs and program.vcs.vcs: 588 | vcs[key] = program.vcs.get_code_id() 589 | else: 590 | print('Program not under (known) version control system') 591 | vcs[key] = testcode2.compatibility.compat_input( 592 | 'Enter revision id for %s: ' % (key)) 593 | 594 | # Benchmark label from vcs info. 595 | if len(vcs) == 1: 596 | benchmark = vcs.popitem()[1] 597 | else: 598 | benchmark = [] 599 | for (key, code_id) in vcs.items(): 600 | benchmark.append('%s-%s' % (key, code_id)) 601 | benchmark = '.'.join(benchmark) 602 | 603 | # Create benchmarks. 604 | for test in tests: 605 | test.create_new_benchmarks(benchmark, copy_files_since) 606 | 607 | # update userconfig file. 608 | if userconfig: 609 | config = testcode2.compatibility.configparser.RawConfigParser() 610 | config.optionxform = str # Case sensitive file. 611 | config.read(userconfig) 612 | if insert_id: 613 | ids = config.get('user', 'benchmark').split() 614 | if benchmark in ids: 615 | ids.remove(benchmark) 616 | ids.insert(0, benchmark) 617 | benchmark = ' '.join(ids) 618 | if len(benchmark.split()) > 1: 619 | print('Setting new benchmarks in userconfig to be: %s.' % 620 | (benchmark)) 621 | else: 622 | print('Setting new benchmark in userconfig to be: %s.' % 623 | (benchmark)) 624 | config.set('user', 'benchmark', benchmark) 625 | userconfig = open(userconfig, 'w') 626 | config.write(userconfig) 627 | userconfig.close() 628 | 629 | #--- info output --- 630 | 631 | def start_status(tests, running, verbose=1): 632 | '''Print a header containing useful information. 633 | 634 | tests: list of tests. 635 | running: true if tests are to be run. 636 | verbose: level of verbosity in output (no output if <1). 637 | ''' 638 | 639 | if verbose > 0: 640 | exes = [test.test_program.exe for test in tests] 641 | exes = testcode2.compatibility.compat_set(exes) 642 | if running: 643 | for exe in exes: 644 | print('Using executable: %s.' % (exe)) 645 | # All tests use the same test_id and benchmark. 646 | print('Test id: %s.' % (tests[0].test_program.test_id)) 647 | if len(tests[0].test_program.benchmark) > 1: 648 | benchmark_ids = ', '.join(tests[0].test_program.benchmark) 649 | print('Benchmarks: %s.' % (benchmark_ids)) 650 | else: 651 | print('Benchmark: %s.' % (tests[0].test_program.benchmark[0])) 652 | print('') 653 | 654 | def end_status(tests, not_checked=0, verbose=1, final=True): 655 | '''Print a footer containing useful information. 656 | 657 | tests: list of tests. 658 | not_checked: number of tests not checked (ie not run or compared). 659 | verbose: level of verbosity in output. A summary footer is produced if greater 660 | than 0; otherwise a minimal status line is printed out. 661 | final: final call (so print a goodbye messge). 662 | ''' 663 | 664 | def pluralise(string, num): 665 | '''Return plural form (just by adding s) to string if num > 1.''' 666 | if num > 1: 667 | string = string+'s' 668 | return string 669 | 670 | def select_tests(stat_key, tests, statuses): 671 | '''Select a subset of tests. 672 | 673 | (test.name, test.path) is included if the test object contains at least 674 | one test of the desired status (stat_key).''' 675 | test_subset = [(test.name, test.path) for (test, status) 676 | in zip(tests, statuses) if status[stat_key] != 0] 677 | return sorted(test_subset) 678 | 679 | def format_test_subset(subset): 680 | '''Format each entry in the list returned by select_tests.''' 681 | subset_fmt = [] 682 | for (name, path) in subset: 683 | if os.path.abspath(name) == os.path.abspath(path): 684 | entry = name 685 | else: 686 | entry = '%s (test name: %s)' % (path, name) 687 | if entry not in subset_fmt: 688 | subset_fmt.append(entry) 689 | return subset_fmt 690 | 691 | statuses = [test.get_status() for test in tests] 692 | npassed = sum(status['passed'] for status in statuses) 693 | nwarning = sum(status['warning'] for status in statuses) 694 | nfailed = sum(status['failed'] for status in statuses) 695 | nunknown = sum(status['unknown'] for status in statuses) 696 | nskipped = sum(status['skipped'] for status in statuses) 697 | nran = sum(status['ran'] for status in statuses) 698 | failures = format_test_subset(select_tests('failed', tests, statuses)) 699 | warnings = format_test_subset(select_tests('warning', tests, statuses)) 700 | skipped = format_test_subset(select_tests('skipped', tests, statuses)) 701 | # Treat warnings as passes but add a note about how many warnings. 702 | npassed += nwarning 703 | # Treat skipped tests as tests which weren't run. 704 | nran -= nskipped 705 | 706 | # Pedantic. 707 | warning = pluralise('warning', nwarning) 708 | ran_test = pluralise('test', nran) 709 | failed_test = pluralise('test', nfailed) 710 | skipped_test = pluralise('test', nskipped) 711 | 712 | add_info_msg = [] 713 | if nwarning != 0: 714 | add_info_msg.append('%s %s' % (nwarning, warning)) 715 | if nskipped != 0: 716 | add_info_msg.append('%s skipped' % (nskipped,)) 717 | if nunknown != 0: 718 | add_info_msg.append('%s unknown' % (nunknown,)) 719 | if not_checked != 0: 720 | add_info_msg.append('%s not checked' % (not_checked,)) 721 | add_info_msg = ', '.join(add_info_msg) 722 | if add_info_msg: 723 | add_info_msg = ' (%s)' % (add_info_msg,) 724 | 725 | if nran == 0: 726 | print('No tests to run.') 727 | elif verbose > 0: 728 | if verbose < 2: 729 | print('') # Obsessive formatting. 730 | msg = '%s%s out of %s %s passed%s.' 731 | if final: 732 | msg = 'All done. %s' % (msg,) 733 | if npassed == nran: 734 | print(msg % ('', npassed, nran, ran_test, add_info_msg)) 735 | else: 736 | print(msg % ('ERROR: only ', npassed, nran, ran_test, add_info_msg)) 737 | if failures: 738 | print('Failed %s in:\n\t%s' % (failed_test, '\n\t'.join(failures))) 739 | if warnings: 740 | print('%s in:\n\t%s' % (warning.title(), '\n\t'.join(warnings))) 741 | if skipped and verbose > 1: 742 | print('Skipped %s in:\n\t%s' % (skipped_test, '\n\t'.join(skipped))) 743 | else: 744 | print(' [%s/%s%s]'% (npassed, nran, add_info_msg)) 745 | 746 | # ternary operator not in python 2.4. :-( 747 | ret_val = 0 748 | if nran != npassed: 749 | ret_val = 1 750 | 751 | return ret_val 752 | 753 | #--- main runner --- 754 | 755 | def main(args): 756 | '''main controller procedure. 757 | 758 | args: command-line arguments passed to testcode2. 759 | ''' 760 | 761 | start_time = time.time() 762 | 763 | (options, actions) = parse_cmdline_args(args) 764 | 765 | # Shortcut names to options used multiple times. 766 | verbose = options.verbose 767 | userconfig = options.userconfig 768 | reuse_id = 'run' not in actions and testcode2.compatibility.compat_any( 769 | [action in actions for action in ['compare', 'diff', 'recheck']] 770 | ) 771 | 772 | (user_options, test_programs, tests) = init_tests(userconfig, 773 | options.jobconfig, options.test_id, reuse_id, 774 | options.executable, options.category, options.nprocs, 775 | options.benchmark, options.user_option, 776 | options.job_option) 777 | 778 | ret_val = 0 779 | if not (len(actions) == 1 and 'tidy' in actions): 780 | start_status(tests, 'run' in actions, verbose) 781 | if 'run' in actions: 782 | run_tests(tests, verbose, options.queue_system, options.tot_nprocs) 783 | ret_val = end_status(tests, 0, verbose) 784 | if 'recheck' in actions: 785 | not_checked = recheck_tests(tests, verbose, options.queue_system, 786 | options.tot_nprocs, options.first_run) 787 | ret_val = end_status(tests, not_checked, verbose) 788 | if 'compare' in actions: 789 | not_checked = compare_tests(tests, verbose) 790 | ret_val = end_status(tests, not_checked, verbose) 791 | if 'diff' in actions: 792 | diff_tests(tests, user_options['diff'], verbose) 793 | if 'tidy' in actions: 794 | tidy_tests(tests, options.older_than) 795 | if 'make-benchmarks' in actions: 796 | make_benchmarks(test_programs, tests, userconfig, start_time, 797 | options.insert) 798 | 799 | return ret_val 800 | 801 | if __name__ == '__main__': 802 | 803 | try: 804 | sys.exit(main(sys.argv[1:])) 805 | except testcode2.exceptions.TestCodeError: 806 | err = sys.exc_info()[1] 807 | print(err) 808 | sys.exit(1) 809 | -------------------------------------------------------------------------------- /docs/.static/dummy_file: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsspencer/testcode/2258047a0d09714898572244f274f8dc47efadf3/docs/.static/dummy_file -------------------------------------------------------------------------------- /docs/.templates/dummy_file: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jsspencer/testcode/2258047a0d09714898572244f274f8dc47efadf3/docs/.templates/dummy_file -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = .build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 16 | 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | help: 20 | @echo "Please use \`make ' where is one of" 21 | @echo " html to make standalone HTML files" 22 | @echo " dirhtml to make HTML files named index.html in directories" 23 | @echo " singlehtml to make a single large HTML file" 24 | @echo " pickle to make pickle files" 25 | @echo " json to make JSON files" 26 | @echo " htmlhelp to make HTML files and a HTML help project" 27 | @echo " qthelp to make HTML files and a qthelp project" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 31 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 32 | @echo " text to make text files" 33 | @echo " man to make manual pages" 34 | @echo " texinfo to make Texinfo files" 35 | @echo " info to make Texinfo files and run them through makeinfo" 36 | @echo " gettext to make PO message catalogs" 37 | @echo " changes to make an overview of all changed/added/deprecated items" 38 | @echo " linkcheck to check all external links for integrity" 39 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 40 | 41 | clean: 42 | -rm -rf $(BUILDDIR)/* 43 | 44 | html: 45 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 48 | 49 | dirhtml: 50 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 51 | @echo 52 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 53 | 54 | singlehtml: 55 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 56 | @echo 57 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 58 | 59 | pickle: 60 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 61 | @echo 62 | @echo "Build finished; now you can process the pickle files." 63 | 64 | json: 65 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 66 | @echo 67 | @echo "Build finished; now you can process the JSON files." 68 | 69 | htmlhelp: 70 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 71 | @echo 72 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 73 | ".hhp project file in $(BUILDDIR)/htmlhelp." 74 | 75 | qthelp: 76 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 77 | @echo 78 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 79 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 80 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/testcode.qhcp" 81 | @echo "To view the help file:" 82 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/testcode.qhc" 83 | 84 | devhelp: 85 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 86 | @echo 87 | @echo "Build finished." 88 | @echo "To view the help file:" 89 | @echo "# mkdir -p $$HOME/.local/share/devhelp/testcode" 90 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/testcode" 91 | @echo "# devhelp" 92 | 93 | epub: 94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 95 | @echo 96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 97 | 98 | latex: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo 101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 103 | "(use \`make latexpdf' here to do that automatically)." 104 | 105 | latexpdf: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo "Running LaTeX files through pdflatex..." 108 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 110 | 111 | text: 112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 113 | @echo 114 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 115 | 116 | man: 117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 118 | @echo 119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 120 | 121 | texinfo: 122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 123 | @echo 124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 125 | @echo "Run \`make' in that directory to run these through makeinfo" \ 126 | "(use \`make info' here to do that automatically)." 127 | 128 | info: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo "Running Texinfo files through makeinfo..." 131 | make -C $(BUILDDIR)/texinfo info 132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 133 | 134 | gettext: 135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 136 | @echo 137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 138 | 139 | changes: 140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 141 | @echo 142 | @echo "The overview file is in $(BUILDDIR)/changes." 143 | 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | doctest: 151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 152 | @echo "Testing of doctests in the sources finished, look at the " \ 153 | "results in $(BUILDDIR)/doctest/output.txt." 154 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # testcode documentation build configuration file, created by 5 | # sphinx-quickstart on Fri May 25 22:35:45 2012. 6 | # 7 | # This file is execfile()d with the current directory set to its containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys, os 16 | 17 | # If extensions (or modules to document with autodoc) are in another directory, 18 | # add these directories to sys.path here. If the directory is relative to the 19 | # documentation root, use os.path.abspath to make it absolute, like shown here. 20 | #sys.path.insert(0, os.path.abspath('.')) 21 | 22 | # -- General configuration ----------------------------------------------------- 23 | 24 | # If your documentation needs a minimal Sphinx version, state it here. 25 | #needs_sphinx = '1.0' 26 | 27 | # Add any Sphinx extension module names here, as strings. They can be extensions 28 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 29 | extensions = ['sphinx.ext.autodoc'] 30 | 31 | # Add any paths that contain templates here, relative to this directory. 32 | templates_path = ['.templates'] 33 | 34 | # The suffix of source filenames. 35 | source_suffix = '.rst' 36 | 37 | # The encoding of source files. 38 | #source_encoding = 'utf-8-sig' 39 | 40 | # The master toctree document. 41 | master_doc = 'index' 42 | 43 | # General information about the project. 44 | project = 'testcode' 45 | copyright = '2012, James Spencer' 46 | 47 | # The version info for the project you're documenting, acts as replacement for 48 | # |version| and |release|, also used in various other places throughout the 49 | # built documents. 50 | # 51 | # The short X.Y version. 52 | version = 'dev' 53 | # The full version, including alpha/beta/rc tags. 54 | release = 'dev' 55 | 56 | # The language for content autogenerated by Sphinx. Refer to documentation 57 | # for a list of supported languages. 58 | #language = None 59 | 60 | # There are two options for replacing |today|: either, you set today to some 61 | # non-false value, then it is used: 62 | #today = '' 63 | # Else, today_fmt is used as the format for a strftime call. 64 | #today_fmt = '%B %d, %Y' 65 | 66 | # List of patterns, relative to source directory, that match files and 67 | # directories to ignore when looking for source files. 68 | exclude_patterns = ['.build'] 69 | 70 | # The reST default role (used for this markup: `text`) to use for all documents. 71 | #default_role = None 72 | 73 | # If true, '()' will be appended to :func: etc. cross-reference text. 74 | #add_function_parentheses = True 75 | 76 | # If true, the current module name will be prepended to all description 77 | # unit titles (such as .. function::). 78 | #add_module_names = True 79 | 80 | # If true, sectionauthor and moduleauthor directives will be shown in the 81 | # output. They are ignored by default. 82 | #show_authors = False 83 | 84 | # The name of the Pygments (syntax highlighting) style to use. 85 | pygments_style = 'sphinx' 86 | 87 | # A list of ignored prefixes for module index sorting. 88 | #modindex_common_prefix = [] 89 | 90 | 91 | # -- Options for HTML output --------------------------------------------------- 92 | 93 | # The theme to use for HTML and HTML Help pages. See the documentation for 94 | # a list of builtin themes. 95 | on_rtd = os.environ.get('READTHEDOCS', None) == 'True' 96 | if on_rtd: 97 | html_theme = 'default' 98 | else: 99 | html_theme = 'haiku' 100 | 101 | # Theme options are theme-specific and customize the look and feel of a theme 102 | # further. For a list of options available for each theme, see the 103 | # documentation. 104 | #html_theme_options = {} 105 | 106 | # Add any paths that contain custom themes here, relative to this directory. 107 | #html_theme_path = [] 108 | 109 | # The name for this set of Sphinx documents. If None, it defaults to 110 | # " v documentation". 111 | #html_title = None 112 | 113 | # A shorter title for the navigation bar. Default is the same as html_title. 114 | #html_short_title = None 115 | 116 | # The name of an image file (relative to this directory) to place at the top 117 | # of the sidebar. 118 | #html_logo = None 119 | 120 | # The name of an image file (within the static path) to use as favicon of the 121 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 122 | # pixels large. 123 | #html_favicon = None 124 | 125 | # Add any paths that contain custom static files (such as style sheets) here, 126 | # relative to this directory. They are copied after the builtin static files, 127 | # so a file named "default.css" will overwrite the builtin "default.css". 128 | html_static_path = ['.static'] 129 | 130 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 131 | # using the given strftime format. 132 | #html_last_updated_fmt = '%b %d, %Y' 133 | 134 | # If true, SmartyPants will be used to convert quotes and dashes to 135 | # typographically correct entities. 136 | #html_use_smartypants = True 137 | 138 | # Custom sidebar templates, maps document names to template names. 139 | #html_sidebars = {} 140 | 141 | # Additional templates that should be rendered to pages, maps page names to 142 | # template names. 143 | #html_additional_pages = {} 144 | 145 | # If false, no module index is generated. 146 | #html_domain_indices = True 147 | 148 | # If false, no index is generated. 149 | #html_use_index = True 150 | 151 | # If true, the index is split into individual pages for each letter. 152 | #html_split_index = False 153 | 154 | # If true, links to the reST sources are added to the pages. 155 | #html_show_sourcelink = True 156 | 157 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 158 | #html_show_sphinx = True 159 | 160 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 161 | #html_show_copyright = True 162 | 163 | # If true, an OpenSearch description file will be output, and all pages will 164 | # contain a tag referring to it. The value of this option must be the 165 | # base URL from which the finished HTML is served. 166 | #html_use_opensearch = '' 167 | 168 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 169 | #html_file_suffix = None 170 | 171 | # Output file base name for HTML help builder. 172 | htmlhelp_basename = 'testcodedoc' 173 | 174 | 175 | # -- Options for LaTeX output -------------------------------------------------- 176 | 177 | latex_elements = { 178 | # The paper size ('letterpaper' or 'a4paper'). 179 | #'papersize': 'letterpaper', 180 | 181 | # The font size ('10pt', '11pt' or '12pt'). 182 | #'pointsize': '10pt', 183 | 184 | # Additional stuff for the LaTeX preamble. 185 | #'preamble': '', 186 | } 187 | 188 | # Grouping the document tree into LaTeX files. List of tuples 189 | # (source start file, target name, title, author, documentclass [howto/manual]). 190 | latex_documents = [ 191 | ('index', 'testcode.tex', 'testcode Documentation', 192 | 'James Spencer', 'manual'), 193 | ] 194 | 195 | # The name of an image file (relative to this directory) to place at the top of 196 | # the title page. 197 | #latex_logo = None 198 | 199 | # For "manual" documents, if this is true, then toplevel headings are parts, 200 | # not chapters. 201 | #latex_use_parts = False 202 | 203 | # If true, show page references after internal links. 204 | #latex_show_pagerefs = False 205 | 206 | # If true, show URL addresses after external links. 207 | #latex_show_urls = False 208 | 209 | # Documents to append as an appendix to all manuals. 210 | #latex_appendices = [] 211 | 212 | # If false, no module index is generated. 213 | #latex_domain_indices = True 214 | 215 | 216 | # -- Options for manual page output -------------------------------------------- 217 | 218 | # One entry per manual page. List of tuples 219 | # (source start file, name, description, authors, manual section). 220 | man_pages = [ 221 | ('index', 'testcode', 'testcode Documentation', 222 | ['James Spencer'], 1) 223 | ] 224 | 225 | # If true, show URL addresses after external links. 226 | #man_show_urls = False 227 | 228 | 229 | # -- Options for Texinfo output ------------------------------------------------ 230 | 231 | # Grouping the document tree into Texinfo files. List of tuples 232 | # (source start file, target name, title, author, 233 | # dir menu entry, description, category) 234 | texinfo_documents = [ 235 | ('index', 'testcode', 'testcode Documentation', 236 | 'James Spencer', 'testcode', 'One line description of project.', 237 | 'Miscellaneous'), 238 | ] 239 | 240 | # Documents to append as an appendix to all manuals. 241 | #texinfo_appendices = [] 242 | 243 | # If false, no module index is generated. 244 | #texinfo_domain_indices = True 245 | 246 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 247 | #texinfo_show_urls = 'footnote' 248 | -------------------------------------------------------------------------------- /docs/configuration_files.rst: -------------------------------------------------------------------------------- 1 | .. _config: 2 | 3 | Configuration files 4 | =================== 5 | 6 | For convenience, tests can be specified via configuration files rather than 7 | using the testcode API directly. These configuration files are required for 8 | work with the command-line interface. 9 | 10 | The two configuration files are, by default, :ref:`jobconfig` and 11 | :ref:`userconfig` in the working directory. Different names and/or paths can 12 | be specified if required. 13 | 14 | Both configuration files take options in the ini format (as understood by 15 | Python's `configparser `_ module). For example:: 16 | 17 | [section_1] 18 | a = 2 19 | b = test_option 20 | 21 | [section_2] 22 | v = 4.5 23 | hello = world 24 | 25 | defines an ini file with two sections (named 'section_1' and 'section_2'), each 26 | with two variables set. 27 | 28 | .. note:: 29 | 30 | Any paths can either be absolute or relative to the directory containing 31 | the configuration file. The full path need not be given for any program 32 | which exists on the user's PATH. Environment variables in **program** names will be expanded. 33 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | testcode 2 | ======== 3 | 4 | testcode is a python module for testing for regression errors in numerical 5 | (principally scientific) software. Essentially testcode runs a set of 6 | calculations, and compares the output data to that generated by a previous 7 | calculation (which is regarded to be "correct"). It is designed to be 8 | lightweight and highly portable: it can be used both as part of the development 9 | process and to verify the correctness of a binary on a new architecture. 10 | testcode requires python 2.4-3.4. If these are not available, then `pypy 11 | `_ is recommended---for this purpose pypy serves as 12 | a portable, self-contained python implementation but this is a tiny aspect of 13 | the pypy project. 14 | 15 | testcode can run a set of tests and check the calculated data is within a the 16 | desired tolerance of results contained in previous output (using an internal 17 | data extraction engine, a user-supplied data extraction program or 18 | a user-supplied verification program). The programs to be tested can be run in 19 | serial and in parallel and tests can be run in either locally or submitted to 20 | a compute cluster running a queueing system such as PBS. Previous tests can be 21 | compared and diffed against other tests or benchmarks. 22 | 23 | testcode provides access to these features via an API. The supplied 24 | command-line interface, :ref:`testcode.py`, should be sufficient for most 25 | purposes. The command-line interface utilises simple :ref:`configuration files 26 | `, wich makes it easy to customise to the local environment and to add 27 | new tests. 28 | 29 | .. toctree:: 30 | :maxdepth: 1 31 | 32 | installation 33 | configuration_files 34 | jobconfig 35 | userconfig 36 | verification 37 | testcode.py 38 | 39 | Indices and tables 40 | ================== 41 | 42 | * :ref:`genindex` 43 | * :ref:`modindex` 44 | * :ref:`search` 45 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | testcode2 is designed to be very lightweight and portable, so it can easily and 5 | quickly be used on a variety of machines. Typically only downloading the 6 | testcode2 package is required. 7 | 8 | If the :ref:`testcode.py` script is used, then no additional installation steps 9 | are required assuming the directory structure is preserved. If the 10 | ``testcode2`` module is used or the files are split up and installed elsewhere, 11 | then the ``testcode2`` module must be able to be found by python (i.e. exists 12 | on $PYTHONPATH). 13 | -------------------------------------------------------------------------------- /docs/jobconfig.rst: -------------------------------------------------------------------------------- 1 | .. _jobconfig: 2 | 3 | jobconfig 4 | ========= 5 | 6 | The jobconfig file defines the tests to run. If a section named 'categories' 7 | exists, then it gives labels to sets of tests. All other sections are assumed 8 | to individually define a test. 9 | 10 | Tests 11 | ----- 12 | 13 | A test is assumed to reside in the directory given by the name of the test 14 | section. For example:: 15 | 16 | [carbon_dioxide_ccsd] 17 | inputs_args = ('co2.inp','') 18 | 19 | would define a test in the ``carbon_dioxide_ccsd`` subdirectory relative to the 20 | ``jobconfig`` configuration file, with the input file as ``co2.inp`` (in the 21 | ``carbon_dioxide_ccsd`` subdirectory) with no additional arguments to be passed 22 | to the test program. All input and output files related to the test are 23 | assumed to be contained within the test subdirectory. 24 | 25 | The following options are permitted: 26 | 27 | inputs_args [inputs and arguments format (see :ref:`below `)] 28 | Input filename and associated arguments to be passed to the test program. 29 | No default. 30 | min_nprocs [integer] 31 | Minimum number of processors to run test on. Cannot be overridden by the 32 | '--processors' command-line option. Default: 0. 33 | max_nprocs [integer] 34 | Maximum number of processors to run test on. Cannot be overridden by the 35 | '--processors' command-line option. Default: 2^31-1 or 2^63-1. 36 | nprocs [integer] 37 | Number of processors to run the test on. Zero indicates to run the test 38 | purely in serial, without using an external program such as mpirun to 39 | launch the test program. Default: 0. 40 | output [string] 41 | Filename to which the output is written if the output is not written to 42 | standard output. The output file is moved to the specific testcode test 43 | filename at the end of the calculation before the test output is validated 44 | against the benchmark output. Wildcards are allowed so long as the pattern 45 | only matches a single file at the end of the calculation. Default: 46 | inherits from setting in :ref:`userconfig`. 47 | path [string] 48 | Set path (relative to the directory containing the ``jobconfig`` 49 | configuration file) of the test. The test is run in this directory and so 50 | input filenames need to be relative to it. If the given path contains 51 | wildcards, then this is expanded and an individual test is created for each 52 | path that maches the pattern. Note that Python's configparser restricts 53 | the use of special characters in section names and hence some patterns can 54 | only be accomplished by explicitly using the path option. Default: test 55 | name (i.e. the name of the section defining the test). 56 | run_concurrent [boolean] 57 | If true then subtests defined by the inputs_args option are allowed to run 58 | concurrently rather than consecutively, assuming enough processors are 59 | available. Default: false. 60 | submit_template [string] 61 | Path to a template of a submit script used to submit jobs to a queueing 62 | system. testcode will replace the string given in submit_pattern with the 63 | command(s) to run the test. The submit script must do all other actions (e.g. 64 | setting environment variables, loading modules, copying files from the test 65 | directory to a local disk and copying files back afterwards). No default. 66 | program [string] 67 | Program name (appropriate section heading in :ref:`userconfig`) to use to 68 | run the test. Default: specified in the [user] section of 69 | :ref:`userconfig`. 70 | tolerance [tolerance format (see :ref:`tolerance`)] 71 | Tolerances for comparing test output to the benchmark output. Default: 72 | inherits from the settings in :ref:`userconfig`. 73 | 74 | If a test is defined via a category/path containing wildcards and explicitly, 75 | then the explicit category will inherit any settings from the wildcard 76 | definition. For example, given the subdirectories ``t1`` and ``t2``, each 77 | containing tests, the definition:: 78 | 79 | [t*] 80 | inputs_args = ('test.in', '') 81 | [t1] 82 | nprocs = 2 83 | 84 | is entirely equivalent to:: 85 | 86 | [t1] 87 | nprocs = 2 88 | inputs_args = ('test.in', '') 89 | [t2] 90 | inputs_args = ('test.in', '') 91 | 92 | .. note:: 93 | 94 | Explicitly defining a test multiple times, e.g.:: 95 | 96 | [t1] 97 | inputs_args = ('inp1', '') 98 | [t1] 99 | inputs_args = ('inp2', '') 100 | 101 | is not permitted and the resultant settings are not uniquely defined. 102 | 103 | Test categories 104 | --------------- 105 | 106 | For the purposes of selecting a subset of the tests in :ref:`testcode.py`, each 107 | test is automatically placed in two separate categories, one labelled by the 108 | test's name and the other by the test's path. A test can hence be referred to 109 | by either its path or by its name (which are identical by default). 110 | 111 | Additional categories can be specified in the [categories] section. This makes 112 | it very easy to select subsets of the tests to run. For example:: 113 | 114 | [categories] 115 | cat1 = t1 t2 116 | cat2 = t3 t4 117 | cat3 = cat1 t3 118 | 119 | defines three categories (`cat`, `cat2` and `cat3`), each containing a subset 120 | of the overall tests. A category may contain another category so long as 121 | circular dependencies are avoided. There are two special categories, `_all_` 122 | and `_default_`. The `_all_` category contains, by default, all tests and 123 | should not be changed under any circumstances. The `_default_` category can 124 | be set; if it is not specified then it is set to be the `_all_` category. 125 | 126 | .. _inputs: 127 | 128 | Program inputs and arguments 129 | ---------------------------- 130 | 131 | The inputs and arguments must be given in a specific format. As with the 132 | :ref:`tolerance format `, the inputs and arguments are specified 133 | using a comma-separated list of python tuples. Each tuple (basically 134 | a comma-separated list enclosed in parantheses) contains two elements: the name 135 | of an input file and the associated arguments, in that order, represents 136 | a subtest belonging to the given test. Both elements must be quoted. If the 137 | input filename contains wildcard, then those wildcards are expanded to find all 138 | files in the test subdirectory which match that pattern; the expanded list is 139 | sorted in alphanumerical order. A separate subtest (with the same arguments 140 | string) is then created for each file matching the pattern. used to construct 141 | the command to run. A null string (``''``) should be used to represent the 142 | absence of an input file or arguments. By default subtests run in the order 143 | they are specified. For example:: 144 | 145 | inputs_args = ('test.inp', '') 146 | 147 | defines a single subtest, with input filename ``test.inp`` and no arguments, 148 | 149 | :: 150 | 151 | inputs_args = ('test.inp', ''), ('test2.inp', '--verbose') 152 | 153 | defines two subtests, with an additional argument for the second subtest, and 154 | 155 | :: 156 | 157 | inputs_args = ('test*.inp', '') 158 | 159 | defines a subtest for each file matching the pattern ``test*inp`` in the 160 | subdirectory of the test. 161 | -------------------------------------------------------------------------------- /docs/testcode.py.rst: -------------------------------------------------------------------------------- 1 | .. _testcode.py: 2 | 3 | testcode.py 4 | =========== 5 | 6 | .. only:: html 7 | 8 | testcode.py - a command-line interface to testcode. 9 | 10 | Synopsis 11 | -------- 12 | 13 | testcode.py [options] [action1 [action2...]] 14 | 15 | Description 16 | ----------- 17 | 18 | Run a set of actions on a set of tests. 19 | 20 | Requires two configuration files, :ref:`jobconfig` and :ref:`userconfig`. See 21 | testcode documentation for further details. 22 | 23 | testcode.py provides a command-line interface to testcode, a simple framework 24 | for comparing output from (principally numeric) programs to previous output to 25 | reveal regression errors or miscompilation. 26 | 27 | Actions 28 | ------- 29 | 30 | ''run'' is th default action. 31 | 32 | compare 33 | compare set of test outputs from a previous testcode run against the 34 | benchmark outputs. 35 | diff 36 | diff set of test outputs from a previous testcode run against the benchmark 37 | outputs. 38 | make-benchmarks 39 | create a new set of benchmarks and update the :ref:`userconfig` file with 40 | the new benchmark id. Also runs the 'run' action unless the 'compare' 41 | action or 'recheck' action is also given. 42 | recheck 43 | compare set of test outputs from a previous testcode run against 44 | benchmark outputs and rerun any failed tests. 45 | run 46 | run a set of tests and compare against the benchmark outputs. 47 | tidy 48 | Remove files from previous testcode runs from the test directories. 49 | 50 | Options 51 | ------- 52 | 53 | -h, --help 54 | show this help message and exit 55 | -b BENCHMARK, --benchmark=BENCHMARK 56 | Set the file ID of the benchmark files. If BENCHMARK is in the format 57 | t:ID, then the test files with the corresponding ID are used. This 58 | allows two sets of tests to be compared. Default: specified in the [user] 59 | section of the :ref:`userconfig` file. 60 | -c CATEGORY, --category=CATEGORY 61 | Select the category/group of tests. Can be specified multiple times. 62 | Wildcards or parent directories can be used to select multiple directories 63 | by their path. Default: use the `_default_` category if run is an action 64 | unless make-benchmarks is an action. All other cases use the `_all_` 65 | category by default. The `_default_` category contains all tests unless 66 | otherwise set in the :ref:`jobconfig` file. 67 | -e EXECUTABLE, --executable=EXECUTABLE 68 | Set the executable(s) to be used to run the tests. Can be a path or name 69 | of an option in the :ref:`userconfig` file, in which case all test programs are 70 | set to use that value, or in the format program_name=value, which affects 71 | only the specified program. Only relevant to the run action. Default: exe 72 | variable set for each program listed in the :ref:`userconfig` file. 73 | -f, --first-run 74 | Run tests that were not were not run in the previous testcode run. Only 75 | relevant to the recheck action. Default: False. 76 | -i, --insert 77 | Insert the new benchmark into the existing list of benchmarks in userconfig 78 | rather than overwriting it. Only relevant to the make-benchmarks action. 79 | Default: False. 80 | --jobconfig=JOBCONFIG 81 | Set path to the job configuration file. Default: jobconfig. 82 | --job-option=JOB_OPTION 83 | Override/add setting to :ref:`jobconfig`. Takes three arguments. Format: 84 | section_name option_name value. Default: none. 85 | --older-than=OLDER_THAN 86 | Set the age (in days) of files to remove. Only relevant to the tidy 87 | action. Default: 14 days. 88 | -p NPROCS, --processors=NPROCS 89 | Set the number of processors to run each test on. Only relevant to the run 90 | action. Default: run tests as serial jobs. 91 | -q, --quiet 92 | Print only minimal output. Default: False. 93 | -s QUEUE_SYSTEM, --submit=QUEUE_SYSTEM 94 | Submit tests to a queueing system of the specified type. Only PBS system 95 | is currently implemented. Only relevant to the run action. Default: none. 96 | -t TEST_ID, --test-id=TEST_ID 97 | Set the file ID of the test outputs. If TEST_ID is in the format b:ID, then 98 | the benchmark files with the corresponding ID are used. This allows two 99 | sets of benchmarks to be compared. Default: unique filename based upon 100 | date if running tests and most recent test_id if comparing tests. 101 | --total-processors=TOT_NPROCS 102 | Set the total number of processors to use to run as many tests as possible 103 | at the same time. Relevant only to the run option. Default: run all tests 104 | concurrently run if --submit is used; run tests sequentially otherwise. 105 | --userconfig=USERCONFIG 106 | Set path to the user configuration file. Default: userconfig. 107 | --user-option=USER_OPTION 108 | Override/add setting to :ref:`userconfig`. Takes three arguments. Format: 109 | section_name option_name value. Default: none. 110 | -v, --verbose 111 | Increase verbosity of output. Can be specified up to two times. 112 | The default behaviour is to print out the test and its status. (See the 113 | --quiet option to suppress even this.) Specify -v or --verbose once to 114 | show (if relevant) which data values caused warnings or failures. 115 | Specify -v or --verbose twice to see all (external) commands run and all 116 | data extracted from running the tests. Using the maximum verbosity level 117 | is highly recommended for debugging. 118 | 119 | Exit status 120 | ----------- 121 | 122 | 1 if one or more tests fail (run and compare actions only) and 0 otherwise. 123 | 124 | License 125 | ------- 126 | 127 | Modified BSD License. See LICENSE in the source code for more details. 128 | 129 | Bugs 130 | ---- 131 | 132 | Contact James Spencer (j.spencer@imperial.ac.uk) regarding bug reports, 133 | suggestions for improvements or code contributions. 134 | -------------------------------------------------------------------------------- /docs/userconfig.rst: -------------------------------------------------------------------------------- 1 | .. _userconfig: 2 | 3 | userconfig 4 | ========== 5 | 6 | The userconfig file must contain at least two sections. One section must be 7 | entitled 'user' and contains various user settings. Any other section is 8 | assumed to define a program to be tested, where the program is referred to 9 | internally by its section name. This makes it possible for a set of tests to 10 | cover multiple, heavily intertwined, programs. It is, however, far better to 11 | have a distinct set of tests for each program where possible. 12 | 13 | [user] section 14 | -------------- 15 | 16 | The following options are allowed in the [user] section: 17 | 18 | benchmark [string] 19 | Specify the ID of the benchmark to compare to. This should be set running 20 | 21 | .. code-block bash 22 | 23 | $ testcode.py make-benchmarks 24 | 25 | The format of the benchmark files is'benchmark.out.ID.inp=INPUT_FILE.arg=ARGS'. 26 | The 'inp' and/or 'arg' section is not included if it is empty. 27 | 28 | Multiple benchmarks can be used by providing a space-separated list of IDs. The first 29 | ID in the list which corresponds to an existing benchmark filename is used to 30 | validate the test. 31 | date_fmt [string] 32 | Format of the date string used to uniquely label test outputs. This must 33 | be a valid date format string (see `Python documenation 34 | `_). Default: %d%m%Y. 35 | default_program [string] 36 | Default program used to run each test. Only needs to be set if 37 | multiple program sections are specified. No default. 38 | diff [string] 39 | Program used to diff test and benchmark outputs. Default: diff. 40 | tolerance [tolerance format (see :ref:`below `.)] 41 | Default tolerance(s) used to compare all tests to their respective 42 | benchmarks. Default: absolute tolerance 10^-10; no relative tolerance set. 43 | 44 | [program_name] section(s) 45 | ------------------------- 46 | 47 | The following options are allowed to specify a program (called 'program_name') 48 | to be tested: 49 | 50 | data_tag [string] 51 | Data tag to be used to extract data from test and benchmark output. See 52 | :ref:`verification` for more details. No default. 53 | ignore_fields [space-separated list of strings] 54 | Specify the fields (e.g. column headings in the output from the extraction 55 | program) to ignore. This can be used to include, say, timing information 56 | in the test output for performance comparison without causing failure of 57 | tests. Spaces within a string can be escaped by quoting the string. No 58 | default. 59 | exe [string] 60 | Path to the program executable. No default. 61 | extract_fn [string] 62 | A python function (in the form module_name.function_name) which extracts 63 | data from test and benchmark outputs for comparison. See :ref:`verification` 64 | for details. If a space-separated pair of strings are given, the first is 65 | appended to sys.path before the module is imported. Otherwise the desired 66 | module **must** exist on PYTHONPATH. The feature requires python 2.7 or 67 | python 3.1+. 68 | extract_args [string] 69 | Arguments to supply to the extraction program. Default: null string. 70 | extract_cmd_template [string] 71 | Template of command used to extract data from output(s) with the following 72 | substitutions made: 73 | 74 | tc.extract 75 | replaced with the extraction program. 76 | tc.args 77 | replaced with extract_args. 78 | tc.file 79 | replaced with (as required) the filename of the test output or the 80 | filename of the benchmark output. 81 | tc.bench 82 | replaced with the filename of the benchmark output. 83 | tc.test 84 | replaced with the filename of the test output. 85 | 86 | Default: tc.extract tc.args tc.file if verify is False and 87 | tc.extract tc.args tc.test tc.bench if verify is True. 88 | extract_program [string] 89 | Path to program to use to extract data from test and benchmark output. 90 | See :ref:`verification` for more details. No default. 91 | extract_fmt [string] 92 | Format of the data returned by extraction program. See :ref:`verification` 93 | for more details. Can only take values table or yaml. Default: table. 94 | launch_parallel [string] 95 | Command template inserted before run_cmd_template when running the test program in 96 | parallel. tc.nprocs is replaced with the number of processors a test uses (see 97 | run_cmd_template). If tc.nprocs does not appear, then testcode has no control over 98 | the number of processors a test is run on. Default: mpirun -np tc.nprocs. 99 | run_cmd_template [string] 100 | Template of command used to run the program on the test with the following 101 | substitutions made: 102 | 103 | tc.program 104 | replaced with the program to be tested. 105 | tc.args 106 | replaced with the arguments of the test. 107 | tc.input 108 | replaced with the input filename of the test. 109 | tc.output 110 | replaced with the filename for the standard output. The filename 111 | is selected at runtime. 112 | tc.error 113 | replaced with the filename for the error output. The filename is 114 | selected at runtime. 115 | tc.nprocs 116 | replaced with the number of processors the test is run on. 117 | 118 | Default: 'tc.program tc.args tc.input > tc.output 2> tc.error'. The complete command 119 | used to invoke the program is run_cmd_template in serial runs and launch_parallel 120 | run_cmd_template in parallel runs, where launch_parallel is specified above. The 121 | parallel version is only used if the number of processors to run a test on is greater 122 | than zero. 123 | skip_args [string] 124 | Arguments to supply to the program to test whether to skip the comparison 125 | of the test and benchmark. Default: null string. 126 | skip_cmd_template [string] 127 | Template of command used to test whether test was successfully run or 128 | whether the comparison of the benchmark and test output should be skipped. 129 | See :ref:`below ` for more details. The following strings in the 130 | template are replaced: 131 | 132 | tc.skip 133 | replaced with skip_program. 134 | tc.args 135 | replaced with skip_args. 136 | tc.test 137 | replaced with the filename of the test output. 138 | tc.error 139 | replaced with the filename for the error output. 140 | 141 | Default: tc.skip tc.args tc.test. 142 | skip_program [string] 143 | Path to the program to test whether to skip the comparison of the test and 144 | benchmark. If null, then this test is not performed. Default: null string. 145 | submit_pattern [string] 146 | String in the submit template to be replaced by the run command. Default: 147 | testcode.run_cmd. 148 | tolerance [tolerance format (see :ref:`below `.)] 149 | Default tolerance for tests of this type. Default: inherits from 150 | [user]. 151 | verify [boolean] 152 | True if the extraction program compares the benchmark and test 153 | outputs directly. See :ref:`verification` for more details. Default: 154 | False. 155 | vcs [string] 156 | Version control system used for the source code. This is used to 157 | label the benchmarks. The program binary is assumed to be in the same 158 | directory tree as the source code. Supported values are: hg, git and svn 159 | and None. If vcs is set to None, then the version id of the program is 160 | requested interactively when benchmarks are produced. Default: None. 161 | 162 | Most settings are optional and need only be set if certain functionality is 163 | required or the default is not appropriate. Note that at least one of data_tag, 164 | extract_fn or extract_program must be supplied and are used in that order of 165 | precedence. 166 | 167 | In addition, the following variables are used, if present, as default settings 168 | for all tests of this type: 169 | 170 | * inputs_args (no default) 171 | * nprocs (default: 0) 172 | * min_nprocs (default: 0) 173 | * max_nprocs (default: 2^31-1 or 2^63-1) 174 | * output (no default) 175 | * run_concurrent (defailt: false) 176 | * submit_template 177 | 178 | See :ref:`jobconfig` for more details. 179 | 180 | All other settings are assumed to be paths to other versions of the program 181 | (e.g. a stable version). Using one of these versions instead of the one listed 182 | under the 'exe' variable can be selected by an option to :ref:`testcode.py`. 183 | 184 | .. _tolerance: 185 | 186 | Tolerance format 187 | ---------------- 188 | 189 | The format for the tolerance for the data is very specific. Individual 190 | tolerance elements are specified in a comma-separated list. Each individual 191 | tolerance element is a python tuple (essentially a comma-separated list 192 | enclosed in parentheses) consisting of, in order, the absolute tolerance, the 193 | relative tolerance, the label of the field to which the tolerances apply and 194 | a boolean value specifying the strictness of the tolerance (see below). The 195 | labels must be quoted. If no label is supplied (or is set to None) then the 196 | setting is taken to be the default tolerance to be applied to all data. If the 197 | strictness value is not given, the tolerance is assumed to be strict. For 198 | example, the setting:: 199 | 200 | (1e-8, 1.e-6), (1.e-4, 1.e-4, 'Force') 201 | 202 | uses an absolute tolerance of 10^-8 and a relative tolerance of 10^-6 by 203 | default and an absolte tolerance and a relative tolerance of 10^-4 for data 204 | items labelled with 'Force' (i.e. in columns headed by 'Force' using an 205 | external data extraction program or labelled 'Force' by the internal data 206 | extraction program using data tags). If a tolerance is set to None, then it is 207 | ignored. At least one of the tolerances must be set. 208 | 209 | A strict tolerance requires both the test value to be within the absolute and 210 | relative tolerance of the benchmark value in order to be considered to pass. 211 | This is the default behaviour. A non-strict tolerance only requires the test 212 | value to be within the absolute or relative tolerance of the benchmark value. 213 | For example:: 214 | 215 | (1e-8, 1e-6, None, False), (1e-10, 1e-10, 'Energy') 216 | 217 | sets the default absolute and relative tolerances to be 10^-8 and 10^-6 218 | respectively and sets the default tolerance to be non-strict except for the 219 | 'Energy' values, which have a strict absolute and relative tolerances of 220 | 10^-10. If only one of the tolerances is set, then the strict and non-strict 221 | settings are equivalent. 222 | 223 | Alternatively, the tolerance can be labelled by a regular expression, in which case any 224 | data labels which match the regular expression will use that tolerance unless there is 225 | a tolerance with that specific label (i.e. exact matches override a regular 226 | expression match). Note that this is the case even if the tolerance using the exact 227 | tolerance is defined in :ref:`userconfig` and the regular expression match is 228 | defined in :ref:`jobconfig`. 229 | 230 | .. _skip: 231 | 232 | Skipping tests 233 | -------------- 234 | 235 | Sometimes a test should not be compared to the benchmark---for example, if the 236 | version of the program does not support a given feature or can only be run in 237 | parallel. testcode supports this by running a command to detect whether a test 238 | should be skipped. 239 | 240 | If the skipped program is set, then the skipped command is ran before 241 | extracting data from output files. For example, if 242 | 243 | skip_program = grep 244 | skip_args = "is not implemented." 245 | 246 | are set, then testcode will run: 247 | 248 | .. code-block:: bash 249 | 250 | grep "is not implemented." test_file 251 | 252 | where test_file is the test output file. If grep returns 0 (i.e. 253 | test_file contains the string "is not implemented") then the test is 254 | marked as skipped and the test file is not compared to the benchmark. 255 | -------------------------------------------------------------------------------- /docs/verification.rst: -------------------------------------------------------------------------------- 1 | .. _verification: 2 | 3 | Test verification 4 | ================= 5 | 6 | testcode compares selected data from an output with previously obtained output 7 | (the 'benchmark'); a test passes if all data is within a desired tolerance. 8 | The data can be compared using an absolute tolerance and/or a relative 9 | tolerance. testcode needs some way of knowing what data from the output files 10 | should be validated. There are four options. 11 | 12 | * label output with a 'data tag' 13 | 14 | If a data tag is supplied, then testcode will search each output file for 15 | lines starting with that tag. The first numerical entry on those lines will 16 | then be checked against the benchmark. For example, if the data tag is set 17 | to be '[QA]', and the line 18 | 19 | [QA] Energy = 1.23456 eV 20 | 21 | appears in the test output, then testcode will ensure the value 1.23456 is 22 | identical (within the specified tolerance) to the equivalent line in the 23 | benchmark output. The text preceding the value is used to label that data 24 | item; lines with identical text but different values are handled but it is 25 | assumed that such lines always come in the same (relative) order. 26 | 27 | * user-supplied data extraction python function 28 | 29 | An arbitrary python module can be imported and a function contained in the 30 | module called with a test or benchmark output filename as its sole argument. 31 | The function must return the extracted data from the output file as a python 32 | dict with keys labelling each data item (corresponding to the keys used for 33 | setting tolerances) and lists or tuples as values containing the data to be 34 | compared. For example:: 35 | 36 | { 37 | 'val 1': [1.2, 8.7], 38 | 'val 2': [2, 4], 39 | 'val 3': [3.32, 17.2], 40 | } 41 | 42 | Each entry need not contain the same amount of data:: 43 | 44 | { 45 | 'val 1': [1.2, 8.7], 46 | 'val 2': [2, 4], 47 | 'val 3': [3.32, 17.2], 48 | 'val 4': [11.22], 49 | 'val 5': [221.0], 50 | } 51 | 52 | * user-supplied data extraction program 53 | 54 | An external program can be used to extract data from the test and benchmark 55 | output. The program must print the data to be compared in an output file in 56 | either a tabular format (default) or in a YAML format to standard output. 57 | Using YAML format requires the `PyYAML `_ module to be 58 | installed. 59 | 60 | tabular format 61 | A row of text is assumed to start a table. Multiple tables are permitted, 62 | but each table must be square (i.e. no gaps and the same number of elements 63 | on each row) and hence each column heading must contain no spaces. For 64 | example, a single table is of the format:: 65 | 66 | val_1 val_2 val3 67 | 1.2 2 3.32 68 | 8.7 4 17.2 69 | 70 | and a table containing multiple subtables:: 71 | 72 | val_1 val_2 val3 73 | 1.2 2 3.32 74 | 8.7 4 17.2 75 | val_4 val_5 76 | 11.22 221.0 77 | 78 | Tables need not be beautifully presented: the amount of whitespace 79 | between each table cell is not important, so long as there's at least one 80 | space separating adjacent cells. 81 | 82 | Column headings are used to label the data in the subsequent rows. These 83 | labels can be used to specify different tolerances for different types of 84 | data. 85 | 86 | YAML format 87 | The format accepted is a very restricted subset of YAML. Specifically, 88 | only one YAML document is accepted and that document must contain 89 | a single block mapping. Each key in the block mapping can contain 90 | a single data element to be compared or block sequence containing 91 | a series of data elements to be compared. However, block sequences may 92 | not be nested. The equivalent YAML formats for the two examples given 93 | above are:: 94 | 95 | val_1: 96 | - 1.2 97 | - 8.7 98 | val_2: 99 | - 2 100 | - 4 101 | val_3: 102 | - 3.32 103 | - 17.2 104 | 105 | and:: 106 | 107 | val_1: 108 | - 1.2 109 | - 8.7 110 | val_2: 111 | - 2 112 | - 4 113 | val_3: 114 | - 3.32 115 | - 17.2 116 | val_4: 11.22 117 | val_5: 221.0 118 | 119 | See the `PyYAML documentation 120 | `_ for more details. 121 | 122 | Non-numerical values apart from the column headings in tabular ouput are 123 | required to be equal (within python's definition of equality for a given 124 | object). 125 | 126 | * user-supplied verification program 127 | 128 | An external program can be used to validate the test output; the program must 129 | set an exit status of 0 to indicate the test passed and a non-zero value to 130 | indicate failure. 131 | -------------------------------------------------------------------------------- /lib/testcode2/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | testcode2 3 | --------- 4 | 5 | A framework for regression testing numerical programs. 6 | 7 | :copyright: (c) 2012 James Spencer. 8 | :license: modified BSD; see LICENSE for more details. 9 | ''' 10 | 11 | import glob 12 | import os 13 | import pipes 14 | import shutil 15 | import subprocess 16 | import sys 17 | import warnings 18 | 19 | try: 20 | import yaml 21 | _HAVE_YAML = True 22 | except ImportError: 23 | _HAVE_YAML = False 24 | 25 | try: 26 | import importlib 27 | _HAVE_IMPORTLIB_ = True 28 | except ImportError: 29 | _HAVE_IMPORTLIB_ = False 30 | 31 | import testcode2.dir_lock as dir_lock 32 | import testcode2.exceptions as exceptions 33 | import testcode2.queues as queues 34 | import testcode2.compatibility as compat 35 | import testcode2.util as util 36 | import testcode2.validation as validation 37 | 38 | DIR_LOCK = dir_lock.DirLock() 39 | 40 | # Do not change! Bad things will happen... 41 | _FILESTEM_TUPLE = ( 42 | ('test', 'test.out'), 43 | ('error', 'test.err'), 44 | ('benchmark', 'benchmark.out'), 45 | ) 46 | _FILESTEM_DICT = dict( _FILESTEM_TUPLE ) 47 | # We can change FILESTEM if needed. 48 | # However, this should only be done to compare two sets of test output or two 49 | # sets of benchmarks. 50 | # Bad things will happen if tests are run without the default FILESTEM! 51 | FILESTEM = dict( _FILESTEM_TUPLE ) 52 | 53 | class TestProgram: 54 | '''Store and access information about the program being tested.''' 55 | def __init__(self, name, exe, test_id, benchmark, **kwargs): 56 | 57 | # Set sane defaults (mostly null) for keyword arguments. 58 | 59 | self.name = name 60 | 61 | # Running 62 | self.exe = exe 63 | self.test_id = test_id 64 | self.run_cmd_template = ('tc.program tc.args tc.input > ' 65 | 'tc.output 2> tc.error') 66 | self.launch_parallel = 'mpirun -np tc.nprocs' 67 | self.submit_pattern = 'testcode.run_cmd' 68 | 69 | # dummy job with default settings (e.g tolerance) 70 | self.default_test_settings = None 71 | 72 | # Analysis 73 | self.benchmark = benchmark 74 | self.ignore_fields = [] 75 | self.data_tag = None 76 | self.extract_cmd_template = 'tc.extract tc.args tc.file' 77 | self.extract_program = None 78 | self.extract_args = '' 79 | self.extract_fmt = 'table' 80 | self.skip_cmd_template = 'tc.skip tc.args tc.test' 81 | self.skip_program = None 82 | self.skip_args = '' 83 | self.verify = False 84 | self.extract_fn = None 85 | 86 | # Info 87 | self.vcs = None 88 | 89 | # Set values passed in as keyword options. 90 | for (attr, val) in kwargs.items(): 91 | setattr(self, attr, val) 92 | 93 | # If using an external verification program, then set the default 94 | # extract command template. 95 | if self.verify and 'extract_cmd_template' not in kwargs: 96 | self.extract_cmd_template = 'tc.extract tc.args tc.test tc.bench' 97 | 98 | if self.extract_fn: 99 | if _HAVE_IMPORTLIB_: 100 | self.extract_fn = self.extract_fn.split() 101 | if len(self.extract_fn) == 2: 102 | sys.path.append(self.extract_fn[0]) 103 | (mod, fn) = self.extract_fn[-1].rsplit('.', 1) 104 | mod = importlib.import_module(mod) 105 | self.extract_fn = mod.__getattribute__(fn) 106 | elif self.extract_program: 107 | warnings.warn('importlib not available. Will attempt to ' 108 | 'analyse data via an external script.') 109 | self.extract_fn = None 110 | else: 111 | raise exceptions.TestCodeError('importlib not available and ' 112 | 'no data extraction program supplied.') 113 | 114 | # Can we actually extract the data? 115 | if self.extract_fmt == 'yaml' and not _HAVE_YAML: 116 | err = 'YAML data format cannot be used: PyYAML is not installed.' 117 | raise exceptions.TestCodeError(err) 118 | 119 | def run_cmd(self, input_file, args, nprocs=0): 120 | '''Create run command.''' 121 | output_file = util.testcode_filename(FILESTEM['test'], self.test_id, 122 | input_file, args) 123 | error_file = util.testcode_filename(FILESTEM['error'], self.test_id, 124 | input_file, args) 125 | 126 | # Need to escape filenames for passing them to the shell. 127 | exe = pipes.quote(self.exe) 128 | output_file = pipes.quote(output_file) 129 | error_file = pipes.quote(error_file) 130 | 131 | cmd = self.run_cmd_template.replace('tc.program', exe) 132 | if type(input_file) is str: 133 | input_file = pipes.quote(input_file) 134 | cmd = cmd.replace('tc.input', input_file) 135 | else: 136 | cmd = cmd.replace('tc.input', '') 137 | if type(args) is str: 138 | cmd = cmd.replace('tc.args', args) 139 | else: 140 | cmd = cmd.replace('tc.args', '') 141 | cmd = cmd.replace('tc.output', output_file) 142 | cmd = cmd.replace('tc.error', error_file) 143 | if nprocs > 0 and self.launch_parallel: 144 | cmd = '%s %s' % (self.launch_parallel, cmd) 145 | cmd = cmd.replace('tc.nprocs', str(nprocs)) 146 | return cmd 147 | 148 | def extract_cmd(self, path, input_file, args): 149 | '''Create extraction command(s).''' 150 | test_file = util.testcode_filename(FILESTEM['test'], self.test_id, 151 | input_file, args) 152 | bench_file = self.select_benchmark_file(path, input_file, args) 153 | cmd = self.extract_cmd_template 154 | cmd = cmd.replace('tc.extract', pipes.quote(self.extract_program)) 155 | cmd = cmd.replace('tc.args', self.extract_args) 156 | if self.verify: 157 | # Single command to compare benchmark and test outputs. 158 | cmd = cmd.replace('tc.test', pipes.quote(test_file)) 159 | cmd = cmd.replace('tc.bench', pipes.quote(bench_file)) 160 | return (cmd,) 161 | else: 162 | # Need to return commands to extract data from the test and 163 | # benchmark outputs. 164 | test_cmd = cmd.replace('tc.file', pipes.quote(test_file)) 165 | bench_cmd = cmd.replace('tc.file', pipes.quote(bench_file)) 166 | return (bench_cmd, test_cmd) 167 | 168 | def skip_cmd(self, input_file, args): 169 | '''Create skip command.''' 170 | test_file = util.testcode_filename(FILESTEM['test'], self.test_id, 171 | input_file, args) 172 | error_file = util.testcode_filename(FILESTEM['error'], self.test_id, 173 | input_file, args) 174 | cmd = self.skip_cmd_template 175 | cmd = cmd.replace('tc.skip', pipes.quote(self.skip_program)) 176 | cmd = cmd.replace('tc.args', self.skip_args) 177 | cmd = cmd.replace('tc.test', pipes.quote(test_file)) 178 | cmd = cmd.replace('tc.error', pipes.quote(error_file)) 179 | return cmd 180 | 181 | def select_benchmark_file(self, path, input_file, args): 182 | '''Find the first benchmark file out of all benchmark IDs which exists.''' 183 | 184 | benchmark = None 185 | benchmarks = [] 186 | for bench_id in self.benchmark: 187 | benchfile = util.testcode_filename(FILESTEM['benchmark'], bench_id, 188 | input_file, args) 189 | benchmarks.append(benchfile) 190 | if os.path.exists(os.path.join(path, benchfile)): 191 | benchmark = benchfile 192 | break 193 | if not benchmark: 194 | err = 'No benchmark found in %s. Checked for: %s.' 195 | raise exceptions.TestCodeError(err % (path, ', '.join(benchmarks))) 196 | return benchmark 197 | 198 | class Test: 199 | '''Store and execute a test.''' 200 | def __init__(self, name, test_program, path, **kwargs): 201 | 202 | self.name = name 203 | 204 | # program 205 | self.test_program = test_program 206 | 207 | # running 208 | self.path = path 209 | self.inputs_args = None 210 | self.output = None 211 | self.nprocs = 0 212 | self.min_nprocs = 0 213 | self.max_nprocs = compat.maxint 214 | self.submit_template = None 215 | # Run jobs in this concurrently rather than consecutively? 216 | # Only used when setting tests up in testcode2.config: if true then 217 | # each pair of input file and arguments are assigned to a different 218 | # Test object rather than a single Test object. 219 | self.run_concurrent = False 220 | 221 | # Analysis 222 | self.default_tolerance = None 223 | self.tolerances = {} 224 | 225 | # Set values passed in as keyword options. 226 | for (attr, val) in kwargs.items(): 227 | setattr(self, attr, val) 228 | 229 | if not self.inputs_args: 230 | self.inputs_args = [('', '')] 231 | 232 | self.status = dict( (inp_arg, None) for inp_arg in self.inputs_args ) 233 | 234 | # 'Decorate' functions which require a directory lock in order for file 235 | # access to be thread-safe. 236 | # As we use the in_dir decorator, which requires knowledge of the test 237 | # directory (a per-instance property), we cannot use the @decorator 238 | # syntactic sugar. Fortunately we can still modify them at 239 | # initialisation time. Thank you python for closures! 240 | self.start_job = DIR_LOCK.in_dir(self.path)(self._start_job) 241 | self.move_output_to_test_output = DIR_LOCK.in_dir(self.path)( 242 | self._move_output_to_test_output) 243 | self.move_old_output_files = DIR_LOCK.in_dir(self.path)( 244 | self._move_old_output_files) 245 | self.verify_job = DIR_LOCK.in_dir(self.path)(self._verify_job) 246 | self.skip_job = DIR_LOCK.in_dir(self.path)(self._skip_job) 247 | 248 | def __hash__(self): 249 | return hash(self.path) 250 | 251 | def __eq__(self, other): 252 | if not isinstance(other, self.__class__): 253 | return False 254 | else: 255 | # Compare values we care about... 256 | cmp_vals = ['test_program', 'path', 'inputs_args', 'output', 257 | 'nprocs', 'min_nprocs', 'max_nprocs', 'submit_template', 258 | 'default_tolerance', 'tolerances', 'status'] 259 | comparison = tuple(getattr(other, cmp_val) == getattr(self, cmp_val) for cmp_val in cmp_vals) 260 | return compat.compat_all(comparison) 261 | 262 | def run_test(self, verbose=1, cluster_queue=None, rundir=None): 263 | '''Run all jobs in test.''' 264 | 265 | try: 266 | # Construct tests. 267 | test_cmds = [] 268 | test_files = [] 269 | for (test_input, test_arg) in self.inputs_args: 270 | if (test_input and 271 | not os.path.exists(os.path.join(self.path,test_input))): 272 | err = 'Input file does not exist: %s' % (test_input,) 273 | raise exceptions.RunError(err) 274 | test_cmds.append(self.test_program.run_cmd(test_input, test_arg, 275 | self.nprocs)) 276 | test_files.append(util.testcode_filename(FILESTEM['test'], 277 | self.test_program.test_id, test_input, test_arg)) 278 | 279 | # Move files matching output pattern out of the way. 280 | self.move_old_output_files(verbose) 281 | 282 | # Run tests one-at-a-time locally or submit job in single submit 283 | # file to a queueing system. 284 | if cluster_queue: 285 | if self.output: 286 | for (ind, test) in enumerate(test_cmds): 287 | # Don't quote self.output if it contains any wildcards 288 | # (assume the user set it up correctly!) 289 | out = self.output 290 | if not compat.compat_any(wild in self.output for wild in 291 | ['*', '?', '[', '{']): 292 | out = pipes.quote(self.output) 293 | test_cmds[ind] = '%s; mv %s %s' % (test_cmds[ind], 294 | out, pipes.quote(test_files[ind])) 295 | test_cmds = ['\n'.join(test_cmds)] 296 | for (ind, test) in enumerate(test_cmds): 297 | job = self.start_job(test, cluster_queue, verbose) 298 | job.wait() 299 | # Analyse tests as they finish. 300 | if cluster_queue: 301 | # Did all of them at once. 302 | for (test_input, test_arg) in self.inputs_args: 303 | self.verify_job(test_input, test_arg, verbose, rundir) 304 | else: 305 | # Did one job at a time. 306 | (test_input, test_arg) = self.inputs_args[ind] 307 | err = [] 308 | if self.output: 309 | try: 310 | self.move_output_to_test_output(test_files[ind]) 311 | except exceptions.RunError: 312 | err.append(sys.exc_info()[1]) 313 | status = validation.Status() 314 | if job.returncode != 0: 315 | err.insert(0, 'Error running job. Return code: %i' 316 | % job.returncode) 317 | (status, msg) = self.skip_job(test_input, test_arg, 318 | verbose) 319 | if status.skipped(): 320 | self._update_status(status, (test_input, test_arg)) 321 | if verbose > 0 and verbose < 3: 322 | sys.stdout.write( 323 | util.info_line(self.path, 324 | test_input, test_arg, rundir) 325 | ) 326 | status.print_status(msg, verbose) 327 | elif err: 328 | # re-raise first error we hit. 329 | raise exceptions.RunError(err[0]) 330 | else: 331 | self.verify_job(test_input, test_arg, verbose, rundir) 332 | sys.stdout.flush() 333 | except exceptions.RunError: 334 | err = sys.exc_info()[1] 335 | if verbose > 2: 336 | err = 'Test(s) in %s failed.\n%s' % (self.path, err) 337 | status = validation.Status([False]) 338 | self._update_status(status, (test_input, test_arg)) 339 | if verbose > 0 and verbose < 3: 340 | info_line = util.info_line(self.path, test_input, test_arg, rundir) 341 | sys.stdout.write(info_line) 342 | status.print_status(err, verbose) 343 | # Shouldn't run remaining tests after such a catastrophic failure. 344 | # Mark all remaining tests as skipped so the user knows that they 345 | # weren't run. 346 | err = 'Previous test in %s caused a system failure.' % (self.path) 347 | status = validation.Status(name='skipped') 348 | for ((test_input, test_arg), stat) in self.status.items(): 349 | if not self.status[(test_input,test_arg)]: 350 | self._update_status(status, (test_input, test_arg)) 351 | if verbose > 2: 352 | cmd = self.test_program.run_cmd(test_input, test_arg, 353 | self.nprocs) 354 | print('Test using %s in %s' % (cmd, self.path)) 355 | elif verbose > 0: 356 | info_line = util.info_line(self.path, test_input, 357 | test_arg, rundir) 358 | sys.stdout.write(info_line) 359 | status.print_status(err, verbose) 360 | sys.stdout.flush() 361 | 362 | def _start_job(self, cmd, cluster_queue=None, verbose=1): 363 | '''Start test running. Requires directory lock. 364 | 365 | IMPORTANT: use self.start_job rather than self._start_job if using multiple 366 | threads. 367 | 368 | Decorated to start_job, which acquires directory lock and enters self.path 369 | first, during initialisation.''' 370 | 371 | if cluster_queue: 372 | tp_ptr = self.test_program 373 | submit_file = '%s.%s' % (os.path.basename(self.submit_template), 374 | tp_ptr.test_id) 375 | job = queues.ClusterQueueJob(submit_file, system=cluster_queue) 376 | job.create_submit_file(tp_ptr.submit_pattern, cmd, 377 | self.submit_template) 378 | if verbose > 2: 379 | print('Submitting tests using %s (template submit file) in %s' 380 | % (self.submit_template, self.path)) 381 | job.start_job() 382 | else: 383 | # Run locally via subprocess. 384 | if verbose > 2: 385 | print('Running test using %s in %s\n' % (cmd, self.path)) 386 | try: 387 | job = subprocess.Popen(cmd, shell=True) 388 | except OSError: 389 | # slightly odd syntax in order to be compatible with python 2.5 390 | # and python 2.6/3 391 | err = 'Execution of test failed: %s' % (sys.exc_info()[1],) 392 | raise exceptions.RunError(err) 393 | 394 | # Return either Popen object or ClusterQueueJob object. Both have 395 | # a wait method which returns only once job has finished. 396 | return job 397 | 398 | def _move_output_to_test_output(self, test_files_out): 399 | '''Move output to the testcode output file. Requires directory lock. 400 | 401 | This is used when a program writes to standard output rather than to STDOUT. 402 | 403 | IMPORTANT: use self.move_output_to_test_output rather than 404 | self._move_output_to_test_output if using multiple threads. 405 | 406 | Decorated to move_output_to_test_output, which acquires the directory lock and 407 | enters self.path. 408 | ''' 409 | # self.output might be a glob which works with e.g. 410 | # mv self.output test_files[ind] 411 | # if self.output matches only one file. Reproduce that 412 | # here so that running tests through the queueing system 413 | # and running tests locally have the same behaviour. 414 | out_files = glob.glob(self.output) 415 | if len(out_files) == 1: 416 | shutil.move(out_files[0], test_files_out) 417 | else: 418 | err = ('Output pattern (%s) matches %s files (%s).' 419 | % (self.output, len(out_files), out_files)) 420 | raise exceptions.RunError(err) 421 | 422 | def _move_old_output_files(self, verbose=1): 423 | '''Move output to the testcode output file. Requires directory lock. 424 | 425 | This is used when a program writes to standard output rather than to STDOUT. 426 | 427 | IMPORTANT: use self.move_oold_output_files rather than 428 | self._move_old_output_files if using multiple threads. 429 | 430 | Decorated to move_old_output_files, which acquires the directory lock and 431 | enters self.path. 432 | ''' 433 | if self.output: 434 | old_out_files = glob.glob(self.output) 435 | if old_out_files: 436 | out_dir = 'test.prev.output.%s' % (self.test_program.test_id) 437 | if verbose > 2: 438 | print('WARNING: found existing files matching output ' 439 | 'pattern: %s.' % self.output) 440 | print('WARNING: moving existing output files (%s) to %s.\n' 441 | % (', '.join(old_out_files), out_dir)) 442 | if not os.path.exists(out_dir): 443 | os.mkdir(out_dir) 444 | for out_file in old_out_files: 445 | shutil.move(out_file, out_dir) 446 | 447 | def _verify_job(self, input_file, args, verbose=1, rundir=None): 448 | '''Check job against benchmark. 449 | 450 | Assume function is executed in self.path. 451 | 452 | IMPORTANT: use self.verify_job rather than self._verify_job if using multiple 453 | threads. 454 | 455 | Decorated to verify_job, which acquires directory lock and enters self.path 456 | first, during initialisation.''' 457 | # We already have DIR_LOCK, so use _skip_job instead of skip_job. 458 | (status, msg) = self._skip_job(input_file, args, verbose) 459 | try: 460 | if self.test_program.verify and not status.skipped(): 461 | (status, msg) = self.verify_job_external(input_file, args, 462 | verbose) 463 | elif not status.skipped(): 464 | (bench_out, test_out) = self.extract_data(input_file, args, 465 | verbose) 466 | (comparable, status, msg) = validation.compare_data(bench_out, 467 | test_out, self.default_tolerance, self.tolerances, 468 | self.test_program.ignore_fields) 469 | if verbose > 2: 470 | # Include data tables in output. 471 | if comparable: 472 | # Combine test and benchmark dictionaries. 473 | data_table = util.pretty_print_table( 474 | ['benchmark', 'test'], 475 | [bench_out, test_out]) 476 | else: 477 | # Print dictionaries separately--couldn't even compare 478 | # them! 479 | data_table = '\n'.join(( 480 | util.pretty_print_table(['benchmark'], [bench_out]), 481 | util.pretty_print_table(['test '], [test_out]))) 482 | if msg.strip(): 483 | # join data table with error message from 484 | # validation.compare_data. 485 | msg = '\n'.join((msg, data_table)) 486 | else: 487 | msg = data_table 488 | except (exceptions.AnalysisError, exceptions.TestCodeError): 489 | if msg.strip(): 490 | msg = '%s\n%s' % (msg, sys.exc_info()[1]) 491 | else: 492 | msg = sys.exc_info()[1] 493 | status = validation.Status([False]) 494 | 495 | self._update_status(status, (input_file, args)) 496 | if verbose > 0 and verbose < 3: 497 | info_line = util.info_line(self.path, input_file, args, rundir) 498 | sys.stdout.write(info_line) 499 | status.print_status(msg, verbose) 500 | 501 | return (status, msg) 502 | 503 | def _skip_job(self, input_file, args, verbose=1): 504 | '''Run user-supplied command to check if test should be skipped. 505 | 506 | IMPORTANT: use self.skip_job rather than self._skip_job if using multiple 507 | threads. 508 | 509 | Decorated to skip_job, which acquires directory lock and enters self.path 510 | first, during initialisation.''' 511 | status = validation.Status() 512 | if self.test_program.skip_program: 513 | cmd = self.test_program.skip_cmd(input_file, args) 514 | try: 515 | if verbose > 2: 516 | print('Testing whether to skip test using %s in %s.' % 517 | (cmd, self.path)) 518 | skip_popen = subprocess.Popen(cmd, shell=True, 519 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) 520 | skip_popen.wait() 521 | if skip_popen.returncode == 0: 522 | # skip this test 523 | status = validation.Status(name='skipped') 524 | except OSError: 525 | # slightly odd syntax in order to be compatible with python 526 | # 2.5 and python 2.6/3 527 | if verbose > 2: 528 | print('Test to skip test: %s' % (sys.exc_info()[1],)) 529 | return (status, '') 530 | 531 | def verify_job_external(self, input_file, args, verbose=1): 532 | '''Run user-supplied verifier script. 533 | 534 | Assume function is executed in self.path.''' 535 | verify_cmd, = self.test_program.extract_cmd(self.path, input_file, args) 536 | try: 537 | if verbose > 2: 538 | print('Analysing test using %s in %s.' % 539 | (verify_cmd, self.path)) 540 | verify_popen = subprocess.Popen(verify_cmd, shell=True, 541 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) 542 | verify_popen.wait() 543 | except OSError: 544 | # slightly odd syntax in order to be compatible with python 2.5 545 | # and python 2.6/3 546 | err = 'Analysis of test failed: %s' % (sys.exc_info()[1],) 547 | raise exceptions.AnalysisError(err) 548 | output = verify_popen.communicate()[0].decode('utf-8') 549 | if verbose < 2: 550 | # Suppress output. (hackhack) 551 | output = '' 552 | if verify_popen.returncode == 0: 553 | return (validation.Status([True]), output) 554 | else: 555 | return (validation.Status([False]), output) 556 | 557 | def extract_data(self, input_file, args, verbose=1): 558 | '''Extract data from output file. 559 | 560 | Assume function is executed in self.path.''' 561 | tp_ptr = self.test_program 562 | data_files = [ 563 | tp_ptr.select_benchmark_file(self.path, input_file, args), 564 | util.testcode_filename(FILESTEM['test'], 565 | tp_ptr.test_id, input_file, args), 566 | ] 567 | if tp_ptr.data_tag: 568 | # Using internal data extraction function. 569 | if verbose > 2: 570 | print('Analysing output using data_tag %s in %s on files %s.' % 571 | (tp_ptr.data_tag, self.path, ' and '.join(data_files))) 572 | outputs = [util.extract_tagged_data(tp_ptr.data_tag, dfile) 573 | for dfile in data_files] 574 | elif tp_ptr.extract_fn: 575 | if verbose > 2: 576 | print('Analysing output using function %s in %s on files %s.' % 577 | (tp_ptr.extract_fn.__name__, self.path, 578 | ' and '.join(data_files))) 579 | outputs = [tp_ptr.extract_fn(dfile) for dfile in data_files] 580 | else: 581 | # Using external data extraction script. 582 | # Get extraction commands. 583 | extract_cmds = tp_ptr.extract_cmd(self.path, input_file, args) 584 | 585 | # Extract data. 586 | outputs = [] 587 | for cmd in extract_cmds: 588 | try: 589 | if verbose > 2: 590 | print('Analysing output using %s in %s.' % 591 | (cmd, self.path)) 592 | extract_popen = subprocess.Popen(cmd, shell=True, 593 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) 594 | extract_popen.wait() 595 | except OSError: 596 | # slightly odd syntax in order to be compatible with python 597 | # 2.5 and python 2.6/3 598 | err = 'Analysing output failed: %s' % (sys.exc_info()[1],) 599 | raise exceptions.AnalysisError(err) 600 | # Convert data string from extract command to dictionary format. 601 | if extract_popen.returncode != 0: 602 | err = extract_popen.communicate()[1].decode('utf-8') 603 | err = 'Analysing output failed: %s' % (err) 604 | raise exceptions.AnalysisError(err) 605 | data_string = extract_popen.communicate()[0].decode('utf-8') 606 | if self.test_program.extract_fmt == 'table': 607 | outputs.append(util.dict_table_string(data_string)) 608 | elif self.test_program.extract_fmt == 'yaml': 609 | outputs.append({}) 610 | # convert values to be in a tuple so the format matches 611 | # that from dict_table_string. 612 | # ensure all keys are strings so they can be sorted 613 | # (different data types cause problems!) 614 | for (key, val) in yaml.safe_load(data_string).items(): 615 | if isinstance(val, list): 616 | outputs[-1][str(key)] = tuple(val) 617 | else: 618 | outputs[-1][str(key)] = tuple((val,)) 619 | 620 | return tuple(outputs) 621 | 622 | def create_new_benchmarks(self, benchmark, copy_files_since=None, 623 | copy_files_path='testcode_data'): 624 | '''Copy the test files to benchmark files.''' 625 | 626 | oldcwd = os.getcwd() 627 | os.chdir(self.path) 628 | 629 | test_files = [] 630 | for (inp, arg) in self.inputs_args: 631 | test_file = util.testcode_filename(FILESTEM['test'], 632 | self.test_program.test_id, inp, arg) 633 | err_file = util.testcode_filename(FILESTEM['error'], 634 | self.test_program.test_id, inp, arg) 635 | bench_file = util.testcode_filename(_FILESTEM_DICT['benchmark'], 636 | benchmark, inp, arg) 637 | test_files.extend((test_file, err_file, bench_file)) 638 | shutil.copy(test_file, bench_file) 639 | 640 | if copy_files_since: 641 | if not os.path.isdir(copy_files_path): 642 | os.mkdir(copy_files_path) 643 | if os.path.isdir(copy_files_path): 644 | for data_file in glob.glob('*'): 645 | if (os.path.isfile(data_file) and 646 | os.stat(data_file)[-2] >= copy_files_since and 647 | data_file not in test_files): 648 | bench_data_file = os.path.join(copy_files_path, 649 | data_file) 650 | # shutil.copy can't overwrite files so remove old ones 651 | # with the same name. 652 | if os.path.exists(bench_data_file): 653 | os.unlink(bench_data_file) 654 | shutil.copy(data_file, bench_data_file) 655 | 656 | os.chdir(oldcwd) 657 | 658 | def _update_status(self, status, inp_arg): 659 | '''Update self.status with success of a test.''' 660 | if status: 661 | self.status[inp_arg] = status 662 | else: 663 | # Something went wrong. Store a Status failed object. 664 | self.status[inp_arg] = validation.Status([False]) 665 | 666 | def get_status(self): 667 | '''Get number of passed and number of ran tasks.''' 668 | # If there's an object (other than None/False) in the corresponding 669 | # dict entry in self.status, then that test must have ran (albeit not 670 | # necessarily successfuly!). 671 | status = {} 672 | status['passed'] = sum(True for stat in self.status.values() 673 | if stat and stat.passed()) 674 | status['warning'] = sum(True for stat in self.status.values() 675 | if stat and stat.warning()) 676 | status['skipped'] = sum(True for stat in self.status.values() 677 | if stat and stat.skipped()) 678 | status['failed'] = sum(True for stat in self.status.values() 679 | if stat and stat.failed()) 680 | status['unknown'] = sum(True for stat in self.status.values() 681 | if stat and stat.unknown()) 682 | status['ran'] = sum(True for stat in self.status.values() if stat) 683 | return status 684 | -------------------------------------------------------------------------------- /lib/testcode2/_functools_dummy.py: -------------------------------------------------------------------------------- 1 | ''' 2 | testcode2._dummy_functools 3 | -------------------------- 4 | 5 | Dummy stub functions of required functools objects used. 6 | 7 | This means that we can use python 2.4 and advanced features in later versions 8 | of python. 9 | 10 | :copyright: (c) 2012 James Spencer. 11 | :license: modified BSD; see LICENSE for more details. 12 | ''' 13 | 14 | def wraps(func1): 15 | '''Upgrade from python 2.4 to use functools.wraps.''' 16 | def wrapper(func2): 17 | '''Upgrade from python 2.4 to use functools.wraps.''' 18 | def decorated_func(*args, **kwargs): 19 | '''Upgrade from python 2.4 to use functools.wraps.''' 20 | return func2(*args, **kwargs) 21 | return decorated_func 22 | return wrapper 23 | -------------------------------------------------------------------------------- /lib/testcode2/ansi.py: -------------------------------------------------------------------------------- 1 | ''' 2 | testcode2.ansi 3 | -------------- 4 | 5 | (A subset of) ANSI codes and functions to wrap codes around strings. 6 | 7 | :copyright: (c) 2012 James Spencer. 8 | :license: modified BSD; see LICENSE for more details. 9 | ''' 10 | 11 | import sys 12 | 13 | ANSI_START = '\033[' 14 | ANSI_END = '\033[0m' 15 | 16 | ANSI_SGR = dict( 17 | bold=1, 18 | ) 19 | 20 | ANSI_INTENSITY = dict( 21 | normal=0, 22 | bright=60, 23 | ) 24 | 25 | ANSI_COLOUR = dict( 26 | black=30, 27 | red=31, 28 | green=32, 29 | yellow=33, 30 | blue=34, 31 | magenta=35, 32 | cyan=36, 33 | white=37, 34 | ) 35 | 36 | def ansi_format(string, colour='black', intensity='normal', style=None, 37 | override=False): 38 | '''Return string wrapped in the appropriate ANSI codes. 39 | 40 | Note: if not writing to a true terminal and override is false, then the string 41 | is not modified. 42 | ''' 43 | 44 | if sys.stdout.isatty() or override: 45 | code = str(ANSI_COLOUR[colour]+ANSI_INTENSITY[intensity]) 46 | if style: 47 | code += ';%s' % (ANSI_SGR[style]) 48 | code += 'm' 49 | return ANSI_START + code + string + ANSI_END + ANSI_END 50 | else: 51 | return string 52 | -------------------------------------------------------------------------------- /lib/testcode2/compatibility.py: -------------------------------------------------------------------------------- 1 | ''' 2 | testcode2.compatibility 3 | ----------------------- 4 | 5 | Functions for compatibility with different versions of python. 6 | 7 | testcode2 is developed using python 3.2; these statements exist to enable 8 | testcode to function transparently (i.e. without using 2to3) on python 2.4 9 | onwards. 10 | 11 | Rather than using conditional statements in the main source code, instead place 12 | the required statements in this module and then use (e.g.) 13 | 14 | import testcode2.compatibility as compat 15 | 16 | var = compat.compat_set([1,2,3,1]) 17 | 18 | in the main source code. 19 | 20 | :copyright: (c) 2012 James Spencer. 21 | :license: modified BSD; see LICENSE for more details. 22 | ''' 23 | 24 | import sys 25 | 26 | ### python 2.4 ### 27 | 28 | # Import from the sets module if sets are not part of the language. 29 | try: 30 | compat_set = set 31 | except NameError: 32 | from sets import Set as compat_set 33 | 34 | # Any and all don't exist in python <2.5. Define our own in pure python. 35 | try: 36 | compat_all = all 37 | except NameError: 38 | def compat_all(iterable): 39 | '''all(iterable) -> bool 40 | 41 | Return True if bool(x) is True for all values x in the iterable. 42 | ''' 43 | for val in iterable: 44 | if not val: 45 | return False 46 | return True 47 | try: 48 | compat_any = any 49 | except NameError: 50 | def compat_any(iterable): 51 | '''any(iterable) -> bool 52 | 53 | Return True if bool(x) is True for any x in the iterable. 54 | ''' 55 | for val in iterable: 56 | if val: 57 | return True 58 | 59 | try: 60 | import functools 61 | except ImportError: 62 | import testcode2._functools_dummy as functools 63 | 64 | ### python 2.4, python 2.5 ### 65 | 66 | # math.isnan was introduced in python 2.6, so need a workaround for 2.4 and 2.5. 67 | try: 68 | from math import isnan 69 | except ImportError: 70 | def isnan(val): 71 | '''Return True if x is a NaN (not a number), and False otherwise. 72 | 73 | :param float val: number. 74 | 75 | Replacement for math.isnan for python <2.6. 76 | This is not guaranteed to be portable, but does work under Linux. 77 | ''' 78 | return type(val) is float and val != val 79 | 80 | try: 81 | # python >=2.6 82 | from ast import literal_eval 83 | except ImportError: 84 | # python 2.4, 2.5 85 | from compiler import parse 86 | from compiler import ast 87 | def literal_eval(node_or_string): 88 | """Safely evaluate a node/string containing a Python expression. 89 | 90 | Thestring or node provided may only consist of the following Python literal 91 | structures: strings, numbers, tuples, lists, dicts, booleans, and None. 92 | 93 | Essentially a backport of the literal_eval function in python 2.6 onwards. 94 | From: http://mail.python.org/pipermail/python-list/2009-September/1219992.html 95 | """ 96 | _safe_names = {'None': None, 'True': True, 'False': False} 97 | if isinstance(node_or_string, basestring): 98 | node_or_string = parse(node_or_string, mode='eval') 99 | if isinstance(node_or_string, ast.Expression): 100 | node_or_string = node_or_string.node 101 | def _convert(node): 102 | '''Convert node/string to expression.''' 103 | if isinstance(node, ast.Const) and isinstance(node.value, 104 | (basestring, int, float, long, complex)): 105 | return node.value 106 | elif isinstance(node, ast.Tuple): 107 | return tuple(_convert(element) for element in node.nodes) 108 | elif isinstance(node, ast.List): 109 | return list(_convert(element) for element in node.nodes) 110 | elif isinstance(node, ast.Dict): 111 | return dict((_convert(k), _convert(v)) for k, v 112 | in node.items) 113 | elif isinstance(node, ast.Name): 114 | if node.name in _safe_names: 115 | return _safe_names[node.name] 116 | elif isinstance(node, ast.UnarySub): 117 | return -_convert(node.expr) 118 | raise ValueError('malformed string') 119 | return _convert(node_or_string) 120 | 121 | # os.path.relpath was introduced in python 2.6. 122 | try: 123 | from os.path import relpath 124 | except ImportError: 125 | import os.path 126 | def relpath(path, start=os.path.curdir): 127 | """Return a relative version of a path""" 128 | 129 | if not path: 130 | raise ValueError("no path specified") 131 | 132 | filter_null = lambda lll: [x for x in lll if x] 133 | 134 | start_list = filter_null(os.path.abspath(start).split(os.path.sep)) 135 | path_list = filter_null(os.path.abspath(path).split(os.path.sep)) 136 | 137 | common = len(os.path.commonprefix([start_list, path_list])) 138 | 139 | rel_list = [os.pardir] * (len(start_list)-common) + path_list[common:] 140 | if not rel_list: 141 | return os.path.curdir 142 | return os.path.join(*rel_list) 143 | 144 | ### python 2 ### 145 | 146 | try: 147 | import configparser 148 | except ImportError: 149 | import ConfigParser as configparser 150 | 151 | try: 152 | compat_input = raw_input 153 | except NameError: 154 | compat_input = input 155 | 156 | try: 157 | maxint = sys.maxint 158 | except AttributeError: 159 | maxint = sys.maxsize 160 | -------------------------------------------------------------------------------- /lib/testcode2/config.py: -------------------------------------------------------------------------------- 1 | ''' 2 | testcode2.config 3 | ---------------- 4 | 5 | Parse jobconfig and userconfig ini files. 6 | 7 | :copyright: (c) 2012 James Spencer. 8 | :license: modified BSD; see LICENSE for more details. 9 | ''' 10 | 11 | import copy 12 | import glob 13 | import os 14 | import shlex 15 | import subprocess 16 | import time 17 | import warnings 18 | 19 | import testcode2 20 | import testcode2.compatibility as compat 21 | import testcode2.exceptions as exceptions 22 | import testcode2.util as util 23 | import testcode2.validation as validation 24 | import testcode2.vcs as vcs 25 | 26 | def eval_nested_tuple(string): 27 | nested_tuple = compat.literal_eval(string) 28 | if isinstance(nested_tuple[0], (list, tuple)): 29 | return nested_tuple 30 | else: 31 | # Append a comma to the option to ensure literal_eval returns a tuple 32 | # of tuples, even if the option only contains a single tuple. 33 | return compat.literal_eval('%s,' % string) 34 | 35 | def parse_tolerance_tuple(val): 36 | '''Parse (abs_tol,rel_tol,name,strict).''' 37 | if len(val) >= 4: 38 | strict = val[3] 39 | else: 40 | strict = True 41 | if len(val) >= 3: 42 | name = val[2] 43 | else: 44 | name = None 45 | if len(val) >= 2: 46 | rel_tol = val[1] 47 | else: 48 | rel_tol = None 49 | if len(val) >= 1: 50 | abs_tol = val[0] 51 | else: 52 | abs_tol = None 53 | return (name, validation.Tolerance(name, abs_tol, rel_tol, strict)) 54 | 55 | def parse_userconfig(config_file, executables=None, test_id=None, 56 | settings=None): 57 | '''Parse the user options and job types from the userconfig file. 58 | 59 | config_file: location of the userconfig file, either relative or absolute.''' 60 | 61 | if executables is None: 62 | executables = {} 63 | 64 | if not os.path.exists(config_file): 65 | raise exceptions.TestCodeError( 66 | 'User configuration file %s does not exist.' % (config_file) 67 | ) 68 | # paths to programs can be specified relative to the config 69 | # file. 70 | config_directory = os.path.dirname(os.path.abspath(config_file)) 71 | 72 | userconfig = compat.configparser.RawConfigParser() 73 | userconfig.optionxform = str # Case sensitive file. 74 | userconfig.read(config_file) 75 | 76 | # Alter config file with additional settings provided. 77 | if settings: 78 | for (section_key, section) in settings.items(): 79 | for (option_key, value) in section.items(): 80 | userconfig.set(section_key, option_key, value) 81 | 82 | # Sensible defaults for the user options. 83 | user_options = dict(benchmark=None, date_fmt='%d%m%Y', 84 | tolerance='(1.e-10,None)', output_files=None, diff='diff') 85 | 86 | if userconfig.has_section('user'): 87 | user_options.update(dict(userconfig.items('user'))) 88 | userconfig.remove_section('user') 89 | user_options['tolerance'] = dict( 90 | (parse_tolerance_tuple(item) 91 | for item in eval_nested_tuple(user_options['tolerance'])) 92 | ) 93 | if user_options['benchmark']: 94 | user_options['benchmark'] = user_options['benchmark'].split() 95 | else: 96 | raise exceptions.TestCodeError( 97 | 'user section in userconfig does not exist.' 98 | ) 99 | 100 | if not userconfig.sections(): 101 | raise exceptions.TestCodeError( 102 | 'No job types specified in userconfig.' 103 | ) 104 | 105 | test_program_options = ('run_cmd_template', 106 | 'launch_parallel', 'ignore_fields', 'data_tag', 'extract_cmd_template', 107 | 'extract_fn', 'extract_program', 'extract_args', 'extract_fmt', 108 | 'verify', 'vcs', 'skip_program', 'skip_args', 'skip_cmd_template') 109 | default_test_options = ('inputs_args', 'output', 'nprocs', 110 | 'min_nprocs', 'max_nprocs', 'submit_template',) 111 | test_programs = {} 112 | for section in userconfig.sections(): 113 | tp_dict = {} 114 | tolerances = copy.deepcopy(user_options['tolerance']) 115 | # Read in possible TestProgram settings. 116 | for item in test_program_options: 117 | if userconfig.has_option(section, item): 118 | tp_dict[item] = userconfig.get(section, item) 119 | if 'ignore_fields' in tp_dict: 120 | tp_dict['ignore_fields'] = shlex.split(tp_dict['ignore_fields']) 121 | if section in executables: 122 | exe = executables[section] 123 | elif '_tc_all' in executables: 124 | exe = executables['_tc_all'] 125 | else: 126 | exe = 'exe' 127 | if userconfig.has_option(section, exe): 128 | # exe is set to be a key rather than the path to an executable. 129 | # Expand. 130 | exe = userconfig.get(section, exe) 131 | # Create a default test settings. 132 | # First, tolerances... 133 | if userconfig.has_option(section, 'tolerance'): 134 | for item in ( 135 | eval_nested_tuple(userconfig.get(section, 'tolerance')) 136 | ): 137 | (name, tol) = parse_tolerance_tuple(item) 138 | tolerances[name] = tol 139 | test_dict = dict( 140 | default_tolerance=tolerances[None], 141 | tolerances=tolerances, 142 | ) 143 | # Other settings... 144 | for item in default_test_options: 145 | if userconfig.has_option(section, item): 146 | test_dict[item] = userconfig.get(section, item) 147 | if userconfig.has_option(section, 'run_concurrent'): 148 | test_dict['run_concurrent'] = \ 149 | userconfig.getboolean(section, 'run_concurrent') 150 | # Programs can be specified relative to the config directory. 151 | exe = set_program_name(exe, config_directory) 152 | if 'extract_program' in tp_dict: 153 | tp_dict['extract_program'] = set_program_name( 154 | tp_dict['extract_program'], config_directory) 155 | if 'skip_program' in tp_dict: 156 | tp_dict['skip_program'] = set_program_name( 157 | tp_dict['skip_program'], config_directory) 158 | if 'submit_template' in test_dict: 159 | test_dict['submit_template'] = os.path.join(config_directory, 160 | test_dict['submit_template']) 161 | for key in ('nprocs', 'max_nprocs', 'min_nprocs'): 162 | if key in test_dict: 163 | test_dict[key] = int(test_dict[key]) 164 | if 'inputs_args' in test_dict: 165 | # format: (input, arg), (input, arg)' 166 | test_dict['inputs_args'] = ( 167 | eval_nested_tuple(test_dict['inputs_args'])) 168 | # Create a default test. 169 | tp_dict['default_test_settings'] = testcode2.Test(None, None, None, 170 | **test_dict) 171 | if 'vcs' in tp_dict: 172 | tp_dict['vcs'] = vcs.VCSRepository(tp_dict['vcs'], 173 | os.path.dirname(exe)) 174 | program = testcode2.TestProgram(section, exe, test_id, 175 | user_options['benchmark'], **tp_dict) 176 | test_programs[section] = program 177 | 178 | if len(test_programs) == 1: 179 | # only one program; set default program which helpfully is the most 180 | # recent value of section from the previous loop. 181 | user_options['default_program'] = section 182 | 183 | return (user_options, test_programs) 184 | 185 | def parse_jobconfig(config_file, user_options, test_programs, settings=None): 186 | '''Parse the test configurations from the jobconfig file. 187 | 188 | config_file: location of the jobconfig file, either relative or absolute.''' 189 | 190 | if not os.path.exists(config_file): 191 | raise exceptions.TestCodeError( 192 | 'Job configuration file %s does not exist.' % (config_file) 193 | ) 194 | 195 | # paths to the test directories can be specified relative to the config 196 | # file. 197 | config_directory = os.path.dirname(os.path.abspath(config_file)) 198 | 199 | jobconfig = compat.configparser.RawConfigParser() 200 | jobconfig.optionxform = str # Case sensitive file. 201 | jobconfig.read(config_file) 202 | 203 | # Alter config file with additional settings provided. 204 | if settings: 205 | for (section_key, section) in settings.items(): 206 | for (option_key, value) in section.items(): 207 | jobconfig.set(section_key, option_key, value) 208 | 209 | # Parse job categories. 210 | # Just store as list of test names for now. 211 | if jobconfig.has_section('categories'): 212 | test_categories = dict(jobconfig.items('categories')) 213 | for (key, val) in test_categories.items(): 214 | test_categories[key] = val.split() 215 | jobconfig.remove_section('categories') 216 | else: 217 | test_categories = {} 218 | 219 | # Parse individual sections for tests. 220 | # Note that sections/paths may contain globs and hence correspond to 221 | # multiple tests. 222 | # First, find out the tests each section corresponds to. 223 | test_sections = [] 224 | for section in jobconfig.sections(): 225 | # Expand any globs in the path/section name and create individual Test 226 | # objects for each one. 227 | if jobconfig.has_option(section, 'path'): 228 | path = os.path.join(config_directory, 229 | jobconfig.get(section, 'path')) 230 | jobconfig.remove_option(section, 'path') 231 | globbed_tests = [(section, os.path.abspath(test_path)) 232 | for test_path in glob.glob(path)] 233 | else: 234 | path = os.path.join(config_directory, section) 235 | globbed_tests = [(test_path, os.path.abspath(test_path)) 236 | for test_path in glob.glob(path)] 237 | test_sections.append((section, globbed_tests)) 238 | test_sections.sort(key=lambda sec_info: len(sec_info[1]), reverse=True) 239 | test_info = {} 240 | for (section, globbed_tests) in test_sections: 241 | test_dict = {} 242 | # test program 243 | if jobconfig.has_option(section, 'program'): 244 | test_program = test_programs[jobconfig.get(section, 'program')] 245 | else: 246 | test_program = test_programs[user_options['default_program']] 247 | # tolerances 248 | if jobconfig.has_option(section, 'tolerance'): 249 | test_dict['tolerances'] = {} 250 | for item in ( 251 | eval_nested_tuple(jobconfig.get(section,'tolerance')) 252 | ): 253 | (name, tol) = parse_tolerance_tuple(item) 254 | test_dict['tolerances'][name] = tol 255 | jobconfig.remove_option(section, 'tolerance') 256 | if None in test_dict['tolerances']: 257 | test_dict['default_tolerance'] = test_dict['tolerances'][None] 258 | # inputs and arguments 259 | if jobconfig.has_option(section, 'inputs_args'): 260 | # format: (input, arg), (input, arg)' 261 | test_dict['inputs_args'] = ( 262 | eval_nested_tuple(jobconfig.get(section, 'inputs_args'))) 263 | jobconfig.remove_option(section, 'inputs_args') 264 | if jobconfig.has_option(section, 'run_concurrent'): 265 | test_dict['run_concurrent'] = \ 266 | jobconfig.getboolean(section, 'run_concurrent') 267 | jobconfig.remove_option(section, 'run_concurrent') 268 | # Other options. 269 | for option in jobconfig.options(section): 270 | test_dict[option] = jobconfig.get(section, option) 271 | for key in ('nprocs', 'max_nprocs', 'min_nprocs'): 272 | if key in test_dict: 273 | test_dict[key] = int(test_dict[key]) 274 | if 'submit_template' in test_dict: 275 | test_dict['submit_template'] = os.path.join(config_directory, 276 | test_dict['submit_template']) 277 | for (name, path) in globbed_tests: 278 | # Need to take care with tolerances: want to *update* existing 279 | # tolerance dictionary rather than overwrite it. 280 | # This means we can't just use test_dict to update the relevant 281 | # dictionary in test_info. 282 | tol = None 283 | if (name, path) in test_info: 284 | # Just update existing info. 285 | test = test_info[(name, path)] 286 | if 'tolerances' in test_dict: 287 | test[1]['tolerances'].update(test_dict['tolerances']) 288 | tol = test_dict.pop('tolerances') 289 | test[0] = test_program 290 | test[1].update(test_dict) 291 | if tol: 292 | test_dict['tolerances'] = tol 293 | else: 294 | # Create new test_info value. 295 | # Merge with default values. 296 | # Default test options. 297 | default_test = test_program.default_test_settings 298 | test = dict( 299 | inputs_args=default_test.inputs_args, 300 | output=default_test.output, 301 | default_tolerance=default_test.default_tolerance, 302 | tolerances = copy.deepcopy(default_test.tolerances), 303 | nprocs=default_test.nprocs, 304 | min_nprocs=default_test.min_nprocs, 305 | max_nprocs=default_test.max_nprocs, 306 | run_concurrent=default_test.run_concurrent, 307 | submit_template=default_test.submit_template, 308 | ) 309 | if 'tolerances' in test_dict: 310 | test['tolerances'].update(test_dict['tolerances']) 311 | tol = test_dict.pop('tolerances') 312 | test.update(test_dict) 313 | # restore tolerances for next test in the glob. 314 | if tol: 315 | test_dict['tolerances'] = tol 316 | test_info[(name, path)] = [test_program, copy.deepcopy(test)] 317 | 318 | # Now create the tests (after finding out what the input files are). 319 | tests = [] 320 | for ((name, path), (test_program, test_dict)) in test_info.items(): 321 | old_dir = os.getcwd() 322 | os.chdir(path) 323 | # Expand any globs in the input files. 324 | inputs_args = [] 325 | for input_arg in test_dict['inputs_args']: 326 | # Be a little forgiving for the input_args config option. 327 | # If we're given ('input'), then clearly the user meant for the 328 | # args option to be empty. However, literal_eval returns 329 | # a string rather than a tuple in such cases, which causes 330 | # problems. 331 | if isinstance(input_arg, str): 332 | inp = input_arg 333 | arg = '' 334 | elif len(input_arg) == 2: 335 | inp = input_arg[0] 336 | arg = input_arg[1] 337 | else: 338 | inp = input_arg[0] 339 | arg = '' 340 | if inp: 341 | # the test, error and benchmark filenames contain the input 342 | # filename, so we need to filter them out. 343 | inp_files = sorted(glob.glob(inp)) 344 | if not inp_files: 345 | err = 'Cannot find input file %s in %s.' % (inp, path) 346 | warnings.warn(err) 347 | continue 348 | # We use a glob for the input argument to avoid the 349 | # case where the argument is empty and hence a pattern 350 | # such as *.inp also matches files like 351 | # test.out.test_id.inp=x.inp and hence considering 352 | # previous output files to actually be an input file in 353 | # their own right. 354 | test_files = [ 355 | util.testcode_filename(stem[1], '*', '*', arg) 356 | for stem in testcode2._FILESTEM_TUPLE 357 | ] 358 | testcode_files = [] 359 | for tc_file in test_files: 360 | testcode_files.extend(glob.glob(tc_file)) 361 | for inp_file in inp_files: 362 | if inp_file not in testcode_files: 363 | inputs_args.append((inp_file, arg)) 364 | else: 365 | inputs_args.append((inp, arg)) 366 | test_dict['inputs_args'] = tuple(inputs_args) 367 | os.chdir(old_dir) 368 | # Create test. 369 | if test_dict['run_concurrent']: 370 | for input_arg in test_dict['inputs_args']: 371 | test_dict['inputs_args'] = (input_arg,) 372 | tests.append(testcode2.Test(name, test_program, path, 373 | **test_dict)) 374 | else: 375 | tests.append(testcode2.Test(name, test_program, path, **test_dict)) 376 | 377 | return (tests, test_categories) 378 | 379 | def get_unique_test_id(tests, reuse_id=False, date_fmt='%d%m%Y'): 380 | '''Find a unique test id based upon the date and previously run tests.''' 381 | todays_id = time.strftime(date_fmt) 382 | newest_file = None 383 | test_id = '0'*len(todays_id) 384 | for test in tests: 385 | test_globs = glob.glob('%s*' % 386 | os.path.join(test.path, testcode2.FILESTEM['test']) 387 | ) 388 | for test_file in test_globs: 389 | if (not newest_file or 390 | os.stat(test_file)[-2] > os.stat(newest_file)[-2]): 391 | newest_file = test_file 392 | # keep track of the latest file with today's test_id (in case 393 | # the most recent test was run with a user-specified test_id). 394 | newest_test_id = util.testcode_file_id( 395 | newest_file, testcode2.FILESTEM['test'] 396 | ) 397 | if newest_test_id[:len(todays_id)] == todays_id: 398 | test_id = newest_test_id 399 | if reuse_id: 400 | # Want test_id to be the most recent set of tests. 401 | if not newest_file: 402 | err = 'Cannot find any previous test outputs.' 403 | raise exceptions.TestCodeError(err) 404 | test_id = util.testcode_file_id(newest_file, testcode2.FILESTEM['test']) 405 | elif test_id[:len(todays_id)] == todays_id: 406 | # Have run at more than one test today already. Create unique id. 407 | if len(test_id) == len(todays_id): 408 | test_id = 1 409 | else: 410 | test_id = int(test_id[len(todays_id)+1:]) + 1 411 | test_id = '%s-%s' % (todays_id, test_id) 412 | else: 413 | # First test of the day! 414 | test_id = todays_id 415 | return test_id 416 | 417 | def select_tests(all_tests, test_categories, selected_categories, prefix=''): 418 | '''Return the set of tests contained by the selected test categories.''' 419 | test_categories['_all_'] = [test.path for test in all_tests] 420 | if ('_default_' in selected_categories 421 | and '_default_' not in test_categories): 422 | selected_categories = ['_all_'] 423 | # Recursively expand job categories. 424 | while compat.compat_any( 425 | cat in test_categories for cat in selected_categories 426 | ): 427 | tmp = [] 428 | for cat in selected_categories: 429 | if cat in test_categories: 430 | tmp.extend(test_categories[cat]) 431 | else: 432 | # cat has been fully expanded and now refers to a test 433 | # contained within the directory named cat. 434 | tmp.append(cat) 435 | selected_categories = tmp 436 | # Select tests to run. 437 | tests = [] 438 | parent = lambda pdir, cdir: \ 439 | not os.path.relpath(cdir, start=pdir).startswith(os.pardir) 440 | for cat in compat.compat_set(selected_categories): 441 | # test paths are relative to the config directory but absolute paths 442 | # are stored . 443 | found = False 444 | cat_paths = glob.glob(os.path.join(prefix, cat)) 445 | for test in all_tests: 446 | if cat == test.name: 447 | found = True 448 | tests.append(test) 449 | elif compat.compat_any(os.path.exists(path) and 450 | os.path.samefile(path, test.path) for path in cat_paths): 451 | found = True 452 | tests.append(test) 453 | elif compat.compat_any(parent(path, test.path) 454 | for path in cat_paths): 455 | # test contained within a subdirectory of a cat_path. 456 | found = True 457 | tests.append(test) 458 | if not found: 459 | print('WARNING: %s test/category not found.\n' % cat) 460 | # Only want to run each test once. 461 | tests = list(compat.compat_set(tests)) 462 | return tests 463 | 464 | def set_program_name(program, relative_path): 465 | '''Set a full path to the given program. 466 | 467 | If the program exists on PATH, then return the full path to that program. 468 | Otherwise, assume program is given relative to relative_path and hence return 469 | the full path. 470 | ''' 471 | program_path = os.path.join(relative_path, program) 472 | program_path = os.path.expandvars(program_path) 473 | if not os.path.exists(program_path): 474 | # Program not supplied as a relative or full path. 475 | # Does program exist on the user's path? 476 | which_popen = subprocess.Popen(['which', program], 477 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) 478 | which_popen.wait() 479 | if which_popen.returncode == 0: 480 | # Program is on user's path. 481 | # Return full path to program. 482 | program_path = which_popen.communicate()[0].decode('utf-8').strip() 483 | else: 484 | # Cannot find program. 485 | # This still allows us to manipulate previously run tests, just not 486 | # run new ones... 487 | print('WARNING: cannot find program: %s.' % (program)) 488 | # Allow things to proceed with the original path -- the user might 489 | # know what they're doing and the above tests are not always 490 | # sufficient (e.g. if using cygwin but using an MPI implementation 491 | # which requires a Windows-based path). 492 | program_path = program 493 | 494 | return program_path 495 | -------------------------------------------------------------------------------- /lib/testcode2/dir_lock.py: -------------------------------------------------------------------------------- 1 | ''' 2 | testcode2.dir_lock 3 | ------------------ 4 | 5 | Threading lock initialisation and helper. 6 | 7 | :copyright: (c) 2012 James Spencer. 8 | :license: modified BSD; see LICENSE for more details. 9 | ''' 10 | 11 | import os 12 | import threading 13 | import testcode2.compatibility as compat 14 | 15 | class DirLock: 16 | '''Helper class for working with threading locks.''' 17 | def __init__(self): 18 | self.lock = threading.Lock() 19 | def with_lock(self, func): 20 | '''Decorate function to be executed whilst holding the lock. 21 | 22 | :param function func: arbitary function. 23 | ''' 24 | @compat.functools.wraps(func) 25 | def decorated_func(*args, **kwargs): 26 | '''Function decorated by Lock.with_lock.''' 27 | self.lock.acquire() 28 | try: 29 | return func(*args, **kwargs) 30 | finally: 31 | self.lock.release() 32 | return decorated_func 33 | def in_dir(self, ddir): 34 | '''Decorate function so it is executed in the given directory ddir. 35 | 36 | The thread executing the function holds the lock whilst entering ddir and 37 | executing the function. This makes such actions thread-safe with respect to 38 | the directory location but is not appropriate for computationally-demanding 39 | functions. 40 | 41 | :param string ddir: directory in which the decorated function is executed. 42 | ''' 43 | # Because we wish to use this as a decorator with arguments passed to 44 | # the decorator, we must return a wrapper function which in turn 45 | # returns the decorated function. See the excellent explanation of 46 | # decorators at: http://stackoverflow.com/a/1594484 47 | def wrapper(func): 48 | '''Wrap func to hold lock whilst being executed in ddir. 49 | 50 | :param string func: arbitrary function. 51 | ''' 52 | @compat.functools.wraps(func) 53 | @self.with_lock 54 | def decorated_func(*args, **kwargs): 55 | '''Function decorated by Lock.in_dir.''' 56 | cwd = os.getcwd() 57 | os.chdir(ddir) 58 | try: 59 | val = func(*args, **kwargs) 60 | except Exception: 61 | # func has raised an error. Return to the original 62 | # directory and then re-raise the error to allow the caller 63 | # to handle it. 64 | os.chdir(cwd) 65 | raise 66 | os.chdir(cwd) 67 | return val 68 | return decorated_func 69 | return wrapper 70 | -------------------------------------------------------------------------------- /lib/testcode2/exceptions.py: -------------------------------------------------------------------------------- 1 | ''' 2 | testcode2.exceptions 3 | -------------------- 4 | 5 | Custom exceptions. Initialise signal handler for the interrupt signal. 6 | 7 | :copyright: (c) 2012 James Spencer. 8 | :license: modified BSD; see LICENSE for more details. 9 | ''' 10 | 11 | import signal 12 | import sys 13 | 14 | def signal_handler(sig, frame): 15 | '''Capture signal and leave quietly.''' 16 | print('Signal: %s has been caught. Bye!' % (sig)) 17 | sys.exit(1) 18 | 19 | 20 | class RunError(Exception): 21 | '''Exception used for errors running test jobs.''' 22 | pass 23 | 24 | 25 | class AnalysisError(Exception): 26 | '''Exception used for errors running test jobs.''' 27 | pass 28 | 29 | 30 | class TestCodeError(Exception): 31 | '''Top level exception for testcode errors.''' 32 | pass 33 | 34 | signal.signal(signal.SIGINT, signal_handler) # Listen out for Ctrl-C. 35 | -------------------------------------------------------------------------------- /lib/testcode2/queues.py: -------------------------------------------------------------------------------- 1 | ''' 2 | testcode.queues 3 | --------------- 4 | 5 | Access to external queueing systems. 6 | 7 | :copyright: (c) 2012 James Spencer. 8 | :license: modified BSD; see LICENSE for more details. 9 | ''' 10 | 11 | import os.path 12 | import subprocess 13 | import sys 14 | import time 15 | 16 | import testcode2.exceptions as exceptions 17 | 18 | class ClusterQueueJob: 19 | '''Interface to external queueing system. 20 | 21 | :param string submit_file: filename of submit script to be submitted to the 22 | queueing system. 23 | :param string system: name of queueing system. Currently only an interface to 24 | PBS is implemented. 25 | ''' 26 | def __init__(self, submit_file, system='PBS'): 27 | self.job_id = None 28 | self.submit_file = submit_file 29 | self.system = system 30 | if self.system == 'PBS': 31 | self.submit_cmd = 'qsub' 32 | self.queue_cmd = 'qstat' 33 | self.job_id_column = 0 34 | self.status_column = 4 35 | self.finished_status = 'C' 36 | else: 37 | err = 'Queueing system not implemented: %s' % self.system 38 | raise exceptions.RunError(err) 39 | def create_submit_file(self, pattern, string, template): 40 | '''Create a submit file. 41 | 42 | Replace pattern in the template file with string and place the result in 43 | self.submit_file. 44 | 45 | :param string pattern: string in template to be replaced. 46 | :param string string: string to replace pattern in template. 47 | :param string template: filename of file containing the template submit script. 48 | ''' 49 | # get template 50 | if not os.path.exists(template): 51 | err = 'Submit file template does not exist: %s.' % (template,) 52 | raise exceptions.RunError(err) 53 | ftemplate = open(template) 54 | submit = ftemplate.read() 55 | ftemplate.close() 56 | # replace marker with our commands 57 | submit = submit.replace(pattern, string) 58 | # write to submit script 59 | fsubmit = open(self.submit_file, 'w') 60 | fsubmit.write(submit) 61 | fsubmit.close() 62 | def start_job(self): 63 | '''Submit job to cluster queue.''' 64 | submit_cmd = [self.submit_cmd, self.submit_file] 65 | try: 66 | submit_popen = subprocess.Popen(submit_cmd, stdout=subprocess.PIPE, 67 | stderr=subprocess.STDOUT) 68 | submit_popen.wait() 69 | self.job_id = submit_popen.communicate()[0].strip().decode('utf-8') 70 | except OSError: 71 | # 'odd' syntax so exceptions work with python 2.5 and python 2.6/3. 72 | err = 'Error submitting job: %s' % (sys.exc_info()[1],) 73 | raise exceptions.RunError(err) 74 | def wait(self): 75 | '''Returns when job has finished running on the cluster.''' 76 | running = True 77 | # Don't ask the queueing system for the job itself but rather parse the 78 | # output from all current jobs and look gor the job in question. 79 | # This works around the problem where the job_id is not a sufficient 80 | # handle to query the system directly (e.g. on the CMTH cluster). 81 | qstat_cmd = [self.queue_cmd] 82 | while running: 83 | time.sleep(15) 84 | qstat_popen = subprocess.Popen(qstat_cmd, stdout=subprocess.PIPE, 85 | stderr=subprocess.PIPE) 86 | qstat_popen.wait() 87 | if qstat_popen.returncode != 0: 88 | err = ('Error inspecting queue system: %s' % 89 | qstat_popen.communicate()) 90 | raise exceptions.RunError(err) 91 | qstat_out = qstat_popen.communicate()[0] 92 | # Assume job has finished unless it appears in the qstat output. 93 | running = False 94 | for line in qstat_out.splitlines(): 95 | words = line.split() 96 | if words[self.job_id_column] == self.job_id: 97 | running = words[self.status_column] != self.finished_status 98 | break 99 | -------------------------------------------------------------------------------- /lib/testcode2/util.py: -------------------------------------------------------------------------------- 1 | ''' 2 | testcode2.util 3 | -------------- 4 | 5 | Utility functions. 6 | 7 | :copyright: (c) 2012 James Spencer. 8 | :license: modified BSD; see LICENSE for more details. 9 | ''' 10 | 11 | import os.path 12 | import re 13 | import sys 14 | 15 | import testcode2.compatibility as compat 16 | import testcode2.exceptions as exceptions 17 | 18 | def testcode_filename(stem, file_id, inp, args): 19 | '''Construct filename in testcode format.''' 20 | filename = '%s.%s' % (stem, file_id) 21 | if inp: 22 | filename = '%s.inp=%s' % (filename, inp) 23 | if args: 24 | filename = '%s.args=%s' % (filename, args) 25 | filename = filename.replace(' ','_') 26 | filename = filename.replace('/', '_') 27 | return filename 28 | 29 | def testcode_file_id(filename, stem): 30 | '''Extract the file_id from a filename in the testcode format.''' 31 | filename = os.path.basename(filename) 32 | file_id = filename.replace('%s.' % (stem), '') 33 | file_id = re.sub(r'\.inp=.*', '', file_id) 34 | file_id = re.sub(r'\.args=.*', '', file_id) 35 | return file_id 36 | 37 | 38 | def try_floatify(val): 39 | '''Convert val to a float if possible.''' 40 | try: 41 | return float(val) 42 | except ValueError: 43 | return val 44 | 45 | def extract_tagged_data(data_tag, filename): 46 | '''Extract data from lines marked by the data_tag in filename.''' 47 | if not os.path.exists(filename): 48 | err = 'Cannot extract data: file %s does not exist.' % (filename) 49 | raise exceptions.AnalysisError(err) 50 | data_file = open(filename) 51 | # Data tag is the first non-space character in the line. 52 | # e.g. extract data from lines: 53 | # data_tag Energy: 1.256743 a.u. 54 | data_tag_regex = re.compile('^ *%s' % (re.escape(data_tag))) 55 | data = {} 56 | for line in data_file.readlines(): 57 | if data_tag_regex.match(line): 58 | # This is a line containing info to be tested. 59 | words = line.split() 60 | key = [] 61 | # name of data is string after the data_tag and preceeding the 62 | # (numerical) data. only use the first number in the line, with 63 | # the key taken from all proceeding information. 64 | for word in words[1:]: 65 | val = try_floatify(word) 66 | if val != word: 67 | break 68 | else: 69 | key.append(word) 70 | if key[-1] in ("=",':'): 71 | key.pop() 72 | key = '_'.join(key) 73 | if key[-1] in ("=",':'): 74 | key = key[:-1] 75 | if not key: 76 | key = 'data' 77 | if key in data: 78 | data[key].append(val) 79 | else: 80 | data[key] = [val] 81 | # We shouldn't change the data from this point: convert entries to tuples. 82 | for (key, val) in data.items(): 83 | data[key] = tuple(val) 84 | return data 85 | 86 | def dict_table_string(table_string): 87 | '''Read a data table from a string into a dictionary. 88 | 89 | The first row and any subsequent rows containing no numbers are assumed to form 90 | headers of a subtable, and so form the keys for the subsequent subtable. 91 | 92 | Values, where possible, are converted to floats. 93 | 94 | e.g. a b c a -> {'a':(1,4,7,8), 'b':(2,5), 'c':(3,6)} 95 | 1 2 3 7 96 | 4 5 6 8 97 | and 98 | a b c -> {'a':(1,4,7), 'b':(2,5,8), 'c':(3,6), 'd':(9), 'e':(6)} 99 | 1 2 3 100 | 4 5 6 101 | a b d e 102 | 7 8 9 6 103 | ''' 104 | data = [i.split() for i in table_string.splitlines()] 105 | # Convert to numbers where appropriate 106 | data = [[try_floatify(val) for val in dline] for dline in data] 107 | data_dict = {} 108 | head = [] 109 | for dline in data: 110 | # Test if all items are strings; if so start a new subtable. 111 | # We actually test if all items are not floats, as python 3 can return 112 | # a bytes variable from subprocess whereas (e.g.) python 2.4 returns a 113 | # str. Testing for this is problematic as the bytes type does not 114 | # exist in python 2.4. Fortunately we have converted all items to 115 | # floats if possible, so can just test for the inverse condition... 116 | if compat.compat_all(type(val) is not float for val in dline): 117 | # header of new subtable 118 | head = dline 119 | for val in head: 120 | if val not in data_dict: 121 | data_dict[val] = [] 122 | else: 123 | if len(dline) > len(head): 124 | err = 'Table missing column heading(s):\n%s' % (table_string) 125 | raise exceptions.AnalysisError(err) 126 | for (ind, val) in enumerate(dline): 127 | # Add data to appropriate key. 128 | # Note that this handles the case where the same column heading 129 | # occurs multiple times in the same subtable and does not 130 | # overwrite the previous column with the same heading. 131 | data_dict[head[ind]].append(val) 132 | # We shouldn't change the data from this point: convert entries to tuples. 133 | for (key, val) in data_dict.items(): 134 | data_dict[key] = tuple(val) 135 | return data_dict 136 | 137 | def wrap_list_strings(word_list, width): 138 | '''Create a list of strings of a given width from a list of words. 139 | 140 | This is, to some extent, a version of textwrap.wrap but without the 'feature' 141 | of removing additional whitespace.''' 142 | wrapped_strings = [] 143 | clen = 0 144 | cstring = [] 145 | for string in word_list: 146 | if clen + len(string) + len(cstring) <= width: 147 | cstring.append(string) 148 | clen += len(string) 149 | else: 150 | wrapped_strings.append(' '.join(cstring)) 151 | cstring = [string] 152 | clen = len(string) 153 | if cstring: 154 | wrapped_strings.append(' '.join(cstring)) 155 | return wrapped_strings 156 | 157 | 158 | def pretty_print_table(labels, dicts): 159 | '''Print data in dictionaries of identical size in a tabular format.''' 160 | # Fill in the dicts with missing data. 161 | # This can be hit if the missing data fields are ignored... 162 | for dict1 in dicts: 163 | for key in dict1.keys(): 164 | if type(dict1[key]) is tuple or type(dict1[key]) is list: 165 | nitems = len(dict1[key]) 166 | val = ('n/a',)*nitems 167 | iterable = True 168 | else: 169 | val = 'n/a' 170 | iterable = False 171 | for dict2 in dicts: 172 | if key not in dict2: 173 | dict2[key] = val 174 | elif iterable and nitems != len(dict2[key]): 175 | dict2[key] += nitems - len(dict2[key]) 176 | # Loop through all elements in order to calculate the field width. 177 | # Create header line as we go. 178 | fmt = dict(_tc_label='%%-%is' % (max(len(str(label)) for label in labels))) 179 | header = [] 180 | for key in sorted(dicts[0].keys()): 181 | fmt[key] = len(str(key)) 182 | nitems = 1 183 | if type(dicts[0][key]) is tuple or type(dicts[0][key]) is list: 184 | nitems = len(dicts[0][key]) 185 | for dval in dicts: 186 | for item in dval[key]: 187 | fmt[key] = max(fmt[key], len(str(item))) 188 | else: 189 | fmt[key] = max(len(str(dval[key])) for dval in dicts) 190 | fmt[key] = max(fmt[key], len(str(key))) 191 | # Finished processing all data items with this key. 192 | # Covert from field width into a format statement. 193 | fmt[key] = '%%-%is' % (fmt[key]) 194 | for item in range(nitems): 195 | header.append(fmt[key] % (key)) 196 | # Wrap header line and insert key/label at the start of each line. 197 | key = fmt['_tc_label'] % ('') 198 | header = wrap_list_strings(header, 70) 199 | header = ['%s %s' % (key, line_part) for line_part in header] 200 | # Printing without a new line is different in python 2 and python 3, so for 201 | # ease we construct the formatting for the line and then print it. 202 | lines = [ header ] 203 | for (ind, label) in enumerate(labels): 204 | line = [fmt['_tc_label'] % (label)] 205 | line = [] 206 | for key in sorted(dicts[ind].keys()): 207 | if type(dicts[ind][key]) is tuple or type(dicts[ind][key]) is list: 208 | for item in range(len(dicts[ind][key])): 209 | line.append(fmt[key] % (dicts[ind][key][item])) 210 | else: 211 | line.append(fmt[key] % (dicts[ind][key])) 212 | # Wrap line and insert key/label at the start of each line. 213 | key = fmt['_tc_label'] % (label) 214 | line = wrap_list_strings(line, 70) 215 | line = ['%s %s' % (key, line_part) for line_part in line] 216 | lines.extend([line]) 217 | # Now actually form table. Due to line wrapping we might actually form 218 | # several subtables. As each line has the same number of items (or 219 | # should!), this is quite simple. 220 | table = [] 221 | for ind in range(len(lines[0])): 222 | table.append('\n'.join([line[ind] for line in lines])) 223 | table = '\n'.join(table) 224 | return (table or 225 | 'No data for %s.' % ('; '.join(label.strip() for label in labels))) 226 | 227 | def info_line(path, input_file, args, rundir): 228 | '''Produce a (terse) string describing a test.''' 229 | if rundir: 230 | path = compat.relpath(path, rundir) 231 | info_line = path 232 | if input_file: 233 | info_line += ' - %s' % (input_file) 234 | if args: 235 | info_line += ' (arg(s): %s)' % (args) 236 | info_line += ': ' 237 | return info_line 238 | -------------------------------------------------------------------------------- /lib/testcode2/validation.py: -------------------------------------------------------------------------------- 1 | ''' 2 | testcode2.validation 3 | -------------------- 4 | 5 | Classes and functions for comparing data. 6 | 7 | :copyright: (c) 2012 James Spencer. 8 | :license: modified BSD; see LICENSE for more details. 9 | ''' 10 | 11 | import re 12 | import sys 13 | import warnings 14 | 15 | import testcode2.ansi as ansi 16 | import testcode2.compatibility as compat 17 | import testcode2.exceptions as exceptions 18 | 19 | class Status: 20 | '''Enum-esque object for storing whether an object passed a comparison. 21 | 22 | bools: iterable of boolean objects. If all booleans are True (False) then the 23 | status is set to pass (fail) and if only some booleans are True, the 24 | status is set to warning (partial pass). 25 | status: existing status to use. bools is ignored if status is supplied. 26 | name: name of status (unknown, skipped, passed, partial, failed) to use. 27 | Setting name overrides bools and status. 28 | ''' 29 | def __init__(self, bools=None, status=None, name=None): 30 | (self._unknown, self._skipped) = (-2, -1) 31 | (self._passed, self._partial, self._failed) = (0, 1, 2) 32 | if name is not None: 33 | setattr(self, 'status', getattr(self, '_'+name)) 34 | elif status is not None: 35 | self.status = status 36 | elif bools: 37 | if compat.compat_all(bools): 38 | self.status = self._passed 39 | elif compat.compat_any(bools): 40 | self.status = self._partial 41 | else: 42 | self.status = self._failed 43 | else: 44 | self.status = self._unknown 45 | def unknown(self): 46 | '''Return true if stored status is unknown.''' 47 | return self.status == self._unknown 48 | def skipped(self): 49 | '''Return true if stored status is skipped.''' 50 | return self.status == self._skipped 51 | def passed(self): 52 | '''Return true if stored status is passed.''' 53 | return self.status == self._passed 54 | def warning(self): 55 | '''Return true if stored status is a partial pass.''' 56 | return self.status == self._partial 57 | def failed(self): 58 | '''Return true if stored status is failed.''' 59 | return self.status == self._failed 60 | def print_status(self, msg=None, verbose=1, vspace=True): 61 | '''Print status. 62 | 63 | msg: optional message to print out after status. 64 | verbose: 0: suppress all output except for . (for pass), U (for unknown), 65 | W (for warning/partial pass) and F (for fail) without a newline. 66 | 1: print 'Passed', 'Unknown', 'WARNING' or '**FAILED**'. 67 | 2: as for 1 plus print msg (if supplied). 68 | 3: as for 2 plus print a blank line. 69 | vspace: print out extra new line afterwards if verbose > 1. 70 | ''' 71 | if verbose > 0: 72 | if self.status == self._unknown: 73 | print('Unknown.') 74 | elif self.status == self._passed: 75 | print('Passed.') 76 | elif self.status == self._skipped: 77 | print('%s.' % ansi.ansi_format('SKIPPED', 'blue')) 78 | elif self.status == self._partial: 79 | print('%s.' % ansi.ansi_format('WARNING', 'blue')) 80 | else: 81 | print('%s.' % ansi.ansi_format('**FAILED**', 'red', 'normal', 'bold')) 82 | if msg and verbose > 1: 83 | print(msg) 84 | if vspace and verbose > 1: 85 | print('') 86 | else: 87 | if self.status == self._unknown: 88 | sys.stdout.write('U') 89 | elif self.status == self._skipped: 90 | sys.stdout.write('S') 91 | elif self.status == self._passed: 92 | sys.stdout.write('.') 93 | elif self.status == self._partial: 94 | sys.stdout.write('W') 95 | else: 96 | sys.stdout.write('F') 97 | sys.stdout.flush() 98 | def __add__(self, other): 99 | '''Add two status objects. 100 | 101 | Return the maximum level (ie most "failed") status.''' 102 | return Status(status=max(self.status, other.status)) 103 | 104 | class Tolerance: 105 | '''Store absolute and relative tolerances 106 | 107 | Given are regarded as equal if they are within these tolerances. 108 | 109 | name: name of tolerance object. 110 | absolute: threshold for absolute difference between two numbers. 111 | relative: threshold for relative difference between two numbers. 112 | strict: if true, then require numbers to be within both thresholds. 113 | ''' 114 | def __init__(self, name='', absolute=None, relative=None, strict=True): 115 | self.name = name 116 | self.absolute = absolute 117 | self.relative = relative 118 | if not self.absolute and not self.relative: 119 | err = 'Neither absolute nor relative tolerance given.' 120 | raise exceptions.TestCodeError(err) 121 | self.strict = strict 122 | def __repr__(self): 123 | return (self.absolute, self.relative, self.strict).__repr__() 124 | def __hash__(self): 125 | return hash(self.name) 126 | def __eq__(self, other): 127 | return (isinstance(other, self.__class__) and 128 | self.__dict__ == other.__dict__) 129 | def validate(self, test_val, benchmark_val, key=''): 130 | '''Compare test and benchmark values to within the tolerances.''' 131 | status = Status([True]) 132 | msg = ['values are within tolerance.'] 133 | compare = '(Test: %s. Benchmark: %s.)' % (test_val, benchmark_val) 134 | try: 135 | # Check float is not NaN (which we can't compare). 136 | if compat.isnan(test_val) or compat.isnan(benchmark_val): 137 | status = Status([False]) 138 | msg = ['cannot compare NaNs.'] 139 | else: 140 | # Check if values are within tolerances. 141 | (status_absolute, msg_absolute) = \ 142 | self.validate_absolute(benchmark_val, test_val) 143 | (status_relative, msg_relative) = \ 144 | self.validate_relative(benchmark_val, test_val) 145 | if self.absolute and self.relative and not self.strict: 146 | # Require only one of thresholds to be met. 147 | status = Status([status_relative.passed(), 148 | status_absolute.passed()]) 149 | else: 150 | # Only have one or other of thresholds (require active one 151 | # to be met) or have both and strict mode is on (require 152 | # both to be met). 153 | status = status_relative + status_absolute 154 | err_stat = '' 155 | if status.warning(): 156 | err_stat = 'Warning: ' 157 | elif status.failed(): 158 | err_stat = 'ERROR: ' 159 | msg = [] 160 | if self.absolute and msg_absolute: 161 | msg.append('%s%s %s' % (err_stat, msg_absolute, compare)) 162 | if self.relative and msg_relative: 163 | msg.append('%s%s %s' % (err_stat, msg_relative, compare)) 164 | except TypeError: 165 | if test_val != benchmark_val: 166 | # require test and benchmark values to be equal (within python's 167 | # definition of equality). 168 | status = Status([False]) 169 | msg = ['values are different. ' + compare] 170 | if key and msg: 171 | msg.insert(0, key) 172 | msg = '\n '.join(msg) 173 | else: 174 | msg = '\n'.join(msg) 175 | return (status, msg) 176 | 177 | def validate_absolute(self, benchmark_val, test_val): 178 | '''Compare test and benchmark values to the absolute tolerance.''' 179 | if self.absolute: 180 | diff = test_val - benchmark_val 181 | err = abs(diff) 182 | passed = err < self.absolute 183 | msg = '' 184 | if not passed: 185 | msg = ('absolute error %.2e greater than %.2e.' % 186 | (err, self.absolute)) 187 | else: 188 | passed = True 189 | msg = 'No absolute tolerance set. Passing without checking.' 190 | return (Status([passed]), msg) 191 | 192 | def validate_relative(self, benchmark_val, test_val): 193 | '''Compare test and benchmark values to the relative tolerance.''' 194 | if self.relative: 195 | diff = test_val - benchmark_val 196 | if benchmark_val == 0 and diff == 0: 197 | err = 0 198 | elif benchmark_val == 0: 199 | err = float("Inf") 200 | else: 201 | err = abs(diff/benchmark_val) 202 | passed = err < self.relative 203 | msg = '' 204 | if not passed: 205 | msg = ('relative error %.2e greater than %.2e.' % 206 | (err, self.relative)) 207 | else: 208 | passed = True 209 | msg = 'No relative tolerance set. Passing without checking.' 210 | return (Status([passed]), msg) 211 | 212 | 213 | def compare_data(benchmark, test, default_tolerance, tolerances, 214 | ignore_fields=None): 215 | '''Compare two data dictionaries.''' 216 | ignored_params = compat.compat_set(ignore_fields or tuple()) 217 | bench_params = compat.compat_set(benchmark) - ignored_params 218 | test_params = compat.compat_set(test) - ignored_params 219 | # Check both the key names and the number of keys in case there are 220 | # different numbers of duplicate keys. 221 | comparable = (bench_params == test_params) 222 | key_counts = dict((key,0) for key in bench_params | test_params) 223 | for (key, val) in benchmark.items(): 224 | if key not in ignored_params: 225 | key_counts[key] += len(val) 226 | for (key, val) in test.items(): 227 | if key not in ignored_params: 228 | key_counts[key] -= len(val) 229 | comparable = comparable and compat.compat_all(kc == 0 for kc in key_counts.values()) 230 | status = Status() 231 | msg = [] 232 | 233 | if not comparable: 234 | status = Status([False]) 235 | bench_only = bench_params - test_params 236 | test_only = test_params - bench_params 237 | msg.append('Different sets of data extracted from benchmark and test.') 238 | if bench_only: 239 | msg.append(" Data only in benchmark: %s." % ", ".join(bench_only)) 240 | if test_only: 241 | msg.append(" Data only in test: %s." % ", ".join(test_only)) 242 | bench_more = [key for key in key_counts 243 | if key_counts[key] > 0 and key not in bench_only] 244 | test_more = [key for key in key_counts 245 | if key_counts[key] < 0 and key not in test_only] 246 | if bench_more: 247 | msg.append(" More data in benchmark than in test: %s." % 248 | ", ".join(bench_more)) 249 | if test_more: 250 | msg.append(" More data in test than in benchmark: %s." % 251 | ", ".join(test_more)) 252 | 253 | for param in (bench_params & test_params): 254 | param_tol = tolerances.get(param, default_tolerance) 255 | if param_tol == default_tolerance: 256 | # See if there's a regex that matches. 257 | tol_matches = [tol for tol in tolerances.values() 258 | if tol.name and re.match(tol.name, param)] 259 | if tol_matches: 260 | param_tol = tol_matches[0] 261 | if len(tol_matches) > 1: 262 | warnings.warn('Multiple tolerance regexes match. ' 263 | 'Using %s.' % (param_tol.name)) 264 | for bench_value, test_value in zip(benchmark[param], test[param]): 265 | key_status, err = param_tol.validate(test_value, bench_value, param) 266 | status += key_status 267 | if not key_status.passed() and err: 268 | msg.append(err) 269 | 270 | return (comparable, status, "\n".join(msg)) 271 | -------------------------------------------------------------------------------- /lib/testcode2/vcs.py: -------------------------------------------------------------------------------- 1 | ''' 2 | testcode2.vcs 3 | ------------- 4 | 5 | Lightweight access to required version control system functions. 6 | 7 | :copyright: (c) 2012 James Spencer. 8 | :license: modified BSD; see LICENSE for more details. 9 | ''' 10 | 11 | import os 12 | import subprocess 13 | 14 | class VCSRepository(object): 15 | '''Handle information about a version control repository. 16 | 17 | vcs: version control system used. Currently git, mercurial and subversion are supported. 18 | repository: (local) directory containing a checked-out version of the repository. 19 | remote_repository: remote location of the repository. 20 | ''' 21 | def __init__(self, vcs, repository, remote_repository=None): 22 | if vcs in ['svn', 'git', 'hg']: 23 | self.vcs = vcs 24 | else: 25 | self.vcs = None 26 | self.repository = repository 27 | if remote_repository: 28 | self.remote_repository = remote_repository 29 | 30 | def get_code_id(self): 31 | '''Return the id (i.e. version number or hash) of the VCS repository.''' 32 | old_dir = os.getcwd() 33 | os.chdir(self.repository) 34 | code_id = 'UNKNOWN' 35 | id_popen = None 36 | if self.vcs == 'svn': 37 | id_popen = subprocess.Popen(['svnversion', '.'], 38 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) 39 | elif self.vcs == 'git': 40 | id_popen = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'], 41 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) 42 | elif self.vcs == 'hg': 43 | id_popen = subprocess.Popen(['hg', 'id', '-i'], 44 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) 45 | if id_popen: 46 | id_popen.wait() 47 | code_id = id_popen.communicate()[0].decode('utf-8').strip() 48 | os.chdir(old_dir) 49 | return (code_id) 50 | --------------------------------------------------------------------------------