├── nim.cfg ├── tests ├── threads.test ├── tunits.test ├── hello │ ├── hello.test │ ├── hello_size.test │ └── hello_multiple.test ├── threads.nim ├── hello.nim └── tunits.nim ├── ntu.nim.cfg ├── testutils.nim ├── .editorconfig ├── .gitignore ├── config.nims ├── testutils ├── nimbletasks.nim ├── unittests.nim ├── moduletests.nim ├── markdown_reports.nim ├── helpers.nim ├── fuzzing.nim ├── fuzzing │ ├── fuzzing_on_windows.md │ └── readme.md ├── fuzzing_engines.nim ├── readme.md ├── config.nim └── spec.nim ├── .github └── workflows │ └── ci.yml ├── scripts └── install_honggfuzz.sh ├── README.md ├── testutils.nimble └── ntu.nim /nim.cfg: -------------------------------------------------------------------------------- 1 | -d:nimOldCaseObjects 2 | -------------------------------------------------------------------------------- /tests/threads.test: -------------------------------------------------------------------------------- 1 | program = "threads" 2 | -------------------------------------------------------------------------------- /tests/tunits.test: -------------------------------------------------------------------------------- 1 | program = "tunits" 2 | -------------------------------------------------------------------------------- /ntu.nim.cfg: -------------------------------------------------------------------------------- 1 | --threads:on 2 | --path="$config" 3 | -------------------------------------------------------------------------------- /testutils.nim: -------------------------------------------------------------------------------- 1 | import 2 | testutils/unittests 3 | 4 | export 5 | unittests 6 | 7 | -------------------------------------------------------------------------------- /tests/hello/hello.test: -------------------------------------------------------------------------------- 1 | program = "../hello" 2 | [Output] 3 | stdout = "hello world\n" 4 | -------------------------------------------------------------------------------- /tests/threads.nim: -------------------------------------------------------------------------------- 1 | when compileOption("threads"): 2 | quit(0) 3 | else: 4 | quit(1) 5 | -------------------------------------------------------------------------------- /tests/hello/hello_size.test: -------------------------------------------------------------------------------- 1 | program = "../hello" 2 | max_size = 800000 3 | release 4 | --opt:size 5 | os = "linux,macosx" 6 | -------------------------------------------------------------------------------- /tests/hello.nim: -------------------------------------------------------------------------------- 1 | import std/os 2 | 3 | if paramCount() == 1: 4 | echo "hello ", paramStr(1) 5 | else: 6 | echo "hello world" 7 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | indent_style = space 3 | insert_final_newline = true 4 | indent_size = 2 5 | trim_trailing_whitespace = true 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # ignore all executable files 2 | * 3 | !*.* 4 | !*/ 5 | *.exe 6 | 7 | nimcache/ 8 | nimble.paths 9 | nimble.develop 10 | -------------------------------------------------------------------------------- /config.nims: -------------------------------------------------------------------------------- 1 | # begin Nimble config (version 2) 2 | when withDir(thisDir(), system.fileExists("nimble.paths")): 3 | include "nimble.paths" 4 | # end Nimble config 5 | -------------------------------------------------------------------------------- /testutils/nimbletasks.nim: -------------------------------------------------------------------------------- 1 | template addTestutilsTasks* = 2 | task moduleTests, "Run all module tests": 3 | let (files, errCode) = gorgeEx("git grep -l 'tests:'") 4 | echo files 5 | 6 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | workflow_dispatch: 8 | 9 | jobs: 10 | build: 11 | uses: status-im/nimbus-common-workflow/.github/workflows/common.yml@main 12 | -------------------------------------------------------------------------------- /tests/tunits.nim: -------------------------------------------------------------------------------- 1 | import unittest2 2 | 3 | suite "goats": 4 | test "pigs": 5 | echo "oink" 6 | check true 7 | 8 | test "horses": 9 | expect ValueError: 10 | echo "ney" 11 | raise newException(ValueError, "you made an error") 12 | -------------------------------------------------------------------------------- /tests/hello/hello_multiple.test: -------------------------------------------------------------------------------- 1 | program = "../hello" 2 | 3 | # my aim is true 4 | [Output] 5 | stdout = "hello world\n" 6 | 7 | # option 2 8 | [Output_larry_is_a_good_boy] 9 | args = "larry" 10 | stdout = "hello larry\n" 11 | 12 | # option 47 13 | [Output_stevie_is_a_good_boy] 14 | args = "stephen" 15 | stdout = "hello stephen\n" 16 | -------------------------------------------------------------------------------- /testutils/unittests.nim: -------------------------------------------------------------------------------- 1 | import 2 | unittest2 3 | 4 | export 5 | unittest2 6 | 7 | template procSuite*(name: string, body: untyped) = 8 | proc suitePayload = 9 | suite name, body 10 | 11 | suitePayload() 12 | 13 | template asyncTest*(name, body: untyped) = 14 | test name: 15 | proc scenario {.async.} = body 16 | waitFor scenario() 17 | 18 | -------------------------------------------------------------------------------- /scripts/install_honggfuzz.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eu 4 | 5 | sudo apt-get update 6 | sudo apt-get install binutils-dev 7 | sudo apt-get install libunwind8-dev 8 | 9 | git clone https://github.com/google/honggfuzz.git /tmp/honggfuzz 10 | 11 | pushd /tmp/honggfuzz 12 | make 13 | sudo make install DESTDIR=/opt/honggfuzz 14 | popd 15 | 16 | rm -rf /tmp/honggfuzz 17 | 18 | -------------------------------------------------------------------------------- /testutils/moduletests.nim: -------------------------------------------------------------------------------- 1 | {.used.} 2 | 3 | template tests*(body: untyped) = 4 | template payload = 5 | when not declared(unittest2): 6 | import unittest2 7 | 8 | body 9 | 10 | when defined(testutils_test_build): 11 | payload() 12 | else: 13 | when not compiles(payload()): 14 | payload() 15 | 16 | template programMain*(body: untyped) {.dirty.} = 17 | proc main {.raises: [CatchableError].} = 18 | body 19 | 20 | when isMainModule and not defined(testutils_test_build): 21 | main() 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Testutils 2 | 3 | Testutils now is a home to: 4 | 5 | * [Testrunner](testutils/readme.md) 6 | [![Build Status](https://travis-ci.org/status-im/nim-testutils.svg?branch=master)](https://travis-ci.org/status-im/nim-testutils) 7 | [![Build status](https://ci.appveyor.com/api/projects/status/ayqsnuvcpwo2nh6m/branch/master?svg=true)](https://ci.appveyor.com/project/nimbus/nim-testutils/branch/master) 8 | * [Fuzzing](testutils/fuzzing/readme.md) 9 | * [Fuzzing on Windows](testutils/fuzzing/fuzzing_on_windows.md) 10 | 11 | ## License 12 | Apache2 or MIT 13 | -------------------------------------------------------------------------------- /testutils.nimble: -------------------------------------------------------------------------------- 1 | mode = ScriptMode.Verbose 2 | 3 | packageName = "testutils" 4 | version = "0.8.0" 5 | author = "Status Research & Development GmbH" 6 | description = "A unittest framework" 7 | license = "Apache License 2.0" 8 | skipDirs = @["tests"] 9 | bin = @["ntu"] 10 | #srcDir = "testutils" 11 | 12 | requires "nim >= 1.6.0", 13 | "unittest2" 14 | 15 | proc execCmd(cmd: string) = 16 | echo "execCmd: " & cmd 17 | exec cmd 18 | 19 | proc execTest(test: string) = 20 | let test = "ntu test " & test 21 | execCmd "nim c --mm:refc -f -r " & test 22 | execCmd "nim c --mm:refc -d:release -r " & test 23 | execCmd "nim c --mm:refc -d:danger -r " & test 24 | execCmd "nim cpp --mm:refc -r " & test 25 | execCmd "nim cpp --mm:refc -d:danger -r " & test 26 | if (NimMajor, NimMinor) > (1, 6): 27 | execCmd "nim c --mm:orc -f -r " & test 28 | execCmd "nim c --mm:orc -d:release -r " & test 29 | execCmd "nim c --mm:orc -d:danger -r " & test 30 | execCmd "nim cpp --mm:orc -r " & test 31 | execCmd "nim cpp --mm:orc -d:danger -r " & test 32 | 33 | execCmd "nim c --gc:arc --exceptions:goto -r " & test 34 | when false: 35 | # we disable gc:arc test here because Nim cgen 36 | # generate something not acceptable for clang 37 | # and failed on windows 64 bit too 38 | # TODO https://github.com/nim-lang/Nim/issues/22101 39 | execCmd "nim cpp --gc:arc --exceptions:goto -r " & test 40 | 41 | task test, "run tests for travis": 42 | execTest("tests") 43 | -------------------------------------------------------------------------------- /testutils/markdown_reports.nim: -------------------------------------------------------------------------------- 1 | import algorithm, sequtils, strutils, strformat, tables 2 | 3 | type 4 | Status* {.pure.} = enum OK, Fail, Skip 5 | 6 | proc generateReport*(title: string; data: OrderedTable[string, OrderedTable[string, Status]]; 7 | width = 63; withTotals = true) = 8 | ## Generate a markdown report from test data and write it to a file with the given title. 9 | ## The table keys are sections, and the nested tables map tests to statuses. 10 | let symbol: array[Status, string] = ["+", "-", " "] 11 | var raw = "" 12 | var okCountTotal = 0 13 | var failCountTotal = 0 14 | var skipCountTotal = 0 15 | raw.add(title & "\n") 16 | raw.add("===\n") 17 | for section, statuses in data: 18 | raw.add("## " & section & "\n") 19 | raw.add("```diff\n") 20 | var sortedStatuses = statuses 21 | sortedStatuses.sort do (a: (string, Status), b: (string, Status)) -> int: 22 | cmp(a[0], b[0]) 23 | var okCount = 0 24 | var failCount = 0 25 | var skipCount = 0 26 | for name, final in sortedStatuses: 27 | let padded = alignLeft(name, width) 28 | raw.add(&"{symbol[final]} {padded[0 ..< width]} {$final}\n") 29 | case final 30 | of Status.OK: okCount += 1 31 | of Status.Fail: failCount += 1 32 | of Status.Skip: skipCount += 1 33 | raw.add("```\n") 34 | if withTotals: 35 | let sum = okCount + failCount + skipCount 36 | okCountTotal += okCount 37 | failCountTotal += failCount 38 | skipCountTotal += skipCount 39 | raw.add("OK: $1/$4 Fail: $2/$4 Skip: $3/$4\n" % [$okCount, $failCount, $skipCount, $sum]) 40 | 41 | if withTotals: 42 | let sumTotal = okCountTotal + failCountTotal + skipCountTotal 43 | raw.add("\n---TOTAL---\n") 44 | raw.add("OK: $1/$4 Fail: $2/$4 Skip: $3/$4\n" % [$okCountTotal, $failCountTotal, 45 | $skipCountTotal, $sumTotal]) 46 | writeFile(title & ".md", raw) 47 | -------------------------------------------------------------------------------- /testutils/helpers.nim: -------------------------------------------------------------------------------- 1 | import std/os 2 | import std/osproc 3 | import std/strutils 4 | import std/streams 5 | import std/pegs 6 | 7 | type 8 | CompileInfo* = object 9 | templFile*: string 10 | errorFile*: string 11 | errorLine*, errorColumn*: int 12 | templLine*, templColumn*: int 13 | msg*: string 14 | fullMsg*: string 15 | compileTime*: float 16 | exitCode*: int 17 | 18 | let 19 | # Error pegs, taken from testament tester 20 | pegLineTemplate = 21 | peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' 'template/generic instantiation from here'.*" 22 | pegLineError = 23 | peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' ('Error') ':' \s* {.*}" 24 | pegOtherError = peg"'Error:' \s* {.*}" 25 | pegError = pegLineError / pegOtherError 26 | pegSuccess = peg"'Hint: operation successful' {[^;]*} '; ' {\d+} '.' {\d+} .*" 27 | 28 | # Timestamp pegs 29 | # peg for unix timestamp, basically any float with 6 digits after the decimal 30 | # Not ideal - could also improve by checking for the location in the line 31 | pegUnixTimestamp = peg"{\d+} '.' {\d\d\d\d\d\d} \s" 32 | # peg for timestamp with format yyyy-MM-dd HH:mm:sszzz 33 | pegRfcTimestamp = peg"{\d\d\d\d} '-' {\d\d} '-' {\d\d} ' ' {\d\d} ':' {\d\d} ':' {\d\d} {'+' / '-'} {\d\d} ':' {\d\d} \s" 34 | # Thread/process id is unpredictable.. 35 | pegXid* = peg"""'tid' (('=') / ('":') / (': ') / (': ') / ('=') / ('>')) \d+""" 36 | 37 | proc cmpIgnorePegs*(a, b: string, pegs: varargs[Peg]): bool = 38 | ## true when input strings are equal without regard to supplied pegs 39 | var 40 | aa = a 41 | bb = b 42 | for peg in pegs: 43 | aa = aa.replace(peg, "dummy") 44 | bb = bb.replace(peg, "dummy") 45 | result = aa == bb 46 | 47 | proc cmpIgnoreTimestamp*(a, b: string, timestamp = ""): bool = 48 | ## true when input strings are equal without regard to supplied timestamp form 49 | if timestamp.len == 0: 50 | result = cmpIgnorePegs(a, b, pegXid) 51 | elif timestamp == "RfcTime": 52 | result = cmpIgnorePegs(a, b, pegRfcTimestamp, pegXid) 53 | elif timestamp == "UnixTime": 54 | result = cmpIgnorePegs(a, b, pegUnixTimestamp, pegXid) 55 | 56 | proc cmpIgnoreDefaultTimestamps*(a, b: string): bool = 57 | ## true when input strings are equal without regard to timestamp 58 | if cmpIgnorePegs(a, b, pegRfcTimestamp, pegXid): 59 | result = true 60 | elif cmpIgnorePegs(a, b, pegUnixTimestamp, pegXid): 61 | result = true 62 | 63 | proc parseCompileStream*(p: Process, output: Stream): CompileInfo = 64 | ## parsing compiler output (based on testament tester) 65 | result.exitCode = -1 66 | var 67 | line = newStringOfCap(120) 68 | suc, err, tmpl = "" 69 | 70 | while true: 71 | if output.readLine(line): 72 | if line =~ pegError: 73 | # `err` should contain the last error/warning message 74 | err = line 75 | elif line =~ pegLineTemplate and err == "": 76 | # `tmpl` contains the last template expansion before the error 77 | tmpl = line 78 | elif line =~ pegSuccess: 79 | suc = line 80 | 81 | if err != "": 82 | result.fullMsg.add(line.string & "\p") 83 | else: 84 | result.exitCode = peekExitCode(p) 85 | if result.exitCode != -1: 86 | break 87 | 88 | if tmpl =~ pegLineTemplate: 89 | result.templFile = extractFilename(matches[0]) 90 | result.templLine = parseInt(matches[1]) 91 | result.templColumn = parseInt(matches[2]) 92 | if err =~ pegLineError: 93 | result.errorFile = extractFilename(matches[0]) 94 | result.errorLine = parseInt(matches[1]) 95 | result.errorColumn = parseInt(matches[2]) 96 | result.msg = matches[3] 97 | elif err =~ pegOtherError: 98 | result.msg = matches[0] 99 | elif suc =~ pegSuccess: 100 | result.msg = suc 101 | result.compileTime = parseFloat(matches[1] & "." & matches[2]) 102 | 103 | proc parseExecuteOutput*() = discard 104 | -------------------------------------------------------------------------------- /testutils/fuzzing.nim: -------------------------------------------------------------------------------- 1 | import os, streams, strutils, chronicles, macros, stew/ptrops 2 | 3 | when not defined(windows): 4 | import posix 5 | 6 | # if user forget to import chronicles 7 | # they still can compile without mysterious 8 | # error such as "undeclared identifier: 'activeChroniclesStream'" 9 | export chronicles 10 | 11 | proc suicide() = 12 | # For code we want to fuzz, SIGSEGV is needed on unwanted exceptions. 13 | # However, this is only needed when fuzzing with afl. 14 | when not defined(windows): 15 | discard kill(getpid(), SIGSEGV) 16 | else: 17 | discard 18 | 19 | template fuzz(body) = 20 | when defined(llvmFuzzer): 21 | body 22 | else: 23 | try: 24 | body 25 | except Exception as e: 26 | error "Fuzzer input created exception", exception=e.name, trace=e.repr, 27 | msg=e.msg 28 | suicide() 29 | 30 | when not defined(llvmFuzzer): 31 | proc readStdin(): seq[byte] = 32 | let s = if paramCount() > 0: newFileStream(paramStr(1)) 33 | else: newFileStream(stdin) 34 | if s.isNil: 35 | chronicles.error "Error opening input stream" 36 | suicide() 37 | # We use binary files as with hex we can get lots of "not hex" failures 38 | var input = s.readAll() 39 | s.close() 40 | # Remove newline if it is there 41 | input.removeSuffix 42 | result = cast[seq[byte]](input) 43 | 44 | proc NimMain() {.importc: "NimMain".} 45 | 46 | # The default init, gets redefined when init template is used. 47 | template initImpl(): untyped = 48 | when defined(llvmFuzzer): 49 | proc fuzzerInit(): cint {.exportc: "LLVMFuzzerInitialize".} = 50 | NimMain() 51 | 52 | return 0 53 | else: 54 | discard 55 | 56 | template init*(body: untyped) {.dirty.} = 57 | ## Init block to do any initialisation for the fuzzing test. 58 | ## 59 | ## For AFL this is currently only cosmetic and will be run each time, before 60 | ## the test block. 61 | ## 62 | ## For LLVM fuzzers this will only be run once. So only put data which is 63 | ## stateless or make sure everything gets properply reset for each new run 64 | ## in the test block. 65 | when defined(llvmFuzzer): 66 | template initImpl() {.dirty.} = 67 | bind NimMain 68 | 69 | proc fuzzerInit(): cint {.exportc: "LLVMFuzzerInitialize".} = 70 | NimMain() 71 | 72 | body 73 | 74 | return 0 75 | else: 76 | template initImpl(): untyped {.dirty.} = 77 | bind fuzz 78 | fuzz: body 79 | 80 | template test*(body: untyped): untyped = 81 | ## Test block to do the actual test that will be fuzzed in a loop. 82 | ## 83 | ## Within this test block there is access to the payload OpenArray which 84 | ## contains the payload provided by the fuzzer. 85 | mixin initImpl 86 | initImpl() 87 | when defined(llvmFuzzer): 88 | proc fuzzerCall(data: ptr byte, len: csize): 89 | cint {.exportc: "LLVMFuzzerTestOneInput".} = 90 | template payload(): auto = 91 | makeOpenArray(data, len) 92 | 93 | body 94 | else: 95 | when not defined(windows): 96 | var payload {.inject.} = readStdin() 97 | 98 | fuzz: body 99 | else: 100 | proc fuzzerCall() {.exportc: "AFLmain", dynlib, cdecl.} = 101 | var payload {.inject.} = readStdin() 102 | fuzz: body 103 | 104 | fuzzerCall() 105 | 106 | when defined(clangfast) and not defined(llvmFuzzer): 107 | ## Can be used for deferred instrumentation. 108 | ## Should be placed on a suitable location in the code where the delayed 109 | ## cloning can take place (e.g. NOT after creation of threads) 110 | proc aflInit*() {.importc: "__AFL_INIT", noDecl.} 111 | ## Can be used for persistent mode. 112 | ## Should be used as value for controlling a loop around a test case. 113 | ## Test case should be able to handle repeated inputs. No repeated fork() will 114 | ## be done. 115 | # TODO: Lets use this in the test block when afl-clang-fast is used? 116 | proc aflLoopImpl(count: cuint): cint {.importc: "__AFL_LOOP", noDecl.} 117 | template aflLoop*(body: untyped): untyped = 118 | while aflLoopImpl(1000) != 0: 119 | `body` 120 | else: 121 | proc aflInit*() = discard 122 | template aflLoop*(body: untyped): untyped = `body` 123 | -------------------------------------------------------------------------------- /testutils/fuzzing/fuzzing_on_windows.md: -------------------------------------------------------------------------------- 1 | # Fuzzing on Windows 2 | 3 | This is a supplemental guide to fuzzing on windows platform. 4 | 5 | ## Windows Subsystem for Linux(WSL) Ubuntu 20.04 6 | 7 | Grab Ubuntu from Windows Store and install libFuzzer and afl. 8 | But don't forget to update or upgrade the database if you start with a 'blank' Ubuntu. 9 | 10 | ```sh 11 | sudo apt update 12 | sudo apt upgrade 13 | ``` 14 | 15 | ### Install clang and libFuzzer 16 | 17 | Pick your clang version: 9, 10, or 11. In this example I'll use clang-10. 18 | 19 | ```sh 20 | sudo apt install build-essential 21 | sudo apt-get install clang-10 lldb-10 lld-10 22 | sudo apt-get install libfuzzer-10-dev 23 | ``` 24 | 25 | Now copy the symlink in `/usr/bin` or whatever location of clang-10. 26 | 27 | ```sh 28 | $> which clang-10 29 | /usr/bin/clang-10 # this is the result of 'which clang-10' 30 | $> sudo cp -P /usr/bin/clang-10 /usr/bin/clang 31 | ``` 32 | 33 | ### Install afl 34 | 35 | ```sh 36 | git clone https://github.com/google/AFL 37 | cd AFL 38 | make 39 | sudo make install 40 | ``` 41 | 42 | Now go back to [Fuzzing instructions for Linux](readme.md) 43 | 44 | ## Real Windows instructions 45 | 46 | There are a lot of things you need to install on Windows. 47 | 48 | ### Compiling with libFuzzer 49 | 50 | * Download and install Clang 11 for Windows [here](https://llvm.org/builds/) 51 | * Download and install Visual Studio 2019 [here](https://visualstudio.microsoft.com/downloads/) 52 | 53 | You don't need to install all of the Visual Studio components, you only need to 54 | choose “Desktop development with C++”. That will be enough and only download less than 2GB instead of 4GB+. 55 | Perhaps you wonder why need to install two compiler? The answer is: libFuzzer does not work with MingW-GCC. 56 | 57 | If you already prepare your test case, the instruction to build the binary is exactly the same with Linux version. 58 | 59 | ```Nim 60 | nim c -d:libFuzzer -d:release -d:chronicles_log_level=fatal --noMain --cc=clang --passC="-fsanitize=fuzzer" --passL="-fsanitize=fuzzer" testcase 61 | ``` 62 | 63 | Now go back to [Starting the Fuzzer using libFuzzer](readme.md#Starting-the-Fuzzer) 64 | 65 | 66 | ### Compiling with winafl 67 | 68 | We will use the same Visual Studio compiler like libFuzzer. 69 | 70 | * Download and install Visual Studio 2019 [here](https://visualstudio.microsoft.com/downloads/) 71 | 72 | Now open one of this terminal from VS 2019: 73 | 74 | * Developer PowerShell for VS 2019 75 | * x64 Native Tools Command Prompt for VS 2019 76 | * x86 Native Tools Command Prompt for VS 2019 77 | 78 | ### Download and build winafl 79 | 80 | No need to install cmake, VS 2019 already included cmake in it's installation package. 81 | 82 | ```sh 83 | git clone https://github.com/googleprojectzero/winafl 84 | cd winafl 85 | git submodule update --init --recursive 86 | ``` 87 | 88 | #### 32/64 bit build using VS 2017 89 | 90 | ```sh 91 | mkdir build32 92 | cd build32 93 | cmake -G"Visual Studio 15 2017" .. -DINTELPT=1 94 | cmake --build . --config Release 95 | 96 | mkdir build64 97 | cd build64 98 | cmake -G"Visual Studio 15 2017 Win64" .. -DINTELPT=1 99 | cmake --build . --config Release 100 | ``` 101 | 102 | #### 32/64 bit build using VS 2019 103 | 104 | ``` 105 | mkdir build32 106 | cd build32 107 | cmake -G"Visual Studio 16 2019" .. -DINTELPT=1 -Ax86 108 | cmake --build . --config Release 109 | 110 | mkdir build64 111 | cd build64 112 | cmake -G"Visual Studio 16 2019" .. -DINTELPT=1 -Ax64 113 | cmake --build . --config Release 114 | ``` 115 | 116 | Either you use VS 2017 or VS 2019, you'll get the binary in: 117 | 118 | `winafl/build64/bin/Release` or `winafl/build32/bin/Release` 119 | 120 | If you only need to use it occasionally, you can use this command to add the winafl binary path to 121 | you env `PATH` instead of polluting it system wide. 122 | 123 | * PowerShell: ```$env:path = ($pwd).path + "\bin\Release;" + $env:path``` 124 | * CMD Command Prompt: ```set PATH=%CD%\bin\Release;%PATH%``` 125 | 126 | #### Compiling testcase 127 | 128 | Compiling the testcase is simpler than Linux version, you don't need to use afl-gcc or afl-clang, 129 | you can use clang, vcc, or mingw-gcc as you like. 130 | 131 | ```Nim 132 | nim c -d:afl -d:noSignalHandler -d:release -d:chronicles_log_level=fatal testcase 133 | ``` 134 | 135 | #### Starting the Fuzzer 136 | 137 | Now run the command from Command Prompt terminal, the `@@` will not work with PowerShell. 138 | Winafl needs the input data to be read from a file, not from stdin, that's why the presence of `@@`. 139 | 140 | ```sh 141 | afl-fuzz.exe -i inDir -o outDir -P -t 20000 -- -coverage_module testcase.exe -fuzz_iterations 20 -target_module testcase.exe -target_method AFLmain -nargs 2 -- testcase.exe @@ 142 | ``` 143 | 144 | * `inDir` is a directory containing a small but valid input file that makes sense to the program. 145 | * `outDir` will be the location of generated testcase corpus. 146 | * replace both `testcase.exe` with your executable binary. 147 | * `-P` is Intel PT selector 148 | * `-t` timeout in msec 149 | -------------------------------------------------------------------------------- /testutils/fuzzing_engines.nim: -------------------------------------------------------------------------------- 1 | import strformat 2 | import os except dirExists 3 | 4 | const 5 | aflGcc = "--cc=gcc " & 6 | "--gcc.exe=afl-gcc " & 7 | "--gcc.linkerexe=afl-gcc" 8 | 9 | aflClang = "--cc=clang " & 10 | "--clang.exe=afl-clang " & 11 | "--clang.linkerexe=afl-clang" 12 | 13 | aflClangFast = "--cc=clang " & 14 | "--clang.exe=afl-clang-fast " & 15 | "--clang.linkerexe=afl-clang-fast " & 16 | "-d:clangfast" 17 | 18 | libFuzzerClang = "--cc=clang " & 19 | "--passC='-fsanitize=fuzzer,address' " & 20 | "--passL='-fsanitize=fuzzer,address'" 21 | 22 | honggfuzzClang = "--cc=clang " & 23 | "--clang.exe=hfuzz-clang " & 24 | "--clang.linkerexe=hfuzz-clang " 25 | 26 | # Can also test in debug mode obviously, but might be slower 27 | # Can turn on more logging, in case of libFuzzer it will get very verbose though 28 | defaultFlags = "-d:release -d:chronicles_log_level=fatal " & 29 | "--hints:off --warnings:off --verbosity:0" 30 | 31 | type 32 | FuzzingEngine* = enum 33 | libFuzzer 34 | honggfuzz 35 | afl 36 | 37 | AflCompiler* = enum 38 | gcc = aflGcc, 39 | clang = aflClang, 40 | clangFast = aflClangFast 41 | 42 | const 43 | defaultFuzzingEngine* = libFuzzer 44 | 45 | when not defined(nimscript): 46 | import os, osproc 47 | 48 | template exec(cmd: string) = 49 | discard execCmd(cmd) 50 | 51 | template mkDir(dir: string) = 52 | createDir dir 53 | 54 | template withDir*(dir: string; body: untyped): untyped = 55 | ## Changes the current directory temporarily. 56 | ## 57 | ## If you need a permanent change, use the `cd() <#cd,string>`_ proc. 58 | ## Usage example: 59 | ## 60 | ## .. code-block:: nim 61 | ## withDir "foo": 62 | ## # inside foo 63 | ## #back to last dir 64 | var curDir = getCurrentDir() 65 | try: 66 | setCurrentDir(dir) 67 | body 68 | finally: 69 | setCurrentDir(curDir) 70 | 71 | template q(x: string): string = 72 | quoteShell x 73 | 74 | proc aflCompile*(target: string, c: AflCompiler) = 75 | let aflOptions = &"-d:afl -d:noSignalHandler {$c}" 76 | let compileCmd = &"nim c {defaultFlags} {aflOptions} {q target}" 77 | exec compileCmd 78 | 79 | proc aflExec*(target: string, 80 | inputDir: string, 81 | resultsDir: string, 82 | cleanStart = false) = 83 | let exe = target.addFileExt(ExeExt) 84 | if not dirExists(inputDir): 85 | # create a input dir with one 0 file for afl 86 | mkDir(inputDir) 87 | # TODO: improve 88 | withDir inputDir: exec "echo '0' > test" 89 | 90 | var fuzzCmd: string 91 | # if there is an output dir already, continue fuzzing from previous run 92 | if (not dirExists(resultsDir)) or cleanStart: 93 | fuzzCmd = &"afl-fuzz -i {q inputDir} -o {q resultsDir} -M fuzzer01 -- {q exe}" 94 | else: 95 | fuzzCmd = &"afl-fuzz -i - -o {q resultsDir} -M fuzzer01 -- {q exe}" 96 | exec fuzzCmd 97 | 98 | proc libFuzzerCompile*(target: string) = 99 | let libFuzzerOptions = &"-d:llvmFuzzer --noMain {libFuzzerClang}" 100 | let compileCmd = &"nim c {defaultFlags} {libFuzzerOptions} {q target}" 101 | exec compileCmd 102 | 103 | proc libFuzzerExec*(target: string, corpusDir: string) = 104 | if not dirExists(corpusDir): 105 | # libFuzzer is OK when starting with empty corpus dir 106 | mkDir(corpusDir) 107 | 108 | exec &"{q target} {q corpusDir}" 109 | 110 | proc honggfuzzCompile*(target: string) = 111 | let honggfuzzOptions = &"-d:llvmFuzzer --noMain {honggfuzzClang}" 112 | let compileCmd = &"nim c {defaultFlags} {honggfuzzOptions} {q target}" 113 | exec compileCmd 114 | 115 | proc honggfuzzExec*(target: string, corpusDir: string, outputDir: string) = 116 | #if not dirExists(corpusDir): 117 | # # libFuzzer is OK when starting with empty corpus dir 118 | # mkDir(corpusDir) 119 | 120 | # TODO: 121 | # Other useful parameters: 122 | # --threads|-n VALUE 123 | # Number of concurrent fuzzing threads (default: number of CPUs / 2) 124 | # --workspace|-W VALUE 125 | # Workspace directory to save crashes & runtime files (default: '.') 126 | # --crashdir VALUE 127 | # Directory where crashes are saved to (default: workspace directory) 128 | # --covdir_new VALUE 129 | # New coverage (beyond the dry-run fuzzing phase) is written to this separate directory 130 | # --dict|-w VALUE 131 | # Dictionary file. Format:http://llvm.org/docs/LibFuzzer.html#dictionaries 132 | exec &"honggfuzz --persistent --input {q corpusDir} --output {q outputDir} -- {q target}" 133 | 134 | proc runFuzzer*(targetPath: string, fuzzer: FuzzingEngine, corpusDir: string) = 135 | let 136 | (path, target, ext) = splitFile(targetPath) 137 | compiledExe = addFileExt(path / target, ExeExt) 138 | corpusDir = if corpusDir.len > 0: corpusDir 139 | else: path / "corpus" 140 | 141 | case fuzzer 142 | of afl: 143 | aflCompile(targetPath, clang) 144 | aflExec(compiledExe, corpusDir, path / "results") 145 | 146 | of libFuzzer: 147 | libFuzzerCompile(targetPath) 148 | libFuzzerExec(compiledExe, corpusDir) 149 | 150 | of honggfuzz: 151 | honggfuzzCompile(targetPath) 152 | honggfuzzExec(compiledExe, corpusDir, path / "results") 153 | 154 | -------------------------------------------------------------------------------- /testutils/readme.md: -------------------------------------------------------------------------------- 1 | # Testrunner [![Build Status](https://travis-ci.org/status-im/nim-testutils.svg?branch=master)](https://travis-ci.org/status-im/nim-testutils) 2 | [![Build status](https://ci.appveyor.com/api/projects/status/ayqsnuvcpwo2nh6m/branch/master?svg=true)](https://ci.appveyor.com/project/nimbus/nim-testutils/branch/master) 3 | 4 | ## Usage 5 | 6 | Command syntax: 7 | 8 | ```sh 9 | Usage: 10 | ntu COMMAND [options] 11 | 12 | Available commands: 13 | 14 | $ ntu test [options] 15 | 16 | Run the test(s) specified at path. Will search recursively for test files 17 | provided path is a directory. 18 | 19 | Options: 20 | --backends:"c cpp js objc" Run tests for specified targets 21 | --include:"test1 test2" Run only listed tests (space/comma separated) 22 | --exclude:"test1 test2" Skip listed tests (space/comma separated) 23 | --update Rewrite failed tests with new output 24 | --sort:"source,test" Sort the tests by program and/or test mtime 25 | --reverse Reverse the order of tests 26 | --random Shuffle the order of tests 27 | --help Display this help and exit 28 | 29 | $ ntu fuzz [options] 30 | 31 | Start a fuzzing test with a Nim module based on testutils/fuzzing. 32 | 33 | Options: 34 | --fuzzer:libFuzzer The fuzzing engine to use. 35 | Possible values: libFuzzer, honggfuzz, afl 36 | --corpus: A directory with initial input cases 37 | ``` 38 | 39 | The runner will look recursively for all `*.test` files at given path. 40 | 41 | ## Test file options 42 | 43 | The test files follow the configuration file syntax (similar as `.ini`), see 44 | also [nim parsecfg module](https://nim-lang.org/docs/parsecfg.html). 45 | 46 | ### Required 47 | 48 | - **program**: A test file should have at minimum a program name. This is the name 49 | of the nim source minus the `.nim` extension. 50 | 51 | ### Optional 52 | 53 | - **max_size**: To check the maximum size of the binary, in bytes. 54 | - **timestamp_peg**: If you don't want to use the default timestamps, you can define 55 | your own timestamp peg here. 56 | - **compile_error**: When expecting a compilation failure, the error message that 57 | should be expected. 58 | - **error_file**: When expecting a compilation failure, the source file where the 59 | error should occur. 60 | - **os**: Space and/or comma separated list of operating systems for which the 61 | test should be run. Defaults to `"linux, macosx, windows"`. Tests meant for a 62 | different OS than the host will be marked as `SKIPPED`. 63 | - **--skip**: This will simply skip the test (will not be marked as failure). 64 | 65 | ### Forwarded Options 66 | Any other options or key-value pairs will be forwarded to the nim compiler. 67 | 68 | A **key-value** pair will become a conditional symbol + value (`-d:SYMBOL(:VAL)`) 69 | for the nim compiler, e.g. for `-d:chronicles_timestamps="UnixTime"` the test 70 | file requires: 71 | ```ini 72 | chronicles_timestamps="UnixTime" 73 | ``` 74 | If only a key is given, an empty value will be forwarded. 75 | 76 | An **option** will be forwarded as is to the nim compiler, e.g. this can be 77 | added in a test file: 78 | ```ini 79 | --opt:size 80 | ``` 81 | 82 | ### Verifying Expected Output 83 | 84 | For outputs to be compared, the output string should be set to the output name 85 | (`stdout` or _filename_) from within an _Output_ section: 86 | 87 | ```ini 88 | [Output] 89 | stdout="""expected stdout output""" 90 | file.log="""expected file output""" 91 | ``` 92 | 93 | Triple quotes can be used for multiple lines. 94 | 95 | ### Supplying Command-line Arguments 96 | 97 | Optionally specify command-line arguments as an escaped string in the following 98 | syntax inside any _Output_ section: 99 | 100 | ```ini 101 | [Output] 102 | args = "--title \"useful title\"" 103 | ``` 104 | 105 | ### Multiple Invocations 106 | 107 | Multiple _Output_ sections denote multiple test program invocations. Any 108 | failure of the test program to match its expected outputs will short-circuit 109 | and fail the test. 110 | 111 | ```ini 112 | [Output] 113 | stdout = "" 114 | args = "--no-output" 115 | 116 | [Output_newlines] 117 | stdout = "\n\n" 118 | args = "--newlines" 119 | ``` 120 | 121 | ### Updating Expected Outputs 122 | 123 | Pass the `--update` argument to `ntu` to rewrite any failing test with 124 | the new outputs of the test. 125 | 126 | ### Concurrent Test Execution 127 | 128 | When built with threads, `ntu` will run multiple test invocations 129 | defined in each test file simultaneously. You can specify `nothreads` 130 | in the _preamble_ to disable this behavior. 131 | 132 | ```ini 133 | nothreads = true 134 | 135 | [Output_1st_serial] 136 | args = "--first" 137 | 138 | [Output_2nd_serial] 139 | args = "--second" 140 | ``` 141 | 142 | The failure of any test will, when possible, short-circuit all other tests 143 | defined in the same file. 144 | 145 | ### CPU Affinity 146 | 147 | Specify `affinity` to clamp the first _N_ concurrent test threads to the first 148 | _N_ CPU cores. 149 | 150 | ```ini 151 | affinity = true 152 | 153 | [Output_1st_core] 154 | args = "--first" 155 | 156 | [Output_2nd_core] 157 | args = "--second" 158 | ``` 159 | 160 | ### Testing Alternate Backends 161 | 162 | By default, `ntu` builds tests using Nim's C backend. 163 | Specify the `--backends` command-line option to build and run run tests with 164 | the backends of your choice. 165 | 166 | ```sh 167 | $ ntu test --backends="c cpp" tests 168 | ``` 169 | 170 | ### Setting the Order of Tests 171 | 172 | By default, `ntu` will order test compilation and execution according to the 173 | modification time of the test program source. You can choose to sort by test 174 | program mtime, too. 175 | 176 | ```sh 177 | $ ntu test --sort:test suite/ 178 | ``` 179 | 180 | You can `--reverse` or `--random`ize the order of tests, too. 181 | 182 | ### More Examples 183 | 184 | See `chonicles`, where `testutils` was born: 185 | - https://github.com/status-im/nim-chronicles/tree/master/tests 186 | 187 | 188 | ## License 189 | Apache2 or MIT 190 | -------------------------------------------------------------------------------- /testutils/fuzzing/readme.md: -------------------------------------------------------------------------------- 1 | # Fuzzing 2 | ## tldr: 3 | * [Install afl](#Install-afl). 4 | * Create a testcase. 5 | * Run: `nim fuzz.nims afl testfolder/testcase.nim` 6 | 7 | Or 8 | 9 | * [Install libFuzzer](#Install-libFuzzer) (comes with LLVM). 10 | * Create a testcase. 11 | * Run: `nim fuzz.nims libFuzzer testfolder/testcase.nim` 12 | 13 | ## Fuzzing Helpers 14 | There are two convenience templates which will help you set up a quick fuzzing 15 | test. 16 | 17 | These are the mandatory `test` block and the optional `init` block. 18 | 19 | Example usage: 20 | ```nim 21 | test: 22 | var rlp = rlpFromBytes(payload) 23 | discard rlp.inspect() 24 | ``` 25 | 26 | Any unhandled `Exception` will result in a failure of the testcase. If certain 27 | `Exception`s are to be allowed to occur within the test, they should be caught. 28 | 29 | E.g.: 30 | ```nim 31 | test: 32 | try: 33 | var rlp = rlpFromBytes(payload) 34 | discard rlp.inspect() 35 | except RlpError as e: 36 | debug "Inspect failed", err = e.msg 37 | ``` 38 | 39 | ## Supported Fuzzers 40 | The two templates can prepare the code for both 41 | [afl](http://lcamtuf.coredump.cx/afl/), 42 | [afl++](https://github.com/AFLplusplus/AFLplusplus) and 43 | [libFuzzer](http://llvm.org/docs/LibFuzzer.html). 44 | 45 | You will need to install first the fuzzer you want to use. 46 | 47 | ### Install afl 48 | 49 | ```sh 50 | # Ubuntu / Debian 51 | sudo apt-get install afl++ 52 | 53 | # Fedora 54 | dnf install american-fuzzy-lop 55 | # for usage with clang & clang-fast you will have to install 56 | # american-fuzzy-lop-clang or american-fuzzy-lop-clang-fast 57 | 58 | # Arch Linux 59 | pacman -S afl 60 | 61 | # NixOS 62 | nix-env -i afl 63 | 64 | ``` 65 | 66 | ### Install libFuzzer 67 | 68 | LibFuzzer is part of llvm and will be installed together with llvm-libs in 69 | recent versions. Installing clang should install llvm-libs. 70 | ```sh 71 | # Ubuntu / Debian 72 | sudo apt-get install clang 73 | 74 | # Fedora 75 | dnf install clang 76 | 77 | # Arch Linux 78 | pacman -S clang 79 | 80 | # NixOS 81 | nix-env -iA nixos.clang_7 nixos.llvm_7 82 | ``` 83 | 84 | ## Compiling & Starting the Fuzzer 85 | ### Scripted helper 86 | There is a nimscript helper to compile & start the fuzzer: 87 | ```sh 88 | # for afl 89 | nim fuzz.nims afl testcase.nim 90 | 91 | # for libFuzzer 92 | nim fuzz.nims libFuzzer testcase.nim 93 | ``` 94 | ### Manually with afl 95 | #### Compiling 96 | With gcc: 97 | ```sh 98 | nim c -d:afl -d:release -d:chronicles_log_level=fatal -d:noSignalHandler --cc=gcc --gcc.exe=afl-gcc --gcc.linkerexe=afl-gcc testcase.nim 99 | ``` 100 | The `afl` define is specifically required for the `init` and `test` 101 | templates. 102 | 103 | You typically want to fuzz in `-d:release` and probably also want to lower down 104 | the logging. But this is not strictly necessary. 105 | 106 | There is also a nimscript task in `config.nims` for this: 107 | ``` 108 | nim c build_afl testcase.nim 109 | ``` 110 | 111 | With clang: 112 | ```sh 113 | # afl-clang 114 | nim c -d:afl -d:noSignalHandler --cc=clang --clang.exe=afl-clang --clang.linkerexe=afl-clang ftestcase.nim 115 | # afl-clang-fast 116 | nim c -d:afl -d:noSignalHandler --cc=clang --clang.exe=afl-clang-fast --clang.linkerexe=afl-clang-fast testcase.nim 117 | ``` 118 | 119 | #### Starting the Fuzzer 120 | 121 | To start the fuzzer: 122 | ```sh 123 | afl-fuzz -i input -o results -- ./testcase 124 | ``` 125 | 126 | To rerun it without losing previous results/corpus: 127 | ```sh 128 | afl-fuzz -i - -o results -- ./testcase 129 | ``` 130 | 131 | To run several parallel fuzzing sessions: 132 | ```sh 133 | # Start master fuzzer 134 | afl-fuzz -i input -o results -M fuzzer01 -- ./testcase 135 | # Start slaves (usually 1 per core available) 136 | afl-fuzz -i input -o results -S fuzzer02 -- ./testcase 137 | afl-fuzz -i input -o results -S fuzzer03 -- ./testcase 138 | # add more if needed 139 | ``` 140 | 141 | When compiled with `-d:afl` the resulting application can also be run 142 | manually by providing it input data, e.g.: 143 | ```sh 144 | ./testcase < testfile 145 | ``` 146 | 147 | During debugging you might not want the testcase to generate a segmentation 148 | fault on exceptions. You can do this by rebuilding the test without the `-d:afl` 149 | flag. Changing to `-d:debug` will also help but might also change the 150 | behaviour. 151 | 152 | ### Manually with libFuzzer 153 | #### Compiling 154 | ```sh 155 | nim c -d:libFuzzer -d:release -d:chronicles_log_level=fatal --noMain --cc=clang --passC="-fsanitize=fuzzer" --passL="-fsanitize=fuzzer" testcase.nim 156 | ``` 157 | 158 | The `libFuzzer` define is specifically required for the `init` and `test` 159 | templates. 160 | 161 | You typically want to fuzz in `-d:release` and probably also want to lower down 162 | the logging. But this is not strictly necessary. 163 | 164 | There is also a nimscript task in `config.nims` for compiling: 165 | ``` 166 | nim c build_libFuzzer testcase.nim 167 | ``` 168 | 169 | #### Starting the Fuzzer 170 | Starting the fuzzer is as simple as running the compiled program: 171 | ```sh 172 | ./testcase corpus_dir -runs=1000000 173 | ``` 174 | 175 | To see the available options: 176 | ```sh 177 | ./testcase test=1 178 | ``` 179 | 180 | Parallel fuzzing on 8 cores: 181 | ```sh 182 | ./fuzz-libfuzzer -jobs=8 -workers=8 183 | ``` 184 | 185 | You can also use the application to verify a specific test case: 186 | ```sh 187 | ./testcase input_file 188 | ``` 189 | 190 | ## Additional notes 191 | The `init` template, when used with **afl**, is only cosmetic. It will be 192 | run before each test block, compared to libFuzzer, where it will be run only 193 | once. 194 | 195 | In case of using afl with `alf-clang-fast` you can make use of `aflInit()` proc 196 | and `aflLoop()` template. 197 | 198 | `aflInit()` will allow using what is called deferred instrumentation. Basically, 199 | the forking of the process will only happen after this call, where normally it 200 | is done right before `main()`. 201 | 202 | `aflLoop:` will allow for (experimental) persistant mode. It will run the test 203 | in loop (1000 iterations) with different payloads. This is more comparable with 204 | libFuzzer. 205 | 206 | These calls are enabled with `-d:clangfast`, and have to be manually added. 207 | They are currently not part of the `test` or `init` templates. 208 | -------------------------------------------------------------------------------- /testutils/config.nim: -------------------------------------------------------------------------------- 1 | import 2 | std/[sequtils, hashes, os, parseopt, strutils, algorithm], 3 | fuzzing_engines 4 | 5 | const 6 | Usage = """ 7 | 8 | Usage: 9 | ntu COMMAND [options] 10 | 11 | Available commands: 12 | 13 | $ ntu test [options] 14 | 15 | Run the test(s) specified at path. Will search recursively for test files 16 | provided path is a directory. 17 | 18 | Options: 19 | --backends:"c cpp js objc" Run tests for specified targets 20 | --include:"test1 test2" Run only listed tests (space/comma separated) 21 | --exclude:"test1 test2" Skip listed tests (space/comma separated) 22 | --update Rewrite failed tests with new output 23 | --sort:"source,test" Sort the tests by program and/or test mtime 24 | --reverse Reverse the order of tests 25 | --random Shuffle the order of tests 26 | --help Display this help and exit 27 | 28 | $ ntu fuzz [options] 29 | 30 | Start a fuzzing test with a Nim module based on testutils/fuzzing. 31 | 32 | Options: 33 | --fuzzer:libFuzzer The fuzzing engine to use. 34 | Possible values: libFuzzer, honggfuzz, afl 35 | --corpus: A directory with initial input cases 36 | 37 | """.unindent.strip 38 | 39 | type 40 | FlagKind* = enum 41 | UpdateOutputs = "--update" 42 | UseThreads = "--threads:on" 43 | DebugBuild = "--define:debug" 44 | ReleaseBuild = "--define:release" 45 | DangerBuild = "--define:danger" 46 | CpuAffinity = "--affinity" 47 | 48 | SortBy* {.pure.} = enum 49 | Random = "random" 50 | Source = "source" 51 | Test = "test" 52 | Reverse = "reverse" 53 | 54 | Command* = enum 55 | noCommand 56 | test 57 | fuzz 58 | 59 | TestConfig* = object 60 | case cmd*: Command 61 | of test: 62 | path*: string 63 | includedTests*: seq[string] 64 | excludedTests*: seq[string] 65 | 66 | flags*: set[FlagKind] 67 | # options 68 | backendNames*: seq[string] 69 | orderBy*: set[SortBy] 70 | of fuzz: 71 | fuzzer*: FuzzingEngine 72 | corpusDir*: string 73 | target*: string 74 | of noCommand: 75 | discard 76 | 77 | const 78 | defaultFlags = {UseThreads} 79 | compilerFlags* = {DebugBuild, ReleaseBuild, DangerBuild, UseThreads} 80 | # --define:testutilsBackends="cpp js" 81 | testutilsBackends* {.strdefine.} = "c" 82 | defaultSort = {Source, Reverse} 83 | 84 | proc `backends=`*(config: var TestConfig; inputs: seq[string]) = 85 | config.backendNames = inputs.sorted 86 | 87 | proc `backends=`*(config: var TestConfig; input: string) = 88 | config.backends = input.split(" ") 89 | 90 | proc backends*(config: TestConfig): seq[string] = 91 | result = config.backendNames 92 | 93 | proc hash*(config: TestConfig): Hash = 94 | var h: Hash = 0 95 | h = h !& config.backends.hash 96 | h = h !& hash(ReleaseBuild in config.flags) 97 | h = h !& hash(DangerBuild in config.flags) 98 | h = h !& hash(UseThreads notin config.flags) 99 | result = !$h 100 | 101 | proc compilationFlags*(config: TestConfig): string = 102 | for flag in compilerFlags * config.flags: 103 | result &= " " & $flag 104 | 105 | proc cache*(config: TestConfig; backend: string): string = 106 | ## return the path to the nimcache for the given backend and 107 | ## compile-time flags 108 | result = getTempDir() 109 | result = result / "testutils-nimcache-$#-$#" % [ backend, 110 | $getCurrentProcessId() ] 111 | 112 | proc processArguments*(): TestConfig = 113 | ## consume the arguments supplied to ntu and yield a computed 114 | ## configuration object 115 | var 116 | opt = initOptParser() 117 | 118 | func toSet[SortBy](list: seq[SortBy]): set[SortBy] = 119 | for element in list.items: 120 | result.incl element 121 | 122 | for kind, key, value in opt.getopt: 123 | if result.cmd == noCommand: 124 | doAssert kind == cmdArgument 125 | result.cmd = parseEnum[Command](key) 126 | if result.cmd == test: 127 | result.flags = defaultFlags 128 | result.backends = testutilsBackends 129 | result.orderBy = defaultSort 130 | continue 131 | 132 | case result.cmd 133 | of test: 134 | case kind 135 | of cmdArgument: 136 | if result.path == "": 137 | result.path = absolutePath(key) 138 | of cmdLongOption, cmdShortOption: 139 | case key.toLowerAscii 140 | of "help", "h": 141 | quit(Usage, QuitSuccess) 142 | of "reverse", "random": 143 | let 144 | flag = parseEnum[SortBy](value) 145 | if flag in result.orderBy: 146 | result.orderBy.excl flag 147 | else: 148 | result.orderBy.incl flag 149 | of "sort": 150 | result.orderBy = toSet value.split(",").mapIt parseEnum[SortBy](it) 151 | of "backend", "backends", "targets", "t": 152 | result.backends = value 153 | of "release", "danger": 154 | result.flags.incl ReleaseBuild 155 | result.flags.incl DangerBuild 156 | of "nothreads": 157 | result.flags.excl UseThreads 158 | of "update": 159 | result.flags.incl UpdateOutputs 160 | of "include": 161 | result.includedTests.add value.split(Whitespace + {','}) 162 | of "exclude": 163 | result.excludedTests.add value.split(Whitespace + {','}) 164 | else: 165 | quit(Usage) 166 | of cmdEnd: 167 | quit(Usage) 168 | 169 | of fuzz: 170 | case kind 171 | of cmdArgument: 172 | result.target = key 173 | of cmdLongOption, cmdShortOption: 174 | case key.toLowerAscii: 175 | of "f", "fuzzer": 176 | result.fuzzer = parseEnum[FuzzingEngine](value) 177 | of "c", "corpus": 178 | result.corpusDir = absolutePath(value) 179 | else: 180 | quit(Usage) 181 | else: 182 | echo "got kind ", kind 183 | quit(Usage) 184 | 185 | of noCommand: 186 | discard 187 | 188 | case result.cmd 189 | of test: 190 | if result.path == "": 191 | quit(Usage) 192 | of fuzz: 193 | if result.target == "": 194 | quit(Usage) 195 | else: 196 | quit(Usage) 197 | 198 | func shouldSkip*(config: TestConfig, name: string): bool = 199 | ## true if the named test should be skipped 200 | if name in config.excludedTests: 201 | result = true 202 | elif config.includedTests.len > 0: 203 | if name notin config.includedTests: 204 | result = true 205 | -------------------------------------------------------------------------------- /testutils/spec.nim: -------------------------------------------------------------------------------- 1 | import std/hashes 2 | import std/os 3 | import std/parsecfg 4 | import std/strutils 5 | import std/streams 6 | import std/strtabs 7 | 8 | import testutils/config 9 | 10 | type 11 | TestOutputs* = StringTableRef 12 | TestSpec* = ref object 13 | section*: string 14 | args*: string 15 | config*: TestConfig 16 | path*: string 17 | pathComponents*: tuple[dir, name, ext: string] 18 | skip*: bool 19 | program*: string 20 | flags*: string 21 | outputs*: TestOutputs 22 | timestampPeg*: string 23 | errorMsg*: string 24 | maxSize*: int64 25 | compileError*: string 26 | errorFile*: string 27 | errorLine*: int 28 | errorColumn*: int 29 | os*: seq[string] 30 | child*: TestSpec 31 | 32 | const 33 | DefaultOses = @["linux", "macosx", "windows"] 34 | 35 | proc hash*(spec: TestSpec): Hash = 36 | var h: Hash = 0 37 | h = h !& spec.config.hash 38 | h = h !& spec.flags.hash 39 | h = h !& spec.os.hash 40 | result = !$h 41 | 42 | proc binaryHash*(spec: TestSpec; backend: string): Hash = 43 | ## hash the backend, any compilation flags, and defines, etc. 44 | var h: Hash = 0 45 | h = h !& backend.hash 46 | h = h !& spec.os.hash 47 | h = h !& hash(spec.config.flags * compilerFlags) 48 | h = h !& hash(spec.flags) 49 | h = h !& spec.program.hash 50 | h = h !& spec.pathComponents.name.hash 51 | result = !$h 52 | 53 | template name*(spec: TestSpec): string = 54 | spec.pathComponents.name 55 | 56 | proc newTestOutputs*(): StringTableRef = 57 | result = newStringTable(mode = modeStyleInsensitive) 58 | 59 | proc clone*(spec: TestSpec): TestSpec = 60 | ## create the parent of this test and set the child reference appropriately 61 | result = new(TestSpec) 62 | result[] = spec[] 63 | result.outputs = newTestOutputs() 64 | result.args = "" 65 | result.child = spec 66 | 67 | func stage*(spec: TestSpec): string = 68 | ## the name of the output section for the test 69 | ## Output_test_section_name 70 | let 71 | # @["", "test_section_name"] 72 | names = spec.section.split("Output") 73 | result = names[^1].replace("_", " ").strip 74 | 75 | proc source*(spec: TestSpec): string = 76 | result = absolutePath(spec.pathComponents.dir / spec.program.addFileExt(".nim")) 77 | 78 | proc binary*(spec: TestSpec; backend: string): string = 79 | ## some day this will make more sense 80 | result = (spec.pathComponents.dir / spec.pathComponents.name).addFileExt(ExeExt) 81 | if dirExists(result): 82 | result = result.addFileExt("out") 83 | 84 | proc binary*(spec: TestSpec): string {.deprecated.} = 85 | ## the output binary (execution input) of the test 86 | result = spec.binary("c") 87 | 88 | iterator binaries*(spec: TestSpec): string = 89 | ## enumerate binary targets for each backend specified by the test 90 | for backend in spec.config.backends.items: 91 | yield spec.binary(backend) 92 | 93 | proc defaults(spec: var TestSpec) = 94 | ## assert some default values for a given spec 95 | spec.os = DefaultOses 96 | spec.outputs = newTestOutputs() 97 | 98 | proc consumeConfigEvent(spec: var TestSpec; event: CfgEvent) = 99 | ## parse a specification supplied prior to any sections 100 | case event.key 101 | of "program": 102 | spec.program = event.value 103 | of "timestamp_peg": 104 | spec.timestampPeg = event.value 105 | of "max_size": 106 | try: 107 | spec.maxSize = parseInt(event.value) 108 | except ValueError: 109 | echo "Parsing warning: value of " & event.key & 110 | " is not a number (value = " & event.value & ")." 111 | of "compile_error": 112 | spec.compileError = event.value 113 | of "error_file": 114 | spec.errorFile = event.value 115 | of "os": 116 | spec.os = event.value.normalize.split({','} + Whitespace) 117 | of "affinity": 118 | spec.config.flags.incl CpuAffinity 119 | of "threads": 120 | spec.config.flags.incl UseThreads 121 | of "nothreads": 122 | spec.config.flags.excl UseThreads 123 | of "release", "danger", "debug": 124 | spec.config.flags.incl parseEnum[FlagKind]("--define:" & event.key) 125 | else: 126 | let 127 | flag = "--define:$#:$#" % [event.key, event.value] 128 | spec.flags.add flag.quoteShell & " " 129 | 130 | proc rewriteTestFile*(spec: TestSpec; outputs: TestOutputs) = 131 | ## rewrite a test file with updated outputs after having run the tests 132 | var 133 | test = loadConfig(spec.path) 134 | # take the opportunity to update an args statement if necessary 135 | if spec.args != "": 136 | test.setSectionKey(spec.section, "args", spec.args) 137 | else: 138 | test.delSectionKey(spec.section, "args") 139 | # delete the old test outputs for completeness 140 | for name, expected in spec.outputs.pairs: 141 | test.delSectionKey(spec.section, name) 142 | # add the new test outputs 143 | for name, expected in outputs.pairs: 144 | test.setSectionKey(spec.section, name, expected) 145 | test.writeConfig(spec.path) 146 | 147 | proc parseTestFile*(config: TestConfig; filePath: string): TestSpec = 148 | ## parse a test input file into a spec 149 | result = new(TestSpec) 150 | result.defaults 151 | result.path = absolutePath(filePath) 152 | result.pathComponents = splitFile result.path 153 | result.config = config 154 | block escapeBlock: 155 | var 156 | f = newFileStream(result.path, fmRead) 157 | if f == nil: 158 | # XXX crash? 159 | echo "Parsing error: cannot open " & result.path 160 | break escapeBlock 161 | 162 | var 163 | outputSection = false 164 | p: CfgParser 165 | p.open(f, result.path) 166 | try: 167 | while true: 168 | var e = next(p) 169 | case e.kind 170 | of cfgEof: 171 | break 172 | of cfgError: 173 | # XXX crash? 174 | echo "Parsing warning:" & e.msg 175 | of cfgSectionStart: 176 | # starts with Output 177 | if e.section[0..len"Output"-1].cmpIgnoreCase("Output") == 0: 178 | if outputSection: 179 | # create our parent; the eternal chain 180 | result = result.clone 181 | outputSection = true 182 | result.section = e.section 183 | of cfgKeyValuePair: 184 | if outputSection: 185 | if e.key.cmpIgnoreStyle("args") == 0: 186 | result.args = e.value 187 | else: 188 | result.outputs[e.key] = e.value 189 | else: 190 | result.consumeConfigEvent(e) 191 | of cfgOption: 192 | case e.key 193 | of "skip": 194 | result.skip = true 195 | else: 196 | # this for for, eg. --opt:size 197 | result.flags &= ("--$#:$#" % [e.key, e.value]).quoteShell & " " 198 | finally: 199 | close p 200 | 201 | # we catch this in ntu and crash there if needed 202 | if result.program == "": 203 | echo "Parsing error: no program value" 204 | -------------------------------------------------------------------------------- /ntu.nim: -------------------------------------------------------------------------------- 1 | import 2 | std/[hashes, random, tables, sequtils, strtabs, strutils, 3 | os, osproc, terminal, times, pegs, algorithm], 4 | testutils/[spec, config, helpers, fuzzing_engines] 5 | 6 | #[ 7 | 8 | The runner will look recursively for all *.test files at given path. A 9 | test file should have at minimum a program name. This is the name of the 10 | nim source minus the .nim extension) 11 | 12 | ]# 13 | 14 | 15 | # Code is here and there influenced by nim testament tester and unittest 16 | # module. 17 | 18 | const 19 | defaultOptions = "--verbosity:1 --warnings:on --skipUserCfg:on " 20 | backendOrder = @["c", "cpp", "js"] 21 | 22 | type 23 | TestStatus* = enum 24 | OK 25 | FAILED 26 | SKIPPED 27 | INVALID 28 | 29 | #[ 30 | If needed, pass more info to the logresult via a TestResult object 31 | TestResult = object 32 | status: TestStatus 33 | compileTime: float 34 | fileSize: uint 35 | ]# 36 | 37 | ThreadPayload = object 38 | core: int 39 | spec: TestSpec 40 | 41 | TestThread = Thread[ThreadPayload] 42 | TestError* = enum 43 | SourceFileNotFound 44 | ExeFileNotFound 45 | OutputFileNotFound 46 | CompileError 47 | RuntimeError 48 | OutputsDiffer 49 | FileSizeTooLarge 50 | CompileErrorDiffers 51 | 52 | BackendTests = TableRef[string, seq[TestSpec]] 53 | 54 | proc logFailure(test: TestSpec; error: TestError; 55 | data: varargs[string] = [""]) = 56 | case error 57 | of SourceFileNotFound: 58 | styledEcho(fgYellow, styleBright, "source file not found: ", 59 | resetStyle, test.source) 60 | of ExeFileNotFound: 61 | styledEcho(fgYellow, styleBright, "executable file not found: ", 62 | resetStyle, test.binary) 63 | of OutputFileNotFound: 64 | styledEcho(fgYellow, styleBright, "file not found: ", 65 | resetStyle, data[0]) 66 | of CompileError: 67 | styledEcho(fgYellow, styleBright, "compile error:\p", 68 | resetStyle, data[0]) 69 | of RuntimeError: 70 | styledEcho(fgYellow, styleBright, "runtime error:\p", 71 | resetStyle, data[0]) 72 | of OutputsDiffer: 73 | styledEcho(fgYellow, styleBright, "outputs are different:\p", 74 | resetStyle,"Expected output to $#:\p$#" % [data[0], data[1]], 75 | "Resulted output to $#:\p$#" % [data[0], data[2]]) 76 | of FileSizeTooLarge: 77 | styledEcho(fgYellow, styleBright, "file size is too large: ", 78 | resetStyle, data[0] & " > " & $test.maxSize) 79 | of CompileErrorDiffers: 80 | styledEcho(fgYellow, styleBright, "compile error is different:\p", 81 | resetStyle, data[0]) 82 | 83 | styledEcho(fgCyan, styleBright, "compiler: ", resetStyle, 84 | "$# $# $# $#" % [defaultOptions, 85 | test.flags, 86 | test.config.compilationFlags, 87 | test.source]) 88 | 89 | template withinDir(dir: string; body: untyped): untyped = 90 | ## run the body with a specified directory, returning to current dir 91 | let 92 | cwd = getCurrentDir() 93 | setCurrentDir(dir) 94 | try: 95 | body 96 | finally: 97 | setCurrentDir(cwd) 98 | 99 | proc logResult(testName: string, status: TestStatus, time: float) = 100 | var color = block: 101 | case status 102 | of OK: fgGreen 103 | of FAILED: fgRed 104 | of SKIPPED: fgYellow 105 | of INVALID: fgRed 106 | styledEcho(styleBright, color, "[", $status, "] ", 107 | resetStyle, testName, 108 | fgYellow, " ", time.formatFloat(ffDecimal, 3), " s") 109 | 110 | proc logResult(testName: string, status: TestStatus) = 111 | var color = block: 112 | case status 113 | of OK: fgGreen 114 | of FAILED: fgRed 115 | of SKIPPED: fgYellow 116 | of INVALID: fgRed 117 | styledEcho(styleBright, color, "[", $status, "] ", 118 | resetStyle, testName) 119 | 120 | template time(duration, body): untyped = 121 | let t0 = epochTime() 122 | block: 123 | body 124 | duration = epochTime() - t0 125 | 126 | proc composeOutputs(test: TestSpec, stdout: string): TestOutputs = 127 | ## collect the outputs for the given test 128 | result = newTestOutputs() 129 | for name, expected in test.outputs.pairs: 130 | if name == "stdout": 131 | result[name] = stdout 132 | else: 133 | if not fileExists(name): 134 | continue 135 | result[name] = readFile(name) 136 | removeFile(name) 137 | 138 | proc cmpOutputs(test: TestSpec, outputs: TestOutputs): TestStatus = 139 | ## compare the test program's outputs to those expected by the test 140 | result = OK 141 | for name, expected in test.outputs.pairs: 142 | if name notin outputs: 143 | logFailure(test, OutputFileNotFound, name) 144 | result = FAILED 145 | continue 146 | 147 | let 148 | testOutput = outputs[name] 149 | 150 | # Would be nice to do a real diff here instead of simple compare 151 | if test.timestampPeg.len > 0: 152 | if not cmpIgnorePegs(testOutput, expected, 153 | peg(test.timestampPeg), pegXid): 154 | logFailure(test, OutputsDiffer, name, expected, testOutput) 155 | result = FAILED 156 | else: 157 | if not cmpIgnoreDefaultTimestamps(testOutput, expected): 158 | logFailure(test, OutputsDiffer, name, expected, testOutput) 159 | result = FAILED 160 | 161 | proc compile(test: TestSpec; backend: string): TestStatus = 162 | ## compile the test program for the requested backends 163 | block escapeBlock: 164 | if not fileExists(test.source): 165 | logFailure(test, SourceFileNotFound) 166 | result = FAILED 167 | break escapeBlock 168 | 169 | let 170 | binary = test.binary(backend) 171 | var 172 | cmd = findExe("nim") 173 | cmd &= " " & backend 174 | cmd &= " --nimcache:" & test.config.cache(backend) 175 | cmd &= " --out:" & binary 176 | cmd &= " " & defaultOptions 177 | cmd &= " " & test.flags 178 | cmd &= " " & test.config.compilationFlags 179 | cmd &= " " & test.source.quoteShell 180 | var 181 | c = parseCmdLine(cmd) 182 | p = startProcess(command=c[0], args=c[1.. ^1], 183 | options={poStdErrToStdOut, poUsePath}) 184 | 185 | try: 186 | let 187 | compileInfo = parseCompileStream(p, p.outputStream) 188 | 189 | if compileInfo.exitCode != 0: 190 | if test.compileError.len == 0: 191 | logFailure(test, CompileError, compileInfo.fullMsg) 192 | result = FAILED 193 | break escapeBlock 194 | else: 195 | if test.compileError == compileInfo.msg and 196 | (test.errorFile.len == 0 or test.errorFile == compileInfo.errorFile) and 197 | (test.errorLine == 0 or test.errorLine == compileInfo.errorLine) and 198 | (test.errorColumn == 0 or test.errorColumn == compileInfo.errorColumn): 199 | result = OK 200 | else: 201 | logFailure(test, CompileErrorDiffers, compileInfo.fullMsg) 202 | result = FAILED 203 | break escapeBlock 204 | 205 | # Lets also check file size here as it kinda belongs to the 206 | # compilation result 207 | if test.maxSize != 0: 208 | var size = getFileSize(binary) 209 | if size > test.maxSize: 210 | logFailure(test, FileSizeTooLarge, $size) 211 | result = FAILED 212 | break escapeBlock 213 | 214 | result = OK 215 | finally: 216 | close(p) 217 | 218 | proc threadedExecute(payload: ThreadPayload) {.thread.} 219 | 220 | proc spawnTest(child: var Thread[ThreadPayload]; test: TestSpec; 221 | core: int): bool = 222 | ## invoke a single test on the given thread/core; true if we 223 | ## pinned the test to the given core 224 | assert core >= 0 225 | child.createThread(threadedExecute, 226 | ThreadPayload(core: core, spec: test)) 227 | # set cpu affinity if requested (and cores remain) 228 | if CpuAffinity in test.config.flags: 229 | if core < countProcessors(): 230 | child.pinToCpu core 231 | result = true 232 | 233 | proc execute(test: TestSpec): TestStatus = 234 | ## invoke a single test and return a status 235 | var 236 | # FIXME: pass a backend 237 | cmd = test.binary 238 | # output the test stage if necessary 239 | if test.stage.len > 0: 240 | echo 20.spaces & test.stage 241 | 242 | if not fileExists(cmd): 243 | result = FAILED 244 | logFailure(test, ExeFileNotFound) 245 | else: 246 | withinDir parentDir(test.path): 247 | cmd = cmd.quoteShell & " " & test.args 248 | let 249 | (output, exitCode) = execCmdEx(cmd) 250 | if exitCode != 0: 251 | # parseExecuteOutput() # Need to parse the run time failures? 252 | logFailure(test, RuntimeError, output) 253 | result = FAILED 254 | else: 255 | let 256 | outputs = test.composeOutputs(output) 257 | result = test.cmpOutputs(outputs) 258 | # perform an update of the testfile if requested and required 259 | if UpdateOutputs in test.config.flags and result == FAILED: 260 | test.rewriteTestFile(outputs) 261 | # we'll call this a `skip` because it's not strictly a failure 262 | # and we want any dependent testing to proceed as usual. 263 | result = SKIPPED 264 | 265 | proc executeAll(test: TestSpec): TestStatus = 266 | ## run a test and any dependent children, yielding a single status 267 | when false and compileOption("threads"): 268 | # TODO parallel execution disabled because `threadedExecute` raises an 269 | # uncaught exception on failure causing ntu to crash 270 | try: 271 | var 272 | thread: TestThread 273 | # we spawn and join the test here so that it can receive 274 | # cpu affinity via the standard thread.pinToCpu method 275 | discard thread.spawnTest(test, 0) 276 | thread.joinThreads 277 | except: 278 | # any thread(?) exception is a failure 279 | result = FAILED 280 | else: 281 | # unthreaded serial test execution 282 | result = SKIPPED 283 | var test = test 284 | while test != nil and result in {OK, SKIPPED}: 285 | result = test.execute 286 | test = test.child 287 | 288 | proc threadedExecute(payload: ThreadPayload) {.thread.} = 289 | ## a thread in which we'll perform a test execution given the payload 290 | var 291 | result = FAILED 292 | if payload.spec.child == nil: 293 | {.gcsafe.}: 294 | result = payload.spec.execute 295 | else: 296 | try: 297 | var 298 | child: TestThread 299 | discard child.spawnTest(payload.spec.child, payload.core + 1) 300 | {.gcsafe.}: 301 | result = payload.spec.execute 302 | child.joinThreads 303 | except: 304 | result = FAILED 305 | if result == FAILED: 306 | raise newException(CatchableError, payload.spec.stage & " failed") 307 | 308 | proc optimizeOrder(tests: seq[TestSpec]; 309 | order: set[SortBy]): seq[TestSpec] = 310 | ## order the tests by how recently each was modified 311 | template whenWritten(path: string): Time = 312 | path.getFileInfo(followSymlink = true).lastWriteTime 313 | 314 | result = tests 315 | for s in SortBy.low .. SortBy.high: 316 | if s in order: 317 | case s 318 | of Test: 319 | result = result.sortedByIt it.path.whenWritten 320 | of Source: 321 | result = result.sortedByIt it.source.whenWritten 322 | of Reverse: 323 | result.reverse 324 | of Random: 325 | result.shuffle 326 | 327 | proc scanTestPath(path: string): seq[string] = 328 | ## add any tests found at the given path 329 | if fileExists(path): 330 | result.add path 331 | else: 332 | for file in walkDirRec path: 333 | if file.endsWith ".test": 334 | result.add file 335 | 336 | proc test(test: TestSpec; backend: string): TestStatus = 337 | let 338 | config = test.config 339 | var 340 | duration: float 341 | 342 | try: 343 | time duration: 344 | # perform all tests in the test file 345 | result = test.executeAll 346 | finally: 347 | logResult(test.name, result, duration) 348 | 349 | proc buildBackendTests(config: TestConfig; 350 | tests: seq[TestSpec]): BackendTests = 351 | ## build the table mapping backend to test inputs 352 | result = newTable[string, seq[TestSpec]](4) 353 | for spec in tests.items: 354 | for backend in config.backends.items: 355 | assert backend != "" 356 | if backend in result: 357 | if spec notin result[backend]: 358 | result[backend].add spec 359 | else: 360 | result[backend] = @[spec] 361 | 362 | proc removeCaches(config: TestConfig; backend: string) = 363 | ## cleanup nimcache directories between backend runs 364 | removeDir config.cache(backend) 365 | 366 | # we want to run tests on "native", first. 367 | proc performTesting(config: TestConfig; 368 | backend: string; tests: seq[TestSpec]): TestStatus = 369 | var 370 | successful, skipped, invalid, failed = 0 371 | dedupe: CountTable[Hash] 372 | 373 | assert backend != "" 374 | 375 | # perform each test in an optimized order 376 | for spec in tests.optimizeOrder(config.orderBy).items: 377 | 378 | block escapeBlock: 379 | if spec.program.len == 0: 380 | # a program name is bare minimum of a test file 381 | result = INVALID 382 | invalid.inc 383 | logResult(spec.program & " for " & spec.name, result) 384 | break escapeBlock 385 | 386 | if spec.skip or hostOS notin spec.os or config.shouldSkip(spec.name): 387 | result = SKIPPED 388 | skipped.inc 389 | logResult(spec.program & " for " & spec.name, result) 390 | break escapeBlock 391 | 392 | let 393 | build = spec.binaryHash(backend) 394 | if build notin dedupe: 395 | dedupe.inc build 396 | # compile the test program for all backends 397 | var 398 | duration: float 399 | try: 400 | time duration: 401 | result = compile(spec, backend) 402 | if result != OK: 403 | failed.inc 404 | break escapeBlock 405 | finally: 406 | logResult("compiled " & spec.program & " for " & spec.name, 407 | result, duration) 408 | 409 | if result == OK: 410 | case spec.test(backend) 411 | of OK: 412 | successful.inc 413 | of SKIPPED: 414 | skipped.inc 415 | of FAILED: 416 | failed.inc 417 | of INVALID: 418 | invalid.inc 419 | 420 | let nonSuccesful = skipped + invalid + failed 421 | styledEcho(styleBright, "Finished run for $#: $#/$# OK, $# SKIPPED, $# FAILED, $# INVALID" % 422 | [backend, $successful, $(tests.len), 423 | $skipped, $failed, $invalid]) 424 | 425 | for spec in tests.items: 426 | try: 427 | # this may fail in 64-bit AppVeyor images with "The process cannot 428 | # access the file because it is being used by another process. 429 | # [OSError]" 430 | let 431 | fn = spec.binary(backend) 432 | if fileExists(fn): 433 | removeFile(fn) 434 | except CatchableError as e: 435 | echo e.msg 436 | 437 | if 0 == tests.len - successful - nonSuccesful: 438 | config.removeCaches(backend) 439 | 440 | if failed != 0: 441 | result = FAILED 442 | elif invalid != 0: 443 | result = INVALID 444 | else: 445 | result = OK 446 | 447 | proc main(): int = 448 | let config = processArguments() 449 | 450 | case config.cmd 451 | of Command.test: 452 | let testFiles = scanTestPath(config.path) 453 | if testFiles.len == 0: 454 | styledEcho(styleBright, "No test files found") 455 | result = 1 456 | else: 457 | var 458 | tests = testFiles.mapIt config.parseTestFile(it) 459 | backends = config.buildBackendTests(tests) 460 | 461 | # c > cpp > js 462 | for backend in backendOrder: 463 | assert backend != "" 464 | # if we actually need to do anything on the given backend 465 | if backend notin backends: 466 | continue 467 | let 468 | tests = backends[backend] 469 | try: 470 | if OK != config.performTesting(backend, tests): 471 | quit QuitFailure 472 | finally: 473 | backends.del(backend) 474 | 475 | for backend, tests in backends.pairs: 476 | assert backend != "" 477 | if OK != config.performTesting(backend, tests): 478 | quit QuitFailure 479 | of Command.fuzz: 480 | runFuzzer(config.target, config.fuzzer, config.corpusDir) 481 | of noCommand: 482 | discard 483 | 484 | when isMainModule: 485 | quit main() 486 | --------------------------------------------------------------------------------