├── tests ├── empty.toml ├── invalid_path.toml ├── echo_user.sh ├── infinite_loop.sh ├── file_to_test_stdin.txt ├── non_executable.toml ├── invalid_test.toml ├── infinite_loop.toml ├── invalid_test_key.toml ├── print_stdin.py ├── test_bad_types.toml ├── test_build_command.toml ├── invalid_output_file.toml ├── duplicate_stderr.toml ├── duplicate_stdout.toml ├── echo_user_test.toml └── test_stdin.toml ├── requirements.txt ├── Dockerfile ├── install.sh ├── update_version.sh ├── .github └── workflows │ └── pythonapp.yml ├── LICENSE ├── test_jenrik.toml ├── README.md └── jenrik /tests/empty.toml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | toml 2 | colored 3 | termcolor 4 | -------------------------------------------------------------------------------- /tests/invalid_path.toml: -------------------------------------------------------------------------------- 1 | binary_path = "./toitoi" 2 | -------------------------------------------------------------------------------- /tests/echo_user.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo $USER 4 | -------------------------------------------------------------------------------- /tests/infinite_loop.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | sleep inf 3 | -------------------------------------------------------------------------------- /tests/file_to_test_stdin.txt: -------------------------------------------------------------------------------- 1 | Hello 2 | world 3 | 1 4 | 2 5 | 3 6 | -------------------------------------------------------------------------------- /tests/non_executable.toml: -------------------------------------------------------------------------------- 1 | binary_path = "./non_executable.toml" 2 | -------------------------------------------------------------------------------- /tests/invalid_test.toml: -------------------------------------------------------------------------------- 1 | binary_path = "../jenrik" 2 | 3 | my_bad_test = "toto" 4 | -------------------------------------------------------------------------------- /tests/infinite_loop.toml: -------------------------------------------------------------------------------- 1 | binary_path = "./infinite_loop.sh" 2 | 3 | [invalid_test] 4 | args = [] 5 | status = 0 6 | -------------------------------------------------------------------------------- /tests/invalid_test_key.toml: -------------------------------------------------------------------------------- 1 | binary_path = "../jenrik" 2 | 3 | [invalid_test] 4 | args = ["-h"] 5 | status = 1 6 | toto = 12 7 | -------------------------------------------------------------------------------- /tests/print_stdin.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | 5 | for line in sys.stdin: 6 | print(line, end="") 7 | -------------------------------------------------------------------------------- /tests/test_bad_types.toml: -------------------------------------------------------------------------------- 1 | binary_path = "../jenrik" 2 | 3 | [invalid_env] 4 | args = ["-h"] 5 | status = 1 6 | env = 123 7 | -------------------------------------------------------------------------------- /tests/test_build_command.toml: -------------------------------------------------------------------------------- 1 | binary_path = "../jenrik" 2 | 3 | build_command = 12 4 | 5 | [testtoto] 6 | args = [] 7 | status = 1 8 | -------------------------------------------------------------------------------- /tests/invalid_output_file.toml: -------------------------------------------------------------------------------- 1 | binary_path = "../jenrik" 2 | 3 | [invalid_test] 4 | args = ["-h"] 5 | status = 1 6 | stderr_file="./README.md" 7 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:latest 2 | WORKDIR /app/ 3 | COPY . /app/ 4 | RUN pip3 install -r ./requirements.txt 5 | CMD ["./jenrik", "./test_jenrik.toml"] 6 | -------------------------------------------------------------------------------- /tests/duplicate_stderr.toml: -------------------------------------------------------------------------------- 1 | binary_path = "../jenrik" 2 | 3 | [invalid_test] 4 | args = ["-h"] 5 | status = 1 6 | stderr_file="./toto" 7 | stderr="tutu" 8 | -------------------------------------------------------------------------------- /tests/duplicate_stdout.toml: -------------------------------------------------------------------------------- 1 | binary_path = "../jenrik" 2 | 3 | [invalid_test] 4 | args = ["-h"] 5 | status = 1 6 | stdout_file="./toto" 7 | stdout="tutu" 8 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if (( $EUID != 0 )); then 4 | echo "Please run as root" 5 | exit 1 6 | fi 7 | 8 | sudo cp ./jenrik /usr/bin/jenrik 9 | echo "Done !" 10 | -------------------------------------------------------------------------------- /tests/echo_user_test.toml: -------------------------------------------------------------------------------- 1 | binary_path = "echo_user.sh" 2 | 3 | [test] 4 | args = [] 5 | status = 0 6 | stdout="jenrik\n" 7 | env.USER = "jenrik" 8 | 9 | [test_with_add_env] 10 | args = [] 11 | status = 0 12 | stdout="jenrik;toto\n" 13 | env.USER = "jenrik" 14 | add_env.USER = ";toto" 15 | -------------------------------------------------------------------------------- /tests/test_stdin.toml: -------------------------------------------------------------------------------- 1 | binary_path = "./print_stdin.py" 2 | 3 | [test_stdin] 4 | args = [] 5 | status = 0 6 | stdin = ["Hello", "world"] 7 | stdout = "Hello\nworld\n" 8 | timeout = 1 9 | 10 | [test_stdin_file] 11 | args = [] 12 | status = 0 13 | stdin_file = "file_to_test_stdin.txt" 14 | stdout = "Hello\nworld\n1\n2\n3\n\n" 15 | timeout = 1 16 | -------------------------------------------------------------------------------- /update_version.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # There are two places where we need to change the version number 4 | # - in the README 5 | # - in the jenrik source code 6 | 7 | # First we get the current version from the readme 8 | current_version=$(grep version README.md -A 2 | grep "^v" | cut -d ' ' '-f2-') 9 | 10 | # The new_version is the current one + 0.1 11 | new_version=$(echo ${current_version} + 0.01 | bc) 12 | 13 | # Now we save it 14 | 15 | # In README 16 | sed -i "" "s/${current_version}/${new_version}/" README.md 17 | 18 | # In jenrik 19 | sed -i "" "s/${current_version}/${new_version}/" jenrik 20 | -------------------------------------------------------------------------------- /.github/workflows/pythonapp.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: JenRik 5 | 6 | on: 7 | push: 8 | branches: [ master, dev ] 9 | pull_request: 10 | branches: [ master, dev ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Set up Python 3.8 20 | uses: actions/setup-python@v1 21 | with: 22 | python-version: 3.8 23 | - name: Install dependencies 24 | run: | 25 | python -m pip install --upgrade pip 26 | pip install -r requirements.txt 27 | - name: Test with JenRik 28 | run: | 29 | ./jenrik test_jenrik.toml 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Yohann 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /test_jenrik.toml: -------------------------------------------------------------------------------- 1 | binary_path = "./jenrik" 2 | 3 | [invalid_arg] 4 | args = ["-h"] 5 | status = 1 6 | 7 | [no_args] 8 | args = [] 9 | status = 1 10 | 11 | [init_only] 12 | args = ["init"] 13 | status = 1 14 | 15 | [invalid_test_file] 16 | args = ["toito"] 17 | stderr = "toito : file not found\n" 18 | status = 1 19 | 20 | [invalid_test_file_but_toml] 21 | args = ["toito.toml"] 22 | stderr = "toito.toml : file not found\n" 23 | status = 1 24 | 25 | [overwrite_init_test_jenrik_toml] 26 | args = ["init", "jenrik"] 27 | stderr = "test_jenrik.toml already exists, can't init the file\n" 28 | status = 1 29 | 30 | [test_working_init] 31 | args = ["init", "my_bin"] 32 | status = 0 33 | post = "rm -f test_my_bin.toml" 34 | 35 | [empty_toml] 36 | args = ["tests/empty.toml"] 37 | status = 1 38 | stderr = "Could not find binary_path key in tests/empty.toml\n" 39 | 40 | [invalid_binary_path] 41 | args = ["tests/invalid_path.toml"] 42 | status = 1 43 | stderr = "./toitoi : file not found\n" 44 | 45 | [invalid_test_key] 46 | args = ["tests/invalid_test_key.toml"] 47 | status = 1 48 | stderr = "invalid_test: Unknown key : toto\n" 49 | 50 | [duplicate_stdout] 51 | args = ["tests/duplicate_stdout.toml"] 52 | status = 1 53 | stderr = "invalid_test: Incompatible keys, 'stdout' and 'stdout_file'\n" 54 | 55 | [duplicate_stderr] 56 | args = ["tests/duplicate_stderr.toml"] 57 | status = 1 58 | stderr = "invalid_test: Incompatible keys, 'stderr' and 'stderr_file'\n" 59 | 60 | [bad_test] 61 | args = ["tests/invalid_test.toml"] 62 | status = 1 63 | stderr = "Invalid test : 'my_bad_test toto'\n" 64 | 65 | [invalid_ouput_file] 66 | args = ["tests/invalid_output_file.toml"] 67 | status = 1 68 | stderr = "tests/README.md : file not found\n" 69 | 70 | [test_pipe_basic_stdout] 71 | args = [] 72 | status = 1 73 | pipe_stdout = "| grep \"USAGE\"" 74 | stdout = "USAGE : ./jenrik file.jrk | init path_to_binary\n" 75 | 76 | [test_pipe_two_pipes_stdout] 77 | args = [] 78 | status = 1 79 | pipe_stdout = "| grep \"USAGE\" | cut -d ' ' -f1" 80 | stdout = "USAGE\n" 81 | 82 | [test_pipe_stderr] 83 | args = ["tests/duplicate_stderr.toml"] 84 | status = 1 85 | pipe_stderr = "| cut -d ':' -f1" 86 | stderr = "invalid_test\n" 87 | 88 | [test_timeout_success] 89 | args = [] 90 | status = 1 91 | timeout= 0.1 92 | 93 | [test_timeout_fail] 94 | args = [] 95 | status = 1 96 | timeout= 0 97 | should_fail = true 98 | 99 | [test_timeout_loop] 100 | args = ["tests/infinite_loop.toml"] 101 | status = 0 102 | timeout = 0.5 103 | should_fail = true 104 | 105 | [test_status_should_fail] 106 | args = [] 107 | status = 0 108 | should_fail = true 109 | 110 | [test_stdin] 111 | args = ["tests/test_stdin.toml"] 112 | status = 0 113 | 114 | [test_env] 115 | args = ["tests/echo_user_test.toml"] 116 | status = 0 117 | 118 | [test_bad_env_values] 119 | args = ["tests/test_bad_types.toml"] 120 | status = 1 121 | 122 | [test_repeat_5_times] 123 | args = [] 124 | status = 1 125 | repeat = 5 126 | 127 | [test_invalid_build_command] 128 | args = ["tests/test_build_command.toml"] 129 | status = 1 130 | stderr = "build_command value must be a string\n" 131 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # JenRik 2 | 3 | JenRik is a simple but powerful testing tool. 4 | 5 | ![](https://github.com/Yohannfra/JenRik/workflows/JenRik/badge.svg) 6 | 7 | The main idea was to write a generic binary testing tool.\ 8 | JenRik simply parse a [toml](https://github.com/toml-lang/toml) 9 | file containing the tests and run them. 10 | 11 | ## Installation 12 | 13 | #### Dependencies: 14 | The dependencies are the python parser for toml and termcolor. Install both with: 15 | ``` 16 | pip install -r requirements.txt 17 | ``` 18 | 19 | #### Installation from sources: 20 | ```bash 21 | git clone https://github.com/Yohannfra/JenRik 22 | cd JenRik 23 | sudo ./install.sh 24 | ``` 25 | 26 | ## Quick Start 27 | 28 | ##### Let's say we need to test this basic python script. 29 | 30 | my_prog.py : 31 | ```python 32 | #!/usr/bin/python3 33 | import sys 34 | 35 | if len(sys.argv) == 1: 36 | print("No arguments given") 37 | exit(1) 38 | else: 39 | print(sys.argv[1]) 40 | ``` 41 | 42 | ##### First we need to initialize a test file for our my_prog.py 43 | 44 | ```bash 45 | $ jenrik init ./my_prog.py 46 | ``` 47 | 48 | It will create a *test_my_prog.toml* file with this content: 49 | ```toml 50 | binary_path = "my_prog.py" 51 | 52 | # A sample test 53 | [test1] 54 | args = ["-h"] 55 | status = 0 56 | stdout="" 57 | stderr="" 58 | ``` 59 | 60 | The first line ```binary_path = "my_prog.py"``` indicates the path to the binary to test. 61 | 62 | The line ```[test1]``` define a test and the following values are parts of it: 63 | ```toml 64 | args = ["-h"] # the command line arguments 65 | status = 0 # the expected exit status 66 | stdout="" # the expected stdout (not tested if empty) 67 | stderr="" # the expected stderr (not tested if empty) 68 | ``` 69 | 70 | Now we can write the tests for our my_prog.py. First delete the sample test and write this instead 71 | ```toml 72 | [test_no_arguments] 73 | args = [] 74 | status = 1 75 | stdout="No arguments given\n" 76 | 77 | [with_args] 78 | args = ["Hello"] 79 | status = 0 80 | stdout="Hello\n" 81 | stderr="" 82 | ``` 83 | 84 | ⚠ **Each test requires at least the args and status values !** 85 | 86 | There are many other available commands: 87 | - **pre** : run a shell command before executing the test 88 | - **post** : run a shell command after executing the test 89 | - **stderr_file** : compare your program stderr with the content of a given file 90 | - **stdout_file** : compare your program stdout with the content of a given file 91 | - **pipe_stdout** : redirect your program stdout to a specified shell command before checking it 92 | - **pipe_stderr** : redirect your program stderr to a specified shell command before checking it 93 | - **should_fail** : make the test success if it fails 94 | - **timeout** : make the test fail if it times out, after killing it (SIGTERM) (the time is given in seconds) 95 | - **stdin** : write in the stdin of the process 96 | - **stdin_file** : write in the stdin of the process from the content of a file 97 | - **env** : change environment variable(s) (replace the value with the given one) 98 | - **add_env** : change environment variable(s) (append the given value to environment value) 99 | - **repeat** : repeat the test x times 100 | 101 | ### Example of how to use some commands 102 | 103 | - **pre** is usefull if you need to prepare a file needed by your programm for a test 104 | - **post** is mainly usefull to cleanup after a test 105 | - **stderr_file** and **stdout_file** are usefull if the output of you program is on multiples lines or if it's a lot of text and you don't want it written in you test file. 106 | - **should_fail** is used if you want a test fail to be its success 107 | - **add_env** is mainly used for environment variables like PATH (when you want to append a value to the existing one) 108 | - **repeat** is super usefull if you want to test a proram that relies on some random data. It makes it easier to run many tests to check if it's always working 109 | 110 | ⚠ **Don't forget that the paths are all relatives to the test file.** 111 | 112 | If you want more examples on how to write tests you should see this [file](test_jenrik.toml) 113 | 114 | Here is a quick example of how to use all availables commands 115 | 116 | ```toml 117 | # args 118 | args = [] 119 | args = ["-h"] 120 | args = ["1", "2", "3"] 121 | 122 | # status 123 | status = 1 124 | 125 | # stdout 126 | stdout="Hello\n" 127 | 128 | # stderr 129 | stderr="Hello err\n" 130 | 131 | # pre 132 | pre = "touch test.txt && echo 'hello' > test.txt" 133 | 134 | # post 135 | post = "rm -f test.txt" 136 | 137 | # stderr_file 138 | stderr_file = "./my_file.txt" 139 | 140 | # stdout_file 141 | stdout_file = "./my_file.txt" 142 | 143 | # pipe_stdout 144 | pipe_stdout = "| grep 'Usage'" 145 | 146 | # pipe_stderr 147 | pipe_stderr = "| cut -d ':' -f1" 148 | 149 | # should_fail (true or false) 150 | should_fail = true 151 | 152 | # timeout (in seconds) 153 | timeout = 0.4 154 | 155 | # stdin 156 | stdin = ["Hello", "World"] 157 | 158 | # stdin_file 159 | stdin_file = "my_stdin.txt" 160 | 161 | # env 162 | env.USER = "toto" 163 | env.TERM = "xterm" 164 | 165 | # add_env 166 | add_env.PATH= ":~/.my_folder" 167 | 168 | # repeat 169 | repeat = 12 170 | ``` 171 | 172 | See [Usage](#Usage) to run the tests 173 | 174 | ## Usage 175 | Once you have written the test file you just have to : 176 | ``` 177 | jenrik test_my_prog.toml 178 | ``` 179 | 180 | The output will look like that 181 | ``` 182 | test_no_arguments : OK 183 | with_args : OK 184 | 185 | Summary ./my_prog.py: 2 tests ran 186 | 2 : OK 187 | 0 : KO 188 | ``` 189 | 190 | 191 | ### Build 192 | 193 | If you want to build your programm when calling jenrik you can use the **build_command** option.\ 194 | It will run it before executing the tests. eg: 195 | 196 | ```toml 197 | binary_path = "./my_program" 198 | 199 | build_command = "make" 200 | 201 | [test_example] 202 | args = [] 203 | status = 1 204 | # ... 205 | ``` 206 | 207 | ### Exit status 208 | 209 | If a parsing error or a configuration error occurs then the exit status is 1\ 210 | Otherwise the exit status is the number of failed tests 211 | 212 | ## Tests 213 | 214 | JenRik tests itself. 215 | ``` 216 | jenrik test_jenrik.toml 217 | ``` 218 | 219 | You can also run JenRik tests within a docker 220 | ``` 221 | $ docker build -t image_jenrik . 222 | $ docker run image_jenrik:latest 223 | ``` 224 | 225 | ## Current version 226 | ``` 227 | v 1.10 228 | ``` 229 | 230 | ## License 231 | This project is licensed under the terms of the MIT license. 232 | -------------------------------------------------------------------------------- /jenrik: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import toml 5 | import os 6 | import subprocess 7 | import colored 8 | from colored import stylize 9 | import shlex 10 | import time 11 | 12 | JENRIK_VERSION = "1.10" 13 | 14 | TESTS_KEYS = [ 15 | {'name' : 'args', 'type': [list]}, 16 | {'name' : 'status', 'type': [int]}, 17 | {'name' : 'stdout', 'type': [str]}, 18 | {'name' : 'stderr', 'type': [str]}, 19 | {'name' : 'pre', 'type': [str, list]}, 20 | {'name' : 'post', 'type': [str, list]}, 21 | {'name' : 'stdout_file', 'type': [str]}, 22 | {'name' : 'stderr_file', 'type': [str]}, 23 | {'name' : 'pipe_stdout', 'type': [str]}, 24 | {'name' : 'pipe_stderr', 'type': [str]}, 25 | {'name' : 'timeout', 'type': [int, float]}, 26 | {'name' : 'should_fail', 'type': [bool]}, 27 | {'name' : 'stdin', 'type': [list]}, 28 | {'name' : 'stdin_file', 'type': [str]}, 29 | {'name' : 'env', 'type': [dict]}, 30 | {'name' : 'add_env', 'type': [dict]}, 31 | {'name' : 'repeat', 'type': [int]} 32 | ] 33 | 34 | REQUIERED_KEYS = ['args', 'status'] 35 | 36 | INCOMPATIBLES_KEYS = [('stdout', 'stdout_file'), 37 | ('stderr', 'stderr_file'), 38 | ('stdin', 'stdin_file')] 39 | 40 | # 0 Normal | 1 Quiet | 2 Super Quiet 41 | QUIET_LEVEL = 0 42 | 43 | def print_help(binary_name): 44 | """ Print a basic help showing how to use Jenerik """ 45 | print(f"USAGE : {binary_name} file.jrk | init path_to_binary") 46 | print("\tinit\t\tcreate a basic test file for the given binary") 47 | print("\t--version\tprint version information and exit") 48 | print("\t--help\tprint this help and exit") 49 | print("\t-q\trun in quiet mode (doesn't show the diffs)") 50 | print("\t-qq\trun in super quiet mode (just print the file name and OK/KO)") 51 | 52 | def get_file_content(fp): 53 | """ open a file and return its content """ 54 | if os.path.exists(fp): 55 | if not os.access(fp, os.R_OK): 56 | sys.exit(f"{fp} : is not readable") 57 | elif os.path.isdir(fp): 58 | sys.exit(f"{fp} : is a directory") 59 | else: 60 | sys.exit(f"{fp} : file not found") 61 | try: 62 | f = open(fp, 'r') 63 | fc = f.read() 64 | f.close() 65 | except: 66 | sys.exit(f"{fp} : could not open and read file") 67 | return fc 68 | 69 | def init_file(fp): 70 | """ Create a default test file """ 71 | test_file_name = 'test_' + fp + '.toml' 72 | 73 | default_file_content = [ 74 | f"binary_path = \"{fp}\"\n\n", 75 | "# A sample test\n", 76 | "[test1]\n", 77 | "args = [\"-h\"]\n", 78 | "status = 0\n", 79 | "stdout=\"\"\n", 80 | "stderr=\"\"\n", 81 | ] 82 | 83 | if os.path.exists(test_file_name): 84 | sys.exit(f"{test_file_name} already exists, can't init the file") 85 | try: 86 | f = open(test_file_name, 'w') 87 | except: 88 | sys.exit(f"Could not create file {test_file_name}") 89 | for line in default_file_content: 90 | f.write(line) 91 | f.close() 92 | print(f"Initialized {test_file_name} with success") 93 | 94 | 95 | def check_binary_validity(binary_path, relative_path): 96 | """ Check if the binary path is a valid executable file """ 97 | if os.path.exists(relative_path + binary_path): 98 | if not os.access(relative_path + binary_path, os.X_OK): 99 | sys.exit(f"{binary_path} : is not executable") 100 | elif os.path.isdir(relative_path + binary_path): 101 | sys.exit(f"{binary_path} : is a directory") 102 | else: 103 | sys.exit(f"{binary_path} : file not found") 104 | 105 | 106 | def check_values_validities(test_name, values): 107 | """ check if all keys values have the good type """ 108 | for key in TESTS_KEYS: 109 | if key['name'] not in values.keys(): 110 | continue 111 | good_type = False 112 | for t in key['type']: 113 | if type(values[key['name']]) == t: 114 | good_type = True 115 | break 116 | if good_type == False: 117 | sys.exit(f"{test_name}: {key['name']} value type must be in {key['type']}") 118 | 119 | for key_pair in INCOMPATIBLES_KEYS: 120 | if key_pair[0] in values.keys() and key_pair[1] in values.keys(): 121 | sys.exit(f"{test_name}: Incompatible keys, '{key_pair[0]}' and '{key_pair[1]}'") 122 | 123 | 124 | def check_tests_validity(test_name, values): 125 | """ Check if all the fieds of the test are known and are valids.""" 126 | if type(values) != dict: 127 | sys.exit(f"Invalid test : '{test_name} {values}'") 128 | for key in REQUIERED_KEYS: 129 | if key not in values.keys(): 130 | sys.exit(test_name + ": Missing field : " + key) 131 | 132 | for key in values: 133 | if key not in [d['name'] for d in TESTS_KEYS]: 134 | sys.exit(f"{test_name}: Unknown key : {key}") 135 | check_values_validities(test_name, values) 136 | 137 | 138 | def run_build_command(build_command): 139 | """ run the build command """ 140 | if type(build_command) != str: 141 | sys.exit(f"build_command value must be a string") 142 | os.system(build_command) 143 | 144 | def check_test_file_validity(content, fp, relative_path): 145 | """ Check if the toml test file is valid """ 146 | binary_path = "" 147 | test_suite = {} 148 | 149 | for key in content.keys(): 150 | if key == "binary_path": 151 | binary_path = content[key] 152 | check_binary_validity(binary_path, relative_path) 153 | elif key == "build_command": 154 | run_build_command(content[key]) 155 | else: 156 | check_tests_validity(key, content[key]) 157 | test_suite[key] = content[key] 158 | 159 | if binary_path == "": 160 | sys.exit(f"Could not find binary_path key in {fp}") 161 | 162 | return (relative_path + binary_path), test_suite 163 | 164 | 165 | class Tester: 166 | """ The class containing everything to run the tests """ 167 | 168 | def __init__(self, binary_path, test_suite, relative_path): 169 | self.test_suite = test_suite 170 | self.binary_path = binary_path 171 | self.count_tests = 0 172 | self.test_should_fail = -1 173 | self.count_failed_tests = 0 174 | self.relative_path = relative_path 175 | 176 | 177 | def print_test_sucess(self): 178 | """ print a message if test success """ 179 | if QUIET_LEVEL == 2: 180 | return 181 | if self.test_should_fail == 0: 182 | return self.print_test_failed("Test should have failed") 183 | print(stylize('OK', colored.fg('green'))) 184 | 185 | 186 | def print_test_failed(self, e): 187 | """ print a message if test fails """ 188 | if QUIET_LEVEL == 2: 189 | return 190 | if self.test_should_fail == 1: 191 | return self.print_test_sucess() 192 | self.count_failed_tests += 1 193 | print(stylize('KO', colored.fg('red')), end=" : ") 194 | print(e) 195 | 196 | 197 | def print_diff(self, t1, t2): 198 | if QUIET_LEVEL > 0: 199 | return 200 | len1 = len(t1) 201 | len2 = len(t2) 202 | len_max = len1 if len1 > len2 else len2 203 | print("-" * 30) 204 | print("Expected:") 205 | sys.stdout.write("'") 206 | for c in range(len1): 207 | if c < len2: 208 | sys.stdout.write(t1[c]) 209 | else: 210 | sys.stdout.write(stylize(t1[c], colored.fg("green"))) 211 | sys.stdout.write("'\n") 212 | print("But got:") 213 | sys.stdout.write("'") 214 | for i in range(len_max): 215 | if i > len2 - 1: 216 | pass 217 | elif i < len1 and t1[i] == t2[i]: 218 | sys.stdout.write(t2[i]) 219 | else: 220 | sys.stdout.write(stylize(t2[i], colored.fg('red'))) 221 | sys.stdout.write("'\n") 222 | print("-" * 30) 223 | 224 | def comp_output_file(self, output_file, output, output_name): 225 | """ compare an output with a given file """ 226 | output_file = (self.relative_path + output_file).replace('/./', '/') 227 | fc = get_file_content(output_file) 228 | if output != fc: 229 | self.print_test_failed(f"Invalid {output_name}") 230 | self.print_diff(fc, output) 231 | return True 232 | return False 233 | 234 | 235 | def apply_pipe(self, output, pipe): 236 | """ apply a pipe command on a given output """ 237 | if pipe == "": 238 | return output 239 | output = os.popen('echo ' + shlex.quote(output.rstrip("\n")) + ' ' + pipe).read() 240 | return output 241 | 242 | 243 | def check_test_results(self, values, stdout, stderr, status): 244 | """ check the tests results """ 245 | if 'pipe_stdout' in values: 246 | stdout = self.apply_pipe(stdout, values['pipe_stdout']) 247 | if 'pipe_stderr' in values: 248 | stderr = self.apply_pipe(stderr, values['pipe_stderr']) 249 | 250 | if values['status'] != status: 251 | self.print_test_failed("Invalid exit status, " 252 | f"expected {values['status']} but got {status}") 253 | elif 'stdout' in values and values['stdout'] != "" \ 254 | and values['stdout'] != stdout: 255 | self.print_test_failed("Invalid stdout") 256 | self.print_diff(values['stdout'], stdout) 257 | elif 'stderr' in values and values['stderr'] != "" \ 258 | and values['stderr'] != stderr: 259 | self.print_test_failed("Invalid stderr") 260 | self.print_diff(values['stderr'], stderr) 261 | elif 'stdout_file' in values and self.comp_output_file(values['stdout_file'], stdout, 'stdout'): 262 | pass 263 | elif 'stderr_file' in values and self.comp_output_file(values['stderr_file'], stderr, 'stderr'): 264 | pass 265 | else: 266 | self.print_test_sucess() 267 | 268 | def run_pre_post_command(self, command): 269 | """ run pre and post commands """ 270 | if type(command) == str and command != "": 271 | os.system(command) 272 | elif type(command) == list and command != []: 273 | for c in command: 274 | if c != "" and type(command) == str: 275 | os.system(c) 276 | 277 | 278 | def fill_env(self, values): 279 | my_env = os.environ.copy() 280 | if 'env' in values: 281 | for v in values['env'].keys(): 282 | my_env[v] = values['env'][v] 283 | if 'add_env' in values: 284 | for v in values['add_env'].keys(): 285 | my_env[v] = my_env[v] + values['add_env'][v] 286 | return my_env 287 | 288 | 289 | def fill_stdin(self, values, process): 290 | """ fill the process stdin """ 291 | if 'stdin' in values: 292 | for v in values['stdin']: 293 | process.stdin.write((v + "\n").encode()) 294 | elif 'stdin_file' in values: 295 | input_file = (self.relative_path + values['stdin_file']).replace('/./', '/') 296 | fc = get_file_content(input_file) 297 | for line in fc.split('\n'): 298 | process.stdin.write((line + "\n").encode()) 299 | 300 | def run_test(self, values, test_name, repeat_count): 301 | """ run the test in a subprocess """ 302 | self.count_tests += 1 303 | 304 | if 'pre' in values: 305 | self.run_pre_post_command(values['pre']) 306 | my_env = self.fill_env(values) 307 | test_args = [self.binary_path] + values['args'] 308 | try: 309 | process = subprocess.Popen(test_args, stdin=subprocess.PIPE, 310 | stdout=subprocess.PIPE, stderr=subprocess.PIPE, 311 | env=my_env) 312 | except OSError as e: 313 | sys.exit(f"An error occured while executing your binary: {e}") 314 | try: 315 | if 'stdin' in values or 'stdin_file' in values: 316 | self.fill_stdin(values, process) 317 | stdout, stderr = process.communicate(timeout=values.get('timeout', None)) 318 | if 'stdin' in values : 319 | process.stdin.close() 320 | except subprocess.TimeoutExpired: 321 | process.kill() 322 | self.print_test_failed(f"Test timed out : terminated after {values['timeout']}s") 323 | else: 324 | self.check_test_results(values, stdout.decode('utf-8'), 325 | stderr.decode('utf-8'), process.returncode) 326 | if 'post' in values: 327 | self.run_pre_post_command(values['post']) 328 | if 'repeat' in values and values['repeat'] - repeat_count > 0: 329 | if QUIET_LEVEL != 2: 330 | print(f" - Repeat {repeat_count + 1} {test_name}: " , end="") 331 | self.run_test(values, test_name, repeat_count + 1) 332 | 333 | def print_summary(self): 334 | """ print a summary of the tests results """ 335 | if QUIET_LEVEL == 2: 336 | if self.count_failed_tests > 0: 337 | print(f"{self.binary_path}: {stylize('KO', colored.fg('red'))}") 338 | else: 339 | print(f"{self.binary_path}: {stylize('OK', colored.fg('green'))}") 340 | else: 341 | count_success = self.count_tests - self.count_failed_tests 342 | print(f"\nSummary {self.binary_path}: {self.count_tests} tests ran") 343 | print(f"{count_success} : {stylize('OK', colored.fg('green'))}") 344 | print(f"{self.count_failed_tests} : {stylize('KO', colored.fg('red'))}") 345 | 346 | def launch(self): 347 | """ launch the tests on the test suite """ 348 | for test in self.test_suite: 349 | self.test_should_fail = -1 350 | if 'should_fail' in self.test_suite[test].keys(): 351 | self.test_should_fail = self.test_suite[test]['should_fail'] 352 | if QUIET_LEVEL != 2: 353 | print(f"{test} : ", end='') 354 | self.run_test(self.test_suite[test], test, 0) 355 | self.print_summary() 356 | return self.count_failed_tests 357 | 358 | def start_jenrik(fp): 359 | file_content = get_file_content(fp) 360 | content = toml.loads(file_content) # Parse the toml file 361 | relative_path = "/".join(fp.split('/')[0:-1]) + '/' 362 | if '/' == relative_path and '/' not in fp: # dirty but works 363 | relative_path = './' 364 | binary_path, test_suite = check_test_file_validity(content, fp, relative_path) 365 | binary_path = binary_path.replace('././', './') 366 | tester = Tester(binary_path, test_suite, relative_path) 367 | exit(tester.launch()) 368 | 369 | def main(argc, argv): 370 | global QUIET_LEVEL 371 | 372 | if "-q" in argv: 373 | QUIET_LEVEL = 1 374 | argv.remove("-q") 375 | argc -= 1 376 | if "-qq" in argv: 377 | QUIET_LEVEL = 2 378 | argv.remove("-qq") 379 | argc -= 1 380 | 381 | if argc == 2 and argv[1] == '--version': 382 | return print(f"jenrik v{JENRIK_VERSION}") 383 | if argc == 2 and argv[1] == '--help': 384 | return print_help(argv[0]) 385 | if argc == 1 or argc > 3 or argc == 3 and argv[1] != 'init': 386 | print_help(argv[0]) 387 | exit(1) 388 | if argc == 3: 389 | init_file(argv[2]) 390 | elif argc == 2: 391 | start_jenrik(argv[1]) 392 | 393 | if __name__ == '__main__': 394 | main(len(sys.argv), sys.argv) 395 | --------------------------------------------------------------------------------