├── .coveragerc ├── .github └── workflows │ ├── linting.yml │ └── tests.yml ├── .gitignore ├── Jenkinsfile ├── LICENSE ├── Makefile ├── README.md ├── kirk ├── libkirk ├── __init__.py ├── data.py ├── events.py ├── export.py ├── framework.py ├── host.py ├── io.py ├── ltp.py ├── ltx.py ├── ltx_sut.py ├── main.py ├── monitor.py ├── plugin.py ├── qemu.py ├── results.py ├── scheduler.py ├── session.py ├── ssh.py ├── sut.py ├── tempfile.py ├── tests │ ├── __init__.py │ ├── conftest.py │ ├── test_events.py │ ├── test_export.py │ ├── test_host.py │ ├── test_io.py │ ├── test_ltp.py │ ├── test_ltx.py │ ├── test_main.py │ ├── test_monitor.py │ ├── test_plugin.py │ ├── test_qemu.py │ ├── test_scheduler.py │ ├── test_session.py │ ├── test_ssh.py │ ├── test_sut.py │ └── test_tempfile.py └── ui.py ├── pylint.ini ├── pyproject.toml └── utils └── json2html.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | *tests* 4 | *test_* 5 | *.tox* 6 | kirk/ui.py 7 | kirk/main.py 8 | 9 | [paths] 10 | source = kirk 11 | 12 | [report] 13 | # Regexes for lines to exclude from consideration 14 | exclude_lines = 15 | # Don't complain about missing debug-only code: 16 | def __repr__ 17 | 18 | # Don't complain if tests don't hit defensive assertion code: 19 | raise AssertionError 20 | raise NotImplementedError 21 | 22 | # Don't complain about python3 support 23 | unicode = str 24 | basestring = str 25 | -------------------------------------------------------------------------------- /.github/workflows/linting.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022 Andrea Cervesato 2 | 3 | name: "Lint packages" 4 | on: [push, pull_request] 5 | 6 | jobs: 7 | build: 8 | runs-on: ubuntu-22.04 9 | 10 | strategy: 11 | fail-fast: false 12 | matrix: 13 | python-version: ["3.9"] 14 | 15 | steps: 16 | - name: Show OS 17 | run: cat /etc/os-release 18 | 19 | - name: Git checkout 20 | uses: actions/checkout@v3 21 | 22 | - name: Set up Python ${{ matrix.python-version }} 23 | uses: actions/setup-python@v4 24 | with: 25 | python-version: ${{ matrix.python-version }} 26 | 27 | - name: Install dependencies 28 | run: python3 -m pip install pylint 29 | 30 | - name: Test kirk 31 | run: ./kirk --help 32 | 33 | - name: Lint with pylint 34 | run: pylint --rcfile=pylint.ini libkirk 35 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2022 Andrea Cervesato 2 | 3 | name: "Test packages" 4 | on: [push, pull_request] 5 | 6 | env: 7 | PYTHON_PKGS: pytest<8.3.5 pytest-asyncio<1.0 build 8 | 9 | jobs: 10 | python3-deprecated: 11 | runs-on: ubuntu-22.04 12 | 13 | container: 14 | image: opensuse/leap:latest 15 | 16 | steps: 17 | - name: Show OS 18 | run: cat /etc/os-release 19 | 20 | - name: Git checkout 21 | uses: actions/checkout@v1 22 | 23 | - name: Set up Python 24 | run: zypper install -y python3-pip make 25 | 26 | - name: Install dependencies 27 | run: pip install $PYTHON_PKGS 28 | 29 | - name: Build package 30 | run: python3 -m build 31 | 32 | - name: Test with pytest 33 | run: pytest -m "not qemu and not ssh and not ltx" 34 | 35 | python3: 36 | runs-on: ubuntu-22.04 37 | 38 | strategy: 39 | fail-fast: false 40 | matrix: 41 | python-version: [ 42 | "3.7", 43 | "3.8", 44 | "3.9", 45 | "3.10", 46 | "3.11", 47 | "3.12", 48 | "3.13" 49 | ] 50 | 51 | steps: 52 | - name: Show OS 53 | run: cat /etc/os-release 54 | 55 | - name: Git checkout 56 | uses: actions/checkout@v3 57 | 58 | - name: Set up Python ${{ matrix.python-version }} 59 | uses: actions/setup-python@v4 60 | with: 61 | python-version: ${{ matrix.python-version }} 62 | 63 | - name: Install dependencies 64 | run: python3 -m pip install $PYTHON_PKGS 65 | 66 | - name: Build package 67 | run: python3 -m build 68 | 69 | - name: Test with pytest 70 | run: python3 -m pytest -m "not qemu and not ssh and not ltx" 71 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | dist/ 3 | *.pyc 4 | *.egg-info 5 | __pycache__ 6 | *.log 7 | output.json 8 | .coverage 9 | htmlcov 10 | .venv* 11 | *.qcow2 12 | .vscode 13 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | pipeline { 2 | agent { 3 | node { 4 | label "kirk" 5 | } 6 | } 7 | environment { 8 | TEST_QEMU_IMAGE = "/data/image.qcow2" 9 | TEST_QEMU_PASSWORD = "root" 10 | TEST_SSH_USERNAME = "auto" 11 | TEST_SSH_PASSWORD = "auto1234" 12 | TEST_SSH_KEY_FILE = "/data/jenkins_rsa" 13 | TEST_LTX_BINARY = "/data/ltx" 14 | } 15 | stages { 16 | stage("Test host") { 17 | steps { 18 | catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { 19 | sh 'coverage run -a -m pytest -m "not qemu and not ssh" --junit-xml=results-host.xml' 20 | } 21 | } 22 | } 23 | stage("Test SSH") { 24 | steps { 25 | catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { 26 | sh 'coverage run -a -m pytest -m "ssh" --junit-xml=results-ssh.xml' 27 | } 28 | } 29 | } 30 | stage("Test Qemu") { 31 | steps { 32 | catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { 33 | sh 'coverage run -a -m pytest -m "qemu" --junit-xml=results-qemu.xml' 34 | } 35 | } 36 | } 37 | stage("Test LTX") { 38 | steps { 39 | catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { 40 | sh 'coverage run -a -m pytest -m "ltx" --junit-xml=results-ltx.xml' 41 | } 42 | } 43 | } 44 | } 45 | post { 46 | always { 47 | sh 'coverage xml -o coverage.xml' 48 | cobertura coberturaReportFile: 'coverage.xml' 49 | junit 'results-*.xml' 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-2.0-or-later 2 | # Copyright (c) 2023 SUSE LLC 3 | # 4 | # Install script for Linux Testing Project 5 | 6 | top_srcdir ?= ../.. 7 | 8 | include $(top_srcdir)/include/mk/env_pre.mk 9 | 10 | BASE_DIR := $(abspath $(DESTDIR)/$(prefix)) 11 | 12 | install: 13 | mkdir -p $(BASE_DIR)/libkirk 14 | 15 | install -m 00644 $(top_srcdir)/tools/kirk/libkirk/*.py $(BASE_DIR)/libkirk 16 | install -m 00775 $(top_srcdir)/tools/kirk/kirk $(BASE_DIR)/kirk 17 | 18 | cd $(BASE_DIR) && ln -sf kirk runltp-ng 19 | 20 | include $(top_srcdir)/include/mk/generic_leaf_target.mk 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | What is Kirk? 2 | ============= 3 | 4 | Kirk application is a fork of [runltp-ng](https://github.com/linux-test-project/runltp-ng) 5 | and it's the official [LTP](https://github.com/linux-test-project) tests 6 | executor. It provides support for remote testing via Qemu, SSH, LTX, parallel 7 | execution and much more. 8 | 9 | ```bash 10 | Host information 11 | 12 | Hostname: susy 13 | Python: 3.6.15 (default, Sep 23 2021, 15:41:43) [GCC] 14 | Directory: /tmp/kirk.acer/tmp1n8pa6gy 15 | 16 | Connecting to SUT: host 17 | 18 | Starting suite: math 19 | --------------------- 20 | abs01: pass (0.003s) 21 | atof01: pass (0.004s) 22 | float_bessel: pass (1.174s) 23 | float_exp_log: pass (1.423s) 24 | float_iperb: pass (0.504s) 25 | float_power: pass (1.161s) 26 | float_trigo: pass (1.208s) 27 | fptest01: pass (0.006s) 28 | fptest02: pass (0.004s) 29 | nextafter01: pass (0.001s) 30 | 31 | 32 | Execution time: 5.895s 33 | 34 | Suite: math 35 | Total runs: 10 36 | Runtime: 5.488s 37 | Passed: 22 38 | Failed: 0 39 | Skipped: 0 40 | Broken: 0 41 | Warnings: 0 42 | Kernel: Linux 6.4.0-150600.23.50-default 43 | Machine: x86_64 44 | Arch: x86_64 45 | RAM: 15573156 kB 46 | Swap: 2095424 kB 47 | Distro: opensuse-leap 15.6 48 | 49 | Disconnecting from SUT: host 50 | ``` 51 | 52 | Quickstart 53 | ---------- 54 | 55 | The tool works out of the box by running `kirk` script. 56 | Minimum python requirement is 3.6+ and *optional* dependences are the following: 57 | 58 | - [asyncssh](https://pypi.org/project/asyncssh/) for SSH support 59 | - [msgpack](https://pypi.org/project/msgpack/) for LTX support 60 | 61 | `kirk` will detect if dependences are installed and activate the corresponding 62 | support. If no dependences are provided by the OS's package manager, 63 | `virtualenv` can be used to install them: 64 | 65 | ```bash 66 | # download source code 67 | git clone git@github.com:acerv/kirk.git 68 | cd kirk 69 | 70 | # create your virtual environment (python-3.6+) 71 | virtualenv .venv 72 | 73 | # activate virtualenv 74 | source .venv/bin/activate 75 | 76 | # SSH support 77 | pip install asyncssh 78 | 79 | # LTX support 80 | pip install msgpack 81 | 82 | # run kirk 83 | ./kirk --help 84 | ``` 85 | 86 | Some basic commands are the following: 87 | 88 | ```bash 89 | # run LTP syscalls testing suite on host 90 | ./kirk --run-suite syscalls 91 | 92 | # run LTP syscalls testing suite on qemu VM 93 | ./kirk --sut qemu:image=folder/image.qcow2:user=root:password=root \ 94 | --run-suite syscalls 95 | 96 | # run LTP syscalls testing suite via SSH 97 | ./kirk --sut ssh:host=myhost.com:user=root:key_file=myhost_id_rsa \ 98 | --run-suite syscalls 99 | 100 | # run LTP syscalls testing suite in parallel on host using 16 workers 101 | ./kirk --run-suite syscalls --workers 16 102 | 103 | # run LTP syscalls testing suite in parallel via SSH using 16 workers 104 | ./kirk --sut ssh:host=myhost.com:user=root:key_file=myhost_id_rsa \ 105 | --run-suite syscalls --workers 16 106 | 107 | # pass environment variables (list of key=value separated by ':') 108 | ./kirk --run-suite net.features \ 109 | --env 'VIRT_PERF_THRESHOLD=180:LTP_NET_FEATURES_IGNORE_PERFORMANCE_FAILURE=1' 110 | ``` 111 | 112 | It's possible to run a single command before running testing suites using 113 | `--run-command` option as following: 114 | 115 | ```bash 116 | ./kirk --run-command /mnt/setup.sh \ 117 | --sut qemu:image=folder/image.qcow2:virtfs=/home/user/tests:user=root:password=root \ 118 | --run-suite syscalls 119 | ``` 120 | 121 | Every session has a temporary directory that can be found in 122 | `//kirk.`. Inside this folder there's a symlink 123 | called `latest`, pointing to the latest session's temporary directory. 124 | 125 | In certain cases, `kirk` sessions can be restored. This can be really helpful 126 | when we need to restore the last session after a system crash: 127 | 128 | ```bash 129 | # restore the latest session 130 | ./kirk --restore /tmp/kirk./latest --run-suite syscalls 131 | ``` 132 | 133 | Setting up console for Qemu 134 | --------------------------- 135 | 136 | To enable console on a tty device for a VM do: 137 | 138 | - open `/etc/default/grub` 139 | - add `console=$tty_name, console=tty0` to `GRUB_CMDLINE_LINUX` 140 | - run `grub-mkconfig -o /boot/grub/grub.cfg` 141 | 142 | Where `$tty_name` should be `ttyS0`, unless virtio serial type is used (i.e. 143 | if you set the `serial=virtio` backend option, then use `hvc0`) 144 | 145 | Implementing SUT 146 | ---------------- 147 | 148 | Sometimes we need to cover complex testing scenarios, where the SUT uses 149 | particular protocols and infrastructures, in order to communicate with our 150 | host machine and to execute tests binaries. 151 | 152 | For this reason, `kirk` provides a plugin system to recognize custom SUT 153 | class implementations inside the `libkirk` folder. Please check `host.py` 154 | or `ssh.py` implementations for more details. 155 | 156 | Once a new SUT class is implemented and placed inside the `libkirk` folder, 157 | `kirk -s help` command can be used to see if application correctly 158 | recognise it. 159 | 160 | Implementing Framework 161 | ---------------------- 162 | 163 | Every testing framework has it's own setup, defining tests folders, data and 164 | variables. For this reason, `Framework` class provides a generic API that, once 165 | implemented, permits to define a specific testing framework. The class 166 | implementation must be included inside the `libkirk` folder and it will be 167 | used as an abstraction layer between `kirk` scheduler and the specific testing 168 | framework. 169 | 170 | Development 171 | ----------- 172 | 173 | The application is validated using `pytest` and `pylint`. 174 | To run unittests: 175 | 176 | ```bash 177 | pytest 178 | ``` 179 | 180 | To run linting checks: 181 | 182 | ```bash 183 | pylint --rcfile=pylint.ini ./libkirk 184 | ``` 185 | -------------------------------------------------------------------------------- /kirk: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | .. module:: kirk 4 | :platform: Linux 5 | :synopsis: script to run the main entry point 6 | .. moduleauthor:: Andrea Cervesato 7 | """ 8 | 9 | # pylint: disable=invalid-name 10 | import os 11 | import sys 12 | 13 | # include tool library 14 | sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) 15 | 16 | if __name__ == "__main__": 17 | import libkirk.main 18 | libkirk.main.run() 19 | -------------------------------------------------------------------------------- /libkirk/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: __init__ 3 | :platform: Linux 4 | :synopsis: application package definition 5 | 6 | .. moduleauthor:: Andrea Cervesato 7 | """ 8 | import sys 9 | import signal 10 | import typing 11 | import asyncio 12 | from libkirk.events import EventsHandler 13 | 14 | 15 | # Kirk version 16 | __version__ = '2.1' 17 | 18 | 19 | class KirkException(Exception): 20 | """ 21 | The most generic exception that is raised by any module when 22 | something bad happens. 23 | """ 24 | pass 25 | 26 | 27 | events = EventsHandler() 28 | 29 | 30 | def get_event_loop() -> asyncio.BaseEventLoop: 31 | """ 32 | Return the current asyncio event loop. 33 | """ 34 | loop = None 35 | 36 | try: 37 | loop = asyncio.get_running_loop() 38 | except (AttributeError, RuntimeError): 39 | pass 40 | 41 | if not loop: 42 | try: 43 | loop = asyncio.get_event_loop() 44 | except RuntimeError: 45 | pass 46 | 47 | if not loop: 48 | loop = asyncio.new_event_loop() 49 | asyncio.set_event_loop(loop) 50 | 51 | return loop 52 | 53 | 54 | def create_task(coro: typing.Coroutine) -> asyncio.Task: 55 | """ 56 | Create a new task. 57 | """ 58 | loop = get_event_loop() 59 | task = loop.create_task(coro) 60 | 61 | return task 62 | 63 | 64 | def all_tasks(loop: asyncio.AbstractEventLoop) -> list: 65 | """ 66 | Return the list of all running tasks for the specific loop. 67 | """ 68 | tasks = None 69 | 70 | # pylint: disable=no-member 71 | if sys.version_info >= (3, 7): 72 | tasks = asyncio.all_tasks(loop=loop) 73 | else: 74 | tasks = asyncio.Task.all_tasks(loop=loop) 75 | 76 | return tasks 77 | 78 | 79 | def cancel_tasks(loop: asyncio.AbstractEventLoop) -> None: 80 | """ 81 | Cancel all asyncio running tasks. 82 | """ 83 | tasks = all_tasks(loop) 84 | if not tasks: 85 | return 86 | 87 | for task in tasks: 88 | if task.cancelled() or task.done(): 89 | continue 90 | 91 | task.cancel() 92 | 93 | # pylint: disable=deprecated-argument 94 | if sys.version_info >= (3, 10): 95 | loop.run_until_complete( 96 | asyncio.gather(*tasks, return_exceptions=True)) 97 | else: 98 | loop.run_until_complete( 99 | asyncio.gather(*tasks, loop=loop, return_exceptions=True)) 100 | 101 | for task in tasks: 102 | if task.cancelled(): 103 | continue 104 | 105 | if task.exception() is not None: 106 | loop.call_exception_handler({ 107 | 'message': 'unhandled exception during asyncio.run() shutdown', 108 | 'exception': task.exception(), 109 | 'task': task, 110 | }) 111 | 112 | 113 | def to_thread(callback: callable, *args: typing.Any) -> typing.Any: 114 | """ 115 | Run callback inside a thread. This is useful for blocking I/O operations. 116 | """ 117 | loop = get_event_loop() 118 | return loop.run_in_executor(None, callback, *args) 119 | 120 | 121 | __all__ = [ 122 | "KirkException", 123 | "events", 124 | "get_event_loop", 125 | ] 126 | -------------------------------------------------------------------------------- /libkirk/data.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: data 3 | :platform: Linux 4 | :synopsis: module containing input data handling 5 | 6 | .. moduleauthor:: Andrea Cervesato 7 | """ 8 | import logging 9 | 10 | LOGGER = logging.getLogger("kirk.data") 11 | 12 | 13 | class Suite: 14 | """ 15 | Testing suite definition class. 16 | """ 17 | 18 | def __init__(self, name: str, tests: list) -> None: 19 | """ 20 | :param name: name of the testing suite 21 | :type name: str 22 | :param tests: tests of the suite 23 | :type tests: list 24 | """ 25 | self._name = name 26 | self._tests = tests 27 | 28 | def __repr__(self) -> str: 29 | return \ 30 | f"name: '{self._name}', " \ 31 | f"tests: {self._tests}" 32 | 33 | @property 34 | def name(self): 35 | """ 36 | Name of the testing suite. 37 | """ 38 | return self._name 39 | 40 | @name.setter 41 | def name(self, value: str): 42 | """ 43 | Set the suite name. 44 | """ 45 | if not value: 46 | raise ValueError("empty suite name") 47 | 48 | self._name = value 49 | 50 | @property 51 | def tests(self): 52 | """ 53 | Tests definitions. 54 | """ 55 | return self._tests 56 | 57 | 58 | class Test: 59 | """ 60 | Test definition class. 61 | """ 62 | 63 | def __init__(self, **kwargs: dict) -> None: 64 | """ 65 | :param name: name of the test 66 | :type name: str 67 | :param cmd: command to execute 68 | :type cmd: str 69 | :param cwd: current working directory of the command 70 | :type cwd: str 71 | :param env: environment variables used to run the command 72 | :type env: dict 73 | :param args: list of arguments 74 | :type args: list(str) 75 | :param parallelizable: if True, test can be run in parallel 76 | :type parallelizable: bool 77 | """ 78 | self._name = kwargs.get("name", None) 79 | self._cmd = kwargs.get("cmd", None) 80 | self._args = kwargs.get("args", []) 81 | self._cwd = kwargs.get("cwd", None) 82 | self._env = kwargs.get("env", {}) 83 | self._parallelizable = kwargs.get("parallelizable", False) 84 | 85 | def __repr__(self) -> str: 86 | return \ 87 | f"name: '{self._name}', " \ 88 | f"commmand: '{self._cmd}', " \ 89 | f"arguments: {self._args}, " \ 90 | f"cwd: '{self._cwd}', " \ 91 | f"environ: '{self._env}', " \ 92 | f"parallelizable: {self._parallelizable}" 93 | 94 | @property 95 | def name(self): 96 | """ 97 | Name of the test. 98 | """ 99 | return self._name 100 | 101 | @property 102 | def command(self): 103 | """ 104 | Command to execute test. 105 | """ 106 | return self._cmd 107 | 108 | @property 109 | def arguments(self): 110 | """ 111 | Arguments of the command. 112 | """ 113 | return self._args 114 | 115 | @property 116 | def parallelizable(self): 117 | """ 118 | If True, test can be run in parallel. 119 | """ 120 | return self._parallelizable 121 | 122 | @property 123 | def cwd(self): 124 | """ 125 | Current working directory. 126 | """ 127 | return self._cwd 128 | 129 | @property 130 | def env(self): 131 | """ 132 | Environment variables 133 | """ 134 | return self._env 135 | 136 | @property 137 | def full_command(self): 138 | """ 139 | Return the full command, with arguments as well. 140 | For example, if `command="ls"` and `arguments="-l -a"`, 141 | `full_command="ls -l -a"`. 142 | """ 143 | cmd = self.command 144 | if len(self.arguments) > 0: 145 | cmd += ' ' 146 | cmd += ' '.join(self.arguments) 147 | 148 | return cmd 149 | -------------------------------------------------------------------------------- /libkirk/events.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: events 3 | :platform: Linux 4 | :synopsis: events handler implementation module 5 | 6 | .. moduleauthor:: Andrea Cervesato 7 | """ 8 | import typing 9 | import logging 10 | import asyncio 11 | 12 | 13 | class Event: 14 | """ 15 | An event to process. 16 | """ 17 | 18 | def __init__(self, ordered: bool = False) -> None: 19 | """ 20 | :param ordered: True if coroutines must be processed in order 21 | :type ordered: bool 22 | """ 23 | self._coros = [] 24 | self._ordered = ordered 25 | 26 | def remove(self, coro: typing.Coroutine) -> None: 27 | """ 28 | Remove a specific coroutine associated to the event. 29 | :param coro: coroutine to remove 30 | :type coro: typing.Coroutine 31 | """ 32 | for item in self._coros: 33 | if item == coro: 34 | self._coros.remove(coro) 35 | break 36 | 37 | def has_coros(self) -> bool: 38 | """ 39 | Check if there are still available registrations. 40 | """ 41 | return len(self._coros) > 0 42 | 43 | def register(self, coro: typing.Coroutine) -> None: 44 | """ 45 | Register a new coroutine. 46 | """ 47 | self._coros.append(coro) 48 | 49 | def create_tasks(self, *args: list, **kwargs: dict) -> list: 50 | """ 51 | Create tasks to run according to registered coroutines. 52 | :param args: Arguments to be passed to callback functions execution. 53 | :type args: list 54 | :param kwargs: Keyword arguments to be passed to callback functions 55 | execution. 56 | :type kwargs: dict 57 | """ 58 | tasks = [coro(*args, **kwargs) for coro in self._coros] 59 | 60 | if self._ordered: 61 | return tasks 62 | 63 | return [asyncio.gather(*tasks)] 64 | 65 | 66 | class EventsHandler: 67 | """ 68 | This class implements event loop and events handling. 69 | """ 70 | 71 | def __init__(self) -> None: 72 | self._logger = logging.getLogger("kirk.events") 73 | self._tasks = asyncio.Queue() 74 | self._lock = asyncio.Lock() 75 | self._events = {} 76 | self._stop = False 77 | 78 | # register a default event used to notify internal 79 | # errors in the our application 80 | self._events["internal_error"] = Event() 81 | 82 | def _get_event(self, name: str) -> Event: 83 | """ 84 | Return an event according to its `name`. 85 | """ 86 | return self._events.get(name, None) 87 | 88 | def reset(self) -> None: 89 | """ 90 | Reset the entire events queue. 91 | """ 92 | self._logger.info("Reset events queue") 93 | self._events.clear() 94 | 95 | def is_registered(self, event_name: str) -> bool: 96 | """ 97 | Returns True if ``event_name`` is registered. 98 | :param event_name: name of the event 99 | :type event_name: str 100 | :returns: True if registered, False otherwise 101 | """ 102 | if not event_name: 103 | raise ValueError("event_name is empty") 104 | 105 | evt = self._get_event(event_name) 106 | if not evt: 107 | return False 108 | 109 | return evt.has_coros() 110 | 111 | def register(self, event_name: str, coro: typing.Coroutine, ordered: bool = False) -> None: 112 | """ 113 | Register an event with ``event_name``. 114 | :param event_name: name of the event 115 | :type event_name: str 116 | :param coro: coroutine associated with ``event_name`` 117 | :type coro: Coroutine 118 | :param ordered: if True, the event will raise coroutines in the order 119 | they arrive 120 | :type ordered: bool 121 | """ 122 | if not event_name: 123 | raise ValueError("event_name is empty") 124 | 125 | if not coro: 126 | raise ValueError("coro is empty") 127 | 128 | self._logger.info("Register event: %s", repr(event_name)) 129 | 130 | evt = self._get_event(event_name) 131 | if not evt: 132 | evt = Event(ordered=ordered) 133 | self._events[event_name] = evt 134 | 135 | evt.register(coro) 136 | 137 | def unregister(self, event_name: str, coro: typing.Coroutine = None) -> None: 138 | """ 139 | Unregister a single event coroutine with event_name`. If `coro` is None, 140 | all coroutines registered will be removed. 141 | :param event_name: name of the event 142 | :type event_name: str 143 | :param coro: coroutine to unregister 144 | :type coro: typing.Coroutine 145 | """ 146 | if not event_name: 147 | raise ValueError("event_name is empty") 148 | 149 | if not coro: 150 | raise ValueError("coro is empty") 151 | 152 | if not self.is_registered(event_name): 153 | raise ValueError(f"{event_name} is not registered") 154 | 155 | self._logger.info( 156 | "Unregister event: %s -> %s", repr(event_name), repr(coro)) 157 | 158 | if coro: 159 | self._events[event_name].remove(coro) 160 | else: 161 | del self._events[event_name] 162 | 163 | async def fire(self, event_name: str, *args: list, **kwargs: dict) -> None: 164 | """ 165 | Fire a specific event. 166 | :param event_name: name of the event 167 | :type event_name: str 168 | :param args: Arguments to be passed to callback functions execution. 169 | :type args: list 170 | :param kwargs: Keyword arguments to be passed to callback functions 171 | execution. 172 | :type kwargs: dict 173 | """ 174 | if not event_name: 175 | raise ValueError("event_name is empty") 176 | 177 | evt = self._get_event(event_name) 178 | if not evt: 179 | return 180 | 181 | for task in evt.create_tasks(*args, **kwargs): 182 | await self._tasks.put(task) 183 | 184 | async def _consume(self) -> None: 185 | """ 186 | Consume the next event. 187 | """ 188 | # asyncio.queue::get() will wait until an item is available 189 | # without blocking the application 190 | task = await self._tasks.get() 191 | if not task: 192 | return 193 | 194 | # pylint: disable=broad-except 195 | try: 196 | await task 197 | except asyncio.CancelledError: 198 | pass 199 | except Exception as exc: 200 | if "internal_error" not in self._events: 201 | return 202 | 203 | self._logger.info("Exception catched") 204 | self._logger.error(exc) 205 | 206 | ievt = self._get_event("internal_error") 207 | ievt.create_tasks(exc, task.get_name()) 208 | if ievt: 209 | await asyncio.gather(*ievt) 210 | finally: 211 | self._tasks.task_done() 212 | 213 | async def stop(self) -> None: 214 | """ 215 | Stop the event loop. 216 | """ 217 | self._logger.info("Stopping event loop") 218 | 219 | self._stop = True 220 | 221 | # indicate producer is done 222 | await self._tasks.put(None) 223 | 224 | async with self._lock: 225 | pass 226 | 227 | # consume the last tasks 228 | while not self._tasks.empty(): 229 | await self._consume() 230 | 231 | self._logger.info("Event loop stopped") 232 | 233 | async def start(self) -> None: 234 | """ 235 | Start the event loop. 236 | """ 237 | self._stop = False 238 | 239 | try: 240 | async with self._lock: 241 | self._logger.info("Starting event loop") 242 | 243 | while not self._stop: 244 | await self._consume() 245 | 246 | self._logger.info("Event loop completed") 247 | except asyncio.CancelledError: 248 | await self.stop() 249 | -------------------------------------------------------------------------------- /libkirk/export.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: export 3 | :platform: Linux 4 | :synopsis: module containing exporters definition 5 | 6 | .. moduleauthor:: Andrea Cervesato 7 | """ 8 | import os 9 | import json 10 | import logging 11 | from libkirk import KirkException 12 | from libkirk.io import AsyncFile 13 | from libkirk.results import ResultStatus 14 | 15 | 16 | class ExporterError(KirkException): 17 | """ 18 | Raised when an error occurs during Exporter operations. 19 | """ 20 | 21 | 22 | class Exporter: 23 | """ 24 | A class used to export Results into report file. 25 | """ 26 | 27 | async def save_file(self, results: list, path: str) -> None: 28 | """ 29 | Save report into a file by taking information from SUT and testing 30 | results. 31 | :param results: list of suite results to export. 32 | :type results: list(SuiteResults) 33 | :param path: path of the file to save. 34 | :type path: str 35 | """ 36 | raise NotImplementedError() 37 | 38 | 39 | class JSONExporter(Exporter): 40 | """ 41 | Export testing results into a JSON file. 42 | """ 43 | 44 | def __init__(self) -> None: 45 | self._logger = logging.getLogger("kirk.json") 46 | 47 | # pylint: disable=too-many-locals 48 | async def save_file(self, results: list, path: str) -> None: 49 | if not results or len(results) == 0: 50 | raise ValueError("results is empty") 51 | 52 | if not path: 53 | raise ValueError("path is empty") 54 | 55 | if os.path.exists(path): 56 | raise ExporterError(f"'{path}' already exists") 57 | 58 | self._logger.info("Exporting JSON report into %s", path) 59 | 60 | results_json = [] 61 | 62 | for result in results: 63 | for test_report in result.tests_results: 64 | status = "" 65 | if test_report.status == ResultStatus.PASS: 66 | status = "pass" 67 | elif test_report.status == ResultStatus.BROK: 68 | status = "brok" 69 | elif test_report.status == ResultStatus.WARN: 70 | status = "warn" 71 | elif test_report.status == ResultStatus.CONF: 72 | status = "conf" 73 | else: 74 | status = "fail" 75 | 76 | data_test = { 77 | "test_fqn": test_report.test.name, 78 | "status": status, 79 | "test": { 80 | "command": test_report.test.command, 81 | "arguments": test_report.test.arguments, 82 | "log": test_report.stdout, 83 | "retval": [str(test_report.return_code)], 84 | "duration": test_report.exec_time, 85 | "failed": test_report.failed, 86 | "passed": test_report.passed, 87 | "broken": test_report.broken, 88 | "skipped": test_report.skipped, 89 | "warnings": test_report.warnings, 90 | "result": status, 91 | }, 92 | } 93 | 94 | results_json.append(data_test) 95 | 96 | data = { 97 | "results": results_json, 98 | "stats": { 99 | "runtime": sum(result.exec_time for result in results), 100 | "passed": sum(result.passed for result in results), 101 | "failed": sum(result.failed for result in results), 102 | "broken": sum(result.broken for result in results), 103 | "skipped": sum(result.skipped for result in results), 104 | "warnings": sum(result.warnings for result in results) 105 | }, 106 | "environment": { 107 | "distribution": results[0].distro, 108 | "distribution_version": results[0].distro_ver, 109 | "kernel": results[0].kernel, 110 | "arch": results[0].arch, 111 | "cpu": results[0].cpu, 112 | "swap": results[0].swap, 113 | "RAM": results[0].ram, 114 | }, 115 | } 116 | 117 | async with AsyncFile(path, "w+") as outfile: 118 | text = json.dumps(data, indent=4) 119 | await outfile.write(text) 120 | 121 | self._logger.info("Report exported") 122 | -------------------------------------------------------------------------------- /libkirk/framework.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: framework 3 | :platform: Linux 4 | :synopsis: framework definition 5 | 6 | .. moduleauthor:: Andrea Cervesato 7 | """ 8 | from libkirk import KirkException 9 | from libkirk.sut import SUT 10 | from libkirk.data import Test 11 | from libkirk.data import Suite 12 | from libkirk.plugin import Plugin 13 | from libkirk.results import TestResults 14 | 15 | 16 | class FrameworkError(KirkException): 17 | """ 18 | A generic framework exception. 19 | """ 20 | 21 | 22 | class Framework(Plugin): 23 | """ 24 | Framework definition. Implement this class if you need to support more 25 | testing frameworks inside the application. 26 | """ 27 | 28 | async def get_suites(self, sut: SUT) -> list: 29 | """ 30 | Return the list of available suites inside SUT. 31 | :param sut: SUT object to communicate with 32 | :type sut: SUT 33 | :returns: list 34 | """ 35 | raise NotImplementedError() 36 | 37 | async def find_command(self, sut: SUT, command: str) -> Test: 38 | """ 39 | Search for command inside Framework folder and, if it's not found, 40 | search for command in the operating system. Then return a Test object 41 | which can be used to execute command. 42 | :param sut: SUT object to communicate with 43 | :type sut: SUT 44 | :param command: command to execute 45 | :type command: str 46 | :returns: Test 47 | """ 48 | raise NotImplementedError() 49 | 50 | async def find_suite(self, sut: SUT, name: str) -> Suite: 51 | """ 52 | Search for suite with given name inside SUT. 53 | :param sut: SUT object to communicate with 54 | :type sut: SUT 55 | :param suite: name of the suite 56 | :type suite: str 57 | :returns: Suite 58 | """ 59 | raise NotImplementedError() 60 | 61 | async def read_result( 62 | self, 63 | test: Test, 64 | stdout: str, 65 | retcode: int, 66 | exec_t: float) -> TestResults: 67 | """ 68 | Return test results accoding with runner output and Test definition. 69 | :param test: Test definition object 70 | :type test: Test 71 | :param stdout: test stdout 72 | :type stdout: str 73 | :param retcode: test return code 74 | :type retcode: int 75 | :param exec_t: test execution time in seconds 76 | :type exec_t: float 77 | :returns: TestResults 78 | """ 79 | raise NotImplementedError() 80 | -------------------------------------------------------------------------------- /libkirk/host.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: host 3 | :platform: Linux 4 | :synopsis: module containing host SUT implementation 5 | 6 | .. moduleauthor:: Andrea Cervesato 7 | """ 8 | import os 9 | import time 10 | import signal 11 | import asyncio 12 | import logging 13 | import contextlib 14 | from asyncio.subprocess import Process 15 | from libkirk.io import AsyncFile 16 | from libkirk.sut import SUT 17 | from libkirk.sut import IOBuffer 18 | from libkirk.sut import SUTError 19 | from libkirk.sut import KernelPanicError 20 | 21 | 22 | class HostSUT(SUT): 23 | """ 24 | SUT implementation using host's shell. 25 | """ 26 | BUFFSIZE = 1024 27 | 28 | def __init__(self) -> None: 29 | self._logger = logging.getLogger("kirk.host") 30 | self._fetch_lock = asyncio.Lock() 31 | self._procs = [] 32 | self._running = False 33 | self._stop = False 34 | 35 | def setup(self, **kwargs: dict) -> None: 36 | pass 37 | 38 | @property 39 | def config_help(self) -> dict: 40 | # cwd and env are given by default, so no options are needed 41 | return {} 42 | 43 | @property 44 | def name(self) -> str: 45 | return "host" 46 | 47 | @property 48 | def parallel_execution(self) -> bool: 49 | return True 50 | 51 | @property 52 | async def is_running(self) -> bool: 53 | return self._running 54 | 55 | @staticmethod 56 | async def _process_alive(proc: Process) -> bool: 57 | """ 58 | Return True if process is alive and running. 59 | """ 60 | with contextlib.suppress(asyncio.TimeoutError): 61 | returncode = await asyncio.wait_for(proc.wait(), 1e-6) 62 | if returncode is not None: 63 | return False 64 | 65 | return True 66 | 67 | async def _kill_process(self, proc: Process) -> None: 68 | """ 69 | Kill a process and all its subprocesses. 70 | """ 71 | self._logger.info("Kill process %d", proc.pid) 72 | 73 | try: 74 | os.killpg(os.getpgid(proc.pid), signal.SIGKILL) 75 | except ProcessLookupError: 76 | # process has been killed already 77 | pass 78 | 79 | async def ping(self) -> float: 80 | if not await self.is_running: 81 | raise SUTError("SUT is not running") 82 | 83 | ret = await self.run_command("test .") 84 | reply_t = ret["exec_time"] 85 | 86 | return reply_t 87 | 88 | async def communicate(self, iobuffer: IOBuffer = None) -> None: 89 | if await self.is_running: 90 | raise SUTError("SUT is running") 91 | 92 | self._running = True 93 | 94 | async def stop(self, iobuffer: IOBuffer = None) -> None: 95 | if not await self.is_running: 96 | return 97 | 98 | self._logger.info("Stopping SUT") 99 | self._stop = True 100 | 101 | try: 102 | if self._procs: 103 | self._logger.info( 104 | "Terminating %d process(es)", 105 | len(self._procs)) 106 | 107 | for proc in self._procs: 108 | await self._kill_process(proc) 109 | 110 | await asyncio.gather(*[ 111 | proc.wait() for proc in self._procs 112 | ]) 113 | 114 | self._logger.info("Process(es) terminated") 115 | 116 | if self._fetch_lock.locked(): 117 | self._logging.info("Terminating data fetch") 118 | 119 | async with self._fetch_lock: 120 | pass 121 | finally: 122 | self._stop = False 123 | self._running = False 124 | self._logger.info("SUT has stopped") 125 | 126 | async def run_command( 127 | self, 128 | command: str, 129 | cwd: str = None, 130 | env: dict = None, 131 | iobuffer: IOBuffer = None) -> dict: 132 | if not command: 133 | raise ValueError("command is empty") 134 | 135 | if not await self.is_running: 136 | raise SUTError("SUT is not running") 137 | 138 | self._logger.info("Executing command: '%s'", command) 139 | 140 | ret = None 141 | proc = None 142 | t_end = 0 143 | stdout = "" 144 | 145 | try: 146 | kwargs = { 147 | "stdout": asyncio.subprocess.PIPE, 148 | "stderr": asyncio.subprocess.STDOUT, 149 | "cwd": cwd, 150 | "start_new_session": True 151 | } 152 | 153 | if env: 154 | # commands can fail when env is defined, so we just skip 155 | # env usage if dictionary is empty 156 | kwargs["env"] = env 157 | 158 | proc = await asyncio.create_subprocess_shell(command, **kwargs) 159 | 160 | self._procs.append(proc) 161 | 162 | t_start = time.time() 163 | panic = False 164 | 165 | while True: 166 | line = await proc.stdout.read(self.BUFFSIZE) 167 | sline = line.decode(encoding="utf-8", errors="ignore") 168 | 169 | if iobuffer: 170 | await iobuffer.write(sline) 171 | 172 | stdout += sline 173 | panic = "Kernel panic" in stdout[-2*self.BUFFSIZE:] 174 | 175 | if not await self._process_alive(proc): 176 | break 177 | 178 | await proc.wait() 179 | 180 | t_end = time.time() - t_start 181 | 182 | if panic: 183 | raise KernelPanicError() 184 | finally: 185 | if proc: 186 | self._procs.remove(proc) 187 | 188 | await self._kill_process(proc) 189 | await proc.wait() 190 | 191 | ret = { 192 | "command": command, 193 | "stdout": stdout, 194 | "returncode": proc.returncode, 195 | "exec_time": t_end, 196 | } 197 | 198 | self._logger.debug("return data=%s", ret) 199 | 200 | self._logger.info("Command executed") 201 | 202 | return ret 203 | 204 | async def fetch_file(self, target_path: str) -> bytes: 205 | if not target_path: 206 | raise ValueError("target path is empty") 207 | 208 | if not os.path.isfile(target_path): 209 | raise SUTError(f"'{target_path}' file doesn't exist") 210 | 211 | if not await self.is_running: 212 | raise SUTError("SUT is not running") 213 | 214 | async with self._fetch_lock: 215 | self._logger.info("Downloading '%s'", target_path) 216 | 217 | retdata = bytes() 218 | 219 | try: 220 | async with AsyncFile(target_path, 'rb') as ftarget: 221 | retdata = await ftarget.read() 222 | except IOError as err: 223 | raise SUTError(err) 224 | 225 | self._logger.info("File copied") 226 | 227 | return retdata 228 | -------------------------------------------------------------------------------- /libkirk/io.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: io 3 | :platform: Linux 4 | :synopsis: module for handling I/O blocking operations 5 | 6 | .. moduleauthor:: Andrea Cervesato 7 | """ 8 | import libkirk 9 | 10 | 11 | class AsyncFile: 12 | """ 13 | Handle files in asynchronous way by running operations inside a separate 14 | thread. 15 | """ 16 | 17 | def __init__(self, filename: str, mode='r') -> None: 18 | """ 19 | :param filename: file to open 20 | :type filename: str 21 | :param mode: mode to open the file 22 | :type mode: str 23 | """ 24 | self._filename = filename 25 | self._mode = mode 26 | self._file = None 27 | 28 | async def __aenter__(self): 29 | await self.open() 30 | return self 31 | 32 | async def __aexit__(self, exc_type, exc_val, exc_tb): 33 | await self.close() 34 | 35 | def __aiter__(self): 36 | return self 37 | 38 | async def __anext__(self): 39 | if 'r' not in self._mode: 40 | raise ValueError("File must be open in read mode") 41 | 42 | line = None 43 | if self._file: 44 | line = await libkirk.to_thread(self._file.readline) 45 | if not line: 46 | raise StopAsyncIteration 47 | 48 | return line 49 | 50 | async def open(self) -> None: 51 | """ 52 | Open the file according to the mode. 53 | """ 54 | if self._file: 55 | return 56 | 57 | def _open(): 58 | if 'b' in self._mode: 59 | # pylint: disable=unspecified-encoding 60 | return open(self._filename, self._mode) 61 | 62 | return open(self._filename, self._mode, encoding='utf-8') 63 | 64 | self._file = await libkirk.to_thread(_open) 65 | 66 | async def close(self) -> None: 67 | """ 68 | Close the file. 69 | """ 70 | if not self._file: 71 | return 72 | 73 | await libkirk.to_thread(self._file.close) 74 | self._file = None 75 | 76 | async def seek(self, pos: int) -> None: 77 | """ 78 | Asynchronous version of `seek()`. 79 | :param pos: position to search 80 | :type pos: int 81 | """ 82 | if not self._file: 83 | return 84 | 85 | await libkirk.to_thread(self._file.seek, pos) 86 | 87 | async def tell(self) -> int: 88 | """ 89 | Asynchronous version of `tell()`. 90 | :returns: current file position or None if file is not open. 91 | """ 92 | if not self._file: 93 | return None 94 | 95 | return await libkirk.to_thread(self._file.tell) 96 | 97 | async def read(self, size: int = -1) -> str: 98 | """ 99 | Asynchronous version of `read()`. 100 | :returns: data that has been read or None if file is not open. 101 | """ 102 | if not self._file: 103 | return None 104 | 105 | return await libkirk.to_thread(self._file.read, size) 106 | 107 | async def readline(self) -> str: 108 | """ 109 | Asynchronous version of `readline()`. 110 | :returns: data that has been read or None if file is not open. 111 | """ 112 | if not self._file: 113 | return None 114 | 115 | return await libkirk.to_thread(self._file.readline) 116 | 117 | async def write(self, data: str) -> None: 118 | """ 119 | Asynchronous version of `write()`. 120 | :param data: data to write inside file 121 | :type data: str 122 | """ 123 | if not self._file: 124 | return 125 | 126 | await libkirk.to_thread(self._file.write, data) 127 | -------------------------------------------------------------------------------- /libkirk/ltp.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: ltp 3 | :platform: Linux 4 | :synopsis: LTP framework definition 5 | 6 | .. moduleauthor:: Andrea Cervesato 7 | """ 8 | import os 9 | import re 10 | import json 11 | import shlex 12 | import logging 13 | from libkirk.results import TestResults 14 | from libkirk.results import ResultStatus 15 | from libkirk.sut import SUT 16 | from libkirk.data import Suite 17 | from libkirk.data import Test 18 | from libkirk.framework import Framework 19 | from libkirk.framework import FrameworkError 20 | 21 | 22 | class LTPFramework(Framework): 23 | """ 24 | Linux Test Project framework definition. 25 | """ 26 | 27 | PARALLEL_BLACKLIST = [ 28 | "needs_root", 29 | "needs_device", 30 | "mount_device", 31 | "mntpoint", 32 | "resource_file", 33 | "format_device", 34 | "save_restore", 35 | "max_runtime" 36 | ] 37 | 38 | def __init__(self) -> None: 39 | self._logger = logging.getLogger("libkirk.ltp") 40 | self._root = None 41 | self._env = None 42 | self._max_runtime = None 43 | self._tc_folder = None 44 | 45 | @ property 46 | def config_help(self) -> dict: 47 | return { 48 | "root": "LTP install folder", 49 | "max_runtime": "filter out all tests above this time value", 50 | } 51 | 52 | def setup(self, **kwargs: dict) -> None: 53 | self._root = os.environ.get("LTPROOT", "/opt/ltp") 54 | self._env = { 55 | "LTPROOT": self._root, 56 | "TMPDIR": "/tmp", 57 | "LTP_COLORIZE_OUTPUT": "1", 58 | } 59 | 60 | env = kwargs.get("env", None) 61 | if env: 62 | self._env.update(env) 63 | 64 | timeout = kwargs.get("test_timeout", None) 65 | if timeout: 66 | self._env["LTP_TIMEOUT_MUL"] = str((timeout * 0.9) / 300.0) 67 | 68 | root = kwargs.get("root", None) 69 | if root: 70 | self._root = root 71 | self._env["LTPROOT"] = self._root 72 | 73 | self._tc_folder = os.path.join(self._root, "testcases", "bin") 74 | 75 | runtime = kwargs.get("max_runtime", None) 76 | 77 | if runtime: 78 | try: 79 | runtime = float(runtime) 80 | except TypeError: 81 | raise FrameworkError("max_runtime must be an integer") 82 | 83 | self._max_runtime = runtime 84 | 85 | async def _read_path(self, sut: SUT) -> dict: 86 | """ 87 | Read PATH and initialize it with testcases folder as well. 88 | """ 89 | env = self._env.copy() 90 | if 'PATH' in env: 91 | env["PATH"] = env["PATH"] + f":{self._tc_folder}" 92 | else: 93 | ret = await sut.run_command("echo -n $PATH") 94 | if ret["returncode"] != 0: 95 | raise FrameworkError("Can't read PATH variable") 96 | 97 | tcases = os.path.join(self._root, "testcases", "bin") 98 | env["PATH"] = ret["stdout"].strip() + f":{tcases}" 99 | 100 | self._logger.debug("PATH=%s", env["PATH"]) 101 | 102 | return env 103 | 104 | def _is_addable(self, test_params: dict) -> bool: 105 | """ 106 | Check if test has to be added or not, according with test parameters 107 | from metadata. 108 | """ 109 | addable = True 110 | 111 | # filter out max_runtime tests when required 112 | if self._max_runtime: 113 | runtime = test_params.get("max_runtime") 114 | if runtime: 115 | try: 116 | runtime = float(runtime) 117 | if runtime >= self._max_runtime: 118 | self._logger.info( 119 | "max_runtime is bigger than %f", 120 | self._max_runtime) 121 | addable = False 122 | except TypeError: 123 | self._logger.error( 124 | "metadata contains wrong max_runtime type: %s", 125 | runtime) 126 | 127 | return addable 128 | 129 | # pylint: disable=too-many-locals 130 | async def _read_runtest( 131 | self, 132 | sut: SUT, 133 | suite_name: str, 134 | content: str, 135 | metadata: dict = None) -> Suite: 136 | """ 137 | It reads a runtest file content and it returns a Suite object. 138 | """ 139 | self._logger.info("collecting testing suite: %s", suite_name) 140 | 141 | metadata_tests = None 142 | if metadata: 143 | self._logger.info("Reading metadata content") 144 | metadata_tests = metadata.get("tests", None) 145 | 146 | env = await self._read_path(sut) 147 | 148 | tests = [] 149 | lines = content.split('\n') 150 | 151 | for line in lines: 152 | if not line.strip() or line.strip().startswith("#"): 153 | continue 154 | 155 | self._logger.debug("Test declaration: %s", line) 156 | 157 | parts = shlex.split(line) 158 | if len(parts) < 2: 159 | raise FrameworkError( 160 | "runtest file is not defining test command") 161 | 162 | test_name = parts[0] 163 | test_cmd = parts[1] 164 | test_args = [] 165 | 166 | if len(parts) >= 3: 167 | test_args = parts[2:] 168 | 169 | parallelizable = True 170 | 171 | if not metadata_tests: 172 | # no metadata no party 173 | parallelizable = False 174 | else: 175 | test_params = metadata_tests.get(test_name, None) 176 | if test_params: 177 | self._logger.info( 178 | "Found %s test params in metadata", test_name) 179 | self._logger.debug("params=%s", test_params) 180 | 181 | if test_params is None: 182 | # this probably means test is not using new LTP API, 183 | # so we can't decide if test can run in parallel or not 184 | parallelizable = False 185 | else: 186 | if not self._is_addable(test_params): 187 | continue 188 | 189 | for blacklist_param in self.PARALLEL_BLACKLIST: 190 | if blacklist_param in test_params: 191 | parallelizable = False 192 | break 193 | 194 | if not parallelizable: 195 | self._logger.info("Test '%s' is not parallelizable", test_name) 196 | else: 197 | self._logger.info("Test '%s' is parallelizable", test_name) 198 | 199 | test = Test( 200 | name=test_name, 201 | cmd=test_cmd, 202 | args=test_args, 203 | cwd=self._tc_folder, 204 | env=env, 205 | parallelizable=parallelizable) 206 | 207 | tests.append(test) 208 | 209 | self._logger.debug("test: %s", test) 210 | 211 | self._logger.debug("Collected tests: %d", len(tests)) 212 | 213 | suite = Suite(suite_name, tests) 214 | 215 | self._logger.debug(suite) 216 | self._logger.info("Collected testing suite: %s", suite_name) 217 | 218 | return suite 219 | 220 | @property 221 | def name(self) -> str: 222 | return "ltp" 223 | 224 | async def get_suites(self, sut: SUT) -> list: 225 | if not sut: 226 | raise ValueError("SUT is None") 227 | 228 | ret = await sut.run_command(f"test -d {self._root}") 229 | if ret["returncode"] != 0: 230 | raise FrameworkError(f"LTP folder doesn't exist: {self._root}") 231 | 232 | runtest_dir = os.path.join(self._root, "runtest") 233 | ret = await sut.run_command(f"test -d {runtest_dir}") 234 | if ret["returncode"] != 0: 235 | raise FrameworkError(f"'{runtest_dir}' doesn't exist inside SUT") 236 | 237 | ret = await sut.run_command(f"ls --format=single-column {runtest_dir}") 238 | stdout = ret["stdout"] 239 | if ret["returncode"] != 0: 240 | raise FrameworkError(f"command failed with: {stdout}") 241 | 242 | suites = [line for line in stdout.split('\n') if line] 243 | return suites 244 | 245 | async def find_command(self, sut: SUT, command: str) -> Test: 246 | if not sut: 247 | raise ValueError("SUT is None") 248 | 249 | if not command: 250 | raise ValueError("command is empty") 251 | 252 | cmd_args = shlex.split(command) 253 | cwd = None 254 | env = None 255 | 256 | ret = await sut.run_command(f"test -d {self._tc_folder}") 257 | if ret["returncode"] == 0: 258 | cwd = self._tc_folder 259 | env = await self._read_path(sut) 260 | 261 | test = Test( 262 | name=cmd_args[0], 263 | cmd=cmd_args[0], 264 | args=cmd_args[1:] if len(cmd_args) > 0 else None, 265 | cwd=cwd, 266 | env=env, 267 | parallelizable=False) 268 | 269 | return test 270 | 271 | async def find_suite(self, sut: SUT, name: str) -> Suite: 272 | if not sut: 273 | raise ValueError("SUT is None") 274 | 275 | if not name: 276 | raise ValueError("name is empty") 277 | 278 | ret = await sut.run_command(f"test -d {self._root}") 279 | if ret["returncode"] != 0: 280 | raise FrameworkError(f"LTP folder doesn't exist: {self._root}") 281 | 282 | suite_path = os.path.join(self._root, "runtest", name) 283 | 284 | ret = await sut.run_command(f"test -f {suite_path}") 285 | if ret["returncode"] != 0: 286 | raise FrameworkError(f"'{name}' suite doesn't exist") 287 | 288 | runtest_data = await sut.fetch_file(suite_path) 289 | runtest_str = runtest_data.decode(encoding="utf-8", errors="ignore") 290 | 291 | metadata_path = os.path.join( 292 | self._root, 293 | "metadata", 294 | "ltp.json" 295 | ) 296 | metadata_dict = None 297 | ret = await sut.run_command(f"test -f {metadata_path}") 298 | if ret["returncode"] == 0: 299 | metadata_data = await sut.fetch_file(metadata_path) 300 | metadata_dict = json.loads(metadata_data) 301 | 302 | suite = await self._read_runtest(sut, name, runtest_str, metadata_dict) 303 | 304 | return suite 305 | 306 | async def read_result( 307 | self, 308 | test: Test, 309 | stdout: str, 310 | retcode: int, 311 | exec_t: float) -> TestResults: 312 | # get rid of colors from stdout 313 | stdout = re.sub(r'\u001b\[[0-9;]+[a-zA-Z]', '', stdout) 314 | 315 | match = re.search( 316 | r"Summary:\n" 317 | r"passed\s*(?P\d+)\n" 318 | r"failed\s*(?P\d+)\n" 319 | r"broken\s*(?P\d+)\n" 320 | r"skipped\s*(?P\d+)\n" 321 | r"warnings\s*(?P\d+)\n", 322 | stdout 323 | ) 324 | 325 | passed = 0 326 | failed = 0 327 | skipped = 0 328 | broken = 0 329 | skipped = 0 330 | warnings = 0 331 | error = retcode == -1 332 | status = ResultStatus.PASS 333 | 334 | if match: 335 | passed = int(match.group("passed")) 336 | failed = int(match.group("failed")) 337 | skipped = int(match.group("skipped")) 338 | broken = int(match.group("broken")) 339 | skipped = int(match.group("skipped")) 340 | warnings = int(match.group("warnings")) 341 | else: 342 | passed = stdout.count("TPASS") 343 | failed = stdout.count("TFAIL") 344 | skipped = stdout.count("TSKIP") 345 | broken = stdout.count("TBROK") 346 | warnings = stdout.count("TWARN") 347 | 348 | if passed == 0 and \ 349 | failed == 0 and \ 350 | skipped == 0 and \ 351 | broken == 0 and \ 352 | warnings == 0: 353 | # if no results are given, this is probably an 354 | # old test implementation that fails when return 355 | # code is != 0 356 | if retcode == 0: 357 | passed = 1 358 | elif retcode == 4: 359 | warnings = 1 360 | elif retcode == 32: 361 | skipped = 1 362 | elif not error: 363 | failed = 1 364 | 365 | if retcode == 0: 366 | status = ResultStatus.PASS 367 | elif retcode in (2, -1): 368 | status = ResultStatus.BROK 369 | elif retcode == 4: 370 | status = ResultStatus.WARN 371 | elif retcode == 32: 372 | status = ResultStatus.CONF 373 | else: 374 | status = ResultStatus.FAIL 375 | 376 | if error: 377 | broken = 1 378 | 379 | result = TestResults( 380 | test=test, 381 | failed=failed, 382 | passed=passed, 383 | broken=broken, 384 | skipped=skipped, 385 | warnings=warnings, 386 | exec_time=exec_t, 387 | retcode=retcode, 388 | stdout=stdout, 389 | status=status, 390 | ) 391 | 392 | return result 393 | -------------------------------------------------------------------------------- /libkirk/ltx_sut.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: ltx 3 | :platform: Linux 4 | :synopsis: module containing LTX communication class 5 | 6 | .. moduleauthor:: Andrea Cervesato 7 | """ 8 | import os 9 | import time 10 | import asyncio 11 | import logging 12 | import importlib 13 | from libkirk.sut import SUT 14 | from libkirk.sut import SUTError 15 | from libkirk.sut import IOBuffer 16 | from libkirk.ltx import Request 17 | from libkirk.ltx import Requests 18 | from libkirk.ltx import LTX 19 | from libkirk.ltx import LTXError 20 | 21 | 22 | class LTXSUT(SUT): 23 | """ 24 | A SUT using LTX as executor. 25 | """ 26 | 27 | def __init__(self) -> None: 28 | self._logger = logging.getLogger("kirk.ltx") 29 | self._release_lock = asyncio.Lock() 30 | self._fetch_lock = asyncio.Lock() 31 | self._stdout = '' 32 | self._stdin = '' 33 | self._stdout_fd = -1 34 | self._stdin_fd = -1 35 | self._tmpdir = None 36 | self._ltx = None 37 | self._slots = [] 38 | 39 | @property 40 | def name(self) -> str: 41 | return "ltx" 42 | 43 | @property 44 | def config_help(self) -> dict: 45 | return { 46 | "stdin": "transport stdin file", 47 | "stdout": "transport stdout file", 48 | } 49 | 50 | def setup(self, **kwargs: dict) -> None: 51 | if not importlib.util.find_spec('msgpack'): 52 | raise SUTError("'msgpack' library is not available") 53 | 54 | self._logger.info("Initialize SUT") 55 | 56 | self._tmpdir = kwargs.get("tmpdir", None) 57 | self._stdin = kwargs.get("stdin", None) 58 | self._stdout = kwargs.get("stdout", None) 59 | 60 | if not os.path.exists(self._stdin): 61 | raise SUTError(f"'{self._stdin}' stdin file doesn't exist") 62 | 63 | if not os.path.exists(self._stdout): 64 | raise SUTError(f"'{self._stdout}' stdout file doesn't exist") 65 | 66 | @property 67 | def parallel_execution(self) -> bool: 68 | return True 69 | 70 | @property 71 | async def is_running(self) -> bool: 72 | if self._ltx: 73 | return self._ltx.connected 74 | 75 | return False 76 | 77 | async def stop(self, iobuffer: IOBuffer = None) -> None: 78 | if not await self.is_running: 79 | return 80 | 81 | if self._slots: 82 | requests = [] 83 | for slot_id in self._slots: 84 | requests.append(Requests.kill(slot_id)) 85 | 86 | if requests: 87 | await self._send_requests(requests) 88 | 89 | while self._slots: 90 | await asyncio.sleep(1e-2) 91 | 92 | try: 93 | await self._ltx.disconnect() 94 | except LTXError as err: 95 | raise SUTError(err) 96 | 97 | while await self.is_running: 98 | await asyncio.sleep(1e-2) 99 | 100 | try: 101 | if self._stdin_fd != -1: 102 | os.close(self._stdin_fd) 103 | 104 | if self._stdout_fd != -1: 105 | os.close(self._stdout_fd) 106 | except OSError as err: 107 | # LTX can exit before we close file, so we skip 108 | # 'Bad file descriptor' error message 109 | if err.errno == 9: 110 | pass 111 | 112 | async def _send_requests(self, requests: list) -> list: 113 | """ 114 | Send requests and check for LTXError. 115 | """ 116 | reply = None 117 | try: 118 | reply = await self._ltx.gather(requests) 119 | except LTXError as err: 120 | raise SUTError(err) 121 | 122 | return reply 123 | 124 | async def _reserve_slot(self) -> int: 125 | """ 126 | Reserve an execution slot. 127 | """ 128 | async with self._release_lock: 129 | slot_id = -1 130 | for i in range(0, Request.MAX_SLOTS): 131 | if i not in self._slots: 132 | slot_id = i 133 | break 134 | 135 | if slot_id == -1: 136 | raise SUTError("No execution slots available") 137 | 138 | self._slots.append(slot_id) 139 | 140 | return slot_id 141 | 142 | async def _release_slot(self, slot_id: int) -> None: 143 | """ 144 | Release an execution slot. 145 | """ 146 | if slot_id in self._slots: 147 | self._slots.remove(slot_id) 148 | 149 | async def ping(self) -> float: 150 | if not await self.is_running: 151 | raise SUTError("SUT is not running") 152 | 153 | req = Requests.ping() 154 | start_t = time.monotonic() 155 | replies = await self._send_requests([req]) 156 | 157 | return (replies[req][0] * 1e-9) - start_t 158 | 159 | async def communicate(self, iobuffer: IOBuffer = None) -> None: 160 | if await self.is_running: 161 | raise SUTError("SUT is already running") 162 | 163 | self._stdin_fd = os.open(self._stdin, os.O_WRONLY) 164 | self._stdout_fd = os.open(self._stdout, os.O_RDONLY) 165 | 166 | self._ltx = LTX(self._stdin_fd, self._stdout_fd) 167 | 168 | try: 169 | await self._ltx.connect() 170 | except LTXError as err: 171 | raise SUTError(err) 172 | 173 | await self._send_requests([Requests.version()]) 174 | 175 | async def run_command( 176 | self, 177 | command: str, 178 | cwd: str = None, 179 | env: dict = None, 180 | iobuffer: IOBuffer = None) -> dict: 181 | if not command: 182 | raise ValueError("command is empty") 183 | 184 | if not await self.is_running: 185 | raise SUTError("SUT is not running") 186 | 187 | self._logger.info("Running command: %s", repr(command)) 188 | 189 | slot_id = await self._reserve_slot() 190 | ret = None 191 | 192 | try: 193 | start_t = time.monotonic() 194 | 195 | requests = [] 196 | if cwd: 197 | requests.append(Requests.cwd(slot_id, cwd)) 198 | 199 | if env: 200 | for key, value in env.items(): 201 | requests.append(Requests.env(slot_id, key, value)) 202 | 203 | async def _stdout_coro(data): 204 | if iobuffer: 205 | await iobuffer.write(data) 206 | 207 | exec_req = Requests.execute( 208 | slot_id, 209 | command, 210 | stdout_coro=_stdout_coro) 211 | 212 | requests.append(exec_req) 213 | replies = await self._send_requests(requests) 214 | reply = replies[exec_req] 215 | 216 | ret = { 217 | "command": command, 218 | "stdout": reply[3], 219 | "exec_time": (reply[0] * 1e-9) - start_t, 220 | "returncode": reply[2], 221 | } 222 | 223 | self._logger.debug(ret) 224 | finally: 225 | await self._release_slot(slot_id) 226 | 227 | self._logger.info("Command executed") 228 | 229 | return ret 230 | 231 | async def fetch_file(self, target_path: str) -> bytes: 232 | if not target_path: 233 | raise ValueError("target path is empty") 234 | 235 | if not await self.is_running: 236 | raise SUTError("SSH connection is not present") 237 | 238 | async with self._fetch_lock: 239 | req = Requests.get_file(target_path) 240 | replies = await self._send_requests([req]) 241 | reply = replies[req] 242 | 243 | return reply[1] 244 | -------------------------------------------------------------------------------- /libkirk/monitor.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: ui 3 | :platform: Linux 4 | :synopsis: modules used to generate real-time data from the executor 5 | 6 | .. moduleauthor:: Andrea Cervesato 7 | """ 8 | import json 9 | import asyncio 10 | import logging 11 | import libkirk 12 | from libkirk.io import AsyncFile 13 | from libkirk.data import Test 14 | from libkirk.data import Suite 15 | from libkirk.results import TestResults 16 | from libkirk.results import SuiteResults 17 | 18 | 19 | # pylint: disable=missing-function-docstring 20 | # pylint: disable=too-many-public-methods 21 | class JSONFileMonitor: 22 | """ 23 | Monitor the current executor status and it redirects events to a file 24 | using JSON format. 25 | """ 26 | 27 | def __init__(self, path: str) -> None: 28 | """ 29 | :param path: path of the file 30 | :type path: str 31 | """ 32 | self._logging = logging.getLogger("libkirk.monitor") 33 | self._logging.info("File to monitor: %s", path) 34 | self._lock = asyncio.Lock() 35 | 36 | self._path = path 37 | self._events = { 38 | "session_restore": self.session_restore, 39 | "session_started": self.session_started, 40 | "session_stopped": self.session_stopped, 41 | "sut_stdout": self.sut_stdout, 42 | "sut_start": self.sut_start, 43 | "sut_stop": self.sut_stop, 44 | "sut_restart": self.sut_restart, 45 | "sut_not_responding": self.sut_not_responding, 46 | "run_cmd_start": self.run_cmd_start, 47 | "run_cmd_stop": self.run_cmd_stop, 48 | "test_stdout": self.test_stdout, 49 | "test_started": self.test_started, 50 | "test_completed": self.test_completed, 51 | "test_timed_out": self.test_timed_out, 52 | "suite_started": self.suite_started, 53 | "suite_completed": self.suite_completed, 54 | "suite_timeout": self.suite_timeout, 55 | "session_warning": self.session_warning, 56 | "session_error": self.session_error, 57 | "kernel_panic": self.kernel_panic, 58 | "kernel_tainted": self.kernel_tainted 59 | } 60 | 61 | async def start(self) -> None: 62 | """ 63 | Attach to events and start writing data inside the monitor file. 64 | """ 65 | self._logging.info("Start monitoring") 66 | 67 | for name, coro in self._events.items(): 68 | libkirk.events.register(name, coro) 69 | 70 | async def stop(self) -> None: 71 | """ 72 | Stop monitoring events. 73 | """ 74 | self._logging.info("Stop monitoring") 75 | 76 | for name, coro in self._events.items(): 77 | libkirk.events.unregister(name, coro) 78 | 79 | async def _write(self, msg_type: str, msg: str) -> None: 80 | """ 81 | Write a message to the JSON file. 82 | """ 83 | data = { 84 | "type": msg_type, 85 | "message": msg, 86 | } 87 | 88 | data_str = json.dumps(data) 89 | 90 | async with self._lock: 91 | async with AsyncFile(self._path, 'w') as fdata: 92 | await fdata.write(data_str) 93 | 94 | @staticmethod 95 | def _test_to_dict(test: Test) -> dict: 96 | """ 97 | Convert test into a dict which can be converted to JSON. 98 | """ 99 | data = { 100 | "name": test.name, 101 | "command": test.command, 102 | "arguments": test.arguments, 103 | "parallelizable": test.parallelizable, 104 | "cwd": test.cwd, 105 | "env": test.env, 106 | } 107 | 108 | return data 109 | 110 | def _suite_to_dict(self, suite: Suite) -> dict: 111 | """ 112 | Translate suite into a dict which can be converted into JSON. 113 | """ 114 | data = { 115 | "name": suite.name, 116 | "tests": {} 117 | } 118 | 119 | tests = [] 120 | for test in suite.tests: 121 | tests.append(self._test_to_dict(test)) 122 | 123 | data["tests"] = tests 124 | 125 | return data 126 | 127 | async def session_restore(self, restore: str) -> None: 128 | await self._write("session_restore", {"restore": restore}) 129 | 130 | async def session_started(self, tmpdir: str) -> None: 131 | await self._write("session_started", {"tmpdir": tmpdir}) 132 | 133 | async def session_stopped(self) -> None: 134 | await self._write("session_stopped", {}) 135 | 136 | async def sut_stdout(self, sut: str, data: str) -> None: 137 | await self._write("sut_stdout", { 138 | "sut": sut, 139 | "data": data, 140 | }) 141 | 142 | async def sut_start(self, sut: str) -> None: 143 | await self._write("sut_start", {"sut": sut}) 144 | 145 | async def sut_stop(self, sut: str) -> None: 146 | await self._write("sut_stop", {"sut": sut}) 147 | 148 | async def sut_restart(self, sut: str) -> None: 149 | await self._write("sut_restart", {"sut": sut}) 150 | 151 | async def sut_not_responding(self) -> None: 152 | await self._write("sut_not_responding", {}) 153 | 154 | async def run_cmd_start(self, cmd: str) -> None: 155 | await self._write("run_cmd_start", {"cmd": cmd}) 156 | 157 | async def run_cmd_stop( 158 | self, 159 | command: str, 160 | stdout: str, 161 | returncode: int) -> None: 162 | await self._write("run_cmd_stop", { 163 | "command": command, 164 | "stdout": stdout, 165 | "returncode": returncode, 166 | }) 167 | 168 | async def test_stdout(self, test: Test, data: str) -> None: 169 | await self._write("test_stdout", { 170 | "test": self._test_to_dict(test), 171 | "data": data, 172 | }) 173 | 174 | async def test_started(self, test: Test) -> None: 175 | await self._write("test_started", { 176 | "test": self._test_to_dict(test), 177 | }) 178 | 179 | async def test_completed(self, results: TestResults) -> None: 180 | await self._write("test_completed", { 181 | "test": self._test_to_dict(results.test), 182 | "stdout": results.stdout, 183 | "status": results.status, 184 | "exec_time": results.exec_time, 185 | "passed": results.passed, 186 | "failed": results.failed, 187 | "broken": results.broken, 188 | "skipped": results.skipped, 189 | "warnings": results.warnings, 190 | }) 191 | 192 | async def test_timed_out(self, test: Test, timeout: int) -> None: 193 | await self._write("test_started", { 194 | "test": self._test_to_dict(test), 195 | "timeout": timeout 196 | }) 197 | 198 | async def suite_started(self, suite: Suite) -> None: 199 | await self._write("suite_started", self._suite_to_dict(suite)) 200 | 201 | async def suite_completed( 202 | self, 203 | results: SuiteResults, 204 | exec_time: float) -> None: 205 | data = { 206 | "suite": self._suite_to_dict(results.suite), 207 | "exec_time": exec_time, 208 | "total_run": len(results.suite.tests), 209 | "passed": results.passed, 210 | "failed": results.failed, 211 | "skipped": results.skipped, 212 | "broken": results.broken, 213 | "warnings": results.warnings, 214 | "kernel_version": results.kernel, 215 | "cpu": results.cpu, 216 | "arch": results.arch, 217 | "ram": results.ram, 218 | "swap": results.swap, 219 | "distro": results.distro, 220 | "distro_version": results.distro_ver 221 | } 222 | 223 | await self._write("suite_completed", data) 224 | 225 | async def suite_timeout(self, suite: Suite, timeout: float) -> None: 226 | await self._write("suite_timeout", { 227 | "suite": self._suite_to_dict(suite), 228 | "timeout": timeout, 229 | }) 230 | 231 | async def session_warning(self, msg: str) -> None: 232 | await self._write("session_warning", {"message": msg}) 233 | 234 | async def session_error(self, error: str) -> None: 235 | await self._write("session_error", {"error": error}) 236 | 237 | async def kernel_panic(self) -> None: 238 | await self._write("kernel_panic", {}) 239 | 240 | async def kernel_tainted(self, message: str) -> None: 241 | await self._write("kernel_tainted", {"message": message}) 242 | -------------------------------------------------------------------------------- /libkirk/plugin.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: plugin 3 | :platform: Linux 4 | :synopsis: generic plugin handling 5 | 6 | .. moduleauthor:: Andrea Cervesato 7 | """ 8 | import os 9 | import inspect 10 | import importlib 11 | import importlib.util 12 | 13 | 14 | class Plugin: 15 | """ 16 | Generic plugin definition. 17 | """ 18 | 19 | def setup(self, **kwargs: dict) -> None: 20 | """ 21 | Initialize plugin using configuration dictionary. 22 | :param kwargs: SUT configuration 23 | :type kwargs: dict 24 | """ 25 | raise NotImplementedError() 26 | 27 | @property 28 | def config_help(self) -> dict: 29 | """ 30 | Associate each configuration option with a help message. 31 | This is used by the main menu application to generate --help message. 32 | :returns: dict 33 | """ 34 | raise NotImplementedError() 35 | 36 | @property 37 | def name(self) -> str: 38 | """ 39 | Name of the plugin. 40 | """ 41 | raise NotImplementedError() 42 | 43 | 44 | def discover(mytype: type, folder: str) -> list: 45 | """ 46 | Discover ``mytype`` implementations inside a specific folder. 47 | """ 48 | if not folder or not os.path.isdir(folder): 49 | raise ValueError("Discover folder doesn't exist") 50 | 51 | loaded_obj = [] 52 | 53 | for myfile in os.listdir(folder): 54 | if not myfile.endswith('.py'): 55 | continue 56 | 57 | path = os.path.join(folder, myfile) 58 | if not os.path.isfile(path): 59 | continue 60 | 61 | spec = importlib.util.spec_from_file_location('obj', path) 62 | module = importlib.util.module_from_spec(spec) 63 | spec.loader.exec_module(module) 64 | 65 | members = inspect.getmembers(module, inspect.isclass) 66 | for _, klass in members: 67 | if klass.__module__ != module.__name__ or \ 68 | klass is mytype or \ 69 | klass in loaded_obj: 70 | continue 71 | 72 | if issubclass(klass, mytype): 73 | loaded_obj.append(klass()) 74 | 75 | if len(loaded_obj) > 0: 76 | loaded_obj.sort(key=lambda x: x.name) 77 | 78 | return loaded_obj 79 | -------------------------------------------------------------------------------- /libkirk/results.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: data 3 | :platform: Linux 4 | :synopsis: module containing suites data definition 5 | 6 | .. moduleauthor:: Andrea Cervesato 7 | """ 8 | from libkirk.data import Test 9 | from libkirk.data import Suite 10 | 11 | 12 | class ResultStatus: 13 | """ 14 | Overall status of the test. This is a specific flag that is used to 15 | recognize final test status. For example, we might have 10 tests passing 16 | inside a single test binary, but the overall status of the test is fine, so 17 | we assign a PASS status. 18 | """ 19 | # regular test run 20 | PASS = 0 21 | 22 | # test broken result 23 | BROK = 2 24 | 25 | # test warns 26 | WARN = 4 27 | 28 | # test failure 29 | FAIL = 16 30 | 31 | # test configuration error 32 | CONF = 32 33 | 34 | 35 | class Results: 36 | """ 37 | Base class for results. 38 | """ 39 | 40 | @property 41 | def exec_time(self) -> float: 42 | """ 43 | Execution time. 44 | :returns: float 45 | """ 46 | raise NotImplementedError() 47 | 48 | @property 49 | def failed(self) -> int: 50 | """ 51 | Number of TFAIL. 52 | :returns: int 53 | """ 54 | raise NotImplementedError() 55 | 56 | @property 57 | def passed(self) -> int: 58 | """ 59 | Number of TPASS. 60 | :returns: int 61 | """ 62 | raise NotImplementedError() 63 | 64 | @property 65 | def broken(self) -> int: 66 | """ 67 | Number of TBROK. 68 | :returns: int 69 | """ 70 | raise NotImplementedError() 71 | 72 | @property 73 | def skipped(self) -> int: 74 | """ 75 | Number of TSKIP. 76 | :returns: int 77 | """ 78 | raise NotImplementedError() 79 | 80 | @property 81 | def warnings(self) -> int: 82 | """ 83 | Number of TWARN. 84 | :returns: int 85 | """ 86 | raise NotImplementedError() 87 | 88 | 89 | class TestResults(Results): 90 | """ 91 | Test results definition. 92 | """ 93 | 94 | def __init__(self, **kwargs) -> None: 95 | """ 96 | :param test: Test object declaration 97 | :type test: Test 98 | :param failed: number of TFAIL 99 | :type failed: int 100 | :param passed: number of TPASS 101 | :type passed: int 102 | :param broken: number of TBROK 103 | :type broken: int 104 | :param skipped: number of TSKIP 105 | :type skipped: int 106 | :param warnings: number of TWARN 107 | :type warnings: int 108 | :param exec_time: time for test's execution 109 | :type exec_time: float 110 | :param status: overall status of the test 111 | :type status: int 112 | :param retcode: return code of the executed test 113 | :type retcode: int 114 | :param stdout: stdout of the test 115 | :type stdout: str 116 | """ 117 | self._test = kwargs.get("test", None) 118 | self._failed = max(kwargs.get("failed", 0), 0) 119 | self._passed = max(kwargs.get("passed", 0), 0) 120 | self._broken = max(kwargs.get("broken", 0), 0) 121 | self._skipped = max(kwargs.get("skipped", 0), 0) 122 | self._warns = max(kwargs.get("warnings", 0), 0) 123 | self._exec_t = max(kwargs.get("exec_time", 0.0), 0.0) 124 | self._retcode = kwargs.get("retcode", 0) 125 | self._status = kwargs.get("status", ResultStatus.PASS) 126 | self._stdout = kwargs.get("stdout", None) 127 | 128 | if not self._test: 129 | raise ValueError("Empty test object") 130 | 131 | def __repr__(self) -> str: 132 | return \ 133 | f"test: '{self._test}', " \ 134 | f"failed: '{self._failed}', " \ 135 | f"passed: {self._passed}, " \ 136 | f"broken: {self._broken}, " \ 137 | f"skipped: {self._skipped}, " \ 138 | f"warnins: {self._warns}, " \ 139 | f"status: {self._status}, " \ 140 | f"exec_time: {self._exec_t}, " \ 141 | f"retcode: {self._retcode}, " \ 142 | f"stdout: {repr(self._stdout)}" 143 | 144 | @property 145 | def test(self) -> Test: 146 | """ 147 | Test object declaration. 148 | :returns: Test 149 | """ 150 | return self._test 151 | 152 | @property 153 | def return_code(self) -> int: 154 | """ 155 | Return code after execution. 156 | :returns: int 157 | """ 158 | return self._retcode 159 | 160 | @property 161 | def stdout(self) -> str: 162 | """ 163 | Return the ending stdout. 164 | :returns: str 165 | """ 166 | return self._stdout 167 | 168 | @property 169 | def status(self) -> int: 170 | """ 171 | Overall test result status. 172 | :returns: int 173 | """ 174 | return self._status 175 | 176 | @property 177 | def exec_time(self) -> float: 178 | return self._exec_t 179 | 180 | @property 181 | def failed(self) -> int: 182 | return self._failed 183 | 184 | @property 185 | def passed(self) -> int: 186 | return self._passed 187 | 188 | @property 189 | def broken(self) -> int: 190 | return self._broken 191 | 192 | @property 193 | def skipped(self) -> int: 194 | return self._skipped 195 | 196 | @property 197 | def warnings(self) -> int: 198 | return self._warns 199 | 200 | 201 | class SuiteResults(Results): 202 | """ 203 | Testing suite results definition. 204 | """ 205 | 206 | def __init__(self, **kwargs) -> None: 207 | """ 208 | :param suite: Test object declaration 209 | :type suite: Suite 210 | :param tests: List of the tests results 211 | :type tests: list(TestResults) 212 | :param distro: distribution name 213 | :type distro: str 214 | :param distro_ver: distribution version 215 | :type distro_ver: str 216 | :param kernel: kernel version 217 | :type kernel: str 218 | :param arch: OS architecture 219 | :type arch: str 220 | """ 221 | self._suite = kwargs.get("suite", None) 222 | self._tests = kwargs.get("tests", []) 223 | self._distro = kwargs.get("distro", None) 224 | self._distro_ver = kwargs.get("distro_ver", None) 225 | self._kernel = kwargs.get("kernel", None) 226 | self._arch = kwargs.get("arch", None) 227 | self._cpu = kwargs.get("cpu", None) 228 | self._swap = kwargs.get("swap", None) 229 | self._ram = kwargs.get("ram", None) 230 | 231 | if not self._suite: 232 | raise ValueError("Empty suite object") 233 | 234 | def __repr__(self) -> str: 235 | return \ 236 | f"suite: '{self._suite}', " \ 237 | f"tests: '{self._tests}', " \ 238 | f"distro: {self._distro}, " \ 239 | f"distro_ver: {self._distro_ver}, " \ 240 | f"kernel: {self._kernel}, " \ 241 | f"arch: {self._arch}, " \ 242 | f"cpu: {self._cpu}, " \ 243 | f"swap: {self._swap}, " \ 244 | f"ram: {self._ram}" 245 | 246 | @property 247 | def suite(self) -> Suite: 248 | """ 249 | Suite object declaration. 250 | :returns: Suite 251 | """ 252 | return self._suite 253 | 254 | @property 255 | def tests_results(self) -> list: 256 | """ 257 | Results of all tests. 258 | :returns: list(TestResults) 259 | """ 260 | return self._tests 261 | 262 | def _get_result(self, attr: str) -> int: 263 | """ 264 | Return the total number of results. 265 | """ 266 | res = 0 267 | for test in self._tests: 268 | res += getattr(test, attr) 269 | 270 | return res 271 | 272 | @property 273 | def distro(self) -> str: 274 | """ 275 | Distribution name. 276 | """ 277 | return self._distro 278 | 279 | @property 280 | def distro_ver(self) -> str: 281 | """ 282 | Distribution version. 283 | """ 284 | return self._distro_ver 285 | 286 | @property 287 | def kernel(self) -> str: 288 | """ 289 | Kernel version. 290 | """ 291 | return self._kernel 292 | 293 | @property 294 | def arch(self) -> str: 295 | """ 296 | Operating system architecture. 297 | """ 298 | return self._arch 299 | 300 | @property 301 | def cpu(self) -> str: 302 | """ 303 | Current CPU type. 304 | """ 305 | return self._cpu 306 | 307 | @property 308 | def swap(self) -> str: 309 | """ 310 | Current swap memory occupation. 311 | """ 312 | return self._swap 313 | 314 | @property 315 | def ram(self) -> str: 316 | """ 317 | Current RAM occupation. 318 | """ 319 | return self._ram 320 | 321 | @property 322 | def exec_time(self) -> float: 323 | return self._get_result("exec_time") 324 | 325 | @property 326 | def failed(self) -> int: 327 | return self._get_result("failed") 328 | 329 | @property 330 | def passed(self) -> int: 331 | return self._get_result("passed") 332 | 333 | @property 334 | def broken(self) -> int: 335 | return self._get_result("broken") 336 | 337 | @property 338 | def skipped(self) -> int: 339 | return self._get_result("skipped") 340 | 341 | @property 342 | def warnings(self) -> int: 343 | return self._get_result("warnings") 344 | -------------------------------------------------------------------------------- /libkirk/ssh.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: ssh 3 | :platform: Linux 4 | :synopsis: module defining SSH SUT 5 | 6 | .. moduleauthor:: Andrea Cervesato 7 | """ 8 | import time 9 | import asyncio 10 | import logging 11 | import importlib 12 | import contextlib 13 | from libkirk.sut import SUT 14 | from libkirk.sut import SUTError 15 | from libkirk.sut import IOBuffer 16 | from libkirk.sut import KernelPanicError 17 | 18 | try: 19 | import asyncssh 20 | import asyncssh.misc 21 | 22 | class MySSHClientSession(asyncssh.SSHClientSession): 23 | """ 24 | Custom SSHClientSession used to store stdout during execution of commands 25 | and to check if Kernel Panic has occured in the system. 26 | """ 27 | 28 | def __init__(self, iobuffer: IOBuffer): 29 | self._output = [] 30 | self._iobuffer = iobuffer 31 | self._panic = False 32 | 33 | def data_received(self, data, _) -> None: 34 | """ 35 | Override default data_received callback, storing stdout/stderr inside 36 | a buffer and checking for kernel panic. 37 | """ 38 | self._output.append(data) 39 | 40 | if self._iobuffer: 41 | asyncio.ensure_future(self._iobuffer.write(data)) 42 | 43 | if "Kernel panic" in data: 44 | self._panic = True 45 | 46 | def kernel_panic(self) -> bool: 47 | """ 48 | True if command triggered a kernel panic during its execution. 49 | """ 50 | return self._panic 51 | 52 | def get_output(self) -> list: 53 | """ 54 | Return the list containing stored stdout/stderr messages. 55 | """ 56 | return self._output 57 | except ModuleNotFoundError: 58 | pass 59 | 60 | 61 | # pylint: disable=too-many-instance-attributes 62 | class SSHSUT(SUT): 63 | """ 64 | A SUT that is using SSH protocol con communicate and transfer data. 65 | """ 66 | 67 | def __init__(self) -> None: 68 | self._logger = logging.getLogger("kirk.ssh") 69 | self._tmpdir = None 70 | self._host = None 71 | self._port = None 72 | self._reset_cmd = None 73 | self._user = None 74 | self._password = None 75 | self._key_file = None 76 | self._sudo = False 77 | self._known_hosts = None 78 | self._session_sem = None 79 | self._stop = False 80 | self._conn = None 81 | self._downloader = None 82 | self._channels = [] 83 | 84 | @property 85 | def name(self) -> str: 86 | return "ssh" 87 | 88 | @property 89 | def config_help(self) -> dict: 90 | return { 91 | "host": "IP address of the SUT (default: localhost)", 92 | "port": "TCP port of the service (default: 22)", 93 | "user": "name of the user (default: root)", 94 | "password": "root password", 95 | "key_file": "private key location", 96 | "reset_cmd": "command to reset the remote SUT", 97 | "sudo": "use sudo to access to root shell (default: 0)", 98 | "known_hosts": "path to custom known_hosts file (optional)", 99 | } 100 | 101 | async def _reset(self, iobuffer: IOBuffer = None) -> None: 102 | """ 103 | Run the reset command on host. 104 | """ 105 | if not self._reset_cmd: 106 | return 107 | 108 | self._logger.info("Executing reset command: %s", repr(self._reset_cmd)) 109 | 110 | proc = await asyncio.create_subprocess_shell( 111 | self._reset_cmd, 112 | stdout=asyncio.subprocess.PIPE, 113 | stderr=asyncio.subprocess.PIPE) 114 | 115 | while True: 116 | line = await proc.stdout.read(1024) 117 | if line: 118 | sline = line.decode(encoding="utf-8", errors="ignore") 119 | 120 | if iobuffer: 121 | await iobuffer.write(sline) 122 | 123 | with contextlib.suppress(asyncio.TimeoutError): 124 | returncode = await asyncio.wait_for(proc.wait(), 1e-6) 125 | if returncode is not None: 126 | break 127 | 128 | await proc.wait() 129 | 130 | self._logger.info("Reset command has been executed") 131 | 132 | def _create_command(self, cmd: str, cwd: str, env: dict) -> str: 133 | """ 134 | Create command to send to SSH client. 135 | """ 136 | args = [] 137 | 138 | if cwd: 139 | args.append(f"cd {cwd} && ") 140 | 141 | if env: 142 | for key, value in env.items(): 143 | args.append(f"export {key}={value} && ") 144 | 145 | args.append(cmd) 146 | 147 | script = ''.join(args) 148 | if self._sudo: 149 | script = f"sudo /bin/sh -c '{script}'" 150 | 151 | return script 152 | 153 | def setup(self, **kwargs: dict) -> None: 154 | if not importlib.util.find_spec('asyncssh'): 155 | raise SUTError("'asyncssh' library is not available") 156 | 157 | self._logger.info("Initialize SUT") 158 | 159 | self._tmpdir = kwargs.get("tmpdir", None) 160 | self._host = kwargs.get("host", "localhost") 161 | self._port = kwargs.get("port", 22) 162 | self._reset_cmd = kwargs.get("reset_cmd", None) 163 | self._user = kwargs.get("user", "root") 164 | self._password = kwargs.get("password", None) 165 | self._key_file = kwargs.get("key_file", None) 166 | self._known_hosts = kwargs.get("known_hosts", "~/.ssh/known_hosts") 167 | 168 | if self._known_hosts == "/dev/null": 169 | self._known_hosts = None 170 | 171 | try: 172 | self._port = int(kwargs.get("port", "22")) 173 | 174 | if 1 > self._port > 65535: 175 | raise ValueError() 176 | except ValueError: 177 | raise SUTError("'port' must be an integer between 1-65535") 178 | 179 | try: 180 | self._sudo = int(kwargs.get("sudo", 0)) == 1 181 | except ValueError: 182 | raise SUTError("'sudo' must be 0 or 1") 183 | 184 | @property 185 | def parallel_execution(self) -> bool: 186 | return True 187 | 188 | @property 189 | async def is_running(self) -> bool: 190 | return self._conn is not None 191 | 192 | async def communicate(self, iobuffer: IOBuffer = None) -> None: 193 | if await self.is_running: 194 | raise SUTError("SUT is already running") 195 | 196 | try: 197 | self._conn = None 198 | if self._key_file: 199 | priv_key = asyncssh.read_private_key(self._key_file) 200 | 201 | self._conn = await asyncssh.connect( 202 | host=self._host, 203 | port=self._port, 204 | username=self._user, 205 | client_keys=[priv_key], 206 | known_hosts=self._known_hosts) 207 | else: 208 | self._conn = await asyncssh.connect( 209 | host=self._host, 210 | port=self._port, 211 | username=self._user, 212 | password=self._password, 213 | known_hosts=self._known_hosts) 214 | 215 | # read maximum number of sessions and limit `run_command` 216 | # concurrent calls to that by using a semaphore 217 | ret = await self._conn.run( 218 | r'sed -n "s/^MaxSessions\s*\([[:digit:]]*\)/\1/p" ' 219 | '/etc/ssh/sshd_config') 220 | 221 | max_sessions = ret.stdout or 10 222 | 223 | self._logger.info("Maximum SSH sessions: %d", max_sessions) 224 | self._session_sem = asyncio.Semaphore(max_sessions) 225 | except asyncssh.misc.Error as err: 226 | if not self._stop: 227 | raise SUTError(err) from err 228 | 229 | async def stop(self, iobuffer: IOBuffer = None) -> None: 230 | if not await self.is_running: 231 | return 232 | 233 | self._stop = True 234 | try: 235 | if self._channels: 236 | self._logger.info("Killing %d process(es)", 237 | len(self._channels)) 238 | 239 | for proc in self._channels: 240 | proc.kill() 241 | await proc.wait_closed() 242 | 243 | self._channels.clear() 244 | 245 | if self._downloader: 246 | await self._downloader.close() 247 | 248 | self._logger.info("Closing connection") 249 | self._conn.close() 250 | await self._conn.wait_closed() 251 | self._logger.info("Connection closed") 252 | 253 | await self._reset(iobuffer=iobuffer) 254 | finally: 255 | self._stop = False 256 | self._conn = None 257 | 258 | async def ping(self) -> float: 259 | if not await self.is_running: 260 | raise SUTError("SUT is not running") 261 | 262 | start_t = time.time() 263 | 264 | self._logger.info("Ping %s:%d", self._host, self._port) 265 | 266 | try: 267 | await self._conn.run("test .", check=True) 268 | except asyncssh.Error as err: 269 | raise SUTError(err) from err 270 | 271 | end_t = time.time() - start_t 272 | 273 | self._logger.info("SUT replied after %.3f seconds", end_t) 274 | 275 | return end_t 276 | 277 | async def run_command( 278 | self, 279 | command: str, 280 | cwd: str = None, 281 | env: dict = None, 282 | iobuffer: IOBuffer = None) -> dict: 283 | if not command: 284 | raise ValueError("command is empty") 285 | 286 | if not await self.is_running: 287 | raise SUTError("SSH connection is not present") 288 | 289 | async with self._session_sem: 290 | cmd = self._create_command(command, cwd, env) 291 | ret = None 292 | start_t = 0 293 | stdout = [] 294 | panic = False 295 | channel = None 296 | session = None 297 | 298 | try: 299 | self._logger.info("Running command: %s", repr(command)) 300 | 301 | channel, session = await self._conn.create_session( 302 | lambda: MySSHClientSession(iobuffer), 303 | cmd 304 | ) 305 | 306 | self._channels.append(channel) 307 | start_t = time.time() 308 | 309 | await channel.wait_closed() 310 | 311 | panic = session.kernel_panic() 312 | stdout = session.get_output() 313 | finally: 314 | if channel: 315 | self._channels.remove(channel) 316 | 317 | ret = { 318 | "command": command, 319 | "returncode": channel.get_returncode(), 320 | "exec_time": time.time() - start_t, 321 | "stdout": "".join(stdout) 322 | } 323 | 324 | if panic: 325 | raise KernelPanicError() 326 | 327 | self._logger.info("Command executed") 328 | self._logger.debug(ret) 329 | 330 | return ret 331 | 332 | async def fetch_file(self, target_path: str) -> bytes: 333 | if not target_path: 334 | raise ValueError("target path is empty") 335 | 336 | if not await self.is_running: 337 | raise SUTError("SSH connection is not present") 338 | 339 | data = None 340 | try: 341 | ret = await self._conn.run( 342 | f"cat {target_path}", 343 | check=True, 344 | encoding=None) 345 | 346 | data = ret.stdout 347 | except asyncssh.Error as err: 348 | if not self._stop: 349 | raise SUTError(err) from err 350 | 351 | return data 352 | -------------------------------------------------------------------------------- /libkirk/sut.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: sut 3 | :platform: Linux 4 | :synopsis: sut definition 5 | 6 | .. moduleauthor:: Andrea Cervesato 7 | """ 8 | import re 9 | import asyncio 10 | from libkirk import KirkException 11 | from libkirk.plugin import Plugin 12 | 13 | 14 | class SUTError(KirkException): 15 | """ 16 | Raised when an error occurs in SUT. 17 | """ 18 | 19 | 20 | class KernelPanicError(SUTError): 21 | """ 22 | Raised during kernel panic. 23 | """ 24 | 25 | 26 | class IOBuffer: 27 | """ 28 | IO stdout buffer. The API is similar to ``IO`` types. 29 | """ 30 | 31 | async def write(self, data: str) -> None: 32 | """ 33 | Write data. 34 | """ 35 | raise NotImplementedError() 36 | 37 | 38 | TAINTED_MSG = [ 39 | "proprietary module was loaded", 40 | "module was force loaded", 41 | "kernel running on an out of specification system", 42 | "module was force unloaded", 43 | "processor reported a Machine Check Exception (MCE)", 44 | "bad page referenced or some unexpected page flags", 45 | "taint requested by userspace application", 46 | "kernel died recently, i.e. there was an OOPS or BUG", 47 | "ACPI table overridden by user", 48 | "kernel issued warning", 49 | "staging driver was loaded", 50 | "workaround for bug in platform firmware applied", 51 | "externally-built (“out-of-tree”) module was loaded", 52 | "unsigned module was loaded", 53 | "soft lockup occurred", 54 | "kernel has been live patched", 55 | "auxiliary taint, defined for and used by distros", 56 | "kernel was built with the struct randomization plugin" 57 | ] 58 | 59 | 60 | class SUT(Plugin): 61 | """ 62 | SUT abstraction class. It could be a remote host, a local host, a virtual 63 | machine instance, etc. 64 | """ 65 | 66 | @property 67 | def parallel_execution(self) -> bool: 68 | """ 69 | If True, SUT supports commands parallel execution. 70 | """ 71 | raise NotImplementedError() 72 | 73 | @property 74 | async def is_running(self) -> bool: 75 | """ 76 | Return True if SUT is running. 77 | """ 78 | raise NotImplementedError() 79 | 80 | async def ping(self) -> float: 81 | """ 82 | If SUT is replying and it's available, ping will return time needed to 83 | wait for SUT reply. 84 | :returns: float 85 | """ 86 | raise NotImplementedError() 87 | 88 | async def communicate(self, iobuffer: IOBuffer = None) -> None: 89 | """ 90 | Start communicating with the SUT. 91 | :param iobuffer: buffer used to write SUT stdout 92 | :type iobuffer: IOBuffer 93 | """ 94 | raise NotImplementedError() 95 | 96 | async def stop(self, iobuffer: IOBuffer = None) -> None: 97 | """ 98 | Stop the current SUT session. 99 | :param iobuffer: buffer used to write SUT stdout 100 | :type iobuffer: IOBuffer 101 | """ 102 | raise NotImplementedError() 103 | 104 | async def run_command( 105 | self, 106 | command: str, 107 | cwd: str = None, 108 | env: dict = None, 109 | iobuffer: IOBuffer = None) -> dict: 110 | """ 111 | Coroutine to run command on target. 112 | :param command: command to execute 113 | :type command: str 114 | :param cwd: current working directory 115 | :type cwd: str 116 | :param env: environment variables 117 | :type env: dict 118 | :param iobuffer: buffer used to write SUT stdout 119 | :type iobuffer: IOBuffer 120 | :returns: dictionary containing command execution information 121 | 122 | { 123 | "command": , 124 | "returncode": , 125 | "stdout": , 126 | "exec_time": , 127 | } 128 | 129 | If None is returned, then callback failed. 130 | """ 131 | raise NotImplementedError() 132 | 133 | async def fetch_file(self, target_path: str) -> bytes: 134 | """ 135 | Fetch file from target path and return data from target path. 136 | :param target_path: path of the file to download from target 137 | :type target_path: str 138 | :returns: bytes contained in target_path 139 | """ 140 | raise NotImplementedError() 141 | 142 | async def ensure_communicate( 143 | self, 144 | iobuffer: IOBuffer = None, 145 | retries: int = 10) -> None: 146 | """ 147 | Ensure that `communicate` is completed, retrying as many times we 148 | want in case of `KirkException` error. After each `communicate` error 149 | the SUT is stopped and a new communication is tried. 150 | :param iobuffer: buffer used to write SUT stdout 151 | :type iobuffer: IOBuffer 152 | :param retries: number of times we retry communicating with SUT 153 | :type retries: int 154 | """ 155 | retries = max(retries, 1) 156 | 157 | for retry in range(retries): 158 | try: 159 | await self.communicate(iobuffer=iobuffer) 160 | break 161 | except KirkException as err: 162 | if retry >= retries - 1: 163 | raise err 164 | 165 | await self.stop(iobuffer=iobuffer) 166 | 167 | async def get_info(self) -> dict: 168 | """ 169 | Return SUT information. 170 | :returns: dict 171 | 172 | { 173 | "distro": str, 174 | "distro_ver": str, 175 | "kernel": str, 176 | "arch": str, 177 | "cpu" : str, 178 | "swap" : str, 179 | "ram" : str, 180 | } 181 | 182 | """ 183 | # create suite results 184 | async def _run_cmd(cmd: str) -> str: 185 | """ 186 | Run command, check for returncode and return command's stdout. 187 | """ 188 | stdout = "unknown" 189 | try: 190 | ret = await asyncio.wait_for(self.run_command(cmd), 1.5) 191 | if ret["returncode"] == 0: 192 | stdout = ret["stdout"].rstrip() 193 | except asyncio.TimeoutError: 194 | pass 195 | 196 | return stdout 197 | 198 | distro, \ 199 | distro_ver, \ 200 | kernel, \ 201 | arch, \ 202 | cpu, \ 203 | meminfo = await asyncio.gather(*[ 204 | _run_cmd(". /etc/os-release && echo \"$ID\""), 205 | _run_cmd(". /etc/os-release && echo \"$VERSION_ID\""), 206 | _run_cmd("uname -s -r -v"), 207 | _run_cmd("uname -m"), 208 | _run_cmd("uname -p"), 209 | _run_cmd("cat /proc/meminfo") 210 | ]) 211 | 212 | memory = "unknown" 213 | swap = "unkown" 214 | 215 | if meminfo: 216 | mem_m = re.search(r'MemTotal:\s+(?P\d+\s+kB)', meminfo) 217 | if mem_m: 218 | memory = mem_m.group('memory') 219 | 220 | swap_m = re.search(r'SwapTotal:\s+(?P\d+\s+kB)', meminfo) 221 | if swap_m: 222 | swap = swap_m.group('swap') 223 | 224 | ret = { 225 | "distro": distro, 226 | "distro_ver": distro_ver, 227 | "kernel": kernel, 228 | "arch": arch, 229 | "cpu": cpu, 230 | "ram": memory, 231 | "swap": swap 232 | } 233 | 234 | return ret 235 | 236 | _tainted_lock = asyncio.Lock() 237 | _tainted_status = asyncio.Queue(maxsize=1) 238 | 239 | async def get_tainted_info(self) -> tuple: 240 | """ 241 | Return information about kernel if tainted. 242 | :returns: (int, list[str]) 243 | """ 244 | if self._tainted_lock.locked() and self._tainted_status.qsize() > 0: 245 | status = await self._tainted_status.get() 246 | return status 247 | 248 | async with self._tainted_lock: 249 | ret = await self.run_command("cat /proc/sys/kernel/tainted") 250 | if ret["returncode"] != 0: 251 | raise SUTError("Can't read tainted kernel information") 252 | 253 | tainted_num = len(TAINTED_MSG) 254 | code = ret["stdout"].strip() 255 | 256 | # output is likely message in stderr 257 | if not code.isdigit(): 258 | raise SUTError(code) 259 | 260 | code = int(code) 261 | bits = format(code, f"0{tainted_num}b")[::-1] 262 | 263 | messages = [] 264 | for i in range(0, tainted_num): 265 | if bits[i] == "1": 266 | msg = TAINTED_MSG[i] 267 | messages.append(msg) 268 | 269 | if self._tainted_status.qsize() > 0: 270 | await self._tainted_status.get() 271 | 272 | await self._tainted_status.put((code, messages)) 273 | 274 | return code, messages 275 | -------------------------------------------------------------------------------- /libkirk/tempfile.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: tempfile 3 | :platform: Linux 4 | :synopsis: module that contains temporary files handling 5 | 6 | .. moduleauthor:: Andrea Cervesato 7 | """ 8 | import os 9 | import pwd 10 | import shutil 11 | import pathlib 12 | import tempfile 13 | 14 | 15 | class TempDir: 16 | """ 17 | Temporary directory handler. 18 | """ 19 | SYMLINK_NAME = "latest" 20 | FOLDER_PREFIX = "kirk." 21 | 22 | def __init__(self, root: str = None, max_rotate: int = 5) -> None: 23 | """ 24 | :param root: root directory (i.e. /tmp). If None, TempDir will handle 25 | requests without adding any file or directory. 26 | :type root: str | None 27 | :param max_rotate: maximum number of temporary directories 28 | :type max_rotate: int 29 | """ 30 | if root and not os.path.isdir(root): 31 | raise ValueError(f"root folder doesn't exist: {root}") 32 | 33 | self._root = root 34 | if root: 35 | self._root = os.path.abspath(root) 36 | 37 | self._max_rotate = max(max_rotate, 0) 38 | self._folder = self._rotate() 39 | 40 | def _rotate(self) -> str: 41 | """ 42 | Check for old folders and remove them, then create a new one and return 43 | its full path. 44 | """ 45 | if not self._root: 46 | return "" 47 | 48 | name = pwd.getpwuid(os.getuid()).pw_name 49 | tmpbase = os.path.join(self._root, f"{self.FOLDER_PREFIX}{name}") 50 | 51 | os.makedirs(tmpbase, exist_ok=True) 52 | 53 | # delete the first max_rotate items 54 | sorted_paths = sorted( 55 | pathlib.Path(tmpbase).iterdir(), 56 | key=os.path.getmtime) 57 | 58 | # don't consider latest symlink 59 | num_paths = len(sorted_paths) - 1 60 | 61 | if num_paths >= self._max_rotate: 62 | max_items = num_paths - self._max_rotate + 1 63 | paths = sorted_paths[:max_items] 64 | 65 | for path in paths: 66 | if path.name == self.SYMLINK_NAME: 67 | continue 68 | 69 | shutil.rmtree(str(path.resolve())) 70 | 71 | # create a new folder 72 | folder = tempfile.mkdtemp(dir=tmpbase) 73 | 74 | # create symlink to the latest temporary directory 75 | latest = os.path.join(tmpbase, self.SYMLINK_NAME) 76 | if os.path.islink(latest): 77 | os.remove(latest) 78 | 79 | os.symlink( 80 | folder, 81 | os.path.join(tmpbase, self.SYMLINK_NAME), 82 | target_is_directory=True) 83 | 84 | return folder 85 | 86 | @property 87 | def root(self) -> str: 88 | """ 89 | The root folder. For example, if temporary folder is 90 | "/tmp/kirk.acer/tmpf547ftxv" the method will return "/tmp". 91 | If root folder has not been given during object creation, this 92 | method returns an empty string. 93 | """ 94 | return self._root if self._root else "" 95 | 96 | @property 97 | def abspath(self) -> str: 98 | """ 99 | Absolute path of the temporary directory. 100 | """ 101 | return self._folder 102 | 103 | def mkdir(self, path: str) -> None: 104 | """ 105 | Create a directory inside temporary directory. 106 | :param path: path of the directory 107 | :type path: str 108 | :returns: folder path. 109 | """ 110 | if not self._folder: 111 | return 112 | 113 | dpath = os.path.join(self._folder, path) 114 | os.mkdir(dpath) 115 | 116 | def mkfile(self, path: str, content: bytes) -> None: 117 | """ 118 | Create a file inside temporary directory. 119 | :param path: path of the file 120 | :type path: str 121 | :param content: file content 122 | :type content: str 123 | """ 124 | if not self._folder: 125 | return 126 | 127 | fpath = os.path.join(self._folder, path) 128 | with open(fpath, "w+", encoding="utf-8") as mypath: 129 | mypath.write(content) 130 | -------------------------------------------------------------------------------- /libkirk/tests/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: __init__ 3 | :platform: Linux 4 | :synopsis: Entry point of the testing suite 5 | .. moduleauthor:: Andrea Cervesato 6 | """ 7 | 8 | import os 9 | import sys 10 | 11 | # include library 12 | sys.path.insert(0, os.path.abspath(os.path.dirname(os.path.dirname(__file__)))) 13 | -------------------------------------------------------------------------------- /libkirk/tests/conftest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Generic stuff for pytest. 3 | """ 4 | import libkirk 5 | import pytest 6 | from libkirk.results import TestResults 7 | from libkirk.sut import SUT 8 | from libkirk.framework import Framework 9 | from libkirk.data import Suite 10 | from libkirk.data import Test 11 | 12 | 13 | @pytest.fixture(scope="session") 14 | def event_loop(): 15 | """ 16 | Current event loop. Keep it in session scope, otherwise tests which 17 | will use same coroutines will be associated to different event_loop. 18 | In this way, pytest-asyncio plugin will work properly. 19 | """ 20 | loop = libkirk.get_event_loop() 21 | 22 | yield loop 23 | 24 | if not loop.is_closed(): 25 | loop.close() 26 | 27 | 28 | class DummyFramework(Framework): 29 | """ 30 | A generic framework created for testing. 31 | """ 32 | 33 | def __init__(self) -> None: 34 | self._root = None 35 | 36 | def setup(self, **kwargs: dict) -> None: 37 | self._root = kwargs.get("root", "/") 38 | self._env = kwargs.get("env", None) 39 | 40 | @property 41 | def name(self) -> str: 42 | return "dummy" 43 | 44 | @property 45 | def config_help(self) -> dict: 46 | return {} 47 | 48 | async def get_suites(self, sut: SUT) -> list: 49 | return ["suite01", "suite02", "sleep", "environ", "kernel_panic"] 50 | 51 | async def find_command(self, sut: SUT, command: str) -> Test: 52 | return Test(name=command, cmd=command) 53 | 54 | async def find_suite(self, sut: SUT, name: str) -> Suite: 55 | if name in "suite01": 56 | test0 = Test( 57 | name="test01", 58 | cwd=self._root, 59 | env=self._env, 60 | cmd="echo", 61 | args=["-n", "ciao0"], 62 | parallelizable=False) 63 | 64 | test1 = Test( 65 | name="test02", 66 | cwd=self._root, 67 | env=self._env, 68 | cmd="echo", 69 | args=["-n", "ciao0"], 70 | parallelizable=False) 71 | 72 | return Suite(name, [test0, test1]) 73 | if name == "suite02": 74 | test0 = Test( 75 | name="test01", 76 | cwd=self._root, 77 | env=self._env, 78 | cmd="echo", 79 | args=["-n", "ciao0"], 80 | parallelizable=False) 81 | 82 | test1 = Test( 83 | name="test02", 84 | cwd=self._root, 85 | env=self._env, 86 | cmd="sleep", 87 | args=["0.2", "&&", "echo", "-n", "ciao1"], 88 | parallelizable=True) 89 | 90 | return Suite(name, [test0, test1]) 91 | elif name == "sleep": 92 | test0 = Test( 93 | name="test01", 94 | cwd=self._root, 95 | env=self._env, 96 | cmd="sleep", 97 | args=["2"], 98 | parallelizable=False) 99 | 100 | test1 = Test( 101 | name="test02", 102 | cwd=self._root, 103 | env=self._env, 104 | cmd="sleep", 105 | args=["2"], 106 | parallelizable=False) 107 | 108 | return Suite(name, [test0, test1]) 109 | elif name == "environ": 110 | test0 = Test( 111 | name="test01", 112 | cwd=self._root, 113 | env=self._env, 114 | cmd="echo", 115 | args=["-n", "$hello"], 116 | parallelizable=False) 117 | 118 | return Suite(name, [test0]) 119 | elif name == "kernel_panic": 120 | test0 = Test( 121 | name="test01", 122 | cwd=self._root, 123 | env=self._env, 124 | cmd="echo", 125 | args=["Kernel", "panic"], 126 | parallelizable=False) 127 | 128 | test1 = Test( 129 | name="test01", 130 | cwd=self._root, 131 | env=self._env, 132 | cmd="sleep", 133 | args=["0.2"], 134 | parallelizable=False) 135 | 136 | return Suite(name, [test0, test1]) 137 | 138 | return None 139 | 140 | async def read_result( 141 | self, 142 | test: Test, 143 | stdout: str, 144 | retcode: int, 145 | exec_t: float) -> TestResults: 146 | passed = 0 147 | failed = 0 148 | skipped = 0 149 | broken = 0 150 | skipped = 0 151 | warnings = 0 152 | error = retcode == -1 153 | 154 | if retcode == 0: 155 | passed = 1 156 | elif retcode == 4: 157 | warnings = 1 158 | elif retcode == 32: 159 | skipped = 1 160 | elif not error: 161 | failed = 1 162 | 163 | if error: 164 | broken = 1 165 | 166 | result = TestResults( 167 | test=test, 168 | passed=passed, 169 | failed=failed, 170 | broken=broken, 171 | skipped=skipped, 172 | warnings=warnings, 173 | exec_time=exec_t, 174 | retcode=retcode, 175 | stdout=stdout, 176 | ) 177 | 178 | return result 179 | 180 | 181 | @pytest.fixture 182 | def dummy_framework(): 183 | """ 184 | A fummy framework implementation used for testing. 185 | """ 186 | obj = DummyFramework() 187 | obj.setup(root="/tmp") 188 | yield obj 189 | -------------------------------------------------------------------------------- /libkirk/tests/test_events.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unittest for events module. 3 | """ 4 | import asyncio 5 | import pytest 6 | import libkirk 7 | 8 | 9 | @pytest.fixture(autouse=True) 10 | def cleanup(): 11 | """ 12 | Cleanup all events after each test. 13 | """ 14 | yield 15 | libkirk.events.reset() 16 | 17 | 18 | def test_reset(): 19 | """ 20 | Test reset method. 21 | """ 22 | async def funct(): 23 | pass 24 | 25 | libkirk.events.register("myevent", funct) 26 | assert libkirk.events.is_registered("myevent") 27 | 28 | libkirk.events.reset() 29 | assert not libkirk.events.is_registered("myevent") 30 | 31 | 32 | def test_register_errors(): 33 | """ 34 | Test register method during errors. 35 | """ 36 | async def funct(): 37 | pass 38 | 39 | with pytest.raises(ValueError): 40 | libkirk.events.register(None, funct) 41 | 42 | with pytest.raises(ValueError): 43 | libkirk.events.register("myevent", None) 44 | 45 | 46 | def test_register(): 47 | """ 48 | Test register method. 49 | """ 50 | async def funct(): 51 | pass 52 | 53 | libkirk.events.register("myevent", funct) 54 | assert libkirk.events.is_registered("myevent") 55 | 56 | 57 | def test_unregister_errors(): 58 | """ 59 | Test unregister method during errors. 60 | """ 61 | with pytest.raises(ValueError): 62 | libkirk.events.unregister(None) 63 | 64 | 65 | def test_unregister_all(): 66 | """ 67 | Test unregister method removing all coroutine 68 | from the events list. 69 | """ 70 | async def funct1(): 71 | pass 72 | 73 | async def funct2(): 74 | pass 75 | 76 | assert not libkirk.events.is_registered("myevent") 77 | 78 | # register events first 79 | libkirk.events.register("myevent", funct1) 80 | assert libkirk.events.is_registered("myevent") 81 | 82 | libkirk.events.register("myevent", funct2) 83 | assert libkirk.events.is_registered("myevent") 84 | 85 | # unregister events one by one 86 | libkirk.events.unregister("myevent", funct1) 87 | assert libkirk.events.is_registered("myevent") 88 | 89 | libkirk.events.unregister("myevent", funct2) 90 | assert not libkirk.events.is_registered("myevent") 91 | 92 | 93 | def test_unregister_single(): 94 | """ 95 | Test unregister method removing a single coroutine 96 | from the events list. 97 | """ 98 | async def funct(): 99 | pass 100 | 101 | libkirk.events.register("myevent", funct) 102 | assert libkirk.events.is_registered("myevent") 103 | 104 | libkirk.events.unregister("myevent", funct) 105 | assert not libkirk.events.is_registered("myevent") 106 | 107 | 108 | @pytest.mark.asyncio 109 | async def test_fire_errors(): 110 | """ 111 | Test fire method during errors. 112 | """ 113 | with pytest.raises(ValueError): 114 | await libkirk.events.fire(None, "prova") 115 | 116 | 117 | @pytest.mark.asyncio 118 | async def test_fire(): 119 | """ 120 | Test fire method. 121 | """ 122 | times = 100 123 | called = [] 124 | 125 | async def diehard(error, name): 126 | assert error is not None 127 | assert name is not None 128 | 129 | async def tofire(param): 130 | called.append(param) 131 | 132 | async def start(): 133 | await libkirk.events.start() 134 | 135 | async def run(): 136 | for i in range(times): 137 | await libkirk.events.fire("myevent", i) 138 | 139 | while len(called) < times: 140 | await asyncio.sleep(1e-3) 141 | 142 | await libkirk.events.stop() 143 | 144 | libkirk.events.register("myevent", tofire) 145 | assert libkirk.events.is_registered("myevent") 146 | 147 | libkirk.events.register("internal_error", diehard) 148 | assert libkirk.events.is_registered("internal_error") 149 | 150 | libkirk.create_task(start()) 151 | await run() 152 | 153 | while len(called) < times: 154 | asyncio.sleep(1e-3) 155 | 156 | called.sort() 157 | for i in range(times): 158 | assert called[i] == i 159 | -------------------------------------------------------------------------------- /libkirk/tests/test_export.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unit tests for Exporter implementations. 3 | """ 4 | import json 5 | import asyncio 6 | import pytest 7 | from libkirk.data import Test 8 | from libkirk.data import Suite 9 | from libkirk.results import SuiteResults, TestResults, ResultStatus 10 | from libkirk.export import JSONExporter 11 | 12 | 13 | pytestmark = pytest.mark.asyncio 14 | 15 | 16 | class TestJSONExporter: 17 | """ 18 | Test JSONExporter class implementation. 19 | """ 20 | 21 | async def test_save_file_bad_args(self): 22 | """ 23 | Test save_file method with bad arguments. 24 | """ 25 | exporter = JSONExporter() 26 | 27 | with pytest.raises(ValueError): 28 | await exporter.save_file(list(), "") 29 | 30 | with pytest.raises(ValueError): 31 | await exporter.save_file(None, "") 32 | 33 | with pytest.raises(ValueError): 34 | await exporter.save_file([0, 1], None) 35 | 36 | async def test_save_file(self, tmpdir): 37 | """ 38 | Test save_file method. 39 | """ 40 | # create suite/test metadata objects 41 | tests = [ 42 | Test(name="ls0", cmd="ls"), 43 | Test(name="ls1", cmd="ls", args=["-l"]), 44 | Test(name="ls2", cmd="ls", args=["--error"]) 45 | ] 46 | suite0 = Suite("ls_suite0", tests) 47 | 48 | # create results objects 49 | tests_res = [ 50 | TestResults( 51 | test=tests[0], 52 | failed=0, 53 | passed=1, 54 | broken=0, 55 | skipped=0, 56 | warnings=0, 57 | exec_time=1, 58 | retcode=0, 59 | stdout="folder\nfile.txt", 60 | status=0, 61 | ), 62 | TestResults( 63 | test=tests[1], 64 | failed=0, 65 | passed=1, 66 | broken=0, 67 | skipped=0, 68 | warnings=0, 69 | exec_time=1, 70 | retcode=0, 71 | stdout="folder\nfile.txt", 72 | status=0, 73 | ), 74 | TestResults( 75 | test=tests[2], 76 | failed=1, 77 | passed=0, 78 | broken=0, 79 | skipped=0, 80 | warnings=0, 81 | exec_time=1, 82 | retcode=1, 83 | stdout="", 84 | status=ResultStatus.FAIL, 85 | ), 86 | ] 87 | 88 | suite_res = [ 89 | SuiteResults( 90 | suite=suite0, 91 | tests=tests_res, 92 | distro="openSUSE-Leap", 93 | distro_ver="15.3", 94 | kernel="5.17", 95 | arch="x86_64", 96 | cpu="x86_64", 97 | swap="10 kB", 98 | ram="1000 kB", 99 | exec_time=3), 100 | ] 101 | 102 | exporter = JSONExporter() 103 | tasks = [] 104 | 105 | for i in range(100): 106 | output = tmpdir / f"output{i}.json" 107 | tasks.append(exporter.save_file(suite_res, str(output))) 108 | 109 | await asyncio.gather(*tasks, return_exceptions=True) 110 | 111 | for i in range(100): 112 | data = None 113 | 114 | output = tmpdir / f"output{i}.json" 115 | with open(str(output), 'r') as json_data: 116 | data = json.load(json_data) 117 | 118 | assert len(data["results"]) == 3 119 | assert data["results"][0] == { 120 | "test": { 121 | "command": "ls", 122 | "arguments": [], 123 | "failed": 0, 124 | "passed": 1, 125 | "broken": 0, 126 | "skipped": 0, 127 | "warnings": 0, 128 | "duration": 1, 129 | "result": "pass", 130 | "log": "folder\nfile.txt", 131 | "retval": [ 132 | "0" 133 | ], 134 | }, 135 | "status": "pass", 136 | "test_fqn": "ls0", 137 | } 138 | assert data["results"][1] == { 139 | "test": { 140 | "command": "ls", 141 | "arguments": ["-l"], 142 | "failed": 0, 143 | "passed": 1, 144 | "broken": 0, 145 | "skipped": 0, 146 | "warnings": 0, 147 | "duration": 1, 148 | "result": "pass", 149 | "log": "folder\nfile.txt", 150 | "retval": [ 151 | "0" 152 | ], 153 | }, 154 | "status": "pass", 155 | "test_fqn": "ls1", 156 | } 157 | assert data["results"][2] == { 158 | "test": { 159 | "command": "ls", 160 | "arguments": ["--error"], 161 | "failed": 1, 162 | "passed": 0, 163 | "broken": 0, 164 | "skipped": 0, 165 | "warnings": 0, 166 | "duration": 1, 167 | "result": "fail", 168 | "log": "", 169 | "retval": [ 170 | "1" 171 | ], 172 | }, 173 | "status": "fail", 174 | "test_fqn": "ls2", 175 | } 176 | 177 | assert data["environment"] == { 178 | "distribution_version": "15.3", 179 | "distribution": "openSUSE-Leap", 180 | "kernel": "5.17", 181 | "arch": "x86_64", 182 | "cpu": "x86_64", 183 | "swap": "10 kB", 184 | "RAM": "1000 kB", 185 | } 186 | assert data["stats"] == { 187 | "runtime": 3, 188 | "passed": 2, 189 | "failed": 1, 190 | "broken": 0, 191 | "skipped": 0, 192 | "warnings": 0, 193 | } 194 | -------------------------------------------------------------------------------- /libkirk/tests/test_host.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unittests for host SUT implementations. 3 | """ 4 | import pytest 5 | from libkirk.host import HostSUT 6 | from libkirk.tests.test_sut import _TestSUT 7 | from libkirk.tests.test_session import _TestSession 8 | 9 | 10 | pytestmark = pytest.mark.asyncio 11 | 12 | 13 | @pytest.fixture 14 | async def sut(): 15 | _sut = HostSUT() 16 | _sut.setup() 17 | 18 | yield _sut 19 | 20 | if await _sut.is_running: 21 | await _sut.stop() 22 | 23 | 24 | class TestHostSUT(_TestSUT): 25 | """ 26 | Test HostSUT implementation. 27 | """ 28 | 29 | @pytest.fixture 30 | def sut_stop_sleep(self, request): 31 | """ 32 | Host SUT test doesn't require time sleep in `test_stop_communicate`. 33 | """ 34 | return request.param * 0 35 | 36 | async def test_fetch_file_stop(self): 37 | pytest.skip(reason="Coroutines don't support I/O file handling") 38 | 39 | 40 | class TestHostSession(_TestSession): 41 | """ 42 | Test Session implementation. 43 | """ 44 | -------------------------------------------------------------------------------- /libkirk/tests/test_io.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unittests for io module. 3 | """ 4 | import pytest 5 | from libkirk.io import AsyncFile 6 | 7 | pytestmark = pytest.mark.asyncio 8 | 9 | 10 | async def test_seek(tmpdir): 11 | """ 12 | Test `seek()` method. 13 | """ 14 | myfile = tmpdir / "myfile" 15 | 16 | with open(myfile, "w", encoding="utf-8") as fdata: 17 | fdata.write("kirkdata") 18 | 19 | async with AsyncFile(myfile, 'r') as fdata: 20 | await fdata.seek(4) 21 | assert await fdata.read() == "data" 22 | 23 | 24 | async def test_tell(tmpdir): 25 | """ 26 | Test `tell()` method. 27 | """ 28 | myfile = tmpdir / "myfile" 29 | 30 | with open(myfile, "w", encoding="utf-8") as fdata: 31 | fdata.write("kirkdata") 32 | 33 | async with AsyncFile(myfile, 'r') as fdata: 34 | await fdata.seek(4) 35 | assert await fdata.tell() == 4 36 | 37 | 38 | async def test_read(tmpdir): 39 | """ 40 | Test `read()` method. 41 | """ 42 | myfile = tmpdir / "myfile" 43 | 44 | with open(myfile, "w", encoding="utf-8") as fdata: 45 | fdata.write("kirkdata") 46 | 47 | async with AsyncFile(myfile, 'r') as fdata: 48 | assert await fdata.read() == "kirkdata" 49 | 50 | 51 | async def test_write(tmpdir): 52 | """ 53 | Test `write()` method. 54 | """ 55 | myfile = tmpdir / "myfile" 56 | 57 | async with AsyncFile(myfile, "w") as fdata: 58 | await fdata.write("kirkdata") 59 | 60 | with open(myfile, "r", encoding="utf-8") as fdata: 61 | assert fdata.read() == "kirkdata" 62 | 63 | 64 | async def test_readline(tmpdir): 65 | """ 66 | Test `readline()` method. 67 | """ 68 | myfile = tmpdir / "myfile" 69 | 70 | with open(myfile, "w", encoding="utf-8") as fdata: 71 | fdata.write("kirkdata\n") 72 | fdata.write("kirkdata\n") 73 | 74 | async with AsyncFile(myfile, 'r') as fdata: 75 | assert await fdata.readline() == "kirkdata\n" 76 | assert await fdata.readline() == "kirkdata\n" 77 | assert await fdata.readline() == '' 78 | 79 | 80 | async def test_file_no_open(tmpdir): 81 | """ 82 | Test a file when it's not open. 83 | """ 84 | myfile = tmpdir / "myfile" 85 | 86 | with open(myfile, "w", encoding="utf-8") as fdata: 87 | fdata.write("kirkdata") 88 | 89 | fdata = AsyncFile(myfile, 'r') 90 | await fdata.seek(4) 91 | assert not await fdata.tell() 92 | assert not await fdata.read() 93 | assert not await fdata.readline() 94 | await fdata.write("faaaa") 95 | await fdata.close() 96 | await fdata.close() 97 | await fdata.close() 98 | 99 | 100 | async def test_open(tmpdir): 101 | """ 102 | Test `open()` method. 103 | """ 104 | myfile = tmpdir / "myfile" 105 | fdata = AsyncFile(myfile, 'w') 106 | await fdata.open() 107 | await fdata.open() 108 | try: 109 | await fdata.write("ciao") 110 | finally: 111 | await fdata.close() 112 | await fdata.close() 113 | 114 | 115 | async def test_mutliple_open_close(tmpdir): 116 | """ 117 | Test `open()` and `close()` methods when open/close multiple times. 118 | """ 119 | myfile = tmpdir / "myfile" 120 | fdata = AsyncFile(myfile, 'w') 121 | await fdata.open() 122 | await fdata.open() 123 | await fdata.open() 124 | await fdata.close() 125 | await fdata.close() 126 | -------------------------------------------------------------------------------- /libkirk/tests/test_ltp.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test Framework implementations. 3 | """ 4 | import os 5 | import json 6 | import pytest 7 | from libkirk.data import Test 8 | from libkirk.ltp import LTPFramework 9 | from libkirk.host import HostSUT 10 | 11 | pytestmark = pytest.mark.asyncio 12 | 13 | 14 | class TestLTPFramework: 15 | """ 16 | Inherit this class to implement framework tests. 17 | """ 18 | TESTS_NUM = 6 19 | SUITES_NUM = 3 20 | 21 | @pytest.fixture 22 | async def sut(self): 23 | """ 24 | Host SUT communication object. 25 | """ 26 | obj = HostSUT() 27 | obj.setup() 28 | 29 | await obj.communicate() 30 | yield obj 31 | await obj.stop() 32 | 33 | @pytest.fixture 34 | def framework(self, tmpdir): 35 | """ 36 | LTP framework object. 37 | """ 38 | fw = LTPFramework() 39 | fw.setup(root=str(tmpdir)) 40 | 41 | yield fw 42 | 43 | @pytest.fixture(autouse=True) 44 | def prepare_tmpdir(self, tmpdir): 45 | """ 46 | Prepare the temporary directory adding runtest folder. 47 | """ 48 | # create simple testing suites 49 | content = "" 50 | for i in range(self.TESTS_NUM): 51 | content += f"test0{i} echo ciao\n" 52 | 53 | testcases = tmpdir.mkdir("testcases").mkdir("bin") 54 | runtest = tmpdir.mkdir("runtest") 55 | 56 | for i in range(self.SUITES_NUM): 57 | suite = runtest / f"suite{i}" 58 | suite.write(content) 59 | 60 | # create a suite that is executing slower than the others 61 | # and it's parallelizable 62 | content = "" 63 | for i in range(self.TESTS_NUM, self.TESTS_NUM * 2): 64 | content += f"slow_test0{i} sleep 0.05\n" 65 | 66 | suite = runtest / f"slow_suite" 67 | suite.write(content) 68 | 69 | tests = {} 70 | for i in range(self.TESTS_NUM, self.TESTS_NUM * 2): 71 | name = f"slow_test0{i}" 72 | tests[name] = {"max_runtime": "10"} 73 | 74 | metadata_d = {"tests": tests} 75 | metadata = tmpdir.mkdir("metadata") / "ltp.json" 76 | metadata.write(json.dumps(metadata_d)) 77 | 78 | # create shell test 79 | test_sh = testcases / "test.sh" 80 | test_sh.write("#!/bin/bash\necho $1 $2\n") 81 | 82 | async def test_name(self, framework): 83 | """ 84 | Test that name property is not empty. 85 | """ 86 | assert framework.name == "ltp" 87 | 88 | async def test_get_suites(self, framework, sut, tmpdir): 89 | """ 90 | Test get_suites method. 91 | """ 92 | suites = await framework.get_suites(sut) 93 | assert "suite0" in suites 94 | assert "suite1" in suites 95 | assert "suite2" in suites 96 | assert "slow_suite" in suites 97 | 98 | async def test_find_command(self, framework, sut, tmpdir): 99 | """ 100 | Test find_command method. 101 | """ 102 | test = await framework.find_command(sut, "test.sh ciao bepi") 103 | assert test.name == "test.sh" 104 | assert test.command == "test.sh" 105 | assert test.arguments == ["ciao", "bepi"] 106 | assert not test.parallelizable 107 | assert test.cwd == tmpdir / "testcases" / "bin" 108 | assert test.env 109 | 110 | async def test_find_suite(self, framework, sut, tmpdir): 111 | """ 112 | Test find_suite method. 113 | """ 114 | for i in range(self.SUITES_NUM): 115 | suite = await framework.find_suite(sut, f"suite{i}") 116 | assert len(suite.tests) == self.TESTS_NUM 117 | 118 | for j in range(self.TESTS_NUM): 119 | test = suite.tests[j] 120 | assert test.name == f"test0{j}" 121 | assert test.command == "echo" 122 | assert test.arguments == ["ciao"] 123 | assert test.cwd == os.path.join( 124 | str(tmpdir), 125 | "testcases", 126 | "bin") 127 | assert not test.parallelizable 128 | assert "LTPROOT" in test.env 129 | assert "TMPDIR" in test.env 130 | assert "LTP_COLORIZE_OUTPUT" in test.env 131 | 132 | suite = await framework.find_suite(sut, "slow_suite") 133 | assert len(suite.tests) == self.TESTS_NUM 134 | 135 | for test in suite.tests: 136 | assert test.command == "sleep" 137 | assert test.arguments == ["0.05"] 138 | assert test.cwd == os.path.join( 139 | str(tmpdir), 140 | "testcases", 141 | "bin") 142 | assert not test.parallelizable 143 | assert "LTPROOT" in test.env 144 | assert "TMPDIR" in test.env 145 | assert "LTP_COLORIZE_OUTPUT" in test.env 146 | 147 | async def test_find_suite_max_runtime(self, sut, tmpdir): 148 | """ 149 | Test find_suite method when max_runtime is defined. 150 | """ 151 | framework = LTPFramework() 152 | framework.setup(root=str(tmpdir), max_runtime=5) 153 | 154 | suite = await framework.find_suite(sut, "slow_suite") 155 | assert len(suite.tests) == 0 156 | 157 | async def test_read_result_passed(self, framework): 158 | """ 159 | Test read_result method when test passes. 160 | """ 161 | test = Test(name="test", cmd="echo", args="ciao") 162 | result = await framework.read_result(test, 'ciao\n', 0, 0.1) 163 | assert result.passed == 1 164 | assert result.failed == 0 165 | assert result.broken == 0 166 | assert result.skipped == 0 167 | assert result.warnings == 0 168 | assert result.exec_time == 0.1 169 | assert result.test == test 170 | assert result.return_code == 0 171 | assert result.stdout == "ciao\n" 172 | 173 | async def test_read_result_failure(self, framework): 174 | """ 175 | Test read_result method when test fails. 176 | """ 177 | test = Test(name="test", cmd="echo") 178 | result = await framework.read_result(test, '', 1, 0.1) 179 | assert result.passed == 0 180 | assert result.failed == 1 181 | assert result.broken == 0 182 | assert result.skipped == 0 183 | assert result.warnings == 0 184 | assert result.exec_time == 0.1 185 | assert result.test == test 186 | assert result.return_code == 1 187 | assert result.stdout == "" 188 | 189 | async def test_read_result_broken(self, framework): 190 | """ 191 | Test read_result method when test is broken. 192 | """ 193 | test = Test(name="test", cmd="echo") 194 | result = await framework.read_result(test, '', -1, 0.1) 195 | assert result.passed == 0 196 | assert result.failed == 0 197 | assert result.broken == 1 198 | assert result.skipped == 0 199 | assert result.warnings == 0 200 | assert result.exec_time == 0.1 201 | assert result.test == test 202 | assert result.return_code == -1 203 | assert result.stdout == "" 204 | 205 | async def test_read_result_skipped(self, framework): 206 | """ 207 | Test read_result method when test has skip. 208 | """ 209 | test = Test(name="test", cmd="echo") 210 | result = await framework.read_result( 211 | test, "mydata", 32, 0.1) 212 | assert result.passed == 0 213 | assert result.failed == 0 214 | assert result.broken == 0 215 | assert result.skipped == 1 216 | assert result.warnings == 0 217 | assert result.exec_time == 0.1 218 | assert result.test == test 219 | assert result.return_code == 32 220 | assert result.stdout == "mydata" 221 | -------------------------------------------------------------------------------- /libkirk/tests/test_ltx.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unittests for ltx module. 3 | """ 4 | import os 5 | import time 6 | import signal 7 | import asyncio.subprocess 8 | import pytest 9 | from libkirk.ltx import LTX 10 | from libkirk.ltx import Requests 11 | from libkirk.ltx_sut import LTXSUT 12 | from libkirk.tests.test_sut import _TestSUT 13 | from libkirk.tests.test_session import _TestSession 14 | 15 | pytestmark = [pytest.mark.asyncio, pytest.mark.ltx] 16 | 17 | TEST_LTX_BINARY = os.environ.get("TEST_LTX_BINARY", None) 18 | 19 | if not TEST_LTX_BINARY or not os.path.isfile(TEST_LTX_BINARY): 20 | pytestmark.append(pytest.mark.skip( 21 | reason="TEST_LTX_BINARY doesn't exist")) 22 | 23 | 24 | class TestLTX: 25 | """ 26 | Test LTX implementation. 27 | """ 28 | 29 | @pytest.fixture 30 | async def ltx(self, tmpdir): 31 | """ 32 | LTX handler. 33 | """ 34 | stdin_path = str(tmpdir / 'transport.in') 35 | stdout_path = str(tmpdir / 'transport.out') 36 | 37 | os.mkfifo(stdin_path) 38 | os.mkfifo(stdout_path) 39 | 40 | stdin = os.open(stdin_path, os.O_RDWR | os.O_NONBLOCK) 41 | stdout = os.open(stdout_path, os.O_RDWR) 42 | 43 | proc = await asyncio.subprocess.create_subprocess_shell( 44 | TEST_LTX_BINARY, 45 | stdin=stdin, 46 | stdout=stdout) 47 | 48 | try: 49 | async with LTX(stdin, stdout) as handle: 50 | yield handle 51 | finally: 52 | proc.kill() 53 | 54 | async def test_version(self, ltx): 55 | """ 56 | Test version request. 57 | """ 58 | req = Requests.version() 59 | replies = await ltx.gather([req]) 60 | assert replies[req][0] == "0.1" 61 | 62 | async def test_ping(self, ltx): 63 | """ 64 | Test ping request. 65 | """ 66 | start_t = time.monotonic() 67 | req = Requests.ping() 68 | replies = await ltx.gather([req]) 69 | assert start_t < replies[req][0] * 1e-9 < time.monotonic() 70 | 71 | async def test_execute(self, ltx): 72 | """ 73 | Test execute request. 74 | """ 75 | stdout = [] 76 | 77 | async def _stdout_coro(data): 78 | stdout.append(data) 79 | 80 | start_t = time.monotonic() 81 | req = Requests.execute(0, "uname", stdout_coro=_stdout_coro) 82 | replies = await ltx.gather([req]) 83 | reply = replies[req] 84 | 85 | assert ''.join(stdout) == "Linux\n" 86 | assert start_t < reply[0] * 1e-9 < time.monotonic() 87 | assert reply[1] == 1 88 | assert reply[2] == 0 89 | assert reply[3] == "Linux\n" 90 | 91 | async def test_execute_builtin(self, ltx): 92 | """ 93 | Test execute request with builtin command. 94 | """ 95 | stdout = [] 96 | 97 | async def _stdout_coro(data): 98 | stdout.append(data) 99 | 100 | start_t = time.monotonic() 101 | req = Requests.execute( 102 | 0, "echo -n ciao", stdout_coro=_stdout_coro) 103 | replies = await ltx.gather([req]) 104 | reply = replies[req] 105 | 106 | assert ''.join(stdout) == "ciao" 107 | assert start_t < reply[0] * 1e-9 < time.monotonic() 108 | assert reply[1] == 1 109 | assert reply[2] == 0 110 | assert reply[3] == "ciao" 111 | 112 | async def test_execute_multiple(self, ltx): 113 | """ 114 | Test multiple execute request in a row. 115 | """ 116 | times = 2 117 | stdout = [] 118 | 119 | async def _stdout_coro(data): 120 | stdout.append(data) 121 | 122 | req = [] 123 | for slot in range(times): 124 | req.append(Requests.execute( 125 | slot, 126 | "echo -n ciao", 127 | stdout_coro=_stdout_coro)) 128 | 129 | start_t = time.monotonic() 130 | replies = await ltx.gather(req) 131 | end_t = time.monotonic() 132 | 133 | for reply in replies.values(): 134 | assert start_t < reply[0] * 1e-9 < end_t 135 | assert reply[1] == 1 136 | assert reply[2] == 0 137 | assert reply[3] == "ciao" 138 | 139 | for data in stdout: 140 | assert data == "ciao" 141 | 142 | async def test_set_file(self, ltx, tmp_path): 143 | """ 144 | Test set_file request. 145 | """ 146 | data = b'AaXa\x00\x01\x02Zz' * 1024 147 | pfile = tmp_path / 'file.bin' 148 | 149 | req = Requests.set_file(str(pfile), data) 150 | await ltx.gather([req]) 151 | 152 | assert pfile.read_bytes() == data 153 | 154 | async def test_get_file(self, ltx, tmp_path): 155 | """ 156 | Test get_file request. 157 | """ 158 | pfile = tmp_path / 'file.bin' 159 | pfile.write_bytes(b'AaXa\x00\x01\x02Zz' * 1024) 160 | 161 | req = Requests.get_file(str(pfile)) 162 | replies = await ltx.gather([req]) 163 | 164 | assert replies[req][0] == str(pfile) 165 | assert pfile.read_bytes() == replies[req][1] 166 | 167 | async def test_kill(self, ltx): 168 | """ 169 | Test kill method. 170 | """ 171 | start_t = time.monotonic() 172 | exec_req = Requests.execute(0, "sleep 1") 173 | kill_req = Requests.kill(0) 174 | replies = await ltx.gather([exec_req, kill_req]) 175 | reply = replies[exec_req] 176 | 177 | assert start_t < reply[0] * 1e-9 < time.monotonic() 178 | assert reply[1] == 2 179 | assert reply[2] == signal.SIGKILL 180 | assert reply[3] == "" 181 | 182 | async def test_env(self, ltx): 183 | """ 184 | Test env request. 185 | """ 186 | start_t = time.monotonic() 187 | env_req = Requests.env(0, "HELLO", "CIAO") 188 | exec_req = Requests.execute(0, "echo -n $HELLO") 189 | replies = await ltx.gather([env_req, exec_req]) 190 | reply = replies[exec_req] 191 | 192 | assert start_t < reply[0] * 1e-9 < time.monotonic() 193 | assert reply[1] == 1 194 | assert reply[2] == 0 195 | assert reply[3] == "CIAO" 196 | 197 | async def test_env_multiple(self, ltx): 198 | """ 199 | Test env request. 200 | """ 201 | start_t = time.monotonic() 202 | env_req = Requests.env(128, "HELLO", "CIAO") 203 | exec_req = Requests.execute(0, "echo -n $HELLO") 204 | replies = await ltx.gather([env_req, exec_req]) 205 | reply = replies[exec_req] 206 | 207 | assert start_t < reply[0] * 1e-9 < time.monotonic() 208 | assert reply[1] == 1 209 | assert reply[2] == 0 210 | assert reply[3] == "CIAO" 211 | 212 | async def test_cwd(self, ltx, tmpdir): 213 | """ 214 | Test cwd request. 215 | """ 216 | path = str(tmpdir) 217 | 218 | start_t = time.monotonic() 219 | env_req = Requests.cwd(0, path) 220 | exec_req = Requests.execute(0, "echo -n $PWD") 221 | replies = await ltx.gather([env_req, exec_req]) 222 | reply = replies[exec_req] 223 | 224 | assert start_t < reply[0] * 1e-9 < time.monotonic() 225 | assert reply[1] == 1 226 | assert reply[2] == 0 227 | assert reply[3] == path 228 | 229 | async def test_cwd_multiple(self, ltx, tmpdir): 230 | """ 231 | Test cwd request on multiple slots. 232 | """ 233 | path = str(tmpdir) 234 | 235 | start_t = time.monotonic() 236 | env_req = Requests.cwd(128, path) 237 | exec_req = Requests.execute(0, "echo -n $PWD") 238 | replies = await ltx.gather([env_req, exec_req]) 239 | reply = replies[exec_req] 240 | 241 | assert start_t < reply[0] * 1e-9 < time.monotonic() 242 | assert reply[1] == 1 243 | assert reply[2] == 0 244 | assert reply[3] == path 245 | 246 | async def test_all_together(self, ltx, tmp_path): 247 | """ 248 | Test all requests together. 249 | """ 250 | data = b'AaXa\x00\x01\x02Zz' * 1024 251 | pfile = tmp_path / 'file.bin' 252 | 253 | requests = [] 254 | requests.append(Requests.version()) 255 | requests.append(Requests.set_file(str(pfile), data)) 256 | requests.append(Requests.ping()) 257 | requests.append(Requests.env(0, "HELLO", "CIAO")) 258 | requests.append(Requests.execute(0, "sleep 5")) 259 | requests.append(Requests.kill(0)) 260 | requests.append(Requests.get_file(str(pfile))) 261 | 262 | await ltx.gather(requests) 263 | 264 | 265 | @pytest.fixture 266 | async def sut(tmpdir): 267 | """ 268 | LTXSUT instance object. 269 | """ 270 | stdin_path = str(tmpdir / 'transport.in') 271 | stdout_path = str(tmpdir / 'transport.out') 272 | 273 | os.mkfifo(stdin_path) 274 | os.mkfifo(stdout_path) 275 | 276 | stdin = os.open(stdin_path, os.O_RDONLY | os.O_NONBLOCK) 277 | stdout = os.open(stdout_path, os.O_RDWR) 278 | 279 | proc = await asyncio.subprocess.create_subprocess_shell( 280 | TEST_LTX_BINARY, 281 | stdin=stdin, 282 | stdout=stdout) 283 | 284 | sut = LTXSUT() 285 | sut.setup( 286 | cwd=str(tmpdir), 287 | env=dict(HELLO="WORLD"), 288 | stdin=stdin_path, 289 | stdout=stdout_path) 290 | 291 | yield sut 292 | 293 | if await sut.is_running: 294 | await sut.stop() 295 | 296 | proc.kill() 297 | 298 | 299 | class TestLTXSUT(_TestSUT): 300 | """ 301 | Test HostSUT implementation. 302 | """ 303 | 304 | async def test_fetch_file_stop(self): 305 | pytest.skip(reason="LTX doesn't support stop for GET_FILE") 306 | 307 | 308 | class TestLTXSession(_TestSession): 309 | """ 310 | Test Session implementation using LTX SUT. 311 | """ 312 | -------------------------------------------------------------------------------- /libkirk/tests/test_main.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unittests for main module. 3 | """ 4 | import os 5 | import sys 6 | import pwd 7 | import json 8 | import pytest 9 | import libkirk.main 10 | 11 | 12 | class TestMain: 13 | """ 14 | The the main module entry point. 15 | """ 16 | 17 | @pytest.fixture(autouse=True) 18 | def setup(self, dummy_framework): 19 | """ 20 | Setup main before running tests. 21 | """ 22 | libkirk.main.LOADED_FRAMEWORK.append(dummy_framework) 23 | 24 | def read_report(self, temp) -> dict: 25 | """ 26 | Check if report file contains the given number of tests. 27 | """ 28 | name = pwd.getpwuid(os.getuid()).pw_name 29 | report = str(temp / f"kirk.{name}" / "latest" / "results.json") 30 | assert os.path.isfile(report) 31 | 32 | # read report and check if all suite's tests have been executed 33 | report_d = None 34 | with open(report, 'r', encoding="utf-8") as report_f: 35 | report_d = json.loads(report_f.read()) 36 | 37 | return report_d 38 | 39 | def test_wrong_options(self): 40 | """ 41 | Test wrong options. 42 | """ 43 | cmd_args = [ 44 | "--run-command1234", "ls" 45 | ] 46 | 47 | with pytest.raises(SystemExit) as excinfo: 48 | libkirk.main.run(cmd_args=cmd_args) 49 | 50 | assert excinfo.value.code == 2 51 | 52 | def test_run_command(self, tmpdir): 53 | """ 54 | Test --run-command option. 55 | """ 56 | temp = tmpdir.mkdir("temp") 57 | cmd_args = [ 58 | "--tmp-dir", str(temp), 59 | "--run-command", "ls" 60 | ] 61 | 62 | with pytest.raises(SystemExit) as excinfo: 63 | libkirk.main.run(cmd_args=cmd_args) 64 | 65 | assert excinfo.value.code == libkirk.main.RC_OK 66 | 67 | @pytest.mark.skipif(sys.version_info < (3, 8), reason="requires python3.8+") 68 | def test_run_command_timeout(self, tmpdir): 69 | """ 70 | Test --run-command option with timeout. 71 | """ 72 | temp = tmpdir.mkdir("temp") 73 | cmd_args = [ 74 | "--tmp-dir", str(temp), 75 | "--run-command", "ls", 76 | "--exec-timeout", "0" 77 | ] 78 | 79 | with pytest.raises(SystemExit) as excinfo: 80 | libkirk.main.run(cmd_args=cmd_args) 81 | 82 | assert excinfo.value.code == libkirk.main.RC_ERROR 83 | 84 | def test_run_suite(self, tmpdir): 85 | """ 86 | Test --run-suite option. 87 | """ 88 | temp = tmpdir.mkdir("temp") 89 | cmd_args = [ 90 | "--tmp-dir", str(temp), 91 | "--framework", "dummy", 92 | "--run-suite", "suite01" 93 | ] 94 | 95 | with pytest.raises(SystemExit) as excinfo: 96 | libkirk.main.run(cmd_args=cmd_args) 97 | 98 | assert excinfo.value.code == libkirk.main.RC_OK 99 | 100 | report = self.read_report(temp) 101 | assert len(report["results"]) == 2 102 | 103 | def test_run_suite_timeout(self, tmpdir): 104 | """ 105 | Test --run-suite option with timeout. 106 | """ 107 | temp = tmpdir.mkdir("temp") 108 | cmd_args = [ 109 | "--tmp-dir", str(temp), 110 | "--framework", "dummy", 111 | "--run-suite", "suite01", 112 | "--suite-timeout", "0" 113 | ] 114 | 115 | with pytest.raises(SystemExit) as excinfo: 116 | libkirk.main.run(cmd_args=cmd_args) 117 | 118 | assert excinfo.value.code == libkirk.main.RC_OK 119 | 120 | report = self.read_report(temp) 121 | assert len(report["results"]) == 2 122 | 123 | for param in report["results"]: 124 | assert param["test"]["passed"] == 0 125 | assert param["test"]["failed"] == 0 126 | assert param["test"]["broken"] == 0 127 | assert param["test"]["warnings"] == 0 128 | assert param["test"]["skipped"] == 1 129 | 130 | def test_run_suite_verbose(self, tmpdir, capsys): 131 | """ 132 | Test --run-suite option with --verbose. 133 | """ 134 | temp = tmpdir.mkdir("temp") 135 | cmd_args = [ 136 | "--tmp-dir", str(temp), 137 | "--framework", "dummy", 138 | "--run-suite", "suite01", 139 | "--verbose", 140 | ] 141 | 142 | with pytest.raises(SystemExit) as excinfo: 143 | libkirk.main.run(cmd_args=cmd_args) 144 | 145 | assert excinfo.value.code == libkirk.main.RC_OK 146 | 147 | captured = capsys.readouterr() 148 | assert "ciao0\n" in captured.out 149 | 150 | @pytest.mark.xfail(reason="This test passes if run alone. capsys bug?") 151 | def test_run_suite_no_colors(self, tmpdir, capsys): 152 | """ 153 | Test --run-suite option with --no-colors. 154 | """ 155 | temp = tmpdir.mkdir("temp") 156 | cmd_args = [ 157 | "--tmp-dir", str(temp), 158 | "--framework", "dummy", 159 | "--run-suite", "suite01", 160 | "--no-colors", 161 | ] 162 | 163 | with pytest.raises(SystemExit) as excinfo: 164 | libkirk.main.run(cmd_args=cmd_args) 165 | 166 | assert excinfo.value.code == libkirk.main.RC_OK 167 | 168 | out, _ = capsys.readouterr() 169 | assert "test00: pass" in out 170 | 171 | def test_restore_suite(self, tmpdir): 172 | """ 173 | Test --restore option. 174 | """ 175 | temp = tmpdir.mkdir("temp") 176 | 177 | # run a normal session 178 | cmd_args = [ 179 | "--tmp-dir", str(temp), 180 | "--framework", "dummy", 181 | "--run-suite", "suite01" 182 | ] 183 | 184 | with pytest.raises(SystemExit) as excinfo: 185 | libkirk.main.run(cmd_args=cmd_args) 186 | 187 | assert excinfo.value.code == libkirk.main.RC_OK 188 | 189 | report = self.read_report(temp) 190 | assert len(report["results"]) == 2 191 | 192 | # restore session 193 | name = pwd.getpwuid(os.getuid()).pw_name 194 | cmd_args = [ 195 | "--tmp-dir", str(temp), 196 | "--restore", f"{str(temp)}/kirk.{name}/latest", 197 | "--framework", "dummy", 198 | "--run-suite", "suite01", "environ" 199 | ] 200 | 201 | with pytest.raises(SystemExit) as excinfo: 202 | libkirk.main.run(cmd_args=cmd_args) 203 | 204 | assert excinfo.value.code == libkirk.main.RC_OK 205 | 206 | report = self.read_report(temp) 207 | assert len(report["results"]) == 1 208 | 209 | def test_json_report(self, tmpdir): 210 | """ 211 | Test --json-report option. 212 | """ 213 | temp = tmpdir.mkdir("temp") 214 | report = str(tmpdir / "report.json") 215 | cmd_args = [ 216 | "--tmp-dir", str(temp), 217 | "--framework", "dummy", 218 | "--run-suite", "suite01", 219 | "--json-report", report 220 | ] 221 | 222 | with pytest.raises(SystemExit) as excinfo: 223 | libkirk.main.run(cmd_args=cmd_args) 224 | 225 | assert excinfo.value.code == libkirk.main.RC_OK 226 | assert os.path.isfile(report) 227 | 228 | report_a = self.read_report(temp) 229 | assert len(report_a["results"]) == 2 230 | 231 | report_b = None 232 | with open(report, 'r', encoding="utf-8") as report_f: 233 | report_b = json.loads(report_f.read()) 234 | 235 | assert report_a == report_b 236 | 237 | def test_skip_tests(self, tmpdir): 238 | """ 239 | Test --skip-tests option. 240 | """ 241 | temp = tmpdir.mkdir("temp") 242 | cmd_args = [ 243 | "--tmp-dir", str(temp), 244 | "--framework", "dummy", 245 | "--run-suite", "suite01", 246 | "--skip-tests", "test0[23]" 247 | ] 248 | 249 | with pytest.raises(SystemExit) as excinfo: 250 | libkirk.main.run(cmd_args=cmd_args) 251 | 252 | assert excinfo.value.code == libkirk.main.RC_OK 253 | 254 | report = self.read_report(temp) 255 | assert len(report["results"]) == 1 256 | 257 | def test_skip_file(self, tmpdir): 258 | """ 259 | Test --skip-file option. 260 | """ 261 | skipfile = tmpdir / "skipfile" 262 | skipfile.write("test02\ntest03") 263 | 264 | temp = tmpdir.mkdir("temp") 265 | cmd_args = [ 266 | "--tmp-dir", str(temp), 267 | "--framework", "dummy", 268 | "--run-suite", "suite01", 269 | "--skip-file", str(skipfile) 270 | ] 271 | 272 | with pytest.raises(SystemExit) as excinfo: 273 | libkirk.main.run(cmd_args=cmd_args) 274 | 275 | assert excinfo.value.code == libkirk.main.RC_OK 276 | 277 | report = self.read_report(temp) 278 | assert len(report["results"]) == 1 279 | 280 | def test_skip_tests_and_file(self, tmpdir): 281 | """ 282 | Test --skip-file option with --skip-tests. 283 | """ 284 | skipfile = tmpdir / "skipfile" 285 | skipfile.write("test03") 286 | 287 | temp = tmpdir.mkdir("temp") 288 | cmd_args = [ 289 | "--tmp-dir", str(temp), 290 | "--framework", "dummy", 291 | "--run-suite", "suite01", 292 | "--skip-tests", "test01", 293 | "--skip-file", str(skipfile) 294 | ] 295 | 296 | with pytest.raises(SystemExit) as excinfo: 297 | libkirk.main.run(cmd_args=cmd_args) 298 | 299 | assert excinfo.value.code == libkirk.main.RC_OK 300 | 301 | report = self.read_report(temp) 302 | assert len(report["results"]) == 1 303 | 304 | def test_workers(self, tmpdir): 305 | """ 306 | Test --workers option. 307 | """ 308 | temp = tmpdir.mkdir("temp") 309 | 310 | # run on multiple workers 311 | cmd_args = [ 312 | "--tmp-dir", str(temp), 313 | "--framework", "dummy", 314 | "--run-suite", "suite01", 315 | "--workers", str(os.cpu_count()), 316 | ] 317 | 318 | with pytest.raises(SystemExit) as excinfo: 319 | libkirk.main.run(cmd_args=cmd_args) 320 | 321 | assert excinfo.value.code == libkirk.main.RC_OK 322 | 323 | report = self.read_report(temp) 324 | assert len(report["results"]) == 2 325 | 326 | def test_sut_help(self): 327 | """ 328 | Test "--sut help" command and check if SUT class(es) are loaded. 329 | """ 330 | cmd_args = [ 331 | "--sut", "help" 332 | ] 333 | 334 | with pytest.raises(SystemExit) as excinfo: 335 | libkirk.main.run(cmd_args=cmd_args) 336 | 337 | assert excinfo.value.code == libkirk.main.RC_OK 338 | assert len(libkirk.main.LOADED_SUT) > 0 339 | 340 | def test_framework_help(self): 341 | """ 342 | Test "--framework help" command and check if Framework class(es) 343 | are loaded. 344 | """ 345 | cmd_args = [ 346 | "--framework", "help" 347 | ] 348 | 349 | with pytest.raises(SystemExit) as excinfo: 350 | libkirk.main.run(cmd_args=cmd_args) 351 | 352 | assert excinfo.value.code == libkirk.main.RC_OK 353 | assert len(libkirk.main.LOADED_FRAMEWORK) > 0 354 | 355 | def test_env(self, tmpdir): 356 | """ 357 | Test --env option. 358 | """ 359 | temp = tmpdir.mkdir("temp") 360 | cmd_args = [ 361 | "--tmp-dir", str(temp), 362 | "--framework", "dummy", 363 | "--run-suite", "environ", 364 | "--env", "hello=ciao" 365 | ] 366 | 367 | with pytest.raises(SystemExit) as excinfo: 368 | libkirk.main.run(cmd_args=cmd_args) 369 | 370 | assert excinfo.value.code == libkirk.main.RC_OK 371 | 372 | report = self.read_report(temp) 373 | assert len(report["results"]) == 1 374 | assert report["results"][0]["test"]["log"] == "ciao" 375 | 376 | def test_suite_iterate(self, tmpdir): 377 | """ 378 | Test --suite-iterate option. 379 | """ 380 | temp = tmpdir.mkdir("temp") 381 | cmd_args = [ 382 | "--tmp-dir", str(temp), 383 | "--framework", "dummy", 384 | "--run-suite", "suite01", 385 | "--suite-iterate", "4", 386 | ] 387 | 388 | with pytest.raises(SystemExit) as excinfo: 389 | libkirk.main.run(cmd_args=cmd_args) 390 | 391 | assert excinfo.value.code == libkirk.main.RC_OK 392 | 393 | report = self.read_report(temp) 394 | assert len(report["results"]) == 8 395 | 396 | def test_randomize(self, tmpdir): 397 | """ 398 | Test --randomize option. 399 | """ 400 | num_of_suites = 10 401 | 402 | temp = tmpdir.mkdir("temp") 403 | cmd_args = [ 404 | "--tmp-dir", str(temp), 405 | "--framework", "dummy", 406 | "--run-suite", 407 | ] 408 | cmd_args.extend(["suite01"] * num_of_suites) 409 | cmd_args.append("--randomize") 410 | 411 | with pytest.raises(SystemExit) as excinfo: 412 | libkirk.main.run(cmd_args=cmd_args) 413 | 414 | assert excinfo.value.code == libkirk.main.RC_OK 415 | 416 | report = self.read_report(temp) 417 | assert len(report["results"]) == 2 * num_of_suites 418 | 419 | tests_names = [] 420 | for test in report["results"]: 421 | tests_names.append(test["test_fqn"]) 422 | 423 | assert ["test01", "test02"] * num_of_suites != tests_names 424 | 425 | def test_runtime(self, tmpdir): 426 | """ 427 | Test --runtime option. 428 | """ 429 | temp = tmpdir.mkdir("temp") 430 | cmd_args = [ 431 | "--tmp-dir", str(temp), 432 | "--framework", "dummy", 433 | "--run-suite", "suite01", 434 | "--runtime", "1", 435 | ] 436 | 437 | with pytest.raises(SystemExit) as excinfo: 438 | libkirk.main.run(cmd_args=cmd_args) 439 | 440 | assert excinfo.value.code == libkirk.main.RC_OK 441 | 442 | report = self.read_report(temp) 443 | assert len(report["results"]) >= 2 444 | -------------------------------------------------------------------------------- /libkirk/tests/test_monitor.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unittest for monitor module. 3 | """ 4 | import json 5 | import asyncio 6 | import pytest 7 | import libkirk 8 | from libkirk.io import AsyncFile 9 | from libkirk.monitor import JSONFileMonitor 10 | 11 | pytestmark = pytest.mark.asyncio 12 | 13 | MONITOR_FILE = "monitor.json" 14 | 15 | 16 | @pytest.fixture(autouse=True) 17 | async def monitor(tmpdir): 18 | """ 19 | Fixture containing json file monitor. 20 | """ 21 | fpath = tmpdir / MONITOR_FILE 22 | 23 | # fill the file with garbage data before writing 24 | with open(fpath, 'w', encoding="utf-8") as data: 25 | data.write("garbage") 26 | 27 | obj = JSONFileMonitor(fpath) 28 | await obj.start() 29 | yield 30 | await obj.stop() 31 | 32 | 33 | @pytest.fixture(autouse=True) 34 | async def run_events(): 35 | """ 36 | Run kirk events at the beginning of the test 37 | and stop it at the end of the test. 38 | """ 39 | async def start(): 40 | await libkirk.events.start() 41 | 42 | libkirk.create_task(start()) 43 | yield 44 | await libkirk.events.stop() 45 | 46 | 47 | @pytest.fixture 48 | async def read_monitor(tmpdir): 49 | """ 50 | Read a single line inside the monitor file. 51 | """ 52 | fpath = tmpdir / MONITOR_FILE 53 | 54 | async def _read(): 55 | async with AsyncFile(fpath, 'r') as fdata: 56 | data = None 57 | while not data: 58 | data = await fdata.readline() 59 | 60 | return data 61 | 62 | async def _wrap(position, msg): 63 | for _ in range(0, position - 1): 64 | await asyncio.wait_for(_read(), 1) 65 | 66 | data = await asyncio.wait_for(_read(), 1) 67 | assert data == msg 68 | 69 | return _wrap 70 | 71 | 72 | async def test_single_write(read_monitor): 73 | """ 74 | Test if a single event will cause to write inside the monitor file 75 | only once. 76 | """ 77 | msg = json.dumps({ 78 | 'type': "session_stopped", 79 | 'message': {} 80 | }) 81 | 82 | for _ in range(1, 10): 83 | await libkirk.events.fire("session_stopped") 84 | await read_monitor(1, msg) 85 | 86 | 87 | @pytest.mark.xfail(reason="This test passes if run alone") 88 | async def test_override_events(tmpdir, read_monitor): 89 | """ 90 | Test if we are correctly writing data inside monitor file. 91 | """ 92 | await libkirk.events.fire("session_started", str(tmpdir)) 93 | await libkirk.events.fire("kernel_panic") 94 | await libkirk.events.fire("session_stopped") 95 | 96 | msg = json.dumps({ 97 | 'type': "session_stopped", 98 | 'message': {} 99 | }) 100 | 101 | await read_monitor(3, msg) 102 | -------------------------------------------------------------------------------- /libkirk/tests/test_plugin.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unittests for framework module. 3 | """ 4 | import libkirk 5 | import libkirk.plugin 6 | from libkirk.sut import SUT 7 | from libkirk.framework import Framework 8 | 9 | 10 | def test_sut(tmpdir): 11 | """ 12 | Test if SUT implementations are correctly loaded. 13 | """ 14 | suts = [] 15 | suts.append(tmpdir / "sutA.py") 16 | suts.append(tmpdir / "sutB.py") 17 | suts.append(tmpdir / "sutC.txt") 18 | 19 | for index in range(0, len(suts)): 20 | suts[index].write( 21 | "from libkirk.sut import SUT\n\n" 22 | f"class SUT{index}(SUT):\n" 23 | " @property\n" 24 | " def name(self) -> str:\n" 25 | f" return 'mysut{index}'\n" 26 | ) 27 | 28 | suts = libkirk.plugin.discover(SUT, str(tmpdir)) 29 | 30 | assert len(suts) == 2 31 | 32 | 33 | def test_framework(tmpdir): 34 | """ 35 | Test if Framework implementations are correctly loaded. 36 | """ 37 | suts = [] 38 | suts.append(tmpdir / "frameworkA.py") 39 | suts.append(tmpdir / "frameworkB.py") 40 | suts.append(tmpdir / "frameworkC.txt") 41 | 42 | for index in range(0, len(suts)): 43 | suts[index].write( 44 | "from libkirk.framework import Framework\n\n" 45 | f"class Framework{index}(Framework):\n" 46 | " @property\n" 47 | " def name(self) -> str:\n" 48 | f" return 'fw{index}'\n" 49 | ) 50 | 51 | suts = libkirk.plugin.discover(Framework, str(tmpdir)) 52 | 53 | assert len(suts) == 2 54 | -------------------------------------------------------------------------------- /libkirk/tests/test_qemu.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test SUT implementations. 3 | """ 4 | import os 5 | import pytest 6 | from libkirk.qemu import QemuSUT 7 | from libkirk.sut import KernelPanicError 8 | from libkirk.tests.test_sut import _TestSUT 9 | from libkirk.tests.test_sut import Printer 10 | from libkirk.tests.test_session import _TestSession 11 | 12 | pytestmark = [pytest.mark.asyncio, pytest.mark.qemu] 13 | 14 | TEST_QEMU_IMAGE = os.environ.get("TEST_QEMU_IMAGE", None) 15 | TEST_QEMU_USERNAME = os.environ.get("TEST_QEMU_USERNAME", None) 16 | TEST_QEMU_PASSWORD = os.environ.get("TEST_QEMU_PASSWORD", None) 17 | TEST_QEMU_KERNEL = os.environ.get("TEST_QEMU_KERNEL", None) 18 | TEST_QEMU_BUSYBOX = os.environ.get("TEST_QEMU_BUSYBOX", None) 19 | 20 | if not TEST_QEMU_IMAGE: 21 | pytestmark.append(pytest.mark.skip( 22 | reason="TEST_QEMU_IMAGE not defined")) 23 | 24 | if not TEST_QEMU_USERNAME: 25 | pytestmark.append(pytest.mark.skip( 26 | reason="TEST_QEMU_USERNAME not defined")) 27 | 28 | if not TEST_QEMU_PASSWORD: 29 | pytestmark.append(pytest.mark.skip( 30 | reason="TEST_QEMU_PASSWORD not defined")) 31 | 32 | 33 | class _TestQemuSUT(_TestSUT): 34 | """ 35 | Test Qemu SUT implementation. 36 | """ 37 | 38 | async def test_kernel_panic(self, sut): 39 | """ 40 | Test kernel panic recognition. 41 | """ 42 | iobuff = Printer() 43 | 44 | await sut.communicate(iobuffer=iobuff) 45 | await sut.run_command( 46 | "echo 'Kernel panic\nThis is a generic message' > /tmp/panic.txt", 47 | iobuffer=iobuff) 48 | 49 | with pytest.raises(KernelPanicError): 50 | await sut.run_command( 51 | "cat /tmp/panic.txt", 52 | iobuffer=iobuff) 53 | 54 | async def test_fetch_file_stop(self): 55 | pytest.skip(reason="Coroutines don't support I/O file handling") 56 | 57 | 58 | @pytest.fixture 59 | async def sut_isa(tmpdir): 60 | """ 61 | Qemu instance using ISA. 62 | """ 63 | iobuff = Printer() 64 | 65 | runner = QemuSUT() 66 | runner.setup( 67 | tmpdir=str(tmpdir), 68 | image=TEST_QEMU_IMAGE, 69 | user=TEST_QEMU_USERNAME, 70 | password=TEST_QEMU_PASSWORD, 71 | serial="isa") 72 | 73 | yield runner 74 | 75 | if await runner.is_running: 76 | await runner.stop(iobuffer=iobuff) 77 | 78 | 79 | @pytest.fixture 80 | async def sut_virtio(tmpdir): 81 | """ 82 | Qemu instance using VirtIO. 83 | """ 84 | runner = QemuSUT() 85 | runner.setup( 86 | tmpdir=str(tmpdir), 87 | image=TEST_QEMU_IMAGE, 88 | user=TEST_QEMU_USERNAME, 89 | password=TEST_QEMU_PASSWORD, 90 | serial="virtio") 91 | 92 | yield runner 93 | 94 | if await runner.is_running: 95 | await runner.stop() 96 | 97 | 98 | class TestQemuSUTISA(_TestQemuSUT): 99 | """ 100 | Test QemuSUT implementation using ISA protocol. 101 | """ 102 | 103 | @pytest.fixture 104 | async def sut(self, sut_isa): 105 | yield sut_isa 106 | 107 | 108 | class TestQemuSUTVirtIO(_TestQemuSUT): 109 | """ 110 | Test QemuSUT implementation using VirtIO protocol. 111 | """ 112 | 113 | @pytest.fixture 114 | async def sut(self, sut_virtio): 115 | yield sut_virtio 116 | 117 | 118 | class TestSessionQemuISA(_TestSession): 119 | """ 120 | Test Session using Qemu with ISA protocol. 121 | """ 122 | 123 | @pytest.fixture 124 | async def sut(self, sut_isa): 125 | yield sut_isa 126 | 127 | 128 | class TestSessionQemuVirtIO(_TestSession): 129 | """ 130 | Test Session using Qemu with ISA protocol. 131 | """ 132 | 133 | @pytest.fixture 134 | async def sut(self, sut_virtio): 135 | yield sut_virtio 136 | 137 | 138 | @pytest.mark.skipif( 139 | not TEST_QEMU_KERNEL, 140 | reason="TEST_QEMU_KERNEL not defined") 141 | @pytest.mark.skipif( 142 | not TEST_QEMU_BUSYBOX, 143 | reason="TEST_QEMU_BUSYBOX not defined") 144 | class TestQemuSUTBusybox(_TestQemuSUT): 145 | """ 146 | Test QemuSUT implementation using kernel/initrd functionality with 147 | busybox initramfs image. 148 | """ 149 | 150 | @pytest.fixture 151 | async def sut(self, tmpdir): 152 | """ 153 | Qemu instance using kernel/initrd. 154 | """ 155 | runner = QemuSUT() 156 | runner.setup( 157 | tmpdir=str(tmpdir), 158 | kernel=TEST_QEMU_KERNEL, 159 | initrd=TEST_QEMU_BUSYBOX, 160 | prompt="/ #") 161 | 162 | yield runner 163 | 164 | if await runner.is_running: 165 | await runner.stop() 166 | -------------------------------------------------------------------------------- /libkirk/tests/test_session.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unittests for the session module. 3 | """ 4 | import json 5 | import asyncio 6 | import pytest 7 | from libkirk.session import Session 8 | from libkirk.tempfile import TempDir 9 | 10 | 11 | pytestmark = pytest.mark.asyncio 12 | 13 | 14 | @pytest.fixture 15 | async def sut(): 16 | """ 17 | SUT communication object. 18 | """ 19 | raise NotImplementedError() 20 | 21 | 22 | class _TestSession: 23 | """ 24 | Test for Session class. 25 | """ 26 | 27 | @pytest.fixture 28 | async def session(self, tmpdir, sut, dummy_framework): 29 | """ 30 | Session communication object. 31 | """ 32 | session = Session( 33 | tmpdir=TempDir(str(tmpdir)), 34 | framework=dummy_framework, 35 | sut=sut) 36 | 37 | yield session 38 | 39 | await asyncio.wait_for(session.stop(), timeout=30) 40 | 41 | async def test_run(self, session): 42 | """ 43 | Test run method when executing suites. 44 | """ 45 | await session.run(suites=["suite01", "suite02"]) 46 | 47 | async def test_run_pattern(self, tmpdir, session): 48 | """ 49 | Test run method when executing tests filtered out with a pattern. 50 | """ 51 | report = str(tmpdir / "report.json") 52 | await session.run( 53 | suites=["suite01", "suite02"], 54 | pattern="test01|test02", 55 | report_path=report) 56 | 57 | with open(report, "r", encoding="utf-8") as report_file: 58 | report_data = json.loads(report_file.read()) 59 | assert len(report_data["results"]) == 4 60 | 61 | async def test_run_report(self, tmpdir, session): 62 | """ 63 | Test run method when executing suites, generating a report. 64 | """ 65 | report = str(tmpdir / "report.json") 66 | await session.run( 67 | suites=["suite01", "suite02"], 68 | report_path=report) 69 | 70 | with open(report, "r") as report_file: 71 | report_data = json.loads(report_file.read()) 72 | assert len(report_data["results"]) == 4 73 | 74 | async def test_run_stop(self, session): 75 | """ 76 | Test stop method during run. We are not going to generate any results 77 | file, because we are not even sure some tests will be executed. 78 | """ 79 | async def stop(): 80 | await asyncio.sleep(0.2) 81 | await session.stop() 82 | 83 | await asyncio.gather(*[ 84 | session.run(suites=["sleep"]), 85 | stop(), 86 | ]) 87 | 88 | async def test_run_command(self, session): 89 | """ 90 | Test run method when running a single command. 91 | """ 92 | await session.run(command="test") 93 | 94 | async def test_run_command_stop(self, session): 95 | """ 96 | Test stop when runnig a command. 97 | """ 98 | async def stop(): 99 | await asyncio.sleep(0.1) 100 | await asyncio.wait_for(session.stop(), timeout=30) 101 | 102 | await asyncio.gather(*[ 103 | session.run(command="sleep 1"), 104 | stop() 105 | ]) 106 | 107 | async def test_run_skip_tests(self, tmpdir, session): 108 | """ 109 | Test run method when executing suites. 110 | """ 111 | report = str(tmpdir / "report.json") 112 | await session.run( 113 | suites=["suite01", "suite02"], 114 | skip_tests="test0[23]", 115 | report_path=report) 116 | 117 | with open(report, "r", encoding="utf-8") as report_file: 118 | report_data = json.loads(report_file.read()) 119 | assert len(report_data["results"]) == 2 120 | 121 | @pytest.mark.parametrize( 122 | "iterate,expect", 123 | [ 124 | (0, 4), 125 | (1, 4), 126 | (3, 12), 127 | ] 128 | ) 129 | async def test_run_suite_iterate(self, tmpdir, session, iterate, expect): 130 | """ 131 | Test run method when executing a testing suite multiple times. 132 | """ 133 | report = str(tmpdir / "report.json") 134 | await session.run( 135 | suites=["suite01", "suite02"], 136 | suite_iterate=iterate, 137 | report_path=report) 138 | 139 | with open(report, "r", encoding="utf-8") as report_file: 140 | report_data = json.loads(report_file.read()) 141 | assert len(report_data["results"]) == expect 142 | 143 | async def test_run_randomize(self, tmpdir, session): 144 | """ 145 | Test run method when executing shuffled tests. 146 | """ 147 | num_of_suites = 50 148 | 149 | report = str(tmpdir / "report.json") 150 | await session.run( 151 | suites=["suite01"] * num_of_suites, 152 | randomize=True, 153 | report_path=report) 154 | 155 | report_data = None 156 | with open(report, "r", encoding="utf-8") as report_file: 157 | report_data = json.loads(report_file.read()) 158 | 159 | assert len(report_data["results"]) == 2 * num_of_suites 160 | 161 | tests_names = [] 162 | for test in report_data["results"]: 163 | tests_names.append(test["test_fqn"]) 164 | 165 | assert ["test01", "test02"] * num_of_suites != tests_names 166 | 167 | async def test_run_runtime(self, tmpdir, session): 168 | """ 169 | Test run method when executing suites for a certain amount of time. 170 | """ 171 | report = str(tmpdir / "report.json") 172 | await session.run( 173 | suites=["suite01"], 174 | runtime=1, 175 | report_path=report) 176 | 177 | with open(report, "r", encoding="utf-8") as report_file: 178 | report_data = json.loads(report_file.read()) 179 | assert len(report_data["results"]) >= 2 180 | -------------------------------------------------------------------------------- /libkirk/tests/test_ssh.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unittests for ssh module. 3 | """ 4 | import os 5 | import subprocess 6 | import asyncio 7 | import pytest 8 | import pytest 9 | from libkirk.sut import IOBuffer 10 | from libkirk.sut import KernelPanicError 11 | from libkirk.ssh import SSHSUT 12 | from libkirk.tests.test_sut import _TestSUT 13 | from libkirk.tests.test_session import _TestSession 14 | 15 | pytestmark = [pytest.mark.asyncio, pytest.mark.ssh] 16 | 17 | TEST_SSH_USERNAME = os.environ.get("TEST_SSH_USERNAME", None) 18 | TEST_SSH_PASSWORD = os.environ.get("TEST_SSH_PASSWORD", None) 19 | TEST_SSH_KEY_FILE = os.environ.get("TEST_SSH_KEY_FILE", None) 20 | 21 | if not TEST_SSH_USERNAME: 22 | pytestmark.append(pytest.mark.skip( 23 | reason="TEST_SSH_USERNAME not defined")) 24 | 25 | if not TEST_SSH_PASSWORD: 26 | pytestmark.append(pytest.mark.skip( 27 | reason="TEST_SSH_PASSWORD not defined")) 28 | 29 | if not TEST_SSH_KEY_FILE: 30 | pytestmark.append(pytest.mark.skip( 31 | reason="TEST_SSH_KEY_FILE not defined")) 32 | 33 | 34 | @pytest.fixture 35 | def config(): 36 | """ 37 | Base configuration to connect to SUT. 38 | """ 39 | raise NotImplementedError() 40 | 41 | 42 | @pytest.fixture 43 | async def sut(config): 44 | """ 45 | SSH SUT communication object. 46 | """ 47 | _sut = SSHSUT() 48 | _sut.setup(**config) 49 | 50 | yield _sut 51 | 52 | if await _sut.is_running: 53 | await _sut.stop() 54 | 55 | 56 | class _TestSSHSUT(_TestSUT): 57 | """ 58 | Test SSHSUT implementation using username/password. 59 | """ 60 | 61 | async def test_reset_cmd(self, config): 62 | """ 63 | Test reset_cmd option. 64 | """ 65 | kwargs = dict(reset_cmd="echo ciao") 66 | kwargs.update(config) 67 | 68 | sut = SSHSUT() 69 | sut.setup(**kwargs) 70 | await sut.communicate() 71 | 72 | class MyBuffer(IOBuffer): 73 | data = "" 74 | 75 | async def write(self, data: str) -> None: 76 | self.data = data 77 | # wait for data inside the buffer 78 | await asyncio.sleep(0.1) 79 | 80 | buffer = MyBuffer() 81 | await sut.stop(iobuffer=buffer) 82 | 83 | assert buffer.data == 'ciao\n' 84 | 85 | @pytest.mark.parametrize("enable", ["0", "1"]) 86 | async def test_sudo(self, config, enable): 87 | """ 88 | Test sudo parameter. 89 | """ 90 | kwargs = dict(sudo=enable) 91 | kwargs.update(config) 92 | 93 | sut = SSHSUT() 94 | sut.setup(**kwargs) 95 | await sut.communicate() 96 | ret = await sut.run_command("whoami") 97 | 98 | if enable == "1": 99 | assert ret["stdout"] == "root\n" 100 | else: 101 | assert ret["stdout"] != "root\n" 102 | 103 | async def test_kernel_panic(self, sut): 104 | """ 105 | Test kernel panic recognition. 106 | """ 107 | await sut.communicate() 108 | 109 | with pytest.raises(KernelPanicError): 110 | await sut.run_command( 111 | "echo 'Kernel panic\nThis is a generic message'") 112 | 113 | async def test_stderr(self, sut): 114 | """ 115 | Test if we are correctly reading stderr. 116 | """ 117 | await sut.communicate() 118 | 119 | ret = await sut.run_command(">&2 echo ciao_stderr && echo ciao_stdout") 120 | assert ret["stdout"] == "ciao_stdout\nciao_stderr\n" 121 | 122 | async def test_long_stdout(self, sut): 123 | """ 124 | Test really long stdout. 125 | """ 126 | await sut.communicate() 127 | 128 | result = subprocess.run( 129 | "tr -dc 'a-zA-Z0-9' None: 23 | self._logger = logging.getLogger("test.host") 24 | 25 | async def write(self, data: str) -> None: 26 | print(data, end="") 27 | 28 | 29 | @pytest.fixture 30 | def sut(): 31 | """ 32 | Expose the SUT implementation via this fixture in order to test it. 33 | """ 34 | raise NotImplementedError() 35 | 36 | 37 | class _TestSUT: 38 | """ 39 | Generic tests for SUT implementation. 40 | """ 41 | 42 | _logger = logging.getLogger("test.asyncsut") 43 | 44 | async def test_config_help(self, sut): 45 | """ 46 | Test if config_help has the right type. 47 | """ 48 | assert isinstance(sut.config_help, dict) 49 | 50 | async def test_ping_no_running(self, sut): 51 | """ 52 | Test ping method with no running sut. 53 | """ 54 | with pytest.raises(SUTError): 55 | await sut.ping() 56 | 57 | async def test_ping(self, sut): 58 | """ 59 | Test ping method. 60 | """ 61 | await sut.communicate(iobuffer=Printer()) 62 | ping_t = await sut.ping() 63 | assert ping_t > 0 64 | 65 | async def test_get_info(self, sut): 66 | """ 67 | Test get_info method. 68 | """ 69 | await sut.communicate(iobuffer=Printer()) 70 | info = await sut.get_info() 71 | 72 | assert info["distro"] 73 | assert info["distro_ver"] 74 | assert info["kernel"] 75 | assert info["arch"] 76 | 77 | async def test_get_tainted_info(self, sut): 78 | """ 79 | Test get_tainted_info. 80 | """ 81 | await sut.communicate(iobuffer=Printer()) 82 | code, messages = await sut.get_tainted_info() 83 | 84 | assert code >= 0 85 | assert isinstance(messages, list) 86 | 87 | async def test_communicate(self, sut): 88 | """ 89 | Test communicate method. 90 | """ 91 | await sut.communicate(iobuffer=Printer()) 92 | with pytest.raises(SUTError): 93 | await sut.communicate(iobuffer=Printer()) 94 | 95 | async def test_ensure_communicate(self, sut): 96 | """ 97 | Test ensure_communicate method. 98 | """ 99 | await sut.ensure_communicate(iobuffer=Printer()) 100 | with pytest.raises(SUTError): 101 | await sut.ensure_communicate(iobuffer=Printer(), retries=1) 102 | 103 | @pytest.fixture 104 | def sut_stop_sleep(self, request): 105 | """ 106 | Setup sleep time before calling stop after communicate. 107 | By changing multiply factor it's possible to tweak stop sleep and 108 | change the behaviour of `test_stop_communicate`. 109 | """ 110 | return request.param * 1.0 111 | 112 | @pytest.mark.parametrize("sut_stop_sleep", [1, 2], indirect=True) 113 | async def test_communicate_stop(self, sut, sut_stop_sleep): 114 | """ 115 | Test stop method when running communicate. 116 | """ 117 | async def stop(): 118 | await asyncio.sleep(sut_stop_sleep) 119 | await sut.stop(iobuffer=Printer()) 120 | 121 | await asyncio.gather(*[ 122 | sut.communicate(iobuffer=Printer()), 123 | stop() 124 | ], return_exceptions=True) 125 | 126 | async def test_run_command(self, sut): 127 | """ 128 | Execute run_command once. 129 | """ 130 | await sut.communicate(iobuffer=Printer()) 131 | res = await sut.run_command("echo 0") 132 | 133 | assert res["returncode"] == 0 134 | assert int(res["stdout"]) == 0 135 | assert 0 < res["exec_time"] < time.time() 136 | 137 | async def test_run_command_stop(self, sut): 138 | """ 139 | Execute run_command once, then call stop(). 140 | """ 141 | await sut.communicate(iobuffer=Printer()) 142 | 143 | async def stop(): 144 | await asyncio.sleep(0.2) 145 | await sut.stop(iobuffer=Printer()) 146 | 147 | async def test(): 148 | res = await sut.run_command("sleep 2") 149 | 150 | assert res["returncode"] != 0 151 | assert 0 < res["exec_time"] < 2 152 | 153 | await asyncio.gather(*[ 154 | test(), 155 | stop() 156 | ]) 157 | 158 | async def test_run_command_parallel(self, sut): 159 | """ 160 | Execute run_command in parallel. 161 | """ 162 | if not sut.parallel_execution: 163 | pytest.skip(reason="Parallel execution is not supported") 164 | 165 | await sut.communicate(iobuffer=Printer()) 166 | 167 | exec_count = os.cpu_count() 168 | coros = [sut.run_command(f"echo {i}") 169 | for i in range(exec_count)] 170 | 171 | results = await asyncio.gather(*coros) 172 | 173 | for data in results: 174 | assert data["returncode"] == 0 175 | assert 0 <= int(data["stdout"]) < exec_count 176 | assert 0 < data["exec_time"] < time.time() 177 | 178 | async def test_run_command_stop_parallel(self, sut): 179 | """ 180 | Execute multiple run_command in parallel, then call stop(). 181 | """ 182 | if not sut.parallel_execution: 183 | pytest.skip(reason="Parallel execution is not supported") 184 | 185 | await sut.communicate(iobuffer=Printer()) 186 | 187 | async def stop(): 188 | await asyncio.sleep(0.2) 189 | await sut.stop(iobuffer=Printer()) 190 | 191 | async def test(): 192 | exec_count = os.cpu_count() 193 | coros = [sut.run_command("sleep 2") 194 | for i in range(exec_count)] 195 | results = await asyncio.gather(*coros, return_exceptions=True) 196 | 197 | for data in results: 198 | if not isinstance(data, dict): 199 | # we also have stop() return 200 | continue 201 | 202 | assert data["returncode"] != 0 203 | assert 0 < data["exec_time"] < 2 204 | 205 | await asyncio.gather(*[ 206 | test(), 207 | stop() 208 | ]) 209 | 210 | async def test_fetch_file_bad_args(self, sut): 211 | """ 212 | Test fetch_file method with bad arguments. 213 | """ 214 | await sut.communicate(iobuffer=Printer()) 215 | 216 | with pytest.raises(ValueError): 217 | await sut.fetch_file(None) 218 | 219 | with pytest.raises(SUTError): 220 | await sut.fetch_file('this_file_doesnt_exist') 221 | 222 | async def test_fetch_file(self, sut): 223 | """ 224 | Test fetch_file method. 225 | """ 226 | await sut.communicate(iobuffer=Printer()) 227 | 228 | for i in range(0, 5): 229 | myfile = f"/tmp/myfile{i}" 230 | await sut.run_command(f"echo -n 'mytests' > {myfile}") 231 | data = await sut.fetch_file(myfile) 232 | 233 | assert data == b"mytests" 234 | 235 | async def test_fetch_file_stop(self, sut): 236 | """ 237 | Test stop method when running fetch_file. 238 | """ 239 | target = "/tmp/target_file" 240 | await sut.communicate(iobuffer=Printer()) 241 | 242 | async def fetch(): 243 | await sut.run_command(f"truncate -s {1024*1024*1024} {target}"), 244 | await sut.fetch_file(target) 245 | 246 | async def stop(): 247 | await asyncio.sleep(2) 248 | await sut.stop(iobuffer=Printer()) 249 | 250 | libkirk.create_task(fetch()) 251 | 252 | await stop() 253 | 254 | async def test_cwd(self, sut): 255 | """ 256 | Test CWD constructor argument. 257 | """ 258 | await sut.communicate(iobuffer=Printer()) 259 | 260 | ret = await sut.run_command( 261 | "echo -n $PWD", 262 | cwd="/tmp", 263 | iobuffer=Printer()) 264 | 265 | assert ret["returncode"] == 0 266 | assert ret["stdout"].strip() == "/tmp" 267 | 268 | async def test_env(self, sut): 269 | """ 270 | Test ENV constructor argument. 271 | """ 272 | await sut.communicate(iobuffer=Printer()) 273 | 274 | ret = await sut.run_command( 275 | "echo -n $HELLO", 276 | env=dict(HELLO="ciao"), 277 | iobuffer=Printer()) 278 | 279 | assert ret["returncode"] == 0 280 | assert ret["stdout"].strip() == "ciao" 281 | -------------------------------------------------------------------------------- /libkirk/tests/test_tempfile.py: -------------------------------------------------------------------------------- 1 | """ 2 | Unittest for temporary module. 3 | """ 4 | import os 5 | import pytest 6 | from libkirk.tempfile import TempDir 7 | 8 | 9 | class TestTempDir: 10 | """ 11 | Test the TempDir class implementation. 12 | """ 13 | 14 | def test_constructor(self): 15 | """ 16 | Test TempDir constructor. 17 | """ 18 | with pytest.raises(ValueError): 19 | TempDir(root="this_folder_doesnt_exist") 20 | 21 | # for some reasons, following test fails on systems which are slow 22 | # to release directories after remove (in particular remote containers) 23 | # even after os.sync or time.sleep. So we XFAIL this test by default 24 | @pytest.mark.xfail 25 | def test_rotate(self, tmpdir): 26 | """ 27 | Test folders rotation. 28 | """ 29 | max_rotate = 5 30 | plus_rotate = 5 31 | 32 | currdir = str(tmpdir) 33 | 34 | tempdir = None 35 | for _ in range(0, max_rotate + plus_rotate): 36 | tempdir = TempDir(currdir, max_rotate=max_rotate) 37 | 38 | assert tempdir.abspath is not None 39 | assert tempdir.abspath == os.readlink( 40 | os.path.join(tempdir.abspath, "..", tempdir.SYMLINK_NAME)) 41 | 42 | os.sync() 43 | 44 | total = 0 45 | for _, dirs, _ in os.walk(os.path.join(tempdir.abspath, "..")): 46 | for mydir in dirs: 47 | if mydir != "latest": 48 | total += 1 49 | 50 | assert total == max_rotate 51 | 52 | def test_rotate_empty_root(self): 53 | """ 54 | Test folders rotation with empty root. 55 | """ 56 | tempdir = TempDir(None) 57 | assert not os.path.isdir(tempdir.abspath) 58 | 59 | def test_mkdir(self, tmpdir): 60 | """ 61 | Test mkdir method. 62 | """ 63 | tempdir = TempDir(str(tmpdir)) 64 | tempdir.mkdir("myfolder") 65 | assert os.path.isdir(os.path.join(tempdir.abspath, "myfolder")) 66 | 67 | for i in range(0, 10): 68 | tempdir.mkdir(f"myfolder/{i}") 69 | assert os.path.isdir(os.path.join( 70 | tempdir.abspath, f"myfolder/{i}")) 71 | 72 | def test_mkdir_no_root(self): 73 | """ 74 | Test mkdir method without root. 75 | """ 76 | tempdir = TempDir(None) 77 | tempdir.mkdir("myfolder") 78 | assert not os.path.isdir(os.path.join(tempdir.abspath, "myfolder")) 79 | 80 | def test_mkfile(self, tmpdir): 81 | """ 82 | Test mkfile method. 83 | """ 84 | content = "mystuff" 85 | tempdir = TempDir(str(tmpdir)) 86 | 87 | for i in range(0, 10): 88 | tempdir.mkfile(f"myfile{i}", content) 89 | 90 | pos = os.path.join(tempdir.abspath, f"myfile{i}") 91 | assert os.path.isfile(pos) 92 | assert open(pos, "r").read() == "mystuff" 93 | 94 | def test_mkfile_no_root(self): 95 | """ 96 | Test mkfile method without root. 97 | """ 98 | content = "mystuff" 99 | tempdir = TempDir(None) 100 | 101 | tempdir.mkfile("myfile", content) 102 | assert not os.path.isfile(os.path.join(tempdir.abspath, "myfile")) 103 | 104 | def test_mkdir_mkfile(self, tmpdir): 105 | """ 106 | Test mkfile after mkdir. 107 | """ 108 | content = "mystuff" 109 | tempdir = TempDir(str(tmpdir)) 110 | 111 | tempdir.mkdir("mydir") 112 | tempdir.mkfile("mydir/myfile", content) 113 | 114 | pos = os.path.join(tempdir.abspath, "mydir", "myfile") 115 | assert os.path.isfile(pos) 116 | assert open(pos, "r").read() == "mystuff" 117 | -------------------------------------------------------------------------------- /pylint.ini: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # Specify a configuration file. 4 | #rcfile= 5 | 6 | # Python code to execute, usually for sys.path manipulation such as 7 | # pygtk.require(). 8 | #init-hook= 9 | 10 | # Profiled execution. 11 | profile=no 12 | 13 | # Add files or directories to the blacklist. They should be base names, not 14 | # paths. 15 | ignore=CVS,tests 16 | 17 | # Pickle collected data for later comparisons. 18 | persistent=yes 19 | 20 | # List of plugins (as comma separated values of python modules names) to load, 21 | # usually to register additional checkers. 22 | load-plugins= 23 | 24 | # Deprecated. It was used to include message's id in output. Use --msg-template 25 | # instead. 26 | include-ids=no 27 | 28 | # Deprecated. It was used to include symbolic ids of messages in output. Use 29 | # --msg-template instead. 30 | symbols=no 31 | 32 | # Use multiple processes to speed up Pylint. 33 | jobs=1 34 | 35 | # Allow loading of arbitrary C extensions. Extensions are imported into the 36 | # active Python interpreter and may run arbitrary code. 37 | unsafe-load-any-extension=no 38 | 39 | # A comma-separated list of package or module names from where C extensions may 40 | # be loaded. Extensions are loading into the active Python interpreter and may 41 | # run arbitrary code 42 | extension-pkg-whitelist= 43 | 44 | # Allow optimization of some AST trees. This will activate a peephole AST 45 | # optimizer, which will apply various small optimizations. For instance, it can 46 | # be used to obtain the result of joining multiple strings with the addition 47 | # operator. Joining a lot of strings can lead to a maximum recursion error in 48 | # Pylint and this flag can prevent that. It has one side effect, the resulting 49 | # AST will be different than the one from reality. 50 | optimize-ast=no 51 | 52 | 53 | [MESSAGES CONTROL] 54 | 55 | # Only show warnings with the listed confidence levels. Leave empty to show 56 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED 57 | confidence= 58 | 59 | # Enable the message, report, category or checker with the given id(s). You can 60 | # either give multiple identifier separated by comma (,) or put this option 61 | # multiple time. See also the "--disable" option for examples. 62 | #enable= 63 | 64 | # Disable the message, report, category or checker with the given id(s). You 65 | # can either give multiple identifiers separated by comma (,) or put this 66 | # option multiple times (only on the command line, not in the configuration 67 | # file where it should appear only once).You can also use "--disable=all" to 68 | # disable everything first and then reenable specific checks. For example, if 69 | # you want to run only the similarities checker, you can use "--disable=all 70 | # --enable=similarities". If you want to run only the classes checker, but have 71 | # no Warning level messages displayed, use"--disable=all --enable=classes 72 | # --disable=W" 73 | disable=E1608,W1627,E1601,E1603,E1602,E1605,E1604,E1607,E1606,W1621,W1620,W1623,W1622,W1625,W1624,W1609,W1608,W1607,W1606,W1605,W1604,W1603,W1602,W1601,W1639,W1640,I0021,W1638,I0020,W1618,W1619,W1630,W1626,W1637,W1634,W1635,W1610,W1611,W1612,W1613,W1614,W1615,W1616,W1617,W1632,W1633,W0704,W1628,W1629,W1636,I0011,C0304,C0326,C0330,R0801,W0107,R0205,W0707 74 | 75 | 76 | [REPORTS] 77 | 78 | # Set the output format. Available formats are text, parseable, colorized, msvs 79 | # (visual studio) and html. You can also give a reporter class, eg 80 | # mypackage.mymodule.MyReporterClass. 81 | output-format=parseable 82 | 83 | # Put messages in a separate file for each module / package specified on the 84 | # command line instead of printing them on stdout. Reports (if any) will be 85 | # written in a file name "pylint_global.[txt|html]". 86 | files-output=no 87 | 88 | # Tells whether to display a full report or only the messages 89 | reports=yes 90 | 91 | # Python expression which should return a note less than 10 (10 is the highest 92 | # note). You have access to the variables errors warning, statement which 93 | # respectively contain the number of errors / warnings messages and the total 94 | # number of statements analyzed. This is used by the global evaluation report 95 | # (RP0004). 96 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 97 | 98 | # Add a comment according to your evaluation note. This is used by the global 99 | # evaluation report (RP0004). 100 | comment=no 101 | 102 | # Template used to display messages. This is a python new-style format string 103 | # used to format the message information. See doc for all details 104 | #msg-template= 105 | 106 | 107 | [LOGGING] 108 | 109 | # Logging modules to check that the string format arguments are in logging 110 | # function parameter format 111 | logging-modules=logging 112 | 113 | 114 | [SPELLING] 115 | 116 | # Spelling dictionary name. Available dictionaries: none. To make it working 117 | # install python-enchant package. 118 | spelling-dict= 119 | 120 | # List of comma separated words that should not be checked. 121 | spelling-ignore-words= 122 | 123 | # A path to a file that contains private dictionary; one word per line. 124 | spelling-private-dict-file= 125 | 126 | # Tells whether to store unknown words to indicated private dictionary in 127 | # --spelling-private-dict-file option instead of raising a message. 128 | spelling-store-unknown-words=no 129 | 130 | 131 | [MISCELLANEOUS] 132 | 133 | # List of note tags to take in consideration, separated by a comma. 134 | notes=XXX 135 | 136 | 137 | [TYPECHECK] 138 | 139 | # Tells whether missing members accessed in mixin class should be ignored. A 140 | # mixin class is detected if its name ends with "mixin" (case insensitive). 141 | ignore-mixin-members=yes 142 | 143 | # List of module names for which member attributes should not be checked 144 | # (useful for modules/projects where namespaces are manipulated during runtime 145 | # and thus existing member attributes cannot be deduced by static analysis 146 | ignored-modules= 147 | 148 | # List of classes names for which member attributes should not be checked 149 | # (useful for classes with attributes dynamically set). 150 | ignored-classes=SQLObject 151 | 152 | # When zope mode is activated, add a predefined set of Zope acquired attributes 153 | # to generated-members. 154 | zope=no 155 | 156 | # List of members which are set dynamically and missed by pylint inference 157 | # system, and so shouldn't trigger E0201 when accessed. Python regular 158 | # expressions are accepted. 159 | generated-members=REQUEST,acl_users,aq_parent 160 | 161 | 162 | [BASIC] 163 | 164 | # Required attributes for module, separated by a comma 165 | required-attributes= 166 | 167 | # List of builtins function names that should not be used, separated by a comma 168 | bad-functions=map,filter,input 169 | 170 | # Good variable names which should always be accepted, separated by a comma 171 | good-names=i,j,k,ex,Run,_ 172 | 173 | # Bad variable names which should always be refused, separated by a comma 174 | bad-names=foo,bar,baz,toto,tutu,tata 175 | 176 | # Colon-delimited sets of names that determine each other's naming style when 177 | # the name regexes allow several styles. 178 | name-group= 179 | 180 | # Include a hint for the correct naming format with invalid-name 181 | include-naming-hint=no 182 | 183 | # Regular expression matching correct function names 184 | function-rgx=[a-z_][a-z0-9_]{2,30}$ 185 | 186 | # Naming hint for function names 187 | function-name-hint=[a-z_][a-z0-9_]{2,30}$ 188 | 189 | # Regular expression matching correct variable names 190 | variable-rgx=[a-z_][a-z0-9_]{2,30}$ 191 | 192 | # Naming hint for variable names 193 | variable-name-hint=[a-z_][a-z0-9_]{2,30}$ 194 | 195 | # Regular expression matching correct constant names 196 | const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 197 | 198 | # Naming hint for constant names 199 | const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 200 | 201 | # Regular expression matching correct attribute names 202 | attr-rgx=[a-z_][a-z0-9_]{2,30}$ 203 | 204 | # Naming hint for attribute names 205 | attr-name-hint=[a-z_][a-z0-9_]{2,30}$ 206 | 207 | # Regular expression matching correct argument names 208 | argument-rgx=[a-z_][a-z0-9_]{2,30}$ 209 | 210 | # Naming hint for argument names 211 | argument-name-hint=[a-z_][a-z0-9_]{2,30}$ 212 | 213 | # Regular expression matching correct class attribute names 214 | class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 215 | 216 | # Naming hint for class attribute names 217 | class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 218 | 219 | # Regular expression matching correct inline iteration names 220 | inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ 221 | 222 | # Naming hint for inline iteration names 223 | inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ 224 | 225 | # Regular expression matching correct class names 226 | class-rgx=[A-Z_][a-zA-Z0-9]+$ 227 | 228 | # Naming hint for class names 229 | class-name-hint=[A-Z_][a-zA-Z0-9]+$ 230 | 231 | # Regular expression matching correct module names 232 | module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 233 | 234 | # Naming hint for module names 235 | module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 236 | 237 | # Regular expression matching correct method names 238 | method-rgx=[a-z_][a-z0-9_]{2,30}$ 239 | 240 | # Naming hint for method names 241 | method-name-hint=[a-z_][a-z0-9_]{2,30}$ 242 | 243 | # Regular expression which should only match function or class names that do 244 | # not require a docstring. 245 | no-docstring-rgx=__.*__ 246 | 247 | # Minimum line length for functions/classes that require docstrings, shorter 248 | # ones are exempt. 249 | docstring-min-length=-1 250 | 251 | 252 | [FORMAT] 253 | 254 | # Maximum number of characters on a single line. 255 | max-line-length=100 256 | 257 | # Regexp for a line that is allowed to be longer than the limit. 258 | ignore-long-lines=^\s*(# )??$ 259 | 260 | # Allow the body of an if to be on the same line as the test if there is no 261 | # else. 262 | single-line-if-stmt=no 263 | 264 | # List of optional constructs for which whitespace checking is disabled 265 | no-space-check=trailing-comma,dict-separator 266 | 267 | # Maximum number of lines in a module 268 | max-module-lines=1000 269 | 270 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 271 | # tab). 272 | indent-string=' ' 273 | 274 | # Number of spaces of indent required inside a hanging or continued line. 275 | indent-after-paren=4 276 | 277 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 278 | expected-line-ending-format= 279 | 280 | 281 | [SIMILARITIES] 282 | 283 | # Minimum lines number of a similarity. 284 | min-similarity-lines=4 285 | 286 | # Ignore comments when computing similarities. 287 | ignore-comments=yes 288 | 289 | # Ignore docstrings when computing similarities. 290 | ignore-docstrings=yes 291 | 292 | # Ignore imports when computing similarities. 293 | ignore-imports=no 294 | 295 | 296 | [VARIABLES] 297 | 298 | # Tells whether we should check for unused import in __init__ files. 299 | init-import=no 300 | 301 | # A regular expression matching the name of dummy variables (i.e. expectedly 302 | # not used). 303 | dummy-variables-rgx=_$|dummy 304 | 305 | # List of additional names supposed to be defined in builtins. Remember that 306 | # you should avoid to define new builtins when possible. 307 | additional-builtins= 308 | 309 | # List of strings which can identify a callback function by name. A callback 310 | # name must start or end with one of those strings. 311 | callbacks=cb_,_cb 312 | 313 | 314 | [IMPORTS] 315 | 316 | # Deprecated modules which should not be used, separated by a comma 317 | deprecated-modules=regsub,TERMIOS,Bastion,rexec 318 | 319 | # Create a graph of every (i.e. internal and external) dependencies in the 320 | # given file (report RP0402 must not be disabled) 321 | import-graph= 322 | 323 | # Create a graph of external dependencies in the given file (report RP0402 must 324 | # not be disabled) 325 | ext-import-graph= 326 | 327 | # Create a graph of internal dependencies in the given file (report RP0402 must 328 | # not be disabled) 329 | int-import-graph= 330 | 331 | 332 | [CLASSES] 333 | 334 | # List of interface methods to ignore, separated by a comma. This is used for 335 | # instance to not check methods defines in Zope's Interface base class. 336 | ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by 337 | 338 | # List of method names used to declare (i.e. assign) instance attributes. 339 | defining-attr-methods=__init__,__new__,setUp 340 | 341 | # List of valid names for the first argument in a class method. 342 | valid-classmethod-first-arg=cls 343 | 344 | # List of valid names for the first argument in a metaclass class method. 345 | valid-metaclass-classmethod-first-arg=mcs 346 | 347 | # List of member names, which should be excluded from the protected access 348 | # warning. 349 | exclude-protected=_asdict,_fields,_replace,_source,_make 350 | 351 | 352 | [DESIGN] 353 | 354 | # Maximum number of arguments for function / method 355 | max-args=7 356 | 357 | # Argument names that match this expression will be ignored. Default to name 358 | # with leading underscore 359 | ignored-argument-names=_.* 360 | 361 | # Maximum number of locals for function / method body 362 | max-locals=15 363 | 364 | # Maximum number of return / yield for function / method body 365 | max-returns=6 366 | 367 | # Maximum number of branch for function / method body 368 | max-branches=18 369 | 370 | # Maximum number of statements in function / method body 371 | max-statements=50 372 | 373 | # Maximum number of parents for a class (see R0901). 374 | max-parents=7 375 | 376 | # Maximum number of attributes for a class (see R0902). 377 | max-attributes=12 378 | 379 | # Minimum number of public methods for a class (see R0903). 380 | min-public-methods=0 381 | 382 | # Maximum number of public methods for a class (see R0904). 383 | max-public-methods=20 384 | 385 | 386 | [EXCEPTIONS] 387 | 388 | # Exceptions that will emit a warning when being caught. Defaults to 389 | # "Exception" 390 | overgeneral-exceptions=Exception 391 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "kirk" 7 | dynamic = ["version"] 8 | description = "All-in-one Linux Testing Framework" 9 | readme = "README.md" 10 | license = {file = "LICENSE"} 11 | requires-python = ">=3.6" 12 | keywords = ["testing", "linux", "development", "ltp", "linux-test-project"] 13 | authors = [ 14 | {name = "Linux Test Project", email = "ltp@lists.linux.it" } 15 | ] 16 | maintainers = [ 17 | {name = "Andrea Cervesato", email = "andrea.cervesato@suse.com"} 18 | ] 19 | classifiers = [ 20 | "Natural Language :: English", 21 | "Intended Audience :: Developers", 22 | "Operating System :: OS Independent", 23 | "Programming Language :: Python", 24 | "Programming Language :: Python :: 3.6", 25 | "Programming Language :: Python :: 3.7", 26 | "Programming Language :: Python :: 3.8", 27 | "Programming Language :: Python :: 3.9", 28 | "Programming Language :: Python :: 3.10", 29 | "Programming Language :: Python :: 3.11", 30 | "Programming Language :: Python :: 3.12", 31 | "Programming Language :: Python :: 3.13", 32 | "Topic :: Software Development :: Testing", 33 | ] 34 | 35 | [project.urls] 36 | "Homepage" = "https://github.com/linux-test-project/kirk" 37 | "Bug Reports" = "https://github.com/linux-test-project/kirk/issues" 38 | 39 | [tool.setuptools.dynamic] 40 | version = {attr = "libkirk.__version__"} 41 | 42 | [tool.setuptools.packages.find] 43 | include = ["libkirk"] 44 | exclude = ["libkirk.tests"] 45 | 46 | [project.scripts] 47 | kirk = "libkirk.main:run" 48 | 49 | [project.optional-dependencies] 50 | ssh = ["asyncssh <= 2.17.0"] 51 | ltx = ["msgpack <= 1.1.0"] 52 | 53 | [tool.setuptools] 54 | include-package-data = true 55 | 56 | [tool.pytest.ini_options] 57 | asyncio_mode = "auto" 58 | addopts = "-v" 59 | testpaths = ["libkirk/tests"] 60 | filterwarnings = [ 61 | "ignore::DeprecationWarning", 62 | "ignore::pytest.PytestCollectionWarning", 63 | ] 64 | log_cli = "true" 65 | log_level = "DEBUG" 66 | markers = [ 67 | "ssh", 68 | "qemu", 69 | "ltx", 70 | ] 71 | -------------------------------------------------------------------------------- /utils/json2html.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: GPL-2.0-or-later 3 | # Copyright (c) 2025 Cyril Hrubis 4 | """ 5 | This script parses JSON results from kirk and produces a HTML page. 6 | """ 7 | import os 8 | import json 9 | import argparse 10 | from datetime import timedelta 11 | from html import escape 12 | 13 | _HTML_HEADER = """ 14 | 15 | 16 | LTP results 17 | 75 | 158 | 159 | 160 |
161 |
162 |

LTP Results

""" 163 | 164 | _HTML_FOOTER = """
165 |
166 | 182 | 183 | """ 184 | 185 | 186 | def _generate_environment(environment): 187 | """ 188 | Generates HTML environment table. 189 | """ 190 | out = [] 191 | out.append(" ") 192 | out.append(" ") 193 | out.append(" ") 194 | out.append(" ") 195 | 196 | print("\n".join(out)) 197 | 198 | for key in environment: 199 | out = [] 200 | 201 | out.append(" ") 202 | out.append(f" ") 203 | out.append(f" ") 204 | out.append(" ") 205 | 206 | print("\n".join(out)) 207 | 208 | print("
Environment information
{key}{environment[key]}
") 209 | 210 | 211 | def _generate_stats(stats): 212 | """ 213 | Generates HTML overall statistics. 214 | """ 215 | 216 | out = [] 217 | out.append(" ") 218 | out.append(" ") 219 | out.append(" ") 220 | out.append(" ") 221 | out.append(" ") 222 | out.append(f" ") 223 | out.append(f" ") 224 | out.append(f" ") 225 | out.append(f" ") 226 | out.append(f" ") 227 | out.append(f" ") 228 | out.append(" ") 229 | out.append("
Overall results
Runtime: {str(timedelta(seconds=stats['runtime']))}Passed: {stats['passed']}Skipped: {stats['skipped']}Failed: {stats['failed']}Broken: {stats['broken']}Warnings: {stats['warnings']}
") 230 | 231 | print("\n".join(out)) 232 | 233 | 234 | _RESULT_TABLE_HEADER = """
235 |
236 | Hide Passed 237 | Hide Skipped 238 | Filter by ID: 239 |
240 |
241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | """ 251 | 252 | 253 | def _generate_results(results): 254 | """ 255 | generates html result table. 256 | """ 257 | print(_RESULT_TABLE_HEADER) 258 | 259 | for res in results: 260 | overall = 'pass' 261 | 262 | test = res['test'] 263 | 264 | if test['failed'] > 0: 265 | overall = 'fail' 266 | elif test['broken'] > 0: 267 | overall = 'brok' 268 | elif test['warnings'] > 0: 269 | overall = 'warn' 270 | elif test['skipped'] > 0: 271 | overall = 'skip' 272 | 273 | out = [] 274 | 275 | out.append(f" ") 276 | out.append(f" ") 277 | out.append(f" ") 278 | out.append(f" ") 279 | out.append(f" ") 280 | out.append(f" ") 281 | out.append(f" ") 282 | out.append(f" ") 283 | out.append(" ") 284 | out.append(" ") 285 | out.append(" ") 289 | out.append(" ") 290 | 291 | print("\n".join(out)) 292 | 293 | print("
Test ID ↕Duration ↕Passes ↕Skips ↕Fails ↕Broken ↕Warns ↕
{res['test_fqn']}{test['duration']:.2f}{test['passed']}{test['skipped']}{test['failed']}{test['broken']}{test['warnings']}
") 286 | out.append(" 
")
287 |         out.append(escape(test['log']) + "
") 288 | out.append("
") 294 | 295 | 296 | def _generate_html(results_path): 297 | """ 298 | Generates HTML results. 299 | """ 300 | print(_HTML_HEADER) 301 | 302 | with open(results_path, 'r', encoding="utf-8") as file: 303 | results = json.load(file) 304 | 305 | _generate_environment(results['environment']) 306 | _generate_stats(results['stats']) 307 | _generate_results(results['results']) 308 | 309 | print(_HTML_FOOTER) 310 | 311 | 312 | def _file_exists(filepath): 313 | """ 314 | Check if the given file path exists. 315 | """ 316 | if not os.path.isfile(filepath): 317 | raise argparse.ArgumentTypeError( 318 | f"The file '{filepath}' does not exist.") 319 | return filepath 320 | 321 | 322 | def run(): 323 | """ 324 | Entry point of the script. 325 | """ 326 | parser = argparse.ArgumentParser( 327 | description="Script to generate simple HTML result table.") 328 | 329 | parser.add_argument( 330 | '-r', 331 | '--results', 332 | type=_file_exists, 333 | required=True, 334 | help='kirk results.json file location') 335 | 336 | args = parser.parse_args() 337 | 338 | _generate_html(args.results) 339 | 340 | 341 | if __name__ == "__main__": 342 | run() 343 | --------------------------------------------------------------------------------