├── skt
    ├── __init__.py
    ├── misc.py
    ├── executable.py
    └── runner.py
├── tests
    ├── __init__.py
    ├── assets
    │   ├── 0.xml
    │   ├── testing_state.cfg
    │   ├── testing_state_no_runner.cfg
    │   ├── beaker_fail_results.xml
    │   ├── beaker_pass_results.xml
    │   ├── beaker_skip_and_fail.xml
    │   ├── beaker_cancel_results.xml
    │   ├── beaker_results2.xml
    │   ├── beaker_results3.xml
    │   ├── beaker_recipe_set_panic_results.xml
    │   ├── beaker_recipe_set_results.xml
    │   ├── beaker_results.xml
    │   ├── actual_rc.cfg
    │   ├── beaker_recipe_set_fail_results.xml
    │   ├── test.xml
    │   ├── beaker_wait_pass.xml
    │   └── beaker_aborted_some.xml
    ├── test_executable.py
    ├── misc.py
    └── test_runner.py
├── .gitignore
├── MANIFEST.in
├── .travis.yml
├── setup.py
├── tox.ini
├── CHANGELOG.md
├── setup.cfg
├── CONTRIBUTING.md
├── README.md
└── LICENSE


/skt/__init__.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/tests/__init__.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/tests/assets/0.xml:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *.pyc
2 | *.swp
3 | *.swo
4 | skt.egg-info
5 | .tox/
6 | .coverage
7 | 


--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include README.md
2 | include LICENSE
3 | include skt/*
4 | include tests/*
5 | 


--------------------------------------------------------------------------------
/tests/assets/testing_state.cfg:
--------------------------------------------------------------------------------
1 | [jobs]
2 | foo=bar
3 | 
4 | [state]
5 | jobs=J:123456
6 | 
7 | [runner]
8 | jobtemplate=test_template.xml
9 | 


--------------------------------------------------------------------------------
/tests/assets/testing_state_no_runner.cfg:
--------------------------------------------------------------------------------
1 | [jobs]
2 | foo=bar
3 | 
4 | [state]
5 | jobs=J:123456
6 | patchwork_01=http://example.com/patch/1
7 | patchwork_02=http://example.com/patch/1
8 | foo=bar
9 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | language: python
 2 | dist: xenial
 3 | python:
 4 |     - "3.7"
 5 | env:
 6 |     - TOX_ENV=flake8,pylint
 7 | install:
 8 |     # Install skt using pip to ensure dependencies are downloaded correctly.
 9 |     - pip install .[dev]
10 |     - pip install coveralls tox
11 | script:
12 |     - tox -e $(echo py${TRAVIS_PYTHON_VERSION} | tr -d .) -e ${TOX_ENV}
13 |     - pip uninstall -y skt
14 | 


--------------------------------------------------------------------------------
/tests/assets/beaker_fail_results.xml:
--------------------------------------------------------------------------------
 1 | <recipeSet>
 2 |   <recipe id="1234" result='Fail' status='Completed'>
 3 |     <logs>
 4 |       <log name='console.log' href="http://example.com/">
 5 |         TEST RESULT
 6 |       </log>
 7 |     </logs>
 8 |     <task name='/test/misc/machineinfo'>
 9 |       <logs>
10 |         <log name='machinedesc.log' href="http://example.com/machinedesc.log">
11 |         </log>
12 |         <log name='lshw.log' href="http://example.com/lshw.log">
13 |         </log>
14 |       </logs>
15 |     </task>
16 |     <task name='/distribution/install' result='Fail'>
17 |     </task>
18 |   </recipe>
19 | </recipeSet>
20 | 


--------------------------------------------------------------------------------
/tests/assets/beaker_pass_results.xml:
--------------------------------------------------------------------------------
 1 | <recipeSet>
 2 |   <recipe result='Pass' status='Completed'>
 3 |     <logs>
 4 |       <log name='console.log' href="http://example.com/">
 5 |         TEST RESULT
 6 |       </log>
 7 |     </logs>
 8 |     <task name='/test/misc/machineinfo'>
 9 |       <logs>
10 |         <log name='machinedesc.log' href="http://example.com/machinedesc.log">
11 |         </log>
12 |         <log name='lshw.log' href="http://example.com/lshw.log">
13 |         </log>
14 |       </logs>
15 |     </task>
16 |     <task name='/distribution/install' result='Pass' status='Completed'>
17 |     </task>
18 |   </recipe>
19 | </recipeSet>
20 | 


--------------------------------------------------------------------------------
/tests/assets/beaker_skip_and_fail.xml:
--------------------------------------------------------------------------------
 1 | <job id="0001">
 2 |   <whiteboard>
 3 |     skt 4.17.0-rc1+ 1234567890.tar.gz [noavc] [noselinux]
 4 |   </whiteboard>
 5 |     <recipeSet>
 6 |     <recipe result='Fail' status='Completed' id="123123">
 7 |       <logs>
 8 |         <log name='console.log' href="http://example.com/">
 9 |           TEST RESULT
10 |         </log>
11 |       </logs>
12 |       <task name='/distribution/install' result='Pass' status="Completed"/>
13 |       <task name='/test/we/skipped' result="Skip" status="Completed"/>
14 |       <task name='/test/we/ran' result="Fail" status="Completed"/>
15 |     </recipe>
16 |   </recipeSet>
17 | </job>
18 | 


--------------------------------------------------------------------------------
/tests/assets/beaker_cancel_results.xml:
--------------------------------------------------------------------------------
 1 | <xml result='Fail' status='Cancelled'>
 2 |   <whiteboard>
 3 |     skt 4.17.0-rc1+ 1234567890.tar.gz [noavc] [noselinux]
 4 |   </whiteboard>
 5 |   <recipeSet>
 6 |     <recipe>
 7 |       <logs>
 8 |         <log name='console.log' href="http://example.com/">
 9 |           TEST RESULT
10 |         </log>
11 |       </logs>
12 |       <task name='/test/misc/machineinfo'>
13 |         <logs>
14 |           <log name='machinedesc.log' href="http://example.com/machinedesc.log">
15 |           </log>
16 |           <log name='lshw.log' href="http://example.com/lshw.log">
17 |           </log>
18 |         </logs>
19 |       </task>
20 |     </recipe>
21 |   </recipeSet>
22 | </xml>
23 | 


--------------------------------------------------------------------------------
/tests/assets/beaker_results2.xml:
--------------------------------------------------------------------------------
 1 | <job id='101010' result='Pass' status='Completed'>
 2 |   <whiteboard>
 3 |     skt 4.17.0-rc1+ 1234567890.tar.gz [noavc] [noselinux]
 4 |   </whiteboard>
 5 |   <recipeSet id="123">
 6 |     <recipe result='Warn' system='machine.beaker.org' id="456">
 7 |       <hostRequires>
 8 |       </hostRequires>
 9 |       <logs>
10 |         <log name='console.log' href="http://example.com/">
11 |           TEST RESULT
12 |         </log>
13 |       </logs>
14 |       <task name='/test/misc/machineinfo' result="Warn">
15 |         <logs>
16 |           <log name='machinedesc.log' href="http://example.com/machinedesc.log">
17 |           </log>
18 |           <log name='lshw.log' href="http://example.com/lshw.log">
19 |           </log>
20 |         </logs>
21 |       </task>
22 |     </recipe>
23 |   </recipeSet>
24 | </job>
25 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | # Copyright (c) 2017-2020 Red Hat, Inc. All rights reserved. This copyrighted
 3 | # material is made available to anyone wishing to use, modify, copy, or
 4 | # redistribute it subject to the terms and conditions of the GNU General
 5 | # Public License v.2 or later.
 6 | #
 7 | # This program is distributed in the hope that it will be useful, but WITHOUT
 8 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 9 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
10 | # details.
11 | #
12 | # You should have received a copy of the GNU General Public License
13 | # along with this program; if not, write to the Free Software Foundation, Inc.,
14 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
15 | """Install skt using setuptools."""
16 | from setuptools import setup
17 | 
18 | setup()
19 | 


--------------------------------------------------------------------------------
/tests/assets/beaker_results3.xml:
--------------------------------------------------------------------------------
 1 | <job id='101010' result='Pass' status='Completed'>
 2 |   <whiteboard>
 3 |     skt 4.17.0-rc1+ 1234567890.tar.gz [noavc] [noselinux]
 4 |   </whiteboard>
 5 |   <recipeSet id="123">
 6 |     <recipe result='Warn' system='machine.beaker.org' id="456">
 7 |       <hostRequires>
 8 |       </hostRequires>
 9 |       <logs>
10 |         <log name='console.log' href="http://example.com/">
11 |           TEST RESULT
12 |         </log>
13 |       </logs>
14 |       <task name="/test/misc/boottest" result="Pass"><fetch url="kpkginstall"/></task>
15 |       <task name='/test/misc/machineinfo' result="Pass">
16 |         <logs>
17 |           <log name='machinedesc.log' href="http://example.com/machinedesc.log">
18 |           </log>
19 |           <log name='lshw.log' href="http://example.com/lshw.log">
20 |           </log>
21 |         </logs>
22 |       </task>
23 | 
24 |     </recipe>
25 |   </recipeSet>
26 | 
27 | </job>
28 | 


--------------------------------------------------------------------------------
/tests/assets/beaker_recipe_set_panic_results.xml:
--------------------------------------------------------------------------------
 1 | <recipeSet>
 2 |   <recipe system='machine.beaker.org' result='Fail' status='Completed'>
 3 |     <hostRequires>
 4 |       <and>
 5 |         <arch op="=" value="x86_64"/>
 6 |       </and>
 7 |     </hostRequires>
 8 |     <logs>
 9 |       <log name='console.log' href="http://example.com/console">
10 |         TEST RESULT
11 |       </log>
12 |     </logs>
13 |     <task name='/test/misc/machineinfo' result='Pass' status='Completed'>
14 |       <logs>
15 |         <log name='machinedesc.log' href="http://example.com/machinedesc.log">
16 |         </log>
17 |         <log name='lshw.log' href="http://example.com/lshw.log">
18 |         </log>
19 |       </logs>
20 |     </task>
21 |     <task name='/distribution/kpkginstall' result='Panic' status='Aborted'>
22 |       <fetch url="https://github.com/CKI-project/tests-beaker/archive/master.zip#distribution/kpkginstall"/>
23 |     </task>
24 |   </recipe>
25 | </recipeSet>
26 | 


--------------------------------------------------------------------------------
/tox.ini:
--------------------------------------------------------------------------------
 1 | [tox]
 2 | envlist =
 3 |     flake8
 4 |     pylint
 5 | 
 6 | [testenv]
 7 | passenv = TRAVIS TRAVIS_*
 8 | deps =
 9 |     coverage
10 |     coveralls
11 | commands =
12 |     coverage run --branch --source=skt -m unittest discover tests
13 |     coverage report -m
14 |     coveralls
15 | install_command=pip install {opts} {packages} .[dev] https://gitlab.com/cki-project/cki-lib/-/archive/master/cki-lib-master.zip https://gitlab.com/cki-project/datadefinition/-/archive/master/datadefinition-master.zip
16 | 
17 | [testenv:flake8]
18 | passenv = CI TRAVIS TRAVIS_*
19 | basepython =
20 |     python3.7
21 | commands =
22 |     flake8 --show-source .
23 | 
24 | [testenv:pylint]
25 | passenv = CI TRAVIS TRAVIS_*
26 | basepython =
27 |     python3.7
28 | commands =
29 |     # Disable R0801 in pylint that checks for duplicate content in multiple
30 |     # files. See https://github.com/PyCQA/pylint/issues/214 for details.
31 |     pylint -d R0801 --ignored-classes=responses tests
32 | 


--------------------------------------------------------------------------------
/tests/assets/beaker_recipe_set_results.xml:
--------------------------------------------------------------------------------
 1 | <recipeSet id="1234">
 2 |   <recipe id="5678" system='machine.beaker.org' result='Pass' status='Completed'>
 3 |     <hostRequires>
 4 |       <and>
 5 |         <arch op="=" value="x86_64"/>
 6 |       </and>
 7 |     </hostRequires>
 8 |     <logs>
 9 |       <log name='console.log' href="http://example.com/">
10 |         TEST RESULT
11 |       </log>
12 |     </logs>
13 |     <task name='/test/misc/machineinfo' result='Pass' status='Completed'>
14 |       <logs>
15 |         <log name='machinedesc.log' href="http://example.com/machinedesc.log">
16 |         </log>
17 |         <log name='lshw.log' href="http://example.com/lshw.log">
18 |         </log>
19 |       </logs>
20 |     </task>
21 |     <task name='/distribution/kpkginstall' result='Pass' status='Completed'>
22 |       <fetch url="https://github.com/CKI-project/tests-beaker/archive/master.zip#distribution/kpkginstall"/>
23 |     </task>
24 |     <task name='/test/we/ran' result='Pass' status='Completed' />
25 |   </recipe>
26 | </recipeSet>
27 | 


--------------------------------------------------------------------------------
/tests/assets/beaker_results.xml:
--------------------------------------------------------------------------------
 1 | <job id='101010' result='Fail' status='Completed'>
 2 |   <whiteboard>
 3 |     skt 4.17.0-rc1+ 1234567890.tar.gz [noavc] [noselinux]
 4 |   </whiteboard>
 5 |   <recipeSet id="123">
 6 |     <recipe result='Warn' system='machine.beaker.org' id="456">
 7 |       <hostRequires>
 8 |       </hostRequires>
 9 |       <logs>
10 |         <log name='console.log' href="http://example.com/">
11 |           TEST RESULT
12 |         </log>
13 |       </logs>
14 |       <task name="/test/misc/boottest" result="Fail">
15 |                 <params>
16 |           <param name="CKI_WAIVED" value="true"/>
17 |         </params>
18 |         <fetch url="kpkginstall"/></task>
19 |       <task name='/test/misc/machineinfo' result="Fail">
20 |         <params>
21 |           <param name="CKI_WAIVED" value="true"/>
22 |         </params>
23 |         <logs>
24 |           <log name='machinedesc.log' href="http://example.com/machinedesc.log">
25 |           </log>
26 |           <log name='lshw.log' href="http://example.com/lshw.log">
27 |           </log>
28 |         </logs>
29 |       </task>
30 |     </recipe>
31 |   </recipeSet>
32 | </job>
33 | 


--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
 1 | # Changelog for skt
 2 | 
 3 | ## June 2019
 4 | 
 5 | * All commands except for `skt run` dropped; this has been replaced with
 6 |   GitLab pipeline yaml. SKT serves only to submit and watch Beaker jobs.
 7 | 
 8 | ## June 2018
 9 | 
10 | * Arguments for `skt merge` should now be used multiple times instead of
11 |   supplying multiple values for each option. This affects the following
12 |   arguments:
13 | 
14 |   * `--patch` (formerly `--patchlist`)
15 |   * `--pw`
16 |   * `--merge-ref`
17 | 
18 |   Example: `skt merge --pw URL1 --pw URL2 --pw URL3`
19 | 
20 | * Arguments for reporter are now explicitly defined and JSON strings are no
21 |   longer required. Users can specify the following arguments:
22 | 
23 |   * `--reporter`: type of reporter to use (`stdio` and `mail` supported)
24 |     *(required)*
25 |   * `--mail-from`: email address of the sender *(required)*
26 |   * `--mail-to`: email address of the recipient *(required)*
27 |   * `--mail-subject`: subject line of the email *(optional)*
28 |   * `--mail-header`: additional header to add to the email *(optional)*
29 | 
30 |   The `--mail-to` and `--mail-header` arguments can be specified more than once to select multiple recipients or add multiple headers.
31 | 


--------------------------------------------------------------------------------
/tests/assets/actual_rc.cfg:
--------------------------------------------------------------------------------
 1 | [state]
 2 | workdir = /builds/cki-project/cki-pipeline/workdir
 3 | git_url = git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable-rc.git
 4 | stage_merge = pass
 5 | make_target = targz-pkg
 6 | commit_message_title = Linux 5.4.11-rc1
 7 | tag = -ed7e2ec
 8 | kernel_type = upstream
 9 | test_hash = ed7e2ecd1b93e8f35aa93d1bd185bf9a4ad1e1dd
10 | kernel_arch = x86_64
11 | stage_build = pass
12 | kernel_version = 5.4.11-rc1-ed7e2ec.cki
13 | make_opts = -j30 INSTALL_MOD_STRIP=1 targz-pkg
14 | build_job_url = https://xci32.lab.eng.rdu2.redhat.com/cki-project/cki-pipeline/-/jobs/553679
15 | tarball_file = kernel-stable-x86_64-ed7e2ecd1b93e8f35aa93d1bd185bf9a4ad1e1dd.tar.gz
16 | config_file = kernel-stable-x86_64-ed7e2ecd1b93e8f35aa93d1bd185bf9a4ad1e1dd.config
17 | kernel_package_url = https://artifacts.cki-project.org/pipelines/377632/kernel-stable-x86_64-ed7e2ecd1b93e8f35aa93d1bd185bf9a4ad1e1dd.tar.gz
18 | kernel_config_url = https://artifacts.cki-project.org/pipelines/377632/kernel-stable-x86_64-ed7e2ecd1b93e8f35aa93d1bd185bf9a4ad1e1dd.config
19 | debug_kernel = no
20 | stage_setup = pass
21 | retcode = 0
22 | 
23 | 
24 | [runner]
25 | jobtemplate = beaker.xml
26 | blacklist = beaker-blacklist.txt
27 | 


--------------------------------------------------------------------------------
/setup.cfg:
--------------------------------------------------------------------------------
 1 | [metadata]
 2 | name = skt
 3 | description = "Sonic Kernel Testing"
 4 | long_description = file: README.md
 5 | version = 1
 6 | author = Red Hat, Inc.
 7 | license = GPLv2+
 8 | 
 9 | [options]
10 | # Automatically find all files beneath the skt directory and include them.
11 | packages = find:
12 | # Parse the MANIFEST.in file and include those files, too.
13 | include_package_data = True
14 | # Let pip install dependencies automatically.
15 | install_requires = jinja2>=2.10
16 |                    defusedxml
17 |                    requests
18 |                    enum34
19 | 
20 | # The beaker-client package breaks some Python packaging rules and tries to
21 | # store content in the system /etc directory. This causes a Sandbox violation
22 | # in Circle-CI.
23 | [options.extras_require]
24 | beaker = beaker-client
25 |          python-krbV
26 | dev = pylint
27 |       flake8
28 |       mock
29 |       responses
30 | 
31 | [options.entry_points]
32 | # Set up an executable 'skt' that calls the main() function in
33 | # skt/executable.py.
34 | console_scripts =
35 |     skt = skt.executable:main
36 | 
37 | [options.packages.find]
38 | # Don't include the /tests directory when we search for python files.
39 | exclude =
40 |     tests
41 | 


--------------------------------------------------------------------------------
/tests/assets/beaker_recipe_set_fail_results.xml:
--------------------------------------------------------------------------------
 1 | <recipeSet>
 2 |   <recipe system='machine.beaker.org' result='Fail' status='Completed'>
 3 |     <hostRequires>
 4 |       <and>
 5 |         <arch op="=" value="x86_64"/>
 6 |       </and>
 7 |     </hostRequires>
 8 |     <logs>
 9 |       <log name='console.log' href="http://example.com/">
10 |         TEST RESULT
11 |       </log>
12 |     </logs>
13 |     <task name='/test/misc/machineinfo' result='Pass' status='Completed'>
14 |       <logs>
15 |         <log name='machinedesc.log' href="http://example.com/machinedesc.log">
16 |         </log>
17 |         <log name='lshw.log' href="http://example.com/lshw.log">
18 |         </log>
19 |       </logs>
20 |     </task>
21 |     <task name='/distribution/kpkginstall' result='Pass' status='Completed'>
22 |       <fetch url="https://github.com/CKI-project/tests-beaker/archive/master.zip#distribution/kpkginstall"/>
23 |     </task>
24 |     <task name='/a/failed/waived/test' result='Fail' status='Completed'>
25 |       <params>
26 |         <param name="CKI_WAIVED" value="true"/>
27 |       </params>
28 |     </task>
29 |     <task name='/a/passed/waived/test' result='Pass' status='Completed'>
30 |       <params>
31 |         <param name="CKI_WAIVED" value="true"/>
32 |       </params>
33 |     </task>
34 |     <task name='/a/test/with/no/waive' result='Fail' status='Completed'></task>
35 |   </recipe>
36 | </recipeSet>
37 | 


--------------------------------------------------------------------------------
/skt/misc.py:
--------------------------------------------------------------------------------
 1 | # Copyright (c) 2017-2020 Red Hat, Inc. All rights reserved. This copyrighted
 2 | # material is made available to anyone wishing to use, modify, copy, or
 3 | # redistribute it subject to the terms and conditions of the GNU General
 4 | # Public License v.2 or later.
 5 | #
 6 | # This program is distributed in the hope that it will be useful, but WITHOUT
 7 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 8 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
 9 | # details.
10 | #
11 | # You should have received a copy of the GNU General Public License
12 | # along with this program; if not, write to the Free Software Foundation, Inc.,
13 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
14 | """Functions and constants used by multiple parts of skt."""
15 | 
16 | # SKT Result
17 | SKT_SUCCESS = 0
18 | SKT_FAIL = 1
19 | SKT_ERROR = 2
20 | SKT_BOOT = 3
21 | 
22 | 
23 | def is_task_waived(task):
24 |     """ Check XML param to see if the test is waived.
25 |         Args:
26 |             task: xml node
27 | 
28 |         Returns: True if test is waived, otherwise False
29 |     """
30 |     is_task_waived_val = False
31 |     for param in task.findall('.//param'):
32 |         try:
33 |             if param.attrib.get('name').lower() == 'cki_waived' and \
34 |                     param.attrib.get('value').lower() == 'true':
35 |                 is_task_waived_val = True
36 |                 break
37 |         except ValueError:
38 |             pass
39 | 
40 |     return is_task_waived_val
41 | 


--------------------------------------------------------------------------------
/tests/assets/test.xml:
--------------------------------------------------------------------------------
 1 | <job>
 2 |   <whiteboard>skt 3.10.xx [noavc] [noselinux]</whiteboard>
 3 |   <recipeSet>
 4 |     <recipe ks_meta="harness='restraint-rhts beakerlib-redhat'" kernel_options="selinux=0" whiteboard="3.10.xx">
 5 |       <distroRequires>
 6 |         <and>
 7 |           <distro_family op="=" value="Fedora26"/>
 8 |           <distro_tag op="=" value="RELEASED"/>
 9 |           <distro_variant op="=" value="Server"/>
10 |           <distro_arch op="=" value="x86_64"/>
11 |         </and>
12 |       </distroRequires>
13 |       <hostRequires>
14 |         <and>
15 |           <arch op="=" value="x86_64"/>
16 |         </and>
17 |       </hostRequires>
18 |       <repos/>
19 |       <partitions/>
20 |       <ks_appends/>
21 |       <task name="/distribution/install" role="STANDALONE">
22 |         <params/>
23 |       </task>
24 |       <task name="/distribution/command" role="None">
25 |         <params>
26 |           <param name="CMDS_TO_RUN" value="rm /usr/share/restraint/plugins/report_result.d/10_avc_check"/>
27 |         </params>
28 |       </task>
29 |         <task name="/test/misc/machineinfo">
30 |         <fetch url="https://github.com/RH-FMK/tests-beaker.git?master#test/misc/machineinfo"/>
31 |       </task>
32 |       <task name="/distribution/kpkginstall" role="STANDALONE">
33 |         <fetch url="https://github.com/RH-FMK/tests-beaker.git?master#distribution/kpkginstall"/>
34 |         <params>
35 |           <param name="KPKG_URL" value="##KPKG_URL##"/>
36 |           <param name="KVER" value="3.10.xx"/>
37 |         </params>
38 |       </task>
39 |     </recipe>
40 |   </recipeSet>
41 | </job>
42 | 


--------------------------------------------------------------------------------
/tests/assets/beaker_wait_pass.xml:
--------------------------------------------------------------------------------
 1 | <job id="0001">
 2 |   <whiteboard>
 3 |     skt 4.17.0-rc1+ 1234567890.tar.gz [noavc] [noselinux]
 4 |   </whiteboard>
 5 |   <recipeSet>
 6 |     <recipe result='Pass' status='Completed' id="123">
 7 |       <logs>
 8 |         <log name='console.log' href="http://example.com/">
 9 |           TEST RESULT
10 |         </log>
11 |       </logs>
12 |       <task name='/test/misc/machineinfo' result="Warn">
13 |         <logs>
14 |           <log name='machinedesc.log' href="http://example.com/machinedesc.log">
15 |           </log>
16 |           <log name='lshw.log' href="http://example.com/lshw.log">
17 |           </log>
18 |         </logs>
19 |       </task>
20 |       <task name='/distribution/install' result='Pass'>
21 |       </task>
22 |     </recipe>
23 |   </recipeSet>
24 |   <recipeSet>
25 |     <recipe result='Pass' status='Completed' id="456456">
26 |       <logs>
27 |         <log name='console.log' href="http://example.com/">
28 |           TEST RESULT
29 |         </log>
30 |       </logs>
31 |       <task name='/test/misc/machineinfo'>
32 |         <logs>
33 |           <log name='machinedesc.log' href="http://example.com/machinedesc.log">
34 |           </log>
35 |           <log name='lshw.log' href="http://example.com/lshw.log">
36 |           </log>
37 |         </logs>
38 |       </task>
39 |       <task name='/distribution/install' result='Pass'>
40 |       </task>
41 |     </recipe>
42 |   </recipeSet>
43 |     <recipeSet>
44 |     <recipe result='Pass' status='Waiting' id="123123">
45 |       <logs>
46 |         <log name='console.log' href="http://example.com/">
47 |           TEST RESULT
48 |         </log>
49 |       </logs>
50 | 
51 |       <task name='/distribution/install' result='Pass' />
52 |       <task name='/test/we/ran' result="Pass">
53 |         <params>
54 |           <param name="CKI_WAIVED" value="true"/>
55 |         </params>
56 |       </task>
57 | 
58 |     </recipe>
59 |   </recipeSet>
60 | </job>
61 | 


--------------------------------------------------------------------------------
/tests/assets/beaker_aborted_some.xml:
--------------------------------------------------------------------------------
 1 | <job id="0001">
 2 |   <whiteboard>
 3 |     skt 4.17.0-rc1+ 1234567890.tar.gz [noavc] [noselinux]
 4 |   </whiteboard>
 5 |   <recipeSet id="123">
 6 |     <recipe result='Warn' status='Aborted' id="123">
 7 |       <logs>
 8 |         <log name='console.log' href="http://example.com/">
 9 |           TEST RESULT
10 |         </log>
11 |       </logs>
12 |       <task name='/test/misc/machineinfo' result="Warn">
13 |         <params>
14 |           <param name="CKI_WAIVED" value="true"/>
15 |         </params>
16 |         <logs>
17 |           <log name='machinedesc.log' href="http://example.com/machinedesc.log">
18 |           </log>
19 |           <log name='lshw.log' href="http://example.com/lshw.log">
20 |           </log>
21 |         </logs>
22 |       </task>
23 |       <task name='/distribution/install' result='Fail'>
24 |       </task>
25 |     </recipe>
26 |   </recipeSet>
27 |   <recipeSet id="456">
28 |     <recipe result='Pass' status='Completed' id="456456">
29 |       <logs>
30 |         <log name='console.log' href="http://example.com/">
31 |           TEST RESULT
32 |         </log>
33 |       </logs>
34 |       <task name='/test/misc/machineinfo'>
35 |         <params>
36 |           <param name="CKI_WAIVED" value="true"/>
37 |         </params>
38 |         <logs>
39 |           <log name='machinedesc.log' href="http://example.com/machinedesc.log">
40 |           </log>
41 |           <log name='lshw.log' href="http://example.com/lshw.log">
42 |           </log>
43 |         </logs>
44 |       </task>
45 |       <task name='/distribution/install' result='Fail'>
46 |       </task>
47 |     </recipe>
48 |   </recipeSet>
49 |     <recipeSet id="789">
50 |     <recipe result='Warn' status='Waiting' id="123123">
51 |       <logs>
52 |         <log name='console.log' href="http://example.com/">
53 |           TEST RESULT
54 |         </log>
55 |       </logs>
56 |       <task name='/test/misc/machineinfo'>
57 |         <params>
58 |           <param name="CKI_WAIVED" value="true"/>
59 |         </params>
60 |         <logs>
61 |           <log name='machinedesc.log' href="http://example.com/machinedesc.log">
62 |           </log>
63 |           <log name='lshw.log' href="http://example.com/lshw.log">
64 |           </log>
65 |         </logs>
66 |       </task>
67 |       <task name='/distribution/install' result='Fail'>
68 |       </task>
69 |     </recipe>
70 |   </recipeSet>
71 | </job>
72 | 


--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
 1 | # Contributing guidelines
 2 | 
 3 | This file provides guidance for developers and code reviewers that work on
 4 | `skt`.
 5 | 
 6 | ## Bugs
 7 | 
 8 | Please report all bugs using [GitHub
 9 | Issues](https://github.com/RH-FMK/skt/issues/new) within the `skt`
10 | repository.
11 | 
12 | ## Submitting patches
13 | 
14 | All patches should be submitted via GitHub's pull requests. When submitting
15 | a patch, please do the following:
16 | 
17 | * Limit the first line of your commit message to 50 characters
18 | * Describe the bug you found or the feature that is missing from the project
19 | * Describe how your patch fixes the bug or improves the project
20 | * Make sure the code keeps working and making sense after each commit, and
21 |   does not require any further commits to fix it.
22 | * Make sure each commit contains only one complete logical change, no more, no
23 |   less.
24 | * Always document each new module, class, or function.
25 | * Add or update the documentation when the logic or the interface of an
26 |   existing module, class, or function changes.
27 | * Monitor the results of the CI jobs when you submit the pull request and fix
28 |   any issues found in those tests
29 | * Read review comments carefully, discuss the points you disagree with,
30 |   and address all outstanding comments with each respin, so the comments are
31 |   not lost and there's no backtracking.
32 | * Reply to review comments and update the patches quickly, preferably within
33 |   one day, so the reviewer has fresh memory of the code, and review finishes
34 |   sooner.
35 | 
36 | Code quality guidelines are available in the
37 | [rh-fmk/meta repository](https://github.com/RH-FMK/meta/blob/master/CODING.md).
38 | 
39 | ## Reviewing patches
40 | 
41 | Code reviewers must maintain the code quality within the project and review
42 | patches on a regular basis. Reviewers should:
43 | 
44 | * Strive to provide timely feedback for patches and respins, preferably
45 |   replying within a day.
46 | * Feedback should be constructive (*"I would suggest that you..."* rather
47 |   than *"I don't like this"*)
48 | * Identify areas for improvement, especially with test coverage
49 | * Test the changes being reviewed with each respin.
50 | * Provide as complete feedback as possible with each respin to minimize
51 |   number of iterations.
52 | * Focus on getting changes into a "good enough" shape and merged sooner, but
53 |   describe desirable improvements to be required for further submissions.
54 | * Stay on topic of the changes and improvements to minimize stray
55 |   conversations and unnecessary argument.
56 | 


--------------------------------------------------------------------------------
/tests/test_executable.py:
--------------------------------------------------------------------------------
  1 | # Copyright (c) 2018-2019 Red Hat, Inc. All rights reserved. This copyrighted
  2 | # material is made available to anyone wishing to use, modify, copy, or
  3 | # redistribute it subject to the terms and conditions of the GNU General Public
  4 | # License v.2 or later.
  5 | #
  6 | # This program is distributed in the hope that it will be useful, but WITHOUT
  7 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  8 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
  9 | # details.
 10 | #
 11 | # You should have received a copy of the GNU General Public License along with
 12 | # this program; if not, write to the Free Software Foundation, Inc., 51
 13 | # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 14 | """Test cases for runner module."""
 15 | import logging
 16 | import os
 17 | import signal
 18 | import threading
 19 | import time
 20 | import unittest
 21 | 
 22 | import mock
 23 | from rcdefinition.rc_data import SKTData
 24 | 
 25 | from skt import executable
 26 | from skt.runner import BeakerRunner
 27 | from tests import misc
 28 | 
 29 | ASSETS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'assets')
 30 | RC_EXAMPLE = open(os.path.join(ASSETS_DIR, 'actual_rc.cfg')).read()
 31 | 
 32 | 
 33 | def trigger_signal():
 34 |     """ Send SIGTERM to self after 2 seconds."""
 35 |     time.sleep(2)
 36 |     pid = os.getpid()
 37 |     os.kill(pid, signal.SIGTERM)
 38 | 
 39 | 
 40 | class TestExecutable(unittest.TestCase):
 41 |     """Test cases for executable module."""
 42 | 
 43 |     def setUp(self) -> None:
 44 |         self.myrunner = BeakerRunner(**misc.DEFAULT_ARGS)
 45 | 
 46 |     def test_full_path_relative(self):
 47 |         """Verify that full_path() expands a relative path."""
 48 |         filename = "somefile"
 49 |         result = executable.full_path(filename)
 50 |         expected_path = "{}/{}".format(os.getcwd(), filename)
 51 |         self.assertEqual(expected_path, result)
 52 | 
 53 |     def test_full_path_user_directory(self):
 54 |         """Verify that full_path() expands a user directory path."""
 55 |         filename = "somefile"
 56 |         result = executable.full_path("~/{}".format(filename))
 57 |         expected_path = "{}/{}".format(os.path.expanduser('~'), filename)
 58 |         self.assertEqual(expected_path, result)
 59 | 
 60 |     def test_setup_logging(self):
 61 |         """Ensure that setup_logging works and sets-up to what we expect."""
 62 |         verbose = False
 63 |         executable.setup_logging(verbose)
 64 | 
 65 |         requests_lgr = logging.getLogger('requests')
 66 |         urllib3_lgr = logging.getLogger('urllib3')
 67 | 
 68 |         requests_level = logging.getLevelName(requests_lgr.getEffectiveLevel())
 69 |         urllib3_level = logging.getLevelName(urllib3_lgr.getEffectiveLevel())
 70 | 
 71 |         self.assertEqual(requests_level, 'WARNING')
 72 |         self.assertEqual(urllib3_level, 'WARNING')
 73 | 
 74 |         current_logger = logging.getLogger('executable')
 75 |         self.assertEqual(current_logger.getEffectiveLevel(), logging.WARNING -
 76 |                          (verbose * 10))
 77 | 
 78 |     @mock.patch('skt.executable.BeakerRunner')
 79 |     @mock.patch('builtins.open', create=True)
 80 |     @mock.patch('subprocess.Popen')
 81 |     @mock.patch('logging.error')
 82 |     @mock.patch('subprocess.call')
 83 |     @mock.patch('skt.runner.BeakerRunner._BeakerRunner__jobsubmit')
 84 |     def test_cleanup_called(self, mock_jobsubmit, mock_call, mock_log_err,
 85 |                             mock_popen, mock_open, mock_runner):
 86 |         """Ensure BeakerRunner.signal_handler works."""
 87 |         # pylint: disable=W0613,R0913
 88 |         mock_runner.return_value = self.myrunner
 89 | 
 90 |         mock_jobsubmit.return_value = "J:0001"
 91 | 
 92 |         mock_call.return_value = 0
 93 |         mock_popen.return_value = 0
 94 | 
 95 |         thread = threading.Thread(target=trigger_signal)
 96 | 
 97 |         try:
 98 |             thread.start()
 99 |             # it's fine to call this directly, no need to mock
100 |             skt_data = SKTData.deserialize(RC_EXAMPLE)
101 | 
102 |             executable.cmd_run(skt_data)
103 |             thread.join()
104 |         except (KeyboardInterrupt, SystemExit):
105 |             logging.info('Thread cancelling...')
106 | 
107 |         self.assertTrue(executable.cmd_run.cleanup_done)
108 | 


--------------------------------------------------------------------------------
/tests/misc.py:
--------------------------------------------------------------------------------
  1 | # Copyright (c) 2018-2019 Red Hat, Inc. All rights reserved. This copyrighted
  2 | # material is made available to anyone wishing to use, modify, copy, or
  3 | # redistribute it subject to the terms and conditions of the GNU General Public
  4 | # License v.2 or later.
  5 | #
  6 | # This program is distributed in the hope that it will be useful, but WITHOUT
  7 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  8 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
  9 | # details.
 10 | #
 11 | # You should have received a copy of the GNU General Public License along with
 12 | # this program; if not, write to the Free Software Foundation, Inc., 51
 13 | # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 14 | """Miscellaneous for tests."""
 15 | import os
 16 | 
 17 | import mock
 18 | from defusedxml.ElementTree import fromstring
 19 | 
 20 | ASSETS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'assets')
 21 | SCRIPT_PATH = os.path.dirname(__file__)
 22 | DEFAULT_ARGS = {
 23 |     'jobtemplate': '{}/assets/test.xml'.format(SCRIPT_PATH)
 24 | }
 25 | INV_TEMPLATE_ARGS = {
 26 |     'jobtemplate': '{}/assets/0.xml'.format(SCRIPT_PATH)
 27 | }
 28 | 
 29 | 
 30 | def get_asset_path(filename):
 31 |     """Return the absolute path of an asset passed as parameter.
 32 | 
 33 |     Args:
 34 |         filename: Asset's filename.
 35 |     Returns:
 36 |         The absolute path of the corresponding asset.
 37 |     """
 38 |     return os.path.join(ASSETS_DIR, filename)
 39 | 
 40 | 
 41 | def get_asset_content(filename):
 42 |     """Return the content of an asset passed as parameter.
 43 | 
 44 |     Args:
 45 |         filename: Asset's filename.
 46 |     Returns:
 47 |         The content of the corresponding asset.
 48 |     """
 49 |     with open(get_asset_path(filename)) as asset:
 50 |         return asset.read()
 51 | 
 52 | 
 53 | def fake_cancel_pending_jobs(sself):
 54 |     """Cancel pending job without calling 'bkr cancel'.
 55 | 
 56 |     Args:
 57 |         sself: BeakerRunner
 58 |     Returns:
 59 |         None
 60 |     """
 61 |     # pylint: disable=protected-access,
 62 |     for job_id in set(sself.job_to_recipe_set_map):
 63 |         sself._BeakerRunner__forget_taskspec(job_id)
 64 | 
 65 | 
 66 | def exec_on(myrunner, mock_jobsubmit, xml_asset_file, max_aborted,
 67 |             alt_state=None):
 68 |     """Simulate getting live results from Beaker.
 69 |     Feed skt/runner with an XML and change it after a couple of runs.
 70 | 
 71 |     Args:
 72 |         myrunner:       BeakerRunner
 73 |         mock_jobsubmit: mock object for __jobsubmit
 74 |         xml_asset_file: xml filename to use
 75 |         max_aborted:    Maximum number of allowed aborted jobs. Abort the
 76 |                         whole stage if the number is reached.
 77 |         alt_state:      if set, represents a state to transition the job to
 78 |     Returns:
 79 |         BeakerRunner run() result
 80 | 
 81 |     """
 82 |     # pylint: disable=W0613,too-many-arguments
 83 |     def fake_getresultstree(sself, taskspec):
 84 |         """Fakt getresultstree. Change state of last recipe on 3rd loop.
 85 | 
 86 |         Args:
 87 |              sself:    BeakerRunner
 88 |              taskspec: ID of the job, recipe or recipe set.
 89 |         Returns:
 90 |             xml root
 91 |         """
 92 | 
 93 |         if alt_state:
 94 |             if fake_getresultstree.run_count > 2:
 95 |                 result = fromstring(get_asset_content(xml_asset_file))
 96 | 
 97 |                 recipe = result.findall('.//recipe')[-1]
 98 |                 recipe.attrib['status'] = alt_state
 99 | 
100 |                 sself.recipe_set_results[taskspec] = result
101 |                 return result
102 | 
103 |             fake_getresultstree.run_count += 1
104 | 
105 |         result = fromstring(get_asset_content(xml_asset_file))
106 |         sself.recipe_set_results[taskspec] = result
107 |         return result
108 | 
109 |     fake_getresultstree.run_count = 1
110 |     # fake cancel_pending_jobs so 'bkr cancel' isn't run
111 |     mock1 = mock.patch('skt.runner.BeakerRunner.cancel_pending_jobs',
112 |                        fake_cancel_pending_jobs)
113 |     mock1.start()
114 | 
115 |     # fake getresultstree, so we can change XML input during testrun
116 |     mock2 = mock.patch('skt.runner.BeakerRunner.getresultstree',
117 |                        fake_getresultstree)
118 |     mock2.start()
119 | 
120 |     url = "http://machine1.example.com/builds/1234567890.tar.gz"
121 |     release = "4.17.0-rc1"
122 |     wait = True
123 | 
124 |     mock_jobsubmit.return_value = "J:0001"
125 | 
126 |     # no need to wait 60 seconds
127 |     # though beaker_pass_results.xml only needs one iteration
128 |     myrunner.watchdelay = 0.01
129 | 
130 |     result = myrunner.run(url, max_aborted, release, wait)
131 | 
132 |     mock1.stop()
133 |     mock2.stop()
134 |     return result
135 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | skt - sonic kernel testing
  2 | ==========================
  3 | 
  4 | THIS PROJECT HAS MOVED TO https://gitlab.com/cki-project/skt/.
  5 | 
  6 | [![Travis CI Build Status][travis_badge]][travis_page]
  7 | [![Test Coverage Status][coveralls_badge]][coveralls_page]
  8 | 
  9 | Skt is a tool for monitoring Beaker jobs and resubmitting them.
 10 | 
 11 | Dependencies
 12 | ------------
 13 | 
 14 | Install dependencies needed for running skt like this:
 15 | 
 16 |     sudo dnf install -y python3 beaker-client
 17 | 
 18 | Extra dependencies needed for running the testsuite:
 19 | 
 20 |     sudo dnf install -y python3-mock
 21 | 
 22 | Run tests
 23 | ---------
 24 | 
 25 | To run all tests execute:
 26 | 
 27 |     python3 -m unittest discover tests
 28 | 
 29 | To run some specific tests, you can execute a specific test like this:
 30 | 
 31 |     python3 -m unittest tests.test_runner
 32 | 
 33 | 
 34 | Installation
 35 | ------------
 36 | 
 37 | Install `skt` directly from git:
 38 | 
 39 |     pip install git+https://github.com/RH-FMK/skt
 40 | 
 41 | If support for beaker is required, install ``skt`` with the ``beaker``
 42 | extras:
 43 | 
 44 |     pip install git+https://github.com/rh-fmk/skt.git#egg-project[beaker]
 45 | 
 46 | Test the `skt` executable by printing the help text:
 47 | 
 48 |     skt -h
 49 | 
 50 | Usage
 51 | -----
 52 | 
 53 | The `skt` tool implements several "commands", and each of those accepts its
 54 | own command-line options and arguments. However there are also several
 55 | "global" command-line options, shared by all the commands. To get a summary of
 56 | the global options and available commands, run `skt --help`. To get a
 57 | summary of particular command's options and arguments, run `skt <COMMAND>
 58 | --help`, where `<COMMAND>` would be the command of interest.
 59 | 
 60 | Most of command-line options can also be read by `skt` from its configuration
 61 | file, which is specified using the global `--rc` command-line option. However,
 62 | there are some command-line options which cannot be stored in the configuration
 63 | file, and there are some options read from the configuration file by some `skt`
 64 | commands, which cannot be passed via the command line. Some of the latter are
 65 | required for operation.
 66 | 
 67 | Most `skt` commands can write their state to the configuration file as they
 68 | work, so that the other commands can take the workflow task over from them.
 69 | Some commands can receive that state from the command line, via options, but
 70 | some require some information stored in the configuration file. For this
 71 | reason, to support a complete workflow, it is necessary to always make the
 72 | commands transfer their state via the configuration file.
 73 | 
 74 | To separate the actual configuration from the specific workflow's state, and
 75 | to prevent separate tasks from interfering with each other, you can store your
 76 | configuration in a separate (e.g. read-only) file, copy it to a new file each
 77 | time you want to do something, then discard the file after the task is
 78 | complete. Note that reusing a configuration file with state added can break
 79 | some commands in unexpected ways. That includes repeating a previous command
 80 | after the next command in the workflow has already ran.
 81 | 
 82 | The following commands are supported by `skt`:
 83 | 
 84 | * `run`
 85 |     - Run tests on a built kernel using the specified "runner". Only
 86 |       "Beaker" runner is currently supported. This command expects `publish`
 87 |       command to have completed succesfully.
 88 | 
 89 | Currently, skt is being used only to monitor Beaker test results. Section below
 90 | describes this.
 91 | 
 92 | All the following commands use the `-vv` option to increase verbosity of the
 93 | command's output, so it's easier to debug problems. Remove the option for
 94 | quieter, shorter output.
 95 | 
 96 | ### Run
 97 | 
 98 | To run the tests you will need access to a
 99 | [Beaker](https://beaker-project.org/) instance configured to the point where
100 | `bkr whoami` completes successfully. You will also need Beaker job XML file,
101 |  which runs the tests. 
102 | Below is an example of this file. Note that it won't work as is.
103 | 
104 | ```XML
105 | <job>
106 |   <whiteboard>skt kernel-version</whiteboard>
107 |   <recipeSet>
108 |     <recipe whiteboard="kernel-version">
109 |       <distroRequires>
110 |         <and>
111 |           <distro_family op="=" value="Fedora26"/>
112 |           <distro_tag op="=" value="RELEASED"/>
113 |           <distro_variant op="=" value="Server"/>
114 |           <distro_arch op="=" value="x86_64"/>
115 |         </and>
116 |       </distroRequires>
117 |       <hostRequires>
118 |         <and>
119 |           <arch op="=" value="x86_64"/>
120 |         </and>
121 |       </hostRequires>
122 |       <repos/>
123 |       <partitions/>
124 |       <ks_appends/>
125 |       <task name="/distribution/install" role="STANDALONE"/>
126 |       <task name="/distribution/kpkginstall" role="STANDALONE">
127 |         <params>
128 |           <param name="KPKG_URL" value="http://url_to_kernel"/>
129 |           <param name="KVER" value="kernel-version"/>
130 |         </params>
131 |       </task>
132 |     </recipe>
133 |   </recipeSet>
134 | </job>
135 | ```
136 | 
137 | Provided you have both Beaker access and a suitable job XML file, you can
138 | run the tests with the built kernel as such:
139 | 
140 |     skt --rc <SKTRC> --state --workdir <WORKDIR> -vv run --wait
141 | 
142 | The `<SKTRC>` is a config file with contents like this:
143 | 
144 | [runner]
145 | jobtemplate=beaker.xml
146 | jobowner=username
147 | blacklist=beaker-blacklist.txt
148 | 
149 | Here, `<jobtemplate>` is the name of the file with the Beaker job XML
150 | file. If you remove the `--wait` option, the command will return once the
151 | job was submitted. Otherwise it will wait for its completion and report the
152 | result.
153 | 
154 | In case running on specific hosts is not desired, one can use a simple text
155 | file containing one hostname per line, and pass the file via `blacklist`
156 | parameter. Tests will not attempt to run on machines which names are specified
157 | in the file. This is useful for example as a temporary fix in case the hardware
158 | is buggy and the maintainer of the pool doesn't have time to exclude it from
159 | the pool.
160 | 
161 | Developer Guide
162 | ---------------
163 | 
164 | Developers can test changes to `skt` by using "development mode" from python's
165 | `setuptools` package. First, `cd` to the directory where `skt` is cloned and
166 | run:
167 | 
168 |     pip install --user -e .
169 | 
170 | This installs `skt` in a mode where any changes within the repo are
171 | immediately available simply by running `skt`. There is no need to repeatedly
172 | run `pip install .` after each change.
173 | 
174 | Using a virtual environment is highly recommended. This keeps `skt` and all
175 | its dependencies in a separate Python environment. Developers can build a
176 | virtual environment for skt quickly:
177 | 
178 |     virtualenv ~/skt-venv/
179 |     source ~/skt-venv/bin/activate
180 |     pip install -e .
181 | 
182 | To deactivate the virtual environment, simply run `deactivate`.
183 | 
184 | License
185 | -------
186 | skt is distributed under GPLv2 license.
187 | 
188 | This program is free software: you can redistribute it and/or modify
189 | it under the terms of the GNU General Public License as published by
190 | the Free Software Foundation, either version 2 of the License, or
191 | (at your option) any later version.
192 | 
193 | This program is distributed in the hope that it will be useful,
194 | but WITHOUT ANY WARRANTY; without even the implied warranty of
195 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
196 | GNU General Public License for more details.
197 | 
198 | You should have received a copy of the GNU General Public License
199 | along with this program. If not, see <http://www.gnu.org/licenses/>.
200 | 
201 | [travis_badge]: https://travis-ci.org/RH-FMK/skt.svg?branch=master
202 | [travis_page]: https://travis-ci.org/RH-FMK/skt
203 | [coveralls_badge]: https://coveralls.io/repos/github/RH-FMK/skt/badge.svg?branch=master
204 | [coveralls_page]: https://coveralls.io/github/RH-FMK/skt?branch=master
205 | 


--------------------------------------------------------------------------------
/skt/executable.py:
--------------------------------------------------------------------------------
  1 | # Copyright (c) 2017-2020 Red Hat, Inc. All rights reserved. This copyrighted
  2 | # material is made available to anyone wishing to use, modify, copy, or
  3 | # redistribute it subject to the terms and conditions of the GNU General
  4 | # Public License v.2 or later.
  5 | #
  6 | # This program is distributed in the hope that it will be useful, but WITHOUT
  7 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  8 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
  9 | # details.
 10 | #
 11 | # You should have received a copy of the GNU General Public License
 12 | # along with this program; if not, write to the Free Software Foundation, Inc.,
 13 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 14 | """SKT entry point and argument parsing."""
 15 | import argparse
 16 | import atexit
 17 | import itertools
 18 | import logging
 19 | import os
 20 | import signal
 21 | import sys
 22 | import tempfile
 23 | 
 24 | from rcdefinition.rc_data import SKTData
 25 | 
 26 | from skt.misc import SKT_ERROR
 27 | from skt.runner import BeakerRunner
 28 | 
 29 | LOGGER = logging.getLogger()
 30 | 
 31 | 
 32 | def full_path(path):
 33 |     """Get an absolute path to a file."""
 34 |     return os.path.abspath(os.path.expanduser(path))
 35 | 
 36 | 
 37 | def cmd_run(skt_data):
 38 |     """
 39 |     Run tests on a built kernel using the specified "runner". Only "Beaker"
 40 |     runner is currently supported.
 41 | 
 42 |     Args:
 43 |         skt_data: SKTData, parsed rc config file overriden with cmd-line args
 44 |     """
 45 |     jobtemplate = skt_data.runner.jobtemplate
 46 |     jobowner = skt_data.runner.jobowner
 47 |     blacklist = skt_data.runner.blacklist
 48 |     runner = BeakerRunner(jobtemplate, jobowner, blacklist)
 49 |     try:
 50 |         cmd_run.cleanup_done
 51 |     except AttributeError:
 52 |         cmd_run.cleanup_done = False
 53 | 
 54 |     def cleanup_handler():
 55 |         """ Save SKT job state (jobs, recipesets, retcode) to rc-file and mark
 56 |             in runner that cleanup_handler() ran. Jobs are not cancelled, see
 57 |             ticket #1140.
 58 | 
 59 |             Returns:
 60 |                  None
 61 |         """
 62 |         # Don't run cleanup handler twice by accident.
 63 |         # NOTE: Also, the code to cancel any jobs was removed on purpose.
 64 |         if cmd_run.cleanup_done:
 65 |             return
 66 | 
 67 |         skt_data.state.jobs = ' '.join(runner.job_to_recipe_set_map.keys())
 68 |         skt_data.state.recipesets = ' '.join(
 69 |             list(itertools.chain.from_iterable(
 70 |                 runner.job_to_recipe_set_map.values()
 71 |             ))
 72 |         )
 73 | 
 74 |         skt_data.state.retcode = runner.retcode
 75 |         with open(skt_data.state.rc, 'w') as fhandle:
 76 |             fhandle.write(skt_data.serialize())
 77 | 
 78 |         # NOTE: Don't cancel jobs. Per ticket #1140, Beaker jobs must continue
 79 |         # to run when a timeout is reached and skt is killed in the GitLab
 80 |         # pipeline.
 81 |         cmd_run.cleanup_done = True
 82 | 
 83 |     def signal_handler(sig, frame):
 84 |         # pylint: disable=unused-argument
 85 |         """
 86 |         Handle SIGTERM|SIGINT: call cleanup_handler() and exit.
 87 |         """
 88 |         cleanup_handler()
 89 | 
 90 |         sys.exit(SKT_ERROR)
 91 | 
 92 |     atexit.register(cleanup_handler)
 93 |     signal.signal(signal.SIGINT, signal_handler)
 94 |     signal.signal(signal.SIGTERM, signal_handler)
 95 |     return runner.run(skt_data.state.kernel_package_url,
 96 |                       skt_data.state.max_aborted_count,
 97 |                       skt_data.state.kernel_version,
 98 |                       skt_data.state.wait,
 99 |                       arch=skt_data.state.kernel_arch)
100 | 
101 | 
102 | def setup_logging(verbose):
103 |     """
104 |     Setup the root logger.
105 | 
106 |     Args:
107 |         verbose:    Verbosity level to setup log message filtering.
108 |     """
109 |     logging.basicConfig(format="%(asctime)s %(levelname)8s   %(message)s")
110 |     LOGGER.setLevel(logging.WARNING - (verbose * 10))
111 |     logging.getLogger('requests').setLevel(logging.WARNING)
112 |     logging.getLogger('urllib3').setLevel(logging.WARNING)
113 | 
114 | 
115 | def setup_parser():
116 |     """
117 |     Create an skt command line parser.
118 |     PLEASE SET DEFAULTS IN post_fixture() only.
119 | 
120 |     Returns:
121 |         The created parser.
122 |     """
123 |     parser = argparse.ArgumentParser()
124 | 
125 |     # These arguments apply to all commands within skt
126 |     parser.add_argument("-d", "--workdir", type=str, help="Path to work dir")
127 |     parser.add_argument("-v", "--verbose", help="Increase verbosity level",
128 |                         action="count", default=0)
129 |     parser.add_argument("--rc", help="Path to rc file", required=True)
130 |     parser.add_argument(
131 |         "--state",
132 |         help=(
133 |             "Save/read state from 'state' section of rc file"
134 |         ),
135 |         action="store_true",
136 |         default=False
137 |     )
138 | 
139 |     subparsers = parser.add_subparsers()
140 | 
141 |     # These arguments apply to the 'run' skt command
142 |     parser_run = subparsers.add_parser("run", add_help=False)
143 |     parser_run.add_argument('--max-aborted-count', type=int,
144 |                             help='Ignore <count> aborted jobs to work around '
145 |                                  'temporary infrastructure issues. Defaults '
146 |                                  'to 3.')
147 |     parser_run.add_argument("--wait", action="store_true",
148 |                             help="Do not exit until tests are finished")
149 | 
150 |     parser_run.add_argument("-h", "--help", help="Run sub-command help",
151 |                             action="help")
152 | 
153 |     return parser
154 | 
155 | 
156 | def post_fixture(skt_data):
157 |     """ Modifies skt configuration to set defaults or modify params."""
158 |     # Get an absolute path for the work directory
159 |     if skt_data.state.workdir:
160 |         skt_data.state.workdir = full_path(skt_data.state.workdir)
161 |     else:
162 |         skt_data.state.workdir = tempfile.mkdtemp()
163 | 
164 |     # Assign default --wait value if not specified
165 |     if not skt_data.state.wait:
166 |         skt_data.state.wait = True
167 | 
168 |     # Assign default max aborted count if it's not defined in config file
169 |     if not skt_data.state.max_aborted_count:
170 |         skt_data.state.max_aborted_count = 3
171 | 
172 |     # Get absolute path to blacklist file
173 |     if skt_data.runner.blacklist:
174 |         skt_data.runner.blacklist = full_path(skt_data.runner.blacklist)
175 | 
176 |     return skt_data
177 | 
178 | 
179 | def override_config_with_cmdline(args, skt_data):
180 |     """Override skt config data with cmd-line args.
181 |     Args:
182 |         args: argparse.Namespace, cmd-line args
183 |         skt_data: SKTData, parsed config file setiings
184 |     Returns:
185 |         updated skt_data with overriden values
186 |     """
187 | 
188 |     for key, value in vars(args).items():
189 |         setattr(skt_data.state, key, value)
190 | 
191 |     return skt_data
192 | 
193 | 
194 | def load_skt_config_data(args):
195 |     """Load settings from config file and override with cmd-line args.
196 |     Args:
197 |         args: argparse.Namespace, parsed cmd-line args
198 |     Returns:
199 |         skt_data: SKTData, parsed rc config file
200 |     """
201 |     # make sure path to rc-file is absolute
202 |     args.rc = full_path(args.rc)
203 | 
204 |     with open(args.rc) as fhandle:
205 |         skt_data = SKTData.deserialize(fhandle.read())  # type: SKTData
206 | 
207 |     # override config file values with cmd-line args
208 |     skt_data = override_config_with_cmdline(args, skt_data)
209 | 
210 |     return skt_data
211 | 
212 | 
213 | def main():
214 |     """This is the main entry point used by setup.cfg."""
215 |     # pylint: disable=protected-access
216 |     try:
217 |         parser = setup_parser()
218 |         args = parser.parse_args()
219 |         skt_data = load_skt_config_data(args)
220 | 
221 |         setup_logging(skt_data.state.verbose)
222 | 
223 |         skt_data = post_fixture(skt_data)
224 | 
225 |         retcode = cmd_run(skt_data)
226 | 
227 |         sys.exit(retcode)
228 |     except KeyboardInterrupt:
229 |         print("\nExited at user request.")
230 |         sys.exit(130)
231 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                     GNU GENERAL PUBLIC LICENSE
  2 |                        Version 2, June 1991
  3 | 
  4 |  Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
  5 |  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  6 |  Everyone is permitted to copy and distribute verbatim copies
  7 |  of this license document, but changing it is not allowed.
  8 | 
  9 |                             Preamble
 10 | 
 11 |   The licenses for most software are designed to take away your
 12 | freedom to share and change it.  By contrast, the GNU General Public
 13 | License is intended to guarantee your freedom to share and change free
 14 | software--to make sure the software is free for all its users.  This
 15 | General Public License applies to most of the Free Software
 16 | Foundation's software and to any other program whose authors commit to
 17 | using it.  (Some other Free Software Foundation software is covered by
 18 | the GNU Lesser General Public License instead.)  You can apply it to
 19 | your programs, too.
 20 | 
 21 |   When we speak of free software, we are referring to freedom, not
 22 | price.  Our General Public Licenses are designed to make sure that you
 23 | have the freedom to distribute copies of free software (and charge for
 24 | this service if you wish), that you receive source code or can get it
 25 | if you want it, that you can change the software or use pieces of it
 26 | in new free programs; and that you know you can do these things.
 27 | 
 28 |   To protect your rights, we need to make restrictions that forbid
 29 | anyone to deny you these rights or to ask you to surrender the rights.
 30 | These restrictions translate to certain responsibilities for you if you
 31 | distribute copies of the software, or if you modify it.
 32 | 
 33 |   For example, if you distribute copies of such a program, whether
 34 | gratis or for a fee, you must give the recipients all the rights that
 35 | you have.  You must make sure that they, too, receive or can get the
 36 | source code.  And you must show them these terms so they know their
 37 | rights.
 38 | 
 39 |   We protect your rights with two steps: (1) copyright the software, and
 40 | (2) offer you this license which gives you legal permission to copy,
 41 | distribute and/or modify the software.
 42 | 
 43 |   Also, for each author's protection and ours, we want to make certain
 44 | that everyone understands that there is no warranty for this free
 45 | software.  If the software is modified by someone else and passed on, we
 46 | want its recipients to know that what they have is not the original, so
 47 | that any problems introduced by others will not reflect on the original
 48 | authors' reputations.
 49 | 
 50 |   Finally, any free program is threatened constantly by software
 51 | patents.  We wish to avoid the danger that redistributors of a free
 52 | program will individually obtain patent licenses, in effect making the
 53 | program proprietary.  To prevent this, we have made it clear that any
 54 | patent must be licensed for everyone's free use or not licensed at all.
 55 | 
 56 |   The precise terms and conditions for copying, distribution and
 57 | modification follow.
 58 | 
 59 |                     GNU GENERAL PUBLIC LICENSE
 60 |    TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
 61 | 
 62 |   0. This License applies to any program or other work which contains
 63 | a notice placed by the copyright holder saying it may be distributed
 64 | under the terms of this General Public License.  The "Program", below,
 65 | refers to any such program or work, and a "work based on the Program"
 66 | means either the Program or any derivative work under copyright law:
 67 | that is to say, a work containing the Program or a portion of it,
 68 | either verbatim or with modifications and/or translated into another
 69 | language.  (Hereinafter, translation is included without limitation in
 70 | the term "modification".)  Each licensee is addressed as "you".
 71 | 
 72 | Activities other than copying, distribution and modification are not
 73 | covered by this License; they are outside its scope.  The act of
 74 | running the Program is not restricted, and the output from the Program
 75 | is covered only if its contents constitute a work based on the
 76 | Program (independent of having been made by running the Program).
 77 | Whether that is true depends on what the Program does.
 78 | 
 79 |   1. You may copy and distribute verbatim copies of the Program's
 80 | source code as you receive it, in any medium, provided that you
 81 | conspicuously and appropriately publish on each copy an appropriate
 82 | copyright notice and disclaimer of warranty; keep intact all the
 83 | notices that refer to this License and to the absence of any warranty;
 84 | and give any other recipients of the Program a copy of this License
 85 | along with the Program.
 86 | 
 87 | You may charge a fee for the physical act of transferring a copy, and
 88 | you may at your option offer warranty protection in exchange for a fee.
 89 | 
 90 |   2. You may modify your copy or copies of the Program or any portion
 91 | of it, thus forming a work based on the Program, and copy and
 92 | distribute such modifications or work under the terms of Section 1
 93 | above, provided that you also meet all of these conditions:
 94 | 
 95 |     a) You must cause the modified files to carry prominent notices
 96 |     stating that you changed the files and the date of any change.
 97 | 
 98 |     b) You must cause any work that you distribute or publish, that in
 99 |     whole or in part contains or is derived from the Program or any
100 |     part thereof, to be licensed as a whole at no charge to all third
101 |     parties under the terms of this License.
102 | 
103 |     c) If the modified program normally reads commands interactively
104 |     when run, you must cause it, when started running for such
105 |     interactive use in the most ordinary way, to print or display an
106 |     announcement including an appropriate copyright notice and a
107 |     notice that there is no warranty (or else, saying that you provide
108 |     a warranty) and that users may redistribute the program under
109 |     these conditions, and telling the user how to view a copy of this
110 |     License.  (Exception: if the Program itself is interactive but
111 |     does not normally print such an announcement, your work based on
112 |     the Program is not required to print an announcement.)
113 | 
114 | These requirements apply to the modified work as a whole.  If
115 | identifiable sections of that work are not derived from the Program,
116 | and can be reasonably considered independent and separate works in
117 | themselves, then this License, and its terms, do not apply to those
118 | sections when you distribute them as separate works.  But when you
119 | distribute the same sections as part of a whole which is a work based
120 | on the Program, the distribution of the whole must be on the terms of
121 | this License, whose permissions for other licensees extend to the
122 | entire whole, and thus to each and every part regardless of who wrote it.
123 | 
124 | Thus, it is not the intent of this section to claim rights or contest
125 | your rights to work written entirely by you; rather, the intent is to
126 | exercise the right to control the distribution of derivative or
127 | collective works based on the Program.
128 | 
129 | In addition, mere aggregation of another work not based on the Program
130 | with the Program (or with a work based on the Program) on a volume of
131 | a storage or distribution medium does not bring the other work under
132 | the scope of this License.
133 | 
134 |   3. You may copy and distribute the Program (or a work based on it,
135 | under Section 2) in object code or executable form under the terms of
136 | Sections 1 and 2 above provided that you also do one of the following:
137 | 
138 |     a) Accompany it with the complete corresponding machine-readable
139 |     source code, which must be distributed under the terms of Sections
140 |     1 and 2 above on a medium customarily used for software interchange; or,
141 | 
142 |     b) Accompany it with a written offer, valid for at least three
143 |     years, to give any third party, for a charge no more than your
144 |     cost of physically performing source distribution, a complete
145 |     machine-readable copy of the corresponding source code, to be
146 |     distributed under the terms of Sections 1 and 2 above on a medium
147 |     customarily used for software interchange; or,
148 | 
149 |     c) Accompany it with the information you received as to the offer
150 |     to distribute corresponding source code.  (This alternative is
151 |     allowed only for noncommercial distribution and only if you
152 |     received the program in object code or executable form with such
153 |     an offer, in accord with Subsection b above.)
154 | 
155 | The source code for a work means the preferred form of the work for
156 | making modifications to it.  For an executable work, complete source
157 | code means all the source code for all modules it contains, plus any
158 | associated interface definition files, plus the scripts used to
159 | control compilation and installation of the executable.  However, as a
160 | special exception, the source code distributed need not include
161 | anything that is normally distributed (in either source or binary
162 | form) with the major components (compiler, kernel, and so on) of the
163 | operating system on which the executable runs, unless that component
164 | itself accompanies the executable.
165 | 
166 | If distribution of executable or object code is made by offering
167 | access to copy from a designated place, then offering equivalent
168 | access to copy the source code from the same place counts as
169 | distribution of the source code, even though third parties are not
170 | compelled to copy the source along with the object code.
171 | 
172 |   4. You may not copy, modify, sublicense, or distribute the Program
173 | except as expressly provided under this License.  Any attempt
174 | otherwise to copy, modify, sublicense or distribute the Program is
175 | void, and will automatically terminate your rights under this License.
176 | However, parties who have received copies, or rights, from you under
177 | this License will not have their licenses terminated so long as such
178 | parties remain in full compliance.
179 | 
180 |   5. You are not required to accept this License, since you have not
181 | signed it.  However, nothing else grants you permission to modify or
182 | distribute the Program or its derivative works.  These actions are
183 | prohibited by law if you do not accept this License.  Therefore, by
184 | modifying or distributing the Program (or any work based on the
185 | Program), you indicate your acceptance of this License to do so, and
186 | all its terms and conditions for copying, distributing or modifying
187 | the Program or works based on it.
188 | 
189 |   6. Each time you redistribute the Program (or any work based on the
190 | Program), the recipient automatically receives a license from the
191 | original licensor to copy, distribute or modify the Program subject to
192 | these terms and conditions.  You may not impose any further
193 | restrictions on the recipients' exercise of the rights granted herein.
194 | You are not responsible for enforcing compliance by third parties to
195 | this License.
196 | 
197 |   7. If, as a consequence of a court judgment or allegation of patent
198 | infringement or for any other reason (not limited to patent issues),
199 | conditions are imposed on you (whether by court order, agreement or
200 | otherwise) that contradict the conditions of this License, they do not
201 | excuse you from the conditions of this License.  If you cannot
202 | distribute so as to satisfy simultaneously your obligations under this
203 | License and any other pertinent obligations, then as a consequence you
204 | may not distribute the Program at all.  For example, if a patent
205 | license would not permit royalty-free redistribution of the Program by
206 | all those who receive copies directly or indirectly through you, then
207 | the only way you could satisfy both it and this License would be to
208 | refrain entirely from distribution of the Program.
209 | 
210 | If any portion of this section is held invalid or unenforceable under
211 | any particular circumstance, the balance of the section is intended to
212 | apply and the section as a whole is intended to apply in other
213 | circumstances.
214 | 
215 | It is not the purpose of this section to induce you to infringe any
216 | patents or other property right claims or to contest validity of any
217 | such claims; this section has the sole purpose of protecting the
218 | integrity of the free software distribution system, which is
219 | implemented by public license practices.  Many people have made
220 | generous contributions to the wide range of software distributed
221 | through that system in reliance on consistent application of that
222 | system; it is up to the author/donor to decide if he or she is willing
223 | to distribute software through any other system and a licensee cannot
224 | impose that choice.
225 | 
226 | This section is intended to make thoroughly clear what is believed to
227 | be a consequence of the rest of this License.
228 | 
229 |   8. If the distribution and/or use of the Program is restricted in
230 | certain countries either by patents or by copyrighted interfaces, the
231 | original copyright holder who places the Program under this License
232 | may add an explicit geographical distribution limitation excluding
233 | those countries, so that distribution is permitted only in or among
234 | countries not thus excluded.  In such case, this License incorporates
235 | the limitation as if written in the body of this License.
236 | 
237 |   9. The Free Software Foundation may publish revised and/or new versions
238 | of the General Public License from time to time.  Such new versions will
239 | be similar in spirit to the present version, but may differ in detail to
240 | address new problems or concerns.
241 | 
242 | Each version is given a distinguishing version number.  If the Program
243 | specifies a version number of this License which applies to it and "any
244 | later version", you have the option of following the terms and conditions
245 | either of that version or of any later version published by the Free
246 | Software Foundation.  If the Program does not specify a version number of
247 | this License, you may choose any version ever published by the Free Software
248 | Foundation.
249 | 
250 |   10. If you wish to incorporate parts of the Program into other free
251 | programs whose distribution conditions are different, write to the author
252 | to ask for permission.  For software which is copyrighted by the Free
253 | Software Foundation, write to the Free Software Foundation; we sometimes
254 | make exceptions for this.  Our decision will be guided by the two goals
255 | of preserving the free status of all derivatives of our free software and
256 | of promoting the sharing and reuse of software generally.
257 | 
258 |                             NO WARRANTY
259 | 
260 |   11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
268 | REPAIR OR CORRECTION.
269 | 
270 |   12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
278 | POSSIBILITY OF SUCH DAMAGES.
279 | 
280 |                      END OF TERMS AND CONDITIONS
281 | 
282 |             How to Apply These Terms to Your New Programs
283 | 
284 |   If you develop a new program, and you want it to be of the greatest
285 | possible use to the public, the best way to achieve this is to make it
286 | free software which everyone can redistribute and change under these terms.
287 | 
288 |   To do so, attach the following notices to the program.  It is safest
289 | to attach them to the start of each source file to most effectively
290 | convey the exclusion of warranty; and each file should have at least
291 | the "copyright" line and a pointer to where the full notice is found.
292 | 
293 |     <one line to give the program's name and a brief idea of what it does.>
294 |     Copyright (C) <year>  <name of author>
295 | 
296 |     This program is free software; you can redistribute it and/or modify
297 |     it under the terms of the GNU General Public License as published by
298 |     the Free Software Foundation; either version 2 of the License, or
299 |     (at your option) any later version.
300 | 
301 |     This program is distributed in the hope that it will be useful,
302 |     but WITHOUT ANY WARRANTY; without even the implied warranty of
303 |     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
304 |     GNU General Public License for more details.
305 | 
306 |     You should have received a copy of the GNU General Public License along
307 |     with this program; if not, write to the Free Software Foundation, Inc.,
308 |     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
309 | 
310 | Also add information on how to contact you by electronic and paper mail.
311 | 
312 | If the program is interactive, make it output a short notice like this
313 | when it starts in an interactive mode:
314 | 
315 |     Gnomovision version 69, Copyright (C) year name of author
316 |     Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
317 |     This is free software, and you are welcome to redistribute it
318 |     under certain conditions; type `show c' for details.
319 | 
320 | The hypothetical commands `show w' and `show c' should show the appropriate
321 | parts of the General Public License.  Of course, the commands you use may
322 | be called something other than `show w' and `show c'; they could even be
323 | mouse-clicks or menu items--whatever suits your program.
324 | 
325 | You should also get your employer (if you work as a programmer) or your
326 | school, if any, to sign a "copyright disclaimer" for the program, if
327 | necessary.  Here is a sample; alter the names:
328 | 
329 |   Yoyodyne, Inc., hereby disclaims all copyright interest in the program
330 |   `Gnomovision' (which makes passes at compilers) written by James Hacker.
331 | 
332 |   <signature of Ty Coon>, 1 April 1989
333 |   Ty Coon, President of Vice
334 | 
335 | This General Public License does not permit incorporating your program into
336 | proprietary programs.  If your program is a subroutine library, you may
337 | consider it more useful to permit linking proprietary applications with the
338 | library.  If this is what you want to do, use the GNU Lesser General
339 | Public License instead of this License.
340 | 


--------------------------------------------------------------------------------
/tests/test_runner.py:
--------------------------------------------------------------------------------
  1 | # Copyright (c) 2018-2019 Red Hat, Inc. All rights reserved. This copyrighted
  2 | # material is made available to anyone wishing to use, modify, copy, or
  3 | # redistribute it subject to the terms and conditions of the GNU General Public
  4 | # License v.2 or later.
  5 | #
  6 | # This program is distributed in the hope that it will be useful, but WITHOUT
  7 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  8 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
  9 | # details.
 10 | #
 11 | # You should have received a copy of the GNU General Public License along with
 12 | # this program; if not, write to the Free Software Foundation, Inc., 51
 13 | # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 14 | """Test cases for runner module."""
 15 | import re
 16 | import subprocess
 17 | import tempfile
 18 | import unittest
 19 | 
 20 | import mock
 21 | from defusedxml.ElementTree import fromstring
 22 | from defusedxml.ElementTree import tostring
 23 | 
 24 | from skt import runner
 25 | from skt.misc import SKT_FAIL, SKT_SUCCESS, SKT_ERROR
 26 | from tests import misc
 27 | 
 28 | 
 29 | class TestRunner(unittest.TestCase):
 30 |     """Test cases for runner module."""
 31 |     # (Too many public methods) pylint: disable=too-many-public-methods
 32 | 
 33 |     def setUp(self):
 34 |         """Set up test fixtures"""
 35 |         self.myrunner = runner.BeakerRunner(**misc.DEFAULT_ARGS)
 36 | 
 37 |         self.max_aborted = 3
 38 | 
 39 |         # mock helper method to always return 'cki' to avoid mocking annoying
 40 |         # subprocess call
 41 |         self.mock1 = mock.patch('skt.runner.BeakerRunner.get_recipset_group',
 42 |                                 lambda sself, taskspec: 'cki')
 43 |         self.mock1.start()
 44 | 
 45 |     def tearDown(self):
 46 |         self.mock1.stop()
 47 | 
 48 |     def test_get_kpkginstall_task(self):
 49 |         """ Ensure get_kpkginstall_task works."""
 50 |         recipe_xml = """<recipe><task name="Boot test">
 51 |         <fetch url="kpkginstall"/></task>
 52 |         <task><fetch url="kpkginstall"/></task></recipe>"""
 53 |         recipe_node = fromstring(recipe_xml)
 54 | 
 55 |         ret_node = self.myrunner.get_kpkginstall_task(recipe_node)
 56 |         self.assertEqual(ret_node.attrib['name'], 'Boot test')
 57 |         self.assertEqual(ret_node.find('fetch').attrib['url'], 'kpkginstall')
 58 | 
 59 |     def test_get_recipe_test_list_1st(self):
 60 |         """ Ensure get_recipe_test_list works. First task is skipped."""
 61 |         recipe_xml = """<recipe><task result="Skip" /><task name="good2" />
 62 |         </recipe>"""
 63 | 
 64 |         recipe_node = fromstring(recipe_xml)
 65 | 
 66 |         ret_list = self.myrunner.get_recipe_test_list(recipe_node)
 67 |         self.assertEqual(ret_list, ['good2'])
 68 | 
 69 |     def test_get_recipe_test_list_2nd(self):
 70 |         """ Ensure get_recipe_test_list works. Second task is skipped."""
 71 |         recipe_xml = """<recipe><task name="good1" /><task result="Skip" />
 72 |         </recipe>"""
 73 | 
 74 |         recipe_node = fromstring(recipe_xml)
 75 | 
 76 |         ret_list = self.myrunner.get_recipe_test_list(recipe_node)
 77 |         self.assertEqual(ret_list, ['good1'])
 78 | 
 79 |     def test_get_recipe_test_list(self):
 80 |         """ Ensure get_recipe_test_list works. No task skipped."""
 81 |         recipe_xml = """<recipe><task name="good1" /><task name="good2" />
 82 |            </recipe>"""
 83 | 
 84 |         recipe_node = fromstring(recipe_xml)
 85 | 
 86 |         ret_list = self.myrunner.get_recipe_test_list(recipe_node)
 87 |         self.assertEqual(ret_list, ['good1', 'good2'])
 88 | 
 89 |     @mock.patch('subprocess.Popen')
 90 |     def test_jobsubmit(self, mock_popen):
 91 |         """ Ensure __jobsubmit works."""
 92 |         self.myrunner.jobowner = 'beaker-gods'
 93 | 
 94 |         args = ["bkr", "job-submit", "--job-owner=beaker-gods", "-"]
 95 | 
 96 |         mock_popen.return_value.returncode = 0
 97 |         mock_popen.return_value.communicate.return_value = \
 98 |             (bytearray("Submitted: ['123']", 'utf-8'), '')
 99 | 
100 |         # pylint: disable=W0212,E1101
101 |         self.myrunner._BeakerRunner__jobsubmit('<xml />')
102 | 
103 |         mock_popen.assert_called_once_with(args, stdin=subprocess.PIPE,
104 |                                            stdout=subprocess.PIPE,
105 |                                            stderr=subprocess.PIPE)
106 | 
107 |     @mock.patch('subprocess.Popen')
108 |     def test_jobsubmit_exc(self, mock_popen):
109 |         """ Ensure __jobsubmit doesn't parse invalid bkr output."""
110 |         # pylint: disable=W0212,E1101
111 |         self.myrunner.jobowner = 'beaker-gods'
112 | 
113 |         args = ["bkr", "job-submit", "--job-owner=beaker-gods", "-"]
114 | 
115 |         mock_popen.return_value.returncode = 0
116 |         mock_popen.return_value.communicate.return_value = \
117 |             ("Submitted: a horse", '')
118 | 
119 |         with self.assertRaises(Exception) as exc:
120 |             self.myrunner._BeakerRunner__jobsubmit('<xml />')
121 |             self.assertEqual(exc.message, 'Unable to submit the job!')
122 |             mock_popen.assert_called_once_with(args, stdin=subprocess.PIPE,
123 |                                                stdout=subprocess.PIPE)
124 | 
125 |     @mock.patch('builtins.open', create=True)
126 |     @mock.patch('subprocess.Popen')
127 |     def test_cancel_pending_jobs(self, mock_popen, mock_open):
128 |         """ Ensure cancel_pending_jobs works."""
129 |         # pylint: disable=W0212,E1101,W0613
130 |         j_jobid = 'J:123'
131 |         setid = '456'
132 |         test_xml = bytearray(
133 |             """<xml><whiteboard>yeah-that-whiteboard</whiteboard>
134 |             <recipeSet id="{}" /></xml>""".format(setid), 'utf-8')
135 | 
136 |         mock_popen.return_value.returncode = 0
137 |         mock_popen.return_value.communicate.return_value = (test_xml, '')
138 | 
139 |         self.myrunner._BeakerRunner__add_to_watchlist(j_jobid)
140 | 
141 |         mock_popen.assert_called()
142 | 
143 |         binary = 'bkr'
144 |         args = ['job-cancel', j_jobid]
145 | 
146 |         attrs = {'communicate.return_value': ('output', 'error'),
147 |                  'returncode': 0}
148 |         mock_popen.configure_mock(**attrs)
149 | 
150 |         self.myrunner.cancel_pending_jobs()
151 | 
152 |         mock_popen.assert_called_with([binary] + args)
153 | 
154 |     @mock.patch('logging.error')
155 |     def test_load_blacklist_fail(self, mock_logging_err):
156 |         """Ensure BeakerRunner.__load_blacklist() works"""
157 |         # pylint: disable=W0212,E1101,W0613
158 |         r_nr = self.myrunner
159 |         inv = 'blah-such-files-dont-usually-exist'
160 |         with self.assertRaises(Exception) as exc:
161 |             r_nr.blacklisted = self.myrunner._BeakerRunner__load_blacklist(inv)
162 | 
163 |             self.assertEqual(exc.message, ('Can\'t access %s!', inv))
164 | 
165 |     def test_load_blacklist(self):
166 |         """Ensure BeakerRunner.__load_blacklist() works"""
167 |         # pylint: disable=W0212,E1101
168 |         hostnames = ['host1', 'host2']
169 |         with tempfile.NamedTemporaryFile('w') as temp:
170 |             temp.write('\n'.join(hostnames) + '\n')
171 |             temp.seek(0)
172 | 
173 |             myrunner = self.myrunner
174 | 
175 |             myrunner.blacklisted = self.myrunner._BeakerRunner__load_blacklist(
176 |                 temp.name)
177 | 
178 |         self.assertEqual(hostnames, self.myrunner.blacklisted)
179 | 
180 |     def test_blacklist_test_force(self):
181 |         """ Ensure blacklist_hreq does not override when force= is set."""
182 |         # pylint: disable=W0212,E1101
183 |         initial = """<hostRequires force="srv" />"""
184 | 
185 |         hreq_node = fromstring(initial)
186 | 
187 |         # load blacklist ['host1', 'host2']
188 |         self.test_load_blacklist()
189 | 
190 |         etree_result = self.myrunner._BeakerRunner__blacklist_hreq(hreq_node)
191 |         # result must be the same as initial
192 |         result = tostring(etree_result).decode('utf-8')
193 |         self.assertEqual(re.sub(r'[\s]+', '', initial),
194 |                          re.sub(r'[\s]+', '', result))
195 | 
196 |     def test_blacklist_hreq_noand(self):
197 |         """ Ensure blacklist_hreq works without <and> element."""
198 |         # pylint: disable=W0212,E1101
199 |         initial = """<hostRequires></hostRequires>"""
200 | 
201 |         exp_result = """<hostRequires><and>
202 |         <hostname op="!=" value="host1"/>
203 |         <hostname op="!=" value="host2"/></and></hostRequires>"""
204 | 
205 |         hreq_node = fromstring(initial)
206 | 
207 |         # load blacklist ['host1', 'host2']
208 |         self.test_load_blacklist()
209 | 
210 |         etree_result = self.myrunner._BeakerRunner__blacklist_hreq(hreq_node)
211 |         result = tostring(etree_result).decode('utf-8')
212 |         self.assertEqual(re.sub(r'[\s]+', '', exp_result),
213 |                          re.sub(r'[\s]+', '', result))
214 | 
215 |     def test_blacklist_hreq_nohostnames(self):
216 |         """ Ensure blacklist_hreq works without hostnames."""
217 |         # pylint: disable=W0212,E1101
218 |         initial = """<hostRequires><system_type value="Machine"/><and>
219 |         <hypervisor op="=" value=""/></and></hostRequires>"""
220 | 
221 |         exp_result = """<hostRequires><system_type value="Machine"/><and>
222 |         <hypervisor op="=" value=""/></and></hostRequires>"""
223 | 
224 |         hreq_node = fromstring(initial)
225 | 
226 |         self.myrunner.blacklisted = []
227 |         etree_result = self.myrunner._BeakerRunner__blacklist_hreq(hreq_node)
228 |         result = tostring(etree_result).decode('utf-8')
229 |         self.assertEqual(re.sub(r'[\s]+', '', exp_result),
230 |                          re.sub(r'[\s]+', '', result))
231 | 
232 |     def test_blacklist_hreq_whnames(self):
233 |         """ Ensure blacklist_hreq works with hostnames."""
234 |         # pylint: disable=W0212,E1101
235 |         initial = """<hostRequires><system_type value="Machine"/><and>
236 |         <hypervisor op="=" value=""/></and></hostRequires>"""
237 | 
238 |         exp_result = """<hostRequires><system_type value="Machine"/><and>
239 |         <hypervisor op="=" value=""/><hostname op="!=" value="host1"/>
240 |         <hostname op="!=" value="host2"/></and></hostRequires>"""
241 | 
242 |         hreq_node = fromstring(initial)
243 | 
244 |         # load blacklist ['host1', 'host2']
245 |         self.test_load_blacklist()
246 | 
247 |         etree_result = self.myrunner._BeakerRunner__blacklist_hreq(hreq_node)
248 |         result = tostring(etree_result).decode('utf-8')
249 |         self.assertEqual(re.sub(r'[\s]+', '', exp_result),
250 |                          re.sub(r'[\s]+', '', result))
251 | 
252 |     @mock.patch('builtins.open', create=True)
253 |     @mock.patch('subprocess.Popen')
254 |     def test_add_to_watchlist(self, mock_popen, mock_open):
255 |         """Ensure __add_to_watchlist() works."""
256 |         # pylint: disable=W0212,E1101,W0613
257 |         j_jobid = 'J:123'
258 |         setid = '456'
259 |         s_setid = 'RS:{}'.format(setid)
260 |         test_xml = bytearray(
261 |             """<xml><whiteboard>yeah-that-whiteboard</whiteboard>
262 |             <recipeSet id="{}" /></xml>""".format(setid), 'utf-8')
263 | 
264 |         mock_popen.return_value.returncode = 0
265 |         mock_popen.return_value.communicate.return_value = (test_xml, '')
266 | 
267 |         self.myrunner._BeakerRunner__add_to_watchlist(j_jobid)
268 |         mock_popen.assert_called_once_with(
269 |             ["bkr", "job-results", "--prettyxml", j_jobid],
270 |             stdout=subprocess.PIPE,
271 |             stderr=subprocess.PIPE
272 |         )
273 | 
274 |         # test that whiteboard was parsed OK
275 |         self.assertEqual(self.myrunner.whiteboard, 'yeah-that-whiteboard')
276 | 
277 |         # test that job_to_recipe mapping was updated
278 |         self.assertEqual(self.myrunner.job_to_recipe_set_map[j_jobid],
279 |                          {s_setid})
280 | 
281 |         # test that watchlist contains RS
282 |         self.assertIn(s_setid, self.myrunner.watchlist)
283 | 
284 |         # test that no recipes completed
285 |         self.assertEqual(self.myrunner.completed_recipes[s_setid], set())
286 | 
287 |     @mock.patch('builtins.open', create=True)
288 |     @mock.patch('subprocess.Popen')
289 |     def test_getresultstree(self, mock_popen, mock_open):
290 |         """Ensure getresultstree() works."""
291 |         # pylint: disable=W0613
292 |         test_xml = bytearray("<xml><test>TEST</test></xml>", 'utf-8')
293 |         mock_popen.return_value.returncode = 0
294 |         mock_popen.return_value.communicate.return_value = (test_xml, '')
295 |         result = self.myrunner.getresultstree('RS:123')
296 |         self.assertEqual(next(x.text for x in result.iter('test')), 'TEST')
297 | 
298 |     def test_forget_taskspec_withr(self):
299 |         """Ensure __forget_taskspec() works with recipe sets."""
300 |         # pylint: disable=protected-access,E1101
301 |         self.myrunner.job_to_recipe_set_map = {"J:00001": ["RS:00001"]}
302 |         result = self.myrunner._BeakerRunner__forget_taskspec("RS:00001")
303 |         self.assertIsNone(result)
304 |         self.assertEqual(self.myrunner.job_to_recipe_set_map, {})
305 | 
306 |     @mock.patch('logging.info')
307 |     def test_getresults_pass(self, mock_logging):
308 |         """Ensure __getresults() works."""
309 |         # pylint: disable=W0212,E1101
310 |         self.myrunner.job_to_recipe_set_map = {'jobid': set(['recipeset'])}
311 |         self.myrunner.recipe_set_results['recipeset'] = fromstring(
312 |             misc.get_asset_content('beaker_recipe_set_results.xml')
313 |         )
314 | 
315 |         result = self.myrunner._BeakerRunner__getresults()
316 |         self.assertEqual(result, 0)
317 |         mock_logging.assert_called()
318 | 
319 |     @mock.patch('logging.error')
320 |     def test_getresults_aborted(self, mock_logging):
321 |         """Ensure __getresults() handles all aborted / cancelled jobs."""
322 |         # pylint: disable=W0212,E1101
323 |         result = self.myrunner._BeakerRunner__getresults()
324 |         self.assertEqual(result, 2)
325 |         mock_logging.assert_called()
326 | 
327 |     @mock.patch('logging.info')
328 |     def test_getresults_failure(self, mock_logging):
329 |         """Ensure __getresults() handles a job failure."""
330 |         # pylint: disable=W0212,E1101
331 |         self.myrunner.job_to_recipe_set_map = {'jobid': set(['recipeset'])}
332 |         self.myrunner.recipe_set_results['recipeset'] = fromstring(
333 |             misc.get_asset_content('beaker_fail_results.xml')
334 |         )
335 | 
336 |         result = self.myrunner._BeakerRunner__getresults()
337 |         self.assertEqual(result, 1)
338 |         mock_logging.assert_called()
339 | 
340 |     def test_recipe_set_to_job(self):
341 |         """Ensure __recipe_set_to_job() works."""
342 |         # pylint: disable=W0212,E1101
343 |         beaker_xml = misc.get_asset_content('beaker_recipe_set_results.xml')
344 |         xml_parsed = fromstring(beaker_xml)
345 | 
346 |         result = self.myrunner._BeakerRunner__recipe_set_to_job(xml_parsed)
347 |         self.assertEqual(result.tag, 'job')
348 | 
349 |         result = self.myrunner._BeakerRunner__recipe_set_to_job(xml_parsed,
350 |                                                                 samehost=True)
351 |         self.assertEqual(result.tag, 'job')
352 | 
353 |     def test_recipe_set_to_job_whst(self):
354 |         """Ensure __recipe_set_to_job() works with hostname."""
355 |         # pylint: disable=W0212,E1101
356 |         beaker_xml = """<recipeSet><recipe><hostRequires>
357 |         <hostname op="!=" value="hst1"/></hostRequires></recipe></recipeSet>"""
358 |         xml_parsed = fromstring(beaker_xml)
359 | 
360 |         result = self.myrunner._BeakerRunner__recipe_set_to_job(xml_parsed)
361 | 
362 |         # check that <hostname op="!=" value="hst1"/> wasn't removed
363 |         self.assertEqual(len(result.findall('.//hostname')), 1)
364 |         self.assertEqual(tostring(result.find('.//hostname')).decode(),
365 |                          '<hostname op="!=" value="hst1" />')
366 | 
367 |     @mock.patch('skt.runner.BeakerRunner._BeakerRunner__jobsubmit')
368 |     def test_run(self, mock_jobsubmit):
369 |         """Ensure BeakerRunner.run works."""
370 |         url = "http://machine1.example.com/builds/1234567890.tar.gz"
371 |         release = "4.17.0-rc1"
372 |         wait = False
373 | 
374 |         mock_jobsubmit.return_value = "J:0001"
375 | 
376 |         result = self.myrunner.run(url, self.max_aborted, release, wait)
377 |         self.assertEqual(result, 0)
378 | 
379 |     @mock.patch('logging.error')
380 |     def test_run_fail(self, mock_logging_err):
381 |         """Ensure BeakerRunner.run errors on invalid xml."""
382 |         # pylint: disable=W0613
383 |         url = "http://machine1.example.com/builds/1234567890.tar.gz"
384 |         release = "4.17.0-rc1"
385 |         wait = True
386 |         inv_runner = runner.BeakerRunner(**misc.INV_TEMPLATE_ARGS)
387 | 
388 |         result = inv_runner.run(url, self.max_aborted, release, wait)
389 |         self.assertEqual(result, 2)
390 | 
391 |     @mock.patch('skt.runner.BeakerRunner.getresultstree')
392 |     @mock.patch('skt.runner.BeakerRunner._BeakerRunner__jobsubmit')
393 |     def test_run_wait(self, mock_jobsubmit, mock_getresultstree):
394 |         """Ensure BeakerRunner.run works."""
395 |         url = "http://machine1.example.com/builds/1234567890.tar.gz"
396 |         release = "4.17.0-rc1"
397 |         wait = True
398 |         self.myrunner.whiteboard = 'test'
399 | 
400 |         beaker_xml = misc.get_asset_content('beaker_pass_results.xml')
401 |         mock_getresultstree.return_value = fromstring(beaker_xml)
402 |         mock_jobsubmit.return_value = "J:0001"
403 | 
404 |         # no need to wait 60 seconds
405 |         # though beaker_pass_results.xml only needs one iteration
406 |         self.myrunner.watchdelay = 0.1
407 |         result = self.myrunner.run(url, self.max_aborted, release, wait)
408 | 
409 |         self.assertEqual(result, 0)
410 | 
411 |     @mock.patch('logging.warning')
412 |     @mock.patch('logging.error')
413 |     @mock.patch('skt.runner.BeakerRunner._BeakerRunner__jobsubmit')
414 |     def test_run_wait3(self, mock_logging, mock_logging_err, mock_jobsubmit):
415 |         """Ensure BeakerRunner.run works."""
416 |         # pylint: disable=W0613
417 | 
418 |         # beaker_results2.xml doesn't have passed tasks, so result is SKT_FAIL
419 |         result = misc.exec_on(self.myrunner, mock_jobsubmit,
420 |                               'beaker_results2.xml', 1, 'Completed')
421 |         self.assertEqual(SKT_FAIL, result)
422 | 
423 |     @mock.patch('logging.warning')
424 |     @mock.patch('logging.error')
425 |     @mock.patch('skt.runner.BeakerRunner._BeakerRunner__jobsubmit')
426 |     def test_run_wait4(self, mock_logging, mock_logging_err, mock_jobsubmit):
427 |         """Ensure BeakerRunner.run works."""
428 |         # pylint: disable=W0613
429 | 
430 |         result = misc.exec_on(self.myrunner, mock_jobsubmit,
431 |                               'beaker_results3.xml', 1, 'Completed')
432 |         self.assertEqual(SKT_SUCCESS, result)
433 | 
434 |     @mock.patch('logging.warning')
435 |     @mock.patch('logging.error')
436 |     @mock.patch('skt.runner.BeakerRunner._BeakerRunner__jobsubmit')
437 |     def test_run_wait5(self, mock_logging, mock_logging_err, mock_jobsubmit):
438 |         """Ensure BeakerRunner.run works."""
439 |         # pylint: disable=W0613
440 | 
441 |         result = misc.exec_on(self.myrunner, mock_jobsubmit,
442 |                               'beaker_results3.xml', 2, 'Completed')
443 |         self.assertEqual(SKT_SUCCESS, result)
444 | 
445 |     @mock.patch('logging.warning')
446 |     @mock.patch('logging.error')
447 |     @mock.patch('skt.runner.BeakerRunner._BeakerRunner__jobsubmit')
448 |     def test_run_wait6(self, mock_logging, mock_logging_err, mock_jobsubmit):
449 |         """Ensure BeakerRunner.run works."""
450 |         # pylint: disable=W0613
451 |         # abort right-away ( 0 allowed)
452 |         misc.exec_on(self.myrunner, mock_jobsubmit, 'beaker_aborted_some.xml',
453 |                      0)
454 | 
455 |     @mock.patch('logging.warning')
456 |     @mock.patch('logging.error')
457 |     @mock.patch('skt.runner.BeakerRunner._BeakerRunner__jobsubmit')
458 |     def test_run_wait7(self, mock_logging, mock_logging_err, mock_jobsubmit):
459 |         """Ensure BeakerRunner.run works."""
460 |         # pylint: disable=W0613
461 | 
462 |         # abort later on, change last recipe to Aborted
463 |         result = misc.exec_on(self.myrunner, mock_jobsubmit,
464 |                               'beaker_aborted_some.xml', 0, 'Aborted')
465 |         self.assertEqual(SKT_ERROR, result)
466 | 
467 |     @mock.patch('logging.warning')
468 |     @mock.patch('logging.error')
469 |     @mock.patch('skt.runner.BeakerRunner._BeakerRunner__jobsubmit')
470 |     def test_run_wait8(self, mock_logging, mock_logging_err, mock_jobsubmit):
471 |         """Ensure BeakerRunner.run works."""
472 |         # pylint: disable=W0613
473 | 
474 |         result = misc.exec_on(self.myrunner, mock_jobsubmit,
475 |                               'beaker_aborted_some.xml', 5, 'Cancelled')
476 |         self.assertEqual(SKT_ERROR, result)
477 | 
478 |     @mock.patch('skt.runner.BeakerRunner.getresultstree')
479 |     @mock.patch('skt.runner.BeakerRunner._BeakerRunner__jobsubmit')
480 |     def test_waived_hidden(self, mock_jobsubmit, mock_getresultstree):
481 |         """ Ensure that waived tests don't affect overall test result."""
482 | 
483 |         beaker_xml = misc.get_asset_content('beaker_results.xml')
484 |         mock_getresultstree.return_value = fromstring(beaker_xml)
485 |         mock_jobsubmit.return_value = "J:0001"
486 | 
487 |         # no need to wait 60 seconds
488 |         # though beaker_pass_results.xml only needs one iteration
489 |         self.myrunner.watchdelay = 0.1
490 | 
491 |         result = misc.exec_on(self.myrunner, mock_jobsubmit,
492 |                               'beaker_results.xml', 5, 'Completed')
493 |         self.assertEqual(SKT_SUCCESS, result)
494 | 
495 |     @mock.patch('logging.error')
496 |     @mock.patch('logging.warning')
497 |     @mock.patch('skt.runner.BeakerRunner.getresultstree')
498 |     @mock.patch('skt.runner.BeakerRunner._BeakerRunner__jobsubmit')
499 |     def test_waived_fails2(self, mock_jobsubmit, mock_getresultstree,
500 |                            mock_warning, mock_error):
501 |         """ Ensure that waived tests don't affect overall
502 |             test result. This tests test abort."""
503 |         # pylint: disable=unused-argument
504 | 
505 |         beaker_xml = misc.get_asset_content('beaker_aborted_some.xml')
506 |         mock_getresultstree.return_value = fromstring(beaker_xml)
507 |         mock_jobsubmit.return_value = "J:0001"
508 | 
509 |         # no need to wait 60 seconds
510 |         # though beaker_pass_results.xml only needs one iteration
511 |         self.myrunner.watchdelay = 0.1
512 | 
513 |         result = misc.exec_on(self.myrunner, mock_jobsubmit,
514 |                               'beaker_aborted_some.xml', 5,
515 |                               'Completed')
516 | 
517 |         self.assertEqual(result, SKT_ERROR)
518 | 
519 |     @mock.patch('logging.error')
520 |     @mock.patch('logging.warning')
521 |     @mock.patch('skt.runner.BeakerRunner.getresultstree')
522 |     @mock.patch('skt.runner.BeakerRunner._BeakerRunner__jobsubmit')
523 |     def test_waived_abort(self, mock_jobsubmit, mock_getresultstree,
524 |                           mock_warning, mock_error):
525 |         """ Ensure that one test failing and one waived test aborting
526 |             leads to failure. """
527 |         # pylint: disable=unused-argument
528 | 
529 |         beaker_xml = misc.get_asset_content('beaker_aborted_some.xml')
530 |         mock_getresultstree.return_value = fromstring(beaker_xml)
531 |         mock_jobsubmit.return_value = "J:0001"
532 | 
533 |         # no need to wait 60 seconds
534 |         # though beaker_pass_results.xml only needs one iteration
535 |         self.myrunner.watchdelay = 0.1
536 | 
537 |         result = misc.exec_on(self.myrunner, mock_jobsubmit,
538 |                               'beaker_aborted_some.xml', 5, 'Aborted')
539 | 
540 |         self.assertEqual(result, SKT_ERROR)
541 | 
542 |     @mock.patch('logging.error')
543 |     @mock.patch('logging.warning')
544 |     @mock.patch('skt.runner.BeakerRunner.getresultstree')
545 |     @mock.patch('skt.runner.BeakerRunner._BeakerRunner__jobsubmit')
546 |     def test_fail_and_skip(self, mock_jobsubmit, mock_getresultstree,
547 |                            mock_warning, mock_error):
548 |         """ Ensure that a job with failed tasks and skipped tests
549 |             returns SKT_FAIL."""
550 |         # pylint: disable=unused-argument
551 | 
552 |         beaker_xml = misc.get_asset_content('beaker_skip_and_fail.xml')
553 |         mock_getresultstree.return_value = fromstring(beaker_xml)
554 |         mock_jobsubmit.return_value = "J:0001"
555 | 
556 |         # no need to wait 60 seconds
557 |         # though beaker_pass_results.xml only needs one iteration
558 |         self.myrunner.watchdelay = 0.1
559 | 
560 |         # For the purposes of this test it's not necessary to flip the state
561 |         # of the fake Beaker XML job to 'Completed', the asset file already has
562 |         # that state.
563 |         result = misc.exec_on(self.myrunner, mock_jobsubmit,
564 |                               'beaker_skip_and_fail.xml', 5)
565 | 
566 |         # see method description for details why SKT_FAIL
567 |         self.assertEqual(SKT_FAIL, result)
568 | 


--------------------------------------------------------------------------------
/skt/runner.py:
--------------------------------------------------------------------------------
  1 | # Copyright (c) 2017-2020 Red Hat, Inc. All rights reserved. This copyrighted
  2 | # material is made available to anyone wishing to use, modify, copy, or
  3 | # redistribute it subject to the terms and conditions of the GNU General
  4 | # Public License v.2 or later.
  5 | #
  6 | # This program is distributed in the hope that it will be useful, but WITHOUT
  7 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  8 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
  9 | # details.
 10 | #
 11 | # You should have received a copy of the GNU General Public License
 12 | # along with this program; if not, write to the Free Software Foundation, Inc.,
 13 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 14 | """Class for managing Runner."""
 15 | import copy
 16 | import logging
 17 | import os
 18 | import pathlib
 19 | import platform
 20 | import re
 21 | import subprocess
 22 | import sys
 23 | import time
 24 | import traceback
 25 | 
 26 | from defusedxml.ElementTree import fromstring
 27 | from defusedxml.ElementTree import tostring
 28 | from defusedxml.ElementTree import ParseError
 29 | 
 30 | from skt.misc import SKT_SUCCESS, SKT_FAIL, SKT_ERROR, SKT_BOOT
 31 | from skt.misc import is_task_waived
 32 | from cki_lib.misc import safe_popen, retry_safe_popen
 33 | 
 34 | 
 35 | class ConditionCheck:
 36 |     def __init__(self, retval, **kwargs):
 37 |         self.retval = retval
 38 |         self.kwargs = kwargs
 39 | 
 40 |     def __str__(self):
 41 |         values = ' '.join([f'{arg}={self.kwargs[arg]}' for arg in self.kwargs])
 42 | 
 43 |         return f'retval={self.retval} {values}'
 44 | 
 45 |     def __call__(self, task, is_task_waived_func, prev_task):
 46 |         """ Evaluates the condition and return retval if matched, else None.
 47 | 
 48 |             Args:
 49 |                 task: defusedxml of the task node
 50 |                 is_task_waived_func: function used to test whether the task
 51 |                                      is waived
 52 |                 prev_task: task that was run before this one, or None if this
 53 |                            task is the first task in the recipe
 54 |         """
 55 |         task_results = {
 56 |             'result': task.attrib.get('result'),
 57 |             'status': task.attrib.get('status'),
 58 |             'waived': is_task_waived_func(task),
 59 |             'prev_task_panicked_and_waived':
 60 |                 True if prev_task is not None and (
 61 |                     is_task_waived_func(prev_task) and
 62 |                     prev_task.attrib.get('result') == 'Panic'
 63 |                 ) else False
 64 |         }
 65 | 
 66 |         if not self.kwargs:
 67 |             # don't match empty conditions as satisfied
 68 |             return None
 69 | 
 70 |         for arg in self.kwargs:
 71 |             if task_results[arg] != self.kwargs[arg]:
 72 |                 # the status entry doesn't match all the conditions
 73 |                 return None
 74 | 
 75 |         # the status entry matches all the conditions
 76 |         return self.retval
 77 | 
 78 | 
 79 | result_condition_checks = [
 80 |     # This contains objects that will return <retval> (first parameter), when
 81 |     # all the specified conditions are met. Empty conditions with no keywords
 82 |     # are never met.
 83 | 
 84 |     # Previous task was waived and panicked, which causes the next
 85 |     # task to abort. The task is waived for a reason, return
 86 |     # SKT_SUCCESS.
 87 |     ConditionCheck(SKT_SUCCESS, result='Warn', waived=False, status='Aborted',
 88 |                    prev_task_panicked_and_waived=True),
 89 | 
 90 |     # A non-waived task panicked, return SKT_FAIL and don't confuse
 91 |     # this with infra-errors.
 92 |     ConditionCheck(SKT_FAIL, result='Panic', waived=False),
 93 | 
 94 |     # A non-waived tasked aborted, return SKT_ERROR, possible
 95 |     # infra issue.
 96 |     ConditionCheck(SKT_ERROR, result='Warn',  waived=False, status='Aborted'),
 97 | 
 98 |     # The rest of the fall-through conditions.
 99 |     ConditionCheck(SKT_FAIL, result='Warn', waived=False),
100 |     ConditionCheck(SKT_FAIL, result='Fail', waived=False),
101 | ]
102 | 
103 | 
104 | class BeakerRunner:
105 |     """Beaker test runner"""
106 |     # pylint: disable=too-many-instance-attributes
107 | 
108 |     def __init__(self, jobtemplate, jobowner=None, blacklist=None):
109 |         """
110 |         Initialize a runner executing tests on Beaker.
111 | 
112 |         Args:
113 |             jobtemplate:    Path to a Beaker job template. Can contain a tilde
114 |                             expression ('~' or '~user') to be expanded into
115 |                             the current user's home directory.
116 |             jobowner:       Name of a Beaker user on whose behalf the job
117 |                             should be submitted, or None, if the owner should
118 |                             be the current user.
119 |             blacklist:      Path to file containing hostnames to blacklist from
120 |                             running on, one hostname per line.
121 |         """
122 |         # Beaker job template file path
123 |         # FIXME Move expansion up the call stack, as this limits the class
124 |         # usefulness, because tilde is a valid path character.
125 |         self.template = os.path.expanduser(jobtemplate)
126 |         # Name of a Beaker user on whose behalf the job should be submitted,
127 |         # or None, if the owner should be the current user.
128 |         self.jobowner = jobowner
129 |         self.blacklisted = self.__load_blacklist(blacklist)
130 |         # Delay between checks of Beaker job statuses, seconds
131 |         self.watchdelay = 60
132 |         # Set of recipe sets that didn't complete yet
133 |         self.watchlist = set()
134 |         self.whiteboard = ''
135 |         self.job_to_recipe_set_map = {}
136 |         self.recipe_set_results = {}
137 |         # Keep a set of completed recipes per set so we don't check them again
138 |         self.completed_recipes = {}
139 |         self.aborted_count = 0
140 |         # Set up the default, allowing for overrides with each run
141 |         self.max_aborted = 3
142 |         # Marks that we've had too many retries on infra-issues.
143 |         self.has_aborted = False
144 | 
145 |         # the actual retcode to return is stored here
146 |         self.retcode = SKT_ERROR
147 | 
148 |         logging.info("beaker template: %s", self.template)
149 | 
150 |     @classmethod
151 |     def __load_blacklist(cls, filepath):
152 |         hostnames = []
153 | 
154 |         try:
155 |             with open(filepath, 'r') as fileh:
156 |                 for line in fileh:
157 |                     line = line.strip()
158 |                     if line:
159 |                         hostnames.append(line)
160 |         except (IOError, OSError) as exc:
161 |             logging.error('Can\'t access %s!', filepath)
162 |             raise exc
163 |         except TypeError:
164 |             logging.info('No hostname blacklist file passed')
165 | 
166 |         logging.info('Blacklisted hostnames: %s', hostnames)
167 |         return hostnames
168 | 
169 |     def get_recipset_group(self, taskspec):
170 |         for (jid, rset) in self.job_to_recipe_set_map.items():
171 |             if taskspec in rset:
172 |                 return self.getresultstree(jid).attrib['group']
173 | 
174 |         return None
175 | 
176 |     def getresultstree(self, taskspec):
177 |         """
178 |         Retrieve Beaker results for taskspec in Beaker's native XML format.
179 | 
180 |         Args:
181 |             taskspec:   ID of the job, recipe or recipe set.
182 | 
183 |         Returns:
184 |             etree node representing the results.
185 |         """
186 |         args = ["bkr", "job-results", "--prettyxml", taskspec]
187 | 
188 |         err_strings = ["ProtocolError", "503 Service Unavailable"]
189 |         stdout, stderr, returncode = retry_safe_popen(err_strings, args,
190 |                                                       stderr=subprocess.PIPE,
191 |                                                       stdout=subprocess.PIPE)
192 | 
193 |         if returncode:
194 |             logging.warning(stdout)
195 |             logging.warning(stderr)
196 |             raise RuntimeError('failed getting Beaker job-results')
197 | 
198 |         # return Beaker results parsed xml
199 |         results = fromstring(stdout)
200 |         self.recipe_set_results[taskspec] = results
201 |         return results
202 | 
203 |     def __forget_taskspec(self, recipe_set_id):
204 |         """
205 |         Remove recipe set from self.job_to_recipe_set_map and self.watchlist
206 |         (if applicable).
207 | 
208 |         Args:
209 |             recipe_set_id: recipe set (RS:xxxxx) ID.
210 |         """
211 |         self.watchlist.discard(recipe_set_id)
212 |         deljids = set()
213 |         for (jid, rset) in self.job_to_recipe_set_map.items():
214 |             if recipe_set_id in rset:
215 |                 rset.remove(recipe_set_id)
216 |                 if not rset:
217 |                     deljids.add(jid)
218 |         for jid in deljids:
219 |             del self.job_to_recipe_set_map[jid]
220 | 
221 |     def _not_booting(self, recipe):
222 |         """
223 |         Check if the kernel we should test failed to boot. In these cases, the
224 |         Boot test throws EWD. We need to check that EWD wasn't hit sooner (e.g.
225 |         the distro failed to install).
226 | 
227 |         Returns:
228 |             True if the issue is caused by a kernel not booting,
229 |             False otherwise.
230 |         """
231 |         is_boot_test = False
232 | 
233 |         for task in recipe.findall('task'):
234 |             if task.attrib['name'] == "Boot test":
235 |                 is_boot_test = True
236 | 
237 |             for res in task.findall('.//results/'):
238 |                 if res.text and 'External Watchdog Expired' in res.text:
239 |                     if is_boot_test:
240 |                         return True
241 |                     else:
242 |                         return False
243 | 
244 |             if is_boot_test:
245 |                 # If we got here it means that we got past the boot without
246 |                 # hitting EWD. Since we want to only check the boot failure and
247 |                 # not test troubles, return here.
248 |                 return False
249 | 
250 |     def decide_run_result_by_task(self, recipe_result, recipe_id):
251 |         """ Return result of a single recipe decided by tasks. The conditions
252 |             to test are read from result_condition_checks in their natural
253 |             specified order.
254 | 
255 |             Args:
256 |                 recipe_result: a defused xml
257 |                 recipe_id: id of the recipe from the XML, prefixed with R:
258 |             Returns:
259 |                 retval, msg where retval is a return code like SKT_SUCCESS,
260 |                             SKT_BOOT, ... and msg is an explanation of why
261 | 
262 |         """
263 |         # If the recipe passed, then there's little to do.
264 |         if recipe_result.attrib.get('result') == 'Pass':
265 |             return SKT_SUCCESS, f'recipeid {recipe_id} passed all tests'
266 | 
267 |         if self._not_booting(recipe_result):
268 |             return SKT_BOOT, f'recipeid {recipe_id} hit EWD in boottest!'
269 | 
270 |         if self.has_aborted:
271 |             return SKT_ERROR, 'too many aborted recipes!'
272 | 
273 |         prev_task = None
274 |         for task in recipe_result.findall('task'):
275 |             for cond_check in result_condition_checks:
276 |                 retval = cond_check(task, is_task_waived, prev_task)
277 |                 if retval is not None:
278 |                     return retval, f'recipeid {recipe_id} -> {str(cond_check)}'
279 | 
280 |             # remember the previous task
281 |             prev_task = task
282 | 
283 |         # It's possible that failing tests were just waived...
284 |         return SKT_SUCCESS, f'recipeid {recipe_id} passed with waived tests'
285 | 
286 |     def __getresults(self):
287 |         """
288 |         Get return code based on the job results. This processes all recipes.
289 |         The priority is (from highest to lowest):
290 |         # 0) Infra issue - all tests aborted or cancelled
291 |         # 1) Unwaived infra issue in any recipe
292 |         # 2) Boot failure
293 |         # 3) Unwaived test failure
294 |         # 4) All tests OK or alle OK with test waived
295 | 
296 |         Returns:
297 |             SKT_SUCCESS if all jobs passed,
298 |             SKT_FAIL in case of failures, and
299 |             SKT_ERROR in case of infrastructure failures.
300 |             SKT_BOOT in case of boot failure.
301 |         """
302 |         if not self.job_to_recipe_set_map:
303 |             # We forgot every job / recipe set
304 |             logging.error('All test sets aborted or were cancelled!')
305 |             return SKT_ERROR
306 | 
307 |         rcpid_and_results = []
308 |         for _, recipe_sets in self.job_to_recipe_set_map.items():
309 |             for recipe_set_id in recipe_sets:
310 |                 results = self.recipe_set_results[recipe_set_id]
311 |                 for recipe_result in results.findall('.//recipe'):
312 |                     rcpid = 'R:' + recipe_result.attrib.get('id')
313 |                     ret, msg = self.decide_run_result_by_task(recipe_result,
314 |                                                               rcpid)
315 | 
316 |                     # log output of decide_run_result_by_task for final rc only
317 |                     logging.info(msg)
318 | 
319 |                     rcpid_and_results.append((rcpid, ret))
320 | 
321 |         for ret in [SKT_ERROR, SKT_BOOT, SKT_FAIL]:
322 |             for rcpid, result in rcpid_and_results:
323 |                 if ret == result:
324 |                     logging.info(f'Failure ({ret}) in recipeid {rcpid}'
325 |                                  f' detected!')
326 |                     return ret
327 | 
328 |         logging.info('Testing passed!')
329 |         return SKT_SUCCESS
330 | 
331 |     def __blacklist_hreq(self, host_requires):
332 |         """
333 |         Make sure recipe excludes blacklisted hosts.
334 | 
335 |         Args:
336 |             host_requires: etree node representing "hostRequires" node from the
337 |                            recipe.
338 | 
339 |         Returns:
340 |             Modified "hostRequires" etree node.
341 |         """
342 | 
343 |         if host_requires.get('force'):
344 |             # don't add blacklist if the host is forced
345 |             return host_requires
346 | 
347 |         and_node = host_requires.find('and')
348 |         if and_node is None:
349 |             and_node = fromstring('<and />')
350 |             host_requires.append(and_node)
351 | 
352 |         invalid_entries_reported = False
353 |         for disabled in self.blacklisted:
354 |             try:
355 |                 hostname = fromstring(f'<hostname op="!=" value="{disabled}" '
356 |                                       f'/>')
357 |                 and_node.append(hostname)
358 |             except ParseError:
359 |                 # do not accept or try to quote any html/xml values; only
360 |                 # plaintext values like "host1" are accepted
361 |                 if not invalid_entries_reported:
362 |                     logging.info('The blacklist or a part of it is invalid!')
363 |                     invalid_entries_reported = True
364 | 
365 |         return host_requires
366 | 
367 |     def __recipe_set_to_job(self, recipe_set, samehost=False):
368 |         tmp = copy.deepcopy(recipe_set)
369 | 
370 |         try:
371 |             group = self.get_recipset_group('RS:{}'.format(recipe_set.
372 |                                                            attrib['id']))
373 |         except KeyError:
374 |             # don't set group later on
375 |             group = None
376 | 
377 |         for recipe in tmp.findall('recipe'):
378 |             hreq = recipe.find("hostRequires")
379 |             if samehost:
380 |                 hostname = hreq.find('hostname')
381 |                 if hostname is not None:
382 |                     hreq.remove(hostname)
383 | 
384 |                 value = recipe.attrib.get("system")
385 |                 hostname = fromstring(f'<hostname op="=" value="{value}"/>')
386 |                 hreq.append(hostname)
387 |             else:
388 |                 new_hreq = self.__blacklist_hreq(hreq)
389 |                 recipe.remove(hreq)
390 |                 recipe.append(new_hreq)
391 | 
392 |         newwb = fromstring("<whiteboard/>")
393 |         newwb.text = "%s [RS:%s]" % (self.whiteboard, tmp.attrib.get("id"))
394 | 
395 |         newroot = fromstring("<job/>")
396 |         if group:
397 |             newroot.attrib['group'] = group
398 | 
399 |         newroot.append(newwb)
400 |         newroot.append(tmp)
401 | 
402 |         return newroot
403 | 
404 |     def cancel_pending_jobs(self):
405 |         """
406 |         Cancel all recipe sets from self.watchlist.
407 |         Cancelling a part of a job leads to cancelling the entire job.
408 |         So we cancel a job if any of its recipesets is in the watchlist.
409 |         """
410 |         logging.info('Cancelling pending jobs!')
411 | 
412 |         # Update all recipe_set_ids before canceling, so we don't gey KeyError
413 |         # later on (nitpick).
414 |         for recipe_set_id in self.watchlist.copy():
415 |             self.getresultstree(recipe_set_id)
416 | 
417 |         for job_id in set(self.job_to_recipe_set_map):
418 |             _, _, ret = safe_popen(['bkr', 'job-cancel', job_id])
419 |             if ret:
420 |                 logging.info('Failed to cancel the remaining recipe sets!')
421 | 
422 |     def __handle_test_abort(self, recipe, recipe_id, recipe_set_id, root):
423 |         if self._not_booting(recipe):
424 |             return
425 | 
426 |         retval, _ = self.decide_run_result_by_task(recipe, recipe_id)
427 |         if retval == SKT_SUCCESS:
428 |             # A task that is waived aborted or panicked. Waived tasks are
429 |             # appended to the end of the recipe, so we should be able to
430 |             # safely ignore this.
431 |             return
432 | 
433 |         logging.warning('%s from %s aborted!',
434 |                         recipe_id,
435 |                         recipe_set_id)
436 |         self.aborted_count += 1
437 | 
438 |         if self.aborted_count < self.max_aborted:
439 |             logging.warning('Resubmitting aborted %s',
440 |                             recipe_set_id)
441 |             newjob = self.__recipe_set_to_job(root)
442 |             newjobid = self.__jobsubmit(tostring(newjob))
443 |             self.__add_to_watchlist(newjobid)
444 | 
445 |         self.watchlist.discard(recipe_set_id)
446 | 
447 |     def __handle_test_fail(self, recipe, recipe_id):
448 |         # Something in the recipe set really reported failure
449 |         test_failure = False
450 |         # set to True when test failed, but is waived
451 |         waiving_skip = False
452 | 
453 |         if self.get_kpkginstall_task(recipe) is None:
454 |             # we don't waive the kernel-install task :-)
455 |             # Assume the kernel was installed by default and
456 |             # everything is a test
457 |             test_failure = True
458 | 
459 |         elif self.decide_run_result_by_task(recipe, recipe_id)[0]\
460 |                 == SKT_SUCCESS:
461 |             # A task that is waived failed. Waived tasks are
462 |             # appended to the end of the recipe, so we should be able to
463 |             # safely ignore this.
464 |             waiving_skip = True
465 |             # set this just fyi - we will continue anyway
466 |             test_failure = True
467 |         else:
468 |             test_list = self.get_recipe_test_list(recipe)
469 | 
470 |             for task in recipe.findall('task'):
471 |                 result = task.attrib.get('result')
472 | 
473 |                 if result != 'Pass' and result != 'Skip':
474 |                     if task.attrib.get('name') in test_list:
475 |                         test_failure = True
476 | 
477 |                     break
478 | 
479 |         return test_failure, waiving_skip
480 | 
481 |     def __watchloop(self):
482 |         while self.watchlist:
483 |             time.sleep(self.watchdelay)
484 |             if self.max_aborted <= self.aborted_count:
485 |                 self.has_aborted = True
486 |                 # Remove / cancel all the remaining recipe set IDs and abort
487 |                 self.cancel_pending_jobs()
488 |                 return
489 | 
490 |             for recipe_set_id in self.watchlist.copy():
491 |                 root = self.getresultstree(recipe_set_id)
492 |                 recipes = root.findall('.//recipe')
493 | 
494 |                 for recipe in recipes:
495 |                     result = recipe.attrib.get('result')
496 |                     status = recipe.attrib.get('status')
497 |                     recipe_id = 'R:' + recipe.attrib.get('id')
498 |                     if status not in ['Completed', 'Aborted', 'Cancelled'] or \
499 |                             recipe_id in self.completed_recipes[recipe_set_id]:
500 |                         # continue watching unfinished recipes
501 |                         continue
502 | 
503 |                     logging.info("%s status changed to %s", recipe_id, status)
504 |                     self.completed_recipes[recipe_set_id].add(recipe_id)
505 |                     if len(self.completed_recipes[recipe_set_id]) == \
506 |                             len(recipes):
507 |                         try:
508 |                             self.watchlist.remove(recipe_set_id)
509 |                         except KeyError:
510 |                             pass
511 |                         self.recipe_set_results[recipe_set_id] = root
512 | 
513 |                     if result == 'Pass':
514 |                         # some recipe passed, nothing to do here
515 |                         continue
516 | 
517 |                     if status == 'Cancelled':
518 |                         # job got cancelled for some reason, there's probably
519 |                         # an external reason
520 |                         logging.error('Cancelled run detected! Cancelling the '
521 |                                       'rest of runs and aborting!')
522 |                         self.cancel_pending_jobs()
523 |                         return
524 | 
525 |                     if result == 'Warn' and status == 'Aborted':
526 |                         self.__handle_test_abort(recipe, recipe_id,
527 |                                                  recipe_set_id, root)
528 |                         continue
529 | 
530 |                     # check for test failure
531 |                     test_failure, waive_skip = \
532 |                         self.__handle_test_fail(recipe, recipe_id)
533 |                     if waive_skip:
534 |                         logging.info("recipe %s waived task(s) failed",
535 |                                      recipe_id)
536 |                         continue
537 | 
538 |                     if not test_failure:
539 |                         # Recipe failed before the tested kernel was installed
540 |                         self.__forget_taskspec(recipe_set_id)
541 |                         self.aborted_count += 1
542 | 
543 |                         if self.aborted_count < self.max_aborted:
544 |                             logging.warning('Infrastructure-related problem '
545 |                                             'found, resubmitting %s',
546 |                                             recipe_set_id)
547 |                             newjob = self.__recipe_set_to_job(root)
548 |                             newjobid = self.__jobsubmit(tostring(newjob))
549 |                             self.__add_to_watchlist(newjobid)
550 | 
551 |     def __add_to_watchlist(self, jobid):
552 |         root = self.getresultstree(jobid)
553 | 
554 |         if not self.whiteboard:
555 |             self.whiteboard = root.find("whiteboard").text
556 | 
557 |         self.job_to_recipe_set_map[jobid] = set()
558 |         for recipe_set in root.findall("recipeSet"):
559 |             set_id = "RS:%s" % recipe_set.attrib.get("id")
560 |             self.job_to_recipe_set_map[jobid].add(set_id)
561 |             self.watchlist.add(set_id)
562 |             self.completed_recipes[set_id] = set()
563 |             logging.info("added %s to watchlist", set_id)
564 | 
565 |     def wait(self, jobid):
566 |         """
567 |         Add jobid to watchlist, enter watchloop and wait for jobid to finish.
568 | 
569 |         Args:
570 |             jobid: id of a Beaker job like 1234
571 | 
572 |         """
573 |         self.__add_to_watchlist(jobid)
574 |         self.__watchloop()
575 | 
576 |     def get_recipe_test_list(self, recipe_node):
577 |         """
578 |         Retrieve the list of tests which ran for a particular recipe. All tasks
579 |         after kpkginstall (including the kpkginstall task itself), which were
580 |         not skipped, are interpreted as ran tests. If the kpkginstall task
581 |         doesn't exist, assume every task is a test and the kernel was installed
582 |         by default.
583 | 
584 |         Args:
585 |             recipe_node: ElementTree node representing the recipe, extracted
586 |                          from Beaker XML or result XML.
587 | 
588 |         Returns:
589 |             List of test names that ran.
590 |         """
591 |         test_list = []
592 |         after_kpkg = True if self.get_kpkginstall_task(recipe_node) is None \
593 |             else False
594 | 
595 |         for test_task in recipe_node.findall('task'):
596 |             fetch = test_task.find('fetch')
597 |             if fetch is not None and \
598 |                     'kpkginstall' in fetch.attrib.get('url', ''):
599 |                 after_kpkg = True
600 | 
601 |             if after_kpkg and test_task.attrib.get('result') != 'Skip':
602 |                 test_list.append(test_task.attrib.get('name'))
603 | 
604 |         return test_list
605 | 
606 |     @classmethod
607 |     def get_kpkginstall_task(cls, recipe_node):
608 |         """
609 |         Return a kpkginstall task node for a given recipe.
610 | 
611 |         Returns:
612 |             Etree node representing kpkginstall task, None if there is no such
613 |             task.
614 |         """
615 |         for task in recipe_node.findall('task'):
616 |             fetch = task.find('fetch')
617 |             if fetch is not None and \
618 |                     'kpkginstall' in fetch.attrib.get('url', ''):
619 |                 return task
620 | 
621 |         return None
622 | 
623 |     def __jobsubmit(self, xml):
624 |         # pylint: disable=no-self-use
625 |         jobid = None
626 |         args = ["bkr", "job-submit"]
627 | 
628 |         if self.jobowner is not None:
629 |             args += ["--job-owner=%s" % self.jobowner]
630 | 
631 |         args += ["-"]
632 |         err_strings = ["connection to beaker.engineering.redhat.com failed",
633 |                        "Can't connect to MySQL server on"]
634 |         stdout, stderr, retcode = retry_safe_popen(err_strings, args,
635 |                                                    stdin_data=xml,
636 |                                                    stdin=subprocess.PIPE,
637 |                                                    stderr=subprocess.PIPE,
638 |                                                    stdout=subprocess.PIPE)
639 | 
640 |         for line in stdout.split("\n"):
641 |             match = re.match(r"^Submitted: \['([^']+)'\]$", line)
642 |             if match:
643 |                 jobid = match.group(1)
644 |                 break
645 | 
646 |         if not jobid:
647 |             logging.info(f'retcode={retcode}, stderr={stderr}')
648 |             logging.info(stdout)
649 |             raise Exception('Unable to submit the job!')
650 | 
651 |         logging.info("submitted jobid: %s", jobid)
652 | 
653 |         return jobid
654 | 
655 |     def add_blacklist2recipes(self, job_xml_tree):
656 |         """ Make sure blacklist is added to all recipes.
657 | 
658 |             Args:
659 |                job_xml_tree: ElementTree.Element with all recipeSets/recipes
660 | 
661 |         """
662 |         for recipe in job_xml_tree.findall('recipeSet/recipe'):
663 |             hreq = recipe.find('hostRequires')
664 |             new_hreq = self.__blacklist_hreq(hreq)
665 |             recipe.remove(hreq)
666 |             recipe.append(new_hreq)
667 | 
668 |     def run(self, url, max_aborted, release, wait=False,
669 |             arch=platform.machine()):
670 |         """
671 |         Run tests in Beaker.
672 | 
673 |         Args:
674 |             url:         URL pointing to kernel tarball.
675 |             max_aborted: Maximum number of allowed aborted jobs. Abort the
676 |                          whole stage if the number is reached.
677 |             release:     NVR of the tested kernel.
678 |             wait:        False if skt should exit after submitting the jobs,
679 |                          True if it should wait for them to finish.
680 |             arch:        Architecture of the machine the tests should run on,
681 |                          in a format accepted by Beaker. Defaults to
682 |                          architecture of the current machine skt is running on
683 |                          if not specified.
684 | 
685 |         Returns:
686 |             ret where ret can be
687 |                    SKT_SUCCESS if everything passed
688 |                    SKT_FAIL if testing failed
689 |                    SKT_ERROR in case of infrastructure error (exceptions are
690 |                                                               logged)
691 |                    SKT_BOOT if the boot test failed
692 |         """
693 |         # pylint: disable=too-many-arguments
694 |         self.watchlist = set()
695 |         self.job_to_recipe_set_map = {}
696 |         self.recipe_set_results = {}
697 |         self.completed_recipes = {}
698 |         self.aborted_count = 0
699 |         self.max_aborted = max_aborted
700 | 
701 |         try:
702 |             text = pathlib.Path(self.template).read_text()
703 |             job_xml_tree = fromstring(text)
704 |             # add blacklist to all recipes
705 |             self.add_blacklist2recipes(job_xml_tree)
706 | 
707 |             # convert etree to xml and submit the job to Beaker
708 |             jobid = self.__jobsubmit(tostring(job_xml_tree))
709 | 
710 |             if wait:
711 |                 # wait for completion, resubmit jobs as needed
712 |                 self.wait(jobid)
713 |                 # get return code and report it
714 |                 self.retcode = self.__getresults()
715 |                 logging.debug(
716 |                     "Got return code when gathering results: %s", self.retcode
717 |                 )
718 |             else:
719 |                 # not waiting -> change retcode to success
720 |                 self.retcode = SKT_SUCCESS
721 | 
722 |         except (Exception, BaseException) as e:
723 |             if isinstance(e, SystemExit):
724 |                 sys.stderr.write('SystemExit exception caught\n')
725 |                 raise
726 |             else:
727 |                 exc = sys.exc_info()
728 |                 logging.error('\n'.join(traceback.format_exception(*exc)))
729 | 
730 |         return self.retcode
731 | 


--------------------------------------------------------------------------------