├── .github ├── cijoe-docker │ └── Dockerfile ├── cijoe │ └── README.rst └── workflows │ ├── cijoe_docker.yml │ └── verify_and_publish.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .readthedocs.yaml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── CONTRIBUTORS.md ├── LICENSE ├── Makefile ├── README.rst ├── docs ├── .gitignore ├── Dockerfile ├── Makefile ├── emit.py ├── requirements.txt └── source │ ├── 000_prep.cmd │ ├── 000_prep.out │ ├── 050_usage_help.cmd │ ├── 050_usage_help.out │ ├── 100_inject.cmd │ ├── 100_inject.out │ ├── 150_cijoe_resources.cmd │ ├── 150_cijoe_resources.out │ ├── 160_cijoe_resources_compl.cmd │ ├── 160_cijoe_resources_compl.out │ ├── 200_quickstart.cmd │ ├── 200_quickstart.err │ ├── 200_quickstart.out │ ├── 250_quickstart.cmd │ ├── 250_quickstart.out │ ├── 250_quickstart.txt │ ├── 300_lint.cmd │ ├── 300_lint.err │ ├── 300_lint.out │ ├── 400_usage_script_all.cmd │ ├── 400_usage_script_all.out │ ├── 400_usage_script_local.cmd │ ├── 400_usage_script_local.out │ ├── 410_wait.cmd │ ├── 410_wait.out │ ├── 420_usage_workflow_all.cmd │ ├── 420_usage_workflow_all.out │ ├── 430_wait.cmd │ ├── 430_wait.out │ ├── 450_usage_workflow_subset.cmd │ ├── 450_usage_workflow_subset.out │ ├── _static │ ├── .keep │ ├── cijoe-networked.drawio.png │ ├── environment.png │ └── logo.png │ ├── conf.py │ ├── configs │ └── index.rst │ ├── figs │ └── environment.dia │ ├── index.rst │ ├── packages │ ├── core │ │ ├── index.rst │ │ └── scripts │ │ │ ├── cmdrunner.rst │ │ │ ├── example_script_default.rst │ │ │ ├── example_script_testrunner.rst │ │ │ ├── get.rst │ │ │ ├── put.rst │ │ │ ├── reporter.rst │ │ │ ├── repository_prep.rst │ │ │ └── testrunner.rst │ ├── index.rst │ ├── linux │ │ ├── index.rst │ │ └── scripts │ │ │ ├── build_kdebs.rst │ │ │ ├── null_blk.rst │ │ │ └── sysinfo.rst │ ├── qemu │ │ ├── index.rst │ │ └── scripts │ │ │ ├── build.rst │ │ │ ├── guest_initialize.rst │ │ │ ├── guest_kill.rst │ │ │ ├── guest_start.rst │ │ │ ├── install.rst │ │ │ └── qemu_version.rst │ └── system_imaging │ │ ├── index.rst │ │ └── scripts │ │ ├── diskimage_from_cloudimage.rst │ │ └── dockerimage_from_diskimage.rst │ ├── prerequisites │ └── index.rst │ ├── resources │ ├── auxiliary │ │ └── index.rst │ ├── index.rst │ └── templates │ │ ├── index.rst │ │ ├── template.html.jinja2 │ │ └── template.py │ ├── scripts │ └── index.rst │ ├── testrunner │ └── index.rst │ ├── usage │ └── index.rst │ └── workflows │ └── index.rst ├── pyproject.toml ├── src └── cijoe │ ├── cli │ ├── __init__.py │ ├── __main__.py │ └── cli.py │ ├── core │ ├── __init__.py │ ├── analyser.py │ ├── auxiliary │ │ ├── .keep │ │ ├── __init__.py │ │ ├── cijoe-completions │ │ └── example.perfreq │ ├── command.py │ ├── configs │ │ ├── __init__.py │ │ ├── example_config_default.toml │ │ ├── example_config_get_put.toml │ │ ├── example_config_testrunner.toml │ │ └── transport-ssh.toml │ ├── errors.py │ ├── misc.py │ ├── processing.py │ ├── resources.py │ ├── scripts │ │ ├── __init__.py │ │ ├── cmdrunner.py │ │ ├── example_script_default.py │ │ ├── example_script_testrunner.py │ │ ├── get.py │ │ ├── put.py │ │ ├── reporter.py │ │ ├── repository_prep.py │ │ └── testrunner.py │ ├── templates │ │ ├── .keep │ │ ├── __init__.py │ │ ├── example-tmp-workflow.yaml.jinja2 │ │ └── report-workflow.html.jinja2 │ ├── transport.py │ ├── workflows │ │ ├── __init__.py │ │ ├── example_workflow_default.yaml │ │ ├── example_workflow_get_put.yaml │ │ └── example_workflow_testrunner.yaml │ └── worklets │ │ ├── README.rst │ │ └── __init__.py │ ├── linux │ ├── __init__.py │ ├── configs │ │ ├── __init__.py │ │ ├── example_config_build_kdebs.toml │ │ └── example_config_null_blk.toml │ ├── null_blk.py │ ├── scripts │ │ ├── __init__.py │ │ ├── build_kdebs.py │ │ ├── null_blk.py │ │ └── sysinfo.py │ └── workflows │ │ ├── __init__.py │ │ ├── example_workflow_build_kdebs.yaml │ │ └── example_workflow_null_blk.yaml │ ├── pytest_plugin │ ├── __init__.py │ └── hooks_and_fixtures.py │ ├── qemu │ ├── __init__.py │ ├── auxiliary │ │ └── __init__.py │ ├── configs │ │ ├── __init__.py │ │ ├── example_config_build.toml │ │ ├── example_config_guest_aarch64.toml │ │ └── example_config_guest_x86_64.toml │ ├── scripts │ │ ├── __init__.py │ │ ├── build.py │ │ ├── guest_initialize.py │ │ ├── guest_kill.py │ │ ├── guest_start.py │ │ ├── guest_wait_for_termination.py │ │ ├── install.py │ │ └── qemu_version.py │ ├── workflows │ │ ├── __init__.py │ │ ├── example_workflow_build.yaml │ │ ├── example_workflow_guest_aarch64.yaml │ │ └── example_workflow_guest_x86_64.yaml │ └── wrapper.py │ └── system_imaging │ ├── __init__.py │ ├── auxiliary │ ├── Dockerfile │ ├── __init__.py │ ├── cloudinit-freebsd-metadata.meta │ ├── cloudinit-freebsd-userdata.user │ ├── cloudinit-linux-alpine-userdata.user │ ├── cloudinit-linux-common-metadata.meta │ ├── cloudinit-linux-common-userdata.user │ └── dockerignore │ ├── configs │ ├── __init__.py │ ├── example_config_aarch64.toml │ └── example_config_x86_64.toml │ ├── scripts │ ├── __init__.py │ ├── diskimage_from_cloudimage.py │ └── dockerimage_from_diskimage.py │ └── workflows │ ├── __init__.py │ ├── example_workflow_aarch64.yaml │ └── example_workflow_x86_64.yaml └── tests ├── core ├── aux_loglevel.py ├── conftest.py ├── test_cli_example.py ├── test_collector.py ├── test_commands.py ├── test_loglevel.py ├── test_transfer.py └── test_workflow.py └── linux └── test_null_blk.py /.github/cijoe-docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # Introduction 2 | # ============ 3 | # 4 | # This Dockerfile sets up an environment to run cijoe in CI pipelines, 5 | # such as those provided by GitHub Actions. It supports: 6 | # 7 | # * Building, installing, and running cijoe. 8 | # * Running QEMU-guest in Docker: 9 | # - Ideal for using custom virtual machines on GitHub Actions. 10 | # * Running Docker-in-Docker (DinD): 11 | # - Useful for executing 'docker build' within a Docker container. 12 | # 13 | # Based on Debian, this image leverages one of the most renowned Linux distributions, 14 | # offering a familiar environment for Ubuntu users. Debian provides freedom, 15 | # stability, and a vast selection of up-to-date packages. 16 | # 17 | # Custom QEMU 18 | # =========== 19 | # 20 | # For specific use cases, you might require a more recent version of QEMU. 21 | # In such cases, this image can serve as a base, allowing you to extend it 22 | # by building a custom version of QEMU. This approach can provide newer 23 | # features or enable functionality not available in upstream versions. 24 | FROM debian:bookworm 25 | 26 | WORKDIR /opt 27 | 28 | RUN apt-get -qy update && \ 29 | apt-get -qy \ 30 | -o "Dpkg::Options::=--force-confdef" \ 31 | -o "Dpkg::Options::=--force-confold" upgrade \ 32 | && \ 33 | apt-get -qy autoclean 34 | 35 | RUN apt-get -qy -f install --no-install-recommends \ 36 | bc \ 37 | bison \ 38 | bridge-utils \ 39 | build-essential \ 40 | ca-certificates \ 41 | cloud-image-utils \ 42 | cpio \ 43 | debhelper-compat \ 44 | docker.io \ 45 | flex \ 46 | fuse3 \ 47 | genisoimage \ 48 | git \ 49 | guestmount \ 50 | htop \ 51 | kmod \ 52 | libelf-dev \ 53 | libglib2.0-dev \ 54 | libguestfs-tools \ 55 | libssl-dev \ 56 | linux-image-amd64 \ 57 | lshw \ 58 | meson \ 59 | neovim \ 60 | nodejs \ 61 | openssh-server \ 62 | pciutils \ 63 | pipx \ 64 | procps \ 65 | python3-build \ 66 | python3-jinja2 \ 67 | qemu-efi-aarch64 \ 68 | qemu-kvm \ 69 | qemu-system-arm \ 70 | qemu-system-x86 \ 71 | qemu-utils \ 72 | rsync \ 73 | ssh \ 74 | time \ 75 | && \ 76 | apt-get -qy clean && \ 77 | apt-get -qy autoremove && \ 78 | rm -rf /var/lib/apt/lists/* && \ 79 | pipx ensurepath 80 | 81 | # Proide the cijoe-version as argument to the build and expose it in ENV 82 | ARG CIJOE_VERSION 83 | ENV CIJOE_VERSION=$CIJOE_VERSION 84 | 85 | # Setup environment variables for pipx 86 | ENV PIPX_HOME=/root/.local/pipx 87 | ENV PATH=/root/.local/bin::$PATH 88 | 89 | # Install cijoe itself 90 | RUN pipx install cijoe==$CIJOE_VERSION && \ 91 | pipx inject cijoe coverage --force && \ 92 | pipx inject cijoe pytest-cov --force 93 | 94 | # 95 | # Modified SSH Configuration, this is done enable the use of ssh to localhost 96 | # without being prompted 97 | # 98 | 99 | # Setup SSH 100 | RUN echo "PermitRootLogin yes" >> /etc/ssh/sshd_config && \ 101 | systemctl enable ssh && \ 102 | service ssh restart && \ 103 | mkdir -p /root/.ssh && \ 104 | chmod 0700 /root/.ssh && \ 105 | ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N "" && \ 106 | cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys 107 | 108 | # Don't want any of that hastle 109 | RUN echo "Host *" >> /root/.ssh/config && \ 110 | echo " StrictHostKeyChecking no" >> /root/.ssh/config && \ 111 | echo " NoHostAuthenticationForLocalhost yes" >> /root/.ssh/config && \ 112 | echo " UserKnownHostsFile=/dev/null" >> /root/.ssh/config && \ 113 | chmod 0400 /root/.ssh/config 114 | 115 | CMD ["bash"] 116 | -------------------------------------------------------------------------------- /.github/cijoe/README.rst: -------------------------------------------------------------------------------- 1 | cijoe in github 2 | =============== 3 | 4 | These are cijoe files used in github workflows for cijoe itself. 5 | -------------------------------------------------------------------------------- /.github/workflows/cijoe_docker.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Build and deploy a Docker image for cijoe with access to Docker and QEMU. 3 | # 4 | # The image is built and pushed to GitHub Container Registry (GHCR) when: 5 | # 1. Triggered manually via workflow_dispatch. 6 | # 2. Changes are pushed to 'cijoe_docker' branch 7 | # 8 | # Refer to `.github/workflows/cijoe_docker.yml` for details on this workflow. 9 | name: cijoe_docker 10 | 11 | on: 12 | workflow_dispatch: 13 | push: 14 | branches: 15 | - 'cijoe_docker' 16 | 17 | env: 18 | DOCKER_IMAGE: ghcr.io/${{ github.repository_owner }}/cijoe-docker 19 | DOCKER_TAG: v0.9.52 20 | 21 | jobs: 22 | build_and_push: 23 | runs-on: ubuntu-latest 24 | 25 | steps: 26 | - name: Check out repository 27 | uses: actions/checkout@v4.2.2 28 | 29 | - name: Authenticate to GitHub Container Registry 30 | uses: docker/login-action@v3.3.0 31 | with: 32 | registry: ghcr.io 33 | username: ${{ github.actor }} 34 | password: ${{ secrets.GITHUB_TOKEN }} 35 | 36 | - name: Build Docker image 37 | run: | 38 | docker buildx build \ 39 | --build-arg CIJOE_VERSION=${{ env.DOCKER_TAG }} \ 40 | --tag ${{ env.DOCKER_IMAGE }}:latest \ 41 | --tag ${{ env.DOCKER_IMAGE }}:${{ env.DOCKER_TAG }} \ 42 | --file .github/cijoe-docker/Dockerfile \ 43 | . 44 | 45 | - name: Push Docker image to registry 46 | run: | 47 | docker push ${{ env.DOCKER_IMAGE }}:latest 48 | docker push ${{ env.DOCKER_IMAGE }}:${{ env.DOCKER_TAG }} 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .coverage 2 | cijoe-output* 3 | tags 4 | selftest_results 5 | *.egg-info 6 | build 7 | dist 8 | .cache 9 | .idea 10 | __pycache__ 11 | *.img 12 | *.pyc 13 | *.zip 14 | .vscode/ 15 | .venv/ -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | default_language_version: 3 | python: python3 4 | 5 | repos: 6 | - repo: https://github.com/jumanjihouse/pre-commit-hook-yamlfmt 7 | rev: 0.2.3 8 | hooks: 9 | - id: yamlfmt 10 | name: YAML-format 11 | args: 12 | - '--mapping=2' 13 | - '--sequence=2' 14 | - '--offset=0' 15 | - '--width=120' 16 | - '--preserve-quotes' 17 | types: [file] 18 | files: \.(yaml|yml|config|workflow)$ 19 | 20 | - repo: https://github.com/psf/black 21 | rev: 24.10.0 22 | hooks: 23 | - id: black 24 | name: Python-format-black 25 | 26 | - repo: https://github.com/pycqa/isort 27 | rev: 5.13.2 28 | hooks: 29 | - id: isort 30 | name: Python-format-isort 31 | 32 | - repo: https://github.com/astral-sh/ruff-pre-commit 33 | rev: v0.6.9 34 | hooks: 35 | - id: ruff 36 | name: Python-lint-ruff 37 | args: [--fix] 38 | 39 | - repo: https://github.com/pre-commit/mirrors-mypy 40 | rev: v1.12.0 41 | hooks: 42 | - id: mypy 43 | name: Python-lint-mypy 44 | exclude: "docs/source/conf.py" 45 | additional_dependencies: 46 | - types-paramiko 47 | - types-PyYAML 48 | - types-requests 49 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # .readthedocs.yaml 3 | # Read the Docs configuration file 4 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 5 | 6 | # Required 7 | version: 2 8 | 9 | # Set the version of Python and other tools you might need 10 | build: 11 | os: ubuntu-20.04 12 | tools: 13 | python: "3.10" 14 | 15 | # Build documentation in the docs/ directory with Sphinx 16 | sphinx: 17 | configuration: docs/source/conf.py 18 | 19 | # If using Sphinx, optionally build your docs in additional formats such as PDF 20 | # formats: 21 | # - pdf 22 | 23 | # Optionally declare the Python requirements required to build your docs 24 | python: 25 | install: 26 | - requirements: docs/requirements.txt 27 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | The repository is tagged using semantic versioning, e.g. `v0.0.3`. The `master` 4 | branch consist of the latest state of CIJOE with possible hot-fixes since 5 | the last version tag, consider `master` as a preview of the next version. 6 | 7 | Changes are described in this file in a section named matching the version tag. 8 | Sections with "(Upcoming)" describe changes on the roadmap for CIJOE. 9 | 10 | Changes on the `master` branch, from the latest version tag up to and including 11 | HEAD can be subject to a git rebase. 12 | 13 | ## 0.9.2 14 | 15 | Added a naive path-sanitizer, switched to using git to "clean", fixed a style-issue. 16 | 17 | ## 0.9.1 18 | 19 | Fixed doc generation via read-the-docs 20 | 21 | ## 0.9.0 22 | 23 | A complete re-implementation and a switch of license. 24 | 25 | 26 | ## 0.2.1 27 | 28 | * Replaced the ``::`` module-separator with ``.`` in hooks and modules. 29 | 30 | * Fix to quick-start documentation 31 | 32 | ## 0.2.0 33 | 34 | * Replaced the ``::`` module-seperator with ``.`` 35 | 36 | * Removed all examples and provided them in an example package 'cijoe-pkg-example' 37 | 38 | * Removed deprecated Bash-module named ``board`` 39 | 40 | * Removed deprecated reference environment ``refenv-u1604`` 41 | 42 | * A complete overhaul of the documentation 43 | 44 | * ``extractor:fio_json_read``: add 'name' and 'stddev' to metric-context 45 | 46 | * ``bin:cij_metric_dump``: tool to collect all metrics and dump them to stdout 47 | 48 | ## 0.1.42 49 | 50 | * Add tool ``cij_plotter`` capable of producing plots based of metrics extraced from testcases 51 | 52 | * Add tools for testcase metric extraction and analysis 53 | 54 | * Support for using the Python tools without entering the CIJOE shell 55 | 56 | * Support for multiple testplans per test-run 57 | - Arguments to ``cij_runner`` has changed 58 | Use: ``--testplan`` to provide one or more testplans (this replaces positional arg) 59 | Use: ``--env`` to provide invironment file (this replaces positional arg) 60 | - Structure of ``cij_runner`` output has changed 61 | Before: ``/testsuite_ident/...`` 62 | Now: ``/testplan_ident/testsuite_ident/...`` 63 | In other words, testsuites and nested beneath testplans in the test-result output. 64 | 65 | ## 0.0.35 66 | 67 | * Added option tot disable colors in bash-output-helpers 68 | * Added option to initialize qemu-boot image from cloud-image 69 | * Cleanup SSH module 70 | 71 | ## 0.0.34 72 | 73 | * Fixed typo in sysinf hook 74 | * Removed use of dmesg-hook in `example_01_usage.plan` 75 | * Added `example_02_usage.plan` using the dmesg-hook 76 | 77 | ## 0.0.33 78 | 79 | * Added Dockerfile for interactive use of CIJOE 80 | * Fixed junit-representation 81 | 82 | ## 0.0.32 83 | 84 | * The lock-hook will no longer create lock-files on the test-target, the 85 | lock-hook will as such only protect one-self against one-self, e.g. abort when 86 | the same environment is being used. When using shared resources your resource 87 | manager or CI system should provide mutual exclusion to the test-target. 88 | 89 | * The Makefile now defaults to doing user-mode uninstall + install. 90 | 91 | * Reporter: CSS and JavaScript are now embedded to avoid requiring 92 | network-access to external resources when reading the reports. 93 | 94 | ## 0.0.28 95 | 96 | * Fixes... 97 | 98 | ## 0.0.27 99 | 100 | * mod/qemu: added EARLY RESET and examples of device state and error-injection 101 | 102 | ## 0.0.26 103 | 104 | * fixes... 105 | 106 | ## 0.0.25 107 | 108 | * mod/qemu: re-done to align with the NVMe/OCSSD support in `refenv/qemu` 109 | 110 | ## 0.0.24 111 | 112 | * hooks/sysinf: added collection of kernel config 113 | 114 | ## v0.0.23 115 | 116 | * `bin/cij_runner`: added primitive interrupt handler 117 | * hooks: fixed invalid error-messaging 118 | * testcases/tlint: fixed description 119 | 120 | ## v0.0.22 121 | 122 | * mod/fio: fixed showcmd for remote fio 123 | * hooks/pblk: added comment on requirements 124 | * docs: added placeholder for descr. of packages 125 | * selftest: changed messaging on error to reduce confusion 126 | * build: changed messaging on error to reduce confusion 127 | * `bin/cij_tlint`: fixed missing use-of-nonexistant check 128 | 129 | ## v0.0.21 130 | 131 | * selftest: fixed warning-message 132 | * mod/qemu: fixed return of 'qemu::is_running' and added 'qemu::wait' 133 | * mod/qemu: exported path to NVMe/OCSSD device via QEMU_NVME_IMAGE_FPATH 134 | 135 | ## v0.0.20 136 | 137 | * Selftest fix 138 | 139 | ## v0.0.19 140 | 141 | * Refined selftesting for reuse by cijoe packages 142 | 143 | ## v0.0.18 144 | 145 | * Nothing but the version number 146 | 147 | ## v0.0.17 148 | 149 | * Yet another bunch of fixes 150 | * Changed license from BSD-2 to Apache 151 | 152 | ## v0.0.16 153 | 154 | * CI fixes for automatic deployment 155 | 156 | ## v0.0.15 157 | 158 | * Bunch of Python 3 and style fixes 159 | * Deprecated pblk-hooks for specific params 160 | 161 | ## v0.0.14 162 | 163 | * `cij_reporter`: fixed rendering of elapsed wall-clock 164 | * Updated qemu module to match new 'qemu' with OCSSD support in 'qemu-img' 165 | 166 | ## v0.0.13 167 | 168 | * A myriad of cleanup and fixes 169 | 170 | * Deprecated Python Libraries 171 | - nvm.py incomplete Python interface for liblightnvm using CLI 172 | - spdk.py testcases implemented for liblightnvm in Python, this is handled 173 | better by the liblightnvm testcases themselves, hence deprecated 174 | 175 | ## v0.0.12 176 | 177 | * Bumped version number 178 | * Added 'clean' to release-script 179 | 180 | ## v0.0.11 181 | 182 | * Added option to define testcases "inline" in testplan 183 | - It used to rely on a specific testsuite file 184 | - It now uses inline, when it is defined, testsuite otherwise 185 | * Added testplan/testsuite alias 186 | - To be used for briefly describing how a set of testcases relates to the 187 | testplan 188 | * Expanded usage examples 189 | * Fixes to environment sourcing and lnvm module 190 | * Fixed prefix to interactive shell 191 | 192 | ## v0.0.5 193 | 194 | ## Test-Runner 195 | 196 | * Changed `cij_runner` arguments to positional 197 | 198 | ## Example environment definitions 199 | 200 | * Expanded declarations of reference environment 201 | 202 | ## v0.0.4 203 | 204 | Added this CHANGELOG, a bunch of style fixes and a couple of logic fixes, and 205 | some functionality changes. 206 | 207 | # Shell Modules 208 | 209 | * Renamed `get_fib_range` to `cij::get_fib_range` 210 | * `get_exp_2_range` to `cij::get_exp_2_range` 211 | * vdbench: prefixed vars with `VDBENCH_` 212 | 213 | # Tools 214 | 215 | Changed `cij_reporter` it now takes the output path as positional argument 216 | instead of optional named argument. E.g.: 217 | 218 | ```bash 219 | # How it was 220 | cij_reporter --output /path/to/output 221 | 222 | # How it is now 223 | cij_reporter /path/to/output 224 | ``` 225 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | We're glad you want to contribute! This document will help answer common 4 | questions you may have during your first contribution. 5 | 6 | ## Contribution Process 7 | 8 | We have a 3 step process for contributions: 9 | 10 | 1. Commit changes to a git branch, making sure to sign-off those changes for the 11 | [Developer Certificate of Origin](#developer-certification-of-origin-dco). 12 | 2. Create a GitHub Pull Request for your change, following the instructions in 13 | the pull request template. 14 | 3. Perform a [Code Review](#code-review-process) with the project maintainers on 15 | the pull request. 16 | 17 | ### Developer Certification of Origin (DCO) 18 | 19 | Licensing is very important to open source projects. It helps ensure the 20 | software continues to be available under the terms that the author desired. 21 | 22 | CIJOE uses the Apache 2.0 LICENSE to strike a balance between open contribution 23 | and allowing you to use the software however you would like to. 24 | 25 | The license tells you what rights you have that are provided by the copyright 26 | holder. It is important that the contributor fully understands what rights they 27 | are licensing and agrees to them. Sometimes the copyright holder isn't the 28 | contributor, such as when the contributor is doing work on behalf of a company. 29 | 30 | To make a good faith effort to ensure these criteria are met, CIJOE requires 31 | the Developer Certificate of Origin (DCO) process to be followed. 32 | 33 | The DCO is an attestation attached to every contribution made by every 34 | developer. In the commit message of the contribution, the developer simply adds 35 | a Signed-off-by statement and thereby agrees to the DCO, which you can find 36 | below or at . 37 | 38 | ``` 39 | Developer's Certificate of Origin 1.1 40 | 41 | By making a contribution to this project, I certify that: 42 | 43 | (a) The contribution was created in whole or in part by me and I 44 | have the right to submit it under the open source license 45 | indicated in the file; or 46 | 47 | (b) The contribution is based upon previous work that, to the 48 | best of my knowledge, is covered under an appropriate open 49 | source license and I have the right under that license to 50 | submit that work with modifications, whether created in whole 51 | or in part by me, under the same open source license (unless 52 | I am permitted to submit under a different license), as 53 | Indicated in the file; or 54 | 55 | (c) The contribution was provided directly to me by some other 56 | person who certified (a), (b) or (c) and I have not modified 57 | it. 58 | 59 | (d) I understand and agree that this project and the contribution 60 | are public and that a record of the contribution (including 61 | all personal information I submit with it, including my 62 | sign-off) is maintained indefinitely and may be redistributed 63 | consistent with this project or the open source license(s) 64 | involved. 65 | ``` 66 | 67 | #### DCO Sign-Off Methods 68 | 69 | The DCO requires a sign-off message in the following format appear on each 70 | commit in the pull request: 71 | 72 | ``` 73 | Signed-off-by: Simon A. F. Lund 74 | ``` 75 | 76 | The DCO text can either be manually added to your commit body, or you can add 77 | either **-s** or **--signoff** to your usual git commit commands. If you forget 78 | to add the sign-off you can also amend a previous commit with the sign-off by 79 | running **git commit --amend -s**. If you've pushed your changes to GitHub 80 | already you'll need to force push your branch after this with **git push -f**. 81 | -------------------------------------------------------------------------------- /CONTRIBUTORS.md: -------------------------------------------------------------------------------- 1 | CIJOE Contributors 2 | ================== 3 | 4 | The following people have contributed to the implementation of CIJOE: 5 | 6 | * Simon Andreas Frimann Lund, [CNEX Labs, Inc.] 7 | * Yingjun Yang, [CNEX Labs, Inc.] 8 | * Hans Holmberg, [CNEX Labs, Inc.] 9 | * Javier González, [CNEX Labs, Inc.] 10 | * Weifeng Guo, [CNEX Labs, Inc.] 11 | * Niclas Hedam, [IT University of Copenhagen] 12 | 13 | --- 14 | 15 | [CNEX Labs Inc.]: https://www.cnexlabs.com/ 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2022, Simon A. F. Lund 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # This Makefile serves as convenient command-line auto-completion 3 | # 4 | PROJECT_NAME=cijoe 5 | BUILD=pyproject-build 6 | PIPX=pipx 7 | PYTEST="$(shell pipx environment --value PIPX_LOCAL_VENVS)/${PROJECT_NAME}/bin/pytest" 8 | PYTHON_SYS=python3 9 | TWINE=twine 10 | CIJOE_VERSION=$(shell cd src; python3 -c "from cijoe import core;print(core.__version__)") 11 | 12 | define default-help 13 | # invoke: 'make uninstall', 'make install' 14 | endef 15 | .PHONY: default 16 | default: build 17 | @echo "## ${PROJECT_NAME}: make default" 18 | @echo "## ${PROJECT_NAME}: make default [DONE]" 19 | 20 | define all-help 21 | # Do all: clean uninstall build install 22 | endef 23 | .PHONY: all 24 | all: uninstall clean deps build install info test 25 | 26 | define deps-help 27 | # Dependencies for building cijoe and uploading it to PyPI 28 | endef 29 | .PHONY: deps 30 | deps: 31 | ${PIPX} install build || true 32 | ${PIPX} install twine || true 33 | 34 | define info-help 35 | # Dump various Python / tooling information 36 | endef 37 | .PHONY: info 38 | info: 39 | @echo "## ${PROJECT_NAME}: make info" 40 | ${BUILD} --version || true 41 | ${PIPX} --version || true 42 | ${PIPX} environment || true 43 | ${PYTEST} --version || true 44 | ${PYTHON_SYS} --version || true 45 | ${TWINE} --version || true 46 | @echo "## ${PROJECT_NAME}: make info [DONE]" 47 | 48 | define docker-help 49 | # drop into a docker instance with the repository bind-mounted at /tmp/source 50 | endef 51 | .PHONY: docker 52 | docker: 53 | @echo "## ${PROJECT_NAME}: docker" 54 | @docker run -it \ 55 | -w /tmp/source \ 56 | --mount type=bind,source="$(shell pwd)",target=/tmp/source \ 57 | ghcr.io/refenv/cijoe-docker:latest \ 58 | bash 59 | @echo "## ${PROJECT_NAME}: docker [DONE]" 60 | 61 | 62 | define docker-build-help 63 | # Build docker image 64 | endef 65 | .PHONY: docker-build 66 | docker-build: 67 | @echo "## ${PROJECT_NAME}: docker" 68 | @docker buildx build \ 69 | --build-arg CIJOE_VERSION=${CIJOE_VERSION} \ 70 | --tag ghcr.io/refenv/cijoe-docker:latest \ 71 | --file .github/cijoe-docker/Dockerfile \ 72 | . 73 | @echo "## ${PROJECT_NAME}: docker [DONE]" 74 | 75 | define docker-kvm-help 76 | # drop into a kvm-able docker instance with the repository bind-mounted at /tmp/source 77 | # 78 | # Previously, this utilized the "--privileged" flag this is replaced with 79 | # "--device / dev/kvm" to avoid unnecessary privilege escalation / collision 80 | # when running multiple instances. 81 | endef 82 | .PHONY: docker-kvm 83 | docker-kvm: 84 | @echo "## ${PROJECT_NAME}: docker" 85 | @docker run -it \ 86 | --device=/dev/kvm \ 87 | --device=/dev/fuse \ 88 | --cap-add=SYS_ADMIN \ 89 | --security-opt apparmor=unconfined \ 90 | -w /tmp/source \ 91 | --mount type=bind,source="$(shell pwd)",target=/tmp/source \ 92 | ghcr.io/refenv/cijoe-docker \ 93 | bash 94 | @echo "## ${PROJECT_NAME}: docker [DONE]" 95 | 96 | define format-help 97 | # run code format (style, code-conventions and language-integrity) on staged changes 98 | endef 99 | .PHONY: format 100 | format: 101 | @echo "## ${PROJECT_NAME}: format" 102 | @pre-commit run 103 | @echo "## ${PROJECT_NAME}: format [DONE]" 104 | 105 | define format-all-help 106 | # run code format (style, code-conventions and language-integrity) on staged and committed changes 107 | endef 108 | .PHONY: format-all 109 | format-all: 110 | @echo "## ${PROJECT_NAME}: format-all" 111 | @pre-commit run --all-files 112 | @echo "## ${PROJECT_NAME}: format-all [DONE]" 113 | 114 | define build-help 115 | # Build the package (sdist and wheel using sdist) 116 | endef 117 | .PHONY: build 118 | build: 119 | @echo "## ${PROJECT_NAME}: make build" 120 | @${BUILD} 121 | @echo "## ${PROJECT_NAME}: make build [DONE]" 122 | 123 | define install-help 124 | # install for current user 125 | endef 126 | .PHONY: install 127 | install: 128 | @echo "## ${PROJECT_NAME}: make install" 129 | @${PIPX} install dist/*.tar.gz --force --python python3 130 | @${PIPX} inject cijoe coverage --include-apps --include-deps --force 131 | @${PIPX} inject cijoe pytest-cov --force 132 | @echo "## ${PROJECT_NAME}: make install [DONE]" 133 | 134 | define install-source-help 135 | # install for current user 136 | endef 137 | .PHONY: install-source 138 | install-source: 139 | @echo "## ${PROJECT_NAME}: make install" 140 | @${PIPX} install . --editable --force --python python3 141 | @${PIPX} inject cijoe coverage --include-apps --include-deps --force 142 | @${PIPX} inject cijoe pytest-cov --force 143 | @echo "## ${PROJECT_NAME}: make install [DONE]" 144 | 145 | define uninstall-help 146 | # uninstall 147 | # 148 | # Prefix with 'sudo' when uninstalling a system-wide installation 149 | endef 150 | .PHONY: uninstall 151 | uninstall: 152 | @echo "## ${PROJECT_NAME}: make uninstall" 153 | @${PIPX} uninstall ${PROJECT_NAME} || echo "Cannot uninstall => That is OK" 154 | @echo "## ${PROJECT_NAME}: make uninstall [DONE]" 155 | 156 | define test-help 157 | # Run pytest on the testcase-test 158 | endef 159 | .PHONY: test 160 | test: 161 | @echo "## ${PROJECT_NAME}: make test" 162 | @${PYTEST} --cov --cov-branch --config src/cijoe/core/configs/example_config_default.toml -s 163 | @echo "## ${PROJECT_NAME}: make test [DONE]" 164 | 165 | define release-help 166 | # Run release with twine 167 | endef 168 | .PHONY: release 169 | release: all 170 | @echo "## ${PROJECT_NAME}: make release" 171 | @echo -n "# rel: "; date 172 | @${TWINE} upload dist/* 173 | @echo "## ${PROJECT_NAME}: make release" 174 | 175 | define clean-help 176 | # clean the Python build dirs (build, dist) 177 | endef 178 | .PHONY: clean 179 | clean: 180 | @echo "## ${PROJECT_NAME}: clean" 181 | @git clean -fdx || echo "Failed git-clean ==> That is OK" 182 | @echo "## ${PROJECT_NAME}: clean [DONE]" 183 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | cijoe: tools for systems development and testing 2 | ================================================ 3 | 4 | .. image:: https://img.shields.io/pypi/v/cijoe.svg 5 | :target: https://pypi.org/project/cijoe 6 | :alt: PyPI 7 | 8 | .. image:: https://github.com/refenv/cijoe/workflows/selftest/badge.svg 9 | :target: https://github.com/refenv/cijoe/actions 10 | :alt: Build Status 11 | 12 | .. image:: https://readthedocs.org/projects/cijoe/badge/?version=latest 13 | :target: https://cijoe.readthedocs.io/en/latest/?badge=latest 14 | :alt: Documentation Status 15 | 16 | .. image:: https://coveralls.io/repos/github/refenv/cijoe/badge.svg?branch=main 17 | :target: https://coveralls.io/github/refenv/cijoe?branch=main 18 | :alt: Coverage status 19 | 20 | Tools for systems development and testing. 21 | 22 | Please take a look at the documentation for how to install and use ``cijoe``: 23 | 24 | * `Quickstart Guide`_ 25 | * `Usage`_ 26 | 27 | If you find bugs or need help then feel free to submit an `Issue`_. If you want 28 | to get involved head over to the `GitHub page`_ to get the source code and 29 | submit a `Pull request`_ with your changes. 30 | 31 | .. _Quickstart Guide: https://cijoe.readthedocs.io/ 32 | .. _Usage: https://cijoe.readthedocs.io/ 33 | .. _GitHub page: https://github.com/refenv/cijoe 34 | .. _Pull request: https://github.com/refenv/cijoe/pulls 35 | .. _Issue: https://github.com/refenv/cijoe/issues 36 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | default.config 2 | example.workflow 3 | venv 4 | cijoe-archive 5 | cijoe-config.toml 6 | cijoe-script.py 7 | cijoe-workflow.yaml 8 | -------------------------------------------------------------------------------- /docs/Dockerfile: -------------------------------------------------------------------------------- 1 | # Base image 2 | FROM fedora:40 3 | 4 | # Install required packages: python3, pipx, make, and git 5 | RUN dnf install -y \ 6 | git \ 7 | hostname \ 8 | make \ 9 | pciutils \ 10 | pipx \ 11 | pyproject-rpm-macros \ 12 | python3 \ 13 | python3-pip \ 14 | && \ 15 | dnf clean all \ 16 | && \ 17 | dnf autoremove 18 | 19 | # Create a non-root user and group, set home directory 20 | RUN useradd -m -s /bin/bash developer \ 21 | && mkdir /cijoe \ 22 | && chown developer:developer /cijoe 23 | 24 | # Switch to the non-root user 25 | USER developer 26 | 27 | # Set working directory for the container 28 | WORKDIR /home/developer 29 | 30 | # Ensure PATH and environment is correct for pipx 31 | ENV PIPX_HOME=/home/developer/.local/pipx 32 | ENV PATH=/home/developer/.local/bin:/home/developer/.local/pipx/venvs/bin:$PATH 33 | 34 | # Configure pipx for the non-root user 35 | RUN pipx ensurepath 36 | 37 | # Default command 38 | CMD ["/bin/bash"] 39 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | SPHINX_SOURCEDIR=source 2 | SPHINX_BUILDDIR=build 3 | PYTHON=$(shell pipx environment --value PIPX_LOCAL_VENVS)/cijoe/bin/python 4 | 5 | .PHONY: common 6 | common: clean build view 7 | 8 | .PHONY: all 9 | all: venv-setup kmdo clean build view 10 | 11 | define build-help 12 | # Remove auto-generated documentation the source-dir and the generated docs in build-dir 13 | endef 14 | .PHONY: clean 15 | clean: 16 | @echo "## clean" 17 | rm -rf "${SPHINX_BUILDDIR}" 18 | rm -rf "${SPHINX_SOURCEDIR}/api" 19 | 20 | .PHONY: emit-section-packages 21 | emit-section-packages: 22 | @echo "## section:packages" 23 | @rm -rf "${SPHINX_SOURCEDIR}/packages/" 24 | ${PYTHON} ./emit.py 25 | 26 | .PHONY: docker-build 27 | docker-build: 28 | docker build -t cijoe-docgen . 29 | 30 | .PHONY: docker 31 | docker: docker-build 32 | docker run -it --rm \ 33 | -v $(shell pwd)/../:/cijoe \ 34 | --user $(shell id -u):$(shell id -g) \ 35 | cijoe-docgen \ 36 | bash -c "cd /cijoe && make all && cd docs && make all" 37 | 38 | define apidoc-help 39 | # Generate api documentation using Sphinx autodoc 40 | endef 41 | .PHONY: apidoc 42 | apidoc: 43 | @echo "## apidoc" 44 | sphinx-apidoc --implicit-namespaces -e -H "API" -M -f --tocfile index -o ${SPHINX_SOURCEDIR}/api ../src/cijoe 45 | 46 | define kmdo-help 47 | # Generate api documentation using Sphinx autodoc 48 | endef 49 | .PHONY: kmdo 50 | kmdo: 51 | @echo "## kmdo" 52 | kmdo source 53 | 54 | define build-help 55 | # Build the documentation (invoke sphinx) 56 | endef 57 | .PHONY: build 58 | build: 59 | @echo "## build" 60 | sphinx-build -b html ${SPHINX_SOURCEDIR} ${SPHINX_BUILDDIR} 61 | 62 | define build-help 63 | # Open the HTML documentation 64 | endef 65 | .PHONY: view 66 | view: 67 | @echo "## open docs" 68 | xdg-open "${SPHINX_BUILDDIR}/index.html" || open "${SPHINX_BUILDDIR}/index.html" 69 | 70 | define build-py-env-help 71 | # Setup a virtual environment using pipx 72 | endef 73 | .PHONY: venv-setup 74 | venv-setup: 75 | pipx install ../dist/cijoe-*.tar.gz --force --include-deps 76 | xargs -a requirements.txt -I {} pipx inject cijoe {} --force --include-deps 77 | -------------------------------------------------------------------------------- /docs/emit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Produce documentation for all cijoe packages 4 | """ 5 | import re 6 | import subprocess 7 | from pathlib import Path 8 | from pprint import pprint 9 | 10 | from jinja2 import Template 11 | 12 | from cijoe.core.resources import get_resources 13 | 14 | DOC_ROOT_PKGS = "packages" 15 | 16 | TEMPLATE_PKG_OVERVIEW = """.. 17 | .. This file is generated by emit.py any manual changes are overwritten 18 | .. 19 | 20 | .. _sec-packages: 21 | 22 | ========== 23 | Packages 24 | ========== 25 | 26 | A **cijoe** package is a collection of :ref:`sec-resources`, which may optionally 27 | include a Python module, packaged within a Python package. Including a Python module is 28 | useful when multiple :ref:`sec-resources-scripts` share common implementation details. 29 | 30 | The packages covered in this section are **built-in**, meaning they are included by 31 | default with the **cijoe** installation. You can also create your own package, and the 32 | **cijoe** infrastructure will automatically recognize it, allowing it to be loaded in 33 | the same way as the **built-in** :ref:`sec-resources`. 34 | 35 | If you prefer not to create and distribute a Python package via PyPi, **cijoe** can 36 | still collect and use your locally available :ref:`sec-resources` without additional 37 | steps. 38 | 39 | .. toctree:: 40 | :maxdepth: 2 41 | :hidden: 42 | 43 | {% for pkg_name, script_name in packages.items() %} 44 | {{ pkg_name }}/index.rst 45 | {%- endfor %} 46 | """ 47 | 48 | TEMPLATE_PKG_INDEX = """ 49 | .. _sec-packages-{{ pkg_name }}: 50 | 51 | {{ pkg_name }} 52 | {% set pkg_name_len = pkg_name | length -%} 53 | {{ "=" * pkg_name_len }} 54 | 55 | These are the scripts provided in the package, they are listed by the **full** 56 | name that you can use to refer to them in a workflow. 57 | 58 | Scripts 59 | ------- 60 | 61 | .. toctree:: 62 | :maxdepth: 2 63 | :hidden: 64 | 65 | {% for script_name in scripts %} 66 | scripts/{{ script_name }}.rst 67 | {%- endfor %} 68 | """ 69 | 70 | TEMPLATE_SCRIPT_INDEX = """ 71 | .. _sec-packages-{{ pkg_name }}-{{script_name}}: 72 | 73 | {% set script_title = pkg_name + '.' + script_name -%} 74 | {% set script_title_len = script_title | length -%} 75 | {{ script_title }} 76 | {{ "~" * script_title_len }} 77 | 78 | .. automodule:: cijoe.{{ pkg_name }}.scripts.{{ script_name }} 79 | :members: 80 | 81 | CLI arguments 82 | ------------- 83 | {{ options }} 84 | """ 85 | 86 | 87 | def setup_templates(): 88 | return { 89 | "pkg_overview": Template(TEMPLATE_PKG_OVERVIEW), 90 | "pkg_index": Template(TEMPLATE_PKG_INDEX), 91 | "script_index": Template(TEMPLATE_SCRIPT_INDEX), 92 | } 93 | 94 | 95 | def main(): 96 | packages = {} 97 | templates = setup_templates() 98 | 99 | for key, props in get_resources().get("scripts").items(): 100 | if "." not in key: 101 | continue 102 | 103 | pkg_name, script_name = key.split(".") 104 | if pkg_name not in packages: 105 | packages[pkg_name] = [] 106 | 107 | packages[pkg_name].append(script_name) 108 | packages[pkg_name].sort() 109 | 110 | # Create package page 111 | pkgs_index_rst = Path("source") / DOC_ROOT_PKGS / "index.rst" 112 | pkgs_index_rst.parent.mkdir(parents=True, exist_ok=True) 113 | with pkgs_index_rst.open("w") as rst: 114 | rst.write(templates["pkg_overview"].render(packages=packages)) 115 | 116 | for pkg_name, scripts in packages.items(): 117 | pkg_path = Path("source") / DOC_ROOT_PKGS / pkg_name 118 | 119 | # Create package index 120 | pkg_path.mkdir(parents=True, exist_ok=True) 121 | pkg_index_rst = pkg_path / "index.rst" 122 | with pkg_index_rst.open("w") as rst: 123 | rst.write(templates["pkg_index"].render(pkg_name=pkg_name, scripts=scripts)) 124 | 125 | for script_name in scripts: 126 | script_path = pkg_path / "scripts" / f"{script_name}.rst" 127 | 128 | res = subprocess.run( 129 | ["cijoe", f"{pkg_name}.{script_name}", "--help"], 130 | capture_output=True, 131 | text=True, 132 | ) 133 | options = next( 134 | args for args in res.stdout.split("\n\n") if args.startswith("options") 135 | ) 136 | options = re.sub( 137 | r"\n {2,}([^-])", r" \1", options 138 | ) # put arg help on same line as argument. 139 | options = re.sub( 140 | r" (-.*?)\s{2,}(.*)", r"\n* ``\1``\n\n \2", options 141 | ) # setup as bullet points 142 | 143 | # Create package index 144 | script_path.parent.mkdir(parents=True, exist_ok=True) 145 | with script_path.open("w") as rst: 146 | rst.write( 147 | templates["script_index"].render( 148 | pkg_name=pkg_name, script_name=script_name, options=options 149 | ) 150 | ) 151 | 152 | 153 | if __name__ == "__main__": 154 | main() 155 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | cijoe 2 | kmdo 3 | pydata-sphinx-theme 4 | sphinx==8 5 | sphinx-autopackagesummary 6 | sphinxcontrib-bibtex 7 | sphinxcontrib-jquery 8 | sphinx-copybutton 9 | sphinx-tabs -------------------------------------------------------------------------------- /docs/source/000_prep.cmd: -------------------------------------------------------------------------------- 1 | rm -r cijoe-output || true 2 | rm -r cijoe-example-* || true 3 | rm cijoe-output.tar.gz || true 4 | -------------------------------------------------------------------------------- /docs/source/000_prep.out: -------------------------------------------------------------------------------- 1 | 2 | rm: cannot remove 'cijoe-output': No such file or directory 3 | 4 | 5 | rm: cannot remove 'cijoe-example-*': No such file or directory 6 | 7 | 8 | rm: cannot remove 'cijoe-output.tar.gz': No such file or directory 9 | -------------------------------------------------------------------------------- /docs/source/050_usage_help.cmd: -------------------------------------------------------------------------------- 1 | cijoe --help 2 | -------------------------------------------------------------------------------- /docs/source/050_usage_help.out: -------------------------------------------------------------------------------- 1 | usage: cijoe [-h] [--config CONFIG] [--workflow WORKFLOW] [--output OUTPUT] 2 | [--log-level] [--monitor] [--no-report] [--skip-report] 3 | [--tag TAG] [--archive] [--produce-report] [--integrity-check] 4 | [--resources] [--example [EXAMPLE]] [--version] 5 | [step ...] 6 | 7 | options: 8 | -h, --help show this help message and exit 9 | 10 | workflow: 11 | Run workflow at '-w', using config at '-c', and output at '-o' 12 | 13 | step Given a workflow; one or more workflow steps to run. 14 | Else; one or more cijoe Python scripts to run. 15 | (default: None) 16 | --config CONFIG, -c CONFIG 17 | Path to the Configuration file. (default: cijoe- 18 | config.toml) 19 | --workflow WORKFLOW, -w WORKFLOW 20 | Path to workflow file. (default: cijoe-workflow.yaml) 21 | --output OUTPUT, -o OUTPUT 22 | Path to output directory. (default: /cijoe/docs/cijoe- 23 | output) 24 | --log-level, -l Increase log-level. Provide '-l' for info and '-ll' 25 | for debug. (default: None) 26 | --monitor, -m Dump command output to stdout (default: False) 27 | --no-report, -n Skip the producing, and opening, a report at the end 28 | of the workflow-run (default: False) 29 | --skip-report, -s Skip the report opening at the end of the workflow-run 30 | (default: True) 31 | --tag TAG, -t TAG Tags to identify a workflow-run. This will be prefixed 32 | while storing in archive (default: None) 33 | 34 | utilities: 35 | Workflow, and workflow-related utilities 36 | 37 | --archive, -a Move the output at '-o / --output' to 'cijoe- 38 | archive/YYYY-MM-DD_HH:MM:SS (default: False) 39 | --produce-report, -p Produce report, and open it, for output at '-o / 40 | --output' and exit. (default: None) 41 | --integrity-check, -i 42 | Check integrity of workflow at '-w / --workflow' and 43 | exit. (default: False) 44 | --resources, -r List collected resources and exit. (default: False) 45 | --example [EXAMPLE], -e [EXAMPLE] 46 | Emits the given example. When no example is given, 47 | then it prints a list of available examples. (default: 48 | None) 49 | --version, -v Print the version number of 'cijoe' and exit. 50 | (default: False) 51 | 52 | -------------------------------------------------------------------------------- /docs/source/100_inject.cmd: -------------------------------------------------------------------------------- 1 | # Add a Python package to the cijoe venv provided by pipx 2 | pipx inject cijoe matplotlib -------------------------------------------------------------------------------- /docs/source/100_inject.out: -------------------------------------------------------------------------------- 1 | 2 | 3 | injected package matplotlib into venv cijoe 4 | 5 | installing matplotlib... 6 | done! ✨ 🌟 ✨ 7 | -------------------------------------------------------------------------------- /docs/source/150_cijoe_resources.cmd: -------------------------------------------------------------------------------- 1 | cijoe --resources 2 | -------------------------------------------------------------------------------- /docs/source/160_cijoe_resources_compl.cmd: -------------------------------------------------------------------------------- 1 | cijoe -r | grep compl 2 | -------------------------------------------------------------------------------- /docs/source/160_cijoe_resources_compl.out: -------------------------------------------------------------------------------- 1 | - ident: core.cijoe-completions 2 | path: /home/developer/.local/pipx/venvs/cijoe/lib64/python3.12/site-packages/cijoe/core/auxiliary/cijoe-completions 3 | 4 | -------------------------------------------------------------------------------- /docs/source/200_quickstart.cmd: -------------------------------------------------------------------------------- 1 | # Install cijoe into a pipx-managed virtual-environment 2 | pipx install cijoe 3 | 4 | # Print a list of bundled usage examples 5 | cijoe --example 6 | 7 | # Produce example script, config, and workflow 8 | cijoe --example core.default 9 | 10 | # Execute the workflow 11 | cijoe cijoe-example-core.default/cijoe-workflow.yaml \ 12 | --config cijoe-example-core.default/cijoe-config.toml 13 | 14 | -------------------------------------------------------------------------------- /docs/source/200_quickstart.err: -------------------------------------------------------------------------------- 1 | 2 | 3 | 'cijoe' already seems to be installed. Not modifying existing installation in 4 | '/home/safl/.local/share/pipx/venvs/cijoe'. Pass '--force' to force 5 | installation. 6 | 7 | 8 | 9 | 10 | 11 | 12 | core.default 13 | fio.default 14 | gha.default 15 | linux.default 16 | linux.null_blk 17 | qemu.build 18 | qemu.guest_aarch64 19 | qemu.guest_x86_64 20 | system_imaging.aarch64 21 | system_imaging.x86_64 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | ERROR:cli:main(): config(cijoe-config.toml) does not exist; exiting 36 | -------------------------------------------------------------------------------- /docs/source/200_quickstart.out: -------------------------------------------------------------------------------- 1 | 2 | 3 | 'cijoe' already seems to be installed. Not modifying existing installation in 4 | '/home/developer/.local/pipx/venvs/cijoe'. Pass '--force' to force 5 | installation. 6 | 7 | 8 | 9 | 10 | 11 | 12 | core.default 13 | fio.default 14 | gha.default 15 | linux.default 16 | linux.null_blk 17 | qemu.build 18 | qemu.guest_aarch64 19 | qemu.guest_x86_64 20 | system_imaging.aarch64 21 | system_imaging.x86_64 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /docs/source/250_quickstart.cmd: -------------------------------------------------------------------------------- 1 | cijoe --produce-report || true 2 | -------------------------------------------------------------------------------- /docs/source/250_quickstart.out: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /docs/source/250_quickstart.txt: -------------------------------------------------------------------------------- 1 | cijoe --produce-report 2 | -------------------------------------------------------------------------------- /docs/source/300_lint.cmd: -------------------------------------------------------------------------------- 1 | # Check format of workflow and verify existance of the scripts used 2 | cijoe --integrity-check \ 3 | --config cijoe-example-core.default/cijoe-config.toml \ 4 | cijoe-example-core.default/cijoe-workflow.yaml 5 | -------------------------------------------------------------------------------- /docs/source/300_lint.err: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ERROR:cli:main(): config(cijoe-example-core.default/cijoe-config.yaml) does not exist; exiting 5 | -------------------------------------------------------------------------------- /docs/source/300_lint.out: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /docs/source/400_usage_script_all.cmd: -------------------------------------------------------------------------------- 1 | # Run a script directly 2 | cijoe --config cijoe-example-core.default/cijoe-config.toml \ 3 | core.example_script_default -------------------------------------------------------------------------------- /docs/source/400_usage_script_all.out: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /docs/source/400_usage_script_local.cmd: -------------------------------------------------------------------------------- 1 | # Run a local script directly 2 | cijoe --config cijoe-example-core.default/cijoe-config.toml \ 3 | cijoe-example-core.default/cijoe-script.py -------------------------------------------------------------------------------- /docs/source/400_usage_script_local.out: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /docs/source/410_wait.cmd: -------------------------------------------------------------------------------- 1 | sleep 2 2 | -------------------------------------------------------------------------------- /docs/source/410_wait.out: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /docs/source/420_usage_workflow_all.cmd: -------------------------------------------------------------------------------- 1 | # Run the workflow 2 | cijoe cijoe-example-core.default/cijoe-workflow.yaml \ 3 | --config cijoe-example-core.default/cijoe-config.toml 4 | -------------------------------------------------------------------------------- /docs/source/420_usage_workflow_all.out: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /docs/source/430_wait.cmd: -------------------------------------------------------------------------------- 1 | sleep 2 2 | -------------------------------------------------------------------------------- /docs/source/430_wait.out: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /docs/source/450_usage_workflow_subset.cmd: -------------------------------------------------------------------------------- 1 | # Run a subset of the steps in the workflow 2 | cijoe cijoe-example-core.default/cijoe-workflow.yaml \ 3 | --config cijoe-example-core.default/cijoe-config.toml \ 4 | inline_commands 5 | -------------------------------------------------------------------------------- /docs/source/450_usage_workflow_subset.out: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /docs/source/_static/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/docs/source/_static/.keep -------------------------------------------------------------------------------- /docs/source/_static/cijoe-networked.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/docs/source/_static/cijoe-networked.drawio.png -------------------------------------------------------------------------------- /docs/source/_static/environment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/docs/source/_static/environment.png -------------------------------------------------------------------------------- /docs/source/_static/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/docs/source/_static/logo.png -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # For the full list of built-in configuration values, see the documentation: 4 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 5 | 6 | # -- Project information ----------------------------------------------------- 7 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 8 | from cijoe.core import __version__ 9 | 10 | project = "cijoe" 11 | copyright = "2024, Simon A. F. Lund" 12 | author = "Simon A. F. Lund" 13 | release = __version__ 14 | 15 | # -- General configuration --------------------------------------------------- 16 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 17 | 18 | extensions = [ 19 | "pydata_sphinx_theme", 20 | "sphinx_copybutton", 21 | "sphinx.ext.autodoc", 22 | "sphinx.ext.coverage", 23 | "sphinx.ext.extlinks", 24 | "sphinx.ext.imgmath", 25 | "sphinx.ext.napoleon", 26 | "sphinx.ext.todo", 27 | "sphinx_tabs.tabs", 28 | ] 29 | 30 | templates_path = ["_templates"] 31 | exclude_patterns = [] 32 | 33 | # -- Options for HTML output ------------------------------------------------- 34 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 35 | 36 | numfig = True 37 | 38 | html_theme = "pydata_sphinx_theme" 39 | html_static_path = ["_static"] 40 | html_logo = "_static/logo.png" 41 | html_sidebars = { 42 | "configs**": [], 43 | "introduction**": [], 44 | "prereq**": [], 45 | "scripts**": [], 46 | "testrunner**": [], 47 | "usage**": [], 48 | "workflows**": [], 49 | } 50 | html_theme_options = { 51 | "header_links_before_dropdown": 8, 52 | "collapse_navigation": False, 53 | "navigation_depth": 4, 54 | "navigation_with_keys": False, 55 | "navbar_align": "left", 56 | "show_version_warning_banner": True, 57 | } 58 | html_context = { 59 | "default_mode": "dark", 60 | } 61 | 62 | 63 | extlinks = { 64 | "ansible": ("https://www.ansible.com/%s", None), 65 | "argparse": ("https://docs.python.org/3/library/argparse.html%s", None), 66 | "chef": ("https://www.chef.io/%s", None), 67 | "docker": ("https://www.docker.com/%s", None), 68 | "expect": ("https://en.wikipedia.org/wiki/Expect%s", None), 69 | "fabric": ("https://www.fabfile.org/%s", None), 70 | "github": ("https://github.com/%s", None), 71 | "gitlab": ("https://gitlab.com/%s", None), 72 | "invocations": ("https://invocations.readthedocs.io/en/latest/%s", None), 73 | "invoke": ("https://www.pyinvoke.org/%s", None), 74 | "jenkins": ("https://www.jenkins.io/%s", None), 75 | "jinja": ("https://jinja.palletsprojects.com/en/3.1.x/%s", None), 76 | "just": ("https://github.com/casey/just%s", None), 77 | "make": ("https://en.wikipedia.org/wiki/Make_(software)%s", None), 78 | "paramiko": ("https://www.paramiko.org/%s", None), 79 | "paramiko_client": ("https://docs.paramiko.org/en/latest/api/client.html%s", None), 80 | "pep668": ("https://peps.python.org/pep-0668/%s", None), 81 | "pipx": ("https://pypa.github.io/pipx/%s", None), 82 | "posix_sh": ( 83 | "https://pubs.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html%s", 84 | None, 85 | ), 86 | "puppet": ("https://puppet.com/%s", None), 87 | "pytest": ("https://docs.pytest.org/%s", None), 88 | "python_argparse": ("https://docs.python.org/3/library/argparse.html%s", None), 89 | "python": ("https://www.python.org/%s", None), 90 | "python_logging": ("https://docs.python.org/3/library/logging.html%s", None), 91 | "qemu": ("https://www.qemu.org/%s", None), 92 | "toml": ("https://toml.io/en/%s", None), 93 | "travis": ("https://travis-ci.org/%s", None), 94 | "yaml": ("https://yaml.org/%s", None), 95 | "windows_ssh": ( 96 | "https://learn.microsoft.com/en-us/windows-server/administration/openssh/" 97 | "openssh-overview%s", 98 | None, 99 | ), 100 | } 101 | -------------------------------------------------------------------------------- /docs/source/figs/environment.dia: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/docs/source/figs/environment.dia -------------------------------------------------------------------------------- /docs/source/packages/core/index.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-core: 3 | 4 | core 5 | ==== 6 | 7 | These are the scripts provided in the package, they are listed by the **full** 8 | name that you can use to refer to them in a workflow. 9 | 10 | Scripts 11 | ------- 12 | 13 | .. toctree:: 14 | :maxdepth: 2 15 | :hidden: 16 | 17 | 18 | scripts/cmdrunner.rst 19 | scripts/example_script_default.rst 20 | scripts/example_script_testrunner.rst 21 | scripts/get.rst 22 | scripts/put.rst 23 | scripts/reporter.rst 24 | scripts/repository_prep.rst 25 | scripts/testrunner.rst -------------------------------------------------------------------------------- /docs/source/packages/core/scripts/cmdrunner.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-core-cmdrunner: 3 | 4 | core.cmdrunner 5 | ~~~~~~~~~~~~~~ 6 | 7 | .. automodule:: cijoe.core.scripts.cmdrunner 8 | :members: 9 | 10 | CLI arguments 11 | ------------- 12 | options: 13 | 14 | * ``-h, --help`` 15 | 16 | show this help message and exit 17 | 18 | * ``--commands COMMANDS [COMMANDS ...]`` 19 | 20 | The commands to be run 21 | 22 | * ``--transport TRANSPORT`` 23 | 24 | The key of the transport from the cijoe config file on which the commands should be run. Use 'initiator' if the commands should be run locally. Defaults to the first transport in the config file ('initiator' if none are defined). -------------------------------------------------------------------------------- /docs/source/packages/core/scripts/example_script_default.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-core-example_script_default: 3 | 4 | core.example_script_default 5 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~ 6 | 7 | .. automodule:: cijoe.core.scripts.example_script_default 8 | :members: 9 | 10 | CLI arguments 11 | ------------- 12 | options: 13 | 14 | * ``-h, --help`` 15 | 16 | show this help message and exit 17 | 18 | * ``--repeat REPEAT`` 19 | 20 | Amount of times the message will be repeated -------------------------------------------------------------------------------- /docs/source/packages/core/scripts/example_script_testrunner.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-core-example_script_testrunner: 3 | 4 | core.example_script_testrunner 5 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 6 | 7 | .. automodule:: cijoe.core.scripts.example_script_testrunner 8 | :members: 9 | 10 | CLI arguments 11 | ------------- 12 | options: 13 | 14 | * ``-h, --help`` 15 | 16 | show this help message and exit -------------------------------------------------------------------------------- /docs/source/packages/core/scripts/get.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-core-get: 3 | 4 | core.get 5 | ~~~~~~~~ 6 | 7 | .. automodule:: cijoe.core.scripts.get 8 | :members: 9 | 10 | CLI arguments 11 | ------------- 12 | options: 13 | 14 | * ``-h, --help`` 15 | 16 | show this help message and exit 17 | 18 | * ``--src SRC`` 19 | 20 | path to the file on remote machine 21 | 22 | * ``--dst DST`` 23 | 24 | path to where the file should be placed on the initiator 25 | 26 | * ``--transport TRANSPORT`` 27 | 28 | The name of the transport which should be considered as the remote machine. Use 'initiator' if the commands should be run locally. Defaults to the first transport in the config file ('initiator' if none are defined). -------------------------------------------------------------------------------- /docs/source/packages/core/scripts/put.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-core-put: 3 | 4 | core.put 5 | ~~~~~~~~ 6 | 7 | .. automodule:: cijoe.core.scripts.put 8 | :members: 9 | 10 | CLI arguments 11 | ------------- 12 | options: 13 | 14 | * ``-h, --help`` 15 | 16 | show this help message and exit 17 | 18 | * ``--src SRC`` 19 | 20 | path to the file on initiator 21 | 22 | * ``--dst DST`` 23 | 24 | path to where the file should be placed on the remote machine 25 | 26 | * ``--transport TRANSPORT`` 27 | 28 | The name of the transport which should be considered as the remote machine. Use 'initiator' if the commands should be run locally. Defaults to the first transport in the config file ('initiator' if none are defined). -------------------------------------------------------------------------------- /docs/source/packages/core/scripts/reporter.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-core-reporter: 3 | 4 | core.reporter 5 | ~~~~~~~~~~~~~ 6 | 7 | .. automodule:: cijoe.core.scripts.reporter 8 | :members: 9 | 10 | CLI arguments 11 | ------------- 12 | options: 13 | 14 | * ``-h, --help`` 15 | 16 | show this help message and exit 17 | 18 | * ``--report_open {true,false}`` 19 | 20 | Whether or not the generated report should be opened (in a browser) -------------------------------------------------------------------------------- /docs/source/packages/core/scripts/repository_prep.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-core-repository_prep: 3 | 4 | core.repository_prep 5 | ~~~~~~~~~~~~~~~~~~~~ 6 | 7 | .. automodule:: cijoe.core.scripts.repository_prep 8 | :members: 9 | 10 | CLI arguments 11 | ------------- 12 | options: 13 | 14 | * ``-h, --help`` 15 | 16 | show this help message and exit -------------------------------------------------------------------------------- /docs/source/packages/core/scripts/testrunner.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-core-testrunner: 3 | 4 | core.testrunner 5 | ~~~~~~~~~~~~~~~ 6 | 7 | .. automodule:: cijoe.core.scripts.testrunner 8 | :members: 9 | 10 | CLI arguments 11 | ------------- 12 | options: 13 | 14 | * ``-h, --help`` 15 | 16 | show this help message and exit 17 | 18 | * ``--run_local {true,false}`` 19 | 20 | Whether 'pytest' should be executed in same environment as 'cijoe' 21 | 22 | * ``--random_order {true,false}`` 23 | 24 | Whether the tests should be run in random order 25 | 26 | * ``--args ARGS`` 27 | 28 | Additional args given to 'pytest' -------------------------------------------------------------------------------- /docs/source/packages/index.rst: -------------------------------------------------------------------------------- 1 | .. 2 | .. This file is generated by emit.py any manual changes are overwritten 3 | .. 4 | 5 | .. _sec-packages: 6 | 7 | ========== 8 | Packages 9 | ========== 10 | 11 | A **cijoe** package is a collection of :ref:`sec-resources`, which may optionally 12 | include a Python module, packaged within a Python package. Including a Python module is 13 | useful when multiple :ref:`sec-resources-scripts` share common implementation details. 14 | 15 | The packages covered in this section are **built-in**, meaning they are included by 16 | default with the **cijoe** installation. You can also create your own package, and the 17 | **cijoe** infrastructure will automatically recognize it, allowing it to be loaded in 18 | the same way as the **built-in** :ref:`sec-resources`. 19 | 20 | If you prefer not to create and distribute a Python package via PyPi, **cijoe** can 21 | still collect and use your locally available :ref:`sec-resources` without additional 22 | steps. 23 | 24 | .. toctree:: 25 | :maxdepth: 2 26 | :hidden: 27 | 28 | 29 | core/index.rst 30 | linux/index.rst 31 | qemu/index.rst 32 | system_imaging/index.rst -------------------------------------------------------------------------------- /docs/source/packages/linux/index.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-linux: 3 | 4 | linux 5 | ===== 6 | 7 | These are the scripts provided in the package, they are listed by the **full** 8 | name that you can use to refer to them in a workflow. 9 | 10 | Scripts 11 | ------- 12 | 13 | .. toctree:: 14 | :maxdepth: 2 15 | :hidden: 16 | 17 | 18 | scripts/build_kdebs.rst 19 | scripts/null_blk.rst 20 | scripts/sysinfo.rst -------------------------------------------------------------------------------- /docs/source/packages/linux/scripts/build_kdebs.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-linux-build_kdebs: 3 | 4 | linux.build_kdebs 5 | ~~~~~~~~~~~~~~~~~ 6 | 7 | .. automodule:: cijoe.linux.scripts.build_kdebs 8 | :members: 9 | 10 | CLI arguments 11 | ------------- 12 | options: 13 | 14 | * ``-h, --help`` 15 | 16 | show this help message and exit 17 | 18 | * ``--local_version LOCAL_VERSION`` 19 | 20 | Path to local version of kdebs 21 | 22 | * ``--run_local {true,false}`` 23 | 24 | Whether or not to execute in the same environment as 'cijoe'. -------------------------------------------------------------------------------- /docs/source/packages/linux/scripts/null_blk.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-linux-null_blk: 3 | 4 | linux.null_blk 5 | ~~~~~~~~~~~~~~ 6 | 7 | .. automodule:: cijoe.linux.scripts.null_blk 8 | :members: 9 | 10 | CLI arguments 11 | ------------- 12 | options: 13 | 14 | * ``-h, --help`` 15 | 16 | show this help message and exit 17 | 18 | * ``--do {insert,remove}`` 19 | 20 | The commands to be run on the nullblk module. -------------------------------------------------------------------------------- /docs/source/packages/linux/scripts/sysinfo.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-linux-sysinfo: 3 | 4 | linux.sysinfo 5 | ~~~~~~~~~~~~~ 6 | 7 | .. automodule:: cijoe.linux.scripts.sysinfo 8 | :members: 9 | 10 | CLI arguments 11 | ------------- 12 | options: 13 | 14 | * ``-h, --help`` 15 | 16 | show this help message and exit -------------------------------------------------------------------------------- /docs/source/packages/qemu/index.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-qemu: 3 | 4 | qemu 5 | ==== 6 | 7 | These are the scripts provided in the package, they are listed by the **full** 8 | name that you can use to refer to them in a workflow. 9 | 10 | Scripts 11 | ------- 12 | 13 | .. toctree:: 14 | :maxdepth: 2 15 | :hidden: 16 | 17 | 18 | scripts/build.rst 19 | scripts/guest_initialize.rst 20 | scripts/guest_kill.rst 21 | scripts/guest_start.rst 22 | scripts/install.rst 23 | scripts/qemu_version.rst -------------------------------------------------------------------------------- /docs/source/packages/qemu/scripts/build.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-qemu-build: 3 | 4 | qemu.build 5 | ~~~~~~~~~~ 6 | 7 | .. automodule:: cijoe.qemu.scripts.build 8 | :members: 9 | 10 | CLI arguments 11 | ------------- 12 | options: 13 | 14 | * ``-h, --help`` 15 | 16 | show this help message and exit -------------------------------------------------------------------------------- /docs/source/packages/qemu/scripts/guest_initialize.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-qemu-guest_initialize: 3 | 4 | qemu.guest_initialize 5 | ~~~~~~~~~~~~~~~~~~~~~ 6 | 7 | .. automodule:: cijoe.qemu.scripts.guest_initialize 8 | :members: 9 | 10 | CLI arguments 11 | ------------- 12 | options: 13 | 14 | * ``-h, --help`` 15 | 16 | show this help message and exit 17 | 18 | * ``--guest_name GUEST_NAME`` 19 | 20 | Name of the qemu guest. 21 | 22 | * ``--system_image_name SYSTEM_IMAGE_NAME`` 23 | 24 | Name of the system image. This will overwrite any system image name defined in the configuration file. -------------------------------------------------------------------------------- /docs/source/packages/qemu/scripts/guest_kill.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-qemu-guest_kill: 3 | 4 | qemu.guest_kill 5 | ~~~~~~~~~~~~~~~ 6 | 7 | .. automodule:: cijoe.qemu.scripts.guest_kill 8 | :members: 9 | 10 | CLI arguments 11 | ------------- 12 | options: 13 | 14 | * ``-h, --help`` 15 | 16 | show this help message and exit 17 | 18 | * ``--guest_name GUEST_NAME`` 19 | 20 | Name of the qemu guest. -------------------------------------------------------------------------------- /docs/source/packages/qemu/scripts/guest_start.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-qemu-guest_start: 3 | 4 | qemu.guest_start 5 | ~~~~~~~~~~~~~~~~ 6 | 7 | .. automodule:: cijoe.qemu.scripts.guest_start 8 | :members: 9 | 10 | CLI arguments 11 | ------------- 12 | options: 13 | 14 | * ``-h, --help`` 15 | 16 | show this help message and exit 17 | 18 | * ``--guest_name GUEST_NAME`` 19 | 20 | Name of the qemu guest. -------------------------------------------------------------------------------- /docs/source/packages/qemu/scripts/install.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-qemu-install: 3 | 4 | qemu.install 5 | ~~~~~~~~~~~~ 6 | 7 | .. automodule:: cijoe.qemu.scripts.install 8 | :members: 9 | 10 | CLI arguments 11 | ------------- 12 | options: 13 | 14 | * ``-h, --help`` 15 | 16 | show this help message and exit -------------------------------------------------------------------------------- /docs/source/packages/qemu/scripts/qemu_version.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-qemu-qemu_version: 3 | 4 | qemu.qemu_version 5 | ~~~~~~~~~~~~~~~~~ 6 | 7 | .. automodule:: cijoe.qemu.scripts.qemu_version 8 | :members: 9 | 10 | CLI arguments 11 | ------------- 12 | options: 13 | 14 | * ``-h, --help`` 15 | 16 | show this help message and exit -------------------------------------------------------------------------------- /docs/source/packages/system_imaging/index.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-system_imaging: 3 | 4 | system_imaging 5 | ============== 6 | 7 | These are the scripts provided in the package, they are listed by the **full** 8 | name that you can use to refer to them in a workflow. 9 | 10 | Scripts 11 | ------- 12 | 13 | .. toctree:: 14 | :maxdepth: 2 15 | :hidden: 16 | 17 | 18 | scripts/diskimage_from_cloudimage.rst 19 | scripts/dockerimage_from_diskimage.rst -------------------------------------------------------------------------------- /docs/source/packages/system_imaging/scripts/diskimage_from_cloudimage.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-system_imaging-diskimage_from_cloudimage: 3 | 4 | system_imaging.diskimage_from_cloudimage 5 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 6 | 7 | .. automodule:: cijoe.system_imaging.scripts.diskimage_from_cloudimage 8 | :members: 9 | 10 | CLI arguments 11 | ------------- 12 | options: 13 | 14 | * ``-h, --help`` 15 | 16 | show this help message and exit 17 | 18 | * ``--pattern PATTERN`` 19 | 20 | Pattern for image names to build -------------------------------------------------------------------------------- /docs/source/packages/system_imaging/scripts/dockerimage_from_diskimage.rst: -------------------------------------------------------------------------------- 1 | 2 | .. _sec-packages-system_imaging-dockerimage_from_diskimage: 3 | 4 | system_imaging.dockerimage_from_diskimage 5 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 6 | 7 | .. automodule:: cijoe.system_imaging.scripts.dockerimage_from_diskimage 8 | :members: 9 | 10 | CLI arguments 11 | ------------- 12 | options: 13 | 14 | * ``-h, --help`` 15 | 16 | show this help message and exit 17 | 18 | * ``--pattern PATTERN`` 19 | 20 | Pattern for image names to build -------------------------------------------------------------------------------- /docs/source/prerequisites/index.rst: -------------------------------------------------------------------------------- 1 | .. _sec-prerequisites: 2 | 3 | Prerequisites 4 | ============= 5 | 6 | **TLDR;** To install and run **cijoe**, the requirements are: 7 | 8 | * **initiator**: :python:`Python <>` and :pipx:`pipx <>` 9 | * **target**: ssh-server 10 | 11 | The diagram below illustrates what the **initiator** and **target** systems 12 | cover. 13 | 14 | .. figure:: ../_static/cijoe-networked.drawio.png 15 | :alt: Development Environment 16 | :align: center 17 | 18 | The "core" agentless functionality of **cijoe**; run commands and transfer 19 | files 20 | 21 | 22 | .. _sec-prerequisites-initiator: 23 | 24 | Initiator 25 | --------- 26 | 27 | The **initiator** is the system where the **cijoe** command-line tool is invoked 28 | and the **cijoe** scripts are being executed, requiring :python:`Python <>` 29 | along with the necessary :python:`Python <>` package dependencies. With the 30 | adoption of :pep668:`PEP 668 <>` by Linux distributions, providing a virtual 31 | environment for Python packages is now mandatory rather than a recommended 32 | practice. 33 | 34 | For command-line utilities, :pipx:`pipx <>` offers a convenient way to install 35 | tools within a :python:`Python <>` virtual environment (venv), ensuring the 36 | correct environment variables are set for CLI endpoints to function properly. 37 | Therefore, the initiator must meet the following requirements: 38 | 39 | * :python:`Python <>` >= 3.9 40 | * :pipx:`pipx <>` 41 | 42 | It is recommended to install :python:`Python <>` and :pipx:`pipx <>` via 43 | the package manager of the **initiator** system. To ensure that **pipx** is 44 | correctly, then run the following: 45 | 46 | .. code-block:: shell 47 | 48 | pipx ensurepath 49 | 50 | .. note:: 51 | The **initiator** uses a pure-Python (3.6+) implementation of the SSHv2 52 | protocol (:paramiko:`Paramiko <>`), thus, it does not interfere with your 53 | existing **SSH** setup and does not rely on system crypto/ssl libraries. 54 | 55 | After running this, reload your environment by either logging out completely or 56 | starting a new shell session. With :python:`Python <>` and :pipx:`pipx <>` in 57 | place, then install **cijoe**:: 58 | 59 | pipx install cijoe --include-deps 60 | 61 | Check that it installed correctly, by invoking the **cijoe** command-line tool: 62 | 63 | .. literalinclude:: ../050_usage_help.cmd 64 | :language: console 65 | 66 | You should then see: 67 | 68 | .. literalinclude:: ../050_usage_help.out 69 | :language: console 70 | 71 | With this in place, then go ahead and check up on 72 | your :ref:`sec-prerequisites-target` configuration. 73 | 74 | .. _sec-prerequisites-target: 75 | 76 | Target 77 | ------ 78 | 79 | The **target** is where **commands** are executed via **ssh** and files 80 | are transferred using **scp**, both of which must be properly installed and 81 | configured. As an agentless system, **cijoe** is minimally intrusive and thus 82 | require no additional software installation on the **target**. 83 | 84 | However, **cijoe** assumes certain conditions in the environment. To simplify 85 | specific tasks, this section outlines these assumptions and provides the 86 | necessary tweaks. 87 | 88 | * You can access the **target** system from the **initiator** via **SSH** 89 | 90 | - E.g. test that you can ``ssh foo@example`` 91 | - You setup your **ssh** credentials in the **cijoe** config file 92 | - The **ssh** credentials on your system is **not** re-used 93 | 94 | This means that the following is available on the **target**: 95 | 96 | * sshd (e.g. openssh-server) 97 | 98 | - This is considered **agentless** since it is not specific to **cijoe** and 99 | it is generally available, even on :windows_ssh:`Windows <>`. 100 | 101 | * scp 102 | 103 | - This comes with the sshd installation 104 | 105 | * The **commands** you specify must exist 106 | 107 | - It is recommended to **not** use shell-specific commands, as doing so 108 | will reduce script portability. Additionally, advanced shell functionality 109 | is usually unnecessary since Python capabilities are available on the 110 | **initiator** side where your **cijoe** script is executing. Therefore, keep 111 | commands simple. 112 | 113 | - If you run a benchmark suite, then install it first. **cijoe** can help you 114 | do so by writing a **cijoe** script that does the installation of the tool, 115 | transfer it with ``cijoe.put()`` and ``cijoe.run()`` to invoke installation. 116 | Or you can use some other means of **provisioning**. 117 | 118 | .. _sec-prerequisites-target-setup: 119 | 120 | Target Setup 121 | ------------ 122 | 123 | The following subsections describe system setup / configuration of the target 124 | **system**. 125 | 126 | .. _sec-prerequisites-target-config-user: 127 | 128 | user 129 | ~~~~ 130 | 131 | These configurations would be considered **unsafe** had they been for a 132 | production environment facing the public internet. However, do keep in mind that 133 | the **target** system will often be a system spun up in an **adhoc** fashion 134 | inside a DevOps infrastructure, cloud-service, or a local lab. 135 | 136 | Should you be using a **target** system where ``root`` login over 137 | **ssh** is not acceptable, then you can use a **non-root** user 138 | with :ref:`sec-prerequisites-target-config-sudo`. 139 | 140 | Or simply use a **non-root** user and do not run any **commands** requiring 141 | elevated privileges. In such a scenario, then request permission for your user 142 | to certain devices and tools that you need. 143 | 144 | .. _sec-prerequisites-target-config-sshd: 145 | 146 | sshd login 147 | ~~~~~~~~~~ 148 | 149 | The following are a couple of sshd settings that it is recommended that you 150 | apply, the configuration file is usually available at e.g. 151 | ``/etc/ssh/sshd_config``: 152 | 153 | .. code-block:: shell 154 | 155 | PermitRootLogin yes 156 | PasswordAuthentication yes 157 | 158 | These options enable, as their names suggest, the ability to log in with the 159 | ``root`` user, and to use **password** authentification. These options are 160 | disabled by default since it can be considered **dangerous** if for instance the 161 | machine is facing the public internet. 162 | 163 | .. _sec-prerequisites-target-config-sudo: 164 | 165 | Passwordless sudo 166 | ~~~~~~~~~~~~~~~~~ 167 | 168 | There are also scenarios, where you want to execute using a **non-root** user. 169 | In such cases, having **sudo** capabilities is desirable. For the user named 170 | ``foo``, then invoke ``visudo`` and add a line such as: 171 | 172 | .. code-block:: shell 173 | 174 | foo ALL=(ALL) NOPASSWD: ALL 175 | 176 | Allowing the ``foo`` user to run ``sudo`` without being prompted for a 177 | password can be especially useful when automating commands. This avoids the 178 | need for additional tools or scripting workarounds, such as those relying 179 | on :expect:`expect-like <>` functionality. By bypassing the password prompt, 180 | you simplify the process of running commands in scripts that require elevated 181 | privileges. 182 | -------------------------------------------------------------------------------- /docs/source/resources/auxiliary/index.rst: -------------------------------------------------------------------------------- 1 | .. _sec-resources-auxiliary: 2 | 3 | Auxiliary 4 | ========= 5 | 6 | Auxiliary files are files which do not fall under any of the other kinds of 7 | resources. These are useful for providing things like scripts for ``fio``, 8 | configuration files for some system, hardware specifications, and other means 9 | of data. 10 | 11 | For the **automatic collection** of resources to find auxiliary files, they 12 | must either: 13 | 14 | a. be located in sub directory of the current working directory (``cwd``) 15 | called ``auxiliary``, or 16 | b. be a :python:`Python <>` script that is **not** considered a **cijoe** 17 | :ref:`script `. 18 | 19 | Auxiliary files can be accessed in the ``auxiliary`` resources. 20 | 21 | .. code-block:: python 22 | 23 | resources = get_resources() 24 | template_path = resources["auxiliary"] -------------------------------------------------------------------------------- /docs/source/resources/index.rst: -------------------------------------------------------------------------------- 1 | .. _sec-resources: 2 | 3 | =========== 4 | Resources 5 | =========== 6 | 7 | In **cijoe** the most essential **resources** are :ref:`sec-resources-scripts`, 8 | :ref:`sec-resources-workflows`, and :ref:`sec-resources-configs`. 9 | In addition to these are :ref:`Auxiliary files `, and 10 | :ref:`sec-resources-templates`. 11 | 12 | Resources are **automatically collected** from installed 13 | **cijoe** :ref:`sec-packages` as well as the current working directory 14 | (``cwd``) and any sub-directory with a max depth of 2. 15 | 16 | When writing a :ref:`script ` then resources are accessed 17 | as follows:: 18 | 19 | from cijoe.core.resources import get_resources 20 | 21 | resources = get_resources() 22 | 23 | 24 | This is convenient as you don't need to worry about the location of 25 | your :ref:`sec-resources-auxiliary` files -- they are readily available. 26 | 27 | On the command-line a quick way to list all available resources: 28 | 29 | 30 | .. literalinclude:: ../150_cijoe_resources.cmd 31 | :language: bash 32 | 33 | 34 | .. literalinclude:: ../150_cijoe_resources.out 35 | :language: bash 36 | 37 | 38 | This is useful when you want to create a modified version of a script, review 39 | its details, or verify that all expected resources are available in your 40 | installation. 41 | 42 | There are gold hidden in the **resources** for the example, **cijoe** provides a 43 | bash-completion script. 44 | 45 | The following sections describe the different types of resources. 46 | 47 | .. toctree:: 48 | :maxdepth: 2 49 | :includehidden: 50 | :hidden: 51 | 52 | templates/index.rst 53 | auxiliary/index.rst 54 | -------------------------------------------------------------------------------- /docs/source/resources/templates/index.rst: -------------------------------------------------------------------------------- 1 | .. _sec-resources-templates: 2 | 3 | Templates 4 | ========= 5 | 6 | In **cijoe** templates refer to :jinja:`Jinja <>` templates with the 7 | ``.jinja2`` file extension. Any such file that is reachable by the **automatic 8 | collection** of resources, will be available as a template. 9 | 10 | Consider the simple HTML template below with filename ``template.html.jinja2``. 11 | 12 | .. literalinclude:: ./template.html.jinja2 13 | :language: html 14 | 15 | See the official Jinja :jinja:`Template Designer Documentation ` for 16 | more information on how to construct Jinja templates. 17 | 18 | In a **cijoe** :ref:`script `, you can access the HTML 19 | template via the **cijoe** resources. The example script below will populate 20 | the template with the initiator's hostname, generating a new file with the 21 | HTML. 22 | 23 | .. literalinclude:: ./template.py 24 | :language: python -------------------------------------------------------------------------------- /docs/source/resources/templates/template.html.jinja2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |

Hello {{ name }}!

5 | 6 | -------------------------------------------------------------------------------- /docs/source/resources/templates/template.py: -------------------------------------------------------------------------------- 1 | """ 2 | example of using templates 3 | ========================== 4 | 5 | An example of how to use a template from the cijoe resources. 6 | 7 | The script renders a Jinja template with filename `template.html.jinja2` that 8 | takes `name` as a parameter, and creates a new html file from the template 9 | where all instances of "{{ name }}" has been replaced with the initiator's 10 | hostname. 11 | 12 | """ 13 | 14 | from argparse import Namespace 15 | 16 | import jinja2 17 | 18 | from cijoe.core.command import Cijoe 19 | from cijoe.core.resources import get_resources 20 | 21 | 22 | def main(args: Namespace, cijoe: Cijoe): 23 | resources = get_resources() 24 | template_path = resources["templates"]["template.html"].path 25 | 26 | jinja_env = jinja2.Environment( 27 | autoescape=True, loader=jinja2.FileSystemLoader(template_path.parent) 28 | ) 29 | template = jinja_env.get_template(template_path.name) 30 | 31 | err, state = cijoe.run_local("hostname") 32 | if err: 33 | return err 34 | 35 | hostname = state.output().strip() 36 | 37 | with open("hello.html", "a") as file: 38 | html = template.render({"name": hostname}) 39 | file.write(html) 40 | 41 | return 0 42 | -------------------------------------------------------------------------------- /docs/source/testrunner/index.rst: -------------------------------------------------------------------------------- 1 | .. _sec-usage-testrunner: 2 | 3 | ============ 4 | Testrunner 5 | ============ 6 | 7 | **cijoe** provides a test runner implemented as a :pytest:`pytest <>` plugin, 8 | wrapped in a **cijoe** script named :ref:`core.testrunner `. 9 | The plugin is included with the **cijoe** package, but to use it, you must ensure 10 | that :pytest:`pytest<>` has access to the rest of the **cijoe** virtual 11 | environment (venv) and that the ``pytest`` command-line tool is available. 12 | 13 | This can be easily achieved by using **pipx** to install **cijoe** with its 14 | dependencies: 15 | 16 | .. code-block:: bash 17 | 18 | pipx install cijoe --include-deps 19 | 20 | The test runner can be used in two main ways: 21 | 22 | 1. Directly via the :pytest:`pytest <>` command-line: ``pytest --config cijoe-config.toml`` 23 | 24 | 2. Via a **cijoe** workflow, with a step using the :ref:`core.testrunner 25 | `, executed through the ``cijoe`` command-line tool. 26 | 27 | While the first method may be more familiar and require no further explanation, 28 | the test runner was specifically designed to be used within a **cijoe** 29 | workflow and command-line interface. 30 | 31 | The intent of using :pytest:`pytest <>` in this context is based on the 32 | assumption that, since **cijoe** is a Python project and the :ref:`scripts 33 | ` are also Python-based, the users of **cijoe** are 34 | likely to be familiar with writing tests using :pytest:`pytest <>`. They are 35 | presumed to be aware of general pytest usage and capabilities, allowing them to 36 | leverage that prior knowledge. 37 | 38 | However, there are key differences in how :pytest:`pytest <>` is applied here, 39 | which may seem unfamiliar or awkward to experienced :pytest:`pytest <>` users. 40 | The focus of the following subsections is to highlight and clarify these 41 | essential differences. 42 | 43 | Usage 44 | ===== 45 | 46 | In a :ref:`workflow ` the :ref:`core.testrunner 47 | ` is inserted as a step with arguments like below: 48 | 49 | .. code-block:: yaml 50 | 51 | - name: run_tests 52 | uses: core.testrunner 53 | with: 54 | args: '-k "filtering" my_tests' 55 | random_order: false 56 | run_local: false 57 | 58 | This will result in the following invocation on the initiator: 59 | 60 | .. code-block:: bash 61 | 62 | pytest \ 63 | --config cijoe-config.toml \ 64 | --output output \ 65 | -k "filtering" my_tests 66 | 67 | The key difference between invoking the ``pytest`` command-line tool directly 68 | and using the **cijoe** script :ref:`core.testrunner ` 69 | in the **cijoe** workflow is that the latter integrates the **pytest** report into 70 | **cijoe**, producing a cohesive and standalone report. -------------------------------------------------------------------------------- /docs/source/usage/index.rst: -------------------------------------------------------------------------------- 1 | .. _sec-usage: 2 | 3 | ======= 4 | Usage 5 | ======= 6 | 7 | The entry point for **usage** of **cijoe** is the command-line tool ``cijoe``. 8 | Much like :ansible:`Ansible <>` requires playbooks and inventories, 9 | :make:`make <>` needs a ``Makefile``, and GitHub Actions relies on workflows, 10 | then **cijoe** also requires input to function, these are: 11 | 12 | - An individual :ref:`script ` to execute **or** a 13 | collection of :ref:`scripts `, ordered, and documented 14 | in a :ref:`workflow ` 15 | - A :ref:`configuration file `, providing all the values 16 | that your :ref:`script(s) ` need 17 | 18 | For guidance on creating these files, refer to the :ref:`sec-resources` section. 19 | For the rest of the :ref:`sec-usage` section, and subsections, we will use the 20 | example script, workflow, and configuration file generated by running: 21 | 22 | .. literalinclude:: ../200_quickstart.cmd 23 | :lines: 4-5 24 | 25 | The command above by default produces the example provided by the **core** 26 | **cijoe** package, however, as you can see in section :ref:`sec-packages`, then 27 | there are multiple packages and examples for using them are produced in the same 28 | manner: 29 | 30 | * ``cijoe --example qemu`` 31 | 32 | - Producing example resources for the :ref:`qemu package ` 33 | 34 | * ``cijoe --example linux`` 35 | 36 | - Producing example resources for the :ref:`Linux package ` 37 | 38 | The following sections describe the use of :ref:`sec-resources-scripts`, 39 | :ref:`sec-resources-workflows`, the remainder of the current section 40 | provides subsections with information provided for reference on 41 | all :ref:`sec-usage-cli`, :ref:`sec-usage-evars`, and behaverial information 42 | on :ref:`sec-usage-sp`. 43 | 44 | 45 | .. _sec-usage-cli: 46 | 47 | CLI Arguments 48 | ============= 49 | 50 | When in doubt, then you can always consult the ``cijoe`` command-line arguments: 51 | 52 | .. literalinclude:: ../050_usage_help.cmd 53 | 54 | Which yields the following output: 55 | 56 | .. literalinclude:: ../050_usage_help.out 57 | 58 | 59 | .. _sec-usage-sp: 60 | 61 | Search Paths 62 | ============ 63 | 64 | The :ref:`sec-usage-cli` for the positional argument, and config-files 65 | (``--c / --config``) and workflows (``-w / --workflow``) by default search for files 66 | named ``cijoe-workflow.yaml`` and ``cijoe-config.toml``, respectfully. These files 67 | are searched for, in order, in the following locations: 68 | 69 | ``$PWD`` 70 | In your current working directory 71 | 72 | ``$PWD/.cijoe`` 73 | In the subfolder named ``.cijoe`` of your current working directory 74 | 75 | ``$HOME/.cijoe`` 76 | In a subfolder of your home-directory named ``.cijoe`` 77 | 78 | ``$HOME/.config/cijoe`` 79 | In a subfolder of of the ``.config`` folder in your home-directory named 80 | ``cijoe`` 81 | 82 | In addition to these search paths for the **cijoe** configuration file, then 83 | the :ref:`environment variable ` named ``CIJOE_DEFAULT_CONFIG`` 84 | can be utilized to directly set the path to the configuration file instead 85 | providing it via the ``--config`` command-line option. 86 | 87 | .. _sec-usage-evars: 88 | 89 | Environment Variables 90 | ===================== 91 | 92 | The following environment variables modify the bahavior of the ``cijoe`` 93 | command-line tool. 94 | 95 | CIJOE_DISABLE_SSH_ENV_INJECT 96 | When this is set, the environment variables passed to 97 | ``cijoe.run(..., env={your: vars})`` will not be passed on to the SSH 98 | transport. 99 | 100 | CIJOE_DEFAULT_CONFIG 101 | When set, the value will be used as the default for the command-line 102 | ``-c/--config`` argument. 103 | 104 | CIJOE_DEFAULT_WORKFLOW 105 | When set, the value will be used as the default for the positional 106 | command-line argument. 107 | 108 | 109 | .. _sec-usage-docker: 110 | 111 | Docker Image 112 | ============ 113 | 114 | There is a **cijoe** Docker image available at 115 | :github:`GitHub `, which contains 116 | everything needed to build, install and run **cijoe**. This includes all the 117 | tools required by the built-in packages of **cijoe**, for example 118 | :docker:`Docker <>` and :qemu:`QEMU <>`. 119 | 120 | 121 | A prebuilt **cijoe** Docker image is available at 122 | :github:`GitHub `. This image provides 123 | a ready-to-use environment containing all necessary dependencies to build, install, 124 | and run **cijoe**. It includes tools required by built-in **cijoe** packages, such as 125 | :docker:`Docker <>` and :qemu:`QEMU <>`. 126 | 127 | 128 | Pulling the Docker Image 129 | ------------------------ 130 | 131 | To download the latest version of the **cijoe** Docker image, use: 132 | 133 | .. code-block:: bash 134 | 135 | docker pull ghcr.io/refenv/cijoe-docker:latest 136 | 137 | For a specific version, replace `latest` with the desired tag, e.g.: 138 | 139 | .. code-block:: bash 140 | 141 | docker pull ghcr.io/refenv/cijoe-docker:v0.9.50 142 | 143 | Using the `latest` tag ensures you get the newest available image, while 144 | using a specific version provides stability across runs. 145 | 146 | 147 | GitHub Actions 148 | -------------- 149 | 150 | The following GitHub Actions workflow demonstrates how to use the **cijoe** 151 | Docker image to run the `core.default` example and upload the **cijoe** report 152 | as an artifact. 153 | 154 | .. code-block:: yaml 155 | 156 | name: cijoe_example 157 | 158 | on: 159 | workflow_dispatch: 160 | 161 | jobs: 162 | run_cijoe_example: 163 | runs-on: ubuntu-22.04 164 | container: 165 | image: ghcr.io/refenv/cijoe-docker:latest 166 | 167 | steps: 168 | - name: Set up Python 3.12 169 | uses: actions/setup-python@v5.3.0 170 | with: 171 | python-version: "3.12" 172 | 173 | - name: Generate cijoe example 174 | run: | 175 | $(which cijoe) --example core.default 176 | mv ./cijoe-example-core.default ./example 177 | 178 | - name: Execute workflow 179 | run: | 180 | $(which cijoe) --monitor -l \ 181 | --config ./example/cijoe-config.toml \ 182 | --workflow ./example/cijoe-workflow.yaml 183 | 184 | - name: Upload report 185 | if: always() 186 | uses: actions/upload-artifact@v4.3.0 187 | with: 188 | name: report-cijoe_example 189 | path: cijoe-output/* 190 | if-no-files-found: error -------------------------------------------------------------------------------- /docs/source/workflows/index.rst: -------------------------------------------------------------------------------- 1 | .. _sec-resources-workflows: 2 | 3 | =========== 4 | Workflows 5 | =========== 6 | 7 | Workflows enable the organized execution of commands and scripts. After 8 | execution, a report is generated, containing the status and embedded 9 | documentation of the workflow and scripts in a self-contained format. To run the 10 | workflow produced by ``cijoe --example``, use the following command: 11 | 12 | .. literalinclude:: ../420_usage_workflow_all.cmd 13 | :language: bash 14 | 15 | The command will execute **all** the 16 | :ref:`steps ` in the workflow. To run a subset of 17 | steps, you can specify the step name(s) as arguments to the ``cijoe`` tool, 18 | similar to how targets are specified in a :make:`Makefile <>`: 19 | 20 | .. literalinclude:: ../450_usage_workflow_subset.cmd 21 | :language: bash 22 | 23 | With the above, only the step named **builtin_script** will be executed. This 24 | becomes even more useful when utilizing **cijoe** bash completions. 25 | 26 | There are a couple of workflow-specific options. See 27 | the :ref:`sec-resources-configs` section for reference. 28 | 29 | 30 | .. _sec-resources-workflows-content: 31 | 32 | Content Overview 33 | ================ 34 | 35 | Let's take a look at what the workflow file produced by ``cijoe --example core.default`` 36 | looks like: 37 | 38 | .. literalinclude:: ../../../src/cijoe/core/workflows/example_workflow_default.yaml 39 | :language: yaml 40 | 41 | At a first glance, then it might feel a bit similar to GitHub Actions workflow, 42 | but dramatically simpler since: 43 | 44 | * There are **no** logic operators 45 | 46 | * There **is** simple variable substitution using 47 | 48 | - Values from configuration file 49 | - Values from environment variables on **initiator** 50 | 51 | * Minimal amount of "magic" keys 52 | 53 | - ``doc``: Describe what the workflow does using multi-line plain-text 54 | - ``steps``: Ordered list of scripts, to inline-commands, to run 55 | 56 | Descriptions of the content is provided in the following subsections. 57 | 58 | .. _sec-resources-workflows-steps: 59 | 60 | Steps 61 | ===== 62 | 63 | Although **cijoe** aims to be simple, with minimal "magic" and a low learning 64 | curve, there is some **yaml-magic** involved in the workflow steps. A step can 65 | take one of two forms: either as :ref:`sec-resources-workflows-inline-commands` 66 | or as :ref:`sec-resources-workflows-step-scripts`. 67 | 68 | Both forms require that a step **must** have a **name**. This allows subsets 69 | of steps to be executed via the ``cijoe`` command-line tool. When naming steps, 70 | follow these conventions: 71 | 72 | * Letters: a-z 73 | * Numbers: 0-9 74 | * Special characters: `-` and `_` 75 | * Must be lowercase 76 | * Must **not** start with a number 77 | 78 | In short, use the typical lowercase identifier convention. 79 | 80 | .. _sec-resources-workflows-inline-commands: 81 | 82 | Inline Commands 83 | --------------- 84 | 85 | A step with **inline commands** take the form: 86 | 87 | .. literalinclude:: ../../../src/cijoe/core/workflows/example_workflow_default.yaml 88 | :language: yaml 89 | :lines: 24-27 90 | 91 | Each line in a multi-line string is executed. It is implemented as a call to 92 | ``cijoe.run(command)``. Thus, the above notation for **inline commands** turn 93 | into execution of functions in the **cijoe** Python module: 94 | 95 | .. code-block:: python 96 | 97 | cijoe.run("cat /proc/cpuinfo") 98 | cijoe.run("hostname") 99 | 100 | .. note:: 101 | This is implemented in **cijoe** as "syntactic-sugar" for 102 | running the built-in script :ref:`core.cmdrunner `. 103 | Thus, have a look at :ref:`sec-resources-workflows-step-scripts` to see what 104 | this **unfolds** as. 105 | 106 | 107 | .. _sec-resources-workflows-step-scripts: 108 | 109 | Steps with Scripts 110 | ------------------ 111 | 112 | When a step runs a script, then you give it a **name** and you tell it 113 | 114 | .. literalinclude:: ../../../src/cijoe/core/workflows/example_workflow_default.yaml 115 | :language: yaml 116 | :lines: 29-34 117 | 118 | Take note of the "magic" keys: 119 | 120 | uses 121 | Name of the script to run, without the ``.py`` extension. Packaged scripts 122 | include a prefix, such as ``core.`` or ``linux.``. As in the 123 | example above where the script ``cmdrunner.py`` from the ``core`` package 124 | is used (:ref:`core.cmdrunner `). 125 | 126 | with 127 | Everything under this key is passed to the script's entry function: 128 | ``main(args, cijoe)`` in the ``args`` argument. 129 | 130 | 131 | .. _sec-resources-workflows-linting: 132 | 133 | Linting 134 | ------- 135 | 136 | When you write a workflow yourself it can be nice to check whether it is valid 137 | without running it. You can do so by running: 138 | 139 | .. literalinclude:: ../300_lint.cmd 140 | :language: bash 141 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=54", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "cijoe" 7 | dynamic = ["version"] 8 | description = "A loosely coupled approach to systems development and testing" 9 | readme = { file = "README.rst", content-type = "text/x-rst" } 10 | license = { file = "LICENSE" } 11 | requires-python = ">=3.9" 12 | keywords = ["systems", "development", "testing"] 13 | authors = [ 14 | { name = "Simon A. F. Lund", email = "os@safl.dk" } 15 | ] 16 | maintainers = [ 17 | { name = "Simon A. F. Lund", email = "os@safl.dk" } 18 | ] 19 | classifiers = [ 20 | "Development Status :: 4 - Beta", 21 | "Environment :: Console", 22 | "Framework :: Pytest", 23 | "Intended Audience :: Developers", 24 | "Intended Audience :: System Administrators", 25 | "License :: OSI Approved :: BSD License", 26 | "Programming Language :: Python", 27 | "Topic :: Software Development :: Testing", 28 | "Topic :: Software Development", 29 | "Topic :: Text Processing", 30 | "Topic :: Utilities" 31 | ] 32 | 33 | dependencies = [ 34 | "jinja2", 35 | "paramiko", 36 | "pytest", 37 | "pytest-random-order>=1.0.0", 38 | "pytest-reportlog", 39 | "pyyaml", 40 | "requests", 41 | "psutil", 42 | "scp", 43 | "tomli>=1.1.0; python_version < '3.11'", 44 | "tomli-w", 45 | "watchdog" 46 | ] 47 | 48 | [project.urls] 49 | homepage = "https://cijoe.readthedocs.io/" 50 | documentation = "https://cijoe.readthedocs.io/" 51 | repository = "https://github.com/refenv/cijoe" 52 | changelog = "https://github.com/refenv/cijoe/blob/main/CHANGELOG.md" 53 | 54 | [project.entry-points."console_scripts"] 55 | cijoe = "cijoe.cli.cli:main" 56 | 57 | [project.entry-points."pytest11"] 58 | cijoe = "cijoe.pytest_plugin.hooks_and_fixtures" 59 | 60 | [tool.setuptools] 61 | zip-safe = false 62 | 63 | [tool.setuptools.packages.find] 64 | where = ["src"] 65 | namespaces = true 66 | 67 | [tool.setuptools.dynamic] 68 | version = { attr = "cijoe.core.__version__" } 69 | 70 | [tool.setuptools.package-data] 71 | "*" = ["*.html", "*.config", "*.toml", "*.yaml", "*.workflow", "*.user", "*.meta", "*.jinja2", "auxiliary/*"] 72 | 73 | [tool.pytest.ini_options] 74 | addopts = "-W ignore::DeprecationWarning" 75 | 76 | [tool.black] 77 | line-length = 88 78 | target-version = ['py310'] 79 | 80 | [tool.isort] 81 | profile = "black" 82 | line_length = 88 83 | 84 | [tool.ruff] 85 | line-length = 88 86 | lint.extend-ignore = ["E203", "F401", "F811", "E501"] 87 | 88 | [tool.mypy] 89 | ignore_missing_imports = true 90 | exclude = "docs/source/conf.py" 91 | 92 | [tool.coverage.run] 93 | branch = true 94 | relative_files = true 95 | source_pkgs = [ 96 | "cijoe.core", 97 | "cijoe.cli", 98 | "cijoe.linux", 99 | "cijoe.qemu", 100 | "cijoe.system_imaging", 101 | ] 102 | 103 | # Setting variable to enable debugging when running coverage 104 | # debug = ["trace", "config", "sys", "data", "premain", "pybehave", "pathmap", "dataio", "plugin", "sql", "sqldata"] 105 | 106 | [tool.coverage.paths] 107 | source = [ 108 | "src/cijoe", 109 | "*/site-packages/cijoe/", 110 | ] 111 | -------------------------------------------------------------------------------- /src/cijoe/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/cli/__init__.py -------------------------------------------------------------------------------- /src/cijoe/cli/__main__.py: -------------------------------------------------------------------------------- 1 | from cijoe.cli import cli 2 | 3 | cli.main() 4 | -------------------------------------------------------------------------------- /src/cijoe/core/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.9.52" 2 | -------------------------------------------------------------------------------- /src/cijoe/core/analyser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Library functions for performance requirements and normalization of metrics 4 | """ 5 | import copy 6 | import dataclasses 7 | import os 8 | import re 9 | from typing import List, Tuple 10 | 11 | import yaml 12 | 13 | from cijoe.core.errors import InvalidRangeError, UnknownUnitError 14 | 15 | UNITS = { 16 | # general 17 | "": 1, # no unit 18 | "B": 1, # bytes 19 | "k": 1000, # kilo 20 | "M": 1000**2, # mega 21 | "G": 1000**3, # giga 22 | # kibi 23 | "KiB": 1024**1, # kibibytes 24 | "MiB": 1024**2, # mibibytes 25 | "GiB": 1024**3, # gibibytes 26 | "TiB": 1024**4, # tibibytes 27 | # kilo 28 | "kB": 1000**1, # kilobytes 29 | "MB": 1000**2, # megabytes 30 | "GB": 1000**3, # gigabytes 31 | "TB": 1000**4, # gigabytes 32 | # time 33 | "nsec": 1 / 1000**3, # nanoseconds 34 | "usec": 1 / 1000**2, # microseconds 35 | "msec": 1 / 1000**1, # milliseconds 36 | "sec": 1, # seconds 37 | "min": 60, # minutes 38 | } 39 | 40 | 41 | class Range: 42 | """ 43 | Range implements parsing and validation of mathematical range notation, 44 | e.g. `[-5;100[` which translates to "must be >= -5 and < 100". 45 | """ 46 | 47 | # pylint: disable=no-self-use 48 | # pylint: disable=too-few-public-methods 49 | 50 | _rng_re = re.compile( 51 | r"^(?P\[|\])\s*(?P-inf|-?\d+(\.\d*)?)\s*;" # [1.0; 52 | r"\s*(?Pinf|-?\d+(\.\d*)?)\s*(?P\[|\])" # 1.0] 53 | rf"\s*(?P({'|'.join(UNITS)}))$" # ms 54 | ) 55 | 56 | def __init__(self, rng: str): 57 | match = self._rng_re.match(rng) 58 | if not match: 59 | raise InvalidRangeError(f'invalid syntax or unit for "{rng}"') 60 | 61 | rng_start = float(match["rstart"]) 62 | rng_end = float(match["rend"]) 63 | if rng_start > rng_end: 64 | raise InvalidRangeError( 65 | "expected lower bound <= upper bound, " f"{rng_start} <= {rng_end}" 66 | ) 67 | 68 | # NOTE: _rng_re enforces that match["unit"] exists in UNITS. 69 | unit_val = UNITS[match["unit"]] 70 | 71 | self._rng_start = rng_start 72 | self._rng_end = rng_end 73 | self._elower = match["elower"] 74 | self._eupper = match["eupper"] 75 | self._unit = match["unit"] 76 | 77 | self._check_lower = self._make_check_lower( 78 | match["elower"], rng_start * unit_val 79 | ) 80 | self._check_upper = self._make_check_upper(match["eupper"], rng_end * unit_val) 81 | 82 | def contains(self, val: float) -> bool: 83 | """Check whether n is contained in range. 84 | 85 | val must be given in the base unit of the measurement, e.g. seconds for 86 | time and bytes for storage. 87 | """ 88 | return self._check_lower(val) and self._check_upper(val) 89 | 90 | def _make_check_lower(self, edge_lower: str, rng_start: float): 91 | if edge_lower == "[": 92 | return lambda n: n >= rng_start 93 | if edge_lower == "]": 94 | return lambda n: n > rng_start 95 | raise InvalidRangeError("invalid input _make_check_lower") 96 | 97 | def _make_check_upper(self, edge_upper: str, rng_end: float): 98 | if edge_upper == "[": 99 | return lambda n: n < rng_end 100 | if edge_upper == "]": 101 | return lambda n: n <= rng_end 102 | raise InvalidRangeError("invalid input _make_check_upper") 103 | 104 | def format_val(self, val: float) -> str: 105 | """Formats and returns val using the unit of the range. 106 | 107 | Example: 108 | range: "[250; 750]usec" 109 | val: 0.0005 110 | output: "500 usec" 111 | """ 112 | 113 | val_conv = val / UNITS[self._unit] 114 | return f"{val_conv:.3f} {self._unit}" 115 | 116 | def __str__(self): 117 | return ( 118 | f"{self._elower}{self._rng_start};" 119 | f"{self._rng_end}{self._eupper} {self._unit}" 120 | ) 121 | 122 | 123 | def to_base_unit(val: float, unit: str = "") -> float: 124 | """Converts val in the given unit to its base unit. 125 | Example: 126 | val: 100, unit: 'KiB' 127 | output: 102400 (bytes) 128 | 129 | val: 500, unit: 'msec' 130 | output: 0.5 (seconds) 131 | """ 132 | unit_scalar = UNITS.get(unit, None) 133 | if not unit_scalar: 134 | raise UnknownUnitError(f"Unit '{unit}' is not supported") 135 | 136 | return val * unit_scalar 137 | -------------------------------------------------------------------------------- /src/cijoe/core/auxiliary/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/core/auxiliary/.keep -------------------------------------------------------------------------------- /src/cijoe/core/auxiliary/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/core/auxiliary/__init__.py -------------------------------------------------------------------------------- /src/cijoe/core/auxiliary/cijoe-completions: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # The function that will perform the autocomplete 4 | _cijoe_completion() { 5 | local cur prev yaml_file toml_file values options 6 | 7 | # Get the current word (the one being completed) and the previous word 8 | cur="${COMP_WORDS[COMP_CWORD]}" 9 | prev="${COMP_WORDS[COMP_CWORD-1]}" 10 | 11 | # Define the possible options for cijoe 12 | options="--help --config -c --workflow -w --output -o --log-level -l --no-report -n --skip-report -s --tag -t --archive -a --produce-report -p --monitor -m --integrity-check -i --resources -r --example -e --version -v" 13 | 14 | # Provide options completion if the current word starts with -- 15 | if [[ "$cur" == --* ]]; then 16 | COMPREPLY=( $(compgen -W "$options" -- "$cur") ) 17 | return 18 | fi 19 | 20 | # Check if the previous word is --config or -c, which specifies the TOML file 21 | if [[ "$prev" == "--config" || "$prev" == "-c" ]]; then 22 | # Complete with TOML files or directories (allow both files and paths) 23 | COMPREPLY=( $(compgen -o plusdirs -f -- "$cur") ) 24 | for i in "${!COMPREPLY[@]}"; do 25 | # Append "/" to directories only if needed and ensure it doesn't append for the full path 26 | if [[ -d "${COMPREPLY[$i]}" ]]; then 27 | if [[ "${cur}" != */ ]]; then 28 | COMPREPLY[$i]="${COMPREPLY[$i]}/" # Append "/" to directories only if it doesn't already have one 29 | fi 30 | elif [[ ! "${COMPREPLY[$i]}" == *.toml ]]; then 31 | unset "COMPREPLY[$i]" # Remove non-TOML files 32 | fi 33 | done 34 | return 35 | fi 36 | 37 | # Check if the previous word is --workflow or -w, which specifies the YAML file 38 | if [[ "$prev" == "--workflow" || "$prev" == "-w" ]]; then 39 | # Complete with YAML files or directories (allow both files and paths) 40 | COMPREPLY=( $(compgen -o plusdirs -f -- "$cur") ) 41 | for i in "${!COMPREPLY[@]}"; do 42 | # Append "/" to directories only if needed and ensure it doesn't append for the full path 43 | if [[ -d "${COMPREPLY[$i]}" ]]; then 44 | if [[ "${cur}" != */ ]]; then 45 | COMPREPLY[$i]="${COMPREPLY[$i]}/" # Append "/" to directories only if it doesn't already have one 46 | fi 47 | elif [[ ! "${COMPREPLY[$i]}" == *.yaml ]]; then 48 | unset "COMPREPLY[$i]" # Remove non-YAML files 49 | fi 50 | done 51 | return 52 | fi 53 | 54 | # Search for the YAML file in the command-line arguments 55 | yaml_file="" 56 | for ((i=1; i.,:;"*': 17 | ident = ident.replace(illegal, "_") 18 | 19 | return ident 20 | 21 | 22 | def download(url: str, path: Path): 23 | """Downloads a file over http(s), returns (err, path).""" 24 | 25 | path = Path(path).resolve() 26 | if path.is_dir(): 27 | path = path / url.split("/")[-1] 28 | if not (path.parent.is_dir() and path.parent.exists()): 29 | return errno.EINVAL, path 30 | 31 | with requests.get(url, stream=True) as request: 32 | request.raise_for_status() 33 | with path.open("wb") as local: 34 | for chunk in request.iter_content(chunk_size=8192): 35 | local.write(chunk) 36 | 37 | return 0, path 38 | 39 | 40 | def get_checksums_from_url(url_checksum: str): 41 | """ 42 | Downloads checksum(s) from given url to a temporary directory, returns 43 | the hashing algorithm and the contents of the file. The algorithm is 44 | found based on the filename of the downloaded checksum file. 45 | Returns (err, checksums, algorithm). 46 | """ 47 | 48 | dir = TemporaryDirectory() 49 | err, path_checksum = download(url_checksum, Path(dir.name).resolve()) 50 | if err: 51 | log.error(f"download({url_checksum}), {path_checksum}: failed") 52 | return err, None, None 53 | 54 | with open(path_checksum, "r") as checksum_f: 55 | # Loop through valid hash algorithms to find the one that matches the 56 | # filename of the checksum 57 | for algorithm in hashlib.algorithms_guaranteed: 58 | if algorithm in path_checksum.name.lower(): 59 | return 0, checksum_f.read(), algorithm 60 | 61 | log.error( 62 | "error: downloaded checksum does contain the name of a valid hash algorithm" 63 | ) 64 | return 1, None, None 65 | 66 | 67 | def download_and_verify(url: str, url_checksum: str, path: Path): 68 | """ 69 | Downloads a file over http(s). The file is only downloaded if checksums are 70 | not equal and if the checksum at url_checksum matches the downloaded file, 71 | returns (err, path). 72 | """ 73 | 74 | path = Path(path).resolve() 75 | if path.is_dir(): 76 | path = path / url.split("/")[-1] 77 | if not (path.parent.is_dir() and path.parent.exists()): 78 | return errno.EINVAL, path 79 | 80 | err, verification, algorithm = get_checksums_from_url(url_checksum) 81 | if err: 82 | log.error(f"error when downloading checksum ({url_checksum})") 83 | 84 | # The checksum of the existing file at path 85 | checksum = None 86 | checksum_path = path.with_suffix(path.suffix + f".{algorithm}sum") 87 | if path.exists() and checksum_path.exists(): 88 | with open(checksum_path, "r") as f: 89 | checksum = f.read() 90 | 91 | # If the file does not already exists or if checksums do not match, 92 | # download from url 93 | if not checksum or checksum not in verification: 94 | log.info(f"Downloading file from {url}") 95 | 96 | with NamedTemporaryFile() as dwnld: 97 | err, dwnld_path = download(url, Path(dwnld.name).resolve()) 98 | if err: 99 | log.error(f"download({url}), {dwnld_path}: failed") 100 | return err, None 101 | 102 | # Check that downloaded checksum file contains the checksum of 103 | # the downloaded file 104 | new_checksum = hashlib.file_digest(dwnld, algorithm).hexdigest() 105 | if new_checksum not in verification: 106 | log.error( 107 | f"error while downloading file: checksum ({new_checksum}) not in checksum file:\n{verification}" 108 | ) 109 | return 1, None 110 | 111 | # Move contents of temporary file to the given path 112 | shutil.copyfile(dwnld_path, path) 113 | with open(checksum_path, "w") as f: 114 | f.write(new_checksum) 115 | 116 | return 0, path 117 | -------------------------------------------------------------------------------- /src/cijoe/core/processing.py: -------------------------------------------------------------------------------- 1 | """ 2 | Helper functions for processing the output produced by a workflow 3 | """ 4 | 5 | import json 6 | import time 7 | from pathlib import Path 8 | from typing import Any, Dict, Union 9 | 10 | from cijoe.core.misc import ENCODING, sanitize_ident 11 | from cijoe.core.resources import dict_from_yamlfile 12 | 13 | 14 | def cmd_number_from_path(path): 15 | """Extracts the numerical part after 'cmd_' in the filename stem.""" 16 | 17 | return int(path.stem.split("_")[1]) 18 | 19 | 20 | def runlog_from_path(path: Path): 21 | """Produce a dict of command-dicts with paths to .output and .state files""" 22 | 23 | run: Dict[str, Dict[str, Any]] = {} 24 | 25 | if not (path.is_dir() and path.exists()): 26 | return run 27 | 28 | for cmd_path in sorted(path.glob("cmd_*.*"), key=cmd_number_from_path): 29 | stem = cmd_path.stem 30 | suffix = cmd_path.suffix[1:] 31 | if suffix not in ["output", "state"]: 32 | continue 33 | 34 | if stem not in run: 35 | run[stem] = { 36 | "output_path": None, 37 | "output": "", 38 | "state": {}, 39 | "state_path": None, 40 | } 41 | 42 | run[stem][f"{suffix}_path"] = cmd_path 43 | if suffix == "output": 44 | with run[stem][f"{suffix}_path"].open( 45 | encoding=ENCODING, errors="replace" 46 | ) as content: 47 | run[stem][f"{suffix}"] = content.read() 48 | elif suffix == "state": 49 | yaml_dict = dict_from_yamlfile(run[stem][f"{suffix}_path"]) 50 | if not yaml_dict["is_done"]: 51 | yaml_dict["elapsed"] = time.time() - yaml_dict["begin"] 52 | run[stem][f"{suffix}"] = yaml_dict 53 | 54 | return run 55 | 56 | 57 | def longrepr_to_string(longrepr): 58 | """Extract pytest crash/traceback/longrepr info from pytest structure""" 59 | 60 | lines = [] 61 | 62 | lines.append("# crashinfo") 63 | reprcrash = longrepr.get("reprcrash", {}) 64 | for key, value in reprcrash.items(): 65 | lines.append(f"{key}: {value}") 66 | 67 | entries = longrepr.get("reprtraceback", {"reprentries": []}).get("reprentries", []) 68 | for entry in entries: 69 | if entry is None: 70 | continue 71 | 72 | data = entry.get("data") 73 | if data is None: 74 | continue 75 | 76 | reprfuncargs = data.get("reprfuncargs") 77 | if reprfuncargs is None: 78 | continue 79 | 80 | reprargs = reprfuncargs.get("args") 81 | if reprargs is None: 82 | continue 83 | 84 | lines.append("") 85 | lines.append("# test-args") 86 | 87 | for argline in reprargs: 88 | lines.append(":".join(argline)) 89 | 90 | lines.append("") 91 | lines.append("# test-output-lines") 92 | for dataline in entry.get("data", {"lines": []}).get("lines", []): 93 | lines.append(dataline) 94 | 95 | return "\n".join(lines) 96 | 97 | 98 | def testreport_from_file(path: Path): 99 | """Parse the given 'pytest-reportlog' output into a restreport dict""" 100 | 101 | results: Dict[str, Dict[str, Any]] = { 102 | "status": {"failed": 0, "passed": 0, "skipped": 0, "total": 0}, 103 | "tests": {}, 104 | } 105 | 106 | logpath = path / "testreport.log" 107 | if not logpath.exists(): 108 | return {} 109 | 110 | with logpath.open() as logfile: 111 | for count, line in enumerate(logfile.readlines()): 112 | result = json.loads(line) 113 | if result["$report_type"] != "TestReport": 114 | continue 115 | 116 | nodeid: str = result["nodeid"] 117 | if nodeid not in results["tests"]: 118 | try: 119 | comp = nodeid.split("::") 120 | group_left = comp[0] 121 | group_right = "".join(comp[1:]) 122 | except Exception: 123 | group_left, group_right = (nodeid, nodeid) 124 | 125 | results["tests"][nodeid] = { 126 | "group_left": group_left, 127 | "group_right": group_right, 128 | "count": count, 129 | "nodeid": nodeid, 130 | "duration": 0.0, 131 | "outcome": [], 132 | "runlog": {}, 133 | "longrepr": "", 134 | } 135 | if isinstance(result["longrepr"], list): 136 | results["tests"][nodeid]["longrepr"] += "\n".join( 137 | [str(item) for item in result["longrepr"]] 138 | ) 139 | elif isinstance(result["longrepr"], dict): 140 | results["tests"][nodeid]["longrepr"] += longrepr_to_string( 141 | result["longrepr"] 142 | ) 143 | 144 | results["tests"][nodeid]["duration"] += result["duration"] 145 | results["tests"][nodeid]["outcome"] += [result["outcome"]] 146 | 147 | runlog = runlog_from_path(path / sanitize_ident(result["nodeid"])) 148 | if runlog: 149 | results["tests"][nodeid]["runlog"] = runlog 150 | 151 | for nodeid, testcase in results["tests"].items(): 152 | results["status"]["total"] += 1 153 | for key in ["failed", "skipped", "passed"]: 154 | if key in testcase["outcome"]: 155 | results["status"][key] += 1 156 | break 157 | 158 | if results["status"]["total"]: 159 | return results 160 | 161 | return {} 162 | 163 | 164 | def artifacts_in_path(path: Path): 165 | """Returns a list of paths to artifacts""" 166 | 167 | if not path.exists(): 168 | return [] 169 | 170 | artifacts = [] 171 | for artifact_dir in Path(path).rglob("artifacts"): 172 | for artifact in artifact_dir.rglob("*"): 173 | artifacts.append(artifact.relative_to(path)) 174 | 175 | return sorted(artifacts) 176 | 177 | 178 | def process_workflow_output(args, cijoe): 179 | workflow_state = dict_from_yamlfile(args.output / "workflow.state") 180 | workflow_state["config"] = cijoe.config.options 181 | workflow_state["artifacts"] = artifacts_in_path(args.output) 182 | 183 | # workflow_state["artifacts"] = artifacts_in_path( 184 | # args.output, args.output / "artifacts" 185 | # ) 186 | 187 | for step in workflow_state["steps"]: 188 | if "extras" not in step: 189 | step["extras"] = {} 190 | 191 | if step["status"]["started"] > 0 and step["status"]["elapsed"] == 0: 192 | step["status"]["elapsed"] = time.time() - step["status"]["started"] 193 | 194 | step_path = args.output / step["id"] 195 | if not step_path.exists(): 196 | continue 197 | 198 | # artifacts = artifacts_in_path(args.output, step_path / "artifacts") 199 | # artifacts = artifacts_in_path(step_path / "artifacts") 200 | # if artifacts: 201 | # step["extras"]["artifacts"] = artifacts 202 | 203 | runlog = runlog_from_path(step_path) 204 | if runlog: 205 | step["extras"]["runlog"] = runlog 206 | 207 | testreport = testreport_from_file(step_path) 208 | if testreport: 209 | step["extras"]["testreport"] = testreport 210 | 211 | return workflow_state 212 | -------------------------------------------------------------------------------- /src/cijoe/core/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/core/scripts/__init__.py -------------------------------------------------------------------------------- /src/cijoe/core/scripts/cmdrunner.py: -------------------------------------------------------------------------------- 1 | """ 2 | cmdrunner 3 | ========= 4 | 5 | Executes a list of commands in the given order. Note that multi-line commands are not 6 | support, each line or list of strings are treated as individual commands. 7 | 8 | Retargetable: True 9 | ------------------ 10 | """ 11 | 12 | import errno 13 | import logging as log 14 | from argparse import ArgumentParser 15 | 16 | 17 | def add_args(parser: ArgumentParser): 18 | parser.add_argument( 19 | "--commands", nargs="+", type=str, help="The commands to be run" 20 | ) 21 | parser.add_argument( 22 | "--transport", 23 | type=str, 24 | default=None, 25 | help=( 26 | "The key of the transport from the cijoe config file on which the commands should be run. " 27 | "Use 'initiator' if the commands should be run locally. " 28 | "Defaults to the first transport in the config file ('initiator' if none are defined)." 29 | ), 30 | ) 31 | 32 | 33 | def main(args, cijoe): 34 | """Run commands one at a time via cijoe.run()""" 35 | 36 | err = 0 37 | if "commands" not in args: 38 | log.error("missing step-argument: with.commands") 39 | return errno.EINVAL 40 | 41 | for cmd in args.commands: 42 | err, state = cijoe.run(cmd, transport_name=args.transport) 43 | if err: 44 | break 45 | 46 | return err 47 | -------------------------------------------------------------------------------- /src/cijoe/core/scripts/example_script_default.py: -------------------------------------------------------------------------------- 1 | """ 2 | cijoe example script 3 | ==================== 4 | 5 | The script is a modified "Hello, World!" example. It repeatedly prints a message a set 6 | number of times and allows parameterization of the message content. 7 | 8 | The purpose of this script is to demonstrate how to run commands and supply input to the 9 | script using a configuration file, environment variables, command-line arguments and 10 | workflow step arguments. 11 | 12 | An example of using the core infrastructure of cijoe: 13 | 14 | * cijoe.run(command) 15 | 16 | - Error-handling; checking return-code 17 | - Output processing state.output() 18 | 19 | Input is given to scripts via configuration-files, environment variables and from 20 | workflow-step-arguments, this is demonstrated as the first thing in the script. 21 | 22 | cijoe also has primitives for transferring data: 23 | 24 | * cijoe.get(src, dst) 25 | 26 | - Transfer 'src' directory or file from **target** to 'dst' on **initiator** 27 | 28 | * cijoe.put(src, dst) 29 | 30 | - Transfer 'src' directory or file from **initiator** to 'dst' on **target** 31 | 32 | These are not used in the example code below, but you can experiment and try adding 33 | them yourself. 34 | """ 35 | 36 | import logging as log 37 | from argparse import ArgumentParser, Namespace 38 | 39 | from cijoe.core.command import Cijoe 40 | 41 | 42 | def add_args(parser: ArgumentParser): 43 | """Optional function for defining command-line arguments for this script""" 44 | parser.add_argument( 45 | "--repeat", 46 | type=int, 47 | default=1, 48 | help="Amount of times the message will be repeated", 49 | ) 50 | 51 | 52 | def main(args: Namespace, cijoe: Cijoe): 53 | """Entry-point of the cijoe-script""" 54 | 55 | # Grab message from the configuration-file 56 | message = cijoe.getconf("example.message", "Hello World!") 57 | 58 | # When executed via workflow, grab the step-argument 59 | repeat = args.repeat 60 | if repeat < 1: 61 | log.error(f"Invalid step-argument: repeat({repeat}) < 1") 62 | return 1 63 | 64 | log.info(f"Will echo the message({message}), repeat({repeat}) times") 65 | 66 | # Now, execute a command that echoes the 'message' 'repeat' number of times 67 | for _ in range(1, repeat + 1): 68 | err, state = cijoe.run(f"echo '{message}'") 69 | if "Hello" not in state.output(): 70 | log.error("Something went wrong") 71 | return 1 72 | 73 | log.info("Success!") 74 | 75 | return 0 76 | -------------------------------------------------------------------------------- /src/cijoe/core/scripts/example_script_testrunner.py: -------------------------------------------------------------------------------- 1 | from argparse import Namespace 2 | 3 | from cijoe.core.command import Cijoe 4 | from cijoe.core.scripts.testrunner import pytest_remote 5 | 6 | 7 | def test_hello_world(cijoe: Cijoe): 8 | err, state = cijoe.run("echo 'Hello World!'") 9 | 10 | assert not err 11 | assert "Hello" in state.output() 12 | 13 | 14 | def test_config_message(cijoe: Cijoe): 15 | message = cijoe.getconf("testrunner.message") 16 | assert message 17 | 18 | cijoe.run("hostname") 19 | cijoe.run("lspci") 20 | 21 | err, state = cijoe.run(f"echo '{message}'") 22 | 23 | assert not err 24 | assert "Hello" in state.output() 25 | 26 | 27 | def test_true(cijoe: Cijoe): 28 | assert True 29 | 30 | 31 | def main(args: Namespace, cijoe: Cijoe): 32 | """ 33 | This main function is not run as part of the example workflow in the 34 | core.testrunner example, but must be here in order for it to be 35 | elicited as script when running `cijoe --example core.testrunner`. 36 | """ 37 | 38 | return 0 39 | -------------------------------------------------------------------------------- /src/cijoe/core/scripts/get.py: -------------------------------------------------------------------------------- 1 | """ 2 | get 3 | === 4 | 5 | Copies a file from remote to local. 6 | 7 | Retargetable: True 8 | ------------------ 9 | """ 10 | 11 | import errno 12 | import logging as log 13 | from argparse import ArgumentParser 14 | 15 | 16 | def add_args(parser: ArgumentParser): 17 | parser.add_argument("--src", type=str, help="path to the file on remote machine") 18 | parser.add_argument( 19 | "--dst", 20 | type=str, 21 | help="path to where the file should be placed on the initiator", 22 | ) 23 | parser.add_argument( 24 | "--transport", 25 | type=str, 26 | default=None, 27 | help=( 28 | "The name of the transport which should be considered as the remote machine. " 29 | "Use 'initiator' if the commands should be run locally. " 30 | "Defaults to the first transport in the config file ('initiator' if none are defined)." 31 | ), 32 | ) 33 | 34 | 35 | def main(args, cijoe): 36 | """Copies the file at args.src on the remote machine to args.dst on the local machine""" 37 | 38 | if not ("src" in args and "dst" in args): 39 | log.error("missing step-argument: with.src and/or with.dst") 40 | return errno.EINVAL 41 | 42 | return int(not cijoe.get(args.src, args.dst, transport_name=args.transport)) 43 | -------------------------------------------------------------------------------- /src/cijoe/core/scripts/put.py: -------------------------------------------------------------------------------- 1 | """ 2 | put 3 | === 4 | 5 | Copies a file from local to remote. 6 | 7 | Retargetable: True 8 | ------------------ 9 | """ 10 | 11 | import errno 12 | import logging as log 13 | from argparse import ArgumentParser 14 | 15 | 16 | def add_args(parser: ArgumentParser): 17 | parser.add_argument("--src", type=str, help="path to the file on initiator") 18 | parser.add_argument( 19 | "--dst", 20 | type=str, 21 | help="path to where the file should be placed on the remote machine", 22 | ) 23 | parser.add_argument( 24 | "--transport", 25 | type=str, 26 | default=None, 27 | help=( 28 | "The name of the transport which should be considered as the remote machine. " 29 | "Use 'initiator' if the commands should be run locally. " 30 | "Defaults to the first transport in the config file ('initiator' if none are defined)." 31 | ), 32 | ) 33 | 34 | 35 | def main(args, cijoe): 36 | """Copies the file at args.src on the local machine to args.dst on the remote machine""" 37 | 38 | if not ("src" in args and "dst" in args): 39 | log.error("missing step-argument: with.src and/or with.dst") 40 | return errno.EINVAL 41 | 42 | return int(not cijoe.put(args.src, args.dst, transport_name=args.transport)) 43 | -------------------------------------------------------------------------------- /src/cijoe/core/scripts/reporter.py: -------------------------------------------------------------------------------- 1 | """ 2 | Report generator 3 | ================ 4 | 5 | Generates a HTML report in the workflow output directory. 6 | 7 | Retargtable: false 8 | ------------------ 9 | 10 | The report-generator works on the files generated by cijoe on the host which is 11 | executing cijoe. Thus, no need to make this re-targetable. 12 | """ 13 | 14 | import logging as log 15 | import webbrowser 16 | from argparse import ArgumentParser, _StoreAction 17 | from datetime import datetime 18 | 19 | import jinja2 20 | import yaml 21 | 22 | from cijoe.core.processing import process_workflow_output 23 | from cijoe.core.resources import get_resources 24 | 25 | 26 | def add_args(parser: ArgumentParser): 27 | class StringToBoolAction(_StoreAction): 28 | def __call__(self, parser, namespace, values, option_string=None): 29 | setattr(namespace, self.dest, values == "true") 30 | 31 | parser.add_argument( 32 | "--report_open", 33 | choices=["true", "false"], 34 | default=False, 35 | action=StringToBoolAction, 36 | help="Whether or not the generated report should be opened (in a browser)", 37 | ) 38 | 39 | 40 | def to_yaml(value): 41 | return yaml.dump(value) 42 | 43 | 44 | def elapsed_txt(value): 45 | minutes, seconds = divmod(float(value), 60.0) 46 | hours, minutes = divmod(minutes, 60.0) 47 | 48 | txt = [] 49 | if hours: 50 | txt.append(f"{hours:.0f} hour") 51 | if minutes: 52 | txt.append(f"{minutes:.0f} min") 53 | txt.append(f"{seconds:0.2f} sec") 54 | 55 | return " ".join(txt) 56 | 57 | 58 | def timestamp_to_txt(value): 59 | return datetime.fromtimestamp(float(value)).strftime("%d-%m-%Y, %H:%M:%S") 60 | 61 | 62 | def main(args, cijoe): 63 | """Produce a HTML report of the 'workflow.state' file in 'args.output'""" 64 | 65 | report_open = args.report_open 66 | 67 | resources = get_resources() 68 | 69 | template_path = resources["templates"]["core.report-workflow.html"].path 70 | report_path = args.output / "report.html" 71 | 72 | log.info(f"template: {template_path}") 73 | log.info(f"report: {report_path}") 74 | 75 | workflow_state = process_workflow_output(args, cijoe) 76 | 77 | jinja_env = jinja2.Environment( 78 | autoescape=True, loader=jinja2.FileSystemLoader(template_path.parent) 79 | ) 80 | jinja_env.filters["to_yaml"] = to_yaml 81 | jinja_env.filters["elapsed_txt"] = elapsed_txt 82 | jinja_env.filters["timestamp_to_txt"] = timestamp_to_txt 83 | template = jinja_env.get_template(template_path.name) 84 | 85 | with (report_path).open("w") as report: 86 | report.write(template.render(workflow_state)) 87 | 88 | if report_open: 89 | webbrowser.open("file://%s" % report_path.resolve()) 90 | 91 | return 0 92 | -------------------------------------------------------------------------------- /src/cijoe/core/scripts/repository_prep.py: -------------------------------------------------------------------------------- 1 | """ 2 | repository_prep 3 | =============== 4 | 5 | For every key in the configuration which has a subkey named "repository", then 6 | following is done: 7 | 8 | * git clone repository.remote # if ! exists(repository.path) 9 | * git checkout [repository.branch,repository.tag] # if repository.{branch,tag} 10 | * git pull --rebase # if repository.branch 11 | * git status 12 | 13 | The intended usage of this script is to prepare a repositories in a recently 14 | provisioned system. Such as a done by 'qemu.provision'. 15 | 16 | Configuration 17 | ------------- 18 | 19 | Ensure that the "repository" has sensible values for: 20 | 21 | * remote: url of the repository to clone 22 | 23 | * branch: name of the branch to check out and rebase 24 | * tag: name of the tag to check out 25 | 26 | * run_local: Optionally, set 'run_local' to True, in case the repos should just be 27 | checked out locally, instead of on the remote. 28 | 29 | Retargetable: True 30 | ------------------ 31 | """ 32 | 33 | import errno 34 | import logging as log 35 | from pathlib import Path 36 | 37 | 38 | def main(args, cijoe): 39 | """Clone, checkout branch and pull""" 40 | 41 | err, _ = cijoe.run("git --version") 42 | if err: 43 | log.error("Looks like git is not available") 44 | return err 45 | 46 | for repos in [ 47 | r["repository"] for r in cijoe.config.options.values() if "repository" in r 48 | ]: 49 | run = cijoe.run_local if repos.get("run_local", False) else cijoe.run 50 | 51 | repos_root = Path(repos["path"]).parent 52 | 53 | err, _ = run(f"mkdir -p {repos_root}") 54 | if err: 55 | log.error("failed creating repos_root({repos_root}; giving up") 56 | return err 57 | 58 | err, _ = run( 59 | f"[ ! -d {repos['path']} ] &&" 60 | f" git clone {repos['remote']} {repos['path']} --recursive" 61 | ) 62 | if err: 63 | log.info("either already cloned or failed cloning; continuing optimisticly") 64 | 65 | err, _ = run("git fetch --all", cwd=repos["path"]) 66 | if err: 67 | log.info("fetching failed; continuing optimisticly") 68 | 69 | do_checkout = repos.get("branch", repos.get("tag", None)) 70 | if do_checkout: 71 | err, _ = run(f"git checkout {do_checkout}", cwd=repos["path"]) 72 | if err: 73 | log.error("Failed checking out; giving up") 74 | return err 75 | else: 76 | log.info("no 'branch' nor 'tag' key; skipping checkout") 77 | 78 | if "branch" in repos.keys(): 79 | err, _ = run("git pull --rebase", cwd=repos["path"]) 80 | if err: 81 | log.error("failed pulling; giving up") 82 | return err 83 | 84 | err, _ = run("git submodule update --init --recursive", cwd=repos["path"]) 85 | if err: 86 | log.info("Updating submodules failed; continuin optimisticly") 87 | 88 | err, _ = run("git status", cwd=repos["path"]) 89 | if err: 90 | log.error("failed 'git status'; giving up") 91 | return err 92 | 93 | return 0 94 | -------------------------------------------------------------------------------- /src/cijoe/core/scripts/testrunner.py: -------------------------------------------------------------------------------- 1 | """ 2 | testrunner 3 | ========== 4 | 5 | This is the CIJOE testrunner, it is implemented as a pytest-plugin. By doing 6 | so, then there is little to nothing to learn, assuming, that you have used 7 | pytest before. 8 | 9 | The "special" thing is the pytest-plugin, it provides pytest command-line 10 | arguments: ``-c / --config`` and ``-o / --output``. These behave just like the 11 | ``cijoe`` counter-part. Within, your tests, then a ``cijoe`` fixture is 12 | provided. It will give the tests access to a cijoe-instance with the given 13 | config. Quite neat :) 14 | 15 | Thus, arbitrary testing of third-party test-suites, tools benchmarks etc. is 16 | possible with something as simple as:: 17 | 18 | def test_foo(cijoe): 19 | 20 | err, state = cijoe.run("execute-some-testsuite") 21 | assert not err 22 | 23 | Requires the following pytest plugins for correct behaviour: 24 | 25 | * cijoe, fixtures providing 'cijoe' object and "--config" and "--output" 26 | pytest-arguments to instantiate cijoe. 27 | 28 | * report-log, dump testnode-status as JSON, this is consumed by 'core.report' 29 | to produce an overview of testcases and link them with the cijoe-captured 30 | output and auxiliary files. 31 | 32 | Invocation of pytest is done in one of the following two ways, and controlled 33 | by ``args.run_local``, with boolean value True / False. The default is 34 | True. 35 | 36 | * ``args.run_local: True`` 37 | 38 | This is the most common, invoking pytest locally, which in turn will be using 39 | the same config as all other cijoe-scripts. To clarify, cijoe will execute 40 | 'pytest' in the same environment/system where the ``cijoe`` cli was executed. 41 | 42 | * ``args.run_local: False`` 43 | 44 | This is a special-case, where a collection of pytests uses cijoe, but only the 45 | configuration, the that the pytest verify is Python code / statements / 46 | expressions, not CIJOE command executions cijoe.run(). In order ot run these 47 | remotely, then the code must be available, and then it does the following: 48 | 49 | - Create a copy of the currently used cijoe-config and remove the transport section if any is there 50 | - Transfer cijoe-config-copy to remote 51 | - Invoke pytest remotely using cijoe-config-copy 52 | - Download the testreport.log from remote 53 | 54 | Why this? This allows for executing pytests on a remote system which does not 55 | use cijoe.run(). Such as tests implemented in Python. 56 | """ 57 | 58 | import copy 59 | import logging as log 60 | import uuid 61 | from argparse import ArgumentParser, _StoreAction 62 | from pathlib import Path 63 | 64 | from cijoe.core.resources import dict_to_tomlfile 65 | 66 | 67 | def add_args(parser: ArgumentParser): 68 | class StringToBoolAction(_StoreAction): 69 | def __call__(self, parser, namespace, values, option_string=None): 70 | setattr(namespace, self.dest, values == "true") 71 | 72 | parser.add_argument( 73 | "--run_local", 74 | choices=["true", "false"], 75 | default=True, 76 | action=StringToBoolAction, 77 | help="Whether 'pytest' should be executed in same environment as 'cijoe'", 78 | ) 79 | parser.add_argument( 80 | "--random_order", 81 | choices=["true", "false"], 82 | default=True, 83 | action=StringToBoolAction, 84 | help=( 85 | "Whether the tests should be run in random order. " 86 | "This is generally recommended, as it helps reduce inter-test " 87 | "dependencies and assumptions about the environment's state" 88 | ), 89 | ) 90 | parser.add_argument( 91 | "--args", type=str, help="Additional arguments passed verbatim to 'pytest'." 92 | ) 93 | 94 | 95 | def pytest_cmdline(args, config_path, output_path, reportlog_path): 96 | """Contruct pytest command-line arguments given args and paths""" 97 | 98 | log.info(f"config_path({config_path})") 99 | log.info(f"output_path({output_path})") 100 | log.info(f"reportlog_path({reportlog_path})") 101 | 102 | cmdline = ["pytest"] 103 | if args.config: 104 | cmdline.append("--config") 105 | cmdline.append(config_path) 106 | cmdline += ["--output", output_path] 107 | cmdline += ["--report-log", reportlog_path] 108 | 109 | if "args" in args: 110 | cmdline += args.args.split(" ") 111 | 112 | random_order = args.random_order 113 | cmdline += ["--random-order"] if random_order else [] 114 | 115 | return cmdline 116 | 117 | 118 | def pytest_remote(args, cijoe): 119 | """ 120 | Run pytest on remote, that is, transfer config, execute pytest on the 121 | remote and transfer the testreport.log from remote to local 122 | """ 123 | 124 | err, state = cijoe.run("pwd") 125 | if err: 126 | log.error("Failed querying 'pwd'") 127 | return err 128 | 129 | cwd = Path(state.output().strip()) 130 | rand = str(uuid.uuid4())[:8] 131 | 132 | # Construct config based on current config, but without "ssh" transport 133 | config_stem = f"cijoe-config-{rand}.toml" 134 | config_path = cwd / config_stem 135 | config_path_local = args.output / cijoe.output_ident / config_stem 136 | 137 | config = copy.deepcopy(cijoe.config.options) 138 | if "transport" in config.get("cijoe", {}): 139 | del config["cijoe"]["transport"] 140 | 141 | dict_to_tomlfile(config, config_path_local) 142 | cijoe.put(str(config_path_local), config_path) 143 | 144 | # Construct pytest command-line and execute it 145 | reportlog_stem = f"cijoe-testreport-{rand}.log" 146 | reportlog_path = cwd / reportlog_stem 147 | reportlog_path_local = args.output / cijoe.output_ident / "testreport.log" 148 | 149 | # Construct the output-path and the equivalent local-path 150 | output_stem = f"cijoe-output-{rand}" 151 | output_path = cwd / output_stem 152 | output_path_local = args.output / cijoe.output_ident / "output_pytest" 153 | 154 | cmdline = pytest_cmdline( 155 | args, 156 | str(config_path), 157 | str(output_path), 158 | str(reportlog_path), 159 | ) 160 | 161 | err, _ = cijoe.run(" ".join(cmdline)) 162 | 163 | # Retrieve testlog 164 | cijoe.get(str(reportlog_path), str(reportlog_path_local)) 165 | log.info(reportlog_path_local) 166 | 167 | # TODO: Retrieve output 168 | cijoe.get(str(output_path), str(output_path_local)) 169 | 170 | # Cleanup: remove "artifacts" on remote 171 | cijoe.run(f"rm {config_path}") 172 | cijoe.run(f"rm {reportlog_path}") 173 | cijoe.run(f"rm -r {output_path}") 174 | 175 | return err 176 | 177 | 178 | def pytest_local(args, cijoe): 179 | """ 180 | Run pytest locally, that is, execute on the same system on which cijoe-cli 181 | was executed. The cijoe config provided via 'args.config' is forwarded to 182 | the pytest-cijoe-plugin unmodified. 183 | """ 184 | 185 | cmdline = pytest_cmdline( 186 | args, 187 | str(args.config), 188 | str(args.output / cijoe.output_ident), 189 | str(args.output / cijoe.output_ident / "testreport.log"), 190 | ) 191 | 192 | err, _ = cijoe.run_local(" ".join(cmdline)) 193 | 194 | return err 195 | 196 | 197 | def main(args, cijoe): 198 | """Invoke the pytest + cijoe-plugin test-runner""" 199 | return (pytest_local if args.run_local else pytest_remote)(args, cijoe) 200 | -------------------------------------------------------------------------------- /src/cijoe/core/templates/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/core/templates/.keep -------------------------------------------------------------------------------- /src/cijoe/core/templates/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Templates 3 | """ 4 | -------------------------------------------------------------------------------- /src/cijoe/core/templates/example-tmp-workflow.yaml.jinja2: -------------------------------------------------------------------------------- 1 | --- 2 | doc: | 3 | Temporary standalone script 4 | 5 | steps: 6 | {% for step in steps %} 7 | - name: {{step | replace(".", "_")}} 8 | uses: {{step}} 9 | {% endfor %} -------------------------------------------------------------------------------- /src/cijoe/core/workflows/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This package is a container for workflow files 3 | """ 4 | -------------------------------------------------------------------------------- /src/cijoe/core/workflows/example_workflow_default.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | doc: | 3 | This is a workflow file, it serves as an example on how to run commands and 4 | scripts, the structure intentionally mimics that of GitHUB actions, however, 5 | the keys you see here are all there is. 6 | 7 | Running commands, as you can see below, looks just like running commands in a 8 | GitHUB Workflow 9 | 10 | * Add the 'run' key with a value of multi-line string 11 | 12 | Using scripts, it is similar to that of a GitHUB action 13 | 14 | * Add the 'uses' key with the name of the script 15 | - packaged scripts have a namespaced name e.g. "my_pkg.my_script" 16 | - non-packaged do not e.g. "my_script" 17 | * Add the 'with' key providing arguments to the script 18 | 19 | The commands and the scripts are passed an instance of cijoe which they can 20 | use to call run() / get() / put(), with an output-directory matching the 21 | current step. This is it, end of story. 22 | 23 | steps: 24 | - name: inline_commands 25 | run: | 26 | cat /proc/cpuinfo 27 | hostname 28 | 29 | - name: script_with_args 30 | uses: core.cmdrunner 31 | with: 32 | commands: 33 | - cat /proc/cpuinfo 34 | - hostname 35 | 36 | - name: example_script 37 | uses: core.example_script_default 38 | with: 39 | repeat: 2 40 | 41 | # 42 | # Uncomment the lines below to include the non-packaged script 43 | # 'cijoe-script.py', from the directory where you ran 'cijoe --example' 44 | # 45 | #- name: custom_script 46 | # uses: cijoe-script 47 | -------------------------------------------------------------------------------- /src/cijoe/core/workflows/example_workflow_get_put.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | doc: | 3 | This workflow file is an example of how to use core.put and core.get. 4 | 5 | steps: 6 | - name: guest_initialize 7 | uses: qemu.guest_initialize 8 | with: 9 | guest_name: generic-uefi-tcg-aarch64 10 | system_image_name: debian-12-aarch64 11 | 12 | - name: guest_start 13 | uses: qemu.guest_start 14 | with: 15 | guest_name: generic-uefi-tcg-aarch64 16 | 17 | - name: guest_check 18 | run: | 19 | hostname 20 | 21 | - name: create_file 22 | run: | 23 | echo "Hello World" > /tmp/hello.txt 24 | with: 25 | transport: initiator 26 | 27 | - name: put 28 | uses: core.put 29 | with: 30 | src: /tmp/hello.txt 31 | dst: /tmp/hello_from_initiator.txt 32 | 33 | - name: get 34 | uses: core.get 35 | with: 36 | src: /tmp/hello_from_initiator.txt 37 | dst: /tmp/hello_from_target.txt 38 | 39 | - name: check 40 | run: | 41 | cat /tmp/hello_from_target.txt 42 | with: 43 | transport: initiator 44 | 45 | - name: guest_kill 46 | uses: qemu.guest_kill 47 | with: 48 | guest_name: generic-uefi-tcg-aarch64 49 | -------------------------------------------------------------------------------- /src/cijoe/core/workflows/example_workflow_testrunner.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | doc: | 3 | This workflow file is an example of how to use the CIJOE testrunner. 4 | 5 | The testrunner script takes three optional arguments. The purpose of this 6 | workflow is to run the testrunner with different combinations of these 7 | arguments. 8 | 9 | steps: 10 | - name: diskimage_from_cloudimage 11 | uses: system_imaging.diskimage_from_cloudimage 12 | with: 13 | pattern: "debian-12-x86_64" 14 | 15 | - name: guest_initialize 16 | uses: qemu.guest_initialize 17 | with: 18 | guest_name: generic-bios-kvm-x86_64 19 | system_image_name: debian-12-x86_64 20 | 21 | - name: guest_start 22 | uses: qemu.guest_start 23 | with: 24 | guest_name: generic-bios-kvm-x86_64 25 | 26 | - name: guest_check 27 | run: | 28 | hostname 29 | 30 | - name: install_cijoe 31 | run: | 32 | apt-get -qy install python3 pipx 33 | rm -f /usr/bin/py.test || true 34 | rm -f /usr/bin/pytest || true 35 | pipx install cijoe --include-deps 36 | pipx inject cijoe --force pytest --include-apps 37 | pipx ensurepath 38 | 39 | - name: testrunner_local 40 | uses: core.testrunner 41 | with: 42 | run_local: true 43 | random_order: false 44 | args: "cijoe-example-core.testrunner/" 45 | 46 | - name: create_dir 47 | run: | 48 | rm -rf /tmp/cijoe-example-core.testrunner/ 49 | mkdir -p /tmp/cijoe-example-core.testrunner/ 50 | 51 | - name: transfer_tests 52 | uses: core.put 53 | with: 54 | src: "{{ local.env.PWD }}/cijoe-example-core.testrunner/" 55 | dst: "/tmp/cijoe-example-core.testrunner/" 56 | 57 | - name: testrunner_remote 58 | uses: core.testrunner 59 | with: 60 | run_local: false 61 | random_order: true 62 | args: "/tmp/cijoe-example-core.testrunner/" 63 | 64 | - name: testrunner_keywords 65 | uses: core.testrunner 66 | with: 67 | args: "-k 'true' cijoe-example-core.testrunner/" 68 | 69 | - name: guest_kill 70 | uses: qemu.guest_kill 71 | with: 72 | guest_name: generic-bios-kvm-x86_64 73 | -------------------------------------------------------------------------------- /src/cijoe/core/worklets/README.rst: -------------------------------------------------------------------------------- 1 | Worklets 2 | ======== 3 | 4 | These are now deprecated and now use the more common name of "scripts". For 5 | backward compatibility, then functions named "worklet_entry" can still be used. 6 | -------------------------------------------------------------------------------- /src/cijoe/core/worklets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/core/worklets/__init__.py -------------------------------------------------------------------------------- /src/cijoe/linux/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/linux/__init__.py -------------------------------------------------------------------------------- /src/cijoe/linux/configs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/linux/configs/__init__.py -------------------------------------------------------------------------------- /src/cijoe/linux/configs/example_config_build_kdebs.toml: -------------------------------------------------------------------------------- 1 | [linux.repository] 2 | remote = "https://github.com/torvalds/linux.git" 3 | path = "{{ local.env.HOME }}/git/linux" 4 | 5 | # 6 | # null_blk 7 | # 8 | # The keys of 'null_blk' are passed verbatim to 'modprobe null_blk', consult the Kernel docs for a 9 | # description of the different keys 10 | # 11 | # When the nullblk kernel module is loaded it will create 'nr_devices' nullblk instances, when 12 | # 'nr_devices' is 0, then they can be created via sysfs instead 13 | [null_blk] 14 | nr_devices = 4 15 | queue_mode = 2 16 | home_node = 0 17 | bs = 512 18 | irqmode = 2 19 | gb = 14 20 | completion_nsec = 10 21 | submit_queue = 1 22 | hw_queue_depth = 64 23 | memory_backed = 1 24 | # MQ 25 | use_per_node_hctx = 0 26 | no_sched = 0 27 | blocking = 0 28 | shared_tags = 0 29 | # Zoned 30 | zoned = 0 31 | zone_size = 256 32 | zone_nr_conv = 0 33 | -------------------------------------------------------------------------------- /src/cijoe/linux/null_blk.py: -------------------------------------------------------------------------------- 1 | """ 2 | null_blk module, helpers to load/unload null block instances 3 | 4 | To use it, one must have permissions to modprobe, rmmod and modify the /syscfg 5 | 6 | NOTE: This is **fully** re-targetable, that is, changing transport changes where it 7 | is running. 8 | 9 | For reference, see: https://docs.kernel.org/block/null_blk.html 10 | 11 | TODO 12 | - Implement initialization of nullblk instances via /sys/kernel, this allows for 13 | different device instances instead of N instances with the same configuration. 14 | For example, one can instantiate a regular block-device as well as a zoned 15 | block-device. 16 | 17 | retargetable: True 18 | """ 19 | 20 | NULLBLK_MODULE_NAME = "null_blk" 21 | NULLBLK_SYSPATH = "/sys/kernel/config/nullb" 22 | 23 | 24 | def insert(cijoe, config=None): 25 | """Load the 'null_blk' kernel module using parameters defined in the config""" 26 | 27 | if config is None: 28 | config = cijoe.config.options.get("null_blk") 29 | 30 | nullblk_params = ( 31 | " ".join([f"{k}={v}" for k, v in config.items()]) 32 | if config.get("nr_devices") 33 | else "" 34 | ) 35 | 36 | return cijoe.run(f"modprobe {NULLBLK_MODULE_NAME} {nullblk_params}") 37 | 38 | 39 | def remove(cijoe): 40 | """Remove the null_blk kernel module""" 41 | 42 | # This can be used when instanttation via SYSPATH, however, commented out as it is 43 | # not useful yet. 44 | # cijoe.run(f"rmdir {NULLBLK_SYSPATH}/nullb*") 45 | 46 | return cijoe.run(f"modprobe -r {NULLBLK_MODULE_NAME}") 47 | -------------------------------------------------------------------------------- /src/cijoe/linux/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/linux/scripts/__init__.py -------------------------------------------------------------------------------- /src/cijoe/linux/scripts/build_kdebs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Linux Custom Kernel as Debian Package 3 | ===================================== 4 | 5 | There are a myriad of ways to build and install a custom Linux kernel. This worklet 6 | builds it as a Debian package. The generated .deb packages are stored in 7 | cijoe.output_path. 8 | 9 | Retargetable: False 10 | ------------------- 11 | 12 | It is intended to be run "locally" since, currently the collection of the generated 13 | .debs are not retrieved via cijoe.get(), doing so would make it retargetable. 14 | """ 15 | 16 | import logging as log 17 | from argparse import ArgumentParser, _StoreAction 18 | from pathlib import Path 19 | 20 | 21 | def add_args(parser: ArgumentParser): 22 | class StringToBoolAction(_StoreAction): 23 | def __call__(self, parser, namespace, values, option_string=None): 24 | setattr(namespace, self.dest, values == "true") 25 | 26 | parser.add_argument( 27 | "--local_version", 28 | type=str, 29 | default="custom", 30 | help="Path to local version of kdebs", 31 | ) 32 | parser.add_argument( 33 | "--run_local", 34 | choices=["true", "false"], 35 | default=True, 36 | action=StringToBoolAction, 37 | help="Whether or not to execute in the same environment as 'cijoe'.", 38 | ) 39 | 40 | 41 | def main(args, cijoe): 42 | """Configure, build and collect the build-artifacts""" 43 | 44 | path = cijoe.getconf("linux.repository.path") 45 | if not path: 46 | log.error("missing config: linux.repository.path") 47 | return 1 48 | 49 | repos = Path(path).resolve() 50 | err, _ = cijoe.run(f"[ -d {repos} ]") 51 | if err: 52 | return err 53 | 54 | run = cijoe.run_local if args.run_local else cijoe.run 55 | 56 | commands = [ 57 | "[ -f .config ] && rm .config || true", 58 | "yes " " | make olddefconfig", 59 | "./scripts/config --disable CONFIG_DEBUG_INFO", 60 | "./scripts/config --disable SYSTEM_TRUSTED_KEYS", 61 | "./scripts/config --disable SYSTEM_REVOCATION_KEYS", 62 | f"yes '' | make -j$(nproc) bindeb-pkg LOCALVERSION={args.local_version}", 63 | f"mkdir -p {cijoe.output_path}/artifacts/linux", 64 | f"mv ../*.deb {cijoe.output_path}/artifacts/linux", 65 | f"mv ../*.changes {cijoe.output_path}/artifacts/linux", 66 | f"mv ../*.buildinfo {cijoe.output_path}/artifacts/linux", 67 | ] 68 | for cmd in commands: 69 | err, _ = run(cmd, cwd=str(repos)) 70 | if err: 71 | return err 72 | 73 | return 0 74 | -------------------------------------------------------------------------------- /src/cijoe/linux/scripts/null_blk.py: -------------------------------------------------------------------------------- 1 | """ 2 | insert / remove null_blk 3 | ======================== 4 | 5 | Insert or remove null_blk instances, based on the value of args.do 6 | 7 | Retargetable: True 8 | ------------------ 9 | """ 10 | 11 | import errno 12 | from argparse import ArgumentParser 13 | 14 | import cijoe.linux.null_blk as null_blk 15 | 16 | 17 | def add_args(parser: ArgumentParser): 18 | parser.add_argument( 19 | "--do", 20 | choices=["insert", "remove"], 21 | default="insert", 22 | help="The commands to be run on the nullblk module.", 23 | ) 24 | 25 | 26 | def main(args, cijoe): 27 | """Insert or remove the null_blk""" 28 | 29 | do = args.do 30 | if do == "insert": 31 | err, _ = null_blk.insert(cijoe) 32 | elif do == "remove": 33 | err, _ = null_blk.remove(cijoe) 34 | else: 35 | err = errno.EINVAL 36 | 37 | return err 38 | -------------------------------------------------------------------------------- /src/cijoe/linux/scripts/sysinfo.py: -------------------------------------------------------------------------------- 1 | """ 2 | collect Linux system information 3 | ================================ 4 | 5 | Collects a bunch of information about the system kernel and hardware. 6 | 7 | Retargetable: True 8 | ------------------ 9 | """ 10 | 11 | 12 | def main(args, cijoe): 13 | """Collect Linux system information""" 14 | 15 | commands = [ 16 | "hostname", 17 | "lsb_release --all || cat /etc/os-release", 18 | "uname -a", 19 | "cat /boot/config-$(uname -r)", 20 | "set", 21 | "lsblk", 22 | "lscpu", 23 | "lslocks", 24 | "lslogins", 25 | "lsmem", 26 | "lsmod", 27 | "lspci", 28 | ] 29 | 30 | err = 0 31 | for cmd in commands: 32 | err, state = cijoe.run(cmd) 33 | if err: 34 | err = err 35 | 36 | return err 37 | -------------------------------------------------------------------------------- /src/cijoe/linux/workflows/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/linux/workflows/__init__.py -------------------------------------------------------------------------------- /src/cijoe/linux/workflows/example_workflow_build_kdebs.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | doc: | 3 | This workflow builds Linux kernel as .deb installable packages 4 | 5 | NOTE, if you switch 'run_local' to 'False', then you have to collect the kdebs yourself. 6 | 7 | steps: 8 | - name: sysinfo 9 | uses: linux.sysinfo 10 | 11 | - name: repository 12 | uses: core.repository_prep 13 | 14 | - name: build 15 | uses: linux.build_kdebs 16 | with: 17 | run_local: true 18 | -------------------------------------------------------------------------------- /src/cijoe/linux/workflows/example_workflow_null_blk.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | doc: | 3 | This is an example of utilizing some of the Linux worklets and helpers 4 | 5 | steps: 6 | - name: diskimage_from_cloudimage 7 | uses: system_imaging.diskimage_from_cloudimage 8 | with: 9 | pattern: "debian-12-x86_64" 10 | 11 | - name: guest_initialize 12 | uses: qemu.guest_initialize 13 | with: 14 | guest_name: generic-bios-kvm-x86_64 15 | system_image_name: debian-12-x86_64 16 | 17 | - name: guest_start 18 | uses: qemu.guest_start 19 | with: 20 | guest_name: generic-bios-kvm-x86_64 21 | 22 | - name: guest_check 23 | run: | 24 | hostname 25 | 26 | - name: sysinfo 27 | uses: linux.sysinfo 28 | 29 | - name: null_blk_insert 30 | uses: linux.null_blk 31 | 32 | - name: list 33 | run: lsblk 34 | 35 | - name: null_blk_remove 36 | uses: linux.null_blk 37 | with: 38 | do: remove 39 | 40 | - name: guest_kill 41 | uses: qemu.guest_kill 42 | with: 43 | guest_name: generic-bios-kvm-x86_64 44 | -------------------------------------------------------------------------------- /src/cijoe/pytest_plugin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/pytest_plugin/__init__.py -------------------------------------------------------------------------------- /src/cijoe/pytest_plugin/hooks_and_fixtures.py: -------------------------------------------------------------------------------- 1 | """ 2 | pytest-plugin API 3 | ================= 4 | 5 | The plugin provides a cijoe-instance readily available as a test-fixture, setup per 6 | test with a nodeid-defined output-directory. For example:: 7 | 8 | def test_foo(cijoe): 9 | err, _ = cijoe.run("hostname") 10 | asssert not err 11 | 12 | To provide the cijoe-instance a configuration and output directory must be provided. 13 | These are given via pytest, e.g.:: 14 | 15 | pytest --config default.toml --output /tmp/foo 16 | 17 | In case no arguments are provided, defaults are used. 18 | """ 19 | 20 | from pathlib import Path 21 | 22 | import pytest 23 | 24 | from cijoe.core.command import Cijoe, default_output_path 25 | from cijoe.core.resources import Collector, Config 26 | 27 | pytest.cijoe_instance = None 28 | 29 | 30 | def pytest_addoption(parser): 31 | """ 32 | Add options ``--config`` and ``--output`` to pytest, these will be used for the 33 | instantiation of cijoe. 34 | """ 35 | 36 | collector = Collector() 37 | collector.collect() 38 | 39 | parser.addoption( 40 | "--config", 41 | action="store", 42 | type=Path, 43 | help="Path to cijoe configuration", 44 | default=str(collector.resources["configs"]["core.example_config_default"]), 45 | ) 46 | parser.addoption( 47 | "--output", 48 | action="store", 49 | type=Path, 50 | help="Path to cijoe output directory", 51 | default=default_output_path(), 52 | ) 53 | 54 | 55 | def pytest_configure(config): 56 | """ 57 | Initializes the cijoe instance using pytest-options ``--config`` and ``--output`` 58 | 59 | The cijoe-instance is stored in ``pytest.cijoe_instance``, this might appear as bad 60 | form. However, it is a common pytest-pattern for enabling access to state otherwise 61 | only accessible to tests and fixtures. 62 | 63 | Why would this be needed? Well, for large paramaterizations, e.g. to generate input 64 | to pytest.mark.parametrize(). Here it is convenient to be able to access the same 65 | cijoe instance, and for example generate test-parametrization based on the content 66 | of the cijoe-configuration. 67 | """ 68 | 69 | cijoe_config_path = config.getoption("--config") 70 | 71 | cijoe_config = Config.from_path(cijoe_config_path) 72 | if cijoe_config is None: 73 | raise Exception(f"Failed loading config({cijoe_config_path})") 74 | 75 | pytest.cijoe_instance = Cijoe( 76 | cijoe_config, 77 | config.getoption("--output"), 78 | False, 79 | ) 80 | if pytest.cijoe_instance is None: 81 | raise Exception("Failed instantiating cijoe") 82 | 83 | 84 | def pytest_terminal_summary(terminalreporter, exitstatus, config): 85 | """ 86 | Prints out a notice that cijoe is beeing used along with the values of the 87 | ``--config`` and ``--output`` options. 88 | """ 89 | 90 | terminalreporter.ensure_newline() 91 | terminalreporter.section( 92 | "-={[ CIJOE pytest-plugin ]}=-", sep="-", blue=True, bold=True 93 | ) 94 | terminalreporter.line("config: %r" % config.getoption("--config")) 95 | terminalreporter.line("output: %r" % config.getoption("--output")) 96 | 97 | 98 | @pytest.fixture 99 | def cijoe(request): 100 | """ 101 | Provides a cijoe-instance, initialized with the pytest-options: ``--config``, and 102 | ``--output`` and with a per-test customization of the output directory. 103 | """ 104 | 105 | if pytest.cijoe_instance is None: 106 | raise Exception("Invalid configuration or instance") 107 | 108 | pytest.cijoe_instance.set_output_ident(request.node.nodeid) 109 | 110 | return pytest.cijoe_instance 111 | -------------------------------------------------------------------------------- /src/cijoe/qemu/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/qemu/__init__.py -------------------------------------------------------------------------------- /src/cijoe/qemu/auxiliary/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/qemu/auxiliary/__init__.py -------------------------------------------------------------------------------- /src/cijoe/qemu/configs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/qemu/configs/__init__.py -------------------------------------------------------------------------------- /src/cijoe/qemu/configs/example_config_build.toml: -------------------------------------------------------------------------------- 1 | # 2 | # This part of the CIJOE qemu package is not retargetable, that means, this 3 | # scripts only run on the initiator, thus, there is no "cijoe.transport" 4 | # configuration here. 5 | # 6 | 7 | [cijoe.workflow] 8 | fail_fast=true 9 | 10 | # Used by: qemu.build.py 11 | [qemu.repository] 12 | remote = "https://github.com/qemu/qemu.git" 13 | path = "{{ local.env.HOME }}/git/qemu" 14 | tag = "v9.2.0" 15 | 16 | # Used by: qemu.build.py 17 | [qemu.build] 18 | prefix = "{{ local.env.HOME }}/opt/qemu" 19 | 20 | # Used by: the qemu.*.py scripts 21 | [qemu] 22 | img_bin = "{{ local.env.HOME }}/opt/qemu/bin/qemu-img" 23 | 24 | [qemu.systems.x86_64] 25 | bin = "{{ local.env.HOME }}/opt/qemu/bin/qemu-system-x86_64" 26 | 27 | [qemu.systems.aarch64] 28 | bin = "{{ local.env.HOME }}/opt/qemu/bin/qemu-system-aarch64" -------------------------------------------------------------------------------- /src/cijoe/qemu/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/qemu/scripts/__init__.py -------------------------------------------------------------------------------- /src/cijoe/qemu/scripts/build.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Build qemu system and default tools 4 | =================================== 5 | 6 | This script automates the process of building QEMU from its source repository at 7 | the given repository path. It configures the build environment with VirtFS and 8 | debugging enabled. 9 | 10 | Configuration 11 | ------------- 12 | 13 | * ``qemu.repository.path``: str 14 | 15 | Path to the qemu repository on the target machine. 16 | 17 | * ``qemu.build.prefix``: str 18 | 19 | Prefix given to the qemu configuration in the `--prefix` argument 20 | 21 | Retargetable: True 22 | ------------------ 23 | """ 24 | import errno 25 | import logging as log 26 | from pathlib import Path 27 | 28 | 29 | def main(args, cijoe): 30 | """Build qemu""" 31 | 32 | repos_path = cijoe.getconf("qemu.repository.path", None) 33 | if not repos_path: 34 | log.error("missing qemu.repository.path") 35 | return errno.EINVAL 36 | 37 | prefix = cijoe.getconf("qemu.build.prefix") 38 | if not prefix: 39 | log.error("missing qemu.build.prefix") 40 | return errno.EINVAL 41 | 42 | err, _ = cijoe.run(f'[ -d "{repos_path}" ]') 43 | if err: 44 | log.error(f"No qemu git-repository at repos({repos_path})") 45 | return err 46 | 47 | build_dir = Path(repos_path) / "build" 48 | 49 | err, _ = cijoe.run(f"mkdir -p {build_dir}") 50 | if err: 51 | return err 52 | 53 | configure_args = [ 54 | f"--prefix={prefix}", 55 | "--audio-drv-list=''", 56 | "--disable-docs", 57 | "--disable-glusterfs", 58 | "--disable-libnfs", 59 | "--disable-libusb", 60 | "--disable-opengl", 61 | "--disable-sdl", 62 | "--disable-smartcard", 63 | "--disable-spice", 64 | "--disable-virglrenderer", 65 | "--disable-vnc", 66 | "--disable-vte", 67 | "--disable-xen", 68 | "--enable-debug", 69 | "--enable-virtfs", 70 | "--target-list=x86_64-softmmu,aarch64-softmmu", 71 | ] 72 | err, _ = cijoe.run("../configure " + " ".join(configure_args), cwd=build_dir) 73 | if err: 74 | return err 75 | 76 | err, _ = cijoe.run("make -j $(nproc)", cwd=build_dir) 77 | if err: 78 | return err 79 | 80 | return 0 81 | -------------------------------------------------------------------------------- /src/cijoe/qemu/scripts/guest_initialize.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Guest initialization 4 | ==================== 5 | 6 | This script initializes a guest environment by setting up the necessary resources 7 | for guest-instance management. It creates a directory structure to store files 8 | related to the guest, such as: 9 | 10 | - PID files 11 | - Monitor files 12 | - Serial input/output logs 13 | - Other instance-specific metadata 14 | 15 | Additionally, the script prepares guest storage, including the boot drive, 16 | using an existing `.qcow2` image (e.g., created via Cloud-init, Packer, etc.). 17 | 18 | Configuration 19 | ------------- 20 | 21 | * ``qemu.guests..system_image_name``: str 22 | 23 | Name of the system image. This will be overwritten if also given as script 24 | argument. 25 | 26 | * ``system-imaging.images..disk``: dict 27 | 28 | A dictionary containing the path to the disk image and, if this path does not 29 | exist, a URL from where the disk image cna be downloaded. 30 | 31 | Retargetable: False 32 | ------------------- 33 | """ 34 | import errno 35 | import logging as log 36 | from argparse import ArgumentParser 37 | from pathlib import Path 38 | 39 | from cijoe.core.misc import download, download_and_verify 40 | from cijoe.qemu.wrapper import Guest 41 | 42 | 43 | def add_args(parser: ArgumentParser): 44 | parser.add_argument("--guest_name", type=str, help="Name of the qemu guest.") 45 | parser.add_argument( 46 | "--system_image_name", 47 | type=str, 48 | help="Name of the system image. This will overwrite any system image name defined in the configuration file.", 49 | ) 50 | 51 | 52 | def main(args, cijoe): 53 | """Provision using an existing boot image""" 54 | 55 | if "guest_name" not in args: 56 | log.error("missing argument: guest_name") 57 | return errno.EINVAL 58 | 59 | guest = Guest(cijoe, cijoe.config, args.guest_name) 60 | 61 | system_image_name = cijoe.getconf( 62 | f"qemu.guests.{args.guest_name}.system_image_name", None 63 | ) 64 | if "system_image_name" in args: 65 | system_image_name = args.system_image_name 66 | 67 | if system_image_name is None: 68 | log.error("qemu.guests.THIS.system_image_name is not set") 69 | return errno.EINVAL 70 | 71 | if ( 72 | disk := cijoe.getconf(f"system-imaging.images.{system_image_name}.disk", None) 73 | ) is None: 74 | log.error(f"system-imaging.images.{system_image_name}.disk is not set") 75 | return errno.EINVAL 76 | 77 | if not (diskimage_path := Path(disk.get("path"))).exists(): 78 | if (disk_url := disk.get("url", None)) is None: 79 | log.error( 80 | f"Cannot download; no 'url' in configuration-file for disk({disk})" 81 | ) 82 | return errno.EINVAL 83 | 84 | diskimage_path.parent.mkdir(exist_ok=True, parents=True) 85 | 86 | disk_url_checksum = disk.get("url_checksum", None) 87 | err, path = ( 88 | download_and_verify(disk_url, disk_url_checksum, diskimage_path) 89 | if disk_url_checksum 90 | else download(disk_url, diskimage_path) 91 | ) 92 | if err: 93 | log.error(f"err({err}, path({path})") 94 | return err 95 | 96 | err = guest.initialize(diskimage_path) 97 | if err: 98 | log.error(f"guest.initialize({diskimage_path}); err({err})") 99 | return err 100 | 101 | return 0 102 | -------------------------------------------------------------------------------- /src/cijoe/qemu/scripts/guest_kill.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Kill a qemu guest 4 | ================= 5 | 6 | Shutdown qemu guests by killing the process using the pid associated with the 7 | given guest name. 8 | 9 | Note: The script will not fail if the guest does not exist. 10 | 11 | Retargetable: False 12 | ------------------- 13 | """ 14 | import logging as log 15 | from argparse import ArgumentParser 16 | 17 | from cijoe.qemu.wrapper import Guest 18 | 19 | 20 | def add_args(parser: ArgumentParser): 21 | parser.add_argument("--guest_name", type=str, help="Name of the qemu guest.") 22 | 23 | 24 | def main(args, cijoe): 25 | """Kill a qemu guest""" 26 | 27 | if "guest_name" not in args: 28 | log.error("missing argument: guest_name") 29 | return 1 30 | 31 | guest = Guest(cijoe, cijoe.config, args.guest_name) 32 | 33 | return guest.kill() 34 | -------------------------------------------------------------------------------- /src/cijoe/qemu/scripts/guest_start.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Start a qemu guest 4 | ================== 5 | 6 | Starts the qemu guest with the given guest name. Fails if the guest is not up 7 | within 180 seconds. 8 | 9 | Retargetable: False 10 | ------------------- 11 | """ 12 | import errno 13 | import logging as log 14 | from argparse import ArgumentParser 15 | 16 | from cijoe.qemu.wrapper import Guest 17 | 18 | 19 | def add_args(parser: ArgumentParser): 20 | parser.add_argument("--guest_name", type=str, help="Name of the qemu guest.") 21 | 22 | 23 | def main(args, cijoe): 24 | """Start a qemu guest""" 25 | 26 | if "guest_name" not in args: 27 | log.error("missing argument: guest_name") 28 | return 1 29 | 30 | guest = Guest(cijoe, cijoe.config, args.guest_name) 31 | 32 | err = guest.start() 33 | if err: 34 | log.error(f"guest.start() : err({err})") 35 | return err 36 | 37 | started = guest.is_up(timeout=180) 38 | if not started: 39 | log.error("guest.is_up() : False") 40 | return errno.EAGAIN 41 | 42 | return 0 43 | -------------------------------------------------------------------------------- /src/cijoe/qemu/scripts/guest_wait_for_termination.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Wait for qemu guest termination 4 | =============================== 5 | 6 | Note: The script will not fail if the guest does not exist. 7 | Note: This script does not itself terminate the qemu guest. 8 | 9 | Retargetable: False 10 | ------------------- 11 | """ 12 | import logging as log 13 | import time 14 | from argparse import ArgumentParser 15 | 16 | from cijoe.qemu.wrapper import Guest 17 | 18 | 19 | def add_args(parser: ArgumentParser): 20 | parser.add_argument("--guest_name", type=str, help="Name of the qemu guest.") 21 | parser.add_argument( 22 | "--timeout", 23 | type=int, 24 | default=60, 25 | help="Amount of seconds to wait for the qemu guest to terminate.", 26 | ) 27 | 28 | 29 | def main(args, cijoe): 30 | """Wait for termination of qemu guest""" 31 | 32 | if "guest_name" not in args: 33 | log.error("missing argument: guest_name") 34 | return 1 35 | 36 | guest = Guest(cijoe, cijoe.config, args.guest_name) 37 | 38 | start = time.time() 39 | err, terminated = guest.wait_for_termination(args.timeout) 40 | end = time.time() 41 | 42 | if terminated: 43 | log.info( 44 | f"Guest({args.guest_name}) terminated gracefully in {end-start} seconds" 45 | ) 46 | else: 47 | log.warning( 48 | f"Guest({args.guest_name}) was not terminated within {args.timeout} seconds" 49 | ) 50 | 51 | return err 52 | -------------------------------------------------------------------------------- /src/cijoe/qemu/scripts/install.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Install qemu 4 | ============ 5 | 6 | This script installs qemu on the host machine. It is assumed that qemu has been 7 | built in advance at the given repository path. 8 | 9 | Configuration 10 | ------------- 11 | 12 | * ``qemu.repository.path``: str 13 | 14 | Path to the qemu repository on the target machine. 15 | 16 | Retargetable: False 17 | ------------------- 18 | """ 19 | import errno 20 | import logging as log 21 | from pathlib import Path 22 | 23 | 24 | def main(args, cijoe): 25 | """Install qemu""" 26 | 27 | path = cijoe.getconf("qemu.repository.path", None) 28 | if not path: 29 | log.error("missing 'qemu.repository.path' in configuration file") 30 | return errno.EINVAL 31 | 32 | build_dir = Path(path) / "build" 33 | 34 | err, _ = cijoe.run_local("make install", cwd=build_dir) 35 | 36 | return err 37 | -------------------------------------------------------------------------------- /src/cijoe/qemu/scripts/qemu_version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Determine version of qemu-img and qemu-system-bins 4 | ================================================== 5 | 6 | This script records the qemu-img, qemu-system-x86_64, qemu-system-aarch64, etc. versions 7 | in the runlogs, ensuring the version used by the qemu.wrapper is always available for 8 | inspection in reports. 9 | 10 | Retargetable: False 11 | ------------------- 12 | """ 13 | from cijoe.qemu.wrapper import qemu_img, qemu_system 14 | 15 | 16 | def main(args, cijoe): 17 | """Install qemu""" 18 | 19 | errors = [] 20 | 21 | err, _ = qemu_img(cijoe, "--version") 22 | errors.append(err) 23 | 24 | for system_label in cijoe.getconf("qemu.systems", {}).keys(): 25 | err, _ = qemu_system(cijoe, system_label, "--version") 26 | errors.append(err) 27 | 28 | for err in errors: 29 | if err: 30 | return err 31 | 32 | return 0 33 | -------------------------------------------------------------------------------- /src/cijoe/qemu/workflows/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/qemu/workflows/__init__.py -------------------------------------------------------------------------------- /src/cijoe/qemu/workflows/example_workflow_build.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | doc: | 3 | This workflow demonstrates how to build qemu from source 4 | 5 | * Build qemu from source (x86_64 and aarch64) 6 | * Install qemu 7 | * Check the qemu system version -- which the helper resolves to 8 | 9 | This is done via scripts, which in turn are utilizing helper-functions from 10 | cijoe.qemu.wrapper. 11 | 12 | steps: 13 | - name: repository_prep 14 | uses: core.repository_prep 15 | 16 | - name: build 17 | uses: qemu.build 18 | 19 | - name: install 20 | uses: qemu.install 21 | 22 | - name: qemu_system_version 23 | uses: qemu.qemu_version 24 | -------------------------------------------------------------------------------- /src/cijoe/qemu/workflows/example_workflow_guest_aarch64.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | doc: | 3 | This use of the 'qemu' shows how to initialize, start, and stop a qemu guest 4 | 5 | * Create a diskimage using a cloudimage 6 | 7 | - This is done using the 'system_imaging' package 8 | - Using Debian cloudinit as a reference 9 | 10 | * Initialize a 'system_imaging' provided disk image 11 | 12 | - See the 'system_imaging' package for examples of creating disk-images 13 | 14 | * Start the guest 15 | * Run a command within the guest (via SSH) 16 | * Stop the guest again 17 | 18 | This is done via scripts, which in turn are utilizing helper-functions from 19 | cijoe.qemu.wrapper. 20 | 21 | steps: 22 | - name: diskimage_from_cloudimage 23 | uses: system_imaging.diskimage_from_cloudimage 24 | with: 25 | pattern: "debian-12-aarch64" 26 | 27 | - name: guest_initialize 28 | uses: qemu.guest_initialize 29 | with: 30 | guest_name: generic-uefi-tcg-aarch64 31 | system_image_name: debian-12-aarch64 32 | 33 | - name: guest_start 34 | uses: qemu.guest_start 35 | with: 36 | guest_name: generic-uefi-tcg-aarch64 37 | 38 | - name: guest_check 39 | run: | 40 | hostname 41 | 42 | - name: guest_shutdown 43 | run: | 44 | sudo shutdown -h now 45 | 46 | - name: guest_wait 47 | uses: qemu.guest_wait_for_termination 48 | with: 49 | guest_name: generic-uefi-tcg-aarch64 50 | timeout: 60 51 | 52 | - name: guest_kill 53 | uses: qemu.guest_kill 54 | with: 55 | guest_name: generic-uefi-tcg-aarch64 56 | -------------------------------------------------------------------------------- /src/cijoe/qemu/workflows/example_workflow_guest_x86_64.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | doc: | 3 | This use of the 'qemu' shows how to initialize, start, and stop a qemu guest 4 | 5 | * Create a diskimage using a cloudimage 6 | 7 | - This is done using the 'system_imaging' package 8 | - Using Debian cloudinit as a reference 9 | 10 | * Initialize a 'system_imaging' provided disk image 11 | 12 | - See the 'system_imaging' package for examples of creating disk-images 13 | 14 | * Start the guest 15 | * Run a command within the guest (via SSH) 16 | * Stop the guest again 17 | 18 | This is done via scripts, which in turn are utilizing helper-functions from 19 | cijoe.qemu.wrapper. 20 | 21 | steps: 22 | - name: diskimage_from_cloudimage 23 | uses: system_imaging.diskimage_from_cloudimage 24 | with: 25 | pattern: "debian-12-x86_64" 26 | 27 | - name: guest_initialize 28 | uses: qemu.guest_initialize 29 | with: 30 | guest_name: generic-bios-kvm-x86_64 31 | system_image_name: debian-12-x86_64 32 | 33 | - name: guest_start 34 | uses: qemu.guest_start 35 | with: 36 | guest_name: generic-bios-kvm-x86_64 37 | 38 | - name: guest_check 39 | run: | 40 | hostname 41 | 42 | - name: guest_shutdown 43 | run: | 44 | sudo shutdown -h now 45 | 46 | - name: guest_wait 47 | uses: qemu.guest_wait_for_termination 48 | with: 49 | guest_name: generic-bios-kvm-x86_64 50 | timeout: 60 51 | 52 | - name: guest_kill 53 | uses: qemu.guest_kill 54 | with: 55 | guest_name: generic-bios-kvm-x86_64 56 | -------------------------------------------------------------------------------- /src/cijoe/system_imaging/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/system_imaging/__init__.py -------------------------------------------------------------------------------- /src/cijoe/system_imaging/auxiliary/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM scratch 2 | COPY . / 3 | CMD ["/bin/bash"] 4 | -------------------------------------------------------------------------------- /src/cijoe/system_imaging/auxiliary/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/system_imaging/auxiliary/__init__.py -------------------------------------------------------------------------------- /src/cijoe/system_imaging/auxiliary/cloudinit-freebsd-metadata.meta: -------------------------------------------------------------------------------- 1 | instance-id: i-cijoe-freebsd 2 | local-hostname: cijoe-freebsd 3 | -------------------------------------------------------------------------------- /src/cijoe/system_imaging/auxiliary/cloudinit-freebsd-userdata.user: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | disable_root: False 3 | ssh_pwauth: True 4 | package_update: true 5 | package_upgrade: true 6 | packages: 7 | - fio 8 | - htop 9 | - nvme-cli 10 | - pciutils 11 | write_files: 12 | - path: /etc/ssh/sshd_config 13 | content: | 14 | PermitRootLogin yes 15 | PermitEmptyPasswords yes 16 | runcmd: 17 | - [pw, usermod, root, -w, none] 18 | - [sed, -E, -i.bak, 's/auth[[:space:]]+required[[:space:]]+pam_unix\.so[[:space:]]+no_warn[[:space:]]+try_first_pass/& nullok/', /etc/pam.d/sshd] 19 | final_message: "The system is finally up, after $UPTIME seconds" 20 | power_state: 21 | mode: poweroff 22 | message: So long and thanks for all the fish 23 | timeout: 30 24 | condition: True 25 | -------------------------------------------------------------------------------- /src/cijoe/system_imaging/auxiliary/cloudinit-linux-alpine-userdata.user: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | datasource_list: [NoCloud, ConfigDrive] 3 | disable_root: false 4 | ssh_pwauth: true 5 | 6 | chpasswd: 7 | expire: false 8 | users: 9 | - {name: root, password: root, type: text} 10 | 11 | package_update: true 12 | package_upgrade: true 13 | packages: 14 | - htop 15 | - lshw 16 | - pciutils 17 | - busybox 18 | 19 | write_files: 20 | - path: /etc/ssh/sshd_config 21 | content: | 22 | PermitRootLogin yes 23 | PermitEmptyPasswords yes 24 | PasswordAuthentication yes 25 | owner: root:root 26 | permissions: '0644' 27 | 28 | runcmd: 29 | - service sshd restart 30 | 31 | final_message: "The system is up, after $UPTIME seconds" 32 | power_state: 33 | mode: poweroff 34 | message: So long and thanks for all the fish 35 | timeout: 30 36 | condition: true 37 | -------------------------------------------------------------------------------- /src/cijoe/system_imaging/auxiliary/cloudinit-linux-common-metadata.meta: -------------------------------------------------------------------------------- 1 | instance-id: i-cijoe-linux 2 | local-hostname: cijoe-linux 3 | -------------------------------------------------------------------------------- /src/cijoe/system_imaging/auxiliary/cloudinit-linux-common-userdata.user: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | disable_root: false 3 | ssh_pwauth: true 4 | 5 | chpasswd: 6 | expire: false 7 | users: 8 | - {name: root, password: root, type: text} 9 | 10 | package_update: true 11 | package_upgrade: true 12 | packages: 13 | - htop 14 | - lshw 15 | - pciutils 16 | 17 | write_files: 18 | - path: /etc/ssh/sshd_config.d/00-cijoe.conf 19 | content: | 20 | PermitRootLogin yes 21 | PermitEmptyPasswords yes 22 | PasswordAuthentication yes 23 | owner: root:root 24 | permissions: '0644' 25 | 26 | runcmd: 27 | - systemctl restart ssh 28 | 29 | final_message: "The system is up, after $UPTIME seconds" 30 | power_state: 31 | mode: poweroff 32 | message: So long and thanks for all the fish 33 | timeout: 30 34 | condition: true 35 | -------------------------------------------------------------------------------- /src/cijoe/system_imaging/auxiliary/dockerignore: -------------------------------------------------------------------------------- 1 | /etc/fstab 2 | /dev/* 3 | /proc/* 4 | /sys/* -------------------------------------------------------------------------------- /src/cijoe/system_imaging/configs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/system_imaging/configs/__init__.py -------------------------------------------------------------------------------- /src/cijoe/system_imaging/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/system_imaging/scripts/__init__.py -------------------------------------------------------------------------------- /src/cijoe/system_imaging/scripts/diskimage_from_cloudimage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Create disk image using a cloud image 4 | ===================================== 5 | 6 | This will produce disk images for all the system images described in config. section: 7 | 8 | * ``system_imaging.images`` 9 | 10 | You can reduce this by providing a case-incensitive fnmatch pattern as input to the 11 | script via workflow, such as these:: 12 | 13 | # This will build all images 14 | with: 15 | pattern: "*" 16 | 17 | # This will build all those starting with "debian" 18 | with: 19 | pattern: "debian*" 20 | 21 | Retargetable: False 22 | ------------------- 23 | 24 | This script only runs on the iniator; due to the use of 'shutil', 'download' etc. 25 | """ 26 | import errno 27 | import logging as log 28 | import shutil 29 | from argparse import ArgumentParser 30 | from fnmatch import fnmatch 31 | from pathlib import Path 32 | from pprint import pformat 33 | 34 | from cijoe.core.misc import download 35 | from cijoe.qemu.wrapper import Guest, qemu_img 36 | 37 | 38 | def add_args(parser: ArgumentParser): 39 | parser.add_argument("--pattern", type=str, help="Pattern for image names to build") 40 | 41 | 42 | def diskimage_from_cloudimage(cijoe, image: dict): 43 | """ 44 | Build a diskimage, using qemu and cloudimage, and copy it to the diskimage location 45 | """ 46 | 47 | if not (cloud := image.get("cloud", {})): 48 | log.error("missing .cloud entry in configuration file") 49 | return errno.EINVAL 50 | 51 | cloud_image_path = Path(cloud.get("path")) 52 | cloud_image_url = cloud.get("url") 53 | cloud_image_metadata_path = Path(cloud.get("metadata_path")) 54 | cloud_image_userdata_path = Path(cloud.get("userdata_path")) 55 | 56 | if not (disk := image.get("disk", {})): 57 | log.error("missing .disk entry in configuration file") 58 | return errno.EINVAL 59 | 60 | if not cloud_image_path.exists(): 61 | cloud_image_path.parent.mkdir(parents=True, exist_ok=True) 62 | 63 | err, path = download(cloud_image_url, cloud_image_path) 64 | if err: 65 | log.error(f"download({cloud_image_url}), {cloud_image_path}: failed") 66 | return err 67 | 68 | if (system_label := image.get("system_label", None)) is None: 69 | log.error("missing .system_label entry in configuration file") 70 | pass 71 | 72 | # Get the first guest with a matching system_label 73 | guest_name = None 74 | for cur_guest_name, cur_guest in cijoe.getconf("qemu.guests", {}).items(): 75 | guest_system_label = cur_guest.get("system_label", None) 76 | if guest_system_label is None: 77 | log.error(f"guest_name({cur_guest_name}) is missing 'system_label'") 78 | return errno.EINVAL 79 | 80 | if guest_system_label == system_label: 81 | guest_name = cur_guest_name 82 | break 83 | 84 | if guest_name is None: 85 | log.error("Could not find a guest to use for diskimage creation") 86 | return errno.EINVAL 87 | 88 | guest = Guest(cijoe, cijoe.config, guest_name) 89 | guest.kill() # Ensure the guest is *not* running 90 | guest.initialize(cloud_image_path) # Initialize using the cloudimage 91 | 92 | # Create seed.img, with data and meta embedded 93 | guest_metadata_path = guest.guest_path / "meta-data" 94 | err, _ = cijoe.run_local(f"cp {cloud_image_metadata_path} {guest_metadata_path}") 95 | guest_userdata_path = guest.guest_path / "user-data" 96 | err, _ = cijoe.run_local(f"cp {cloud_image_userdata_path} {guest_userdata_path}") 97 | 98 | # This uses mkisofs instead of cloud-localds, such that it works on macOS and Linux, 99 | # the 'mkisofs' should be available with 'cdrtools' 100 | seed_img = guest.guest_path / "seed.img" 101 | cloud_cmd = " ".join( 102 | [ 103 | "mkisofs", 104 | "-output", 105 | f"{seed_img}", 106 | "-volid", 107 | "cidata", 108 | "-joliet", 109 | "-rock", 110 | str(guest_userdata_path), 111 | str(guest_metadata_path), 112 | ] 113 | ) 114 | err, _ = cijoe.run_local(cloud_cmd) 115 | if err: 116 | log.error(f"Failed creating {seed_img}") 117 | return err 118 | 119 | # Additional args to pass to the guest when starting it 120 | system_args = [] 121 | 122 | system_args += ["-cdrom", f"{seed_img}"] 123 | 124 | # When not daemonized then this will block until the machine shuts down, which is 125 | # what we want, as we want to wait for the cloudinit process to finalize 126 | err = guest.start(daemonize=False, extra_args=system_args) 127 | if err: 128 | log.error("Failure starting guest or during cloudinit process") 129 | return err 130 | 131 | # Copy to disk-location 132 | disk_path = Path(disk.get("path")) 133 | disk_path.parent.mkdir(parents=True, exist_ok=True) 134 | err, _ = cijoe.run_local(f"cp {guest.boot_img} {disk_path}") 135 | if err: 136 | log.error(f"Failed copying to {disk_path}") 137 | return err 138 | 139 | # Resize the .qcow file This still requires that the partitions are resized with 140 | # e.g. growpart as part of the cloud-init process 141 | cijoe.run_local(f"qemu-img info {disk_path}") 142 | 143 | err, _ = cijoe.run_local(f"qemu-img resize {disk_path} 12G") 144 | if err: 145 | log.error("Failed resizing .qcow image") 146 | return err 147 | 148 | cijoe.run_local(f"qemu-img info {disk_path}") 149 | 150 | # Compute sha256sum of the disk-image 151 | err, _ = cijoe.run_local(f"sha256sum {disk_path} > {disk_path}.sha256") 152 | if err: 153 | log.error(f"Failed computing sha256 sum of disk_path({disk_path})") 154 | return err 155 | 156 | cijoe.run_local(f"ls -la {disk_path}") 157 | cijoe.run_local(f"cat {disk_path}.sha256") 158 | 159 | return 0 160 | 161 | 162 | def main(args, cijoe): 163 | """Provision a qemu-guest using a cloud-init image""" 164 | 165 | if "pattern" not in args: 166 | log.error("missing step-argument: with.pattern") 167 | return errno.EINVAL 168 | pattern = args.pattern 169 | 170 | log.info(f"Got pattern({pattern})") 171 | 172 | entry_name = "system-imaging.images" 173 | images = cijoe.getconf(entry_name, {}) 174 | if not images: 175 | log.error(f"missing: '{entry_name}' in configuration file") 176 | return errno.EINVAL 177 | 178 | build_status = {} 179 | for image_name, image in images.items(): 180 | if not fnmatch(image_name.lower(), pattern.lower()): 181 | log.info(f"image_name({image_name}); did not match pattern({pattern}") 182 | continue 183 | 184 | build_status[image_name] = False 185 | log.info(f"image_name({image_name}); matched pattern({pattern})") 186 | 187 | err = diskimage_from_cloudimage(cijoe, image) 188 | if err: 189 | log.error(f"failed build_and_copy(); err({err})") 190 | return err 191 | 192 | build_status[image_name] = True 193 | 194 | count = sum([1 for status in build_status.values() if status]) 195 | log.info(f"Build count({count}) disk images; status: {pformat(build_status)}") 196 | 197 | if not count: 198 | log.error(f"did not build anything, count({count}); invalid with.pattern?") 199 | return errno.EINVAL 200 | 201 | return 0 202 | -------------------------------------------------------------------------------- /src/cijoe/system_imaging/scripts/dockerimage_from_diskimage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Create docker image using a disk image 4 | ====================================== 5 | 6 | This will produce dockers images for all the system images described in config. section: 7 | 8 | * ``system_imaging.images`` 9 | 10 | You can reduce this by providing a case-incensitive fnmatch pattern as input to the 11 | script via workflow, such as these:: 12 | 13 | # This will build all images 14 | with: 15 | pattern: "*" 16 | 17 | # This will build all those starting with debian 18 | with: 19 | pattern: "debian*" 20 | 21 | Retargetable: False 22 | ------------------- 23 | 24 | This script only runs on the iniator; due to the use of 'shutil', 'download' etc. 25 | """ 26 | import logging as log 27 | import shutil 28 | from argparse import ArgumentParser 29 | from fnmatch import fnmatch 30 | from pathlib import Path 31 | 32 | from cijoe.core.misc import download 33 | from cijoe.core.resources import get_resources 34 | 35 | 36 | def add_args(parser: ArgumentParser): 37 | parser.add_argument("--pattern", type=str, help="Pattern for image names to build") 38 | 39 | 40 | def dockerimage_from_diskimage(cijoe, image): 41 | resources = get_resources() 42 | 43 | err, state = cijoe.run_local("mktemp -d") # Create temporary directory 44 | if err: 45 | log.error("Failed creating workdir") 46 | return 47 | 48 | workdir = Path(state.output().strip()) 49 | mountpoint = workdir / "mount" 50 | 51 | shutil.copyfile( 52 | str(resources.get("auxiliary", {}).get("system_imaging.Dockerfile").path), 53 | str(workdir / "Dockerfile"), 54 | ) 55 | shutil.copyfile( 56 | str(resources.get("auxiliary", {}).get("system_imaging.dockerignore").path), 57 | str(workdir / ".dockerignore"), 58 | ) 59 | 60 | err, _ = cijoe.run_local(f'mkdir -p "{mountpoint}"') 61 | 62 | commands = [ 63 | f"guestmount -a {image['disk']['path']} -i --ro {mountpoint}", 64 | f"docker build -t {image['docker']['name']}:{image['docker']['tag']} -f {workdir}/Dockerfile {mountpoint}", 65 | f"guestunmount {mountpoint}", 66 | ] 67 | for command in commands: 68 | err, _ = cijoe.run_local(command) 69 | if err: 70 | log.error(f"command({command}); err({err})") 71 | return err 72 | 73 | cijoe.run_local(f'echo "Needs cleanup!" && find {workdir}') 74 | cijoe.run_local( 75 | f'echo "Run with: docker run -it {image["docker"]["name"]}:{image["docker"]["tag"]} bash"' 76 | ) 77 | 78 | return 0 79 | 80 | 81 | def main(args, cijoe): 82 | """Create a docker image using the content of a .qcow2 image""" 83 | 84 | if "pattern" not in args: 85 | log.error("missing step-argument: with.pattern") 86 | return 1 87 | pattern = args.pattern 88 | 89 | log.info(f"Got pattern({pattern})") 90 | 91 | entry_name = "system-imaging.images" 92 | images = cijoe.getconf(entry_name, {}) 93 | if not images: 94 | log.error(f"missing: '{entry_name}' in configuration file") 95 | return 1 96 | 97 | count = 0 98 | for image_name, image in cijoe.getconf("system-imaging.images", {}).items(): 99 | if not fnmatch(image_name.lower(), pattern.lower()): 100 | log.info(f"image_name({image_name}); did not match pattern({pattern}") 101 | continue 102 | 103 | log.info(f"image_name({image_name}); matched pattern({pattern})") 104 | 105 | err = dockerimage_from_diskimage(cijoe, image) 106 | if err: 107 | log.error(f"failed dockerimage_from_diskimage(); err({err})") 108 | return err 109 | 110 | count += 1 111 | 112 | if not count: 113 | log.error(f"did not build anything, count({count}); invalid with.pattern?") 114 | return err 115 | 116 | return 0 117 | -------------------------------------------------------------------------------- /src/cijoe/system_imaging/workflows/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/refenv/cijoe/1629fb3196e5a2a0dcf7c15549c3ae8b3d9c2b25/src/cijoe/system_imaging/workflows/__init__.py -------------------------------------------------------------------------------- /src/cijoe/system_imaging/workflows/example_workflow_aarch64.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | doc: | 3 | Build disk and docker images 4 | ============================ 5 | 6 | Here is what the workflow consumes and produces: 7 | 8 | * cloud image ==> disk image (.qcow2) ==> docker image 9 | 10 | This images are usable for: 11 | 12 | * Disk image for a virtual machine 13 | 14 | - This can be e.g. a qemu-guest or other host-vmm / hypervisor 15 | 16 | * Disk image for a physical machine 17 | 18 | - By writing to bootable media using qemu-img 19 | 20 | * Docker image 21 | 22 | - Run locally using Docker engine, desktop e.g. GitHUB 23 | 24 | It uses the 'system_imaging' section of the configuration file as input, the 25 | system images matching 'pattern' are processed. 26 | 27 | NOTE: This creates images for all aarch64 system images described in the 28 | "system_imaging" section of the cijoe configuration file. 29 | 30 | steps: 31 | - name: diskimage_from_cloudimage 32 | uses: system_imaging.diskimage_from_cloudimage 33 | with: 34 | pattern: "*aarch64*" 35 | 36 | - name: dockerimage_from_diskimage 37 | uses: system_imaging.dockerimage_from_diskimage 38 | with: 39 | pattern: "*aarch64*" 40 | -------------------------------------------------------------------------------- /src/cijoe/system_imaging/workflows/example_workflow_x86_64.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | doc: | 3 | Build disk and docker images 4 | ============================ 5 | 6 | Here is what the workflow consumes and produces: 7 | 8 | * cloud image ==> disk image (.qcow2) ==> docker image 9 | 10 | This images are usable for: 11 | 12 | * Disk image for a virtual machine 13 | 14 | - This can be e.g. a qemu-guest or other host-vmm / hypervisor 15 | 16 | * Disk image for a physical machine 17 | 18 | - By writing to bootable media using qemu-img 19 | 20 | * Docker image 21 | 22 | - Run locally using Docker engine, desktop e.g. GitHUB 23 | 24 | It uses the 'system_imaging' section of the configuration file as input, the 25 | system images matching 'pattern' are processed. 26 | 27 | NOTE: This creates images for all x86_64 system images described in the 28 | "system_imaging" section of the cijoe configuration file. This could be 29 | swapped with aarch64, however, it has not been tested yet. 30 | 31 | steps: 32 | - name: diskimage_from_cloudimage 33 | uses: system_imaging.diskimage_from_cloudimage 34 | with: 35 | pattern: "*x86_64*" 36 | 37 | - name: dockerimage_from_diskimage 38 | uses: system_imaging.dockerimage_from_diskimage 39 | with: 40 | pattern: "*x86_64*" 41 | -------------------------------------------------------------------------------- /tests/core/aux_loglevel.py: -------------------------------------------------------------------------------- 1 | import logging as log 2 | 3 | 4 | def main(args, cijoe): 5 | log.critical("critical") 6 | log.error("error") 7 | log.warning("warning") 8 | log.info("info") 9 | log.debug("debug") 10 | 11 | return 0 12 | -------------------------------------------------------------------------------- /tests/core/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from cijoe.core.resources import Collector 4 | 5 | 6 | @pytest.fixture(autouse=True) 7 | def reset_collector(): 8 | # This fixture runs automatically before each test 9 | collector = Collector() 10 | collector.reset() 11 | -------------------------------------------------------------------------------- /tests/core/test_cli_example.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | 4 | from cijoe.cli.cli import SEARCH_PATHS, search_for_file 5 | 6 | 7 | def test_cli_example_emit_listing(cijoe): 8 | err, _ = cijoe.run("cijoe --example") 9 | assert not err 10 | 11 | 12 | def test_cli_example_emit_specific(cijoe, tmp_path): 13 | err, state = cijoe.run("cijoe --example") 14 | assert not err 15 | 16 | examples = [line.strip() for line in state.output().splitlines()] 17 | for example in examples: 18 | err, state = cijoe.run(f"cijoe --example {example}", cwd=tmp_path) 19 | assert not err 20 | 21 | 22 | def test_cli_example_emit_all_in_package(cijoe, tmp_path): 23 | err, state = cijoe.run("cijoe --example") 24 | assert not err 25 | 26 | examples = [line.strip() for line in state.output().splitlines()] 27 | for pkg_name in list(set([line.split(".")[0] for line in examples])): 28 | err, state = cijoe.run(f"cijoe --example {pkg_name}", cwd=tmp_path) 29 | assert not err 30 | 31 | 32 | def test_emit_example_core(cijoe, tmp_path): 33 | err, _ = cijoe.run("cijoe --example core.default", cwd=tmp_path) 34 | assert not err 35 | 36 | 37 | def test_cli_version(cijoe): 38 | err, _ = cijoe.run("cijoe --version") 39 | pass 40 | assert not err 41 | 42 | 43 | def test_cli_resources(cijoe): 44 | err, _ = cijoe.run("cijoe --resources") 45 | assert not err 46 | 47 | 48 | def test_cli_integration_check(cijoe, tmp_path): 49 | # Get a temporary path and change directory so it is possible 50 | # to rerun test and it wont pollute the test environment 51 | 52 | err, _ = cijoe.run("cijoe --example core.default", cwd=tmp_path) 53 | assert not err 54 | 55 | err, _ = cijoe.run( 56 | "cijoe --integrity-check ", 57 | cwd=tmp_path / "cijoe-example-core.default", 58 | ) 59 | assert not err 60 | 61 | 62 | def test_cli_environment_variables(cijoe): 63 | # This test needs to be run in a session since we set a environment 64 | # variable and it should not pollute. 65 | os.environ["HELLO_WORLD"] = "true" 66 | message = cijoe.getconf("hello.world", None) 67 | assert message 68 | 69 | os.environ["HELLO_WORLD"] = "1" 70 | message = cijoe.getconf("hello.world", None) 71 | 72 | os.environ["HELLO_WORLD"] = "0x1" 73 | message = cijoe.getconf("hello.world", None) 74 | assert message == 1 75 | 76 | os.environ["HELLO_WORLD"] = "Hello World!" 77 | message = cijoe.getconf("hello.world", None) 78 | assert message == "Hello World!" 79 | 80 | # This should fail since 0xg is not a valid hex value 81 | os.environ["HELLO_WORLD"] = "0xg" 82 | message = cijoe.getconf("hello.world", None) 83 | 84 | 85 | def test_cli_search_for_file_exists(): 86 | filename = "tmpfile.txt" 87 | file = (SEARCH_PATHS[0] / filename).resolve() 88 | file.write_text("Hello World!") 89 | 90 | path = search_for_file(Path(filename)) 91 | assert path 92 | 93 | file.unlink() 94 | 95 | 96 | def test_cli_search_for_file_not_exists(): 97 | filename = "tmpfile.txt" 98 | 99 | path = search_for_file(Path(filename)) 100 | assert not path 101 | 102 | 103 | def test_cli_search_for_file_absolute_exists(): 104 | filename = "tmpfile.txt" 105 | file = (SEARCH_PATHS[0] / filename).resolve() 106 | file.write_text("Hello World!") 107 | 108 | path = search_for_file(file) 109 | assert path 110 | 111 | file.unlink() 112 | 113 | 114 | def test_cli_search_for_file_absolute_not_exists(): 115 | filename = "tmpfile.txt" 116 | file = SEARCH_PATHS[0] / filename 117 | 118 | path = search_for_file(file) 119 | assert not path 120 | -------------------------------------------------------------------------------- /tests/core/test_collector.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import cijoe.core 4 | from cijoe.core.resources import Collector 5 | 6 | CORE_RESOURCE_COUNTS = { 7 | "configs": 4, 8 | "templates": 2, 9 | "auxiliary": 2, 10 | "scripts": 8, 11 | } 12 | 13 | 14 | def test_resource_collection(): 15 | """Check that the expected amount of resources are collected""" 16 | 17 | collector = Collector() 18 | collector.collect() 19 | 20 | for category, count in CORE_RESOURCE_COUNTS.items(): 21 | assert ( 22 | len( 23 | [ 24 | r 25 | for r in collector.resources[category].keys() 26 | if r.startswith("core") 27 | ] 28 | ) 29 | == count 30 | ), f"Invalid count({count}) for category({category})" 31 | 32 | 33 | def test_collect_scripts_from_path(): 34 | """Uses the core package, to have something to collect.""" 35 | 36 | collector = Collector() 37 | collector.collect_from_path( 38 | Path(__file__) 39 | .parent.parent.parent.joinpath("src", "cijoe", "core", "scripts") 40 | .resolve() 41 | ) 42 | 43 | assert ( 44 | len(collector.resources["scripts"]) == CORE_RESOURCE_COUNTS["scripts"] 45 | ), "Failed collecting from path" 46 | 47 | 48 | def test_collect_scripts_from_packages(): 49 | collector = Collector() 50 | collector.collect_from_packages(cijoe.core.__path__, cijoe.core.__name__ + ".") 51 | 52 | assert ( 53 | len(collector.resources["scripts"]) == CORE_RESOURCE_COUNTS["scripts"] 54 | ), "Failed collecting from packages" 55 | 56 | 57 | def test_compare_from_path_with_from_package(): 58 | """This is just to give hint to whether it is 'from_path' or 'from_packages'""" 59 | 60 | collector_path = Collector() 61 | collector_path.collect_from_path(Path(__file__).parent.parent.joinpath("scripts")) 62 | 63 | collector_pkgs = Collector() 64 | collector_pkgs.collect_from_packages(cijoe.core.__path__, cijoe.core.__name__ + ".") 65 | 66 | assert len(collector_path.resources["scripts"]) == len( 67 | collector_pkgs.resources["scripts"] 68 | ) 69 | 70 | 71 | def test_collect_from_empty_path(): 72 | """This should return an empty dictionary""" 73 | 74 | collector = Collector() 75 | collector.collect_from_path("/tmp") 76 | 77 | assert len(collector.resources["scripts"]) == 0, "Did not expect to find any" 78 | -------------------------------------------------------------------------------- /tests/core/test_commands.py: -------------------------------------------------------------------------------- 1 | def test_hello(): 2 | assert True 3 | 4 | 5 | def test_default(cijoe): 6 | err, state = cijoe.run("ls -lh") 7 | 8 | assert err == 0 9 | 10 | 11 | def test_default_again(cijoe): 12 | err, state = cijoe.run("ls") 13 | 14 | assert err == 0 15 | -------------------------------------------------------------------------------- /tests/core/test_loglevel.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | from pathlib import Path 3 | 4 | 5 | def run_loglevel_script(tmp_path, loglevel) -> subprocess.CompletedProcess[str]: 6 | config_path = (tmp_path / "test-config-empty.toml").resolve() 7 | config_path.write_text("") 8 | 9 | script_file = Path(__file__).absolute().parent / "aux_loglevel.py" 10 | 11 | command = [ 12 | "cijoe", 13 | str(script_file), 14 | "--config", 15 | str(config_path), 16 | "--no-report", 17 | ] 18 | 19 | if loglevel: 20 | command += [f"-{loglevel*'l'}"] 21 | 22 | return subprocess.run( 23 | command, 24 | cwd=str(tmp_path), 25 | capture_output=True, 26 | text=True, 27 | ) 28 | 29 | 30 | def test_no_loglevel_parameter(tmp_path): 31 | result = run_loglevel_script(tmp_path, 0) 32 | 33 | assert result.returncode == 0 34 | assert "critical" in result.stderr 35 | assert "error" in result.stderr 36 | assert "warning" not in result.stderr 37 | assert "info" not in result.stderr 38 | assert "debug" not in result.stderr 39 | 40 | 41 | def test_one_loglevel_parameter(tmp_path): 42 | result = run_loglevel_script(tmp_path, 1) 43 | 44 | assert result.returncode == 0 45 | assert "warning" in result.stderr 46 | assert "info" in result.stderr 47 | assert "debug" not in result.stderr 48 | 49 | 50 | def test_two_loglevel_parameter(tmp_path): 51 | result = run_loglevel_script(tmp_path, 2) 52 | 53 | assert result.returncode == 0 54 | assert "debug" in result.stderr 55 | 56 | 57 | def test_more_loglevel_parameter(tmp_path): 58 | result = run_loglevel_script(tmp_path, 5) 59 | 60 | assert result.returncode == 0 61 | assert "debug" in result.stderr 62 | -------------------------------------------------------------------------------- /tests/core/test_transfer.py: -------------------------------------------------------------------------------- 1 | import filecmp 2 | import os 3 | import tempfile 4 | 5 | from cijoe.core.misc import ENCODING 6 | 7 | 8 | def test_push_pull(cijoe): 9 | """Create a file, fill it, push it, pull it, and compare""" 10 | 11 | with tempfile.NamedTemporaryFile( 12 | encoding=ENCODING, mode="a", delete=False 13 | ) as test_file: 14 | test_file.write("".join([chr(65 + (i % 24)) for i in range(4096)])) 15 | test_file.flush() 16 | 17 | assert cijoe.put(test_file.name, "foo"), "Failed push()" 18 | 19 | assert cijoe.get("foo", "bar"), "Failed pull()" 20 | 21 | pulled = os.path.join(cijoe.output_path, cijoe.output_ident, "bar") 22 | 23 | assert filecmp.cmp(test_file.name, pulled, shallow=False), "Failed cmp()" 24 | -------------------------------------------------------------------------------- /tests/core/test_workflow.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import subprocess 3 | from argparse import Namespace 4 | from pathlib import Path 5 | 6 | import yaml 7 | 8 | from cijoe.cli.cli import cli_integrity_check 9 | from cijoe.core.processing import runlog_from_path 10 | from cijoe.core.resources import get_resources 11 | 12 | WORKFLOW_SKELETON = { 13 | "doc": "Some description", 14 | "steps": [ 15 | {"name": "foo", "uses": "core.example_script_default"}, 16 | ], 17 | } 18 | 19 | 20 | def test_workflow_load(): 21 | resources = get_resources() 22 | 23 | config = resources["configs"]["core.example_config_default"] 24 | assert config 25 | 26 | errors = config.load() 27 | assert not errors 28 | 29 | workflow = resources["workflows"]["core.example_workflow_default"] 30 | assert workflow 31 | 32 | errors = workflow.load(Namespace(), config, []) 33 | assert not errors 34 | 35 | 36 | def test_workflow_lint_valid_workflow(tmp_path): 37 | 38 | config_path = (tmp_path / "test-config-empty.toml").resolve() 39 | config_path.write_text("") 40 | 41 | data = copy.deepcopy(WORKFLOW_SKELETON) 42 | 43 | workflow_file = (tmp_path / "workflow.yaml").resolve() 44 | workflow_file.write_text(yaml.dump(data)) 45 | 46 | result = subprocess.run( 47 | [ 48 | "cijoe", 49 | str(workflow_file), 50 | "--integrity-check", 51 | "--config", 52 | str(config_path), 53 | ], 54 | cwd=str(tmp_path), 55 | ) 56 | assert result.returncode == 0 57 | 58 | 59 | def test_workflow_lint_invalid_step_name(tmp_path): 60 | 61 | config_path = (tmp_path / "test-config-empty.toml").resolve() 62 | config_path.write_text("") 63 | 64 | data = copy.deepcopy(WORKFLOW_SKELETON) 65 | data.get("steps", []).append( 66 | {"name": "cannot have spaces", "with": "core.example_script_default"} 67 | ) 68 | 69 | workflow_file = (tmp_path / "workflow.yaml").resolve() 70 | workflow_file.write_text(yaml.dump(data)) 71 | 72 | result = subprocess.run( 73 | [ 74 | "cijoe", 75 | str(workflow_file), 76 | "--integrity-check", 77 | "--config", 78 | str(config_path), 79 | ], 80 | cwd=str(tmp_path), 81 | ) 82 | assert result.returncode != 0 83 | 84 | 85 | def test_workflow_report_command_ordering(tmp_path): 86 | 87 | config_path = (tmp_path / "test-config-empty.toml").resolve() 88 | config_path.write_text("") 89 | 90 | data = copy.deepcopy(WORKFLOW_SKELETON) 91 | data["steps"].append( 92 | { 93 | "name": "many_commands", 94 | "uses": "core.example_script_default", 95 | "with": {"repeat": 100}, 96 | } 97 | ) 98 | 99 | output_path = (tmp_path / "output").resolve() 100 | workflow_file = (tmp_path / "workflow.yaml").resolve() 101 | workflow_file.write_text(yaml.dump(data)) 102 | 103 | result = subprocess.run( 104 | [ 105 | "cijoe", 106 | str(workflow_file), 107 | "--output", 108 | str(output_path), 109 | "--config", 110 | str(config_path), 111 | ], 112 | cwd=str(tmp_path), 113 | ) 114 | assert result.returncode == 0 115 | 116 | for count, key in enumerate( 117 | runlog_from_path(output_path / "002_many_commands").keys(), 1 118 | ): 119 | val = int(Path(key).stem.split("_")[1]) 120 | assert count == val 121 | -------------------------------------------------------------------------------- /tests/linux/test_null_blk.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import cijoe.linux.null_blk as null_blk 4 | 5 | 6 | def skip_when_config_has_no_remote(cijoe): 7 | """Skip testing when configuration is module not enabled""" 8 | 9 | transport = cijoe.config.options.get("cijoe", {}).get("transport", None) 10 | if not transport: 11 | pytest.skip(reason="skipping as there is no remote transport defined") 12 | 13 | 14 | def test_insert(cijoe): 15 | """Test the creation of null_block via module-load""" 16 | 17 | skip_when_config_has_no_remote(cijoe) 18 | 19 | config = cijoe.config.options.get("null_blk", None) 20 | assert config, "Invalid environment configuration" 21 | 22 | nr_devices = int(config.get("nr_devices")) 23 | assert nr_devices, "!nr_devices, only module-load instances are supported" 24 | 25 | err, _ = null_blk.insert(cijoe) 26 | assert not err, "Failed inserting kernel module" 27 | 28 | err, _ = cijoe.run("lsblk") 29 | assert not err, "Failed listing block devices" 30 | 31 | for n in range(nr_devices): 32 | err, _ = cijoe.run(f"file /dev/nullb{n}") 33 | assert not err 34 | 35 | 36 | def test_remove(cijoe): 37 | skip_when_config_has_no_remote(cijoe) 38 | 39 | err, _ = null_blk.remove(cijoe) 40 | assert not err, "Failed removing kernel module" 41 | --------------------------------------------------------------------------------