├── .gitignore ├── requirements.txt ├── vagrant ├── test_scripts │ ├── test-arch-sanitizers-gcc.sh │ ├── test-rawhide-selinux.sh │ ├── test-arch.sh │ ├── test-arch-coverage.sh │ └── test-arch-sanitizers-clang.sh ├── boxes │ ├── Vagrantfile_archlinux_systemd │ ├── Vagrantfile_rawhide_selinux │ ├── rawhide_selinux.sh │ └── archlinux_systemd.sh ├── bootstrap_scripts │ ├── rawhide-selinux.sh │ ├── arch-coverage.sh │ ├── arch.sh │ ├── arch-sanitizers-gcc.sh │ └── arch-sanitizers-clang.sh ├── workarounds │ └── build-shared-libs.sh ├── vagrant-ci-wrapper.sh ├── vagrantfiles │ ├── Vagrantfile_rawhide │ └── Vagrantfile_arch ├── vagrant-setup.sh ├── vagrant-build.sh └── vagrant-make-cache.sh ├── jenkins ├── runners │ ├── upstream-vagrant-rawhide-selinux.sh │ ├── upstream-coverage.sh │ ├── rhel7-centos7.sh │ ├── rhel9-centos9-sanitizers.sh │ ├── upstream-vagrant-archlinux.sh │ ├── upstream-vagrant-archlinux-stable.sh │ ├── upstream-centos9s.sh │ ├── upstream-centos9s-stable.sh │ ├── rhel9-centos9.sh │ ├── upstream-vagrant-archlinux-sanitizers.sh │ ├── rhel8-centos8.sh │ ├── rhel-cron-build.sh │ ├── upstream-vagrant-archlinux-sanitizers-stable.sh │ ├── vagrant-make-cache.sh │ └── upstream-cron-build.sh ├── jenkins-configuration.md └── jenkins.yaml ├── .github └── workflows │ ├── differential-shellcheck.yml │ └── codeql.yml ├── utils ├── kexec.sh ├── artifacts-copy-file.sh ├── reposync.sh └── generate-index.sh ├── agent ├── testsuite-rhel7.sh ├── bootstrap-rhel7.sh ├── testsuite-rhel8.sh ├── testsuite.sh ├── testsuite-rhel9.sh ├── bootstrap.sh └── bootstrap-rhel8.sh ├── README.md └── common └── task-control.sh /.gitignore: -------------------------------------------------------------------------------- 1 | duffy.key 2 | *.sw* 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | duffy[client]>=3.3.5 2 | httpx 3 | -------------------------------------------------------------------------------- /vagrant/test_scripts/test-arch-sanitizers-gcc.sh: -------------------------------------------------------------------------------- 1 | test-arch-sanitizers-clang.sh -------------------------------------------------------------------------------- /jenkins/runners/upstream-vagrant-rawhide-selinux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Note: this script MUST be self-contained - i.e. it MUST NOT source any 4 | # external scripts as it is used as a bootstrap script, thus it's 5 | # fetched and executed without rest of this repository 6 | # 7 | # Example usage in Jenkins 8 | # #!/bin/sh 9 | # 10 | # set -e 11 | # 12 | # curl -q -o runner.sh https://../upstream-vagrant-rawhide-selinux.sh 13 | # chmod +x runner.sh 14 | # ./runner.sh 15 | set -eu 16 | set -o pipefail 17 | 18 | ARGS=() 19 | 20 | git clone https://github.com/systemd/systemd-centos-ci 21 | cd systemd-centos-ci 22 | 23 | ./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 --vagrant rawhide-selinux ${ARGS:+"${ARGS[@]}"} 24 | -------------------------------------------------------------------------------- /.github/workflows/differential-shellcheck.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # https://github.com/redhat-plumbers-in-action/differential-shellcheck#readme 3 | 4 | name: Differential ShellCheck 5 | on: 6 | push: 7 | branches: 8 | - main 9 | pull_request: 10 | branches: 11 | - main 12 | 13 | permissions: 14 | contents: read 15 | 16 | jobs: 17 | lint: 18 | runs-on: ubuntu-latest 19 | 20 | permissions: 21 | security-events: write 22 | 23 | steps: 24 | - name: Repository checkout 25 | uses: actions/checkout@v3 26 | with: 27 | fetch-depth: 0 28 | 29 | - name: Differential ShellCheck 30 | uses: redhat-plumbers-in-action/differential-shellcheck@v4 31 | with: 32 | token: ${{ secrets.GITHUB_TOKEN }} 33 | -------------------------------------------------------------------------------- /jenkins/runners/upstream-coverage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Note: this script MUST be self-contained - i.e. it MUST NOT source any 4 | # external scripts as it is used as a bootstrap script, thus it's 5 | # fetched and executed without rest of this repository 6 | # 7 | # Example usage in Jenkins 8 | # #!/bin/sh 9 | # 10 | # set -e 11 | # 12 | # curl -q -o runner.sh https://../upstream-coverage.sh 13 | # chmod +x runner.sh 14 | # ./runner.sh 15 | set -eu 16 | set -o pipefail 17 | 18 | ARGS=() 19 | 20 | if [[ -v ghprbPullId && -n "$ghprbPullId" ]]; then 21 | ARGS+=(--pr "$ghprbPullId") 22 | fi 23 | 24 | git clone https://github.com/systemd/systemd-centos-ci 25 | cd systemd-centos-ci 26 | 27 | ./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 --vagrant arch-coverage ${ARGS:+"${ARGS[@]}"} 28 | -------------------------------------------------------------------------------- /jenkins/runners/rhel7-centos7.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Note: this script MUST be self-contained - i.e. it MUST NOT source any 4 | # external scripts as it is used as a bootstrap script, thus it's 5 | # fetched and executed without rest of this repository 6 | # 7 | # Example usage in Jenkins 8 | # #!/bin/sh 9 | # 10 | # set -e 11 | # 12 | # curl -q -o runner.sh https://../rhel7-centos7.sh 13 | # chmod +x runner.sh 14 | # ./runner.sh 15 | set -eu 16 | set -o pipefail 17 | 18 | ARGS=() 19 | 20 | if [[ -v ghprbPullId && -n "$ghprbPullId" ]]; then 21 | ARGS+=(--pr "$ghprbPullId") 22 | fi 23 | 24 | git clone https://github.com/systemd/systemd-centos-ci 25 | cd systemd-centos-ci 26 | 27 | ./agent-control.py --pool virt-ec2-t2-centos-7-x86_64 \ 28 | --bootstrap-script="bootstrap-rhel7.sh" \ 29 | --testsuite-script="testsuite-rhel7.sh" \ 30 | ${ARGS:+"${ARGS[@]}"} 31 | -------------------------------------------------------------------------------- /utils/kexec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Simple wrapper around kexec stuff to avoid doing unnecessary kexec when already 3 | # running the latest installed kernel 4 | set -eu 5 | set -o pipefail 6 | 7 | if [[ "${1:-}" == "-u" ]]; then 8 | dnf --refresh -y update kernel 9 | fi 10 | 11 | RUNNING_KERNEL="$(uname -r)" 12 | LATEST_KERNEL="$(rpm -q kernel --qf '%{VERSION}-%{RELEASE}.%{ARCH}\n' | sort -Vr | head -n1)" 13 | 14 | echo "Running kernel: $RUNNING_KERNEL" 15 | echo "Latest installed kernel: $LATEST_KERNEL" 16 | 17 | if [[ "$RUNNING_KERNEL" == "$LATEST_KERNEL" ]]; then 18 | echo "Already running the latest kernel, skipping kexec" 19 | exit 0 20 | fi 21 | 22 | if ! command -v kexec >/dev/null; then 23 | dnf install -y kexec-tools 24 | fi 25 | 26 | echo "Loading & executing kernel $LATEST_KERNEL" 27 | kexec --initrd="/boot/initramfs-$LATEST_KERNEL.img" --reuse-cmdline --load "/boot/vmlinuz-$LATEST_KERNEL" 28 | kexec --exec 29 | -------------------------------------------------------------------------------- /jenkins/runners/rhel9-centos9-sanitizers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Note: this script MUST be self-contained - i.e. it MUST NOT source any 4 | # external scripts as it is used as a bootstrap script, thus it's 5 | # fetched and executed without rest of this repository 6 | # 7 | # Example usage in Jenkins 8 | # #!/bin/sh 9 | # 10 | # set -e 11 | # 12 | # curl -q -o runner.sh https://../rhel9-centos9-sanitizers.sh 13 | # chmod +x runner.sh 14 | # ./runner.sh 15 | set -eu 16 | set -o pipefail 17 | 18 | ARGS=() 19 | 20 | if [[ -v ghprbPullId && -n "$ghprbPullId" ]]; then 21 | ARGS+=(--pr "$ghprbPullId") 22 | fi 23 | 24 | git clone https://github.com/systemd/systemd-centos-ci 25 | cd systemd-centos-ci 26 | 27 | ./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 \ 28 | --bootstrap-script="bootstrap-rhel9.sh" \ 29 | --bootstrap-args="-h unified -z" \ 30 | --testsuite-script="testsuite-rhel9-sanitizers.sh" \ 31 | --kexec \ 32 | ${ARGS:+"${ARGS[@]}"} 33 | -------------------------------------------------------------------------------- /utils/artifacts-copy-file.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Single purpose script to copy a file from one place to another using rsync, 3 | # since the CentOS CI artifact server supports only rsync/sftp. 4 | # Disclaimer: this doesn't work in all cases - it was written to "simply" rename 5 | # a file using the rsync protocol, so please bear that in mind. 6 | set -eu 7 | set -o pipefail 8 | 9 | SRC="${1:?Missing argument: source}" 10 | DEST="${2:?Missing argument: destination}" 11 | [[ "$SRC" == */* ]] && SRC_DIR="${SRC%/*}" || SRC_DIR="." 12 | [[ "$DEST" == */* ]] && DEST_DIR="${DEST%/*}" || DEST_DIR="." 13 | TEMP_DIR="$(mktemp -d "$PWD/.sync-dirXXX")" 14 | 15 | # shellcheck disable=SC2064 16 | trap "cd && rm -fr '$TEMP_DIR'" EXIT 17 | 18 | pushd "$TEMP_DIR" 19 | mkdir -p "$SRC_DIR" "$DEST_DIR" 20 | # Crucial line, otherwise we won't be able to access the web directory listing 21 | chmod -R o+rx . 22 | 23 | rsync -av "systemd@artifacts.ci.centos.org:/srv/artifacts/systemd/$SRC" "$SRC_DIR" 24 | mv -v "$SRC" "$DEST" 25 | rm -fr "$SRC" 26 | rsync -av . "systemd@artifacts.ci.centos.org:/srv/artifacts/systemd/" 27 | 28 | popd 29 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # vi: ts=2 sw=2 et: 3 | 4 | name: "CodeQL" 5 | 6 | on: 7 | push: 8 | branches: 9 | - main 10 | pull_request: 11 | branches: 12 | - main 13 | 14 | permissions: 15 | contents: read 16 | 17 | jobs: 18 | analyze: 19 | name: Analyze 20 | runs-on: ubuntu-22.04 21 | concurrency: 22 | group: ${{ github.workflow }}-${{ matrix.language }}-${{ github.ref }} 23 | cancel-in-progress: true 24 | permissions: 25 | actions: read 26 | security-events: write 27 | 28 | strategy: 29 | fail-fast: false 30 | matrix: 31 | language: ['python'] 32 | 33 | steps: 34 | - name: Checkout repository 35 | uses: actions/checkout@v3 36 | 37 | - name: Initialize CodeQL 38 | uses: github/codeql-action/init@v2 39 | with: 40 | languages: ${{ matrix.language }} 41 | queries: +security-extended,security-and-quality 42 | 43 | - name: Autobuild 44 | uses: github/codeql-action/autobuild@v2 45 | 46 | - name: Perform CodeQL Analysis 47 | uses: github/codeql-action/analyze@v2 48 | -------------------------------------------------------------------------------- /jenkins/runners/upstream-vagrant-archlinux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Note: this script MUST be self-contained - i.e. it MUST NOT source any 4 | # external scripts as it is used as a bootstrap script, thus it's 5 | # fetched and executed without rest of this repository 6 | # 7 | # Example usage in Jenkins 8 | # #!/bin/sh 9 | # 10 | # set -e 11 | # 12 | # curl -q -o runner.sh https://../upstream-vagrant-archlinux.sh 13 | # chmod +x runner.sh 14 | # ./runner.sh 15 | set -eu 16 | set -o pipefail 17 | 18 | ARGS=() 19 | 20 | if [[ -v ghprbPullId && -n "$ghprbPullId" ]]; then 21 | ARGS+=(--pr "$ghprbPullId") 22 | 23 | # We're not testing the main branch, so let's see if the PR scope 24 | # is something we should indeed test 25 | SCOPE_RX='(^(catalog|factory|hwdb|meson.*|network|(?!mkosi)[^\.].*\.d|rules|src|test|units))' 26 | git fetch -fu origin "refs/pull/${ghprbPullId:?}/merge" 27 | if ! git diff --name-only "origin/${ghprbTargetBranch:?}" FETCH_HEAD | grep -P "$SCOPE_RX"; then 28 | echo "Changes in this PR don't seem relevant, skipping..." 29 | exit 0 30 | fi 31 | fi 32 | 33 | git clone https://github.com/systemd/systemd-centos-ci 34 | cd systemd-centos-ci 35 | 36 | ./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 --vagrant arch ${ARGS:+"${ARGS[@]}"} 37 | -------------------------------------------------------------------------------- /jenkins/runners/upstream-vagrant-archlinux-stable.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Note: this script MUST be self-contained - i.e. it MUST NOT source any 4 | # external scripts as it is used as a bootstrap script, thus it's 5 | # fetched and executed without rest of this repository 6 | # 7 | # Example usage in Jenkins 8 | # #!/bin/sh 9 | # 10 | # set -e 11 | # 12 | # curl -q -o runner.sh https://../upstream-vagrant-archlinux-stable.sh 13 | # chmod +x runner.sh 14 | # ./runner.sh 15 | set -eu 16 | set -o pipefail 17 | 18 | ARGS=() 19 | 20 | if [[ -v ghprbPullId && -n "$ghprbPullId" ]]; then 21 | ARGS+=(--pr "$ghprbPullId") 22 | 23 | # We're not testing the main branch, so let's see if the PR scope 24 | # is something we should indeed test 25 | SCOPE_RX='(^(catalog|factory|hwdb|meson.*|network|(?!mkosi)[^\.].*\.d|rules|src|test|units))' 26 | git fetch -fu origin "refs/pull/${ghprbPullId:?}/merge" 27 | if ! git diff --name-only "origin/${ghprbTargetBranch:?}" FETCH_HEAD | grep -P "$SCOPE_RX"; then 28 | echo "Changes in this PR don't seem relevant, skipping..." 29 | exit 0 30 | fi 31 | fi 32 | 33 | git clone https://github.com/systemd/systemd-centos-ci 34 | cd systemd-centos-ci 35 | 36 | ./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 \ 37 | --bootstrap-args='-s https://github.com/systemd/systemd-stable.git' \ 38 | --vagrant arch \ 39 | ${ARGS:+"${ARGS[@]}"} 40 | -------------------------------------------------------------------------------- /jenkins/runners/upstream-centos9s.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Note: this script MUST be self-contained - i.e. it MUST NOT source any 4 | # external scripts as it is used as a bootstrap script, thus it's 5 | # fetched and executed without rest of this repository 6 | # 7 | # Example usage in Jenkins 8 | # #!/bin/sh 9 | # 10 | # set -e 11 | # 12 | # curl -q -o runner.sh https://../upstream-centos9.sh 13 | # chmod +x runner.sh 14 | # ./runner.sh 15 | set -eu 16 | set -o pipefail 17 | 18 | ARGS=() 19 | 20 | if [[ -v ghprbPullId && -n "$ghprbPullId" ]]; then 21 | ARGS+=(--pr "$ghprbPullId") 22 | 23 | # We're not testing the main branch, so let's see if the PR scope 24 | # is something we should indeed test 25 | # 26 | # Let's make the regex here less strict, so we can, for example, test man page 27 | # generation and other low-impact changes 28 | SCOPE_RX='(^(catalog|factory|hwdb|man|meson.*|network|(?!mkosi)[^\.].*\.d|rules|src|test|tools|units))' 29 | git fetch -fu origin "refs/pull/${ghprbPullId:?}/merge" 30 | if ! git diff --name-only "origin/${ghprbTargetBranch:?}" FETCH_HEAD | grep -P "$SCOPE_RX"; then 31 | echo "Changes in this PR don't seem relevant, skipping..." 32 | exit 0 33 | fi 34 | fi 35 | 36 | git clone https://github.com/systemd/systemd-centos-ci 37 | cd systemd-centos-ci 38 | 39 | ./agent-control.py --pool virt-ec2-t2-centos-9s-x86_64 \ 40 | --testsuite-args="-n" \ 41 | --kdump-collect \ 42 | ${ARGS:+"${ARGS[@]}"} 43 | -------------------------------------------------------------------------------- /jenkins/runners/upstream-centos9s-stable.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Note: this script MUST be self-contained - i.e. it MUST NOT source any 4 | # external scripts as it is used as a bootstrap script, thus it's 5 | # fetched and executed without rest of this repository 6 | # 7 | # Example usage in Jenkins 8 | # #!/bin/sh 9 | # 10 | # set -e 11 | # 12 | # curl -q -o runner.sh https://../upstream-centos9-stable.sh 13 | # chmod +x runner.sh 14 | # ./runner.sh 15 | set -eu 16 | set -o pipefail 17 | 18 | ARGS=() 19 | 20 | if [[ -v ghprbPullId && -n "$ghprbPullId" ]]; then 21 | ARGS+=(--pr "$ghprbPullId") 22 | 23 | # We're not testing the main branch, so let's see if the PR scope 24 | # is something we should indeed test 25 | # 26 | # Let's make the regex here less strict, so we can, for example, test man page 27 | # generation and other low-impact changes 28 | SCOPE_RX='(^(catalog|factory|hwdb|man|meson.*|network|(?!mkosi)[^\.].*\.d|rules|src|test|tools|units))' 29 | git fetch -fu origin "refs/pull/${ghprbPullId:?}/merge" 30 | if ! git diff --name-only "origin/${ghprbTargetBranch:?}" FETCH_HEAD | grep -P "$SCOPE_RX"; then 31 | echo "Changes in this PR don't seem relevant, skipping..." 32 | exit 0 33 | fi 34 | fi 35 | 36 | git clone https://github.com/systemd/systemd-centos-ci 37 | cd systemd-centos-ci 38 | 39 | ./agent-control.py --pool virt-ec2-t2-centos-9s-x86_64 \ 40 | --bootstrap-args="-s https://github.com/systemd/systemd-stable.git" \ 41 | --testsuite-args="-n" \ 42 | --kdump-collect \ 43 | ${ARGS:+"${ARGS[@]}"} 44 | -------------------------------------------------------------------------------- /vagrant/boxes/Vagrantfile_archlinux_systemd: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby ts=4 sw=4 et: 3 | 4 | Vagrant.configure("2") do |config| 5 | config.vm.define :archlinux_systemd 6 | config.vm.box = "archlinux/archlinux" 7 | # Don't replace the original Vagrant's insecure key 8 | config.ssh.insert_key = false 9 | config.vm.synced_folder '.', '/vagrant', disabled: true 10 | 11 | ovmf_nvram = "/tmp/OVMF_VARS.arch.fd" 12 | 13 | # Note: CentOS CI infra specific overrides - you may want to change them 14 | # to run the VM locally 15 | config.vm.provider :libvirt do |libvirt| 16 | libvirt.driver = ENV.fetch("VAGRANT_DRIVER", "kvm") 17 | libvirt.memory = ENV.fetch("VAGRANT_MEMORY", "8192") 18 | libvirt.cpus = ENV.fetch("VAGRANT_CPUS", "8") 19 | 20 | # Pass through /dev/random from the host to the VM 21 | libvirt.random :model => 'random' 22 | 23 | libvirt.machine_type = "q35" 24 | 25 | # Emulate UEFI using OVMF 26 | libvirt.loader = "/usr/share/edk2/ovmf/OVMF_CODE.fd" 27 | libvirt.nvram = ovmf_nvram 28 | 29 | # Emulate TPM 2.0 using swtpm 30 | libvirt.tpm_model = "tpm-crb" 31 | libvirt.tpm_type = "emulator" 32 | libvirt.tpm_version = "2.0" 33 | end 34 | 35 | config.trigger.before [:up, :provision] do |trigger| 36 | trigger.run = {inline: "cp -u /usr/share/edk2/ovmf/OVMF_VARS.fd #{ovmf_nvram}"} 37 | end 38 | 39 | config.trigger.after [:destroy] do |trigger| 40 | trigger.run = {inline: "rm #{ovmf_nvram}"} 41 | end 42 | 43 | config.vm.provision "shell", privileged: true, path: __dir__ + "/archlinux_systemd.sh" 44 | end 45 | -------------------------------------------------------------------------------- /vagrant/boxes/Vagrantfile_rawhide_selinux: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby ts=4 sw=4 et: 3 | 4 | Vagrant.configure("2") do |config| 5 | config.vm.define :rawhide_selinux 6 | config.vm.box = "fedora-rawhide-cloud" 7 | config.vm.box_url = "https://dl.fedoraproject.org/pub/fedora/linux/development/rawhide/Cloud/x86_64/images/BOX-NAME-PLACEHOLDER" 8 | # Don't replace the original Vagrant's insecure key 9 | config.ssh.insert_key = false 10 | config.vm.synced_folder '.', '/vagrant', disabled: true 11 | 12 | ovmf_nvram = "/tmp/OVMF_VARS.arch.fd" 13 | 14 | # Note: CentOS CI infra specific overrides - you may want to change them 15 | # to run the VM locally 16 | config.vm.provider :libvirt do |libvirt| 17 | libvirt.driver = ENV.fetch("VAGRANT_DRIVER", "kvm") 18 | libvirt.memory = ENV.fetch("VAGRANT_MEMORY", "8192") 19 | libvirt.cpus = ENV.fetch("VAGRANT_CPUS", "8") 20 | 21 | # Pass through /dev/random from the host to the VM 22 | libvirt.random :model => 'random' 23 | 24 | libvirt.machine_type = "q35" 25 | 26 | # Emulate UEFI using OVMF 27 | libvirt.loader = "/usr/share/edk2/ovmf/OVMF_CODE.fd" 28 | libvirt.nvram = ovmf_nvram 29 | 30 | # Emulate TPM 2.0 using swtpm 31 | libvirt.tpm_model = "tpm-crb" 32 | libvirt.tpm_type = "emulator" 33 | libvirt.tpm_version = "2.0" 34 | end 35 | 36 | config.trigger.before [:up, :provision] do |trigger| 37 | trigger.run = {inline: "cp -u /usr/share/edk2/ovmf/OVMF_VARS.fd #{ovmf_nvram}"} 38 | end 39 | 40 | config.trigger.after [:destroy] do |trigger| 41 | trigger.run = {inline: "rm #{ovmf_nvram}"} 42 | end 43 | 44 | config.vm.provision "shell", privileged: true, path: __dir__ + "/rawhide_selinux.sh" 45 | end 46 | -------------------------------------------------------------------------------- /jenkins/runners/rhel9-centos9.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Note: this script MUST be self-contained - i.e. it MUST NOT source any 4 | # external scripts as it is used as a bootstrap script, thus it's 5 | # fetched and executed without rest of this repository 6 | # 7 | # Example usage in Jenkins 8 | # #!/bin/sh 9 | # 10 | # set -e 11 | # 12 | # curl -q -o runner.sh https://../rhel9-centos9.sh 13 | # chmod +x runner.sh 14 | # ./runner.sh 15 | set -eu 16 | set -o pipefail 17 | 18 | ARGS=() 19 | 20 | at_exit() { 21 | # Correctly collect artifacts from all cron jobs and generate a nice 22 | # directory structure 23 | if find . -name "artifacts_*" | grep -q "."; then 24 | mkdir _artifacts_all 25 | mv artifacts_* _artifacts_all 26 | mv _artifacts_all artifacts_all 27 | 28 | utils/generate-index.sh artifacts_all index.html 29 | fi 30 | } 31 | 32 | trap at_exit EXIT 33 | 34 | if [[ -v ghprbPullId && -n "$ghprbPullId" ]]; then 35 | ARGS+=(--pr "$ghprbPullId") 36 | fi 37 | 38 | git clone https://github.com/systemd/systemd-centos-ci 39 | cd systemd-centos-ci 40 | 41 | # C9S job with unified cgroup hierarchy 42 | ./agent-control.py --no-index --pool virt-ec2-t2-centos-9s-x86_64 \ 43 | --bootstrap-script="bootstrap-rhel9.sh" \ 44 | --bootstrap-args="-h unified" \ 45 | --testsuite-script="testsuite-rhel9.sh" \ 46 | --testsuite-args="-n" \ 47 | ${ARGS:+"${ARGS[@]}"} 48 | 49 | # C9S job with legacy cgroup hierarchy 50 | ./agent-control.py --no-index --pool virt-ec2-t2-centos-9s-x86_64 \ 51 | --bootstrap-script="bootstrap-rhel9.sh" \ 52 | --bootstrap-args="-h legacy" \ 53 | --testsuite-script="testsuite-rhel9.sh" \ 54 | --testsuite-args="-n" \ 55 | ${ARGS:+"${ARGS[@]}"} 56 | -------------------------------------------------------------------------------- /jenkins/runners/upstream-vagrant-archlinux-sanitizers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Note: this script MUST be self-contained - i.e. it MUST NOT source any 4 | # external scripts as it is used as a bootstrap script, thus it's 5 | # fetched and executed without rest of this repository 6 | # 7 | # Example usage in Jenkins 8 | # #!/bin/sh 9 | # 10 | # set -e 11 | # 12 | # curl -q -o runner.sh https://../upstream-vagrant-archlinux-sanitizers.sh 13 | # chmod +x runner.sh 14 | # ./runner.sh 15 | set -eu 16 | set -o pipefail 17 | 18 | at_exit() { 19 | # Correctly collect artifacts from all sanitizer jobs and generate a nice 20 | # directory structure 21 | if find . -name "artifacts_*" | grep -q "."; then 22 | mkdir _artifacts_all 23 | mv artifacts_* _artifacts_all 24 | mv _artifacts_all artifacts_all 25 | 26 | utils/generate-index.sh artifacts_all index.html 27 | fi 28 | } 29 | 30 | trap at_exit EXIT 31 | 32 | ARGS=() 33 | 34 | if [[ -v ghprbPullId && -n "$ghprbPullId" ]]; then 35 | ARGS+=(--pr "$ghprbPullId") 36 | 37 | # We're not testing the main branch, so let's see if the PR scope 38 | # is something we should indeed test 39 | SCOPE_RX='(^(catalog|factory|hwdb|meson.*|network|(?!mkosi)[^\.].*\.d|rules|src|test|units))' 40 | git fetch -fu origin "refs/pull/${ghprbPullId:?}/merge" 41 | if ! git diff --name-only "origin/${ghprbTargetBranch:?}" FETCH_HEAD | grep -P "$SCOPE_RX"; then 42 | echo "Changes in this PR don't seem relevant, skipping..." 43 | exit 0 44 | fi 45 | fi 46 | 47 | git clone https://github.com/systemd/systemd-centos-ci 48 | cd systemd-centos-ci 49 | 50 | #./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 \ 51 | # --vagrant arch-sanitizers-gcc \ 52 | # --no-index \ 53 | # ${ARGS:+"${ARGS[@]}"} 54 | 55 | ./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 \ 56 | --vagrant arch-sanitizers-clang \ 57 | --no-index \ 58 | ${ARGS:+"${ARGS[@]}"} 59 | -------------------------------------------------------------------------------- /jenkins/runners/rhel8-centos8.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Note: this script MUST be self-contained - i.e. it MUST NOT source any 4 | # external scripts as it is used as a bootstrap script, thus it's 5 | # fetched and executed without rest of this repository 6 | # 7 | # Example usage in Jenkins 8 | # #!/bin/sh 9 | # 10 | # set -e 11 | # 12 | # curl -q -o runner.sh https://../rhel8-centos8.sh 13 | # chmod +x runner.sh 14 | # ./runner.sh 15 | set -eu 16 | set -o pipefail 17 | 18 | ARGS=() 19 | TARGET_BRANCH="${ghprbTargetBranch:-main}" 20 | 21 | at_exit() { 22 | # Correctly collect artifacts from all cron jobs and generate a nice 23 | # directory structure 24 | if find . -name "artifacts_*" | grep -q "."; then 25 | mkdir _artifacts_all 26 | mv artifacts_* _artifacts_all 27 | mv _artifacts_all artifacts_all 28 | 29 | utils/generate-index.sh artifacts_all index.html 30 | fi 31 | } 32 | 33 | trap at_exit EXIT 34 | 35 | if [[ -v ghprbPullId && -n "$ghprbPullId" ]]; then 36 | ARGS+=(--pr "$ghprbPullId") 37 | fi 38 | 39 | git clone https://github.com/systemd/systemd-centos-ci 40 | cd systemd-centos-ci 41 | 42 | # RHEL 8 job with legacy cgroup hierarchy 43 | ./agent-control.py --no-index --pool virt-ec2-t2-centos-9s-x86_64 \ 44 | --bootstrap-script="bootstrap-rhel8.sh" \ 45 | --bootstrap-args="-h legacy" \ 46 | --testsuite-script="testsuite-rhel8.sh" \ 47 | ${ARGS:+"${ARGS[@]}"} 48 | 49 | # RHEL 8 supports unified cgroups since RHEL 8.2, so ignore RHEL 8.0 and 50 | # RHEL 8.1 branches 51 | if [[ "$TARGET_BRANCH" != "rhel-8.0.0" && "$TARGET_BRANCH" != "rhel-8.1.0" ]]; then 52 | # RHEL 8 job with unified cgroup hierarchy 53 | ./agent-control.py --no-index --pool virt-ec2-t2-centos-9s-x86_64 \ 54 | --bootstrap-script="bootstrap-rhel8.sh" \ 55 | --bootstrap-args="-h unified" \ 56 | --testsuite-script="testsuite-rhel8.sh" \ 57 | ${ARGS:+"${ARGS[@]}"} 58 | fi 59 | -------------------------------------------------------------------------------- /vagrant/boxes/rawhide_selinux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eux 3 | set -o pipefail 4 | 5 | dnf clean all 6 | # Try to get the new GPG repo keys. In some situations this might fail, usually 7 | # when we try to update the fedora-gpg-keys by two versions (e.g. 35 -> 37). In 8 | # that case use a relatively ugly hack to force-get the latest GPG keys from 9 | # the Rawhide repository. 10 | if ! dnf -y update fedora-repos fedora-gpg-keys; then 11 | dnf -y --nogpgcheck --disablerepo '*' \ 12 | --repofrompath the-true-rawhide,https://dl.fedoraproject.org/pub/fedora/linux/development/rawhide/Everything/x86_64/os/ \ 13 | update fedora-repos fedora-gpg-keys 14 | fi 15 | 16 | # FIXME: disable the openh264 repo, since it has broken signatures on current Rawhide 17 | # See: https://pagure.io/releng/issue/12275 18 | dnf config-manager setopt fedora-cisco-openh264.enabled=0 19 | 20 | # Upgrade the system 21 | dnf upgrade -y 22 | 23 | # Install NFS tools required by Vagrant's "synced folder" functionality 24 | dnf install -y nfs-utils libnfs-utils portmap 25 | # Install build & test dependencies 26 | dnf install -y attr busybox cryptsetup dnf5-plugins dosfstools fedpkg git jq nc qemu-kvm rpm-build rpmdevtools rust socat \ 27 | strace time tmt tpm2-tss-devel util-linux-script 'python3dist(jinja2)' 28 | dnf builddep -y dracut systemd 29 | 30 | # Unlock root account and set its password to 'vagrant' to allow root login 31 | # via ssh 32 | echo "vagrant" | passwd --stdin 33 | passwd -S root 34 | # Fedora's default for PermitRootLogin= is 'prohibit-password' which breaks 35 | # Vagrant 'insert_key' feature 36 | echo "PermitRootLogin yes" >>/etc/ssh/sshd_config 37 | 38 | # Configure NTP (chronyd) 39 | dnf install -y chrony 40 | systemctl enable --now chronyd 41 | systemctl status chronyd 42 | 43 | # Disable 'quiet' mode on the kernel command line and forward everything 44 | # to ttyS0 instead of just tty0, so we can collect it using QEMU's 45 | # -serial file:xxx feature 46 | sed -i '/GRUB_CMDLINE_LINUX_DEFAULT/ { s/quiet//; s/"$/ console=ttyS0"/ }' /etc/default/grub 47 | grub2-mkconfig -o /boot/grub2/grub.cfg 48 | -------------------------------------------------------------------------------- /jenkins/jenkins-configuration.md: -------------------------------------------------------------------------------- 1 | Notes describing current Jenkins configuration in case it needs to be redeployed. 2 | 3 | Note: most of the configuration has been translated to a yaml format parsed 4 | by the Configuration as Code plugin. Simply copy the `jenkins.yaml` to 5 | `/var/lib/jenkins/jenkins.yaml` and reload the configuration under 6 | Configure System -> Configuration as Code -> Actions -> Reload existing configuration 7 | 8 | Do ^ _after_ installing all the plugins below. 9 | 10 | # Manage Jenkins 11 | ## Configure System 12 | ### GitHub Pull Request Builder 13 | - Credentials -> Add -> Jenkins 14 | - Kind: Secret Text 15 | - Secret: token generated at https://github.com/settings/tokens (classic token with scope: repo:status) 16 | - Description: GH/mrc0mmand 17 | 18 | ## Manage Nodes and Clouds 19 | ### Configure Clouds 20 | - Kubernetes -> Kubernetes Cloud Details -> Set "Concurrency Limit" to 10 21 | 22 | ## Plugins 23 | - ANSI Color (https://plugins.jenkins.io/ansicolor) 24 | - Configuration as Code (https://plugins.jenkins.io/configuration-as-code/) 25 | - Embeddable Build Status (https://plugins.jenkins.io/embeddable-build-status) 26 | - GitHub Pull Request Builder (https://plugins.jenkins.io/ghprb) 27 | - Mailer (https://plugins.jenkins.io/mailer) 28 | - Naginator (https://plugins.jenkins.io/naginator) 29 | - OWASP Markup Formatter (https://plugins.jenkins.io/antisamy-markup-formatter) 30 | - Timestamper (https://plugins.jenkins.io/timestamper) 31 | - URL SCM (https://plugins.jenkins.io/URLSCM) 32 | - Workspace Cleanup (https://plugins.jenkins.io/ws-cleanup) 33 | 34 | ## Unwanted plugins 35 | - Blue Ocean 36 | - Display URL for Blue Ocean 37 | - Personalization for Blue Ocean 38 | - ... 39 | 40 | # Useful links & stuff 41 | - cico-workspace image: https://quay.io/repository/centosci/cico-workspace 42 | - cico-workspace template: https://github.com/CentOS/ansible-infra-playbooks/blob/master/templates/openshift/jenkins-ci-workspace.yml 43 | - spawn a debug pod: `oc run cico-workspace-debug --image quay.io/centosci/cico-workspace:latest --attach=false --leave-stdin-open --tty --stdin --command -- /bin/bash` 44 | - (re)attach: `oc exec -it cico-workspace-debug -- bash 45 | -------------------------------------------------------------------------------- /jenkins/runners/rhel-cron-build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # shellcheck disable=SC2181,SC2317 3 | 4 | # Note: this script MUST be self-contained - i.e. it MUST NOT source any 5 | # external scripts as it is used as a bootstrap script, thus it's 6 | # fetched and executed without rest of this repository 7 | # 8 | # Example usage in Jenkins 9 | # #!/bin/sh 10 | # 11 | # set -e 12 | # 13 | # curl -q -o runner.sh https://../rhel-cron-build.sh 14 | # chmod +x runner.sh 15 | # ./runner.sh 16 | set -eu 17 | set -o pipefail 18 | 19 | at_exit() { 20 | # Correctly collect artifacts from all cron jobs and generate a nice 21 | # directory structure 22 | if find . -name "artifacts_*" | grep -q "."; then 23 | mkdir _artifacts_all 24 | mv artifacts_* _artifacts_all 25 | mv _artifacts_all artifacts_all 26 | 27 | utils/generate-index.sh artifacts_all index.html 28 | fi 29 | } 30 | 31 | trap at_exit EXIT 32 | 33 | ARGS=() 34 | FAILED=() 35 | PASSED=() 36 | EC=0 37 | 38 | git clone https://github.com/systemd/systemd-centos-ci 39 | cd systemd-centos-ci 40 | 41 | run_rhel9_full_unified() { 42 | # C9S job with unified cgroup hierarchy 43 | ./agent-control.py --no-index --pool virt-ec2-t2-centos-9s-x86_64 \ 44 | --bootstrap-script="bootstrap-rhel9.sh" \ 45 | --bootstrap-args="-h unified" \ 46 | --testsuite-script="testsuite-rhel9.sh" \ 47 | ${ARGS:+"${ARGS[@]}"} 48 | } 49 | 50 | run_rhel9_full_legacy() { 51 | # C9S job with legacy cgroup hierarchy 52 | ./agent-control.py --no-index --pool virt-ec2-t2-centos-9s-x86_64 \ 53 | --bootstrap-script="bootstrap-rhel9.sh" \ 54 | --bootstrap-args="-h legacy" \ 55 | --testsuite-script="testsuite-rhel9.sh" \ 56 | ${ARGS:+"${ARGS[@]}"} 57 | } 58 | 59 | for job in run_rhel9_full_{unified,legacy}; do 60 | if ! "$job"; then 61 | FAILED+=("$job") 62 | EC=$((EC + 1)) 63 | else 64 | PASSED+=("$job") 65 | fi 66 | done 67 | 68 | echo "PASSED TASKS:" 69 | printf " %s\n" "${PASSED[@]}" 70 | echo 71 | echo "FAILED TASKS:" 72 | printf " %s\n" "${FAILED[@]}" 73 | 74 | exit $EC 75 | -------------------------------------------------------------------------------- /jenkins/runners/upstream-vagrant-archlinux-sanitizers-stable.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Note: this script MUST be self-contained - i.e. it MUST NOT source any 4 | # external scripts as it is used as a bootstrap script, thus it's 5 | # fetched and executed without rest of this repository 6 | # 7 | # Example usage in Jenkins 8 | # #!/bin/sh 9 | # 10 | # set -e 11 | # 12 | # curl -q -o runner.sh https://../upstream-vagrant-archlinux-sanitizers-stable.sh 13 | # chmod +x runner.sh 14 | # ./runner.sh 15 | set -eu 16 | set -o pipefail 17 | 18 | at_exit() { 19 | # Correctly collect artifacts from all sanitizer jobs and generate a nice 20 | # directory structure 21 | if find . -name "artifacts_*" | grep -q "."; then 22 | mkdir _artifacts_all 23 | mv artifacts_* _artifacts_all 24 | mv _artifacts_all artifacts_all 25 | 26 | utils/generate-index.sh artifacts_all index.html 27 | fi 28 | } 29 | 30 | trap at_exit EXIT 31 | 32 | ARGS=() 33 | 34 | if [[ -v ghprbPullId && -n "$ghprbPullId" ]]; then 35 | ARGS+=(--pr "$ghprbPullId") 36 | 37 | # We're not testing the main branch, so let's see if the PR scope 38 | # is something we should indeed test 39 | SCOPE_RX='(^(catalog|factory|hwdb|meson.*|network|(?!mkosi)[^\.].*\.d|rules|src|test|units))' 40 | git fetch -fu origin "refs/pull/${ghprbPullId:?}/merge" 41 | if ! git diff --name-only "origin/${ghprbTargetBranch:?}" FETCH_HEAD | grep -P "$SCOPE_RX"; then 42 | echo "Changes in this PR don't seem relevant, skipping..." 43 | exit 0 44 | fi 45 | fi 46 | 47 | git clone https://github.com/systemd/systemd-centos-ci 48 | cd systemd-centos-ci 49 | 50 | # Run both jobs, since we don't run the other-half-of-this-job in cron in this case 51 | ./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 \ 52 | --bootstrap-args='-s https://github.com/systemd/systemd-stable.git' \ 53 | --no-index \ 54 | --vagrant arch-sanitizers-gcc \ 55 | ${ARGS:+"${ARGS[@]}"} 56 | 57 | ./agent-control.py --pool metal-ec2-c5n-centos-9s-x86_64 \ 58 | --bootstrap-args='-s https://github.com/systemd/systemd-stable.git' \ 59 | --no-index \ 60 | --vagrant arch-sanitizers-clang \ 61 | ${ARGS:+"${ARGS[@]}"} 62 | -------------------------------------------------------------------------------- /agent/testsuite-rhel7.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | # shellcheck disable=SC2155 3 | 4 | LIB_ROOT="$(dirname "$0")/../common" 5 | # shellcheck source=common/task-control.sh 6 | . "$LIB_ROOT/task-control.sh" "testsuite-logs-rhel7" || exit 1 7 | # shellcheck source=common/utils.sh 8 | . "$LIB_ROOT/utils.sh" || exit 1 9 | 10 | # EXIT signal handler 11 | at_exit() { 12 | set +e 13 | exectask "journalctl-testsuite" "journalctl -b --no-pager" 14 | } 15 | 16 | set -eu 17 | set -o pipefail 18 | 19 | trap at_exit EXIT 20 | 21 | # Install test dependencies 22 | exectask "yum-depinstall" \ 23 | "yum -y install net-tools strace nc busybox e2fsprogs quota dnsmasq qemu-kvm python-enum34" 24 | 25 | set +e 26 | 27 | ### TEST PHASE ### 28 | pushd systemd || { echo >&2 "Can't pushd to systemd"; exit 1; } 29 | 30 | # Run the internal unit tests (make check) 31 | exectask "make-check" "make check" 32 | if [[ -f "test-suite.log" ]]; then 33 | cat test-suite.log 34 | exectask "make-check-full" "cat test-suite.log" 35 | fi 36 | 37 | ## Integration test suite ## 38 | SKIP_LIST=() 39 | 40 | centos_ensure_qemu_symlink 41 | 42 | for t in test/TEST-??-*; do 43 | if [[ ${#SKIP_LIST[@]} -ne 0 ]] && in_set "$t" "${SKIP_LIST[@]}"; then 44 | echo -e "[SKIP] Skipping test $t\n" 45 | continue 46 | fi 47 | 48 | rm -fr /var/tmp/systemd-test* 49 | 50 | ## Configure test environment 51 | # Explicitly set paths to initramfs and kernel images (for QEMU tests) 52 | export INITRD="/boot/initramfs-$(uname -r).img" 53 | export KERNEL_BIN="/boot/vmlinuz-$(uname -r)" 54 | # Explicitly enable user namespaces 55 | export KERNEL_APPEND="user_namespace.enable=1" 56 | # Set timeouts for QEMU and nspawn tests to kill them in case they get stuck 57 | export QEMU_TIMEOUT=600 58 | export NSPAWN_TIMEOUT=600 59 | 60 | if ! exectask "${t##*/}" "make -C $t clean setup run"; then 61 | # Each integration test dumps the system journal when something breaks 62 | rsync -amq /var/tmp/systemd-test*/journal "$LOGDIR/${t##*/}" &>/dev/null || : 63 | fi 64 | done 65 | 66 | ## Other integration tests ## 67 | TEST_LIST=( 68 | "test/test-exec-deserialization.py" 69 | ) 70 | 71 | for t in "${TEST_LIST[@]}"; do 72 | if [[ ! -f $t ]]; then 73 | echo "Test '$t' not found, skipping..." 74 | continue 75 | fi 76 | exectask "${t##*/}" "timeout 15m ./$t" 77 | done 78 | 79 | # Summary 80 | show_task_summary 81 | 82 | finish_and_exit 83 | -------------------------------------------------------------------------------- /vagrant/bootstrap_scripts/rawhide-selinux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Vagrant provider for a standard systemd setup 3 | 4 | set -eux 5 | set -o pipefail 6 | 7 | whoami 8 | uname -a 9 | 10 | # Do a system upgrade 11 | dnf upgrade -y 12 | 13 | # Let's make the $BUILD_DIR for meson reside outside of the NFS volume mounted 14 | # under /build to avoid certain race conditions, like: 15 | # /usr/bin/ld: error: /build/build/src/udev/libudev.so.1: file too short 16 | # The same path must be exported in the respective tests scripts (vagrant-test.sh, 17 | # etc.) so the unit & integration tests can find the compiled binaries 18 | # Note: avoid using /tmp or /var/tmp, as certain tests use binaries from the 19 | # buildir in combination with PrivateTmp=true 20 | export BUILD_DIR="${BUILD_DIR:-/systemd-meson-build}" 21 | 22 | # Use systemd repo path specified by SYSTEMD_ROOT 23 | pushd /build 24 | 25 | # Dump list of installed packages 26 | rpm -qa > vagrant-rawhide-installed-pkgs.txt 27 | # Dump additional OS info 28 | { 29 | echo "### CPUINFO ###" 30 | cat /proc/cpuinfo 31 | echo "### MEMINFO ###" 32 | cat /proc/meminfo 33 | echo "### VERSION ###" 34 | cat /proc/version 35 | } > vagrant-rawhide-osinfo.txt 36 | 37 | # Switch SELinux to permissive mode after reboot, so we catch all possible 38 | # AVCs, not just the first one 39 | setenforce 0 40 | sed -ri 's/^SELINUX=\w+$/SELINUX=permissive/' /etc/selinux/config 41 | cat /etc/selinux/config 42 | 43 | # Build & install latest systemd 44 | rm -fr "$BUILD_DIR" 45 | meson setup "$BUILD_DIR" \ 46 | --werror \ 47 | -Dc_args='-fno-omit-frame-pointer -ftrapv' \ 48 | -Ddebug=true \ 49 | --optimization=g \ 50 | -Dsysvinit-path=/etc/rc.d/init.d \ 51 | -Drc-local=/etc/rc.d/rc.local \ 52 | -Ddefault-dnssec=no \ 53 | -Dtests=true \ 54 | -Dinstall-tests=true 55 | ninja -C "$BUILD_DIR" install 56 | popd 57 | 58 | # Install the latest SELinux policy 59 | fedpkg clone -a selinux-policy 60 | pushd selinux-policy 61 | dnf -y builddep selinux-policy.spec 62 | ./make-rhat-patches.sh 63 | fedpkg local 64 | dnf install -y noarch/selinux-policy-* 65 | popd 66 | 67 | # Force relabel on next boot 68 | fixfiles -v -F onboot 69 | 70 | # Build & install latest dracut-ng 71 | git clone https://github.com/dracut-ng/dracut-ng 72 | pushd dracut-ng 73 | ./configure 74 | make -j "$(nproc)" 75 | make install 76 | # Fix SELinux labels on module files 77 | restorecon -Rv /usr/lib/dracut 78 | dracut --version 79 | 80 | systemd-analyze set-log-level debug 81 | systemd-analyze set-log-target console 82 | -------------------------------------------------------------------------------- /vagrant/test_scripts/test-rawhide-selinux.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | # shellcheck disable=SC2155 3 | 4 | DISTRO="${1:-unspecified}" 5 | SCRIPT_DIR="$(dirname "$0")" 6 | # This variable is automagically consumed by the "framework" for integration tests 7 | # See respective bootstrap script under vagrant/bootstrap_scripts/ for reasoning 8 | export BUILD_DIR="${BUILD_DIR:-/systemd-meson-build}" 9 | 10 | # Following scripts are copied from the systemd-centos-ci/common directory 11 | # by vagrant-build.sh 12 | # shellcheck source=common/task-control.sh 13 | . "$SCRIPT_DIR/task-control.sh" "vagrant-$DISTRO-testsuite" || exit 1 14 | # shellcheck source=common/utils.sh 15 | . "$SCRIPT_DIR/utils.sh" || exit 1 16 | 17 | at_exit() { 18 | set +e 19 | exectask "journalctl-testsuite" "journalctl -b -o short-monotonic --no-hostname --no-pager" 20 | } 21 | 22 | trap at_exit EXIT 23 | 24 | pushd /build || { echo >&2 "Can't pushd to /build"; exit 1; } 25 | 26 | ## SETUP ## 27 | set -ex 28 | # Download & prepare the Fedora SELinux test suite 29 | fedpkg co -a tests/selinux 30 | pushd selinux 31 | cat >plans/systemd.fmf </dev/null; then 16 | dnf -y install "$VAGRANT_PKG_URL" 17 | fi 18 | 19 | # Install necessary dependencies 20 | dnf -y install 'dnf-command(download)' 'dnf-command(builddep)' perl cpio diffutils git libarchive spectool 21 | 22 | # Workaround for current Vagrant's DSO hell 23 | # --- 24 | # The krb5-libs RPM is compiled with --with-crypto-impl=openssl, which 25 | # includes symbols, that are not available in the Vagrant's embedded OpenSSL 26 | # library, causing errors like: 27 | # /opt/vagrant/embedded/lib64/libk5crypto.so.3: undefined symbol: EVP_KDF_ctrl, version OPENSSL_1_1_1b 28 | 29 | # Workaround this by compiling a local version of the krb5-libs using the 30 | # builtin crypto implementation and copying the built libraries into 31 | # the embedded lib dir. 32 | # 33 | # See: 34 | # https://github.com/hashicorp/vagrant/issues/11020 35 | # https://github.com/vagrant-libvirt/vagrant-libvirt/issues/1031 36 | # https://github.com/vagrant-libvirt/vagrant-libvirt/issues/943 37 | ( 38 | BUILD_DIR="$(mktemp -d)" 39 | pushd "$BUILD_DIR" 40 | dnf -y install gcc byacc tar make 41 | dnf download --source krb5 42 | rpm2cpio krb5-*.src.rpm | cpio -imdV 43 | tar xf krb5-*.tar.gz 44 | cd krb5-*/src 45 | ./configure --with-crypto-impl=builtin 46 | make -j 47 | cp -a lib/crypto/libk5crypto.* "$OUT_DIR/" 48 | # We need to build libssh (below) with the just compiled libkrb5crypto.so 49 | cp -a lib/crypto/libk5crypto.* /opt/vagrant/embedded/lib64/ 50 | popd 51 | rm -fr "$BUILD_DIR" 52 | ) 53 | 54 | # pam_wrapper (libssh dep) is not, for some reason, in C8S repositories, *sigh* 55 | ( 56 | BUILD_DIR="$(mktemp -d)" 57 | pushd "$BUILD_DIR" 58 | git clone https://git.centos.org/rpms/pam_wrapper 59 | cd pam_wrapper 60 | git checkout c8s || git checkout c8 61 | dnf -y --enablerepo powertools builddep SPECS/pam_wrapper.spec 62 | mkdir SOURCES 63 | spectool -g -C SOURCES SPECS/pam_wrapper.spec 64 | rpmbuild -ba --define "_topdir $PWD" SPECS/pam_wrapper.spec 65 | dnf -y install "RPMS/$(uname -m)/"pam_wrapper*.rpm 66 | popd 67 | rm -fr "$BUILD_DIR" 68 | ) 69 | 70 | # Same as above, but for libssh 71 | ( 72 | BUILD_DIR="$(mktemp -d)" 73 | pushd "$BUILD_DIR" 74 | dnf download --source libssh 75 | dnf -y --enablerepo powertools builddep libssh*.src.rpm 76 | rpm2cpio libssh-*.src.rpm | cpio -imdV 77 | tar xf libssh-*.tar.xz 78 | cd "$(find . -maxdepth 1 -name "libssh-*" -type d)" 79 | mkdir build 80 | cd build 81 | cmake .. -DOPENSSL_ROOT_DIR=/opt/vagrant/embedded/ -DCMAKE_PREFIX_PATH=/opt/vagrant/embedded/ 82 | make -j 83 | cp -a lib/libssh* "$OUT_DIR/" 84 | popd 85 | rm -fr "$BUILD_DIR" 86 | ) 87 | 88 | pushd "$OUT_DIR" 89 | tar -pczvf vagrant-shared-libs.tar.gz --exclude "*.sh" --exclude "*.gz" --remove-files -- * 90 | popd 91 | -------------------------------------------------------------------------------- /vagrant/bootstrap_scripts/arch-coverage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # shellcheck disable=SC2155 3 | # Vagrant provider for a standard systemd setup 4 | 5 | set -eux 6 | set -o pipefail 7 | 8 | whoami 9 | uname -a 10 | 11 | # The custom CentOS CI box should be updated and provide necessary 12 | # build & test dependencies 13 | 14 | # Let's make the $BUILD_DIR for meson reside outside of the NFS volume mounted 15 | # under /build to avoid certain race conditions, like: 16 | # /usr/bin/ld: error: /build/build/src/udev/libudev.so.1: file too short 17 | # The same path must be exported in the respective tests scripts (vagrant-test.sh, 18 | # etc.) so the unit & integration tests can find the compiled binaries 19 | # Note: avoid using /tmp or /var/tmp, as certain tests use binaries from the 20 | # buildir in combination with PrivateTmp=true 21 | export BUILD_DIR="${BUILD_DIR:-/systemd-meson-build}" 22 | 23 | # Use systemd repo path specified by SYSTEMD_ROOT 24 | pushd /build 25 | 26 | # Dump list of installed packages 27 | pacman -Q > vagrant-arch-installed-pkgs.txt 28 | # Dump additional OS info 29 | { 30 | echo "### CPUINFO ###" 31 | cat /proc/cpuinfo 32 | echo "### MEMINFO ###" 33 | cat /proc/meminfo 34 | echo "### VERSION ###" 35 | cat /proc/version 36 | } > vagrant-arch-osinfo.txt 37 | 38 | rm -fr "$BUILD_DIR" 39 | # Build phase 40 | meson setup "$BUILD_DIR" \ 41 | --werror \ 42 | -Dc_args='-fno-omit-frame-pointer -ftrapv' \ 43 | -Ddebug=true \ 44 | --optimization=0 \ 45 | -Db_coverage=true \ 46 | -Dlog-trace=true \ 47 | -Dfexecve=true \ 48 | -Dslow-tests=true \ 49 | -Dfuzz-tests=true \ 50 | -Dtests=unsafe \ 51 | -Dinstall-tests=true \ 52 | -Ddbuspolicydir=/usr/share/dbus-1/system.d \ 53 | -Dlocalegen-path=/usr/bin/locale-gen 54 | ninja -C "$BUILD_DIR" 55 | 56 | # Install the universal coverage uploader 57 | # https://github.com/coverallsapp/coverage-reporter 58 | curl -L https://coveralls.io/coveralls-linux.tar.gz | tar -xz -C /usr/local/bin 59 | coveralls --version 60 | 61 | # Manually install upstream D-Bus config file for org.freedesktop.network1 62 | # so systemd-networkd testsuite can use potentially new/updated methods 63 | cp -fv src/network/org.freedesktop.network1.conf /usr/share/dbus-1/system.d/ 64 | 65 | # Manually install upstream systemd-networkd service unit files in case a PR 66 | # introduces a change in them 67 | # See: https://github.com/systemd/systemd/pull/14415#issuecomment-579307925 68 | cp -fv "$BUILD_DIR/units/systemd-networkd.service" /usr/lib/systemd/system/systemd-networkd.service 69 | cp -fv "$BUILD_DIR/units/systemd-networkd-wait-online.service" /usr/lib/systemd/system/systemd-networkd-wait-online.service 70 | 71 | # In order to be able to collect all coverage reports, we need to run 72 | # the systemd-networkd test suite from the build dir, which means we need to 73 | # create certain symlinks manually (since usually they're created by meson 74 | # during the 'install' step). 75 | 76 | # Support udevadm/systemd-udevd merge efforts from 77 | # https://github.com/systemd/systemd/pull/15918 78 | # The udevadm -> systemd-udevd symlink is created in the install phase which 79 | # we don't execute in sanitizer runs, so let's create it manually where 80 | # we need it 81 | if [[ -x "$BUILD_DIR/udevadm" && ! -x "$BUILD_DIR/systemd-udevd" ]]; then 82 | ln -frsv "$BUILD_DIR/udevadm" "$BUILD_DIR/systemd-udevd" 83 | fi 84 | 85 | popd 86 | -------------------------------------------------------------------------------- /vagrant/bootstrap_scripts/arch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # shellcheck disable=SC2155 3 | # Vagrant provider for a standard systemd setup 4 | 5 | set -eux 6 | set -o pipefail 7 | 8 | whoami 9 | uname -a 10 | 11 | # The custom CentOS CI box should be updated and provide necessary 12 | # build & test dependencies 13 | 14 | # Let's make the $BUILD_DIR for meson reside outside of the NFS volume mounted 15 | # under /build to avoid certain race conditions, like: 16 | # /usr/bin/ld: error: /build/build/src/udev/libudev.so.1: file too short 17 | # The same path must be exported in the respective tests scripts (vagrant-test.sh, 18 | # etc.) so the unit & integration tests can find the compiled binaries 19 | # Note: avoid using /tmp or /var/tmp, as certain tests use binaries from the 20 | # buildir in combination with PrivateTmp=true 21 | export BUILD_DIR="${BUILD_DIR:-/systemd-meson-build}" 22 | 23 | # Use systemd repo path specified by SYSTEMD_ROOT 24 | pushd /build 25 | 26 | # FIXME: monkey-patch systemd's mkinitcpio hook to install libkmod as well, 27 | # as it's a dlopen() dep since [0]. Drop this once a new systemd release 28 | # is out and the hook in Arch is tweaked. 29 | # [0] https://github.com/systemd/systemd/pull/31131 30 | sed -i 's/qrencode; do/qrencode kmod; do/' /usr/lib/initcpio/install/systemd 31 | 32 | # Dump list of installed packages 33 | pacman -Q > vagrant-arch-installed-pkgs.txt 34 | # Dump additional OS info 35 | { 36 | echo "### CPUINFO ###" 37 | cat /proc/cpuinfo 38 | echo "### MEMINFO ###" 39 | cat /proc/meminfo 40 | echo "### VERSION ###" 41 | cat /proc/version 42 | } > vagrant-arch-osinfo.txt 43 | 44 | rm -fr "$BUILD_DIR" 45 | # Build phase 46 | # shellcheck disable=SC2046 47 | meson setup "$BUILD_DIR" \ 48 | --werror \ 49 | -Dc_args='-fno-omit-frame-pointer -ftrapv' \ 50 | -Ddebug=true \ 51 | --optimization=g \ 52 | -Dlog-trace=true \ 53 | -Dfexecve=true \ 54 | -Dslow-tests=true \ 55 | -Dfuzz-tests=true \ 56 | -Dtests=unsafe \ 57 | -Dinstall-tests=true \ 58 | -Ddbuspolicydir=/usr/share/dbus-1/system.d \ 59 | -Dlocalegen-path=/usr/bin/locale-gen \ 60 | $(grep -q default-network meson_options.txt && echo -Ddefault-network=true) \ 61 | -Dman=true \ 62 | -Dhtml=true 63 | ninja -C "$BUILD_DIR" 64 | ninja -C "$BUILD_DIR" install 65 | 66 | # Make sure the revision we just compiled is actually bootable 67 | ( 68 | # We need a custom initrd (with the systemd module) for integration tests 69 | # See vagrant-test.sh for reasoning 70 | export INITRD="$(mktemp /var/tmp/initrd-testsuite-XXX.img)" 71 | mkinitcpio -c /dev/null -A base,systemd,modconf,block,filesystems,keyboard,fsck -g "$INITRD" 72 | # Enable as much debug logging as we can to make debugging easier 73 | # (especially for boot issues) 74 | export KERNEL_APPEND="debug systemd.log_level=debug rd.systemd.log_target=console" 75 | export QEMU_TIMEOUT=600 76 | # Skip the nspawn version of the test 77 | export TEST_NO_NSPAWN=1 78 | # Enforce nested KVM 79 | export TEST_NESTED_KVM=1 80 | 81 | if ! make -C test/TEST-01-BASIC clean setup run clean-again; then 82 | rsync -amq /var/tmp/systemd-test*/system.journal vagrant-sanity-boot-check.journal >/dev/null || : 83 | exit 1 84 | fi 85 | 86 | rm -fv "$INITRD" 87 | ) 2>&1 | grep --text --line-buffered '^' | tee vagrant-arch-sanity-boot-check.log 88 | 89 | popd 90 | -------------------------------------------------------------------------------- /vagrant/vagrant-ci-wrapper.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | # Auxiliary script for the CentOS CI infrastructure. 3 | # 4 | # This script basically checks out the requested branch of the systemd/systemd 5 | # repository, installs & configures Vagrant, and runs the configured test suite 6 | # in the Vagrant container on given distributions. 7 | 8 | LIB_ROOT="$(dirname "$0")/../common" 9 | # shellcheck source=common/task-control.sh 10 | . "$LIB_ROOT/task-control.sh" "vagrant-logs" || exit 1 11 | # shellcheck source=common/utils.sh 12 | . "$LIB_ROOT/utils.sh" || exit 1 13 | 14 | at_exit() { 15 | set +e 16 | journalctl -b -o short-monotonic --no-hostname --no-pager >"$LOGDIR/journalctl-vagrant-host.log" 17 | # Copy over all vagrant-related artifacts, so the Jenkins artifact plugin 18 | # can gather them for further investigation 19 | if [[ -v SYSTEMD_ROOT ]]; then 20 | cp -r "$SYSTEMD_ROOT"/vagrant-* "$LOGDIR" 21 | fi 22 | # Collect QEMU serial console logs (see the configuration in a respective 23 | # Vagrantfile) 24 | cp /tmp/vagrant-*-console.log "$LOGDIR" 25 | } 26 | 27 | REPO_URL="https://github.com/systemd/systemd.git" 28 | SCRIPT_ROOT="$(dirname "$0")" 29 | # Supported distros: 30 | # 31 | # Arch Linux with sanitizers (Address Sanitizer, Undefined Behavior Sanitizer 32 | # Runs only a selected part of the test suite, see vagrant-test-sanitizers.sh 33 | # for more information 34 | # distro-tag: arch-sanitizers-gcc or arch-sanitizers-clang 35 | # 36 | # "Standalone" Arch Linux 37 | # Runs unit tests, fuzzers, and integration tests 38 | # distro-tag: arch 39 | DISTRO="" 40 | REMOTE_REF="" 41 | 42 | set -eu 43 | set -o pipefail 44 | 45 | while getopts "d:r:s:" opt; do 46 | case "$opt" in 47 | d) 48 | DISTRO="$OPTARG" 49 | ;; 50 | r) 51 | REMOTE_REF="$OPTARG" 52 | ;; 53 | s) 54 | REPO_URL="$OPTARG" 55 | ;; 56 | ?) 57 | exit 1 58 | ;; 59 | *) 60 | echo "Usage: $0 -d DISTRO_TAG [-r REMOTE_REF] [-s SOURCE_REPO_URL]" 61 | exit 1 62 | esac 63 | done 64 | 65 | if [[ -z "$DISTRO" ]]; then 66 | echo >&2 "Missing argument: distro tag" 67 | fi 68 | 69 | # Fetch the upstream systemd repo 70 | test -e systemd && rm -rf systemd 71 | echo "Cloning repo: $REPO_URL" 72 | git clone "$REPO_URL" systemd 73 | export SYSTEMD_ROOT="$PWD/systemd" 74 | 75 | trap at_exit EXIT 76 | 77 | pushd systemd || { echo >&2 "Can't pushd to systemd"; exit 1; } 78 | git_checkout_pr "$REMOTE_REF" 79 | 80 | # Create a Coveralls configuration file if the Coveralls token is present 81 | # (the file is provided by the agent-control.py script) 82 | if [[ -f /.coveralls.token ]]; then 83 | ( 84 | set +x 85 | cat >.coveralls.yml <&1 | tee "$LOGDIR/console-$DISTRO.log" 104 | -------------------------------------------------------------------------------- /jenkins/jenkins.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # Jenkins configuration for the Configuration as a Code plugin (JCasC) 3 | # Docs & examples: 4 | # - https://github.com/jenkinsci/configuration-as-code-plugin 5 | # 6 | # yamllint disable rule:line-length 7 | 8 | jenkins: 9 | authorizationStrategy: 10 | globalMatrix: 11 | # Allow viewing jobs to anonymous users 12 | # Note: this _replaces_ the whole permissions matrix 13 | permissions: 14 | - "Job/Discover:anonymous" 15 | - "Job/Read:anonymous" 16 | - "Job/ViewStatus:anonymous" 17 | - "Overall/Administer:mrc0mmand-admin-edit-view" 18 | - "Overall/Read:anonymous" 19 | 20 | globalNodeProperties: 21 | # Set LANG=en_US.utf8 everywhere to avoid encoding issues when viewing 22 | # logs in a browser 23 | - envVars: 24 | env: 25 | - key: LANG 26 | value: en_US.utf8 27 | 28 | # Note: don't add anything to the clouds.kubernetes section here, as it 29 | # completely _overrides_ the pod templates provided by OCP 30 | 31 | markupFormatter: 32 | # OWASP Markup Formatter plugin 33 | rawHtml: 34 | disableSyntaxHighlighting: false 35 | mode: EXCLUSIVE 36 | numExecutors: 0 37 | primaryView: 38 | all: 39 | name: "All" 40 | remotingSecurity: 41 | enabled: true 42 | 43 | systemMessage: | 44 |

45 | 46 | CI jobs for both upstream and downstream (RHEL) systemd repositories.

47 | 48 | Upstream: https://github.com/systemd/systemd
49 | Upstream-stable: https://github.com/systemd/systemd-stable
50 | RHEL 7: https://github.com/redhat-plumbers/systemd-rhel7
51 | RHEL 8: https://github.com/redhat-plumbers/systemd-rhel8
52 | RHEL 9: https://github.com/redhat-plumbers/systemd-rhel9
53 |
54 | Upstream test coverage: Coverage Status 55 |
56 |
57 | CI scripts: https://github.com/systemd/systemd-centos-ci
58 |
59 |
60 | Something's broken or missing? Ping mrc0mmand@LiberaChat or open an issue in the systemd CentOS CI GitHub repo. 61 | 62 | views: 63 | - all: 64 | name: "All" 65 | - list: 66 | includeRegex: "^rhel.+" 67 | name: "CentOS - RHEL" 68 | - list: 69 | includeRegex: "(?!.+-stable)upstream-.+" 70 | name: "Upstream" 71 | - list: 72 | includeRegex: "upstream-.+-stable$" 73 | name: "Upstream-stable" 74 | 75 | unclassified: 76 | ansiColorBuildWrapper: 77 | globalColorMapName: "xterm" 78 | ghprbTrigger: 79 | adminlist: "mrc0mmand" 80 | manageWebhooks: false 81 | location: 82 | # Jenkins in OCP overrides this on each startup 83 | # See: https://pagure.io/centos-infra/issue/1103 84 | adminAddress: builder@jenkins-systemd.apps.ocp.cloud.ci.centos.org 85 | mailer: 86 | smtpHost: smtp.ci.centos.org 87 | -------------------------------------------------------------------------------- /utils/reposync.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DOWNLOAD_LOCATION="${1:-.}" 4 | # JOB_URL is exported by Jenkins 5 | # See: https://jenkins-systemd.apps.ocp.cloud.ci.centos.org/env-vars.html/ 6 | ARTIFACT_BASE_DIR="${JOB_URL:?}/lastSuccessfulBuild/artifact/$DOWNLOAD_LOCATION" 7 | 8 | at_exit() { 9 | rm -f repo-config.repo 10 | } 11 | 12 | set -eu 13 | set -o pipefail 14 | 15 | trap at_exit EXIT 16 | 17 | sync_repo() { 18 | local repo_link="${1:?}" 19 | local repo_id="${2:?}" 20 | local local_repo_id="${3:?}" 21 | local release_ver="${4:?}" 22 | local arches gpg_key_url gpg_key_name 23 | 24 | IFS=" " read -ra arches <<<"${5:?}" 25 | 26 | rm -f repo-config.repo 27 | curl -s -o repo-config.repo "$repo_link" 28 | 29 | # Check if the original repository configuration contains a URL to a GPG key 30 | # If so, parse it and download it 31 | gpg_key_url="$(awk -F= '/^gpgkey=/ { print $2 }' repo-config.repo)" 32 | if [[ -n "$gpg_key_url" ]]; then 33 | gpg_key_name="${gpg_key_url##*/}" 34 | curl -s -o "$gpg_key_name" "$gpg_key_url" 35 | fi 36 | 37 | for arch in "${arches[@]}"; do 38 | # Make a local copy of the original repository packages 39 | dnf reposync --norepopath --newest-only --download-metadata \ 40 | --arch="${arch},noarch" --forcearch="$arch" \ 41 | --config="repo-config.repo" --repoid="$repo_id" \ 42 | --releasever="$release_ver" \ 43 | --download-path="$DOWNLOAD_LOCATION/$local_repo_id/$arch" 44 | done 45 | 46 | # Create a repo file 47 | cat >"$DOWNLOAD_LOCATION/$local_repo_id/$local_repo_id.repo" << EOF 48 | [$local_repo_id] 49 | name=Mirror of $repo_id Copr repo (\$basearch) 50 | baseurl=$ARTIFACT_BASE_DIR/$local_repo_id/\$basearch/ 51 | skip_if_unavailable=False 52 | enabled=1 53 | # Disable modular filtering for this repository, so we can override certain 54 | # module packages with our own 55 | module_hotfixes=1 56 | EOF 57 | # Copy over the downloaded GPG key, if any 58 | if [[ -f "$gpg_key_name" ]]; then 59 | mv "$gpg_key_name" "$DOWNLOAD_LOCATION/$local_repo_id/$gpg_key_name" 60 | echo "gpgkey=$ARTIFACT_BASE_DIR/$local_repo_id/$gpg_key_name" >>"$DOWNLOAD_LOCATION/$local_repo_id/$local_repo_id.repo" 61 | fi 62 | } 63 | 64 | # centos-8-stream 65 | ORIGINAL_REPO="https://copr.fedorainfracloud.org/coprs/mrc0mmand/systemd-centos-ci-centos8/repo/centos-stream-8/mrc0mmand-systemd-centos-ci-centos8-centos-stream-8.repo" 66 | ORIGINAL_REPO_ID="copr:copr.fedorainfracloud.org:mrc0mmand:systemd-centos-ci-centos8" 67 | LOCAL_REPO_ID="mrc0mmand-systemd-centos-ci-centos8-stream8" 68 | sync_repo "$ORIGINAL_REPO" "$ORIGINAL_REPO_ID" "$LOCAL_REPO_ID" 8 "x86_64 aarch64 ppc64le" 69 | 70 | # centos-9-stream 71 | ORIGINAL_REPO="https://copr.fedorainfracloud.org/coprs/mrc0mmand/systemd-centos-ci-centos9/repo/centos-stream-9/mrc0mmand-systemd-centos-ci-centos9-centos-stream-9.repo" 72 | ORIGINAL_REPO_ID="copr:copr.fedorainfracloud.org:mrc0mmand:systemd-centos-ci-centos9" 73 | LOCAL_REPO_ID="mrc0mmand-systemd-centos-ci-centos9-stream9" 74 | sync_repo "$ORIGINAL_REPO" "$ORIGINAL_REPO_ID" "$LOCAL_REPO_ID" 9 "x86_64 aarch64 ppc64le s390x" 75 | 76 | # centos-9-stream patched kernel (temporary) 77 | # FIXME: drop once https://issues.redhat.com/browse/RHEL-32384 is resolved 78 | ORIGINAL_REPO="https://copr.fedorainfracloud.org/coprs/mrc0mmand/c9s-kernel-debug/repo/centos-stream-9/mrc0mmand-c9s-kernel-debug-centos-stream-9.repo" 79 | ORIGINAL_REPO_ID="copr:copr.fedorainfracloud.org:mrc0mmand:c9s-kernel-debug" 80 | LOCAL_REPO_ID="mrc0mmand-c9s-kernel-debug-stream9" 81 | sync_repo "$ORIGINAL_REPO" "$ORIGINAL_REPO_ID" "$LOCAL_REPO_ID" 9 "x86_64" 82 | -------------------------------------------------------------------------------- /vagrant/bootstrap_scripts/arch-sanitizers-gcc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eux 4 | set -o pipefail 5 | 6 | whoami 7 | uname -a 8 | 9 | # The custom CentOS CI box should be updated and provide necessary 10 | # build & test dependencies 11 | 12 | # Let's make the $BUILD_DIR for meson reside outside of the NFS volume mounted 13 | # under /build to avoid certain race conditions, like: 14 | # /usr/bin/ld: error: /build/build/src/udev/libudev.so.1: file too short 15 | # The same path must be exported in the respective tests scripts (vagrant-test.sh, 16 | # etc.) so the unit & integration tests can find the compiled binaries 17 | # Note: avoid using /tmp or /var/tmp, as certain tests use binaries from the 18 | # buildir in combination with PrivateTmp=true 19 | export BUILD_DIR="${BUILD_DIR:-/systemd-meson-build}" 20 | 21 | # Use systemd repo path specified by SYSTEMD_ROOT 22 | pushd /build 23 | 24 | # Dump list of installed packages 25 | pacman -Q > vagrant-arch-sanitizers-gcc-installed-pkgs.txt 26 | # Dump additional OS info 27 | { 28 | echo "### CPUINFO ###" 29 | cat /proc/cpuinfo 30 | echo "### MEMINFO ###" 31 | cat /proc/meminfo 32 | echo "### VERSION ###" 33 | cat /proc/version 34 | } > vagrant-arch-sanitizers-gcc-osinfo.txt 35 | 36 | rm -fr "$BUILD_DIR" 37 | # Build phase 38 | # Compile systemd with the Address Sanitizer (ASan) and Undefined Behavior 39 | # Sanitizer (UBSan) 40 | 41 | # FIXME (--as-needed) 42 | # Since version 10, both gcc and clang started to ignore certain linker errors 43 | # when compiling with -fsanitize=address. This eventually leads up to -lcrypt 44 | # not being correctly propagated, but the fact is masked by the aforementioned 45 | # issue. However, when the binary attempts to load a symbol from the libcrypt 46 | # binary, it crashes since it's not linked correctly against it. 47 | # Negating the -Wl,--as-needed used by default by -Wl,--no-as-needed seems to 48 | # help in this case. 49 | # 50 | # See: 51 | # https://bugzilla.redhat.com/show_bug.cgi?id=1827338#c3 52 | # https://github.com/systemd/systemd-centos-ci/issues/247 53 | 54 | meson setup "$BUILD_DIR" \ 55 | --werror \ 56 | -Dc_args='-fno-omit-frame-pointer -ftrapv' \ 57 | -Dcpp_args='-fno-omit-frame-pointer -ftrapv' \ 58 | -Db_asneeded=false `# See the FIXME (--as-needed) above` \ 59 | -Ddebug=true \ 60 | --optimization=g \ 61 | -Dlog-trace=true \ 62 | -Dfexecve=true \ 63 | -Dtests=unsafe \ 64 | -Dslow-tests=true \ 65 | -Dinstall-tests=true \ 66 | -Ddbuspolicydir=/usr/share/dbus-1/system.d \ 67 | -Dlocalegen-path=/usr/bin/locale-gen \ 68 | -Dman=false \ 69 | -Db_sanitize=address,undefined 70 | ninja -C "$BUILD_DIR" 71 | 72 | # Manually install upstream D-Bus config file for org.freedesktop.network1 73 | # so systemd-networkd testsuite can use potentially new/updated methods 74 | cp -fv src/network/org.freedesktop.network1.conf /usr/share/dbus-1/system.d/ 75 | 76 | # Manually install upstream systemd-networkd* service unit files in case a PR 77 | # introduces a change in them 78 | # See: https://github.com/systemd/systemd/pull/14415#issuecomment-579307925 79 | cp -fv "$BUILD_DIR/units/systemd-networkd.service" /usr/lib/systemd/system/systemd-networkd.service 80 | cp -fv "$BUILD_DIR/units/systemd-networkd-wait-online.service" /usr/lib/systemd/system/systemd-networkd-wait-online.service 81 | 82 | # Support udevadm/systemd-udevd merge efforts from 83 | # https://github.com/systemd/systemd/pull/15918 84 | # The udevadm -> systemd-udevd symlink is created in the install phase which 85 | # we don't execute in sanitizer runs, so let's create it manually where 86 | # we need it 87 | if [[ -x "$BUILD_DIR/udevadm" && ! -x "$BUILD_DIR/systemd-udevd" ]]; then 88 | ln -frsv "$BUILD_DIR/udevadm" "$BUILD_DIR/systemd-udevd" 89 | fi 90 | 91 | popd 92 | -------------------------------------------------------------------------------- /agent/bootstrap-rhel7.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | # shellcheck disable=SC2155 3 | 4 | LIB_ROOT="$(dirname "$0")/../common" 5 | # shellcheck source=common/task-control.sh 6 | . "$LIB_ROOT/task-control.sh" "bootstrap-logs-rhel7" || exit 1 7 | # shellcheck source=common/utils.sh 8 | . "$LIB_ROOT/utils.sh" || exit 1 9 | 10 | REMOTE_REF="" 11 | 12 | # EXIT signal handler 13 | at_exit() { 14 | # Let's collect some build-related logs 15 | set +e 16 | rsync -amq /var/tmp/systemd-test*/journal "$LOGDIR" &>/dev/null || : 17 | exectask "journalctl-bootstrap" "journalctl -b --no-pager" 18 | exectask "list-of-installed-packages" "rpm -qa" 19 | } 20 | 21 | set -eu 22 | set -o pipefail 23 | 24 | trap at_exit EXIT 25 | 26 | # Parse optional script arguments 27 | # Note: in RHEL7 version of the bootstrap script this is kind of pointless 28 | # (since it's parsing only a single option), but it allows us to have 29 | # a single interface in agent-control.py for all RHEL versions without 30 | # additional hassle 31 | while getopts "r:" opt; do 32 | case "$opt" in 33 | r) 34 | REMOTE_REF="$OPTARG" 35 | ;; 36 | ?) 37 | exit 1 38 | ;; 39 | *) 40 | echo "Usage: $0 [-r REMOTE_REF]" 41 | exit 1 42 | esac 43 | done 44 | 45 | # Install necessary dependencies 46 | # - systemd-* packages are necessary for correct users/groups to be created 47 | cmd_retry yum -y install systemd-journal-gateway systemd-resolved rpm-build yum-utils net-tools strace nc busybox e2fsprogs quota dnsmasq qemu-kvm 48 | cmd_retry yum-builddep -y systemd 49 | 50 | # Fetch the downstream systemd repo 51 | test -e systemd && rm -rf systemd 52 | git clone https://github.com/redhat-plumbers/systemd-rhel7 systemd 53 | pushd systemd || { echo >&2 "Can't pushd to systemd"; exit 1; } 54 | 55 | git_checkout_pr "$REMOTE_REF" 56 | 57 | # It's impossible to keep the local SELinux policy database up-to-date with 58 | # arbitrary pull request branches we're testing against. 59 | # Disable SELinux on the test hosts and avoid false positives. 60 | if setenforce 0; then 61 | echo SELINUX=disabled >/etc/selinux/config 62 | fi 63 | 64 | # Compile systemd 65 | ( 66 | ./autogen.sh 67 | CONFIGURE_OPTS=( 68 | --with-sysvinit-path=/etc/rc.d/init.d 69 | --with-rc-local-script-path-start=/etc/rc.d/rc.local 70 | --disable-timesyncd 71 | --disable-kdbus 72 | --disable-terminal 73 | --enable-gtk-doc 74 | --enable-compat-libs 75 | --disable-sysusers 76 | --disable-ldconfig 77 | --enable-lz4 78 | ) 79 | ./configure "${CONFIGURE_OPTS[@]}" 80 | make -j "$(nproc)" 81 | make install 82 | ) 2>&1 | tee "$LOGDIR/build.log" 83 | 84 | # Let's check if the new systemd at least boots before rebooting the system 85 | ( 86 | # Ensure the initrd contains the same systemd version as the one we're 87 | # trying to test 88 | dracut -f --filesystems "xfs ext4" 89 | 90 | centos_ensure_qemu_symlink 91 | 92 | ## Configure test environment 93 | # Explicitly set paths to initramfs and kernel images (for QEMU tests) 94 | export INITRD="/boot/initramfs-$(uname -r).img" 95 | export KERNEL_BIN="/boot/vmlinuz-$(uname -r)" 96 | # Set timeout for QEMU tests to kill them in case they get stuck 97 | export QEMU_TIMEOUT=600 98 | # Disable nspawn version of the test 99 | export TEST_NO_NSPAWN=1 100 | 101 | if ! make -C test/TEST-01-BASIC clean setup run clean; then 102 | rsync -amq /var/tmp/systemd-test*/journal "$LOGDIR/" &>/dev/null || : 103 | fi 104 | ) 2>&1 | tee "$LOGDIR/sanity-boot-check.log" 105 | 106 | echo "-----------------------------" 107 | echo "- REBOOT THE MACHINE BEFORE -" 108 | echo "- CONTINUING -" 109 | echo "-----------------------------" 110 | -------------------------------------------------------------------------------- /vagrant/vagrantfiles/Vagrantfile_rawhide: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby ts=4 sw=4 et: 3 | 4 | @ui = Vagrant::UI::Colored.new 5 | 6 | Vagrant.configure("2") do |config| 7 | config.vm.define :rawhide_selinux_ci 8 | 9 | # Use our updated & cached Vagrant box (see vagrant/vagrant-make-cache.sh) 10 | if ENV["VAGRANT_TEST_IMAGE"] then 11 | config.vm.box = "rawhide_selinux-new" 12 | config.vm.box_url = "https://artifacts.ci.centos.org/systemd/vagrant_boxes/rawhide_selinux-new" 13 | else 14 | config.vm.box = "rawhide_selinux" 15 | config.vm.box_url = "https://artifacts.ci.centos.org/systemd/vagrant_boxes/rawhide_selinux" 16 | end 17 | 18 | # Disable the default /vagrant share, since we don't use it anyway 19 | config.vm.synced_folder ".", "/vagrant", disabled: true 20 | # NFSv4 supports TCP only and requires 'no_root_squash' (instead of the default 21 | # 'all_squash') in /etc/exports 22 | config.vm.synced_folder ENV["SYSTEMD_ROOT"], "/build", 23 | type: "nfs", 24 | nfs_version: 4.0, 25 | nfs_udp: false, 26 | linux__nfs_options: ["rw", "no_subtree_check", "no_root_squash", "async"], 27 | mount_options: ["rw", "async", "fsc", "rsize=32768", "wsize=32768", 28 | "hard", "noatime", "actimeo=2"] 29 | 30 | # Bump the boot timeout from 5 to 10 minutes, since the filesystem relabeling 31 | # can take (slightly) longer than that 32 | config.vm.boot_timeout = 600 33 | 34 | # Set the default user for `vagrant ssh` to root and insert the Vagrant's 35 | # SSH key to allow passwordless SSH login. This requires the SSH account 36 | # to be unlocked, have its password set to 'vagrant', and the PermitRootLogin= 37 | # directive in /etc/ssh/sshd_config set to 'yes'. 38 | config.ssh.username = "root" 39 | config.ssh.password = "vagrant" 40 | config.ssh.insert_key = "true" 41 | 42 | ovmf_nvram = "/tmp/OVMF_VARS.arch.fd" 43 | 44 | config.vm.provider :libvirt do |libvirt| 45 | libvirt.driver = ENV.fetch("VAGRANT_DRIVER", "kvm") 46 | libvirt.memory = ENV.fetch("VAGRANT_MEMORY", "8192") 47 | libvirt.cpus = ENV.fetch("VAGRANT_CPUS", "8") 48 | 49 | # Collect output from a serial console into a file to make debugging easier 50 | # The -nographic option allows us to collect BIOS messages as well 51 | libvirt.qemuargs :value => "-nographic" 52 | # This file needs to be collected later by vagrant-ci-wrapper.sh 53 | libvirt.serial :type => "file", :source => {:path => "/tmp/vagrant-arch-serial-console.log"} 54 | 55 | # Pass through /dev/random from the host to the VM 56 | libvirt.random :model => 'random' 57 | 58 | # Enable nested KVM 59 | libvirt.nested = true 60 | libvirt.cpu_mode = "host-model" 61 | 62 | libvirt.machine_type = "q35" 63 | 64 | # Emulate UEFI using OVMF 65 | libvirt.loader = "/usr/share/edk2/ovmf/OVMF_CODE.fd" 66 | libvirt.nvram = ovmf_nvram 67 | 68 | # Emulate TPM 2.0 using swtpm 69 | libvirt.tpm_model = "tpm-crb" 70 | libvirt.tpm_type = "emulator" 71 | libvirt.tpm_version = "2.0" 72 | end 73 | 74 | config.trigger.before [:up, :provision] do |trigger| 75 | trigger.run = {inline: "cp -u /usr/share/edk2/ovmf/OVMF_VARS.fd #{ovmf_nvram}"} 76 | end 77 | 78 | config.trigger.after [:destroy] do |trigger| 79 | trigger.run = {inline: "rm #{ovmf_nvram}"} 80 | end 81 | 82 | # Use an external bootstrap script if set 83 | # Use an external bootstrap script if set 84 | if ENV["VAGRANT_BOOTSTRAP_SCRIPT"] then 85 | @ui.info("Using a custom bootstrap script: " + ENV["VAGRANT_BOOTSTRAP_SCRIPT"]) 86 | config.vm.provision "shell", 87 | privileged: true, 88 | path: ENV["VAGRANT_BOOTSTRAP_SCRIPT"] 89 | else 90 | @ui.info("No bootstrap script given (use VAGRANT_BOOTSTRAP_SCRIPT env variable to fix this)") 91 | end 92 | end 93 | -------------------------------------------------------------------------------- /utils/generate-index.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | # This script generates a simple index HTML page to make possible 4 | # investigation easier. 5 | # Note: this is a single-purpose script which heavily depends on the directory 6 | # structure created by the systemd CentOS CI Jenkins job, as well as 7 | # its environment (i.e. env variables) 8 | 9 | set -e 10 | set -o pipefail 11 | 12 | if [[ $# -ne 2 ]]; then 13 | echo >&2 "Usage: $0 artifacts_dir index_file" 14 | exit 1 15 | fi 16 | 17 | # Custom path to the 'tree' binary in CentOS CI 18 | export PATH="/home/systemd/bin:$PATH" 19 | ARTIFACTS_DIR="$1" 20 | INDEX_FILE="$2" 21 | CSS_FILE="$INDEX_FILE.css" 22 | 23 | if [[ ! -d "$ARTIFACTS_DIR" ]]; then 24 | echo >&2 "'$ARTIFACTS_DIR' is not a directory" 25 | exit 1 26 | fi 27 | 28 | PR="${ghprbPullId:-N/A}" 29 | PR_URL="${ghprbPullLink:-#}" 30 | 31 | # Generate a nice HTML directory listing using the tree utility 32 | tree --charset=utf-8 -C -T "systemd CentOS CI (PR#$PR)" -H "$ARTIFACTS_DIR" "$ARTIFACTS_DIR" -o "$INDEX_FILE" 33 | 34 | # Add some useful info below the main title 35 | ADDITIONAL_INFO_FILE="$(mktemp)" 36 | cat >"$ADDITIONAL_INFO_FILE" < 38 | 39 | 40 | 41 | 42 | 43 |
Build URL: $BUILD_URL
Console log: $BUILD_URL/console
PR title: ${ghprbPullTitle:-N/A}
Date: $(date --rfc-3339=seconds)
44 |
45 | 46 | EOF 47 | # 1) Add a newline after the tag, so we can use sed's pattern matching 48 | sed -i "s##\n#" "$INDEX_FILE" 49 | # 2) Append contents of the $ADDITIONAL_INFO_FILE after the tag 50 | sed -i "/<\/h1>/ r $ADDITIONAL_INFO_FILE" "$INDEX_FILE" 51 | # Delete the temporary file 52 | rm -f "$ADDITIONAL_INFO_FILE" 53 | 54 | # Use a relatively ugly sed to append a red cross after each "_FAIL" log file 55 | sed -i -r 's/(_FAIL.log)(<\/a>)/\1 \❌\2/g' "$INDEX_FILE" 56 | 57 | # Completely unnecessary workaround for CentOS CI Jenkins' CSP, which disallows 58 | # inline CSS (but I want my colored links) 59 | # Part 1: extract the inline CSS 60 | grep --text -Pzo '(?s)(?<=)' "$INDEX_FILE" | sed -e '//d' >"$CSS_FILE" 61 | # Part 2: link it back to the original index file 62 | sed -i "//a" "$INDEX_FILE" 63 | 64 | LANDING_URL="${BUILD_URL}artifact/${PWD##"$WORKSPACE"}/index.html" 65 | 66 | # As we can't expect to have 'cowsay' installed, let's make our own oversimplified 67 | # version of it for absolutely no apparent reason. The picture below is, of course, 68 | # borrowed from the cowsay package. 69 | echo -n ' '; for ((i = 0; i < ${#LANDING_URL} + 2; i++)); do echo -n '_'; done 70 | echo -ne "\n< $LANDING_URL >\n" 71 | echo -n ' '; for ((i = 0; i < ${#LANDING_URL} + 2; i++)); do echo -n '-'; done 72 | # shellcheck disable=SC1004,SC2028 73 | echo ' 74 | \ ^ /^ 75 | \ / \ // \ 76 | \ |\___/| / \// .\ 77 | \ /O O \__ / // | \ \ *----* 78 | / / \/_/ // | \ \ \ | 79 | @___@` \/_ // | \ \ \/\ \ 80 | 0/0/| \/_ // | \ \ \ \ 81 | 0/0/0/0/| \/// | \ \ | | 82 | 0/0/0/0/0/_|_ / ( // | \ _\ | / 83 | 0/0/0/0/0/0/`/,_ _ _/ ) ; -. | _ _\.-~ / / 84 | ,-} _ *-.|.-~-. .~ ~ 85 | \ \__/ `/\ / ~-. _ .-~ / 86 | \____(oo) *. } { / 87 | ( (--) .----~-.\ \-` .~ 88 | //__\\ \__ Ack! ///.----..< \ _ -~ 89 | // \\ ///-._ _ _ _ _ _ _{^ - - - - ~ 90 | ' 91 | -------------------------------------------------------------------------------- /vagrant/vagrantfiles/Vagrantfile_arch: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby ts=4 sw=4 et: 3 | 4 | @ui = Vagrant::UI::Colored.new 5 | 6 | Vagrant.configure("2") do |config| 7 | config.vm.define :archlinux_systemd_ci 8 | 9 | # Use our updated & cached Vagrant box (see vagrant/vagrant-make-cache.sh) 10 | if ENV["VAGRANT_TEST_IMAGE"] then 11 | config.vm.box = "archlinux_systemd-new" 12 | config.vm.box_url = "https://artifacts.ci.centos.org/systemd/vagrant_boxes/archlinux_systemd-new" 13 | else 14 | config.vm.box = "archlinux_systemd" 15 | config.vm.box_url = "https://artifacts.ci.centos.org/systemd/vagrant_boxes/archlinux_systemd" 16 | end 17 | 18 | # Disable the default /vagrant share, since we don't use it anyway 19 | config.vm.synced_folder ".", "/vagrant", disabled: true 20 | # NFSv4 supports TCP only and requires 'no_root_squash' (instead of the default 21 | # 'all_squash') in /etc/exports 22 | # 23 | # Note: if you're frustrated future me trying to get this work locally, make 24 | # sure that: 25 | # - firewall-cmd --zone=libvirt --add-service={nfs,nfs3,mountd,rpc-bind} 26 | # - check if the VM didn't get two IP addresses, if so remove the one that 27 | # doesn't match the IP in /etc/exports on the host 28 | config.vm.synced_folder ENV["SYSTEMD_ROOT"], "/build", 29 | type: "nfs", 30 | nfs_version: 4.0, 31 | nfs_udp: false, 32 | linux__nfs_options: ["rw", "no_subtree_check", "no_root_squash", "async"], 33 | mount_options: ["rw", "async", "fsc", "rsize=32768", "wsize=32768", 34 | "hard", "noatime", "actimeo=2"] 35 | 36 | # Set the default user for `vagrant ssh` to root and insert the Vagrant's 37 | # SSH key to allow passwordless SSH login. This requires the SSH account 38 | # to be unlocked, have its password set to 'vagrant', and the PermitRootLogin= 39 | # directive in /etc/ssh/sshd_config set to 'yes'. 40 | config.ssh.username = "root" 41 | config.ssh.password = "vagrant" 42 | config.ssh.insert_key = "true" 43 | 44 | ovmf_nvram = "/tmp/OVMF_VARS.arch.fd" 45 | 46 | # Note: CentOS CI infra specific overrides - you may want to change them 47 | # to run the VM locally 48 | config.vm.provider :libvirt do |libvirt| 49 | libvirt.driver = ENV.fetch("VAGRANT_DRIVER", "kvm") 50 | libvirt.memory = ENV.fetch("VAGRANT_MEMORY", "8192") 51 | libvirt.cpus = ENV.fetch("VAGRANT_CPUS", "8") 52 | 53 | # Collect output from a serial console into a file to make debugging easier 54 | # The -nographic option allows us to collect BIOS messages as well 55 | libvirt.qemuargs :value => "-nographic" 56 | # This file needs to be collected later by vagrant-ci-wrapper.sh 57 | libvirt.serial :type => "file", :source => {:path => "/tmp/vagrant-arch-serial-console.log"} 58 | 59 | # Pass through /dev/random from the host to the VM 60 | libvirt.random :model => 'random' 61 | 62 | # Enable nested KVM 63 | libvirt.nested = true 64 | libvirt.cpu_mode = "host-model" 65 | 66 | libvirt.machine_type = "q35" 67 | 68 | # Emulate UEFI using OVMF 69 | libvirt.loader = "/usr/share/edk2/ovmf/OVMF_CODE.fd" 70 | libvirt.nvram = ovmf_nvram 71 | 72 | # Emulate TPM 2.0 using swtpm 73 | libvirt.tpm_model = "tpm-crb" 74 | libvirt.tpm_type = "emulator" 75 | libvirt.tpm_version = "2.0" 76 | end 77 | 78 | config.ssh.extra_args = [ 79 | "-o", "ConnectionAttempts=60", 80 | "-o", "ConnectTimeout=180", 81 | "-o", "ServerAliveInterval=60", 82 | "-o", "TCPKeepAlive=no" 83 | ] 84 | 85 | config.trigger.before [:up, :provision] do |trigger| 86 | trigger.run = {inline: "cp -u /usr/share/edk2/ovmf/OVMF_VARS.fd #{ovmf_nvram}"} 87 | end 88 | 89 | config.trigger.after [:destroy] do |trigger| 90 | trigger.run = {inline: "rm #{ovmf_nvram}"} 91 | end 92 | 93 | # Use an external bootstrap script if set 94 | if ENV["VAGRANT_BOOTSTRAP_SCRIPT"] then 95 | @ui.info("Using a custom bootstrap script: " + ENV["VAGRANT_BOOTSTRAP_SCRIPT"]) 96 | config.vm.provision "shell", 97 | privileged: true, 98 | path: ENV["VAGRANT_BOOTSTRAP_SCRIPT"] 99 | else 100 | @ui.info("No bootstrap script given (use VAGRANT_BOOTSTRAP_SCRIPT env variable to fix this)") 101 | end 102 | end 103 | -------------------------------------------------------------------------------- /vagrant/vagrant-setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | # Simple script which installs Vagrant to a machine from CentOS CI 3 | # infrastructure if not already installed. The script also installs 4 | # vagrant-libvirt module to support "proper" virtualization (kvm/qemu) instead 5 | # of default containers. 6 | 7 | LIB_ROOT="$(dirname "$0")/../common" 8 | # shellcheck source=common/utils.sh 9 | . "$LIB_ROOT/utils.sh" || exit 1 10 | 11 | VAGRANT_PKG_URL="https://releases.hashicorp.com/vagrant/2.4.3/vagrant-2.4.3-1.x86_64.rpm" 12 | WORKAROUNDS_DIR="$(dirname "$(readlink -f "$0")")/workarounds" 13 | 14 | set -eu 15 | set -o pipefail 16 | 17 | # Set up nested KVM 18 | # Let's make all errors "soft", at least for now, as we're still perfectly 19 | # fine with running tests without nested KVM 20 | if KVM_MODULE_NAME="$(lsmod | grep -m1 -Eo '(kvm_intel|kvm_amd)')"; then 21 | echo "[vagrant-setup] Detected KVM module: $KVM_MODULE_NAME" 22 | # Attempt to reload the detected KVM module with nested=1 parameter 23 | if modprobe -v -r "$KVM_MODULE_NAME" && modprobe -v "$KVM_MODULE_NAME" nested=1; then 24 | # The reload was successful, check if the module's 'nested' parameter 25 | # confirms that nested KVM is indeed enabled 26 | KVM_MODULE_NESTED="$(< "/sys/module/$KVM_MODULE_NAME/parameters/nested")" || : 27 | echo "[vagrant-setup] /sys/module/$KVM_MODULE_NAME/parameters/nested: $KVM_MODULE_NESTED" 28 | 29 | if [[ "$KVM_MODULE_NESTED" =~ (1|Y) ]]; then 30 | echo "[vagrant-setup] Nested KVM is enabled" 31 | else 32 | echo "[vagrant-setup] Failed to enable nested KVM" 33 | fi 34 | else 35 | echo "[vagrant-setup] Failed to reload module '$KVM_MODULE_NAME'" 36 | fi 37 | else 38 | echo "[vagrant-setup] No KVM module found, can't setup nested KVM" 39 | fi 40 | 41 | # Configure NTP (chronyd) 42 | if ! rpm -q chrony; then 43 | cmd_retry dnf -y install chrony 44 | fi 45 | 46 | systemctl enable --now chronyd 47 | systemctl status --no-pager chronyd 48 | 49 | # Set tuned to throughput-performance if available 50 | if command -v tuned-adm 2>/dev/null; then 51 | tuned-adm profile throughput-performance 52 | tuned-adm active 53 | tuned-adm verify 54 | fi 55 | 56 | # Configure Vagrant 57 | if ! vagrant version 2>/dev/null; then 58 | # Install Vagrant 59 | cmd_retry dnf -y install "$VAGRANT_PKG_URL" 60 | fi 61 | 62 | # To speed up Vagrant jobs, let's use a pre-compiled bundle with the necessary 63 | # shared libraries. See vagrant/workarounds/build-shared-libs.sh for more info. 64 | if [[ -e "$WORKAROUNDS_DIR/vagrant-shared-libs.tar.gz" ]]; then 65 | echo "[vagrant-setup] Found pre-compiled shared libs, unpacking..." 66 | command -v tar >/dev/null || dnf -q -y install tar 67 | tar -xzvf "$WORKAROUNDS_DIR/vagrant-shared-libs.tar.gz" -C /opt/vagrant/embedded/lib64/ 68 | fi 69 | 70 | if ! vagrant plugin list | grep vagrant-libvirt; then 71 | # Install vagrant-libvirt dependencies 72 | cmd_retry dnf -y --enablerepo crb install gcc libguestfs-tools-c libvirt libvirt-devel libgcrypt make qemu-kvm ruby-devel 73 | # Start libvirt daemon 74 | systemctl start libvirtd 75 | systemctl status libvirtd 76 | # Sanity-check if Vagrant is correctly installed 77 | vagrant version 78 | # Install vagrant-libvirt plugin 79 | # See: https://github.com/vagrant-libvirt/vagrant-libvirt 80 | # Env variables taken from https://github.com/vagrant-libvirt/vagrant-libvirt#possible-problems-with-plugin-installation-on-linux 81 | export CONFIGURE_ARGS='with-ldflags="-L/opt/vagrant/embedded/lib64 -L/opt/vagrant/embedded/lib" with-libvirt-include=/usr/include/libvirt with-libvirt-lib=/usr/lib' 82 | export GEM_HOME=~/.vagrant.d/gems 83 | export GEM_PATH=$GEM_HOME:/opt/vagrant/embedded/gems 84 | export PATH=/opt/vagrant/embedded/bin:$PATH 85 | cmd_retry vagrant plugin install vagrant-libvirt 86 | fi 87 | 88 | vagrant --version 89 | vagrant plugin list 90 | 91 | cmd_retry dnf -y install edk2-ovmf 92 | 93 | # Configure NFS for Vagrant's shared folders 94 | rpm -q nfs-utils || cmd_retry dnf -y install nfs-utils 95 | systemctl stop nfs-server 96 | systemctl start proc-fs-nfsd.mount 97 | lsmod | grep -E '^nfs$' || modprobe -v nfs 98 | lsmod | grep -E '^nfsd$' || modprobe -v nfsd 99 | echo 10 > /proc/sys/fs/nfs/nlm_grace_period 100 | echo 10 > /proc/fs/nfsd/nfsv4gracetime 101 | echo 10 > /proc/fs/nfsd/nfsv4leasetime 102 | systemctl enable nfs-server 103 | systemctl restart nfs-server 104 | systemctl status nfs-server 105 | sleep 10 106 | -------------------------------------------------------------------------------- /vagrant/bootstrap_scripts/arch-sanitizers-clang.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eux 4 | set -o pipefail 5 | 6 | whoami 7 | uname -a 8 | 9 | # The custom CentOS CI box should be updated and provide necessary 10 | # build & test dependencies 11 | 12 | # Let's make the $BUILD_DIR for meson reside outside of the NFS volume mounted 13 | # under /build to avoid certain race conditions, like: 14 | # /usr/bin/ld: error: /build/build/src/udev/libudev.so.1: file too short 15 | # The same path must be exported in the respective tests scripts (vagrant-test.sh, 16 | # etc.) so the unit & integration tests can find the compiled binaries 17 | # Note: avoid using /tmp or /var/tmp, as certain tests use binaries from the 18 | # buildir in combination with PrivateTmp=true 19 | export BUILD_DIR="${BUILD_DIR:-/systemd-meson-build}" 20 | 21 | # Use systemd repo path specified by SYSTEMD_ROOT 22 | pushd /build 23 | 24 | # Dump list of installed packages 25 | pacman -Q > vagrant-arch-sanitizers-clang-installed-pkgs.txt 26 | # Dump additional OS info 27 | { 28 | echo "### CPUINFO ###" 29 | cat /proc/cpuinfo 30 | echo "### MEMINFO ###" 31 | cat /proc/meminfo 32 | echo "### VERSION ###" 33 | cat /proc/version 34 | } > vagrant-arch-sanitizers-clang-osinfo.txt 35 | 36 | rm -fr "$BUILD_DIR" 37 | # Build phase 38 | # Compile systemd with the Address Sanitizer (ASan) and Undefined Behavior 39 | # Sanitizer (UBSan) using llvm/clang 40 | export CC=clang 41 | export CXX=clang++ 42 | 43 | # Unlike gcc's ASan, clang's ASan DSO is in a non-standard path, thus any binary 44 | # compiled with -shared-libasan using clang will fail to start. Let's add the 45 | # necessary path to the ldconfig cache to avoid that. 46 | if ! ASAN_RT_PATH="$(${CC:-clang} --print-file-name "libclang_rt.asan-$(uname -m).so")"; then 47 | echo >&2 "Couldn't detect path to the clang's ASan RT library" 48 | exit 1 49 | fi 50 | 51 | if ! ldconfig -p | grep -- "$ASAN_RT_PATH" >/dev/null; then 52 | LDCONFIG_PATH="$(mktemp /etc/ld.so.conf.d/99-clang-libasan-XXX.conf)" 53 | echo "Adding ${ASAN_RT_PATH%/*} to ldconfig cache (using $LDCONFIG_PATH)" 54 | echo "${ASAN_RT_PATH%/*}" >"$LDCONFIG_PATH" 55 | ldconfig 56 | 57 | if ! ldconfig -p | grep -- "$ASAN_RT_PATH" >/dev/null; then 58 | echo >&2 "Failed to add $ASAN_RT_PATH to ldconfig cache" 59 | exit 1 60 | fi 61 | fi 62 | 63 | # FIXME (--as-needed) 64 | # Since version 10, both gcc and clang started to ignore certain linker errors 65 | # when compiling with -fsanitize=address. This eventually leads up to -lcrypt 66 | # not being correctly propagated, but the fact is masked by the aforementioned 67 | # issue. However, when the binary attempts to load a symbol from the libcrypt 68 | # binary, it crashes since it's not linked correctly against it. 69 | # Negating the -Wl,--as-needed used by default by -Wl,--no-as-needed seems to 70 | # help in this case. 71 | # 72 | # See: 73 | # https://bugzilla.redhat.com/show_bug.cgi?id=1827338#c3 74 | # https://github.com/systemd/systemd-centos-ci/issues/247 75 | # 76 | # FIXME (-fno-sanitize=function) 77 | # systemd's hashmap implementation fails miserably with -fsanitize=function, 78 | # so let's disable it until [0] is resolved. 79 | # 80 | # [0] https://github.com/systemd/systemd/issues/29972 81 | 82 | meson setup "$BUILD_DIR" \ 83 | --werror \ 84 | -Dc_args='-Og -fno-omit-frame-pointer -ftrapv -shared-libasan -fno-sanitize=function' \ 85 | -Dc_link_args="-shared-libasan" \ 86 | -Dcpp_args='-Og -fno-omit-frame-pointer -ftrapv -shared-libasan -fno-sanitize=function' \ 87 | -Dcpp_link_args="-shared-libasan" \ 88 | -Db_asneeded=false `# See the FIXME (--as-needed) above` \ 89 | -Ddebug=true \ 90 | --optimization=g \ 91 | -Dlog-trace=true \ 92 | -Dfexecve=true \ 93 | -Dtests=unsafe \ 94 | -Dslow-tests=true \ 95 | -Dfuzz-tests=true \ 96 | -Dinstall-tests=true \ 97 | -Ddbuspolicydir=/usr/share/dbus-1/system.d \ 98 | -Dlocalegen-path=/usr/bin/locale-gen \ 99 | -Dman=false \ 100 | -Db_sanitize=address,undefined \ 101 | -Db_lundef=false # See https://github.com/mesonbuild/meson/issues/764 102 | ninja -C "$BUILD_DIR" 103 | 104 | # Manually install upstream D-Bus config file for org.freedesktop.network1 105 | # so systemd-networkd testsuite can use potentially new/updated methods 106 | cp -fv src/network/org.freedesktop.network1.conf /usr/share/dbus-1/system.d/ 107 | 108 | # Manually install upstream systemd-networkd service unit files in case a PR 109 | # introduces a change in them 110 | # See: https://github.com/systemd/systemd/pull/14415#issuecomment-579307925 111 | cp -fv "$BUILD_DIR/units/systemd-networkd.service" /usr/lib/systemd/system/systemd-networkd.service 112 | cp -fv "$BUILD_DIR/units/systemd-networkd-wait-online.service" /usr/lib/systemd/system/systemd-networkd-wait-online.service 113 | if [[ -f "$BUILD_DIR/units/systemd-networkd-dhcp-server.service" ]]; then 114 | # Introduced in https://github.com/systemd/systemd/pull/30021 115 | cp -v "$BUILD_DIR/units/systemd-networkd-dhcp-server.service" /usr/lib/systemd/system/systemd-networkd-dhcp-server.service 116 | fi 117 | 118 | # Support udevadm/systemd-udevd merge efforts from 119 | # https://github.com/systemd/systemd/pull/15918 120 | # The udevadm -> systemd-udevd symlink is created in the install phase which 121 | # we don't execute in sanitizer runs, so let's create it manually where 122 | # we need it 123 | if [[ -x "$BUILD_DIR/udevadm" && ! -x "$BUILD_DIR/systemd-udevd" ]]; then 124 | ln -frsv "$BUILD_DIR/udevadm" "$BUILD_DIR/systemd-udevd" 125 | fi 126 | 127 | popd 128 | -------------------------------------------------------------------------------- /vagrant/vagrant-build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | # This script is part of the systemd Vagrant test suite for CentOS CI. 3 | # 4 | # The script takes a distro string as the only argument and then tries to fetch, 5 | # build, and configure a Vagrant VM according to the respective Vagrantfile, 6 | # if such Vagrantfile exists. On success, vagrant-test.sh script is executed 7 | # inside the VM and the test artifacts are stored in a respective vagrant-$DISTRO_STRING 8 | # folder in the $SYSTEMD_ROOT directory for further investigation. 9 | 10 | # The passed distro string is split into two parts, for example: 11 | # DISTRO_STRING=arch-sanitizers-clang is split into 'arch', which identifies 12 | # a respective Vagrantfile and 'sanitizers-clang' which denotes the test type. 13 | # Also, the distro string as a whole identifies the bootstrap script, which 14 | # will be executed as part of the Vagrantfile. 15 | 16 | if ! vagrant version; then 17 | echo >&2 "Missing vagrant package, consider running 'vagrant-setup.sh'" 18 | exit 1 19 | fi 20 | 21 | if [[ $# -ne 1 ]]; then 22 | echo >&2 "Usage: $0 " 23 | exit 1 24 | fi 25 | 26 | set -eu 27 | set -o pipefail 28 | 29 | VAGRANT_ROOT="$(dirname "$(readlink -f "$0")")" 30 | VAGRANT_FILES="$VAGRANT_ROOT/vagrantfiles" 31 | VAGRANT_BOOTSTRAP_SCRIPTS="$VAGRANT_ROOT/bootstrap_scripts" 32 | VAGRANT_TEST_SCRIPTS="$VAGRANT_ROOT/test_scripts" 33 | WITH_SANITIZERS=false 34 | WITH_COVERAGE=false 35 | DISTRO_STRING="${1,,}" 36 | # Takes the first part of the hyphen-delimited distro string as a distro name 37 | # e.g.: if DISTRO_STRING=arch-sanitizers-clang then DISTRO=arch 38 | DISTRO="${DISTRO_STRING%%-*}" 39 | 40 | # To test generated images before they get into "production" vagran-make-cache.sh 41 | # suffixes them with "-new", so they're not picked automatically by the pipeline 42 | # and we can test them separately first without affecting the CI. 43 | if [[ $DISTRO_STRING =~ -new$ ]]; then 44 | # Tell the respective Vagrantfile to use a test box name/url instead of the 45 | # regular ("production") one 46 | export VAGRANT_TEST_IMAGE=1 47 | # Drop the '-new' suffix from the distro string to use the "standard" 48 | # bootstrap scripts 49 | DISTRO_STRING="${DISTRO_STRING%-new}" 50 | fi 51 | 52 | # If the distro string is in "-sanitizers-*" format, we're testing 53 | # systemd using various sanitizers (ASan, UBSan, etc.) and due to performance 54 | # issue we want to skip certain steps (like reboot and integration tests). 55 | if [[ $DISTRO_STRING =~ -sanitizers- ]]; then 56 | WITH_SANITIZERS=true 57 | elif [[ $DISTRO_STRING =~ -coverage ]]; then 58 | WITH_COVERAGE=true 59 | fi 60 | 61 | # Decide which Vagrant file to use 62 | VAGRANT_FILE="$VAGRANT_FILES/Vagrantfile_$DISTRO" 63 | # Use a custom external script, so we can use one Vagrantfile with multiple 64 | # different scripts instead of having multiple Vagrantfiles with inlined 65 | # scripts 66 | BOOTSTRAP_SCRIPT="${VAGRANT_BOOTSTRAP_SCRIPT:-$VAGRANT_BOOTSTRAP_SCRIPTS/$DISTRO_STRING.sh}" 67 | # Pick the respective test script for given distro string 68 | TEST_SCRIPT="${VAGRANT_TEST_SCRIPT:-$VAGRANT_TEST_SCRIPTS/test-$DISTRO_STRING.sh}" 69 | 70 | if [[ ! -f $VAGRANT_FILE ]]; then 71 | echo >&2 "No Vagrantfile found for distro '$DISTRO'" 72 | exit 1 73 | fi 74 | 75 | if [[ ! -f $BOOTSTRAP_SCRIPT ]]; then 76 | echo >&2 "Bootstrap script '$BOOTSTRAP_SCRIPT' not found" 77 | exit 1 78 | fi 79 | 80 | if [[ ! -f $TEST_SCRIPT ]]; then 81 | echo >&2 "Test script '$TEST_SCRIPT' not found" 82 | exit 1 83 | fi 84 | 85 | # Configure environment (following env variables are used in the respective 86 | # Vagrantfile) 87 | export SYSTEMD_ROOT="${SYSTEMD_ROOT:-$HOME/systemd}" 88 | export VAGRANT_DRIVER="${VAGRANT_DRIVER:-kvm}" 89 | export VAGRANT_MEMORY="${VAGRANT_MEMORY:-131072}" 90 | export VAGRANT_CPUS="${VAGRANT_CPUS:-$(nproc)}" 91 | export VAGRANT_BOOTSTRAP_SCRIPT="$BOOTSTRAP_SCRIPT" 92 | 93 | # Absolute systemd git root path on the host machine 94 | TEST_DIR="$(mktemp -d "$SYSTEMD_ROOT/vagrant-$DISTRO_STRING-config.XXX")" 95 | # Relative path (i.e just the vagrant-$DISTRO_STRING.XXX dir) used for navigation 96 | # in the guest VM 97 | RELATIVE_TEST_DIR="${TEST_DIR##*/}" 98 | 99 | # Copy the target Vagrant file to the test dir 100 | cp "$VAGRANT_FILE" "$TEST_DIR/Vagrantfile" 101 | cp "$VAGRANT_ROOT/../common/task-control.sh" "$TEST_DIR/task-control.sh" 102 | cp "$VAGRANT_ROOT/../common/utils.sh" "$TEST_DIR/utils.sh" 103 | pushd "$TEST_DIR" || { echo >&2 "Can't pushd to $TEST_DIR"; exit 1; } 104 | # Copy the test script to the test dir 105 | # Note: -L is necessary, since some test script may be symlinks, to avoid 106 | # code duplication 107 | cp -L "$TEST_SCRIPT" "$TEST_DIR/" 108 | 109 | # Provision the machine 110 | vagrant up --no-tty --provider=libvirt 111 | 112 | set +e 113 | 114 | if $WITH_SANITIZERS || $WITH_COVERAGE; then 115 | # Skip the reboot/reload when running with sanitizers, as it in most cases 116 | # causes boot to timeout or die completely 117 | # Run tests with sanitizers 118 | vagrant ssh -c "cd /build && $RELATIVE_TEST_DIR/${TEST_SCRIPT##*/} $DISTRO_STRING" 119 | SSH_EC=$? 120 | else 121 | # Reboot the VM to "apply" the new systemd 122 | timeout 5m vagrant reload 123 | case $? in 124 | 0) 125 | ;; 126 | 124) 127 | echo >&2 "Timeout during machine reboot" 128 | exit 124 129 | ;; 130 | *) 131 | echo >&2 "Failed to reboot the VM using 'vagrant reload'" 132 | exit 1 133 | ;; 134 | esac 135 | # Run tests 136 | vagrant ssh -c "cd /build && $RELATIVE_TEST_DIR/${TEST_SCRIPT##*/} $DISTRO_STRING" 137 | SSH_EC=$? 138 | fi 139 | 140 | if [[ $SSH_EC -ne 0 ]]; then 141 | echo >&2 "'vagrant ssh' exited with an unexpected EC: $SSH_EC" 142 | exit $SSH_EC 143 | fi 144 | 145 | # Destroy the VM 146 | vagrant destroy -f 147 | popd 148 | -------------------------------------------------------------------------------- /vagrant/vagrant-make-cache.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Script which updates Vagrant boxes used by systemd CentOS CI 3 | # with build dependencies and other configuration, so we don't have to that in 4 | # every CI job. These updated images are then stored on the CentOS CI artifact 5 | # server and can be reused by other CI jobs. 6 | # This script is intended to run in CentOS CI Jenkins periodically, to keep 7 | # the images up-to-date. 8 | # 9 | # In time of writing this, the setup phase, which consists of updating the 10 | # base system in the container and installing build/test dependencies, took 11 | # around 10 minutes, which is pretty substantial time slice for a CI run, which 12 | # takes ~45 minutes. 13 | 14 | set -eux 15 | set -o pipefail 16 | 17 | DUFFY_SSH_KEY="/root/.ssh/duffy.key" 18 | VAGRANT_ROOT="$(dirname "$(readlink -f "$0")")" 19 | VAGRANT_FILE="$VAGRANT_ROOT/boxes/${1:?Missing argument: Vagrantfile}" 20 | 21 | if [[ ! -e "$DUFFY_SSH_KEY" ]]; then 22 | echo >&2 "Missing Duffy SSH key, can't continue" 23 | exit 1 24 | fi 25 | 26 | if [[ ! -f "$VAGRANT_FILE" ]]; then 27 | echo >&2 "Couldn't find Vagrantfile '$VAGRANT_FILE'" 28 | exit 1 29 | fi 30 | 31 | # Disable SELinux on the test hosts and avoid false positives. 32 | sestatus | grep -E "SELinux status:\s*disabled" || setenforce 0 33 | 34 | # Install vagrant if not already installed 35 | "$VAGRANT_ROOT"/vagrant-setup.sh 36 | 37 | # Stop firewalld 38 | systemctl -q is-enabled firewalld && systemctl disable --now firewalld 39 | systemctl restart libvirtd 40 | 41 | mkdir vagrant_cache 42 | pushd vagrant_cache 43 | 44 | # The URL for Fedora Rawhide Vagrant box changes over time, so let's attempt 45 | # to get the latest one 46 | if [[ "${VAGRANT_FILE##*/}" == "Vagrantfile_rawhide_selinux" ]]; then 47 | echo "Fetching Vagrant name URL for Fedora Rawhide" 48 | 49 | if ! BOX_NAME="$(curl -s https://dl.fedoraproject.org/pub/fedora/linux/development/rawhide/Cloud/x86_64/images/ | grep -Po -m1 '(?<=")Fedora.*?vagrant.libvirt.box(?=")')"; then 50 | echo >&2 "Failed to fetch the box name for Fedora Rawhide (got: $BOX_NAME)" 51 | exit 1 52 | fi 53 | 54 | echo "Using '$BOX_NAME' as the Fedora Rawhide box name" 55 | 56 | sed -i "/config.vm.box_url/s/BOX-NAME-PLACEHOLDER/$BOX_NAME/" "$VAGRANT_FILE" 57 | fi 58 | 59 | echo "Installing btrfs-aware packages" 60 | dnf -y install centos-release-hyperscale-experimental 61 | dnf -y install btrfs-progs kernel libguestfs-tools-c 62 | # Add the btrfs-progs package to the guestfs image 63 | echo "btrfs-progs" >>/usr/lib64/guestfs/supermin.d/packages 64 | 65 | # Start a VM described in the Vagrantfile with all provision steps 66 | export VAGRANT_DRIVER="${VAGRANT_DRIVER:-kvm}" 67 | export VAGRANT_MEMORY="${VAGRANT_MEMORY:-32768}" 68 | export VAGRANT_CPUS="${VAGRANT_CPUS:-$(nproc)}" 69 | 70 | # Provision the VM 71 | cp "$VAGRANT_FILE" Vagrantfile 72 | cp "$VAGRANT_ROOT"/boxes/*.sh . 73 | vagrant up --no-tty --provider=libvirt 74 | # Make sure the VM is bootable after running the provision script 75 | timeout -v 5m vagrant reload 76 | # shellcheck disable=SC2016 77 | vagrant ssh -c 'sudo bootctl status' 78 | vagrant halt 79 | 80 | # Workaround for `virt-{sysprep,resize,...}` - work with the image via qemu directly 81 | # instead of using libvirt 82 | export LIBGUESTFS_BACKEND=direct 83 | 84 | # A very convoluted way to expand the backing image from the original 20G (ATTOW) 85 | # to something more usable 86 | if [[ "${VAGRANT_FILE##*/}" == "Vagrantfile_archlinux_systemd" ]]; then 87 | LIBVIRT_IMAGE="/var/lib/libvirt/images/vagrant_cache_archlinux_systemd.img" 88 | 89 | # Convert the qcow2 image back into a raw one so it's easier to manipulate with 90 | qemu-img convert -f qcow2 -O raw "$LIBVIRT_IMAGE" image.raw 91 | # Prepare a sparse file for the expanded image 92 | truncate -r image.raw new_image.raw 93 | truncate -s +100G new_image.raw 94 | # Expand the last partition in the image 95 | LAST_PART="$(virt-filesystems --parts -a image.raw | sort -r | head -n1)" 96 | virt-resize --expand "$LAST_PART" image.raw new_image.raw 97 | # Convert the new image back to qcow2 and copy it back to the libvirt storage 98 | # (and hope for the best) 99 | qemu-img convert -f raw -O qcow2 new_image.raw "$LIBVIRT_IMAGE" 100 | rm -f image.raw 101 | # Check if the new image boots 102 | vagrant up 103 | vagrant ssh -c 'lsblk; df -h /' 104 | # Also, sanity-check the whole BLS stuff 105 | # shellcheck disable=SC2016 106 | vagrant ssh -c 'sudo bash -xec "ls -lR $(bootctl -x); cat /etc/machine-id; [[ -e $(bootctl -x)/$( archlinux_systemd-new 114 | BOX_NAME="${VAGRANT_FILE##*/Vagrantfile_}-new" 115 | # You guessed it, another workaround - let's include the original Vagrantfile 116 | # as well, as it usually contains important settings which make the box 117 | # actually bootable. For this, we need to detect the location of the box, from 118 | # the original box name (i.e. generic/arch, see the beautiful awk below), and 119 | # then transform it to a path to the Vagrantfile, which contains the box name, 120 | # but all slashes are replaced by "-VAGRANTSLASH-" (and that's what the bash 121 | # substitution is for) 122 | ORIGINAL_BOX_NAME="$(awk 'match($0, /^[^#]*config.vm.box\s*=\s*"([^"]+)"/, m) { print m[1]; exit 0; }' "$VAGRANT_FILE")" 123 | ORIGINAL_VAGRANTFILE="$(find "$HOME/.vagrant.d/boxes/${ORIGINAL_BOX_NAME//\//-VAGRANTSLASH-}/" -type f -name Vagrantfile)" 124 | if [[ ! -e "$ORIGINAL_VAGRANTFILE" ]]; then 125 | echo >&2 "Failed to locate the original Vagrantfile for '$ORIGINAL_BOX_NAME' box" 126 | exit 1 127 | fi 128 | # Tell virt-sysprep to not truncate the already existing /etc/machine-id, since 129 | # we use it in paths to kernel and initrd images on the ESP 130 | export VAGRANT_LIBVIRT_VIRT_SYSPREP_OPERATIONS="defaults,-machine-id" 131 | vagrant package --no-tty --output "$BOX_NAME" --vagrantfile "$ORIGINAL_VAGRANTFILE" 132 | # Remove the VM we just packaged 133 | vagrant destroy -f 134 | 135 | # Check if we can build a usable VM from the just packaged box 136 | TEST_DIR="$(mktemp -d testbox.XXX)" 137 | 138 | vagrant box remove -f testbox || : 139 | vagrant box add --name testbox "$BOX_NAME" 140 | pushd "$TEST_DIR" 141 | cat >Vagrantfile < 'random' 154 | libvirt.machine_type = "q35" 155 | libvirt.loader = "/usr/share/edk2/ovmf/OVMF_CODE.fd" 156 | libvirt.nvram = "/tmp/OVMF_VARS.fd" 157 | end 158 | 159 | end 160 | EOF 161 | cp -fvL /usr/share/edk2/ovmf/OVMF_VARS.fd /tmp 162 | chmod o+rw /tmp/OVMF_VARS.fd 163 | vagrant up --no-tty --provider=libvirt 164 | # shellcheck disable=SC2016 165 | vagrant ssh -c 'bash -exc "bootctl status; uname -a; id; [[ $UID == 0 ]]"' 166 | 167 | # Cleanup 168 | vagrant halt 169 | vagrant destroy -f 170 | vagrant box remove -f testbox 171 | popd && rm -fr "$TEST_DIR" 172 | 173 | # Upload the box to the CentOS CI artifact storage 174 | 175 | # Little workaround to create a proper directory hierarchy on the server 176 | mkdir vagrant_boxes 177 | mv "$BOX_NAME" vagrant_boxes 178 | 179 | rsync -e "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i $DUFFY_SSH_KEY" -av "vagrant_boxes" systemd@artifacts.ci.centos.org:/srv/artifacts/systemd/ 180 | echo "Box URL: https://artifacts.ci.centos.org/systemd/vagrant_boxes/$BOX_NAME" 181 | -------------------------------------------------------------------------------- /vagrant/test_scripts/test-arch.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | # shellcheck disable=SC2155 3 | # This script is part of the systemd Vagrant test suite for CentOS CI and 4 | # it's expected to be executed in a Vagrant VM configured by vagrant-build.sh 5 | # script. 6 | # Majority of this script is copied from the systemd-centos-ci/agent/testsuite.sh 7 | # script with some modifications to support other distributions. Test dependencies 8 | # for each distribution must be installed prior executing this script. 9 | 10 | DISTRO="${1:-unspecified}" 11 | SCRIPT_DIR="$(dirname "$0")" 12 | # This variable is automagically consumed by the "framework" for integration tests 13 | # See respective bootstrap script under vagrant/bootstrap_scripts/ for reasoning 14 | export BUILD_DIR="${BUILD_DIR:-/systemd-meson-build}" 15 | 16 | # Following scripts are copied from the systemd-centos-ci/common directory 17 | # by vagrant-build.sh 18 | # shellcheck source=common/task-control.sh 19 | . "$SCRIPT_DIR/task-control.sh" "vagrant-$DISTRO-testsuite" || exit 1 20 | # shellcheck source=common/utils.sh 21 | . "$SCRIPT_DIR/utils.sh" || exit 1 22 | 23 | at_exit() { 24 | set +e 25 | exectask "journalctl-testsuite" "journalctl -b -o short-monotonic --no-hostname --no-pager" 26 | } 27 | 28 | trap at_exit EXIT 29 | 30 | bootctl status 31 | 32 | # Enable systemd-coredump 33 | if ! coredumpctl_init; then 34 | echo >&2 "Failed to configure systemd-coredump/coredumpctl" 35 | exit 1 36 | fi 37 | 38 | # Collect any coredumps that happened during boot 39 | if ! exectask "coredumpctl_collect_boot" "coredumpctl_collect"; then 40 | echo >&2 "Detected coredump(s) during system bootup" 41 | exit 1 42 | fi 43 | 44 | # Disable swap, since it seems to cause CPU soft lock-ups in some cases 45 | swapoff -av 46 | swapon --show 47 | 48 | pushd /build || { echo >&2 "Can't pushd to /build"; exit 1; } 49 | 50 | # Run the internal unit tests (make check) 51 | exectask "ninja-test" "meson test -C $BUILD_DIR --print-errorlogs --timeout-multiplier=3" 52 | [[ -d "$BUILD_DIR/meson-logs" ]] && rsync -amq --include '*.txt' --include '*/' --exclude '*' "$BUILD_DIR/meson-logs" "$LOGDIR" 53 | 54 | # Ignore any coredumps generated by unit tests, as there's a lot of intentional crashes 55 | coredumpctl_set_ts 56 | 57 | ## Integration test suite ## 58 | # Prepare a custom-tailored initrd image (with the systemd module included). 59 | # This is necessary, as the default mkinitcpio config includes only the udev module, 60 | # which breaks certain things, like setting global env variables for systemd from 61 | # the kernel command line. 62 | # The exported INITRD variable is picked up by all following integration tests 63 | export INITRD="$(mktemp /var/tmp/initrd-testsuite-XXX.img)" 64 | if ! mkinitcpio -c /dev/null -A base,systemd,sd-encrypt,modconf,block,filesystems,keyboard,fsck -g "$INITRD"; then 65 | echo >&2 "Failed to generate initrd, can't continue" 66 | exit 1 67 | fi 68 | 69 | # Don't strip systemd binaries installed into test images, so we can get nice 70 | # stack traces when something crashes 71 | export STRIP_BINARIES=no 72 | 73 | # Initialize the 'base' image (default.img) on which the other images are based 74 | exectask "setup-the-base-image" "make -C test/TEST-01-BASIC clean setup TESTDIR=/var/tmp/systemd-test-TEST-01-BASIC" 75 | 76 | # Parallelized tasks 77 | EXECUTED_LIST=() 78 | FLAKE_LIST=( 79 | "test/TEST-16-EXTEND-TIMEOUT" # flaky test 80 | "test/TEST-25-IMPORT" # flaky when paralellized (systemd/systemd#13973) 81 | ) 82 | SKIP_LIST=( 83 | "test/TEST-61-UNITTESTS-QEMU" # redundant test, runs the same tests as TEST-02, but only QEMU (systemd/systemd#19969) 84 | "${FLAKE_LIST[@]}" 85 | ) 86 | 87 | ## Other integration tests ## 88 | # Enqueue the "other" tests first. The networkd testsuite has quite a long 89 | # runtime without requiring too much resources, hence it can run in parallel 90 | # with the "standard" integration tests, saving ~30 minutes ATTOW 91 | TEST_LIST=( 92 | "test/test-network/systemd-networkd-tests.py" 93 | ) 94 | 95 | # Prepare environment for the systemd-networkd testsuite 96 | systemctl disable --now dhcpcd dnsmasq 97 | systemctl reload dbus.service 98 | 99 | for t in "${TEST_LIST[@]}"; do 100 | exectask_p "${t##*/}" "/bin/time -v -- timeout -k 60s 60m ./$t" 101 | done 102 | 103 | # Shared test env variables 104 | # 105 | # Set timeouts for QEMU and nspawn tests to kill them in case they get stuck 106 | export QEMU_TIMEOUT=900 107 | export NSPAWN_TIMEOUT=900 108 | # Enforce nested KVM 109 | export TEST_NESTED_KVM=1 110 | 111 | for t in test/TEST-??-*; do 112 | if [[ ${#SKIP_LIST[@]} -ne 0 ]] && in_set "$t" "${SKIP_LIST[@]}"; then 113 | echo -e "[SKIP] Skipping test $t\n" 114 | continue 115 | fi 116 | 117 | ## Configure test environment 118 | # Tell the test framework to copy the base image for each test, so we 119 | # can run them in parallel 120 | export TEST_PARALLELIZE=1 121 | # Set the test dir to something predictable so we can refer to it later 122 | export TESTDIR="/var/tmp/systemd-test-${t##*/}" 123 | # Set QEMU_SMP appropriately (regarding the parallelism) 124 | # OPTIMAL_QEMU_SMP is part of the common/task-control.sh file 125 | export QEMU_SMP=$OPTIMAL_QEMU_SMP 126 | # Use a "unique" name for each nspawn container to prevent scope clash 127 | export NSPAWN_ARGUMENTS="--machine=${t##*/}" 128 | 129 | # Skipped test don't create the $TESTDIR automatically, so do it explicitly 130 | # otherwise the `touch` command would fail 131 | mkdir -p "$TESTDIR" 132 | rm -f "$TESTDIR/pass" 133 | 134 | exectask_p "${t##*/}" "/bin/time -v -- make -C $t setup run && touch $TESTDIR/pass" 135 | EXECUTED_LIST+=("$t") 136 | done 137 | 138 | # Wait for remaining running tasks 139 | exectask_p_finish 140 | 141 | for t in "${FLAKE_LIST[@]}"; do 142 | ## Configure test environment 143 | # Set the test dir to something predictable so we can refer to it later 144 | export TESTDIR="/var/tmp/systemd-test-${t##*/}" 145 | # Set QEMU_SMP appropriately (regarding the parallelism) 146 | # OPTIMAL_QEMU_SMP is part of the common/task-control.sh file 147 | export QEMU_SMP=$(nproc) 148 | 149 | # Suffix the $TESTDIR of each retry with an index to tell them apart 150 | export MANGLE_TESTDIR=1 151 | exectask_retry "${t##*/}" "/bin/time -v -- make -C $t setup run && touch \$TESTDIR/pass" 152 | 153 | # Retried tasks are suffixed with an index, so update the $EXECUTED_LIST 154 | # array accordingly to correctly find the respective journals 155 | for ((i = 1; i <= TASK_RETRY_DEFAULT; i++)); do 156 | [[ -d "/var/tmp/systemd-test-${t##*/}_${i}" ]] && EXECUTED_LIST+=("${t}_${i}") 157 | done 158 | done 159 | 160 | # Save journals created by integration tests 161 | for t in "${EXECUTED_LIST[@]}"; do 162 | testdir="/var/tmp/systemd-test-${t##*/}" 163 | if [[ -f "$testdir/system.journal" ]]; then 164 | # Filter out test-specific coredumps which are usually intentional 165 | # Note: $COREDUMPCTL_EXCLUDE_MAP resides in common/utils.sh 166 | if [[ -v "COREDUMPCTL_EXCLUDE_MAP[$t]" ]]; then 167 | export COREDUMPCTL_EXCLUDE_RX="${COREDUMPCTL_EXCLUDE_MAP[$t]}" 168 | fi 169 | # Attempt to collect coredumps from test-specific journals as well 170 | exectask "${t##*/}_coredumpctl_collect" "coredumpctl_collect '$testdir/'" 171 | # Make sure to not propagate the custom coredumpctl filter override 172 | [[ -v COREDUMPCTL_EXCLUDE_RX ]] && unset -v COREDUMPCTL_EXCLUDE_RX 173 | 174 | # Keep the journal files only if the associated test case failed 175 | if [[ ! -f "$testdir/pass" ]]; then 176 | rsync -aq "$testdir/system.journal" "$LOGDIR/${t##*/}/" 177 | fi 178 | fi 179 | 180 | # Clean the no longer necessary test artifacts 181 | [[ -d "$t" ]] && make -C "$t" clean-again > /dev/null 182 | done 183 | 184 | # Collect coredumps using the coredumpctl utility, if any 185 | exectask "coredumpctl_collect" "coredumpctl_collect" 186 | 187 | # Summary 188 | show_task_summary 189 | 190 | exectask "journalctl-testsuite" "journalctl -b -o short-monotonic --no-hostname --no-pager" 191 | 192 | finish_and_exit 193 | -------------------------------------------------------------------------------- /vagrant/boxes/archlinux_systemd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eux 3 | set -o pipefail 4 | 5 | whoami 6 | 7 | # We should have a working TPM 2.0 device 8 | stat /dev/tpm0 9 | [[ "$(>/etc/pacman.conf <>/etc/ssh/sshd_config 64 | 65 | # Create /etc/localtime 66 | systemd-firstboot --timezone=UTC 67 | 68 | # Change the default locale from C.UTF-8 69 | localectl set-locale en_US.UTF-8 70 | 71 | # Enable GDM by default (to catch certain systemd-logind related issues) 72 | systemctl enable gdm.service 73 | systemctl set-default graphical.target 74 | systemctl get-default 75 | 76 | cd /tmp 77 | 78 | # Since makepkg won't run under root, we need to create a separate use and give it 79 | # access to paswordless sudo 80 | useradd --create-home builder 81 | mkdir -p /etc/sudoers.d 82 | echo "builder ALL=(ALL) NOPASSWD: ALL" >/etc/sudoers.d/builder 83 | 84 | # Compile & install tgt (iSCSI target utils) 85 | pacman --needed --noconfirm -S docbook-xsl libxslt perl-config-general 86 | git clone --depth=1 https://github.com/fujita/tgt 87 | pushd tgt 88 | make sbindir=/usr/bin ISCSI=1 89 | make sbindir=/usr/bin install 90 | install -Dm644 scripts/tgtd.service /usr/lib/systemd/system/tgtd.service 91 | # Workaround for a race condition when starting tgtd 92 | # See: 93 | # * https://bugzilla.redhat.com/show_bug.cgi?id=848942 94 | # * https://src.fedoraproject.org/rpms/scsi-target-utils/c/3a25fe7a57200b61ecebcec0d867671597080196?branch=rawhide 95 | # * https://src.fedoraproject.org/rpms/scsi-target-utils/c/5de1dd10b8804e555c1c010c1af52a4415155971?branch=rawhide 96 | sed -i '/^ExecStart=\/usr\/sbin\/tgtd/aExecStartPost=\/bin\/sleep 5' /usr/lib/systemd/system/tgtd.service 97 | popd 98 | rm -fr tgt 99 | tgtadm --version 100 | 101 | # Compile & install netlabel_tools 102 | pacman --needed --noconfirm -S autoconf automake gcc libtool make pkg-config 103 | git clone --depth=1 https://github.com/netlabel/netlabel_tools 104 | pushd netlabel_tools 105 | ./autogen.sh 106 | ./configure --prefix=/usr 107 | make -j 108 | make install 109 | popd 110 | rm -fr netlabel_tools 111 | netlabelctl --help 112 | 113 | # Compile & install radvd 114 | # FIXME: drop once [0] is released & lands in Arch Linux 115 | # [0] https://github.com/radvd-project/radvd/pull/141 116 | pacman --needed --noconfirm -S autoconf automake byacc flex gcc libbsd libtool make pkg-config 117 | git clone --depth=1 https://github.com/radvd-project/radvd 118 | pushd radvd 119 | ./autogen.sh 120 | ./configure --prefix=/usr --sysconfdir=/etc --mandir=/usr/share/man 121 | make -j 122 | make install 123 | popd 124 | rm -fr radvd 125 | # radvd returns 1 for both --version and --help, so we need to be a bit more creative here 126 | (radvd --version || :) |& grep "^Version" 127 | 128 | # Compile & install dfuzzer 129 | pacman --needed --noconfirm -S docbook-xsl gcc glib2 libxslt meson pkg-config 130 | git clone --depth=1 https://github.com/dbus-fuzzer/dfuzzer 131 | pushd dfuzzer 132 | meson build 133 | ninja -C build install 134 | popd 135 | rm -fr dfuzzer 136 | dfuzzer --version 137 | 138 | # Remove the makepkg user 139 | rm /etc/sudoers.d/builder 140 | userdel -fr builder 141 | 142 | # Replace systemd-networkd with dhcpcd as the network manager for the default 143 | # interface, so we can run the systemd-networkd test suite without having 144 | # to worry about the network staying up 145 | # 146 | # Tell systemd-networkd to ignore eth0 netdev, so we can keep it up 147 | # during the systemd-networkd testsuite 148 | cat >/etc/systemd/network/10-eth0.network <"$BOOT/loader/entries/arch.conf" <&2 "Failed to configure systemd-coredump/coredumpctl" 47 | exit 1 48 | fi 49 | 50 | # Collect any coredumps that happened during boot 51 | if ! exectask "coredumpctl_collect_boot" "coredumpctl_collect"; then 52 | echo >&2 "Detected coredump(s) during system bootup" 53 | exit 1 54 | fi 55 | fi 56 | 57 | set +e 58 | 59 | ### TEST PHASE ### 60 | pushd systemd || { echo >&2 "Can't pushd to systemd"; exit 1; } 61 | 62 | # FIXME: test-journal-flush 63 | # A particularly ugly workaround for the flaky test-journal-flush. As the issue 64 | # presented so far only in the QEMU TEST-24, let's skip it just there, instead 65 | # of disabling it completely (even in the `meson test`). 66 | # 67 | # See: systemd/systemd#17963 68 | # shellcheck disable=SC2016 69 | sed -i '/TEST_LIST=/aTEST_LIST=("${TEST_LIST[@]/\\/usr\\/lib\\/systemd\\/tests\\/test-journal-flush}")' test/TEST-24-UNIT-TESTS/testsuite.sh 70 | 71 | # FIXME: test-barrier 72 | # This test is flaky on systems under load, which happens intermittently due 73 | # to how meson runs the tests (in parallel). 74 | # 75 | # See: 76 | # https://github.com/systemd/systemd/commit/fd23f9c9a70e1214507641d327da40d1688b74d7 77 | # https://github.com/systemd/systemd/commit/a1e3f0f38b43e68ff9ea33ab1935aed4edf6ed7f 78 | echo 'int main(void) { return 77; }' >src/test/test-barrier.c 79 | 80 | # FIXME (not really): test-execute 81 | # This test is not compatible with newer packages on RHEL 9 and backporting all the necessary stuff would be a 82 | # major PITA. Since we run it in other CIs (both externally and internally), let's just not bother here. 83 | echo 'int main(void) { return 77; }' >src/test/test-execute.c 84 | 85 | # FIXME (also not really): test-fs-util with newer kernel 86 | # This test fails with newer kernels, because we're missing a couple of patches in RHEL 8. Since there's no 87 | # CentOS 8 (nor CentOS Stream 8), we run the tests on C9S and hit this issue. It's very unlikely we'll backport 88 | # the necessary patches to RHEL 8, so drop the failing test for now. 89 | # 90 | # See: 91 | # - https://github.com/redhat-plumbers/systemd-rhel8/issues/434 92 | sed -i '/a = strjoina(p, "\/lnk");/,/timespec_load/d' src/test/test-fs-util.c 93 | 94 | # Run the internal unit tests (make check) 95 | exectask "ninja-test" "meson test -C build --print-errorlogs --timeout-multiplier=3" 96 | # Copy over meson test artifacts 97 | [[ -d "build/meson-logs" ]] && rsync -amq --include '*.txt' --include '*/' --exclude '*' "build/meson-logs" "$LOGDIR" 98 | 99 | # Ignore any coredumps generated by unit tests, as there's a lot of intentional crashes 100 | coredumpctl_set_ts 101 | 102 | ## Integration test suite ## 103 | CHECK_LIST=() 104 | SKIP_LIST=( 105 | "test/TEST-16-EXTEND-TIMEOUT" # flaky test 106 | ) 107 | 108 | if [[ "$CGROUP_HIERARCHY" == "legacy" ]]; then 109 | # This test explicitly requires unified cgroup hierarchy 110 | SKIP_LIST+=("test/TEST-19-DELEGATE") 111 | fi 112 | 113 | centos_ensure_qemu_symlink 114 | 115 | ## Generate a custom-tailored initrd for the integration tests 116 | # The host initrd contains multipath modules & services which are unused 117 | # in the integration tests and sometimes cause unexpected failures. Let's build 118 | # a custom initrd used solely by the integration tests 119 | # 120 | # Set a path to the custom initrd into the INITRD variable which is read by 121 | # the integration test suite "framework" 122 | export INITRD="/var/tmp/ci-initramfs-$(uname -r).img" 123 | # Copy over the original initrd, as we want to keep the custom installed 124 | # files we installed during the bootstrap phase (i.e. we want to keep the 125 | # command line arguments the original initrd was built with) 126 | cp -fv "/boot/initramfs-$(uname -r).img" "$INITRD" 127 | # Rebuild the original initrd without the multipath module 128 | dracut -o "multipath rngd dbus-broker" -a dbus-daemon --filesystems ext4 --rebuild "$INITRD" 129 | 130 | for t in test/TEST-??-*; do 131 | if [[ ${#SKIP_LIST[@]} -ne 0 ]] && in_set "$t" "${SKIP_LIST[@]}"; then 132 | echo -e "[SKIP] Skipping test $t\n" 133 | continue 134 | fi 135 | 136 | ## Configure test environment 137 | # Explicitly set paths to initramfs and kernel images (for QEMU tests) 138 | # See $INITRD above 139 | export KERNEL_BIN="/boot/vmlinuz-$(uname -r)" 140 | export KERNEL_APPEND="enforcing=0 $CGROUP_KERNEL_ARGS" 141 | # Set timeouts for QEMU and nspawn tests to kill them in case they get stuck 142 | export QEMU_TIMEOUT=1200 143 | export NSPAWN_TIMEOUT=600 144 | # Set the test dir to something predictable so we can refer to it later 145 | export TESTDIR="/var/tmp/systemd-test-${t##*/}" 146 | # Set QEMU_SMP appropriately (regarding the parallelism) 147 | # OPTIMAL_QEMU_SMP is part of the common/task-control.sh file 148 | export QEMU_SMP=$OPTIMAL_QEMU_SMP 149 | # Work around 'Fatal glibc error: CPU does not support x86-64-v2' 150 | # See: 151 | # - https://bugzilla.redhat.com/show_bug.cgi?id=2060839 152 | # - https://access.redhat.com/solutions/6833751 153 | export QEMU_OPTIONS="-cpu Nehalem" 154 | # Several syscalls we need are not whitelisted by default in RHEL 8's systemd, but systemd knows 155 | # about them, so let's tell it to whitelist all known syscalls 156 | export NSPAWN_ARGUMENTS="--system-call-filter=@known" 157 | 158 | # Suffix the $TESTDIR of each retry with an index to tell them apart 159 | export MANGLE_TESTDIR=1 160 | # FIXME: retry each task again if it fails (i.e. run each task twice at most) 161 | # to work around intermittent QEMU soft lockups/ACPI timer errors 162 | # 163 | # Suffix the $TESTDIR of each retry with an index to tell them apart 164 | exectask_retry_p "${t##*/}" "/bin/time -v -- make -C $t setup run && touch \$TESTDIR/pass; rm -fv \$TESTDIR/*.img; test -e \$TESTDIR/pass" 165 | # Retried tasks are suffixed with an index, so update the $CHECK_LIST 166 | # array with all possible task names correctly find the respective journals 167 | # shellcheck disable=SC2207 168 | CHECK_LIST+=($(seq -f "${t}_%g" 1 "$TASK_RETRY_DEFAULT")) 169 | done 170 | 171 | # Wait for remaining running tasks 172 | exectask_p_finish 173 | 174 | # Save journals created by integration tests 175 | for t in "${CHECK_LIST[@]}"; do 176 | testdir="/var/tmp/systemd-test-${t##*/}" 177 | if [[ -d "$testdir/journal" ]]; then 178 | if [[ $COLLECT_COREDUMPS -ne 0 ]]; then 179 | # Attempt to collect coredumps from test-specific journals as well 180 | exectask "${t##*/}_coredumpctl_collect" "coredumpctl_collect '$testdir/journal'" 181 | fi 182 | # Keep the journal files only if the associated test case failed 183 | if [[ ! -f "$testdir/pass" ]]; then 184 | rsync -aq "$testdir/journal" "$LOGDIR/${t##*/}" 185 | fi 186 | fi 187 | done 188 | 189 | ## Other integration tests ## 190 | TEST_LIST=( 191 | "test/test-exec-deserialization.py" 192 | # "test/test-network/systemd-networkd-tests.py" 193 | ) 194 | 195 | for t in "${TEST_LIST[@]}"; do 196 | exectask "${t##*/}" "/bin/time -v -- ./$t" 197 | done 198 | 199 | if [[ $COLLECT_COREDUMPS -ne 0 ]]; then 200 | # Collect coredumps using the coredumpctl utility, if any 201 | exectask "coredumpctl_collect" "coredumpctl_collect" 202 | fi 203 | 204 | # Summary 205 | show_task_summary 206 | 207 | finish_and_exit 208 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # About 2 | 3 | This project contains code to build, install and test [systemd](https://github.com/systemd/systemd/) 4 | on test machines provisioned from a pool provided by the [CentOS CI](https://wiki.centos.org/QaWiki/CI) project, 5 | using their [Duffy](https://wiki.centos.org/QaWiki/CI/Duffy) API. 6 | 7 | As of right now these scripts provide CI for [upstream](https://github.com/systemd/systemd) 8 | on CentOS Stream 8 and Arch Linux, and for [RHEL downstream](https://github.com/redhat-plumbers) 9 | on supported CentOS releases. 10 | 11 | # Structure 12 | 13 | The main entrypoint is the `agent-control.py` script, which is responsible for 14 | getting a provisioned machine from the Duffy pool, run the respective CI scripts, 15 | and gather results. To make this work with GitHub pull requests, the `jenkins/` 16 | directory contains *glue* scripts which generate a correct set of arguments for 17 | the `agent-control.py` script for each PR. 18 | 19 | The CI scripts are scattered among several directories: 20 | 21 | * `agent/` 22 | 23 | Bootstrap and test runner scripts used to run tests directly on the provisioned 24 | machines (i.e. for running tests on CentOS (Stream) X). 25 | 26 | * `common/` 27 | 28 | Various *libraries* and support functions used by other scripts. 29 | 30 | * `jenkins/` 31 | 32 | Scripts which are responsible for feeding `agent-control.py` correct parameters 33 | based on data from the Jenkins GitHub plugin. 34 | 35 | These scripts are used directly in the *Execute shell* step of each Jenkins job 36 | in following manner: 37 | 38 | ```bash 39 | #!/bin/sh 40 | 41 | set -e 42 | 43 | curl -q -o runner.sh https://raw.githubusercontent.com/systemd/systemd-centos-ci/main/jenkins/runners/upstream-centos8.sh 44 | chmod +x runner.sh 45 | ./runner.sh 46 | ``` 47 | 48 | * `utils/` 49 | 50 | Various utility scripts used by the pipeline. 51 | 52 | * `vagrant/` 53 | 54 | Setup, build, and test runner scripts to test systemd on other distributions than CentOS 55 | using Vagrant VMs. 56 | 57 | # Pipelines 58 | 59 | Jenkins instance: https://jenkins-systemd.apps.ocp.cloud.ci.centos.org/ 60 | 61 | ## Upstream on CentOS Stream 8 (upstream-centos8) 62 | 63 | ``` 64 | agent-control.py +-> agent/bootstrap.sh +-> reboot +-> agent/testsuite.sh 65 | ``` 66 | 67 | To test compatibility of the upstream systemd with older kernels, this job builds, installs, and 68 | tests an upstream PR on a CentOS Stream 8 baremetal machine. 69 | 70 | To achieve this, `agent-control.py` runs `agent/bootstrap.sh` script to fetch, build, and install 71 | the respective PR (along with other necessary dependencies), reboots the machine, and executes 72 | `agent/testsuite.sh` to to the actual testing. 73 | 74 | ## Downstream (RHEL) on CentOS 7 and CentOS Stream 8/9 (rhelX-centosX) 75 | 76 | The same worklflow as above, but for systemd in RHEL: 77 | 78 | * [RHEL 7/CentOS 7](https://github.com/redhat-plumbers/systemd-rhel7) 79 | 80 | ``` 81 | agent-control.py +-> agent/bootstrap-rhel7.sh +-> reboot +-> agent/testsuite-rhel7.sh 82 | ``` 83 | 84 | * [RHEL 8/CentOS Stream 8](https://github.com/redhat-plumbers/systemd-rhel8) 85 | 86 | ``` 87 | agent-control.py +-> agent/bootstrap-rhel8.sh +-> reboot +-> agent/testsuite-rhel8.sh 88 | ``` 89 | 90 | * [RHEL 9/CentOS Stream 9](https://github.com/redhat-plumbers/systemd-rhel9) 91 | 92 | ``` 93 | agent-control.py +-> agent/bootstrap-rhel9.sh +-> reboot +-> agent/testsuite-rhel9.sh 94 | ``` 95 | 96 | ## Upstream on Arch Linux using Vagrant (upstream-vagrant-archlinux) 97 | 98 | To achieve the exact opposite of the testing on CentOS Stream 8, this pipeline check the compatibility 99 | of systemd with the latest versions of kernel and other components. As the CentOS CI 100 | pool provides only CentOS machines, this pipeline introduces an intermediary in form of 101 | a [Vagrant](https://www.vagrantup.com) VM along with [vagrant-libvirt](https://github.com/vagrant-libvirt/vagrant-libvirt) 102 | plugin to spin up a virtual machine in which we do the actual testing. 103 | 104 | ### Structure 105 | 106 | Each VM consists of two images: the base image (Vagrant Box) and a runtime image. 107 | 108 | Vagrant Box templates are stored under `vagrant/boxes/` and the images themselves 109 | are rebuilt every few days by a separate job to keep the package set up-to-date, 110 | but to not slow down the CI pipeline. 111 | 112 | The templates for each VM used in the CI can be found in `vagrant/vagrantfiles/`. 113 | As the majority of the current (single) Vagrantfile remains identical across several 114 | instances of the VMs except for the bootstrap phase, the bootstrap scripts were 115 | separated into their own directory - `vagrant/bootstrap_scripts/` and the Vagrantfile 116 | itself is now parametrized, to avoid code duplication. 117 | 118 | The test scripts are in the `vagrant/test_scripts/` directory, and follow the 119 | same naming convention as the bootstrap scripts above, except for being prefixed 120 | with `test-`. 121 | 122 | ### Pipeline 123 | 124 | ``` 125 | agent-control.py +-> vagrant/vagrant-ci-wrapper.sh +-> vagrant/vagrant-build.sh +-> [reboot] +-> vagrant/test_scripts/test--.sh 126 | + ^ + ^ 127 | | | | | 128 | v + v + 129 | vagrant/vagrant-setup.sh vagrant/vagrantfiles/Vagrantfile_ 130 | + ^ 131 | | | 132 | v + 133 | vagrant/bootstrap-scripts/-.sh 134 | 135 | ``` 136 | 137 | The pipeline consists of several steps (scripts): 138 | 139 | 1. `vagrant/vagrant-ci-wrapper.sh` 140 | 141 | This script acts as a bootstrap where it checks out the systemd repository 142 | on a revision based on information passed down by the respective Jenkins 143 | glue script, configures the underlying system for Vagrant (by calling 144 | `vagrant/vagrant-setup.sh`) and the test suite, and starts the VM build process 145 | by executing `vagrant/vagrant-build.sh` 146 | 147 | 2. `vagrant/vagrant-build.sh` 148 | 149 | Builds a *runtime* VM image based on the base one (Vagrant Box). The Vagrantfile 150 | for each VM (`vagrant/vagrantfiles/`) can reference an external bootstrap script 151 | (`vagrant/bootstrap_scripts/`) making itself reusable for multiple different 152 | scenarios. 153 | 154 | When the VM is built, the local systemd repo is mounted into it over NFS 155 | and the bootstrap script is executed, which builds the checked out revision 156 | and makes other necessary changes to the VM itself. 157 | 158 | After this step the VM is rebooted (*reloaded* in the Vagrant terms) and the 159 | test runner script (`vagrant/test_scripts/*.sh`) is executed inside. 160 | 161 | ## Upstream on Arch Linux with sanitizers using Vagrant (upstream-vagrant-archlinux-sanitizers) 162 | 163 | To tackle the question of security a little bit, this job does *almost* the same thing 164 | as the one above, but here systemd is compiled with [Address Sanitizer](https://github.com/google/sanitizers/wiki/AddressSanitizer) 165 | and [Undefined Behavior Sanitizer](https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html). In this 166 | case we skip the *reboot* step of the pipeline, since we don't install the compiled 167 | systemd revision anyway. 168 | 169 | ## (Auxiliary) Rebuild the Vagrant images (vagrant-make-cache) 170 | 171 | ``` 172 | agent-control.py +-> vagrant/vagrant-make-cache.sh 173 | ``` 174 | 175 | To keep the images up-to-date but to not slow every CI pipeline down while doing so, 176 | this job's sole purpose is to rebuild the base images (Vagrant Boxes) every few days 177 | (based on Jenkins cron) and upload it to the artifacts server, where it can be 178 | used by the respective Vagrantfile. 179 | 180 | ## (Auxiliary) Mirror the Copr repo with CentOS (Stream) dependencies (reposync) 181 | 182 | ``` 183 | utils/reposync.sh 184 | ``` 185 | 186 | We need a couple of newer dependencies for systemd on CentOS (Stream) to cover the most 187 | recent features. For this we have a bunch of Copr repositories [0][1] with necessary packages. 188 | 189 | [0] https://copr.fedorainfracloud.org/coprs/mrc0mmand/systemd-centos-ci-centos8/ \ 190 | [1] https://copr.fedorainfracloud.org/coprs/mrc0mmand/systemd-centos-ci-centos9/ 191 | 192 | However, we found out that the infrastructure Copr is running in is quite unreliable 193 | so a fair share of our jobs was failing just because they couldn't install dependencies. 194 | Having a local mirror in the CentOS CI infrastructure definitely helps in this case and 195 | the purpose of this script is to update it every few hours (again, using Jenkins cron). 196 | 197 | ## (Auxiliary) CI for the CI repository (ci-build) 198 | 199 | To make sure changes to this repository don't break the CI pipeline, this job 200 | provides a way to run a specific revision of the CI scripts (from a PR) against 201 | the main branch of the respective systemd repository. 202 | 203 | As this requires twiddling around with various bits and knobs given which part 204 | of the CI pipeline we want to test (instead of just running everything and pointlessly 205 | waiting many hours until it finishes), the job configuration is stored in the Jenkins 206 | job itself and thus requires access to the instance. 207 | 208 | -------------------------------------------------------------------------------- /agent/testsuite.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | # shellcheck disable=SC2155 3 | 4 | LIB_ROOT="$(dirname "$0")/../common" 5 | # shellcheck source=common/task-control.sh 6 | . "$LIB_ROOT/task-control.sh" "testsuite-logs-$(uname -m)" || exit 1 7 | # shellcheck source=common/utils.sh 8 | . "$LIB_ROOT/utils.sh" || exit 1 9 | 10 | export BUILD_DIR="${BUILD_DIR:-/systemd-meson-build}" 11 | 12 | # EXIT signal handler 13 | at_exit() { 14 | set +e 15 | exectask "journalctl-testsuite" "journalctl -b -o short-monotonic --no-hostname --no-pager" 16 | } 17 | 18 | set -eu 19 | set -o pipefail 20 | 21 | trap at_exit EXIT 22 | 23 | while getopts "ns" opt; do 24 | case "$opt" in 25 | n) 26 | echo "[NOTICE] Running only nspawn-based tests" 27 | export TEST_NO_QEMU=1 28 | ;; 29 | s) 30 | SKIP_MAN_CHECK=1 31 | ;; 32 | ?) 33 | exit 1 34 | ;; 35 | *) 36 | echo "Usage: $0 [-n] [-s]" 37 | exit 1 38 | esac 39 | done 40 | 41 | ### TEST PHASE ### 42 | # Enable systemd-coredump 43 | if ! coredumpctl_init; then 44 | echo >&2 "Failed to configure systemd-coredump/coredumpctl" 45 | exit 1 46 | fi 47 | 48 | # Collect any coredumps that happened during boot 49 | if ! exectask "coredumpctl_collect_boot" "coredumpctl_collect"; then 50 | echo >&2 "Detected coredump(s) during system bootup" 51 | exit 1 52 | fi 53 | 54 | centos_ensure_qemu_symlink 55 | 56 | set +e 57 | 58 | ### TEST PHASE ### 59 | pushd systemd || { echo >&2 "Can't pushd to systemd"; exit 1; } 60 | 61 | # Run the internal unit tests (make check) 62 | exectask "ninja-test" "meson test -C $BUILD_DIR --print-errorlogs --timeout-multiplier=3" 63 | # Copy over meson test artifacts 64 | [[ -d "$BUILD_DIR/meson-logs" ]] && rsync -amq --include '*.txt' --include '*/' --exclude '*' "$BUILD_DIR/meson-logs" "$LOGDIR" 65 | 66 | if [[ "${SKIP_MAN_CHECK:-0}" -eq 0 ]]; then 67 | # If we're not testing the main branch (the first diff) check if the tested 68 | # branch doesn't contain only man-related changes. If so, skip the integration 69 | # tests 70 | MAIN_BRANCH="$(git rev-parse --abbrev-ref origin/HEAD)" 71 | if ! git diff --quiet "$MAIN_BRANCH" HEAD && ! git diff "$(git merge-base "$MAIN_BRANCH" HEAD)" --name-only | grep -vE "^man/" >/dev/null; then 72 | echo "Detected man-only PR, skipping integration tests" 73 | finish_and_exit 74 | fi 75 | fi 76 | 77 | # Ignore any coredumps generated by unit tests, as there's a lot of intentional crashes 78 | coredumpctl_set_ts 79 | 80 | ## Integration test suite ## 81 | CHECK_LIST=() 82 | FLAKE_LIST=( 83 | "test/TEST-16-EXTEND-TIMEOUT" # flaky test, see below 84 | "test/TEST-63-PATH" # flaky when the AWS region is under heavy load 85 | ) 86 | SKIP_LIST=( 87 | "test/TEST-61-UNITTESTS-QEMU" # redundant test, runs the same tests as TEST-02, but only QEMU (systemd/systemd#19969) 88 | "${FLAKE_LIST[@]}" 89 | ) 90 | 91 | ## Generate a custom-tailored initrd for the integration tests 92 | # The host initrd contains multipath modules & services which are unused 93 | # in the integration tests and sometimes cause unexpected failures. Let's build 94 | # a custom initrd used solely by the integration tests 95 | # 96 | # Set a path to the custom initrd into the INITRD variable which is read by 97 | # the integration test suite "framework" 98 | export INITRD="/var/tmp/ci-initramfs-$(uname -r).img" 99 | DRACUT_OPTS=() 100 | [[ -x /usr/lib/systemd/systemd-executor ]] && DRACUT_OPTS+=(--install /usr/lib/systemd/systemd-executor) 101 | # Copy over the original initrd, as we want to keep the custom installed 102 | # files we installed during the bootstrap phase (i.e. we want to keep the 103 | # command line arguments the original initrd was built with) 104 | cp -fv "/boot/initramfs-$(uname -r).img" "$INITRD" 105 | # Rebuild the original initrd with the dm-crypt modules and without the multipath module 106 | dracut "${DRACUT_OPTS[@]}" -a crypt -o "multipath rngd nfs" --filesystems ext4 --rebuild "$INITRD" 107 | # Don't strip systemd binaries installed into test images, so we can get nice 108 | # stack traces when something crashes 109 | export STRIP_BINARIES=no 110 | 111 | # Initialize the 'base' image (default.img) on which the other images are based 112 | exectask "setup-the-base-image" "make -C test/TEST-01-BASIC clean setup TESTDIR=/var/tmp/systemd-test-TEST-01-BASIC" 113 | 114 | ## Other integration tests ## 115 | # Enqueue the "other" tests first. The networkd testsuite has quite a long 116 | # runtime without requiring too much resources, hence it can run in parallel 117 | # with the "standard" integration tests, saving ~30 minutes ATTOW (this excludes 118 | # dusty nodes, where any kind of parallelism leads to unstable tests) 119 | TEST_LIST=( 120 | "test/test-network/systemd-networkd-tests.py" 121 | ) 122 | 123 | for t in "${TEST_LIST[@]}"; do 124 | exectask_p "${t##*/}" "/bin/time -v -- timeout -k 60s 60m ./$t" 125 | done 126 | 127 | # Shared test env variables 128 | export KERNEL_APPEND="enforcing=0 watchdog_thresh=60 workqueue.watchdog_thresh=120" 129 | # Explicitly set paths to initramfs and kernel images (for QEMU tests) 130 | # See $INITRD above 131 | export KERNEL_BIN="/boot/vmlinuz-$(uname -r)" 132 | # Set timeouts for QEMU and nspawn tests to kill them in case they get stuck 133 | export QEMU_TIMEOUT=1800 134 | export NSPAWN_TIMEOUT=900 135 | export QEMU_OPTIONS="-cpu max" 136 | 137 | # Let's re-shuffle the test list a bit by placing the most expensive tests 138 | # in the front, so they can run in background while we go through the rest 139 | # of the list 140 | readarray -t INTEGRATION_TESTS < <( 141 | [[ -d test/TEST-64-UDEV-STORAGE ]] && echo test/TEST-64-UDEV-STORAGE 142 | find test/ -maxdepth 1 -type d -name "TEST-??-*" ! -name "TEST-64-UDEV-STORAGE" | sort 143 | ) 144 | 145 | for t in "${INTEGRATION_TESTS[@]}"; do 146 | if [[ ${#SKIP_LIST[@]} -ne 0 ]] && in_set "$t" "${SKIP_LIST[@]}"; then 147 | echo -e "[SKIP] Skipping test $t\n" 148 | continue 149 | fi 150 | 151 | ## Configure test environment 152 | # Tell the test framework to copy the base image for each test, so we 153 | # can run them in parallel 154 | export TEST_PARALLELIZE=1 155 | # Set the test dir to something predictable so we can refer to it later 156 | export TESTDIR="/var/tmp/systemd-test-${t##*/}" 157 | # Set QEMU_SMP appropriately (regarding the parallelism) 158 | # OPTIMAL_QEMU_SMP is part of the common/task-control.sh file 159 | export QEMU_SMP=$OPTIMAL_QEMU_SMP 160 | 161 | # FIXME: retry each task again if it fails (i.e. run each task twice at most) 162 | # to work around intermittent QEMU soft lockups/ACPI timer errors 163 | # 164 | # Suffix the $TESTDIR of each retry with an index to tell them apart 165 | export MANGLE_TESTDIR=1 166 | exectask_retry_p "${t##*/}" "/bin/time -v -- make -C $t setup run && touch \$TESTDIR/pass; rm -fv \$TESTDIR/*.img; test -e \$TESTDIR/pass" 167 | # Retried tasks are suffixed with an index, so update the $CHECK_LIST 168 | # array with all possible task names correctly find the respective journals 169 | # shellcheck disable=SC2207 170 | CHECK_LIST+=($(seq -f "${t}_%g" 1 "$TASK_RETRY_DEFAULT")) 171 | done 172 | 173 | # Wait for remaining running tasks 174 | exectask_p_finish 175 | 176 | for t in "${FLAKE_LIST[@]}"; do 177 | # For older stable branches 178 | if [[ ! -d "$t" ]]; then 179 | echo "Test '$t' is not available, skipping..." 180 | continue 181 | fi 182 | 183 | ## Configure test environment 184 | # Set the test dir to something predictable so we can refer to it later 185 | export TESTDIR="/var/tmp/systemd-test-${t##*/}" 186 | # Set QEMU_SMP appropriately (regarding the parallelism) 187 | # OPTIMAL_QEMU_SMP is part of the common/task-control.sh file 188 | export QEMU_SMP=$(nproc) 189 | 190 | # Suffix the $TESTDIR of each retry with an index to tell them apart 191 | export MANGLE_TESTDIR=1 192 | exectask_retry "${t##*/}" "/bin/time -v -- make -C $t setup run && touch \$TESTDIR/pass; rm -fv \$TESTDIR/*.img; test -e \$TESTDIR/pass" 193 | 194 | # Retried tasks are suffixed with an index, so update the $CHECK_LIST 195 | # array accordingly to correctly find the respective journals 196 | for ((i = 1; i <= TASK_RETRY_DEFAULT; i++)); do 197 | [[ -d "/var/tmp/systemd-test-${t##*/}_${i}" ]] && CHECK_LIST+=("${t}_${i}") 198 | done 199 | done 200 | 201 | for t in "${CHECK_LIST[@]}"; do 202 | testdir="/var/tmp/systemd-test-${t##*/}" 203 | if [[ -f "$testdir/system.journal" ]]; then 204 | # Filter out test-specific coredumps which are usually intentional 205 | # Note: $COREDUMPCTL_EXCLUDE_MAP resides in common/utils.sh 206 | # Note2: strip the "_X" suffix added by exectask_retry*() 207 | if [[ -v "COREDUMPCTL_EXCLUDE_MAP[${t%_[0-9]}]" ]]; then 208 | export COREDUMPCTL_EXCLUDE_RX="${COREDUMPCTL_EXCLUDE_MAP[${t%_[0-9]}]}" 209 | fi 210 | # Attempt to collect coredumps from test-specific journals as well 211 | exectask "${t##*/}_coredumpctl_collect" "coredumpctl_collect '$testdir/'" 212 | # Make sure to not propagate the custom coredumpctl filter override 213 | [[ -v COREDUMPCTL_EXCLUDE_RX ]] && unset -v COREDUMPCTL_EXCLUDE_RX 214 | 215 | # Keep the journal files only if the associated test case failed 216 | if [[ ! -f "$testdir/pass" ]]; then 217 | rsync -aq "$testdir/system.journal" "$LOGDIR/${t##*/}/" 218 | fi 219 | fi 220 | 221 | # Clean the no longer necessary test artifacts 222 | [[ -d "$t" ]] && make -C "$t" clean-again > /dev/null 223 | done 224 | 225 | # Collect coredumps using the coredumpctl utility, if any 226 | exectask "coredumpctl_collect" "coredumpctl_collect" 227 | 228 | # Summary 229 | show_task_summary 230 | 231 | finish_and_exit 232 | -------------------------------------------------------------------------------- /vagrant/test_scripts/test-arch-coverage.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | # shellcheck disable=SC2155 3 | # This script is part of the systemd Vagrant test suite for CentOS CI and 4 | # it's expected to be executed in a Vagrant VM configured by vagrant-build.sh 5 | # script. 6 | # Majority of this script is copied from the systemd-centos-ci/agent/testsuite.sh 7 | # script with some modifications to support other distributions. Test dependencies 8 | # for each distribution must be installed prior executing this script. 9 | 10 | DISTRO="${1:-unspecified}" 11 | SCRIPT_DIR="$(dirname "$0")" 12 | # This variable is automagically consumed by the "framework" for integration tests 13 | # See respective bootstrap script under vagrant/bootstrap_scripts/ for reasoning 14 | export BUILD_DIR="${BUILD_DIR:-/systemd-meson-build}" 15 | # Consumed by coredumpctl_init()/coredumpctl_collect() 16 | export COREDUMPCTL_BIN="$BUILD_DIR/coredumpctl" 17 | 18 | # Following scripts are copied from the systemd-centos-ci/common directory 19 | # by vagrant-build.sh 20 | # shellcheck source=common/task-control.sh 21 | . "$SCRIPT_DIR/task-control.sh" "vagrant-$DISTRO-testsuite" || exit 1 22 | # shellcheck source=common/utils.sh 23 | . "$SCRIPT_DIR/utils.sh" || exit 1 24 | 25 | at_exit() { 26 | set +e 27 | exectask "journalctl-testsuite" "journalctl -b -o short-monotonic --no-hostname --no-pager" 28 | } 29 | 30 | trap at_exit EXIT 31 | 32 | COVERAGE_DIR="$(mktemp -d)" 33 | if ! meson_get_bool "$BUILD_DIR" "b_coverage"; then 34 | echo >&2 "systemd is not built with -Db_coverage=true, can't collect coverage" 35 | exit 1 36 | fi 37 | 38 | # Enable systemd-coredump 39 | if ! coredumpctl_init; then 40 | echo >&2 "Failed to configure systemd-coredump/coredumpctl" 41 | exit 1 42 | fi 43 | 44 | # Collect any coredumps that happened during boot 45 | if ! exectask "coredumpctl_collect_boot" "coredumpctl_collect"; then 46 | echo >&2 "Detected coredump(s) during system bootup" 47 | exit 1 48 | fi 49 | 50 | # Disable swap, since it seems to cause CPU soft lock-ups in some cases 51 | swapoff -av 52 | 53 | pushd /build || { echo >&2 "Can't pushd to /build"; exit 1; } 54 | 55 | # Tweak $BUILD_DIR's permissions, so anybody can read & write the gcov metadata 56 | setfacl --recursive --modify="d:u::rwX,d:g::rwX,d:o:rwX" --modify="u::rwX,g::rwX,o:rwX" "$BUILD_DIR" 57 | 58 | # Collect initial coverage 59 | exectask "lcov_collect-build_dir-initial" "lcov_collect $COVERAGE_DIR/build_dir_initial.coverage-info $BUILD_DIR --initial" 60 | 61 | exectask "ninja-test" "GCOV_ERROR_FILE=$LOGDIR/ninja-test-gcov-errors.log meson test -C $BUILD_DIR --print-errorlogs --timeout-multiplier=5" 62 | exectask "lcov_collect-build_dir-ninja-test" "lcov_collect $COVERAGE_DIR/unit-tests.coverage-info $BUILD_DIR" 63 | [[ -d "$BUILD_DIR/meson-logs" ]] && rsync -amq --include '*.txt' --include '*/' --exclude '*' "$BUILD_DIR/meson-logs" "$LOGDIR" 64 | 65 | # Ignore any coredumps generated by unit tests, as there's a lot of intentional crashes 66 | coredumpctl_set_ts 67 | 68 | ## Integration test suite ## 69 | # Prepare a custom-tailored initrd image (with the systemd module included). 70 | # This is necessary, as the default mkinitcpio config includes only the udev module, 71 | # which breaks certain things, like setting global env variables for systemd from 72 | # the kernel command line. 73 | # The exported INITRD variable is picked up by all following integration tests 74 | export INITRD="$(mktemp /var/tmp/initrd-testsuite-XXX.img)" 75 | if ! mkinitcpio -c /dev/null -A base,systemd,sd-encrypt,modconf,block,filesystems,keyboard,fsck -g "$INITRD"; then 76 | echo >&2 "Failed to generate initrd, can't continue" 77 | exit 1 78 | fi 79 | 80 | # Initialize the 'base' image (default.img) on which the other images are based 81 | exectask "setup-the-base-image" "make -C test/TEST-01-BASIC clean setup TESTDIR=/var/tmp/systemd-test-TEST-01-BASIC" 82 | 83 | # Parallelized tasks 84 | EXECUTED_LIST=() 85 | FLAKE_LIST=( 86 | "test/TEST-16-EXTEND-TIMEOUT" # flaky test 87 | "test/TEST-25-IMPORT" # flaky when paralellized (systemd/systemd#13973) 88 | ) 89 | SKIP_LIST=( 90 | "test/TEST-61-UNITTESTS-QEMU" # redundant test, runs the same tests as TEST-02, but only QEMU (systemd/systemd#19969) 91 | "${FLAKE_LIST[@]}" 92 | ) 93 | 94 | ## Other integration tests ## 95 | # Prepare environment for the systemd-networkd testsuite 96 | systemctl disable --now dhcpcd dnsmasq 97 | systemctl reload dbus.service 98 | 99 | exectask_p "systemd-networkd-tests.py" \ 100 | "/bin/time -v -- timeout -k 60s 60m test/test-network/systemd-networkd-tests.py --build-dir=$BUILD_DIR --debug --with-coverage" 101 | 102 | for t in test/TEST-??-*; do 103 | if [[ ${#SKIP_LIST[@]} -ne 0 ]] && in_set "$t" "${SKIP_LIST[@]}"; then 104 | echo -e "[SKIP] Skipping test $t\n" 105 | continue 106 | fi 107 | 108 | ## Configure test environment 109 | # Tell the test framework to copy the base image for each test, so we 110 | # can run them in parallel 111 | export TEST_PARALLELIZE=1 112 | # Set timeouts for QEMU and nspawn tests to kill them in case they get stuck 113 | export QEMU_TIMEOUT=900 114 | export NSPAWN_TIMEOUT=900 115 | # Set the test dir to something predictable so we can refer to it later 116 | export TESTDIR="/var/tmp/systemd-test-${t##*/}" 117 | # Set QEMU_SMP appropriately (regarding the parallelism) 118 | # OPTIMAL_QEMU_SMP is part of the common/task-control.sh file 119 | export QEMU_SMP=$OPTIMAL_QEMU_SMP 120 | # Enforce nested KVM 121 | export TEST_NESTED_KVM=1 122 | # Use a "unique" name for each nspawn container to prevent scope clash 123 | export NSPAWN_ARGUMENTS="--machine=${t##*/}" 124 | 125 | # Skipped test don't create the $TESTDIR automatically, so do it explicitly 126 | # otherwise the `touch` command would fail 127 | mkdir -p "$TESTDIR" 128 | rm -f "$TESTDIR/pass" 129 | 130 | exectask_p "${t##*/}" "/bin/time -v -- make -C $t setup run && touch $TESTDIR/pass" 131 | EXECUTED_LIST+=("$t") 132 | done 133 | 134 | # Wait for remaining running tasks 135 | exectask_p_finish 136 | 137 | for t in "${FLAKE_LIST[@]}"; do 138 | ## Configure test environment 139 | # Set timeouts for QEMU and nspawn tests to kill them in case they get stuck 140 | export QEMU_TIMEOUT=600 141 | export NSPAWN_TIMEOUT=600 142 | # Set the test dir to something predictable so we can refer to it later 143 | export TESTDIR="/var/tmp/systemd-test-${t##*/}" 144 | # Set QEMU_SMP appropriately (regarding the parallelism) 145 | # OPTIMAL_QEMU_SMP is part of the common/task-control.sh file 146 | export QEMU_SMP=$(nproc) 147 | # Enforce nested KVM 148 | export TEST_NESTED_KVM=1 149 | 150 | # Suffix the $TESTDIR of each retry with an index to tell them apart 151 | export MANGLE_TESTDIR=1 152 | exectask_retry "${t##*/}" "/bin/time -v -- make -C $t setup run && touch \$TESTDIR/pass" 153 | 154 | # Retried tasks are suffixed with an index, so update the $EXECUTED_LIST 155 | # array accordingly to correctly find the respective journals 156 | for ((i = 1; i <= TASK_RETRY_DEFAULT; i++)); do 157 | [[ -d "/var/tmp/systemd-test-${t##*/}_${i}" ]] && EXECUTED_LIST+=("${t}_${i}") 158 | done 159 | done 160 | 161 | # Save journals created by integration tests 162 | for t in "${EXECUTED_LIST[@]}"; do 163 | testname="${t##*/}" 164 | testdir="/var/tmp/systemd-test-$testname" 165 | 166 | if [[ -f "$testdir/system.journal" ]]; then 167 | # Filter out test-specific coredumps which are usually intentional 168 | # Note: $COREDUMPCTL_EXCLUDE_MAP resides in common/utils.sh 169 | if [[ -v "COREDUMPCTL_EXCLUDE_MAP[$t]" ]]; then 170 | export COREDUMPCTL_EXCLUDE_RX="${COREDUMPCTL_EXCLUDE_MAP[$t]}" 171 | fi 172 | # Attempt to collect coredumps from test-specific journals as well 173 | exectask "${testname}_coredumpctl_collect" "coredumpctl_collect '$testdir/'" 174 | # Make sure to not propagate the custom coredumpctl filter override 175 | [[ -v COREDUMPCTL_EXCLUDE_RX ]] && unset -v COREDUMPCTL_EXCLUDE_RX 176 | 177 | # Keep the journal files only if the associated test case failed 178 | if [[ ! -f "$testdir/pass" ]]; then 179 | rsync -aq "$testdir/system.journal" "$LOGDIR/$testname/" 180 | fi 181 | fi 182 | 183 | if [[ -f "$testdir/coverage-info" ]]; then 184 | cp "$testdir/coverage-info" "$COVERAGE_DIR/${testname}.coverage-info" 185 | fi 186 | 187 | # Clean the no longer necessary test artifacts 188 | [[ -d "$t" ]] && make -C "$t" clean-again > /dev/null 189 | done 190 | 191 | # Collect coverage metadata from the $BUILD_DIR (since we use the just-built nspawn 192 | # and other tools + networkd test suite) 193 | exectask "lcov_collect-build_dir-final" "lcov_collect $COVERAGE_DIR/build_dir.coverage-info $BUILD_DIR" 194 | 195 | # Collect coredumps using the coredumpctl utility, if any 196 | exectask "coredumpctl_collect" "coredumpctl_collect" 197 | 198 | # Merge all "coverage-info" files from the integration tests into one file 199 | exectask "lcov-merge-coverage" "lcov_merge everything.coverage-info $COVERAGE_DIR" 200 | exectask "lcov-dump-coverage-report" "lcov --ignore-errors inconsistent --list everything.coverage-info" 201 | # Coveralls repo token is set via the .coveralls.yml configuration file generated 202 | # in vagrant/vagrant-ci-wrapper.sh 203 | exectask "coveralls-upload" "coveralls report --format=lcov everything.coverage-info" 204 | # Copy the final coverage report to artifacts for local analysis if needed 205 | cp -fv "everything.coverage-info" "$LOGDIR/" 206 | 207 | # If the test logs contain lines like: 208 | # 209 | # ...systemd-resolved[735885]: profiling:/systemd-meson-build/src/shared/libsystemd-shared-250.a.p/base-filesystem.c.gcda:Cannot open 210 | # 211 | # it means we're possibly missing some coverage since gcov can't write the stats, 212 | # usually due to the sandbox being too restrictive (e.g. ProtectSystem=yes, 213 | # ProtectHome=yes) or the $BUILD_DIR being inaccessible to non-root users - see 214 | # `setfacl` stuff above. 215 | _check_for_missing_coverage() { 216 | local ec=0 217 | 218 | while read -r file; do 219 | echo "*** Processing file $file ***" 220 | ! grep -E "profiling:.+?gcda:[Cc]annot open" "$file" || ec=1 221 | done < <(find "$LOGDIR" -maxdepth 1 -name "*.log" ! -regex ".*/TEST-\(02\|82\)-.*.log" ! -name "check_for_missing_coverage*.log") 222 | 223 | return $ec 224 | } 225 | exectask "check_for_missing_coverage" "_check_for_missing_coverage" 226 | 227 | # Summary 228 | show_task_summary 229 | finish_and_exit 230 | -------------------------------------------------------------------------------- /agent/testsuite-rhel9.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | # shellcheck disable=SC2155 3 | 4 | LIB_ROOT="$(dirname "$0")/../common" 5 | # The common/utils.sh include needs to come first, as it includes definition 6 | # of print_cgroup_hierarchy() 7 | # shellcheck source=common/utils.sh 8 | . "$LIB_ROOT/utils.sh" || exit 1 9 | # shellcheck source=common/task-control.sh 10 | . "$LIB_ROOT/task-control.sh" "testsuite-logs-$(print_cgroup_hierarchy)-rhel9" || exit 1 11 | 12 | # EXIT signal handler 13 | at_exit() { 14 | set +e 15 | exectask "journalctl-testsuite" "journalctl -b -o short-monotonic --no-hostname --no-pager" 16 | } 17 | 18 | set -eu 19 | set -o pipefail 20 | 21 | trap at_exit EXIT 22 | 23 | if [[ "${1:-}" == "-n" ]]; then 24 | echo "[NOTICE] Running only nspawn-based tests" 25 | export TEST_NO_QEMU=1 26 | fi 27 | 28 | ### SETUP PHASE ### 29 | CGROUP_HIERARCHY="$(print_cgroup_hierarchy)" 30 | 31 | echo "Current cgroup hierarchy: $CGROUP_HIERARCHY" 32 | # Reflect the current cgroup hierarchy in each test VM 33 | if [[ "$CGROUP_HIERARCHY" == unified ]]; then 34 | CGROUP_KERNEL_ARGS=("systemd.unified_cgroup_hierarchy=1" "systemd.legacy_systemd_cgroup_controller=0") 35 | else 36 | CGROUP_KERNEL_ARGS=("systemd.unified_cgroup_hierarchy=0" "systemd.legacy_systemd_cgroup_controller=1") 37 | fi 38 | 39 | # Enable systemd-coredump 40 | if ! coredumpctl_init; then 41 | echo >&2 "Failed to configure systemd-coredump/coredumpctl" 42 | exit 1 43 | fi 44 | 45 | # Collect any coredumps that happened during boot 46 | if ! exectask "coredumpctl_collect_boot" "coredumpctl_collect"; then 47 | echo >&2 "Detected coredump(s) during system bootup" 48 | exit 1 49 | fi 50 | 51 | if [[ $(cat /proc/sys/user/max_user_namespaces) -le 0 ]]; then 52 | echo >&2 "user.max_user_namespaces must be > 0" 53 | exit 1 54 | fi 55 | 56 | set +e 57 | 58 | ### TEST PHASE ### 59 | pushd systemd || { echo >&2 "Can't pushd to systemd"; exit 1; } 60 | 61 | # Following issues should be fixed by the rebase in RHEL 9.2, but let's keep 62 | # them around for possible z-streams 63 | if "build/systemctl" --version | grep -q "systemd 250"; then 64 | # FIXME: test-journal-flush 65 | # A particularly ugly workaround for the flaky test-journal-flush. As the issue 66 | # presented so far only in the QEMU TEST-02, let's skip it just there, instead 67 | # of disabling it completely (even in the `meson test`). 68 | # 69 | # See: systemd/systemd#17963 70 | # shellcheck disable=SC2016 71 | sed -i '/mapfile -t TEST_LIST/aTEST_LIST=("${TEST_LIST[@]/\\/usr\\/lib\\/systemd\\/tests\\/test-journal-flush}")' test/units/testsuite-02.sh 72 | 73 | # FIXME: test-loop-block 74 | # This test is flaky due to uevent mess, and requires a kernel change. 75 | # 76 | # See: 77 | # systemd/systemd#17469 78 | # systemd/systemd#18166 79 | echo 'int main(void) { return 77; }' > src/test/test-loop-block.c 80 | 81 | # FIXME: test-seccomp 82 | # This test became flaky once again, so disable it temporarily until the reason 83 | # is found out. 84 | # 85 | # See: systemd/systemd#17078 86 | echo 'int main(void) { return 77; }' > src/test/test-seccomp.c 87 | 88 | # FIXME: test-barrier 89 | # This test is flaky on systems under load, which happens intermittently due 90 | # to how meson runs the tests (in parallel). 91 | # 92 | # See: 93 | # https://github.com/systemd/systemd/commit/fd23f9c9a70e1214507641d327da40d1688b74d7 94 | # https://github.com/systemd/systemd/commit/a1e3f0f38b43e68ff9ea33ab1935aed4edf6ed7f 95 | echo 'int main(void) { return 77; }' > src/test/test-barrier.c 96 | fi 97 | 98 | # Run the internal unit tests (make check) 99 | exectask "ninja-test" "meson test -C build --print-errorlogs --timeout-multiplier=3" 100 | # Copy over meson test artifacts 101 | [[ -d "build/meson-logs" ]] && rsync -amq --include '*.txt' --include '*/' --exclude '*' "build/meson-logs" "$LOGDIR" 102 | 103 | # If we're not testing the main branch (the first diff) check if the tested 104 | # branch doesn't contain only man-related changes. If so, skip the integration 105 | # tests 106 | if ! git diff --quiet main HEAD && ! git diff "$(git merge-base main HEAD)" --name-only | grep -vE "^man/" >/dev/null; then 107 | echo "Detected man-only PR, skipping integration tests" 108 | finish_and_exit 109 | fi 110 | 111 | # Ignore any coredumps generated by unit tests, as there's a lot of intentional crashes 112 | coredumpctl_set_ts 113 | 114 | ## Integration test suite ## 115 | EXECUTED_LIST=() 116 | FLAKE_LIST=( 117 | "test/TEST-16-EXTEND-TIMEOUT" # flaky test, see below 118 | "test/TEST-50-DISSECT" # flaky test, see below (systemd/systemd#17469) 119 | "test/TEST-75-RESOLVED" # flaky test 120 | ) 121 | SKIP_LIST=( 122 | "test/TEST-29-PORTABLE" # we don't ship portabled in RHEL 9 123 | "test/TEST-30-ONCLOCKCHANGE" # we don't ship timesyncd in RHEL 9 124 | "test/TEST-61-UNITTESTS-QEMU" # redundant test, runs the same tests as TEST-02, but only QEMU (systemd/systemd#19969) 125 | "${FLAKE_LIST[@]}" 126 | ) 127 | 128 | if [[ "$CGROUP_HIERARCHY" == "legacy" ]]; then 129 | # These test (or parts of them) explicitly require unified cgroup hierarchy 130 | SKIP_LIST+=( 131 | "test/TEST-19-DELEGATE" 132 | "test/TEST-74-AUX-UTILS" 133 | ) 134 | fi 135 | 136 | # Skip TEST-70-TPM2 on RHEL/C9S < 9.2, as it's not supported there and 137 | # we lack a lot of patches to make the test work 138 | if "build/systemctl" --version | grep -q "systemd 250"; then 139 | SKIP_LIST+=("test/TEST-70-TPM2") 140 | fi 141 | 142 | centos_ensure_qemu_symlink 143 | 144 | ## Generate a custom-tailored initrd for the integration tests 145 | # The host initrd contains multipath modules & services which are unused 146 | # in the integration tests and sometimes cause unexpected failures. Let's build 147 | # a custom initrd used solely by the integration tests 148 | # 149 | # Set a path to the custom initrd into the INITRD variable which is read by 150 | # the integration test suite "framework" 151 | export INITRD="/var/tmp/ci-initramfs-$(uname -r).img" 152 | # Copy over the original initrd, as we want to keep the custom installed 153 | # files we installed during the bootstrap phase (i.e. we want to keep the 154 | # command line arguments the original initrd was built with) 155 | cp -fv "/boot/initramfs-$(uname -r).img" "$INITRD" 156 | # Rebuild the original initrd without the multipath module 157 | dracut -a crypt -o "multipath rngd" --rebuild "$INITRD" 158 | 159 | # Initialize the 'base' image (default.img) on which the other images are based 160 | exectask "setup-the-base-image" "make -C test/TEST-01-BASIC clean setup TESTDIR=/var/tmp/systemd-test-TEST-01-BASIC" 161 | 162 | # Shared test env variables 163 | # 164 | # Explicitly set paths to initramfs and kernel images (for QEMU tests) 165 | # See $INITRD above 166 | export KERNEL_BIN="/boot/vmlinuz-$(uname -r)" 167 | # Explicitly enable user namespaces 168 | export KERNEL_APPEND="${CGROUP_KERNEL_ARGS[*]}" 169 | # Set timeouts for QEMU and nspawn tests to kill them in case they get stuck 170 | export QEMU_TIMEOUT=1800 171 | export NSPAWN_TIMEOUT=600 172 | # Work around 'Fatal glibc error: CPU does not support x86-64-v2' 173 | # See: 174 | # - https://bugzilla.redhat.com/show_bug.cgi?id=2060839 175 | # - https://access.redhat.com/solutions/6833751 176 | export QEMU_OPTIONS="-cpu Nehalem" 177 | 178 | # Let's re-shuffle the test list a bit by placing the most expensive tests 179 | # in the front, so they can run in background while we go through the rest 180 | # of the list 181 | readarray -t INTEGRATION_TESTS < <( 182 | echo test/TEST-64-UDEV-STORAGE 183 | find test/ -maxdepth 1 -type d -name "TEST-??-*" ! -name "TEST-64-UDEV-STORAGE" | sort 184 | ) 185 | 186 | for t in "${INTEGRATION_TESTS[@]}"; do 187 | if [[ ${#SKIP_LIST[@]} -ne 0 ]] && in_set "$t" "${SKIP_LIST[@]}"; then 188 | echo -e "[SKIP] Skipping test $t\n" 189 | continue 190 | fi 191 | 192 | ## Configure test environment 193 | # Tell the test framework to copy the base image for each test, so we 194 | # can run them in parallel 195 | export TEST_PARALLELIZE=1 196 | # Set the test dir to something predictable so we can refer to it later 197 | export TESTDIR="/var/tmp/systemd-test-${t##*/}" 198 | # Set QEMU_SMP appropriately (regarding the parallelism) 199 | # OPTIMAL_QEMU_SMP is part of the common/task-control.sh file 200 | export QEMU_SMP=$OPTIMAL_QEMU_SMP 201 | # Use a "unique" name for each nspawn container to prevent scope clash 202 | export NSPAWN_ARGUMENTS="--machine=${t##*/}" 203 | 204 | # Skipped test don't create the $TESTDIR automatically, so do it explicitly 205 | # otherwise the `touch` command would fail 206 | mkdir -p "$TESTDIR" 207 | rm -f "$TESTDIR/pass" 208 | 209 | exectask_p "${t##*/}" "/bin/time -v -- make -C $t setup run && touch $TESTDIR/pass" 210 | EXECUTED_LIST+=("$t") 211 | done 212 | 213 | # Wait for remaining running tasks 214 | exectask_p_finish 215 | 216 | for t in "${FLAKE_LIST[@]}"; do 217 | ## Configure test environment 218 | # Set the test dir to something predictable so we can refer to it later 219 | export TESTDIR="/var/tmp/systemd-test-${t##*/}" 220 | # Set QEMU_SMP appropriately (regarding the parallelism) 221 | # OPTIMAL_QEMU_SMP is part of the common/task-control.sh file 222 | export QEMU_SMP=$(nproc) 223 | 224 | # Suffix the $TESTDIR of each retry with an index to tell them apart 225 | export MANGLE_TESTDIR=1 226 | exectask_retry "${t##*/}" "/bin/time -v -- make -C $t setup run && touch \$TESTDIR/pass" 227 | 228 | # Retried tasks are suffixed with an index, so update the $EXECUTED_LIST 229 | # array accordingly to correctly find the respective journals 230 | for ((i = 1; i <= TASK_RETRY_DEFAULT; i++)); do 231 | [[ -d "/var/tmp/systemd-test-${t##*/}_${i}" ]] && EXECUTED_LIST+=("${t}_${i}") 232 | done 233 | done 234 | 235 | for t in "${EXECUTED_LIST[@]}"; do 236 | testdir="/var/tmp/systemd-test-${t##*/}" 237 | if [[ -f "$testdir/system.journal" ]]; then 238 | # Filter out test-specific coredumps which are usually intentional 239 | # Note: $COREDUMPCTL_EXCLUDE_MAP resides in common/utils.sh 240 | if [[ -v "COREDUMPCTL_EXCLUDE_MAP[$t]" ]]; then 241 | export COREDUMPCTL_EXCLUDE_RX="${COREDUMPCTL_EXCLUDE_MAP[$t]}" 242 | fi 243 | # Attempt to collect coredumps from test-specific journals as well 244 | exectask "${t##*/}_coredumpctl_collect" "coredumpctl_collect '$testdir/'" 245 | # Make sure to not propagate the custom coredumpctl filter override 246 | [[ -v COREDUMPCTL_EXCLUDE_RX ]] && unset -v COREDUMPCTL_EXCLUDE_RX 247 | 248 | # Keep the journal files only if the associated test case failed 249 | if [[ ! -f "$testdir/pass" ]]; then 250 | rsync -aq "$testdir/system.journal" "$LOGDIR/${t##*/}/" 251 | fi 252 | fi 253 | 254 | # Clean the no longer necessary test artifacts 255 | [[ -d "$t" ]] && make -C "$t" clean-again > /dev/null 256 | done 257 | 258 | ## Other integration tests ## 259 | TEST_LIST=( 260 | "test/test-exec-deserialization.py" 261 | ) 262 | 263 | for t in "${TEST_LIST[@]}"; do 264 | exectask "${t##*/}" "/bin/time -v -- timeout -k 60s 60m ./$t" 265 | done 266 | 267 | # Collect coredumps using the coredumpctl utility, if any 268 | exectask "coredumpctl_collect" "coredumpctl_collect" 269 | 270 | # Summary 271 | show_task_summary 272 | 273 | finish_and_exit 274 | -------------------------------------------------------------------------------- /agent/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | # shellcheck disable=SC2155 3 | 4 | LIB_ROOT="$(dirname "$0")/../common" 5 | # shellcheck source=common/task-control.sh 6 | . "$LIB_ROOT/task-control.sh" "bootstrap-logs-$(uname -m)" || exit 1 7 | # shellcheck source=common/utils.sh 8 | . "$LIB_ROOT/utils.sh" || exit 1 9 | 10 | export BUILD_DIR="${BUILD_DIR:-/systemd-meson-build}" 11 | REPO_URL="${REPO_URL:-https://github.com/systemd/systemd.git}" 12 | REMOTE_REF="" 13 | 14 | # EXIT signal handler 15 | at_exit() { 16 | # Let's collect some build-related logs 17 | set +e 18 | exectask "journalctl-bootstrap" "journalctl -b -o short-monotonic --no-hostname --no-pager" 19 | exectask "list-of-installed-packages" "rpm -qa" 20 | } 21 | 22 | set -eu 23 | set -o pipefail 24 | 25 | trap at_exit EXIT 26 | 27 | # Parse optional script arguments 28 | while getopts "r:s:" opt; do 29 | case "$opt" in 30 | r) 31 | REMOTE_REF="$OPTARG" 32 | ;; 33 | s) 34 | REPO_URL="$OPTARG" 35 | ;; 36 | ?) 37 | exit 1 38 | ;; 39 | *) 40 | echo "Usage: $0 [-r REMOTE_REF] [-s SOURCE_REPO_URL]" 41 | exit 1 42 | esac 43 | done 44 | 45 | ADDITIONAL_DEPS=( 46 | attr 47 | bind-utils 48 | bpftool 49 | busybox 50 | clang 51 | cryptsetup 52 | device-mapper-event 53 | device-mapper-multipath 54 | dfuzzer 55 | dhcp-client 56 | dhcp-server 57 | dnsmasq 58 | dosfstools 59 | e2fsprogs 60 | elfutils 61 | elfutils-devel 62 | evemu 63 | expect 64 | fsverity-utils # EPEL 65 | gcc-c++ 66 | glibc-langpack-en 67 | gnutls-utils 68 | integritysetup 69 | iproute-tc 70 | iscsi-initiator-utils 71 | jq 72 | kernel-modules-extra 73 | keyutils 74 | knot 75 | knot-dnssecutils 76 | libarchive-devel 77 | libasan 78 | libbpf-devel 79 | libfdisk-devel 80 | libfido2-devel 81 | libpwquality-devel 82 | libzstd-devel 83 | llvm 84 | lvm2 85 | make 86 | mdadm 87 | mtools 88 | net-tools 89 | netlabel_tools 90 | nftables 91 | nmap-ncat 92 | nvme-cli 93 | opensc 94 | openssl-devel 95 | pcre2-devel 96 | python3-jinja2 97 | python3-pefile # EPEL 98 | python3-pexpect 99 | python3-psutil 100 | python3-pyelftools # EPEL 101 | python3-pyparsing 102 | python3-pytest 103 | qrencode-devel 104 | quota 105 | rust 106 | screen 107 | scsi-target-utils 108 | selinux-policy-devel 109 | socat 110 | softhsm # EPEL 111 | squashfs-tools 112 | strace 113 | stress # EPEL 114 | stress-ng 115 | time 116 | tpm2-tools 117 | tpm2-tss-devel 118 | veritysetup 119 | vim-common 120 | wget 121 | zstd 122 | ) 123 | 124 | if [[ "$(uname -m)" != ppc64le ]]; then 125 | # There's noo qemu-kvm and swtpm on ppc64le 126 | ADDITIONAL_DEPS+=(qemu-kvm swtpm) 127 | fi 128 | 129 | cmd_retry dnf -y install epel-release epel-next-release dnf-plugins-core gdb 130 | cmd_retry dnf -y config-manager --enable epel --enable epel-next --enable crb 131 | # Local mirror of https://copr.fedorainfracloud.org/coprs/mrc0mmand/systemd-centos-ci-centos9/ 132 | cmd_retry dnf -y config-manager --add-repo "https://jenkins-systemd.apps.ocp.cloud.ci.centos.org/job/reposync/lastSuccessfulBuild/artifact/repos/mrc0mmand-systemd-centos-ci-centos9-stream9/mrc0mmand-systemd-centos-ci-centos9-stream9.repo" 133 | cmd_retry dnf -y update 134 | # FIXME: temporarily skip the crb-source repo, since it currently has borked signatures 135 | # (and we don't need it in this particular case) 136 | dnf config-manager --save --setopt crb-source.skip_if_unavailable=1 137 | # --skip-unavailable is necessary for archs without gnu-efi (like ppc64le) 138 | cmd_retry dnf -y --skip-unavailable builddep systemd 139 | cmd_retry dnf -y install "${ADDITIONAL_DEPS[@]}" 140 | # Remove setroubleshoot-server if it's installed, since we don't use it anyway 141 | # and it's causing some weird performance issues 142 | if rpm -q setroubleshoot-server; then 143 | dnf -y remove setroubleshoot-server 144 | fi 145 | 146 | # Fetch the upstream systemd repo 147 | test -e systemd && rm -rf systemd 148 | echo "Cloning repo: $REPO_URL" 149 | git clone "$REPO_URL" systemd 150 | pushd systemd || { echo >&2 "Can't pushd to systemd"; exit 1; } 151 | 152 | git_checkout_pr "$REMOTE_REF" 153 | 154 | # It's impossible to keep the local SELinux policy database up-to-date with 155 | # arbitrary pull request branches we're testing against. 156 | # Set SELinux to permissive on the test hosts to avoid false positives, but 157 | # to still allow running tests which require SELinux. 158 | if setenforce 0; then 159 | echo SELINUX=permissive >/etc/selinux/config 160 | fi 161 | 162 | # Disable firewalld (needed for systemd-networkd tests) 163 | systemctl -q is-enabled firewalld && systemctl disable --now firewalld 164 | 165 | # Set tuned to throughput-performance if available 166 | if command -v tuned-adm 2>/dev/null; then 167 | tuned-adm profile throughput-performance 168 | tuned-adm active 169 | tuned-adm verify 170 | fi 171 | 172 | # Enable systemd-coredump 173 | if ! coredumpctl_init; then 174 | echo >&2 "Failed to configure systemd-coredump/coredumpctl" 175 | exit 1 176 | fi 177 | 178 | # Compile systemd 179 | # - slow-tests=true: enables slow tests 180 | # - fuzz-tests=true: enables fuzzy tests using libasan installed above 181 | # - tests=unsafe: enable unsafe tests, which might change the environment 182 | # - install-tests=true: necessary for test/TEST-24-UNIT-TESTS 183 | ( 184 | # Make sure we copy over the meson logs even if the compilation fails 185 | # shellcheck disable=SC2064 186 | trap "[[ -d $BUILD_DIR/meson-logs ]] && cp -r $BUILD_DIR/meson-logs '$LOGDIR'" EXIT 187 | 188 | # Skip ukify on ppc64le, since EFI is not supported there 189 | [[ "$(uname -m)" == ppc64le ]] && UKIFY=false || UKIFY=true 190 | 191 | meson setup "$BUILD_DIR" \ 192 | -Dc_args='-fno-omit-frame-pointer -ftrapv -Og' \ 193 | -Dcpp_args='-Og' \ 194 | -Ddebug=true \ 195 | --werror \ 196 | -Dlog-trace=true \ 197 | -Dslow-tests=true \ 198 | -Dfuzz-tests=true \ 199 | -Dtests=unsafe \ 200 | -Dinstall-tests=true \ 201 | -Ddbuspolicydir=/etc/dbus-1/system.d \ 202 | -Dukify="$UKIFY" \ 203 | -Dman=true \ 204 | -Dhtml=true 205 | ninja -C "$BUILD_DIR" 206 | ) 2>&1 | tee "$LOGDIR/build.log" 207 | 208 | # shellcheck disable=SC2119 209 | coredumpctl_set_ts 210 | 211 | # Install the compiled systemd 212 | ninja -C "$BUILD_DIR" install 213 | 214 | LATEST_KERNEL="$(rpm -q kernel --qf "%{EVR}.%{ARCH}\n" | sort -Vr | head -n1)" 215 | # FIXME: drop once https://github.com/systemd/systemd/pull/27890 lands 216 | DRACUT_OPTS=() 217 | [[ -x /usr/lib/systemd/systemd-executor ]] && DRACUT_OPTS+=(--install /usr/lib/systemd/systemd-executor) 218 | 219 | # Let's check if the new systemd at least boots before rebooting the system 220 | # As the CentOS' systemd-nspawn version is too old, we have to use QEMU 221 | ( 222 | # Ensure the initrd contains the same systemd version as the one we're 223 | # trying to test 224 | # Also, rebuild the original initrd without the multipath module, see 225 | # comments in `testsuite.sh` for the explanation 226 | export INITRD="/var/tmp/ci-sanity-initramfs-$(uname -r).img" 227 | cp -fv "/boot/initramfs-$(uname -r).img" "$INITRD" 228 | dracut "${DRACUT_OPTS[@]}" --kver "$LATEST_KERNEL" -o "multipath rngd" --filesystems ext4 --rebuild "$INITRD" 229 | 230 | centos_ensure_qemu_symlink 231 | 232 | ## Configure test environment 233 | # Explicitly set paths to initramfs (see above) and kernel images 234 | # (for QEMU tests) 235 | export KERNEL_BIN="/boot/vmlinuz-$LATEST_KERNEL" 236 | export KERNEL_VER="$LATEST_KERNEL" 237 | # Enable kernel debug output for easier debugging when something goes south 238 | export KERNEL_APPEND="debug systemd.log_level=debug rd.systemd.log_target=console systemd.default_standard_output=journal+console" 239 | # Set timeout for QEMU tests to kill them in case they get stuck 240 | export QEMU_TIMEOUT=600 241 | # Disable nspawn version of the test 242 | export TEST_NO_NSPAWN=1 243 | export QEMU_OPTIONS="-cpu max" 244 | 245 | if ! make -C test/TEST-01-BASIC clean setup run clean-again; then 246 | rsync -amq /var/tmp/systemd-tests/systemd-test*/system.journal "$LOGDIR/sanity-boot-check.journal" >/dev/null || : 247 | exit 1 248 | fi 249 | 250 | rm -fv "$INITRD" 251 | ) 2>&1 | tee "$LOGDIR/sanity-boot-check.log" 252 | 253 | # Tell systemd-networkd to ignore eth0 netdev, so we can keep it up during the 254 | # systemd-networkd testsuite 255 | cat >/etc/systemd/network/10-eth0.network <&2 "Missing systemd module in the initramfs image, can't continue..." 276 | lsinitrd "/boot/initramfs-$(uname -r).img" 277 | exit 1 278 | fi 279 | 280 | # For some reason the C9S AMIs have BLS in grub switched off. If that's the case 281 | # let's re-enable it before tweaking the grub configuration further. 282 | if grep -q "GRUB_ENABLE_BLSCFG=false" /etc/default/grub; then 283 | sed -i 's/GRUB_ENABLE_BLSCFG=false/GRUB_ENABLE_BLSCFG=true/' /etc/default/grub 284 | grub2-mkconfig -o /boot/grub2/grub.cfg 285 | fi 286 | 287 | GRUBBY_ARGS=( 288 | # As the RTC on CentOS CI machines is notoriously incorrect, let's override 289 | # it early in the boot process to properly execute units using 290 | # ConditionNeedsUpdate= 291 | # See: https://github.com/systemd/systemd/issues/15724#issuecomment-628194867 292 | # Update: if the original time difference is too big, the mtime of 293 | # /etc/.updated is already too far in the future, so it doesn't matter if 294 | # we correct the time during the next boot, since it's still going to be 295 | # in the past. Let's just explicitly override all ConditionNeedsUpdate= 296 | # directives to true to fix this once and for all 297 | "systemd.condition-needs-update=1" 298 | # Also, store & reuse the current (and corrected) time & date, as it doesn't 299 | # persist across reboots without this kludge and can (actually it does) 300 | # interfere with running tests 301 | "systemd.clock_usec=$(($(date +%s%N) / 1000 + 1))" 302 | # Reboot the machine on kernel panic 303 | "panic=3" 304 | ) 305 | # Make sure the latest kernel is the one we're going to boot into 306 | grubby --set-default "/boot/vmlinuz-$LATEST_KERNEL" 307 | grubby --args="${GRUBBY_ARGS[*]}" --update-kernel="$(grubby --default-kernel)" 308 | # Check if the $GRUBBY_ARGS were applied correctly 309 | for arg in "${GRUBBY_ARGS[@]}"; do 310 | if ! grep -q -r "$arg" /boot/loader/entries/; then 311 | echo >&2 "Kernel parameter '$arg' was not found in /boot/loader/entries/*.conf" 312 | exit 1 313 | fi 314 | done 315 | 316 | # coredumpctl_collect takes an optional argument, which upsets shellcheck 317 | # shellcheck disable=SC2119 318 | coredumpctl_collect 319 | 320 | # Configure kdump 321 | #kdumpctl status || kdumpctl restart 322 | #kdumpctl showmem 323 | #kdumpctl rebuild 324 | 325 | echo "-----------------------------" 326 | echo "- REBOOT THE MACHINE BEFORE -" 327 | echo "- CONTINUING -" 328 | echo "-----------------------------" 329 | -------------------------------------------------------------------------------- /agent/bootstrap-rhel8.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | # shellcheck disable=SC2155 3 | 4 | LIB_ROOT="$(dirname "$0")/../common" 5 | # shellcheck source=common/task-control.sh 6 | . "$LIB_ROOT/task-control.sh" "bootstrap-logs-rhel8" || exit 1 7 | # shellcheck source=common/utils.sh 8 | . "$LIB_ROOT/utils.sh" || exit 1 9 | 10 | REPO_URL="${REPO_URL:-https://github.com/redhat-plumbers/systemd-rhel8}" 11 | CGROUP_HIERARCHY="legacy" 12 | REMOTE_REF="" 13 | 14 | # EXIT signal handler 15 | at_exit() { 16 | # Let's collect some build-related logs 17 | set +e 18 | rsync -amq /var/tmp/systemd-test*/journal "$LOGDIR" &>/dev/null || : 19 | exectask "journalctl-bootstrap" "journalctl -b -o short-monotonic --no-hostname --no-pager" 20 | exectask "list-of-installed-packages" "rpm -qa" 21 | } 22 | 23 | set -eu 24 | set -o pipefail 25 | 26 | trap at_exit EXIT 27 | 28 | # Parse optional script arguments 29 | while getopts "r:h:" opt; do 30 | case "$opt" in 31 | h) 32 | CGROUP_HIERARCHY="$OPTARG" 33 | if [[ "$CGROUP_HIERARCHY" != legacy && "$CGROUP_HIERARCHY" != unified ]]; then 34 | echo "Invalid cgroup hierarchy specified: $CGROUP_HIERARCHY" 35 | exit 1 36 | fi 37 | ;; 38 | r) 39 | REMOTE_REF="$OPTARG" 40 | ;; 41 | ?) 42 | exit 1 43 | ;; 44 | *) 45 | echo "Usage: $0 [-h CGROUP_HIERARCHY] [-r REMOTE_REF]" 46 | exit 1 47 | esac 48 | done 49 | 50 | ADDITIONAL_DEPS=( 51 | busybox 52 | cryptsetup 53 | dhclient 54 | dnsmasq 55 | e2fsprogs 56 | expect 57 | gdb 58 | jq 59 | libasan 60 | libubsan 61 | make 62 | net-tools 63 | nmap-ncat 64 | perl-IPC-SysV 65 | perl-Time-HiRes 66 | plymouth # * 67 | qemu-kvm 68 | quota 69 | socat 70 | strace 71 | time 72 | wget 73 | ) 74 | # * We need plymouth with EL8 systemd, as we miss a couple of patches that make 75 | # it an optional dependency (like e4e039bce4462b45e413bb687ab593b6ecc886e3 and 76 | # follow-ups) 77 | 78 | # Install and enable EPEL 79 | cmd_retry dnf -y install epel-release epel-next-release dnf-plugins-core 80 | cmd_retry dnf config-manager --enable epel --enable crb 81 | # Upgrade the machine to get the most recent environment 82 | cmd_retry dnf -y upgrade 83 | # Install systemd's build dependencies 84 | cmd_retry dnf -y builddep systemd 85 | cmd_retry dnf -y install "${ADDITIONAL_DEPS[@]}" 86 | # Use the Nmap's version of nc, since TEST-13-NSPAWN-SMOKE doesn't seem to work 87 | # with the OpenBSD version present on CentOS 8 88 | if alternatives --display nmap; then 89 | alternatives --set nmap /usr/bin/ncat 90 | alternatives --display nmap 91 | fi 92 | 93 | # Fetch the systemd repo 94 | test -e systemd && rm -rf systemd 95 | git clone "$REPO_URL" systemd 96 | pushd systemd || { echo >&2 "Can't pushd to systemd"; exit 1; } 97 | 98 | git_checkout_pr "$REMOTE_REF" 99 | 100 | # Optionally cherry-pick some commits to make the test work/stable before they 101 | # reach RHEL 8 branches 102 | git remote add upstream "https://github.com/systemd/systemd" 103 | git fetch upstream 104 | # test: make TEST-27 non-racy 105 | git show 324ca05459422b55cb6fa04318552541159c239a | git apply --verbose --recount || : 106 | 107 | # It's impossible to keep the local SELinux policy database up-to-date with 108 | # arbitrary pull request branches we're testing against. 109 | # Disable SELinux on the test hosts and avoid false positives. 110 | if setenforce 0; then 111 | echo SELINUX=permissive >/etc/selinux/config 112 | fi 113 | 114 | # Set tuned to throughput-performance if available 115 | if command -v tuned-adm 2>/dev/null; then 116 | tuned-adm profile throughput-performance 117 | tuned-adm active 118 | tuned-adm verify 119 | fi 120 | 121 | # Enable systemd-coredump 122 | if ! coredumpctl_init; then 123 | echo >&2 "Failed to configure systemd-coredump/coredumpctl" 124 | exit 1 125 | fi 126 | 127 | # Compile systemd 128 | # - slow-tests=true: enable slow tests 129 | # - tests=unsafe: enable unsafe tests, which might change the environment 130 | # - install-tests=true: necessary for test/TEST-24-UNIT-TESTS 131 | ( 132 | # Make sure we copy over the meson logs even if the compilation fails 133 | # shellcheck disable=SC2064 134 | trap "[[ -d $PWD/build/meson-logs ]] && cp -r $PWD/build/meson-logs '$LOGDIR'" EXIT 135 | # shellcheck disable=SC2191 136 | CONFIGURE_OPTS=( 137 | # RHEL8 options 138 | -Dsysvinit-path=/etc/rc.d/init.d 139 | -Drc-local=/etc/rc.d/rc.local 140 | -Ddns-servers='' 141 | -Ddev-kvm-mode=0666 142 | -Dkmod=true 143 | -Dxkbcommon=true 144 | -Dblkid=true 145 | -Dseccomp=true 146 | -Dima=true 147 | -Dselinux=true 148 | -Dapparmor=false 149 | -Dpolkit=true 150 | -Dxz=true 151 | -Dzlib=true 152 | -Dbzip2=true 153 | -Dlz4=true 154 | -Dpam=true 155 | -Dacl=true 156 | -Dsmack=true 157 | -Dgcrypt=true 158 | -Daudit=true 159 | -Delfutils=true 160 | -Dlibcryptsetup=true 161 | -Delfutils=true 162 | -Dqrencode=false 163 | -Dgnutls=true 164 | -Dmicrohttpd=true 165 | -Dlibidn2=true 166 | -Dlibiptc=false 167 | -Dlibcurl=true 168 | -Defi=true 169 | -Dtpm=true 170 | -Dhwdb=true 171 | -Dsysusers=true 172 | -Ddefault-kill-user-processes=false 173 | -Dtests=unsafe 174 | -Dinstall-tests=true 175 | -Dtty-gid=5 176 | -Dusers-gid=100 177 | -Dnobody-user=nobody 178 | -Dnobody-group=nobody 179 | -Dsplit-usr=false 180 | -Dsplit-bin=true 181 | -Db_lto=false 182 | -Dnetworkd=false 183 | -Dtimesyncd=false 184 | -Ddefault-hierarchy=legacy 185 | # Custom options 186 | -Dslow-tests=true 187 | -Dtests=unsafe 188 | -Dinstall-tests=true 189 | # FIXME?: --werror 190 | # gcc in RHEL 9 has some additional warnings enabled for which we don't have the respective 191 | # patches in RHEL 8's systemd 192 | #--werror 193 | -Dman=true 194 | -Dhtml=true 195 | ) 196 | 197 | # Ignore compiler's `unused-function` warnings on rhel-8.{1,2}.0 based branches, 198 | # since we disable the `test-cap-util` test there in a rather crude way, leading 199 | # up to valid but expected warnings 200 | if git diff --quiet ..remotes/origin/rhel-8.1.0 || git diff --quiet ..remotes/origin/rhel-8.2.0; then 201 | CONFIGURE_OPTS+=(-Dc_args='-g -O0 -ftrapv -Wno-error=unused-function') 202 | else 203 | CONFIGURE_OPTS+=(-Dc_args='-g -O0 -ftrapv') 204 | fi 205 | 206 | meson build "${CONFIGURE_OPTS[@]}" 207 | ninja -C build 208 | ) 2>&1 | tee "$LOGDIR/build.log" 209 | 210 | # Install the compiled systemd 211 | ninja -C build install 212 | 213 | # Create necessary systemd users/groups 214 | getent group systemd-resolve &>/dev/null || groupadd -r -g 193 systemd-resolve 2>&1 215 | getent passwd systemd-resolve &>/dev/null || useradd -r -u 193 -l -g systemd-resolve -d / -s /sbin/nologin -c "systemd Resolver" systemd-resolve &>/dev/null 216 | 217 | LATEST_KERNEL="$(rpm -q kernel --qf "%{EVR}.%{ARCH}\n" | sort -Vr | head -n1)" 218 | # Configure the selected cgroup hierarchy for both the host machine and each 219 | # integration test VM 220 | if [[ "$CGROUP_HIERARCHY" == unified ]]; then 221 | CGROUP_KERNEL_ARGS="systemd.unified_cgroup_hierarchy=1 systemd.legacy_systemd_cgroup_controller=0" 222 | else 223 | CGROUP_KERNEL_ARGS="systemd.unified_cgroup_hierarchy=0 systemd.legacy_systemd_cgroup_controller=1" 224 | fi 225 | 226 | # Disable irqbalance, as we don't really need it and it sometimes SIGTRAPs, causing spurious 227 | # coredumps 228 | systemctl disable --now irqbalance 229 | 230 | # dbus-broker on RHEL 9 is not compatible with systemd < 243, so replace it with dbus-daemon 231 | dnf -y install dbus-daemon 232 | systemctl disable dbus-broker 233 | systemctl disable --global dbus-broker 234 | systemctl enable dbus-daemon 235 | systemctl enable --global dbus-daemon 236 | 237 | # Several test/test-functions tweaks to make it work with RHEL 9 238 | # 239 | # RHEL 8's test/test-functions overwrites $QEMU_OPTIONS instead of appending to 240 | # it, so setting it as usual won't work. Let's, temporarily, patch 241 | # test/test-functions directly 242 | sed -i '0,/QEMU_OPTIONS=".*/s//&\n-cpu Nehalem \\/' test/test-functions 243 | # Make it work on systems where dbus-broker is the default 244 | sed -i '/dbus.service/d' test/test-functions 245 | sed -i '/dbus.socket/a\ inst /etc/systemd/system/dbus.service' test/test-functions 246 | 247 | # Let's check if the new systemd at least boots before rebooting the system 248 | ( 249 | # Ensure the initrd contains the same systemd version as the one we're 250 | # trying to test 251 | # Also, rebuild the original initrd without the multipath module, see 252 | # comments in `testsuite.sh` for the explanation 253 | export INITRD="/var/tmp/ci-sanity-initramfs-$(uname -r).img" 254 | cp -fv "/boot/initramfs-$(uname -r).img" "$INITRD" 255 | dracut --kver "$LATEST_KERNEL" -o "multipath rngd dbus-broker" -a dbus-daemon --filesystems ext4 --rebuild "$INITRD" 256 | 257 | centos_ensure_qemu_symlink 258 | 259 | ## Configure test environment 260 | # Explicitly set paths to initramfs (see above) and kernel images 261 | # (for QEMU tests) 262 | export KERNEL_BIN="/boot/vmlinuz-$LATEST_KERNEL" 263 | export KERNEL_VER="$LATEST_KERNEL" 264 | # Enable kernel debug output for easier debugging when something goes south 265 | export KERNEL_APPEND="debug systemd.log_level=debug rd.systemd.log_target=console $CGROUP_KERNEL_ARGS" 266 | # Set timeout for QEMU tests to kill them in case they get stuck 267 | export QEMU_TIMEOUT=600 268 | # Disable nspawn version of the test 269 | export TEST_NO_NSPAWN=1 270 | # Work around 'Fatal glibc error: CPU does not support x86-64-v2' 271 | # See: 272 | # - https://bugzilla.redhat.com/show_bug.cgi?id=2060839 273 | # - https://access.redhat.com/solutions/6833751 274 | export QEMU_OPTIONS="-cpu Nehalem" 275 | 276 | make -C test/TEST-01-BASIC clean setup run clean 277 | 278 | rm -fv "$INITRD" 279 | ) 2>&1 | tee "$LOGDIR/sanity-boot-check.log" 280 | 281 | # The new systemd binary boots, so let's issue a daemon-reexec to use it. 282 | # This is necessary, since at least once we got into a situation where 283 | # the old systemd binary was incompatible with the unit files on disk and 284 | # prevented the system from reboot 285 | SYSTEMD_LOG_LEVEL=debug systemctl daemon-reexec 286 | [[ -n "${XDG_RUNTIME_DIR:-}" ]] && SYSTEMD_LOG_LEVEL=debug systemctl --user daemon-reexec 287 | 288 | dracut -o dbus-broker -a dbus-daemon -f --regenerate-all 289 | 290 | # Check if the new dracut image contains the systemd module to avoid issues 291 | # like systemd/systemd#11330 292 | if ! lsinitrd -m "/boot/initramfs-$(uname -r).img" | grep "^systemd$"; then 293 | echo >&2 "Missing systemd module in the initramfs image, can't continue..." 294 | lsinitrd "/boot/initramfs-$(uname -r).img" 295 | exit 1 296 | fi 297 | 298 | # Switch between cgroups v1 (legacy) or cgroups v2 (unified) if requested 299 | echo "Configuring $CGROUP_HIERARCHY cgroup hierarchy using '$CGROUP_KERNEL_ARGS'" 300 | 301 | # Make sure the latest kernel is the one we're going to boot into 302 | grubby --set-default "/boot/vmlinuz-$LATEST_KERNEL" 303 | grubby --args="$CGROUP_KERNEL_ARGS" --update-kernel="$(grubby --default-kernel)" 304 | # grub on RHEL 8 uses BLS 305 | grep -r "systemd.unified_cgroup_hierarchy" /boot/loader/entries/ 306 | 307 | # coredumpctl_collect takes an optional argument, which upsets shellcheck 308 | # shellcheck disable=SC2119 309 | coredumpctl_collect 310 | 311 | echo "-----------------------------" 312 | echo "- REBOOT THE MACHINE BEFORE -" 313 | echo "- CONTINUING -" 314 | echo "-----------------------------" 315 | -------------------------------------------------------------------------------- /vagrant/test_scripts/test-arch-sanitizers-clang.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | # shellcheck disable=SC2155 3 | # This script is part of the systemd Vagrant test suite for CentOS CI and 4 | # it's expected to be executed in a Vagrant VM configured by vagrant-build.sh 5 | # script. 6 | # Majority of this script is copied from the systemd-centos-ci/agent/testsuite.sh 7 | # script with some modifications to support other distributions. Test dependencies 8 | # for each distribution must be installed prior executing this script. 9 | 10 | DISTRO="${1:-unspecified}" 11 | SCRIPT_DIR="$(dirname "$0")" 12 | # This variable is automagically consumed by the "framework" for integration tests 13 | # See respective bootstrap script under vagrant/bootstrap_scripts/ for reasoning 14 | export BUILD_DIR="${BUILD_DIR:-/systemd-meson-build}" 15 | # Consumed by coredumpctl_init()/coredumpctl_collect() 16 | export COREDUMPCTL_BIN="$BUILD_DIR/coredumpctl" 17 | 18 | # Following scripts are copied from the systemd-centos-ci/common directory by vagrant-builder.sh 19 | # shellcheck source=common/task-control.sh 20 | . "$SCRIPT_DIR/task-control.sh" "vagrant-$DISTRO-testsuite" || exit 1 21 | # shellcheck source=common/utils.sh 22 | . "$SCRIPT_DIR/utils.sh" || exit 1 23 | 24 | at_exit() { 25 | set +e 26 | exectask "journalctl-testsuite" "journalctl -b -o short-monotonic --no-hostname --no-pager" 27 | } 28 | 29 | trap at_exit EXIT 30 | 31 | bootctl status 32 | 33 | # Enable systemd-coredump 34 | if ! coredumpctl_init; then 35 | echo >&2 "Failed to configure systemd-coredump/coredumpctl" 36 | exit 1 37 | fi 38 | 39 | pushd /build || { echo >&2 "Can't pushd to /build"; exit 1; } 40 | 41 | ## Sanitizer-specific options 42 | export ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:detect_invalid_pointer_pairs=2:handle_ioctl=1:print_cmdline=1:disable_coredump=0:use_madv_dontdump=1 43 | export UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 44 | 45 | # Dump current ASan config 46 | ASAN_OPTIONS="${ASAN_OPTIONS:+$ASAN_OPTIONS:}help=1" "$BUILD_DIR/systemctl" is-system-running &>"$LOGDIR/asan_config.txt" 47 | 48 | ## Disable certain flaky tests 49 | # FIXME: test-execute 50 | # This test occasionally timeouts when running under sanitizers. Until the root 51 | # cause is figured out, let's temporarily skip this test to not disturb CI runs. 52 | echo 'int main(void) { return 77; }' > src/test/test-execute.c 53 | 54 | # Run the internal unit tests (make check) 55 | exectask "ninja-test_sanitizers" "meson test -C $BUILD_DIR --print-errorlogs --timeout-multiplier=3" 56 | exectask "check-meson-logs-for-sanitizer-errors" "cat $BUILD_DIR/meson-logs/testlog*.txt | check_for_sanitizer_errors" 57 | [[ -d "$BUILD_DIR/meson-logs" ]] && rsync -amq --include '*.txt' --include '*/' --exclude '*' "$BUILD_DIR/meson-logs" "$LOGDIR" 58 | 59 | # Ignore any coredumps generated by unit tests, as there's a lot of intentional crashes 60 | coredumpctl_set_ts 61 | 62 | ## Run TEST-01-BASIC under sanitizers 63 | # Prepare a custom-tailored initrd image (with the systemd module included). 64 | # This is necessary, as the default mkinitcpio config includes only the udev module, 65 | # which breaks certain things, like setting global env variables for systemd from 66 | # the kernel command line. 67 | # The exported INITRD variable is picked up by all following integration tests 68 | export INITRD="$(mktemp /var/tmp/initrd-testsuite-XXX.img)" 69 | if ! mkinitcpio -c /dev/null -A base,systemd,sd-encrypt,modconf,block,filesystems,keyboard,fsck -g "$INITRD"; then 70 | echo >&2 "Failed to generate initrd, can't continue" 71 | exit 1 72 | fi 73 | # Set timeouts for QEMU and nspawn tests to kill them in case they get stuck 74 | export QEMU_TIMEOUT=1500 75 | export NSPAWN_TIMEOUT=1200 76 | # Set QEMU_SMP to speed things up 77 | export QEMU_SMP=$(nproc) 78 | # Arch Linux requires booting with initrd, as all commonly used filesystems 79 | # are compiled in as modules 80 | export SKIP_INITRD=no 81 | # Enforce nested KVM 82 | export TEST_NESTED_KVM=1 83 | # Bump the SUT memory to 8G, mainly for dfuzzer 84 | export QEMU_MEM=8G 85 | # Don't strip systemd binaries installed into test images, so we can get nice 86 | # stack traces when something crashes 87 | export STRIP_BINARIES=no 88 | 89 | # Enable systemd-coredump 90 | if ! coredumpctl_init; then 91 | echo >&2 "Failed to configure systemd-coredump/coredumpctl" 92 | exit 1 93 | fi 94 | 95 | # Disable swap, since it seems to cause CPU soft lock-ups in some cases 96 | swapoff -av 97 | 98 | # As running integration tests with broken systemd can be quite time consuming 99 | # (usually we need to wait for the test to timeout, see $QEMU_TIMEOUT and 100 | # $NSPAWN_TIMEOUT above), let's try to sanity check systemd first by running 101 | # the basic integration test under systemd-nspawn 102 | # 103 | # If the sanity check passes we can be at least somewhat sure the systemd 104 | # 'core' is stable and we can run the rest of the selected integration tests. 105 | # 1) Run it under systemd-nspawn 106 | export TESTDIR="/var/tmp/TEST-01-BASIC_sanitizers-nspawn" 107 | rm -fr "$TESTDIR" 108 | exectask "TEST-01-BASIC_sanitizers-nspawn" "make -C test/TEST-01-BASIC clean setup run clean-again TEST_NO_QEMU=1 && touch $TESTDIR/pass" 109 | NSPAWN_EC=$? 110 | # Each integration test dumps the system journal when something breaks 111 | [[ ! -f "$TESTDIR/pass" ]] && rsync -aq "$TESTDIR/system.journal" "$LOGDIR/${TESTDIR##*/}/" 112 | 113 | if [[ $NSPAWN_EC -eq 0 ]]; then 114 | # 2) The sanity check passed, let's run the other half of the TEST-01-BASIC 115 | # (under QEMU) and possibly other selected tests 116 | export TESTDIR="/var/tmp/systemd-test-TEST-01-BASIC_sanitizers-qemu" 117 | rm -fr "$TESTDIR" 118 | exectask "TEST-01-BASIC_sanitizers-qemu" "make -C test/TEST-01-BASIC clean setup run TEST_NO_NSPAWN=1 && touch $TESTDIR/pass" 119 | 120 | # Prepare environment for the systemd-networkd testsuite 121 | systemctl disable --now dhcpcd dnsmasq 122 | systemctl reload dbus.service 123 | 124 | # Run the systemd-networkd testsuite "in the background" while we run other 125 | # integration tests, since it doesn't require much resources and should not 126 | # interfere with them (and vice versa), saving a non-insignificant amount 127 | # of time 128 | exectask_p "systemd-networkd-tests.py" \ 129 | "/bin/time -v -- timeout -k 60s 60m test/test-network/systemd-networkd-tests.py --build-dir=$BUILD_DIR --debug --asan-options=$ASAN_OPTIONS --ubsan-options=$UBSAN_OPTIONS" 130 | 131 | # Run certain other integration tests under sanitizers to cover bigger 132 | # systemd subcomponents (but only if TEST-01-BASIC passed, so we can 133 | # be somewhat sure the 'base' systemd components work). 134 | CHECK_LIST=() 135 | INTEGRATION_TESTS=( 136 | test/TEST-04-JOURNAL # systemd-journald 137 | test/TEST-07-PID1 # PID1 & core stuff 138 | test/TEST-13-NSPAWN-SMOKE # Old test name for stable branches 139 | test/TEST-13-NSPAWN # systemd-nspawn 140 | test/TEST-15-DROPIN # dropin logic 141 | test/TEST-17-UDEV # systemd-udevd 142 | test/TEST-19-CGROUP 143 | test/TEST-21-DFUZZER # fuzz all systemd D-Bus interfaces 144 | test/TEST-22-TMPFILES # systemd-tmpfiles 145 | test/TEST-23-TYPE-EXEC # Old test name for stable branches 146 | test/TEST-23-UNIT-FILE 147 | test/TEST-24-CRYPTSETUP # systemd-cryptsetup + generator 148 | test/TEST-26-SYSTEMCTL # systemctl & friends 149 | test/TEST-29-PORTABLE # systemd-portabled 150 | test/TEST-34-DYNAMICUSERMIGRATE 151 | test/TEST-35-LOGIN # systemd-logind 152 | test/TEST-45-TIMEDATE # systemd-timedated 153 | test/TEST-46-HOMED # systemd-homed 154 | test/TEST-50-DISSECT # systemd-dissect 155 | test/TEST-54-CREDS # credentials & stuff 156 | test/TEST-55-OOMD # systemd-oomd 157 | test/TEST-58-REPART # systemd-repart 158 | test/TEST-64-UDEV-STORAGE # systemd-udevd with various storage setups 159 | test/TEST-65-ANALYZE # systemd-analyze 160 | test/TEST-70-TPM2 # systemd-cryptenroll 161 | test/TEST-71-HOSTNAME # systemd-hostnamed 162 | test/TEST-72-SYSUPDATE # systemd-sysupdate 163 | test/TEST-73-LOCALE # systemd-localed 164 | test/TEST-74-AUX-UTILS # auxiliary utils (busctl, sd-delta, sd-mount, ...) 165 | test/TEST-75-RESOLVED # systemd-resolved 166 | test/TEST-81-GENERATORS # various systemd generators 167 | ) 168 | 169 | for t in "${INTEGRATION_TESTS[@]}"; do 170 | # Some of the newer tests might not be available in stable branches, 171 | # so let's skip them instead of failing 172 | if [[ ! -d "$t" ]]; then 173 | echo "Test '$t' is not available, skipping..." 174 | continue 175 | fi 176 | 177 | export TEST_PARALLELIZE=1 178 | export QEMU_SMP=$OPTIMAL_QEMU_SMP 179 | # Set the test dir to something predictable so we can refer to it later 180 | export TESTDIR="/var/tmp/systemd-test-${t##*/}" 181 | 182 | # Suffix the $TESTDIR of each retry with an index to tell them apart 183 | export MANGLE_TESTDIR=1 184 | exectask_retry_p "${t##*/}" "/bin/time -v -- make -C $t setup run && touch \$TESTDIR/pass" 185 | 186 | # Retried tasks are suffixed with an index, so update the $CHECK_LIST 187 | # array with all possible task names correctly find the respective journals 188 | # shellcheck disable=SC2207 189 | CHECK_LIST+=($(seq -f "${t}_%g" 1 "$TASK_RETRY_DEFAULT")) 190 | done 191 | 192 | exectask_p_finish 193 | 194 | # Save journals created by integration tests 195 | for t in "TEST-01-BASIC_sanitizers-qemu" "${CHECK_LIST[@]}"; do 196 | testdir="/var/tmp/systemd-test-${t##*/}" 197 | if [[ -f "$testdir/system.journal" ]]; then 198 | # Filter out test-specific coredumps which are usually intentional 199 | # Note: $COREDUMPCTL_EXCLUDE_MAP resides in common/utils.sh 200 | # Note2: since all tests in this run are using the `exectask_retry` 201 | # runner, they're always suffixed with '_X' 202 | if [[ -v "COREDUMPCTL_EXCLUDE_MAP[${t%_[0-9]}]" ]]; then 203 | export COREDUMPCTL_EXCLUDE_RX="${COREDUMPCTL_EXCLUDE_MAP[${t%_[0-9]}]}" 204 | fi 205 | # Attempt to collect coredumps from test-specific journals as well 206 | exectask "${t##*/}_coredumpctl_collect" "coredumpctl_collect '$testdir/'" 207 | # Make sure to not propagate the custom coredumpctl filter override 208 | [[ -v COREDUMPCTL_EXCLUDE_RX ]] && unset -v COREDUMPCTL_EXCLUDE_RX 209 | 210 | # Check for sanitizer errors in test journals 211 | exectask "${t##*/}_sanitizer_errors" "$BUILD_DIR/journalctl -o short-monotonic --no-hostname --file $testdir/system.journal | check_for_sanitizer_errors" 212 | # Keep the journal files only if the associated test case failed 213 | if [[ ! -f "$testdir/pass" ]]; then 214 | rsync -aq "$testdir/system.journal" "$LOGDIR/${t##*/}/" 215 | fi 216 | fi 217 | done 218 | 219 | exectask "check-networkd-log-for-sanitizer-errors" "cat $LOGDIR/systemd-networkd-tests.py*.log | check_for_sanitizer_errors" 220 | fi 221 | 222 | # Check the test logs for sanitizer errors as well, since some tests may 223 | # output the "interesting" information only to the console. 224 | _check_test_logs_for_sanitizer_errors() { 225 | local ec=0 226 | 227 | while read -r file; do 228 | echo "*** Processing file $file ***" 229 | check_for_sanitizer_errors < "$file" || ec=1 230 | done < <(find "$LOGDIR" -maxdepth 1 -name "TEST-*.log" ! -name "*_sanitizer_*" ! -name "*_coredumpctl_*") 231 | 232 | return $ec 233 | } 234 | exectask "test_logs_sanitizer_errors" "_check_test_logs_for_sanitizer_errors" 235 | exectask "check-journal-for-sanitizer-errors" "journalctl -o short-monotonic --no-hostname -b | check_for_sanitizer_errors" 236 | # Collect coredumps using the coredumpctl utility, if any 237 | exectask "coredumpctl_collect" "coredumpctl_collect" 238 | 239 | # Summary 240 | show_task_summary 241 | finish_and_exit 242 | -------------------------------------------------------------------------------- /common/task-control.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/bash 2 | 3 | set -u 4 | set -o pipefail 5 | 6 | # Internal logging helpers which make use of the internal call stack to get 7 | # the function name of the caller 8 | _log() { echo "[${FUNCNAME[1]}] $1"; } 9 | _err() { echo >&2 "[${FUNCNAME[1]}] $1"; } 10 | _echo() { [[ "${TASK_LOG_LEVEL:-1}" -ne 0 ]] && echo "$@"; } 11 | 12 | if [[ -n "$1" ]]; then 13 | LOGDIR="$(mktemp -d "$PWD/$1.XXX")" 14 | else 15 | LOGDIR="$(mktemp -d "$PWD/testsuite-logs.XXX")" 16 | fi 17 | declare -r LOGDIR 18 | 19 | # We need to use files to track the passed/failed tasks, as in certain cases 20 | # we use subprocesses which can't modify variables in the parent process 21 | declare -r PASSED_TASKS_STATE="$LOGDIR/.passed_tasks" 22 | declare -r FAILED_TASKS_STATE="$LOGDIR/.failed_tasks" 23 | # Initialize the state files 24 | : >"$PASSED_TASKS_STATE" 25 | : >"$FAILED_TASKS_STATE" 26 | # Variables for parallel tasks 27 | declare -A TASK_QUEUE=() 28 | # Default number of retries for exectask_retry() 29 | declare -ri TASK_RETRY_DEFAULT=2 30 | # Try to determine the optimal values for parallel execution using the nproc 31 | # utility. If that fails, fall back to using default values for necessary 32 | # variables. 33 | if NPROC=$(nproc); then 34 | OPTIMAL_QEMU_SMP=2 35 | MAX_QUEUE_SIZE=$((NPROC / OPTIMAL_QEMU_SMP)) 36 | if [[ $MAX_QUEUE_SIZE -lt 1 ]]; then 37 | # We have enough CPUs for only one concurrent task 38 | OPTIMAL_QEMU_SMP=$NPROC 39 | MAX_QUEUE_SIZE=1 40 | elif [[ $MAX_QUEUE_SIZE -gt 4 ]]; then 41 | # Cap the max # of parallel jobs, otherwise we start hitting I/O limits 42 | # causing unexpected test fails 43 | MAX_QUEUE_SIZE=4 44 | OPTIMAL_QEMU_SMP=$((NPROC / MAX_QUEUE_SIZE)) 45 | fi 46 | else 47 | # Using nproc failed, let's fall back to defaults, which can be overridden 48 | # from the outside. 49 | MAX_QUEUE_SIZE=${MAX_QUEUE_SIZE:-1} 50 | OPTIMAL_QEMU_SMP=${OPTIMAL_QEMU_SMP:-1} 51 | fi 52 | 53 | echo "[TASK-CONTROL] OPTIMAL_QEMU_SMP = $OPTIMAL_QEMU_SMP" 54 | echo "[TASK-CONTROL] MAX_QUEUE_SIZE = $MAX_QUEUE_SIZE" 55 | 56 | # Active wait for PID to finish 57 | # - print '.' every 10 seconds 58 | # - return the exit code of the waited for process 59 | # Arguments 60 | # - PID (must be a child of current shell) 61 | waitforpid() { 62 | local pid="${1:?Missing PID}" 63 | local counter=0 64 | local ec 65 | SECONDS=0 66 | 67 | _echo "Waiting for PID $pid to finish" 68 | while kill -0 "$pid" 2>/dev/null; do 69 | if ((counter++ % 100 == 0)); then 70 | _echo -n "." 71 | fi 72 | sleep .1 73 | done 74 | 75 | wait "$pid" 76 | ec=$? 77 | 78 | _echo 79 | _echo "PID $pid finished with EC $ec in ${SECONDS}s" 80 | 81 | return $ec 82 | } 83 | 84 | # Convert passed exit code to a "human readable" message 85 | # - EC == 0: PASS, FAIL otherwise 86 | # - rename the log file accordingly (PASS/FAIL suffix) 87 | # - update internal counters 88 | # - dump the passed log file in case of FAIL 89 | # Arguments 90 | # $1 - exit code to process 91 | # $2 - path to log file "belonging" to the exit code 92 | # $3 - task name 93 | # $4 - ignore EC (i.e. don't update statistics with this task's results) 94 | # takes int (0: don't ignore, !0: ignore; default: 0) [optional] 95 | printresult() { 96 | local task_ec="${1:?Missing task exit code}" 97 | local task_logfile="${2:?Missing task log file}" 98 | local task_name="${3:?Missing task name}" 99 | local ignore_ec="${4:-0}" 100 | # Let's rename the target log file according to the test result (PASS/FAIL) 101 | local logfile_base="${task_logfile%.*}" # Log file path without the extension 102 | local logfile_ext="${task_logfile##*.}" # Log file extension without the leading dot 103 | local new_logfile 104 | 105 | # Determine the log's new name 106 | if [[ $task_ec -eq 0 ]]; then 107 | new_logfile="${logfile_base}_PASS.${logfile_ext}" 108 | else 109 | new_logfile="${logfile_base}_FAIL.${logfile_ext}" 110 | fi 111 | 112 | # Attempt to rename the log file. If we don't succeed, continue with the old one 113 | if mv "$task_logfile" "$new_logfile"; then 114 | task_logfile="$new_logfile" 115 | else 116 | _err "Log rename failed" 117 | fi 118 | 119 | # Don't update internal counters if we want to ignore task's EC 120 | if [[ $ignore_ec -eq 0 ]]; then 121 | if [[ $task_ec -eq 0 ]]; then 122 | echo "$task_name" >>"$PASSED_TASKS_STATE" 123 | echo "[RESULT] $task_name - PASS (log file: $task_logfile)" 124 | else 125 | cat "$task_logfile" 126 | echo "$task_name" >>"$FAILED_TASKS_STATE" 127 | echo "[RESULT] $task_name - FAIL (EC: $task_ec) (log file: $task_logfile)" 128 | fi 129 | else 130 | echo "[IGNORED RESULT] $task_name - EC: $task_ec (log file: $task_logfile)" 131 | fi 132 | } 133 | 134 | # Execute given task "silently": 135 | # - redirect stdout/stderr to a given log file 136 | # - show a simple progress "bar" 137 | # - dump the log on error 138 | # Arguments 139 | # $1 - task name 140 | # $2 - task command 141 | # $3 - ignore EC (i.e. don't update statistics with this task's results) 142 | # takes int (0: don't ignore, !0: ignore; default: 0) [optional] 143 | exectask() { 144 | local task_name="${1:?Missing task name}" 145 | local task_command="${2:?Missing task command}" 146 | local ignore_ec="${3:-0}" 147 | local logfile="$LOGDIR/$task_name.log" 148 | touch "$logfile" 149 | 150 | echo "[TASK] $task_name ($task_command)" 151 | echo "[TASK START] $(date)" >>"$logfile" 152 | 153 | # shellcheck disable=SC2086 154 | eval $task_command &>>"$logfile" & 155 | local pid=$! 156 | waitforpid $pid 157 | local ec=$? 158 | echo "[TASK END] $(date)" >>"$logfile" 159 | 160 | printresult $ec "$logfile" "$task_name" "$ignore_ec" 161 | echo 162 | 163 | return $ec 164 | } 165 | 166 | # Execute given task "silently" and retry it n-times in case the task fails: 167 | # - redirect stdout/stderr to a given log file 168 | # - show a simple progress "bar" 169 | # - dump the log on error 170 | # - retry the task up to n times in case it fails 171 | # Essentially the same function as exectask(), but for flaky tests. 172 | # 173 | # Arguments 174 | # $1 - task name 175 | # $2 - task command 176 | # $3 - # of retries (default: 3) [optional] 177 | exectask_retry() { 178 | local task_name="${1:?Missing task name}" 179 | local task_command="${2:?Missing task command}" 180 | local retries="${3:-$TASK_RETRY_DEFAULT}" 181 | local ec=0 182 | local orig_testdir orig_nspawn_arguments 183 | 184 | for ((i = 1; i <= retries; i++)); do 185 | local logfile="$LOGDIR/${task_name}_${i}.log" 186 | local pid 187 | 188 | touch "$logfile" 189 | 190 | echo "[TASK] $task_name ($task_command) [try $i/$retries]" 191 | echo "[TASK START] $(date)" >>"$logfile" 192 | 193 | # Make sure each retry has a unique state dir ($TESTDIR) and container 194 | # name (passed in $NSPAWN_ARGUMENTS), so we don't overwrite results 195 | # of previous retries or die because of a name clash. 196 | # Note: this is relevant only for the integration tests (test/TEST-??-*) 197 | if [[ "${MANGLE_TESTDIR:-0}" -ne 0 ]]; then 198 | # Suffix the $TESTDIR for each retry by its index if requested 199 | orig_testdir="${orig_testdir:-$TESTDIR}" 200 | export TESTDIR="${orig_testdir}_${i}" 201 | mkdir -p "$TESTDIR" 202 | rm -f "$TESTDIR/pass" 203 | 204 | # Also, set a unique name for each nspawn container to prevent scope clash 205 | orig_nspawn_arguments="${orig_nspawn_arguments:-${NSPAWN_ARGUMENTS:-}}" 206 | export NSPAWN_ARGUMENTS="$orig_nspawn_arguments --machine=${task_name}--${i}" 207 | fi 208 | 209 | # shellcheck disable=SC2086 210 | eval $task_command &>>"$logfile" & 211 | pid=$! 212 | waitforpid $pid 213 | ec=$? 214 | echo "[TASK END] $(date)" >>"$logfile" 215 | 216 | if [[ $ec -eq 0 ]]; then 217 | # Task passed => report the result & bail out early 218 | printresult $ec "$logfile" "$task_name" 0 219 | echo 220 | break 221 | else 222 | # Task failed => check if we still have retries left. If so, report 223 | # the result as "ignored" and continue to the next retry. Otherwise, 224 | # report the result as failed. 225 | printresult $ec "$logfile" "$task_name" $((i < retries)) 226 | echo 227 | fi 228 | done 229 | 230 | return $ec 231 | } 232 | 233 | # Execute given task in parallel fashion: 234 | # - redirect stdout/stderr to a given log file 235 | # - return after inserting the task into the queue (or wait until there's 236 | # a free spot) 237 | # - dump the log on error 238 | # Arguments 239 | # $1 - task name 240 | # $2 - task command 241 | exectask_p() { 242 | local task_name="${1:?Missing task name}" 243 | local task_command="${2:?Missing task command}" 244 | local key 245 | 246 | while [[ ${#TASK_QUEUE[@]} -ge $MAX_QUEUE_SIZE ]]; do 247 | for key in "${!TASK_QUEUE[@]}"; do 248 | if ! kill -0 "${TASK_QUEUE[$key]}" &>/dev/null; then 249 | # Task has finished, drop it from the queue 250 | wait "${TASK_QUEUE[$key]}" 251 | unset "TASK_QUEUE[$key]" 252 | # Break from inner for loop and outer while loop to skip 253 | # the sleep below when we find a free slot in the queue 254 | break 2 255 | fi 256 | done 257 | 258 | # Precisely* calculated constant to keep the spinlock from burning the CPU(s) 259 | sleep 0.01 260 | done 261 | 262 | TASK_LOG_LEVEL=0 exectask "$task_name" "$task_command" & 263 | TASK_QUEUE[$task_name]=$! 264 | 265 | return 0 266 | } 267 | 268 | # Execute given task in parallel fashion: 269 | # - redirect stdout/stderr to a given log file 270 | # - return after inserting the task into the queue (or wait until there's 271 | # a free spot) 272 | # - dump the log on error 273 | # Arguments 274 | # $1 - task name 275 | # $2 - task command 276 | # $3 - # of retries (default: 3) [optional] 277 | exectask_retry_p() { 278 | local task_name="${1:?Missing task name}" 279 | local task_command="${2:?Missing task command}" 280 | local retries="${3:-$TASK_RETRY_DEFAULT}" 281 | local key 282 | 283 | while [[ ${#TASK_QUEUE[@]} -ge $MAX_QUEUE_SIZE ]]; do 284 | for key in "${!TASK_QUEUE[@]}"; do 285 | if ! kill -0 "${TASK_QUEUE[$key]}" &>/dev/null; then 286 | # Task has finished, drop it from the queue 287 | wait "${TASK_QUEUE[$key]}" 288 | unset "TASK_QUEUE[$key]" 289 | # Break from inner for loop and outer while loop to skip 290 | # the sleep below when we find a free slot in the queue 291 | break 2 292 | fi 293 | done 294 | 295 | # Precisely* calculated constant to keep the spinlock from burning the CPU(s) 296 | sleep 0.01 297 | done 298 | 299 | TASK_LOG_LEVEL=0 exectask_retry "$task_name" "$task_command" "$retries" & 300 | TASK_QUEUE[$task_name]=$! 301 | 302 | return 0 303 | } 304 | # Wait for the remaining tasks in the parallel tasks queue 305 | exectask_p_finish() { 306 | local key 307 | 308 | echo "[INFO] Waiting for remaining running parallel tasks" 309 | 310 | for key in "${!TASK_QUEUE[@]}"; do 311 | echo "[INFO] Waiting for task '$key' to finish..." 312 | wait "${TASK_QUEUE[$key]}" 313 | unset "TASK_QUEUE[$key]" 314 | done 315 | } 316 | 317 | # Show summary about executed tasks 318 | show_task_summary() { 319 | local failed passed 320 | 321 | failed="$(wc -l <"$FAILED_TASKS_STATE")" 322 | passed="$(wc -l <"$PASSED_TASKS_STATE")" 323 | 324 | echo 325 | echo "TEST SUMMARY:" 326 | echo "-------------" 327 | echo "PASSED: $passed" 328 | echo "FAILED: $failed" 329 | echo "TOTAL: $((passed + failed))" 330 | 331 | if [[ $failed -ne 0 ]]; then 332 | echo 333 | echo "FAILED TASKS:" 334 | echo "-------------" 335 | sort "$FAILED_TASKS_STATE" 336 | fi 337 | } 338 | 339 | finish_and_exit() { 340 | local failed 341 | 342 | failed="$(wc -l <"$FAILED_TASKS_STATE")" || exit 1 343 | [[ "$failed" -eq 0 ]] && exit 0 || exit 1 344 | } 345 | --------------------------------------------------------------------------------