├── utils ├── common │ ├── .gitattributes │ ├── parse_gtest_list.py │ ├── publish_subunit_result.py │ └── gtest2subunit.py ├── linux │ ├── .gitattributes │ ├── all.sh │ ├── pathutils.sh │ ├── exec.sh │ ├── openstack.sh │ ├── common.sh │ └── wsman.py └── windows │ ├── all.psm1 │ ├── run_with_timeout.ps1 │ ├── msys.psm1 │ ├── git.psm1 │ ├── common.ps1 │ ├── pathutils.psm1 │ ├── tests.psm1 │ └── windows.psm1 ├── tools ├── get_todos.sh ├── sysprep_windows.ps1 └── lint.sh ├── test_host ├── install_python_requirements.ps1 ├── set_userspace_crashdump_location.ps1 └── run_tests.ps1 └── get-bin.py /utils/common/.gitattributes: -------------------------------------------------------------------------------- 1 | * text eol=lf 2 | -------------------------------------------------------------------------------- /utils/linux/.gitattributes: -------------------------------------------------------------------------------- 1 | * text eol=lf 2 | -------------------------------------------------------------------------------- /tools/get_todos.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SCRIPT_DIR=$(dirname "$BASH_SOURCE") 4 | PROJ_DIR="$SCRIPT_DIR/../" 5 | 6 | pushd $PROJ_DIR 7 | grep -RPizo ' *# TODO.*\n( *#.*\n)*' 8 | -------------------------------------------------------------------------------- /test_host/install_python_requirements.ps1: -------------------------------------------------------------------------------- 1 | $ErrorActionPreference = "Stop" 2 | $ProgressPreference = "SilentlyContinue" 3 | 4 | python -m pip install ` 5 | os-testr six python-dateutil requests prettytable 6 | -------------------------------------------------------------------------------- /utils/linux/all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | basedir_utils=$(dirname "$BASH_SOURCE") 4 | 5 | source "$basedir_utils/common.sh" 6 | source "$basedir_utils/exec.sh" 7 | source "$basedir_utils/pathutils.sh" 8 | source "$basedir_utils/openstack.sh" 9 | -------------------------------------------------------------------------------- /utils/windows/all.psm1: -------------------------------------------------------------------------------- 1 | $scriptLocation = [System.IO.Path]::GetDirectoryName( 2 | $myInvocation.MyCommand.Definition) 3 | 4 | . "$scriptLocation\common.ps1" 5 | 6 | import-module "$scriptLocation\pathutils.psm1" 7 | import-module "$scriptLocation\windows.psm1" 8 | import-module "$scriptLocation\msys.psm1" 9 | import-module "$scriptLocation\tests.psm1" 10 | import-module "$scriptLocation\git.psm1" 11 | 12 | Export-ModuleMember -function "*" 13 | -------------------------------------------------------------------------------- /utils/common/parse_gtest_list.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import sys 4 | import re 5 | 6 | def strip_comments(line): 7 | return re.sub(" *# .*", "", line) 8 | 9 | 10 | def is_test_class_name(line): 11 | return re.match(r"[^ ].*\.", line) 12 | 13 | 14 | for line in sys.stdin: 15 | line = strip_comments(line).strip() 16 | 17 | if is_test_class_name(line): 18 | test_class_name = line.strip(".") 19 | else: 20 | test_name = line 21 | print("%s.%s" % (test_class_name, test_name)) 22 | -------------------------------------------------------------------------------- /utils/windows/run_with_timeout.ps1: -------------------------------------------------------------------------------- 1 | Param( 2 | [Parameter(Mandatory=$true)] 3 | [string]$cmd, 4 | [Parameter(Mandatory=$true)] 5 | [int]$timeoutSec, 6 | [string]$jobName 7 | ) 8 | 9 | $ErrorActionPreference = "Stop" 10 | $ProgressPreference = "SilentlyContinue" 11 | 12 | $scriptLocation = [System.IO.Path]::GetDirectoryName( 13 | $myInvocation.MyCommand.Definition) 14 | 15 | . "$scriptLocation\common.ps1" 16 | 17 | import-module "$scriptLocation\windows.psm1" 18 | 19 | run_as_job $jobName 20 | _iex_with_timeout $cmd $timeoutSec 21 | -------------------------------------------------------------------------------- /utils/windows/msys.psm1: -------------------------------------------------------------------------------- 1 | $scriptLocation = [System.IO.Path]::GetDirectoryName( 2 | $myInvocation.MyCommand.Definition) 3 | 4 | . "$scriptLocation\common.ps1" 5 | 6 | function tar_msys() { 7 | # We're doing some plumbing to get tar working properly without 8 | # messing with PATH. This probably means that this can't be piped. 9 | # 10 | # msys provides tar. Apparently, Windows 10 includes tarbsd as well, 11 | # but it won't work with msys' bzip2. We don't want to add 12 | # the msys' bin dir to PATH "globally", as that will break other 13 | # binaries, e.g. "cmd". 14 | 15 | $pathBackup = "$($env:PATH)" 16 | $env:PATH ="$msysBinDir;$($env:PATH)" 17 | 18 | try { 19 | tar.exe @args 20 | if ($LASTEXITCODE) { 21 | throw "Command failed: tar $args" 22 | } 23 | } 24 | finally { 25 | $env:PATH = $pathBackup 26 | } 27 | } 28 | 29 | function convert_to_msys_path($path) { 30 | # We'll need posix paths. 31 | # c:\test\abcd becomes /c/test/abcd 32 | return ($path -replace "^([a-z]):\\",'/$1/') -replace "\\", "/" 33 | } 34 | -------------------------------------------------------------------------------- /tools/sysprep_windows.ps1: -------------------------------------------------------------------------------- 1 | # We need to run sysprep each time we update the Windows image. 2 | 3 | $cbsInitDir = "C:\Program Files\Cloudbase Solutions\Cloudbase-Init" 4 | $cbsInitLogDir = "$cbsInitDir\log" 5 | $unattendFile = "$cbsInitDir\conf\Unattend.xml" 6 | 7 | function clear_eventlog () 8 | { 9 | $Logs = Get-EventLog -List | ForEach {$_.Log} 10 | $Logs | % {Clear-EventLog -Log $_ } 11 | Get-EventLog -List 12 | } 13 | 14 | function cleanup_cbsinit () { 15 | get-service cloudbase* | stop-service 16 | rm -Recurse -Force "$cbsInitLogDir\*" 17 | } 18 | 19 | function remove_unneeded_apps () { 20 | # Some pre-installed games may prevent us from doing the sysprep. 21 | $apps = Get-AppxPackage -AllUsers ` 22 | | ? {$_.Name -notmatch "microsoft|windows|-|InputApp"} 23 | $apps += Get-AppxPackage -AllUsers | ? {$_.Name -match "bing" } 24 | 25 | $apps | % { echo "Removing $_.Name"; Remove-AppxPackage -AllUsers $_ } 26 | } 27 | 28 | 29 | clear_eventlog 30 | remove_unneeded_apps 31 | 32 | ipconfig /release 33 | 34 | C:\Windows\System32\Sysprep\sysprep.exe ` 35 | /generalize /oobe /shutdown ` 36 | /unattend:$unattendFile 37 | -------------------------------------------------------------------------------- /utils/linux/pathutils.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function ensure_dir_empty () { 4 | local DIR=$1 5 | 6 | log_summary "Cleanning up dir: $1" 7 | rm -rf $DIR 8 | mkdir -p $DIR 9 | } 10 | 11 | function cifs_to_unc_path () { 12 | echo $1 | tr / "\\" 2> /dev/null 13 | } 14 | 15 | function ensure_share_unmounted () { 16 | local MOUNT=$1 17 | local MOUNTPOINTS 18 | local MOUNTED_SHARE 19 | 20 | MOUNT=$(echo $MOUNT | tr "\\" "/" 2> /dev/null) 21 | MOUNTPOINTS=$(mount | tr "\\" "/" 2> /dev/null | \ 22 | grep -E "(^| )$MOUNT " | awk '{print $3}') 23 | MOUNTED_SHARE=$(mount | tr "\\" "/" 2> /dev/null | \ 24 | grep -E "(^| )$MOUNT " | awk 'NR==1 {print $1}') 25 | 26 | if [[ -z $MOUNTPOINTS ]]; then 27 | log_summary "\"$MOUNT\" is not mounted. Skipping unmount." 28 | else 29 | for mountpoint in $MOUNTPOINTS; do 30 | log_summary "Unmounting \"$MOUNTED_SHARE\" - \"$mountpoint\"." 31 | sudo umount $mountpoint 32 | done 33 | 34 | if [[ $(is_wsl) ]]; then 35 | net.exe use $(cifs_to_unc_path $MOUNTED_SHARE) /delete || \ 36 | log_summary "Failed to remove SMB mapping through net use." 37 | fi 38 | fi 39 | } 40 | -------------------------------------------------------------------------------- /utils/windows/git.psm1: -------------------------------------------------------------------------------- 1 | $scriptLocation = [System.IO.Path]::GetDirectoryName( 2 | $myInvocation.MyCommand.Definition) 3 | 4 | . "$scriptLocation\common.ps1" 5 | 6 | function git_clone_pull($path, $url, $ref="main", $shallow=$false) 7 | { 8 | log_message "Cloning / pulling: $url, branch: $ref. Path: $path." 9 | 10 | pushd . 11 | try 12 | { 13 | if (!(Test-Path -path $path)) 14 | { 15 | if ($shallow) { 16 | safe_exec "git clone -q -b $ref $url $path --depth=1" 17 | } 18 | else { 19 | safe_exec "git clone -q $url $path" 20 | } 21 | 22 | cd $path 23 | } 24 | else 25 | { 26 | cd $path 27 | 28 | safe_exec "git remote set-url origin $url" 29 | safe_exec "git reset --hard" 30 | safe_exec "git clean -f -d" 31 | safe_exec "git fetch" 32 | } 33 | 34 | safe_exec "git checkout $ref" 35 | 36 | if ((git tag) -contains $ref) { 37 | log_message "Got tag $ref instead of a branch." 38 | log_message "Skipping doing a pull." 39 | } 40 | elseif ($(git log -1 --pretty=format:"%H").StartsWith($ref)){ 41 | log_message "Got a commit id instead of a branch." 42 | log_message "Skipping doing a pull." 43 | } 44 | else { 45 | safe_exec "git pull" 46 | } 47 | } 48 | finally 49 | { 50 | popd 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /tools/lint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SCRIPT_DIR=$(dirname "$BASH_SOURCE") 4 | PROJ_DIR=$(realpath "$SCRIPT_DIR/../") 5 | 6 | FAILED=0 7 | 8 | function run_check() { 9 | local pattern="$1" 10 | local err_msg="$2" 11 | 12 | local occurrences 13 | occurrences=$(grep -nRIP "$pattern" \ 14 | --exclude-dir='.*' $PROJ_DIR) 15 | if [[ ! -z $occurrences ]]; then 16 | FAILED=1 17 | echo -e "$err_msg\n" 18 | echo -e "$occurrences\n" 19 | fi 20 | } 21 | 22 | function check_local_command_substitution() { 23 | echo "Checking for misuse of local variables and command substitution." 24 | 25 | pattern='local [a-zA-Z0-9_]+=(\$\(|`)' 26 | err_msg=" 27 | Local variables are used in conjunction with command substitution. This will 28 | prevent non-zero return codes from being intercepted. Please declare the 29 | local variables separately." 30 | 31 | run_check "$pattern" "$err_msg" 32 | } 33 | 34 | function check_empty_line_whitespaces() { 35 | echo "Checking for empty lines containing whitespaces." 36 | 37 | pattern='^[ ]+$' 38 | err_msg="Empty line contains whitespaces." 39 | 40 | run_check "$pattern" "$err_msg" 41 | } 42 | 43 | function check_trailing_whitespaces() { 44 | echo "Checking for trailing whitespaces." 45 | 46 | pattern='[^ ]* +$' 47 | err_msg="Trailing whitespaces." 48 | 49 | run_check "$pattern" "$err_msg" 50 | } 51 | 52 | 53 | check_local_command_substitution 54 | check_empty_line_whitespaces 55 | check_trailing_whitespaces 56 | 57 | exit $FAILED 58 | -------------------------------------------------------------------------------- /test_host/set_userspace_crashdump_location.ps1: -------------------------------------------------------------------------------- 1 | Param( 2 | [string]$dumpDir, 3 | [int]$dumpCount, 4 | [int]$dumpType 5 | ) 6 | 7 | $ErrorActionPreference = "Stop" 8 | $ProgressPreference = "SilentlyContinue" 9 | 10 | $scriptLocation = [System.IO.Path]::GetDirectoryName( 11 | $myInvocation.MyCommand.Definition) 12 | 13 | import-module "$scriptLocation\..\utils\windows\all.psm1" 14 | 15 | 16 | $werRegPath = "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting" 17 | $localDumpsRegPath = "$werRegPath\LocalDumps" 18 | 19 | if (!(Test-path $localDumpsRegPath)) { 20 | log_message "Creating $localDumpsRegPath" 21 | New-Item -Path $localDumpsRegPath 22 | } 23 | 24 | if ($PSBoundParameters.ContainsKey('dumpDir')) { 25 | log_message "setting $localDumpsRegPath\DumpFolder to $dumpDir" 26 | Set-ItemProperty ` 27 | -Path $localDumpsRegPath -Name "DumpFolder" ` 28 | -Value $dumpDir -Type ExpandString 29 | 30 | if (!(Test-path $dumpDir)) { 31 | log_message "creating folder $dumpDir" 32 | mkdir $dumpDir 33 | } 34 | } 35 | 36 | if ($PSBoundParameters.ContainsKey('dumpCount')) { 37 | log_message "setting $localDumpsRegPath\DumpCount to $dumpCount" 38 | Set-ItemProperty ` 39 | -Path $localDumpsRegPath -Name "DumpCount" ` 40 | -Value $dumpCount -Type DWord 41 | } 42 | 43 | if ($PSBoundParameters.ContainsKey('dumpType')) { 44 | log_message "setting $localDumpsRegPath\DumpType to $dumpType" 45 | Set-ItemProperty ` 46 | -Path $localDumpsRegPath -Name "DumpType" ` 47 | -Value $dumpType -Type DWord 48 | } 49 | -------------------------------------------------------------------------------- /utils/linux/exec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | basedir_utils=$(dirname "$BASH_SOURCE") 4 | 5 | function exec_with_retry2 () { 6 | local MAX_RETRIES=$1 7 | local INTERVAL=$2 8 | local COUNTER=0 9 | 10 | while [ $COUNTER -lt $MAX_RETRIES ]; do 11 | local EXIT=0 12 | eval '${@:3}' || EXIT=$? 13 | if [ $EXIT -eq 0 ]; then 14 | return 0 15 | fi 16 | let COUNTER=COUNTER+1 17 | 18 | if [ -n "$INTERVAL" ]; then 19 | sleep $INTERVAL 20 | fi 21 | done 22 | return $EXIT 23 | } 24 | 25 | function exec_with_retry () { 26 | local CMD=$1 27 | local MAX_RETRIES=${2-10} 28 | local INTERVAL=${3-0} 29 | 30 | exec_with_retry2 $MAX_RETRIES $INTERVAL $CMD 31 | } 32 | 33 | function run_wsman_cmd() { 34 | local HOST=$1 35 | local USERNAME=$2 36 | local CERT_PATH=$3 37 | local CERT_KEY_PATH=$4 38 | local CMD=$5 39 | local PS=$6 40 | 41 | local ARGS=("$basedir_utils/wsman.py" \ 42 | -U https://$HOST:5986/wsman \ 43 | -u $USERNAME -k $CERT_PATH -K $CERT_KEY_PATH) 44 | if [ ! -z $PS ]; then 45 | ARGS+=("-P") 46 | fi 47 | 48 | ARGS+=("$CMD") 49 | echo ${ARGS[@]} 50 | 51 | python ${ARGS[@]} 52 | } 53 | 54 | function run_wsman_ps() { 55 | local HOST=$1 56 | local USERNAME=$2 57 | local CERT_PATH=$3 58 | local CERT_KEY_PATH=$4 59 | local CMD="${@:5}" 60 | 61 | run_wsman_cmd $HOST $USERNAME $CERT_PATH \ 62 | $CERT_KEY_PATH "$CMD" "powershell" 63 | } 64 | 65 | function run_ssh_cmd () { 66 | local SSHUSER_HOST=$1 67 | local SSHKEY=$2 68 | local CMD="${@:3}" 69 | 70 | ssh -t -o 'PasswordAuthentication no' \ 71 | -o 'StrictHostKeyChecking no' \ 72 | -o 'UserKnownHostsFile /dev/null' \ 73 | -i $SSHKEY $SSHUSER_HOST "$CMD" 74 | } 75 | 76 | function wait_for_listening_port () { 77 | local HOST=$1 78 | local PORT=$2 79 | exec_with_retry "nc -z -w15 $HOST $PORT" 15 10 80 | } 81 | -------------------------------------------------------------------------------- /utils/windows/common.ps1: -------------------------------------------------------------------------------- 1 | $ErrorActionPreference = "Stop" 2 | $ProgressPreference = "SilentlyContinue" 3 | 4 | $scriptLocation = [System.IO.Path]::GetDirectoryName( 5 | $myInvocation.MyCommand.Definition) 6 | 7 | function get_utc_iso8601_time() { 8 | Get-Date(Get-Date).ToUniversalTime() -uformat '+%Y-%m-%dT%H:%M:%S.000Z' 9 | } 10 | 11 | function log_message($message) { 12 | echo "[$(get_utc_iso8601_time)] $message" 13 | } 14 | 15 | function iex_with_timeout() { 16 | Param( 17 | [Parameter(Mandatory=$true)] 18 | [string]$cmd, 19 | [Parameter(Mandatory=$true)] 20 | [int]$timeoutSec 21 | ) 22 | 23 | & "$scriptLocation\run_with_timeout.ps1" "$cmd" $timeoutSec 24 | } 25 | 26 | function _iex_with_timeout() { 27 | # This is not completely safe. If the job times out while having 28 | # children processes, its children will not be stopped. The calling 29 | # script may hang because of this, especially when ran remotely. 30 | Param( 31 | [Parameter(Mandatory=$true)] 32 | [string]$cmd, 33 | [Parameter(Mandatory=$true)] 34 | [int]$timeoutSec 35 | ) 36 | 37 | log_message "Executing cmd with timeout ($timeoutSec s): $cmd" 38 | $job = start-job -ArgumentList $cmd -ScriptBlock { 39 | param($c) 40 | iex $c 41 | if ($LASTEXITCODE) { 42 | throw "Command returned non-zero code($LASTEXITCODE): `"$c`"." 43 | } 44 | } 45 | 46 | try { 47 | wait-job $job -timeout $timeoutSec 48 | 49 | if ($job.State -notin @("Completed", "Failed")) { 50 | throw "Command timed out ($($timeoutSec)s): `"$cmd`"." 51 | } 52 | receive-job $job 53 | } 54 | finally { 55 | stop-job $job 56 | remove-job $job 57 | } 58 | } 59 | 60 | function safe_exec($cmd) { 61 | # The idea is to prevent powershell from treating stderr output 62 | # as an error when calling executables, relying on the return code 63 | # (which unfortunately doesn't always happen by default, 64 | # especially in case of remote sessions). 65 | cmd /c "$cmd 2>&1" 66 | if ($LASTEXITCODE) { 67 | throw "Command failed: $cmd" 68 | } 69 | } 70 | 71 | function get_unix_time () { 72 | [int](Get-Date(Get-Date).ToUniversalTime() -uformat "%s") 73 | } 74 | -------------------------------------------------------------------------------- /get-bin.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import argparse 3 | import requests 4 | import sys 5 | import urllib.parse 6 | from string import Template 7 | 8 | SHAMAN_SEARCH = 'https://shaman.ceph.com/api/search/?project=$project&distros=$distro/$distrover&ref=$branchname&sha1=$sha1' # noqa 9 | CHACRA_BIN = 'https://$chacra_host/binaries/$project/$ref/$sha1/$distro/$distrover/x86_64/flavors/default/$filename' # noqa 10 | 11 | PROJECT = 'ceph' 12 | DISTRO = 'windows' 13 | DISTROVER = '1809' 14 | BRANCHNAME = 'main' 15 | SHA1 = 'latest' 16 | FILENAME = 'ceph.zip' 17 | 18 | 19 | def getbin(project, distro, distrover, branchname, sha1, filename): 20 | resp = requests.get(Template(SHAMAN_SEARCH).substitute( 21 | project=project, 22 | distro=distro, 23 | distrover=distrover, 24 | branchname=branchname, 25 | sha1=sha1, 26 | )) 27 | resp.raise_for_status() 28 | resp_json = resp.json() 29 | if len(resp_json) == 0: 30 | raise RuntimeError(f'no results found at {resp.url}') 31 | chacra_host = urllib.parse.urlparse(resp_json[0]['url']).netloc 32 | chacra_ref = resp_json[0]['ref'] 33 | chacra_sha1 = resp_json[0]['sha1'] 34 | print('got chacra host {}, ref {}, sha1 {} from {}'.format( 35 | chacra_host, chacra_ref, chacra_sha1, resp.url)) 36 | resp = requests.get(Template(CHACRA_BIN).substitute( 37 | chacra_host=chacra_host, 38 | project=project, 39 | ref=chacra_ref, 40 | sha1=chacra_sha1, 41 | distro=distro, 42 | distrover=distrover, 43 | filename=filename, 44 | ), stream=True) 45 | resp.raise_for_status() 46 | print(f'got file from {resp.url}') 47 | with open(filename, 'wb') as f: 48 | for chunk in resp.iter_content(chunk_size=None, decode_unicode=True): 49 | print('.',) 50 | f.write(chunk) 51 | 52 | 53 | def main(): 54 | parser = argparse.ArgumentParser() 55 | parser.add_argument('--project', default=PROJECT) 56 | parser.add_argument('--distro', '-D', default=DISTRO) 57 | parser.add_argument('--distrover', '-V', default=DISTROVER) 58 | parser.add_argument('--branchname', '-b', default=BRANCHNAME) 59 | parser.add_argument('--sha1', '-s', default=SHA1) 60 | parser.add_argument('--filename', '-f', default=FILENAME) 61 | args = parser.parse_args() 62 | 63 | getbin(args.project, 64 | args.distro, 65 | args.distrover, 66 | args.branchname, 67 | args.sha1, 68 | args.filename) 69 | return 0 70 | 71 | 72 | if __name__ == '__main__': 73 | sys.exit(main()) 74 | -------------------------------------------------------------------------------- /utils/common/publish_subunit_result.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # A simple helper for publishing subunit test result records. 4 | 5 | import argparse 6 | import datetime 7 | import sys 8 | 9 | import six 10 | import subunit 11 | from subunit import iso8601 12 | 13 | parser = argparse.ArgumentParser( 14 | description='Helper publishing subunit test results.') 15 | parser.add_argument('--test-id', required=True) 16 | parser.add_argument('--status', default="success", 17 | choices=("exists", "xfail", "uxsuccess", 18 | "success", "fail", "skip", 19 | "none", "inprogress")) 20 | parser.add_argument('--start-time', 21 | help='Test start time in Epoch format.') 22 | parser.add_argument('--stop-time', 23 | help='Test stop time in Epoch format.') 24 | parser.add_argument('--stdin-attachment', action='store_true', 25 | help='Read the attachment from stdin.') 26 | parser.set_defaults(stdin_attachment=False) 27 | parser.add_argument('--attachment-path', 28 | help="Pass an attachment file path.") 29 | parser.add_argument('--attachment-string', 30 | help='Pass a string record attachment.') 31 | parser.add_argument('--output-file', 32 | help='If no output is specified, stdout will be used.') 33 | parser.add_argument('--attachment-name', 34 | default="details") 35 | parser.add_argument('--attachment-type', 36 | default="text/plain") 37 | 38 | 39 | args = parser.parse_args() 40 | 41 | start_time = None 42 | stop_time = None 43 | 44 | to_bytes = lambda x: six.b(x) if isinstance(x, six.string_types) else x 45 | 46 | if args.start_time: 47 | start_time = datetime.datetime.fromtimestamp( 48 | float(args.start_time)).replace(tzinfo=iso8601.UTC) 49 | if args.stop_time: 50 | stop_time = datetime.datetime.fromtimestamp( 51 | float(args.stop_time)).replace(tzinfo=iso8601.UTC) 52 | 53 | output_file = None 54 | try: 55 | if args.output_file: 56 | output_file = open(args.output_file, 'ab') 57 | output_handle = output_file 58 | else: 59 | output_handle = sys.stdout 60 | 61 | attachment = to_bytes(args.attachment_string or "") + b"\n" 62 | if args.stdin_attachment: 63 | attachment += sys.stdin.read() 64 | 65 | if args.attachment_path: 66 | with open(args.attachment_path, 'rb') as f: 67 | attachment += f.read() 68 | 69 | output = subunit.v2.StreamResultToBytes(output_handle) 70 | output.startTestRun() 71 | output.status(timestamp=start_time, 72 | test_id=args.test_id) 73 | output.status(test_status=args.status, 74 | timestamp=stop_time, test_id=args.test_id, 75 | file_name=args.attachment_name, 76 | file_bytes=to_bytes(attachment), 77 | mime_type=args.attachment_type) 78 | output.stopTestRun() 79 | finally: 80 | if output_file: 81 | output_file.close() 82 | 83 | -------------------------------------------------------------------------------- /utils/linux/openstack.sh: -------------------------------------------------------------------------------- 1 | function get_instance_state() { 2 | local INSTANCE_ID=$1 3 | ensure_vars_set INSTANCE_ID 4 | nova show $INSTANCE_ID | grep " status " | awk '{print $4}' 5 | } 6 | 7 | function wait_for_instance_state () { 8 | local INSTANCE_ID=$1 9 | local EXPECTED_STATE=$2 10 | local TIMEOUT=${3:-180} 11 | local POLL_INTERVAL=${4:-2} 12 | local INSTANCE_STATE 13 | 14 | SECONDS=0 15 | TRIES=0 16 | 17 | required_vars=(INSTANCE_ID EXPECTED_STATE TIMEOUT POLL_INTERVAL) 18 | ensure_vars_set $required_vars 19 | 20 | INSTANCE_STATE=$(get_instance_state $INSTANCE_ID) 21 | 22 | while [[ $SECONDS -lt $TIMEOUT ]] && \ 23 | [[ ! ( ${INSTANCE_STATE^^} =~ "ERROR" \ 24 | || $INSTANCE_STATE == $EXPECTED_STATE ) ]]; do 25 | 26 | sleep $POLL_INTERVAL 27 | INSTANCE_STATE=$(get_instance_state $INSTANCE_ID) 28 | done 29 | 30 | if [[ ${INSTANCE_STATE^^} =~ "ERROR" ]]; then 31 | nova show $INSTANCE_ID 32 | log_summary "Instance $INSTANCE_ID entered error state"\ 33 | "($INSTANCE_STATE)." 34 | return 1 35 | fi 36 | 37 | if [[ $INSTANCE_STATE != $EXPECTED_STATE ]]; then 38 | log_summary "Timeout ($SECONDS s) waiting for instance" \ 39 | "$INSTANCE_ID to become $EXPECTED_STATE." \ 40 | "Current state: $INSTANCE_STATE." 41 | return 1 42 | else 43 | log_summary "Instance $INSTANCE_ID reached expected" \ 44 | "state: $INSTANCE_STATE." 45 | fi 46 | } 47 | 48 | function wait_for_instance_boot () { 49 | local INSTANCE_ID=$1 50 | local TIMEOUT=${2:-180} 51 | local POLL_INTERVAL=${3:-2} 52 | 53 | required_vars=(INSTANCE_ID EXPECTED_STATE TIMEOUT POLL_INTERVAL) 54 | ensure_vars_set $required_vars 55 | 56 | wait_for_instance_state $INSTANCE_ID "ACTIVE" $TIMEOUT $POLL_INTERVAL 57 | } 58 | 59 | function boot_vm() { 60 | # This function doesn't wait for the instance to spawn, call 61 | # 'wait_for_instance_boot' after booting the vm. 62 | local VMID 63 | local _ERR_OPTS 64 | 65 | _ERR_OPTS=$(set +o | grep err) 66 | set +eE 67 | 68 | VMID=$(nova boot $@ | grep " id " | cut -d "|" -f 3) 69 | 70 | local NOVABOOT_EXIT=$? 71 | $_ERR_OPTS 72 | 73 | if [ $NOVABOOT_EXIT -ne 0 ]; then 74 | nova show "$VMID" 75 | nova delete $VMID 76 | log_summary "Failed to create VM: $VMID" 77 | return 1 78 | fi 79 | echo $VMID 80 | } 81 | 82 | function get_vm_ip() { 83 | local vmName=$1 84 | local vms 85 | local vmCount 86 | vms=`nova list | grep $vmName` 87 | vmCount=`echo $vms | wc -l` 88 | 89 | if [[ -z $vms ]]; then 90 | log_summary "Could not find vm $vmName." 91 | return 1 92 | fi 93 | 94 | if [[ $vmCount -gt 1 ]]; then 95 | log_summary "Found multiple vms." 96 | return 1 97 | fi 98 | 99 | local ip 100 | ip=`echo $vms | cut -d '|' -f 7 | sed -r 's/.*=//'` 101 | if [[ -z $ip ]]; then 102 | log_summary "Failed to retrieve vm \"$vmName\" ip." 103 | return 1; 104 | fi 105 | 106 | echo $ip 107 | } 108 | 109 | function delete_vm_if_exists() { 110 | local VM_ID=$1 111 | 112 | if [[ -z $VM_ID ]]; then 113 | log_summary "No vm id specified. Skipping delete." 114 | fi 115 | 116 | if [[ $(nova list | grep $VM_ID) ]]; then 117 | log_summary "Deleting vm $VM_ID" 118 | nova delete $VM_ID 119 | else 120 | log_summary "VM $VM_ID does not exist. Skipping delete." 121 | fi 122 | } 123 | -------------------------------------------------------------------------------- /utils/common/gtest2subunit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import argparse 4 | import datetime 5 | import sys 6 | 7 | import dateutil.parser 8 | import six 9 | import subunit 10 | from subunit import iso8601 11 | from xml.etree import ElementTree 12 | 13 | parser = argparse.ArgumentParser( 14 | description='Google test xml output to subunit converter.') 15 | parser.add_argument('--xml-path', required=True) 16 | parser.add_argument('--subunit-path', required=True) 17 | parser.add_argument('--test-prefix', 18 | help='Use the specified prefix for the test ids. ' 19 | 'This is useful when having multiple runs of the ' 20 | 'same tests (e.g. compiled for different ' 21 | 'architectures)') 22 | 23 | args = parser.parse_args() 24 | 25 | to_bytes = lambda x: six.b(x) if isinstance(x, six.string_types) else x 26 | 27 | class Gtest2Subunit(object): 28 | def __init__(self, gtest_xml_path, subunit_path, test_prefix=None): 29 | self._xml_path = gtest_xml_path 30 | self._subunit_path = subunit_path 31 | self._test_prefix = test_prefix 32 | 33 | def _emit_test_result(self, stream, test_id, status, details, 34 | start_time, stop_time): 35 | output = subunit.v2.StreamResultToBytes(stream) 36 | output.startTestRun() 37 | output.status(timestamp=start_time, 38 | test_id=test_id) 39 | output.status(test_status=status, 40 | timestamp=stop_time, test_id=test_id, 41 | file_name="details", 42 | file_bytes=to_bytes(details), 43 | mime_type="text/plain") 44 | output.stopTestRun() 45 | 46 | def convert(self): 47 | with open(self._subunit_path, 'ab') as f: 48 | self._convert(f) 49 | 50 | def _convert(self, subunit_stream): 51 | et = ElementTree.parse(self._xml_path) 52 | 53 | root = et.getroot() 54 | assert root.tag == "testsuites" 55 | 56 | for testsuite in root: 57 | testsuite_name = testsuite.get("name") 58 | 59 | assert testsuite.tag == "testsuite" 60 | assert testsuite_name 61 | 62 | testsuite_timestamp = testsuite.get("timestamp") 63 | 64 | for test in testsuite: 65 | test_name = test.get("name") 66 | assert test_name 67 | 68 | test_id = "%s.%s" % (testsuite_name, test_name) 69 | if self._test_prefix: 70 | test_id = "%s.%s" % (self._test_prefix, test_id) 71 | 72 | test_timestamp = ( 73 | test.get("timestamp") or 74 | testsuite_timestamp or 75 | datetime.datetime.utcnow().isoformat()) 76 | start_time = dateutil.parser.parse( 77 | test_timestamp).replace(tzinfo=iso8601.UTC) 78 | 79 | test_duration = float(test.get("time", 0)) 80 | end_time = start_time + datetime.timedelta( 81 | seconds=test_duration) 82 | 83 | failures = test.findall("./failure") 84 | failure_messages = [failure.get("message") 85 | for failure in failures] 86 | 87 | status = "fail" if failures else "success" 88 | 89 | error_details = "\n\n".join(failure_messages) 90 | self._emit_test_result( 91 | subunit_stream, test_id, status, 92 | to_bytes(error_details), 93 | start_time, end_time) 94 | 95 | 96 | if __name__ == "__main__": 97 | converter = Gtest2Subunit(args.xml_path, 98 | args.subunit_path, 99 | args.test_prefix) 100 | converter.convert() 101 | -------------------------------------------------------------------------------- /utils/linux/common.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | basedir_utils=$(dirname "$BASH_SOURCE") 4 | 5 | TIMESTAMP_FORMAT=${TIMESTAMP_FORMAT:-"%F_%H:%M:%S%:::z"} 6 | 7 | set -o pipefail 8 | 9 | function log_message () { 10 | local SCRIPT_NAME 11 | 12 | [[ $LOG_SCRIPT_NAME ]] && SCRIPT_NAME=" ($(basename $0))" 13 | 14 | local MSG=[$(date -uIseconds)] 15 | MSG="$MSG$SCRIPT_NAME" 16 | MSG="$MSG $@" 17 | 18 | echo -e "$MSG" 19 | } 20 | 21 | function log_warning () { 22 | log_message "WARNING: $@" 23 | } 24 | 25 | function log_summary () { 26 | local _XTRACE 27 | _XTRACE=$(set +o | grep xtrace) 28 | set +o xtrace 29 | log_message "$@" 30 | 31 | if [ ! -z $LOGGING_CONFIGURED ]; then 32 | log_message "$@" >&3 33 | log_message "$@" >> $LOG_SUMMARY_FILE 34 | fi 35 | 36 | # restore xtrace 37 | $_XTRACE 38 | } 39 | 40 | trap err_trap ERR 41 | function err_trap () { 42 | local r=$? 43 | set +o xtrace 44 | 45 | log_summary "${0##*/} failed." 46 | 47 | if [ ! -z $LOGGING_CONFIGURED ]; then 48 | log_summary "Full log: $LOG_FILE." 49 | tail -n 15 $LOG_FILE >&3 50 | fi 51 | 52 | exit $r 53 | } 54 | 55 | function die () { 56 | set +o xtrace 57 | log_summary "$@" 58 | 59 | exit 1 60 | } 61 | 62 | function setup_logging () { 63 | if [ ! -z $LOGGING_CONFIGURED ]; then 64 | # Logging already configured. 65 | return 66 | fi 67 | 68 | local default_log_name="$(basename $0 | sed 's/\..*//')" 69 | local log_dir=$1 70 | local log_name=${2:-$default_log_name} 71 | 72 | if [ -z $log_dir ]; then 73 | log_message "Log dir not specified." 74 | return 75 | fi 76 | 77 | mkdir -p $log_dir 78 | 79 | LOG_FILE="$log_dir/$log_name.log" 80 | LOG_SUMMARY_FILE="$log_dir/$log_name.summary.log" 81 | 82 | # Save original fds. 83 | exec 3>&1 84 | exec 4>&2 85 | 86 | exec 1> $LOG_FILE 2>&1 87 | rm -f $LOG_SUMMARY_FILE 88 | 89 | set -o xtrace 90 | LOGGING_CONFIGURED="1" 91 | } 92 | 93 | function ensure_vars_set () { 94 | local MISSING_VARS=() 95 | 96 | while test $# -gt 0 97 | do 98 | local var=$1 99 | 100 | if [[ -z ${!var} ]]; then 101 | MISSING_VARS+=($var) 102 | fi 103 | shift 104 | done 105 | 106 | if [ ! -z $MISSING_VARS ]; then 107 | die "The following variables must" \ 108 | "be set: ${MISSING_VARS[@]}" 109 | fi 110 | } 111 | 112 | function check_running_pid () { 113 | local pid=$1 114 | local stopped="" 115 | 116 | ps -p $pid &> /dev/null || stopped=1 117 | 118 | if [[ $stopped ]]; then 119 | return 1; 120 | fi 121 | } 122 | 123 | function kill_if_running () { 124 | local pid=$1 125 | local kill_message=${2:-"Killing process: $1."} 126 | local running="" 127 | 128 | check_running_pid $1 && running=1 129 | if [[ $running ]]; then 130 | log_message $kill_message 131 | kill -9 $pid &> /dev/null 132 | fi 133 | } 134 | 135 | function is_wsl () { 136 | cat /proc/version | grep Microsoft 137 | } 138 | 139 | function str_to_bool () { 140 | local str=${1,,} 141 | if [[ $str == "1" || $str == "yes" || $str == "true" ]]; then 142 | return 0 143 | else 144 | return 1 145 | fi 146 | } 147 | 148 | function log_git_info () { 149 | git status 150 | git remote -v 151 | git log --oneline -n 10 --format="%h %cI %s" 152 | } 153 | 154 | function log_ci_scripts_git_info() { 155 | pushd $basedir_utils 156 | log_git_info 157 | popd 158 | } 159 | 160 | function set_git_ci_creds () { 161 | # We may have to cherry pick some patches, in which case those 162 | # creds need to be set. 163 | git config --global user.email "android-ci@cloudbasesolutions.com" 164 | git config --global user.name "Android Cloudbase CI" 165 | } 166 | -------------------------------------------------------------------------------- /utils/linux/wsman.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Copyright 2013 Cloudbase Solutions Srl 4 | # All Rights Reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 7 | # not use this file except in compliance with the License. You may obtain 8 | # a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations 16 | # under the License. 17 | 18 | import base64 19 | import getopt 20 | import six 21 | import sys 22 | 23 | from winrm import protocol 24 | 25 | 26 | def print_usage(): 27 | print ("%s -U -u -p " 28 | "[-k ] [-K ] [-P] " 29 | " [cmd_args]" % 30 | sys.argv[0]) 31 | 32 | 33 | def parse_args(): 34 | # TODO: switch to argparse 35 | username = None 36 | password = None 37 | url = None 38 | cmd = None 39 | cert_pem_path = None 40 | cert_key_pem_path = None 41 | powershell = False 42 | 43 | try: 44 | show_usage = False 45 | opts, args = getopt.getopt(sys.argv[1:], "hU:u:p:c:k:K:P") 46 | for opt, arg in opts: 47 | if opt == "-h": 48 | show_usage = True 49 | if opt == "-U": 50 | url = arg 51 | elif opt == "-u": 52 | username = arg 53 | elif opt == "-p": 54 | password = arg 55 | elif opt == "-k": 56 | cert_pem_path = arg 57 | elif opt == "-K": 58 | cert_key_pem_path = arg 59 | elif opt == "-P": 60 | powershell = True 61 | 62 | cmd = args 63 | 64 | have_certs = cert_pem_path and cert_key_pem_path 65 | have_auth = username and (password or have_certs) 66 | if show_usage or not (url and have_auth and cmd): 67 | print_usage() 68 | exit(1) 69 | 70 | except getopt.GetoptError: 71 | print_usage() 72 | exit(1) 73 | 74 | return (url, username, password, cert_pem_path, 75 | cert_key_pem_path, cmd, powershell) 76 | 77 | def _parse_command(command, powershell): 78 | if isinstance(command, list) or isinstance(command, tuple): 79 | command = " ".join([six.text_type(c) for c in command]) 80 | 81 | if powershell: 82 | command = "$ProgressPreference = \"SilentlyContinue\"; " + command 83 | b64_command = base64.b64encode(command.encode("utf_16_le")) 84 | command = ("powershell.exe -ExecutionPolicy RemoteSigned " 85 | "-NonInteractive -EncodedCommand %s" % b64_command) 86 | return command 87 | 88 | def run_wsman_cmd(url, username, password, 89 | cert_pem_path, cert_key_pem_path, 90 | cmd, powershell): 91 | protocol.Protocol.DEFAULT_TIMEOUT = 3600 92 | 93 | cmd = _parse_command(cmd, powershell) 94 | use_cert = bool(cert_pem_path and cert_key_pem_path) 95 | transport = ("ssl" 96 | if use_cert else "plaintext") 97 | 98 | p = protocol.Protocol(endpoint=url, 99 | transport=transport, 100 | username=username, 101 | password=password, 102 | cert_pem=cert_pem_path, 103 | cert_key_pem=cert_key_pem_path) 104 | 105 | shell_id = p.open_shell() 106 | 107 | command_id = p.run_command(shell_id, cmd) 108 | std_out, std_err, status_code = p.get_command_output(shell_id, command_id) 109 | 110 | p.cleanup_command(shell_id, command_id) 111 | p.close_shell(shell_id) 112 | 113 | return (std_out, std_err, status_code) 114 | 115 | 116 | def main(): 117 | exit_code = 0 118 | 119 | (url, username, password, 120 | cert_pem_path, cert_key_pem_path, 121 | cmd, powershell) = parse_args() 122 | 123 | std_out, std_err, exit_code = run_wsman_cmd(url, username, password, 124 | cert_pem_path, 125 | cert_key_pem_path, 126 | cmd, 127 | powershell) 128 | sys.stderr.write(std_err) 129 | sys.stdout.write(std_out) 130 | 131 | sys.exit(exit_code) 132 | 133 | 134 | if __name__ == "__main__": 135 | main() 136 | -------------------------------------------------------------------------------- /utils/windows/pathutils.psm1: -------------------------------------------------------------------------------- 1 | $scriptLocation = [System.IO.Path]::GetDirectoryName( 2 | $myInvocation.MyCommand.Definition) 3 | 4 | . "$scriptLocation\common.ps1" 5 | 6 | function get_full_path($path) { 7 | # Unlike Resolve-Path, this doesn't throw an exception if the path does not exist. 8 | return ( 9 | $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath( 10 | $path) 11 | ) 12 | } 13 | 14 | function check_path($path) { 15 | log_message "Ensuring that `"$PATH`" exists." 16 | if (!(test-path $path)) { 17 | throw "Could not find path: `"$path`"." 18 | } 19 | } 20 | 21 | function get_link_target($linkPath) { 22 | $linkPath = Resolve-Path $linkPath 23 | $basePath = Split-Path $linkPath 24 | $link = Split-Path -leaf $linkPath 25 | $dir = cmd /c dir /a:l $basePath | findstr /C:"$link" 26 | $regx = $link + '\ *\[(.*?)\]' 27 | 28 | $Matches = $null 29 | $found = $dir -match $regx 30 | if ($found) { 31 | if ($Matches[1]) { 32 | # We'll try to resolve relative paths. 33 | pushd $basePath 34 | $target = get_full_path $Matches[1] 35 | popd 36 | return $target 37 | } 38 | } 39 | return '' 40 | } 41 | 42 | function delete_symlink($link) { 43 | log_message "Deleting link `"$link`"." 44 | fsutil reparsepoint delete $link 45 | remove-item $link -Force -Confirm:$false 46 | } 47 | 48 | function ensure_symlink($target, $link, $isDir) { 49 | log_message ("Ensuring symlink exists: $link -> $target. " + 50 | "Directory: $isDir") 51 | 52 | $target = get_full_path $target 53 | $link = get_full_path $link 54 | 55 | if ($target -eq $link) { 56 | log_message "$target IS $link. Skipping creating a symlink." 57 | } 58 | 59 | $shouldCreate = $false 60 | if (test-path $link) { 61 | $existing_target = get_link_target $link 62 | if (!($existing_target)) { 63 | throw ("Cannot create symlink. $link already exists " + 64 | "but is not a symlink") 65 | } 66 | 67 | if ($existing_target -ne $target) { 68 | log_message "Recreating symlink. Current target: $existing_target" 69 | delete_symlink $link 70 | $shouldCreate = $true 71 | } 72 | } 73 | else { 74 | $shouldCreate = $true 75 | } 76 | 77 | if ($shouldCreate) { 78 | $dirArg = "" 79 | if ($isDir) { 80 | $dirArg = "/D" 81 | } 82 | 83 | log_message "cmd /c mklink $dirArg $link $target" 84 | iex "cmd /c mklink $dirArg $link $target" 85 | if ($LASTEXITCODE) { 86 | throw "Failed to create symlink." 87 | } 88 | } 89 | } 90 | 91 | function ensure_binary_available($bin) { 92 | log_message ("Ensuring that the following " + 93 | "executable is available: `"$bin`".") 94 | if (!(where.exe $bin)) { 95 | throw ("Could not find `"$bin`". Make sure that it's installed " + 96 | "and its path is included in PATH.") 97 | } 98 | } 99 | 100 | function ensure_dir_exists($path) { 101 | if (!(test-path $path)) { 102 | mkdir -force $path | out-null 103 | } 104 | } 105 | 106 | function ensure_dir_empty($path) { 107 | # Ensure that the specified dir exists and is empty. 108 | if (test-path $path) { 109 | log_message "Directory already exists. Cleaning it up: `"$path`"." 110 | rm -recurse -force $path 111 | } 112 | mkdir $path | out-null 113 | } 114 | 115 | function check_remove_dir($path) { 116 | # Ensure that the specified dir exists and is empty. 117 | if (test-path $path) { 118 | log_message "Removing dir: `"$path`"." 119 | rm -recurse -force $path 120 | } 121 | } 122 | 123 | function check_remove_file($path, $silent=$false) { 124 | # Ensure that the specified dir exists and is empty. 125 | if (test-path $path) { 126 | if (! $silent) { 127 | log_message "Removing file: `"$path`"." 128 | } 129 | rm -force $path 130 | } 131 | } 132 | 133 | function extract_zip($src, $dest) { 134 | # Make sure to use full paths. 135 | Add-Type -AssemblyName System.IO.Compression.FileSystem 136 | 137 | log_message "Extracting zip: `"$src`" -> `"$dest`"." 138 | [System.IO.Compression.ZipFile]::ExtractToDirectory($src, $dest) 139 | } 140 | 141 | function ensure_smb_share($shareName, $sharePath) { 142 | log_message ("Ensuring that the `"$shareName`" SMB share " + 143 | "exists, exporting `"$sharePath`".") 144 | $sharePath = get_full_path $sharePath 145 | 146 | $existingShare = get-smbshare -name $shareName -ErrorAction ignore 147 | if ($existingShare) { 148 | $existingSharePath = get_full_path $existingShare.Path 149 | if ($existingSharePath -ne $sharePath) { 150 | throw ("Share `"$shareName`" already exists but it exports" + 151 | "a different path: `"$existingSharePath`". " + 152 | "Was requested: `"$sharePath`".") 153 | } 154 | log_message ("Share `"$shareName`" already exists, " + 155 | "exporting the requested " + 156 | "path: `"$sharePath`".") 157 | } 158 | else { 159 | log_message "Share $shareName does not exist. Creating it." 160 | ensure_dir_exists $sharePath 161 | New-SmbShare -Name $shareName -Path $sharePath 162 | } 163 | } 164 | 165 | function grant_smb_share_access($shareName, $accountName, 166 | $accessRight="Full") { 167 | log_message ("Granting `"$accessRight`" SMB share access " + 168 | "on `"$shareName`" to `"accountName`".") 169 | Grant-SmbShareAccess -AccountName $accountName ` 170 | -AccessRight $accessRight ` 171 | -name $shareName -Force 172 | set-smbpathacl $shareName 173 | } 174 | -------------------------------------------------------------------------------- /utils/windows/tests.psm1: -------------------------------------------------------------------------------- 1 | $scriptLocation = [System.IO.Path]::GetDirectoryName( 2 | $myInvocation.MyCommand.Definition) 3 | 4 | . "$scriptLocation\common.ps1" 5 | 6 | function add_subunit_result($outputFile, $testName, $result, $startTime, 7 | $stopTime, $details, $attachedFile) { 8 | $tempFile = [System.IO.Path]::GetTempFileName() 9 | 10 | cmd /c "echo $details >> $tempFile" 11 | cmd /c "echo '' >> $tempFile" 12 | if ($attachedFile) { 13 | # CMD 'type' properly handles binary files, unlike PS. 14 | cmd /c "type $attachedFile >> $tempFile" 15 | } 16 | 17 | python "$scriptLocation\..\common\publish_subunit_result.py" ` 18 | --test-id=$testName --status=$result --start-time=$startTime ` 19 | --stop-time $stopTime --output-file=$outputFile ` 20 | --attachment-path $tempFile 21 | 22 | check_remove_file $tempFile -silent $true 23 | } 24 | 25 | function add_subunit_failure($outputFile, $testName, $startTime, 26 | $stopTime, $details, $attachedFile) { 27 | add_subunit_result $outputFile $testName "fail" $startTime ` 28 | $stopTime $details $attachedFile 29 | } 30 | 31 | function add_subunit_success($outputFile, $testName, $startTime, 32 | $stopTime, $details, $attachedFile) { 33 | add_subunit_result $outputFile $testName "success" $startTime ` 34 | $stopTime $details $attachedFile 35 | } 36 | 37 | function gtest2subunit($xmlPath, $subunitPath, $testPrefix) { 38 | safe_exec ( 39 | "python `"$scriptLocation\..\common\gtest2subunit.py`" " + 40 | "--xml-path=$xmlPath --subunit-path=$subunitPath " + 41 | "--test-prefix=$testPrefix") 42 | } 43 | 44 | function generate_subunit_report($subunitPath, $reportDir, $reportName) { 45 | # Generate some user friendly reports based on the subunit binary file. 46 | $textResultFile = "$reportDir\$reportName.txt" 47 | $htmlResultFile = "$reportDir\$reportName.html" 48 | 49 | safe_exec "subunit2html $subunitPath $htmlResultFile" 50 | 51 | try { 52 | safe_exec "type $subunitPath | subunit-trace" | ` 53 | Out-File -Encoding ascii -FilePath $textResultFile 54 | } catch { 55 | # subunit-trace returns a non-zero exit code when the subunit file is 56 | # invalid OR when there are failed tests. This function is only 57 | # concerned in generating the test report and shouldn't fail if there 58 | # are test failures. 59 | log_message "subunit-trace reports failures: $_" 60 | } 61 | } 62 | 63 | function run_gtest($binPath, $resultDir, $timeout=-1, $testFilter, $testSuffix) { 64 | $binName = (split-path -leaf $binPath) -replace ".exe$","" 65 | $testName = $binName + $testSuffix 66 | $xmlOutputPath = join-path $resultDir ($testName + "_results.xml") 67 | $consoleOutputPath = join-path $resultDir ($testName + "_results.log") 68 | $gtestFilterArg = "" 69 | 70 | if ($testFilter) { 71 | $gtestFilterArg = "--gtest_filter=`"$testFilter`"" 72 | } 73 | 74 | $cmd = ("cmd /c '$binPath --gtest_output=xml:$xmlOutputPath $gtestFilterArg " + 75 | ">> $consoleOutputPath 2>&1'") 76 | 77 | echo $cmd | Out-File -Encoding ascii -FilePath $consoleOutputPath 78 | iex_with_timeout $cmd $timeout 79 | } 80 | 81 | function run_test($binPath, $resultDir, $timeout=-1, $testArgs, $testSuffix) { 82 | $binName = (split-path -leaf $binPath) -replace ".exe$","" 83 | $testName = $binName + $testSuffix 84 | $consoleOutputPath = join-path $resultDir ($testName + "_results.log") 85 | 86 | $cmd = ("cmd /c '$binPath $testArgs " + 87 | ">> $consoleOutputPath 2>&1'") 88 | 89 | echo $cmd | Out-File -Encoding ascii -FilePath $consoleOutputPath 90 | iex_with_timeout $cmd $timeout 91 | } 92 | 93 | function run_test_subunit($binPath, $resultDir, 94 | $subunitOutputPath, $timeout=-1, 95 | $testArgs, $testSuffix) { 96 | $binName = (split-path -leaf $binPath) -replace ".exe$","" 97 | $testName = $binName + $testSuffix 98 | $consoleOutputPath = join-path $resultDir ($testName + "_results.log") 99 | 100 | $startTime = get_unix_time 101 | try { 102 | run_test $binPath $resultDir $timeout $testArgs $testSuffix 103 | } 104 | catch { 105 | $errMsg = $_.Exception.Message 106 | $failed = $true 107 | throw 108 | } 109 | finally { 110 | $stopTime = get_unix_time 111 | 112 | if ($failed) { 113 | if (! $errMsg ) { 114 | $errMsg = "Test failed: $testName." 115 | } 116 | add_subunit_failure $subunitOutputPath $testName ` 117 | $startTime $stopTime ` 118 | $errMsg $consoleOutputPath 119 | } 120 | else { 121 | $testDetails = "Test passed: $testName" 122 | add_subunit_success $subunitOutputPath $testName ` 123 | $startTime $stopTime ` 124 | $testDetails $consoleOutputPath 125 | } 126 | } 127 | } 128 | 129 | function get_gtest_list($binPath) { 130 | safe_exec ("$binPath --gtest_list_tests | " + 131 | "python `"$scriptLocation\..\common\parse_gtest_list.py`"") 132 | } 133 | 134 | function run_gtest_subunit($binPath, $resultDir, $timeout=-1, $testFilter, 135 | $subunitOutputPath, $testSuffix) { 136 | $binName = (split-path -leaf $binPath) -replace ".exe$","" 137 | $testName = $binName + $testSuffix 138 | $xmlOutputPath = join-path $resultDir ($testName + "_results.xml") 139 | $consoleOutputPath = join-path $resultDir ($testName + "_results.log") 140 | 141 | $startTime = get_unix_time 142 | try { 143 | run_gtest $binPath $resultDir $timeout $testFilter $testSuffix 144 | } 145 | catch { 146 | $errMsg = $_.Exception.Message 147 | throw 148 | } 149 | finally { 150 | $stopTime = get_unix_time 151 | if (test-path $xmlOutputPath) { 152 | gtest2subunit $xmlOutputPath $subunitOutputPath $testName 153 | } 154 | else { 155 | if (! $errMsg ) { 156 | $errMsg = "Missing output xml." 157 | } 158 | add_subunit_failure $subunitOutputPath $testName ` 159 | $startTime $stopTime ` 160 | $errMsg $consoleOutputPath 161 | } 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /utils/windows/windows.psm1: -------------------------------------------------------------------------------- 1 | $scriptLocation = [System.IO.Path]::GetDirectoryName( 2 | $myInvocation.MyCommand.Definition) 3 | 4 | . "$scriptLocation\common.ps1" 5 | 6 | function _set_env($key, $val, $target="User") { 7 | $acceptableTargetList = @("process", "user", "machine") 8 | if ($target.ToLower() -notin $acceptableTargetList) { 9 | throw ("Cannot set environment variable `"$key`" to `"$val`". " + 10 | "Unsupported target: `"$target`".") 11 | } 12 | 13 | $varTarget = [System.EnvironmentVariableTarget]::$target 14 | 15 | [System.Environment]::SetEnvironmentVariable($key, $val, $varTarget) 16 | } 17 | 18 | function set_env($key, $val, $target="User") { 19 | log_message ("Setting environment value: `"$key`" = `"$val`". " + 20 | "Target: `"$target`".") 21 | _set_env $key $val $target 22 | # You'll always want to set the "Process" target as well, so that 23 | # it applies to the current process. Just to avoid some weird issues, 24 | # we're setting it here. 25 | _set_env $key $val "Process" 26 | } 27 | 28 | function get_env($var) { 29 | [System.Environment]::GetEnvironmentVariable($var) 30 | } 31 | 32 | function env_path_var_contains($path, $var="PATH") { 33 | # This may used for %PATH% and similar environment variables, 34 | # e.g. PYHTONPATH. 35 | $normPath = $path.Replace("\", "\\").Trim("\") 36 | (get_env $var) -imatch "(?:^|;)$normPath\\?(?:$|;)" 37 | } 38 | 39 | function add_to_env_path($path, $target="User", $var="PATH"){ 40 | if (!(env_path_var_contains $path $var)) { 41 | log_message "Adding `"$path`" to %$var%." 42 | 43 | $currentPath = get_env $var 44 | $newPath = "$currentPath;$path".Trim(";") 45 | 46 | set_env $var $newPath $target 47 | } 48 | else { 49 | log_message "%$var% already contains `"$path`"." 50 | } 51 | } 52 | 53 | function check_elevated() { 54 | log_message "Checking elevated permissions." 55 | 56 | $identity = [System.Security.Principal.WindowsIdentity]::GetCurrent() 57 | $principal = new-object System.Security.Principal.WindowsPrincipal( 58 | $identity) 59 | $adminRole = [System.Security.Principal.WindowsBuiltInRole]::Administrator 60 | $elevated = $principal.IsInRole($adminRole) 61 | if (!$elevated) { 62 | throw "This script requires elevated privileges." 63 | } 64 | } 65 | 66 | function stop_processes($name) { 67 | log_message "Stopping process(es): `"$name`"." 68 | get-process $name | stop-process -PassThru | ` 69 | % { log_message "Stopped process: `"$($_.Name)`"." } 70 | } 71 | 72 | function check_windows_feature($featureName) { 73 | log_message("Ensuring that the following Windows feature is available: " + 74 | "`"$featureName`".") 75 | 76 | $feature = Get-WindowsOptionalFeature -FeatureName "$featureName" -Online 77 | if (!($feature)) { 78 | throw "Could not find Windows feature: `"$featureName`"." 79 | } 80 | if ($feature.Count -gt 1) { 81 | # We're going to allow wildcards. 82 | log_message ("WARNING: Found multiple features matching " + 83 | "the specified name: $($feature.FeatureName). " + 84 | "Will ensure that all of them are enabled.") 85 | log_message $msg 86 | } 87 | 88 | $feature | ` 89 | ? { $_.State -eq "Enabled" } | ` 90 | % { log_message ("The following Windows feature is available: " + 91 | "`"$($_.FeatureName)`".")} 92 | $disabledFeatures = @() 93 | $feature | ` 94 | ? { $_.State -ne "Enabled" } | ` 95 | % { $disabledFeatures += $_.FeatureName } 96 | 97 | if ($disabledFeatures) { 98 | throw "The following Windows features are not enabled: $missingFeatures." 99 | } 100 | } 101 | 102 | function enable_rdp_access() { 103 | log_message "Enabling RDP access." 104 | 105 | Set-ItemProperty ` 106 | -Path "HKLM:\\System\\CurrentControlSet\\Control\\Terminal Server" ` 107 | -Name "fDenyTSConnections" -Value 0 108 | Enable-NetFirewallRule -DisplayGroup "Remote Desktop" 109 | } 110 | 111 | $ErrorActionPreference = "Stop" 112 | $ProgressPreference = "SilentlyContinue" 113 | 114 | $JobMgrDefinition = @" 115 | using System; 116 | using System.Text; 117 | using System.Runtime.InteropServices; 118 | public class JobMgr 119 | { 120 | public enum JOBOBJECTINFOCLASS 121 | { 122 | AssociateCompletionPortInformation = 7, 123 | BasicLimitInformation = 2, 124 | BasicUIRestrictions = 4, 125 | EndOfJobTimeInformation = 6, 126 | ExtendedLimitInformation = 9, 127 | SecurityLimitInformation = 5, 128 | GroupInformation = 11 129 | } 130 | 131 | [StructLayout(LayoutKind.Sequential)] 132 | struct JOBOBJECT_BASIC_LIMIT_INFORMATION 133 | { 134 | public Int64 PerProcessUserTimeLimit; 135 | public Int64 PerJobUserTimeLimit; 136 | public UInt32 LimitFlags; 137 | public UIntPtr MinimumWorkingSetSize; 138 | public UIntPtr MaximumWorkingSetSize; 139 | public UInt32 ActiveProcessLimit; 140 | public Int64 Affinity; 141 | public UInt32 PriorityClass; 142 | public UInt32 SchedulingClass; 143 | } 144 | 145 | 146 | [StructLayout(LayoutKind.Sequential)] 147 | struct IO_COUNTERS 148 | { 149 | public UInt64 ReadOperationCount; 150 | public UInt64 WriteOperationCount; 151 | public UInt64 OtherOperationCount; 152 | public UInt64 ReadTransferCount; 153 | public UInt64 WriteTransferCount; 154 | public UInt64 OtherTransferCount; 155 | } 156 | 157 | [StructLayout(LayoutKind.Sequential)] 158 | struct JOBOBJECT_EXTENDED_LIMIT_INFORMATION 159 | { 160 | public JOBOBJECT_BASIC_LIMIT_INFORMATION BasicLimitInformation; 161 | public IO_COUNTERS IoInfo; 162 | public UIntPtr ProcessMemoryLimit; 163 | public UIntPtr JobMemoryLimit; 164 | public UIntPtr PeakProcessMemoryUsed; 165 | public UIntPtr PeakJobMemoryUsed; 166 | } 167 | 168 | [DllImport("kernel32.dll", CharSet = CharSet.Unicode)] 169 | public static extern IntPtr CreateJobObject( 170 | IntPtr lpJobAttributes, string lpName); 171 | 172 | [DllImport("kernel32.dll")] 173 | public static extern bool AssignProcessToJobObject( 174 | IntPtr hJob, IntPtr hProcess); 175 | 176 | [DllImport("kernel32.dll")] 177 | public static extern bool SetInformationJobObject( 178 | IntPtr hJob, JOBOBJECTINFOCLASS JobObjectInfoClass, 179 | IntPtr lpJobObjectInfo, uint cbJobObjectInfoLength); 180 | 181 | [DllImport("kernel32.dll")] 182 | public static extern IntPtr GetCurrentProcess(); 183 | 184 | 185 | private const UInt32 JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000; 186 | public void register_current_proc_as_job(string jobName) 187 | { 188 | IntPtr hJob = CreateJobObject(IntPtr.Zero, jobName); 189 | 190 | JOBOBJECT_BASIC_LIMIT_INFORMATION info = 191 | new JOBOBJECT_BASIC_LIMIT_INFORMATION(); 192 | info.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; 193 | 194 | JOBOBJECT_EXTENDED_LIMIT_INFORMATION extendedInfo = 195 | new JOBOBJECT_EXTENDED_LIMIT_INFORMATION(); 196 | extendedInfo.BasicLimitInformation = info; 197 | 198 | int length = Marshal.SizeOf( 199 | typeof(JOBOBJECT_EXTENDED_LIMIT_INFORMATION)); 200 | IntPtr extendedInfoPtr = Marshal.AllocHGlobal(length); 201 | Marshal.StructureToPtr(extendedInfo, extendedInfoPtr, false); 202 | 203 | SetInformationJobObject(hJob, 204 | JOBOBJECTINFOCLASS.ExtendedLimitInformation, 205 | extendedInfoPtr, 206 | (uint)length); 207 | 208 | IntPtr hProcess = GetCurrentProcess(); 209 | bool blRc = AssignProcessToJobObject(hJob, hProcess); 210 | 211 | Marshal.FreeHGlobal(extendedInfoPtr); 212 | } 213 | } 214 | "@ 215 | 216 | 217 | function run_as_job($jobName) { 218 | # This will ensure that this process's children will be part of the same 219 | # job object, being stopped along with it when it exits. 220 | if (! $jobName) { 221 | $jobName = [guid]::NewGuid().Guid 222 | } 223 | Add-Type -TypeDefinition $JobMgrDefinition 224 | 225 | $mgr = New-Object JobMgr 226 | $mgr.register_current_proc_as_job($jobName) 227 | } 228 | -------------------------------------------------------------------------------- /test_host/run_tests.ps1: -------------------------------------------------------------------------------- 1 | Param( 2 | [string]$testDir="${env:SystemDrive}\ceph", 3 | [string]$resultDir="${env:SystemDrive}\workspace\test_results", 4 | [string]$cephConfig="$env:ProgramData\ceph\ceph.conf", 5 | [int]$testSuiteTimeout=1800, 6 | [int]$workerCount=8, 7 | [bool]$skipSlowTests=$false, 8 | [string]$rbdPoolName="rbd" 9 | ) 10 | 11 | $ErrorActionPreference = "Stop" 12 | $ProgressPreference = "SilentlyContinue" 13 | 14 | $isolatedTestsMapping = $false 15 | $env:CEPH_CONF = $cephConfig 16 | 17 | $scriptLocation = [System.IO.Path]::GetDirectoryName( 18 | $myInvocation.MyCommand.Definition) 19 | 20 | import-module "$scriptLocation\..\utils\windows\all.psm1" 21 | 22 | function clear_test_stats() { 23 | $env:TEST_FAILED = "0" 24 | } 25 | 26 | function validate_test_run() { 27 | if ($env:TEST_FAILED -ne "0") { 28 | throw "One or more test suites have failed" 29 | } 30 | 31 | log_message "All the tests have passed." 32 | } 33 | 34 | function notify_starting_test($testDescription, $testType) { 35 | log_message "Running test: $testType $testDescription." 36 | } 37 | 38 | function notify_successful_test($testDescription, $testType) { 39 | log_message "$testType $testDescription passed." 40 | } 41 | 42 | function notify_failed_test($testDescription, $testType, $errMsg) { 43 | # We're going to resume running tests even if one of the suite fails, 44 | # throwing an error at the end of the run. 45 | $env:TEST_FAILED = "1" 46 | 47 | log_message "$testType $testDescription failed. Error: $errMsg" 48 | } 49 | 50 | function get_isolated_tests($testFileName, $isolatedTestsMapping) { 51 | $isolatedTests = @() 52 | foreach ($isolatedTestPattern in $isolatedTestsMapping.Keys) { 53 | if ($testFileName -match $isolatedTestPattern) { 54 | $isolatedTests += $isolatedTestsMapping[$isolatedTestPattern] 55 | } 56 | } 57 | $isolatedTests 58 | } 59 | 60 | function get_matching_pattern($str, $patternList) { 61 | foreach ($pattern in $patternList) { 62 | if ($str -match $pattern) { 63 | return $pattern 64 | } 65 | } 66 | } 67 | 68 | function run_tests_from_dir( 69 | $testdir, $resultDir, $pattern, 70 | $isolatedTestsMapping, 71 | $testType, 72 | $runIsolatedTests, 73 | $subunitOutFile, 74 | $nonGTestList, 75 | $workerCount=8) { 76 | $testList = ls -Recurse $testdir | ` 77 | ? { $_.Name -match $pattern } 78 | 79 | $rsp = [RunspaceFactory]::CreateRunspacePool($workerCount, $workerCount) 80 | $rsp.Open() 81 | $jobs = @() 82 | foreach($testBinary in $testList) { 83 | $testPath = $testBinary.FullName 84 | $testBinName = (split-path -leaf $testPath) -replace ".exe$","" 85 | $testResultDir = join-path $resultDir "out\${testBinName}" 86 | ensure_dir_exists $testResultDir 87 | 88 | $isolatedTests = get_isolated_tests $testBinary.Name $isolatedTestsMapping 89 | $testFilter = $isolatedTests -join ":" 90 | 91 | $nonGTestPattern = get_matching_pattern $testBinary.Name $nonGTestList.Keys 92 | if ($nonGTestPattern) { 93 | $isGtest = $false 94 | $testArgs = $nonGTestList[$nonGTestPattern] 95 | $tags = "$testType[standalone]" 96 | } 97 | else { 98 | $isGtest = $true 99 | $testArgs = "" 100 | $tags = "$testType[googletest]" 101 | } 102 | 103 | if ((! $runIsolatedTests) -and $testFilter -eq "*") { 104 | # This whole test suite is skipped. We won't pass this filter to 105 | # the binary as it may not use the GTest framework. 106 | continue 107 | } 108 | if ($runIsolatedTests -and (! $testFilter)) { 109 | # No isolated tests for this suite. 110 | continue 111 | } 112 | if ((! $runIsolatedTests) -and $testFilter) { 113 | $testFilter = "-$testFilter" 114 | } 115 | 116 | # We had issues with corrupted subunit files, so each job will 117 | # stream to a separate file. We'll merge those files afterwards. 118 | $uid = [guid]::NewGuid().ToString() 119 | $subunitTempFile = $subunitOutFile + ".tmp." + $uid 120 | 121 | if ($testArgs -is [hashtable]) { 122 | $testArgsMap = $testArgs 123 | } else { 124 | $testArgsMap = @{$testArgs=""} 125 | } 126 | 127 | $idx = 0 128 | foreach($testArgs in $testArgsMap.Keys) { 129 | $testSuffix = $testArgsMap[$testArgs] 130 | if ($testSuffix) { 131 | $testSuffix = "." + $testSuffix 132 | } elseif ($testArgsMap.Keys.Length > 1) { 133 | $testSuffix = "." + [string]$idx 134 | } 135 | 136 | $testName = $testBinary.Name -replace ".exe$","" 137 | $testName += $testSuffix 138 | 139 | notify_starting_test $testName $tags 140 | 141 | $job = [Powershell]::Create().AddScript({ 142 | Param( 143 | [Parameter(Mandatory=$true)] 144 | [string]$utilsModuleLocation, 145 | [Parameter(Mandatory=$true)] 146 | [string]$testPath, 147 | [Parameter(Mandatory=$true)] 148 | [string]$resultDir, 149 | [Parameter(Mandatory=$true)] 150 | [int]$testSuiteTimeout, 151 | [Parameter(Mandatory=$true)] 152 | [string]$subunitOutFile, 153 | [string]$testFilter, 154 | [bool]$isGtest, 155 | [string]$testArgs, 156 | [string]$testSuffix 157 | ) 158 | import-module $utilsModuleLocation 159 | try { 160 | if ($isGtest) { 161 | run_gtest_subunit ` 162 | $testPath $resultDir $testSuiteTimeout ` 163 | $testFilter $subunitOutFile $testSuffix 164 | } 165 | else { 166 | run_test_subunit ` 167 | $testPath $resultDir $subunitOutFile ` 168 | $testSuiteTimeout $testArgs $testSuffix 169 | } 170 | 171 | return @{success=$true} 172 | } 173 | catch { 174 | $errMsg = $_.Exception.Message 175 | return @{success=$false; errMsg=$errMsg} 176 | } 177 | }).AddParameters(@{ 178 | utilsModuleLocation="$scriptLocation\..\utils\windows\all.psm1"; 179 | testPath=$testPath; 180 | resultDir=$testResultDir; 181 | testSuiteTimeout=$testSuiteTimeout; 182 | subunitOutFile=$subunitTempFile; 183 | testFilter=$testFilter; 184 | isGtest=$isGtest; 185 | testArgs=$testArgs; 186 | testSuffix=$testSuffix 187 | }) 188 | $job.RunspacePool = $rsp 189 | $jobs += @{ 190 | Job=$job; 191 | Result=$job.BeginInvoke(); 192 | TestName=$testName; 193 | TestType=$tags 194 | } 195 | $idx++; 196 | } 197 | } 198 | 199 | do { 200 | Start-Sleep -seconds 10 201 | [System.Object[]]$finished = $jobs | ? {$_.Result.IsCompleted -eq $true} 202 | log_message "Finished $($finished.Count) out of $($jobs.Count) jobs." 203 | } while ($finished.Count -lt $jobs.Count) 204 | 205 | foreach($r in $jobs) { 206 | $result = $r.Job.EndInvoke($r.Result) 207 | if($result.success) { 208 | notify_successful_test $r.TestName $r.TestType 209 | } 210 | else { 211 | notify_failed_test $r.TestName $r.TestType $result.errMsg 212 | } 213 | } 214 | 215 | # Aggregate the results 216 | ls ($subunitOutFile + ".tmp.*") | % { 217 | $subunitTempFile = $_.FullName 218 | gc -encoding byte $subunitTempFile -Read 512 | ac -encoding byte $subunitOutFile 219 | rm $subunitTempFile 220 | } 221 | } 222 | 223 | function run_tests() { 224 | $subunitFile = "$resultDir\subunit.out" 225 | $testPattern="^unittest.*.exe$|^ceph_test.*.exe$" 226 | # Tests that aren't using the google test framework or require specific 227 | # arguments and will have to begin run differently. The following mapping 228 | # provides the arguments needed by each test. 229 | $nonGTestList=@{ 230 | "ceph_test_timers.exe"=""; 231 | "ceph_test_rados_delete_pools_parallel.exe"=""; 232 | "ceph_test_rados_list_parallel.exe"=""; 233 | "ceph_test_rados_open_pools_parallel.exe"=""; 234 | "ceph_test_rados_watch_notify.exe"=""; 235 | "ceph_test_rados_op_speed"=""; 236 | "ceph_test_librbd_fsx.exe"=@{ 237 | "$rbdPoolName test_librbd_fsx0 -N 3000 -d"="librbd"; 238 | "$rbdPoolName test_librbd_fsx1 -N 3000 -M -L -r 4096 -w 4096 -d"="wnbd" 239 | }; 240 | } 241 | # Tests that aren't supposed to be run automatically. 242 | $manualTests=@{ 243 | "ceph_test_mutate.exe"="*"; 244 | "ceph_test_rewrite_latency.exe"="*"; 245 | } 246 | $excludedTests=@{ 247 | # This test passes but it leaks a "sleep" subprocess which 248 | # hangs powershell's job mechanism. 249 | "unittest_subprocess.exe"="SubProcessTimed.SubshellTimedout"; 250 | # TODO: the following tests seem to use the AioTestData semaphore incorrectly. 251 | # The same AioTestData structure is reused for multiple async callbacks, which 252 | # implies multiple completion notifications. However, the semaphore is initialized 253 | # with 0, so "wait" will return before subsequent "notify" calls complete. 254 | # For this reason, we can have situations in which "notify" callbacks occur after 255 | # the test completes and the AioTestData structure and semaphore are cleaned up, 256 | # which leads to a crash: 257 | # * https://pastebin.com/raw/gh7CytHA 258 | # * https://pastebin.com/raw/2Yu90uEG 259 | "ceph_test_rados_striper_api_aio.exe"=@( 260 | "*.Flush*", 261 | "*.RoundTrip*", 262 | "*.IsSafe*") 263 | "ceph_test_signal_handlers.exe"="*"; 264 | # TODO - we may stick to the client tests, but this may also 265 | # involve RGW, which we haven't covered yet. 266 | "ceph_test_admin_socket_output.exe"="*"; 267 | # The following tests fail because of the pseudo-random 268 | # numbers generated by rand, getting increased chance of 269 | # false positives. 270 | "unittest_bloom_filter"=@( 271 | "BloomFilter.SweepInt", 272 | "BloomFilter.CompressibleSweep"); 273 | # execinfo.h isn't available on Windows, this may need a 274 | # different approach. 275 | "unittest_back_trace.exe"="*"; 276 | # Those tests throttle some operations. We're meeting the 277 | # high threshold but we miss the low threshold. 278 | "unittest_throttle.exe"="*"; 279 | # The Mingw C runtime seems to have some issues. Among others, 280 | # the "%z" flag isn't handled properly by strftime. 281 | "unittest_time.exe"="TimePoints.stringify" 282 | "unittest_utime.exe"=@( 283 | "utime_t.localtime", 284 | "utime_t.parse_date"); 285 | # Flaky test, fails on Linux as well 286 | "ceph_test_rados_api_io_pp.exe"="LibRadosIoECPP.CrcZeroWrite"; 287 | # The following tests are affected by errno conversions 288 | "ceph_test_rados_api_snapshots_pp.exe"=` 289 | # EOLDSNAPC is defined as 85, which overlaps with ERESTART, 290 | # which will be converted to EINTR 291 | "LibRadosSnapshotsSelfManagedPP.OrderSnap"; 292 | # cls_helo.cc:write_return_data returns 42, which will be converted 293 | # TODO: ensure that this won't affect the rados/rbd (e.g. we may 294 | # end up converting values other than error codes, which is wrong). 295 | "ceph_test_rados_api_tier_pp.exe"="*"; 296 | # TODO: look into this. seems like a local error (ECANCELED) gets 297 | # converted to the unix value, yet the test expects the host error. 298 | "ceph_test_rados_api_aio_pp.exe"="LibRadosAio.OmapPP"; 299 | # TODO: some watch timeout is not honored. Watch3 seems to be broken. 300 | "ceph_test_rados_api_watch_notify.exe"="LibRadosWatchNotify.Watch3Timeout"; 301 | "ceph_test_rados_api_watch_notify_pp.exe"="*WatchNotify3*"; 302 | # TODO 303 | "ceph_test_lazy_omap_stats.exe"="*"; 304 | # seems broken. When shutting down timers, it asserts that a lock is 305 | # set but nobody's locking it. It passes if CEPH_DEBUG_MUTEX is 306 | # disabled, in which case such checks always return 1. 307 | "ceph_test_timers.exe"="*"; 308 | # For some reason, the following tests gets WSAECONNREFUSED errors but 309 | # only when running under a powershell job, passing otherwise. 310 | "unittest_perf_counters*"="*"; 311 | # TODO: the SnaptrimStats test fails on Linux as well, it most probably 312 | # requires some specific configuration. 313 | "ceph_test_rados_api_snapshots_stats.exe"="*SnaptrimStats*"; 314 | "ceph_test_rados_api_snapshots_stats_pp.exe"="*SnaptrimStats*"; 315 | # TODO: Need debugging 316 | "unittest_compression.exe"="*"; 317 | # Fails on the CI env; 318 | "ceph_test_rados_api_cls_remote_reads.exe"="*"; 319 | "unittest_confutils.exe"=@( 320 | "ConfUtils.ParseFiles0"); 321 | "unittest_fair_mutex.exe"=@( 322 | "FairMutex.fair"); 323 | "unittest_mempool.exe"=@( 324 | "mempool.check_shard_select"); 325 | # bufferlist benchmark tests can take more than 30 minutes, for which 326 | # reason we'll disable them by default. 327 | "unittest_bufferlist.exe"=@( 328 | "BufferList.read_file", 329 | "*Bench*", 330 | "*bench*"); 331 | } 332 | $slowTestList=@{ 333 | } 334 | # The following tests have to be run separately. 335 | $isolatedTests=@{ 336 | "unittest_admin_socket.exe"="*"; 337 | # Multiple libcephfs tests try to access or remove the same paths. 338 | # Also, we're skipping flaky delegation tests: 339 | # https://github.com/ceph/ceph/pull/52427#issuecomment-1640325664 340 | "ceph_test_libcephfs*"="-LibCephFS.Deleg*"; 341 | } 342 | 343 | # TODO: fix merging hashtables, allow the same suite to have some excluded 344 | # tests and some marked as slow tests. 345 | ($manualTests.Keys + $isolatedTests.Keys) | ForEach-Object { $excludedTests += @{$_="*"} } 346 | if ($skipSlowTests) { 347 | $slowTestList.Keys | ForEach-Object { $excludedTests += @{$_="*"} } 348 | } 349 | 350 | log_message "Running unit tests." 351 | log_message "Using subunit file: $subunitFile" 352 | 353 | run_tests_from_dir -testdir $testDir ` 354 | -resultDir $resultDir ` 355 | -pattern $testPattern ` 356 | -isolatedTestsMapping $excludedTests ` 357 | -runIsolatedTests $false ` 358 | -testType "" ` 359 | -subunitOutFile $subunitFile ` 360 | -workerCount $workerCount ` 361 | -nonGTestList $nonGTestList 362 | 363 | # Various tests that are known to crash, hang or cannot be run in parallel. 364 | log_message "Running isolated unit tests." 365 | run_tests_from_dir -testdir $testDir ` 366 | -resultDir $resultDir ` 367 | -pattern $testPattern ` 368 | -isolatedTestsMapping $isolatedTests ` 369 | -runIsolatedTests $true ` 370 | -testType "[isolated]" ` 371 | -subunitOutFile $subunitFile ` 372 | -workerCount 1 ` 373 | -nonGTestList $nonGTestList 374 | 375 | try { 376 | generate_subunit_report $subunitFile $resultDir "test_results" 377 | } catch { 378 | log_message "subunit report failure: $_" 379 | if (test-path $subunitFile) { 380 | $subunitHash = (Get-FileHash $subunitFile -Algorithm SHA256).Hash 381 | log_message "subunit file: $subunitFile" 382 | log_message "subunit sha256 checksum: $subunitHash" 383 | } else { 384 | log_message "missing subunit file: $subunitFile" 385 | } 386 | } 387 | } 388 | 389 | ensure_dir_exists $resultDir 390 | 391 | clear_test_stats 392 | 393 | run_tests 394 | 395 | validate_test_run 396 | --------------------------------------------------------------------------------