/`` will be uploaded via
14 | a cron job. Your builders public key will need to be added to that machine.
15 | Rsync or scp can be used to upload the files.
16 |
17 | llvmbisect uses some regexes in llvmlab.py to parse the comiler information.
18 | The tar file you upload will need to match those regexes. For example:
19 | ``clang-r249497-t13154-b13154.tar.gz``
20 |
--------------------------------------------------------------------------------
/zorg/buildbot/builders/sanitizers/buildbot_fast.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | set -x
4 | set -e
5 | set -u
6 |
7 | HERE="$(cd $(dirname $0) && pwd)"
8 | . ${HERE}/buildbot_functions.sh
9 |
10 | ROOT=`pwd`
11 | PLATFORM=`uname`
12 | export PATH="/usr/local/bin:$PATH"
13 |
14 | USE_GIT=1
15 |
16 | CHECK_LIBCXX=${CHECK_LIBCXX:-1}
17 | CHECK_LLD=${CHECK_LLD:-1}
18 | LLVM=$ROOT/llvm
19 | CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_PARALLEL_LINK_JOBS=20"
20 |
21 | clobber
22 |
23 | # Stage 1
24 |
25 | build_stage1_clang_at_revison
26 |
27 | CMAKE_COMMON_OPTIONS="$CMAKE_COMMON_OPTIONS -DLLVM_ENABLE_ASSERTIONS=ON"
28 |
29 | buildbot_update
30 |
31 | # Stage 2 / Memory Sanitizer
32 |
33 | build_stage2_msan
34 |
35 | check_stage2_msan
36 |
37 | # Stage 2 / AddressSanitizer
38 |
39 | build_stage2_asan
40 |
41 | check_stage2_asan
42 |
43 | # Stage 2 / UndefinedBehaviorSanitizer
44 |
45 | build_stage2_ubsan
46 |
47 | check_stage2_ubsan
48 |
--------------------------------------------------------------------------------
/tasks/tasktool/tasktool/hooks/mk-sshrun-build:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | set -eu
3 |
4 | USERDIR="$1"
5 | TASKSCRIPT="$2"
6 | BUILDCONFIG="$3"
7 | BUILDNAME="$4"
8 |
9 | TASK_DIR="$(dirname "${TASKSCRIPT}")"
10 | export GIT_WORK_TREE="$(git -C "${TASK_DIR}" rev-parse --show-toplevel)"
11 | GIT_DIR="$(git -C "${TASK_DIR}" rev-parse --git-dir)"
12 | export GIT_DIR="$(cd "${TASK_DIR}" ; cd "${GIT_DIR}" ; pwd)"
13 |
14 | . check-config.sh
15 | . "${USERDIR}/config"
16 |
17 | . check-clean-repo.sh
18 |
19 | HOST="$(whoami)@$(hostname -f)"
20 | CONFIG_URL="${HOST}:${GIT_WORK_TREE}"
21 | CONFIG_REV="$(git rev-parse HEAD)"
22 |
23 | . mk-build-id.sh
24 | echo "BUILDNAME=\"${BUILDNAME}_${BUILDID}\""
25 | if [ -n "${REMOTE_TRYDIR+x}" ]; then
26 | echo "REMOTE_TRYDIR=\"${REMOTE_TRYDIR}\""
27 | fi
28 | tail -n +4 header_sshrun
29 | . mk-build-common.sh
30 |
31 | mkdir -p "${USERDIR}/results"
32 | echo "RESULT_HOST='${HOST}'"
33 | echo "RESULT_FILE='${USERDIR}/results/${BUILDNAME}_${BUILDID}.tar.xz'"
34 | tail -n +4 footer_sshrun
35 |
--------------------------------------------------------------------------------
/tasks/tasktool/setup.py:
--------------------------------------------------------------------------------
1 | from setuptools import setup, find_packages
2 |
3 | setup(
4 | name='tasktool',
5 | version='1.0',
6 | author='Matthias Braun',
7 | author_email='matze@braunis.de',
8 | license='BSD',
9 | description='CI Task Runner',
10 | classifiers=[
11 | 'Development Status :: 5 - Production/Stable',
12 | 'Environment :: Console',
13 | 'Intended Audience :: Developers',
14 | 'License :: OSI Approved :: BSD License',
15 | 'Operating System :: POSIX',
16 | 'Programming Language :: Python',
17 | 'Programming Language :: Unix Shell',
18 | 'Topic :: Software Development :: Build Tools',
19 | 'Topic :: Software Development :: Quality Assurance'
20 | 'Topic :: Software Development :: Testing',
21 | ],
22 | package_data={
23 | 'tasktool': ['hooks/*'],
24 | },
25 | packages=find_packages(),
26 | entry_points={
27 | 'console_scripts': ['task=tasktool.task:main'],
28 | },
29 | )
30 |
--------------------------------------------------------------------------------
/llvmlab/llvmlab/ui/templates/latest_release.html:
--------------------------------------------------------------------------------
1 | {% extends "layout.html" %}
2 | {% block title %}latest release{% endblock %}
3 | {% block body %}
4 |
5 | {# The builds archive table... #}
6 |
7 | Latest Release
8 |
9 |
10 | FIXME: THIS PAGE IS JUST A PLACEHOLDER, IT IS NOT LIVE!
11 |
12 |
13 | The latest released sources are available here: llvm-rNNNNNN .
14 |
15 |
16 | Binary packages of the latest release:
17 |
18 |
19 |
20 | Product
21 | OS
22 | Arch
23 | Archive
24 |
25 |
26 |
27 | {% for build in ci_config.published_builds %}
28 |
29 | {{ build.product }}
30 | {{ build.os }}
31 | {{ build.arch }}
32 | {{
33 | build.archive_name }}
35 | {% endfor %}
36 |
37 |
38 | {% endblock %}
39 |
--------------------------------------------------------------------------------
/test/lit.cfg:
--------------------------------------------------------------------------------
1 | # -*- Python -*-
2 |
3 | import os
4 | import platform
5 | import lit
6 |
7 | # Configuration file for the 'lit' test runner.
8 |
9 | # name: The name of this test suite.
10 | config.name = 'Zorg'
11 |
12 | # testFormat: The test format to use to interpret tests.
13 | #
14 | # For now we require '&&' between commands, until they get globally killed and
15 | # the test runner updated.
16 | execute_external = platform.system() != 'Windows'
17 | config.test_format = lit.formats.ShTest(execute_external)
18 |
19 | # suffixes: A list of file extensions to treat as test files.
20 | config.suffixes = ['.py']
21 |
22 | # test_source_root: The root path where tests are located.
23 | config.test_source_root = os.path.dirname(__file__)
24 | config.test_exec_root = config.test_source_root
25 |
26 | config.target_triple = None
27 |
28 | config.environment['PYTHONPATH'] = os.path.join(config.test_source_root, '..')
29 |
30 |
31 | src_root = os.path.join(config.test_source_root, '..')
32 |
33 | config.substitutions.append(('%{src_root}', src_root))
--------------------------------------------------------------------------------
/tasks/utils/normalize_compiler.sh:
--------------------------------------------------------------------------------
1 | # Get rid of the overly deep directory structures in the compiler artifacts
2 | #
3 | # After this script has run you should have $WORKSPACE/compiler/bin/clang etc.
4 | cd "compiler"
5 | # For the people with full xcode paths
6 | if [ -d "Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr" ]; then
7 | if [ -d "usr" ]; then
8 | rm -rf usr
9 | fi
10 | mv Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/* .
11 | rm -rf Applications
12 | elif [ -d */bin ]; then
13 | # For people that have exactly 1 toplevel directory in their artifact
14 | # */bin/clang -> bin/clang
15 | dir=$(dirname */bin)
16 | for subdir in "$dir"/*
17 | do
18 | subdir_basename=$(basename $subdir)
19 | if [ -d $subdir_basename ]; then
20 | rm -rf $subdir_basename
21 | fi
22 | mv "$dir"/$subdir_basename .
23 | done
24 | fi
25 |
26 | # Make sure there is a clang now
27 | cd "${WORKSPACE}"
28 | ${WORKSPACE}/compiler/bin/clang -v
29 |
--------------------------------------------------------------------------------
/zorg/buildbot/commands/SuppressionDejaGNUCommand.py:
--------------------------------------------------------------------------------
1 | import re
2 | import StandardizedTest
3 |
4 | class SuppressionDejaGNUCommand(StandardizedTest.StandardizedTest):
5 | kRunningRE = re.compile(r'Running (.*) ...')
6 | kRunningRE = re.compile(r'Running (.*) ...')
7 | kTestStateLineRE = re.compile(r'(FAIL|PASS|XFAIL|XPASS|UNRESOLVED): (.*)')
8 |
9 | testLogName = 'dg.sum'
10 |
11 | def parseLog(self, lines):
12 | results = []
13 |
14 | test_suite = None
15 | for ln in lines:
16 | m = self.kRunningRE.match(ln)
17 | if m is not None:
18 | test_suite, = m.groups()
19 | continue
20 |
21 | m = self.kTestStateLineRE.match(ln)
22 | if m is not None:
23 | code,name = m.groups()
24 | results.append((code, name, None))
25 | continue
26 |
27 | return results
28 |
29 | if __name__ == '__main__':
30 | import sys
31 | t = SuppressionDejaGNUCommand()
32 | for res in t.parseLog(open(sys.argv[1]).readlines()):
33 | print res
34 |
--------------------------------------------------------------------------------
/llvmlab/llvmlab/ui/static/jquery/1.5/LICENSE.txt:
--------------------------------------------------------------------------------
1 | Copyright (c) 2011 John Resig, http://jquery.com/
2 |
3 | Permission is hereby granted, free of charge, to any person obtaining
4 | a copy of this software and associated documentation files (the
5 | "Software"), to deal in the Software without restriction, including
6 | without limitation the rights to use, copy, modify, merge, publish,
7 | distribute, sublicense, and/or sell copies of the Software, and to
8 | permit persons to whom the Software is furnished to do so, subject to
9 | the following conditions:
10 |
11 | The above copyright notice and this permission notice shall be
12 | included in all copies or substantial portions of the Software.
13 |
14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
21 |
--------------------------------------------------------------------------------
/llvmlab/llvmlab/ui/templates/buildbot_monitor.html:
--------------------------------------------------------------------------------
1 | {% extends "layout.html" %}
2 | {% block title %}buildbot monitor{% endblock %}
3 | {% block body %}
4 |
5 | Buildbot Status Monitor
6 |
7 | Monitoring: {{ bb_status.master_url }}
8 |
9 | {% for name,builds in bb_status.builders|dictsort %}
10 |
11 | {{ name }}
12 |
13 |
14 |
15 |
16 | Number
17 | Source Stamp
18 | Result
19 | Start Time
20 | End Time
21 | Slave
22 |
23 |
24 | {% for build in builds %}
25 |
26 | {{ build.number }}
27 | {{ build.source_stamp }}
28 | {{ build.result }}
29 | {{ build.start_time }}
30 | {{ build.end_time }}
31 | {{ build.slave }}
32 |
33 | {% endfor %}
34 |
35 |
36 | {% endfor %}
37 |
38 | Event Log
39 |
40 |
41 |
42 | Time Event
44 | {% for time,event in bb_status.event_log|reverse %}
45 |
46 | {{ time }}
47 | {{ event }}
48 |
49 | {% endfor %}
50 |
51 |
52 | {% endblock %}
53 |
--------------------------------------------------------------------------------
/zorg/buildbot/commands/BatchFileDownload.py:
--------------------------------------------------------------------------------
1 | from buildbot.steps.transfer import FileDownload
2 |
3 | class BatchFileDownload(FileDownload):
4 | # FIXME: It would be nice to form a BatchedShellCommand step out of this.
5 | def __init__(self, **kwargs):
6 | if 'command' in kwargs:
7 | if 'mastersrc' in kwargs:
8 | raise ValueError,"Unexpected 'mastersrc' argument."
9 | if 'slavedest' in kwargs:
10 | raise ValueError,"Unexpected 'slavedest' argument."
11 |
12 | # This is the initial construction, create a temporary
13 | # batch file to run the command.
14 | import os
15 | import tempfile
16 |
17 | command = kwargs.pop('command')
18 | tf = tempfile.NamedTemporaryFile(delete=False)
19 | print >>tf, '@echo on'
20 | print >>tf, ' '.join('"%s"' % a for a in command)
21 | tf.close()
22 |
23 | remotename = kwargs.get('name', 'batched-command')
24 | kwargs['mastersrc'] = os.path.abspath(tf.name)
25 | kwargs['slavedest'] = '%s.bat' % remotename
26 |
27 | FileDownload.__init__(self, **kwargs)
28 |
--------------------------------------------------------------------------------
/tasks/tasktool/tasktool/hooks/check-clean-repo.sh:
--------------------------------------------------------------------------------
1 | # One of the most common errors is to forget to add files/changes to the config
2 | # repository. So we stop the process if the git repository shows any untracked
3 | # files or non-comitted changes.
4 | # TODO: Add a commandline flag or similar to override/ignore the checks?
5 |
6 | # Check that repo is clean
7 | if ! git diff-index --quiet HEAD --; then
8 | echo 1>&2 "Found uncommitted changes, the build probably won't work."
9 | echo 1>&2 "Note: Use `git status` to view them."
10 | exit 1
11 | fi
12 |
13 | # Check that the taskscript is actually part of the repo
14 | if ! git ls-files "${TASKSCRIPT}" --error-unmatch > /dev/null; then
15 | echo 1>&2 "Task '${TASKSCRIPT}' is not tracked by git."
16 | echo 1>&2 "Note: Add and commit the file to the git repository."
17 | exit 1
18 | fi
19 |
20 | # Check for untracked files
21 | if [ -n "$(git ls-files --others --exclude-standard)" ]; then
22 | echo 1>&2 "Error: There are untracked files around, the build probably won't work."
23 | echo 1>&2 "Note: - Add the files and commit them"
24 | echo 1>&2 "Note: - Or add them to \`.gitignore\` or \`.git/info/exclude\`"
25 | exit 1
26 | fi
27 |
--------------------------------------------------------------------------------
/tasks/tasktool/tasktool/hooks/try.sb:
--------------------------------------------------------------------------------
1 | ;; Sandbox profile for try executions so they can only write/remove stuff
2 | ;; within their workspace directory (TODO: Restrict reading too)
3 | (version 1)
4 | (allow default)
5 |
6 | (define (home-regex home-relative-regex)
7 | (regex (string-append "^" (regex-quote (param "HOME")) home-relative-regex)))
8 | (define (home-subpath home-relative-subpath)
9 | (subpath (string-append (param "HOME") home-relative-subpath)))
10 | (define (home-literal home-relative-literal)
11 | (literal (string-append (param "HOME") home-relative-literal)))
12 |
13 | (deny file-write*)
14 | (allow file-write*
15 | (subpath (param "WORKSPACE"))
16 | ; temp files/dirs
17 | (subpath "/private/tmp")
18 | (subpath "/private/var/folders")
19 | (subpath "/private/var/tmp")
20 | (home-subpath "/Library/Caches/pip")
21 | ; for /dev/null, /dev/dtracehelper possibly others
22 | (subpath "/dev")
23 | )
24 |
25 | ;; TODO: Restrict reading to a minimum. Something like:
26 | ;;(deny file-read*)
27 | ;;(allow file-read*
28 | ;; (subpath "/bin")
29 | ;; (subpath "/sbin")
30 | ;; (subpath "/usr/bin")
31 | ;; (subpath "/usr/local/bin")
32 | ;; (subpath "/Applications/Xcode.app")
33 | ;;)
34 |
--------------------------------------------------------------------------------
/codesign/debugsign/dbsign/logger.py:
--------------------------------------------------------------------------------
1 | # COPYRIGHT LINE: FIXME
2 |
3 | """
4 | dbsign.logger
5 |
6 | logging configuration for debugsign
7 | """
8 |
9 |
10 | from __future__ import print_function
11 |
12 | import logging
13 | import os
14 |
15 |
16 | DATE_FORMAT = "%H:%M:%S"
17 | LOG_FORMAT = "%(asctime)s %(name)13s:%(lineno)-4s %(levelname)7s %(message)s"
18 | BASE_LOGLEVEL = logging.WARNING
19 | LOGLEVEL = os.getenv('DEBUGSIGN_LOGLEVEL', BASE_LOGLEVEL)
20 |
21 | logging.basicConfig(format=LOG_FORMAT,
22 | datefmt=DATE_FORMAT,
23 | level=LOGLEVEL)
24 |
25 | _root = logging.getLogger()
26 | log = logging.getLogger(__name__)
27 |
28 |
29 | def get_logger(name): # type: (str) -> logging.Logger
30 | log.debug("Fetching logger for '%s'", name)
31 | return logging.getLogger(name)
32 |
33 |
34 | def set_level(level): # type: (int) -> int
35 | # must be done on root logger or it won't propagage to other loggers
36 | _root.setLevel(level)
37 | return _root.getEffectiveLevel()
38 |
39 |
40 | def normalize(level): # type: (int) -> int
41 | if level < logging.DEBUG:
42 | return logging.DEBUG
43 | if level > logging.CRITICAL:
44 | return logging.CRITICAL
45 | return level
46 |
--------------------------------------------------------------------------------
/llvmlab/llvmlab/user.py:
--------------------------------------------------------------------------------
1 | """
2 | LLVM-Lab User Objects
3 | """
4 |
5 | from llvmlab import util
6 |
7 | class User(util.simple_repr_mixin):
8 | @staticmethod
9 | def fromdata(data):
10 | version = data['version']
11 | if version != 0:
12 | raise ValueError, "Unknown version"
13 |
14 | return User(data['id'], data['passhash'],
15 | data['name'], data['email'],
16 | data['htpasswd'])
17 |
18 | def todata(self):
19 | return { 'version' : 0,
20 | 'id' : self.id,
21 | 'passhash' : self.passhash,
22 | 'name' : self.name,
23 | 'email' : self.email,
24 | 'htpasswd' : self.htpasswd }
25 |
26 | def __init__(self, id, passhash, name, email, htpasswd):
27 | self.id = id
28 | self.passhash = passhash
29 | self.name = name
30 | self.email = email
31 | self.htpasswd = htpasswd
32 |
33 | def has_lab_access(self):
34 | """has_lab_access() -> bool
35 |
36 | Is this user allowed access to the lab? Users with lab access can
37 | add/modify/remove machines and update other physical lab information.
38 | """
39 | return True
40 |
--------------------------------------------------------------------------------
/tasks/lnt-ctmark.sh:
--------------------------------------------------------------------------------
1 | SUBMIT_URL="$(build arg --optional SUBMIT_URL)"
2 | SUBMIT_NAME="$(build arg --optional SUBMIT_NAME)"
3 | SUBMIT_ORDER="$(build arg --optional SUBMIT_ORDER)"
4 | # This is should be used to supply the cmake cache file to use
5 | LNT_FLAGS="$(build arg --optional LNT_FLAGS)"
6 |
7 | # Set defaults for missing/empty parameters.
8 | : ${SUBMIT_URL:='http://104.154.54.203/db_default/v4/nts/submitRun'}
9 | : ${SUBMIT_NAME:="${JOB_NAME-}"}
10 |
11 | # A generic ctmark run designed to run as a recurring jenkins job with varying
12 | # cmake caches and submission to an lnt server.
13 | build get compiler
14 | build get test-suite
15 | build get lnt
16 |
17 | DEPENDENCY_FILES="${TASKDIR}"/lnt-testsuite.dep
18 | . "${TASKDIR}"/utils/check_dependencies.sh
19 | . "${TASKDIR}"/utils/normalize_compiler.sh
20 | . "${TASKDIR}"/utils/venv.sh
21 | . "${TASKDIR}"/utils/venv_lit.sh
22 | . "${TASKDIR}"/utils/venv_lnt.sh
23 |
24 | LNT_FLAGS+=" --cmake-define TEST_SUITE_RUN_BENCHMARKS=Off"
25 | LNT_FLAGS+=" --build-threads 1"
26 | LNT_FLAGS+=" --cmake-define TEST_SUITE_SUBDIRS=\"CTMark\""
27 | . "${TASKDIR}"/utils/lnt_test_suite.sh
28 | . "${TASKDIR}"/utils/lnt_submit.sh
29 |
30 | . "${TASKDIR}"/utils/lnt_move_results.sh
31 | . "${TASKDIR}"/utils/lnt_check_no_errors.sh
32 |
--------------------------------------------------------------------------------
/llvmlab/llvmlab/ui/templates/phase_popup.html:
--------------------------------------------------------------------------------
1 | {# Fragment template for rendering an individual phase pop-up #}
2 |
3 |
7 |
8 |
9 | Builder Status
11 | {% for name,builds in phased_builds|dictsort %}
12 |
13 | {{
14 | name }}
16 | {% for build in builds %}
17 | {% if build.end_time == None %}
18 | {% set kind = "running" %}
19 | {% set text = "RUNNING" %}
20 | {% elif build.result == 0 %}
21 | {% set kind = "success" %}
22 | {% set text = "PASS" %}
23 | {% else %}
24 | {% set kind = "failure" %}
25 | {% set text = "FAIL" %}
26 | {% endif %}
27 |
28 | {{ text }}
31 |
32 | {% endfor %}
33 |
34 |
35 | {% endfor %}
36 |
37 |
--------------------------------------------------------------------------------
/tasks/tasktool/tasktool/hooks/mk-submit-build:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -eu
3 |
4 | USERDIR="$1"
5 | TASKSCRIPT="$2"
6 | BUILDCONFIG="$3"
7 |
8 | TASK_DIR="$(dirname "${TASKSCRIPT}")"
9 | export GIT_WORK_TREE="$(git -C "${TASK_DIR}" rev-parse --show-toplevel)"
10 | GIT_DIR="$(git -C "${TASK_DIR}" rev-parse --git-dir)"
11 | export GIT_DIR="$(cd "${TASK_DIR}" ; cd "${GIT_DIR}" ; pwd)"
12 |
13 | . check-config.sh
14 | . "${USERDIR}/config"
15 |
16 | . check-clean-repo.sh
17 |
18 | # We need a branch name for remote submission
19 | BRANCH="$(git symbolic-ref HEAD)"
20 | BRANCH="${BRANCH##refs/heads/}"
21 | if test "$(git rev-parse HEAD)" != "$(git rev-parse $BRANCH)"; then
22 | echo 1>&2 "HEAD is not at branch for submission"
23 | exit 1
24 | fi
25 |
26 | # Make sure the branch is actually pushed to the remote
27 | if [ -z ${REMOTE+x} ]; then
28 | REMOTE="$(git config branch.${BRANCH}.remote)"
29 | if [ -z ${REMOTE+x} ]; then
30 | REMOTE="origin"
31 | fi
32 | fi
33 | if [ "$(git rev-parse $REMOTE/$BRANCH)" != "$(git rev-parse $BRANCH)" ]; then
34 | echo 1>&2 "Branch $BRANCH is not up to date on remote $REMOTE"
35 | exit 1
36 | fi
37 |
38 | CONFIG_URL="$(git remote get-url $REMOTE)"
39 | CONFIG_REV="$BRANCH"
40 |
41 | . mk-build-id.sh
42 | tail -n +4 header_submit
43 | . mk-build-common.sh
44 |
--------------------------------------------------------------------------------
/llvmbisect/llvmlab/shell.py:
--------------------------------------------------------------------------------
1 | """
2 | shell like utilities
3 | """
4 |
5 | import os
6 |
7 |
8 | def execute(args):
9 | import subprocess
10 | """execute(command) - Run the given command (or argv list) in a shell and
11 | return the exit code."""
12 | return subprocess.Popen(args).wait()
13 |
14 |
15 | def capture(args, include_stderr=False):
16 | import subprocess
17 | """capture(command) - Run the given command (or argv list) in a shell and
18 | return the standard output."""
19 | stderr = subprocess.PIPE
20 | if include_stderr:
21 | stderr = subprocess.STDOUT
22 | p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=stderr)
23 | out, _ = p.communicate()
24 | return p.wait(), out
25 |
26 |
27 | def mkdir_p(path):
28 | """mkdir_p(path) - Make the "path" directory, if it does not exist; this
29 | will also make directories for any missing parent directories."""
30 | import errno
31 |
32 | if not path or os.path.exists(path):
33 | return
34 |
35 | parent = os.path.dirname(path)
36 | if parent != path:
37 | mkdir_p(parent)
38 |
39 | try:
40 | os.mkdir(path)
41 | except OSError as e:
42 | # Ignore EEXIST, which may occur during a race condition.
43 | if e.errno != errno.EEXIST:
44 | raise
45 |
--------------------------------------------------------------------------------
/zorg/buildbot/builders/annotated/clang-windows.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 |
3 | import os
4 | import sys
5 | import annotated_builder
6 |
7 | def main(argv):
8 | ap = annotated_builder.get_argument_parser()
9 | args = ap.parse_args(argv[1:])
10 |
11 | projects = ['llvm', 'clang', 'lld', 'debuginfo-tests']
12 | stages = 2
13 | extra_cmake_args = [
14 | '-DCMAKE_BUILD_TYPE=Release',
15 | '-DLLVM_ENABLE_PDB=ON',
16 | '-DLLVM_ENABLE_ASSERTIONS=ON',
17 | '-DLLVM_TARGETS_TO_BUILD=all',
18 | ]
19 | check_targets = ['check-llvm', 'check-clang', 'check-lld', 'check-debuginfo']
20 |
21 | # Check both stage 1 and stage 2.
22 | check_stages = [True] * stages
23 |
24 | compiler = 'clang-cl'
25 | linker = 'lld-link'
26 |
27 | builder = annotated_builder.AnnotatedBuilder()
28 | builder.run_steps(projects=projects,
29 | stages=stages,
30 | extra_cmake_args=extra_cmake_args,
31 | check_targets=check_targets,
32 | check_stages=check_stages,
33 | compiler=compiler,
34 | linker=linker,
35 | jobs=args.jobs)
36 |
37 |
38 | if __name__ == '__main__':
39 | sys.path.append(os.path.dirname(__file__))
40 | sys.exit(main(sys.argv))
41 |
--------------------------------------------------------------------------------
/llvmbisect/setup.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | from setuptools import setup, find_packages
4 |
5 | # setuptools expects to be invoked from within the directory of setup.py, but it
6 | # is nice to allow:
7 | # python path/to/setup.py install
8 | # to work (for scripts, etc.)
9 | os.chdir(os.path.dirname(os.path.abspath(__file__)))
10 |
11 | setup(
12 | name = "llvmbisect",
13 | version = "1.0",
14 |
15 | author = "Daniel Dunbar and Chris Matthews",
16 | author_email = "chris.matthews@apple.com",
17 | url = 'http://lab.llvm.org',
18 | license = 'Apache-2.0 with LLVM exception',
19 |
20 | description = "Compiler bisection service.",
21 | keywords = 'testing compiler performance development llvm',
22 |
23 | classifiers=[
24 | 'Development Status :: 4 - Beta',
25 | 'Environment :: Console',
26 | 'Intended Audience :: Developers',
27 | 'License :: OSI Approved :: Apache-2.0 with LLVM exception',
28 | 'Natural Language :: English',
29 | 'Operating System :: OS Independent',
30 | 'Progamming Language :: Python',
31 | 'Topic :: Software Development :: Quality Assurance',
32 | 'Topic :: Software Development :: Testing',
33 | ],
34 |
35 | packages = find_packages(),
36 |
37 | scripts = ['bin/llvmlab'],
38 |
39 | install_requires=['requests'],
40 | )
41 |
--------------------------------------------------------------------------------
/llvmlab/llvmlab/util.py:
--------------------------------------------------------------------------------
1 | import colorsys
2 |
3 | __all__ = []
4 |
5 | class simple_repr_mixin(object):
6 | def __repr__(self):
7 | return "%s(%s)" % (self.__class__.__name__,
8 | ", ".join("%s=%r" % (k,v)
9 | for k,v in sorted(self.__dict__.items())))
10 |
11 | def sorted(items):
12 | items = list(items)
13 | items.sort()
14 | return items
15 |
16 | def make_dark_color(h):
17 | h = h % 1.
18 | s = 0.95
19 | v = 0.8
20 | return colorsys.hsv_to_rgb(h,0.9+s*.1,v)
21 |
22 | class multidict(object):
23 | def __init__(self, elts=()):
24 | self.data = {}
25 | for key,value in elts:
26 | self[key] = value
27 |
28 | def __contains__(self, item):
29 | return item in self.data
30 | def __getitem__(self, item):
31 | return self.data[item]
32 | def __setitem__(self, key, value):
33 | if key in self.data:
34 | self.data[key].append(value)
35 | else:
36 | self.data[key] = [value]
37 | def items(self):
38 | return self.data.items()
39 | def values(self):
40 | return self.data.values()
41 | def keys(self):
42 | return self.data.keys()
43 | def __len__(self):
44 | return len(self.data)
45 | def get(self, key, default=None):
46 | return self.data.get(key, default)
47 |
--------------------------------------------------------------------------------
/llvmlab/llvmlab/data.py:
--------------------------------------------------------------------------------
1 | """
2 | LLVM-Lab Data Management
3 | """
4 |
5 | from llvmlab import util
6 | from llvmlab import machine, user
7 |
8 | class Data(util.simple_repr_mixin):
9 | @staticmethod
10 | def fromdata(data):
11 | version = data['version']
12 | if version != 0:
13 | raise ValueError, "Unknown version"
14 |
15 | users = [user.User.fromdata(u)
16 | for u in data['users']]
17 | machines = [machine.Machine.fromdata(u)
18 | for u in data['machines']]
19 | return Data(users, machines)
20 |
21 | def todata(self):
22 | return { 'version' : 0,
23 | 'users' : [item.todata()
24 | for item in self.users.values()
25 | if item is not self.admin_user],
26 | 'machines' : [item.todata()
27 | for item in self.machines.values()] }
28 |
29 | def __init__(self, users, machines):
30 | self.machines = dict((item.id, item) for item in machines)
31 | self.users = dict((item.id, item) for item in users)
32 | self.admin_user = None
33 |
34 | def set_admin_user(self, user):
35 | if user.id in self.users:
36 | raise ValueError, "database contains admin user %r" % user.id
37 |
38 | self.admin_user = user
39 | self.users[user.id] = user
40 |
--------------------------------------------------------------------------------
/llvmlab/llvmlab/ui/templates/index.html:
--------------------------------------------------------------------------------
1 | {% extends "layout.html" %}
2 | {% set components = [] %}
3 | {% block title %}home{% endblock %}
4 | {% block body %}
5 |
6 |
19 |
20 | Welcome to the LLVM Laboratory!
21 |
22 | The LLVM laboratory is a set of machines and software to support
23 | the LLVM project, for use by LLVM developers and
24 | community members.
25 |
26 | Sponsors
27 |
28 | Access Softek : a software development and consulting company with a
29 | specialization in compiler development, is pleased to help the LLVM development
30 | effort by hosting the LLVM lab in its Berkeley office.
31 |
32 | For more information about us please take a look at our very out-of-date
33 | website {{ "www.accesssoftek.com"|urlize }} and if you are
34 | an LLVM developer passing through Berkeley, drop in and say hi.
35 |
36 | {% endblock %}
37 |
--------------------------------------------------------------------------------
/llvmlab/llvmlab/ui/templates/layout.html:
--------------------------------------------------------------------------------
1 |
3 |
4 |
5 | {{ self.title() }}
7 |
24 | {% for message in get_flashed_messages() %}
25 |
{{ message }}
26 | {% endfor %}
27 |
{% block body %}{% endblock %}
29 |
37 |
38 |
39 |
--------------------------------------------------------------------------------
/tasks/utils/test_suite_compile.sh:
--------------------------------------------------------------------------------
1 | echo "@@@ Test-Suite Compile @@@"
2 | rm -rf "test-suite-build"
3 | mkdir -p "test-suite-build"
4 | cd "test-suite-build"
5 | TEST_SUITE_CMAKE_FLAGS+="" # Make sure variable is defined
6 | # Note that we prepend CMAKE_C_COMPILER to ensure it is set before cache files
7 | # (-C flags) are executed.
8 | TEST_SUITE_CMAKE_FLAGS="-DCMAKE_C_COMPILER=\"${WORKSPACE}/compiler/bin/clang\" ${TEST_SUITE_CMAKE_FLAGS}"
9 |
10 | # Default value
11 | : ${TEST_SUITE_PGO_BUILD:=}
12 |
13 | TEST_SUITE_CMAKE_FLAGS+=" -GNinja"
14 | TEST_SUITE_CMAKE_FLAGS+=" ${WORKSPACE}/test-suite"
15 | if [ "${TEST_SUITE_PGO_BUILD}" = "1" ]; then
16 | TEST_SUITE_CMAKE_FLAGS+=" -DTEST_SUITE_RUN_TYPE=train"
17 | TEST_SUITE_CMAKE_FLAGS+=" -DTEST_SUITE_PROFILE_GENERATE=On"
18 | fi
19 | eval cmake ${TEST_SUITE_CMAKE_FLAGS}
20 |
21 | TEST_SUITE_NINJA_FLAGS+="" # Make sure variable is defined
22 | eval ninja "${TEST_SUITE_NINJA_FLAGS}"
23 |
24 | if [ "${TEST_SUITE_PGO_BUILD}" = "1" ]; then
25 | if [ -z ${TEST_SUITE_LIT_TARGETS+x} ]; then
26 | TEST_SUITE_LIT_TARGETS="."
27 | fi
28 | TEST_SUITE_LIT_FLAGS+="" # make sure variable is defined
29 | eval lit -j1 ${TEST_SUITE_LIT_FLAGS} ${TEST_SUITE_LIT_TARGETS}
30 | # Rerun cmake with flipped profile flags
31 | eval cmake -DTEST_SUITE_PROFILE_GENERATE=Off -DTEST_SUITE_PROFILE_USE=On -DTEST_SUITE_RUN_TYPE=ref .
32 | eval ninja "${TEST_SUITE_NINJA_FLAGS}"
33 | fi
34 |
35 | cd "${WORKSPACE}"
36 | echo "@@@@@@"
37 |
--------------------------------------------------------------------------------
/llvmlab/docs/_templates/index.html:
--------------------------------------------------------------------------------
1 | {% extends "layout.html" %}
2 | {% set title = 'llvmlab' %}
3 | {% block body %}
4 | llvmlab - LLVM Dashboard
5 |
6 |
7 | llvmlab is a dashboard for the LLVM project.
8 |
9 | This is the web page for the llvmlab software itself, the current online
10 | version of the server is available
11 | at http://lab.llvm.org/lab/ .
12 |
13 |
14 | Documentation
15 |
16 |
17 |
18 |
19 | Contents for a complete overview
21 |
22 | Search page search the documentation
24 |
25 |
26 | General Index all functions, classes, terms
28 |
29 |
30 |
31 | Source
32 | llvmlab is available in the LLVM "zorg" SVN repository:
33 | svn co
34 |
35 | http://llvm.org/svn/llvm-project/zorg/trunk/llvmlab .
36 |
37 | {% endblock %}
38 |
--------------------------------------------------------------------------------
/codesign/debugsign/dbsign/result.py:
--------------------------------------------------------------------------------
1 | # COPYRIGHT LINE: FIXME
2 |
3 | """
4 | dbsign.result
5 |
6 | result classes for debugsign
7 | """
8 |
9 | from __future__ import print_function
10 |
11 | import dbsign.logger as logger
12 |
13 |
14 | log = logger.get_logger(__name__)
15 |
16 |
17 | #
18 | # Result class
19 | #
20 |
21 | class Result(object):
22 | def __init__(self, value): # type: () -> ()
23 | self._checked = False
24 | self._value = value
25 |
26 | def __del__(self): # type: () -> ()
27 | assert self._checked
28 |
29 | def __nonzero__(self): # type: () -> bool
30 | raise NotImplementedError("{} does not support boolean evaluation".
31 | format(self.__class__.__name__))
32 |
33 | def __repr__(self): # type: () -> str
34 | return "{0.__class__.__name__}({0._value!r})".format(self)
35 |
36 | @property
37 | def checked(self): # type: () -> bool
38 | return self._checked
39 |
40 | @property
41 | def value(self): # type: () -> str
42 | self._checked = True
43 | return self._value
44 |
45 | def renew(self): # type: () -> Result
46 | self._checked = False
47 | return self
48 |
49 |
50 | class Failure(Result):
51 | def __nonzero__(self): # type: () -> bool
52 | self._checked = True
53 | return False
54 |
55 |
56 | class Success(Result):
57 | def __nonzero__(self): # type: () -> bool
58 | self._checked = True
59 | return True
60 |
--------------------------------------------------------------------------------
/zorg/buildbot/util/reloading.py:
--------------------------------------------------------------------------------
1 | """
2 | Helper module for defining a utility which will reload all of the
3 | modules used in the system.
4 |
5 | This is important when using buildbot's reload command.
6 | """
7 |
8 | import os
9 | import sys
10 |
11 | time_cache = {}
12 |
13 | def path_starts_with_one_of(path, paths):
14 | for p in paths:
15 | if path.startswith(p):
16 | return True
17 |
18 | def reload_all(only_paths = [], log = False):
19 | # Reload all modules in sys.modules which have changed.
20 | for module in sys.modules.values():
21 | if not hasattr(module, '__file__'):
22 | continue
23 | path = getattr(module, '__file__')
24 | if not path:
25 | continue
26 | if os.path.splitext(path)[1] in ['.pyc', '.pyo', '.pyd']:
27 | path = path[:-1]
28 |
29 | # Never reload ourselves, we don't want to kill the cache.
30 | if path == __file__:
31 | continue
32 |
33 | # If we were given a limited path list, only reload modules
34 | # with paths under that.
35 | if only_paths and not path_starts_with_one_of(path, only_paths):
36 | continue
37 |
38 | if os.path.isfile(path):
39 | mtime = os.stat(path).st_mtime
40 | if path not in time_cache or mtime != time_cache[path]:
41 | if log:
42 | print >>sys.stderr, "note: reloading %r" % path
43 | time_cache[path] = mtime
44 | reload(module)
45 |
--------------------------------------------------------------------------------
/zorg/jenkins/jobs/delete_old_jobs.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | import glob
3 | import os
4 | import subprocess
5 | import sys
6 | import time
7 | import xml.etree.ElementTree as ET
8 |
9 | new_jobs = set()
10 | for g in glob.glob('build/jenkins/job/*'):
11 | new_jobs.add(os.path.basename(g))
12 |
13 | if len(new_jobs) == 0:
14 | print "No new jobs?!?"
15 | sys.exit(1)
16 |
17 | query = subprocess.check_output(['util/query.sh', 'api/xml?tree=jobs[name,description]'], )
18 |
19 | existing_jobs = set()
20 | tree = ET.fromstring(query)
21 | for job in tree.findall('.//job'):
22 | name = job.find('name').text
23 | description_e = job.find('description')
24 | if description_e is None:
25 | continue
26 | description = description_e.text
27 | if description is None:
28 | continue
29 | if '$$job generated from ' in description:
30 | existing_jobs.add(name.strip())
31 | if len(existing_jobs) == 0:
32 | print "No existing jobs?!?"
33 | sys.exit(1)
34 |
35 | # We should have already uploaded all the new jobs
36 | missing = new_jobs - existing_jobs
37 | if len(missing) > 0:
38 | print "Missing jobs?!?"
39 | sys.exit(1)
40 |
41 | to_delete = existing_jobs - new_jobs
42 | if len(to_delete) > 0:
43 | print ""
44 | print ""
45 | print "Will delete the following jobs:"
46 | for jobname in to_delete:
47 | print " %s" % jobname
48 | print "You have 5 seconds to abort"
49 | time.sleep(5)
50 | for jobname in to_delete:
51 | subprocess.check_call(['util/delete_job.sh', jobname])
52 |
--------------------------------------------------------------------------------
/zorg/jenkins/jobs/util/delete_old_jobs.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | import glob
3 | import os
4 | import subprocess
5 | import sys
6 | import time
7 | import xml.etree.ElementTree as ET
8 |
9 | new_jobs = set()
10 | for g in glob.glob('build/jenkins/job/*'):
11 | new_jobs.add(os.path.basename(g))
12 |
13 | if len(new_jobs) == 0:
14 | print "No new jobs?!?"
15 | sys.exit(1)
16 |
17 | query = subprocess.check_output(['util/query.sh', 'api/xml?tree=jobs[name,description]'], )
18 |
19 | existing_jobs = set()
20 | tree = ET.fromstring(query)
21 | for job in tree.findall('.//job'):
22 | name = job.find('name').text
23 | description_e = job.find('description')
24 | if description_e is None:
25 | continue
26 | description = description_e.text
27 | if description is None:
28 | continue
29 | if '$$GENERATED BY llvm-ci-tasks$$' in description:
30 | existing_jobs.add(name.strip())
31 | if len(existing_jobs) == 0:
32 | print "No existing jobs?!?"
33 | sys.exit(1)
34 |
35 | # We should have already uploaded all the new jobs
36 | missing = new_jobs - existing_jobs
37 | if len(missing) > 0:
38 | print "Missing jobs?!?"
39 | sys.exit(1)
40 |
41 | to_delete = existing_jobs - new_jobs
42 | if len(to_delete) > 0:
43 | print ""
44 | print ""
45 | print "Will delete the following jobs:"
46 | for jobname in to_delete:
47 | print " %s" % jobname
48 | print "You have 5 seconds to abort"
49 | time.sleep(5)
50 | for jobname in to_delete:
51 | subprocess.check_call(['util/delete_job.sh', jobname])
52 |
--------------------------------------------------------------------------------
/tasks/tasktool/tasktool/hooks/submit:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | set -eu -o pipefail
3 |
4 | USERDIR="$1"
5 | BUILDSCRIPT="$2"
6 | BUILDNAME="$3"
7 |
8 | . check-config.sh
9 | . "${USERDIR}/config"
10 |
11 | CURL_ARGS=" -s -XPOST"
12 | CURL_ARGS+=" -i"
13 | CURL_ARGS+=" -u \"${JENKINS_USER}:${JENKINS_TOKEN}\""
14 | CURL_ARGS+=" \"${JENKINS_URL}/job/${JENKINS_SUBMIT_PROJECT}/buildWithParameters\""
15 | CURL_ARGS+=" --data-urlencode BUILDSCRIPT@\"$BUILDSCRIPT\""
16 | CURL_ARGS+=" --data-urlencode BUILDNAME=\"${BUILDNAME}\""
17 | if [ -n "${JENKINS_SUBMIT_NODE+}" ]; then
18 | CURL_ARGS+=" --data-urlencode where=\"${JENKINS_SUBMIT_NODE}\""
19 | fi
20 |
21 | eval curl $CURL_ARGS | grep Location
22 |
23 | # TODO: Get a handle on something that allows us to track the build as it goes
24 | # through jenkins. This is hard to do because:
25 | # - Jenkins gives us back a handle for the build in the Queue (the "Location"
26 | # header in the answer)
27 | # - Once the build is executed the queue handle gets updated with a handle
28 | # to the actual build. However queue handle becomes invalid after a while.
29 | # => You have to actively poll/query for the queue item changing into a build to
30 | # catch it :-/
31 | #
32 | # Alternatively our builds are annotated with a unique ID and we set the jenkins
33 | # build name to that. We could search all the jenkins builds to find a specific
34 | # ID (probably good enough for a single node, is that efficient when we have
35 | # to search mutliple or all nodes?).
36 |
37 | # For now the user has to search/track himself:
38 | echo ""
39 | echo "Note: Build results at ${JENKINS_URL}/job/${JENKINS_SUBMIT_PROJECT}"
40 |
--------------------------------------------------------------------------------
/tasks/tasktool/tasktool/hooks/exec-try-build:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -eu
3 |
4 | USERDIR="$1"
5 | BUILDSCRIPT="$2"
6 | BUILDNAME="$3"
7 |
8 | # Import config if present.
9 | [ -e "${USERDIR}/config" ] && . "${USERDIR}/config"
10 |
11 | # Create workspace directory.
12 | if [ -z ${TRYDIR+x} ]; then
13 | TRYDIR="$TMPDIR"
14 | if [ -z ${TRYDIR+x} ]; then
15 | TRYDIR="/tmp"
16 | fi
17 | fi
18 |
19 | export WORKSPACE="${TRYDIR}/${BUILDNAME}"
20 | mkdir "${TRYDIR}/${BUILDNAME}"
21 | echo "Workspace: ${WORKSPACE}"
22 |
23 | # Work with a minimum PATH
24 | export PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
25 |
26 | # Clear environment from some typical user-specific settings.
27 | # TODO: Should we rather start with a blank environment?
28 | unset CPATH
29 | unset LIBRARY_PATH
30 | unset MANPATH
31 | unset PKG_CONFIG_PATH
32 |
33 | LOGFILE="${WORKSPACE}/log.txt"
34 | SANDBOXPROFILE="$(pwd)/try.sb"
35 | cd "${WORKSPACE}"
36 | : ${HOME:=/tmp}
37 |
38 | if [ -z "${TASK_TRY_NO_SANDBOX:-}" ]; then
39 | /bin/sh "${BUILDSCRIPT}" 2>&1 | tee "$LOGFILE"
40 | else
41 | # Note that sandboxes don't work recursively, so we set TASK_TRY_NO_SANDBOX
42 | # for the child.
43 | TASK_TRY_NO_SANDBOX=1 sandbox-exec -f "${SANDBOXPROFILE}" -D WORKSPACE="$WORKSPACE" -D HOME="$HOME" /bin/sh "${BUILDSCRIPT}" 2>&1 | tee "$LOGFILE"
44 | fi
45 |
46 | # Gather results
47 | if [ -d "${WORKSPACE}/result" ]; then
48 | mkdir -p "${USERDIR}/results"
49 | RESULTFILE="${USERDIR}/results/${BUILDNAME}.tar.xz"
50 | cd "${WORKSPACE}/result"
51 | shopt -s dotglob nullglob
52 | tar -c --xz -f "${RESULTFILE}" *
53 | echo 1>&2 "Results written to ${RESULTFILE}"
54 | fi
55 |
--------------------------------------------------------------------------------
/zorg/buildbot/commands/LICENSE.chromium.TXT:
--------------------------------------------------------------------------------
1 | // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
2 | //
3 | // Redistribution and use in source and binary forms, with or without
4 | // modification, are permitted provided that the following conditions are
5 | // met:
6 | //
7 | // * Redistributions of source code must retain the above copyright
8 | // notice, this list of conditions and the following disclaimer.
9 | // * Redistributions in binary form must reproduce the above
10 | // copyright notice, this list of conditions and the following disclaimer
11 | // in the documentation and/or other materials provided with the
12 | // distribution.
13 | // * Neither the name of Google Inc. nor the names of its
14 | // contributors may be used to endorse or promote products derived from
15 | // this software without specific prior written permission.
16 | //
17 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 |
--------------------------------------------------------------------------------
/codesign/debugsign/unittests/test_logger.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python -tt
2 |
3 | """
4 | unittests.test_logger
5 | """
6 |
7 | try:
8 | import unittest2 as unittest
9 | except ImportError:
10 | import unittest
11 |
12 | import logging
13 |
14 | import dbsign.logger as L
15 |
16 |
17 | class TestLogger(unittest.TestCase):
18 | def test_logger(self): # type: () -> ()
19 | # This test assumes the initial, intended semantics of the various
20 | # loggers returned. If semantics change, so must the tests.
21 | init_level = L._root.level
22 | test_level = 7
23 | log_name = "TestLoggerLogger"
24 | log = L.get_logger(log_name)
25 |
26 | with self.subTest(method=L.get_logger):
27 | self.assertEqual(log.__class__, logging.Logger)
28 | self.assertEqual(log_name, log.name)
29 | # Returned loggers should default to 0 (inherit parent level)
30 | self.assertEqual(0, log.level)
31 |
32 | with self.subTest(method=L.set_level):
33 | # Assert set_level() sets root level, not sublogger level
34 | L.set_level(test_level)
35 | self.assertEqual(test_level, L._root.level)
36 | self.assertEqual(0, log.level)
37 |
38 | # Restore and verify
39 | L.set_level(init_level)
40 | self.assertEqual(init_level, L._root.level)
41 | self.assertEqual(0, log.level)
42 |
43 | def test_normalize(self): # type: () -> ()
44 | self.assertEqual(logging.WARN, L.normalize(logging.WARN))
45 | self.assertEqual(logging.DEBUG, L.normalize(logging.DEBUG - 1))
46 | self.assertEqual(logging.CRITICAL, L.normalize(logging.CRITICAL + 1))
47 |
--------------------------------------------------------------------------------
/codesign/debugsign/README.md:
--------------------------------------------------------------------------------
1 | # LLDB Codesigning Identities
2 |
3 | The **`debugsign`** tool contains the logic necessary to configure
4 | and enable automation-friendly codesigning for LLDB builds on Darwin
5 | systems.
6 |
7 | The **import** sub-command requires a P12 archive containing a
8 | codesigning identity. A P12 archive compatible with the
9 | **`debugsign`** tool can produced using the following procedure:
10 |
11 | # Generating a Codesigning Identity
12 |
13 | * Open **Keychain Access**
14 | * -> **App Menu**
15 | * -> **Certificate Assistant**
16 | * -> **Create a Certificate**
17 |
18 | In the dialog that appears:
19 |
20 | * Name: **`lldb_codesign`**
21 | * Certificate Type: **`Code Signing`**
22 | * use defaults for all other values and complete the dialog sequence
23 |
24 | The identity (private key, public key, self-signed certificate) will
25 | be created in the default login keychain. Export the identity:
26 |
27 | * Select **login** keychain
28 | * Select **My Certificates**
29 | * Select entire **lldb_codesign** item (not sub-components)
30 | * **File** -> **Export Items**
31 | * Select File Format: **`Personal Information Exchange (.p12)`**
32 | * When asked for a password to protect the exported archive, enter
33 | **`lldb_codesign`** for both options.
34 | * If **Keychain Access** asks for permission to export, click **Allow**
35 |
36 | The resulting ``.p12`` file can be passed to **`debugsign`**'s import
37 | sub-command.
38 |
39 | # Using the debugsign Script
40 |
41 | Once you've created the **`.p12`** archive, you can use it to enable
42 | codesigning on any number of machines for as long as the generated
43 | certificate is valid.
44 |
45 | The general use case for ``debugsign`` is described in the script's
46 | help output: **`debugsign help`**
47 |
48 |
--------------------------------------------------------------------------------
/zorg/buildbot/builders/sanitizers/buildbot_gn.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | set -exu
4 |
5 | HERE="$(cd $(dirname $0) && pwd)"
6 | . ${HERE}/buildbot_functions.sh
7 |
8 | ROOT=`pwd`
9 | PLATFORM=`uname`
10 |
11 | USE_GIT=1
12 |
13 | CHECK_LIBCXX=${CHECK_LIBCXX:-0}
14 | CHECK_LLD=${CHECK_LLD:-1}
15 | STAGE1_CLOBBER=
16 | STAGE2_DIR=llvm_build
17 |
18 | CLOBBER="gn ${STAGE2_DIR}"
19 | clobber
20 |
21 | LLVM=$ROOT/llvm
22 | CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_PARALLEL_LINK_JOBS=20"
23 | build_stage1_clang_at_revison
24 |
25 | echo @@@BUILD_STEP build GN@@@
26 | [[ -d gn ]] || git clone https://gn.googlesource.com/gn
27 | (
28 | export PATH="$PATH:$(readlink -f ${STAGE1_DIR}/bin)"
29 | cd gn
30 | git checkout d3304fbba9e39a5e996cbc8c769499a1079a8743
31 | python build/gen.py
32 | ninja -C out
33 | ) || { echo @@@STEP_EXCEPTION@@@ ; exit 1 ; }
34 |
35 | buildbot_update
36 |
37 | (
38 | echo @@@BUILD_STEP sync sources from cmake@@@
39 | export PATH=$(readlink -f gn/out/):$PATH
40 | cd $LLVM/..
41 | $LLVM/utils/gn/build/sync_source_lists_from_cmake.py
42 | ) || echo @@@STEP_WARNINGS@@@
43 |
44 | (
45 | echo @@@BUILD_STEP configure@@@
46 | export PATH=$(readlink -f gn/out/):$PATH
47 | $LLVM/utils/gn/gn.py gen ${STAGE2_DIR} \
48 | --list --short --overrides-only \
49 | --args="clang_base_path=\"$(readlink -f ${STAGE1_DIR})\""
50 | ) || echo @@@STEP_FAILURE@@@
51 |
52 | (
53 | cd $STAGE2_DIR
54 | for TARGET in "" $(ninja -t targets | grep -o "^check-[^:]*") ; do
55 | echo @@@BUILD_STEP ninja $TARGET@@@
56 | if [[ "$TARGET" == "check-clang-tools" ]] ; then
57 | STEP_ERROR=@@@STEP_WARNINGS@@@
58 | else
59 | STEP_ERROR=@@@STEP_FAILURE@@@
60 | fi
61 | ninja $TARGET || echo $STEP_ERROR
62 | done
63 | )
64 |
--------------------------------------------------------------------------------
/zorg/buildbot/builders/sanitizers/buildbot_android.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | set -x
4 | set -e
5 | set -u
6 |
7 | HERE="$(cd $(dirname $0) && pwd)"
8 | . ${HERE}/buildbot_functions.sh
9 | . ${HERE}/buildbot_android_functions.sh
10 |
11 | ROOT=`pwd`
12 | PLATFORM=`uname`
13 | LOCAL_IPS=`hostname -I`
14 | export PATH="/usr/local/bin:$PATH"
15 |
16 | USE_GIT=1
17 |
18 | CHECK_LIBCXX=${CHECK_LIBCXX:-1}
19 | CHECK_LLD=${CHECK_LLD:-1}
20 | LLVM=$ROOT/llvm
21 | CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS:-}"
22 | CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS:-} -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_PARALLEL_LINK_JOBS=20"
23 |
24 | if [ -e /usr/include/plugin-api.h ]; then
25 | CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS} -DLLVM_BINUTILS_INCDIR=/usr/include"
26 | fi
27 |
28 | clobber
29 |
30 | download_android_tools r16
31 |
32 | # Stage 1
33 |
34 | build_stage1_clang_at_revison
35 | ### From now on we use just-built Clang as a host compiler ###
36 |
37 | buildbot_update
38 |
39 | CMAKE_COMMON_OPTIONS="$CMAKE_COMMON_OPTIONS -DLLVM_ENABLE_ASSERTIONS=ON"
40 |
41 | build_stage2_android
42 |
43 | # Android NDK has no iconv.h which is requred by LIBXML2.
44 | CMAKE_COMMON_OPTIONS="${CMAKE_COMMON_OPTIONS} -DLLVM_LIBXML2_ENABLED=OFF"
45 |
46 | build_android_ndk aarch64 arm64
47 | build_android_ndk arm arm
48 | build_android_ndk i686 x86
49 |
50 | echo @@@BUILD_STEP run cmake@@@
51 | configure_android aarch64 aarch64-linux-android
52 | # Testing armv7 instead of plain arm to work around
53 | # https://code.google.com/p/android/issues/detail?id=68779
54 | configure_android arm armv7-linux-androideabi
55 | configure_android i686 i686-linux-android
56 |
57 | build_android aarch64
58 | build_android arm
59 | build_android i686
60 |
61 | test_android i686:x86 aarch64:arm64-v8a arm:armeabi-v7a
62 |
--------------------------------------------------------------------------------
/zorg/jenkins/email.template:
--------------------------------------------------------------------------------
1 |
2 |
3 | Report
4 |
19 |
20 |
21 | [${currentBuild.currentResult} ] ${env.JOB_NAME} [${env.BUILD_DISPLAY_NAME}]
22 |
23 |
24 | Build URL: ${env.BUILD_URL} Project: ${env.JOB_NAME} Build duration: ${hudson.Util.getTimeSpanString(System.currentTimeMillis() - currentBuild.startTimeInMillis)}
28 |
29 | Issues
30 | <% if (log_summary == "") { %>
31 | No known issues detected
32 | <% } else { %>
33 | ${log_summary}
34 | <% } %>
35 | Build Log
36 |
37 | Changes
38 |
39 | <% def changeSets = currentBuild.getChangeSets()
40 | def hadChanges = false
41 | changeSets.each { changeSet ->
42 | %>
43 | <% changeSet.each { cs ->
44 | hadChanges = true
45 | aUser = cs.author %>
46 | Commit ${cs.revision} by <%= aUser != null ? aUser.displayName : it.author.displayName %>: ${cs.msg}
47 |
48 | <% cs.affectedFiles.each { %>
49 | ${it.editType.name} : ${it.path}
52 |
53 | No Changes
59 | <% } %>
60 |
61 |
62 |
63 |
--------------------------------------------------------------------------------
/llvmlab/llvmlab/ui/templates/build_chart.html:
--------------------------------------------------------------------------------
1 | {% extends "layout.html" %}
2 | {% block title %}build chart{% endblock %}
3 | {% block head %}
4 |
27 | {% endblock %}
28 | {% block body %}
29 |
30 | Build Chart
31 |
32 |
33 | {% for slave,builds in build_chart.data|dictsort %}
34 |
35 | {{ slave }}
36 |
37 |
38 | {% for row in builds %}
39 |
40 |
41 |
45 | {% for item in row %}
46 |
47 |
54 | {% endfor %}
55 |
57 |
58 | {% endfor %}
59 |
60 |
61 | {% endfor %}
62 |
63 |
64 | {% endblock %}
65 |
--------------------------------------------------------------------------------
/llvmlab/setup.py:
--------------------------------------------------------------------------------
1 | import llvmlab
2 | import os
3 |
4 | from setuptools import setup, find_packages
5 |
6 | # setuptools expects to be invoked from within the directory of setup.py, but it
7 | # is nice to allow:
8 | # python path/to/setup.py install
9 | # to work (for scripts, etc.)
10 | os.chdir(os.path.dirname(os.path.abspath(__file__)))
11 |
12 | setup(
13 | name = "llvmlab",
14 | version = llvmlab.__version__,
15 |
16 | author = llvmlab.__author__,
17 | author_email = llvmlab.__email__,
18 | url = 'http://lab.llvm.org',
19 | license = 'Apache-2.0 with LLVM exception',
20 |
21 | description = "lab.llvm.org Web App",
22 | keywords = 'web testing performance development llvm',
23 |
24 | classifiers=[
25 | 'Development Status :: 1 - Planning',
26 | 'Environment :: Web Environment',
27 | 'Intended Audience :: Developers',
28 | 'License :: OSI Approved :: Apache-2.0 with LLVM exception',
29 | 'Natural Language :: English',
30 | 'Operating System :: OS Independent',
31 | 'Progamming Language :: Python',
32 | 'Topic :: Software Development :: Quality Assurance',
33 | 'Topic :: Software Development :: Testing',
34 | ],
35 |
36 | # Additional resource extensions we use.
37 | package_data = { 'llvmlab.llvmlabtool' : ['*.sample'],
38 | 'llvmlab.ui' : ['static/*.ico',
39 | 'static/*.css',
40 | 'static/*.js',
41 | 'static/jquery/1.5/*.js',
42 | 'templates/*.html' ] },
43 |
44 | zip_safe = False,
45 | packages = find_packages(),
46 | entry_points = {
47 | 'console_scripts': [
48 | 'llvmlab = llvmlab.llvmlabtool:main',
49 | ],
50 | },
51 | install_requires=['Flask'],
52 | )
53 |
--------------------------------------------------------------------------------
/llvmbisect/llvmlab/test_llvmlab.py:
--------------------------------------------------------------------------------
1 | # RUN: pythong test_llvmlab.py
2 | import unittest
3 |
4 | import os
5 | import shutil
6 | import tempfile
7 | from . import ci
8 |
9 | class TestLLVMLabCI(unittest.TestCase):
10 |
11 | def setUp(self):
12 | self.workdir = tempfile.mkdtemp()
13 | print self.workdir
14 | os.chdir(self.workdir)
15 |
16 | def tearDown(self):
17 | shutil.rmtree(self.workdir)
18 |
19 | def test_bisect(self):
20 | ci.action_bisect("llvmlab", ["--min-rev", "219719",
21 | "--max-rev", "219899",
22 | "bash", "-c",
23 | "%(path)s/bin/clang -v | grep b700"])
24 |
25 | def test_ls(self):
26 | """Check that you can """
27 | builds = ci.action_ls("llvmlab", [])
28 | self.assertIn("clang-stage1-configure-RA_build", builds)
29 | compilers = ci.action_ls("llvmlab",
30 | ["clang-stage1-configure-RA_build"])
31 | compiler_revs = [x.revision for x in compilers]
32 | self.assertIn(219899, compiler_revs)
33 |
34 | def test_fetch_noargs(self):
35 | """ """
36 | path = ci.action_fetch("llvmlab", ["clang-stage1-configure-RA_build"])
37 | self.assertTrue(os.path.isdir(path), "Fetch did not get a compiler?")
38 |
39 | def test_fetch_arg(self):
40 | """ """
41 | path = ci.action_fetch("llvmlab",
42 | ["--update-link", "clang_link",
43 | "clang-stage1-configure-RA_build",
44 | "clang-r219899-t2014-10-15_20-42-53-b808"])
45 | self.assertTrue(os.path.isdir(path), "Fetch did not get a compiler?")
46 |
47 |
48 | def run_tests():
49 | suite = unittest.TestLoader().loadTestsFromTestCase(TestLLVMLabCI)
50 | unittest.TextTestRunner(verbosity=2).run(suite)
51 |
--------------------------------------------------------------------------------
/tasks/tasktool/tasktool/repos/artifact_server.py:
--------------------------------------------------------------------------------
1 | '''
2 | Get artifact from an artifact server.
3 |
4 | The server should present a directory structure over http/https. The leaf
5 | directories should contain a file called `last_good_build.properties`
6 | (typically generated by jenkins) containing a line like:
7 |
8 | ARTIFACT=subdir/buz.tar.gz
9 |
10 | which is then resolved as the latest version available for this directory.
11 | '''
12 | import os
13 | import sys
14 | import tasktool.utils as utils
15 |
16 |
17 | def verify(config):
18 | if config.get('url') is None:
19 | raise Exception("No 'url' specified")
20 |
21 |
22 | def resolve_latest(config):
23 | assert(config['type'] == 'artifact_server')
24 | url = config['url']
25 | if not url.endswith('/'):
26 | url += '/'
27 | latest_info_file = config.get('latest_info_file',
28 | 'last_good_build.properties')
29 | latest_url = url + latest_info_file
30 | props = utils.check_output(['curl', '-s', latest_url])
31 |
32 | res = None
33 | latest_key = config.pop('latest_key', 'ARTIFACT')
34 | for line in props.split('\n'):
35 | line = line.strip()
36 | if line.startswith('%s=' % latest_key):
37 | res = line[len(latest_key)+1:]
38 | break
39 |
40 | # We usually have something like, try to resolve this to a single URL
41 | # URL=https://myserver.com/mybuild/
42 | # ARTIFACT=mybuild/blabla.tar.gz
43 | dirname = os.path.dirname(res) + "/"
44 | full_url = url
45 | if full_url.endswith(dirname):
46 | full_url = full_url[:-len(dirname)]
47 | full_url += res
48 | config['url'] = full_url
49 | config['type'] = 'url'
50 |
51 |
52 | def get_artifact(config, dest_dir):
53 | raise Exception("A2 artifact not resolved to URL?!?")
54 |
55 |
56 | def repro_arg(config, dest_dir):
57 | raise Exception("A2 artifact not resolved to URL?!?")
58 |
--------------------------------------------------------------------------------
/dep/tests/assets/brew_ninja_not_installed.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "name": "ninja",
4 | "full_name": "ninja",
5 | "desc": "Small build system for use with gyp or CMake",
6 | "homepage": "https://ninja-build.org/",
7 | "oldname": null,
8 | "aliases": [],
9 | "versions": {
10 | "stable": "1.8.2",
11 | "bottle": true,
12 | "devel": null,
13 | "head": "HEAD"
14 | },
15 | "revision": 0,
16 | "version_scheme": 0,
17 | "installed": [],
18 | "linked_keg": null,
19 | "pinned": false,
20 | "outdated": false,
21 | "keg_only": false,
22 | "dependencies": [],
23 | "recommended_dependencies": [],
24 | "optional_dependencies": [],
25 | "build_dependencies": [],
26 | "conflicts_with": [],
27 | "caveats": null,
28 | "requirements": [],
29 | "options": [
30 | {
31 | "option": "--without-test",
32 | "description": "Don't run build-time tests"
33 | }
34 | ],
35 | "bottle": {
36 | "stable": {
37 | "rebuild": 0,
38 | "cellar": ":any_skip_relocation",
39 | "prefix": "/usr/local",
40 | "root_url": "https://homebrew.bintray.com/bottles",
41 | "files": {
42 | "high_sierra": {
43 | "url": "https://homebrew.bintray.com/bottles/ninja-1.8.2.high_sierra.bottle.tar.gz",
44 | "sha256": "eeba4fff08b3ed4b308250fb650f7d06630acd18465900ba0e27cecfe925a6cc"
45 | },
46 | "sierra": {
47 | "url": "https://homebrew.bintray.com/bottles/ninja-1.8.2.sierra.bottle.tar.gz",
48 | "sha256": "90ecf90948f0fa65c82011d79338d7c5ca2a4d0cb7cb8dc3892243f749fbe2eb"
49 | },
50 | "el_capitan": {
51 | "url": "https://homebrew.bintray.com/bottles/ninja-1.8.2.el_capitan.bottle.tar.gz",
52 | "sha256": "675165ce642fa811e1a0a363be0ba66a7b907d46056f89fd20938aa33e7d59f7"
53 | }
54 | }
55 | }
56 | }
57 | }
58 | ]
59 |
--------------------------------------------------------------------------------
/zorg/llvmlab/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | lab.llvm.org site specific customizations for the LLVM Lab web app.
3 | """
4 |
5 | import os
6 | import sys
7 |
8 | # Allow direct import of the master configuration files.
9 | g_master_dir = os.path.join(os.path.dirname(__file__),
10 | "..", "..", "buildbot", "llvmlab", "master")
11 | if g_master_dir not in sys.path:
12 | sys.path.append(g_master_dir)
13 | try:
14 | from config import phase_config
15 | except:
16 | # FIXME: Remove this once actual phase_config object is in place.
17 | class DummyConfig:
18 | phases = []
19 | phase_config = DummyConfig()
20 |
21 | import llvmlab.ci.config
22 | import llvmlab.ci.summary
23 |
24 | def construct_config():
25 | phases = []
26 | builders = []
27 | published_builds = []
28 |
29 | # Add hard coded builders.
30 | builders.append(llvmlab.ci.config.Builder("Validated Build"))
31 |
32 | for phase in phase_config.phases:
33 | # Add the phase object.
34 | phase_builder = "phase%d - %s" % (phase['number'], phase['name'])
35 | phases.append(llvmlab.ci.config.Phase(
36 | phase['title'], phase['number'], phase_builder,
37 | [b['name'] for b in phase['builders']],
38 | phase['description']))
39 |
40 | # Add the builder objects.
41 | builders.append(llvmlab.ci.config.Builder(phase_builder))
42 | for b in phase['builders']:
43 | builders.append(llvmlab.ci.config.Builder(b['name']))
44 |
45 | return llvmlab.ci.config.Config(phases, builders, published_builds,
46 | "Validated Build")
47 |
48 | def register(app):
49 | # Construct the LLVM Lab dashboard configuration object directly from the
50 | # buildbot phase_config module.
51 | config = construct_config()
52 | app.config.summary = llvmlab.ci.summary.Summary(
53 | config, app.config.status)
54 |
55 | print >>sys.stderr, "note: loaded lab.llvm.org extensions"
56 |
--------------------------------------------------------------------------------
/llvmlab/tests/basic.py:
--------------------------------------------------------------------------------
1 | import os
2 | import unittest
3 |
4 | import llvmlab
5 | from llvmlab.ui import app
6 |
7 | class TestCase(unittest.TestCase):
8 | def setUp(self):
9 | self.app = app.App.create_test_instance().test_client()
10 |
11 | def tearDown(self):
12 | pass
13 |
14 | def login(self, username, password):
15 | return self.app.post('/login', data=dict(
16 | username=username,
17 | password=password), follow_redirects=True)
18 |
19 | def logout(self):
20 | return self.app.get('/logout', follow_redirects=True)
21 |
22 | class TestBasic(TestCase):
23 | def test_index(self):
24 | rv = self.app.get('/')
25 | assert rv.data.startswith('login.""" in rv.data
35 |
36 | # Check that the login page shows what we expect.
37 | rv = self.app.get('/login')
38 | assert """admin """ in rv.data
43 |
44 | # Check that the login page shows something sensible.
45 | rv = self.app.get('/login')
46 | assert """You are already logged in.""" in rv.data
47 |
48 | # Check that we can access the users page now.
49 | rv = self.app.get('/users')
50 | assert """You must login .""" not in rv.data
51 | assert """Administrator """ in rv.data
52 |
53 | # Log out.
54 | rv = self.logout()
55 | assert "Logged in" not in rv.data
56 |
57 | if __name__ == '__main__':
58 | unittest.main()
59 |
--------------------------------------------------------------------------------
/llvmbisect/llvmlab/scripts.py:
--------------------------------------------------------------------------------
1 | """
2 | Utilities for building the llvmlab multi-tool.
3 | """
4 |
5 | import os
6 | import sys
7 |
8 |
9 | class Tool(object):
10 | """
11 | This object defines a generic command line tool instance, which dynamically
12 | builds its commands from a module dictionary.
13 |
14 | Example usage::
15 |
16 | import scripts
17 |
18 | def action_foo(name, args):
19 | "the foo command"
20 |
21 | ...
22 |
23 | tool = scripts.Tool(locals())
24 | if __name__ == '__main__':
25 | tool.main(sys.argv)
26 |
27 | Any function beginning with "action_" is considered a tool command. It's
28 | name is defined by the function name suffix. Underscores in the function
29 | name are converted to '-' in the command line syntax. Actions ending ith
30 | "-debug" are not listed in the help.
31 | """
32 |
33 | def __init__(self, locals):
34 | # Create the list of commands.
35 | self.commands = dict((name[7:].replace('_', '-'), f)
36 | for name, f in locals.items()
37 | if name.startswith('action_'))
38 |
39 | def usage(self, name):
40 | print >>sys.stderr, "Usage: %s command [options]" % (
41 | os.path.basename(name))
42 | print >>sys.stderr
43 | print >>sys.stderr, "Available commands:"
44 | cmds_width = max(map(len, self.commands))
45 | for name, func in sorted(self.commands.items()):
46 | if name.endswith("-debug"):
47 | continue
48 |
49 | print >>sys.stderr, " %-*s - %s" % (cmds_width, name,
50 | func.__doc__)
51 | sys.exit(1)
52 |
53 | def main(self, args):
54 | if len(args) < 2 or args[1] not in self.commands:
55 | if len(args) >= 2:
56 | print >>sys.stderr, "error: invalid command %r\n" % args[1]
57 | self.usage(args[0])
58 |
59 | cmd = args[1]
60 | return self.commands[cmd]('%s %s' % (args[0], cmd), args[2:])
61 |
--------------------------------------------------------------------------------
/tasks/tasktool/tasktool/repos/git.py:
--------------------------------------------------------------------------------
1 | import sys
2 | import tasktool.utils as utils
3 |
4 |
5 | def _git_ls_remote(url, ref):
6 | refs = utils.check_output(['git', 'ls-remote', url, ref])
7 | revs = {}
8 | for line in refs.split("\n"):
9 | line = line.strip()
10 | if line == "":
11 | continue
12 | rev, _, name = line.partition('\t')
13 | revs[name] = rev
14 |
15 | return revs
16 |
17 |
18 | def verify(config):
19 | if config.get('url') is None:
20 | raise Exception("No 'url' specified")
21 |
22 |
23 | def resolve_latest(config):
24 | assert(config['type'] == 'git')
25 | rev = config.get('rev')
26 | default_rev = config.pop('default_rev', None)
27 | if default_rev is None:
28 | default_rev = "refs/heads/master"
29 | if rev is not None:
30 | return
31 | url = config['url']
32 | revs = _git_ls_remote(url, default_rev)
33 | # Look for an exact match
34 | if default_rev is not None:
35 | rev = revs.get(default_rev, None)
36 | # Otherwise hope that we only have a single match.
37 | if rev is None:
38 | if len(revs) == 0:
39 | raise Exception("No refs matching '%s' found for "
40 | "repository '%s'\n" % (default_rev, url))
41 | if len(revs) > 1:
42 | raise Exception("Found multiple refs matching '%s' for "
43 | "repository '%s': %s\n" %
44 | (default_rev, url, ", ".join(revs.keys())))
45 | rev = revs.values()[0]
46 | config['rev'] = rev
47 |
48 |
49 | def get_artifact(config, dest_dir):
50 | url = config['url']
51 | rev = config['rev']
52 | # Note: -s is slightly dangerous for long running builds on the local
53 | # machine where the user changes the reference repo. But it's also a good
54 | # bit faster.
55 | utils.check_call(['git', 'clone', '-q', '-n', '-s', url, dest_dir])
56 | utils.check_call(['git', '--git-dir=%s/.git' % dest_dir,
57 | '--work-tree=%s' % dest_dir, 'checkout', '-q', rev])
58 |
59 |
60 | def repro_arg(config, dest_dir):
61 | return '-r %s=%s' % (dest_dir, config['rev'])
62 |
--------------------------------------------------------------------------------
/llvm-lnt/blacklist:
--------------------------------------------------------------------------------
1 | nts\..*O3.*.code_size
2 | nts\..*Os.*.code_size
3 | nts\..*O0.*.code_size
4 | nts\..*Oz.*\.compile_time
5 | nts\..*\.execution_time
6 | nts\..*.SingleSource/.*
7 | nts\..*.MultiSource/Applications/aha/.*
8 | nts\..*.MultiSource/Applications/ALAC/.*
9 | nts\..*.MultiSource/Applications/d/make_dparser.*
10 | nts\..*.MultiSource/Applications/hbd/hbd.*
11 | nts\..*.MultiSource/Applications/hexxagon/hexxagon.*
12 | nts\..*.MultiSource/Applications/JM/ldecod/ldecod.*
13 | nts\..*.MultiSource/Applications/lambda-0.1.3/lambda.*
14 | nts\..*.MultiSource/Applications/lua/lua.*
15 | nts\..*.MultiSource/Applications/minisat/minisat.*
16 | nts\..*.MultiSource/Applications/oggenc/oggenc.*
17 | nts\..*.MultiSource/Applications/sgefa/sgefa.*
18 | nts\..*.MultiSource/Applications/SIBsim4/SIBsim4.*
19 | nts\..*.MultiSource/Applications/siod/siod.*
20 | nts\..*.MultiSource/Applications/spiff/spiff.*
21 | nts\..*.MultiSource/Applications/viterbi/viterbi.*
22 | nts\..*.MultiSource/Benchmarks/ASCI_Purple/SMG2000/smg2000.*
23 | nts\..*.MultiSource/Benchmarks/ASC_Sequoia/.*
24 | nts\..*.MultiSource/Benchmarks/BitBench/.*
25 | nts\..*.MultiSource/Benchmarks/Fhourstones.*
26 | nts\..*.MultiSource/Benchmarks/FreeBench/.*
27 | nts\..*.MultiSource/Benchmarks/llubenchmark/.*
28 | nts\..*.MultiSource/Benchmarks/MallocBench/.*
29 | nts\..*.MultiSource/Benchmarks/McCat/.*
30 | nts\..*.MultiSource/Benchmarks/mediabench/.*
31 | nts\..*.MultiSource/Benchmarks/MiBench/automotive.*
32 | nts\..*.MultiSource/Benchmarks/MiBench/consumer-jpeg/consumer-jpeg.*
33 | nts\..*.MultiSource/Benchmarks/MiBench/consumer-lame/consumer-lame.*
34 | nts\..*.MultiSource/Benchmarks/MiBench/network-.*
35 | nts\..*.MultiSource/Benchmarks/MiBench/security-.*
36 | nts\..*.MultiSource/Benchmarks/MiBench/telecomm-.*
37 | nts\..*.MultiSource/Benchmarks/nbench/nbench.*
38 | nts\..*.MultiSource/Benchmarks/NPB-serial/is/is.*
39 | nts\..*.MultiSource/Benchmarks/Olden/.*
40 | nts\..*.MultiSource/Benchmarks/PAQ8p/paq8p.*
41 | nts\..*.MultiSource/Benchmarks/Prolangs-C.*
42 | nts\..*.MultiSource/Benchmarks/Ptrdist/.*
43 | nts\..*.MultiSource/Benchmarks/SciMark2-C/.*
44 | nts\..*.MultiSource/Benchmarks/sim/.*
45 | nts\..*.MultiSource/Benchmarks/Trimaran/.*
46 | nts\..*.MultiSource/Benchmarks/TSVC/.*
47 | nts\..*.MultiSource/Benchmarks/VersaBench/.*
48 |
--------------------------------------------------------------------------------
/llvmlab/llvmlab/ci/summary.py:
--------------------------------------------------------------------------------
1 | """
2 | Summary information for the dashboard.
3 |
4 | Connects the active CI configuration with the status information to track the
5 | current high-level status.
6 | """
7 |
8 | import sys
9 |
10 | class Summary(object):
11 | def __init__(self, config, status):
12 | self.config = config
13 | self.status = status
14 |
15 | def get_current_status(self):
16 | """
17 | get_current_status() -> ...
18 |
19 | Compute information on the current status.
20 | """
21 |
22 | # For each configured builder, compute the:
23 | # a. current build(s),
24 | # b. last passing build,
25 | # c. last failing build,
26 | # d. last completed build
27 | info = {}
28 | for builder in self.config.builders:
29 | builds = self.status.builders.get(builder.name)
30 |
31 | current = []
32 | passing = failing = completed = None
33 | if builds is None:
34 | # FIXME: Logging!
35 | print >>sys.stderr, "warning: no status for '%s'" % (
36 | builder.name)
37 | info[builder.name] = None
38 | else:
39 | for build in builds[::-1]:
40 | # Check if this is an active build.
41 | if build.start_time is not None and build.end_time is None:
42 | current.append(build)
43 | continue
44 |
45 | # Otherwise, check the status.
46 | if completed is None:
47 | completed = build
48 |
49 | # Track the (a) most recent passing build and (b) oldest
50 | # failure which happened after a passing build.
51 | if passing is None:
52 | if build.result == 0:
53 | passing = build
54 | else:
55 | failing = build
56 |
57 | info[builder.name] = {
58 | 'current' : current,
59 | 'passing' : passing,
60 | 'failing' : failing,
61 | 'completed' : completed }
62 |
63 | return info
64 |
--------------------------------------------------------------------------------
/zorg/buildbot/builders/SanitizerBuilder.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | import buildbot
4 | import buildbot.process.factory
5 | from buildbot.steps.source import SVN, Git
6 | from buildbot.process.properties import WithProperties
7 | from zorg.buildbot.commands.AnnotatedCommand import AnnotatedCommand
8 |
9 | def getSanitizerBuildFactory(
10 | clean=False,
11 | env=None,
12 | timeout=1200):
13 |
14 | # Prepare environmental variables. Set here all env we want everywhere.
15 | merged_env = {
16 | "TERM" : "dumb", # Make sure Clang doesn't use color escape sequences.
17 | }
18 | # Clobber bot if we need a clean build.
19 | if clean:
20 | merged_env["BUILDBOT_CLOBBER"] = "1"
21 | # Overwrite pre-set items with the given ones, so user can set anything.
22 | if env is not None:
23 | merged_env.update(env)
24 |
25 | f = buildbot.process.factory.BuildFactory()
26 |
27 | # Determine the build directory.
28 | f.addStep(buildbot.steps.shell.SetProperty(name="get_builddir",
29 | command=["pwd"],
30 | property="builddir",
31 | description="set build dir",
32 | workdir=".",
33 | env=merged_env))
34 |
35 | sanitizer_buildbot = "sanitizer_buildbot"
36 | sanitizer_script_dir = os.path.join(sanitizer_buildbot, "sanitizers")
37 | # Get sanitizer buildbot scripts.
38 | f.addStep(SVN(name='svn-sanitizer-buildbot',
39 | mode='update',
40 | svnurl='http://llvm.org/svn/llvm-project/zorg/trunk/'
41 | 'zorg/buildbot/builders/sanitizers',
42 | workdir=sanitizer_script_dir,
43 | alwaysUseLatest=True))
44 |
45 | sanitizer_script = os.path.join("..", sanitizer_script_dir, "buildbot_selector.py")
46 |
47 | # Run annotated command for sanitizer.
48 | f.addStep(AnnotatedCommand(name="annotate",
49 | description="annotate",
50 | timeout=timeout,
51 | haltOnFailure=True,
52 | command="python " + sanitizer_script,
53 | env=merged_env))
54 | return f
55 |
--------------------------------------------------------------------------------
/zorg/buildbot/builders/sanitizers/buildbot_fuzzer.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | set -x
4 | set -e
5 | set -u
6 |
7 | HERE="$(cd $(dirname $0) && pwd)"
8 | . ${HERE}/buildbot_functions.sh
9 |
10 | ROOT=`pwd`
11 | PLATFORM=`uname`
12 | export PATH="/usr/local/bin:$PATH"
13 |
14 | USE_GIT=1
15 |
16 | CHECK_LIBCXX=${CHECK_LIBCXX:-0}
17 | CHECK_LLD=${CHECK_LLD:-1}
18 | MAKE_JOBS=${MAX_MAKE_JOBS:-$(nproc)}
19 | LLVM=$ROOT/llvm
20 | LIBFUZZER=$LLVM/lib/Fuzzer
21 | # No assertions. Need to clean up the existing assertion failures first.
22 | # Also, the Fuzzer does not provide reproducers on assertion failures yet.
23 | CMAKE_COMMON_OPTIONS="-GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=OFF -DLLVM_PARALLEL_LINK_JOBS=8 -DLIBFUZZER_ENABLE_TESTS=ON"
24 |
25 | CLOBBER=fuzzer-test-suite
26 | STAGE1_CLOBBER="RUNDIR-* $LIBFUZZER"
27 | clobber
28 |
29 | # Make sure asan intercepts SIGABRT so that the fuzzer can print the test cases
30 | # for assertion failures.
31 | # export ASAN_OPTIONS=handle_abort=1:strip_path_prefix=build/llvm/
32 |
33 | buildbot_update
34 |
35 | # Stage 1
36 |
37 | build_stage1_clang
38 |
39 | # export ASAN_SYMBOLIZER_PATH="${llvm_symbolizer_path}"
40 | export PATH="$(readlink -f ${STAGE1_DIR}/bin):$PATH"
41 |
42 | echo @@@BUILD_STEP check-fuzzer@@@
43 |
44 | (cd ${STAGE1_DIR} && ninja check-fuzzer) || echo @@@STEP_FAILURE@@@
45 |
46 | echo @@@BUILD_STEP get fuzzer-test-suite @@@
47 | [ ! -e fuzzer-test-suite ] && git clone https://github.com/google/fuzzer-test-suite.git
48 | (cd fuzzer-test-suite && git pull)
49 |
50 | RunFuzzerTest() {
51 | echo @@@BUILD_STEP test "$1" fuzzer@@@
52 | ln -sf $LIBFUZZER .
53 | export FUZZING_ENGINE=fsanitize_fuzzer
54 | `pwd`/fuzzer-test-suite/build-and-test.sh "$1"
55 | }
56 |
57 | ulimit -t 3600
58 |
59 | RunFuzzerTest re2-2014-12-09 || echo @@@STEP_FAILURE@@@
60 | RunFuzzerTest c-ares-CVE-2016-5180 || echo @@@STEP_FAILURE@@@
61 | RunFuzzerTest openssl-1.0.1f || echo @@@STEP_FAILURE@@@
62 | RunFuzzerTest openssl-1.0.2d || echo @@@STEP_FAILURE@@@
63 | RunFuzzerTest libxml2-v2.9.2 || echo @@@STEP_FAILURE@@@
64 | RunFuzzerTest libpng-1.2.56 || echo @@@STEP_FAILURE@@@
65 | RunFuzzerTest libssh-2017-1272 || echo @@@STEP_FAILURE@@@
66 | RunFuzzerTest proj4-2017-08-14 || echo @@@STEP_FAILURE@@@
67 | #RunFuzzerTest woff2-2016-05-06 || echo @@@STEP_WARNINGS@@@ # Often can't find the bug in the given time.
68 |
--------------------------------------------------------------------------------
/codesign/debugsign/unittests/test_ansi.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python -tt
2 |
3 | """
4 | unittests.test_ansi
5 | """
6 |
7 | try:
8 | import unittest2 as unittest
9 | except ImportError:
10 | import unittest
11 |
12 | import mock
13 | import re
14 | import StringIO
15 | import sys
16 |
17 | import dbsign.ansi as A
18 |
19 |
20 | class TestAnsi(unittest.TestCase):
21 | def setUp(self):
22 | self.test_text = "The quick brown fox, etc."
23 | self.codes = A._ANSI_CODES
24 | re_ansi = r"\033\[.*?m"
25 | ansi_msg_pattern = r'(?P{0})(?P.*?)(?P{0})'
26 | self.re_ansi_msg = re.compile(ansi_msg_pattern.format(re_ansi))
27 |
28 | @unittest.skipUnless(sys.stdout.isatty(), A.WARN("requires tty"))
29 | def test_ansi_tty(self):
30 | msg = self.test_text
31 |
32 | for color in self.codes:
33 | with self.subTest(color):
34 | ansi_msg = A.ANSI(color, msg)
35 | self.assertTrue(ansi_msg.startswith(self.codes[color]))
36 | self.assertTrue(ansi_msg.endswith(self.codes['clear']))
37 |
38 | @unittest.skipUnless(sys.stdout.isatty(), A.WARN("requires tty"))
39 | def test_ansi_convenience_tty(self):
40 | msg = self.test_text
41 | funcs = [A.OK, A.INFO, A.WARN, A.ERROR]
42 |
43 | for func in funcs:
44 | with self.subTest(func=func.func_name):
45 | ansi_msg = func(msg)
46 | m = self.re_ansi_msg.match(ansi_msg)
47 | self.assertTrue(m)
48 | self.assertIn(m.group('begin'), self.codes.values())
49 | self.assertEqual(m.group('middle'), msg)
50 | self.assertEqual(m.group('end'), self.codes['clear'])
51 |
52 | @mock.patch('sys.stdout', new_callable=StringIO.StringIO)
53 | def test_ansi_notty(self, mock_stdout):
54 | msg = self.test_text
55 |
56 | for color in self.codes:
57 | with self.subTest(color):
58 | ansi_msg = A.ANSI(color, msg)
59 | self.assertEqual(msg, ansi_msg)
60 |
61 | @mock.patch('sys.stdout', new_callable=StringIO.StringIO)
62 | def test_ansi_convenience_notty(self, mock_stdout):
63 | msg = self.test_text
64 | funcs = [A.OK, A.INFO, A.WARN, A.ERROR]
65 |
66 | for func in funcs:
67 | with self.subTest(func=func.func_name):
68 | self.assertEqual(msg, func(msg))
69 |
--------------------------------------------------------------------------------
/zorg/buildbot/builders/sanitizers/buildbot_selector.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 |
3 | import os
4 | import subprocess
5 | import sys
6 |
7 | THIS_DIR=os.path.dirname(sys.argv[0])
8 |
9 |
10 | def bash(path):
11 | return 'bash ' + os.path.join(THIS_DIR, path)
12 |
13 | def cmd_call(path):
14 | return 'call ' + os.path.join(THIS_DIR, path)
15 |
16 | BOT_ASSIGNMENT = {
17 | 'sanitizer-ppc64le-linux': bash('buildbot_cmake.sh'),
18 | 'sanitizer-ppc64be-linux': bash('buildbot_cmake.sh'),
19 | 'sanitizer-x86_64-linux': bash('buildbot_cmake.sh'),
20 | 'sanitizer-x86_64-linux-fast': bash('buildbot_fast.sh'),
21 | 'sanitizer-x86_64-linux-autoconf': bash('buildbot_standard.sh'),
22 | 'sanitizer-x86_64-linux-fuzzer': bash('buildbot_fuzzer.sh'),
23 | 'sanitizer-x86_64-linux-android': bash('buildbot_android.sh'),
24 | 'sanitizer-x86_64-linux-bootstrap': bash('buildbot_bootstrap_asan.sh'),
25 | 'sanitizer-x86_64-linux-bootstrap-msan': bash('buildbot_bootstrap_msan.sh'),
26 | 'sanitizer-x86_64-linux-bootstrap-ubsan': bash('buildbot_bootstrap_ubsan.sh'),
27 | 'sanitizer-x86_64-linux-gn': 'true',
28 | }
29 |
30 | BOT_ADDITIONAL_ENV = {
31 | 'sanitizer-ppc64le-linux': { 'HAVE_NINJA': '1', 'CHECK_LIBCXX': '0', 'CHECK_LLD': '0' },
32 | 'sanitizer-ppc64be-linux': { 'HAVE_NINJA': '1', 'CHECK_LIBCXX': '0', 'CHECK_LLD': '0' },
33 | 'sanitizer-x86_64-linux': { 'HAVE_NINJA' : '1' },
34 | 'sanitizer-x86_64-linux-bootstrap': {},
35 | 'sanitizer-x86_64-linux-fast': {},
36 | 'sanitizer-x86_64-linux-autoconf': {},
37 | 'sanitizer-x86_64-linux-fuzzer': {},
38 | 'sanitizer-x86_64-linux-android': {},
39 | 'sanitizer-x86_64-linux-bootstrap-msan': {},
40 | 'sanitizer-x86_64-linux-bootstrap-ubsan': {},
41 | 'sanitizer-x86_64-linux-gn': {},
42 | }
43 |
44 | def Main():
45 | builder = os.environ.get('BUILDBOT_BUILDERNAME')
46 | print "builder name: %s" % (builder)
47 | cmd = BOT_ASSIGNMENT.get(builder)
48 | if not cmd:
49 | sys.stderr.write('ERROR - unset/invalid builder name\n')
50 | sys.exit(1)
51 |
52 | print "%s runs: %s\n" % (builder, cmd)
53 | sys.stdout.flush()
54 |
55 | bot_env = os.environ
56 | add_env = BOT_ADDITIONAL_ENV.get(builder)
57 | for var in add_env:
58 | bot_env[var] = add_env[var]
59 | if 'TMPDIR' in bot_env:
60 | del bot_env['TMPDIR']
61 |
62 | retcode = subprocess.call(cmd, env=bot_env, shell=True)
63 | sys.exit(retcode)
64 |
65 |
66 | if __name__ == '__main__':
67 | Main()
68 |
--------------------------------------------------------------------------------
/zorg/jenkins/relay.groovy:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env groovy
2 | @NonCPS
3 | private def get_matching_jobs(pattern) {
4 | def jobs = []
5 | for (job in Jenkins.getInstance().getAllItems(Job)) {
6 | def jobname = job.getName()
7 | def m = jobname =~ pattern
8 | if (m) {
9 | def shortname = m[0][1]
10 | jobs.push([shortname, jobname])
11 | }
12 | }
13 | return jobs
14 | }
15 |
16 | private def basename(path) {
17 | return path.drop(path.lastIndexOf('/') + 1)
18 | }
19 |
20 | private def relay_steps(job_pattern, artifact_url, last_good_properties_url) {
21 | // The upstream jobs triggering the relay produce a
22 | // "last_good_build.properties" file that contains a reference to the
23 | // compiler artifact that should be used for this run and which llvm
24 | // revision it is based on.
25 | propfile = basename(last_good_properties_url)
26 | sh """
27 | rm -f ${propfile}
28 | curl -fksSO "${last_good_properties_url}"
29 | """
30 | def props = readProperties file: propfile
31 | def artifact = "http://labmaster2.local/artifacts/${props.ARTIFACT}"
32 | currentBuild.setDisplayName("${props.GIT_DISTANCE}-${props.GIT_SHA}")
33 |
34 | // Trigger all jobs with names matching the `job_pattern` regex.
35 | def joblist = get_matching_jobs(job_pattern)
36 | def parallel_builds = [:]
37 | for (j in joblist) {
38 | def shortname = j[0]
39 | def jobname = j[1]
40 | parallel_builds[shortname] = {
41 | def job_params = [
42 | [$class: 'StringParameterValue',
43 | name: 'ARTIFACT',
44 | value: artifact],
45 | [$class: 'StringParameterValue',
46 | name: 'GIT_SHA',
47 | value: props.GIT_SHA],
48 | [$class: 'StringParameterValue',
49 | name: 'GIT_DISTANCE',
50 | value: props.GIT_DISTANCE],
51 | ]
52 | build job: jobname, parameters: job_params
53 | }
54 | }
55 | parallel parallel_builds
56 | }
57 |
58 | def pipeline(job_pattern,
59 | artifact_url='http://labmaster2.local/artifacts/',
60 | last_good_properties_url='http://labmaster2.local/artifacts/clang-stage1-RA/last_good_build.properties') {
61 | node('master') {
62 | stage('main') {
63 | relay_steps job_pattern, artifact_url, last_good_properties_url
64 | }
65 | }
66 | }
67 | return this
68 |
--------------------------------------------------------------------------------
/codesign/debugsign/dbsign/shell.py:
--------------------------------------------------------------------------------
1 | # COPYRIGHT LINE: FIXME
2 |
3 | """
4 | dbsign.shell
5 |
6 | shell routines for debugsign
7 | """
8 |
9 | from __future__ import print_function
10 |
11 | import os
12 | from subprocess import PIPE, Popen
13 |
14 | import dbsign.logger
15 |
16 |
17 | log = dbsign.logger.get_logger(__name__)
18 |
19 |
20 | class ShellCommand(object):
21 | """
22 | Represents the result of a shell command
23 | """
24 | def __init__(self, args, code, stdout, stderr):
25 | # type: (list[str], int, str, str) -> ()
26 | self.data = {
27 | 'args': args,
28 | 'code': code,
29 | 'stdout': stdout,
30 | 'stderr': stderr,
31 | }
32 |
33 | def __eq__(self, rhs): # type: (ShellCommand) -> bool
34 | return self.data == rhs.data
35 |
36 | def __getattr__(self, attr): # type: (str) -> T
37 | if attr in self.data:
38 | return self.data[attr]
39 | raise AttributeError(attr)
40 |
41 | def __nonzero__(self): # type: () -> bool
42 | return self.code == 0
43 |
44 | def __repr__(self): # type: () -> str
45 | repr_fmt = "{0}(args={1.args!r}, code={1.code!r},"
46 | repr_fmt += " stdout={1.stdout!r}, stderr={1.stderr!r})"
47 | return repr_fmt.format(self.__class__.__name__, self)
48 |
49 |
50 | def __run(args, stdin=None): # type: (list[str], str) -> ShellCommand
51 | """internal function to run shell commands"""
52 | try:
53 | p = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE)
54 | stdout, stderr = p.communicate(input=stdin)
55 | except OSError as os_err:
56 | log.debug('Unable to execute command: %s: %s', args, os_err)
57 | raise
58 |
59 | cmd = ShellCommand(args, code=p.returncode, stdout=stdout, stderr=stderr)
60 | log.debug(cmd)
61 | return cmd
62 |
63 |
64 | def run(args, stdin=None): # type: (list[str]) -> ShellCommand
65 | """Run a regular (non-sudo) command"""
66 | log.debug("run(args=%s)", repr(args))
67 |
68 | if os.path.basename(args[0]).startswith('su'):
69 | log.info('run() called with illegal command `%s`', args)
70 | raise RuntimeError('Unauthorized use of run; use sudo_run')
71 |
72 | return __run(args, stdin)
73 |
74 |
75 | def sudo_run(args, stdin=None): # type: (list[str]) -> ShellCommand
76 | """Run a command with root privileges using sudo"""
77 | log.debug("sudo_run(args=%s)", repr(args))
78 |
79 | args.insert(0, 'sudo')
80 |
81 | return __run(args, stdin)
82 |
--------------------------------------------------------------------------------
/zorg/buildbot/builders/annotated/sanitizer-windows.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 |
3 | import os
4 | import sys
5 | import annotated_builder
6 | import util
7 |
8 |
9 | class SanitizerAnnotatedBuilder(annotated_builder.AnnotatedBuilder):
10 |
11 | """Customizes the 'build' step of the generic AnnotatedBuilder"""
12 |
13 | def build(self, stage_name, build_dir, jobs=None):
14 | # The basic idea here is to run 'ninja compiler-rt ; ninja clang lld'.
15 | # This ensures that portability issues in compiler-rt code are found
16 | # first. Then, we only build clang and lld, the primary dependencies of
17 | # the sanitizer test suites, to keep cycle time low. This means there
18 | # are still some remaining test dependencies (FileCheck) that may be
19 | # compiled during the check step, but there shouldn't be that many.
20 | self.report_build_step('%s build' % (stage_name,))
21 | self.halt_on_failure()
22 | base_cmd = ['ninja']
23 | if jobs:
24 | base_cmd += ['-j', str(jobs)]
25 | early_targets = ['compiler-rt']
26 | late_targets = ['clang', 'lld']
27 | util.report_run_cmd(base_cmd + early_targets, cwd=build_dir)
28 | util.report_run_cmd(base_cmd + late_targets, cwd=build_dir)
29 |
30 |
31 | def main(argv):
32 | ap = annotated_builder.get_argument_parser()
33 | args = ap.parse_args(argv[1:])
34 |
35 | projects = ['llvm', 'clang', 'lld', 'compiler-rt']
36 | stages = 1
37 | extra_cmake_args = [
38 | '-DCMAKE_BUILD_TYPE=Release',
39 | '-DLLVM_ENABLE_PDB=ON',
40 | '-DLLVM_ENABLE_ASSERTIONS=ON',
41 | '-DLLVM_TARGETS_TO_BUILD=X86',
42 | ]
43 | check_targets = ['check-asan', 'check-asan-dynamic', 'check-sanitizer',
44 | 'check-ubsan', 'check-fuzzer', 'check-cfi',
45 | 'check-profile', 'check-builtins']
46 |
47 | # These arguments are a bit misleading, they really mean use cl.exe for
48 | # stage1 instead of GCC.
49 | compiler = 'clang-cl'
50 | linker = 'lld-link'
51 |
52 | builder = SanitizerAnnotatedBuilder()
53 | builder.run_steps(stages=stages,
54 | projects=projects,
55 | extra_cmake_args=extra_cmake_args,
56 | check_targets=check_targets,
57 | compiler=compiler,
58 | linker=linker,
59 | jobs=args.jobs)
60 |
61 |
62 | if __name__ == '__main__':
63 | sys.path.append(os.path.dirname(__file__))
64 | sys.exit(main(sys.argv))
65 |
--------------------------------------------------------------------------------
/zorg/buildbot/process/properties.py:
--------------------------------------------------------------------------------
1 | # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
2 | # See https://llvm.org/LICENSE.txt for license information.
3 | # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
4 |
5 | from buildbot.interfaces import IRenderable
6 | from buildbot.process.properties import WithProperties
7 | from zope.interface import implements
8 |
9 |
10 | class InterpolateToNativePath(WithProperties):
11 | """
12 | This is a marker class, used to indicate that we
13 | want to interpolate build properties as a paths with
14 | the correct buildslave path separator.
15 | """
16 |
17 | implements(IRenderable)
18 | compare_attrs = ('fmtstring', 'args')
19 |
20 | def __init__(self, fmtstring, *args, **lambda_subs):
21 | WithProperties.__init__(self, fmtstring, *args, **lambda_subs)
22 |
23 | def getRenderingFor(self, build):
24 | # Upcall the base class first.
25 | p = WithProperties.getRenderingFor(self, build)
26 |
27 | # Then we need to normalize the path for
28 | # watever is native on the buildslave.
29 | # Note: Do not call normpath here, as it could
30 | # change the path meaning if links used.
31 | slave = build.build.slavebuilder.slave
32 |
33 | return slave.path_module.normcase(p)
34 |
35 |
36 | class InterpolateToPosixPath(WithProperties):
37 | """
38 | This is a marker class, used to indicate that we
39 | want to interpolate build properties as a paths with
40 | POSIX path separator.
41 | """
42 |
43 | implements(IRenderable)
44 | compare_attrs = ('fmtstring', 'args')
45 |
46 | def __init__(self, fmtstring, *args, **lambda_subs):
47 | WithProperties.__init__(self, fmtstring, *args, **lambda_subs)
48 |
49 | def getRenderingFor(self, build):
50 | # Upcall the base class first.
51 | p = WithProperties.getRenderingFor(self, build)
52 |
53 | # Then we need to figure out the buildslave OS:
54 | slave = build.build.slavebuilder.slave
55 | if slave.slave_system == 'posix':
56 | # Note: Do not call normpath here, as it could
57 | # change the path meaning if links used.
58 | p = slave.path_module.normcase(p)
59 | elif slave.slave_system in ('win32', 'nt'):
60 | # Normalize the path first, then replace
61 | # the path separator to the POSIX one.
62 | p = slave.path_module.normcase(p).replace('\\','/')
63 | else:
64 | # Return the string as is.
65 | pass
66 |
67 | return p
68 |
--------------------------------------------------------------------------------
/zorg/buildbot/builders/sanitizers/buildbot_standard.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | set -x
4 | set -e
5 | set -u
6 |
7 | # dump buildbot env
8 | env
9 |
10 | HERE="$(dirname $0)"
11 | . ${HERE}/buildbot_functions.sh
12 |
13 | USE_GIT=1
14 |
15 | TSAN_DEBUG_BUILD_DIR=tsan_debug_build
16 | TSAN_FULL_DEBUG_BUILD_DIR=tsan_full_debug_build
17 | TSAN_RELEASE_BUILD_DIR=tsan_release_build
18 |
19 | CLOBBER="$TSAN_DEBUG_BUILD_DIR $TSAN_FULL_DEBUG_BUILD_DIR $TSAN_RELEASE_BUILD_DIR"
20 | clobber
21 |
22 | ROOT=`pwd`
23 | PLATFORM=`uname`
24 | MAKE_JOBS=${MAX_MAKE_JOBS:-$(nproc)}
25 |
26 | CHECK_LIBCXX=${CHECK_LIBCXX:-1}
27 | CHECK_LLD=${CHECK_LLD:-1}
28 |
29 | LLVM=${ROOT}/llvm
30 | CMAKE_COMMON_OPTIONS="-DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_BUILD_EXTERNAL_COMPILER_RT=ON"
31 |
32 | function build_tsan {
33 | local build_dir=$1
34 | local extra_cmake_args=$2
35 | local targets="clang llvm-symbolizer llvm-config FileCheck not"
36 | if [ ! -d $build_dir ]; then
37 | mkdir $build_dir
38 | fi
39 | if [[ "$USE_GIT" != "0" ]]; then
40 | extra_cmake_args="${extra_cmake_args} -DLLVM_ENABLE_PROJECTS='clang;compiler-rt'"
41 | fi
42 | (cd $build_dir && CC="$3" CXX="$4" cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \
43 | ${CMAKE_COMMON_OPTIONS} ${extra_cmake_args} \
44 | ${LLVM})
45 | (cd $build_dir && make -j$MAKE_JOBS ${targets}) || echo @@@STEP_FAILURE@@@
46 | (cd $build_dir && make compiler-rt-clear) || echo @@@STEP_FAILURE@@@
47 | (cd $build_dir && make -j$MAKE_JOBS tsan) || echo @@@STEP_FAILURE@@@
48 | }
49 |
50 | buildbot_update
51 |
52 | echo @@@BUILD_STEP build fresh clang + debug compiler-rt@@@
53 | build_tsan "${TSAN_DEBUG_BUILD_DIR}" "-DCOMPILER_RT_DEBUG=ON" gcc g++
54 |
55 | echo @@@BUILD_STEP test tsan in debug compiler-rt build@@@
56 | (cd $TSAN_DEBUG_BUILD_DIR && make -j$MAKE_JOBS check-tsan) || echo @@@STEP_FAILURE@@@
57 |
58 | echo @@@BUILD_STEP build tsan with stats and debug output@@@
59 | build_tsan "${TSAN_FULL_DEBUG_BUILD_DIR}" "-DCOMPILER_RT_DEBUG=ON -DCOMPILER_RT_TSAN_DEBUG_OUTPUT=ON -DLLVM_INCLUDE_TESTS=OFF" gcc g++
60 |
61 | echo @@@BUILD_STEP build release tsan with clang@@@
62 | build_tsan "${TSAN_RELEASE_BUILD_DIR}" "-DCOMPILER_RT_DEBUG=OFF" "$ROOT/$TSAN_DEBUG_BUILD_DIR/bin/clang" "$ROOT/$TSAN_DEBUG_BUILD_DIR/bin/clang++"
63 |
64 | echo @@@BUILD_STEP tsan analyze@@@
65 | BIN=$(mktemp -t tsan_exe.XXXXXXXX)
66 | echo "int main() {return 0;}" | $TSAN_RELEASE_BUILD_DIR/bin/clang -x c++ - -fsanitize=thread -O2 -o ${BIN}
67 | COMPILER_RT=$LLVM/projects/compiler-rt
68 | if [[ "$USE_GIT" != "0" ]]; then
69 | COMPILER_RT=$LLVM/../compiler-rt
70 | fi
71 | $COMPILER_RT/lib/tsan/check_analyze.sh ${BIN} || echo @@@STEP_FAILURE@@@
72 |
--------------------------------------------------------------------------------
/zorg/buildbot/conditions/FileConditions.py:
--------------------------------------------------------------------------------
1 | from buildbot.process.buildstep import LoggedRemoteCommand
2 | from buildbot.interfaces import BuildSlaveTooOldError
3 | import stat
4 |
5 | from twisted.python import log
6 |
7 | class FileExists(object):
8 | """I check a file existence on the buildslave. I return True if the file
9 | with the given name exists, False if the file does not exist or that is
10 | a directory.
11 |
12 | Use me with doStepIf to make a build step conditional to existence of some
13 | file. For example
14 |
15 | doStepIf=FileExists('build/configure')
16 | """
17 |
18 | def __init__(self, filename):
19 | self.filename = filename
20 |
21 | def __call__(self, step):
22 | slavever = step.slaveVersion('stat')
23 | if not slavever:
24 | raise BuildSlaveTooOldError("slave is too old, does not know "
25 | "about stat")
26 |
27 | def commandComplete(cmd):
28 | if cmd.rc != 0:
29 | return False
30 |
31 | s = cmd.updates["stat"][-1]
32 | filemode = s[stat.ST_MODE]
33 | if stat.S_ISREG(filemode) or stat.S_ISLNK(filemode):
34 | # True only if this is a file or a link and not any other file
35 | # system object.
36 | return True
37 | else:
38 | return False
39 |
40 | cmd = LoggedRemoteCommand('stat', {'file': self.filename})
41 | d = step.runCommand(cmd)
42 | d.addCallback(lambda res: commandComplete(cmd))
43 | return d
44 |
45 | class FileDoesNotExist(object):
46 | """I check a file existence on the buildslave. I return False if
47 | the file with the given name exists or that is a directory, True if the
48 | file does not exist.
49 |
50 | Use me with doStepIf to make a build step conditional to nonexistence
51 | of some file. For example
52 |
53 | doStepIf=FileDoesNotExist('build/configure')
54 | """
55 |
56 | def __init__(self, filename):
57 | self.filename = filename
58 |
59 | def __call__(self, step):
60 | slavever = step.slaveVersion('stat')
61 | if not slavever:
62 | raise BuildSlaveTooOldError("slave is too old, does not know "
63 | "about stat")
64 |
65 | def commandComplete(cmd):
66 | # False if any filesystem object with the given name exists.
67 | return (cmd.rc != 0)
68 |
69 | cmd = LoggedRemoteCommand('stat', {'file': self.filename})
70 | d = step.runCommand(cmd)
71 | d.addCallback(lambda res: commandComplete(cmd))
72 | return d
--------------------------------------------------------------------------------
/codesign/debugsign/unittests/test_result.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python -tt
2 |
3 | """
4 | unittests.test_result
5 | """
6 |
7 | try:
8 | import unittest2 as unittest
9 | except ImportError:
10 | import unittest
11 |
12 | import mock
13 | import StringIO
14 |
15 | import dbsign.result as R
16 |
17 |
18 | class TestResult(unittest.TestCase):
19 | def setUp(self):
20 | self.msg = "The quick brown fox."
21 |
22 | def test_result_failure(self):
23 | value = self.msg
24 | failure = R.Failure(value)
25 |
26 | self.assertFalse(failure.checked)
27 | self.assertEqual(value, failure.value)
28 | self.assertFalse(failure)
29 | self.assertTrue(failure.checked)
30 | self.assertRegexpMatches(repr(failure), value)
31 |
32 | def test_result_success(self):
33 | value = self.msg
34 | success = R.Success(value)
35 |
36 | self.assertFalse(success.checked)
37 | self.assertEqual(success.value, value)
38 | self.assertTrue(success)
39 | self.assertTrue(success.checked)
40 | self.assertRegexpMatches(repr(success), value)
41 |
42 | def test_abstract(self):
43 | text = self.msg
44 | error_text = "does not support boolean evaluation"
45 | res = R.Result(text)
46 |
47 | self.assertEqual(text, res.value)
48 | with self.assertRaisesRegexp(NotImplementedError, error_text):
49 | bool(res)
50 |
51 | def test_renew(self):
52 | msg = self.msg
53 |
54 | def fn(res_type):
55 | # create instance
56 | res = res_type(msg)
57 | self.assertFalse(res.checked)
58 | # invoke __nonzero__()
59 | bool(res)
60 | self.assertTrue(res.checked)
61 | # return "renewed" object
62 | return res.renew()
63 |
64 | for res_type in (R.Success, R.Failure):
65 | with self.subTest(resultType=res_type.__name__):
66 | res = fn(res_type)
67 | self.assertEqual(res_type, res.__class__)
68 | self.assertFalse(res.checked)
69 | bool(res)
70 | self.assertTrue(res.checked)
71 |
72 | @mock.patch('sys.stderr', new_callable=StringIO.StringIO)
73 | def test_unchecked_asserts(self, mock_stderr):
74 | for res_type in (R.Result, R.Success, R.Failure):
75 | with self.subTest(resultType=res_type.__name__):
76 | with self.assertRaises(AssertionError):
77 | # generate an instance of the passed class
78 | res = res_type(res_type.__name__)
79 | # call destructor to simulate program termination
80 | res.__del__()
81 |
--------------------------------------------------------------------------------
/zorg/buildbot/util/ConfigEmailLookup.py:
--------------------------------------------------------------------------------
1 | import buildbot
2 | import zope
3 | import os
4 |
5 | from datetime import datetime, timedelta
6 | from twisted.python import log
7 |
8 | class ConfigEmailLookup(buildbot.util.ComparableMixin):
9 | """
10 | Email lookup implementation which searchs a user specified configuration
11 | file to match commit authors to email addresses.
12 | """
13 |
14 | # TODO: Document this class.
15 | # Class loads llvm_authors from file and reload if the file was updated.
16 |
17 | zope.interface.implements(buildbot.interfaces.IEmailLookup)
18 | compare_attrs = ["author_filename", "default_address", "only_addresses"]
19 |
20 | def __init__(self, author_filename, default_address, only_addresses = None, update_interval=timedelta(hours=1)):
21 | from ConfigParser import ConfigParser
22 |
23 | self.author_filename = author_filename
24 | self.default_address = default_address
25 | self.only_addresses = only_addresses
26 | self.update_interval = update_interval
27 |
28 | self.config_parser = ConfigParser()
29 | self.config_parser.read(self.author_filename)
30 |
31 | self.time_checked = datetime.utcnow()
32 | self.time_loaded = datetime.utcfromtimestamp(os.path.getmtime(self.author_filename))
33 |
34 | if only_addresses:
35 | import re
36 | self.address_match_p = re.compile(only_addresses).match
37 | else:
38 | self.address_match_p = lambda addr: True
39 |
40 | def getAddress(self, name):
41 |
42 | # For a multiple types of pollers/schedulers we can get an email address or
43 | # a fully qualified name/email in 'name' parameter. Check and return name if it
44 | # contains email address.
45 | if name is not None and '@' in name:
46 | return name
47 |
48 | try:
49 |
50 | if (datetime.utcnow() - self.time_checked) >= timedelta(minutes=1):
51 | self.time_checked = datetime.utcnow()
52 | current_mtime = datetime.utcfromtimestamp(os.path.getmtime(self.author_filename))
53 |
54 | if (current_mtime != self.time_loaded) and ((datetime.utcnow() - current_mtime) >= timedelta(minutes=1)):
55 | # Reload the list of authors.
56 | self.config_parser.read(self.author_filename)
57 | self.time_loaded = current_mtime
58 | log.msg('Reloaded file %s (mtime=%s) at %s' % (self.author_filename, self.time_loaded, self.time_checked))
59 |
60 | except:
61 | log.msg('Cannot load the %s file.' % self.author_filename)
62 | pass
63 |
64 | try:
65 | email = self.config_parser.get("authors", name)
66 | except:
67 | return self.default_address
68 |
69 | if self.address_match_p(email):
70 | return email
71 |
72 | return self.default_address
73 |
--------------------------------------------------------------------------------
/dep/tests/assets/brew_cmake_installed.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "name": "cmake",
4 | "full_name": "cmake",
5 | "desc": "Cross-platform make",
6 | "homepage": "https://www.cmake.org/",
7 | "oldname": null,
8 | "aliases": [],
9 | "versions": {
10 | "stable": "3.10.2",
11 | "bottle": true,
12 | "devel": null,
13 | "head": "HEAD"
14 | },
15 | "revision": 0,
16 | "version_scheme": 0,
17 | "installed": [
18 | {
19 | "version": "3.10.0",
20 | "used_options": [],
21 | "built_as_bottle": true,
22 | "poured_from_bottle": true,
23 | "runtime_dependencies": [],
24 | "installed_as_dependency": false,
25 | "installed_on_request": true
26 | },
27 | {
28 | "version": "3.10.2",
29 | "used_options": [],
30 | "built_as_bottle": true,
31 | "poured_from_bottle": true,
32 | "runtime_dependencies": [],
33 | "installed_as_dependency": false,
34 | "installed_on_request": true
35 | }
36 | ],
37 | "linked_keg": "3.10.0",
38 | "pinned": false,
39 | "outdated": false,
40 | "keg_only": false,
41 | "dependencies": [
42 | "sphinx-doc"
43 | ],
44 | "recommended_dependencies": [],
45 | "optional_dependencies": [],
46 | "build_dependencies": [
47 | "sphinx-doc"
48 | ],
49 | "conflicts_with": [],
50 | "caveats": null,
51 | "requirements": [],
52 | "options": [
53 | {
54 | "option": "--without-docs",
55 | "description": "Don't build man pages"
56 | },
57 | {
58 | "option": "--with-completion",
59 | "description": "Install Bash completion (Has potential problems with system bash)"
60 | }
61 | ],
62 | "bottle": {
63 | "stable": {
64 | "rebuild": 0,
65 | "cellar": ":any_skip_relocation",
66 | "prefix": "/usr/local",
67 | "root_url": "https://homebrew.bintray.com/bottles",
68 | "files": {
69 | "high_sierra": {
70 | "url": "https://homebrew.bintray.com/bottles/cmake-3.10.2.high_sierra.bottle.tar.gz",
71 | "sha256": "7ad98f403e21c76cfd0789d83acffce92cb29aad3d2ea9b4fe8b2c05de8f33b9"
72 | },
73 | "sierra": {
74 | "url": "https://homebrew.bintray.com/bottles/cmake-3.10.2.sierra.bottle.tar.gz",
75 | "sha256": "878aeaeda98df7a8940bdccc42b7ff22195980a62410428f8febc7cd03a1c681"
76 | },
77 | "el_capitan": {
78 | "url": "https://homebrew.bintray.com/bottles/cmake-3.10.2.el_capitan.bottle.tar.gz",
79 | "sha256": "05efc0e612c16eabdf8e509545d18a7e1016de2c6cb6deb597b49ffd590dcc0a"
80 | }
81 | }
82 | }
83 | }
84 | }
85 | ]
--------------------------------------------------------------------------------
/zorg/buildbot/builders/AnnotatedBuilder.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | import buildbot
4 | from buildbot.process.properties import WithProperties
5 | from buildbot.steps.shell import SetProperty, ShellCommand
6 | from buildbot.steps.source import SVN
7 | from zorg.buildbot.commands.AnnotatedCommand import AnnotatedCommand
8 | from zorg.buildbot.process.factory import LLVMBuildFactory
9 |
10 |
11 | def getAnnotatedBuildFactory(
12 | script,
13 | clean=False,
14 | depends_on_projects=None,
15 | env=None,
16 | timeout=1200):
17 | """
18 | Returns a new build factory that uses AnnotatedCommand, which
19 | allows the build to be run by version-controlled scripts that do
20 | not require a buildmaster restart to update.
21 | """
22 |
23 | f = LLVMBuildFactory(
24 | depends_on_projects=depends_on_projects,
25 | llvm_srcdir='llvm.src')
26 |
27 | if clean:
28 | f.addStep(SetProperty(property='clean', command='echo 1'))
29 |
30 | # We normally use the clean property to indicate that we want a
31 | # clean build, but AnnotatedCommand uses the clobber property
32 | # instead. Therefore, set clobber if clean is set to a truthy
33 | # value. This will cause AnnotatedCommand to set
34 | # BUILDBOT_CLOBBER=1 in the environment, which is how we
35 | # communicate to the script that we need a clean build.
36 | f.addStep(SetProperty(
37 | property='clobber',
38 | command='echo 1',
39 | doStepIf=lambda step: step.build.getProperty('clean', False)))
40 |
41 | merged_env = {
42 | 'TERM': 'dumb' # Be cautious and disable color output from all tools.
43 | }
44 | if env is not None:
45 | # Overwrite pre-set items with the given ones, so user can set
46 | # anything.
47 | merged_env.update(env)
48 |
49 | scripts_dir = "annotated"
50 | f.addStep(SVN(name='update-annotate-scripts',
51 | mode='update',
52 | svnurl='http://llvm.org/svn/llvm-project/zorg/trunk/'
53 | 'zorg/buildbot/builders/annotated',
54 | workdir=scripts_dir,
55 | alwaysUseLatest=True))
56 |
57 | # Explicitly use '/' as separator, because it works on *nix and Windows.
58 | script_path = "../%s/%s" % (scripts_dir, script)
59 | f.addStep(AnnotatedCommand(name="annotate",
60 | description="annotate",
61 | timeout=timeout,
62 | haltOnFailure=True,
63 | command=WithProperties(
64 | "python %(script)s --jobs=%(jobs:-)s",
65 | script=lambda _: script_path),
66 | env=merged_env))
67 | return f
68 |
--------------------------------------------------------------------------------
/llvmbisect/llvmlab/algorithm.py:
--------------------------------------------------------------------------------
1 | """Handy algorithms."""
2 |
3 |
4 | def bisect(predicate, list):
5 | """
6 | bisect(predicate, list) -> item or None
7 |
8 | Given a test predicate and a list of items, search return the first item in
9 | the list for which the predicate succeeds, or None if no such item is
10 | found.
11 |
12 | The list is assumed to be ordered such that (predicate(i) for i in list) is
13 | monotonic. If this condition is not met, the returned item is guaranteed to
14 | satisfy the predicate and the item preceeding it is guaranteed to fail the
15 | predicate, but that is all. Additionally, if the last item does not pass
16 | the predicate, such an item might not be found.
17 |
18 | This function is optimized for the case where the searched for item is near
19 | the beginning of the list.
20 | """
21 |
22 | if not list:
23 | return None
24 |
25 | lo = 0
26 | hi = len(list)-1
27 |
28 | # Check first item immediately.
29 | if predicate(list[lo]):
30 | return list[lo]
31 |
32 | # Invariants:
33 | # not predicate(list[lo])
34 | # predicate(list[hi])
35 |
36 | # Binary search region.
37 | while lo + 1 != hi:
38 | mid = (lo + hi) // 2
39 | if predicate(list[mid]):
40 | hi = mid
41 | else:
42 | lo = mid
43 |
44 | return list[hi]
45 |
46 |
47 | def gallop(predicate, list):
48 | """
49 | gallop(predicate, list) -> list or None
50 |
51 | Given a test predicate and a list of items, reduce the search space
52 | assuming the searched for item is near the beginning of the list.
53 |
54 | The list is assumed to be ordered such that (predicate(i) for i in list) is
55 | monotonic. If this condition is not met, the returned item is guaranteed to
56 | satisfy the predicate and the item preceeding it is guaranteed to fail the
57 | predicate, but that is all. Additionally, if the last item does not pass
58 | the predicate, such an item might not be found.
59 | """
60 |
61 | if not list:
62 | return None
63 |
64 | # Check first item immediately.
65 | if predicate(list[0]):
66 | return list[0:1]
67 |
68 | # Invariants:
69 | # not predicate(list[lo])
70 |
71 | # Gallop to find initial search range, under the assumption that we are
72 | # most likely looking for something at the head of this list.
73 | lo = 0
74 | hi = 1
75 | while hi < len(list):
76 | if predicate(list[hi]):
77 | break
78 | lo, hi = hi, hi + (hi - lo)*2
79 |
80 | # If we galloped past the end, limit the hi range.
81 | if hi >= len(list):
82 | hi = len(list) - 1
83 | if hi == lo or not predicate(list[hi]):
84 | return None
85 | return list[lo:hi+1]
86 |
--------------------------------------------------------------------------------
/llvmbisect/llvmlab/gcs.py:
--------------------------------------------------------------------------------
1 | """Integration with Google Cloud Storage.
2 |
3 | """
4 | import os
5 | import requests
6 | import urllib3
7 |
8 | # Root URL to use for our queries.
9 | DEFAULT_GCS = "https://www.googleapis.com/storage/v1/"
10 |
11 | # Specify a different root URL if compilers are hosted elsewhere.
12 | GCS = os.getenv("GCS_SERVER", DEFAULT_GCS)
13 |
14 | DEFAULT_BUCKET = "llvm-build-artifacts"
15 |
16 | BUCKET = os.getenv("BUCKET", DEFAULT_BUCKET)
17 |
18 | class HttpClient(object):
19 | def __init__(self):
20 | self.session = requests.Session()
21 | # Retry after 0s, 0.2s, 0.4s.
22 | retry = urllib3.util.retry.Retry(total=3, backoff_factor=0.1)
23 | adapter = requests.adapters.HTTPAdapter(max_retries=retry)
24 | self.session.mount('https://', adapter)
25 |
26 | def get(self, url, **kwargs):
27 | if "timeout" not in kwargs:
28 | kwargs["timeout"] = 5 # seconds
29 | return self.session.get(url, **kwargs)
30 |
31 | HTTP_CLIENT = HttpClient()
32 |
33 |
34 | def fetch_builders():
35 | """Each build kind is stored as a folder in the GCS bucket.
36 | List all the folders in the bucket, which is our list of possible
37 | compilers.
38 | """
39 | params = {'delimiter': "/", 'fields': "prefixes,nextPageToken"}
40 | r = HTTP_CLIENT.get(GCS + "b/" + BUCKET + "/o", params=params)
41 | r.raise_for_status()
42 | reply_data = r.json()
43 | assert "nextPageToken" not in reply_data.keys(), "Too many builders!"
44 | folders = reply_data['prefixes']
45 | no_slashes = [x.replace("/", "") for x in folders]
46 | return no_slashes
47 |
48 |
49 | def fetch_builds(project):
50 | """Given a builder name, get the list of all the files stored for that
51 | builder.
52 | """
53 | assert project is not None
54 | all_data = {'items':[]}
55 | params = {'delimiter': "/",
56 | "fields": "nextPageToken,kind,items(name, mediaLink)",
57 | 'prefix': project + "/"}
58 | r = HTTP_CLIENT.get(GCS + "b/" + BUCKET + "/o", params=params)
59 | r.raise_for_status()
60 | reply_data = r.json()
61 | all_data['items'].extend(reply_data['items'])
62 | while reply_data.get('nextPageToken'):
63 | params['pageToken'] = reply_data['nextPageToken']
64 | r = HTTP_CLIENT.get(GCS + "b/" + BUCKET + "/o", params=params)
65 | r.raise_for_status()
66 | reply_data = r.json()
67 | all_data['items'].extend(reply_data['items'])
68 | return all_data
69 |
70 | # Dunno what this could be moved up to?
71 | CHUNK_SIZE = 5124288
72 |
73 |
74 | def get_compiler(url, filename):
75 | """Get the compiler at the url, and save to filename."""
76 | r = HTTP_CLIENT.get(url)
77 | r.raise_for_status()
78 | with open(filename, 'wb') as fd:
79 | for chunk in r.iter_content(CHUNK_SIZE):
80 | fd.write(chunk)
81 | return filename
82 |
--------------------------------------------------------------------------------
/llvmlab/docs/intro.rst:
--------------------------------------------------------------------------------
1 | .. _intro:
2 |
3 | Introduction
4 | ============
5 |
6 | Installation
7 | ------------
8 |
9 | These are the (current) rough steps to get a working llvmlab server:
10 |
11 | 1. Install llvmlab::
12 |
13 | python setup.py install
14 |
15 | It is recommended that you install llvmlab into a virtualenv. If you are
16 | developing the software, you presumably want to use::
17 |
18 | python setup.py develop
19 |
20 | 2. Create a new llvmlab installation::
21 |
22 | llvmlab create \
23 | --master-url http://example.com:8010 \
24 | --plugin-module zorg.llvmlab
25 | path/to/install-dir
26 |
27 | This will create the llvmlab configuration file, the default database, and a
28 | .wsgi wrapper to create the application. If using this instance for
29 | development, you may want to add the ``--debug-server`` argument to default
30 | to running the server in debug mode.
31 |
32 | The ``--master-url`` should be used to point the lab at the buildbot
33 | installation it is intended to monitor. You can monitor ``lab.llvm.org`` for
34 | quick testing purposes, but **please** do not leave this running for an
35 | extended time, as it puts a certain amount of load on the buildbot
36 | installation. If you want to run longer tests, please run a local buildbot
37 | master and monitor that.
38 |
39 | The ``--plugin-module`` argument is required in order for the dashboard to
40 | work, it is how the dashboard loads the information about the buildbot
41 | configuration. The module path is expected to be importable, so you may need
42 | to extend the PYTHONPATH to support that (e.g.,
43 | ``PYTHONPATH=/path/to/zorg/repo`` would allow the default ``lab.llvm.org``
44 | plugin named above to be imported).
45 |
46 | If using this instance for deployment, you *certainly* want to provide the
47 | ``--admin-email`` and ``--admin-password`` arguments to override the
48 | defaults. You may also need to modify the generated ``app.cfg`` file to
49 | change the default SMTP relay server (used for mailing error messages).
50 |
51 | You can execute the generated WSGI app directly to run with the builtin web
52 | server, or use::
53 |
54 | env LLVMLAB_CONFIG=/path/to/instance/lab.cfg llvmlab runserver
55 |
56 | which may eventually provide additional command line options. Neither of
57 | these servers is recommended for production use.
58 |
59 | 3. Add the 'app.wsgi' app to your Apache configuration. You should set also
60 | configure the WSGIDaemonProcess and WSGIProcessGroup variables if not
61 | already done.
62 |
63 | If running in a virtualenv you will need to configure that as well; see the
64 | `modwsgi wiki `_.
65 |
66 |
67 | Architecture
68 | ------------
69 |
70 | The llvmlab web app is currently implemented as a WSGI web app using Flask.
71 |
--------------------------------------------------------------------------------
/zorg/jenkins/jobs/util/make_pipeline.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | import sys
3 | import re
4 | from xml.sax.saxutils import escape
5 |
6 | pipeline_git_url = sys.argv[1]
7 | pipeline_git_path = sys.argv[2]
8 | description = open(sys.argv[3]).read()
9 |
10 | template = '''\
11 |
12 |
13 | {description}
14 | false
15 |
16 |
17 | false
18 |
19 |
20 | false
21 | false
22 |
23 | {extra_properties}
24 |
25 |
26 |
27 | 2
28 |
29 |
30 | {pipeline_git_url}
31 |
32 |
33 |
34 |
35 | */master
36 |
37 |
38 | false
39 |
42 | {pipeline_git_path}
43 | true
44 |
45 | false
47 |
48 | '''
49 |
50 | # Bad hack to set throttle parameter if 'relay' is in the jobname
51 | extra_properties = ''
52 | if 'relay' in pipeline_git_path:
53 | extra_properties += '''
54 |
55 | 1
56 | 1
57 | true
59 | project
60 | false
61 |
63 | '''
64 |
65 | variables = {
66 | "pipeline_git_url": escape(pipeline_git_url),
67 | "pipeline_git_path": escape(pipeline_git_path),
68 | "extra_properties": extra_properties,
69 | }
70 | variables['description'] = escape(description.format(**variables))
71 |
72 | sys.stdout.write(template.format(**variables))
73 |
--------------------------------------------------------------------------------
/llvmlab/llvmlab/ui/static/style.css:
--------------------------------------------------------------------------------
1 | body {
2 | background-color: #F0F3FF;
3 | font-family: "Lucida Grande", "Lucida Sans Unicode", Arial, Verdana,
4 | Helvetica, sans-serif;
5 | font-size: small;
6 | }
7 |
8 | div.body {
9 | color: #000;
10 | margin: 20px;
11 | }
12 | div.body a {
13 | color: #333;
14 | }
15 |
16 | div.header {
17 | background-color: #50505A;
18 | color: #FFF;
19 | font-size: 125%;
20 | padding: 10px 10px;
21 | text-align: left;
22 | }
23 | div.header a {
24 | color: #FFF;
25 | }
26 |
27 | div.footer {
28 | background-color: #50505A;
29 | color: #FFF;
30 | font-size: 75%;
31 | padding: 10px 10px;
32 | text-align: right;
33 | }
34 | div.footer a {
35 | color: #FFF;
36 | }
37 |
38 | div.flash_messages {
39 | margin: 10px;
40 | }
41 | div.flash_message {
42 | background: #CEE5F5;
43 | padding: 10px;
44 | border: 1px solid #AACBE2;
45 | }
46 |
47 | div.popup {
48 | border-radius: 7px;
49 | -webkit-border-radius: 7px;
50 | -moz-border-radius: 7px;
51 | position: absolute;
52 | background-color: #FFFFFF;
53 | padding: 4px 4px 4px 4px;
54 | float: left;
55 | display: none;
56 | border-width: 1px;
57 | border-style: solid;
58 | }
59 |
60 | div.popup_inner {
61 | border-radius: 3px;
62 | -webkit-border-radius: 3px;
63 | -moz-border-radius: 3px;
64 | background-color: #F0F3EF;
65 | padding: 4px 4px 4px 4px;
66 | }
67 |
68 | /* Dashboard Features */
69 |
70 | a.help-link {
71 | padding: 0px;
72 | font-size: 75%;
73 | vertical-align: super;
74 | }
75 |
76 | td.phase-cell {
77 | padding: 4px 4px 4px 4px;
78 | color: #333333;
79 | background-color: #eee;
80 | }
81 |
82 | .phase-cell {
83 | text-align: center;
84 | height: 20px;
85 | padding: 0 2px;
86 | line-height: 0;
87 | white-space: nowrap;
88 | }
89 |
90 | .phase-cell a {
91 | opacity: 0.85;
92 | border-width: 1px;
93 | border-style: solid;
94 | border-radius: 4px;
95 | -webkit-border-radius: 4px;
96 | -moz-border-radius: 4px;
97 | display: block;
98 | width: 90%;
99 | height: 20px;
100 | line-height: 20px;
101 | margin-left: auto;
102 | margin-right: auto;
103 | }
104 |
105 |
106 | /* Build Status Indicators */
107 | .success {
108 | color: #FFFFFF;
109 | background-color: #8f8;
110 | border-color: #4F8530;
111 | }
112 | .failure {
113 | color: #FFFFFF;
114 | background-color: #f99;
115 | border-color: #A77272;
116 | }
117 | .warnings {
118 | color: #FFFFFF;
119 | background-color: #ffc343;
120 | border-color: #C29D46;
121 | }
122 | .running {
123 | color: #666666;
124 | background-color: #ff6;
125 | border-color: #C5C56D;
126 | }
127 | .unknown {
128 | border-width: 1px;
129 | border-style: solid;
130 | border-color: #aaa;
131 | background-color: #fff;
132 | }
133 |
--------------------------------------------------------------------------------
/zorg/buildbot/commands/MakeCommand.py:
--------------------------------------------------------------------------------
1 | import re
2 |
3 | from buildbot.process.properties import WithProperties
4 | from buildbot.steps.shell import WarningCountingShellCommand
5 |
6 | class MakeCommand(WarningCountingShellCommand):
7 |
8 | @staticmethod
9 | def sanitize_kwargs(kwargs):
10 | # kwargs we could get and must not pass through
11 | # to the buildstep.RemoteShellCommand constructor.
12 | # Note: This is a workaround of the buildbot design issue,
13 | # thus should be removed once the original issue gets fixed.
14 | consume_kwargs = [
15 | "jobs",
16 | ]
17 |
18 | sanitized_kwargs = kwargs.copy()
19 | for k in consume_kwargs:
20 | if k in sanitized_kwargs.keys():
21 | del sanitized_kwargs[k]
22 |
23 | return sanitized_kwargs
24 |
25 |
26 | def __init__(self, prefixCommand=None, options=None, targets=None, **kwargs):
27 | self.prefixCommand = prefixCommand
28 | self.targets = targets
29 |
30 | command = []
31 | if prefixCommand:
32 | command += prefixCommand
33 |
34 | command += ["make"]
35 |
36 | if options is None:
37 | self.options = list()
38 | else:
39 | self.options = list(options)
40 |
41 | j_opt = re.compile(r'^-j$|^-j\d+$')
42 |
43 | # We can get jobs in the options. If so, we would use that.
44 | if not any(j_opt.search(opt) for opt in self.options if isinstance(opt, basestring)):
45 | # Otherwise let's see if we got it in the kwargs.
46 | if kwargs.get('jobs', None):
47 | self.options += ["-j", kwargs['jobs']]
48 | else:
49 | # Use the property if option was not explicitly
50 | # specified.
51 | command += [
52 | WithProperties("%(jobs:+-j)s"),
53 | WithProperties("%(jobs:-)s"),
54 | ]
55 |
56 | if self.options:
57 | command += self.options
58 |
59 | if targets:
60 | command += targets
61 |
62 | # Remove here all the kwargs any of our LLVM buildbot command could consume.
63 | # Note: We will remove all the empty items from the command at start, as we
64 | # still didn't get yet WithProperties rendered.
65 | sanitized_kwargs = self.sanitize_kwargs(kwargs)
66 |
67 | sanitized_kwargs["command"] = command
68 |
69 | # And upcall to let the base class do its work
70 | WarningCountingShellCommand.__init__(self, **sanitized_kwargs)
71 |
72 | self.addFactoryArguments(prefixCommand=prefixCommand,
73 | options=self.options,
74 | targets=targets)
75 |
76 |
77 | def start(self):
78 | # Don't forget to remove all the empty items from the command,
79 | # which we could get because of WithProperties rendered as empty strings.
80 | self.command = filter(bool, self.command)
81 | # Then upcall.
82 | WarningCountingShellCommand.start(self)
83 |
--------------------------------------------------------------------------------
/zorg/buildbot/commands/DejaGNUCommand.py:
--------------------------------------------------------------------------------
1 | import re
2 | import urllib
3 | import buildbot
4 | import buildbot.status.builder
5 | from buildbot.status.results import FAILURE, SUCCESS
6 | import buildbot.steps.shell
7 | from buildbot.process.buildstep import LogLineObserver
8 | from buildbot.steps.shell import Test
9 |
10 | class DejaGNULogObserver(LogLineObserver):
11 | kStartLineRE = re.compile(r'Running .*/(.*/.*\.exp) \.\.\.');
12 | kFinishLineRE = re.compile(r'testcase .*/(.*/.*\.exp) completed in .* seconds');
13 | kTestStateLineRE = re.compile(r'(FAIL|PASS|XFAIL|XPASS|KFAIL|KPASS|UNRESOLVED|UNTESTED|UNSUPPORTED): .*')
14 | failingCodes = set(['FAIL', 'XPASS', 'KPASS', 'UNRESOLVED'])
15 | def __init__(self):
16 | LogLineObserver.__init__(self)
17 | self.resultCounts = {}
18 | self.currentLines = ''
19 | self.currentFailed = False
20 | self.anyFailed = False
21 | def outLineReceived(self, line):
22 | if len(self.currentLines):
23 | self.currentLines += '\n' + line
24 | m = self.kTestStateLineRE.search(line)
25 | if m:
26 | resultCode, = m.groups()
27 | if resultCode in self.failingCodes:
28 | self.hasFailed = True
29 | self.currentFailed = True
30 | self.anyFailed = True
31 | if not resultCode in self.resultCounts:
32 | self.resultCounts[resultCode] = 0
33 | self.resultCounts[resultCode] += 1
34 | m = self.kFinishLineRE.match(line)
35 | if m:
36 | name, = m.groups()
37 | if self.currentFailed:
38 | self.step.addCompleteLog(name.replace('/', '__'), self.currentLines)
39 | self.currentLines = ''
40 | else:
41 | m = self.kStartLineRE.match(line)
42 | if m:
43 | self.currentLines = line
44 | self.currentFailed = False
45 |
46 | class DejaGNUCommand(Test):
47 | resultNames = {'FAIL':'unexpected failures',
48 | 'PASS':'expected passes',
49 | 'XFAIL':'expected failures',
50 | 'XPASS':'unexpected passes',
51 | 'KFAIL':'known failures',
52 | 'KPASS':'unknown passes',
53 | 'UNRESOLVED':'unresolved testcases',
54 | 'UNTESTED':'untested testcases',
55 | 'UNSUPPORTED':'unsupported tests'}
56 |
57 | def __init__(self, ignore=[], flaky=[], max_logs=20,
58 | *args, **kwargs):
59 | Test.__init__(self, *args, **kwargs)
60 | self.logObserver = DejaGNULogObserver()
61 | self.addLogObserver('gdb.log', self.logObserver)
62 |
63 | def evaluateCommand(self, cmd):
64 | if self.logObserver.anyFailed:
65 | return FAILURE
66 | return SUCCESS
67 |
68 | def describe(self, done=False):
69 | description = Test.describe(self, done)
70 | for name, count in self.logObserver.resultCounts.iteritems():
71 | description.append('{0} {1}'.format(count, self.resultNames[name]))
72 | return description
73 |
--------------------------------------------------------------------------------
/zorg/buildbot/util/InformativeMailNotifier.py:
--------------------------------------------------------------------------------
1 | import buildbot
2 | from buildbot import util, interfaces
3 | from zope.interface import implements
4 | from buildbot.status import builder, mail
5 |
6 | if buildbot.version[:5] >= '0.8.7':
7 | def get_change_string(build):
8 | data = ''
9 | ss_list = build.getSourceStamps()
10 | if ss_list:
11 | data += 'CHANGES:\n'
12 | for ss in ss_list:
13 | data += '\n\n'.join([c.asText() for c in ss.changes])
14 | data += '\n\n'
15 | else:
16 | data += 'NO SOURCE STAMP (CHANGES UNAVAILABLE)'
17 | data += '\n\n'
18 | return data
19 | else:
20 | def get_change_string(build):
21 | data = ''
22 | ss = build.getSourceStamp()
23 | if ss:
24 | data += 'CHANGES:\n'
25 | data += '\n\n'.join([c.asText() for c in ss.changes])
26 | data += '\n\n'
27 | else:
28 | data += 'NO SOURCE STAMP (CHANGES UNAVAILABLE)'
29 | data += '\n\n'
30 | return data
31 |
32 | class InformativeMailNotifier(mail.MailNotifier):
33 | """MailNotifier subclass which provides additional information about the
34 | build failure inside the email."""
35 |
36 | implements(interfaces.IEmailSender)
37 | compare_attrs = (mail.MailNotifier.compare_attrs +
38 | ["num_lines", "only_failure_logs"])
39 |
40 | # Remove messageFormatter from the compare_attrs, that would lead to
41 | # recursion, and is checked by the class test.
42 | compare_attrs.remove("messageFormatter")
43 |
44 | def __init__(self,
45 | num_lines = 10, only_failure_logs = True,
46 | *attrs, **kwargs):
47 | mail.MailNotifier.__init__(self,
48 | messageFormatter=self.informative_formatter,
49 | *attrs, **kwargs)
50 | self.num_lines = num_lines
51 | self.only_failure_logs = only_failure_logs
52 |
53 | # Adapt to work with 0.8.3...
54 | if not hasattr(self, 'defaultMessage'):
55 | self.defaultMessage = mail.defaultMessage
56 |
57 | def informative_formatter(self, mode, name, build, results, status):
58 | # Get the standard message.
59 | data = self.defaultMessage(mode, name, build, results, status)['body']
60 | data += '\n' + '='*80 + '\n\n'
61 |
62 | # Append additional information on the changes.
63 | data += get_change_string(build)
64 |
65 | # Append log files.
66 | if self.num_lines:
67 | data += 'LOGS:\n'
68 | for logf in build.getLogs():
69 | logStep = logf.getStep()
70 | logStatus,_ = logStep.getResults()
71 | if (self.only_failure_logs and logStatus != builder.FAILURE):
72 | continue
73 |
74 | trailingLines = logf.getText().splitlines()[-self.num_lines:]
75 | data += "Last %d lines of '%s':\n" % (self.num_lines,
76 | logf.getName())
77 | data += '\t' + '\n\t'.join(trailingLines)
78 | data += '\n\n'
79 |
80 | return { 'body' : data,
81 | 'type' : 'plain' }
82 |
--------------------------------------------------------------------------------
/zorg/jenkins/jobs/jobs/lldb-cmake-sanitized:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env groovy
2 | pipeline {
3 | agent { label 'green-dragon-23' }
4 | parameters {
5 | string(name: 'GIT_REVISION', defaultValue: '*/master', description: 'Git revision to build')
6 | string(name: 'ARTIFACT', defaultValue: 'clang-stage1-RA/latest', description: 'Compiler artifact to use for building the project')
7 | string(name: 'BUILD_TYPE', defaultValue: 'Release', description: 'Default CMake build type; one of: Release, Debug, ...')
8 | string(name: 'CLEAN', defaultValue: "false", description: 'Whether or not to clean the build directory before building')
9 | }
10 | stages {
11 | stage('Checkout') {
12 | steps {
13 | dir('llvm-project') {
14 | checkout([$class: 'GitSCM', branches: [[name: params.GIT_REVISION]], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-project.git']]])
15 | }
16 | dir('llvm-zorg') {
17 | checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[url: 'http://labmaster3.local/git/llvm-zorg.git']]])
18 | }
19 | }
20 | }
21 | stage('Fetch Host Compiler') {
22 | steps {
23 | timeout(10) {
24 | sh '''
25 | python llvm-zorg/zorg/jenkins/monorepo_build.py fetch
26 | '''
27 | }
28 | }
29 | }
30 | stage('Build') {
31 | steps {
32 | timeout(90) {
33 | sh '''
34 | set -u
35 | rm -rf build.properties
36 |
37 | cd llvm-project
38 | git tag -a -m "First Commit" first_commit 97724f18c79c7cc81ced24239eb5e883bf1398ef || true
39 |
40 | git_desc=$(git describe --match "first_commit")
41 |
42 | export GIT_DISTANCE=$(echo ${git_desc} | cut -f 2 -d "-")
43 |
44 | sha=$(echo ${git_desc} | cut -f 3 -d "-")
45 | export GIT_SHA=${sha:1}
46 |
47 | cd -
48 |
49 | export PATH=$PATH:/usr/bin:/usr/local/bin
50 |
51 | python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-sanitized build \
52 | --assertions \
53 | --projects="clang;libcxx;libcxxabi;lld;lldb" \
54 | --cmake-type=Release
55 | '''
56 | }
57 | }
58 | }
59 | stage('Test') {
60 | steps {
61 | timeout(120) {
62 | sh '''
63 | set -u
64 | export PATH=$PATH:/usr/bin:/usr/local/bin
65 |
66 | rm -rf test/results.xml
67 |
68 | # Running too many asanified threads is too stressful for the kernel and we get >90% system time.
69 | export MAX_PARALLEL_TESTS=$(sysctl hw.physicalcpu |awk '{print $2}')
70 |
71 | python llvm-zorg/zorg/jenkins/monorepo_build.py lldb-cmake-sanitized test
72 | '''
73 | }
74 | }
75 | }
76 | }
77 |
78 | post {
79 | always {
80 | scanForIssues tool: clang()
81 | junit 'test/results.xml'
82 | }
83 | }
84 | }
85 |
--------------------------------------------------------------------------------
/zorg/buildbot/commands/NightlyTestCommand.py:
--------------------------------------------------------------------------------
1 | import itertools
2 | import re
3 | import os
4 |
5 | import buildbot
6 | import buildbot.steps.shell
7 |
8 | class NightlyTestCommand(buildbot.steps.shell.Test):
9 |
10 | def __init__(self, xfails=[], *args, **kwargs):
11 | buildbot.steps.shell.Test.__init__(self, *args, **kwargs)
12 |
13 | self.expectedFailures = set(xfails)
14 | self.addFactoryArguments(xfails=list(xfails))
15 |
16 | def evaluateCommand(self, cmd):
17 | # Always fail if the command itself failed.
18 | #
19 | # Disabled for now, nightlytest is so broken.
20 | #if cmd.rc != 0:
21 | # return buildbot.status.builder.FAILURE
22 |
23 | failures = {}
24 | xfailures = {}
25 | xpasses = {}
26 | num_passes = 0
27 | num_tests = 0
28 | for item in parse_report(self.getLog('report').readlines()):
29 | name = item.pop('Program')
30 | for key,value in item.items():
31 | if '/' in key:
32 | continue
33 | kname = '%s.%s' % (key,name)
34 | if value == '*':
35 | if kname in self.expectedFailures:
36 | xfailures[key] = xfailures.get(key,[])
37 | xfailures[key].append(kname)
38 | else:
39 | failures[key] = failures.get(key,[])
40 | failures[key].append(kname)
41 | else:
42 | if kname in self.expectedFailures:
43 | xpasses[key] = xpasses.get(key,[])
44 | xpasses[key].append(kname)
45 | else:
46 | num_passes += 1
47 | num_tests += 1
48 |
49 | num_fails = num_xfails = num_xpasses = 0
50 | for type,items in failures.items():
51 | if len(items) == num_tests: # Assume these are disabled.
52 | continue
53 | self.addCompleteLog('fail.%s' % type, '\n'.join(items) + '\n')
54 | num_fails += len(items)
55 | for type,items in xfailures.items():
56 | self.addCompleteLog('xfail.%s' % type, '\n'.join(items) + '\n')
57 | num_xfails += len(items)
58 | for type,items in xpasses.items():
59 | self.addCompleteLog('xpass.%s' % type, '\n'.join(items) + '\n')
60 | num_xpasses += len(items)
61 |
62 | self.setTestResults(total=(num_passes + num_fails + num_xfails +
63 | num_xpasses),
64 | failed=num_fails,
65 | passed=num_passes + num_xpasses,
66 | warnings=num_xfails + num_xpasses)
67 | if num_fails:
68 | return buildbot.status.builder.FAILURE
69 | return buildbot.status.builder.SUCCESS
70 |
71 | def parse_report(lines):
72 | def split_row(ln):
73 | return ln.split()
74 |
75 | header = None
76 | for ln in lines:
77 | if not ln.strip():
78 | continue
79 |
80 | if header is None:
81 | ln = ln.replace(' compile', '_compile')
82 | ln = ln.replace(' codegen', '_codegen')
83 | header = split_row(ln)
84 | else:
85 | row = split_row(ln)
86 | yield dict(zip(header, split_row(ln)))
87 |
--------------------------------------------------------------------------------
/tasks/tasktool/tasktool/utils.py:
--------------------------------------------------------------------------------
1 | from pipes import quote
2 | import inspect
3 | import os
4 | import subprocess
5 | import sys
6 | import textwrap
7 |
8 | verbose = False
9 |
10 |
11 | def _logged_catched(func, commandline, *args, **kwargs):
12 | commandline_string = commandline
13 | if isinstance(commandline, list) or isinstance(commandline, tuple):
14 | commandline_string = " ".join([quote(arg) for arg in commandline])
15 |
16 | if verbose:
17 | cwd = kwargs.get('cwd', None)
18 | if cwd is not None:
19 | sys.stderr.write("In Directory: %s\n" % cwd)
20 | sys.stderr.write("$ %s\n" % commandline_string)
21 | try:
22 | return func(commandline, *args, **kwargs)
23 | except subprocess.CalledProcessError as e:
24 | sys.stderr.write("Error while executing: $ %s\n" %
25 | commandline_string)
26 | if e.output is not None:
27 | sys.stderr.write(str(e.output))
28 | sys.exit(e.returncode)
29 | except OSError as e:
30 | sys.stderr.write("Error while executing $ %s\n" % commandline_string)
31 | sys.stderr.write("Error: %s\n" % e)
32 | sys.exit(1)
33 |
34 |
35 | def check_call(commandline, *args, **kwargs):
36 | """Opinionated version of subprocess.check_call: Logs commandline in
37 | verbose mode, and for returncode != 0 print a message and exit the
38 | application."""
39 | return _logged_catched(subprocess.check_call, commandline, *args, **kwargs)
40 |
41 |
42 | def check_output(commandline, *args, **kwargs):
43 | """Opinionated version of subprocess.check_output: Logs commandline in
44 | verbose mode, and for returncode != 0 print a message and exit the
45 | application."""
46 | return _logged_catched(subprocess.check_output, commandline, *args,
47 | **kwargs)
48 |
49 |
50 | def call(commandline, *args, **kwargs):
51 | """Opinionated version of subprocess.check_output: Logs commandline in
52 | verbose mode, and exit with message the if command does not exist."""
53 | return _logged_catched(subprocess.call, commandline, *args, **kwargs)
54 |
55 |
56 | def _print_help(commands, argv, docstring):
57 | if docstring:
58 | description = inspect.cleandoc(docstring)
59 | sys.stderr.write(description)
60 | sys.stderr.write("\n\n")
61 | sys.stderr.write("Usage:\n")
62 | twc = textwrap.TextWrapper(expand_tabs=True, initial_indent=' ',
63 | subsequent_indent=' ', width=78)
64 | for name, func in sorted(commands.items(), key=lambda x: x[0]):
65 | sys.stderr.write(" %s %s [arg]...\n" %
66 | (os.path.basename(argv[0]), name))
67 | if func.__doc__:
68 | docstring = inspect.cleandoc(func.__doc__)
69 | sys.stderr.write('\n'.join(twc.wrap(docstring)) + '\n')
70 |
71 |
72 | def run_subcommand(commands, argv, docstring=None):
73 | if len(argv) < 2:
74 | _print_help(commands, argv, docstring)
75 | sys.stderr.write("\nError: No command specified!\n")
76 | sys.exit(1)
77 |
78 | commandname = argv[1]
79 | if commandname == 'help' or commandname == '--help':
80 | _print_help(commands, argv, docstring)
81 | sys.exit(0)
82 |
83 | command = commands.get(argv[1])
84 | if command is None:
85 | sys.stderr.write("Error: Unknown command '%s'\n" % argv[1])
86 | sys.exit(1)
87 |
88 | args = argv[2:]
89 | command(args)
90 |
--------------------------------------------------------------------------------
/codesign/debugsign/unittests/test_commands.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python -tt
2 |
3 | """
4 | unittests.test_commands
5 | """
6 |
7 | from __future__ import print_function
8 |
9 | try:
10 | import unittest2 as unittest
11 | except ImportError:
12 | import unittest
13 |
14 | import mock
15 | import os
16 | import StringIO
17 |
18 | import dbsign.commands as C
19 | import dbsign.logger as L
20 | from dbsign.result import Failure, Success
21 | import dbsign.shell as sh
22 |
23 |
24 | log = L.get_logger(__name__)
25 |
26 |
27 | def dummy_sudo_run(params, *args):
28 | if sh.UNSAFE_FLAG in os.environ:
29 | return dummy_run(params, *args)
30 | else:
31 | raise RuntimeError()
32 |
33 |
34 | def dummy_run(params, *args):
35 | return sh.ShellCommand(params, 0, 'dummy_stdout', 'dummy_stderr')
36 |
37 |
38 | class TestCommands(unittest.TestCase):
39 | def setUp(self):
40 | pass
41 |
42 | def tearDown(self):
43 | pass
44 |
45 | @mock.patch('dbsign.shell.sudo_run')
46 | @mock.patch('__builtin__.print')
47 | def test_auth_sudo(self, _print, _sudo_run):
48 | _sudo_run.side_effect = [
49 | Success('good1'), # run 1
50 | Failure('bad2'), Success('good2'), # run 2
51 | Failure('bad3'), Failure('bad3'), # run 3
52 | ]
53 | self.assertTrue(C._auth_sudo())
54 | self.assertTrue(C._auth_sudo())
55 | self.assertFalse(C._auth_sudo())
56 |
57 | @mock.patch('dbsign.shell.run')
58 | @mock.patch('sys.stdout', new_callable=StringIO.StringIO)
59 | def test_run_linter(self, mock_stdout, new_run):
60 | new_run.side_effect = dummy_run
61 | C._run_linter()
62 | new_run.assert_called()
63 | self.assertIn('flake8', new_run.call_args[0][0])
64 |
65 | @mock.patch('__builtin__.print')
66 | def test_cmd_help(self, _print):
67 | parser = mock.MagicMock()
68 | code = C.cmd_help(parser)
69 | self.assertTrue(_print.called)
70 | self.assertTrue(parser.format_help.called)
71 | self.assertEqual(0, code)
72 |
73 | @mock.patch('dbsign.security.unlock_keychain')
74 | @mock.patch('__builtin__.print')
75 | def test_cmd_prep(self, _print, _unlock):
76 | _unlock.side_effect = (Success('good'), Failure('bad'))
77 | self.assertEqual(0, C.cmd_prep())
78 | self.assertEqual(1, C.cmd_prep())
79 |
80 | @mock.patch('dbsign.commands._run_unittests')
81 | @mock.patch('dbsign.commands._auth_sudo')
82 | @mock.patch('__builtin__.print')
83 | def test_cmd_test(self, _print, _auth_sudo, _run_unit):
84 | _run_unit.side_effect = [(), (1, 2, 3)]
85 | self.assertEqual(0, C.cmd_test())
86 | self.assertEqual(1, _print.call_count)
87 |
88 | _print.reset_mock()
89 | self.assertEqual(3, C.cmd_test())
90 | self.assertEqual(2, _print.call_count)
91 |
92 | @mock.patch('unittest.TextTestRunner().run()')
93 | @mock.patch('unittest.TestLoader().discover()')
94 | @unittest.skip('TODO')
95 | def test_run_unittests(self, _test_discover, _test_run):
96 | _test_discover.return_value = ('test1', 'test2', 'test3')
97 | _test_run.return_value = mock.MagicMock()
98 |
99 | problems = C._run_unittests()
100 | self.assertEqual(0, len(problems))
101 |
102 | problems = C._run_unittests()
103 | self.assertEqual(0, len(problems))
104 |
--------------------------------------------------------------------------------