├── .gitignore ├── .taskcluster.yml ├── .travis.yml ├── .travis_rename_releases.sh └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /taskcluster-worker 2 | /taskcluster-worker.exe 3 | /.vagrant 4 | coverage.out 5 | /.bash_history 6 | /engines/qemu/test-image/tinycore-setup.tar.zst 7 | /engines/qemu/test-image/tinycore-worker.tar.zst 8 | /mock-config.yml 9 | /qemu-config.yml 10 | /vendor/*/ 11 | /config.env 12 | /examples/ubuntu-image/cache/ 13 | /examples/windows-image/cache/ 14 | -------------------------------------------------------------------------------- /.taskcluster.yml: -------------------------------------------------------------------------------- 1 | # The version is always required 2 | version: 0 3 | # Top level metadata is always required 4 | metadata: 5 | name: "TaskCluster Worker Tests" 6 | description: "These tests should ensure that any new commits against taskcluster worker codebase are tested across all supported worker types in gekco." 7 | owner: "{{ event.head.user.email }}" # the user who sent the pr/push e-mail will be inserted here 8 | source: "{{ event.head.repo.url }}" # the repo where the pr came from will be inserted here 9 | tasks: 10 | 11 | 12 | ########################################################## 13 | ##################### linux 64 build ##################### 14 | ########################################################## 15 | 16 | - provisionerId: "{{ taskcluster.docker.provisionerId }}" 17 | workerType: "{{ taskcluster.docker.workerType }}" 18 | metadata: 19 | name: "Build/test 64 bit taskcluster-worker on Debian GNU/Linux 8 (jessie)" 20 | description: "This builds the 64 bit linux version of taskcluster-worker" 21 | owner: "{{ event.head.user.email }}" # the user who sent the pr/push e-mail will be inserted here 22 | source: "{{ event.head.repo.url }}" # the repo where the pr came from will be inserted here 23 | extra: 24 | github: 25 | # Events that will trigger this task 26 | events: 27 | - push 28 | scopes: 29 | - secrets:get:repo:github.com/taskcluster/taskcluster-worker 30 | - docker-worker:cache:taskcluster-worker-checkout-1 31 | payload: 32 | features: 33 | taskclusterProxy: true 34 | maxRunTime: 3600 35 | image: golang:1.10-stretch 36 | command: 37 | - /bin/bash 38 | - -vxec 39 | - | 40 | curl -s http://taskcluster/secrets/v1/secret/repo:github.com/taskcluster/taskcluster-worker | sed -n 's/.*"b64_encoded_credentials_script": "\(.*\)".*/\1/p' | base64 -d > env_private.sh 41 | source env_private.sh 42 | go get -v "github.com/kardianos/govendor" 43 | mkdir -p "${GOPATH}/src/github.com/taskcluster" 44 | cd "${GOPATH}/src/github.com/taskcluster" 45 | if [ ! -d taskcluster-worker/.git ]; then rm -rf taskcluster-worker; git clone '{{ event.head.repo.url }}' 'taskcluster-worker'; fi 46 | cd 'taskcluster-worker' 47 | git fetch --tags 48 | git checkout -f '{{ event.head.sha }}' 49 | git clean -d -f -e '/vendor/*' -x 50 | # sometimes `govendor sync` fails first time, so run again, if fails first time 51 | govendor sync || govendor sync 52 | make reinstall check 53 | artifacts: 54 | public/build/taskcluster-worker-linux-amd64: 55 | path: "/go/bin/taskcluster-worker" 56 | expires: "{{ '2 weeks' | $fromNow }}" 57 | type: file 58 | cache: 59 | taskcluster-worker-checkout-1: /go/src 60 | 61 | 62 | ########################################################## 63 | #################### Windows 2012 R2 ##################### 64 | ########################################################## 65 | 66 | - provisionerId: aws-provisioner-v1 67 | workerType: win2012r2-cu 68 | metadata: 69 | name: "Build/test 64 bit taskcluster-worker on Windows 2012 R2" 70 | description: "This builds the 64 bit Windows version of taskcluster-worker" 71 | owner: "{{ event.head.user.email }}" # the user who sent the pr/push e-mail will be inserted here 72 | source: "{{ event.head.repo.url }}" # the repo where the pr came from will be inserted here 73 | extra: 74 | github: 75 | # Events that will trigger this task 76 | events: 77 | - push 78 | scopes: 79 | - generic-worker:cache:taskcluster-worker-checkout-1 80 | payload: 81 | maxRunTime: 3600 82 | artifacts: 83 | - expires: "{{ '2 weeks' | $fromNow }}" 84 | type: file 85 | path: public/build/taskcluster-worker-windows-amd64.exe 86 | command: 87 | - 'set GOROOT=%CD%\go1.10\go' 88 | - 'set GOPATH=%CD%\gopath' 89 | - 'set PATH=%CD%\git\cmd;%GOPATH%\bin;%GOROOT%\bin;%PATH%' 90 | - 'mkdir public\build' 91 | - 'mklink public\build\taskcluster-worker-windows-amd64.exe "%GOPATH%\bin\taskcluster-worker.exe"' 92 | - 'go get -v github.com/kardianos/govendor' 93 | - 'if not exist "%GOPATH%\src\github.com\taskcluster" mkdir "%GOPATH%\src\github.com\taskcluster"' 94 | - 'cd "%GOPATH%\src\github.com\taskcluster"' 95 | - 'git config --global core.autocrlf false' 96 | - 'if not exist taskcluster-worker git clone {{ event.head.repo.url }} taskcluster-worker' 97 | - 'cd taskcluster-worker' 98 | - 'git fetch --tags' 99 | - 'git checkout -f {{ event.head.sha }}' 100 | - 'git clean -d -f -e /vendor/* -x' 101 | - ':: sometimes `govendor sync` fails first time, so run again, if fails first time' 102 | - 'govendor sync || govendor sync' 103 | - 'C:\taskcluster-worker-test-creds.cmd' 104 | - '"C:\mozilla-build\msys\bin\bash.exe" --login -c "cd ${GOPATH}/src/github.com/taskcluster/taskcluster-worker && make reinstall check"' 105 | mounts: 106 | - cacheName: taskcluster-worker-checkout-1 107 | directory: gopath/src 108 | - content: 109 | url: https://storage.googleapis.com/golang/go1.10.windows-amd64.zip 110 | directory: go1.10 111 | format: zip 112 | - content: 113 | url: https://github.com/git-for-windows/git/releases/download/v2.16.2.windows.1/MinGit-2.16.2-64-bit.zip 114 | directory: git 115 | format: zip 116 | 117 | 118 | ########################################################## 119 | #################### OS X 10.10 build #################### 120 | ########################################################## 121 | 122 | - provisionerId: pmoore-manual 123 | workerType: mac-os-x 124 | metadata: 125 | name: "Build/test 64 bit taskcluster-worker on Mac OS X 10.10" 126 | description: "This builds the 64 bit OS X version of taskcluster-worker" 127 | owner: "{{ event.head.user.email }}" # the user who sent the pr/push e-mail will be inserted here 128 | source: "{{ event.head.repo.url }}" # the repo where the pr came from will be inserted here 129 | extra: 130 | github: 131 | # Events that will trigger this task 132 | events: 133 | - push 134 | scopes: 135 | - generic-worker:cache:taskcluster-worker-checkout-1 136 | payload: 137 | maxRunTime: 3600 138 | artifacts: 139 | - expires: "{{ '2 weeks' | $fromNow }}" 140 | type: file 141 | path: public/build/taskcluster-worker-darwin-amd64 142 | command: 143 | - - /bin/bash 144 | - -vxec 145 | - | 146 | export GOROOT="$(pwd)/go1.10/go" 147 | export GOPATH="$(pwd)/gopath" 148 | export PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" 149 | source "${TASKCLUSTER_CREDS_BOOTSTRAP}" 150 | go get -v "github.com/kardianos/govendor" 151 | mkdir -p "${GOPATH}/src/github.com/taskcluster" 152 | cd "${GOPATH}/src/github.com/taskcluster" 153 | if [ ! -d taskcluster-worker/.git ]; then rm -rf taskcluster-worker; git clone '{{ event.head.repo.url }}' 'taskcluster-worker'; fi 154 | cd 'taskcluster-worker' 155 | git fetch --tags 156 | git checkout -f '{{ event.head.sha }}' 157 | git clean -d -f -e '/vendor/*' -x 158 | # sometimes `govendor sync` fails first time, so run again, if fails first time 159 | govendor sync || govendor sync 160 | make reinstall check 161 | cd ../../../../.. 162 | mkdir -p public/build 163 | mv "${GOPATH}/bin/taskcluster-worker" public/build/taskcluster-worker-darwin-amd64 164 | mounts: 165 | - cacheName: taskcluster-worker-checkout-1 166 | directory: gopath/src 167 | - content: 168 | url: https://storage.googleapis.com/golang/go1.10.darwin-amd64.tar.gz 169 | directory: go1.10 170 | format: tar.gz 171 | 172 | ########################################################## 173 | #################### linux QEMU tests #################### 174 | ########################################################## 175 | 176 | 177 | - provisionerId: terraform-packet 178 | workerType: tc-worker-qemu-v1 179 | metadata: 180 | name: "Test taskcluster-worker QEMU engine on QEMU engine" 181 | description: "Builds and tests tc-worker with QEMU engine..." 182 | owner: "{{ event.head.user.email }}" # the user who sent the pr/push e-mail will be inserted here 183 | source: "{{ event.head.repo.url }}" # the repo where the pr came from will be inserted here 184 | extra: 185 | github: 186 | # Events that will trigger this task 187 | events: 188 | - push 189 | payload: 190 | image: 'https://s3-us-west-2.amazonaws.com/public-qemu-images/repository/github.com/taskcluster/taskcluster-worker/ubuntu-worker.tar.zst' 191 | command: 192 | - clone-and-exec.sh 193 | - make 194 | - tc-worker-env-tests 195 | maxRunTime: '30 min' 196 | env: 197 | REPOSITORY: '{{ event.head.repo.url }}' 198 | REVISION: '{{ event.head.sha }}' 199 | 200 | ########################################################## 201 | ##################### upload-docs ##################### 202 | ########################################################## 203 | 204 | - provisionerId: "{{ taskcluster.docker.provisionerId }}" 205 | workerType: "{{ taskcluster.docker.workerType }}" 206 | metadata: 207 | name: "Upload docs for taskcluster-worker" 208 | description: "Builds the 64 bit linux version of taskcluster-worker, generates and uploads docs" 209 | owner: "{{ event.head.user.email }}" # the user who sent the pr/push e-mail will be inserted here 210 | source: "{{ event.head.repo.url }}" # the repo where the pr came from will be inserted here 211 | extra: 212 | github: 213 | # Events that will trigger this task 214 | events: 215 | - push 216 | branches: 217 | - master 218 | scopes: 219 | - auth:aws-s3:read-write:taskcluster-raw-docs/taskcluster-worker/ 220 | - docker-worker:cache:taskcluster-worker-checkout-1 221 | payload: 222 | features: 223 | taskclusterProxy: true 224 | maxRunTime: 3600 225 | image: golang:1.10-stretch 226 | command: 227 | - /bin/bash 228 | - -vxec 229 | - | 230 | go get -v "github.com/kardianos/govendor" 231 | mkdir -p "${GOPATH}/src/github.com/taskcluster" 232 | cd "${GOPATH}/src/github.com/taskcluster" 233 | if [ ! -d taskcluster-worker/.git ]; then rm -rf taskcluster-worker; git clone '{{ event.head.repo.url }}' 'taskcluster-worker'; fi 234 | cd 'taskcluster-worker' 235 | git fetch --tags 236 | git checkout -f '{{ event.head.sha }}' 237 | git clean -d -f -e '/vendor/*' -x 238 | # sometimes `govendor sync` fails first time, so run again, if fails first time 239 | govendor sync || govendor sync 240 | # build tc-worker 241 | make install 242 | # generate docs 243 | taskcluster-worker schema config -o docs/config-schema.json 244 | taskcluster-worker schema payload examples/docker-config.yml -f json -o docs/docker-engine-payload-schema.json 245 | # install taskcluster-lib-docs 246 | apt-get update -y 247 | curl -sL https://deb.nodesource.com/setup_8.x | bash - 248 | apt-get install -y nodejs 249 | npm install -g taskcluster-lib-docs@3.3.2 250 | # upload docs 251 | export DOCS_PROJECT=taskcluster-worker DOCS_TIER=workers DOCS_FOLDER=docs DOCS_README=README.md 252 | # Hack because we forgot to wrap with node binary in taskcluster-lib-docs 253 | # See: https://github.com/taskcluster/taskcluster-lib-docs/pull/26 254 | nodejs `which upload-project-docs` 255 | cache: 256 | taskcluster-worker-checkout-1: /go/src 257 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: '1.10' 4 | 5 | env: 6 | - MY_GOOS=linux 7 | MY_GOARCH=amd64 8 | 9 | matrix: 10 | include: 11 | - os: osx 12 | env: 13 | - MY_GOOS=darwin 14 | - MY_GOARCH=amd64 15 | # - os: linux 16 | # env: 17 | # - MY_GOOS=windows 18 | # - MY_GOARCH=amd64 19 | # - os: linux 20 | # env: 21 | # - MY_GOOS=windows 22 | # - MY_GOARCH=386 23 | - os: linux 24 | env: 25 | - MY_GOOS=linux 26 | - MY_GOARCH=amd64 27 | 28 | before_install: 29 | - go get github.com/kardianos/govendor 30 | - govendor sync 31 | # exporting variables here is a hack to have them available in deploy section. 32 | # it works, just don't ask me why. -- wcosta 33 | - export GOOS=${MY_GOOS} 34 | - export GOARCH=${MY_GOARCH} 35 | - go version 36 | - go env 37 | # install cover tools if not on windows 38 | - "test $GOOS == windows || go get github.com/axw/gocov/gocov github.com/mattn/goveralls golang.org/x/tools/cmd/cover github.com/pierrre/gotestcover" 39 | 40 | install: 41 | - "echo No go get steps since we have vendored packages..." 42 | 43 | script: 44 | # note - below, || is used rather than && to ensure we only get zero exit code if all is ok 45 | - "test $GOOS != windows || make install" 46 | - "test $GOOS == windows || (make prechecks install check && ${GOPATH}/bin/gotestcover -v -coverprofile=coverage.report $(go list ./... | grep -v /vendor/) && go tool cover -func=coverage.report)" 47 | 48 | after_script: 49 | - "test $GOOS == windows || ($HOME/gopath/bin/goveralls -coverprofile=coverage.report -service=travis-ci)" 50 | 51 | before_deploy: 52 | - "source .travis_rename_releases.sh" 53 | 54 | deploy: 55 | api_key: 56 | secure: C+zuSru+yiMrFnyIllZukhTDt9L1krfbZQ9hNCFhjPtLkL+ytISlklw7A9p7NpSJvd7kGqYOjbFq3nZDsUnZG7S4QJ9u/U3xD2BcBrwVTnUWDZv/iit65m7xQSCdkLv+4myhaqH6dvcU6TNjZNrcEmBKxTBXgoSd4OpI89TifCLjcbcrKFrv8JXsTwsYH2N68AXV5CAxjl7X79hicZm82TIyisgcXKm96n7Y2kxPrVDvvQ508blD52EQsBszpzNh9F2THw3uglfeHFCYYUBzntSMbYKRtMTXNrtlvFpJUadIuzvB2DxBQIAoM3NDnEOVIbjLMjT2Fzvq5cVmBFyY7ymxEO7R9wC0IHaYruloXrpG1Ss+tK+Yom/hwXXeoHCWvFFDKzMWFaG/TYDKCmoSLRh65XAEWuueyo8g/ONybzpb377XnqMDa6imT0FtfDww3U+sqpV7DtpgilIRqodINVPXhS62sU648ZSi4DIY66DjUkX8MHL9fXKQkFHwTK6bS7sJctzjUCWkxUC0a7QpRndaSBdojK7/7kSC7yQJ+7j2jdZojAQcAz+is8erQlDHM9E8sCNTpyaAW75Koh9w/hdpR/U0Z8mKtdqwYlxpG7SqaKpJj2VfeBx2NsiK7oRPpmXU6Cye+jAKobUW96IKvHsCzdOvgOoPI4cX7qxDEEs= 57 | file: 58 | - "${RELEASE_FILE}" 59 | provider: releases 60 | skip_cleanup: true 61 | on: 62 | repo: taskcluster/taskcluster-worker 63 | tags: true 64 | 65 | # encrpyt channel name to get around issue 66 | # https://github.com/travis-ci/travis-ci/issues/1094 67 | notifications: 68 | irc: 69 | channels: 70 | # encrpyted string was "irc.mozilla.org#taskcluster-bots" 71 | - secure: "oTVLnjY0DGp95VEO6+PkEgRdg6CFQkLRBiEYbS/Ak0o4wdD4nrZ1XGpMcl9HwGsy6jZ6m0TfvT4Db2RXfmaA5r0rvt0miZkHTrgqrn5qy/XzDSykP8x724+/wan3iCDtqeMI+pSzZqyxMDwuY0M01NlEK4+6sZUL310LBGFS56S4qinAYrgAgDs8vRB+n6Nv3eA4uaxIXN7+6TmOq8cZyHUMO5J+qSe5TjcQt0mheb1gcUupvUhPOTk95oQVm7YH1iYIcKdWB4msrziRGo9zC1hnVupAaanD4avJRZ9i5EQDVl9YicgzhfotwW4Sh8SXVs1DIL20kqyw0AvB4wdNxo/Dgakdg6KjZvmNjEEbWWydWAZrbXz2e6Qa6HKy9R5F/eClfbHCC9c1ebi35mh9OLgnhPpjXTYH3eaiFjk6POaiBktzIUKTM8SaZpKeuyxZPe6dJDhsYPkDohhv2dMnGDxoJOHVoUza+d8NcuJ4wCD3amAigCPmG3mIKO1dv5GPVBnj9j08Iq3ign0qGPXeu+ouBQ6PtTM1wL3MWJktD0xsMAS+WY3NXFVDSBe593jweMBDCQAs0i8+5BgPOvlAh0vDzxvPpA85761lD8GGXIxRk7AdRja0h7cNLJP5zsxQxcI2MZPpTbrCpgeYJZI+r4Txw3+gLItpcNpmPWEnRFQ=" 72 | on_failure: always 73 | on_success: change 74 | template: 75 | - "%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}" 76 | - "Change view : %{compare_url}" 77 | - "Build details : %{build_url}" 78 | - "Commit message : %{commit_message}" 79 | -------------------------------------------------------------------------------- /.travis_rename_releases.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # The deploy is called per arch and os combination - so we only release one file here. 4 | # We just need to work out which file we built, rename it to something unique, and 5 | # set an environment variable for its location that we can use in .travis.yml for 6 | # publishing back to github. 7 | 8 | # linux, darwin: 9 | FILE_EXT="" 10 | [ "${GOOS}" == "windows" ] && FILE_EXT=".exe" 11 | 12 | # let's rename the release file because it has a 1:1 mapping with what it is called on 13 | # github releases, and therefore the name for each platform needs to be unique so that 14 | # they don't overwrite each other. Set a variable that can be used in .travis.yml 15 | export RELEASE_FILE="${TRAVIS_BUILD_DIR}/taskcluster-worker-${TRAVIS_TAG:1}-${GOOS}-${GOARCH}${FILE_EXT}" 16 | 17 | # when `go install` cross-copmiled, this will be in a subdir; otherwise, it will be in GOPATH/bin. So take 18 | # whichever one we can find 19 | for INSTALLED_FILE in "${GOPATH}/bin/${GOOS}_${GOARCH}/taskcluster-worker${FILE_EXT}" "${GOPATH}/bin/taskcluster-worker${FILE_EXT}" ; do 20 | [ -f "${INSTALLED_FILE}" ] && break 21 | done 22 | mv "${INSTALLED_FILE}" "${RELEASE_FILE}" 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ** NOTE ** 2 | 3 | This worker was never completed. The two common Taskcluster worker implementations are [docker-worker](https://github.com/taskcluster/docker-worker) and [generic-worker](https://docs.taskcluster.net/docs/reference/workers/generic-worker). 4 | --------------------------------------------------------------------------------