├── .codeclimate.yml ├── .docker-entrypoint.sh ├── .dockerignore ├── .editorconfig ├── .example.env ├── .github └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .snapcraft.yaml ├── .snapcraft └── daemon.start ├── .travis.yml ├── .vagrant-provision.sh ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── Vagrantfile ├── amqp_canceller.go ├── amqp_canceller_test.go ├── amqp_job.go ├── amqp_job_queue.go ├── amqp_job_test.go ├── amqp_log_writer.go ├── amqp_log_writer_factory.go ├── amqp_log_writer_test.go ├── amqp_test.go ├── backend ├── backend.go ├── docker.go ├── docker_test.go ├── ec2.go ├── fake.go ├── gce.go ├── gce_test.go ├── jupiterbrain.go ├── local.go ├── lxd.go ├── openstack.go ├── package.go ├── package_test.go ├── progresser.go ├── start_attributes.go ├── start_attributes_test.go ├── text_progresser.go └── text_progresser_test.go ├── build_script_generator.go ├── build_script_generator_test.go ├── build_trace_persister.go ├── canceller.go ├── canceller_test.go ├── cli.go ├── cli_test.go ├── cmd └── travis-worker │ ├── main.go │ └── main_test.go ├── config ├── config.go ├── config_test.go └── provider_config.go ├── context └── package.go ├── doc.go ├── errors └── errors.go ├── example-payload-premium.json ├── example-payload.json ├── file_job.go ├── file_job_queue.go ├── file_log_writer.go ├── go.mod ├── go.sum ├── help.go ├── http_job.go ├── http_job_queue.go ├── http_job_queue_test.go ├── http_job_test.go ├── http_log_part_sink.go ├── http_log_part_sink_test.go ├── http_log_writer.go ├── http_log_writer_test.go ├── image ├── api_selector.go ├── api_selector_test.go ├── doc.go ├── env_selector.go ├── env_selector_test.go ├── manager.go ├── params.go └── selector.go ├── job.go ├── job_queue.go ├── job_test.go ├── log_writer.go ├── log_writer_factory.go ├── metrics ├── memstats.go └── package.go ├── multi_source_job_queue.go ├── multi_source_job_queue_test.go ├── package.go ├── package_test.go ├── processor.go ├── processor_pool.go ├── processor_test.go ├── ratelimit ├── ratelimit.go └── ratelimit_test.go ├── remote └── package.go ├── remote_controller.go ├── script ├── clean ├── docker-push ├── drain-logs ├── fmtpolice ├── fold-coverprofiles ├── http-job-test ├── http-job-test-internal ├── lintall ├── list-packages ├── publish-example-payload ├── send-docker-hub-trigger ├── smoke └── smoke-docker ├── sentry_logrus_hook.go ├── ssh └── package.go ├── step_check_cancellation.go ├── step_download_trace.go ├── step_generate_script.go ├── step_open_log_writer.go ├── step_open_log_writer_test.go ├── step_run_script.go ├── step_send_received.go ├── step_sleep.go ├── step_start_instance.go ├── step_subscribe_cancellation.go ├── step_transform_build_json.go ├── step_transform_build_json_test.go ├── step_update_state.go ├── step_upload_script.go ├── step_write_worker_info.go ├── step_write_worker_info_test.go ├── systemd-wrapper ├── systemd.service ├── version.go └── winrm └── package.go /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | --- 2 | engines: 3 | gofmt: 4 | enabled: true 5 | golint: 6 | enabled: true 7 | govet: 8 | enabled: true 9 | fixme: 10 | enabled: true 11 | exclude_paths: 12 | - vendor/**/* 13 | ratings: 14 | paths: 15 | - src 16 | -------------------------------------------------------------------------------- /.docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | if [ "${1:0:1}" = '-' ]; then 5 | set -- travis-worker "$@" 6 | fi 7 | 8 | exec "$@" 9 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | [*.go] 2 | indent_style = tab 3 | indent_size = 4 4 | -------------------------------------------------------------------------------- /.example.env: -------------------------------------------------------------------------------- 1 | ## WORKER CONFIGURATION 2 | # To get an idea of what should be in this file, check out /etc/default/travis-worker on a Travis worker host. 3 | 4 | 5 | export TRAVIS_WORKER_PROVIDER_NAME='docker' 6 | export TRAVIS_WORKER_DOCKER_ENDPOINT=unix:///var/run/docker.sock 7 | #export TRAVIS_WORKER_DOCKER_CERT_PATH="/etc/docker/secret-stuff" 8 | export TRAVIS_WORKER_DOCKER_HOST='tcp://localhost:1234' 9 | export TRAVIS_WORKER_DOCKER_PRIVILEGED='false' 10 | export TRAVIS_WORKER_DOCKER_NATIVE=true 11 | 12 | 13 | ############# 14 | ### QUEUE ### 15 | ############# 16 | 17 | ### Valid values for TRAVIS_WORKER_QUEUE_TYPE: 'amqp', 'file', 'http' 18 | export TRAVIS_WORKER_QUEUE_TYPE='file' 19 | 20 | ### QUEUE: AMQP ### 21 | export TRAVIS_WORKER_AMQP_URI='amqp://fixme' 22 | 23 | ### QUEUE: FILE ### 24 | # TRAVIS_WORKER_QUEUE_NAME: This directory will be created when you run `make` 25 | export TRAVIS_WORKER_QUEUE_NAME='builds.ec2' 26 | 27 | 28 | ######################### 29 | ### EXTERNAL SERVICES ### 30 | ######################### 31 | 32 | ### Job board (not needed if you're using a local file-based queue) 33 | export TRAVIS_WORKER_JOB_BOARD_URL='https://travis-worker:API_KEY@job-board-staging.travis-ci.org' 34 | 35 | ### Travis build 36 | ### Note: TRAVIS_WORKER_BUILD_API_URI: This can be found in the env of the job board app, e.g.: heroku config:get JOB_BOARD_BUILD_API_ORG_URL -a job-board-staging 37 | export TRAVIS_WORKER_BUILD_API_URI='https://x:API_KEY@build-staging.travis-ci.org/script' 38 | 39 | ############# 40 | ### OTHER ### 41 | ############# 42 | 43 | 44 | # TRAVIS_WORKER_POOL_SIZE: This should be set to the number of available CPUs divided by the CPUs allocated per job. 45 | # For example, our EC2 production worker hosts have 32 CPUs available, and each job gets 2 CPUs, this number could be set to 16. 46 | # But because the host also needs some CPU time we set it a bit lower than that, to 13 47 | export TRAVIS_WORKER_POOL_SIZE=6 48 | 49 | # CPUs per job is controlled via `TRAVIS_WORKER_DOCKER_CPUS` and `TRAVIS_WORKER_DOCKER_CPU_SET_SIZE` 50 | # See: https://github.com/travis-ci/worker/blob/master/backend/docker.go#L46 51 | # CPUs defaults to 2, CPU set size defaults to number of available cpus on the machine 52 | 53 | export TRAVIS_WORKER_PPROF_PORT=6060 54 | 55 | export TRAVIS_WORKER_DEFAULT_DIST='trusty' 56 | export TRAVIS_WORKER_TRAVIS_SITE='org' 57 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## What is the problem that this PR is trying to fix? 2 | 3 | ## What approach did you choose and why? 4 | 5 | ## How can you test this? 6 | 7 | ## What feedback would you like, if any? 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | gin-bin 27 | *.log 28 | 29 | *.coverprofile 30 | *coverage.html 31 | 32 | .*env 33 | .build/ 34 | bin/travis-worker 35 | 36 | tmp/ 37 | VERSION 38 | VERSION_TAG 39 | VERSION_SHA1 40 | CURRENT_SHA1 41 | GIT_DESCRIPTION 42 | .vagrant/ 43 | 44 | *_rsa* 45 | vendor/** 46 | .deps-fetched 47 | !vendor/manifest 48 | /*-success 49 | .crossdeps 50 | build/ 51 | .write-test 52 | .envrc 53 | 54 | builds.*/ 55 | 56 | .vscode 57 | /.idea/ 58 | -------------------------------------------------------------------------------- /.snapcraft.yaml: -------------------------------------------------------------------------------- 1 | name: travis-worker 2 | base: core18 3 | version: git 4 | grade: stable 5 | summary: Travis CI worker 6 | description: |- 7 | This snap is for internal use by Travis to run worker nodes. 8 | confinement: strict 9 | 10 | apps: 11 | daemon: 12 | command: daemon.start 13 | daemon: simple 14 | plugs: 15 | - lxd 16 | - network 17 | - network-bind 18 | travis-worker: 19 | command: bin/travis-worker 20 | plugs: 21 | - lxd 22 | - network 23 | - network-bind 24 | 25 | parts: 26 | travis-worker: 27 | source: . 28 | build-packages: 29 | - gcc 30 | plugin: go 31 | go-packages: 32 | - github.com/travis-ci/worker/cmd/travis-worker 33 | go-importpath: github.com/travis-ci/worker 34 | go-channel: 1.21/stable 35 | prime: 36 | - bin/travis-worker 37 | override-build: |- 38 | set -eux 39 | 40 | cd ${SNAPCRAFT_PROJECT_DIR} 41 | VERSION="$(git describe --always --dirty --tags 2>/dev/null)" 42 | REVISION="$(git rev-parse HEAD)" 43 | REVISION_URL="https://github.com/travis-ci/worker/tree/${REVISION}" 44 | GENERATED="$(date -u +'%Y-%m-%dT%H:%M:%S%z')" 45 | COPYRIGHT="$(grep -i ^copyright LICENSE | sed 's/^[Cc]opyright //')" 46 | 47 | cd ${SNAPCRAFT_PART_SRC} 48 | sed -i "s#VersionString =.*#VersionString = \"${VERSION}\"#g" version.go 49 | sed -i "s#RevisionString =.*#RevisionString = \"${REVISION}\"#g" version.go 50 | sed -i "s#RevisionURLString =.*#RevisionURLString = \"${REVISION_URL}\"#g" version.go 51 | sed -i "s#GeneratedString =.*#GeneratedString = \"${GENERATED}\"#g" version.go 52 | sed -i "s#CopyrightString =.*#CopyrightString = \"${COPYRIGHT}\"#g" version.go 53 | 54 | set +ex 55 | snapcraftctl build 56 | set -ex 57 | 58 | wrappers: 59 | plugin: dump 60 | source: .snapcraft/ 61 | -------------------------------------------------------------------------------- /.snapcraft/daemon.start: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | export LXD_DIR=/var/snap/lxd/common/lxd/ 3 | 4 | # Source the worker configuration 5 | if [ ! -e "${SNAP_COMMON}/worker.env" ]; then 6 | echo "travis-worker isn't configured, please write its configuration to ${SNAP_COMMON}/worker.env" >&2 7 | exit 1 8 | fi 9 | . ${SNAP_COMMON}/worker.env 10 | 11 | exec travis-worker 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 1.22.x 4 | 5 | dist: focal 6 | 7 | git: 8 | depth: 10 9 | 10 | # Build master and PRs which merge into those branches 11 | # We don't automatically build other branches when they're pushed; create a PR to cause the CI to run. 12 | branches: 13 | only: 14 | - master 15 | 16 | cache: 17 | directories: 18 | - vendor 19 | - $HOME/.cache/go-build 20 | - $HOME/gopath/bin 21 | - $HOME/gopath/pkg/mod 22 | 23 | services: 24 | - rabbitmq 25 | - docker 26 | - redis 27 | 28 | env: 29 | global: 30 | - AMQP_URI="amqp://" 31 | - GOPATH="$HOME/gopath" 32 | - PATH="bin:$HOME/gopath/bin:$HOME/bin:$PATH" 33 | - CHECKOUT_ROOT="$HOME/gopath/src/github.com/travis-ci/worker" 34 | - GO15VENDOREXPERIMENT='1' 35 | - REDIS_URL="redis://" 36 | 37 | stages: 38 | - name: test 39 | if: (type = push OR type = pull_request) AND branch = master 40 | 41 | before_cache: 42 | - make clean 43 | 44 | jobs: 45 | include: 46 | - stage: test 47 | name: lint 48 | 49 | script: 50 | - make deps 51 | - make lintall 52 | 53 | - stage: test 54 | name: linux 55 | 56 | script: 57 | - make deps 58 | - GO111MODULE=on make build 59 | - mkdir -p build/linux/amd64 60 | - cp ${GOPATH%%:*}/bin/travis-worker build/linux/amd64 61 | - make test-no-cover 62 | - make test-cover 63 | - make smoke 64 | 65 | addons: 66 | apt: 67 | packages: 68 | - rabbitmq-server 69 | artifacts: 70 | paths: 71 | - ./build/linux/amd64/travis-worker 72 | target_paths: 73 | - travis-ci/worker/$TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER 74 | - travis-ci/worker/$(git describe --always --dirty --tags) 75 | - travis-ci/worker/$TRAVIS_BRANCH 76 | 77 | - stage: test 78 | name: crossbuild 79 | 80 | script: 81 | - go mod vendor 82 | - GO111MODULE=on make build/darwin/amd64/travis-worker 83 | 84 | addons: 85 | artifacts: 86 | paths: 87 | - ./build/darwin/amd64/travis-worker 88 | target_paths: 89 | - travis-ci/worker/$TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER 90 | - travis-ci/worker/$(git describe --always --dirty --tags) 91 | - travis-ci/worker/$TRAVIS_BRANCH 92 | 93 | - stage: test 94 | name: docker 95 | if: type != 'pull_request' && env(VAULT_PASS) is present && env(VAULT_USERNAME) is present 96 | 97 | script: 98 | - vault login --no-print -method=userpass username=$VAULT_USERNAME password=$VAULT_PASS 99 | - vault kv get -field=secret gcp/gcr-sa-key > /tmp/gcr_key.json; 100 | - gcloud -q auth activate-service-account --key-file /tmp/gcr_key.json; 101 | - gcloud -q config set project ${GCE_PROJECT} 102 | - gcloud auth configure-docker 103 | - make docker-build 104 | - make docker-push 105 | 106 | addons: 107 | snaps: 108 | # google-cloud-sdk is available on focal 109 | # - name: google-cloud-sdk 110 | - name: vault 111 | apt: 112 | update: true 113 | packages: 114 | - docker-ce 115 | 116 | - stage: test 117 | name: http-job-test 118 | 119 | script: 120 | - GO111MODULE=on make build 121 | - mkdir -p build/linux/amd64 122 | - cp ${GOPATH%%:*}/bin/travis-worker build/linux/amd64 123 | - make http-job-test 124 | -------------------------------------------------------------------------------- /.vagrant-provision.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | 4 | main() { 5 | set -o xtrace 6 | 7 | export DEBIAN_FRONTEND=noninteractive 8 | 9 | sudo apt update -y 10 | sudo apt install -y \ 11 | build-essential \ 12 | git \ 13 | redis-tools 14 | 15 | if ! gimme version; then 16 | curl -sSL \ 17 | -o /usr/local/bin/gimme \ 18 | https://raw.githubusercontent.com/travis-ci/gimme/master/gimme 19 | chmod +x /usr/local/bin/gimme 20 | fi 21 | 22 | sudo -u vagrant HOME=/home/vagrant bash -c 'gimme 1.8.3' 23 | 24 | if ! docker version; then 25 | curl -sSL https://get.docker.io | sudo bash 26 | fi 27 | 28 | docker run -d -p 5672:5672 --name rabbitmq rabbitmq:3-management 29 | docker run -d -p 6379:6379 --name redis redis 30 | 31 | cat >/home/vagrant/.bash_profile < 3 | 4 | COPY . /go/src/github.com/travis-ci/worker 5 | WORKDIR /go/src/github.com/travis-ci/worker 6 | ENV CGO_ENABLED 0 7 | RUN make build 8 | 9 | FROM alpine:latest 10 | RUN apk --no-cache add ca-certificates curl bash 11 | 12 | COPY --from=builder /go/bin/travis-worker /usr/local/bin/travis-worker 13 | COPY --from=builder /go/src/github.com/travis-ci/worker/systemd.service /app/systemd.service 14 | COPY --from=builder /go/src/github.com/travis-ci/worker/systemd-wrapper /app/systemd-wrapper 15 | COPY --from=builder /go/src/github.com/travis-ci/worker/.docker-entrypoint.sh /docker-entrypoint.sh 16 | 17 | VOLUME ["/var/tmp"] 18 | STOPSIGNAL SIGINT 19 | 20 | ENTRYPOINT ["/docker-entrypoint.sh"] 21 | CMD ["/usr/local/bin/travis-worker"] 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2018 Travis CI GmbH 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL := bash 2 | 3 | PACKAGE_CHECKOUT := $(shell echo ${PWD}) 4 | PACKAGE := github.com/travis-ci/worker 5 | ALL_PACKAGES := $(PACKAGE) $(shell script/list-packages) $(PACKAGE)/cmd/... 6 | 7 | VERSION_VAR := $(PACKAGE).VersionString 8 | VERSION_VALUE ?= $(shell git describe --always --dirty --tags 2>/dev/null) 9 | REV_VAR := $(PACKAGE).RevisionString 10 | REV_VALUE ?= $(shell git rev-parse HEAD 2>/dev/null || echo "'???'") 11 | REV_URL_VAR := $(PACKAGE).RevisionURLString 12 | REV_URL_VALUE ?= https://github.com/travis-ci/worker/tree/$(shell git rev-parse HEAD 2>/dev/null || echo "'???'") 13 | GENERATED_VAR := $(PACKAGE).GeneratedString 14 | GENERATED_VALUE ?= $(shell date -u +'%Y-%m-%dT%H:%M:%S%z') 15 | COPYRIGHT_VAR := $(PACKAGE).CopyrightString 16 | COPYRIGHT_VALUE ?= $(shell grep -i ^copyright LICENSE | sed 's/^[Cc]opyright //') 17 | DOCKER_IMAGE_REPO ?= gcr.io/travis-ci-prod-services-1/worker 18 | DOCKER_DEST ?= $(DOCKER_IMAGE_REPO):$(VERSION_VALUE) 19 | 20 | DOCKER ?= docker 21 | GO ?= go 22 | GOPATH := $(shell echo $${GOPATH%%:*}) 23 | GOPATH_BIN := $(GOPATH)/bin 24 | GOBUILD_LDFLAGS ?= \ 25 | -X '$(VERSION_VAR)=$(VERSION_VALUE)' \ 26 | -X '$(REV_VAR)=$(REV_VALUE)' \ 27 | -X '$(REV_URL_VAR)=$(REV_URL_VALUE)' \ 28 | -X '$(GENERATED_VAR)=$(GENERATED_VALUE)' \ 29 | -X '$(COPYRIGHT_VAR)=$(COPYRIGHT_VALUE)' 30 | 31 | export GO15VENDOREXPERIMENT 32 | export DOCKER_DEST 33 | 34 | COVERPROFILES := \ 35 | root-coverage.coverprofile \ 36 | backend-coverage.coverprofile \ 37 | config-coverage.coverprofile \ 38 | image-coverage.coverprofile 39 | CROSSBUILD_BINARIES := \ 40 | build/darwin/amd64/travis-worker \ 41 | build/linux/amd64/travis-worker 42 | 43 | SHFMT_URL := https://github.com/mvdan/sh/releases/download/v2.5.0/shfmt_v2.5.0_linux_amd64 44 | 45 | %-coverage.coverprofile: 46 | $(GO) test -covermode=count -coverprofile=$@ \ 47 | -tags netgo -ldflags "$(GOBUILD_LDFLAGS)" \ 48 | $(PACKAGE)/$(subst -,/,$(subst root,,$(subst -coverage.coverprofile,,$@))) 49 | 50 | .PHONY: % 51 | %: 52 | ./script/$@ 53 | 54 | .PHONY: all 55 | all: clean test 56 | 57 | .PHONY: test 58 | test: .deps-fetched lintall build fmtpolice test-no-cover test-cover 59 | 60 | .PHONY: test-cover 61 | test-cover: coverage.html 62 | 63 | .PHONY: test-no-cover 64 | test-no-cover: 65 | $(GO) test -race -tags netgo -ldflags "$(GOBUILD_LDFLAGS)" $(ALL_PACKAGES) 66 | 67 | coverage.html: coverage.coverprofile 68 | $(GO) tool cover -html=$^ -o $@ 69 | 70 | coverage.coverprofile: $(COVERPROFILES) 71 | ./script/fold-coverprofiles $^ > $@ 72 | $(GO) tool cover -func=$@ 73 | 74 | .PHONY: build 75 | build: .deps-fetched 76 | $(GO) install -tags netgo -ldflags "$(GOBUILD_LDFLAGS)" $(ALL_PACKAGES) 77 | 78 | .PHONY: crossbuild 79 | crossbuild: .deps-fetched $(CROSSBUILD_BINARIES) 80 | 81 | .PHONY: docker-build 82 | docker-build: 83 | $(DOCKER) build -t $(DOCKER_DEST) . 84 | 85 | build/darwin/amd64/travis-worker: 86 | GOARCH=amd64 GOOS=darwin CGO_ENABLED=0 \ 87 | $(GO) build -o build/darwin/amd64/travis-worker \ 88 | -ldflags "$(GOBUILD_LDFLAGS)" $(PACKAGE)/cmd/travis-worker 89 | 90 | build/linux/amd64/travis-worker: 91 | GOARCH=amd64 GOOS=linux CGO_ENABLED=0 \ 92 | $(GO) build -o build/linux/amd64/travis-worker \ 93 | -ldflags "$(GOBUILD_LDFLAGS)" $(PACKAGE)/cmd/travis-worker 94 | 95 | .PHONY: distclean 96 | distclean: clean 97 | rm -rf .deps-fetched build/ 98 | 99 | .PHONY: deps 100 | deps: .ensure-shfmt .ensure-golangci-lint .deps-fetched 101 | 102 | .deps-fetched: go.mod 103 | GO111MODULE=on $(GO) mod download 104 | GO111MODULE=on $(GO) mod vendor 105 | touch $@ 106 | 107 | .PHONY: .ensure-shfmt 108 | .ensure-shfmt: 109 | if ! shfmt -version 2>/dev/null; then \ 110 | curl -o $(GOPATH_BIN)/shfmt -sSL $(SHFMT_URL); \ 111 | chmod +x $(GOPATH_BIN)/shfmt; \ 112 | shfmt -version; \ 113 | fi 114 | 115 | .PHONY: .ensure-golangci-lint 116 | .ensure-golangci-lint: 117 | curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(HOME)/bin 118 | if ! command -v $(go env GOPATH)/bin/golangci-lint &>/dev/null; then \ 119 | $(HOME)/bin/golangci-lint --version; \ 120 | fi 121 | 122 | .PHONY: annotations 123 | annotations: 124 | @git grep -E '(TODO|FIXME|XXX):' | grep -v -E 'Makefile|vendor/' 125 | 126 | $(DOCKER_ENV_FILE): 127 | env | grep ^DOCKER | sort >$@ 128 | echo 'DOCKER_DEST=$(DOCKER_DEST)' >>$@ 129 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | Vagrant.configure('2') do |config| 4 | config.vm.box = 'ubuntu/trusty64' 5 | config.vm.synced_folder( 6 | '.', '/home/vagrant/go/src/github.com/travis-ci/worker' 7 | ) 8 | config.vm.provision 'shell', path: '.vagrant-provision.sh' 9 | 10 | config.vm.provider 'virtualbox' do |v| 11 | v.memory = 1024 12 | v.cpus = 2 13 | end 14 | 15 | config.vm.provider 'aws' do |aws, override| 16 | aws.access_key_id = ENV['AWS_ACCESS_KEY'] 17 | aws.secret_access_key = ENV['AWS_SECRET_KEY'] 18 | aws.keypair_name = ENV['AWS_KEYPAIR'] 19 | 20 | aws.ami = 'ami-7747d01e' 21 | aws.instance_type = 'c3.4xlarge' 22 | override.ssh.username = 'vagrant' 23 | override.ssh.private_key_path = ENV['AWS_SSH_PRIVATE_KEY'] 24 | aws.block_device_mapping = [ 25 | { 26 | 'DeviceName' => '/dev/sda1', 27 | 'Ebs.VolumeSize' => 100 28 | } 29 | ] 30 | aws.tags = { 31 | 'Name' => 'travis-worker-vagrant-trusty' 32 | } 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /amqp_canceller.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | gocontext "context" 8 | 9 | amqp "github.com/rabbitmq/amqp091-go" 10 | "github.com/sirupsen/logrus" 11 | "github.com/travis-ci/worker/context" 12 | ) 13 | 14 | type cancelCommand struct { 15 | Type string `json:"type"` 16 | JobID uint64 `json:"job_id"` 17 | Source string `json:"source"` 18 | Reason string `json:"reason"` 19 | } 20 | 21 | // AMQPCanceller is responsible for listening to a command queue on AMQP and 22 | // dispatching the commands to the right place. Currently the only valid command 23 | // is the 'cancel job' command. 24 | type AMQPCanceller struct { 25 | conn *amqp.Connection 26 | ctx gocontext.Context 27 | cancellationBroadcaster *CancellationBroadcaster 28 | } 29 | 30 | // NewAMQPCanceller creates a new AMQPCanceller. No network traffic 31 | // occurs until you call Run() 32 | func NewAMQPCanceller(ctx gocontext.Context, conn *amqp.Connection, cancellationBroadcaster *CancellationBroadcaster) *AMQPCanceller { 33 | ctx = context.FromComponent(ctx, "canceller") 34 | 35 | return &AMQPCanceller{ 36 | ctx: ctx, 37 | conn: conn, 38 | 39 | cancellationBroadcaster: cancellationBroadcaster, 40 | } 41 | } 42 | 43 | // Run will make the AMQPCanceller listen to the worker command queue and 44 | // start dispatching any incoming commands. 45 | func (d *AMQPCanceller) Run() { 46 | amqpChan, err := d.conn.Channel() 47 | logger := context.LoggerFromContext(d.ctx).WithFields(logrus.Fields{ 48 | "self": "amqp_canceller", 49 | "inst": fmt.Sprintf("%p", d), 50 | }) 51 | if err != nil { 52 | logger.WithField("err", err).Error("couldn't open channel") 53 | return 54 | } 55 | defer amqpChan.Close() 56 | 57 | err = amqpChan.Qos(1, 0, false) 58 | if err != nil { 59 | logger.WithField("err", err).Error("couldn't set prefetch") 60 | return 61 | } 62 | 63 | err = amqpChan.ExchangeDeclare("worker.commands", "fanout", false, false, false, false, nil) 64 | if err != nil { 65 | logger.WithField("err", err).Error("couldn't declare exchange") 66 | return 67 | } 68 | 69 | queue, err := amqpChan.QueueDeclare("", true, false, true, false, nil) 70 | if err != nil { 71 | logger.WithField("err", err).Error("couldn't declare queue") 72 | return 73 | } 74 | 75 | err = amqpChan.QueueBind(queue.Name, "", "worker.commands", false, nil) 76 | if err != nil { 77 | logger.WithField("err", err).Error("couldn't bind queue to exchange") 78 | return 79 | } 80 | 81 | deliveries, err := amqpChan.Consume(queue.Name, "commands", false, true, false, false, nil) 82 | if err != nil { 83 | logger.WithField("err", err).Error("couldn't consume queue") 84 | return 85 | } 86 | 87 | for delivery := range deliveries { 88 | err := d.processCommand(delivery) 89 | if err != nil { 90 | logger.WithField("err", err).WithField("delivery", delivery).Error("couldn't process delivery") 91 | } 92 | 93 | err = delivery.Ack(false) 94 | if err != nil { 95 | logger.WithField("err", err).WithField("delivery", delivery).Error("couldn't ack delivery") 96 | } 97 | } 98 | } 99 | 100 | func (d *AMQPCanceller) processCommand(delivery amqp.Delivery) error { 101 | command := &cancelCommand{} 102 | logger := context.LoggerFromContext(d.ctx).WithFields(logrus.Fields{ 103 | "self": "amqp_canceller", 104 | "inst": fmt.Sprintf("%p", d), 105 | }) 106 | err := json.Unmarshal(delivery.Body, command) 107 | if err != nil { 108 | logger.WithField("err", err).Error("unable to parse JSON") 109 | return err 110 | } 111 | 112 | if command.Type != "cancel_job" { 113 | logger.WithField("command", command.Type).Error("unknown worker command") 114 | return nil 115 | } 116 | 117 | d.cancellationBroadcaster.Broadcast(CancellationCommand{JobID: command.JobID, Reason: command.Reason}) 118 | 119 | return nil 120 | } 121 | 122 | func tryClose(ch chan<- struct{}) (closedNow bool) { 123 | closedNow = true 124 | defer func() { 125 | if x := recover(); x != nil { 126 | closedNow = false 127 | } 128 | }() 129 | 130 | close(ch) 131 | 132 | return 133 | } 134 | -------------------------------------------------------------------------------- /amqp_canceller_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | gocontext "context" 8 | 9 | "github.com/pborman/uuid" 10 | amqp "github.com/rabbitmq/amqp091-go" 11 | "github.com/travis-ci/worker/context" 12 | ) 13 | 14 | func newTestAMQPCanceller(t *testing.T, cancellationBroadcaster *CancellationBroadcaster) *AMQPCanceller { 15 | amqpConn, _ := setupAMQPConn(t) 16 | 17 | uuid := uuid.NewRandom() 18 | ctx := context.FromUUID(gocontext.TODO(), uuid.String()) 19 | 20 | return NewAMQPCanceller(ctx, amqpConn, cancellationBroadcaster) 21 | } 22 | 23 | func TestNewAMQPCanceller(t *testing.T) { 24 | if newTestAMQPCanceller(t, NewCancellationBroadcaster()) == nil { 25 | t.Fail() 26 | } 27 | } 28 | 29 | func TestAMQPCanceller_Run(t *testing.T) { 30 | cancellationBroadcaster := NewCancellationBroadcaster() 31 | canceller := newTestAMQPCanceller(t, cancellationBroadcaster) 32 | 33 | errChan := make(chan interface{}) 34 | 35 | go func() { 36 | defer func() { 37 | errChan <- recover() 38 | }() 39 | canceller.Run() 40 | }() 41 | 42 | select { 43 | case <-time.After(3 * time.Second): 44 | case err := <-errChan: 45 | if err != nil { 46 | t.Error(err) 47 | } 48 | } 49 | } 50 | 51 | func TestAMQPCanceller_processCommand(t *testing.T) { 52 | cancellationBroadcaster := NewCancellationBroadcaster() 53 | canceller := newTestAMQPCanceller(t, cancellationBroadcaster) 54 | 55 | err := canceller.processCommand(amqp.Delivery{Body: []byte("{")}) 56 | if err == nil { 57 | t.Fatalf("no JSON parsing error returned") 58 | } 59 | 60 | errChan := make(chan error) 61 | 62 | go func() { 63 | delivery := amqp.Delivery{Body: []byte(`{ 64 | "type": "cancel_job", 65 | "job_id": 123, 66 | "source": "tests" 67 | }`), 68 | } 69 | errChan <- canceller.processCommand(delivery) 70 | }() 71 | 72 | select { 73 | case <-time.After(3 * time.Second): 74 | t.Fatalf("hit potential deadlock condition") 75 | case err := <-errChan: 76 | if err != nil { 77 | t.Error(err) 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /amqp_job_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "encoding/json" 5 | "strings" 6 | "testing" 7 | "time" 8 | 9 | gocontext "context" 10 | 11 | "github.com/bitly/go-simplejson" 12 | amqp "github.com/rabbitmq/amqp091-go" 13 | "github.com/stretchr/testify/assert" 14 | "github.com/travis-ci/worker/backend" 15 | ) 16 | 17 | type fakeAMQPAcknowledger struct { 18 | lastAckTag uint64 19 | lastAckMult bool 20 | lastNackTag uint64 21 | lastNackMult bool 22 | lastNackReq bool 23 | lastRejectTag uint64 24 | lastRejectReq bool 25 | } 26 | 27 | func (a *fakeAMQPAcknowledger) Ack(tag uint64, mult bool) error { 28 | a.lastAckTag = tag 29 | a.lastAckMult = mult 30 | return nil 31 | } 32 | 33 | func (a *fakeAMQPAcknowledger) Nack(tag uint64, mult bool, req bool) error { 34 | a.lastNackTag = tag 35 | a.lastNackMult = mult 36 | a.lastNackReq = req 37 | return nil 38 | } 39 | 40 | func (a *fakeAMQPAcknowledger) Reject(tag uint64, req bool) error { 41 | a.lastRejectTag = tag 42 | a.lastRejectReq = req 43 | return nil 44 | } 45 | 46 | func newTestAMQPJob(t *testing.T) *amqpJob { 47 | amqpConn, logChan := setupAMQPConn(t) 48 | 49 | payload := &JobPayload{ 50 | Type: "job:test", 51 | Job: JobJobPayload{ 52 | ID: uint64(123), 53 | Number: "1", 54 | QueuedAt: new(time.Time), 55 | }, 56 | Build: BuildPayload{ 57 | ID: uint64(456), 58 | Number: "1", 59 | }, 60 | UUID: "870f986d-a88f-4801-86cc-3d2dbc6c80da", 61 | Config: map[string]interface{}{}, 62 | Timeouts: TimeoutsPayload{ 63 | HardLimit: uint64(9000), 64 | LogSilence: uint64(8001), 65 | }, 66 | } 67 | startAttributes := &backend.StartAttributes{ 68 | Language: "go", 69 | Dist: "trusty", 70 | } 71 | 72 | body, err := json.Marshal(payload) 73 | if err != nil { 74 | t.Error(err) 75 | } 76 | 77 | delivery := amqp.Delivery{ 78 | Body: body, 79 | Acknowledger: &fakeAMQPAcknowledger{}, 80 | } 81 | 82 | rawPayload, err := simplejson.NewJson(body) 83 | if err != nil { 84 | t.Error(err) 85 | } 86 | 87 | stateUpdatePool := newStateUpdatePool(amqpConn, 1) 88 | 89 | return &amqpJob{ 90 | conn: amqpConn, 91 | logWriterChan: logChan, 92 | stateUpdatePool: stateUpdatePool, 93 | delivery: delivery, 94 | payload: payload, 95 | rawPayload: rawPayload, 96 | startAttributes: startAttributes, 97 | received: time.Now().Add(-3 * time.Minute), 98 | started: time.Now().Add(-2 * time.Minute), 99 | finished: time.Now().Add(-3 * time.Second), 100 | } 101 | } 102 | 103 | func TestAMQPJob(t *testing.T) { 104 | job := newTestAMQPJob(t) 105 | 106 | if job.Payload() == nil { 107 | t.Fatalf("payload not set") 108 | } 109 | 110 | if job.RawPayload() == nil { 111 | t.Fatalf("raw payload not set") 112 | } 113 | 114 | if job.StartAttributes() == nil { 115 | t.Fatalf("start attributes not set") 116 | } 117 | 118 | if job.GoString() == "" { 119 | t.Fatalf("go string is empty") 120 | } 121 | } 122 | 123 | func TestAMQPJob_GoString(t *testing.T) { 124 | job := newTestAMQPJob(t) 125 | 126 | str := job.GoString() 127 | 128 | if !strings.HasPrefix(str, "&amqpJob{") && !strings.HasSuffix(str, "}") { 129 | t.Fatalf("go string has unexpected format: %q", str) 130 | } 131 | } 132 | 133 | func TestAMQPJob_Error(t *testing.T) { 134 | job := newTestAMQPJob(t) 135 | 136 | err := job.Error(gocontext.TODO(), "wat") 137 | if err != nil { 138 | t.Error(err) 139 | } 140 | } 141 | 142 | func TestAMQPJob_Requeue(t *testing.T) { 143 | job := newTestAMQPJob(t) 144 | ctx := gocontext.TODO() 145 | 146 | err := job.Requeue(ctx) 147 | if err != nil { 148 | t.Error(err) 149 | } 150 | 151 | acker := job.delivery.Acknowledger.(*fakeAMQPAcknowledger) 152 | if acker.lastAckMult { 153 | t.Fatalf("last ack multiple was true") 154 | } 155 | } 156 | 157 | func TestAMQPJob_Received(t *testing.T) { 158 | job := newTestAMQPJob(t) 159 | 160 | err := job.Received(gocontext.TODO()) 161 | if err != nil { 162 | t.Error(err) 163 | } 164 | } 165 | 166 | func TestAMQPJob_Started(t *testing.T) { 167 | job := newTestAMQPJob(t) 168 | 169 | err := job.Started(gocontext.TODO()) 170 | if err != nil { 171 | t.Error(err) 172 | } 173 | } 174 | 175 | func TestAMQPJob_Finish(t *testing.T) { 176 | job := newTestAMQPJob(t) 177 | ctx := gocontext.TODO() 178 | 179 | err := job.Finish(ctx, FinishStatePassed) 180 | if err != nil { 181 | t.Error(err) 182 | } 183 | } 184 | 185 | func TestAMQPJob_createStateUpdateBody(t *testing.T) { 186 | job := newTestAMQPJob(t) 187 | body := job.createStateUpdateBody(gocontext.TODO(), "foo") 188 | 189 | assert.Equal(t, "foo", body["state"]) 190 | 191 | for _, key := range []string{ 192 | "finished_at", 193 | "id", 194 | "meta", 195 | "queued_at", 196 | "received_at", 197 | "started_at", 198 | } { 199 | assert.Contains(t, body, key) 200 | } 201 | 202 | assert.Equal(t, body["meta"].(map[string]interface{})["instance_id"], nil) 203 | 204 | job.received = time.Time{} 205 | assert.NotContains(t, job.createStateUpdateBody(gocontext.TODO(), "foo"), "received_at") 206 | 207 | job.Payload().Job.QueuedAt = nil 208 | assert.NotContains(t, job.createStateUpdateBody(gocontext.TODO(), "foo"), "queued_at") 209 | 210 | job.started = time.Time{} 211 | assert.NotContains(t, job.createStateUpdateBody(gocontext.TODO(), "foo"), "started_at") 212 | 213 | job.finished = time.Time{} 214 | assert.NotContains(t, job.createStateUpdateBody(gocontext.TODO(), "foo"), "finished_at") 215 | } 216 | -------------------------------------------------------------------------------- /amqp_log_writer_factory.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | gocontext "context" 5 | "time" 6 | 7 | amqp "github.com/rabbitmq/amqp091-go" 8 | ) 9 | 10 | type AMQPLogWriterFactory struct { 11 | conn *amqp.Connection 12 | withLogSharding bool 13 | logWriterChan *amqp.Channel 14 | } 15 | 16 | func NewAMQPLogWriterFactory(conn *amqp.Connection, sharded bool) (*AMQPLogWriterFactory, error) { 17 | channel, err := conn.Channel() 18 | if err != nil { 19 | return nil, err 20 | } 21 | 22 | if sharded { 23 | // This exchange should be declared as sharded using a policy that matches its name. 24 | err = channel.ExchangeDeclare("reporting.jobs.logs_sharded", "x-modulus-hash", true, false, false, false, nil) 25 | if err != nil { 26 | return nil, err 27 | } 28 | } else { 29 | _, err = channel.QueueDeclare("reporting.jobs.logs", true, false, false, false, nil) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | err = channel.QueueBind("reporting.jobs.logs", "reporting.jobs.logs", "reporting", false, nil) 35 | if err != nil { 36 | return nil, err 37 | } 38 | } 39 | 40 | return &AMQPLogWriterFactory{ 41 | conn: conn, 42 | withLogSharding: sharded, 43 | logWriterChan: channel, 44 | }, nil 45 | } 46 | 47 | func (l *AMQPLogWriterFactory) LogWriter(ctx gocontext.Context, defaultLogTimeout time.Duration, job Job) (LogWriter, error) { 48 | logTimeout := time.Duration(job.Payload().Timeouts.LogSilence) * time.Second 49 | if logTimeout == 0 { 50 | logTimeout = defaultLogTimeout 51 | } 52 | 53 | return newAMQPLogWriter(ctx, l.logWriterChan, job.Payload().Job.ID, logTimeout, l.withLogSharding) 54 | } 55 | 56 | func (l *AMQPLogWriterFactory) Cleanup() error { 57 | l.logWriterChan.Close() 58 | return l.conn.Close() 59 | } 60 | -------------------------------------------------------------------------------- /amqp_log_writer_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "testing" 8 | "time" 9 | 10 | "github.com/pborman/uuid" 11 | workerctx "github.com/travis-ci/worker/context" 12 | ) 13 | 14 | func TestAMQPLogWriterWrite(t *testing.T) { 15 | amqpConn, amqpChan := setupAMQPConn(t) 16 | defer amqpConn.Close() 17 | defer amqpChan.Close() 18 | 19 | uuid := uuid.NewRandom() 20 | ctx := workerctx.FromUUID(context.TODO(), uuid.String()) 21 | 22 | logWriter, err := newAMQPLogWriter(ctx, amqpChan, 4, time.Hour, false) 23 | if err != nil { 24 | t.Fatal(err) 25 | } 26 | logWriter.SetMaxLogLength(1000) 27 | 28 | _, err = fmt.Fprintf(logWriter, "Hello, ") 29 | if err != nil { 30 | t.Error(err) 31 | } 32 | _, err = fmt.Fprintf(logWriter, "world!") 33 | if err != nil { 34 | t.Error(err) 35 | } 36 | 37 | // Close the log writer to force it to flush out the buffer 38 | err = logWriter.Close() 39 | if err != nil { 40 | t.Error(err) 41 | } 42 | 43 | delivery, ok, err := amqpChan.Get("reporting.jobs.logs", true) 44 | if err != nil { 45 | t.Error(err) 46 | } 47 | if !ok { 48 | t.Error("expected log message, but there was none") 49 | } 50 | 51 | var lp amqpLogPart 52 | 53 | err = json.Unmarshal(delivery.Body, &lp) 54 | if err != nil { 55 | t.Error(err) 56 | } 57 | 58 | expected := amqpLogPart{ 59 | JobID: 4, 60 | Content: "Hello, world!", 61 | Number: 0, 62 | UUID: uuid.String(), 63 | Final: false, 64 | } 65 | 66 | if expected != lp { 67 | t.Errorf("log part is %#v, expected %#v", lp, expected) 68 | } 69 | } 70 | 71 | func TestAMQPLogWriterClose(t *testing.T) { 72 | amqpConn, amqpChan := setupAMQPConn(t) 73 | defer amqpConn.Close() 74 | defer amqpChan.Close() 75 | 76 | uuid := uuid.NewRandom() 77 | ctx := workerctx.FromUUID(context.TODO(), uuid.String()) 78 | 79 | logWriter, err := newAMQPLogWriter(ctx, amqpChan, 4, time.Hour, false) 80 | if err != nil { 81 | t.Fatal(err) 82 | } 83 | logWriter.SetMaxLogLength(1000) 84 | 85 | // Close the log writer to force it to flush out the buffer 86 | err = logWriter.Close() 87 | if err != nil { 88 | t.Error(err) 89 | } 90 | 91 | delivery, ok, err := amqpChan.Get("reporting.jobs.logs", true) 92 | if err != nil { 93 | t.Error(err) 94 | } 95 | if !ok { 96 | t.Error("expected log message, but there was none") 97 | } 98 | 99 | var lp amqpLogPart 100 | 101 | err = json.Unmarshal(delivery.Body, &lp) 102 | if err != nil { 103 | t.Error(err) 104 | } 105 | 106 | expected := amqpLogPart{ 107 | JobID: 4, 108 | Content: "", 109 | Number: 0, 110 | UUID: uuid.String(), 111 | Final: true, 112 | } 113 | 114 | if expected != lp { 115 | t.Errorf("log part is %#v, expected %#v", lp, expected) 116 | } 117 | } 118 | 119 | func noCancel() {} 120 | 121 | func TestAMQPMaxLogLength(t *testing.T) { 122 | amqpConn, amqpChan := setupAMQPConn(t) 123 | defer amqpConn.Close() 124 | defer amqpChan.Close() 125 | 126 | uuid := uuid.NewRandom() 127 | ctx := workerctx.FromUUID(context.TODO(), uuid.String()) 128 | 129 | logWriter, err := newAMQPLogWriter(ctx, amqpChan, 4, time.Hour, false) 130 | if err != nil { 131 | t.Fatal(err) 132 | } 133 | logWriter.SetMaxLogLength(4) 134 | logWriter.SetCancelFunc(noCancel) 135 | 136 | _, err = fmt.Fprintf(logWriter, "1234") 137 | if err != nil { 138 | t.Error(err) 139 | } 140 | if logWriter.MaxLengthReached() { 141 | t.Error("max length should not be reached yet") 142 | } 143 | 144 | _, _ = fmt.Fprintf(logWriter, "5") 145 | if !logWriter.MaxLengthReached() { 146 | t.Error("expected MaxLengthReached to be true") 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /amqp_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | amqp "github.com/rabbitmq/amqp091-go" 8 | ) 9 | 10 | func setupAMQPConn(t *testing.T) (*amqp.Connection, *amqp.Channel) { 11 | if os.Getenv("AMQP_URI") == "" { 12 | t.Skip("skipping amqp test since there is no AMQP_URI") 13 | } 14 | 15 | amqpConn, err := amqp.Dial(os.Getenv("AMQP_URI")) 16 | if err != nil { 17 | t.Fatal(err) 18 | } 19 | 20 | logChan, err := amqpConn.Channel() 21 | if err != nil { 22 | t.Fatal(err) 23 | } 24 | 25 | err = logChan.ExchangeDeclare("reporting", "topic", true, false, false, false, nil) 26 | if err != nil { 27 | return nil, nil 28 | } 29 | _, err = logChan.QueueDeclare("reporting.jobs.logs", true, false, false, false, nil) 30 | if err != nil { 31 | t.Error(err) 32 | } 33 | 34 | err = logChan.QueueBind("reporting.jobs.logs", "reporting.jobs.logs", "reporting", false, nil) 35 | if err != nil { 36 | return nil, nil 37 | } 38 | 39 | _, err = logChan.QueuePurge("reporting.jobs.logs", false) 40 | if err != nil { 41 | t.Error(err) 42 | } 43 | 44 | return amqpConn, logChan 45 | } 46 | -------------------------------------------------------------------------------- /backend/backend.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "sort" 7 | "sync" 8 | 9 | "github.com/travis-ci/worker/config" 10 | ) 11 | 12 | var ( 13 | backendRegistry = map[string]*Backend{} 14 | backendRegistryMutex sync.Mutex 15 | ) 16 | 17 | var ErrDownloadTraceNotImplemented = errors.New("DownloadTrace not implemented") 18 | 19 | // Backend wraps up an alias, backend provider help, and a factory func for a 20 | // given backend provider wheee 21 | type Backend struct { 22 | Alias string 23 | HumanReadableName string 24 | ProviderHelp map[string]string 25 | ProviderFunc func(*config.ProviderConfig) (Provider, error) 26 | } 27 | 28 | // Register adds a backend to the registry! 29 | func Register(alias, humanReadableName string, providerHelp map[string]string, providerFunc func(*config.ProviderConfig) (Provider, error)) { 30 | backendRegistryMutex.Lock() 31 | defer backendRegistryMutex.Unlock() 32 | 33 | backendRegistry[alias] = &Backend{ 34 | Alias: alias, 35 | HumanReadableName: humanReadableName, 36 | ProviderHelp: providerHelp, 37 | ProviderFunc: providerFunc, 38 | } 39 | } 40 | 41 | // NewBackendProvider looks up a backend by its alias and returns a provider via 42 | // the factory func on the registered *Backend 43 | func NewBackendProvider(alias string, cfg *config.ProviderConfig) (Provider, error) { 44 | backendRegistryMutex.Lock() 45 | defer backendRegistryMutex.Unlock() 46 | 47 | backend, ok := backendRegistry[alias] 48 | if !ok { 49 | return nil, fmt.Errorf("unknown backend provider: %s", alias) 50 | } 51 | 52 | return backend.ProviderFunc(cfg) 53 | } 54 | 55 | // EachBackend calls a given function for each registered backend 56 | func EachBackend(f func(*Backend)) { 57 | backendRegistryMutex.Lock() 58 | defer backendRegistryMutex.Unlock() 59 | 60 | backendAliases := []string{} 61 | for backendAlias := range backendRegistry { 62 | backendAliases = append(backendAliases, backendAlias) 63 | } 64 | 65 | sort.Strings(backendAliases) 66 | 67 | for _, backendAlias := range backendAliases { 68 | f(backendRegistry[backendAlias]) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /backend/fake.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "io" 7 | "time" 8 | 9 | "github.com/travis-ci/worker/config" 10 | ) 11 | 12 | func init() { 13 | Register("fake", "Fake", map[string]string{ 14 | "LOG_OUTPUT": "faked log output to write", 15 | "RUN_SLEEP": "faked runtime sleep duration", 16 | "ERROR": "error out all jobs (useful for testing requeue storms)", 17 | }, newFakeProvider) 18 | } 19 | 20 | type fakeProvider struct { 21 | cfg *config.ProviderConfig 22 | } 23 | 24 | func newFakeProvider(cfg *config.ProviderConfig) (Provider, error) { 25 | return &fakeProvider{cfg: cfg}, nil 26 | } 27 | 28 | func (p *fakeProvider) SupportsProgress() bool { 29 | return false 30 | } 31 | 32 | func (p *fakeProvider) StartWithProgress(ctx context.Context, startAttributes *StartAttributes, _ Progresser) (Instance, error) { 33 | return p.Start(ctx, startAttributes) 34 | } 35 | 36 | func (p *fakeProvider) Start(ctx context.Context, _ *StartAttributes) (Instance, error) { 37 | var ( 38 | dur time.Duration 39 | err error 40 | ) 41 | 42 | if p.cfg.IsSet("STARTUP_DURATION") { 43 | dur, err = time.ParseDuration(p.cfg.Get("STARTUP_DURATION")) 44 | if err != nil { 45 | return nil, err 46 | } 47 | } 48 | 49 | return &fakeInstance{p: p, startupDuration: dur}, nil 50 | } 51 | 52 | func (p *fakeProvider) Setup(ctx context.Context) error { return nil } 53 | 54 | type fakeInstance struct { 55 | p *fakeProvider 56 | 57 | startupDuration time.Duration 58 | } 59 | 60 | func (i *fakeInstance) Warmed() bool { 61 | return false 62 | } 63 | 64 | func (i *fakeInstance) SupportsProgress() bool { 65 | return false 66 | } 67 | 68 | func (i *fakeInstance) UploadScript(ctx context.Context, script []byte) error { 69 | return nil 70 | } 71 | 72 | func (i *fakeInstance) RunScript(ctx context.Context, writer io.Writer) (*RunResult, error) { 73 | if i.p.cfg.Get("ERROR") == "true" { 74 | return &RunResult{Completed: false}, errors.New("fake provider is configured to error all jobs") 75 | } 76 | 77 | if i.p.cfg.IsSet("RUN_SLEEP") { 78 | rs, err := time.ParseDuration(i.p.cfg.Get("RUN_SLEEP")) 79 | if err != nil { 80 | return &RunResult{Completed: false}, err 81 | } 82 | time.Sleep(rs) 83 | } 84 | 85 | _, err := writer.Write([]byte(i.p.cfg.Get("LOG_OUTPUT"))) 86 | if err != nil { 87 | return &RunResult{Completed: false}, err 88 | } 89 | 90 | return &RunResult{Completed: true}, nil 91 | } 92 | 93 | func (i *fakeInstance) DownloadTrace(ctx context.Context) ([]byte, error) { 94 | return nil, ErrDownloadTraceNotImplemented 95 | } 96 | 97 | func (i *fakeInstance) Stop(ctx context.Context) error { 98 | return nil 99 | } 100 | 101 | func (i *fakeInstance) ID() string { 102 | return "fake" 103 | } 104 | 105 | func (i *fakeInstance) ImageName() string { 106 | return "fake" 107 | } 108 | 109 | func (i *fakeInstance) StartupDuration() time.Duration { 110 | return i.startupDuration 111 | } 112 | -------------------------------------------------------------------------------- /backend/gce_test.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "net/http/httptest" 9 | "net/url" 10 | "os" 11 | "testing" 12 | 13 | "github.com/stretchr/testify/assert" 14 | "github.com/travis-ci/worker/config" 15 | ) 16 | 17 | type gceTestResponse struct { 18 | Status int 19 | Headers map[string]string 20 | Body string 21 | } 22 | 23 | type gceTestServer struct { 24 | Responses *gceTestResponseMap 25 | } 26 | 27 | type gceTestResponseMap struct { 28 | Map map[string]*gceTestResponse 29 | } 30 | 31 | func (s *gceTestServer) ServeHTTP(w http.ResponseWriter, req *http.Request) { 32 | q := req.URL.Query() 33 | origURL := q.Get("_orig_req_url") 34 | 35 | resp, ok := s.Responses.Map[origURL] 36 | if !ok { 37 | w.WriteHeader(http.StatusNotFound) 38 | fmt.Fprintf(w, "OOPS NOPE! %v", origURL) 39 | return 40 | } 41 | 42 | for key, value := range resp.Headers { 43 | w.Header().Set(key, value) 44 | } 45 | 46 | w.WriteHeader(resp.Status) 47 | _, _ = io.WriteString(w, resp.Body) 48 | } 49 | 50 | func gceTestSetupGCEServer(resp *gceTestResponseMap) *httptest.Server { 51 | if resp == nil { 52 | resp = &gceTestResponseMap{Map: map[string]*gceTestResponse{}} 53 | } 54 | 55 | return httptest.NewServer(&gceTestServer{Responses: resp}) 56 | } 57 | 58 | type gceTestRequestLog struct { 59 | Reqs []*http.Request 60 | } 61 | 62 | func (rl *gceTestRequestLog) Add(req *http.Request) { 63 | if rl.Reqs == nil { 64 | rl.Reqs = []*http.Request{} 65 | } 66 | 67 | rl.Reqs = append(rl.Reqs, req) 68 | } 69 | 70 | func gceTestSetup(t *testing.T, cfg *config.ProviderConfig, resp *gceTestResponseMap) (*gceProvider, *http.Transport, *gceTestRequestLog) { 71 | if cfg == nil { 72 | cfg = config.ProviderConfigFromMap(map[string]string{ 73 | "ACCOUNT_JSON": "{}", 74 | "PROJECT_ID": "project_id", 75 | "IMAGE_ALIASES": "foo", 76 | "IMAGE_ALIAS_FOO": "default", 77 | }) 78 | } 79 | 80 | server := gceTestSetupGCEServer(resp) 81 | reqs := &gceTestRequestLog{} 82 | 83 | gceCustomHTTPTransportLock.Lock() 84 | transport := &http.Transport{ 85 | Proxy: func(req *http.Request) (*url.URL, error) { 86 | reqs.Add(req) 87 | 88 | u, err := url.Parse(server.URL) 89 | if err != nil { 90 | return nil, err 91 | } 92 | q := u.Query() 93 | q.Set("_orig_req_url", req.URL.String()) 94 | u.RawQuery = q.Encode() 95 | return u, nil 96 | }, 97 | } 98 | gceCustomHTTPTransport = transport 99 | 100 | p, err := newGCEProvider(cfg) 101 | 102 | gceCustomHTTPTransport = nil 103 | gceCustomHTTPTransportLock.Unlock() 104 | 105 | if err != nil { 106 | t.Fatal(err) 107 | } 108 | 109 | provider := p.(*gceProvider) 110 | return provider, transport, reqs 111 | } 112 | 113 | func gceTestTeardown(p *gceProvider) { 114 | if p.cfg.IsSet("TEMP_DIR") { 115 | _ = os.RemoveAll(p.cfg.Get("TEMP_DIR")) 116 | } 117 | } 118 | 119 | func TestNewGCEProvider(t *testing.T) { 120 | p, _, _ := gceTestSetup(t, nil, nil) 121 | defer gceTestTeardown(p) 122 | } 123 | 124 | func TestGCEProvider_SetupMakesRequests(t *testing.T) { 125 | p, _, rl := gceTestSetup(t, nil, nil) 126 | err := p.Setup(context.TODO()) 127 | 128 | assert.NotNil(t, err) 129 | assert.True(t, len(rl.Reqs) >= 1) 130 | } 131 | -------------------------------------------------------------------------------- /backend/local.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "os" 8 | "os/exec" 9 | "path/filepath" 10 | "time" 11 | 12 | gocontext "context" 13 | 14 | "github.com/travis-ci/worker/config" 15 | ) 16 | 17 | var ( 18 | errNoScriptUploaded = fmt.Errorf("no script uploaded") 19 | localHelp = map[string]string{ 20 | "SCRIPTS_DIR": "directory where generated scripts will be written", 21 | } 22 | ) 23 | 24 | func init() { 25 | Register("local", "Local", localHelp, newLocalProvider) 26 | } 27 | 28 | type localProvider struct { 29 | cfg *config.ProviderConfig 30 | scriptsDir string 31 | } 32 | 33 | func newLocalProvider(cfg *config.ProviderConfig) (Provider, error) { 34 | scriptsDir, _ := os.Getwd() 35 | 36 | if cfg.IsSet("SCRIPTS_DIR") { 37 | scriptsDir = cfg.Get("SCRIPTS_DIR") 38 | } 39 | 40 | if scriptsDir == "" { 41 | scriptsDir = os.TempDir() 42 | } 43 | 44 | return &localProvider{cfg: cfg, scriptsDir: scriptsDir}, nil 45 | } 46 | 47 | func (p *localProvider) SupportsProgress() bool { 48 | return false 49 | } 50 | 51 | func (p *localProvider) StartWithProgress(ctx gocontext.Context, startAttributes *StartAttributes, _ Progresser) (Instance, error) { 52 | return p.Start(ctx, startAttributes) 53 | } 54 | 55 | func (p *localProvider) Start(ctx gocontext.Context, startAttributes *StartAttributes) (Instance, error) { 56 | return newLocalInstance(p) 57 | } 58 | 59 | func (p *localProvider) Setup(ctx gocontext.Context) error { return nil } 60 | 61 | type localInstance struct { 62 | p *localProvider 63 | 64 | scriptPath string 65 | } 66 | 67 | func newLocalInstance(p *localProvider) (*localInstance, error) { 68 | return &localInstance{ 69 | p: p, 70 | }, nil 71 | } 72 | 73 | func (i *localInstance) Warmed() bool { 74 | return false 75 | } 76 | 77 | func (i *localInstance) SupportsProgress() bool { 78 | return false 79 | } 80 | 81 | func (i *localInstance) UploadScript(ctx gocontext.Context, script []byte) error { 82 | scriptPath := filepath.Join(i.p.scriptsDir, fmt.Sprintf("build-%v.sh", time.Now().UTC().UnixNano())) 83 | f, err := os.Create(scriptPath) 84 | if err != nil { 85 | return err 86 | } 87 | 88 | i.scriptPath = scriptPath 89 | 90 | scriptBuf := bytes.NewBuffer(script) 91 | _, err = io.Copy(f, scriptBuf) 92 | return err 93 | } 94 | 95 | func (i *localInstance) RunScript(ctx gocontext.Context, writer io.Writer) (*RunResult, error) { 96 | if i.scriptPath == "" { 97 | return &RunResult{Completed: false}, errNoScriptUploaded 98 | } 99 | 100 | cmd := exec.Command("bash", i.scriptPath) 101 | cmd.Stdout = writer 102 | cmd.Stderr = writer 103 | 104 | err := cmd.Start() 105 | if err != nil { 106 | return &RunResult{Completed: false}, err 107 | } 108 | 109 | errChan := make(chan error) 110 | go func() { 111 | errChan <- cmd.Wait() 112 | }() 113 | 114 | select { 115 | case err := <-errChan: 116 | if err != nil { 117 | return &RunResult{Completed: false}, err 118 | } 119 | return &RunResult{Completed: true}, nil 120 | case <-ctx.Done(): 121 | err = ctx.Err() 122 | if err != nil { 123 | return &RunResult{Completed: false}, err 124 | } 125 | return &RunResult{Completed: true}, nil 126 | } 127 | } 128 | 129 | func (i *localInstance) DownloadTrace(ctx gocontext.Context) ([]byte, error) { 130 | return nil, ErrDownloadTraceNotImplemented 131 | } 132 | 133 | func (i *localInstance) Stop(ctx gocontext.Context) error { 134 | return nil 135 | } 136 | 137 | func (i *localInstance) ID() string { 138 | return fmt.Sprintf("local:%s", i.scriptPath) 139 | } 140 | 141 | func (i *localInstance) ImageName() string { 142 | return "" 143 | } 144 | 145 | func (i *localInstance) StartupDuration() time.Duration { return zeroDuration } 146 | -------------------------------------------------------------------------------- /backend/package_test.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | gocontext "context" 5 | "fmt" 6 | "math/rand" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/travis-ci/worker/context" 12 | ) 13 | 14 | func Test_asBool(t *testing.T) { 15 | for s, b := range map[string]bool{ 16 | "yes": true, 17 | "on": true, 18 | "1": true, 19 | "boo": true, 20 | "0": false, 21 | "99": true, 22 | "a": true, 23 | "off": false, 24 | "no": false, 25 | "fafafaf": true, 26 | "true": true, 27 | "false": false, 28 | "": false, 29 | } { 30 | assert.Equal(t, b, asBool(s)) 31 | } 32 | } 33 | 34 | func Test_hostnameFromContext(t *testing.T) { 35 | jobID := rand.Uint64() 36 | 37 | for _, tc := range []struct{ r, n string }{ 38 | { 39 | r: "friendly/fribble", 40 | n: fmt.Sprintf("travis-job-friendly-fribble-%v", jobID), 41 | }, 42 | { 43 | r: "very-SiLlY.nAmE.wat/por-cu___-pine", 44 | n: fmt.Sprintf("travis-job-very-silly-nam-por-cu-pine-%v", jobID), 45 | }, 46 | } { 47 | ctx := context.FromRepository(context.FromJobID(gocontext.TODO(), jobID), tc.r) 48 | assert.Equal(t, tc.n, hostnameFromContext(ctx)) 49 | } 50 | 51 | randName := hostnameFromContext(gocontext.TODO()) 52 | randParts := strings.Split(randName, "-") 53 | assert.Len(t, randParts, 9) 54 | assert.Equal(t, "unk", randParts[2]) 55 | assert.Equal(t, "unk", randParts[3]) 56 | } 57 | 58 | func Test_str2Map(t *testing.T) { 59 | s := "foo:bar,bang:baz Hello:World, extra space:justBecause sillychars:butwhy%3F encodedspace:yup+, colonInside:why%3Anot" 60 | m := str2map(s, " ,") 61 | e := map[string]string{ 62 | "foo": "bar", 63 | "bang": "baz", 64 | "Hello": "World", 65 | "extra": "", 66 | "space": "justBecause", 67 | "sillychars": "butwhy?", 68 | "encodedspace": "yup ", 69 | "colonInside": "why:not", 70 | } 71 | assert.Equal(t, e, m) 72 | } 73 | -------------------------------------------------------------------------------- /backend/progresser.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | type ProgressState int 4 | 5 | const ( 6 | ProgressNeutral ProgressState = iota 7 | ProgressSuccess 8 | ProgressFailure 9 | ) 10 | 11 | func (ps ProgressState) String() string { 12 | switch ps { 13 | case ProgressSuccess: 14 | return "success" 15 | case ProgressFailure: 16 | return "failure" 17 | case ProgressNeutral: 18 | return "neutral" 19 | default: 20 | return "unknown" 21 | } 22 | } 23 | 24 | type ProgressEntry struct { 25 | Message string `json:"message"` 26 | State ProgressState `json:"state"` 27 | Interrupts bool `json:"interrupts"` 28 | Continues bool `json:"continues"` 29 | Raw bool `json:"raw"` 30 | } 31 | 32 | type Progresser interface { 33 | Progress(*ProgressEntry) 34 | } 35 | 36 | type NullProgresser struct{} 37 | 38 | func (np *NullProgresser) Progress(_ *ProgressEntry) {} 39 | -------------------------------------------------------------------------------- /backend/start_attributes.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type VmConfig struct { 8 | GpuCount int64 `json:"gpu_count"` 9 | GpuType string `json:"gpu_type"` 10 | Zone string `json:"zone"` 11 | } 12 | 13 | // StartAttributes contains some parts of the config which can be used to 14 | // determine the type of instance to boot up (for example, what image to use) 15 | type StartAttributes struct { 16 | Language string `json:"language"` 17 | OsxImage string `json:"osx_image"` 18 | Dist string `json:"dist"` 19 | Arch string `json:"arch"` 20 | Group string `json:"group"` 21 | OS string `json:"os"` 22 | ImageName string `json:"image_name"` 23 | 24 | // The VMType isn't stored in the config directly, but in the top level of 25 | // the job payload, see the worker.JobPayload struct. 26 | VMType string `json:"-"` 27 | 28 | // The VMConfig isn't stored in the config directly, but in the top level of 29 | // the job payload, see the worker.JobPayload struct. 30 | VMConfig VmConfig `json:"-"` 31 | 32 | // The VMSize isn't stored in the config directly, but in the top level of 33 | // the job payload, see the worker.JobPayload struct. 34 | VMSize string `json:"-"` 35 | 36 | // Warmer isn't stored in the config directly, but in the top level of 37 | // the job payload, see the worker.JobPayload struct. 38 | Warmer bool `json:"-"` 39 | 40 | // HardTimeout isn't stored in the config directly, but is injected 41 | // from the processor 42 | HardTimeout time.Duration `json:"-"` 43 | 44 | // ProgressType isn't stored in the config directly, but is injected from 45 | // the processor 46 | ProgressType string `json:"-"` 47 | } 48 | 49 | // SetDefaults sets any missing required attributes to the default values provided 50 | func (sa *StartAttributes) SetDefaults(lang, dist, arch, group, os, vmType string, vmConfig VmConfig) { 51 | if sa.Language == "" { 52 | sa.Language = lang 53 | } 54 | 55 | if sa.Dist == "" { 56 | sa.Dist = dist 57 | } 58 | 59 | if sa.Arch == "" { 60 | sa.Arch = arch 61 | } 62 | 63 | if sa.Group == "" { 64 | sa.Group = group 65 | } 66 | 67 | if sa.OS == "" { 68 | sa.OS = os 69 | } 70 | 71 | if sa.VMType == "" { 72 | sa.VMType = vmType 73 | } 74 | 75 | if sa.VMConfig.GpuCount == 0 { 76 | sa.VMConfig.GpuCount = vmConfig.GpuCount 77 | } 78 | 79 | if sa.VMConfig.GpuType == "" { 80 | sa.VMConfig.GpuType = vmConfig.GpuType 81 | } 82 | 83 | if sa.VMConfig.Zone == "" { 84 | sa.VMConfig.Zone = vmConfig.Zone 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /backend/start_attributes_test.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | var ( 10 | testStartAttributesTestCases = []struct { 11 | A []*StartAttributes 12 | O []*StartAttributes 13 | }{ 14 | { 15 | A: []*StartAttributes{ 16 | {Language: ""}, 17 | {Language: "ruby"}, 18 | {Language: "python", Dist: "trusty"}, 19 | {Language: "python", Dist: "trusty", Arch: "amd64", Group: "edge"}, 20 | {Language: "python", Dist: "frob", Arch: "amd64", Group: "edge", OS: "flob"}, 21 | {Language: "python", Dist: "frob", Arch: "amd64", OsxImage: "", Group: "edge", OS: "flob"}, 22 | {Language: "python", Dist: "frob", Arch: "amd64", OsxImage: "", Group: "edge", OS: "flob", VMType: "premium"}, 23 | {Language: "python", Dist: "frob", Arch: "amd64", OsxImage: "", Group: "edge", OS: "flob", VMType: "premium", VMConfig: VmConfig{GpuCount: 0, GpuType: "", Zone: ""}}, 24 | }, 25 | O: []*StartAttributes{ 26 | {Language: "default", Dist: "precise", Arch: "amd64", Group: "stable", OS: "linux", VMType: "default"}, 27 | {Language: "ruby", Dist: "precise", Arch: "amd64", Group: "stable", OS: "linux", VMType: "default"}, 28 | {Language: "python", Dist: "trusty", Arch: "amd64", Group: "stable", OS: "linux", VMType: "default"}, 29 | {Language: "python", Dist: "trusty", Arch: "amd64", Group: "edge", OS: "linux", VMType: "default"}, 30 | {Language: "python", Dist: "frob", Arch: "amd64", Group: "edge", OS: "flob", VMType: "default"}, 31 | {Language: "python", Dist: "frob", Arch: "amd64", OsxImage: "", Group: "edge", OS: "flob", VMType: "default"}, 32 | {Language: "python", Dist: "frob", Arch: "amd64", OsxImage: "", Group: "edge", OS: "flob", VMType: "premium"}, 33 | {Language: "python", Dist: "frob", Arch: "amd64", OsxImage: "", Group: "edge", OS: "flob", VMType: "premium", VMConfig: VmConfig{}}, 34 | }, 35 | }, 36 | } 37 | ) 38 | 39 | func TestStartAttributes(t *testing.T) { 40 | sa := &StartAttributes{} 41 | assert.Equal(t, "", sa.Dist) 42 | assert.Equal(t, "", sa.Arch) 43 | assert.Equal(t, "", sa.Group) 44 | assert.Equal(t, "", sa.Language) 45 | assert.Equal(t, "", sa.OS) 46 | assert.Equal(t, "", sa.OsxImage) 47 | assert.Equal(t, "", sa.VMType) 48 | assert.Equal(t, VmConfig{GpuCount: 0, GpuType: "", Zone: ""}, sa.VMConfig) 49 | } 50 | 51 | func TestStartAttributes_SetDefaults(t *testing.T) { 52 | for _, tc := range testStartAttributesTestCases { 53 | for i, sa := range tc.A { 54 | expected := tc.O[i] 55 | sa.SetDefaults("default", "precise", "amd64", "stable", "linux", "default", sa.VMConfig) 56 | assert.Equal(t, expected, sa) 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /backend/text_progresser.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | var textProgressHeads = map[ProgressState]string{ 8 | ProgressSuccess: "✓ ", 9 | ProgressFailure: "✗ ", 10 | ProgressNeutral: "• ", 11 | } 12 | 13 | type TextProgresser struct { 14 | w io.Writer 15 | } 16 | 17 | func NewTextProgresser(w io.Writer) *TextProgresser { 18 | if w == nil { 19 | w = io.Discard 20 | } 21 | return &TextProgresser{w: w} 22 | } 23 | 24 | func (tp *TextProgresser) Progress(entry *ProgressEntry) { 25 | head := textProgressHeads[entry.State] 26 | tail := "\r\n" 27 | 28 | if entry.Interrupts { 29 | head = "\r\n" + head 30 | } 31 | 32 | if entry.Continues { 33 | tail = "" 34 | } 35 | 36 | if entry.Raw { 37 | head = "" 38 | tail = "" 39 | } 40 | 41 | _, _ = tp.w.Write([]byte(head + entry.Message + tail)) 42 | } 43 | -------------------------------------------------------------------------------- /backend/text_progresser_test.go: -------------------------------------------------------------------------------- 1 | package backend 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestNewTextProgresser(t *testing.T) { 12 | w := &bytes.Buffer{} 13 | tp := NewTextProgresser(w) 14 | 15 | assert.NotNil(t, tp) 16 | assert.NotNil(t, tp.w) 17 | 18 | tp = NewTextProgresser(nil) 19 | assert.NotNil(t, tp) 20 | assert.NotNil(t, tp.w) 21 | assert.Equal(t, tp.w, io.Discard) 22 | } 23 | 24 | var testTextProgresserProgressCases = []struct { 25 | e *ProgressEntry 26 | o string 27 | }{ 28 | {e: &ProgressEntry{Message: "hello"}, 29 | o: "• hello\r\n"}, 30 | {e: &ProgressEntry{Message: "yay", State: ProgressSuccess}, 31 | o: "✓ yay\r\n"}, 32 | {e: &ProgressEntry{Message: "oh no", State: ProgressFailure}, 33 | o: "✗ oh no\r\n"}, 34 | {e: &ProgressEntry{Message: "hello...", Continues: true}, 35 | o: "• hello..."}, 36 | {e: &ProgressEntry{Message: "yay...", State: ProgressSuccess, Continues: true}, 37 | o: "✓ yay..."}, 38 | {e: &ProgressEntry{Message: "oh no...", State: ProgressFailure, Continues: true}, 39 | o: "✗ oh no..."}, 40 | {e: &ProgressEntry{Message: "hello!", Interrupts: true}, 41 | o: "\r\n• hello!\r\n"}, 42 | {e: &ProgressEntry{Message: "yay!", State: ProgressSuccess, Interrupts: true}, 43 | o: "\r\n✓ yay!\r\n"}, 44 | {e: &ProgressEntry{Message: "oh no!", State: ProgressFailure, Interrupts: true}, 45 | o: "\r\n✗ oh no!\r\n"}, 46 | {e: &ProgressEntry{Message: "hello!...", Interrupts: true, Continues: true}, 47 | o: "\r\n• hello!..."}, 48 | {e: &ProgressEntry{Message: "yay!...", State: ProgressSuccess, Interrupts: true, Continues: true}, 49 | o: "\r\n✓ yay!..."}, 50 | {e: &ProgressEntry{Message: "oh no!...", State: ProgressFailure, Interrupts: true, Continues: true}, 51 | o: "\r\n✗ oh no!..."}, 52 | {e: &ProgressEntry{Message: "hello", Raw: true}, 53 | o: "hello"}, 54 | {e: &ProgressEntry{Message: "yay", State: ProgressSuccess, Raw: true}, 55 | o: "yay"}, 56 | {e: &ProgressEntry{Message: "oh no", State: ProgressFailure, Raw: true}, 57 | o: "oh no"}, 58 | } 59 | 60 | func TestTextProgresser_Progress(t *testing.T) { 61 | w := &bytes.Buffer{} 62 | tp := NewTextProgresser(w) 63 | 64 | for _, entry := range testTextProgresserProgressCases { 65 | buf := &bytes.Buffer{} 66 | tp.w = buf 67 | tp.Progress(entry.e) 68 | assert.Equal(t, entry.o, buf.String()) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /build_script_generator.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "bytes" 5 | "crypto/tls" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "net/url" 10 | "strconv" 11 | "time" 12 | 13 | gocontext "context" 14 | 15 | "github.com/travis-ci/worker/config" 16 | "github.com/travis-ci/worker/metrics" 17 | ) 18 | 19 | // A BuildScriptGeneratorError is sometimes used by the Generate method on a 20 | // BuildScriptGenerator to return more metadata about an error. 21 | type BuildScriptGeneratorError struct { 22 | error 23 | 24 | // true when this error can be recovered by retrying later 25 | Recover bool 26 | } 27 | 28 | // A BuildScriptGenerator generates a build script for a given job payload. 29 | type BuildScriptGenerator interface { 30 | Generate(gocontext.Context, Job) ([]byte, error) 31 | } 32 | 33 | type webBuildScriptGenerator struct { 34 | URL string 35 | aptCacheHost string 36 | npmCacheHost string 37 | paranoid bool 38 | fixResolvConf bool 39 | fixEtcHosts bool 40 | cacheType string 41 | cacheFetchTimeout int 42 | cachePushTimeout int 43 | s3CacheOptions s3BuildCacheOptions 44 | 45 | httpClient *http.Client 46 | } 47 | 48 | type s3BuildCacheOptions struct { 49 | scheme string 50 | region string 51 | bucket string 52 | accessKeyID string 53 | secretAccessKey string 54 | } 55 | 56 | // NewBuildScriptGenerator creates a generator backed by an HTTP API. 57 | func NewBuildScriptGenerator(cfg *config.Config) BuildScriptGenerator { 58 | return &webBuildScriptGenerator{ 59 | URL: cfg.BuildAPIURI, 60 | aptCacheHost: cfg.BuildAptCache, 61 | npmCacheHost: cfg.BuildNpmCache, 62 | paranoid: cfg.BuildParanoid, 63 | fixResolvConf: cfg.BuildFixResolvConf, 64 | fixEtcHosts: cfg.BuildFixEtcHosts, 65 | cacheType: cfg.BuildCacheType, 66 | cacheFetchTimeout: int(cfg.BuildCacheFetchTimeout.Seconds()), 67 | cachePushTimeout: int(cfg.BuildCachePushTimeout.Seconds()), 68 | s3CacheOptions: s3BuildCacheOptions{ 69 | scheme: cfg.BuildCacheS3Scheme, 70 | region: cfg.BuildCacheS3Region, 71 | bucket: cfg.BuildCacheS3Bucket, 72 | accessKeyID: cfg.BuildCacheS3AccessKeyID, 73 | secretAccessKey: cfg.BuildCacheS3SecretAccessKey, 74 | }, 75 | httpClient: &http.Client{ 76 | Transport: &http.Transport{ 77 | TLSClientConfig: &tls.Config{ 78 | InsecureSkipVerify: cfg.BuildAPIInsecureSkipVerify, 79 | }, 80 | }, 81 | }, 82 | } 83 | } 84 | 85 | func (g *webBuildScriptGenerator) Generate(ctx gocontext.Context, job Job) ([]byte, error) { 86 | payload := job.RawPayload() 87 | 88 | if g.aptCacheHost != "" { 89 | payload.SetPath([]string{"hosts", "apt_cache"}, g.aptCacheHost) 90 | } 91 | if g.npmCacheHost != "" { 92 | payload.SetPath([]string{"hosts", "npm_cache"}, g.npmCacheHost) 93 | } 94 | 95 | payload.Set("paranoid", g.paranoid) 96 | payload.Set("fix_resolv_conf", g.fixResolvConf) 97 | payload.Set("fix_etc_hosts", g.fixEtcHosts) 98 | 99 | if g.cacheType != "" { 100 | payload.SetPath([]string{"cache_options", "type"}, g.cacheType) 101 | payload.SetPath([]string{"cache_options", "fetch_timeout"}, g.cacheFetchTimeout) 102 | payload.SetPath([]string{"cache_options", "push_timeout"}, g.cachePushTimeout) 103 | payload.SetPath([]string{"cache_options", "s3", "scheme"}, g.s3CacheOptions.scheme) 104 | payload.SetPath([]string{"cache_options", "s3", "region"}, g.s3CacheOptions.region) 105 | payload.SetPath([]string{"cache_options", "s3", "bucket"}, g.s3CacheOptions.bucket) 106 | payload.SetPath([]string{"cache_options", "s3", "access_key_id"}, g.s3CacheOptions.accessKeyID) 107 | payload.SetPath([]string{"cache_options", "s3", "secret_access_key"}, g.s3CacheOptions.secretAccessKey) 108 | } 109 | 110 | b, err := payload.Encode() 111 | if err != nil { 112 | return nil, err 113 | } 114 | 115 | var token string 116 | u, err := url.Parse(g.URL) 117 | if err != nil { 118 | return nil, err 119 | } 120 | if u.User != nil { 121 | token = u.User.Username() 122 | u.User = nil 123 | } 124 | 125 | jp := job.Payload() 126 | if jp != nil { 127 | q := u.Query() 128 | q.Set("job_id", strconv.FormatUint(jp.Job.ID, 10)) 129 | q.Set("source", "worker") 130 | u.RawQuery = q.Encode() 131 | } 132 | 133 | buf := bytes.NewBuffer(b) 134 | req, err := http.NewRequest("POST", u.String(), buf) 135 | if err != nil { 136 | return nil, err 137 | } 138 | if token != "" { 139 | req.Header.Set("Authorization", "token "+token) 140 | } 141 | req.Header.Set("User-Agent", fmt.Sprintf("worker-go v=%v rev=%v d=%v", VersionString, RevisionString, GeneratedString)) 142 | req.Header.Set("Content-Type", "application/json") 143 | 144 | startRequest := time.Now() 145 | 146 | resp, err := g.httpClient.Do(req) 147 | if err != nil { 148 | return nil, err 149 | } 150 | 151 | defer resp.Body.Close() 152 | metrics.TimeSince("worker.job.script.api", startRequest) 153 | 154 | body, err := io.ReadAll(resp.Body) 155 | if err != nil { 156 | return nil, err 157 | } 158 | 159 | if resp.StatusCode >= 500 { 160 | return nil, BuildScriptGeneratorError{error: fmt.Errorf("server error: %q", string(body)), Recover: true} 161 | } else if resp.StatusCode >= 400 { 162 | return nil, BuildScriptGeneratorError{error: fmt.Errorf("client error: %q", string(body)), Recover: false} 163 | } 164 | 165 | return body, nil 166 | } 167 | -------------------------------------------------------------------------------- /build_script_generator_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "net/http/httptest" 8 | "testing" 9 | 10 | "github.com/bitly/go-simplejson" 11 | "github.com/stretchr/testify/require" 12 | "github.com/travis-ci/worker/config" 13 | ) 14 | 15 | func TestBuildScriptGenerator(t *testing.T) { 16 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 17 | fmt.Fprintln(w, "Hello, client") 18 | })) 19 | defer ts.Close() 20 | 21 | gen := NewBuildScriptGenerator(&config.Config{BuildAPIURI: ts.URL}) 22 | 23 | script, err := gen.Generate(context.TODO(), &fakeJob{ 24 | rawPayload: simplejson.New(), 25 | }) 26 | require.Nil(t, err) 27 | require.Equal(t, []byte("Hello, client\n"), script) 28 | } 29 | -------------------------------------------------------------------------------- /build_trace_persister.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "bytes" 5 | "strconv" 6 | 7 | gocontext "context" 8 | 9 | "github.com/aws/aws-sdk-go/aws" 10 | "github.com/aws/aws-sdk-go/aws/session" 11 | "github.com/aws/aws-sdk-go/service/s3" 12 | "github.com/travis-ci/worker/config" 13 | ) 14 | 15 | // credentials are configured via env: 16 | // * AWS_ACCESS_KEY_ID 17 | // * AWS_SECRET_ACCESS_KEY 18 | // 19 | // or via the shared creds file ~/.aws/credentials 20 | // 21 | // or via the EC2 instance IAM role 22 | 23 | // A BuildTracePersister persists a build trace. (duh) 24 | type BuildTracePersister interface { 25 | Persist(gocontext.Context, Job, []byte) error 26 | } 27 | 28 | type s3BuildTracePersister struct { 29 | bucket string 30 | keyPrefix string 31 | region string 32 | } 33 | 34 | // NewBuildTracePersister creates a build trace persister backed by S3 35 | func NewBuildTracePersister(cfg *config.Config) BuildTracePersister { 36 | if !cfg.BuildTraceEnabled { 37 | return nil 38 | } 39 | 40 | return &s3BuildTracePersister{ 41 | bucket: cfg.BuildTraceS3Bucket, 42 | keyPrefix: cfg.BuildTraceS3KeyPrefix, 43 | region: cfg.BuildTraceS3Region, 44 | } 45 | } 46 | 47 | // TODO: cache aws session for reuse? goroutine pool? 48 | // TODO: handle job restarts -- archive separate trace per run? use job UUID? 49 | // perhaps we can rely on s3 versioning for this. 50 | 51 | func (p *s3BuildTracePersister) Persist(ctx gocontext.Context, job Job, buf []byte) error { 52 | sess, err := session.NewSession(&aws.Config{Region: aws.String(p.region)}) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | key := p.keyPrefix + strconv.FormatUint(job.Payload().Job.ID, 10) 58 | 59 | _, err = s3.New(sess).PutObject(&s3.PutObjectInput{ 60 | Bucket: aws.String(p.bucket), 61 | Key: aws.String(key), 62 | ACL: aws.String("private"), 63 | Body: bytes.NewReader(buf), 64 | ContentLength: aws.Int64(int64(len(buf))), 65 | ContentType: aws.String("application/octet-stream"), 66 | ContentDisposition: aws.String("attachment"), 67 | ServerSideEncryption: aws.String("AES256"), 68 | }) 69 | 70 | return err 71 | } 72 | -------------------------------------------------------------------------------- /canceller.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import "sync" 4 | 5 | type CancellationCommand struct { 6 | JobID uint64 7 | Reason string 8 | } 9 | 10 | // A CancellationBroadcaster allows you to subscribe to and unsubscribe from 11 | // cancellation messages for a given job ID. 12 | type CancellationBroadcaster struct { 13 | registryMutex sync.Mutex 14 | registry map[uint64][](chan CancellationCommand) 15 | } 16 | 17 | // NewCancellationBroadcaster sets up a new cancellation broadcaster with an 18 | // empty registry. 19 | func NewCancellationBroadcaster() *CancellationBroadcaster { 20 | return &CancellationBroadcaster{ 21 | registry: make(map[uint64][](chan CancellationCommand)), 22 | } 23 | } 24 | 25 | // Broadcast broacasts a cancellation message to all currently subscribed 26 | // cancellers. 27 | func (cb *CancellationBroadcaster) Broadcast(command CancellationCommand) { 28 | cb.registryMutex.Lock() 29 | defer cb.registryMutex.Unlock() 30 | 31 | chans := cb.registry[command.JobID] 32 | delete(cb.registry, command.JobID) 33 | 34 | for _, ch := range chans { 35 | ch <- command 36 | close(ch) 37 | } 38 | } 39 | 40 | // Subscribe will set up a subscription for cancellation messages for the 41 | // given job ID. When a cancellation message comes in, the returned channel 42 | // will be closed. 43 | func (cb *CancellationBroadcaster) Subscribe(id uint64) <-chan CancellationCommand { 44 | cb.registryMutex.Lock() 45 | defer cb.registryMutex.Unlock() 46 | 47 | if _, ok := cb.registry[id]; !ok { 48 | cb.registry[id] = make([](chan CancellationCommand), 0, 1) 49 | } 50 | 51 | ch := make(chan CancellationCommand, 1) 52 | cb.registry[id] = append(cb.registry[id], ch) 53 | 54 | return ch 55 | } 56 | 57 | // Unsubscribe removes an existing subscription for the channel. 58 | func (cb *CancellationBroadcaster) Unsubscribe(id uint64, ch <-chan CancellationCommand) { 59 | cb.registryMutex.Lock() 60 | defer cb.registryMutex.Unlock() 61 | 62 | // If there's no registered channels for the given ID, just return 63 | if _, ok := cb.registry[id]; !ok { 64 | return 65 | } 66 | 67 | // If there's only one element, remove the key 68 | if len(cb.registry[id]) <= 1 { 69 | delete(cb.registry, id) 70 | return 71 | } 72 | 73 | var chanIndex int = -1 74 | for i, registeredChan := range cb.registry[id] { 75 | if registeredChan == ch { 76 | chanIndex = i 77 | break 78 | } 79 | } 80 | if chanIndex == -1 { 81 | // Channel is already removed 82 | return 83 | } 84 | 85 | // Remove element at index by putting the last element in that place, and 86 | // then shrinking the slice to remove the last element. 87 | cb.registry[id][chanIndex] = cb.registry[id][len(cb.registry[id])-1] 88 | cb.registry[id] = cb.registry[id][:len(cb.registry[id])-1] 89 | } 90 | -------------------------------------------------------------------------------- /canceller_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestCancellationBroadcaster(t *testing.T) { 8 | cb := NewCancellationBroadcaster() 9 | 10 | ch1_1 := cb.Subscribe(1) 11 | ch1_2 := cb.Subscribe(1) 12 | ch1_3 := cb.Subscribe(1) 13 | ch2 := cb.Subscribe(2) 14 | 15 | cb.Unsubscribe(1, ch1_2) 16 | 17 | cb.Broadcast(CancellationCommand{JobID: 1, Reason: "42"}) 18 | cb.Broadcast(CancellationCommand{JobID: 1, Reason: "42"}) 19 | 20 | assertReceived(t, "ch1_1", ch1_1, CancellationCommand{JobID: 1, Reason: "42"}) 21 | assertWaiting(t, "ch1_2", ch1_2) 22 | assertReceived(t, "ch1_3", ch1_3, CancellationCommand{JobID: 1, Reason: "42"}) 23 | assertWaiting(t, "ch2", ch2) 24 | } 25 | 26 | func assertReceived(t *testing.T, name string, ch <-chan CancellationCommand, expected CancellationCommand) { 27 | select { 28 | case val, ok := (<-ch): 29 | if ok { 30 | if expected != val { 31 | t.Errorf("expected to receive %v, got %v", expected, val) 32 | } 33 | } else { 34 | t.Errorf("expected %s to not be closed, but it was closed", name) 35 | } 36 | default: 37 | t.Errorf("expected %s to receive a value, but it didn't", name) 38 | } 39 | } 40 | 41 | func assertWaiting(t *testing.T, name string, ch <-chan CancellationCommand) { 42 | select { 43 | case _, ok := (<-ch): 44 | if ok { 45 | t.Errorf("expected %s to not be closed and not have a value, but it received a value", name) 46 | } else { 47 | t.Errorf("expected %s to not be closed and not have a value, but it was closed", name) 48 | } 49 | default: 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /cli_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | "time" 9 | 10 | gocontext "context" 11 | 12 | "github.com/sirupsen/logrus" 13 | "github.com/stretchr/testify/assert" 14 | "github.com/travis-ci/worker/context" 15 | ) 16 | 17 | func TestNewCLI(t *testing.T) { 18 | c := NewCLI(nil) 19 | assert.NotNil(t, c) 20 | assert.Nil(t, c.c) 21 | assert.True(t, c.bootTime.String() < time.Now().UTC().String()) 22 | } 23 | 24 | func TestCLI_heartbeatHandler(t *testing.T) { 25 | i := NewCLI(nil) 26 | i.heartbeatSleep = time.Duration(0) 27 | i.heartbeatErrSleep = time.Duration(0) 28 | 29 | ctx, cancel := gocontext.WithCancel(gocontext.Background()) 30 | logger := context.LoggerFromContext(ctx).WithField("self", "cli_test") 31 | i.ctx = ctx 32 | i.cancel = cancel 33 | i.logger = logger 34 | 35 | logrus.SetLevel(logrus.FatalLevel) 36 | 37 | i.ProcessorPool = NewProcessorPool(&ProcessorPoolConfig{ 38 | Context: ctx, 39 | }, nil, nil, nil, nil) 40 | 41 | n := 0 42 | done := make(chan struct{}) 43 | 44 | mux := http.NewServeMux() 45 | mux.HandleFunc(`/`, func(w http.ResponseWriter, req *http.Request) { 46 | n++ 47 | switch n { 48 | case 1: 49 | t.Logf("responding 404") 50 | w.WriteHeader(http.StatusNotFound) 51 | fmt.Fprintf(w, "no\n") 52 | return 53 | case 2: 54 | t.Logf("responding 200 with busted JSON") 55 | w.WriteHeader(http.StatusOK) 56 | fmt.Fprintf(w, "{bork\n") 57 | return 58 | case 3: 59 | t.Logf("responding 200 with unexpected JSON") 60 | w.WriteHeader(http.StatusOK) 61 | fmt.Fprintf(w, `{"etat": "sous"}`) 62 | return 63 | case 4: 64 | t.Logf("responding 200 with JSON of mismatched type") 65 | w.WriteHeader(http.StatusOK) 66 | fmt.Fprintf(w, `{"state": true}`) 67 | return 68 | case 5: 69 | t.Logf("responding 200 with state=up") 70 | w.WriteHeader(http.StatusOK) 71 | fmt.Fprintf(w, `{"state": "up"}`) 72 | return 73 | default: 74 | t.Logf("responding 200 with state=down") 75 | w.WriteHeader(http.StatusOK) 76 | fmt.Fprintf(w, `{"state": "down"}`) 77 | done <- struct{}{} 78 | return 79 | } 80 | }) 81 | 82 | ts := httptest.NewServer(mux) 83 | defer ts.Close() 84 | 85 | go i.heartbeatHandler(ts.URL, "") 86 | 87 | for { 88 | select { 89 | case <-done: 90 | cancel() 91 | return 92 | default: 93 | time.Sleep(time.Millisecond) 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /cmd/travis-worker/main.go: -------------------------------------------------------------------------------- 1 | // Package main implements the CLI for the travis-worker binary 2 | package main 3 | 4 | import ( 5 | "os" 6 | 7 | "github.com/travis-ci/worker" 8 | "github.com/travis-ci/worker/config" 9 | "gopkg.in/urfave/cli.v1" 10 | ) 11 | 12 | func main() { 13 | app := cli.NewApp() 14 | app.Usage = "Travis Worker" 15 | app.Version = worker.VersionString 16 | app.Author = "Travis CI GmbH" 17 | app.Email = "contact+travis-worker@travis-ci.com" 18 | app.Copyright = worker.CopyrightString 19 | 20 | app.Flags = config.Flags 21 | app.Action = runWorker 22 | 23 | _ = app.Run(os.Args) 24 | } 25 | 26 | func runWorker(c *cli.Context) error { 27 | workerCLI := worker.NewCLI(c) 28 | canRun, err := workerCLI.Setup() 29 | if err != nil { 30 | return err 31 | } 32 | if canRun { 33 | workerCLI.Run() 34 | } 35 | return nil 36 | } 37 | -------------------------------------------------------------------------------- /cmd/travis-worker/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "testing" 4 | 5 | func TestNothing(t *testing.T) { 6 | if false { 7 | t.Fail() 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "os" 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "gopkg.in/urfave/cli.v1" 12 | ) 13 | 14 | func runAppTest(t *testing.T, args []string, action func(*cli.Context) error) { 15 | app := cli.NewApp() 16 | app.Flags = Flags 17 | app.Action = action 18 | _ = app.Run(append([]string{"whatever"}, args...)) 19 | } 20 | 21 | func TestFromCLIContext(t *testing.T) { 22 | runAppTest(t, []string{}, func(c *cli.Context) error { 23 | cfg := FromCLIContext(c) 24 | 25 | assert.NotNil(t, cfg) 26 | return nil 27 | }) 28 | } 29 | 30 | func TestFromCLIContext_SetsBoolFlags(t *testing.T) { 31 | runAppTest(t, []string{ 32 | "--build-api-insecure-skip-verify", 33 | "--build-fix-etc-hosts", 34 | "--build-fix-resolv-conf", 35 | "--build-paranoid", 36 | "--sentry-hook-errors", 37 | "--skip-shutdown-on-log-timeout", 38 | }, func(c *cli.Context) error { 39 | cfg := FromCLIContext(c) 40 | 41 | assert.True(t, cfg.BuildAPIInsecureSkipVerify, "BuildAPIInsecureSkipVerify") 42 | assert.True(t, cfg.BuildFixEtcHosts, "BuildFixEtcHosts") 43 | assert.True(t, cfg.BuildFixResolvConf, "BuildFixResolvConf") 44 | assert.True(t, cfg.BuildParanoid, "BuildParanoid") 45 | assert.True(t, cfg.SentryHookErrors, "SentryHookErrors") 46 | assert.True(t, cfg.SkipShutdownOnLogTimeout, "SkipShutdownOnLogTimeout") 47 | 48 | return nil 49 | }) 50 | } 51 | 52 | func TestFromCLIContext_SetsStringFlags(t *testing.T) { 53 | runAppTest(t, []string{ 54 | "--amqp-uri=amqp://", 55 | "--amqp-tls-cert=cert", 56 | "--base-dir=dir", 57 | "--build-api-uri=http://build/api", 58 | "--build-apt-cache=cache", 59 | "--build-cache-s3-access-key-id=id", 60 | "--build-cache-s3-bucket=bucket", 61 | "--build-cache-s3-region=region", 62 | "--build-cache-s3-scheme=scheme", 63 | "--build-cache-s3-secret-access-key=key", 64 | "--build-cache-type=type", 65 | "--build-npm-cache=cache", 66 | "--default-dist=dist", 67 | "--default-arch=arch", 68 | "--default-group=group", 69 | "--default-language=language", 70 | "--default-os=os", 71 | "--hostname=hostname", 72 | "--librato-email=email", 73 | "--librato-source=source", 74 | "--librato-token=token", 75 | "--logs-amqp-uri=amqp://logs", 76 | "--provider-name=provider", 77 | "--queue-name=name", 78 | "--queue-type=type", 79 | "--sentry-dsn=dsn", 80 | }, func(c *cli.Context) error { 81 | cfg := FromCLIContext(c) 82 | 83 | assert.Equal(t, "amqp://", cfg.AmqpURI, "AmqpURI") 84 | assert.Equal(t, "cert", cfg.AmqpTlsCert, "AmqpTlsCert") 85 | assert.Equal(t, "dir", cfg.BaseDir, "BaseDir") 86 | assert.Equal(t, "http://build/api", cfg.BuildAPIURI, "BuildAPIURI") 87 | assert.Equal(t, "cache", cfg.BuildAptCache, "BuildAptCache") 88 | assert.Equal(t, "id", cfg.BuildCacheS3AccessKeyID, "BuildCacheS3AccessKeyID") 89 | assert.Equal(t, "bucket", cfg.BuildCacheS3Bucket, "BuildCacheS3Bucket") 90 | assert.Equal(t, "scheme", cfg.BuildCacheS3Scheme, "BuildCacheS3Scheme") 91 | assert.Equal(t, "key", cfg.BuildCacheS3SecretAccessKey, "BuildCacheS3SecretAccessKey") 92 | assert.Equal(t, "type", cfg.BuildCacheType, "BuildCacheType") 93 | assert.Equal(t, "cache", cfg.BuildNpmCache, "BuildNpmCache") 94 | assert.Equal(t, "dist", cfg.DefaultDist, "DefaultDist") 95 | assert.Equal(t, "arch", cfg.DefaultArch, "DefaultArch") 96 | assert.Equal(t, "group", cfg.DefaultGroup, "DefaultGroup") 97 | assert.Equal(t, "language", cfg.DefaultLanguage, "DefaultLanguage") 98 | assert.Equal(t, "os", cfg.DefaultOS, "DefaultOS") 99 | assert.Equal(t, "hostname", cfg.Hostname, "Hostname") 100 | assert.Equal(t, "email", cfg.LibratoEmail, "LibratoEmail") 101 | assert.Equal(t, "source", cfg.LibratoSource, "LibratoSource") 102 | assert.Equal(t, "token", cfg.LibratoToken, "LibratoToken") 103 | assert.Equal(t, "amqp://logs", cfg.LogsAmqpURI, "LogsAmqpURI") 104 | assert.Equal(t, "provider", cfg.ProviderName, "ProviderName") 105 | assert.Equal(t, "name", cfg.QueueName, "QueueName") 106 | assert.Equal(t, "type", cfg.QueueType, "QueueType") 107 | assert.Equal(t, "dsn", cfg.SentryDSN, "SentryDSN") 108 | 109 | return nil 110 | }) 111 | } 112 | 113 | func TestFromCLIContext_SetsIntFlags(t *testing.T) { 114 | runAppTest(t, []string{ 115 | "--pool-size=42", 116 | }, func(c *cli.Context) error { 117 | cfg := FromCLIContext(c) 118 | 119 | assert.Equal(t, 42, cfg.PoolSize, "PoolSize") 120 | 121 | return nil 122 | }) 123 | } 124 | 125 | func TestFromCLIContext_SetsDurationFlags(t *testing.T) { 126 | runAppTest(t, []string{ 127 | "--file-polling-interval=42s", 128 | "--hard-timeout=2h", 129 | "--log-timeout=11m", 130 | "--script-upload-timeout=2m", 131 | "--startup-timeout=3m", 132 | "--build-cache-fetch-timeout=7m", 133 | "--build-cache-push-timeout=8m", 134 | }, func(c *cli.Context) error { 135 | cfg := FromCLIContext(c) 136 | 137 | assert.Equal(t, 42*time.Second, cfg.FilePollingInterval, "FilePollingInterval") 138 | assert.Equal(t, 2*time.Hour, cfg.HardTimeout, "HardTimeout") 139 | assert.Equal(t, 11*time.Minute, cfg.LogTimeout, "LogTimeout") 140 | assert.Equal(t, 2*time.Minute, cfg.ScriptUploadTimeout, "ScriptUploadTimeout") 141 | assert.Equal(t, 3*time.Minute, cfg.StartupTimeout, "StartupTimeout") 142 | assert.Equal(t, 7*time.Minute, cfg.BuildCacheFetchTimeout, "BuildCacheFetchTimeout") 143 | assert.Equal(t, 8*time.Minute, cfg.BuildCachePushTimeout, "BuildCachePushTimeout") 144 | 145 | return nil 146 | }) 147 | } 148 | 149 | func TestFromCLIContext_SetsProviderConfig(t *testing.T) { 150 | i := fmt.Sprintf("%v", rand.Int()) 151 | os.Setenv("TRAVIS_WORKER_FAKE_FOO", i) 152 | 153 | runAppTest(t, []string{ 154 | "--provider-name=fake", 155 | }, func(c *cli.Context) error { 156 | cfg := FromCLIContext(c) 157 | 158 | assert.NotNil(t, cfg.ProviderConfig) 159 | assert.Equal(t, i, cfg.ProviderConfig.Get("FOO")) 160 | 161 | return nil 162 | }) 163 | } 164 | -------------------------------------------------------------------------------- /config/provider_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "os" 7 | "sort" 8 | "strings" 9 | "sync" 10 | ) 11 | 12 | // ProviderConfig is the part of a configuration specific to a provider. 13 | type ProviderConfig struct { 14 | sync.Mutex 15 | 16 | cfgMap map[string]string 17 | } 18 | 19 | // GoString formats the ProviderConfig as valid Go syntax. This makes 20 | // ProviderConfig implement fmt.GoStringer. 21 | func (pc *ProviderConfig) GoString() string { 22 | return fmt.Sprintf("&ProviderConfig{cfgMap: %#v}", pc.cfgMap) 23 | } 24 | 25 | // Each loops over all configuration settings and calls the given function with 26 | // the key and value. The settings are sorted so f i called with the keys in 27 | // alphabetical order. 28 | func (pc *ProviderConfig) Each(f func(string, string)) { 29 | keys := []string{} 30 | for key := range pc.cfgMap { 31 | keys = append(keys, key) 32 | } 33 | 34 | sort.Strings(keys) 35 | 36 | for _, key := range keys { 37 | f(key, pc.Get(key)) 38 | } 39 | } 40 | 41 | // Get the value of a setting with the given key. The empty string is returned 42 | // if the setting could not be found. 43 | func (pc *ProviderConfig) Get(key string) string { 44 | pc.Lock() 45 | defer pc.Unlock() 46 | 47 | if value, ok := pc.cfgMap[key]; ok { 48 | return value 49 | } 50 | 51 | return "" 52 | } 53 | 54 | // Set the value of a setting with the given key. 55 | func (pc *ProviderConfig) Set(key, value string) { 56 | pc.Lock() 57 | defer pc.Unlock() 58 | 59 | pc.cfgMap[key] = value 60 | } 61 | 62 | // Unset removes the given key from the config map 63 | func (pc *ProviderConfig) Unset(key string) { 64 | pc.Lock() 65 | defer pc.Unlock() 66 | 67 | delete(pc.cfgMap, key) 68 | } 69 | 70 | // IsSet returns true if a setting with the given key exists, or false if it 71 | // does not. 72 | func (pc *ProviderConfig) IsSet(key string) bool { 73 | pc.Lock() 74 | defer pc.Unlock() 75 | 76 | _, ok := pc.cfgMap[key] 77 | return ok 78 | } 79 | 80 | // ProviderConfigFromEnviron dynamically builds a *ProviderConfig from the 81 | // environment by loading values from keys with prefixes that match either the 82 | // uppercase provider name + "_" or "TRAVIS_WORKER_" + uppercase provider name + 83 | // "_", e.g., for provider "foo": 84 | // 85 | // env: TRAVIS_WORKER_FOO_BAR=ham FOO_BAZ=bones 86 | // map equiv: {"BAR": "ham", "BAZ": "bones"} 87 | func ProviderConfigFromEnviron(providerName string) *ProviderConfig { 88 | upperProvider := strings.ToUpper(providerName) 89 | 90 | pc := &ProviderConfig{cfgMap: map[string]string{}} 91 | 92 | for _, prefix := range []string{ 93 | "TRAVIS_WORKER_" + upperProvider + "_", 94 | upperProvider + "_", 95 | } { 96 | for _, e := range os.Environ() { 97 | if strings.HasPrefix(e, prefix) { 98 | pair := strings.SplitN(e, "=", 2) 99 | 100 | key := strings.ToUpper(strings.TrimPrefix(pair[0], prefix)) 101 | value := pair[1] 102 | if !strings.HasSuffix(key, "ACCOUNT_JSON") { 103 | unescapedValue, err := url.QueryUnescape(value) 104 | if err == nil { 105 | value = unescapedValue 106 | } 107 | } 108 | 109 | pc.Set(key, value) 110 | } 111 | } 112 | } 113 | 114 | return pc 115 | } 116 | 117 | // ProviderConfigFromMap creates a provider configuration backed by the given 118 | // map. Useful for testing a provider. 119 | func ProviderConfigFromMap(cfgMap map[string]string) *ProviderConfig { 120 | return &ProviderConfig{cfgMap: cfgMap} 121 | } 122 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package worker implements the core of Worker. 3 | 4 | The main coordinating component in the core is the Processor, and is a good 5 | place to start when exploring the code. It gets the jobs off the job queue and 6 | calls the right things in order to run it. The ProcessorPool starts up the 7 | required number of Processors to get the concurrency that's wanted for a single 8 | Worker instance. 9 | */ 10 | package worker 11 | -------------------------------------------------------------------------------- /errors/errors.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | type JobAbortError interface { 4 | UserFacingErrorMessage() string 5 | } 6 | 7 | type wrappedJobAbortError struct { 8 | err error 9 | } 10 | 11 | func NewWrappedJobAbortError(err error) error { 12 | return &wrappedJobAbortError{ 13 | err: err, 14 | } 15 | } 16 | 17 | // we do not implement Cause(), because we want 18 | // errors.Cause() to bottom out here 19 | 20 | func (abortErr wrappedJobAbortError) Error() string { 21 | return abortErr.err.Error() 22 | } 23 | 24 | func (abortErr wrappedJobAbortError) UserFacingErrorMessage() string { 25 | return abortErr.err.Error() 26 | } 27 | -------------------------------------------------------------------------------- /example-payload-premium.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "test", 3 | "vm_type": "premium", 4 | "vm_config": { 5 | "gpu_count": 1, 6 | "gpu_type": "nvidia-tesla-v100", 7 | "zone": "us-central1-a" 8 | }, 9 | "queue": "builds.gce", 10 | "config": { 11 | "os": "linux", 12 | "dist": "trusty", 13 | "sudo": "required", 14 | "group": "stable", 15 | "python": "2.7", 16 | "script": [ 17 | "tox" 18 | ], 19 | ".result": "configured", 20 | "language": "python", 21 | "before_install": [ 22 | "pip install ." 23 | ] 24 | }, 25 | "env_vars": [ 26 | { 27 | "name": "DOCKER_REGISTRY_URL", 28 | "value": "hub.docker.com", 29 | "public": true 30 | } 31 | ], 32 | "job": { 33 | "id": 374404043, 34 | "number": "64.1", 35 | "commit": "f692a0340c210b1e53a4aa06e07ba7b688fde2c3", 36 | "commit_range": "176d0f413640...f692a0340c21", 37 | "commit_message": "also pull ruby:2.3.5", 38 | "branch": "master", 39 | "ref": null, 40 | "tag": null, 41 | "pull_request": false, 42 | "state": "passed", 43 | "secure_env_enabled": true, 44 | "secure_env_removed": false, 45 | "debug_options": {}, 46 | "queued_at": "2018-05-21T14:34:58Z", 47 | "allow_failure": false, 48 | "stage_name": null 49 | }, 50 | "source": { 51 | "id": 374404042, 52 | "number": "64", 53 | "event_type": "push" 54 | }, 55 | "repository": { 56 | "id": 8660711, 57 | "github_id": 40738658, 58 | "private": false, 59 | "slug": "soulshake/startup-threads-cli", 60 | "source_url": "https://github.com/soulshake/startup-threads-cli.git", 61 | "source_host": "github.com", 62 | "api_url": "https://api.github.com/repos/soulshake/startup-threads-cli", 63 | "last_build_id": 374404042, 64 | "last_build_number": "64", 65 | "last_build_started_at": "2018-05-21T14:35:30Z", 66 | "last_build_finished_at": "2018-05-21T14:36:55Z", 67 | "last_build_duration": 143, 68 | "last_build_state": "passed", 69 | "default_branch": "master", 70 | "description": "Send t-shirts from your Startup Threads account from the command line" 71 | }, 72 | "ssh_key": null, 73 | "timeouts": { 74 | "hard_limit": 3000, 75 | "log_silence": null 76 | }, 77 | "enterprise": false, 78 | "prefer_https": false 79 | } 80 | -------------------------------------------------------------------------------- /example-payload.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "test", 3 | "vm_type": "default", 4 | "queue": "builds.ec2", 5 | "config": { 6 | "language": "ruby", 7 | "script": "echo Hello World", 8 | ".result": "configured", 9 | "os": "linux", 10 | "group": "stable", 11 | "dist": "trusty" 12 | }, 13 | "env_vars": [], 14 | "job": { 15 | "id": 721270, 16 | "number": "445.1", 17 | "commit": "fad82cc76a5df58e749b82f7f2ab10ede52ba730", 18 | "commit_range": "0faba6fc0a3e...fad82cc76a5d", 19 | "commit_message": "updating contents of RANDOM", 20 | "branch": "master", 21 | "ref": null, 22 | "tag": null, 23 | "pull_request": false, 24 | "state": "passed", 25 | "secure_env_enabled": true, 26 | "secure_env_removed": false, 27 | "debug_options": {}, 28 | "queued_at": "2018-05-14T14:26:50Z", 29 | "allow_failure": false, 30 | "stage_name": null 31 | }, 32 | "source": { 33 | "id": 721269, 34 | "number": "445", 35 | "event_type": "push" 36 | }, 37 | "repository": { 38 | "id": 44461, 39 | "github_id": 55685806, 40 | "private": false, 41 | "slug": "igorwwwwwwwwwwwwwwwwwwww/hello-world", 42 | "source_url": "https://github.com/igorwwwwwwwwwwwwwwwwwwww/hello-world.git", 43 | "source_host": "github.com", 44 | "api_url": "https://api.github.com/repos/igorwwwwwwwwwwwwwwwwwwww/hello-world", 45 | "last_build_id": 721269, 46 | "last_build_number": "445", 47 | "last_build_started_at": "2018-05-14T14:26:52Z", 48 | "last_build_finished_at": "2018-05-14T14:27:06Z", 49 | "last_build_duration": 14, 50 | "last_build_state": "passed", 51 | "default_branch": "master", 52 | "description": "Hi!" 53 | }, 54 | "ssh_key": null, 55 | "timeouts": { 56 | "hard_limit": 3000, 57 | "log_silence": null 58 | }, 59 | "cache_settings": { 60 | "fetch_timeout": 1200, 61 | "push_timeout": 7200, 62 | "s3": { 63 | "access_key_id": "nopenotanaccesskey", 64 | "aws_signature_version": "4", 65 | "bucket": "travis-cache-staging-org", 66 | "hostname": "s3.amazonaws.com", 67 | "secret_access_key": "vEry_SeCrEt_AcCeSs_KeY" 68 | }, 69 | "type": "s3" 70 | }, 71 | "enterprise": false, 72 | "prefer_https": false 73 | } 74 | -------------------------------------------------------------------------------- /file_job.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | "time" 8 | 9 | gocontext "context" 10 | 11 | "github.com/bitly/go-simplejson" 12 | "github.com/sirupsen/logrus" 13 | "github.com/travis-ci/worker/backend" 14 | "github.com/travis-ci/worker/context" 15 | "github.com/travis-ci/worker/metrics" 16 | ) 17 | 18 | type fileJob struct { 19 | createdFile string 20 | receivedFile string 21 | startedFile string 22 | finishedFile string 23 | logFile string 24 | bytes []byte 25 | payload *JobPayload 26 | rawPayload *simplejson.Json 27 | startAttributes *backend.StartAttributes 28 | finishState FinishState 29 | requeued bool 30 | } 31 | 32 | func (j *fileJob) Payload() *JobPayload { 33 | return j.payload 34 | } 35 | 36 | func (j *fileJob) RawPayload() *simplejson.Json { 37 | return j.rawPayload 38 | } 39 | 40 | func (j *fileJob) StartAttributes() *backend.StartAttributes { 41 | return j.startAttributes 42 | } 43 | 44 | func (j *fileJob) FinishState() FinishState { 45 | return j.finishState 46 | } 47 | 48 | func (j *fileJob) Requeued() bool { 49 | return j.requeued 50 | } 51 | 52 | func (j *fileJob) Received(_ gocontext.Context) error { 53 | return os.Rename(j.createdFile, j.receivedFile) 54 | } 55 | 56 | func (j *fileJob) Started(_ gocontext.Context) error { 57 | return os.Rename(j.receivedFile, j.startedFile) 58 | } 59 | 60 | func (j *fileJob) Error(ctx gocontext.Context, errMessage string) error { 61 | log, err := j.LogWriter(ctx, time.Minute) 62 | if err != nil { 63 | return err 64 | } 65 | 66 | _, err = log.WriteAndClose([]byte(errMessage)) 67 | if err != nil { 68 | return err 69 | } 70 | 71 | return j.Finish(ctx, FinishStateErrored) 72 | } 73 | 74 | func (j *fileJob) Requeue(ctx gocontext.Context) error { 75 | context.LoggerFromContext(ctx).WithField("self", "file_job").Info("requeueing job") 76 | 77 | metrics.Mark("worker.job.requeue") 78 | 79 | j.requeued = true 80 | 81 | var err error 82 | 83 | for _, fname := range []string{ 84 | j.receivedFile, 85 | j.startedFile, 86 | j.finishedFile, 87 | } { 88 | err = os.Rename(fname, j.createdFile) 89 | if err == nil { 90 | return nil 91 | } 92 | } 93 | 94 | return err 95 | } 96 | 97 | func (j *fileJob) Finish(ctx gocontext.Context, state FinishState) error { 98 | context.LoggerFromContext(ctx).WithFields(logrus.Fields{ 99 | "state": state, 100 | "self": "file_job", 101 | }).Info("finishing job") 102 | 103 | metrics.Mark(fmt.Sprintf("travis.worker.job.finish.%s", state)) 104 | 105 | err := os.Rename(j.startedFile, j.finishedFile) 106 | if err != nil { 107 | return err 108 | } 109 | 110 | return os.WriteFile(strings.Replace(j.finishedFile, ".json", ".state", -1), 111 | []byte(state), os.FileMode(0644)) 112 | } 113 | 114 | func (j *fileJob) LogWriter(ctx gocontext.Context, defaultLogTimeout time.Duration) (LogWriter, error) { 115 | logTimeout := time.Duration(j.payload.Timeouts.LogSilence) * time.Second 116 | if logTimeout == 0 { 117 | logTimeout = defaultLogTimeout 118 | } 119 | 120 | return newFileLogWriter(ctx, j.logFile, logTimeout) 121 | } 122 | 123 | func (j *fileJob) SetupContext(ctx gocontext.Context) gocontext.Context { return ctx } 124 | 125 | func (j *fileJob) Name() string { return "file" } 126 | -------------------------------------------------------------------------------- /file_job_queue.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | "time" 10 | 11 | gocontext "context" 12 | 13 | "github.com/bitly/go-simplejson" 14 | "github.com/sirupsen/logrus" 15 | "github.com/travis-ci/worker/backend" 16 | "github.com/travis-ci/worker/context" 17 | ) 18 | 19 | // FileJobQueue is a JobQueue that uses directories for input, state, and output 20 | type FileJobQueue struct { 21 | queue string 22 | pollingInterval time.Duration 23 | 24 | buildJobChan chan Job 25 | 26 | baseDir string 27 | createdDir string 28 | receivedDir string 29 | startedDir string 30 | finishedDir string 31 | logDir string 32 | 33 | DefaultLanguage, DefaultDist, DefaultArch, DefaultGroup, DefaultOS string 34 | } 35 | 36 | // NewFileJobQueue creates a *FileJobQueue from a base directory and queue name 37 | func NewFileJobQueue(baseDir, queue string, pollingInterval time.Duration) (*FileJobQueue, error) { 38 | _, err := os.Stat(baseDir) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | fd, err := os.Create(filepath.Join(baseDir, ".write-test")) 44 | if err != nil { 45 | return nil, err 46 | } 47 | 48 | defer fd.Close() 49 | 50 | createdDir := filepath.Join(baseDir, queue, "10-created.d") 51 | receivedDir := filepath.Join(baseDir, queue, "30-received.d") 52 | startedDir := filepath.Join(baseDir, queue, "50-started.d") 53 | finishedDir := filepath.Join(baseDir, queue, "70-finished.d") 54 | logDir := filepath.Join(baseDir, queue, "log") 55 | 56 | for _, dirname := range []string{createdDir, receivedDir, startedDir, finishedDir, logDir} { 57 | err := os.MkdirAll(dirname, os.FileMode(0755)) 58 | if err != nil { 59 | return nil, err 60 | } 61 | } 62 | 63 | return &FileJobQueue{ 64 | queue: queue, 65 | pollingInterval: pollingInterval, 66 | 67 | baseDir: baseDir, 68 | createdDir: createdDir, 69 | receivedDir: receivedDir, 70 | startedDir: startedDir, 71 | finishedDir: finishedDir, 72 | logDir: logDir, 73 | }, nil 74 | } 75 | 76 | // Jobs returns a channel of jobs from the created directory 77 | func (f *FileJobQueue) Jobs(ctx gocontext.Context) (<-chan Job, error) { 78 | if f.buildJobChan == nil { 79 | f.buildJobChan = make(chan Job) 80 | go f.pollInDirForJobs(ctx) 81 | } 82 | return f.buildJobChan, nil 83 | } 84 | 85 | func (f *FileJobQueue) pollInDirForJobs(ctx gocontext.Context) { 86 | for { 87 | f.pollInDirTick(ctx) 88 | time.Sleep(f.pollingInterval) 89 | } 90 | } 91 | 92 | func (f *FileJobQueue) pollInDirTick(ctx gocontext.Context) { 93 | logger := context.LoggerFromContext(ctx).WithField("self", "file_job_queue") 94 | entries, err := os.ReadDir(f.createdDir) 95 | if err != nil { 96 | logger.WithField("err", err).Error("input directory read error") 97 | return 98 | } 99 | 100 | logger.WithFields(logrus.Fields{ 101 | "entries": entries, 102 | "file_job_queue": fmt.Sprintf("%p", f), 103 | }).Debug("entries") 104 | 105 | for _, entry := range entries { 106 | if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { 107 | continue 108 | } 109 | 110 | buildJob := &fileJob{ 111 | createdFile: filepath.Join(f.createdDir, entry.Name()), 112 | payload: &JobPayload{}, 113 | startAttributes: &backend.StartAttributes{}, 114 | } 115 | startAttrs := &jobPayloadStartAttrs{Config: &backend.StartAttributes{}} 116 | 117 | fb, err := os.ReadFile(buildJob.createdFile) 118 | if err != nil { 119 | logger.WithField("err", err).Error("input file read error") 120 | continue 121 | } 122 | 123 | err = json.Unmarshal(fb, buildJob.payload) 124 | if err != nil { 125 | logger.WithField("err", err).Error("payload JSON parse error, skipping") 126 | continue 127 | } 128 | 129 | err = json.Unmarshal(fb, &startAttrs) 130 | if err != nil { 131 | logger.WithField("err", err).Error("start attributes JSON parse error, skipping") 132 | continue 133 | } 134 | 135 | buildJob.rawPayload, err = simplejson.NewJson(fb) 136 | if err != nil { 137 | logger.WithField("err", err).Error("raw payload JSON parse error, skipping") 138 | continue 139 | } 140 | 141 | buildJob.startAttributes = startAttrs.Config 142 | buildJob.startAttributes.VMConfig = buildJob.payload.VMConfig 143 | buildJob.startAttributes.VMType = buildJob.payload.VMType 144 | buildJob.startAttributes.SetDefaults(f.DefaultLanguage, f.DefaultDist, f.DefaultArch, f.DefaultGroup, f.DefaultOS, VMTypeDefault, VMConfigDefault) 145 | buildJob.receivedFile = filepath.Join(f.receivedDir, entry.Name()) 146 | buildJob.startedFile = filepath.Join(f.startedDir, entry.Name()) 147 | buildJob.finishedFile = filepath.Join(f.finishedDir, entry.Name()) 148 | buildJob.logFile = filepath.Join(f.logDir, strings.Replace(entry.Name(), ".json", ".log", -1)) 149 | buildJob.bytes = fb 150 | 151 | f.buildJobChan <- buildJob 152 | } 153 | } 154 | 155 | // Name returns the name of this queue type, wow! 156 | func (q *FileJobQueue) Name() string { 157 | return "file" 158 | } 159 | 160 | // Cleanup is a no-op 161 | func (f *FileJobQueue) Cleanup() error { 162 | return nil 163 | } 164 | -------------------------------------------------------------------------------- /file_log_writer.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "os" 5 | "time" 6 | 7 | gocontext "context" 8 | ) 9 | 10 | type fileLogWriter struct { 11 | ctx gocontext.Context 12 | logFile string 13 | fd *os.File 14 | 15 | timer *time.Timer 16 | timeout time.Duration 17 | } 18 | 19 | func newFileLogWriter(ctx gocontext.Context, logFile string, timeout time.Duration) (LogWriter, error) { 20 | fd, err := os.Create(logFile) 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | return &fileLogWriter{ 26 | ctx: ctx, 27 | logFile: logFile, 28 | fd: fd, 29 | 30 | timer: time.NewTimer(time.Hour), 31 | timeout: timeout, 32 | }, nil 33 | } 34 | 35 | func (w *fileLogWriter) Write(b []byte) (int, error) { 36 | return w.fd.Write(b) 37 | } 38 | 39 | func (w *fileLogWriter) Close() error { 40 | return w.fd.Close() 41 | } 42 | 43 | func (w *fileLogWriter) SetMaxLogLength(n int) {} 44 | 45 | func (w *fileLogWriter) SetJobStarted(meta *JobStartedMeta) {} 46 | 47 | func (w *fileLogWriter) SetCancelFunc(cancel gocontext.CancelFunc) {} 48 | 49 | func (w *fileLogWriter) MaxLengthReached() bool { 50 | return false 51 | } 52 | 53 | func (w *fileLogWriter) Timeout() <-chan time.Time { 54 | return w.timer.C 55 | } 56 | 57 | func (w *fileLogWriter) WriteAndClose(b []byte) (int, error) { 58 | n, err := w.Write(b) 59 | if err != nil { 60 | return n, err 61 | } 62 | 63 | err = w.Close() 64 | return n, err 65 | } 66 | -------------------------------------------------------------------------------- /help.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "sort" 7 | 8 | "github.com/travis-ci/worker/backend" 9 | "gopkg.in/urfave/cli.v1" 10 | ) 11 | 12 | var ( 13 | cliHelpPrinter = cli.HelpPrinter 14 | ) 15 | 16 | const ( 17 | providerHelpHeader = ` 18 | All provider options must be given as environment variables of the form: 19 | 20 | $[TRAVIS_WORKER_]{UPCASE_PROVIDER_NAME}_{UPCASE_UNDERSCORED_KEY} 21 | ^------------^ 22 | optional namespace 23 | 24 | e.g.: 25 | 26 | TRAVIS_WORKER_DOCKER_HOST='tcp://127.0.0.1:4243' 27 | TRAVIS_WORKER_DOCKER_PRIVILEGED='true' 28 | 29 | ` 30 | ) 31 | 32 | func init() { 33 | cli.HelpPrinter = helpPrinter 34 | } 35 | 36 | func helpPrinter(w io.Writer, templ string, data interface{}) { 37 | cliHelpPrinter(w, templ, data) 38 | 39 | fmt.Fprint(w, providerHelpHeader) 40 | 41 | margin := 4 42 | maxLen := 0 43 | 44 | backend.EachBackend(func(b *backend.Backend) { 45 | for itemKey := range b.ProviderHelp { 46 | if len(itemKey) > maxLen { 47 | maxLen = len(itemKey) 48 | } 49 | } 50 | }) 51 | 52 | itemFmt := fmt.Sprintf("%%%ds - %%s\n", maxLen+margin) 53 | 54 | backend.EachBackend(func(b *backend.Backend) { 55 | fmt.Fprintf(w, "\n%s provider help:\n\n", b.HumanReadableName) 56 | 57 | sortedKeys := []string{} 58 | for key := range b.ProviderHelp { 59 | sortedKeys = append(sortedKeys, key) 60 | } 61 | 62 | sort.Strings(sortedKeys) 63 | 64 | for _, key := range sortedKeys { 65 | fmt.Printf(itemFmt, key, b.ProviderHelp[key]) 66 | } 67 | }) 68 | 69 | fmt.Println("") 70 | } 71 | -------------------------------------------------------------------------------- /http_job_queue_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/http/httptest" 7 | "net/url" 8 | "strings" 9 | "testing" 10 | "time" 11 | 12 | gocontext "context" 13 | 14 | "github.com/stretchr/testify/assert" 15 | ) 16 | 17 | func TestHTTPJobQueue(t *testing.T) { 18 | hjq, err := NewHTTPJobQueue(nil, "test", "fake", "fake", nil) 19 | assert.Nil(t, err) 20 | assert.NotNil(t, hjq) 21 | } 22 | 23 | func TestHTTPJobQueue_Jobs(t *testing.T) { 24 | mux := http.NewServeMux() 25 | mux.HandleFunc(`/jobs/pop`, func(w http.ResponseWriter, req *http.Request) { 26 | w.WriteHeader(http.StatusOK) 27 | fmt.Fprintf(w, `{"job_id":"100001"}`) 28 | }) 29 | mux.HandleFunc(`/jobs/100001/claim`, func(w http.ResponseWriter, req *http.Request) { 30 | w.WriteHeader(http.StatusOK) 31 | }) 32 | mux.HandleFunc(`/jobs/100001`, func(w http.ResponseWriter, req *http.Request) { 33 | w.WriteHeader(http.StatusOK) 34 | fmt.Fprint(w, strings.Replace(`{ 35 | "data": { 36 | "type": "job", 37 | "job": { 38 | "id": 100001, 39 | "number": "42.1", 40 | "queued_at": "2011-04-01T11:05:55Z" 41 | }, 42 | "source": { 43 | "id": 100001, 44 | "number": "42" 45 | }, 46 | "repository": { 47 | "id": 8490324, 48 | "slug": "travis-ci/nonexistent-repository" 49 | }, 50 | "uuid": "fafafaf", 51 | "config": {}, 52 | "vm_type": "test", 53 | "meta": { 54 | "state_update_count": 0 55 | } 56 | } 57 | }`, "\t", " ", -1)) 58 | }) 59 | 60 | mux.HandleFunc(`/`, func(w http.ResponseWriter, req *http.Request) { 61 | t.Fatalf("unknown URL requested: %#v", req.URL.Path) 62 | }) 63 | jobBoardServer := httptest.NewServer(mux) 64 | defer jobBoardServer.Close() 65 | 66 | jobBoardURL, _ := url.Parse(jobBoardServer.URL) 67 | hjq, err := NewHTTPJobQueue(jobBoardURL, "test", "fake", "fake", nil) 68 | assert.Nil(t, err) 69 | assert.NotNil(t, hjq) 70 | 71 | ctx := gocontext.TODO() 72 | buildJobChan, err := hjq.Jobs(ctx) 73 | assert.Nil(t, err) 74 | assert.NotNil(t, buildJobChan) 75 | 76 | select { 77 | case job := <-buildJobChan: 78 | assert.NotNil(t, job) 79 | case <-time.After(time.Second): 80 | t.Fatalf("failed to recv job") 81 | } 82 | } 83 | 84 | func TestHTTPJobQueue_Name(t *testing.T) { 85 | hjq, err := NewHTTPJobQueue(nil, "test", "fake", "fake", nil) 86 | assert.Nil(t, err) 87 | assert.Equal(t, "http", hjq.Name()) 88 | } 89 | 90 | func TestHTTPJobQueue_Cleanup(t *testing.T) { 91 | hjq, err := NewHTTPJobQueue(nil, "test", "fake", "fake", nil) 92 | assert.Nil(t, err) 93 | assert.Nil(t, hjq.Cleanup()) 94 | } 95 | -------------------------------------------------------------------------------- /http_job_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | "net/http/httptest" 8 | "strings" 9 | "testing" 10 | 11 | gocontext "context" 12 | 13 | "github.com/bitly/go-simplejson" 14 | "github.com/travis-ci/worker/backend" 15 | ) 16 | 17 | func newTestHTTPJob(t *testing.T) *httpJob { 18 | jobPayload := &JobPayload{ 19 | Type: "job:test", 20 | Job: JobJobPayload{ 21 | ID: uint64(123), 22 | Number: "1", 23 | }, 24 | Build: BuildPayload{ 25 | ID: uint64(456), 26 | Number: "1", 27 | }, 28 | UUID: "870f986d-a88f-4801-86cc-3d2dbc6c80da", 29 | Config: map[string]interface{}{}, 30 | Timeouts: TimeoutsPayload{ 31 | HardLimit: uint64(9000), 32 | LogSilence: uint64(8001), 33 | }, 34 | } 35 | jsp := jobScriptPayload{ 36 | Name: "main", 37 | Encoding: "base64", 38 | Content: "IyEvdXNyL2Jpbi9lbnYgYmFzaAplY2hvIHd1dAo=", 39 | } 40 | startAttributes := &backend.StartAttributes{ 41 | Language: "go", 42 | Dist: "trusty", 43 | } 44 | 45 | body, err := json.Marshal(jobPayload) 46 | if err != nil { 47 | t.Error(err) 48 | } 49 | 50 | rawPayload, err := simplejson.NewJson(body) 51 | if err != nil { 52 | t.Error(err) 53 | } 54 | 55 | return &httpJob{ 56 | payload: &httpJobPayload{ 57 | Data: jobPayload, 58 | JobScript: jsp, 59 | JWT: "huh", 60 | ImageName: "yeap", 61 | }, 62 | rawPayload: rawPayload, 63 | startAttributes: startAttributes, 64 | deleteSelf: func(_ gocontext.Context) error { return nil }, 65 | } 66 | } 67 | 68 | func TestHTTPJob(t *testing.T) { 69 | job := newTestHTTPJob(t) 70 | 71 | if job.Payload() == nil { 72 | t.Fatalf("payload not set") 73 | } 74 | 75 | if job.RawPayload() == nil { 76 | t.Fatalf("raw payload not set") 77 | } 78 | 79 | if job.StartAttributes() == nil { 80 | t.Fatalf("start attributes not set") 81 | } 82 | 83 | if job.GoString() == "" { 84 | t.Fatalf("go string is empty") 85 | } 86 | } 87 | 88 | func TestHTTPJob_GoString(t *testing.T) { 89 | job := newTestHTTPJob(t) 90 | 91 | str := job.GoString() 92 | 93 | if !strings.HasPrefix(str, "&httpJob{") && !strings.HasSuffix(str, "}") { 94 | t.Fatalf("go string has unexpected format: %q", str) 95 | } 96 | } 97 | 98 | func TestHTTPJob_Error(t *testing.T) { 99 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 100 | if r.Method == "PUT" || r.Method == "DELETE" { 101 | w.WriteHeader(http.StatusNoContent) 102 | } 103 | })) 104 | defer ts.Close() 105 | job := newTestHTTPJob(t) 106 | job.payload.JobStateURL = ts.URL 107 | job.payload.JobPartsURL = ts.URL 108 | 109 | err := job.Error(gocontext.TODO(), "wat") 110 | if err != nil { 111 | t.Error(err) 112 | } 113 | } 114 | 115 | func TestHTTPJob_Requeue(t *testing.T) { 116 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 117 | fmt.Fprintln(w, "hai") 118 | })) 119 | defer ts.Close() 120 | 121 | job := newTestHTTPJob(t) 122 | job.payload.JobStateURL = ts.URL 123 | job.payload.JobPartsURL = ts.URL 124 | 125 | ctx := gocontext.TODO() 126 | 127 | err := job.Requeue(ctx) 128 | if err != nil { 129 | t.Error(err) 130 | } 131 | } 132 | 133 | func TestHTTPJob_Received(t *testing.T) { 134 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 135 | fmt.Fprintln(w, "hai") 136 | })) 137 | defer ts.Close() 138 | job := newTestHTTPJob(t) 139 | job.payload.JobStateURL = ts.URL 140 | job.payload.JobPartsURL = ts.URL 141 | 142 | err := job.Received(gocontext.TODO()) 143 | if err != nil { 144 | t.Error(err) 145 | } 146 | } 147 | 148 | func TestHTTPJob_Started(t *testing.T) { 149 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 150 | fmt.Fprintln(w, "hai") 151 | })) 152 | defer ts.Close() 153 | job := newTestHTTPJob(t) 154 | job.payload.JobStateURL = ts.URL 155 | job.payload.JobPartsURL = ts.URL 156 | 157 | err := job.Started(gocontext.TODO()) 158 | if err != nil { 159 | t.Error(err) 160 | } 161 | } 162 | 163 | func TestHTTPJob_Finish(t *testing.T) { 164 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 165 | if r.Method == "DELETE" { 166 | w.WriteHeader(http.StatusNoContent) 167 | } 168 | fmt.Fprintln(w, "hai") 169 | })) 170 | defer ts.Close() 171 | 172 | job := newTestHTTPJob(t) 173 | job.payload.JobStateURL = ts.URL 174 | job.payload.JobPartsURL = ts.URL 175 | 176 | ctx := gocontext.TODO() 177 | 178 | err := job.Finish(ctx, FinishStatePassed) 179 | if err != nil { 180 | t.Error(err) 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /http_log_part_sink_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "testing" 7 | 8 | gocontext "context" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestNewHTTPLogPartSink(t *testing.T) { 14 | ctx, cancel := gocontext.WithCancel(gocontext.TODO()) 15 | cancel() 16 | lps := newHTTPLogPartSink( 17 | ctx, 18 | "http://example.org/log-parts/multi", 19 | uint64(1000)) 20 | 21 | assert.NotNil(t, lps) 22 | } 23 | 24 | func TestHTTPLogPartSink_flush(t *testing.T) { 25 | lss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 26 | if r.Method == "POST" { 27 | w.WriteHeader(http.StatusNoContent) 28 | return 29 | } 30 | w.WriteHeader(http.StatusNotImplemented) 31 | })) 32 | defer lss.Close() 33 | 34 | httpLogPartSinksByURLMutex.Lock() 35 | httpLogPartSinksByURL[lss.URL] = newHTTPLogPartSink(gocontext.TODO(), lss.URL, uint64(1000)) 36 | httpLogPartSinksByURLMutex.Unlock() 37 | 38 | ctx := gocontext.TODO() 39 | lps := newHTTPLogPartSink(ctx, lss.URL, uint64(10)) 40 | lps.flush(gocontext.TODO()) 41 | _ = lps.Add(ctx, &httpLogPart{ 42 | JobID: uint64(4), 43 | Content: "wat", 44 | Number: 3, 45 | Final: false, 46 | }) 47 | 48 | lps.partsBufferMutex.Lock() 49 | assert.Len(t, lps.partsBuffer, 1) 50 | lps.partsBufferMutex.Unlock() 51 | 52 | lps.flush(ctx) 53 | 54 | lps.partsBufferMutex.Lock() 55 | assert.Len(t, lps.partsBuffer, 0) 56 | lps.partsBufferMutex.Unlock() 57 | } 58 | -------------------------------------------------------------------------------- /http_log_writer.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | gocontext "context" 8 | 9 | "github.com/sirupsen/logrus" 10 | "github.com/travis-ci/worker/context" 11 | ) 12 | 13 | type httpLogPart struct { 14 | Content string 15 | Final bool 16 | JobID uint64 17 | Number uint64 18 | Token string 19 | } 20 | 21 | type httpLogWriter struct { 22 | ctx gocontext.Context 23 | cancel gocontext.CancelFunc 24 | jobID uint64 25 | authToken string 26 | 27 | closeChan chan struct{} 28 | 29 | logPartNumber uint64 30 | 31 | bytesWritten int 32 | maxLength int 33 | maxLengthReached bool 34 | 35 | lps *httpLogPartSink 36 | 37 | timer *time.Timer 38 | timeout time.Duration 39 | } 40 | 41 | func newHTTPLogWriter(ctx gocontext.Context, url string, authToken string, jobID uint64, timeout time.Duration) (*httpLogWriter, error) { 42 | writer := &httpLogWriter{ 43 | ctx: context.FromComponent(ctx, "log_writer"), 44 | jobID: jobID, 45 | authToken: authToken, 46 | closeChan: make(chan struct{}), 47 | timer: time.NewTimer(time.Hour), 48 | timeout: timeout, 49 | lps: getHTTPLogPartSinkByURL(url), 50 | } 51 | 52 | return writer, nil 53 | } 54 | 55 | func (w *httpLogWriter) Write(p []byte) (int, error) { 56 | if w.closed() { 57 | return 0, fmt.Errorf("attempted write to closed log") 58 | } 59 | 60 | logger := context.LoggerFromContext(w.ctx).WithFields(logrus.Fields{ 61 | "self": "http_log_writer", 62 | "inst": fmt.Sprintf("%p", w), 63 | }) 64 | 65 | logger.WithFields(logrus.Fields{ 66 | "length": len(p), 67 | "bytes": string(p), 68 | }).Debug("begin writing bytes") 69 | 70 | w.timer.Reset(w.timeout) 71 | 72 | w.bytesWritten += len(p) 73 | if w.bytesWritten > w.maxLength { 74 | logger.Info("wrote past maximum log length - cancelling context") 75 | w.maxLengthReached = true 76 | if w.cancel == nil { 77 | logger.Error("cancel function does not exist") 78 | } else { 79 | w.cancel() 80 | } 81 | return 0, nil 82 | } 83 | 84 | err := w.lps.Add(w.ctx, &httpLogPart{ 85 | Content: string(p), 86 | JobID: w.jobID, 87 | Number: w.logPartNumber, 88 | Token: w.authToken, 89 | }) 90 | if err != nil { 91 | context.LoggerFromContext(w.ctx).WithFields(logrus.Fields{ 92 | "err": err, 93 | "self": "http_log_writer", 94 | }).Error("could not add log part to sink") 95 | return 0, err 96 | } 97 | 98 | w.logPartNumber++ 99 | return len(p), err 100 | } 101 | 102 | func (w *httpLogWriter) Close() error { 103 | if w.closed() { 104 | return nil 105 | } 106 | 107 | w.timer.Stop() 108 | 109 | close(w.closeChan) 110 | 111 | err := w.lps.Add(w.ctx, &httpLogPart{ 112 | Final: true, 113 | JobID: w.jobID, 114 | Number: w.logPartNumber, 115 | Token: w.authToken, 116 | }) 117 | 118 | if err != nil { 119 | context.LoggerFromContext(w.ctx).WithFields(logrus.Fields{ 120 | "err": err, 121 | "self": "http_log_writer", 122 | }).Error("could not add log part to sink") 123 | return err 124 | } 125 | 126 | w.logPartNumber++ 127 | return nil 128 | } 129 | 130 | func (w *httpLogWriter) Timeout() <-chan time.Time { 131 | return w.timer.C 132 | } 133 | 134 | func (w *httpLogWriter) SetMaxLogLength(bytes int) { 135 | w.maxLength = bytes 136 | } 137 | 138 | func (w *httpLogWriter) SetJobStarted(meta *JobStartedMeta) {} 139 | 140 | func (w *httpLogWriter) SetCancelFunc(cancel gocontext.CancelFunc) { 141 | w.cancel = cancel 142 | } 143 | 144 | func (w *httpLogWriter) MaxLengthReached() bool { 145 | return w.maxLengthReached 146 | } 147 | 148 | func (w *httpLogWriter) WriteAndClose(p []byte) (int, error) { 149 | if w.closed() { 150 | return 0, fmt.Errorf("log already closed") 151 | } 152 | 153 | w.timer.Stop() 154 | 155 | close(w.closeChan) 156 | 157 | err := w.lps.Add(w.ctx, &httpLogPart{ 158 | Content: string(p), 159 | JobID: w.jobID, 160 | Number: w.logPartNumber, 161 | Token: w.authToken, 162 | }) 163 | 164 | if err != nil { 165 | context.LoggerFromContext(w.ctx).WithFields(logrus.Fields{ 166 | "err": err, 167 | "self": "http_log_writer", 168 | }).Error("could not add log part to sink") 169 | return 0, err 170 | } 171 | w.logPartNumber++ 172 | 173 | err = w.lps.Add(w.ctx, &httpLogPart{ 174 | Final: true, 175 | JobID: w.jobID, 176 | Number: w.logPartNumber, 177 | Token: w.authToken, 178 | }) 179 | 180 | if err != nil { 181 | context.LoggerFromContext(w.ctx).WithFields(logrus.Fields{ 182 | "err": err, 183 | "self": "http_log_writer", 184 | }).Error("could not add log part to sink") 185 | return 0, err 186 | } 187 | w.logPartNumber++ 188 | return len(p), nil 189 | } 190 | 191 | func (w *httpLogWriter) closed() bool { 192 | select { 193 | case <-w.closeChan: 194 | return true 195 | default: 196 | return false 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /http_log_writer_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | gocontext "context" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func buildTestHTTPLogWriter() (gocontext.CancelFunc, *httpLogWriter, error) { 13 | ctx, cancel := gocontext.WithCancel(gocontext.TODO()) 14 | hlw, err := newHTTPLogWriter( 15 | ctx, 16 | "https://jobs.example.org/foo", 17 | "fafafaf", 18 | 1337, 19 | time.Second) 20 | 21 | hlw.SetMaxLogLength(100) 22 | hlw.SetCancelFunc(noCancel) 23 | 24 | return cancel, hlw, err 25 | } 26 | 27 | func TestNewHTTPLogWriter(t *testing.T) { 28 | cancel, hlw, err := buildTestHTTPLogWriter() 29 | defer cancel() 30 | 31 | assert.Nil(t, err) 32 | assert.NotNil(t, hlw) 33 | assert.Implements(t, (*LogWriter)(nil), hlw) 34 | } 35 | 36 | func TestHTTPLogWriter_Write(t *testing.T) { 37 | cancel, hlw, _ := buildTestHTTPLogWriter() 38 | defer cancel() 39 | 40 | assert.NotNil(t, hlw) 41 | n, err := hlw.Write([]byte("it's a hot one out there")) 42 | assert.Nil(t, err) 43 | assert.True(t, n > 0) 44 | } 45 | 46 | func TestHTTPLogWriter_Write_HitsMaxLogLength(t *testing.T) { 47 | cancel, hlw, _ := buildTestHTTPLogWriter() 48 | defer cancel() 49 | 50 | assert.NotNil(t, hlw) 51 | 52 | hlw.bytesWritten = 1000 53 | 54 | n, err := hlw.Write([]byte("there's a strong wind blowing")) 55 | assert.Nil(t, err) 56 | assert.True(t, hlw.MaxLengthReached()) 57 | assert.Equal(t, 0, n) 58 | } 59 | 60 | func TestHTTPLogWriter_Write_HitsMaxLogLength_CannotWriteAndClose(t *testing.T) { 61 | cancel, hlw, _ := buildTestHTTPLogWriter() 62 | defer cancel() 63 | 64 | assert.NotNil(t, hlw) 65 | 66 | hlw.bytesWritten = 1000 67 | mbs := hlw.lps.maxBufferSize 68 | hlw.lps.maxBufferSize = 0 69 | defer func() { hlw.lps.maxBufferSize = mbs }() 70 | 71 | n, err := hlw.Write([]byte("looks like rain mmm hmm")) 72 | assert.Nil(t, err) 73 | assert.True(t, hlw.MaxLengthReached()) 74 | assert.Equal(t, 0, n) 75 | } 76 | 77 | func TestHTTPLogWriter_Write_HitsMaxLogLength_CannotWriteAndClose_LogClosed(t *testing.T) { 78 | cancel, hlw, _ := buildTestHTTPLogWriter() 79 | defer cancel() 80 | 81 | assert.NotNil(t, hlw) 82 | 83 | hlw.bytesWritten = 1000 84 | err := hlw.Close() 85 | assert.Nil(t, err) 86 | 87 | n, err := hlw.Write([]byte("a storm is a comin")) 88 | assert.NotNil(t, err) 89 | assert.Equal(t, 0, n) 90 | } 91 | -------------------------------------------------------------------------------- /image/doc.go: -------------------------------------------------------------------------------- 1 | // Package image contains logic for image selection logic. 2 | // 3 | // Worker supports two ways of selecting the image to use for a compute 4 | // instance: An APISelector that talks to job-board 5 | // (https://github.com/travis-ci/job-board) and an ENVSelector that gets the 6 | // data from environment variables. 7 | package image 8 | -------------------------------------------------------------------------------- /image/env_selector.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | import ( 4 | gocontext "context" 5 | "strings" 6 | 7 | "github.com/travis-ci/worker/config" 8 | ) 9 | 10 | // EnvSelector implements Selector for environment-based mappings 11 | type EnvSelector struct { 12 | c *config.ProviderConfig 13 | 14 | lookup map[string]string 15 | } 16 | 17 | // NewEnvSelector builds a new EnvSelector from the given *config.ProviderConfig 18 | func NewEnvSelector(c *config.ProviderConfig) (*EnvSelector, error) { 19 | es := &EnvSelector{c: c} 20 | es.buildLookup() 21 | return es, nil 22 | } 23 | 24 | func (es *EnvSelector) buildLookup() { 25 | lookup := map[string]string{} 26 | 27 | es.c.Each(func(key, value string) { 28 | if strings.HasPrefix(key, "IMAGE_") { 29 | lookup[strings.ToLower(strings.Replace(key, "IMAGE_", "", -1))] = value 30 | } 31 | }) 32 | 33 | es.lookup = lookup 34 | } 35 | 36 | func (es *EnvSelector) Select(ctx gocontext.Context, params *Params) (string, error) { 37 | imageName := "default" 38 | 39 | for _, key := range es.buildCandidateKeys(params) { 40 | if key == "" { 41 | continue 42 | } 43 | 44 | if s, ok := es.lookup[key]; ok { 45 | imageName = s 46 | break 47 | } 48 | } 49 | 50 | // check for one level of indirection 51 | if selected, ok := es.lookup[imageName]; ok { 52 | return selected, nil 53 | } 54 | return imageName, nil 55 | } 56 | 57 | func (es *EnvSelector) buildCandidateKeys(params *Params) []string { 58 | fullKey := []string{} 59 | candidateKeys := []string{} 60 | 61 | hasLang := params.Language != "" 62 | hasDist := params.Dist != "" 63 | hasGroup := params.Group != "" 64 | hasOS := params.OS != "" 65 | 66 | if params.OS == "osx" && params.OsxImage != "" { 67 | if hasLang { 68 | candidateKeys = append(candidateKeys, "osx_image_"+params.OsxImage+"_"+params.Language) 69 | } 70 | candidateKeys = append(candidateKeys, "osx_image_"+params.OsxImage) 71 | } 72 | 73 | if hasDist && hasGroup && hasLang { 74 | candidateKeys = append(candidateKeys, "dist_"+params.Dist+"_group_"+params.Group+"_"+params.Language) 75 | candidateKeys = append(candidateKeys, params.Dist+"_"+params.Group+"_"+params.Language) 76 | } 77 | 78 | if hasDist && hasLang { 79 | candidateKeys = append(candidateKeys, "dist_"+params.Dist+"_"+params.Language) 80 | candidateKeys = append(candidateKeys, params.Dist+"_"+params.Language) 81 | } 82 | 83 | if hasGroup && hasLang { 84 | candidateKeys = append(candidateKeys, "group_"+params.Group+"_"+params.Language) 85 | candidateKeys = append(candidateKeys, params.Group+"_"+params.Language) 86 | } 87 | 88 | if hasOS && hasLang { 89 | candidateKeys = append(candidateKeys, "os_"+params.OS+"_"+params.Language) 90 | candidateKeys = append(candidateKeys, params.OS+"_"+params.Language) 91 | } 92 | 93 | if hasDist { 94 | candidateKeys = append(candidateKeys, "default_dist_"+params.Dist) 95 | candidateKeys = append(candidateKeys, "dist_"+params.Dist) 96 | candidateKeys = append(candidateKeys, params.Dist) 97 | } 98 | 99 | if hasGroup { 100 | candidateKeys = append(candidateKeys, "default_group_"+params.Group) 101 | candidateKeys = append(candidateKeys, "group_"+params.Group) 102 | candidateKeys = append(candidateKeys, params.Group) 103 | } 104 | 105 | if hasLang { 106 | candidateKeys = append(candidateKeys, "language_"+params.Language) 107 | candidateKeys = append(candidateKeys, params.Language) 108 | candidateKeys = append(candidateKeys, params.Language) 109 | } 110 | 111 | if hasOS { 112 | candidateKeys = append(candidateKeys, "default_os_"+params.OS) 113 | candidateKeys = append(candidateKeys, "os_"+params.OS) 114 | candidateKeys = append(candidateKeys, params.OS) 115 | } 116 | 117 | return append([]string{strings.Join(fullKey, "_")}, candidateKeys...) 118 | } 119 | -------------------------------------------------------------------------------- /image/env_selector_test.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/travis-ci/worker/config" 10 | ) 11 | 12 | var ( 13 | testEnvSelectorMaps = []*struct { 14 | E map[string]string 15 | O []*testEnvCase 16 | }{ 17 | { 18 | E: map[string]string{ 19 | "IMAGE_DEFAULT": "travis:default", 20 | "IMAGE_LINUX": "travis:linux", 21 | "IMAGE_DIST_XENIAL": "travis:xenial", 22 | "IMAGE_DIST_XENIAL_PYTHON": "travis:py3k", 23 | "IMAGE_GROUP_EDGE": "travis:edge", 24 | "IMAGE_GROUP_EDGE_RUBY": "travis:ruby9001", 25 | "IMAGE_LANGUAGE_RUBY": "travis:ruby8999", 26 | "IMAGE_PYTHON": "travis:python", 27 | }, 28 | O: []*testEnvCase{ 29 | {E: "travis:default", P: &Params{Language: "clojure"}}, 30 | {E: "travis:default", P: &Params{Language: "java"}}, 31 | {E: "travis:edge", P: &Params{Language: "java", Group: "edge"}}, 32 | {E: "travis:linux", P: &Params{Language: "bf", Group: "wat", OS: "linux"}}, 33 | {E: "travis:py3k", P: &Params{Language: "python", Dist: "xenial"}}, 34 | {E: "travis:ruby8999", P: &Params{Language: "ruby"}}, 35 | {E: "travis:ruby9001", P: &Params{Language: "ruby", Group: "edge"}}, 36 | {E: "travis:xenial", P: &Params{Language: "java", Dist: "xenial"}}, 37 | }, 38 | }, 39 | { 40 | E: map[string]string{ 41 | "IMAGE_DEFAULT": "travisci/ci-garnet:packer-1410230255-fafafaf", 42 | "IMAGE_DIST_BIONIC": "registry.business.com/fancy/ubuntu:bionic", 43 | "IMAGE_DIST_TRUSTY": "travisci/ci-connie:packer-1420290255-fafafaf", 44 | "IMAGE_DIST_TRUSTY_RUBY": "registry.business.com/travisci/ci-ruby:whatever", 45 | "IMAGE_GROUP_EDGE_PYTHON": "travisci/ci-garnet:packer-1530230255-fafafaf", 46 | "IMAGE_LANGUAGE_RUBY": "registry.business.com/travisci/ci-ruby:whatever", 47 | }, 48 | O: []*testEnvCase{ 49 | {E: "registry.business.com/fancy/ubuntu:bionic", P: &Params{Language: "bash", Dist: "bionic"}}, 50 | {E: "registry.business.com/fancy/ubuntu:bionic", P: &Params{Language: "ruby", Dist: "bionic"}}, 51 | {E: "registry.business.com/travisci/ci-ruby:whatever", P: &Params{Language: "ruby", Dist: "trusty"}}, 52 | {E: "registry.business.com/travisci/ci-ruby:whatever", P: &Params{Language: "ruby"}}, 53 | {E: "travisci/ci-connie:packer-1420290255-fafafaf", P: &Params{Language: "wat", Dist: "trusty"}}, 54 | {E: "travisci/ci-garnet:packer-1410230255-fafafaf", P: &Params{Language: "python", Group: "stable"}}, 55 | {E: "travisci/ci-garnet:packer-1410230255-fafafaf", P: &Params{Language: "python"}}, 56 | {E: "travisci/ci-garnet:packer-1410230255-fafafaf", P: &Params{Language: "wat"}}, 57 | {E: "travisci/ci-garnet:packer-1530230255-fafafaf", P: &Params{Language: "python", Group: "edge"}}, 58 | }, 59 | }, 60 | } 61 | ) 62 | 63 | type testEnvCase struct { 64 | E string 65 | P *Params 66 | } 67 | 68 | func TestNewEnvSelector(t *testing.T) { 69 | assert.Panics(t, func() { _, _ = NewEnvSelector(nil) }) 70 | assert.NotPanics(t, func() { 71 | _, _ = NewEnvSelector(config.ProviderConfigFromMap(map[string]string{})) 72 | }) 73 | } 74 | 75 | func TestEnvSelector_Select(t *testing.T) { 76 | for _, tesm := range testEnvSelectorMaps { 77 | es, err := NewEnvSelector(config.ProviderConfigFromMap(tesm.E)) 78 | if err != nil { 79 | t.Fatal(err) 80 | } 81 | 82 | for _, tc := range tesm.O { 83 | actual, _ := es.Select(context.TODO(), tc.P) 84 | assert.Equal(t, tc.E, actual, fmt.Sprintf("%#v %q", tc.P, tc.E)) 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /image/manager.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | import ( 4 | gocontext "context" 5 | "fmt" 6 | "net/url" 7 | "runtime" 8 | 9 | lxd "github.com/canonical/lxd/client" 10 | lxdapi "github.com/canonical/lxd/shared/api" 11 | "github.com/sirupsen/logrus" 12 | "github.com/travis-ci/worker/context" 13 | "golang.org/x/sync/errgroup" 14 | ) 15 | 16 | var ( 17 | arch = runtime.GOARCH 18 | infra = fmt.Sprintf("lxd-%s", arch) 19 | 20 | tags = []string{"os:linux"} 21 | groups = []string{"group:stable", "group:edge", "group:dev"} 22 | ) 23 | 24 | func NewManager(ctx gocontext.Context, selector *APISelector, imagesServerURL *url.URL) (*Manager, error) { 25 | logger := context.LoggerFromContext(ctx).WithField("self", "image_manager") 26 | 27 | client, err := lxd.ConnectLXDUnix("", nil) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | return &Manager{ 33 | client: client, 34 | selector: selector, 35 | logger: logger, 36 | imagesServerURL: imagesServerURL, 37 | }, nil 38 | } 39 | 40 | type Manager struct { 41 | client lxd.ContainerServer 42 | selector *APISelector 43 | logger *logrus.Entry 44 | imagesServerURL *url.URL 45 | } 46 | 47 | func (m *Manager) Load(imageName string) error { 48 | ok, err := m.Exists(imageName) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | if ok { 54 | m.logger.WithFields(logrus.Fields{ 55 | "image_name": imageName, 56 | }).Info("image already present") 57 | 58 | return nil 59 | } 60 | 61 | imageURL := m.imageUrl(imageName) 62 | 63 | m.logger.WithFields(logrus.Fields{ 64 | "image_name": imageName, 65 | "image_url": imageURL, 66 | }).Info("importing image") 67 | 68 | return m.importImage(imageName, imageURL) 69 | } 70 | 71 | func (m *Manager) Exists(imageName string) (bool, error) { 72 | images, err := m.client.GetImages() 73 | if err != nil { 74 | return false, err 75 | } 76 | 77 | for _, img := range images { 78 | for _, alias := range img.Aliases { 79 | if alias.Name == imageName { 80 | return true, nil 81 | } 82 | } 83 | } 84 | 85 | return false, nil 86 | } 87 | 88 | func (m *Manager) Update(ctx gocontext.Context) error { 89 | m.logger.WithFields(logrus.Fields{ 90 | "arch": arch, 91 | "infra": infra, 92 | }).Info("updating lxc images") 93 | 94 | images := []*apiSelectorImageRef{} 95 | 96 | for _, group := range groups { 97 | imagesGroup, err := m.selector.SelectAll(ctx, infra, append(tags, group)) 98 | if err != nil { 99 | return err 100 | } 101 | 102 | images = append(images, imagesGroup...) 103 | } 104 | 105 | g, _ := errgroup.WithContext(ctx) 106 | for _, img := range images { 107 | img := img 108 | 109 | g.Go(func() error { 110 | m.logger.WithFields(logrus.Fields{ 111 | "image_name": img.Name, 112 | "infra": infra, 113 | "group": img.Group(), 114 | }).Info("updating image") 115 | 116 | return m.Load(img.Name) 117 | 118 | }) 119 | 120 | } 121 | 122 | return g.Wait() 123 | } 124 | 125 | func (m *Manager) Cleanup() error { 126 | // TODO 127 | 128 | return nil 129 | } 130 | 131 | func (m *Manager) importImage(imageName, imgURL string) error { 132 | op, err := m.client.CreateImage( 133 | lxdapi.ImagesPost{ 134 | Filename: imageName, 135 | Source: &lxdapi.ImagesPostSource{ 136 | Type: "url", 137 | URL: imgURL, 138 | }, 139 | Aliases: []lxdapi.ImageAlias{ 140 | lxdapi.ImageAlias{Name: imageName}, 141 | }, 142 | }, nil, 143 | ) 144 | if err != nil { 145 | return err 146 | } 147 | 148 | err = op.Wait() 149 | if err != nil { 150 | return err 151 | } 152 | 153 | alias, _, err := m.client.GetImageAlias(imageName) 154 | if err != nil { 155 | return err 156 | } 157 | 158 | image, _, err := m.client.GetImage(alias.Target) 159 | if err != nil { 160 | return err 161 | } 162 | 163 | containerName := fmt.Sprintf("%s-warmup", imageName) 164 | 165 | rop, err := m.client.CreateContainerFromImage( 166 | m.client, 167 | *image, 168 | lxdapi.ContainersPost{ 169 | Name: containerName, 170 | }, 171 | ) 172 | if err != nil { 173 | return err 174 | } 175 | 176 | err = rop.Wait() 177 | if err != nil { 178 | return err 179 | } 180 | 181 | defer func() { 182 | op, err := m.client.DeleteContainer(containerName) 183 | if err != nil { 184 | m.logger.WithFields(logrus.Fields{ 185 | "error": err, 186 | "container_name": containerName, 187 | }).Error("failed to delete container") 188 | return 189 | } 190 | 191 | err = op.Wait() 192 | if err != nil { 193 | m.logger.WithFields(logrus.Fields{ 194 | "error": err, 195 | "container_name": containerName, 196 | }).Error("failed to delete container") 197 | return 198 | } 199 | }() 200 | 201 | return err 202 | } 203 | 204 | func (m *Manager) imageUrl(name string) string { 205 | u := *m.imagesServerURL 206 | u.Path = fmt.Sprintf("/images/travis/%s", name) 207 | return u.String() 208 | } 209 | -------------------------------------------------------------------------------- /image/params.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | type Params struct { 4 | Infra string 5 | Language string 6 | OsxImage string 7 | Dist string 8 | Group string 9 | OS string 10 | 11 | JobID uint64 12 | Repo string 13 | GpuVMType string 14 | } 15 | -------------------------------------------------------------------------------- /image/selector.go: -------------------------------------------------------------------------------- 1 | package image 2 | 3 | import gocontext "context" 4 | 5 | // Selector is the interface for selecting an image! 6 | type Selector interface { 7 | Select(gocontext.Context, *Params) (string, error) 8 | } 9 | -------------------------------------------------------------------------------- /job.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "time" 5 | 6 | gocontext "context" 7 | 8 | "github.com/bitly/go-simplejson" 9 | "github.com/travis-ci/worker/backend" 10 | ) 11 | 12 | const ( 13 | VMTypeDefault = "default" 14 | VMTypePremium = "premium" 15 | ) 16 | 17 | var VMConfigDefault = backend.VmConfig{GpuCount: 0, GpuType: ""} 18 | 19 | type jobPayloadStartAttrs struct { 20 | Config *backend.StartAttributes `json:"config"` 21 | VmConfig *backend.VmConfig `json:"vm_config"` 22 | } 23 | 24 | type httpJobPayloadStartAttrs struct { 25 | Data *jobPayloadStartAttrs `json:"data"` 26 | } 27 | 28 | // JobPayload is the payload we receive over RabbitMQ. 29 | type JobPayload struct { 30 | Type string `json:"type"` 31 | Job JobJobPayload `json:"job"` 32 | Build BuildPayload `json:"source"` 33 | Repository RepositoryPayload `json:"repository"` 34 | UUID string `json:"uuid"` 35 | Config map[string]interface{} `json:"config"` 36 | Timeouts TimeoutsPayload `json:"timeouts,omitempty"` 37 | VMType string `json:"vm_type"` 38 | VMConfig backend.VmConfig `json:"vm_config"` 39 | VMSize string `json:"vm_size"` 40 | Meta JobMetaPayload `json:"meta"` 41 | Queue string `json:"queue"` 42 | Trace bool `json:"trace"` 43 | Warmer bool `json:"warmer"` 44 | } 45 | 46 | // JobMetaPayload contains meta information about the job. 47 | type JobMetaPayload struct { 48 | StateUpdateCount uint `json:"state_update_count"` 49 | } 50 | 51 | // JobJobPayload contains information about the job. 52 | type JobJobPayload struct { 53 | ID uint64 `json:"id"` 54 | Number string `json:"number"` 55 | QueuedAt *time.Time `json:"queued_at"` 56 | } 57 | 58 | // BuildPayload contains information about the build. 59 | type BuildPayload struct { 60 | ID uint64 `json:"id"` 61 | Number string `json:"number"` 62 | } 63 | 64 | // RepositoryPayload contains information about the repository. 65 | type RepositoryPayload struct { 66 | ID uint64 `json:"id"` 67 | Slug string `json:"slug"` 68 | } 69 | 70 | // TimeoutsPayload contains information about any custom timeouts. The timeouts 71 | // are given in seconds, and a value of 0 means no custom timeout is set. 72 | type TimeoutsPayload struct { 73 | HardLimit uint64 `json:"hard_limit"` 74 | LogSilence uint64 `json:"log_silence"` 75 | } 76 | 77 | // FinishState is the state that a job finished with (such as pass/fail/etc.). 78 | // You should not provide a string directly, but use one of the FinishStateX 79 | // constants defined in this package. 80 | type FinishState string 81 | 82 | // Valid finish states for the FinishState type 83 | const ( 84 | FinishStatePassed FinishState = "passed" 85 | FinishStateFailed FinishState = "failed" 86 | FinishStateErrored FinishState = "errored" 87 | FinishStateCancelled FinishState = "cancelled" 88 | ) 89 | 90 | // A Job ties togeher all the elements required for a build job 91 | type Job interface { 92 | Payload() *JobPayload 93 | RawPayload() *simplejson.Json 94 | StartAttributes() *backend.StartAttributes 95 | FinishState() FinishState 96 | Requeued() bool 97 | 98 | Received(gocontext.Context) error 99 | Started(gocontext.Context) error 100 | Error(gocontext.Context, string) error 101 | Requeue(gocontext.Context) error 102 | Finish(gocontext.Context, FinishState) error 103 | 104 | LogWriter(gocontext.Context, time.Duration) (LogWriter, error) 105 | Name() string 106 | SetupContext(gocontext.Context) gocontext.Context 107 | } 108 | -------------------------------------------------------------------------------- /job_queue.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | gocontext "context" 5 | ) 6 | 7 | // JobQueue is the minimal interface needed by a ProcessorPool 8 | type JobQueue interface { 9 | Jobs(gocontext.Context) (<-chan Job, error) 10 | Name() string 11 | Cleanup() error 12 | } 13 | -------------------------------------------------------------------------------- /job_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | var jsonPayload = ` 12 | { 13 | "type": "test", 14 | "vm_type": "default", 15 | "queue": "builds.docker", 16 | "config": { 17 | "language": "rust", 18 | "rust": "stable", 19 | "branches": { 20 | "only": ["master"] 21 | }, 22 | "os": "linux", 23 | ".result": "configured", 24 | "global_env": [], 25 | "group": "stable", 26 | "dist": "precise", 27 | "arch": "amd64" 28 | }, 29 | "env_vars": [], 30 | "job": { 31 | "id": 191312240, 32 | "number": "11.1", 33 | "commit": "8a7bc35dbeac4d6c93c2ab92db864cdd63684b04", 34 | "commit_range": "32507736dce8...8a7bc35dbeac", 35 | "commit_message": "Updates Readme", 36 | "branch": "master", 37 | "ref": null, 38 | "tag": null, 39 | "pull_request": false, 40 | "state": "queued", 41 | "secure_env_enabled": true, 42 | "debug_options": {}, 43 | "queued_at": "2017-01-12T15:00:00Z" 44 | }, 45 | "source": { 46 | "id": 191312239, 47 | "number": "11", 48 | "event_type": "push" 49 | }, 50 | "repository": { 51 | "id": 11342078, 52 | "github_id": 76659567, 53 | "slug": "lukaspustina/axfrnotify", 54 | "source_url": "https://github.com/lukaspustina/axfrnotify.git", 55 | "api_url": "https://api.github.com/repos/lukaspustina/axfrnotify", 56 | "last_build_id": 190973807, 57 | "last_build_number": "10", 58 | "last_build_started_at": "2017-01-11T14:38:56Z", 59 | "last_build_finished_at": "2017-01-11T14:40:41Z", 60 | "last_build_duration": 284, 61 | "last_build_state": "passed", 62 | "default_branch": "master", 63 | "description": "axfrnotify sends an NOTIFY message to a secondary name server to initiate a zone refresh for a specific domain name." 64 | }, 65 | "ssh_key": null, 66 | "timeouts": { 67 | "hard_limit": null, 68 | "log_silence": null 69 | }, 70 | "cache_settings": {} 71 | } 72 | ` 73 | 74 | func TestUnmarshalJobPayload(t *testing.T) { 75 | var job JobPayload 76 | 77 | err := json.Unmarshal([]byte(jsonPayload), &job) 78 | assert.NoError(t, err) 79 | 80 | assert.NotNil(t, job.Job.QueuedAt) 81 | assert.Exactly(t, time.Unix(1484233200, 0).In(time.UTC), *job.Job.QueuedAt) 82 | } 83 | -------------------------------------------------------------------------------- /log_writer.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "io" 5 | "time" 6 | 7 | gocontext "context" 8 | ) 9 | 10 | var ( 11 | // LogWriterTick is how often the buffer should be flushed out and sent to 12 | // travis-logs. 13 | LogWriterTick = 500 * time.Millisecond 14 | 15 | // LogChunkSize is a bit of a magic number, calculated like this: The 16 | // maximum Pusher payload is 10 kB (or 10 KiB, who knows, but let's go with 17 | // 10 kB since that is smaller). Looking at the travis-logs source, the 18 | // current message overhead (i.e. the part of the payload that isn't 19 | // the content of the log part) is 42 bytes + the length of the JSON- 20 | // encoded ID and the length of the JSON-encoded sequence number. A 64- 21 | // bit number is up to 20 digits long, so that means (assuming we don't 22 | // go over 64-bit numbers) the overhead is up to 82 bytes. That means 23 | // we can send up to 9918 bytes of content. However, the JSON-encoded 24 | // version of a string can be significantly longer than the raw bytes. 25 | // Worst case that I could find is "<", which with the Go JSON encoder 26 | // becomes "\u003c" (i.e. six bytes long). So, given a string of just 27 | // left angle brackets, the string would become six times as long, 28 | // meaning that the longest string we can take is 1653. We could still 29 | // get errors if we go over 64-bit numbers, but I find the likeliness 30 | // of that happening to both the sequence number, the ID, and us maxing 31 | // out the worst-case logs to be quite unlikely, so I'm willing to live 32 | // with that. --Sarah 33 | LogChunkSize = 1653 34 | ) 35 | 36 | // JobStartedMeta is metadata that is useful for computing time to first 37 | // log line downstream, and breaking it down into further dimensions. 38 | type JobStartedMeta struct { 39 | QueuedAt *time.Time `json:"queued_at"` 40 | Repo string `json:"repo"` 41 | Queue string `json:"queue"` 42 | Infra string `json:"infra"` 43 | } 44 | 45 | // LogWriter is primarily an io.Writer that will send all bytes to travis-logs 46 | // for processing, and also has some utility methods for timeouts and log length 47 | // limiting. Each LogWriter is tied to a given job, and can be gotten by calling 48 | // the LogWriter() method on a Job. 49 | type LogWriter interface { 50 | io.WriteCloser 51 | WriteAndClose([]byte) (int, error) 52 | Timeout() <-chan time.Time 53 | SetMaxLogLength(int) 54 | SetJobStarted(meta *JobStartedMeta) 55 | SetCancelFunc(gocontext.CancelFunc) 56 | MaxLengthReached() bool 57 | } 58 | -------------------------------------------------------------------------------- /log_writer_factory.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | gocontext "context" 5 | "time" 6 | ) 7 | 8 | type LogWriterFactory interface { 9 | LogWriter(gocontext.Context, time.Duration, Job) (LogWriter, error) 10 | Cleanup() error 11 | } 12 | -------------------------------------------------------------------------------- /metrics/memstats.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "runtime" 5 | "time" 6 | 7 | "github.com/rcrowley/go-metrics" 8 | ) 9 | 10 | // ReportMemstatsMetrics will send runtime Memstats metrics every 10 seconds, 11 | // and will block forever. 12 | func ReportMemstatsMetrics() { 13 | memStats := &runtime.MemStats{} 14 | lastSampleTime := time.Now() 15 | var lastPauseNs uint64 16 | var lastNumGC uint64 17 | 18 | sleep := 10 * time.Second 19 | 20 | for { 21 | runtime.ReadMemStats(memStats) 22 | 23 | now := time.Now() 24 | 25 | metrics.GetOrRegisterGauge("travis.worker.goroutines", metrics.DefaultRegistry).Update(int64(runtime.NumGoroutine())) 26 | metrics.GetOrRegisterGauge("travis.worker.memory.allocated", metrics.DefaultRegistry).Update(int64(memStats.Alloc)) 27 | metrics.GetOrRegisterGauge("travis.worker.memory.mallocs", metrics.DefaultRegistry).Update(int64(memStats.Mallocs)) 28 | metrics.GetOrRegisterGauge("travis.worker.memory.frees", metrics.DefaultRegistry).Update(int64(memStats.Frees)) 29 | metrics.GetOrRegisterGauge("travis.worker.memory.gc.total_pause", metrics.DefaultRegistry).Update(int64(memStats.PauseTotalNs)) 30 | metrics.GetOrRegisterGauge("travis.worker.memory.gc.heap", metrics.DefaultRegistry).Update(int64(memStats.HeapAlloc)) 31 | metrics.GetOrRegisterGauge("travis.worker.memory.gc.stack", metrics.DefaultRegistry).Update(int64(memStats.StackInuse)) 32 | 33 | if lastPauseNs > 0 { 34 | pauseSinceLastSample := memStats.PauseTotalNs - lastPauseNs 35 | metrics.GetOrRegisterGauge("travis.worker.memory.gc.pause_per_second", metrics.DefaultRegistry).Update(int64(float64(pauseSinceLastSample) / sleep.Seconds())) 36 | } 37 | lastPauseNs = memStats.PauseTotalNs 38 | 39 | countGC := int(uint64(memStats.NumGC) - lastNumGC) 40 | if lastNumGC > 0 { 41 | diff := float64(countGC) 42 | diffTime := now.Sub(lastSampleTime).Seconds() 43 | metrics.GetOrRegisterGauge("travis.worker.memory.gc.gc_per_second", metrics.DefaultRegistry).Update(int64(diff / diffTime)) 44 | } 45 | 46 | if countGC > 0 { 47 | if countGC > 256 { 48 | countGC = 256 49 | } 50 | 51 | for i := 0; i < countGC; i++ { 52 | idx := int((memStats.NumGC-uint32(i))+255) % 256 53 | pause := time.Duration(memStats.PauseNs[idx]) 54 | metrics.GetOrRegisterTimer("travis.worker.memory.gc.pause", metrics.DefaultRegistry).Update(pause) 55 | } 56 | } 57 | 58 | lastNumGC = uint64(memStats.NumGC) 59 | lastSampleTime = now 60 | 61 | time.Sleep(sleep) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /metrics/package.go: -------------------------------------------------------------------------------- 1 | // Package metrics provides easy methods to send metrics 2 | package metrics 3 | 4 | import ( 5 | "time" 6 | 7 | "github.com/rcrowley/go-metrics" 8 | ) 9 | 10 | // Mark increases the meter metric with the given name by 1 11 | func Mark(name string) { 12 | metrics.GetOrRegisterMeter(name, metrics.DefaultRegistry).Mark(1) 13 | } 14 | 15 | // TimeSince increases the timer metric with the given name by the time since the given time 16 | func TimeSince(name string, since time.Time) { 17 | metrics.GetOrRegisterTimer(name, metrics.DefaultRegistry).UpdateSince(since) 18 | } 19 | 20 | // TimeDuration increases the timer metric with the given name by the given duration 21 | func TimeDuration(name string, duration time.Duration) { 22 | metrics.GetOrRegisterTimer(name, metrics.DefaultRegistry).Update(duration) 23 | } 24 | 25 | // Gauge sets a gauge metric to a given value 26 | func Gauge(name string, value int64) { 27 | metrics.GetOrRegisterGauge(name, metrics.DefaultRegistry).Update(value) 28 | } 29 | -------------------------------------------------------------------------------- /multi_source_job_queue.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "time" 7 | 8 | gocontext "context" 9 | 10 | "github.com/sirupsen/logrus" 11 | "github.com/travis-ci/worker/context" 12 | "github.com/travis-ci/worker/metrics" 13 | ) 14 | 15 | type MultiSourceJobQueue struct { 16 | queues []JobQueue 17 | } 18 | 19 | func NewMultiSourceJobQueue(queues ...JobQueue) *MultiSourceJobQueue { 20 | return &MultiSourceJobQueue{queues: queues} 21 | } 22 | 23 | // Jobs returns a Job channel that selects over each source queue Job channel 24 | func (msjq *MultiSourceJobQueue) Jobs(ctx gocontext.Context) (outChan <-chan Job, err error) { 25 | logger := context.LoggerFromContext(ctx).WithFields(logrus.Fields{ 26 | "self": "multi_source_job_queue", 27 | "inst": fmt.Sprintf("%p", msjq), 28 | }) 29 | 30 | buildJobChan := make(chan Job) 31 | outChan = buildJobChan 32 | 33 | buildJobChans := map[string]<-chan Job{} 34 | 35 | for i, queue := range msjq.queues { 36 | jc, err := queue.Jobs(ctx) 37 | if err != nil { 38 | logger.WithFields(logrus.Fields{ 39 | "err": err, 40 | "name": queue.Name(), 41 | }).Error("failed to get job chan from queue") 42 | return nil, err 43 | } 44 | qName := fmt.Sprintf("%s.%d", queue.Name(), i) 45 | buildJobChans[qName] = jc 46 | } 47 | 48 | go func() { 49 | for { 50 | for queueName, bjc := range buildJobChans { 51 | var job Job = nil 52 | jobSendBegin := time.Now() 53 | logger = logger.WithField("queue_name", queueName) 54 | 55 | logger.Debug("about to receive job") 56 | select { 57 | case job = <-bjc: 58 | if job == nil { 59 | logger.Debug("skipping nil job") 60 | continue 61 | } 62 | jobID := uint64(0) 63 | if job.Payload() != nil { 64 | jobID = job.Payload().Job.ID 65 | } 66 | 67 | logger.WithField("job_id", jobID).Debug("about to send job to multi source output channel") 68 | buildJobChan <- job 69 | 70 | metrics.TimeSince("travis.worker.job_queue.multi.blocking_time", jobSendBegin) 71 | logger.WithFields(logrus.Fields{ 72 | "job_id": jobID, 73 | "source": queueName, 74 | "send_duration_ms": time.Since(jobSendBegin).Seconds() * 1e3, 75 | }).Info("sent job to multi source output channel") 76 | case <-ctx.Done(): 77 | return 78 | case <-time.After(time.Second): 79 | continue 80 | } 81 | } 82 | } 83 | }() 84 | 85 | return outChan, nil 86 | } 87 | 88 | // Name builds a name from each source queue name 89 | func (msjq *MultiSourceJobQueue) Name() string { 90 | s := []string{} 91 | for _, queue := range msjq.queues { 92 | s = append(s, queue.Name()) 93 | } 94 | 95 | return strings.Join(s, ",") 96 | } 97 | 98 | // Cleanup runs cleanup for each source queue 99 | func (msjq *MultiSourceJobQueue) Cleanup() error { 100 | for _, queue := range msjq.queues { 101 | err := queue.Cleanup() 102 | if err != nil { 103 | return err 104 | } 105 | } 106 | return nil 107 | } 108 | -------------------------------------------------------------------------------- /multi_source_job_queue_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | 8 | gocontext "context" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/travis-ci/worker/context" 12 | ) 13 | 14 | func TestNewMultiSourceJobQueue(t *testing.T) { 15 | ctx := gocontext.TODO() 16 | logger := context.LoggerFromContext(ctx) 17 | jq0 := &fakeJobQueue{c: make(chan Job)} 18 | jq1 := &fakeJobQueue{c: make(chan Job)} 19 | msjq := NewMultiSourceJobQueue(jq0, jq1) 20 | 21 | assert.NotNil(t, msjq) 22 | 23 | buildJobChan, err := msjq.Jobs(ctx) 24 | assert.Nil(t, err) 25 | assert.NotNil(t, buildJobChan) 26 | 27 | done := make(chan struct{}) 28 | 29 | go func() { 30 | logger.Debugf("about to <-%#v [\"buildJobChan\"]", buildJobChan) 31 | <-buildJobChan 32 | logger.Debugf("<-%#v [\"buildJobChan\"]", buildJobChan) 33 | logger.Debugf("about to <-%#v [\"buildJobChan\"]", buildJobChan) 34 | <-buildJobChan 35 | logger.Debugf("<-%#v [\"buildJobChan\"]", buildJobChan) 36 | logger.Debugf("about to %#v [\"done\"] <- {}", done) 37 | done <- struct{}{} 38 | logger.Debugf("%#v [\"done\"] <- {}", done) 39 | }() 40 | 41 | go func() { 42 | logger.Debugf("about to %#v [\"jq0.c\"] <- &fakeJob{}", jq0.c) 43 | jq0.c <- &fakeJob{} 44 | logger.Debugf("%#v [\"jq0.c\"] <- &fakeJob{}", jq0.c) 45 | 46 | logger.Debugf("about to %#v [\"done\"] <- {}", done) 47 | done <- struct{}{} 48 | logger.Debugf("%#v [\"done\"] <- {}", done) 49 | }() 50 | 51 | go func() { 52 | logger.Debugf("about to %#v [\"jq1.c\"] <- &fakeJob{}", jq1.c) 53 | jq1.c <- &fakeJob{} 54 | logger.Debugf("%#v [\"jq1.c\"] <- &fakeJob{}", jq1.c) 55 | 56 | logger.Debugf("about to %#v [\"done\"] <- {}", done) 57 | done <- struct{}{} 58 | logger.Debugf("%#v [\"done\"] <- {}", done) 59 | }() 60 | 61 | doneCount := 0 62 | for doneCount < 3 { 63 | logger.Debugf("entering for loop") 64 | timeout := 5 * time.Second 65 | select { 66 | case <-time.After(timeout): 67 | assert.FailNow(t, fmt.Sprintf("jobs were not received within %v", timeout)) 68 | case <-done: 69 | logger.Debugf("<-%#v [\"done\"]", done) 70 | doneCount++ 71 | } 72 | } 73 | } 74 | 75 | func TestMultiSourceJobQueue_Name(t *testing.T) { 76 | jq0 := &fakeJobQueue{c: make(chan Job)} 77 | jq1 := &fakeJobQueue{c: make(chan Job)} 78 | msjq := NewMultiSourceJobQueue(jq0, jq1) 79 | assert.Equal(t, "fake,fake", msjq.Name()) 80 | } 81 | 82 | func TestMultiSourceJobQueue_Cleanup(t *testing.T) { 83 | jq0 := &fakeJobQueue{c: make(chan Job)} 84 | jq1 := &fakeJobQueue{c: make(chan Job)} 85 | msjq := NewMultiSourceJobQueue(jq0, jq1) 86 | err := msjq.Cleanup() 87 | assert.Nil(t, err) 88 | 89 | assert.True(t, jq0.cleanedUp) 90 | assert.True(t, jq1.cleanedUp) 91 | } 92 | 93 | func TestMultiSourceJobQueue_Jobs_uniqueChannels(t *testing.T) { 94 | jq0 := &fakeJobQueue{c: make(chan Job)} 95 | jq1 := &fakeJobQueue{c: make(chan Job)} 96 | msjq := NewMultiSourceJobQueue(jq0, jq1) 97 | 98 | buildJobChan0, err := msjq.Jobs(gocontext.TODO()) 99 | assert.Nil(t, err) 100 | assert.NotNil(t, buildJobChan0) 101 | 102 | buildJobChan1, err := msjq.Jobs(gocontext.TODO()) 103 | assert.Nil(t, err) 104 | assert.NotNil(t, buildJobChan1) 105 | 106 | assert.NotEqual(t, fmt.Sprintf("%#v", buildJobChan0), fmt.Sprintf("%#v", buildJobChan1)) 107 | } 108 | -------------------------------------------------------------------------------- /package.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "strings" 7 | ) 8 | 9 | const ( 10 | travisFoldStart = "travis_fold:start:%s\r\033[0K" 11 | travisFoldEnd = "travis_fold:end:%s\r\033[0K" 12 | ) 13 | 14 | func writeFold(w io.Writer, name string, b []byte) (int, error) { 15 | folded := []byte(fmt.Sprintf(travisFoldStart, name)) 16 | folded = append(folded, b...) 17 | 18 | if string(folded[len(folded)-1]) != "\n" { 19 | folded = append(folded, []byte("\n")...) 20 | } 21 | 22 | folded = append(folded, []byte(fmt.Sprintf(travisFoldEnd, name))...) 23 | return w.Write(folded) 24 | } 25 | 26 | func writeFoldStart(w io.Writer, name string, b []byte) (int, error) { 27 | folded := []byte(fmt.Sprintf(travisFoldStart, name)) 28 | folded = append(folded, b...) 29 | return w.Write(folded) 30 | } 31 | 32 | func writeFoldEnd(w io.Writer, name string, b []byte) (int, error) { 33 | folded := b 34 | folded = append(folded, []byte(fmt.Sprintf(travisFoldEnd, name))...) 35 | return w.Write(folded) 36 | } 37 | 38 | func stringSplitSpace(s string) []string { 39 | parts := []string{} 40 | for _, part := range strings.Split(s, " ") { 41 | parts = append(parts, strings.TrimSpace(part)) 42 | } 43 | return parts 44 | } 45 | -------------------------------------------------------------------------------- /package_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "os" 5 | "time" 6 | 7 | gocontext "context" 8 | 9 | simplejson "github.com/bitly/go-simplejson" 10 | "github.com/pkg/errors" 11 | "github.com/sirupsen/logrus" 12 | "github.com/travis-ci/worker/backend" 13 | ) 14 | 15 | func init() { 16 | logrus.SetLevel(logrus.FatalLevel) 17 | if os.Getenv("TRAVIS_WORKER_TEST_DEBUG") == "1" { 18 | logrus.SetLevel(logrus.DebugLevel) 19 | } 20 | } 21 | 22 | type fakeJobQueue struct { 23 | c chan Job 24 | 25 | cleanedUp bool 26 | } 27 | 28 | func (jq *fakeJobQueue) Jobs(ctx gocontext.Context) (<-chan Job, error) { 29 | return (<-chan Job)(jq.c), nil 30 | } 31 | 32 | func (jq *fakeJobQueue) Name() string { return "fake" } 33 | 34 | func (jq *fakeJobQueue) Cleanup() error { 35 | jq.cleanedUp = true 36 | return nil 37 | } 38 | 39 | type fakeJob struct { 40 | payload *JobPayload 41 | rawPayload *simplejson.Json 42 | startAttributes *backend.StartAttributes 43 | finishState FinishState 44 | requeued bool 45 | 46 | events []string 47 | 48 | hasBrokenLogWriter bool 49 | } 50 | 51 | func (fj *fakeJob) Payload() *JobPayload { 52 | return fj.payload 53 | } 54 | 55 | func (fj *fakeJob) RawPayload() *simplejson.Json { 56 | return fj.rawPayload 57 | } 58 | 59 | func (fj *fakeJob) StartAttributes() *backend.StartAttributes { 60 | return fj.startAttributes 61 | } 62 | 63 | func (fj *fakeJob) FinishState() FinishState { 64 | return fj.finishState 65 | } 66 | 67 | func (fj *fakeJob) Requeued() bool { 68 | return fj.requeued 69 | } 70 | 71 | func (fj *fakeJob) Received(ctx gocontext.Context) error { 72 | select { 73 | case <-ctx.Done(): 74 | return ctx.Err() 75 | default: 76 | } 77 | fj.events = append(fj.events, "received") 78 | return nil 79 | } 80 | 81 | func (fj *fakeJob) Started(ctx gocontext.Context) error { 82 | select { 83 | case <-ctx.Done(): 84 | return ctx.Err() 85 | default: 86 | } 87 | fj.events = append(fj.events, "started") 88 | return nil 89 | } 90 | 91 | func (fj *fakeJob) Error(ctx gocontext.Context, msg string) error { 92 | select { 93 | case <-ctx.Done(): 94 | return ctx.Err() 95 | default: 96 | } 97 | fj.events = append(fj.events, "errored") 98 | return nil 99 | } 100 | 101 | func (fj *fakeJob) Requeue(ctx gocontext.Context) error { 102 | fj.requeued = true 103 | select { 104 | case <-ctx.Done(): 105 | return ctx.Err() 106 | default: 107 | } 108 | fj.events = append(fj.events, "requeued") 109 | return nil 110 | } 111 | 112 | func (fj *fakeJob) Finish(ctx gocontext.Context, state FinishState) error { 113 | fj.finishState = state 114 | select { 115 | case <-ctx.Done(): 116 | return ctx.Err() 117 | default: 118 | } 119 | fj.events = append(fj.events, string(state)) 120 | return nil 121 | } 122 | 123 | func (fj *fakeJob) LogWriter(_ gocontext.Context, _ time.Duration) (LogWriter, error) { 124 | return &fakeLogWriter{broken: fj.hasBrokenLogWriter}, nil 125 | } 126 | 127 | func (j *fakeJob) SetupContext(ctx gocontext.Context) gocontext.Context { return ctx } 128 | 129 | func (fj *fakeJob) Name() string { return "fake" } 130 | 131 | type fakeLogWriter struct { 132 | broken bool 133 | } 134 | 135 | func (flw *fakeLogWriter) Write(_ []byte) (int, error) { 136 | if flw.broken { 137 | return 0, errors.New("failed to write") 138 | } 139 | return 0, nil 140 | } 141 | 142 | func (flw *fakeLogWriter) Close() error { 143 | if flw.broken { 144 | return errors.New("failed to close") 145 | } 146 | return nil 147 | } 148 | 149 | func (flw *fakeLogWriter) WriteAndClose(_ []byte) (int, error) { 150 | if flw.broken { 151 | return 0, errors.New("failed to write and close") 152 | } 153 | return 0, nil 154 | } 155 | 156 | func (flw *fakeLogWriter) Timeout() <-chan time.Time { 157 | return make(chan time.Time) 158 | } 159 | 160 | func (flw *fakeLogWriter) SetMaxLogLength(_ int) {} 161 | 162 | func (flw *fakeLogWriter) SetJobStarted(meta *JobStartedMeta) {} 163 | 164 | func (flw *fakeLogWriter) SetCancelFunc(_ gocontext.CancelFunc) {} 165 | 166 | func (flw *fakeLogWriter) MaxLengthReached() bool { 167 | return false 168 | } 169 | -------------------------------------------------------------------------------- /processor_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "context" 5 | "reflect" 6 | "testing" 7 | "time" 8 | 9 | simplejson "github.com/bitly/go-simplejson" 10 | "github.com/pborman/uuid" 11 | "github.com/travis-ci/worker/backend" 12 | "github.com/travis-ci/worker/config" 13 | workerctx "github.com/travis-ci/worker/context" 14 | ) 15 | 16 | type buildScriptGeneratorFunction func(context.Context, Job) ([]byte, error) 17 | 18 | func (bsg buildScriptGeneratorFunction) Generate(ctx context.Context, job Job) ([]byte, error) { 19 | return bsg(ctx, job) 20 | } 21 | 22 | func TestProcessor(t *testing.T) { 23 | t.Skip("brittle test is brittle :scream_cat:") 24 | for i, tc := range []struct { 25 | runSleep time.Duration 26 | hardTimeout time.Duration 27 | stateEvents []string 28 | isCancelled bool 29 | hasBrokenLogWriter bool 30 | }{ 31 | { 32 | runSleep: 0 * time.Second, 33 | hardTimeout: 5 * time.Second, 34 | stateEvents: []string{"received", "started", string(FinishStatePassed)}, 35 | }, 36 | { 37 | runSleep: 5 * time.Second, 38 | hardTimeout: 6 * time.Second, 39 | stateEvents: []string{"received", "started", string(FinishStateCancelled)}, 40 | isCancelled: true, 41 | }, 42 | { 43 | runSleep: 0 * time.Second, 44 | hardTimeout: 5 * time.Second, 45 | stateEvents: []string{"received", "started", "requeued"}, 46 | hasBrokenLogWriter: true, 47 | }, 48 | { 49 | runSleep: 5 * time.Second, 50 | hardTimeout: 4 * time.Second, 51 | stateEvents: []string{"received", "started", string(FinishStateErrored)}, 52 | }, 53 | } { 54 | jobID := uint64(100 + i) 55 | uuid := uuid.NewRandom() 56 | ctx := workerctx.FromProcessor(context.TODO(), uuid.String()) 57 | 58 | provider, err := backend.NewBackendProvider("fake", config.ProviderConfigFromMap(map[string]string{ 59 | "RUN_SLEEP": tc.runSleep.String(), 60 | "LOG_OUTPUT": "hello, world", 61 | })) 62 | if err != nil { 63 | t.Error(err) 64 | } 65 | 66 | generator := buildScriptGeneratorFunction(func(ctx context.Context, job Job) ([]byte, error) { 67 | return []byte("hello, world"), nil 68 | }) 69 | 70 | jobChan := make(chan Job) 71 | jobQueue := &fakeJobQueue{c: jobChan} 72 | cancellationBroadcaster := NewCancellationBroadcaster() 73 | 74 | processor, err := NewProcessor(ctx, "test-hostname", jobQueue, nil, provider, generator, nil, cancellationBroadcaster, ProcessorConfig{ 75 | Config: &config.Config{ 76 | HardTimeout: tc.hardTimeout, 77 | LogTimeout: time.Second, 78 | ScriptUploadTimeout: 3 * time.Second, 79 | StartupTimeout: 4 * time.Second, 80 | MaxLogLength: 4500000, 81 | PayloadFilterExecutable: "filter.py", 82 | }, 83 | }) 84 | if err != nil { 85 | t.Error(err) 86 | } 87 | 88 | doneChan := make(chan struct{}) 89 | go func() { 90 | processor.Run() 91 | doneChan <- struct{}{} 92 | }() 93 | 94 | rawPayload, _ := simplejson.NewJson([]byte("{}")) 95 | 96 | job := &fakeJob{ 97 | rawPayload: rawPayload, 98 | payload: &JobPayload{ 99 | Type: "job:test", 100 | Job: JobJobPayload{ 101 | ID: jobID, 102 | Number: "3.1", 103 | }, 104 | Build: BuildPayload{ 105 | ID: 1, 106 | Number: "3", 107 | }, 108 | Repository: RepositoryPayload{ 109 | ID: 4, 110 | Slug: "green-eggs/ham", 111 | }, 112 | UUID: "foo-bar", 113 | Config: map[string]interface{}{}, 114 | Timeouts: TimeoutsPayload{}, 115 | }, 116 | startAttributes: &backend.StartAttributes{}, 117 | hasBrokenLogWriter: tc.hasBrokenLogWriter, 118 | } 119 | 120 | if tc.isCancelled { 121 | go func(sl time.Duration, i uint64) { 122 | time.Sleep(sl) 123 | cancellationBroadcaster.Broadcast(CancellationCommand{JobID: i}) 124 | }(tc.runSleep-1, jobID) 125 | } 126 | 127 | jobChan <- job 128 | 129 | processor.GracefulShutdown() 130 | <-doneChan 131 | 132 | if processor.ProcessedCount != 1 { 133 | t.Errorf("processor.ProcessedCount = %d, expected %d", processor.ProcessedCount, 1) 134 | } 135 | 136 | if !reflect.DeepEqual(tc.stateEvents, job.events) { 137 | t.Errorf("job.events = %#v, expected %#v", job.events, tc.stateEvents) 138 | } 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /ratelimit/ratelimit_test.go: -------------------------------------------------------------------------------- 1 | package ratelimit 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestRateLimit(t *testing.T) { 12 | if os.Getenv("REDIS_URL") == "" { 13 | t.Skip("skipping redis test since there is no REDIS_URL") 14 | } 15 | 16 | if time.Now().Minute() > 58 { 17 | t.Log("Note: The TestRateLimit test is known to have a bug if run near the top of the hour. Since the rate limiter isn't a moving window, it could end up checking against two different buckets on either side of the top of the hour, so if you see that just re-run it after you've passed the top of the hour.") 18 | } 19 | 20 | rateLimiter := NewRateLimiter(os.Getenv("REDIS_URL"), fmt.Sprintf("worker-test-rl-%d", os.Getpid()), false, time.Minute) 21 | 22 | ok, err := rateLimiter.RateLimit(context.TODO(), "slow", 2, time.Hour) 23 | if err != nil { 24 | t.Fatalf("rate limiter error: %v", err) 25 | } 26 | if !ok { 27 | t.Fatal("expected to not get rate limited, but was limited") 28 | } 29 | 30 | ok, err = rateLimiter.RateLimit(context.TODO(), "slow", 2, time.Hour) 31 | if err != nil { 32 | t.Fatalf("rate limiter error: %v", err) 33 | } 34 | if !ok { 35 | t.Fatal("expected to not get rate limited, but was limited") 36 | } 37 | 38 | ok, err = rateLimiter.RateLimit(context.TODO(), "slow", 2, time.Hour) 39 | if err != nil { 40 | t.Fatalf("rate limiter error: %v", err) 41 | } 42 | if ok { 43 | t.Fatal("expected to get rate limited, but was not limited") 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /remote/package.go: -------------------------------------------------------------------------------- 1 | package remote 2 | 3 | import "io" 4 | 5 | type Remoter interface { 6 | UploadFile(path string, data []byte) (bool, error) 7 | DownloadFile(path string) ([]byte, error) 8 | RunCommand(command string, output io.Writer) (int32, error) 9 | Close() error 10 | } 11 | -------------------------------------------------------------------------------- /script/clean: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | 4 | main() { 5 | shopt -s nullglob 6 | 7 | cd "$(dirname "$(dirname "${BASH_SOURCE[0]}")")" 8 | 9 | rm -rvf \ 10 | "${GOPATH%%:*}/bin/travis-worker" \ 11 | ./*coverage.coverprofile \ 12 | ./tmp/deb \ 13 | ./tmp/output \ 14 | .docker.env \ 15 | CURRENT_SHA1 \ 16 | GIT_DESCRIPTION \ 17 | VERSION \ 18 | VERSION_SHA1 \ 19 | VERSION_TAG \ 20 | coverage.html 21 | 22 | find "${GOPATH%%:*}/pkg" -wholename '*travis-ci/worker*.a' | 23 | grep -v vendor/ | 24 | xargs rm -vf 25 | } 26 | 27 | main "$@" 28 | -------------------------------------------------------------------------------- /script/docker-push: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | 4 | main() { 5 | set +o xtrace 6 | 7 | if [[ "${DOCKER_CREDS}" == 'quay' ]]; then 8 | DOCKER_LOGIN_EMAIL=. 9 | DOCKER_LOGIN_USERNAME="${QUAY_DOCKER_LOGIN_USERNAME}" 10 | DOCKER_LOGIN_PASSWORD="${QUAY_DOCKER_LOGIN_PASSWORD}" 11 | DOCKER_LOGIN_SERVER="${QUAY_DOCKER_LOGIN_SERVER}" 12 | fi 13 | 14 | [[ ${DOCKER_LOGIN_USERNAME} ]] || { 15 | __log error="missing \$DOCKER_LOGIN_USERNAME" 16 | exit 1 17 | } 18 | 19 | [[ ${DOCKER_LOGIN_PASSWORD} ]] || { 20 | __log error="missing \$DOCKER_LOGIN_PASSWORD" 21 | exit 1 22 | } 23 | 24 | [[ ${DOCKER_DEST} ]] || { 25 | __log error="missing \$DOCKER_DEST" 26 | exit 1 27 | } 28 | 29 | local login_args=( 30 | -u "${DOCKER_LOGIN_USERNAME}" 31 | --password-stdin 32 | ) 33 | 34 | if [[ "${DOCKER_LOGIN_EMAIL}" ]]; then 35 | login_args=("${login_args[@]}" -e "${DOCKER_LOGIN_EMAIL}") 36 | fi 37 | 38 | __log 'msg="docker login"' 39 | if [[ "${DOCKER_LOGIN_SERVER}" ]]; then 40 | echo "${DOCKER_LOGIN_PASSWORD}" | 41 | docker login "${login_args[@]}" "${DOCKER_LOGIN_SERVER}" 42 | else 43 | echo "${DOCKER_LOGIN_PASSWORD}" | docker login "${login_args[@]}" 44 | fi 45 | 46 | : "${DOCKER_PUSH_RETRIES:=6}" 47 | : "${DOCKER_LOGOUT_POST_PUSH:=0}" 48 | 49 | local attempt=0 50 | local sleep_interval=10 51 | local push_start 52 | local duration 53 | 54 | while true; do 55 | __log "msg=\"docker push\" dest=${DOCKER_DEST} attempt=$((attempt + 1))" 56 | push_start=$(date +%s) 57 | if docker push "${DOCKER_DEST}"; then 58 | local now_ts 59 | now_ts="$(date +%s)" 60 | duration="$((now_ts - push_start))" 61 | __log 'msg="docker push complete"' \ 62 | "dest=${DOCKER_DEST} " \ 63 | "duration=${duration}s" 64 | 65 | if [[ ${DOCKER_LOGOUT_POST_PUSH} == 1 ]]; then 66 | __log 'msg="docker logout"' 67 | # shellcheck disable=SC2086 68 | docker logout ${DOCKER_LOGIN_SERVER} 69 | __log 'msg="docker logout complete"' 70 | fi 71 | exit 0 72 | fi 73 | 74 | attempt=$((attempt + 1)) 75 | 76 | if [[ $attempt -gt ${DOCKER_PUSH_RETRIES} ]]; then 77 | break 78 | fi 79 | 80 | __log "msg=\"sleeping\" interval=$((attempt * sleep_interval))" 81 | sleep $((attempt * sleep_interval)) 82 | done 83 | 84 | exit 86 85 | } 86 | 87 | __log() { 88 | echo "time=$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$@" 89 | } 90 | 91 | main "$@" 92 | -------------------------------------------------------------------------------- /script/drain-logs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | import pika 3 | import json 4 | 5 | def on_message(channel, method_frame, header_frame, body): 6 | log_part = json.loads(body) 7 | print(log_part['log'], end='') 8 | if log_part['final']: 9 | print() 10 | print('---') 11 | print() 12 | 13 | connection = pika.BlockingConnection() 14 | channel = connection.channel() 15 | channel.basic_consume(on_message, queue='reporting.jobs.logs', no_ack=True) 16 | try: 17 | channel.start_consuming() 18 | except KeyboardInterrupt: 19 | channel.stop_consuming() 20 | connection.close() 21 | -------------------------------------------------------------------------------- /script/fmtpolice: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | 4 | main() { 5 | if [[ -z "${1}" ]]; then 6 | git ls-files '*.go' | while read -r f; do 7 | __gofmt_check "${f}" 8 | done 9 | else 10 | find "${1}" -type f -iname '*.go' | while read -r f; do 11 | __gofmt_check "${f}" 12 | done 13 | fi 14 | 15 | echo 16 | echo "***** ALL HAPPY *****" 17 | } 18 | 19 | __gofmt_check() { 20 | local f="${1}" 21 | gofmt "${f}" | if ! diff -u "${f}" -; then 22 | echo "fmtpolice:${f} NOK" 23 | exit 1 24 | else 25 | echo "fmtpolice:${f} OK" 26 | fi 27 | } 28 | 29 | main "$@" 30 | -------------------------------------------------------------------------------- /script/fold-coverprofiles: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | 4 | main() { 5 | if [[ $# -lt 1 ]]; then 6 | echo "Usage: $(basename "${0}") [coverprofile, coverprofile, ...]" 7 | exit 1 8 | fi 9 | 10 | : "${GO:=go}" 11 | : "${OUT_TMP:=$(mktemp worker.XXXXX)}" 12 | : "${PACKAGE:=github.com/travis-ci/worker}" 13 | 14 | trap 'rm -f '"${OUT_TMP}" EXIT TERM QUIT 15 | 16 | "${GO}" test \ 17 | -covermode=count \ 18 | "-coverprofile=${OUT_TMP}" \ 19 | "${GOBUILD_LDFLAGS[*]}" \ 20 | "${PACKAGE}" 1>&2 21 | 22 | echo 'mode: count' 23 | grep -h -v 'mode: count' "${OUT_TMP}" || true 24 | grep -h -v 'mode: count' "${@}" 25 | } 26 | 27 | main "$@" 28 | -------------------------------------------------------------------------------- /script/http-job-test: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | 4 | main() { 5 | : "${JOB_BOARD_CLONE_DIR:=$(mktemp -d /tmp/job-board-clone.XXXXXXX)}" 6 | : "${JOB_BOARD_CLONE_URL:=https://github.com/travis-ci/job-board.git}" 7 | : "${JOB_BOARD_CLONE_BRANCH:=ga-tbt181-job_board}" 8 | 9 | trap __cleanup EXIT QUIT INT 10 | 11 | git clone --branch "${JOB_BOARD_CLONE_BRANCH}" \ 12 | "${JOB_BOARD_CLONE_URL}" "${JOB_BOARD_CLONE_DIR}" 13 | 14 | docker run -d \ 15 | --name job-board-redis \ 16 | redis 17 | 18 | docker run -d \ 19 | --name job-board-postgres \ 20 | -e POSTGRES_PASSWORD=yay \ 21 | postgres 22 | 23 | docker exec \ 24 | --user postgres \ 25 | job-board-postgres bash -c 'while ! pg_isready; do sleep 1; done' 26 | 27 | # despite pg_isready reporting the server as up, we still have to wait a bit 28 | # for createdb to be able to successfully connect 29 | sleep 1 30 | 31 | docker exec \ 32 | --user postgres \ 33 | job-board-postgres createdb job_board_test 34 | 35 | docker exec \ 36 | --user postgres \ 37 | job-board-postgres psql -l 38 | 39 | docker run \ 40 | --rm \ 41 | --name travis-worker-http-job-test \ 42 | --link job-board-postgres:postgres \ 43 | --link job-board-redis:redis \ 44 | -v "${JOB_BOARD_CLONE_DIR}:/usr/src/app" \ 45 | -v "${TOP:=$(git rev-parse --show-toplevel)}:/worker" \ 46 | -w /usr/src/app \ 47 | "ruby:3.2.5" \ 48 | /worker/script/http-job-test-internal 49 | } 50 | 51 | __cleanup() { 52 | if [[ "${JOB_BOARD_CLONE_DIR}" ]]; then 53 | rm -rf "${JOB_BOARD_CLONE_DIR}" 54 | fi 55 | docker rm -f job-board-{postgres,redis} 56 | } 57 | 58 | main "$@" 59 | -------------------------------------------------------------------------------- /script/http-job-test-internal: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | 4 | main() { 5 | export PGHOST=postgres 6 | export PGPASSWORD="${POSTGRES_ENV_POSTGRES_PASSWORD}" 7 | export PGPORT="${POSTGRES_PORT_5432_TCP_PORT}" 8 | export PGUSER=postgres 9 | export REDIS_PORT="${REDIS_PORT_6379_TCP_PORT}" 10 | 11 | export DATABASE_URL="postgres://${PGUSER}:${PGPASSWORD}@${PGHOST}:${PGPORT}/job_board_test" 12 | export INTEGRATION_SPECS=1 13 | export REDIS_URL="redis://redis:${REDIS_PORT}/0" 14 | 15 | apt-get update 16 | apt-get install -y perl postgresql-client 17 | script/install-sqitch 18 | eval "$(perl -I ~/perl5/lib/perl5/ '-Mlocal::lib')" 19 | sqitch deploy 20 | sqitch verify 21 | 22 | install -D /worker/build/linux/amd64/travis-worker \ 23 | ./spec/support/bin/travis-worker 24 | 25 | gem install bundler -v 2.4.19 26 | bundle install 27 | find . -type d -exec chmod a+rwx {} \; 28 | find . -type f -exec chmod a+rw {} \; 29 | exec bundle exec rspec ./spec/integration/worker_interaction_spec.rb 30 | } 31 | 32 | main "$@" 33 | -------------------------------------------------------------------------------- /script/lintall: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | 4 | main() { 5 | pushd "$(dirname "$(dirname "${BASH_SOURCE[0]}")")" 6 | 7 | # : "${CYCLOMAX:=20}" 8 | # : "${DEADLINE:=1m}" 9 | # : "${PACKAGE:=github.com/travis-ci/worker}" 10 | 11 | # : "${ENABLED_LINTER_FLAGS:=-E goimports -E gofmt -E goconst -E deadcode -E golint -E vet}" 12 | 13 | # GOLANGCILINT_ARGS="--disable-all ${ENABLED_LINTER_FLAGS}" 14 | # GOLANGCILINT_ARGS="${GOLANGCILINT_ARGS} --deadline=${DEADLINE}" 15 | # GOLANGCILINT_ARGS="${GOLANGCILINT_ARGS} --vendor" 16 | # GOLANGCILINT_ARGS="${GOLANGCILINT_ARGS} --tests" 17 | 18 | # if [[ "${DEBUG}" ]]; then 19 | # GOLANGCILINT_ARGS="${GOLANGCILINT_ARGS} --debug" 20 | # else 21 | # GOLANGCILINT_ARGS="${GOLANGCILINT_ARGS} --errors" 22 | # fi 23 | 24 | # IFS=" " read -r -a gmlargs_array <<<"$GOLANGCILINT_ARGS" 25 | 26 | set -o xtrace 27 | "$HOME"/bin/golangci-lint run 28 | # $(go env GOPATH)/bin/golangci-lint run "${gmlargs_array[@]}" 29 | git grep -l '^#!/usr/bin/env bash' | xargs shfmt -i 2 -w 30 | git grep -l '^#!/usr/bin/env bash' | xargs shellcheck 31 | } 32 | 33 | main "$@" 34 | -------------------------------------------------------------------------------- /script/list-packages: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | 4 | main() { 5 | cd "$(dirname "$(dirname "${BASH_SOURCE[0]}")")" 6 | 7 | find . -maxdepth 1 -type d | grep -v '^\.$' | sed 's@./@@' | while read -r pkg; do 8 | if [[ "_$(bash -c "shopt -s nullglob; echo ${pkg}/*.go")" == "_" ]]; then 9 | continue 10 | fi 11 | echo "github.com/travis-ci/worker/${pkg}" 12 | done 13 | } 14 | 15 | main "$@" 16 | -------------------------------------------------------------------------------- /script/publish-example-payload: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | import os 3 | import pika 4 | 5 | 6 | top = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) 7 | 8 | with open(os.path.join(top, 'example-payload.json'), 'r') as f: 9 | data = f.read() 10 | 11 | connection = pika.BlockingConnection(pika.ConnectionParameters('localhost')) 12 | channel = connection.channel() 13 | 14 | channel.basic_publish(exchange='', 15 | routing_key=os.environ["TRAVIS_WORKER_QUEUE_NAME"], 16 | body=data) 17 | print(" [x] Sent 'Enqueued the thing!'") 18 | 19 | connection.close() 20 | -------------------------------------------------------------------------------- /script/send-docker-hub-trigger: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script will trigger a multi-stage Docker build at the Docker hub repo referenced by $DOCKER_HUB_TRIGGER_URL. 4 | 5 | if [[ -n "$TRAVIS_PULL_REQUEST" && "$TRAVIS_PULL_REQUEST" != 'false' ]]; then 6 | echo "This is a pull request (${TRAVIS_PULL_REQUEST}); not triggering Docker hub build" 7 | exit 0 8 | fi 9 | 10 | [ -z "$TRAVIS_PULL_REQUEST" ] && echo "No TRAVIS_PULL_REQUEST env var found. Aborting." && exit 1 11 | [ -z "$DOCKER_HUB_TRIGGER_URL" ] && echo "No DOCKER_HUB_TRIGGER_URL env var found. Aborting." && exit 1 12 | [ -z "$TRAVIS_BRANCH" ] && echo "No TRAVIS_BRANCH env var found. Aborting." && exit 1 13 | 14 | echo "Triggering Docker Hub build on branch ${TRAVIS_BRANCH}" 15 | curl -H "Content-Type: application/json" \ 16 | --data '{"source_type":"Branch","source_name":"'${TRAVIS_BRANCH}'"}' \ 17 | -X POST "$DOCKER_HUB_TRIGGER_URL" 18 | -------------------------------------------------------------------------------- /script/smoke: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | 4 | main() { 5 | if [[ $DEBUG ]]; then 6 | set -o xtrace 7 | fi 8 | 9 | : "${OS:=$(go env GOHOSTOS)}" 10 | : "${ARCH:=$(go env GOHOSTARCH)}" 11 | 12 | cd "$(dirname "$(dirname "${BASH_SOURCE[0]}")")" 13 | 14 | git checkout HEAD go.sum 15 | git diff --exit-code 16 | git diff --cached --exit-code 17 | "./build/${OS}/${ARCH}/travis-worker" --version 18 | "./build/${OS}/${ARCH}/travis-worker" -v | grep -v '\?' 19 | "./build/${OS}/${ARCH}/travis-worker" --help 20 | "./build/${OS}/${ARCH}/travis-worker" -h 21 | diff -q \ 22 | <("./build/${OS}/${ARCH}/travis-worker" --help) \ 23 | <("./build/${OS}/${ARCH}/travis-worker" -h) 24 | N="${RANDOM}" 25 | TRAVIS_WORKER_FOO_N="${N}" "./build/${OS}/${ARCH}/travis-worker" \ 26 | --echo-config \ 27 | --provider-name=foo | grep "^export TRAVIS_WORKER_FOO_N=\"${N}\"\$" 28 | "./build/${OS}/${ARCH}/travis-worker" --echo-config 29 | "./build/${OS}/${ARCH}/travis-worker" --list-backend-providers 30 | } 31 | 32 | main "$@" 33 | -------------------------------------------------------------------------------- /script/smoke-docker: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | 4 | main() { 5 | if [[ -f "${DOCKER_ENV_FILE}" ]]; then 6 | # shellcheck source=/dev/null 7 | source "${DOCKER_ENV_FILE}" 8 | fi 9 | 10 | [[ ${DOCKER_DEST} ]] || { 11 | __log error="missing \$DOCKER_DEST" 12 | exit 1 13 | } 14 | 15 | unset TRAVIS_WORKER_DOCKER_CREDS 16 | unset TRAVIS_WORKER_DOCKER_LOGIN_PASSWORD 17 | unset TRAVIS_WORKER_DOCKER_LOGIN_USERNAME 18 | unset DOCKER_CREDS 19 | unset DOCKER_LOGIN_PASSWORD 20 | unset DOCKER_LOGIN_USERNAME 21 | 22 | set -o xtrace 23 | docker run --rm "${DOCKER_DEST}" --version &>/dev/null 24 | docker run --rm "${DOCKER_DEST}" travis-worker --version &>/dev/null 25 | docker run --rm "${DOCKER_DEST}" --help &>/dev/null 26 | docker run --rm "${DOCKER_DEST}" travis-worker --help &>/dev/null 27 | docker run --rm "${DOCKER_DEST}" curl --version &>/dev/null 28 | docker run --rm "${DOCKER_DEST}" curl https://www.google.com &>/dev/null 29 | } 30 | 31 | __log() { 32 | echo "time=$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$@" 33 | } 34 | 35 | main "$@" 36 | -------------------------------------------------------------------------------- /sentry_logrus_hook.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/getsentry/raven-go" 8 | "github.com/sirupsen/logrus" 9 | ) 10 | 11 | var ( 12 | severityMap = map[logrus.Level]raven.Severity{ 13 | logrus.DebugLevel: raven.DEBUG, 14 | logrus.InfoLevel: raven.INFO, 15 | logrus.WarnLevel: raven.WARNING, 16 | logrus.ErrorLevel: raven.ERROR, 17 | logrus.FatalLevel: raven.FATAL, 18 | logrus.PanicLevel: raven.FATAL, 19 | } 20 | ) 21 | 22 | // SentryHook delivers logs to a sentry server 23 | type SentryHook struct { 24 | Timeout time.Duration 25 | client *raven.Client 26 | levels []logrus.Level 27 | } 28 | 29 | // NewSentryHook creates a hook to be added to an instance of logger and 30 | // initializes the raven client. This method sets the timeout to 100 31 | // milliseconds. 32 | func NewSentryHook(DSN string, levels []logrus.Level) (*SentryHook, error) { 33 | client, err := raven.New(DSN) 34 | if err != nil { 35 | return nil, err 36 | } 37 | return &SentryHook{100 * time.Millisecond, client, levels}, nil 38 | } 39 | 40 | // Fire is called when an event should be sent to sentry 41 | func (hook *SentryHook) Fire(entry *logrus.Entry) error { 42 | defer func() { 43 | if r := recover(); r != nil { 44 | entry.Logger.WithField("panic", r).Error("paniced when trying to send log to sentry") 45 | } 46 | }() 47 | 48 | packet := &raven.Packet{ 49 | Message: entry.Message, 50 | Timestamp: raven.Timestamp(entry.Time), 51 | Level: severityMap[entry.Level], 52 | Platform: "go", 53 | } 54 | 55 | if serverName, ok := entry.Data["server_name"]; ok { 56 | packet.ServerName = serverName.(string) 57 | delete(entry.Data, "server_name") 58 | } 59 | packet.Extra = map[string]interface{}(entry.Data) 60 | 61 | if errMaybe, ok := packet.Extra["err"]; ok { 62 | if err, ok := errMaybe.(error); ok { 63 | packet.Extra["err"] = err.Error() 64 | } 65 | } 66 | 67 | packet.Interfaces = append(packet.Interfaces, raven.NewStacktrace(4, 3, []string{"github.com/travis-ci/worker"})) 68 | 69 | _, errCh := hook.client.Capture(packet, nil) 70 | if hook.Timeout != 0 { 71 | timeoutCh := time.After(hook.Timeout) 72 | select { 73 | case err := <-errCh: 74 | return err 75 | case <-timeoutCh: 76 | return fmt.Errorf("no response from sentry server in %s", hook.Timeout) 77 | } 78 | } 79 | return nil 80 | } 81 | 82 | // Levels returns the available logging levels. 83 | func (hook *SentryHook) Levels() []logrus.Level { 84 | return hook.levels 85 | } 86 | -------------------------------------------------------------------------------- /ssh/package.go: -------------------------------------------------------------------------------- 1 | package ssh 2 | 3 | import ( 4 | "crypto" 5 | "crypto/x509" 6 | "encoding/pem" 7 | "io" 8 | "os" 9 | "time" 10 | 11 | "golang.org/x/crypto/ssh" 12 | 13 | "github.com/pkg/errors" 14 | "github.com/pkg/sftp" 15 | ) 16 | 17 | type Dialer interface { 18 | Dial(address, username string, timeout time.Duration) (Connection, error) 19 | } 20 | type Connection interface { 21 | UploadFile(path string, data []byte) (bool, error) 22 | DownloadFile(path string) ([]byte, error) 23 | RunCommand(command string, output io.Writer) (int32, error) 24 | Close() error 25 | } 26 | 27 | func FormatPublicKey(key interface{}) ([]byte, error) { 28 | pubKey, err := ssh.NewPublicKey(key) 29 | if err != nil { 30 | return nil, errors.Wrap(err, "couldn't use public key") 31 | } 32 | 33 | return ssh.MarshalAuthorizedKey(pubKey), nil 34 | } 35 | 36 | type AuthDialer struct { 37 | authMethods []ssh.AuthMethod 38 | } 39 | 40 | func NewDialerWithKey(key crypto.Signer) (*AuthDialer, error) { 41 | signer, err := ssh.NewSignerFromKey(key) 42 | if err != nil { 43 | return nil, errors.Wrap(err, "couldn't create signer from SSH key") 44 | } 45 | 46 | return &AuthDialer{ 47 | authMethods: []ssh.AuthMethod{ssh.PublicKeys(signer)}, 48 | }, nil 49 | } 50 | 51 | func NewDialerWithPassword(password string) (*AuthDialer, error) { 52 | return &AuthDialer{ 53 | authMethods: []ssh.AuthMethod{ssh.Password(password)}, 54 | }, nil 55 | } 56 | 57 | func NewDialerWithKeyWithoutPassPhrase(pemBytes []byte) (*AuthDialer, error) { 58 | signer, err := ssh.ParsePrivateKey(pemBytes) 59 | if err != nil { 60 | return nil, errors.Wrap(err, "couldn't create signer from SSH key") 61 | } 62 | return &AuthDialer{ 63 | authMethods: []ssh.AuthMethod{ssh.PublicKeys(signer)}, 64 | }, nil 65 | } 66 | 67 | func NewDialer(keyPath, keyPassphrase string) (*AuthDialer, error) { 68 | file, err := os.ReadFile(keyPath) 69 | if err != nil { 70 | return nil, errors.Wrap(err, "couldn't read SSH key") 71 | } 72 | 73 | block, _ := pem.Decode(file) 74 | if block == nil { 75 | return nil, errors.Errorf("ssh key does not contain a valid PEM block") 76 | } 77 | 78 | if keyPassphrase == "" { 79 | return NewDialerWithKeyWithoutPassPhrase(file) 80 | } 81 | 82 | der, err := x509.DecryptPEMBlock(block, []byte(keyPassphrase)) //nolint:staticcheck 83 | if err != nil { 84 | return nil, errors.Wrap(err, "couldn't decrypt SSH key") 85 | } 86 | 87 | key, err := x509.ParsePKCS1PrivateKey(der) 88 | if err != nil { 89 | return nil, errors.Wrap(err, "couldn't parse SSH key") 90 | } 91 | 92 | return NewDialerWithKey(key) 93 | } 94 | 95 | func (d *AuthDialer) Dial(address, username string, timeout time.Duration) (Connection, error) { 96 | client, err := ssh.Dial("tcp", address, &ssh.ClientConfig{ 97 | User: username, 98 | Auth: d.authMethods, 99 | Timeout: timeout, 100 | // TODO: Verify server public key against something (optionally)? 101 | HostKeyCallback: ssh.InsecureIgnoreHostKey(), 102 | }) 103 | if err != nil { 104 | return nil, errors.Wrap(err, "couldn't connect to SSH server") 105 | } 106 | 107 | return &sshConnection{client: client}, nil 108 | } 109 | 110 | type sshConnection struct { 111 | client *ssh.Client 112 | } 113 | 114 | func (c *sshConnection) UploadFile(path string, data []byte) (bool, error) { 115 | sftp, err := sftp.NewClient(c.client) 116 | if err != nil { 117 | return false, errors.Wrap(err, "couldn't create SFTP client") 118 | } 119 | defer sftp.Close() 120 | 121 | _, err = sftp.Lstat(path) 122 | if err == nil { 123 | return true, errors.New("file already existed") 124 | } 125 | 126 | f, err := sftp.Create(path) 127 | if err != nil { 128 | return false, errors.Wrap(err, "couldn't create file") 129 | } 130 | 131 | _, err = f.Write(data) 132 | if err != nil { 133 | return false, errors.Wrap(err, "couldn't write contents to file") 134 | } 135 | 136 | return false, nil 137 | } 138 | 139 | func (c *sshConnection) DownloadFile(path string) ([]byte, error) { 140 | sftp, err := sftp.NewClient(c.client) 141 | if err != nil { 142 | return nil, errors.Wrap(err, "couldn't create SFTP client") 143 | } 144 | defer sftp.Close() 145 | 146 | // TODO: enforce file size limit 147 | 148 | _, err = sftp.Lstat(path) 149 | if err != nil { 150 | return nil, errors.Wrap(err, "couldn't stat file") 151 | } 152 | 153 | f, err := sftp.OpenFile(path, os.O_RDONLY) 154 | if err != nil { 155 | return nil, errors.Wrap(err, "couldn't open file") 156 | } 157 | 158 | buf, err := io.ReadAll(f) 159 | if err != nil { 160 | return nil, errors.Wrap(err, "couldn't read contents of file") 161 | } 162 | 163 | return buf, nil 164 | } 165 | 166 | func (c *sshConnection) RunCommand(command string, output io.Writer) (int32, error) { 167 | session, err := c.client.NewSession() 168 | if err != nil { 169 | return 0, errors.Wrap(err, "error creating SSH session") 170 | } 171 | defer session.Close() 172 | 173 | err = session.RequestPty("xterm", 40, 80, ssh.TerminalModes{}) 174 | if err != nil { 175 | return 0, errors.Wrap(err, "error requesting PTY") 176 | } 177 | 178 | session.Stdout = output 179 | session.Stderr = output 180 | 181 | err = session.Run(command) 182 | 183 | if err == nil { 184 | return 0, nil 185 | } 186 | 187 | switch err := err.(type) { 188 | case *ssh.ExitError: 189 | return int32(err.ExitStatus()), nil 190 | default: 191 | return 0, errors.Wrap(err, "error running script") 192 | } 193 | } 194 | 195 | func (c *sshConnection) Close() error { 196 | return c.client.Close() 197 | } 198 | -------------------------------------------------------------------------------- /step_check_cancellation.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | gocontext "context" 5 | "errors" 6 | "fmt" 7 | 8 | "github.com/mitchellh/multistep" 9 | "github.com/travis-ci/worker/context" 10 | "go.opencensus.io/trace" 11 | ) 12 | 13 | var JobCancelledError = errors.New("job cancelled") 14 | 15 | type stepCheckCancellation struct{} 16 | 17 | func (s *stepCheckCancellation) Run(state multistep.StateBag) multistep.StepAction { 18 | cancelChan := state.Get("cancelChan").(<-chan CancellationCommand) 19 | 20 | ctx := state.Get("ctx").(gocontext.Context) 21 | 22 | _, span := trace.StartSpan(ctx, "CheckCancellation.Run") 23 | defer span.End() 24 | 25 | select { 26 | case command := <-cancelChan: 27 | ctx := state.Get("ctx").(gocontext.Context) 28 | buildJob := state.Get("buildJob").(Job) 29 | if _, ok := state.GetOk("logWriter"); ok { 30 | logWriter := state.Get("logWriter").(LogWriter) 31 | s.writeLogAndFinishWithState(ctx, logWriter, buildJob, FinishStateCancelled, fmt.Sprintf("\n\nDone: Job Cancelled\n\n%s", command.Reason)) 32 | } else { 33 | err := buildJob.Finish(ctx, FinishStateCancelled) 34 | if err != nil { 35 | context.LoggerFromContext(ctx).WithField("err", err).WithField("state", FinishStateCancelled).Error("couldn't update job state") 36 | } 37 | } 38 | state.Put("err", JobCancelledError) 39 | return multistep.ActionHalt 40 | default: 41 | } 42 | 43 | return multistep.ActionContinue 44 | } 45 | 46 | func (s *stepCheckCancellation) Cleanup(state multistep.StateBag) {} 47 | 48 | func (s *stepCheckCancellation) writeLogAndFinishWithState(ctx gocontext.Context, logWriter LogWriter, buildJob Job, state FinishState, logMessage string) { 49 | ctx, span := trace.StartSpan(ctx, "WriteLogAndFinishWithState.CheckCancellation") 50 | defer span.End() 51 | 52 | _, err := logWriter.WriteAndClose([]byte(logMessage)) 53 | if err != nil { 54 | context.LoggerFromContext(ctx).WithField("err", err).Error("couldn't write final log message") 55 | } 56 | 57 | err = buildJob.Finish(ctx, state) 58 | if err != nil { 59 | context.LoggerFromContext(ctx).WithField("err", err).WithField("state", state).Error("couldn't update job state") 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /step_download_trace.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "os" 5 | "time" 6 | 7 | gocontext "context" 8 | 9 | "github.com/mitchellh/multistep" 10 | "github.com/pkg/errors" 11 | "github.com/sirupsen/logrus" 12 | "github.com/travis-ci/worker/backend" 13 | "github.com/travis-ci/worker/context" 14 | "github.com/travis-ci/worker/metrics" 15 | "go.opencensus.io/trace" 16 | ) 17 | 18 | type stepDownloadTrace struct { 19 | persister BuildTracePersister 20 | } 21 | 22 | func (s *stepDownloadTrace) Run(state multistep.StateBag) multistep.StepAction { 23 | if s.persister == nil { 24 | return multistep.ActionContinue 25 | } 26 | 27 | ctx := state.Get("ctx").(gocontext.Context) 28 | 29 | defer context.TimeSince(ctx, "step_download_trace_run", time.Now()) 30 | 31 | ctx, span := trace.StartSpan(ctx, "DownloadTrace.Run") 32 | defer span.End() 33 | 34 | buildJob := state.Get("buildJob").(Job) 35 | processedAt := state.Get("processedAt").(time.Time) 36 | 37 | instance := state.Get("instance").(backend.Instance) 38 | 39 | logger := context.LoggerFromContext(ctx).WithField("self", "step_download_trace") 40 | 41 | // ctx, cancel := gocontext.WithTimeout(ctx, s.uploadTimeout) 42 | // defer cancel() 43 | 44 | // downloading the trace is best-effort, so we continue in any case 45 | 46 | if !buildJob.Payload().Trace { 47 | return multistep.ActionContinue 48 | } 49 | 50 | buf, err := instance.DownloadTrace(ctx) 51 | if err != nil { 52 | span.SetStatus(trace.Status{ 53 | Code: trace.StatusCodeUnavailable, 54 | Message: err.Error(), 55 | }) 56 | 57 | if err == backend.ErrDownloadTraceNotImplemented || os.IsNotExist(errors.Cause(err)) { 58 | logger.WithFields(logrus.Fields{ 59 | "err": err, 60 | }).Info("skipping trace download") 61 | 62 | return multistep.ActionContinue 63 | } 64 | 65 | metrics.Mark("worker.job.trace.download.error") 66 | 67 | logger.WithFields(logrus.Fields{ 68 | "err": err, 69 | }).Error("couldn't download trace") 70 | context.CaptureError(ctx, err) 71 | 72 | return multistep.ActionContinue 73 | } 74 | 75 | logger.WithFields(logrus.Fields{ 76 | "since_processed_ms": time.Since(processedAt).Seconds() * 1e3, 77 | }).Info("downloaded trace") 78 | 79 | err = s.persister.Persist(ctx, buildJob, buf) 80 | 81 | if err != nil { 82 | metrics.Mark("worker.job.trace.persist.error") 83 | 84 | span.SetStatus(trace.Status{ 85 | Code: trace.StatusCodeUnavailable, 86 | Message: err.Error(), 87 | }) 88 | 89 | logger.WithFields(logrus.Fields{ 90 | "err": err, 91 | }).Error("couldn't persist trace") 92 | context.CaptureError(ctx, err) 93 | 94 | return multistep.ActionContinue 95 | } 96 | 97 | logger.WithFields(logrus.Fields{ 98 | "since_processed_ms": time.Since(processedAt).Seconds() * 1e3, 99 | }).Info("persisted trace") 100 | 101 | return multistep.ActionContinue 102 | } 103 | 104 | func (s *stepDownloadTrace) Cleanup(state multistep.StateBag) { 105 | // Nothing to clean up 106 | } 107 | -------------------------------------------------------------------------------- /step_generate_script.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "time" 5 | 6 | gocontext "context" 7 | 8 | "github.com/cenk/backoff" 9 | "github.com/mitchellh/multistep" 10 | "github.com/travis-ci/worker/context" 11 | "go.opencensus.io/trace" 12 | ) 13 | 14 | type stepGenerateScript struct { 15 | generator BuildScriptGenerator 16 | } 17 | 18 | func (s *stepGenerateScript) Run(state multistep.StateBag) multistep.StepAction { 19 | buildJob := state.Get("buildJob").(Job) 20 | ctx := state.Get("ctx").(gocontext.Context) 21 | 22 | defer context.TimeSince(ctx, "step_generate_script_run", time.Now()) 23 | 24 | ctx, span := trace.StartSpan(ctx, "GenerateScript.Run") 25 | defer span.End() 26 | 27 | logger := context.LoggerFromContext(ctx).WithField("self", "step_generate_script") 28 | 29 | b := backoff.NewExponentialBackOff() 30 | b.MaxInterval = 10 * time.Second 31 | b.MaxElapsedTime = time.Minute 32 | 33 | var script []byte 34 | var err error 35 | switch job := buildJob.(type) { 36 | case BuildScriptGenerator: 37 | logger.Info("using job to get script") 38 | script, err = job.Generate(ctx, buildJob) 39 | default: 40 | logger.Info("using build script generator to generate script") 41 | err = backoff.Retry(func() (err error) { 42 | script, err = s.generator.Generate(ctx, buildJob) 43 | return 44 | }, b) 45 | } 46 | 47 | if err != nil { 48 | state.Put("err", err) 49 | 50 | span.SetStatus(trace.Status{ 51 | Code: trace.StatusCodeUnavailable, 52 | Message: err.Error(), 53 | }) 54 | 55 | logger.WithField("err", err).Error("couldn't generate build script, erroring job") 56 | err := buildJob.Error(ctx, "An error occurred while generating the build script.") 57 | if err != nil { 58 | logger.WithField("err", err).Error("couldn't requeue job") 59 | } 60 | 61 | return multistep.ActionHalt 62 | } 63 | 64 | logger.Info("generated script") 65 | 66 | state.Put("script", script) 67 | 68 | return multistep.ActionContinue 69 | } 70 | 71 | func (s *stepGenerateScript) Cleanup(multistep.StateBag) { 72 | // Nothing to clean up 73 | } 74 | -------------------------------------------------------------------------------- /step_open_log_writer.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "time" 5 | 6 | gocontext "context" 7 | 8 | "github.com/mitchellh/multistep" 9 | "github.com/sirupsen/logrus" 10 | "github.com/travis-ci/worker/context" 11 | "go.opencensus.io/trace" 12 | ) 13 | 14 | type stepOpenLogWriter struct { 15 | maxLogLength int 16 | defaultLogTimeout time.Duration 17 | } 18 | 19 | func (s *stepOpenLogWriter) Run(state multistep.StateBag) multistep.StepAction { 20 | ctx := state.Get("ctx").(gocontext.Context) 21 | buildJob := state.Get("buildJob").(Job) 22 | logWriterFactory := state.Get("logWriterFactory") 23 | logger := context.LoggerFromContext(ctx).WithField("self", "step_open_log_writer") 24 | 25 | ctx, span := trace.StartSpan(ctx, "OpenLogWriter.Run") 26 | defer span.End() 27 | 28 | var logWriter LogWriter 29 | var err error 30 | 31 | if logWriterFactory != nil { 32 | logWriter, err = logWriterFactory.(LogWriterFactory).LogWriter(ctx, s.defaultLogTimeout, buildJob) 33 | } else { 34 | logWriter, err = buildJob.LogWriter(ctx, s.defaultLogTimeout) 35 | } 36 | if err != nil { 37 | state.Put("err", err) 38 | 39 | logger.WithFields(logrus.Fields{ 40 | "err": err, 41 | "log_timeout": s.defaultLogTimeout, 42 | }).Error("couldn't open a log writer, attempting requeue") 43 | context.CaptureError(ctx, err) 44 | 45 | err := buildJob.Requeue(ctx) 46 | if err != nil { 47 | logger.WithField("err", err).Error("couldn't requeue job") 48 | } 49 | 50 | return multistep.ActionHalt 51 | } 52 | logWriter.SetMaxLogLength(s.maxLogLength) 53 | 54 | state.Put("logWriter", logWriter) 55 | 56 | return multistep.ActionContinue 57 | } 58 | 59 | func (s *stepOpenLogWriter) Cleanup(state multistep.StateBag) { 60 | ctx := state.Get("ctx").(gocontext.Context) 61 | 62 | _, span := trace.StartSpan(ctx, "OpenLogWriter.Cleanup") 63 | defer span.End() 64 | 65 | logWriter, ok := state.Get("logWriter").(LogWriter) 66 | if ok { 67 | logWriter.Close() 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /step_open_log_writer_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "testing" 5 | 6 | gocontext "context" 7 | 8 | "github.com/mitchellh/multistep" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func setupStepOpenLogWriter() (*stepOpenLogWriter, multistep.StateBag) { 13 | s := &stepOpenLogWriter{} 14 | 15 | ctx := gocontext.TODO() 16 | 17 | job := &fakeJob{ 18 | payload: &JobPayload{ 19 | Type: "job:test", 20 | Job: JobJobPayload{ 21 | ID: 2, 22 | Number: "3.1", 23 | }, 24 | Build: BuildPayload{ 25 | ID: 1, 26 | Number: "3", 27 | }, 28 | Repository: RepositoryPayload{ 29 | ID: 4, 30 | Slug: "green-eggs/ham", 31 | }, 32 | UUID: "foo-bar", 33 | Config: map[string]interface{}{}, 34 | Timeouts: TimeoutsPayload{}, 35 | }, 36 | } 37 | 38 | state := &multistep.BasicStateBag{} 39 | state.Put("ctx", ctx) 40 | state.Put("buildJob", job) 41 | 42 | return s, state 43 | } 44 | 45 | func TestStepOpenLogWriter_Run(t *testing.T) { 46 | s, state := setupStepOpenLogWriter() 47 | action := s.Run(state) 48 | assert.Equal(t, multistep.ActionContinue, action) 49 | assert.NotNil(t, state.Get("logWriter")) 50 | } 51 | -------------------------------------------------------------------------------- /step_send_received.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | gocontext "context" 5 | "time" 6 | 7 | "github.com/mitchellh/multistep" 8 | "github.com/sirupsen/logrus" 9 | "github.com/travis-ci/worker/context" 10 | "go.opencensus.io/trace" 11 | ) 12 | 13 | type stepSendReceived struct{} 14 | 15 | func (s *stepSendReceived) Run(state multistep.StateBag) multistep.StepAction { 16 | buildJob := state.Get("buildJob").(Job) 17 | ctx := state.Get("ctx").(gocontext.Context) 18 | 19 | defer context.TimeSince(ctx, "step_send_received_run", time.Now()) 20 | 21 | ctx, span := trace.StartSpan(ctx, "SendReceived.Run") 22 | defer span.End() 23 | 24 | err := buildJob.Received(ctx) 25 | if err != nil { 26 | context.LoggerFromContext(ctx).WithFields(logrus.Fields{ 27 | "err": err, 28 | "self": "step_send_received", 29 | }).Error("couldn't send received event") 30 | } 31 | 32 | return multistep.ActionContinue 33 | } 34 | 35 | func (s *stepSendReceived) Cleanup(state multistep.StateBag) {} 36 | -------------------------------------------------------------------------------- /step_sleep.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | gocontext "context" 5 | "time" 6 | 7 | "github.com/mitchellh/multistep" 8 | "github.com/travis-ci/worker/context" 9 | "go.opencensus.io/trace" 10 | ) 11 | 12 | type stepSleep struct { 13 | duration time.Duration 14 | } 15 | 16 | func (s *stepSleep) Run(state multistep.StateBag) multistep.StepAction { 17 | ctx := state.Get("ctx").(gocontext.Context) 18 | 19 | defer context.TimeSince(ctx, "step_sleep_run", time.Now()) 20 | 21 | _, span := trace.StartSpan(ctx, "Sleep.Run") 22 | defer span.End() 23 | 24 | time.Sleep(s.duration) 25 | 26 | return multistep.ActionContinue 27 | } 28 | 29 | func (s *stepSleep) Cleanup(state multistep.StateBag) {} 30 | -------------------------------------------------------------------------------- /step_start_instance.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "time" 5 | 6 | gocontext "context" 7 | 8 | "github.com/mitchellh/multistep" 9 | "github.com/pkg/errors" 10 | "github.com/sirupsen/logrus" 11 | "github.com/travis-ci/worker/backend" 12 | "github.com/travis-ci/worker/context" 13 | workererrors "github.com/travis-ci/worker/errors" 14 | "go.opencensus.io/trace" 15 | ) 16 | 17 | type stepStartInstance struct { 18 | provider backend.Provider 19 | startTimeout time.Duration 20 | } 21 | 22 | func (s *stepStartInstance) Run(state multistep.StateBag) multistep.StepAction { 23 | buildJob := state.Get("buildJob").(Job) 24 | ctx := state.Get("ctx").(gocontext.Context) 25 | logWriter := state.Get("logWriter").(LogWriter) 26 | 27 | logger := context.LoggerFromContext(ctx).WithField("self", "step_start_instance") 28 | 29 | defer context.TimeSince(ctx, "step_start_instance_run", time.Now()) 30 | 31 | ctx, span := trace.StartSpan(ctx, "StartInstance.Run") 32 | defer span.End() 33 | 34 | logger.Info("starting instance") 35 | 36 | preTimeoutCtx := ctx 37 | 38 | ctx, cancel := gocontext.WithTimeout(ctx, s.startTimeout) 39 | defer cancel() 40 | 41 | startTime := time.Now() 42 | 43 | var ( 44 | instance backend.Instance 45 | err error 46 | ) 47 | 48 | if s.provider.SupportsProgress() && buildJob.StartAttributes().ProgressType != "" { 49 | var progresser backend.Progresser 50 | switch buildJob.StartAttributes().ProgressType { 51 | case "text": 52 | progresser = backend.NewTextProgresser(logWriter) 53 | _, _ = writeFoldStart(logWriter, "step_start_instance", []byte("\033[33;1mStarting instance\033[0m\r\n")) 54 | defer func() { 55 | _, err := writeFoldEnd(logWriter, "step_start_instance", []byte("")) 56 | if err != nil { 57 | logger.WithFields(logrus.Fields{ 58 | "err": err, 59 | }).Error("couldn't write fold end") 60 | } 61 | }() 62 | default: 63 | logger.WithField("progress_type", buildJob.StartAttributes().ProgressType).Warn("unknown progress type") 64 | progresser = &backend.NullProgresser{} 65 | } 66 | 67 | instance, err = s.provider.StartWithProgress(ctx, buildJob.StartAttributes(), progresser) 68 | } else { 69 | instance, err = s.provider.Start(ctx, buildJob.StartAttributes()) 70 | } 71 | 72 | if err != nil { 73 | state.Put("err", err) 74 | 75 | span.SetStatus(trace.Status{ 76 | Code: trace.StatusCodeUnavailable, 77 | Message: err.Error(), 78 | }) 79 | 80 | jobAbortErr, ok := errors.Cause(err).(workererrors.JobAbortError) 81 | if ok { 82 | logWriter.WriteAndClose([]byte(jobAbortErr.UserFacingErrorMessage())) 83 | 84 | err = buildJob.Finish(preTimeoutCtx, FinishStateErrored) 85 | if err != nil { 86 | logger.WithField("err", err).WithField("state", FinishStateErrored).Error("couldn't mark job as finished") 87 | } 88 | 89 | return multistep.ActionHalt 90 | } 91 | 92 | logger.WithFields(logrus.Fields{ 93 | "err": err, 94 | "start_timeout": s.startTimeout, 95 | }).Error("couldn't start instance, attempting requeue") 96 | context.CaptureError(ctx, err) 97 | 98 | err := buildJob.Requeue(preTimeoutCtx) 99 | if err != nil { 100 | logger.WithField("err", err).Error("couldn't requeue job") 101 | } 102 | 103 | return multistep.ActionHalt 104 | } 105 | 106 | logger.WithFields(logrus.Fields{ 107 | "boot_duration_ms": time.Since(startTime).Seconds() * 1e3, 108 | "instance_id": instance.ID(), 109 | "image_name": instance.ImageName(), 110 | "version": VersionString, 111 | "warmed": instance.Warmed(), 112 | }).Info("started instance") 113 | 114 | state.Put("instance", instance) 115 | 116 | return multistep.ActionContinue 117 | } 118 | 119 | func (s *stepStartInstance) Cleanup(state multistep.StateBag) { 120 | ctx := state.Get("ctx").(gocontext.Context) 121 | 122 | defer context.TimeSince(ctx, "step_start_instance_cleanup", time.Now()) 123 | 124 | ctx, span := trace.StartSpan(ctx, "StartInstance.Cleanup") 125 | defer span.End() 126 | 127 | instance, ok := state.Get("instance").(backend.Instance) 128 | logger := context.LoggerFromContext(ctx).WithField("self", "step_start_instance") 129 | if !ok { 130 | logger.Info("no instance to stop") 131 | return 132 | } 133 | 134 | skipShutdown, ok := state.Get("skipShutdown").(bool) 135 | if ok && skipShutdown { 136 | logger.WithField("instance", instance).Error("skipping shutdown, VM will be left running") 137 | return 138 | } 139 | 140 | if err := instance.Stop(ctx); err != nil { 141 | logger.WithFields(logrus.Fields{"err": err, "instance": instance}).Warn("couldn't stop instance") 142 | } else { 143 | logger.Info("stopped instance") 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /step_subscribe_cancellation.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | gocontext "context" 5 | 6 | "github.com/mitchellh/multistep" 7 | "go.opencensus.io/trace" 8 | ) 9 | 10 | type stepSubscribeCancellation struct { 11 | cancellationBroadcaster *CancellationBroadcaster 12 | } 13 | 14 | func (s *stepSubscribeCancellation) Run(state multistep.StateBag) multistep.StepAction { 15 | ctx := state.Get("ctx").(gocontext.Context) 16 | 17 | _, span := trace.StartSpan(ctx, "SubscribeCancellation.Run") 18 | defer span.End() 19 | 20 | if s.cancellationBroadcaster == nil { 21 | ch := make(chan CancellationCommand) 22 | state.Put("cancelChan", (<-chan CancellationCommand)(ch)) 23 | return multistep.ActionContinue 24 | } 25 | 26 | buildJob := state.Get("buildJob").(Job) 27 | ch := s.cancellationBroadcaster.Subscribe(buildJob.Payload().Job.ID) 28 | state.Put("cancelChan", ch) 29 | 30 | return multistep.ActionContinue 31 | } 32 | 33 | func (s *stepSubscribeCancellation) Cleanup(state multistep.StateBag) { 34 | if s.cancellationBroadcaster == nil { 35 | return 36 | } 37 | 38 | ctx := state.Get("ctx").(gocontext.Context) 39 | 40 | _, span := trace.StartSpan(ctx, "SubscribeCancellation.Cleanup") 41 | defer span.End() 42 | 43 | buildJob := state.Get("buildJob").(Job) 44 | ch := state.Get("cancelChan").(<-chan CancellationCommand) 45 | s.cancellationBroadcaster.Unsubscribe(buildJob.Payload().Job.ID, ch) 46 | } 47 | -------------------------------------------------------------------------------- /step_transform_build_json.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os/exec" 7 | "strings" 8 | 9 | "github.com/mitchellh/multistep" 10 | "github.com/travis-ci/worker/context" 11 | "go.opencensus.io/trace" 12 | gocontext "golang.org/x/net/context" 13 | ) 14 | 15 | type stepTransformBuildJSON struct { 16 | payloadFilterExecutable string 17 | } 18 | 19 | type EnvVar struct { 20 | Name string 21 | Public bool 22 | Value string 23 | } 24 | 25 | func (s *stepTransformBuildJSON) Run(state multistep.StateBag) multistep.StepAction { 26 | buildJob := state.Get("buildJob").(Job) 27 | ctx := state.Get("ctx").(gocontext.Context) 28 | 29 | ctx, span := trace.StartSpan(ctx, "TransformBuildJSON.Run") 30 | defer span.End() 31 | 32 | if s.payloadFilterExecutable == "" { 33 | context.LoggerFromContext(ctx).Info("skipping json transformation, no filter executable defined") 34 | return multistep.ActionContinue 35 | } 36 | 37 | context.LoggerFromContext(ctx).Info(fmt.Sprintf("calling filter executable: %s", s.payloadFilterExecutable)) 38 | 39 | payload := buildJob.RawPayload() 40 | 41 | cmd := exec.Command(s.payloadFilterExecutable) 42 | rawJson, err := payload.MarshalJSON() 43 | if err != nil { 44 | context.LoggerFromContext(ctx).Info(fmt.Sprintf("failed to marshal json: %v", err)) 45 | return multistep.ActionContinue 46 | } 47 | 48 | cmd.Stdin = strings.NewReader(string(rawJson)) 49 | 50 | var out bytes.Buffer 51 | cmd.Stdout = &out 52 | 53 | err = cmd.Run() 54 | if err != nil { 55 | context.LoggerFromContext(ctx).Info(fmt.Sprintf("failed to run filter executable: %v", err)) 56 | return multistep.ActionContinue 57 | } 58 | 59 | err = payload.UnmarshalJSON(out.Bytes()) 60 | if err != nil { 61 | context.LoggerFromContext(ctx).Info(fmt.Sprintf("failed to unmarshal json: %v", err)) 62 | return multistep.ActionContinue 63 | } 64 | 65 | context.LoggerFromContext(ctx).Info("replaced the build json") 66 | 67 | return multistep.ActionContinue 68 | } 69 | 70 | func (s *stepTransformBuildJSON) Cleanup(multistep.StateBag) { 71 | // Nothing to clean up 72 | } 73 | -------------------------------------------------------------------------------- /step_transform_build_json_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "testing" 5 | 6 | gocontext "golang.org/x/net/context" 7 | 8 | "github.com/bitly/go-simplejson" 9 | "github.com/mitchellh/multistep" 10 | "github.com/stretchr/testify/assert" 11 | "github.com/travis-ci/worker/backend" 12 | "github.com/travis-ci/worker/config" 13 | ) 14 | 15 | func setupStepTransformBuildJSON(cfg *config.ProviderConfig) (*stepTransformBuildJSON, multistep.StateBag) { 16 | s := &stepTransformBuildJSON{} 17 | 18 | bp, _ := backend.NewBackendProvider("fake", cfg) 19 | 20 | ctx := gocontext.TODO() 21 | instance, _ := bp.Start(ctx, nil) 22 | 23 | rawPayload, _ := simplejson.NewJson([]byte("{}")) 24 | 25 | job := &fakeJob{ 26 | rawPayload: rawPayload, 27 | } 28 | 29 | state := &multistep.BasicStateBag{} 30 | state.Put("ctx", ctx) 31 | state.Put("buildJob", job) 32 | state.Put("instance", instance) 33 | 34 | return s, state 35 | } 36 | 37 | func TestStepTransformBuildJSON_Run(t *testing.T) { 38 | 39 | cfg := config.ProviderConfigFromMap(map[string]string{ 40 | "PAYLOAD_FILTER_EXECUTABLE": "", 41 | }) 42 | 43 | s, state := setupStepTransformBuildJSON(cfg) 44 | action := s.Run(state) 45 | assert.Equal(t, multistep.ActionContinue, action) 46 | } 47 | 48 | func TestStepTransformBuildJSON_RunWithExecutableConfigured(t *testing.T) { 49 | 50 | cfg := config.ProviderConfigFromMap(map[string]string{ 51 | "PAYLOAD_FILTER_EXECUTABLE": "/usr/local/bin/filter.py", 52 | }) 53 | 54 | s, state := setupStepTransformBuildJSON(cfg) 55 | action := s.Run(state) 56 | assert.Equal(t, multistep.ActionContinue, action) 57 | } 58 | -------------------------------------------------------------------------------- /step_update_state.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | gocontext "context" 5 | "time" 6 | 7 | "github.com/mitchellh/multistep" 8 | "github.com/sirupsen/logrus" 9 | "github.com/travis-ci/worker/backend" 10 | "github.com/travis-ci/worker/context" 11 | "go.opencensus.io/trace" 12 | ) 13 | 14 | type stepUpdateState struct{} 15 | 16 | func (s *stepUpdateState) Run(state multistep.StateBag) multistep.StepAction { 17 | ctx := state.Get("ctx").(gocontext.Context) 18 | buildJob := state.Get("buildJob").(Job) 19 | instance := state.Get("instance").(backend.Instance) 20 | processedAt := state.Get("processedAt").(time.Time) 21 | logWriter := state.Get("logWriter").(LogWriter) 22 | 23 | logger := context.LoggerFromContext(ctx).WithField("self", "step_update_state") 24 | 25 | instanceID := instance.ID() 26 | if instanceID != "" { 27 | ctx = context.FromInstanceID(ctx, instanceID) 28 | state.Put("ctx", ctx) 29 | } 30 | 31 | defer context.TimeSince(ctx, "step_update_state_run", time.Now()) 32 | 33 | ctx, span := trace.StartSpan(ctx, "UpdateState.Run") 34 | defer span.End() 35 | 36 | logWriter.SetJobStarted(&JobStartedMeta{ 37 | QueuedAt: buildJob.Payload().Job.QueuedAt, 38 | Repo: buildJob.Payload().Repository.Slug, 39 | Queue: buildJob.Payload().Queue, 40 | Infra: state.Get("infra").(string), 41 | }) 42 | 43 | err := buildJob.Started(ctx) 44 | if err != nil { 45 | context.LoggerFromContext(ctx).WithFields(logrus.Fields{ 46 | "err": err, 47 | "self": "step_update_state", 48 | "instance_id": instanceID, 49 | }).Error("couldn't mark job as started") 50 | } 51 | 52 | logger.WithFields(logrus.Fields{ 53 | "since_processed_ms": time.Since(processedAt).Seconds() * 1e3, 54 | "action": "run", 55 | }).Info("marked job as started") 56 | 57 | return multistep.ActionContinue 58 | } 59 | 60 | func (s *stepUpdateState) Cleanup(state multistep.StateBag) { 61 | buildJob := state.Get("buildJob").(Job) 62 | ctx := state.Get("ctx").(gocontext.Context) 63 | processedAt := state.Get("processedAt").(time.Time) 64 | 65 | instance := state.Get("instance").(backend.Instance) 66 | instanceID := instance.ID() 67 | if instanceID != "" { 68 | ctx = context.FromInstanceID(ctx, instanceID) 69 | state.Put("ctx", ctx) 70 | } 71 | 72 | defer context.TimeSince(ctx, "step_update_state_cleanup", time.Now()) 73 | 74 | ctx, span := trace.StartSpan(ctx, "UpdateState.Cleanup") 75 | defer span.End() 76 | 77 | logger := context.LoggerFromContext(ctx).WithField("self", "step_update_state") 78 | logger.WithFields(logrus.Fields{ 79 | "since_processed_ms": time.Since(processedAt).Seconds() * 1e3, 80 | "action": "cleanup", 81 | }).Info("cleaning up") 82 | 83 | mresult, ok := state.GetOk("scriptResult") 84 | 85 | if ok { 86 | result := mresult.(*backend.RunResult) 87 | 88 | var err error 89 | 90 | switch result.ExitCode { 91 | case 0: 92 | err = buildJob.Finish(ctx, FinishStatePassed) 93 | case 1: 94 | err = buildJob.Finish(ctx, FinishStateFailed) 95 | default: 96 | err = buildJob.Finish(ctx, FinishStateErrored) 97 | } 98 | 99 | if err != nil { 100 | context.LoggerFromContext(ctx).WithFields(logrus.Fields{ 101 | "err": err, 102 | "self": "step_update_state", 103 | "instance_id": instanceID, 104 | }).Error("couldn't mark job as finished") 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /step_upload_script.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "time" 5 | 6 | gocontext "context" 7 | 8 | "github.com/mitchellh/multistep" 9 | "github.com/pkg/errors" 10 | "github.com/sirupsen/logrus" 11 | "github.com/travis-ci/worker/backend" 12 | "github.com/travis-ci/worker/context" 13 | "github.com/travis-ci/worker/metrics" 14 | "go.opencensus.io/trace" 15 | ) 16 | 17 | type stepUploadScript struct { 18 | uploadTimeout time.Duration 19 | } 20 | 21 | func (s *stepUploadScript) Run(state multistep.StateBag) multistep.StepAction { 22 | ctx := state.Get("ctx").(gocontext.Context) 23 | buildJob := state.Get("buildJob").(Job) 24 | logWriter := state.Get("logWriter").(LogWriter) 25 | processedAt := state.Get("processedAt").(time.Time) 26 | 27 | instance := state.Get("instance").(backend.Instance) 28 | script := state.Get("script").([]byte) 29 | 30 | logger := context.LoggerFromContext(ctx).WithField("self", "step_upload_script") 31 | 32 | defer context.TimeSince(ctx, "step_upload_script_run", time.Now()) 33 | 34 | ctx, span := trace.StartSpan(ctx, "UploadScript.Run") 35 | defer span.End() 36 | 37 | preTimeoutCtx := ctx 38 | 39 | ctx, cancel := gocontext.WithTimeout(ctx, s.uploadTimeout) 40 | defer cancel() 41 | 42 | if instance.SupportsProgress() && buildJob.StartAttributes().ProgressType == "text" { 43 | _, _ = writeFoldStart(logWriter, "step_upload_script", []byte("\033[33;1mUploading script\033[0m\r\n")) 44 | defer func() { 45 | _, err := writeFoldEnd(logWriter, "step_upload_script", []byte("")) 46 | if err != nil { 47 | logger.WithFields(logrus.Fields{ 48 | "err": err, 49 | }).Error("couldn't write fold end") 50 | } 51 | }() 52 | } 53 | 54 | err := instance.UploadScript(ctx, script) 55 | if err != nil { 56 | state.Put("err", err) 57 | 58 | span.SetStatus(trace.Status{ 59 | Code: trace.StatusCodeUnavailable, 60 | Message: err.Error(), 61 | }) 62 | 63 | errMetric := "worker.job.upload.error" 64 | if errors.Cause(err) == backend.ErrStaleVM { 65 | errMetric += ".stalevm" 66 | } 67 | metrics.Mark(errMetric) 68 | 69 | logger.WithFields(logrus.Fields{ 70 | "err": err, 71 | "upload_timeout": s.uploadTimeout, 72 | }).Error("couldn't upload script, attempting requeue") 73 | context.CaptureError(ctx, err) 74 | 75 | err := buildJob.Requeue(preTimeoutCtx) 76 | if err != nil { 77 | logger.WithField("err", err).Error("couldn't requeue job") 78 | } 79 | 80 | return multistep.ActionHalt 81 | } 82 | 83 | logger.WithFields(logrus.Fields{ 84 | "since_processed_ms": time.Since(processedAt).Seconds() * 1e3, 85 | }).Info("uploaded script") 86 | 87 | return multistep.ActionContinue 88 | } 89 | 90 | func (s *stepUploadScript) Cleanup(state multistep.StateBag) { 91 | // Nothing to clean up 92 | } 93 | -------------------------------------------------------------------------------- /step_write_worker_info.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | gocontext "context" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/mitchellh/multistep" 9 | "github.com/travis-ci/worker/backend" 10 | "go.opencensus.io/trace" 11 | ) 12 | 13 | type stepWriteWorkerInfo struct { 14 | } 15 | 16 | func (s *stepWriteWorkerInfo) Run(state multistep.StateBag) multistep.StepAction { 17 | logWriter := state.Get("logWriter").(LogWriter) 18 | buildJob := state.Get("buildJob").(Job) 19 | instance := state.Get("instance").(backend.Instance) 20 | ctx := state.Get("ctx").(gocontext.Context) 21 | 22 | _, span := trace.StartSpan(ctx, "WriteWorkerInfo.Run") 23 | defer span.End() 24 | 25 | if hostname, ok := state.Get("hostname").(string); ok && hostname != "" { 26 | _, _ = writeFold(logWriter, "worker_info", []byte(strings.Join([]string{ 27 | "\033[33;1mWorker information\033[0m", 28 | fmt.Sprintf("hostname: %s", hostname), 29 | fmt.Sprintf("version: %s %s", VersionString, RevisionURLString), 30 | fmt.Sprintf("instance: %s %s (via %s)", instance.ID(), instance.ImageName(), buildJob.Name()), 31 | fmt.Sprintf("startup: %v", instance.StartupDuration()), 32 | }, "\n"))) 33 | } 34 | 35 | return multistep.ActionContinue 36 | } 37 | 38 | func (s *stepWriteWorkerInfo) Cleanup(state multistep.StateBag) {} 39 | -------------------------------------------------------------------------------- /step_write_worker_info_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | "time" 7 | 8 | gocontext "context" 9 | 10 | "github.com/mitchellh/multistep" 11 | "github.com/stretchr/testify/assert" 12 | "github.com/travis-ci/worker/backend" 13 | "github.com/travis-ci/worker/config" 14 | ) 15 | 16 | type byteBufferLogWriter struct { 17 | *bytes.Buffer 18 | } 19 | 20 | func (w *byteBufferLogWriter) Close() error { 21 | return nil 22 | } 23 | 24 | func (w *byteBufferLogWriter) WriteAndClose(p []byte) (int, error) { 25 | return w.Write(p) 26 | } 27 | 28 | func (w *byteBufferLogWriter) Timeout() <-chan time.Time { 29 | return make(<-chan time.Time) 30 | } 31 | 32 | func (w *byteBufferLogWriter) SetMaxLogLength(m int) { 33 | } 34 | 35 | func (w *byteBufferLogWriter) SetJobStarted(meta *JobStartedMeta) { 36 | } 37 | 38 | func (w *byteBufferLogWriter) SetCancelFunc(_ gocontext.CancelFunc) { 39 | } 40 | 41 | func (w *byteBufferLogWriter) MaxLengthReached() bool { 42 | return false 43 | } 44 | 45 | func setupStepWriteWorkerInfo() (*stepWriteWorkerInfo, *byteBufferLogWriter, multistep.StateBag) { 46 | s := &stepWriteWorkerInfo{} 47 | 48 | bp, _ := backend.NewBackendProvider("fake", 49 | config.ProviderConfigFromMap(map[string]string{ 50 | "STARTUP_DURATION": "42.17s", 51 | })) 52 | 53 | ctx := gocontext.TODO() 54 | instance, _ := bp.Start(ctx, nil) 55 | 56 | logWriter := &byteBufferLogWriter{ 57 | bytes.NewBufferString(""), 58 | } 59 | 60 | state := &multistep.BasicStateBag{} 61 | state.Put("ctx", ctx) 62 | state.Put("logWriter", logWriter) 63 | state.Put("instance", instance) 64 | state.Put("hostname", "frizzlefry.example.local") 65 | state.Put("buildJob", &fakeJob{payload: &JobPayload{Job: JobJobPayload{ID: 4}}}) 66 | 67 | return s, logWriter, state 68 | } 69 | 70 | func TestStepWriteWorkerInfo_Run(t *testing.T) { 71 | s, logWriter, state := setupStepWriteWorkerInfo() 72 | 73 | s.Run(state) 74 | 75 | out := logWriter.String() 76 | assert.Contains(t, out, "travis_fold:start:worker_info\r\033[0K") 77 | assert.Contains(t, out, "\033[33;1mWorker information\033[0m\n") 78 | assert.Contains(t, out, "\nhostname: frizzlefry.example.local\n") 79 | assert.Contains(t, out, "\nversion: "+VersionString+" "+RevisionURLString+"\n") 80 | assert.Contains(t, out, "\ninstance: fake fake (via fake)\n") 81 | assert.Contains(t, out, "\nstartup: 42.17s\n") 82 | assert.Contains(t, out, "\ntravis_fold:end:worker_info\r\033[0K") 83 | } 84 | -------------------------------------------------------------------------------- /systemd-wrapper: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -o errexit 3 | 4 | main() { 5 | local name="${1:-travis-worker}" 6 | 7 | docker stop "${name}" &>/dev/null || true 8 | docker rm -f "${name}" &>/dev/null || true 9 | 10 | local env_file 11 | env_file="$(tfw writeenv travis-worker)" 12 | 13 | set -o allexport 14 | # shellcheck source=/dev/null 15 | source "${env_file}" 16 | 17 | if [[ "${TRAVIS_WORKER_PROVIDER_NAME}" == gce ]]; then 18 | local gce_zone 19 | gce_zone="$(__fetch_gce_zone)" 20 | if [ -z "${gce_zone}" ]; then 21 | gce_zone=us-central1-b 22 | fi 23 | echo "TRAVIS_WORKER_GCE_ZONE=${gce_zone}" >>"${env_file}" 24 | fi 25 | 26 | if [[ ! "${TRAVIS_WORKER_LIBRATO_SOURCE}" ]]; then 27 | local librato_source 28 | librato_source="$(__build_librato_source "$(hostname)" "${name}")" 29 | echo "TRAVIS_WORKER_LIBRATO_SOURCE=${librato_source}" >>"${env_file}" 30 | fi 31 | 32 | if [ -f "${TRAVIS_WORKER_PRESTART_HOOK}" ]; then 33 | "${TRAVIS_WORKER_PRESTART_HOOK}" 34 | fi 35 | 36 | exec docker run \ 37 | --rm \ 38 | --name "${name}" \ 39 | --hostname "$(hostname)" \ 40 | --userns host \ 41 | -v /var/tmp:/var/tmp \ 42 | -v /var/run:/var/run \ 43 | --env-file "${env_file}" \ 44 | "${TRAVIS_WORKER_SELF_IMAGE}" travis-worker 45 | } 46 | 47 | __fetch_gce_zone() { 48 | curl -sSL \ 49 | "http://metadata.google.internal/computeMetadata/v1/instance/zone" \ 50 | -H "Metadata-Flavor: Google" | 51 | awk -F/ '{ print $NF }' 52 | } 53 | 54 | __build_librato_source() { 55 | local host_name="${1}" 56 | local name="${2}" 57 | 58 | if [[ "${name}" == "travis-worker" ]]; then 59 | echo "${host_name}" 60 | return 61 | fi 62 | 63 | echo "${host_name}-${name/travis-worker-/}" 64 | } 65 | 66 | main "$@" 67 | -------------------------------------------------------------------------------- /systemd.service: -------------------------------------------------------------------------------- 1 | # Service definition to be installed somewhere in the systemd load path 2 | [Unit] 3 | Description=Travis Worker 4 | 5 | [Service] 6 | ExecStart=___SYSTEMD_WRAPPER___ 7 | ExecStopPost=/bin/sleep 5 8 | Restart=always 9 | WorkingDirectory=/ 10 | User=travis 11 | 12 | [Install] 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /version.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "runtime" 8 | 9 | "gopkg.in/urfave/cli.v1" 10 | ) 11 | 12 | var ( 13 | // VersionString is the git describe version set at build time 14 | VersionString = "?" 15 | // RevisionString is the git revision set at build time 16 | RevisionString = "?" 17 | // RevisionURLString is the full URL to the revision set at build time 18 | RevisionURLString = "?" 19 | // GeneratedString is the build date set at build time 20 | GeneratedString = "?" 21 | // CopyrightString is the copyright set at build time 22 | CopyrightString = "?" 23 | ) 24 | 25 | func init() { 26 | cli.VersionPrinter = customVersionPrinter 27 | _ = os.Setenv("VERSION", VersionString) 28 | _ = os.Setenv("REVISION", RevisionString) 29 | _ = os.Setenv("GENERATED", GeneratedString) 30 | } 31 | 32 | func customVersionPrinter(c *cli.Context) { 33 | fmt.Printf("%v v=%v rev=%v d=%v go=%v\n", filepath.Base(c.App.Name), 34 | VersionString, RevisionString, GeneratedString, runtime.Version()) 35 | } 36 | -------------------------------------------------------------------------------- /winrm/package.go: -------------------------------------------------------------------------------- 1 | package winrm 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io" 8 | "time" 9 | 10 | "github.com/masterzen/winrm" 11 | "github.com/packer-community/winrmcp/winrmcp" 12 | ) 13 | 14 | var errNotImplemented = fmt.Errorf("method not implemented") 15 | 16 | func New(host string, port int, username, password string) (*Remoter, error) { 17 | 18 | endpoint := &winrm.Endpoint{ 19 | Host: host, 20 | Port: port, 21 | HTTPS: true, 22 | Insecure: true, 23 | } 24 | 25 | params := *winrm.DefaultParameters 26 | params.Timeout = "PT2H" 27 | 28 | winrmClient, err := winrm.NewClientWithParameters( 29 | endpoint, username, password, ¶ms) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | shell, err := winrmClient.CreateShell() 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | if err := shell.Close(); err != nil { 40 | return nil, err 41 | } 42 | 43 | return &Remoter{ 44 | username: username, 45 | password: password, 46 | winrmClient: winrmClient, 47 | endpoint: endpoint, 48 | }, nil 49 | } 50 | 51 | type Remoter struct { 52 | username string 53 | password string 54 | winrmClient *winrm.Client 55 | endpoint *winrm.Endpoint 56 | } 57 | 58 | func (r *Remoter) UploadFile(path string, data []byte) (bool, error) { 59 | wcp, err := r.newCopyClient() 60 | if err != nil { 61 | return false, err 62 | } 63 | 64 | err = wcp.Write(path, bytes.NewReader(data)) 65 | return false, err 66 | } 67 | 68 | func (r *Remoter) DownloadFile(path string) ([]byte, error) { 69 | return nil, errNotImplemented 70 | } 71 | 72 | func (r *Remoter) RunCommand(command string, output io.Writer) (int32, error) { 73 | ctx, cancel := context.WithCancel(context.Background()) 74 | defer cancel() 75 | exitCode, err := r.winrmClient.RunWithContext(ctx, command, output, output) 76 | return int32(exitCode), err 77 | } 78 | 79 | func (r *Remoter) Close() error { 80 | return nil 81 | } 82 | 83 | func (r *Remoter) newCopyClient() (*winrmcp.Winrmcp, error) { 84 | addr := fmt.Sprintf("%s:%d", r.endpoint.Host, r.endpoint.Port) 85 | 86 | config := &winrmcp.Config{ 87 | Auth: winrmcp.Auth{ 88 | User: r.username, 89 | Password: r.password, 90 | }, 91 | Https: true, 92 | Insecure: true, 93 | OperationTimeout: 180 * time.Second, 94 | MaxOperationsPerShell: 30, 95 | TransportDecorator: nil, 96 | } 97 | 98 | return winrmcp.New(addr, config) 99 | } 100 | --------------------------------------------------------------------------------