├── .gitignore ├── .go-version ├── .travis.yml ├── Deps ├── LICENSE ├── Makefile ├── README.md ├── UPLOAD_USAGE.txt ├── USAGE.in.md ├── USAGE.md ├── USAGE.txt ├── artifact ├── artifact.go ├── artifact_test.go ├── options.go └── result.go ├── artifacts.go ├── artifacts_test.go ├── client ├── artifact_putter.go ├── client.go └── client_test.go ├── deploy ├── env ├── env.go └── env_test.go ├── install ├── logging ├── multi_line_formatter.go └── multi_line_formatter_test.go ├── markdownify-usage ├── path ├── path.go ├── path_test.go ├── set.go └── set_test.go ├── test-upload └── upload ├── artifacts_provider.go ├── artifacts_provider_test.go ├── null_provider.go ├── options.go ├── options_test.go ├── s3_provider.go ├── s3_provider_test.go ├── upload_provider.go ├── upload_test.go ├── uploader.go ├── uploader_test.go └── util.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | 25 | *.coverprofile 26 | *coverage.html 27 | 28 | /artifacts 29 | /artifacts.test 30 | 31 | /.env 32 | /.gox-bootstrap 33 | /.gox-install 34 | /.deps 35 | /build/ 36 | /SHA256SUMS 37 | -------------------------------------------------------------------------------- /.go-version: -------------------------------------------------------------------------------- 1 | 1.3.3 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | sudo: false 3 | go: 1.18.2 4 | env: 5 | global: 6 | - PATH=$HOME/gopath/bin:$HOME/bin:$PATH 7 | - ARTIFACTS_CONCURRENCY=5 8 | - ARTIFACTS_S3_BUCKET=travis-ci-gmbh 9 | - ARTIFACTS_TARGET_PATHS="artifacts/$TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER:artifacts/$TRAVIS_COMMIT" 10 | - ARTIFACTS_PERMISSIONS=public-read 11 | - ARTIFACTS_CACHE_CONTROL='public, max-age=315360000' 12 | - ARTIFACTS_LOG_FORMAT=multiline 13 | - ARTIFACTS_DEBUG=1 14 | before_install: 15 | - go get github.com/meatballhat/deppy 16 | - go get golang.org/x/tools/cmd/cover 17 | - deppy restore 18 | - go install -a -race std 19 | script: 20 | - make distclean all crossbuild 21 | - if [[ $TRAVIS_SECURE_ENV_VARS == true && $TRAVIS_PULL_REQUEST == false ]] ; then ./deploy ; fi 22 | - travis_retry ./install 23 | -------------------------------------------------------------------------------- /Deps: -------------------------------------------------------------------------------- 1 | { 2 | "ImportPath": "github.com/travis-ci/artifacts", 3 | "GoVersion": "go1.3.1", 4 | "Packages": [ 5 | "github.com/travis-ci/artifacts", 6 | "github.com/travis-ci/artifacts/artifact", 7 | "github.com/travis-ci/artifacts/client", 8 | "github.com/travis-ci/artifacts/env", 9 | "github.com/travis-ci/artifacts/logging", 10 | "github.com/travis-ci/artifacts/path", 11 | "github.com/travis-ci/artifacts/upload" 12 | ], 13 | "Deps": [ 14 | { 15 | "ImportPath": "github.com/Sirupsen/logrus", 16 | "Comment": "v0.5.1", 17 | "Rev": "736840a898fb560e3d93413410a4cb1371f42f71" 18 | }, 19 | { 20 | "ImportPath": "github.com/codegangsta/cli", 21 | "Comment": "1.2.0-22-g687db20", 22 | "Rev": "687db20fc379d1686465a28e9959707cd1acc990" 23 | }, 24 | { 25 | "ImportPath": "github.com/dustin/go-humanize", 26 | "Rev": "cb7b800be3f0238405be0e57d481740cfc4fb285" 27 | }, 28 | { 29 | "ImportPath": "github.com/mitchellh/goamz/aws", 30 | "Rev": "caaaea8b30ee15616494ee68abd5d8ebbbef05cf" 31 | }, 32 | { 33 | "ImportPath": "github.com/mitchellh/goamz/s3", 34 | "Rev": "caaaea8b30ee15616494ee68abd5d8ebbbef05cf" 35 | } 36 | ] 37 | } 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Dan Buch 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PACKAGE := github.com/travis-ci/artifacts 2 | SUBPACKAGES := \ 3 | $(PACKAGE)/artifact \ 4 | $(PACKAGE)/client \ 5 | $(PACKAGE)/env \ 6 | $(PACKAGE)/logging \ 7 | $(PACKAGE)/path \ 8 | $(PACKAGE)/upload 9 | 10 | COVERPROFILES := \ 11 | artifact-coverage.coverprofile \ 12 | env-coverage.coverprofile \ 13 | logging-coverage.coverprofile \ 14 | path-coverage.coverprofile \ 15 | upload-coverage.coverprofile 16 | 17 | VERSION_VAR := main.VersionString 18 | REPO_VERSION := $(shell git describe --always --dirty --tags) 19 | 20 | REV_VAR := main.RevisionString 21 | REPO_REV := $(shell git rev-parse -q HEAD) 22 | 23 | GO ?= go 24 | GOX ?= gox 25 | DEPPY ?= deppy 26 | GOBUILD_LDFLAGS := -ldflags "\ 27 | -X '$(VERSION_VAR)=$(REPO_VERSION)' \ 28 | -X '$(REV_VAR)=$(REPO_REV)' \ 29 | " 30 | GOBUILD_FLAGS ?= 31 | GOTEST_FLAGS ?= 32 | GOX_OSARCH ?= freebsd/amd64 linux/amd64 darwin/amd64 windows/amd64 linux/ppc64le linux/aarch64 33 | GOX_FLAGS ?= -output="build/{{.OS}}/{{.Arch}}/{{.Dir}}" -osarch="$(GOX_OSARCH)" 34 | 35 | TRAVIS_BUILD_DIR ?= . 36 | export TRAVIS_BUILD_DIR 37 | 38 | .PHONY: all 39 | all: clean test USAGE.txt UPLOAD_USAGE.txt USAGE.md 40 | 41 | .PHONY: test 42 | test: build fmtpolice test-deps .test 43 | 44 | .PHONY: quicktest 45 | quicktest: 46 | $(GO) test $(GOTEST_FLAGS) $(PACKAGE) $(SUBPACKAGES) 47 | 48 | .PHONY: .test 49 | .test: test-race coverage.html 50 | 51 | .PHONY: test-deps 52 | test-deps: 53 | $(GO) test -i $(GOBUILD_LDFLAGS) $(PACKAGE) $(SUBPACKAGES) 54 | 55 | .PHONY: test-race 56 | test-race: 57 | $(GO) test -race $(GOTEST_FLAGS) $(GOBUILD_LDFLAGS) $(PACKAGE) $(SUBPACKAGES) 58 | 59 | coverage.html: coverage.coverprofile 60 | $(GO) tool cover -html=$^ -o $@ 61 | 62 | coverage.coverprofile: $(COVERPROFILES) 63 | $(GO) test -v -covermode=count -coverprofile=$@.tmp $(GOBUILD_LDFLAGS) $(PACKAGE) 64 | echo 'mode: count' > $@ 65 | grep -h -v 'mode: count' $@.tmp >> $@ 66 | $(RM) $@.tmp 67 | grep -h -v 'mode: count' $^ >> $@ 68 | $(GO) tool cover -func=$@ 69 | 70 | path-coverage.coverprofile: 71 | $(GO) test -v -covermode=count -coverprofile=$@ $(GOBUILD_LDFLAGS) $(PACKAGE)/path 72 | 73 | upload-coverage.coverprofile: 74 | $(GO) test -v -covermode=count -coverprofile=$@ $(GOBUILD_LDFLAGS) $(PACKAGE)/upload 75 | 76 | env-coverage.coverprofile: 77 | $(GO) test -v -covermode=count -coverprofile=$@ $(GOBUILD_LDFLAGS) $(PACKAGE)/env 78 | 79 | logging-coverage.coverprofile: 80 | $(GO) test -v -covermode=count -coverprofile=$@ $(GOBUILD_LDFLAGS) $(PACKAGE)/logging 81 | 82 | artifact-coverage.coverprofile: 83 | $(GO) test -v -covermode=count -coverprofile=$@ $(GOBUILD_LDFLAGS) $(PACKAGE)/artifact 84 | 85 | USAGE.txt: build 86 | $${GOPATH%%:*}/bin/artifacts help | grep -v -E '^(VERSION|\s+v[0-9]\.[0-9]\.[0-9])' > $@ 87 | 88 | UPLOAD_USAGE.txt: build 89 | $${GOPATH%%:*}/bin/artifacts help upload > $@ 90 | 91 | USAGE.md: USAGE.txt UPLOAD_USAGE.txt $(shell git ls-files '*.go') 92 | ./markdownify-usage < USAGE.in.md > USAGE.md 93 | 94 | .PHONY: build 95 | build: deps .build 96 | 97 | .PHONY: .build 98 | .build: 99 | $(GO) install $(GOBUILD_FLAGS) $(GOBUILD_LDFLAGS) $(PACKAGE) 100 | 101 | .PHONY: crossbuild 102 | crossbuild: deps 103 | $(GOX) $(GOX_FLAGS) $(GOBUILD_FLAGS) $(GOBUILD_LDFLAGS) $(PACKAGE) 104 | 105 | .PHONY: deps 106 | deps: .gox-install .deps 107 | 108 | .deps: 109 | $(DEPPY) restore && touch $@ 110 | 111 | .gox-install: 112 | $(GO) get -x github.com/mitchellh/gox > $@ 113 | 114 | .PHONY: distclean 115 | distclean: clean 116 | $(RM) -v .gox-* .deps 117 | $(RM) -rv ./build 118 | 119 | .PHONY: clean 120 | clean: 121 | $(RM) -v $${GOPATH%%:*}/bin/artifacts 122 | $(RM) -v coverage.html $(COVERPROFILES) 123 | $(GO) clean $(PACKAGE) $(SUBPACKAGES) || true 124 | if [ -d $${GOPATH%%:*}/pkg ] ; then \ 125 | find $${GOPATH%%:*}/pkg -wholename '*travis-ci/artifacts*.a' | xargs $(RM) -fv || true; \ 126 | fi 127 | 128 | .PHONY: save 129 | save: 130 | $(DEPPY) save $(PACKAGE) $(SUBPACKAGES) 131 | 132 | .PHONY: fmtpolice 133 | fmtpolice: 134 | set -e; for f in $(shell git ls-files '*.go'); do gofmt $$f | diff -u $$f - ; done 135 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Travis CI Artifacts Uploader 2 | ============================ 3 | 4 | A smart little Go app to help aid in uploading build artifacts. 5 | 6 | ## Installation 7 | 8 | There are pre-built binaries of the latest stable build for 64-bit 9 | Linux, OSX, and Windows available here via the following links. Please 10 | note that the tests run on 64-bit Linux. 11 | 12 | * [Linux/amd64](https://s3.amazonaws.com/travis-ci-gmbh/artifacts/stable/build/linux/amd64/artifacts) 13 | * [Linux/ppc64le](https://s3.amazonaws.com/travis-ci-gmbh/artifacts/stable/build/linux/ppc64le/artifacts) 14 | * [OSX/amd64](https://s3.amazonaws.com/travis-ci-gmbh/artifacts/stable/build/darwin/amd64/artifacts) 15 | * [Windows/amd64](https://s3.amazonaws.com/travis-ci-gmbh/artifacts/stable/build/windows/amd64/artifacts.exe) 16 | * [FreeBSD/amd64](https://s3.amazonaws.com/travis-ci-gmbh/artifacts/stable/build/freebsd/amd64/artifacts) 17 | * [SHA-256 checksums](https://s3.amazonaws.com/travis-ci-gmbh/artifacts/stable/SHA256SUMS) 18 | 19 | There is also an [install script](./install) for Linux and OSX that may 20 | be used like so: 21 | 22 | ``` bash 23 | curl -sL https://raw.githubusercontent.com/travis-ci/artifacts/master/install | bash 24 | ``` 25 | 26 | ## Usage 27 | 28 | Once the binary is in your `$PATH` and has been made executable, you 29 | will have access to the help system via `artifacts help` (also available 30 | [here](./USAGE.md)). 31 | 32 | ### S3 ENVIRONMENT COMPATIBILITY 33 | 34 | In addition to the environment variables listed above for defining the 35 | access key, secret, and bucket, some additional variables will also work. 36 | 37 | #### environment variables accepted for "key" 38 | 39 | 0. `ARTIFACTS_KEY` 40 | 0. `ARTIFACTS_AWS_ACCESS_KEY` 41 | 0. `AWS_ACCESS_KEY_ID` 42 | 0. `AWS_ACCESS_KEY` 43 | 44 | #### environment variables accepted for "secret" 45 | 46 | 0. `ARTIFACTS_SECRET` 47 | 0. `ARTIFACTS_AWS_SECRET_KEY` 48 | 0. `AWS_SECRET_ACCESS_KEY` 49 | 0. `AWS_SECRET_KEY` 50 | 51 | #### environment variables accepted for "bucket" 52 | 53 | 0. `ARTIFACTS_BUCKET` 54 | 0. `ARTIFACTS_S3_BUCKET` 55 | 56 | #### environment variables accepted for "region" 57 | 58 | 0. `ARTIFACTS_REGION` 59 | 0. `ARTIFACTS_S3_REGION` 60 | 61 | 62 | ### EXAMPLES 63 | 64 | #### Example: logs and coverage 65 | 66 | In this case, the key and secret are passed as command line flags and 67 | the `log/` and `coverage/` directories are passed as positional path 68 | arguments: 69 | 70 | ``` bash 71 | artifacts upload \ 72 | --key AKIT339AFIY655O3Q9DZ \ 73 | --secret 48TmqyraUyJ7Efpegi6Lfd10yUskAMB0G2TtRCX1 \ 74 | --bucket my-fancy-bucket \ 75 | log/ coverage/ 76 | ``` 77 | 78 | The same operation using environment variables would look like this: 79 | 80 | ``` bash 81 | export ARTIFACTS_KEY="AKIT339AFIY655O3Q9DZ" 82 | export ARTIFACTS_SECRET="48TmqyraUyJ7Efpegi6Lfd10yUskAMB0G2TtRCX1" 83 | export ARTIFACTS_BUCKET="my-fancy-bucket" 84 | export ARTIFACTS_PATHS="log/:coverage/" 85 | 86 | artifacts upload 87 | ``` 88 | 89 | #### Example: untracked files 90 | 91 | In order to upload all of the untracked files (according to git), one 92 | might do this: 93 | 94 | ``` bash 95 | artifacts upload \ 96 | --key AKIT339AFIY655O3Q9DZ \ 97 | --secret 48TmqyraUyJ7Efpegi6Lfd10yUskAMB0G2TtRCX1 \ 98 | --bucket my-fancy-bucket \ 99 | $(git ls-files -o) 100 | ``` 101 | 102 | The same operation using environment variables would look like this: 103 | 104 | ``` bash 105 | export ARTIFACTS_KEY="AKIT339AFIY655O3Q9DZ" 106 | export ARTIFACTS_SECRET="48TmqyraUyJ7Efpegi6Lfd10yUskAMB0G2TtRCX1" 107 | export ARTIFACTS_BUCKET="my-fancy-bucket" 108 | export ARTIFACTS_PATHS="$(git ls-files -o | tr "\n" ":")" 109 | 110 | artifacts upload 111 | ``` 112 | 113 | #### Example: multiple target paths 114 | 115 | Specifying one or more custom target path will override the default of 116 | `artifacts/$TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER`. Multiple target paths 117 | must be specified in ':'-delimited strings: 118 | 119 | ``` bash 120 | artifacts upload \ 121 | --key AKIT339AFIY655O3Q9DZ \ 122 | --secret 48TmqyraUyJ7Efpegi6Lfd10yUskAMB0G2TtRCX1 \ 123 | --bucket my-fancy-bucket \ 124 | --target-paths "artifacts/$TRAVIS_REPO_SLUG/$TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER:artifacts/$TRAVIS_REPO_SLUG/$TRAVIS_COMMIT" \ 125 | $(git ls-files -o) 126 | ``` 127 | 128 | The same operation using environment variables would look like this: 129 | 130 | ``` bash 131 | export ARTIFACTS_TARGET_PATHS="artifacts/$TRAVIS_REPO_SLUG/$TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER:artifacts/$TRAVIS_REPO_SLUG/$TRAVIS_COMMIT" 132 | export ARTIFACTS_KEY="AKIT339AFIY655O3Q9DZ" 133 | export ARTIFACTS_SECRET="48TmqyraUyJ7Efpegi6Lfd10yUskAMB0G2TtRCX1" 134 | export ARTIFACTS_BUCKET="my-fancy-bucket" 135 | export ARTIFACTS_PATHS="$(git ls-files -o | tr "\n" ":")" 136 | 137 | artifacts upload 138 | ``` 139 | -------------------------------------------------------------------------------- /UPLOAD_USAGE.txt: -------------------------------------------------------------------------------- 1 | NAME: 2 | upload - upload some artifacts! 3 | 4 | USAGE: 5 | command upload [command options] [arguments...] 6 | 7 | DESCRIPTION: 8 | 9 | Upload a set of local paths to an artifact repository. The paths may be 10 | provided as either positional command-line arguments or as the $ARTIFACTS_PATHS 11 | environment variable, which should be :-delimited. 12 | 13 | Paths may be either files or directories. Any path provided will be walked for 14 | all child entries. Each entry will have its mime type detected based first on 15 | the file extension, then by sniffing up to the first 512 bytes via the net/http 16 | function "DetectContentType". 17 | 18 | 19 | OPTIONS: 20 | --key, -k upload credentials key *REQUIRED* (default "") [$ARTIFACTS_KEY] 21 | --bucket, -b destination bucket *REQUIRED* (default "") [$ARTIFACTS_BUCKET] 22 | --cache-control artifact cache-control header value (default "private") [$ARTIFACTS_CACHE_CONTROL] 23 | --permissions artifact access permissions (default "private") [$ARTIFACTS_PERMISSIONS] 24 | --secret, -s upload credentials secret *REQUIRED* (default "") [$ARTIFACTS_SECRET] 25 | --s3-region region used when storing to S3 (default "us-east-1") [$ARTIFACTS_REGION] 26 | --repo-slug, -r repo owner/name slug (default "") [$ARTIFACTS_REPO_SLUG] 27 | --build-number build number (default "") [$ARTIFACTS_BUILD_NUMBER] 28 | --build-id build id (default "") [$ARTIFACTS_BUILD_ID] 29 | --job-number job number (default "") [$ARTIFACTS_JOB_NUMBER] 30 | --job-id job id (default "") [$ARTIFACTS_JOB_ID] 31 | --concurrency upload worker concurrency (default "5") [$ARTIFACTS_CONCURRENCY] 32 | --max-size max combined size of uploaded artifacts (default "1048576000") [$ARTIFACTS_MAX_SIZE] 33 | --upload-provider, -p artifact upload provider (artifacts, s3, null) (default "s3") [$ARTIFACTS_UPLOAD_PROVIDER] 34 | --retries number of upload retries per artifact (default "2") [$ARTIFACTS_RETRIES] 35 | --target-paths, -t artifact target paths (':'-delimited) (default "[:]") [$ARTIFACTS_TARGET_PATHS] 36 | --working-dir working directory (default ".") [$ARTIFACTS_WORKING_DIR] 37 | --save-host, -H artifact save host (default "") [$ARTIFACTS_SAVE_HOST] 38 | --auth-token, -T artifact save auth token (default "") [$ARTIFACTS_AUTH_TOKEN] 39 | 40 | -------------------------------------------------------------------------------- /USAGE.in.md: -------------------------------------------------------------------------------- 1 | Travis CI Artifacts Uploader Usage 2 | ================================== 3 | 4 | ___GENERATED_WARNING___ 5 | 6 | ## global options 7 | 8 | ___USAGE___ 9 | 10 | ## upload 11 | 12 | The upload commmand may be used to upload arbitrary files to an artifact 13 | repository. The only such artifact repository currently supported is 14 | S3. All of the required arguments may be provided as command line 15 | arguments or environment variables. 16 | 17 | ___UPLOAD_USAGE___ 18 | 19 | 20 | -------------------------------------------------------------------------------- /USAGE.md: -------------------------------------------------------------------------------- 1 | Travis CI Artifacts Uploader Usage 2 | ================================== 3 | 4 | :boom: **Warning: file generated from [USAGE.in.md](./USAGE.in.md) ** :boom: 5 | 6 | 7 | ## global options 8 | 9 | 10 | ### NAME 11 | artifacts - manage your artifacts! 12 | 13 | ### USAGE 14 | `artifacts [global options] command [command options] [arguments...]` 15 | 16 | ### COMMANDS 17 | * `upload, u` upload some artifacts! 18 | * `help, h` Shows a list of commands or help for one command 19 | 20 | ### GLOBAL OPTIONS 21 | * `--log-format, -f` log output format (text, json, or multiline) [`$ARTIFACTS_LOG_FORMAT`] 22 | * `--debug, -D` set log level to debug [`$ARTIFACTS_DEBUG`] 23 | * `--quiet, -q` set log level to panic [`$ARTIFACTS_QUIET`] 24 | * `--help, -h` show help 25 | * `--version, -v` print the version 26 | 27 | ## upload 28 | 29 | The upload commmand may be used to upload arbitrary files to an artifact 30 | repository. The only such artifact repository currently supported is 31 | S3. All of the required arguments may be provided as command line 32 | arguments or environment variables. 33 | 34 | 35 | ### NAME 36 | upload - upload some artifacts! 37 | 38 | ### USAGE 39 | `command upload [command options] [arguments...]` 40 | 41 | ### DESCRIPTION 42 | Upload a set of local paths to an artifact repository. The paths may be 43 | provided as either positional command-line arguments or as the `$ARTIFACTS_PATHS` 44 | environment variable, which should be :-delimited. 45 | Paths may be either files or directories. Any path provided will be walked for 46 | all child entries. Each entry will have its mime type detected based first on 47 | the file extension, then by sniffing up to the first 512 bytes via the net/http 48 | function "DetectContentType". 49 | 50 | ### OPTIONS 51 | * `--key, -k` upload credentials key *REQUIRED* (default "") [`$ARTIFACTS_KEY`] 52 | * `--bucket, -b` destination bucket *REQUIRED* (default "") [`$ARTIFACTS_BUCKET`] 53 | * `--cache-control` artifact cache-control header value (default "private") [`$ARTIFACTS_CACHE_CONTROL`] 54 | * `--permissions` artifact access permissions (default "private") [`$ARTIFACTS_PERMISSIONS`] 55 | * `--secret, -s` upload credentials secret *REQUIRED* (default "") [`$ARTIFACTS_SECRET`] 56 | * `--s`3-region region used when storing to S3 (default "us-east-1") [`$ARTIFACTS_REGION`] 57 | * `--repo-slug, -r` repo owner/name slug (default "") [`$ARTIFACTS_REPO_SLUG`] 58 | * `--build-number` build number (default "") [`$ARTIFACTS_BUILD_NUMBER`] 59 | * `--build-id` build id (default "") [`$ARTIFACTS_BUILD_ID`] 60 | * `--job-number` job number (default "") [`$ARTIFACTS_JOB_NUMBER`] 61 | * `--job-id` job id (default "") [`$ARTIFACTS_JOB_ID`] 62 | * `--concurrency` upload worker concurrency (default "5") [`$ARTIFACTS_CONCURRENCY`] 63 | * `--max-size` max combined size of uploaded artifacts (default "1048576000") [`$ARTIFACTS_MAX_SIZE`] 64 | * `--upload-provider, -p` artifact upload provider (artifacts, s3, null) (default "s3") [`$ARTIFACTS_UPLOAD_PROVIDER`] 65 | * `--retries` number of upload retries per artifact (default "2") [`$ARTIFACTS_RETRIES`] 66 | * `--target-paths, -t` artifact target paths (':'-delimited) (default "[:]") [`$ARTIFACTS_TARGET_PATHS`] 67 | * `--working-dir` working directory (default ".") [`$ARTIFACTS_WORKING_DIR`] 68 | * `--save-host, -H` artifact save host (default "") [`$ARTIFACTS_SAVE_HOST`] 69 | * `--auth-token, -T` artifact save auth token (default "") [`$ARTIFACTS_AUTH_TOKEN`] 70 | 71 | 72 | -------------------------------------------------------------------------------- /USAGE.txt: -------------------------------------------------------------------------------- 1 | NAME: 2 | artifacts - manage your artifacts! 3 | 4 | USAGE: 5 | artifacts [global options] command [command options] [arguments...] 6 | 7 | 8 | COMMANDS: 9 | upload, u upload some artifacts! 10 | help, h Shows a list of commands or help for one command 11 | 12 | GLOBAL OPTIONS: 13 | --log-format, -f log output format (text, json, or multiline) [$ARTIFACTS_LOG_FORMAT] 14 | --debug, -D set log level to debug [$ARTIFACTS_DEBUG] 15 | --quiet, -q set log level to panic [$ARTIFACTS_QUIET] 16 | --help, -h show help 17 | --version, -v print the version 18 | 19 | -------------------------------------------------------------------------------- /artifact/artifact.go: -------------------------------------------------------------------------------- 1 | package artifact 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "mime" 7 | "net/http" 8 | "os" 9 | "path" 10 | "path/filepath" 11 | "strings" 12 | 13 | "github.com/mitchellh/goamz/s3" 14 | ) 15 | 16 | const ( 17 | defaultCtype = "application/octet-stream" 18 | ) 19 | 20 | // Artifact is the thing that gets uploaded or whatever 21 | type Artifact struct { 22 | RepoSlug string 23 | BuildNumber string 24 | BuildID string 25 | JobNumber string 26 | JobID string 27 | 28 | Source string 29 | Dest string 30 | Prefix string 31 | Perm s3.ACL 32 | 33 | UploadResult *Result 34 | } 35 | 36 | // New creates a new *Artifact 37 | func New(prefix, source, dest string, opts *Options) *Artifact { 38 | return &Artifact{ 39 | Prefix: prefix, 40 | Source: source, 41 | Dest: dest, 42 | 43 | RepoSlug: opts.RepoSlug, 44 | BuildNumber: opts.BuildNumber, 45 | BuildID: opts.BuildID, 46 | JobNumber: opts.JobNumber, 47 | JobID: opts.JobID, 48 | Perm: opts.Perm, 49 | 50 | UploadResult: &Result{}, 51 | } 52 | } 53 | 54 | // ContentType makes it easier to find the perfect match 55 | func (a *Artifact) ContentType() string { 56 | ctype := mime.TypeByExtension(path.Ext(a.Source)) 57 | if ctype != "" { 58 | return ctype 59 | } 60 | 61 | f, err := os.Open(a.Source) 62 | if err != nil { 63 | return defaultCtype 64 | } 65 | 66 | var buf bytes.Buffer 67 | 68 | _, err = io.CopyN(&buf, f, int64(512)) 69 | if err != nil && err != io.EOF { 70 | return defaultCtype 71 | } 72 | 73 | return http.DetectContentType(buf.Bytes()) 74 | } 75 | 76 | // Reader makes an io.Reader out of the filepath 77 | func (a *Artifact) Reader() (io.Reader, error) { 78 | f, err := os.Open(a.Source) 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | return f, nil 84 | } 85 | 86 | // Size reports the size of the artifact 87 | func (a *Artifact) Size() (uint64, error) { 88 | fi, err := os.Stat(a.Source) 89 | if err != nil { 90 | return uint64(0), nil 91 | } 92 | 93 | return uint64(fi.Size()), nil 94 | } 95 | 96 | // FullDest calculates the full remote destination path 97 | func (a *Artifact) FullDest() string { 98 | return strings.TrimLeft(filepath.Join(a.Prefix, a.Dest), "/") 99 | } 100 | -------------------------------------------------------------------------------- /artifact/artifact_test.go: -------------------------------------------------------------------------------- 1 | package artifact 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "path/filepath" 9 | "testing" 10 | 11 | "github.com/mitchellh/goamz/s3" 12 | ) 13 | 14 | type testPath struct { 15 | Path string 16 | ContentType string 17 | Valid bool 18 | } 19 | 20 | var ( 21 | testTmp, err = ioutil.TempDir("", "artifacts-test-upload") 22 | testArtifactPathDir = filepath.Join(testTmp, "artifact") 23 | testArtifactPaths = []*testPath{ 24 | &testPath{ 25 | Path: filepath.Join(testArtifactPathDir, "foo"), 26 | ContentType: "text/plain; charset=utf-8", 27 | Valid: true, 28 | }, 29 | &testPath{ 30 | Path: filepath.Join(testArtifactPathDir, "foo.csv"), 31 | ContentType: "text/csv; charset=utf-8", 32 | Valid: true, 33 | }, 34 | &testPath{ 35 | Path: filepath.Join(testArtifactPathDir, "nonexistent"), 36 | ContentType: defaultCtype, 37 | Valid: false, 38 | }, 39 | &testPath{ 40 | Path: filepath.Join(testArtifactPathDir, "unreadable"), 41 | ContentType: defaultCtype, 42 | Valid: false, 43 | }, 44 | } 45 | ) 46 | 47 | func init() { 48 | if err != nil { 49 | log.Panicf("game over: %v\n", err) 50 | } 51 | 52 | err = os.MkdirAll(testArtifactPathDir, 0755) 53 | if err != nil { 54 | log.Panicf("game over: %v\n", err) 55 | } 56 | 57 | for _, p := range testArtifactPaths { 58 | if filepath.Base(p.Path) == "nonexistent" { 59 | continue 60 | } 61 | 62 | fd, err := os.Create(p.Path) 63 | if err != nil { 64 | log.Panicf("game over: %v\n", err) 65 | } 66 | 67 | defer fd.Close() 68 | 69 | for i := 0; i < 512; i++ { 70 | fmt.Fprintf(fd, "something\n") 71 | } 72 | 73 | if filepath.Base(p.Path) == "unreadable" { 74 | fd.Chmod(0000) 75 | } 76 | } 77 | } 78 | 79 | func TestNewArtifact(t *testing.T) { 80 | a := New("bucket", "/foo/bar", "linux/foo", &Options{ 81 | Perm: s3.PublicRead, 82 | RepoSlug: "owner/foo", 83 | }) 84 | if a == nil { 85 | t.Fatalf("new artifact is nil") 86 | } 87 | 88 | if a.Prefix != "bucket" { 89 | t.Fatalf("prefix not set correctly: %v", a.Prefix) 90 | } 91 | 92 | if a.Dest != "linux/foo" { 93 | t.Fatalf("destination not set correctly: %v", a.Dest) 94 | } 95 | 96 | if a.Perm != s3.PublicRead { 97 | t.Fatalf("s3 perm not set correctly: %v", a.Perm) 98 | } 99 | 100 | if a.UploadResult == nil { 101 | t.Fatalf("result not initialized") 102 | } 103 | 104 | if a.UploadResult.OK { 105 | t.Fatalf("result initialized with OK as true") 106 | } 107 | 108 | if a.UploadResult.Err != nil { 109 | t.Fatalf("result initialized with non-nil Err") 110 | } 111 | } 112 | 113 | func TestArtifactContentType(t *testing.T) { 114 | for _, p := range testArtifactPaths { 115 | a := New("bucket", p.Path, "linux/foo", &Options{ 116 | Perm: s3.PublicRead, 117 | RepoSlug: "owner/foo", 118 | }) 119 | if a == nil { 120 | t.Fatalf("new artifact is nil") 121 | } 122 | 123 | actualCtype := a.ContentType() 124 | if p.ContentType != actualCtype { 125 | t.Fatalf("%v: %v != %v", p.Path, p.ContentType, actualCtype) 126 | } 127 | } 128 | } 129 | 130 | func TestArtifactReader(t *testing.T) { 131 | for _, p := range testArtifactPaths { 132 | if !p.Valid { 133 | continue 134 | } 135 | 136 | a := New("bucket", p.Path, "linux/foo", &Options{ 137 | Perm: s3.PublicRead, 138 | RepoSlug: "owner/foo", 139 | }) 140 | if a == nil { 141 | t.Fatalf("new artifact is nil") 142 | } 143 | 144 | reader, err := a.Reader() 145 | if err != nil { 146 | t.Fatalf("error getting reader: %v", err) 147 | } 148 | 149 | _, err = ioutil.ReadAll(reader) 150 | if err != nil { 151 | t.Error(err) 152 | } 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /artifact/options.go: -------------------------------------------------------------------------------- 1 | package artifact 2 | 3 | import ( 4 | "github.com/mitchellh/goamz/s3" 5 | ) 6 | 7 | // Options encapsulates stuff specific to artifacts. Ugh. 8 | type Options struct { 9 | RepoSlug string 10 | BuildNumber string 11 | BuildID string 12 | JobNumber string 13 | JobID string 14 | Perm s3.ACL 15 | } 16 | -------------------------------------------------------------------------------- /artifact/result.go: -------------------------------------------------------------------------------- 1 | package artifact 2 | 3 | // Result contains some lame simple crap about things done with artifacts 4 | type Result struct { 5 | OK bool 6 | Err error 7 | } 8 | -------------------------------------------------------------------------------- /artifacts.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/Sirupsen/logrus" 8 | "github.com/codegangsta/cli" 9 | "github.com/travis-ci/artifacts/logging" 10 | "github.com/travis-ci/artifacts/upload" 11 | ) 12 | 13 | var ( 14 | // VersionString contains the compiled-in version number 15 | VersionString = "?" 16 | // RevisionString contains the compiled-in git rev 17 | RevisionString = "?" 18 | ) 19 | 20 | func main() { 21 | app := buildApp() 22 | app.Run(os.Args) 23 | } 24 | 25 | func buildApp() *cli.App { 26 | app := cli.NewApp() 27 | app.Name = "artifacts" 28 | app.Usage = "manage your artifacts!" 29 | app.Version = fmt.Sprintf("%s revision=%s", VersionString, RevisionString) 30 | app.Flags = []cli.Flag{ 31 | cli.StringFlag{ 32 | Name: "log-format, f", 33 | EnvVar: "ARTIFACTS_LOG_FORMAT", 34 | Usage: "log output format (text, json, or multiline)", 35 | }, 36 | cli.BoolFlag{ 37 | Name: "debug, D", 38 | EnvVar: "ARTIFACTS_DEBUG", 39 | Usage: "set log level to debug", 40 | }, 41 | cli.BoolFlag{ 42 | Name: "quiet, q", 43 | EnvVar: "ARTIFACTS_QUIET", 44 | Usage: "set log level to panic", 45 | }, 46 | } 47 | app.Commands = []cli.Command{ 48 | { 49 | Name: "upload", 50 | ShortName: "u", 51 | Usage: "upload some artifacts!", 52 | Description: upload.CommandDescription, 53 | Flags: upload.DefaultOptions.Flags(), 54 | Action: runUpload, 55 | }, 56 | } 57 | 58 | return app 59 | } 60 | 61 | func runUpload(c *cli.Context) { 62 | log := configureLog(c) 63 | 64 | opts := upload.NewOptions() 65 | opts.UpdateFromCLI(c) 66 | 67 | if err := opts.Validate(); err != nil { 68 | log.Fatal(err) 69 | } 70 | 71 | if err := upload.Upload(opts, log); err != nil { 72 | log.Fatal(err) 73 | } 74 | } 75 | 76 | func configureLog(c *cli.Context) *logrus.Logger { 77 | log := logrus.New() 78 | 79 | switch c.GlobalString("log-format") { 80 | case "json": 81 | log.Formatter = &logrus.JSONFormatter{} 82 | case "multiline": 83 | log.Formatter = &logging.MultiLineFormatter{} 84 | default: 85 | log.Formatter = &logrus.TextFormatter{} 86 | } 87 | 88 | if c.GlobalBool("debug") { 89 | log.Level = logrus.DebugLevel 90 | log.Debug("setting log level to debug") 91 | } 92 | 93 | if c.GlobalBool("quiet") { 94 | log.Level = logrus.PanicLevel 95 | } 96 | 97 | return log 98 | } 99 | -------------------------------------------------------------------------------- /artifacts_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestBuildApp(t *testing.T) { 8 | app := buildApp() 9 | if app == nil { 10 | t.Errorf("app is nil") 11 | } 12 | 13 | if app.Name != "artifacts" { 14 | t.Errorf("unexpected app name: %v", app.Name) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /client/artifact_putter.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import "github.com/travis-ci/artifacts/artifact" 4 | 5 | // ArtifactPutter is the interface used to put artifacts 6 | type ArtifactPutter interface { 7 | PutArtifact(*artifact.Artifact) error 8 | } 9 | -------------------------------------------------------------------------------- /client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "path" 8 | "time" 9 | 10 | "github.com/Sirupsen/logrus" 11 | "github.com/travis-ci/artifacts/artifact" 12 | ) 13 | 14 | var ( 15 | errFailedPut = fmt.Errorf("failed to put artifact to artifacts service") 16 | 17 | defaultRetryInterval = 3 * time.Second 18 | ) 19 | 20 | // Client does stuff with the server 21 | type Client struct { 22 | SaveHost string 23 | Token string 24 | RetryInterval time.Duration 25 | 26 | log *logrus.Logger 27 | } 28 | 29 | // New creates a new *Client 30 | func New(host, token string, log *logrus.Logger) *Client { 31 | return &Client{ 32 | SaveHost: host, 33 | Token: token, 34 | RetryInterval: defaultRetryInterval, 35 | 36 | log: log, 37 | } 38 | } 39 | 40 | // PutArtifact puts ... an ... artifact 41 | func (c *Client) PutArtifact(a *artifact.Artifact) error { 42 | reader, err := a.Reader() 43 | if err != nil { 44 | return err 45 | } 46 | 47 | // e.g. hostname.example.org/owner/repo/jobs/123456/path/to/artifact 48 | fullURL := fmt.Sprintf("%s/%s", 49 | c.SaveHost, 50 | path.Join(a.RepoSlug, "jobs", a.JobID, a.Dest)) 51 | 52 | c.log.WithFields(logrus.Fields{ 53 | "url": fullURL, 54 | "source": a.Source, 55 | }).Debug("putting artifact to url") 56 | 57 | req, err := http.NewRequest("PUT", fullURL, reader) 58 | if err != nil { 59 | return err 60 | } 61 | 62 | size, err := a.Size() 63 | if err != nil { 64 | return err 65 | } 66 | 67 | req.Header.Set("Artifacts-Repo-Slug", a.RepoSlug) 68 | req.Header.Set("Artifacts-Source", a.Source) 69 | req.Header.Set("Artifacts-Dest", a.FullDest()) 70 | req.Header.Set("Artifacts-Job-Number", a.JobNumber) 71 | req.Header.Set("Artifacts-Size", fmt.Sprintf("%d", size)) 72 | 73 | client := &http.Client{} 74 | resp, err := client.Do(req) 75 | if err != nil { 76 | return err 77 | } 78 | 79 | if resp.StatusCode != 200 { 80 | return errFailedPut 81 | } 82 | 83 | defer resp.Body.Close() 84 | body, err := ioutil.ReadAll(resp.Body) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | c.log.WithFields(logrus.Fields{ 90 | "artifact": a, 91 | "response": string(body), 92 | }).Debug("successfully uploaded artifact") 93 | 94 | return nil 95 | } 96 | -------------------------------------------------------------------------------- /client/client_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/Sirupsen/logrus" 7 | ) 8 | 9 | func TestNew(t *testing.T) { 10 | log := logrus.New() 11 | log.Level = logrus.PanicLevel 12 | c := New("host.example.com", "foo-bar", log) 13 | 14 | if c.SaveHost != "host.example.com" { 15 | t.Fatalf("SaveHost %v != host.example.com", c.SaveHost) 16 | } 17 | 18 | if c.Token != "foo-bar" { 19 | t.Fatalf("Token %v != foo-bar", c.Token) 20 | } 21 | 22 | if c.RetryInterval != defaultRetryInterval { 23 | t.Fatalf("RetryInterval %v != %v", c.RetryInterval, defaultRetryInterval) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /deploy: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | SHA256SUMMER=sha256sum 6 | if ! sha256sum --version &>/dev/null ; then 7 | SHA256SUMMER=gsha256sum 8 | fi 9 | 10 | export ARTIFACTS_PATHS="$( 11 | git ls-files -o | grep -v '\.deps' | tr "\n" ":" 12 | ):SHA256SUMS" 13 | $SHA256SUMMER ${ARTIFACTS_PATHS//:/ } > SHA256SUMS 14 | 15 | echo " TRAVIS_BRANCH=\"$TRAVIS_BRANCH\"" 16 | echo " TRAVIS_GO_VERSION=\"$TRAVIS_GO_VERSION\"" 17 | echo " TRAVIS_PULL_REQUEST=\"$TRAVIS_PULL_REQUEST\"" 18 | echo " TRAVIS_TAG=\"$TRAVIS_TAG\"" 19 | 20 | if [[ "x$TRAVIS_BRANCH" == "xmaster" ]] && 21 | [[ "x$TRAVIS_GO_VERSION" == "x1.13.5" ]] && 22 | [[ "x$TRAVIS_PULL_REQUEST" == "xfalse" ]] && 23 | [[ "x$TRAVIS_TAG" == "x" ]] ; then 24 | echo " # Deploying as 'stable'" 25 | export ARTIFACTS_TARGET_PATHS="$ARTIFACTS_TARGET_PATHS:artifacts/stable" 26 | else 27 | echo " # Not deploying as 'stable'" 28 | fi 29 | 30 | echo " ARTIFACTS_TARGET_PATHS=$ARTIFACTS_TARGET_PATHS" 31 | exec ${GOPATH%%:*}/bin/artifacts upload 32 | -------------------------------------------------------------------------------- /env/env.go: -------------------------------------------------------------------------------- 1 | package env 2 | 3 | import ( 4 | "os" 5 | "strconv" 6 | "strings" 7 | ) 8 | 9 | // CascadeMatch is like Cascade, but also returns which env var 10 | // was used to retrieve the value 11 | func CascadeMatch(keys []string, dflt string) (string, string) { 12 | for _, key := range keys { 13 | value := strings.TrimSpace(os.Getenv(key)) 14 | if value == "" { 15 | continue 16 | } 17 | return value, key 18 | } 19 | 20 | return dflt, "" 21 | } 22 | 23 | // Slice returns a string slice from the env given a delimiter 24 | func Slice(key, delim string, dflt []string) []string { 25 | value := strings.TrimSpace(os.Getenv(key)) 26 | if value == "" { 27 | return dflt 28 | } 29 | 30 | ret := []string{} 31 | for _, part := range strings.Split(value, delim) { 32 | trimmed := strings.TrimSpace(part) 33 | if trimmed != "" { 34 | ret = append(ret, trimmed) 35 | } 36 | } 37 | 38 | return expandSlice(ret) 39 | } 40 | 41 | // Uint returns an uint from the env 42 | func Uint(key string, dflt uint64) uint64 { 43 | value := strings.TrimSpace(os.Getenv(key)) 44 | if value == "" { 45 | return dflt 46 | } 47 | 48 | uintVal, err := strconv.ParseUint(value, 10, 64) 49 | if err != nil { 50 | return dflt 51 | } 52 | 53 | return uintVal 54 | } 55 | 56 | func expandSlice(vars []string) []string { 57 | expanded := []string{} 58 | for _, s := range vars { 59 | expanded = append(expanded, os.ExpandEnv(s)) 60 | } 61 | return expanded 62 | } 63 | -------------------------------------------------------------------------------- /env/env_test.go: -------------------------------------------------------------------------------- 1 | package env 2 | 3 | import ( 4 | "os" 5 | "reflect" 6 | "testing" 7 | ) 8 | 9 | func init() { 10 | os.Setenv("FOO", "1") 11 | os.Setenv("BAR", "") 12 | os.Setenv("BAZ", "a:b:c::") 13 | os.Setenv("MOAR", "32GB") 14 | } 15 | 16 | type sliceCase struct { 17 | expected []string 18 | actual []string 19 | } 20 | 21 | func TestSlice(t *testing.T) { 22 | for _, c := range []sliceCase{ 23 | sliceCase{ 24 | expected: []string{"a", "b", "c"}, 25 | actual: Slice("BAZ", ":", []string{}), 26 | }, 27 | sliceCase{ 28 | expected: []string{"1"}, 29 | actual: Slice("FOO", ":", []string{}), 30 | }, 31 | sliceCase{ 32 | expected: []string{"z", "y", "x"}, 33 | actual: Slice("NOPE", ":", []string{"z", "y", "x"}), 34 | }, 35 | } { 36 | if !reflect.DeepEqual(c.expected, c.actual) { 37 | t.Fatalf("%v != %v", c.expected, c.actual) 38 | } 39 | } 40 | } 41 | 42 | func TestUint(t *testing.T) { 43 | for actual, expected := range map[uint64]uint64{ 44 | Uint("FOO", uint64(4)): uint64(1), 45 | Uint("BAR", uint64(3)): uint64(3), 46 | Uint("BAZ", uint64(5)): uint64(5), 47 | Uint("NOPE", uint64(3)): uint64(3), 48 | } { 49 | if expected != actual { 50 | t.Fatalf("%v != %v", expected, actual) 51 | } 52 | } 53 | } 54 | 55 | func TestExpandSlice(t *testing.T) { 56 | for _, c := range []sliceCase{ 57 | sliceCase{ 58 | expected: []string{"1,a:b:c::", "32GB", ""}, 59 | actual: expandSlice([]string{"$FOO,${BAZ}", "$MOAR", "${BAR}"}), 60 | }, 61 | sliceCase{ 62 | expected: []string{"", ""}, 63 | actual: expandSlice([]string{"$NOPE", "${NOPE}"}), 64 | }, 65 | } { 66 | if !reflect.DeepEqual(c.expected, c.actual) { 67 | t.Fatalf("%v != %v", c.expected, c.actual) 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /install: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | ARTIFACTS_DEST=${ARTIFACTS_DEST:-$HOME/bin/artifacts} 6 | OS=$(uname | tr '[:upper:]' '[:lower:]') 7 | ARCH=$(uname -m) 8 | if [[ $ARCH == x86_64 ]] ; then 9 | ARCH=amd64 10 | fi 11 | 12 | mkdir -p $(dirname "$ARTIFACTS_DEST") 13 | curl -sL -o "$ARTIFACTS_DEST" \ 14 | https://s3.amazonaws.com/travis-ci-gmbh/artifacts/stable/build/$OS/$ARCH/artifacts 15 | chmod +x "$ARTIFACTS_DEST" 16 | PATH="$(dirname "$ARTIFACTS_DEST"):$PATH" artifacts -v 17 | -------------------------------------------------------------------------------- /logging/multi_line_formatter.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strings" 7 | 8 | "github.com/Sirupsen/logrus" 9 | ) 10 | 11 | // MultiLineFormatter is a logrus-compatible formatter for multi-line output 12 | type MultiLineFormatter struct{} 13 | 14 | // Format creates a formatted entry 15 | func (f *MultiLineFormatter) Format(entry *logrus.Entry) ([]byte, error) { 16 | var serialized []byte 17 | 18 | levelText := strings.ToUpper(entry.Level.String()) 19 | 20 | msg := fmt.Sprintf("%s: %s\n", levelText, entry.Message) 21 | if levelText == "ERROR" { 22 | msg = fmt.Sprintf("\033[31;1m%s\033[0m", msg) 23 | } 24 | serialized = append(serialized, []byte(msg)...) 25 | 26 | keys := []string{} 27 | for k := range entry.Data { 28 | if k != "time" { 29 | keys = append(keys, k) 30 | } 31 | } 32 | 33 | sort.Strings(keys) 34 | 35 | for _, k := range keys { 36 | v := entry.Data[k] 37 | serialized = append(serialized, []byte(fmt.Sprintf(" %v: %v\n", k, v))...) 38 | } 39 | 40 | return append(serialized, '\n'), nil 41 | } 42 | -------------------------------------------------------------------------------- /logging/multi_line_formatter_test.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/Sirupsen/logrus" 7 | ) 8 | 9 | func TestFormat(t *testing.T) { 10 | formatter := &MultiLineFormatter{} 11 | log := logrus.New() 12 | entry := &logrus.Entry{ 13 | Logger: log, 14 | Level: logrus.InfoLevel, 15 | Message: "something", 16 | Data: logrus.Fields{"foo": "bar"}, 17 | } 18 | 19 | bytes, err := formatter.Format(entry) 20 | if err != nil { 21 | t.Error(err) 22 | } 23 | 24 | expected := "INFO: something\n foo: bar\n\n" 25 | actual := string(bytes) 26 | 27 | if expected != actual { 28 | t.Logf("%q != %q", expected, actual) 29 | t.Fail() 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /markdownify-usage: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # vim:fileencoding=utf-8 3 | 4 | require 'digest/sha2' 5 | 6 | def mangle(text) 7 | text.gsub(/\t/, ' ') 8 | .gsub(/^(NAME|USAGE|COMMANDS|GLOBAL OPTIONS|OPTIONS|DESCRIPTION):/, '### \1') 9 | .gsub(/^ ([a-z]+, [a-z]) /, '* `\1`') 10 | .gsub(/^\s+/, '') 11 | .gsub(/^(--[-a-z]+, -[a-zA-Z])/, '* `\1`') 12 | .gsub(/^(--[-a-z]+)/, '* `\1`') 13 | .gsub(/(\$[A-Z_]{2,})/, '`\1`') 14 | .gsub(/^### `([A-Z `]+)`$/, '### \1') 15 | .gsub(/\[([-a-z_ ])\]/, '[`\1`]') 16 | .gsub(/^(command upload.*|artifacts \[global.*)/, '`\1`') 17 | .gsub(/\s+$/, '') 18 | .gsub(/^### /, "\n### ") 19 | end 20 | 21 | generated_warning = ':boom: ' \ 22 | '**Warning: file generated from [USAGE.in.md](./USAGE.in.md) ** ' \ 23 | ":boom:\n" 24 | 25 | exe = "#{ENV['GOPATH'].split(':').first}/bin/artifacts" 26 | usage = `#{exe} help 2>&1`.chomp.split("\n").reject do |line| 27 | line =~ /^VERSION|\s+v\d\.\d\.\d/ 28 | end.join("\n") 29 | 30 | output = $stdin.read.sub(/___GENERATED_WARNING___/, generated_warning) 31 | .sub(/___USAGE___/, mangle(usage)) 32 | .sub(/___UPLOAD_USAGE___/, mangle(`#{exe} help upload 2>&1`.chomp)) 33 | output = output.gsub(/___CHECKSUM___/, Digest::SHA256.base64digest(output)) 34 | $stdout.puts(output) 35 | -------------------------------------------------------------------------------- /path/path.go: -------------------------------------------------------------------------------- 1 | package path 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "strings" 7 | ) 8 | 9 | // Path is path-like. Bonkers. 10 | type Path struct { 11 | Root string 12 | From string 13 | To string 14 | } 15 | 16 | // New makes a new *Path. Crazy! 17 | func New(root, from, to string) *Path { 18 | return &Path{ 19 | Root: root, 20 | From: from, 21 | To: to, 22 | } 23 | } 24 | 25 | // Fullpath returns the full file/dir path 26 | func (p *Path) Fullpath() string { 27 | if p.IsAbs() || strings.HasPrefix(p.From, "/") { 28 | return p.From 29 | } 30 | 31 | return filepath.Join(p.Root, p.From) 32 | } 33 | 34 | // IsDir tells if the path is a directory! 35 | func (p *Path) IsDir() bool { 36 | fi, err := os.Stat(p.From) 37 | if err != nil { 38 | return false 39 | } 40 | 41 | return fi.IsDir() 42 | } 43 | 44 | // IsExists tells if the path exists locally 45 | func (p *Path) IsExists() bool { 46 | _, err := os.Stat(p.From) 47 | return err == nil 48 | } 49 | 50 | // IsAbs tells if the path is absolute! 51 | func (p *Path) IsAbs() bool { 52 | asRelpath := filepath.Join(p.Root, p.From) 53 | _, err := os.Stat(asRelpath) 54 | if err == nil { 55 | return false 56 | } 57 | 58 | return p.IsExists() 59 | } 60 | -------------------------------------------------------------------------------- /path/path_test.go: -------------------------------------------------------------------------------- 1 | package path 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "path/filepath" 9 | "testing" 10 | ) 11 | 12 | var ( 13 | testTmp, err = ioutil.TempDir("", "artifacts-test-path") 14 | testSomethingPath = filepath.Join(testTmp, "something") 15 | testSomethingBoop = filepath.Join(testSomethingPath, "boop") 16 | 17 | fullPathTests = map[string][]string{ 18 | "/abc/ham": []string{"/abc", "ham", "bone"}, 19 | "/flim": []string{"/nope", "/flim", "flam"}, 20 | testSomethingBoop: []string{"/bogus", testSomethingBoop, "boop"}, 21 | } 22 | isAbsTests = map[string]bool{ 23 | "fiddle/faddle": false, 24 | testSomethingBoop: true, 25 | } 26 | isDirTests = map[string]bool{ 27 | testSomethingPath: true, 28 | "this/had/better/not/work": false, 29 | } 30 | ) 31 | 32 | func init() { 33 | if err != nil { 34 | log.Panicf("game over: %v\n", err) 35 | } 36 | 37 | err = os.MkdirAll(testSomethingPath, 0755) 38 | if err != nil { 39 | log.Panicf("game over: %v\n", err) 40 | } 41 | 42 | fd, err := os.Create(testSomethingBoop) 43 | if err != nil { 44 | log.Panicf("game over: %v\n", err) 45 | } 46 | 47 | defer fd.Close() 48 | fmt.Fprintf(fd, "something\n") 49 | } 50 | 51 | func TestNew(t *testing.T) { 52 | p := New("/xyz", "foo", "bar") 53 | 54 | if p.Root != "/xyz" { 55 | t.Fail() 56 | } 57 | 58 | if p.From != "foo" { 59 | t.Fail() 60 | } 61 | 62 | if p.To != "bar" { 63 | t.Fail() 64 | } 65 | } 66 | 67 | func TestPathIsAbs(t *testing.T) { 68 | for path, truth := range isAbsTests { 69 | p := New("/whatever", path, "somewhere") 70 | if p.IsAbs() != truth { 71 | t.Errorf("path %v IsAbs != %v\n", path, truth) 72 | } 73 | } 74 | } 75 | 76 | func TestPathFullpath(t *testing.T) { 77 | for expected, args := range fullPathTests { 78 | actual := New(args[0], args[1], args[2]).Fullpath() 79 | if expected != actual { 80 | t.Errorf("%v != %v", expected, actual) 81 | } 82 | } 83 | } 84 | 85 | func TestPathIsDir(t *testing.T) { 86 | for path, truth := range isDirTests { 87 | p := New("/whatever", path, "somewhere") 88 | if p.IsDir() != truth { 89 | t.Errorf("path %v IsDir != %v\n", path, truth) 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /path/set.go: -------------------------------------------------------------------------------- 1 | package path 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | ) 7 | 8 | // Set is a set of paths and their behaviors 9 | type Set struct { 10 | sync.Mutex 11 | paths map[string]*Path 12 | } 13 | 14 | // NewSet creates a new *Set 15 | func NewSet() *Set { 16 | return &Set{ 17 | paths: map[string]*Path{}, 18 | } 19 | } 20 | 21 | // Add adds a path to the set 22 | func (ps *Set) Add(p *Path) { 23 | ps.Lock() 24 | defer ps.Unlock() 25 | ps.paths[fmt.Sprintf("%#v", p)] = p 26 | } 27 | 28 | // All returns each path in the pathset 29 | func (ps *Set) All() []*Path { 30 | ps.Lock() 31 | defer ps.Unlock() 32 | 33 | all := []*Path{} 34 | for _, p := range ps.paths { 35 | all = append(all, p) 36 | } 37 | 38 | return all 39 | } 40 | -------------------------------------------------------------------------------- /path/set_test.go: -------------------------------------------------------------------------------- 1 | package path 2 | 3 | import "testing" 4 | 5 | func TestNewSet(t *testing.T) { 6 | ps := NewSet() 7 | if len(ps.All()) > 0 { 8 | t.Fatalf("new Set has non-empty paths") 9 | } 10 | } 11 | 12 | func TestSetDeDuping(t *testing.T) { 13 | ps := NewSet() 14 | 15 | ps.Add(New("/", "foo", "bar")) 16 | ps.Add(New("/", "foo", "bar")) 17 | 18 | if len(ps.All()) > 1 { 19 | t.Fatalf("duplicate path was not de-duped") 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /test-upload: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | TMPPATH=$(mktemp "${TMPDIR:-/tmp}/artifacts-test.XXX") 6 | trap "rm -f $TMPPATH" EXIT KILL TERM 7 | 8 | echo "hey there" > $TMPPATH 9 | 10 | export ARTIFACTS_CONCURRENCY=3 11 | export ARTIFACTS_RETRIES=1 12 | export ARTIFACTS_PATHS="$(git ls-files -o | grep -v '.env' | tr "\n" ":"):$TMPPATH" 13 | export PATH="${GOPATH%%:*}/bin:$PATH" 14 | 15 | exec artifacts ${CMD:-upload} "$@" 16 | -------------------------------------------------------------------------------- /upload/artifacts_provider.go: -------------------------------------------------------------------------------- 1 | package upload 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/Sirupsen/logrus" 7 | "github.com/dustin/go-humanize" 8 | "github.com/travis-ci/artifacts/artifact" 9 | "github.com/travis-ci/artifacts/client" 10 | ) 11 | 12 | var ( 13 | defaultProviderRetryInterval = 3 * time.Second 14 | ) 15 | 16 | type artifactsProvider struct { 17 | RetryInterval time.Duration 18 | 19 | opts *Options 20 | log *logrus.Logger 21 | 22 | overrideClient client.ArtifactPutter 23 | } 24 | 25 | func newArtifactsProvider(opts *Options, log *logrus.Logger) *artifactsProvider { 26 | return &artifactsProvider{ 27 | RetryInterval: defaultProviderRetryInterval, 28 | 29 | opts: opts, 30 | log: log, 31 | } 32 | } 33 | 34 | func (ap *artifactsProvider) Upload(id string, opts *Options, 35 | in chan *artifact.Artifact, out chan *artifact.Artifact, done chan bool) { 36 | 37 | cl := ap.getClient() 38 | 39 | for a := range in { 40 | err := ap.uploadFile(cl, a) 41 | if err != nil { 42 | a.UploadResult.OK = false 43 | a.UploadResult.Err = err 44 | } else { 45 | a.UploadResult.OK = true 46 | } 47 | out <- a 48 | } 49 | 50 | done <- true 51 | return 52 | } 53 | 54 | func (ap *artifactsProvider) uploadFile(cl client.ArtifactPutter, a *artifact.Artifact) error { 55 | retries := uint64(0) 56 | 57 | for { 58 | err := ap.rawUpload(cl, a) 59 | if err == nil { 60 | return nil 61 | } 62 | if retries < ap.opts.Retries { 63 | retries++ 64 | ap.log.WithFields(logrus.Fields{ 65 | "artifact": a.Source, 66 | "retry": retries, 67 | "err": err, 68 | }).Debug("retrying") 69 | time.Sleep(ap.RetryInterval) 70 | continue 71 | } else { 72 | return err 73 | } 74 | } 75 | return nil 76 | } 77 | 78 | func (ap *artifactsProvider) rawUpload(cl client.ArtifactPutter, a *artifact.Artifact) error { 79 | ctype := a.ContentType() 80 | size, err := a.Size() 81 | if err != nil { 82 | return err 83 | } 84 | 85 | ap.log.WithFields(logrus.Fields{ 86 | "percent_max_size": pctMax(size, ap.opts.MaxSize), 87 | "max_size": humanize.Bytes(ap.opts.MaxSize), 88 | "source": a.Source, 89 | "dest": a.FullDest(), 90 | "content_type": ctype, 91 | "cache_control": ap.opts.CacheControl, 92 | }).Debug("more artifact details") 93 | 94 | return cl.PutArtifact(a) 95 | } 96 | 97 | func (ap *artifactsProvider) getClient() client.ArtifactPutter { 98 | if ap.overrideClient != nil { 99 | ap.log.WithField("client", ap.overrideClient).Debug("using override client") 100 | return ap.overrideClient 101 | } 102 | 103 | ap.log.Debug("creating new client") 104 | return client.New(ap.opts.ArtifactsSaveHost, ap.opts.ArtifactsAuthToken, ap.log) 105 | } 106 | 107 | func (ap *artifactsProvider) Name() string { 108 | return "artifacts" 109 | } 110 | -------------------------------------------------------------------------------- /upload/artifacts_provider_test.go: -------------------------------------------------------------------------------- 1 | package upload 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | 8 | "github.com/mitchellh/goamz/s3" 9 | "github.com/travis-ci/artifacts/artifact" 10 | ) 11 | 12 | type nullPutter struct { 13 | Putted []*artifact.Artifact 14 | } 15 | 16 | func (np *nullPutter) PutArtifact(a *artifact.Artifact) error { 17 | if np.Putted == nil { 18 | np.Putted = []*artifact.Artifact{} 19 | } 20 | np.Putted = append(np.Putted, a) 21 | return nil 22 | } 23 | 24 | func TestArtifactsProviderDefaults(t *testing.T) { 25 | opts := NewOptions() 26 | log := getPanicLogger() 27 | ap := newArtifactsProvider(opts, log) 28 | 29 | if ap.RetryInterval != defaultProviderRetryInterval { 30 | t.Fatalf("RetryInterval %v != %v", ap.RetryInterval, defaultProviderRetryInterval) 31 | } 32 | 33 | if ap.opts != opts { 34 | t.Fatalf("opts %v != %v", ap.opts, opts) 35 | } 36 | 37 | if ap.log != log { 38 | t.Fatalf("log %v != %v", ap.log, log) 39 | } 40 | 41 | if ap.Name() != "artifacts" { 42 | t.Fatalf("Name %v != artifacts", ap.Name()) 43 | } 44 | } 45 | 46 | func TestArtifactsUpload(t *testing.T) { 47 | opts := NewOptions() 48 | log := getPanicLogger() 49 | ap := newArtifactsProvider(opts, log) 50 | ap.overrideClient = &nullPutter{} 51 | 52 | in := make(chan *artifact.Artifact) 53 | out := make(chan *artifact.Artifact) 54 | done := make(chan bool) 55 | 56 | go ap.Upload("test-0", opts, in, out, done) 57 | 58 | go func() { 59 | for _, p := range testArtifactPaths { 60 | if !p.Valid { 61 | continue 62 | } 63 | 64 | a := artifact.New("bucket", p.Path, "linux/foo", &artifact.Options{ 65 | Perm: s3.PublicRead, 66 | RepoSlug: "owner/foo", 67 | }) 68 | 69 | in <- a 70 | fmt.Printf("---> Fed artifact: %#v\n", a) 71 | } 72 | close(in) 73 | }() 74 | 75 | accum := []*artifact.Artifact{} 76 | for { 77 | select { 78 | case <-time.After(5 * time.Second): 79 | t.Fatalf("took too long oh derp") 80 | case a := <-out: 81 | accum = append(accum, a) 82 | case <-done: 83 | if len(accum) == 0 { 84 | t.Fatalf("nothing uploaded") 85 | } 86 | return 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /upload/null_provider.go: -------------------------------------------------------------------------------- 1 | package upload 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | 7 | "github.com/Sirupsen/logrus" 8 | "github.com/travis-ci/artifacts/artifact" 9 | ) 10 | 11 | var ( 12 | errUploadFailed = fmt.Errorf("upload failed") 13 | ) 14 | 15 | type nullProvider struct { 16 | SourcesToFail []string 17 | 18 | Log *logrus.Logger 19 | } 20 | 21 | func newNullProvider(sourcesToFail []string, log *logrus.Logger) *nullProvider { 22 | if sourcesToFail == nil { 23 | sourcesToFail = []string{} 24 | } 25 | if log == nil { 26 | log = logrus.New() 27 | } 28 | return &nullProvider{ 29 | SourcesToFail: sourcesToFail, 30 | 31 | Log: log, 32 | } 33 | } 34 | 35 | func (np *nullProvider) Upload(id string, opts *Options, 36 | in chan *artifact.Artifact, out chan *artifact.Artifact, done chan bool) { 37 | 38 | sort.Strings(np.SourcesToFail) 39 | lenSrc := len(np.SourcesToFail) 40 | 41 | for a := range in { 42 | idx := sort.SearchStrings(np.SourcesToFail, a.Source) 43 | if idx < 0 || idx >= lenSrc { 44 | a.UploadResult.OK = false 45 | a.UploadResult.Err = errUploadFailed 46 | } else { 47 | a.UploadResult.OK = true 48 | } 49 | np.Log.WithField("artifact", a).Debug("not really uploading") 50 | out <- a 51 | } 52 | 53 | done <- true 54 | } 55 | 56 | func (np *nullProvider) Name() string { 57 | return "null" 58 | } 59 | -------------------------------------------------------------------------------- /upload/options.go: -------------------------------------------------------------------------------- 1 | package upload 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "reflect" 7 | "strconv" 8 | "strings" 9 | 10 | "github.com/codegangsta/cli" 11 | "github.com/dustin/go-humanize" 12 | "github.com/travis-ci/artifacts/env" 13 | ) 14 | 15 | const ( 16 | sizeChars = "BKMGTPEZYbkmgtpezy" 17 | 18 | // CommandDescription is the string used to describe the 19 | // "upload" command in the command line help system 20 | CommandDescription = ` 21 | Upload a set of local paths to an artifact repository. The paths may be 22 | provided as either positional command-line arguments or as the $ARTIFACTS_PATHS 23 | environment variable, which should be :-delimited. 24 | 25 | Paths may be either files or directories. Any path provided will be walked for 26 | all child entries. Each entry will have its mime type detected based first on 27 | the file extension, then by sniffing up to the first 512 bytes via the net/http 28 | function "DetectContentType". 29 | ` 30 | ) 31 | 32 | var ( 33 | DefaultOptions = NewOptions() 34 | 35 | optsMaps = map[string]map[string]string{ 36 | "cli": map[string]string{ 37 | "AccessKey": "key, k", 38 | "BucketName": "bucket, b", 39 | "CacheControl": "cache-control", 40 | "Perm": "permissions", 41 | "SecretKey": "secret, s", 42 | "S3Region": "s3-region", 43 | 44 | "RepoSlug": "repo-slug, r", 45 | "BuildNumber": "build-number", 46 | "BuildID": "build-id", 47 | "JobNumber": "job-number", 48 | "JobID": "job-id", 49 | 50 | "Concurrency": "concurrency", 51 | "MaxSize": "max-size", 52 | "Paths": "", 53 | "Provider": "upload-provider, p", 54 | "Retries": "retries", 55 | "TargetPaths": "target-paths, t", 56 | "WorkingDir": "working-dir", 57 | 58 | "ArtifactsSaveHost": "save-host, H", 59 | "ArtifactsAuthToken": "auth-token, T", 60 | }, 61 | "doc": map[string]string{ 62 | "AccessKey": "upload credentials key *REQUIRED*", 63 | "BucketName": "destination bucket *REQUIRED*", 64 | "CacheControl": "artifact cache-control header value", 65 | "Perm": "artifact access permissions", 66 | "SecretKey": "upload credentials secret *REQUIRED*", 67 | "S3Region": "region used when storing to S3", 68 | 69 | "RepoSlug": "repo owner/name slug", 70 | "BuildNumber": "build number", 71 | "BuildID": "build id", 72 | "JobNumber": "job number", 73 | "JobID": "job id", 74 | 75 | "Concurrency": "upload worker concurrency", 76 | "MaxSize": "max combined size of uploaded artifacts", 77 | "Paths": "", 78 | "Provider": "artifact upload provider (artifacts, s3, null)", 79 | "Retries": "number of upload retries per artifact", 80 | "TargetPaths": "artifact target paths (':'-delimited)", 81 | "WorkingDir": "working directory", 82 | 83 | "ArtifactsSaveHost": "artifact save host", 84 | "ArtifactsAuthToken": "artifact save auth token", 85 | }, 86 | "env": map[string]string{ 87 | "AccessKey": "ARTIFACTS_KEY,ARTIFACTS_AWS_ACCESS_KEY,AWS_ACCESS_KEY_ID,AWS_ACCESS_KEY", 88 | "BucketName": "ARTIFACTS_BUCKET,ARTIFACTS_S3_BUCKET", 89 | "CacheControl": "ARTIFACTS_CACHE_CONTROL", 90 | "Perm": "ARTIFACTS_PERMISSIONS", 91 | "SecretKey": "ARTIFACTS_SECRET,ARTIFACTS_AWS_SECRET_KEY,AWS_SECRET_ACCESS_KEY,AWS_SECRET_KEY", 92 | "S3Region": "ARTIFACTS_REGION,ARTIFACTS_S3_REGION", 93 | 94 | "RepoSlug": "ARTIFACTS_REPO_SLUG,TRAVIS_REPO_SLUG", 95 | "BuildNumber": "ARTIFACTS_BUILD_NUMBER,TRAVIS_BUILD_NUMBER", 96 | "BuildID": "ARTIFACTS_BUILD_ID,TRAVIS_BUILD_ID", 97 | "JobNumber": "ARTIFACTS_JOB_NUMBER,TRAVIS_JOB_NUMBER", 98 | "JobID": "ARTIFACTS_JOB_ID,TRAVIS_JOB_ID", 99 | 100 | "Concurrency": "ARTIFACTS_CONCURRENCY", 101 | "MaxSize": "ARTIFACTS_MAX_SIZE", 102 | "Paths": "ARTIFACTS_PATHS", 103 | "Provider": "ARTIFACTS_UPLOAD_PROVIDER", 104 | "Retries": "ARTIFACTS_RETRIES", 105 | "TargetPaths": "ARTIFACTS_TARGET_PATHS", 106 | "WorkingDir": "ARTIFACTS_WORKING_DIR,TRAVIS_BUILD_DIR,PWD", 107 | 108 | "ArtifactsSaveHost": "ARTIFACTS_SAVE_HOST", 109 | "ArtifactsAuthToken": "ARTIFACTS_AUTH_TOKEN", 110 | }, 111 | "default": map[string]string{ 112 | "AccessKey": "", 113 | "BucketName": "", 114 | "CacheControl": "private", 115 | "Perm": "private", 116 | "SecretKey": "", 117 | "S3Region": "us-east-1", 118 | 119 | "RepoSlug": "", 120 | "BuildNumber": "", 121 | "BuildID": "", 122 | "JobNumber": "", 123 | "JobID": "", 124 | 125 | "Concurrency": "5", 126 | "MaxSize": fmt.Sprintf("%d", 1024*1024*1000), 127 | "Paths": "", 128 | "Provider": "s3", 129 | "Retries": "2", 130 | "TargetPaths": "artifacts/$TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER", 131 | "WorkingDir": ".", 132 | 133 | "ArtifactsSaveHost": "", 134 | "ArtifactsAuthToken": "", 135 | }, 136 | } 137 | ) 138 | 139 | // Options is used in the call to Upload 140 | type Options struct { 141 | AccessKey string 142 | BucketName string 143 | CacheControl string 144 | Perm string 145 | SecretKey string 146 | S3Region string 147 | 148 | RepoSlug string 149 | BuildNumber string 150 | BuildID string 151 | JobNumber string 152 | JobID string 153 | 154 | Concurrency uint64 155 | MaxSize uint64 156 | Paths []string 157 | Provider string 158 | Retries uint64 159 | TargetPaths []string 160 | WorkingDir string 161 | 162 | ArtifactsSaveHost string 163 | ArtifactsAuthToken string 164 | } 165 | 166 | // NewOptions makes some *Options with defaults! 167 | func NewOptions() *Options { 168 | opts := &Options{} 169 | opts.reset() 170 | return opts 171 | } 172 | 173 | func (opts *Options) Flags() []cli.Flag { 174 | dflt := DefaultOptions 175 | flags := []cli.Flag{} 176 | 177 | s := reflect.ValueOf(dflt).Elem() 178 | t := s.Type() 179 | 180 | for i := 0; i < s.NumField(); i++ { 181 | f := s.Field(i) 182 | if !f.CanSet() { 183 | continue 184 | } 185 | 186 | tf := t.Field(i) 187 | name := optsMaps["cli"][tf.Name] 188 | if name == "" { 189 | continue 190 | } 191 | 192 | flags = append(flags, cli.StringFlag{ 193 | Name: name, 194 | EnvVar: strings.Split(optsMaps["env"][tf.Name], ",")[0], 195 | Usage: fmt.Sprintf("%v (default %q)", 196 | optsMaps["doc"][tf.Name], 197 | fmt.Sprintf("%v", f.Interface())), 198 | }) 199 | } 200 | 201 | return flags 202 | } 203 | 204 | func (opts *Options) reset() { 205 | s := reflect.ValueOf(opts).Elem() 206 | t := s.Type() 207 | 208 | for i := 0; i < s.NumField(); i++ { 209 | f := s.Field(i) 210 | if !f.CanSet() { 211 | continue 212 | } 213 | 214 | tf := t.Field(i) 215 | dflt := os.ExpandEnv(optsMaps["default"][tf.Name]) 216 | envKeys := strings.Split(optsMaps["env"][tf.Name], ",") 217 | value, envVar := env.CascadeMatch(envKeys, dflt) 218 | 219 | if value == "" { 220 | continue 221 | } 222 | 223 | if envVar == "" { 224 | envVar = envKeys[0] 225 | } 226 | 227 | value = os.ExpandEnv(value) 228 | 229 | k := f.Kind() 230 | switch k { 231 | case reflect.String: 232 | f.SetString(value) 233 | case reflect.Uint64: 234 | uintVal, err := strconv.ParseUint(dflt, 10, 64) 235 | if err != nil { 236 | fmt.Fprintf(os.Stderr, "ERROR: %v", err) 237 | } else { 238 | f.SetUint(env.Uint(envVar, uintVal)) 239 | } 240 | case reflect.Slice: 241 | sliceValue := env.Slice(envVar, ":", strings.Split(":", dflt)) 242 | f.Set(reflect.ValueOf(sliceValue)) 243 | default: 244 | panic(fmt.Sprintf("unknown kind wat: %v", k)) 245 | } 246 | } 247 | } 248 | 249 | // UpdateFromCLI overlays a *cli.Context onto internal options 250 | func (opts *Options) UpdateFromCLI(c *cli.Context) { 251 | s := reflect.ValueOf(opts).Elem() 252 | t := s.Type() 253 | 254 | for i := 0; i < s.NumField(); i++ { 255 | f := s.Field(i) 256 | if !f.CanSet() { 257 | continue 258 | } 259 | 260 | tf := t.Field(i) 261 | 262 | names := optsMaps["cli"][tf.Name] 263 | nameParts := strings.Split(names, ",") 264 | if len(nameParts) < 1 { 265 | continue 266 | } 267 | 268 | name := nameParts[0] 269 | value := c.String(name) 270 | if value == "" { 271 | continue 272 | } 273 | 274 | switch name { 275 | case "concurrency", "retries": 276 | intVal, err := strconv.ParseUint(value, 10, 64) 277 | if err == nil { 278 | f.SetUint(intVal) 279 | } 280 | case "max-size": 281 | if strings.ContainsAny(value, sizeChars) { 282 | b, err := humanize.ParseBytes(value) 283 | if err == nil { 284 | opts.MaxSize = b 285 | } 286 | } else { 287 | intVal, err := strconv.ParseUint(value, 10, 64) 288 | if err == nil { 289 | opts.MaxSize = intVal 290 | } 291 | } 292 | case "target-paths": 293 | tp := []string{} 294 | for _, part := range strings.Split(value, ":") { 295 | trimmed := strings.TrimSpace(part) 296 | if trimmed != "" { 297 | tp = append(tp, trimmed) 298 | } 299 | } 300 | opts.TargetPaths = tp 301 | default: 302 | if f.Kind() == reflect.String { 303 | f.SetString(value) 304 | } 305 | } 306 | } 307 | 308 | for _, arg := range c.Args() { 309 | opts.Paths = append(opts.Paths, arg) 310 | } 311 | } 312 | 313 | // Validate checks for validity! 314 | func (opts *Options) Validate() error { 315 | if opts.Provider == "s3" { 316 | return opts.validateS3() 317 | } 318 | 319 | return nil 320 | } 321 | 322 | func (opts *Options) validateS3() error { 323 | if opts.BucketName == "" { 324 | return fmt.Errorf("no bucket name given") 325 | } 326 | 327 | if opts.AccessKey == "" { 328 | return fmt.Errorf("no access key given") 329 | } 330 | 331 | if opts.SecretKey == "" { 332 | return fmt.Errorf("no secret key given") 333 | } 334 | 335 | return nil 336 | } 337 | -------------------------------------------------------------------------------- /upload/options_test.go: -------------------------------------------------------------------------------- 1 | package upload 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func TestOptionsValidate(t *testing.T) { 9 | os.Clearenv() 10 | opts := NewOptions() 11 | opts.Provider = "null" 12 | 13 | if opts.Validate() != nil { 14 | t.Fatalf("default options were invalid") 15 | } 16 | } 17 | 18 | func TestOptionsValidateS3(t *testing.T) { 19 | os.Clearenv() 20 | opts := NewOptions() 21 | opts.Provider = "s3" 22 | 23 | err := opts.Validate() 24 | if err == nil { 25 | t.Fatalf("default options were valid for s3") 26 | } 27 | 28 | if err.Error() != "no bucket name given" { 29 | t.Fatalf("default options did not fail on missing bucket name") 30 | } 31 | 32 | opts.BucketName = "foo" 33 | err = opts.Validate() 34 | if err == nil { 35 | t.Fatalf("options with only bucket name were valid for s3") 36 | } 37 | 38 | if err.Error() != "no access key given" { 39 | t.Fatalf("options did not fail on missing access key") 40 | } 41 | 42 | opts.AccessKey = "AZ123" 43 | err = opts.Validate() 44 | if err == nil { 45 | t.Fatalf("options with only bucket name were valid for s3") 46 | } 47 | 48 | if err.Error() != "no secret key given" { 49 | t.Fatalf("options did not fail on missing secret key") 50 | } 51 | 52 | opts.SecretKey = "ZYX321" 53 | if opts.Validate() != nil { 54 | t.Fatalf("valid s3 options were deemed invalid") 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /upload/s3_provider.go: -------------------------------------------------------------------------------- 1 | package upload 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/Sirupsen/logrus" 8 | "github.com/dustin/go-humanize" 9 | "github.com/mitchellh/goamz/aws" 10 | "github.com/mitchellh/goamz/s3" 11 | "github.com/travis-ci/artifacts/artifact" 12 | ) 13 | 14 | var ( 15 | nilAuth aws.Auth 16 | ) 17 | 18 | type s3Provider struct { 19 | RetryInterval time.Duration 20 | 21 | opts *Options 22 | log *logrus.Logger 23 | 24 | overrideConn *s3.S3 25 | overrideAuth aws.Auth 26 | } 27 | 28 | func newS3Provider(opts *Options, log *logrus.Logger) *s3Provider { 29 | return &s3Provider{ 30 | RetryInterval: defaultProviderRetryInterval, 31 | 32 | opts: opts, 33 | log: log, 34 | 35 | overrideAuth: nilAuth, 36 | } 37 | } 38 | 39 | func (s3p *s3Provider) Upload(id string, opts *Options, in chan *artifact.Artifact, out chan *artifact.Artifact, done chan bool) { 40 | auth, err := s3p.getAuth(opts.AccessKey, opts.SecretKey) 41 | 42 | if err != nil { 43 | s3p.log.WithFields(logrus.Fields{ 44 | "uploader": id, 45 | "err": err, 46 | }).Error("uploader failed to get aws auth") 47 | done <- true 48 | return 49 | } 50 | 51 | conn := s3p.getConn(auth) 52 | bucket := conn.Bucket(opts.BucketName) 53 | 54 | if bucket == nil { 55 | s3p.log.WithFields(logrus.Fields{ 56 | "uploader": id, 57 | }).Warn("uploader failed to get bucket") 58 | done <- true 59 | return 60 | } 61 | 62 | for a := range in { 63 | err := s3p.uploadFile(opts, bucket, a) 64 | if err != nil { 65 | a.UploadResult.OK = false 66 | a.UploadResult.Err = err 67 | } else { 68 | a.UploadResult.OK = true 69 | } 70 | out <- a 71 | } 72 | 73 | done <- true 74 | return 75 | } 76 | 77 | func (s3p *s3Provider) uploadFile(opts *Options, b *s3.Bucket, a *artifact.Artifact) error { 78 | retries := uint64(0) 79 | 80 | for { 81 | err := s3p.rawUpload(opts, b, a) 82 | if err == nil { 83 | return nil 84 | } 85 | if retries < opts.Retries { 86 | retries++ 87 | s3p.log.WithFields(logrus.Fields{ 88 | "artifact": a.Source, 89 | "retry": retries, 90 | "err": err, 91 | }).Debug("retrying") 92 | time.Sleep(s3p.RetryInterval) 93 | continue 94 | } else { 95 | return err 96 | } 97 | } 98 | return nil 99 | } 100 | 101 | func (s3p *s3Provider) rawUpload(opts *Options, b *s3.Bucket, a *artifact.Artifact) error { 102 | dest := a.FullDest() 103 | reader, err := a.Reader() 104 | if err != nil { 105 | return err 106 | } 107 | 108 | ctype := a.ContentType() 109 | size, err := a.Size() 110 | if err != nil { 111 | return err 112 | } 113 | 114 | s3p.log.WithFields(logrus.Fields{ 115 | "download_url": fmt.Sprintf("%s/%s/%s", s3p.getRegion().S3Endpoint, b.Name, dest), 116 | }).Info(fmt.Sprintf("uploading: %s (size: %s)", a.Source, humanize.Bytes(size))) 117 | 118 | s3p.log.WithFields(logrus.Fields{ 119 | "percent_max_size": pctMax(size, opts.MaxSize), 120 | "max_size": humanize.Bytes(opts.MaxSize), 121 | "source": a.Source, 122 | "dest": dest, 123 | "bucket": b.Name, 124 | "content_type": ctype, 125 | "cache_control": opts.CacheControl, 126 | }).Debug("more artifact details") 127 | 128 | err = b.PutReaderHeader(dest, reader, int64(size), 129 | map[string][]string{ 130 | "Content-Type": []string{ctype}, 131 | "Cache-Control": []string{opts.CacheControl}, 132 | }, a.Perm) 133 | if err != nil { 134 | return err 135 | } 136 | 137 | return nil 138 | } 139 | 140 | func (s3p *s3Provider) getConn(auth aws.Auth) *s3.S3 { 141 | if s3p.overrideConn != nil { 142 | s3p.log.WithField("conn", s3p.overrideConn).Debug("using override connection") 143 | return s3p.overrideConn 144 | } 145 | 146 | return s3.New(auth, s3p.getRegion()) 147 | } 148 | 149 | func (s3p *s3Provider) getAuth(accessKey, secretKey string) (aws.Auth, error) { 150 | if s3p.overrideAuth != nilAuth { 151 | s3p.log.WithField("auth", s3p.overrideAuth).Debug("using override auth") 152 | return s3p.overrideAuth, nil 153 | } 154 | 155 | s3p.log.Debug("creating new auth") 156 | return aws.GetAuth(accessKey, secretKey) 157 | } 158 | 159 | func (s3p *s3Provider) getRegion() aws.Region { 160 | region, ok := aws.Regions[s3p.opts.S3Region] 161 | 162 | if !ok { 163 | s3p.log.WithFields(logrus.Fields{ 164 | "region": s3p.opts.S3Region, 165 | "default": DefaultOptions.S3Region, 166 | }).Warn(fmt.Sprintf("invalid region, defaulting to %s", DefaultOptions.S3Region)) 167 | region = aws.Regions[DefaultOptions.S3Region] 168 | } 169 | 170 | return region 171 | } 172 | 173 | func (s3p *s3Provider) Name() string { 174 | return "s3" 175 | } 176 | -------------------------------------------------------------------------------- /upload/s3_provider_test.go: -------------------------------------------------------------------------------- 1 | package upload 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | 8 | "github.com/mitchellh/goamz/aws" 9 | "github.com/mitchellh/goamz/s3" 10 | "github.com/mitchellh/goamz/s3/s3test" 11 | "github.com/travis-ci/artifacts/artifact" 12 | ) 13 | 14 | var ( 15 | s3srv = &localS3Server{ 16 | config: &s3test.Config{ 17 | Send409Conflict: true, 18 | }, 19 | } 20 | testS3 *s3.S3 21 | ) 22 | 23 | func init() { 24 | s3srv.SetUp() 25 | testS3 = s3.New(s3srv.Auth, s3srv.Region) 26 | err = testS3.Bucket("bucket").PutBucket(s3.ACL("public-read")) 27 | if err != nil { 28 | panic(err) 29 | } 30 | } 31 | 32 | type localS3Server struct { 33 | Auth aws.Auth 34 | Region aws.Region 35 | srv *s3test.Server 36 | config *s3test.Config 37 | } 38 | 39 | func (s *localS3Server) SetUp() { 40 | if s.srv != nil { 41 | return 42 | } 43 | 44 | srv, err := s3test.NewServer(s.config) 45 | if err != nil { 46 | panic(err) 47 | } 48 | 49 | s.srv = srv 50 | s.Region = aws.Region{ 51 | Name: "faux-region-9000", 52 | S3Endpoint: srv.URL(), 53 | S3LocationConstraint: true, 54 | } 55 | } 56 | 57 | func TestNewS3Provider(t *testing.T) { 58 | s3p := newS3Provider(NewOptions(), getPanicLogger()) 59 | 60 | if s3p.RetryInterval != defaultProviderRetryInterval { 61 | t.Fatalf("RetryInterval %v != %v", s3p.RetryInterval, defaultProviderRetryInterval) 62 | } 63 | 64 | if s3p.Name() != "s3" { 65 | t.Fatalf("Name %v != s3", s3p.Name()) 66 | } 67 | } 68 | 69 | func TestS3ProviderUpload(t *testing.T) { 70 | opts := NewOptions() 71 | s3p := newS3Provider(opts, getPanicLogger()) 72 | s3p.overrideConn = testS3 73 | s3p.overrideAuth = aws.Auth{ 74 | AccessKey: "whatever", 75 | SecretKey: "whatever", 76 | Token: "whatever", 77 | } 78 | 79 | in := make(chan *artifact.Artifact) 80 | out := make(chan *artifact.Artifact) 81 | done := make(chan bool) 82 | 83 | go s3p.Upload("test-0", opts, in, out, done) 84 | 85 | go func() { 86 | for _, p := range testArtifactPaths { 87 | if !p.Valid { 88 | continue 89 | } 90 | 91 | a := artifact.New("bucket", p.Path, "linux/foo", &artifact.Options{ 92 | Perm: s3.PublicRead, 93 | RepoSlug: "owner/foo", 94 | }) 95 | 96 | in <- a 97 | fmt.Printf("---> Fed artifact: %#v\n", a) 98 | } 99 | close(in) 100 | }() 101 | 102 | accum := []*artifact.Artifact{} 103 | for { 104 | select { 105 | case <-time.After(5 * time.Second): 106 | t.Fatalf("took too long oh derp") 107 | case a := <-out: 108 | accum = append(accum, a) 109 | case <-done: 110 | if len(accum) == 0 { 111 | t.Fatalf("nothing uploaded") 112 | } 113 | return 114 | } 115 | } 116 | } 117 | 118 | func TestS3ProviderRegionOption(t *testing.T) { 119 | opts := NewOptions() 120 | 121 | for input, output := range map[string]string{ 122 | "us-west-2": "us-west-2", 123 | "bogus-9000": "us-east-1", 124 | } { 125 | opts.S3Region = input 126 | s3p := newS3Provider(opts, getPanicLogger()) 127 | 128 | region := s3p.getRegion() 129 | if region.Name != output { 130 | t.Fatalf("region %v != %v", region.Name, output) 131 | } 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /upload/upload_provider.go: -------------------------------------------------------------------------------- 1 | package upload 2 | 3 | import ( 4 | "github.com/travis-ci/artifacts/artifact" 5 | ) 6 | 7 | type uploadProvider interface { 8 | Upload(string, *Options, 9 | chan *artifact.Artifact, chan *artifact.Artifact, chan bool) 10 | Name() string 11 | } 12 | -------------------------------------------------------------------------------- /upload/upload_test.go: -------------------------------------------------------------------------------- 1 | package upload 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "path/filepath" 9 | ) 10 | 11 | type testPath struct { 12 | Path string 13 | ContentType string 14 | Valid bool 15 | } 16 | 17 | var ( 18 | testTmp, err = ioutil.TempDir("", "artifacts-test-upload") 19 | testArtifactPathDir = filepath.Join(testTmp, "artifact") 20 | testArtifactPaths = []*testPath{ 21 | &testPath{ 22 | Path: filepath.Join(testArtifactPathDir, "foo"), 23 | ContentType: "text/plain; charset=utf-8", 24 | Valid: true, 25 | }, 26 | &testPath{ 27 | Path: filepath.Join(testArtifactPathDir, "foo.csv"), 28 | ContentType: "text/csv; charset=utf-8", 29 | Valid: true, 30 | }, 31 | &testPath{ 32 | Path: filepath.Join(testArtifactPathDir, "nonexistent"), 33 | ContentType: "application/octet-stream", 34 | Valid: false, 35 | }, 36 | &testPath{ 37 | Path: filepath.Join(testArtifactPathDir, "unreadable"), 38 | ContentType: "application/octet-stream", 39 | Valid: false, 40 | }, 41 | } 42 | ) 43 | 44 | func init() { 45 | os.Clearenv() 46 | 47 | if err != nil { 48 | log.Panicf("game over: %v\n", err) 49 | } 50 | 51 | err = os.MkdirAll(testArtifactPathDir, 0755) 52 | if err != nil { 53 | log.Panicf("game over: %v\n", err) 54 | } 55 | 56 | for _, p := range testArtifactPaths { 57 | if filepath.Base(p.Path) == "nonexistent" { 58 | continue 59 | } 60 | 61 | fd, err := os.Create(p.Path) 62 | if err != nil { 63 | log.Panicf("game over: %v\n", err) 64 | } 65 | 66 | defer fd.Close() 67 | 68 | for i := 0; i < 512; i++ { 69 | fmt.Fprintf(fd, "something\n") 70 | } 71 | 72 | if filepath.Base(p.Path) == "unreadable" { 73 | fd.Chmod(0000) 74 | } 75 | } 76 | } 77 | 78 | func setenvs(e map[string]string) error { 79 | for k, v := range e { 80 | err := os.Setenv(k, v) 81 | if err != nil { 82 | return err 83 | } 84 | } 85 | 86 | return nil 87 | } 88 | -------------------------------------------------------------------------------- /upload/uploader.go: -------------------------------------------------------------------------------- 1 | package upload 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | "sync" 9 | "time" 10 | 11 | "github.com/Sirupsen/logrus" 12 | "github.com/dustin/go-humanize" 13 | "github.com/mitchellh/goamz/s3" 14 | "github.com/travis-ci/artifacts/artifact" 15 | "github.com/travis-ci/artifacts/path" 16 | ) 17 | 18 | const ( 19 | defaultPublicCacheControl = "public, max-age=315360000" 20 | ) 21 | 22 | type uploader struct { 23 | Opts *Options 24 | Paths *path.Set 25 | RetryInterval time.Duration 26 | Provider uploadProvider 27 | 28 | log *logrus.Logger 29 | curSize *maxSizeTracker 30 | startTime time.Time 31 | } 32 | 33 | type maxSizeTracker struct { 34 | sync.Mutex 35 | Current uint64 36 | } 37 | 38 | // Upload does the deed! 39 | func Upload(opts *Options, log *logrus.Logger) error { 40 | return newUploader(opts, log).Upload() 41 | } 42 | 43 | func newUploader(opts *Options, log *logrus.Logger) *uploader { 44 | var provider uploadProvider 45 | 46 | if opts.CacheControl == "" { 47 | opts.CacheControl = defaultPublicCacheControl 48 | } 49 | 50 | if opts.Provider == "" { 51 | opts.Provider = "s3" 52 | } 53 | 54 | switch opts.Provider { 55 | case "artifacts": 56 | provider = newArtifactsProvider(opts, log) 57 | case "s3": 58 | provider = newS3Provider(opts, log) 59 | case "null": 60 | provider = newNullProvider(nil, log) 61 | default: 62 | log.WithFields(logrus.Fields{ 63 | "provider": opts.Provider, 64 | }).Warn("unrecognized provider, using s3 instead") 65 | provider = newS3Provider(opts, log) 66 | } 67 | 68 | u := &uploader{ 69 | Opts: opts, 70 | Paths: path.NewSet(), 71 | Provider: provider, 72 | 73 | log: log, 74 | startTime: time.Now(), 75 | } 76 | 77 | for _, s := range opts.Paths { 78 | parts := strings.SplitN(s, ":", 2) 79 | if len(parts) < 2 { 80 | parts = append(parts, "") 81 | } 82 | 83 | p := path.New(opts.WorkingDir, parts[0], parts[1]) 84 | log.WithFields(logrus.Fields{"path": p}).Debug("adding path") 85 | u.Paths.Add(p) 86 | } 87 | 88 | return u 89 | } 90 | 91 | func (u *uploader) Upload() error { 92 | u.log.Debug("starting upload") 93 | u.startTime = time.Now() 94 | done := make(chan bool) 95 | allDone := uint64(0) 96 | inChan := u.files() 97 | outChan := make(chan *artifact.Artifact) 98 | failed := []*artifact.Artifact{} 99 | 100 | defer func() { 101 | if len(failed) == 0 { 102 | return 103 | } 104 | 105 | for _, a := range failed { 106 | u.log.WithFields(logrus.Fields{ 107 | "err": a.UploadResult.Err, 108 | }).Error(fmt.Sprintf("failed to upload: %s", a.Source)) 109 | } 110 | }() 111 | 112 | u.log.WithFields(logrus.Fields{ 113 | "bucket": u.Opts.BucketName, 114 | "cache_control": u.Opts.CacheControl, 115 | "permissions": u.Opts.Perm, 116 | }).Info("uploading with settings") 117 | 118 | u.log.WithFields(logrus.Fields{ 119 | "working_dir": u.Opts.WorkingDir, 120 | "target_paths": u.Opts.TargetPaths, 121 | "concurrency": u.Opts.Concurrency, 122 | "max_size": u.Opts.MaxSize, 123 | "retries": u.Opts.Retries, 124 | }).Debug("other upload settings") 125 | 126 | for i := uint64(0); i < u.Opts.Concurrency; i++ { 127 | u.log.WithFields(logrus.Fields{ 128 | "uploader": i, 129 | }).Debug("starting uploader worker") 130 | 131 | go u.Provider.Upload(fmt.Sprintf("%d", i), u.Opts, inChan, outChan, done) 132 | } 133 | 134 | for { 135 | select { 136 | case outArtifact := <-outChan: 137 | if outArtifact != nil && !outArtifact.UploadResult.OK { 138 | failed = append(failed, outArtifact) 139 | } 140 | case <-done: 141 | allDone++ 142 | if allDone >= u.Opts.Concurrency { 143 | return nil 144 | } 145 | } 146 | } 147 | 148 | return nil 149 | } 150 | 151 | func (u *uploader) artifactFeederLoop(path *path.Path, artifacts chan *artifact.Artifact) error { 152 | to, from, root := path.To, path.From, path.Root 153 | u.log.WithField("path", path).Debug("incoming path") 154 | 155 | if path.IsDir() { 156 | root = filepath.Join(root, from) 157 | u.log.WithField("root", root).Debug("path is dir, so setting root to root+from") 158 | } 159 | 160 | artifactOpts := &artifact.Options{ 161 | Perm: s3.ACL(u.Opts.Perm), 162 | RepoSlug: u.Opts.RepoSlug, 163 | BuildNumber: u.Opts.BuildNumber, 164 | BuildID: u.Opts.BuildID, 165 | JobNumber: u.Opts.JobNumber, 166 | JobID: u.Opts.JobID, 167 | } 168 | 169 | filepath.Walk(path.Fullpath(), func(source string, info os.FileInfo, err error) error { 170 | if info != nil && info.IsDir() { 171 | u.log.WithField("path", source).Debug("skipping directory") 172 | return nil 173 | } 174 | 175 | relPath := strings.Replace(strings.Replace(source, root, "", -1), root+"/", "", -1) 176 | dest := relPath 177 | if len(to) > 0 { 178 | if path.IsDir() { 179 | dest = filepath.Join(to, relPath) 180 | } else { 181 | dest = to 182 | } 183 | } 184 | 185 | for _, targetPath := range u.Opts.TargetPaths { 186 | err := func() error { 187 | u.curSize.Lock() 188 | defer u.curSize.Unlock() 189 | 190 | a := artifact.New(targetPath, source, dest, artifactOpts) 191 | 192 | size, err := a.Size() 193 | if err != nil { 194 | return err 195 | } 196 | 197 | u.curSize.Current += size 198 | logFields := logrus.Fields{ 199 | "current_size": humanize.Bytes(u.curSize.Current), 200 | "max_size": humanize.Bytes(u.Opts.MaxSize), 201 | "percent_max_size": pctMax(size, u.Opts.MaxSize), 202 | "artifact": relPath, 203 | "artifact_size": humanize.Bytes(size), 204 | } 205 | 206 | if u.curSize.Current > u.Opts.MaxSize { 207 | msg := "max-size would be exceeded" 208 | u.log.WithFields(logFields).Error(msg) 209 | return fmt.Errorf(msg) 210 | } 211 | 212 | u.log.WithFields(logFields).Debug("queueing artifact") 213 | artifacts <- a 214 | return nil 215 | }() 216 | if err != nil { 217 | return err 218 | } 219 | } 220 | return nil 221 | }) 222 | 223 | return nil 224 | } 225 | 226 | func (u *uploader) artifactFeeder(artifacts chan *artifact.Artifact) error { 227 | u.curSize = &maxSizeTracker{Current: uint64(0)} 228 | 229 | i := 0 230 | for _, path := range u.Paths.All() { 231 | u.artifactFeederLoop(path, artifacts) 232 | i++ 233 | } 234 | 235 | u.log.WithFields(logrus.Fields{ 236 | "total_size": humanize.Bytes(u.curSize.Current), 237 | "count": i, 238 | "time_elapsed": time.Since(u.startTime), 239 | }).Debug("done feeding artifacts") 240 | 241 | close(artifacts) 242 | return nil 243 | } 244 | 245 | func (u *uploader) files() chan *artifact.Artifact { 246 | artifacts := make(chan *artifact.Artifact) 247 | go u.artifactFeeder(artifacts) 248 | return artifacts 249 | } 250 | -------------------------------------------------------------------------------- /upload/uploader_test.go: -------------------------------------------------------------------------------- 1 | package upload 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/Sirupsen/logrus" 8 | ) 9 | 10 | var ( 11 | isDebug = os.Getenv("ARTIFACTS_DEBUG") != "" 12 | ) 13 | 14 | func setUploaderEnv() { 15 | setenvs(map[string]string{ 16 | "TRAVIS_BUILD_NUMBER": "3", 17 | "TRAVIS_JOB_NUMBER": "3.2", 18 | "ARTIFACTS_S3_BUCKET": "foo", 19 | "ARTIFACTS_TARGET_PATHS": "baz:artifacts/$TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER", 20 | "ARTIFACTS_PATHS": "bin/:derp", 21 | "ARTIFACTS_UPLOAD_PROVIDER": "null", 22 | }) 23 | } 24 | 25 | func getPanicLogger() *logrus.Logger { 26 | log := logrus.New() 27 | log.Level = logrus.PanicLevel 28 | if isDebug { 29 | log.Level = logrus.DebugLevel 30 | } 31 | return log 32 | } 33 | 34 | func getTestUploader() *uploader { 35 | setUploaderEnv() 36 | 37 | log := getPanicLogger() 38 | u := newUploader(NewOptions(), log) 39 | u.Provider = newNullProvider(nil, log) 40 | return u 41 | } 42 | 43 | func TestNewUploader(t *testing.T) { 44 | u := getTestUploader() 45 | if u == nil { 46 | t.Errorf("options are %v", u) 47 | } 48 | 49 | if u.Opts.BucketName != "foo" { 50 | t.Errorf("bucket name is %v", u.Opts.BucketName) 51 | } 52 | 53 | if len(u.Opts.TargetPaths) != 2 { 54 | t.Errorf("target paths length != 2: %v", len(u.Opts.TargetPaths)) 55 | } 56 | 57 | if u.Opts.TargetPaths[0] != "baz" { 58 | t.Errorf("target paths[0] != baz: %v", u.Opts.TargetPaths) 59 | } 60 | 61 | if u.Opts.TargetPaths[1] != "artifacts/3/3.2" { 62 | t.Errorf("target paths[1] != artifacts/3/3.2: %v", u.Opts.TargetPaths) 63 | } 64 | 65 | if len(u.Paths.All()) != 2 { 66 | t.Errorf("all paths length != 2: %v", len(u.Paths.All())) 67 | } 68 | } 69 | 70 | var testOptsProviderCases = map[string]string{ 71 | "artifacts": "artifacts", 72 | "s3": "s3", 73 | "null": "null", 74 | "foo": "s3", 75 | "": "s3", 76 | } 77 | 78 | func TestNewUploaderProviderOptions(t *testing.T) { 79 | opts := NewOptions() 80 | for opt, name := range testOptsProviderCases { 81 | opts.Provider = opt 82 | u := newUploader(opts, getPanicLogger()) 83 | if u.Provider.Name() != name { 84 | t.Fatalf("new uploader does not have %s provider: %q != %q", 85 | name, u.Provider.Name(), name) 86 | } 87 | } 88 | } 89 | 90 | func TestNewUploaderUnsetCacheControlOption(t *testing.T) { 91 | opts := NewOptions() 92 | opts.CacheControl = "" 93 | u := newUploader(opts, getPanicLogger()) 94 | if u.Opts.CacheControl != defaultPublicCacheControl { 95 | t.Fatalf("new uploader cache control option not defaulted") 96 | } 97 | } 98 | 99 | func TestUpload(t *testing.T) { 100 | setUploaderEnv() 101 | err := Upload(NewOptions(), getPanicLogger()) 102 | if err != nil { 103 | t.Errorf("go boom: %v", err) 104 | } 105 | } 106 | 107 | func TestUploaderUpload(t *testing.T) { 108 | u := getTestUploader() 109 | if u == nil { 110 | t.Errorf("options are %v", u) 111 | } 112 | 113 | err := u.Upload() 114 | if err != nil { 115 | t.Errorf("failed to not really upload: %v", err) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /upload/util.go: -------------------------------------------------------------------------------- 1 | package upload 2 | 3 | func pctMax(artifactSize, maxSize uint64) float64 { 4 | return float64(100.0) * (float64(artifactSize) / float64(maxSize)) 5 | } 6 | --------------------------------------------------------------------------------