├── .gitignore ├── .arclint ├── client ├── run_server.sh ├── Dockerfile ├── run_test.sh ├── fig.yml ├── fakes3 │ └── Dockerfile ├── testserver │ └── testserver.go ├── client_unit_test.go ├── client.go └── client_test.go ├── migrations ├── 4_byte_array.sql ├── 2_index_artifactid_size.sql ├── 3_add_relative_path.sql ├── 1_initial.sql └── README ├── common ├── version.go ├── reqcontext │ └── reqcontext.go ├── clock.go ├── sentry │ └── sentry.go └── stats │ └── stats.go ├── ci ├── README.txt ├── run_tests.sh └── run_integration_tests.sh ├── model ├── logchunk.go ├── bucketstate_string.go ├── artifactstate_string.go ├── bucket.go └── artifact.go ├── migration.go ├── database ├── dberrortype_string.go ├── database.go ├── mock_Database.go ├── gorp_database.go └── bindata.go ├── support ├── bootstrap-vagrant.sh └── bootstrap-ubuntu.sh ├── Vagrantfile ├── README.md ├── Makefile ├── api ├── common.go ├── buckethandler_test.go ├── buckethandler.go ├── logchunkreader.go ├── logchunkreader_test.go ├── artifacthandler_test.go └── artifacthandler.go ├── server.go └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | .arcconfig 2 | .vagrant 3 | *.deb 4 | coverage.out 5 | env 6 | junit.xml 7 | test.output 8 | -------------------------------------------------------------------------------- /.arclint: -------------------------------------------------------------------------------- 1 | { 2 | "exclude": "(_Database.go$|_string.go$|^database/bindata.go$)", 3 | "linters": { 4 | "sample": { 5 | "type": "golint" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /client/run_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export GOMAXPROCS=4 4 | cd /go/src/github.com/dropbox/changes-artifacts 5 | go run server.go -migrations-only 6 | go run server.go -verbose 7 | -------------------------------------------------------------------------------- /migrations/4_byte_array.sql: -------------------------------------------------------------------------------- 1 | -- +migrate Up 2 | ALTER TABLE logchunk ADD COLUMN content_bytes BYTEA; 3 | 4 | -- +migrate Down 5 | ALTER TABLE logchunk DROP COLUMN content_bytes; 6 | -------------------------------------------------------------------------------- /common/version.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | const version = "0.0.1" 4 | 5 | var gitVersion string 6 | 7 | func GetVersion() string { 8 | return version + "-" + gitVersion 9 | } 10 | -------------------------------------------------------------------------------- /migrations/2_index_artifactid_size.sql: -------------------------------------------------------------------------------- 1 | -- +migrate Up 2 | CREATE INDEX logchunk_artifactid_size_last ON logchunk (artifactid, size DESC NULLS LAST); 3 | 4 | -- +migrate Down 5 | DROP INDEX logchunk_artifactid_size_last; 6 | -------------------------------------------------------------------------------- /ci/README.txt: -------------------------------------------------------------------------------- 1 | Scripts used by Changes to setup and run tests for Changes Artifacts. 2 | 3 | 'setup.sh' is used to prepare the test environment (assuming a bare Ubuntu instance). 4 | 'run_tests.sh' will run all tests once the env is set up complete. 5 | -------------------------------------------------------------------------------- /migrations/3_add_relative_path.sql: -------------------------------------------------------------------------------- 1 | -- +migrate Up 2 | ALTER TABLE artifact ADD COLUMN relativepath VARCHAR(255); 3 | UPDATE artifact SET relativepath = name WHERE relativepath IS NULL; 4 | 5 | -- +migrate Down 6 | ALTER TABLE artifact DROP COLUMN "relativepath"; 7 | -------------------------------------------------------------------------------- /model/logchunk.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | type LogChunk struct { 4 | // Automatically-generated unique id. 5 | Id int64 6 | ArtifactId int64 7 | ByteOffset int64 8 | Size int64 9 | ContentBytes []byte `db:"content_bytes"` 10 | } 11 | -------------------------------------------------------------------------------- /ci/run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eux 2 | 3 | pwd 4 | ls -lh 5 | 6 | cd $GOPATH/src/github.com/dropbox/changes-artifacts/ 7 | go get -v ./... 8 | 9 | go get -v github.com/jstemmer/go-junit-report 10 | 11 | go test -short -v ./... | tee test.output | go-junit-report > junit.xml 12 | cat test.output 13 | -------------------------------------------------------------------------------- /client/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:latest 2 | MAINTAINER Anup Chenthamarakshan 3 | 4 | RUN go get -v github.com/go-martini/martini gopkg.in/amz.v1/s3 gopkg.in/gorp.v1 github.com/lib/pq github.com/martini-contrib/render github.com/stretchr/testify golang.org/x/tools/cmd/cover github.com/cenkalti/backoff 5 | -------------------------------------------------------------------------------- /migration.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // go get github.com/jteeuwen/go-bindata/... to install go-bindata 4 | 5 | // We don't really care about modtime, because all migrations will be applied regardless of their 6 | // timestamp. 7 | //go:generate go-bindata -pkg database -modtime=1 -o database/bindata.go migrations/ 8 | -------------------------------------------------------------------------------- /client/run_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd /go/src/github.com/dropbox/changes-artifacts 4 | 5 | # Import missing dependencies 6 | go get -v ./... 7 | 8 | /mnt/run_server.sh & 9 | # No need to wait here. Test implicitly waits for the server to be running. 10 | go test -v ./client/... && echo '\n\n\n' && go test -cover ./client/... 11 | pkill -9 artifacts 12 | -------------------------------------------------------------------------------- /model/bucketstate_string.go: -------------------------------------------------------------------------------- 1 | // generated by stringer -type=BucketState; DO NOT EDIT 2 | 3 | package model 4 | 5 | import "fmt" 6 | 7 | const _BucketState_name = "UNKNOWNOPENCLOSEDTIMEDOUT" 8 | 9 | var _BucketState_index = [...]uint8{0, 7, 11, 17, 25} 10 | 11 | func (i BucketState) String() string { 12 | if i >= BucketState(len(_BucketState_index)-1) { 13 | return fmt.Sprintf("BucketState(%d)", i) 14 | } 15 | return _BucketState_name[_BucketState_index[i]:_BucketState_index[i+1]] 16 | } 17 | -------------------------------------------------------------------------------- /database/dberrortype_string.go: -------------------------------------------------------------------------------- 1 | // generated by stringer -type=DBErrorType; DO NOT EDIT 2 | 3 | package database 4 | 5 | import "fmt" 6 | 7 | const _DBErrorType_name = "INTERNALVALIDATION_FAILUREENTITY_NOT_FOUND" 8 | 9 | var _DBErrorType_index = [...]uint8{0, 8, 26, 42} 10 | 11 | func (i DBErrorType) String() string { 12 | if i < 0 || i >= DBErrorType(len(_DBErrorType_index)-1) { 13 | return fmt.Sprintf("DBErrorType(%d)", i) 14 | } 15 | return _DBErrorType_name[_DBErrorType_index[i]:_DBErrorType_index[i+1]] 16 | } 17 | -------------------------------------------------------------------------------- /client/fig.yml: -------------------------------------------------------------------------------- 1 | test: 2 | build: . 3 | command: /mnt/run_test.sh 4 | volumes: 5 | - ~/gopath/src/github.com/dropbox/changes-artifacts:/go/src/github.com/dropbox/changes-artifacts 6 | - run_test.sh:/mnt/run_test.sh 7 | - run_server.sh:/mnt/run_server.sh 8 | links: 9 | - artifactsdb 10 | - fakes3 11 | 12 | artifactsdb: 13 | image: postgres:latest 14 | environment: 15 | - POSTGRES_USER=artifacts 16 | - POSTGRES_PASSWORD=artifacts 17 | 18 | fakes3: 19 | build: fakes3 20 | ports: 21 | - "4569:4569" 22 | -------------------------------------------------------------------------------- /support/bootstrap-vagrant.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eux 2 | 3 | cd /vagrant/ 4 | 5 | support/bootstrap-ubuntu.sh 6 | 7 | echo "alias work='cd \$GOPATH/src/github.com/dropbox/changes-artifacts'" | sudo tee /etc/profile.d/work-alias.sh 8 | 9 | export PATH=/usr/local/go/bin:$PATH 10 | export GOPATH=~/ 11 | 12 | # Install dependencies for 'go generate' 13 | sudo chown -R `whoami` ~/src 14 | go get -v github.com/vektra/mockery/cmd/mockery 15 | go get -v github.com/jteeuwen/go-bindata/go-bindata 16 | go get -v golang.org/x/tools/cmd/stringer 17 | -------------------------------------------------------------------------------- /ci/run_integration_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -x 2 | 3 | cd $GOPATH/src/github.com/dropbox/changes-artifacts/ 4 | export PATH=$GOPATH/bin:$PATH 5 | 6 | go get -v github.com/jstemmer/go-junit-report 7 | go get -v ./... 8 | 9 | go run server.go -migrations-only 10 | 11 | PORT=3000 go run server.go -verbose & 12 | sudo fakes3 -r /var/cache/fakes3 -p 4569 & 13 | 14 | go test -v -race -cover $@ ./... | tee test.output | go-junit-report > junit.xml 15 | 16 | statuscode=${PIPESTATUS[0]} 17 | 18 | pkill -9 server 19 | sudo pkill -9 fakes3 20 | 21 | cat test.output 22 | 23 | exit $statuscode 24 | -------------------------------------------------------------------------------- /model/artifactstate_string.go: -------------------------------------------------------------------------------- 1 | // generated by stringer -type=ArtifactState; DO NOT EDIT 2 | 3 | package model 4 | 5 | import "fmt" 6 | 7 | const _ArtifactState_name = "UNKNOWN_ARTIFACT_STATEERRORAPPENDINGAPPEND_COMPLETEWAITING_FOR_UPLOADUPLOADINGUPLOADEDDEADLINE_EXCEEDEDCLOSED_WITHOUT_DATA" 8 | 9 | var _ArtifactState_index = [...]uint8{0, 22, 27, 36, 51, 69, 78, 86, 103, 122} 10 | 11 | func (i ArtifactState) String() string { 12 | if i >= ArtifactState(len(_ArtifactState_index)-1) { 13 | return fmt.Sprintf("ArtifactState(%d)", i) 14 | } 15 | return _ArtifactState_name[_ArtifactState_index[i]:_ArtifactState_index[i+1]] 16 | } 17 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! 5 | VAGRANTFILE_API_VERSION = "2" 6 | 7 | Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| 8 | config.vm.box = "ubuntu/trusty64" 9 | 10 | config.vm.provider "virtualbox" do |v| 11 | v.memory = 2048 12 | v.cpus = 4 13 | end 14 | 15 | config.ssh.forward_agent = true 16 | 17 | config.vm.synced_folder "./", "/home/vagrant/src/github.com/dropbox/changes-artifacts", owner: "vagrant", group: "vagrant" 18 | 19 | config.vm.provision :shell, :privileged => false, :path => "support/bootstrap-vagrant.sh" 20 | end 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ***NOTICE: THIS REPO IS NO LONGER UPDATED*** 2 | 3 | 4 | Changes Artifacts 5 | ================= 6 | An artifact server and client for use with changes. Used for storing 7 | the results of builds in Amazon S3. 8 | 9 | This project is in its infancy - even more so than changes itself - 10 | and so is unstable. 11 | 12 | There is a test suite and test environment for this project - run 13 | "fig up" in the client/ directory to run it. This runs against 14 | fake-s3 so there is no need for S3 credentials. 15 | 16 | Building deb package 17 | -------------------- 18 | 19 | ``` 20 | host$ vagrant up 21 | host$ vagrant ssh 22 | VM$ sudo chown -R vagrant:vagrant src/ 23 | VM$ work 24 | VM$ make 25 | ``` 26 | -------------------------------------------------------------------------------- /client/fakes3/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ruby:latest 2 | MAINTAINER Anup Chenthamarakshan 3 | 4 | # Almost entirely lifted from https://github.com/spurious-io/s3/blob/master/Dockerfile 5 | # and https://registry.hub.docker.com/u/lphoward/fake-s3/dockerfile/ 6 | 7 | RUN mkdir -p /var/data/fakes3 8 | WORKDIR /var/data/fakes3 9 | 10 | RUN gem install fakes3 11 | 12 | # Not having below line makes PutObject requests terribly slow in some cases (not sure what) 13 | RUN cd /usr/local/lib/ruby && grep -l -ri ':DoNotReverseLookup *=> nil' * | xargs sed -i "s/:DoNotReverseLookup *=> nil/:DoNotReverseLookup => true/" 14 | 15 | EXPOSE 4569 16 | 17 | ENTRYPOINT ["fakes3", "-r" ,"/var/data/fakes3", "-p", "4569"] 18 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Shamelessly copied from https://github.com/dropbox/changes-client/blob/master/Makefile 2 | 3 | BIN=${GOPATH}/bin/changes-artifacts 4 | 5 | # Revision shows date of latest commit and abbreviated commit SHA 6 | # E.g., 1438708515-753e183 7 | REV=`git show -s --format=%ct-%h HEAD` 8 | 9 | deb: 10 | @echo "Compiling changes-artifacts" 11 | @make install 12 | 13 | @echo "Setting up temp build folder" 14 | rm -rf /tmp/changes-artifacts-build 15 | mkdir -p /tmp/changes-artifacts-build/usr/bin 16 | cp $(BIN) /tmp/changes-artifacts-build/usr/bin/changes-artifacts 17 | 18 | @echo "Creating .deb file" 19 | fpm -s dir -t deb -n "changes-artifacts" -v "`$(BIN) --version`" -C /tmp/changes-artifacts-build . 20 | 21 | install: 22 | @make deps 23 | go clean -i ./... 24 | go install -ldflags "-X github.com/dropbox/changes-artifacts/common.gitVersion $(REV)" -v ./... 25 | 26 | deps: 27 | go get -v ./... 28 | -------------------------------------------------------------------------------- /migrations/1_initial.sql: -------------------------------------------------------------------------------- 1 | -- +migrate Up 2 | -- SQL in section 'Up' is executed when this migration is applied 3 | CREATE TABLE IF NOT EXISTS bucket ( 4 | dateclosed TIMESTAMP WITH TIME ZONE, 5 | datecreated TIMESTAMP WITH TIME ZONE, 6 | id TEXT NOT NULL PRIMARY KEY, 7 | owner TEXT, 8 | state TEXT 9 | ); 10 | CREATE TABLE IF NOT EXISTS artifact ( 11 | bucketid TEXT, 12 | datecreated TIMESTAMP WITH TIME ZONE, 13 | id BIGSERIAL NOT NULL PRIMARY KEY, 14 | name TEXT, 15 | s3url TEXT, 16 | size BIGINT, 17 | state TEXT, 18 | deadlinemins TEXT, 19 | UNIQUE (bucketid, name) 20 | ); 21 | CREATE TABLE IF NOT EXISTS logchunk ("id" BIGSERIAL NOT NULL PRIMARY KEY , "artifactid" BIGINT, "byteoffset" BIGINT, "size" BIGINT, "content" TEXT); 22 | 23 | -- +migrate Down 24 | -- SQL section 'Down' is executed when this migration is rolled back 25 | DROP TABLE bucket; 26 | DROP TABLE artifact; 27 | DROP TABLE logchunk; 28 | -------------------------------------------------------------------------------- /model/bucket.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import "time" 4 | 5 | //go:generate stringer -type=BucketState 6 | type BucketState uint 7 | 8 | // Please remember to update the mapping to strings. 9 | const ( 10 | // A Bucket should never be in this state. 11 | UNKNOWN BucketState = iota 12 | 13 | // Accepting new artifacts and appends to existing artifacts 14 | OPEN 15 | 16 | // No further changes to this bucket. No new artifacts or appends to existing ones. 17 | CLOSED 18 | 19 | // Similar to `CLOSED`. Was forcibly closed because it was not explicitly closed before deadline. 20 | // TODO This isn't implemented yet. Implement it. 21 | TIMEDOUT 22 | ) 23 | 24 | type Bucket struct { 25 | DateClosed time.Time `json:"dateClosed"` 26 | DateCreated time.Time `json:"dateCreated"` 27 | // Must be globally unique even between different owners. Other than that, it can 28 | // be arbitrary. 29 | Id string `json:"id"` 30 | // A characteristic string signifying what service owns the bucket. 31 | Owner string `json:"owner"` 32 | State BucketState `json:"state"` 33 | } 34 | -------------------------------------------------------------------------------- /common/reqcontext/reqcontext.go: -------------------------------------------------------------------------------- 1 | package reqcontext 2 | 3 | import ( 4 | "net/http" 5 | 6 | "golang.org/x/net/context" 7 | 8 | "github.com/go-martini/martini" 9 | ) 10 | 11 | // Unexported to avoid collisions 12 | type key int 13 | 14 | const requestKey key = 0 15 | 16 | func withReq(ctx context.Context, req *http.Request) context.Context { 17 | return context.WithValue(ctx, requestKey, req) 18 | } 19 | 20 | // ReqFromContext fetches an instance of http.Request embedded in given context.Context. 21 | func ReqFromContext(ctx context.Context) (*http.Request, bool) { 22 | r, ok := ctx.Value(requestKey).(*http.Request) 23 | return r, ok 24 | } 25 | 26 | // ContextHandler returns a middleware handler that populates a Context instance with current 27 | // request and sentry information. There are no requirements on where the handler is to be installed 28 | // in the handler chain. 29 | func ContextHandler(rootCtx context.Context) martini.Handler { 30 | return func(res http.ResponseWriter, req *http.Request, c martini.Context) { 31 | c.Map(withReq(rootCtx, req)) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /migrations/README: -------------------------------------------------------------------------------- 1 | Artifacts Store Database Migrations 2 | =================================== 3 | 4 | Add any migration scripts titled _.sql to this folder. Scripts will be 5 | executed in order. Please see https://github.com/rubenv/sql-migrate#writing-migrations for 6 | guidelines on how to write the migration scripts. 7 | 8 | NOTE: Please run `go generate -v ./...` after making any changes to this folder. We are embedding the 9 | migration script as a go file (this translation is performed by `go generate`). Without running `go 10 | generate`, the migration file will *NOT* be included and applied. 11 | 12 | Testing database migrations (backwards compatiblity): 13 | ----------------------------------------------------- 14 | During client integration tests, we execute the same migrations that we would on prod. By default, 15 | these tests will run against the previously deployed version (this *should* match the version on 16 | prod). To verify that tests continue to run after migrations, bump up the migration level stored in 17 | client_test.go#setupDB. 18 | -------------------------------------------------------------------------------- /api/common.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "math/rand" 7 | 8 | "golang.org/x/net/context" 9 | 10 | "github.com/dropbox/changes-artifacts/common/sentry" 11 | "github.com/martini-contrib/render" 12 | ) 13 | 14 | // LogAndRespondWithErrorf posts a JSON-serialized error message and statuscode on the HTTP response 15 | // object (using Martini render). 16 | // Log the error message to Sentry. 17 | func LogAndRespondWithErrorf(ctx context.Context, render render.Render, code int, errStr string, params ...interface{}) { 18 | msg := fmt.Sprintf(errStr, params...) 19 | sentry.ReportError(ctx, errors.New(msg)) 20 | render.JSON(code, map[string]string{"error": msg}) 21 | } 22 | 23 | // LogAndRespondWithError posts a JSON-serialized error and statuscode on the HTTP response object 24 | // (using Martini render). 25 | // Log the error message to Sentry. 26 | func LogAndRespondWithError(ctx context.Context, render render.Render, code int, err error) { 27 | sentry.ReportError(ctx, err) 28 | render.JSON(code, map[string]string{"error": err.Error()}) 29 | } 30 | 31 | // RespondWithErrorf posts a JSON-serialized error message and statuscode on the HTTP response 32 | // object (using Martini render). 33 | func RespondWithErrorf(ctx context.Context, render render.Render, code int, errStr string, params ...interface{}) { 34 | msg := fmt.Sprintf(errStr, params...) 35 | render.JSON(code, map[string]string{"error": msg}) 36 | } 37 | 38 | // RespondWithError posts a JSON-serialized error and statuscode on the HTTP response object 39 | // (using Martini render). 40 | func RespondWithError(ctx context.Context, render render.Render, code int, err error) { 41 | render.JSON(code, map[string]string{"error": err.Error()}) 42 | } 43 | 44 | const alphabet = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" 45 | 46 | func randString(n int) string { 47 | strBytes := make([]byte, n) 48 | for i := range strBytes { 49 | strBytes[i] = alphabet[rand.Intn(len(alphabet))] 50 | } 51 | return string(strBytes) 52 | } 53 | -------------------------------------------------------------------------------- /support/bootstrap-ubuntu.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eux 2 | 3 | export DEBIAN_FRONTEND=noninteractive 4 | 5 | install_go() { 6 | GO_VERSION=1.6 7 | re=\\bgo$GO_VERSION\\b 8 | 9 | if [ -x /usr/local/go/bin/go ] && [[ `/usr/local/go/bin/go version` =~ $re ]] 10 | then 11 | echo "Go binary already installed" 12 | return 13 | fi 14 | 15 | sudo rm -rf /usr/local/go 16 | echo "Installing Go binary...." 17 | cd /tmp 18 | wget -O go${GO_VERSION}.linux-amd64.tar.gz -q https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz 19 | sudo tar -C /usr/local -xzf "go${GO_VERSION}.linux-amd64.tar.gz" 20 | echo "Installed Go binary...." 21 | 22 | echo 'export PATH=/usr/local/go/bin:$PATH' | sudo tee /etc/profile.d/golang.sh 23 | echo 'export GOPATH=~/' | sudo tee /etc/profile.d/gopath.sh 24 | 25 | /usr/local/go/bin/go version 26 | 27 | # Install git (required for go get) 28 | sudo apt-get install -y git 29 | } 30 | 31 | install_fpm() { 32 | # Install gem first 33 | sudo apt-get install -y ruby-dev gcc 34 | sudo gem install fpm --no-ri --no-rdoc 35 | } 36 | 37 | install_fakes3() { 38 | # Install gem first 39 | sudo apt-get install -y ruby-dev gcc 40 | sudo gem install fakes3 41 | sudo mkdir -p /var/cache/fakes3 42 | 43 | # Required for integration tests, OK to be run many times 44 | echo "127.0.0.1 fakes3" | sudo tee -a /etc/hosts 45 | } 46 | 47 | install_postgres() { 48 | PG_INSTALLED=1 49 | dpkg -s postgresql-9.3 >/dev/null 2>&1 || PG_INSTALLED=0 50 | if [ $PG_INSTALLED -ne 1 ] 51 | then 52 | sudo apt-get install -y postgresql-9.3 53 | echo "127.0.0.1 artifactsdb" | sudo tee -a /etc/hosts 54 | sudo -u postgres psql -U postgres << EOF 55 | CREATE ROLE artifacts LOGIN password 'artifacts'; 56 | CREATE DATABASE artifacts ENCODING 'UTF8' OWNER artifacts; 57 | EOF 58 | 59 | sudo sed -i "s/peer$/md5/g" /etc/postgresql/9.3/main/pg_hba.conf 60 | sudo service postgresql restart 61 | fi 62 | } 63 | 64 | # Required to make sure postgres install doesn't fail :( 65 | sudo apt-get -y update 66 | 67 | install_go 68 | install_fpm 69 | install_fakes3 70 | install_postgres 71 | -------------------------------------------------------------------------------- /model/artifact.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | //go:generate stringer -type=ArtifactState 9 | type ArtifactState uint 10 | 11 | // NOTE: Do not reorder. Always append new entries to the bottom. Any existing entries which 12 | // are deprecated should be renamed from FOO to DEPRECATED_FOO and left in the same position. 13 | // 14 | // Please remember to update StateString 15 | const ( 16 | UNKNOWN_ARTIFACT_STATE ArtifactState = 0 17 | 18 | // Error during streamed upload. 19 | ERROR ArtifactState = 1 20 | 21 | // Log file being streamed in chunks. We currently store them as LogChunks. 22 | APPENDING ArtifactState = 2 23 | 24 | // Once the artifact has been finalized (or the bucket closed), the artifact which was being 25 | // appended will be marked for compaction and upload to S3. 26 | APPEND_COMPLETE ArtifactState = 3 27 | 28 | // The artifact is waiting for a file upload request to stream through to S3. 29 | WAITING_FOR_UPLOAD ArtifactState = 4 30 | 31 | // If the artifact is in LogChunks, it is now being merged and uploaded to S3. 32 | // Else, the file is being passed through to S3 directly from the client. 33 | UPLOADING ArtifactState = 5 34 | 35 | // Terminal state: the artifact is in S3 in its entirety. 36 | UPLOADED ArtifactState = 6 37 | 38 | // Deadline exceeded before APPEND_COMPLETE OR UPLOADED 39 | DEADLINE_EXCEEDED ArtifactState = 7 40 | 41 | // Artifact was closed without any appends or upload operation. 42 | CLOSED_WITHOUT_DATA ArtifactState = 8 43 | ) 44 | 45 | type Artifact struct { 46 | BucketId string `json:"bucketId"` 47 | DateCreated time.Time `json:"dateCreated"` 48 | // Auto-generated globally unique id. 49 | Id int64 `json:"id"` 50 | // id that must be unique within a bucket (but not necessairly globally). 51 | // For streamed artifacts this is often the file name. 52 | Name string `json:"name"` 53 | // This is deterministically generated as // but in case we wish to 54 | // switch conventions later we store it. 55 | S3URL string `json:"s3URL"` 56 | Size int64 `json:"size"` 57 | State ArtifactState `json:"state"` 58 | DeadlineMins uint `json:"deadlineMins"` 59 | RelativePath string `json:"relativePath"` 60 | } 61 | 62 | func (a *Artifact) DefaultS3URL() string { 63 | return fmt.Sprintf("/%s/%s", a.BucketId, a.Name) 64 | } 65 | -------------------------------------------------------------------------------- /common/clock.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | // Lifted from https://github.com/buddyfs/buddystore/blob/master/clock.go 4 | 5 | import ( 6 | "sync" 7 | "time" 8 | 9 | "github.com/stretchr/testify/mock" 10 | ) 11 | 12 | type Clock interface { 13 | Now() time.Time 14 | AfterFunc(time.Duration, func()) *time.Timer 15 | } 16 | 17 | type RealClock struct { 18 | // Implements: 19 | // ClockIface 20 | } 21 | 22 | var _ Clock = new(RealClock) 23 | 24 | func (r *RealClock) Now() time.Time { 25 | return time.Now() 26 | } 27 | 28 | func (r *RealClock) AfterFunc(d time.Duration, f func()) *time.Timer { 29 | return time.AfterFunc(d, f) 30 | } 31 | 32 | type MockClock struct { 33 | frozen bool 34 | currentTime time.Time 35 | lock sync.RWMutex 36 | 37 | // AfterFunc simulation 38 | nextEvent func() 39 | nextEventTimer time.Time 40 | nextEventSet bool 41 | 42 | mock.Mock 43 | // Implements: 44 | // ClockIface 45 | } 46 | 47 | var _ Clock = new(MockClock) 48 | 49 | func NewMockClock() *MockClock { 50 | return new(MockClock) 51 | } 52 | 53 | func NewFrozenClock() *MockClock { 54 | return new(MockClock).Freeze() 55 | } 56 | 57 | func NewRealClock() Clock { 58 | return new(RealClock) 59 | } 60 | 61 | func (m *MockClock) Freeze() *MockClock { 62 | m.lock.Lock() 63 | defer m.lock.Unlock() 64 | 65 | m.frozen = true 66 | m.currentTime = time.Now() 67 | 68 | return m 69 | } 70 | 71 | func (m *MockClock) Advance(d time.Duration) *MockClock { 72 | m.lock.Lock() 73 | 74 | if !m.frozen { 75 | m.lock.Unlock() 76 | panic("Cannot advance live clock. Call MockClock.Freeze() first.") 77 | } 78 | 79 | m.currentTime = m.currentTime.Add(d) 80 | m.lock.Unlock() 81 | 82 | m.lock.RLock() 83 | defer m.lock.RUnlock() 84 | 85 | if m.nextEventSet { 86 | if m.currentTime.After(m.nextEventTimer) { 87 | // TODO: This is a synchronous call to avoid race conditions. 88 | m.nextEvent() 89 | } 90 | } 91 | 92 | return m 93 | } 94 | 95 | func (m *MockClock) Now() time.Time { 96 | m.lock.RLock() 97 | defer m.lock.RUnlock() 98 | 99 | if m.frozen { 100 | return m.currentTime 101 | } 102 | 103 | return time.Now() 104 | } 105 | 106 | func (m *MockClock) AfterFunc(d time.Duration, f func()) *time.Timer { 107 | m.lock.Lock() 108 | defer m.lock.Unlock() 109 | 110 | m.Mock.Called(d, f) 111 | 112 | m.nextEventTimer = m.currentTime.Add(d) 113 | m.nextEvent = f 114 | m.nextEventSet = true 115 | 116 | return time.NewTimer(d) 117 | } 118 | -------------------------------------------------------------------------------- /database/database.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/dropbox/changes-artifacts/model" 7 | _ "github.com/vektra/mockery" // Required to generate MockDatabase 8 | ) 9 | 10 | //go:generate stringer -type=DBErrorType 11 | type DBErrorType int 12 | 13 | const ( 14 | INTERNAL DBErrorType = iota 15 | 16 | // Fields not set or invalid 17 | VALIDATION_FAILURE 18 | 19 | // Entity not found in the database 20 | ENTITY_NOT_FOUND 21 | ) 22 | 23 | type DatabaseError struct { 24 | errStr string 25 | errType DBErrorType 26 | } 27 | 28 | func (dbe *DatabaseError) Error() string { 29 | return fmt.Sprintf("DatabaseError[%s]: %s", dbe.errType, dbe.errStr) 30 | } 31 | 32 | func (dbe *DatabaseError) GetError() error { 33 | if dbe != nil { 34 | return fmt.Errorf(dbe.Error()) 35 | } 36 | return nil 37 | } 38 | 39 | // We do this to ensure that DatabaseError implements Error. 40 | var _ error = new(DatabaseError) 41 | 42 | func MockDatabaseError() *DatabaseError { 43 | return &DatabaseError{errStr: "MOCK ERROR", errType: INTERNAL} 44 | } 45 | 46 | func WrapInternalDatabaseError(err error) *DatabaseError { 47 | if err == nil { 48 | return nil 49 | } 50 | 51 | return &DatabaseError{errStr: err.Error(), errType: INTERNAL} 52 | } 53 | 54 | func NewValidationError(format string, args ...interface{}) *DatabaseError { 55 | if format == "" { 56 | panic("Error formatting NewValidationError") 57 | } 58 | if len(args) > 0 { 59 | return &DatabaseError{errStr: fmt.Sprintf(format, args...), errType: VALIDATION_FAILURE} 60 | } 61 | return &DatabaseError{errStr: format, errType: VALIDATION_FAILURE} 62 | } 63 | 64 | func NewEntityNotFoundError(format string, args ...interface{}) *DatabaseError { 65 | if format == "" { 66 | panic("Error formatting NewEntityNotFoundError") 67 | } 68 | if len(args) > 0 { 69 | return &DatabaseError{errStr: fmt.Sprintf(format, args...), errType: ENTITY_NOT_FOUND} 70 | } 71 | return &DatabaseError{errStr: format, errType: ENTITY_NOT_FOUND} 72 | } 73 | 74 | func (dbe *DatabaseError) EntityNotFound() bool { 75 | return dbe != nil && dbe.errType == ENTITY_NOT_FOUND 76 | } 77 | 78 | //go:generate mockery -name=Database -inpkg 79 | type Database interface { 80 | // Register all DB table<->object mappings in memory 81 | RegisterEntities() 82 | 83 | // Bucket instance is expected to have id, datecreated, state and owner field set. 84 | InsertBucket(*model.Bucket) *DatabaseError 85 | 86 | InsertArtifact(*model.Artifact) *DatabaseError 87 | 88 | InsertLogChunk(*model.LogChunk) *DatabaseError 89 | 90 | // Bucket instance is expected to have id, datecreated, state and owner field set. 91 | UpdateBucket(*model.Bucket) *DatabaseError 92 | 93 | // TODO: Pagination and/or other forms of filtering 94 | ListBuckets() ([]model.Bucket, *DatabaseError) 95 | 96 | GetBucket(string) (*model.Bucket, *DatabaseError) 97 | 98 | ListArtifactsInBucket(string) ([]model.Artifact, *DatabaseError) 99 | 100 | UpdateArtifact(*model.Artifact) *DatabaseError 101 | 102 | ListLogChunksInArtifact(artifactID int64, offset int64, limit int64) ([]model.LogChunk, *DatabaseError) 103 | 104 | // Delete list of log chunks, primarily used to clean up log chunks after merging and uploading 105 | DeleteLogChunksForArtifact(int64) (int64, *DatabaseError) 106 | 107 | GetArtifactByName(bucket string, name string) (*model.Artifact, *DatabaseError) 108 | 109 | // Get last logchunk seen for an artifact. 110 | GetLastLogChunkSeenForArtifact(int64) (*model.LogChunk, *DatabaseError) 111 | } 112 | -------------------------------------------------------------------------------- /common/sentry/sentry.go: -------------------------------------------------------------------------------- 1 | package sentry 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | 8 | "golang.org/x/net/context" 9 | 10 | "github.com/dropbox/changes-artifacts/common" 11 | "github.com/dropbox/changes-artifacts/common/reqcontext" 12 | "github.com/getsentry/raven-go" 13 | "github.com/go-martini/martini" 14 | ) 15 | 16 | // getSentryClient returns a reporter which logs to Sentry if sentryDsn is provided. 17 | // Logs to standard logger if sentryDsn is nil. 18 | func getSentryClient(env string, sentryDsn string) *raven.Client { 19 | if sentryDsn == "" { 20 | return nil 21 | } 22 | 23 | sentryClient, err := raven.NewClient(sentryDsn, map[string]string{ 24 | "version": common.GetVersion(), 25 | "env": env, 26 | }) 27 | 28 | if err != nil { 29 | log.Println("Error creating a Sentry client:", err) 30 | return nil 31 | } 32 | 33 | return sentryClient 34 | } 35 | 36 | type key int 37 | 38 | const errReporterKey key = 0 39 | 40 | // CreateAndInstallSentryClient installs a Sentry client to the supplied context. 41 | // If an empty dsn is provided, the installed client will be nil. 42 | func CreateAndInstallSentryClient(ctx context.Context, env string, dsn string) context.Context { 43 | sentryClient := getSentryClient(env, dsn) 44 | if sentryClient != nil { 45 | ctx = context.WithValue(ctx, errReporterKey, sentryClient) 46 | } 47 | 48 | return ctx 49 | } 50 | 51 | func getErrorReporterFromContext(ctx context.Context) *raven.Client { 52 | r, ok := ctx.Value(errReporterKey).(*raven.Client) 53 | if !ok { 54 | // If no error reporter is installed, we'll use log.Print methods. 55 | return nil 56 | } 57 | 58 | return r 59 | } 60 | 61 | // ReportError logs an error to Sentry if a sentry client is installed in the context. 62 | // Logs to standard logger otherwise. 63 | func ReportError(ctx context.Context, err error) { 64 | sentryClient := getErrorReporterFromContext(ctx) 65 | reportError(ctx, sentryClient, err) 66 | } 67 | 68 | // ReportMessage logs a message to Sentry if a sentry client is installed. 69 | // Logs to standard logger otherwise. 70 | func ReportMessage(ctx context.Context, msg string) { 71 | sentryClient := getErrorReporterFromContext(ctx) 72 | 73 | reportMessage(ctx, sentryClient, msg) 74 | } 75 | 76 | func reportError(ctx context.Context, sentryClient *raven.Client, err error) { 77 | if sentryClient != nil { 78 | req, ok := reqcontext.ReqFromContext(ctx) 79 | 80 | if ok { 81 | sentryClient.CaptureError(err, map[string]string{}, raven.NewHttp(req)) 82 | } else { 83 | sentryClient.CaptureError(err, map[string]string{}) 84 | } 85 | } 86 | 87 | log.Printf("[Sentry Error] %v\n", err) 88 | } 89 | 90 | func reportMessage(ctx context.Context, sentryClient *raven.Client, msg string) { 91 | if sentryClient != nil { 92 | req, ok := reqcontext.ReqFromContext(ctx) 93 | 94 | if ok { 95 | sentryClient.CaptureMessage(msg, map[string]string{}, raven.NewHttp(req)) 96 | } else { 97 | sentryClient.CaptureMessage(msg, map[string]string{}) 98 | } 99 | } 100 | 101 | log.Printf("[Sentry Message] %v\n", msg) 102 | } 103 | 104 | // PanicHandler intercepts panic's from request handlers and sends them to Sentry. 105 | // Exception is re-panic'd to be handled up the chain. 106 | func PanicHandler() martini.Handler { 107 | return func(res http.ResponseWriter, req *http.Request, ctx context.Context, c martini.Context) { 108 | defer func() { 109 | if e := recover(); e != nil { 110 | log.Printf("Caught exception %v\n", e) 111 | if err, ok := e.(error); ok { 112 | ReportError(ctx, err) 113 | } else { 114 | ReportMessage(ctx, fmt.Sprintf("Caught error %s", e)) 115 | } 116 | panic(e) 117 | } 118 | }() 119 | c.Next() 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /common/stats/stats.go: -------------------------------------------------------------------------------- 1 | package stats 2 | 3 | import ( 4 | "expvar" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | "os" 9 | "sync" 10 | "time" 11 | 12 | "github.com/gin-gonic/gin" 13 | "github.com/quipo/statsd" 14 | ) 15 | 16 | // Stat holds a named integer counter, which is exported to expvar and statsd 17 | type Stat struct { 18 | key string 19 | exp *expvar.Int 20 | } 21 | 22 | // NewStat creates a named statistic with the given name 23 | func NewStat(key string) *Stat { 24 | return &Stat{key, expvar.NewInt(key)} 25 | } 26 | 27 | // Add increments the integer value stored in this stat 28 | func (v *Stat) Add(delta int64) { 29 | v.exp.Add(delta) 30 | lock.RLock() 31 | defer lock.RUnlock() 32 | stats.Incr(v.key, delta) 33 | } 34 | 35 | func (v *Stat) String() string { 36 | return v.exp.String() 37 | } 38 | 39 | // TimingStat holds a timer statistic, internally represented as two counters, total time consumed 40 | // and number of observations. 41 | type TimingStat struct { 42 | timeNs, counter *Stat 43 | } 44 | 45 | // NewTimingStat creates a timing statistic with the given name 46 | func NewTimingStat(key string) *TimingStat { 47 | timeNs := NewStat(key + "_time_ns") 48 | counter := NewStat(key + "_count") 49 | return &TimingStat{timeNs: timeNs, counter: counter} 50 | } 51 | 52 | // AddTimeSince calculates time elapsed since given start time and adds it to the stat. 53 | func (ts *TimingStat) AddTimeSince(start time.Time) { 54 | ts.Add(time.Since(start)) 55 | } 56 | 57 | // Add notes a timing event of given duration (increases observation count by one and adds time 58 | // duration to total time spent in event) 59 | func (ts *TimingStat) Add(delta time.Duration) { 60 | ts.counter.Add(1) 61 | ts.timeNs.Add(delta.Nanoseconds()) 62 | } 63 | 64 | // Interval between batched stats updates pushed to the statsd instance. 65 | const updateInterval = 5 * time.Second 66 | 67 | var requestCounter = NewStat("requests") 68 | var lock sync.RWMutex 69 | var noopClient = &statsd.NoopClient{} 70 | var stats statsd.Statsd = noopClient 71 | 72 | // CreateStatsdClient creates a local instances of a statsd client. Any errors will be logged to 73 | // console and ignored. 74 | func CreateStatsdClient(statsdURL, statsdPrefix string) error { 75 | lock.Lock() 76 | defer lock.Unlock() 77 | 78 | if stats != noopClient { 79 | // Already initialized. Don't overwrite 80 | return nil 81 | } 82 | 83 | if statsdURL != "" { 84 | hostname, err := os.Hostname() 85 | if err != nil { 86 | log.Printf("Could not read hostname. Using default noop statsd client: %s", err) 87 | return err 88 | } 89 | prefix := fmt.Sprintf("%s.%s.artifacts.", statsdPrefix, hostname) 90 | 91 | statsdClient := statsd.NewStatsdClient(statsdURL, prefix) 92 | 93 | if statsdClient != nil { 94 | stats = statsd.NewStatsdBuffer(updateInterval, statsdClient) 95 | } 96 | } else { 97 | log.Println("No statsd URL provided. Using default noop statsd client") 98 | } 99 | 100 | return nil 101 | } 102 | 103 | // ShutdownStatsdClient flushes any outstanding stats and terminates connections to statsd. 104 | func ShutdownStatsdClient() { 105 | stats.Close() 106 | } 107 | 108 | // Counter counts number of requests made to the server 109 | func Counter() gin.HandlerFunc { 110 | return func(_ *gin.Context) { 111 | requestCounter.Add(1) 112 | } 113 | } 114 | 115 | // Handler display a JSON object showing number of requests received 116 | // Copied from https://golang.org/src/expvar/expvar.go#L305 117 | func Handler(res http.ResponseWriter, req *http.Request) { 118 | res.Header().Set("Content-Type", "application/json; charset=utf-8") 119 | fmt.Fprintf(res, "{\n") 120 | first := true 121 | expvar.Do(func(kv expvar.KeyValue) { 122 | if !first { 123 | fmt.Fprintf(res, ",\n") 124 | } 125 | first = false 126 | fmt.Fprintf(res, "%q: %s", kv.Key, kv.Value) 127 | }) 128 | fmt.Fprintf(res, "\n}\n") 129 | } 130 | -------------------------------------------------------------------------------- /api/buckethandler_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/dropbox/changes-artifacts/common" 8 | "github.com/dropbox/changes-artifacts/database" 9 | "github.com/dropbox/changes-artifacts/model" 10 | "github.com/stretchr/testify/mock" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestCreateBucket(t *testing.T) { 15 | mockdb := &database.MockDatabase{} 16 | 17 | // Used to verify creation timestamp. 18 | mockClock := common.NewFrozenClock() 19 | 20 | // var bucket *model.Bucket 21 | var err error 22 | 23 | // Bad request 24 | _, err = CreateBucket(mockdb, mockClock, "", "owner") 25 | require.Error(t, err) 26 | 27 | _, err = CreateBucket(mockdb, mockClock, "id", "") 28 | require.Error(t, err) 29 | 30 | // DB error 31 | mockdb.On("GetBucket", "id").Return(nil, database.WrapInternalDatabaseError(fmt.Errorf("Internal Error"))).Once() 32 | _, err = CreateBucket(mockdb, mockClock, "id", "owner") 33 | require.Error(t, err) 34 | 35 | // Entity exists 36 | mockdb.On("GetBucket", "id").Return(&model.Bucket{}, nil).Once() 37 | _, err = CreateBucket(mockdb, mockClock, "id", "owner") 38 | require.Error(t, err) 39 | 40 | // DB error while creating bucket 41 | mockdb.On("GetBucket", "id").Return(nil, database.NewEntityNotFoundError("ENF")).Once() 42 | mockdb.On("InsertBucket", mock.AnythingOfType("*model.Bucket")).Return(database.WrapInternalDatabaseError(fmt.Errorf("INT"))).Once() 43 | _, err = CreateBucket(mockdb, mockClock, "id", "owner") 44 | require.Error(t, err) 45 | 46 | // Successfully created bucket 47 | mockdb.On("GetBucket", "id").Return(nil, database.NewEntityNotFoundError("ENF")).Once() 48 | mockdb.On("InsertBucket", mock.AnythingOfType("*model.Bucket")).Return(nil).Once() 49 | bucket, err := CreateBucket(mockdb, mockClock, "id", "owner") 50 | require.NoError(t, err) 51 | require.NotNil(t, bucket) 52 | 53 | mockdb.AssertExpectations(t) 54 | } 55 | 56 | func TestCloseBucket(t *testing.T) { 57 | mockdb := &database.MockDatabase{} 58 | 59 | // We're using this to verify closing timestamp. 60 | mockClock := common.NewFrozenClock() 61 | 62 | // If bucket is not currently open, return failure 63 | bucket := &model.Bucket{State: model.CLOSED} 64 | require.Error(t, CloseBucket(nil, bucket, mockdb, nil, nil)) 65 | 66 | bucket_id := "bucket_id_1" 67 | 68 | // If DB throws error in any step, return failure 69 | bucket = &model.Bucket{State: model.OPEN, Id: bucket_id} 70 | mockdb.On("UpdateBucket", bucket).Return(database.WrapInternalDatabaseError(fmt.Errorf("foo"))).Once() 71 | require.Error(t, CloseBucket(nil, bucket, mockdb, nil, mockClock)) 72 | 73 | bucket = &model.Bucket{State: model.OPEN, Id: bucket_id} 74 | mockdb.On("UpdateBucket", bucket).Return(nil).Once() 75 | mockdb.On("ListArtifactsInBucket", bucket.Id).Return(nil, database.WrapInternalDatabaseError(fmt.Errorf("err"))).Once() 76 | require.Error(t, CloseBucket(nil, bucket, mockdb, nil, mockClock)) 77 | 78 | // Closing bucket with no artifacts successfully. Verify bucket state and dateclosed. 79 | bucket = &model.Bucket{State: model.OPEN, Id: bucket_id} 80 | mockdb.On("UpdateBucket", bucket).Return(nil).Once() 81 | mockdb.On("ListArtifactsInBucket", bucket.Id).Return([]model.Artifact{}, nil).Once() 82 | require.NoError(t, CloseBucket(nil, bucket, mockdb, nil, mockClock)) 83 | require.Equal(t, model.CLOSED, bucket.State) 84 | require.Equal(t, mockClock.Now(), bucket.DateClosed) 85 | 86 | // Closing bucket with no artifacts successfully. Verify bucket state and dateclosed. 87 | bucket = &model.Bucket{State: model.OPEN, Id: bucket_id} 88 | artifact := model.Artifact{Id: 20, State: model.UPLOADED} 89 | mockdb.On("UpdateBucket", bucket).Return(nil).Once() 90 | mockdb.On("ListArtifactsInBucket", bucket.Id).Return([]model.Artifact{artifact}, nil).Once() 91 | 92 | require.NoError(t, CloseBucket(nil, bucket, mockdb, nil, mockClock)) 93 | require.Equal(t, model.CLOSED, bucket.State) 94 | require.Equal(t, mockClock.Now(), bucket.DateClosed) 95 | 96 | mockdb.AssertExpectations(t) 97 | } 98 | -------------------------------------------------------------------------------- /client/testserver/testserver.go: -------------------------------------------------------------------------------- 1 | package testserver 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "sync" 7 | "testing" 8 | ) 9 | 10 | type request struct { 11 | method string 12 | url string 13 | responseCode int 14 | responseBytes string 15 | shouldHang bool 16 | } 17 | 18 | // TestServer wraps around httptest.Server to support expectations and timeout tests. 19 | // Use ExpectAndRespond and ExpectAndHang methods to indicate expected requests in the order they 20 | // should arrive. 21 | // 22 | // Currently, parallel requests are not supported. 23 | type TestServer struct { 24 | reqChain []request 25 | t *testing.T 26 | s *httptest.Server 27 | URL string 28 | waiter *sync.Cond 29 | reqChainLock sync.Mutex 30 | } 31 | 32 | // NewTestServer creates a new TestServer associate with given testing.T instance 33 | func NewTestServer(t *testing.T) *TestServer { 34 | var m sync.Mutex 35 | ts := &TestServer{t: t, waiter: sync.NewCond(&m)} 36 | ts.run() 37 | ts.URL = ts.s.URL 38 | return ts 39 | } 40 | 41 | // ExpectAndRespond specifies the next request to be expected (using request method and url) and 42 | // specifies what response needs to be provided. 43 | func (ts *TestServer) ExpectAndRespond(method string, url string, responseCode int, responseBytes string) *TestServer { 44 | return ts.insertNextReq(request{ 45 | method: method, 46 | url: url, 47 | responseCode: responseCode, 48 | responseBytes: responseBytes, 49 | shouldHang: false, 50 | }) 51 | } 52 | 53 | // ExpectAndHang specifies the next request expected, the server hangs and the request will not be 54 | // responded to. This is useful to test client timeouts. To stop the hanging server, call 55 | // CloseAndAssertExpectations. 56 | func (ts *TestServer) ExpectAndHang(method string, url string) *TestServer { 57 | return ts.insertNextReq(request{ 58 | method: method, 59 | url: url, 60 | shouldHang: true, 61 | }) 62 | } 63 | 64 | func (ts *TestServer) insertNextReq(nextReq request) *TestServer { 65 | defer ts.reqChainLock.Unlock() 66 | ts.reqChainLock.Lock() 67 | ts.reqChain = append(ts.reqChain, nextReq) 68 | 69 | return ts 70 | } 71 | 72 | func (ts *TestServer) popNextReq() (nextReq request, ok bool) { 73 | defer ts.reqChainLock.Unlock() 74 | ts.reqChainLock.Lock() 75 | 76 | if len(ts.reqChain) > 0 { 77 | nextReq, ts.reqChain = ts.reqChain[0], ts.reqChain[1:] 78 | ok = true 79 | } else { 80 | ok = false 81 | } 82 | 83 | return 84 | } 85 | 86 | func (ts *TestServer) run() { 87 | ts.s = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 88 | ts.t.Logf("Received request: %s %s\n", r.Method, r.URL) 89 | 90 | nextReq, ok := ts.popNextReq() 91 | 92 | if !ok { 93 | w.WriteHeader(http.StatusExpectationFailed) 94 | ts.t.Fatalf("Unexpected request %s %s\n", r.Method, r.URL) 95 | } 96 | 97 | if nextReq.method != r.Method || nextReq.url != r.URL.String() { 98 | w.WriteHeader(http.StatusExpectationFailed) 99 | ts.t.Fatalf("Expected request: %s %s\nGot request: %s %s", nextReq.method, nextReq.url, r.Method, r.URL) 100 | } 101 | 102 | if nextReq.shouldHang { 103 | ts.t.Log("Hanging on response") 104 | ts.waiter.L.Lock() 105 | ts.waiter.Wait() 106 | ts.waiter.L.Unlock() 107 | } else { 108 | ts.t.Logf("Responding with status %d\n", nextReq.responseCode) 109 | w.WriteHeader(nextReq.responseCode) 110 | w.Write([]byte(nextReq.responseBytes)) 111 | } 112 | })) 113 | 114 | ts.t.Logf("Running httptest server on %s\n", ts.s.URL) 115 | } 116 | 117 | // CloseAndAssertExpectations will stop any hanging requests and shutdown the test server. Any 118 | // remaining expectations will flag a test error. 119 | func (ts *TestServer) CloseAndAssertExpectations() { 120 | ts.reqChainLock.Lock() 121 | defer ts.reqChainLock.Unlock() 122 | if len(ts.reqChain) != 0 { 123 | ts.t.Fatalf("Some expected requests were never called, next one being %s %s", 124 | ts.reqChain[0].method, ts.reqChain[0].url) 125 | } 126 | 127 | ts.waiter.L.Lock() 128 | ts.waiter.Broadcast() 129 | ts.waiter.L.Unlock() 130 | ts.s.Close() 131 | } 132 | -------------------------------------------------------------------------------- /api/buckethandler.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | 8 | "golang.org/x/net/context" 9 | 10 | "github.com/dropbox/changes-artifacts/common" 11 | "github.com/dropbox/changes-artifacts/database" 12 | "github.com/dropbox/changes-artifacts/model" 13 | "github.com/martini-contrib/render" 14 | "gopkg.in/amz.v1/s3" 15 | ) 16 | 17 | type HttpError struct { 18 | errCode int 19 | errStr string 20 | } 21 | 22 | func (he *HttpError) Error() string { 23 | return he.errStr 24 | } 25 | 26 | func NewHttpError(code int, format string, args ...interface{}) *HttpError { 27 | if len(args) > 0 { 28 | return &HttpError{errCode: code, errStr: fmt.Sprintf(format, args...)} 29 | } 30 | return &HttpError{errCode: code, errStr: format} 31 | } 32 | 33 | func NewWrappedHttpError(code int, err error) *HttpError { 34 | return &HttpError{errCode: code, errStr: err.Error()} 35 | } 36 | 37 | // Ensure that HttpError implements error 38 | var _ error = new(HttpError) 39 | 40 | func ListBuckets(ctx context.Context, r render.Render, db database.Database) { 41 | if buckets, err := db.ListBuckets(); err != nil { 42 | LogAndRespondWithError(ctx, r, http.StatusBadRequest, err) 43 | } else { 44 | r.JSON(http.StatusOK, buckets) 45 | } 46 | } 47 | 48 | func CreateBucket(db database.Database, clk common.Clock, bucketId string, owner string) (*model.Bucket, *HttpError) { 49 | if bucketId == "" { 50 | return nil, NewHttpError(http.StatusBadRequest, "Bucket ID not provided") 51 | } 52 | 53 | if len(owner) == 0 { 54 | return nil, NewHttpError(http.StatusBadRequest, "Bucket Owner not provided") 55 | } 56 | 57 | _, err := db.GetBucket(bucketId) 58 | if err != nil && !err.EntityNotFound() { 59 | return nil, NewWrappedHttpError(http.StatusInternalServerError, err) 60 | } 61 | if err == nil { 62 | return nil, NewHttpError(http.StatusBadRequest, "Entity exists") 63 | } 64 | 65 | var bucket model.Bucket 66 | bucket.Id = bucketId 67 | bucket.DateCreated = clk.Now() 68 | bucket.State = model.OPEN 69 | bucket.Owner = owner 70 | if err := db.InsertBucket(&bucket); err != nil { 71 | return nil, NewWrappedHttpError(http.StatusBadRequest, err) 72 | } 73 | return &bucket, nil 74 | } 75 | 76 | func HandleCreateBucket(ctx context.Context, r render.Render, req *http.Request, db database.Database, clk common.Clock) { 77 | var createBucketReq struct { 78 | ID string 79 | Owner string 80 | } 81 | 82 | if err := json.NewDecoder(req.Body).Decode(&createBucketReq); err != nil { 83 | LogAndRespondWithErrorf(ctx, r, http.StatusBadRequest, "Malformed JSON request") 84 | return 85 | } 86 | 87 | if bucket, err := CreateBucket(db, clk, createBucketReq.ID, createBucketReq.Owner); err != nil { 88 | LogAndRespondWithError(ctx, r, err.errCode, err) 89 | } else { 90 | r.JSON(http.StatusOK, bucket) 91 | } 92 | } 93 | 94 | func HandleGetBucket(ctx context.Context, r render.Render, bucket *model.Bucket) { 95 | if bucket == nil { 96 | LogAndRespondWithErrorf(ctx, r, http.StatusBadRequest, "No bucket specified") 97 | return 98 | } 99 | 100 | r.JSON(http.StatusOK, bucket) 101 | } 102 | 103 | // HandleCloseBucket handles the HTTP request to close a bucket. See CloseBucket for details. 104 | func HandleCloseBucket(ctx context.Context, r render.Render, db database.Database, bucket *model.Bucket, s3Bucket *s3.Bucket, clk common.Clock) { 105 | if bucket == nil { 106 | LogAndRespondWithErrorf(ctx, r, http.StatusBadRequest, "No bucket specified") 107 | return 108 | } 109 | 110 | if err := CloseBucket(ctx, bucket, db, s3Bucket, clk); err != nil { 111 | LogAndRespondWithError(ctx, r, http.StatusBadRequest, err) 112 | } else { 113 | r.JSON(http.StatusOK, bucket) 114 | } 115 | return 116 | } 117 | 118 | // CloseBucket closes a bucket, preventing further updates. All artifacts associated with the bucket 119 | // are also marked closed. If the bucket is already closed, an error is returned. 120 | func CloseBucket(ctx context.Context, bucket *model.Bucket, db database.Database, s3Bucket *s3.Bucket, clk common.Clock) error { 121 | if bucket.State != model.OPEN { 122 | return fmt.Errorf("Bucket is already closed") 123 | } 124 | 125 | bucket.State = model.CLOSED 126 | bucket.DateClosed = clk.Now() 127 | if err := db.UpdateBucket(bucket); err != nil { 128 | return err 129 | } 130 | 131 | if artifacts, err := db.ListArtifactsInBucket(bucket.Id); err != nil { 132 | return err 133 | } else { 134 | for _, artifact := range artifacts { 135 | if err := CloseArtifact(ctx, &artifact, db, s3Bucket, false); err != nil { 136 | return err 137 | } 138 | } 139 | } 140 | 141 | return nil 142 | } 143 | -------------------------------------------------------------------------------- /api/logchunkreader.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "io" 7 | "os" 8 | "unicode/utf8" 9 | 10 | "github.com/dropbox/changes-artifacts/database" 11 | "github.com/dropbox/changes-artifacts/model" 12 | ) 13 | 14 | // Number of bytes to read ahead while making DB queries. 15 | const ReadaheadBytes = 1 * 1024 * 1024 16 | 17 | // logChunkReader presents an io.ReadSeeker interface to sequentially reading logchunks for an 18 | // artifact from the database. It stores any un-Read() bytes in an internal buffer to avoid reading 19 | // logchunks multiple times from the database. Also, an optional readahead size can be specified 20 | // while reading large byte ranges (or the entire artifact) to prepopulate the cache using lesser 21 | // number of DB queries (instead of many short queries, each of which has a non-trivial overhead on 22 | // the DB). 23 | type logChunkReader struct { 24 | artifact *model.Artifact 25 | db database.Database 26 | offset int64 27 | readAhead int64 28 | unread bytes.Buffer // Readahead buffer of bytes which were read from DB but not yet read by client. 29 | // This can be replaced with a global cache because logchunks are immutable to reduce even more 30 | // hits to the DB. 31 | } 32 | 33 | func newLogChunkReader(artifact *model.Artifact, db database.Database) *logChunkReader { 34 | return &logChunkReader{artifact: artifact, db: db} 35 | } 36 | 37 | func newLogChunkReaderWithReadahead(artifact *model.Artifact, db database.Database) *logChunkReader { 38 | return &logChunkReader{artifact: artifact, db: db, readAhead: min(artifact.Size, ReadaheadBytes)} 39 | } 40 | 41 | func (lcr *logChunkReader) Read(p []byte) (int, error) { 42 | offsetInSlice := int64(0) 43 | if len(p) == 0 { 44 | return 0, nil 45 | } 46 | 47 | if lcr.unread.Len() > 0 { 48 | n, err := lcr.unread.Read(p) 49 | offsetInSlice += int64(n) 50 | lcr.offset += int64(n) 51 | 52 | if err != nil { 53 | return n, err 54 | } else { 55 | // If we've already filled buffer, return 56 | if n == len(p) { 57 | if lcr.offset == lcr.artifact.Size { 58 | return n, io.EOF 59 | } 60 | return n, nil 61 | } 62 | } 63 | } 64 | 65 | if lcr.offset < 0 || lcr.offset >= lcr.artifact.Size { 66 | return int(offsetInSlice), io.EOF 67 | } 68 | 69 | bytesToRead := min(lcr.artifact.Size-lcr.offset, int64(len(p))-offsetInSlice) 70 | if bytesToRead+offsetInSlice != int64(len(p)) { 71 | p = p[0 : bytesToRead+offsetInSlice] 72 | } 73 | 74 | chunks, err := lcr.db.ListLogChunksInArtifact(lcr.artifact.Id, lcr.offset, min(lcr.artifact.Size, lcr.offset+max(bytesToRead, lcr.readAhead))) 75 | // TODO: Should we retry? This appears to be the best place to do so. 76 | if err != nil { 77 | return int(offsetInSlice), err 78 | } 79 | 80 | for _, chunk := range chunks { 81 | if bytesToRead == 0 { 82 | lcr.unread.Write(chunk.ContentBytes) 83 | } 84 | if chunk.ByteOffset < lcr.offset+bytesToRead && chunk.ByteOffset+chunk.Size > lcr.offset { 85 | startByte := lcr.offset - chunk.ByteOffset 86 | bytesToReadFromChunk := min(bytesToRead, chunk.Size-startByte) 87 | 88 | copy(p[offsetInSlice:], chunk.ContentBytes[startByte:startByte+bytesToReadFromChunk]) 89 | lcr.offset += bytesToReadFromChunk 90 | offsetInSlice += bytesToReadFromChunk 91 | bytesToRead -= bytesToReadFromChunk 92 | 93 | if bytesToReadFromChunk < chunk.Size-startByte { 94 | lcr.unread.Write(chunk.ContentBytes[startByte+bytesToReadFromChunk:]) 95 | } 96 | } 97 | } 98 | 99 | if lcr.offset == lcr.artifact.Size { 100 | return len(p), io.EOF 101 | } 102 | return len(p), nil 103 | } 104 | 105 | var errInvalidSeek = errors.New("Invalid seek target") 106 | 107 | func (lcr *logChunkReader) setOffset(offset int64) (int64, error) { 108 | if offset >= 0 && offset <= lcr.artifact.Size { 109 | delta := int(offset - lcr.offset) 110 | if delta >= 0 && delta < lcr.unread.Len() { 111 | lcr.unread.Next(delta) 112 | } else { 113 | lcr.unread.Reset() 114 | } 115 | lcr.offset = offset 116 | return lcr.offset, nil 117 | } 118 | 119 | return lcr.offset, errInvalidSeek 120 | } 121 | 122 | func (lcr *logChunkReader) Seek(offset int64, whence int) (int64, error) { 123 | if whence == os.SEEK_SET { 124 | // Seek WRT start of file 125 | return lcr.setOffset(offset) 126 | } 127 | 128 | if whence == os.SEEK_CUR { 129 | // Seek WRT current offset 130 | return lcr.setOffset(lcr.offset + offset) 131 | } 132 | 133 | return lcr.setOffset(lcr.artifact.Size + offset) 134 | } 135 | 136 | var _ io.ReadSeeker = (*logChunkReader)(nil) 137 | 138 | var errInvalidStartingRune = errors.New("Read() started in the middle of a rune") 139 | 140 | // Read valid UTF-8 content from provided io.Reader. 141 | // If underlying reader starts in the middle of a rune, an error is returned. 142 | // If reader ends in the middle of a rune, the last (invalid) rune is discarded. Note that the 143 | // underlying reader will now start reading from the middle of a rune. 144 | func runeLimitedRead(r io.Reader, p []byte) (int, error) { 145 | n, err := r.Read(p) 146 | if n == 0 { 147 | return n, err 148 | } 149 | 150 | // If first byte is not a valid rune starting byte, returned error 151 | if n > 0 && !utf8.RuneStart(p[0]) { 152 | return 0, errInvalidStartingRune 153 | } 154 | 155 | // The following code is a lightly modified version of utf8#Valid() 156 | for i := 0; i < n; { 157 | if p[i] < utf8.RuneSelf { 158 | // Skip single byte rune 159 | i++ 160 | continue 161 | } 162 | 163 | r, size := utf8.DecodeRune(p[i:]) 164 | if size == 1 && r == utf8.RuneError { 165 | return i, err 166 | } 167 | i += size 168 | } 169 | 170 | return n, err 171 | } 172 | -------------------------------------------------------------------------------- /database/mock_Database.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import "github.com/stretchr/testify/mock" 4 | 5 | import "github.com/dropbox/changes-artifacts/model" 6 | import _ "github.com/vektra/mockery" 7 | 8 | type MockDatabase struct { 9 | mock.Mock 10 | } 11 | 12 | func (_m *MockDatabase) RegisterEntities() { 13 | _m.Called() 14 | } 15 | func (_m *MockDatabase) InsertBucket(_a0 *model.Bucket) *DatabaseError { 16 | ret := _m.Called(_a0) 17 | 18 | var r0 *DatabaseError 19 | if rf, ok := ret.Get(0).(func(*model.Bucket) *DatabaseError); ok { 20 | r0 = rf(_a0) 21 | } else { 22 | if ret.Get(0) != nil { 23 | r0 = ret.Get(0).(*DatabaseError) 24 | } 25 | } 26 | 27 | return r0 28 | } 29 | func (_m *MockDatabase) InsertArtifact(_a0 *model.Artifact) *DatabaseError { 30 | ret := _m.Called(_a0) 31 | 32 | var r0 *DatabaseError 33 | if rf, ok := ret.Get(0).(func(*model.Artifact) *DatabaseError); ok { 34 | r0 = rf(_a0) 35 | } else { 36 | if ret.Get(0) != nil { 37 | r0 = ret.Get(0).(*DatabaseError) 38 | } 39 | } 40 | 41 | return r0 42 | } 43 | func (_m *MockDatabase) InsertLogChunk(_a0 *model.LogChunk) *DatabaseError { 44 | ret := _m.Called(_a0) 45 | 46 | var r0 *DatabaseError 47 | if rf, ok := ret.Get(0).(func(*model.LogChunk) *DatabaseError); ok { 48 | r0 = rf(_a0) 49 | } else { 50 | if ret.Get(0) != nil { 51 | r0 = ret.Get(0).(*DatabaseError) 52 | } 53 | } 54 | 55 | return r0 56 | } 57 | func (_m *MockDatabase) UpdateBucket(_a0 *model.Bucket) *DatabaseError { 58 | ret := _m.Called(_a0) 59 | 60 | var r0 *DatabaseError 61 | if rf, ok := ret.Get(0).(func(*model.Bucket) *DatabaseError); ok { 62 | r0 = rf(_a0) 63 | } else { 64 | if ret.Get(0) != nil { 65 | r0 = ret.Get(0).(*DatabaseError) 66 | } 67 | } 68 | 69 | return r0 70 | } 71 | func (_m *MockDatabase) ListBuckets() ([]model.Bucket, *DatabaseError) { 72 | ret := _m.Called() 73 | 74 | var r0 []model.Bucket 75 | if rf, ok := ret.Get(0).(func() []model.Bucket); ok { 76 | r0 = rf() 77 | } else { 78 | if ret.Get(0) != nil { 79 | r0 = ret.Get(0).([]model.Bucket) 80 | } 81 | } 82 | 83 | var r1 *DatabaseError 84 | if rf, ok := ret.Get(1).(func() *DatabaseError); ok { 85 | r1 = rf() 86 | } else { 87 | if ret.Get(1) != nil { 88 | r1 = ret.Get(1).(*DatabaseError) 89 | } 90 | } 91 | 92 | return r0, r1 93 | } 94 | func (_m *MockDatabase) GetBucket(_a0 string) (*model.Bucket, *DatabaseError) { 95 | ret := _m.Called(_a0) 96 | 97 | var r0 *model.Bucket 98 | if rf, ok := ret.Get(0).(func(string) *model.Bucket); ok { 99 | r0 = rf(_a0) 100 | } else { 101 | if ret.Get(0) != nil { 102 | r0 = ret.Get(0).(*model.Bucket) 103 | } 104 | } 105 | 106 | var r1 *DatabaseError 107 | if rf, ok := ret.Get(1).(func(string) *DatabaseError); ok { 108 | r1 = rf(_a0) 109 | } else { 110 | if ret.Get(1) != nil { 111 | r1 = ret.Get(1).(*DatabaseError) 112 | } 113 | } 114 | 115 | return r0, r1 116 | } 117 | func (_m *MockDatabase) ListArtifactsInBucket(_a0 string) ([]model.Artifact, *DatabaseError) { 118 | ret := _m.Called(_a0) 119 | 120 | var r0 []model.Artifact 121 | if rf, ok := ret.Get(0).(func(string) []model.Artifact); ok { 122 | r0 = rf(_a0) 123 | } else { 124 | if ret.Get(0) != nil { 125 | r0 = ret.Get(0).([]model.Artifact) 126 | } 127 | } 128 | 129 | var r1 *DatabaseError 130 | if rf, ok := ret.Get(1).(func(string) *DatabaseError); ok { 131 | r1 = rf(_a0) 132 | } else { 133 | if ret.Get(1) != nil { 134 | r1 = ret.Get(1).(*DatabaseError) 135 | } 136 | } 137 | 138 | return r0, r1 139 | } 140 | func (_m *MockDatabase) UpdateArtifact(_a0 *model.Artifact) *DatabaseError { 141 | ret := _m.Called(_a0) 142 | 143 | var r0 *DatabaseError 144 | if rf, ok := ret.Get(0).(func(*model.Artifact) *DatabaseError); ok { 145 | r0 = rf(_a0) 146 | } else { 147 | if ret.Get(0) != nil { 148 | r0 = ret.Get(0).(*DatabaseError) 149 | } 150 | } 151 | 152 | return r0 153 | } 154 | func (_m *MockDatabase) ListLogChunksInArtifact(_a0 int64, _a1 int64, _a2 int64) ([]model.LogChunk, *DatabaseError) { 155 | ret := _m.Called(_a0, _a1, _a2) 156 | 157 | var r0 []model.LogChunk 158 | if rf, ok := ret.Get(0).(func(int64, int64, int64) []model.LogChunk); ok { 159 | r0 = rf(_a0, _a1, _a2) 160 | } else { 161 | if ret.Get(0) != nil { 162 | r0 = ret.Get(0).([]model.LogChunk) 163 | } 164 | } 165 | 166 | var r1 *DatabaseError 167 | if rf, ok := ret.Get(1).(func(int64, int64, int64) *DatabaseError); ok { 168 | r1 = rf(_a0, _a1, _a2) 169 | } else { 170 | if ret.Get(1) != nil { 171 | r1 = ret.Get(1).(*DatabaseError) 172 | } 173 | } 174 | 175 | return r0, r1 176 | } 177 | func (_m *MockDatabase) DeleteLogChunksForArtifact(_a0 int64) (int64, *DatabaseError) { 178 | ret := _m.Called(_a0) 179 | 180 | var r0 int64 181 | if rf, ok := ret.Get(0).(func(int64) int64); ok { 182 | r0 = rf(_a0) 183 | } else { 184 | r0 = ret.Get(0).(int64) 185 | } 186 | 187 | var r1 *DatabaseError 188 | if rf, ok := ret.Get(1).(func(int64) *DatabaseError); ok { 189 | r1 = rf(_a0) 190 | } else { 191 | if ret.Get(1) != nil { 192 | r1 = ret.Get(1).(*DatabaseError) 193 | } 194 | } 195 | 196 | return r0, r1 197 | } 198 | func (_m *MockDatabase) GetArtifactByName(bucket string, name string) (*model.Artifact, *DatabaseError) { 199 | ret := _m.Called(bucket, name) 200 | 201 | var r0 *model.Artifact 202 | if rf, ok := ret.Get(0).(func(string, string) *model.Artifact); ok { 203 | r0 = rf(bucket, name) 204 | } else { 205 | if ret.Get(0) != nil { 206 | r0 = ret.Get(0).(*model.Artifact) 207 | } 208 | } 209 | 210 | var r1 *DatabaseError 211 | if rf, ok := ret.Get(1).(func(string, string) *DatabaseError); ok { 212 | r1 = rf(bucket, name) 213 | } else { 214 | if ret.Get(1) != nil { 215 | r1 = ret.Get(1).(*DatabaseError) 216 | } 217 | } 218 | 219 | return r0, r1 220 | } 221 | func (_m *MockDatabase) GetLastLogChunkSeenForArtifact(_a0 int64) (*model.LogChunk, *DatabaseError) { 222 | ret := _m.Called(_a0) 223 | 224 | var r0 *model.LogChunk 225 | if rf, ok := ret.Get(0).(func(int64) *model.LogChunk); ok { 226 | r0 = rf(_a0) 227 | } else { 228 | if ret.Get(0) != nil { 229 | r0 = ret.Get(0).(*model.LogChunk) 230 | } 231 | } 232 | 233 | var r1 *DatabaseError 234 | if rf, ok := ret.Get(1).(func(int64) *DatabaseError); ok { 235 | r1 = rf(_a0) 236 | } else { 237 | if ret.Get(1) != nil { 238 | r1 = ret.Get(1).(*DatabaseError) 239 | } 240 | } 241 | 242 | return r0, r1 243 | } 244 | -------------------------------------------------------------------------------- /database/gorp_database.go: -------------------------------------------------------------------------------- 1 | package database 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/dropbox/changes-artifacts/common/stats" 7 | "github.com/dropbox/changes-artifacts/model" 8 | "gopkg.in/gorp.v1" 9 | ) 10 | 11 | type GorpDatabase struct { 12 | dbmap *gorp.DbMap 13 | } 14 | 15 | func NewGorpDatabase(dbmap *gorp.DbMap) *GorpDatabase { 16 | return &GorpDatabase{dbmap: dbmap} 17 | } 18 | 19 | func verifyBucketFields(bucket *model.Bucket) *DatabaseError { 20 | if len(bucket.Id) == 0 { 21 | return NewValidationError("Bucket.ID not set") 22 | } 23 | 24 | if bucket.State != model.OPEN && bucket.State != model.CLOSED && bucket.State != model.TIMEDOUT { 25 | return NewValidationError("Bucket in unknown state") 26 | } 27 | 28 | if len(bucket.Owner) == 0 { 29 | return NewValidationError("Bucket owner not set") 30 | } 31 | 32 | return nil 33 | } 34 | 35 | func (db *GorpDatabase) RegisterEntities() { 36 | // Add bucket non-autoincrementing ID field. 37 | db.dbmap.AddTableWithName(model.Bucket{}, "bucket").SetKeys(false, "Id") 38 | 39 | // Add artifact autoincrementing ID field. 40 | db.dbmap.AddTableWithName(model.Artifact{}, "artifact"). 41 | SetKeys(true, "Id"). 42 | SetUniqueTogether("BucketID", "Name") 43 | 44 | // Add logchunk autoincrementing ID field. 45 | db.dbmap.AddTableWithName(model.LogChunk{}, "logchunk").SetKeys(true, "Id") 46 | } 47 | 48 | var insertBucketTimer = stats.NewTimingStat("insert_bucket") 49 | 50 | func (db *GorpDatabase) InsertBucket(bucket *model.Bucket) *DatabaseError { 51 | defer insertBucketTimer.AddTimeSince(time.Now()) 52 | 53 | if err := verifyBucketFields(bucket); err != nil { 54 | return err 55 | } 56 | 57 | return WrapInternalDatabaseError(db.dbmap.Insert(bucket)) 58 | } 59 | 60 | var insertArtifactTimer = stats.NewTimingStat("insert_artifact") 61 | 62 | func (db *GorpDatabase) InsertArtifact(artifact *model.Artifact) *DatabaseError { 63 | defer insertArtifactTimer.AddTimeSince(time.Now()) 64 | return WrapInternalDatabaseError(db.dbmap.Insert(artifact)) 65 | } 66 | 67 | var insertLogChunkTimer = stats.NewTimingStat("insert_logchunk") 68 | 69 | func (db *GorpDatabase) InsertLogChunk(logChunk *model.LogChunk) *DatabaseError { 70 | defer insertLogChunkTimer.AddTimeSince(time.Now()) 71 | return WrapInternalDatabaseError(db.dbmap.Insert(logChunk)) 72 | } 73 | 74 | var updateBucketTimer = stats.NewTimingStat("update_bucket") 75 | 76 | func (db *GorpDatabase) UpdateBucket(bucket *model.Bucket) *DatabaseError { 77 | defer updateBucketTimer.AddTimeSince(time.Now()) 78 | if err := verifyBucketFields(bucket); err != nil { 79 | return err 80 | } 81 | 82 | _, err := db.dbmap.Update(bucket) 83 | return WrapInternalDatabaseError(err) 84 | } 85 | 86 | func (db *GorpDatabase) ListBuckets() ([]model.Bucket, *DatabaseError) { 87 | buckets := []model.Bucket{} 88 | // NOTE: Hardcoded limit of 25 buckets below. 89 | // Because of the large number of buckets (2.5M+ and increasing), its not feasible to list all 90 | // buckets at /buckets. Instead, we show only the latest 25 buckets. This endpoint is not 91 | // particularly useful and is not used by any client. 92 | if _, err := db.dbmap.Select(&buckets, "SELECT * FROM bucket ORDER BY datecreated LIMIT 25"); err != nil { 93 | return nil, WrapInternalDatabaseError(err) 94 | } 95 | 96 | return buckets, nil 97 | } 98 | 99 | var getBucketTimer = stats.NewTimingStat("get_bucket") 100 | 101 | func (db *GorpDatabase) GetBucket(id string) (*model.Bucket, *DatabaseError) { 102 | defer getBucketTimer.AddTimeSince(time.Now()) 103 | if bucket, err := db.dbmap.Get(model.Bucket{}, id); err != nil && !gorp.NonFatalError(err) { 104 | return nil, WrapInternalDatabaseError(err) 105 | } else if bucket == nil { 106 | return nil, NewEntityNotFoundError("Entity %s not found", id) 107 | } else { 108 | return bucket.(*model.Bucket), nil 109 | } 110 | } 111 | 112 | var listArtifactsTimer = stats.NewTimingStat("list_artifacts") 113 | 114 | func (db *GorpDatabase) ListArtifactsInBucket(bucketId string) ([]model.Artifact, *DatabaseError) { 115 | defer listArtifactsTimer.AddTimeSince(time.Now()) 116 | artifacts := []model.Artifact{} 117 | if _, err := db.dbmap.Select(&artifacts, "SELECT * FROM artifact WHERE bucketid = :bucketid", 118 | map[string]interface{}{"bucketid": bucketId}); err != nil && !gorp.NonFatalError(err) { 119 | return nil, WrapInternalDatabaseError(err) 120 | } 121 | 122 | return artifacts, nil 123 | } 124 | 125 | func (db *GorpDatabase) UpdateArtifact(artifact *model.Artifact) *DatabaseError { 126 | _, err := db.dbmap.Update(artifact) 127 | if !gorp.NonFatalError(err) { 128 | return WrapInternalDatabaseError(err) 129 | } 130 | 131 | return nil 132 | } 133 | 134 | var listLogChunksTimer = stats.NewTimingStat("list_logchunks") 135 | 136 | func (db *GorpDatabase) ListLogChunksInArtifact(artifactID int64, byteBegin int64, byteEnd int64) ([]model.LogChunk, *DatabaseError) { 137 | defer listLogChunksTimer.AddTimeSince(time.Now()) 138 | logChunks := []model.LogChunk{} 139 | if _, err := db.dbmap.Select(&logChunks, 140 | `SELECT * FROM logchunk 141 | WHERE artifactid = :artifactid AND byteoffset < :limit AND size + byteoffset >= :offset 142 | ORDER BY byteoffset ASC`, 143 | map[string]interface{}{"artifactid": artifactID, "offset": byteBegin, "limit": byteEnd}); err != nil && !gorp.NonFatalError(err) { 144 | return nil, WrapInternalDatabaseError(err) 145 | } 146 | 147 | return logChunks, nil 148 | } 149 | 150 | var deleteLogChunksTimer = stats.NewTimingStat("delete_logchunks") 151 | 152 | // DeleteLogChunksForArtifact deletes all log chunks for an artifact. 153 | // Returns (number of deleted rows, err) 154 | func (db *GorpDatabase) DeleteLogChunksForArtifact(artifactID int64) (int64, *DatabaseError) { 155 | defer deleteLogChunksTimer.AddTimeSince(time.Now()) 156 | res, err := db.dbmap.Exec("DELETE FROM logchunk WHERE artifactid = $1", artifactID) 157 | if err != nil && !gorp.NonFatalError(err) { 158 | rows, _ := res.RowsAffected() 159 | return rows, WrapInternalDatabaseError(err) 160 | } 161 | 162 | rows, err := res.RowsAffected() 163 | if err != nil && !gorp.NonFatalError(err) { 164 | return rows, WrapInternalDatabaseError(err) 165 | } 166 | 167 | return rows, nil 168 | } 169 | 170 | var getArtifactTimer = stats.NewTimingStat("get_artifact") 171 | 172 | func (db *GorpDatabase) GetArtifactByName(bucketId string, artifactName string) (*model.Artifact, *DatabaseError) { 173 | defer getArtifactTimer.AddTimeSince(time.Now()) 174 | var artifact model.Artifact 175 | if err := db.dbmap.SelectOne(&artifact, "SELECT * FROM artifact WHERE bucketid = :bucketid AND name = :artifactname", 176 | map[string]string{"bucketid": bucketId, "artifactname": artifactName}); err != nil && !gorp.NonFatalError(err) { 177 | return nil, WrapInternalDatabaseError(err) 178 | } 179 | 180 | return &artifact, nil 181 | } 182 | 183 | var getLastLogChunkTimer = stats.NewTimingStat("get_last_logchunk") 184 | 185 | // GetLastLogChunkSeenForArtifact returns the last full logchunk present in the database associated 186 | // with artifact. 187 | func (db *GorpDatabase) GetLastLogChunkSeenForArtifact(artifactID int64) (*model.LogChunk, *DatabaseError) { 188 | defer getLastLogChunkTimer.AddTimeSince(time.Now()) 189 | var logChunk model.LogChunk 190 | if err := db.dbmap.SelectOne(&logChunk, "SELECT * FROM logchunk WHERE artifactid = :artifactid ORDER BY byteoffset DESC LIMIT 1", 191 | map[string]interface{}{"artifactid": artifactID}); err != nil && !gorp.NonFatalError(err) { 192 | return nil, WrapInternalDatabaseError(err) 193 | } 194 | return &logChunk, nil 195 | } 196 | 197 | // Ensure GorpDatabase implements Database 198 | var _ Database = new(GorpDatabase) 199 | -------------------------------------------------------------------------------- /api/logchunkreader_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "net/http" 7 | "os" 8 | "testing" 9 | 10 | "github.com/dropbox/changes-artifacts/database" 11 | "github.com/dropbox/changes-artifacts/model" 12 | ) 13 | 14 | func makeChunks(offset int, chunks ...string) []model.LogChunk { 15 | logChunks := make([]model.LogChunk, len(chunks)) 16 | 17 | for i, chunk := range chunks { 18 | logChunks[i] = model.LogChunk{ByteOffset: int64(offset), Size: int64(len(chunk)), ContentBytes: []byte(chunk)} 19 | offset += len(chunk) 20 | } 21 | 22 | return logChunks 23 | } 24 | 25 | func TestLogChunkReaderSeek(t *testing.T) { 26 | type testcase struct { 27 | offset int 28 | whence int 29 | expectError bool 30 | expectedOffset int 31 | } 32 | 33 | cases := []testcase{ 34 | {offset: 1, whence: os.SEEK_SET, expectError: false, expectedOffset: 1}, 35 | {offset: 1025, whence: os.SEEK_CUR, expectError: false, expectedOffset: 1025}, 36 | {offset: 5, whence: os.SEEK_CUR, expectError: false, expectedOffset: 5}, 37 | {offset: -10, whence: os.SEEK_END, expectError: false, expectedOffset: 1015}, 38 | {offset: -1, whence: os.SEEK_SET, expectError: true, expectedOffset: 0}, 39 | {offset: 1026, whence: os.SEEK_CUR, expectError: true, expectedOffset: 0}, 40 | {offset: 1, whence: os.SEEK_END, expectError: true, expectedOffset: 0}, 41 | } 42 | 43 | for _, c := range cases { 44 | lcr := newLogChunkReader(&model.Artifact{Size: 1025}, nil) 45 | 46 | n, err := lcr.Seek(int64(c.offset), c.whence) 47 | if n != int64(c.expectedOffset) { 48 | t.Errorf("After Seek(%d, %d), expected offset: %d, actual offset: %d", c.offset, c.whence, c.expectedOffset, n) 49 | } 50 | if c.expectError { 51 | if err == nil { 52 | t.Errorf("Expected error during Seek(%d, %d), none received", c.offset, c.whence) 53 | } 54 | } else if err != nil { 55 | t.Errorf("Unexpected error during Seek(%d, %d): %s", c.offset, c.whence, err) 56 | } 57 | } 58 | } 59 | 60 | func TestLogChunkReaderRead(t *testing.T) { 61 | type mockDBCall struct { 62 | offset int 63 | limit int 64 | logChunks []model.LogChunk 65 | } 66 | 67 | type unit struct { 68 | readBufLen int 69 | expectEOF bool 70 | expectError bool 71 | expectBytes string 72 | 73 | expectDBCall *mockDBCall 74 | } 75 | 76 | type testcase struct { 77 | artifactSize int64 78 | units []unit 79 | } 80 | 81 | cases := []testcase{ 82 | { 83 | artifactSize: 5, 84 | units: []unit{ 85 | {readBufLen: 0, expectError: false}, // Empty p 86 | {readBufLen: 1, expectBytes: "0", expectError: false, expectDBCall: &mockDBCall{offset: 0, limit: 5, logChunks: makeChunks(0, "01234")}}, 87 | {readBufLen: 3, expectBytes: "123", expectError: false}, // Read from cache 88 | {readBufLen: 2, expectBytes: "4", expectEOF: true}, // Read from cache and truncate p 89 | {readBufLen: 1, expectEOF: true}, // Read at EOF 90 | }, 91 | }, 92 | { 93 | artifactSize: 5, 94 | units: []unit{ 95 | {readBufLen: 5, expectBytes: "01234", expectEOF: true, expectDBCall: &mockDBCall{offset: 0, limit: 5, logChunks: makeChunks(0, "01234")}}, 96 | }, 97 | }, 98 | { 99 | artifactSize: 5, 100 | units: []unit{ 101 | {readBufLen: 3, expectBytes: "012", expectError: false, expectDBCall: &mockDBCall{offset: 0, limit: 5, logChunks: makeChunks(0, "01234")}}, 102 | {readBufLen: 2, expectBytes: "34", expectEOF: true}, // Read from cache till EOF 103 | }, 104 | }, 105 | { 106 | artifactSize: 5, 107 | units: []unit{ 108 | {readBufLen: 2, expectBytes: "01", expectError: false, expectDBCall: &mockDBCall{offset: 0, limit: 5, logChunks: makeChunks(0, "012")}}, 109 | {readBufLen: 4, expectBytes: "234", expectEOF: true, expectDBCall: &mockDBCall{offset: 3, limit: 5, logChunks: makeChunks(3, "34")}}, 110 | }, 111 | }, 112 | { 113 | artifactSize: 5, 114 | units: []unit{ 115 | {readBufLen: 2, expectBytes: "01", expectError: false, expectDBCall: &mockDBCall{offset: 0, limit: 5, logChunks: makeChunks(0, "01", "23")}}, 116 | {readBufLen: 4, expectBytes: "234", expectEOF: true, expectDBCall: &mockDBCall{offset: 4, limit: 5, logChunks: makeChunks(4, "4")}}, 117 | }, 118 | }, 119 | } 120 | 121 | for _, c := range cases { 122 | mockdb := &database.MockDatabase{} 123 | lcr := newLogChunkReaderWithReadahead(&model.Artifact{Size: c.artifactSize, Id: 12345}, mockdb) 124 | for _, unit := range c.units { 125 | if unit.expectDBCall != nil { 126 | mockdb.On("ListLogChunksInArtifact", int64(12345), int64(unit.expectDBCall.offset), int64(unit.expectDBCall.limit)).Return(unit.expectDBCall.logChunks, nil).Once() 127 | } 128 | 129 | p := make([]byte, unit.readBufLen) 130 | 131 | n, err := lcr.Read(p) 132 | if n != len(unit.expectBytes) { 133 | t.Errorf("Mismatch in number of bytes read during Read(%d): Expected: %d, Actual: %d", unit.readBufLen, len(unit.expectBytes), n) 134 | } 135 | if !bytes.Equal([]byte(unit.expectBytes), p[:n]) { 136 | t.Errorf("Mismatch in content read during Read(%d): Expected: '%s', Actual: '%s'", unit.readBufLen, unit.expectBytes, p[:n]) 137 | } 138 | 139 | if unit.expectEOF { 140 | if err != io.EOF { 141 | t.Errorf("Expected EOF during Read(%d), none received", unit.readBufLen) 142 | } 143 | } else if unit.expectError { 144 | if err != nil { 145 | t.Errorf("Expected error during Read(%d), none received", unit.readBufLen) 146 | } 147 | } else if err != nil { 148 | t.Errorf("Unexpected error during Read(%d): %s", unit.readBufLen, err) 149 | } 150 | 151 | mockdb.AssertExpectations(t) 152 | } 153 | } 154 | } 155 | 156 | func TestLogChunkReaderCacheInvalidation(t *testing.T) { 157 | mockdb := &database.MockDatabase{} 158 | lcr := newLogChunkReader(&model.Artifact{Id: 123, Size: 11}, mockdb) 159 | 160 | mockdb.On("ListLogChunksInArtifact", int64(123), int64(0), int64(7)).Return(makeChunks(0, "012345", "67891"), nil).Once() 161 | bts := make([]byte, 7) 162 | lcr.Read(bts) 163 | mockdb.AssertExpectations(t) 164 | 165 | // No-op seek 166 | lcr.Seek(0, os.SEEK_CUR) 167 | if lcr.unread.Len() != 4 { 168 | t.Errorf("Cache expected to be 4 bytes, but is of size %d bytes", lcr.unread.Len()) 169 | } 170 | 171 | // Seek 1 byte ahead, advance cache 172 | lcr.Seek(1, os.SEEK_CUR) 173 | if lcr.unread.Len() != 3 { 174 | t.Errorf("Cache expected to be 3 bytes, but is of size %d bytes", lcr.unread.Len()) 175 | } 176 | 177 | // Seek 1 byte behind, invalidate cache 178 | lcr.Seek(-1, os.SEEK_CUR) 179 | if lcr.unread.Len() != 0 { 180 | t.Errorf("Cache expected to be empty, but is of size %d bytes", lcr.unread.Len()) 181 | } 182 | 183 | lcr.Seek(0, os.SEEK_SET) 184 | mockdb.On("ListLogChunksInArtifact", int64(123), int64(0), int64(7)).Return(makeChunks(0, "012345", "678"), nil).Once() 185 | lcr.Read(bts) 186 | mockdb.AssertExpectations(t) 187 | 188 | // Seek beyond end of cache, invalidate cache 189 | lcr.Seek(3, os.SEEK_CUR) 190 | if lcr.unread.Len() != 0 { 191 | t.Errorf("Cache expected to be empty, but is of size %d bytes", lcr.unread.Len()) 192 | } 193 | } 194 | 195 | func TestGetByteRangeFromRequest(t *testing.T) { 196 | type testcase struct { 197 | query string 198 | artifactSize int 199 | expectedBegin int 200 | expectedEnd int 201 | expectError bool 202 | } 203 | cases := []testcase{ 204 | {query: "/chunked", artifactSize: 0, expectedBegin: 0, expectedEnd: 0, expectError: true}, 205 | {query: "/chunked", artifactSize: 100, expectedBegin: 0, expectedEnd: 99, expectError: false}, 206 | {query: "/chunked?limit=0", artifactSize: 100, expectedBegin: 0, expectedEnd: 99, expectError: false}, 207 | {query: "/chunked?limit=5", artifactSize: 100, expectedBegin: 0, expectedEnd: 4, expectError: false}, 208 | {query: "/chunked?offset=0", artifactSize: 100, expectedBegin: 0, expectedEnd: 99, expectError: false}, 209 | {query: "/chunked?offset=0&limit=0", artifactSize: 100, expectedBegin: 0, expectedEnd: 99, expectError: false}, 210 | {query: "/chunked?offset=0&limit=11", artifactSize: 100, expectedBegin: 0, expectedEnd: 10, expectError: false}, 211 | {query: "/chunked?offset=4&limit=0", artifactSize: 100, expectedBegin: 4, expectedEnd: 99, expectError: false}, 212 | {query: "/chunked?offset=4&limit=20", artifactSize: 100, expectedBegin: 4, expectedEnd: 23, expectError: false}, 213 | {query: "/chunked?offset=85&limit=20", artifactSize: 100, expectedBegin: 85, expectedEnd: 99, expectError: false}, 214 | {query: "/chunked?offset=85&limit=20", artifactSize: 30, expectedBegin: 0, expectedEnd: 30, expectError: true}, 215 | {query: "/chunked?offset=-1&limit=20", artifactSize: 100, expectedBegin: 0, expectedEnd: 19, expectError: false}, 216 | {query: "/chunked?offset=0&limit=-1", artifactSize: 100, expectedBegin: 0, expectedEnd: 99, expectError: false}, 217 | {query: "/chunked?offset=99", artifactSize: 100, expectedBegin: 99, expectedEnd: 99, expectError: false}, 218 | {query: "/chunked", artifactSize: 2000000, expectedBegin: 0, expectedEnd: 999999, expectError: false}, 219 | {query: "/chunked?limit=2000000", artifactSize: 2000000, expectedBegin: 0, expectedEnd: 999999, expectError: false}, 220 | } 221 | 222 | for _, c := range cases { 223 | artifact := &model.Artifact{Size: int64(c.artifactSize)} 224 | req, _ := http.NewRequest("GET", c.query, nil) 225 | 226 | begin, end, err := getByteRangeFromRequest(req, artifact) 227 | if int(begin) != c.expectedBegin || int(end) != c.expectedEnd { 228 | t.Errorf("Byte range mismatch for request %s => Expected: Begin=%d End=%d, Actual: Begin=%d End=%d", c.query, c.expectedBegin, c.expectedEnd, begin, end) 229 | } 230 | 231 | if c.expectError && err == nil { 232 | t.Errorf("Expected error for request %s, but got none", c.query) 233 | } 234 | 235 | if !c.expectError && err != nil { 236 | t.Errorf("Expected no error for request %s, but got %s", c.query, err) 237 | } 238 | } 239 | } 240 | -------------------------------------------------------------------------------- /server.go: -------------------------------------------------------------------------------- 1 | // Main entrypoint for Artifact Server. 2 | // 3 | // This is a REST API (no html frontend) implemented using Martini. 4 | // 5 | // Database: Postgresql 6 | // Storage: S3 7 | package main 8 | 9 | import ( 10 | "database/sql" 11 | "encoding/json" 12 | "flag" 13 | "fmt" 14 | "io/ioutil" 15 | "log" 16 | "math/rand" 17 | "net/http" 18 | _ "net/http/pprof" 19 | "os" 20 | "time" 21 | 22 | "golang.org/x/net/context" 23 | 24 | "gopkg.in/amz.v1/aws" 25 | "gopkg.in/amz.v1/s3" 26 | "gopkg.in/gorp.v1" 27 | "gopkg.in/tylerb/graceful.v1" 28 | 29 | "github.com/dropbox/changes-artifacts/api" 30 | "github.com/dropbox/changes-artifacts/common" 31 | "github.com/dropbox/changes-artifacts/common/sentry" 32 | "github.com/dropbox/changes-artifacts/common/stats" 33 | "github.com/dropbox/changes-artifacts/database" 34 | "github.com/dropbox/changes-artifacts/model" 35 | "github.com/gin-gonic/gin" 36 | _ "github.com/lib/pq" 37 | "github.com/martini-contrib/render" 38 | "github.com/rubenv/sql-migrate" 39 | ) 40 | 41 | type RenderOnGin struct { 42 | render.Render // This makes sure we don't have to create dummies for methods we don't use 43 | 44 | ginCtx *gin.Context 45 | } 46 | 47 | func (r RenderOnGin) JSON(statusCode int, obj interface{}) { 48 | r.ginCtx.JSON(statusCode, obj) 49 | } 50 | 51 | var _ render.Render = (*RenderOnGin)(nil) 52 | 53 | func HomeHandler(c *gin.Context) { 54 | c.String(http.StatusOK, "Hello, I am Artie Facts, the artifact store that stores artifacts.") 55 | } 56 | 57 | func VersionHandler(c *gin.Context) { 58 | c.String(http.StatusOK, common.GetVersion()) 59 | } 60 | 61 | func bindBucket(ctx context.Context, r render.Render, gc *gin.Context, db database.Database) { 62 | bucketId := gc.Param("bucket_id") 63 | bucket, err := db.GetBucket(bucketId) 64 | 65 | if err != nil && err.EntityNotFound() { 66 | // Don't log this error to Sentry 67 | // Changes will hit this endpoint for non-existant buckets very often. 68 | api.RespondWithErrorf(ctx, r, http.StatusNotFound, "Bucket not found") 69 | gc.Abort() 70 | return 71 | } 72 | 73 | if err != nil { 74 | api.LogAndRespondWithError(ctx, r, http.StatusInternalServerError, err) 75 | gc.Abort() 76 | return 77 | } 78 | 79 | if bucket == nil { 80 | api.LogAndRespondWithErrorf(ctx, r, http.StatusBadRequest, "Got nil bucket without error for bucket: %s", bucketId) 81 | gc.Abort() 82 | return 83 | } 84 | 85 | gc.Set("bucket", bucket) 86 | } 87 | 88 | func bindArtifact(ctx context.Context, r render.Render, gc *gin.Context, db database.Database) *model.Artifact { 89 | bucketId := gc.Param("bucket_id") 90 | artifactName := gc.Param("artifact_name") 91 | artifact, err := db.GetArtifactByName(bucketId, artifactName) 92 | 93 | if err != nil && err.EntityNotFound() { 94 | api.LogAndRespondWithErrorf(ctx, r, http.StatusNotFound, "Artifact not found") 95 | gc.Abort() 96 | return nil 97 | } 98 | 99 | if err != nil { 100 | api.LogAndRespondWithError(ctx, r, http.StatusInternalServerError, err) 101 | gc.Abort() 102 | return nil 103 | } 104 | 105 | if artifact == nil { 106 | api.LogAndRespondWithErrorf(ctx, r, http.StatusBadRequest, "Got nil artifact without error for artifact: %s/%s", bucketId, artifactName) 107 | gc.Abort() 108 | return nil 109 | } 110 | 111 | gc.Set("artifact", artifact) 112 | return artifact 113 | } 114 | 115 | type config struct { 116 | DbConnstr string 117 | CorsURLs string 118 | Env string 119 | S3Server string 120 | S3Region string 121 | S3Bucket string 122 | S3AccessKey string 123 | S3SecretKey string 124 | SentryDSN string 125 | StatsdPrefix string 126 | StatsdURL string 127 | } 128 | 129 | var defaultConfig = config{ 130 | CorsURLs: "http://dropboxlocalhost.com:5000", 131 | DbConnstr: "postgres://artifacts:artifacts@artifactsdb/artifacts?sslmode=disable", 132 | S3Server: "http://fakes3:4569", 133 | S3Region: "fakes3", 134 | S3Bucket: "artifacts", 135 | } 136 | 137 | func getConfigFrom(configFile string) config { 138 | var conf config 139 | 140 | if configFile == "" { 141 | return defaultConfig 142 | } 143 | 144 | if content, err := ioutil.ReadFile(configFile); err != nil { 145 | log.Fatalf("Unable to open config file %s\n", configFile) 146 | } else { 147 | if err := json.Unmarshal(content, &conf); err != nil { 148 | log.Fatalf("Unable to decode config file %s\n", configFile) 149 | } else { 150 | return conf 151 | } 152 | } 153 | 154 | return config{} 155 | } 156 | 157 | func performMigrations(db *sql.DB) error { 158 | migrations := &migrate.AssetMigrationSource{ 159 | Asset: database.Asset, 160 | AssetDir: database.AssetDir, 161 | Dir: "migrations", 162 | } 163 | 164 | n, err := migrate.Exec(db, "postgres", migrations, migrate.Up) 165 | log.Printf("Completed %d migrations\n", n) 166 | if err != nil { 167 | log.Println("Error completing DB migration:", err) 168 | } 169 | 170 | return err 171 | } 172 | 173 | func getListenAddr() string { 174 | port := os.Getenv("PORT") 175 | 176 | if len(port) == 0 { 177 | // By default, we use port 3000 178 | port = "3000" 179 | } 180 | 181 | // Host is "" => any address (IPv4/IPv6) 182 | hostPort := ":" + port 183 | log.Printf("About to listen on %s", hostPort) 184 | 185 | return hostPort 186 | } 187 | 188 | func main() { 189 | var flagConfigFile string 190 | flag.StringVar(&flagConfigFile, "config", "", "JSON Config file containing DB parameters and S3 information") 191 | 192 | flagVerbose := flag.Bool("verbose", false, "Enable request logging") 193 | flagLogDBQueries := flag.Bool("log-db-queries", false, "Enable DB query logging (Use with care, will dump raw logchunk contents to logfile)") 194 | 195 | showVersion := flag.Bool("version", false, "Show version number and quit") 196 | 197 | onlyPerformMigrations := flag.Bool("migrations-only", false, "Only perform database migrations and quit") 198 | 199 | dbMaxIdleConns := flag.Int("db-max-idle-conns", 20, "Maximum number of idle connections to the DB") 200 | 201 | dbMaxOpenConns := flag.Int("db-max-open-conns", 50, "Maximum number of open connections to the DB") 202 | 203 | shutdownTimeout := flag.Duration("shutdown-timeout", 15*time.Second, "Time to wait before closing active connections after SIGTERM signal has been recieved") 204 | 205 | flag.Parse() 206 | log.SetFlags(log.Lmicroseconds | log.Lshortfile) 207 | 208 | // Required for artifact name deduplication (not strictly necessary, but good to have) 209 | rand.Seed(time.Now().UTC().UnixNano()) 210 | 211 | if *showVersion { 212 | fmt.Println(common.GetVersion()) 213 | return 214 | } 215 | 216 | conf := getConfigFrom(flagConfigFile) 217 | 218 | // ----- BEGIN DB Connections Setup ----- 219 | db, err := sql.Open("postgres", conf.DbConnstr) 220 | 221 | if err != nil { 222 | log.Fatalf("Could not connect to the database: %v\n", err) 223 | } 224 | 225 | if *onlyPerformMigrations { 226 | err := performMigrations(db) 227 | db.Close() 228 | if err != nil { 229 | os.Exit(1) 230 | } 231 | 232 | return 233 | } 234 | 235 | db.SetMaxIdleConns(*dbMaxIdleConns) 236 | db.SetMaxOpenConns(*dbMaxOpenConns) 237 | defer db.Close() 238 | dbmap := &gorp.DbMap{Db: db, Dialect: gorp.PostgresDialect{}} 239 | if *flagLogDBQueries { 240 | dbmap.TraceOn("[gorp]", log.New(os.Stdout, "artifacts:", log.Lmicroseconds)) 241 | } 242 | gdb := database.NewGorpDatabase(dbmap) 243 | // ----- END DB Connections Setup ----- 244 | 245 | // ----- BEGIN AWS Connections ----- 246 | var region aws.Region 247 | var auth aws.Auth 248 | if conf.S3Region == "fakes3" { 249 | region = aws.Region{ 250 | Name: conf.S3Region, 251 | S3Endpoint: conf.S3Server, 252 | Sign: aws.SignV2, 253 | } 254 | 255 | auth = aws.Auth{} 256 | } else { 257 | region = aws.Regions[conf.S3Region] 258 | auth = aws.Auth{AccessKey: conf.S3AccessKey, SecretKey: conf.S3SecretKey} 259 | } 260 | 261 | s3Client := s3.New(auth, region) 262 | 263 | bucket := s3Client.Bucket(conf.S3Bucket) 264 | // ----- END AWS Connections ----- 265 | 266 | gdb.RegisterEntities() 267 | 268 | stats.CreateStatsdClient(conf.StatsdURL, conf.StatsdPrefix) 269 | defer stats.ShutdownStatsdClient() 270 | 271 | g := gin.New() 272 | g.Use(gin.Recovery()) 273 | 274 | if *flagVerbose { 275 | g.Use(gin.Logger()) 276 | } 277 | 278 | realClock := new(common.RealClock) 279 | 280 | rootCtx := context.Background() 281 | rootCtx = sentry.CreateAndInstallSentryClient(rootCtx, conf.Env, conf.SentryDSN) 282 | g.Use(stats.Counter()) 283 | 284 | g.GET("/", HomeHandler) 285 | g.GET("/version", VersionHandler) 286 | g.GET("/buckets", func(gc *gin.Context) { 287 | api.ListBuckets(rootCtx, &RenderOnGin{ginCtx: gc}, gdb) 288 | }) 289 | g.POST("/buckets/", func(gc *gin.Context) { 290 | api.HandleCreateBucket(rootCtx, &RenderOnGin{ginCtx: gc}, gc.Request, gdb, realClock) 291 | }) 292 | g.POST("/buckets/:bucket_id/artifacts/:artifact_name", func(gc *gin.Context) { 293 | render := &RenderOnGin{ginCtx: gc} 294 | afct := bindArtifact(rootCtx, render, gc, gdb) 295 | if !gc.IsAborted() { 296 | api.PostArtifact(rootCtx, render, gc.Request, gdb, bucket, afct) 297 | } 298 | }) 299 | 300 | br := g.Group("/buckets/:bucket_id", func(gc *gin.Context) { 301 | bindBucket(rootCtx, &RenderOnGin{ginCtx: gc}, gc, gdb) 302 | }) 303 | { 304 | br.GET("", func(gc *gin.Context) { 305 | bkt := gc.MustGet("bucket").(*model.Bucket) 306 | api.HandleGetBucket(rootCtx, &RenderOnGin{ginCtx: gc}, bkt) 307 | }) 308 | br.POST("/close", func(gc *gin.Context) { 309 | bkt := gc.MustGet("bucket").(*model.Bucket) 310 | api.HandleCloseBucket(rootCtx, &RenderOnGin{ginCtx: gc}, gdb, bkt, bucket, realClock) 311 | }) 312 | br.GET("/artifacts/", func(gc *gin.Context) { 313 | bkt := gc.MustGet("bucket").(*model.Bucket) 314 | api.ListArtifacts(rootCtx, &RenderOnGin{ginCtx: gc}, gc.Request, gdb, bkt) 315 | }) 316 | br.POST("/artifacts", func(gc *gin.Context) { 317 | bkt := gc.MustGet("bucket").(*model.Bucket) 318 | api.HandleCreateArtifact(rootCtx, &RenderOnGin{ginCtx: gc}, gc.Request, gdb, bkt) 319 | }) 320 | 321 | ar := br.Group("/artifacts/:artifact_name", func(gc *gin.Context) { 322 | bindArtifact(rootCtx, &RenderOnGin{ginCtx: gc}, gc, gdb) 323 | }) 324 | { 325 | ar.GET("", func(gc *gin.Context) { 326 | afct := gc.MustGet("artifact").(*model.Artifact) 327 | api.HandleGetArtifact(rootCtx, &RenderOnGin{ginCtx: gc}, afct) 328 | }) 329 | ar.POST("/close", func(gc *gin.Context) { 330 | afct := gc.MustGet("artifact").(*model.Artifact) 331 | api.HandleCloseArtifact(rootCtx, &RenderOnGin{ginCtx: gc}, gdb, bucket, afct) 332 | }) 333 | ar.GET("/content", func(gc *gin.Context) { 334 | afct := gc.MustGet("artifact").(*model.Artifact) 335 | api.GetArtifactContent(rootCtx, &RenderOnGin{ginCtx: gc}, gc.Request, gc.Writer, gdb, bucket, afct) 336 | }) 337 | ar.GET("/chunked", func(gc *gin.Context) { 338 | if conf.CorsURLs != "" { 339 | gc.Writer.Header().Add("Access-Control-Allow-Origin", conf.CorsURLs) 340 | } 341 | afct := gc.MustGet("artifact").(*model.Artifact) 342 | api.GetArtifactContentChunks(rootCtx, &RenderOnGin{ginCtx: gc}, gc.Request, gc.Writer, gdb, bucket, afct) 343 | }) 344 | } 345 | } 346 | 347 | http.Handle("/", g) 348 | 349 | // If the process gets a SIGTERM, it will close listening port allowing another server to bind and 350 | // begin listening immediately. Any ongoing connections will be given 15 seconds (by default) to 351 | // complete, after which they are forcibly terminated. 352 | graceful.Run(getListenAddr(), *shutdownTimeout, nil) 353 | } 354 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2014 Dropbox, Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /client/client_unit_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "net/http" 5 | "testing" 6 | "time" 7 | 8 | "golang.org/x/net/context" 9 | 10 | "github.com/dropbox/changes-artifacts/client/testserver" 11 | "github.com/dropbox/changes-artifacts/model" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestNewBucketSuccessStateWithWrongName(t *testing.T) { 16 | ts := testserver.NewTestServer(t) 17 | defer ts.CloseAndAssertExpectations() 18 | 19 | client := NewArtifactStoreClient(ts.URL) 20 | 21 | ts.ExpectAndRespond("POST", "/buckets/", http.StatusOK, `{"Id": "1234"}`) 22 | 23 | b, err := client.NewBucket("foo", "bar", 32) 24 | require.Nil(t, b) 25 | require.Error(t, err) 26 | require.False(t, err.IsRetriable(), "Error %s should not be retriable", err) 27 | } 28 | 29 | func TestNewBucketSuccessfully(t *testing.T) { 30 | ts := testserver.NewTestServer(t) 31 | defer ts.CloseAndAssertExpectations() 32 | 33 | client := NewArtifactStoreClient(ts.URL) 34 | 35 | ts.ExpectAndRespond("POST", "/buckets/", http.StatusOK, `{"Id": "foo"}`) 36 | 37 | b, err := client.NewBucket("foo", "bar", 32) 38 | require.NotNil(t, b) 39 | require.NoError(t, err) 40 | } 41 | 42 | func TestGetBucketSuccessStateWithWrongName(t *testing.T) { 43 | ts := testserver.NewTestServer(t) 44 | defer ts.CloseAndAssertExpectations() 45 | 46 | client := NewArtifactStoreClient(ts.URL) 47 | 48 | ts.ExpectAndRespond("GET", "/buckets/foo", http.StatusOK, `{"Id": "1234"}`) 49 | 50 | b, err := client.GetBucket("foo") 51 | require.Nil(t, b) 52 | require.Error(t, err) 53 | require.False(t, err.IsRetriable(), "Error %s should not be retriable", err) 54 | } 55 | 56 | func TestGetBucketSuccessfully(t *testing.T) { 57 | ts := testserver.NewTestServer(t) 58 | defer ts.CloseAndAssertExpectations() 59 | 60 | client := NewArtifactStoreClient(ts.URL) 61 | 62 | ts.ExpectAndRespond("GET", "/buckets/foo", http.StatusOK, `{"Id": "foo"}`) 63 | 64 | b, err := client.GetBucket("foo") 65 | require.NotNil(t, b) 66 | require.NoError(t, err) 67 | } 68 | 69 | func TestNewStreamedArtifactSuccessfully(t *testing.T) { 70 | ts := testserver.NewTestServer(t) 71 | defer ts.CloseAndAssertExpectations() 72 | 73 | client := NewArtifactStoreClient(ts.URL) 74 | 75 | ts.ExpectAndRespond("POST", "/buckets/", http.StatusOK, `{"Id": "foo"}`) 76 | ts.ExpectAndRespond("POST", "/buckets/foo/artifacts", http.StatusOK, `{"Name": "artifact"}`) 77 | 78 | b, _ := client.NewBucket("foo", "bar", 32) 79 | sa, err := b.NewStreamedArtifact("artifact", 10) 80 | require.NotNil(t, sa) 81 | require.NoError(t, err) 82 | } 83 | 84 | func TestNewBucketErrors(t *testing.T) { 85 | testErrorCombinations(t, func(*testserver.TestServer, *ArtifactStoreClient) interface{} { return nil }, "POST", "/buckets/", 86 | func(c *ArtifactStoreClient, _ interface{}) (interface{}, *ArtifactsError) { 87 | return c.NewBucket("foo", "bar", 12) 88 | }) 89 | } 90 | 91 | func TestGetBucketErrors(t *testing.T) { 92 | testErrorCombinations(t, func(*testserver.TestServer, *ArtifactStoreClient) interface{} { return nil }, "GET", "/buckets/foo", 93 | func(c *ArtifactStoreClient, _ interface{}) (interface{}, *ArtifactsError) { 94 | return c.GetBucket("foo") 95 | }) 96 | } 97 | 98 | func TestCreateStreamingArtifactErrors(t *testing.T) { 99 | testErrorCombinations(t, 100 | func(ts *testserver.TestServer, c *ArtifactStoreClient) interface{} { 101 | ts.ExpectAndRespond("POST", "/buckets/", http.StatusOK, `{"Id": "foo"}`) 102 | b, _ := c.NewBucket("foo", "bar", 32) 103 | return b 104 | }, 105 | "POST", "/buckets/foo/artifacts", 106 | func(c *ArtifactStoreClient, b interface{}) (interface{}, *ArtifactsError) { 107 | return b.(*Bucket).NewStreamedArtifact("artifact", 10) 108 | }) 109 | } 110 | 111 | func TestCreateChunkedArtifactErrors(t *testing.T) { 112 | testErrorCombinations(t, 113 | func(ts *testserver.TestServer, c *ArtifactStoreClient) interface{} { 114 | ts.ExpectAndRespond("POST", "/buckets/", http.StatusOK, `{"Id": "foo"}`) 115 | b, _ := c.NewBucket("foo", "bar", 32) 116 | return b 117 | }, 118 | "POST", "/buckets/foo/artifacts", 119 | func(c *ArtifactStoreClient, b interface{}) (interface{}, *ArtifactsError) { 120 | return b.(*Bucket).NewChunkedArtifact("artifact") 121 | }) 122 | } 123 | 124 | func TestGetArtifactErrors(t *testing.T) { 125 | testErrorCombinations(t, 126 | func(ts *testserver.TestServer, c *ArtifactStoreClient) interface{} { 127 | ts.ExpectAndRespond("POST", "/buckets/", http.StatusOK, `{"Id": "foo"}`) 128 | ts.ExpectAndRespond("POST", "/buckets/foo/artifacts", http.StatusOK, `{"Id": "bar"}`) 129 | b, _ := c.NewBucket("foo", "bar", 32) 130 | b.NewStreamedArtifact("bar", 1234) 131 | return b 132 | }, 133 | "GET", "/buckets/foo/artifacts/bar", 134 | func(c *ArtifactStoreClient, b interface{}) (interface{}, *ArtifactsError) { 135 | return b.(*Bucket).GetArtifact("bar") 136 | }) 137 | } 138 | 139 | func testErrorCombinations(t *testing.T, 140 | prerun func(*testserver.TestServer, *ArtifactStoreClient) interface{}, 141 | method string, 142 | url string, 143 | test func(c *ArtifactStoreClient, obj interface{}) (interface{}, *ArtifactsError)) { 144 | { 145 | ts := testserver.NewTestServer(t) 146 | client := NewArtifactStoreClient(ts.URL) 147 | obj := prerun(ts, client) 148 | 149 | ts.CloseAndAssertExpectations() 150 | // Server is missing, network error 151 | op, err := test(client, obj) 152 | require.Nil(t, op) 153 | require.Error(t, err) 154 | require.True(t, err.IsRetriable(), "Error %s should be retriable", err) 155 | } 156 | 157 | { 158 | // Server threw internal error 159 | ts := testserver.NewTestServer(t) 160 | defer ts.CloseAndAssertExpectations() 161 | 162 | client := NewArtifactStoreClient(ts.URL) 163 | obj := prerun(ts, client) 164 | ts.ExpectAndRespond(method, url, http.StatusInternalServerError, `{"error": "Something bad happened"}`) 165 | 166 | op, err := test(client, obj) 167 | require.Nil(t, op) 168 | require.Error(t, err) 169 | require.True(t, err.IsRetriable(), "Error %s should be retriable", err) 170 | } 171 | 172 | { 173 | // Server indicated client error 174 | ts := testserver.NewTestServer(t) 175 | defer ts.CloseAndAssertExpectations() 176 | 177 | client := NewArtifactStoreClient(ts.URL) 178 | obj := prerun(ts, client) 179 | ts.ExpectAndRespond(method, url, http.StatusBadRequest, `{"error": "Bad client"}`) 180 | 181 | op, err := test(client, obj) 182 | require.Nil(t, op) 183 | require.Error(t, err) 184 | require.False(t, err.IsRetriable(), "Error %s should not be retriable", err) 185 | } 186 | 187 | { 188 | // Proxy error - server was unreachable 189 | ts := testserver.NewTestServer(t) 190 | defer ts.CloseAndAssertExpectations() 191 | 192 | client := NewArtifactStoreClient(ts.URL) 193 | obj := prerun(ts, client) 194 | ts.ExpectAndRespond(method, url, http.StatusBadGateway, `Foo`) 195 | 196 | op, err := test(client, obj) 197 | require.Nil(t, op) 198 | require.Error(t, err) 199 | require.True(t, err.IsRetriable(), "Error %s should be retriable", err) 200 | } 201 | 202 | { 203 | // Proxy/server error - mangled output 204 | ts := testserver.NewTestServer(t) 205 | defer ts.CloseAndAssertExpectations() 206 | 207 | client := NewArtifactStoreClient(ts.URL) 208 | obj := prerun(ts, client) 209 | ts.ExpectAndRespond(method, url, http.StatusOK, ``) 210 | 211 | op, err := test(client, obj) 212 | require.Nil(t, op) 213 | require.Error(t, err) 214 | require.False(t, err.IsRetriable(), "Error %s should not be retriable", err) 215 | } 216 | 217 | { 218 | // Proxy/server hangs and times out 219 | ts := testserver.NewTestServer(t) 220 | defer ts.CloseAndAssertExpectations() 221 | client := NewArtifactStoreClientWithContext(ts.URL, 100*time.Millisecond, context.Background()) 222 | obj := prerun(ts, client) 223 | ts.ExpectAndHang(method, url) 224 | 225 | op, err := test(client, obj) 226 | require.Nil(t, op) 227 | require.Error(t, err) 228 | require.True(t, err.IsRetriable(), "Error %s should be retriable", err) 229 | } 230 | } 231 | 232 | func TestUploadLogChunksAndFlushSuccessfully(t *testing.T) { 233 | ts := testserver.NewTestServer(t) 234 | defer ts.CloseAndAssertExpectations() 235 | 236 | client := NewArtifactStoreClient(ts.URL) 237 | 238 | ts.ExpectAndRespond("POST", "/buckets/", http.StatusOK, `{"Id": "foo"}`) 239 | ts.ExpectAndRespond("POST", "/buckets/foo/artifacts", http.StatusOK, `{"Name": "artifact"}`) 240 | 241 | b, _ := client.NewBucket("foo", "bar", 32) 242 | sa, err := b.NewChunkedArtifact("artifact") 243 | require.NotNil(t, sa) 244 | require.NoError(t, err) 245 | 246 | { 247 | // Content request might come later, even as late as Flush() 248 | ts.ExpectAndRespond("POST", "/buckets/foo/artifacts/artifact", 200, `{}`) 249 | err := sa.AppendLog("console contents") 250 | require.NoError(t, err) 251 | } 252 | 253 | { 254 | // Content request might come later, even as late as Flush() 255 | ts.ExpectAndRespond("POST", "/buckets/foo/artifacts/artifact", 200, `{}`) 256 | err := sa.AppendLog("more console contents") 257 | require.NoError(t, err) 258 | } 259 | 260 | { 261 | err := sa.Flush() 262 | require.NoError(t, err) 263 | } 264 | } 265 | 266 | func TestUploadLogChunksAndCloseSuccessfully(t *testing.T) { 267 | ts := testserver.NewTestServer(t) 268 | defer ts.CloseAndAssertExpectations() 269 | 270 | client := NewArtifactStoreClient(ts.URL) 271 | 272 | ts.ExpectAndRespond("POST", "/buckets/", http.StatusOK, `{"Id": "foo"}`) 273 | ts.ExpectAndRespond("POST", "/buckets/foo/artifacts", http.StatusOK, `{"Name": "artifact"}`) 274 | 275 | b, _ := client.NewBucket("foo", "bar", 32) 276 | sa, err := b.NewChunkedArtifact("artifact") 277 | require.NotNil(t, sa) 278 | require.NoError(t, err) 279 | 280 | { 281 | // Content request might come later, even as late as Flush() 282 | ts.ExpectAndRespond("POST", "/buckets/foo/artifacts/artifact", 200, `{}`) 283 | err := sa.AppendLog("console contents") 284 | require.NoError(t, err) 285 | } 286 | 287 | { 288 | // Content request might come later, even as late as Flush() 289 | ts.ExpectAndRespond("POST", "/buckets/foo/artifacts/artifact", 200, `{}`) 290 | err := sa.AppendLog("more console contents") 291 | require.NoError(t, err) 292 | } 293 | 294 | { 295 | ts.ExpectAndRespond("POST", "/buckets/foo/artifacts/artifact/close", 200, `{}`) 296 | err := sa.Close() 297 | require.NoError(t, err) 298 | } 299 | } 300 | 301 | func TestPushLogChunkServerSucceedOnRetry(t *testing.T) { 302 | ts := testserver.NewTestServer(t) 303 | defer ts.CloseAndAssertExpectations() 304 | 305 | client := NewArtifactStoreClient(ts.URL) 306 | 307 | ts.ExpectAndRespond("POST", "/buckets/", http.StatusOK, `{"Id": "foo"}`) 308 | ts.ExpectAndRespond("POST", "/buckets/foo/artifacts", http.StatusOK, `{"Name": "artifact"}`) 309 | 310 | b, _ := client.NewBucket("foo", "bar", 32) 311 | sa, err := b.NewChunkedArtifact("artifact") 312 | require.NotNil(t, sa) 313 | require.NoError(t, err) 314 | 315 | { 316 | // Fail with a retriable error first 317 | ts.ExpectAndRespond("POST", "/buckets/foo/artifacts/artifact", 500, `{}`) 318 | // Then succeed on retry 319 | ts.ExpectAndRespond("POST", "/buckets/foo/artifacts/artifact", 200, `{}`) 320 | err := sa.AppendLog("console contents") 321 | require.NoError(t, err) 322 | } 323 | 324 | { 325 | ts.ExpectAndRespond("POST", "/buckets/foo/artifacts/artifact/close", 200, `{}`) 326 | err := sa.Close() 327 | require.NoError(t, err) 328 | } 329 | } 330 | 331 | func TestPushLogChunkServerFailWithTerminalError(t *testing.T) { 332 | ts := testserver.NewTestServer(t) 333 | defer ts.CloseAndAssertExpectations() 334 | 335 | client := NewArtifactStoreClientWithContext(ts.URL, 100*time.Millisecond, context.Background()) 336 | 337 | ts.ExpectAndRespond("POST", "/buckets/", http.StatusOK, `{"Id": "foo"}`) 338 | ts.ExpectAndRespond("POST", "/buckets/foo/artifacts", http.StatusOK, `{"Name": "artifact"}`) 339 | 340 | b, _ := client.NewBucket("foo", "bar", 32) 341 | sa, err := b.NewChunkedArtifact("artifact") 342 | require.NotNil(t, sa) 343 | require.NoError(t, err) 344 | 345 | { 346 | // Fail with a terminal error 347 | ts.ExpectAndRespond("POST", "/buckets/foo/artifacts/artifact", 400, `{}`) 348 | err := sa.AppendLog("console contents") 349 | require.NoError(t, err) 350 | err = sa.Close() 351 | require.Error(t, err) 352 | require.False(t, err.IsRetriable()) 353 | } 354 | } 355 | 356 | func TestPushLogChunkCancelledContext(t *testing.T) { 357 | ts := testserver.NewTestServer(t) 358 | defer ts.CloseAndAssertExpectations() 359 | 360 | ctx, cancel := context.WithCancel(context.Background()) 361 | client := NewArtifactStoreClientWithContext(ts.URL, 100*time.Millisecond, ctx) 362 | 363 | ts.ExpectAndRespond("POST", "/buckets/", http.StatusOK, `{"Id": "foo"}`) 364 | ts.ExpectAndRespond("POST", "/buckets/foo/artifacts", http.StatusOK, `{"Name": "artifact"}`) 365 | 366 | b, _ := client.NewBucket("foo", "bar", 32) 367 | sa, err := b.NewChunkedArtifact("artifact") 368 | require.NotNil(t, sa) 369 | require.NoError(t, err) 370 | 371 | // Cancel the context to prevent any further requests 372 | cancel() 373 | 374 | { 375 | err := sa.AppendLog("console contents") 376 | require.NoError(t, err) 377 | err = sa.Close() 378 | require.Error(t, err) 379 | require.False(t, err.IsRetriable()) 380 | } 381 | } 382 | 383 | func TestArtifactURL(t *testing.T) { 384 | client := NewArtifactStoreClient("http://foo") 385 | bucket := &Bucket{ 386 | bucket: &model.Bucket{ 387 | Id: "bkt", 388 | }, 389 | client: client, 390 | } 391 | 392 | { 393 | streamedArtifact := &StreamedArtifact{ 394 | ArtifactImpl: &ArtifactImpl{ 395 | artifact: &model.Artifact{ 396 | Name: "safct", 397 | }, 398 | bucket: bucket, 399 | }, 400 | } 401 | require.Equal(t, "http://foo/buckets/bkt/artifacts/safct/content", streamedArtifact.GetContentURL()) 402 | } 403 | 404 | { 405 | chunkedArtifact := &ChunkedArtifact{ 406 | ArtifactImpl: &ArtifactImpl{ 407 | artifact: &model.Artifact{ 408 | Name: "cafct", 409 | }, 410 | bucket: bucket, 411 | }, 412 | } 413 | require.Equal(t, "http://foo/buckets/bkt/artifacts/cafct/content", chunkedArtifact.GetContentURL()) 414 | } 415 | } 416 | -------------------------------------------------------------------------------- /database/bindata.go: -------------------------------------------------------------------------------- 1 | // Code generated by go-bindata. 2 | // sources: 3 | // migrations/1_initial.sql 4 | // migrations/2_index_artifactid_size.sql 5 | // migrations/3_add_relative_path.sql 6 | // migrations/4_byte_array.sql 7 | // migrations/README 8 | // DO NOT EDIT! 9 | 10 | package database 11 | 12 | import ( 13 | "bytes" 14 | "compress/gzip" 15 | "fmt" 16 | "io" 17 | "strings" 18 | "os" 19 | "time" 20 | "io/ioutil" 21 | "path/filepath" 22 | ) 23 | 24 | func bindataRead(data []byte, name string) ([]byte, error) { 25 | gz, err := gzip.NewReader(bytes.NewBuffer(data)) 26 | if err != nil { 27 | return nil, fmt.Errorf("Read %q: %v", name, err) 28 | } 29 | 30 | var buf bytes.Buffer 31 | _, err = io.Copy(&buf, gz) 32 | clErr := gz.Close() 33 | 34 | if err != nil { 35 | return nil, fmt.Errorf("Read %q: %v", name, err) 36 | } 37 | if clErr != nil { 38 | return nil, err 39 | } 40 | 41 | return buf.Bytes(), nil 42 | } 43 | 44 | type asset struct { 45 | bytes []byte 46 | info os.FileInfo 47 | } 48 | 49 | type bindataFileInfo struct { 50 | name string 51 | size int64 52 | mode os.FileMode 53 | modTime time.Time 54 | } 55 | 56 | func (fi bindataFileInfo) Name() string { 57 | return fi.name 58 | } 59 | func (fi bindataFileInfo) Size() int64 { 60 | return fi.size 61 | } 62 | func (fi bindataFileInfo) Mode() os.FileMode { 63 | return fi.mode 64 | } 65 | func (fi bindataFileInfo) ModTime() time.Time { 66 | return fi.modTime 67 | } 68 | func (fi bindataFileInfo) IsDir() bool { 69 | return false 70 | } 71 | func (fi bindataFileInfo) Sys() interface{} { 72 | return nil 73 | } 74 | 75 | var _migrations1_initialSql = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x94\x92\x41\x73\xba\x30\x10\xc5\xef\x7c\x8a\x1d\x2e\xea\xfc\xf5\xf4\x3f\x7a\xc2\x9a\xb6\x99\x22\x2a\x84\xa9\xf6\x16\xc3\xaa\x19\x31\x30\x10\xc6\xb6\x9f\xbe\x09\x88\xd2\xce\xb4\xb6\xc7\xf7\xf2\xd8\xfd\xed\xb2\xa3\x11\xfc\x3b\xca\x5d\xc1\x35\x42\x9c\x3b\x46\x46\x4b\x1f\xa4\x82\x12\x85\x96\x99\x82\x5e\x9c\xf7\x40\x96\x80\xaf\x28\x2a\x8d\x09\x9c\xf6\xa8\x40\xef\x8d\xd5\x7c\x67\x43\x46\xf0\x3c\x4f\x25\x26\xce\x5d\x48\x3c\x46\x80\x79\x13\x9f\x00\xbd\x87\x60\xce\x80\xac\x68\xc4\x22\xd8\x54\xe2\x80\x1a\xfa\x0e\x40\x62\xfa\x89\x34\x2b\x4d\x3d\x46\x67\x24\x62\xde\x6c\x01\xcf\x94\x3d\xd6\x12\x5e\xe6\x01\x19\xb6\xb1\x02\xb9\xbe\x91\x93\xe6\x99\xac\x58\xdd\x2c\x88\x7d\x1f\x16\x21\x9d\x79\xe1\x1a\x9e\xc8\xda\xbe\x67\x27\x85\x45\x1d\xb1\xaa\xd4\x76\x5a\xab\x9c\xc1\xf8\x27\x60\x5e\x68\xb9\xe5\xa2\x41\x6e\xe8\xcf\x9d\xfe\x08\x37\xa1\x0f\x11\x09\xa9\xe7\x7f\x4b\xa8\xf8\x11\xaf\x80\xff\xab\x22\xbd\x2a\xf9\x8e\xb6\x02\x0d\xbe\xd0\xd7\x10\xc8\x93\x54\x2a\x3c\x4a\x55\x5e\xcc\x38\xa0\xcb\x98\x40\xbf\x45\x1e\xd6\xe5\x07\x37\xa6\x4d\xb3\x9d\xd8\x57\xea\x00\x7d\x57\x26\xee\x0d\x66\x18\x82\xdb\xae\xe7\x9c\xb6\x7c\xe0\x6e\xde\x34\x66\xdb\x6d\x89\xba\x63\xda\x11\x3a\x52\x64\x4a\xa3\x32\x01\x0b\x6c\xa0\x9c\xee\x15\x4e\xcd\xcf\x6a\xef\xf0\x72\x84\xd6\xfc\xd5\x19\x16\x59\x9a\x9a\xd7\x0d\x17\x07\x67\x1a\xce\x17\xe7\x49\x9b\x4d\x8c\xbb\x56\x4b\xff\xc9\x6c\x77\x30\x76\x3e\x02\x00\x00\xff\xff\x08\x1b\xf1\xb5\x19\x03\x00\x00") 76 | 77 | func migrations1_initialSqlBytes() ([]byte, error) { 78 | return bindataRead( 79 | _migrations1_initialSql, 80 | "migrations/1_initial.sql", 81 | ) 82 | } 83 | 84 | func migrations1_initialSql() (*asset, error) { 85 | bytes, err := migrations1_initialSqlBytes() 86 | if err != nil { 87 | return nil, err 88 | } 89 | 90 | info := bindataFileInfo{name: "migrations/1_initial.sql", size: 793, mode: os.FileMode(436), modTime: time.Unix(1, 0)} 91 | a := &asset{bytes: bytes, info: info} 92 | return a, nil 93 | } 94 | 95 | var _migrations2_index_artifactid_sizeSql = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\xd2\xd5\x55\xd0\xce\xcd\x4c\x2f\x4a\x2c\x49\x55\x08\x2d\xe0\x72\x0e\x72\x75\x0c\x71\x55\xf0\xf4\x73\x71\x8d\x50\xc8\xc9\x4f\x4f\xce\x28\xcd\xcb\x8e\x4f\x2c\x2a\xc9\x4c\x4b\x4c\x2e\xc9\x4c\x89\x2f\xce\xac\x4a\x8d\xcf\x49\x2c\x2e\x51\xf0\xf7\x83\x2b\x50\xd0\x40\xa8\xd0\x51\x00\x29\x51\x70\x71\x0d\x76\x56\xf0\x0b\xf5\xf1\x09\x56\xf0\x71\x0c\x0e\xd1\xb4\xe6\xe2\x42\xb6\xca\x25\xbf\x3c\x8f\xcb\x25\xc8\x3f\x80\x18\xab\xac\xb9\x00\x01\x00\x00\xff\xff\x72\xaa\x21\x25\xa6\x00\x00\x00") 96 | 97 | func migrations2_index_artifactid_sizeSqlBytes() ([]byte, error) { 98 | return bindataRead( 99 | _migrations2_index_artifactid_sizeSql, 100 | "migrations/2_index_artifactid_size.sql", 101 | ) 102 | } 103 | 104 | func migrations2_index_artifactid_sizeSql() (*asset, error) { 105 | bytes, err := migrations2_index_artifactid_sizeSqlBytes() 106 | if err != nil { 107 | return nil, err 108 | } 109 | 110 | info := bindataFileInfo{name: "migrations/2_index_artifactid_size.sql", size: 166, mode: os.FileMode(436), modTime: time.Unix(1, 0)} 111 | a := &asset{bytes: bytes, info: info} 112 | return a, nil 113 | } 114 | 115 | var _migrations3_add_relative_pathSql = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\xd2\xd5\x55\xd0\xce\xcd\x4c\x2f\x4a\x2c\x49\x55\x08\x2d\xe0\x72\xf4\x09\x71\x0d\x52\x08\x71\x74\xf2\x71\x55\x48\x2c\x2a\xc9\x4c\x4b\x4c\x2e\x51\x70\x74\x71\x51\x70\xf6\xf7\x09\xf5\xf5\x53\x28\x4a\xcd\x49\x2c\xc9\x2c\x4b\x2d\x48\x2c\xc9\x50\x08\x73\x0c\x72\xf6\x70\x0c\xd2\x30\x32\x35\xd5\xb4\xe6\x0a\x0d\x70\x71\x0c\x41\xd2\x15\xec\x1a\x82\xaa\xdc\x56\x21\x2f\x31\x37\x55\x21\xdc\xc3\x35\xc8\x15\x55\xc6\x33\x58\xc1\x2f\xd4\xc7\xc7\x9a\x8b\x0b\xd9\x39\x2e\xf9\xe5\x79\xd8\x1d\xe4\x12\xe4\x1f\x00\x73\x91\x12\xb2\x49\x4a\xd6\x5c\x80\x00\x00\x00\xff\xff\xd3\x97\xe9\xe4\xd1\x00\x00\x00") 116 | 117 | func migrations3_add_relative_pathSqlBytes() ([]byte, error) { 118 | return bindataRead( 119 | _migrations3_add_relative_pathSql, 120 | "migrations/3_add_relative_path.sql", 121 | ) 122 | } 123 | 124 | func migrations3_add_relative_pathSql() (*asset, error) { 125 | bytes, err := migrations3_add_relative_pathSqlBytes() 126 | if err != nil { 127 | return nil, err 128 | } 129 | 130 | info := bindataFileInfo{name: "migrations/3_add_relative_path.sql", size: 209, mode: os.FileMode(436), modTime: time.Unix(1, 0)} 131 | a := &asset{bytes: bytes, info: info} 132 | return a, nil 133 | } 134 | 135 | var _migrations4_byte_arraySql = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\xd2\xd5\x55\xd0\xce\xcd\x4c\x2f\x4a\x2c\x49\x55\x08\x2d\xe0\x72\xf4\x09\x71\x0d\x52\x08\x71\x74\xf2\x71\x55\xc8\xc9\x4f\x4f\xce\x28\xcd\xcb\x56\x70\x74\x71\x51\x70\xf6\xf7\x09\xf5\xf5\x53\x48\xce\xcf\x2b\x49\xcd\x2b\x89\x4f\xaa\x2c\x49\x2d\x56\x70\x8a\x0c\x71\x75\xb4\xe6\xe2\x42\x36\xc4\x25\xbf\x3c\x0f\xbb\x31\x2e\x41\xfe\x01\x58\xcd\xb1\xe6\x02\x04\x00\x00\xff\xff\x4f\xc8\xc7\x20\x86\x00\x00\x00") 136 | 137 | func migrations4_byte_arraySqlBytes() ([]byte, error) { 138 | return bindataRead( 139 | _migrations4_byte_arraySql, 140 | "migrations/4_byte_array.sql", 141 | ) 142 | } 143 | 144 | func migrations4_byte_arraySql() (*asset, error) { 145 | bytes, err := migrations4_byte_arraySqlBytes() 146 | if err != nil { 147 | return nil, err 148 | } 149 | 150 | info := bindataFileInfo{name: "migrations/4_byte_array.sql", size: 134, mode: os.FileMode(436), modTime: time.Unix(1, 0)} 151 | a := &asset{bytes: bytes, info: info} 152 | return a, nil 153 | } 154 | 155 | var _migrationsReadme = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\x9c\x53\xcd\x6e\xdb\x30\x0c\xbe\xeb\x29\x08\xf4\xd2\x06\x89\x73\x0f\xba\x02\x2d\xb2\xe3\x7e\x80\x06\xd8\x31\x91\x2d\xda\x16\x2a\x4b\xaa\x48\xc5\xf3\xdb\x8f\xb2\x97\x60\xcd\x0e\x03\x66\x20\x87\x48\x24\xbf\x3f\xea\x39\xb1\x6d\x75\xc3\x04\xaf\x1c\x12\xc2\x5e\xb3\xae\x35\x21\x7c\xb1\x5d\xd2\x6c\x83\x27\xf5\xe9\xdf\x9f\x52\xcf\xc6\x80\xf6\x13\x0c\x97\x3e\xa0\x26\xd9\x28\x83\xd9\xb2\x43\x03\x8f\x84\xef\x47\x1f\x9e\x8e\x8f\xd4\x87\xc4\xc7\x01\x89\x74\x87\x4f\x15\xbd\x3b\xe0\x00\xdc\x5b\x82\x36\x38\x83\xa9\x82\xd7\xdf\xbd\xa3\x75\x0e\x6a\x54\xf8\x13\x9b\xcc\x32\xc5\xfa\xeb\x20\x08\x69\xae\xfd\xee\xb0\x10\x26\x44\xe8\x99\x23\xed\xb6\xdb\xce\x72\x9f\xeb\xaa\x09\xc3\x36\xe5\x1a\xfd\x79\x2b\x20\x9b\x85\x1a\xde\x8d\xc9\xb2\xf5\xdd\xe6\x4a\xb5\xe0\x26\xd5\x65\x6b\xd0\x59\x8f\x04\xc2\xbe\x0f\x63\x61\x55\x6a\x51\xb8\xe1\xdf\xc2\x2a\xa5\xbe\x7e\x3b\x7c\xde\x5d\x08\xa4\xec\xe1\xd4\x05\xe8\xd0\x63\xc1\x81\xcd\x19\xaa\x6d\x55\x55\x27\xd0\x2d\x63\x82\x41\xbf\x09\xec\xec\x52\xd3\x6b\xdf\x09\xd0\xad\xee\x1f\x08\x5a\x52\xc0\xa1\x46\x63\x4a\xb1\x20\xab\x5b\x64\xd0\x04\x1a\x04\xa9\xb5\x0e\xe1\x7e\x1e\xc0\x49\x7b\x72\x4b\x95\xfc\x8d\x98\x44\xd2\x20\x86\xd5\xd3\x07\x52\xa7\x07\x01\x11\x77\x42\xe6\x42\xd8\x17\x0c\xb9\x57\xd7\xfb\xf5\x8d\xd8\x19\x63\x8e\x61\x25\x6a\x57\x12\x86\x64\xd0\xb8\x6c\xb0\xe4\x2d\xbf\x18\x9d\x45\x23\x66\x1c\x90\x8a\xad\x60\x2e\x2b\xf4\x87\xbf\xf7\xb5\x6e\xde\x46\x9d\x0c\x81\x84\x12\xe5\xb4\x76\x96\xa7\x87\x9d\xda\xfc\xcf\xa7\xf6\x39\x15\xa8\x46\xa0\x3d\x0b\x21\xc6\x0b\x5f\x16\x16\xb4\x86\x51\x4c\x5c\x76\x66\xd6\x43\x7a\xf8\xc0\x87\x7b\xcd\xa5\x66\x0c\xd9\x99\x12\x77\x4c\xc1\x54\xf0\x32\x81\xc1\x56\x67\xc7\x6b\x25\x6d\xa2\x61\x1e\xb7\xe8\x2f\xf9\xea\x4e\x5b\x4f\x3c\xcf\x8c\x09\xcf\x36\x64\x72\xa5\x29\xba\x30\x89\x23\x67\x4c\x54\x58\x2c\xa1\xac\x64\xd3\x65\xfe\x4a\x92\xe7\xa6\x9f\x9b\x2e\x05\xc1\xab\x02\x29\x69\x1c\x42\x39\xb4\xed\xb4\x90\x5a\x00\x9b\xe0\xc5\xcb\x8c\x65\x41\x66\xdc\x65\x81\xae\x02\xd6\x50\xe7\x21\x42\x8e\x37\x69\x39\x3c\xa3\x03\x2a\x4f\xb9\xbc\x15\xb5\x18\x74\x2c\x43\xab\x2e\xdc\x11\x72\x8e\xfb\x97\x4a\xfd\x0a\x00\x00\xff\xff\x23\xa1\xeb\xb5\xf7\x03\x00\x00") 156 | 157 | func migrationsReadmeBytes() ([]byte, error) { 158 | return bindataRead( 159 | _migrationsReadme, 160 | "migrations/README", 161 | ) 162 | } 163 | 164 | func migrationsReadme() (*asset, error) { 165 | bytes, err := migrationsReadmeBytes() 166 | if err != nil { 167 | return nil, err 168 | } 169 | 170 | info := bindataFileInfo{name: "migrations/README", size: 1015, mode: os.FileMode(436), modTime: time.Unix(1, 0)} 171 | a := &asset{bytes: bytes, info: info} 172 | return a, nil 173 | } 174 | 175 | // Asset loads and returns the asset for the given name. 176 | // It returns an error if the asset could not be found or 177 | // could not be loaded. 178 | func Asset(name string) ([]byte, error) { 179 | cannonicalName := strings.Replace(name, "\\", "/", -1) 180 | if f, ok := _bindata[cannonicalName]; ok { 181 | a, err := f() 182 | if err != nil { 183 | return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err) 184 | } 185 | return a.bytes, nil 186 | } 187 | return nil, fmt.Errorf("Asset %s not found", name) 188 | } 189 | 190 | // MustAsset is like Asset but panics when Asset would return an error. 191 | // It simplifies safe initialization of global variables. 192 | func MustAsset(name string) []byte { 193 | a, err := Asset(name) 194 | if (err != nil) { 195 | panic("asset: Asset(" + name + "): " + err.Error()) 196 | } 197 | 198 | return a 199 | } 200 | 201 | // AssetInfo loads and returns the asset info for the given name. 202 | // It returns an error if the asset could not be found or 203 | // could not be loaded. 204 | func AssetInfo(name string) (os.FileInfo, error) { 205 | cannonicalName := strings.Replace(name, "\\", "/", -1) 206 | if f, ok := _bindata[cannonicalName]; ok { 207 | a, err := f() 208 | if err != nil { 209 | return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err) 210 | } 211 | return a.info, nil 212 | } 213 | return nil, fmt.Errorf("AssetInfo %s not found", name) 214 | } 215 | 216 | // AssetNames returns the names of the assets. 217 | func AssetNames() []string { 218 | names := make([]string, 0, len(_bindata)) 219 | for name := range _bindata { 220 | names = append(names, name) 221 | } 222 | return names 223 | } 224 | 225 | // _bindata is a table, holding each asset generator, mapped to its name. 226 | var _bindata = map[string]func() (*asset, error){ 227 | "migrations/1_initial.sql": migrations1_initialSql, 228 | "migrations/2_index_artifactid_size.sql": migrations2_index_artifactid_sizeSql, 229 | "migrations/3_add_relative_path.sql": migrations3_add_relative_pathSql, 230 | "migrations/4_byte_array.sql": migrations4_byte_arraySql, 231 | "migrations/README": migrationsReadme, 232 | } 233 | 234 | // AssetDir returns the file names below a certain 235 | // directory embedded in the file by go-bindata. 236 | // For example if you run go-bindata on data/... and data contains the 237 | // following hierarchy: 238 | // data/ 239 | // foo.txt 240 | // img/ 241 | // a.png 242 | // b.png 243 | // then AssetDir("data") would return []string{"foo.txt", "img"} 244 | // AssetDir("data/img") would return []string{"a.png", "b.png"} 245 | // AssetDir("foo.txt") and AssetDir("notexist") would return an error 246 | // AssetDir("") will return []string{"data"}. 247 | func AssetDir(name string) ([]string, error) { 248 | node := _bintree 249 | if len(name) != 0 { 250 | cannonicalName := strings.Replace(name, "\\", "/", -1) 251 | pathList := strings.Split(cannonicalName, "/") 252 | for _, p := range pathList { 253 | node = node.Children[p] 254 | if node == nil { 255 | return nil, fmt.Errorf("Asset %s not found", name) 256 | } 257 | } 258 | } 259 | if node.Func != nil { 260 | return nil, fmt.Errorf("Asset %s not found", name) 261 | } 262 | rv := make([]string, 0, len(node.Children)) 263 | for childName := range node.Children { 264 | rv = append(rv, childName) 265 | } 266 | return rv, nil 267 | } 268 | 269 | type bintree struct { 270 | Func func() (*asset, error) 271 | Children map[string]*bintree 272 | } 273 | var _bintree = &bintree{nil, map[string]*bintree{ 274 | "migrations": &bintree{nil, map[string]*bintree{ 275 | "1_initial.sql": &bintree{migrations1_initialSql, map[string]*bintree{ 276 | }}, 277 | "2_index_artifactid_size.sql": &bintree{migrations2_index_artifactid_sizeSql, map[string]*bintree{ 278 | }}, 279 | "3_add_relative_path.sql": &bintree{migrations3_add_relative_pathSql, map[string]*bintree{ 280 | }}, 281 | "4_byte_array.sql": &bintree{migrations4_byte_arraySql, map[string]*bintree{ 282 | }}, 283 | "README": &bintree{migrationsReadme, map[string]*bintree{ 284 | }}, 285 | }}, 286 | }} 287 | 288 | // RestoreAsset restores an asset under the given directory 289 | func RestoreAsset(dir, name string) error { 290 | data, err := Asset(name) 291 | if err != nil { 292 | return err 293 | } 294 | info, err := AssetInfo(name) 295 | if err != nil { 296 | return err 297 | } 298 | err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755)) 299 | if err != nil { 300 | return err 301 | } 302 | err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) 303 | if err != nil { 304 | return err 305 | } 306 | err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) 307 | if err != nil { 308 | return err 309 | } 310 | return nil 311 | } 312 | 313 | // RestoreAssets restores an asset under the given directory recursively 314 | func RestoreAssets(dir, name string) error { 315 | children, err := AssetDir(name) 316 | // File 317 | if err != nil { 318 | return RestoreAsset(dir, name) 319 | } 320 | // Dir 321 | for _, child := range children { 322 | err = RestoreAssets(dir, filepath.Join(name, child)) 323 | if err != nil { 324 | return err 325 | } 326 | } 327 | return nil 328 | } 329 | 330 | func _filePath(dir, name string) string { 331 | cannonicalName := strings.Replace(name, "\\", "/", -1) 332 | return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) 333 | } 334 | 335 | -------------------------------------------------------------------------------- /client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "log" 11 | "net/http" 12 | "path/filepath" 13 | "time" 14 | 15 | "golang.org/x/net/context" 16 | "golang.org/x/net/context/ctxhttp" 17 | 18 | "github.com/cenkalti/backoff" 19 | "github.com/dropbox/changes-artifacts/model" 20 | ) 21 | 22 | const MAX_PENDING_REPORTS = 100 23 | 24 | // Default timeout for any HTTP requests. On timeout, mark the operation as failed, but retriable. 25 | const DefaultReqTimeout = 30 * time.Second 26 | 27 | func ignoreBody(body io.ReadCloser, err *ArtifactsError) *ArtifactsError { 28 | if body != nil { 29 | body.Close() 30 | } 31 | 32 | return err 33 | } 34 | 35 | type ArtifactsError struct { 36 | errStr string 37 | retriable bool 38 | } 39 | 40 | func (e *ArtifactsError) Error() string { 41 | return e.errStr 42 | } 43 | 44 | func (e *ArtifactsError) IsRetriable() bool { 45 | return e.retriable 46 | } 47 | 48 | func NewRetriableError(errStr string) *ArtifactsError { 49 | return &ArtifactsError{retriable: true, errStr: errStr} 50 | } 51 | 52 | func NewRetriableErrorf(format string, args ...interface{}) *ArtifactsError { 53 | return NewRetriableError(fmt.Sprintf(format, args...)) 54 | } 55 | 56 | func NewTerminalError(errStr string) *ArtifactsError { 57 | return &ArtifactsError{retriable: false, errStr: errStr} 58 | } 59 | 60 | func NewTerminalErrorf(format string, args ...interface{}) *ArtifactsError { 61 | return NewTerminalError(fmt.Sprintf(format, args...)) 62 | } 63 | 64 | type ArtifactStoreClient struct { 65 | server string 66 | ctx context.Context 67 | timeout time.Duration 68 | } 69 | 70 | func NewArtifactStoreClient(serverURL string) *ArtifactStoreClient { 71 | return NewArtifactStoreClientWithContext(serverURL, DefaultReqTimeout, context.TODO()) 72 | } 73 | 74 | // NewArtifactStoreClientWithContext creates a new client with given context and per-request 75 | // timeout. 76 | func NewArtifactStoreClientWithContext(serverURL string, timeout time.Duration, ctx context.Context) *ArtifactStoreClient { 77 | return &ArtifactStoreClient{server: serverURL, timeout: timeout, ctx: ctx} 78 | } 79 | 80 | func (c *ArtifactStoreClient) getAPI(path string) (io.ReadCloser, *ArtifactsError) { 81 | url := c.server + path 82 | ctx, cancel := context.WithTimeout(c.ctx, c.timeout) 83 | defer cancel() 84 | 85 | if resp, err := ctxhttp.Get(ctx, nil, url); err != nil { 86 | return nil, NewRetriableError(err.Error()) 87 | } else { 88 | if resp.StatusCode != http.StatusOK { 89 | return nil, determineResponseError(resp, url, "POST") 90 | } 91 | return resp.Body, nil 92 | } 93 | } 94 | 95 | func (c *ArtifactStoreClient) postAPIJSON(path string, params map[string]interface{}) (io.ReadCloser, *ArtifactsError) { 96 | mJSON, err := json.Marshal(params) 97 | if err != nil { 98 | // Marshalling is deterministic so we can't retry in this scenario. 99 | return nil, NewTerminalError(err.Error()) 100 | } 101 | 102 | return c.postAPI(path, "application/json", bytes.NewReader(mJSON)) 103 | } 104 | 105 | func (c *ArtifactStoreClient) postAPI(path string, contentType string, body io.Reader) (io.ReadCloser, *ArtifactsError) { 106 | url := c.server + path 107 | ctx, cancel := context.WithTimeout(c.ctx, c.timeout) 108 | defer cancel() 109 | 110 | if resp, err := ctxhttp.Post(ctx, nil, url, contentType, body); err != nil { 111 | // If there was an error connecting to the server, it is likely to be transient and should be 112 | // retried. 113 | return nil, NewRetriableError(err.Error()) 114 | } else { 115 | if resp.StatusCode != http.StatusOK { 116 | return nil, determineResponseError(resp, url, "POST") 117 | } 118 | return resp.Body, nil 119 | } 120 | } 121 | 122 | func (c *ArtifactStoreClient) parseBucketFromResponse(body io.ReadCloser) (*Bucket, *ArtifactsError) { 123 | bText, err := ioutil.ReadAll(body) 124 | if err != nil { 125 | return nil, NewRetriableError(err.Error()) 126 | } 127 | body.Close() 128 | 129 | bucket := new(model.Bucket) 130 | if err := json.Unmarshal(bText, bucket); err != nil { 131 | return nil, NewTerminalError(err.Error()) 132 | } 133 | 134 | return &Bucket{ 135 | client: c, 136 | bucket: bucket, 137 | }, nil 138 | } 139 | 140 | // Determine the parse error for the response, which corresponds 141 | // to the "error" field in its json-encoded body. 142 | func parseErrorForResponse(body io.ReadCloser) (string, error) { 143 | var bJson map[string]string 144 | 145 | bText, err := ioutil.ReadAll(body) 146 | if err != nil { 147 | return "", err 148 | } 149 | body.Close() 150 | 151 | err = json.Unmarshal(bText, &bJson) 152 | if err != nil { 153 | return "", err 154 | } 155 | parseError, ok := bJson["error"] 156 | if !ok { 157 | return "", errors.New("Response body did not contain error key") 158 | } 159 | return parseError, nil 160 | } 161 | 162 | // Return either a terminal or retriable error for a failed response 163 | // depending on the type of status code in the response, and format 164 | // it in a nice way showing the url and method. 165 | func determineResponseError(resp *http.Response, url string, method string) *ArtifactsError { 166 | parsedError, err := parseErrorForResponse(resp.Body) 167 | if err != nil { 168 | parsedError = fmt.Sprintf("Unknown error, could not parse body: %s", err.Error()) 169 | } 170 | if resp.StatusCode >= 500 { 171 | // Server error. Maybe DB is unreachable. Can be retried. 172 | return NewRetriableErrorf("Error %d [%s %s] %s", resp.StatusCode, method, url, parsedError) 173 | } 174 | return NewTerminalErrorf("Error %d [%s %s] %s", resp.StatusCode, method, url, parsedError) 175 | } 176 | 177 | func (c *ArtifactStoreClient) GetBucket(bucketName string) (*Bucket, *ArtifactsError) { 178 | body, err := c.getAPI(fmt.Sprintf("/buckets/%s", bucketName)) 179 | 180 | if err != nil { 181 | return nil, err 182 | } 183 | 184 | bucket, e := c.parseBucketFromResponse(body) 185 | 186 | if e != nil { 187 | return nil, e 188 | } 189 | 190 | if bucket.bucket.Id != bucketName { 191 | return nil, NewTerminalError("Bucket created with wrong name") 192 | } 193 | 194 | return bucket, nil 195 | } 196 | 197 | // XXX deadlineMins is not used. Is this planned for something? 198 | func (c *ArtifactStoreClient) NewBucket(bucketName string, owner string, deadlineMins int) (*Bucket, *ArtifactsError) { 199 | body, err := c.postAPIJSON("/buckets/", map[string]interface{}{ 200 | "id": bucketName, 201 | "owner": owner, 202 | }) 203 | 204 | if err != nil { 205 | return nil, err 206 | } 207 | 208 | bucket, err := c.parseBucketFromResponse(body) 209 | if err != nil { 210 | return nil, err 211 | } 212 | 213 | if bucket.bucket.Id != bucketName { 214 | return nil, NewTerminalError("Bucket created with wrong name") 215 | } 216 | 217 | return bucket, err 218 | } 219 | 220 | type Bucket struct { 221 | client *ArtifactStoreClient 222 | bucket *model.Bucket 223 | } 224 | 225 | func (b *Bucket) parseArtifactFromResponse(body io.ReadCloser) (Artifact, *ArtifactsError) { 226 | bText, err := ioutil.ReadAll(body) 227 | if err != nil { 228 | return nil, NewRetriableError(err.Error()) 229 | } 230 | body.Close() 231 | 232 | artifact := new(model.Artifact) 233 | if err := json.Unmarshal(bText, artifact); err != nil { 234 | return nil, NewTerminalError(err.Error()) 235 | } 236 | 237 | return &ArtifactImpl{ 238 | artifact: artifact, 239 | bucket: b, 240 | }, nil 241 | } 242 | 243 | // Creates a new chunked artifact whose size does not have to be known. The name 244 | // acts as an id for the artifact. Because of additional overhead if the size is 245 | // already known then `NewStreamedArtifact` may be more applicable. 246 | func (b *Bucket) NewChunkedArtifact(name string) (*ChunkedArtifact, *ArtifactsError) { 247 | body, err := b.client.postAPIJSON(fmt.Sprintf("/buckets/%s/artifacts", b.bucket.Id), map[string]interface{}{ 248 | "chunked": true, 249 | "name": name, 250 | }) 251 | 252 | if err != nil { 253 | return nil, err 254 | } 255 | 256 | artifact, err := b.parseArtifactFromResponse(body) 257 | 258 | if err != nil { 259 | return nil, err 260 | } 261 | 262 | return (&ChunkedArtifact{ArtifactImpl: artifact.(*ArtifactImpl)}).init(), nil 263 | } 264 | 265 | // NewStreamedArtifact creates a new streamed (fixed-size) artifact given a file path and size. 266 | // The artifact name (which serves as its id) is computed from the file name (this is a hint to the 267 | // server which is free to modify the artifact name). 268 | // 269 | // The artifact does not actually get uploaded here - that will need to be perfomeed in 270 | // UploadArtifact. The artifact will only be complete when the server has received exactly "size" 271 | // bytes. This is only suitable for static content such as files. 272 | func (b *Bucket) NewStreamedArtifact(path string, size int64) (*StreamedArtifact, *ArtifactsError) { 273 | name := filepath.Base(path) 274 | body, err := b.client.postAPIJSON(fmt.Sprintf("/buckets/%s/artifacts", b.bucket.Id), map[string]interface{}{ 275 | "chunked": false, 276 | "name": name, 277 | "size": size, 278 | "relativePath": path, 279 | }) 280 | 281 | if err != nil { 282 | return nil, err 283 | } 284 | 285 | artifact, err := b.parseArtifactFromResponse(body) 286 | 287 | if err != nil { 288 | return nil, err 289 | } 290 | 291 | return &StreamedArtifact{ 292 | ArtifactImpl: artifact.(*ArtifactImpl), 293 | }, nil 294 | } 295 | 296 | func (b *Bucket) GetArtifact(name string) (Artifact, *ArtifactsError) { 297 | body, err := b.client.getAPI(fmt.Sprintf("/buckets/%s/artifacts/%s", b.bucket.Id, name)) 298 | 299 | if err != nil { 300 | return nil, err 301 | } 302 | 303 | return b.parseArtifactFromResponse(body) 304 | } 305 | 306 | func (b *Bucket) ListArtifacts() ([]Artifact, *ArtifactsError) { 307 | body, err := b.client.getAPI(fmt.Sprintf("/buckets/%s/artifacts/", b.bucket.Id)) 308 | if err != nil { 309 | return nil, err 310 | } 311 | 312 | return b.parseArtifactListFromResponse(body) 313 | } 314 | 315 | func (b *Bucket) parseArtifactListFromResponse(body io.ReadCloser) ([]Artifact, *ArtifactsError) { 316 | bText, err := ioutil.ReadAll(body) 317 | if err != nil { 318 | return nil, NewRetriableError(err.Error()) 319 | } 320 | body.Close() 321 | 322 | artifacts := []model.Artifact{} 323 | if err := json.Unmarshal(bText, &artifacts); err != nil { 324 | return nil, NewTerminalError(err.Error()) 325 | } 326 | 327 | wrappedArtifacts := make([]Artifact, len(artifacts)) 328 | for i, _ := range artifacts { 329 | wrappedArtifacts[i] = &ArtifactImpl{ 330 | artifact: &artifacts[i], 331 | bucket: b, 332 | } 333 | } 334 | 335 | return wrappedArtifacts, nil 336 | } 337 | 338 | func (b *Bucket) Close() *ArtifactsError { 339 | return ignoreBody(b.client.postAPIJSON(fmt.Sprintf("/buckets/%s/close", b.bucket.Id), map[string]interface{}{})) 340 | } 341 | 342 | type Artifact interface { 343 | // Returns a read-only copy of the raw model.Artifact instance associated with the artifact 344 | GetArtifactModel() *model.Artifact 345 | 346 | // Returns a handle to the bucket containing the artifact 347 | GetBucket() *Bucket 348 | 349 | // Return raw contents of the artifact (artifact file or text of a log stream) 350 | // as an io.ReadCloser. It is the responsibility of the caller to close the 351 | // io.ReadCloser 352 | GetContent() (io.ReadCloser, *ArtifactsError) 353 | 354 | // Returns a direct link to the raw contents of this artifact 355 | GetContentURL() string 356 | } 357 | 358 | type ArtifactImpl struct { 359 | artifact *model.Artifact 360 | bucket *Bucket 361 | } 362 | 363 | func (ai *ArtifactImpl) GetArtifactModel() *model.Artifact { 364 | return ai.artifact 365 | } 366 | 367 | func (ai *ArtifactImpl) GetBucket() *Bucket { 368 | return ai.bucket 369 | } 370 | 371 | // GetContentURL returns a direct link to the raw contents of an artifact 372 | func (ai *ArtifactImpl) GetContentURL() string { 373 | return fmt.Sprintf("%s/buckets/%s/artifacts/%s/content", ai.bucket.client.server, ai.bucket.bucket.Id, ai.artifact.Name) 374 | } 375 | 376 | // A chunked artifact is one which can be sent in chunks of 377 | // varying size. It is only complete upon the client manually 378 | // telling the server that it is complete, and is useful for 379 | // logs and other other artifacts whose size is not known 380 | // at the same they are streaming. 381 | type ChunkedArtifact struct { 382 | *ArtifactImpl 383 | offset int 384 | bytestream chan []byte 385 | fatalErr chan *ArtifactsError 386 | complete chan bool 387 | } 388 | 389 | func (artifact *ChunkedArtifact) init() *ChunkedArtifact { 390 | artifact.offset = 0 391 | artifact.bytestream = make(chan []byte, MAX_PENDING_REPORTS) 392 | artifact.complete = make(chan bool) 393 | artifact.fatalErr = make(chan *ArtifactsError) 394 | go artifact.pushLogChunks() 395 | 396 | return artifact 397 | } 398 | 399 | func (artifact *ChunkedArtifact) Flush() *ArtifactsError { 400 | close(artifact.bytestream) 401 | 402 | select { 403 | case <-artifact.bucket.client.ctx.Done(): 404 | return NewTerminalError(artifact.bucket.client.ctx.Err().Error()) 405 | case err := <-artifact.fatalErr: 406 | return err 407 | case _ = <-artifact.complete: 408 | // Recreate the stream and start pushing again. 409 | artifact.bytestream = make(chan []byte, MAX_PENDING_REPORTS) 410 | go artifact.pushLogChunks() 411 | return nil 412 | } 413 | } 414 | 415 | // A streamed artifact is a fixed-size artifact whose size 416 | // is known at the start of the transfer. It is therefore 417 | // not suitable for logs but useful for static files, etc. 418 | type StreamedArtifact struct { 419 | *ArtifactImpl 420 | } 421 | 422 | func newTicker() *backoff.Ticker { 423 | b := backoff.NewExponentialBackOff() 424 | b.InitialInterval = 100 * time.Millisecond 425 | b.MaxInterval = 5 * time.Second 426 | b.MaxElapsedTime = 15 * time.Second 427 | 428 | return backoff.NewTicker(b) 429 | } 430 | 431 | func (artifact *ChunkedArtifact) pushLogChunks() { 432 | var err *ArtifactsError 433 | for logChunk := range artifact.bytestream { 434 | ticker := newTicker() 435 | for { 436 | // If our parent context has been cancelled, we discard state and get out. 437 | select { 438 | case <-ticker.C: 439 | case <-artifact.bucket.client.ctx.Done(): 440 | log.Println("Client context has closed during log upload. Bailing out without any further log upload operations.") 441 | ticker.Stop() 442 | return 443 | } 444 | 445 | err = ignoreBody(artifact.bucket.client.postAPIJSON(fmt.Sprintf("/buckets/%s/artifacts/%s", artifact.bucket.bucket.Id, artifact.artifact.Name), map[string]interface{}{ 446 | "size": len(logChunk), 447 | "bytes": logChunk, 448 | "byteoffset": artifact.offset, 449 | })) 450 | 451 | if err != nil { 452 | if err.IsRetriable() { 453 | // Let's retry the request after a backoff 454 | continue 455 | } else { 456 | // This either means the server lost some of our logchunks (because of a rollback), or a 457 | // proxy timeout caused the previous request to succeed at the server but the proxy 458 | // returned an error. 459 | // 460 | // TODO: We should try to salvage the situation by seeing if we can skip a logchunk. 461 | // For now, bail. 462 | artifact.fatalErr <- err 463 | return 464 | } 465 | } 466 | 467 | artifact.offset += len(logChunk) 468 | ticker.Stop() 469 | break 470 | } 471 | } 472 | 473 | if err != nil { 474 | artifact.fatalErr <- err 475 | } else { 476 | artifact.complete <- true 477 | } 478 | } 479 | 480 | // Appends the log chunk to the stream. This is asynchronous so any errors 481 | // in sending will occur when closing the artifact. 482 | func (artifact *ChunkedArtifact) AppendLog(chunk string) *ArtifactsError { 483 | // TODO: There is no reason for this method to take in a string anymore. 484 | // Update to use []byte or support io.Writer interface. 485 | artifact.bytestream <- []byte(chunk) 486 | 487 | return nil 488 | } 489 | 490 | func (a *StreamedArtifact) UploadArtifact(stream io.Reader) *ArtifactsError { 491 | url := fmt.Sprintf("/buckets/%s/artifacts/%s", a.bucket.bucket.Id, a.artifact.Name) 492 | ticker := newTicker() 493 | defer ticker.Stop() 494 | 495 | for { 496 | // If our parent context has been cancelled, we discard state and get out. 497 | select { 498 | case <-ticker.C: 499 | case <-a.bucket.client.ctx.Done(): 500 | return NewTerminalError("Client context has closed during artifact upload. Bailing out without any further retries.") 501 | } 502 | 503 | err := ignoreBody(a.bucket.client.postAPI(url, "application/octet-stream", stream)) 504 | 505 | if err == nil { 506 | // TODO: Verify that the artifact that was stored matches the one we just uploaded. 507 | return nil 508 | } 509 | 510 | if err.IsRetriable() { 511 | // Let's retry the request after a backoff 512 | continue 513 | } else { 514 | return err 515 | } 516 | } 517 | } 518 | 519 | func (a *ChunkedArtifact) Close() *ArtifactsError { 520 | if err := a.Flush(); err != nil { 521 | return err 522 | } 523 | 524 | return ignoreBody(a.bucket.client.postAPIJSON(fmt.Sprintf("/buckets/%s/artifacts/%s/close", a.bucket.bucket.Id, a.artifact.Name), map[string]interface{}{})) 525 | } 526 | 527 | func (a ArtifactImpl) GetContent() (io.ReadCloser, *ArtifactsError) { 528 | url := fmt.Sprintf("/buckets/%s/artifacts/%s/content", a.bucket.bucket.Id, a.artifact.Name) 529 | return a.bucket.client.getAPI(url) 530 | } 531 | -------------------------------------------------------------------------------- /client/client_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "bytes" 5 | "database/sql" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "net/http" 10 | "strconv" 11 | "strings" 12 | "sync" 13 | "testing" 14 | "time" 15 | 16 | "gopkg.in/gorp.v1" 17 | 18 | "github.com/dropbox/changes-artifacts/database" 19 | "github.com/dropbox/changes-artifacts/model" 20 | _ "github.com/lib/pq" 21 | "github.com/rubenv/sql-migrate" 22 | "github.com/stretchr/testify/require" 23 | ) 24 | 25 | func setup(tb testing.TB) *ArtifactStoreClient { 26 | url := "http://localhost:3000" 27 | 28 | // Always wait for server to be running first, before setting up the DB. 29 | // Otherwise, our surgery with the DB will interfere with any setup being done on the server 30 | // during startup. 31 | waitForServer(tb, url) 32 | setupDB(tb) 33 | 34 | return NewArtifactStoreClient(url) 35 | } 36 | 37 | func waitForServer(tb testing.TB, url string) { 38 | retries := 1000 39 | 40 | for retries > 0 { 41 | if _, err := http.Get(url); err != nil { 42 | retries-- 43 | time.Sleep(10 * time.Millisecond) 44 | } else { 45 | fmt.Println("*********** SERVER ONLINE ***********") 46 | return 47 | } 48 | } 49 | 50 | tb.Fatalf("Artifacts server did not come up in time") 51 | } 52 | 53 | func setupDB(tb testing.TB) { 54 | DB_STR := "postgres://artifacts:artifacts@artifactsdb/artifacts?sslmode=disable" 55 | 56 | db, err := sql.Open("postgres", DB_STR) 57 | if err != nil { 58 | tb.Fatalf("Error connecting to Postgres: %s", err) 59 | } 60 | 61 | dbmap := &gorp.DbMap{Db: db, Dialect: gorp.PostgresDialect{}} 62 | gdb := database.NewGorpDatabase(dbmap) 63 | gdb.RegisterEntities() 64 | 65 | migrations := &migrate.AssetMigrationSource{ 66 | Asset: database.Asset, 67 | AssetDir: database.AssetDir, 68 | Dir: "migrations", 69 | } 70 | 71 | if n, err := migrate.Exec(db, "postgres", migrations, migrate.Down); err != nil { 72 | tb.Fatalf("Error resetting Postgres DB: %s", err) 73 | } else { 74 | fmt.Printf("Completed %d DOWN migrations\n", n) 75 | } 76 | 77 | // `maxMigrations` below is the maximum number of migration levels to be performed. 78 | // This is left here to make it easy to verify that database upgrades are backwards compatible. 79 | // After a new migration is added, this number should be bumped up. 80 | const maxMigrations = 4 81 | if n, err := migrate.ExecMax(db, "postgres", migrations, migrate.Up, maxMigrations); err != nil { 82 | tb.Fatalf("Error recreating Postgres DB: %s", err) 83 | } else { 84 | fmt.Printf("Completed %d UP migrations\n", n) 85 | } 86 | 87 | fmt.Println("************* DB RESET **************") 88 | } 89 | 90 | func TestCreateAndGetBucket(t *testing.T) { 91 | if testing.Short() { 92 | t.Skip("Skipping end-to-end test in short mode.") 93 | } 94 | 95 | client := setup(t) 96 | 97 | var bucket *Bucket 98 | var err error 99 | 100 | bucket, err = client.NewBucket("", "ownername", 31) 101 | require.Nil(t, bucket) 102 | require.Error(t, err) 103 | 104 | bucket, err = client.NewBucket("bucketname", "", 31) 105 | require.Nil(t, bucket) 106 | require.Error(t, err) 107 | 108 | bucket, err = client.NewBucket("bucketname", "ownername", 31) 109 | require.NotNil(t, bucket) 110 | require.Equal(t, "bucketname", bucket.bucket.Id) 111 | require.Equal(t, "ownername", bucket.bucket.Owner) 112 | require.Equal(t, model.OPEN, bucket.bucket.State) 113 | require.NoError(t, err) 114 | 115 | // Duplicate 116 | bucket, err = client.NewBucket("bucketname", "ownername", 31) 117 | require.Nil(t, bucket) 118 | require.Error(t, err) 119 | 120 | bucket, err = client.GetBucket("bucketname") 121 | require.NotNil(t, bucket) 122 | require.Equal(t, "bucketname", bucket.bucket.Id) 123 | require.Equal(t, "ownername", bucket.bucket.Owner) 124 | require.Equal(t, model.OPEN, bucket.bucket.State) 125 | require.NoError(t, err) 126 | 127 | bucket, err = client.GetBucket("bucketnotfound") 128 | require.Nil(t, bucket) 129 | require.Error(t, err) 130 | } 131 | 132 | func TestCreateAndGetChunkedArtifact(t *testing.T) { 133 | bucketName := "bucketName" 134 | ownerName := "ownerName" 135 | artifactName := "artifactName" 136 | 137 | if testing.Short() { 138 | t.Skip("Skipping end-to-end test in short mode.") 139 | } 140 | 141 | client := setup(t) 142 | 143 | var bucket *Bucket 144 | var artifact Artifact 145 | var err error 146 | 147 | bucket, err = client.NewBucket(bucketName, ownerName, 31) 148 | require.NotNil(t, bucket) 149 | require.NoError(t, err) 150 | 151 | artifact, err = bucket.NewChunkedArtifact(artifactName) 152 | require.NotNil(t, artifact) 153 | require.Equal(t, artifactName, artifact.GetArtifactModel().Name) 154 | require.Equal(t, model.APPENDING, artifact.GetArtifactModel().State) 155 | // require.Equal will crib if the types are not identical. 156 | require.Equal(t, int64(0), artifact.GetArtifactModel().Size) 157 | require.Equal(t, bucketName, artifact.GetArtifactModel().BucketId) 158 | require.Empty(t, artifact.GetArtifactModel().S3URL) 159 | require.NoError(t, err) 160 | 161 | artifact, err = bucket.GetArtifact(artifactName) 162 | require.NotNil(t, artifact) 163 | require.Equal(t, artifactName, artifact.GetArtifactModel().Name) 164 | require.Equal(t, model.APPENDING, artifact.GetArtifactModel().State) 165 | // require.Equal will crib if the types are not identical. 166 | require.Equal(t, int64(0), artifact.GetArtifactModel().Size) 167 | require.Equal(t, bucketName, artifact.GetArtifactModel().BucketId) 168 | require.Empty(t, artifact.GetArtifactModel().S3URL) 169 | require.NoError(t, err) 170 | } 171 | 172 | func TestCreateAndGetStreamedArtifact(t *testing.T) { 173 | bucketName := "bucketName" 174 | ownerName := "ownerName" 175 | artifactName := "artifactName" 176 | fileSize := int64(100) 177 | 178 | if testing.Short() { 179 | t.Skip("Skipping end-to-end test in short mode.") 180 | } 181 | 182 | client := setup(t) 183 | 184 | var bucket *Bucket 185 | var artifact Artifact 186 | var err error 187 | 188 | bucket, err = client.NewBucket(bucketName, ownerName, 31) 189 | require.NotNil(t, bucket) 190 | require.NoError(t, err) 191 | 192 | artifact, err = bucket.NewStreamedArtifact(artifactName, fileSize) 193 | require.NotNil(t, artifact) 194 | require.Equal(t, artifactName, artifact.GetArtifactModel().Name) 195 | require.Equal(t, model.WAITING_FOR_UPLOAD, artifact.GetArtifactModel().State) 196 | require.Equal(t, fileSize, artifact.GetArtifactModel().Size) 197 | require.Equal(t, bucketName, artifact.GetArtifactModel().BucketId) 198 | require.Empty(t, artifact.GetArtifactModel().S3URL) 199 | require.NoError(t, err) 200 | 201 | artifact, err = bucket.GetArtifact(artifactName) 202 | require.NotNil(t, artifact) 203 | require.Equal(t, artifactName, artifact.GetArtifactModel().Name) 204 | require.Equal(t, model.WAITING_FOR_UPLOAD, artifact.GetArtifactModel().State) 205 | require.Equal(t, fileSize, artifact.GetArtifactModel().Size) 206 | require.Equal(t, bucketName, artifact.GetArtifactModel().BucketId) 207 | require.Empty(t, artifact.GetArtifactModel().S3URL) 208 | require.NoError(t, err) 209 | } 210 | 211 | func TestCloseBucket(t *testing.T) { 212 | bucketName := "bucketName" 213 | ownerName := "ownerName" 214 | chunkedArtifactName := "chunkedArt" 215 | streamedArtifactName := "streamedArt" 216 | fileSize := int64(100) 217 | 218 | if testing.Short() { 219 | t.Skip("Skipping end-to-end test in short mode.") 220 | } 221 | 222 | client := setup(t) 223 | 224 | var bucket *Bucket 225 | var artifact Artifact 226 | var err error 227 | 228 | bucket, err = client.NewBucket(bucketName, ownerName, 31) 229 | require.NotNil(t, bucket) 230 | require.NoError(t, err) 231 | 232 | artifact, err = bucket.NewChunkedArtifact(chunkedArtifactName) 233 | require.NotNil(t, artifact) 234 | require.NoError(t, err) 235 | 236 | artifact, err = bucket.NewStreamedArtifact(streamedArtifactName, fileSize) 237 | require.NotNil(t, artifact) 238 | require.NoError(t, err) 239 | 240 | require.NoError(t, bucket.Close()) 241 | 242 | // Verify bucket is closed 243 | bucket, err = client.GetBucket(bucketName) 244 | require.NotNil(t, bucket) 245 | require.Equal(t, bucketName, bucket.bucket.Id) 246 | require.Equal(t, ownerName, bucket.bucket.Owner) 247 | require.Equal(t, model.CLOSED, bucket.bucket.State) 248 | require.NoError(t, err) 249 | 250 | // Verify chunked artifact is closed 251 | artifact, err = bucket.GetArtifact(chunkedArtifactName) 252 | require.NotNil(t, artifact) 253 | require.Equal(t, chunkedArtifactName, artifact.GetArtifactModel().Name) 254 | // Since we didn't write any content to the streamed artifact, it should be closed without data. 255 | require.Equal(t, model.CLOSED_WITHOUT_DATA, artifact.GetArtifactModel().State) 256 | // require.Equal will crib if the types are not identical. 257 | require.Equal(t, int64(0), artifact.GetArtifactModel().Size) 258 | require.Equal(t, bucketName, artifact.GetArtifactModel().BucketId) 259 | require.Empty(t, artifact.GetArtifactModel().S3URL) 260 | require.NoError(t, err) 261 | 262 | // Verify streamed artifact is closed 263 | artifact, err = bucket.GetArtifact(streamedArtifactName) 264 | require.NotNil(t, artifact) 265 | require.Equal(t, streamedArtifactName, artifact.GetArtifactModel().Name) 266 | require.Equal(t, model.CLOSED_WITHOUT_DATA, artifact.GetArtifactModel().State) 267 | require.Equal(t, fileSize, artifact.GetArtifactModel().Size) 268 | require.Equal(t, bucketName, artifact.GetArtifactModel().BucketId) 269 | require.Empty(t, artifact.GetArtifactModel().S3URL) 270 | require.NoError(t, err) 271 | 272 | // Adding artifacts to a closed bucket 273 | artifact, err = bucket.NewChunkedArtifact("artifact") 274 | require.Nil(t, artifact) 275 | require.Error(t, err) 276 | 277 | // Adding artifacts to a closed bucket 278 | artifact, err = bucket.NewStreamedArtifact("artifact", 200) 279 | require.Nil(t, artifact) 280 | require.Error(t, err) 281 | } 282 | 283 | func TestAppendToChunkedArtifact(t *testing.T) { 284 | bucketName := "bucketName" 285 | ownerName := "ownerName" 286 | artifactName := "artifactName" 287 | 288 | if testing.Short() { 289 | t.Skip("Skipping end-to-end test in short mode.") 290 | } 291 | 292 | client := setup(t) 293 | 294 | var bucket *Bucket 295 | var artifact Artifact 296 | var cartifact *ChunkedArtifact 297 | var err error 298 | 299 | bucket, err = client.NewBucket(bucketName, ownerName, 31) 300 | require.NotNil(t, bucket) 301 | require.NoError(t, err) 302 | 303 | cartifact, err = bucket.NewChunkedArtifact(artifactName) 304 | require.NotNil(t, cartifact) 305 | require.NoError(t, err) 306 | 307 | // Append chunk to artifact. 308 | require.NoError(t, cartifact.AppendLog("0123456789")) 309 | 310 | // Append another chunk to artifact. 311 | require.NoError(t, cartifact.AppendLog("9876543210")) 312 | 313 | // Flush and verify contents. 314 | require.NoError(t, cartifact.Flush()) 315 | 316 | artifact, err = bucket.GetArtifact(artifactName) 317 | require.NotNil(t, artifact) 318 | require.Equal(t, model.APPENDING, artifact.GetArtifactModel().State) 319 | // require.Equal will crib if the types are not identical. 320 | require.Equal(t, int64(20), artifact.GetArtifactModel().Size) 321 | require.Equal(t, bucketName, artifact.GetArtifactModel().BucketId) 322 | require.Empty(t, artifact.GetArtifactModel().S3URL) 323 | require.NoError(t, err) 324 | 325 | // Append yet another chunk to artifact. 326 | require.NoError(t, cartifact.AppendLog("9876543210")) 327 | // Close the artifact 328 | require.NoError(t, cartifact.Close()) 329 | 330 | // Verify it exists on S3 331 | artifact, err = bucket.GetArtifact(artifactName) 332 | require.NotNil(t, artifact) 333 | require.Equal(t, model.UPLOADED, artifact.GetArtifactModel().State) 334 | // require.Equal will crib if the types are not identical. 335 | require.Equal(t, int64(30), artifact.GetArtifactModel().Size) 336 | require.Equal(t, bucketName, artifact.GetArtifactModel().BucketId) 337 | require.Equal(t, fmt.Sprintf("/%s/%s", bucketName, artifactName), artifact.GetArtifactModel().S3URL) 338 | require.NoError(t, err) 339 | 340 | content, err := cartifact.GetContent() 341 | require.NoError(t, err) 342 | var buf bytes.Buffer 343 | buf.ReadFrom(content) 344 | require.Equal(t, 30, buf.Len()) 345 | } 346 | 347 | func TestChunkedArtifactWriteIncompleteUnicodeCharacters(t *testing.T) { 348 | const bucketName = "bucketName" 349 | const ownerName = "ownerName" 350 | const artifactName = "artifactName" 351 | 352 | if testing.Short() { 353 | t.Skip("Skipping end-to-end test in short mode.") 354 | } 355 | 356 | client := setup(t) 357 | 358 | bucket, err := client.NewBucket(bucketName, ownerName, 31) 359 | require.NotNil(t, bucket) 360 | require.NoError(t, err) 361 | 362 | // Append bytes to artifacts. 363 | cartifact, err := bucket.NewChunkedArtifact(artifactName) 364 | require.NotNil(t, cartifact) 365 | require.NoError(t, err) 366 | 367 | { 368 | var buf bytes.Buffer 369 | buf.WriteRune('☃') // Rune of length 3 bytes 370 | require.NoError(t, cartifact.AppendLog(string(buf.Next(2)))) // Send only first two bytes 371 | require.NoError(t, cartifact.AppendLog(string(buf.Next(1)))) // Send last byte 372 | require.NoError(t, cartifact.Flush()) 373 | } 374 | 375 | { 376 | reader, err := cartifact.GetContent() 377 | require.NoError(t, err) 378 | 379 | var buf bytes.Buffer 380 | buf.ReadFrom(reader) 381 | r, size, buferr := buf.ReadRune() 382 | require.NoError(t, buferr) 383 | require.Equal(t, '☃', r) 384 | require.Equal(t, 3, size) 385 | } 386 | } 387 | 388 | func TestChunkedArtifactBytes(t *testing.T) { 389 | const bucketName = "bucketName" 390 | const ownerName = "ownerName" 391 | const artifactName = "artifactName" 392 | 393 | if testing.Short() { 394 | t.Skip("Skipping end-to-end test in short mode.") 395 | } 396 | 397 | client := setup(t) 398 | 399 | bucket, err := client.NewBucket(bucketName, ownerName, 31) 400 | require.NotNil(t, bucket) 401 | require.NoError(t, err) 402 | 403 | // Append bytes to artifacts. 404 | for i := 0; i <= 255; i++ { 405 | cartifact, err := bucket.NewChunkedArtifact(artifactName + strconv.Itoa(i)) 406 | require.NotNil(t, cartifact) 407 | require.NoError(t, err) 408 | require.NoError(t, cartifact.AppendLog(string(byte(i)))) 409 | require.NoError(t, cartifact.Flush()) 410 | } 411 | } 412 | 413 | func TestPostStreamedArtifact(t *testing.T) { 414 | bucketName := "bucketName" 415 | ownerName := "ownerName" 416 | artifactName := "postedStreamedArtifactName" 417 | 418 | if testing.Short() { 419 | t.Skip("Skipping end-to-end test in short mode.") 420 | } 421 | 422 | client := setup(t) 423 | 424 | var bucket *Bucket 425 | var artifact Artifact 426 | var streamedArtifact *StreamedArtifact 427 | var err error 428 | 429 | bucket, err = client.NewBucket(bucketName, ownerName, 31) 430 | require.NotNil(t, bucket) 431 | require.NoError(t, err) 432 | 433 | streamedArtifact, err = bucket.NewStreamedArtifact(artifactName, 30) 434 | require.NotNil(t, streamedArtifact) 435 | require.NoError(t, err) 436 | 437 | rd := bytes.NewReader([]byte("012345678998765432100123456789")) 438 | // Append chunk to artifact. 439 | require.NoError(t, streamedArtifact.UploadArtifact(rd)) 440 | 441 | artifact, err = bucket.GetArtifact(artifactName) 442 | require.NotNil(t, artifact) 443 | require.Equal(t, model.UPLOADED, artifact.GetArtifactModel().State) 444 | // require.Equal will crib if the types are not identical. 445 | require.Equal(t, int64(30), artifact.GetArtifactModel().Size) 446 | require.Equal(t, bucketName, artifact.GetArtifactModel().BucketId) 447 | require.Equal(t, fmt.Sprintf("/%s/%s", bucketName, artifactName), artifact.GetArtifactModel().S3URL) 448 | require.NoError(t, err) 449 | 450 | // Verify it exists on S3 451 | content, err := streamedArtifact.GetContent() 452 | require.NoError(t, err) 453 | var buf bytes.Buffer 454 | buf.ReadFrom(content) 455 | require.Equal(t, 30, buf.Len()) 456 | } 457 | 458 | func TestCreateAndListArtifacts(t *testing.T) { 459 | bucketName := "bucketName" 460 | ownerName := "ownerName" 461 | artifactName1 := "artifactName1" 462 | artifactName2 := "artifactName2" 463 | 464 | if testing.Short() { 465 | t.Skip("Skipping end-to-end test in short mode.") 466 | } 467 | 468 | client := setup(t) 469 | 470 | var bucket *Bucket 471 | var streamedArtifact *StreamedArtifact 472 | var chunkedArtifact *ChunkedArtifact 473 | var err error 474 | var artifacts []Artifact 475 | 476 | bucket, err = client.NewBucket(bucketName, ownerName, 31) 477 | require.NotNil(t, bucket) 478 | require.NoError(t, err) 479 | 480 | streamedArtifact, err = bucket.NewStreamedArtifact(artifactName1, 30) 481 | require.NotNil(t, streamedArtifact) 482 | require.NoError(t, err) 483 | 484 | chunkedArtifact, err = bucket.NewChunkedArtifact(artifactName2) 485 | require.NotNil(t, chunkedArtifact) 486 | require.NoError(t, err) 487 | 488 | artifacts, err = bucket.ListArtifacts() 489 | require.NotNil(t, artifacts) 490 | require.Nil(t, err) 491 | require.Len(t, artifacts, 2) 492 | 493 | // We really shouldn't worry about order here. 494 | require.Equal(t, artifactName1, artifacts[0].GetArtifactModel().Name) 495 | require.Equal(t, artifactName2, artifacts[1].GetArtifactModel().Name) 496 | } 497 | 498 | func TestCreateDuplicateArtifactRace(t *testing.T) { 499 | bucketName := "bucketName" 500 | ownerName := "ownerName" 501 | artifactName := "artifactName" 502 | 503 | if testing.Short() { 504 | t.Skip("Skipping end-to-end test in short mode.") 505 | } 506 | 507 | client := setup(t) 508 | 509 | bucket, err := client.NewBucket(bucketName, ownerName, 31) 510 | require.NotNil(t, bucket) 511 | require.NoError(t, err) 512 | 513 | // More parallelism will hit postgres connection limit. 514 | // http://www.postgresql.org/docs/current/static/runtime-config-connection.html#GUC-MAX-CONNECTIONS 515 | createdArtifactNames := make([]string, 20) 516 | 517 | // Create many duplicates 518 | var wg sync.WaitGroup 519 | 520 | for i := range createdArtifactNames { 521 | wg.Add(1) 522 | go func(counter int) { 523 | defer wg.Done() 524 | artifact, e := bucket.NewStreamedArtifact(artifactName, 1) 525 | require.NoError(t, e) 526 | require.NotNil(t, artifact) 527 | createdArtifactNames[counter] = artifact.GetArtifactModel().Name 528 | }(i) 529 | } 530 | 531 | wg.Wait() 532 | 533 | seen := make(map[string]bool) 534 | 535 | for _, created := range createdArtifactNames { 536 | if !strings.HasPrefix(created, artifactName) { 537 | t.Fatalf("Expected created artifact name to have prefix '%s', got '%s'", artifactName, created) 538 | } 539 | 540 | if seen[created] { 541 | t.Fatalf("Duplicate artifact name: %s", created) 542 | } else { 543 | seen[created] = true 544 | } 545 | } 546 | } 547 | 548 | func gen4KString() string { 549 | var buf bytes.Buffer 550 | for i := 0; i < 400; i++ { 551 | buf.WriteString("0123456789") 552 | } 553 | return buf.String() 554 | } 555 | 556 | func BenchmarkMergingArtifacts(b *testing.B) { 557 | bucketName := "bucketName" 558 | ownerName := "ownerName" 559 | artifactName := "artifactName" 560 | 561 | if testing.Short() { 562 | b.Skip("Skipping end-to-end test in short mode.") 563 | } 564 | 565 | client := setup(b) 566 | 567 | bucket, _ := client.NewBucket(bucketName, ownerName, 31) 568 | require.NotNil(b, bucket) 569 | 570 | artifact, _ := bucket.NewChunkedArtifact(artifactName) 571 | require.NotNil(b, artifact) 572 | 573 | // Typical logchunk of size ~4KB 574 | str := gen4KString() 575 | 576 | // Typical artifact of ~1000 logchunks 577 | for i := 0; i < 1000; i++ { 578 | artifact.AppendLog(str) 579 | } 580 | // Make sure all logchunks are sent to API/DB. 581 | artifact.Flush() 582 | 583 | // Exercise logchunk stitching flow. 584 | // Both fetching content and merging before writing to S3 use same logic to fetch chunks. 585 | // 586 | // 1000x4K chunks runs in ~20ms on a 4-core laptop (with synchronous_commit=off,fsync=off in 587 | // Postgres) 588 | b.ResetTimer() 589 | for i := 0; i < b.N; i++ { 590 | reader, _ := artifact.GetContent() 591 | io.Copy(ioutil.Discard, reader) 592 | } 593 | } 594 | 595 | func BenchmarkAppendLog(b *testing.B) { 596 | bucketName := "bucketName" 597 | ownerName := "ownerName" 598 | artifactName := "artifactName" 599 | 600 | if testing.Short() { 601 | b.Skip("Skipping end-to-end test in short mode.") 602 | } 603 | 604 | client := setup(b) 605 | 606 | bucket, _ := client.NewBucket(bucketName, ownerName, 31) 607 | require.NotNil(b, bucket) 608 | 609 | artifact, _ := bucket.NewChunkedArtifact(artifactName) 610 | require.NotNil(b, artifact) 611 | 612 | // Typical logchunk of size ~4KB 613 | str := gen4KString() 614 | 615 | // Each 4K chunks sent takes ~1.3ms on a 4-core laptop (with synchronous_commit=off,fsync=off in 616 | // Postgres) 617 | b.ResetTimer() 618 | for i := 0; i < b.N; i++ { 619 | artifact.AppendLog(str) 620 | } 621 | // Make sure all logchunks are sent to API/DB. 622 | artifact.Flush() 623 | } 624 | -------------------------------------------------------------------------------- /api/artifacthandler_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net/http" 7 | "net/http/httptest" 8 | "testing" 9 | 10 | "golang.org/x/net/context" 11 | 12 | "gopkg.in/amz.v1/aws" 13 | "gopkg.in/amz.v1/s3" 14 | "gopkg.in/amz.v1/s3/s3test" 15 | 16 | "github.com/dropbox/changes-artifacts/common/sentry" 17 | "github.com/dropbox/changes-artifacts/database" 18 | "github.com/dropbox/changes-artifacts/model" 19 | "github.com/stretchr/testify/mock" 20 | "github.com/stretchr/testify/require" 21 | ) 22 | 23 | func getS3Bucket(t *testing.T, url string, doCreate bool) *s3.Bucket { 24 | s3Client := s3.New(aws.Auth{AccessKey: "abc", SecretKey: "123"}, aws.Region{ 25 | Name: "fake-artifacts-test-region", 26 | S3Endpoint: url, 27 | S3LocationConstraint: true, 28 | Sign: aws.SignV2, 29 | }) 30 | 31 | s3Bucket := s3Client.Bucket("fake-artifacts-store-bucket") 32 | if doCreate { 33 | if err := s3Bucket.PutBucket(s3.Private); err != nil { 34 | t.Fatalf("Error creating s3 bucket: %s\n", err) 35 | } 36 | } 37 | 38 | return s3Bucket 39 | } 40 | 41 | func fakeS3ServerWithBucket(t *testing.T, f http.HandlerFunc) (*httptest.Server, *s3.Bucket) { 42 | ts := httptest.NewServer(http.HandlerFunc(f)) 43 | t.Logf("Fake S3 Server up at %s\n", ts.URL) 44 | 45 | return ts, getS3Bucket(t, ts.URL, false) 46 | } 47 | 48 | func testS3ServerWithBucket(t *testing.T) (*s3test.Server, *s3.Bucket) { 49 | s3Server, err := s3test.NewServer(&s3test.Config{Send409Conflict: true}) 50 | if err != nil { 51 | t.Fatalf("Error bringing up fake s3 server: %s\n", err) 52 | } 53 | 54 | t.Logf("S3 Test Server up at %s\n", s3Server.URL()) 55 | 56 | return s3Server, getS3Bucket(t, s3Server.URL(), true) 57 | } 58 | 59 | func TestCreateArtifact(t *testing.T) { 60 | mockdb := &database.MockDatabase{} 61 | 62 | { 63 | // Create artifact with no name 64 | artifact, err := CreateArtifact(createArtifactReq{}, &model.Bucket{ 65 | State: model.CLOSED, 66 | }, mockdb) 67 | require.Nil(t, artifact) 68 | require.Error(t, err) 69 | } 70 | 71 | { 72 | // Create artifact in closed bucket 73 | artifact, err := CreateArtifact(createArtifactReq{ 74 | Name: "aName", 75 | }, &model.Bucket{ 76 | State: model.CLOSED, 77 | }, mockdb) 78 | require.Nil(t, artifact) 79 | require.Error(t, err) 80 | } 81 | 82 | // ---------- BEGIN Streamed artifact creation -------------- 83 | { 84 | artifact, err := CreateArtifact(createArtifactReq{ 85 | Name: "aName", 86 | Size: 0, // Invalid size 87 | Chunked: false, 88 | }, &model.Bucket{ 89 | State: model.OPEN, 90 | Id: "bName", 91 | }, mockdb) 92 | require.Nil(t, artifact) 93 | require.Error(t, err) 94 | } 95 | 96 | { 97 | artifact, err := CreateArtifact(createArtifactReq{ 98 | Name: "aName", 99 | Size: MaxArtifactSizeBytes + 1, // Invalid size 100 | Chunked: false, 101 | }, &model.Bucket{ 102 | State: model.OPEN, 103 | Id: "bName", 104 | }, mockdb) 105 | require.Nil(t, artifact) 106 | require.Error(t, err) 107 | } 108 | 109 | { 110 | // Fail inserting into DB once and also fail getting artifact 111 | mockdb.On("InsertArtifact", mock.AnythingOfType("*model.Artifact")).Return(database.WrapInternalDatabaseError(fmt.Errorf("Err"))).Once() 112 | mockdb.On("GetArtifactByName", "bName", "aName").Return(nil, database.WrapInternalDatabaseError(fmt.Errorf("Err"))).Once() 113 | artifact, err := CreateArtifact(createArtifactReq{ 114 | Name: "aName", 115 | Size: 10, 116 | Chunked: false, 117 | }, &model.Bucket{ 118 | State: model.OPEN, 119 | Id: "bName", 120 | }, mockdb) 121 | require.Nil(t, artifact) 122 | require.Error(t, err) 123 | require.Equal(t, http.StatusInternalServerError, err.errCode) 124 | mockdb.AssertExpectations(t) 125 | } 126 | 127 | { 128 | // Successfully create artifact 129 | mockdb.On("InsertArtifact", mock.AnythingOfType("*model.Artifact")).Return(nil).Once() 130 | artifact, err := CreateArtifact(createArtifactReq{ 131 | Name: "aName", 132 | Size: 10, 133 | Chunked: false, 134 | }, &model.Bucket{ 135 | State: model.OPEN, 136 | Id: "bName", 137 | }, mockdb) 138 | require.NoError(t, err) 139 | require.NotNil(t, artifact) 140 | require.Equal(t, "aName", artifact.Name) 141 | require.Equal(t, "bName", artifact.BucketId) 142 | require.Equal(t, model.WAITING_FOR_UPLOAD, artifact.State) 143 | require.Equal(t, int64(10), artifact.Size) 144 | mockdb.AssertExpectations(t) 145 | } 146 | // ---------- END Streamed artifact creation -------------- 147 | 148 | // ---------- BEGIN Chunked artifact creation -------------- 149 | { 150 | // Fail on inserting into DB 151 | mockdb.On("InsertArtifact", mock.AnythingOfType("*model.Artifact")).Return(database.WrapInternalDatabaseError(fmt.Errorf("Err"))).Once() 152 | mockdb.On("GetArtifactByName", "bName", "aName").Return(nil, database.WrapInternalDatabaseError(fmt.Errorf("Err"))).Once() 153 | artifact, err := CreateArtifact(createArtifactReq{ 154 | Name: "aName", 155 | Chunked: true, 156 | }, &model.Bucket{ 157 | State: model.OPEN, 158 | Id: "bName", 159 | }, mockdb) 160 | require.Error(t, err) 161 | require.Nil(t, artifact) 162 | mockdb.AssertExpectations(t) 163 | } 164 | 165 | { 166 | // Successfully create artifact 167 | mockdb.On("InsertArtifact", mock.AnythingOfType("*model.Artifact")).Return(nil).Once() 168 | artifact, err := CreateArtifact(createArtifactReq{ 169 | Name: "aName", 170 | Chunked: true, 171 | DeadlineMins: 20, 172 | }, &model.Bucket{ 173 | State: model.OPEN, 174 | Id: "bName", 175 | }, mockdb) 176 | require.NoError(t, err) 177 | require.NotNil(t, artifact) 178 | require.Equal(t, "aName", artifact.Name) 179 | require.Equal(t, "bName", artifact.BucketId) 180 | require.Equal(t, model.APPENDING, artifact.State) 181 | require.Equal(t, int64(0), artifact.Size) 182 | mockdb.AssertExpectations(t) 183 | } 184 | // ---------- END Streamed artifact creation -------------- 185 | 186 | // ---------- BEGIN Duplicate artifact name --------------- 187 | { 188 | mockdb.On("InsertArtifact", mock.AnythingOfType("*model.Artifact")).Return(database.MockDatabaseError()).Once() 189 | mockdb.On("GetArtifactByName", "bName", "aName").Return(&model.Artifact{}, nil).Once() 190 | mockdb.On("InsertArtifact", mock.AnythingOfType("*model.Artifact")).Return(nil).Once() 191 | artifact, err := CreateArtifact(createArtifactReq{ 192 | Name: "aName", 193 | Chunked: true, 194 | DeadlineMins: 20, 195 | }, &model.Bucket{ 196 | State: model.OPEN, 197 | Id: "bName", 198 | }, mockdb) 199 | require.NoError(t, err) 200 | require.NotNil(t, artifact) 201 | // "Random" string below is deterministically produced because we use a deterministic seed 202 | // during unit tests. (https://golang.org/pkg/math/rand/#Seed) 203 | require.Equal(t, "aName.dup.rfBd5", artifact.Name) 204 | require.Equal(t, "bName", artifact.BucketId) 205 | require.Equal(t, model.APPENDING, artifact.State) 206 | require.Equal(t, int64(0), artifact.Size) 207 | mockdb.AssertExpectations(t) 208 | } 209 | 210 | { 211 | mockdb.On("InsertArtifact", mock.AnythingOfType("*model.Artifact")).Return(database.MockDatabaseError()).Times(6) 212 | mockdb.On("GetArtifactByName", "bName", mock.Anything).Return(&model.Artifact{}, nil).Times(5) 213 | artifact, err := CreateArtifact(createArtifactReq{ 214 | Name: "aName", 215 | Chunked: true, 216 | DeadlineMins: 20, 217 | }, &model.Bucket{ 218 | State: model.OPEN, 219 | Id: "bName", 220 | }, mockdb) 221 | require.Nil(t, artifact) 222 | require.Error(t, err) 223 | mockdb.AssertExpectations(t) 224 | } 225 | // ---------- END Duplicate artifact name ----------------- 226 | } 227 | 228 | func getExpectedLogChunkToBeWritten() *model.LogChunk { 229 | return &model.LogChunk{ 230 | Size: 2, 231 | ArtifactId: 10, 232 | ContentBytes: []byte("ab"), 233 | } 234 | } 235 | 236 | func getLogChunkToAppend() *createLogChunkReq { 237 | return &createLogChunkReq{ 238 | Size: 2, 239 | Bytes: []byte("ab"), 240 | ByteOffset: 0, 241 | } 242 | } 243 | 244 | func TestAppendLogChunk(t *testing.T) { 245 | mockdb := &database.MockDatabase{} 246 | 247 | // Already completed artifact 248 | require.Error(t, AppendLogChunk(context.Background(), mockdb, &model.Artifact{ 249 | State: model.APPEND_COMPLETE, 250 | }, nil)) 251 | 252 | // Size 0 chunk 253 | require.Error(t, AppendLogChunk(context.Background(), mockdb, &model.Artifact{ 254 | State: model.APPENDING, 255 | }, &createLogChunkReq{ 256 | Size: 0, 257 | })) 258 | 259 | // Blank string chunk 260 | require.Error(t, AppendLogChunk(context.Background(), mockdb, &model.Artifact{ 261 | State: model.APPENDING, 262 | }, &createLogChunkReq{ 263 | Size: 1, 264 | Content: "", 265 | })) 266 | 267 | // Mismatch between size and content chunk 268 | require.Error(t, AppendLogChunk(context.Background(), mockdb, &model.Artifact{ 269 | State: model.APPENDING, 270 | }, &createLogChunkReq{ 271 | Content: "ab", 272 | Size: 1, 273 | })) 274 | 275 | // Last logchunk was repeated. 276 | mockdb.On("GetLastLogChunkSeenForArtifact", int64(10)).Return(&model.LogChunk{Size: 2, ContentBytes: []byte("ab")}, nil).Once() 277 | require.NoError(t, AppendLogChunk(context.Background(), mockdb, &model.Artifact{ 278 | State: model.APPENDING, 279 | Id: 10, 280 | Size: 2, 281 | }, getLogChunkToAppend())) 282 | 283 | // Unexpected logchunk 284 | mockdb.On("GetLastLogChunkSeenForArtifact", int64(10)).Return(&model.LogChunk{Size: 3}, nil).Once() 285 | require.Error(t, AppendLogChunk(context.Background(), mockdb, &model.Artifact{ 286 | State: model.APPENDING, 287 | Id: 10, 288 | Size: 2, 289 | }, getLogChunkToAppend())) 290 | 291 | mockdb.On("UpdateArtifact", mock.AnythingOfType("*model.Artifact")).Return(database.MockDatabaseError()).Once() 292 | require.Error(t, AppendLogChunk(context.Background(), mockdb, &model.Artifact{ 293 | State: model.APPENDING, 294 | Id: 10, 295 | Size: 0, 296 | }, getLogChunkToAppend())) 297 | 298 | mockdb.On("UpdateArtifact", mock.AnythingOfType("*model.Artifact")).Return(nil).Once() 299 | mockdb.On("InsertLogChunk", getExpectedLogChunkToBeWritten()).Return(database.MockDatabaseError()).Once() 300 | require.Error(t, AppendLogChunk(context.Background(), mockdb, &model.Artifact{ 301 | State: model.APPENDING, 302 | Id: 10, 303 | Size: 0, 304 | }, getLogChunkToAppend())) 305 | 306 | mockdb.On("UpdateArtifact", mock.AnythingOfType("*model.Artifact")).Return(nil).Once() 307 | mockdb.On("InsertLogChunk", getExpectedLogChunkToBeWritten()).Return(nil).Once() 308 | require.NoError(t, AppendLogChunk(context.Background(), mockdb, &model.Artifact{ 309 | State: model.APPENDING, 310 | Id: 10, 311 | Size: 0, 312 | }, getLogChunkToAppend())) 313 | 314 | mockdb.AssertExpectations(t) 315 | } 316 | 317 | func TestPutArtifactErrorChecks(t *testing.T) { 318 | // Chunked artifacts 319 | require.Error(t, PutArtifact(context.Background(), &model.Artifact{State: model.APPENDING}, nil, nil, PutArtifactReq{})) 320 | require.Error(t, PutArtifact(context.Background(), &model.Artifact{State: model.APPEND_COMPLETE}, nil, nil, PutArtifactReq{})) 321 | 322 | // Already being uploaded elsewhere 323 | require.Error(t, PutArtifact(context.Background(), &model.Artifact{State: model.UPLOADING}, nil, nil, PutArtifactReq{})) 324 | 325 | // Already completed upload 326 | require.Error(t, PutArtifact(context.Background(), &model.Artifact{State: model.UPLOADED}, nil, nil, PutArtifactReq{})) 327 | 328 | require.Error(t, PutArtifact(context.Background(), &model.Artifact{State: model.WAITING_FOR_UPLOAD}, nil, nil, PutArtifactReq{ 329 | ContentLength: "", 330 | })) 331 | 332 | require.Error(t, PutArtifact(context.Background(), &model.Artifact{State: model.WAITING_FOR_UPLOAD}, nil, nil, PutArtifactReq{ 333 | ContentLength: "foo", 334 | })) 335 | 336 | // Size mismatch 337 | require.Error(t, PutArtifact(context.Background(), &model.Artifact{State: model.WAITING_FOR_UPLOAD, Size: 10}, nil, nil, PutArtifactReq{ 338 | ContentLength: "20", 339 | })) 340 | } 341 | 342 | func TestPutArtifactToS3Successfully(t *testing.T) { 343 | mockdb := &database.MockDatabase{} 344 | 345 | // First change to UPLOADING state... 346 | mockdb.On("UpdateArtifact", &model.Artifact{ 347 | State: model.UPLOADING, 348 | Size: 10, 349 | Name: "TestPutArtifact__artifactName", 350 | BucketId: "TestPutArtifact__bucketName", 351 | }).Return(nil).Once() 352 | 353 | // Then change to UPLOADED state. 354 | mockdb.On("UpdateArtifact", &model.Artifact{ 355 | State: model.UPLOADED, 356 | Size: 10, 357 | S3URL: "/TestPutArtifact__bucketName/TestPutArtifact__artifactName", 358 | Name: "TestPutArtifact__artifactName", 359 | BucketId: "TestPutArtifact__bucketName", 360 | }).Return(nil).Once() 361 | 362 | s3Server, s3Bucket := testS3ServerWithBucket(t) 363 | require.NoError(t, PutArtifact(context.Background(), &model.Artifact{ 364 | State: model.WAITING_FOR_UPLOAD, 365 | Size: 10, 366 | Name: "TestPutArtifact__artifactName", 367 | BucketId: "TestPutArtifact__bucketName", 368 | }, mockdb, s3Bucket, PutArtifactReq{ 369 | ContentLength: "10", 370 | Body: bytes.NewBufferString("0123456789"), 371 | })) 372 | 373 | s3Server.Quit() 374 | 375 | // mockdb.AssertExpectations(t) 376 | } 377 | 378 | func TestPutArtifactShortWrite(t *testing.T) { 379 | mockdb := &database.MockDatabase{} 380 | 381 | // First change to UPLOADING state... 382 | mockdb.On("UpdateArtifact", &model.Artifact{ 383 | State: model.UPLOADING, 384 | Size: 10, 385 | Name: "TestPutArtifact__artifactName", 386 | BucketId: "TestPutArtifact__bucketName", 387 | }).Return(nil).Once() 388 | 389 | // Then change to ERROR state. 390 | mockdb.On("UpdateArtifact", &model.Artifact{ 391 | State: model.ERROR, 392 | Size: 10, 393 | Name: "TestPutArtifact__artifactName", 394 | BucketId: "TestPutArtifact__bucketName", 395 | }).Return(nil).Once() 396 | 397 | s3Server, s3Bucket := testS3ServerWithBucket(t) 398 | require.Error(t, PutArtifact(context.Background(), &model.Artifact{ 399 | State: model.WAITING_FOR_UPLOAD, 400 | Size: 10, 401 | Name: "TestPutArtifact__artifactName", 402 | BucketId: "TestPutArtifact__bucketName", 403 | }, mockdb, s3Bucket, PutArtifactReq{ 404 | ContentLength: "10", 405 | Body: bytes.NewBufferString("012345678"), 406 | })) 407 | 408 | s3Server.Quit() 409 | } 410 | 411 | func TestPutArtifactToS3WithS3Errors(t *testing.T) { 412 | mockdb := &database.MockDatabase{} 413 | 414 | // First change to UPLOADING state... 415 | mockdb.On("UpdateArtifact", &model.Artifact{ 416 | State: model.UPLOADING, 417 | Size: 10, 418 | Name: "TestPutArtifact__artifactName", 419 | BucketId: "TestPutArtifact__bucketName", 420 | }).Return(nil).Once() 421 | 422 | // Then change to UPLOADED state. 423 | mockdb.On("UpdateArtifact", &model.Artifact{ 424 | State: model.ERROR, 425 | Size: 10, 426 | Name: "TestPutArtifact__artifactName", 427 | BucketId: "TestPutArtifact__bucketName", 428 | }).Return(nil).Once() 429 | 430 | s3Server, s3Bucket := testS3ServerWithBucket(t) 431 | // Terminate the s3 server to simulate s3 errors. We don't differentiate 432 | // between different s3 errors. So, this should handle all cases. 433 | s3Server.Quit() 434 | 435 | require.Error(t, PutArtifact(context.Background(), &model.Artifact{ 436 | State: model.WAITING_FOR_UPLOAD, 437 | Size: 10, 438 | Name: "TestPutArtifact__artifactName", 439 | BucketId: "TestPutArtifact__bucketName", 440 | }, mockdb, s3Bucket, PutArtifactReq{ 441 | ContentLength: "10", 442 | Body: bytes.NewBufferString("0123456789"), 443 | })) 444 | 445 | // mockdb.AssertExpectations(t) 446 | } 447 | 448 | func TestPutArtifactToS3WithRetries(t *testing.T) { 449 | mockdb := &database.MockDatabase{} 450 | 451 | // First change to UPLOADING state... 452 | mockdb.On("UpdateArtifact", &model.Artifact{ 453 | State: model.UPLOADING, 454 | Size: 10, 455 | Name: "TestPutArtifact__artifactName", 456 | BucketId: "TestPutArtifact__bucketName", 457 | }).Return(nil).Once() 458 | 459 | // Then change to UPLOADED state. 460 | mockdb.On("UpdateArtifact", &model.Artifact{ 461 | State: model.UPLOADED, 462 | Size: 10, 463 | Name: "TestPutArtifact__artifactName", 464 | BucketId: "TestPutArtifact__bucketName", 465 | S3URL: "/TestPutArtifact__bucketName/TestPutArtifact__artifactName", 466 | }).Return(nil).Once() 467 | 468 | reqCounter := 0 469 | s3Server, s3Bucket := fakeS3ServerWithBucket(t, func(w http.ResponseWriter, r *http.Request) { 470 | reqCounter++ 471 | 472 | if reqCounter <= 2 { 473 | // Simulate failure first 2 times 474 | w.WriteHeader(http.StatusInternalServerError) 475 | } else { 476 | // Pass on 3rd attempt 477 | w.WriteHeader(http.StatusOK) 478 | } 479 | }) 480 | 481 | require.NoError(t, PutArtifact(context.Background(), &model.Artifact{ 482 | State: model.WAITING_FOR_UPLOAD, 483 | Size: 10, 484 | Name: "TestPutArtifact__artifactName", 485 | BucketId: "TestPutArtifact__bucketName", 486 | }, mockdb, s3Bucket, PutArtifactReq{ 487 | ContentLength: "10", 488 | Body: bytes.NewBufferString("0123456789"), 489 | })) 490 | 491 | s3Server.Close() 492 | } 493 | 494 | func TestMergeLogChunks(t *testing.T) { 495 | mockdb := &database.MockDatabase{} 496 | 497 | // Merging log chunks not valid in following states 498 | require.Error(t, MergeLogChunks(nil, &model.Artifact{State: model.WAITING_FOR_UPLOAD}, mockdb, nil)) 499 | require.Error(t, MergeLogChunks(nil, &model.Artifact{State: model.APPENDING}, mockdb, nil)) 500 | require.Error(t, MergeLogChunks(nil, &model.Artifact{State: model.UPLOADED}, mockdb, nil)) 501 | require.Error(t, MergeLogChunks(nil, &model.Artifact{State: model.UPLOADING}, mockdb, nil)) 502 | 503 | // Closing an empty artifact with no errors 504 | mockdb.On("UpdateArtifact", &model.Artifact{ 505 | State: model.CLOSED_WITHOUT_DATA, 506 | }).Return(nil).Once() 507 | require.NoError(t, MergeLogChunks(nil, &model.Artifact{State: model.APPEND_COMPLETE, Size: 0}, mockdb, nil)) 508 | 509 | // Closing an empty artifact with db errors 510 | mockdb.On("UpdateArtifact", &model.Artifact{ 511 | State: model.CLOSED_WITHOUT_DATA, 512 | }).Return(database.MockDatabaseError()).Once() 513 | require.Error(t, MergeLogChunks(nil, &model.Artifact{State: model.APPEND_COMPLETE, Size: 0}, mockdb, nil)) 514 | 515 | // ----- BEGIN Closing an artifact with some log chunks 516 | // DB Error while updating artifact 517 | mockdb.On("UpdateArtifact", &model.Artifact{ 518 | State: model.UPLOADING, 519 | Size: 10, 520 | }).Return(database.MockDatabaseError()).Once() 521 | require.Error(t, MergeLogChunks(nil, &model.Artifact{State: model.APPEND_COMPLETE, Size: 10}, mockdb, nil)) 522 | 523 | { 524 | // DB Error while fetching logchunks 525 | mockdb.On("UpdateArtifact", &model.Artifact{ 526 | Id: 2, 527 | State: model.UPLOADING, 528 | Size: 10, 529 | }).Return(nil).Once() 530 | mockdb.On("ListLogChunksInArtifact", int64(2), int64(0), int64(10)).Return(nil, database.MockDatabaseError()).Times(MaxUploadAttempts) 531 | s3Server, s3Bucket := testS3ServerWithBucket(t) 532 | require.Error(t, MergeLogChunks(nil, &model.Artifact{Id: 2, State: model.APPEND_COMPLETE, Size: 10}, mockdb, s3Bucket)) 533 | s3Server.Quit() 534 | } 535 | 536 | { 537 | // Stitching chunks succeeds, but uploading to S3 fails 538 | mockdb.On("UpdateArtifact", &model.Artifact{ 539 | Id: 2, 540 | State: model.UPLOADING, 541 | Size: 10, 542 | Name: "TestMergeLogChunks__artifactName", 543 | BucketId: "TestMergeLogChunks__bucketName", 544 | }).Return(nil).Once() 545 | mockdb.On("ListLogChunksInArtifact", int64(2), int64(0), int64(10)).Return([]model.LogChunk{ 546 | model.LogChunk{ByteOffset: 0, Size: 5, ContentBytes: []byte("01234")}, 547 | model.LogChunk{ByteOffset: 5, Size: 5, ContentBytes: []byte("56789")}, 548 | }, nil).Once() 549 | s3Server, s3Bucket := testS3ServerWithBucket(t) 550 | s3Server.Quit() 551 | require.Error(t, MergeLogChunks(sentry.CreateAndInstallSentryClient(context.TODO(), "", ""), 552 | &model.Artifact{ 553 | Id: 2, 554 | State: model.APPEND_COMPLETE, 555 | Size: 10, 556 | Name: "TestMergeLogChunks__artifactName", 557 | BucketId: "TestMergeLogChunks__bucketName", 558 | }, mockdb, s3Bucket)) 559 | } 560 | 561 | // Stitching chunks and uploading to S3 successfully (but deleting logchunks fail) 562 | mockdb = &database.MockDatabase{} 563 | mockdb.On("UpdateArtifact", &model.Artifact{ 564 | Id: 2, 565 | State: model.UPLOADING, 566 | Size: 10, 567 | Name: "TestMergeLogChunks__artifactName", 568 | BucketId: "TestMergeLogChunks__bucketName", 569 | }).Return(nil).Once() 570 | mockdb.On("ListLogChunksInArtifact", int64(2), int64(0), int64(10)).Return([]model.LogChunk{ 571 | model.LogChunk{ByteOffset: 0, Size: 5, ContentBytes: []byte("01234")}, 572 | model.LogChunk{ByteOffset: 5, Size: 5, ContentBytes: []byte("56789")}, 573 | }, nil).Once() 574 | mockdb.On("UpdateArtifact", &model.Artifact{ 575 | Id: 2, 576 | State: model.UPLOADED, 577 | S3URL: "/TestMergeLogChunks__bucketName/TestMergeLogChunks__artifactName", 578 | Name: "TestMergeLogChunks__artifactName", 579 | BucketId: "TestMergeLogChunks__bucketName", 580 | Size: 10, 581 | }).Return(nil).Once() 582 | mockdb.On("DeleteLogChunksForArtifact", int64(2)).Return(int64(0), database.MockDatabaseError()).Once() 583 | s3Server, s3Bucket := testS3ServerWithBucket(t) 584 | require.NoError(t, MergeLogChunks(sentry.CreateAndInstallSentryClient(context.TODO(), "", ""), 585 | &model.Artifact{ 586 | Id: 2, 587 | State: model.APPEND_COMPLETE, 588 | Size: 10, 589 | Name: "TestMergeLogChunks__artifactName", 590 | BucketId: "TestMergeLogChunks__bucketName", 591 | }, mockdb, s3Bucket)) 592 | s3Server.Quit() 593 | 594 | // Stitching chunks and uploading to S3 successfully 595 | mockdb.On("UpdateArtifact", &model.Artifact{ 596 | Id: 3, 597 | State: model.UPLOADING, 598 | Size: 10, 599 | Name: "TestMergeLogChunks__artifactName", 600 | BucketId: "TestMergeLogChunks__bucketName", 601 | }).Return(nil).Once() 602 | mockdb.On("ListLogChunksInArtifact", int64(3), int64(0), int64(10)).Return([]model.LogChunk{ 603 | model.LogChunk{ByteOffset: 0, Size: 5, ContentBytes: []byte("01234")}, 604 | model.LogChunk{ByteOffset: 5, Size: 5, ContentBytes: []byte("56789")}, 605 | }, nil).Once() 606 | mockdb.On("DeleteLogChunksForArtifact", int64(3)).Return(int64(2), nil).Once() 607 | mockdb.On("UpdateArtifact", &model.Artifact{ 608 | Id: 3, 609 | State: model.UPLOADED, 610 | S3URL: "/TestMergeLogChunks__bucketName/TestMergeLogChunks__artifactName", 611 | Name: "TestMergeLogChunks__artifactName", 612 | BucketId: "TestMergeLogChunks__bucketName", 613 | Size: 10, 614 | }).Return(nil).Once() 615 | s3Server, s3Bucket = testS3ServerWithBucket(t) 616 | require.NoError(t, MergeLogChunks(nil, &model.Artifact{ 617 | Id: 3, 618 | State: model.APPEND_COMPLETE, 619 | Size: 10, 620 | Name: "TestMergeLogChunks__artifactName", 621 | BucketId: "TestMergeLogChunks__bucketName", 622 | }, mockdb, s3Bucket)) 623 | s3Server.Quit() 624 | // ----- END Closing an artifact with some log chunks 625 | } 626 | -------------------------------------------------------------------------------- /api/artifacthandler.go: -------------------------------------------------------------------------------- 1 | // A REST api for the artifact store, implemented using Martini. 2 | // 3 | // Each "Handle" function acts as a handler for a request and is 4 | // routed with Martini (routing is hanlded elsewhere). 5 | package api 6 | 7 | import ( 8 | "bytes" 9 | "encoding/json" 10 | "errors" 11 | "fmt" 12 | "io" 13 | "log" 14 | "net/http" 15 | "net/url" 16 | "os" 17 | "path/filepath" 18 | "strconv" 19 | "time" 20 | 21 | "golang.org/x/net/context" 22 | 23 | "github.com/dropbox/changes-artifacts/common/sentry" 24 | "github.com/dropbox/changes-artifacts/common/stats" 25 | "github.com/dropbox/changes-artifacts/database" 26 | "github.com/dropbox/changes-artifacts/model" 27 | "github.com/martini-contrib/render" 28 | "github.com/moshee/airlift/contentdisposition" 29 | "gopkg.in/amz.v1/s3" 30 | ) 31 | 32 | const DEFAULT_DEADLINE = 30 33 | 34 | // Format string to construct unique artifact names to recover from duplicate artifact name. 35 | const DuplicateArtifactNameFormat = "%s.dup.%s" 36 | 37 | // Maximum number of duplicate file name resolution attempts before failing with an internal error. 38 | const MaxDuplicateFileNameResolutionAttempts = 5 39 | 40 | // Maximum artifact size => 200 MB 41 | const MaxArtifactSizeBytes = 200 * 1024 * 1024 42 | 43 | // Maximum number of bytes to fetch while returning chunked response. 44 | const MaxChunkedRequestBytes = 1000000 45 | 46 | // MaxUploadAttempts is the maximum number of attempts to upload an artifact to S3. 47 | const MaxUploadAttempts = 3 48 | 49 | var bytesUploadedCounter = stats.NewStat("bytes_uploaded") 50 | 51 | type createArtifactReq struct { 52 | Name string 53 | Chunked bool 54 | Size int64 55 | DeadlineMins uint 56 | RelativePath string 57 | } 58 | 59 | type createLogChunkReq struct { 60 | ByteOffset int64 61 | Size int64 62 | Content string // DEPRECATED in favor of Bytes 63 | Bytes []byte 64 | } 65 | 66 | // CreateArtifact creates a new artifact in a open bucket. 67 | // 68 | // If an artifact with the same name already exists in the same bucket, we attempt to rename the 69 | // artifact by adding a suffix. 70 | // If the request specifies a chunked artifact, the size field is ignored and always set to zero. 71 | // If the request is for a streamed artifact, size is mandatory. 72 | // A relative path field may be specified to preserve the original file name and path. If no path is 73 | // specified, the original artifact name is used by default. 74 | func CreateArtifact(req createArtifactReq, bucket *model.Bucket, db database.Database) (*model.Artifact, *HttpError) { 75 | if len(req.Name) == 0 { 76 | return nil, NewHttpError(http.StatusBadRequest, "Artifact name not provided") 77 | } 78 | 79 | if bucket.State != model.OPEN { 80 | return nil, NewHttpError(http.StatusBadRequest, "Bucket is already closed") 81 | } 82 | 83 | artifact := new(model.Artifact) 84 | 85 | artifact.Name = req.Name 86 | artifact.BucketId = bucket.Id 87 | artifact.DateCreated = time.Now() 88 | 89 | if req.DeadlineMins == 0 { 90 | artifact.DeadlineMins = DEFAULT_DEADLINE 91 | } else { 92 | artifact.DeadlineMins = req.DeadlineMins 93 | } 94 | 95 | if req.Chunked { 96 | artifact.State = model.APPENDING 97 | } else { 98 | if req.Size == 0 { 99 | return nil, NewHttpError(http.StatusBadRequest, "Cannot create a new upload artifact without size.") 100 | } else if req.Size > MaxArtifactSizeBytes { 101 | return nil, NewHttpError(http.StatusRequestEntityTooLarge, fmt.Sprintf("Entity '%s' (size %d) is too large (limit %d)", req.Name, req.Size, MaxArtifactSizeBytes)) 102 | } 103 | artifact.Size = req.Size 104 | artifact.State = model.WAITING_FOR_UPLOAD 105 | } 106 | 107 | if req.RelativePath == "" { 108 | // Use artifact name provided as default relativePath 109 | artifact.RelativePath = req.Name 110 | } else { 111 | artifact.RelativePath = req.RelativePath 112 | } 113 | 114 | // Attempt to insert artifact and retry with a different name if it fails. 115 | if err := db.InsertArtifact(artifact); err != nil { 116 | for attempt := 1; attempt <= MaxDuplicateFileNameResolutionAttempts; attempt++ { 117 | // Unable to create new artifact - if an artifact already exists, the above insert failed 118 | // because of a collision. 119 | if _, err := db.GetArtifactByName(bucket.Id, artifact.Name); err != nil { 120 | // This could be a transient DB error (down/unreachable), in which case we expect the client 121 | // to retry. There is no value in attempting alternate artifact names. 122 | // 123 | // We have no means of verifying there was a name collision - bail with an internal error. 124 | return nil, NewHttpError(http.StatusInternalServerError, err.Error()) 125 | } 126 | 127 | // File name collision - attempt to resolve 128 | artifact.Name = fmt.Sprintf(DuplicateArtifactNameFormat, req.Name, randString(5)) 129 | if err := db.InsertArtifact(artifact); err == nil { 130 | return artifact, nil 131 | } 132 | } 133 | 134 | return nil, NewHttpError(http.StatusInternalServerError, "Exceeded retry limit avoiding duplicates") 135 | } 136 | 137 | return artifact, nil 138 | } 139 | 140 | func HandleCreateArtifact(ctx context.Context, r render.Render, req *http.Request, db database.Database, bucket *model.Bucket) { 141 | if bucket == nil { 142 | LogAndRespondWithErrorf(ctx, r, http.StatusBadRequest, "No bucket specified") 143 | return 144 | } 145 | 146 | var createReq createArtifactReq 147 | 148 | if err := json.NewDecoder(req.Body).Decode(&createReq); err != nil { 149 | LogAndRespondWithError(ctx, r, http.StatusBadRequest, err) 150 | return 151 | } 152 | 153 | if artifact, err := CreateArtifact(createReq, bucket, db); err != nil { 154 | LogAndRespondWithError(ctx, r, err.errCode, err) 155 | } else { 156 | r.JSON(http.StatusOK, artifact) 157 | } 158 | } 159 | 160 | func ListArtifacts(ctx context.Context, r render.Render, req *http.Request, db database.Database, bucket *model.Bucket) { 161 | if bucket == nil { 162 | LogAndRespondWithErrorf(ctx, r, http.StatusBadRequest, "No bucket specified") 163 | return 164 | } 165 | 166 | artifacts, err := db.ListArtifactsInBucket(bucket.Id) 167 | if err != nil { 168 | LogAndRespondWithError(ctx, r, http.StatusInternalServerError, err) 169 | return 170 | } 171 | 172 | r.JSON(http.StatusOK, artifacts) 173 | } 174 | 175 | func HandleGetArtifact(ctx context.Context, r render.Render, artifact *model.Artifact) { 176 | if artifact == nil { 177 | LogAndRespondWithErrorf(ctx, r, http.StatusBadRequest, "No bucket specified") 178 | return 179 | } 180 | 181 | r.JSON(http.StatusOK, artifact) 182 | } 183 | 184 | // AppendLogChunk appends a logchunk to an artifact. 185 | // If the logchunk position does not match the current end of artifact, an error is returned. 186 | // An exception to this is made when the last seen logchunk is repeated, which is silently ignored 187 | // without an error. 188 | func AppendLogChunk(ctx context.Context, db database.Database, artifact *model.Artifact, logChunkReq *createLogChunkReq) *HttpError { 189 | if artifact.State != model.APPENDING { 190 | return NewHttpError(http.StatusBadRequest, fmt.Sprintf("Unexpected artifact state: %s", artifact.State)) 191 | } 192 | 193 | if logChunkReq.Size <= 0 { 194 | return NewHttpError(http.StatusBadRequest, "Invalid chunk size %d", logChunkReq.Size) 195 | } 196 | 197 | var contentBytes []byte 198 | if len(logChunkReq.Bytes) != 0 { 199 | // If request sent Bytes, use Bytes. 200 | if int64(len(logChunkReq.Bytes)) != logChunkReq.Size { 201 | return NewHttpError(http.StatusBadRequest, "Content length %d does not match indicated size %d", len(logChunkReq.Bytes), logChunkReq.Size) 202 | } 203 | contentBytes = logChunkReq.Bytes 204 | } else { 205 | // Otherwise, allow Content, for now. 206 | if len(logChunkReq.Content) == 0 { 207 | return NewHttpError(http.StatusBadRequest, "Empty content string") 208 | } 209 | 210 | if int64(len(logChunkReq.Content)) != logChunkReq.Size { 211 | return NewHttpError(http.StatusBadRequest, "Content length %d does not match indicated size %d", len(logChunkReq.Content), logChunkReq.Size) 212 | } 213 | contentBytes = []byte(logChunkReq.Content) 214 | } 215 | 216 | // Find previous chunk in DB - append only 217 | nextByteOffset := artifact.Size 218 | if nextByteOffset != logChunkReq.ByteOffset { 219 | // There is a possibility the previous logchunk is being retried - we need to handle cases where 220 | // a server/proxy time out caused the client not to get an ACK when it successfully uploaded the 221 | // previous logchunk, due to which it is retrying. 222 | // 223 | // This is a best-effort check - if we encounter DB errors or any mismatch in the chunk 224 | // contents, we ignore this test and claim that a range mismatch occured. 225 | if nextByteOffset != 0 && nextByteOffset == logChunkReq.ByteOffset+logChunkReq.Size { 226 | if prevLogChunk, err := db.GetLastLogChunkSeenForArtifact(artifact.Id); err == nil { 227 | if prevLogChunk != nil && prevLogChunk.ByteOffset == logChunkReq.ByteOffset && prevLogChunk.Size == logChunkReq.Size && bytes.Equal(prevLogChunk.ContentBytes, contentBytes) { 228 | sentry.ReportMessage(ctx, fmt.Sprintf("Received duplicate chunk for artifact %v of size %d at byte %d", artifact.Id, logChunkReq.Size, logChunkReq.ByteOffset)) 229 | return nil 230 | } 231 | } 232 | } 233 | 234 | return NewHttpError(http.StatusBadRequest, "Overlapping ranges detected, expected offset: %d, actual offset: %d", nextByteOffset, logChunkReq.ByteOffset) 235 | } 236 | 237 | // Expand artifact size - redundant after above change. 238 | if artifact.Size < logChunkReq.ByteOffset+logChunkReq.Size { 239 | artifact.Size = logChunkReq.ByteOffset + logChunkReq.Size 240 | if err := db.UpdateArtifact(artifact); err != nil { 241 | return NewHttpError(http.StatusInternalServerError, err.Error()) 242 | } 243 | } 244 | 245 | logChunk := &model.LogChunk{ 246 | ArtifactId: artifact.Id, 247 | ByteOffset: logChunkReq.ByteOffset, 248 | ContentBytes: contentBytes, 249 | Size: logChunkReq.Size, 250 | } 251 | 252 | if err := db.InsertLogChunk(logChunk); err != nil { 253 | return NewHttpError(http.StatusBadRequest, "Error updating log chunk: %s", err) 254 | } 255 | return nil 256 | } 257 | 258 | // PostArtifact updates content associated with an artifact. 259 | // 260 | // If the artifact is streamed (uploaded in one shot), PutArtifact is invoked to stream content 261 | // directly through to S3. 262 | // 263 | // If the artifact is chunked (appended chunk by chunk), verify that the position being written to 264 | // matches the current end of artifact, insert a new log chunk at that position and move the end of 265 | // file forward. 266 | func PostArtifact(ctx context.Context, r render.Render, req *http.Request, db database.Database, s3bucket *s3.Bucket, artifact *model.Artifact) { 267 | if artifact == nil { 268 | LogAndRespondWithErrorf(ctx, r, http.StatusBadRequest, "No artifact specified") 269 | return 270 | } 271 | 272 | switch artifact.State { 273 | case model.WAITING_FOR_UPLOAD: 274 | contentLengthStr := req.Header.Get("Content-Length") 275 | contentLength, err := strconv.ParseInt(contentLengthStr, 10, 64) // string, base, bits 276 | if err != nil { 277 | LogAndRespondWithErrorf(ctx, r, http.StatusBadRequest, "Couldn't parse Content-Length as int64") 278 | } else if contentLength != artifact.Size { 279 | LogAndRespondWithErrorf(ctx, r, http.StatusBadRequest, "Content-Length does not match artifact size") 280 | } else if err = PutArtifact(ctx, artifact, db, s3bucket, PutArtifactReq{ContentLength: contentLengthStr, Body: req.Body}); err != nil { 281 | LogAndRespondWithError(ctx, r, http.StatusInternalServerError, err) 282 | } else { 283 | r.JSON(http.StatusOK, artifact) 284 | } 285 | return 286 | 287 | case model.UPLOADING: 288 | LogAndRespondWithErrorf(ctx, r, http.StatusBadRequest, "Artifact is currently being updated") 289 | return 290 | 291 | case model.UPLOADED: 292 | LogAndRespondWithErrorf(ctx, r, http.StatusBadRequest, "Artifact already uploaded") 293 | return 294 | 295 | case model.APPENDING: 296 | // TODO: Treat contents as a JSON request containing a chunk. 297 | logChunkReq := new(createLogChunkReq) 298 | if err := json.NewDecoder(req.Body).Decode(logChunkReq); err != nil { 299 | LogAndRespondWithError(ctx, r, http.StatusBadRequest, err) 300 | return 301 | } 302 | 303 | if err := AppendLogChunk(ctx, db, artifact, logChunkReq); err != nil { 304 | LogAndRespondWithError(ctx, r, err.errCode, err) 305 | return 306 | } 307 | 308 | r.JSON(http.StatusOK, artifact) 309 | return 310 | 311 | case model.APPEND_COMPLETE: 312 | LogAndRespondWithErrorf(ctx, r, http.StatusBadRequest, "Artifact is closed for further appends") 313 | return 314 | } 315 | } 316 | 317 | // CloseArtifact closes an artifact for further writes and begins process of merging and uploading 318 | // the artifact. This operation is only valid for artifacts which are being uploaded in chunks. 319 | // In all other cases, an error is returned. 320 | func CloseArtifact(ctx context.Context, artifact *model.Artifact, db database.Database, s3bucket *s3.Bucket, failIfAlreadyClosed bool) error { 321 | switch artifact.State { 322 | case model.UPLOADED: 323 | // Already closed. Nothing to do here. 324 | fallthrough 325 | case model.APPEND_COMPLETE: 326 | // This artifact will be eventually shipped to S3. No change required. 327 | return nil 328 | 329 | case model.APPENDING: 330 | artifact.State = model.APPEND_COMPLETE 331 | if err := db.UpdateArtifact(artifact); err != nil { 332 | return err 333 | } 334 | 335 | return MergeLogChunks(ctx, artifact, db, s3bucket) 336 | 337 | case model.WAITING_FOR_UPLOAD: 338 | // Streaming artifact was not uploaded 339 | artifact.State = model.CLOSED_WITHOUT_DATA 340 | if err := db.UpdateArtifact(artifact); err != nil { 341 | return err 342 | } 343 | 344 | return nil 345 | 346 | default: 347 | return fmt.Errorf("Unexpected artifact state: %s", artifact.State) 348 | } 349 | } 350 | 351 | // Merges all of the individual chunks into a single object and stores it on s3. 352 | // The log chunks are stored in the database, while the object is uploaded to s3. 353 | func MergeLogChunks(ctx context.Context, artifact *model.Artifact, db database.Database, s3bucket *s3.Bucket) error { 354 | switch artifact.State { 355 | case model.APPEND_COMPLETE: 356 | // TODO: Reimplement using GorpDatabase 357 | // If the file is empty, don't bother creating an object on S3. 358 | if artifact.Size == 0 { 359 | artifact.State = model.CLOSED_WITHOUT_DATA 360 | artifact.S3URL = "" 361 | 362 | // Conversion between *DatabaseEror and error is tricky. If we don't do this, a nil 363 | // *DatabaseError can become a non-nil error. 364 | return db.UpdateArtifact(artifact).GetError() 365 | } 366 | 367 | // XXX Do we need to commit here or is this handled transparently? 368 | artifact.State = model.UPLOADING 369 | if err := db.UpdateArtifact(artifact); err != nil { 370 | return err 371 | } 372 | 373 | fileName := artifact.DefaultS3URL() 374 | 375 | r := newLogChunkReaderWithReadahead(artifact, db) 376 | 377 | if err := uploadArtifactToS3(s3bucket, fileName, artifact.Size, r); err != nil { 378 | return err 379 | } 380 | 381 | // XXX This is a long operation and should probably be asynchronous from the 382 | // actual HTTP request, and the client should poll to check when its uploaded. 383 | artifact.State = model.UPLOADED 384 | artifact.S3URL = fileName 385 | if err := db.UpdateArtifact(artifact); err != nil { 386 | return err 387 | } 388 | 389 | // From this point onwards, we will not send back any errors back to the user. If we are 390 | // unable to delete logchunks, we log it to Sentry instead. 391 | if _, err := db.DeleteLogChunksForArtifact(artifact.Id); err != nil { 392 | sentry.ReportError(ctx, err) 393 | return nil 394 | } 395 | 396 | return nil 397 | 398 | case model.WAITING_FOR_UPLOAD: 399 | fallthrough 400 | case model.ERROR: 401 | fallthrough 402 | case model.APPENDING: 403 | fallthrough 404 | case model.UPLOADED: 405 | fallthrough 406 | case model.UPLOADING: 407 | return fmt.Errorf("Artifact can only be merged when in APPEND_COMPLETE state, but state is %s", artifact.State) 408 | default: 409 | return fmt.Errorf("Illegal artifact state! State code is %d", artifact.State) 410 | } 411 | } 412 | 413 | // HandleCloseArtifact handles the HTTP request to close an artifact. See CloseArtifact for details. 414 | func HandleCloseArtifact(ctx context.Context, r render.Render, db database.Database, s3bucket *s3.Bucket, artifact *model.Artifact) { 415 | if artifact == nil { 416 | LogAndRespondWithErrorf(ctx, r, http.StatusBadRequest, "No artifact specified") 417 | return 418 | } 419 | 420 | if err := CloseArtifact(ctx, artifact, db, s3bucket, true); err != nil { 421 | LogAndRespondWithError(ctx, r, http.StatusBadRequest, err) 422 | return 423 | } 424 | 425 | r.JSON(http.StatusOK, map[string]interface{}{}) 426 | } 427 | 428 | func intParam(v url.Values, paramName string, fallback int64) int64 { 429 | if value := v.Get(paramName); value != "" { 430 | if intVal, err := strconv.Atoi(value); err == nil { 431 | return int64(intVal) 432 | } 433 | 434 | // TODO: should we should raise an error here because the query is badly formatted? 435 | } 436 | 437 | return fallback 438 | } 439 | 440 | func min(a, b int64) int64 { 441 | if a < b { 442 | return a 443 | } 444 | return b 445 | } 446 | 447 | func max(a, b int64) int64 { 448 | if a > b { 449 | return a 450 | } 451 | return b 452 | } 453 | 454 | var errReadBeyondEOF = errors.New("Reading beyond EOF") 455 | 456 | func getByteRangeFromRequest(req *http.Request, artifact *model.Artifact) (int64, int64, error) { 457 | queryParams := req.URL.Query() 458 | 459 | // byteRangeBegin and byteRangeEnd are inclusive range markers within an artifact. 460 | byteRangeBegin := max(intParam(queryParams, "offset", 0), 0) 461 | if byteRangeBegin >= artifact.Size { 462 | return 0, artifact.Size, errReadBeyondEOF 463 | } 464 | 465 | limit := max(intParam(queryParams, "limit", 0), 0) 466 | if limit == 0 { 467 | limit = MaxChunkedRequestBytes 468 | } 469 | limit = min(limit, MaxChunkedRequestBytes) 470 | 471 | byteRangeEnd := min(artifact.Size-1, byteRangeBegin+limit-1) 472 | 473 | return byteRangeBegin, byteRangeEnd, nil 474 | } 475 | 476 | // GetArtifactContentChunks lists artifact contents in a chunked form. Useful to poll for updates to 477 | // chunked artifacts. All artifact types are supported and chunks can be requested from arbitrary 478 | // locations within artifacts. 479 | // 480 | // This is primarily meant for Changes UI for log following. If you need to fetch byte ranges from the 481 | // store, it should be available directly at /content 482 | // 483 | // URL query parameters offset and limit can be used to control range of chunks to be fetched. 484 | // offset -> byte offset of the start of the range to be fetched (defaults to beginning of artifact) 485 | // limit -> number of bytes to be fetched (defaults to 100KB) 486 | // 487 | // Negative values for any query parameter will cause it to be set to 0 (default) 488 | func GetArtifactContentChunks(ctx context.Context, r render.Render, req *http.Request, res http.ResponseWriter, db database.Database, s3bucket *s3.Bucket, artifact *model.Artifact) { 489 | if artifact == nil { 490 | LogAndRespondWithErrorf(ctx, r, http.StatusBadRequest, "No artifact specified") 491 | return 492 | } 493 | 494 | type Chunk struct { 495 | ID int64 `json:"id"` 496 | Offset int64 `json:"offset"` 497 | Size int64 `json:"size"` 498 | Text string `json:"text"` 499 | } 500 | 501 | type Result struct { 502 | Chunks []Chunk `json:"chunks"` 503 | EOF bool `json:"eof"` 504 | NextOffset int64 `json:"nextOffset"` 505 | } 506 | 507 | byteRangeBegin, byteRangeEnd, err := getByteRangeFromRequest(req, artifact) 508 | 509 | if err != nil { 510 | // If given range is not valid, steer client to a valid range. 511 | r.JSON(http.StatusOK, &Result{Chunks: []Chunk{}, EOF: err == errReadBeyondEOF && artifact.State == model.UPLOADED, NextOffset: byteRangeEnd}) 512 | return 513 | } 514 | 515 | switch artifact.State { 516 | case model.UPLOADING: 517 | // No data to report right now. Wait till upload to S3 completes. 518 | fallthrough 519 | case model.WAITING_FOR_UPLOAD: 520 | // Upload hasn't started. No data to report. Try again later. 521 | r.JSON(http.StatusOK, &Result{Chunks: []Chunk{}, NextOffset: byteRangeBegin}) 522 | return 523 | case model.UPLOADED: 524 | // Fetch from S3 525 | url := s3bucket.SignedURL(artifact.S3URL, time.Now().Add(30*time.Minute)) 526 | rq, err := http.NewRequest("GET", url, nil) 527 | if err != nil { 528 | LogAndRespondWithError(ctx, r, http.StatusInternalServerError, err) 529 | return 530 | } 531 | rq.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", byteRangeBegin, byteRangeEnd)) 532 | resp, err := http.DefaultClient.Do(rq) 533 | if err != nil { 534 | LogAndRespondWithError(ctx, r, http.StatusInternalServerError, err) 535 | return 536 | } 537 | if resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusOK { 538 | LogAndRespondWithErrorf(ctx, r, http.StatusBadRequest, fmt.Sprintf("Bad status code %d", resp.StatusCode)) 539 | return 540 | } 541 | var buf bytes.Buffer 542 | n, err := buf.ReadFrom(resp.Body) 543 | if err != nil { 544 | LogAndRespondWithError(ctx, r, http.StatusInternalServerError, err) 545 | return 546 | } 547 | 548 | nextOffset := byteRangeBegin + int64(n) 549 | r.JSON(http.StatusOK, &Result{ 550 | Chunks: []Chunk{Chunk{Offset: byteRangeBegin, Size: int64(n), Text: buf.String()}}, 551 | EOF: nextOffset == artifact.Size, 552 | NextOffset: nextOffset, 553 | }) 554 | return 555 | case model.APPENDING: 556 | fallthrough 557 | case model.APPEND_COMPLETE: 558 | // Pick from log chunks 559 | rd := newLogChunkReader(artifact, db) 560 | rd.Seek(byteRangeBegin, os.SEEK_SET) 561 | 562 | bts := make([]byte, byteRangeEnd-byteRangeBegin+1) 563 | n, err := runeLimitedRead(rd, bts) 564 | if err != nil && err != io.EOF { 565 | LogAndRespondWithError(ctx, r, http.StatusInternalServerError, err) 566 | return 567 | } 568 | 569 | if n > 0 { 570 | r.JSON(http.StatusOK, &Result{ 571 | Chunks: []Chunk{Chunk{Offset: byteRangeBegin, Size: int64(n), Text: string(bts[:n])}}, 572 | NextOffset: byteRangeBegin + int64(n), 573 | }) 574 | } else { 575 | r.JSON(http.StatusOK, &Result{Chunks: []Chunk{}, NextOffset: byteRangeBegin}) 576 | } 577 | return 578 | } 579 | } 580 | 581 | func GetArtifactContent(ctx context.Context, r render.Render, req *http.Request, res http.ResponseWriter, db database.Database, s3bucket *s3.Bucket, artifact *model.Artifact) { 582 | if artifact == nil { 583 | LogAndRespondWithErrorf(ctx, r, http.StatusBadRequest, "No artifact specified") 584 | return 585 | } 586 | 587 | switch artifact.State { 588 | case model.UPLOADED: 589 | // Fetch from S3 590 | url := s3bucket.SignedURL(artifact.S3URL, time.Now().Add(30*time.Minute)) 591 | rq, err := http.NewRequest("GET", url, nil) 592 | if byteRanges := req.Header.Get("Range"); byteRanges != "" { 593 | // If request contains Range: headers, pass them right through to S3. 594 | // TODO(anupc): Validation? We're sending user input through to the data store. 595 | rq.Header.Add("Range", byteRanges) 596 | } 597 | resp, err := http.DefaultClient.Do(rq) 598 | if err != nil { 599 | LogAndRespondWithError(ctx, r, http.StatusInternalServerError, err) 600 | return 601 | } 602 | if resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusOK { 603 | LogAndRespondWithErrorf(ctx, r, http.StatusInternalServerError, fmt.Sprintf("Bad status code %d recieved from S3", resp.StatusCode)) 604 | return 605 | } 606 | contentdisposition.SetFilename(res, filepath.Base(artifact.RelativePath)) 607 | res.Header().Add("Content-Length", strconv.Itoa(int(artifact.Size))) 608 | if n, err := io.CopyN(res, resp.Body, artifact.Size); err != nil { 609 | sentry.ReportError(ctx, fmt.Errorf("Error transferring artifact (for artifact %s/%s, bytes (%d/%d) read): %s", artifact.BucketId, artifact.Name, n, artifact.Size, err)) 610 | return 611 | } 612 | return 613 | case model.UPLOADING: 614 | // Not done uploading to S3 yet. Error. 615 | LogAndRespondWithErrorf(ctx, r, http.StatusNotFound, "Waiting for content to complete uploading") 616 | return 617 | case model.APPENDING: 618 | fallthrough 619 | case model.APPEND_COMPLETE: 620 | // Pick from log chunks 621 | contentdisposition.SetFilename(res, filepath.Base(artifact.RelativePath)) 622 | // All written bytes are immutable. So, unless size changes, all previously read contents can be cached. 623 | res.Header().Add("ETag", strconv.Itoa(int(artifact.Size))) 624 | http.ServeContent(res, req, filepath.Base(artifact.RelativePath), time.Time{}, newLogChunkReaderWithReadahead(artifact, db)) 625 | return 626 | case model.WAITING_FOR_UPLOAD: 627 | // Not started yet. Error 628 | LogAndRespondWithErrorf(ctx, r, http.StatusNotFound, "Waiting for content to get uploaded") 629 | return 630 | } 631 | } 632 | 633 | func uploadArtifactToS3(bucket *s3.Bucket, artifactName string, artifactSize int64, contentReader io.ReadSeeker) error { 634 | attempts := 0 635 | 636 | for { 637 | attempts++ 638 | // Rewind Seeker to beginning, required if we had already read a few bytes from it before. 639 | if _, err := contentReader.Seek(0, os.SEEK_SET); err != nil { 640 | return err 641 | } 642 | 643 | if err := bucket.PutReader(artifactName, contentReader, artifactSize, "binary/octet-stream", s3.PublicRead); err != nil { 644 | if attempts < MaxUploadAttempts { 645 | log.Printf("[Attempt %d/%d] Error uploading to S3: %s", attempts, MaxUploadAttempts, err) 646 | continue 647 | } 648 | return fmt.Errorf("Error uploading to S3: %s", err) 649 | } 650 | 651 | bytesUploadedCounter.Add(artifactSize) 652 | return nil 653 | } 654 | 655 | return nil // This should never happen - only here to satisfy the compiler 656 | } 657 | 658 | type PutArtifactReq struct { 659 | ContentLength string 660 | Body io.Reader 661 | } 662 | 663 | // PutArtifact writes a streamed artifact to S3. The entire file contents are streamed directly 664 | // through to S3. If S3 is not accessible, we don't make any attempt to buffer on disk and fail 665 | // immediately. 666 | func PutArtifact(ctx context.Context, artifact *model.Artifact, db database.Database, bucket *s3.Bucket, req PutArtifactReq) error { 667 | if artifact.State != model.WAITING_FOR_UPLOAD { 668 | return fmt.Errorf("Expected artifact to be in state WAITING_FOR_UPLOAD: %s", artifact.State) 669 | } 670 | 671 | // New file being inserted into DB. 672 | // Mark status change to UPLOADING and start uploading to S3. 673 | // 674 | // First, verify that the size of the content being uploaded matches our expected size. 675 | var fileSize int64 676 | var err error 677 | 678 | if req.ContentLength != "" { 679 | fileSize, err = strconv.ParseInt(req.ContentLength, 10, 64) // string, base, bits 680 | // This should never happen if a sane HTTP client is used. Nonetheless ... 681 | if err != nil { 682 | return fmt.Errorf("Invalid Content-Length specified") 683 | } 684 | } else { 685 | // This too should never happen if a sane HTTP client is used. Nonetheless ... 686 | return fmt.Errorf("Content-Length not specified") 687 | } 688 | 689 | if fileSize != artifact.Size { 690 | return fmt.Errorf("Content length %d does not match expected file size %d", fileSize, artifact.Size) 691 | } 692 | 693 | artifact.State = model.UPLOADING 694 | if err := db.UpdateArtifact(artifact); err != nil { 695 | return err 696 | } 697 | 698 | cleanupAndReturn := func(err error) error { 699 | // TODO: Is there a better way to detect and handle errors? 700 | // Use a channel to signify upload completion. In defer, check if the channel is empty. If 701 | // yes, mark error. Else ignore. 702 | if err != nil { 703 | // TODO: s/ERROR/WAITING_FOR_UPLOAD/ ? 704 | sentry.ReportError(ctx, err) 705 | artifact.State = model.ERROR 706 | err2 := db.UpdateArtifact(artifact) 707 | if err2 != nil { 708 | log.Printf("Error while handling error: %s", err2.Error()) 709 | } 710 | return err 711 | } 712 | 713 | return nil 714 | } 715 | 716 | b := new(bytes.Buffer) 717 | // Note: Storing entire contents of uploaded artifact in memory can cause OOMS. 718 | if n, err := io.CopyN(b, req.Body, artifact.Size); err != nil { 719 | return cleanupAndReturn(fmt.Errorf("Error reading from request body (for artifact %s/%s, bytes (%d/%d) read): %s", artifact.BucketId, artifact.Name, n, artifact.Size, err)) 720 | } 721 | fileName := artifact.DefaultS3URL() 722 | 723 | if err := uploadArtifactToS3(bucket, fileName, artifact.Size, bytes.NewReader(b.Bytes())); err != nil { 724 | return cleanupAndReturn(err) 725 | } 726 | 727 | artifact.State = model.UPLOADED 728 | artifact.S3URL = fileName 729 | if err := db.UpdateArtifact(artifact); err != nil { 730 | return err 731 | } 732 | return nil 733 | } 734 | --------------------------------------------------------------------------------