├── .gitignore ├── .github ├── CODEOWNERS └── workflows │ └── actions.yml ├── CHANGELOG.md ├── nfpm.yaml ├── main.go ├── downloader ├── downloader_test.go └── downloader.go ├── method ├── method_test.go └── method.go ├── Makefile ├── README.md └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | apt-s3_* 2 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # CODEOWNERS file 2 | # This file defines who should review code changes in this repository. 3 | 4 | * @zendesk/guardians-composite 5 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Unreeleased Changes 2 | 3 | # 0.1.1 (2019-04-08) 4 | * switch from `panic` to `os.Exit(1)` 5 | 6 | # 0.1.0 (2019-04-02) 7 | * initial release supporting `apt` and interactive usage 8 | -------------------------------------------------------------------------------- /nfpm.yaml: -------------------------------------------------------------------------------- 1 | name: "apt-s3" 2 | arch: "${ARCH}" 3 | platform: "linux" 4 | version: "${VERSION}" 5 | section: "default" 6 | priority: "extra" 7 | conflicts: 8 | - apt-transport-s3 9 | replaces: 10 | - apt-transport-s3 11 | provides: 12 | - apt-s3 13 | depends: 14 | - ca-certificates 15 | maintainer: "Rob Bayerl " 16 | vendor: "Zendesk, Inc." 17 | description: | 18 | apt transport method for repositories hosted in S3. 19 | homepage: "https://github.com/rbayerl/apt-s3" 20 | license: "Apache-2.0" 21 | contents: 22 | - src: apt-s3 23 | dst: /usr/local/bin/apt-s3 24 | - src: /usr/local/bin/apt-s3 25 | dst: /usr/lib/apt/methods/s3 26 | type: symlink 27 | - src: LICENSE 28 | dst: /usr/share/doc/apt-s3/copyright 29 | -------------------------------------------------------------------------------- /.github/workflows/actions.yml: -------------------------------------------------------------------------------- 1 | name: repo-checks 2 | on: [push] 3 | jobs: 4 | main: 5 | name: go-lang-matrix 6 | runs-on: ubuntu-latest 7 | env: 8 | APP_ENV: test 9 | GO111MODULE: on 10 | GITHUB_TOKEN: ${{ secrets.ORG_GITHUB_TOKEN }} 11 | strategy: 12 | matrix: 13 | go: 14 | - "1.12" 15 | - "1.11" 16 | steps: 17 | - uses: zendesk/checkout@v2 18 | with: 19 | fetch-depth: 0 20 | - uses: zendesk/setup-go@v2 21 | with: 22 | go-version: ${{ matrix.go }} 23 | - name: test go ${{ matrix.go }} 24 | run: | 25 | curl -sSLO https://github.com/goreleaser/nfpm/releases/download/v2.11.3/nfpm_2.11.3_Linux_x86_64.tar.gz 26 | mkdir bin 27 | tar xfz nfpm_2.11.3_Linux_x86_64.tar.gz -C bin 28 | export PATH=./bin:$PATH 29 | make 30 | 31 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "os" 8 | "regexp" 9 | "runtime" 10 | 11 | "github.com/zendesk/apt-s3/method" 12 | ) 13 | 14 | var ( 15 | downloadUri = flag.String("download", "", "S3 URI for downloading a single file") 16 | downloadPath = flag.String("path", "", "Path to download file to") 17 | versionFlag = flag.Bool("version", false, "Show version") 18 | Version = "master" 19 | ) 20 | 21 | func main() { 22 | m := method.New() 23 | programName := os.Args[0] 24 | 25 | flag.Parse() 26 | 27 | if *versionFlag { 28 | fmt.Printf("%s %s (Go version: %s)\n", programName, Version, runtime.Version()) 29 | os.Exit(0) 30 | // Called outside of apt to download a file 31 | } else if *downloadUri != "" { 32 | if match, _ := regexp.MatchString("s3://.*\\.s3.*\\.amazonaws\\.com/.*", *downloadUri); !match { 33 | log.Fatalf("Incorrect bucket format.\nExpected: s3://.s3-.amazonaws.com/path/to/file\n") 34 | } else { 35 | filename, err := m.Downloader.DownloadFile(*downloadUri, *downloadPath) 36 | if err != nil { 37 | log.Fatal(err) 38 | } 39 | fmt.Printf("Downloaded %s\n", filename) 40 | os.Exit(0) 41 | } 42 | } else { 43 | m.Start() 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /downloader/downloader_test.go: -------------------------------------------------------------------------------- 1 | package downloader 2 | 3 | import "testing" 4 | 5 | func TestParseURI(t *testing.T) { 6 | tests := []struct { 7 | uri string 8 | bucket string 9 | region string 10 | key string 11 | filename string 12 | }{ 13 | { 14 | uri: "s3://my-bucket.s3-us-west-1.amazonaws.com/path/to/file", 15 | bucket: "my-bucket", 16 | region: "us-west-1", 17 | key: "path/to/file", 18 | filename: "file", 19 | }, 20 | { 21 | uri: "s3://my-bucket.s3.amazonaws.com/path/to/file", 22 | bucket: "my-bucket", 23 | region: "us-east-1", 24 | key: "path/to/file", 25 | filename: "file", 26 | }, 27 | } 28 | 29 | d := &Downloader{} 30 | 31 | for _, test := range tests { 32 | bucket, region, key, filename := d.parseURI(test.uri) 33 | if bucket != test.bucket { 34 | t.Errorf("parseURI() bucket == %s (expected %s)", bucket, test.bucket) 35 | } 36 | if region != test.region { 37 | t.Errorf("parseURI() region == %s (expected %s)", region, test.region) 38 | } 39 | if key != test.key { 40 | t.Errorf("parseURI() key == %s (expected %s)", key, test.key) 41 | } 42 | if filename != test.filename { 43 | t.Errorf("parseURI() filename == %s (expected %s)", filename, test.filename) 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /method/method_test.go: -------------------------------------------------------------------------------- 1 | package method 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "io" 7 | "os" 8 | "os/exec" 9 | "strings" 10 | "testing" 11 | ) 12 | 13 | func TestSendCapabilities(t *testing.T) { 14 | old := os.Stdout 15 | r, w, _ := os.Pipe() 16 | os.Stdout = w 17 | 18 | m := &Method{} 19 | 20 | m.sendCapabilities() 21 | 22 | w.Close() 23 | os.Stdout = old 24 | 25 | var buf bytes.Buffer 26 | io.Copy(&buf, r) 27 | if buf.String() != "100 Capabilities\nSend-Config: true\nPipeline: true\nSingle-Instance: yes\n\n" { 28 | t.Errorf("sendCapabilities() unexpected string %s", buf.String()) 29 | } 30 | } 31 | 32 | func TestFindLine(t *testing.T) { 33 | lines := []string{ 34 | "Not this line", 35 | "Foo: bar", 36 | "Not this line either", 37 | } 38 | 39 | m := &Method{} 40 | key := m.findLine("Foo", lines) 41 | if key != "bar" { 42 | t.Errorf("findLine() Foo = %s (expected bar)", key) 43 | } 44 | } 45 | 46 | func TestHandleError(t *testing.T) { 47 | if os.Getenv("TEST_EXIT") == "1" { 48 | m := &Method{} 49 | m.handleError("s3://foobar.s3.amazonaws.com/foo", errors.New("Foobar error")) 50 | return 51 | } 52 | r, w, _ := os.Pipe() 53 | cmd := exec.Command("go", "test", "github.com/zendesk/apt-s3/method", "-test.run=TestHandleError") 54 | cmd.Env = append(os.Environ(), "TEST_EXIT=1") 55 | cmd.Stdout = w 56 | cmd.Run() 57 | w.Close() 58 | var buf bytes.Buffer 59 | io.Copy(&buf, r) 60 | if !strings.Contains(buf.String(), "400 URI Failure\nMessage: Foobar error\nURI: s3://foobar.s3.amazonaws.com/foo\n\n") { 61 | t.Errorf("handleError() unexpected error message %s", buf.String()) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VERSION := $(shell git describe --tags) 2 | LDFLAGS := -ldflags='-X "main.Version=$(VERSION)"' 3 | 4 | ARCHITECTURES = amd64 arm64 5 | BUILD_TARGETS = $(patsubst %, apt-s3_$(VERSION)_%, $(ARCHITECTURES)) 6 | PACKAGE_TARGETS = $(patsubst %, apt-s3_$(VERSION)_%.deb, $(ARCHITECTURES)) 7 | 8 | all: test $(PACKAGE_TARGETS) 9 | 10 | $(BUILD_TARGETS): apt-s3_$(VERSION)_% : build-deps 11 | GOOS=linux GOARCH=$* go build $(LDFLAGS) -o $@ 12 | 13 | $(PACKAGE_TARGETS): apt-s3_$(VERSION)_%.deb : apt-s3_$(VERSION)_% 14 | cp apt-s3_$(VERSION)_$* apt-s3 # Workaround, nfpm does not support env vars in contents 15 | VERSION=$(VERSION) ARCH=$* nfpm pkg --target $@ 16 | rm apt-s3 17 | 18 | build-deps: 19 | go get ./... 20 | 21 | clean: 22 | rm -f apt-s3_* apt-s3_*.deb 23 | 24 | release: $(PACKAGE_TARGETS) tag 25 | ifndef GITHUB_TOKEN 26 | $(error GITHUB_TOKEN is not set!) 27 | endif 28 | $(eval URL := $(shell curl -sS -H "Authorization: token $$GITHUB_TOKEN" -H "Content-Type: application/json" -X POST -d '{"tag_name":"$(VERSION)","name":"v$(VERSION)"}' https://api.github.com/repos/zendesk/apt-s3/releases | awk -F\" /assets_url/'{sub(/api/, "uploads", $$4); print $$4 }')) 29 | $(foreach arch,$(ARCHITECTURES),\ 30 | $(shell curl -sS -H "Authorization: token $$GITHUB_TOKEN" -H "Content-Type: application/octet-stream" -X POST --data-binary "@apt-s3_$(VERSION)_$(arch)" $(URL)?name=apt-s3_$(VERSION)_$(arch) >/dev/null)\ 31 | $(shell curl -sS -H "Authorization: token $$GITHUB_TOKEN" -H "Content-Type: application/octet-stream" -X POST --data-binary "@apt-s3_$(VERSION)_$(arch).deb" $(URL)?name=apt-s3_$(VERSION)_$(arch).deb >/dev/null)\ 32 | ) 33 | 34 | tag: 35 | git tag $(VERSION) 36 | git push --tags 37 | 38 | test: build-deps 39 | go test -v ./... 40 | 41 | .PHONY: all build-deps clean release tag test 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # apt-s3 2 | ![repo-checks](https://github.com/zendesk/apt-s3/workflows/repo-checks/badge.svg) 3 | 4 | `apt-s3` is an [APT Method Interface](http://www.fifi.org/doc/libapt-pkg-doc/method.html/) written in Go to use a private S3 bucket as an `apt` repository on Debian based systems. Similar projects exist, but they all have their caveats: 5 | * Many are completely unmaintained 6 | * Most require `python` and some require additional `pip` packages 7 | * Some only use the default AWS authentication methods 8 | * This means any application specific credentials in a Docker container must also have access to the S3 bucket or `apt` breaks entirely 9 | * Most set the region globally so they only support a single S3 region at a time 10 | * Some place the API keys in the S3 URI 11 | * This means they are leaked every time `apt-get update` or `apt-get install` is run 12 | * Some do not use the AWS SDK 13 | * None of them expose an interactive component for downloading files 14 | 15 | ## Installation 16 | 17 | The only requirement for `apt-s3` is the `ca-certificates` package and its dependencies. 18 | 19 | Installation is as easy as downloading the binary or deb package from our [releases](https://github.com/zendesk/apt-s3/releases) page. 20 | 21 | ### Package Installation 22 | 23 | Download the package and install it with `dpkg -i /path/to/package.deb`. If you see the error message below simply run `apt-get install -f` to fix it. 24 | ``` 25 | dpkg: dependency problems prevent configuration of apt-s3: 26 | apt-s3 depends on ca-certificates; however: 27 | Package ca-certificates is not installed. 28 | ``` 29 | 30 | ### Binary Installation 31 | 32 | Download the binary and move it to `/usr/lib/apt/methods/s3`. 33 | 34 | ## Usage 35 | 36 | Simply create an apt list file in the proper format to start using `apt-s3` with apt. 37 | ```bash 38 | export BUCKET_NAME=my-s3-bucket 39 | export BUCKET_REGION=us-east-1 40 | 41 | echo "deb s3://${BUCKET_NAME}.s3-${BUCKET_REGION}.amazonaws.com/ stable main" > /etc/apt/sources.list.d/s3bucket.list 42 | ``` 43 | 44 | ### Credentials File 45 | 46 | `/etc/apt/s3creds` is checked before using the default AWS credential methods. The file has a format similar to `~/.aws/credentials`, but profiles are ignored. 47 | 48 | ``` 49 | aws_access_key_id = foo 50 | aws_secret_access_key = foobar123 51 | aws_session_token = not-normally-needed 52 | ``` 53 | 54 | ### Interactive Usage 55 | 56 | To download a file using `apt-s3` simply use the `-download` flag. Run `apt-s3 -help` for usage info. 57 | 58 | ```bash 59 | export BUCKET_NAME=my-s3-bucket 60 | export BUCKET_REGION=us-east-1 61 | 62 | apt-s3 -download s3:/${BUCKET_NAME}.s3-${BUCKET_REGION}.amazonaws.com/file -path /tmp/file 63 | ``` 64 | 65 | ## Building 66 | 67 | Use the Makefile to build the binary and .deb package (requires [nfpm](https://github.com/goreleaser/nfpm) to be installed and in the `$PATH`). 68 | 69 | ```bash 70 | $ make 71 | ``` 72 | 73 | ## Releasing a New Version 74 | 75 | To release a new version you will need a few things: 76 | 77 | 1. Write access to this repo 78 | 2. [A personal access token](https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line) 79 | 3. [nfpm](https://github.com/goreleaser/nfpm) installed and in the `$PATH` 80 | 81 | Once everything is set up follow these steps to create a release and upload assets: 82 | 83 | ```bash 84 | export GITHUB_TOKEN= 85 | # bumping the version programatically can be easily done with `awk` 86 | export VERSION=$(git describe --tags | awk -F. -v OFS=. '{ $3++ } 1') # use $1 for major/$2 for minor/$3 for patch 87 | make VERSION=$VERSION release 88 | ``` 89 | 90 | ## Contributing 91 | 92 | Improvements are always welcome. Please follow these steps to contribute: 93 | 94 | 1. Fork repo 95 | 2. Submit a Pull Request with a detailed explanation of changes 96 | 3. Receive a :+1: from a core team member 97 | 4. Core team will merge your changes 98 | 99 | ## License 100 | 101 | Use of this software is subject to important terms and conditions as set forth in the [LICENSE](LICENSE) file. 102 | -------------------------------------------------------------------------------- /method/method.go: -------------------------------------------------------------------------------- 1 | // Package method interacts with apt via stdin and stdout 2 | package method 3 | 4 | import ( 5 | "bufio" 6 | "bytes" 7 | "crypto/md5" 8 | "crypto/sha1" 9 | "crypto/sha256" 10 | "crypto/sha512" 11 | "fmt" 12 | "hash" 13 | "io" 14 | "io/ioutil" 15 | "os" 16 | "strings" 17 | 18 | "github.com/zendesk/apt-s3/downloader" 19 | ) 20 | 21 | type Method struct { 22 | Downloader *downloader.Downloader 23 | } 24 | 25 | func New() *Method { 26 | m := &Method{ 27 | Downloader: downloader.New(), 28 | } 29 | return m 30 | } 31 | 32 | // calculateHash calculates and returns a single hash. Used by calculateHashes 33 | func (m *Method) calculateHash(h hash.Hash, f []byte) (string, error) { 34 | if _, err := io.Copy(h, bytes.NewReader(f)); err != nil { 35 | return "", err 36 | } 37 | 38 | return fmt.Sprintf("%x", h.Sum(nil)), nil 39 | } 40 | 41 | // calculateHashes returns md5, sha1, sha256, and sha512 hashes of the downloaded file 42 | func (m *Method) calculateHashes(filename string) (string, string, string, string, error) { 43 | f, err := ioutil.ReadFile(filename) 44 | if err != nil { 45 | return "", "", "", "", err 46 | } 47 | 48 | md5h, err := m.calculateHash(md5.New(), f) 49 | if err != nil { 50 | return "", "", "", "", err 51 | } 52 | sha1h, err := m.calculateHash(sha1.New(), f) 53 | if err != nil { 54 | return "", "", "", "", err 55 | } 56 | sha256h, err := m.calculateHash(sha256.New(), f) 57 | if err != nil { 58 | return "", "", "", "", err 59 | } 60 | sha512h, err := m.calculateHash(sha512.New(), f) 61 | if err != nil { 62 | return "", "", "", "", err 63 | } 64 | 65 | return md5h, sha1h, sha256h, sha512h, nil 66 | } 67 | 68 | // sendCapabilities tells apt what this method is capable of 69 | func (m *Method) sendCapabilities() { 70 | fmt.Printf("100 Capabilities\nSend-Config: true\nPipeline: true\nSingle-Instance: yes\n\n") 71 | } 72 | 73 | // findLine finds the line that starts with key: and returns the value 74 | func (m *Method) findLine(key string, lines []string) string { 75 | for i := 0; i < len(lines); i++ { 76 | linesSs := strings.Split(lines[i], ": ") 77 | if linesSs[0] == key { 78 | return linesSs[1] 79 | } 80 | } 81 | 82 | return "" 83 | } 84 | 85 | // UriStart downloads a file from S3 and tells apt about when the download 86 | // starts and is finished 87 | func (m *Method) UriStart(lines []string) error { 88 | uri := m.findLine("URI", lines) 89 | path := m.findLine("Filename", lines) 90 | 91 | lastModified, size, err := m.Downloader.GetFileAttributes(uri) 92 | if err != nil { 93 | m.handleError(uri, err) 94 | } 95 | 96 | fmt.Printf("200 URI Start\nLast-Modified: %s\nSize: %d\nURI: %s\n\n", lastModified, size, uri) 97 | 98 | filename, err := m.Downloader.DownloadFile(uri, path) 99 | if err != nil { 100 | m.handleError(uri, err) 101 | } 102 | md5Hash, sha1Hash, sha256Hash, sha512Hash, err := m.calculateHashes(filename) 103 | if err != nil { 104 | return err 105 | } 106 | fmt.Printf("201 URI Done\nFilename: %s\nLast-Modified: %s\n", filename, lastModified) 107 | fmt.Printf("MD5-Hash: %s\nMD5Sum-Hash: %s\nSHA1-Hash: %s\n", md5Hash, md5Hash, sha1Hash) 108 | fmt.Printf("SHA256-Hash: %s\nSHA512-Hash: %s\n", sha256Hash, sha512Hash) 109 | fmt.Printf("Size: %d\nURI: %s\n\n", size, uri) 110 | 111 | return nil 112 | } 113 | 114 | // handleError sends an error message to os.Stdout in a format which apt 115 | // understands 116 | func (m *Method) handleError(uri string, err error) { 117 | fmt.Printf("400 URI Failure\nMessage: %s\nURI: %s\n\n", strings.TrimRight(fmt.Sprintln(err), "\n"), uri) 118 | os.Exit(1) 119 | } 120 | 121 | // Start watches os.Stdin for a "600 URI Acquire" message from apt which 122 | // triggers UriStart 123 | func (m *Method) Start() { 124 | var lines []string 125 | scanner := bufio.NewScanner(os.Stdin) 126 | m.sendCapabilities() 127 | 128 | for scanner.Scan() { 129 | t := scanner.Text() 130 | if t != "" { 131 | lines = append(lines, t) 132 | } else { 133 | if len(lines) > 0 { 134 | if lines[0] == "600 URI Acquire" { 135 | if err := m.UriStart(lines); err != nil { 136 | m.handleError(strings.Split(lines[1], ": ")[1], err) 137 | } 138 | } 139 | lines = make([]string, 0) 140 | } 141 | } 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /downloader/downloader.go: -------------------------------------------------------------------------------- 1 | // Package downloader parses an s3 URI and downloads the specified file to the 2 | // filesystem. 3 | package downloader 4 | 5 | import ( 6 | "bufio" 7 | "errors" 8 | "os" 9 | "strings" 10 | 11 | "github.com/aws/aws-sdk-go/aws" 12 | "github.com/aws/aws-sdk-go/aws/awserr" 13 | "github.com/aws/aws-sdk-go/aws/credentials" 14 | "github.com/aws/aws-sdk-go/aws/session" 15 | "github.com/aws/aws-sdk-go/service/s3" 16 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 17 | ) 18 | 19 | // Downloader tracks the region and Session and only recreates the Session 20 | // if the region has changed 21 | type Downloader struct { 22 | region string 23 | sess *session.Session 24 | } 25 | 26 | func New() *Downloader { 27 | d := &Downloader{} 28 | return d 29 | } 30 | 31 | // getValue parses a string and returns the value assigned to a key 32 | func (d *Downloader) getValue(line string) string { 33 | splitLine := strings.Split(line, " = ") 34 | return (splitLine[len(splitLine)-1]) 35 | } 36 | 37 | // credentialsFromFile loads AWS credentials from a non-standard path 38 | func (d *Downloader) credentialsFromFile(fileName string) (string, string, string, error) { 39 | var accessKey, secretKey, token string 40 | 41 | file, err := os.Open(fileName) 42 | if err != nil { 43 | return "", "", "", err 44 | } 45 | defer file.Close() 46 | 47 | scanner := bufio.NewScanner(file) 48 | for scanner.Scan() { 49 | switch { 50 | case strings.Contains(scanner.Text(), "aws_access_key_id"): 51 | accessKey = d.getValue(scanner.Text()) 52 | case strings.Contains(scanner.Text(), "aws_secret_access_key"): 53 | secretKey = d.getValue(scanner.Text()) 54 | case strings.Contains(scanner.Text(), "aws_session_token"): 55 | token = d.getValue(scanner.Text()) 56 | } 57 | } 58 | if err := scanner.Err(); err != nil { 59 | return "", "", "", err 60 | } 61 | 62 | return accessKey, secretKey, token, nil 63 | } 64 | 65 | // loadCredentials sets up a Session using credentials found in /etc/apt/s3creds 66 | // or using the default configuration supported by AWS if /etc/apt/s3creds does 67 | // not exist 68 | func (d *Downloader) loadCredentials(region string) (*session.Session, error) { 69 | var config aws.Config 70 | var sess *session.Session 71 | 72 | if _, err := os.Stat("/etc/apt/s3creds"); err == nil { 73 | accessKey, secretKey, token, err := d.credentialsFromFile("/etc/apt/s3creds") 74 | if err != nil { 75 | return nil, err 76 | } 77 | config = aws.Config{ 78 | Region: aws.String(region), 79 | Credentials: credentials.NewStaticCredentials(accessKey, secretKey, token), 80 | } 81 | } else if os.IsNotExist(err) { 82 | config = aws.Config{Region: aws.String(region)} 83 | } 84 | sess, err := session.NewSession(&config) 85 | 86 | return sess, err 87 | } 88 | 89 | // parseUri takes an S3 URI s3://.s3-.amazonaws.com/key/file 90 | // and returns the bucket, region, key, and filename 91 | func (d *Downloader) parseURI(keyString string) (string, string, string, string) { 92 | var region string 93 | ss := strings.Split(keyString, "/") 94 | bucketSs := strings.Split(ss[2], ".") 95 | bucket := bucketSs[0] 96 | regionSs := strings.Split(bucketSs[1], "-") 97 | // Default to us-east-1 if just .s3.amazonaws.com is passed 98 | if len(regionSs) == 1 { 99 | region = "us-east-1" 100 | } else { 101 | region = strings.Join(regionSs[1:], "-") 102 | } 103 | key := strings.Join(ss[3:], "/") 104 | filename := ss[len(ss)-1] 105 | return bucket, region, key, filename 106 | } 107 | 108 | // GetFileAttributes queries the object in S3 and returns the timestamp and 109 | // size in the format expected by apt 110 | func (d *Downloader) GetFileAttributes(s3Uri string) (string, int64, error) { 111 | var err error 112 | bucket, region, key, _ := d.parseURI(s3Uri) 113 | 114 | if d.region != region { 115 | d.region = region 116 | d.sess, err = d.loadCredentials(region) 117 | if err != nil { 118 | return "", -1, err 119 | } 120 | } 121 | 122 | svc := s3.New(d.sess) 123 | 124 | result, err := svc.GetObject(&s3.GetObjectInput{ 125 | Bucket: aws.String(bucket), 126 | Key: aws.String(key), 127 | }) 128 | if err != nil { 129 | if aerr, ok := err.(awserr.Error); ok { 130 | return "", -1, errors.New(strings.Join(strings.Split(aerr.Error(), "\n"), " ")) 131 | } 132 | } 133 | 134 | return result.LastModified.Format("2006-01-02T15:04:05+00:00"), *result.ContentLength, nil 135 | } 136 | 137 | // DownloadFile pulls the file from an S3 bucket and writes it to the specified 138 | // path 139 | func (d *Downloader) DownloadFile(s3Uri string, path string) (string, error) { 140 | var err error 141 | bucket, region, key, filename := d.parseURI(s3Uri) 142 | if path != "" { 143 | filename = path 144 | } 145 | 146 | if d.region != region { 147 | d.region = region 148 | d.sess, err = d.loadCredentials(region) 149 | if err != nil { 150 | return "", err 151 | } 152 | } 153 | downloader := s3manager.NewDownloader(d.sess) 154 | 155 | f, err := os.Create(filename) 156 | if err != nil { 157 | return "", err 158 | } 159 | 160 | if _, err := downloader.Download(f, &s3.GetObjectInput{ 161 | Bucket: aws.String(bucket), 162 | Key: aws.String(key), 163 | }); err != nil { 164 | os.Remove(filename) 165 | return "", err 166 | } 167 | return filename, nil 168 | } 169 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2019 Zendesk 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | --------------------------------------------------------------------------------