├── .dockerignore ├── .github ├── dependabot.yml ├── release.yml ├── stale.yml └── workflows │ ├── goreleaser-run.yml │ ├── goreleaser-test.yml │ ├── ko-build.yml │ ├── ko-test.yaml │ └── main.yml ├── .gitignore ├── LICENSE ├── README.md ├── ci └── .goreleaser.yml ├── go.mod ├── go.sum ├── internal ├── cachecontrol │ ├── cache-control.go │ └── cache-control_test.go └── uploader │ ├── mediastore │ ├── container.go │ └── mediastore.go │ ├── s3 │ └── s3.go │ └── uploader.go └── main.go /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | dist 3 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gomod 4 | directory: "/" 5 | schedule: 6 | interval: weekly 7 | open-pull-requests-limit: 99 8 | - package-ecosystem: github-actions 9 | directory: "/" 10 | schedule: 11 | interval: daily 12 | open-pull-requests-limit: 99 13 | -------------------------------------------------------------------------------- /.github/release.yml: -------------------------------------------------------------------------------- 1 | changelog: 2 | exclude: 3 | authors: 4 | - dependabot 5 | -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | daysUntilStale: 30 2 | daysUntilClose: 5 3 | exemptLabels: 4 | - bug 5 | staleLabel: stale 6 | markComment: > 7 | This issue has been automatically marked as stale because it has not had 8 | recent activity. It will be closed if no further activity occurs. Thank you 9 | for your contributions. 10 | closeComment: false 11 | -------------------------------------------------------------------------------- /.github/workflows/goreleaser-run.yml: -------------------------------------------------------------------------------- 1 | name: goreleaser 2 | on: 3 | push: 4 | tags: 5 | - "*" 6 | 7 | permissions: 8 | contents: write 9 | 10 | jobs: 11 | release: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v4.1.6 15 | with: 16 | fetch-depth: 0 17 | 18 | - name: fetch tags 19 | run: git fetch --tags --force 20 | 21 | - uses: actions/setup-go@v5.0.1 22 | with: 23 | go-version: "1.22.0" 24 | 25 | - uses: goreleaser/goreleaser-action@v5.1.0 26 | env: 27 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 28 | with: 29 | distribution: goreleaser 30 | version: latest 31 | args: release -f ci/.goreleaser.yml 32 | -------------------------------------------------------------------------------- /.github/workflows/goreleaser-test.yml: -------------------------------------------------------------------------------- 1 | name: goreleaser-test 2 | on: 3 | push: 4 | branches: 5 | - main 6 | paths: 7 | - ci/.goreleaser.yml 8 | - .github/workflows/goreleaser-test.yml 9 | pull_request: 10 | paths: 11 | - ci/.goreleaser.yml 12 | - .github/workflows/goreleaser-test.yml 13 | 14 | jobs: 15 | test-goreleaser: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v4.1.6 19 | 20 | - uses: actions/setup-go@v5.0.1 21 | with: 22 | go-version: "1.22.0" 23 | 24 | - uses: goreleaser/goreleaser-action@v5.1.0 25 | with: 26 | distribution: goreleaser 27 | version: latest 28 | args: release --snapshot -f ci/.goreleaser.yml 29 | -------------------------------------------------------------------------------- /.github/workflows/ko-build.yml: -------------------------------------------------------------------------------- 1 | name: ko-build 2 | on: 3 | push: 4 | branches: 5 | - main 6 | tags: 7 | - "*" 8 | 9 | jobs: 10 | build-and-push: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4.1.6 14 | 15 | - name: docker meta 16 | id: meta 17 | uses: docker/metadata-action@v5.5.1 18 | with: 19 | images: fsouza/s3-upload-proxy 20 | sep-tags: "," 21 | tags: | 22 | type=ref,event=branch 23 | type=ref,event=pr 24 | type=semver,pattern={{version}} 25 | type=semver,pattern={{major}}.{{minor}} 26 | type=semver,pattern={{major}} 27 | 28 | - name: login to docker hub 29 | uses: docker/login-action@v3.1.0 30 | with: 31 | username: ${{ secrets.DOCKER_USERNAME }} 32 | password: ${{ secrets.DOCKER_PASSWORD }} 33 | 34 | - uses: actions/setup-go@v5.0.1 35 | id: setup-go 36 | with: 37 | go-version: "1.22.0" 38 | 39 | - uses: ko-build/setup-ko@v0.6 40 | 41 | - name: ko build 42 | run: | 43 | ko build --platform=all --bare --tags="$(echo "${RAW_TAGS}" | sed -e 's;fsouza/s3-upload-proxy:;;g')" 44 | env: 45 | KO_DEFAULTBASEIMAGE: fsouza/alpine-base 46 | KO_DOCKER_REPO: fsouza/s3-upload-proxy 47 | RAW_TAGS: ${{ steps.meta.outputs.tags }} 48 | -------------------------------------------------------------------------------- /.github/workflows/ko-test.yaml: -------------------------------------------------------------------------------- 1 | name: ko-test 2 | on: 3 | pull_request: 4 | branches: 5 | - main 6 | 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v4.1.6 12 | 13 | - name: docker meta 14 | id: meta 15 | uses: docker/metadata-action@v5.5.1 16 | with: 17 | images: fsouza/s3-upload-proxy 18 | sep-tags: "," 19 | tags: | 20 | type=ref,event=branch 21 | type=ref,event=pr 22 | type=semver,pattern={{version}} 23 | type=semver,pattern={{major}}.{{minor}} 24 | type=semver,pattern={{major}} 25 | 26 | - uses: actions/setup-go@v5.0.1 27 | id: setup-go 28 | with: 29 | go-version: "1.22.0" 30 | 31 | - uses: ko-build/setup-ko@v0.6 32 | 33 | - name: ko build 34 | run: | 35 | ko build --platform=all --bare --tags="$(echo "${RAW_TAGS}" | sed -e 's;fsouza/s3-upload-proxy:;;g')" 36 | env: 37 | KO_DEFAULTBASEIMAGE: fsouza/alpine-base 38 | KO_DOCKER_REPO: ko.local/fsouza/s3-upload-proxy 39 | RAW_TAGS: ${{ steps.meta.outputs.tags }} 40 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | on: 3 | push: 4 | branches: 5 | - main 6 | 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | jobs: 12 | test: 13 | name: tests 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v4.1.6 17 | 18 | - uses: actions/setup-go@v5.0.1 19 | with: 20 | go-version: "1.22.0" 21 | 22 | - name: run-tests 23 | run: go test -race -vet all ./... 24 | 25 | staticcheck: 26 | name: staticcheck 27 | runs-on: ubuntu-latest 28 | steps: 29 | - uses: actions/checkout@v4.1.6 30 | 31 | - uses: actions/setup-go@v5.0.1 32 | id: setup-go 33 | with: 34 | go-version: "1.22.0" 35 | 36 | - run: | 37 | go install honnef.co/go/tools/cmd/staticcheck@master 38 | 39 | - name: get staticcheck version 40 | id: get-staticcheck-version 41 | run: | 42 | echo "version=$(staticcheck --version | sed -e 's/,//g')" >>${GITHUB_OUTPUT} 43 | 44 | - name: staticcheck cache 45 | id: staticcheck-cache 46 | uses: actions/cache@v4.0.2 47 | with: 48 | path: ~/.cache/staticcheck 49 | key: "${{ steps.get-staticcheck-version.outputs.version }}-${{ steps.setup-go.outputs.go-version }}" 50 | 51 | - name: run staticcheck 52 | run: | 53 | staticcheck ./... 54 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | s3-upload-proxy 2 | dist 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) Francisco Souza 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 17 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 19 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 20 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 21 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 22 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # s3-upload-proxy 2 | 3 | [![Build Status](https://github.com/fsouza/s3-upload-proxy/workflows/Build/badge.svg)](https://github.com/fsouza/s3-upload-proxy/actions?query=branch:main+workflow:Build) 4 | 5 | Tool for proxying HTTP uploads to S3 buckets and Elemental MediaStore 6 | Containers (added later, bad naming :D). Useful for private network protected environments. 7 | 8 | ## Running locally 9 | 10 | Make sure you have [latest Go](https://golang.org/doc/install), then make sure 11 | you have AWS credentials properly configured (s3-upload-proxy uses the [default 12 | credential provider 13 | chain](https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default), 14 | so you can use environment variables or file-based configuration). 15 | 16 | Having Go and AWS credentials, just set the environment variable 17 | `BUCKET_NAME`, build and start the process: 18 | 19 | ``` 20 | % export BUCKET_NAME=some-bucket 21 | % go build -o s3-upload-proxy 22 | % ./s3-upload-proxy 23 | ``` 24 | 25 | If you want to use MediaStore, the environment variable `UPLOAD_DRIVER` 26 | should be set to "mediastore" and `BUCKET_NAME` must be set to the container 27 | name on mediastore: 28 | 29 | ``` 30 | % export UPLOAD_DRIVER=mediastore BUCKET_NAME=mediastoretest 31 | % go build -o s3-upload-proxy 32 | % ./s3-upload-proxy 33 | ``` 34 | 35 | ## Environment variables 36 | 37 | s3-upload-proxy configuration's is defined using the following environment 38 | variables: 39 | 40 | | Variable | Default value | Required | Description | 41 | | --------------------------- | ------------- | -------- | -------------------------------------------------------------------------------- | 42 | | UPLOAD_DRIVER | s3 | No | Upload driver to use (options are "mediastore" or "s3") | 43 | | BUCKET_NAME | | Yes | Name of the S3 bucket or the mediastore container (depends on the upload driver) | 44 | | HEALTHCHECK_PATH | /healthcheck | No | Path for healthcheck | 45 | | HTTP_PORT | 80 | No | Port to bind (unsigned int) | 46 | | LOG_LEVEL | debug | No | Logging level | 47 | | CACHE_CONTROL_RULES | | No | JSON array with cache control rules (see below) | 48 | | MEDIASTORE_CHUNKED_TRANSFER | false | No | Whether to enable chunked transfer with MediaStore for lower latency | 49 | 50 | ## Defining cache-control rules 51 | 52 | The tool also allow configuration for cache-control rules. The value of the 53 | environment variable `CACHE_CONTROL_RULES` is a JSON array with the rules. An 54 | example: 55 | 56 | ``` 57 | % export CACHE_CONTROL_RULES='[{"regexp":".mp4$","value":"public, max-age=3600"},{"regexp":".ts$","value":"public, max-age=2, s-maxage=999999"},{"regexp":".m3u8$","value":"private"}]' 58 | ``` 59 | 60 | Notice that the extension must include the dot. 61 | 62 | Also available on Docker Hub: https://hub.docker.com/r/fsouza/s3-upload-proxy/. 63 | -------------------------------------------------------------------------------- /ci/.goreleaser.yml: -------------------------------------------------------------------------------- 1 | builds: 2 | - env: 3 | - CGO_ENABLED=0 4 | goos: 5 | - darwin 6 | - linux 7 | goarch: 8 | - amd64 9 | - arm64 10 | archives: 11 | - files: 12 | - LICENSE 13 | - README.md 14 | checksum: 15 | name_template: "checksums.txt" 16 | snapshot: 17 | name_template: "{{ .Tag }}-next" 18 | changelog: 19 | use: github-native 20 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/fsouza/s3-upload-proxy 2 | 3 | require ( 4 | github.com/aws/aws-sdk-go v1.53.0 5 | github.com/kelseyhightower/envconfig v1.4.0 6 | ) 7 | 8 | require github.com/jmespath/go-jmespath v0.4.0 // indirect 9 | 10 | go 1.22 11 | 12 | toolchain go1.22.0 13 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/aws/aws-sdk-go v1.53.0 h1:MMo1x1ggPPxDfHMXJnQudTbGXYlD4UigUAud1DJxPVo= 2 | github.com/aws/aws-sdk-go v1.53.0/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= 3 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 4 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 6 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 7 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 8 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 9 | github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= 10 | github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= 11 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 12 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 13 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 14 | golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= 15 | golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= 16 | golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= 17 | golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= 18 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 19 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 20 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 21 | -------------------------------------------------------------------------------- /internal/cachecontrol/cache-control.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Francisco Souza. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package cachecontrol 6 | 7 | import ( 8 | "bytes" 9 | "encoding/json" 10 | "regexp" 11 | ) 12 | 13 | // Rules is a list of cache control rules. 14 | type Rules []Rule 15 | 16 | // Rule is a mapping of regular expressions to cache-control string rules. 17 | type Rule struct { 18 | Regexp *jregexp `json:"regexp"` 19 | Value string `json:"value"` 20 | } 21 | 22 | // Set loads the list of rules as a JSON-string, allowing values of type Rules 23 | // to be used with envconfig. 24 | func (c *Rules) Set(value string) error { 25 | return json.Unmarshal([]byte(value), c) 26 | } 27 | 28 | // HeaderValue returns the matching cache control rule for the given file name. 29 | func (c Rules) HeaderValue(fileName string) *string { 30 | for _, rule := range c { 31 | if rule.Regexp.re.MatchString(fileName) { 32 | cacheControl := rule.Value 33 | return &cacheControl 34 | } 35 | } 36 | 37 | return nil 38 | } 39 | 40 | type jregexp struct { 41 | re *regexp.Regexp 42 | } 43 | 44 | func (r *jregexp) UnmarshalJSON(data []byte) (err error) { 45 | expr := string(bytes.Trim(data, `"`)) 46 | r.re, err = regexp.Compile(expr) 47 | return err 48 | } 49 | -------------------------------------------------------------------------------- /internal/cachecontrol/cache-control_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Francisco Souza. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package cachecontrol 6 | 7 | import ( 8 | "reflect" 9 | "regexp" 10 | "testing" 11 | 12 | "github.com/aws/aws-sdk-go/aws" 13 | "github.com/kelseyhightower/envconfig" 14 | ) 15 | 16 | func TestCacheControlRulesCanBeLoadedFromEnv(t *testing.T) { 17 | t.Setenv("RULES", `[{"regexp":".mp4$","value":"public, max-age=123456"},{"regexp":".html$","value":"public, max-age=60"}]`) 18 | var value struct { 19 | Rules Rules `envconfig:"RULES"` 20 | } 21 | expectedRules := map[string]Rule{ 22 | regexp.MustCompile(`.mp4$`).String(): {Value: "public, max-age=123456"}, 23 | regexp.MustCompile(`.html$`).String(): {Value: "public, max-age=60"}, 24 | } 25 | err := envconfig.Process("", &value) 26 | if err != nil { 27 | t.Fatal(err) 28 | } 29 | gotRules := map[string]Rule{} 30 | for _, r := range value.Rules { 31 | re := r.Regexp.re 32 | r.Regexp = nil 33 | gotRules[re.String()] = r 34 | } 35 | if !reflect.DeepEqual(gotRules, expectedRules) { 36 | t.Errorf("wrong rules returned\nwant %#v\ngot %#v", expectedRules, gotRules) 37 | } 38 | } 39 | 40 | func TestCacheControlRulesInvalidJSON(t *testing.T) { 41 | t.Setenv("RULES", `[{"regexp:".mp4"},{"regexp":".html",`) 42 | var value struct { 43 | Rules Rules `envconfig:"RULES"` 44 | } 45 | err := envconfig.Process("", &value) 46 | if err == nil { 47 | t.Fatal("unexpected error") 48 | } 49 | } 50 | 51 | func TestCacheControlHeaderValue(t *testing.T) { 52 | rules := Rules{ 53 | Rule{Regexp: &jregexp{re: regexp.MustCompile(`\.mp4$`)}, Value: "public, max-age=123456"}, 54 | Rule{Regexp: &jregexp{re: regexp.MustCompile(`\.html$`)}, Value: "public, max-age=60"}, 55 | Rule{Regexp: &jregexp{re: regexp.MustCompile(`master_.+\.m3u8$`)}, Value: "private"}, 56 | Rule{Regexp: &jregexp{re: regexp.MustCompile(`master\.m3u8$`)}, Value: "public, max-age=1"}, 57 | Rule{Regexp: &jregexp{re: regexp.MustCompile(`\.webm$`)}, Value: "public, max-age=2, s-maxage=123456"}, 58 | Rule{Regexp: &jregexp{re: regexp.MustCompile(`\.mp3$`)}, Value: "public, s-maxage=123456"}, 59 | } 60 | tests := []struct { 61 | input string 62 | expected *string 63 | }{ 64 | { 65 | "https://github.com/some/file.mp4", 66 | aws.String("public, max-age=123456"), 67 | }, 68 | { 69 | "file.mp4", 70 | aws.String("public, max-age=123456"), 71 | }, 72 | { 73 | "some/path/index.html", 74 | aws.String("public, max-age=60"), 75 | }, 76 | { 77 | "video/master.m3u8", 78 | aws.String("public, max-age=1"), 79 | }, 80 | { 81 | "video/master_720p.m3u8", 82 | aws.String("private"), 83 | }, 84 | { 85 | "file.mp3", 86 | aws.String("public, s-maxage=123456"), 87 | }, 88 | { 89 | "video.webm", 90 | aws.String("public, max-age=2, s-maxage=123456"), 91 | }, 92 | { 93 | "some/path/audio.ogg", 94 | nil, 95 | }, 96 | } 97 | for _, test := range tests { 98 | test := test 99 | t.Run(test.input, func(t *testing.T) { 100 | value := rules.HeaderValue(test.input) 101 | if !reflect.DeepEqual(value, test.expected) { 102 | t.Errorf("wrong value returned\nwant %#v\ngot %#v", test.expected, value) 103 | } 104 | }) 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /internal/uploader/mediastore/container.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Francisco Souza. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package mediastore 6 | 7 | import ( 8 | "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/aws/session" 10 | "github.com/aws/aws-sdk-go/service/mediastore" 11 | "github.com/aws/aws-sdk-go/service/mediastoredata" 12 | ) 13 | 14 | func (u *msUploader) getDataClientForContainer(name string) (*mediastoredata.MediaStoreData, error) { 15 | v, ok := u.containers.Load(name) 16 | if !ok { 17 | client, err := u.newDataClient(name) 18 | if err != nil { 19 | return nil, err 20 | } 21 | v = client 22 | u.containers.Store(name, v) 23 | } 24 | return v.(*mediastoredata.MediaStoreData), nil 25 | } 26 | 27 | func (u *msUploader) newDataClient(containerName string) (*mediastoredata.MediaStoreData, error) { 28 | resp, err := u.client.DescribeContainer(&mediastore.DescribeContainerInput{ 29 | ContainerName: aws.String(containerName), 30 | }) 31 | if err != nil { 32 | return nil, err 33 | } 34 | sess, err := session.NewSession(aws.NewConfig().WithEndpoint(*resp.Container.Endpoint)) 35 | if err != nil { 36 | return nil, err 37 | } 38 | client := mediastoredata.New(sess) 39 | return client, nil 40 | } 41 | -------------------------------------------------------------------------------- /internal/uploader/mediastore/mediastore.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Francisco Souza. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package mediastore 6 | 7 | import ( 8 | "sync" 9 | 10 | "github.com/aws/aws-sdk-go/aws" 11 | "github.com/aws/aws-sdk-go/aws/session" 12 | "github.com/aws/aws-sdk-go/service/mediastore" 13 | "github.com/aws/aws-sdk-go/service/mediastoredata" 14 | "github.com/fsouza/s3-upload-proxy/internal/uploader" 15 | ) 16 | 17 | // DriverOptions are the set of options that can change how the mediastore 18 | // driver behaves. 19 | type DriverOptions struct { 20 | ChunkedTransfer bool 21 | } 22 | 23 | // New returns an uploader that sends objects to Elemental MediaStore. 24 | func New(options DriverOptions) (uploader.Uploader, error) { 25 | u := msUploader{uploadAvailability: mediastoredata.UploadAvailabilityStandard} 26 | if options.ChunkedTransfer { 27 | u.uploadAvailability = mediastoredata.UploadAvailabilityStreaming 28 | } 29 | sess, err := session.NewSession() 30 | if err != nil { 31 | return nil, err 32 | } 33 | u.client = mediastore.New(sess) 34 | return &u, nil 35 | } 36 | 37 | type msUploader struct { 38 | client *mediastore.MediaStore 39 | containers sync.Map 40 | uploadAvailability string 41 | } 42 | 43 | func (u *msUploader) Upload(options uploader.Options) error { 44 | client, err := u.getDataClientForContainer(options.Bucket) 45 | if err != nil { 46 | return err 47 | } 48 | input := mediastoredata.PutObjectInput{ 49 | Path: aws.String(options.Path), 50 | ContentType: options.ContentType, 51 | CacheControl: options.CacheControl, 52 | Body: aws.ReadSeekCloser(options.Body), 53 | UploadAvailability: aws.String(u.uploadAvailability), 54 | } 55 | _, err = client.PutObjectWithContext(options.Context, &input) 56 | return err 57 | } 58 | 59 | func (u *msUploader) Delete(options uploader.Options) error { 60 | client, err := u.getDataClientForContainer(options.Bucket) 61 | if err != nil { 62 | return err 63 | } 64 | input := mediastoredata.DeleteObjectInput{Path: aws.String(options.Path)} 65 | _, err = client.DeleteObjectWithContext(options.Context, &input) 66 | return err 67 | } 68 | -------------------------------------------------------------------------------- /internal/uploader/s3/s3.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Francisco Souza. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package s3 6 | 7 | import ( 8 | "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/aws/session" 10 | "github.com/aws/aws-sdk-go/service/s3" 11 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 12 | "github.com/fsouza/s3-upload-proxy/internal/uploader" 13 | ) 14 | 15 | // New returns an uploader that sends objects to S3. 16 | func New() (uploader.Uploader, error) { 17 | u := s3Uploader{} 18 | sess, err := session.NewSession() 19 | if err != nil { 20 | return nil, err 21 | } 22 | u.client = s3.New(sess) 23 | u.upload = s3manager.NewUploader(sess) 24 | return &u, nil 25 | } 26 | 27 | type s3Uploader struct { 28 | client *s3.S3 29 | upload *s3manager.Uploader 30 | } 31 | 32 | func (u *s3Uploader) Upload(options uploader.Options) error { 33 | _, err := u.upload.UploadWithContext(options.Context, &s3manager.UploadInput{ 34 | Bucket: aws.String(options.Bucket), 35 | Key: aws.String(options.Path), 36 | Body: options.Body, 37 | ContentType: options.ContentType, 38 | CacheControl: options.CacheControl, 39 | }) 40 | return err 41 | } 42 | 43 | func (u *s3Uploader) Delete(options uploader.Options) error { 44 | _, err := u.client.DeleteObjectWithContext(options.Context, &s3.DeleteObjectInput{ 45 | Bucket: aws.String(options.Bucket), 46 | Key: aws.String(options.Path), 47 | }) 48 | return err 49 | } 50 | -------------------------------------------------------------------------------- /internal/uploader/uploader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Francisco Souza. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package uploader 6 | 7 | import ( 8 | "context" 9 | "io" 10 | ) 11 | 12 | // Uploader is an interface used to upload objects to remote object store 13 | // servers. 14 | type Uploader interface { 15 | Upload(Options) error 16 | Delete(Options) error 17 | } 18 | 19 | // Options presents the set of options to the Upload method. 20 | type Options struct { 21 | Context context.Context 22 | Bucket string 23 | Path string 24 | Body io.Reader 25 | ContentType *string 26 | CacheControl *string 27 | } 28 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Francisco Souza. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "errors" 9 | "fmt" 10 | "log" 11 | "log/slog" 12 | "mime" 13 | "net" 14 | "net/http" 15 | "os" 16 | "path/filepath" 17 | "strings" 18 | "time" 19 | 20 | "github.com/fsouza/s3-upload-proxy/internal/cachecontrol" 21 | "github.com/fsouza/s3-upload-proxy/internal/uploader" 22 | "github.com/fsouza/s3-upload-proxy/internal/uploader/mediastore" 23 | "github.com/fsouza/s3-upload-proxy/internal/uploader/s3" 24 | "github.com/kelseyhightower/envconfig" 25 | ) 26 | 27 | // Config is the configuration of the s3-uploader. 28 | type Config struct { 29 | BucketName string `envconfig:"BUCKET_NAME" required:"true"` 30 | UploadDriver string `envconfig:"UPLOAD_DRIVER" default:"s3"` 31 | HealthcheckPath string `envconfig:"HEALTHCHECK_PATH" default:"/healthcheck"` 32 | HTTPPort int `envconfig:"HTTP_PORT" default:"80"` 33 | LogLevel string `envconfig:"LOG_LEVEL" default:"debug"` 34 | CacheControl cachecontrol.Rules `envconfig:"CACHE_CONTROL_RULES"` 35 | ChunkedTransfer bool `envconfig:"MEDIASTORE_CHUNKED_TRANSFER"` 36 | } 37 | 38 | func loadConfig() (Config, error) { 39 | var cfg Config 40 | err := envconfig.Process("", &cfg) 41 | if err != nil { 42 | return cfg, err 43 | } 44 | if cfg.UploadDriver != "s3" && cfg.UploadDriver != "mediastore" { 45 | return cfg, errors.New(`invalid UPLOAD_DRIVER, valid options are "s3" and "mediastore"`) 46 | } 47 | if cfg.ChunkedTransfer && cfg.UploadDriver != "mediastore" { 48 | return cfg, errors.New("MEDIASTORE_CHUNKED_TRANSFERS should only be defined for the mediastore UPLOAD_DRIVER") 49 | } 50 | return cfg, nil 51 | } 52 | 53 | func (c *Config) uploader() (uploader.Uploader, error) { 54 | if c.UploadDriver == "s3" { 55 | return s3.New() 56 | } 57 | if c.UploadDriver == "mediastore" { 58 | return mediastore.New(mediastore.DriverOptions{ChunkedTransfer: c.ChunkedTransfer}) 59 | } 60 | return nil, fmt.Errorf("invalid upload driver %q", c.UploadDriver) 61 | } 62 | 63 | func (c *Config) logger() *slog.Logger { 64 | levels := map[string]slog.Level{ 65 | "debug": slog.LevelDebug, 66 | "info": slog.LevelInfo, 67 | "warning": slog.LevelWarn, 68 | "warn": slog.LevelWarn, 69 | "error": slog.LevelError, 70 | } 71 | opts := slog.HandlerOptions{Level: levels[c.LogLevel]} 72 | return slog.New(slog.NewTextHandler(os.Stderr, &opts)) 73 | } 74 | 75 | func healthcheck(w http.ResponseWriter, r *http.Request) { 76 | if r.Method != "GET" { 77 | http.Error(w, "method not allowed", http.StatusMethodNotAllowed) 78 | return 79 | } 80 | w.WriteHeader(http.StatusOK) 81 | } 82 | 83 | func main() { 84 | cfg, err := loadConfig() 85 | if err != nil { 86 | log.Fatal(err) 87 | } 88 | logger := cfg.logger() 89 | 90 | uper, err := cfg.uploader() 91 | if err != nil { 92 | logger.Error("failed to create uploader", err) 93 | os.Exit(1) 94 | } 95 | 96 | http.HandleFunc(cfg.HealthcheckPath, healthcheck) 97 | http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 98 | start := time.Now() 99 | defer r.Body.Close() 100 | if r.Method != http.MethodPost && r.Method != http.MethodPut && r.Method != http.MethodDelete { 101 | http.Error(w, "method not allowed", http.StatusMethodNotAllowed) 102 | return 103 | } 104 | 105 | key := strings.TrimLeft(r.URL.Path, "/") 106 | contentType := mime.TypeByExtension(filepath.Ext(key)) 107 | logger := logger.With( 108 | slog.String("bucket", cfg.BucketName), 109 | slog.String("objectkey", key), 110 | slog.String("contentType", contentType), 111 | ) 112 | options := uploader.Options{ 113 | Bucket: cfg.BucketName, 114 | Path: key, 115 | Body: r.Body, 116 | ContentType: stringPtr(contentType), 117 | Context: r.Context(), 118 | CacheControl: cfg.CacheControl.HeaderValue(key), 119 | } 120 | switch r.Method { 121 | case http.MethodPost, http.MethodPut: 122 | err = uper.Upload(options) 123 | if err != nil { 124 | logger.Error("failed to upload file", err) 125 | http.Error(w, err.Error(), http.StatusInternalServerError) 126 | return 127 | } 128 | logger.Debug(fmt.Sprintf("finished upload in %s", time.Since(start))) 129 | case http.MethodDelete: 130 | err = uper.Delete(options) 131 | if err != nil { 132 | logger.Error("failed to delete file", err) 133 | http.Error(w, err.Error(), http.StatusInternalServerError) 134 | return 135 | } 136 | logger.Debug(fmt.Sprintf("deleted in %s", time.Since(start))) 137 | } 138 | fmt.Fprintln(w, "OK") 139 | }) 140 | 141 | listenAddr := fmt.Sprintf(":%d", cfg.HTTPPort) 142 | listener, err := net.Listen("tcp", listenAddr) 143 | if err != nil { 144 | logger.Error("failed to start listener", err) 145 | os.Exit(1) 146 | } 147 | defer listener.Close() 148 | logger.Info(fmt.Sprintf("listening on %s", listener.Addr())) 149 | http.Serve(listener, nil) 150 | } 151 | 152 | // stringPtr makes empty strings a nil pointer. 153 | func stringPtr(input string) *string { 154 | if input == "" { 155 | return nil 156 | } 157 | return &input 158 | } 159 | --------------------------------------------------------------------------------