├── VERSION ├── catalog-info.yaml ├── .gitignore ├── Makefile ├── .github ├── workflows │ └── notify-ci-status.yml └── PULL_REQUEST_TEMPLATE.md ├── .circleci └── config.yml ├── go.mod ├── cmd ├── README.md └── p3.go ├── README.md ├── golang.mk ├── go.sum ├── LICENSE ├── pathio_test.go ├── pathio.go └── gen_mock_s3handler.go /VERSION: -------------------------------------------------------------------------------- 1 | 5.1.0 2 | 3 | -------------------------------------------------------------------------------- /catalog-info.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: backstage.io/v1alpha1 2 | kind: Component 3 | metadata: 4 | name: pathio 5 | description: go library for transparently writing to and reading from different types of paths (supports stdout, s3, and fs) 6 | owner: unknown 7 | spec: 8 | type: unknown 9 | lifecycle: production 10 | owner: unknown 11 | system: Clever 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # JetBrains 11 | .idea 12 | 13 | # Architecture specific extensions/prefixes 14 | *.[568vq] 15 | [568vq].out 16 | 17 | *.cgo1.go 18 | *.cgo2.c 19 | _cgo_defun.c 20 | _cgo_gotypes.go 21 | _cgo_export.* 22 | 23 | _testmain.go 24 | 25 | *.exe 26 | *.test 27 | p3 28 | 29 | 30 | vendor/ 31 | bin/ 32 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include golang.mk 2 | .DEFAULT_GOAL := test # override default goal set in library makefile 3 | 4 | SHELL := /bin/bash 5 | PKG = github.com/Clever/pathio/v5 6 | PKGS := $(shell go list ./... | grep -v /vendor | grep -v /tools) 7 | $(eval $(call golang-version-check,1.24)) 8 | .PHONY: build test 9 | 10 | build: 11 | go build -o build/p3 $(PKG)/cmd 12 | 13 | audit-gen: gen 14 | $(if \ 15 | $(shell git status -s), \ 16 | $(error "Generated files are not up to date. Please commit the results of `make gen`") \ 17 | @echo "") 18 | 19 | gen: 20 | go generate 21 | 22 | test: $(PKGS) 23 | $(PKGS): golang-test-all-strict-deps 24 | $(call golang-test-all-strict,$@) 25 | 26 | install_deps: 27 | go mod vendor 28 | go build -o bin/mockgen ./vendor/github.com/golang/mock/mockgen 29 | -------------------------------------------------------------------------------- /.github/workflows/notify-ci-status.yml: -------------------------------------------------------------------------------- 1 | name: Notify CI status 2 | 3 | on: 4 | check_suite: 5 | types: [completed] 6 | status: 7 | 8 | jobs: 9 | call-workflow: 10 | if: >- 11 | (github.event.branches[0].name == github.event.repository.default_branch && 12 | (github.event.state == 'error' || github.event.state == 'failure')) || 13 | (github.event.check_suite.head_branch == github.event.repository.default_branch && 14 | github.event.check_suite.conclusion != 'success') 15 | uses: Clever/ci-scripts/.github/workflows/reusable-notify-ci-status.yml@master 16 | secrets: 17 | CIRCLE_CI_INTEGRATIONS_URL: ${{ secrets.CIRCLE_CI_INTEGRATIONS_URL }} 18 | CIRCLE_CI_INTEGRATIONS_USERNAME: ${{ secrets.CIRCLE_CI_INTEGRATIONS_USERNAME }} 19 | CIRCLE_CI_INTEGRATIONS_PASSWORD: ${{ secrets.CIRCLE_CI_INTEGRATIONS_PASSWORD }} 20 | SLACK_BOT_TOKEN: ${{ secrets.DAPPLE_BOT_TOKEN }} 21 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Clever Coding Standards Agreement 2 | 3 | - [ ] Author and Review Statement, "We agree this code adheres to our [Clever Global Coding Standards](https://app.getguru.com/folders/ibabX5oT/Engineering-Standards-Best-Practices?activeCard=a8a444f4-9149-4ec7-a0fd-8ba42519d93e) and other applicable [Coding Standards](https://app.getguru.com/folders/ibabX5oT/Engineering-Standards-Best-Practices)" 4 | 5 | ## Clever Coding Standards Agreement 6 | 7 | - [ ] Author and Review Statement, "We agree this code adheres to our [Clever Global Coding Standards](https://app.getguru.com/folders/ibabX5oT/Engineering-Standards-Best-Practices?activeCard=a8a444f4-9149-4ec7-a0fd-8ba42519d93e) and other applicable [Coding Standards](https://app.getguru.com/folders/ibabX5oT/Engineering-Standards-Best-Practices)" 8 | 9 | ## JIRA 10 | [Link to JIRA](insert url here) 11 | 12 | ## Overview 13 | (insert PR description here) 14 | 15 | ## Testing 16 | (how did you test this) 17 | 18 | ## Rollout 19 | (are there any special rollout considerations? specific steps? risks?) 20 | 21 | ## Rollback 22 | (specific steps? risks?) -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build: 4 | working_directory: ~/go/src/github.com/Clever/pathio 5 | docker: 6 | - image: cimg/go:1.24 7 | environment: 8 | GOPRIVATE: github.com/Clever/* 9 | CIRCLE_ARTIFACTS: /tmp/circleci-artifacts 10 | CIRCLE_TEST_REPORTS: /tmp/circleci-test-results 11 | steps: 12 | - run: 13 | command: cd $HOME && git clone --depth 1 -v https://github.com/Clever/ci-scripts.git && cd ci-scripts && git show --oneline -s 14 | name: Clone ci-scripts 15 | - checkout 16 | - setup_remote_docker 17 | - run: 18 | command: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS 19 | name: Set up CircleCI artifacts directories 20 | - run: 21 | command: git config --global "url.ssh://git@github.com/Clever".insteadOf "https://github.com/Clever" 22 | - run: 23 | name: Add github.com to known hosts 24 | command: mkdir -p ~/.ssh && touch ~/.ssh/known_hosts && echo 'github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=' >> ~/.ssh/known_hosts 25 | - run: make install_deps 26 | - run: make test 27 | - run: make audit-gen 28 | - run: make build 29 | - run: if [ "${CIRCLE_BRANCH}" == "master" ]; then $HOME/ci-scripts/circleci/github-release $GH_RELEASE_TOKEN; fi; 30 | - run: $HOME/ci-scripts/circleci/catalog-sync $CATAPULT_URL $CATAPULT_USER $CATAPULT_PASS pathio utility 31 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Clever/pathio/v5 2 | 3 | go 1.24 4 | 5 | require ( 6 | github.com/alecthomas/kingpin/v2 v2.4.0 7 | github.com/aws/aws-sdk-go-v2 v1.36.4 8 | github.com/aws/aws-sdk-go-v2/config v1.29.16 9 | github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.79 10 | github.com/aws/aws-sdk-go-v2/service/s3 v1.80.2 11 | github.com/aws/smithy-go v1.22.2 12 | github.com/golang/mock v1.6.0 13 | github.com/stretchr/testify v1.8.2 14 | ) 15 | 16 | require ( 17 | github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect 18 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect 19 | github.com/aws/aws-sdk-go-v2/credentials v1.17.69 // indirect 20 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.31 // indirect 21 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35 // indirect 22 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35 // indirect 23 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect 24 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.35 // indirect 25 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect 26 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.3 // indirect 27 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.16 // indirect 28 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.16 // indirect 29 | github.com/aws/aws-sdk-go-v2/service/sso v1.25.4 // indirect 30 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.2 // indirect 31 | github.com/aws/aws-sdk-go-v2/service/sts v1.33.21 // indirect 32 | github.com/davecgh/go-spew v1.1.1 // indirect 33 | github.com/pmezard/go-difflib v1.0.0 // indirect 34 | github.com/xhit/go-str2duration/v2 v2.1.0 // indirect 35 | golang.org/x/mod v0.4.2 // indirect 36 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect 37 | golang.org/x/tools v0.1.1 // indirect 38 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 39 | gopkg.in/yaml.v3 v3.0.1 // indirect 40 | ) 41 | 42 | tool github.com/golang/mock/mockgen 43 | -------------------------------------------------------------------------------- /cmd/README.md: -------------------------------------------------------------------------------- 1 | # p3 2 | -- 3 | 4 | The p3 executable exist to make it easier to manually test pathio. 5 | 6 | ## Usage 7 | 8 | From the project root: 9 | 10 | ``` 11 | make build 12 | 13 | # Uploading a local file to s3 14 | ./build/p3 upload s3://BUCKET/KEY /LOCAL_FILE 15 | 16 | # Downloading an s3 file 17 | ./build/p3 download s3://BUCKET/KEY /LOCAL_FILE 18 | 19 | # Listing contents of an s3 bucket or local file path 20 | ./build/p3 list s3://BUCKET/KEY 21 | ./build/p3 list LOCAL_FILE 22 | 23 | # Check if the s3 or local file path exists 24 | ./build/p3 exists s3://BUCKET/KEY 25 | ./build/p3 exists LOCAL_FILE 26 | 27 | # Delete the s3 object or local file 28 | ./build/p3 delete s3://BUCKET/KEY 29 | ./build/p3 delete LOCAL_FILE 30 | 31 | # Write the contents of the provided string to an s3 object or local file 32 | ./build/p3 write "hello world" s3://BUCKET/KEY 33 | ./build/p3 write "hello world" LOCAL_FILE 34 | 35 | ``` 36 | 37 | Notes for testing: 38 | 39 | * The optional flag `--profile=` can be used for allowing p3 to authenticate using a profile instead of environment 40 | variables. 41 | * This does require you to have profile 42 | under [~/.aws/credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-where) 43 | 44 | ### CLI --help 45 | 46 | ``` 47 | usage: p3 [] [ ...] 48 | 49 | Flags: 50 | --[no-]help Show context-sensitive help (also try --help-long and --help-man). 51 | --profile="" AWS profile to use in lieu of the AWS_SECRET_ACCESS_KEY and AWS_ACCESS_KEY_ID environment variables 52 | 53 | Commands: 54 | help [...] 55 | Show help. 56 | 57 | list 58 | list contents of an S3 path 59 | 60 | download 61 | download contents of an S3 path to a local file 62 | 63 | upload 64 | upload contents of a local file to an S3 path 65 | 66 | delete 67 | delete contents of an S3 path 68 | 69 | exists 70 | check if the s3 path exists 71 | 72 | write 73 | copy contents of a string to a file 74 | 75 | ``` -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pathio 2 | 3 | [![GoDoc](https://godoc.org/github.com/Clever/pathio/v5?status.svg)](https://godoc.org/github.com/Clever/pathio/v5) 4 | 5 | ``` 6 | go get "github.com/Clever/pathio/v5" 7 | ``` 8 | 9 | Package pathio is a package that allows writing to and reading from different 10 | types of paths transparently. It supports two types of paths: 11 | 12 | 1. Local file paths 13 | 2. S3 File Paths (s3://bucket/key) 14 | 15 | Note that using s3 paths requires setting two environment variables 16 | 17 | 1. AWS_SECRET_ACCESS_KEY 18 | 2. AWS_ACCESS_KEY_ID 19 | 20 | ## Usage 21 | 22 | Pathio has a very easy to use interface, with 5 main functions: 23 | 24 | ``` 25 | import "github.com/Clever/pathio/v5" 26 | var err error 27 | ``` 28 | 29 | ## Initialization 30 | 31 | Initializing a new Client: 32 | 33 | ``` 34 | ctx := context.Background() 35 | // Load the default AWS configuration 36 | awsConfig, err := awsV2Config.LoadDefaultConfig(ctx, awsV2Config.WithDefaultRegion("us-east-1")) 37 | if err != nil { 38 | log.Fatalf("failed to load AWS config: %v", err) 39 | } 40 | pathioClient := pathio.NewClient(ctx, &awsConfig) 41 | 42 | ``` 43 | 44 | Using the Default Client (Import the Package): 45 | 46 | ``` 47 | import ( 48 | "github.com/Clever/pathio/v5" 49 | ) 50 | ``` 51 | 52 | ``` 53 | pathioClient := pathio.DefaultClient 54 | arcReader, err := pathioClient.Reader(wd.Input.Archive) 55 | ``` 56 | 57 | ### ListFiles 58 | 59 | ``` 60 | // func ListFiles(path string) ([]string, error) 61 | files, err = pathio.ListFiles("s3://bucket/my/key") // s3 62 | files, err = pathio.ListFiles("/home/me") // local 63 | ``` 64 | 65 | ### Write / WriteReader 66 | 67 | ``` 68 | // func Write(path string, input []byte) error 69 | toWrite := []byte("hello world\n") 70 | err = pathio.Write("s3://bucket/my/key", toWrite) // s3 71 | err = pathio.Write("/home/me/hello_world", toWrite) // local 72 | 73 | // func WriteReader(path string, input io.ReadSeeker) error 74 | toWriteReader, err := os.Open("test.txt") // this implements Read and Seek 75 | err = pathio.WriteReader("s3://bucket/my/key", toWriteReader) // s3 76 | err = pathio.WriteReader("/home/me/hello_world", toWriteReader) // local 77 | ``` 78 | 79 | ### Read 80 | 81 | ``` 82 | // func Reader(path string) (rc io.ReadCloser, err error) 83 | reader, err = pathio.Reader("s3://bucket/key/to/read") // s3 84 | reader, err = pathio.Reader("/home/me/file/to/read") // local 85 | ``` 86 | 87 | ### Delete 88 | 89 | ``` 90 | // func Delete(path string) error 91 | err = pathio.Delete("s3://bucket/key/to/read") // s3 92 | err = pathio.Delete("/home/me/file/to/read") // local 93 | ``` 94 | -------------------------------------------------------------------------------- /cmd/p3.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "log" 8 | "os" 9 | "strings" 10 | "time" 11 | 12 | "github.com/alecthomas/kingpin/v2" 13 | awsV2Config "github.com/aws/aws-sdk-go-v2/config" 14 | 15 | "github.com/Clever/pathio/v5" 16 | ) 17 | 18 | var ( 19 | awsProfile = kingpin.Flag("profile", "AWS profile to use in lieu of the AWS_SECRET_ACCESS_KEY and AWS_ACCESS_KEY_ID environment variables").Default("").String() 20 | 21 | listCommand = kingpin.Command("list", "list contents of an S3 path") 22 | listPath = listCommand.Arg("file_path", "S3 or local path to list the contents").Required().String() 23 | 24 | downloadCommand = kingpin.Command("download", "download contents of an S3 path to a local file") 25 | downloadS3Path = downloadCommand.Arg("s3_path", "S3 path to download").Required().String() 26 | downloadLocalPath = downloadCommand.Arg("local_path", "local file to write to").Required().String() 27 | 28 | uploadCommand = kingpin.Command("upload", "upload contents of a local file to an S3 path") 29 | uploadS3Path = uploadCommand.Arg("s3_path", "S3 path to upload").Required().String() 30 | uploadLocalPath = uploadCommand.Arg("local_path", "local file to write to").Required().String() 31 | 32 | deleteCommand = kingpin.Command("delete", "delete contents of an S3 path") 33 | deletePath = deleteCommand.Arg("file_path", "S3 path or local file path to delete").Required().String() 34 | 35 | existsCommand = kingpin.Command("exists", "check if the s3 path exists") 36 | existsPath = existsCommand.Arg("path", "S3 path or local file path to check existence of").Required().String() 37 | 38 | writeCommand = kingpin.Command("write", "copy contents of a string to a file") 39 | contents = writeCommand.Arg("contents", "string to write to a file").Required().String() 40 | toPath = writeCommand.Arg("destination_path", "the local file path or S3 path to be written to").Required().String() 41 | 42 | presignedURLCommand = kingpin.Command("presigned-url", "generate a presigned URL for an S3 path") 43 | presignedURLPath = presignedURLCommand.Arg("path", "S3 path to generate a presigned URL for").Required().String() 44 | ) 45 | 46 | func newPathioClientWithS3() *pathio.Client { 47 | ctx := context.Background() 48 | var optFns []func(*awsV2Config.LoadOptions) error 49 | if awsProfile != nil { 50 | optFns = append(optFns, awsV2Config.WithSharedConfigProfile(*awsProfile)) 51 | } 52 | cfg, err := awsV2Config.LoadDefaultConfig(ctx, optFns...) 53 | if err != nil { 54 | log.Fatalf("error building p3 aws config: %v", err) 55 | } 56 | return pathio.NewClient(ctx, &cfg) 57 | } 58 | 59 | func main() { 60 | command := kingpin.Parse() 61 | 62 | switch command { 63 | // Pathio's ListFiles 64 | case listCommand.FullCommand(): 65 | listCommandFn() 66 | // Pathio's Reader 67 | case downloadCommand.FullCommand(): 68 | downloadCommandFn() 69 | // Pathio's WriteReader 70 | case uploadCommand.FullCommand(): 71 | uploadCommandFn() 72 | // Pathio's Delete 73 | case deleteCommand.FullCommand(): 74 | deleteCommandFn() 75 | // Pathio's Exists 76 | case existsCommand.FullCommand(): 77 | existsCommandFn() 78 | // Pathio's Write 79 | case writeCommand.FullCommand(): 80 | writeCommandFn() 81 | // Pathio's GeneratePresignedURL 82 | case presignedURLCommand.FullCommand(): 83 | presignedURLCommandFn() 84 | default: 85 | log.Fatalf("unknown command: %s", command) 86 | } 87 | } 88 | 89 | func listCommandFn() { 90 | var client pathio.Pathio 91 | if strings.HasPrefix(*listPath, "s3://") { 92 | client = newPathioClientWithS3() 93 | } else { 94 | client = pathio.DefaultClient 95 | } 96 | 97 | results, err := client.ListFiles(*listPath) 98 | if err != nil { 99 | log.Fatalf("error list file path: %s", err) 100 | } 101 | for _, result := range results { 102 | fmt.Println(result) 103 | } 104 | } 105 | 106 | func downloadCommandFn() { 107 | client := newPathioClientWithS3() 108 | 109 | file, err := os.Create(*downloadLocalPath) 110 | if err != nil { 111 | log.Fatalf("Error creating local file: %s", err) 112 | } 113 | defer file.Close() 114 | reader, err := client.Reader(*downloadS3Path) 115 | if err != nil { 116 | log.Fatalf("Failed to find s3 file: %s", err) 117 | } 118 | defer reader.Close() 119 | _, err = io.Copy(file, reader) 120 | if err != nil { 121 | log.Fatalf("Failed to download and write s3 file: %s", err) 122 | } 123 | fmt.Printf("Downloaded %s to %s\n", *downloadS3Path, *downloadLocalPath) 124 | } 125 | 126 | func uploadCommandFn() { 127 | client := newPathioClientWithS3() 128 | 129 | file, err := os.Open(*uploadLocalPath) 130 | if err != nil { 131 | log.Fatalf("Error opening file to upload: %s", err) 132 | } 133 | defer file.Close() 134 | err = client.WriteReader(*uploadS3Path, file) 135 | if err != nil { 136 | log.Fatalf("Error uploading file: %s", err) 137 | } 138 | fmt.Printf("Uploaded %s to %s\n", *uploadLocalPath, *uploadS3Path) 139 | } 140 | 141 | func deleteCommandFn() { 142 | var client pathio.Pathio 143 | if strings.HasPrefix(*deletePath, "s3://") { 144 | client = newPathioClientWithS3() 145 | } else { 146 | client = pathio.DefaultClient 147 | } 148 | 149 | err := client.Delete(*deletePath) 150 | if err != nil { 151 | log.Fatalf("error deleting file: %s", err) 152 | } 153 | fmt.Printf("Deleted %s successfully\n", *deletePath) 154 | } 155 | 156 | func existsCommandFn() { 157 | var client pathio.Pathio 158 | if strings.HasPrefix(*existsPath, "s3://") { 159 | client = newPathioClientWithS3() 160 | } else { 161 | client = pathio.DefaultClient 162 | } 163 | 164 | exists, err := client.Exists(*existsPath) 165 | if err != nil { 166 | log.Fatalf("error checking if file exists: %s", err) 167 | } 168 | if exists { 169 | fmt.Printf("%s exists\n", *existsPath) 170 | } else { 171 | fmt.Printf("%s does not exist\n", *existsPath) 172 | } 173 | } 174 | 175 | func writeCommandFn() { 176 | var client pathio.Pathio 177 | if strings.HasPrefix(*toPath, "s3://") { 178 | client = newPathioClientWithS3() 179 | } else { 180 | client = pathio.DefaultClient 181 | } 182 | 183 | err := client.Write(*toPath, []byte(*contents)) 184 | if err != nil { 185 | log.Fatalf("error checking if file exists: %s", err) 186 | } 187 | fmt.Printf("Wrote contents to: %s\n", *toPath) 188 | } 189 | 190 | func presignedURLCommandFn() { 191 | client := newPathioClientWithS3() 192 | 193 | presignedURL, err := client.GeneratePresignedURL(*presignedURLPath, 1*time.Hour) 194 | if err != nil { 195 | log.Fatalf("error generating presigned URL: %s", err) 196 | } 197 | fmt.Printf("Presigned URL: %s\n", presignedURL) 198 | } 199 | -------------------------------------------------------------------------------- /golang.mk: -------------------------------------------------------------------------------- 1 | # This is the default Clever Golang Makefile. 2 | # It is stored in the dev-handbook repo, github.com/Clever/dev-handbook 3 | # Please do not alter this file directly. 4 | GOLANG_MK_VERSION := 1.3.1 5 | 6 | SHELL := /bin/bash 7 | SYSTEM := $(shell uname -a | cut -d" " -f1 | tr '[:upper:]' '[:lower:]') 8 | .PHONY: golang-test-deps golang-ensure-curl-installed 9 | 10 | # set timezone to UTC for golang to match circle and deploys 11 | export TZ=UTC 12 | 13 | # go build flags for use across all commands which accept them 14 | export GOFLAGS := -mod=vendor $(GOFLAGS) 15 | 16 | # if the gopath includes several directories, use only the first 17 | GOPATH=$(shell echo $$GOPATH | cut -d: -f1) 18 | 19 | # This block checks and confirms that the proper Go toolchain version is installed. 20 | # It uses ^ matching in the semver sense -- you can be ahead by a minor 21 | # version, but not a major version (patch is ignored). 22 | # arg1: golang version 23 | define golang-version-check 24 | _ := $(if \ 25 | $(shell \ 26 | expr >/dev/null \ 27 | `go version | cut -d" " -f3 | cut -c3- | cut -d. -f2 | sed -E 's/beta[0-9]+//'` \ 28 | \>= `echo $(1) | cut -d. -f2` \ 29 | \& \ 30 | `go version | cut -d" " -f3 | cut -c3- | cut -d. -f1` \ 31 | = `echo $(1) | cut -d. -f1` \ 32 | && echo 1), \ 33 | @echo "", \ 34 | $(error must be running Go version ^$(1) - you are running $(shell go version | cut -d" " -f3 | cut -c3-))) 35 | endef 36 | 37 | # FGT is a utility that exits with 1 whenever any stderr/stdout output is recieved. 38 | # We pin its version since its a simple tool that does its job as-is; 39 | # so we're defended against it breaking or changing in the future. 40 | FGT := $(GOPATH)/bin/fgt 41 | $(FGT): 42 | go install -mod=readonly github.com/GeertJohan/fgt@262f7b11eec07dc7b147c44641236f3212fee89d 43 | 44 | golang-ensure-curl-installed: 45 | @command -v curl >/dev/null 2>&1 || { echo >&2 "curl not installed. Please install curl."; exit 1; } 46 | 47 | # Golint is a tool for linting Golang code for common errors. 48 | # We pin its version because an update could add a new lint check which would make 49 | # previously passing tests start failing without changing our code. 50 | # this package is deprecated and frozen 51 | # Infra recommendation is to eventually move to https://github.com/golangci/golangci-lint so don't fail on linting error for now 52 | GOLINT := $(GOPATH)/bin/golint 53 | $(GOLINT): 54 | go install -mod=readonly golang.org/x/lint/golint@738671d3881b9731cc63024d5d88cf28db875626 55 | 56 | # golang-fmt-deps requires the FGT tool for checking output 57 | golang-fmt-deps: $(FGT) 58 | 59 | # golang-fmt checks that all golang files in the pkg are formatted correctly. 60 | # arg1: pkg path 61 | define golang-fmt 62 | @echo "FORMATTING $(1)..." 63 | @PKG_PATH=$$(go list -f '{{.Dir}}' $(1)); $(FGT) gofmt -l=true $${PKG_PATH}/*.go 64 | endef 65 | 66 | # golang-lint-deps requires the golint tool for golang linting. 67 | golang-lint-deps: $(GOLINT) 68 | 69 | # golang-lint calls golint on all golang files in the pkg. 70 | # arg1: pkg path 71 | define golang-lint 72 | @echo "LINTING $(1)..." 73 | @PKG_PATH=$$(go list -f '{{.Dir}}' $(1)); find $${PKG_PATH}/*.go -type f | grep -v gen_ | xargs $(GOLINT) 74 | endef 75 | 76 | # golang-lint-deps-strict requires the golint tool for golang linting. 77 | golang-lint-deps-strict: $(GOLINT) $(FGT) 78 | 79 | # golang-test-deps is here for consistency 80 | golang-test-deps: 81 | 82 | # golang-test uses the Go toolchain to run all tests in the pkg. 83 | # arg1: pkg path 84 | define golang-test 85 | @echo "TESTING $(1)..." 86 | @go test -v $(1) 87 | endef 88 | 89 | # golang-test-strict-deps is here for consistency 90 | golang-test-strict-deps: 91 | 92 | # golang-test-strict uses the Go toolchain to run all tests in the pkg with the race flag 93 | # arg1: pkg path 94 | define golang-test-strict 95 | @echo "TESTING $(1)..." 96 | @go test -v -race $(1) 97 | endef 98 | 99 | # golang-test-strict-cover-deps is here for consistency 100 | golang-test-strict-cover-deps: 101 | 102 | # golang-test-strict-cover uses the Go toolchain to run all tests in the pkg with the race and cover flag. 103 | # appends coverage results to coverage.txt 104 | # arg1: pkg path 105 | define golang-test-strict-cover 106 | @echo "TESTING $(1)..." 107 | @go test -v -race -cover -coverprofile=profile.tmp -covermode=atomic $(1) 108 | @if [ -f profile.tmp ]; then \ 109 | cat profile.tmp | tail -n +2 >> coverage.txt; \ 110 | rm profile.tmp; \ 111 | fi; 112 | endef 113 | 114 | # golang-vet-deps is here for consistency 115 | golang-vet-deps: 116 | 117 | # golang-vet uses the Go toolchain to vet all the pkg for common mistakes. 118 | # arg1: pkg path 119 | define golang-vet 120 | @echo "VETTING $(1)..." 121 | @go vet $(1) 122 | endef 123 | 124 | # golang-test-all-deps installs all dependencies needed for different test cases. 125 | golang-test-all-deps: golang-fmt-deps golang-lint-deps golang-test-deps golang-vet-deps 126 | 127 | # golang-test-all calls fmt, lint, vet and test on the specified pkg. 128 | # arg1: pkg path 129 | define golang-test-all 130 | $(call golang-fmt,$(1)) 131 | $(call golang-lint,$(1)) 132 | $(call golang-vet,$(1)) 133 | $(call golang-test,$(1)) 134 | endef 135 | 136 | # golang-test-all-strict-deps: installs all dependencies needed for different test cases. 137 | golang-test-all-strict-deps: golang-fmt-deps golang-lint-deps-strict golang-test-strict-deps golang-vet-deps 138 | 139 | # golang-test-all-strict calls fmt, lint, vet and test on the specified pkg with strict 140 | # requirements that no errors are thrown while linting. 141 | # arg1: pkg path 142 | define golang-test-all-strict 143 | $(call golang-fmt,$(1)) 144 | $(call golang-lint,$(1)) 145 | $(call golang-vet,$(1)) 146 | $(call golang-test-strict,$(1)) 147 | endef 148 | 149 | # golang-test-all-strict-cover-deps: installs all dependencies needed for different test cases. 150 | golang-test-all-strict-cover-deps: golang-fmt-deps golang-lint-deps-strict golang-test-strict-cover-deps golang-vet-deps 151 | 152 | # golang-test-all-strict-cover calls fmt, lint, vet and test on the specified pkg with strict and cover 153 | # requirements that no errors are thrown while linting. 154 | # arg1: pkg path 155 | define golang-test-all-strict-cover 156 | $(call golang-fmt,$(1)) 157 | $(call golang-lint,$(1)) 158 | $(call golang-vet,$(1)) 159 | $(call golang-test-strict-cover,$(1)) 160 | endef 161 | 162 | # golang-build: builds a golang binary 163 | # arg1: pkg path 164 | # arg2: executable name 165 | define golang-build 166 | @echo "BUILDING $(2)..." 167 | @CGO_ENABLED=0 go build -o bin/$(2) $(1); 168 | endef 169 | 170 | # golang-debug-build: builds a golang binary with debugging capabilities 171 | # arg1: pkg path 172 | # arg2: executable name 173 | define golang-debug-build 174 | @echo "BUILDING $(2) FOR DEBUG..." 175 | @CGO_ENABLED=0 go build -gcflags="all=-N -l" -o bin/$(2) $(1); 176 | endef 177 | 178 | # golang-cgo-build: builds a golang binary with CGO 179 | # arg1: pkg path 180 | # arg2: executable name 181 | define golang-cgo-build 182 | @echo "BUILDING $(2) WITH CGO ..." 183 | @CGO_ENABLED=1 go build -installsuffix cgo -o bin/$(2) $(1); 184 | endef 185 | 186 | # golang-setup-coverage: set up the coverage file 187 | golang-setup-coverage: 188 | @echo "mode: atomic" > coverage.txt 189 | 190 | # golang-update-makefile downloads latest version of golang.mk 191 | golang-update-makefile: 192 | @wget https://raw.githubusercontent.com/Clever/dev-handbook/master/make/golang-v1.mk -O /tmp/golang.mk 2>/dev/null 193 | @if ! grep -q $(GOLANG_MK_VERSION) /tmp/golang.mk; then cp /tmp/golang.mk golang.mk && echo "golang.mk updated"; else echo "golang.mk is up-to-date"; fi 194 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY= 2 | github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= 3 | github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= 4 | github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= 5 | github.com/aws/aws-sdk-go-v2 v1.36.4 h1:GySzjhVvx0ERP6eyfAbAuAXLtAda5TEy19E5q5W8I9E= 6 | github.com/aws/aws-sdk-go-v2 v1.36.4/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= 7 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= 8 | github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= 9 | github.com/aws/aws-sdk-go-v2/config v1.29.16 h1:XkruGnXX1nEZ+Nyo9v84TzsX+nj86icbFAeust6uo8A= 10 | github.com/aws/aws-sdk-go-v2/config v1.29.16/go.mod h1:uCW7PNjGwZ5cOGZ5jr8vCWrYkGIhPoTNV23Q/tpHKzg= 11 | github.com/aws/aws-sdk-go-v2/credentials v1.17.69 h1:8B8ZQboRc3uaIKjshve/XlvJ570R7BKNy3gftSbS178= 12 | github.com/aws/aws-sdk-go-v2/credentials v1.17.69/go.mod h1:gPME6I8grR1jCqBFEGthULiolzf/Sexq/Wy42ibKK9c= 13 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.31 h1:oQWSGexYasNpYp4epLGZxxjsDo8BMBh6iNWkTXQvkwk= 14 | github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.31/go.mod h1:nc332eGUU+djP3vrMI6blS0woaCfHTe3KiSQUVTMRq0= 15 | github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.79 h1:mGo6WGWry+s5GEf2GLfw3zkHad109FQmtvBV3VYQ8mA= 16 | github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.79/go.mod h1:siwnpWxHYFSSge7Euw9lGMgQBgvRyym352mCuGNHsMQ= 17 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35 h1:o1v1VFfPcDVlK3ll1L5xHsaQAFdNtZ5GXnNR7SwueC4= 18 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35/go.mod h1:rZUQNYMNG+8uZxz9FOerQJ+FceCiodXvixpeRtdESrU= 19 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35 h1:R5b82ubO2NntENm3SAm0ADME+H630HomNJdgv+yZ3xw= 20 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35/go.mod h1:FuA+nmgMRfkzVKYDNEqQadvEMxtxl9+RLT9ribCwEMs= 21 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= 22 | github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= 23 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.35 h1:th/m+Q18CkajTw1iqx2cKkLCij/uz8NMwJFPK91p2ug= 24 | github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.35/go.mod h1:dkJuf0a1Bc8HAA0Zm2MoTGm/WDC18Td9vSbrQ1+VqE8= 25 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= 26 | github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= 27 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.3 h1:VHPZakq2L7w+RLzV54LmQavbvheFaR2u1NomJRSEfcU= 28 | github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.3/go.mod h1:DX1e/lkbsAt0MkY3NgLYuH4jQvRfw8MYxTe9feR7aXM= 29 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.16 h1:/ldKrPPXTC421bTNWrUIpq3CxwHwRI/kpc+jPUTJocM= 30 | github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.16/go.mod h1:5vkf/Ws0/wgIMJDQbjI4p2op86hNW6Hie5QtebrDgT8= 31 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.16 h1:2HuI7vWKhFWsBhIr2Zq8KfFZT6xqaId2XXnXZjkbEuc= 32 | github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.16/go.mod h1:BrwWnsfbFtFeRjdx0iM1ymvlqDX1Oz68JsQaibX/wG8= 33 | github.com/aws/aws-sdk-go-v2/service/s3 v1.80.2 h1:T6Wu+8E2LeTUqzqQ/Bh1EoFNj1u4jUyveMgmTlu9fDU= 34 | github.com/aws/aws-sdk-go-v2/service/s3 v1.80.2/go.mod h1:chSY8zfqmS0OnhZoO/hpPx/BHfAIL80m77HwhRLYScY= 35 | github.com/aws/aws-sdk-go-v2/service/sso v1.25.4 h1:EU58LP8ozQDVroOEyAfcq0cGc5R/FTZjVoYJ6tvby3w= 36 | github.com/aws/aws-sdk-go-v2/service/sso v1.25.4/go.mod h1:CrtOgCcysxMvrCoHnvNAD7PHWclmoFG78Q2xLK0KKcs= 37 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.2 h1:XB4z0hbQtpmBnb1FQYvKaCM7UsS6Y/u8jVBwIUGeCTk= 38 | github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.2/go.mod h1:hwRpqkRxnQ58J9blRDrB4IanlXCpcKmsC83EhG77upg= 39 | github.com/aws/aws-sdk-go-v2/service/sts v1.33.21 h1:nyLjs8sYJShFYj6aiyjCBI3EcLn1udWrQTjEF+SOXB0= 40 | github.com/aws/aws-sdk-go-v2/service/sts v1.33.21/go.mod h1:EhdxtZ+g84MSGrSrHzZiUm9PYiZkrADNja15wtRJSJo= 41 | github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= 42 | github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= 43 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 44 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 45 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 46 | github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= 47 | github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= 48 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 49 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 50 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 51 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 52 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 53 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 54 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 55 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 56 | github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= 57 | github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 58 | github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= 59 | github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= 60 | github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 61 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 62 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 63 | golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= 64 | golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 65 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 66 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 67 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 68 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 69 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 70 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 71 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 72 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 73 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 74 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= 75 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 76 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 77 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 78 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 79 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 80 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 81 | golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= 82 | golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= 83 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 84 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 85 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 86 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 87 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 88 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 89 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 90 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 91 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 92 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 93 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | Copyright 2014 Clever, Inc. 179 | 180 | Licensed under the Apache License, Version 2.0 (the "License"); 181 | you may not use this file except in compliance with the License. 182 | You may obtain a copy of the License at 183 | 184 | http://www.apache.org/licenses/LICENSE-2.0 185 | 186 | Unless required by applicable law or agreed to in writing, software 187 | distributed under the License is distributed on an "AS IS" BASIS, 188 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 189 | See the License for the specific language governing permissions and 190 | limitations under the License. 191 | -------------------------------------------------------------------------------- /pathio_test.go: -------------------------------------------------------------------------------- 1 | package pathio 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "context" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "os" 11 | "strings" 12 | "testing" 13 | "time" 14 | 15 | "github.com/aws/aws-sdk-go-v2/aws" 16 | "github.com/aws/aws-sdk-go-v2/service/s3" 17 | s3Types "github.com/aws/aws-sdk-go-v2/service/s3/types" 18 | "github.com/golang/mock/gomock" 19 | "github.com/stretchr/testify/assert" 20 | ) 21 | 22 | func TestParseS3Path(t *testing.T) { 23 | bucketName, s3path, err := parseS3Path("s3://clever-files/directory/path") 24 | assert.Nil(t, err) 25 | assert.Equal(t, bucketName, "clever-files") 26 | assert.Equal(t, s3path, "directory/path") 27 | 28 | bucketName, s3path, err = parseS3Path("s3://clever-files/directory") 29 | assert.Nil(t, err) 30 | assert.Equal(t, bucketName, "clever-files") 31 | assert.Equal(t, s3path, "directory") 32 | } 33 | 34 | func TestParseInvalidS3Path(t *testing.T) { 35 | _, _, err := parseS3Path("s3://") 36 | assert.EqualError(t, err, "invalid s3 path s3://") 37 | 38 | _, _, err = parseS3Path("s3://ag-ge") 39 | assert.EqualError(t, err, "invalid s3 path s3://ag-ge") 40 | } 41 | 42 | func TestFileReader(t *testing.T) { 43 | // Create a temporary file and write some data to it 44 | file, err := os.CreateTemp("/tmp", "pathioFileReaderTest") 45 | assert.Nil(t, err) 46 | text := "fileReaderTest" 47 | _ = os.WriteFile(file.Name(), []byte(text), 0644) 48 | 49 | reader, err := Reader(file.Name()) 50 | assert.Nil(t, err) 51 | line, _, err := bufio.NewReader(reader).ReadLine() 52 | assert.Nil(t, err) 53 | assert.Equal(t, string(line), text) 54 | } 55 | 56 | func TestWriteToFilePath(t *testing.T) { 57 | file, err := os.CreateTemp("/tmp", "writeToPathTest") 58 | assert.Nil(t, err) 59 | defer os.Remove(file.Name()) 60 | 61 | assert.Nil(t, Write(file.Name(), []byte("testout"))) 62 | output, err := os.ReadFile(file.Name()) 63 | assert.Nil(t, err) 64 | assert.Equal(t, "testout", string(output)) 65 | } 66 | 67 | func TestDefaultClientHasContext(t *testing.T) { 68 | client := DefaultClient.(*Client) 69 | assert.NotNil(t, client.ctx, "DefaultClient should have a valid context to prevent panics") 70 | } 71 | 72 | func TestS3ConnectionInformation(t *testing.T) { 73 | testCases := []struct { 74 | desc string 75 | path string 76 | region string 77 | expectedBucket string 78 | expectedKey string 79 | expectedError string 80 | }{ 81 | { 82 | desc: "ValidS3PathWithRegion", 83 | path: "s3://test-bucket/path/to/file.txt", 84 | region: "us-west-2", 85 | expectedBucket: "test-bucket", 86 | expectedKey: "path/to/file.txt", 87 | }, 88 | { 89 | desc: "InvalidS3Path", 90 | path: "s3://invalid", 91 | region: "", 92 | expectedError: "invalid s3 path s3://invalid", 93 | }, 94 | } 95 | 96 | for _, tc := range testCases { 97 | t.Run(tc.desc, func(t *testing.T) { 98 | ctx := context.Background() 99 | client := &Client{ 100 | ctx: ctx, 101 | providedConfig: &aws.Config{}, 102 | } 103 | 104 | conn, err := client.s3ConnectionInformation(tc.path, tc.region) 105 | 106 | if tc.expectedError != "" { 107 | assert.Error(t, err) 108 | assert.Contains(t, err.Error(), tc.expectedError) 109 | return 110 | } 111 | 112 | assert.NoError(t, err) 113 | assert.Equal(t, tc.expectedBucket, conn.bucket) 114 | assert.Equal(t, tc.expectedKey, conn.key) 115 | }) 116 | } 117 | } 118 | 119 | func TestGetRegionForBucket(t *testing.T) { 120 | testCases := []struct { 121 | desc string 122 | bucketName string 123 | mockConstraint s3Types.BucketLocationConstraint 124 | mockError error 125 | expectedRegion string 126 | expectedError string 127 | }{ 128 | { 129 | desc: "RegionFound", 130 | bucketName: "test-bucket", 131 | mockConstraint: s3Types.BucketLocationConstraint("us-west-1"), 132 | expectedRegion: "us-west-1", 133 | }, 134 | { 135 | desc: "DefaultRegion", 136 | bucketName: "test-bucket", 137 | mockConstraint: "", 138 | expectedRegion: "us-east-1", 139 | }, 140 | { 141 | desc: "Error", 142 | bucketName: "test-bucket", 143 | mockError: errors.New("access denied"), 144 | expectedError: "failed to get location for bucket 'test-bucket', access denied", 145 | }, 146 | } 147 | 148 | for _, tc := range testCases { 149 | t.Run(tc.desc, func(t *testing.T) { 150 | ctrl := gomock.NewController(t) 151 | mockHandler := NewMocks3Handler(ctrl) 152 | 153 | expectedParams := &s3.GetBucketLocationInput{ 154 | Bucket: aws.String(tc.bucketName), 155 | } 156 | 157 | if tc.mockError != nil { 158 | mockHandler.EXPECT(). 159 | GetBucketLocation(gomock.Any(), expectedParams). 160 | Return(&s3.GetBucketLocationOutput{}, tc.mockError) 161 | } else { 162 | mockHandler.EXPECT(). 163 | GetBucketLocation(gomock.Any(), expectedParams). 164 | Return(&s3.GetBucketLocationOutput{LocationConstraint: tc.mockConstraint}, nil) 165 | } 166 | 167 | region, err := getRegionForBucket(context.Background(), mockHandler, tc.bucketName) 168 | 169 | if tc.expectedError != "" { 170 | assert.Error(t, err) 171 | assert.EqualError(t, err, tc.expectedError) 172 | return 173 | } 174 | 175 | assert.NoError(t, err) 176 | assert.Equal(t, tc.expectedRegion, region) 177 | }) 178 | } 179 | } 180 | 181 | func TestS3Calls(t *testing.T) { 182 | testCases := []struct { 183 | desc string 184 | testCase func(svc *Mocks3Handler, t *testing.T) 185 | }{ 186 | { 187 | desc: "GetRegionForBucketSuccess", 188 | testCase: func(svc *Mocks3Handler, t *testing.T) { 189 | name, region := "bucket", "region" 190 | output := s3.GetBucketLocationOutput{LocationConstraint: s3Types.BucketLocationConstraint(region)} 191 | params := s3.GetBucketLocationInput{Bucket: aws.String(name)} 192 | svc.EXPECT().GetBucketLocation(gomock.Any(), ¶ms).Return(&output, nil) 193 | foundRegion, _ := getRegionForBucket(context.TODO(), svc, name) 194 | assert.Equal(t, region, foundRegion) 195 | }, 196 | }, 197 | { 198 | desc: "GetRegionForBucketDefault", 199 | testCase: func(svc *Mocks3Handler, t *testing.T) { 200 | name := "bucket" 201 | output := s3.GetBucketLocationOutput{LocationConstraint: ""} 202 | svc.EXPECT().GetBucketLocation(gomock.Any(), gomock.Any()).Return(&output, nil) 203 | foundRegion, _ := getRegionForBucket(context.TODO(), svc, name) 204 | assert.Equal(t, "us-east-1", foundRegion) 205 | }, 206 | }, 207 | { 208 | desc: "GetRegionForBucketError", 209 | testCase: func(svc *Mocks3Handler, t *testing.T) { 210 | name, err := "bucket", "Error!" 211 | output := s3.GetBucketLocationOutput{LocationConstraint: ""} 212 | svc.EXPECT().GetBucketLocation(gomock.Any(), gomock.Any()).Return(&output, errors.New(err)) 213 | _, foundErr := getRegionForBucket(context.TODO(), svc, name) 214 | assert.Equal(t, foundErr, fmt.Errorf("failed to get location for bucket '%s', %s", name, err)) 215 | }, 216 | }, 217 | { 218 | desc: "S3FileReaderSuccess", 219 | testCase: func(svc *Mocks3Handler, t *testing.T) { 220 | bucket, key, value := "bucket", "key", "value" 221 | reader := io.NopCloser(bytes.NewBuffer([]byte(value))) 222 | output := s3.GetObjectOutput{Body: reader} 223 | params := s3.GetObjectInput{ 224 | Bucket: aws.String(bucket), 225 | Key: aws.String(key), 226 | } 227 | svc.EXPECT().GetObject(gomock.Any(), ¶ms).Return(&output, nil) 228 | foundReader, _ := s3FileReader(context.TODO(), s3Connection{svc, bucket, key}) 229 | body := make([]byte, len(value)) 230 | _, err := foundReader.Read(body) 231 | assert.NoError(t, err) 232 | assert.Equal(t, string(body), value) 233 | }, 234 | }, 235 | { 236 | desc: "S3FileReaderError", 237 | testCase: func(svc *Mocks3Handler, t *testing.T) { 238 | bucket, key, err := "bucket", "key", "Error!" 239 | params := s3.GetObjectInput{ 240 | Bucket: aws.String(bucket), 241 | Key: aws.String(key), 242 | } 243 | output := s3.GetObjectOutput{} 244 | svc.EXPECT().GetObject(gomock.Any(), ¶ms).Return(&output, errors.New(err)) 245 | _, foundErr := s3FileReader(context.TODO(), s3Connection{svc, bucket, key}) 246 | assert.Equal(t, foundErr.Error(), err) 247 | }, 248 | }, 249 | { 250 | desc: "S3FileWriterSuccess", 251 | testCase: func(svc *Mocks3Handler, t *testing.T) { 252 | bucket, key := "bucket", "key" 253 | input := bytes.NewReader(make([]byte, 0)) 254 | output := s3.PutObjectOutput{} 255 | params := s3.PutObjectInput{ 256 | Bucket: aws.String(bucket), 257 | Key: aws.String(key), 258 | Body: input, 259 | ServerSideEncryption: "AES256", 260 | } 261 | svc.EXPECT().PutObject(gomock.Any(), ¶ms).Return(&output, nil) 262 | foundErr := writeToS3(context.TODO(), s3Connection{svc, bucket, key}, input, false) 263 | assert.Equal(t, foundErr, nil) 264 | }, 265 | }, 266 | { 267 | desc: "S3FileWriterError", 268 | testCase: func(svc *Mocks3Handler, t *testing.T) { 269 | bucket, key, err := "bucket", "key", "Error!" 270 | input := bytes.NewReader(make([]byte, 0)) 271 | output := s3.PutObjectOutput{} 272 | params := s3.PutObjectInput{ 273 | Bucket: aws.String(bucket), 274 | Key: aws.String(key), 275 | Body: input, 276 | ServerSideEncryption: "AES256", 277 | } 278 | svc.EXPECT().PutObject(gomock.Any(), ¶ms).Return(&output, errors.New(err)) 279 | foundErr := writeToS3(context.TODO(), s3Connection{svc, bucket, key}, input, false) 280 | assert.Equal(t, foundErr.Error(), err) 281 | }, 282 | }, 283 | { 284 | desc: "S3FileWriterSuccessNoEncryption", 285 | testCase: func(svc *Mocks3Handler, t *testing.T) { 286 | bucket, key := "bucket", "key" 287 | input := bytes.NewReader(make([]byte, 0)) 288 | output := s3.PutObjectOutput{} 289 | params := s3.PutObjectInput{ 290 | Bucket: aws.String(bucket), 291 | Key: aws.String(key), 292 | Body: input, 293 | } 294 | svc.EXPECT().PutObject(gomock.Any(), ¶ms).Return(&output, nil) 295 | foundErr := writeToS3(context.TODO(), s3Connection{svc, bucket, key}, input, true) 296 | assert.Equal(t, foundErr, nil) 297 | }, 298 | }, 299 | { 300 | desc: "S3ListFiles", 301 | testCase: func(svc *Mocks3Handler, t *testing.T) { 302 | bucket, key := "bucket", "key" 303 | output := []*s3.ListObjectsV2Output{ 304 | { 305 | Contents: []s3Types.Object{ 306 | {Key: aws.String("file1")}, 307 | }, 308 | CommonPrefixes: []s3Types.CommonPrefix{ 309 | {Prefix: aws.String("prefix/")}, 310 | }, 311 | IsTruncated: aws.Bool(false), 312 | }, 313 | } 314 | 315 | params := s3.ListObjectsV2Input{ 316 | Bucket: aws.String(bucket), 317 | Prefix: aws.String(key), 318 | Delimiter: aws.String("/"), 319 | } 320 | 321 | svc.EXPECT().ListAllObjects(gomock.Any(), ¶ms).Return(output, nil) 322 | files, err := lsS3(context.TODO(), s3Connection{svc, bucket, key}) 323 | assert.NoError(t, err) 324 | assert.Equal(t, []string{"prefix/", "file1"}, files) 325 | }, 326 | }, 327 | { 328 | desc: "S3ListFilesRecurse", 329 | testCase: func(svc *Mocks3Handler, t *testing.T) { 330 | bucket, key := "bucket", "key" 331 | 332 | output := []*s3.ListObjectsV2Output{ 333 | { 334 | Contents: []s3Types.Object{ 335 | {Key: aws.String("file1")}, 336 | }, 337 | CommonPrefixes: []s3Types.CommonPrefix{ 338 | {Prefix: aws.String("prefix/")}, 339 | {Prefix: aws.String("prefix2/")}, 340 | }, 341 | IsTruncated: aws.Bool(true), 342 | NextContinuationToken: aws.String("file1"), 343 | }, 344 | { 345 | Contents: []s3Types.Object{ 346 | {Key: aws.String("file2")}, 347 | }, 348 | CommonPrefixes: []s3Types.CommonPrefix{ 349 | {Prefix: aws.String("prefix2/")}, 350 | }, 351 | IsTruncated: aws.Bool(false), 352 | }, 353 | } 354 | 355 | params := s3.ListObjectsV2Input{ 356 | Bucket: aws.String(bucket), 357 | Prefix: aws.String(key), 358 | Delimiter: aws.String("/"), 359 | } 360 | 361 | svc.EXPECT().ListAllObjects(gomock.Any(), ¶ms).Return(output, nil) 362 | 363 | files, err := lsS3(context.TODO(), s3Connection{svc, bucket, key}) 364 | assert.NoError(t, err) 365 | assert.Equal(t, []string{"prefix/", "prefix2/", "file1", "file2"}, files) 366 | }, 367 | }, 368 | } 369 | for _, spec := range testCases { 370 | t.Run(spec.desc, func(t *testing.T) { 371 | c := gomock.NewController(t) 372 | svc := NewMocks3Handler(c) 373 | spec.testCase(svc, t) 374 | c.Finish() 375 | }) 376 | } 377 | } 378 | 379 | func TestGeneratePresignedURLE2E(t *testing.T) { 380 | testCases := []struct { 381 | desc string 382 | path string 383 | expiration time.Duration 384 | expectedError string 385 | mockURL string 386 | mockError error 387 | setupMocks func(*Mocks3Handler, string, string, time.Duration) 388 | }{ 389 | { 390 | desc: "ValidS3PathSuccess", 391 | path: "s3://test-bucket/path/to/file.txt", 392 | expiration: 1 * time.Hour, 393 | mockURL: "https://test-bucket.s3.amazonaws.com/path/to/file.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=...", 394 | setupMocks: func(mock *Mocks3Handler, bucket, key string, expiration time.Duration) { 395 | mock.EXPECT(). 396 | GeneratePresignedURL(gomock.Any(), bucket, key, expiration). 397 | Return("https://test-bucket.s3.amazonaws.com/path/to/file.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=...", nil) 398 | }, 399 | }, 400 | { 401 | desc: "InvalidPath", 402 | path: "/local/path/file.txt", 403 | expiration: 1 * time.Hour, 404 | expectedError: "path is not an S3 path (s3://bucket/key), got: /local/path/file.txt", 405 | }, 406 | { 407 | desc: "InvalidS3Path", 408 | path: "s3://invalid", 409 | expiration: 1 * time.Hour, 410 | expectedError: "invalid s3 path s3://invalid", 411 | }, 412 | { 413 | desc: "S3HandlerError", 414 | path: "s3://test-bucket/path/to/file.txt", 415 | expiration: 1 * time.Hour, 416 | mockError: errors.New("access denied"), 417 | setupMocks: func(mock *Mocks3Handler, bucket, key string, expiration time.Duration) { 418 | mock.EXPECT(). 419 | GeneratePresignedURL(gomock.Any(), bucket, key, expiration). 420 | Return("", errors.New("access denied")) 421 | }, 422 | }, 423 | { 424 | desc: "ShortExpiration", 425 | path: "s3://test-bucket/path/to/file.txt", 426 | expiration: 5 * time.Minute, 427 | mockURL: "https://test-bucket.s3.amazonaws.com/path/to/file.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=...", 428 | setupMocks: func(mock *Mocks3Handler, bucket, key string, expiration time.Duration) { 429 | mock.EXPECT(). 430 | GeneratePresignedURL(gomock.Any(), bucket, key, expiration). 431 | Return("https://test-bucket.s3.amazonaws.com/path/to/file.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=...", nil) 432 | }, 433 | }, 434 | { 435 | desc: "LongExpiration", 436 | path: "s3://test-bucket/path/to/file.txt", 437 | expiration: 7 * 24 * time.Hour, // 7 days 438 | mockURL: "https://test-bucket.s3.amazonaws.com/path/to/file.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=...", 439 | setupMocks: func(mock *Mocks3Handler, bucket, key string, expiration time.Duration) { 440 | mock.EXPECT(). 441 | GeneratePresignedURL(gomock.Any(), bucket, key, expiration). 442 | Return("https://test-bucket.s3.amazonaws.com/path/to/file.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=...", nil) 443 | }, 444 | }, 445 | } 446 | 447 | for _, tc := range testCases { 448 | t.Run(tc.desc, func(t *testing.T) { 449 | ctrl := gomock.NewController(t) 450 | defer ctrl.Finish() 451 | 452 | mockHandler := NewMocks3Handler(ctrl) 453 | client := &Client{ 454 | ctx: context.Background(), 455 | } 456 | 457 | if strings.HasPrefix(tc.path, "s3://") && tc.expectedError == "" { 458 | // Parse the S3 path to get bucket and key 459 | bucket, key, err := parseS3Path(tc.path) 460 | if err != nil { 461 | assert.Error(t, err) 462 | assert.Contains(t, err.Error(), tc.expectedError) 463 | return 464 | } 465 | 466 | // Set up mock generation of presigned URL 467 | if tc.setupMocks != nil { 468 | tc.setupMocks(mockHandler, bucket, key, tc.expiration) 469 | } 470 | 471 | // Create s3Connection with our mock 472 | s3Conn := s3Connection{ 473 | handler: mockHandler, 474 | bucket: bucket, 475 | key: key, 476 | } 477 | 478 | // Call the generatePresignedS3URL function 479 | url, err := generatePresignedS3URL(context.Background(), s3Conn, tc.expiration) 480 | 481 | if tc.mockError != nil { 482 | assert.Error(t, err) 483 | assert.EqualError(t, err, tc.mockError.Error()) 484 | assert.Empty(t, url) 485 | return 486 | } 487 | 488 | assert.NoError(t, err) 489 | assert.Equal(t, tc.mockURL, url) 490 | } else { 491 | // Handling non-S3 paths 492 | url, err := client.GeneratePresignedURL(tc.path, tc.expiration) 493 | 494 | if tc.expectedError != "" { 495 | assert.Error(t, err) 496 | assert.Contains(t, err.Error(), tc.expectedError) 497 | assert.Empty(t, url) 498 | return 499 | } 500 | 501 | assert.NoError(t, err) 502 | assert.NotEmpty(t, url) 503 | } 504 | }) 505 | } 506 | } 507 | 508 | func TestPackageLevelGeneratePresignedURLE2E(t *testing.T) { 509 | testCases := []struct { 510 | desc string 511 | path string 512 | expiration time.Duration 513 | expectedError string 514 | }{ 515 | { 516 | desc: "InvalidPath", 517 | path: "/local/path/file.txt", 518 | expiration: 1 * time.Hour, 519 | expectedError: "path is not an S3 path (s3://bucket/key), got: /local/path/file.txt", 520 | }, 521 | { 522 | desc: "InvalidS3Path", 523 | path: "s3://invalid", 524 | expiration: 1 * time.Hour, 525 | expectedError: "invalid s3 path s3://invalid", 526 | }, 527 | } 528 | 529 | for _, tc := range testCases { 530 | t.Run(tc.desc, func(t *testing.T) { 531 | url, err := GeneratePresignedURL(tc.path, tc.expiration) 532 | 533 | assert.Error(t, err) 534 | assert.Contains(t, err.Error(), tc.expectedError) 535 | assert.Empty(t, url) 536 | }) 537 | } 538 | } 539 | -------------------------------------------------------------------------------- /pathio.go: -------------------------------------------------------------------------------- 1 | // Package pathio is a package that allows writing to and reading from different types of paths transparently. 2 | // It supports two types of paths: 3 | // 1. Local file paths 4 | // 2. S3 File Paths (s3://bucket/key) 5 | // 6 | // Note that using s3 paths requires setting two environment variables 7 | // 1. AWS_SECRET_ACCESS_KEY 8 | // 2. AWS_ACCESS_KEY_ID 9 | package pathio 10 | 11 | import ( 12 | "bytes" 13 | "context" 14 | "errors" 15 | "fmt" 16 | "io" 17 | "log" 18 | "os" 19 | "path/filepath" 20 | "strings" 21 | "time" 22 | 23 | "github.com/aws/aws-sdk-go-v2/aws" 24 | awsV2Config "github.com/aws/aws-sdk-go-v2/config" 25 | "github.com/aws/aws-sdk-go-v2/feature/s3/manager" 26 | "github.com/aws/aws-sdk-go-v2/service/s3" 27 | s3Types "github.com/aws/aws-sdk-go-v2/service/s3/types" 28 | "github.com/aws/smithy-go" 29 | ) 30 | 31 | const ( 32 | defaultLocation = "us-east-1" 33 | aesAlgo = "AES256" 34 | ) 35 | 36 | // generate a mock for Pathio 37 | //go:generate bin/mockgen -source=$GOFILE -destination=gen_mock_s3handler.go -package=pathio 38 | 39 | // Pathio is a defined interface for accessing both S3 and local files. 40 | type Pathio interface { 41 | Reader(path string) (rc io.ReadCloser, err error) 42 | Write(path string, input []byte) error 43 | WriteReader(path string, input io.ReadSeeker) error 44 | Delete(path string) error 45 | ListFiles(path string) ([]string, error) 46 | Exists(path string) (bool, error) 47 | GeneratePresignedURL(path string, expiration time.Duration) (string, error) 48 | } 49 | 50 | // Client is the pathio client used to access the local file system and S3. 51 | // To configure options on the client, create a new Client and call its methods 52 | // directly. 53 | // 54 | // &Client{ 55 | // disableS3Encryption: true, // disables encryption 56 | // Region: "us-east-1", // hardcodes the s3 region, instead of looking it up 57 | // }.Write(...) 58 | type Client struct { 59 | ctx context.Context 60 | disableS3Encryption bool 61 | Region string 62 | providedConfig *aws.Config 63 | } 64 | 65 | // DefaultClient is the default pathio client called by the Reader, Writer, and 66 | // WriteReader methods. It has S3 encryption enabled. 67 | var DefaultClient Pathio = &Client{ 68 | ctx: context.Background(), 69 | } 70 | 71 | // NewClient creates a new client that utilizes the provided AWS config. This can 72 | // be leveraged to enforce more limited permissions. 73 | func NewClient(ctx context.Context, cfg *aws.Config) *Client { 74 | return &Client{ 75 | ctx: ctx, 76 | providedConfig: cfg, 77 | } 78 | } 79 | 80 | // Reader calls DefaultClient's Reader method. 81 | func Reader(path string) (rc io.ReadCloser, err error) { 82 | return DefaultClient.Reader(path) 83 | } 84 | 85 | // Write calls DefaultClient's Write method. 86 | func Write(path string, input []byte) error { 87 | return DefaultClient.Write(path, input) 88 | } 89 | 90 | // WriteReader calls DefaultClient's WriteReader method. 91 | func WriteReader(path string, input io.ReadSeeker) error { 92 | return DefaultClient.WriteReader(path, input) 93 | } 94 | 95 | // Delete calls DefaultClient's Delete method. 96 | func Delete(path string) error { 97 | return DefaultClient.Delete(path) 98 | } 99 | 100 | // ListFiles calls DefaultClient's ListFiles method. 101 | func ListFiles(path string) ([]string, error) { 102 | return DefaultClient.ListFiles(path) 103 | } 104 | 105 | // Exists calls DefaultClient's Exists method. 106 | func Exists(path string) (bool, error) { 107 | return DefaultClient.Exists(path) 108 | } 109 | 110 | // GeneratePresignedURL calls DefaultClient's GeneratePresignedURL method. 111 | func GeneratePresignedURL(path string, expiration time.Duration) (string, error) { 112 | return DefaultClient.GeneratePresignedURL(path, expiration) 113 | } 114 | 115 | // S3API defines the interfaces that pathio needs for AWS access. 116 | type S3API interface { 117 | GetBucketLocation(ctx context.Context, params *s3.GetBucketLocationInput, optFns ...func(*s3.Options)) (*s3.GetBucketLocationOutput, error) 118 | 119 | s3.ListObjectsV2APIClient // embedded for s3's ListObjectsV2() 120 | s3.HeadObjectAPIClient // embedded for s3's HeadObject() 121 | manager.DownloadAPIClient // embedded for s3's GetObject() 122 | 123 | manager.UploadAPIClient // embedded for s3's PutObject() 124 | DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) 125 | } 126 | 127 | // s3Handler defines the wrapper interface that pathio uses for AWS access 128 | type s3Handler interface { 129 | GetBucketLocation(ctx context.Context, input *s3.GetBucketLocationInput) (*s3.GetBucketLocationOutput, error) 130 | GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) 131 | DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) 132 | PutObject(ctx context.Context, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) 133 | ListObjects(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) 134 | // ListAllObjects will construct and use a ListObjectsV2 Paginator to fetch all results based on the supplied ListObjectsV2Input 135 | ListAllObjects(ctx context.Context, input *s3.ListObjectsV2Input) ([]*s3.ListObjectsV2Output, error) 136 | HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) 137 | GeneratePresignedURL(ctx context.Context, bucket, key string, expiration time.Duration) (string, error) 138 | } 139 | 140 | type s3Connection struct { 141 | handler s3Handler 142 | bucket string 143 | key string 144 | } 145 | 146 | // Reader returns an io.Reader for the specified path. The path can either be a local file path 147 | // or an S3 path. It is the caller's responsibility to close rc. 148 | func (c *Client) Reader(path string) (rc io.ReadCloser, err error) { 149 | if strings.HasPrefix(path, "s3://") { 150 | s3Conn, err := c.s3ConnectionInformation(path, c.Region) 151 | if err != nil { 152 | return nil, err 153 | } 154 | return s3FileReader(c.ctx, s3Conn) 155 | } 156 | // Local file path 157 | return os.Open(path) 158 | } 159 | 160 | // Write writes a byte array to the specified path. The path can be either a local file path or an 161 | // S3 path. 162 | func (c *Client) Write(path string, input []byte) error { 163 | return c.WriteReader(path, bytes.NewReader(input)) 164 | } 165 | 166 | // WriteReader writes all the data read from the specified io.Reader to the 167 | // output path. The path can either a local file path or an S3 path. 168 | func (c *Client) WriteReader(path string, input io.ReadSeeker) error { 169 | // return the file pointer to the start before reading from it when writing 170 | if offset, err := input.Seek(0, io.SeekStart); err != nil || offset != 0 { 171 | return fmt.Errorf("failed to reset the file pointer to 0. offset: %d; error %s", offset, err) 172 | } 173 | 174 | if strings.HasPrefix(path, "s3://") { 175 | s3Conn, err := c.s3ConnectionInformation(path, c.Region) 176 | if err != nil { 177 | return err 178 | } 179 | return writeToS3(c.ctx, s3Conn, input, c.disableS3Encryption) 180 | } 181 | return writeToLocalFile(path, input) 182 | } 183 | 184 | // Delete deletes the object at the specified path. The path can be either 185 | // a local file path or an S3 path. 186 | func (c *Client) Delete(path string) error { 187 | if strings.HasPrefix(path, "s3://") { 188 | s3Conn, err := c.s3ConnectionInformation(path, c.Region) 189 | if err != nil { 190 | return err 191 | } 192 | return deleteS3Object(c.ctx, s3Conn) 193 | } 194 | // Local file path 195 | return os.Remove(path) 196 | } 197 | 198 | // ListFiles lists all the files/directories in the directory. It does not recurse 199 | func (c *Client) ListFiles(path string) ([]string, error) { 200 | if strings.HasPrefix(path, "s3://") { 201 | s3Conn, err := c.s3ConnectionInformation(path, c.Region) 202 | if err != nil { 203 | return nil, err 204 | } 205 | return lsS3(c.ctx, s3Conn) 206 | } 207 | return lsLocal(path) 208 | } 209 | 210 | // Exists determines if a path does or does not exist. 211 | // NOTE: S3 is eventually consistent so keep in mind that there is a delay. 212 | func (c *Client) Exists(path string) (bool, error) { 213 | if strings.HasPrefix(path, "s3://") { 214 | s3Conn, err := c.s3ConnectionInformation(path, c.Region) 215 | if err != nil { 216 | return false, err 217 | } 218 | return existsS3(c.ctx, s3Conn) 219 | } 220 | return existsLocal(path) 221 | } 222 | 223 | // GeneratePresignedURL generates a pre-signed URL for the specified S3 object. 224 | // The path must be an S3 path (s3://bucket/key). The expiration time determines 225 | // how long the URL will be valid. 226 | func (c *Client) GeneratePresignedURL(path string, expiration time.Duration) (string, error) { 227 | if strings.HasPrefix(path, "s3://") { 228 | s3Conn, err := c.s3ConnectionInformation(path, c.Region) 229 | if err != nil { 230 | return "", err 231 | } 232 | return generatePresignedS3URL(c.ctx, s3Conn, expiration) 233 | } 234 | return "", fmt.Errorf("path is not an S3 path (s3://bucket/key), got: %s", path) 235 | } 236 | 237 | func existsS3(ctx context.Context, s3Conn s3Connection) (bool, error) { 238 | _, err := s3Conn.handler.HeadObject(ctx, &s3.HeadObjectInput{ 239 | Bucket: aws.String(s3Conn.bucket), 240 | Key: aws.String(s3Conn.key), 241 | }) 242 | if err != nil { 243 | var apiError smithy.APIError 244 | if errors.As(err, &apiError) { 245 | var notFound *s3Types.NotFound 246 | switch { 247 | case errors.As(apiError, ¬Found): 248 | return false, nil 249 | default: 250 | return false, err 251 | } 252 | } 253 | return false, err 254 | } 255 | return true, nil 256 | } 257 | 258 | func existsLocal(path string) (bool, error) { 259 | _, err := os.Stat(path) 260 | if os.IsNotExist(err) { 261 | return false, nil 262 | } 263 | return err == nil, err 264 | } 265 | 266 | func lsS3(ctx context.Context, s3Conn s3Connection) ([]string, error) { 267 | params := s3.ListObjectsV2Input{ 268 | Bucket: aws.String(s3Conn.bucket), 269 | Prefix: aws.String(s3Conn.key), 270 | Delimiter: aws.String("/"), 271 | } 272 | var finalResults []string 273 | 274 | // s3 ListObjects limits the response to 1000 objects and marks as truncated if there were more 275 | // In this case we set a Marker that the next query will start from. 276 | // We also ensure that prefixes are not duplicated 277 | pages, err := s3Conn.handler.ListAllObjects(ctx, ¶ms) 278 | if err != nil { 279 | return nil, err 280 | } 281 | for _, page := range pages { 282 | if len(page.CommonPrefixes) > 0 && elementInSlice(finalResults, *page.CommonPrefixes[0].Prefix) { 283 | page.CommonPrefixes = page.CommonPrefixes[1:] 284 | } 285 | results := make([]string, len(page.Contents)+len(page.CommonPrefixes)) 286 | for i, val := range page.CommonPrefixes { 287 | results[i] = *val.Prefix 288 | } 289 | for i, val := range page.Contents { 290 | results[i+len(page.CommonPrefixes)] = *val.Key 291 | } 292 | finalResults = append(finalResults, results...) 293 | } 294 | return finalResults, nil 295 | } 296 | 297 | func elementInSlice(slice []string, elem string) bool { 298 | for _, v := range slice { 299 | if elem == v { 300 | return true 301 | } 302 | } 303 | return false 304 | } 305 | 306 | func lsLocal(path string) ([]string, error) { 307 | resp, err := os.ReadDir(path) 308 | if err != nil { 309 | return nil, err 310 | } 311 | results := make([]string, len(resp)) 312 | for i, val := range resp { 313 | results[i] = val.Name() 314 | } 315 | return results, nil 316 | } 317 | 318 | // s3FileReader converts an S3Path into an io.ReadCloser 319 | func s3FileReader(ctx context.Context, s3Conn s3Connection) (io.ReadCloser, error) { 320 | params := s3.GetObjectInput{ 321 | Bucket: aws.String(s3Conn.bucket), 322 | Key: aws.String(s3Conn.key), 323 | } 324 | resp, err := s3Conn.handler.GetObject(ctx, ¶ms) 325 | if err != nil { 326 | return nil, err 327 | } 328 | return resp.Body, nil 329 | } 330 | 331 | // writeToS3 uploads the given file to S3 332 | func writeToS3(ctx context.Context, s3Conn s3Connection, input io.ReadSeeker, disableEncryption bool) error { 333 | params := s3.PutObjectInput{ 334 | Bucket: aws.String(s3Conn.bucket), 335 | Key: aws.String(s3Conn.key), 336 | Body: input, 337 | } 338 | if !disableEncryption { 339 | params.ServerSideEncryption = aesAlgo 340 | } 341 | _, err := s3Conn.handler.PutObject(ctx, ¶ms) 342 | return err 343 | } 344 | 345 | // deleteS3Object deletes the file on S3 at the given path 346 | func deleteS3Object(ctx context.Context, s3Conn s3Connection) error { 347 | params := s3.DeleteObjectInput{ 348 | Bucket: aws.String(s3Conn.bucket), 349 | Key: aws.String(s3Conn.key), 350 | } 351 | 352 | _, err := s3Conn.handler.DeleteObject(ctx, ¶ms) 353 | return err 354 | } 355 | 356 | // generatePresignedS3URL generates a pre-signed URL for the specified S3 object 357 | func generatePresignedS3URL(ctx context.Context, s3Conn s3Connection, expiration time.Duration) (string, error) { 358 | return s3Conn.handler.GeneratePresignedURL(ctx, s3Conn.bucket, s3Conn.key, expiration) 359 | } 360 | 361 | // writeToLocalFile writes the given file locally 362 | func writeToLocalFile(path string, input io.ReadSeeker) error { 363 | if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { 364 | return err 365 | } 366 | file, err := os.Create(path) 367 | defer file.Close() 368 | if err != nil { 369 | return err 370 | } 371 | _, err = io.Copy(file, input) 372 | return err 373 | } 374 | 375 | // parseS3path parses an S3 path (s3://bucket/key) and returns a bucket, key, error tuple 376 | func parseS3Path(path string) (string, string, error) { 377 | // S3 path names are of the form s3://bucket/key 378 | stringsArray := strings.SplitN(path, "/", 4) 379 | if len(stringsArray) < 4 { 380 | return "", "", fmt.Errorf("invalid s3 path %s", path) 381 | } 382 | bucketName := stringsArray[2] 383 | // Everything after the third slash is the key 384 | key := stringsArray[3] 385 | return bucketName, key, nil 386 | } 387 | 388 | // s3ConnectionInformation parses the s3 path and returns the s3 connection from the 389 | // correct region, as well as the bucket, and key 390 | func (c *Client) s3ConnectionInformation(path, region string) (s3Connection, error) { 391 | bucket, key, err := parseS3Path(path) 392 | if err != nil { 393 | return s3Connection{}, err 394 | } 395 | 396 | // If no region passed in, look up region in S3 397 | if region == "" { 398 | region, err = getRegionForBucket(c.ctx, c.newS3Handler(c.ctx, defaultLocation), bucket) 399 | if err != nil { 400 | return s3Connection{}, err 401 | } 402 | } 403 | 404 | return s3Connection{c.newS3Handler(c.ctx, region), bucket, key}, nil 405 | } 406 | 407 | // getRegionForBucket looks up the region name for the given bucket 408 | func getRegionForBucket(ctx context.Context, svc s3Handler, name string) (string, error) { 409 | // Any region will work for the region lookup, but the request MUST use 410 | // PathStyle 411 | params := s3.GetBucketLocationInput{ 412 | Bucket: aws.String(name), 413 | } 414 | resp, err := svc.GetBucketLocation(ctx, ¶ms) 415 | if err != nil { 416 | return "", fmt.Errorf("failed to get location for bucket '%s', %s", name, err) 417 | } 418 | if resp.LocationConstraint == "" { 419 | return defaultLocation, nil 420 | } 421 | return string(resp.LocationConstraint), nil 422 | } 423 | 424 | type liveS3Handler struct { 425 | liveS3 S3API 426 | // Store the full S3 client for presigned URL generation 427 | s3Client *s3.Client 428 | } 429 | 430 | func (m *liveS3Handler) GetBucketLocation(ctx context.Context, input *s3.GetBucketLocationInput) (*s3.GetBucketLocationOutput, error) { 431 | return m.liveS3.GetBucketLocation(ctx, input) 432 | } 433 | 434 | func (m *liveS3Handler) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { 435 | return m.liveS3.GetObject(ctx, input) 436 | } 437 | 438 | func (m *liveS3Handler) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { 439 | return m.liveS3.DeleteObject(ctx, input) 440 | } 441 | 442 | func (m *liveS3Handler) PutObject(ctx context.Context, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) { 443 | return m.liveS3.PutObject(ctx, input) 444 | } 445 | 446 | func (m *liveS3Handler) ListObjects(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { 447 | return m.liveS3.ListObjectsV2(ctx, input) 448 | } 449 | 450 | // ListAllObjects will utilize a ListObjectsV2Paginator to collate all responses 451 | func (m *liveS3Handler) ListAllObjects(ctx context.Context, input *s3.ListObjectsV2Input) ([]*s3.ListObjectsV2Output, error) { 452 | // code reference: https://github.com/aws/aws-sdk-go-v2/blob/example/service/s3/listObjects/v0.2.9/example/service/s3/listObjects/listObjects.go 453 | var pages []*s3.ListObjectsV2Output 454 | pager := s3.NewListObjectsV2Paginator(m.liveS3, input) 455 | 456 | for pager.HasMorePages() { 457 | page, err := pager.NextPage(ctx) 458 | if err != nil { 459 | return nil, err 460 | } 461 | 462 | pages = append(pages, page) 463 | } 464 | 465 | return pages, nil 466 | } 467 | 468 | func (m *liveS3Handler) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { 469 | return m.liveS3.HeadObject(ctx, input) 470 | } 471 | 472 | func (m *liveS3Handler) GeneratePresignedURL(ctx context.Context, bucket, key string, expiration time.Duration) (string, error) { 473 | if m.s3Client == nil { 474 | return "", fmt.Errorf("S3 client not available for presigned URL generation") 475 | } 476 | 477 | presignClient := s3.NewPresignClient(m.s3Client) 478 | 479 | request, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{ 480 | Bucket: aws.String(bucket), 481 | Key: aws.String(key), 482 | }, func(opts *s3.PresignOptions) { 483 | opts.Expires = expiration 484 | }) 485 | if err != nil { 486 | return "", err 487 | } 488 | 489 | return request.URL, nil 490 | } 491 | 492 | func (c *Client) newS3Handler(ctx context.Context, region string) *liveS3Handler { 493 | if c.providedConfig != nil { 494 | s3Client := s3.NewFromConfig(*c.providedConfig, func(o *s3.Options) { 495 | o.Region = region 496 | o.UsePathStyle = true 497 | }) 498 | return &liveS3Handler{ 499 | liveS3: s3Client, 500 | s3Client: s3Client, 501 | } 502 | } 503 | 504 | awsConfig, err := awsV2Config.LoadDefaultConfig(ctx, awsV2Config.WithRegion(region)) 505 | if err != nil { 506 | log.Fatalf("aws v2 config error: %s", err.Error()) 507 | } 508 | 509 | s3Client := s3.NewFromConfig(awsConfig) 510 | return &liveS3Handler{ 511 | liveS3: s3Client, 512 | s3Client: s3Client, 513 | } 514 | } 515 | -------------------------------------------------------------------------------- /gen_mock_s3handler.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: pathio.go 3 | 4 | // Package pathio is a generated GoMock package. 5 | package pathio 6 | 7 | import ( 8 | context "context" 9 | io "io" 10 | reflect "reflect" 11 | time "time" 12 | 13 | s3 "github.com/aws/aws-sdk-go-v2/service/s3" 14 | gomock "github.com/golang/mock/gomock" 15 | ) 16 | 17 | // MockPathio is a mock of Pathio interface. 18 | type MockPathio struct { 19 | ctrl *gomock.Controller 20 | recorder *MockPathioMockRecorder 21 | } 22 | 23 | // MockPathioMockRecorder is the mock recorder for MockPathio. 24 | type MockPathioMockRecorder struct { 25 | mock *MockPathio 26 | } 27 | 28 | // NewMockPathio creates a new mock instance. 29 | func NewMockPathio(ctrl *gomock.Controller) *MockPathio { 30 | mock := &MockPathio{ctrl: ctrl} 31 | mock.recorder = &MockPathioMockRecorder{mock} 32 | return mock 33 | } 34 | 35 | // EXPECT returns an object that allows the caller to indicate expected use. 36 | func (m *MockPathio) EXPECT() *MockPathioMockRecorder { 37 | return m.recorder 38 | } 39 | 40 | // Delete mocks base method. 41 | func (m *MockPathio) Delete(path string) error { 42 | m.ctrl.T.Helper() 43 | ret := m.ctrl.Call(m, "Delete", path) 44 | ret0, _ := ret[0].(error) 45 | return ret0 46 | } 47 | 48 | // Delete indicates an expected call of Delete. 49 | func (mr *MockPathioMockRecorder) Delete(path interface{}) *gomock.Call { 50 | mr.mock.ctrl.T.Helper() 51 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockPathio)(nil).Delete), path) 52 | } 53 | 54 | // Exists mocks base method. 55 | func (m *MockPathio) Exists(path string) (bool, error) { 56 | m.ctrl.T.Helper() 57 | ret := m.ctrl.Call(m, "Exists", path) 58 | ret0, _ := ret[0].(bool) 59 | ret1, _ := ret[1].(error) 60 | return ret0, ret1 61 | } 62 | 63 | // Exists indicates an expected call of Exists. 64 | func (mr *MockPathioMockRecorder) Exists(path interface{}) *gomock.Call { 65 | mr.mock.ctrl.T.Helper() 66 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exists", reflect.TypeOf((*MockPathio)(nil).Exists), path) 67 | } 68 | 69 | // GeneratePresignedURL mocks base method. 70 | func (m *MockPathio) GeneratePresignedURL(path string, expiration time.Duration) (string, error) { 71 | m.ctrl.T.Helper() 72 | ret := m.ctrl.Call(m, "GeneratePresignedURL", path, expiration) 73 | ret0, _ := ret[0].(string) 74 | ret1, _ := ret[1].(error) 75 | return ret0, ret1 76 | } 77 | 78 | // GeneratePresignedURL indicates an expected call of GeneratePresignedURL. 79 | func (mr *MockPathioMockRecorder) GeneratePresignedURL(path, expiration interface{}) *gomock.Call { 80 | mr.mock.ctrl.T.Helper() 81 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GeneratePresignedURL", reflect.TypeOf((*MockPathio)(nil).GeneratePresignedURL), path, expiration) 82 | } 83 | 84 | // ListFiles mocks base method. 85 | func (m *MockPathio) ListFiles(path string) ([]string, error) { 86 | m.ctrl.T.Helper() 87 | ret := m.ctrl.Call(m, "ListFiles", path) 88 | ret0, _ := ret[0].([]string) 89 | ret1, _ := ret[1].(error) 90 | return ret0, ret1 91 | } 92 | 93 | // ListFiles indicates an expected call of ListFiles. 94 | func (mr *MockPathioMockRecorder) ListFiles(path interface{}) *gomock.Call { 95 | mr.mock.ctrl.T.Helper() 96 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListFiles", reflect.TypeOf((*MockPathio)(nil).ListFiles), path) 97 | } 98 | 99 | // Reader mocks base method. 100 | func (m *MockPathio) Reader(path string) (io.ReadCloser, error) { 101 | m.ctrl.T.Helper() 102 | ret := m.ctrl.Call(m, "Reader", path) 103 | ret0, _ := ret[0].(io.ReadCloser) 104 | ret1, _ := ret[1].(error) 105 | return ret0, ret1 106 | } 107 | 108 | // Reader indicates an expected call of Reader. 109 | func (mr *MockPathioMockRecorder) Reader(path interface{}) *gomock.Call { 110 | mr.mock.ctrl.T.Helper() 111 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reader", reflect.TypeOf((*MockPathio)(nil).Reader), path) 112 | } 113 | 114 | // Write mocks base method. 115 | func (m *MockPathio) Write(path string, input []byte) error { 116 | m.ctrl.T.Helper() 117 | ret := m.ctrl.Call(m, "Write", path, input) 118 | ret0, _ := ret[0].(error) 119 | return ret0 120 | } 121 | 122 | // Write indicates an expected call of Write. 123 | func (mr *MockPathioMockRecorder) Write(path, input interface{}) *gomock.Call { 124 | mr.mock.ctrl.T.Helper() 125 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockPathio)(nil).Write), path, input) 126 | } 127 | 128 | // WriteReader mocks base method. 129 | func (m *MockPathio) WriteReader(path string, input io.ReadSeeker) error { 130 | m.ctrl.T.Helper() 131 | ret := m.ctrl.Call(m, "WriteReader", path, input) 132 | ret0, _ := ret[0].(error) 133 | return ret0 134 | } 135 | 136 | // WriteReader indicates an expected call of WriteReader. 137 | func (mr *MockPathioMockRecorder) WriteReader(path, input interface{}) *gomock.Call { 138 | mr.mock.ctrl.T.Helper() 139 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteReader", reflect.TypeOf((*MockPathio)(nil).WriteReader), path, input) 140 | } 141 | 142 | // MockS3API is a mock of S3API interface. 143 | type MockS3API struct { 144 | ctrl *gomock.Controller 145 | recorder *MockS3APIMockRecorder 146 | } 147 | 148 | // MockS3APIMockRecorder is the mock recorder for MockS3API. 149 | type MockS3APIMockRecorder struct { 150 | mock *MockS3API 151 | } 152 | 153 | // NewMockS3API creates a new mock instance. 154 | func NewMockS3API(ctrl *gomock.Controller) *MockS3API { 155 | mock := &MockS3API{ctrl: ctrl} 156 | mock.recorder = &MockS3APIMockRecorder{mock} 157 | return mock 158 | } 159 | 160 | // EXPECT returns an object that allows the caller to indicate expected use. 161 | func (m *MockS3API) EXPECT() *MockS3APIMockRecorder { 162 | return m.recorder 163 | } 164 | 165 | // AbortMultipartUpload mocks base method. 166 | func (m *MockS3API) AbortMultipartUpload(arg0 context.Context, arg1 *s3.AbortMultipartUploadInput, arg2 ...func(*s3.Options)) (*s3.AbortMultipartUploadOutput, error) { 167 | m.ctrl.T.Helper() 168 | varargs := []interface{}{arg0, arg1} 169 | for _, a := range arg2 { 170 | varargs = append(varargs, a) 171 | } 172 | ret := m.ctrl.Call(m, "AbortMultipartUpload", varargs...) 173 | ret0, _ := ret[0].(*s3.AbortMultipartUploadOutput) 174 | ret1, _ := ret[1].(error) 175 | return ret0, ret1 176 | } 177 | 178 | // AbortMultipartUpload indicates an expected call of AbortMultipartUpload. 179 | func (mr *MockS3APIMockRecorder) AbortMultipartUpload(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { 180 | mr.mock.ctrl.T.Helper() 181 | varargs := append([]interface{}{arg0, arg1}, arg2...) 182 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AbortMultipartUpload", reflect.TypeOf((*MockS3API)(nil).AbortMultipartUpload), varargs...) 183 | } 184 | 185 | // CompleteMultipartUpload mocks base method. 186 | func (m *MockS3API) CompleteMultipartUpload(arg0 context.Context, arg1 *s3.CompleteMultipartUploadInput, arg2 ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error) { 187 | m.ctrl.T.Helper() 188 | varargs := []interface{}{arg0, arg1} 189 | for _, a := range arg2 { 190 | varargs = append(varargs, a) 191 | } 192 | ret := m.ctrl.Call(m, "CompleteMultipartUpload", varargs...) 193 | ret0, _ := ret[0].(*s3.CompleteMultipartUploadOutput) 194 | ret1, _ := ret[1].(error) 195 | return ret0, ret1 196 | } 197 | 198 | // CompleteMultipartUpload indicates an expected call of CompleteMultipartUpload. 199 | func (mr *MockS3APIMockRecorder) CompleteMultipartUpload(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { 200 | mr.mock.ctrl.T.Helper() 201 | varargs := append([]interface{}{arg0, arg1}, arg2...) 202 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteMultipartUpload", reflect.TypeOf((*MockS3API)(nil).CompleteMultipartUpload), varargs...) 203 | } 204 | 205 | // CreateMultipartUpload mocks base method. 206 | func (m *MockS3API) CreateMultipartUpload(arg0 context.Context, arg1 *s3.CreateMultipartUploadInput, arg2 ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error) { 207 | m.ctrl.T.Helper() 208 | varargs := []interface{}{arg0, arg1} 209 | for _, a := range arg2 { 210 | varargs = append(varargs, a) 211 | } 212 | ret := m.ctrl.Call(m, "CreateMultipartUpload", varargs...) 213 | ret0, _ := ret[0].(*s3.CreateMultipartUploadOutput) 214 | ret1, _ := ret[1].(error) 215 | return ret0, ret1 216 | } 217 | 218 | // CreateMultipartUpload indicates an expected call of CreateMultipartUpload. 219 | func (mr *MockS3APIMockRecorder) CreateMultipartUpload(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { 220 | mr.mock.ctrl.T.Helper() 221 | varargs := append([]interface{}{arg0, arg1}, arg2...) 222 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateMultipartUpload", reflect.TypeOf((*MockS3API)(nil).CreateMultipartUpload), varargs...) 223 | } 224 | 225 | // DeleteObject mocks base method. 226 | func (m *MockS3API) DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) { 227 | m.ctrl.T.Helper() 228 | varargs := []interface{}{ctx, params} 229 | for _, a := range optFns { 230 | varargs = append(varargs, a) 231 | } 232 | ret := m.ctrl.Call(m, "DeleteObject", varargs...) 233 | ret0, _ := ret[0].(*s3.DeleteObjectOutput) 234 | ret1, _ := ret[1].(error) 235 | return ret0, ret1 236 | } 237 | 238 | // DeleteObject indicates an expected call of DeleteObject. 239 | func (mr *MockS3APIMockRecorder) DeleteObject(ctx, params interface{}, optFns ...interface{}) *gomock.Call { 240 | mr.mock.ctrl.T.Helper() 241 | varargs := append([]interface{}{ctx, params}, optFns...) 242 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteObject", reflect.TypeOf((*MockS3API)(nil).DeleteObject), varargs...) 243 | } 244 | 245 | // GetBucketLocation mocks base method. 246 | func (m *MockS3API) GetBucketLocation(ctx context.Context, params *s3.GetBucketLocationInput, optFns ...func(*s3.Options)) (*s3.GetBucketLocationOutput, error) { 247 | m.ctrl.T.Helper() 248 | varargs := []interface{}{ctx, params} 249 | for _, a := range optFns { 250 | varargs = append(varargs, a) 251 | } 252 | ret := m.ctrl.Call(m, "GetBucketLocation", varargs...) 253 | ret0, _ := ret[0].(*s3.GetBucketLocationOutput) 254 | ret1, _ := ret[1].(error) 255 | return ret0, ret1 256 | } 257 | 258 | // GetBucketLocation indicates an expected call of GetBucketLocation. 259 | func (mr *MockS3APIMockRecorder) GetBucketLocation(ctx, params interface{}, optFns ...interface{}) *gomock.Call { 260 | mr.mock.ctrl.T.Helper() 261 | varargs := append([]interface{}{ctx, params}, optFns...) 262 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBucketLocation", reflect.TypeOf((*MockS3API)(nil).GetBucketLocation), varargs...) 263 | } 264 | 265 | // GetObject mocks base method. 266 | func (m *MockS3API) GetObject(arg0 context.Context, arg1 *s3.GetObjectInput, arg2 ...func(*s3.Options)) (*s3.GetObjectOutput, error) { 267 | m.ctrl.T.Helper() 268 | varargs := []interface{}{arg0, arg1} 269 | for _, a := range arg2 { 270 | varargs = append(varargs, a) 271 | } 272 | ret := m.ctrl.Call(m, "GetObject", varargs...) 273 | ret0, _ := ret[0].(*s3.GetObjectOutput) 274 | ret1, _ := ret[1].(error) 275 | return ret0, ret1 276 | } 277 | 278 | // GetObject indicates an expected call of GetObject. 279 | func (mr *MockS3APIMockRecorder) GetObject(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { 280 | mr.mock.ctrl.T.Helper() 281 | varargs := append([]interface{}{arg0, arg1}, arg2...) 282 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObject", reflect.TypeOf((*MockS3API)(nil).GetObject), varargs...) 283 | } 284 | 285 | // HeadObject mocks base method. 286 | func (m *MockS3API) HeadObject(arg0 context.Context, arg1 *s3.HeadObjectInput, arg2 ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { 287 | m.ctrl.T.Helper() 288 | varargs := []interface{}{arg0, arg1} 289 | for _, a := range arg2 { 290 | varargs = append(varargs, a) 291 | } 292 | ret := m.ctrl.Call(m, "HeadObject", varargs...) 293 | ret0, _ := ret[0].(*s3.HeadObjectOutput) 294 | ret1, _ := ret[1].(error) 295 | return ret0, ret1 296 | } 297 | 298 | // HeadObject indicates an expected call of HeadObject. 299 | func (mr *MockS3APIMockRecorder) HeadObject(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { 300 | mr.mock.ctrl.T.Helper() 301 | varargs := append([]interface{}{arg0, arg1}, arg2...) 302 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HeadObject", reflect.TypeOf((*MockS3API)(nil).HeadObject), varargs...) 303 | } 304 | 305 | // ListObjectsV2 mocks base method. 306 | func (m *MockS3API) ListObjectsV2(arg0 context.Context, arg1 *s3.ListObjectsV2Input, arg2 ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) { 307 | m.ctrl.T.Helper() 308 | varargs := []interface{}{arg0, arg1} 309 | for _, a := range arg2 { 310 | varargs = append(varargs, a) 311 | } 312 | ret := m.ctrl.Call(m, "ListObjectsV2", varargs...) 313 | ret0, _ := ret[0].(*s3.ListObjectsV2Output) 314 | ret1, _ := ret[1].(error) 315 | return ret0, ret1 316 | } 317 | 318 | // ListObjectsV2 indicates an expected call of ListObjectsV2. 319 | func (mr *MockS3APIMockRecorder) ListObjectsV2(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { 320 | mr.mock.ctrl.T.Helper() 321 | varargs := append([]interface{}{arg0, arg1}, arg2...) 322 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListObjectsV2", reflect.TypeOf((*MockS3API)(nil).ListObjectsV2), varargs...) 323 | } 324 | 325 | // PutObject mocks base method. 326 | func (m *MockS3API) PutObject(arg0 context.Context, arg1 *s3.PutObjectInput, arg2 ...func(*s3.Options)) (*s3.PutObjectOutput, error) { 327 | m.ctrl.T.Helper() 328 | varargs := []interface{}{arg0, arg1} 329 | for _, a := range arg2 { 330 | varargs = append(varargs, a) 331 | } 332 | ret := m.ctrl.Call(m, "PutObject", varargs...) 333 | ret0, _ := ret[0].(*s3.PutObjectOutput) 334 | ret1, _ := ret[1].(error) 335 | return ret0, ret1 336 | } 337 | 338 | // PutObject indicates an expected call of PutObject. 339 | func (mr *MockS3APIMockRecorder) PutObject(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { 340 | mr.mock.ctrl.T.Helper() 341 | varargs := append([]interface{}{arg0, arg1}, arg2...) 342 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutObject", reflect.TypeOf((*MockS3API)(nil).PutObject), varargs...) 343 | } 344 | 345 | // UploadPart mocks base method. 346 | func (m *MockS3API) UploadPart(arg0 context.Context, arg1 *s3.UploadPartInput, arg2 ...func(*s3.Options)) (*s3.UploadPartOutput, error) { 347 | m.ctrl.T.Helper() 348 | varargs := []interface{}{arg0, arg1} 349 | for _, a := range arg2 { 350 | varargs = append(varargs, a) 351 | } 352 | ret := m.ctrl.Call(m, "UploadPart", varargs...) 353 | ret0, _ := ret[0].(*s3.UploadPartOutput) 354 | ret1, _ := ret[1].(error) 355 | return ret0, ret1 356 | } 357 | 358 | // UploadPart indicates an expected call of UploadPart. 359 | func (mr *MockS3APIMockRecorder) UploadPart(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { 360 | mr.mock.ctrl.T.Helper() 361 | varargs := append([]interface{}{arg0, arg1}, arg2...) 362 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UploadPart", reflect.TypeOf((*MockS3API)(nil).UploadPart), varargs...) 363 | } 364 | 365 | // Mocks3Handler is a mock of s3Handler interface. 366 | type Mocks3Handler struct { 367 | ctrl *gomock.Controller 368 | recorder *Mocks3HandlerMockRecorder 369 | } 370 | 371 | // Mocks3HandlerMockRecorder is the mock recorder for Mocks3Handler. 372 | type Mocks3HandlerMockRecorder struct { 373 | mock *Mocks3Handler 374 | } 375 | 376 | // NewMocks3Handler creates a new mock instance. 377 | func NewMocks3Handler(ctrl *gomock.Controller) *Mocks3Handler { 378 | mock := &Mocks3Handler{ctrl: ctrl} 379 | mock.recorder = &Mocks3HandlerMockRecorder{mock} 380 | return mock 381 | } 382 | 383 | // EXPECT returns an object that allows the caller to indicate expected use. 384 | func (m *Mocks3Handler) EXPECT() *Mocks3HandlerMockRecorder { 385 | return m.recorder 386 | } 387 | 388 | // DeleteObject mocks base method. 389 | func (m *Mocks3Handler) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { 390 | m.ctrl.T.Helper() 391 | ret := m.ctrl.Call(m, "DeleteObject", ctx, input) 392 | ret0, _ := ret[0].(*s3.DeleteObjectOutput) 393 | ret1, _ := ret[1].(error) 394 | return ret0, ret1 395 | } 396 | 397 | // DeleteObject indicates an expected call of DeleteObject. 398 | func (mr *Mocks3HandlerMockRecorder) DeleteObject(ctx, input interface{}) *gomock.Call { 399 | mr.mock.ctrl.T.Helper() 400 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteObject", reflect.TypeOf((*Mocks3Handler)(nil).DeleteObject), ctx, input) 401 | } 402 | 403 | // GeneratePresignedURL mocks base method. 404 | func (m *Mocks3Handler) GeneratePresignedURL(ctx context.Context, bucket, key string, expiration time.Duration) (string, error) { 405 | m.ctrl.T.Helper() 406 | ret := m.ctrl.Call(m, "GeneratePresignedURL", ctx, bucket, key, expiration) 407 | ret0, _ := ret[0].(string) 408 | ret1, _ := ret[1].(error) 409 | return ret0, ret1 410 | } 411 | 412 | // GeneratePresignedURL indicates an expected call of GeneratePresignedURL. 413 | func (mr *Mocks3HandlerMockRecorder) GeneratePresignedURL(ctx, bucket, key, expiration interface{}) *gomock.Call { 414 | mr.mock.ctrl.T.Helper() 415 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GeneratePresignedURL", reflect.TypeOf((*Mocks3Handler)(nil).GeneratePresignedURL), ctx, bucket, key, expiration) 416 | } 417 | 418 | // GetBucketLocation mocks base method. 419 | func (m *Mocks3Handler) GetBucketLocation(ctx context.Context, input *s3.GetBucketLocationInput) (*s3.GetBucketLocationOutput, error) { 420 | m.ctrl.T.Helper() 421 | ret := m.ctrl.Call(m, "GetBucketLocation", ctx, input) 422 | ret0, _ := ret[0].(*s3.GetBucketLocationOutput) 423 | ret1, _ := ret[1].(error) 424 | return ret0, ret1 425 | } 426 | 427 | // GetBucketLocation indicates an expected call of GetBucketLocation. 428 | func (mr *Mocks3HandlerMockRecorder) GetBucketLocation(ctx, input interface{}) *gomock.Call { 429 | mr.mock.ctrl.T.Helper() 430 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBucketLocation", reflect.TypeOf((*Mocks3Handler)(nil).GetBucketLocation), ctx, input) 431 | } 432 | 433 | // GetObject mocks base method. 434 | func (m *Mocks3Handler) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { 435 | m.ctrl.T.Helper() 436 | ret := m.ctrl.Call(m, "GetObject", ctx, input) 437 | ret0, _ := ret[0].(*s3.GetObjectOutput) 438 | ret1, _ := ret[1].(error) 439 | return ret0, ret1 440 | } 441 | 442 | // GetObject indicates an expected call of GetObject. 443 | func (mr *Mocks3HandlerMockRecorder) GetObject(ctx, input interface{}) *gomock.Call { 444 | mr.mock.ctrl.T.Helper() 445 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetObject", reflect.TypeOf((*Mocks3Handler)(nil).GetObject), ctx, input) 446 | } 447 | 448 | // HeadObject mocks base method. 449 | func (m *Mocks3Handler) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { 450 | m.ctrl.T.Helper() 451 | ret := m.ctrl.Call(m, "HeadObject", ctx, input) 452 | ret0, _ := ret[0].(*s3.HeadObjectOutput) 453 | ret1, _ := ret[1].(error) 454 | return ret0, ret1 455 | } 456 | 457 | // HeadObject indicates an expected call of HeadObject. 458 | func (mr *Mocks3HandlerMockRecorder) HeadObject(ctx, input interface{}) *gomock.Call { 459 | mr.mock.ctrl.T.Helper() 460 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HeadObject", reflect.TypeOf((*Mocks3Handler)(nil).HeadObject), ctx, input) 461 | } 462 | 463 | // ListAllObjects mocks base method. 464 | func (m *Mocks3Handler) ListAllObjects(ctx context.Context, input *s3.ListObjectsV2Input) ([]*s3.ListObjectsV2Output, error) { 465 | m.ctrl.T.Helper() 466 | ret := m.ctrl.Call(m, "ListAllObjects", ctx, input) 467 | ret0, _ := ret[0].([]*s3.ListObjectsV2Output) 468 | ret1, _ := ret[1].(error) 469 | return ret0, ret1 470 | } 471 | 472 | // ListAllObjects indicates an expected call of ListAllObjects. 473 | func (mr *Mocks3HandlerMockRecorder) ListAllObjects(ctx, input interface{}) *gomock.Call { 474 | mr.mock.ctrl.T.Helper() 475 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListAllObjects", reflect.TypeOf((*Mocks3Handler)(nil).ListAllObjects), ctx, input) 476 | } 477 | 478 | // ListObjects mocks base method. 479 | func (m *Mocks3Handler) ListObjects(ctx context.Context, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { 480 | m.ctrl.T.Helper() 481 | ret := m.ctrl.Call(m, "ListObjects", ctx, input) 482 | ret0, _ := ret[0].(*s3.ListObjectsV2Output) 483 | ret1, _ := ret[1].(error) 484 | return ret0, ret1 485 | } 486 | 487 | // ListObjects indicates an expected call of ListObjects. 488 | func (mr *Mocks3HandlerMockRecorder) ListObjects(ctx, input interface{}) *gomock.Call { 489 | mr.mock.ctrl.T.Helper() 490 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListObjects", reflect.TypeOf((*Mocks3Handler)(nil).ListObjects), ctx, input) 491 | } 492 | 493 | // PutObject mocks base method. 494 | func (m *Mocks3Handler) PutObject(ctx context.Context, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) { 495 | m.ctrl.T.Helper() 496 | ret := m.ctrl.Call(m, "PutObject", ctx, input) 497 | ret0, _ := ret[0].(*s3.PutObjectOutput) 498 | ret1, _ := ret[1].(error) 499 | return ret0, ret1 500 | } 501 | 502 | // PutObject indicates an expected call of PutObject. 503 | func (mr *Mocks3HandlerMockRecorder) PutObject(ctx, input interface{}) *gomock.Call { 504 | mr.mock.ctrl.T.Helper() 505 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PutObject", reflect.TypeOf((*Mocks3Handler)(nil).PutObject), ctx, input) 506 | } 507 | --------------------------------------------------------------------------------