├── .all-contributorsrc
├── .github
    └── workflows
    │   ├── golangci-lint.yml
    │   └── test.yml
├── .gitignore
├── Dockerfile
├── LICENSE
├── Makefile
├── README.md
├── Version
├── browse.go
├── browse_test.go
├── caddyfile.go
├── caddyfile_test.go
├── errors.go
├── example
    ├── Caddyfile
    ├── LOCALSTACK_EXAMPLE.md
    ├── awslocal
    │   └── populate.sh
    └── docker-compose.yml
├── go.mod
├── go.sum
├── s3proxy.go
├── s3proxy_test.go
├── tag.sh
└── testdata
    ├── _404.txt
    ├── default_error_page.txt
    ├── inner
        └── index.html
    ├── test.json
    └── to-delete.json


/.all-contributorsrc:
--------------------------------------------------------------------------------
 1 | {
 2 |   "projectName": "caddy-s3-proxy",
 3 |   "repoType": "github",
 4 |   "files": [
 5 |     "README.md"
 6 |   ],
 7 |   "commit": true,
 8 |   "imageSize": 100,
 9 |   "contributorsPerLine": 7,
10 |   "contributorsSortAlphabetically": false,
11 |   "badgeTemplate": "[![All Contributors](https://img.shields.io/badge/all_contributors-<%= contributors.length %>-orange.svg?style=flat-square)](#contributors)",
12 |   "contributorTemplate": "<a href=\"<%= contributor.profile %>\"><img src=\"<%= contributor.avatar_url %>\" width=\"<%= options.imageSize %>px;\" alt=\"\"/><br /><sub><b><%= contributor.name %></b></sub></a>",
13 |   "skipCi": true,
14 |   "contributors": [
15 |     {
16 |       "login": "rayjlinden",
17 |       "name": "rayjlinden",
18 |       "avatar_url": "https://avatars0.githubusercontent.com/u/42587610?v=4",
19 |       "profile": "https://github.com/rayjlinden",
20 |       "contributions": [
21 |         "code",
22 |         "doc"
23 |       ]
24 |     },
25 |     {
26 |       "login": "gilbsgilbs",
27 |       "name": "Gilbert Gilb's",
28 |       "avatar_url": "https://avatars2.githubusercontent.com/u/3407667?v=4",
29 |       "profile": "https://github.com/gilbsgilbs",
30 |       "contributions": [
31 |         "code",
32 |         "doc",
33 |         "test"
34 |       ]
35 |     },
36 |     {
37 |       "login": "christoph-kluge",
38 |       "name": "Christoph Kluge",
39 |       "avatar_url": "https://avatars3.githubusercontent.com/u/1446269?v=4",
40 |       "profile": "https://github.com/christoph-kluge",
41 |       "contributions": [
42 |         "bug",
43 |         "code"
44 |       ]
45 |     }
46 |   ],
47 |   "projectOwner": "lindenlab",
48 |   "repoHost": "https://github.com"
49 | }
50 | 


--------------------------------------------------------------------------------
/.github/workflows/golangci-lint.yml:
--------------------------------------------------------------------------------
 1 | name: golangci-lint
 2 | on:
 3 |   push:
 4 |     tags:
 5 |       - v*
 6 |     branches:
 7 |       - master
 8 |   pull_request:
 9 | jobs:
10 |   golangci:
11 |     name: lint
12 |     runs-on: ubuntu-latest
13 |     steps:
14 |       - uses: actions/setup-go@v4
15 |         with:
16 |           go-version: '1.17'
17 |           cache: false
18 |       - uses: actions/checkout@v3
19 |       - name: golangci-lint
20 |         uses: golangci/golangci-lint-action@v3
21 |         with:
22 |           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
23 |           version: v1.29
24 | 
25 |           # golangci-lint seems to be randomly slow. Increasing the timeout circumvents this issue.
26 |           args: --timeout 5m
27 | 
28 |           # See here for more info on this action:
29 |           # https://github.com/golangci/golangci-lint-action
30 | 


--------------------------------------------------------------------------------
/.github/workflows/test.yml:
--------------------------------------------------------------------------------
 1 | on: [push, pull_request]
 2 | 
 3 | name: Test
 4 | jobs:
 5 |   test:
 6 |     strategy:
 7 |       matrix:
 8 |         go-version: [1.14.x, 1.15.x]
 9 |     runs-on: ubuntu-latest
10 |     services:
11 |       localstack:
12 |         image: localstack/localstack:latest
13 |         ports:
14 |           - 4566:4566
15 |         env:
16 |           SERVICES: s3
17 |     env:
18 |       AWS_SECRET_ACCESS_KEY: dummy
19 |       AWS_ACCESS_KEY_ID: dummy
20 |       AWS_REGION: dummy
21 |       AWS_ENDPOINT: http://localhost:4566
22 |     steps:
23 |       - name: Install Go
24 |         uses: actions/setup-go@v2
25 |         with:
26 |           go-version: ${{ matrix.go-version }}
27 |       - name: Checkout code
28 |         uses: actions/checkout@v2
29 |       - name: Test
30 |         run: go test -v -cover ./...
31 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | caddy
2 | cover.*
3 | 


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM debian:buster-slim
2 | 
3 | COPY caddy /usr/bin/caddy
4 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "[]"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright [yyyy] [name of copyright owner]
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | export GO111MODULE=on
 2 | VERSION := $(shell cat Version)
 3 | COVER_TARGET ?= 30
 4 | 
 5 | # Set for running tests against localstack
 6 | export AWS_SECRET_ACCESS_KEY=dummy
 7 | export AWS_ACCESS_KEY_ID=dummy
 8 | export AWS_REGION=dummy
 9 | export AWS_ENDPOINT=http://localhost:4566
10 | 
11 | .PHONY: build
12 | build: caddy
13 | 
14 | caddy: *.go go.mod Makefile
15 | 	# go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest  -- install xcaddy if you don't have it
16 | 	xcaddy build --output caddy --with github.com/lindenlab/caddy-s3-proxy=${CURDIR}
17 | 
18 | .PHONY: docker
19 | docker: caddy  ## build a docker image for caddy with the s3proxy
20 | 	@docker build -t caddy .
21 | 
22 | .PHONY: test
23 | test:  ## Run go test on source base
24 | 	@go test --race
25 | 
26 | .PHONY: cover
27 | cover:  ## Generate test coverage results
28 | 	@go test -gcflags=-l --covermode=count -coverprofile cover.profile ${PKGS}
29 | 	@go tool cover -html cover.profile -o cover.html
30 | 	@go tool cover -func cover.profile -o cover.func
31 | 	@tail -n 1 cover.func | awk '{if (int($$3) >= ${COVER_TARGET}) {print "Coverage good: " $$3} else {print "Coverage is less than ${COVER_TARGET}%: " $$3; exit 1}}'
32 | 
33 | .PHONY: lint
34 | lint:  ## Run golint on source base
35 | 	@golangci-lint run ./...
36 | 
37 | .PHONY: localstack
38 | localstack:  ## Launch localstack to run tests against
39 | 	@docker-compose -f example/docker-compose.yml up -d localstack
40 | 
41 | .PHONY: example
42 | example: docker  ## Run docker-compose up in the example directory
43 | 	@docker-compose -f example/docker-compose.yml up
44 | 
45 | .DEFAULT_GOAL := help
46 | .PHONY: help
47 | help:   ## Display this help message
48 | 	@grep -E '^[ a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
49 | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-15s\033[0m %s\n", $$1, $$2}'
50 | 
51 | .PHONY: clean
52 | clean:  ## Delete any generated files
53 | 	@rm -f caddy
54 | 
55 | .PHONY: version
56 | version:  ## Show the version the Makefile will build
57 | 	@echo ${VERSION}
58 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | [![golangci-lint Actions Status](https://github.com/lindenlab/caddy-s3-proxy/workflows/golangci-lint/badge.svg)](https://github.com/lindenlab/caddy-s3-proxy/actions)
  2 | [![Test Actions Status](https://github.com/lindenlab/caddy-s3-proxy/workflows/Test/badge.svg)](https://github.com/lindenlab/caddy-s3-proxy/actions)
  3 | [![All Contributors](https://img.shields.io/badge/all_contributors-2-orange.svg?style=flat-square)](#contributors)
  4 | 
  5 | # caddy-s3-proxy
  6 | 
  7 | caddy-s3-proxy allows you to proxy requests directly from S3.
  8 | 
  9 | S3 does have the website option, in which case, a normal reverse proxy could be used to display S3 data.
 10 | However, it is sometimes inconvient to do that.  This module lets you access S3 data even if website access
 11 | is not configured on your bucket.
 12 | 
 13 | ## Making a version of caddy with this plugin
 14 | 
 15 | With caddy 2 you can use [xcaddy](https://github.com/caddyserver/xcaddy) to build a version of caddy
 16 | with this plugin installed.  To install xcaddy do:
 17 | ```
 18 | go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest
 19 | ```
 20 | 
 21 | This repo has a Makefile to make it easier to build a new version of caddy with this plugin.  Just type:
 22 | ```
 23 | make build
 24 | ```
 25 | 
 26 | You can run ```make docker``` do build a local image you can test with.
 27 | 
 28 | ## Configuration
 29 | The Caddyfile directive would look something like this:
 30 | ```
 31 | 	s3proxy [<matcher>] {
 32 | 		bucket <bucket_name>
 33 | 		region <region_name>
 34 |                 profile <aws profile>
 35 | 		index  <list of index file names>
 36 | 		endpoint <alternative S3 endpoint>
 37 | 		root   <key prefix>
 38 | 		enable_put
 39 | 		enable_delete
 40 |  		force_path_style
 41 | 		errors <http status> <S3 key to a custom error page for this http status>
 42 | 		errors <S3 key to a default error page>
 43 | 		browse [<path to template>]
 44 | 	}
 45 | ```
 46 | 
 47 | |  option   |  type  |  required | default | help |
 48 | |-----------|:------:|-----------|---------|------|
 49 | | bucket              | string   | yes |                          | S3 bucket name |
 50 | | region              | string   | yes-ish  |  env AWS_REGION          | S3 region - if not give in the Caddyfile then AWS_REGION env var must be set.|
 51 | | profile             | string   | no  |  empty string            | AWS profile if using shared credentials files. |
 52 | | endpoint            | string   | no  |  aws default             | S3 hostname |
 53 | | index               | string[] | no  |  [index.html, index.txt] | Index files to look up for dir path |
 54 | | root                | string   | no  |    | Set a "prefix" to be added to key |
 55 | | enable_put          | bool     | no  | false   | Allow PUT method to be sent through proxy |
 56 | | enable_delete       | bool     | no  | false   | Allow DELETE method to be sent through proxy |
 57 | | force_path_style    | bool     | no  | false   | Set this to `true` to force S3 request to use path-style addressing |
 58 | | use_accelerate      | bool     | no  | false   | Set this to `true` to enable S3 Accelerate feature |
 59 | | errors              | [int, ] string | no |  | Custom error page or use "pass_through" to write nothing for errors. |
 60 | | browse              | [string] | no |  | Turns on a directory view for partial keys, an optional path to a template can be given |
 61 | 
 62 | ## Credentials
 63 | 
 64 | This module uses the default providor chain to get credentials for access to S3.  This provides several more
 65 | secure options to provide credentials for accessing S3 without putting the credentials in the Caddyfile.
 66 | The methods include (and are looked for in this order):
 67 | 
 68 | 1) Environment variables.  I.e. AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
 69 | 
 70 | 2) Shared credentials file.  (Located at ~/.aws/credentials)
 71 |    (You may pass the optional profile directive to select specific credentials.)
 72 | 
 73 | 3) If your application uses an ECS task definition or RunTask API operation, IAM role for tasks.
 74 | 
 75 | 4) If your application is running on an Amazon EC2 instance, IAM role for Amazon EC2.
 76 | 
 77 | For much more detail on the various options for setting AWS credentials see here:
 78 | https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html
 79 | 
 80 | ## Works with localstack!
 81 | 
 82 | The s3 proxy works great with localstack for local testing.  Just set the endpoint directive to your localstack
 83 | instance.  You will also want to set the `force_path_style` directive as well since localstack currently does not
 84 | support virtual style addressing.  In fact, all of our examples use localstack - check out the examples directory.
 85 | 
 86 | ## Handling errors
 87 | 
 88 | When accessing S3 you may get errors like keyNotFound, bucket does not exist, or ACL permissions problems.  By default
 89 | this proxy will map those errors to an http error - like 404, 403 or 500.
 90 | 
 91 | However, with the `errors` directive you have a couple of more options.  You can specify a S3 key that may contain HTML
 92 | to display rather than just returning an error code.  This can be done for a specific error or all errors.  For example,
 93 | ```
 94 | errors 403 /key/path/to/permissionerr.html
 95 | errors /key/path/to/defaulterr.html
 96 | ```
 97 | This will display the page permissionerr.html for any 403 errors and defaulterr.html for all other errors.
 98 | 
 99 | There is a special option to "pass through" on an error and let the next Caddy handler deal with the request.  For example,
100 | ```
101 | errors 404 pass_through
102 | errors /key/path/to/defaulterr.html
103 | ```
104 | 
105 | Will pass 404 errors onto the next handler.  All other errors will show the page defaulterr.html.
106 | 
107 | Note: The `errors` direction only applies to GET method requests.  PUT and DELETE errors just return the code.
108 | 
109 | ## Examples you can play with
110 | 
111 | In the examples directory is an example of using the s3proxy with localstack.
112 | Localstack contains a working version of S3 you can use for local development.
113 | 
114 | Check out the examples [here](example/LOCALSTACK_EXAMPLE.md).
115 | You can also just run ```make example``` to build a docker image with the plugin and launch the compose example.
116 | 
117 | # Contributors
118 | 
119 | A big thank you to folks who have contributed to this project!
120 | 
121 | <!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
122 | <!-- prettier-ignore-start -->
123 | <!-- markdownlint-disable -->
124 | <table>
125 |   <tr>
126 |     <td align="center"><a href="https://github.com/rayjlinden"><img src="https://avatars0.githubusercontent.com/u/42587610?v=4" width="100px;" alt=""/><br /><sub><b>rayjlinden</b></sub></a></td>
127 |     <td align="center"><a href="https://github.com/gilbsgilbs"><img src="https://avatars2.githubusercontent.com/u/3407667?v=4" width="100px;" alt=""/><br /><sub><b>Gilbert Gilb's</b></sub></a></td>
128 |     <td align="center"><a href="https://github.com/christoph-kluge"><img src="https://avatars3.githubusercontent.com/u/1446269?v=4" width="100px;" alt=""/><br /><sub><b>Christoph Kluge</b></sub></a></td>
129 |   </tr>
130 | </table>
131 | 
132 | <!-- markdownlint-enable -->
133 | <!-- prettier-ignore-end -->
134 | <!-- ALL-CONTRIBUTORS-LIST:END -->
135 | 
136 | 


--------------------------------------------------------------------------------
/Version:
--------------------------------------------------------------------------------
1 | 0.5.8
2 | 


--------------------------------------------------------------------------------
/browse.go:
--------------------------------------------------------------------------------
  1 | package caddys3proxy
  2 | 
  3 | import (
  4 | 	"bytes"
  5 | 	"encoding/json"
  6 | 	"html/template"
  7 | 	"net/http"
  8 | 	"net/url"
  9 | 	"path"
 10 | 	"strconv"
 11 | 	"strings"
 12 | 	"sync"
 13 | 
 14 | 	"github.com/aws/aws-sdk-go/aws"
 15 | 	"github.com/aws/aws-sdk-go/service/s3"
 16 | 	"github.com/dustin/go-humanize"
 17 | )
 18 | 
 19 | var bufPool = sync.Pool{
 20 | 	New: func() interface{} {
 21 | 		return new(bytes.Buffer)
 22 | 	},
 23 | }
 24 | 
 25 | type PageObj struct {
 26 | 	Count    int64  `json:"count"`
 27 | 	Items    []Item `json:"items"`
 28 | 	MoreLink string `json:"more"`
 29 | }
 30 | 
 31 | type Item struct {
 32 | 	Name         string `json:"name"`
 33 | 	IsDir        bool   `json:"is_dir"`
 34 | 	Key          string `json:"key"`
 35 | 	Url          string `json:"url"`
 36 | 	Size         string `json:"size"`
 37 | 	LastModified string `json:"last_modified"`
 38 | }
 39 | 
 40 | // GenerateJson generates JSON output for the PageObj
 41 | func (po PageObj) GenerateJson(w http.ResponseWriter) error {
 42 | 	buf := bufPool.Get().(*bytes.Buffer)
 43 | 	buf.Reset()
 44 | 	defer bufPool.Put(buf)
 45 | 
 46 | 	err := json.NewEncoder(buf).Encode(po)
 47 | 	if err != nil {
 48 | 		return err
 49 | 	}
 50 | 
 51 | 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 52 | 	_, err = buf.WriteTo(w)
 53 | 	return err
 54 | }
 55 | 
 56 | func (p S3Proxy) ConstructListObjInput(r *http.Request, key string) s3.ListObjectsV2Input {
 57 | 	// We need to strip the first '/' from the key to make it a valid prefix
 58 | 	prefix := strings.TrimPrefix(key, "/")
 59 | 
 60 | 	input := s3.ListObjectsV2Input{
 61 | 		Bucket:    aws.String(p.Bucket),
 62 | 		Prefix:    aws.String(prefix),
 63 | 		Delimiter: aws.String("/"),
 64 | 	}
 65 | 
 66 | 	nextToken := r.URL.Query().Get("next")
 67 | 	if nextToken != "" {
 68 | 		input.ContinuationToken = aws.String(nextToken)
 69 | 	}
 70 | 
 71 | 	maxPerPage := r.URL.Query().Get("max")
 72 | 	if maxPerPage != "" {
 73 | 		maxKeys, err := strconv.ParseInt(maxPerPage, 10, 64)
 74 | 		if err == nil && maxKeys > 0 && maxKeys <= 1000 {
 75 | 			input.MaxKeys = aws.Int64(maxKeys)
 76 | 		}
 77 | 	}
 78 | 
 79 | 	return input
 80 | }
 81 | 
 82 | // GenerateHtml generates html output for the PageObj
 83 | func (po PageObj) GenerateHtml(w http.ResponseWriter, template *template.Template) error {
 84 | 	buf := bufPool.Get().(*bytes.Buffer)
 85 | 	buf.Reset()
 86 | 	defer bufPool.Put(buf)
 87 | 
 88 | 	err := template.Execute(buf, po)
 89 | 	if err != nil {
 90 | 		return err
 91 | 	}
 92 | 
 93 | 	w.Header().Set("Content-Type", "text/html; charset=utf-8")
 94 | 	_, err = buf.WriteTo(w)
 95 | 	return err
 96 | }
 97 | 
 98 | func (p S3Proxy) MakePageObj(result *s3.ListObjectsV2Output) PageObj {
 99 | 	po := PageObj{}
100 | 	po.Count = *result.KeyCount
101 | 	if result.NextContinuationToken != nil {
102 | 		var nextUrl url.URL
103 | 		queryItems := nextUrl.Query()
104 | 
105 | 		queryItems.Add("next", *result.NextContinuationToken)
106 | 		if result.MaxKeys != nil {
107 | 			queryItems.Add("max", strconv.FormatInt(*result.MaxKeys, 10))
108 | 		}
109 | 		nextUrl.RawQuery = queryItems.Encode()
110 | 		po.MoreLink = nextUrl.String()
111 | 	}
112 | 
113 | 	for _, dir := range result.CommonPrefixes {
114 | 		name := path.Base(*dir.Prefix)
115 | 		dirPath := "./" + name + "/"
116 | 		po.Items = append(po.Items, Item{
117 | 			Url:   dirPath,
118 | 			Name:  name,
119 | 			IsDir: true,
120 | 		})
121 | 	}
122 | 	for _, obj := range result.Contents {
123 | 		name := path.Base(*obj.Key)
124 | 		itemPath := "./" + name
125 | 		size := humanize.Bytes(uint64(*obj.Size))
126 | 		timeAgo := humanize.Time(*obj.LastModified)
127 | 		po.Items = append(po.Items, Item{
128 | 			Name:         name,
129 | 			Key:          *obj.Key,
130 | 			Url:          itemPath,
131 | 			Size:         size,
132 | 			LastModified: timeAgo,
133 | 			IsDir:        false,
134 | 		})
135 | 	}
136 | 
137 | 	return po
138 | }
139 | 
140 | // This is a lame ass default template - needs to get better
141 | const defaultBrowseTemplate = `<!DOCTYPE html>
142 | <html>
143 |         <body>
144 |                 <ul>
145 |                 {{- range .PageObj }}
146 |                 <li>
147 |                 {{- if .IsDir}}
148 |                 <a href="{{html .Url}}">{{html .Name}}</a>
149 |                 {{- else}}
150 |                 <a href="{{html .Url}}">{{html .Name}}</a> Size: {{html .Size}} Last Modified: {{html .LastModified}}
151 |                 {{- end}}
152 |                 </li>
153 |                 {{- end }}
154 |                 </ul>
155 | 		<p>number of items: {{ .Count }}</p>
156 | 		{{- if .MoreLink }}
157 | 		<a href="{{ html .MoreLink }}">more...</a>
158 | 		{{- end }}
159 |         </body>
160 | </html>`
161 | 


--------------------------------------------------------------------------------
/browse_test.go:
--------------------------------------------------------------------------------
  1 | package caddys3proxy
  2 | 
  3 | import (
  4 | 	"net/http"
  5 | 	"net/url"
  6 | 	"reflect"
  7 | 	"testing"
  8 | 	"time"
  9 | 
 10 | 	"github.com/aws/aws-sdk-go/aws"
 11 | 	"github.com/aws/aws-sdk-go/service/s3"
 12 | )
 13 | 
 14 | func TestConstructListObjInput(t *testing.T) {
 15 | 	type testCase struct {
 16 | 		name        string
 17 | 		key         string
 18 | 		bucket      string
 19 | 		queryString string
 20 | 		expected    s3.ListObjectsV2Input
 21 | 	}
 22 | 
 23 | 	testCases := []testCase{
 24 | 		testCase{
 25 | 			name:   "no query options",
 26 | 			bucket: "myBucket",
 27 | 			key:    "/mypath/",
 28 | 			expected: s3.ListObjectsV2Input{
 29 | 				Bucket:    aws.String("myBucket"),
 30 | 				Delimiter: aws.String("/"),
 31 | 				Prefix:    aws.String("mypath/"),
 32 | 			},
 33 | 		},
 34 | 		testCase{
 35 | 			name:        "max option",
 36 | 			bucket:      "myBucket",
 37 | 			key:         "/mypath/",
 38 | 			queryString: "?max=20",
 39 | 			expected: s3.ListObjectsV2Input{
 40 | 				Bucket:    aws.String("myBucket"),
 41 | 				Delimiter: aws.String("/"),
 42 | 				Prefix:    aws.String("mypath/"),
 43 | 				MaxKeys:   aws.Int64(20),
 44 | 			},
 45 | 		},
 46 | 		testCase{
 47 | 			name:        "max with next",
 48 | 			bucket:      "myBucket",
 49 | 			key:         "/mypath/",
 50 | 			queryString: "?max=20&next=FOO",
 51 | 			expected: s3.ListObjectsV2Input{
 52 | 				Bucket:            aws.String("myBucket"),
 53 | 				Delimiter:         aws.String("/"),
 54 | 				Prefix:            aws.String("mypath/"),
 55 | 				MaxKeys:           aws.Int64(20),
 56 | 				ContinuationToken: aws.String("FOO"),
 57 | 			},
 58 | 		},
 59 | 	}
 60 | 	for _, tc := range testCases {
 61 | 		r := http.Request{}
 62 | 		u, _ := url.Parse(tc.queryString)
 63 | 		r.URL = u
 64 | 		p := S3Proxy{
 65 | 			Bucket: tc.bucket,
 66 | 		}
 67 | 		result := p.ConstructListObjInput(&r, tc.key)
 68 | 		if !reflect.DeepEqual(tc.expected, result) {
 69 | 			t.Errorf("Expected obj %v, got %v.", tc.expected, result)
 70 | 		}
 71 | 	}
 72 | }
 73 | 
 74 | func TestMakePageObj(t *testing.T) {
 75 | 	p := S3Proxy{}
 76 | 	listOutput := s3.ListObjectsV2Output{
 77 | 		KeyCount:              aws.Int64(20),
 78 | 		NextContinuationToken: aws.String("next_token"),
 79 | 		MaxKeys:               aws.Int64(20),
 80 | 		CommonPrefixes: []*s3.CommonPrefix{
 81 | 			&s3.CommonPrefix{
 82 | 				Prefix: aws.String("/mydir"),
 83 | 			},
 84 | 			&s3.CommonPrefix{
 85 | 				Prefix: aws.String("/otherdir"),
 86 | 			},
 87 | 		},
 88 | 		Contents: []*s3.Object{
 89 | 			&s3.Object{
 90 | 				Key:          aws.String("/path/to/myobj"),
 91 | 				Size:         aws.Int64(1024),
 92 | 				LastModified: aws.Time(time.Date(1845, time.November, 10, 23, 0, 0, 0, time.UTC)),
 93 | 			},
 94 | 		},
 95 | 	}
 96 | 
 97 | 	result := p.MakePageObj(&listOutput)
 98 | 	expected := PageObj{
 99 | 		Count:    20,
100 | 		MoreLink: "?max=20&next=next_token",
101 | 		Items: []Item{
102 | 			Item{
103 | 				Url:   "./mydir/",
104 | 				IsDir: true,
105 | 				Name:  "mydir",
106 | 			},
107 | 			Item{
108 | 				Url:   "./otherdir/",
109 | 				IsDir: true,
110 | 				Name:  "otherdir",
111 | 			},
112 | 			Item{
113 | 				Url:          "./myobj",
114 | 				Key:          "/path/to/myobj",
115 | 				IsDir:        false,
116 | 				Name:         "myobj",
117 | 				Size:         "1.0 kB",
118 | 				LastModified: "a long while ago",
119 | 			},
120 | 		},
121 | 	}
122 | 
123 | 	if !reflect.DeepEqual(expected, result) {
124 | 		t.Errorf("Expected obj %v, got %v.", expected, result)
125 | 	}
126 | }
127 | 


--------------------------------------------------------------------------------
/caddyfile.go:
--------------------------------------------------------------------------------
  1 | package caddys3proxy
  2 | 
  3 | import (
  4 | 	"strconv"
  5 | 
  6 | 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
  7 | 	"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
  8 | 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
  9 | )
 10 | 
 11 | func init() {
 12 | 	httpcaddyfile.RegisterHandlerDirective("s3proxy", parseCaddyfile)
 13 | }
 14 | 
 15 | // parseCaddyfile parses the s3proxy directive. It enables the proxying
 16 | // requests to S3 and configures it with this syntax:
 17 | //
 18 | //    s3proxy [<matcher>] {
 19 | //        root   <path to prefix S3 key with>
 20 | //        region <aws region>
 21 | //        profile <aws profile>
 22 | //        bucket <s3 bucket name>
 23 | //        index  <files...>
 24 | //        hide   <file patterns...>
 25 | //        endpoint <alternative endpoint>
 26 | //        enable_put
 27 | //        enable_delete
 28 | //        force_path_style
 29 | //        use_accelerate
 30 | //        errors [<http code>] [<s3 key to error page>|pass_through]
 31 | //        browse [<template file>]
 32 | //    }
 33 | //
 34 | func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {
 35 | 	return parseCaddyfileWithDispenser(h.Dispenser)
 36 | }
 37 | 
 38 | func parseCaddyfileWithDispenser(h *caddyfile.Dispenser) (*S3Proxy, error) {
 39 | 	var b S3Proxy
 40 | 
 41 | 	h.NextArg() // skip block beginning: "s3proxy"
 42 | parseLoop:
 43 | 	for h.NextBlock(0) {
 44 | 		switch h.Val() {
 45 | 		case "endpoint":
 46 | 			if !h.AllArgs(&b.Endpoint) {
 47 | 				return nil, h.ArgErr()
 48 | 			}
 49 | 		case "region":
 50 | 			if !h.AllArgs(&b.Region) {
 51 | 				return nil, h.ArgErr()
 52 | 			}
 53 | 		case "profile":
 54 | 			if !h.AllArgs(&b.Profile) {
 55 | 				return nil, h.ArgErr()
 56 | 			}
 57 | 		case "root":
 58 | 			if !h.AllArgs(&b.Root) {
 59 | 				return nil, h.ArgErr()
 60 | 			}
 61 | 		case "hide":
 62 | 			b.Hide = h.RemainingArgs()
 63 | 			if len(b.Hide) == 0 {
 64 | 				return nil, h.ArgErr()
 65 | 			}
 66 | 		case "bucket":
 67 | 			if !h.AllArgs(&b.Bucket) {
 68 | 				return nil, h.ArgErr()
 69 | 			}
 70 | 			if b.Bucket == "" {
 71 | 				break parseLoop
 72 | 			}
 73 | 		case "index":
 74 | 			b.IndexNames = h.RemainingArgs()
 75 | 			if len(b.IndexNames) == 0 {
 76 | 				return nil, h.ArgErr()
 77 | 			}
 78 | 		case "enable_put":
 79 | 			b.EnablePut = true
 80 | 		case "enable_delete":
 81 | 			b.EnableDelete = true
 82 | 		case "force_path_style":
 83 | 			b.S3ForcePathStyle = true
 84 | 		case "use_accelerate":
 85 | 			b.S3UseAccelerate = true
 86 | 		case "browse":
 87 | 			b.EnableBrowse = true
 88 | 			args := h.RemainingArgs()
 89 | 			if len(args) == 1 {
 90 | 				b.BrowseTemplate = args[0]
 91 | 			}
 92 | 			if len(args) > 1 {
 93 | 				return nil, h.ArgErr()
 94 | 			}
 95 | 		case "error_page", "errors":
 96 | 			if b.ErrorPages == nil {
 97 | 				b.ErrorPages = make(map[int]string)
 98 | 			}
 99 | 
100 | 			args := h.RemainingArgs()
101 | 			if len(args) == 1 {
102 | 				b.DefaultErrorPage = args[0]
103 | 			} else if len(args) == 2 {
104 | 				httpStatusStr := args[0]
105 | 				s3KeyOrPassThrough := args[1]
106 | 
107 | 				httpStatus, err := strconv.Atoi(httpStatusStr)
108 | 				if err != nil {
109 | 					return nil, h.Errf("'%s' is not a valid HTTP status code", httpStatusStr)
110 | 				}
111 | 
112 | 				b.ErrorPages[httpStatus] = s3KeyOrPassThrough
113 | 			} else {
114 | 				return nil, h.ArgErr()
115 | 			}
116 | 		default:
117 | 			return nil, h.Errf("%s not a valid s3proxy option", h.Val())
118 | 		}
119 | 	}
120 | 	if b.Bucket == "" {
121 | 		return nil, h.Err("bucket must be set and not empty")
122 | 	}
123 | 
124 | 	return &b, nil
125 | }
126 | 


--------------------------------------------------------------------------------
/caddyfile_test.go:
--------------------------------------------------------------------------------
  1 | package caddys3proxy
  2 | 
  3 | import (
  4 | 	"reflect"
  5 | 	"testing"
  6 | 
  7 | 	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
  8 | )
  9 | 
 10 | type testCase struct {
 11 | 	desc      string
 12 | 	input     string
 13 | 	shouldErr bool
 14 | 	errString string
 15 | 	obj       S3Proxy
 16 | }
 17 | 
 18 | func TestParseCaddyfile(t *testing.T) {
 19 | 	testCases := []testCase{
 20 | 		testCase{
 21 | 			desc: "bad sub directive",
 22 | 			input: `s3proxy {
 23 | 				foo
 24 | 			}`,
 25 | 			shouldErr: true,
 26 | 			errString: "Testfile:2 - Error during parsing: foo not a valid s3proxy option",
 27 | 		},
 28 | 		testCase{
 29 | 			desc: "bucket bad # args",
 30 | 			input: `s3proxy {
 31 | 			bucket
 32 | 			}`,
 33 | 			shouldErr: true,
 34 | 			errString: "Testfile:2 - Error during parsing: Wrong argument count or unexpected line ending after 'bucket'",
 35 | 		},
 36 | 		testCase{
 37 | 			desc: "bucket empty string",
 38 | 			input: `s3proxy {
 39 | 				bucket ""
 40 | 			}`,
 41 | 			shouldErr: true,
 42 | 			errString: "Testfile:2 - Error during parsing: bucket must be set and not empty",
 43 | 		},
 44 | 		testCase{
 45 | 			desc: "bucket missing",
 46 | 			input: `s3proxy {
 47 | 				region foo
 48 | 			}`,
 49 | 			shouldErr: true,
 50 | 			errString: "Testfile:3 - Error during parsing: bucket must be set and not empty",
 51 | 		},
 52 | 		testCase{
 53 | 			desc: "endpoint bad # args",
 54 | 			input: `s3proxy {
 55 | 				endpoint
 56 | 			}`,
 57 | 			shouldErr: true,
 58 | 			errString: "Testfile:2 - Error during parsing: Wrong argument count or unexpected line ending after 'endpoint'",
 59 | 		},
 60 | 		testCase{
 61 | 			desc: "region bad # args",
 62 | 			input: `s3proxy {
 63 | 				region one two
 64 | 			}`,
 65 | 			shouldErr: true,
 66 | 			errString: "Testfile:2 - Error during parsing: Wrong argument count or unexpected line ending after 'one'",
 67 | 		},
 68 | 		testCase{
 69 | 			desc: "root bad # args",
 70 | 			input: `s3proxy {
 71 | 				root one two
 72 | 			}`,
 73 | 			shouldErr: true,
 74 | 			errString: "Testfile:2 - Error during parsing: Wrong argument count or unexpected line ending after 'one'",
 75 | 		},
 76 | 		testCase{
 77 | 			desc: "errors on invalid HTTP status for errors",
 78 | 			input: `s3proxy {
 79 | 				bucket mybucket
 80 | 				errors invalid "path/to/404.html"
 81 | 			}`,
 82 | 			shouldErr: true,
 83 | 			errString: "Testfile:3 - Error during parsing: 'invalid' is not a valid HTTP status code",
 84 | 		},
 85 | 		testCase{
 86 | 			desc: "errors on too many arguments for errors",
 87 | 			input: `s3proxy {
 88 | 				bucket mybucket
 89 | 				errors 403 "path/to/404.html" "what's this?"
 90 | 			}`,
 91 | 			shouldErr: true,
 92 | 			errString: "Testfile:3 - Error during parsing: Wrong argument count or unexpected line ending after 'what's this?'",
 93 | 		},
 94 | 		testCase{
 95 | 			desc: "endpoint gets set",
 96 | 			input: `s3proxy {
 97 | 				bucket mybucket
 98 | 				endpoint myvalue
 99 | 				region myregion
100 | 			}`,
101 | 			shouldErr: false,
102 | 			obj: S3Proxy{
103 | 				Bucket:   "mybucket",
104 | 				Endpoint: "myvalue",
105 | 				Region:   "myregion",
106 | 			},
107 | 		},
108 | 		testCase{
109 | 			desc: "enable pu",
110 | 			input: `s3proxy {
111 | 				bucket mybucket
112 | 				enable_put
113 | 			}`,
114 | 			shouldErr: false,
115 | 			obj: S3Proxy{
116 | 				Bucket:    "mybucket",
117 | 				EnablePut: true,
118 | 			},
119 | 		},
120 | 		testCase{
121 | 			desc: "enable delete",
122 | 			input: `s3proxy {
123 | 				bucket mybucket
124 | 				enable_delete
125 | 			}`,
126 | 			shouldErr: false,
127 | 			obj: S3Proxy{
128 | 				Bucket:       "mybucket",
129 | 				EnableDelete: true,
130 | 			},
131 | 		},
132 | 		testCase{
133 | 			desc: "enable error pages",
134 | 			input: `s3proxy {
135 | 				bucket mybucket
136 | 				errors 404 "path/to/404.html"
137 | 				errors 403 "path/to/403.html"
138 | 				errors "path/to/default_error.html"
139 | 			}`,
140 | 			shouldErr: false,
141 | 			obj: S3Proxy{
142 | 				Bucket: "mybucket",
143 | 				ErrorPages: map[int]string{
144 | 					403: "path/to/403.html",
145 | 					404: "path/to/404.html",
146 | 				},
147 | 				DefaultErrorPage: "path/to/default_error.html",
148 | 			},
149 | 		},
150 | 		testCase{
151 | 			desc: "hide files",
152 | 			input: `s3proxy {
153 | 				bucket mybucket
154 | 				hide foo.txt _*
155 | 			}`,
156 | 			shouldErr: false,
157 | 			obj: S3Proxy{
158 | 				Bucket: "mybucket",
159 | 				Hide:   []string{"foo.txt", "_*"},
160 | 			},
161 | 		},
162 | 		testCase{
163 | 			desc: "hide files - missing arg",
164 | 			input: `s3proxy {
165 | 				bucket mybucket
166 | 				hide
167 | 			}`,
168 | 			shouldErr: true,
169 | 			errString: "Testfile:3 - Error during parsing: Wrong argument count or unexpected line ending after 'hide'",
170 | 		},
171 | 		testCase{
172 | 			desc: "index test",
173 | 			input: `s3proxy {
174 | 				bucket mybucket
175 | 				index i.htm i.html
176 | 			}`,
177 | 			shouldErr: false,
178 | 			obj: S3Proxy{
179 | 				Bucket:     "mybucket",
180 | 				IndexNames: []string{"i.htm", "i.html"},
181 | 			},
182 | 		},
183 | 		testCase{
184 | 			desc: "index - missing arg",
185 | 			input: `s3proxy {
186 | 				bucket mybucket
187 | 				index
188 | 			}`,
189 | 			shouldErr: true,
190 | 			errString: "Testfile:3 - Error during parsing: Wrong argument count or unexpected line ending after 'index'",
191 | 		},
192 | 	}
193 | 
194 | 	for _, tc := range testCases {
195 | 		d := caddyfile.NewTestDispenser(tc.input)
196 | 		prox, err := parseCaddyfileWithDispenser(d)
197 | 
198 | 		if tc.shouldErr {
199 | 			if err == nil {
200 | 				t.Errorf("Test case '%s' expected an err and did not get one", tc.desc)
201 | 			}
202 | 			if prox != nil {
203 | 				t.Errorf("Test case '%s' expected an nil obj but it was not nil", tc.desc)
204 | 			}
205 | 			if err.Error() != tc.errString {
206 | 				t.Errorf("Test case '%s' expected err '%s' but got '%s'", tc.desc, tc.errString, err.Error())
207 | 			}
208 | 		} else {
209 | 			if err != nil {
210 | 				t.Errorf("Test case '%s' unexpected err '%s'", tc.desc, err.Error())
211 | 			}
212 | 			if prox == nil {
213 | 				t.Errorf("Test case '%s' return object was nil", tc.desc)
214 | 				continue
215 | 			}
216 | 			if !reflect.DeepEqual(*prox, tc.obj) {
217 | 				t.Errorf("Test case '%s' expected Endpoint of  '%#v' but got '%#v'", tc.desc, tc.obj, prox)
218 | 			}
219 | 		}
220 | 	}
221 | }
222 | 


--------------------------------------------------------------------------------
/errors.go:
--------------------------------------------------------------------------------
  1 | package caddys3proxy
  2 | 
  3 | import (
  4 | 	"net/http"
  5 | 
  6 | 	"github.com/aws/aws-sdk-go/aws/awserr"
  7 | 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
  8 | )
  9 | 
 10 | // Mapping of aws error codes to http status
 11 | // See: https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
 12 | var awsErrorCodesMapping = map[string]int{
 13 | 	"AccessDenied":                                   http.StatusForbidden,
 14 | 	"AccountProblem":                                 http.StatusForbidden,
 15 | 	"AllAccessDisabled":                              http.StatusForbidden,
 16 | 	"AmbiguousGrantByEmailAddress":                   http.StatusBadRequest,
 17 | 	"AuthorizationHeaderMalformed":                   http.StatusBadRequest,
 18 | 	"BadDigest":                                      http.StatusBadRequest,
 19 | 	"BucketAlreadyExists":                            http.StatusConflict,
 20 | 	"BucketAlreadyOwnedByYou":                        http.StatusConflict,
 21 | 	"BucketNotEmpty":                                 http.StatusConflict,
 22 | 	"CredentialsNotSupported":                        http.StatusBadRequest,
 23 | 	"CrossLocationLoggingProhibited":                 http.StatusForbidden,
 24 | 	"EntityTooSmall":                                 http.StatusBadRequest,
 25 | 	"EntityTooLarge":                                 http.StatusBadRequest,
 26 | 	"ExpiredToken":                                   http.StatusBadRequest,
 27 | 	"IllegalLocationConstraintException":             http.StatusBadRequest,
 28 | 	"IllegalVersioningConfigurationException":        http.StatusBadRequest,
 29 | 	"IncompleteBody":                                 http.StatusBadRequest,
 30 | 	"IncorrectNumberOfFilesInPostRequest":            http.StatusBadRequest,
 31 | 	"InlineDataTooLarge":                             http.StatusBadRequest,
 32 | 	"InternalError":                                  http.StatusInternalServerError,
 33 | 	"InvalidAccessKeyId":                             http.StatusForbidden,
 34 | 	"InvalidAccessPoint":                             http.StatusBadRequest,
 35 | 	"InvalidArgument":                                http.StatusBadRequest,
 36 | 	"InvalidBucketName":                              http.StatusBadRequest,
 37 | 	"InvalidBucketState":                             http.StatusConflict,
 38 | 	"InvalidDigest":                                  http.StatusBadRequest,
 39 | 	"InvalidEncryptionAlgorithmError":                http.StatusBadRequest,
 40 | 	"InvalidLocationConstraint":                      http.StatusBadRequest,
 41 | 	"InvalidObjectState":                             http.StatusForbidden,
 42 | 	"InvalidPart":                                    http.StatusBadRequest,
 43 | 	"InvalidPartOrder":                               http.StatusBadRequest,
 44 | 	"InvalidPayer":                                   http.StatusForbidden,
 45 | 	"InvalidPolicyDocument":                          http.StatusBadRequest,
 46 | 	"InvalidRange":                                   http.StatusRequestedRangeNotSatisfiable,
 47 | 	"InvalidRequest":                                 http.StatusBadRequest,
 48 | 	"InvalidSecurity":                                http.StatusForbidden,
 49 | 	"InvalidSOAPRequest":                             http.StatusBadRequest,
 50 | 	"InvalidStorageClass":                            http.StatusBadRequest,
 51 | 	"InvalidTargetBucketForLogging":                  http.StatusBadRequest,
 52 | 	"InvalidToken":                                   http.StatusBadRequest,
 53 | 	"InvalidURI":                                     http.StatusBadRequest,
 54 | 	"KeyTooLongError":                                http.StatusBadRequest,
 55 | 	"MalformedACLError":                              http.StatusBadRequest,
 56 | 	"MalformedPOSTRequest":                           http.StatusBadRequest,
 57 | 	"MalformedXML":                                   http.StatusBadRequest,
 58 | 	"MaxMessageLengthExceeded":                       http.StatusBadRequest,
 59 | 	"MaxPostPreDataLengthExceededError":              http.StatusBadRequest,
 60 | 	"MetadataTooLarge":                               http.StatusBadRequest,
 61 | 	"MethodNotAllowed":                               http.StatusMethodNotAllowed,
 62 | 	"MissingContentLength":                           http.StatusLengthRequired,
 63 | 	"MissingRequestBodyError":                        http.StatusBadRequest,
 64 | 	"MissingSecurityElement":                         http.StatusBadRequest,
 65 | 	"MissingSecurityHeader":                          http.StatusBadRequest,
 66 | 	"NoLoggingStatusForKey":                          http.StatusBadRequest,
 67 | 	"NoSuchBucket":                                   http.StatusNotFound,
 68 | 	"NoSuchBucketPolicy":                             http.StatusNotFound,
 69 | 	"NoSuchKey":                                      http.StatusNotFound,
 70 | 	"NoSuchLifecycleConfiguration":                   http.StatusNotFound,
 71 | 	"NoSuchUpload":                                   http.StatusNotFound,
 72 | 	"NoSuchVersion":                                  http.StatusNotFound,
 73 | 	"NotImplemented":                                 http.StatusNotImplemented,
 74 | 	"NotSignedUp":                                    http.StatusForbidden,
 75 | 	"OperationAborted":                               http.StatusConflict,
 76 | 	"PermanentRedirect":                              http.StatusMovedPermanently,
 77 | 	"PreconditionFailed":                             http.StatusPreconditionFailed,
 78 | 	"Redirect":                                       http.StatusTemporaryRedirect,
 79 | 	"RestoreAlreadyInProgress":                       http.StatusConflict,
 80 | 	"RequestIsNotMultiPartContent":                   http.StatusBadRequest,
 81 | 	"RequestTimeout":                                 http.StatusBadRequest,
 82 | 	"RequestTimeTooSkewed":                           http.StatusForbidden,
 83 | 	"RequestTorrentOfBucketError":                    http.StatusBadRequest,
 84 | 	"ServerSideEncryptionConfigurationNotFoundError": http.StatusBadRequest,
 85 | 	"ServiceUnavailable":                             http.StatusServiceUnavailable,
 86 | 	"SignatureDoesNotMatch":                          http.StatusForbidden,
 87 | 	"SlowDown":                                       http.StatusServiceUnavailable,
 88 | 	"TemporaryRedirect":                              http.StatusTemporaryRedirect,
 89 | 	"TokenRefreshRequired":                           http.StatusBadRequest,
 90 | 	"TooManyAccessPoints":                            http.StatusBadRequest,
 91 | 	"TooManyBuckets":                                 http.StatusBadRequest,
 92 | 	"UnexpectedContent":                              http.StatusBadRequest,
 93 | 	"UnresolvableGrantByEmailAddress":                http.StatusBadRequest,
 94 | 	"UserKeyMustBeSpecified":                         http.StatusBadRequest,
 95 | 	"NoSuchAccessPoint":                              http.StatusBadRequest,
 96 | 	"InvalidTag":                                     http.StatusBadRequest,
 97 | 	"MalformedPolicy":                                http.StatusBadRequest,
 98 | 
 99 | 	// This one is not defined in the doc above - but it happens...
100 | 	// https://github.com/aws/aws-sdk-go/issues/3637
101 | 	"NotModified": http.StatusNotModified,
102 | }
103 | 
104 | func convertToCaddyError(err error) caddyhttp.HandlerError {
105 | 	caddyErr, isCaddyErr := err.(caddyhttp.HandlerError)
106 | 	if isCaddyErr {
107 | 		// Already a caddy error
108 | 		return caddyErr
109 | 	}
110 | 	if aerr, ok := err.(awserr.Error); ok {
111 | 		//If aws error look up status code in above table
112 | 		if code, ok := awsErrorCodesMapping[aerr.Code()]; ok {
113 | 			return caddyhttp.Error(code, aerr)
114 | 		}
115 | 	}
116 | 
117 | 	// Any other error is a 500
118 | 	return caddyhttp.Error(http.StatusInternalServerError, err)
119 | }
120 | 


--------------------------------------------------------------------------------
/example/Caddyfile:
--------------------------------------------------------------------------------
 1 | {
 2 | 	order s3proxy last
 3 | 	debug
 4 | }
 5 | 
 6 | :80 {
 7 | 	log
 8 | 	respond /health-check "OK" 200
 9 | 	respond /test "This is working" 200
10 | 
11 | 	# Example using strip prefix
12 | 	route /dev/testing/coverage-reports/* {
13 | 		uri strip_prefix /dev/testing/coverage-reports
14 |                 s3proxy {
15 |                         region "us-west-2"
16 |                         bucket "test-results"
17 | 			endpoint "http://localstack:4566/"
18 | 			force_path_style
19 |                 }
20 | 	}
21 | 
22 | 	route /animals/* {
23 | 		# It can be useful to have the root and s3proxy
24 | 		# in the same route to limit the scope of the root.
25 | 		# Especially important if also doing a file server.
26 | 		root * /a/long/path/we/have/for
27 |                 s3proxy {
28 |                         region "us-west-2"
29 |                         bucket "bkt"
30 | 			endpoint "http://localstack:4566/"
31 | 			enable_put
32 | 			enable_delete
33 | 			browse
34 | 			force_path_style
35 |                 }
36 | 	}
37 | 
38 | 	# To have the s3prox without the route directive you
39 | 	# need to set the order which we do at the top of this file.
40 |         s3proxy /* {
41 | 		region "us-west-2"
42 | 		bucket "my-bucket"
43 | 		index index.html
44 | 		endpoint "http://localstack:4566/"
45 | 		force_path_style
46 | 	}
47 | }
48 | 
49 | 


--------------------------------------------------------------------------------
/example/LOCALSTACK_EXAMPLE.md:
--------------------------------------------------------------------------------
  1 | 
  2 | # localstack example
  3 | 
  4 | This dir contains an example of using cadd with the caddy-s3-proxy against localstack.  localstack is a mock version of S3 and other AWS services you can run locally.
  5 | 
  6 | 
  7 | ## Running the demo
  8 | 
  9 | You need to have docker and docker-compose installed.
 10 | 
 11 | First you need a docker image running caddy with the s3proxy installed.
 12 | You can type ```make docker``` to do that.
 13 | 
 14 | Then cd into this directory and type:
 15 | ```
 16 | docker-compose up
 17 | ```
 18 | 
 19 | The script awslocal/populate.sh populates the S3 buckets of localstack
 20 | with some sample content to so a few examples of how to use s3proxy.
 21 | Each of the following examples are configured in the Caddyfile and 
 22 | you can play with them by hitting http://localhost.  
 23 | 
 24 | ## Example #1 - Basic config
 25 | 
 26 | Here is a basic config for s3proxy:
 27 | ```
 28 | {
 29 |         order s3proxy last
 30 | }
 31 | 
 32 | s3proxy {
 33 | 	region "us-west-2"
 34 | 	bucket "my-bucket"
 35 | 	index index.html
 36 | 	endpoint "http://localstack:4566/"
 37 |     force_path_style
 38 | }
 39 | ```
 40 | 
 41 | With localstack you need to use the endpoint directive.  However, in
 42 | normal use on AWS it is not needed.  The bucket directive is required
 43 | and with this config any path to caddy is simply used as a key to
 44 | fetch content in the bucket.
 45 | 
 46 | The region is also required.  (However, it can also be set by setting
 47 | the environment variable AWS_REGION.)
 48 | 
 49 | Once you have launch docker-compose you can try this config out with the following curl:
 50 | ```
 51 | curl localhost/hello.txt
 52 | ```
 53 | which should return "hello world".
 54 | 
 55 | 
 56 | Of course, the content can also be html and seen via a browser.  Open your
 57 | browser to *http://localhost* and you should see a sample web page.  This
 58 | eample is also utilizing the "index" directive which tells the s3proxy
 59 | that if the "key" appears to be a directory - look for an object called
 60 | index.html to display instead.
 61 | 
 62 | BTW, the default values for the index directive are index.html and index.htm
 63 | so the sample use of the index directive here is not even needed.
 64 | 
 65 | ## Example #2 - Using uri strip_prefix 
 66 | 
 67 | Sometimes you do not want all of your sites "path" to be used as a
 68 | key into the bucket.  Here is an example where only the last part of
 69 | the path is used to create the key in the bucket.  
 70 | ```
 71 |         route /dev/testing/coverage-reports/* {
 72 |                 uri strip_prefix /dev/testing/coverage-reports
 73 |                 s3proxy {
 74 |                         region "us-west-2"
 75 |                         bucket "test-results"
 76 |                         endpoint "http://localstack:4566/"
 77 | 		                force_path_style
 78 |                 }
 79 |         }
 80 | ```
 81 | Here we are using the "uri strip_prefix" directive to strip the path 
 82 | /dev/testing/coverage-reports from the url befor calling the s3proxy.
 83 | So the web site path of /dev/testing/coverage-reports/2/report.txt will 
 84 | return the s3 object with the key of /2/report.txt
 85 | 
 86 | Try it out with the following curl:
 87 | ```
 88 | curl localhost/dev/testing/coverage-reports/2/report.txt
 89 | ```
 90 | 
 91 | ## Example #3 - using the root directive
 92 | 
 93 | Also, the first part of your key may also not be something you want in
 94 | your website path.  You can use the root directive to define the "prefix"
 95 | to your S3 key that gets prepended to your path before getting an object
 96 | from S3.
 97 | 
 98 | Here is an example config:
 99 | ```
100 |         route /animals/* {
101 |                 root * /a/long/path/we/have/for
102 |                 s3proxy {
103 |                         region "us-west-2"
104 |                         bucket "bkt"
105 |                         endpoint "http://localstack:4566/"
106 | 			enable_put
107 | 			enable_delete
108 | 		                force_path_style
109 |                 }
110 |         }
111 | ```
112 | 
113 | You can try it out like this:
114 | ```
115 | curl localhost/animals/dog.txt
116 | ```
117 | 
118 | In this case the website request path of /animals/dog.txt
119 | will return the S3 object of /a/long/path/we/have/for/animals/dog.txt
120 | 
121 | 
122 | ## Example #4 - put and delete operations
123 | 
124 | It is also possible to send PUT and DELETE operations through the proxy.  Of course,
125 | you will want to ensure this is locked down with proper authentication!  By default,
126 | the operations are not allow - but you can turn support for them on with the
127 | *enable_put* and *enable_delete* directives.
128 | 
129 | In the above example, these options have been turned on.  You can try it out woth
130 | some curl commands.  (Be sure to set the Content-Type header when loading data.)
131 | 
132 | ```
133 | curl -X PUT -d "COW GOES MOO"  -H "Content-Type: text/plain" localhost/animals/cow.txt
134 | curl localhost/animals/cow.txt
135 | 
136 | curl -X DELETE  localhost/animals/cow.txt
137 | curl -i localhost/animals/cow.txt
138 | ```
139 | 


--------------------------------------------------------------------------------
/example/awslocal/populate.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | set -x
 3 | awslocal s3 mb s3://my-bucket
 4 | echo "hello world" | awslocal s3 cp --no-guess-mime-type --content-type text/html - s3://my-bucket/hello.txt
 5 | cat << EOF >  index.html
 6 | <!DOCTYPE html><title>s3proxy Examples</title>
 7 | <h1>s3proxy Examples</h1>
 8 | <p>This is a silly sample site served by caddy and s3proxy.</p>
 9 | EOF
10 | awslocal s3 cp index.html s3://my-bucket/index.html
11 | 
12 | awslocal s3 mb s3://test-results
13 | echo "test reports" | awslocal s3 cp - s3://test-results/index.html
14 | echo "resuults for test 1" | awslocal s3 cp - s3://test-results/1/report.txt
15 | echo "resuults for test 2" | awslocal s3 cp - s3://test-results/2/report.txt
16 | echo "resuults for test 57" | awslocal s3 cp - s3://test-results/57/report.txt
17 | echo "resuults for test 784" | awslocal s3 cp - s3://test-results/784/report.txt
18 | 
19 | awslocal s3 mb s3://bkt
20 | echo "CAT" | awslocal s3 cp - s3://bkt/a/long/path/we/have/for/animals/cat.txt
21 | echo "DOG" | awslocal s3 cp - s3://bkt/a/long/path/we/have/for/animals/dog.txt
22 | echo "COW" | awslocal s3 cp - s3://bkt/a/long/path/we/have/for/animals/cow.txt
23 | echo "BAT" | awslocal s3 cp - s3://bkt/a/long/path/we/have/for/animals/bat.txt
24 | set +x
25 | 
26 | 


--------------------------------------------------------------------------------
/example/docker-compose.yml:
--------------------------------------------------------------------------------
 1 | 
 2 | version: '2.1'
 3 | 
 4 | services:
 5 |   localstack:
 6 |     image: localstack/localstack
 7 |     ports:
 8 |       - "4566-4599:4566-4599"
 9 |     environment:
10 |       - SERVICES=s3
11 |       - DEBUG=1
12 |     volumes:
13 |       - ./awslocal:/docker-entrypoint-initaws.d
14 |   caddy:
15 |     image: caddy
16 |     volumes:
17 |       - ./Caddyfile:/Caddyfile
18 |     ports:
19 |       - "80:80"
20 |     links:
21 |       - localstack
22 |     command: caddy  run --config /Caddyfile
23 | 


--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
 1 | module github.com/lindenlab/caddy-s3-proxy
 2 | 
 3 | go 1.13
 4 | 
 5 | require (
 6 | 	github.com/alecthomas/chroma v0.8.2 // indirect
 7 | 	github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect
 8 | 	github.com/aryann/difflib v0.0.0-20210328193216-ff5ff6dc229b // indirect
 9 | 	github.com/aws/aws-sdk-go v1.44.272
10 | 	github.com/caddyserver/caddy/v2 v2.6.4
11 | 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
12 | 	github.com/dustin/go-humanize v1.0.1
13 | 	github.com/google/cel-go v0.13.0 // indirect
14 | 	github.com/google/cel-spec v0.4.0 // indirect
15 | 	github.com/jsternberg/zap-logfmt v1.2.0 // indirect
16 | 	github.com/klauspost/cpuid v1.3.1 // indirect
17 | 	github.com/lucas-clemente/quic-go v0.19.3 // indirect
18 | 	github.com/naoina/go-stringutil v0.1.0 // indirect
19 | 	github.com/naoina/toml v0.1.1 // indirect
20 | 	github.com/smallstep/certificates v0.23.2 // indirect
21 | 	github.com/smallstep/cli v0.15.2 // indirect
22 | 	github.com/spf13/cobra v1.7.0 // indirect
23 | 	github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691 // indirect
24 | 	go.step.sm/crypto v0.23.2 // indirect
25 | 	go.step.sm/linkedca v0.19.1 // indirect
26 | 	go.uber.org/multierr v1.6.0 // indirect
27 | 	go.uber.org/zap v1.24.0
28 | 	gopkg.in/yaml.v3 v3.0.1 // indirect
29 | )
30 | 


--------------------------------------------------------------------------------
/s3proxy.go:
--------------------------------------------------------------------------------
  1 | package caddys3proxy
  2 | 
  3 | import (
  4 | 	"bytes"
  5 | 	"errors"
  6 | 	"fmt"
  7 | 	"html/template"
  8 | 	"io"
  9 | 	"io/ioutil"
 10 | 	"net/http"
 11 | 	"path"
 12 | 	"path/filepath"
 13 | 	"reflect"
 14 | 	"strings"
 15 | 	"time"
 16 | 
 17 | 	"github.com/aws/aws-sdk-go/aws"
 18 | 	"github.com/aws/aws-sdk-go/aws/awserr"
 19 | 	"github.com/aws/aws-sdk-go/aws/session"
 20 | 	"github.com/aws/aws-sdk-go/service/s3"
 21 | 	caddy "github.com/caddyserver/caddy/v2"
 22 | 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
 23 | 	"go.uber.org/zap"
 24 | )
 25 | 
 26 | var defaultIndexNames = []string{"index.html", "index.txt"}
 27 | 
 28 | func init() {
 29 | 	caddy.RegisterModule(S3Proxy{})
 30 | }
 31 | 
 32 | // S3Proxy implements a proxy to return, set, delete or browse objects from S3
 33 | type S3Proxy struct {
 34 | 	// The path to the root of the site. Default is `{http.vars.root}` if set,
 35 | 	// Or if not set the value is "" - meaning use the whole path as a key.
 36 | 	Root string `json:"root,omitempty"`
 37 | 
 38 | 	// The AWS region the bucket is hosted in
 39 | 	Region string `json:"region,omitempty"`
 40 | 
 41 | 	// The AWS profile to use if mulitple profiles are specified in creds
 42 | 	Profile string `json:"profile,omitempty"`
 43 | 
 44 | 	// The name of the S3 bucket
 45 | 	Bucket string `json:"bucket,omitempty"`
 46 | 
 47 | 	// Use non-standard endpoint for S3
 48 | 	Endpoint string `json:"endpoint,omitempty"`
 49 | 
 50 | 	// The names of files to try as index files if a folder is requested.
 51 | 	IndexNames []string `json:"index_names,omitempty"`
 52 | 
 53 | 	// A glob pattern used to hide matching key paths (returning a 404)
 54 | 	Hide []string
 55 | 
 56 | 	// Flag to determine if PUT operations are allowed (default false)
 57 | 	EnablePut bool
 58 | 
 59 | 	// Flag to determine if DELETE operations are allowed (default false)
 60 | 	EnableDelete bool
 61 | 
 62 | 	// Flag to enable browsing of "directories" in S3 (paths that end with a /)
 63 | 	EnableBrowse bool
 64 | 
 65 | 	// Path to a template file to use for generating browse dir html page
 66 | 	BrowseTemplate string
 67 | 
 68 | 	// Mapping of HTTP error status to S3 keys or pass through option.
 69 | 	ErrorPages map[int]string `json:"error_pages,omitempty"`
 70 | 
 71 | 	// S3 key to a default error page or pass through option.
 72 | 	DefaultErrorPage string `json:"default_error_page,omitempty"`
 73 | 
 74 | 	// Set this to `true` to force the request to use path-style addressing.
 75 | 	S3ForcePathStyle bool `json:"force_path_style,omitempty"`
 76 | 
 77 | 	// Set this to `true` to enable S3 Accelerate feature.
 78 | 	S3UseAccelerate bool `json:"use_accelerate,omitempty"`
 79 | 
 80 | 	client      *s3.S3
 81 | 	dirTemplate *template.Template
 82 | 	log         *zap.Logger
 83 | }
 84 | 
 85 | // CaddyModule returns the Caddy module information.
 86 | func (S3Proxy) CaddyModule() caddy.ModuleInfo {
 87 | 	return caddy.ModuleInfo{
 88 | 		ID:  "http.handlers.s3proxy",
 89 | 		New: func() caddy.Module { return new(S3Proxy) },
 90 | 	}
 91 | }
 92 | 
 93 | func (p *S3Proxy) Provision(ctx caddy.Context) (err error) {
 94 | 	p.log = ctx.Logger(p)
 95 | 
 96 | 	if p.Root == "" {
 97 | 		p.Root = "{http.vars.root}"
 98 | 	}
 99 | 
100 | 	if p.IndexNames == nil {
101 | 		p.IndexNames = defaultIndexNames
102 | 	}
103 | 
104 | 	if p.ErrorPages == nil {
105 | 		p.ErrorPages = make(map[int]string)
106 | 	}
107 | 
108 | 	if p.EnableBrowse {
109 | 		var tpl *template.Template
110 | 		var err error
111 | 
112 | 		if p.BrowseTemplate != "" {
113 | 			tpl, err = template.ParseFiles(p.BrowseTemplate)
114 | 			if err != nil {
115 | 				return fmt.Errorf("parsing browse template file: %v", err)
116 | 			}
117 | 		} else {
118 | 			tpl, err = template.New("default_listing").Parse(defaultBrowseTemplate)
119 | 			if err != nil {
120 | 				return fmt.Errorf("parsing default browse template: %v", err)
121 | 			}
122 | 		}
123 | 		p.dirTemplate = tpl
124 | 	}
125 | 
126 | 	var config aws.Config
127 | 
128 | 	// If Region is not specified NewSession will look for it from an env value AWS_REGION
129 | 	if p.Region != "" {
130 | 		config.Region = aws.String(p.Region)
131 | 	}
132 | 
133 | 	if p.Endpoint != "" {
134 | 		config.Endpoint = aws.String(p.Endpoint)
135 | 	}
136 | 
137 | 	if p.S3ForcePathStyle {
138 | 		config.S3ForcePathStyle = aws.Bool(p.S3ForcePathStyle)
139 | 	}
140 | 
141 | 	if p.S3UseAccelerate {
142 | 		config.S3UseAccelerate = aws.Bool(p.S3UseAccelerate)
143 | 	}
144 | 
145 | 	sess, err := session.NewSessionWithOptions(session.Options{
146 | 		Profile:           p.Profile,
147 | 		Config:            config,
148 | 		SharedConfigState: session.SharedConfigEnable,
149 | 	})
150 | 	if err != nil {
151 | 		p.log.Error("could not create AWS session",
152 | 			zap.String("error", err.Error()),
153 | 		)
154 | 		return err
155 | 	}
156 | 
157 | 	// Create S3 service client
158 | 	p.client = s3.New(sess)
159 | 	p.log.Info("S3 proxy initialized for bucket: " + p.Bucket)
160 | 	p.log.Debug("config values",
161 | 		zap.String("endpoint", p.Endpoint),
162 | 		zap.String("region", p.Region),
163 | 		zap.String("profile", p.Profile),
164 | 		zap.Bool("enable_put", p.EnablePut),
165 | 		zap.Bool("enable_delete", p.EnableDelete),
166 | 		zap.String("default_error_page", p.DefaultErrorPage),
167 | 		zap.Bool("enable_browse", p.EnableBrowse),
168 | 		zap.Bool("force_path_style", p.S3ForcePathStyle),
169 | 		zap.Bool("use_accelerate", p.S3UseAccelerate),
170 | 	)
171 | 
172 | 	return nil
173 | }
174 | 
175 | func (p S3Proxy) getS3Object(bucket string, path string, headers http.Header) (*s3.GetObjectOutput, error) {
176 | 	oi := &s3.GetObjectInput{
177 | 		Bucket: aws.String(bucket),
178 | 		Key:    aws.String(path),
179 | 	}
180 | 
181 | 	if rg := headers.Get("Range"); rg != "" {
182 | 		oi = oi.SetRange(rg)
183 | 	}
184 | 	if ifMatch := headers.Get("If-Match"); ifMatch != "" {
185 | 		oi = oi.SetIfMatch(ifMatch)
186 | 	}
187 | 	if ifNoneMatch := headers.Get("If-None-Match"); ifNoneMatch != "" {
188 | 		oi = oi.SetIfNoneMatch(ifNoneMatch)
189 | 	}
190 | 	if ifModifiedSince := headers.Get("If-Modified-Since"); ifModifiedSince != "" {
191 | 		t, err := time.Parse(http.TimeFormat, ifModifiedSince)
192 | 		if err == nil {
193 | 			oi = oi.SetIfModifiedSince(t)
194 | 		}
195 | 	}
196 | 	if ifUnmodifiedSince := headers.Get("If-Unmodified-Since"); ifUnmodifiedSince != "" {
197 | 		t, err := time.Parse(http.TimeFormat, ifUnmodifiedSince)
198 | 		if err == nil {
199 | 			oi = oi.SetIfUnmodifiedSince(t)
200 | 		}
201 | 	}
202 | 
203 | 	p.log.Debug("get from S3",
204 | 		zap.String("bucket", bucket),
205 | 		zap.String("key", path),
206 | 	)
207 | 
208 | 	// TODO: GetObject could return the aws error InternalError, if that happens it is best practice to retry the
209 | 	// the call.  That retry logic should go here...
210 | 	return p.client.GetObject(oi)
211 | }
212 | 
213 | func joinPath(root string, uriPath string) string {
214 | 	isDir := uriPath[len(uriPath)-1:] == "/"
215 | 	newPath := path.Join(root, uriPath)
216 | 	if isDir && newPath != "/" {
217 | 		// Join will strip the ending /
218 | 		// add it back if it was there as it implies a dir view
219 | 		return newPath + "/"
220 | 	}
221 | 	return newPath
222 | }
223 | 
224 | func makeAwsString(str string) *string {
225 | 	if str == "" {
226 | 		return nil
227 | 	}
228 | 	return aws.String(str)
229 | }
230 | 
231 | func (p S3Proxy) PutHandler(w http.ResponseWriter, r *http.Request, key string) error {
232 | 	isDir := strings.HasSuffix(key, "/")
233 | 	if isDir || !p.EnablePut {
234 | 		err := errors.New("method not allowed")
235 | 		return caddyhttp.Error(http.StatusMethodNotAllowed, err)
236 | 	}
237 | 
238 | 	// The request gives us r.Body a ReadCloser.  However, Put needs a ReadSeeker.
239 | 	// So we need to read the entire object in memory and create the ReadSeeker.
240 | 	// TODO: this will not work well for very large files - will run out of memory
241 | 	buf, err := ioutil.ReadAll(r.Body)
242 | 	if err != nil {
243 | 		return convertToCaddyError(err)
244 | 	}
245 | 
246 | 	oi := s3.PutObjectInput{
247 | 		Bucket:             aws.String(p.Bucket),
248 | 		Key:                aws.String(key),
249 | 		CacheControl:       makeAwsString(r.Header.Get("Cache-Control")),
250 | 		ContentDisposition: makeAwsString(r.Header.Get("Content-Disposition")),
251 | 		ContentEncoding:    makeAwsString(r.Header.Get("Content-Encoding")),
252 | 		ContentLanguage:    makeAwsString(r.Header.Get("Content-Language")),
253 | 		ContentType:        makeAwsString(r.Header.Get("Content-Type")),
254 | 		Body:               bytes.NewReader(buf),
255 | 	}
256 | 	po, err := p.client.PutObject(&oi)
257 | 	if err != nil {
258 | 		return convertToCaddyError(err)
259 | 	}
260 | 
261 | 	setStrHeader(w, "ETag", po.ETag)
262 | 
263 | 	return nil
264 | }
265 | 
266 | func (p S3Proxy) DeleteHandler(w http.ResponseWriter, r *http.Request, key string) error {
267 | 	isDir := strings.HasSuffix(key, "/")
268 | 	if isDir || !p.EnableDelete {
269 | 		err := errors.New("method not allowed")
270 | 		return caddyhttp.Error(http.StatusMethodNotAllowed, err)
271 | 	}
272 | 
273 | 	di := s3.DeleteObjectInput{
274 | 		Bucket: aws.String(p.Bucket),
275 | 		Key:    aws.String(key),
276 | 	}
277 | 	_, err := p.client.DeleteObject(&di)
278 | 	if err != nil {
279 | 		return convertToCaddyError(err)
280 | 	}
281 | 
282 | 	return nil
283 | }
284 | 
285 | func (p S3Proxy) BrowseHandler(w http.ResponseWriter, r *http.Request, key string) error {
286 | 
287 | 	input := p.ConstructListObjInput(r, key)
288 | 
289 | 	result, err := p.client.ListObjectsV2(&input)
290 | 	if err != nil {
291 | 		p.log.Debug("error in ListObjectsV2",
292 | 			zap.String("bucket", p.Bucket),
293 | 			zap.String("key", key),
294 | 			zap.String("err", err.Error()),
295 | 		)
296 | 		return convertToCaddyError(err)
297 | 	}
298 | 
299 | 	pageObj := p.MakePageObj(result)
300 | 
301 | 	if r.Header.Get("Content-type") == "application/json" {
302 | 		// Give JSON output of dir
303 | 		err = pageObj.GenerateJson(w)
304 | 	} else {
305 | 		// Generate html response of dir
306 | 		err = pageObj.GenerateHtml(w, p.dirTemplate)
307 | 	}
308 | 	if err != nil {
309 | 		return convertToCaddyError(err)
310 | 	}
311 | 	return nil
312 | }
313 | 
314 | func (p S3Proxy) writeResponseFromGetObject(w http.ResponseWriter, obj *s3.GetObjectOutput) error {
315 | 	// Copy headers from AWS response to our response
316 | 	setStrHeader(w, "Cache-Control", obj.CacheControl)
317 | 	setStrHeader(w, "Content-Disposition", obj.ContentDisposition)
318 | 	setStrHeader(w, "Content-Encoding", obj.ContentEncoding)
319 | 	setStrHeader(w, "Content-Language", obj.ContentLanguage)
320 | 	setStrHeader(w, "Content-Range", obj.ContentRange)
321 | 	setStrHeader(w, "Content-Type", obj.ContentType)
322 | 	setStrHeader(w, "ETag", obj.ETag)
323 | 	setStrHeader(w, "Expires", obj.Expires)
324 | 	setTimeHeader(w, "Last-Modified", obj.LastModified)
325 | 
326 | 	// Adds all custom headers which where used on this object
327 | 	for key, value := range obj.Metadata {
328 | 		setStrHeader(w, key, value)
329 | 	}
330 | 
331 | 	var err error
332 | 	if obj.Body != nil {
333 | 		// io.Copy will set Content-Length
334 | 		w.Header().Del("Content-Length")
335 | 		_, err = io.Copy(w, obj.Body)
336 | 	}
337 | 
338 | 	return err
339 | }
340 | 
341 | func (p S3Proxy) serveErrorPage(w http.ResponseWriter, s3Key string) error {
342 | 	obj, err := p.getS3Object(p.Bucket, s3Key, nil)
343 | 	if err != nil {
344 | 		return err
345 | 	}
346 | 
347 | 	if err := p.writeResponseFromGetObject(w, obj); err != nil {
348 | 		return err
349 | 	}
350 | 
351 | 	return nil
352 | }
353 | 
354 | // ServeHTTP implements the main entry point for a request for the caddyhttp.Handler interface.
355 | func (p S3Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
356 | 	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
357 | 
358 | 	fullPath := joinPath(repl.ReplaceAll(p.Root, ""), r.URL.Path)
359 | 
360 | 	var err error
361 | 	switch r.Method {
362 | 	case http.MethodGet:
363 | 		err = p.GetHandler(w, r, fullPath)
364 | 	case http.MethodPut:
365 | 		err = p.PutHandler(w, r, fullPath)
366 | 	case http.MethodDelete:
367 | 		err = p.DeleteHandler(w, r, fullPath)
368 | 	default:
369 | 		err = caddyhttp.Error(http.StatusMethodNotAllowed, errors.New("method not allowed"))
370 | 	}
371 | 	if err == nil {
372 | 		// Success!
373 | 		return nil
374 | 	}
375 | 
376 | 	// Make the err a caddyErr if it is not already
377 | 	caddyErr, isCaddyErr := err.(caddyhttp.HandlerError)
378 | 	if !isCaddyErr {
379 | 		caddyErr = caddyhttp.Error(http.StatusInternalServerError, err)
380 | 	}
381 | 
382 | 	// If non OK status code - WriteHeader - except for GET method, where we still need to process more
383 | 	if r.Method != http.MethodGet {
384 | 		if caddyErr.StatusCode != 0 {
385 | 			w.WriteHeader(caddyErr.StatusCode)
386 | 		}
387 | 		return caddyErr
388 | 	}
389 | 
390 | 	// Certain errors we will not pass through
391 | 	if caddyErr.StatusCode == http.StatusNotModified ||
392 | 		caddyErr.StatusCode == http.StatusPreconditionFailed ||
393 | 		caddyErr.StatusCode == http.StatusRequestedRangeNotSatisfiable {
394 | 		w.WriteHeader(caddyErr.StatusCode)
395 | 		return caddyErr
396 | 	}
397 | 
398 | 	// process errors directive
399 | 	doPassThrough, doS3ErrorPage, s3Key := p.determineErrorsAction(caddyErr.StatusCode)
400 | 	if doPassThrough {
401 | 		return next.ServeHTTP(w, r)
402 | 	}
403 | 
404 | 	if caddyErr.StatusCode != 0 {
405 | 		w.WriteHeader(caddyErr.StatusCode)
406 | 	}
407 | 	if doS3ErrorPage {
408 | 		if err := p.serveErrorPage(w, s3Key); err != nil {
409 | 			// Just log the error as we don't want to swallow the parent error.
410 | 			p.log.Error("error serving error page",
411 | 				zap.String("bucket", p.Bucket),
412 | 				zap.String("key", s3Key),
413 | 				zap.String("err", err.Error()),
414 | 			)
415 | 		}
416 | 	}
417 | 	return caddyErr
418 | }
419 | 
420 | func (p S3Proxy) determineErrorsAction(statusCode int) (bool, bool, string) {
421 | 	var s3Key string
422 | 	if errorPageS3Key, hasErrorPageForCode := p.ErrorPages[statusCode]; hasErrorPageForCode {
423 | 		s3Key = errorPageS3Key
424 | 	} else if p.DefaultErrorPage != "" {
425 | 		s3Key = p.DefaultErrorPage
426 | 	}
427 | 
428 | 	if strings.ToLower(s3Key) == "pass_through" {
429 | 		return true, false, ""
430 | 	}
431 | 
432 | 	return false, s3Key != "", s3Key
433 | }
434 | 
435 | func (p S3Proxy) GetHandler(w http.ResponseWriter, r *http.Request, fullPath string) error {
436 | 	// If file is hidden - return 404
437 | 	if fileHidden(fullPath, p.Hide) {
438 | 		return caddyhttp.Error(http.StatusNotFound, nil)
439 | 	}
440 | 
441 | 	isDir := strings.HasSuffix(fullPath, "/")
442 | 	var obj *s3.GetObjectOutput
443 | 	var err error
444 | 
445 | 	if isDir && len(p.IndexNames) > 0 {
446 | 		for _, indexPage := range p.IndexNames {
447 | 			indexPath := path.Join(fullPath, indexPage)
448 | 			obj, err = p.getS3Object(p.Bucket, indexPath, r.Header)
449 | 			caddyErr := convertToCaddyError(err)
450 | 			if err == nil || caddyErr.StatusCode == 304 {
451 | 				// We found an index!
452 | 				isDir = false
453 | 				break
454 | 			} else {
455 | 				logIt := true
456 | 				if aerr, ok := err.(awserr.Error); ok {
457 | 					// Getting no such key here could be rather common
458 | 					// So only log a warning if we get any other type of error
459 | 					if aerr.Code() != s3.ErrCodeNoSuchKey {
460 | 						logIt = false
461 | 					}
462 | 				}
463 | 				if logIt {
464 | 					p.log.Warn("error when looking for index",
465 | 						zap.String("bucket", p.Bucket),
466 | 						zap.String("key", fullPath),
467 | 						zap.String("err", err.Error()),
468 | 					)
469 | 				}
470 | 			}
471 | 		}
472 | 	}
473 | 
474 | 	// If this is still a dir then browse or throw an error
475 | 	if isDir {
476 | 		if p.EnableBrowse {
477 | 			return p.BrowseHandler(w, r, fullPath)
478 | 		} else {
479 | 			err = errors.New("can not view a directory")
480 | 			return caddyhttp.Error(http.StatusForbidden, err)
481 | 		}
482 | 	}
483 | 
484 | 	// Get the obj from S3 (skip if we already did when looking for an index)
485 | 	if obj == nil {
486 | 		obj, err = p.getS3Object(p.Bucket, fullPath, r.Header)
487 | 	}
488 | 	if err != nil {
489 | 		caddyErr := convertToCaddyError(err)
490 | 		if caddyErr.StatusCode == http.StatusNotFound {
491 | 			// Log as debug as this one may be quite common
492 | 			p.log.Debug("not found",
493 | 				zap.String("bucket", p.Bucket),
494 | 				zap.String("key", fullPath),
495 | 				zap.String("err", caddyErr.Error()),
496 | 			)
497 | 		} else {
498 | 			p.log.Error("failed to get object",
499 | 				zap.String("bucket", p.Bucket),
500 | 				zap.String("key", fullPath),
501 | 				zap.String("err", caddyErr.Error()),
502 | 			)
503 | 		}
504 | 
505 | 		return caddyErr
506 | 	}
507 | 
508 | 	return p.writeResponseFromGetObject(w, obj)
509 | }
510 | 
511 | func setStrHeader(w http.ResponseWriter, key string, value *string) {
512 | 	if value != nil && len(*value) > 0 {
513 | 		w.Header().Add(key, *value)
514 | 	}
515 | }
516 | 
517 | func setTimeHeader(w http.ResponseWriter, key string, value *time.Time) {
518 | 	if value != nil && !reflect.DeepEqual(*value, time.Time{}) {
519 | 		w.Header().Add(key, value.UTC().Format(http.TimeFormat))
520 | 	}
521 | }
522 | 
523 | // fileHidden returns true if filename is hidden
524 | // according to the hide list.
525 | func fileHidden(filename string, hide []string) bool {
526 | 	sep := string(filepath.Separator)
527 | 	var components []string
528 | 
529 | 	for _, h := range hide {
530 | 		if !strings.Contains(h, sep) {
531 | 			// if there is no separator in h, then we assume the user
532 | 			// wants to hide any files or folders that match that
533 | 			// name; thus we have to compare against each component
534 | 			// of the filename, e.g. hiding "bar" would hide "/bar"
535 | 			// as well as "/foo/bar/baz" but not "/barstool".
536 | 			if len(components) == 0 {
537 | 				components = strings.Split(filename, sep)
538 | 			}
539 | 			for _, c := range components {
540 | 				if c == h {
541 | 					return true
542 | 				}
543 | 			}
544 | 		} else if strings.HasPrefix(filename, h) {
545 | 			// otherwise, if there is a separator in h, and
546 | 			// filename is exactly prefixed with h, then we
547 | 			// can do a prefix match so that "/foo" matches
548 | 			// "/foo/bar" but not "/foobar".
549 | 			withoutPrefix := strings.TrimPrefix(filename, h)
550 | 			if strings.HasPrefix(withoutPrefix, sep) {
551 | 				return true
552 | 			}
553 | 		}
554 | 
555 | 		// in the general case, a glob match will suffice
556 | 		if hidden, _ := filepath.Match(h, filename); hidden {
557 | 			return true
558 | 		}
559 | 	}
560 | 
561 | 	return false
562 | }
563 | 


--------------------------------------------------------------------------------
/s3proxy_test.go:
--------------------------------------------------------------------------------
  1 | package caddys3proxy
  2 | 
  3 | import (
  4 | 	"bytes"
  5 | 	"context"
  6 | 	"fmt"
  7 | 	"io"
  8 | 	"math/rand"
  9 | 	"mime"
 10 | 	"net/http"
 11 | 	"net/http/httptest"
 12 | 	"os"
 13 | 	"path/filepath"
 14 | 	"reflect"
 15 | 	"strings"
 16 | 	"testing"
 17 | 	"time"
 18 | 
 19 | 	"go.uber.org/zap"
 20 | 
 21 | 	"github.com/aws/aws-sdk-go/aws/awserr"
 22 | 	caddy "github.com/caddyserver/caddy/v2"
 23 | 
 24 | 	"github.com/aws/aws-sdk-go/aws"
 25 | 	"github.com/aws/aws-sdk-go/aws/session"
 26 | 	"github.com/aws/aws-sdk-go/service/s3"
 27 | )
 28 | 
 29 | type jTestCase struct {
 30 | 	root     string
 31 | 	path     string
 32 | 	expected string
 33 | }
 34 | 
 35 | func TestJoinPath(t *testing.T) {
 36 | 	testCases := []jTestCase{
 37 | 		jTestCase{
 38 | 			root:     "",
 39 | 			path:     "/foo",
 40 | 			expected: "/foo",
 41 | 		},
 42 | 		jTestCase{
 43 | 			root:     "",
 44 | 			path:     "/",
 45 | 			expected: "/",
 46 | 		},
 47 | 		jTestCase{
 48 | 			root:     "/",
 49 | 			path:     "/",
 50 | 			expected: "/",
 51 | 		},
 52 | 		jTestCase{
 53 | 			root:     "/",
 54 | 			path:     "/foo",
 55 | 			expected: "/foo",
 56 | 		},
 57 | 		jTestCase{
 58 | 			root:     "/cat",
 59 | 			path:     "/dog",
 60 | 			expected: "/cat/dog",
 61 | 		},
 62 | 		jTestCase{
 63 | 			root:     "/cat/",
 64 | 			path:     "/dog",
 65 | 			expected: "/cat/dog",
 66 | 		},
 67 | 		jTestCase{
 68 | 			root:     "/cat/",
 69 | 			path:     "/dog/",
 70 | 			expected: "/cat/dog/",
 71 | 		},
 72 | 		jTestCase{
 73 | 			root:     "",
 74 | 			path:     "/dog/",
 75 | 			expected: "/dog/",
 76 | 		},
 77 | 	}
 78 | 	for _, tc := range testCases {
 79 | 		r := joinPath(tc.root, tc.path)
 80 | 		if r != tc.expected {
 81 | 			t.Errorf("When joining '%s' and '%s' we expected '%s' but got '%s'", tc.root, tc.path, tc.expected, r)
 82 | 		}
 83 | 	}
 84 | }
 85 | 
 86 | func TestFileHidden(t *testing.T) {
 87 | 	for i, tc := range []struct {
 88 | 		inputHide []string
 89 | 		inputPath string
 90 | 		expect    bool
 91 | 	}{
 92 | 		{
 93 | 			inputHide: nil,
 94 | 			inputPath: "",
 95 | 			expect:    false,
 96 | 		},
 97 | 		{
 98 | 			inputHide: []string{".gitignore"},
 99 | 			inputPath: "/.gitignore",
100 | 			expect:    true,
101 | 		},
102 | 		{
103 | 			inputHide: []string{".git"},
104 | 			inputPath: "/.gitignore",
105 | 			expect:    false,
106 | 		},
107 | 		{
108 | 			inputHide: []string{"/.git"},
109 | 			inputPath: "/.gitignore",
110 | 			expect:    false,
111 | 		},
112 | 		{
113 | 			inputHide: []string{".git"},
114 | 			inputPath: "/.git",
115 | 			expect:    true,
116 | 		},
117 | 		{
118 | 			inputHide: []string{".git"},
119 | 			inputPath: "/.git/foo",
120 | 			expect:    true,
121 | 		},
122 | 		{
123 | 			inputHide: []string{".git"},
124 | 			inputPath: "/foo/.git/bar",
125 | 			expect:    true,
126 | 		},
127 | 		{
128 | 			inputHide: []string{"/prefix"},
129 | 			inputPath: "/prefix/foo",
130 | 			expect:    true,
131 | 		},
132 | 		{
133 | 			inputHide: []string{"/foo/*/bar"},
134 | 			inputPath: "/foo/asdf/bar",
135 | 			expect:    true,
136 | 		},
137 | 		{
138 | 			inputHide: []string{"/foo"},
139 | 			inputPath: "/foo",
140 | 			expect:    true,
141 | 		},
142 | 		{
143 | 			inputHide: []string{"/foo"},
144 | 			inputPath: "/foobar",
145 | 			expect:    false,
146 | 		},
147 | 	} {
148 | 		// for Windows' sake
149 | 		tc.inputPath = filepath.FromSlash(tc.inputPath)
150 | 		for i := range tc.inputHide {
151 | 			tc.inputHide[i] = filepath.FromSlash(tc.inputHide[i])
152 | 		}
153 | 
154 | 		actual := fileHidden(tc.inputPath, tc.inputHide)
155 | 		if actual != tc.expect {
156 | 			t.Errorf("Test %d: Is %s hidden in %v? Got %t but expected %t",
157 | 				i, tc.inputPath, tc.inputHide, actual, tc.expect)
158 | 		}
159 | 	}
160 | }
161 | 
162 | func newS3Client(t *testing.T) *s3.S3 {
163 | 	endpoint := os.Getenv("AWS_ENDPOINT")
164 | 	if endpoint == "" {
165 | 		t.Skip("Skipping test because AWS_ENDPOINT environment variable is not set.")
166 | 	}
167 | 
168 | 	config := aws.Config{
169 | 		S3ForcePathStyle: aws.Bool(true),
170 | 		Endpoint:         aws.String(endpoint),
171 | 	}
172 | 
173 | 	sess, err := session.NewSession(&config)
174 | 	if err != nil {
175 | 		t.Fatal(err)
176 | 	}
177 | 
178 | 	return s3.New(sess)
179 | }
180 | 
181 | func setupTestBucket(t *testing.T, client *s3.S3) string {
182 | 	bucketName := fmt.Sprintf(
183 | 		"caddy-s3-proxy-testdata-%d-%d",
184 | 		time.Now().UnixNano(),
185 | 		rand.Int(),
186 | 	)
187 | 	testDataDir := "testdata"
188 | 
189 | 	_, err := client.CreateBucket(&s3.CreateBucketInput{
190 | 		Bucket: aws.String(bucketName),
191 | 	})
192 | 	if awsErr, isAwsErr := err.(awserr.Error); isAwsErr {
193 | 		if awsErr.Code() == s3.ErrCodeBucketAlreadyExists {
194 | 			err = nil
195 | 		}
196 | 	}
197 | 	if err != nil {
198 | 		t.Fatal(err)
199 | 	}
200 | 
201 | 	if err := filepath.Walk(testDataDir, func(p string, info os.FileInfo, err error) error {
202 | 		if info.IsDir() {
203 | 			return nil
204 | 		}
205 | 
206 | 		if err != nil {
207 | 			return err
208 | 		}
209 | 
210 | 		key := strings.TrimPrefix(p, testDataDir)
211 | 		contentType := mime.TypeByExtension(filepath.Ext(p))
212 | 		if contentType == "" {
213 | 			contentType = "application/octet-stream"
214 | 		}
215 | 
216 | 		file, err := os.Open(p)
217 | 		if err != nil {
218 | 			return err
219 | 		}
220 | 		defer file.Close()
221 | 
222 | 		if _, err := client.PutObject(&s3.PutObjectInput{
223 | 			Bucket:      aws.String(bucketName),
224 | 			Key:         aws.String(key),
225 | 			ContentType: aws.String(contentType),
226 | 			Body:        file,
227 | 		}); err != nil {
228 | 			return err
229 | 		}
230 | 
231 | 		return nil
232 | 	}); err != nil {
233 | 		t.Fatal(err)
234 | 	}
235 | 
236 | 	return bucketName
237 | }
238 | 
239 | func TestProxy(t *testing.T) {
240 | 	client := newS3Client(t)
241 | 	bucketName := setupTestBucket(t, client)
242 | 
243 | 	for _, tc := range []struct {
244 | 		name                 string
245 | 		proxy                S3Proxy
246 | 		method               string
247 | 		body                 []byte
248 | 		headers              http.Header
249 | 		path                 string
250 | 		expectedCode         int
251 | 		expectedHeaders      http.Header
252 | 		expectedResponseText string
253 | 		expectsEmptyResponse bool
254 | 	}{
255 | 		{
256 | 			name:                 "can get simple JSON object",
257 | 			proxy:                S3Proxy{Bucket: bucketName},
258 | 			method:               http.MethodGet,
259 | 			path:                 "/test.json",
260 | 			expectedCode:         http.StatusOK,
261 | 			expectedResponseText: `{"foo": "bar"}`,
262 | 			expectedHeaders: http.Header{
263 | 				"Content-Type": []string{"application/json"},
264 | 			},
265 | 		},
266 | 		{
267 | 			name:                 "hidden file are not served",
268 | 			proxy:                S3Proxy{Bucket: bucketName, Hide: []string{"test.json"}},
269 | 			method:               http.MethodGet,
270 | 			path:                 "/test.json",
271 | 			expectedCode:         http.StatusNotFound,
272 | 			expectsEmptyResponse: true,
273 | 		},
274 | 		{
275 | 			name:                 "can't post",
276 | 			proxy:                S3Proxy{Bucket: bucketName},
277 | 			method:               http.MethodPost,
278 | 			path:                 "/cannot-post",
279 | 			expectedCode:         http.StatusMethodNotAllowed,
280 | 			expectsEmptyResponse: true,
281 | 		},
282 | 		{
283 | 			name:                 "can't delete if not allowed",
284 | 			proxy:                S3Proxy{Bucket: bucketName},
285 | 			method:               http.MethodDelete,
286 | 			path:                 "/cannot-delete",
287 | 			expectedCode:         http.StatusMethodNotAllowed,
288 | 			expectsEmptyResponse: true,
289 | 		},
290 | 		{
291 | 			name:                 "can delete if allowed",
292 | 			proxy:                S3Proxy{Bucket: bucketName, EnableDelete: true},
293 | 			method:               http.MethodDelete,
294 | 			path:                 "/to-delete.json",
295 | 			expectedCode:         http.StatusOK,
296 | 			expectsEmptyResponse: true,
297 | 		},
298 | 		{
299 | 			name:                 "can't put if not allowed",
300 | 			proxy:                S3Proxy{Bucket: bucketName},
301 | 			method:               http.MethodPut,
302 | 			path:                 "/cannot-put",
303 | 			expectedCode:         http.StatusMethodNotAllowed,
304 | 			expectsEmptyResponse: true,
305 | 		},
306 | 		{
307 | 			name:                 "can put if allowed",
308 | 			proxy:                S3Proxy{Bucket: bucketName, EnablePut: true},
309 | 			method:               http.MethodPut,
310 | 			path:                 "/can-put",
311 | 			body:                 []byte("some content"),
312 | 			expectedCode:         http.StatusOK,
313 | 			expectsEmptyResponse: true,
314 | 		},
315 | 		{
316 | 			name:                 "serves index.html",
317 | 			proxy:                S3Proxy{Bucket: bucketName, IndexNames: []string{"index.html"}},
318 | 			method:               http.MethodGet,
319 | 			path:                 "/inner/",
320 | 			expectedCode:         http.StatusOK,
321 | 			expectedResponseText: "my index.html",
322 | 		},
323 | 		{
324 | 			name:   "returns 304 If-None-Match on index",
325 | 			proxy:  S3Proxy{Bucket: bucketName, IndexNames: []string{"index.html"}},
326 | 			method: http.MethodGet,
327 | 			path:   "/inner/",
328 | 			headers: http.Header{
329 | 				"If-None-Match": []string{`"44bacca965de5aef310706cc55c4a7b0"`},
330 | 			},
331 | 			expectedCode:         http.StatusNotModified,
332 | 			expectsEmptyResponse: true,
333 | 		},
334 | 		{
335 | 			name:                 "cannot browse",
336 | 			proxy:                S3Proxy{Bucket: bucketName},
337 | 			method:               http.MethodGet,
338 | 			path:                 "/inner/",
339 | 			expectedCode:         http.StatusForbidden,
340 | 			expectsEmptyResponse: true,
341 | 		},
342 | 		{
343 | 			name:         "returns 404 if not found",
344 | 			proxy:        S3Proxy{Bucket: bucketName},
345 | 			method:       http.MethodGet,
346 | 			path:         "/doesnt-exist",
347 | 			expectedCode: http.StatusNotFound,
348 | 		},
349 | 		{
350 | 			name: "returns 404 page if 404 error page is set",
351 | 			proxy: S3Proxy{
352 | 				Bucket:           bucketName,
353 | 				ErrorPages:       map[int]string{404: "_404.txt"},
354 | 				DefaultErrorPage: "default_error_page.txt",
355 | 			},
356 | 			method:               http.MethodGet,
357 | 			path:                 "/doesnt-exist",
358 | 			expectedCode:         http.StatusNotFound,
359 | 			expectedResponseText: `this is 404`,
360 | 		},
361 | 		{
362 | 			name: "returns default page if default error page is set",
363 | 			proxy: S3Proxy{
364 | 				Bucket:           bucketName,
365 | 				DefaultErrorPage: "default_error_page.txt",
366 | 			},
367 | 			method:               http.MethodGet,
368 | 			path:                 "/doesnt-exist",
369 | 			expectedCode:         http.StatusNotFound,
370 | 			expectedResponseText: `this is a default error page`,
371 | 		},
372 | 		{
373 | 			name:   "returns range",
374 | 			proxy:  S3Proxy{Bucket: bucketName},
375 | 			method: http.MethodGet,
376 | 			path:   "/test.json",
377 | 			headers: http.Header{
378 | 				"Range": []string{"bytes=0-4"},
379 | 			},
380 | 			expectedCode:         http.StatusOK,
381 | 			expectedResponseText: `{"foo`,
382 | 		},
383 | 		{
384 | 			name:   "returns 200 code If-Match",
385 | 			proxy:  S3Proxy{Bucket: bucketName},
386 | 			method: http.MethodGet,
387 | 			path:   "/test.json",
388 | 			headers: http.Header{
389 | 				"If-Match": []string{`"a38212e01d6f419c9bd303b304a99e9b"`},
390 | 			},
391 | 			expectedCode:         http.StatusOK,
392 | 			expectedResponseText: `{"foo": "bar"}`,
393 | 		},
394 | 		{
395 | 			name:   "returns 412 If-Match",
396 | 			proxy:  S3Proxy{Bucket: bucketName},
397 | 			method: http.MethodGet,
398 | 			path:   "/test.json",
399 | 			headers: http.Header{
400 | 				"If-Match": []string{`"no good etag"`},
401 | 			},
402 | 			expectedCode:         http.StatusPreconditionFailed,
403 | 			expectsEmptyResponse: true,
404 | 		},
405 | 		{
406 | 			name:   "returns 304 If-None-Match",
407 | 			proxy:  S3Proxy{Bucket: bucketName},
408 | 			method: http.MethodGet,
409 | 			path:   "/test.json",
410 | 			headers: http.Header{
411 | 				"If-None-Match": []string{`"a38212e01d6f419c9bd303b304a99e9b"`},
412 | 			},
413 | 			expectedCode:         http.StatusNotModified,
414 | 			expectsEmptyResponse: true,
415 | 		},
416 | 		{
417 | 			name:   "returns 200 If-None-Match",
418 | 			proxy:  S3Proxy{Bucket: bucketName},
419 | 			method: http.MethodGet,
420 | 			path:   "/test.json",
421 | 			headers: http.Header{
422 | 				"If-None-Match": []string{`"no good etag"`},
423 | 			},
424 | 			expectedCode:         http.StatusOK,
425 | 			expectedResponseText: `{"foo": "bar"}`,
426 | 		},
427 | 		{
428 | 			name:   "returns 200 If-Unmodified-Since",
429 | 			proxy:  S3Proxy{Bucket: bucketName},
430 | 			method: http.MethodGet,
431 | 			path:   "/test.json",
432 | 			headers: http.Header{
433 | 				"If-Unmodified-Since": []string{`Thu, 05 May 2568 07:28:00 GMT`},
434 | 			},
435 | 			expectedCode:         http.StatusOK,
436 | 			expectedResponseText: `{"foo": "bar"}`,
437 | 		},
438 | 		{
439 | 			name:   "returns 412 If-Unmodified-Since",
440 | 			proxy:  S3Proxy{Bucket: bucketName},
441 | 			method: http.MethodGet,
442 | 			path:   "/test.json",
443 | 			headers: http.Header{
444 | 				"If-Unmodified-Since": []string{`Wed, 21 Oct 2015 07:28:00 GMT`},
445 | 			},
446 | 			expectedCode:         http.StatusPreconditionFailed,
447 | 			expectsEmptyResponse: true,
448 | 		},
449 | 		{
450 | 			name:   "returns 200 If-Modified-Since",
451 | 			proxy:  S3Proxy{Bucket: bucketName},
452 | 			method: http.MethodGet,
453 | 			path:   "/test.json",
454 | 			headers: http.Header{
455 | 				"If-Modified-Since": []string{`Thu, 05 May 2568 07:28:00 GMT`},
456 | 			},
457 | 			expectedCode:         http.StatusNotModified,
458 | 			expectsEmptyResponse: true,
459 | 		},
460 | 		{
461 | 			name:   "returns 412 If-Modified-Since",
462 | 			proxy:  S3Proxy{Bucket: bucketName},
463 | 			method: http.MethodGet,
464 | 			path:   "/test.json",
465 | 			headers: http.Header{
466 | 				"If-Modified-Since": []string{`Wed, 21 Oct 2015 07:28:00 GMT`},
467 | 			},
468 | 			expectedCode:         http.StatusOK,
469 | 			expectedResponseText: `{"foo": "bar"}`,
470 | 		},
471 | 	} {
472 | 		t.Run(tc.name, func(t *testing.T) {
473 | 			var body io.Reader
474 | 			if tc.body != nil {
475 | 				body = bytes.NewReader(tc.body)
476 | 			}
477 | 
478 | 			req, err := http.NewRequest(tc.method, tc.path, body)
479 | 			if err != nil {
480 | 				t.Fatal(err)
481 | 			}
482 | 			repl := caddy.NewReplacer()
483 | 			ctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)
484 | 			req = req.WithContext(ctx)
485 | 			req.Header = tc.headers
486 | 
487 | 			recorder := httptest.NewRecorder()
488 | 
489 | 			tc.proxy.client = client
490 | 			tc.proxy.log = zap.NewExample()
491 | 
492 | 			_ = tc.proxy.ServeHTTP(recorder, req, nil)
493 | 
494 | 			// Check HTTP status code
495 | 			if tc.expectedCode != 0 && recorder.Code != tc.expectedCode {
496 | 				t.Errorf("Expected code %d, got %d.", tc.expectedCode, recorder.Code)
497 | 			}
498 | 
499 | 			// Check response headers
500 | 			respHeaders := recorder.Header()
501 | 			for k, v := range tc.expectedHeaders {
502 | 				if !reflect.DeepEqual(respHeaders.Values(k), v) {
503 | 					t.Errorf("Expected headers %v, got %v.", tc.expectedHeaders, respHeaders.Values(k))
504 | 				}
505 | 			}
506 | 
507 | 			// Check response body
508 | 			if tc.expectedResponseText != "" && tc.expectedResponseText != strings.TrimSpace(recorder.Body.String()) {
509 | 				t.Errorf(
510 | 					"Expected response text %s, got %s.",
511 | 					tc.expectedResponseText,
512 | 					recorder.Body.String(),
513 | 				)
514 | 			}
515 | 
516 | 			// Check if response should be empty
517 | 			if tc.expectsEmptyResponse && recorder.Body.Len() != 0 {
518 | 				t.Errorf("Expected response body to be empty, got %s.", recorder.Body.String())
519 | 			}
520 | 		})
521 | 	}
522 | }
523 | 


--------------------------------------------------------------------------------
/tag.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | BRANCH=$DRONE_COMMIT_BRANCH
 3 | if [ "$BRANCH" = "" ]; then
 4 | 	BRANCH=$(git rev-parse --abbrev-ref HEAD)
 5 | fi
 6 | 
 7 | 
 8 | calc_version () {
 9 | 	VERSION_FILE=$1
10 | 	DIR=$(dirname $VERSION_FILE)
11 | 	if [ "$DIR" = "." ]; then
12 | 		MODULE_PATH=
13 | 	else
14 | 		MODULE_PATH=$DIR/
15 | 	fi
16 | 
17 | 	echo $MODULE_PATH"v$(cat $VERSION_FILE)"
18 | }
19 | 
20 | if [ "$1" = "check_version" ]; then
21 | 	shift
22 | 	for var in "$@"
23 | 	do
24 | 		VERSION=$(calc_version $var)
25 | 		VER_EXIST=$(git tag -l $VERSION)
26 | 		echo $VER_EXIST
27 | 		if [ "$DRONE_COMMIT_BRANCH" != master ] && [ -n "$VER_EXIST" ]; then echo "Need to update $var - exiting" && exit 1; fi
28 | 	done
29 | 	exit
30 | fi
31 | 
32 | git checkout $BRANCH
33 | git pull origin $BRANCH
34 | 
35 | for var in "$@"
36 | do
37 | 	VERSION=$(calc_version $var)
38 | 
39 | 	if [ "$BRANCH" != "master" ]; then
40 | 		git describe --match "$VERSION-pre.*" --abbrev=0 HEAD --tags 2> /dev/null
41 | 		COUNTER=1
42 | 		while [  $COUNTER -lt 15 ]; do
43 | 			tag="$VERSION-pre.$COUNTER"
44 | 			git describe --match $tag --abbrev=0 HEAD --tags 2> /dev/null
45 | 			if [ $? != 0 ]; then
46 | 				break;
47 | 			fi
48 | 	        	let COUNTER=COUNTER+1 
49 | 		done
50 | 
51 | 		VERSION=$tag
52 | 	fi
53 | 	git tag $VERSION
54 | done
55 | 
56 | git push --tags origin $BRANCH 
57 | 
58 | 


--------------------------------------------------------------------------------
/testdata/_404.txt:
--------------------------------------------------------------------------------
1 | this is 404
2 | 


--------------------------------------------------------------------------------
/testdata/default_error_page.txt:
--------------------------------------------------------------------------------
1 | this is a default error page
2 | 


--------------------------------------------------------------------------------
/testdata/inner/index.html:
--------------------------------------------------------------------------------
1 | my index.html
2 | 


--------------------------------------------------------------------------------
/testdata/test.json:
--------------------------------------------------------------------------------
1 | {"foo": "bar"}
2 | 


--------------------------------------------------------------------------------
/testdata/to-delete.json:
--------------------------------------------------------------------------------
1 | {"foo": "bar"}
2 | 


--------------------------------------------------------------------------------