├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── build-rpm.sh ├── build-srpm.sh ├── config.json.dist ├── s3proxy.spec └── src └── s3proxy ├── config.go ├── countinghash.go ├── credentialcache.go ├── encryption.go ├── encryption_test.go ├── log.go ├── main.go ├── metadata.go └── proxy_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | *~ 3 | /bin 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 1.1.1 3 | sudo: false 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/abustany/s3proxy.svg?branch=master)](https://travis-ci.org/abustany/s3proxy) 2 | 3 | Simple authenticating proxy for AWS S3 4 | ====================================== 5 | 6 | This proxy allows any application supporting HTTP proxies to access files in a 7 | private S3 bucket (upload or download). The authorization headers are only sent 8 | if the proxy detects a S3 URL (of the form `*.s3.amazonaws.com/*`). Multiple 9 | buckets can be configured with different settings. 10 | 11 | The proxy supports fetching tokens from an IAM role, so you don't have to store 12 | the keys in clear text in the configuration file when running on an EC2 instance 13 | with a properly configured role. 14 | 15 | Transparent client-side AES encryption is supported. The size of your encryption 16 | key (16, 24, or 32 characters) will determine whether 128, 192 or 256 bit 17 | encryption is used. When encryption is used, files are encrypted on the fly 18 | during upload, and decrypted during download. Encryption keys are defined per 19 | bucket. 20 | 21 | The difference between client side encryption and the server side encryption 22 | also available in S3 is that with client side encryption, you keys are never 23 | stored on Amazon servers. 24 | 25 | Build 26 | ===== 27 | You'll need Go 1.1 to compile s3proxy. Note that the Go tools are only needed 28 | for compiling s3proxy, the resulting binary does not depend on any external 29 | libraries. 30 | 31 | - Export GOPATH to the root directory of s3proxy 32 | - Run go install s3proxy 33 | 34 | You should now have a s3proxy binary in bin/s3proxy 35 | 36 | Setup 37 | ===== 38 | - Copy config.json.dist to a file somewhere and edit the values inside 39 | - Start the proxy, passing the path to the config file as the only command line 40 | parameter 41 | 42 | Future 43 | ====== 44 | - Support wildcards in bucket configurations? 45 | -------------------------------------------------------------------------------- /build-rpm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | MYDIR="$(dirname $(readlink -m $0))" 6 | 7 | if [ -d "rpm" ]; then 8 | echo "A folder called rpm already exists, please delete it before running this script." 9 | exit 1 10 | fi 11 | 12 | mkdir rpm 13 | 14 | TOPDIR="$(readlink -m rpm)" 15 | 16 | rpmbuild -bb --define "%s3proxy_intree_build 1" --define "%_topdir $TOPDIR" --define "%_sourcedir $MYDIR" $MYDIR/s3proxy.spec 17 | 18 | echo "RPM built in $TOPDIR/RPMS" 19 | -------------------------------------------------------------------------------- /build-srpm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | MYDIR="$(dirname $(readlink -m $0))" 6 | 7 | GIT_DIR=$(cd $MYDIR && readlink -m $(git rev-parse --git-dir)) 8 | 9 | export GIT_DIR 10 | 11 | # Functions copied from the specfile 12 | latestcommitstamp() { 13 | git rev-list -n1 --format=format:%ct HEAD | tail -n1 14 | } 15 | 16 | releasestr() { 17 | echo $(date -u +%Y%m%dT%H%M%SZ --date=@$(latestcommitstamp)).git$(git rev-list --abbrev-commit -n1 HEAD) 18 | } 19 | 20 | if [ -d "rpm" ]; then 21 | echo "A folder called rpm already exists, please delete it before running this script." 22 | exit 1 23 | fi 24 | 25 | mkdir -p rpm/SOURCES rpm/SPECS 26 | 27 | RPMDIR="$(readlink -m rpm)" 28 | 29 | RELEASESTR=$(releasestr) 30 | FOLDER_NAME="s3proxy-${RELEASESTR}" 31 | 32 | git archive --format=tar --prefix="${FOLDER_NAME}/" HEAD | bzip2 -c > "$RPMDIR/SOURCES/${FOLDER_NAME}.tar.bz2" 33 | 34 | SPECFILE="$RPMDIR/SPECS/s3proxy.spec" 35 | 36 | echo -e "# AUTO GENERATED BY build-srpm.sh, DO NOT EDIT BY HAND\n" > $SPECFILE 37 | 38 | # We need to expand those macros now, since in the chroot build environment we 39 | # won't have access to git etc. 40 | sed \ 41 | -e "/^%define latestcommitstamp /d" \ 42 | -e "s/^%define releasestr .\\+/%define releasestr $RELEASESTR/" \ 43 | s3proxy.spec >> $SPECFILE 44 | 45 | rpmbuild -bs --define "%_topdir $RPMDIR" $SPECFILE 46 | 47 | echo "SRPM built in $RPMDIR/SRPMS/" 48 | -------------------------------------------------------------------------------- /config.json.dist: -------------------------------------------------------------------------------- 1 | { 2 | "Server": { 3 | "Address": "127.0.0.1", 4 | "Port": 18000, 5 | "DisableKeepAlives": false 6 | }, 7 | 8 | "Buckets": { 9 | "AWS bucket name": { 10 | "AccessKeyId": "", 11 | "SecretAccessKey": "" 12 | }, 13 | "AWS bucket name using IAM roles": { 14 | }, 15 | "AWS bucket name using IAM roles and encryption": { 16 | "EncryptionKey": "0123456789ABCDEF" 17 | }, 18 | "AWS bucket name using IAM roles and retrying once if a request fails (default is 0 - no retries)": { 19 | "RetryCount": 1 20 | }, 21 | "AWS bucket name using IAM roles and retrying once after 200 ms": { 22 | "RetryCount": 1, 23 | "RetryDelay": 200 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /s3proxy.spec: -------------------------------------------------------------------------------- 1 | # This specfile is a bit "weird" in the sense that it's meant to be run directly 2 | # off the git repository, not with a tarball in the RPM source dir. 3 | # Check the build-rpm.sh and build-srpm.sh scripts to see how to use it. 4 | 5 | # debuginfo not supported with Go 6 | %global debug_package %{nil} 7 | 8 | %define latestcommitstamp %(git rev-list -n1 --format=format:%ct HEAD | tail -n1) 9 | %define releasestr %(date -u +%Y%m%dT%H%M%SZ --date=@%{latestcommitstamp}).git%(git rev-list --abbrev-commit -n1 HEAD) 10 | 11 | %if %{?s3proxy_intree_build:%{s3proxy_intree_build}}%{!?s3proxy_intree_build:0} 12 | %define s3proxy_src_dir $RPM_SOURCE_DIR 13 | %else 14 | %define s3proxy_src_dir $(readlink -m .) 15 | %endif 16 | 17 | Name: s3proxy 18 | Version: 0 19 | Release: %{releasestr}%{?dist} 20 | Summary: HTTP proxy authenticating requests to AWS S3 buckets 21 | 22 | Group: Applications/Internet 23 | License: GPLv2 24 | URL: https://github.com/abustany/s3proxy 25 | 26 | BuildRequires: golang >= 1.1 27 | 28 | %if %{?s3proxy_intree_build:%{s3proxy_intree_build}}%{!?s3proxy_intree_build:0} 29 | # No need for a Source field since we're (ab)using $RPM_SOURCE_DIR 30 | %else 31 | Source0: s3proxy-%{releasestr}.tar.bz2 32 | %endif 33 | 34 | %description 35 | S3Proxy is an HTTP proxy that can be configured to authenticate requests to AWS 36 | S3 buckets. That allows applications to access private buckets as normal 37 | websites, without needing to know the API keys. 38 | 39 | %prep 40 | %if %{?s3proxy_intree_build:%{s3proxy_intree_build}}%{!?s3proxy_intree_build:0} 41 | %else 42 | %setup -q -n s3proxy-%{releasestr} 43 | %endif 44 | 45 | %build 46 | export GOPATH="%{s3proxy_src_dir}" 47 | go install s3proxy 48 | 49 | %check 50 | export GOPATH="%{s3proxy_src_dir}" 51 | go test s3proxy 52 | 53 | %install 54 | install -D -m 0755 %{s3proxy_src_dir}/bin/s3proxy $RPM_BUILD_ROOT%{_bindir}/s3proxy 55 | install -D -m 0644 %{s3proxy_src_dir}/config.json.dist \ 56 | $RPM_BUILD_ROOT%{_sysconfdir}/s3proxy/config.json.dist 57 | 58 | %files 59 | %defattr(-,root,root,-) 60 | %doc %{s3proxy_src_dir}/README.md %{s3proxy_src_dir}/LICENSE 61 | %{_sysconfdir}/s3proxy/config.json.dist 62 | %{_bindir}/s3proxy 63 | 64 | %changelog 65 | * Tue Jul 09 2013 Adrien Bustany - 0 66 | - Initial specfile 67 | -------------------------------------------------------------------------------- /src/s3proxy/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | ) 8 | 9 | type ServerConfig struct { 10 | Address string 11 | Port uint16 12 | DisableKeepAlives bool 13 | } 14 | 15 | type BucketConfig struct { 16 | AccessKeyId string 17 | SecretAccessKey string 18 | EncryptionKey string 19 | RetryCount int 20 | RetryDelay int 21 | } 22 | 23 | type Config struct { 24 | Server *ServerConfig 25 | Buckets map[string]*BucketConfig 26 | } 27 | 28 | func parseConfig(filename string) (*Config, error) { 29 | fd, err := os.Open(filename) 30 | 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | defer fd.Close() 36 | 37 | decoder := json.NewDecoder(fd) 38 | 39 | c := &Config{} 40 | 41 | err = decoder.Decode(c) 42 | 43 | if err != nil { 44 | return nil, err 45 | } 46 | 47 | if c.Server.Address == "" { 48 | return nil, fmt.Errorf("Missing config parameter Server.Address") 49 | } 50 | 51 | if c.Server.Port <= 0 { 52 | return nil, fmt.Errorf("Missing or invalid config parameter Server.Port") 53 | } 54 | 55 | for name, config := range c.Buckets { 56 | // Empty AccessKeyId means "use instance profile" 57 | if config.AccessKeyId != "" && config.SecretAccessKey == "" { 58 | return nil, fmt.Errorf("Missing config parameter SecretAccessKey for bucket '%s'", name) 59 | } 60 | 61 | if config.RetryCount < 0 { 62 | config.RetryCount = 0 63 | } 64 | 65 | if config.RetryDelay < 0 { 66 | config.RetryDelay = 0 67 | } 68 | } 69 | 70 | return c, nil 71 | } 72 | -------------------------------------------------------------------------------- /src/s3proxy/countinghash.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "hash" 5 | ) 6 | 7 | type CountingHash struct { 8 | h hash.Hash 9 | count uint64 10 | } 11 | 12 | func (h *CountingHash) Write(p []byte) (int, error) { 13 | n, err := h.h.Write(p) 14 | 15 | h.count += uint64(n) 16 | 17 | return n, err 18 | } 19 | 20 | func (h *CountingHash) Sum(b []byte) []byte { 21 | return h.h.Sum(b) 22 | } 23 | 24 | func (h *CountingHash) Size() int { 25 | return h.h.Size() 26 | } 27 | 28 | func (h *CountingHash) BlockSize() int { 29 | return h.h.BlockSize() 30 | } 31 | 32 | func (h *CountingHash) Count() uint64 { 33 | return h.count 34 | } 35 | 36 | func NewCountingHash(h hash.Hash) *CountingHash { 37 | return &CountingHash{ 38 | h, 39 | 0, 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/s3proxy/credentialcache.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | type CredentialCache struct { 13 | client *http.Client 14 | cache map[string]*Credentials 15 | instanceRole string 16 | cacheMutex sync.Mutex 17 | } 18 | 19 | func NewCredentialCache() *CredentialCache { 20 | return &CredentialCache{ 21 | client: &http.Client{}, 22 | cache: make(map[string]*Credentials), 23 | } 24 | } 25 | 26 | var AmzIAMEndpoint = "http://169.254.169.254/latest/meta-data/iam/security-credentials/" 27 | 28 | func (c *CredentialCache) discoverInstanceRole() (string, error) { 29 | rsp, err := c.client.Get(AmzIAMEndpoint) 30 | 31 | if err != nil { 32 | return "", fmt.Errorf("Cannot make HTTP request: %s", err) 33 | } 34 | 35 | if rsp.StatusCode != http.StatusOK { 36 | return "", fmt.Errorf("Unexpected HTTP status code: %s", rsp.Status) 37 | } 38 | 39 | data, err := ioutil.ReadAll(rsp.Body) 40 | 41 | if err != nil { 42 | return "", fmt.Errorf("Error while reading HTTP response body: %s", err) 43 | } 44 | 45 | return string(data), nil 46 | } 47 | 48 | func (c *CredentialCache) FetchRoleCredentials(role string) (*Credentials, error) { 49 | url := AmzIAMEndpoint + role 50 | 51 | resp, err := c.client.Get(url) 52 | 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | defer resp.Body.Close() 58 | 59 | if resp.StatusCode != http.StatusOK { 60 | return nil, fmt.Errorf("Error response from the IAM endpoint: %s", resp.Status) 61 | } 62 | 63 | payload := struct { 64 | Code string 65 | LastUpdated string 66 | AccessKeyId string 67 | SecretAccessKey string 68 | Token string 69 | Expiration string 70 | }{} 71 | 72 | decoder := json.NewDecoder(resp.Body) 73 | 74 | err = decoder.Decode(&payload) 75 | 76 | if err != nil { 77 | return nil, err 78 | } 79 | 80 | expiration, err := time.Parse(time.RFC3339, payload.Expiration) 81 | 82 | if err != nil { 83 | return nil, err 84 | } 85 | 86 | return &Credentials{ 87 | payload.AccessKeyId, 88 | payload.SecretAccessKey, 89 | payload.Token, 90 | expiration, 91 | }, nil 92 | } 93 | 94 | func (c *CredentialCache) GetRoleCredentials() (*Credentials, error) { 95 | c.cacheMutex.Lock() 96 | defer c.cacheMutex.Unlock() 97 | 98 | if c.instanceRole == "" { 99 | var err error 100 | 101 | c.instanceRole, err = c.discoverInstanceRole() 102 | 103 | if err != nil { 104 | return nil, fmt.Errorf("Error while fetching instance role: %s", err) 105 | } 106 | } 107 | 108 | credentials := c.cache[c.instanceRole] 109 | 110 | if credentials == nil || time.Now().After(credentials.Expiration) { 111 | var err error 112 | 113 | credentials, err = c.FetchRoleCredentials(c.instanceRole) 114 | 115 | if err != nil { 116 | return nil, err 117 | } 118 | 119 | c.cache[c.instanceRole] = credentials 120 | } 121 | 122 | return credentials, nil 123 | } 124 | -------------------------------------------------------------------------------- /src/s3proxy/encryption.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "crypto/aes" 6 | "crypto/cipher" 7 | "crypto/rand" 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | ) 12 | 13 | func SetupCipher(info *BucketInfo, ivReader io.Reader) (cipher.Block, []byte, error) { 14 | block, err := aes.NewCipher([]byte(info.Config.EncryptionKey)) 15 | 16 | if err != nil { 17 | return nil, nil, err 18 | } 19 | 20 | iv := make([]byte, block.BlockSize()) 21 | n, err := io.ReadFull(ivReader, iv) 22 | 23 | if n != len(iv) || err != nil { 24 | return nil, nil, fmt.Errorf("Cannot build IV: %s", err) 25 | } 26 | 27 | return block, iv, nil 28 | } 29 | 30 | func SetupReadEncryption(input io.Reader, info *BucketInfo) (io.ReadCloser, int64, error) { 31 | block, iv, err := SetupCipher(info, input) 32 | 33 | if err != nil { 34 | return nil, -1, err 35 | } 36 | 37 | decrypter := cipher.NewCFBDecrypter(block, iv[:]) 38 | 39 | reader := &cipher.StreamReader{ 40 | S: decrypter, 41 | R: input, 42 | } 43 | 44 | return ioutil.NopCloser(reader), int64(len(iv)), nil 45 | } 46 | 47 | func SetupWriteEncryption(input io.Reader, info *BucketInfo) (io.ReadCloser, int64, error) { 48 | block, iv, err := SetupCipher(info, rand.Reader) 49 | 50 | if err != nil { 51 | return nil, -1, err 52 | } 53 | 54 | ivReader := bytes.NewReader(iv) 55 | encrypter := cipher.NewCFBEncrypter(block, iv[:]) 56 | 57 | reader := &cipher.StreamReader{ 58 | S: encrypter, 59 | R: input, 60 | } 61 | 62 | return ioutil.NopCloser(io.MultiReader(ivReader, reader)), int64(len(iv)), nil 63 | } 64 | -------------------------------------------------------------------------------- /src/s3proxy/encryption_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | func byteArrayEquals(a []byte, b []byte) bool { 11 | if len(a) != len(b) { 12 | return false 13 | } 14 | 15 | equals := true 16 | 17 | for i := range a { 18 | if a[i] != b[i] { 19 | equals = false 20 | break 21 | } 22 | } 23 | 24 | return equals 25 | } 26 | 27 | func TestEncryption(t *testing.T) { 28 | bucketInfo := &BucketInfo{ 29 | Config: &BucketConfig{ 30 | EncryptionKey: "0123456789ABCDEF", 31 | }, 32 | } 33 | 34 | bucketInfoWrongKey := &BucketInfo{ 35 | Config: &BucketConfig{ 36 | EncryptionKey: "WRONG_KEY_OH_NO!", 37 | }, 38 | } 39 | 40 | payload := "The privilege of absurdity; to which no living creature is subject but man only." 41 | 42 | const NEncryptionRounds = 3 43 | encryptedPayloads := make([][]byte, NEncryptionRounds) 44 | 45 | // We use a random IV, so we should get different encrypted payloads for 46 | // each round 47 | for i := 0; i < NEncryptionRounds; i++ { 48 | // Encrypt 49 | encReader, _, err := SetupWriteEncryption(strings.NewReader(payload), bucketInfo) 50 | 51 | if err != nil { 52 | t.Fatalf("Error while setting up encryption: %s", err) 53 | } 54 | 55 | encryptedPayloads[i], err = ioutil.ReadAll(encReader) 56 | 57 | if err != nil { 58 | t.Fatalf("Error while encrypting data: %s", err) 59 | } 60 | 61 | encReader.Close() 62 | 63 | if i > 0 && byteArrayEquals(encryptedPayloads[i-1], encryptedPayloads[i]) { 64 | t.Fatalf("Two identical encrypted payloads for two different encryption rounds") 65 | } 66 | 67 | // Decrypt with correct key 68 | decReader, _, err := SetupReadEncryption(bytes.NewReader(encryptedPayloads[i]), bucketInfo) 69 | 70 | if err != nil { 71 | t.Fatalf("Error while setting up decryption: %s", err) 72 | } 73 | 74 | decryptedPayload, err := ioutil.ReadAll(decReader) 75 | 76 | if err != nil { 77 | t.Fatalf("Error while decrypting data: %s", err) 78 | } 79 | 80 | decReader.Close() 81 | 82 | if string(decryptedPayload) != payload { 83 | t.Fatalf("Decrypted payload does not match original (decrypted: '%s' original: '%s')", string(decryptedPayload), payload) 84 | } 85 | 86 | // Decrypt with incorrect key 87 | decReader, _, err = SetupReadEncryption(bytes.NewReader(encryptedPayloads[i]), bucketInfoWrongKey) 88 | 89 | if err != nil { 90 | t.Fatalf("Error while setting up decryption: %s", err) 91 | } 92 | 93 | decryptedPayload, err = ioutil.ReadAll(decReader) 94 | 95 | if err != nil { 96 | t.Fatalf("Error while decrypting data: %s", err) 97 | } 98 | 99 | decReader.Close() 100 | 101 | if string(decryptedPayload) == payload { 102 | t.Fatalf("Getting correct payload while decrypting with wrong key!") 103 | } 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/s3proxy/log.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | ) 9 | 10 | var InfoLogger *log.Logger 11 | var ErrorLogger *log.Logger 12 | 13 | const LogInfoPrefix = "INFO " 14 | const LogErrorPrefix = "ERROR " 15 | 16 | func makeLogger(output io.Writer, prefix string) *log.Logger { 17 | return log.New(output, prefix, log.LstdFlags|log.Lshortfile) 18 | } 19 | 20 | func init() { 21 | InfoLogger = makeLogger(ioutil.Discard, LogInfoPrefix) 22 | ErrorLogger = makeLogger(os.Stderr, LogErrorPrefix) 23 | } 24 | 25 | func enableDebugMode(enable bool) { 26 | var w = ioutil.Discard 27 | 28 | if enable { 29 | w = os.Stdout 30 | } 31 | 32 | InfoLogger = makeLogger(w, LogInfoPrefix) 33 | } 34 | -------------------------------------------------------------------------------- /src/s3proxy/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "crypto/hmac" 6 | "crypto/md5" 7 | "crypto/sha1" 8 | "encoding/base64" 9 | "flag" 10 | "fmt" 11 | "io" 12 | "io/ioutil" 13 | "net/http" 14 | "os" 15 | "sort" 16 | "strconv" 17 | "strings" 18 | "time" 19 | ) 20 | 21 | func usage() { 22 | fmt.Fprintf(os.Stderr, "Usage: %s CONFIG_FILE\n", os.Args[0]) 23 | fmt.Fprintf(os.Stderr, "HTTP proxy that authenticates S3 requests\n") 24 | } 25 | 26 | type Credentials struct { 27 | AccessKeyId string 28 | SecretAccessKey string 29 | Token string 30 | Expiration time.Time 31 | } 32 | 33 | type BucketInfo struct { 34 | Name string 35 | VirtualHost bool 36 | Config *BucketConfig 37 | } 38 | 39 | type ProxyHandler struct { 40 | config *Config 41 | client *http.Client 42 | credentialCache *CredentialCache 43 | } 44 | 45 | const S3ProxyMetadataHeader = "X-Amz-Meta-S3proxy" 46 | const S3ProxyMetadataVersion = byte(0x00) 47 | 48 | func (h *ProxyHandler) GetBucketSecurityCredentials(c *BucketConfig) (*Credentials, error) { 49 | if c.AccessKeyId != "" { 50 | return &Credentials{ 51 | AccessKeyId: c.AccessKeyId, 52 | SecretAccessKey: c.SecretAccessKey, 53 | }, nil 54 | } 55 | 56 | return h.credentialCache.GetRoleCredentials() 57 | } 58 | 59 | var AwsDomain = "s3.amazonaws.com" 60 | 61 | func (h *ProxyHandler) GetBucketInfo(r *http.Request) *BucketInfo { 62 | var portIdx = strings.IndexRune(r.Host, ':') 63 | 64 | if portIdx == -1 { 65 | portIdx = len(r.Host) 66 | } 67 | 68 | host := r.Host[0:portIdx] 69 | 70 | if !strings.HasSuffix(host, AwsDomain) { 71 | return nil 72 | } 73 | 74 | var bucketName string 75 | // Whether the URL was using bucket.s3.amazonaws.com instead of s3.amazonaws.com/bucket/ 76 | var bucketVirtualHost = false 77 | 78 | if len(host) > len(AwsDomain) { 79 | bucketName = host[0 : len(host)-len(AwsDomain)-1] 80 | bucketVirtualHost = true 81 | } else { 82 | tokens := strings.Split(r.URL.Path, "/") 83 | 84 | // Split produces empty tokens which we are not interested in 85 | for _, t := range tokens { 86 | if t == "" { 87 | continue 88 | } 89 | 90 | bucketName = t 91 | break 92 | } 93 | } 94 | 95 | return &BucketInfo{ 96 | Name: bucketName, 97 | VirtualHost: bucketVirtualHost, 98 | Config: h.config.Buckets[bucketName], 99 | } 100 | } 101 | 102 | func (h *ProxyHandler) PreRequestEncryptionHook(r *http.Request, innerRequest *http.Request, info *BucketInfo) (*CountingHash, error) { 103 | if info == nil || info.Config == nil || info.Config.EncryptionKey == "" || r.Method != "PUT" { 104 | return nil, nil 105 | } 106 | 107 | // If this is a "copy" PUT, we should send no body at all 108 | for k, _ := range r.Header { 109 | if strings.HasPrefix(strings.ToLower(k), "x-amz-copy-source") { 110 | return nil, nil 111 | } 112 | } 113 | 114 | encryptedInput, extralen, err := SetupWriteEncryption(r.Body, info) 115 | 116 | if err != nil { 117 | return nil, err 118 | } 119 | 120 | // Since encryption transforms the data, after the inner request succeeds, 121 | // we'll match the MD5s of the transformed data, and mangle the etag in the 122 | // response we send to the client with the MD5 of the untransformed data if 123 | // they match. 124 | innerBodyHash := NewCountingHash(md5.New()) 125 | teereader := io.TeeReader(encryptedInput, innerBodyHash) 126 | innerRequest.Body = ioutil.NopCloser(teereader) 127 | 128 | if length := innerRequest.ContentLength; length != -1 { 129 | innerRequest.ContentLength += extralen 130 | innerRequest.Header.Set("Content-Length", strconv.FormatInt(innerRequest.ContentLength, 10)) 131 | } 132 | 133 | InfoLogger.Print("Encrypting the request") 134 | 135 | return innerBodyHash, nil 136 | } 137 | 138 | func (h *ProxyHandler) PostRequestEncryptionHook(r *http.Request, innerResponse *http.Response, info *BucketInfo) (io.ReadCloser, error) { 139 | if info == nil || info.Config == nil || info.Config.EncryptionKey == "" { 140 | return innerResponse.Body, nil 141 | } 142 | 143 | if r.Method != "GET" && r.Method != "HEAD" { 144 | return innerResponse.Body, nil 145 | } 146 | 147 | if innerResponse.StatusCode >= 300 { 148 | return innerResponse.Body, nil 149 | } 150 | 151 | // When listing folders, the returned data is not going to be encrypted 152 | if strings.HasSuffix(r.URL.Path, "/") { 153 | InfoLogger.Print("Directory listing request, skipping decryption") 154 | return innerResponse.Body, nil 155 | } 156 | 157 | InfoLogger.Print("Decrypting the response") 158 | 159 | // If we had cached encrypted metadata, decrypt it and return it to the client 160 | if encryptedMetadata := innerResponse.Header.Get(S3ProxyMetadataHeader); encryptedMetadata != "" { 161 | var metadataBytes []byte 162 | _, err := fmt.Sscanf(encryptedMetadata, "%x", &metadataBytes) 163 | 164 | if err != nil { 165 | return nil, err 166 | } 167 | 168 | decReader, _, err := SetupReadEncryption(bytes.NewReader(metadataBytes), info) 169 | 170 | if err != nil { 171 | return nil, err 172 | } 173 | 174 | metadata, err := UnserializeObjectMetadata(decReader) 175 | 176 | if err != nil { 177 | return nil, err 178 | } 179 | 180 | delete(innerResponse.Header, S3ProxyMetadataHeader) 181 | innerResponse.Header.Set("Etag", metadata.Etag) 182 | innerResponse.Header.Set("Content-Length", fmt.Sprintf("%d", metadata.Size)) 183 | 184 | InfoLogger.Printf("Overwrote the response headers with the cached version (Etag: %s, Content-Length: %d)", metadata.Etag, metadata.Size) 185 | } 186 | 187 | if r.Method == "HEAD" { 188 | return innerResponse.Body, nil 189 | } 190 | 191 | decryptedReader, minuslen, err := SetupReadEncryption(innerResponse.Body, info) 192 | 193 | if err != nil { 194 | return nil, err 195 | } 196 | 197 | if length := innerResponse.ContentLength; length != -1 { 198 | innerResponse.ContentLength -= minuslen 199 | innerResponse.Header.Set("Content-Length", strconv.FormatInt(innerResponse.ContentLength, 10)) 200 | } 201 | 202 | return decryptedReader, nil 203 | } 204 | 205 | func (h *ProxyHandler) SignRequest(r *http.Request, info *BucketInfo) error { 206 | // See http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#ConstructingTheAuthenticationHeader 207 | 208 | if info == nil || info.Config == nil { 209 | return nil 210 | } 211 | credentials, err := h.GetBucketSecurityCredentials(info.Config) 212 | 213 | if err != nil { 214 | return err 215 | } 216 | 217 | dateStr := r.Header.Get("Date") 218 | 219 | if dateStr == "" && r.Header.Get("x-amz-date") == "" { 220 | dateStr = time.Now().UTC().Format(time.RFC1123Z) 221 | r.Header.Set("Date", dateStr) 222 | } 223 | 224 | if credentials.Token != "" { 225 | r.Header.Add("x-amz-security-token", credentials.Token) 226 | } 227 | 228 | canonicalizedResource := bytes.NewBuffer(nil) 229 | 230 | if info.VirtualHost { 231 | canonicalizedResource.WriteString("/" + info.Name) 232 | } 233 | 234 | canonicalizedResource.WriteString(r.URL.Path) 235 | 236 | canonicalizedAmzHeaders := bytes.NewBuffer(nil) 237 | 238 | amzHeaders := []string{} 239 | 240 | for k, _ := range r.Header { 241 | if !strings.HasPrefix(strings.ToLower(k), "x-amz-") { 242 | continue 243 | } 244 | 245 | amzHeaders = append(amzHeaders, k) 246 | } 247 | 248 | sort.Strings(amzHeaders) 249 | 250 | for _, k := range amzHeaders { 251 | canonicalizedAmzHeaders.WriteString(strings.ToLower(k)) 252 | canonicalizedAmzHeaders.WriteString(":") 253 | canonicalizedAmzHeaders.WriteString(strings.Join(r.Header[k], ",")) 254 | canonicalizedAmzHeaders.WriteString("\n") 255 | } 256 | 257 | buf := bytes.NewBuffer(nil) 258 | 259 | buf.WriteString(r.Method) 260 | buf.WriteString("\n") 261 | 262 | buf.WriteString(r.Header.Get("Content-MD5")) 263 | buf.WriteString("\n") 264 | 265 | buf.WriteString(r.Header.Get("Content-Type")) 266 | buf.WriteString("\n") 267 | 268 | buf.WriteString(dateStr) 269 | buf.WriteString("\n") 270 | 271 | buf.WriteString(canonicalizedAmzHeaders.String()) 272 | buf.WriteString(canonicalizedResource.String()) 273 | 274 | signature := hmac.New(sha1.New, ([]byte)(credentials.SecretAccessKey)) 275 | signature.Write(buf.Bytes()) 276 | 277 | signature64 := bytes.NewBuffer(nil) 278 | 279 | b64encoder := base64.NewEncoder(base64.StdEncoding, signature64) 280 | b64encoder.Write(signature.Sum(nil)) 281 | b64encoder.Close() 282 | 283 | signatureHdr := fmt.Sprintf("AWS %s:%s", credentials.AccessKeyId, signature64.String()) 284 | 285 | r.Header.Set("Authorization", signatureHdr) 286 | 287 | InfoLogger.Printf("Signed request (signature: %s )", signatureHdr) 288 | 289 | return nil 290 | } 291 | 292 | func failRequest(w http.ResponseWriter, format string, args ...interface{}) { 293 | const ErrorFooter = "\n\nGreetings, the S3Proxy\n" 294 | 295 | w.WriteHeader(http.StatusInternalServerError) 296 | fmt.Fprintf(w, format + ErrorFooter, args...) 297 | ErrorLogger.Printf(format, args...) 298 | } 299 | 300 | func (h *ProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 301 | InfoLogger.Printf("%s %s (Host: %s)", r.Method, r.URL, r.Host) 302 | 303 | info := h.GetBucketInfo(r) 304 | 305 | if info == nil { 306 | InfoLogger.Print("Not an S3 request") 307 | } else { 308 | if info.Config == nil { 309 | InfoLogger.Printf("No configuration for S3 bucket %s", info.Name) 310 | } else { 311 | InfoLogger.Printf("Handling request for bucket %s", info.Name) 312 | } 313 | } 314 | 315 | innerRequest := &http.Request{ 316 | Method: r.Method, 317 | URL: r.URL, 318 | Proto: r.Proto, 319 | ProtoMajor: r.ProtoMajor, 320 | ProtoMinor: r.ProtoMinor, 321 | Header: r.Header, 322 | Body: r.Body, 323 | ContentLength: r.ContentLength, 324 | TransferEncoding: r.TransferEncoding, 325 | Close: r.Close, 326 | Host: r.Host, 327 | Form: r.Form, 328 | PostForm: r.PostForm, 329 | MultipartForm: r.MultipartForm, 330 | Trailer: r.Trailer, 331 | } 332 | 333 | innerRequest.URL.Scheme = "http" 334 | innerRequest.URL.Host = r.Host 335 | 336 | var originalBodyHash *CountingHash 337 | 338 | dataCheckNeeded := r.Method == "PUT" && info != nil 339 | 340 | if dataCheckNeeded { 341 | originalBodyHash = NewCountingHash(md5.New()) 342 | 343 | teereader := io.TeeReader(r.Body, originalBodyHash) 344 | r.Body = ioutil.NopCloser(teereader) 345 | } 346 | 347 | err := h.SignRequest(innerRequest, info) 348 | 349 | if err != nil { 350 | failRequest(w, "Error while signing the request: %s", err) 351 | return 352 | } 353 | 354 | innerBodyHash, err := h.PreRequestEncryptionHook(r, innerRequest, info) 355 | 356 | if err != nil { 357 | failRequest(w, "Error while setting up encryption: %s", err) 358 | return 359 | } 360 | 361 | maxRetryCount := 0 362 | retryCount := 0 363 | retryDelay := 0 364 | 365 | if info != nil && info.Config != nil { 366 | maxRetryCount = info.Config.RetryCount 367 | retryDelay = info.Config.RetryDelay 368 | } 369 | 370 | var innerResponse *http.Response 371 | 372 | for { 373 | innerResponse, err = h.client.Do(innerRequest) 374 | 375 | if err == nil && innerResponse.StatusCode < 300 { 376 | break 377 | } 378 | 379 | requestFailed := (err != nil || innerResponse.StatusCode >= http.StatusInternalServerError) 380 | 381 | if retryCount < maxRetryCount && requestFailed { 382 | InfoLogger.Printf("Request to %s failed, retrying after %d ms (try %d out of %d)", innerRequest.URL, retryDelay, 1+retryCount, 1+maxRetryCount) 383 | 384 | retryCount++ 385 | time.Sleep(time.Duration(retryDelay) * time.Millisecond) 386 | continue 387 | } 388 | 389 | if err != nil { 390 | failRequest(w, "Error while serving the request: %s", err) 391 | return 392 | } 393 | 394 | // We had a 5xx response, but no error from the HTTP client: just 395 | // forward the response, that will get to the client 396 | 397 | // Do not try to update the metadata if the request failed 398 | dataCheckNeeded = false 399 | 400 | break 401 | } 402 | 403 | defer func() { 404 | if innerResponse != nil && innerResponse.Body != nil { 405 | innerResponse.Body.Close() 406 | } 407 | }() 408 | 409 | if dataCheckNeeded { 410 | awsEtag := innerResponse.Header.Get("Etag") 411 | 412 | bodyHash := innerBodyHash 413 | 414 | if bodyHash == nil { 415 | bodyHash = originalBodyHash 416 | } 417 | 418 | innerEtag := fmt.Sprintf("\"%.0x\"", bodyHash.Sum(nil)) 419 | originalEtag := fmt.Sprintf("\"%.0x\"", originalBodyHash.Sum(nil)) 420 | 421 | // if the Etags don't match, we can leave whatever value there 422 | if innerEtag == awsEtag { 423 | innerResponse.Header["Etag"] = []string{originalEtag} 424 | } 425 | 426 | // Let's also store the original metadata in S3, so we can use it later 427 | // for HEAD and GET requests (if we uploaded any data). We encrypt the 428 | // metadata too. 429 | if innerBodyHash != nil { 430 | metadata := &ObjectMetadata{ 431 | originalBodyHash.Count(), 432 | originalEtag, 433 | } 434 | 435 | err = h.UpdateObjectMetadata(innerRequest.URL, metadata, r.Header, info) 436 | 437 | if err != nil { 438 | failRequest(w, "Error while updating metadata: %s", err) 439 | return 440 | } 441 | } 442 | } 443 | 444 | responseReader, err := h.PostRequestEncryptionHook(r, innerResponse, info) 445 | 446 | if err != nil { 447 | failRequest(w, "Error while setting up decryption: %s", err) 448 | return 449 | } 450 | 451 | for k, vs := range innerResponse.Header { 452 | for _, v := range vs { 453 | w.Header().Add(k, v) 454 | } 455 | } 456 | 457 | w.WriteHeader(innerResponse.StatusCode) 458 | io.Copy(w, responseReader) 459 | } 460 | 461 | func NewProxyHandler(config *Config) *ProxyHandler { 462 | transport := &http.Transport{ 463 | Proxy: http.ProxyFromEnvironment, 464 | DisableKeepAlives: config.Server.DisableKeepAlives, 465 | } 466 | 467 | return &ProxyHandler{ 468 | config, 469 | &http.Client{ 470 | Transport: transport, 471 | }, 472 | NewCredentialCache(), 473 | } 474 | } 475 | 476 | func main() { 477 | var debugMode = flag.Bool("debug", false, "Enable debug messages") 478 | 479 | flag.Parse() 480 | 481 | if len(flag.Args()) != 1 { 482 | usage() 483 | os.Exit(1) 484 | } 485 | 486 | if *debugMode { 487 | enableDebugMode(*debugMode) 488 | InfoLogger.Print("Enabling debug messages") 489 | } 490 | 491 | config, err := parseConfig(flag.Args()[0]) 492 | 493 | if err != nil { 494 | ErrorLogger.Printf("Error while parsing the configuration file: %s\n", err) 495 | os.Exit(1) 496 | } 497 | 498 | handler := NewProxyHandler(config) 499 | 500 | listenAddress := fmt.Sprintf("%s:%d", config.Server.Address, config.Server.Port) 501 | http.ListenAndServe(listenAddress, handler) 502 | } 503 | -------------------------------------------------------------------------------- /src/s3proxy/metadata.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/binary" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "net/http" 11 | "net/url" 12 | "strings" 13 | ) 14 | 15 | type ObjectMetadata struct { 16 | Size uint64 17 | Etag string 18 | } 19 | 20 | func SerializeObjectMetadata(m *ObjectMetadata, w io.Writer) error { 21 | _, err := w.Write([]byte{S3ProxyMetadataVersion}) 22 | 23 | if err != nil { 24 | return err 25 | } 26 | 27 | encodedSize := make([]byte, 8) 28 | n := binary.PutUvarint(encodedSize, m.Size) 29 | 30 | _, err = w.Write(encodedSize[0:n]) 31 | 32 | if err != nil { 33 | return err 34 | } 35 | 36 | _, err = io.Copy(w, strings.NewReader(m.Etag)) 37 | 38 | return err 39 | } 40 | 41 | func UnserializeObjectMetadata(r io.Reader) (*ObjectMetadata, error) { 42 | bufReader := bufio.NewReader(r) 43 | 44 | metadataVersion := make([]byte, 1) 45 | 46 | if n, err := bufReader.Read(metadataVersion); err != nil || n != len(metadataVersion) { 47 | return nil, fmt.Errorf("Cannot read metadata version: %s", err) 48 | } 49 | 50 | if metadataVersion[0] != S3ProxyMetadataVersion { 51 | return nil, fmt.Errorf("Invalid metadata version: %x", metadataVersion) 52 | } 53 | 54 | size, err := binary.ReadUvarint(bufReader) 55 | 56 | if err != nil { 57 | return nil, err 58 | } 59 | 60 | etag, err := ioutil.ReadAll(bufReader) 61 | 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | return &ObjectMetadata{ 67 | size, 68 | string(etag), 69 | }, nil 70 | } 71 | 72 | func (h *ProxyHandler) UpdateObjectMetadata(objectUrl *url.URL, metadata *ObjectMetadata, originalHeaders http.Header, info *BucketInfo) error { 73 | if info == nil || info.Config == nil { 74 | return nil 75 | } 76 | 77 | serializedMetadata := bytes.NewBuffer(nil) 78 | 79 | err := SerializeObjectMetadata(metadata, serializedMetadata) 80 | 81 | if err != nil { 82 | return err 83 | } 84 | 85 | encReader, _, err := SetupWriteEncryption(bytes.NewReader(serializedMetadata.Bytes()), info) 86 | 87 | if err != nil { 88 | return err 89 | } 90 | 91 | encryptedMetadata, err := ioutil.ReadAll(encReader) 92 | 93 | if err != nil { 94 | return err 95 | } 96 | 97 | requestUrl := *objectUrl 98 | 99 | var objectPath string 100 | 101 | if info.VirtualHost { 102 | objectPath = info.Name + requestUrl.Path 103 | } else { 104 | objectPath = requestUrl.Path 105 | } 106 | 107 | metadataRequest, err := http.NewRequest("PUT", objectUrl.String(), nil) 108 | 109 | if err != nil { 110 | return err 111 | } 112 | 113 | metadataRequest.Header.Set("x-amz-copy-source", objectPath) 114 | metadataRequest.Header.Set("x-amz-metadata-directive", "REPLACE") 115 | metadataRequest.Header.Set(S3ProxyMetadataHeader, fmt.Sprintf("%.0x", encryptedMetadata)) 116 | 117 | for k, vs := range originalHeaders { 118 | if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") || k == S3ProxyMetadataHeader { 119 | continue 120 | } 121 | 122 | metadataRequest.Header[k] = vs 123 | } 124 | 125 | err = h.SignRequest(metadataRequest, info) 126 | 127 | if err != nil { 128 | return err 129 | } 130 | 131 | metadataResponse, err := h.client.Do(metadataRequest) 132 | 133 | if err != nil { 134 | return err 135 | } 136 | 137 | if metadataResponse.StatusCode != http.StatusOK { 138 | return fmt.Errorf("Unexpected HTTP status: %s", metadataResponse.Status) 139 | } 140 | 141 | defer metadataResponse.Body.Close() 142 | 143 | return nil 144 | } 145 | -------------------------------------------------------------------------------- /src/s3proxy/proxy_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net" 7 | "net/http" 8 | "net/url" 9 | "strings" 10 | "sync/atomic" 11 | "testing" 12 | "time" 13 | ) 14 | 15 | var testConfig = &Config{ 16 | &ServerConfig{}, 17 | map[string]*BucketConfig{ 18 | "testbucket": { 19 | "AccessKey", 20 | "SecretKey", 21 | "", 22 | 0, 23 | 0, 24 | }, 25 | "testbucket2": { 26 | "AccessKey", 27 | "SecretKey2", 28 | "", 29 | 1, 30 | 0, 31 | }, 32 | "testbucket3": { 33 | "", 34 | "", 35 | "", 36 | 1, 37 | 0, 38 | }, 39 | "testbucket4": { 40 | "AccessKey", 41 | "SecretKey2", 42 | "", 43 | 1, 44 | 300, 45 | }, 46 | }, 47 | } 48 | 49 | type IAMServer struct { 50 | Port int 51 | RequestCount int32 52 | Creds map[string]*Credentials 53 | l net.Listener 54 | } 55 | 56 | // Fails one out of two requests 57 | type FailingServer struct { 58 | Port int 59 | RequestCount int32 60 | l net.Listener 61 | } 62 | 63 | func GetListeningAddress() (net.Listener, int) { 64 | var l net.Listener 65 | 66 | addr := &net.TCPAddr{ 67 | IP: net.IPv4(127, 0, 0, 1), 68 | Port: 0, // random port 69 | } 70 | 71 | l, err := net.ListenTCP("tcp", addr) 72 | 73 | if err != nil { 74 | panic(fmt.Sprintf("Could not start TCP listener: %s", err)) 75 | } 76 | 77 | tcpaddr, err := net.ResolveTCPAddr("tcp", l.Addr().String()) 78 | 79 | if err != nil { 80 | panic(fmt.Sprintf("Could not parse TCP address: %s", err)) 81 | } 82 | 83 | return l, tcpaddr.Port 84 | } 85 | 86 | func NewIAMServer() *IAMServer { 87 | listener, port := GetListeningAddress() 88 | 89 | server := &IAMServer{ 90 | port, 91 | 0, 92 | make(map[string]*Credentials), 93 | listener, 94 | } 95 | 96 | go http.Serve(listener, server) 97 | 98 | // There is no way to ensure the HTTP server properly started :/ 99 | time.Sleep(50 * time.Millisecond) 100 | 101 | return server 102 | } 103 | 104 | func (s *IAMServer) Close() { 105 | s.l.Close() 106 | } 107 | 108 | func (s *IAMServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { 109 | var pathPrefix = "/latest/meta-data/iam/security-credentials/" 110 | 111 | if !strings.HasPrefix(r.URL.Path, pathPrefix) { 112 | w.WriteHeader(http.StatusNotFound) 113 | return 114 | } 115 | 116 | roleName := r.URL.Path[len(pathPrefix):] 117 | 118 | // Discovery request 119 | if roleName == "" { 120 | role := "testrole" 121 | w.WriteHeader(http.StatusOK) 122 | w.Write([]byte(role)) 123 | return 124 | } 125 | 126 | creds := s.Creds[roleName] 127 | 128 | if creds == nil { 129 | w.WriteHeader(http.StatusNotFound) 130 | return 131 | } 132 | 133 | // Needs to be atomic since each request is served in a separate goroutine 134 | atomic.AddInt32(&s.RequestCount, 1) 135 | 136 | payload := struct { 137 | Code string 138 | LastUpdated string 139 | Type string 140 | AccessKeyId string 141 | SecretAccessKey string 142 | Token string 143 | Expiration string 144 | }{ 145 | "Success", 146 | "2013-07-05T12:32:43Z", 147 | "AWS-HMAC", 148 | creds.AccessKeyId, 149 | creds.SecretAccessKey, 150 | creds.Token, 151 | creds.Expiration.Format(time.RFC3339), 152 | } 153 | 154 | jsondata, _ := json.Marshal(payload) 155 | 156 | w.Write(jsondata) 157 | } 158 | 159 | func NewFailingServer() *FailingServer { 160 | listener, port := GetListeningAddress() 161 | 162 | server := &FailingServer{ 163 | port, 164 | 0, 165 | listener, 166 | } 167 | 168 | go http.Serve(listener, server) 169 | 170 | // There is no way to ensure the HTTP server properly started :/ 171 | time.Sleep(50 * time.Millisecond) 172 | 173 | return server 174 | } 175 | 176 | func (s *FailingServer) Close() { 177 | s.l.Close() 178 | } 179 | 180 | func (s *FailingServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { 181 | atomic.AddInt32(&s.RequestCount, 1) 182 | 183 | if strings.HasSuffix(r.URL.Path, "/failtcp") { 184 | hijacker, ok := w.(http.Hijacker) 185 | 186 | if !ok { 187 | panic("ResponseWriter is not a Hijacker") 188 | } 189 | 190 | conn, _, _ := hijacker.Hijack() 191 | 192 | conn.Close() 193 | } else if strings.HasSuffix(r.URL.Path, "/failhttp") { 194 | w.WriteHeader(http.StatusInternalServerError) 195 | w.Write([]byte("HTTP error")) 196 | } else if strings.HasSuffix(r.URL.Path, "/notfound") { 197 | w.WriteHeader(http.StatusNotFound) 198 | w.Write([]byte("HTTP not found")) 199 | } else { 200 | w.WriteHeader(http.StatusOK) 201 | w.Write([]byte("HTTP OK")) 202 | } 203 | } 204 | 205 | func compareCreds(t *testing.T, creds *Credentials, expected *Credentials) bool { 206 | if creds == nil && expected != nil { 207 | t.Errorf("Unexpected nil credentials") 208 | return false 209 | } 210 | 211 | if creds != nil && expected == nil { 212 | t.Errorf("Unexpected non-nil credentials") 213 | return false 214 | } 215 | 216 | if creds == nil { 217 | return true 218 | } 219 | 220 | if creds.AccessKeyId != expected.AccessKeyId { 221 | t.Errorf("Invalid access key ID: expected '%s', got '%s'", expected.AccessKeyId, creds.AccessKeyId) 222 | return false 223 | } 224 | 225 | if creds.SecretAccessKey != expected.SecretAccessKey { 226 | t.Errorf("Invalid secret access key: expected '%s', got '%s'", expected.SecretAccessKey, creds.SecretAccessKey) 227 | return false 228 | } 229 | 230 | if creds.Token != expected.Token { 231 | t.Errorf("Invalid token: expected '%s', got '%s'", expected.Token, creds.Token) 232 | return false 233 | } 234 | 235 | exp1 := creds.Expiration.Format(time.RFC3339) 236 | exp2 := expected.Expiration.Format(time.RFC3339) 237 | 238 | if exp1 != exp2 { 239 | t.Errorf("Invalid expiration: expected '%s', got '%s'", exp1, exp2) 240 | return false 241 | } 242 | 243 | return true 244 | } 245 | 246 | func SetupFakeIAMServer() *IAMServer { 247 | server := NewIAMServer() 248 | 249 | // Overrides definition in credentialcache.go 250 | AmzIAMEndpoint = fmt.Sprintf("http://127.0.0.1:%d/latest/meta-data/iam/security-credentials/", server.Port) 251 | 252 | testRoleCreds := &Credentials{ 253 | "AccessKey", 254 | "SecretKey", 255 | "SecretToken", 256 | time.Now().Add(1 * time.Hour), 257 | } 258 | 259 | server.Creds["testrole"] = testRoleCreds 260 | 261 | return server 262 | } 263 | 264 | func TestRoles(t *testing.T) { 265 | server := SetupFakeIAMServer() 266 | defer server.Close() 267 | 268 | testData := []struct { 269 | Name string 270 | Role string 271 | Creds *Credentials 272 | Expiration time.Time 273 | Cached bool 274 | }{ 275 | { 276 | "First request for an IAM role token (uncached)", 277 | "testrole", 278 | server.Creds["testrole"], 279 | time.Now().Add(1 * time.Hour), 280 | false, 281 | }, 282 | { 283 | "Request for a not-expired-yet IAM role token", 284 | "testrole", 285 | server.Creds["testrole"], 286 | time.Now().Add(1 * time.Hour), 287 | true, 288 | }, 289 | { 290 | "Request for an expired IAM role token", 291 | "testrole", 292 | server.Creds["testrole"], 293 | time.Now().Add(1 * time.Hour), 294 | true, 295 | }, 296 | } 297 | 298 | c := NewCredentialCache() 299 | 300 | for _, d := range testData { 301 | currentCount := server.RequestCount 302 | 303 | if c := server.Creds[d.Role]; c != nil { 304 | c.Expiration = d.Expiration 305 | } 306 | 307 | creds, err := c.GetRoleCredentials() 308 | 309 | if d.Creds != nil && err != nil { 310 | t.Errorf("Unexpected error while fetching role for case '%s': %s", d.Name, err) 311 | continue 312 | } 313 | 314 | if !compareCreds(t, creds, d.Creds) { 315 | continue 316 | } 317 | 318 | if creds == nil { 319 | continue 320 | } 321 | 322 | hasBeenCached := server.RequestCount == currentCount 323 | 324 | if d.Cached != hasBeenCached { 325 | t.Errorf("Unexpected caching behaviour for case '%s' (has been cached: %v / should have been cached: %v)", d.Name, hasBeenCached, d.Cached) 326 | continue 327 | } 328 | } 329 | } 330 | 331 | func TestSignature(t *testing.T) { 332 | server := SetupFakeIAMServer() 333 | defer server.Close() 334 | 335 | h := NewProxyHandler(testConfig) 336 | 337 | // The signatures for those were generated with s3cmd 338 | testData := []struct { 339 | Name string 340 | Host string 341 | Path string 342 | Signature string 343 | }{ 344 | { 345 | "Configured bucket using virtual host", 346 | "testbucket.s3.amazonaws.com", 347 | "/folder/file", 348 | "RpgjresZ/ancNZM3iABqOSLeTnE=", 349 | }, 350 | { 351 | "Configured bucket with no virtual host", 352 | "s3.amazonaws.com", 353 | "/testbucket/folder/file", 354 | "RpgjresZ/ancNZM3iABqOSLeTnE=", 355 | }, 356 | { 357 | "Other configured bucket (test multi bucket setup)", 358 | "s3.amazonaws.com", 359 | "/testbucket2/folder/file", 360 | "/tkaeRrXgID3wbSFLHduukEkLjo=", 361 | }, 362 | { 363 | "Bucket with IAM role", 364 | "testbucket3.s3.amazonaws.com", 365 | "/folder/file", 366 | "Fucd5+FvRyP4ptezvxITdFa6wmc=", 367 | }, 368 | { 369 | "Unconfigured bucket", 370 | "anotherbucket.s3.amazonaws.com", 371 | "/", 372 | "", 373 | }, 374 | { 375 | "Website (not a bucket)", 376 | "atotallyunrelatedwebsite.org", 377 | "/index.html", 378 | "", 379 | }, 380 | } 381 | 382 | for _, d := range testData { 383 | requestUrl, _ := url.Parse("http://" + d.Host + d.Path) 384 | 385 | request := &http.Request{ 386 | Method: "GET", 387 | Host: d.Host, 388 | URL: requestUrl, 389 | Header: map[string][]string{ 390 | "Date": []string{"Tue, 09 Jul 2013 13:38:52 GMT"}, 391 | }, 392 | } 393 | 394 | // If the request should *not* be signed, make sure it's the case by 395 | // generating a header that will get overwritten if the request gets 396 | // signed. 397 | if d.Signature == "" { 398 | request.Header.Add("Authorization", "XXX") 399 | } 400 | 401 | info := h.GetBucketInfo(request) 402 | err := h.SignRequest(request, info) 403 | 404 | if err != nil { 405 | t.Errorf("Unexpected error while signing for test case '%s': %s", d.Name, err) 406 | continue 407 | } 408 | 409 | auth := request.Header.Get("Authorization") 410 | 411 | if d.Signature == "" { 412 | if auth != "XXX" { 413 | t.Errorf("Unexpected authorization header for test case '%s'", d.Name) 414 | } 415 | 416 | continue 417 | } 418 | 419 | if auth == "" { 420 | t.Errorf("Missing authorization header for test case '%s'", d.Name) 421 | continue 422 | } 423 | 424 | authTokens := strings.Split(auth, ":") 425 | 426 | if len(authTokens) != 2 { 427 | t.Errorf("Invalid authorization header for test case '%s'", d.Name) 428 | continue 429 | } 430 | 431 | if authTokens[0] != "AWS AccessKey" { 432 | t.Errorf("Invalid access key for test case '%s'", d.Name) 433 | continue 434 | } 435 | 436 | if authTokens[1] != d.Signature { 437 | t.Errorf("Invalid signature for test case '%s'\n Expected: '%s\n Got '%s'", d.Name, d.Signature, authTokens[1]) 438 | continue 439 | } 440 | } 441 | } 442 | 443 | func createProxiedRequest(url string, proxyport int) *http.Request { 444 | r, err := http.NewRequest("GET", url, nil) 445 | 446 | if err != nil { 447 | panic(fmt.Sprintf("Cannot create HTTP request: %s", err)) 448 | } 449 | 450 | r.Header.Add("Host", r.URL.Host) 451 | r.URL.Host = fmt.Sprintf("127.0.0.1:%d", proxyport) 452 | 453 | return r 454 | } 455 | 456 | func TestRetries(t *testing.T) { 457 | // Get the ProxyHandler to think AWS is 127.0.0.1 so we get the retries 458 | AwsDomain = "127.0.0.1" 459 | 460 | handler := NewProxyHandler(testConfig) 461 | listener, port := GetListeningAddress() 462 | 463 | go http.Serve(listener, handler) 464 | time.Sleep(50 * time.Millisecond) 465 | defer listener.Close() 466 | 467 | failingServer := NewFailingServer() 468 | defer failingServer.Close() 469 | 470 | client := &http.Client{} 471 | 472 | serverURL := fmt.Sprintf("http://127.0.0.1:%d", failingServer.Port) 473 | 474 | testData := []struct { 475 | Name string 476 | Path string 477 | ExpectedCode int 478 | RequestCount int32 479 | MinimumRequestTime int 480 | }{ 481 | { 482 | "A good request should pass and not retry", 483 | "/testbucket2/ok", 484 | http.StatusOK, 485 | 1, 486 | 0, 487 | }, 488 | { 489 | "A 404 should not retry", 490 | "/testbucket2/notfound", 491 | http.StatusNotFound, 492 | 1, 493 | 0, 494 | }, 495 | { 496 | "A TCP error should retry if specified", 497 | "/testbucket2/failtcp", 498 | http.StatusInternalServerError, 499 | 2, 500 | 0, 501 | }, 502 | { 503 | "A TCP error should not retry if not specified", 504 | "/testbucket/failtcp", 505 | http.StatusInternalServerError, 506 | 1, 507 | 0, 508 | }, 509 | { 510 | "An HTTP error should retry if specified", 511 | "/testbucket2/failhttp", 512 | http.StatusInternalServerError, 513 | 2, 514 | 0, 515 | }, 516 | { 517 | "An HTTP error should not retry if not specified", 518 | "/testbucket/failhttp", 519 | http.StatusInternalServerError, 520 | 1, 521 | 0, 522 | }, 523 | { 524 | "Throttled buckets should not retry too fast", 525 | "/testbucket4/failhttp", 526 | http.StatusInternalServerError, 527 | 2, 528 | 300, 529 | }, 530 | } 531 | 532 | for _, d := range testData { 533 | currentReqCount := failingServer.RequestCount 534 | 535 | startTime := time.Now() 536 | 537 | resp, err := client.Do(createProxiedRequest(serverURL+d.Path, port)) 538 | 539 | if err != nil { 540 | t.Fatalf("Error while doing request for case '%s': %s", d.Name, err) 541 | } 542 | 543 | if resp.StatusCode != d.ExpectedCode { 544 | t.Fatalf("Unexpected HTTP status %d for test case '%s', expected %d", resp.StatusCode, d.Name, d.ExpectedCode) 545 | } 546 | 547 | reqCount := failingServer.RequestCount - currentReqCount 548 | 549 | if reqCount != d.RequestCount { 550 | t.Fatalf("Invalid request count for case '%s': %d (expected %d)", d.Name, reqCount, d.RequestCount) 551 | } 552 | 553 | reqDuration := time.Now().Sub(startTime) 554 | 555 | if reqDuration < time.Duration(d.MinimumRequestTime)*time.Millisecond { 556 | t.Fatalf("Retries went too fast, should have taken at least %d milliseconds", d.MinimumRequestTime) 557 | } 558 | } 559 | } 560 | --------------------------------------------------------------------------------