├── .expeditor ├── config.yml └── verify.pipeline.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── rebar.config ├── rebar.lock ├── rebar3 ├── src ├── http_uri_.erl ├── mini_s3.app.src ├── mini_s3.erl ├── ms3_http.erl └── ms3_xml.erl └── test └── mini_s3_tests.erl /.expeditor/config.yml: -------------------------------------------------------------------------------- 1 | # Documentation available at https://expeditor.chef.io/docs/getting-started/ 2 | slack: 3 | notify_channel: 4 | - chef-server-notify 5 | 6 | github: 7 | delete_branch_on_merge: true 8 | 9 | pipelines: 10 | - verify: 11 | description: Pull Request validation tests 12 | public: true 13 | -------------------------------------------------------------------------------- /.expeditor/verify.pipeline.yml: -------------------------------------------------------------------------------- 1 | --- 2 | steps: 3 | - label: ":erlang: 24" 4 | command: 5 | - hab pkg install core/erlang24 -bf 6 | - make all 7 | expeditor: 8 | executor: 9 | docker: 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.beam 2 | deps 3 | *.tmproj 4 | _build 5 | *.plt 6 | ebin/*.beam 7 | ebin/*.app 8 | .eunit 9 | /.concrete/DEV_MODE 10 | .rebar -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Simple Makefile for rebar3 based erlang project 3 | # 4 | # 5 | # Use rebar3 from either: 6 | # - ./rebar3 7 | # - rebar3 on the PATH (found via which) 8 | # - Downloaded from $REBAR3_URL 9 | # 10 | REBAR3_URL=https://s3.amazonaws.com/rebar3/rebar3 11 | ifeq ($(wildcard rebar3),rebar3) 12 | REBAR3 = $(CURDIR)/rebar3 13 | endif 14 | 15 | # Fallback to rebar on PATH 16 | REBAR3 ?= $(shell which rebar3) 17 | 18 | # And finally, prep to download rebar if all else fails 19 | ifeq ($(REBAR3),) 20 | REBAR3 = ./rebar3 21 | endif 22 | 23 | all: $(REBAR3) 24 | @$(REBAR3) do clean, compile, eunit, dialyzer 25 | 26 | rel: all 27 | @$(REBAR3) release 28 | 29 | test: 30 | @$(REBAR3) eunit ct 31 | 32 | dialyzer: 33 | @$(REBAR3) dialyzer 34 | 35 | xref: 36 | @$(REBAR3) xref 37 | 38 | update: 39 | @$(REBAR3) update 40 | 41 | install: $(REBAR3) distclean update 42 | 43 | distclean: 44 | @rm -rf _build 45 | 46 | $(REBAR3): 47 | curl -Lo rebar3 $(REBAR3_URL) || wget $(REBAR3_URL) 48 | chmod a+x rebar3 49 | 50 | travis: all 51 | @echo "Travis'd!" 52 | 53 | .PHONY: test 54 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mini_s3 2 | 3 | mini_s3 is a simple s3 client API for erlang. This is used by the Chef 4 | Server which can be found at https://github.com/chef/chef-server 5 | 6 | # DEVELOPMENT 7 | 8 | To build and test mini_s3, run: 9 | 10 | make all 11 | 12 | This project uses eunit for testing. Please consider adding a unit 13 | test when submitting a new feature or bug fix. 14 | 15 | ## Signing Your Commits 16 | 17 | This project utilizes a Developer Certificate of Origin (DCO) to 18 | ensure that each commit was written by the author or that the author 19 | has the appropriate rights necessary to contribute the change. The 20 | project utilizes [Developer Certificate of Origin, Version 1.1](http://developercertificate.org/) 21 | 22 | ``` 23 | Developer Certificate of Origin 24 | Version 1.1 25 | 26 | Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 27 | 660 York Street, Suite 102, 28 | San Francisco, CA 94110 USA 29 | 30 | Everyone is permitted to copy and distribute verbatim copies of this 31 | license document, but changing it is not allowed. 32 | 33 | 34 | Developer's Certificate of Origin 1.1 35 | 36 | By making a contribution to this project, I certify that: 37 | 38 | (a) The contribution was created in whole or in part by me and I 39 | have the right to submit it under the open source license 40 | indicated in the file; or 41 | 42 | (b) The contribution is based upon previous work that, to the best 43 | of my knowledge, is covered under an appropriate open source 44 | license and I have the right under that license to submit that 45 | work with modifications, whether created in whole or in part 46 | by me, under the same open source license (unless I am 47 | permitted to submit under a different license), as indicated 48 | in the file; or 49 | 50 | (c) The contribution was provided directly to me by some other 51 | person who certified (a), (b) or (c) and I have not modified 52 | it. 53 | 54 | (d) I understand and agree that this project and the contribution 55 | are public and that a record of the contribution (including all 56 | personal information I submit with it, including my sign-off) is 57 | maintained indefinitely and may be redistributed consistent with 58 | this project or the open source license(s) involved. 59 | ``` 60 | 61 | Each commit must include a DCO which looks like this 62 | 63 | `Signed-off-by: Joe Smith ` 64 | 65 | The project requires that the name used is your real name. Neither 66 | anonymous contributors nor those utilizing pseudonyms will be 67 | accepted. 68 | 69 | Git makes it easy to add this line to your commit messages. 70 | 71 | 1. Make sure the `user.name` and `user.email` are set in your git configs. 72 | 2. Use `-s` or `--signoff` to add the Signed-off-by line to the end of the commit message. 73 | 74 | # LICENSE 75 | 76 | Copyright 2011-2016 Chef Software, Inc. All Rights Reserved. 77 | 78 | Licensed under the Apache License, Version 2.0 (the "License"); you 79 | may not use this file except in compliance with the License. You may 80 | obtain a copy of the License at 81 | 82 | http://www.apache.org/licenses/LICENSE-2.0 83 | 84 | Unless required by applicable law or agreed to in writing, software 85 | distributed under the License is distributed on an "AS IS" BASIS, 86 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 87 | implied. See the License for the specific language governing 88 | permissions and limitations under the License. 89 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | %% -*- mode: erlang -*- 2 | %% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- 3 | %% ex: ts=4 sw=4 ft=erlang et 4 | {deps, [ 5 | {envy, ".*", {git, "https://github.com/markan/envy", {branch, "master"}}}, 6 | {erlcloud,".*", {git, "https://github.com/chef/erlcloud", {branch, "lbaker/presigned-headers"}}} 7 | ]}. 8 | 9 | {profiles, [{ test, [ 10 | {deps, [meck]}, 11 | {erl_opts, [nowarn_export_all]} 12 | ] 13 | }]}. 14 | 15 | {erl_opts, [debug_info, warnings_as_errors]}. 16 | -------------------------------------------------------------------------------- /rebar.lock: -------------------------------------------------------------------------------- 1 | {"1.2.0", 2 | [{<<"base16">>,{pkg,<<"base16">>,<<"1.0.0">>},1}, 3 | {<<"eini">>,{pkg,<<"eini">>,<<"1.2.9">>},1}, 4 | {<<"envy">>, 5 | {git,"https://github.com/markan/envy", 6 | {ref,"0148fb4b7ed0e188511578e98b42d6e7dde0ebd1"}}, 7 | 0}, 8 | {<<"erlcloud">>, 9 | {git,"https://github.com/chef/erlcloud", 10 | {branch,"lbaker/presigned-headers"}}, 11 | 0}, 12 | {<<"jsx">>,{pkg,<<"jsx">>,<<"2.11.0">>},1}, 13 | {<<"lhttpc">>, 14 | {git,"https://github.com/erlcloud/lhttpc", 15 | {ref,"8e34985a3cd0ac2a7fc2a88a041554c64d33e74b"}}, 16 | 1}]}. 17 | [ 18 | {pkg_hash,[ 19 | {<<"base16">>, <<"283644E2B21BD5915ACB7178BED7851FB07C6E5749B8FAD68A53C501092176D9">>}, 20 | {<<"eini">>, <<"FCC3CBD49BBDD9A1D9735C7365DAFFCD84481CCE81E6CB80537883AA44AC4895">>}, 21 | {<<"jsx">>, <<"08154624050333919B4AC1B789667D5F4DB166DC50E190C4D778D1587F102EE0">>}]}, 22 | {pkg_hash_ext,[ 23 | {<<"base16">>, <<"02AFD0827E61A7B07093873E063575CA3A2B07520567C7F8CEC7C5D42F052D76">>}, 24 | {<<"eini">>, <<"DA64AE8DB7C2F502E6F20CDF44CD3D9BE364412B87FF49FEBF282540F673DFCB">>}, 25 | {<<"jsx">>, <<"EED26A0D04D217F9EECEFFFB89714452556CF90EB38F290A27A4D45B9988F8C0">>}]} 26 | ]. 27 | -------------------------------------------------------------------------------- /rebar3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/chef/mini_s3/4dd584fce031d35bbe5c4b72a04660b75673ca21/rebar3 -------------------------------------------------------------------------------- /src/http_uri_.erl: -------------------------------------------------------------------------------- 1 | %% 2 | %% %CopyrightBegin% 3 | %% 4 | %% Copyright Ericsson AB 2006-2018. All Rights Reserved. 5 | %% 6 | %% Licensed under the Apache License, Version 2.0 (the "License"); 7 | %% you may not use this file except in compliance with the License. 8 | %% You may obtain a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, software 13 | %% distributed under the License is distributed on an "AS IS" BASIS, 14 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | %% See the License for the specific language governing permissions and 16 | %% limitations under the License. 17 | %% 18 | %% %CopyrightEnd% 19 | %% 20 | %% 21 | %% This is from chapter 3, Syntax Components, of RFC 3986: 22 | %% 23 | %% The generic URI syntax consists of a hierarchical sequence of 24 | %% components referred to as the scheme, authority, path, query, and 25 | %% fragment. 26 | %% 27 | %% URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ] 28 | %% 29 | %% hier-part = "//" authority path-abempty 30 | %% / path-absolute 31 | %% / path-rootless 32 | %% / path-empty 33 | %% 34 | %% The scheme and path components are required, though the path may be 35 | %% empty (no characters). When authority is present, the path must 36 | %% either be empty or begin with a slash ("/") character. When 37 | %% authority is not present, the path cannot begin with two slash 38 | %% characters ("//"). These restrictions result in five different ABNF 39 | %% rules for a path (Section 3.3), only one of which will match any 40 | %% given URI reference. 41 | %% 42 | %% The following are two example URIs and their component parts: 43 | %% 44 | %% foo://example.com:8042/over/there?name=ferret#nose 45 | %% \_/ \______________/\_________/ \_________/ \__/ 46 | %% | | | | | 47 | %% scheme authority path query fragment 48 | %% | _____________________|__ 49 | %% / \ / \ 50 | %% urn:example:animal:ferret:nose 51 | %% 52 | %% scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) 53 | %% authority = [ userinfo "@" ] host [ ":" port ] 54 | %% userinfo = *( unreserved / pct-encoded / sub-delims / ":" ) 55 | %% 56 | %% 57 | 58 | -module(http_uri_). 59 | 60 | -export([parse/1, parse/2, 61 | scheme_defaults/0, 62 | encode/1, decode/1]). 63 | 64 | -export_type([uri/0, 65 | user_info/0, 66 | scheme/0, default_scheme_port_number/0, 67 | host/0, 68 | path/0, 69 | query/0, 70 | fragment/0]). 71 | 72 | -type uri() :: string() | binary(). 73 | -type user_info() :: string() | binary(). 74 | -type scheme() :: atom(). 75 | -type host() :: string() | binary(). 76 | -type path() :: string() | binary(). 77 | -type query() :: string() | binary(). 78 | -type fragment() :: string() | binary(). 79 | -type port_number() :: inet:port_number(). 80 | -type default_scheme_port_number() :: port_number(). 81 | -type hex_uri() :: string() | binary(). %% Hexadecimal encoded URI. 82 | -type maybe_hex_uri() :: string() | binary(). %% A possibly hexadecimal encoded URI. 83 | 84 | -type scheme_defaults() :: [{scheme(), default_scheme_port_number()}]. 85 | -type scheme_validation_fun() :: fun((SchemeStr :: string() | binary()) -> 86 | valid | {error, Reason :: term()}). 87 | 88 | %%%========================================================================= 89 | %%% API 90 | %%%========================================================================= 91 | 92 | -spec scheme_defaults() -> scheme_defaults(). 93 | scheme_defaults() -> 94 | [{http, 80}, 95 | {https, 443}, 96 | {ftp, 21}, 97 | {ssh, 22}, 98 | {sftp, 22}, 99 | {tftp, 69}]. 100 | 101 | -type parse_result() :: 102 | {scheme(), user_info(), host(), port_number(), path(), query()} | 103 | {scheme(), user_info(), host(), port_number(), path(), query(), 104 | fragment()}. 105 | 106 | -spec parse(uri()) -> {ok, parse_result()} | {error, term()}. 107 | parse(AbsURI) -> 108 | parse(AbsURI, []). 109 | 110 | -spec parse(uri(), [Option]) -> {ok, parse_result()} | {error, term()} when 111 | Option :: {ipv6_host_with_brackets, boolean()} | 112 | {scheme_defaults, scheme_defaults()} | 113 | {fragment, boolean()} | 114 | {scheme_validation_fun, scheme_validation_fun() | none}. 115 | parse(AbsURI, Opts) -> 116 | case parse_scheme(AbsURI, Opts) of 117 | {error, Reason} -> 118 | {error, Reason}; 119 | {Scheme, DefaultPort, Rest} -> 120 | case (catch parse_uri_rest(Scheme, DefaultPort, Rest, Opts)) of 121 | {ok, Result} -> 122 | {ok, Result}; 123 | {error, Reason} -> 124 | {error, {Reason, Scheme, AbsURI}}; 125 | _ -> 126 | {error, {malformed_url, Scheme, AbsURI}} 127 | end 128 | end. 129 | 130 | reserved() -> 131 | sets:from_list([$;, $:, $@, $&, $=, $+, $,, $/, $?, 132 | $#, $[, $], $<, $>, $\", ${, $}, $|, %" 133 | $\\, $', $^, $%, $ ]). 134 | 135 | -spec encode(uri()) -> hex_uri(). 136 | encode(URI) when is_list(URI) -> 137 | Reserved = reserved(), 138 | lists:append([uri_encode(Char, Reserved) || Char <- URI]); 139 | encode(URI) when is_binary(URI) -> 140 | Reserved = reserved(), 141 | << <<(uri_encode_binary(Char, Reserved))/binary>> || <> <= URI >>. 142 | 143 | -spec decode(maybe_hex_uri()) -> uri(). 144 | decode(String) when is_list(String) -> 145 | do_decode(String); 146 | decode(String) when is_binary(String) -> 147 | do_decode_binary(String). 148 | 149 | do_decode([$%,Hex1,Hex2|Rest]) -> 150 | [hex2dec(Hex1)*16+hex2dec(Hex2)|do_decode(Rest)]; 151 | do_decode([First|Rest]) -> 152 | [First|do_decode(Rest)]; 153 | do_decode([]) -> 154 | []. 155 | 156 | do_decode_binary(<<$%, Hex:2/binary, Rest/bits>>) -> 157 | <<(binary_to_integer(Hex, 16)), (do_decode_binary(Rest))/binary>>; 158 | do_decode_binary(<>) -> 159 | <>; 160 | do_decode_binary(<<>>) -> 161 | <<>>. 162 | 163 | %%%======================================================================== 164 | %%% Internal functions 165 | %%%======================================================================== 166 | 167 | which_scheme_defaults(Opts) -> 168 | Key = scheme_defaults, 169 | case lists:keysearch(Key, 1, Opts) of 170 | {value, {Key, SchemeDefaults}} -> 171 | SchemeDefaults; 172 | false -> 173 | scheme_defaults() 174 | end. 175 | 176 | parse_scheme(AbsURI, Opts) -> 177 | case split_uri(AbsURI, ":", {error, no_scheme}, 1, 1) of 178 | {error, no_scheme} -> 179 | {error, no_scheme}; 180 | {SchemeStr, Rest} -> 181 | case extract_scheme(SchemeStr, Opts) of 182 | {error, Error} -> 183 | {error, Error}; 184 | {ok, Scheme} -> 185 | SchemeDefaults = which_scheme_defaults(Opts), 186 | case lists:keysearch(Scheme, 1, SchemeDefaults) of 187 | {value, {Scheme, DefaultPort}} -> 188 | {Scheme, DefaultPort, Rest}; 189 | false -> 190 | {Scheme, no_default_port, Rest} 191 | end 192 | end 193 | end. 194 | 195 | extract_scheme(Str, Opts) -> 196 | case lists:keysearch(scheme_validation_fun, 1, Opts) of 197 | {value, {scheme_validation_fun, Fun}} when is_function(Fun) -> 198 | case Fun(Str) of 199 | valid -> 200 | {ok, to_atom(http_util:to_lower(Str))}; 201 | {error, Error} -> 202 | {error, Error} 203 | end; 204 | _ -> 205 | {ok, to_atom(http_util:to_lower(Str))} 206 | end. 207 | 208 | to_atom(S) when is_list(S) -> 209 | list_to_atom(S); 210 | to_atom(S) when is_binary(S) -> 211 | binary_to_atom(S, unicode). 212 | 213 | parse_uri_rest(Scheme, DefaultPort, <<"//", URIPart/binary>>, Opts) -> 214 | {Authority, PathQueryFragment} = 215 | split_uri(URIPart, "[/?#]", {URIPart, <<"">>}, 1, 0), 216 | {RawPath, QueryFragment} = 217 | split_uri(PathQueryFragment, "[?#]", {PathQueryFragment, <<"">>}, 1, 0), 218 | {Query, Fragment} = 219 | split_uri(QueryFragment, "#", {QueryFragment, <<"">>}, 1, 0), 220 | {UserInfo, HostPort} = split_uri(Authority, "@", {<<"">>, Authority}, 1, 1), 221 | {Host, Port} = parse_host_port(Scheme, DefaultPort, HostPort, Opts), 222 | Path = path(RawPath), 223 | case lists:keyfind(fragment, 1, Opts) of 224 | {fragment, true} -> 225 | {ok, {Scheme, UserInfo, Host, Port, Path, Query, Fragment}}; 226 | _ -> 227 | {ok, {Scheme, UserInfo, Host, Port, Path, Query}} 228 | end; 229 | parse_uri_rest(Scheme, DefaultPort, "//" ++ URIPart, Opts) -> 230 | {Authority, PathQueryFragment} = 231 | split_uri(URIPart, "[/?#]", {URIPart, ""}, 1, 0), 232 | {RawPath, QueryFragment} = 233 | split_uri(PathQueryFragment, "[?#]", {PathQueryFragment, ""}, 1, 0), 234 | {Query, Fragment} = 235 | split_uri(QueryFragment, "#", {QueryFragment, ""}, 1, 0), 236 | {UserInfo, HostPort} = split_uri(Authority, "@", {"", Authority}, 1, 1), 237 | {Host, Port} = parse_host_port(Scheme, DefaultPort, HostPort, Opts), 238 | Path = path(RawPath), 239 | case lists:keyfind(fragment, 1, Opts) of 240 | {fragment, true} -> 241 | {ok, {Scheme, UserInfo, Host, Port, Path, Query, Fragment}}; 242 | _ -> 243 | {ok, {Scheme, UserInfo, Host, Port, Path, Query}} 244 | end. 245 | 246 | 247 | %% In this version of the function, we no longer need 248 | %% the Scheme argument, but just in case... 249 | parse_host_port(_Scheme, DefaultPort, <<"[", HostPort/binary>>, Opts) -> %ipv6 250 | {Host, ColonPort} = split_uri(HostPort, "\\]", {HostPort, <<"">>}, 1, 1), 251 | Host2 = maybe_ipv6_host_with_brackets(Host, Opts), 252 | {_, Port} = split_uri(ColonPort, ":", {<<"">>, DefaultPort}, 0, 1), 253 | {Host2, int_port(Port)}; 254 | parse_host_port(_Scheme, DefaultPort, "[" ++ HostPort, Opts) -> %ipv6 255 | {Host, ColonPort} = split_uri(HostPort, "\\]", {HostPort, ""}, 1, 1), 256 | Host2 = maybe_ipv6_host_with_brackets(Host, Opts), 257 | {_, Port} = split_uri(ColonPort, ":", {"", DefaultPort}, 0, 1), 258 | {Host2, int_port(Port)}; 259 | 260 | parse_host_port(_Scheme, DefaultPort, HostPort, _Opts) -> 261 | {Host, Port} = split_uri(HostPort, ":", {HostPort, DefaultPort}, 1, 1), 262 | {Host, int_port(Port)}. 263 | 264 | split_uri(UriPart, SplitChar, NoMatchResult, SkipLeft, SkipRight) -> 265 | case re:run(UriPart, SplitChar, [{capture, first}]) of 266 | {match, [{Match, _}]} -> 267 | {string:slice(UriPart, 0, Match + 1 - SkipLeft), 268 | string:slice(UriPart, Match + SkipRight, string:length(UriPart))}; 269 | nomatch -> 270 | NoMatchResult 271 | end. 272 | 273 | maybe_ipv6_host_with_brackets(Host, Opts) when is_binary(Host) -> 274 | case lists:keysearch(ipv6_host_with_brackets, 1, Opts) of 275 | {value, {ipv6_host_with_brackets, true}} -> 276 | <<"[", Host/binary, "]">>; 277 | _ -> 278 | Host 279 | end; 280 | maybe_ipv6_host_with_brackets(Host, Opts) -> 281 | case lists:keysearch(ipv6_host_with_brackets, 1, Opts) of 282 | {value, {ipv6_host_with_brackets, true}} -> 283 | "[" ++ Host ++ "]"; 284 | _ -> 285 | Host 286 | end. 287 | 288 | int_port(Port) when is_integer(Port) -> 289 | Port; 290 | int_port(Port) when is_binary(Port) -> 291 | binary_to_integer(Port); 292 | int_port(Port) when is_list(Port) -> 293 | list_to_integer(Port); 294 | %% This is the case where no port was found and there was no default port 295 | int_port(no_default_port) -> 296 | throw({error, no_default_port}). 297 | 298 | path(<<"">>) -> 299 | <<"/">>; 300 | path("") -> 301 | "/"; 302 | path(Path) -> 303 | Path. 304 | 305 | uri_encode(Char, Reserved) -> 306 | case sets:is_element(Char, Reserved) of 307 | true -> 308 | [ $% | http_util:integer_to_hexlist(Char)]; 309 | false -> 310 | [Char] 311 | end. 312 | 313 | uri_encode_binary(Char, Reserved) -> 314 | case sets:is_element(Char, Reserved) of 315 | true -> 316 | << $%, (integer_to_binary(Char, 16))/binary >>; 317 | false -> 318 | <> 319 | end. 320 | 321 | hex2dec(X) when (X>=$0) andalso (X=<$9) -> X-$0; 322 | hex2dec(X) when (X>=$A) andalso (X=<$F) -> X-$A+10; 323 | hex2dec(X) when (X>=$a) andalso (X=<$f) -> X-$a+10. 324 | -------------------------------------------------------------------------------- /src/mini_s3.app.src: -------------------------------------------------------------------------------- 1 | %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- 2 | %% ex: ts=4 sw=4 ft=erlang et 3 | %% Copyright 2010 Brian Buchanan. All Rights Reserved. 4 | %% Copyright 2012 Opscode, Inc. All Rights Reserved. 5 | %% 6 | %% This file is provided to you under the Apache License, 7 | %% Version 2.0 (the "License"); you may not use this file 8 | %% except in compliance with the License. You may obtain 9 | %% a copy of the License at 10 | %% 11 | %% http://www.apache.org/licenses/LICENSE-2.0 12 | %% 13 | %% Unless required by applicable law or agreed to in writing, 14 | %% software distributed under the License is distributed on an 15 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | %% KIND, either express or implied. See the License for the 17 | %% specific language governing permissions and limitations 18 | %% under the License. 19 | %% 20 | 21 | {application, mini_s3, 22 | [{description, "A Robust Amazon S3 client for Erlang"}, 23 | {vsn, "0.0.1"}, 24 | {registered, []}, 25 | {applications, [erlcloud, 26 | stdlib, 27 | kernel, 28 | sasl, 29 | public_key, 30 | crypto, 31 | ssl, 32 | xmerl]}, 33 | {modules, []}, 34 | {env, []} 35 | ] 36 | }. 37 | -------------------------------------------------------------------------------- /src/mini_s3.erl: -------------------------------------------------------------------------------- 1 | %% -*- erlang-indent-level: 4;indent-tabs-mode: nil; fill-column: 92 -*- 2 | %% ex: ts=4 sw=4 et 3 | %% Amazon Simple Storage Service (S3) 4 | %% Copyright 2010 Brian Buchanan. All Rights Reserved. 5 | %% Copyright 2012 Opscode, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | 22 | -module(mini_s3). 23 | 24 | -export([ 25 | copy_object/5, 26 | copy_object/6, 27 | create_bucket/3, 28 | create_bucket/4, 29 | delete_bucket/2, 30 | delete_object/2, 31 | delete_object/3, 32 | delete_object_version/3, 33 | delete_object_version/4, 34 | get_bucket_attribute/2, 35 | get_bucket_attribute/3, 36 | get_object/4, 37 | get_object_acl/2, 38 | get_object_acl/3, 39 | get_object_acl/4, 40 | get_object_torrent/2, 41 | get_object_torrent/3, 42 | get_object_metadata/4, 43 | list_buckets/1, 44 | list_objects/2, 45 | list_objects/3, 46 | list_object_versions/2, 47 | list_object_versions/3, 48 | new/3, 49 | new/4, 50 | new/5, 51 | put_object/6, 52 | s3_url/6, 53 | s3_url/7, 54 | set_bucket_attribute/3, 55 | set_bucket_attribute/4, 56 | set_object_acl/3, 57 | set_object_acl/4 58 | ]). 59 | 60 | -export([make_authorization/10, 61 | manual_start/0, 62 | expiration_time_v4/1, 63 | universaltime/0 64 | ]). 65 | 66 | -include_lib("xmerl/include/xmerl.hrl"). 67 | -include_lib("erlcloud/include/erlcloud_aws.hrl"). 68 | 69 | -ifdef(TEST). 70 | -compile([export_all, nowarn_export_all]). 71 | -endif. 72 | 73 | -export([expiration_time/1]). 74 | 75 | -type s3_bucket_attribute_name() :: acl 76 | | location 77 | | logging 78 | | request_payment 79 | | versioning 80 | | notification. 81 | 82 | -type s3_bucket_acl() :: private 83 | | public_read 84 | | public_read_write 85 | | authenticated_read 86 | | bucket_owner_read 87 | | bucket_owner_full_control. 88 | 89 | -type s3_location_constraint() :: none 90 | | us_west_1 91 | | eu 92 | | 'us-east-1' 93 | | 'us-east-2' 94 | | 'us-west-1' 95 | | 'us-west-2' 96 | | 'ca-central-1' 97 | | 'eu-west-1' 98 | | 'eu-west-2' 99 | | 'eu-west-3' 100 | | 'eu-north-1' 101 | | 'eu-central-1' 102 | | 'ap-south-1' 103 | | 'ap-southeast-1' 104 | | 'ap-southeast-2' 105 | | 'ap-northeast-1' 106 | | 'ap-northeast-2' 107 | | 'ap-northeast-3' 108 | | 'ap-east-1' 109 | | 'me-south-1' 110 | | 'sa-east-1'. 111 | 112 | -export_type([aws_config/0, 113 | s3_bucket_attribute_name/0, 114 | s3_bucket_acl/0, 115 | s3_location_constraint/0]). 116 | 117 | -type bucket_access_type() :: vhost | path. 118 | 119 | %% This is a helper function that exists to make development just a 120 | %% wee bit easier 121 | -spec manual_start() -> ok. 122 | manual_start() -> 123 | application:start(crypto), 124 | application:start(public_key), 125 | application:start(ssl), 126 | application:start(inets). 127 | 128 | -spec discern_ipv(string()) -> pos_integer(). 129 | discern_ipv(Url) -> 130 | case lists:member($[, Url) andalso lists:member($], Url) of 131 | true -> 6; 132 | false -> 4 133 | end. 134 | 135 | -spec new(string() | binary(), string() | binary(), string()) -> aws_config(). 136 | new(AccessKeyID, SecretAccessKey, Url) -> 137 | Ipv = discern_ipv(Url), 138 | 139 | Parse = 140 | case Url of 141 | % uri_string:parse fails in certain cases of missing scheme. detect and fix. 142 | [$h, $t, $t, $p, $:, $/, $/ | _] -> uri_string:parse( Url ); 143 | [$h, $t, $t, $p, $s, $:, $/, $/ | _] -> uri_string:parse( Url ); 144 | _ -> uri_string:parse(["no-scheme://", Url]) 145 | end, 146 | 147 | Path0 = maps:get(path , Parse ), 148 | Host0 = maps:get(host , Parse, undefined), 149 | Scheme0 = maps:get(scheme, Parse, undefined), 150 | Port0 = maps:get(port , Parse, undefined), 151 | 152 | % detect and repair any erroneous parse. 153 | % uri_string:parse parses URLs shaped as "host" or "host:port" incorrectly, e.g.: 154 | % 155 | % % should be #{host => "host"} 156 | % > uri_string:parse("host"). 157 | % #{path => "host"} 158 | % 159 | % % should be #{host => "host", port => 80} 160 | % > uri_string:parse("host:80"). 161 | % #{path => "80",scheme => "host"} 162 | {Scheme1, Host1, Path1, Port1} = 163 | case {Scheme0, Host0, Path0, Port0} of 164 | {undefined, undefined, _, undefined} when Path0 /= undefined 165 | -> {undefined, Path0, "", undefined }; 166 | {_, undefined, _, undefined} -> {undefined, Scheme0, "", list_to_integer(Path0)}; 167 | _ -> {Scheme0, Host0, Path0, Port0 } 168 | end, 169 | 170 | Host2 = case Ipv of 4 -> string:lowercase(Host1); _ -> [$[ | Host1 ++ "]"] end, 171 | 172 | {Scheme, Port} = 173 | case {Scheme1, Port1} of 174 | {undefined, undefined} -> {"https://", 443}; 175 | {undefined, 80} -> {"http://", 80}; 176 | {undefined, _} -> {"https://", Port1}; 177 | {"http", undefined} -> {"http://", 80}; 178 | {"http", _} -> {"http://", Port1}; 179 | {"https", undefined} -> {"https://", 443}; 180 | {"https", _} -> {"https://", Port1}; 181 | {"no-scheme", undefined} -> {"https://", 443}; 182 | {"no-scheme", 80} -> {"http://", 80}; 183 | {"no-scheme", _} -> {"https://", Port1}; 184 | _ -> {Scheme1, Port1} 185 | end, 186 | 187 | %% bookshelf wants bucketname after host e.g. https://api.chef-server.dev:443/bookshelf. 188 | %% s3 wants bucketname before host (actually, it takes it either way) e.g. https://bookshelf.api.chef-server.dev:443. 189 | %% 190 | %% UPDATE 191 | %% amazon: "Buckets created after September 30, 2020, will support only virtual hosted-style requests. 192 | %% Path-style requests will continue to be supported for buckets created on or before this date." 193 | %% for further discussion, see: 194 | %% https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/ 195 | %% https://github.com/chef/chef-server/issues/2088 196 | %% https://github.com/chef/chef-server/issues/1911 197 | (erlcloud_s3:new(AccessKeyID, SecretAccessKey, Host2++Path1, Port))#aws_config{s3_scheme=Scheme, s3_bucket_after_host=true, s3_bucket_access_method=path}. 198 | 199 | % old mini_s3: 200 | % -spec new(string(), string(), string(), bucket_access_type()) -> aws_config(). 201 | % mini_s3:new(accesskey, secretaccesskey, host, bucketaccesstype) 202 | -spec new(string() | binary(), string() | binary(), string(), bucket_access_type()) -> aws_config(). 203 | new(AccessKeyID, SecretAccessKey, Host, BucketAccessType) -> 204 | {BucketAccessMethod, BucketAfterHost} = case BucketAccessType of path -> {path, true}; _ -> {vhost, false} end, 205 | Config = new(AccessKeyID, SecretAccessKey, Host), 206 | Config#aws_config{ 207 | s3_bucket_access_method=BucketAccessMethod, 208 | s3_bucket_after_host=BucketAfterHost 209 | }. 210 | 211 | % erlcloud has no new/5, and arguments differ. 212 | % for now, attempting conversion to new/4 (dropping SslOpts). 213 | % see: https://github.com/chef/chef-server/issues/2171 214 | -spec new(string() | binary(), string() | binary(), string(), bucket_access_type(), proplists:proplist()) -> aws_config(). 215 | new(AccessKeyID, SecretAccessKey, Host, BucketAccessType, _SslOpts) -> 216 | new(AccessKeyID, SecretAccessKey, Host, BucketAccessType). 217 | 218 | -define(XMLNS_S3, "http://s3.amazonaws.com/doc/2006-03-01/"). 219 | 220 | -spec copy_object(string(), string(), string(), string(), proplists:proplist()) -> proplists:proplist(). 221 | copy_object(DestBucketName, DestKeyName, SrcBucketName, SrcKeyName, Options) -> 222 | erlcloud_s3:copy_object(DestBucketName, DestKeyName, SrcBucketName, SrcKeyName, Options). 223 | 224 | -spec copy_object(string(), string(), string(), string(), proplists:proplist(), aws_config()) -> proplists:proplist(). 225 | copy_object(DestBucketName, DestKeyName, SrcBucketName, SrcKeyName, Options, Config) -> 226 | erlcloud_s3:copy_object(DestBucketName, DestKeyName, SrcBucketName, SrcKeyName, Options, Config). 227 | 228 | -spec create_bucket(string(), s3_bucket_acl(), s3_location_constraint() | aws_config()) -> ok. 229 | create_bucket(BucketName, ACL, LocationConstraint) -> 230 | erlcloud_s3:create_bucket(BucketName, ACL, LocationConstraint). 231 | 232 | -spec create_bucket(string(), s3_bucket_acl(), s3_location_constraint(), aws_config()) -> ok. 233 | create_bucket(BucketName, ACL, LocationConstraint, Config) -> 234 | erlcloud_s3:create_bucket(BucketName, ACL, LocationConstraint, Config). 235 | 236 | -spec delete_bucket(string(), aws_config()) -> ok. 237 | delete_bucket(BucketName, Config) -> 238 | erlcloud_s3:delete_bucket(BucketName, Config). 239 | 240 | -spec delete_object(string(), string()) -> proplists:proplist(). 241 | delete_object(BucketName, Key) -> 242 | erlcloud_s3:delete_object(BucketName, Key). 243 | 244 | -spec delete_object(string(), string(), aws_config()) -> proplists:proplist(). 245 | delete_object(BucketName, Key, Config) -> 246 | erlcloud_s3:delete_object(BucketName, Key, Config). 247 | 248 | -spec delete_object_version(string(), string(), string()) -> proplists:proplist(). 249 | delete_object_version(BucketName, Key, Version) -> 250 | erlcloud_s3:delete_object_version(BucketName, Key, Version). 251 | 252 | -spec delete_object_version(string(), string(), string(), aws_config()) -> proplists:proplist(). 253 | delete_object_version(BucketName, Key, Version, Config) -> 254 | erlcloud_s3:delete_object_version(BucketName, Key, Version, Config). 255 | 256 | -spec list_buckets(aws_config()) -> proplists:proplist(). 257 | list_buckets(Config) -> 258 | Result = erlcloud_s3:list_buckets(Config), 259 | case proplists:lookup(buckets, Result) of none -> [{buckets, []}]; X -> [X] end. 260 | 261 | -spec list_objects(string(), proplists:proplist()) -> proplists:proplist(). 262 | list_objects(BucketName, Options) -> 263 | erlcloud_s3:list_objects(BucketName, Options). 264 | 265 | -spec list_objects(string(), proplists:proplist(), aws_config()) -> proplists:proplist(). 266 | list_objects(BucketName, Options, Config) -> 267 | List = erlcloud_s3:list_objects(BucketName, Options, Config), 268 | [{name, Name} | Rest] = List, 269 | %[{name, http_uri:decode(Name)} | Rest]. 270 | [{name, decode(Name)} | Rest]. 271 | 272 | -spec get_bucket_attribute(string(), s3_bucket_attribute_name()) -> term(). 273 | get_bucket_attribute(BucketName, AttributeName) -> 274 | erlcloud_s3:get_bucket_attribute(BucketName, AttributeName). 275 | 276 | -spec get_bucket_attribute(string(), s3_bucket_attribute_name(), aws_config()) -> term(). 277 | get_bucket_attribute(BucketName, AttributeName, Config) -> 278 | erlcloud_s3:get_bucket_attribute(BucketName, AttributeName, Config). 279 | 280 | %% Abstraction of universaltime, so it can be mocked via meck 281 | -spec universaltime() -> calendar:datetime(). 282 | universaltime() -> 283 | erlang:universaltime(). 284 | 285 | -spec if_not_empty(string(), iolist()) -> iolist(). 286 | if_not_empty("", _V) -> 287 | ""; 288 | if_not_empty(_, Value) -> 289 | Value. 290 | 291 | -spec s3_url(atom(), string(), string(), non_neg_integer() | {non_neg_integer(), non_neg_integer()}, proplists:proplist(), aws_config()) -> binary(). 292 | s3_url(Method, BucketName0, Key0, {TTL, ExpireWin}, RawHeaders, Config) -> 293 | {Date, Lifetime} = expiration_time_v4({TTL, ExpireWin}), 294 | s3_url(Method, BucketName0, Key0, Lifetime, normalize_host(RawHeaders), Date, Config); 295 | s3_url(Method, BucketName0, Key0, Lifetime, RawHeaders, Config) 296 | when is_list(BucketName0), is_list(Key0), is_tuple(Config) -> 297 | [BucketName, Key] = [ms3_http:url_encode_loose(X) || X <- [BucketName0, Key0]], 298 | RequestURI = erlcloud_s3:make_presigned_v4_url(Lifetime, BucketName, Method, Key, [], normalize_host(RawHeaders), Config), 299 | iolist_to_binary(RequestURI). 300 | 301 | -spec s3_url(atom(), string(), string(), non_neg_integer(), proplists:proplist(), string(), aws_config()) -> binary(). 302 | s3_url(Method, BucketName0, Key0, Lifetime, RawHeaders, Date, Config) 303 | when is_list(BucketName0), is_list(Key0), is_tuple(Config) -> 304 | [BucketName, Key] = [ms3_http:url_encode_loose(X) || X <- [BucketName0, Key0]], 305 | RequestURI = erlcloud_s3:make_presigned_v4_url(Lifetime, BucketName, Method, Key, [], normalize_host(RawHeaders), Date, Config), 306 | iolist_to_binary(RequestURI). 307 | 308 | %----------------------------------------------------------------------------------- 309 | % Lincoln Baker's implementation from scratch of expiration windows for sigv4, 310 | % for making batches of cacheable presigned URLs. 311 | % 312 | % past present future 313 | % | 314 | % ------+------+------+--+---+------+------+------+------ 315 | % | | | | | | | | time 316 | % ------+------+------+--+---+------+------+------+------ 317 | % | ^ | 318 | % x-amz-date ----+ | +---- x-amz-expires 319 | % |-| 320 | % TTL 321 | % |------| 322 | % Lifetime 323 | % 324 | % Given a TTL, x-amz-expires should be set to be the closest expiry-window 325 | % boundary >= present+TTL, ie present+TTL selects the expiry-window. Squelch 326 | % any resulting Lifetime of greater than one week to one week. 327 | % 328 | % 1) Segment all of time into 'windows' of width expiry-window-size. 329 | % 2) Align x-amz-date to nearest expiry-window boundary less than present time. 330 | % 3) Align x-amz-expires to nearest expiry-window boundary greater than present time. 331 | % 4) The right edge of present+TTL is a 'selector' to determine which expiration 332 | % window we are in, thus determining final value of x-amz-expires and Lifetime. 333 | % 5) While x-amz-expires - present < TTL, x-amz-expires += expiry-window-size. 334 | % 6) Lifetime = x-amz-expires - x-amz-date, or WEEKSEC, whichever is less. 335 | % 336 | % USAGE: {Date, Lifetime} = make_expire_win(TTL, ExpireWin) 337 | % 338 | -define(WEEKSEC, 604800). 339 | %-spec make_expire_win(non_neg_integer(), non_neg_integer()) -> {XAmzDate::string(), Lifetime::non_neg_integer()}. 340 | %make_expire_win(TTL, ExpireWinSiz) when ExpireWinSiz > 0 -> 341 | % Present = calendar:datetime_to_gregorian_seconds(calendar:now_to_universal_time(os:timestamp())), 342 | % XAmzDateSec = Present div ExpireWinSiz * ExpireWinSiz, 343 | % ExpirWinMult = ((TTL div ExpireWinSiz) + (case TTL rem ExpireWinSiz > 0 of true -> 1; _ -> 0 end)), 344 | % XAmzExpires = case ExpirWinMult of 0 -> 1; _ -> ExpirWinMult end * ExpireWinSiz + XAmzDateSec, 345 | % Lifetime = 346 | % case (L = XAmzExpires - XAmzDateSec) > ?WEEKSEC of 347 | % true -> ?WEEKSEC; 348 | % _ -> L 349 | % end, 350 | % {erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(XAmzDateSec)), Lifetime}. 351 | 352 | %% calendar:datetime_to_gregorian_seconds({{1970, 1, 1}, {0, 0, 0}}). 353 | %% use midnight instead of EPOCH 354 | -define(EPOCH, 62167219200). 355 | -define(DAY, 86400). 356 | 357 | %% Prajakta's expiration windows, retrofitted from the old sigv2 implementation. 358 | %% @doc Number of seconds since the request is made that a request can be valid for, specified by 359 | %% TimeToLive, which is the number of seconds from "right now" that a request should be 360 | %% valid. If the argument provided is a tuple, we use the interval logic that will only 361 | %% result in Interval / 86400 unique expiration times per day 362 | expiration_time_v4({TimeToLive, Interval}) -> 363 | {{NowY, NowMo, NowD},{_,_,_}} = Now = mini_s3:universaltime(), 364 | NowSecs = calendar:datetime_to_gregorian_seconds(Now), 365 | MidnightSecs = calendar:datetime_to_gregorian_seconds({{NowY, NowMo, NowD},{0,0,0}}), 366 | %% How many seconds are we into today? 367 | TodayOffset = NowSecs - MidnightSecs, 368 | %XAmzDate in seconds starting Midnight. 369 | XAmzDateSecOffset = TodayOffset div Interval * Interval, 370 | %This last interval in a ?DAY is bounded by ?DAY; 371 | NewInterval = case ( XAmzDateSecOffset + Interval ) > ?DAY of 372 | true -> ?DAY - XAmzDateSecOffset; 373 | _ -> Interval 374 | end, 375 | Lifetime = case (L = TimeToLive + NewInterval) > ?WEEKSEC of 376 | true -> ?WEEKSEC; 377 | _ -> L 378 | end, 379 | {erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(XAmzDateSecOffset + MidnightSecs)), Lifetime}. 380 | 381 | %% @doc Number of seconds since the Epoch that a request can be valid for, specified by 382 | %% TimeToLive, which is the number of seconds from "right now" that a request should be 383 | %% valid. If the argument provided is a tuple, we use the interval logic that will only 384 | %% result in Interval / 86400 unique expiration times per day 385 | -spec expiration_time(TimeToLive :: non_neg_integer() | {non_neg_integer(), non_neg_integer()}) -> 386 | Expires::non_neg_integer(). 387 | expiration_time({TimeToLive, Interval}) -> 388 | {{NowY, NowMo, NowD},{_,_,_}} = Now = mini_s3:universaltime(), 389 | NowSecs = calendar:datetime_to_gregorian_seconds(Now), 390 | MidnightSecs = calendar:datetime_to_gregorian_seconds({{NowY, NowMo, NowD},{0,0,0}}), 391 | %% How many seconds are we into today? 392 | TodayOffset = NowSecs - MidnightSecs, 393 | Buffer = case (TodayOffset + Interval) >= ?DAY of 394 | %% true if we're in the day's last interval, don't let it spill into tomorrow 395 | true -> 396 | ?DAY - TodayOffset; 397 | %% false means this interval is bounded by today 398 | _ -> 399 | Interval - (TodayOffset rem Interval) 400 | end, 401 | NowSecs + Buffer - ?EPOCH + TimeToLive; 402 | expiration_time(TimeToLive) -> 403 | Now = calendar:datetime_to_gregorian_seconds(mini_s3:universaltime()), 404 | (Now - ?EPOCH) + TimeToLive. 405 | 406 | -spec get_object(string(), string(), proplists:proplist(), aws_config()) -> proplists:proplist(). 407 | get_object(BucketName, Key, Options, Config) -> 408 | erlcloud_s3:get_object(BucketName, Key, Options, Config). 409 | 410 | -spec get_object_acl(string(), string()) -> proplists:proplist(). 411 | get_object_acl(BucketName, Key) -> 412 | erlcloud_s3:get_object_acl(BucketName, Key). 413 | 414 | -spec get_object_acl(string(), string(), proplists:proplist() | aws_config()) -> proplists:proplist(). 415 | get_object_acl(BucketName, Key, Config) -> 416 | erlcloud_s3:get_object_acl(BucketName, Key, Config). 417 | 418 | -spec get_object_acl(string(), string(), proplists:proplist(), aws_config()) -> proplists:proplist(). 419 | get_object_acl(BucketName, Key, Options, Config) -> 420 | erlcloud_s3:get_object_acl(BucketName, Key, Options, Config). 421 | 422 | -spec get_object_metadata(string(), string(), proplists:proplist(), aws_config()) -> proplists:proplist(). 423 | get_object_metadata(BucketName, Key, Options, Config) -> 424 | erlcloud_s3:get_object_metadata(BucketName, Key, Options, Config). 425 | 426 | -spec get_object_torrent(string(), string()) -> proplists:proplist(). 427 | get_object_torrent(BucketName, Key) -> 428 | erlcloud_s3:get_object_torrent(BucketName, Key). 429 | 430 | -spec get_object_torrent(string(), string(), aws_config()) -> proplists:proplist(). 431 | get_object_torrent(BucketName, Key, Config) -> 432 | erlcloud_s3:get_object_torrent(BucketName, Key, Config). 433 | 434 | -spec list_object_versions(string(), proplists:proplist() | aws_config()) -> proplists:proplist(). 435 | list_object_versions(BucketName, Options) -> 436 | erlcloud_s3:list_object_versions(BucketName, Options). 437 | 438 | -spec list_object_versions(string(), proplists:proplist(), aws_config()) -> proplists:proplist(). 439 | list_object_versions(BucketName, Options, Config) -> 440 | erlcloud_s3:list_object_versions(BucketName, Options, Config). 441 | 442 | -spec put_object(string(), string(), iodata(), proplists:proplist(), [{string(), string()}], aws_config()) -> proplists:proplist(). 443 | put_object(BucketName, Key, Value, Options, HTTPHeaders, Config) -> 444 | erlcloud_s3:put_object(BucketName, Key, Value, Options, HTTPHeaders, Config). 445 | 446 | -spec set_object_acl(string(), string(), proplists:proplist()) -> ok. 447 | set_object_acl(BucketName, Key, ACL) -> 448 | erlcloud_s3:set_object_acl(BucketName, Key, ACL). 449 | 450 | -spec set_object_acl(string(), string(), proplists:proplist(), aws_config()) -> ok. 451 | set_object_acl(BucketName, Key, ACL, Config) -> 452 | erlcloud_s3:set_object_acl(BucketName, Key, ACL, Config). 453 | 454 | -spec set_bucket_attribute(string(), atom(), term()) -> ok. 455 | set_bucket_attribute(BucketName, AttributeName, Value) -> 456 | erlcloud_s3:set_bucket_attribute(BucketName, AttributeName, Value). 457 | 458 | -spec set_bucket_attribute(string(), atom(), term(), aws_config()) -> ok. 459 | set_bucket_attribute(BucketName, AttributeName, Value, Config) -> 460 | erlcloud_s3:set_bucket_attribute(BucketName, AttributeName, Value, Config). 461 | 462 | make_authorization(AccessKeyId, SecretKey, Method, ContentMD5, ContentType, Date, AmzHeaders, 463 | Host, Resource, Subresource) -> 464 | CanonizedAmzHeaders = 465 | [[Name, $:, Value, $\n] || {Name, Value} <- lists:sort(AmzHeaders)], 466 | StringToSign = [string:to_upper(atom_to_list(Method)), $\n, 467 | ContentMD5, $\n, 468 | ContentType, $\n, 469 | Date, $\n, 470 | CanonizedAmzHeaders, 471 | if_not_empty(Host, [$/, Host]), 472 | Resource, 473 | if_not_empty(Subresource, [$?, Subresource])], 474 | Signature = base64:encode(crypto:mac(hmac, sha, SecretKey, StringToSign)), 475 | {StringToSign, ["AWS ", AccessKeyId, $:, Signature]}. 476 | 477 | 478 | % ---------------------------------------------------- 479 | % local functions 480 | % ---------------------------------------------------- 481 | 482 | -spec decode(string() | binary()) -> string() | binary(). 483 | decode(String) when is_list(String) -> 484 | do_decode(String); 485 | decode(String) when is_binary(String) -> 486 | do_decode_binary(String). 487 | 488 | do_decode([$%,Hex1,Hex2|Rest]) -> 489 | [hex2dec(Hex1)*16+hex2dec(Hex2)|do_decode(Rest)]; 490 | do_decode([First|Rest]) -> 491 | [First|do_decode(Rest)]; 492 | do_decode([]) -> 493 | []. 494 | 495 | do_decode_binary(<<$%, Hex:2/binary, Rest/bits>>) -> 496 | <<(binary_to_integer(Hex, 16)), (do_decode_binary(Rest))/binary>>; 497 | do_decode_binary(<>) -> 498 | <>; 499 | do_decode_binary(<<>>) -> 500 | <<>>. 501 | 502 | hex2dec(X) when (X>=$0) andalso (X=<$9) -> X-$0; 503 | hex2dec(X) when (X>=$A) andalso (X=<$F) -> X-$A+10; 504 | hex2dec(X) when (X>=$a) andalso (X=<$f) -> X-$a+10. 505 | 506 | -spec normalize_host(proplists:proplist()) -> proplists:proplist(). 507 | normalize_host(Headers) -> 508 | case proplists:get_value("host", Headers) of 509 | undefined -> Headers; 510 | Host -> [{"host", string:lowercase(Host)} | proplists:delete("host", Headers)] 511 | end. 512 | 513 | % ---------------------------------------------------- 514 | % currently unused 515 | % ---------------------------------------------------- 516 | 517 | %-spec delete_bucket(string()) -> ok. 518 | %delete_bucket(BucketName) -> 519 | % erlcloud_s3:delete_bucket(BucketName). 520 | 521 | %-spec get_object(string(), string(), proplists:proplist()) -> proplists:proplist(). 522 | %get_object(BucketName, Key, Options) -> 523 | % erlcloud_s3:get_object(BucketName, Key, Options). 524 | 525 | %-spec put_object(string(), string(), iodata(), proplists:proplist(), [{string(), string()}] | aws_config()) -> proplists:proplist(). 526 | %put_object(BucketName, Key, Value, Options, HTTPHeaders) -> 527 | % erlcloud_s3:put_object(BucketName, Key, Value, Options, HTTPHeaders). 528 | 529 | % for some functions which don't pass in Configs 530 | %default_config() -> 531 | % Defaults = envy:get(mini_s3, s3_defaults, list), 532 | % case proplists:is_defined(key_id, Defaults) andalso 533 | % proplists:is_defined(secret_access_key, Defaults) of 534 | % true -> 535 | % {key_id, Key} = proplists:lookup(key_id, Defaults), 536 | % {secret_access_key, AccessKey} = 537 | % proplists:lookup(secret_access_key, Defaults), 538 | % #aws_config{access_key_id=Key, secret_access_key=AccessKey}; 539 | % false -> 540 | % throw({error, missing_s3_defaults}) 541 | % end. 542 | -------------------------------------------------------------------------------- /src/ms3_http.erl: -------------------------------------------------------------------------------- 1 | %% -*- erlang-indent-level: 4;indent-tabs-mode: nil; fill-column: 92 -*- 2 | %% ex: ts=4 sw=4 et 3 | %% Copyright 2010 Brian Buchanan. All Rights Reserved. 4 | %% Copyright 2012 Opscode, Inc. All Rights Reserved. 5 | %% 6 | %% This file is provided to you under the Apache License, 7 | %% Version 2.0 (the "License"); you may not use this file 8 | %% except in compliance with the License. You may obtain 9 | %% a copy of the License at 10 | %% 11 | %% http://www.apache.org/licenses/LICENSE-2.0 12 | %% 13 | %% Unless required by applicable law or agreed to in writing, 14 | %% software distributed under the License is distributed on an 15 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | %% KIND, either express or implied. See the License for the 17 | %% specific language governing permissions and limitations 18 | %% under the License. 19 | %% 20 | 21 | -module(ms3_http). 22 | 23 | -export([make_query_string/1, url_encode/1, url_encode_loose/1]). 24 | 25 | make_query_string(Params) -> 26 | string:join([[Key, "=", url_encode(value_to_string(Value))] 27 | || {Key, Value} <- Params, Value =/= none, Value =/= undefined], 28 | "&"). 29 | 30 | value_to_string(Integer) 31 | when is_integer(Integer) -> 32 | integer_to_list(Integer); 33 | value_to_string(Atom) 34 | when is_atom(Atom) -> 35 | atom_to_list(Atom); 36 | value_to_string(Binary) 37 | when is_binary(Binary) -> 38 | Binary; 39 | value_to_string(String) 40 | when is_list(String) -> 41 | String. 42 | 43 | url_encode(Binary) when is_binary(Binary) -> 44 | url_encode(binary_to_list(Binary)); 45 | url_encode(String) -> 46 | url_encode(String, []). 47 | url_encode([], Accum) -> 48 | lists:reverse(Accum); 49 | url_encode([Char|String], Accum) 50 | when Char >= $A, Char =< $Z; 51 | Char >= $a, Char =< $z; 52 | Char >= $0, Char =< $9; 53 | Char =:= $-; Char =:= $_; 54 | Char =:= $.; Char =:= $~ -> 55 | url_encode(String, [Char|Accum]); 56 | url_encode([Char|String], Accum) 57 | when Char >=0, Char =< 255 -> 58 | url_encode(String, [hex_char(Char rem 16), hex_char(Char div 16),$%|Accum]). 59 | 60 | url_encode_loose(Binary) when is_binary(Binary) -> 61 | url_encode_loose(binary_to_list(Binary)); 62 | url_encode_loose(String) -> 63 | url_encode_loose(String, []). 64 | url_encode_loose([], Accum) -> 65 | lists:reverse(Accum); 66 | url_encode_loose([Char|String], Accum) 67 | when Char >= $A, Char =< $Z; 68 | Char >= $a, Char =< $z; 69 | Char >= $0, Char =< $9; 70 | Char =:= $-; Char =:= $_; 71 | Char =:= $.; Char =:= $~; 72 | Char =:= $/; Char =:= $: -> 73 | url_encode_loose(String, [Char|Accum]); 74 | url_encode_loose([Char|String], Accum) 75 | when Char >=0, Char =< 255 -> 76 | url_encode_loose(String, [hex_char(Char rem 16), hex_char(Char div 16),$%|Accum]). 77 | 78 | hex_char(C) 79 | when C >= 0, C =< 9 -> 80 | $0 + C; 81 | hex_char(C) 82 | when C >= 10, C =< 15 -> 83 | $A + C - 10. 84 | -------------------------------------------------------------------------------- /src/ms3_xml.erl: -------------------------------------------------------------------------------- 1 | %% -*- erlang-indent-level: 4;indent-tabs-mode: nil; fill-column: 92 -*- 2 | %% ex: ts=4 sw=4 et 3 | %% Copyright 2010 Brian Buchanan. All Rights Reserved. 4 | %% Copyright 2012 Opscode, Inc. All Rights Reserved. 5 | %% 6 | %% This file is provided to you under the Apache License, 7 | %% Version 2.0 (the "License"); you may not use this file 8 | %% except in compliance with the License. You may obtain 9 | %% a copy of the License at 10 | %% 11 | %% http://www.apache.org/licenses/LICENSE-2.0 12 | %% 13 | %% Unless required by applicable law or agreed to in writing, 14 | %% software distributed under the License is distributed on an 15 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | %% KIND, either express or implied. See the License for the 17 | %% specific language governing permissions and limitations 18 | %% under the License. 19 | %% 20 | 21 | -module(ms3_xml). 22 | 23 | -export([decode/2, get_text/2]). 24 | 25 | -include_lib("xmerl/include/xmerl.hrl"). 26 | 27 | decode(Values, Node) -> 28 | lists:reverse(lists:foldl(fun ({Name, XPath, Type}, 29 | Output) -> 30 | case get_value(XPath, Type, Node) of 31 | undefined -> Output; 32 | Value -> [{Name, Value} | Output] 33 | end 34 | end, 35 | [], Values)). 36 | 37 | get_value(XPath, text, Node) -> 38 | get_text(XPath, Node); 39 | get_value(XPath, optional_text, Node) -> 40 | get_text(XPath, Node, undefined); 41 | get_value(XPath, integer, Node) -> 42 | get_integer(XPath, Node); 43 | get_value(XPath, optional_integer, Node) -> 44 | case get_text(XPath, Node, undefined) of 45 | undefined -> 46 | undefined; 47 | Text -> 48 | list_to_integer(Text) 49 | end; 50 | get_value(XPath, float, Node) -> 51 | get_float(XPath, Node); 52 | get_value(XPath, time, Node) -> 53 | get_time(XPath, Node); 54 | get_value(XPath, list, Node) -> 55 | get_list(XPath, Node); 56 | get_value(XPath, boolean, Node) -> 57 | get_bool(XPath, Node); 58 | get_value(XPath, optional_boolean, Node) -> 59 | case get_text(XPath, Node, undefined) of 60 | undefined -> 61 | undefined; 62 | "true" -> 63 | true; 64 | _ -> 65 | false 66 | end; 67 | get_value(XPath, present, Node) -> 68 | xmerl_xpath:string(XPath, Node) =/= []; 69 | get_value(_XPath, xml, Node) -> 70 | Node; 71 | get_value(XPath, Fun, Node) 72 | when is_function(Fun, 1) -> 73 | Fun(xmerl_xpath:string(XPath, Node)); 74 | get_value(XPath, {single, Fun}, Node) 75 | when is_function(Fun, 1) -> 76 | case xmerl_xpath:string(XPath, Node) of 77 | [] -> undefined; 78 | [SubNode] -> Fun(SubNode) 79 | end; 80 | get_value(XPath, {single, List}, Node) 81 | when is_list(List) -> 82 | case xmerl_xpath:string(XPath, Node) of 83 | [] -> undefined; 84 | [SubNode] -> decode(List, SubNode) 85 | end; 86 | get_value(XPath, {value, Fun}, Node) 87 | when is_function(Fun, 1) -> 88 | Fun(get_text(XPath, Node)); 89 | get_value(XPath, List, Node) 90 | when is_list(List) -> 91 | [decode(List, SubNode) 92 | || SubNode <- xmerl_xpath:string(XPath, Node)]. 93 | 94 | get_float(XPath, Node) -> 95 | list_to_float(get_text(XPath, Node)). 96 | 97 | get_text(#xmlText{value = Value}) -> Value; 98 | get_text(#xmlElement{content = Content}) -> 99 | lists:flatten([get_text(Node) || Node <- Content]). 100 | 101 | get_text(XPath, Doc) -> get_text(XPath, Doc, ""). 102 | 103 | get_text({XPath, AttrName}, Doc, Default) -> 104 | case xmerl_xpath:string(XPath ++ "/@" ++ AttrName, Doc) 105 | of 106 | [] -> Default; 107 | [#xmlAttribute{value = Value} | _] -> Value 108 | end; 109 | get_text(XPath, Doc, Default) -> 110 | case xmerl_xpath:string(XPath ++ "/text()", Doc) of 111 | [] -> Default; 112 | TextNodes -> 113 | lists:flatten([Node#xmlText.value || Node <- TextNodes]) 114 | end. 115 | 116 | get_list(XPath, Doc) -> 117 | [get_text(Node) 118 | || Node <- xmerl_xpath:string(XPath, Doc)]. 119 | 120 | get_integer(XPath, Doc) -> 121 | get_integer(XPath, Doc, 0). 122 | 123 | get_integer(XPath, Doc, Default) -> 124 | case get_text(XPath, Doc) of 125 | "" -> 126 | Default; 127 | Text -> 128 | list_to_integer(Text) 129 | end. 130 | 131 | get_bool(XPath, Doc) -> 132 | case get_text(XPath, Doc, "false") of 133 | "true" -> 134 | true; 135 | _ -> 136 | false 137 | end. 138 | 139 | get_time(XPath, Doc) -> 140 | case get_text(XPath, Doc, undefined) of 141 | undefined -> 142 | undefined; 143 | Time -> 144 | parse_time(Time) 145 | end. 146 | 147 | parse_time(String) -> 148 | case re:run(String, 149 | "^(\\d{4})-(\\d{2})-(\\d{2})T(\\d{2}):(\\d{2})" 150 | ":(\\d{2})(?:\\.\\d+)?Z", 151 | [{capture, all_but_first, list}]) 152 | of 153 | {match, [Yr, Mo, Da, H, M, S]} -> 154 | {{list_to_integer(Yr), list_to_integer(Mo), 155 | list_to_integer(Da)}, 156 | {list_to_integer(H), list_to_integer(M), 157 | list_to_integer(S)}}; 158 | nomatch -> error 159 | end. 160 | -------------------------------------------------------------------------------- /test/mini_s3_tests.erl: -------------------------------------------------------------------------------- 1 | %% -*- erlang-indent-level: 4;indent-tabs-mode: nil; fill-column: 92 -*- 2 | %% ex: ts=4 sw=4 et 3 | %% Copyright 2013 Opscode, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | -module(mini_s3_tests). 20 | 21 | -include_lib("eunit/include/eunit.hrl"). 22 | -include_lib("erlcloud/include/erlcloud_aws.hrl"). 23 | 24 | -spec format_s3_uri(aws_config(), string()) -> string(). 25 | format_s3_uri(Config, Bucket) -> 26 | erlcloud_s3:get_object_url(Bucket, "", Config). 27 | 28 | format_s3_uri_test_() -> 29 | Config = fun(Url, Type) -> 30 | mini_s3:new("", "", Url, Type) 31 | end, 32 | Tests = [ 33 | %% hostname 34 | {"https://my-aws.me.com", vhost, "https://bucket.my-aws.me.com:443/"}, 35 | {"https://my-aws.me.com", path, "https://my-aws.me.com:443/bucket/"}, 36 | 37 | %% ipv4 38 | {"https://192.168.12.13", path, "https://192.168.12.13:443/bucket/"}, 39 | 40 | %% ipv6 41 | {"https://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]", path, 42 | "https://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:443/bucket/"}, 43 | 44 | %% These tests document current behavior. Using 45 | %% vhost with an IP address does not make sense, 46 | %% but leaving as-is for now to avoid adding the 47 | %% is_it_an_ip_or_a_name code. 48 | {"https://192.168.12.13", vhost, "https://bucket.192.168.12.13:443/"}, 49 | 50 | {"https://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]", vhost, 51 | "https://bucket.[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:443/"} 52 | ], 53 | [ ?_assertEqual(Expect, format_s3_uri(Config(Url, Type), "bucket")) 54 | || {Url, Type, Expect} <- Tests ]. 55 | 56 | -define(MIDNIGHT, 63589536000). 57 | -define(DAY, 86400). 58 | -define(HOUR, 3600). 59 | -define(WEEK, 604800). 60 | 61 | expiration_time_test_() -> 62 | Tests = [ 63 | %% {TTLSecs, IntervalSecs, MockedTimestamp, ExpectedExpiry} 64 | {{3600, 900}, {{2015,1,27},{0,0,0}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT))), (?HOUR + 900)}}, 65 | {{3600, 900}, {{2015,1,27},{0,0,10}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT))), (?HOUR + 900)}}, 66 | {{3600, 900}, {{2015,1,27},{0,1,0}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT))), (?HOUR + 900)}}, 67 | {{3600, 900}, {{2015,1,27},{0,1,10}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT))), (?HOUR + 900)}}, 68 | {{3600, 900}, {{2015,1,27},{0,3,0}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT))), (?HOUR + 900)}}, 69 | {{3600, 900}, {{2015,1,27},{0,3,30}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT))), (?HOUR + 900)}}, 70 | {{3600, 900}, {{2015,1,27},{0,5,0}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT))), (?HOUR + 900)}}, 71 | {{3600, 900}, {{2015,1,27},{0,10,0}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT))), (?HOUR + 900)}}, 72 | {{3600, 900}, {{2015,1,27},{0,14,0}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT))), (?HOUR + 900)}}, 73 | {{3600, 900}, {{2015,1,27},{0,14,59}}, {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT))), (?HOUR + 900)}}, 74 | {{3600, 900}, {{2015,1,27},{0,15,0}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT + 900))), (?HOUR + 900)}}, 75 | {{3600, 900}, {{2015,1,27},{0,15,1}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT + 900))), (?HOUR + 900)}}, 76 | {{3600, 900}, {{2015,1,27},{0,29,59}}, {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT + 900))), (?HOUR + 900)}}, 77 | {{3600, 900}, {{2015,1,27},{0,30,0}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT + 1800))), (?HOUR + 900)}}, 78 | {{3600, 900}, {{2015,1,27},{0,44,59}}, {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT + 1800))), (?HOUR + 900)}}, 79 | {{3600, 900}, {{2015,1,27},{0,45,0}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT + 2700))), (?HOUR + 900)}}, 80 | {{3600, 900}, {{2015,1,27},{0,59,59}}, {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT + 2700))), (?HOUR + 900)}}, 81 | {{3600, 900}, {{2015,1,27},{1,0,0}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT + 3600))), (?HOUR + 900)}}, 82 | {{3600, 900}, {{2015,1,27},{23,44,0}}, {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT + ?DAY - 1800))), (?HOUR + 900)}}, 83 | {{3600, 900}, {{2015,1,27},{23,58,0}}, {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT + ?DAY - 900))), (?HOUR + 900)}}, 84 | {{3600, 900}, {{2015,1,28},{0,0,0}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT + ?DAY))), (?HOUR + 900)}}, 85 | 86 | %% There are 86400 seconds in a day. What happens if the interval is not evenly 87 | %% divisible in that time? Take 7m for example. 420 secs goes into a day 205.71 88 | %% times which is a remainder of 300 seconds. We should make sure that we 89 | %% restart the intervals at midnight, so we don't have day to day drift 90 | 91 | {{3600, 420}, {{2015,1,27},{23,59,0}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT + ?DAY - 300))), (?HOUR + 300)}}, 92 | {{3600, 420}, {{2015,1,28},{0,0,0}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT + ?DAY))), (?HOUR + 420)}}, 93 | 94 | {{604800, 420}, {{2015,1,27},{0,0,0}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT))), (?WEEK)}} 95 | 96 | %% Let's test the old functionality too 97 | % {3600, {{2015,1,27},{0,0,0}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT))), (?HOUR)}}, 98 | % {3600, {{2015,1,27},{0,0,1}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT))), (?HOUR + 1)}}, 99 | % {3600, {{2015,1,27},{0,1,1}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT))), (?HOUR + 61)}}, 100 | % {3600, {{2015,1,28},{0,1,1}} , {(erlcloud_aws:iso_8601_basic_time(calendar:gregorian_seconds_to_datetime(?MIDNIGHT + ?DAY))), (?DAY + ?HOUR + 61)}} 101 | ], 102 | 103 | TestFun = fun(Arg, MockedTime) -> 104 | meck:new(mini_s3, [unstick, passthrough]), 105 | meck:expect(mini_s3, universaltime, fun() -> MockedTime end), 106 | Expiry = mini_s3:expiration_time_v4(Arg), 107 | meck:unload(mini_s3), 108 | Expiry 109 | end, 110 | [ ?_assertEqual(Expect, TestFun(Arg, MockedTimestamp)) 111 | || {Arg, MockedTimestamp, Expect} <- Tests]. 112 | 113 | % NOTE: this should be included in make_expire_win_test(), but timeout doesn't work unless the function is named main_test_(). 114 | % note that this test is very sensitive to timing, and will fail if the timing is off. 115 | % for reference: -spec make_expire_win(TTL::non_neg_integer(), WinSize::non_neg_integer()) -> {XAmzDate::string(), Lifetime::non_neg_integer()}. 116 | main_test_() -> 117 | {timeout, 60, 118 | fun() -> 119 | % 10 expiration windows of size 1sec created 1sec apart should be 10 unique windows (i.e. no duplicates) 120 | Set1 = [{timer:sleep(1000), mini_s3:expiration_time_v4({0, 1})} || _ <- [1,2,3,4,5,6,7,8,9,0]], 121 | 10 = length(sets:to_list(sets:from_list(Set1))), 122 | 123 | % 100 expiration windows of large size created quickly should be mostly duplicates 124 | Set2 = [mini_s3:expiration_time_v4({0, 1000}) || _ <- lists:duplicate(100, 0)], 125 | true = 2 >= length(sets:to_list(sets:from_list(Set2))) 126 | end 127 | }. 128 | 129 | expiration_time_v4_test() -> 130 | % test that this property holds: 131 | % lifetime >= ttl; lifetime >= expire_win_size 132 | {_, 1} = mini_s3:expiration_time_v4({0, 1}), 133 | {_, 2} = mini_s3:expiration_time_v4({1, 1}), 134 | {_, 3} = mini_s3:expiration_time_v4({2, 1}), 135 | {_, 100} = mini_s3:expiration_time_v4({0, 100}), 136 | {_, 199} = mini_s3:expiration_time_v4({99, 100}), 137 | {_, 200} = mini_s3:expiration_time_v4({100, 100}), 138 | {_, L0} = mini_s3:expiration_time_v4({101, 100}), 139 | true = L0 >= 101, 140 | {_, 1000} = mini_s3:expiration_time_v4({0, 1000}), 141 | {_, 1999} = mini_s3:expiration_time_v4({999, 1000}), 142 | {_, 2000} = mini_s3:expiration_time_v4({1000, 1000}), 143 | {_, L1} = mini_s3:expiration_time_v4({1001, 1000}), 144 | true = L1 >= 1001. 145 | 146 | new_test() -> 147 | % scheme://host:port 148 | Config0 = mini_s3:new("key", "secret", "http://host:80"), 149 | "http://" = Config0#aws_config.s3_scheme, 150 | "host" = Config0#aws_config.s3_host, 151 | 80 = Config0#aws_config.s3_port, 152 | 153 | Config1 = mini_s3:new("key", "secret", "https://host:80"), 154 | "https://" = Config1#aws_config.s3_scheme, 155 | "host" = Config1#aws_config.s3_host, 156 | 80 = Config1#aws_config.s3_port, 157 | 158 | Config2 = mini_s3:new("key", "secret", "http://host:443"), 159 | "http://" = Config2#aws_config.s3_scheme, 160 | "host" = Config2#aws_config.s3_host, 161 | 443 = Config2#aws_config.s3_port, 162 | 163 | Config3 = mini_s3:new("key", "secret", "https://host:443"), 164 | "https://" = Config3#aws_config.s3_scheme, 165 | "host" = Config3#aws_config.s3_host, 166 | 443 = Config3#aws_config.s3_port, 167 | 168 | Config4 = mini_s3:new("key", "secret", "https://host:23"), 169 | "https://" = Config4#aws_config.s3_scheme, 170 | "host" = Config4#aws_config.s3_host, 171 | 23 = Config4#aws_config.s3_port, 172 | 173 | Config5 = mini_s3:new("key", "secret", "http://host:23"), 174 | "http://" = Config5#aws_config.s3_scheme, 175 | "host" = Config5#aws_config.s3_host, 176 | 23 = Config5#aws_config.s3_port, 177 | 178 | Config00 = mini_s3:new("key", "secret", "http://[1234:1234:1234:1234:1234:1234:1234:1234]:80"), 179 | "http://" = Config00#aws_config.s3_scheme, 180 | "[1234:1234:1234:1234:1234:1234:1234:1234]" = Config00#aws_config.s3_host, 181 | 80 = Config00#aws_config.s3_port, 182 | 183 | 184 | % scheme://host 185 | Config6 = mini_s3:new("key", "secret", "https://host"), 186 | "https://" = Config6#aws_config.s3_scheme, 187 | "host" = Config6#aws_config.s3_host, 188 | 443 = Config6#aws_config.s3_port, 189 | 190 | Config7 = mini_s3:new("key", "secret", "http://host"), 191 | "http://" = Config7#aws_config.s3_scheme, 192 | "host" = Config7#aws_config.s3_host, 193 | 80 = Config7#aws_config.s3_port, 194 | 195 | Config11 = mini_s3:new("key", "secret", "http://[1234:1234:1234:1234:1234:1234:1234:1234]"), 196 | "http://" = Config11#aws_config.s3_scheme, 197 | "[1234:1234:1234:1234:1234:1234:1234:1234]" = Config11#aws_config.s3_host, 198 | 80 = Config11#aws_config.s3_port, 199 | 200 | 201 | % host:port 202 | Config99 = mini_s3:new("key", "secret", "127.0.0.1:4321"), 203 | "https://" = Config99#aws_config.s3_scheme, 204 | "127.0.0.1" = Config99#aws_config.s3_host, 205 | 4321 = Config99#aws_config.s3_port, 206 | 207 | Config8 = mini_s3:new("key", "secret", "host:80"), 208 | "http://" = Config8#aws_config.s3_scheme, 209 | "host" = Config8#aws_config.s3_host, 210 | 80 = Config8#aws_config.s3_port, 211 | 212 | Config9 = mini_s3:new("key", "secret", "host:443"), 213 | "https://" = Config9#aws_config.s3_scheme, 214 | "host" = Config9#aws_config.s3_host, 215 | 443 = Config9#aws_config.s3_port, 216 | 217 | ConfigA = mini_s3:new("key", "secret", "host:23"), 218 | "https://" = ConfigA#aws_config.s3_scheme, 219 | "host" = ConfigA#aws_config.s3_host, 220 | 23 = ConfigA#aws_config.s3_port, 221 | 222 | Config88 = mini_s3:new("key", "secret", "[1234:1234:1234:1234:1234:1234:1234:1234]:80"), 223 | "http://" = Config88#aws_config.s3_scheme, 224 | "[1234:1234:1234:1234:1234:1234:1234:1234]" = Config88#aws_config.s3_host, 225 | 80 = Config88#aws_config.s3_port, 226 | 227 | 228 | % host 229 | ConfigB = mini_s3:new("key", "secret", "host"), 230 | "https://" = ConfigB#aws_config.s3_scheme, 231 | "host" = ConfigB#aws_config.s3_host, 232 | 443 = ConfigB#aws_config.s3_port, 233 | 234 | ConfigBB = mini_s3:new("key", "secret", "[1234:1234:1234:1234:1234:1234:1234:1234]"), 235 | "https://" = ConfigBB#aws_config.s3_scheme, 236 | "[1234:1234:1234:1234:1234:1234:1234:1234]" = ConfigBB#aws_config.s3_host, 237 | 443 = ConfigBB#aws_config.s3_port. 238 | --------------------------------------------------------------------------------