├── .formatter.exs ├── .github ├── dependabot.yml └── workflows │ └── on-push.yml ├── .gitignore ├── .iex.exs ├── .tool-versions ├── CHANGELOG.md ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE.md ├── LICENSE ├── PULL_REQUEST_TEMPLATE.md ├── README.md ├── config ├── config.exs ├── dev.exs ├── prod.exs └── test.exs ├── lib ├── ex_aws.ex └── ex_aws │ ├── auth.ex │ ├── auth │ ├── credentials.ex │ ├── signatures.ex │ └── utils.ex │ ├── behaviour.ex │ ├── config.ex │ ├── config │ ├── auth_cache.ex │ └── defaults.ex │ ├── credentials_ini │ ├── file.ex │ └── provider.ex │ ├── error.ex │ ├── instance_meta.ex │ ├── instance_meta_token_provider.ex │ ├── json │ ├── codec.ex │ └── jsx.ex │ ├── operation.ex │ ├── operation │ ├── json.ex │ ├── query.ex │ ├── query │ │ └── parser.ex │ ├── rest_query.ex │ └── s3.ex │ ├── request.ex │ ├── request │ ├── hackney.ex │ ├── http_client.ex │ ├── req.ex │ └── url.ex │ └── utils.ex ├── mix.exs ├── mix.lock ├── priv └── endpoints.exs └── test ├── alternate_helper.exs ├── default_helper.exs ├── ex_aws ├── auth │ ├── auth_cache_test.exs │ ├── credentials_test.exs │ ├── signatures_test.exs │ └── utils_test.exs ├── auth_test.exs ├── config_test.exs ├── credentials_ini │ └── file_test.exs ├── ex_aws_test.exs ├── instance_meta_test.exs ├── operation │ └── s3_test.exs ├── request │ ├── req_test.exs │ └── url_test.exs ├── request_test.exs └── utils_test.exs ├── telemetry_helper.exs └── test_helper.exs /.formatter.exs: -------------------------------------------------------------------------------- 1 | [ 2 | import_deps: [], 3 | inputs: ["*.{ex,exs}", "priv/*/", "{config,lib,test}/**/*.{ex,exs}"], 4 | subdirectories: ["priv/*/"] 5 | ] 6 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: mix 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | open-pull-requests-limit: 10 8 | ignore: 9 | - dependency-name: ex_doc 10 | versions: 11 | - 0.24.2 12 | - dependency-name: hackney 13 | versions: 14 | - 1.17.1 15 | -------------------------------------------------------------------------------- /.github/workflows/on-push.yml: -------------------------------------------------------------------------------- 1 | name: on-push 2 | on: [push, pull_request] 3 | env: 4 | MIX_ENV: test 5 | AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} 6 | AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 7 | jobs: 8 | build: 9 | runs-on: ubuntu-24.04 10 | # See https://hexdocs.pm/elixir/compatibility-and-deprecations.html#compatibility-between-elixir-and-erlang-otp 11 | strategy: 12 | matrix: 13 | include: 14 | - pair: 15 | otp: "27" 16 | elixir: "1.18" 17 | lint: lint 18 | - pair: 19 | otp: "27" 20 | elixir: "1.17" 21 | 22 | - pair: 23 | otp: "26" 24 | elixir: "1.17" 25 | - pair: 26 | otp: "26" 27 | elixir: "1.16" 28 | - pair: 29 | otp: "26" 30 | elixir: "1.15" 31 | 32 | - pair: 33 | otp: "25" 34 | elixir: "1.17" 35 | - pair: 36 | otp: "25" 37 | elixir: "1.16" 38 | - pair: 39 | otp: "25" 40 | elixir: "1.15" 41 | - pair: 42 | otp: "25" 43 | elixir: "1.14" 44 | 45 | - pair: 46 | otp: "24" 47 | elixir: "1.16" 48 | - pair: 49 | otp: "24" 50 | elixir: "1.15" 51 | - pair: 52 | otp: "24" 53 | elixir: "1.14" 54 | - pair: 55 | otp: "24" 56 | elixir: "1.13" 57 | 58 | steps: 59 | - uses: actions/checkout@v4 60 | - uses: erlef/setup-beam@v1 61 | with: 62 | otp-version: ${{matrix.pair.otp}} 63 | elixir-version: ${{matrix.pair.elixir}} 64 | - uses: rrainn/dynamodb-action@v3.0.0 65 | with: 66 | port: 8000 67 | cors: "*" 68 | - uses: actions/cache@v4 69 | with: 70 | path: | 71 | deps 72 | _build 73 | key: ${{ runner.os }}-mix-${{matrix.pair.otp}}-${{matrix.pair.elixir}}-${{ hashFiles('**/mix.lock') }} 74 | restore-keys: | 75 | ${{ runner.os }}-mix-${{matrix.pair.otp}}-${{matrix.pair.elixir}}- 76 | 77 | - run: mix deps.get 78 | 79 | - run: mix compile 80 | 81 | - run: mix deps.unlock --check-unused 82 | if: ${{matrix.lint}} 83 | 84 | - run: mix format --check-formatted 85 | if: ${{matrix.lint}} 86 | 87 | - run: mix dialyzer 88 | if: ${{matrix.lint}} 89 | 90 | - run: mix test 91 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /doc 2 | /_build 3 | *_build* 4 | /deps 5 | **/deps* 6 | erl_crash.dump 7 | *.ez 8 | /mnesia/dev 9 | /mnesia/test 10 | /mnesia/prod 11 | *.pem 12 | /tmp 13 | 14 | # IntelliJ IDEA files 15 | /.idea 16 | /exaws.iml 17 | .elixir_ls 18 | *~ 19 | -------------------------------------------------------------------------------- /.iex.exs: -------------------------------------------------------------------------------- 1 | alias ExAws.Dynamo 2 | alias ExAws.Kinesis 3 | 4 | Application.ensure_all_started(:hackney) 5 | -------------------------------------------------------------------------------- /.tool-versions: -------------------------------------------------------------------------------- 1 | elixir 1.18.1 2 | erlang 27.2 3 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | v2.5.9 (2025-04-15) 2 | - Endpoint updates 3 | - Fixes for Req support, update minimum version to 0.5.10 4 | - Do not warn when optional deps are not available 5 | - Implement retries for TooManyRequestsException 6 | - Move regexes out of module attributes to fix compatibility with OTP 28 7 | 8 | v2.5.8 (2024-12-13) 9 | - Fix Req :follow_redirects deprecation warning 10 | - Add exclude_patterns to filter out temporary files 11 | 12 | v2.5.7 (2024-10-31) 13 | - Endpoint updates 14 | - Configurable max attempts for client-specific errors (4xx) using `max_attempts_client` in `retries` config 15 | - Fix `follow_redirect` option handling for `req` client 16 | 17 | v2.5.6 (2024-10-09) 18 | - Endpoint updates 19 | 20 | v2.5.5 (2024-09-10) 21 | - Endpoint updates 22 | - Drop support for Elixir 1.12 23 | - Add support for Elixir 1.17 24 | - Add Req request backend 25 | - Stop sending `content-length` header for DELETE and HEAD requests 26 | - Fix handling of `[profile ]` blocks in config file 27 | - Update test platform to Ubuntu 24.04 28 | 29 | v2.5.4 (2024-06-04) 30 | - Endpoint and region updates 31 | - Add support for Elixir 1.16 32 | - Drop support for Elixir 1.11 33 | - Add support for `sso_session` in `.aws/config` 34 | - Add support for EventBridge Pipes service 35 | - Add support for Pinpoint Mobiletargeting service 36 | - Add docs for running DynamoDB locally 37 | - Allow iodata body in s3 requests 38 | 39 | v2.5.3 (2024-03-20) 40 | - Add mappings for Chime 41 | 42 | v2.5.2 (2024-03-19) 43 | - Endpoint updates 44 | 45 | v2.5.1 (2024-01-11) 46 | - Endpoint updates 47 | - Documentation updates 48 | - Fix request error handling when error type is a proplist 49 | - Fixes to ensure Req and Finch clients return the same success fields as hackney 50 | - Add basic support for Personalise service 51 | - Add basic support for Bedrock service 52 | 53 | v2.5.0 54 | - [Breaking] Bump minimum elixir version to 1.11 55 | - Replace retired HTTPotion with Req as default web client 56 | - Endpoint updates 57 | 58 | v2.4.4 59 | - Endpoint updates 60 | - Add new endpoints for Chime SDK Media Pipelines API 61 | 62 | v2.4.3 63 | - Switch default behaviour of credential refreshing so that it must be explicitly enabled with 64 | `refreshable: true`. Having it as the default behaviour was causing breaking issues with ignoring 65 | overridesin places they shouldn't have been. 66 | - Update endpoints 67 | 68 | v2.4.2 69 | - Add name, request, and response data to telemetry 70 | - Force refresh of auth config during long-running streaming operations to avoid failed auth 71 | - Update endpoints 72 | - Update docs 73 | 74 | v2.4.1 75 | - Add support for credentials_process in AWS credentials config 76 | - Service endpoint updates 77 | - Switch to `Config` from `Mix.Config` 78 | 79 | v2.4.0 80 | - Increase minimum elixir version to 1.10 81 | - Add `error_parser` field to operations. This may be optionally populated by services which 82 | need to do service-specific error handling prior to falling back to the default ExAws handling. 83 | 84 | V2.3.4 85 | - Fix crash in authentication for regions without SSO service (#894) 86 | - Service endpoint updates 87 | 88 | v2.3.3 89 | - Imporve resiliency/recovery when authentication token queries fail 90 | - Use `default` profile for `:aws_cli` config when `AWS_PROFILE is undefined 91 | - Include service in telemetry events 92 | - Fix crash generating auth headers for request with empty path 93 | 94 | v2.3.2 95 | - Fix type for IMDSv2 header 96 | - Make IMDSv2 optional, with fallback to v1 97 | - Fix spec for `Config.new/2` 98 | 99 | v2.3.1 100 | - Support container task role credentials in token provider 101 | - Fix issue with ECS instance meta data introduced in 2.3.0 102 | - Fix typespec on `ExAws.Request.HttpClient.request/5` 103 | 104 | v2.3.0 105 | - Raise an exception on S3 operation when bucket is `nil` 106 | - Update regions for transcribe service 107 | - Doc and spec improvements 108 | - Add location service 109 | - Add support for IMDSv2 110 | - Add support for awscli SSO credentials system 111 | 112 | v2.2.10 113 | - Add Athena support in `ca-central-1` 114 | - Add support for `me` region 115 | 116 | v2.2.9 117 | - Add me-south-1 region 118 | - Display `ExAws.Config` docs 119 | 120 | v2.2.8 121 | - Fix compiler warning on Elixir 1.13 122 | - Fix support for explicitly passing in headers 123 | - Add new Rekognition endpoints 124 | 125 | v2.2.7 126 | - `Request.Url`: Fix sanitize with spaces in request params 127 | - Relax `jsx` requirement 128 | - Relax `mime` requirement 129 | - Update CI tests to include elixir 1.12 130 | 131 | v2.2.6 132 | - Increase minimum SweetXML version and disable DTD parsing (#781) 133 | - Pass optional headers to REST requests (#820) 134 | - Restrict mime version to 1.x 135 | - Add config for sagemaker_runtime_a2i 136 | 137 | v2.2.5 138 | - Revert #796 to resolve #814. A more comprehensive fix for #796 is in the works. 139 | 140 | v2.2.4 141 | - Various documentation updates 142 | - Improve performance of space de-duplication on auth headers (#788) 143 | - Include the expected sequence token in error returns where it exists (#791) 144 | - Ensure absolute path for virtual hosted stile S3 URLs (#792) 145 | - Add QuickSight endpoints (#793) 146 | - Tighten up `telemetry` version requirement 147 | - Prevent `:awscli` config values from causing config from the CLI to leak into other values for 148 | which it wasn't specified (#796) 149 | - Update SageMaker endpoints (#804) 150 | - Update SES endpoints (#807) 151 | - Add eu-north-1 endpoint for logs (#811) 152 | 153 | v2.2.3 154 | - Add af-south-1 S3 region 155 | - Add support for telemetry events 156 | 157 | v2.2.2 158 | - Add sa-east-1 region to cognito-idp service 159 | - Support for af-south-1 160 | - Increase minimum hackney version to 1.16 to hopefully reduce instances of people hitting bugs 161 | in older versions 162 | - Include profile in ETS key used for :awscli auth cache 163 | 164 | v2.2.1 165 | - Fix regression in 2.2.0 requiring metadata instance config parameter 166 | - Fix calculation of authentication cache time 167 | 168 | v2.2.0 169 | - Add us-west-1 to list of supported ses services. 170 | - Handle aws errors that do not have a `#` in the type 171 | - [Breaking] Allow STS credentials to be injected by configuration 172 | - This change moves the `ExAws.CredentialsIni` functions into 173 | `ExAws.CredentialsIni.File` and turns the former into a behaviour definition. 174 | Any explicit uses of `ExAws.CredentialsIni.` will need to be 175 | replaced with `ExAws.CredentialsIni.File.`. 176 | 177 | v2.1.9 178 | - Small tweak to correctly handle error responses from DynamoDB local v1.15 179 | 180 | v2.1.8 181 | - Fix regression introduced in 2.1.7 which broke creation of folders (#752) 182 | - Fixes to run cleanly under dialyzer 183 | - Fix ExAws.Request.HttpClient.request spec to include header fields required by S3 184 | - Fix S3 path handling on Windows 185 | - Add Athena for eu-west-2 186 | - Refactor auth cache refreshing (fixes issue #625) 187 | - `mix format` pass 188 | 189 | v2.1.7 190 | 191 | - Various documentation updates 192 | - Add `comprehend` endpoint 193 | - Support firehose in region ca-central-1 194 | - More documentation fixes 195 | - Add github workflow actions for CI 196 | - Add us-east-2 endpoint for SES 197 | - Use :crypto.mac/4 rather than the deprecated :crypto.hmac/3 when available 198 | - Support virtual-host style S3 buckets 199 | - Fix presigned URLs with embedded query parameter strings 200 | - Support reading profile for CLI config from AWS_PROFILE environment variable 201 | 202 | v2.1.6 203 | 204 | - Fixes/updates for various service endpoints 205 | - Add support form Chime, via ex_chime_aws 206 | - Typing fix for HTTP content-lenght header 207 | - Fix warnings for Elixir 1.11 208 | - Increase minimum Elixir version to 1.5 209 | - Update and tidy docs and README 210 | 211 | v2.1.5 212 | 213 | - Elixir 1.11 compatibility tweak 214 | 215 | v2.1.3 216 | 217 | - Relax Jason version 218 | 219 | v2.1.0 220 | 221 | - Slew of bug fixes 222 | - Updated endpoints and regions 223 | - [Breaking] kinesis.tail task renamed to aws.kinesis.tail 224 | 225 | v2.0.2 226 | 227 | - Enhancement: Enable `ExAws.Auth.presigned_url` with custom body. Enables https://github.com/ex-aws/ex_aws_rds/pull/3 228 | - Enhancement: Handle non AWS regions with new default structure. 229 | 230 | v2.0.1 231 | 232 | - Fix regression where mix config region was applies too late. 233 | 234 | ExAws v2.0.0 235 | 236 | - Major Project Split 237 | - Configuration update to support all regions on all services. 238 | 239 | ExAws v1.1.4 240 | 241 | - Further refactoring of EC2, relaxed dependencies 242 | 243 | ExAws v1.1.3 244 | 245 | - Significant refactoring of EC2 246 | - Expansion of CloudFormation functionality 247 | 248 | - DynamoDB: Permit empty lists, add stream_query. 249 | 250 | ExAws v1.1.2 251 | 252 | - New Service: Cloudwatch (initial) 253 | - New Service: ElasticTranscoder 254 | 255 | - Various bug fixes. 256 | 257 | Thanks to our many contributors! 258 | 259 | ExAws v1.1.0 260 | 261 | This update has quite a few changes, many thanks to those who contributed code 262 | this release! 263 | 264 | Enhancements 265 | - New Service: Route53 266 | - New Service: DynamoStreams 267 | - New Service: SES (Partial) 268 | - New Service: STS (Partial) 269 | - SQS: Support for FIFO queues added. 270 | - Improved error message when authentication keys are missing or invalid 271 | - Improved error message when instance role is used locally 272 | 273 | Breaking Changes: 274 | - Elixir 1.4 required for `S3.upload` 275 | - Flow support removed in favor of `Task.async_stream` 276 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============ 3 | 4 | ExAws would not be what it is today without the enormous contributions of many people. This document exists to provide a clear direction for those who want to help with this critical part of the Elixir open source community. 5 | 6 | Thank you all for your help! 7 | 8 | ## ExAws v2.0 Architecture 9 | 10 | In order to manage the growing size of ExAws, the project has been split up. The `:ex_aws` package itself now consists only of 11 | 12 | - Authentication logic 13 | - Request logic 14 | - Core Operations. 15 | 16 | To elaborate on that last point, AWS services can be grouped by their broad approach to API building. You've got ones that focus on JSON, ones that combine JSON + REST, older ones that are all query param based, and so on. There are really just a handful of these operation types, and they live here in ExAws itself. They each implement the `ExAws.Operation` protocol, which is what actually drives the request process. 17 | 18 | Building these operations by hand however is not always pleasant, so to improve on the experience there are dedicated service modules like `ExAws.Dynamo` or `ExAws.S3` that provide Elixir friendly functions each representing some action on the service. These functions build and return the aforementioned operation structs, which can then be passed to `ExAws.request`. 19 | 20 | These service modules now live in their own packages, and have a variety of maintainers. For fixing bugs or adding features to existing services, please simply open a PR or an issue on each service individually. 21 | 22 | ## Adding a Service 23 | 24 | The ExAws organization is happy to accept new services, but be advised that you the contributor are committing to maintain the service. If this is not a responsibility that you want, please simply maintain the code under your own GitHub account. 25 | 26 | After you have built a project following the rest of this document, create an issue on the main `ex_aws` project asking for a repo for the project. After one is created, create a PR from your project to the new, empty project. If the review process is succesful it will be merged, and you will be given commit rights on that repository. 27 | 28 | ### Determine the Operation 29 | 30 | As mentioned, ExAws is built around some common operations and thus the first step of adding a new service is figuring out what operation to use. The Go SDK also uses a similar paradigm, and the JSON documents found there are a useful reference. Find the API you want to use in https://github.com/aws/aws-sdk-go/tree/master/models/apis. If we wanted to build Redshift, we'd go to https://github.com/aws/aws-sdk-go/blob/master/models/apis/redshift/2012-12-01/api-2.json#L6, looking at the `"protocol"` key. In this case it's the `query` protocol so we want the `ExAws.Operation.Query` type. 31 | 32 | What this means is that we need every function within our hypothetical `ExAws.Redshift` module to return an `%ExAws.Operation.Query{}` struct. 33 | 34 | Note that the mapping from protocol to operation type isn't always one-to-one. 35 | For example, the `"rest-json"` protocol uses the `ExAws.Operation.JSON` type. 36 | 37 | ### Build a Project 38 | 39 | We now need to build a new project. 40 | 41 | ```bash 42 | $ mix new ex_aws_redshift --module ExAws.Redshift 43 | ``` 44 | 45 | In this project we want a `lib/ex_aws/redshift.ex` file that looks like: 46 | 47 | ```elixir 48 | defmodule ExAws.Redshift do 49 | 50 | @type describe_cluster_opt :: {:cluster_identifier, String.t} 51 | | {:marker, String.t} 52 | | {:max_records, 20..100} 53 | | {:tags, [String.t]} 54 | @spec describe_clusters(opts :: [describe_cluster_opt]) :: ExAws.Operation.Query.t 55 | def describe_clusters(opts \\ []) do 56 | params = # build params here 57 | request(:describe_clusters, params) 58 | end 59 | 60 | defp request(action, params) do 61 | action_string = action |> Atom.to_string |> Macro.camelize 62 | 63 | %ExAws.Operation.Query{ 64 | path: "/" <> queue, 65 | params: params |> Map.put("Action", action_string), 66 | service: :redshift, 67 | action: action, 68 | parser: &ExAws.RedShift.Parsers.parse/2 69 | } 70 | end 71 | end 72 | ``` 73 | 74 | Notice a few things. The `describe_clusters/0,1` function matches up with the Describe Clusters action found in http://docs.aws.amazon.com/redshift/latest/APIReference/API_DescribeClusters.html. All of the parameters found there are optional, so we put them all in opts. Each of these parameters is given an Elixir friendly name, like `:cluster_identifier` instead of `:ClusterIdentifier`, and the tags are normalized to a nice list of strings instead of the whole `TagKeys.TagKey.N` nonsense. It's up to the function to turn those nice params into what AWS expects. 75 | 76 | Finally, we're gonna have a lot of functions that all create the same operation struct, so we extract it into a `request/2` helper function. This helper function also defines a `parser` function. See the SQS for an example of this. 77 | 78 | ### What about code generation? 79 | 80 | Having seen those JSON files in the go SDK you may be wondering if it would be possible to generate all the service modules based on the contents of those JSON files. There is definitely some merit to this idea, and you are more than welcome to try. 81 | 82 | Unfortunately however my own (Ben Wilson) efforts at this keep hitting dead ends. It's very hard to typespec everything because AWS will have both `Endpoint` as a type and `endpoint` as a type, whereas Elixir requires all types to be lower case. Trying to generate parsers for the responses had similar issues. Since we're in a dynamic language there's far less need to enforce input shape, AWS validates all that stuff anyway. 83 | 84 | For now, it's been simply easier to build helper modules by hand. Who knows, maybe you're up for building ExAws v3.0? 85 | 86 | ## Formatting and code style 87 | 88 | Before opening a PR, please run `mix format` over your changes with a recent Elixir version, and also `mix dialyzer` to make sure you haven't introduced any typing errors. 89 | 90 | ## Running Tests 91 | Running the test suite for ex_aws requires a few things: 92 | 93 | 1. DynamoDB Local must be running 94 | * May be downloaded from http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Tools.DynamoDBLocal.html 95 | * And may be executed with `java -Djava.library.path=/DynamoDBLocal_lib -jar DynamoDBLocal.jar -inMemory` 96 | 2. Requires two environment variables `AWS_ACCESS_KEY_ID`, and `AWS_SECRET_ACCESS_KEY` 97 | * These must be valid AWS credentials, the minimum IAM permissions required are: 98 | ``` 99 | { 100 | "Version": "2012-10-17", 101 | "Statement": [ 102 | { 103 | "Effect": "Allow", 104 | "Action": [ 105 | "dynamodb:ListStreams", 106 | "route53:ListHostedZones", 107 | "kinesis:ListStreams", 108 | "lambda:ListFunctions", 109 | "s3:ListAllMyBuckets", 110 | "sns:ListTopics", 111 | "sqs:ListQueues", 112 | "ec2:DescribeInstances", 113 | "firehose:ListDeliveryStreams", 114 | "ses:VerifyEmailIdentity", 115 | "elastictranscoder:ListPipelines", 116 | "cloudwatch:DescribeAlarms", 117 | "cloudformation:DescribeStacks", 118 | "cloudformation:ListStacks" 119 | ], 120 | "Resource": "*" 121 | } 122 | ] 123 | } 124 | ``` 125 | 126 | The test suite can be run with `AWS_ACCESS_KEY_ID=your-aws-access-key AWS_SECRET_ACCESS_KEY=your-aws-secret-access-key mix test` 127 | 128 | ### Key Management Service 129 | 130 | If running integrate test for Key Management Service. Require an environment variable `TEST_EX_AWS_KEY_ARN`. Please set new CMK to `TEST_EX_AWS_KEY_ARN` for integrate test. 131 | 132 | ## Creating a New Operation 133 | 134 | If you find that you need an operation not currently in ExAws, please create an issue. 135 | -------------------------------------------------------------------------------- /ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | * Do not use the issues tracker for help or support (try Elixir Forum, Slack, IRC, etc.) 2 | * Questions about how to contribute are fine. 3 | 4 | ### Environment 5 | 6 | * Elixir & Erlang versions (elixir --version): 7 | * ExAws version `mix deps |grep ex_aws` 8 | * HTTP client version. IE for hackney do `mix deps | grep hackney` 9 | 10 | ### Current behavior 11 | 12 | Include code samples, errors and stacktraces if appropriate. 13 | 14 | ### Expected behavior 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2014 CargoSense, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Before opening a PR, please make sure you have: 2 | 3 | * Run `mix format` using a recent version of Elixir 4 | * Run `mix dialyzer` to make sure the typing is correct 5 | * Run `mix test` to ensure no tests have broken (also please make sure you've added tests for your particular change, where appropriate). 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ExAws 2 | 3 | 4 | 5 | ![GitHub Workflow Status](https://img.shields.io/github/workflow/status/ex-aws/ex_aws/on-push) 6 | [![hex.pm](https://img.shields.io/hexpm/v/ex_aws.svg)](https://hex.pm/packages/ex_aws) 7 | [![hex.pm](https://img.shields.io/hexpm/dt/ex_aws.svg)](https://hex.pm/packages/ex_aws) 8 | [![hex.pm](https://img.shields.io/hexpm/l/ex_aws.svg)](https://hex.pm/packages/ex_aws) 9 | [![hexdocs.pm](https://img.shields.io/badge/hexdocs-release-lightgreen.svg)](https://hexdocs.pm/ex_aws) 10 | [![github.com](https://img.shields.io/github/last-commit/ex-aws/ex_aws.svg)](https://github.com/ex-aws/ex_aws/commits/master) 11 | 12 | A flexible easy to use set of AWS APIs. 13 | 14 | Available Services: https://github.com/ex-aws?q=service&type=&language= 15 | 16 | ## Getting Started 17 | 18 | ExAws v2.0 breaks out every service into its own package. To use the S3 19 | service, you need both the core `:ex_aws` package as well as the `:ex_aws_s3` 20 | package. 21 | 22 | As with all ExAws services, you'll need a compatible HTTP client (defaults to 23 | `:hackney`) and whatever JSON or XML codecs needed by the services you want to 24 | use. Consult individual service documentation for details on what each service 25 | needs. 26 | 27 | ```elixir 28 | defp deps do 29 | [ 30 | {:ex_aws, "~> 2.1"}, 31 | {:ex_aws_s3, "~> 2.0"}, 32 | {:hackney, "~> 1.9"}, 33 | {:sweet_xml, "~> 0.6"}, 34 | ] 35 | end 36 | ``` 37 | 38 | With these deps you can use `ExAws` precisely as you're used to: 39 | 40 | ``` 41 | # make a request (with the default region) 42 | ExAws.S3.list_objects("my-bucket") |> ExAws.request() 43 | 44 | # or specify the region 45 | ExAws.S3.list_objects("my-bucket") |> ExAws.request(region: "us-west-1") 46 | 47 | # some operations support streaming 48 | ExAws.S3.list_objects("my-bucket") |> ExAws.stream!() |> Enum.to_list() 49 | ``` 50 | 51 | ### AWS Key configuration 52 | 53 | ExAws requires valid AWS keys in order to work properly. ExAws by default does 54 | the equivalent of: 55 | 56 | ```elixir 57 | config :ex_aws, 58 | access_key_id: [{:system, "AWS_ACCESS_KEY_ID"}, :instance_role], 59 | secret_access_key: [{:system, "AWS_SECRET_ACCESS_KEY"}, :instance_role] 60 | ``` 61 | 62 | This means it will try to resolve credentials in order: 63 | 64 | * Look for the AWS standard `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables 65 | * Resolve credentials with IAM 66 | * If running inside ECS and a [task role](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html) has been assigned it will use it 67 | * Otherwise it will fall back to the [instance role](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) 68 | 69 | AWS CLI config files are supported, but require an additional dependency: 70 | 71 | ```elixir 72 | {:configparser_ex, "~> 4.0"} 73 | ``` 74 | 75 | You can then add `{:awscli, "profile_name", timeout}` to the above config and 76 | it will pull information from `~/.aws/config` and `~/.aws/credentials` 77 | 78 | Alternatively, if you already have a profile name set in the `AWS_PROFILE` environment 79 | variable, you can use that with `{:awscli, :system, timeout}` 80 | 81 | ```elixir 82 | config :ex_aws, 83 | access_key_id: [{:system, "AWS_ACCESS_KEY_ID"}, {:awscli, "default", 30}, :instance_role], 84 | secret_access_key: [{:system, "AWS_SECRET_ACCESS_KEY"}, {:awscli, "default", 30}, :instance_role] 85 | ``` 86 | 87 | For role based authentication via `role_arn` and `source_profile` an additional 88 | dependency is required: 89 | 90 | ```elixir 91 | {:ex_aws_sts, "~> 2.0"} 92 | ``` 93 | 94 | Further information on role based authentication is provided in said dependency. 95 | 96 | #### Session token configuration 97 | 98 | Alternatively, you can also provide `AWS_SESSION_TOKEN` to `security_token` to authenticate 99 | with session token: 100 | 101 | ```elixir 102 | config :ex_aws, 103 | access_key_id: {:system, "AWS_ACCESS_KEY_ID"}, 104 | security_token: {:system, "AWS_SESSION_TOKEN"}, 105 | secret_access_key: {:system, "AWS_SECRET_ACCESS_KEY"} 106 | ``` 107 | 108 | ### Hackney configuration 109 | 110 | ExAws by default uses [hackney](https://github.com/benoitc/hackney) to make 111 | HTTP requests to AWS API. You can modify the options as such: 112 | 113 | ```elixir 114 | config :ex_aws, :hackney_opts, 115 | follow_redirect: true, 116 | recv_timeout: 30_000 117 | ``` 118 | 119 | ### AWS Region Configuration. 120 | 121 | You can set the region used by default for requests. 122 | 123 | ```elixir 124 | config :ex_aws, 125 | region: "us-west-2", 126 | ``` 127 | 128 | Alternatively, the region can be set in an environment variable: 129 | 130 | ```elixir 131 | config :ex_aws, 132 | region: {:system, "AWS_REGION"} 133 | ``` 134 | 135 | ### JSON Codec Configuration 136 | 137 | The default JSON codec is Jason. You can choose a different one: 138 | 139 | ```elixir 140 | config :ex_aws, 141 | json_codec: Poison 142 | ``` 143 | 144 | ### Path Normalization 145 | 146 | Paths that include multiple consecutive /'s will by default be normalized 147 | to a single slash. There are cases when paths need to be literal (S3) and 148 | this normalization behaviour can be turned off via configuration: 149 | 150 | ```elixir 151 | config :ex_aws, 152 | normalize_path: false 153 | ``` 154 | 155 | ## Direct Usage 156 | 157 | ExAws can also be used directly without any specific service module. 158 | 159 | You need to figure out how the API of the specific AWS service works, in particular: 160 | 161 | - Protocol (JSON or query). 162 | - Path (depends on the service and the specific operation, usually "/"). 163 | - Service name (used to generate the request signature, 164 | [as described here](https://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html)). 165 | - Request body, query params, HTTP method, and headers (depends on the service 166 | and specific operation). 167 | 168 | You can look for this information in the service's API reference at 169 | [docs.aws.amazon.com](https://docs.aws.amazon.com/index.html) or, for example, 170 | in the Go SDK API models at [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go/tree/master/models/apis) (look for a `api-*.json` file). 171 | 172 | The protocol dictates which operation module to use for the request. If the 173 | protocol is JSON, use `ExAws.Operation.JSON`, if it's query, use 174 | `ExAws.Operation.Query`. 175 | 176 | ### Examples 177 | 178 | #### Redshift DescribeClusters 179 | 180 | ```elixir 181 | action = :describe_clusters 182 | action_string = action |> Atom.to_string |> Macro.camelize 183 | 184 | operation = 185 | %ExAws.Operation.Query{ 186 | path: "/", 187 | params: %{"Action" => action_string}, 188 | service: :redshift, 189 | action: action 190 | } 191 | 192 | ExAws.request(operation) 193 | ``` 194 | 195 | #### ECS RunTask 196 | 197 | ```elixir 198 | data = %{ 199 | taskDefinition: "hello_world", 200 | launchType: "FARGATE", 201 | networkConfiguration: %{ 202 | awsvpcConfiguration: %{ 203 | subnets: ["subnet-1a2b3c4d", "subnet-4d3c2b1a"], 204 | securityGroups: ["sg-1a2b3c4d"], 205 | assignPublicIp: "ENABLED" 206 | } 207 | } 208 | } 209 | 210 | operation = 211 | %ExAws.Operation.JSON{ 212 | http_method: :post, 213 | headers: [ 214 | {"x-amz-target", "AmazonEC2ContainerServiceV20141113.RunTask"}, 215 | {"content-type", "application/x-amz-json-1.1"} 216 | ], 217 | path: "/", 218 | data: data, 219 | service: :ecs 220 | } 221 | 222 | ExAws.request(operation) 223 | ``` 224 | 225 | ## Highlighted Features 226 | 227 | - Easy configuration. 228 | - Minimal dependencies. Choose your favorite JSON codec and HTTP client. 229 | - Elixir streams to automatically retrieve paginated resources. 230 | - Elixir protocols allow easy customization of Dynamo encoding / decoding. 231 | - Simple. ExAws aims to provide a clear and consistent elixir wrapping around 232 | AWS APIs, not abstract them away entirely. For every action in a given AWS 233 | API there is a corresponding function within the appropriate module. Higher 234 | level abstractions like the aforementioned streams are in addition to and not 235 | instead of basic API calls. 236 | 237 | That's it! 238 | 239 | ## Retries 240 | 241 | ExAws will retry failed AWS API requests using exponential backoff per the 242 | "Full Jitter" formula described in 243 | https://www.awsarchitectureblog.com/2015/03/backoff.html 244 | 245 | The algorithm uses three values, which are configurable: 246 | 247 | ```elixir 248 | # default values shown below 249 | 250 | config :ex_aws, :retries, 251 | max_attempts: 10, 252 | base_backoff_in_ms: 10, 253 | max_backoff_in_ms: 10_000 254 | ``` 255 | 256 | * `max_attempts` is the maximum number of possible attempts with backoffs in between each one 257 | * `max_attempts_client` may be set to a different value for client errors (4xx) (default is `max_attempts`) 258 | * `base_backoff_in_ms` corresponds to the `base` value described in the blog post 259 | * `max_backoff_in_ms` corresponds to the `cap` value described in the blog post 260 | 261 | ## Testing 262 | 263 | If you want to run `mix test`, you'll need to have a local `dynamodb` running 264 | on port 8000: 265 | 266 | ```console 267 | docker run --rm -d -p 8000:8000 amazon/dynamodb-local -jar DynamoDBLocal.jar -port 8000 268 | ``` 269 | 270 | For more info please see [Setting up DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html). 271 | 272 | The redirect test will intentionally cause a warning to be issued. 273 | 274 | ## License 275 | 276 | The MIT License (MIT) 277 | 278 | Copyright (c) 2014-2020 CargoSense, Inc. 279 | 280 | Permission is hereby granted, free of charge, to any person obtaining a copy 281 | of this software and associated documentation files (the "Software"), to deal 282 | in the Software without restriction, including without limitation the rights 283 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 284 | copies of the Software, and to permit persons to whom the Software is 285 | furnished to do so, subject to the following conditions: 286 | 287 | The above copyright notice and this permission notice shall be included in 288 | all copies or substantial portions of the Software. 289 | 290 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 291 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 292 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 293 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 294 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 295 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 296 | THE SOFTWARE. 297 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | config :logger, :console, 4 | level: :debug, 5 | format: "$date $time [$level] $metadata$message\n", 6 | metadata: [:user_id] 7 | 8 | import_config "#{Mix.env()}.exs" 9 | -------------------------------------------------------------------------------- /config/dev.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | config :ex_aws, 4 | debug_requests: true, 5 | access_key_id: [{:system, "AWS_ACCESS_KEY_ID"}, :instance_role], 6 | secret_access_key: [{:system, "AWS_SECRET_ACCESS_KEY"}, :instance_role], 7 | region: "us-east-1" 8 | -------------------------------------------------------------------------------- /config/prod.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | -------------------------------------------------------------------------------- /config/test.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | config :logger, level: :warning 4 | 5 | config :ex_aws, 6 | json_codec: Test.JSONCodec, 7 | access_key_id: "testkeyid", 8 | secret_access_key: "secretaccesskey" 9 | 10 | config :ex_aws, :kinesis, 11 | scheme: "https://", 12 | host: "kinesis.us-east-1.amazonaws.com", 13 | region: "us-east-1", 14 | port: 443 15 | 16 | config :ex_aws, :dynamodb, 17 | scheme: "http://", 18 | host: "localhost", 19 | port: 8000, 20 | region: "us-east-1" 21 | 22 | config :ex_aws, :dynamodb_streams, 23 | scheme: "http://", 24 | host: "localhost", 25 | port: 8000, 26 | region: "us-east-1" 27 | 28 | config :ex_aws, :lambda, 29 | host: "lambda.us-east-1.amazonaws.com", 30 | scheme: "https://", 31 | region: "us-east-1", 32 | port: 443 33 | 34 | config :ex_aws, :s3, 35 | scheme: "https://", 36 | host: "s3.amazonaws.com", 37 | region: "us-east-1" 38 | -------------------------------------------------------------------------------- /lib/ex_aws.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws do 2 | @moduledoc """ 3 | Module for making and processing AWS requests. 4 | """ 5 | 6 | use Application 7 | 8 | @behaviour ExAws.Behaviour 9 | 10 | @doc """ 11 | Perform an AWS request. 12 | 13 | First build an operation from one of the services, and then pass it to this 14 | function to perform it. 15 | 16 | If you want to build an operation manually, see: `ExAws.Operation` 17 | 18 | This function takes an optional second parameter of configuration overrides. 19 | This is useful if you want to have certain configuration changed on a per 20 | request basis. 21 | 22 | Also you can configure telemetry metrics with: 23 | 24 | * `:telemetry_event` - The telemetry event name to dispatch the event under. 25 | Defaults to `[:ex_aws, :request]`. 26 | * `:telemetry_options` - Extra options to attach to telemetry event name. 27 | 28 | ## Examples 29 | 30 | If you have one of the service modules installed, you can just use those service 31 | modules like this: 32 | 33 | ExAws.S3.list_buckets |> ExAws.request 34 | 35 | ExAws.S3.list_buckets |> ExAws.request(region: "eu-west-1") 36 | 37 | ExAws.Dynamo.get_object("users", "foo@bar.com") |> ExAws.request 38 | 39 | Alternatively you can create operation structs manually for services 40 | that aren't supported: 41 | 42 | op = %ExAws.Operation.JSON{ 43 | http_method: :post, 44 | service: :dynamodb, 45 | headers: [ 46 | {"x-amz-target", "DynamoDB_20120810.ListTables"}, 47 | {"content-type", "application/x-amz-json-1.0"} 48 | ], 49 | } 50 | 51 | ExAws.request(op) 52 | 53 | ## Telemetry events 54 | 55 | The following events are published: 56 | 57 | * `[:ex_aws, :request, :start]` - dispatched on start every request sent to the AWS. 58 | * `[:ex_aws, :request, :stop]` - dispatched on every response from AWS. 59 | * `[:ex_aws, :request, :exception]` - dispatched after exceptions on request sent to AWS. 60 | 61 | With `:metadata` map including the following fields: 62 | 63 | * `:result` - the request result: `:ok` or `:error` 64 | * `:attempt` - the attempt number 65 | * `:service` - the AWS service 66 | * `:options` - extra options given to the repo operation under 67 | `:telemetry_options` 68 | 69 | """ 70 | @impl ExAws.Behaviour 71 | @spec request(ExAws.Operation.t(), keyword) :: {:ok, term} | {:error, term} 72 | def request(op, config_overrides \\ []) do 73 | ExAws.Operation.perform(op, ExAws.Config.new(op.service, config_overrides)) 74 | end 75 | 76 | @doc """ 77 | Perform an AWS request, raise if it fails. 78 | 79 | Same as `request/1,2` except it will either return the successful response from 80 | AWS or raise an exception. 81 | """ 82 | @impl ExAws.Behaviour 83 | @spec request!(ExAws.Operation.t(), keyword) :: term 84 | def request!(op, config_overrides \\ []) do 85 | case request(op, config_overrides) do 86 | {:ok, result} -> 87 | result 88 | 89 | error -> 90 | raise ExAws.Error, """ 91 | ExAws Request Error! 92 | 93 | #{inspect(error)} 94 | """ 95 | end 96 | end 97 | 98 | @doc """ 99 | Return a stream for the AWS resource. 100 | 101 | ## Examples 102 | 103 | ExAws.S3.list_objects("my-bucket") |> ExAws.stream! 104 | 105 | """ 106 | @impl ExAws.Behaviour 107 | @spec stream!(ExAws.Operation.t(), keyword) :: Enumerable.t() 108 | def stream!(op, config_overrides \\ []) do 109 | ExAws.Operation.stream!(op, ExAws.Config.new(op.service, config_overrides)) 110 | end 111 | 112 | @doc false 113 | @impl Application 114 | def start(_type, _args) do 115 | children = [ 116 | {ExAws.Config.AuthCache, [name: ExAws.Config.AuthCache]}, 117 | {ExAws.InstanceMetaTokenProvider, [name: ExAws.InstanceMetaTokenProvider]} 118 | ] 119 | 120 | opts = [strategy: :one_for_one, name: ExAws.Supervisor] 121 | Supervisor.start_link(children, opts) 122 | end 123 | end 124 | -------------------------------------------------------------------------------- /lib/ex_aws/auth.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Auth do 2 | import ExAws.Auth.Utils 3 | 4 | alias ExAws.Auth.Credentials 5 | alias ExAws.Auth.Signatures 6 | alias ExAws.Request.Url 7 | 8 | @moduledoc false 9 | 10 | @unsignable_headers ["x-amzn-trace-id"] 11 | @unsignable_headers_multi_case ["x-amzn-trace-id", "X-Amzn-Trace-Id"] 12 | 13 | def validate_config(%{disable_headers_signature: true} = config), 14 | do: {:ok, config} 15 | 16 | def validate_config(config) do 17 | with :ok <- get_key(config, :secret_access_key), 18 | :ok <- get_key(config, :access_key_id) do 19 | {:ok, config} 20 | end 21 | end 22 | 23 | defp get_key(config, key) do 24 | case Map.fetch(config, key) do 25 | :error -> 26 | {:error, "Required key: #{inspect(key)} not found in config!"} 27 | 28 | {:ok, nil} -> 29 | {:error, "Required key: #{inspect(key)} is nil in config!"} 30 | 31 | {:ok, val} when is_binary(val) -> 32 | :ok 33 | 34 | {:ok, val} -> 35 | {:error, "Required key: #{inspect(key)} must be a string, but instead is #{inspect(val)}"} 36 | end 37 | end 38 | 39 | def headers(_http_method, _url, _service, %{disable_headers_signature: true}, headers, _body), 40 | do: {:ok, headers} 41 | 42 | def headers(http_method, url, service, config, headers, body) do 43 | with {:ok, config} <- validate_config(config) do 44 | datetime = :calendar.universal_time() 45 | 46 | headers = 47 | [ 48 | {"host", URI.parse(url).authority}, 49 | {"x-amz-date", amz_date(datetime)} 50 | | headers 51 | ] 52 | |> handle_temp_credentials(config) 53 | 54 | auth_header = 55 | auth_header( 56 | http_method, 57 | url, 58 | headers, 59 | body, 60 | service |> service_override(config) |> service_name, 61 | datetime, 62 | config 63 | ) 64 | 65 | {:ok, [{"Authorization", auth_header} | headers]} 66 | end 67 | end 68 | 69 | def presigned_url( 70 | http_method, 71 | url, 72 | service, 73 | datetime, 74 | config, 75 | expires, 76 | query_params \\ [], 77 | body \\ nil, 78 | headers \\ [] 79 | ) do 80 | with {:ok, config} <- validate_config(config) do 81 | service = service_name(service) 82 | signed_headers = presigned_url_headers(url, headers) 83 | 84 | uri = URI.parse(url) 85 | uri_query = query_from_parsed_uri(uri) 86 | 87 | org_query_params = 88 | Enum.reduce(query_params, uri_query, fn {k, v}, acc -> [{to_string(k), v} | acc] end) 89 | 90 | amz_query_params = 91 | build_amz_query_params(service, datetime, config, expires, signed_headers) 92 | 93 | query_to_sign = (org_query_params ++ amz_query_params) |> canonical_query_params() 94 | 95 | amz_query_string = canonical_query_params(amz_query_params) 96 | 97 | query_for_url = 98 | if Enum.any?(org_query_params) do 99 | canonical_query_params(org_query_params) <> "&" <> amz_query_string 100 | else 101 | amz_query_string 102 | end 103 | 104 | path = url |> Url.get_path(service) |> Url.uri_encode() 105 | 106 | signature = 107 | signature( 108 | http_method, 109 | url, 110 | query_to_sign, 111 | signed_headers, 112 | body, 113 | service, 114 | datetime, 115 | config 116 | ) 117 | 118 | {:ok, 119 | "#{uri.scheme}://#{uri.authority}#{path}?#{query_for_url}&X-Amz-Signature=#{signature}"} 120 | end 121 | end 122 | 123 | defp handle_temp_credentials(headers, %{security_token: token}) do 124 | [{"X-Amz-Security-Token", token} | headers] 125 | end 126 | 127 | defp handle_temp_credentials(headers, _), do: headers 128 | 129 | defp auth_header(http_method, url, headers, body, service, datetime, config) do 130 | query = 131 | url 132 | |> URI.parse() 133 | |> query_from_parsed_uri() 134 | |> canonical_query_params() 135 | 136 | signature = signature(http_method, url, query, headers, body, service, datetime, config) 137 | 138 | [ 139 | "AWS4-HMAC-SHA256 Credential=", 140 | Credentials.generate_credential_v4(service, config, datetime), 141 | ",", 142 | "SignedHeaders=", 143 | signed_headers(headers), 144 | ",", 145 | "Signature=", 146 | signature 147 | ] 148 | |> IO.iodata_to_binary() 149 | end 150 | 151 | defp query_from_parsed_uri(%{query: nil}), do: [] 152 | 153 | defp query_from_parsed_uri(%{query: query_string}) do 154 | query_string 155 | |> URI.decode_query() 156 | |> Enum.to_list() 157 | end 158 | 159 | defp signature(http_method, url, query, headers, body, service, datetime, config) do 160 | path = url |> Url.get_path(service) |> Url.uri_encode() 161 | request = build_canonical_request(http_method, path, query, headers, body) 162 | string_to_sign = string_to_sign(request, service, datetime, config) 163 | Signatures.generate_signature_v4(service, config, datetime, string_to_sign) 164 | end 165 | 166 | def build_canonical_request(http_method, path, query, headers, body) do 167 | http_method = http_method |> method_string |> String.upcase() 168 | 169 | headers = headers |> canonical_headers 170 | 171 | header_string = 172 | headers 173 | |> Enum.map(fn {k, v} -> "#{k}:#{remove_dup_spaces(to_string(v))}" end) 174 | |> Enum.join("\n") 175 | 176 | signed_headers_list = signed_headers_value(headers) 177 | 178 | payload = 179 | case body do 180 | nil -> "UNSIGNED-PAYLOAD" 181 | _ -> ExAws.Auth.Utils.hash_sha256(body) 182 | end 183 | 184 | [ 185 | http_method, 186 | "\n", 187 | path, 188 | "\n", 189 | query, 190 | "\n", 191 | header_string, 192 | "\n", 193 | "\n", 194 | signed_headers_list, 195 | "\n", 196 | payload 197 | ] 198 | |> IO.iodata_to_binary() 199 | end 200 | 201 | defp remove_dup_spaces(str), do: remove_dup_spaces(str, "") 202 | defp remove_dup_spaces(str, str), do: str 203 | 204 | defp remove_dup_spaces(str, _last), 205 | do: str |> String.replace(" ", " ") |> remove_dup_spaces(str) 206 | 207 | defp string_to_sign(request, service, datetime, config) do 208 | request = hash_sha256(request) 209 | 210 | """ 211 | AWS4-HMAC-SHA256 212 | #{amz_date(datetime)} 213 | #{Credentials.generate_credential_scope_v4(service, config, datetime)} 214 | #{request} 215 | """ 216 | |> String.trim_trailing() 217 | end 218 | 219 | defp signed_headers(headers) do 220 | headers 221 | |> Enum.map(fn {k, _} -> String.downcase(k) end) 222 | |> Kernel.--(@unsignable_headers) 223 | |> Enum.sort(&(&1 < &2)) 224 | |> Enum.join(";") 225 | end 226 | 227 | defp canonical_query_params(params) do 228 | params 229 | |> Enum.sort(&compare_query_params/2) 230 | |> Enum.map_join("&", &pair/1) 231 | end 232 | 233 | defp compare_query_params({key, value1}, {key, value2}), do: value1 < value2 234 | defp compare_query_params({key_1, _}, {key_2, _}), do: key_1 < key_2 235 | 236 | defp pair({k, _}) when is_list(k) do 237 | raise ArgumentError, "encode_query/1 keys cannot be lists, got: #{inspect(k)}" 238 | end 239 | 240 | defp pair({_, v}) when is_list(v) do 241 | raise ArgumentError, "encode_query/1 values cannot be lists, got: #{inspect(v)}" 242 | end 243 | 244 | defp pair({k, v}) do 245 | URI.encode_www_form(Kernel.to_string(k)) <> "=" <> aws_encode_www_form(Kernel.to_string(v)) 246 | end 247 | 248 | # is basically the same as URI.encode_www_form 249 | # but doesn't use %20 instead of "+" 250 | def aws_encode_www_form(str) when is_binary(str) do 251 | import Bitwise 252 | 253 | for <>, into: "" do 254 | case URI.char_unreserved?(c) do 255 | true -> <> 256 | false -> "%" <> hex(bsr(c, 4)) <> hex(band(c, 15)) 257 | end 258 | end 259 | end 260 | 261 | defp hex(n) when n <= 9, do: <> 262 | defp hex(n), do: <> 263 | 264 | defp canonical_headers(headers) do 265 | headers 266 | |> Enum.reduce([], fn 267 | {k, _v}, acc when k in @unsignable_headers_multi_case -> acc 268 | {k, v}, acc when is_binary(v) -> [{String.downcase(to_string(k)), String.trim(v)} | acc] 269 | {k, v}, acc -> [{String.downcase(to_string(k)), v} | acc] 270 | end) 271 | |> Enum.sort(fn {k1, _}, {k2, _} -> k1 < k2 end) 272 | end 273 | 274 | defp presigned_url_headers(url, headers) do 275 | uri = URI.parse(url) 276 | canonical_headers([{"host", uri.authority} | headers]) 277 | end 278 | 279 | defp build_amz_query_params(service, datetime, config, expires, signed_headers) do 280 | [ 281 | {"X-Amz-Algorithm", "AWS4-HMAC-SHA256"}, 282 | {"X-Amz-Credential", Credentials.generate_credential_v4(service, config, datetime)}, 283 | {"X-Amz-Date", amz_date(datetime)}, 284 | {"X-Amz-Expires", expires}, 285 | {"X-Amz-SignedHeaders", signed_headers_value(signed_headers)} 286 | ] ++ 287 | if config[:security_token] do 288 | [{"X-Amz-Security-Token", config[:security_token]}] 289 | else 290 | [] 291 | end 292 | end 293 | 294 | defp signed_headers_value(headers) do 295 | headers 296 | |> Enum.map(&elem(&1, 0)) 297 | |> Enum.join(";") 298 | end 299 | 300 | defp service_override(service, config) do 301 | if config[:service_override] do 302 | config[:service_override] 303 | else 304 | service 305 | end 306 | end 307 | end 308 | -------------------------------------------------------------------------------- /lib/ex_aws/auth/credentials.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Auth.Credentials do 2 | @moduledoc false 3 | 4 | import ExAws.Auth.Utils, only: [date: 1] 5 | 6 | def generate_credential_v4(service, config, datetime) do 7 | scope = generate_credential_scope_v4(service, config, datetime) 8 | "#{config[:access_key_id]}/#{scope}" 9 | end 10 | 11 | def generate_credential_scope_v4(service, config, datetime) do 12 | "#{date(datetime)}/#{config[:region]}/#{service}/aws4_request" 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/ex_aws/auth/signatures.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Auth.Signatures do 2 | @moduledoc false 3 | import ExAws.Auth.Utils, only: [hmac_sha256: 2, date: 1, bytes_to_hex: 1] 4 | 5 | def generate_signature_v4(service, config, datetime, string_to_sign) do 6 | service 7 | |> signing_key(datetime, config) 8 | |> hmac_sha256(string_to_sign) 9 | |> bytes_to_hex 10 | end 11 | 12 | defp signing_key(service, datetime, config) do 13 | ["AWS4", config[:secret_access_key]] 14 | |> hmac_sha256(date(datetime)) 15 | |> hmac_sha256(config[:region]) 16 | |> hmac_sha256(service) 17 | |> hmac_sha256("aws4_request") 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/ex_aws/auth/utils.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Auth.Utils do 2 | @moduledoc false 3 | 4 | def uri_encode(url), do: ExAws.Request.Url.uri_encode(url) 5 | 6 | def hash_sha256(data) do 7 | :sha256 8 | |> :crypto.hash(data) 9 | |> bytes_to_hex 10 | end 11 | 12 | # :crypto.mac/4 is introduced in Erlang/OTP 22.1 and :crypto.hmac/3 is removed 13 | # in Erlang/OTP 24. The check is needed for backwards compatibility. 14 | # The Code.ensure_loaded/1 call is executed so function_expored?/3 can be used 15 | # to determine which function to use. 16 | Code.ensure_loaded?(:crypto) || IO.warn(":crypto module failed to load") 17 | 18 | case function_exported?(:crypto, :mac, 4) do 19 | true -> 20 | def hmac_sha256(key, data), do: :crypto.mac(:hmac, :sha256, key, data) 21 | 22 | false -> 23 | def hmac_sha256(key, data), do: :crypto.hmac(:sha256, key, data) 24 | end 25 | 26 | def bytes_to_hex(bytes) do 27 | bytes 28 | |> Base.encode16(case: :lower) 29 | end 30 | 31 | def service_name(service), do: service |> Atom.to_string() 32 | 33 | def method_string(method) do 34 | method 35 | |> Atom.to_string() 36 | |> String.upcase() 37 | end 38 | 39 | def date({date, _time}) do 40 | date |> quasi_iso_format 41 | end 42 | 43 | def amz_date({date, time}) do 44 | date = date |> quasi_iso_format 45 | time = time |> quasi_iso_format 46 | 47 | [date, "T", time, "Z"] 48 | |> IO.iodata_to_binary() 49 | end 50 | 51 | def quasi_iso_format({y, m, d}) do 52 | [y, m, d] 53 | |> Enum.map(&Integer.to_string/1) 54 | |> Enum.map(&zero_pad/1) 55 | end 56 | 57 | defp zero_pad(<<_>> = val), do: "0" <> val 58 | defp zero_pad(val), do: val 59 | end 60 | -------------------------------------------------------------------------------- /lib/ex_aws/behaviour.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Behaviour do 2 | @moduledoc """ 3 | A behaviour definition for the core operations of ExAws. 4 | 5 | `ExAws` implements this behaviour. 6 | """ 7 | 8 | @doc "See `ExAws.request/2`." 9 | @callback request(ExAws.Operation.t()) :: {:ok, term} | {:error, term} 10 | 11 | @doc "See `ExAws.request/2`." 12 | @callback request(ExAws.Operation.t(), Keyword.t()) :: {:ok, term} | {:error, term} 13 | 14 | @doc "See `ExAws.request!/2`." 15 | @callback request!(ExAws.Operation.t()) :: term | no_return 16 | 17 | @doc "See `ExAws.request!/2`." 18 | @callback request!(ExAws.Operation.t(), Keyword.t()) :: term | no_return 19 | 20 | @doc "See `ExAws.stream!/2`." 21 | @callback stream!(ExAws.Operation.t()) :: Enumerable.t() 22 | 23 | @doc "See `ExAws.stream!/2`." 24 | @callback stream!(ExAws.Operation.t(), Keyword.t()) :: Enumerable.t() 25 | end 26 | -------------------------------------------------------------------------------- /lib/ex_aws/config.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Config do 2 | @moduledoc """ 3 | Generates the configuration for a service. 4 | 5 | It starts with the defaults for a given environment and then merges in the 6 | common config from the ex_aws config root, and then finally any config 7 | specified for the particular service. 8 | 9 | ## Refreshable fields 10 | 11 | Some fields are marked as refreshable. These fields will be fetched through 12 | the auth cache even if they are passed in as overrides. This is so stale 13 | credentials aren't used, for example, with long running streams. 14 | 15 | This behaviour must be explicitly enabled by passing `refreshable: true` as an option 16 | to Config.new/2 17 | """ 18 | 19 | # TODO: Add proper documentation? 20 | 21 | @common_config [ 22 | :http_client, 23 | :http_opts, 24 | :json_codec, 25 | :access_key_id, 26 | :secret_access_key, 27 | :debug_requests, 28 | :region, 29 | :security_token, 30 | :retries, 31 | :normalize_path, 32 | :telemetry_event, 33 | :telemetry_options 34 | ] 35 | 36 | @instance_role_config [ 37 | :access_key_id, 38 | :secret_access_key, 39 | :security_token 40 | ] 41 | 42 | @awscli_config [ 43 | :source_profile, 44 | :role_arn, 45 | :access_key_id, 46 | :secret_access_key, 47 | :region, 48 | :security_token, 49 | :role_session_name, 50 | :external_id 51 | ] 52 | 53 | @type t :: %{} | Keyword.t() 54 | 55 | @doc """ 56 | Builds a complete set of config for an operation. 57 | 58 | 1. Defaults are pulled from `ExAws.Config.Defaults` 59 | 2. Common values set via e.g `config :ex_aws` are merged in. 60 | 3. Keys set on the individual service e.g `config :ex_aws, :s3` are merged in 61 | 4. Finally, any configuration overrides are merged in 62 | 63 | """ 64 | @spec new(atom, keyword) :: map() 65 | def new(service, opts \\ []) do 66 | overrides = Map.new(opts) 67 | 68 | service 69 | |> build_base(overrides) 70 | |> retrieve_runtime_config() 71 | |> parse_host_for_region() 72 | end 73 | 74 | @doc """ 75 | Builds a minimal HTTP configuration. 76 | """ 77 | def http_config(service, opts \\ []) do 78 | overrides = Map.new(opts) 79 | 80 | build_base(service, overrides) 81 | |> Map.take([:http_client, :http_opts, :json_codec]) 82 | |> retrieve_runtime_config 83 | end 84 | 85 | def build_base(service, overrides \\ %{}) do 86 | common_config = Application.get_all_env(:ex_aws) |> Map.new() |> Map.take(@common_config) 87 | service_config = Application.get_env(:ex_aws, service, []) |> Map.new() 88 | 89 | region = 90 | (Map.get(overrides, :region) || 91 | Map.get(service_config, :region) || 92 | Map.get(common_config, :region) || 93 | "us-east-1") 94 | |> retrieve_runtime_value(%{}) 95 | 96 | defaults = ExAws.Config.Defaults.get(service, region) 97 | 98 | config = 99 | defaults 100 | |> Map.merge(common_config) 101 | |> Map.merge(service_config) 102 | |> add_refreshable_metadata(overrides) 103 | 104 | # (Maybe) do not allow overrides for refreshable config. 105 | overrides = 106 | if refreshable = config[:refreshable] do 107 | Enum.reduce(refreshable, overrides, fn 108 | :awscli, overrides -> Map.drop(overrides, @awscli_config) 109 | :instance_role, overrides -> Map.drop(overrides, @instance_role_config) 110 | end) 111 | else 112 | overrides 113 | end 114 | 115 | Map.merge(config, overrides) 116 | end 117 | 118 | # :awscli and :instance_role both read creds from ExAws.Config.AuthCache which 119 | # is "refreshable". This is useful for long running streams where the creds can 120 | # change while the stream is still running. 121 | defp add_refreshable_metadata(config, %{refreshable: true}) do 122 | refreshable = 123 | Enum.flat_map(config, fn {_k, v} -> List.wrap(v) end) 124 | |> Enum.reduce([], fn 125 | {:awscli, _, _}, acc -> [:awscli | acc] 126 | :instance_role, acc -> [:instance_role | acc] 127 | _, acc -> acc 128 | end) 129 | |> Enum.uniq() 130 | 131 | if refreshable != [] do 132 | Map.put(config, :refreshable, refreshable) 133 | else 134 | config 135 | end 136 | end 137 | 138 | defp add_refreshable_metadata(config, _overrides) do 139 | config 140 | end 141 | 142 | def retrieve_runtime_config(config) do 143 | Enum.reduce(config, config, fn 144 | {:host, host}, config -> 145 | Map.put(config, :host, retrieve_runtime_value(host, config)) 146 | 147 | {:retries, retries}, config -> 148 | Map.put(config, :retries, retries) 149 | 150 | {:http_opts, http_opts}, config -> 151 | Map.put(config, :http_opts, http_opts) 152 | 153 | {:telemetry_event, telemetry_event}, config -> 154 | Map.put(config, :telemetry_event, telemetry_event) 155 | 156 | {:telemetry_options, telemetry_options}, config -> 157 | Map.put(config, :telemetry_options, telemetry_options) 158 | 159 | {:headers, headers}, config -> 160 | Map.put(config, :headers, headers) 161 | 162 | {:refreshable, refreshable}, config -> 163 | Map.put(config, :refreshable, refreshable) 164 | 165 | {k, v}, config -> 166 | case retrieve_runtime_value(v, config) do 167 | %{} = result -> Map.merge(config, result) 168 | value -> Map.put(config, k, value) 169 | end 170 | end) 171 | end 172 | 173 | def retrieve_runtime_value({:system, env_key}, _) do 174 | System.get_env(env_key) 175 | end 176 | 177 | def retrieve_runtime_value(:instance_role, config) do 178 | config 179 | |> ExAws.Config.AuthCache.get() 180 | |> Map.take(@instance_role_config) 181 | |> valid_map_or_nil 182 | end 183 | 184 | def retrieve_runtime_value({:awscli, profile, expiration}, _) do 185 | ExAws.Config.AuthCache.get(profile, expiration * 1000) 186 | |> Map.take(@awscli_config) 187 | |> valid_map_or_nil 188 | end 189 | 190 | def retrieve_runtime_value(values, config) when is_list(values) do 191 | values 192 | |> Stream.map(&retrieve_runtime_value(&1, config)) 193 | |> Enum.find(& &1) 194 | end 195 | 196 | def retrieve_runtime_value(value, _), do: value 197 | 198 | def parse_host_for_region(%{host: {stub, host}, region: region} = config) do 199 | Map.put(config, :host, String.replace(host, stub, region)) 200 | end 201 | 202 | def parse_host_for_region(%{host: map, region: region} = config) when is_map(map) do 203 | case Map.fetch(map, region) do 204 | {:ok, host} -> Map.put(config, :host, host) 205 | :error -> "A host for region #{region} was not found in host map #{inspect(map)}" 206 | end 207 | end 208 | 209 | def parse_host_for_region(config), do: config 210 | 211 | def awscli_auth_adapter, do: Application.get_env(:ex_aws, :awscli_auth_adapter, nil) 212 | 213 | def awscli_auth_credentials(profile, credentials_ini_provider \\ ExAws.CredentialsIni.File) do 214 | case Application.get_env(:ex_aws, :awscli_credentials, nil) do 215 | nil -> 216 | case credentials_ini_provider.security_credentials(profile) do 217 | {:ok, creds} -> creds 218 | {:error, err} -> raise "Recieved error while retrieving security credentials: #{err}" 219 | end 220 | 221 | %{^profile => profile_credentials} -> 222 | profile_credentials 223 | 224 | _otherwise -> 225 | raise("Missing #{profile} in provided credentials.") 226 | end 227 | end 228 | 229 | defp valid_map_or_nil(map) when map == %{}, do: nil 230 | defp valid_map_or_nil(map), do: map 231 | end 232 | -------------------------------------------------------------------------------- /lib/ex_aws/config/auth_cache.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Config.AuthCache do 2 | @moduledoc false 3 | 4 | use GenServer 5 | 6 | # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html 7 | 8 | @refresh_lead_time 300_000 9 | @instance_auth_key :aws_instance_auth 10 | 11 | defmodule AuthConfigAdapter do 12 | @moduledoc false 13 | 14 | @doc "Compute the awscli auth information." 15 | @callback adapt_auth_config(auth :: map, profile :: String.t(), expiration :: integer) :: any 16 | end 17 | 18 | def start_link(opts \\ []) do 19 | GenServer.start_link(__MODULE__, :ok, Keyword.put(opts, :name, __MODULE__)) 20 | end 21 | 22 | def get(config) do 23 | :ets.lookup(__MODULE__, @instance_auth_key) 24 | |> refresh_auth_if_required(config) 25 | end 26 | 27 | def get(profile, expiration) do 28 | case :ets.lookup(__MODULE__, {:awscli, profile}) do 29 | [{{:awscli, ^profile}, auth_config}] -> 30 | auth_config 31 | 32 | [] -> 33 | GenServer.call(__MODULE__, {:refresh_awscli_config, profile, expiration}, 30_000) 34 | end 35 | end 36 | 37 | ## Callbacks 38 | 39 | def init(:ok) do 40 | ets = :ets.new(__MODULE__, [:named_table, read_concurrency: true]) 41 | {:ok, ets} 42 | end 43 | 44 | def handle_call({:refresh_auth, config}, _from, ets) do 45 | auth = refresh_auth(config, ets) 46 | {:reply, auth, ets} 47 | end 48 | 49 | def handle_call({:refresh_awscli_config, profile, expiration}, _from, ets) do 50 | auth = refresh_awscli_config(profile, expiration, ets) 51 | {:reply, auth, ets} 52 | end 53 | 54 | def handle_info({:refresh_auth, config}, ets) do 55 | refresh_auth(config, ets) 56 | {:noreply, ets} 57 | end 58 | 59 | def handle_info({:refresh_awscli_config, profile, expiration}, ets) do 60 | refresh_awscli_config(profile, expiration, ets) 61 | {:noreply, ets} 62 | end 63 | 64 | def refresh_awscli_config(profile, expiration, ets) do 65 | auth = ExAws.Config.awscli_auth_credentials(profile) 66 | 67 | auth = 68 | case ExAws.Config.awscli_auth_adapter() do 69 | nil -> 70 | auth 71 | 72 | adapter -> 73 | attempt_credentials_refresh(adapter, auth, profile, expiration) 74 | end 75 | 76 | Process.send_after(self(), {:refresh_awscli_config, profile, expiration}, expiration) 77 | :ets.insert(ets, {{:awscli, profile}, auth}) 78 | 79 | auth 80 | end 81 | 82 | defp attempt_credentials_refresh(adapter, auth, profile, expiration, retries \\ 6) do 83 | case adapter.adapt_auth_config(auth, profile, expiration) do 84 | {:error, error} when retries == 1 -> 85 | Process.send_after(self(), {:refresh_awscli_config, profile, expiration}, expiration) 86 | 87 | raise "Could't get credentials from auth adapter after 6 retries, last error was #{inspect(error)}" 88 | 89 | {:error, _error} -> 90 | Process.sleep(:rand.uniform(5_000)) 91 | attempt_credentials_refresh(adapter, auth, profile, expiration, retries - 1) 92 | 93 | # Always store a map on AuthCache 94 | auth when is_map(auth) -> 95 | auth 96 | end 97 | end 98 | 99 | defp refresh_auth_if_required([], config) do 100 | GenServer.call(__MODULE__, {:refresh_auth, config}, 30_000) 101 | end 102 | 103 | defp refresh_auth_if_required([{_key, cached_auth}], config) do 104 | if next_refresh_in(cached_auth) > 0 do 105 | cached_auth 106 | else 107 | GenServer.call(__MODULE__, {:refresh_auth, config}, 30_000) 108 | end 109 | end 110 | 111 | defp refresh_auth(config, ets) do 112 | :ets.lookup(__MODULE__, @instance_auth_key) 113 | |> refresh_auth_if_stale(config, ets) 114 | end 115 | 116 | defp refresh_auth_if_stale([], config, ets) do 117 | refresh_auth_now(config, ets) 118 | end 119 | 120 | defp refresh_auth_if_stale([{_key, cached_auth}], config, ets) do 121 | if next_refresh_in(cached_auth) > @refresh_lead_time do 122 | # we still have a valid auth token, so simply return that 123 | cached_auth 124 | else 125 | refresh_auth_now(config, ets) 126 | end 127 | end 128 | 129 | defp refresh_auth_if_stale(_, config, ets), do: refresh_auth_now(config, ets) 130 | 131 | defp refresh_auth_now(config, ets) do 132 | auth = ExAws.InstanceMeta.security_credentials(config) 133 | :ets.insert(ets, {@instance_auth_key, auth}) 134 | Process.send_after(__MODULE__, {:refresh_auth, config}, next_refresh_in(auth)) 135 | auth 136 | end 137 | 138 | defp next_refresh_in(%{expiration: expiration}) do 139 | try do 140 | expires_in_ms = 141 | expiration 142 | |> NaiveDateTime.from_iso8601!() 143 | |> NaiveDateTime.diff(NaiveDateTime.utc_now(), :millisecond) 144 | 145 | # refresh lead_time before auth expires, unless the time has passed 146 | # otherwise refresh needed now 147 | max(0, expires_in_ms - @refresh_lead_time) 148 | rescue 149 | _e -> 0 150 | end 151 | end 152 | 153 | defp next_refresh_in(_), do: 0 154 | end 155 | -------------------------------------------------------------------------------- /lib/ex_aws/config/defaults.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Config.Defaults do 2 | @moduledoc """ 3 | Defaults for each service 4 | """ 5 | 6 | @common %{ 7 | access_key_id: [{:system, "AWS_ACCESS_KEY_ID"}, :instance_role], 8 | secret_access_key: [{:system, "AWS_SECRET_ACCESS_KEY"}, :instance_role], 9 | http_client: ExAws.Request.Hackney, 10 | json_codec: Jason, 11 | retries: [ 12 | max_attempts: 10, 13 | base_backoff_in_ms: 10, 14 | max_backoff_in_ms: 10_000 15 | ], 16 | require_imds_v2: false, 17 | normalize_path: true 18 | } 19 | 20 | @doc """ 21 | Retrieve the default configuration for a service. 22 | """ 23 | @spec defaults(service :: atom) :: map 24 | 25 | def defaults(:dynamodb_streams) do 26 | %{service_override: :dynamodb} 27 | |> Map.merge(defaults(:dynamodb)) 28 | end 29 | 30 | def defaults(:lex_runtime) do 31 | %{service_override: :lex} 32 | |> Map.merge(defaults(:lex)) 33 | end 34 | 35 | def defaults(:lex_models) do 36 | %{service_override: :lex} 37 | |> Map.merge(defaults(:lex)) 38 | end 39 | 40 | def defaults(:"personalize-runtime") do 41 | %{service_override: :personalize} 42 | |> Map.merge(defaults(:personalize)) 43 | end 44 | 45 | def defaults(:"personalize-events") do 46 | %{service_override: :personalize} 47 | |> Map.merge(defaults(:personalize)) 48 | end 49 | 50 | def defaults(:sagemaker_runtime) do 51 | %{service_override: :sagemaker} 52 | |> Map.merge(defaults(:sagemaker)) 53 | end 54 | 55 | def defaults(:sagemaker_runtime_a2i) do 56 | %{service_override: :sagemaker} 57 | |> Map.merge(defaults(:sagemaker)) 58 | end 59 | 60 | def defaults(:iot_data) do 61 | %{service_override: :iotdata} 62 | |> Map.merge(defaults(:iot)) 63 | end 64 | 65 | def defaults(:"session.qldb") do 66 | %{service_override: :qldb} 67 | |> Map.merge(defaults(:qldb)) 68 | end 69 | 70 | def defaults(:ingest_timestream) do 71 | %{service_override: :timestream} 72 | |> Map.merge(defaults(:timestream)) 73 | end 74 | 75 | def defaults(:query_timestream) do 76 | %{service_override: :timestream} 77 | |> Map.merge(defaults(:timestream)) 78 | end 79 | 80 | def defaults(service) when service in [:places, :maps, :geofencing, :tracking, :routes] do 81 | %{service_override: :geo} 82 | |> Map.merge(defaults(:geo)) 83 | end 84 | 85 | def defaults(chime_service) 86 | when chime_service in [ 87 | :"chime-sdk-media-pipelines", 88 | :"chime-sdk-identity", 89 | :"chime-sdk-meetings", 90 | :"chime-sdk-voice" 91 | ] do 92 | %{service_override: :chime} 93 | |> Map.merge(defaults(:chime)) 94 | end 95 | 96 | def defaults(_) do 97 | Map.merge( 98 | %{ 99 | scheme: "https://", 100 | region: "us-east-1", 101 | port: 443 102 | }, 103 | @common 104 | ) 105 | end 106 | 107 | def get(service, region) do 108 | service 109 | |> defaults 110 | |> Map.put(:host, host(service, region)) 111 | end 112 | 113 | def host(service, region) do 114 | partition = 115 | Enum.find(partitions(), fn {regex, _} -> 116 | Regex.run(regex, region) 117 | end) 118 | 119 | with {_, partition} <- partition do 120 | do_host(partition, service, region) 121 | end 122 | end 123 | 124 | defp partitions(), 125 | do: [ 126 | {~r/^(us|eu|af|ap|sa|ca|me)\-\w+-\d?-?\w+$/, "aws"}, 127 | {~r/^cn\-\w+\-\d+$/, "aws-cn"}, 128 | {~r/^us\-gov\-\w+\-\d+$/, "aws-us-gov"} 129 | ] 130 | 131 | defp service_map(:ses), do: "email" 132 | defp service_map(:sagemaker_runtime), do: "runtime.sagemaker" 133 | defp service_map(:sagemaker_runtime_a2i), do: "a2i-runtime.sagemaker" 134 | defp service_map(:lex_runtime), do: "runtime.lex" 135 | defp service_map(:lex_models), do: "models.lex" 136 | defp service_map(:dynamodb_streams), do: "streams.dynamodb" 137 | defp service_map(:iot_data), do: "data.iot" 138 | defp service_map(:ingest_timestream), do: "ingest.timestream" 139 | defp service_map(:query_timestream), do: "query.timestream" 140 | defp service_map(:places), do: "places.geo" 141 | defp service_map(:maps), do: "maps.geo" 142 | defp service_map(:geofencing), do: "geofencing.geo" 143 | defp service_map(:tracking), do: "tracking.geo" 144 | defp service_map(:routes), do: "routes.geo" 145 | 146 | defp service_map(service) do 147 | service 148 | |> to_string 149 | |> String.replace("_", "-") 150 | end 151 | 152 | @external_resource "priv/endpoints.exs" 153 | 154 | @partition_data Code.eval_file("priv/endpoints.exs", File.cwd!()) 155 | |> elem(0) 156 | |> Map.get("partitions") 157 | |> Map.new(fn partition -> 158 | {partition["partition"], partition} 159 | end) 160 | 161 | defp do_host(partition, service_slug, region) do 162 | partition = @partition_data |> Map.fetch!(partition) 163 | partition_name = partition["partition"] 164 | service = service_map(service_slug) 165 | 166 | partition 167 | |> Map.fetch!("services") 168 | |> fetch_or(service, "#{service_slug} not found in partition #{partition_name}") 169 | |> case do 170 | %{"isRegionalized" => false} = data -> 171 | data 172 | |> Map.fetch!("endpoints") 173 | |> Map.values() 174 | |> List.first() 175 | 176 | data -> 177 | data 178 | |> Map.fetch!("endpoints") 179 | |> fetch_or( 180 | region, 181 | "#{service_slug} not supported in region #{region} for partition #{partition_name}" 182 | ) 183 | end 184 | |> case do 185 | %{"hostname" => hostname} -> 186 | hostname 187 | 188 | _ -> 189 | dns_suffix = Map.fetch!(partition, "dnsSuffix") 190 | hostname = Map.fetch!(partition, "defaults") |> Map.fetch!("hostname") 191 | apply_defaults(hostname, service, region, dns_suffix) 192 | end 193 | end 194 | 195 | defp fetch_or(map, key, msg) do 196 | Map.get(map, key) || raise msg 197 | end 198 | 199 | def apply_defaults(hostname, service, region, dns_suffix) do 200 | hostname 201 | |> String.replace("{service}", service) 202 | |> String.replace("{region}", region) 203 | |> String.replace("{dnsSuffix}", dns_suffix) 204 | end 205 | end 206 | -------------------------------------------------------------------------------- /lib/ex_aws/credentials_ini/file.ex: -------------------------------------------------------------------------------- 1 | if Code.ensure_loaded?(ConfigParser) do 2 | defmodule ExAws.CredentialsIni.File do 3 | @moduledoc false 4 | 5 | # as per https://docs.aws.amazon.com/cli/latest/topic/config-vars.html 6 | @valid_config_keys ~w( 7 | aws_access_key_id aws_secret_access_key aws_session_token region 8 | role_arn source_profile credential_source external_id mfa_serial role_session_name credential_process 9 | sso_start_url sso_region sso_account_id sso_role_name sso_session 10 | ) 11 | 12 | @special_merge_keys ~w(sso_session) 13 | 14 | def security_credentials(profile_name) do 15 | config_credentials = profile_from_config(profile_name) 16 | shared_credentials = profile_from_shared_credentials(profile_name) 17 | 18 | case config_credentials do 19 | %{ 20 | sso_start_url: sso_start_url, 21 | sso_account_id: sso_account_id, 22 | sso_role_name: sso_role_name 23 | } -> 24 | sso_cache_key = Map.get(config_credentials, :sso_session, sso_start_url) 25 | config = ExAws.Config.http_config(:sso) 26 | 27 | case get_sso_role_credentials(sso_cache_key, sso_account_id, sso_role_name, config) do 28 | {:ok, sso_creds} -> {:ok, Map.merge(sso_creds, shared_credentials)} 29 | {:error, _} = err -> err 30 | end 31 | 32 | %{credential_process: credential_process} -> 33 | config = ExAws.Config.http_config(:sso) 34 | 35 | case get_credentials_from_process(credential_process, config) do 36 | {:ok, credentials} -> {:ok, Map.merge(credentials, shared_credentials)} 37 | {:error, _} = err -> err 38 | end 39 | 40 | _ -> 41 | {:ok, Map.merge(config_credentials, shared_credentials)} 42 | end 43 | end 44 | 45 | defp get_sso_role_credentials(sso_cache_key, sso_account_id, sso_role_name, config) do 46 | with {_, {:ok, sso_cache_content}} <- 47 | {:read, File.read(get_sso_cache_file(sso_cache_key))}, 48 | {_, 49 | {:ok, %{"expiresAt" => expires_at, "accessToken" => access_token, "region" => region}}} <- 50 | {:decode, config[:json_codec].decode(sso_cache_content)}, 51 | {_, :ok} <- 52 | {:expiration, check_sso_expiration(expires_at)}, 53 | {_, {:ok, sso_creds}} <- 54 | {:sso_creds, 55 | request_sso_role_credentials( 56 | access_token, 57 | region, 58 | sso_account_id, 59 | sso_role_name, 60 | config 61 | )}, 62 | {_, {:ok, reformatted_creds}} <- 63 | {:rename, rename_sso_credential_keys(sso_creds)} do 64 | {:ok, reformatted_creds} 65 | else 66 | {:read, {:error, error}} -> {:error, "Could not read SSO cache file: #{error}"} 67 | {:decode, _} -> {:error, "SSO cache file contains invalid json"} 68 | {:expiration, error} -> error 69 | {:sso_creds, error} -> error 70 | {:rename, error} -> error 71 | end 72 | end 73 | 74 | defp get_sso_cache_file(sso_cache_key) do 75 | hash = :crypto.hash(:sha, sso_cache_key) |> Base.encode16() |> String.downcase() 76 | 77 | System.user_home() 78 | |> Path.join(".aws/sso/cache/#{hash}.json") 79 | end 80 | 81 | defp check_sso_expiration(expires_at_str) do 82 | with {:ok, _} <- check_expiration(expires_at_str) do 83 | :ok 84 | else 85 | {:timestamp, {:error, err}} -> 86 | {:error, "SSO cache file has invalid expiration format: #{err}"} 87 | 88 | {:expires, _} -> 89 | {:error, "SSO access token is expired, refresh the token with `aws sso login`"} 90 | end 91 | end 92 | 93 | defp request_sso_role_credentials( 94 | access_token, 95 | region, 96 | account_id, 97 | role_name, 98 | config 99 | ) do 100 | with {_, {:ok, %{status_code: 200, headers: _headers, body: body_raw}}} <- 101 | {:request, 102 | config[:http_client].request( 103 | :get, 104 | "https://portal.sso.#{region}.amazonaws.com/federation/credentials?account_id=#{account_id}&role_name=#{role_name}", 105 | "", 106 | [{"x-amz-sso_bearer_token", access_token}], 107 | Map.get(config, :http_opts, []) 108 | ) 109 | |> ExAws.Request.maybe_transform_response()}, 110 | {_, {:ok, body}} <- {:decode, config[:json_codec].decode(body_raw)} do 111 | {:ok, body} 112 | else 113 | {:request, {_, %{status_code: status_code} = resp}} -> 114 | {:error, "SSO role credentials request responded with #{status_code}: #{resp}"} 115 | 116 | {:decode, err} -> 117 | {:error, "Could not decode SSO role credentials response: #{err}"} 118 | end 119 | end 120 | 121 | defp rename_sso_credential_keys(%{"roleCredentials" => role_credentials}) do 122 | with {_, access_key} when not is_nil(access_key) <- 123 | {:accessKey, Map.get(role_credentials, "accessKeyId")}, 124 | {_, expiration} when not is_nil(expiration) <- 125 | {:expiration, Map.get(role_credentials, "expiration")}, 126 | {_, secret_access_key} when not is_nil(secret_access_key) <- 127 | {:secretAccess, Map.get(role_credentials, "secretAccessKey")}, 128 | {_, session_token} when not is_nil(session_token) <- 129 | {:sessionToken, Map.get(role_credentials, "sessionToken")} do 130 | {:ok, 131 | %{ 132 | access_key_id: access_key, 133 | expiration: expiration, 134 | secret_access_key: secret_access_key, 135 | security_token: session_token 136 | }} 137 | else 138 | {missing, _} -> {:error, "#{missing} is missing from SSO role credential response"} 139 | end 140 | end 141 | 142 | defp get_credentials_from_process(credential_process, config) do 143 | with {_, {:ok, process_result}} <- 144 | {:process, execute_process(credential_process)}, 145 | {_, {:ok, %{"Version" => 1} = result}} <- 146 | {:decode, config[:json_codec].decode(process_result)}, 147 | {_, {:ok, expiration}} <- 148 | {:expiration, check_credentials_expiration(result)}, 149 | {_, {:ok, reformatted_creds}} <- 150 | {:rename, format_result(result, expiration)} do 151 | {:ok, reformatted_creds} 152 | else 153 | {:process, {:error, error}} -> {:error, "Could not execute process: #{error}"} 154 | {:decode, _} -> {:error, "Credentials process results contains invalid json"} 155 | {:expiration, error} -> error 156 | {:rename, error} -> error 157 | end 158 | end 159 | 160 | defp execute_process(credential_process) do 161 | with [command | args] <- String.split(credential_process), 162 | {result, 0} <- System.cmd(command, args, stderr_to_stdout: true) do 163 | {:ok, result} 164 | else 165 | [] -> {:error, "Could not read command from config file : #{credential_process}"} 166 | {error, exit_code} -> {:error, "Exit code : #{exit_code} - #{error}"} 167 | end 168 | end 169 | 170 | defp format_result(result, nil) do 171 | with {_, access_key} when not is_nil(access_key) <- 172 | {:accessKey, Map.get(result, "AccessKeyId")}, 173 | {_, secret_access_key} when not is_nil(secret_access_key) <- 174 | {:secretAccess, Map.get(result, "SecretAccessKey")} do 175 | {:ok, 176 | %{ 177 | access_key_id: access_key, 178 | secret_access_key: secret_access_key 179 | }} 180 | else 181 | {missing, _} -> {:error, "#{missing} is missing from credentials process response"} 182 | end 183 | end 184 | 185 | defp format_result(result, expiration) do 186 | with {_, access_key} when not is_nil(access_key) <- 187 | {:accessKey, Map.get(result, "AccessKeyId")}, 188 | {_, secret_access_key} when not is_nil(secret_access_key) <- 189 | {:secretAccess, Map.get(result, "SecretAccessKey")}, 190 | {_, session_token} when not is_nil(session_token) <- 191 | {:sessionToken, Map.get(result, "SessionToken")} do 192 | {:ok, 193 | %{ 194 | access_key_id: access_key, 195 | expiration: DateTime.to_unix(expiration), 196 | secret_access_key: secret_access_key, 197 | security_token: session_token 198 | }} 199 | else 200 | {missing, _} -> {:error, "#{missing} is missing from credentials process response"} 201 | end 202 | end 203 | 204 | defp check_credentials_expiration(%{"Expiration" => expiration_str}) do 205 | with {:ok, expiration} <- check_expiration(expiration_str) do 206 | {:ok, expiration} 207 | else 208 | {:timestamp, {:error, err}} -> 209 | {:error, "Process returned invalid expiration format: #{err}"} 210 | 211 | {:expires, _} -> 212 | {:error, "Process returned expired credentials"} 213 | end 214 | end 215 | 216 | defp check_credentials_expiration(_), do: {:ok, nil} 217 | 218 | defp check_expiration(expiration_str) do 219 | with {_, {:ok, expiration, _}} <- {:timestamp, DateTime.from_iso8601(expiration_str)}, 220 | {_, :gt} <- {:expires, DateTime.compare(expiration, DateTime.utc_now())} do 221 | {:ok, expiration} 222 | end 223 | end 224 | 225 | def parse_ini_file({:ok, contents}, :system) do 226 | parse_ini_file({:ok, contents}, profile_name_from_env()) 227 | end 228 | 229 | def parse_ini_file({:ok, contents}, profile_name) do 230 | composite_key = "profile " <> profile_name 231 | 232 | contents 233 | |> ConfigParser.parse_string() 234 | |> case do 235 | {:ok, %{^profile_name => config} = full} -> 236 | merge_special_keys(full, config) 237 | |> strip_key_prefix() 238 | 239 | {:ok, %{^composite_key => config} = full} -> 240 | merge_special_keys(full, config) 241 | |> strip_key_prefix() 242 | 243 | {:ok, %{}} -> 244 | %{} 245 | 246 | _ -> 247 | %{} 248 | end 249 | end 250 | 251 | def parse_ini_file(_, _), do: %{} 252 | 253 | def merge_special_keys(full_config, credentials) do 254 | credentials 255 | |> Map.take(@special_merge_keys) 256 | |> Enum.reduce(credentials, fn {key, val}, acc -> 257 | merge_section = "#{String.replace(key, "_", "-")} #{val}" 258 | 259 | case full_config do 260 | %{^merge_section => config} -> 261 | Map.merge(config, acc) 262 | 263 | _ -> 264 | acc 265 | end 266 | end) 267 | end 268 | 269 | def strip_key_prefix(credentials) do 270 | credentials 271 | |> Map.take(@valid_config_keys) 272 | |> Map.new(fn {key, val} -> 273 | updated_key = 274 | key 275 | |> String.replace_leading("aws_", "") 276 | |> String.to_atom() 277 | 278 | {updated_key, val} 279 | end) 280 | end 281 | 282 | def replace_token_key(credentials) do 283 | case Map.pop(credentials, :session_token) do 284 | {nil, credentials} -> 285 | credentials 286 | 287 | {token, credentials} -> 288 | Map.put(credentials, :security_token, token) 289 | end 290 | end 291 | 292 | defp profile_from_shared_credentials(profile_name) do 293 | System.user_home() 294 | |> Path.join(".aws/credentials") 295 | |> File.read() 296 | |> parse_ini_file(profile_name) 297 | |> replace_token_key 298 | end 299 | 300 | defp profile_from_config(profile_name) do 301 | section = profile_from_name(profile_name) 302 | 303 | System.user_home() 304 | |> Path.join(".aws/config") 305 | |> File.read() 306 | |> parse_ini_file(section) 307 | end 308 | 309 | defp profile_from_name(:system) do 310 | profile_name_from_env() 311 | |> profile_from_name() 312 | end 313 | 314 | defp profile_from_name("default"), do: "default" 315 | defp profile_from_name(other), do: "profile #{other}" 316 | 317 | defp profile_name_from_env() do 318 | System.get_env("AWS_PROFILE") || "default" 319 | end 320 | end 321 | else 322 | defmodule ExAws.CredentialsIni.File do 323 | @moduledoc false 324 | 325 | def security_credentials(_), do: raise("ConfigParser required to use") 326 | def parse_ini_file(_, _), do: raise("ConfigParser required to use") 327 | def replace_token_key(_), do: raise("ConfigParser required to use") 328 | end 329 | end 330 | -------------------------------------------------------------------------------- /lib/ex_aws/credentials_ini/provider.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.CredentialsIni.Provider do 2 | @moduledoc """ 3 | Specifies expected behaviour of a credentials provider. 4 | 5 | A credentials initializer provider is a module that fetches the AWS credentials from different sources. 6 | """ 7 | 8 | @type profile :: String.t() 9 | @type credentials :: map() 10 | 11 | @callback security_credentials(profile) :: credentials 12 | end 13 | -------------------------------------------------------------------------------- /lib/ex_aws/error.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Error do 2 | @moduledoc """ 3 | A generic AWS error. 4 | """ 5 | 6 | defexception [:message] 7 | end 8 | -------------------------------------------------------------------------------- /lib/ex_aws/instance_meta.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.InstanceMeta do 2 | @moduledoc false 3 | 4 | # Provides access to the AWS Instance MetaData 5 | # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html 6 | 7 | # AWS InstanceMetaData URL 8 | @meta_path_root "http://169.254.169.254/latest/meta-data" 9 | 10 | # Endpoint for ECS tasks role credentials 11 | # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html 12 | @task_role_root "http://169.254.170.2" 13 | 14 | def request(config, url, fallback \\ false) do 15 | # If we're using IMDSv2, we will need to pass in session token headers. 16 | headers = get_request_headers(config, fallback) 17 | 18 | case config.http_client.request(:get, url, "", headers, http_opts()) 19 | |> ExAws.Request.maybe_transform_response() do 20 | {:ok, %{status_code: 200, body: body}} -> 21 | body 22 | 23 | {:ok, %{status_code: status_code}} -> 24 | retry_or_raise(config, url, status_code, fallback) 25 | 26 | error -> 27 | raise """ 28 | Instance Meta Error: #{inspect(error)} 29 | 30 | You tried to access the AWS EC2 instance meta, but it could not be reached. 31 | This happens most often when trying to access it from your local computer, 32 | which happens when environment variables are not set correctly prompting 33 | ExAws to fallback to the Instance Meta. 34 | 35 | Please check your key config and make sure they're configured correctly: 36 | 37 | For Example: 38 | ``` 39 | ExAws.Config.new(:s3) 40 | ExAws.Config.new(:dynamodb) 41 | ``` 42 | """ 43 | end 44 | end 45 | 46 | defp retry_or_raise(config, url, 401, false) do 47 | request(config, url, true) 48 | end 49 | 50 | defp retry_or_raise(_config, _url, status_code, _fallback) do 51 | raise """ 52 | Instance Meta Error: HTTP response status code #{inspect(status_code)} 53 | 54 | Please check AWS EC2 IAM role. 55 | """ 56 | end 57 | 58 | def get_request_headers(config, fallback) do 59 | if fallback || Map.get(config, :require_imds_v2) do 60 | ExAws.InstanceMetaTokenProvider.get_headers(config) 61 | else 62 | [] 63 | end 64 | end 65 | 66 | def instance_role(config) do 67 | ExAws.InstanceMeta.request(config, @meta_path_root <> "/iam/security-credentials/") 68 | end 69 | 70 | def task_role_credentials(config) do 71 | case System.get_env("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") do 72 | nil -> 73 | nil 74 | 75 | uri -> 76 | ExAws.InstanceMeta.request(config, @task_role_root <> uri) 77 | |> config.json_codec.decode! 78 | end 79 | end 80 | 81 | def instance_role_credentials(config) do 82 | ExAws.InstanceMeta.request( 83 | config, 84 | @meta_path_root <> "/iam/security-credentials/#{instance_role(config)}" 85 | ) 86 | |> config.json_codec.decode! 87 | end 88 | 89 | def security_credentials(config) do 90 | result = 91 | case task_role_credentials(config) do 92 | nil -> instance_role_credentials(config) 93 | credentials -> credentials 94 | end 95 | 96 | %{ 97 | access_key_id: result["AccessKeyId"], 98 | secret_access_key: result["SecretAccessKey"], 99 | security_token: result["Token"], 100 | expiration: result["Expiration"] 101 | } 102 | end 103 | 104 | defp http_opts do 105 | # Certain solutions like kube2iam will redirect instance meta requests, 106 | # we need to follow those redirects. 107 | defaults = [follow_redirect: true] 108 | 109 | overrides = 110 | Application.get_env(:ex_aws, :metadata, []) 111 | |> Keyword.get(:http_opts, []) 112 | 113 | Keyword.merge(defaults, overrides) 114 | end 115 | end 116 | -------------------------------------------------------------------------------- /lib/ex_aws/instance_meta_token_provider.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.InstanceMetaTokenProvider do 2 | @moduledoc """ 3 | For use with IMDSv2, this module retrieves the metadata session token and refreshes it before expiration. 4 | """ 5 | 6 | # 6 hours 7 | @metadata_token_ttl_seconds 6 * 60 * 60 8 | @genserver_call_timeout_seconds 30 9 | @metadata_token_api_url "http://169.254.169.254/latest/api/token" 10 | # Endpoint for ECS tasks role credentials 11 | # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html 12 | @task_role_root "http://169.254.170.2" 13 | # The header we pass to control the token's time to live 14 | @metadata_token_ttl_header_name "x-aws-ec2-metadata-token-ttl-seconds" 15 | # The header we use to pass the token along to all other metadata calls 16 | @metadata_token_header_name "x-aws-ec2-metadata-token" 17 | 18 | use GenServer 19 | 20 | def start_link(opts \\ []) do 21 | GenServer.start_link(__MODULE__, :ok, opts) 22 | end 23 | 24 | def get(config) do 25 | case :ets.lookup(__MODULE__, :aws_metadata_token) do 26 | [{:aws_metadata_token, token}] -> 27 | token 28 | 29 | [] -> 30 | GenServer.call( 31 | __MODULE__, 32 | {:refresh_token, config}, 33 | @genserver_call_timeout_seconds * 1_000 34 | ) 35 | end 36 | end 37 | 38 | def get_headers(config) do 39 | [{@metadata_token_header_name, get(config)}] 40 | end 41 | 42 | ## Callbacks 43 | 44 | def init(:ok) do 45 | ets = :ets.new(__MODULE__, [:named_table, read_concurrency: true]) 46 | {:ok, ets} 47 | end 48 | 49 | def handle_call({:refresh_token, config}, _from, ets) do 50 | token = refresh_token(config, ets) 51 | {:reply, token, ets} 52 | end 53 | 54 | def handle_info({:refresh_token, config}, ets) do 55 | refresh_token(config, ets) 56 | {:noreply, ets} 57 | end 58 | 59 | def refresh_token(config, ets) do 60 | token = request_token(config) 61 | 62 | # Setting the :no_metadata_token_cache option in tests ensures we can always expect the token request. 63 | unless config[:no_metadata_token_cache] do 64 | :ets.insert(ets, {:aws_metadata_token, token}) 65 | end 66 | 67 | Process.send_after(self(), {:refresh_token, config}, refresh_in(config)) 68 | 69 | token 70 | end 71 | 72 | def refresh_in(_config) do 73 | # Check five minutes prior to expiration, or now, which ever is later. 74 | refresh_in = @metadata_token_ttl_seconds - 5 * 60 75 | max(0, refresh_in * 1_000) 76 | end 77 | 78 | def request_token(config) do 79 | case config.http_client.request( 80 | :put, 81 | metadata_token_api_url(), 82 | "", 83 | token_ttl_seconds_headers(config), 84 | follow_redirect: true 85 | ) 86 | |> ExAws.Request.maybe_transform_response() do 87 | {:ok, %{status_code: 200, body: body}} -> 88 | body 89 | 90 | {:ok, %{status_code: status_code}} -> 91 | raise """ 92 | Instance Meta Error: HTTP response status code #{inspect(status_code)} 93 | 94 | Please check AWS EC2 Instance Metadata Service configuration to make sure the service is configured properly. 95 | """ 96 | 97 | error -> 98 | raise """ 99 | Instance Meta Error: #{inspect(error)} 100 | 101 | You tried to access the AWS EC2 Instance Metadata token API, but it could not be reached. 102 | 103 | Please check AWS EC2 Instance Metadata Service configuration to make sure the service is enabled. 104 | """ 105 | end 106 | end 107 | 108 | defp metadata_token_api_url do 109 | case System.get_env("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") do 110 | nil -> @metadata_token_api_url 111 | uri -> @task_role_root <> uri 112 | end 113 | end 114 | 115 | defp token_ttl_seconds_headers(_config) do 116 | [{@metadata_token_ttl_header_name, Integer.to_string(@metadata_token_ttl_seconds)}] 117 | end 118 | end 119 | -------------------------------------------------------------------------------- /lib/ex_aws/json/codec.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.JSON.Codec do 2 | @moduledoc """ 3 | Defines the specification for a JSON codec. 4 | 5 | ExAws supports the use of your favorite JSON codec provided it fulfills this specification. 6 | Poison fulfills this spec without modification, and is the default. 7 | 8 | See the contents of `ExAws.JSON.JSX` for an example of an alternative implementation. 9 | ## Example 10 | Here for example is the code required to make jsx comply with this spec. 11 | 12 | In your config you would do: 13 | 14 | config :ex_aws, 15 | json_codec: ExAws.JSON.JSX 16 | 17 | defmodule ExAws.JSON.JSX do 18 | @behaviour ExAws.JSON.Codec 19 | 20 | @moduledoc false 21 | 22 | def encode!(%{} = map) do 23 | map |> :jsx.encode 24 | end 25 | 26 | def encode(map) do 27 | try do 28 | {:ok, encode!(map)} 29 | rescue 30 | ArgumentError -> {:error, :badarg} 31 | end 32 | end 33 | 34 | def decode!(string) do 35 | :jsx.decode(string, [:return_maps]) 36 | end 37 | 38 | def decode(string) do 39 | try do 40 | {:ok, decode!(string)} 41 | rescue 42 | ArgumentError -> {:error, :badarg} 43 | end 44 | end 45 | end 46 | 47 | """ 48 | 49 | @callback encode!(%{}) :: String.t() 50 | @callback encode(%{}) :: {:ok, String.t()} | {:error, String.t()} 51 | 52 | @callback decode!(String.t()) :: %{} 53 | @callback decode(String.t()) :: {:ok, %{}} | {:error, %{}} 54 | end 55 | -------------------------------------------------------------------------------- /lib/ex_aws/json/jsx.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.JSON.JSX do 2 | @behaviour ExAws.JSON.Codec 3 | 4 | @moduledoc false 5 | 6 | if Code.ensure_loaded?(:jsx) do 7 | def encode!(%{} = map) do 8 | map |> :jsx.encode() 9 | end 10 | 11 | def decode!(string) do 12 | :jsx.decode(string, [:return_maps]) 13 | end 14 | else 15 | def encode!(_) do 16 | raise ":jsx must be added as a dependency to use this module" 17 | end 18 | 19 | def decode!(_) do 20 | raise ":jsx must be added as a dependency to use this module" 21 | end 22 | end 23 | 24 | def encode(map) do 25 | try do 26 | {:ok, encode!(map)} 27 | rescue 28 | ArgumentError -> {:error, :badarg} 29 | end 30 | end 31 | 32 | def decode(string) do 33 | try do 34 | {:ok, decode!(string)} 35 | rescue 36 | ArgumentError -> {:error, :badarg} 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/ex_aws/operation.ex: -------------------------------------------------------------------------------- 1 | defprotocol ExAws.Operation do 2 | @moduledoc """ 3 | An operation to perform on AWS. 4 | 5 | This module defines a protocol for executing operations on AWS. ExAws ships with 6 | several different modules that each implement the `ExAws.Operation` protocol. These 7 | modules each handle one of the broad categories of AWS service types: 8 | 9 | - `ExAws.Operation.JSON` 10 | - `ExAws.Operation.Query` 11 | - `ExAws.Operation.RestQuery` 12 | - `ExAws.Operation.S3` 13 | 14 | ExAws works by creating a data structure that implements this protocol, and then 15 | calling `perform/2` on it. 16 | """ 17 | 18 | @doc """ 19 | Perform a request on AWS. 20 | 21 | The operation is synchronous, returning a response or an error. 22 | 23 | ## Example 24 | 25 | %ExAws.Operation.JSON{ 26 | data: %{}, 27 | headers: [ 28 | {"x-amz-target", "DynamoDB_20120810.ListTables"}, 29 | {"content-type", "application/x-amz-json-1.0"} 30 | ], 31 | http_method: :post, 32 | params: %{}, 33 | path: "/", 34 | service: :dynamodb, 35 | } |> ExAws.Operation.perform(ExAws.Config.new(:dynamodb)) 36 | 37 | """ 38 | def perform(operation, config) 39 | 40 | @doc """ 41 | Perform a *streaming* request on AWS. 42 | """ 43 | def stream!(operation, config) 44 | end 45 | -------------------------------------------------------------------------------- /lib/ex_aws/operation/json.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Operation.JSON do 2 | @moduledoc """ 3 | Datastructure representing an operation on a JSON based AWS service. 4 | 5 | This module is generally not used directly, but rather is constructed by one 6 | of the relevant AWS services. 7 | 8 | These include: 9 | - DynamoDB 10 | - Kinesis 11 | - Lambda (Rest style) 12 | - ElasticTranscoder 13 | 14 | JSON services are generally pretty simple. You just need to populate the `data` 15 | attribute with whatever request body parameters need converted to JSON, and set 16 | any service specific headers. 17 | 18 | The `before_request` 19 | """ 20 | 21 | defstruct stream_builder: nil, 22 | http_method: :post, 23 | parser: &Function.identity/1, 24 | error_parser: &Function.identity/1, 25 | path: "/", 26 | data: %{}, 27 | params: %{}, 28 | headers: [], 29 | service: nil, 30 | before_request: nil 31 | 32 | @type t :: %__MODULE__{} 33 | 34 | def new(service, opts) do 35 | struct(%__MODULE__{service: service}, opts) 36 | end 37 | end 38 | 39 | defimpl ExAws.Operation, for: ExAws.Operation.JSON do 40 | @type response_t :: %{} | ExAws.Request.error_t() 41 | 42 | def perform(operation, config) do 43 | operation = handle_callbacks(operation, config) 44 | url = ExAws.Request.Url.build(operation, config) 45 | 46 | headers = [ 47 | {"x-amz-content-sha256", ""} | operation.headers 48 | ] 49 | 50 | ExAws.Request.request( 51 | operation.http_method, 52 | url, 53 | operation.data, 54 | headers, 55 | config, 56 | operation.service 57 | ) 58 | |> operation.error_parser.() 59 | |> ExAws.Request.default_aws_error() 60 | |> parse(config) 61 | end 62 | 63 | def stream!(%ExAws.Operation.JSON{stream_builder: nil}, _) do 64 | raise ArgumentError, """ 65 | This operation does not support streaming! 66 | """ 67 | end 68 | 69 | def stream!(%ExAws.Operation.JSON{stream_builder: stream_builder}, config_overrides) do 70 | stream_builder.(config_overrides) 71 | end 72 | 73 | defp handle_callbacks(%{before_request: nil} = op, _), do: op 74 | 75 | defp handle_callbacks(%{before_request: callback} = op, config) do 76 | callback.(op, config) 77 | end 78 | 79 | defp parse({:error, result}, _), do: {:error, result} 80 | defp parse({:ok, %{body: ""}}, _), do: {:ok, %{}} 81 | 82 | defp parse({:ok, %{body: body}}, config) do 83 | {:ok, config[:json_codec].decode!(body)} 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /lib/ex_aws/operation/query.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Operation.Query do 2 | @moduledoc """ 3 | Datastructure representing an operation on a Query based AWS service 4 | 5 | These include: 6 | - SQS 7 | - SNS 8 | - SES 9 | """ 10 | 11 | defstruct path: "/", 12 | params: %{}, 13 | content_encoding: "identity", 14 | service: nil, 15 | action: nil, 16 | parser: &ExAws.Utils.identity/2 17 | 18 | @type t :: %__MODULE__{} 19 | end 20 | 21 | defimpl ExAws.Operation, for: ExAws.Operation.Query do 22 | def perform(operation, config) do 23 | data = operation.params |> URI.encode_query() 24 | 25 | data = 26 | case operation.content_encoding do 27 | "identity" -> data 28 | "gzip" -> :zlib.gzip(data) 29 | end 30 | 31 | url = 32 | operation 33 | |> Map.delete(:params) 34 | |> ExAws.Request.Url.build(config) 35 | 36 | headers = [ 37 | {"content-type", "application/x-www-form-urlencoded"}, 38 | {"content-encoding", operation.content_encoding} 39 | ] 40 | 41 | result = 42 | ExAws.Request.request(:post, url, data, headers, config, operation.service) 43 | |> ExAws.Request.default_aws_error() 44 | 45 | parser = operation.parser 46 | 47 | cond do 48 | is_function(parser, 2) -> 49 | parser.(result, operation.action) 50 | 51 | is_function(parser, 3) -> 52 | parser.(result, operation.action, config) 53 | 54 | true -> 55 | result 56 | end 57 | end 58 | 59 | def stream!(_, _), do: nil 60 | end 61 | -------------------------------------------------------------------------------- /lib/ex_aws/operation/query/parser.ex: -------------------------------------------------------------------------------- 1 | if Code.ensure_loaded?(SweetXml) do 2 | # TODO: Extract this. 3 | defmodule ExAws.Operation.Query.Parser do 4 | @moduledoc false 5 | 6 | defmacro __using__(_opts) do 7 | quote do 8 | import SweetXml, only: [sigil_x: 2] 9 | 10 | def parse({:error, {type, http_status_code, %{body: xml}}}, _) do 11 | parsed_body = 12 | xml 13 | |> SweetXml.parse(dtd: :none) 14 | |> SweetXml.xpath( 15 | ~x"//ErrorResponse", 16 | request_id: ~x"./RequestId/text()"s, 17 | type: ~x"./Error/Type/text()"s, 18 | code: ~x"./Error/Code/text()"s, 19 | message: ~x"./Error/Message/text()"s, 20 | detail: ~x"./Error/Detail/text()"s 21 | ) 22 | 23 | {:error, {type, http_status_code, parsed_body}} 24 | end 25 | end 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /lib/ex_aws/operation/rest_query.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Operation.RestQuery do 2 | @moduledoc false 3 | 4 | defstruct http_method: nil, 5 | path: "/", 6 | params: %{}, 7 | body: "", 8 | service: nil, 9 | action: nil, 10 | parser: &ExAws.Utils.identity/2 11 | 12 | @type t :: %__MODULE__{} 13 | end 14 | 15 | defimpl ExAws.Operation, for: ExAws.Operation.RestQuery do 16 | def perform(operation, config) do 17 | headers = config[:headers] || [] 18 | url = ExAws.Request.Url.build(operation, config) 19 | 20 | ExAws.Request.request( 21 | operation.http_method, 22 | url, 23 | operation.body, 24 | headers, 25 | config, 26 | operation.service 27 | ) 28 | |> ExAws.Request.default_aws_error() 29 | |> operation.parser.(operation.action) 30 | end 31 | 32 | def stream!(%{stream_builder: fun}, config) do 33 | fun.(config) 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /lib/ex_aws/operation/s3.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Operation.S3 do 2 | @moduledoc """ 3 | Holds data necessary for an operation on the S3 service. 4 | """ 5 | 6 | defstruct stream_builder: nil, 7 | parser: &Function.identity/1, 8 | bucket: "", 9 | path: "/", 10 | http_method: nil, 11 | body: "", 12 | resource: "", 13 | params: %{}, 14 | headers: %{}, 15 | service: :s3 16 | 17 | @type t :: %__MODULE__{} 18 | 19 | defimpl ExAws.Operation do 20 | def perform(operation, config) do 21 | body = operation.body 22 | headers = operation.headers 23 | http_method = operation.http_method 24 | 25 | {operation, config} = add_bucket_to_path(operation, config) 26 | 27 | url = 28 | operation 29 | |> add_resource_to_params() 30 | |> ExAws.Request.Url.build(config) 31 | 32 | hashed_payload = ExAws.Auth.Utils.hash_sha256(body) 33 | 34 | headers = 35 | headers 36 | |> Map.put("x-amz-content-sha256", hashed_payload) 37 | |> put_content_length_header(body, http_method) 38 | |> Map.to_list() 39 | 40 | ExAws.Request.request(http_method, url, body, headers, config, operation.service) 41 | |> ExAws.Request.default_aws_error() 42 | |> operation.parser.() 43 | end 44 | 45 | def stream!(%{stream_builder: fun}, config), do: fun.(config) 46 | 47 | defp put_content_length_header(headers, "", :get), do: headers 48 | 49 | defp put_content_length_header(headers, "", :head), do: headers 50 | 51 | defp put_content_length_header(headers, "", :delete), do: headers 52 | 53 | defp put_content_length_header(headers, body, _) do 54 | Map.put(headers, "content-length", IO.iodata_length(body) |> Integer.to_string()) 55 | end 56 | 57 | @spec add_bucket_to_path(operation :: ExAws.Operation.S3.t(), config :: map) :: 58 | {operation :: ExAws.Operation.S3.t(), config :: map} 59 | def add_bucket_to_path(operation, config \\ %{}) 60 | 61 | def add_bucket_to_path(%{bucket: nil}, _config) do 62 | raise "#{__MODULE__}.perform/2 cannot perform operation on `nil` bucket" 63 | end 64 | 65 | def add_bucket_to_path(operation, %{virtual_host: true, host: base_host} = config) do 66 | vhost_domain = "#{operation.bucket}.#{base_host}" 67 | 68 | {put_in(operation.path, ensure_absolute(operation.path)), 69 | Map.put(config, :host, vhost_domain)} 70 | end 71 | 72 | def add_bucket_to_path(operation, config) do 73 | path = "/#{operation.bucket}#{ensure_absolute(operation.path)}" |> expand_dot() 74 | {operation |> Map.put(:path, path), config} 75 | end 76 | 77 | @spec add_resource_to_params(operation :: ExAws.Operation.S3.t()) :: ExAws.Operation.S3.t() 78 | def add_resource_to_params(operation) do 79 | params = operation.params |> Map.new() |> Map.put(operation.resource, 1) 80 | operation |> Map.put(:params, params) 81 | end 82 | 83 | defp ensure_absolute(<<"/", _rest::binary>> = path), do: path 84 | defp ensure_absolute(path), do: "/#{path}" 85 | 86 | # A subset of Elixir's built-in Path.expand/1 - because it's OS-specific 87 | # we can't use it to normalise paths with "." or ".." in them (otherwise 88 | # it breaks on Windows because the /'s become \'s). 89 | defp expand_dot(<<"/", rest::binary>>), do: "/" <> do_expand_dot(rest) 90 | defp expand_dot(path), do: do_expand_dot(path) 91 | 92 | defp do_expand_dot(path), do: do_expand_dot(:binary.split(path, "/", [:global]), []) 93 | defp do_expand_dot([".." | t], [_, _ | acc]), do: do_expand_dot(t, acc) 94 | defp do_expand_dot([".." | t], []), do: do_expand_dot(t, []) 95 | defp do_expand_dot(["." | t], acc), do: do_expand_dot(t, acc) 96 | defp do_expand_dot([h | t], acc), do: do_expand_dot(t, ["/", h | acc]) 97 | defp do_expand_dot([], []), do: "" 98 | defp do_expand_dot([], ["/" | acc]), do: IO.iodata_to_binary(:lists.reverse(acc)) 99 | end 100 | end 101 | -------------------------------------------------------------------------------- /lib/ex_aws/request.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Request do 2 | @moduledoc false 3 | 4 | require Logger 5 | 6 | @type http_status :: pos_integer 7 | @type success_content :: %{body: binary, headers: [{binary, binary}]} 8 | @type success_t :: {:ok, success_content} 9 | @type error_t :: {:error, {:http_error, http_status, binary}} 10 | @type response_t :: success_t | error_t 11 | 12 | def request(http_method, url, data, headers, config, service) do 13 | body = 14 | case data do 15 | [] -> "{}" 16 | d when is_binary(d) -> d 17 | _ -> config[:json_codec].encode!(data) 18 | end 19 | 20 | request_and_retry(http_method, url, service, config, headers, body, {:attempt, 1}) 21 | end 22 | 23 | def request_and_retry(_method, _url, _service, _config, _headers, _req_body, {:error, reason}), 24 | do: {:error, reason} 25 | 26 | def request_and_retry(method, url, service, config, headers, req_body, {:attempt, attempt}) do 27 | full_headers = ExAws.Auth.headers(method, url, service, config, headers, req_body) 28 | 29 | with {:ok, full_headers} <- full_headers do 30 | safe_url = ExAws.Request.Url.sanitize(url, service) 31 | 32 | if config[:debug_requests] do 33 | Logger.debug( 34 | "ExAws: Request URL: #{inspect(safe_url)} HEADERS: #{inspect(full_headers)} BODY: #{inspect(req_body)} ATTEMPT: #{attempt}" 35 | ) 36 | end 37 | 38 | case do_request(config, method, safe_url, req_body, full_headers, attempt, service) do 39 | {:ok, %{status_code: status} = resp} when status in 200..299 or status == 304 -> 40 | {:ok, resp} 41 | 42 | {:ok, %{status_code: status} = _resp} when status == 301 -> 43 | Logger.warning("ExAws: Received redirect, did you specify the correct region?") 44 | {:error, {:http_error, status, "redirected"}} 45 | 46 | {:ok, %{status_code: status} = resp} when status in 400..499 -> 47 | case client_error(resp, config[:json_codec]) do 48 | {:retry, reason} -> 49 | request_and_retry( 50 | method, 51 | url, 52 | service, 53 | config, 54 | headers, 55 | req_body, 56 | attempt_again?(attempt, reason, :client, config) 57 | ) 58 | 59 | {:error, reason} -> 60 | {:error, reason} 61 | end 62 | 63 | {:ok, %{status_code: status} = resp} when status >= 500 -> 64 | body = Map.get(resp, :body) 65 | reason = {:http_error, status, body} 66 | 67 | request_and_retry( 68 | method, 69 | url, 70 | service, 71 | config, 72 | headers, 73 | req_body, 74 | attempt_again?(attempt, reason, :server, config) 75 | ) 76 | 77 | {:error, reason_struct} -> 78 | reason = 79 | case reason_struct do 80 | %{reason: reason} -> reason 81 | [reason: reason] -> reason 82 | end 83 | 84 | Logger.warning( 85 | "ExAws: HTTP ERROR: #{inspect(reason)} for URL: #{inspect(safe_url)} ATTEMPT: #{attempt}" 86 | ) 87 | 88 | request_and_retry( 89 | method, 90 | url, 91 | service, 92 | config, 93 | headers, 94 | req_body, 95 | attempt_again?(attempt, reason, :other, config) 96 | ) 97 | end 98 | end 99 | end 100 | 101 | defp do_request(config, method, safe_url, req_body, full_headers, attempt, service) do 102 | telemetry_event = Map.get(config, :telemetry_event, [:ex_aws, :request]) 103 | telemetry_options = Map.get(config, :telemetry_options, []) 104 | 105 | telemetry_metadata = %{ 106 | options: telemetry_options, 107 | attempt: attempt, 108 | service: service, 109 | request_body: req_body, 110 | operation: extract_operation(full_headers) 111 | } 112 | 113 | :telemetry.span(telemetry_event, telemetry_metadata, fn -> 114 | result = 115 | config[:http_client].request( 116 | method, 117 | safe_url, 118 | req_body, 119 | full_headers, 120 | Map.get(config, :http_opts, []) 121 | ) 122 | |> maybe_transform_response() 123 | 124 | stop_metadata = 125 | case result do 126 | {:ok, %{status_code: status} = resp} when status in 200..299 or status == 304 -> 127 | %{result: :ok, response_body: Map.get(resp, :body)} 128 | 129 | error -> 130 | %{result: :error, error: extract_error(error)} 131 | end 132 | 133 | telemetry_metadata = Map.merge(telemetry_metadata, stop_metadata) 134 | {result, telemetry_metadata} 135 | end) 136 | end 137 | 138 | defp extract_operation(headers), do: Enum.find_value(headers, &match_operation/1) 139 | 140 | defp match_operation({"x-amz-target", value}), do: value 141 | defp match_operation({_key, _value}), do: nil 142 | 143 | defp extract_error({:ok, %{body: body}}), do: body 144 | defp extract_error({:ok, response}), do: response 145 | defp extract_error({:error, error}), do: error 146 | defp extract_error(error), do: error 147 | 148 | def client_error(%{status_code: status, body: body} = error, json_codec) do 149 | case json_codec.decode(body) do 150 | {:ok, %{"__type" => error_type, "message" => message} = err} -> 151 | handle_error(error_type, message, status, err) 152 | 153 | # Rather irritatingly, as of 1.15, the local version of DynamoDB returns this with a 154 | # capital M in "Message" 155 | {:ok, %{"__type" => error_type, "Message" => message} = err} -> 156 | handle_error(error_type, message, status, err) 157 | 158 | _ -> 159 | {:error, {:http_error, status, error}} 160 | end 161 | end 162 | 163 | def client_error(%{status_code: status} = error, _) do 164 | {:error, {:http_error, status, error}} 165 | end 166 | 167 | def handle_aws_error({"ProvisionedThroughputExceededException" = type, message, _}) do 168 | {:retry, {type, message}} 169 | end 170 | 171 | def handle_aws_error({"ThrottlingException" = type, message, _}) do 172 | {:retry, {type, message}} 173 | end 174 | 175 | def handle_aws_error({"TooManyRequestsException" = type, message, _}) do 176 | {:retry, {type, message}} 177 | end 178 | 179 | def handle_aws_error({type, message, %{"expectedSequenceToken" => expected_sequence_token}}) do 180 | {:error, {type, message, expected_sequence_token}} 181 | end 182 | 183 | def handle_aws_error({type, message, err}) do 184 | # Mark as unhandled, might intereset error_parsers. 185 | {:error, {:aws_unhandled, type, message, err}} 186 | end 187 | 188 | # Clear unhandled mark, so Request.request() callers don't see it. 189 | def default_aws_error({:error, {:aws_unhandled, type, message, _}}) do 190 | {:error, {type, message}} 191 | end 192 | 193 | def default_aws_error(result) do 194 | result 195 | end 196 | 197 | defp handle_error(error_type, message, status, err) do 198 | error_type 199 | |> String.split("#") 200 | |> case do 201 | [_, type] -> handle_aws_error({type, message, err}) 202 | [type] -> handle_aws_error({type, message, err}) 203 | _ -> {:error, {:http_error, status, err}} 204 | end 205 | end 206 | 207 | def attempt_again?(attempt, reason, error_type, config) do 208 | max_attempts = 209 | case error_type do 210 | :client -> config[:retries][:client_error_max_attempts] || config[:retries][:max_attempts] 211 | _ -> config[:retries][:max_attempts] 212 | end 213 | 214 | if attempt >= max_attempts do 215 | {:error, reason} 216 | else 217 | attempt |> backoff(config) 218 | {:attempt, attempt + 1} 219 | end 220 | end 221 | 222 | def backoff(attempt, config) do 223 | (config[:retries][:base_backoff_in_ms] * :math.pow(2, attempt)) 224 | |> min(config[:retries][:max_backoff_in_ms]) 225 | |> trunc 226 | |> :rand.uniform() 227 | |> :timer.sleep() 228 | end 229 | 230 | def maybe_transform_response({:ok, %{status: status, body: body, headers: headers}}) do 231 | # Req and Finch use status (rather than status_code) as a key. 232 | {:ok, %{status_code: status, body: body, headers: headers}} 233 | end 234 | 235 | def maybe_transform_response(response), do: response 236 | end 237 | -------------------------------------------------------------------------------- /lib/ex_aws/request/hackney.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Request.Hackney do 2 | @behaviour ExAws.Request.HttpClient 3 | 4 | @moduledoc """ 5 | Configuration for `:hackney`. 6 | 7 | Options can be set for `:hackney` with the following config: 8 | 9 | config :ex_aws, :hackney_opts, 10 | recv_timeout: 30_000 11 | 12 | The default config handles setting the above. 13 | """ 14 | 15 | @default_opts [recv_timeout: 30_000] 16 | 17 | def request(method, url, body \\ "", headers \\ [], http_opts \\ []) do 18 | opts = Application.get_env(:ex_aws, :hackney_opts, @default_opts) 19 | opts = http_opts ++ [:with_body | opts] 20 | 21 | case :hackney.request(method, url, headers, body, opts) do 22 | {:ok, status, headers} -> 23 | {:ok, %{status_code: status, headers: headers}} 24 | 25 | {:ok, status, headers, body} -> 26 | {:ok, %{status_code: status, headers: headers, body: body}} 27 | 28 | {:error, reason} -> 29 | {:error, %{reason: reason}} 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/ex_aws/request/http_client.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Request.HttpClient do 2 | @moduledoc ~S''' 3 | Specifies expected behaviour of an HTTP client. 4 | 5 | ExAws allows you to use your HTTP client of choice, provided that 6 | it can be coerced into complying with this module's specification. 7 | 8 | The default is `:hackney`. 9 | 10 | ## Example: Req 11 | 12 | Here is an example using [Req](https://hexdocs.pm/req/readme.html). 13 | 14 | First, create a module implementing the `ExAws.Request.HttpClient` behaviour. 15 | 16 | ``` 17 | defmodule ExAws.Request.Req do 18 | @moduledoc """ 19 | ExAws HTTP client implementation for Req. 20 | """ 21 | 22 | @behaviour ExAws.Request.HttpClient 23 | 24 | @impl ExAws.Request.HttpClient 25 | def request(method, url, body, headers, _http_opts) do 26 | request = Req.new(decode_body: false, retry: false) 27 | 28 | case Req.request(request, method: method, url: url, body: body, headers: headers) do 29 | {:ok, response} -> 30 | response = %{ 31 | status_code: response.status, 32 | headers: Req.get_headers_list(response), 33 | body: response.body, 34 | } 35 | 36 | {:ok, response} 37 | 38 | {:error, reason} -> 39 | {:error, %{reason: reason}} 40 | end 41 | end 42 | end 43 | ``` 44 | 45 | Then, in build-time config (e.g. config.exs): 46 | 47 | ``` 48 | config :ex_aws, 49 | http_client: ExAws.Request.Req 50 | ``` 51 | 52 | When conforming your selected HTTP Client take note of a few things: 53 | 54 | - The module name doesn't need to follow the same styling as this module it 55 | is simply your own 'HTTP Client', i.e. `MyApp.HttpClient` 56 | 57 | - The request function must accept the methods as described in the 58 | `c:request/5` callback, you can however set these as optional, 59 | i.e. `http_opts \\ []` 60 | 61 | - Ensure the call to your chosen HTTP Client is correct and the return is 62 | in the same format as defined in the `c:request/5` callback 63 | 64 | ## Example: Mojito 65 | 66 | def request(method, url, body, headers, http_opts \\ []) do 67 | Mojito.request(method, url, headers, body, http_opts) 68 | end 69 | 70 | ''' 71 | 72 | @type http_method :: :get | :post | :put | :delete | :options | :head 73 | @callback request( 74 | method :: http_method, 75 | url :: binary, 76 | req_body :: binary, 77 | headers :: [{binary, binary}, ...], 78 | http_opts :: term 79 | ) :: 80 | {:ok, %{status_code: pos_integer, headers: any}} 81 | | {:ok, %{status_code: pos_integer, headers: any, body: binary}} 82 | | {:error, %{reason: any}} 83 | end 84 | -------------------------------------------------------------------------------- /lib/ex_aws/request/req.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Request.Req do 2 | @behaviour ExAws.Request.HttpClient 3 | 4 | @moduledoc """ 5 | Configuration for `m:Req`. 6 | 7 | Options can be set for `m:Req` with the following config: 8 | 9 | config :ex_aws, :req_opts, 10 | receive_timeout: 30_000 11 | 12 | The default config handles setting the above. 13 | """ 14 | 15 | @default_opts [receive_timeout: 30_000] 16 | 17 | @impl true 18 | def request(method, url, body \\ "", headers \\ [], http_opts \\ []) do 19 | http_opts = rename_follow_redirect(http_opts) 20 | 21 | [method: method, url: url, body: body, headers: headers, decode_body: false, retry: false] 22 | |> Keyword.merge(Application.get_env(:ex_aws, :req_opts, @default_opts)) 23 | |> Keyword.merge(http_opts) 24 | |> Req.request() 25 | |> case do 26 | {:ok, resp} -> 27 | {:ok, %{status_code: resp.status, headers: Req.get_headers_list(resp), body: resp.body}} 28 | 29 | {:error, reason} -> 30 | {:error, %{reason: reason}} 31 | end 32 | end 33 | 34 | # Req >= 0.4.0 uses :redirect, but some clients pass the :hackney option 35 | # :follow_redirect. Rename the option for Req to use. 36 | defp rename_follow_redirect(opts) do 37 | {follow, opts} = Keyword.pop(opts, :follow_redirect, false) 38 | 39 | Keyword.put(opts, :redirect, follow) 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/ex_aws/request/url.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Request.Url do 2 | @moduledoc false 3 | 4 | @doc """ 5 | Builds URL for an operation and a config" 6 | """ 7 | def build(operation, config) do 8 | config 9 | |> Map.take([:scheme, :host, :port]) 10 | |> Map.put(:query, query(operation)) 11 | |> Map.put(:path, operation.path) 12 | |> normalize_scheme 13 | |> normalize_path(config.normalize_path) 14 | |> convert_port_to_integer 15 | |> (&struct(URI, &1)).() 16 | |> URI.to_string() 17 | |> String.trim_trailing("?") 18 | end 19 | 20 | defp query(operation) do 21 | operation 22 | |> Map.get(:params, %{}) 23 | |> normalize_params 24 | |> URI.encode_query() 25 | end 26 | 27 | defp normalize_scheme(url) do 28 | url |> Map.update(:scheme, "", &String.replace(&1, "://", "")) 29 | end 30 | 31 | defp normalize_path(url, false), do: url 32 | 33 | defp normalize_path(url, _normalize) do 34 | url |> Map.update(:path, "", &String.replace(&1, ~r/\/{2,}/, "/")) 35 | end 36 | 37 | defp convert_port_to_integer(url = %{port: port}) when is_binary(port) do 38 | {port, _} = Integer.parse(port) 39 | put_in(url[:port], port) 40 | end 41 | 42 | defp convert_port_to_integer(url), do: url 43 | 44 | defp normalize_params(params) when is_map(params) do 45 | params |> Map.delete("") |> Map.delete(nil) 46 | end 47 | 48 | defp normalize_params(params), do: params 49 | 50 | def sanitize(url, service) when service in ["s3", :s3] do 51 | new_path = 52 | url 53 | |> get_path(service) 54 | |> String.replace_prefix("/", "") 55 | |> uri_encode() 56 | 57 | query = 58 | case String.split(url, "?", parts: 2) do 59 | [_] -> nil 60 | [_, ""] -> nil 61 | [_, q] -> q 62 | end 63 | 64 | url 65 | |> URI.parse() 66 | |> Map.put(:fragment, nil) 67 | |> Map.put(:path, "/" <> new_path) 68 | |> Map.put(:query, query) 69 | |> URI.to_string() 70 | |> String.replace("+", "%20") 71 | end 72 | 73 | def sanitize(url, _), do: String.replace(url, "+", "%20") 74 | 75 | def get_path(url, service \\ nil) 76 | # Elixir's URI.parse will treat everything after the # sign 77 | # as a fragment. This is correct, but S3 treats it as part 78 | # of the path of the object. 79 | # 80 | # This will split the URL at the base with the right side 81 | # being the path, except for the query params. 82 | # 83 | # for example: 84 | # `"https://bucket.aws.com/my/path/here+ #3.txt?t=21"` 85 | # https://bucket.aws.com | /my/path/here+ #3.txt?t=21 86 | # ________base__________ | /my/path/here+ #3.txt | t=21 87 | # ________base__________ | /my/path/here+ #3.txt | _params_ 88 | # 89 | # After using `uri_encode` it will be `/my/path/here%2B%20%233.txt` 90 | # 91 | def get_path(url, service) when service in ["s3", :s3] do 92 | base = 93 | url 94 | |> URI.parse() 95 | |> Map.put(:path, nil) 96 | |> Map.put(:query, nil) 97 | |> Map.put(:fragment, nil) 98 | |> URI.to_string() 99 | 100 | [_base, path_with_params] = String.split(url, base, parts: 2) 101 | [path | _query_params] = String.split(path_with_params, "?", parts: 2) 102 | 103 | path 104 | end 105 | 106 | def get_path(url, _), do: URI.parse(url).path || "/" 107 | 108 | def uri_encode(url), do: URI.encode(url, &valid_path_char?/1) 109 | 110 | # Space character 111 | defp valid_path_char?(?\s), do: false 112 | defp valid_path_char?(?/), do: true 113 | 114 | defp valid_path_char?(c) do 115 | URI.char_unescaped?(c) && !URI.char_reserved?(c) 116 | end 117 | end 118 | -------------------------------------------------------------------------------- /lib/ex_aws/utils.ex: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Utils do 2 | @moduledoc false 3 | 4 | def identity(x, _), do: x 5 | 6 | # This isn't tail recursive. However, given that the structures 7 | # being worked upon are relatively shallow, this is ok. 8 | def camelize_keys(opts, kwargs \\ []) do 9 | deep = kwargs[:deep] || false 10 | spec = kwargs[:spec] || %{} 11 | do_cmlz_keys(opts, deep: deep, spec: spec) 12 | end 13 | 14 | defp do_cmlz_keys([%{} | _] = opts, deep: deep, spec: spec) do 15 | Enum.map(opts, &do_cmlz_keys(&1, deep: deep, spec: spec)) 16 | end 17 | 18 | defp do_cmlz_keys(opts = %{}, deep: deep, spec: spec) do 19 | opts 20 | |> Enum.reduce(%{}, fn {k, v}, map -> 21 | if deep do 22 | Map.put(map, camelize_key(k, spec: spec), do_cmlz_keys(v, deep: true, spec: spec)) 23 | else 24 | Map.put(map, camelize_key(k, spec: spec), v) 25 | end 26 | end) 27 | end 28 | 29 | defp do_cmlz_keys(opts, kwargs) do 30 | try do 31 | opts |> Map.new() |> do_cmlz_keys(kwargs) 32 | rescue 33 | [Protocol.UndefinedError, ArgumentError, FunctionClauseError] -> opts 34 | end 35 | end 36 | 37 | def camelize_key(key, [spec: spec] \\ [spec: %{}]) do 38 | spec[key] || key |> maybe_stringify |> camelize 39 | end 40 | 41 | def camelize(string) do 42 | string 43 | |> to_charlist 44 | |> Enum.reduce({true, ~c""}, fn 45 | ?_, {_, acc} -> {true, acc} 46 | ?/, {_, acc} -> {false, [?. | acc]} 47 | char, {false, acc} -> {false, [char | acc]} 48 | char, {true, acc} -> {false, [upcase(char) | acc]} 49 | end) 50 | # charlist 51 | |> elem(1) 52 | # unreverses charlist after reducing 53 | |> Enum.reverse() 54 | |> to_string 55 | end 56 | 57 | def upcase(value) when is_atom(value), do: value |> Atom.to_string() |> String.upcase() 58 | def upcase(value) when is_binary(value), do: String.upcase(value) 59 | def upcase(char), do: :string.to_upper(char) 60 | 61 | def uuid, do: DateTime.utc_now() |> :erlang.phash2() 62 | 63 | def rename_keys(params, mapping) do 64 | mapping = Map.new(mapping) 65 | 66 | Enum.map(params, fn {k, v} -> 67 | {Map.get(mapping, k, k), v} 68 | end) 69 | end 70 | 71 | def format(params, kwargs \\ []) do 72 | prefix = kwargs[:prefix] || "" 73 | spec = kwargs[:spec] || %{} 74 | 75 | case kwargs[:type] || :xml do 76 | :xml -> 77 | xml_format(params, prefix: prefix, spec: spec) 78 | # :json -> json_format(params, prefix: prefix, spec: spec) 79 | end 80 | end 81 | 82 | # NOTE: xml_format is not tail call optimized 83 | # but it is unlikely that any AWS params will ever 84 | # be nested enough for this to cause a stack overflow 85 | 86 | # Indexed formats 87 | 88 | defp xml_format([nested | _] = params, kwargs) when is_map(nested) do 89 | params |> Enum.map(&Map.to_list/1) |> xml_format(kwargs) 90 | end 91 | 92 | defp xml_format([nested | _] = params, prefix: pre, spec: spec) when is_list(nested) do 93 | params 94 | |> Stream.with_index(1) 95 | |> Stream.map(fn {params, i} -> {params, Integer.to_string(i)} end) 96 | |> Stream.flat_map(fn {params, i} -> 97 | xml_format(params, prefix: pre <> dot?(pre) <> i, spec: spec) 98 | end) 99 | |> Enum.to_list() 100 | end 101 | 102 | defp xml_format([param | _] = params, prefix: pre, spec: spec) when is_tuple(param) do 103 | params 104 | |> Stream.map(fn {key, values} -> {maybe_camelize(key, spec: spec), values} end) 105 | |> Stream.flat_map(fn {key, values} -> 106 | xml_format(values, prefix: pre <> dot?(pre) <> key, spec: spec) 107 | end) 108 | |> Enum.to_list() 109 | end 110 | 111 | # Non-indexed formats 112 | defp xml_format(params, kwargs) when is_list(params) do 113 | pre = kwargs[:prefix] 114 | 115 | params 116 | |> Stream.with_index(1) 117 | |> Stream.map(fn {value, i} -> {value, Integer.to_string(i)} end) 118 | |> Stream.map(fn {value, i} -> {pre <> dot?(pre) <> i, value} end) 119 | |> Enum.to_list() 120 | end 121 | 122 | defp xml_format(params, kwargs) when is_map(params) do 123 | xml_format(Map.to_list(params), kwargs) 124 | end 125 | 126 | defp xml_format(value, kwargs), do: [{kwargs[:prefix], value}] 127 | 128 | defp dot?(""), do: "" 129 | defp dot?(_), do: "." 130 | 131 | def filter_nil_params(opts) do 132 | opts 133 | |> Enum.reject(fn {_key, value} -> value == nil end) 134 | |> Enum.into(%{}) 135 | end 136 | 137 | def maybe_camelize(elem, kwargs \\ [spec: %{}]) 138 | def maybe_camelize(elem, kwargs) when is_atom(elem), do: camelize_key(elem, kwargs) 139 | def maybe_camelize(elem, _) when is_bitstring(elem), do: elem 140 | 141 | def maybe_stringify(elem) when is_atom(elem), do: Atom.to_string(elem) 142 | def maybe_stringify(elem) when is_bitstring(elem), do: elem 143 | 144 | defmacro __using__(kwargs) do 145 | camelize_inject = 146 | quote do 147 | [spec: unquote(kwargs[:non_standard_keys] || %{})] ++ kwargs 148 | end 149 | 150 | format_inject = 151 | quote do 152 | [type: unquote(kwargs[:format_type] || :xml)] ++ unquote(camelize_inject) 153 | end 154 | 155 | quote do 156 | import ExAws.Utils, 157 | except: [ 158 | format: 2, 159 | format: 1, 160 | camelize_keys: 2, 161 | camelize_keys: 1, 162 | camelize_key: 2, 163 | camelize_key: 1, 164 | maybe_camelize: 2, 165 | maybe_camelize: 1 166 | ] 167 | 168 | defp format(params, kwargs \\ []), do: ExAws.Utils.format(params, unquote(format_inject)) 169 | 170 | defp camelize_keys(opts, kwargs \\ []), 171 | do: ExAws.Utils.camelize_keys(opts, unquote(camelize_inject)) 172 | 173 | defp camelize_key(opts, kwargs \\ []), 174 | do: ExAws.Utils.camelize_key(opts, unquote(camelize_inject)) 175 | 176 | defp maybe_camelize(opts, kwargs \\ []), 177 | do: ExAws.Utils.maybe_camelize(opts, unquote(camelize_inject)) 178 | end 179 | end 180 | end 181 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Mixfile do 2 | use Mix.Project 3 | 4 | @source_url "https://github.com/ex-aws/ex_aws" 5 | @version "2.5.9" 6 | 7 | def project do 8 | [ 9 | app: :ex_aws, 10 | version: @version, 11 | elixir: "~> 1.13", 12 | elixirc_paths: elixirc_paths(Mix.env()), 13 | description: "Generic AWS client", 14 | name: "ExAws", 15 | source_url: @source_url, 16 | package: package(), 17 | deps: deps(), 18 | docs: docs(), 19 | dialyzer: [ 20 | plt_add_apps: [:mix, :hackney, :configparser_ex, :jsx] 21 | ], 22 | test_coverage: [tool: ExCoveralls], 23 | preferred_cli_env: [ 24 | coveralls: :test, 25 | "coveralls.detail": :test, 26 | "coveralls.post": :test, 27 | "coveralls.html": :test 28 | ], 29 | xref: [ 30 | exclude: [ 31 | :hackney, 32 | Req 33 | ] 34 | ] 35 | ] 36 | end 37 | 38 | def application do 39 | [extra_applications: [:logger, :crypto], mod: {ExAws, []}] 40 | end 41 | 42 | defp elixirc_paths(:test), do: ["lib", "test/support"] 43 | defp elixirc_paths(_), do: ["lib"] 44 | 45 | defp deps() do 46 | [ 47 | {:telemetry, "~> 0.4.3 or ~> 1.0"}, 48 | {:mime, "~> 1.2 or ~> 2.0"}, 49 | {:bypass, "~> 2.1", only: :test}, 50 | {:configparser_ex, "~> 4.0", optional: true}, 51 | {:dialyxir, "~> 1.0", only: [:dev, :test], runtime: false}, 52 | {:ex_doc, "~> 0.16", only: [:dev, :test]}, 53 | {:hackney, "~> 1.16", optional: true}, 54 | {:req, "~> 0.5.10 or ~> 0.6 or ~> 1.0", optional: true}, 55 | {:jason, "~> 1.1", optional: true}, 56 | {:jsx, "~> 2.8 or ~> 3.0", optional: true}, 57 | {:mox, "~> 1.0", only: :test}, 58 | {:sweet_xml, "~> 0.7", optional: true}, 59 | {:excoveralls, "~> 0.10", only: :test} 60 | ] 61 | end 62 | 63 | defp package do 64 | [ 65 | description: description(), 66 | files: ["priv", "lib", "config", "mix.exs", "CHANGELOG.md", "README*", "LICENSE"], 67 | exclude_patterns: ["_build", "deps", "test", "*~"], 68 | maintainers: ["Bernard Duggan", "Ben Wilson"], 69 | licenses: ["MIT"], 70 | links: %{ 71 | Changelog: "#{@source_url}/blob/master/CHANGELOG.md", 72 | GitHub: @source_url 73 | }, 74 | exclude_patterns: [~r/.*~/] 75 | ] 76 | end 77 | 78 | defp description do 79 | """ 80 | AWS client for Elixir. Currently supports Dynamo, DynamoStreams, EC2, 81 | Firehose, Kinesis, KMS, Lambda, RRDS, Route53, S3, SES, SNS, SQS, STS and others. 82 | """ 83 | end 84 | 85 | defp docs do 86 | [ 87 | main: "readme", 88 | source_ref: "v#{@version}", 89 | source_url: @source_url, 90 | extras: ["README.md"] 91 | ] 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"}, 3 | "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"}, 4 | "certifi": {:hex, :certifi, "2.14.0", "ed3bef654e69cde5e6c022df8070a579a79e8ba2368a00acf3d75b82d9aceeed", [:rebar3], [], "hexpm", "ea59d87ef89da429b8e905264fdec3419f84f2215bb3d81e07a18aac919026c3"}, 5 | "configparser_ex": {:hex, :configparser_ex, "4.0.0", "17e2b831cfa33a08c56effc610339b2986f0d82a9caa0ed18880a07658292ab6", [:mix], [], "hexpm", "02e6d1a559361a063cba7b75bc3eb2d6ad7e62730c551cc4703541fd11e65e5b"}, 6 | "cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"}, 7 | "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, 8 | "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"}, 9 | "dialyxir": {:hex, :dialyxir, "1.4.5", "ca1571ac18e0f88d4ab245f0b60fa31ff1b12cbae2b11bd25d207f865e8ae78a", [:mix], [{:erlex, ">= 0.2.7", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b0fb08bb8107c750db5c0b324fa2df5ceaa0f9307690ee3c1f6ba5b9eb5d35c3"}, 10 | "earmark_parser": {:hex, :earmark_parser, "1.4.43", "34b2f401fe473080e39ff2b90feb8ddfeef7639f8ee0bbf71bb41911831d77c5", [:mix], [], "hexpm", "970a3cd19503f5e8e527a190662be2cee5d98eed1ff72ed9b3d1a3d466692de8"}, 11 | "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"}, 12 | "ex_doc": {:hex, :ex_doc, "0.37.3", "f7816881a443cd77872b7d6118e8a55f547f49903aef8747dbcb345a75b462f9", [:mix], [{:earmark_parser, "~> 1.4.42", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "e6aebca7156e7c29b5da4daa17f6361205b2ae5f26e5c7d8ca0d3f7e18972233"}, 13 | "excoveralls": {:hex, :excoveralls, "0.18.5", "e229d0a65982613332ec30f07940038fe451a2e5b29bce2a5022165f0c9b157e", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "523fe8a15603f86d64852aab2abe8ddbd78e68579c8525ae765facc5eae01562"}, 14 | "finch": {:hex, :finch, "0.19.0", "c644641491ea854fc5c1bbaef36bfc764e3f08e7185e1f084e35e0672241b76d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fc5324ce209125d1e2fa0fcd2634601c52a787aff1cd33ee833664a5af4ea2b6"}, 15 | "hackney": {:hex, :hackney, "1.23.0", "55cc09077112bcb4a69e54be46ed9bc55537763a96cd4a80a221663a7eafd767", [:rebar3], [{:certifi, "~> 2.14.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "6cd1c04cd15c81e5a493f167b226a15f0938a84fc8f0736ebe4ddcab65c0b44e"}, 16 | "hpax": {:hex, :hpax, "1.0.2", "762df951b0c399ff67cc57c3995ec3cf46d696e41f0bba17da0518d94acd4aac", [:mix], [], "hexpm", "2f09b4c1074e0abd846747329eaa26d535be0eb3d189fa69d812bfb8bfefd32f"}, 17 | "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, 18 | "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, 19 | "jsx": {:hex, :jsx, "3.1.0", "d12516baa0bb23a59bb35dccaf02a1bd08243fcbb9efe24f2d9d056ccff71268", [:rebar3], [], "hexpm", "0c5cc8fdc11b53cc25cf65ac6705ad39e54ecc56d1c22e4adb8f5a53fb9427f3"}, 20 | "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, 21 | "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, 22 | "makeup_erlang": {:hex, :makeup_erlang, "1.0.2", "03e1804074b3aa64d5fad7aa64601ed0fb395337b982d9bcf04029d68d51b6a7", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "af33ff7ef368d5893e4a267933e7744e46ce3cf1f61e2dccf53a111ed3aa3727"}, 23 | "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, 24 | "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"}, 25 | "mimerl": {:hex, :mimerl, "1.3.0", "d0cd9fc04b9061f82490f6581e0128379830e78535e017f7780f37fea7545726", [:rebar3], [], "hexpm", "a1e15a50d1887217de95f0b9b0793e32853f7c258a5cd227650889b38839fe9d"}, 26 | "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, 27 | "mox": {:hex, :mox, "1.2.0", "a2cd96b4b80a3883e3100a221e8adc1b98e4c3a332a8fc434c39526babafd5b3", [:mix], [{:nimble_ownership, "~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}], "hexpm", "c7b92b3cc69ee24a7eeeaf944cd7be22013c52fcb580c1f33f50845ec821089a"}, 28 | "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, 29 | "nimble_ownership": {:hex, :nimble_ownership, "1.0.0", "3f87744d42c21b2042a0aa1d48c83c77e6dd9dd357e425a038dd4b49ba8b79a1", [:mix], [], "hexpm", "7c16cc74f4e952464220a73055b557a273e8b1b7ace8489ec9d86e9ad56cb2cc"}, 30 | "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, 31 | "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, 32 | "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, 33 | "plug": {:hex, :plug, "1.17.0", "a0832e7af4ae0f4819e0c08dd2e7482364937aea6a8a997a679f2cbb7e026b2e", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f6692046652a69a00a5a21d0b7e11fcf401064839d59d6b8787f23af55b1e6bc"}, 34 | "plug_cowboy": {:hex, :plug_cowboy, "2.6.1", "9a3bbfceeb65eff5f39dab529e5cd79137ac36e913c02067dba3963a26efe9b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "de36e1a21f451a18b790f37765db198075c25875c64834bcc82d90b309eb6613"}, 35 | "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, 36 | "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, 37 | "req": {:hex, :req, "0.5.10", "a3a063eab8b7510785a467f03d30a8d95f66f5c3d9495be3474b61459c54376c", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "8a604815743f8a2d3b5de0659fa3137fa4b1cffd636ecb69b30b2b9b2c2559be"}, 38 | "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, 39 | "sweet_xml": {:hex, :sweet_xml, "0.7.5", "803a563113981aaac202a1dbd39771562d0ad31004ddbfc9b5090bdcd5605277", [:mix], [], "hexpm", "193b28a9b12891cae351d81a0cead165ffe67df1b73fe5866d10629f4faefb12"}, 40 | "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, 41 | "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, 42 | } 43 | -------------------------------------------------------------------------------- /test/alternate_helper.exs: -------------------------------------------------------------------------------- 1 | defmodule Test.KinesisAlt do 2 | def config_root do 3 | Application.get_all_env(:ex_aws) 4 | |> Keyword.put(:http_client, ExAws.Request.Req) 5 | |> Keyword.put(:json_codec, ExAws.JSON.JSX) 6 | end 7 | end 8 | 9 | defmodule Test.Req do 10 | def request(method, url, body, headers, _) do 11 | Req.request(method: method, url: url, body: body, headers: headers) 12 | end 13 | end 14 | 15 | defmodule Test.JSXCodec do 16 | def encode!(%{} = map) do 17 | map 18 | |> Map.to_list() 19 | |> :jsx.encode() 20 | |> case do 21 | "[]" -> "{}" 22 | val -> val 23 | end 24 | end 25 | 26 | def encode(map) do 27 | {:ok, encode!(map)} 28 | end 29 | 30 | def decode!(string) do 31 | :jsx.decode(string) 32 | |> Map.new() 33 | end 34 | 35 | def decode(string) do 36 | {:ok, decode!(string)} 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /test/default_helper.exs: -------------------------------------------------------------------------------- 1 | ## Clients 2 | 3 | ## Other 4 | 5 | defmodule Test.JSONCodec do 6 | @behaviour ExAws.JSON.Codec 7 | 8 | defdelegate encode!(data), to: Jason 9 | defdelegate encode(data), to: Jason 10 | defdelegate decode!(data), to: Jason 11 | defdelegate decode(data), to: Jason 12 | end 13 | -------------------------------------------------------------------------------- /test/ex_aws/auth/auth_cache_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ExAws.AuthCacheTest do 2 | use ExUnit.Case, async: false 3 | 4 | import Mox 5 | 6 | @response {:ok, %{status_code: 200, body: %{}}} 7 | 8 | @config [ 9 | http_client: ExAws.Request.HttpMock, 10 | access_key_id: [{:awscli, "xpto", 1}], 11 | secret_access_key: [{:awscli, "xpto", 1}] 12 | ] 13 | 14 | defmodule SleepAdapter do 15 | @moduledoc false 16 | 17 | @behaviour ExAws.Config.AuthCache.AuthConfigAdapter 18 | 19 | @config %{ 20 | access_key_id: "AKIAIOSFODNN7EXAMPLE", 21 | secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", 22 | region: "us-east-1" 23 | } 24 | 25 | @impl true 26 | def adapt_auth_config(_config, _profile, _expiration) do 27 | Process.sleep(200) 28 | @config 29 | end 30 | end 31 | 32 | setup do 33 | Application.put_env(:ex_aws, :awscli_auth_adapter, SleepAdapter) 34 | 35 | :ok 36 | end 37 | 38 | test "using adapter does not leak dirty cache" do 39 | parent = self() 40 | 41 | op = %ExAws.Operation.S3{ 42 | body: "", 43 | bucket: "", 44 | headers: %{}, 45 | http_method: :get, 46 | params: [], 47 | parser: & &1, 48 | path: "/", 49 | resource: "", 50 | service: :s3, 51 | stream_builder: nil 52 | } 53 | 54 | spawn(fn -> 55 | ExAws.Request.HttpMock 56 | |> expect(:request, fn _method, _url, _body, _headers, _opts -> 57 | @response 58 | end) 59 | 60 | result = ExAws.request(op, @config) 61 | send(parent, result) 62 | end) 63 | 64 | ExAws.Request.HttpMock 65 | |> expect(:request, fn _method, _url, _body, _headers, _opts -> 66 | @response 67 | end) 68 | 69 | Process.sleep(100) 70 | assert ExAws.request(op, @config) == @response 71 | 72 | assert_receive @response, 1000 73 | end 74 | 75 | test "using adapter retries when there is an error" do 76 | # The flaky adapter simulates failures on the adapter side 77 | # for a few a tries and then returns a successful response. 78 | 79 | defmodule FlakyAdapter do 80 | @moduledoc false 81 | 82 | @behaviour ExAws.Config.AuthCache.AuthConfigAdapter 83 | 84 | @config %{ 85 | access_key_id: "AKIAIOSFODNN7EXAMPLE", 86 | secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", 87 | region: "us-east-1" 88 | } 89 | 90 | def init do 91 | :ets.new(__MODULE__, [:named_table, :set, :public, read_concurrency: true]) 92 | :ets.insert(__MODULE__, {:count, 0}) 93 | end 94 | 95 | @impl true 96 | def adapt_auth_config(_config, _profile, _expiration) do 97 | [count: count] = :ets.lookup(__MODULE__, :count) 98 | 99 | if count < 3 do 100 | :ets.insert(__MODULE__, {:count, count + 1}) 101 | {:error, %{status_code: 400, body: "Throttling", reason: "Throttling"}} 102 | else 103 | @config 104 | end 105 | end 106 | end 107 | 108 | FlakyAdapter.init() 109 | Application.put_env(:ex_aws, :awscli_auth_adapter, FlakyAdapter) 110 | 111 | op = %ExAws.Operation.S3{ 112 | body: "", 113 | bucket: "", 114 | headers: %{}, 115 | http_method: :get, 116 | params: [], 117 | parser: & &1, 118 | path: "/", 119 | resource: "", 120 | service: :s3, 121 | stream_builder: nil 122 | } 123 | 124 | ExAws.Request.HttpMock 125 | |> expect(:request, fn _method, _url, _body, _headers, _opts -> 126 | @response 127 | end) 128 | 129 | Process.sleep(100) 130 | assert ExAws.request(op, @config) == @response 131 | end 132 | end 133 | -------------------------------------------------------------------------------- /test/ex_aws/auth/credentials_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Auth.CredentialsTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias ExAws.Auth.Credentials 5 | 6 | test "generate_credential_v4/3 returns the correct value" do 7 | datetime = {{2013, 5, 24}, {0, 0, 0}} 8 | 9 | config = [ 10 | access_key_id: "AKIAIOSFODNN7EXAMPLE", 11 | region: "us-east-1" 12 | ] 13 | 14 | scope = Credentials.generate_credential_v4("s3", config, datetime) 15 | 16 | assert scope == "AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request" 17 | end 18 | 19 | test "generate_credential_scope_v4/3 returns the correct value" do 20 | datetime = {{2013, 5, 24}, {0, 0, 0}} 21 | config = [region: "us-east-1"] 22 | 23 | scope = Credentials.generate_credential_scope_v4("s3", config, datetime) 24 | 25 | assert scope == "20130524/us-east-1/s3/aws4_request" 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /test/ex_aws/auth/signatures_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Auth.SignaturesTest do 2 | use ExUnit.Case, async: true 3 | alias ExAws.Auth.Signatures 4 | 5 | describe "generate_signature_v4/4" do 6 | test "with a basic string to sign" do 7 | config = 8 | ExAws.Config.new( 9 | :s3, 10 | access_key_id: "AKIAIOSFODNN7EXAMPLE", 11 | secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", 12 | region: "us-east-1" 13 | ) 14 | 15 | datetime = {{2016, 8, 29}, {19, 41, 33}} 16 | 17 | signature = Signatures.generate_signature_v4("s3", config, datetime, "hello world") 18 | 19 | assert signature == "690b8431208dae486dd00df93bde9370d8aba587098b9a26bfd07c259df395c9" 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /test/ex_aws/auth/utils_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Auth.UtilsTest do 2 | use ExUnit.Case, async: true 3 | import ExAws.Auth.Utils 4 | 5 | test "quasi_iso_date/1" do 6 | assert quasi_iso_format({2015, 1, 2}) == ["2015", "01", "02"] 7 | assert quasi_iso_format({2015, 11, 12}) == ["2015", "11", "12"] 8 | end 9 | 10 | test "amz_date/1" do 11 | assert amz_date({{2015, 1, 2}, {1, 3, 5}}) == "20150102T010305Z" 12 | assert amz_date({{2015, 11, 22}, {11, 31, 51}}) == "20151122T113151Z" 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /test/ex_aws/auth_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ExAws.AuthTest do 2 | use ExUnit.Case, async: true 3 | 4 | import ExAws.Auth, 5 | only: [ 6 | headers: 6, 7 | build_canonical_request: 5 8 | ] 9 | 10 | import ExAws.Request.Url, 11 | only: [ 12 | uri_encode: 1 13 | ] 14 | 15 | @config %{ 16 | access_key_id: "AKIAIOSFODNN7EXAMPLE", 17 | secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", 18 | region: "us-east-1" 19 | } 20 | 21 | test "build_canonical_request can handle : " do 22 | expected = 23 | "GET\n/bar%3Abaz%40blag\n\n\n\n\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" 24 | 25 | path = URI.parse("http://foo.com/bar:baz@blag").path |> uri_encode 26 | assert build_canonical_request(:get, path, "", %{}, "") == expected 27 | end 28 | 29 | test "build_canonical_request ignores unsignable headers" do 30 | path = URI.parse("http://foo.com/bar:baz@blag").path |> uri_encode 31 | without_unsignable_header = build_canonical_request(:get, path, "", %{}, "") 32 | 33 | with_unsignable_header = 34 | build_canonical_request( 35 | :get, 36 | path, 37 | "", 38 | %{ 39 | "X-Amzn-Trace-Id" => "1-aaaaaaa-bbbbbbbbbbbbb" 40 | }, 41 | "" 42 | ) 43 | 44 | assert with_unsignable_header == without_unsignable_header 45 | end 46 | 47 | test "presigned url" do 48 | # Data taken from example in: 49 | # http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html 50 | http_method = :get 51 | url = "https://examplebucket.s3.amazonaws.com/test.txt" 52 | service = :s3 53 | datetime = {{2013, 5, 24}, {0, 0, 0}} 54 | expires = 86400 55 | actual = ExAws.Auth.presigned_url(http_method, url, service, datetime, @config, expires) 56 | 57 | expected = 58 | "https://examplebucket.s3.amazonaws.com/test.txt" <> 59 | "?X-Amz-Algorithm=AWS4-HMAC-SHA256" <> 60 | "&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request" <> 61 | "&X-Amz-Date=20130524T000000Z" <> 62 | "&X-Amz-Expires=86400" <> 63 | "&X-Amz-SignedHeaders=host" <> 64 | "&X-Amz-Signature=aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404" 65 | 66 | assert {:ok, expected} == actual 67 | end 68 | 69 | test "presigned url with special characters" do 70 | http_method = :get 71 | url = "https://examplebucket.s3.amazonaws.com/folder-one/test+ #3.txt" 72 | service = :s3 73 | datetime = {{2013, 5, 24}, {0, 0, 0}} 74 | expires = 86400 75 | actual = ExAws.Auth.presigned_url(http_method, url, service, datetime, @config, expires) 76 | 77 | expected = 78 | "https://examplebucket.s3.amazonaws.com/folder-one/test%2B%20%233.txt" <> 79 | "?X-Amz-Algorithm=AWS4-HMAC-SHA256" <> 80 | "&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request" <> 81 | "&X-Amz-Date=20130524T000000Z" <> 82 | "&X-Amz-Expires=86400" <> 83 | "&X-Amz-SignedHeaders=host" <> 84 | "&X-Amz-Signature=d1892eeaf3110a6c1a805d8ad7a0c825a72a4255c7f48908922be55a7c4ae753" 85 | 86 | assert {:ok, expected} == actual 87 | end 88 | 89 | test "presigned url with query params" do 90 | # Data taken from example in: 91 | # http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html 92 | http_method = :put 93 | url = "https://examplebucket.s3.amazonaws.com/test.txt" 94 | service = :s3 95 | datetime = {{2013, 5, 24}, {0, 0, 0}} 96 | expires = 86400 97 | query_params = [partNumber: 1, uploadId: "sample.upload.id"] 98 | 99 | actual = 100 | ExAws.Auth.presigned_url( 101 | http_method, 102 | url, 103 | service, 104 | datetime, 105 | @config, 106 | expires, 107 | query_params 108 | ) 109 | 110 | expected = 111 | "https://examplebucket.s3.amazonaws.com/test.txt?partNumber=1&uploadId=sample.upload.id&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20130524T000000Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=1fdac5451b2996880dc23162853ce76e4cf0a05257e430aec59e309ecd126ade" 112 | 113 | assert {:ok, expected} == actual 114 | end 115 | 116 | test "presigned url with empty request body" do 117 | # Required for RDS generate_db_token 118 | # Data taken from botocore https://github.com/boto/botocore/blob/master/tests/unit/test_signers.py#L922 119 | http_method = :get 120 | url = "https://prod-instance.us-east-1.rds.amazonaws.com:3306/" 121 | service = :"rds-db" 122 | datetime = {{2016, 11, 7}, {17, 39, 33}} 123 | expires = 900 124 | query_params = [Action: "connect", DBUser: "someusername"] 125 | body = "" 126 | config = %{access_key_id: "akid", secret_access_key: "skid", region: "us-east-1"} 127 | 128 | actual = 129 | ExAws.Auth.presigned_url( 130 | http_method, 131 | url, 132 | service, 133 | datetime, 134 | config, 135 | expires, 136 | query_params, 137 | body 138 | ) 139 | 140 | expected = 141 | "https://prod-instance.us-east-1.rds.amazonaws.com:3306/?Action=connect&DBUser=someusername&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=akid%2F20161107%2Fus-east-1%2Frds-db%2Faws4_request&X-Amz-Date=20161107T173933Z&X-Amz-Expires=900&X-Amz-SignedHeaders=host&X-Amz-Signature=d1138cdbc0ca63eec012ec0fc6c2267e03642168f5884a7795320d4c18374c61" 142 | 143 | assert {:ok, expected} == actual 144 | end 145 | 146 | test "allow custom host & region" do 147 | config = 148 | ExAws.Config.new(:s3, host: %{"nyc3" => "nyc3.digitaloceanspaces.com"}, region: "nyc3") 149 | 150 | assert config.region == "nyc3" 151 | assert config.host == "nyc3.digitaloceanspaces.com" 152 | 153 | config = ExAws.Config.new(:s3, host: "nyc3.digitaloceanspaces.com", region: "nyc3") 154 | assert config.region == "nyc3" 155 | assert config.host == "nyc3.digitaloceanspaces.com" 156 | end 157 | 158 | describe "headers/6" do 159 | @config ExAws.Config.new(:s3, 160 | host: "nyc3.digitaloceanspaces.com", 161 | region: "eu-west-1", 162 | secret_access_key: "", 163 | access_key_id: "" 164 | ) 165 | 166 | test "builds authentication headers with X-Amzn-Trace-Id" do 167 | assert {:ok, headers} = 168 | headers( 169 | :get, 170 | "https://my-bucket.s3-eu-west-1.amazonaws.com", 171 | :s3, 172 | @config, 173 | [ 174 | {"X-Amzn-Trace-Id", "1-aaaaaaa-bbbbbbbbbbbbb"}, 175 | {"content-type", "application/json"} 176 | ], 177 | _body = "" 178 | ) 179 | 180 | {"Authorization", auth_header} = List.keyfind(headers, "Authorization", 0) 181 | assert String.contains?(auth_header, "x-amz-date") 182 | assert String.contains?(auth_header, "host") 183 | assert String.contains?(auth_header, "content-type") 184 | refute String.contains?(auth_header, "x-amz-security-token") 185 | refute String.contains?(auth_header, "x-amzn-trace-id") 186 | end 187 | 188 | test "keeps unsignable headers in the headers list" do 189 | assert {:ok, headers} = 190 | headers( 191 | :get, 192 | "https://my-bucket.s3-eu-west-1.amazonaws.com", 193 | :s3, 194 | @config, 195 | [ 196 | {"X-Amzn-Trace-Id", "1-aaaaaaa-bbbbbbbbbbbbb"}, 197 | {"content-type", "application/json"} 198 | ], 199 | _body = "" 200 | ) 201 | 202 | assert {"X-Amzn-Trace-Id", "1-aaaaaaa-bbbbbbbbbbbbb"} = 203 | List.keyfind(headers, "X-Amzn-Trace-Id", 0) 204 | end 205 | 206 | test "when security token is provided" do 207 | assert {:ok, headers} = 208 | headers( 209 | :get, 210 | "https://my-bucket.s3-eu-west-1.amazonaws.com", 211 | :s3, 212 | @config |> Map.put(:security_token, "abc"), 213 | [ 214 | {"content-type", "application/json"} 215 | ], 216 | _body = "" 217 | ) 218 | 219 | {"Authorization", auth_header} = List.keyfind(headers, "Authorization", 0) 220 | 221 | assert String.contains?(auth_header, "x-amz-security-token") 222 | end 223 | end 224 | end 225 | -------------------------------------------------------------------------------- /test/ex_aws/config_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ExAws.ConfigTest do 2 | use ExUnit.Case, async: true 3 | 4 | setup do 5 | Application.delete_env(:ex_aws, :awscli_credentials) 6 | 7 | on_exit(fn -> 8 | Application.delete_env(:ex_aws, :awscli_credentials) 9 | end) 10 | end 11 | 12 | test "overrides work properly" do 13 | config = ExAws.Config.new(:s3, region: "us-west-2") 14 | assert config.region == "us-west-2" 15 | end 16 | 17 | test "{:system} style configs work" do 18 | value = "foo" 19 | System.put_env("ExAwsConfigTest", value) 20 | 21 | assert :s3 22 | |> ExAws.Config.new( 23 | access_key_id: {:system, "ExAwsConfigTest"}, 24 | secret_access_key: {:system, "AWS_SECURITY_TOKEN"} 25 | ) 26 | |> Map.get(:access_key_id) == value 27 | end 28 | 29 | test "security_token is configured properly" do 30 | value = "security_token" 31 | System.put_env("AWS_SECURITY_TOKEN", value) 32 | 33 | assert :s3 34 | |> ExAws.Config.new( 35 | access_key_id: {:system, "AWS_SECURITY_TOKEN"}, 36 | security_token: {:system, "AWS_SECURITY_TOKEN"} 37 | ) 38 | |> Map.get(:security_token) == value 39 | end 40 | 41 | test "config file is parsed if no given credentials in configuraion" do 42 | profile = "default" 43 | 44 | Mox.expect(ExAws.Credentials.InitMock, :security_credentials, 1, fn ^profile -> 45 | {:ok, %{region: "eu-west-1"}} 46 | end) 47 | 48 | config = ExAws.Config.awscli_auth_credentials(profile, ExAws.Credentials.InitMock) 49 | 50 | assert config.region == "eu-west-1" 51 | end 52 | 53 | test "profile config returned if given credentials in configuration" do 54 | profile = "default" 55 | 56 | example_credentials = %{ 57 | "default" => %{ 58 | region: "eu-west-1" 59 | } 60 | } 61 | 62 | Application.put_env(:ex_aws, :awscli_credentials, example_credentials) 63 | 64 | Mox.expect(ExAws.Credentials.InitMock, :security_credentials, 0, fn ^profile -> 65 | %{region: "eu-west-1"} 66 | end) 67 | 68 | config = ExAws.Config.awscli_auth_credentials(profile, ExAws.Credentials.InitMock) 69 | 70 | assert config.region == "eu-west-1" 71 | end 72 | 73 | test "error on wrong credentials configuration" do 74 | profile = "other" 75 | 76 | example_credentials = %{ 77 | "default" => %{ 78 | region: "eu-west-1" 79 | } 80 | } 81 | 82 | Application.put_env(:ex_aws, :awscli_credentials, example_credentials) 83 | 84 | Mox.expect(ExAws.Credentials.InitMock, :security_credentials, 0, fn ^profile -> 85 | %{region: "eu-west-1"} 86 | end) 87 | 88 | assert_raise RuntimeError, fn -> 89 | ExAws.Config.awscli_auth_credentials(profile, ExAws.Credentials.InitMock) 90 | end 91 | end 92 | 93 | test "region as a plain string" do 94 | region_value = "us-west-1" 95 | 96 | assert :s3 97 | |> ExAws.Config.new(region: region_value) 98 | |> Map.get(:region) == region_value 99 | end 100 | 101 | test "region as an envar" do 102 | region_value = "us-west-1" 103 | System.put_env("AWS_REGION", region_value) 104 | 105 | assert :s3 106 | |> ExAws.Config.new(region: {:system, "AWS_REGION"}) 107 | |> Map.get(:region) == region_value 108 | end 109 | 110 | test "headers are passed as provided" do 111 | headers = [{"If-Match", "ABC"}] 112 | 113 | assert :s3 114 | |> ExAws.Config.new(headers: headers) 115 | |> Map.get(:headers) == headers 116 | end 117 | end 118 | -------------------------------------------------------------------------------- /test/ex_aws/credentials_ini/file_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ExAws.CredentialsIni.File.FileTest do 2 | use ExUnit.Case, async: true 3 | 4 | test "credentials file is parsed" do 5 | example_credentials = """ 6 | [default] 7 | aws_access_key_id = TESTKEYID 8 | aws_secret_access_key = TESTSECRET 9 | aws_session_token = TESTTOKEN 10 | """ 11 | 12 | credentials = 13 | ExAws.CredentialsIni.File.parse_ini_file({:ok, example_credentials}, "default") 14 | |> ExAws.CredentialsIni.File.replace_token_key() 15 | 16 | assert credentials.access_key_id == "TESTKEYID" 17 | assert credentials.secret_access_key == "TESTSECRET" 18 | assert credentials.security_token == "TESTTOKEN" 19 | end 20 | 21 | test "config file is parsed with sso config" do 22 | example_config = """ 23 | [default] 24 | sso_start_url = https://start.us-gov-home.awsapps.com/directory/somecompany 25 | sso_region = us-gov-west-1 26 | sso_account_id = 123456789101 27 | sso_role_name = SomeRole 28 | region = us-gov-west-1 29 | output = json 30 | """ 31 | 32 | config = ExAws.CredentialsIni.File.parse_ini_file({:ok, example_config}, "default") 33 | 34 | assert config.sso_start_url == "https://start.us-gov-home.awsapps.com/directory/somecompany" 35 | assert config.sso_region == "us-gov-west-1" 36 | assert config.sso_account_id == "123456789101" 37 | assert config.sso_role_name == "SomeRole" 38 | end 39 | 40 | test "config file is parsed with sso config that uses sso_session" do 41 | example_config = """ 42 | [sso-session somecompany] 43 | sso_start_url = https://start.us-gov-home.awsapps.com/directory/somecompany 44 | sso_region = us-gov-west-1 45 | 46 | [default] 47 | sso_session = somecompany 48 | sso_account_id = 123456789101 49 | sso_role_name = SomeRole 50 | region = us-gov-west-1 51 | output = json 52 | """ 53 | 54 | config = ExAws.CredentialsIni.File.parse_ini_file({:ok, example_config}, "default") 55 | 56 | assert config.sso_session == "somecompany" 57 | assert config.sso_start_url == "https://start.us-gov-home.awsapps.com/directory/somecompany" 58 | assert config.sso_region == "us-gov-west-1" 59 | assert config.sso_account_id == "123456789101" 60 | assert config.sso_role_name == "SomeRole" 61 | end 62 | 63 | test "config file is parsed with non-default sso config profile that uses sso_session" do 64 | example_config = """ 65 | [sso-session somecompany] 66 | sso_start_url = https://start.us-gov-home.awsapps.com/directory/somecompany 67 | sso_region = us-gov-west-1 68 | 69 | [profile somecompany] 70 | sso_session = somecompany 71 | sso_account_id = 123456789101 72 | sso_role_name = SomeRole 73 | region = us-gov-west-1 74 | output = json 75 | """ 76 | 77 | config = ExAws.CredentialsIni.File.parse_ini_file({:ok, example_config}, "somecompany") 78 | 79 | assert config.sso_session == "somecompany" 80 | assert config.sso_start_url == "https://start.us-gov-home.awsapps.com/directory/somecompany" 81 | assert config.sso_region == "us-gov-west-1" 82 | assert config.sso_account_id == "123456789101" 83 | assert config.sso_role_name == "SomeRole" 84 | end 85 | 86 | test "{:system} in profile name gets dynamic profile name" do 87 | System.put_env("AWS_PROFILE", "custom-profile") 88 | 89 | example_credentials = """ 90 | [custom-profile] 91 | aws_access_key_id = TESTKEYID 92 | aws_secret_access_key = TESTSECRET 93 | aws_session_token = TESTTOKEN 94 | """ 95 | 96 | credentials = 97 | ExAws.CredentialsIni.File.parse_ini_file({:ok, example_credentials}, :system) 98 | |> ExAws.CredentialsIni.File.replace_token_key() 99 | 100 | assert credentials.access_key_id == "TESTKEYID" 101 | assert credentials.secret_access_key == "TESTSECRET" 102 | assert credentials.security_token == "TESTTOKEN" 103 | end 104 | 105 | test "{:system} in profile name gets dynamic profile name using sso config" do 106 | System.put_env("AWS_PROFILE", "custom-profile") 107 | 108 | example_credentials = """ 109 | [sso-session somecompany] 110 | sso_start_url = https://start.us-gov-home.awsapps.com/directory/somecompany 111 | sso_region = us-gov-west-1 112 | 113 | [profile custom-profile] 114 | sso_session = somecompany 115 | sso_account_id = 123456789101 116 | sso_role_name = SomeRole 117 | region = us-gov-west-1 118 | output = json 119 | """ 120 | 121 | credentials = 122 | ExAws.CredentialsIni.File.parse_ini_file({:ok, example_credentials}, :system) 123 | |> ExAws.CredentialsIni.File.replace_token_key() 124 | 125 | assert credentials.sso_session == "somecompany" 126 | 127 | assert credentials.sso_start_url == 128 | "https://start.us-gov-home.awsapps.com/directory/somecompany" 129 | 130 | assert credentials.sso_region == "us-gov-west-1" 131 | assert credentials.sso_account_id == "123456789101" 132 | assert credentials.sso_role_name == "SomeRole" 133 | end 134 | 135 | test "config file is parsed" do 136 | example_config = """ 137 | [default] 138 | region = eu-west-1 139 | """ 140 | 141 | config = ExAws.CredentialsIni.File.parse_ini_file({:ok, example_config}, "default") 142 | 143 | assert config.region == "eu-west-1" 144 | end 145 | end 146 | -------------------------------------------------------------------------------- /test/ex_aws/ex_aws_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ExAwsTest do 2 | use ExUnit.Case, async: true 3 | 4 | # Skip until I figure out what the issue here is 5 | @tag :skip 6 | test "basic S3 operation works" do 7 | op = %ExAws.Operation.S3{ 8 | body: "", 9 | bucket: "", 10 | headers: %{}, 11 | http_method: :get, 12 | params: [], 13 | parser: & &1, 14 | path: "/", 15 | resource: "", 16 | service: :s3, 17 | stream_builder: nil 18 | } 19 | 20 | assert {:ok, %{body: _}} = ExAws.request(op) 21 | end 22 | 23 | test "basic json operation works" do 24 | TelemetryHelper.attach_telemetry([:ex_aws, :request]) 25 | 26 | op = %ExAws.Operation.JSON{ 27 | http_method: :post, 28 | service: :dynamodb, 29 | headers: [ 30 | {"x-amz-target", "DynamoDB_20120810.ListTables"}, 31 | {"content-type", "application/x-amz-json-1.0"} 32 | ] 33 | } 34 | 35 | assert {:ok, %{"TableNames" => _}} = ExAws.request(op) 36 | 37 | assert_receive {[:ex_aws, :request, :start], %{system_time: _}, 38 | %{ 39 | attempt: 1, 40 | options: [], 41 | request_body: "{}" 42 | }} 43 | 44 | assert_receive {[:ex_aws, :request, :stop], %{duration: _}, 45 | %{ 46 | options: [], 47 | attempt: 1, 48 | request_body: "{}", 49 | result: :ok, 50 | response_body: "{\"TableNames\":[]}" 51 | }} 52 | end 53 | 54 | test "json operation with differently named service" do 55 | op = %ExAws.Operation.JSON{ 56 | before_request: nil, 57 | data: %{}, 58 | headers: [ 59 | {"x-amz-target", "DynamoDBStreams_20120810.ListStreams"}, 60 | {"content-type", "application/x-amz-json-1.0"} 61 | ], 62 | http_method: :post, 63 | params: %{}, 64 | parser: & &1, 65 | path: "/", 66 | service: :dynamodb_streams, 67 | stream_builder: nil 68 | } 69 | 70 | assert {:ok, %{"Streams" => _}} = ExAws.request(op) 71 | end 72 | 73 | test "custom telemetry options" do 74 | TelemetryHelper.attach_telemetry([:ex_aws, :custom_request]) 75 | 76 | op = %ExAws.Operation.JSON{ 77 | http_method: :post, 78 | service: :dynamodb, 79 | headers: [ 80 | {"x-amz-target", "DynamoDB_20120810.ListTables"}, 81 | {"content-type", "application/x-amz-json-1.0"} 82 | ] 83 | } 84 | 85 | options = [telemetry_event: [:ex_aws, :custom_request], telemetry_options: [name: :sample]] 86 | 87 | assert {:ok, %{"TableNames" => _}} = ExAws.request(op, options) 88 | 89 | assert_receive {[:ex_aws, :custom_request, :start], %{system_time: _}, 90 | %{ 91 | attempt: 1, 92 | options: [name: :sample] 93 | }} 94 | 95 | assert_receive {[:ex_aws, :custom_request, :stop], %{duration: _}, 96 | %{ 97 | options: [name: :sample], 98 | attempt: 1, 99 | result: :ok 100 | }} 101 | end 102 | 103 | test "telemetry operation and service" do 104 | TelemetryHelper.attach_telemetry([:ex_aws, :custom_request]) 105 | 106 | op = %ExAws.Operation.JSON{ 107 | http_method: :post, 108 | service: :dynamodb, 109 | headers: [ 110 | {"x-amz-target", "DynamoDB_20120810.ListTables"}, 111 | {"content-type", "application/x-amz-json-1.0"} 112 | ] 113 | } 114 | 115 | options = [telemetry_event: [:ex_aws, :custom_request]] 116 | 117 | assert {:ok, %{"TableNames" => _}} = ExAws.request(op, options) 118 | 119 | assert_receive {[:ex_aws, :custom_request, :start], %{system_time: _}, 120 | %{ 121 | attempt: 1, 122 | operation: "DynamoDB_20120810.ListTables", 123 | service: :dynamodb, 124 | options: [] 125 | }} 126 | 127 | assert_receive {[:ex_aws, :custom_request, :stop], %{duration: _}, 128 | %{ 129 | attempt: 1, 130 | operation: "DynamoDB_20120810.ListTables", 131 | service: :dynamodb, 132 | options: [], 133 | result: :ok 134 | }} 135 | end 136 | 137 | test "invalid request" do 138 | TelemetryHelper.attach_telemetry([:ex_aws, :request]) 139 | 140 | op = %ExAws.Operation.JSON{ 141 | http_method: :post, 142 | service: :dynamodb, 143 | headers: [ 144 | {"x-amz-target", "DynamoDB_20120810.CreateTable"}, 145 | {"content-type", "application/x-amz-json-1.0"} 146 | ] 147 | } 148 | 149 | assert {:error, {"ValidationException", _}} = ExAws.request(op) 150 | 151 | assert_receive {[:ex_aws, :request, :start], %{system_time: _}, 152 | %{ 153 | options: [], 154 | attempt: 1, 155 | request_body: "{}" 156 | }} 157 | 158 | assert_receive {[:ex_aws, :request, :stop], %{duration: _}, 159 | %{ 160 | options: [], 161 | attempt: 1, 162 | request_body: "{}", 163 | result: :error, 164 | error: 165 | ~s({"__type":"com.amazon.coral.validate#ValidationException","Message":"Invalid table/index name. Table/index names must be between 3 and 255 characters long, and may contain only the characters a-z, A-Z, 0-9, '_', '-', and '.'"}) 166 | }} 167 | end 168 | end 169 | -------------------------------------------------------------------------------- /test/ex_aws/instance_meta_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ExAws.InstanceMetaTest do 2 | use ExUnit.Case, async: false 3 | 4 | import Mox 5 | 6 | # Let expect statements apply to ExAws.InstanceMetaTokenProvider process as well 7 | setup :set_mox_from_context 8 | 9 | test "instance_role" do 10 | role_name = "dummy-role" 11 | 12 | ExAws.Request.HttpMock 13 | |> expect(:request, fn _method, _url, _body, _headers, _opts -> 14 | {:ok, %{status_code: 200, body: role_name}} 15 | end) 16 | 17 | config = 18 | ExAws.Config.new(:s3, 19 | http_client: ExAws.Request.HttpMock, 20 | access_key_id: "dummy", 21 | secret_access_key: "dummy", 22 | # Don't cache the metadata token, so we can always expect a request to get the token 23 | no_metadata_token_cache: true 24 | ) 25 | 26 | assert ExAws.InstanceMeta.instance_role(config) == role_name 27 | end 28 | 29 | describe "metadata options" do 30 | setup %{metadata_opts: metadata_opts} do 31 | metadata_opts_old = Application.get_env(:ex_aws, :metadata, nil) 32 | 33 | on_exit(fn -> 34 | case metadata_opts_old do 35 | nil -> 36 | Application.delete_env(:ex_aws, :metadata) 37 | 38 | _other -> 39 | Application.put_env(:ex_aws, :metadata, metadata_opts_old) 40 | end 41 | end) 42 | 43 | Application.put_env(:ex_aws, :metadata, metadata_opts) 44 | end 45 | 46 | @tag metadata_opts: [http_opts: [pool: :ex_aws_metadata]] 47 | test "separate http opts for instance metadata" do 48 | role_name = "dummy-role" 49 | 50 | ExAws.Request.HttpMock 51 | |> expect(:request, fn _method, _url, _body, _headers, opts -> 52 | assert Keyword.get(opts, :pool) == :ex_aws_metadata 53 | {:ok, %{status_code: 200, body: role_name}} 54 | end) 55 | 56 | config = 57 | ExAws.Config.new(:ec2, 58 | http_client: ExAws.Request.HttpMock, 59 | access_key_id: "dummy", 60 | secret_access_key: "dummy", 61 | # Don't cache the metadata token, so we can always expect a request to get the token 62 | no_metadata_token_cache: true 63 | ) 64 | 65 | assert ExAws.InstanceMeta.instance_role(config) == role_name 66 | end 67 | end 68 | 69 | describe "IMDSv2" do 70 | test "when initial metadata request fails with a 401, fallback to IMDSv2 flow" do 71 | role_name = "dummy-role-imdsv2" 72 | 73 | ExAws.Request.HttpMock 74 | |> expect(:request, fn :get, _url, _body, _headers, _opts -> 75 | {:ok, %{status_code: 401, body: ""}} 76 | end) 77 | |> expect(:request, fn :put, _url, _body, _headers, _opts -> 78 | {:ok, %{status_code: 200, body: "dummy-token"}} 79 | end) 80 | |> expect(:request, fn :get, _url, _body, headers, _opts -> 81 | assert Enum.member?(headers, {"x-aws-ec2-metadata-token", "dummy-token"}) 82 | {:ok, %{status_code: 200, body: role_name}} 83 | end) 84 | 85 | config = 86 | ExAws.Config.new(:s3, 87 | http_client: ExAws.Request.HttpMock, 88 | access_key_id: "dummy", 89 | secret_access_key: "dummy", 90 | # Don't cache the metadata token, so we can always expect a request to get the token 91 | no_metadata_token_cache: true 92 | ) 93 | 94 | assert ExAws.InstanceMeta.instance_role(config) == role_name 95 | end 96 | 97 | test "configuration to use IMDSv2 by default" do 98 | role_name = "dummy-role-imdsv2" 99 | 100 | ExAws.Request.HttpMock 101 | |> expect(:request, fn :put, _url, _body, _headers, _opts -> 102 | {:ok, %{status_code: 200, body: "dummy-token"}} 103 | end) 104 | |> expect(:request, fn :get, _url, _body, headers, _opts -> 105 | assert Enum.member?(headers, {"x-aws-ec2-metadata-token", "dummy-token"}) 106 | {:ok, %{status_code: 200, body: role_name}} 107 | end) 108 | 109 | config = 110 | ExAws.Config.new(:s3, 111 | http_client: ExAws.Request.HttpMock, 112 | access_key_id: "dummy", 113 | secret_access_key: "dummy", 114 | # Don't cache the metadata token, so we can always expect a request to get the token 115 | no_metadata_token_cache: true, 116 | require_imds_v2: true 117 | ) 118 | 119 | assert ExAws.InstanceMeta.instance_role(config) == role_name 120 | end 121 | end 122 | end 123 | -------------------------------------------------------------------------------- /test/ex_aws/operation/s3_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Operation.S3Test do 2 | use ExUnit.Case, async: true 3 | 4 | alias Elixir.ExAws.Operation.ExAws.Operation.S3 5 | 6 | def s3_operation(bucket \\ "my-bucket-1") do 7 | %ExAws.Operation.S3{ 8 | body: "", 9 | bucket: bucket, 10 | headers: %{}, 11 | http_method: :get, 12 | params: [], 13 | parser: & &1, 14 | path: "/folder", 15 | resource: "", 16 | service: :s3, 17 | stream_builder: nil 18 | } 19 | end 20 | 21 | test "S3 adds bucket to path when virtual_host is missing from config" do 22 | config = ExAws.Config.new(:s3) 23 | operation = s3_operation() 24 | 25 | {processed_operation, processed_config} = S3.add_bucket_to_path(operation, config) 26 | 27 | assert(processed_config.host == config.host) 28 | assert(processed_operation.path == "/#{operation.bucket}#{operation.path}") 29 | end 30 | 31 | test "S3 adds bucket to path when virtual_host is false" do 32 | config = ExAws.Config.new(:s3) |> Map.put(:virtual_host, false) 33 | operation = s3_operation() 34 | 35 | {processed_operation, processed_config} = S3.add_bucket_to_path(operation, config) 36 | 37 | assert(processed_config.host == config.host) 38 | assert(processed_operation.path == "/#{operation.bucket}#{operation.path}") 39 | end 40 | 41 | test "S3 adds bucket to path when virtual_host is true" do 42 | config = ExAws.Config.new(:s3) |> Map.put(:virtual_host, true) 43 | operation = s3_operation() 44 | 45 | {processed_operation, processed_config} = S3.add_bucket_to_path(operation, config) 46 | 47 | assert(processed_config.host == "#{operation.bucket}.#{config.host}") 48 | assert(processed_operation.path == operation.path) 49 | end 50 | 51 | test "S3 raises when bucket is nil" do 52 | config = ExAws.Config.new(:s3) 53 | operation = s3_operation(nil) 54 | 55 | assert_raise RuntimeError, 56 | "#{S3}.perform/2 cannot perform operation on `nil` bucket", 57 | fn -> S3.add_bucket_to_path(operation, config) end 58 | end 59 | 60 | test "ensure paths with . and .. are correctly resolved" do 61 | config = ExAws.Config.new(:s3) 62 | operation = %{s3_operation() | path: "a/../b/../c/./d"} 63 | 64 | {processed_operation, _processed_config} = S3.add_bucket_to_path(operation, config) 65 | assert processed_operation.path == "/my-bucket-1/c/d" 66 | end 67 | 68 | test "ensure paths ending with / preserve the trailing /" do 69 | config = ExAws.Config.new(:s3) |> Map.put(:virtual_host, false) 70 | operation = %{s3_operation() | path: "folder/"} 71 | 72 | {processed_operation, _processed_config} = S3.add_bucket_to_path(operation, config) 73 | assert processed_operation.path == "/my-bucket-1/folder/" 74 | end 75 | 76 | test "ensure paths ending with / preserve the trailing / (virtual-host)" do 77 | config = ExAws.Config.new(:s3) |> Map.put(:virtual_host, true) 78 | operation = %{s3_operation() | path: "folder/"} 79 | 80 | {processed_operation, _processed_config} = S3.add_bucket_to_path(operation, config) 81 | assert processed_operation.path == "/folder/" 82 | end 83 | end 84 | -------------------------------------------------------------------------------- /test/ex_aws/request/req_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Request.ReqTest do 2 | use ExUnit.Case, async: true 3 | 4 | test "ExAws.Request conformance" do 5 | plug = fn conn -> 6 | attempt = Process.get(:retry_attempt, 0) 7 | 8 | if attempt < 3 do 9 | Process.put(:retry_attempt, attempt + 1) 10 | Plug.Conn.send_resp(conn, 500, "oops") 11 | else 12 | assert conn.host == "test-server" 13 | assert Plug.Conn.get_req_header(conn, "x-foo") == ["bar"] 14 | {:ok, body, conn} = Plug.Conn.read_body(conn) 15 | assert body == ~s|{"message":"hello"}| 16 | Req.Test.json(conn, %{attempt: attempt}) 17 | end 18 | end 19 | 20 | config = %{ 21 | http_client: ExAws.Request.Req, 22 | http_opts: [ 23 | plug: plug 24 | ], 25 | retries: [ 26 | base_backoff_in_ms: 1 27 | ], 28 | json_codec: Jason, 29 | access_key_id: "AKIAIOSFODNN7EXAMPLE", 30 | secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", 31 | region: "us-east-1" 32 | } 33 | 34 | {:ok, resp} = 35 | ExAws.Request.request( 36 | :post, 37 | "https://test-server", 38 | %{message: "hello"}, 39 | [{"x-foo", "bar"}], 40 | config, 41 | :s3 42 | ) 43 | 44 | assert resp.status_code == 200 45 | 46 | assert List.keyfind(resp.headers, "content-type", 0) == 47 | {"content-type", "application/json; charset=utf-8"} 48 | 49 | assert resp.body == ~s|{"attempt":3}| 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /test/ex_aws/request/url_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ExAws.Request.UrlTest do 2 | use ExUnit.Case, async: true 3 | alias ExAws.Request.Url 4 | 5 | describe "build" do 6 | setup do 7 | query = %{ 8 | path: "/path", 9 | params: %{foo: :bar} 10 | } 11 | 12 | config = %{ 13 | scheme: "https", 14 | host: "example.com", 15 | port: 443, 16 | normalize_path: true 17 | } 18 | 19 | {:ok, %{query: query, config: config}} 20 | end 21 | 22 | test "it build urls for query operation", %{query: query, config: config} do 23 | assert Url.build(query, config) == "https://example.com/path?foo=bar" 24 | end 25 | 26 | test "it allows setting custom port", %{query: query, config: config} do 27 | config = config |> Map.put(:port, 4430) 28 | assert Url.build(query, config) == "https://example.com:4430/path?foo=bar" 29 | end 30 | 31 | test "it converts the port to an integer if it is a string", %{query: query, config: config} do 32 | config = config |> Map.put(:port, "4430") 33 | assert Url.build(query, config) == "https://example.com:4430/path?foo=bar" 34 | end 35 | 36 | test "it allows passing scheme with trailing ://", %{query: query, config: config} do 37 | config = config |> Map.put(:scheme, "https://") 38 | assert Url.build(query, config) == "https://example.com/path?foo=bar" 39 | end 40 | 41 | test "it accepts params as a list of keywords", %{query: query, config: config} do 42 | query = query |> Map.put(:params, foo: :bar) 43 | assert Url.build(query, config) == "https://example.com/path?foo=bar" 44 | end 45 | 46 | test "it does not have trailing ? when params is an empty map", %{ 47 | query: query, 48 | config: config 49 | } do 50 | query = query |> Map.put(:params, %{}) 51 | assert Url.build(query, config) == "https://example.com/path" 52 | end 53 | 54 | test "it does not have trailing ? when params is an empty list", %{ 55 | query: query, 56 | config: config 57 | } do 58 | query = query |> Map.put(:params, []) 59 | assert Url.build(query, config) == "https://example.com/path" 60 | end 61 | 62 | test "it accepts query without params key", %{query: query, config: config} do 63 | query = query |> Map.delete(:params) 64 | assert Url.build(query, config) == "https://example.com/path" 65 | end 66 | 67 | test "it cleans up excessive slashes in the path", %{query: query, config: config} do 68 | query = query |> Map.put(:path, "//path///with/too/many//slashes//") 69 | assert Url.build(query, config) == "https://example.com/path/with/too/many/slashes/?foo=bar" 70 | end 71 | 72 | test "it ignores empty parameter key", %{query: query, config: config} do 73 | query = query |> Map.put(:params, %{"foo" => "bar", "" => 1}) 74 | assert Url.build(query, config) == "https://example.com/path?foo=bar" 75 | end 76 | 77 | test "it ignores nil parameter key", %{query: query, config: config} do 78 | query = query |> Map.put(:params, %{"foo" => "bar", nil => 1}) 79 | assert Url.build(query, config) == "https://example.com/path?foo=bar" 80 | end 81 | end 82 | 83 | describe "get_path" do 84 | test "it uses S3-specific URL parsing to keep the path for S3 services" do 85 | url = "https://example.com/uploads/invalid path but+valid//for#s3/haha.txt" 86 | assert Url.get_path(url, :s3) == "/uploads/invalid path but+valid//for#s3/haha.txt" 87 | end 88 | 89 | test "it uses standard URL parsing for the path for non-S3 services" do 90 | url = "https://example.com/uploads/invalid path but+valid//for#i-am-anchor" 91 | assert Url.get_path(url) == "/uploads/invalid path but+valid//for" 92 | end 93 | 94 | test "returns an empty path when there isn't one specified in the url" do 95 | url = "https://example.com" 96 | assert Url.get_path(url) == "/" 97 | end 98 | end 99 | 100 | describe "sanitize" do 101 | setup do 102 | query = %ExAws.Operation.S3{ 103 | body: "", 104 | bucket: "", 105 | headers: %{}, 106 | http_method: :get, 107 | params: %{ 108 | "marker" => "docs/MS FINCH+Paid_25_Oct_2017_16:54:15.pdf" 109 | }, 110 | path: "/", 111 | resource: "", 112 | service: :s3 113 | } 114 | 115 | config = %{ 116 | scheme: "https", 117 | host: "example.com", 118 | port: 443, 119 | normalize_path: true 120 | } 121 | 122 | {:ok, %{query: query, config: config}} 123 | end 124 | 125 | test "it build urls for query operation", %{query: query, config: config} do 126 | assert Url.build(query, config) |> Url.sanitize(query.service) == 127 | "https://example.com/?marker=docs%2FMS%20FINCH%2BPaid_25_Oct_2017_16%3A54%3A15.pdf" 128 | end 129 | end 130 | end 131 | -------------------------------------------------------------------------------- /test/ex_aws/request_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ExAws.RequestTest do 2 | use ExUnit.Case, async: false 3 | import ExUnit.CaptureLog 4 | alias ExAws.JSON.JSX 5 | import Mox 6 | 7 | setup do 8 | {:ok, 9 | config: %{ 10 | http_client: ExAws.Request.HttpMock, 11 | json_codec: JSX, 12 | access_key_id: "AKIAIOSFODNN7EXAMPLE", 13 | secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", 14 | region: "us-east-1", 15 | retries: [ 16 | max_attempts: 5, 17 | base_backoff_in_ms: 1, 18 | max_backoff_in_ms: 20 19 | ] 20 | }, 21 | headers: [ 22 | {"x-amz-bucket-region", "us-east-1"}, 23 | {"x-amz-content-sha256", ExAws.Auth.Utils.hash_sha256("")}, 24 | {"content-length", byte_size("")} 25 | ]} 26 | end 27 | 28 | test "301 redirect", context do 29 | TelemetryHelper.attach_telemetry([:ex_aws, :request]) 30 | 31 | ExAws.Request.HttpMock 32 | |> expect(:request, fn _method, _url, _body, _headers, _opts -> {:ok, %{status_code: 301}} end) 33 | 34 | http_method = :get 35 | url = "https://examplebucket.s3.amazonaws.com/test.txt" 36 | service = :s3 37 | request_body = "" 38 | 39 | assert capture_log(fn -> 40 | assert {:error, {:http_error, 301, "redirected"}} == 41 | ExAws.Request.request_and_retry( 42 | http_method, 43 | url, 44 | service, 45 | context[:config], 46 | context[:headers], 47 | request_body, 48 | {:attempt, 1} 49 | ) 50 | end) =~ "Received redirect, did you specify the correct region?" 51 | 52 | assert_receive {[:ex_aws, :request, :start], %{system_time: _}, 53 | %{ 54 | options: [], 55 | attempt: 1, 56 | service: :s3 57 | }} 58 | 59 | assert_receive {[:ex_aws, :request, :stop], %{duration: _}, 60 | %{ 61 | options: [], 62 | attempt: 1, 63 | service: :s3, 64 | result: :error 65 | }} 66 | end 67 | 68 | test "handles encoding S3 URLs with params", context do 69 | http_method = :get 70 | url = "https://examplebucket.s3.amazonaws.com/test hello #3.txt?acl=21" 71 | service = :s3 72 | request_body = "" 73 | 74 | TelemetryHelper.attach_telemetry([:ex_aws, :request]) 75 | 76 | expect( 77 | ExAws.Request.HttpMock, 78 | :request, 79 | fn _method, url, _body, _headers, _opts -> 80 | assert url == "https://examplebucket.s3.amazonaws.com/test%20hello%20%233.txt?acl=21" 81 | {:ok, %{status_code: 200}} 82 | end 83 | ) 84 | 85 | assert {:ok, %{status_code: 200}} == 86 | ExAws.Request.request_and_retry( 87 | http_method, 88 | url, 89 | service, 90 | context[:config], 91 | context[:headers], 92 | request_body, 93 | {:attempt, 1} 94 | ) 95 | 96 | assert_receive {[:ex_aws, :request, :start], %{system_time: _}, 97 | %{ 98 | options: [], 99 | attempt: 1, 100 | service: :s3 101 | }} 102 | 103 | assert_receive {[:ex_aws, :request, :stop], %{duration: _}, 104 | %{ 105 | options: [], 106 | attempt: 1, 107 | service: :s3, 108 | result: :ok 109 | }} 110 | end 111 | 112 | test "handles encoding S3 URLs without params", context do 113 | http_method = :get 114 | url = "https://examplebucket.s3.amazonaws.com/up//double//test hello+#3.txt" 115 | service = :s3 116 | request_body = "" 117 | 118 | expect( 119 | ExAws.Request.HttpMock, 120 | :request, 121 | fn _method, url, _body, _headers, _opts -> 122 | assert url == "https://examplebucket.s3.amazonaws.com/up//double//test%20hello%2B%233.txt" 123 | {:ok, %{status_code: 200}} 124 | end 125 | ) 126 | 127 | assert {:ok, %{status_code: 200}} == 128 | ExAws.Request.request_and_retry( 129 | http_method, 130 | url, 131 | service, 132 | context[:config], 133 | context[:headers], 134 | request_body, 135 | {:attempt, 1} 136 | ) 137 | end 138 | 139 | test "ProvisionedThroughputExceededException is retried", context do 140 | TelemetryHelper.attach_telemetry([:ex_aws, :request]) 141 | success = mock_provisioned_throughput_response(2) 142 | 143 | http_method = :post 144 | url = "https://kinesis.aws.com/" 145 | service = :kinesis 146 | request_body = "" 147 | 148 | assert {:ok, %{body: success, status_code: 200}} == 149 | ExAws.Request.request_and_retry( 150 | http_method, 151 | url, 152 | service, 153 | context[:config], 154 | context[:headers], 155 | request_body, 156 | {:attempt, 1} 157 | ) 158 | 159 | assert_receive {[:ex_aws, :request, :start], %{system_time: _}, %{attempt: 1}} 160 | assert_receive {[:ex_aws, :request, :stop], %{duration: _}, %{attempt: 1, result: :error}} 161 | assert_receive {[:ex_aws, :request, :start], %{system_time: _}, %{attempt: 2}} 162 | assert_receive {[:ex_aws, :request, :stop], %{duration: _}, %{attempt: 2, result: :error}} 163 | assert_receive {[:ex_aws, :request, :start], %{system_time: _}, %{attempt: 3}} 164 | assert_receive {[:ex_aws, :request, :stop], %{duration: _}, %{attempt: 3, result: :ok}} 165 | end 166 | 167 | test "ProvisionedThroughputExceededException is not retried if client error retries is set to 1", 168 | context do 169 | TelemetryHelper.attach_telemetry([:ex_aws, :request]) 170 | mock_provisioned_throughput_response(1) 171 | 172 | http_method = :post 173 | url = "https://kinesis.aws.com/" 174 | service = :kinesis 175 | request_body = "" 176 | 177 | config = context[:config] |> put_in([:retries, :client_error_max_attempts], 1) 178 | 179 | assert {:error, {"ProvisionedThroughputExceededException", _}} = 180 | ExAws.Request.request_and_retry( 181 | http_method, 182 | url, 183 | service, 184 | config, 185 | context[:headers], 186 | request_body, 187 | {:attempt, 1} 188 | ) 189 | 190 | assert_receive {[:ex_aws, :request, :start], %{system_time: _}, %{attempt: 1}} 191 | assert_receive {[:ex_aws, :request, :stop], %{duration: _}, %{attempt: 1, result: :error}} 192 | refute_receive {[:ex_aws, :request, :start], %{system_time: _}, %{attempt: 2}} 193 | end 194 | 195 | test "Expected sequence token is provided", context do 196 | exception = 197 | "{\"__type\": \"InvalidSequenceTokenException\", \"message\": \"The given sequenceToken is invalid. The next expected sequenceToken is: 49616449618992442982853194240983586320797062450229805234\", \"expectedSequenceToken\": \"49616449618992442982853194240983586320797062450229805234\"}" 198 | 199 | ExAws.Request.HttpMock 200 | |> expect(:request, fn _method, _url, _body, _headers, _opts -> 201 | {:ok, %{status_code: 400, body: exception}} 202 | end) 203 | 204 | http_method = :post 205 | url = "https://kinesis.aws.com/" 206 | service = :kinesis 207 | request_body = "" 208 | 209 | assert {:error, 210 | {"InvalidSequenceTokenException", _, 211 | "49616449618992442982853194240983586320797062450229805234"}} = 212 | ExAws.Request.request_and_retry( 213 | http_method, 214 | url, 215 | service, 216 | context[:config], 217 | context[:headers], 218 | request_body, 219 | {:attempt, 1} 220 | ) 221 | end 222 | 223 | test "Retries on errors, when the error reason is a map", context do 224 | ExAws.Request.HttpMock 225 | |> expect(:request, fn _method, _url, _body, _headers, _opts -> 226 | {:error, %{reason: :closed}} 227 | end) 228 | 229 | http_method = :get 230 | url = "https://examplebucket.s3.amazonaws.com/test.txt" 231 | service = :s3 232 | request_body = "" 233 | 234 | assert {:error, :closed} == 235 | ExAws.Request.request_and_retry( 236 | http_method, 237 | url, 238 | service, 239 | context[:config], 240 | context[:headers], 241 | request_body, 242 | {:attempt, 5} 243 | ) 244 | end 245 | 246 | test "Retries on errors, when the error reason is a keyword list", context do 247 | ExAws.Request.HttpMock 248 | |> expect(:request, fn _method, _url, _body, _headers, _opts -> 249 | {:error, [reason: :closed]} 250 | end) 251 | 252 | http_method = :get 253 | url = "https://examplebucket.s3.amazonaws.com/test.txt" 254 | service = :s3 255 | request_body = "" 256 | 257 | assert {:error, :closed} == 258 | ExAws.Request.request_and_retry( 259 | http_method, 260 | url, 261 | service, 262 | context[:config], 263 | context[:headers], 264 | request_body, 265 | {:attempt, 5} 266 | ) 267 | end 268 | 269 | test "TooManyRequestsException is retried", context do 270 | TelemetryHelper.attach_telemetry([:ex_aws, :request]) 271 | success = mock_too_many_requests_exception(3) 272 | 273 | http_method = :post 274 | url = "https://cognito-idp.eu-west-1.amazonaws.com" 275 | service = :"cognito-idp" 276 | 277 | request_body = 278 | "{\"MessageAction\":\"SUPPRESS\",\"UserAttributes\":[{\"Name\":\"email_verified\",\"Value\":\"False\"},{\"Name\":\"email\",\"Value\":\"user-email@test.com\"}],\"UserPoolId\":\" eu-west-1_abc1dEFGH\",\"Username\":\"user-email@test.com\"}" 279 | 280 | assert {:ok, %{body: success, status_code: 200}} == 281 | ExAws.Request.request_and_retry( 282 | http_method, 283 | url, 284 | service, 285 | context[:config], 286 | context[:headers], 287 | request_body, 288 | {:attempt, 1} 289 | ) 290 | 291 | assert_receive {[:ex_aws, :request, :start], %{system_time: _}, %{attempt: 1}} 292 | assert_receive {[:ex_aws, :request, :stop], %{duration: _}, %{attempt: 1, result: :error}} 293 | assert_receive {[:ex_aws, :request, :start], %{system_time: _}, %{attempt: 2}} 294 | assert_receive {[:ex_aws, :request, :stop], %{duration: _}, %{attempt: 2, result: :error}} 295 | assert_receive {[:ex_aws, :request, :start], %{system_time: _}, %{attempt: 3}} 296 | assert_receive {[:ex_aws, :request, :stop], %{duration: _}, %{attempt: 3, result: :error}} 297 | assert_receive {[:ex_aws, :request, :start], %{system_time: _}, %{attempt: 4}} 298 | assert_receive {[:ex_aws, :request, :stop], %{duration: _}, %{attempt: 4, result: :ok}} 299 | end 300 | 301 | defp mock_provisioned_throughput_response(success_after_retries) do 302 | exception = 303 | "{\"__type\": \"ProvisionedThroughputExceededException\", \"message\": \"Rate exceeded for shard shardId-000000000005 in stream my_stream under account 1234567890.\"}" 304 | 305 | success = 306 | "{\"SequenceNumber\":\"49592207023850419758877078054930583111417627497740632066\",\"ShardId\":\"shardId-000000000000\"}" 307 | 308 | ExAws.Request.HttpMock 309 | |> expect(:request, success_after_retries, fn _method, _url, _body, _headers, _opts -> 310 | {:ok, %{status_code: 400, body: exception}} 311 | end) 312 | |> expect(:request, fn _method, _url, _body, _headers, _opts -> 313 | {:ok, %{status_code: 200, body: success}} 314 | end) 315 | 316 | success 317 | end 318 | 319 | def mock_too_many_requests_exception(success_after_retries) do 320 | exception = "{\"__type\":\"TooManyRequestsException\",\"message\":\"Too many requests\"}" 321 | 322 | success = 323 | "{\"User\":{\"Attributes\":[{\"Name\":\"email\",\"Value\":\"user-email@test.com\"}],\"Enabled\":true,\"UserCreateDate\":1.743179259439E9,\"UserLastModifiedDate\":1.743179259439E9,\"UserStatus\":\"FORCE_CHANGE_PASSWORD\",\"Username\":\"f52064a4-3061-7030-581b-ae8392e97edd\"}}" 324 | 325 | ExAws.Request.HttpMock 326 | |> expect(:request, success_after_retries, fn _method, _url, _body, _headers, _opts -> 327 | {:ok, %{status_code: 400, body: exception}} 328 | end) 329 | |> expect(:request, fn _method, _url, _body, _headers, _opts -> 330 | {:ok, %{status_code: 200, body: success}} 331 | end) 332 | 333 | success 334 | end 335 | end 336 | -------------------------------------------------------------------------------- /test/ex_aws/utils_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ExAws.UtilsTest do 2 | use ExUnit.Case, async: true 3 | import ExAws.Utils 4 | 5 | test "camelize_keys works with maps" do 6 | assert %{"HelloWorld" => "foo"} == 7 | %{hello_world: "foo"} 8 | |> camelize_keys 9 | end 10 | 11 | test "camelize_keys does not go deep unless specified maps" do 12 | assert %{"FooBar" => %{foo_bar: "baz"}} == 13 | %{foo_bar: %{foo_bar: "baz"}} 14 | |> camelize_keys 15 | 16 | assert %{"FooBar" => %{"FooBar" => "baz"}} == 17 | %{foo_bar: %{foo_bar: "baz"}} 18 | |> camelize_keys(deep: true) 19 | end 20 | 21 | test "camelize_keys can handle keyword lists" do 22 | assert %{"FooBar" => %{"FooBar" => "baz"}} == 23 | [foo_bar: [foo_bar: "baz"]] 24 | |> camelize_keys(deep: true) 25 | end 26 | 27 | test "camelize_keys can handle handle lists that aren't keyword lists" do 28 | assert %{"FooBar" => ["foo", "bar"]} == 29 | [foo_bar: ["foo", "bar"]] 30 | |> camelize_keys(deep: true) 31 | end 32 | 33 | test "camelize_keys spec works for non-standard keys" do 34 | assert %{"non-standard" => ["foo", "bar"]} == 35 | [foo_bar: ["foo", "bar"]] 36 | |> camelize_keys(spec: %{foo_bar: "non-standard"}) 37 | end 38 | 39 | test "rename_keys renames keys in a list of keywords" do 40 | assert [d: 1, b: 2, e: 3] == [a: 1, b: 2, c: 3] |> rename_keys(a: :d, c: :e) 41 | end 42 | 43 | test "format (:xml) creates single key value pair" do 44 | assert [{"Key", 1}] == format([key: 1], type: :xml) 45 | end 46 | 47 | test "format (:xml) creates key value pairs from key_template and list" do 48 | assert [{"Key.1", 1}, {"Key.2", 2}] == format([key: [1, 2]], type: :xml) 49 | end 50 | 51 | test "format (:xml) spec works for non-standard keys" do 52 | assert [{"non-standard.1", 1}, {"non-standard.2", 2}] == 53 | format([foo_bar: [1, 2]], spec: %{foo_bar: "non-standard"}, type: :xml) 54 | end 55 | 56 | test "format (:xml) creates key value pairs from list of key_templates" do 57 | expected_return = [ 58 | {"Tag.1.Key", "keyA"}, 59 | {"Tag.1.Value", "ValueA"}, 60 | {"Tag.2.Key", "keyB"}, 61 | {"Tag.2.Value", "keyB"}, 62 | {"Member", "member!"} 63 | ] 64 | 65 | assert expected_return == 66 | format( 67 | [ 68 | tag: [ 69 | [key: "keyA", value: "ValueA"], 70 | [key: "keyB", value: "keyB"] 71 | ], 72 | member: "member!" 73 | ], 74 | type: :xml 75 | ) 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /test/telemetry_helper.exs: -------------------------------------------------------------------------------- 1 | defmodule TelemetryHelper do 2 | alias ExUnit.Callbacks 3 | 4 | def attach_telemetry(prefix) do 5 | name = "ex_aws_test" 6 | parent_pid = self() 7 | 8 | :ok = 9 | :telemetry.attach_many( 10 | name, 11 | [ 12 | prefix ++ [:start], 13 | prefix ++ [:stop], 14 | prefix ++ [:exception] 15 | ], 16 | fn path, args, metadata, _config -> 17 | send(parent_pid, {path, args, metadata}) 18 | end, 19 | nil 20 | ) 21 | 22 | Callbacks.on_exit(fn -> 23 | :telemetry.detach(name) 24 | end) 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | Code.require_file("default_helper.exs", __DIR__) 2 | Code.require_file("alternate_helper.exs", __DIR__) 3 | Code.require_file("telemetry_helper.exs", __DIR__) 4 | 5 | Application.ensure_all_started(:hackney) 6 | Application.ensure_all_started(:jsx) 7 | Application.ensure_all_started(:bypass) 8 | 9 | Mox.defmock(ExAws.Request.HttpMock, for: ExAws.Request.HttpClient) 10 | Mox.defmock(ExAws.Credentials.InitMock, for: ExAws.CredentialsIni.Provider) 11 | 12 | ExUnit.start() 13 | --------------------------------------------------------------------------------