├── .circleci └── config.yml ├── .github └── workflows │ └── scala-steward.yml ├── .gitignore ├── .mergify.yml ├── .scala-steward.conf ├── .scalafmt.conf ├── CODEOWNERS ├── LICENSE ├── README.md ├── build.sbt ├── build └── tag.sh ├── modules ├── auth │ └── src │ │ ├── it │ │ └── scala │ │ │ └── auth │ │ │ └── AwsSignerItSpec.scala │ │ ├── main │ │ └── scala │ │ │ └── auth │ │ │ └── AwsSigner.scala │ │ └── test │ │ ├── resources │ │ └── aws-signv4-test-suite │ │ │ ├── get-header-key-duplicate.authz │ │ │ ├── get-header-key-duplicate.req │ │ │ ├── get-header-value-multiline.authz │ │ │ ├── get-header-value-multiline.req │ │ │ ├── get-header-value-order.authz │ │ │ ├── get-header-value-order.req │ │ │ ├── get-header-value-trim.authz │ │ │ ├── get-header-value-trim.req │ │ │ ├── get-relative-relative.authz │ │ │ ├── get-relative-relative.req │ │ │ ├── get-relative.authz │ │ │ ├── get-relative.req │ │ │ ├── get-slash-dot-slash.authz │ │ │ ├── get-slash-dot-slash.req │ │ │ ├── get-slash-pointless-dot.authz │ │ │ ├── get-slash-pointless-dot.req │ │ │ ├── get-slash.authz │ │ │ ├── get-slash.req │ │ │ ├── get-slashes.authz │ │ │ ├── get-slashes.req │ │ │ ├── get-space.authz │ │ │ ├── get-space.req │ │ │ ├── get-unreserved.authz │ │ │ ├── get-unreserved.req │ │ │ ├── get-utf8.authz │ │ │ ├── get-utf8.req │ │ │ ├── get-vanilla-empty-query-key.authz │ │ │ ├── get-vanilla-empty-query-key.req │ │ │ ├── get-vanilla-query-order-key-case.authz │ │ │ ├── get-vanilla-query-order-key-case.req │ │ │ ├── get-vanilla-query-order-key.authz │ │ │ ├── get-vanilla-query-order-key.req │ │ │ ├── get-vanilla-query-order-value.authz │ │ │ ├── get-vanilla-query-order-value.req │ │ │ ├── get-vanilla-query-unreserved.authz │ │ │ ├── get-vanilla-query-unreserved.req │ │ │ ├── get-vanilla-query.authz │ │ │ ├── get-vanilla-query.req │ │ │ ├── get-vanilla-utf8-query.authz │ │ │ ├── get-vanilla-utf8-query.req │ │ │ ├── get-vanilla.authz │ │ │ ├── get-vanilla.req │ │ │ ├── post-header-key-case.authz │ │ │ ├── post-header-key-case.req │ │ │ ├── post-header-key-sort.authz │ │ │ ├── post-header-key-sort.req │ │ │ ├── post-header-value-case.authz │ │ │ ├── post-header-value-case.req │ │ │ ├── post-sts-header-after.authz │ │ │ ├── post-sts-header-after.req │ │ │ ├── post-sts-header-before.authz │ │ │ ├── post-sts-header-before.req │ │ │ ├── post-vanilla-empty-query-value.authz │ │ │ ├── post-vanilla-empty-query-value.req │ │ │ ├── post-vanilla-query.authz │ │ │ ├── post-vanilla-query.req │ │ │ ├── post-vanilla.authz │ │ │ ├── post-vanilla.req │ │ │ ├── post-x-www-form-urlencoded-parameters.authz │ │ │ ├── post-x-www-form-urlencoded-parameters.req │ │ │ ├── post-x-www-form-urlencoded.authz │ │ │ └── post-x-www-form-urlencoded.req │ │ └── scala │ │ └── auth │ │ ├── AwsSignV4TestSuiteSpec.scala │ │ └── AwsSignerSpec.scala ├── common │ └── src │ │ ├── it │ │ ├── resources │ │ │ └── log4j2-it.xml │ │ └── scala │ │ │ └── common │ │ │ └── IntegrationSpec.scala │ │ ├── main │ │ └── scala │ │ │ └── common │ │ │ ├── CredentialsProvider.scala │ │ │ ├── HttpCodecs.scala │ │ │ ├── headers.scala │ │ │ ├── mediaTypes.scala │ │ │ └── model.scala │ │ └── test │ │ ├── resources │ │ └── log4j2-test.xml │ │ └── scala │ │ └── common │ │ └── UnitSpec.scala └── s3 │ └── src │ ├── it │ ├── resources │ │ └── more.pdf │ └── scala │ │ └── s3 │ │ └── S3Spec.scala │ ├── main │ └── scala │ │ └── s3 │ │ ├── S3.scala │ │ ├── headers.scala │ │ ├── model.scala │ │ └── utils │ │ ├── S3UriDomain.scala │ │ └── S3UriParser.scala │ └── test │ └── scala │ └── s3 │ └── utils │ ├── S3UriDomainSpec.scala │ └── S3UriParserSpec.scala └── project ├── build.properties └── plugins.sbt /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | orbs: 4 | snyk: snyk/snyk@1.1.2 5 | 6 | executors: 7 | docker: 8 | docker: 9 | - image: cimg/openjdk:11.0.17 10 | auth: 11 | username: $OVO_DOCKERHUB_USER 12 | password: $OVO_DOCKERHUB_PASSWORD 13 | environment: 14 | JAVA_OPTS: "-Xmx1g -XX:MaxMetaspaceSize=1g -XX:MetaspaceSize=1g -XX:+CMSClassUnloadingEnabled" 15 | DEFAULT_AWS_REGION: "eu-west-1" 16 | AWS_REGION: "eu-west-1" 17 | 18 | 19 | commands: 20 | generate_sbt_cache_key: 21 | description: "Generate sbt cache key" 22 | steps: 23 | - run: md5sum project/build.properties project/plugins.sbt build.sbt > .sbt_cache_key 24 | 25 | store_sbt_cache: 26 | description: "Store sbt cache" 27 | steps: 28 | - generate_sbt_cache_key 29 | - save_cache: 30 | key: dependencies-{{ checksum ".sbt_cache_key" }} 31 | paths: 32 | - /home/circleci/.ivy2 33 | - /home/circleci/.sbt 34 | - /home/circleci/.cache 35 | 36 | restore_sbt_cache: 37 | description: "Restore sbt cache" 38 | steps: 39 | - generate_sbt_cache_key 40 | - restore_cache: 41 | keys: 42 | - dependencies-{{ checksum ".sbt_cache_key" }} 43 | - dependencies 44 | 45 | add_github_host_ssh_key: 46 | description: "Add the github host SSH key" 47 | steps: 48 | - run: 49 | name: Adding the github host SSH key 50 | command: | 51 | echo 'Adding the github host SSH key...' 52 | mkdir -p -m 0700 ~/.ssh/ 53 | ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts 54 | configure_git_credentials: 55 | description: "Configure git credentials" 56 | steps: 57 | - run: 58 | name: Configure git credentials 59 | command: | 60 | git config user.name ovo-comms-circleci 61 | git config user.email "hello.comms@ovoenergy.com" 62 | 63 | jobs: 64 | job_required_for_mergify: 65 | docker: 66 | - image: "alpine:latest" 67 | steps: 68 | - run: 69 | command: exit 0 70 | 71 | snyk_test: 72 | executor: docker 73 | steps: 74 | - checkout 75 | - snyk/scan: 76 | project: '${CIRCLE_PROJECT_REPONAME}' 77 | severity-threshold: high 78 | fail-on-issues: false 79 | monitor-on-build: true 80 | organization: 'oep-comms' 81 | 82 | build: 83 | 84 | executor: docker 85 | 86 | steps: 87 | 88 | - checkout 89 | 90 | - run: docker login --username $OVO_DOCKERHUB_USER --password $OVO_DOCKERHUB_PASSWORD 91 | 92 | - restore_sbt_cache 93 | 94 | - run: sbt update 95 | 96 | - store_sbt_cache 97 | 98 | - run: sbt headerCheck 99 | - run: sbt scalafmtCheckAll 100 | # TODO Make sure scalafmtCheckAll checks it as well 101 | - run: sbt it:scalafmtCheck 102 | 103 | - run: sbt +test:compile 104 | 105 | - persist_to_workspace: 106 | root: . 107 | paths: 108 | - project/target 109 | - project/project/target 110 | - target 111 | - modules/auth/target 112 | - modules/common/target 113 | - modules/dynamodb/target 114 | - modules/s3/target 115 | 116 | s3_unit_test: 117 | executor: docker 118 | steps: 119 | - checkout 120 | - run: docker login --username $OVO_DOCKERHUB_USER --password $OVO_DOCKERHUB_PASSWORD 121 | - restore_sbt_cache 122 | - attach_workspace: 123 | at: . 124 | 125 | - run: sbt +s3/test 126 | 127 | - store_test_results: 128 | path: modules/s3/target/test-reports 129 | 130 | - store_artifacts: 131 | path: target/unit-test.log 132 | destination: logs/unit-test.log 133 | 134 | auth_unit_test: 135 | 136 | executor: docker 137 | 138 | steps: 139 | 140 | - checkout 141 | - run: docker login --username $OVO_DOCKERHUB_USER --password $OVO_DOCKERHUB_PASSWORD 142 | - restore_sbt_cache 143 | - attach_workspace: 144 | at: . 145 | 146 | - run: sbt +auth/test 147 | 148 | - store_test_results: 149 | path: modules/auth/target/test-reports 150 | 151 | - store_artifacts: 152 | path: target/unit-test.log 153 | destination: logs/unit-test.log 154 | 155 | common_unit_test: 156 | 157 | executor: docker 158 | 159 | steps: 160 | 161 | - checkout 162 | - run: docker login --username $OVO_DOCKERHUB_USER --password $OVO_DOCKERHUB_PASSWORD 163 | - restore_sbt_cache 164 | - attach_workspace: 165 | at: . 166 | 167 | - run: sbt +common/test 168 | 169 | - store_test_results: 170 | path: modules/common/target/test-reports 171 | 172 | - store_artifacts: 173 | path: target/unit-test.log 174 | destination: logs/unit-test.log 175 | 176 | common_integration_test: 177 | 178 | executor: docker 179 | 180 | steps: 181 | 182 | - checkout 183 | - run: docker login --username $OVO_DOCKERHUB_USER --password $OVO_DOCKERHUB_PASSWORD 184 | - restore_sbt_cache 185 | - attach_workspace: 186 | at: . 187 | 188 | - run: sbt +common/it:test 189 | 190 | - store_test_results: 191 | path: modules/common/target/test-reports 192 | 193 | - store_artifacts: 194 | path: target/it-test.log 195 | destination: logs/it-test.log 196 | 197 | auth_integration_test: 198 | 199 | executor: docker 200 | 201 | steps: 202 | 203 | - checkout 204 | - run: docker login --username $OVO_DOCKERHUB_USER --password $OVO_DOCKERHUB_PASSWORD 205 | - restore_sbt_cache 206 | - attach_workspace: 207 | at: . 208 | 209 | - run: sbt +auth/it:test 210 | 211 | - store_test_results: 212 | path: modules/auth/target/test-reports 213 | 214 | - store_artifacts: 215 | path: target/it-test.log 216 | destination: logs/it-test.log 217 | 218 | s3_integration_test: 219 | 220 | executor: docker 221 | 222 | steps: 223 | 224 | - checkout 225 | - run: docker login --username $OVO_DOCKERHUB_USER --password $OVO_DOCKERHUB_PASSWORD 226 | - restore_sbt_cache 227 | - attach_workspace: 228 | at: . 229 | 230 | - run: sbt +s3/it:test 231 | 232 | - store_test_results: 233 | path: modules/s3/target/test-reports 234 | 235 | - store_artifacts: 236 | path: target/it-test.log 237 | destination: logs/it-test.log 238 | 239 | tag: 240 | 241 | executor: docker 242 | 243 | steps: 244 | - checkout 245 | - run: docker login --username $OVO_DOCKERHUB_USER --password $OVO_DOCKERHUB_PASSWORD 246 | - attach_workspace: 247 | at: . 248 | - add_github_host_ssh_key 249 | - configure_git_credentials 250 | - run: build/tag.sh 251 | # Update the workspace to make sure the git tags are available to the 252 | # downstream jobs 253 | - persist_to_workspace: 254 | root: . 255 | paths: 256 | - .git 257 | 258 | release: 259 | 260 | executor: docker 261 | 262 | steps: 263 | - checkout 264 | - run: docker login --username $OVO_DOCKERHUB_USER --password $OVO_DOCKERHUB_PASSWORD 265 | - attach_workspace: 266 | at: . 267 | - restore_sbt_cache 268 | - run: 269 | command: sbt +publish 270 | no_output_timeout: 20m 271 | 272 | workflows: 273 | 274 | main: 275 | jobs: 276 | - snyk_test: 277 | context: 278 | - ovo-internal-public 279 | - comms-internal-build 280 | filters: 281 | branches: 282 | only: 283 | - master 284 | 285 | - build: 286 | context: 287 | - ovo-internal-public 288 | - comms-internal-build 289 | 290 | - auth_unit_test: 291 | context: 292 | - ovo-internal-public 293 | - comms-internal-build 294 | requires: 295 | - build 296 | 297 | - common_unit_test: 298 | context: 299 | - ovo-internal-public 300 | - comms-internal-build 301 | requires: 302 | - build 303 | 304 | - s3_unit_test: 305 | context: 306 | - ovo-internal-public 307 | - comms-internal-build 308 | requires: 309 | - build 310 | 311 | - auth_integration_test: 312 | context: 313 | - ovo-internal-public 314 | - comms-internal-build 315 | - comms-internal-prd 316 | requires: 317 | - build 318 | 319 | - common_integration_test: 320 | context: 321 | - ovo-internal-public 322 | - comms-internal-build 323 | requires: 324 | - build 325 | 326 | - s3_integration_test: 327 | context: 328 | - ovo-internal-public 329 | - comms-internal-build 330 | - comms-internal-prd 331 | requires: 332 | - build 333 | 334 | - job_required_for_mergify: 335 | requires: 336 | - auth_unit_test 337 | - common_unit_test 338 | - s3_unit_test 339 | - auth_integration_test 340 | - common_integration_test 341 | - s3_integration_test 342 | filters: 343 | branches: 344 | ignore: 345 | - master 346 | - deploy-to-uat 347 | 348 | - tag: 349 | context: ovo-internal-public 350 | requires: 351 | - build 352 | - auth_unit_test 353 | - common_unit_test 354 | - s3_unit_test 355 | - auth_integration_test 356 | - common_integration_test 357 | - s3_integration_test 358 | filters: 359 | branches: 360 | only: master 361 | 362 | - release: 363 | context: 364 | - ovo-internal-public 365 | - comms-internal-build 366 | requires: 367 | - tag 368 | 369 | 370 | -------------------------------------------------------------------------------- /.github/workflows/scala-steward.yml: -------------------------------------------------------------------------------- 1 | on: 2 | schedule: 3 | - cron: "0 4 * * 1,4" 4 | workflow_dispatch: 5 | 6 | name: Scala Steward 7 | jobs: 8 | scala-steward: 9 | runs-on: ubuntu-latest 10 | name: Launch Scala Steward 11 | steps: 12 | - name: Launch Scala Steward 13 | uses: scala-steward-org/scala-steward-action@v2 14 | with: 15 | branches: master 16 | author-name: kaluza-libraries 17 | author-email: 123489691+kaluza-libraries@users.noreply.github.com 18 | github-token: ${{ secrets.KALUZA_LIBRARIES_SCALA_STEWARD_TOKEN }} 19 | other-args: "--add-labels" 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Sbt 2 | target/ 3 | .history 4 | 5 | # Idea IDE related 6 | .idea/ 7 | .idea_modules/ 8 | *.iml 9 | .bsp 10 | 11 | # metals 12 | .metals/ 13 | .bloop/ 14 | project/metals.sbt 15 | 16 | # Ensime 17 | .ensime 18 | .ensime_cache/ 19 | 20 | # Mac OSX 21 | .DS_Store 22 | 23 | # Project specific 24 | test.log -------------------------------------------------------------------------------- /.mergify.yml: -------------------------------------------------------------------------------- 1 | queue_rules: 2 | - name: default 3 | merge_conditions: 4 | - and: &base_merge_conditions 5 | - 'check-success=ci/circleci: job_required_for_mergify' 6 | - author=kaluza-libraries 7 | pull_request_rules: 8 | - name: Merge minor version PRs 9 | conditions: 10 | - and: *base_merge_conditions 11 | - and: 12 | - body~=labels:.*semver-minor 13 | actions: 14 | review: 15 | type: APPROVE 16 | queue: 17 | name: default 18 | delete_head_branch: {} 19 | - name: Merge patch version PRs 20 | conditions: 21 | - and: *base_merge_conditions 22 | - and: 23 | - body~=labels:.*semver-patch 24 | actions: 25 | review: 26 | type: APPROVE 27 | queue: 28 | name: default 29 | delete_head_branch: {} 30 | -------------------------------------------------------------------------------- /.scala-steward.conf: -------------------------------------------------------------------------------- 1 | updates.ignore = [ 2 | { groupId = "com.sksamuel.avro4s" } 3 | { groupId = "org.scanamo", artifactId = "scanamo" } 4 | { groupId = "org.scalameta", artifactId = "scalafmt-core" } 5 | ] 6 | -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- 1 | version = 2.4.2 2 | project.git = true 3 | align = none 4 | maxColumn = 100 5 | newlines.afterCurlyLambda = preserve 6 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @ovotech/comms 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2016 OVO Energy 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Scala effectful AWS client 2 | NOTE: 3 | Although this repo is public, we are still treating it as a private repo in terms of governance: there are no stability guarantees, and the changes are entirely driven by the needs of our team. 4 | 5 | The medium term plan is that several actual open source projects will be spawn out of comms-aws into separate libraries. 6 | We don't plan to make this a catch-all bucket to wrap all of the AWS SDK, but to use it as a testbed to produce high-quality, purely functional libraries for some of the AWS services. 7 | 8 | The libraries that we plan to support in the near to medium future are: 9 | 10 | - A purely functional, non blocking, fs2-based DynamoDb client. [dynosaur](https://github.com/ovotech/dynosaur) 11 | - An interface for AWS signing (unclear for now if it will be native or wrap the Java client) 12 | 13 | The libraries that we might support, or might stay as semi-private in comms-aws are: 14 | - An S3 client 15 | 16 | There are no further plans at the moment. 17 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | import sbtrelease.ExtraReleaseCommands 2 | import sbtrelease.ReleaseStateTransformations._ 3 | import sbtrelease.tagsonly.TagsOnly._ 4 | 5 | lazy val fs2Version = "3.12.0" 6 | 7 | lazy val catsEffectVersion = "3.5.2" 8 | 9 | lazy val scalatestVersion = "3.2.19" 10 | 11 | lazy val awsSdkVersion = "2.31.62" 12 | 13 | lazy val scalacheckVersion = "1.18.1" 14 | 15 | lazy val scalatestScalacheckVersion = "3.1.1.1" 16 | 17 | lazy val slf4jVersion = "1.7.36" 18 | 19 | lazy val log4jVersion = "2.24.3" 20 | 21 | lazy val http4sVersion = "0.23.30" 22 | 23 | lazy val http4sBlazeClientVersion = "0.23.17" 24 | 25 | lazy val scalaXmlVersion = "2.4.0" 26 | 27 | lazy val circeVersion = "0.12.2" 28 | 29 | lazy val scodecBitsVersion = "1.1.12" 30 | 31 | lazy val commonCodecVersion = "1.18.0" 32 | 33 | lazy val IntegrationTest = config("it") extend Test 34 | 35 | lazy val noPublishSettings = Seq( 36 | publish := {}, 37 | publishLocal := {}, 38 | publishArtifact := false 39 | ) 40 | 41 | lazy val publicArtifactory = "Artifactory Realm" at "https://kaluza.jfrog.io/artifactory/maven" 42 | 43 | lazy val publishOptions = Seq( 44 | publishTo := Some(publicArtifactory), 45 | credentials += { 46 | for { 47 | usr <- sys.env.get("ARTIFACTORY_USER") 48 | password <- sys.env.get("ARTIFACTORY_PASS") 49 | } yield Credentials("Artifactory Realm", "kaluza.jfrog.io", usr, password) 50 | }.getOrElse(Credentials(Path.userHome / ".ivy2" / ".credentials")), 51 | releaseProcess := Seq[ReleaseStep]( 52 | checkSnapshotDependencies, 53 | releaseStepCommand(ExtraReleaseCommands.initialVcsChecksCommand), 54 | setVersionFromTags(releaseTagPrefix.value), 55 | runClean, 56 | tagRelease, 57 | publishArtifacts, 58 | pushTagsOnly 59 | ) 60 | ) 61 | 62 | lazy val root = (project in file(".")) 63 | .aggregate(auth, common, s3) 64 | .configs(IntegrationTest) 65 | .settings(publishOptions) 66 | .settings( 67 | name := "comms-aws", 68 | inThisBuild( 69 | List( 70 | organization := "com.ovoenergy.comms", 71 | organizationName := "OVO Energy", 72 | organizationHomepage := Some(url("https://ovoenergy.com")), 73 | // TODO Extract contributors from github 74 | developers := List( 75 | Developer( 76 | "filosganga", 77 | "Filippo De Luca", 78 | "filippo.deluca@ovoenergy.com", 79 | url("https://github.com/filosganga") 80 | ), 81 | Developer( 82 | "laurence-bird", 83 | "Laurence Bird", 84 | "laurence.bird@ovoenergy.com", 85 | url("https://github.com/laurence-bird") 86 | ), 87 | Developer( 88 | "SystemFw", 89 | "Fabio Labella", 90 | "fabio.labella@ovoenergy.com", 91 | url("https://github.com/SystemFw") 92 | ), 93 | Developer( 94 | "ZsoltBalvanyos", 95 | "Zsolt Balvanyos", 96 | "zsolt.balvanyos@ovoenergy.com", 97 | url("https://github.com/ZsoltBalvanyos") 98 | ) 99 | ), 100 | startYear := Some(2018), 101 | licenses := Seq("Apache-2.0" -> url("https://opensource.org/licenses/apache-2.0")), 102 | scmInfo := Some( 103 | ScmInfo( 104 | url("https://github.com/ovotech/comms-aws"), 105 | "scm:git:git@github.com:ovotech/comms-aws.git" 106 | ) 107 | ), 108 | scalaVersion := "2.13.16", 109 | crossScalaVersions += "2.12.20", 110 | resolvers ++= Seq( 111 | publicArtifactory 112 | ), 113 | libraryDependencies ++= Seq( 114 | "org.http4s" %% "http4s-core" % http4sVersion, 115 | "org.http4s" %% "http4s-client" % http4sVersion, 116 | "co.fs2" %% "fs2-core" % fs2Version, 117 | "co.fs2" %% "fs2-io" % fs2Version, 118 | "org.slf4j" % "slf4j-api" % slf4jVersion 119 | ), 120 | libraryDependencies ++= Seq( 121 | "org.scalatest" %% "scalatest" % scalatestVersion, 122 | "org.typelevel" %% "cats-effect-testing-scalatest" % "1.6.0", 123 | "org.scalacheck" %% "scalacheck" % scalacheckVersion, 124 | "org.scalatestplus" %% "scalacheck-1-14" % scalatestScalacheckVersion, 125 | "org.apache.logging.log4j" % "log4j-api" % log4jVersion, 126 | "org.apache.logging.log4j" % "log4j-slf4j-impl" % log4jVersion, 127 | "org.http4s" %% "http4s-blaze-client" % http4sBlazeClientVersion 128 | ).map(_ % s"$Test,$IntegrationTest"), 129 | scalafmtOnCompile := true 130 | ) 131 | ) 132 | ) 133 | 134 | lazy val common = (project in file("modules/common")) 135 | .enablePlugins(AutomateHeaderPlugin) 136 | .configs(IntegrationTest) 137 | .settings(publishOptions) 138 | .settings( 139 | name := "comms-aws-common", 140 | scalacOptions -= "-Xfatal-warnings" // enable all options from sbt-tpolecat except fatal warnings 141 | ) 142 | .settings(inConfig(IntegrationTest)(Defaults.itSettings)) 143 | .settings(automateHeaderSettings(IntegrationTest)) 144 | .settings( 145 | libraryDependencies ++= Seq( 146 | "software.amazon.awssdk" % "auth" % awsSdkVersion, 147 | "software.amazon.awssdk" % "sdk-core" % awsSdkVersion % Optional 148 | ) 149 | ) 150 | 151 | lazy val auth = (project in file("modules/auth")) 152 | .enablePlugins(AutomateHeaderPlugin) 153 | .dependsOn(common % s"$Compile->$Compile;$Test->$Test;$IntegrationTest->$IntegrationTest") 154 | .configs(IntegrationTest) 155 | .settings(publishOptions) 156 | .settings( 157 | name := "comms-aws-auth", 158 | scalacOptions -= "-Xfatal-warnings" // enable all options from sbt-tpolecat except fatal warnings 159 | ) 160 | .settings(inConfig(IntegrationTest)(Defaults.itSettings)) 161 | .settings(automateHeaderSettings(IntegrationTest)) 162 | .settings( 163 | libraryDependencies ++= Seq( 164 | "commons-codec" % "commons-codec" % commonCodecVersion, 165 | "software.amazon.awssdk" % "s3" % awsSdkVersion % s"$Test,$IntegrationTest" 166 | ) 167 | ) 168 | 169 | lazy val s3 = (project in file("modules/s3")) 170 | .enablePlugins(AutomateHeaderPlugin) 171 | .dependsOn(common % s"$Compile->$Compile;$Test->$Test;$IntegrationTest->$IntegrationTest", auth) 172 | .configs(IntegrationTest) 173 | .settings(publishOptions) 174 | .settings( 175 | name := "comms-aws-s3", 176 | scalacOptions -= "-Xfatal-warnings" // enable all options from sbt-tpolecat except fatal warnings 177 | ) 178 | .settings(inConfig(IntegrationTest)(Defaults.itSettings)) 179 | .settings(automateHeaderSettings(IntegrationTest)) 180 | .settings( 181 | libraryDependencies ++= Seq( 182 | "org.http4s" %% "http4s-scala-xml" % "0.23.14", 183 | "org.scala-lang.modules" %% "scala-xml" % scalaXmlVersion, 184 | "org.http4s" %% "http4s-blaze-client" % http4sBlazeClientVersion % Optional, 185 | "software.amazon.awssdk" % "s3" % awsSdkVersion % s"$Test,$IntegrationTest" 186 | ) 187 | ) 188 | -------------------------------------------------------------------------------- /build/tag.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # TODO make it smart such that it will not tag again if the commit is already tagged 4 | 5 | set -e 6 | 7 | echo 'Fetching tag from remote...' 8 | git tag -l | xargs git tag -d 9 | git fetch --tags 10 | 11 | #get highest tag number 12 | last_tag=`git describe --abbrev=0 --tags` 13 | current_version=${last_tag#'v'} 14 | 15 | echo "Current version ${current_version}" 16 | 17 | #replace . with space so can split into an array 18 | current_version_parts=(${current_version//./ }) 19 | 20 | #get number parts and increase last one by 1 21 | current_version_major=${current_version_parts[0]} 22 | current_version_minor=${current_version_parts[1]} 23 | current_version_build=${current_version_parts[2]} 24 | 25 | next_version_build=$((current_version_build+1)) 26 | next_version="$current_version_major.$current_version_minor.$next_version_build" 27 | next_tag="v${next_version}" 28 | 29 | echo "Tagging the current commit with ${next_tag}" 30 | 31 | git tag -a ${next_tag} -m "Release version "${next_version} 32 | 33 | echo "Pushing tag ${next_tag} to origin" 34 | git push origin ${next_tag} -------------------------------------------------------------------------------- /modules/auth/src/it/scala/auth/AwsSignerItSpec.scala: -------------------------------------------------------------------------------- 1 | package com.ovoenergy.comms.aws 2 | package auth 3 | 4 | import common._ 5 | import common.model._ 6 | import cats.effect.IO 7 | import cats.effect.testing.scalatest.AsyncIOSpec 8 | import org.http4s.client.Client 9 | import org.http4s.blaze.client.BlazeClientBuilder 10 | import org.http4s.client.dsl.Http4sClientDsl 11 | import org.http4s.Method._ 12 | import org.http4s.headers._ 13 | import org.http4s.{MediaType, Request, Status, Uri} 14 | import org.http4s.client.middleware.{RequestLogger, ResponseLogger} 15 | 16 | import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain 17 | 18 | class AwsSignerItSpec extends IntegrationSpec with Http4sClientDsl[IO] with AsyncIOSpec { 19 | 20 | 21 | // This is our UAT environment 22 | private val esEndpoint = "" 23 | 24 | "AwsSigner" should { 25 | "sign request valid for S3" in { 26 | withHttpClient { client => 27 | val awsSigner = AwsSigner( 28 | CredentialsProvider.default[IO], 29 | Region.`eu-west-1`, 30 | Service.S3 31 | ) 32 | 33 | val requestLogger: Client[IO] => Client[IO] = 34 | RequestLogger[IO](logHeaders = true, logBody = true, redactHeadersWhen = _ => false) 35 | val responseLogger: Client[IO] => Client[IO] = 36 | ResponseLogger[IO](logHeaders = true, logBody = true, redactHeadersWhen = _ => false) 37 | 38 | val signedClient: Client[IO] = awsSigner(requestLogger(responseLogger(client))) 39 | 40 | val req = Request[IO]( 41 | uri = Uri.unsafeFromString("https://s3-eu-west-1.amazonaws.com/ovo-comms-test/more.pdf") 42 | ) 43 | 44 | signedClient.status(req).map(_.isSuccess) 45 | }.asserting(_ shouldBe true) 46 | } 47 | 48 | "sign request valid for S3 with nested paths" in { 49 | withHttpClient { client => 50 | val awsSigner = AwsSigner( 51 | CredentialsProvider.default[IO], 52 | Region.`eu-west-1`, 53 | Service.S3 54 | ) 55 | 56 | val requestLogger: Client[IO] => Client[IO] = 57 | RequestLogger[IO](logHeaders = true, logBody = true, redactHeadersWhen = _ => false) 58 | val responseLogger: Client[IO] => Client[IO] = 59 | ResponseLogger[IO](logHeaders = true, logBody = true, redactHeadersWhen = _ => false) 60 | 61 | val signedClient: Client[IO] = awsSigner(requestLogger(responseLogger(client))) 62 | 63 | val req = Request[IO]( uri = Uri.unsafeFromString("https://s3-eu-west-1.amazonaws.com/ovo-comms-test/test/more.pdf") 64 | ) 65 | 66 | signedClient.status(req) 67 | }.asserting(_ should (not be Status.Unauthorized and not be Status.Forbidden)) 68 | } 69 | 70 | "sign request valid for ES GET" ignore { 71 | withHttpClient { client => 72 | val awsSigner = AwsSigner( 73 | CredentialsProvider.default[IO], 74 | Region.`eu-west-1`, 75 | Service("es") 76 | ) 77 | 78 | val requestLogger: Client[IO] => Client[IO] = 79 | RequestLogger[IO](logHeaders = true, logBody = true, redactHeadersWhen = _ => false) 80 | val responseLogger: Client[IO] => Client[IO] = 81 | ResponseLogger[IO](logHeaders = true, logBody = true, redactHeadersWhen = _ => false) 82 | 83 | val signedClient: Client[IO] = awsSigner(requestLogger(responseLogger(client))) 84 | 85 | for { 86 | status <- signedClient.status(GET(Uri.unsafeFromString(s"$esEndpoint/audit-2018-09/_doc/foo"))) 87 | } yield status 88 | }.asserting(_ should (not be Status.Unauthorized and not be Status.Forbidden)) 89 | } 90 | 91 | "sign request valid for ES POST" ignore { 92 | withHttpClient { client => 93 | val awsSigner = AwsSigner( 94 | CredentialsProvider.default[IO], 95 | Region.`eu-west-1`, 96 | Service("es") 97 | ) 98 | 99 | val requestLogger: Client[IO] => Client[IO] = 100 | RequestLogger[IO](logHeaders = true, logBody = true, redactHeadersWhen = _ => false) 101 | val responseLogger: Client[IO] => Client[IO] = 102 | ResponseLogger[IO](logHeaders = true, logBody = true, redactHeadersWhen = _ => false) 103 | 104 | val signedClient: Client[IO] = awsSigner(requestLogger(responseLogger(client))) 105 | 106 | val body = """ 107 | { 108 | "query" : { 109 | "term": { 110 | "_id": "6fc53f9d-99a0-4938-a486-31fb401c5bc4" 111 | } 112 | } 113 | } 114 | """ 115 | 116 | for { 117 | status <- signedClient.status(POST( 118 | body, 119 | Uri.unsafeFromString(s"$esEndpoint/audit-2018-09/_doc/_search"), 120 | `Content-Type`(MediaType.application.json) 121 | )) 122 | } yield status 123 | }.asserting(_ should (not be Status.Unauthorized and not be Status.Forbidden)) 124 | } 125 | 126 | "sign request valid for ES POST with multiple indexes" ignore { 127 | withHttpClient { client => 128 | val awsSigner = AwsSigner( 129 | CredentialsProvider.default[IO], 130 | Region.`eu-west-1`, 131 | Service("es") 132 | ) 133 | 134 | val requestLogger: Client[IO] => Client[IO] = 135 | RequestLogger[IO](logHeaders = true, logBody = true, redactHeadersWhen = _ => false) 136 | val responseLogger: Client[IO] => Client[IO] = 137 | ResponseLogger[IO](logHeaders = true, logBody = true, redactHeadersWhen = _ => false) 138 | 139 | val signedClient: Client[IO] = awsSigner(requestLogger(responseLogger(client))) 140 | 141 | val body = """ 142 | { 143 | "query" : { 144 | "term": { 145 | "_id": "6fc53f9d-99a0-4938-a486-31fb401c5bc4" 146 | } 147 | } 148 | } 149 | """ 150 | 151 | for { 152 | status <- signedClient.status(POST( 153 | body, 154 | Uri.unsafeFromString( 155 | s"$esEndpoint/audit-2018-09,audit-2018-10,audit-2018-11/_doc/_search?ignore_unavailable=true" 156 | ), 157 | `Content-Type`(MediaType.application.json) 158 | )) 159 | } yield status 160 | }.asserting(_ should (not be Status.Unauthorized and not be Status.Forbidden)) 161 | } 162 | 163 | "sign request valid for ES POST with query" ignore { 164 | withHttpClient { client => 165 | val awsSigner = AwsSigner( 166 | CredentialsProvider.default[IO], 167 | Region.`eu-west-1`, 168 | Service("es") 169 | ) 170 | 171 | val requestLogger: Client[IO] => Client[IO] = 172 | RequestLogger[IO](logHeaders = true, logBody = true, redactHeadersWhen = _ => false) 173 | val responseLogger: Client[IO] => Client[IO] = 174 | ResponseLogger[IO](logHeaders = true, logBody = true, redactHeadersWhen = _ => false) 175 | 176 | val signedClient: Client[IO] = awsSigner(requestLogger(responseLogger(client))) 177 | 178 | val body = """ 179 | { 180 | "query" : { 181 | "term": { 182 | "_id": "6fc53f9d-99a0-4938-a486-31fb401c5bc4" 183 | } 184 | } 185 | } 186 | """ 187 | 188 | for { 189 | status <- signedClient.status(POST( 190 | body, 191 | Uri.unsafeFromString( 192 | s"$esEndpoint/audit-2018-09/_doc/_search?ignore_unavailable=true" 193 | ), 194 | `Content-Type`(MediaType.application.json) 195 | )) 196 | } yield status 197 | }.asserting(_ should (not be Status.Unauthorized and not be Status.Forbidden)) 198 | } 199 | 200 | "sign request valid for ES POST with query and multiple parameters" ignore { 201 | withHttpClient { client => 202 | val awsSigner = AwsSigner( 203 | CredentialsProvider.default[IO], 204 | Region.`eu-west-1`, 205 | Service("es") 206 | ) 207 | 208 | val requestLogger: Client[IO] => Client[IO] = 209 | RequestLogger[IO](logHeaders = true, logBody = true, redactHeadersWhen = _ => false) 210 | val responseLogger: Client[IO] => Client[IO] = 211 | ResponseLogger[IO](logHeaders = true, logBody = true, redactHeadersWhen = _ => false) 212 | 213 | val signedClient: Client[IO] = awsSigner(requestLogger(responseLogger(client))) 214 | 215 | val body = """ 216 | { 217 | "query" : { 218 | "term": { 219 | "_id": "6fc53f9d-99a0-4938-a486-31fb401c5bc4" 220 | } 221 | } 222 | } 223 | """ 224 | 225 | for { 226 | status <- signedClient.status(POST( 227 | body, 228 | Uri.unsafeFromString( 229 | s"$esEndpoint/audit-2018-09/_doc/_search?ignore_unavailable=true&refresh=true" 230 | ), 231 | `Content-Type`(MediaType.application.json) 232 | )) 233 | } yield status 234 | }.asserting(_ should (not be Status.Unauthorized and not be Status.Forbidden)) 235 | } 236 | 237 | "sign request valid for ES POST with query and multiple parameters with comas and stars" ignore { 238 | withHttpClient { client => 239 | val awsSigner = AwsSigner( 240 | CredentialsProvider.default[IO], 241 | Region.`eu-west-1`, 242 | Service("es") 243 | ) 244 | 245 | val requestLogger: Client[IO] => Client[IO] = 246 | RequestLogger[IO](logHeaders = true, logBody = true, redactHeadersWhen = _ => false) 247 | val responseLogger: Client[IO] => Client[IO] = 248 | ResponseLogger[IO](logHeaders = true, logBody = true, redactHeadersWhen = _ => false) 249 | 250 | val signedClient: Client[IO] = awsSigner(requestLogger(responseLogger(client))) 251 | 252 | val body = """ 253 | { 254 | "query" : { 255 | "term": { 256 | "_id": "6fc53f9d-99a0-4938-a486-31fb401c5bc4" 257 | } 258 | } 259 | } 260 | """ 261 | 262 | for { 263 | status <- signedClient.status(POST( 264 | body, 265 | Uri.unsafeFromString( 266 | "/audit-2018-09/_doc/_search?ignore_unavailable=true&refresh=true&foo*=foo&bar,baz=baz" 267 | ), 268 | `Content-Type`(MediaType.application.json) 269 | )) 270 | } yield status 271 | }.asserting(_ should (not be Status.Unauthorized and not be Status.Forbidden)) 272 | } 273 | 274 | "sign request valid for ES POST with star in path" ignore { 275 | withHttpClient { client => 276 | val awsSigner = AwsSigner( 277 | CredentialsProvider.default[IO], 278 | Region.`eu-west-1`, 279 | Service("es") 280 | ) 281 | 282 | val requestLogger: Client[IO] => Client[IO] = 283 | RequestLogger[IO](logHeaders = true, logBody = true, redactHeadersWhen = _ => false) 284 | val responseLogger: Client[IO] => Client[IO] = 285 | ResponseLogger[IO](logHeaders = true, logBody = true, redactHeadersWhen = _ => false) 286 | 287 | val signedClient: Client[IO] = awsSigner(requestLogger(responseLogger(client))) 288 | 289 | val body = """ 290 | { 291 | "query" : { 292 | "term": { 293 | "_id": "6fc53f9d-99a0-4938-a486-31fb401c5bc4" 294 | } 295 | } 296 | } 297 | """ 298 | 299 | for { 300 | status <- signedClient.status(POST( 301 | body, 302 | Uri.unsafeFromString(s"$esEndpoint/audit-*/_doc/_search"), 303 | `Content-Type`(MediaType.application.json) 304 | )) 305 | } yield status 306 | }.asserting(_ should (not be Status.Unauthorized and not be Status.Forbidden)) 307 | } 308 | } 309 | 310 | def withHttpClient[A](f: Client[IO] => IO[A]): IO[A] = { 311 | BlazeClientBuilder[IO].resource 312 | .use(f) 313 | } 314 | 315 | } 316 | -------------------------------------------------------------------------------- /modules/auth/src/main/scala/auth/AwsSigner.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.comms.aws 18 | package auth 19 | 20 | import AwsSigner._ 21 | import common._ 22 | import common.model._ 23 | import headers.{`X-Amz-Content-SHA256`, `X-Amz-Date`, `X-Amz-Security-Token`} 24 | import headers.`X-Amz-Date`._ 25 | import cats.effect.{Resource, Sync} 26 | import cats.implicits._ 27 | 28 | import scala.util.matching.Regex 29 | import java.net.URLEncoder 30 | import java.nio.charset.StandardCharsets 31 | import java.security.MessageDigest 32 | import java.time._ 33 | import java.time.format.DateTimeFormatter 34 | import java.time.temporal.ChronoUnit 35 | import org.slf4j.LoggerFactory 36 | 37 | import javax.crypto.Mac 38 | import javax.crypto.spec.SecretKeySpec 39 | import fs2.hash._ 40 | import org.http4s.{HttpDate, Request, Response, Uri} 41 | import org.http4s.Header.Raw 42 | import org.http4s.client.Client 43 | import org.http4s.headers.{Date, Host} 44 | import org.apache.commons.codec.binary.Hex 45 | import org.http4s.Header.Select.singleHeaders 46 | import org.typelevel.ci._ 47 | 48 | object AwsSigner { 49 | 50 | private val logger = LoggerFactory.getLogger(getClass) 51 | 52 | val dateFormatter: DateTimeFormatter = 53 | DateTimeFormatter.ofPattern("yyyyMMdd") 54 | 55 | val dateTimeFormatter: DateTimeFormatter = 56 | DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmssX") 57 | 58 | val DoubleSlashRegex: Regex = "/{2,}".r 59 | val MultipleSpaceRegex: Regex = "\\s+".r 60 | val EncodedSlashRegex: Regex = "%2F".r 61 | val StarRegex: Regex = """\*""".r 62 | 63 | def encodeHex(bytes: Array[Byte]): String = Hex.encodeHexString(bytes) 64 | 65 | def uriEncode(str: String): String = { 66 | StarRegex.replaceAllIn( 67 | URLEncoder.encode(str, StandardCharsets.UTF_8.name), 68 | "%2A" 69 | ) 70 | } 71 | 72 | def signWithKey( 73 | key: SecretKeySpec, 74 | bytes: Array[Byte], 75 | algorithm: String 76 | ): Array[Byte] = { 77 | val mac = Mac.getInstance(algorithm) 78 | mac.init(key) 79 | mac.doFinal(bytes) 80 | } 81 | 82 | def key( 83 | formattedDate: String, 84 | credentials: Credentials, 85 | region: Region, 86 | service: Service, 87 | algorithm: String 88 | ): SecretKeySpec = { 89 | 90 | def wrapSignature( 91 | signature: SecretKeySpec, 92 | bytes: Array[Byte] 93 | ): SecretKeySpec = 94 | new SecretKeySpec(signWithKey(signature, bytes, algorithm), algorithm) 95 | 96 | val rawKey = new SecretKeySpec( 97 | s"AWS4${credentials.secretAccessKey.value}".getBytes, 98 | algorithm 99 | ) 100 | 101 | val dateKey: SecretKeySpec = 102 | wrapSignature(rawKey, formattedDate.getBytes) 103 | 104 | val dateRegionKey: SecretKeySpec = 105 | wrapSignature(dateKey, region.value.getBytes) 106 | 107 | val dateRegionServiceKey: SecretKeySpec = 108 | wrapSignature(dateRegionKey, service.value.getBytes) 109 | 110 | wrapSignature(dateRegionServiceKey, "aws4_request".getBytes) 111 | } 112 | 113 | def hashBody[F[_]: Sync](request: Request[F]): F[String] = { 114 | request.body 115 | .through(sha256) 116 | .fold(Vector.empty[Byte])(_ :+ _) 117 | .map(xs => encodeHex(xs.toArray)) 118 | .compile 119 | .last 120 | .map(x => x.toRight[Throwable](new IllegalStateException)) 121 | .rethrow 122 | 123 | } 124 | 125 | def extractXAmzDateOrDate[F[_]](request: Request[F]): Option[Instant] = { 126 | request.headers 127 | .get[`X-Amz-Date`] 128 | .map(_.date) 129 | .orElse(request.headers.get[Date].map(_.date)) 130 | .map(_.toInstant) 131 | } 132 | 133 | def fixRequest[F[_]]( 134 | request: Request[F], 135 | credentials: Credentials, 136 | fallbackRequestDateTime: Instant 137 | )(implicit F: Sync[F]): F[Request[F]] = { 138 | 139 | val requestDateTime: Instant = 140 | extractXAmzDateOrDate(request).getOrElse(fallbackRequestDateTime) 141 | 142 | def addHostHeader(r: Request[F]): F[Request[F]] = 143 | if (r.headers.get[Host].isEmpty) { 144 | val uri = r.uri 145 | F.fromOption( 146 | uri.host, 147 | new IllegalArgumentException( 148 | "The request URI must be absolute or the request must have the Host header" 149 | ) 150 | ) 151 | .map(host => r.putHeaders(Host(host.value, r.uri.port))) 152 | } else { 153 | r.pure[F] 154 | } 155 | 156 | def addXAmzDateHeader(r: Request[F]): F[Request[F]] = 157 | (if (r.headers.get[Date].isEmpty && r.headers.get[`X-Amz-Date`].isEmpty) { 158 | r.putHeaders(`X-Amz-Date`(HttpDate.unsafeFromInstant(requestDateTime))) 159 | } else { 160 | r 161 | }).pure[F] 162 | 163 | def addXAmzSecurityTokenHeader(r: Request[F]): F[Request[F]] = 164 | credentials.sessionToken 165 | .fold(r)(sessionToken => r.putHeaders(`X-Amz-Security-Token`(sessionToken))) 166 | .pure[F] 167 | 168 | def addHashedBody(r: Request[F]): F[Request[F]] = 169 | if (r.headers.get[`X-Amz-Content-SHA256`].isEmpty) { 170 | 171 | // TODO Add chunking support for S3 172 | def unChunk(request: Request[F]): F[Request[F]] = 173 | if (r.isChunked) { 174 | val bodyAsBytes: F[List[Byte]] = request.body.compile.toList 175 | bodyAsBytes.map(bs => r.withBodyStream(fs2.Stream.emits(bs))) 176 | } else { 177 | r.pure[F] 178 | } 179 | 180 | for { 181 | unChunked <- unChunk(r) 182 | hashedBody <- hashBody(unChunked) 183 | } yield unChunked.putHeaders(`X-Amz-Content-SHA256`(hashedBody)) 184 | } else { 185 | r.pure[F] 186 | } 187 | 188 | for { 189 | requestWithHost <- addHostHeader(request) 190 | requestWithDate <- addXAmzDateHeader(requestWithHost) 191 | requestWithSessionToken <- addXAmzSecurityTokenHeader(requestWithDate) 192 | requestWithHashedBody <- addHashedBody(requestWithSessionToken) 193 | } yield requestWithHashedBody 194 | } 195 | 196 | def signRequest[F[_]: Sync]( 197 | request: Request[F], 198 | credentials: Credentials, 199 | region: Region, 200 | service: Service 201 | ): F[Request[F]] = { 202 | 203 | // FIXME Algorithm could be customized depending on the service 204 | val algorithm: String = "AWS4-HMAC-SHA256" 205 | val signingAlgorithm: String = "HmacSHA256" 206 | val digestAlgorithm: String = "SHA-256" 207 | 208 | val hashedPayloadF: F[String] = { 209 | val headerValue = request.headers 210 | .get[`X-Amz-Content-SHA256`] 211 | .map(_.hashedContent) 212 | 213 | headerValue.fold(hashBody(request))(_.pure[F]) 214 | } 215 | 216 | val requestDateTimeF = extractXAmzDateOrDate(request) 217 | .fold( 218 | Sync[F].raiseError[Instant]( 219 | new IllegalArgumentException( 220 | "The given request does not have Date or X-Amz-Date header" 221 | ) 222 | ) 223 | )(_.pure[F]) 224 | 225 | (hashedPayloadF, requestDateTimeF).mapN { (hashedPayload, requestDateTime) => 226 | val formattedDateTime = requestDateTime 227 | .atOffset(ZoneOffset.UTC) 228 | .format(AwsSigner.dateTimeFormatter) 229 | 230 | val formattedDate = 231 | requestDateTime 232 | .atOffset(ZoneOffset.UTC) 233 | .format(AwsSigner.dateFormatter) 234 | 235 | val scope = 236 | s"$formattedDate/${region.value}/${service.value}/aws4_request" 237 | 238 | val (canonicalHeaders, signedHeaders) = { 239 | 240 | val grouped = request.headers.headers.groupBy(_.name) 241 | val combined = grouped.mapValues( 242 | _.map(h => MultipleSpaceRegex.replaceAllIn(h.value, " ").trim) 243 | .mkString(",") 244 | ) 245 | 246 | val canonical = combined.toSeq 247 | .sortBy(_._1) 248 | .map { case (k, v) => s"${k.toString.toLowerCase}:$v\n" } 249 | .mkString("") 250 | 251 | val signed: String = 252 | request.headers.headers 253 | .map(_.name.toString.toLowerCase) 254 | .distinct 255 | .sorted 256 | .mkString(";") 257 | 258 | canonical -> signed 259 | } 260 | 261 | val canonicalRequest = { 262 | 263 | val method = request.method.name.toUpperCase 264 | 265 | val canonicalUri = { 266 | val absolutePath = { 267 | if (request.uri.path.startsWithString("/")) request.uri.path 268 | else Uri.Path.unsafeFromString("/").concat(request.uri.path) 269 | } 270 | 271 | // you do not normalize URI paths for requests to Amazon S3 272 | val normalizedPath = if (service != Service.S3) { 273 | DoubleSlashRegex.replaceAllIn(absolutePath.renderString, "/") 274 | } else { 275 | absolutePath.renderString 276 | } 277 | 278 | val handleEmptyPath = if (normalizedPath.isEmpty) "/" else normalizedPath 279 | 280 | val encodedOnceSegments = handleEmptyPath 281 | .split("/", -1) 282 | .map(uriEncode) 283 | 284 | // Normalize URI paths according to RFC 3986. Remove redundant and 285 | // relative path components. Each path segment must be URI-encoded 286 | // twice (except for Amazon S3 which only gets URI-encoded once). 287 | // 288 | // NOTE: This does not seem true at least not for ES 289 | // TODO: Test against dynamodb 290 | // 291 | val encodedTwiceSegments = if (service != Service.S3) { 292 | encodedOnceSegments 293 | } else { 294 | encodedOnceSegments 295 | } 296 | 297 | encodedTwiceSegments.mkString("/") 298 | } 299 | 300 | val canonicalQueryString: String = 301 | request.uri.query.toList 302 | .sortBy(_._1) 303 | .map { 304 | case (a, b) => s"${uriEncode(a)}=${uriEncode(b.getOrElse(""))}" 305 | } 306 | .mkString("&") 307 | 308 | val result = 309 | s"$method\n$canonicalUri\n$canonicalQueryString\n$canonicalHeaders\n$signedHeaders\n$hashedPayload" 310 | 311 | logger.debug(s"canonicalRequest: $result") 312 | 313 | result 314 | } 315 | 316 | val stringToSign = { 317 | val digest = MessageDigest.getInstance(digestAlgorithm) 318 | val hashedRequest = 319 | encodeHex(digest.digest(canonicalRequest.getBytes)) 320 | 321 | val result = s"$algorithm\n$formattedDateTime\n$scope\n$hashedRequest" 322 | 323 | logger.debug(s"stringToSign: $result") 324 | 325 | result 326 | } 327 | 328 | val signingKey: SecretKeySpec = 329 | key(formattedDate, credentials, region, service, signingAlgorithm) 330 | 331 | val signature: String = encodeHex( 332 | signWithKey(signingKey, stringToSign.getBytes, signingAlgorithm) 333 | ) 334 | 335 | val authorizationHeader = { 336 | 337 | val authorizationHeaderValue = 338 | s"$algorithm Credential=${credentials.accessKeyId.value}/$scope, SignedHeaders=$signedHeaders, Signature=$signature" 339 | 340 | Raw(ci"Authorization", authorizationHeaderValue) 341 | } 342 | 343 | request.putHeaders(authorizationHeader) 344 | } 345 | } 346 | 347 | def apply[F[_]]( 348 | credentialsProvider: CredentialsProvider[F], 349 | region: Region, 350 | service: Service 351 | ): AwsSigner[F] = 352 | new AwsSigner[F](credentialsProvider, region, service) 353 | 354 | } 355 | 356 | class AwsSigner[F[_]]( 357 | credentialsProvider: CredentialsProvider[F], 358 | region: Region, 359 | service: Service 360 | ) { 361 | 362 | def apply(client: Client[F])(implicit F: Sync[F]): Client[F] = { 363 | 364 | val sign: Request[F] => Resource[F, Response[F]] = { request => 365 | for { 366 | credentials <- Resource.eval(credentialsProvider.get) 367 | now <- Resource.eval( 368 | Sync[F].delay( 369 | Instant.now(Clock.systemUTC()).truncatedTo(ChronoUnit.SECONDS) 370 | ) 371 | ) 372 | fixed <- Resource.eval(fixRequest(request, credentials, now)) 373 | signed <- Resource.eval( 374 | signRequest(fixed, credentials, region, service) 375 | ) 376 | result <- client.run(signed) 377 | } yield result 378 | } 379 | 380 | Client(sign) 381 | } 382 | 383 | } 384 | -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-header-key-duplicate.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=c9d5ea9f3f72853aea855b47ea873832890dbdd183b4468f858259531a5138ea -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-header-key-duplicate.req: -------------------------------------------------------------------------------- 1 | GET / HTTP/1.1 2 | Host:example.amazonaws.com 3 | My-Header1:value2 4 | My-Header1:value2 5 | My-Header1:value1 6 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-header-value-multiline.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=ba17b383a53190154eb5fa66a1b836cc297cc0a3d70a5d00705980573d8ff790 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-header-value-multiline.req: -------------------------------------------------------------------------------- 1 | GET / HTTP/1.1 2 | Host:example.amazonaws.com 3 | My-Header1:value1 4 | value2 5 | value3 6 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-header-value-order.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=08c7e5a9acfcfeb3ab6b2185e75ce8b1deb5e634ec47601a50643f830c755c01 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-header-value-order.req: -------------------------------------------------------------------------------- 1 | GET / HTTP/1.1 2 | Host:example.amazonaws.com 3 | My-Header1:value4 4 | My-Header1:value1 5 | My-Header1:value3 6 | My-Header1:value2 7 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-header-value-trim.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;my-header2;x-amz-date, Signature=acc3ed3afb60bb290fc8d2dd0098b9911fcaa05412b367055dee359757a9c736 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-header-value-trim.req: -------------------------------------------------------------------------------- 1 | GET / HTTP/1.1 2 | Host:example.amazonaws.com 3 | My-Header1: value1 4 | My-Header2: "a b c" 5 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-relative-relative.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-relative-relative.req: -------------------------------------------------------------------------------- 1 | GET /example1/example2/../.. HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-relative.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-relative.req: -------------------------------------------------------------------------------- 1 | GET /example/.. HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-slash-dot-slash.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-slash-dot-slash.req: -------------------------------------------------------------------------------- 1 | GET /./ HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-slash-pointless-dot.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=ef75d96142cf21edca26f06005da7988e4f8dc83a165a80865db7089db637ec5 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-slash-pointless-dot.req: -------------------------------------------------------------------------------- 1 | GET /./example HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-slash.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-slash.req: -------------------------------------------------------------------------------- 1 | GET // HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-slashes.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=9a624bd73a37c9a373b5312afbebe7a714a789de108f0bdfe846570885f57e84 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-slashes.req: -------------------------------------------------------------------------------- 1 | GET //example// HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-space.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=652487583200325589f1fba4c7e578f72c47cb61beeca81406b39ddec1366741 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-space.req: -------------------------------------------------------------------------------- 1 | GET /example space/ HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-unreserved.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=07ef7494c76fa4850883e2b006601f940f8a34d404d0cfa977f52a65bbf5f24f -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-unreserved.req: -------------------------------------------------------------------------------- 1 | GET /-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-utf8.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=8318018e0b0f223aa2bbf98705b62bb787dc9c0e678f255a891fd03141be5d85 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-utf8.req: -------------------------------------------------------------------------------- 1 | GET /ሴ HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-vanilla-empty-query-key.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=a67d582fa61cc504c4bae71f336f98b97f1ea3c7a6bfe1b6e45aec72011b9aeb -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-vanilla-empty-query-key.req: -------------------------------------------------------------------------------- 1 | GET /?Param1=value1 HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-vanilla-query-order-key-case.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=b97d918cfa904a5beff61c982a1b6f458b799221646efd99d3219ec94cdf2500 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-vanilla-query-order-key-case.req: -------------------------------------------------------------------------------- 1 | GET /?Param2=value2&Param1=value1 HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-vanilla-query-order-key.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=eedbc4e291e521cf13422ffca22be7d2eb8146eecf653089df300a15b2382bd1 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-vanilla-query-order-key.req: -------------------------------------------------------------------------------- 1 | GET /?Param1=value2&Param1=Value1 HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-vanilla-query-order-value.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5772eed61e12b33fae39ee5e7012498b51d56abc0abb7c60486157bd471c4694 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-vanilla-query-order-value.req: -------------------------------------------------------------------------------- 1 | GET /?Param1=value2&Param1=value1 HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-vanilla-query-unreserved.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=9c3e54bfcdf0b19771a7f523ee5669cdf59bc7cc0884027167c21bb143a40197 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-vanilla-query-unreserved.req: -------------------------------------------------------------------------------- 1 | GET /?-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz=-._~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-vanilla-query.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-vanilla-query.req: -------------------------------------------------------------------------------- 1 | GET / HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-vanilla-utf8-query.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=2cdec8eed098649ff3a119c94853b13c643bcf08f8b0a1d91e12c9027818dd04 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-vanilla-utf8-query.req: -------------------------------------------------------------------------------- 1 | GET /?ሴ=bar HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-vanilla.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/get-vanilla.req: -------------------------------------------------------------------------------- 1 | GET / HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-header-key-case.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-header-key-case.req: -------------------------------------------------------------------------------- 1 | POST / HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-header-key-sort.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=c5410059b04c1ee005303aed430f6e6645f61f4dc9e1461ec8f8916fdf18852c -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-header-key-sort.req: -------------------------------------------------------------------------------- 1 | POST / HTTP/1.1 2 | Host:example.amazonaws.com 3 | My-Header1:value1 4 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-header-value-case.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;my-header1;x-amz-date, Signature=cdbc9802e29d2942e5e10b5bccfdd67c5f22c7c4e8ae67b53629efa58b974b7d -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-header-value-case.req: -------------------------------------------------------------------------------- 1 | POST / HTTP/1.1 2 | Host:example.amazonaws.com 3 | My-Header1:VALUE1 4 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-sts-header-after.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-sts-header-after.req: -------------------------------------------------------------------------------- 1 | POST / HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-sts-header-before.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=85d96828115b5dc0cfc3bd16ad9e210dd772bbebba041836c64533a82be05ead -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-sts-header-before.req: -------------------------------------------------------------------------------- 1 | POST / HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z 4 | X-Amz-Security-Token:AQoDYXdzEPT//////////wEXAMPLEtc764bNrC9SAPBSM22wDOk4x4HIZ8j4FZTwdQWLWsKWHGBuFqwAeMicRXmxfpSPfIeoIYRqTflfKD8YUuwthAx7mSEI/qkPpKPi/kMcGdQrmGdeehM4IC1NtBmUpp2wUE8phUZampKsburEDy0KPkyQDYwT7WZ0wq5VSXDvp75YU9HFvlRd8Tx6q6fE8YQcHNVXAkiY9q6d+xo0rKwT38xVqr7ZD0u0iPPkUL64lIZbqBAz+scqKmlzm8FDrypNC9Yjc8fPOLn9FX9KSYvKTr4rvx3iSIlTJabIQwj2ICCR/oLxBA== -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-vanilla-empty-query-value.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=28038455d6de14eafc1f9222cf5aa6f1a96197d7deb8263271d420d138af7f11 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-vanilla-empty-query-value.req: -------------------------------------------------------------------------------- 1 | POST /?Param1=value1 HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-vanilla-query.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=28038455d6de14eafc1f9222cf5aa6f1a96197d7deb8263271d420d138af7f11 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-vanilla-query.req: -------------------------------------------------------------------------------- 1 | POST /?Param1=value1 HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-vanilla.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-vanilla.req: -------------------------------------------------------------------------------- 1 | POST / HTTP/1.1 2 | Host:example.amazonaws.com 3 | X-Amz-Date:20150830T123600Z -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-x-www-form-urlencoded-parameters.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date, Signature=1a72ec8f64bd914b0e42e42607c7fbce7fb2c7465f63e3092b3b0d39fa77a6fe -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-x-www-form-urlencoded-parameters.req: -------------------------------------------------------------------------------- 1 | POST / HTTP/1.1 2 | Content-Type:application/x-www-form-urlencoded; charset=utf8 3 | Host:example.amazonaws.com 4 | X-Amz-Date:20150830T123600Z 5 | Content-Length:13 6 | 7 | Param1=value1 -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-x-www-form-urlencoded.authz: -------------------------------------------------------------------------------- 1 | AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=ff11897932ad3f4e8b18135d722051e5ac45fc38421b1da7b9d196a0fe09473a -------------------------------------------------------------------------------- /modules/auth/src/test/resources/aws-signv4-test-suite/post-x-www-form-urlencoded.req: -------------------------------------------------------------------------------- 1 | POST / HTTP/1.1 2 | Content-Type:application/x-www-form-urlencoded 3 | Host:example.amazonaws.com 4 | X-Amz-Date:20150830T123600Z 5 | Content-Length:13 6 | 7 | Param1=value1 -------------------------------------------------------------------------------- /modules/auth/src/test/scala/auth/AwsSignV4TestSuiteSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.comms.aws 18 | package auth 19 | 20 | import common._ 21 | import headers._ 22 | import model._ 23 | import Credentials._ 24 | import cats.effect.testing.scalatest.AsyncIOSpec 25 | import cats.effect.{IO, Sync} 26 | import org.http4s.client.dsl.Http4sClientDsl 27 | import org.http4s.Method._ 28 | import org.http4s.headers.Authorization 29 | import org.http4s.{HttpDate, Request, Uri} 30 | import org.http4s.syntax.all._ 31 | import org.typelevel.ci.CIStringSyntax 32 | 33 | /* 34 | Validate AwsSigner with test cases from 'AWS Signature Version 4 Test Suite' 35 | https://docs.aws.amazon.com/general/latest/gr/signature-v4-test-suite.html 36 | Download the test cases zip file, extract in the project root dir and run tests. 37 | */ 38 | 39 | class AwsSignV4TestSuiteSpec extends UnitSpec with Http4sClientDsl[IO] with AsyncIOSpec { 40 | import cats.data.EitherT 41 | 42 | val DateFormatter = 43 | java.time.format.DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmssVV") 44 | def parseTestCaseDate(s: String) = 45 | java.time.ZonedDateTime.parse(s, DateFormatter) 46 | 47 | val TestCaseFileBaseDir = 48 | "modules/auth/src/test/resources/aws-signv4-test-suite" 49 | val AuthorizationFileSuffix = ".authz" 50 | val RequestFileSuffix = ".req" 51 | 52 | val AwsRegion = Region.`us-east-1` 53 | val AwsService = Service.TestService 54 | val AwsCredentials = Credentials( 55 | AccessKeyId("AKIDEXAMPLE"), 56 | SecretAccessKey("wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY") 57 | ) 58 | 59 | val expectedFailures = Set( 60 | "get-space", 61 | "post-x-www-form-urlencoded", 62 | "get-unreserved", 63 | "get-header-value-multiline", 64 | "get-relative-relative", 65 | "get-slash-pointless-dot", 66 | "get-slash-dot-slash", 67 | "get-vanilla-utf8-query", 68 | "get-utf8", 69 | "get-vanilla-query-order-value", 70 | "post-x-www-form-urlencoded-parameters", 71 | "get-slashes", 72 | "get-relative", 73 | "get-vanilla-query-unreserved", 74 | "get-vanilla-query-order-key" 75 | ) 76 | 77 | "AwsSigner" should { 78 | getTestCaseFileBaseNames(TestCaseFileBaseDir) foreach { testFile => 79 | s"pass test case '${testFile.getName}'" in { 80 | (for { 81 | testCase <- EitherT(getTestCase[IO](testFile.getAbsolutePath)) 82 | (request, expectedSignature) = testCase 83 | res <- EitherT.right[String](withSignRequest(IO(request)) { signed => 84 | val signature = signed.headers.get("Authorization".ci).get.map(_.value) 85 | IO(signature.toList shouldBe List(expectedSignature)) 86 | }) 87 | } yield res).value.map { 88 | case Left(msg) => fail(msg) 89 | case Right(a) => a 90 | } 91 | } 92 | } 93 | } 94 | 95 | def getTestCaseFileBaseNames(baseDir: String) = { 96 | import java.io.File 97 | 98 | def getRecursiveListOfFiles(dir: File): Array[File] = { 99 | val these = dir.listFiles 100 | these ++ these.filter(_.isDirectory).flatMap(getRecursiveListOfFiles) 101 | } 102 | 103 | getRecursiveListOfFiles(new File(baseDir)).toList 104 | .filter(f => f.getName.endsWith(AuthorizationFileSuffix)) 105 | .map(f => new File(f.getPath.replace(AuthorizationFileSuffix, ""))) 106 | .filter(f => !expectedFailures.contains(f.getName)) 107 | } 108 | 109 | def getTestCase[F[_]](baseFileName: String)(implicit F: Sync[F]) = { 110 | import java.io._ 111 | import cats.implicits._ 112 | import cats.data.EitherT 113 | import cats.effect.Resource 114 | import org.http4s.{Header, Headers} 115 | 116 | def source(fn: String) = 117 | Resource.fromAutoCloseable(F.delay { 118 | scala.io.Source.fromFile(new File(fn), "UTF-8") 119 | }) 120 | def parseRequest(requestText: String): F[Either[String, Request[F]]] = { 121 | val RequestRe = "(?s)(.+?)(\n\n(.+))?".r 122 | val RequestLineRe = "(GET|POST) (.+) HTTP/1.1".r 123 | def headers(rows: List[String]) = 124 | rows.map(_.split(":", 2).toList).collect { 125 | case "X-Amz-Date" :: v :: Nil => 126 | Header.ToRaw.modelledHeadersToRaw( 127 | `X-Amz-Date`(HttpDate.unsafeFromZonedDateTime(parseTestCaseDate(v))) 128 | ) 129 | case k :: v :: Nil => Header.ToRaw.rawToRaw(Header.Raw(k.ci, v)) 130 | } 131 | (requestText match { 132 | case RequestRe(requestSection, _, body) => 133 | requestSection.split("\n").toList match { 134 | case RequestLineRe(methodText, uri) :: headersRows => 135 | val method = methodText match { 136 | case "POST" => POST 137 | case "GET" => GET 138 | } 139 | val request = Request[F]( 140 | method = method, 141 | uri = Uri.unsafeFromString(uri), 142 | headers = Headers(headers(headersRows)) 143 | ) 144 | Right( 145 | Option(body).map(b => request.withEntity(b)).getOrElse(request) 146 | ) 147 | case l => Left(s"unexpected request line syntax: $l") 148 | } 149 | case l => Left(s"unexpected request syntax: '$l'") 150 | }).pure[F] 151 | } 152 | 153 | (for { 154 | request <- EitherT( 155 | source(s"${baseFileName}${RequestFileSuffix}").use(r => parseRequest(r.mkString)) 156 | ) 157 | signature <- EitherT.right[String]( 158 | source(s"${baseFileName}${AuthorizationFileSuffix}") 159 | .use(_.mkString.pure[F]) 160 | ) 161 | } yield request -> signature).value 162 | } 163 | 164 | def withSignRequest[A]( 165 | req: IO[Request[IO]], 166 | region: Region = AwsRegion, 167 | service: Service = AwsService, 168 | credentials: Credentials = AwsCredentials 169 | )(f: Request[IO] => IO[A]): IO[A] = { 170 | for { 171 | request <- req 172 | signedRequest <- AwsSigner.signRequest( 173 | request, 174 | credentials, 175 | region, 176 | service 177 | ) 178 | result <- f(signedRequest) 179 | } yield result 180 | } 181 | 182 | } 183 | -------------------------------------------------------------------------------- /modules/auth/src/test/scala/auth/AwsSignerSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.comms.aws 18 | package auth 19 | 20 | import common._ 21 | import headers._ 22 | import model._ 23 | import Credentials._ 24 | 25 | import java.security.MessageDigest 26 | import java.time.{Instant, LocalDateTime, ZoneOffset} 27 | import java.time.temporal.ChronoUnit 28 | import cats.implicits._ 29 | import cats.effect.IO 30 | import fs2._ 31 | import fs2.hash._ 32 | import org.http4s.client.dsl.Http4sClientDsl 33 | import org.http4s.Method._ 34 | import org.http4s.headers._ 35 | import org.http4s.{Headers, HttpDate, MediaType, Request} 36 | import AwsSigner._ 37 | import cats.effect.testing.scalatest.AsyncIOSpec 38 | import org.http4s.Header.Select.singleHeaders 39 | import org.http4s.syntax.all._ 40 | import org.typelevel.ci.CIStringSyntax 41 | 42 | class AwsSignerSpec extends UnitSpec with Http4sClientDsl[IO] with AsyncIOSpec { 43 | 44 | "digest" should { 45 | "calculate the correct digest" in forAll() { data: Array[Byte] => 46 | val d = MessageDigest.getInstance("SHA-256").digest(data) 47 | val testValue = Stream(data: _*).through(sha256).toList 48 | testValue shouldBe d.toList 49 | } 50 | 51 | "calculate the correct digest for empty body" in { 52 | val md = MessageDigest.getInstance("SHA-256") 53 | val expectedResult = encodeHex(md.digest(Array.empty)) 54 | 55 | note(s"Expected empty body hash: $expectedResult") 56 | 57 | Stream.empty 58 | .covaryOutput[Byte] 59 | .through(sha256) 60 | .fold(Vector.empty[Byte])(_ :+ _) 61 | .map(xs => encodeHex(xs.toArray)) 62 | .toList 63 | .head shouldBe expectedResult 64 | 65 | } 66 | } 67 | 68 | "Request with no body" should { 69 | "not have empty body stream" in { 70 | (for { 71 | req <- IO(GET.apply(uri"https://example.com")) 72 | last <- req.body.compile.last 73 | } yield last).asserting(_ shouldBe None) 74 | } 75 | } 76 | 77 | "uriEncoder" should { 78 | "encode :" in { 79 | uriEncode("foo:bar") shouldBe "foo%3Abar" 80 | } 81 | 82 | "encode %2F" in { 83 | uriEncode("%2F") shouldBe "%252F" 84 | } 85 | } 86 | 87 | "AwsSigner.fixRequest" when { 88 | 89 | "Date header is not defined" when { 90 | "X-Amz-Date is not defined" should { 91 | "Add X-Amz-Date" in { 92 | 93 | val now = Instant.now() 94 | val expectedXAmzDate = `X-Amz-Date`(HttpDate.unsafeFromInstant(now)) 95 | 96 | withFixedRequest( 97 | IO(GET(uri"http://example.com")) 98 | .map(_.removeHeader[Date].removeHeader[`X-Amz-Date`]), 99 | now 100 | ) { r => 101 | IO { 102 | r.headers.get[Date] shouldBe None 103 | r.headers.get[`X-Amz-Date`] shouldBe Some(expectedXAmzDate) 104 | } 105 | } 106 | } 107 | } 108 | 109 | "X-Amz-Date is defined" should { 110 | "not add X-Amz-Date" in { 111 | val now = Instant.now() 112 | val expectedXAmzDate = `X-Amz-Date`( 113 | HttpDate.unsafeFromInstant(now.minus(5, ChronoUnit.MINUTES)) 114 | ) 115 | withFixedRequest( 116 | IO(Request[IO](uri = uri"http://example.com", headers = Headers(expectedXAmzDate))), 117 | now 118 | ) { r => 119 | IO { 120 | r.headers.get[Date] shouldBe None 121 | r.headers.get[`X-Amz-Date`] shouldBe Some(expectedXAmzDate) 122 | } 123 | } 124 | } 125 | } 126 | } 127 | 128 | "Date header is defined" should { 129 | "not add X-Amz-Date" in { 130 | val now = Instant.now() 131 | val expectedDate = 132 | Date(HttpDate.unsafeFromInstant(now.minus(5, ChronoUnit.MINUTES))) 133 | withFixedRequest( 134 | IO(GET(uri"http://example.com")) 135 | .map(_.removeHeader[`X-Amz-Date`].putHeaders(expectedDate)), 136 | now 137 | ) { r => 138 | IO { 139 | r.headers.get[`X-Amz-Date`] shouldBe None 140 | r.headers.get[Date] shouldBe Some(expectedDate) 141 | } 142 | } 143 | } 144 | } 145 | 146 | "Host header is defined" should { 147 | "the uri is absolute" should { 148 | "not add Host header" in { 149 | val expectedHost = Host("foo", 5555) 150 | withFixedRequest( 151 | IO(GET(uri"http://example.com")) 152 | .map(_.putHeaders(expectedHost)) 153 | ) { r => 154 | IO { 155 | r.headers.get[Host] shouldBe Some(expectedHost) 156 | } 157 | }.unsafeRunSync() 158 | } 159 | } 160 | 161 | "the uri is relative" should { 162 | "fail the effect" in { 163 | withFixedRequest(IO(GET(uri"/foo/bar")))(_ => IO.unit).attempt 164 | .asserting(_ shouldBe a[Left[_, _]]) 165 | } 166 | } 167 | } 168 | 169 | "Credentials contain the session token" should { 170 | "add the X-Amz-Security-Token header" in { 171 | 172 | val sessionToken = SessionToken("this-is-a-token") 173 | val credentials = Credentials( 174 | AccessKeyId("FOO"), 175 | SecretAccessKey("BAR"), 176 | sessionToken.some 177 | ) 178 | val expectedXAmzSecurityToken = `X-Amz-Security-Token`(sessionToken) 179 | withFixedRequest( 180 | IO(GET(uri"http://example.com")), 181 | credentials = credentials 182 | ) { r => 183 | IO { 184 | r.headers.get[`X-Amz-Security-Token`] shouldBe Some(expectedXAmzSecurityToken) 185 | } 186 | } 187 | } 188 | } 189 | 190 | "Credentials do not contain the session token" should { 191 | "not add the X-Amz-Security-Token header" in { 192 | 193 | val credentials = 194 | Credentials(AccessKeyId("FOO"), SecretAccessKey("BAR")) 195 | withFixedRequest( 196 | IO(GET(uri"http://example.com")), 197 | credentials = credentials 198 | ) { r => 199 | IO { 200 | r.headers.get[`X-Amz-Security-Token`] shouldBe None 201 | } 202 | } 203 | } 204 | } 205 | } 206 | 207 | "AwsSigner.signRequest" when { 208 | "Date header is not defined" when { 209 | "X-Amz-Date is not defined" should { 210 | "return a failed effect" in { 211 | withSignRequest(IO(GET(uri"http://example.com")))(_ => IO.unit).attempt 212 | .asserting(_ shouldBe a[Left[_, _]]) 213 | } 214 | } 215 | } 216 | } 217 | 218 | "AwsSigner.signRequest" should { 219 | "sign a vanilla GET request correctly" in { 220 | 221 | val expectedAuthorizationValue = 222 | "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5fa00fa31553b73ebf1942676e86291e8372ff2a2260956d9b8aae1d763fbf31" 223 | 224 | val credentials = Credentials( 225 | AccessKeyId("AKIDEXAMPLE"), 226 | SecretAccessKey("wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY") 227 | ) 228 | 229 | val dateTime = LocalDateTime 230 | .parse("20150830T123600Z", AwsSigner.dateTimeFormatter) 231 | .atZone(ZoneOffset.UTC) 232 | 233 | val request = Request[IO]( 234 | uri = uri"/", 235 | headers = Headers( 236 | Host("example.amazonaws.com"), 237 | `X-Amz-Date`(HttpDate.unsafeFromZonedDateTime(dateTime)) 238 | ) 239 | ) 240 | 241 | withSignRequest( 242 | IO(request), 243 | credentials = credentials, 244 | region = Region.`us-east-1`, 245 | service = Service("service") 246 | ) { r => 247 | IO( 248 | r.headers 249 | .get(ci"Authorization") 250 | .get 251 | .head 252 | .value 253 | ) 254 | }.asserting(_ shouldBe expectedAuthorizationValue) 255 | } 256 | 257 | "sign a vanilla POST request correctly" in { 258 | 259 | val expectedAuthorizationValue = 260 | "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=host;x-amz-date, Signature=5da7c1a2acd57cee7505fc6676e4e544621c30862966e37dddb68e92efbe5d6b" 261 | val credentials = Credentials( 262 | AccessKeyId("AKIDEXAMPLE"), 263 | SecretAccessKey("wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY") 264 | ) 265 | val dateTime = LocalDateTime 266 | .parse("20150830T123600Z", AwsSigner.dateTimeFormatter) 267 | .atZone(ZoneOffset.UTC) 268 | 269 | val request = Request[IO]( 270 | method = POST, 271 | uri = uri"/", 272 | headers = Headers( 273 | Host("example.amazonaws.com"), 274 | `X-Amz-Date`(HttpDate.unsafeFromZonedDateTime(dateTime)) 275 | ) 276 | ) 277 | 278 | withSignRequest( 279 | IO(request), 280 | credentials = credentials, 281 | region = Region.`us-east-1`, 282 | service = Service("service") 283 | ) { r => 284 | IO( 285 | r.headers 286 | .get(ci"Authorization") 287 | .get 288 | .head 289 | .value 290 | ) 291 | }.asserting(_ shouldBe expectedAuthorizationValue) 292 | } 293 | 294 | "sign a POST request with body" in { 295 | 296 | val expectedAuthorizationValue = 297 | "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=ff11897932ad3f4e8b18135d722051e5ac45fc38421b1da7b9d196a0fe09473a" 298 | val credentials = Credentials( 299 | AccessKeyId("AKIDEXAMPLE"), 300 | SecretAccessKey("wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY") 301 | ) 302 | val dateTime = LocalDateTime 303 | .parse("20150830T123600Z", AwsSigner.dateTimeFormatter) 304 | .atZone(ZoneOffset.UTC) 305 | 306 | val request = IO( 307 | POST 308 | .apply( 309 | "Param1=value1", 310 | uri"/", 311 | Host("example.amazonaws.com"), 312 | `Content-Type`(MediaType.application.`x-www-form-urlencoded`), 313 | `X-Amz-Date`(HttpDate.unsafeFromZonedDateTime(dateTime)) 314 | ) 315 | ).map(_.removeHeader[`Content-Length`]) 316 | 317 | withSignRequest( 318 | request, 319 | credentials = credentials, 320 | region = Region.`us-east-1`, 321 | service = Service("service") 322 | ) { r => 323 | IO( 324 | r.headers 325 | .get(ci"Authorization") 326 | .get 327 | .head 328 | .value 329 | ) 330 | }.asserting(_ shouldBe expectedAuthorizationValue) 331 | } 332 | } 333 | 334 | def withFixedRequest[A]( 335 | req: IO[Request[IO]], 336 | now: Instant = Instant.now(), 337 | credentials: Credentials = Credentials(AccessKeyId("FOO"), SecretAccessKey("BAR")) 338 | )(f: Request[IO] => IO[A]): IO[A] = { 339 | for { 340 | request <- req 341 | fixedRequest <- AwsSigner.fixRequest(request, credentials, now) 342 | result <- f(fixedRequest) 343 | } yield result 344 | } 345 | 346 | def withSignRequest[A]( 347 | req: IO[Request[IO]], 348 | region: Region = Region.`eu-west-1`, 349 | service: Service = Service.DynamoDb, 350 | credentials: Credentials = Credentials(AccessKeyId("FOO"), SecretAccessKey("BAR")) 351 | )(f: Request[IO] => IO[A]): IO[A] = { 352 | for { 353 | request <- req 354 | signedRequest <- AwsSigner.signRequest( 355 | request, 356 | credentials, 357 | region, 358 | service 359 | ) 360 | result <- f(signedRequest) 361 | } yield result 362 | } 363 | } 364 | -------------------------------------------------------------------------------- /modules/common/src/it/resources/log4j2-it.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /modules/common/src/it/scala/common/IntegrationSpec.scala: -------------------------------------------------------------------------------- 1 | package com.ovoenergy.comms.aws 2 | package common 3 | 4 | import org.scalatest.wordspec.AsyncWordSpec 5 | import org.scalatest.matchers.should.Matchers 6 | 7 | abstract class IntegrationSpec extends AsyncWordSpec with Matchers { 8 | sys.props.put("log4j.configurationFile", "log4j2-it.xml") 9 | } 10 | 11 | -------------------------------------------------------------------------------- /modules/common/src/main/scala/common/CredentialsProvider.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.comms.aws 18 | package common 19 | 20 | import cats.effect.Sync 21 | import cats.implicits._ 22 | 23 | import software.amazon.awssdk.auth.credentials._ 24 | 25 | import model._ 26 | import Credentials._ 27 | 28 | trait CredentialsProvider[F[_]] { 29 | def get: F[Credentials] 30 | } 31 | 32 | object CredentialsProvider { 33 | def default[F[_]: Sync]: CredentialsProvider[F] = 34 | fromAwsCredentialProvider[F]( 35 | AwsCredentialsProviderChain.of(DefaultCredentialsProvider.create()) 36 | ) 37 | 38 | // TODO refresh creds automagically 39 | def fromAwsCredentialProvider[F[_]]( 40 | awsCredentialsProvider: AwsCredentialsProvider 41 | )(implicit F: Sync[F]): CredentialsProvider[F] = 42 | new CredentialsProvider[F] { 43 | override def get: F[Credentials] = 44 | F.delay(awsCredentialsProvider.resolveCredentials()).map { 45 | case creds: AwsSessionCredentials => 46 | Credentials( 47 | AccessKeyId(creds.accessKeyId()), 48 | SecretAccessKey(creds.secretAccessKey()), 49 | SessionToken(creds.sessionToken()).some 50 | ) 51 | case creds => 52 | Credentials( 53 | AccessKeyId(creds.accessKeyId()), 54 | SecretAccessKey(creds.secretAccessKey()), 55 | None 56 | ) 57 | } 58 | } 59 | 60 | def resolveFromEnvironmentVariables: Option[Credentials] = { 61 | 62 | val accessKeyIdEnv = "AWS_ACCESS_KEY_ID" 63 | val secretAccessKey = "AWS_SECRET_ACCESS_KEY" 64 | val sessionTokenEnv = "AWS_SESSION_TOKEN" 65 | 66 | // These are legacy 67 | val accessKey = "AWS_ACCESS_KEY" 68 | val secretKeyEnv = "AWS_SECRET_KEY" 69 | 70 | ( 71 | sys.env 72 | .get(accessKeyIdEnv) 73 | .orElse(sys.env.get(accessKey)) 74 | .map(AccessKeyId.apply), 75 | sys.env 76 | .get(secretKeyEnv) 77 | .orElse(sys.env.get(secretAccessKey)) 78 | .map(SecretAccessKey.apply) 79 | ).mapN { (accessKeyId, secretAccessKey) => 80 | val sessionToken = sys.env 81 | .get(sessionTokenEnv) 82 | .map(SessionToken.apply) 83 | Credentials(accessKeyId, secretAccessKey, sessionToken) 84 | } 85 | } 86 | 87 | def resolveFromSystemProperties: Option[Credentials] = { 88 | 89 | val accessKeyIdProperty = "aws.accessKeyId" 90 | val secretKeyProperty = "aws.secretKey" 91 | val sessionTokenProperty = "aws.sessionToken" 92 | 93 | ( 94 | sys.props.get(accessKeyIdProperty).map(AccessKeyId.apply), 95 | sys.props.get(secretKeyProperty).map(SecretAccessKey.apply) 96 | ).mapN { (accessKeyId, secretAccessKey) => 97 | val sessionToken = sys.props 98 | .get(sessionTokenProperty) 99 | .map(SessionToken.apply) 100 | 101 | Credentials(accessKeyId, secretAccessKey, sessionToken) 102 | } 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /modules/common/src/main/scala/common/HttpCodecs.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.comms.aws 18 | package common 19 | 20 | import model.Credentials._ 21 | 22 | import java.time._ 23 | import format.DateTimeFormatter 24 | 25 | import org.http4s._ 26 | import util.Writer 27 | 28 | import scala.util.Try 29 | import cats.implicits._ 30 | 31 | trait HttpCodecs { 32 | 33 | implicit val httpDateHttpCodec: HttpCodec[HttpDate] = 34 | new HttpCodec[HttpDate] { 35 | 36 | private val dateTimeFormatter = 37 | DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmssX") 38 | 39 | override def parse(s: String): ParseResult[HttpDate] = 40 | Try(ZonedDateTime.parse(s, dateTimeFormatter)).toEither 41 | .leftMap(_ => 42 | ParseFailure( 43 | "Error to parse a datetime", 44 | s"The string `$s` is not a valid datetime" 45 | ) 46 | ) 47 | .flatMap(a => HttpDate.fromEpochSecond(a.toEpochSecond)) 48 | 49 | override def render(writer: Writer, t: HttpDate): writer.type = { 50 | val formattedDateTime = 51 | Instant 52 | .ofEpochSecond(t.epochSecond) 53 | .atOffset(ZoneOffset.UTC) 54 | .format(dateTimeFormatter) 55 | writer << formattedDateTime 56 | } 57 | } 58 | 59 | implicit val sessionTokenHttpCodec: HttpCodec[SessionToken] = 60 | new HttpCodec[SessionToken] { 61 | 62 | override def parse(s: String): ParseResult[SessionToken] = 63 | SessionToken(s).asRight 64 | 65 | override def render(writer: Writer, t: SessionToken): writer.type = 66 | writer << t.value 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /modules/common/src/main/scala/common/headers.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.comms.aws 18 | package common 19 | 20 | import model.Credentials._ 21 | 22 | import java.time._ 23 | import org.http4s.{Header, _} 24 | import org.http4s.Header.Raw 25 | import cats.implicits._ 26 | import org.typelevel.ci.{CIString, _} 27 | 28 | object headers extends HttpCodecs { 29 | 30 | object `X-Amz-Date` { 31 | def name: CIString = ci"X-Amz-Date" 32 | 33 | def parse(s: String): ParseResult[`X-Amz-Date`] = 34 | HttpCodec[HttpDate].parse(s).map(`X-Amz-Date`.apply) 35 | 36 | def unsafeFromInstant(instant: Instant): `X-Amz-Date` = { 37 | `X-Amz-Date`(HttpDate.unsafeFromInstant(instant)) 38 | } 39 | 40 | def unsafeFromDateTime(dateTime: OffsetDateTime): `X-Amz-Date` = { 41 | `X-Amz-Date`(HttpDate.unsafeFromZonedDateTime(dateTime.toZonedDateTime)) 42 | } 43 | 44 | def unsafeFromDateTime(dateTime: ZonedDateTime): `X-Amz-Date` = { 45 | `X-Amz-Date`(HttpDate.unsafeFromZonedDateTime(dateTime)) 46 | } 47 | 48 | def matchHeader(header: Any): Option[`X-Amz-Date`] = header match { 49 | case h: `X-Amz-Date` => h.some 50 | case Raw(n, v) if n == name => parse(v).toOption 51 | case _ => None 52 | } 53 | 54 | implicit val dateInstance: Header[`X-Amz-Date`, Header.Single] = Header.createRendered( 55 | `X-Amz-Date`.name, 56 | _.date, 57 | `X-Amz-Date`.parse 58 | ) 59 | } 60 | 61 | final case class `X-Amz-Date`(date: HttpDate) 62 | 63 | object `X-Amz-Content-SHA256` extends { 64 | val name: CIString = ci"X-Amz-Content-SHA256" 65 | 66 | def matchHeader(header: Any): Option[`X-Amz-Content-SHA256`] = 67 | header match { 68 | case h: `X-Amz-Content-SHA256` => h.some 69 | case Raw(n, v) if n == name => parse(v).toOption 70 | case _ => None 71 | } 72 | 73 | def parse(s: String): ParseResult[`X-Amz-Content-SHA256`] = 74 | `X-Amz-Content-SHA256`(s).asRight 75 | 76 | implicit val contentSha256Instance: Header[`X-Amz-Content-SHA256`, Header.Single] = 77 | Header.createRendered( 78 | `X-Amz-Content-SHA256`.name, 79 | _.hashedContent, 80 | `X-Amz-Content-SHA256`.parse 81 | ) 82 | } 83 | 84 | final case class `X-Amz-Content-SHA256`(hashedContent: String) 85 | 86 | object `X-Amz-Security-Token` { 87 | val name: CIString = ci"X-Amz-Security-Token" 88 | 89 | def matchHeader(header: Any): Option[`X-Amz-Security-Token`] = 90 | header match { 91 | case h: `X-Amz-Security-Token` => h.some 92 | case Raw(n, v) if n == name => parse(v).toOption 93 | case _ => None 94 | } 95 | 96 | def parse(s: String): ParseResult[`X-Amz-Security-Token`] = 97 | HttpCodec[SessionToken].parse(s).map(`X-Amz-Security-Token`.apply) 98 | 99 | implicit val securityTokenInstance: Header[`X-Amz-Security-Token`, Header.Single] = 100 | Header.createRendered( 101 | `X-Amz-Security-Token`.name, 102 | _.sessionToken, 103 | `X-Amz-Security-Token`.parse 104 | ) 105 | } 106 | 107 | final case class `X-Amz-Security-Token`(sessionToken: SessionToken) 108 | 109 | object `X-Amz-Target` { 110 | val name: CIString = ci"X-Amz-Target" 111 | 112 | def matchHeader(header: Any): Option[`X-Amz-Target`] = 113 | header match { 114 | case h: `X-Amz-Target` => h.some 115 | case Raw(n, v) if n == name => parse(v).toOption 116 | case _ => None 117 | } 118 | 119 | def parse(s: String): ParseResult[`X-Amz-Target`] = 120 | `X-Amz-Target`(s).asRight 121 | 122 | implicit val targetInstance: Header[`X-Amz-Target`, Header.Single] = Header.createRendered( 123 | `X-Amz-Target`.name, 124 | _.target, 125 | `X-Amz-Target`.parse 126 | ) 127 | } 128 | 129 | final case class `X-Amz-Target`(target: String) 130 | } 131 | -------------------------------------------------------------------------------- /modules/common/src/main/scala/common/mediaTypes.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.comms.aws 18 | package common 19 | 20 | import org.http4s.MediaType 21 | 22 | object mediaTypes { 23 | 24 | val `application/x-amz-json-1.0`: MediaType = MediaType 25 | .parse("application/x-amz-json-1.0") 26 | .fold(e => throw new Exception(e), identity) 27 | 28 | } 29 | -------------------------------------------------------------------------------- /modules/common/src/main/scala/common/model.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.comms.aws 18 | package common 19 | 20 | object model { 21 | 22 | import Credentials._ 23 | 24 | case class Credentials( 25 | accessKeyId: AccessKeyId, 26 | secretAccessKey: SecretAccessKey, 27 | sessionToken: Option[SessionToken] = None 28 | ) 29 | 30 | object Credentials { 31 | 32 | case class AccessKeyId(value: String) 33 | 34 | case class SecretAccessKey(value: String) { 35 | override def toString = "SecretAccessKey(***)" 36 | } 37 | 38 | case class SessionToken(value: String) { 39 | override def toString = "SessionToken(***)" 40 | } 41 | 42 | } 43 | 44 | case class RequestId(value: String) 45 | 46 | case class Service(value: String) 47 | 48 | object Service { 49 | val S3: Service = Service("s3") 50 | val DynamoDb: Service = Service("dynamodb") 51 | val TestService: Service = Service("service") 52 | } 53 | 54 | case class Region(value: String) 55 | 56 | object Region { 57 | val `us-west-2`: Region = Region("us-west-2") 58 | val `us-west-1`: Region = Region("us-west-1") 59 | val `us-east-2`: Region = Region("us-east-2") 60 | val `us-east-1`: Region = Region("us-east-1") 61 | val `ap-south-1`: Region = Region("ap-south-1") 62 | val `ap-northeast-2`: Region = Region("ap-northeast-2") 63 | val `ap-southeast-1`: Region = Region("ap-southeast-1") 64 | val `ap-southeast-2`: Region = Region("ap-southeast-2") 65 | val `ap-northeast-1`: Region = Region("ap-northeast-1") 66 | val `ca-central-1`: Region = Region("ca-central-1") 67 | val `cn-north-1`: Region = Region("cn-north-1") 68 | val `eu-central-1`: Region = Region("eu-central-1") 69 | val `eu-west-1`: Region = Region("eu-west-1") 70 | val `eu-west-2`: Region = Region("eu-west-2") 71 | val `eu-west-3`: Region = Region("eu-west-3") 72 | val `sa-east-1`: Region = Region("sa-east-1") 73 | val `us-gov-west-1`: Region = Region("us-gov-west-1") 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /modules/common/src/test/resources/log4j2-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /modules/common/src/test/scala/common/UnitSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.comms.aws 18 | package common 19 | 20 | import org.scalatestplus.scalacheck.ScalaCheckDrivenPropertyChecks 21 | import org.scalatest.matchers.should.Matchers 22 | import org.scalatest.wordspec.AsyncWordSpec 23 | 24 | abstract class UnitSpec extends AsyncWordSpec with Matchers with ScalaCheckDrivenPropertyChecks 25 | -------------------------------------------------------------------------------- /modules/s3/src/it/resources/more.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ovotech/comms-aws/d70e7e6c35afbd7a725ed5aa00c5d07d95539860/modules/s3/src/it/resources/more.pdf -------------------------------------------------------------------------------- /modules/s3/src/it/scala/s3/S3Spec.scala: -------------------------------------------------------------------------------- 1 | package com.ovoenergy.comms.aws 2 | package s3 3 | 4 | import cats.implicits._ 5 | import cats.effect._ 6 | import cats.effect.testing.scalatest.AsyncIOSpec 7 | import fs2.io._ 8 | 9 | import java.nio.charset.StandardCharsets.UTF_8 10 | import java.util.UUID 11 | import common.model._ 12 | import common.{CredentialsProvider, IntegrationSpec} 13 | import model._ 14 | import org.http4s.MediaType 15 | 16 | class S3Spec extends IntegrationSpec with AsyncIOSpec { 17 | 18 | val existingKey = Key("more.pdf") 19 | val duplicateKey = Key("duplicate") 20 | val notExistingKey = Key("less.pdf") 21 | val nestedKey = Key("a/b/c") 22 | val slashLeadingKey = Key("/a") 23 | 24 | val existingBucket = Bucket("ovo-comms-test") 25 | val nonExistingBucket = Bucket("ovo-comms-non-existing-bucket") 26 | 27 | val morePdf = IO(getClass.getResourceAsStream("/more.pdf")) 28 | val moreSize = IO { 29 | getClass 30 | .getResource("/more.pdf") 31 | .openConnection() 32 | .getContentLength 33 | .toLong 34 | } 35 | val randomKey = IO(Key(UUID.randomUUID().toString)) 36 | 37 | "headObject" when { 38 | 39 | "the bucket exists" when { 40 | "the key does exist" should { 41 | val key = existingKey 42 | "return the object eTag" in { 43 | withS3 { s3 => 44 | s3.headObject(existingBucket, key) 45 | }.flatMap(IO.fromEither(_)).map(os => os.eTag shouldBe Etag("9fe029056e0841dde3c1b8a169635f6f")) 46 | } 47 | 48 | "return the object metadata" in { 49 | withS3 { s3 => 50 | s3.headObject(existingBucket, key) 51 | }.flatMap { 52 | IO.fromEither(_).map(os => os.metadata shouldBe Map("is-test" -> "true")) 53 | } 54 | } 55 | 56 | "return the object contentLength" in { 57 | withS3 { s3 => 58 | s3.headObject(existingBucket, key) 59 | }.flatMap(IO.fromEither(_)).map { os => 60 | os.contentLength should be > 0L 61 | } 62 | } 63 | } 64 | 65 | "the key does not exist" should { 66 | 67 | "return a Left" in { 68 | withS3 { s3 => 69 | s3.headObject(existingBucket, notExistingKey) 70 | }.unsafeRunSync() shouldBe a[Left[_, _]] 71 | } 72 | 73 | "return NoSuchKey error code" in { 74 | withS3 { s3 => 75 | s3.headObject(existingBucket, notExistingKey) 76 | }.flatMap(IO.fromEither(_)).assertThrowsError[model.Error](error => error.code shouldBe Error.Code("NoSuchKey")) 77 | } 78 | 79 | "return the given key as resource" in { 80 | withS3 { s3 => 81 | s3.headObject(existingBucket, notExistingKey) 82 | }.flatMap(IO.fromEither(_)).assertThrowsError[model.Error] { error => 83 | error.key shouldBe notExistingKey.some 84 | } 85 | } 86 | } 87 | } 88 | 89 | "the bucket does not exist" should { 90 | 91 | "return a Left" in { 92 | withS3 { s3 => 93 | s3.headObject(nonExistingBucket, existingKey) 94 | }.unsafeRunSync() shouldBe a[Left[_, _]] 95 | } 96 | 97 | "return NoSuchBucket error code" in { 98 | withS3 { s3 => 99 | s3.headObject(nonExistingBucket, existingKey) 100 | }.flatMap(IO.fromEither(_)).assertThrowsError[model.Error] { error => 101 | error.code shouldBe Error.Code("NoSuchBucket") 102 | } 103 | } 104 | 105 | "return the given bucket" in { 106 | withS3 { s3 => 107 | s3.headObject(nonExistingBucket, existingKey) 108 | }.flatMap(IO.fromEither(_)).assertThrowsError[model.Error]{ error => 109 | error.bucketName shouldBe nonExistingBucket.some 110 | } 111 | } 112 | 113 | } 114 | 115 | } 116 | 117 | "getObject" when { 118 | 119 | "the bucket exists" when { 120 | "the key does exist" should { 121 | 122 | "return the object eTag" in checkGetObject(existingBucket, existingKey) { objOrError => 123 | objOrError.map(_.summary.eTag) shouldBe Etag("9fe029056e0841dde3c1b8a169635f6f").asRight 124 | } 125 | 126 | "return the object metadata" in checkGetObject(existingBucket, existingKey) { objOrError => 127 | objOrError.map(_.summary.metadata) shouldBe Map("is-test" -> "true").asRight 128 | } 129 | 130 | "return the object contentLength" in checkGetObject(existingBucket, existingKey) { 131 | objOrError => 132 | objOrError.map(_.summary.contentLength > 0L) shouldBe true.asRight 133 | } 134 | 135 | "return the object that can be consumed to a file" in { 136 | withS3 { s3 => 137 | s3.getObject(existingBucket, existingKey) 138 | .map(_.leftWiden[Throwable]) 139 | .rethrow 140 | .use(_.content.compile.toList) 141 | }.unsafeRunSync() should not be empty 142 | } 143 | 144 | // FIXME This test does not pass, but we have verified manually that the connection is getting disposed 145 | "return the object that after been consumed cannot be consumed again" ignore checkGetObject( 146 | existingBucket, 147 | existingKey 148 | ) { objOrError => 149 | IO.fromEither(objOrError).flatMap { obj => 150 | obj.content.compile.toList >> obj.content.compile.toList.attempt 151 | }.asserting(_ shouldBe a[Left[_, _]]) 152 | } 153 | } 154 | 155 | "the key does not exist" should { 156 | 157 | "return a Left" in checkGetObject(existingBucket, notExistingKey) { objOrError => 158 | objOrError shouldBe a[Left[_, _]] 159 | } 160 | 161 | "return NoSuchKey error code" in checkGetObject(existingBucket, notExistingKey) { 162 | objOrError => 163 | objOrError.left.map { error => 164 | error.code 165 | } shouldBe Left(Error.Code("NoSuchKey")) 166 | } 167 | 168 | "return the given key as resource" in checkGetObject(existingBucket, notExistingKey) { 169 | objOrError => 170 | objOrError.left.map { error => 171 | error.key 172 | } shouldBe Left(notExistingKey.some) 173 | } 174 | 175 | } 176 | } 177 | 178 | "the bucket does not exist" should { 179 | 180 | "return a Left" in checkGetObject(nonExistingBucket, existingKey) { objOrError => 181 | objOrError shouldBe a[Left[_, _]] 182 | } 183 | 184 | "return NoSuchBucket error code" in checkGetObject(nonExistingBucket, existingKey) { 185 | objOrError => 186 | objOrError.left.map { error => 187 | error.code 188 | } shouldBe Left(Error.Code("NoSuchBucket")) 189 | } 190 | 191 | "return the given bucket" in checkGetObject(nonExistingBucket, existingKey) { objOrError => 192 | objOrError.left.map { error => 193 | error.bucketName 194 | } shouldBe Left(nonExistingBucket.some) 195 | } 196 | 197 | } 198 | 199 | } 200 | 201 | "putObject" when { 202 | "the bucket exists" when { 203 | "the key does not exist" should { 204 | 205 | "upload the object content with correct Content-Type" in { 206 | val (putResult, getResult) = withS3 { s3 => 207 | val contentIo: IO[ObjectContent[IO]] = moreSize.map { size => 208 | ObjectContent( 209 | readInputStream(morePdf, chunkSize = 64 * 1024), 210 | size, 211 | chunked = true, 212 | mediaType = MediaType.application.pdf 213 | ) 214 | } 215 | 216 | for { 217 | key <- randomKey 218 | content <- contentIo 219 | result <- s3.putObject(existingBucket, key, content) 220 | getResult <- IO(checkGetObject(existingBucket, key)(identity)) 221 | } yield (result, getResult) 222 | 223 | }.unsafeRunSync() 224 | 225 | putResult shouldBe a[Right[_,_]] 226 | getResult.map(_.summary.mediaType) shouldBe Right(MediaType.application.pdf) 227 | 228 | } 229 | 230 | "upload the object content with custom metadata" in { 231 | 232 | val expectedMetadata = Map("test" -> "yes") 233 | 234 | withS3 { s3 => 235 | val contentIo: IO[ObjectContent[IO]] = moreSize.map { size => 236 | ObjectContent( 237 | readInputStream(morePdf, chunkSize = 64 * 1024), 238 | size, 239 | chunked = true 240 | ) 241 | } 242 | 243 | for { 244 | key <- randomKey 245 | content <- contentIo 246 | _ <- s3.putObject(existingBucket, key, content, expectedMetadata) 247 | summary <- s3.headObject(existingBucket, key) 248 | } yield summary 249 | }.flatMap(IO.fromEither(_)).map { summary => 250 | summary.metadata shouldBe expectedMetadata 251 | } 252 | } 253 | } 254 | 255 | "the key is nested" should { 256 | "succeeding" in { 257 | 258 | val content = ObjectContent.fromByteArray[IO](UUID.randomUUID().toString.getBytes(UTF_8)) 259 | 260 | withS3 { s3 => 261 | s3.putObject(existingBucket, nestedKey, content) 262 | }.unsafeRunSync() shouldBe a[Right[_, _]] 263 | 264 | } 265 | } 266 | 267 | "the key has leading slash" should { 268 | "succeeding" in { 269 | 270 | val content = ObjectContent.fromByteArray[IO](UUID.randomUUID().toString.getBytes(UTF_8)) 271 | 272 | withS3 { s3 => 273 | s3.putObject(existingBucket, slashLeadingKey, content) 274 | }.unsafeRunSync() shouldBe a[Right[_, _]] 275 | 276 | } 277 | } 278 | 279 | "the key does exist" should { 280 | "overwrite the existing key" in { 281 | withS3 { s3 => 282 | val content = 283 | ObjectContent.fromByteArray[IO](UUID.randomUUID().toString.getBytes(UTF_8)) 284 | for { 285 | _ <- s3.putObject(existingBucket, duplicateKey, content) 286 | result <- s3.putObject(existingBucket, duplicateKey, content) 287 | } yield result 288 | }.unsafeRunSync() shouldBe a[Right[_, _]] 289 | } 290 | } 291 | } 292 | 293 | "the bucket does not exist" should { 294 | 295 | "return a Left" in { 296 | withS3 { s3 => 297 | s3.putObject( 298 | nonExistingBucket, 299 | existingKey, 300 | ObjectContent.fromByteArray(Array.fill(128 * 1026)(0: Byte)) 301 | ) 302 | }.unsafeRunSync() shouldBe a[Left[_, _]] 303 | } 304 | 305 | "return NoSuchBucket error code" in withS3 { s3 => 306 | s3.putObject( 307 | nonExistingBucket, 308 | existingKey, 309 | ObjectContent.fromByteArray(Array.fill(128 * 1026)(0: Byte)) 310 | ) 311 | }.flatMap(IO.fromEither(_)).assertThrowsError[model.Error] { error => 312 | error.bucketName shouldBe nonExistingBucket.some 313 | } 314 | } 315 | } 316 | 317 | def checkGetObject[A](bucket: Bucket, key: Key)(f: Either[Error, Object[IO]] => A): A = 318 | withS3 { 319 | _.getObject(bucket, key).use(x => IO(f(x))) 320 | }.unsafeRunSync() 321 | 322 | def withS3[A](f: S3[IO] => IO[A]): IO[A] = { 323 | S3.resource(CredentialsProvider.default[IO], Region.`eu-west-1`).use(f) 324 | } 325 | 326 | } 327 | -------------------------------------------------------------------------------- /modules/s3/src/main/scala/s3/S3.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.comms.aws 18 | package s3 19 | 20 | import cats.implicits._ 21 | import cats.effect._ 22 | 23 | import java.nio.ByteBuffer 24 | import org.http4s._ 25 | import org.http4s.headers._ 26 | import org.http4s.Method._ 27 | import org.http4s.Header.Raw 28 | import org.http4s.client.Client 29 | import org.http4s.blaze.client.BlazeClientBuilder 30 | import org.http4s.client.dsl.Http4sClientDsl 31 | 32 | import scala.concurrent.duration._ 33 | import scalaxml._ 34 | 35 | import scala.xml.Elem 36 | import auth.AwsSigner 37 | import model._ 38 | import common._ 39 | import common.model._ 40 | import fs2.text 41 | import com.ovoenergy.comms.aws.s3.headers._ 42 | import org.typelevel.ci.CIStringSyntax 43 | 44 | trait S3[F[_]] { 45 | 46 | def headObject(bucket: Bucket, key: Key): F[Either[Error, ObjectSummary]] 47 | 48 | def getObject(bucket: Bucket, key: Key): Resource[F, Either[Error, Object[F]]] 49 | 50 | def putObject( 51 | bucket: Bucket, 52 | key: Key, 53 | content: ObjectContent[F], 54 | metadata: Map[String, String] = Map.empty 55 | ): F[Either[Error, ObjectPut]] 56 | 57 | } 58 | 59 | object S3 { 60 | 61 | def resource[F[_]: Async]( 62 | credentialsProvider: CredentialsProvider[F], 63 | region: Region, 64 | endpoint: Option[Uri] = None 65 | ): Resource[F, S3[F]] = { 66 | BlazeClientBuilder[F].resource.map(client => 67 | S3.apply(client, credentialsProvider, region, endpoint) 68 | ) 69 | } 70 | 71 | def apply[F[_]: Async]( 72 | client: Client[F], 73 | credentialsProvider: CredentialsProvider[F], 74 | region: Region, 75 | endpoint: Option[Uri] = None, 76 | retryPolicy: RetryPolicy = RetryPolicy.default 77 | ): S3[F] = new S3[F] with Http4sClientDsl[F] { 78 | 79 | val signer = AwsSigner[F](credentialsProvider, region, Service.S3) 80 | val signedClient = signer(client) 81 | 82 | def baseEndpoint(bucket: Bucket) = endpoint.getOrElse { 83 | Uri.unsafeFromString(s"https://${bucket.name}.s3.${region.value}.amazonaws.com") 84 | } 85 | 86 | def uri(bucket: Bucket, key: Key) = { 87 | val bucketEndpoint = baseEndpoint(bucket) 88 | key.value.split("/", -1).foldLeft(bucketEndpoint) { (acc, x) => acc / x } 89 | } 90 | 91 | def getObject( 92 | bucket: Bucket, 93 | key: Key 94 | ): Resource[F, Either[Error, Object[F]]] = 95 | Resource.eval(Sync[F].delay(GET(uri(bucket, key)))).flatMap { req => 96 | signedClient.run(req).evalMap { 97 | case r if r.status.isSuccess => 98 | parseObjectSummary(r).value.rethrow // TODO should lack of etag be an error here? 99 | .map { summary => Object(summary, r.body).asRight[Error] } 100 | case r if r.status.responseClass == Status.ServerError => 101 | fOfBodyString(r).flatMap(errorBodyString => 102 | Sync[F].raiseError[Either[Error, Object[F]]](RetriableServerError(errorBodyString)) 103 | ) 104 | 105 | case r => 106 | r.as[Error].map(_.asLeft[Object[F]]) 107 | } 108 | } 109 | 110 | def headObject( 111 | bucket: Bucket, 112 | key: Key 113 | ): F[Either[Error, ObjectSummary]] = { 114 | 115 | for { 116 | request <- Sync[F].delay(GET(uri(bucket, key))) 117 | result <- withRetry(signedClient.run(request).use { 118 | case r if r.status.isSuccess => 119 | r.as[ObjectSummary].map(_.asRight[Error]) 120 | case r if r.status.responseClass == Status.ServerError => 121 | fOfBodyString(r).flatMap { errorBodyString => 122 | Sync[F] 123 | .raiseError[Either[Error, ObjectSummary]](RetriableServerError(errorBodyString)) 124 | } 125 | case r => 126 | r.as[Error].map(_.asLeft[ObjectSummary]) 127 | }) 128 | } yield result 129 | } 130 | 131 | def putObject( 132 | bucket: Bucket, 133 | key: Key, 134 | content: ObjectContent[F], 135 | metadata: Map[String, String] = Map.empty 136 | ): F[Either[Error, ObjectPut]] = { 137 | 138 | def initHeaders: F[Headers] = 139 | Sync[F] 140 | .fromEither(`Content-Length`.fromLong(content.contentLength)) 141 | .map { contentLength => 142 | Headers( 143 | contentLength, 144 | `Content-Type`(content.mediaType, content.charset), 145 | metadata.map { case (k, v) => Raw(ci"${`X-Amz-Meta-`}$k", v) }.toSeq 146 | ) 147 | } 148 | 149 | val extractContent: F[Array[Byte]] = 150 | if (content.contentLength > ObjectContent.MaxDataLength) { 151 | Sync[F].raiseError { 152 | val msg = 153 | s"The content is too long to be transmitted in a single chunk, max allowed content length: ${ObjectContent.MaxDataLength}" 154 | new IllegalArgumentException(msg) 155 | } 156 | } else { 157 | content.data.chunks.compile 158 | .fold(ByteBuffer.allocate(content.contentLength.toInt))((buffer, chunk) => 159 | buffer put chunk.toByteBuffer 160 | ) 161 | .map(_.array()) 162 | } 163 | 164 | for { 165 | hs <- initHeaders 166 | contentAsSingleChunk <- extractContent 167 | request = PUT(contentAsSingleChunk, uri(bucket, key), hs) 168 | result <- withRetry(signedClient.run(request).use { 169 | case r if r.status.isSuccess => r.as[ObjectPut].map(_.asRight[Error]) 170 | case r if r.status.responseClass == Status.ServerError => 171 | fOfBodyString(r).flatMap { errorBodyString => 172 | Sync[F].raiseError[Either[Error, ObjectPut]](RetriableServerError(errorBodyString)) 173 | } 174 | case r => r.as[Error].map(_.asLeft[ObjectPut]) 175 | }) 176 | } yield result 177 | } 178 | 179 | private def withRetry[A](fa: F[A]): F[A] = 180 | fs2.Stream 181 | .retry(fa, retryPolicy.delay, retryPolicy.nextDelay, retryPolicy.maxAttempts) 182 | .compile 183 | .lastOrError 184 | 185 | implicit def errorEntityDecoder: EntityDecoder[F, Error] = 186 | EntityDecoder[F, Elem].flatMapR { elem => 187 | val code = Option(elem \ "Code") 188 | .filter(_.length == 1) 189 | .map(e => Error.Code(e.text)) 190 | 191 | val message = Option(elem \ "Message") 192 | .filter(_.length == 1) 193 | .map(_.text) 194 | 195 | val requestId = Option(elem \ "RequestId") 196 | .filter(_.length == 1) 197 | .map(id => RequestId(id.text)) 198 | 199 | (code, message, requestId) 200 | .mapN { (code, message, requestId) => 201 | val bucketName = elem.child 202 | .find(_.label == "BucketName") 203 | .map(node => Bucket(node.text)) 204 | val key = 205 | elem.child.find(_.label == "Key").map(node => Key(node.text)) 206 | 207 | Error( 208 | code = code, 209 | requestId = requestId, 210 | message = message, 211 | key = key, 212 | bucketName = bucketName 213 | ) 214 | } 215 | .fold[DecodeResult[F, Error]]( 216 | DecodeResult.failureT( 217 | InvalidMessageBodyFailure( 218 | "Code, RequestId and Message XML elements are mandatory" 219 | ) 220 | ) 221 | )(error => DecodeResult.successT(error)) 222 | } 223 | 224 | implicit def objectPutDecoder: EntityDecoder[F, ObjectPut] = 225 | new EntityDecoder[F, ObjectPut] { 226 | 227 | override def decode( 228 | msg: Media[F], 229 | strict: Boolean 230 | ): DecodeResult[F, ObjectPut] = { 231 | msg.headers 232 | .get[ETag] 233 | .map(t => ObjectPut(Etag(t.tag.tag))) 234 | // TODO InvalidMessageBodyFailure is not correct here as there is no body 235 | .fold[DecodeResult[F, ObjectPut]]( 236 | DecodeResult.failureT( 237 | InvalidMessageBodyFailure("The ETag header must be present") 238 | ) 239 | )(ok => DecodeResult.successT(ok)) 240 | } 241 | 242 | override val consumes: Set[MediaRange] = 243 | MediaRange.standard.values.toSet 244 | } 245 | 246 | implicit def objectSummaryDecoder: EntityDecoder[F, ObjectSummary] = 247 | EntityDecoder.decodeBy(MediaRange.`*/*`)(parseObjectSummary) 248 | 249 | def parseObjectSummary( 250 | response: Media[F] 251 | ): DecodeResult[F, ObjectSummary] = { 252 | 253 | val eTag: DecodeResult[F, Etag] = response.headers 254 | .get[ETag] 255 | .map(t => DecodeResult.successT[F, Etag](Etag(t.tag.tag))) 256 | .getOrElse( 257 | DecodeResult.failureT[F, Etag]( 258 | MalformedMessageBodyFailure( 259 | "ETag header must be present on the response" 260 | ) 261 | ) 262 | ) 263 | 264 | val mediaType: DecodeResult[F, MediaType] = response.headers 265 | .get[`Content-Type`] 266 | .map(_.mediaType) 267 | .fold( 268 | DecodeResult.failureT[F, MediaType]( 269 | MalformedMessageBodyFailure( 270 | "The response needs to contain Media-Type header" 271 | ) 272 | ) 273 | )(DecodeResult.successT[F, MediaType]) 274 | 275 | val charset: DecodeResult[F, Option[Charset]] = response.headers 276 | .get[`Content-Type`] 277 | .flatMap(_.charset) 278 | .traverse(DecodeResult.successT[F, Charset]) 279 | 280 | val contentLength = response.headers 281 | .get[`Content-Length`] 282 | .map(_.length) 283 | .fold( 284 | DecodeResult.failureT[F, Long]( 285 | MalformedMessageBodyFailure( 286 | "The response needs to contain Content-Length header" 287 | ) 288 | ) 289 | )(DecodeResult.successT[F, Long]) 290 | 291 | val metadata: Map[String, String] = response.headers.headers.collect { 292 | case h if h.name.toString.toLowerCase.startsWith(`X-Amz-Meta-`) => 293 | h.name.toString.substring(`X-Amz-Meta-`.length) -> h.value 294 | }.toMap 295 | 296 | (eTag, mediaType, charset, contentLength).mapN { (eTag, mediaType, charset, contentLength) => 297 | ObjectSummary( 298 | eTag, 299 | mediaType, 300 | contentLength, 301 | charset, 302 | metadata 303 | ) 304 | } 305 | } 306 | 307 | } 308 | 309 | private def fOfBodyString[F[_]: Sync](r: Response[F]) = { 310 | r.body.through(text.utf8.decode).compile.string 311 | } 312 | 313 | private case class RetriableServerError(bodyContent: String) extends Exception { 314 | override def getMessage = bodyContent 315 | } 316 | 317 | case class RetryPolicy( 318 | delay: FiniteDuration, 319 | nextDelay: FiniteDuration => FiniteDuration, 320 | maxAttempts: Int, 321 | retriable: Throwable => Boolean 322 | ) 323 | 324 | object RetryPolicy { 325 | // about 50s in total 326 | val default = RetryPolicy( 327 | 50.milliseconds, { prevDuration: FiniteDuration => 328 | (prevDuration.toMillis * 1.25).milliseconds 329 | }, 330 | 25, { 331 | case RetriableServerError(_) => true 332 | case _ => false 333 | } 334 | ) 335 | } 336 | } 337 | -------------------------------------------------------------------------------- /modules/s3/src/main/scala/s3/headers.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.comms.aws 18 | package s3 19 | 20 | import model._ 21 | import org.http4s._ 22 | import syntax.all._ 23 | import Header.Raw 24 | import util.Writer 25 | import cats.implicits._ 26 | import org.typelevel.ci.CIString 27 | 28 | trait HttpCodecs { 29 | 30 | implicit lazy val storageClassHttpCodec: HttpCodec[StorageClass] = 31 | new HttpCodec[StorageClass] { 32 | override def parse(s: String): ParseResult[StorageClass] = 33 | StorageClass 34 | .fromString(s) 35 | .toRight( 36 | new ParseFailure( 37 | "Failed to parse a storage class", 38 | s"$s is nota valid storage class, valid values are: ${StorageClass.values}" 39 | ) 40 | ) 41 | override def render(writer: Writer, t: StorageClass): writer.type = 42 | writer << t.toString 43 | } 44 | 45 | } 46 | 47 | object headers extends HttpCodecs { 48 | 49 | val `X-Amz-Meta-` = "x-amz-meta-" 50 | 51 | object `X-Amz-Storage-Class` { 52 | 53 | val name: CIString = "X-Amz-Storage-Class".ci 54 | 55 | def matchHeader(header: Any): Option[`X-Amz-Storage-Class`] = 56 | header match { 57 | case h: `X-Amz-Storage-Class` => h.some 58 | case Raw(n, v) if n == name => parse(v).toOption 59 | case _ => None 60 | } 61 | 62 | def parse(s: String): ParseResult[`X-Amz-Storage-Class`] = 63 | HttpCodec[StorageClass].parse(s).map(`X-Amz-Storage-Class`.apply) 64 | 65 | } 66 | 67 | final case class `X-Amz-Storage-Class`(storageClass: StorageClass) 68 | 69 | implicit val storageClassInstance: Header[`X-Amz-Storage-Class`, Header.Single] = 70 | Header.createRendered( 71 | `X-Amz-Storage-Class`.name, 72 | _.storageClass, 73 | `X-Amz-Storage-Class`.parse 74 | ) 75 | 76 | } 77 | -------------------------------------------------------------------------------- /modules/s3/src/main/scala/s3/model.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.comms.aws 18 | package s3 19 | 20 | import cats.implicits._ 21 | import cats.effect._ 22 | import fs2._ 23 | 24 | import java.nio.file.{Files, Path} 25 | import org.http4s.{MediaType, Charset} 26 | 27 | import common.model._ 28 | 29 | object model { 30 | 31 | case class ObjectSummary( 32 | eTag: Etag, 33 | mediaType: MediaType, 34 | contentLength: Long, 35 | charset: Option[Charset], 36 | metadata: Map[String, String] 37 | ) 38 | 39 | /** 40 | * The S3 Object. 41 | * @param content The Stream[F, Byte] on the object content. 42 | * @tparam F The effect 43 | */ 44 | case class Object[F[_]](summary: ObjectSummary, content: Stream[F, Byte]) 45 | 46 | case class Error( 47 | code: Error.Code, 48 | requestId: RequestId, 49 | message: String, 50 | key: Option[Key] = None, 51 | bucketName: Option[Bucket] 52 | ) extends Exception(message) 53 | 54 | object Error { 55 | 56 | case class Code(value: String) 57 | 58 | } 59 | 60 | sealed trait StorageClass { 61 | 62 | import StorageClass._ 63 | 64 | override def toString: String = this match { 65 | case Standard => "STANDARD" 66 | case StandardIa => "STANDARD_IA" 67 | case OnezoneIa => "ONEZONE_IA" 68 | case ReducedRedundancy => "REDUCED_REDUNDANCY" 69 | } 70 | } 71 | 72 | object StorageClass { 73 | 74 | case object Standard extends StorageClass 75 | 76 | case object StandardIa extends StorageClass 77 | 78 | case object OnezoneIa extends StorageClass 79 | 80 | case object ReducedRedundancy extends StorageClass 81 | 82 | def values: IndexedSeq[StorageClass] = Vector( 83 | Standard, 84 | StandardIa, 85 | OnezoneIa, 86 | ReducedRedundancy 87 | ) 88 | 89 | def fromString(string: String): Option[StorageClass] = string match { 90 | case "STANDARD" => Standard.some 91 | case "STANDARD_IA" => StandardIa.some 92 | case "ONEZONE_IA" => OnezoneIa.some 93 | case "REDUCED_REDUNDANCY" => ReducedRedundancy.some 94 | case _ => none[StorageClass] 95 | } 96 | 97 | def unsafeFromString(string: String): StorageClass = 98 | fromString(string) 99 | .getOrElse( 100 | throw new IllegalArgumentException( 101 | s"string is not a valid storage class, valid values are: $values" 102 | ) 103 | ) 104 | 105 | } 106 | 107 | case class Bucket(name: String) 108 | 109 | case class Key(value: String) 110 | 111 | case class Etag(value: String) 112 | 113 | case class ObjectPut(etag: Etag) 114 | 115 | // TODO Use something like Byte/KiloByte/Mb/Gb for the length 116 | case class ObjectContent[F[_]]( 117 | data: Stream[F, Byte], 118 | contentLength: Long, 119 | chunked: Boolean, 120 | mediaType: MediaType = MediaType.application.`octet-stream`, 121 | charset: Option[Charset] = None 122 | ) 123 | 124 | object ObjectContent { 125 | 126 | val MaxDataLength: Long = Int.MaxValue.toLong 127 | val ChunkSize: Int = 64 * 1024 128 | 129 | def fromByteArray[F[_]]( 130 | data: Array[Byte], 131 | mediaType: MediaType = MediaType.application.`octet-stream`, 132 | charset: Option[Charset] = None 133 | ): ObjectContent[F] = 134 | ObjectContent[F]( 135 | data = Stream.chunk(Chunk.from(data)).covary[F], 136 | contentLength = data.length.toLong, 137 | mediaType = mediaType, 138 | charset = charset, 139 | chunked = false 140 | ) 141 | 142 | def fromPath[F[_]: Async](path: Path): F[ObjectContent[F]] = 143 | Sync[F] 144 | .delay(Files.size(path)) 145 | .flatTap { contentLength => 146 | Sync[F] 147 | .raiseError[Long] { 148 | new IllegalArgumentException( 149 | "The file must be smaller than MaxDataLength bytes" 150 | ) 151 | } 152 | .whenA(contentLength > MaxDataLength) 153 | } 154 | .map { contentLength => 155 | ObjectContent( 156 | io.file.readAll[F]( 157 | path, 158 | ChunkSize 159 | ), 160 | contentLength, 161 | chunked = contentLength > ChunkSize 162 | ) 163 | } 164 | } 165 | 166 | } 167 | -------------------------------------------------------------------------------- /modules/s3/src/main/scala/s3/utils/S3UriDomain.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.comms.aws 18 | package s3 19 | package utils 20 | 21 | import com.ovoenergy.comms.aws.common.model.Region 22 | 23 | object S3UriDomain { 24 | 25 | def s3UriDomain(forBucket: String, forRegion: Region) = { 26 | if (forRegion == Region.`us-east-1`) { 27 | s"$forBucket.s3.amazonaws.com" 28 | } else { 29 | s"$forBucket.s3-${forRegion.value}.amazonaws.com" 30 | } 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /modules/s3/src/main/scala/s3/utils/S3UriParser.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.comms.aws 18 | package s3 19 | package utils 20 | 21 | import model._ 22 | import cats._ 23 | import cats.implicits._ 24 | import java.net.URLEncoder 25 | import java.net.URI 26 | import java.util.regex.Pattern 27 | import java.net.URLDecoder 28 | 29 | object S3UriParser { 30 | 31 | def getBucketAndKey[F[_]]( 32 | uri: String 33 | )(implicit ME: MonadError[F, Throwable]): F[(Bucket, Key)] = { 34 | for { 35 | uri <- parseS3Uri[F](uri) 36 | bucketAndKey <- if ("s3".equalsIgnoreCase(uri.getScheme)) { 37 | (bucketFromS3SchemeUri[F](uri), keyFromS3SchemeUri[F](uri)).tupled 38 | } else { 39 | (bucketFromHttpUri[F](uri), keyFromHttpUri[F](uri)).tupled 40 | } 41 | } yield bucketAndKey 42 | } 43 | 44 | def getBucket[F[_]]( 45 | uri: String 46 | )(implicit ME: MonadError[F, Throwable]): F[Bucket] = 47 | for { 48 | uri <- parseS3Uri[F](uri) 49 | bucket <- if ("s3".equalsIgnoreCase(uri.getScheme)) { 50 | bucketFromS3SchemeUri[F](uri) 51 | } else { 52 | bucketFromHttpUri[F](uri) 53 | } 54 | } yield bucket 55 | 56 | def getKey[F[_]]( 57 | uri: String 58 | )(implicit ME: MonadError[F, Throwable]): F[Key] = 59 | for { 60 | uri <- parseS3Uri[F](uri) 61 | key <- if ("s3".equalsIgnoreCase(uri.getScheme)) { 62 | keyFromS3SchemeUri[F](uri) 63 | } else { 64 | keyFromHttpUri[F](uri) 65 | } 66 | } yield key 67 | 68 | private def parseS3Uri[F[_]](uri: String)(implicit ME: MonadError[F, Throwable]): F[URI] = 69 | ME.catchNonFatal { 70 | URI.create( 71 | URLEncoder 72 | .encode(uri, "UTF-8") 73 | .replace("%3A", ":") 74 | .replace("%2F", "/") 75 | .replace("+", "%20") 76 | ) 77 | } 78 | 79 | private def bucketFromS3SchemeUri[F[_]]( 80 | uri: URI 81 | )(implicit ME: MonadError[F, Throwable]): F[Bucket] = 82 | uri.getAuthority 83 | .ensureNotNull[F]("S3 Uri does not have an authority section") 84 | .map(Bucket(_)) 85 | 86 | private def keyFromS3SchemeUri[F[_]]( 87 | uri: URI 88 | )(implicit ME: MonadError[F, Throwable]): F[Key] = 89 | uri.getPath 90 | .ensureNotNull[F]("S3 Uri does not have a key") 91 | .map { keyWithLeadingSlash => 92 | Key(keyWithLeadingSlash.substring(1)) 93 | } 94 | 95 | private def bucketFromHttpUri[F[_]]( 96 | uri: URI 97 | )(implicit ME: MonadError[F, Throwable]): F[Bucket] = 98 | for { 99 | tuple <- getHttpUriPathAndMaybePrefix[F](uri) 100 | (uriPath, maybePrefix) = tuple 101 | bucket <- maybePrefix 102 | .fold { 103 | s3BucketFromPath[F](uriPath) 104 | } { prefix => 105 | ME.pure(prefix.substring(0, prefix.length - 1)) 106 | } 107 | } yield Bucket(bucket) 108 | 109 | private def keyFromHttpUri[F[_]]( 110 | uri: URI 111 | )(implicit ME: MonadError[F, Throwable]): F[Key] = 112 | for { 113 | tuple <- getHttpUriPathAndMaybePrefix[F](uri) 114 | (uriPath, maybePrefix) = tuple 115 | key <- maybePrefix.fold { 116 | s3KeyFromPath[F](uriPath) 117 | } { _ => 118 | if (uriPath.isEmpty || uriPath == "/") 119 | ME.raiseError(new Exception("URI does not have a path section")) 120 | else ME.pure(uriPath.substring(1)) 121 | } 122 | } yield Key(key) 123 | 124 | private def getHttpUriPathAndMaybePrefix[F[_]]( 125 | uri: URI 126 | )(implicit ME: MonadError[F, Throwable]): F[(String, Option[String])] = { 127 | val endpointPattern = Pattern.compile("^(.+\\.)?s3[.-]([a-z0-9-]+)\\.") 128 | for { 129 | host <- uri.getHost.ensureNotNull[F]("URI does not have a host section") 130 | matcher = endpointPattern.matcher(host) 131 | _ <- ME.unit.ensure(new Exception("Cannot parse S3 Uri host section"))(_ => matcher.find()) 132 | uriPath <- uri.getPath.ensureNotNull[F]("URI does not have a path section") 133 | maybePrefix = Option(matcher.group(1)) 134 | .filter(!_.isEmpty) 135 | } yield (uriPath, maybePrefix) 136 | } 137 | 138 | private def s3BucketFromPath[F[_]](path: String)( 139 | implicit ME: MonadError[F, Throwable] 140 | ): F[String] = 141 | if ("" == path || "/" == path) { 142 | ME.raiseError(new Exception("S3 path is empty")) 143 | } else { 144 | val index = path.indexOf('/', 1) 145 | ME.catchNonFatal { 146 | if (index == -1) { // https://s3.amazonaws.com/bucket 147 | URLDecoder.decode(path.substring(1), "UTF-8") 148 | } else if (index == (path.length - 1)) { // https://s3.amazonaws.com/bucket/ 149 | URLDecoder.decode(path.substring(1, index), "UTF-8") 150 | } else { // https://s3.amazonaws.com/bucket/key 151 | URLDecoder.decode(path.substring(1, index), "UTF-8") 152 | } 153 | } 154 | } 155 | 156 | private def s3KeyFromPath[F[_]]( 157 | path: String 158 | )(implicit ME: MonadError[F, Throwable]): F[String] = { 159 | if ("" == path || "/" == path) { 160 | ME.raiseError(new Exception("S3 path is empty")) 161 | } else { 162 | val index = path.indexOf('/', 1) 163 | if (index == -1 || index == (path.length - 1)) { // https://s3.amazonaws.com/bucket 164 | ME.raiseError(new Exception("S3 path does not have a key")) 165 | } else { // https://s3.amazonaws.com/bucket/key 166 | ME.catchNonFatal(URLDecoder.decode(path.substring(index + 1), "UTF-8")) 167 | } 168 | } 169 | } 170 | 171 | private implicit class MonadErrorHelpers[T](t: T) { 172 | def ensureNotNull[F[_]](error: => Throwable)(implicit ME: MonadError[F, Throwable]): F[T] = 173 | ME.pure(t).ensure(error)(_ != null) 174 | def ensureNotNull[F[_]](error: String)(implicit ME: MonadError[F, Throwable]): F[T] = 175 | t.ensureNotNull[F](new Exception(error)) 176 | } 177 | 178 | } 179 | -------------------------------------------------------------------------------- /modules/s3/src/test/scala/s3/utils/S3UriDomainSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.comms.aws 18 | package s3 19 | package utils 20 | 21 | import com.ovoenergy.comms.aws.common.UnitSpec 22 | import com.ovoenergy.comms.aws.common.model.Region 23 | import org.scalatest.Assertion 24 | 25 | class S3UriDomainSpec extends UnitSpec { 26 | 27 | "S3UriDomain" should { 28 | 29 | "use the region in the domain name for us-west-2" in assertUsesRegion(Region.`us-west-2`) 30 | "use the region in the domain name for us-west-1" in assertUsesRegion(Region.`us-west-1`) 31 | "use the region in the domain name for us-east-2" in assertUsesRegion(Region.`us-east-2`) 32 | "use the region in the domain name for ap-south-1" in assertUsesRegion(Region.`ap-south-1`) 33 | "use the region in the domain name for ap-northeast-2" in assertUsesRegion( 34 | Region.`ap-northeast-2` 35 | ) 36 | "use the region in the domain name for ap-southeast-1" in assertUsesRegion( 37 | Region.`ap-southeast-1` 38 | ) 39 | "use the region in the domain name for ap-southeast-2" in assertUsesRegion( 40 | Region.`ap-southeast-2` 41 | ) 42 | "use the region in the domain name for ap-northeast-1" in assertUsesRegion( 43 | Region.`ap-northeast-1` 44 | ) 45 | "use the region in the domain name for ca-central-1" in assertUsesRegion(Region.`ca-central-1`) 46 | "use the region in the domain name for cn-north-1" in assertUsesRegion(Region.`cn-north-1`) 47 | "use the region in the domain name for eu-central-1" in assertUsesRegion(Region.`eu-central-1`) 48 | "use the region in the domain name for eu-west-1" in assertUsesRegion(Region.`eu-west-1`) 49 | "use the region in the domain name for eu-west-2" in assertUsesRegion(Region.`eu-west-2`) 50 | "use the region in the domain name for eu-west-3" in assertUsesRegion(Region.`eu-west-3`) 51 | "use the region in the domain name for sa-east-1" in assertUsesRegion(Region.`sa-east-1`) 52 | "use the region in the domain name for us-gov-west-1" in assertUsesRegion( 53 | Region.`us-gov-west-1` 54 | ) 55 | 56 | "handle us-east-1 region differently" in { 57 | S3UriDomain.s3UriDomain("bucketName", Region.`us-east-1`) shouldBe 58 | "bucketName.s3.amazonaws.com" 59 | } 60 | } 61 | 62 | private def assertUsesRegion(region: Region): Assertion = { 63 | S3UriDomain.s3UriDomain("bucketName", region) shouldBe s"bucketName.s3-${region.value}.amazonaws.com" 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /modules/s3/src/test/scala/s3/utils/S3UriParserSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 OVO Energy 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.ovoenergy.comms.aws 18 | package s3 19 | package utils 20 | 21 | import cats.effect.IO 22 | import cats.effect.testing.scalatest.AsyncIOSpec 23 | import com.ovoenergy.comms.aws.common.UnitSpec 24 | import com.ovoenergy.comms.aws.s3.model._ 25 | 26 | class S3UriParserSpec extends UnitSpec with AsyncIOSpec { 27 | 28 | "S3UriParser" should { 29 | 30 | // Bucket 31 | "parse bucket name from authority" in { 32 | val testCase = "s3://some-bucket/object-key" 33 | S3UriParser.getBucket[IO](testCase).asserting(_ shouldBe Bucket("some-bucket")) 34 | } 35 | 36 | "parse bucket name from authority with no key" in { 37 | val testCase = "s3://some-bucket" 38 | S3UriParser.getBucket[IO](testCase).asserting(_ shouldBe Bucket("some-bucket")) 39 | } 40 | 41 | "return fail if authority empty" in { 42 | val testCase = "s3://" 43 | S3UriParser.getBucket[IO](testCase).attempt.asserting(_ shouldBe a[Left[_, _]]) 44 | } 45 | 46 | "parse bucket name from path" in { 47 | val testCase = "https://s3.amazonaws.com/some-bucket/key" 48 | S3UriParser.getBucket[IO](testCase).asserting(_ shouldBe Bucket("some-bucket")) 49 | } 50 | 51 | "parse bucket name from path with no key" in { 52 | val testCase = "https://s3.amazonaws.com/some-bucket" 53 | S3UriParser.getBucket[IO](testCase).asserting(_ shouldBe Bucket("some-bucket")) 54 | } 55 | 56 | "parse bucket name from path with trailing slash" in { 57 | val testCase = "https://s3.amazonaws.com/some-bucket/" 58 | S3UriParser.getBucket[IO](testCase).asserting(_ shouldBe Bucket("some-bucket")) 59 | } 60 | 61 | // Key 62 | "Parse key from s3 format uri" in { 63 | val testCase = "s3://some-bucket/object-key" 64 | S3UriParser.getKey[IO](testCase).asserting(_ shouldBe Key("object-key")) 65 | } 66 | 67 | "Return None from s3 format uri with no key" in { 68 | val testCase = "s3://some-bucket" 69 | S3UriParser.getKey[IO](testCase).attempt.asserting(_ shouldBe a[Left[_, _]]) 70 | } 71 | 72 | "Parse key from regular format uri" in { 73 | val testCase = "https://s3.amazonaws.com/some-bucket/object-key" 74 | S3UriParser.getKey[IO](testCase).asserting(_ shouldBe Key("object-key")) 75 | } 76 | 77 | "Parse multipart key from uri" in { 78 | val testCase = "https://s3.eu-west-1.amazonaws.com/bucket/key-part-1/key-part-2/key-part-3" 79 | S3UriParser.getKey[IO](testCase).asserting(_ shouldBe Key("key-part-1/key-part-2/key-part-3")) 80 | } 81 | 82 | "Return None from regular format uri with no key" in { 83 | val testCase = "https://s3.amazonaws.com/some-bucket/" 84 | S3UriParser.getKey[IO](testCase).attempt.asserting(_ shouldBe a[Left[_, _]]) 85 | } 86 | 87 | } 88 | 89 | } 90 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.11.2 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.3") 2 | addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.11.0") 3 | addSbtPlugin("com.dwijnand" % "sbt-dynver" % "4.1.1") 4 | addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.10.0") 5 | addSbtPlugin("io.github.davidgregory084" % "sbt-tpolecat" % "0.4.4") 6 | addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.10.0-RC1") 7 | addSbtPlugin("fr.qux" % "sbt-release-tags-only" % "0.5.0") 8 | addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.11.1") 9 | 10 | resolvers += Resolver.sonatypeRepo("releases") 11 | --------------------------------------------------------------------------------