├── .dockerignore ├── .github ├── FUNDING.yml ├── dependabot.yml └── workflows │ ├── codeql-analysis.yml │ ├── dockerhubpublish.yml │ ├── pythonapp.yml │ └── pythonpublish.yml ├── .gitignore ├── .gitpod.yml ├── .python-version ├── .travis.yml ├── .vscode ├── launch.json └── settings.json ├── DEVELOP.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── awscurl ├── __init__.py ├── __main__.py ├── awscurl.py └── utils.py ├── ci ├── ci-alpine │ └── Dockerfile ├── ci-centos │ └── Dockerfile └── ci-ubuntu │ └── Dockerfile ├── docs └── note.txt ├── requirements-test.txt ├── requirements.txt ├── run.sh ├── scripts ├── ci-in-docker.sh ├── ci.sh ├── install.sh └── pypi_publish.sh ├── setup.cfg ├── setup.py └── tests ├── __init__.py ├── basic_test.py ├── data └── credentials ├── integration_test.py ├── load_aws_config_test.py ├── stages_test.py ├── unit_test.py └── url_parsing_test.py /.dockerignore: -------------------------------------------------------------------------------- 1 | build/ 2 | venv/ -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [okigan]# Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 13 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "pip" 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "monthly" 12 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | name: "CodeQL" 7 | 8 | on: 9 | push: 10 | branches: [master] 11 | pull_request: 12 | # The branches below must be a subset of the branches above 13 | branches: [master] 14 | schedule: 15 | - cron: '0 9 * * 6' 16 | 17 | jobs: 18 | analyze: 19 | name: Analyze 20 | runs-on: ubuntu-latest 21 | 22 | strategy: 23 | fail-fast: false 24 | matrix: 25 | # Override automatic language detection by changing the below list 26 | # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] 27 | language: ['python'] 28 | # Learn more... 29 | # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection 30 | 31 | steps: 32 | - name: Checkout repository 33 | uses: actions/checkout@v4 34 | 35 | # Initializes the CodeQL tools for scanning. 36 | - name: Initialize CodeQL 37 | uses: github/codeql-action/init@v3 38 | with: 39 | languages: ${{ matrix.language }} 40 | # If you wish to specify custom queries, you can do so here or in a config file. 41 | # By default, queries listed here will override any specified in a config file. 42 | # Prefix the list here with "+" to use these queries and those in the config file. 43 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 44 | 45 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 46 | # If this step fails, then you should remove it and run the build manually (see below) 47 | - name: Autobuild 48 | uses: github/codeql-action/autobuild@v3 49 | 50 | # ℹ️ Command-line programs to run using the OS shell. 51 | # 📚 https://git.io/JvXDl 52 | 53 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 54 | # and modify them (or add more) to build your code if your project 55 | # uses a compiled language 56 | 57 | #- run: | 58 | # make bootstrap 59 | # make release 60 | 61 | - name: Perform CodeQL Analysis 62 | uses: github/codeql-action/analyze@v3 63 | -------------------------------------------------------------------------------- /.github/workflows/dockerhubpublish.yml: -------------------------------------------------------------------------------- 1 | name: Publish Docker images 2 | 3 | on: 4 | release: 5 | types: [published] 6 | workflow_dispatch: 7 | 8 | jobs: 9 | push_to_registry: 10 | name: Push Docker image to Docker Hub 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Check out the repo 14 | uses: actions/checkout@v4 15 | 16 | - name: Log in to Docker Hub 17 | uses: docker/login-action@v3 18 | with: 19 | username: ${{ secrets.DOCKER_USERNAME }} 20 | password: ${{ secrets.DOCKER_PASSWORD }} 21 | 22 | - name: Login to GitHub Container Registry 23 | uses: docker/login-action@v3 24 | with: 25 | registry: ghcr.io 26 | username: ${{ github.repository_owner }} 27 | password: ${{ secrets.GITHUB_TOKEN }} 28 | 29 | - name: Extract metadata (tags, labels) for Docker 30 | id: meta 31 | uses: docker/metadata-action@v5 32 | with: 33 | images: | 34 | okigan/awscurl 35 | ghcr.io/${{ github.repository }} 36 | 37 | - name: Set up QEMU 38 | uses: docker/setup-qemu-action@v3 39 | 40 | - name: Set up Docker Buildx 41 | uses: docker/setup-buildx-action@v3 42 | 43 | - name: Build and push Docker image 44 | uses: docker/build-push-action@v5 45 | with: 46 | context: . 47 | platforms: linux/amd64,linux/arm64 48 | push: true 49 | tags: ${{ steps.meta.outputs.tags }} 50 | labels: ${{ steps.meta.outputs.labels }} -------------------------------------------------------------------------------- /.github/workflows/pythonapp.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push,pull_request,workflow_dispatch] 4 | 5 | jobs: 6 | build: 7 | strategy: 8 | matrix: 9 | runs-on: [ubuntu-20.04, ubuntu-latest, macOS-latest] 10 | python-version: [ '3.6', '3.8', '3.9', '3.11'] 11 | exclude: 12 | - runs-on: ubuntu-latest 13 | python-version: '3.6' 14 | - runs-on: macOS-latest 15 | python-version: '3.6' 16 | fail-fast: false 17 | 18 | runs-on: ${{ matrix.runs-on }} 19 | 20 | env: 21 | AWS_ACCESS_KEY_ID: MOCK_AWS_ACCESS_KEY_ID 22 | AWS_SECRET_ACCESS_KEY: MOCK_AWS_SECRET_ACCESS_KEY 23 | AWS_SESSION_TOKEN: MOCK_AWS_SESSION_TOKEN 24 | 25 | steps: 26 | - uses: actions/checkout@v4 27 | - name: Set up Python ${{ matrix.python-version }} 28 | uses: actions/setup-python@v5 29 | with: 30 | python-version: ${{ matrix.python-version }} 31 | - name: Install dependencies 32 | run: | 33 | python -m pip install --upgrade pip 34 | pip install -r requirements.txt 35 | pip install -r requirements-test.txt 36 | pip install pytest 37 | # - name: Lint with flake8 38 | # run: | 39 | # pip install flake8 40 | # # stop the build if there are Python syntax errors or undefined names 41 | # flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 42 | # # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 43 | # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 44 | - name: Check with pycodestyle 45 | run: | 46 | pycodestyle -v awscurl 47 | - name: Test with pytest 48 | run: | 49 | python --version 50 | pytest -v --cov=awscurl --cov-fail-under=77 --cov-report html 51 | 52 | -------------------------------------------------------------------------------- /.github/workflows/pythonpublish.yml: -------------------------------------------------------------------------------- 1 | name: Upload Python Package 2 | 3 | on: 4 | release: 5 | types: [created] 6 | workflow_dispatch: 7 | 8 | jobs: 9 | deploy: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v4 13 | - name: Set up Python 14 | uses: actions/setup-python@v5 15 | with: 16 | python-version: '3.x' 17 | - name: Install dependencies 18 | run: | 19 | python -m pip install --upgrade pip 20 | pip install setuptools wheel twine 21 | - name: Build and publish 22 | env: 23 | TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} 24 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 25 | run: | 26 | python setup.py sdist bdist_wheel 27 | twine upload dist/* 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | /build/ 3 | /dist/ 4 | /*.egg-info 5 | /venv/ 6 | /.pytest_cache/v/cache/ 7 | /.coverage 8 | /.tox/ 9 | **/*,cover 10 | /htmlcov/ 11 | /.cache/pip/ 12 | /.local/ 13 | -------------------------------------------------------------------------------- /.gitpod.yml: -------------------------------------------------------------------------------- 1 | # List the start up tasks. Learn more https://www.gitpod.io/docs/config-start-tasks/ 2 | tasks: 3 | - init: echo 'init script' # runs during prebuild 4 | command: echo 'start script' 5 | 6 | # List the ports to expose. Learn more https://www.gitpod.io/docs/config-ports/ 7 | ports: 8 | - port: 3000 9 | onOpen: open-preview 10 | 11 | -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.8.16 2 | 3.6.15 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.6" 4 | - "3.7" 5 | - "3.8" 6 | - "3.9" 7 | # command to install dependencies 8 | install: 9 | - pip install -r requirements.txt 10 | - pip install -r requirements-test.txt 11 | 12 | env: 13 | - AWS_ACCESS_KEY_ID=MOCK_AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=MOCK_AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN=MOCK_AWS_SESSION_TOKEN 14 | 15 | # command to run tests 16 | script: 17 | - pycodestyle -v awscurl 18 | - pytest -v --cov=awscurl --cov-fail-under=77 --cov-report html 19 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "awscurl", 9 | "type": "python", 10 | "request": "launch", 11 | "module": "awscurl", 12 | "console": "integratedTerminal", 13 | "justMyCode": true, 14 | "env": { 15 | "PYTHONPATH": "${workspaceFolder}/awscurl" 16 | }, 17 | "args": ["--service","s3", "https://awscurl-sample-bucket.s3.amazonaws.com?q=a{status=b}" ] 18 | } 19 | ] 20 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.testing.pytestArgs": [], 3 | "python.testing.unittestEnabled": false, 4 | "python.testing.pytestEnabled": true 5 | } 6 | -------------------------------------------------------------------------------- /DEVELOP.md: -------------------------------------------------------------------------------- 1 | ## Some useful commands for local development 2 | ### Build docker image 3 | ```sh 4 | 5 | docker build -t awscurl . 6 | 7 | docker run --rm -ti -v "$HOME/.aws:/root/.aws" -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SECURITY_TOKEN -e AWS_PROFILE -e AWS_REGION awscurl "${api_url_base}/api/rxxxxx" 8 | 9 | docker run -it --entrypoint sh awscurl 10 | ``` 11 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build stage 2 | FROM python:3-alpine AS builder 3 | 4 | RUN set -ex && \ 5 | apk add \ 6 | build-base \ 7 | libffi-dev \ 8 | libxml2-dev \ 9 | openssl-dev 10 | 11 | RUN pip install --user botocore 12 | 13 | COPY . /app-source-dir 14 | 15 | RUN pip install -v --user /app-source-dir 16 | 17 | 18 | # Runtime stage 19 | FROM python:3-alpine 20 | 21 | COPY --from=builder /root/.local /root/.local 22 | 23 | ENV PATH=/root/.local/bin/:${PATH} 24 | 25 | ENTRYPOINT ["awscurl"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright 2015 by the contributors to awscurl 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | venv: 2 | python3 -m venv venv 3 | ( \ 4 | . ./venv/bin/activate; \ 5 | which python; \ 6 | pip install --upgrade pip; \ 7 | pip install --upgrade setuptools; \ 8 | pip install -r requirements.txt -r requirements-test.txt; \ 9 | ) 10 | 11 | docker-build: 12 | docker build -t awscurl . 13 | 14 | docker-run: 15 | docker run -it --rm awscurl 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # awscurl [![Donate](https://img.shields.io/badge/donate-github-orange.svg?style=flat-square)](https://github.com/sponsors/okigan) [![Donate](https://img.shields.io/badge/donate-paypal-orange.svg?style=flat-square)](https://www.paypal.com/donate/?business=UDN4FL55J34QC&amount=25) [![Donate](https://img.shields.io/badge/donate-buy_me_a_coffee-orange.svg?style=flat-square)](https://www.buymeacoffee.com/okigan) 2 | 3 | [![PyPI](https://img.shields.io/pypi/v/awscurl.svg)](https://pypi.python.org/pypi/awscurl) 4 | [![Build Status](https://github.com/okigan/awscurl/actions/workflows/pythonapp.yml/badge.svg)](https://github.com/okigan/awscurl) 5 | [![Docker Hub](https://img.shields.io/docker/pulls/okigan/awscurl.svg)](https://hub.docker.com/r/okigan/awscurl) 6 | ![CI badge](https://github.com/okigan/awscurl/workflows/CI/badge.svg?branch=master) 7 | 8 | [![Edit with gitpod](https://img.shields.io/badge/edit--with-gitpod-blue.svg?style=flat-square)](https://gitpod.io/#https://github.com/okigan/awscurl) 9 | [![Edit with vscode](https://img.shields.io/badge/edit--with-vscode-blue.svg?style=flat-square)](https://vscode.dev/github/okigan/awscurl) 10 | [![Edit with github codespaces](https://img.shields.io/badge/edit--with-codespaces-blue.svg?style=flat-square)](https://github.dev/okigan/awscurl) 11 | 12 | curl-like tool with AWS Signature Version 4 request signing. 13 | 14 | ## Features 15 | 16 | * performs requests to AWS services with request signing using curl interface 17 | * supports IAM profile credentials 18 | 19 | ## Overview 20 | 21 | Requests to AWS API must be signed (see [Signing AWS API Requests](http://docs.aws.amazon.com/general/latest/gr/signing_aws_api_requests.html)) 22 | automates the process of signing and makes requests to AWS as simple as a standard curl command. 23 | 24 | ## Installation 25 | 26 | ```sh 27 | pip install awscurl 28 | ``` 29 | 30 | ### Installation from source (bleeding edge) 31 | 32 | ```sh 33 | pip install git+https://github.com/okigan/awscurl 34 | ``` 35 | 36 | ### Installation via Homebrew for MacOS 37 | 38 | ```sh 39 | brew install awscurl 40 | ``` 41 | 42 | #### Running via Docker 43 | 44 | ```sh 45 | docker pull okigan/awscurl # or via docker pull ghcr.io/okigan/awscurl 46 | ``` 47 | 48 | or via Github docker registry 49 | 50 | ```sh 51 | docker pull ghcr.io/okigan/awscurl 52 | ``` 53 | 54 | then 55 | 56 | ```sh 57 | $ docker run --rm -it okigan/awscurl --access_key ACCESS_KEY --secret_key SECRET_KEY --service s3 s3://... 58 | 59 | # or allow access to local credentials as following 60 | $ docker run --rm -it -v "$HOME/.aws:/root/.aws" okigan/awscurl --service s3 s3://... 61 | ``` 62 | 63 | To shorten the length of docker commands use the following alias: 64 | 65 | ```sh 66 | alias awscurl='docker run --rm -ti -v "$HOME/.aws:/root/.aws" -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SECURITY_TOKEN -e AWS_PROFILE okigan/awscurl' 67 | ``` 68 | 69 | This will allow you to run awscurl from within a Docker container as if it was installed on the host system: 70 | 71 | ```sh 72 | awscurl 73 | ``` 74 | 75 | ## Examples 76 | 77 | * Call S3: List bucket content 78 | 79 | ```sh 80 | $ awscurl --service s3 'https://awscurl-sample-bucket.s3.amazonaws.com' | tidy -xml -iq 81 | 82 | 83 | awscurl-sample-bucket 84 | 85 | 86 | 1000 87 | false 88 | 89 | awscurl-sample-file.txt 90 | 2017-07-25T21:27:38.000Z 91 | "d41d8cd98f00b204e9800998ecf8427e" 92 | 0 93 | STANDARD 94 | 95 | 96 | ``` 97 | 98 | * Call EC2: 99 | 100 | ```sh 101 | $ awscurl --service ec2 'https://ec2.amazonaws.com?Action=DescribeRegions&Version=2013-10-15' | tidy -xml -iq 102 | 103 | 104 | 105 | 96511ccd-2d6d-4d63-ad9b-6be6f2c9874d 106 | 107 | 108 | eu-north-1 109 | ec2.eu-north-1.amazonaws.com 110 | 111 | 112 | ap-south-1 113 | ec2.ap-south-1.amazonaws.com 114 | 115 | 116 | 117 | ``` 118 | 119 | * Call API Gateway: 120 | 121 | ```sh 122 | $ awscurl --service execute-api -X POST -d @request.json \ 123 | https://.execute-api.us-east-1.amazonaws.com/ 124 | ``` 125 | 126 | ## Options 127 | 128 | ```sh 129 | usage: __main__.py [-h] [-v] [-i] [-X REQUEST] [-d DATA] [-H HEADER] [-k] [--fail-with-body] [--data-binary] [--region REGION] [--profile PROFILE] [--service SERVICE] 130 | [--access_key ACCESS_KEY] [--secret_key SECRET_KEY] [--security_token SECURITY_TOKEN] [--session_token SESSION_TOKEN] [-L] [-o ] 131 | uri 132 | 133 | Curl AWS request signing 134 | 135 | positional arguments: 136 | uri 137 | 138 | options: 139 | -h, --help show this help message and exit 140 | -v, --verbose verbose flag (default: False) 141 | -i, --include include headers in the output (default: False) 142 | -X REQUEST, --request REQUEST 143 | Specify request command to use (default: GET) 144 | -d DATA, --data DATA HTTP POST data (default: ) 145 | -H HEADER, --header HEADER 146 | HTTP header (default: None) 147 | -k, --insecure Allow insecure server connections when using SSL (default: False) 148 | --fail-with-body Fail on HTTP errors but save the body (default: False) 149 | --data-binary Process HTTP POST data exactly as specified with no extra processing whatsoever. (default: False) 150 | --region REGION AWS region [env var: AWS_DEFAULT_REGION] (default: us-east-1) 151 | --profile PROFILE AWS profile [env var: AWS_PROFILE] (default: default) 152 | --service SERVICE AWS service (default: execute-api) 153 | --access_key ACCESS_KEY 154 | [env var: AWS_ACCESS_KEY_ID] (default: None) 155 | --secret_key SECRET_KEY 156 | [env var: AWS_SECRET_ACCESS_KEY] (default: None) 157 | --security_token SECURITY_TOKEN 158 | [env var: AWS_SECURITY_TOKEN] (default: None) 159 | --session_token SESSION_TOKEN 160 | [env var: AWS_SESSION_TOKEN] (default: None) 161 | -L, --location Follow redirects (default: False) 162 | -o , --output 163 | Write to file instead of stdout (default: ) 164 | 165 | In general, command-line values override environment variables which override defaults. 166 | 167 | ``` 168 | 169 | If you do not specify the `--access_key` or `--secret_key` 170 | (or environment variables), `awscurl` will attempt to use 171 | the credentials you set in `~/.aws/credentials`. If you 172 | do not specify a `--profile` or `AWS_PROFILE`, `awscurl` 173 | uses `default`. 174 | 175 | ## Who uses awscurl 176 | 177 | * [AWS Documentation](https://docs.aws.amazon.com/apigateway/latest/developerguide/apigateway-how-to-call-websocket-api-connections.html) 178 | * [Onica blog](https://onica.com/blog/how-to/how-to-kibana-default-index-pattern/) 179 | * QnA on [StackOverflow](https://stackoverflow.com/search?q=awscurl) 180 | * QnA on [DevOps StackExchange](https://devops.stackexchange.com/search?q=awscurl) 181 | * Examples on [Golfbert](https://golfbert.com/api/samples) 182 | 183 | ## Star History 184 | 185 | [![Star History Chart](https://api.star-history.com/svg?repos=okigan/awscurl)](https://star-history.com/#okigan/awscurl&Date) 186 | 187 | ## Related projects 188 | 189 | * awscurl in Go: 190 | * 191 | * 192 | * awscurl in Lisp: 193 | * awscurl on DockerHub: 194 | * [aws-signature-proxy](https://github.com/sverch/aws-signature-proxy) and related [blog post](https://shaunverch.com/butter/open-source/2019/09/27/butter-days-6.html) 195 | * [aws-sigv4-proxy](https://github.com/awslabs/aws-sigv4-proxy) on awslabs 196 | 197 | ## Last but not least 198 | 199 | * [Sponsor awscurl](https://github.com/sponsors/okigan) 200 | -------------------------------------------------------------------------------- /awscurl/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okigan/awscurl/2a6547300ef5592178159211eb541a1a6e08b683/awscurl/__init__.py -------------------------------------------------------------------------------- /awscurl/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """The main entry point. Invoke as `awscurl' or `python -m awscurl'. 3 | 4 | """ 5 | import sys 6 | from .awscurl import main 7 | 8 | 9 | if __name__ == '__main__': 10 | sys.exit(main()) 11 | -------------------------------------------------------------------------------- /awscurl/awscurl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Awscurl implementation 4 | """ 5 | from __future__ import print_function 6 | 7 | import datetime 8 | import hashlib 9 | import hmac 10 | import os 11 | import pprint 12 | import sys 13 | import re 14 | 15 | from typing import Dict 16 | import urllib 17 | from urllib.parse import quote 18 | 19 | import configparser 20 | import configargparse 21 | import requests 22 | from requests.structures import CaseInsensitiveDict 23 | 24 | 25 | from .utils import sha256_hash, sha256_hash_for_binary_data, sign 26 | 27 | __author__ = 'iokulist' 28 | 29 | IS_VERBOSE = False 30 | 31 | 32 | def __log(*args, **kwargs): 33 | if not IS_VERBOSE: 34 | return 35 | stderr_pp = pprint.PrettyPrinter(stream=sys.stderr) 36 | stderr_pp.pprint(*args, **kwargs) 37 | 38 | 39 | def url_path_to_dict(path): 40 | """http://stackoverflow.com/a/17892757/142207""" 41 | 42 | pattern = (r'^' 43 | r'((?P.+?)://)?' 44 | r'((?P[^/]+?)(:(?P[^/]*?))?@)?' 45 | r'(?P.*?)' 46 | r'(:(?P\d+?))?' 47 | r'(?P/.*?)?' 48 | r'(\?(?P.*?))?' 49 | r'$') 50 | regex = re.compile(pattern) 51 | url_match = regex.match(path) 52 | url_dict = url_match.groupdict() if url_match is not None else None 53 | 54 | if url_dict['path'] is None: 55 | url_dict['path'] = '/' 56 | 57 | if url_dict['query'] is None: 58 | url_dict['query'] = '' 59 | 60 | return url_dict 61 | 62 | 63 | # pylint: disable=too-many-arguments,too-many-locals 64 | def make_request(method, 65 | service, 66 | region, 67 | uri, 68 | headers, 69 | data, 70 | access_key, 71 | secret_key, 72 | security_token, 73 | data_binary, 74 | verify=True, 75 | allow_redirects=False): 76 | """ 77 | # Make HTTP request with AWS Version 4 signing 78 | 79 | :return: http request object 80 | :param method: str 81 | :param service: str 82 | :param region: str 83 | :param uri: str 84 | :param headers: dict 85 | :param data: str 86 | :param access_key: str 87 | :param secret_key: str 88 | :param security_token: str 89 | :param data_binary: bool 90 | :param verify: bool 91 | :param allow_redirects: false 92 | 93 | See also: http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html 94 | """ 95 | 96 | uri_dict = url_path_to_dict(uri) 97 | host = uri_dict['host'] 98 | query = uri_dict['query'] 99 | canonical_uri = uri_dict['path'] 100 | port = uri_dict['port'] 101 | 102 | # Create a date for headers and the credential string 103 | current_time = __now() 104 | amzdate = current_time.strftime('%Y%m%dT%H%M%SZ') 105 | datestamp = current_time.strftime('%Y%m%d') # Date w/o time, used in credential scope 106 | 107 | canonical_request, payload_hash, signed_headers = task_1_create_a_canonical_request( 108 | query, 109 | headers, 110 | port, 111 | host, 112 | amzdate, 113 | method, 114 | data, 115 | security_token, 116 | data_binary, 117 | canonical_uri) 118 | string_to_sign, algorithm, credential_scope = task_2_create_the_string_to_sign( 119 | amzdate, 120 | datestamp, 121 | canonical_request, 122 | service, 123 | region) 124 | signature = task_3_calculate_the_signature( 125 | datestamp, 126 | string_to_sign, 127 | service, 128 | region, 129 | secret_key) 130 | auth_headers = task_4_build_auth_headers_for_the_request( 131 | amzdate, 132 | payload_hash, 133 | algorithm, 134 | credential_scope, 135 | signed_headers, 136 | signature, 137 | access_key, 138 | security_token) 139 | headers.update(auth_headers) 140 | 141 | if data_binary: 142 | return __send_request(uri, data, headers, method, verify, allow_redirects) 143 | else: 144 | return __send_request(uri, data.encode('utf-8'), headers, method, verify, allow_redirects) 145 | 146 | 147 | def remove_default_port(parsed_url): 148 | default_ports = {'http': 80, 'https': 443} 149 | if any(parsed_url.scheme == scheme and parsed_url.port == port 150 | for scheme, port in default_ports.items()): 151 | host = parsed_url.hostname 152 | else: 153 | host = parsed_url.netloc 154 | return host 155 | 156 | 157 | # pylint: disable=too-many-arguments,too-many-locals 158 | def task_1_create_a_canonical_request( 159 | query, 160 | headers: Dict, 161 | port, 162 | host, 163 | amzdate, 164 | method, 165 | data, 166 | security_token, 167 | data_binary, 168 | canonical_uri): 169 | """ 170 | ************* TASK 1: CREATE A CANONICAL REQUEST ************* 171 | http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html 172 | 173 | Step 1 is to define the verb (GET, POST, etc.)--already done. 174 | 175 | Step 2: Create canonical URI--the part of the URI from domain to query 176 | string (use '/' if no path) 177 | canonical_uri = '/' 178 | 179 | Step 3: Create the canonical query string. In this example (a GET 180 | request), 181 | request parameters are in the query string. Query string values must 182 | be URL-encoded (space=%20). The parameters must be sorted by name. 183 | For this example, the query string is pre-formatted in the 184 | request_parameters variable. 185 | """ 186 | canonical_querystring = __normalize_query_string(query) 187 | __log(canonical_querystring) 188 | 189 | # If the host was specified in the HTTP header, ensure that the canonical 190 | # headers are set accordingly 191 | headers = requests.structures.CaseInsensitiveDict(headers) 192 | if 'host' in headers: 193 | fullhost = headers['host'] 194 | else: 195 | fullhost = host + ':' + port if port else host 196 | 197 | fullhost = remove_default_port(urllib.parse.urlparse('//' + fullhost)) 198 | 199 | # Step 4: Create the canonical headers and signed headers. Header names 200 | # and value must be trimmed and lowercase, and sorted in ASCII order. 201 | # Note that there is a trailing \n. 202 | canonical_headers_dict = {'host': fullhost.lower(), 203 | 'x-amz-date': amzdate} 204 | 205 | if security_token: 206 | canonical_headers_dict['x-amz-security-token'] = security_token 207 | 208 | # Step 5: Create the list of signed headers. This lists the headers 209 | # in the canonical_headers list, delimited with ";" and in alpha order. 210 | # Note: The request can include any headers; canonical_headers and 211 | # signed_headers lists those that you want to be included in the 212 | # hash of the request. "Host" and "x-amz-date" are always required. 213 | # already tracked in canonical_headers_dict 214 | 215 | # Step 5.5: Add custom signed headers into the canonical_headers and signed_headers lists. 216 | # Header names must be lowercase, values trimmed, and sorted in ASCII order. 217 | for header, value in sorted(headers.items()): 218 | if "x-amz-" in header.lower(): 219 | canonical_headers_dict[header.lower()] = value.strip() 220 | 221 | sorted_canonical_headers_items = sorted(canonical_headers_dict.items()) 222 | canonical_headers = ''.join(['%s:%s\n' % (k, v) for k, v in sorted_canonical_headers_items]) 223 | signed_headers = ';'.join(k for k, v in sorted_canonical_headers_items) 224 | 225 | # Step 6: Create payload hash (hash of the request body content). For GET 226 | # requests, the payload is an empty string (""). 227 | payload_hash = sha256_hash_for_binary_data(data) if data_binary else sha256_hash(data) 228 | 229 | # Step 7: Combine elements to create create canonical request 230 | canonical_request = (method + '\n' + 231 | requests.utils.quote(canonical_uri) + '\n' + 232 | canonical_querystring + '\n' + 233 | canonical_headers + '\n' + 234 | signed_headers + '\n' + 235 | payload_hash) 236 | 237 | __log('\nCANONICAL REQUEST = ' + canonical_request) 238 | return canonical_request, payload_hash, signed_headers 239 | 240 | 241 | def task_2_create_the_string_to_sign( 242 | amzdate, 243 | datestamp, 244 | canonical_request, 245 | service, 246 | region): 247 | """ 248 | ************* TASK 2: CREATE THE STRING TO SIGN************* 249 | Match the algorithm to the hashing algorithm you use, either SHA-1 or 250 | SHA-256 (recommended) 251 | """ 252 | algorithm = 'AWS4-HMAC-SHA256' 253 | credential_scope = (datestamp + '/' + 254 | region + '/' + 255 | service + '/' + 256 | 'aws4_request') 257 | string_to_sign = (algorithm + '\n' + 258 | amzdate + '\n' + 259 | credential_scope + '\n' + 260 | sha256_hash(canonical_request)) 261 | 262 | __log('\nSTRING_TO_SIGN = ' + string_to_sign) 263 | return string_to_sign, algorithm, credential_scope 264 | 265 | 266 | def task_3_calculate_the_signature( 267 | datestamp, 268 | string_to_sign, 269 | service, 270 | region, 271 | secret_key): 272 | """ 273 | ************* TASK 3: CALCULATE THE SIGNATURE ************* 274 | """ 275 | 276 | def get_signature_key(key, date_stamp, region_name, service_name): 277 | """ 278 | See: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html 279 | 280 | In AWS Signature Version 4, instead of using your AWS access keys to sign a request, you 281 | first create a signing key that is scoped to a specific region and service. For more 282 | information about signing keys, see Introduction to Signing Requests. 283 | """ 284 | k_date = sign(('AWS4' + key).encode('utf-8'), date_stamp) 285 | k_region = sign(k_date, region_name) 286 | k_service = sign(k_region, service_name) 287 | k_signing = sign(k_service, 'aws4_request') 288 | return k_signing 289 | 290 | # Create the signing key using the function defined above. 291 | signing_key = get_signature_key(secret_key, datestamp, region, service) 292 | 293 | # Sign the string_to_sign using the signing_key 294 | encoded = string_to_sign.encode('utf-8') 295 | signature = hmac.new(signing_key, encoded, hashlib.sha256).hexdigest() 296 | return signature 297 | 298 | 299 | def task_4_build_auth_headers_for_the_request( 300 | amzdate, 301 | payload_hash, 302 | algorithm, 303 | credential_scope, 304 | signed_headers, 305 | signature, 306 | access_key, 307 | security_token): 308 | """ 309 | ************* TASK 4: ADD SIGNING INFORMATION TO THE REQUEST *********** 310 | The signing information can be either in a query string value or in a header 311 | named Authorization. This function shows how to use the header. It returns 312 | a headers dict with all the necessary signing headers. 313 | """ 314 | # Create authorization header and add to request headers 315 | authorization_header = ( 316 | algorithm + ' ' + 317 | 'Credential=' + access_key + '/' + credential_scope + ', ' + 318 | 'SignedHeaders=' + signed_headers + ', ' + 319 | 'Signature=' + signature 320 | ) 321 | 322 | # The request can include any headers, but MUST include "host", 323 | # "x-amz-date", and (for this scenario) "Authorization". "host" and 324 | # "x-amz-date" must be included in the canonical_headers and 325 | # signed_headers, as noted earlier. Order here is not significant. 326 | # Python note: The 'host' header is added automatically by the Python 327 | # 'requests' library. 328 | headers = { 329 | 'Authorization': authorization_header, 330 | 'x-amz-date': amzdate, 331 | 'x-amz-content-sha256': payload_hash 332 | } 333 | if security_token is not None: 334 | headers['x-amz-security-token'] = security_token 335 | return headers 336 | 337 | 338 | def __normalize_query_string(query): 339 | parameter_pairs = (list(map(str.strip, s.split("="))) 340 | for s in query.split('&') 341 | if len(s) > 0) 342 | 343 | normalized = '&'.join('%s=%s' % (aws_url_encode(p[0]), aws_url_encode(p[1]) if len(p) > 1 else '') 344 | for p in sorted(parameter_pairs)) 345 | return normalized 346 | 347 | 348 | def aws_url_encode(text): 349 | """ 350 | URI-encode each parameter name and value according to the following rules: 351 | - Do not URI-encode any of the unreserved characters that RFC 3986 defines: A-Z, a-z, 0-9, hyphen (-), 352 | underscore (_), period (.), and tilde (~). 353 | - Percent-encode all other characters with %XY, where X and Y are hexadecimal characters (0-9 and uppercase A-F). 354 | For example, the space character must be encoded as %20 (not using '+', as some encoding schemes do) and 355 | extended UTF-8 characters must be in the form %XY%ZA%BC. 356 | - Double-encode any equals (=) characters in parameter values. 357 | """ 358 | return quote(text, safe='~=').replace('=', '==') 359 | 360 | 361 | def __now(): 362 | return datetime.datetime.utcnow() 363 | 364 | 365 | def __send_request(uri, data, headers, method, verify, allow_redirects): 366 | __log('\nHEADERS++++++++++++++++++++++++++++++++++++') 367 | __log(headers) 368 | 369 | __log('\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++') 370 | __log('Request URL = ' + uri) 371 | 372 | if (verify is False): 373 | import urllib3 374 | urllib3.disable_warnings() 375 | 376 | response = requests.request(method, uri, headers=headers, data=data, verify=verify, allow_redirects=allow_redirects) 377 | 378 | __log('\nRESPONSE++++++++++++++++++++++++++++++++++++') 379 | __log('Response code: %d\n' % response.status_code) 380 | 381 | return response 382 | 383 | 384 | # pylint: disable=too-many-branches 385 | def load_aws_config(access_key, secret_key, security_token, credentials_path, profile): 386 | # type: (str, str, str, str, str) -> Tuple[str, str, str] 387 | """ 388 | Load aws credential configuration, by parsing credential file, then try to fall back to 389 | botocore, by checking (access_key,secret_key) are not (None,None) 390 | """ 391 | if access_key is None or secret_key is None: 392 | try: 393 | exists = os.path.exists(credentials_path) 394 | __log('Credentials file \'{0}\' exists \'{1}\''.format(credentials_path, exists)) 395 | 396 | config = configparser.ConfigParser() 397 | config.read(credentials_path) 398 | 399 | while True: 400 | if access_key is None and config.has_option(profile, "aws_access_key_id"): 401 | access_key = config.get(profile, "aws_access_key_id") 402 | else: 403 | break 404 | 405 | if secret_key is None and config.has_option(profile, "aws_secret_access_key"): 406 | secret_key = config.get(profile, "aws_secret_access_key") 407 | else: 408 | break 409 | 410 | if security_token is None and config.has_option(profile, "aws_session_token"): 411 | security_token = config.get(profile, "aws_session_token") 412 | 413 | break 414 | 415 | except configparser.NoSectionError as exception: 416 | __log('AWS profile \'{0}\' not found'.format(exception.args)) 417 | raise exception 418 | except configparser.NoOptionError as exception: 419 | __log('AWS profile \'{0}\' is missing \'{1}\''.format(profile, exception.args)) 420 | raise exception 421 | except ValueError as exception: 422 | __log(exception) 423 | raise exception 424 | 425 | # try to load instance credentials using botocore 426 | if access_key is None or secret_key is None: 427 | try: 428 | __log("loading botocore package") 429 | import botocore 430 | except ImportError: 431 | __log("botocore package could not be loaded") 432 | botocore = None 433 | 434 | if botocore: 435 | import botocore.session 436 | session = botocore.session.get_session() 437 | cred = session.get_credentials() 438 | access_key, secret_key, security_token = cred.access_key, cred.secret_key, cred.token 439 | 440 | return access_key, secret_key, security_token 441 | 442 | 443 | def normalize_args(args): 444 | if args.access_key == "": 445 | args.access_key = None 446 | if args.secret_key == "": 447 | args.secret_key = None 448 | if args.security_token == "": 449 | args.security_token = None 450 | if args.session_token == "": 451 | args.session_token = None 452 | 453 | 454 | def inner_main(argv): 455 | """ 456 | Awscurl CLI main entry point 457 | """ 458 | # note EC2 ignores Accept header and responds in xml 459 | default_headers = ['Accept: application/xml', 460 | 'Content-Type: application/json'] 461 | 462 | parser = configargparse.ArgumentParser( 463 | description='Curl AWS request signing', 464 | formatter_class=configargparse.ArgumentDefaultsHelpFormatter 465 | ) 466 | 467 | parser.add_argument('-v', '--verbose', action='store_true', 468 | help='verbose flag', default=False) 469 | parser.add_argument('-i', '--include', action='store_true', 470 | help='include headers in the output', default=False) 471 | parser.add_argument('-X', '--request', 472 | help='Specify request command to use', 473 | default='GET') 474 | parser.add_argument('-d', '--data', help='HTTP POST data', default='') 475 | parser.add_argument('-H', '--header', help='HTTP header', action='append') 476 | parser.add_argument('-k', '--insecure', action='store_true', default=False, 477 | help='Allow insecure server connections when using SSL') 478 | parser.add_argument('--fail-with-body', action='store_true', help='Fail on HTTP errors but save the body', default=False) 479 | 480 | parser.add_argument('--data-binary', action='store_true', 481 | help='Process HTTP POST data exactly as specified with ' 482 | 'no extra processing whatsoever.', default=False) 483 | 484 | parser.add_argument('--region', help='AWS region', default='us-east-1', 485 | env_var='AWS_DEFAULT_REGION') 486 | parser.add_argument('--profile', help='AWS profile', default='default', env_var='AWS_PROFILE') 487 | parser.add_argument('--service', help='AWS service', default='execute-api') 488 | parser.add_argument('--access_key', env_var='AWS_ACCESS_KEY_ID') 489 | parser.add_argument('--secret_key', env_var='AWS_SECRET_ACCESS_KEY') 490 | # AWS_SECURITY_TOKEN is deprecated, but kept for backward compatibility 491 | # https://github.com/boto/botocore/blob/c76553d3158b083d818f88c898d8f6d7918478fd/botocore/credentials.py#L260-262 492 | parser.add_argument('--security_token', env_var='AWS_SECURITY_TOKEN') 493 | parser.add_argument('--session_token', env_var='AWS_SESSION_TOKEN') 494 | parser.add_argument('-L', '--location', action='store_true', default=False, 495 | help="Follow redirects") 496 | parser.add_argument('-o', '--output', metavar="", help='Write to file instead of stdout', default='') 497 | 498 | parser.add_argument('uri') 499 | 500 | args = parser.parse_args(argv) 501 | normalize_args(args) 502 | # pylint: disable=global-statement 503 | global IS_VERBOSE 504 | IS_VERBOSE = args.verbose 505 | 506 | if args.verbose: 507 | __log(vars(args)) 508 | 509 | data = args.data 510 | 511 | if data is not None and data.startswith("@"): 512 | filename = data[1:] 513 | read_mode = "rb" if args.data_binary else "r" 514 | with open(filename, read_mode) as post_data_file: 515 | data = post_data_file.read() 516 | 517 | if args.header is None: 518 | args.header = default_headers 519 | 520 | if args.security_token is not None: 521 | args.session_token = args.security_token 522 | del args.security_token 523 | 524 | # pylint: disable=deprecated-lambda 525 | headers = {k: v for (k, v) in map(lambda s: s.split(": "), args.header)} 526 | headers = CaseInsensitiveDict(headers) 527 | 528 | credentials_path = os.path.expanduser("~") + "/.aws/credentials" 529 | args.access_key, args.secret_key, args.session_token = load_aws_config(args.access_key, 530 | args.secret_key, 531 | args.session_token, 532 | credentials_path, 533 | args.profile) 534 | 535 | if args.access_key is None: 536 | raise ValueError('No access key is available') 537 | 538 | if args.secret_key is None: 539 | raise ValueError('No secret key is available') 540 | 541 | response = make_request(args.request, 542 | args.service, 543 | args.region, 544 | args.uri, 545 | headers, 546 | data, 547 | args.access_key, 548 | args.secret_key, 549 | args.session_token, 550 | args.data_binary, 551 | verify=not args.insecure, 552 | allow_redirects=args.location) 553 | 554 | if args.include: 555 | print(response.headers, end='\n\n') 556 | elif IS_VERBOSE: 557 | pprint.PrettyPrinter(stream=sys.stderr).pprint(response.headers) 558 | pprint.PrettyPrinter(stream=sys.stderr).pprint('') 559 | 560 | print(response.text) 561 | 562 | if args.output: 563 | filename = args.output 564 | if args.data_binary: 565 | with open(filename, "wb") as f: 566 | f.write(response.content) 567 | else: 568 | with open(filename, "w") as f: 569 | f.write(response.text) 570 | 571 | exit_code = 0 if response.ok or not args.fail_with_body else 22 572 | 573 | return exit_code 574 | 575 | 576 | def main(): 577 | return inner_main(sys.argv[1:]) 578 | 579 | 580 | if __name__ == '__main__': 581 | sys.exit(main()) 582 | -------------------------------------------------------------------------------- /awscurl/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utilities needed during the signing process 3 | """ 4 | 5 | import hashlib 6 | import hmac 7 | 8 | 9 | def sha256_hash(val): 10 | """ 11 | Sha256 hash of text data. 12 | """ 13 | return hashlib.sha256(val.encode('utf-8')).hexdigest() 14 | 15 | 16 | def sha256_hash_for_binary_data(val): 17 | """ 18 | Sha256 hash of binary data. 19 | """ 20 | return hashlib.sha256(val).hexdigest() 21 | 22 | 23 | def sign(key, msg): 24 | """ 25 | Key derivation functions. 26 | See: http://docs.aws.amazon.com 27 | /general/latest/gr/signature-v4-examples.html 28 | #signature-v4-examples-python 29 | """ 30 | return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest() 31 | -------------------------------------------------------------------------------- /ci/ci-alpine/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | 3 | # RUN echo 'tzdata tzdata/Areas select Europe' | debconf-set-selections && \ 4 | # echo 'tzdata tzdata/Zones/Europe select Paris' | debconf-set-selections && \ 5 | # apk add --no-cache tzdata && \ 6 | # cp /usr/share/zoneinfo/Europe/Paris /etc/localtime && \ 7 | # echo "Europe/Paris" > /etc/timezone && \ 8 | # apk del tzdata 9 | 10 | RUN apk update && \ 11 | apk add sudo curl git build-base autoconf automake libtool \ 12 | openssl-dev readline-dev zlib-dev sqlite-dev ncurses-dev \ 13 | xz-dev tk-dev libffi-dev bzip2-dev bash gcc make musl-dev \ 14 | libffi-dev openssl-dev zlib-dev readline-dev sqlite-dev 15 | 16 | RUN curl https://pyenv.run | bash 17 | 18 | ENV PATH="/root/.pyenv/bin:$PATH" 19 | 20 | RUN eval "$(pyenv init -)" && \ 21 | eval "$(pyenv virtualenv-init -)" 22 | 23 | 24 | WORKDIR /root/workdir 25 | COPY .python-version .python-version 26 | RUN for version in $(cat .python-version); do \ 27 | /root/.pyenv/bin/pyenv install $version; \ 28 | done 29 | 30 | COPY setup.cfg setup.cfg 31 | COPY awscurl awscurl 32 | COPY setup.py setup.py 33 | COPY scripts/ci.sh scripts/ci.sh 34 | COPY requirements.txt requirements.txt 35 | COPY requirements-test.txt requirements-test.txt 36 | COPY tests tests 37 | 38 | # RUN bash -c "source /root/venv/bin/activate && cd dd && tox --recreate" 39 | -------------------------------------------------------------------------------- /ci/ci-centos/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM tgagor/centos 2 | 3 | RUN yum update -y 4 | RUN yum group install -y "Development Tools" 5 | RUN yum install -y libffi-devel readline-devel zlib-devel bzip2-devel sqlite-devel openssl-devel git 6 | 7 | 8 | RUN curl https://pyenv.run | bash 9 | 10 | ENV PATH="/root/.pyenv/bin:$PATH" 11 | RUN eval "$(pyenv init -)" && \ 12 | eval "$(pyenv virtualenv-init -)" 13 | 14 | 15 | WORKDIR /root/workdir 16 | COPY .python-version .python-version 17 | RUN for version in $(cat .python-version); do \ 18 | /root/.pyenv/bin/pyenv install $version; \ 19 | done 20 | 21 | COPY setup.cfg setup.cfg 22 | COPY awscurl awscurl 23 | COPY setup.py setup.py 24 | COPY scripts/ci.sh scripts/ci.sh 25 | COPY requirements.txt requirements.txt 26 | COPY requirements-test.txt requirements-test.txt 27 | COPY tests tests 28 | 29 | # RUN bash -c "source /root/venv/bin/activate && cd dd && tox --recreate" 30 | -------------------------------------------------------------------------------- /ci/ci-ubuntu/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu 2 | 3 | RUN apt update 4 | RUN apt install -y sudo 5 | RUN echo 'tzdata tzdata/Areas select Europe' | debconf-set-selections 6 | RUN echo 'tzdata tzdata/Zones/Europe select Paris' | debconf-set-selections 7 | RUN DEBIAN_FRONTEND="noninteractive" apt install -y tzdata 8 | 9 | RUN apt install -y curl git \ 10 | build-essential \ 11 | autoconf \ 12 | automake \ 13 | libtool \ 14 | libffi-dev libreadline-dev libz-dev libsqlite-dev libssl-dev \ 15 | libreadline-dev libsqlite3-dev wget curl libncurses5-dev libncursesw5-dev \ 16 | xz-utils tk-dev libffi-dev libbz2-dev liblzma-dev git 17 | 18 | RUN curl https://pyenv.run | bash 19 | 20 | ENV PATH="/root/.pyenv/bin:$PATH" 21 | RUN eval "$(pyenv init -)" && \ 22 | eval "$(pyenv virtualenv-init -)" 23 | 24 | 25 | WORKDIR /root/workdir 26 | COPY .python-version .python-version 27 | RUN for version in $(cat .python-version); do \ 28 | /root/.pyenv/bin/pyenv install $version; \ 29 | done 30 | 31 | COPY setup.cfg setup.cfg 32 | COPY awscurl awscurl 33 | COPY setup.py setup.py 34 | COPY scripts/ci.sh scripts/ci.sh 35 | COPY requirements.txt requirements.txt 36 | COPY requirements-test.txt requirements-test.txt 37 | COPY tests tests 38 | 39 | # RUN bash -c "source /root/venv/bin/activate && cd dd && tox --recreate" 40 | -------------------------------------------------------------------------------- /docs/note.txt: -------------------------------------------------------------------------------- 1 | Pypi 2 | pip install twine 3 | python setup.py sdist 4 | twine upload dist/* 5 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | pycodestyle 2 | mock 3 | pytest 4 | pytest-cov 5 | wheel 6 | setuptools 7 | setuptools-rust 8 | build -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | configargparse 3 | configparser 4 | botocore 5 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | set -o errexit 3 | set -o pipefail 4 | set -o nounset 5 | set -o xtrace 6 | 7 | # this script is used to run awscurl from local source -- this would be removed once python modules issue is resolved 8 | 9 | python -m awscurl $@ 10 | -------------------------------------------------------------------------------- /scripts/ci-in-docker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o pipefail 5 | set -o nounset 6 | set -o xtrace 7 | 8 | docker_run() { 9 | local image_type=$1 10 | local script_file=$2 11 | echo "building ${image_type} image -- first time it could take a few minutes" 12 | docker build -t "awscurl-ci-${image_type}" -f "./ci/ci-${image_type}/Dockerfile" . && 13 | docker run --rm -t "awscurl-ci-${image_type}" bash -c "${script_file}" 14 | } 15 | 16 | docker_run "ubuntu" "./scripts/ci.sh" 17 | docker_run "alpine" "./scripts/ci.sh" 18 | docker_run "centos" "./scripts/ci.sh" 19 | -------------------------------------------------------------------------------- /scripts/ci.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DETOX_ROOT_DIR=./build/detox 4 | 5 | grep -v '^ *#' < .python-version | while IFS= read -r PYENV_VERSION 6 | do 7 | # echo PYENV_VERSION="$PYENV_VERSION" 8 | # echo SHELL="$SHELL" 9 | # echo $PATH 10 | # pyenv install -sv "${PYENV_VERSION}" 11 | # https://github.com/pyenv/pyenv/issues/1819#issuecomment-780803524 12 | # /root/.pyenv/bin/pyenv shell "${PYENV_VERSION}" 13 | export PYENV_VERSION="$PYENV_VERSION" 14 | eval "$(pyenv init -)" 15 | 16 | PER_VER_DIR=${DETOX_ROOT_DIR}/v${PYENV_VERSION} 17 | VENV_DIR=${PER_VER_DIR}/venv${PYENV_VERSION} 18 | ( 19 | echo "##### NEW DETOX ENV: " "$(uname) " "${PER_VER_DIR}" " #####" 20 | python3 -m venv "${VENV_DIR}" 21 | source "${VENV_DIR}"/bin/activate 22 | 23 | echo which python="$(which python)" 24 | echo python --version="$(python --version)" 25 | echo pip --version="$(pip --version)" 26 | 27 | PS4='[$(date "+%Y-%m-%d %H:%M:%S")] ' 28 | set -o errexit -o pipefail -o nounset -o xtrace 29 | 30 | pip -q -q install --upgrade pip 31 | # python -m ensurepip --upgrade 32 | # pip install -r requirements.txt 33 | pip -q -q install -r requirements-test.txt 34 | 35 | pycodestyle . 36 | 37 | # python -m build . 38 | pip -q install . 39 | 40 | export AWS_ACCESS_KEY_ID=MOCK_AWS_ACCESS_KEY_ID 41 | export AWS_SECRET_ACCESS_KEY=MOCK_AWS_SECRET_ACCESS_KEY 42 | export AWS_SESSION_TOKEN=MOCK_AWS_SESSION_TOKEN 43 | 44 | pytest \ 45 | --cov=awscurl \ 46 | --cov-fail-under=77 \ 47 | --cov-report html \ 48 | --cov-report=html:"${PER_VER_DIR}"/htmlcov \ 49 | --durations=2 \ 50 | --strict-config 51 | ) 52 | done 53 | -------------------------------------------------------------------------------- /scripts/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/env bash 2 | 3 | set -o errexit 4 | set -o pipefail 5 | set -o nounset 6 | set -o xtrace 7 | 8 | 9 | OS_RELEASE=$(. /etc/os-release; echo "${NAME}") 10 | 11 | if [ "${OS_RELEASE}" = "Ubuntu" ]; then 12 | apt update 13 | apt install -y sudo 14 | echo 'tzdata tzdata/Areas select Europe' | debconf-set-selections 15 | echo 'tzdata tzdata/Zones/Europe select Paris' | debconf-set-selections 16 | DEBIAN_FRONTEND="noninteractive" apt install -y tzdata 17 | 18 | apt install -y curl git \ 19 | build-essential \ 20 | autoconf \ 21 | automake \ 22 | libtool \ 23 | libffi-dev libreadline-dev libz-dev libsqlite-dev libssl-dev \ 24 | libreadline-dev libsqlite3-dev wget curl libncurses5-dev libncursesw5-dev \ 25 | xz-utils tk-dev libffi-dev libbz2-dev liblzma-dev git 26 | elif [ "${OS_RELEASE}" = "CentOS Linux" ]; then 27 | yum update -y 28 | yum group install -y "Development Tools" 29 | yum install -y libffi-devel readline-devel zlib-devel bzip2-devel sqlite-devel openssl-devel git 30 | fi 31 | 32 | curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash 33 | export PATH="/root/.pyenv/bin:$PATH" 34 | if [ -z "${PROMPT_COMMAND:-}" ]; then 35 | export PROMPT_COMMAND="" 36 | fi 37 | eval "$(pyenv init -)" 38 | eval "$(pyenv virtualenv-init -)" 39 | 40 | grep -v '^ *#' < .python-version | while IFS= read -r line 41 | do 42 | pyenv install -s "${line}" 43 | done 44 | -------------------------------------------------------------------------------- /scripts/pypi_publish.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | pip install twine 5 | 6 | python setup.py sdist bdist_wheel 7 | 8 | twine upload dist/* -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [pycodestyle] 2 | count = False 3 | max-line-length = 120 4 | statistics = True 5 | ignore = E501,W291,E225,E123,E266,W504 6 | exclude = build,venv,venv3.6.15,venv3.8.16,.tox 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | __author__ = 'iokulist' 2 | 3 | from setuptools import setup 4 | 5 | # https://github.com/okigan/awscurl/issues/167 6 | # with open("requirements.txt", "r", encoding="utf-8") as f: 7 | # requirements = f.read().splitlines() 8 | 9 | # https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html#summary 10 | 11 | setup( 12 | name='awscurl', 13 | version='0.36', 14 | description='Curl like tool with AWS request signing', 15 | url='http://github.com/okigan/awscurl', 16 | author='Igor Okulist', 17 | author_email='okigan@gmail.com', 18 | license='MIT', 19 | packages=['awscurl'], 20 | # package_data={ 21 | # 'tests': ['*.py'], 22 | # }, 23 | entry_points={ 24 | 'console_scripts': [ 25 | 'awscurl = awscurl.__main__:main', 26 | ], 27 | }, 28 | zip_safe=False, 29 | install_requires=[ 30 | 'requests', 31 | 'configargparse', 32 | 'configparser', 33 | 'urllib3', 34 | 'botocore', 35 | ], 36 | extras_require={ 37 | 'awslibs': [] 38 | } 39 | ) 40 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/okigan/awscurl/2a6547300ef5592178159211eb541a1a6e08b683/tests/__init__.py -------------------------------------------------------------------------------- /tests/basic_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | def increment(x): 5 | return x + 1 6 | 7 | 8 | class ShallPassTest(unittest.TestCase): 9 | def test(self): 10 | self.assertEqual(increment(3), 4) 11 | -------------------------------------------------------------------------------- /tests/data/credentials: -------------------------------------------------------------------------------- 1 | [default] 2 | aws_access_key_id = access_key_id 3 | aws_secret_access_key = secret_access_key 4 | -------------------------------------------------------------------------------- /tests/integration_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import base64 5 | 6 | from unittest import TestCase 7 | 8 | import sys 9 | import os 10 | 11 | # this block resolves issues with pytest/tox, overall project dir structure 12 | # should be updated, some hints at can be found here: 13 | # https://stackoverflow.com/questions/55737714/how-does-a-tox-environment-set-its-sys-path 14 | print(f'sys.path={sys.path}') 15 | this_script_dir=os.path.dirname(os.path.abspath(__file__)) 16 | extra_path=os.path.join(this_script_dir, '..', 'awscurl') 17 | if not os.path.exists(extra_path): 18 | print(f'extra_path does not exist: {extra_path}') 19 | sys.path.append(extra_path) 20 | print(f'sys.path2={sys.path}') 21 | 22 | from awscurl.awscurl import make_request, inner_main # nopep8: E402 23 | 24 | 25 | __author__ = 'iokulist' 26 | 27 | 28 | class TestMakeRequestWithToken(TestCase): 29 | maxDiff = None 30 | 31 | def test_make_request(self, *args, **kvargs): 32 | headers = {} 33 | access_key = base64.b64decode('QUtJQUkyNkxPQU5NSlpLNVNQWUE=').decode("utf-8") 34 | secret_key = base64.b64decode('ekVQbE9URjU0Mys5M0l6UlNnNEVCOEd4cjFQV2NVa1p0TERWSmY4ag==').decode("utf-8") 35 | params = {'method': 'GET', 36 | 'service': 's3', 37 | 'region': 'us-east-1', 38 | 'uri': 'https://awscurl-sample-bucket.s3.amazonaws.com/awscurl-sample-file:.txt?a=b', 39 | 'headers': headers, 40 | 'data': '', 41 | 'access_key': access_key, 42 | 'secret_key': secret_key, 43 | 'security_token': None, 44 | 'data_binary': False} 45 | 46 | r = make_request(**params) 47 | 48 | self.assertEqual(r.status_code, 200) 49 | 50 | 51 | class TestMakeRequestWithTokenAndBinaryData(TestCase): 52 | maxDiff = None 53 | 54 | def test_make_request(self, *args, **kvargs): 55 | headers = {} 56 | access_key = base64.b64decode('QUtJQUkyNkxPQU5NSlpLNVNQWUE=').decode("utf-8") 57 | secret_key = base64.b64decode('ekVQbE9URjU0Mys5M0l6UlNnNEVCOEd4cjFQV2NVa1p0TERWSmY4ag==').decode("utf-8") 58 | params = {'method': 'GET', 59 | 'service': 's3', 60 | 'region': 'us-east-1', 61 | 'uri': 'https://awscurl-sample-bucket.s3.amazonaws.com/awscurl-sample-file:.txt?a=b', 62 | 'headers': headers, 63 | 'data': b'C\xcfI\x91\xc1\xd0\tw<\xa8\x13\x06{=\x9b\xb3\x1c\xfcl\xfe\xb9\xb18zS\xf4%i*Q\xc9v', 64 | 'access_key': access_key, 65 | 'secret_key': secret_key, 66 | 'security_token': None, 67 | 'data_binary': True} 68 | 69 | r = make_request(**params) 70 | 71 | self.assertEqual(r.status_code, 200) 72 | 73 | 74 | class TestMakeRequestWithTokenAndEnglishData(TestCase): 75 | maxDiff = None 76 | 77 | def test_make_request(self, *args, **kvargs): 78 | headers = {} 79 | access_key = base64.b64decode('QUtJQUkyNkxPQU5NSlpLNVNQWUE=').decode("utf-8") 80 | secret_key = base64.b64decode('ekVQbE9URjU0Mys5M0l6UlNnNEVCOEd4cjFQV2NVa1p0TERWSmY4ag==').decode("utf-8") 81 | params = {'method': 'GET', 82 | 'service': 's3', 83 | 'region': 'us-east-1', 84 | 'uri': 'https://awscurl-sample-bucket.s3.amazonaws.com/awscurl-sample-file:.txt?a=b', 85 | 'headers': headers, 86 | 'data': 'Test', 87 | 'access_key': access_key, 88 | 'secret_key': secret_key, 89 | 'security_token': None, 90 | 'data_binary': False} 91 | 92 | r = make_request(**params) 93 | 94 | self.assertEqual(r.status_code, 200) 95 | 96 | 97 | class TestMakeRequestWithTokenAndNonEnglishData(TestCase): 98 | maxDiff = None 99 | 100 | def test_make_request(self, *args, **kvargs): 101 | headers = {} 102 | access_key = base64.b64decode('QUtJQUkyNkxPQU5NSlpLNVNQWUE=').decode("utf-8") 103 | secret_key = base64.b64decode('ekVQbE9URjU0Mys5M0l6UlNnNEVCOEd4cjFQV2NVa1p0TERWSmY4ag==').decode("utf-8") 104 | params = {'method': 'GET', 105 | 'service': 's3', 106 | 'region': 'us-east-1', 107 | 'uri': 'https://awscurl-sample-bucket.s3.amazonaws.com/awscurl-sample-file:.txt?a=b', 108 | 'headers': headers, 109 | 'data': u'テスト', 110 | 'access_key': access_key, 111 | 'secret_key': secret_key, 112 | 'security_token': None, 113 | 'data_binary': False} 114 | 115 | r = make_request(**params) 116 | 117 | self.assertEqual(r.status_code, 200) 118 | 119 | 120 | class TestInnerMainMethod(TestCase): 121 | maxDiff = None 122 | 123 | def test_exit_code_without_fail_option(self, *args, **kwargs): 124 | self.assertEqual( 125 | inner_main(['--verbose', '--service', 's3', 'https://awscurl-sample-bucket.s3.amazonaws.com']), 126 | 0 127 | ) 128 | 129 | def test_exit_code_with_fail_option(self, *args, **kwargs): 130 | self.assertEqual( 131 | inner_main(['--verbose', '--fail-with-body', '--service', 's3', 'https://awscurl-sample-bucket.s3.amazonaws.com']), 132 | 22 133 | ) 134 | 135 | class TestInnerMainMethodEmptyCredentials(TestCase): 136 | maxDiff = None 137 | 138 | def test_exit_code_without_fail_option(self, *args, **kwargs): 139 | self.assertEqual( 140 | inner_main(['--verbose', '--access_key', '', '--secret_key', '', '--session_token', '', '--service', 's3', 141 | 'https://awscurl-sample-bucket.s3.amazonaws.com']), 142 | 0 143 | ) 144 | 145 | def test_exit_code_with_fail_option(self, *args, **kwargs): 146 | self.assertEqual( 147 | inner_main(['--verbose', '--fail-with-body', '--access_key', '', '--secret_key', '', '--session_token', '', '--service', 's3', 148 | 'https://awscurl-sample-bucket.s3.amazonaws.com']), 149 | 22 150 | ) 151 | -------------------------------------------------------------------------------- /tests/load_aws_config_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | 4 | from unittest import TestCase 5 | 6 | from awscurl.awscurl import load_aws_config 7 | 8 | __author__ = 'iokulist' 9 | 10 | 11 | class Test__load_aws_config(TestCase): 12 | def test(self): 13 | access_key, secret_access, token = load_aws_config(None, 14 | None, 15 | None, 16 | "./tests/data/credentials", 17 | "default") 18 | 19 | self.assertEqual([access_key, secret_access, token], ['access_key_id', 'secret_access_key', None]) 20 | 21 | access_key, secret_access, token = load_aws_config(None, 22 | None, 23 | "ttt", 24 | "./tests/data/credentials", 25 | "default") 26 | 27 | self.assertEqual([access_key, secret_access, token], ['access_key_id', 'secret_access_key', 'ttt']) 28 | 29 | # TODO: remove this test as I think it's not valid to loads secret_key if session_key was already provided 30 | # access_key, secret_access, token = load_aws_config('aaa', 31 | # None, 32 | # "ttt", 33 | # "./tests/data/credentials", 34 | # "default") 35 | # 36 | # self.assertEquals([access_key, secret_access, token], ['aaa', None, 'ttt']) 37 | -------------------------------------------------------------------------------- /tests/stages_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Test cases for seprate header calculation stages. 4 | """ 5 | 6 | import json 7 | 8 | from unittest import TestCase 9 | 10 | from awscurl.awscurl import ( 11 | task_1_create_a_canonical_request, 12 | task_2_create_the_string_to_sign, 13 | task_3_calculate_the_signature, 14 | task_4_build_auth_headers_for_the_request) 15 | 16 | 17 | class TestStages(TestCase): 18 | """ 19 | Suite to test all stages. 20 | """ 21 | maxDiff = None 22 | 23 | def test_task_1_create_a_canonical_request(self): 24 | """ 25 | Test the function to create the "canonical" request to match the thing that AWS is hashing 26 | on the server side. 27 | """ 28 | canonical_request, payload_hash, signed_headers = task_1_create_a_canonical_request( 29 | query="Action=DescribeInstances&Version=2013-10-15", 30 | headers=json.loads('{"Content-Type": "application/json", "Accept": "application/xml"}'), 31 | port=None, 32 | host="ec2.amazonaws.com", 33 | amzdate="20190921T022008Z", 34 | method="GET", 35 | data="", 36 | security_token=None, 37 | data_binary=False, 38 | canonical_uri="/") 39 | self.assertEqual(canonical_request, "GET\n" 40 | "/\n" 41 | "Action=DescribeInstances&Version=2013-10-15\n" 42 | "host:ec2.amazonaws.com\n" 43 | "x-amz-date:20190921T022008Z\n" 44 | "\n" 45 | "host;x-amz-date\n" 46 | "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") 47 | self.assertEqual(payload_hash, 48 | "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") 49 | self.assertEqual(signed_headers, "host;x-amz-date") 50 | 51 | def test_task_1_create_a_canonical_request_url_encode_querystring(self): 52 | """ 53 | Test that canonical requests correctly sort and url encode querystring parameters. 54 | """ 55 | canonical_request, payload_hash, signed_headers = task_1_create_a_canonical_request( 56 | query="arg1=true&arg3=c,b,a&arg2=false&noEncoding=ABC-abc_1.23~tilde/slash", 57 | headers=json.loads('{"Content-Type": "application/json", "Accept": "application/xml"}'), 58 | port=None, 59 | host="my-gateway-id.execute-api.us-east-1.amazonaws.com", 60 | amzdate="20190921T022008Z", 61 | method="GET", 62 | data="", 63 | security_token=None, 64 | data_binary=False, 65 | canonical_uri="/stage/my-path") 66 | self.assertEqual(canonical_request, "GET\n" 67 | "/stage/my-path\n" 68 | "arg1=true&arg2=false&arg3=c%2Cb%2Ca&noEncoding=ABC-abc_1.23~tilde%2Fslash\n" 69 | "host:my-gateway-id.execute-api.us-east-1.amazonaws.com\n" 70 | "x-amz-date:20190921T022008Z\n" 71 | "\n" 72 | "host;x-amz-date\n" 73 | "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") 74 | self.assertEqual(payload_hash, 75 | "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") 76 | self.assertEqual(signed_headers, "host;x-amz-date") 77 | 78 | def test_task_2_create_the_string_to_sign(self): 79 | """ 80 | Test the next function that is creating a string in exactly the same way as AWS is on the 81 | server side, to make sure our signature matches. 82 | """ 83 | string_to_sign, algorithm, credential_scope = task_2_create_the_string_to_sign( 84 | amzdate="20190921T022008Z", 85 | datestamp="20190921", 86 | canonical_request="GET\n" 87 | "/\n" 88 | "Action=DescribeInstances&Version=2013-10-15\n" 89 | "host:ec2.amazonaws.com\n" 90 | "x-amz-date:20190921T022008Z\n" 91 | "\n" 92 | "host;x-amz-date\n" 93 | "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", 94 | service="ec2", 95 | region="us-east-1", 96 | ) 97 | self.assertEqual(string_to_sign, "AWS4-HMAC-SHA256\n" 98 | "20190921T022008Z\n" 99 | "20190921/us-east-1/ec2/aws4_request\n" 100 | "4a3b77321aca7e671d4945f0b3b826112e5ca3f2a10c4357e54f518798e7c8ff") 101 | self.assertEqual(algorithm, "AWS4-HMAC-SHA256") 102 | self.assertEqual(credential_scope, "20190921/us-east-1/ec2/aws4_request") 103 | 104 | def test_task_3_calculate_the_signature(self): 105 | """ 106 | Test that we calculate the correct signature from our carefully prepared strings. 107 | """ 108 | signature = task_3_calculate_the_signature( 109 | datestamp="20190921", 110 | string_to_sign="AWS4-HMAC-SHA256\n" 111 | "20190921T022008Z\n" 112 | "20190921/us-east-1/ec2/aws4_request\n" 113 | "4a3b77321aca7e671d4945f0b3b826112e5ca3f2a10c4357e54f518798e7c8ff", 114 | service="ec2", 115 | region="us-east-1", 116 | secret_key="dummytestsecretkey", 117 | ) 118 | self.assertEqual(signature, 119 | "9164aea23e266890838ff6e51eea552e2ee39c63896ac61d91990f200bb16362") 120 | 121 | def test_task_4_build_auth_headers_for_the_request(self): 122 | """ 123 | Test that we are adding the proper headers based on all our calculated information. 124 | """ 125 | new_headers = task_4_build_auth_headers_for_the_request( 126 | amzdate="20190921T022008Z", 127 | payload_hash="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", 128 | algorithm="AWS4-HMAC-SHA256", 129 | credential_scope="20190921/us-east-1/ec2/aws4_request", 130 | signed_headers="host;x-amz-date", 131 | signature="9164aea23e266890838ff6e51eea552e2ee39c63896ac61d91990f200bb16362", 132 | access_key="AKIAIJLPLDILMJV53HCQ", 133 | security_token=None, 134 | ) 135 | self.assertEqual( 136 | new_headers['x-amz-content-sha256'], 137 | 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855') 138 | self.assertNotIn('x-amz-security-token', new_headers) 139 | self.assertEqual( 140 | new_headers['x-amz-date'], 141 | '20190921T022008Z') 142 | self.assertEqual( 143 | new_headers['Authorization'], 144 | 'AWS4-HMAC-SHA256 ' 145 | 'Credential=AKIAIJLPLDILMJV53HCQ/20190921/us-east-1/ec2/aws4_request, ' 146 | 'SignedHeaders=host;x-amz-date, ' 147 | 'Signature=9164aea23e266890838ff6e51eea552e2ee39c63896ac61d91990f200bb16362') 148 | -------------------------------------------------------------------------------- /tests/unit_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: UTF-8 -*- 3 | 4 | import datetime 5 | import json 6 | import sys 7 | 8 | from unittest import TestCase 9 | 10 | from mock import patch 11 | 12 | from awscurl.awscurl import aws_url_encode, make_request 13 | 14 | from requests.exceptions import SSLError 15 | from requests import Response 16 | 17 | import pytest 18 | __author__ = 'iokulist' 19 | 20 | 21 | def my_mock_get(): 22 | class Object(): 23 | pass 24 | 25 | def ss(*args, **kargs): 26 | print("in mock") 27 | response = Object() 28 | response.status_code = 200 29 | response.text = 'some text' 30 | return response 31 | 32 | return ss 33 | 34 | 35 | def my_mock_send_request(): 36 | class Object(): 37 | pass 38 | 39 | def ss(*args, **kargs): 40 | print("in mock") 41 | response = Object() 42 | response.status_code = 200 43 | response.text = 'some text' 44 | return response 45 | 46 | return ss 47 | 48 | 49 | def my_mock_send_request_verify(): 50 | class Object(): 51 | pass 52 | 53 | def ss(uri, data, headers, method, verify, allow_redirects, **kargs): 54 | print("in mock") 55 | if not verify: 56 | raise SSLError 57 | response = Object() 58 | response.status_code = 200 59 | response.text = 'some text' 60 | 61 | return response 62 | 63 | return ss 64 | 65 | 66 | def my_mock_utcnow(): 67 | class Object(): 68 | pass 69 | 70 | def ss(*args, **kargs): 71 | print("in mock") 72 | return datetime.datetime.utcfromtimestamp(0) 73 | 74 | return ss 75 | 76 | 77 | class TestMakeRequest(TestCase): 78 | maxDiff = None 79 | 80 | @patch('requests.get', new_callable=my_mock_get) 81 | @patch('awscurl.awscurl.__send_request', new_callable=my_mock_send_request) 82 | @patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow) 83 | def test_make_request(self, *args, **kvargs): 84 | headers = {} 85 | params = {'method': 'GET', 86 | 'service': 'ec2', 87 | 'region': 'region', 88 | 'uri': 'https://user:pass@host:123/path/?a=b&c=d', 89 | 'headers': headers, 90 | 'data': '', 91 | 'access_key': '', 92 | 'secret_key': '', 93 | 'security_token': '', 94 | 'data_binary': False} 95 | make_request(**params) 96 | 97 | expected = {'x-amz-date': '19700101T000000Z', 98 | 'Authorization': 'AWS4-HMAC-SHA256 Credential=/19700101/region/ec2/aws4_request, SignedHeaders=host;x-amz-date, Signature=de2b9ea384c10b03314afa10532adac358f8c93e3f3dd5bd724eda24a367a7ef', 99 | 'x-amz-content-sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 100 | 'x-amz-security-token': ''} 101 | 102 | self.assertEqual(expected, headers) 103 | 104 | pass 105 | 106 | @patch('requests.get', new_callable=my_mock_get) 107 | @patch('awscurl.awscurl.__send_request', new_callable=my_mock_send_request) 108 | @patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow) 109 | def test_make_request2(self, *args, **kvargs): 110 | 111 | payload = json.dumps({ 112 | "key": "", 113 | }) 114 | creds = { 115 | "access_key": "", 116 | "secret_key": "", 117 | "token": "" 118 | } 119 | 120 | headers = { 121 | "Content-Type": "application/json; charset:UTF-8", 122 | "Connection": "keep-alive", 123 | "Content-Encoding": "amz-1.0", 124 | "x-amz-requestsupertrace": "true" 125 | } 126 | 127 | params = { 128 | 'method':'POST', 129 | 'service':'service-', 130 | 'region':"region-", 131 | 'uri':"", 132 | 'headers': headers, 133 | 'data':payload, 134 | 'data_binary':False, 135 | 'access_key':creds['access_key'], 136 | 'secret_key':creds['secret_key'], 137 | 'security_token':creds['token'], 138 | } 139 | 140 | make_request(**params) 141 | 142 | expected = { 143 | "Content-Type": "application/json; charset:UTF-8", 144 | "Connection": "keep-alive", 145 | "Content-Encoding": "amz-1.0", 146 | "x-amz-requestsupertrace": "true", 147 | "Authorization": "AWS4-HMAC-SHA256 Credential=/19700101/region-/service-/aws4_request, SignedHeaders=host;x-amz-date;x-amz-requestsupertrace;x-amz-security-token, Signature=77e0f17c91f179231fcdf42f4387539b935117600de340ab1904f66302c181d7", 148 | "x-amz-date": "19700101T000000Z", 149 | "x-amz-content-sha256": "4930e13bdc55bb30accf137260ec8fa65b35658360e92a5f8498def3f8ab6144", 150 | "x-amz-security-token": "" 151 | } 152 | 153 | self.assertEqual(expected, headers) 154 | 155 | pass 156 | 157 | 158 | class TestMakeRequestVerifySSLRaises(TestCase): 159 | maxDiff = None 160 | 161 | @patch('awscurl.awscurl.__send_request', new_callable=my_mock_send_request_verify) 162 | @patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow) 163 | def test_make_request(self, *args, **kvargs): 164 | headers = {} 165 | params = {'method': 'GET', 166 | 'service': 'ec2', 167 | 'region': 'region', 168 | 'uri': 'https://user:pass@host:123/path/?a=b&c=d', 169 | 'headers': headers, 170 | 'data': '', 171 | 'access_key': '', 172 | 'secret_key': '', 173 | 'security_token': '', 174 | 'data_binary': False, 175 | 'verify': False, 176 | 'allow_redirects': False} 177 | 178 | with pytest.raises(SSLError): 179 | make_request(**params) 180 | 181 | pass 182 | 183 | 184 | class TestMakeRequestVerifySSLPass(TestCase): 185 | maxDiff = None 186 | 187 | @patch('awscurl.awscurl.__send_request', new_callable=my_mock_send_request_verify) 188 | @patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow) 189 | def test_make_request(self, *args, **kvargs): 190 | headers = {} 191 | params = {'method': 'GET', 192 | 'service': 'ec2', 193 | 'region': 'region', 194 | 'uri': 'https://user:pass@host:123/path/?a=b&c=d', 195 | 'headers': headers, 196 | 'data': '', 197 | 'access_key': '', 198 | 'secret_key': '', 199 | 'security_token': '', 200 | 'data_binary': False, 201 | 'verify': True, 202 | 'allow_redirects': False} 203 | make_request(**params) 204 | 205 | expected = {'x-amz-date': '19700101T000000Z', 206 | 'Authorization': 'AWS4-HMAC-SHA256 Credential=/19700101/region/ec2/aws4_request, SignedHeaders=host;x-amz-date, Signature=de2b9ea384c10b03314afa10532adac358f8c93e3f3dd5bd724eda24a367a7ef', 207 | 'x-amz-content-sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 208 | 'x-amz-security-token': ''} 209 | 210 | self.assertEqual(expected, headers) 211 | 212 | pass 213 | 214 | 215 | class TestMakeRequestWithBinaryData(TestCase): 216 | maxDiff = None 217 | 218 | @patch('requests.get', new_callable=my_mock_get) 219 | @patch('awscurl.awscurl.__send_request', new_callable=my_mock_send_request) 220 | @patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow) 221 | def test_make_request(self, *args, **kvargs): 222 | headers = {} 223 | params = {'method': 'GET', 224 | 'service': 'ec2', 225 | 'region': 'region', 226 | 'uri': 'https://user:pass@host:123/path/?a=b&c=d', 227 | 'headers': headers, 228 | 'data': b'C\xcfI\x91\xc1\xd0\tw<\xa8\x13\x06{=\x9b\xb3\x1c\xfcl\xfe\xb9\xb18zS\xf4%i*Q\xc9v', 229 | 'access_key': '', 230 | 'secret_key': '', 231 | 'security_token': '', 232 | 'data_binary': True} 233 | make_request(**params) 234 | 235 | expected = {'x-amz-date': '19700101T000000Z', 236 | 'Authorization': 'AWS4-HMAC-SHA256 Credential=/19700101/region/ec2/aws4_request, SignedHeaders=host;x-amz-date, Signature=6ebcf316c9bb50bb7b2bbabf128dddde3babbf16badfd31ddc40838e7592d5df', 237 | 'x-amz-content-sha256': '3f514228bd64bbff67daaa80e482aee0e0b0c51891d3a64e4abfa145f4364b99', 238 | 'x-amz-security-token': ''} 239 | 240 | self.assertEqual(expected, headers) 241 | 242 | pass 243 | 244 | 245 | class TestMakeRequestWithToken(TestCase): 246 | maxDiff = None 247 | 248 | @patch('requests.get', new_callable=my_mock_get) 249 | @patch('awscurl.awscurl.__send_request', new_callable=my_mock_send_request) 250 | @patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow) 251 | def test_make_request(self, *args, **kvargs): 252 | headers = {} 253 | params = {'method': 'GET', 254 | 'service': 'ec2', 255 | 'region': 'region', 256 | 'uri': 'https://user:pass@host:123/path/?a=b&c=d', 257 | 'headers': headers, 258 | 'data': '', 259 | 'access_key': 'ABC', 260 | 'secret_key': 'DEF', 261 | 'security_token': 'GHI', 262 | 'data_binary': False} 263 | make_request(**params) 264 | 265 | expected = {'x-amz-date': '19700101T000000Z', 266 | 'x-amz-content-sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 267 | 'Authorization': 'AWS4-HMAC-SHA256 Credential=ABC/19700101/region/ec2/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=e767448ca06e8f3a17548d4193ea29afa759b84f957a71d0a051815f5ebfedfa', 268 | 'x-amz-security-token': 'GHI'} 269 | 270 | self.assertEqual(expected, headers) 271 | 272 | pass 273 | 274 | 275 | class TestMakeRequestWithTokenAndBinaryData(TestCase): 276 | maxDiff = None 277 | 278 | @patch('requests.get', new_callable=my_mock_get) 279 | @patch('awscurl.awscurl.__send_request', new_callable=my_mock_send_request) 280 | @patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow) 281 | def test_make_request(self, *args, **kvargs): 282 | headers = {} 283 | params = {'method': 'GET', 284 | 'service': 'ec2', 285 | 'region': 'region', 286 | 'uri': 'https://user:pass@host:123/path/?a=b&c=d', 287 | 'headers': headers, 288 | 'data': b'C\xcfI\x91\xc1\xd0\tw<\xa8\x13\x06{=\x9b\xb3\x1c\xfcl\xfe\xb9\xb18zS\xf4%i*Q\xc9v', 289 | 'access_key': 'ABC', 290 | 'secret_key': 'DEF', 291 | 'security_token': 'GHI', 292 | 'data_binary': True} 293 | make_request(**params) 294 | 295 | expected = {'x-amz-date': '19700101T000000Z', 296 | 'x-amz-content-sha256': '3f514228bd64bbff67daaa80e482aee0e0b0c51891d3a64e4abfa145f4364b99', 297 | 'Authorization': 'AWS4-HMAC-SHA256 Credential=ABC/19700101/region/ec2/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=edcee42e10d5a4cec5414ebe938edcf292a9a33261809523e2df16281d452c5f', 298 | 'x-amz-security-token': 'GHI'} 299 | 300 | self.assertEqual(expected, headers) 301 | 302 | pass 303 | 304 | 305 | class TestHostFromHeaderUsedInCanonicalHeader(TestCase): 306 | maxDiff = None 307 | 308 | @patch('requests.get', new_callable=my_mock_get) 309 | @patch('awscurl.awscurl.__send_request', new_callable=my_mock_send_request) 310 | @patch('awscurl.awscurl.__now', new_callable=my_mock_utcnow) 311 | def test_make_request(self, *args, **kvargs): 312 | headers = {'host': 'some.other.host.address.com'} 313 | params = {'method': 'GET', 314 | 'service': 'ec2', 315 | 'region': 'region', 316 | 'uri': 'https://user:pass@host:123/path/?a=b&c=d', 317 | 'headers': headers, 318 | 'data': '', 319 | 'access_key': 'ABC', 320 | 'secret_key': 'DEF', 321 | 'security_token': 'GHI', 322 | 'data_binary': False} 323 | make_request(**params) 324 | 325 | expected = {'host': 'some.other.host.address.com', 326 | 'x-amz-date': '19700101T000000Z', 327 | 'x-amz-content-sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 328 | 'Authorization': 'AWS4-HMAC-SHA256 Credential=ABC/19700101/region/ec2/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token, Signature=9cba1c499417655c170f5018b577b9f89154cf9b9827273df54bfa182e5f4273', 329 | 'x-amz-security-token': 'GHI'} 330 | 331 | self.assertEqual(expected, headers) 332 | 333 | pass 334 | 335 | 336 | class TestRequestResponse(TestCase): 337 | maxDiff = None 338 | 339 | @patch('awscurl.awscurl.__send_request') 340 | def test_make_request(self, mocked_resp): 341 | resp = Response() 342 | resp.status_code=200 343 | resp._content = b'{"file_name": "test.yml", "env": "staging", "hash": "\xe5\xad\x97"}' 344 | resp.encoding = 'UTF-8' 345 | mocked_resp.return_value = resp 346 | 347 | headers = {} 348 | params = {'method': 'GET', 349 | 'service': 'ec2', 350 | 'region': 'region', 351 | 'uri': 'https://user:pass@host:123/path/?a=b&c=d', 352 | 'headers': headers, 353 | 'data': b'C\xcfI\x91\xc1\xd0\tw<\xa8\x13\x06{=\x9b\xb3\x1c\xfcl\xfe\xb9\xb18zS\xf4%i*Q\xc9v', 354 | 'access_key': '', 355 | 'secret_key': '', 356 | 'security_token': '', 357 | 'data_binary': True} 358 | r = make_request(**params) 359 | 360 | expected = u'\u5b57' 361 | 362 | ### assert that the unicode character is in the response.text output 363 | self.assertTrue(expected in r.text) 364 | 365 | ### assert that the unicode character is _not_ in the response.text.encode('utf-8') 366 | ### which has been converted to 8-bit string with unicode characters escaped 367 | ### in py2 this raises an exception on the assertion (`expected in x` below) 368 | ### in py3 we can compare the two directly, and the assertion should be false 369 | if sys.version_info[0] == 2: 370 | with self.assertRaises(UnicodeDecodeError): 371 | x = str(r.text.encode('utf-8')) 372 | expected in x 373 | else: 374 | self.assertFalse(expected in str(r.text.encode('utf-8'))) 375 | 376 | pass 377 | 378 | 379 | class TestAwsUrlEncode(TestCase): 380 | def test_aws_url_encode(self): 381 | self.assertEqual(aws_url_encode(""), "") 382 | self.assertEqual(aws_url_encode("AZaz09-_.~"), "AZaz09-_.~") 383 | self.assertEqual(aws_url_encode(" /:@[`{"), "%20%2F%3A%40%5B%60%7B") 384 | self.assertEqual(aws_url_encode("a=,=b"), "a==%2C==b") 385 | self.assertEqual(aws_url_encode("\u0394-\u30a1"), "%CE%94-%E3%82%A1") 386 | 387 | pass 388 | -------------------------------------------------------------------------------- /tests/url_parsing_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | 4 | from unittest import TestCase 5 | 6 | from awscurl import awscurl 7 | 8 | __author__ = 'iokulist' 9 | 10 | 11 | class TestUriParsing(TestCase): 12 | maxDiff = None 13 | 14 | def test(self, *args, **kvargs): 15 | self.assertEqual(awscurl.url_path_to_dict("http://google.com"), 16 | {'host': 'google.com', 17 | 'password': None, 18 | 'path': '/', 19 | 'port': None, 20 | 'query': '', 21 | 'schema': 'http', 22 | 'user': None}) 23 | 24 | self.assertEqual(awscurl.url_path_to_dict("http://user:password@google.com"), 25 | {'host': 'google.com', 26 | 'password': 'password', 27 | 'path': '/', 28 | 'port': None, 29 | 'query': '', 30 | 'schema': 'http', 31 | 'user': 'user'}) 32 | 33 | self.assertEqual(awscurl.url_path_to_dict("http://user:password@google.com/path1/path2"), 34 | {'host': 'google.com', 35 | 'password': 'password', 36 | 'path': '/path1/path2', 37 | 'port': None, 38 | 'query': '', 39 | 'schema': 'http', 40 | 'user': 'user'}) 41 | 42 | self.assertEqual(awscurl.url_path_to_dict("http://google.com/path1/path2/@weird"), 43 | {'host': 'google.com', 44 | 'password': None, 45 | 'path': '/path1/path2/@weird', 46 | 'port': None, 47 | 'query': '', 48 | 'schema': 'http', 49 | 'user': None}) 50 | --------------------------------------------------------------------------------