├── MANIFEST.in ├── .gitignore ├── .travis.yml ├── setup.py ├── README.md ├── example.py ├── LICENSE.txt ├── awsauth.py └── test.py /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE.txt README.md *.py -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | dist/ 3 | requests_aws.egg-info/ 4 | env/ 5 | *.pyc 6 | *.*~ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: python 3 | python: 4 | - "2.6" 5 | - "2.7" 6 | - "3.2" 7 | - "3.3" 8 | - "3.4" 9 | install: 10 | - pip install requests 11 | # command to run tests, e.g. python setup.py test 12 | script: python test.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import sys 4 | 5 | try: 6 | from setuptools import setup 7 | # hush pyflakes 8 | setup 9 | except ImportError: 10 | from distutils.core import setup 11 | 12 | if sys.argv[-1] == 'publish': 13 | os.system('python setup.py sdist upload') 14 | sys.exit() 15 | 16 | setup( 17 | name='requests-aws', 18 | version='0.1.8', 19 | author='Paul Tax', 20 | author_email='paultax@gmail.com', 21 | include_package_data=True, 22 | install_requires=['requests>=0.14.0'], 23 | py_modules=['awsauth'], 24 | url='https://github.com/tax/python-requests-aws', 25 | license='BSD licence, see LICENCE.txt', 26 | description='AWS authentication for Amazon S3 for the python requests module', 27 | long_description=open('README.md').read(), 28 | ) 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #S3 using python-requests 2 | 3 | AWS authentication for Amazon S3 for the wonderful [pyhon requests library](http://python-requests.org) 4 | 5 | [![Build Status](https://travis-ci.org/tax/python-requests-aws.svg?branch=master)](https://travis-ci.org/tax/python-requests-aws) 6 | 7 | - Tested with python 2.7 and python 3 8 | - At the moment only S3 is supported 9 | 10 | ## Usage 11 | 12 | 13 | ```python 14 | import requests 15 | from awsauth import S3Auth 16 | 17 | ACCESS_KEY = 'ACCESSKEYXXXXXXXXXXXX' 18 | SECRET_KEY = 'AWSSECRETKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' 19 | 20 | url = 'http://mybuck.s3.amazonaws.com/file.txt' 21 | s = 'Lola is sweet' 22 | # Creating a file 23 | r = requests.put(url, data=s, auth=S3Auth(ACCESS_KEY, SECRET_KEY)) 24 | 25 | # Downloading a file 26 | r = requests.get(url, auth=S3Auth(ACCESS_KEY, SECRET_KEY)) 27 | if r.text == 'Lola is sweet': 28 | print "It works" 29 | 30 | # Removing a file 31 | r = requests.delete(url, auth=S3Auth(ACCESS_KEY, SECRET_KEY)) 32 | 33 | ``` 34 | 35 | ## Installation 36 | Installing requests-aws is simple with pip: 37 | 38 | ``` 39 | $ pip install requests-aws 40 | ``` 41 | 42 | -------------------------------------------------------------------------------- /example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import print_function 4 | import requests 5 | from awsauth import S3Auth 6 | 7 | ACCESS_KEY = "ACCESSKEYXXXXXXXXXXXX" 8 | SECRET_KEY = "AWSSECRETKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" 9 | 10 | # https://forums.aws.amazon.com/thread.jspa?threadID=28799: 11 | # http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTObjectDELETE.html 12 | acceptable_accesscodes = (200, 204) 13 | 14 | if __name__ == '__main__': 15 | 16 | # Data needs to be in unicode, or it will fail 17 | data = u'Sam is sweet' 18 | 19 | bucket = 'mybucket' 20 | object_name = ['myfile.txt', 'my+file.txt'] 21 | 22 | for o in object_name: 23 | # Creating a file 24 | url = 'http://{0}.s3.amazonaws.com/{1}'.format(bucket, o) 25 | r = requests.put(url, data=data, auth=S3Auth(ACCESS_KEY, SECRET_KEY)) 26 | if r.status_code not in acceptable_accesscodes: 27 | r.raise_for_status() 28 | 29 | # Downloading a file 30 | url = 'http://{0}.s3.amazonaws.com/{1}'.format(bucket, o) 31 | r = requests.get(url, auth=S3Auth(ACCESS_KEY, SECRET_KEY)) 32 | if r.status_code not in acceptable_accesscodes: 33 | r.raise_for_status() 34 | 35 | if r.content == data: 36 | print('Hala Madrid!') 37 | 38 | # Removing a file 39 | url = 'http://{0}.s3.amazonaws.com/{1}'.format(bucket, o) 40 | r = requests.delete(url, auth=S3Auth(ACCESS_KEY, SECRET_KEY)) 41 | if r.status_code not in acceptable_accesscodes: 42 | r.raise_for_status() 43 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2013 Paul Tax All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | 1. Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in 12 | the documentation and/or other materials provided with the 13 | distribution. 14 | 15 | 3. Neither the name of Infrae nor the names of its contributors may 16 | be used to endorse or promote products derived from this software 17 | without specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INFRAE OR 23 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 26 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 27 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 28 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 29 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /awsauth.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import hmac 3 | 4 | from hashlib import sha1 as sha 5 | 6 | py3k = False 7 | try: 8 | from urlparse import urlparse, unquote 9 | from base64 import encodestring 10 | except: 11 | py3k = True 12 | from urllib.parse import urlparse, unquote 13 | from base64 import encodebytes as encodestring 14 | 15 | from email.utils import formatdate 16 | 17 | from requests.auth import AuthBase 18 | 19 | 20 | class S3Auth(AuthBase): 21 | 22 | """Attaches AWS Authentication to the given Request object.""" 23 | 24 | service_base_url = 's3.amazonaws.com' 25 | # List of Query String Arguments of Interest 26 | special_params = [ 27 | 'acl', 'location', 'logging', 'partNumber', 'policy', 'requestPayment', 28 | 'torrent', 'versioning', 'versionId', 'versions', 'website', 'uploads', 29 | 'uploadId', 'response-content-type', 'response-content-language', 30 | 'response-expires', 'response-cache-control', 'delete', 'lifecycle', 31 | 'response-content-disposition', 'response-content-encoding', 'tagging', 32 | 'notification', 'cors' 33 | ] 34 | 35 | def __init__(self, access_key, secret_key, service_url=None): 36 | if service_url: 37 | self.service_base_url = service_url 38 | self.access_key = str(access_key) 39 | self.secret_key = str(secret_key) 40 | 41 | def __call__(self, r): 42 | # Create date header if it is not created yet. 43 | if 'date' not in r.headers and 'x-amz-date' not in r.headers: 44 | r.headers['date'] = formatdate( 45 | timeval=None, 46 | localtime=False, 47 | usegmt=True) 48 | signature = self.get_signature(r) 49 | if py3k: 50 | signature = signature.decode('utf-8') 51 | r.headers['Authorization'] = 'AWS %s:%s' % (self.access_key, signature) 52 | return r 53 | 54 | def get_signature(self, r): 55 | canonical_string = self.get_canonical_string( 56 | r.url, r.headers, r.method) 57 | if py3k: 58 | key = self.secret_key.encode('utf-8') 59 | msg = canonical_string.encode('utf-8') 60 | else: 61 | key = self.secret_key 62 | msg = canonical_string 63 | h = hmac.new(key, msg, digestmod=sha) 64 | return encodestring(h.digest()).strip() 65 | 66 | def get_canonical_string(self, url, headers, method): 67 | parsedurl = urlparse(url) 68 | objectkey = parsedurl.path[1:] 69 | query_args = sorted(parsedurl.query.split('&')) 70 | 71 | bucket = parsedurl.netloc[:-len(self.service_base_url)] 72 | if len(bucket) > 1: 73 | # remove last dot 74 | bucket = bucket[:-1] 75 | 76 | interesting_headers = { 77 | 'content-md5': '', 78 | 'content-type': '', 79 | 'date': ''} 80 | for key in headers: 81 | lk = key.lower() 82 | try: 83 | lk = lk.decode('utf-8') 84 | except: 85 | pass 86 | if headers[key] and (lk in interesting_headers.keys() 87 | or lk.startswith('x-amz-')): 88 | interesting_headers[lk] = headers[key].strip() 89 | 90 | # If x-amz-date is used it supersedes the date header. 91 | if not py3k: 92 | if 'x-amz-date' in interesting_headers: 93 | interesting_headers['date'] = '' 94 | else: 95 | if 'x-amz-date' in interesting_headers: 96 | interesting_headers['date'] = '' 97 | 98 | buf = '%s\n' % method 99 | for key in sorted(interesting_headers.keys()): 100 | val = interesting_headers[key] 101 | if key.startswith('x-amz-'): 102 | buf += '%s:%s\n' % (key, val) 103 | else: 104 | buf += '%s\n' % val 105 | 106 | # append the bucket if it exists 107 | if bucket != '': 108 | buf += '/%s' % bucket 109 | 110 | # add the objectkey. even if it doesn't exist, add the slash 111 | buf += '/%s' % objectkey 112 | 113 | params_found = False 114 | 115 | # handle special query string arguments 116 | for q in query_args: 117 | k = q.split('=')[0] 118 | if k in self.special_params: 119 | buf += '&' if params_found else '?' 120 | params_found = True 121 | 122 | try: 123 | k, v = q.split('=', 1) 124 | 125 | except ValueError: 126 | buf += q 127 | 128 | else: 129 | # Riak CS multipart upload ids look like this, `TFDSheOgTxC2Tsh1qVK73A==`, 130 | # is should be escaped to be included as part of a query string. 131 | # 132 | # A requests mp upload part request may look like 133 | # resp = requests.put( 134 | # 'https://url_here', 135 | # params={ 136 | # 'partNumber': 1, 137 | # 'uploadId': 'TFDSheOgTxC2Tsh1qVK73A==' 138 | # }, 139 | # data='some data', 140 | # auth=S3Auth('access_key', 'secret_key') 141 | # ) 142 | # 143 | # Requests automatically escapes the values in the `params` dict, so now 144 | # our uploadId is `TFDSheOgTxC2Tsh1qVK73A%3D%3D`, 145 | # if we sign the request with the encoded value the signature will 146 | # not be valid, we'll get 403 Access Denied. 147 | # So we unquote, this is no-op if the value isn't encoded. 148 | buf += '{key}={value}'.format(key=k, value=unquote(v)) 149 | 150 | return buf 151 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import requests 4 | import hashlib 5 | import sys 6 | from awsauth import S3Auth 7 | 8 | PY3 = sys.version > '3' 9 | 10 | if PY3: 11 | from base64 import encodebytes as encodestring 12 | else: 13 | from base64 import encodestring 14 | 15 | 16 | TEST_BUCKET = 'testpolpol2' 17 | ACCESS_KEY = 'ACCESSKEYXXXXXXXXXXXX' 18 | SECRET_KEY = 'AWSSECRETKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' 19 | if 'AWS_ACCESS_KEY' in os.environ: 20 | ACCESS_KEY = os.environ['AWS_ACCESS_KEY'] 21 | if 'AWS_SECRET_KEY' in os.environ: 22 | SECRET_KEY = os.environ['AWS_SECRET_KEY'] 23 | 24 | 25 | class TestAWS(unittest.TestCase): 26 | def setUp(self): 27 | self.auth = S3Auth(ACCESS_KEY, SECRET_KEY) 28 | 29 | def get_content_md5(self, data): 30 | hashdig = hashlib.md5(data.encode('utf-8').strip()).digest() 31 | signature = encodestring(hashdig) 32 | if PY3: 33 | return signature.decode('utf-8').strip() 34 | return signature.strip() 35 | 36 | def test_put_get_delete(self): 37 | url = 'http://' + TEST_BUCKET + '.s3.amazonaws.com/myfile.txt' 38 | testdata = 'Sam is sweet' 39 | r = requests.put(url, data=testdata, auth=self.auth) 40 | self.assertEqual(r.status_code, 200) 41 | # Downloading a file 42 | r = requests.get(url, auth=self.auth) 43 | self.assertEqual(r.status_code, 200) 44 | self.assertEqual(r.text, 'Sam is sweet') 45 | # Removing a file 46 | r = requests.delete(url, auth=self.auth) 47 | self.assertEqual(r.status_code, 204) 48 | 49 | def test_put_get_delete_filnamehasplus(self): 50 | testdata = 'Sam is sweet' 51 | filename = 'my+file.txt' 52 | url = 'http://' + TEST_BUCKET + '.s3.amazonaws.com/%s' % (filename) 53 | r = requests.put(url, data=testdata, auth=self.auth) 54 | self.assertEqual(r.status_code, 200) 55 | # Downloading a file 56 | r = requests.get(url, auth=self.auth) 57 | self.assertEqual(r.status_code, 200) 58 | self.assertEqual(r.text, testdata) 59 | # Removing a file 60 | r = requests.delete(url, auth=self.auth) 61 | self.assertEqual(r.status_code, 204) 62 | 63 | def test_put_get_delete_filname_encoded(self): 64 | testdata = 'Sam is sweet' 65 | filename = 'my%20file.txt' 66 | url = 'http://' + TEST_BUCKET + '.s3.amazonaws.com/%s' % (filename) 67 | r = requests.put(url, data=testdata, auth=self.auth) 68 | self.assertEqual(r.status_code, 200) 69 | # Downloading a file 70 | r = requests.get(url, auth=self.auth) 71 | self.assertEqual(r.status_code, 200) 72 | self.assertEqual(r.text, testdata) 73 | # Removing a file 74 | r = requests.delete(url, auth=self.auth) 75 | self.assertEqual(r.status_code, 204) 76 | 77 | def test_put_get_delete_cors(self): 78 | url = 'http://' + TEST_BUCKET + '.s3.amazonaws.com/?cors' 79 | testdata = '\ 80 | \ 81 | *\ 82 | POST\ 83 | 3000\ 84 | Authorization\ 85 | \ 86 | ' 87 | headers = {'content-md5': self.get_content_md5(testdata)} 88 | r = requests.put(url, data=testdata, auth=self.auth, headers=headers) 89 | self.assertEqual(r.status_code, 200) 90 | # Downloading current cors configuration 91 | r = requests.get(url, auth=self.auth) 92 | self.assertEqual(r.status_code, 200) 93 | # Removing removing cors configuration 94 | r = requests.delete(url, auth=self.auth) 95 | self.assertEqual(r.status_code, 204) 96 | 97 | def test_put_get_delete_tagging(self): 98 | url = 'http://' + TEST_BUCKET + '.s3.amazonaws.com/?tagging' 99 | testdata = '\ 100 | \ 101 | \ 102 | Project\ 103 | Project 1\ 104 | \ 105 | \ 106 | ' 107 | headers = {'content-md5': self.get_content_md5(testdata)} 108 | r = requests.put(url, data=testdata, auth=self.auth, headers=headers) 109 | self.assertEqual(r.status_code, 204) 110 | # Downloading current cors configuration 111 | r = requests.get(url, auth=self.auth) 112 | self.assertEqual(r.status_code, 200) 113 | # Removing removing cors configuration 114 | r = requests.delete(url, auth=self.auth) 115 | self.assertEqual(r.status_code, 204) 116 | 117 | def test_put_get_notification(self): 118 | url = 'http://' + TEST_BUCKET + '.s3.amazonaws.com/?notification' 119 | testdata = '' 120 | headers = {'content-md5': self.get_content_md5(testdata)} 121 | r = requests.put(url, data=testdata, auth=self.auth, headers=headers) 122 | self.assertEqual(r.status_code, 200) 123 | # Downloading current cors configuration 124 | r = requests.get(url, auth=self.auth) 125 | self.assertEqual(r.status_code, 200) 126 | # No Delete ?notification API, empty 127 | # tag is default 128 | 129 | def test_canonical_string_not_using_encoded_query_params(self): 130 | url = 'https://bucket.ca.tier3.io/object-name?partNumber=1&uploadId=TFDSheOgTxC2Tsh1qVK73A%3D%3D' # NOQA 131 | headers = { 132 | 'Content-Length': 0, 133 | 'Accept-Encoding': 'gzip, deflate', 134 | 'Accept': '*/*', 135 | 'User-Agent': 'python-requests/2.7.0 CPython/2.7.6 Linux/3.13.0-24-generic', 136 | 'Connection': 'keep-alive', 137 | 'date': 'Fri, 21 Aug 2015 16:08:26 GMT', 138 | } 139 | method = 'PUT' 140 | canonical_string = self.auth.get_canonical_string(url, headers, method) 141 | self.assertTrue('TFDSheOgTxC2Tsh1qVK73A%3D%3D' not in canonical_string) 142 | self.assertTrue('TFDSheOgTxC2Tsh1qVK73A==' in canonical_string) 143 | 144 | url = 'https://bucket.ca.tier3.io/object-name?partNumber=1&uploadId=not%escaped' 145 | canonical_string = self.auth.get_canonical_string(url, headers, method) 146 | self.assertTrue('not%escaped' in canonical_string) 147 | 148 | if __name__ == '__main__': 149 | unittest.main() 150 | --------------------------------------------------------------------------------