├── MANIFEST.in ├── requirements.txt ├── .gitignore ├── test_requirements.txt ├── tox.ini ├── AUTHORS ├── .travis.yml ├── LICENSE ├── requests_ftp ├── __init__.py └── ftp.py ├── tests ├── unit │ └── test_status_code_interpret.py ├── conftest.py ├── simple_proxy.py ├── threadmgr.py ├── simple_ftpd.py ├── test_ftp_proxy.py └── test_ftp.py ├── setup.py └── README.rst /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests>=2.3.0 2 | wsgiref 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | venv/ 2 | env/ 3 | *.pyc 4 | *.egg-info 5 | .tox/ 6 | .cache/ -------------------------------------------------------------------------------- /test_requirements.txt: -------------------------------------------------------------------------------- 1 | pytest==2.8.7 2 | pytest-xdist==1.13.1 3 | pyftpdlib==1.5.0 4 | six==1.10.0 5 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27, py33, py34, py35, pypy 3 | 4 | [testenv] 5 | deps= -r{toxinidir}/test_requirements.txt 6 | commands= py.test {toxinidir}/tests/ 7 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Requests-FTP is written and maintained by Cory Benfield and the following 2 | contributors: 3 | 4 | - Aaron Meurer (@asmeurer) 5 | - David Shea (@dashea) 6 | - Priit Laes (@plaes) 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | python: 4 | - "2.7" 5 | - "3.3" 6 | - "3.4" 7 | - "3.5" 8 | - pypy 9 | 10 | sudo: false 11 | install: 12 | - "pip install -U pip setuptools" 13 | - "pip install ." 14 | - "pip install -r test_requirements.txt" 15 | script: "py.test tests/" 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2012 Cory Benfield 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /requests_ftp/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | requests_ftp FTP transport adapter 3 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 4 | 5 | requests_ftp is a library providing a very stupid transport adapter for use 6 | with the superawesome Python Requests library. 7 | 8 | For full documentation, please see the Github repository. 9 | 10 | :copyright: (c) 2012 by Cory Benfield 11 | :license: Apache 2.0, see LICENSE for more details. 12 | 13 | """ 14 | 15 | __title__ = 'requests-ftp' 16 | __version__ = '0.3.1' 17 | __author__ = 'Cory Benfield' 18 | __license__ = 'Apache 2.0' 19 | __copyright__ = 'Copyright 2012 Cory Benfield' 20 | 21 | from .ftp import FTPAdapter, FTPSession, monkeypatch_session 22 | -------------------------------------------------------------------------------- /tests/unit/test_status_code_interpret.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | from requests_ftp import ftp 3 | 4 | 5 | def test_simple_welcome(): 6 | assert ftp.get_status_code_from_code_response('200 Welcome') == 200 7 | 8 | 9 | def test_ftp_retr_multiline_resp(): 10 | ''' 11 | Example from NASA: 12 | ftp://lasco6.nascom.nasa.gov/pub/lasco/lastimage 13 | /lastimg_C2.gif 14 | The code received is: 15 | '226-File successfully transferred\n226 0.000 seconds' 16 | ''' 17 | assert ftp.get_status_code_from_code_response( 18 | '226-File successfully transferred\n226 0.000 seconds') == 226 19 | 20 | 21 | def test_ftp_retr_multiline_resp_inconsistent_code(): 22 | assert ftp.get_status_code_from_code_response( 23 | '200-File successfully transferred\n226 0.000 seconds') == 226 -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | import pytest 3 | import requests 4 | import requests_ftp 5 | from simple_ftpd import SimpleFTPServer 6 | from simple_proxy import ProxyServer 7 | import threading 8 | 9 | 10 | def pytest_configure(config): 11 | requests_ftp.monkeypatch_session() 12 | 13 | 14 | @pytest.fixture(scope='session') 15 | def ftpd(): 16 | ftp_server = SimpleFTPServer() 17 | ftp_server_thread = threading.Thread(target=ftp_server.serve_forever) 18 | ftp_server_thread.daemon = True 19 | ftp_server_thread.start() 20 | 21 | return ftp_server 22 | 23 | 24 | @pytest.fixture 25 | def session(): 26 | return requests.Session() 27 | 28 | 29 | @pytest.fixture(scope='session') 30 | def proxy(): 31 | proxy_server = ProxyServer() 32 | proxy_server_thread = threading.Thread(target=proxy_server.serve_forever) 33 | proxy_server_thread.daemon = True 34 | proxy_server_thread.start() 35 | 36 | return proxy_server 37 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | 4 | import os 5 | import sys 6 | 7 | try: 8 | from setuptools import setup 9 | except ImportError: 10 | from distutils.core import setup 11 | 12 | APP_NAME = 'requests-ftp' 13 | VERSION = '0.3.1' 14 | AUTHOR = 'Cory Benfield' 15 | LICENSE = 'Apache 2.0' 16 | 17 | try: 18 | LONG_DESC = open('README.rst').read() 19 | except IOError: 20 | LONG_DESC = '' 21 | 22 | # This wrapper stolen wholesale from Requests. 23 | if sys.argv[-1] == 'publish': 24 | os.system('python setup.py sdist upload') 25 | sys.exit() 26 | 27 | requires = ['requests'] 28 | 29 | settings = dict() 30 | 31 | settings.update( 32 | name=APP_NAME, 33 | version=VERSION, 34 | description='FTP Transport Adapter for Requests.', 35 | long_description=LONG_DESC, 36 | author=AUTHOR, 37 | author_email='cory@lukasa.co.uk', 38 | url='http://github.com/Lukasa/requests-ftp', 39 | packages=['requests_ftp'], 40 | package_data={'': ['LICENSE', 'AUTHORS', 'README.rst']}, 41 | package_dir={'requests_ftp': 'requests_ftp'}, 42 | include_package_data=True, 43 | install_requires=requires, 44 | license=LICENSE, 45 | classifiers=( 46 | 'Development Status :: 4 - Beta', 47 | 'Intended Audience :: Developers', 48 | 'Natural Language :: English', 49 | 'License :: OSI Approved :: Apache Software License', 50 | 'Programming Language :: Python', 51 | 'Programming Language :: Python :: 2.7', 52 | ), 53 | ) 54 | 55 | setup(**settings) 56 | -------------------------------------------------------------------------------- /tests/simple_proxy.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | from six.moves import SimpleHTTPServer, socketserver 3 | from six.moves.urllib.request import urlopen 4 | 5 | 6 | class ProxyHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): 7 | """ Proxy handler that calls a user-provided function and 8 | forwards the request via urllib. 9 | 10 | """ 11 | def do_GET(self): 12 | self.server.requests.add(self.path) 13 | data = urlopen(self.path).read() 14 | self.send_response(200) 15 | self.send_header('Content-Length', str(len(data))) 16 | self.end_headers() 17 | self.wfile.write(data) 18 | 19 | 20 | class ProxyServer(socketserver.TCPServer): 21 | allow_reuse_address = True 22 | 23 | port = property(lambda s: s._port, doc="TCP port the server is running on") 24 | 25 | def __init__(self): 26 | """ HTTP proxy server 27 | 28 | Can act as a proxy for whatever kinds of URLs urllib can handle. 29 | Stores requested paths in the set self.requests. 30 | 31 | :param cb: A user-provided method called for each proxy request. The 32 | method should take an argument containing the proxied 33 | request. The return value will be ignored. 34 | """ 35 | 36 | self.requests = set() 37 | 38 | # Create and bind a new TCPServer on any free port 39 | socketserver.TCPServer.__init__(self, ('', 0), ProxyHandler) 40 | self._port = self.socket.getsockname()[1] 41 | 42 | 43 | if __name__ == "__main__": 44 | def cb(path): 45 | print("Proxied request for %s" % path) 46 | proxy = ProxyServer(cb) 47 | print("Proxy running on port %d" % proxy.port) 48 | proxy.serve_forever() 49 | -------------------------------------------------------------------------------- /tests/threadmgr.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | import contextlib 3 | import six 4 | import socket 5 | import sys 6 | import threading 7 | 8 | 9 | class TestThread(threading.Thread): 10 | """ A Thread class that save exceptions raised by the thread. """ 11 | 12 | def __init__(self, exnlist, *args, **kwargs): 13 | self._exnlist = exnlist 14 | super(TestThread, self).__init__(*args, **kwargs) 15 | 16 | def run(self, *args, **kwargs): 17 | try: 18 | super(TestThread, self).run(*args, **kwargs) 19 | except: 20 | self._exnlist.append(sys.exc_info()) 21 | 22 | 23 | @contextlib.contextmanager 24 | def socketServer(target, event=None): 25 | """ Starts a server, running target in a new thread. 26 | 27 | Target is the thread target, and will receive the server socket 28 | and an optional Event as arguments. If event is not None, it 29 | will be passed as the second argument to target and will be 30 | set during the __exit__ phase. 31 | 32 | Returns the TCP port the server is running on. 33 | """ 34 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 35 | goevent = threading.Event() 36 | 37 | try: 38 | s.bind(('', 0)) 39 | port = s.getsockname()[1] 40 | 41 | if event: 42 | args = (s, goevent, event) 43 | else: 44 | args = (s, goevent) 45 | 46 | exn_list = [] 47 | server_thread = TestThread(exn_list, target=target, args=args) 48 | server_thread.daemon = True 49 | server_thread.start() 50 | 51 | goevent.wait(5) 52 | 53 | yield port 54 | 55 | if event: 56 | event.set() 57 | 58 | server_thread.join(1) 59 | 60 | # Check for and re-raise exceptions 61 | if exn_list: 62 | exc_info = exn_list[0] 63 | six.reraise(*exc_info) 64 | finally: 65 | s.close() 66 | -------------------------------------------------------------------------------- /tests/simple_ftpd.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | from pyftpdlib.authorizers import DummyAuthorizer 3 | from pyftpdlib.handlers import FTPHandler 4 | from pyftpdlib.servers import FTPServer 5 | 6 | import shutil 7 | import socket 8 | import tempfile 9 | 10 | 11 | class SimpleFTPServer(FTPServer): 12 | """Starts a simple FTP server on a random free port. """ 13 | 14 | ftp_user = property( 15 | lambda s: 'fakeusername', 16 | doc='User name added for authenticated connections') 17 | ftp_password = property(lambda s: 'qweqwe', doc='Password for ftp_user') 18 | 19 | # Set in __init__ 20 | anon_root = property(lambda s: s._anon_root, doc='Home directory for the anonymous user') 21 | ftp_home = property(lambda s: s._ftp_home, doc='Home directory for ftp_user') 22 | ftp_port = property(lambda s: s._ftp_port, doc='TCP port that the server is listening on') 23 | 24 | def __init__(self): 25 | # Create temp directories for the anonymous and authenticated roots 26 | self._anon_root = tempfile.mkdtemp() 27 | self._ftp_home = tempfile.mkdtemp() 28 | 29 | authorizer = DummyAuthorizer() 30 | authorizer.add_user(self.ftp_user, self.ftp_password, self.ftp_home, perm='elradfmwM') 31 | authorizer.add_anonymous(self.anon_root) 32 | 33 | handler = FTPHandler 34 | handler.authorizer = authorizer 35 | 36 | # Create a socket on any free port 37 | self._ftp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 38 | self._ftp_socket.bind(('', 0)) 39 | self._ftp_port = self._ftp_socket.getsockname()[1] 40 | 41 | # Create a new pyftpdlib server with the socket and handler we've configured 42 | FTPServer.__init__(self, self._ftp_socket, handler) 43 | 44 | def __del__(self): 45 | self.close_all() 46 | 47 | if hasattr(self, '_anon_root'): 48 | shutil.rmtree(self._anon_root, ignore_errors=True) 49 | 50 | if hasattr(self, '_ftp_home'): 51 | shutil.rmtree(self._ftp_home, ignore_errors=True) 52 | 53 | if __name__ == "__main__": 54 | server = SimpleFTPServer() 55 | print("FTPD running on port %d" % server.ftp_port) 56 | print("Anonymous root: %s" % server.anon_root) 57 | print("Authenticated root: %s" % server.ftp_home) 58 | server.serve_forever() 59 | -------------------------------------------------------------------------------- /tests/test_ftp_proxy.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | # Requesting an FTP resource through a proxy will actually create a request 4 | # to a HTTP proxy, and the proxy server will handle the FTP parts. This is 5 | # considered OK for some reason. 6 | 7 | import pytest 8 | 9 | import contextlib 10 | import os 11 | import tempfile 12 | import threading 13 | 14 | import requests 15 | 16 | from threadmgr import socketServer 17 | 18 | 19 | @contextlib.contextmanager 20 | def _prepareTestData(dir): 21 | """ Writes data to the given directory and returns a tuple of (tempname, testdata) """ 22 | with tempfile.NamedTemporaryFile(dir=dir) as f: 23 | # Write ourself the directory 24 | with open(__file__, "rb") as testinput: 25 | testdata = testinput.read() 26 | f.write(testdata) 27 | f.flush() 28 | 29 | yield (os.path.basename(f.name), testdata) 30 | 31 | 32 | def test_proxy_get(ftpd, proxy, session): 33 | # Create a file in the anonymous root and fetch it through a proxy 34 | with _prepareTestData(ftpd.anon_root) as (testfile, testdata): 35 | testurl = 'ftp://127.0.0.1:%d/%s' % (ftpd.ftp_port, testfile) 36 | response = session.get(testurl, proxies={'ftp': 'localhost:%d' % proxy.port}) 37 | 38 | assert response.status_code == requests.codes.ok 39 | 40 | # Check the length 41 | assert response.headers['Content-Length'] == str(len(testdata)) 42 | 43 | # Check the contents 44 | assert response.content == testdata 45 | 46 | # Check that it went through the proxy 47 | assert testurl in proxy.requests 48 | 49 | 50 | def test_proxy_connection_refused(ftpd, session): 51 | # Create and bind a socket but do not listen to ensure we have a port 52 | # that will refuse connections 53 | def target(s, goevent): 54 | goevent.set() 55 | 56 | with socketServer(target) as port: 57 | with pytest.raises(requests.exceptions.ConnectionError): 58 | session.get( 59 | 'ftp://127.0.0.1:%d/' % ftpd.ftp_port, 60 | proxies={'ftp': 'localhost:%d' % port}) 61 | 62 | 63 | def test_proxy_read_timeout(ftpd, session): 64 | # Create and accept a socket, but never respond 65 | def target(s, goevent, event): 66 | s.listen(1) 67 | goevent.set() 68 | (clientsock, _addr) = s.accept() 69 | try: 70 | event.wait(5) 71 | finally: 72 | clientsock.close() 73 | 74 | event = threading.Event() 75 | with socketServer(target, event) as port: 76 | with pytest.raises(requests.exceptions.ReadTimeout): 77 | session.get( 78 | 'ftp://127.0.0.1:%d' % ftpd.ftp_port, 79 | proxies={'ftp': 'localhost:%d' % port}, 80 | timeout=1) 81 | 82 | 83 | def test_proxy_connection_close(ftpd, session): 84 | # Create and accept a socket, then close it 85 | def target(s, goevent): 86 | s.listen(1) 87 | goevent.set() 88 | (clientsock, _addr) = s.accept() 89 | clientsock.close() 90 | 91 | with socketServer(target) as port: 92 | with pytest.raises(requests.exceptions.ConnectionError): 93 | session.get( 94 | 'ftp://127.0.0.1:%d/' % ftpd.ftp_port, 95 | proxies={'ftp': 'localhost:%d' % port}, 96 | timeout=1) 97 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://travis-ci.org/Lukasa/requests-ftp.svg?branch=master 2 | :target: https://travis-ci.org/Lukasa/requests-ftp 3 | 4 | Requests-FTP 5 | ============ 6 | 7 | Requests-FTP is an implementation of a very stupid FTP transport adapter for 8 | use with the awesome `Requests`_ Python library. 9 | 10 | This library is *not* intended to be an example of Transport Adapters best 11 | practices. This library was cowboyed together in about 4 hours of total work, 12 | has no tests, and relies on a few ugly hacks. Instead, it is intended as both 13 | a starting point for future development and a useful example for how to 14 | implement transport adapters. 15 | 16 | Here's how you use it: 17 | 18 | .. code-block:: pycon 19 | 20 | >>> import requests_ftp 21 | >>> s = requests_ftp.FTPSession() 22 | >>> resp = s.list('ftp://127.0.0.1/', auth=('Lukasa', 'notmypass')) 23 | >>> resp.status_code 24 | '226' 25 | >>> print resp.content 26 | ...snip... 27 | >>> resp = s.stor('ftp://127.0.0.1/test.txt', auth=('Lukasa', 'notmypass'), 28 | files={'file': open('report.txt', 'rb')}) 29 | 30 | 31 | Features 32 | -------- 33 | 34 | Almost none! 35 | 36 | - Adds the FTP LIST, STOR, RETR and NLST verbs via a new FTP transport adapter. 37 | - Provides a function that monkeypatches the Requests Session object, exposing 38 | helper methods much like the current ``Session.get()`` and ``Session.post()`` 39 | methods. 40 | - Piggybacks on standard Requests idioms: uses normal Requests models and 41 | access methods, including the tuple form of authentication. 42 | 43 | Does not provide: 44 | 45 | - Connection pooling! One new connection and multiple commands for each 46 | request, including authentication. **Super** inefficient. 47 | - SFTP. Security is for the weak. 48 | - Less common commands. 49 | 50 | Monkey Patching 51 | --------------- 52 | 53 | Sometimes you may want to call a library that uses requests with an ftp URL. 54 | First, check whether the library takes a session parameter. If it does, you 55 | can use either the FTPSession or FTPAdapter class directly, which is the preferred 56 | approach: 57 | 58 | .. code-block:: pycon 59 | 60 | >>> import requests_ftp 61 | >>> import some_library 62 | >>> s = requests_ftp.FTPSession() 63 | >>> resp = some_library.get('ftp://127.0.0.1/', auth=('Lukasa', 'notmypass'), session=s) 64 | 65 | If they do not, either modify the library to add a session parameter, or as an absolute 66 | last resort, use the `monkeypatch_session` function: 67 | 68 | .. code-block:: pycon 69 | 70 | >>> import requests_ftp 71 | >>> requests_ftp.monkeypatch_session() 72 | >>> import some_library 73 | >>> resp = some_library.get('ftp://127.0.0.1/', auth=('Lukasa', 'notmypass')) 74 | 75 | If you expect your code to be used as a library, take particular care to avoid the 76 | `monkeypatch_session` option. 77 | 78 | Important Notes 79 | --------------- 80 | 81 | Many corners have been cut in my rush to get this code finished. The most 82 | obvious problem is that this code does not have *any* tests. This is my highest 83 | priority for fixing. 84 | 85 | More notably, we have the following important caveats: 86 | 87 | - The design of the Requests Transport Adapater means that the STOR method 88 | has to un-encode a multipart form-data encoded body to get the file. This is 89 | painful, and I haven't tested this thoroughly, so it might not work. 90 | - **Massive** assumptions have been made in the use of the STOR method. This 91 | code assumes that there will only be one file included in the files argument. 92 | It also requires that you provide the filename to save as as part of the URL. 93 | This is single-handedly the most brittle part of this adapter. 94 | - This code is not optimised for performance AT ALL. There is some low-hanging 95 | fruit here: we should be able to connection pool relatively easily, and we 96 | can probably avoid making some of the requests we do. 97 | 98 | Contributing 99 | ------------ 100 | 101 | Please do! I would love for this to be developed further by anyone who is 102 | interested. Wherever possible, please provide unit tests for your work (yes, 103 | this is very much a 'do as I say, not as I do' kind of moment). Don't forget 104 | to add your name to AUTHORS. 105 | 106 | License 107 | ------- 108 | 109 | To maximise compatibility with Requests, this code is licensed under the Apache 110 | license. See LICENSE for more details. 111 | 112 | .. _`Requests`: https://github.com/kennethreitz/requests 113 | -------------------------------------------------------------------------------- /tests/test_ftp.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | import pytest 3 | 4 | import contextlib 5 | import os 6 | import tempfile 7 | import threading 8 | 9 | import requests 10 | 11 | from threadmgr import socketServer 12 | 13 | 14 | @contextlib.contextmanager 15 | def _prepareTestData(dir): 16 | """ Writes data to the given directory and returns a tuple of (tempname, testdata) """ 17 | with tempfile.NamedTemporaryFile(dir=dir) as f: 18 | # Write ourself to the directory 19 | with open(__file__, "rb") as testinput: 20 | testdata = testinput.read() 21 | f.write(testdata) 22 | f.flush() 23 | 24 | yield (os.path.basename(f.name), testdata) 25 | 26 | 27 | def test_basic_get(ftpd, session): 28 | # Create a file in the anonymous root and fetch it 29 | with _prepareTestData(ftpd.anon_root) as (testfile, testdata): 30 | response = session.get('ftp://127.0.0.1:%d/%s' % (ftpd.ftp_port, testfile)) 31 | 32 | assert response.status_code == requests.codes.ok 33 | 34 | # Check the length 35 | assert response.headers['Content-Length'] == str(len(testdata)) 36 | 37 | # Check the contents 38 | assert response.content == testdata 39 | 40 | 41 | def test_missing_get(ftpd, session): 42 | # Fetch a file that does not exist, look for a 404 43 | response = session.get("ftp://127.0.0.1:%d/no/such/path" % ftpd.ftp_port) 44 | assert response.status_code == requests.codes.not_found 45 | 46 | 47 | def test_authenticated_get(ftpd, session): 48 | # Create a file in the testuser's root and fetch it 49 | with _prepareTestData(dir=ftpd.ftp_home) as (testfile, testdata): 50 | response = session.get( 51 | 'ftp://127.0.0.1:%d/%s' % (ftpd.ftp_port, testfile), 52 | auth=(ftpd.ftp_user, ftpd.ftp_password)) 53 | 54 | assert response.status_code == requests.codes.ok 55 | assert response.headers['Content-Length'] == str(len(testdata)) 56 | 57 | # Check the contents 58 | assert response.content == testdata 59 | 60 | 61 | def test_basic_retr(ftpd, session): 62 | # Fetch a file with the retr command 63 | with _prepareTestData(dir=ftpd.anon_root) as (testfile, testdata): 64 | response = session.retr("ftp://127.0.0.1:%d/%s" % (ftpd.ftp_port, testfile)) 65 | 66 | assert response.status_code == 226 67 | 68 | 69 | def test_head(ftpd, session): 70 | # Perform a HEAD over an anonymous connection 71 | with _prepareTestData(dir=ftpd.anon_root) as (testfile, testdata): 72 | response = session.head('ftp://127.0.0.1:%d/%s' % (ftpd.ftp_port, testfile)) 73 | 74 | assert response.status_code == requests.codes.ok 75 | assert response.headers['Content-Length'] == str(len(testdata)) 76 | 77 | 78 | def test_connection_refused(session): 79 | # Create and bind a socket but do not listen to ensure we have a port 80 | # that will refuse connections 81 | def target(s, goevent): 82 | goevent.set() 83 | 84 | with socketServer(target) as port: 85 | with pytest.raises(requests.exceptions.ConnectionError): 86 | session.get('ftp://127.0.0.1:%d/' % port) 87 | 88 | 89 | def test_connection_timeout(session): 90 | # Create, bind, and listen on a socket, but never accept 91 | def target(s, goevent): 92 | s.listen(1) 93 | goevent.set() 94 | 95 | with socketServer(target) as port: 96 | with pytest.raises(requests.exceptions.ConnectTimeout): 97 | session.get('ftp://127.0.0.1:%d/' % port, timeout=1) 98 | 99 | 100 | def test_login_timeout(session): 101 | # Create and accept a socket, but stop responding after sending 102 | # the welcome 103 | def target(s, goevent, event): 104 | s.listen(1) 105 | goevent.set() 106 | (clientsock, _addr) = s.accept() 107 | try: 108 | clientsock.send(b'220 welcome\r\n') 109 | # Wait for the exception to be raised in the client 110 | event.wait(5) 111 | finally: 112 | clientsock.close() 113 | 114 | event = threading.Event() 115 | with socketServer(target, event) as port: 116 | with pytest.raises(requests.exceptions.ReadTimeout): 117 | session.get('ftp://127.0.0.1:%d/' % port, timeout=1) 118 | 119 | 120 | def test_connection_close(session): 121 | # Create and accept a socket, then close it after the welcome 122 | def target(s, goevent): 123 | s.listen(1) 124 | goevent.set() 125 | (clientsock, _addr) = s.accept() 126 | try: 127 | clientsock.send(b'220 welcome\r\n') 128 | finally: 129 | clientsock.close() 130 | 131 | with socketServer(target) as port: 132 | with pytest.raises(requests.exceptions.ConnectionError): 133 | session.get('ftp://127.0.0.1:%d/' % port) 134 | 135 | 136 | def test_invalid_response(session): 137 | # Send an invalid welcome 138 | def target(s, goevent): 139 | s.listen(1) 140 | goevent.set() 141 | (clientsock, _addr) = s.accept() 142 | try: 143 | clientsock.send(b'no code\r\n') 144 | finally: 145 | clientsock.close() 146 | 147 | with socketServer(target) as port: 148 | with pytest.raises(requests.exceptions.RequestException): 149 | session.get('ftp://127.0.0.1:%d/' % port) 150 | 151 | 152 | def test_invalid_code(session): 153 | # Send a welcome, then reply with something weird to the USER command 154 | def target(s, goevent): 155 | s.listen(1) 156 | goevent.set() 157 | (clientsock, _addr) = s.accept() 158 | try: 159 | clientsock.send(b'220 welcome\r\n') 160 | clientsock.recv(1024) 161 | clientsock.send(b'125 this makes no sense\r\n') 162 | finally: 163 | clientsock.close() 164 | 165 | with socketServer(target) as port: 166 | with pytest.raises(requests.exceptions.RequestException): 167 | session.get('ftp://127.0.0.1:%d/' % port) 168 | 169 | 170 | def test_unavailable(session): 171 | def target(s, goevent): 172 | s.listen(1) 173 | goevent.set() 174 | (clientsock, _addr) = s.accept() 175 | try: 176 | clientsock.send(b'421 go away\r\n') 177 | finally: 178 | clientsock.close() 179 | 180 | with socketServer(target) as port: 181 | response = session.get('ftp://127.0.0.1:%d/' % port) 182 | assert response.status_code == requests.codes.unavailable 183 | -------------------------------------------------------------------------------- /requests_ftp/ftp.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | import base64 3 | import cgi 4 | import ftplib 5 | from io import BytesIO 6 | import os 7 | import requests 8 | import socket 9 | import logging 10 | 11 | from requests import Response, codes 12 | from requests.compat import urlparse 13 | from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout 14 | from requests.exceptions import RequestException 15 | from requests.hooks import dispatch_hook 16 | from requests.utils import prepend_scheme_if_needed 17 | 18 | log = logging.getLogger(__name__) 19 | 20 | 21 | class FTPSession(requests.Session): 22 | 23 | def __init__(self): 24 | super(FTPSession, self).__init__() 25 | self.mount('ftp://', FTPAdapter()) 26 | 27 | # Define our helper methods. 28 | def list(self, url, **kwargs): 29 | '''Sends an FTP LIST. Returns a Response object.''' 30 | return self.request('LIST', url, **kwargs) 31 | 32 | def retr(self, url, **kwargs): 33 | '''Sends an FTP RETR for a given url. Returns a Response object whose 34 | content field contains the binary data.''' 35 | return self.request('RETR', url, **kwargs) 36 | 37 | def stor(self, url, files=None, **kwargs): 38 | '''Sends an FTP STOR to a given URL. Returns a Response object. Expects 39 | to be given one file by the standard Requests method. The remote 40 | filename will be given by the URL provided.''' 41 | return self.request('STOR', url, files=files, **kwargs) 42 | 43 | def nlst(self, url, **kwargs): 44 | '''Sends an FTP NLST. Returns a Response object.''' 45 | return self.request('NLST', url, **kwargs) 46 | 47 | def size(self, url, **kwargs): 48 | '''Sends an FTP SIZE. Returns a decimal number.''' 49 | return self.request('SIZE', url, **kwargs) 50 | 51 | def mlsd(self, url, **kwargs): 52 | '''Sends an FTP MLSD. Returns a Response object.''' 53 | return self.request('MLSD', url, **kwargs) 54 | 55 | 56 | def monkeypatch_session(): 57 | '''Monkeypatch Requests Sessions to provide all the helper 58 | methods needed for use with FTP.''' 59 | 60 | requests.Session = FTPSession 61 | return 62 | 63 | 64 | def parse_multipart_files(request): 65 | '''Given a prepared request, return a file-like object containing the 66 | original data. This is pretty hacky.''' 67 | # Start by grabbing the pdict. 68 | _, pdict = cgi.parse_header(request.headers['Content-Type']) 69 | 70 | # Now, wrap the multipart data in a BytesIO buffer. This is annoying. 71 | buf = BytesIO() 72 | buf.write(request.body) 73 | buf.seek(0) 74 | 75 | # Parse the data. Simply take the first file. 76 | data = cgi.parse_multipart(buf, pdict) 77 | _, filedata = data.popitem() 78 | buf.close() 79 | 80 | # Get a BytesIO now, and write the file into it. 81 | buf = BytesIO() 82 | buf.write(''.join(filedata)) 83 | buf.seek(0) 84 | 85 | return buf 86 | 87 | 88 | def data_callback_factory(variable): 89 | '''Returns a callback suitable for use by the FTP library. This callback 90 | will repeatedly save data into the variable provided to this function. This 91 | variable should be a file-like structure.''' 92 | def callback(data): 93 | variable.write(data) 94 | if hasattr(variable, "content_len"): 95 | variable.content_len += len(data) 96 | else: 97 | variable.content_len = len(data) 98 | 99 | return 100 | 101 | return callback 102 | 103 | 104 | def build_text_response(request, data, code): 105 | '''Build a response for textual data.''' 106 | return build_response(request, data, code, 'ascii') 107 | 108 | 109 | def build_binary_response(request, data, code): 110 | '''Build a response for data whose encoding is unknown.''' 111 | return build_response(request, data, code, None) 112 | 113 | 114 | def get_status_code_from_code_response(code): 115 | ''' 116 | The idea is to handle complicated code response (even multi lines). 117 | We get the status code in two ways: 118 | - extracting the code from the last valid line in the response 119 | - getting it from the 3 first digits in the code 120 | After a comparison between the two values, 121 | we can safely set the code or raise a warning. 122 | 123 | Examples: 124 | - get_status_code_from_code_response('200 Welcome') == 200 125 | 126 | - multi_line_code = '226-File successfully transferred\n226 0.000 seconds' 127 | get_status_code_from_code_response(multi_line_code) == 226 128 | 129 | - multi_line_with_code_conflicts = '200-File successfully transferred\n226 0.000 seconds' 130 | get_status_code_from_code_response(multi_line_with_code_conflicts) == 226 131 | 132 | For more detail see RFC 959, page 36, on multi-line responses: 133 | https://www.ietf.org/rfc/rfc959.txt 134 | 135 | "Thus the format for multi-line replies is that the first line 136 | will begin with the exact required reply code, followed 137 | immediately by a Hyphen, "-" (also known as Minus), followed by 138 | text. The last line will begin with the same code, followed 139 | immediately by Space , optionally some text, and the Telnet 140 | end-of-line code." 141 | ''' 142 | last_valid_line_from_code = [line for line in code.split('\n') if line][-1] 143 | status_code_from_last_line = int(last_valid_line_from_code.split()[0]) 144 | status_code_from_first_digits = int(code[:3]) 145 | if status_code_from_last_line != status_code_from_first_digits: 146 | log.warning( 147 | 'FTP response status code seems to be inconsistent.\n' 148 | 'Code received: %s, extracted: %s and %s', 149 | code, 150 | status_code_from_last_line, 151 | status_code_from_first_digits 152 | ) 153 | return status_code_from_last_line 154 | 155 | 156 | def build_response(request, data, code, encoding): 157 | '''Builds a response object from the data returned by ftplib, using the 158 | specified encoding.''' 159 | response = Response() 160 | 161 | response.encoding = encoding 162 | 163 | # Fill in some useful fields. 164 | response.raw = data 165 | response.url = request.url 166 | response.request = request 167 | response.status_code = get_status_code_from_code_response(code) 168 | 169 | if hasattr(data, "content_len"): 170 | response.headers['Content-Length'] = str(data.content_len) 171 | 172 | # Make sure to seek the file-like raw object back to the start. 173 | response.raw.seek(0) 174 | 175 | # Run the response hook. 176 | response = dispatch_hook('response', request.hooks, response) 177 | return response 178 | 179 | 180 | class FTPAdapter(requests.adapters.BaseAdapter): 181 | '''A Requests Transport Adapter that handles FTP urls.''' 182 | 183 | def __init__(self): 184 | super(FTPAdapter, self).__init__() 185 | 186 | # Build a dictionary keyed off the methods we support in upper case. 187 | # The values of this dictionary should be the functions we use to 188 | # send the specific queries. 189 | self.func_table = { 190 | 'LIST': self.list, 191 | 'RETR': self.retr, 192 | 'STOR': self.stor, 193 | 'NLST': self.nlst, 194 | 'SIZE': self.size, 195 | 'HEAD': self.head, 196 | 'GET': self.get, 197 | 'MLSD': self.mlsd, 198 | } 199 | 200 | def send(self, request, **kwargs): 201 | '''Sends a PreparedRequest object over FTP. Returns a response object. 202 | ''' 203 | # Get the authentication from the prepared request, if any. 204 | auth = self.get_username_password_from_header(request) 205 | 206 | # Next, get the host and the path. 207 | scheme, host, port, path = self.get_host_and_path_from_url(request) 208 | 209 | # Sort out the timeout. 210 | timeout = kwargs.get('timeout', None) 211 | 212 | # Look for a proxy 213 | proxies = kwargs.get('proxies', {}) 214 | proxy = proxies.get(scheme) 215 | 216 | # If there is a proxy, then we actually want to make a HTTP request 217 | if proxy: 218 | return self.send_proxy(request, proxy, **kwargs) 219 | 220 | # Establish the connection and login if needed. 221 | self.conn = ftplib.FTP() 222 | 223 | # Use a flag to distinguish read vs connection timeouts, and a flat set 224 | # of except blocks instead of a nested try-except, because python 3 225 | # exception chaining makes things weird 226 | connected = False 227 | 228 | try: 229 | self.conn.connect(host, port, timeout) 230 | connected = True 231 | 232 | if auth is not None: 233 | self.conn.login(auth[0], auth[1]) 234 | else: 235 | self.conn.login() 236 | 237 | # Get the method and attempt to find the function to call. 238 | resp = self.func_table[request.method](path, request) 239 | except socket.timeout as e: 240 | # requests distinguishes between connection timeouts and others 241 | if connected: 242 | raise ReadTimeout(e, request=request) 243 | else: 244 | raise ConnectTimeout(e, request=request) 245 | # ftplib raises EOFError if the connection is unexpectedly closed. 246 | # Convert that or any other socket error to a ConnectionError. 247 | except (EOFError, socket.error) as e: 248 | raise ConnectionError(e, request=request) 249 | # Raised for 5xx errors. FTP uses 550 for both ENOENT and EPERM type 250 | # errors, so just translate all of these into a http-ish 404 251 | except ftplib.error_perm as e: 252 | # The exception message is probably from the server, so if it's 253 | # non-ascii, who knows what the encoding is. Latin1 has the 254 | # advantage of not being able to fail. 255 | resp = build_text_response(request, 256 | BytesIO(str(e).encode('latin1')), 257 | str(codes.not_found)) 258 | # 4xx reply, translate to a http 503 259 | except ftplib.error_temp as e: 260 | resp = build_text_response(request, 261 | BytesIO(str(e).encode('latin1')), 262 | str(codes.unavailable)) 263 | # error_reply is an unexpected status code, and error_proto is an 264 | # invalid status code. Error is the generic ftplib error, usually 265 | # raised when a line is too long. Translate all of them to a generic 266 | # RequestException 267 | except (ftplib.error_reply, ftplib.error_proto, ftplib.Error) as e: 268 | raise RequestException(e, request=request) 269 | 270 | # Return the response. 271 | return resp 272 | 273 | def close(self): 274 | '''Dispose of any internal state.''' 275 | # Currently this is a no-op. 276 | pass 277 | 278 | def send_proxy(self, request, proxy, **kwargs): 279 | '''Send an FTP request through a HTTP proxy''' 280 | # Direct the request through a HTTP adapter instead 281 | proxy_url = prepend_scheme_if_needed(proxy, 'http') 282 | s = requests.Session() 283 | adapter = s.get_adapter(proxy_url) 284 | 285 | try: 286 | return adapter.send(request, **kwargs) 287 | finally: 288 | adapter.close() 289 | 290 | def list(self, path, request): 291 | '''Executes the FTP LIST command on the given path.''' 292 | data = BytesIO() 293 | 294 | # To ensure the BytesIO object gets cleaned up, we need to alias its 295 | # close method to the release_conn() method. This is a dirty hack, but 296 | # there you go. 297 | data.release_conn = data.close 298 | 299 | self.conn.cwd(path) 300 | code = self.conn.retrbinary('LIST', data_callback_factory(data)) 301 | 302 | # When that call has finished executing, we'll have all our data. 303 | response = build_text_response(request, data, code) 304 | 305 | # Close the connection. 306 | self.conn.close() 307 | 308 | return response 309 | 310 | def mlsd(self, path, request): 311 | '''Executes the FTP MLSD command on the given path.''' 312 | data = BytesIO() 313 | 314 | # To ensure the BytesIO gets cleaned up, we need to alias its close 315 | # method. See self.list(). 316 | data.release_conn = data.close 317 | 318 | self.conn.cwd(path) 319 | code = self.conn.retrbinary('MLSD', data_callback_factory(data)) 320 | 321 | # When that call has finished executing, we'll have all our data. 322 | response = build_text_response(request, data, code) 323 | 324 | # Close the connection. 325 | self.conn.close() 326 | 327 | return response 328 | 329 | def retr(self, path, request): 330 | '''Executes the FTP RETR command on the given path.''' 331 | data = BytesIO() 332 | 333 | # To ensure the BytesIO gets cleaned up, we need to alias its close 334 | # method. See self.list(). 335 | data.release_conn = data.close 336 | 337 | code = self.conn.retrbinary( 338 | 'RETR ' + path, data_callback_factory(data)) 339 | 340 | response = build_binary_response(request, data, code) 341 | 342 | # Close the connection. 343 | self.conn.close() 344 | 345 | return response 346 | 347 | def get(self, path, request): 348 | '''Executes the FTP RETR command on the given path. 349 | 350 | This is the same as retr except that the FTP server code is 351 | converted to a HTTP 200. 352 | ''' 353 | 354 | response = self.retr(path, request) 355 | 356 | # Errors are handled in send(), so assume everything is ok if we 357 | # made it this far 358 | response.status_code = codes.ok 359 | return response 360 | 361 | def size(self, path, request): 362 | '''Executes the FTP SIZE command on the given path.''' 363 | self.conn.voidcmd( 364 | 'TYPE I') # SIZE is not usually allowed in ASCII mode 365 | 366 | size = self.conn.size(path) 367 | 368 | if not str(size).isdigit(): 369 | self.conn.close() 370 | return None 371 | 372 | data = BytesIO(bytes(size)) 373 | # To ensure the BytesIO gets cleaned up, we need to alias its close 374 | # method to the release_conn() method. This is a dirty hack, but there 375 | # you go. 376 | data.release_conn = data.close 377 | data.content_len = size 378 | 379 | response = build_text_response(request, data, '213') 380 | 381 | self.conn.close() 382 | 383 | return response 384 | 385 | def head(self, path, request): 386 | '''Executes the FTP SIZE command on the given path. 387 | 388 | This is the same as size except that the FTP server code is 389 | converted to a HTTP 200. 390 | ''' 391 | 392 | response = self.size(path, request) 393 | response.status_code = codes.ok 394 | return response 395 | 396 | def stor(self, path, request): 397 | '''Executes the FTP STOR command on the given path.''' 398 | 399 | # First, get the file handle. We assume (bravely) 400 | # that there is only one file to be sent to a given URL. We also 401 | # assume that the filename is sent as part of the URL, not as part of 402 | # the files argument. Both of these assumptions are rarely correct, 403 | # but they are easy. 404 | data = parse_multipart_files(request) 405 | 406 | # Split into the path and the filename. 407 | path, filename = os.path.split(path) 408 | 409 | # Switch directories and upload the data. 410 | self.conn.cwd(path) 411 | code = self.conn.storbinary('STOR ' + filename, data) 412 | 413 | # Close the connection and build the response. 414 | self.conn.close() 415 | 416 | response = build_binary_response(request, BytesIO(), code) 417 | 418 | return response 419 | 420 | def nlst(self, path, request): 421 | '''Executes the FTP NLST command on the given path.''' 422 | data = BytesIO() 423 | 424 | # Alias the close method. 425 | data.release_conn = data.close 426 | 427 | self.conn.cwd(path) 428 | code = self.conn.retrbinary('NLST', data_callback_factory(data)) 429 | 430 | # When that call has finished executing, we'll have all our data. 431 | response = build_text_response(request, data, code) 432 | 433 | # Close the connection. 434 | self.conn.close() 435 | 436 | return response 437 | 438 | def get_username_password_from_header(self, request): 439 | '''Given a PreparedRequest object, reverse the process of adding HTTP 440 | Basic auth to obtain the username and password. Allows the FTP adapter 441 | to piggyback on the basic auth notation without changing the control 442 | flow.''' 443 | auth_header = request.headers.get('Authorization') 444 | 445 | if auth_header: 446 | # The basic auth header is of the form 'Basic xyz'. We want the 447 | # second part. Check that we have the right kind of auth though. 448 | encoded_components = auth_header.split()[:2] 449 | if encoded_components[0] != 'Basic': 450 | raise AuthError('Invalid form of Authentication used.') 451 | else: 452 | encoded = encoded_components[1] 453 | 454 | # Decode the base64 encoded string. 455 | decoded = base64.b64decode(encoded) 456 | 457 | # The auth string was encoded to bytes by requests using latin1, 458 | # and will be encoded to bytes by ftplib (in python 3) using 459 | # latin1. In the meantime, use a str 460 | decoded = decoded.decode('latin1') 461 | 462 | # The string is of the form 'username:password'. Split on the 463 | # colon. 464 | components = decoded.split(':') 465 | username = components[0] 466 | password = components[1] 467 | return (username, password) 468 | else: 469 | # No auth header. Return None. 470 | return None 471 | 472 | def get_host_and_path_from_url(self, request): 473 | '''Given a PreparedRequest object, split the URL in such a manner as to 474 | determine the host and the path. This is a separate method to wrap some 475 | of urlparse's craziness.''' 476 | url = request.url 477 | # scheme, netloc, path, params, query, fragment = urlparse(url) 478 | parsed = urlparse(url) 479 | scheme = parsed.scheme 480 | path = parsed.path 481 | 482 | # If there is a slash on the front of the path, chuck it. 483 | if path.startswith('/'): 484 | path = path[1:] 485 | 486 | host = parsed.hostname 487 | port = parsed.port or 0 488 | 489 | return (scheme, host, port, path) 490 | 491 | 492 | class AuthError(Exception): 493 | '''Denotes an error with authentication.''' 494 | pass 495 | --------------------------------------------------------------------------------