├── .coveragerc ├── .gitignore ├── AUTHORS ├── LICENSE ├── README.md ├── docs └── source │ ├── conf.py │ └── index.rst ├── example └── example.py ├── pywebhdfs ├── __init__.py ├── errors.py ├── operations.py ├── tests │ ├── test_errors.py │ └── test_webhdfs.py └── webhdfs.py ├── setup.cfg ├── setup.py ├── tools ├── pip-requires └── test-requires └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | omit = etc/*,setup.py,*egg*,.tox/*,pywebhdfs/tests/*,pywebhdfs/example/* -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | .coverage 3 | *.pyc 4 | .idea 5 | *.iml 6 | pywebhdfs.egg-info 7 | .tox 8 | *.DS_Store* 9 | nosetests.xml 10 | pywebhdfs/coverage.xml 11 | cache-store 12 | cache-config 13 | *~ -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Many thanks to all the mighty fine contributors to the project, 2 | listed below (in no particular order): 3 | 4 | * Steven Gonzales (stevendgonzales) 5 | * Carlos Marin (carlosmarin) 6 | * Ashish Walia (waliaashish85) 7 | * Eduard Iskandarov (toidi) 8 | * Tatsuo Ikeda (ikeikeikeike) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2013 by Rackspace Hosting, Inc. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyWebHDFS 2 | 3 | **Note:** PyWebHDFS active development has moved to: [https://github.com/pywebhdfs/pywebhdfs](https://github.com/pywebhdfs/pywebhdfs) 4 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # pywebhdfs documentation build configuration file, created by 4 | # sphinx-quickstart on Mon Jun 10 10:20:37 2013. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys 15 | import os 16 | 17 | # If extensions (or modules to document with autodoc) are in another directory, 18 | # add these directories to sys.path here. If the directory is relative to the 19 | # documentation root, use os.path.abspath to make it absolute, like shown here. 20 | #sys.path.insert(0, os.path.abspath('.')) 21 | 22 | # -- General configuration ---------------------------------------------------- 23 | 24 | # If your documentation needs a minimal Sphinx version, state it here. 25 | #needs_sphinx = '1.0' 26 | 27 | # Add any Sphinx extension module names here, as strings. They can be extension 28 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 29 | extensions = [ 30 | 'sphinx.ext.autodoc', 31 | 'sphinx.ext.intersphinx', 32 | 'sphinx.ext.ifconfig'] 33 | 34 | # Add any paths that contain templates here, relative to this directory. 35 | templates_path = ['_templates'] 36 | 37 | # The suffix of source filenames. 38 | source_suffix = '.rst' 39 | 40 | # The encoding of source files. 41 | #source_encoding = 'utf-8-sig' 42 | 43 | # The master toctree document. 44 | master_doc = 'index' 45 | 46 | # General information about the project. 47 | project = u'pywebhdfs' 48 | copyright = u'2013, Steven D. Gonzales' 49 | 50 | # The version info for the project you're documenting, acts as replacement for 51 | # |version| and |release|, also used in various other places throughout the 52 | # built documents. 53 | # 54 | # The short X.Y version. 55 | version = '0.2' 56 | # The full version, including alpha/beta/rc tags. 57 | release = '0.2.3' 58 | 59 | # The language for content autogenerated by Sphinx. Refer to documentation 60 | # for a list of supported languages. 61 | #language = None 62 | 63 | # There are two options for replacing |today|: either, you set today to some 64 | # non-false value, then it is used: 65 | #today = '' 66 | # Else, today_fmt is used as the format for a strftime call. 67 | #today_fmt = '%B %d, %Y' 68 | 69 | # List of patterns, relative to source directory, that match files and 70 | # directories to ignore when looking for source files. 71 | exclude_patterns = [] 72 | 73 | # The reST default role (used for this markup: `text`) to use for all documents 74 | #default_role = None 75 | 76 | # If true, '()' will be appended to :func: etc. cross-reference text. 77 | #add_function_parentheses = True 78 | 79 | # If true, the current module name will be prepended to all description 80 | # unit titles (such as .. function::). 81 | #add_module_names = True 82 | 83 | # If true, sectionauthor and moduleauthor directives will be shown in the 84 | # output. They are ignored by default. 85 | #show_authors = False 86 | 87 | # The name of the Pygments (syntax highlighting) style to use. 88 | pygments_style = 'sphinx' 89 | 90 | # A list of ignored prefixes for module index sorting. 91 | #modindex_common_prefix = [] 92 | 93 | # If true, keep warnings as "system message" paragraphs in the built documents. 94 | #keep_warnings = False 95 | 96 | 97 | # -- Options for HTML output -------------------------------------------------- 98 | 99 | # The theme to use for HTML and HTML Help pages. See the documentation for 100 | # a list of builtin themes. 101 | html_theme = 'default' 102 | 103 | # Theme options are theme-specific and customize the look and feel of a theme 104 | # further. For a list of options available for each theme, see the 105 | # documentation. 106 | #html_theme_options = {} 107 | 108 | # Add any paths that contain custom themes here, relative to this directory. 109 | #html_theme_path = [] 110 | 111 | # The name for this set of Sphinx documents. If None, it defaults to 112 | # " v documentation". 113 | #html_title = None 114 | 115 | # A shorter title for the navigation bar. Default is the same as html_title. 116 | #html_short_title = None 117 | 118 | # The name of an image file (relative to this directory) to place at the top 119 | # of the sidebar. 120 | #html_logo = None 121 | 122 | # The name of an image file (within the static path) to use as favicon of the 123 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 124 | # pixels large. 125 | #html_favicon = None 126 | 127 | # Add any paths that contain custom static files (such as style sheets) here, 128 | # relative to this directory. They are copied after the builtin static files, 129 | # so a file named "default.css" will overwrite the builtin "default.css". 130 | html_static_path = ['_static'] 131 | 132 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 133 | # using the given strftime format. 134 | #html_last_updated_fmt = '%b %d, %Y' 135 | 136 | # If true, SmartyPants will be used to convert quotes and dashes to 137 | # typographically correct entities. 138 | #html_use_smartypants = True 139 | 140 | # Custom sidebar templates, maps document names to template names. 141 | #html_sidebars = {} 142 | 143 | # Additional templates that should be rendered to pages, maps page names to 144 | # template names. 145 | #html_additional_pages = {} 146 | 147 | # If false, no module index is generated. 148 | #html_domain_indices = True 149 | 150 | # If false, no index is generated. 151 | #html_use_index = True 152 | 153 | # If true, the index is split into individual pages for each letter. 154 | #html_split_index = False 155 | 156 | # If true, links to the reST sources are added to the pages. 157 | #html_show_sourcelink = True 158 | 159 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 160 | #html_show_sphinx = True 161 | 162 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 163 | #html_show_copyright = True 164 | 165 | # If true, an OpenSearch description file will be output, and all pages will 166 | # contain a tag referring to it. The value of this option must be the 167 | # base URL from which the finished HTML is served. 168 | #html_use_opensearch = '' 169 | 170 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 171 | #html_file_suffix = None 172 | 173 | # Output file base name for HTML help builder. 174 | htmlhelp_basename = 'pywebhdfsdoc' 175 | 176 | 177 | # -- Options for LaTeX output ------------------------------------------------- 178 | 179 | latex_elements = { 180 | # The paper size ('letterpaper' or 'a4paper'). 181 | #'papersize': 'letterpaper', 182 | 183 | # The font size ('10pt', '11pt' or '12pt'). 184 | #'pointsize': '10pt', 185 | 186 | # Additional stuff for the LaTeX preamble. 187 | #'preamble': '', 188 | } 189 | 190 | # Grouping the document tree into LaTeX files. List of tuples 191 | # (source start file, target name, title, author, documentclass [howto/manual]) 192 | latex_documents = [ 193 | ('index', 'pywebhdfs.tex', u'pywebhdfs Documentation', 194 | u'Steven D. Gonzales', 'manual'), 195 | ] 196 | 197 | # The name of an image file (relative to this directory) to place at the top of 198 | # the title page. 199 | #latex_logo = None 200 | 201 | # For "manual" documents, if this is true, then toplevel headings are parts, 202 | # not chapters. 203 | #latex_use_parts = False 204 | 205 | # If true, show page references after internal links. 206 | #latex_show_pagerefs = False 207 | 208 | # If true, show URL addresses after external links. 209 | #latex_show_urls = False 210 | 211 | # Documents to append as an appendix to all manuals. 212 | #latex_appendices = [] 213 | 214 | # If false, no module index is generated. 215 | #latex_domain_indices = True 216 | 217 | 218 | # -- Options for manual page output ------------------------------------------- 219 | 220 | # One entry per manual page. List of tuples 221 | # (source start file, name, description, authors, manual section). 222 | man_pages = [ 223 | ('index', 'pywebhdfs', u'pywebhdfs Documentation', 224 | [u'Steven D. Gonzales'], 1) 225 | ] 226 | 227 | # If true, show URL addresses after external links. 228 | #man_show_urls = False 229 | 230 | 231 | # -- Options for Texinfo output ----------------------------------------------- 232 | 233 | # Grouping the document tree into Texinfo files. List of tuples 234 | # (source start file, target name, title, author, 235 | # dir menu entry, description, category) 236 | texinfo_documents = [ 237 | ('index', 'pywebhdfs', u'pywebhdfs Documentation', 238 | u'Steven D. Gonzales', 'pywebhdfs', 'One line description of project.', 239 | 'Miscellaneous'), 240 | ] 241 | 242 | # Documents to append as an appendix to all manuals. 243 | #texinfo_appendices = [] 244 | 245 | # If false, no module index is generated. 246 | #texinfo_domain_indices = True 247 | 248 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 249 | #texinfo_show_urls = 'footnote' 250 | 251 | # If true, do not generate a @detailmenu in the "Top" node's menu. 252 | #texinfo_no_detailmenu = False 253 | 254 | 255 | # Example configuration for intersphinx: refer to the Python standard library. 256 | intersphinx_mapping = {'http://docs.python.org/': None} 257 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. pywebhdfs documentation master file, created by 2 | sphinx-quickstart on Mon Jun 10 10:20:37 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | pywebhdfs 0.2.3 documentation 7 | ===================================== 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | .. autoclass:: pywebhdfs.webhdfs.PyWebHdfsClient 14 | :members: __init__, create_file, append_file, read_file, make_dir, rename_file_dir, delete_file_dir, get_file_dir_status, list_dir 15 | -------------------------------------------------------------------------------- /example/example.py: -------------------------------------------------------------------------------- 1 | from pywebhdfs.webhdfs import PyWebHdfsClient 2 | import logging 3 | 4 | logging.basicConfig(level=logging.DEBUG) 5 | _LOG = logging.getLogger(__name__) 6 | 7 | 8 | example_dir = 'user/hdfs/example_dir' 9 | example_file = '{dir}/example.txt'.format(dir=example_dir) 10 | example_data = '01010101010101010101010101010101010101010101\n' 11 | rename_dir = 'user/hdfs/example_rename' 12 | 13 | 14 | #create a new client instance 15 | hdfs = PyWebHdfsClient(host='localhost', port='50070', 16 | user_name='hduser') 17 | 18 | 19 | #create a new directory for the example 20 | print('making new HDFS directory at: {0}\n'.format(example_dir)) 21 | hdfs.make_dir(example_dir) 22 | 23 | # get a dictionary of the directory's status 24 | dir_status = hdfs.get_file_dir_status(example_dir) 25 | print dir_status 26 | 27 | # create a new file on hdfs 28 | print('making new file at: {0}\n'.format(example_file)) 29 | hdfs.create_file(example_file, example_data) 30 | 31 | file_status = hdfs.get_file_dir_status(example_file) 32 | print file_status 33 | 34 | #append to the file created in previous step 35 | print('appending to file at: {0}\n'.format(example_file)) 36 | hdfs.append_file(example_file, example_data) 37 | 38 | file_status = hdfs.get_file_dir_status(example_file) 39 | print file_status 40 | 41 | #read in the data for the file 42 | print('reading data from file at: {0}\n'.format(example_file)) 43 | file_data = hdfs.read_file(example_file) 44 | print file_data 45 | 46 | #rename the example_dir 47 | print('renaming directory from {0} to {1}\n').format(example_dir, rename_dir) 48 | hdfs.rename_file_dir(example_dir, '/{0}'.format(rename_dir)) 49 | 50 | #list the contents of the new directory 51 | listdir_stats = hdfs.list_dir(rename_dir) 52 | print listdir_stats 53 | 54 | example_file = '{dir}/example.txt'.format(dir=rename_dir) 55 | 56 | #delete the example file 57 | print('deleting example file at: {0}'.format(example_file)) 58 | hdfs.delete_file_dir(example_file) 59 | 60 | #list the contents of the directory 61 | listdir_stats = hdfs.list_dir(rename_dir) 62 | print listdir_stats 63 | 64 | #delete the example directory 65 | print('deleting the example directory at: {0}'.format(rename_dir)) 66 | hdfs.delete_file_dir(rename_dir, recursive='true') 67 | -------------------------------------------------------------------------------- /pywebhdfs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ProjectMeniscus/pywebhdfs/0fee16389c1936ace3a6e41716d7d220668d44bb/pywebhdfs/__init__.py -------------------------------------------------------------------------------- /pywebhdfs/errors.py: -------------------------------------------------------------------------------- 1 | 2 | class PyWebHdfsException(Exception): 3 | def __init__(self, msg=str()): 4 | self.msg = msg 5 | super(PyWebHdfsException, self).__init__(self.msg) 6 | 7 | 8 | class BadRequest(PyWebHdfsException): 9 | pass 10 | 11 | 12 | class Unauthorized(PyWebHdfsException): 13 | pass 14 | 15 | 16 | class FileNotFound(PyWebHdfsException): 17 | pass 18 | 19 | 20 | class MethodNotAllowed(PyWebHdfsException): 21 | pass 22 | -------------------------------------------------------------------------------- /pywebhdfs/operations.py: -------------------------------------------------------------------------------- 1 | CREATE = 'CREATE' 2 | APPEND = 'APPEND' 3 | OPEN = 'OPEN' 4 | MKDIRS = 'MKDIRS' 5 | RENAME = 'RENAME' 6 | DELETE = 'DELETE' 7 | GETFILESTATUS = 'GETFILESTATUS' 8 | LISTSTATUS = 'LISTSTATUS' 9 | -------------------------------------------------------------------------------- /pywebhdfs/tests/test_errors.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from pywebhdfs.webhdfs import errors 4 | 5 | 6 | class WhenTestingErrors(unittest.TestCase): 7 | def setUp(self): 8 | pass 9 | 10 | def test_pywebhdfs_exception_is_exception(self): 11 | self.assertIsInstance(errors.PyWebHdfsException(), Exception) 12 | with self.assertRaises(Exception): 13 | raise errors.PyWebHdfsException 14 | 15 | def test_pywebhdfs_exception(self): 16 | msg = 'message' 17 | ex = errors.PyWebHdfsException(msg=msg) 18 | self.assertIs(msg, ex.message) 19 | -------------------------------------------------------------------------------- /pywebhdfs/tests/test_webhdfs.py: -------------------------------------------------------------------------------- 1 | import httplib 2 | import unittest 3 | 4 | from mock import MagicMock 5 | from mock import patch 6 | 7 | from pywebhdfs import errors 8 | from pywebhdfs.webhdfs import PyWebHdfsClient, _raise_pywebhdfs_exception 9 | from pywebhdfs import operations 10 | 11 | 12 | class WhenTestingPyWebHdfsConstructor(unittest.TestCase): 13 | 14 | def test_init_default_args(self): 15 | webhdfs = PyWebHdfsClient() 16 | self.assertEqual('localhost', webhdfs.host) 17 | self.assertEqual('50070', webhdfs.port) 18 | self.assertIsNone(webhdfs.user_name) 19 | 20 | def test_init_args_provided(self): 21 | host = '127.0.0.1' 22 | port = '50075' 23 | user_name = 'myUser' 24 | 25 | webhdfs = PyWebHdfsClient(host=host, port=port, user_name=user_name) 26 | self.assertEqual(host, webhdfs.host) 27 | self.assertEqual(port, webhdfs.port) 28 | self.assertEqual(user_name, webhdfs.user_name) 29 | 30 | 31 | class WhenTestingCreateOperation(unittest.TestCase): 32 | 33 | def setUp(self): 34 | 35 | self.host = 'hostname' 36 | self.port = '00000' 37 | self.user_name = 'username' 38 | self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port, 39 | user_name=self.user_name) 40 | self.response = MagicMock() 41 | self.requests = MagicMock(return_value=self.response) 42 | self.location = 'redirect_uri' 43 | self.path = 'user/hdfs' 44 | self.file_data = '010101' 45 | self.init_response = MagicMock() 46 | self.init_response.headers = {'location': self.location} 47 | self.response = MagicMock() 48 | self.expected_headers = {'content-type': 'application/octet-stream'} 49 | 50 | def test_create_throws_exception_for_no_redirect(self): 51 | 52 | self.init_response.status_code = httplib.BAD_REQUEST 53 | self.response.status_code = httplib.CREATED 54 | self.requests.put.side_effect = [self.init_response, self.response] 55 | with patch('pywebhdfs.webhdfs.requests', self.requests): 56 | with self.assertRaises(errors.PyWebHdfsException): 57 | self.webhdfs.create_file(self.path, self.file_data) 58 | 59 | def test_create_throws_exception_for_not_created(self): 60 | 61 | self.init_response.status_code = httplib.TEMPORARY_REDIRECT 62 | self.response.status_code = httplib.BAD_REQUEST 63 | self.requests.put.side_effect = [self.init_response, self.response] 64 | with patch('pywebhdfs.webhdfs.requests', self.requests): 65 | with self.assertRaises(errors.PyWebHdfsException): 66 | self.webhdfs.create_file(self.path, self.file_data) 67 | 68 | def test_create_returns_file_location(self): 69 | 70 | self.init_response.status_code = httplib.TEMPORARY_REDIRECT 71 | self.response.status_code = httplib.CREATED 72 | self.put_method = MagicMock( 73 | side_effect=[self.init_response, self.response]) 74 | self.requests.put = self.put_method 75 | with patch('pywebhdfs.webhdfs.requests', self.requests): 76 | result = self.webhdfs.create_file(self.path, self.file_data) 77 | self.assertTrue(result) 78 | self.put_method.assert_called_with( 79 | self.location, headers=self.expected_headers, data=self.file_data) 80 | 81 | 82 | class WhenTestingAppendOperation(unittest.TestCase): 83 | 84 | def setUp(self): 85 | 86 | self.host = 'hostname' 87 | self.port = '00000' 88 | self.user_name = 'username' 89 | self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port, 90 | user_name=self.user_name) 91 | self.response = MagicMock() 92 | self.requests = MagicMock(return_value=self.response) 93 | self.location = 'redirect_uri' 94 | self.path = 'user/hdfs' 95 | self.file_data = '010101' 96 | self.init_response = MagicMock() 97 | self.init_response.header = {'location': self.location} 98 | self.response = MagicMock() 99 | 100 | def test_append_throws_exception_for_no_redirect(self): 101 | 102 | self.init_response.status_code = httplib.BAD_REQUEST 103 | self.response.status_code = httplib.OK 104 | self.requests.post.side_effect = [self.init_response, self.response] 105 | with patch('pywebhdfs.webhdfs.requests', self.requests): 106 | with self.assertRaises(errors.PyWebHdfsException): 107 | self.webhdfs.append_file(self.path, self.file_data) 108 | 109 | def test_append_throws_exception_for_not_ok(self): 110 | 111 | self.init_response.status_code = httplib.TEMPORARY_REDIRECT 112 | self.response.status_code = httplib.BAD_REQUEST 113 | self.requests.post.side_effect = [self.init_response, self.response] 114 | with patch('pywebhdfs.webhdfs.requests', self.requests): 115 | with self.assertRaises(errors.PyWebHdfsException): 116 | self.webhdfs.append_file(self.path, self.file_data) 117 | 118 | def test_append_returns_true(self): 119 | 120 | self.init_response.status_code = httplib.TEMPORARY_REDIRECT 121 | self.response.status_code = httplib.OK 122 | self.requests.post.side_effect = [self.init_response, self.response] 123 | with patch('pywebhdfs.webhdfs.requests', self.requests): 124 | result = self.webhdfs.append_file(self.path, self.file_data) 125 | self.assertTrue(result) 126 | 127 | 128 | class WhenTestingOpenOperation(unittest.TestCase): 129 | 130 | def setUp(self): 131 | 132 | self.host = 'hostname' 133 | self.port = '00000' 134 | self.user_name = 'username' 135 | self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port, 136 | user_name=self.user_name) 137 | self.response = MagicMock() 138 | self.requests = MagicMock(return_value=self.response) 139 | self.path = 'user/hdfs' 140 | self.file_data = u'010101' 141 | self.response = MagicMock() 142 | self.response.content = self.file_data 143 | 144 | def test_read_throws_exception_for_not_ok(self): 145 | 146 | self.response.status_code = httplib.BAD_REQUEST 147 | self.requests.get.return_value = self.response 148 | with patch('pywebhdfs.webhdfs.requests', self.requests): 149 | with self.assertRaises(errors.PyWebHdfsException): 150 | self.webhdfs.read_file(self.path) 151 | 152 | def test_read_returns_file(self): 153 | 154 | self.response.status_code = httplib.OK 155 | self.requests.get.return_value = self.response 156 | with patch('pywebhdfs.webhdfs.requests', self.requests): 157 | result = self.webhdfs.read_file(self.path) 158 | self.assertEqual(result, self.file_data) 159 | 160 | 161 | class WhenTestingMkdirOperation(unittest.TestCase): 162 | 163 | def setUp(self): 164 | 165 | self.host = 'hostname' 166 | self.port = '00000' 167 | self.user_name = 'username' 168 | self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port, 169 | user_name=self.user_name) 170 | self.response = MagicMock() 171 | self.requests = MagicMock(return_value=self.response) 172 | self.path = 'user/hdfs' 173 | self.response = MagicMock() 174 | 175 | def test_mkdir_throws_exception_for_not_ok(self): 176 | 177 | self.response.status_code = httplib.BAD_REQUEST 178 | self.requests.put.return_value = self.response 179 | with patch('pywebhdfs.webhdfs.requests', self.requests): 180 | with self.assertRaises(errors.PyWebHdfsException): 181 | self.webhdfs.make_dir(self.path) 182 | 183 | def test_mkdir_returns_true(self): 184 | 185 | self.response.status_code = httplib.OK 186 | self.requests.put.return_value = self.response 187 | with patch('pywebhdfs.webhdfs.requests', self.requests): 188 | result = self.webhdfs.make_dir(self.path) 189 | self.assertTrue(result) 190 | 191 | 192 | class WhenTestingRenameOperation(unittest.TestCase): 193 | 194 | def setUp(self): 195 | 196 | self.host = 'hostname' 197 | self.port = '00000' 198 | self.user_name = 'username' 199 | self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port, 200 | user_name=self.user_name) 201 | self.response = MagicMock() 202 | self.requests = MagicMock(return_value=self.response) 203 | self.path = 'user/hdfs/old_dir' 204 | self.new_path = '/user/hdfs/new_dir' 205 | self.response = MagicMock() 206 | 207 | def test_rename_throws_exception_for_not_ok(self): 208 | 209 | self.response.status_code = httplib.BAD_REQUEST 210 | self.requests.put.return_value = self.response 211 | with patch('pywebhdfs.webhdfs.requests', self.requests): 212 | with self.assertRaises(errors.PyWebHdfsException): 213 | self.webhdfs.rename_file_dir(self.path, self.new_path) 214 | 215 | def test_rename_returns_true(self): 216 | 217 | self.response.status_code = httplib.OK 218 | self.requests.put.return_value = self.response 219 | with patch('pywebhdfs.webhdfs.requests', self.requests): 220 | result = self.webhdfs.rename_file_dir(self.path, self.new_path) 221 | self.assertTrue(result) 222 | 223 | 224 | class WhenTestingDeleteOperation(unittest.TestCase): 225 | 226 | def setUp(self): 227 | 228 | self.host = 'hostname' 229 | self.port = '00000' 230 | self.user_name = 'username' 231 | self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port, 232 | user_name=self.user_name) 233 | self.response = MagicMock() 234 | self.requests = MagicMock(return_value=self.response) 235 | self.path = 'user/hdfs/old_dir' 236 | self.response = MagicMock() 237 | 238 | def test_rename_throws_exception_for_not_ok(self): 239 | 240 | self.response.status_code = httplib.BAD_REQUEST 241 | self.requests.delete.return_value = self.response 242 | with patch('pywebhdfs.webhdfs.requests', self.requests): 243 | with self.assertRaises(errors.PyWebHdfsException): 244 | self.webhdfs.delete_file_dir(self.path) 245 | 246 | def test_rename_returns_true(self): 247 | 248 | self.response.status_code = httplib.OK 249 | self.requests.delete.return_value = self.response 250 | with patch('pywebhdfs.webhdfs.requests', self.requests): 251 | result = self.webhdfs.delete_file_dir(self.path) 252 | self.assertTrue(result) 253 | 254 | 255 | class WhenTestingGetFileStatusOperation(unittest.TestCase): 256 | 257 | def setUp(self): 258 | 259 | self.host = 'hostname' 260 | self.port = '00000' 261 | self.user_name = 'username' 262 | self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port, 263 | user_name=self.user_name) 264 | self.response = MagicMock() 265 | self.requests = MagicMock(return_value=self.response) 266 | self.path = 'user/hdfs/old_dir' 267 | self.response = MagicMock() 268 | self.file_status = { 269 | "FileStatus": { 270 | "accessTime": 0, 271 | "blockSize": 0, 272 | "group": "supergroup", 273 | "length": 0, 274 | "modificationTime": 1320173277227, 275 | "owner": "webuser", 276 | "pathSuffix": "", 277 | "permission": "777", 278 | "replication": 0, 279 | "type": "DIRECTORY" 280 | } 281 | } 282 | self.response.json = MagicMock(return_value=self.file_status) 283 | 284 | def test_get_status_throws_exception_for_not_ok(self): 285 | 286 | self.response.status_code = httplib.BAD_REQUEST 287 | self.requests.get.return_value = self.response 288 | with patch('pywebhdfs.webhdfs.requests', self.requests): 289 | with self.assertRaises(errors.PyWebHdfsException): 290 | self.webhdfs.get_file_dir_status(self.path) 291 | 292 | def test_get_status_returns_true(self): 293 | 294 | self.response.status_code = httplib.OK 295 | self.requests.get.return_value = self.response 296 | with patch('pywebhdfs.webhdfs.requests', self.requests): 297 | result = self.webhdfs.get_file_dir_status(self.path) 298 | 299 | for key in result: 300 | self.assertEqual(result[key], self.file_status[key]) 301 | 302 | 303 | class WhenTestingListDirOperation(unittest.TestCase): 304 | 305 | def setUp(self): 306 | 307 | self.host = 'hostname' 308 | self.port = '00000' 309 | self.user_name = 'username' 310 | self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port, 311 | user_name=self.user_name) 312 | self.response = MagicMock() 313 | self.requests = MagicMock(return_value=self.response) 314 | self.path = 'user/hdfs/old_dir' 315 | self.response = MagicMock() 316 | self.file_status = { 317 | "FileStatuses": { 318 | "FileStatus": [ 319 | { 320 | "accessTime": 0, 321 | "blockSize": 0, 322 | "group": "supergroup", 323 | "length": 24930, 324 | "modificationTime": 1320173277227, 325 | "owner": "webuser", 326 | "pathSuffix": "a.patch", 327 | "permission": "777", 328 | "replication": 0, 329 | "type": "FILE" 330 | }, 331 | { 332 | "accessTime": 0, 333 | "blockSize": 0, 334 | "group": "supergroup", 335 | "length": 0, 336 | "modificationTime": 1320173277227, 337 | "owner": "webuser", 338 | "pathSuffix": "", 339 | "permission": "777", 340 | "replication": 0, 341 | "type": "DIRECTORY" 342 | } 343 | ] 344 | } 345 | } 346 | self.response.json = MagicMock(return_value=self.file_status) 347 | 348 | def test_get_status_throws_exception_for_not_ok(self): 349 | 350 | self.response.status_code = httplib.BAD_REQUEST 351 | self.requests.get.return_value = self.response 352 | with patch('pywebhdfs.webhdfs.requests', self.requests): 353 | with self.assertRaises(errors.PyWebHdfsException): 354 | self.webhdfs.list_dir(self.path) 355 | 356 | def test_get_status_returns_true(self): 357 | 358 | self.response.status_code = httplib.OK 359 | self.requests.get.return_value = self.response 360 | with patch('pywebhdfs.webhdfs.requests', self.requests): 361 | result = self.webhdfs.list_dir(self.path) 362 | 363 | for key in result: 364 | self.assertEqual(result[key], self.file_status[key]) 365 | 366 | 367 | class WhenTestingCreateUri(unittest.TestCase): 368 | 369 | def setUp(self): 370 | self.host = 'hostname' 371 | self.port = '00000' 372 | self.user_name = 'username' 373 | self.path = 'user/hdfs' 374 | self.webhdfs = PyWebHdfsClient(host=self.host, port=self.port, 375 | user_name=self.user_name) 376 | 377 | def test_create_uri_no_kwargs(self): 378 | op = operations.CREATE 379 | uri = 'http://{host}:{port}/webhdfs/v1/' \ 380 | '{path}?op={op}&user.name={user}'\ 381 | .format( 382 | host=self.host, port=self.port, path=self.path, 383 | op=op, user=self.user_name) 384 | result = self.webhdfs._create_uri(self.path, op) 385 | self.assertEqual(uri, result) 386 | 387 | def test_create_uri_with_kwargs(self): 388 | op = operations.CREATE 389 | mykey = 'mykey' 390 | myval = 'myval' 391 | uri = 'http://{host}:{port}/webhdfs/v1/' \ 392 | '{path}?op={op}&{key}={val}' \ 393 | '&user.name={user}' \ 394 | .format( 395 | host=self.host, port=self.port, path=self.path, 396 | op=op, key=mykey, val=myval, user=self.user_name) 397 | result = self.webhdfs._create_uri(self.path, op, mykey=myval) 398 | self.assertEqual(uri, result) 399 | 400 | def test_create_uri_with_unicode_path(self): 401 | op = operations.CREATE 402 | mykey = 'mykey' 403 | myval = 'myval' 404 | path = u'die/Stra\xdfe' 405 | quoted_path = 'die/Stra%C3%9Fe' 406 | uri = 'http://{host}:{port}/webhdfs/v1/' \ 407 | '{path}?op={op}&{key}={val}' \ 408 | '&user.name={user}' \ 409 | .format( 410 | host=self.host, port=self.port, path=quoted_path, 411 | op=op, key=mykey, val=myval, user=self.user_name) 412 | result = self.webhdfs._create_uri(path, op, mykey=myval) 413 | self.assertEqual(uri, result) 414 | 415 | 416 | class WhenTestingRaiseExceptions(unittest.TestCase): 417 | 418 | def test_400_raises_bad_request(self): 419 | with self.assertRaises(errors.BadRequest): 420 | _raise_pywebhdfs_exception(httplib.BAD_REQUEST) 421 | 422 | def test_401_raises_unuathorized(self): 423 | with self.assertRaises(errors.Unauthorized): 424 | _raise_pywebhdfs_exception(httplib.UNAUTHORIZED) 425 | 426 | def test_404_raises_not_found(self): 427 | with self.assertRaises(errors.FileNotFound): 428 | _raise_pywebhdfs_exception(httplib.NOT_FOUND) 429 | 430 | def test_all_other_raises_pywebhdfs_exception(self): 431 | with self.assertRaises(errors.PyWebHdfsException): 432 | _raise_pywebhdfs_exception(httplib.GATEWAY_TIMEOUT) 433 | -------------------------------------------------------------------------------- /pywebhdfs/webhdfs.py: -------------------------------------------------------------------------------- 1 | import httplib 2 | 3 | import requests 4 | try: 5 | from urllib.parse import quote, quote_plus 6 | except ImportError: 7 | from urllib import quote, quote_plus 8 | 9 | from pywebhdfs import errors, operations 10 | 11 | 12 | class PyWebHdfsClient(object): 13 | """ 14 | PyWebHdfsClient is a Python wrapper for the Hadoop WebHDFS REST API 15 | 16 | To use this client: 17 | 18 | >>> from pywebhdfs.webhdfs import PyWebHdfsClient 19 | """ 20 | 21 | def __init__(self, host='localhost', port='50070', user_name=None): 22 | """ 23 | Create a new client for interacting with WebHDFS 24 | 25 | :param host: the ip address or hostname of the HDFS namenode 26 | :param port: the port number for WebHDFS on the namenode 27 | :param user_name: WebHDFS user.name used for authentication 28 | 29 | >>> hdfs = PyWebHdfsClient(host='host',port='50070', user_name='hdfs') 30 | """ 31 | 32 | self.host = host 33 | self.port = port 34 | self.user_name = user_name 35 | 36 | # create base uri to be used in request operations 37 | self.base_uri = 'http://{host}:{port}/webhdfs/v1/'.format( 38 | host=self.host, port=self.port) 39 | 40 | def create_file(self, path, file_data, **kwargs): 41 | """ 42 | Creates a new file on HDFS 43 | 44 | :param path: the HDFS file path without a leading '/' 45 | :param file_data: the initial data to write to the new file 46 | 47 | The function wraps the WebHDFS REST call: 48 | 49 | PUT http://:/webhdfs/v1/?op=CREATE 50 | 51 | [&overwrite=][&blocksize=][&replication=] 52 | [&permission=][&buffersize=] 53 | 54 | The function accepts all WebHDFS optional arguments shown above 55 | 56 | Example: 57 | 58 | >>> hdfs = PyWebHdfsClient(host='host',port='50070', user_name='hdfs') 59 | >>> my_data = '01010101010101010101010101010101' 60 | >>> my_file = 'user/hdfs/data/myfile.txt' 61 | >>> hdfs.create_file(my_file, my_data) 62 | 63 | Example with optional args: 64 | 65 | >>> hdfs.create_file(my_file, my_data, overwrite=True, blocksize=64) 66 | 67 | Or for sending data from file like objects: 68 | 69 | >>> with open('file.data') as file_data: 70 | >>> hdfs.create_file(hdfs_path, data=file_data) 71 | 72 | 73 | Note: The create_file function does not follow automatic redirects but 74 | instead uses a two step call to the API as required in the 75 | WebHDFS documentation 76 | """ 77 | 78 | # make the initial CREATE call to the HDFS namenode 79 | optional_args = kwargs 80 | uri = self._create_uri(path, operations.CREATE, **optional_args) 81 | init_response = requests.put(uri, allow_redirects=False) 82 | 83 | if not init_response.status_code == httplib.TEMPORARY_REDIRECT: 84 | _raise_pywebhdfs_exception( 85 | init_response.status_code, init_response.content) 86 | 87 | # Get the address provided in the location header of the 88 | # initial response from the namenode and make the CREATE request 89 | # to the datanode 90 | uri = init_response.headers['location'] 91 | response = requests.put( 92 | uri, data=file_data, 93 | headers={'content-type': 'application/octet-stream'}) 94 | 95 | if not response.status_code == httplib.CREATED: 96 | _raise_pywebhdfs_exception(response.status_code, response.content) 97 | 98 | return True 99 | 100 | def append_file(self, path, file_data, **kwargs): 101 | """ 102 | Appends to an existing file on HDFS 103 | 104 | :param path: the HDFS file path without a leading '/' 105 | :param file_data: data to append to existing file 106 | 107 | The function wraps the WebHDFS REST call: 108 | 109 | POST http://:/webhdfs/v1/?op=APPEND 110 | 111 | [&buffersize=] 112 | 113 | The function accepts all WebHDFS optional arguments shown above 114 | 115 | Example: 116 | 117 | >>> hdfs = PyWebHdfsClient(host='host',port='50070', user_name='hdfs') 118 | >>> my_data = '01010101010101010101010101010101' 119 | >>> my_file = 'user/hdfs/data/myfile.txt' 120 | >>> hdfs.append_file(my_file, my_data) 121 | 122 | Example with optional args: 123 | 124 | >>> hdfs.append_file(my_file, my_data, overwrite=True, buffersize=4096) 125 | 126 | Note: The append_file function does not follow automatic redirects but 127 | instead uses a two step call to the API as required in the 128 | WebHDFS documentation 129 | 130 | Append is not supported in Hadoop 1.x 131 | """ 132 | 133 | # make the initial APPEND call to the HDFS namenode 134 | optional_args = kwargs 135 | uri = self._create_uri(path, operations.APPEND, **optional_args) 136 | init_response = requests.post(uri, allow_redirects=False) 137 | 138 | if not init_response.status_code == httplib.TEMPORARY_REDIRECT: 139 | _raise_pywebhdfs_exception( 140 | init_response.status_code, init_response.content) 141 | 142 | # Get the address provided in the location header of the 143 | # initial response from the namenode and make the APPEND request 144 | # to the datanode 145 | uri = init_response.headers['location'] 146 | response = requests.post( 147 | uri, data=file_data, 148 | headers={'content-type': 'application/octet-stream'}) 149 | 150 | if not response.status_code == httplib.OK: 151 | _raise_pywebhdfs_exception(response.status_code, response.content) 152 | 153 | return True 154 | 155 | def read_file(self, path, **kwargs): 156 | """ 157 | Reads from a file on HDFS and returns the content 158 | 159 | :param path: the HDFS file path without a leading '/' 160 | 161 | The function wraps the WebHDFS REST call: 162 | 163 | GET http://:/webhdfs/v1/?op=OPEN 164 | 165 | [&offset=][&length=][&buffersize=] 166 | 167 | Note: this function follows automatic redirects 168 | 169 | Example: 170 | 171 | >>> hdfs = PyWebHdfsClient(host='host',port='50070', user_name='hdfs') 172 | >>> my_file = 'user/hdfs/data/myfile.txt' 173 | >>> hdfs.read_file(my_file) 174 | 01010101010101010101010101010101 175 | 01010101010101010101010101010101 176 | 01010101010101010101010101010101 177 | 01010101010101010101010101010101 178 | """ 179 | 180 | optional_args = kwargs 181 | uri = self._create_uri(path, operations.OPEN, **optional_args) 182 | 183 | response = requests.get(uri, allow_redirects=True) 184 | 185 | if not response.status_code == httplib.OK: 186 | _raise_pywebhdfs_exception(response.status_code, response.content) 187 | 188 | return response.content 189 | 190 | def make_dir(self, path, **kwargs): 191 | """ 192 | Create a new directory on HDFS 193 | 194 | :param path: the HDFS file path without a leading '/' 195 | 196 | The function wraps the WebHDFS REST call: 197 | 198 | PUT http://:/webhdfs/v1/?op=MKDIRS 199 | 200 | [&permission=] 201 | 202 | Example: 203 | 204 | >>> hdfs = PyWebHdfsClient(host='host',port='50070', user_name='hdfs') 205 | >>> my_dir = 'user/hdfs/data/new_dir' 206 | >>> hdfs.make_dir(my_dir) 207 | 208 | Example with optional args: 209 | 210 | >>> hdfs.make_dir(my_dir, permission=755) 211 | """ 212 | 213 | optional_args = kwargs 214 | uri = self._create_uri(path, operations.MKDIRS, **optional_args) 215 | 216 | response = requests.put(uri, allow_redirects=True) 217 | 218 | if not response.status_code == httplib.OK: 219 | _raise_pywebhdfs_exception(response.status_code, response.content) 220 | 221 | return True 222 | 223 | def rename_file_dir(self, path, destination_path): 224 | """ 225 | Rename an existing directory or file on HDFS 226 | 227 | :param path: the HDFS file path without a leading '/' 228 | :param destination_path: the new file path name 229 | 230 | The function wraps the WebHDFS REST call: 231 | 232 | PUT :/webhdfs/v1/?op=RENAME&destination= 233 | 234 | Example: 235 | 236 | >>> hdfs = PyWebHdfsClient(host='host',port='50070', user_name='hdfs') 237 | >>> current_dir = 'user/hdfs/data/my_dir' 238 | >>> destination_dir = 'user/hdfs/data/renamed_dir' 239 | >>> hdfs.rename_file_dir(current_dir, destination_dir) 240 | """ 241 | 242 | destination_path = '/' + destination_path.lstrip('/') 243 | uri = self._create_uri(path, operations.RENAME, 244 | destination=destination_path) 245 | 246 | response = requests.put(uri, allow_redirects=True) 247 | 248 | if not response.status_code == httplib.OK: 249 | _raise_pywebhdfs_exception(response.status_code, response.content) 250 | 251 | return True 252 | 253 | def delete_file_dir(self, path, recursive=False): 254 | """ 255 | Delete an existing file or directory from HDFS 256 | 257 | :param path: the HDFS file path without a leading '/' 258 | 259 | The function wraps the WebHDFS REST call: 260 | 261 | DELETE "http://:/webhdfs/v1/?op=DELETE 262 | 263 | [&recursive=] 264 | 265 | Example for deleting a file: 266 | 267 | >>> hdfs = PyWebHdfsClient(host='host',port='50070', user_name='hdfs') 268 | >>> my_file = 'user/hdfs/data/myfile.txt' 269 | >>> hdfs.delete_file_dir(my_file) 270 | 271 | Example for deleting a directory: 272 | 273 | >>> hdfs.delete_file_dir(my_file, recursive=True) 274 | """ 275 | 276 | uri = self._create_uri(path, operations.DELETE, recursive=recursive) 277 | response = requests.delete(uri, allow_redirects=True) 278 | 279 | if not response.status_code == httplib.OK: 280 | _raise_pywebhdfs_exception(response.status_code, response.content) 281 | 282 | return True 283 | 284 | def get_file_dir_status(self, path): 285 | """ 286 | Get the file_status of a single file or directory on HDFS 287 | 288 | :param path: the HDFS file path without a leading '/' 289 | 290 | The function wraps the WebHDFS REST call: 291 | 292 | GET http://:/webhdfs/v1/?op=GETFILESTATUS 293 | 294 | Example for getting file status: 295 | 296 | >>> hdfs = PyWebHdfsClient(host='host',port='50070', user_name='hdfs') 297 | >>> my_file = 'user/hdfs/data/myfile.txt' 298 | >>> hdfs.get_file_dir_status(my_file) 299 | { 300 | "FileStatus":{ 301 | "accessTime":1371737704282, 302 | "blockSize":134217728, 303 | "group":"hdfs", 304 | "length":90, 305 | "modificationTime":1371737704595, 306 | "owner":"hdfs", 307 | "pathSuffix":"", 308 | "permission":"755", 309 | "replication":3, 310 | "type":"FILE" 311 | } 312 | } 313 | 314 | Example for getting directory status: 315 | 316 | >>> my_dir = 'user/hdfs/data/' 317 | >>> hdfs.get_file_dir_status(my_file) 318 | { 319 | "FileStatus":{ 320 | "accessTime":0, 321 | "blockSize":0, 322 | "group":"hdfs", 323 | "length":0, 324 | "modificationTime":1371737704208, 325 | "owner":"hdfs", 326 | "pathSuffix":"", 327 | "permission":"755", 328 | "replication":0, 329 | "type":"DIRECTORY" 330 | } 331 | } 332 | """ 333 | 334 | uri = self._create_uri(path, operations.GETFILESTATUS) 335 | response = requests.get(uri, allow_redirects=True) 336 | 337 | if not response.status_code == httplib.OK: 338 | _raise_pywebhdfs_exception(response.status_code, response.content) 339 | 340 | return response.json() 341 | 342 | def list_dir(self, path): 343 | """ 344 | Get a list of file_status for all files and directories 345 | inside an HDFS directory 346 | 347 | :param path: the HDFS file path without a leading '/' 348 | 349 | The function wraps the WebHDFS REST call: 350 | 351 | GET http://:/webhdfs/v1/?op=LISTSTATUS 352 | 353 | Example for listing a directory: 354 | 355 | >>> hdfs = PyWebHdfsClient(host='host',port='50070', user_name='hdfs') 356 | >>> my_dir = 'user/hdfs' 357 | >>> hdfs.list_dir(my_dir) 358 | { 359 | "FileStatuses":{ 360 | "FileStatus":[ 361 | { 362 | "accessTime":1371737704282, 363 | "blockSize":134217728, 364 | "group":"hdfs", 365 | "length":90, 366 | "modificationTime":1371737704595, 367 | "owner":"hdfs", 368 | "pathSuffix":"example3.txt", 369 | "permission":"755", 370 | "replication":3, 371 | "type":"FILE" 372 | }, 373 | { 374 | "accessTime":1371678467205, 375 | "blockSize":134217728, 376 | "group":"hdfs","length":1057, 377 | "modificationTime":1371678467394, 378 | "owner":"hdfs", 379 | "pathSuffix":"example2.txt", 380 | "permission":"700", 381 | "replication":3, 382 | "type":"FILE" 383 | } 384 | ] 385 | } 386 | } 387 | 388 | """ 389 | 390 | uri = self._create_uri(path, operations.LISTSTATUS) 391 | response = requests.get(uri, allow_redirects=True) 392 | 393 | if not response.status_code == httplib.OK: 394 | _raise_pywebhdfs_exception(response.status_code, response.content) 395 | 396 | return response.json() 397 | 398 | def _create_uri(self, path, operation, **kwargs): 399 | """ 400 | internal function used to construct the WebHDFS request uri based on 401 | the , , and any provided optional arguments 402 | """ 403 | 404 | path_param = quote(path.encode('utf8')) 405 | 406 | # setup the parameter represent the WebHDFS operation 407 | operation_param = '?op={operation}'.format(operation=operation) 408 | 409 | # configure authorization based on provided credentials 410 | auth_param = str() 411 | if self.user_name: 412 | auth_param = '&user.name={user_name}'.format( 413 | user_name=self.user_name) 414 | 415 | # setup any optional parameters 416 | keyword_params = str() 417 | for key in kwargs: 418 | try: 419 | value = quote_plus(kwargs[key].encode('utf8')) 420 | except: 421 | value = str(kwargs[key]).lower() 422 | keyword_params = '{params}&{key}={value}'.format( 423 | params=keyword_params, key=key, value=value) 424 | 425 | # build the complete uri from the base uri and all configured params 426 | uri = '{base_uri}{path}{operation}{keyword_args}{auth}'.format( 427 | base_uri=self.base_uri, path=path_param, 428 | operation=operation_param, keyword_args=keyword_params, 429 | auth=auth_param) 430 | 431 | return uri 432 | 433 | 434 | def _raise_pywebhdfs_exception(resp_code, message=None): 435 | 436 | if resp_code == httplib.BAD_REQUEST: 437 | raise errors.BadRequest(msg=message) 438 | elif resp_code == httplib.UNAUTHORIZED: 439 | raise errors.Unauthorized(msg=message) 440 | elif resp_code == httplib.NOT_FOUND: 441 | raise errors.FileNotFound(msg=message) 442 | elif resp_code == httplib.METHOD_NOT_ALLOWED: 443 | raise errors.MethodNotAllowed(msg=message) 444 | else: 445 | raise errors.PyWebHdfsException(msg=message) 446 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [nosetests] 2 | where=pywebhdfs 3 | nocapture=1 4 | cover-package=pywebhdfs 5 | cover-erase=1 6 | 7 | [build_sphinx] 8 | source-dir = docs/source 9 | build-dir = docs/build 10 | all_files = 1 11 | 12 | [upload_sphinx] 13 | upload-dir = docs/build/html -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | try: 3 | from setuptools import setup, find_packages 4 | except ImportError: 5 | from ez_setup import use_setuptools 6 | use_setuptools() 7 | from setuptools import setup, find_packages 8 | 9 | setup( 10 | name='pywebhdfs', 11 | version='0.2.3', 12 | description='Python wrapper for the Hadoop WebHDFS REST API', 13 | author='Steven D. Gonzales', 14 | author_email='stevendgonzales@gmail.com', 15 | url='https://github.com/ProjectMeniscus/pywebhdfs', 16 | tests_require=[ 17 | "mock", 18 | "nose", 19 | "nosexcover", 20 | "testtools", 21 | "tox" 22 | ], 23 | install_requires=[ 24 | "requests" 25 | ], 26 | test_suite='nose.collector', 27 | zip_safe=False, 28 | include_package_data=True, 29 | packages=find_packages(exclude=['ez_setup']) 30 | ) 31 | -------------------------------------------------------------------------------- /tools/pip-requires: -------------------------------------------------------------------------------- 1 | requests -------------------------------------------------------------------------------- /tools/test-requires: -------------------------------------------------------------------------------- 1 | mock>=1.0.0 2 | nose 3 | nosexcover 4 | testtools 5 | tox 6 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27 3 | 4 | [testenv] 5 | deps = -r{toxinidir}/tools/pip-requires 6 | -r{toxinidir}/tools/test-requires 7 | 8 | [testenv:py27] 9 | commands = nosetests {posargs:--with-xcoverage --with-xunit --cover-package=pywebhdfs} --------------------------------------------------------------------------------