├── isilon ├── exceptions.py ├── __init__.py ├── platform.py ├── namespace.py └── session.py ├── setup.py ├── LICENSE ├── mkapi.py ├── README.md ├── api_example.py └── test_isilon.py /isilon/exceptions.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | class ObjectNotFound(RuntimeError): 4 | """The API has responsed with an HTTP 404 Object not found code, 5 | suppressed for queries as None is returned instead""" 6 | 7 | class APIError(RuntimeError): 8 | """This is an api level error""" 9 | 10 | class ConnectionError(RuntimeError): 11 | """This is an api level error""" 12 | 13 | 14 | 15 | class IsilonLibraryError(RuntimeError): 16 | 17 | """This is a library error, something that the library is trying to automate or protect""" 18 | 19 | -------------------------------------------------------------------------------- /isilon/__init__.py: -------------------------------------------------------------------------------- 1 | import session 2 | import namespace 3 | import platform 4 | 5 | from .exceptions import ObjectNotFound, APIError, ConnectionError, IsilonLibraryError 6 | 7 | class API(object): 8 | 9 | '''Implements higher level functionality to interface with an Isilon cluster''' 10 | 11 | def __init__(self, *args, **kwargs): 12 | 13 | self.session = session.Session(*args, **kwargs) 14 | self.namespace = namespace.Namespace(self.session) 15 | self.platform = platform.Platform(self.session) 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | 3 | setup( 4 | name='isilon', 5 | description='Python library for using the Isilon API', 6 | version='0.1', 7 | author='Matt Robertson', 8 | author_email='sile16@gmail.com', 9 | packages=['isilon'], 10 | requires=['requests'], 11 | license='MIT', 12 | classifiers=( 13 | 'Development Status :: 3 - Alpha', 14 | 'Intended Audience :: Developers', 15 | 'Natural Language :: English', 16 | 'License :: OSI Approved :: MIT License', 17 | 'Programming Language :: Python', 18 | 'Programming Language :: Python :: 2.7' 19 | 20 | )) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Matt Robertson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /mkapi.py: -------------------------------------------------------------------------------- 1 | import isilon 2 | import logging 3 | import argparse 4 | import pprint 5 | 6 | 7 | def main(): 8 | parser = argparse.ArgumentParser(description='list api docs') 9 | parser.add_argument('-url', dest='url', default='', help='filter only paths with this string in it') 10 | parser.add_argument('-method', dest='method', default='get', help='method') 11 | parser.add_argument('-v', action='store_true',dest='verbose',default=False,help='verbose') 12 | 13 | args = parser.parse_args() 14 | 15 | api = isilon.API("192.168.167.101","root","a", secure=False) 16 | pp = pprint.PrettyPrinter(indent=4) 17 | 18 | data = api.session.api_call("GET","/platform/1/?describe&list&all") 19 | 20 | all_keys = {} 21 | 22 | for dir in data['directory']: 23 | 24 | if not args.url or args.url == dir: 25 | 26 | data1 = api.session.api_call("GET","/platform" + dir + "?describe&json") 27 | 28 | # print(data1) 29 | 30 | if not data1 is None: 31 | for k in data1: 32 | all_keys[k] = True 33 | 34 | 35 | all_args = dir + ' Props: ' 36 | 37 | #cloud api's don't have docs yeti 38 | if (not data1 is None): 39 | for key in data1: 40 | if args.method .upper() in key.upper() and 'properties' in data1[key]: 41 | print(key) 42 | if args.verbose: 43 | pp.pprint(data1[key]) 44 | 45 | for prop in data1[key]['properties']: 46 | 47 | all_args = all_args + prop + ', ' 48 | 49 | print(all_args) 50 | 51 | print('') 52 | print("Available keys %s" % str(all_keys)) 53 | 54 | if __name__ == '__main__': 55 | main() 56 | 57 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | python-isilon-api 2 | ================= 3 | 4 | An unofficial python library to make Isilon API calls. This library is incomplete, 5 | not fully tested, and not supported by EMC. I only plan to fill in the features as 6 | I need them. Use at your own risk. 7 | 8 | This is a library I created in order to make a copy tool with the goal 9 | of learning the API. That said, it does work in my environment and 10 | may be useful to others working with the API. 11 | 12 | ####Tested with: 13 | 14 | - OneFS 7.2 15 | - Python 2.7.6 16 | - HTTP Requests 2.5.3 17 | 18 | API Features 19 | ============ 20 | - Automatic Session/Connection management 21 | - Efficient reuse of HTTP connections 22 | - Automatic handling of resume tokens 23 | - Some basic error handling of sessions 24 | - Namespace 25 | - - 26 | 27 | cp.py Features 28 | ============== 29 | - Copies a source directory to a target directory 30 | - Can utilize file cloning (uses SSH connections) 31 | - Uses snaps to create a temporary static source 32 | - Multithreaded 33 | - Intelligently connects to multiple nodes in the cluster 34 | - Applies ACLs to target files 35 | - Can run in verify mode to only compare ACLS 36 | 37 | Example: ./cp.py -c -i kcisilon -u root -p a /ifs/data/test /ifs/data/clones/test1 38 | 39 | ``` 40 | ./cp.py --help 41 | usage: cp.py [-h] -i URL -u USERNAME -p PASSWORD [-t THREADCOUNT] [-c] [-v] 42 | src dst 43 | 44 | Create a copy of a directory 45 | 46 | positional arguments: 47 | src source directory, full path starting with /ifs 48 | dst destination directory, full path starting with /ifs 49 | 50 | optional arguments: 51 | -h, --help show this help message and exit 52 | -i URL IP or DNS name of the cluster 53 | -u USERNAME API Username 54 | -p PASSWORD API Password 55 | -t THREADCOUNT Thread Count, default=16 56 | -c Use sparse cloning technology 57 | -v Verify ACLS only, do not copy or clone 58 | ``` 59 | -------------------------------------------------------------------------------- /api_example.py: -------------------------------------------------------------------------------- 1 | #API Usage Example 2 | #========== 3 | #To use: Modify the fqdn, username, and password 4 | 5 | 6 | import isilon 7 | import time 8 | import logging 9 | 10 | def main(): 11 | fqdn = '192.168.167.101' #change to your cluster 12 | username = 'root' 13 | password = 'a' 14 | 15 | #httplib.HTTPConnection.debuglevel = 1 16 | logging.basicConfig() 17 | logging.getLogger().setLevel(logging.CRITICAL) 18 | logging.captureWarnings(True) 19 | 20 | 21 | #connect, secure=False allows us to bypass the CA validation 22 | api = isilon.API(fqdn, username, password, secure=False) 23 | 24 | # not necessary as it will connect automatically on auth failure 25 | # but this avoids the initial attempt failure 26 | print("Connecting") 27 | api.session.connect() 28 | 29 | 30 | 31 | #Check for old bad snaps and delete 32 | print("Checking for older snaps") 33 | if api.platform.snapshot('testsnap') : 34 | print("We found an existing testsnap, let's delete that...") 35 | api.platform.snapshot_delete('testsnap') 36 | 37 | 38 | #This shows how we can pass params directly to the API though specifically not called 39 | #out as a param for the snapshot_create function. 40 | print("create a snapshot on %s, to expire in 60 seconds" % testfolder ) 41 | api.platform.snapshot_create("testsnap",testfolder,expires=int(time.time()+60)) 42 | 43 | print("confirm test snapshot was created details:") 44 | print(api.platform.snapshot('testsnap')) 45 | 46 | print("Modify the snapshot expire time and rename to testsnap2") 47 | api.platform.snapshot_modify('testsnap',name='testsnap2',expires=int(time.time() + 120)) 48 | 49 | print("Rename back testsnap") 50 | api.platform.snapshot_modify('testsnap2',name='testsnap') 51 | 52 | #debug last API call: 53 | api.session.debug_last() 54 | 55 | #list all snaps 56 | print('\nListing of All Snaps:') 57 | for snap in api.platform.snapshot(limit=2): 58 | print("Name: %s, Path: %s, Created: %s" % (snap['name'], snap['path'], time.ctime(snap['created']) )) 59 | 60 | #cleanup our testnsap 61 | api.platform.snapshot_delete('testsnap') 62 | 63 | 64 | print("list all quotas") 65 | for q in api.platform.quota(): 66 | pcnt_used = 100 * q['usage']['logical'] / q['thresholds']['hard'] 67 | print("Path %s Persona: %s Percent Used: %s" % (q['path'] , q['persona'], pcnt_used)) 68 | 69 | 70 | 71 | 72 | if __name__ == "__main__": 73 | main() -------------------------------------------------------------------------------- /isilon/platform.py: -------------------------------------------------------------------------------- 1 | import json 2 | import requests 3 | import httplib 4 | import logging 5 | import time 6 | 7 | from .exceptions import ( ObjectNotFound, IsilonLibraryError ) 8 | 9 | class Platform(object): 10 | 11 | '''Implements higher level functionality to interface with an Isilon cluster''' 12 | def __init__(self, session): 13 | self.log = logging.getLogger(__name__) 14 | self.log.addHandler(logging.NullHandler()) 15 | self.session = session 16 | self.platform_url = '/platform/' 17 | 18 | def _override(self,params,overrides): 19 | '''copy overrides into params dict, so user can specify additional params not specifically layed out''' 20 | for (k,v) in overrides.items(): 21 | params[k] = v 22 | return params 23 | 24 | def api_call(self,method, url,**kwargs): 25 | '''add the platform prefix to the api call''' 26 | return self.session.api_call(method, self.platform_url + url,**kwargs) 27 | 28 | def api_call_resumeable(self,method,url,**kwargs): 29 | '''add the namespace prefix to the api call''' 30 | return self.session.api_call_resumeable(method,self.platform_url + url,**kwargs) 31 | 32 | def snapshot(self,name="",**kwargs): 33 | '''Get a list of snaps, refer to API docs for other key value pairs accepted as params ''' 34 | #if a specific name is specified we want to return a single object 35 | #else we are going to return a generator function 36 | return self.api_call_resumeable("GET","1/snapshot/snapshots/" + name, params=kwargs) 37 | 38 | def snapshot_create(self,name,path,**kwargs): 39 | '''Create snapshot''' 40 | data = self._override({'name':name , 'path': path }, kwargs) 41 | return self.api_call("POST", "1/snapshot/snapshots", json=data ) 42 | 43 | def snapshot_modify(self,orig_name,**kwargs): 44 | '''Modify snapshot''' 45 | return self.api_call("PUT", "1/snapshot/snapshots/" + orig_name ,json=kwargs) 46 | 47 | def snapshot_delete(self,name,**kwargs): 48 | '''Delete snapshot''' 49 | if name == "": 50 | #This will delete all sanpshots, lets fail and make a seperate func just for that 51 | raise IsilonLibraryError("Empty name field for snapshot delete, use snapshot_delete_all to delete all snaps") 52 | return self.api_call("DELETE", "1/snapshot/snapshots/" + name,params=kwargs) 53 | 54 | def snapshot_delete_all(self,**kwargs): 55 | return self.api_call("DELETE", "1/snapshot/snapshots/",params=kwargs) 56 | 57 | def quota(self,**kwargs): 58 | '''Get a list of quotas, refer to API docs for other key value pairs accepted as params ''' 59 | options={'resolve_names' : True} 60 | #else we are going to return a generator function 61 | return self.api_call_resumeable('GET','1/quota/quotas/', params=self._override(options,kwargs)) 62 | 63 | def quota_create(self,name,quota,**kwargs): 64 | '''Create quota''' 65 | data = self._override({'name':name , 'path': path }, kwargs) 66 | return self.api_call("POST", "1/quota/quotas", json=data ) 67 | 68 | def quota_modify(self,orig_name,**kwargs): 69 | '''Modify quota''' 70 | return self.api_call("PUT", "1/quota/quotas/" + orig_name ,json=kwargs) 71 | 72 | def quota_delete(self,name,**kwargs): 73 | '''Delete quota''' 74 | if name == "": 75 | #This will delete all sanpshots, lets fail and make a seperate func just for that 76 | raise IsilonLibraryError("Empty name field for quota delete, use quota_delete_all") 77 | return self.api_call("DELETE", "1/quota/quotas/" + name,params=kwargs) 78 | 79 | def quota_delete_all(self,**kwargs): 80 | return self.api_call("DELETE","1/quota/quotas/",params=kwargs) 81 | 82 | def hdfs_racks(self,**kwargs): 83 | return self.api_call_resumeable('GET','1/protocols/hdfs/racks',params=kwargs) 84 | 85 | 86 | def config(self): 87 | '''get Config''' 88 | return self.api_call("GET", "1/cluster/config") 89 | 90 | 91 | 92 | -------------------------------------------------------------------------------- /test_isilon.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import isilon 3 | import ssl 4 | import socket 5 | import sys 6 | import requests 7 | import time 8 | 9 | from requests.exceptions import SSLError 10 | from isilon.exceptions import * 11 | 12 | fqdn = '192.168.167.101' 13 | user = 'root' 14 | password = 'a' 15 | testfolder = '/ifs/apitest_' + str(int(time.time())) 16 | testsnap = 'apitest_snap' + str(int(time.time())) 17 | 18 | class IsilonAPI(unittest.TestCase): 19 | def setUp(self): 20 | pass 21 | #self.api = isilon.API(fqdn,user,password,secure=False) 22 | 23 | 24 | def test_bad_ssl_cert(self): 25 | api = isilon.API(fqdn,user,password) 26 | self.assertRaises(requests.exceptions.SSLError,api.session.connect) 27 | 28 | def test_snap_change(self): 29 | #expire test snaps 60 seconds into the future 30 | expires = int(time.time()) + 600 31 | api = isilon.API(fqdn,user,password,secure=False) 32 | api.platform.snapshot_create(testsnap,"/ifs/data",expires=expires) 33 | self.assertGreater(len(api.platform.snapshot(testsnap)),0) 34 | self.assertGreater(len(api.platform.snapshot()),0) 35 | api.platform.snapshot_modify(testsnap,expires=expires+1) 36 | self.assertEqual(api.platform.snapshot(testsnap)[0]['expires'], expires+1) 37 | api.platform.snapshot_delete(testsnap) 38 | 39 | def test_snap_bad_query(self): 40 | api = isilon.API(fqdn,user,password,secure=False) 41 | if api.platform.snapshot("fakename123askksdfkasdfkas"): 42 | self.assertTrue(False) 43 | 44 | def test_empty_resumable_api_set(self): 45 | api = isilon.API(fqdn,user,password,secure=False) 46 | results = api.platform.hdfs_racks() 47 | self.assertEqual(len(results),0) 48 | for x in results: 49 | self.assertTrue(False) 50 | 51 | def test_snap_delete_all(self): 52 | api = isilon.API(fqdn,user,password,secure=False) 53 | self.assertRaises(IsilonLibraryError,api.platform.snapshot_delete,"") 54 | 55 | def test_file_creation(self): 56 | api = isilon.API(fqdn,user,password,secure=False) 57 | 58 | #Create some test files / folders 59 | api.namespace.dir_create(testfolder) 60 | self.assertTrue(api.namespace.is_dir(testfolder)) 61 | 62 | for x in ('a','b'): 63 | subfolder = testfolder + '/' + x 64 | api.namespace.dir_create(subfolder) 65 | for y in range(1,10): 66 | api.namespace.file_create(subfolder + '/' + str(x) + str(y),"test_file") 67 | 68 | #namespace example 69 | dir_a = [] 70 | dir_b = [] 71 | 72 | gena = api.namespace.dir(testfolder + '/a', limit=2, sort='name', dir='ASC') 73 | genb = api.namespace.dir(testfolder + '/b', limit=2, sort='name',dir='DESC') 74 | while True: 75 | try: 76 | itema = gena.next() 77 | itemb = genb.next() 78 | dir_a.append(itema['name']) 79 | dir_b.append(itemb['name']) 80 | except StopIteration: 81 | break 82 | 83 | self.assertEqual(dir_a,[u'a1', u'a2', u'a3', u'a4', u'a5', u'a6', u'a7', u'a8', u'a9']) 84 | self.assertEqual(dir_b,[u'b9', u'b8', u'b7', u'b6', u'b5', u'b4', u'b3', u'b2', u'b1']) 85 | 86 | for x in ('a','b'): 87 | subfolder = testfolder + '/' + x 88 | for y in range(1,10): 89 | self.assertEqual(api.namespace.file(subfolder + '/' + str(x) + str(y)),"test_file") 90 | api.namespace.file_delete(subfolder + '/' + str(x) + str(y)) 91 | 92 | api.namespace.dir_delete(subfolder) 93 | 94 | def test_access_points(self): 95 | api = isilon.API(fqdn,user,password,secure=False) 96 | 97 | #create and get listing again 98 | api.namespace.dir_create(testfolder) 99 | api.namespace.accesspoint_create(name='test_accesspoint',path=testfolder) 100 | 101 | self.assertEqual( api.namespace.accesspoint()['test_accesspoint'], testfolder) 102 | 103 | #test acls 104 | acl = api.namespace.accesspoint_getacl('test_accesspoint') 105 | api.namespace.accesspoint_setacl(name='test_accesspoint',acl=acl) 106 | 107 | #cleanup 108 | api.namespace.accesspoint_delete('test_accesspoint') 109 | 110 | 111 | 112 | 113 | 114 | 115 | if __name__ == '__main__': 116 | unittest.main() 117 | 118 | 119 | -------------------------------------------------------------------------------- /isilon/namespace.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import time 4 | from pprint import pprint 5 | 6 | from .exceptions import ( ObjectNotFound, IsilonLibraryError ) 7 | 8 | 9 | 10 | class Namespace(object): 11 | 12 | '''Implements higher level functionality to interface with an Isilon cluster''' 13 | 14 | def __init__(self, session): 15 | self.log = logging.getLogger(__name__) 16 | self.log.addHandler(logging.NullHandler()) 17 | self.session = session 18 | self.namespace_url = '/namespace' 19 | 20 | #initialize session timeout values 21 | self.timeout = 0 22 | 23 | def _override(self,params,overrides): 24 | '''copy overrides into params dict, so user can specify additional params not specifically layed out''' 25 | for (k,v) in overrides.items(): 26 | params[k] = v 27 | return params 28 | 29 | def api_call(self,method, url,**kwargs): 30 | '''add the namespace prefix to the api call''' 31 | return self.session.api_call(method, self.namespace_url + url,**kwargs) 32 | 33 | def api_call_resumeable(self,method,url,**kwargs): 34 | '''add the namespace prefix to the api call''' 35 | return self.session.api_call_resumeable(method,self.namespace_url + url,**kwargs) 36 | 37 | def accesspoint(self): 38 | r = self.api_call("GET", "") 39 | 40 | data = r['namespaces'] 41 | 42 | #move all the name, value pairs into an actual dictionary for easy use. 43 | 44 | results = {} 45 | for x in data: 46 | results[ x['name'] ] = x['path'] 47 | return results 48 | 49 | 50 | 51 | def accesspoint_create(self,name,path): 52 | data = { 'path': path } 53 | r = self.api_call("PUT", '/' + name.strip('/'), data=json.dumps(data) ) 54 | 55 | def accesspoint_delete(self,name): 56 | r = self.api_call("DELETE", '/' + name.strip('/') ) 57 | 58 | def accesspoint_setacl(self,name,acl): 59 | self.acl_set('/' + name,acl,nsaccess=True) 60 | 61 | def accesspoint_getacl(self,name): 62 | return self.acl('/' + name,nsaccess=True) 63 | 64 | 65 | def acl(self,path,nsaccess=False): 66 | '''get acl''' 67 | options = "?acl" 68 | if nsaccess: 69 | options += "&nsaccess=true" 70 | try: 71 | return self.api_call("GET", path + options) 72 | except ObjectNotFound: 73 | return None 74 | 75 | def acl_set(self,path,acls,nsaccess=False): 76 | '''set acl''' 77 | options = "?acl" 78 | if nsaccess: 79 | options += "&nsaccess=true" 80 | acls['authoritative'] = "acl" 81 | r = self.api_call("PUT", path + options,data=json.dumps(acls)) 82 | 83 | 84 | def metadata(self,path): 85 | '''get metadata''' 86 | options = "?metadata" 87 | try: 88 | data = self.api_call("GET", path + options) 89 | except ObjectNotFound: 90 | return None 91 | 92 | if not 'attrs' in data: 93 | return None 94 | data = data['attrs'] 95 | 96 | #move all the name, value pairs into an actual dictionary for easy use. 97 | results = {} 98 | for x in data: 99 | #print(x) 100 | results[ x['name'] ] = x['value'] 101 | return results 102 | 103 | def metatdata_set(self,path,metadata): 104 | pass 105 | 106 | 107 | def file_copy(self,src_path, dst_path, clone=False, snapshot=None): 108 | '''Copy a file''' 109 | options={'clone' : clone} 110 | if clone and snapshot: 111 | options['snapshot'] = snapshot 112 | headers = { "x-isi-ifs-copy-source" : "/namespace" + src_path } 113 | return self.api_call("PUT", dst_path, params=options, headers=headers) 114 | 115 | def file_create(self, path, data, overwrite=False ): 116 | '''Uploads a file ''' 117 | headers = { "x-isi-ifs-target-type" : "object" , 'content-type' : 'application/octet-stream' } 118 | return self.api_call("PUT", path , data=data, params={'overwrite' : overwrite}, headers=headers) 119 | 120 | def file(self,path,**kwargs): 121 | try: 122 | return self.api_call("GET", path,params=kwargs) 123 | except ObjectNotFound: 124 | return None 125 | 126 | def file_delete(self,path): 127 | return self.api_call("DELETE", path) 128 | 129 | 130 | 131 | def dir(self,path,**kwargs): 132 | '''Get directory listing''' 133 | params = self._override({'detail':'type'},kwargs) 134 | #Resumable catches object not found a returns an empty list 135 | return self.api_call_resumeable("GET", path,params=params) 136 | 137 | def exists(self,path): 138 | if self.metadata(path): 139 | return True 140 | return False 141 | 142 | def is_dir(self,path): 143 | metadata = self.metadata(path) 144 | 145 | if metadata and 'type' in metadata and metadata['type'] == "container" : 146 | return True 147 | return False 148 | 149 | 150 | def dir_create(self,path,recursive=True): 151 | '''Create a new directory''' 152 | headers = { "x-isi-ifs-target-type" : "container" } 153 | options={'recursive' : recursive } 154 | r = self.api_call("PUT", path, params=options, headers=headers) 155 | 156 | 157 | def dir_delete(self,path): 158 | '''delete a directory''' 159 | options="" 160 | r = self.api_call("DELETE", path + options) 161 | 162 | 163 | 164 | 165 | -------------------------------------------------------------------------------- /isilon/session.py: -------------------------------------------------------------------------------- 1 | import json 2 | import requests 3 | import httplib 4 | import logging 5 | import time 6 | from pprint import pprint 7 | import sys 8 | import socket 9 | 10 | from .exceptions import ( ConnectionError, ObjectNotFound, APIError, IsilonLibraryError ) 11 | 12 | 13 | class GenToIter(object): 14 | ''' Converts a generator object into an iterator so we can use len(results) 15 | we do this by passing the total as the first result from our generator function. 16 | Also, makes the actual API call on init rather than on the 17 | first next() as with a generator''' 18 | 19 | def __init__(self,gen): 20 | self.gen=gen 21 | self.length=gen.next() 22 | 23 | def __iter__(self): 24 | return self 25 | 26 | def next(self): 27 | return self.gen.next() 28 | 29 | def __len__(self): 30 | return self.length 31 | 32 | 33 | class Session(object): 34 | 35 | '''Implements higher level functionality to interface with an Isilon cluster''' 36 | 37 | def __init__(self, fqdn, username, password, secure=True, port = 8080, services=('namespace','platform')): 38 | self.log = logging.getLogger(__name__) 39 | self.log.addHandler(logging.NullHandler()) 40 | self.ip = socket.gethostbyname(fqdn) 41 | self.port = port 42 | self.username = username 43 | self.password = password 44 | self.services = services 45 | self.url= "https://" + self.ip + ':' + str(port) 46 | self.session_url = self.url + '/session/1/session' 47 | 48 | #disable invalid security certificate warnings 49 | if(not secure): 50 | requests.packages.urllib3.disable_warnings() 51 | 52 | #Create HTTPS Requests session object 53 | self.s = requests.Session() 54 | self.s.headers.update({'content-type': 'application/json'}) 55 | self.s.verify = secure 56 | 57 | #initialize session timeout values 58 | self.timeout = 0 59 | self.r = None 60 | 61 | 62 | def log_api_call(self,r,lvl): 63 | 64 | self.log.log(lvl, "===========+++ API Call=================================================================") 65 | self.log.log(lvl,"%s %s , HTTP Code: %d" % (r.request.method, r.request.url, r.status_code)) 66 | self.log.log(lvl,"Request Headers: %s" % r.request.headers) 67 | self.log.log(lvl,"Request Data : %s" % (r.request.body)) 68 | self.log.log(lvl,"") 69 | self.log.log(lvl,"Response Headers: %s " % r.headers ) 70 | self.log.log(lvl,"Response Data: %s" % (r.text.strip())) 71 | self.log.log(lvl,"=========================================================================================") 72 | 73 | def debug_last(self): 74 | if self.r: 75 | self.log_api_call(self.r,logging.ERROR) 76 | 77 | def api_call(self,method,url,**kwargs): 78 | 79 | #check to see if there is a valid session 80 | if time.time() > self.timeout: 81 | self.connect() 82 | 83 | #incoming API call uses a relative path 84 | url = self.url + url 85 | 86 | if len(url) > 8198: 87 | self.log.exception("URL Length too long: %s", url) 88 | 89 | #make actual API call 90 | r = self.s.request(method,url,**kwargs) 91 | self.r = r 92 | 93 | #check for authorization issue and retry if we just need to create a new session 94 | if r.status_code == 401: 95 | #self.bad_call(r) 96 | logging.info("Authentication Failure, trying to reconnect session") 97 | self.connect() 98 | r = self.s.request(method,url,**kwargs) 99 | self.r = r 100 | 101 | 102 | if r.status_code == 404: 103 | self.log_api_call(r,logging.DEBUG) 104 | raise ObjectNotFound() 105 | elif r.status_code == 401: 106 | self.log_api_call(r,logging.ERROR) 107 | raise APIError("Authentication failure") 108 | elif r.status_code > 204: 109 | self.log_api_call(r,logging.ERROR) 110 | message = "API Error: %s" % r.text 111 | raise APIError(message) 112 | else: 113 | self.log_api_call(r,logging.DEBUG) 114 | 115 | 116 | #if type json lets return the json directly 117 | if 'content-type' in r.headers: 118 | if 'application/json' == r.headers['content-type']: 119 | return r.json() 120 | 121 | return r.text 122 | 123 | 124 | 125 | def api_call_resumeable(self,method,url,**kwargs): 126 | ''' Returns a generator, lists through all objects even if it requires multiple API calls ''' 127 | def _api_call_resumeable(method,url,**kwargs): 128 | #initialize state for resuming 129 | object_name = None 130 | resume=None 131 | total = sys.maxint 132 | 133 | #Make First API Call 134 | try: 135 | data = self.api_call(method, url, **kwargs) 136 | except ObjectNotFound: 137 | yield 0 138 | return 139 | 140 | #Find the object name we are going to iterate, it should be the only array at the top level 141 | for k,v in data.items(): 142 | if isinstance(v,list): 143 | if not object_name is None: 144 | #found two arrays... also this will break this logic 145 | raise IsilonLibraryError("two arrays found in resumeable api call") 146 | object_name = k 147 | 148 | if object_name is None: 149 | #we can't find the object name, lets throw an exception because this shouldn't happen: 150 | raise IsilonLibraryError("no array found in resumable api call") 151 | 152 | 153 | if 'total' in data: 154 | total = data['total'] 155 | else: 156 | if 'resume' in data and data['resume']: 157 | total = sys.maxint 158 | else: 159 | total = len(data[object_name]) 160 | 161 | yield total 162 | 163 | 164 | 165 | #We will loop through as many api calls as needed to retrieve all items 166 | while True: 167 | 168 | if object_name in data: 169 | for obj in data[object_name]: 170 | yield obj 171 | else: 172 | raise IsilonLibraryError("expected data object is missing") 173 | 174 | #Check for a resume token, is it valid, if so api call the next set of results, else break 175 | if 'resume' in data and data['resume']: 176 | kwargs['params'] = {'resume': data['resume'] } 177 | data = self.api_call(method, url, **kwargs) 178 | else: 179 | break 180 | 181 | #no more resume tokens 182 | return #end of _api_call_resumeable 183 | 184 | results = GenToIter(_api_call_resumeable(method,url,**kwargs)) 185 | #for queries that should return a few results just return a list so that full indexing works 186 | if len(results) < 10: 187 | return list(results) 188 | else: 189 | return results 190 | 191 | 192 | def connect(self): 193 | 194 | #Get an API session cookie from Isilon 195 | #Cookie is automatically added to HTTP requests session 196 | logging.debug("--> creating session") 197 | sessionjson = json.dumps({'username': self.username , 'password': self.password , 'services': self.services}) 198 | 199 | r = self.s.post(self.session_url,data=sessionjson) 200 | if r.status_code != 201 : 201 | #r.raise_for_status() 202 | self.log_api_call(r,logging.ERROR) 203 | raise ConnectionError(r.text) 204 | 205 | #Renew Session 60 seconds prior to our timeout 206 | self.timeout = time.time() + r.json()['timeout_absolute'] - 60 207 | logging.debug("New Session created! Current clock %d, timeout %d" % (time.time(),self.timeout)) 208 | return True 209 | 210 | --------------------------------------------------------------------------------