├── src └── mlab │ ├── __init__.py │ ├── mlabraw.py │ ├── releases.py │ ├── matlabcom.py │ ├── matlabpipe.py │ ├── mlabwrap.py │ └── awmstools.py ├── .gitignore ├── tests └── test_mlab_on_unix.py ├── LICENSE ├── setup.py └── README.rst /src/mlab/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | MLAB calls MATLAB funcitons and looks like a normal python library. 3 | 4 | authors: 5 | Alexander Schmolck 6 | Dani Valevski 7 | Yauhen Yakimovich 8 | 9 | ''' 10 | 11 | import releases 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | 21 | # Installer logs 22 | pip-log.txt 23 | 24 | # Unit test / coverage reports 25 | .coverage 26 | .tox 27 | nosetests.xml 28 | 29 | # Translations 30 | *.mo 31 | 32 | # Mr Developer 33 | .mr.developer.cfg 34 | .project 35 | .pydevproject 36 | -------------------------------------------------------------------------------- /tests/test_mlab_on_unix.py: -------------------------------------------------------------------------------- 1 | import sys 2 | sys.path = ['../src/'] + sys.path 3 | import unittest 4 | from mlab.mlabwrap import MatlabReleaseNotFound 5 | 6 | 7 | class TestMlabUnix(unittest.TestCase): 8 | 9 | def setUp(self): 10 | pass 11 | 12 | def tearDown(self): 13 | pass 14 | 15 | def test_version_discovery(self): 16 | import mlab 17 | instances = mlab.releases.MatlabVersions(globals()) 18 | assert len(instances.pick_latest_release()) > 0 19 | with self.assertRaises(MatlabReleaseNotFound): 20 | mlab_inst = instances.get_mlab_instance('R2010c') 21 | 22 | def test_latest_release(self): 23 | from mlab.releases import latest_release 24 | from matlab import matlabroot 25 | self.assertTrue(len(matlabroot())>0) 26 | matlabroot() 27 | 28 | 29 | if __name__ == '__main__': 30 | unittest.main() 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 Yauhen Yakimovich 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os.path 3 | try: 4 | from setuptools import setup 5 | except ImportError: 6 | from distutils.core import setup 7 | 8 | 9 | def readme(): 10 | try: 11 | import docutils.core 12 | except ImportError: 13 | try: 14 | with open(os.path.join(os.path.dirname(__file__), 15 | 'README.rst')) as f: 16 | return f.read() 17 | except (IOError, OSError): 18 | return '' 19 | with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as f: 20 | document = docutils.core.publish_doctree(f.read()) 21 | nodes = list(document) 22 | description = '' 23 | for node in nodes: 24 | if str(node).startswith('= 0.11', 50 | # ], 51 | ) 52 | -------------------------------------------------------------------------------- /src/mlab/mlabraw.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | ''' 3 | A quick and a bit less dirty hack to wrap matlabpipe/matlabcom as if they 4 | were mlabraw. 5 | 6 | Author: Dani Valevski 7 | Yauhen Yakimovich 8 | 9 | License: MIT 10 | ''' 11 | import platform 12 | is_win = platform.system() == 'Windows' 13 | if is_win: 14 | from matlabcom import MatlabCom as MatlabConnection 15 | from matlabcom import MatlabError as error 16 | from matlabcom import discover_location, find_available_releases 17 | from matlabcom import WindowsMatlabReleaseNotFound as MatlabReleaseNotFound 18 | else: 19 | from matlabpipe import MatlabPipe as MatlabConnection 20 | from matlabpipe import MatlabError as error 21 | from matlabpipe import discover_location, find_available_releases 22 | from matlabpipe import UnixMatlabReleaseNotFound as MatlabReleaseNotFound 23 | try: 24 | import settings 25 | except ImportError: 26 | class settings: 27 | MATLAB_PATH = 'guess' 28 | import traceback 29 | 30 | 31 | _MATLAB_RELEASE='latest' 32 | 33 | 34 | def set_release(matlab_release): 35 | global _MATLAB_RELEASE 36 | _MATLAB_RELEASE=matlab_release 37 | 38 | 39 | def open(): 40 | global _MATLAB_RELEASE 41 | '''Opens MATLAB using specified connection (or DCOM+ protocol on Windows)where matlab_location ''' 42 | if is_win: 43 | ret = MatlabConnection() 44 | ret.open() 45 | return ret 46 | else: 47 | if settings.MATLAB_PATH != 'guess': 48 | matlab_path = settings.MATLAB_PATH + '/bin/matlab' 49 | elif _MATLAB_RELEASE != 'latest': 50 | matlab_path = discover_location(_MATLAB_RELEASE) 51 | else: 52 | # Latest release is found in __init__.by, i.e. higher logical level 53 | raise MatlabReleaseNotFound('Please select a matlab release or set its location.') 54 | try: 55 | ret = MatlabConnection(matlab_path) 56 | ret.open() 57 | except Exception: 58 | #traceback.print_exc(file=sys.stderr) 59 | raise MatlabReleaseNotFound('Could not open matlab, is it in %s?' % matlab_path) 60 | return ret 61 | 62 | def close(matlab): 63 | matlab.close() 64 | 65 | def eval(matlab, exp, log=False): 66 | if log or is_win: 67 | matlab.eval(exp) 68 | else: 69 | matlab.eval(exp, print_expression=False, on_new_output=None) 70 | return '' 71 | 72 | def get(matlab, var_name): 73 | return matlab.get(var_name) 74 | 75 | def put(matlab, var_name, val): 76 | matlab.put({var_name : val}) 77 | -------------------------------------------------------------------------------- /src/mlab/releases.py: -------------------------------------------------------------------------------- 1 | ''' 2 | MLAB calls MATLAB funcitons and looks like a normal python library. 3 | 4 | authors: 5 | Yauhen Yakimovich 6 | 7 | Module wrapping borrowed from `sh` project by Andrew Moffat. 8 | ''' 9 | import os 10 | import sys 11 | from types import ModuleType 12 | from mlabwrap import (MlabWrap, choose_release, find_available_releases, 13 | MatlabReleaseNotFound) 14 | import traceback 15 | 16 | 17 | # TODO: work with ENV 18 | #os.getenv("MLABRAW_CMD_STR", "") 19 | 20 | 21 | def get_available_releases(): 22 | return dict(find_available_releases()) 23 | 24 | 25 | def get_latest_release(available_releases=None): 26 | if not available_releases: 27 | available_releases = dict(find_available_releases()) 28 | versions = available_releases.keys() 29 | latest_release_version = sorted(versions)[-1] 30 | return latest_release_version 31 | 32 | 33 | class MatlabVersions(dict): 34 | 35 | def __init__(self, globs): 36 | self.globs = globs 37 | self.__selected_instance = None 38 | self._available_releases = dict(find_available_releases()) 39 | 40 | def __setitem__(self, k, v): 41 | self.globs[k] = v 42 | 43 | def __getitem__(self, k): 44 | try: return self.globs[k] 45 | except KeyError: pass 46 | 47 | # the only way we'd get to here is if we've tried to 48 | # import * from a repl. so, raise an exception, since 49 | # that's really the only sensible thing to do 50 | if k == "__all__": 51 | raise ImportError('Cannot import * from mlab. Please import mlab ' 52 | 'or import versions individually.') 53 | 54 | if k.startswith("__") and k.endswith("__"): 55 | raise AttributeError 56 | 57 | # how about an return a function variable? 58 | # try: return os.environ[k] 59 | # except KeyError: pass 60 | 61 | # is it a "release version"? 62 | if k.startswith('R') and k in self._available_releases: 63 | self[k] = self.get_mlab_instance(k) 64 | return self[k] 65 | 66 | if k == 'latest_release': 67 | matlab_release = self.pick_latest_release() 68 | instance = self.get_mlab_instance(matlab_release) 69 | self[k] = instance 70 | self[matlab_release] = instance 71 | return instance 72 | 73 | if self.__selected_instance is not None: 74 | instance = self[self.__selected_instance] 75 | try: 76 | return getattr(instance, k) 77 | except AttributeError: 78 | traceback.print_exc(file=sys.stdout) 79 | else: 80 | raise ImportError('Import failed, no MATLAB instance selected. Try import mlab.latest_release') 81 | 82 | 83 | raise ImportError('Failed to import anything for: %s' % k) 84 | 85 | def get_mlab_instance(self, matlab_release): 86 | choose_release(matlab_release) 87 | instance = MlabWrap() 88 | # Make it a module 89 | sys.modules['mlab.releases.' + matlab_release] = instance 90 | sys.modules['matlab'] = instance 91 | return instance 92 | 93 | def pick_latest_release(self): 94 | return get_latest_release(self._available_releases) 95 | 96 | 97 | # this is a thin wrapper around THIS module (we patch sys.modules[__name__]). 98 | # this is in the case that the user does a "from sh import whatever" 99 | # in other words, they only want to import certain programs, not the whole 100 | # system PATH worth of commands. in this case, we just proxy the 101 | # import lookup to our MatlabVersions class 102 | class SelfWrapper(ModuleType): 103 | def __init__(self, self_module): 104 | # this is super ugly to have to copy attributes like this, 105 | # but it seems to be the only way to make reload() behave 106 | # nicely. if i make these attributes dynamic lookups in 107 | # __getattr__, reload sometimes chokes in weird ways... 108 | for attr in ["__builtins__", "__doc__", "__name__", "__package__"]: 109 | setattr(self, attr, getattr(self_module, attr, None)) 110 | # python 3.2 (2.7 and 3.3 work fine) breaks on osx (not ubuntu) 111 | # if we set this to None. and 3.3 needs a value for __path__ 112 | self.__path__ = [] 113 | self.module = self_module 114 | self.instances = MatlabVersions(globals()) 115 | 116 | def __setattr__(self, name, value): 117 | #if hasattr(self, "instances"): self.instances[name] = value 118 | ModuleType.__setattr__(self, name, value) 119 | 120 | def __getattr__(self, name): 121 | if name == "instances": raise AttributeError 122 | if name in dir(self.module): 123 | return getattr(self.module, name) 124 | 125 | return self.instances[name] 126 | 127 | # accept special keywords argument to define defaults for all operations 128 | # that will be processed with given by return SelfWrapper 129 | def __call__(self, **kwargs): 130 | return SelfWrapper(self.self_module, kwargs) 131 | 132 | 133 | 134 | # we're being imported from somewhere 135 | if __name__ != '__main__': 136 | self = sys.modules[__name__] 137 | sys.modules[__name__] = SelfWrapper(self) 138 | -------------------------------------------------------------------------------- /src/mlab/matlabcom.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ A python module for raw communication with Matlab(TM) using COM client 4 | under windows. 5 | 6 | The module sends commands to the matlab process as a COM client. This is only 7 | supported under windows. 8 | 9 | Author: Dani Valevski 10 | Yauhen Yakimovich 11 | 12 | Dependencies: pywin32, numpy 13 | Tested Matlab Versions: 2011a 14 | License: MIT 15 | """ 16 | 17 | import numpy as np 18 | try: 19 | import win32com.client 20 | except: 21 | print 'win32com in missing, please install it' 22 | raise 23 | 24 | def find_available_releases(): 25 | # report we have all versions 26 | return [('R%d%s' % (y,v), '') 27 | for y in range(2006,2015) for v in ('a','b')] 28 | 29 | def discover_location(matlab_release): 30 | pass 31 | 32 | class WindowsMatlabReleaseNotFound(Exception): 33 | '''Raised if specified release was not found.''' 34 | 35 | class MatlabError(Exception): 36 | """Raised when a Matlab evaluation results in an error inside Matlab.""" 37 | pass 38 | 39 | class MatlabConnectionError(Exception): 40 | """Raised for errors related to the Matlab connection.""" 41 | pass 42 | 43 | class MatlabCom(object): 44 | """ Manages a matlab COM client. 45 | 46 | The process can be opened and close with the open/close methods. 47 | To send a command to the matlab shell use 'eval'. 48 | To load numpy data to the matlab shell use 'put'. 49 | To retrieve numpy data from the matlab shell use 'get'. 50 | """ 51 | def __init__(self, matlab_process_path=None, matlab_version=None): 52 | self.client = None 53 | 54 | def open(self, visible=False): 55 | """ Dispatches the matlab COM client. 56 | 57 | Note: If this method fails, try running matlab with the -regserver flag. 58 | """ 59 | if self.client: 60 | raise MatlabConnectionError('Matlab(TM) COM client is still active. Use close to ' 61 | 'close it') 62 | self.client = win32com.client.Dispatch('matlab.application') 63 | self.client.visible = visible 64 | 65 | def close(self): 66 | """ Closes the matlab COM client. 67 | """ 68 | self._check_open() 69 | try: 70 | pass #self.eval('quit();') 71 | except: 72 | pass 73 | del self.client 74 | 75 | def eval(self, expression, identify_erros=True): 76 | """ Evaluates a matlab expression synchronously. 77 | 78 | If identify_erros is true, and the last output line after evaluating the 79 | expressions begins with '???' an excpetion is thrown with the matlab error 80 | following the '???'. 81 | The return value of the function is the matlab output following the call. 82 | """ 83 | #print expression 84 | self._check_open() 85 | ret = self.client.Execute(expression) 86 | #print ret 87 | if identify_erros and ret.rfind('???') != -1: 88 | begin = ret.rfind('???') + 4 89 | raise MatlabError(ret[begin:]) 90 | return ret 91 | 92 | def get(self, names_to_get, convert_to_numpy=True): 93 | """ Loads the requested variables from the matlab com client. 94 | 95 | names_to_get can be either a variable name or a list of variable names. 96 | If it is a variable name, the values is returned. 97 | If it is a list, a dictionary of variable_name -> value is returned. 98 | 99 | If convert_to_numpy is true, the method will all array values to numpy 100 | arrays. Scalars are left as regular python objects. 101 | 102 | """ 103 | self._check_open() 104 | single_itme = isinstance(names_to_get, (unicode, str)) 105 | if single_itme: 106 | names_to_get = [names_to_get] 107 | ret = {} 108 | for name in names_to_get: 109 | ret[name] = self.client.GetWorkspaceData(name, 'base') 110 | # TODO(daniv): Do we really want to reduce dimensions like that? what if this a row vector? 111 | while isinstance(ret[name], (tuple, list)) and len(ret[name]) == 1: 112 | ret[name] = ret[name][0] 113 | if convert_to_numpy and isinstance(ret[name], (tuple, list)): 114 | ret[name] = np.array(ret[name]) 115 | if single_itme: 116 | return ret.values()[0] 117 | return ret 118 | 119 | def put(self, name_to_val): 120 | """ Loads a dictionary of variable names into the matlab com client. 121 | """ 122 | self._check_open() 123 | for name, val in name_to_val.iteritems(): 124 | # First try to put data as a matrix: 125 | try: 126 | self.client.PutFullMatrix(name, 'base', val, None) 127 | except: 128 | self.client.PutWorkspaceData(name, 'base', val) 129 | 130 | def _check_open(self): 131 | if not self.client: 132 | raise MatlabConnectionError('Matlab(TM) process is not active.') 133 | 134 | if __name__ == '__main__': 135 | import unittest 136 | 137 | 138 | class TestMatlabCom(unittest.TestCase): 139 | def setUp(self): 140 | self.matlab = MatlabCom() 141 | self.matlab.open() 142 | 143 | def tearDown(self): 144 | self.matlab.close() 145 | 146 | def test_eval(self): 147 | for i in xrange(100): 148 | ret = self.matlab.eval('disp \'hiush world%s\';' % ('b'*i)) 149 | self.assertTrue('hiush world' in ret) 150 | 151 | def test_put(self): 152 | self.matlab.put({'A' : [1, 2, 3]}) 153 | ret = self.matlab.eval('A') 154 | self.assertTrue('A =' in ret) 155 | 156 | def test_1_element(self): 157 | self.matlab.put({'X': 'string'}) 158 | ret = self.matlab.get('X') 159 | self.assertEquals(ret, 'string') 160 | 161 | def test_get(self): 162 | self.matlab.eval('A = [1 2 3];') 163 | ret = self.matlab.get('A') 164 | self.assertEquals(ret[0], 1) 165 | self.assertEquals(ret[1], 2) 166 | self.assertEquals(ret[2], 3) 167 | 168 | def test_error(self): 169 | self.assertRaises(MatlabError, 170 | self.matlab.eval, 171 | 'no_such_function') 172 | 173 | unittest.main() 174 | -------------------------------------------------------------------------------- /src/mlab/matlabpipe.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | '''A python module for raw communication with Matlab(TM) using pipes under 3 | unix. 4 | 5 | This module exposes the same interface as mlabraw.cpp, so it can be used along 6 | with mlabwrap.py. 7 | The module sends commands to the matlab process using the standard input pipe. 8 | It loads data from/to the matlab process using the undocumented save/load stdio 9 | commands. Only unix (or mac osx) versions of Matlab support pipe communication, 10 | so this module will only work under unix (or mac osx). 11 | 12 | Author: Dani Valevski 13 | Yauhen Yakimovich 14 | 15 | Dependencies: scipy 16 | Tested Matlab Versions: 2009b, 2010a, 2010b, 2011a 17 | License: MIT 18 | ''' 19 | 20 | from cStringIO import StringIO 21 | import fcntl 22 | import numpy as np 23 | import os 24 | import scipy.io as mlabio 25 | import select 26 | import subprocess 27 | import sys 28 | 29 | 30 | class UnixMatlabReleaseNotFound(Exception): 31 | '''Raised if specified release was not found.''' 32 | 33 | 34 | def is_linux(): 35 | return 'linux' in sys.platform 36 | 37 | 38 | _RELEASES = None 39 | 40 | 41 | def _list_releases(): 42 | ''' 43 | Tries to guess matlab process release version and location path on 44 | osx machines. 45 | 46 | The paths we will search are in the format: 47 | /Applications/MATLAB_R[YEAR][VERSION].app/bin/matlab 48 | We will try the latest version first. If no path is found, None is reutrned. 49 | ''' 50 | if is_linux(): 51 | base_path = '/usr/local/MATLAB/R%d%s/bin/matlab' 52 | else: 53 | # assume mac 54 | base_path = '/Applications/MATLAB_R%d%s.app/bin/matlab' 55 | years = range(2050,1990,-1) 56 | release_letters = ('h', 'g', 'f', 'e', 'd', 'c', 'b', 'a') 57 | for year in years: 58 | for letter in release_letters: 59 | release = 'R%d%s' % (year, letter) 60 | matlab_path = base_path % (year, letter) 61 | if os.path.exists(matlab_path): 62 | yield (release, matlab_path) 63 | 64 | 65 | def find_available_releases(): 66 | global _RELEASES 67 | if not _RELEASES: 68 | _RELEASES = list(_list_releases()) 69 | return _RELEASES 70 | 71 | 72 | def is_valid_release_version(version): 73 | '''Checks that the given version code is valid.''' 74 | return version is not None and len(version) == 6 and version[0] == 'R' \ 75 | and int(version[1:5]) in range(1990, 2050) \ 76 | and version[5] in ('h', 'g', 'f', 'e', 'd', 'c', 'b', 'a') 77 | 78 | 79 | def discover_location(release_version): 80 | available_releases = dict(find_available_releases()) 81 | if not available_releases: 82 | raise ValueError('There is no MATLAB installation found. Is it installed?') 83 | if not release_version in available_releases: 84 | raise UnixMatlabReleaseNotFound(release_version) 85 | return available_releases[release_version] 86 | 87 | 88 | find_matlab_process = discover_location 89 | 90 | 91 | def find_matlab_version(process_path): 92 | """ Tries to guess matlab's version according to its process path. 93 | 94 | If we couldn't gues the version, None is returned. 95 | """ 96 | bin_path = os.path.dirname(process_path) 97 | matlab_path = os.path.dirname(bin_path) 98 | matlab_dir_name = os.path.basename(matlab_path) 99 | version = matlab_dir_name 100 | if not is_linux(): 101 | version = matlab_dir_name.replace('MATLAB_', '').replace('.app', '') 102 | if not is_valid_release_version(version): 103 | return None 104 | return version 105 | 106 | 107 | class MatlabError(Exception): 108 | """Raised when a Matlab evaluation results in an error inside Matlab.""" 109 | pass 110 | 111 | 112 | class MatlabConnectionError(Exception): 113 | """Raised for errors related to the Matlab connection.""" 114 | pass 115 | 116 | 117 | class MatlabPipe(object): 118 | """ Manages a connection to a matlab process. 119 | 120 | Matlab version is in the format [YEAR][VERSION] for example: 2011a. 121 | The process can be opened and close with the open/close methods. 122 | To send a command to the matlab shell use 'eval'. 123 | To load numpy data to the matlab shell use 'put'. 124 | To retrieve numpy data from the matlab shell use 'get'. 125 | """ 126 | 127 | def __init__(self, matlab_process_path=None, matlab_version=None): 128 | """ Inits the class. 129 | 130 | matlab path should be a path to the matlab executeable. For example: 131 | /Applications/MATLAB_R2010b.app/bin/matlab 132 | """ 133 | if matlab_version is None and matlab_process_path is not None: 134 | matlab_version = find_matlab_version(matlab_process_path) 135 | if not is_valid_release_version(matlab_version): 136 | raise ValueError('Invalid version code %s' % matlab_version) 137 | if matlab_process_path is None: 138 | matlab_process_path = find_matlab_process(matlab_version) 139 | if not os.path.exists(matlab_process_path): 140 | raise ValueError('Matlab process path %s does not exist' % matlab_process_path) 141 | self.matlab_version = (int(matlab_version[1:5]), matlab_version[5]) 142 | self.matlab_process_path = matlab_process_path 143 | self.process = None 144 | self.command_end_string='___MATLAB_PIPE_COMMAND_ENDED___' 145 | self.expected_output_end = '%s\n>> ' % self.command_end_string 146 | self.stdout_to_read = '' 147 | 148 | def open(self, print_matlab_welcome=False): 149 | '''Opens the matlab process.''' 150 | if self.process and not self.process.returncode: 151 | raise MatlabConnectionError('Matlab(TM) process is still active. Use close to ' 152 | 'close it') 153 | self.process = subprocess.Popen( 154 | [self.matlab_process_path, '-nojvm', '-nodesktop'], 155 | stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 156 | flags = fcntl.fcntl(self.process.stdout, fcntl.F_GETFL) 157 | fcntl.fcntl(self.process.stdout, fcntl.F_SETFL, flags| os.O_NONBLOCK) 158 | 159 | if print_matlab_welcome: 160 | self._sync_output() 161 | else: 162 | self._sync_output(None) 163 | 164 | def close(self): 165 | """ Closes the matlab process. 166 | """ 167 | self._check_open() 168 | self.process.stdin.close() 169 | 170 | def eval(self, 171 | expression, 172 | identify_errors=True, 173 | print_expression=True, 174 | on_new_output=sys.stdout.write): 175 | """ Evaluates a matlab expression synchronously. 176 | 177 | If identify_erros is true, and the last output line after evaluating the 178 | expressions begins with '???' and excpetion is thrown with the matlab error 179 | following the '???'. 180 | If on_new_output is not None, it will be called whenever a new output is 181 | encountered. The default value prints the new output to the screen. 182 | The return value of the function is the matlab output following the call. 183 | """ 184 | self._check_open() 185 | if print_expression: 186 | print expression 187 | self.process.stdin.write(expression) 188 | self.process.stdin.write('\n') 189 | ret = self._sync_output(on_new_output) 190 | # TODO(dani): Use stderr to identify errors. 191 | if identify_errors and ret.rfind('???') != -1: 192 | begin = ret.rfind('???') + 4 193 | end = ret.find('\n', begin) 194 | raise MatlabError(ret[begin:end]) 195 | return ret 196 | 197 | 198 | def put(self, name_to_val, oned_as='row', on_new_output=None): 199 | """ Loads a dictionary of variable names into the matlab shell. 200 | 201 | oned_as is the same as in scipy.io.matlab.savemat function: 202 | oned_as : {'column', 'row'}, optional 203 | If 'column', write 1-D numpy arrays as column vectors. 204 | If 'row', write 1D numpy arrays as row vectors. 205 | """ 206 | self._check_open() 207 | # We can't give stdin to mlabio.savemat because it needs random access :( 208 | temp = StringIO() 209 | mlabio.savemat(temp, name_to_val, oned_as=oned_as) 210 | temp.seek(0) 211 | temp_str = temp.read() 212 | temp.close() 213 | self.process.stdin.write('load stdio;\n') 214 | self._read_until('ack load stdio\n', on_new_output=on_new_output) 215 | self.process.stdin.write(temp_str) 216 | #print 'sent %d kb' % (len(temp_str) / 1024) 217 | self._read_until('ack load finished\n', on_new_output=on_new_output) 218 | self._sync_output(on_new_output=on_new_output) 219 | 220 | def get(self, 221 | names_to_get, 222 | extract_numpy_scalars=True, 223 | on_new_output=None): 224 | """ Loads the requested variables from the matlab shell. 225 | 226 | names_to_get can be either a variable name, a list of variable names, or 227 | None. 228 | If it is a variable name, the values is returned. 229 | If it is a list, a dictionary of variable_name -> value is returned. 230 | If it is None, a dictionary with all variables is returned. 231 | 232 | If extract_numpy_scalars is true, the method will convert numpy scalars 233 | (0-dimension arrays) to a regular python variable. 234 | """ 235 | self._check_open() 236 | single_item = isinstance(names_to_get, (unicode, str)) 237 | if single_item: 238 | names_to_get = [names_to_get] 239 | if names_to_get == None: 240 | self.process.stdin.write('save stdio;\n') 241 | else: 242 | # Make sure that we throw an excpetion if the names are not defined. 243 | for name in names_to_get: 244 | self.eval('%s;' % name, print_expression=False, on_new_output=on_new_output) 245 | #print 'save(\'stdio\', \'%s\');\n' % '\', \''.join(names_to_get) 246 | self.process.stdin.write( 247 | "save('stdio', '%s', '-v7');\n" % '\', \''.join(names_to_get)) 248 | # We have to read to a temp buffer because mlabio.loadmat needs 249 | # random access :( 250 | self._read_until('start_binary\n', on_new_output=on_new_output) 251 | #print 'got start_binary' 252 | temp_str = self._sync_output(on_new_output=on_new_output) 253 | #print 'got all outout' 254 | # Remove expected output and "\n>>" 255 | # TODO(dani): Get rid of the unecessary copy. 256 | # MATLAB 2010a adds an extra >> so we need to remove more spaces. 257 | if self.matlab_version == (2010, 'a'): 258 | temp_str = temp_str[:-len(self.expected_output_end)-6] 259 | else: 260 | temp_str = temp_str[:-len(self.expected_output_end)-3] 261 | temp = StringIO(temp_str) 262 | #print ('____') 263 | #print len(temp_str) 264 | #print ('____') 265 | ret = mlabio.loadmat(temp, chars_as_strings=True, squeeze_me=True) 266 | #print '******' 267 | #print ret 268 | #print '******' 269 | temp.close() 270 | if single_item: 271 | return ret.values()[0] 272 | for key in ret.iterkeys(): 273 | while ret[key].shape and ret[key].shape[-1] == 1: 274 | ret[key] = ret[key][0] 275 | if extract_numpy_scalars: 276 | if isinstance(ret[key], np.ndarray) and not ret[key].shape: 277 | ret[key] = ret[key].tolist() 278 | #print 'done' 279 | return ret 280 | 281 | def _check_open(self): 282 | if not self.process or self.process.returncode: 283 | raise MatlabConnectionError('Matlab(TM) process is not active.') 284 | 285 | def _read_until(self, wait_for_str, on_new_output=sys.stdout.write): 286 | all_output = StringIO() 287 | output_tail = self.stdout_to_read 288 | while not wait_for_str in output_tail: 289 | tail_to_remove = output_tail[:-len(output_tail)] 290 | output_tail = output_tail[-len(output_tail):] 291 | if on_new_output: on_new_output(tail_to_remove) 292 | all_output.write(tail_to_remove) 293 | if not select.select([self.process.stdout], [], [], 10)[0]: 294 | raise MatlabConnectionError('timeout') 295 | new_output = self.process.stdout.read(65536) 296 | output_tail += new_output 297 | chunk_to_take, chunk_to_keep = output_tail.split(wait_for_str, 1) 298 | chunk_to_take += wait_for_str 299 | self.stdout_to_read = chunk_to_keep 300 | if on_new_output: on_new_output(chunk_to_take) 301 | all_output.write(chunk_to_take) 302 | all_output.seek(0) 303 | return all_output.read() 304 | 305 | 306 | """ 307 | def _wait_for(self, wait_for_str, on_new_output=sys.stdout.write): 308 | all_output = StringIO() 309 | output_tail = self.stdout_to_read 310 | byte_count = 0 311 | while not wait_for_str in output_tail: 312 | output_tail = output_tail[:-len(wait_for_str) 313 | #print 'before read' 314 | #print output_tail == wait_for_str 315 | #print repr(output_tail) 316 | #print repr(wait_for_str) 317 | #new_output = self.process.stdout.read(1) 318 | new_output = '' 319 | conn = self.process.stdout 320 | flags = fcntl.fcntl(conn, fcntl.F_GETFL) 321 | if not conn.closed: 322 | fcntl.fcntl(conn, fcntl.F_SETFL, flags| os.O_NONBLOCK) 323 | try: 324 | if not select.select([conn], [], [], 10)[0]: 325 | raise Exception('timeout') 326 | new_output = conn.read(65536) 327 | #try: 328 | #new_output = os.read(self.process.stdout.fileno(), 65536) 329 | #new_output = self.process..readad(65536) 330 | finally: 331 | if not conn.closed: 332 | fcntl.fcntl(conn, fcntl.F_SETFL, flags) 333 | 334 | #print 'after read' 335 | if on_new_output: 336 | on_new_output(new_output) 337 | byte_count += len(new_output) 338 | output_tail += new_output 339 | output_tail = output_tail[-len(wait_for_str):] 340 | all_output.write(new_output) 341 | #if byte_count and byte_count % (2 ** 15) == 0: 342 | #print '%d kb' % (byte_count / 1024) 343 | all_output.seek(0) 344 | print 'recv %d kb' % (byte_count / 1024) 345 | return all_output.read() 346 | """ 347 | def _sync_output(self, on_new_output=sys.stdout.write): 348 | self.process.stdin.write('disp(\'%s\');\n' % self.command_end_string) 349 | return self._read_until(self.expected_output_end, on_new_output) 350 | 351 | 352 | if __name__ == '__main__': 353 | import unittest 354 | 355 | 356 | class TestMatlabPipe(unittest.TestCase): 357 | def setUp(self): 358 | self.matlab = MatlabPipe() 359 | self.matlab.open() 360 | 361 | def tearDown(self): 362 | self.matlab.close() 363 | 364 | def test_eval(self): 365 | for i in xrange(100): 366 | ret = self.matlab.eval('disp \'hiush world%s\';' % ('b'*i)) 367 | self.assertTrue('hiush world' in ret) 368 | 369 | def test_put(self): 370 | self.matlab.put({'A' : [1, 2, 3]}) 371 | ret = self.matlab.eval('A') 372 | self.assertTrue('A =' in ret) 373 | 374 | def test_1_element(self): 375 | self.matlab.put({'X': 'string'}) 376 | ret = self.matlab.get('X') 377 | self.assertEquals(ret, 'string') 378 | 379 | def test_get(self): 380 | self.matlab.eval('A = [1 2 3];') 381 | ret = self.matlab.get('A') 382 | self.assertEquals(ret[0], 1) 383 | self.assertEquals(ret[1], 2) 384 | self.assertEquals(ret[2], 3) 385 | 386 | def test_error(self): 387 | self.assertRaises(MatlabError, 388 | self.matlab.eval, 389 | 'no_such_function') 390 | 391 | unittest.main() 392 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. -*- mode: rst; coding: utf-8; -*- 2 | 3 | ================= 4 | mlab 5 | ================= 6 | 7 | Mlab is a high-level python to Matlab® bridge that lets Matlab look like a normal python library. 8 | 9 | 10 | This python library is based on the work of original mlabwrap project 11 | 12 | http://mlabwrap.sourceforge.net/ 13 | 14 | and Dani Valevski (from Dana Pe'er's lab): 15 | 16 | http://code.google.com/p/danapeerlab/source/browse/trunk/freecell/depends/common/python/matlabpipe.py 17 | 18 | Primer 19 | ------ 20 | 21 | Quick installation:: 22 | 23 | pip install mlab 24 | 25 | Start working with the library by picking a MATLAB release that you have locally installed:: 26 | 27 | from mlab.releases import latest_release as matlab 28 | from matlab import matlabroot 29 | 30 | print matlabroot() 31 | 32 | where **latest_release** is a MlabWrap instance, **matlabroot** is wrapper around MATLAB function. 33 | Please note that matlab module is dynamically created instance, which is in this case referencing 34 | **latest_release** object. 35 | 36 | MATLAB installation discovery mechanism is implemented by mlab.releases module in such a way, that 37 | you have to specify the release version you want to use first, by importing it. Only then you can 38 | import from matlab module:: 39 | 40 | from mlab.releases import R2010b as matlab 41 | from matlab import matlabroot 42 | 43 | Also see mlab.releases.get_available_releases(). 44 | 45 | 46 | ------------ 47 | 48 | 49 | Original README 50 | --------------- 51 | 52 | NOTE Below is the original README from the mlabwrap project. Most of it still 53 | applies, but the underlying implementation is different (COM/Pipes replaced 54 | the use of the MATLAB Engine API). The code samples have been updated for use 55 | with this project. 56 | 57 | 58 | .. contents:: 59 | 60 | Description 61 | ----------- 62 | Mlabwrap is a high-level python to `Matlab®`_ bridge that lets Matlab look 63 | like a normal python library. 64 | 65 | Thanks for your terrific work on this very-useful Python tool! 66 | 67 | -- George A. Blaha, Senior Systems Engineer, 68 | Raytheon Integrated Defense Systems 69 | 70 | 71 | mlab is a repackaging effort to make things up-to-date. 72 | 73 | 74 | .. _Matlab®: 75 | http://www.mathworks.com 76 | 77 | Related 78 | ------- 79 | 80 | Thereis is a copy of mlabwrap v1.1-pre (http://mlabwrap.sourceforge.net/) patched 81 | as described here: 82 | http://sourceforge.net/mailarchive/message.php?msg_id=27312822 83 | 84 | with a patch fixing the error:: 85 | 86 | mlabraw.cpp:225: *error*: invalid conversion from ‘const mwSize*’ to ‘const int*’ 87 | 88 | Also note that in Ubuntu you need to ``sudo apt-get install csh`` 89 | 90 | For details see 91 | http://github.com/aweinstein/mlabwrap 92 | 93 | News 94 | ---- 95 | 96 | **2014-08-26** 1.1.3 Applying patch to add support for Windows via COM. 97 | Credits to Sergey Matyunin, Amro@stackoverflow 98 | 99 | **2013-07-26** 1.1.1 Repacking a library as mlab project. Including code 100 | for Windows (matlabraw.cpp is off for now). 101 | 102 | **2009-10-26** 1.1 fixes an incorrect declaration in ``mlabraw.cpp`` 103 | that caused compilation problems for some users and incorporates a 104 | ``setup.py`` fix for windows suggested by Alan Brooks. More significantly 105 | there is a new spiffy logo! 106 | 107 | **2009-09-14** 1.1-pre finally brings N-D array support, thanks to Vivek 108 | Rathod who joined the project! Also fixed a missing import for saveVarsInMat 109 | (thanks to Nicolas Pinto). 110 | 111 | Since a few people have run into problems that appear to relate to compiling 112 | Matlab® C-extensions in general and aren't mlabwrap-specific, I should probably 113 | stress that in case of any problems that look C-related, verifying whether 114 | engdemo.c works is a great litmus test (see Troubleshooting_ ). 115 | 116 | 117 | **2009-03-23** 1.0.1 is finally out. This is a minor release that fixes some 118 | annoying but mostly minor bugs in mlabwrap (it also slightly improves the 119 | indexing support for proxy-objects, but the exact semantics are still subject 120 | to change.) 121 | 122 | - installation is now easier, in particularly ``LD_LIBRARY_PATH`` no longer 123 | needs to be set and some quoting issues with the matlab call during 124 | installation have been addressed. 125 | 126 | - sparse Matlab® matrices are now handled correctly 127 | (``mlab.sparse([0,0,0,0])`` will now return a proxy for a sparse double 128 | matrix, rather than incorrectly treat at as plain double array and return 129 | junk or crash). 130 | 131 | - replaced the (optional) use of the outdated netcdf package for the 132 | unit-tests with homegrown matlab helper class. 133 | 134 | - several bugs squashed (length of mlabraw.eval'ed strings is checked, better 135 | error-messages etc.) and some small documentation improvements and quite a 136 | few code clean-ups. 137 | 138 | Many thanks to Iain Murray at Toronto and Nicolas Pinto at MIT for letting 139 | themselves be roped into helping me test my stupidly broken release 140 | candidates. 141 | 142 | License 143 | ------- 144 | 145 | mlab (and mlabwrap) is under MIT license, see LICENSE.txt. mlabraw is under a BSD-style 146 | license, see the mlabraw.cpp. 147 | 148 | Download 149 | -------- 150 | 151 | 152 | Installation 153 | ------------ 154 | 155 | mlab should work with python>=2.7 (downto python 2.2, with minor coaxing) and 156 | either numpy_ (recommended) or Numeric (obsolete) installed and Matlab 6, 6.5, 157 | 7.x and 8.x under Linux, OS X® and Windows® (see `OS X`_) on 32- or 64-bit 158 | machines. 159 | 160 | Linux 161 | ''''' 162 | If you're lucky (linux, Matlab binary in ``PATH``):: 163 | 164 | python setup.py install 165 | 166 | (As usual, if you want to install just in your homedir add ``--prefix=$HOME``; 167 | and make sure your ``PYTHONPATH`` is set accordingly.) 168 | 169 | If things do go awry, see Troubleshooting_. 170 | 171 | Windows 172 | ''''''' 173 | 174 | Assuming you have python 2.7.5 (e.g. C:\Python27) and setuptools 175 | ("easy_install.exe") installed and on your PATH. 176 | 177 | 1) Download and install numpy package. You can use packages provided by 178 | Christoph Gohlke: http://www.lfd.uci.edu/~gohlke/pythonlibs/ Also see official 179 | SciPy website for latest status, it might that:: 180 | 181 | easy_install.exe numpy 182 | 183 | would do the trick. 184 | 185 | 186 | 2) You would also need The PyWin32 module by Mark Hammond:: 187 | 188 | easy_install.exe pywin32 189 | 190 | also see Windows in `Troubleshooting`_. 191 | 192 | Documentation 193 | ------------- 194 | - for lazy people 195 | 196 | >>> from mlab.releases import latest_release as matlab 197 | >>> matlab.plot([1,2,3],'-o') 198 | 199 | .. image:: ugly-plot.png 200 | :alt: ugly-plot 201 | 202 | - a slightly prettier example 203 | 204 | >>> from mlab.releases import latest_release as matlab 205 | >>> from numpy import * 206 | >>> xx = arange(-2*pi, 2*pi, 0.2) 207 | >>> matlab.surf(subtract.outer(sin(xx),cos(xx))) 208 | 209 | .. image:: surface-plot.png 210 | :alt: surface-plot 211 | 212 | - for a complete description: 213 | Just run ``pydoc mlab``. 214 | 215 | - for people who like tutorials: 216 | see below 217 | 218 | 219 | Tutorial 220 | '''''''' 221 | 222 | [This is adapted from an email I wrote someone who asked me about mlabwrap. 223 | **Compatibility Note:** Since matlab is becoming increasingly less 224 | ``double``-centric, the default conversion rules might change in post 1.0 225 | mlabwrap; so whilst using ``mlab.plot([1,2,3])`` rather than 226 | ``mlab.plot(array([1.,2.,3.]))`` is fine for interactive use as in the 227 | tutorial below, the latter is recommended for production code.] 228 | 229 | Legend: [...] = omitted output 230 | 231 | Let's say you want to do use Matlab® to calculate the singular value 232 | decomposition of a matrix. So first you import the ``mlab`` pseudo-module and 233 | numpy: 234 | 235 | 236 | >>> from mlab.releases import latest_release as matlab 237 | >>> import numpy 238 | 239 | Now you want to find out what the right function is, so you simply do: 240 | 241 | >>> matlab.lookfor('singular value') 242 | GSVD Generalized Singular Value Decompostion. 243 | SVD Singular value decomposition. 244 | [...] 245 | 246 | Then you look up what ``svd`` actually does, just as you'd look up the 247 | docstring of a python function: 248 | 249 | >>> help(matlab.svd) 250 | mlab_command(*args, **kwargs) 251 | SVD Singular value decomposition. 252 | [U,S,V] = SVD(X) produces a diagonal matrix S, of the same 253 | dimension as X and with nonnegative diagonal elements in 254 | [...] 255 | 256 | Then you try it out: 257 | 258 | >>> matlab.svd(array([[1,2], [1,3]])) 259 | array([[ 3.86432845], 260 | [ 0.25877718]]) 261 | 262 | Notice that we only got 'U' back -- that's because python hasn't got something 263 | like Matlab's multiple value return. Since Matlab functions can have 264 | completely different behavior depending on how many output parameters are 265 | requested, you have to specify explicitly if you want more than 1. So to get 266 | 'U' and also 'S' and 'V' you'd do: 267 | 268 | >>> U, S, V = matlab.svd([[1,2],[1,3]], nout=3) 269 | 270 | The only other possible catch is that Matlab (to a good approximation) 271 | basically represents everything as a double matrix. So there are no 272 | scalars, or 'flat' vectors. They correspond to 1x1 and 1xN matrices 273 | respectively. So, when you pass a flat vector or a scalar to a 274 | mlab-function, it is autoconverted. Also, integer values are automatically 275 | converted to double floats. Here is an example: 276 | 277 | >>> matlab.abs(-1) 278 | array([ [ 1.]]) 279 | 280 | Strings also work as expected: 281 | 282 | >>> matlab.upper('abcde') 283 | 'ABCDE' 284 | 285 | However, although matrices and strings should cover most needs and can be 286 | directly converted, Matlab functions can also return structs or indeed 287 | classes and other types that cannot be converted into python 288 | equivalents. However, rather than just giving up, mlabwrap just hides 289 | this fact from the user by using proxies: 290 | E.g. to create a netlab_ neural net with 2 input, 3 hidden and 1 output node: 291 | 292 | >>> net = matlab.mlp(2,3,1,'logistic') 293 | 294 | Looking at ``net`` reveals that is a proxy: 295 | 296 | >>> net 297 | 299 | type: 'mlp' 300 | nin: 3 301 | nhidden: 3 302 | nout: 3 303 | nwts: 24 304 | outfn: 'linear' 305 | w1: [3x3 double] 306 | b1: [0.0873 -0.0934 0.3629] 307 | w2: [3x3 double] 308 | b2: [-0.6681 0.3572 0.8118] 309 | 310 | When ``net`` or other proxy objects a passed to mlab functions, they are 311 | automatically converted into the corresponding Matlab-objects. So to obtain 312 | a trained network on the 'xor'-problem, one can simply do: 313 | 314 | >>> net = matlab.mlptrain(net, [[1,1], [0,0], [1,0], [0,1]], [0,0,1,1], 1000) 315 | 316 | And test with: 317 | 318 | >>> matlab.mlpfwd(net2, [[1,0]]) 319 | array([ [ 1.]]) 320 | >>> matlab.mlpfwd(net2, [[1,1]]) 321 | array([ [ 7.53175454e-09]]) 322 | 323 | As previously mentioned, normally you shouldn't notice at all when you are 324 | working with proxy objects; they can even be pickled (!), although that is 325 | still somewhat experimental. 326 | 327 | mlabwrap also offers proper error handling and exceptions! So trying to 328 | pass only one input to a net with 2 input nodes raises an Exception: 329 | 330 | 331 | >>> matlab.mlpfwd(net2, 1) 332 | Traceback (most recent call last): 333 | [...] 334 | mlabraw.error: Error using ==> mlpfwd 335 | Dimension of inputs 1 does not match number of model inputs 2 336 | 337 | 338 | Warning messages (and messages to stdout) are also displayed: 339 | 340 | >>> matlab.log(0.) 341 | Warning: Log of zero. 342 | array([ [ -inf]]) 343 | 344 | 345 | Comparison to other existing modules 346 | '''''''''''''''''''''''''''''''''''' 347 | 348 | To get a vague impression just *how* high-level all this, consider attempting to 349 | do something similar to the first example with pymat (upon which the 350 | underlying mlabraw interface to Matlab® is based). 351 | 352 | this: 353 | 354 | >>> A, B, C = matlab.svd([[1,2],[1,3]], 0, nout=3) 355 | 356 | becomes this: 357 | 358 | >>> session = pymat.open() 359 | >>> pymat.put(session, "X", [[1,2], [1,3]]) 360 | >>> pymat.put(session, "cheap", 0) 361 | >>> pymat.eval(session, '[A, B, C] = svd(X, cheap)') 362 | >>> A = pymat.get(session, 'A') 363 | >>> B = pymat.get(session, 'B') 364 | >>> C = pymat.get(session, 'C') 365 | 366 | Plus, there is virtually no error-reporting at all, if something goes wrong in 367 | the ``eval`` step, you'll only notice because the subsequent ``get`` mysteriously 368 | fails. And of course something more fancy like the netlab example above (which 369 | uses proxies to represent matlab class instances in python) would be 370 | impossible to accomplish in pymat in a similar manner. 371 | 372 | However *should* you need low-level access, then that is equally available 373 | (and *with* error reporting); basically just replace ``pymat`` with 374 | ``mlabraw`` above and use ``mlab._session`` as session), i.e 375 | 376 | >>> from mlab.releases import latest_release as matlab 377 | >>> from mlab import mlabraw 378 | >>> mlabraw.put(matlab._session, "X", [[1,2], [1,3]]) 379 | [...] 380 | 381 | Before you resort to this you should ask yourself if it's really a good idea; 382 | the inherent overhead associated with Matlab's C interface appears to be quite 383 | high, so the additional python overhead shouldn't normally matter much -- if 384 | efficiency becomes an issue it's probably better to try to chunk together 385 | several matlab commands in an ``.m``-file in order to reduce the number of 386 | matlab calls. If you're looking for a way to execute "raw" matlab for specific 387 | purposes, ``matlab._do`` is probably a better idea. The low-level ``mlabraw`` 388 | API is much more likely to change in completely backwards incompatible ways in 389 | future versions of mlabwrap. You've been warned. 390 | 391 | What's Missing? 392 | ''''''''''''''' 393 | 394 | - Handling of as arrays of (array) rank 3 or more as well as 395 | non-double/complex arrays (currently everything is converted to 396 | double/complex for passing to Matlab and passing non-double/complex from 397 | Matlab is not not supported). Both should be reasonably easy to implement, 398 | but not that many people have asked for it and I haven't got around to it 399 | yet. 400 | 401 | - Better support for cells. 402 | 403 | - Thread-safety. If you think there's a need please let me know (on the 404 | `StackOverflow tagged query`_); at the moment you can /probably/ get away with 405 | using one seperate MlabWrap object per thread without implementing your own 406 | locking, but even that hasn't been tested. 407 | 408 | 409 | Implementation Notes 410 | '''''''''''''''''''' 411 | 412 | So how does it all work? 413 | 414 | I've got a C extension module (a heavily bug-fixed and somewhat modified 415 | version of pymat, an open-source, low-level python-matlab interface) to take 416 | care of opening Matlab sessions, sending Matlab commands as strings to a 417 | running Matlab session and and converting Numeric arrays (and sequences and 418 | strings...) to Matlab matrices and vice versa. On top of this I then built a 419 | pure python module that with various bells and whistles gives the impression 420 | of providing a Matlab "module". 421 | 422 | This is done by a class that manages a single Matlab session (of which ``matlab`` 423 | is an instance) and creates methods with docstrings on-the-fly. Thus, on the 424 | first call of ``matlab.abs(1)``, the wrapper looks whether there already is a 425 | matching function in the cache. If not, the docstring for ``abs`` is looked up 426 | in Matlab and Matlab's flimsy introspection abilities are used to determine 427 | the number of output arguments (0 or more), then a function with the right 428 | docstring is dynamically created and assigned to ``matlab.abs``. This function 429 | takes care of the conversion of all input parameters and the return values, 430 | using proxies where necessary. Proxy are a bit more involved and the proxy 431 | pickling scheme uses Matlab's ``save`` command to create a binary version of 432 | the proxy's contents which is then pickled, together with the proxy object by 433 | python itself. Hope that gives a vague idea, for more info study the source. 434 | 435 | Troubleshooting 436 | ''''''''''''''' 437 | 438 | Strange hangs under Matlab® R2008a 439 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 440 | 441 | It looks like this particular version of matlab might be broken (I was able to 442 | reproduced the problem with just a stripped down ``engdemo.c`` under 64-bit 443 | linux). R2008b is reported to be working correctly (as are several earlier 444 | versions). 445 | 446 | 447 | matlab not in path 448 | ~~~~~~~~~~~~~~~~~~ 449 | ``setup.py`` will call ``matlab`` in an attempt to query the version and other 450 | information relevant for installation, so it has to be in your ``PATH`` 451 | *unless* you specify everything by hand in ``setup.py``. Of course to be able 452 | to use ``mlabwrap`` in any way ``matlab`` will have to be in your path anyway 453 | (unless that is you set the environment variable ``MLABRAW_CMD_STR`` that 454 | specifies how exactly Matlab® should be called). 455 | 456 | 457 | "Can't open engine" 458 | ~~~~~~~~~~~~~~~~~~~ 459 | If you see something like ``mlabraw.error: Unable to start MATLAB(TM) engine`` 460 | then you may be using an incompatible C++ compiler (or version), or if you're 461 | using unix you might not have ``csh`` installed under ``/bin/csh``, see below. 462 | Try if you can get the ``engdemo.c`` file to work that comes with your Matlab 463 | installation -- `engdemo`_ provides detailed instructions, but in a nutshell: 464 | copy it to a directory where you have write access and do 465 | (assuming Matlab is installed in /opt/MatlabR14 and you're running unix, 466 | otherwise modify as requird):: 467 | 468 | mex -f /opt/MatlabR14/bin/engopts.sh engdemo.c 469 | ./engdemo 470 | 471 | if you get ``Can't start MATLAB engine`` chances are you're trying to use a 472 | compiler version that's not in Mathworks's `list of compatible compilers`_ or 473 | something else with your compiler/Matlab installation is broken that needs to 474 | be resolved before you can successfully build mlabwrap. Chances are that you 475 | or you institution pays a lot of money to the Mathworks, so they should be 476 | happy to give you some tech support. Here's what some user who recently 477 | (2007-02-04) got Matlab 7.04's mex support to work under Ubuntu Edgy after an 478 | exchange with support reported back; apart from installing gcc-3.2.3, he did 479 | the following:: 480 | 481 | The code I'd run (from within Matlab) is... 482 | > mex -setup; # then select: 2 - gcc Mex options 483 | > optsfile = [matlabroot '/bin/engopts.sh']; 484 | > mex -v -f optsfile 'engdemo.c'; 485 | > !./engdemo; 486 | 487 | **Update** John Bender reports that under unix csh needs to be installed in 488 | ``/bin/csh`` for the matlab external engine to work -- since many linux 489 | distros don't install csh by default, you might have to do something like 490 | ``sudo apt-get install csh`` (e.g. under ubuntu or other debian-based 491 | systems). He also pointed out this helpful `engdemo troubleshooting`_ page at 492 | the Mathworks(tm) site. 493 | 494 | 495 | "\`GLIBCXX_3.4.9' not found" on importing mlab (or similar) 496 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 497 | As above, first try to see if you can get engdemo.c to work, because 498 | as long as even the examples that come with Matlab® don't compile, 499 | chances of mlabwrap compiling are rather slim. On the plus-side 500 | if the problem isn't mlabwrap specific, The Mathworks® and/or 501 | Matlab®-specific support forums should be able to help. 502 | 503 | Old Matlab version 504 | ~~~~~~~~~~~~~~~~~~ 505 | If you get something like this on ``python setup.py install``:: 506 | 507 | mlabraw.cpp:634: `engGetVariable' undeclared (first use this function) 508 | 509 | Then you're presumably using an old version of Matlab (i.e. < 6.5); 510 | ``setup.py`` ought to have detected this though (try adjusting 511 | ``MATLAB_VERSION`` by hand and write me a bug report). 512 | 513 | 514 | OS X 515 | ~~~~ 516 | 517 | Josh Marshall tried it under OS X and sent me the following notes (thanks!). 518 | 519 | Notes on running 520 | ................ 521 | 522 | - Before running python, run:: 523 | 524 | export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH$:/Applications/MATLAB701/bin/mac/ 525 | export MLABRAW_CMD_STR=/Applications/MATLAB701/bin/matlab 526 | 527 | [Edit: I'm not sure ``DYLD_LIBRARY_PATH`` modification is still necessary.] 528 | 529 | - As far as graphics commands go, the python interpreter will need to be run 530 | from within the X11 xterm to be able to display anything to the screen. 531 | ie, the command for lazy people 532 | 533 | >>> from mlabwrap import mlab; mlab.plot([1,2,3],'-o') 534 | 535 | won't work unless python is run from an xterm, and the matlab startup 536 | string is 537 | changed to:: 538 | 539 | export MLABRAW_CMD_STR="/Applications/MATLAB701/bin/matlab -nodesktop" 540 | 541 | Windows 542 | ~~~~~~~ 543 | 544 | I'm thankfully not using windows myself, but I try to keep mlabwrap working 545 | under windows, for which I depend on the feedback from windows users. 546 | 547 | Since there are several popular C++ compilers under windows, you might have to 548 | tell setup.py which one you'd like to use (unless it's VC 7). 549 | 550 | George A. Blaha sent me a patch for Borland C++ support; search for "Borland 551 | C++" in setup.py and follow the instructions. 552 | 553 | Dylan T Walker writes mingw32 will also work fine, but for some reason 554 | (distuils glitch?) the following invocation is required:: 555 | 556 | > setup.py build --compiler=mingw32 557 | > setup.py install --skip-build 558 | 559 | 560 | Function Handles and callbacks into python 561 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 562 | 563 | People sometimes try to pass a python function to a matlab function (e.g. 564 | ``matlab.fzero(lambda x: x**2-2, 0)``) which will result in an error messages 565 | because callbacks into python are not implemented (I'm not even it would even 566 | be feasible). Whilst there is no general workaround, in some cases you can 567 | just create an equivalent matlab function on the fly, e.g. do something like 568 | this: ``matlab.fzero(matlab.eval('@(x) x^2-2', 0))``. 569 | 570 | Directly manipulating variables in Matlab® space 571 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 572 | 573 | In certain (rare!) certain cases it might be necessary to directly access or 574 | set a global variable in matlab. In these cases you can use ``matlab._get('SOME_VAR')`` 575 | and ``matlab._set('SOME_VAR', somevalue)``. 576 | 577 | 578 | Support and Feedback 579 | -------------------- 580 | 581 | Post your questions directly on Stack overflow with tags ``matlab``, ``mlab`` 582 | and ``python`` 583 | 584 | .. _StackOverflow tagged query: 585 | http://stackoverflow.com/questions/tagged/matlab+mlab+python?sort=newest&pagesize=50 586 | 587 | 588 | 589 | Credits 590 | ------- 591 | 592 | Yauhen Yakimovich is maintaining the current mlab branch 593 | https://github.com/ewiger 594 | 595 | Amro for recent patch to matlab discovery via COM on Windows 596 | https://github.com/amroamroamro 597 | 598 | Alejandro Weinstein for patches of 1.1pre 599 | https://github.com/aweinstein/mlabwrap 600 | 601 | Alexander Schmolck and Vivek Rathod for mlabwrap: 602 | http://mlabwrap.sourceforge.net/ 603 | 604 | Andrew Sterian for writing pymat without which this module would never have 605 | existed. 606 | 607 | Matthew Brett contributed numpy compatibility and nice setup.py improvements 608 | (which I adapted a bit) to further reduce the need for manual user 609 | intervention for installation. 610 | 611 | I'm only using linux myself -- so I gratefully acknowledge the help of Windows 612 | and OS X users to get things running smoothly under these OSes as well; 613 | particularly those who provided patches to setup.py or mlabraw.cpp (Joris van 614 | Zwieten, George A. Blaha and others). 615 | 616 | Matlab is a registered trademark of `The Mathworks`_. 617 | 618 | .. _The Mathworks: 619 | http://www.mathworks.com 620 | 621 | .. _engdemo troubleshooting: 622 | http://www.mathworks.com/access/helpdesk/help/techdoc/index.html?/access/helpdesk/help/techdoc/matlab_external/f39903.html 623 | 624 | .. _numpy: 625 | http://numpy.scipy.org 626 | 627 | .. _netlab: 628 | http://www.ncrg.aston.ac.uk/netlab/ 629 | 630 | .. _list of compatible compilers: 631 | http://www.mathworks.com/support/tech-notes/1600/1601.html 632 | 633 | .. _Email me: eugeny.yakimovitch@gmail.com 634 | 635 | .. _engdemo: 636 | http://www.mathworks.com/support/solutions/en/data/1-1BSZR/?solution=1-1BSZR 637 | 638 | 639 | -------------------------------------------------------------------------------- /src/mlab/mlabwrap.py: -------------------------------------------------------------------------------- 1 | ############################################################################## 2 | ################## mlabwrap: transparently wraps matlab(tm) ################## 3 | ############################################################################## 4 | ## 5 | ## o author: Alexander Schmolck 6 | ## o created: 2002-05-29 21:51:59+00:40 7 | ## o version: see `__version__` 8 | ## o keywords: matlab wrapper 9 | ## o license: MIT 10 | ## o FIXME: 11 | ## - it seems proxies can somehow still 'disappear', maybe in connection 12 | ## with exceptions in the matlab workspace? 13 | ## - add test that defunct proxy-values are culled from matlab workspace 14 | ## (for some reason ipython seems to keep them alive somehwere, even after 15 | ## a zaphist, should find out what causes that!) 16 | ## - add tests for exception handling! 17 | ## - the proxy getitem/setitem only works quite properly for 1D arrays 18 | ## (matlab's moronic syntax also means that 'foo(bar)(watz)' is not the 19 | ## same as 'tmp = foo(bar); tmp(watz)' -- indeed chances are the former 20 | ## will fail (not, apparently however with 'foo{bar}{watz}' (blech)). This 21 | ## would make it quite hard to the proxy thing 'properly' for nested 22 | ## proxies, so things may break down for complicated cases but can be 23 | ## easily fixed manually e.g.: ``mlab._set('tmp', foo(bar)); 24 | ## mlab._get('tmp',remove=True)[watz]`` 25 | ## - Guess there should be some in principle avoidable problems with 26 | ## assignments to sub-proxies (in addition to the more fundamental, 27 | ## unavoidable problem that ``proxy[index].part = foo`` can't work as 28 | ## expected if ``proxy[index]`` is a marshallable value that doesn't need 29 | ## to be proxied itself; see below for workaround). 30 | ## o XXX: 31 | ## - better support of string 'arrays' 32 | ## - multi-dimensional arrays are unsupported 33 | ## - treatment of lists, tuples and arrays with non-numerical values (these 34 | ## should presumably be wrapped into wrapper classes MlabCell etc.) 35 | ## - should test classes and further improve struct support? 36 | ## - should we transform 1D vectors into row vectors when handing them to 37 | ## matlab? 38 | ## - what should be flattend? Should there be a scalarization opition? 39 | ## - ``autosync_dirs`` is a bit of a hack (and maybe``handle_out``, too)... 40 | ## - is ``global mlab`` in unpickling of proxies OK? 41 | ## - hasattr fun for proxies (__deepcopy__ etc.) 42 | ## - check pickling 43 | ## o TODO: 44 | ## - delattr 45 | ## - better error reporting: test for number of input args etc. 46 | ## - add cloning of proxies. 47 | ## - pickling for nested proxies (and session management for pickling) 48 | ## - more tests 49 | ## o !!!: 50 | ## - matlab complex arrays are intelligently of type 'double' 51 | ## - ``class('func')`` but ``class(var)`` 52 | 53 | """ 54 | mlabwrap 55 | ======== 56 | 57 | This module implements a powerful and simple to use wrapper that makes using 58 | matlab(tm) from python almost completely transparent. To use simply do: 59 | 60 | >>> from mlabwrap import mlab 61 | 62 | and then just use whatever matlab command you like as follows: 63 | 64 | >>> mlab.plot(range(10), 'ro:') 65 | 66 | You can do more than just plotting: 67 | 68 | >>> mlab.sort([3,1,2]) 69 | array([[ 1., 2., 3.]]) 70 | 71 | N.B.: The result here is a 1x3 matrix (and not a flat lenght 3 array) of type 72 | double (and not int), as matlab built around matrices of type double (see 73 | ``MlabWrap._flatten_row_vecs``). 74 | 75 | Matlab(tm)ab, unlike python has multiple value returns. To emulate calls like 76 | ``[a,b] = sort([3,2,1])`` just do: 77 | 78 | >>> mlab.sort([3,1,2], nout=2) 79 | (array([[ 1., 2., 3.]]), array([[ 2., 3., 1.]])) 80 | 81 | For names that are reserved in python (like print) do: 82 | 83 | >>> mlab.print_() 84 | 85 | You can look at the documentation of a matlab function just by using help, 86 | as usual: 87 | 88 | >>> help(mlab.sort) 89 | 90 | In almost all cases that should be enough -- if you need to do trickier 91 | things, then get raw with ``mlab._do``, or build your child class that 92 | handles what you want. 93 | 94 | 95 | Fine points and limitations 96 | --------------------------- 97 | 98 | - Only 2D matrices are directly supported as return values of matlab 99 | functions (arbitrary matlab classes are supported via proxy objects -- 100 | in most cases this shouldn't make much of a difference (as these proxy 101 | objects can be even pickled) -- still this functionality is yet 102 | experimental). 103 | 104 | One potential pitfall with structs (which are currently proxied) is that 105 | setting indices of subarrays ``struct.part[index] = value`` might seem 106 | to have no effect (since ``part`` can be directly represented as a 107 | python array which will be modified without an effect on the proxy 108 | ``struct``'s contents); in that case:: 109 | 110 | some_array[index] = value; struct.part = some_array`` 111 | 112 | will have the desired effect. 113 | 114 | - Matlab doesn't know scalars, or 1D arrays. Consequently all functions 115 | that one might expect to return a scalar or 1D array will return a 1x1 116 | array instead. Also, because matlab(tm) is built around the 'double' 117 | matrix type (which also includes complex matrices), single floats and 118 | integer types will be cast to double. Note that row and column vectors 119 | can be autoconverted automatically to 1D arrays if that is desired (see 120 | ``_flatten_row_vecs``). 121 | 122 | - for matlab(tm) function names like ``print`` that are reserved words in 123 | python, so you have to add a trailing underscore (e.g. ``mlab.print_``). 124 | 125 | - sometimes you will have to specify the number of return arguments of a 126 | function, e.g. ``a,b,c = mlab.foo(nout=3)``. MlabWrap will normally try to 127 | figure out for you whether the function you call returns 0 or more values 128 | (in which case by default only the first value is returned!). For builtins 129 | this might fail (since unfortunately there seems to be no foolproof way to 130 | find out in matlab), but you can always lend a helping hand:: 131 | 132 | mlab.foo = mlab._make_mlab_command('foo', nout=3, doc=mlab.help('foo')) 133 | 134 | Now ``mlab.foo()`` will by default always return 3 values, but you can still 135 | get only one by doing ``mlab.foo(nout=1)`` 136 | 137 | - by default the working directory of matlab(tm) is kept in synch with that of 138 | python to avoid unpleasant surprises. In case this behavior does instaed 139 | cause you unpleasant surprises, you can turn it off with:: 140 | 141 | mlab._autosync_dirs = False 142 | 143 | - you can customize how matlab is called by setting the environment variable 144 | ``MLABRAW_CMD_STR`` (e.g. to add useful opitons like '-nojvm'). For the 145 | rather convoluted semantics see 146 | . 147 | 148 | - if you don't want to use numpy arrays, but something else that's fine 149 | too:: 150 | 151 | >>> import matrix from numpy.core.defmatrix 152 | >>> mlab._array_cast = matrix 153 | >>> mlab.sqrt([[4.], [1.], [0.]]) 154 | matrix([[ 2.], 155 | [ 1.], 156 | [ 0.]]) 157 | 158 | Credits 159 | ------- 160 | 161 | This is really a wrapper around a wrapper (mlabraw) which in turn is a 162 | modified and bugfixed version of Andrew Sterian's pymat 163 | (http://claymore.engineer.gvsu.edu/~steriana/Python/pymat.html), so thanks go 164 | to him for releasing his package as open source. 165 | 166 | 167 | See the docu of ``MlabWrap`` and ``MatlabObjectProxy`` for more information. 168 | """ 169 | 170 | __docformat__ = "restructuredtext en" 171 | __version__ = '1.1' 172 | __author__ = "Alexander Schmolck " 173 | import warnings 174 | from pickle import PickleError 175 | import operator 176 | import os, sys, re 177 | import weakref 178 | import atexit 179 | try: 180 | import numpy 181 | ndarray = numpy.ndarray 182 | except ImportError: 183 | import Numeric 184 | ndarray = Numeric.ArrayType 185 | 186 | 187 | from tempfile import gettempdir 188 | import mlabraw 189 | 190 | from awmstools import update, gensym, slurp, spitOut, isString, escape, strToTempfile, __saveVarsHelper 191 | 192 | #XXX: nested access 193 | def _flush_write_stdout(s): 194 | """Writes `s` to stdout and flushes. Default value for ``handle_out``.""" 195 | sys.stdout.write(s); sys.stdout.flush() 196 | 197 | # XXX I changed this to no longer use weakrefs because it didn't seem 100% 198 | # reliable on second thought; need to check if we need to do something to 199 | # speed up proxy reclamation on the matlab side. 200 | class CurlyIndexer(object): 201 | """A helper class to mimick ``foo{bar}``-style indexing in python.""" 202 | def __init__(self, proxy): 203 | self.proxy = proxy 204 | def __getitem__(self, index): 205 | return self.proxy.__getitem__(index, '{}') 206 | def __setitem__(self, index, value): 207 | self.proxy.__setitem__(index, value, '{}') 208 | 209 | class MlabObjectProxy(object): 210 | """A proxy class for matlab objects that can't be converted to python 211 | types. 212 | 213 | WARNING: There are impedance-mismatch issues between python and matlab 214 | that make designing such a class difficult (e.g. dimensionality, indexing 215 | and ``length`` work fundamentally different in matlab than in python), so 216 | although this class currently tries to transparently support some stuff 217 | (notably (1D) indexing, slicing and attribute access), other operations 218 | (e.g. math operators and in particular __len__ and __iter__) are not yet 219 | supported. Don't depend on the indexing semantics not to change. 220 | 221 | Note: 222 | 223 | Assigning to parts of proxy objects (e.g. ``proxy[index].part = 224 | [[1,2,3]]``) should *largely* work as expected, the only exception 225 | would be if ``proxy.foo[index] = 3`` where ``proxy.foo[index]`` is some 226 | type that can be converted to python (i.e. an array or string, (or 227 | cell, if cell conversion has been enabled)), because then ``proxy.foo`` 228 | returns a new python object. For these cases it's necessary to do:: 229 | 230 | some_array[index] = 3; proxy.foo = some_array 231 | 232 | 233 | """ 234 | def __init__(self, mlabwrap, name, parent=None): 235 | self.__dict__['_mlabwrap'] = mlabwrap 236 | self.__dict__['_name'] = name 237 | """The name is the name of the proxies representation in matlab.""" 238 | self.__dict__['_parent'] = parent 239 | """To fake matlab's ``obj{foo}`` style indexing.""" 240 | def __getstate__(self): 241 | "Experimental pickling support." 242 | if self.__dict__['_parent']: 243 | raise PickleError( 244 | "Only root instances of %s can currently be pickled." % \ 245 | type(self).__name__) 246 | tmp_filename = os.path.join(gettempdir(), "mlab_pickle_%s.mat" % self._mlabwrap._session) 247 | try: 248 | mlab.save(tmp_filename, self._name) 249 | mlab_contents = slurp(tmp_filename, binary=1) 250 | finally: 251 | if os.path.exists(tmp_filename): os.remove(tmp_filename) 252 | 253 | return {'mlab_contents' : mlab_contents, 254 | 'name': self._name} 255 | 256 | 257 | def __setstate__(self, state): 258 | "Experimental unpickling support." 259 | global mlab #XXX this should be dealt with correctly 260 | old_name = state['name'] 261 | mlab_name = "UNPICKLED%s__" % gensym('') 262 | tmp_filename = None 263 | try: 264 | tmp_filename = strToTempfile( 265 | state['mlab_contents'], suffix='.mat', binary=1) 266 | mlabraw.eval(mlab._session, 267 | "TMP_UNPICKLE_STRUCT__ = load('%s', '%s');" % ( 268 | tmp_filename, old_name)) 269 | mlabraw.eval(mlab._session, 270 | "%s = TMP_UNPICKLE_STRUCT__.%s;" % (mlab_name, old_name)) 271 | mlabraw.eval(mlab._session, "clear TMP_UNPICKLE_STRUCT__;") 272 | # XXX 273 | mlab._make_proxy(mlab_name, constructor=lambda *args: self.__init__(*args) or self) 274 | mlabraw.eval(mlab._session, 'clear %s;' % mlab_name) 275 | finally: 276 | if tmp_filename and os.path.exists(tmp_filename): 277 | os.remove(tmp_filename) 278 | # FIXME clear'ing in case of error 279 | 280 | def __repr__(self): 281 | output = [] 282 | self._mlabwrap._do('disp(%s)' % self._name, nout=0, handle_out=output.append) 283 | rep = "".join(output) 284 | klass = self._mlabwrap._do("class(%s)" % self._name) 285 | ## #XXX what about classes? 286 | ## if klass == "struct": 287 | ## rep = "\n" + self._mlabwrap._format_struct(self._name) 288 | ## else: 289 | ## rep = "" 290 | return "<%s of matlab-class: %r; internal name: %r; has parent: %s>\n%s" % ( 291 | type(self).__name__, klass, 292 | self._name, ['yes', 'no'][self._parent is None], 293 | rep) 294 | def __del__(self): 295 | if self._parent is None: 296 | mlabraw.eval(self._mlabwrap._session, 'clear %s;' % self._name) 297 | def _get_part(self, to_get): 298 | if self._mlabwrap._var_type(to_get) in self._mlabwrap._mlabraw_can_convert: 299 | #!!! need assignment to TMP_VAL__ because `mlabraw.get` only works 300 | # with 'atomic' values like ``foo`` and not e.g. ``foo.bar``. 301 | mlabraw.eval(self._mlabwrap._session, "TMP_VAL__=%s" % to_get) 302 | return self._mlabwrap._get('TMP_VAL__', remove=True) 303 | return type(self)(self._mlabwrap, to_get, self) 304 | def _set_part(self, to_set, value): 305 | #FIXME s.a. 306 | if isinstance(value, MlabObjectProxy): 307 | mlabraw.eval(self._mlabwrap._session, "%s = %s;" % (to_set, value._name)) 308 | else: 309 | self._mlabwrap._set("TMP_VAL__", value) 310 | mlabraw.eval(self._mlabwrap._session, "%s = TMP_VAL__;" % to_set) 311 | mlabraw.eval(self._mlabwrap._session, 'clear TMP_VAL__;') 312 | 313 | def __getattr__(self, attr): 314 | if attr == "_": 315 | return self.__dict__.setdefault('_', CurlyIndexer(self)) 316 | else: 317 | return self._get_part("%s.%s" % (self._name, attr)) 318 | def __setattr__(self, attr, value): 319 | self._set_part("%s.%s" % (self._name, attr), value) 320 | # FIXME still have to think properly about how to best translate Matlab semantics here... 321 | def __nonzero__(self): 322 | raise TypeError("%s does not yet implement truth testing" % type(self).__name__) 323 | def __len__(self): 324 | raise TypeError("%s does not yet implement __len__" % type(self).__name__) 325 | def __iter__(self): 326 | raise TypeError("%s does not yet implement iteration" % type(self).__name__) 327 | def _matlab_str_repr(s): 328 | if '\n' not in s: 329 | return "'%s'" % s.replace("'","''") 330 | else: 331 | # Matlab's string literals suck. They can't represent all 332 | # strings, so we need to use sprintf 333 | return "sprintf('%s')" % escape(s).replace("'","''").replace("%", "%%") 334 | _matlab_str_repr = staticmethod(_matlab_str_repr) 335 | #FIXME: those two only work ok for 1D indexing 336 | def _convert_index(self, index): 337 | if isinstance(index, int): 338 | return str(index + 1) # -> matlab 1-based indexing 339 | elif isString(index): 340 | return self._matlab_str_repr(index) 341 | elif isinstance(index, slice): 342 | if index == slice(None,None,None): 343 | return ":" 344 | elif index.step not in (None,1): 345 | raise ValueError("Illegal index for a proxy %r" % index) 346 | else: 347 | start = (index.start or 0) + 1 348 | if start == 0: start_s = 'end' 349 | elif start < 0: start_s = 'end%d' % start 350 | else: start_s = '%d' % start 351 | 352 | if index.stop is None: stop_s = 'end' 353 | elif index.stop < 0: stop_s = 'end%d' % index.stop 354 | else: stop_s = '%d' % index.stop 355 | 356 | return '%s:%s' % (start_s, stop_s) 357 | else: 358 | raise TypeError("Unsupported index type: %r." % type(index)) 359 | def __getitem__(self, index, parens='()'): 360 | """WARNING: Semi-finished, semantics might change because it's not yet 361 | clear how to best bridge the matlab/python impedence match. 362 | HACK: Matlab decadently allows overloading *2* different indexing parens, 363 | ``()`` and ``{}``, hence the ``parens`` option.""" 364 | index = self._convert_index(index) 365 | return self._get_part("".join([self._name,parens[0],index,parens[1]])) 366 | def __setitem__(self, index, value, parens='()'): 367 | """WARNING: see ``__getitem__``.""" 368 | index = self._convert_index(index) 369 | return self._set_part("".join([self._name,parens[0],index,parens[1]]), 370 | value) 371 | 372 | class MlabConversionError(Exception): 373 | """Raised when a mlab type can't be converted to a python primitive.""" 374 | pass 375 | 376 | class MlabWrap(object): 377 | """This class does most of the wrapping work. It manages a single matlab 378 | session (you can in principle have multiple open sessions if you want, 379 | but I can see little use for this, so this feature is largely untested) 380 | and automatically translates all attribute requests (that don't start 381 | with '_') to the appropriate matlab function calls. The details of this 382 | handling can be controlled with a number of instance variables, 383 | documented below.""" 384 | __all__ = [] #XXX a hack, so that this class can fake a module; don't mutate 385 | def __init__(self): 386 | """Create a new matlab(tm) wrapper object. 387 | """ 388 | self._array_cast = None 389 | """specifies a cast for arrays. If the result of an 390 | operation is a numpy array, ``return_type(res)`` will be returned 391 | instead.""" 392 | self._autosync_dirs=True 393 | """`autosync_dirs` specifies whether the working directory of the 394 | matlab session should be kept in sync with that of python.""" 395 | self._flatten_row_vecs = False 396 | """Automatically return 1xn matrices as flat numeric arrays.""" 397 | self._flatten_col_vecs = False 398 | """Automatically return nx1 matrices as flat numeric arrays.""" 399 | self._clear_call_args = True 400 | """Remove the function args from matlab workspace after each function 401 | call. Otherwise they are left to be (partly) overwritten by the next 402 | function call. This saves a function call in matlab but means that the 403 | memory used up by the arguments will remain unreclaimed till 404 | overwritten.""" 405 | self._session = mlabraw.open() 406 | atexit.register(lambda handle=self._session: mlabraw.close(handle)) 407 | self._proxies = weakref.WeakValueDictionary() 408 | """Use ``mlab._proxies.values()`` for a list of matlab object's that 409 | are currently proxied.""" 410 | self._proxy_count = 0 411 | self._mlabraw_can_convert = ('double', 'char') 412 | """The matlab(tm) types that mlabraw will automatically convert for us.""" 413 | self._dont_proxy = {'cell' : False} 414 | """The matlab(tm) types we can handle ourselves with a bit of 415 | effort. To turn on autoconversion for e.g. cell arrays do: 416 | ``mlab._dont_proxy["cell"] = True``.""" 417 | def __del__(self): 418 | mlabraw.close(self._session) 419 | def _format_struct(self, varname): 420 | res = [] 421 | fieldnames = self._do("fieldnames(%s)" % varname) 422 | size = numpy.ravel(self._do("size(%s)" % varname)) 423 | return "%dx%d struct array with fields:\n%s" % ( 424 | size[0], size[1], "\n ".join([""] + fieldnames)) 425 | ## fieldnames 426 | ## fieldvalues = self._do(",".join(["%s.%s" % (varname, fn) 427 | ## for fn in fieldnames]), nout=len(fieldnames)) 428 | ## maxlen = max(map(len, fieldnames)) 429 | ## return "\n".join(["%*s: %s" % (maxlen, (`fv`,`fv`[:20] + '...')[len(`fv`) > 23]) 430 | ## for fv in fieldvalues]) 431 | 432 | def _var_type(self, varname): 433 | mlabraw.eval(self._session, 434 | "TMP_CLS__ = class(%(x)s); if issparse(%(x)s)," 435 | "TMP_CLS__ = [TMP_CLS__,'-sparse']; end;" % dict(x=varname)) 436 | res_type = mlabraw.get(self._session, "TMP_CLS__") 437 | mlabraw.eval(self._session, "clear TMP_CLS__;") # unlikely to need try/finally to ensure clear 438 | return res_type 439 | 440 | def _make_proxy(self, varname, parent=None, constructor=MlabObjectProxy): 441 | """Creates a proxy for a variable. 442 | 443 | XXX create and cache nested proxies also here. 444 | """ 445 | # FIXME why not just use gensym here? 446 | proxy_val_name = "PROXY_VAL%d__" % self._proxy_count 447 | self._proxy_count += 1 448 | mlabraw.eval(self._session, "%s = %s;" % (proxy_val_name, varname)) 449 | res = constructor(self, proxy_val_name, parent) 450 | self._proxies[proxy_val_name] = res 451 | return res 452 | 453 | def _get_cell(self, varname): 454 | # XXX can currently only handle ``{}`` and 1D cells 455 | mlabraw.eval(self._session, 456 | "TMP_SIZE_INFO__ = \ 457 | [all(size(%(vn)s) == 0), \ 458 | min(size(%(vn)s)) == 1 & ndims(%(vn)s) == 2, \ 459 | max(size(%(vn)s))];" % {'vn':varname}) 460 | is_empty, is_rank1, cell_len = map(int, 461 | self._get("TMP_SIZE_INFO__", remove=True).flat) 462 | if is_empty: 463 | return [] 464 | elif is_rank1: 465 | cell_bits = (["TMP%i%s__" % (i, gensym('_')) 466 | for i in range(cell_len)]) 467 | mlabraw.eval(self._session, '[%s] = deal(%s{:});' % 468 | (",".join(cell_bits), varname)) 469 | # !!! this recursive call means we have to take care with 470 | # overwriting temps!!! 471 | return self._get_values(cell_bits) 472 | else: 473 | raise MlabConversionError("Not a 1D cell array") 474 | def _manually_convert(self, varname, vartype): 475 | if vartype == 'cell': 476 | return self._get_cell(varname) 477 | 478 | 479 | def _get_values(self, varnames): 480 | if not varnames: raise ValueError("No varnames") #to prevent clear('') 481 | res = [] 482 | for varname in varnames: 483 | res.append(self._get(varname)) 484 | mlabraw.eval(self._session, "clear('%s');" % "','".join(varnames)) #FIXME wrap try/finally? 485 | return res 486 | 487 | def _do(self, cmd, *args, **kwargs): 488 | """Semi-raw execution of a matlab command. 489 | 490 | Smartly handle calls to matlab, figure out what to do with `args`, 491 | and when to use function call syntax and not. 492 | 493 | If no `args` are specified, the ``cmd`` not ``result = cmd()`` form is 494 | used in Matlab -- this also makes literal Matlab commands legal 495 | (eg. cmd=``get(gca, 'Children')``). 496 | 497 | If ``nout=0`` is specified, the Matlab command is executed as 498 | procedure, otherwise it is executed as function (default), nout 499 | specifying how many values should be returned (default 1). 500 | 501 | **Beware that if you use don't specify ``nout=0`` for a `cmd` that 502 | never returns a value will raise an error** (because assigning a 503 | variable to a call that doesn't return a value is illegal in matlab). 504 | 505 | 506 | ``cast`` specifies which typecast should be applied to the result 507 | (e.g. `int`), it defaults to none. 508 | 509 | XXX: should we add ``parens`` parameter? 510 | """ 511 | handle_out = kwargs.get('handle_out', _flush_write_stdout) 512 | #self._session = self._session or mlabraw.open() 513 | # HACK 514 | if self._autosync_dirs: 515 | mlabraw.eval(self._session, "cd('%s');" % os.getcwd().replace("'", "''")) 516 | nout = kwargs.get('nout', 1) 517 | #XXX what to do with matlab screen output 518 | argnames = [] 519 | tempargs = [] 520 | try: 521 | for count, arg in enumerate(args): 522 | if isinstance(arg, MlabObjectProxy): 523 | argnames.append(arg._name) 524 | else: 525 | nextName = 'arg%d__' % count 526 | argnames.append(nextName) 527 | tempargs.append(nextName) 528 | # have to convert these by hand 529 | ## try: 530 | ## arg = self._as_mlabable_type(arg) 531 | ## except TypeError: 532 | ## raise TypeError("Illegal argument type (%s.:) for %d. argument" % 533 | ## (type(arg), type(count))) 534 | mlabraw.put(self._session, argnames[-1], arg) 535 | 536 | if args: 537 | cmd = "%s(%s)%s" % (cmd, ", ".join(argnames), 538 | ('',';')[kwargs.get('show',0)]) 539 | # got three cases for nout: 540 | # 0 -> None, 1 -> val, >1 -> [val1, val2, ...] 541 | if nout == 0: 542 | handle_out(mlabraw.eval(self._session, cmd)) 543 | return 544 | # deal with matlab-style multiple value return 545 | resSL = ((["RES%d__" % i for i in range(nout)])) 546 | handle_out(mlabraw.eval(self._session, '[%s]=%s;' % (", ".join(resSL), cmd))) 547 | res = self._get_values(resSL) 548 | 549 | if nout == 1: res = res[0] 550 | else: res = tuple(res) 551 | if kwargs.has_key('cast'): 552 | if nout == 0: raise TypeError("Can't cast: 0 nout") 553 | return kwargs['cast'](res) 554 | else: 555 | return res 556 | finally: 557 | if len(tempargs) and self._clear_call_args: 558 | mlabraw.eval(self._session, "clear('%s');" % 559 | "','".join(tempargs)) 560 | # this is really raw, no conversion of [[]] -> [], whatever 561 | def _get(self, name, remove=False): 562 | r"""Directly access a variable in matlab space. 563 | 564 | This should normally not be used by user code.""" 565 | # FIXME should this really be needed in normal operation? 566 | if name in self._proxies: return self._proxies[name] 567 | varname = name 568 | vartype = self._var_type(varname) 569 | if vartype in self._mlabraw_can_convert: 570 | var = mlabraw.get(self._session, varname) 571 | if isinstance(var, ndarray): 572 | if self._flatten_row_vecs and numpy.shape(var)[0] == 1: 573 | var.shape = var.shape[1:2] 574 | elif self._flatten_col_vecs and numpy.shape(var)[1] == 1: 575 | var.shape = var.shape[0:1] 576 | if self._array_cast: 577 | var = self._array_cast(var) 578 | else: 579 | var = None 580 | if self._dont_proxy.get(vartype): 581 | # manual conversions may fail (e.g. for multidimensional 582 | # cell arrays), in that case just fall back on proxying. 583 | try: 584 | var = self._manually_convert(varname, vartype) 585 | except MlabConversionError: pass 586 | if var is None: 587 | # we can't convert this to a python object, so we just 588 | # create a proxy, and don't delete the real matlab 589 | # reference until the proxy is garbage collected 590 | var = self._make_proxy(varname) 591 | if remove: 592 | mlabraw.eval(self._session, "clear('%s');" % varname) 593 | return var 594 | 595 | def _set(self, name, value): 596 | r"""Directly set a variable `name` in matlab space to `value`. 597 | 598 | This should normally not be used in user code.""" 599 | if isinstance(value, MlabObjectProxy): 600 | mlabraw.eval(self._session, "%s = %s;" % (name, value._name)) 601 | else: 602 | ## mlabraw.put(self._session, name, self._as_mlabable_type(value)) 603 | mlabraw.put(self._session, name, value) 604 | 605 | def _make_mlab_command(self, name, nout, doc=None): 606 | def mlab_command(*args, **kwargs): 607 | return self._do(name, *args, **update({'nout':nout}, kwargs)) 608 | mlab_command.__doc__ = "\n" + doc 609 | return mlab_command 610 | 611 | # XXX this method needs some refactoring, but only after it is clear how 612 | # things should be done (e.g. what should be extracted from docstrings and 613 | # how) 614 | def __getattr__(self, attr): 615 | """Magically creates a wrapper to a matlab function, procedure or 616 | object on-the-fly.""" 617 | if re.search(r'\W', attr): # work around ipython <= 0.7.3 bug 618 | raise ValueError("Attributes don't look like this: %r" % attr) 619 | if attr.startswith('__'): raise AttributeError, attr 620 | assert not attr.startswith('_') # XXX 621 | # print_ -> print 622 | if attr[-1] == "_": 623 | name = attr[:-1] 624 | else: 625 | name = attr 626 | try: 627 | nout = self._do("nargout('%s')" % name) 628 | except mlabraw.error, msg: 629 | typ = numpy.ravel(self._do("exist('%s')" % name))[0] 630 | if typ == 0: # doesn't exist 631 | raise AttributeError("No such matlab object: %s" % name) 632 | else: 633 | warnings.warn( 634 | "Couldn't ascertain number of output args" 635 | "for '%s', assuming 1." % name) 636 | nout = 1 637 | doc = self._do("help('%s')" % name) 638 | # play it safe only return 1st if nout >= 1 639 | # XXX are all ``nout>1``s also useable as ``nout==1``s? 640 | nout = nout and 1 641 | mlab_command = self._make_mlab_command(name, nout, doc) 642 | #!!! attr, *not* name, because we might have python keyword name! 643 | setattr(self, attr, mlab_command) 644 | return mlab_command 645 | 646 | 647 | MlabError = mlabraw.error 648 | from mlabraw import MatlabReleaseNotFound, set_release as choose_release, find_available_releases 649 | 650 | 651 | def saveVarsInMat(filename, varNamesStr, outOf=None, **opts): 652 | """Hacky convinience function to dump a couple of python variables in a 653 | .mat file. See `awmstools.saveVars`. 654 | """ 655 | from mlabwrap import mlab 656 | filename, varnames, outOf = __saveVarsHelper( 657 | filename, varNamesStr, outOf, '.mat', **opts) 658 | try: 659 | for varname in varnames: 660 | mlab._set(varname, outOf[varname]) 661 | mlab._do("save('%s','%s')" % (filename, "', '".join(varnames)), nout=0) 662 | finally: 663 | assert varnames 664 | mlab._do("clear('%s')" % "', '".join(varnames), nout=0) 665 | 666 | 667 | all__ = ['saveVarsInMat', 'MlabWrap', 'MlabError', 668 | 'choose_release', 'find_available_releases', 'MatlabReleaseNotFound'] 669 | 670 | -------------------------------------------------------------------------------- /src/mlab/awmstools.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | ############################################################################# 3 | ################## awmstools : Common functions for python ################### 4 | ############################################################################## 5 | ## 6 | ## o author: Alexander Schmolck (A.Schmolck@gmx.net) 7 | ## o created: 2000-04-08T15:52:17+00:00 8 | ## o last changed: $Date: 2009-03-24 02:09:50 $ 9 | ## o license: see file LICENSE 10 | ## o keywords: python, helper functions 11 | ## o requires: python >= 2.4 12 | ## o TODO: 13 | ## - get rid of the bogus test failure (doctest: +ELLIPSIS) 14 | ## - streamline or cull baroque stuff like reverse 15 | ## - cull more stuff and factor into several files 16 | ## - saveVars etc. should have `silent` option or so 17 | ## - not all functions are tested rigorously yet 18 | ## - write a fast merge (kicked old slow recursive one) 19 | ## 20 | ## Sorted in inverse order of uselessness :) The stuff under EXPERIMENTAL is 21 | ## just that: experimental. Expect it not to work, or to disappear or to be 22 | ## incompatibly modified in the future. The rest should be fairly stable. 23 | 24 | """A collection of various convenience functions and classes, small utilities 25 | and 'fixes'. 26 | 27 | Some just save a little bit of typing (`Result`), others are things that 28 | seem to have been forgotten in the standard libraries (`slurp`, 29 | `binarySearch`, `replaceStrs`) or that have a strange behavior 30 | (`os.path.splitext`). Apart from several general purpose utilities for 31 | lists (`flatten`) iterables in general (`window`, `unique`, `union`, 32 | `group` etc.) there are also more special purpose utilities such as various 33 | handy functions and classes for writing scripts (`DryRun`), for debugging 34 | (`makePrintReturner`) and for meta-programming (`gensym`). 35 | 36 | 37 | """ 38 | from __future__ import division 39 | __docformat__ = "restructuredtext en" 40 | __revision__ = "$Id: awmstools.py,v 1.29 2009-03-24 02:09:50 aschmolc Exp $" 41 | __version__ = "0.9" 42 | __author__ = "Alexander Schmolck " 43 | __test__ = {} # this is for doctest 44 | 45 | import bisect 46 | import codecs 47 | import copy 48 | import cPickle 49 | try: from functools import partial # python < 2.5 compatibility 50 | except ImportError: partial = lambda f,*args,**kwargs: lambda *a,**k: f(args+a,update(kwargs,k)) 51 | from itertools import * 52 | import inspect 53 | import itertools 54 | import math 55 | import operator 56 | import os 57 | import getpass 58 | import re 59 | import sys 60 | import tempfile 61 | import time 62 | import types 63 | import urllib2 64 | try: from threading import Lock 65 | except ImportError: Lock = lambda: Null 66 | 67 | try: any 68 | except NameError: any = lambda xs: bool(some(xs)); all = lambda xs: bool(every(xs)) 69 | 70 | class _uniqueClass(object): 71 | """To create a single instance to be used for default values; supports 72 | comparison by-object identity to work around stupid classes that won't allow 73 | comparisons to non-self-instances.""" 74 | def __eq__(a,b): return a is b; 75 | def __ne__(a,b): return a is not b 76 | try: # don't redefine on reload 77 | __unique 78 | except NameError: 79 | __unique = _uniqueClass() 80 | 81 | 82 | if sys.maxint > 1e6*60*60*24*365*100: # see below 83 | # XXX this relies on the GIL & itertools for threadsafety 84 | # but since itertools.count can only count to sys.maxint... 85 | def Counter(start=0): 86 | """A threadsafe counter that let's you keep counting 87 | for at least 100 years at a rate of 1MHz (if `start`= 0). 88 | """ 89 | return itertools.count(start).next 90 | else: 91 | # ... we also need this more generic version 92 | class Counter(object): 93 | """A threadsafe counter that let's you keep counting 94 | for at least 100 years at a rate of 10^6/s (if `start`= 0). 95 | """ 96 | def __init__(self, start=0): 97 | self.lock = Lock() 98 | self.count = start 99 | def __call__(self): 100 | try: 101 | self.lock.acquire() 102 | return self.count 103 | finally: 104 | self.count+=1 105 | self.lock.release() 106 | 107 | # don't redefine on reload 108 | try: 109 | _count 110 | except NameError: 111 | _count = Counter() 112 | _gensyms = {} 113 | def gensym(prefix="GSYM"): 114 | r"""Returns an string that is valid as a unique python variable name. Useful 115 | when creating functions etc. on the fly. Can be used from multiple threads 116 | and is `reload` safe. 117 | """ 118 | return "%s%d" % (prefix, _count()) 119 | 120 | __test__['gensym'] = r""" 121 | >>> import awmstools 122 | >>> bak = awmstools._count 123 | >>> awmstools._count = Counter() 124 | >>> gensym() 125 | 'GSYM0' 126 | >>> gensym() 127 | 'GSYM1' 128 | >>> gensym('FOO') 129 | 'FOO2' 130 | >>> import awmstools 131 | >>> reload(awmstools) and None 132 | >>> awmstools._count = bak 133 | """ 134 | # FIXME test threadsafety at least superficially! 135 | 136 | 137 | #_. FIXES 138 | # Fixes for things in python I'd like to behave differently 139 | 140 | def rexGroups(rex): 141 | """Return the named groups in a regular expression (compiled or as string) 142 | in occuring order. 143 | 144 | >>> rexGroups(r'(?P\w+) +(?P\w+)') 145 | ('name', 'surname') 146 | 147 | """ 148 | if isinstance(rex,basestring): rex = re.compile(rex) 149 | return zip(*sorted([(n,g) for (g,n) in rex.groupindex.items()]))[1] 150 | 151 | 152 | class IndexMaker(object): 153 | """Convinience class to make slices etc. that can be used for creating 154 | indices (mainly because using `slice` is a PITA). 155 | 156 | Examples: 157 | 158 | >>> range(4)[indexme[::-1]] == range(4)[::-1] == [3, 2, 1, 0] 159 | True 160 | >>> indexme[::-1] 161 | slice(None, None, -1) 162 | >>> indexme[0,:] 163 | (0, slice(None, None, None)) 164 | """ 165 | def __getitem__(self, a): 166 | return a 167 | indexme = IndexMaker() 168 | 169 | 170 | # A shortcut for 'infinite' integer e.g. for slicing: ``seq[4:INFI]`` as 171 | # ``seq[4:len(seq)]`` is messy and only works if `seq` isn't an expression 172 | INFI = sys.maxint 173 | # real infinity 174 | INF = 1e999999 175 | 176 | 177 | class Result(object): 178 | """Circumvent python's lack of assignment expression (mainly useful for 179 | writing while loops): 180 | 181 | >>> import re 182 | >>> s = 'one 2 three 4 five 6' 183 | >>> findNumber = Result(re.compile('\d+').search) 184 | >>> while findNumber(s): 185 | ... match = findNumber.result 186 | ... print 'found', `match.group(0)`, 'at position', match.start() 187 | ... s = s[match.end():] 188 | ... 189 | found '2' at position 4 190 | found '4' at position 7 191 | found '6' at position 6 192 | """ 193 | def __init__(self, func): 194 | self.func = func 195 | def __call__(self,*args,**kwargs): 196 | self.result = self.func(*args,**kwargs) 197 | return self.result 198 | 199 | class NullType(object): 200 | r"""Similar to `NoneType` with a corresponding singleton instance `Null` 201 | that, unlike `None` accepts any message and returns itself. 202 | 203 | Examples: 204 | >>> Null("send", a="message")(*"and one more")[ 205 | ... "even index and"].what.you.get.still is Null 206 | True 207 | >>> not Null 208 | True 209 | >>> Null['something'] 210 | Null 211 | >>> Null.something 212 | Null 213 | >>> Null in Null 214 | False 215 | >>> hasattr(Null, 'something') 216 | True 217 | >>> Null.something = "a value" 218 | >>> Null.something 219 | Null 220 | >>> Null == Null 221 | True 222 | >>> Null == 3 223 | False 224 | """ 225 | 226 | def __new__(cls): return Null 227 | def __call__(self, *args, **kwargs): return Null 228 | ## def __getstate__(self, *args): return Null 229 | def __getinitargs__(self): 230 | print "__getinitargs__" 231 | return ('foobar',) 232 | def __getattr__(self, attr): return Null 233 | def __getitem__(self, item): return Null 234 | def __setattr__(self, attr, value): pass 235 | def __setitem__(self, item, value): pass 236 | def __len__(self): return 0 237 | def __iter__(self): return iter([]) 238 | def __contains__(self, item): return False 239 | def __repr__(self): return "Null" 240 | Null = object.__new__(NullType) 241 | 242 | 243 | def div(a,b): 244 | """``div(a,b)`` is like ``a // b`` if ``b`` devides ``a``, otherwise 245 | an `ValueError` is raised. 246 | 247 | >>> div(10,2) 248 | 5 249 | >>> div(10,3) 250 | Traceback (most recent call last): 251 | ... 252 | ValueError: 3 does not divide 10 253 | """ 254 | res, fail = divmod(a,b) 255 | if fail: 256 | raise ValueError("%r does not divide %r" % (b,a)) 257 | else: 258 | return res 259 | 260 | 261 | def ipshuffle(l, random=None): 262 | r"""Shuffle list `l` inplace and return it.""" 263 | import random as _random 264 | _random.shuffle(l, random) 265 | return l 266 | 267 | __test__['ipshuffle'] = r''' 268 | >>> l = [1,2,3] 269 | >>> ipshuffle(l, lambda :0.3) is l 270 | True 271 | >>> l 272 | [2, 3, 1] 273 | >>> l = [1,2,3] 274 | >>> ipshuffle(l, lambda :0.4) is l 275 | True 276 | >>> l 277 | [3, 1, 2] 278 | ''' 279 | 280 | 281 | def shuffle(seq, random=None): 282 | r"""Return shuffled *copy* of `seq`.""" 283 | if isinstance(seq, list): 284 | return ipshuffle(seq[:], random) 285 | elif isString(seq): 286 | # seq[0:0] == "" or u"" 287 | return seq[0:0].join(ipshuffle(list(seq)),random) 288 | else: 289 | return type(seq)(ipshuffle(list(seq),random)) 290 | 291 | __test__['shuffle'] = r''' 292 | >>> l = [1,2,3] 293 | >>> shuffle(l, lambda :0.3) 294 | [2, 3, 1] 295 | >>> l 296 | [1, 2, 3] 297 | >>> shuffle(l, lambda :0.4) 298 | [3, 1, 2] 299 | >>> l 300 | [1, 2, 3] 301 | ''' 302 | 303 | # s = open(file).read() would be a nice shorthand -- unfortunately it doesn't 304 | # work (because the file is never properly closed, at least not under 305 | # Jython). Thus: 306 | 307 | 308 | def _normalizeToFile(maybeFile, mode, expand): 309 | if isinstance(maybeFile, int): 310 | return os.fdopen(maybeFile, mode) 311 | elif isString(maybeFile): 312 | if maybeFile.startswith('http://'): #XXX experimental 313 | return urllib2.urlopen(maybeFile) 314 | else: 315 | if expand: 316 | maybeFile = os.path.expandvars(os.path.expanduser(maybeFile)) 317 | return open(maybeFile, mode) 318 | else: 319 | return maybeFile 320 | 321 | 322 | def slurp(file, binary=False, expand=False): 323 | r"""Read in a complete file `file` as a string 324 | Parameters: 325 | 326 | - `file`: a file handle or a string (`str` or `unicode`). 327 | - `binary`: whether to read in the file in binary mode (default: False). 328 | """ 329 | mode = "r" + ["b",""][not binary] 330 | file = _normalizeToFile(file, mode=mode, expand=expand) 331 | try: return file.read() 332 | finally: file.close() 333 | 334 | # FIXME write proper tests for IO stuff 335 | def withFile(file, func, mode='r', expand=False): 336 | """Pass `file` to `func` and ensure the file is closed afterwards. If 337 | `file` is a string, open according to `mode`; if `expand` is true also 338 | expand user and vars. 339 | """ 340 | file = _normalizeToFile(file, mode=mode, expand=expand) 341 | try: return func(file) 342 | finally: file.close() 343 | 344 | 345 | def slurpLines(file, expand=False): 346 | r"""Read in a complete file (specified by a file handler or a filename 347 | string/unicode string) as list of lines""" 348 | file = _normalizeToFile(file, "r", expand) 349 | try: return file.readlines() 350 | finally: file.close() 351 | 352 | def slurpChompedLines(file, expand=False): 353 | r"""Return ``file`` a list of chomped lines. See `slurpLines`.""" 354 | f=_normalizeToFile(file, "r", expand) 355 | try: return list(chompLines(f)) 356 | finally: f.close() 357 | 358 | def strToTempfile(s, suffix=None, prefix=None, dir=None, binary=False): 359 | """Create a new tempfile, write ``s`` to it and return the filename. 360 | `suffix`, `prefix` and `dir` are like in `tempfile.mkstemp`. 361 | """ 362 | fd, filename = tempfile.mkstemp(**dict((k,v) for (k,v) in 363 | [('suffix',suffix),('prefix',prefix),('dir', dir)] 364 | if v is not None)) 365 | spitOut(s, fd, binary) 366 | return filename 367 | 368 | 369 | def spitOut(s, file, binary=False, expand=False): 370 | r"""Write string `s` into `file` (which can be a string (`str` or 371 | `unicode`) or a `file` instance).""" 372 | mode = "w" + ["b",""][not binary] 373 | file = _normalizeToFile(file, mode=mode, expand=expand) 374 | try: file.write(s) 375 | finally: file.close() 376 | 377 | def spitOutLines(lines, file, expand=False): 378 | r"""Write all the `lines` to `file` (which can be a string/unicode or a 379 | file handler).""" 380 | file = _normalizeToFile(file, mode="w", expand=expand) 381 | try: file.writelines(lines) 382 | finally: file.close() 383 | 384 | 385 | #FIXME should use new subprocess module if possible 386 | def readProcess(cmd, *args): 387 | r"""Similar to `os.popen3`, but returns 2 strings (stdin, stdout) and the 388 | exit code (unlike popen2, exit is 0 if no problems occured (for some 389 | bizarre reason popen2 returns None... ). 390 | 391 | FIXME: only works for UNIX; handling of signalled processes. 392 | """ 393 | import popen2 394 | BUFSIZE=1024 395 | import select 396 | popen = popen2.Popen3((cmd,) + args, capturestderr=True) 397 | which = {id(popen.fromchild): [], 398 | id(popen.childerr): []} 399 | select = Result(select.select) 400 | read = Result(os.read) 401 | try: 402 | popen.tochild.close() # XXX make sure we're not waiting forever 403 | while select([popen.fromchild, popen.childerr], [], []): 404 | readSomething = False 405 | for f in select.result[0]: 406 | while read(f.fileno(), BUFSIZE): 407 | which[id(f)].append(read.result) 408 | readSomething = True 409 | if not readSomething: 410 | break 411 | out, err = ["".join(which[id(f)]) 412 | for f in [popen.fromchild, popen.childerr]] 413 | returncode = popen.wait() 414 | 415 | if os.WIFEXITED(returncode): 416 | exit = os.WEXITSTATUS(returncode) 417 | else: 418 | exit = returncode or 1 # HACK: ensure non-zero 419 | finally: 420 | try: 421 | popen.fromchild.close() 422 | finally: 423 | popen.childerr.close() 424 | return out or "", err or "", exit 425 | 426 | 427 | def silentlyRunProcess(cmd,*args): 428 | """Like `runProcess` but stdout and stderr output is discarded. FIXME: only 429 | works for UNIX!""" 430 | return readProcess(cmd,*args)[2] 431 | 432 | def runProcess(cmd, *args): 433 | """Run `cmd` (which is searched for in the executable path) with `args` and 434 | return the exit status. 435 | 436 | In general (unless you know what you're doing) use:: 437 | 438 | runProcess('program', filename) 439 | 440 | rather than:: 441 | 442 | os.system('program %s' % filename) 443 | 444 | because the latter will not work as expected if `filename` contains 445 | spaces or shell-metacharacters. 446 | 447 | If you need more fine-grained control look at ``os.spawn*``. 448 | """ 449 | from os import spawnvp, P_WAIT 450 | return spawnvp(P_WAIT, cmd, (cmd,) + args) 451 | 452 | 453 | #FIXME; check latest word on this... 454 | def isSeq(obj): 455 | r"""Returns true if obj is a sequence (which does purposefully **not** 456 | include strings, because these are pseudo-sequences that mess 457 | up recursion.)""" 458 | return operator.isSequenceType(obj) and not isinstance(obj, (str, unicode)) 459 | 460 | 461 | 462 | # os.path's splitext is EVIL: it thinks that dotfiles are extensions! 463 | def splitext(p): 464 | r"""Like the normal splitext (in posixpath), but doesn't treat dotfiles 465 | (e.g. .emacs) as extensions. Also uses os.sep instead of '/'.""" 466 | 467 | root, ext = os.path.splitext(p) 468 | # check for dotfiles 469 | if (not root or root[-1] == os.sep): # XXX: use '/' or os.sep here??? 470 | return (root + ext, "") 471 | else: 472 | return root, ext 473 | 474 | 475 | 476 | #_. LIST MANIPULATION 477 | 478 | def bipart(func, seq): 479 | r"""Like a partitioning version of `filter`. Returns 480 | ``[itemsForWhichFuncReturnedFalse, itemsForWhichFuncReturnedTrue]``. 481 | 482 | Example: 483 | 484 | >>> bipart(bool, [1,None,2,3,0,[],[0]]) 485 | [[None, 0, []], [1, 2, 3, [0]]] 486 | """ 487 | 488 | if func is None: func = bool 489 | res = [[],[]] 490 | for i in seq: res[not not func(i)].append(i) 491 | return res 492 | 493 | 494 | def unzip(seq): 495 | r"""Perform the reverse operation to the builtin `zip` function.""" 496 | # XXX should special case for more efficient handling 497 | return zip(*seq) 498 | 499 | def binarySearchPos(seq, item, cmpfunc=cmp): 500 | r"""Return the position of `item` in ordered sequence `seq`, using comparison 501 | function `cmpfunc` (defaults to ``cmp``) and return the first found 502 | position of `item`, or -1 if `item` is not in `seq`. The returned position 503 | is NOT guaranteed to be the first occurence of `item` in `seq`.""" 504 | 505 | if not seq: return -1 506 | left, right = 0, len(seq) - 1 507 | if cmpfunc(seq[left], item) == 1 and \ 508 | cmpfunc(seq[right], item) == -1: 509 | return -1 510 | while left <= right: 511 | halfPoint = (left + right) // 2 512 | comp = cmpfunc(seq[halfPoint], item) 513 | if comp > 0: right = halfPoint - 1 514 | elif comp < 0: left = halfPoint + 1 515 | else: return halfPoint 516 | return -1 517 | 518 | __test__['binarySearchPos'] = r""" 519 | >>> binarySearchPos(range(20), 20) 520 | -1 521 | >>> binarySearchPos(range(20), 1) 522 | 1 523 | >>> binarySearchPos(range(20), 19) 524 | 19 525 | >>> binarySearchPos(range(20), 0) 526 | 0 527 | >>> binarySearchPos(range(1,21,2),4) 528 | -1 529 | >>> binarySearchPos(range(0,20,2), 6) 530 | 3 531 | >>> binarySearchPos(range(19, -1, -1), 3, lambda x,y:-cmp(x,y)) 532 | 16 533 | """ 534 | 535 | 536 | def binarySearchItem(seq, item, cmpfunc=cmp): 537 | r""" Search an ordered sequence `seq` for `item`, using comparison function 538 | `cmpfunc` (defaults to ``cmp``) and return the first found instance of 539 | `item`, or `None` if item is not in `seq`. The returned item is NOT 540 | guaranteed to be the first occurrence of item in `seq`.""" 541 | pos = binarySearchPos(seq, item, cmpfunc) 542 | if pos == -1: raise KeyError("Item not in seq") 543 | else: return seq[pos] 544 | 545 | #XXX could extend this for other sequence types 546 | def rotate(l, steps=1): 547 | r"""Rotates a list `l` `steps` to the left. Accepts 548 | `steps` > `len(l)` or < 0. 549 | 550 | >>> rotate([1,2,3]) 551 | [2, 3, 1] 552 | >>> rotate([1,2,3,4],-2) 553 | [3, 4, 1, 2] 554 | >>> rotate([1,2,3,4],-5) 555 | [4, 1, 2, 3] 556 | >>> rotate([1,2,3,4],1) 557 | [2, 3, 4, 1] 558 | >>> l = [1,2,3]; rotate(l) is not l 559 | True 560 | """ 561 | if len(l): 562 | steps %= len(l) 563 | if steps: 564 | res = l[steps:] 565 | res.extend(l[:steps]) 566 | return res 567 | 568 | def iprotate(l, steps=1): 569 | r"""Like rotate, but modifies `l` in-place. 570 | 571 | >>> l = [1,2,3] 572 | >>> iprotate(l) is l 573 | True 574 | >>> l 575 | [2, 3, 1] 576 | >>> iprotate(iprotate(l, 2), -3) 577 | [1, 2, 3] 578 | 579 | """ 580 | if len(l): 581 | steps %= len(l) 582 | if steps: 583 | firstPart = l[:steps] 584 | del l[:steps] 585 | l.extend(firstPart) 586 | return l 587 | 588 | 589 | 590 | 591 | # XXX: could wrap that in try: except: for non-hashable types, or provide an 592 | # identity function parameter, but well... this is fast and simple (but wastes 593 | # memory) 594 | def unique(iterable): 595 | r"""Returns all unique items in `iterable` in the *same* order (only works 596 | if items in `seq` are hashable). 597 | """ 598 | d = {} 599 | return (d.setdefault(x,x) for x in iterable if x not in d) 600 | 601 | __test__['unique'] = r""" 602 | >>> list(unique(range(3))) == range(3) 603 | True 604 | >>> list(unique([1,1,2,2,3,3])) == range(1,4) 605 | True 606 | >>> list(unique([1])) == [1] 607 | True 608 | >>> list(unique([])) == [] 609 | True 610 | >>> list(unique(['a','a'])) == ['a'] 611 | True 612 | """ 613 | 614 | def notUnique(iterable, reportMax=INF): 615 | """Returns the elements in `iterable` that aren't unique; stops after it found 616 | `reportMax` non-unique elements. 617 | 618 | Examples: 619 | 620 | >>> list(notUnique([1,1,2,2,3,3])) 621 | [1, 2, 3] 622 | >>> list(notUnique([1,1,2,2,3,3], 1)) 623 | [1] 624 | """ 625 | hash = {} 626 | n=0 627 | if reportMax < 1: 628 | raise ValueError("`reportMax` must be >= 1 and is %r" % reportMax) 629 | for item in iterable: 630 | count = hash[item] = hash.get(item, 0) + 1 631 | if count > 1: 632 | yield item 633 | n += 1 634 | if n >= reportMax: 635 | return 636 | __test__['notUnique'] = r""" 637 | >>> list(notUnique(range(3))) 638 | [] 639 | >>> list(notUnique([1])) 640 | [] 641 | >>> list(notUnique([])) 642 | [] 643 | >>> list(notUnique(['a','a'])) 644 | ['a'] 645 | >>> list(notUnique([1,1,2,2,3,3],2)) 646 | [1, 2] 647 | >>> list(notUnique([1,1,2,2,3,3],0)) 648 | Traceback (most recent call last): 649 | [...] 650 | ValueError: `reportMax` must be >= 1 and is 0 651 | """ 652 | 653 | 654 | 655 | def unweave(iterable, n=2): 656 | r"""Divide `iterable` in `n` lists, so that every `n`th element belongs to 657 | list `n`. 658 | 659 | Example: 660 | 661 | >>> unweave((1,2,3,4,5), 3) 662 | [[1, 4], [2, 5], [3]] 663 | """ 664 | res = [[] for i in range(n)] 665 | i = 0 666 | for x in iterable: 667 | res[i % n].append(x) 668 | i += 1 669 | return res 670 | 671 | 672 | 673 | 674 | def weave(*iterables): 675 | r"""weave(seq1 [, seq2] [...]) -> iter([seq1[0], seq2[0] ...]). 676 | 677 | >>> list(weave([1,2,3], [4,5,6,'A'], [6,7,8, 'B', 'C'])) 678 | [1, 4, 6, 2, 5, 7, 3, 6, 8] 679 | 680 | Any iterable will work. The first exhausted iterable determines when to 681 | stop. FIXME rethink stopping semantics. 682 | 683 | >>> list(weave(iter(('is','psu')), ('there','no', 'censorship'))) 684 | ['is', 'there', 'psu', 'no'] 685 | >>> list(weave(('there','no', 'censorship'), iter(('is','psu')))) 686 | ['there', 'is', 'no', 'psu', 'censorship'] 687 | """ 688 | iterables = map(iter, iterables) 689 | while True: 690 | for it in iterables: yield it.next() 691 | 692 | def atIndices(indexable, indices, default=__unique): 693 | r"""Return a list of items in `indexable` at positions `indices`. 694 | 695 | Examples: 696 | 697 | >>> atIndices([1,2,3], [1,1,0]) 698 | [2, 2, 1] 699 | >>> atIndices([1,2,3], [1,1,0,4], 'default') 700 | [2, 2, 1, 'default'] 701 | >>> atIndices({'a':3, 'b':0}, ['a']) 702 | [3] 703 | """ 704 | if default is __unique: 705 | return [indexable[i] for i in indices] 706 | else: 707 | res = [] 708 | for i in indices: 709 | try: 710 | res.append(indexable[i]) 711 | except (IndexError, KeyError): 712 | res.append(default) 713 | return res 714 | 715 | __test__['atIndices'] = r''' 716 | >>> atIndices([1,2,3], [1,1,0,4]) 717 | Traceback (most recent call last): 718 | [...] 719 | IndexError: list index out of range 720 | >>> atIndices({1:2,3:4}, [1,1,0,4]) 721 | Traceback (most recent call last): 722 | [...] 723 | KeyError: 0 724 | >>> atIndices({1:2,3:4}, [1,1,0,4], 'default') 725 | [2, 2, 'default', 'default'] 726 | ''' 727 | 728 | 729 | 730 | 731 | #XXX: should those have reduce like optional end-argument? 732 | #FIXME should s defaul to n? (1 is less typing/better than len(somearg(foo)); ) 733 | def window(iterable, n=2, s=1): 734 | r"""Move an `n`-item (default 2) windows `s` steps (default 1) at a time 735 | over `iterable`. 736 | 737 | Examples: 738 | 739 | >>> list(window(range(6),2)) 740 | [(0, 1), (1, 2), (2, 3), (3, 4), (4, 5)] 741 | >>> list(window(range(6),3)) 742 | [(0, 1, 2), (1, 2, 3), (2, 3, 4), (3, 4, 5)] 743 | >>> list(window(range(6),3, 2)) 744 | [(0, 1, 2), (2, 3, 4)] 745 | >>> list(window(range(5),3,2)) == list(window(range(6),3,2)) 746 | True 747 | """ 748 | assert n >= s 749 | last = [] 750 | for elt in iterable: 751 | last.append(elt) 752 | if len(last) == n: yield tuple(last); last=last[s:] 753 | #FIXME 754 | xwindow = window 755 | # FIXME rename to block? 756 | def group(iterable, n=2, pad=__unique): 757 | r"""Iterate `n`-wise (default pairwise) over `iter`. 758 | Examples: 759 | 760 | >>> for (first, last) in group("Akira Kurosawa John Ford".split()): 761 | ... print "given name: %s surname: %s" % (first, last) 762 | ... 763 | given name: Akira surname: Kurosawa 764 | given name: John surname: Ford 765 | >>> 766 | >>> # both contain the same number of pairs 767 | >>> list(group(range(9))) == list(group(range(8))) 768 | True 769 | >>> # with n=3 770 | >>> list(group(range(10), 3)) 771 | [(0, 1, 2), (3, 4, 5), (6, 7, 8)] 772 | >>> list(group(range(10), 3, pad=0)) 773 | [(0, 1, 2), (3, 4, 5), (6, 7, 8), (9, 0, 0)] 774 | """ 775 | assert n>0 # ensure it doesn't loop forever 776 | if pad is not __unique: it = chain(iterable, (pad,)*(n-1)) 777 | else: it = iter(iterable) 778 | perTuple = xrange(n) 779 | while True: 780 | yield tuple([it.next() for i in perTuple]) 781 | 782 | def iterate(f, n=None, last=__unique): 783 | """ 784 | >>> list(iterate(lambda x:x//2)(128)) 785 | [128, 64, 32, 16, 8, 4, 2, 1, 0] 786 | >>> list(iterate(lambda x:x//2, n=2)(128)) 787 | [128, 64] 788 | """ 789 | if n is not None: 790 | def funciter(start): 791 | for i in xrange(n): yield start; start = f(start) 792 | else: 793 | def funciter(start): 794 | while True: 795 | yield start 796 | last = f(start) 797 | if last == start: return 798 | last, start = start, last 799 | return funciter 800 | 801 | 802 | def dropwhilenot(func, iterable): 803 | """ 804 | >>> list(dropwhilenot(lambda x:x==3, range(10))) 805 | [3, 4, 5, 6, 7, 8, 9] 806 | """ 807 | iterable = iter(iterable) 808 | for x in iterable: 809 | if func(x): break 810 | else: return 811 | yield x 812 | for x in iterable: 813 | yield x 814 | 815 | 816 | def stretch(iterable, n=2): 817 | r"""Repeat each item in `iterable` `n` times. 818 | 819 | Example: 820 | 821 | >>> list(stretch(range(3), 2)) 822 | [0, 0, 1, 1, 2, 2] 823 | """ 824 | times = range(n) 825 | for item in iterable: 826 | for i in times: yield item 827 | 828 | 829 | def splitAt(iterable, indices): 830 | r"""Yield chunks of `iterable`, split at the points in `indices`: 831 | 832 | >>> [l for l in splitAt(range(10), [2,5])] 833 | [[0, 1], [2, 3, 4], [5, 6, 7, 8, 9]] 834 | 835 | splits past the length of `iterable` are ignored: 836 | 837 | >>> [l for l in splitAt(range(10), [2,5,10])] 838 | [[0, 1], [2, 3, 4], [5, 6, 7, 8, 9]] 839 | 840 | 841 | """ 842 | iterable = iter(iterable) 843 | now = 0 844 | for to in indices: 845 | try: 846 | res = [] 847 | for i in range(now, to): res.append(iterable.next()) 848 | except StopIteration: yield res; return 849 | yield res 850 | now = to 851 | res = list(iterable) 852 | if res: yield res 853 | 854 | 855 | __test__['splitAt'] = r""" 856 | >>> [l for l in splitAt(range(10), [1,5])] 857 | [[0], [1, 2, 3, 4], [5, 6, 7, 8, 9]] 858 | >>> [l for l in splitAt(range(10), [2,5,10])] 859 | [[0, 1], [2, 3, 4], [5, 6, 7, 8, 9]] 860 | >>> [l for l in splitAt(range(10), [2,5,9])] 861 | [[0, 1], [2, 3, 4], [5, 6, 7, 8], [9]] 862 | >>> [l for l in splitAt(range(10), [2,5,11])] 863 | [[0, 1], [2, 3, 4], [5, 6, 7, 8, 9]] 864 | 865 | """ 866 | 867 | 868 | 869 | 870 | #_. HASH MANIPULATION 871 | 872 | 873 | # XXX should we use copy or start with empty dict? former is more generic 874 | # (should work for other dict types, too) 875 | def update(d, e): 876 | """Return a copy of dict `d` updated with dict `e`.""" 877 | res = copy.copy(d) 878 | res.update(e) 879 | return res 880 | 881 | def invertDict(d, allowManyToOne=False): 882 | r"""Return an inverted version of dict `d`, so that values become keys and 883 | vice versa. If multiple keys in `d` have the same value an error is 884 | raised, unless `allowManyToOne` is true, in which case one of those 885 | key-value pairs is chosen at random for the inversion. 886 | 887 | Examples: 888 | 889 | >>> invertDict({1: 2, 3: 4}) == {2: 1, 4: 3} 890 | True 891 | >>> invertDict({1: 2, 3: 2}) 892 | Traceback (most recent call last): 893 | File "", line 1, in ? 894 | ValueError: d can't be inverted! 895 | >>> invertDict({1: 2, 3: 2}, allowManyToOne=True).keys() 896 | [2] 897 | """ 898 | res = dict(izip(d.itervalues(), d.iterkeys())) 899 | if not allowManyToOne and len(res) != len(d): 900 | raise ValueError("d can't be inverted!") 901 | return res 902 | 903 | 904 | #_. LISP LIKES 905 | 906 | #AARGH strings are EVIL... 907 | def iflatten(seq, isSeq=isSeq): 908 | r"""Like `flatten` but lazy.""" 909 | for elt in seq: 910 | if isSeq(elt): 911 | for x in iflatten(elt, isSeq): 912 | yield x 913 | else: 914 | yield elt 915 | 916 | __test__['iflatten'] = r""" 917 | >>> type(iflatten([])) 918 | 919 | >>> iflatten([]).next() 920 | Traceback (most recent call last): 921 | File "", line 1, in ? 922 | StopIteration 923 | >>> (a,b,c) = iflatten([1,["2", ([3],)]]) 924 | >>> (a,b,c) == (1, '2', 3) 925 | True 926 | 927 | """ 928 | 929 | def flatten(seq, isSeq=isSeq): 930 | r"""Returns a flattened version of a sequence `seq` as a `list`. 931 | Parameters: 932 | 933 | - `seq`: The sequence to be flattened (any iterable). 934 | - `isSeq`: The function called to determine whether something is a 935 | sequence (default: `isSeq`). *Beware that this function should 936 | **never** test positive for strings, because they are no real 937 | sequences and thus cause infinite recursion.* 938 | 939 | Examples: 940 | 941 | >>> flatten([1,[2,3,(4,[5,6]),7,8]]) 942 | [1, 2, 3, 4, 5, 6, 7, 8] 943 | >>> # flaten only lists 944 | >>> flatten([1,[2,3,(4,[5,6]),7,8]], isSeq=lambda x:isinstance(x, list)) 945 | [1, 2, 3, (4, [5, 6]), 7, 8] 946 | >>> flatten([1,2]) 947 | [1, 2] 948 | >>> flatten([]) 949 | [] 950 | >>> flatten('123') 951 | ['1', '2', '3'] 952 | """ 953 | return [a for elt in seq 954 | for a in (isSeq(elt) and flatten(elt, isSeq) or 955 | [elt])] 956 | 957 | 958 | def countIf(predicate, iterable): 959 | r"""Count all the elements of for which `predicate` returns true.""" 960 | return sum(1 for x in iterable if predicate(x)) 961 | 962 | __test__['countIf'] = r""" 963 | >>> countIf(bool, range(10)) 964 | 9 965 | >>> countIf(bool, range(-1)) 966 | 0 967 | """ 968 | 969 | def positionIf(pred, seq): 970 | """ 971 | >>> positionIf(lambda x: x > 3, range(10)) 972 | 4 973 | """ 974 | for i,e in enumerate(seq): 975 | if pred(e): 976 | return i 977 | return -1 978 | 979 | 980 | 981 | def union(seq1=(), *seqs): 982 | r"""Return the set union of `seq1` and `seqs`, duplicates removed, order random. 983 | 984 | Examples: 985 | >>> union() 986 | [] 987 | >>> union([1,2,3]) 988 | [1, 2, 3] 989 | >>> union([1,2,3], {1:2, 5:1}) 990 | [1, 2, 3, 5] 991 | >>> union((1,2,3), ['a'], "bcd") 992 | ['a', 1, 2, 3, 'd', 'b', 'c'] 993 | >>> union([1,2,3], iter([0,1,1,1])) 994 | [0, 1, 2, 3] 995 | 996 | """ 997 | if not seqs: return list(seq1) 998 | res = set(seq1) 999 | for seq in seqs: 1000 | res.update(set(seq)) 1001 | return list(res) 1002 | 1003 | # by making this a function, we can accomodate for new types later 1004 | def isSet(obj): 1005 | import sets 1006 | return isinstance(obj, (set,sets.BaseSet)) # AARGH -- WTF are they not subclasses!!! 1007 | 1008 | 1009 | 1010 | 1011 | def without(seq1, seq2): 1012 | r"""Return a list with all elements in `seq2` removed from `seq1`, order 1013 | preserved. 1014 | 1015 | Examples: 1016 | 1017 | >>> without([1,2,3,1,2], [1]) 1018 | [2, 3, 2] 1019 | """ 1020 | if isSet(seq2): d2 = seq2 1021 | else: d2 = set(seq2) 1022 | return [elt for elt in seq1 if elt not in d2] 1023 | 1024 | 1025 | def findIf(predicate, iterable): 1026 | """ 1027 | >>> findIf(lambda x: x>20, [10,20,30,40]) 1028 | 30 1029 | >>> findIf(lambda x: x>40, [10,20,30,40]) 1030 | False 1031 | """ 1032 | try: return ifilter(predicate, iterable).next() 1033 | except StopIteration: return False 1034 | 1035 | 1036 | 1037 | def some(predicate, *seqs): 1038 | """ 1039 | >>> some(lambda x: x, [0, False, None]) 1040 | False 1041 | >>> some(lambda x: x, [None, 0, 2, 3]) 1042 | 2 1043 | >>> some(operator.eq, [0,1,2], [2,1,0]) 1044 | True 1045 | >>> some(operator.eq, [1,2], [2,1]) 1046 | False 1047 | """ 1048 | try: 1049 | if len(seqs) == 1: return ifilter(bool,imap(predicate, seqs[0])).next() 1050 | else: return ifilter(bool,starmap(predicate, izip(*seqs))).next() 1051 | except StopIteration: return False 1052 | 1053 | def every(predicate, *iterables): 1054 | r"""Like `some`, but only returns `True` if all the elements of `iterables` 1055 | satisfy `predicate`. 1056 | 1057 | Examples: 1058 | >>> every(bool, []) 1059 | True 1060 | >>> every(bool, [0]) 1061 | False 1062 | >>> every(bool, [1,1]) 1063 | True 1064 | >>> every(operator.eq, [1,2,3],[1,2]) 1065 | True 1066 | >>> every(operator.eq, [1,2,3],[0,2]) 1067 | False 1068 | """ 1069 | try: 1070 | if len(iterables) == 1: ifilterfalse(predicate, iterables[0]).next() 1071 | else: ifilterfalse(bool, starmap(predicate, izip(*iterables))).next() 1072 | except StopIteration: return True 1073 | else: return False 1074 | 1075 | 1076 | 1077 | 1078 | 1079 | #_. TIME AND DATE HANDLING 1080 | 1081 | # XXX: not strictly a time related function, but well... 1082 | def nTimes(n, f, *args, **kwargs): 1083 | r"""Call `f` `n` times with `args` and `kwargs`. 1084 | Useful e.g. for simplistic timing. 1085 | 1086 | Examples: 1087 | 1088 | >>> nTimes(3, sys.stdout.write, 'hallo\n') 1089 | hallo 1090 | hallo 1091 | hallo 1092 | 1093 | """ 1094 | for i in xrange(n): f(*args, **kwargs) 1095 | 1096 | 1097 | def timeCall(*funcAndArgs, **kwargs): 1098 | r"""Return the time (in ms) it takes to call a function (the first 1099 | argument) with the remaining arguments and `kwargs`. 1100 | 1101 | Examples: 1102 | 1103 | To find out how long ``func('foo', spam=1)`` takes to execute, do: 1104 | 1105 | ``timeCall(func, foo, spam=1)`` 1106 | """ 1107 | 1108 | func, args = funcAndArgs[0], funcAndArgs[1:] 1109 | start = time.time() 1110 | func(*args, **kwargs) 1111 | return time.time() - start 1112 | 1113 | __test__['timeCall'] = r""" 1114 | >>> round(timeCall(time.sleep, 1)) 1115 | 1.0 1116 | """ 1117 | 1118 | 1119 | #_. STRING PROCESSING 1120 | 1121 | def isString(obj): 1122 | r"""Return `True` iff `obj` is some type of string (i.e. `str` or 1123 | `unicode`).""" 1124 | return isinstance(obj, basestring) 1125 | 1126 | chomp = partial(re.compile(r"\n+$").sub, '') 1127 | chomp.__doc__ = r"""Discards all trailing newlines in string.""" 1128 | __test__['chomp'] = r""" 1129 | >>> chomp('foobar\n\nf') == 'foobar\n\nf' 1130 | True 1131 | >>> chomp('foobar\n\n') == 'foobar' 1132 | True 1133 | """ 1134 | 1135 | def chompLines(lines): 1136 | r"""Returns a lazy sequence of `lines` sans any trailing newline.""" 1137 | return imap(chomp, lines) 1138 | # faster 2.5 version (not 100% identical, but good enough): 1139 | ## return (L[:-1] if L[-1:] == "\n" else L for L in lines) 1140 | 1141 | __test__['chompLines'] = r""" 1142 | >>> list(chompLines(['\n', 'foo\n', 'foobar\n\nf'])) 1143 | ['', 'foo', 'foobar\n\nf'] 1144 | >>> chompLines(['foo\n', '3', 'foobar\n\nf']).next() 1145 | 'foo' 1146 | """ 1147 | 1148 | def replaceStrs(s, *args): 1149 | r"""Replace all ``(frm, to)`` tuples in `args` in string `s`. 1150 | 1151 | >>> replaceStrs("nothing is better than warm beer", 1152 | ... ('nothing','warm beer'), ('warm beer','nothing')) 1153 | 'warm beer is better than nothing' 1154 | 1155 | """ 1156 | if args == (): return s 1157 | mapping = dict((frm, to) for frm, to in args) 1158 | return re.sub("|".join(map(re.escape, mapping.keys())), 1159 | lambda match:mapping[match.group(0)], s) 1160 | 1161 | def escape(s): 1162 | ur"""Backslash escape non-printable non-ascii characters and ``"'\``. 1163 | 1164 | >>> escape('''a\\string"with'some\tspecial\ncharacters''') 1165 | 'a\\\\string"with\\'some\\tspecial\\ncharacters' 1166 | >>> print eval(escape(u'''u"a\\'unicode\tstring ®"''')) 1167 | a\'unicode string ® 1168 | 1169 | """ 1170 | try: 1171 | return escapeAscii(s) 1172 | except (UnicodeEncodeError, TypeError): 1173 | return escapeUnicode(s) 1174 | def escapeAscii(s): 1175 | return codecs.getwriter('string_escape').encode(s)[0] 1176 | def escapeUnicode(s): 1177 | return codecs.getwriter('unicode_escape').encode(s)[0] 1178 | def unescape(s): 1179 | r"""Inverse of `escape`. 1180 | >>> unescape(r'\x41\n\x42\n\x43') 1181 | 'A\nB\nC' 1182 | >>> unescape(r'\u86c7') 1183 | u'\u86c7' 1184 | >>> unescape(u'ah') 1185 | u'ah' 1186 | """ 1187 | if re.search(r'(?>> lineAndColumnAt("0123\n56", 5) 1205 | (2, 0) 1206 | >>> lineAndColumnAt("0123\n56", 6) 1207 | (2, 1) 1208 | >>> lineAndColumnAt("0123\n56", 0) 1209 | (1, 0) 1210 | """ 1211 | if pos >= len(s): 1212 | raise IndexError("`pos` %d not in string" % pos) 1213 | # *don't* count last '\n', if it is at pos! 1214 | line = s.count('\n',0,pos) 1215 | if line: 1216 | return line + 1, pos - s.rfind('\n',0,pos) - 1 1217 | else: 1218 | return 1, pos 1219 | 1220 | #_. TERMINAL OUTPUT FUNCTIONS 1221 | 1222 | def prin(*args, **kwargs): 1223 | r"""Like ``print``, but a function. I.e. prints out all arguments as 1224 | ``print`` would do. Specify output stream like this:: 1225 | 1226 | print('ERROR', `out="sys.stderr"``). 1227 | 1228 | """ 1229 | print >> kwargs.get('out',None), " ".join([str(arg) for arg in args]) 1230 | __test__['prin'] = r""" 1231 | >>> prin(1,2,3,out=None) 1232 | 1 2 3 1233 | """ 1234 | 1235 | 1236 | def underline(s, lineC='-'): 1237 | r"""Return `s` + newline + enough '-' (or lineC if specified) to underline it 1238 | and a final newline. 1239 | 1240 | Example: 1241 | 1242 | >>> print underline("TOP SECRET", '*'), 1243 | TOP SECRET 1244 | ********** 1245 | 1246 | """ 1247 | return s + "\n" + lineC * len(s) + "\n" 1248 | 1249 | def fitString(s, maxCol=79, newlineReplacement=None): 1250 | r"""Truncate `s` if necessary to fit into a line of width `maxCol` 1251 | (default: 79), also replacing newlines with `newlineReplacement` (default 1252 | `None`: in which case everything after the first newline is simply 1253 | discarded). 1254 | 1255 | Examples: 1256 | 1257 | >>> fitString('12345', maxCol=5) 1258 | '12345' 1259 | >>> fitString('123456', maxCol=5) 1260 | '12...' 1261 | >>> fitString('a line\na second line') 1262 | 'a line' 1263 | >>> fitString('a line\na second line', newlineReplacement='\\n') 1264 | 'a line\\na second line' 1265 | """ 1266 | assert isString(s) 1267 | if '\n' in s: 1268 | if newlineReplacement is None: 1269 | s = s[:s.index('\n')] 1270 | else: 1271 | s = s.replace("\n", newlineReplacement) 1272 | if maxCol is not None and len(s) > maxCol: 1273 | s = "%s..." % s[:maxCol-3] 1274 | return s 1275 | 1276 | #_. EVIL THINGS 1277 | 1278 | def magicGlobals(level=1): 1279 | r"""Return the globals of the *caller*'s caller (default), or `level` 1280 | callers up.""" 1281 | return inspect.getouterframes(inspect.currentframe())[1+level][0].f_globals 1282 | 1283 | def magicLocals(level=1): 1284 | r"""Return the locals of the *caller*'s caller (default) , or `level` 1285 | callers up. 1286 | """ 1287 | return inspect.getouterframes(inspect.currentframe())[1+level][0].f_locals 1288 | 1289 | def thisModule(): 1290 | return sys.modules[sys._getframe(3).f_globals['__name__']] 1291 | 1292 | #_. PERSISTENCE 1293 | 1294 | def __saveVarsHelper(filename, varNamesStr, outOf,extension='.bpickle',**opts): 1295 | filename = os.path.expanduser(filename) 1296 | if outOf is None: outOf = magicGlobals(2) 1297 | if not varNamesStr or not isString(varNamesStr): 1298 | raise ValueError, "varNamesStr must be a string!" 1299 | varnames = varNamesStr.split() 1300 | if not splitext(filename)[1]: filename += extension 1301 | if opts.get("overwrite") == 0 and os.path.exists(filename): 1302 | raise RuntimeError("File already exists") 1303 | return filename, varnames, outOf 1304 | 1305 | def saveVars(filename, varNamesStr, outOf=None, **opts): 1306 | r"""Pickle name and value of all those variables in `outOf` (default: all 1307 | global variables (as seen from the caller)) that are named in 1308 | `varNamesStr` into a file called `filename` (if no extension is given, 1309 | '.bpickle' is appended). Overwrites file without asking, unless you 1310 | specify `overwrite=0`. Load again with `loadVars`. 1311 | 1312 | Thus, to save the global variables ``bar``, ``foo`` and ``baz`` in the 1313 | file 'savedVars' do:: 1314 | 1315 | saveVars('savedVars', 'bar foo baz') 1316 | 1317 | """ 1318 | filename, varnames, outOf = __saveVarsHelper( 1319 | filename, varNamesStr, outOf, **opts) 1320 | print "pickling:\n", "\n".join(sorted(varnames)) 1321 | try: 1322 | f = None 1323 | f = open(filename, "wb") 1324 | 1325 | cPickle.dump(dict(zip(varnames, atIndices(outOf, varnames))), 1326 | f, 1) # UGH: cPickle, unlike pickle doesn't accept bin=1 1327 | finally: 1328 | if f: f.close() 1329 | 1330 | 1331 | #FIXME untested 1332 | def saveDict(filename, d, **kwargs): 1333 | saveVars(filename, " ".join(d.keys()), outOf=d, **kwargs) 1334 | 1335 | 1336 | def addVars(filename, varNamesStr, outOf=None): 1337 | r"""Like `saveVars`, but appends additional variables to file.""" 1338 | filename, varnames, outOf = __saveVarsHelper(filename, varNamesStr, outOf) 1339 | f = None 1340 | try: 1341 | f = open(filename, "rb") 1342 | h = cPickle.load(f) 1343 | f.close() 1344 | 1345 | h.update(dict(zip(varnames, atIndices(outOf, varnames)))) 1346 | f = open(filename, "wb") 1347 | cPickle.dump( h, f , 1 ) 1348 | finally: 1349 | if f: f.close() 1350 | 1351 | def loadDict(filename): 1352 | """Return the variables pickled pickled into `filename` with `saveVars` 1353 | as a dict.""" 1354 | filename = os.path.expanduser(filename) 1355 | if not splitext(filename)[1]: filename += ".bpickle" 1356 | f = None 1357 | try: 1358 | f = open(filename, "rb") 1359 | varH = cPickle.load(f) 1360 | finally: 1361 | if f: f.close() 1362 | return varH 1363 | 1364 | def loadVars(filename, ask=True, into=None, only=None): 1365 | r"""Load variables pickled with `saveVars`. 1366 | Parameters: 1367 | 1368 | - `ask`: If `True` then don't overwrite existing variables without 1369 | asking. 1370 | - `only`: A list to limit the variables to or `None`. 1371 | - `into`: The dictionary the variables should be loaded into (defaults 1372 | to global dictionary). 1373 | """ 1374 | 1375 | filename = os.path.expanduser(filename) 1376 | if into is None: into = magicGlobals() 1377 | varH = loadDict(filename) 1378 | toUnpickle = only or varH.keys() 1379 | alreadyDefined = filter(into.has_key, toUnpickle) 1380 | if alreadyDefined and ask: 1381 | print "The following vars already exist; overwrite (yes/NO)?\n",\ 1382 | "\n".join(alreadyDefined) 1383 | if raw_input() != "yes": 1384 | toUnpickle = without(toUnpickle, alreadyDefined) 1385 | if not toUnpickle: 1386 | print "nothing to unpickle" 1387 | return None 1388 | print "unpickling:\n",\ 1389 | "\n".join(sorted(toUnpickle)) 1390 | for k in varH.keys(): 1391 | if k not in toUnpickle: 1392 | del varH[k] 1393 | into.update(varH) 1394 | 1395 | 1396 | 1397 | #_. SCRIPTING 1398 | 1399 | 1400 | 1401 | def runInfo(prog=None,vers=None,date=None,user=None,dir=None,args=None): 1402 | r"""Create a short info string detailing how a program was invoked. This is 1403 | meant to be added to a history comment field of a data file were it is 1404 | important to keep track of what programs modified it and how. 1405 | 1406 | !!!:`args` should be a **``list``** not a ``str``.""" 1407 | 1408 | return "%(prog)s %(vers)s;" \ 1409 | " run %(date)s by %(usr)s in %(dir)s with: %(args)s'n" % \ 1410 | mkDict(prog=prog or sys.argv[0], 1411 | vers=vers or magicGlobals().get("__version__", ""), 1412 | date=date or isoDateTimeStr(), 1413 | usr=user or getpass.getuser(), 1414 | dir=dir or os.getcwd(), 1415 | args=" ".join(args or sys.argv)) 1416 | 1417 | 1418 | 1419 | 1420 | class DryRun: 1421 | """A little class that is usefull for debugging and testing and for 1422 | programs that should have a "dry run" option. 1423 | 1424 | 1425 | Examples: 1426 | 1427 | >>> import sys 1428 | >>> from os import system 1429 | >>> dry = True 1430 | >>> # that's how you can switch the programms behaviour between dry run 1431 | >>> 1432 | >>> if dry: 1433 | ... # (`out` defaults to stdout, but we need it here for doctest) 1434 | ... run = DryRun(dry=True, out=sys.stdout) 1435 | ... else: 1436 | ... run = DryRun(dry=False, out=sys.stdout) 1437 | ... 1438 | >>> ## IGNORE 2 hacks to get doctest working, please ignore 1439 | >>> def system(x): print "hallo" 1440 | ... 1441 | >>> run.__repr__ = lambda : "" 1442 | >>> ## UNIGNORE 1443 | >>> run(system, 'echo "hallo"') 1444 | system('echo "hallo"') 1445 | >>> # now let's get serious 1446 | >>> run.dry = False 1447 | >>> run(system, 'echo "hallo"') 1448 | hallo 1449 | >>> # just show what command would be run again 1450 | >>> run.dry = True 1451 | >>> # You can also specify how the output for a certain function should be 1452 | ... # look like, e.g. if you just want to print the command for all system 1453 | ... # calls, do the following: 1454 | >>> run.addFormatter(system, lambda x,*args, **kwargs: args[0]) 1455 | 1456 | >>> run(system, 'echo "hallo"') 1457 | echo "hallo" 1458 | >>> # Other functions will still be formated with run.defaultFormatter, which 1459 | ... # gives the following results 1460 | >>> run(int, "010101", 2) 1461 | int('010101', 2) 1462 | >>> # Switch to wet run again: 1463 | >>> run.dry = False 1464 | >>> run(int, "010101", 2) 1465 | 21 1466 | >>> 1467 | 1468 | Caveats: 1469 | 1470 | - remember that arguments might look different from what you specified in 1471 | the source-code, e.g:: 1472 | 1473 | >>> run.dry = True 1474 | >>> run(chr, 0x50) 1475 | chr(80) 1476 | >>> 1477 | 1478 | - 'DryRun(showModule=True)' will try to print the module name where func 1479 | was defined, this might however *not* produce the results you 1480 | expected. For example, a function might be defined in another module 1481 | than the one from which you imported it: 1482 | 1483 | >>> run = DryRun(dry=True, showModule=True) 1484 | >>> run(os.path.join, "path", "file") 1485 | posixpath.join('path', 'file') 1486 | >>> 1487 | 1488 | see `DryRun.__init__` for more details.""" 1489 | 1490 | def __init__(self, dry=True, out=None, showModule=False): 1491 | """ 1492 | Parameters: 1493 | - `dry` : specifies whether to do a try run or not. 1494 | - `out` : is the stream to which all dry runs will be printed 1495 | (default stdout). 1496 | - `showModule` : specifies whether the name of a module in which a 1497 | function was defined is printed (if known). 1498 | """ 1499 | 1500 | self.dry = dry 1501 | self.formatterDict = {} 1502 | self.out = out 1503 | self.showModule = showModule 1504 | def defaultFormatter(func, *args, **kwargs): 1505 | if self.showModule and inspect.getmodule(func): 1506 | funcName = inspect.getmodule(func).__name__ + '.' + func.__name__ 1507 | else: 1508 | funcName = func.__name__ 1509 | return "%s(%s)" % (funcName, 1510 | ", ".join(map(repr,args) + map(lambda x: "%s=%s" % 1511 | tuple(map(repr,x)), kwargs.items()))) 1512 | self.defaultFormatter = defaultFormatter 1513 | def __call__(self, func, *args, **kwargs): 1514 | """Shorcut for self.run.""" 1515 | return self.run(func, *args, **kwargs) 1516 | def run(self, func, *args, **kwargs): 1517 | """Same as ``self.dryRun`` if ``self.dry``, else same as ``self.wetRun``.""" 1518 | if self.dry: 1519 | return self.dryRun(func, *args, **kwargs) 1520 | else: 1521 | return self.wetRun(func, *args, **kwargs) 1522 | def addFormatter(self, func, formatter): 1523 | """Sets the function that is used to format calls to func. formatter is a 1524 | function that is supposed to take these arguments: `func`, `*args` and 1525 | `**kwargs`.""" 1526 | self.formatterDict[func] = formatter 1527 | return self 1528 | def dryRun(self, func, *args, **kwargs): 1529 | """Instead of running function with `*args` and `**kwargs`, just print 1530 | out the function call.""" 1531 | 1532 | print >> self.out, \ 1533 | self.formatterDict.get(func, self.defaultFormatter)(func, *args, **kwargs) 1534 | def wetRun(self, func, *args, **kwargs): 1535 | """Run function with ``*args`` and ``**kwargs``.""" 1536 | return func(*args, **kwargs) 1537 | 1538 | 1539 | 1540 | 1541 | #_. DEBUGGING/INTERACTIVE DEVELOPMENT 1542 | 1543 | def makePrintReturner(pre="", post="" ,out=None): 1544 | r"""Creates functions that print out their argument, (between optional 1545 | `pre` and `post` strings) and return it unmodified. This is usefull for 1546 | debugging e.g. parts of expressions, without having to modify the behavior 1547 | of the program. 1548 | 1549 | Example: 1550 | 1551 | >>> makePrintReturner(pre="The value is:", post="[returning]")(3) 1552 | The value is: 3 [returning] 1553 | 3 1554 | >>> 1555 | """ 1556 | def printReturner(arg): 1557 | myArgs = [pre, arg, post] 1558 | prin(*myArgs, **{'out':out}) 1559 | return arg 1560 | return printReturner 1561 | 1562 | 1563 | 1564 | #_ : Tracing like facilities 1565 | 1566 | # FIXME This is not recursive (and really shouldn't be I guess, at least not 1567 | # without extra argument) but returning a function wrapper here is really not 1568 | # enough, it should be a class (callable hack is of limited workability) 1569 | class ShowWrap(object): 1570 | """A simple way to trace modules or objects 1571 | Example:: 1572 | 1573 | >> import os 1574 | >> os = ShowWrap(os, '->') 1575 | >> os.getenv('SOME_STRANGE_NAME', 'NOT_FOUND') 1576 | -> os.getenv('SOME_STRANGE_NAME','NOT_FOUND') 1577 | 'NOT_FOUND' 1578 | """ 1579 | def __init__(self,module, prefix='-> '): 1580 | self.module=module 1581 | self.prefix=prefix 1582 | def __getattribute__(self, name): 1583 | myDict = super(ShowWrap, self).__getattribute__('__dict__') 1584 | realThing = getattr(myDict['module'], name) 1585 | if not callable(realThing): 1586 | return realThing 1587 | else: 1588 | def show(*x, **y): 1589 | print >>sys.stderr, "%s%s.%s(%s)" % ( 1590 | myDict['prefix'], 1591 | getattr(myDict['module'], '__name__', '?'), 1592 | name, 1593 | ",".join(map(repr,x) + ["%s=%r" % (n, v) for n,v in y.items()])) 1594 | return realThing(*x, **y) 1595 | return show 1596 | 1597 | 1598 | # XXX a nice example of python's expressiveness, but how useful is it? 1599 | def asVerboseContainer(cont, onGet=None, onSet=None, onDel=None): 1600 | """Returns a 'verbose' version of container instance `cont`, that will 1601 | execute `onGet`, `onSet` and `onDel` (if not `None`) every time 1602 | __getitem__, __setitem__ and __delitem__ are called, passing `self`, `key` 1603 | (and `value` in the case of set). E.g: 1604 | 1605 | >>> l = [1,2,3] 1606 | >>> l = asVerboseContainer(l, 1607 | ... onGet=lambda s,k:k==2 and prin('Got two:', k), 1608 | ... onSet=lambda s,k,v:k == v and prin('k == v:', k, v), 1609 | ... onDel=lambda s,k:k == 1 and prin('Deleting one:', k)) 1610 | >>> l 1611 | [1, 2, 3] 1612 | >>> l[1] 1613 | 2 1614 | >>> l[2] 1615 | Got two: 2 1616 | 3 1617 | >>> l[2] = 22 1618 | >>> l[2] = 2 1619 | k == v: 2 2 1620 | >>> del l[2] 1621 | >>> del l[1] 1622 | Deleting one: 1 1623 | 1624 | """ 1625 | class VerboseContainer(type(cont)): 1626 | if onGet: 1627 | def __getitem__(self, key): 1628 | onGet(self, key) 1629 | return super(VerboseContainer, self).__getitem__(key) 1630 | if onSet: 1631 | def __setitem__(self, key, value): 1632 | onSet(self, key, value) 1633 | return super(VerboseContainer, self).__setitem__(key, value) 1634 | if onDel: 1635 | def __delitem__(self, key): 1636 | onDel(self, key) 1637 | return super(VerboseContainer, self).__delitem__(key) 1638 | return VerboseContainer(cont) 1639 | 1640 | 1641 | 1642 | #_. CONVENIENCE 1643 | 1644 | #FIXME untested change 1645 | #FIXME add checks for valid names 1646 | class ezstruct(object): 1647 | r""" 1648 | Examples: 1649 | 1650 | >>> brian = ezstruct(name="Brian", age=30) 1651 | >>> brian.name 1652 | 'Brian' 1653 | >>> brian.age 1654 | 30 1655 | >>> brian.life = "short" 1656 | >>> brian 1657 | ezstruct(age=30, life='short', name='Brian') 1658 | >>> del brian.life 1659 | >>> brian == ezstruct(name="Brian", age=30) 1660 | True 1661 | >>> brian != ezstruct(name="Jesus", age=30) 1662 | True 1663 | >>> len(brian) 1664 | 2 1665 | 1666 | Call the object to create a clone: 1667 | 1668 | >>> brian() is not brian and brian(name="Jesus") == ezstruct(name="Jesus", age=30) 1669 | True 1670 | 1671 | Conversion to/from dict: 1672 | 1673 | >>> ezstruct(**dict(brian)) == brian 1674 | True 1675 | 1676 | Evil Stuff: 1677 | 1678 | >>> brian['name', 'age'] 1679 | ('Brian', 30) 1680 | >>> brian['name', 'age'] = 'BRIAN', 'XXX' 1681 | >>> brian 1682 | ezstruct(age='XXX', name='BRIAN') 1683 | """ 1684 | def __init__(self,**kwargs): 1685 | self.__dict__.update(kwargs) 1686 | def __call__(self, **kwargs): 1687 | import copy 1688 | res = copy.copy(self) 1689 | res.__init__(**kwargs) 1690 | return res 1691 | def __eq__(self, other): 1692 | return self.__dict__ == other.__dict__ 1693 | def __ne__(self, other): 1694 | return not self.__eq__(other) 1695 | def __len__(self): 1696 | return len([k for k in self.__dict__.iterkeys() 1697 | if not (k.startswith('__') or k.endswith('__'))]) 1698 | # FIXME rather perverse 1699 | def __getitem__(self, nameOrNames): 1700 | if isString(nameOrNames): 1701 | return self.__dict__[nameOrNames] 1702 | else: 1703 | return tuple([self.__dict__[n] for n in nameOrNames]) 1704 | # FIXME rather perverse 1705 | def __setitem__(self, nameOrNames, valueOrValues): 1706 | if isString(nameOrNames): 1707 | self.__dict__[nameOrNames] = valueOrValues 1708 | else: 1709 | for (n,v) in zip(nameOrNames, valueOrValues): 1710 | self.__dict__[n] = v 1711 | def __contains__(self, key): 1712 | return key in self.__dict__ and not (key.startswith('__') or key.endswith('__')) 1713 | def __iter__(self): 1714 | for (k,v) in self.__dict__.iteritems(): 1715 | if not (k.startswith('__') or k.endswith('__')): 1716 | yield k,v 1717 | def __repr__(self): 1718 | return mkRepr(self, **vars(self)) 1719 | 1720 | #XXX should we really sort? This makes it impossible for the user to 1721 | #customise sorting by specifying a priority dict, but well... 1722 | #XXX this should do folding... 1723 | 1724 | def mkRepr(instance, *argls, **kwargs): 1725 | r"""Convinience function to implement ``__repr__``. `kwargs` values are 1726 | ``repr`` ed. Special behavior for ``instance=None``: just the 1727 | arguments are formatted. 1728 | 1729 | Example: 1730 | 1731 | >>> class Thing: 1732 | ... def __init__(self, color, shape, taste=None): 1733 | ... self.color, self.shape, self.taste = color, shape, taste 1734 | ... def __repr__(self): 1735 | ... return mkRepr(self, self.color, self.shape, taste=self.taste) 1736 | ... 1737 | >>> maggot = Thing('white', 'cylindrical', 'chicken') 1738 | >>> maggot 1739 | Thing('white', 'cylindrical', taste='chicken') 1740 | >>> Thing('Color # 132942430-214809804-412430988081-241234', 'unkown', taste=maggot) 1741 | Thing('Color # 132942430-214809804-412430988081-241234', 1742 | 'unkown', 1743 | taste=Thing('white', 'cylindrical', taste='chicken')) 1744 | """ 1745 | width=79 1746 | maxIndent=15 1747 | minIndent=2 1748 | args = (map(repr, argls) + ["%s=%r" % (k, v) 1749 | for (k,v) in sorted(kwargs.items())]) or [""] 1750 | if instance is not None: 1751 | start = "%s(" % instance.__class__.__name__ 1752 | args[-1] += ")" 1753 | else: 1754 | start = "" 1755 | if len(start) <= maxIndent and len(start) + len(args[0]) <= width and \ 1756 | max(map(len,args)) <= width: # XXX mag of last condition bit arbitrary 1757 | indent = len(start) 1758 | args[0] = start + args[0] 1759 | if sum(map(len, args)) + 2*(len(args) - 1) <= width: 1760 | return ", ".join(args) 1761 | else: 1762 | indent = minIndent 1763 | args[0] = start + "\n" + " " * indent + args[0] 1764 | return (",\n" + " " * indent).join(args) 1765 | 1766 | __test__['mkRepr'] = r""" 1767 | >>> ezstruct() 1768 | ezstruct() 1769 | >>> ezstruct(a=1,b=2,c=3)(_='foooooooooooooooooooooooooooooooooooooooooooooooo000000000000000000') 1770 | ezstruct( 1771 | _='foooooooooooooooooooooooooooooooooooooooooooooooo000000000000000000', 1772 | a=1, 1773 | b=2, 1774 | c=3) 1775 | >>> ezstruct(a=1,b=2,c=3)(d='foooooooooooooooooooooooooooooooooooooooooooooooo0000000000000000') 1776 | ezstruct(a=1, 1777 | b=2, 1778 | c=3, 1779 | d='foooooooooooooooooooooooooooooooooooooooooooooooo0000000000000000') 1780 | >>> ezstruct(a=1,b=2,c=3) 1781 | ezstruct(a=1, b=2, c=3) 1782 | """ 1783 | 1784 | 1785 | #_. EXPERIMENTAL 1786 | 1787 | def d2attrs(*args, **kwargs): 1788 | """Utility function to remove ``**kwargs`` parsing boiler-plate in 1789 | ``__init__``: 1790 | 1791 | >>> kwargs = dict(name='Bill', age=51, income=1e7) 1792 | >>> self = ezstruct(); d2attrs(kwargs, self, 'income', 'name'); self 1793 | ezstruct(income=10000000.0, name='Bill') 1794 | >>> self = ezstruct(); d2attrs(kwargs, self, 'income', age=0, bloodType='A'); self 1795 | ezstruct(age=51, bloodType='A', income=10000000.0) 1796 | 1797 | To set all keys from ``kwargs`` use: 1798 | 1799 | >>> self = ezstruct(); d2attrs(kwargs, self, 'all!'); self 1800 | ezstruct(age=51, income=10000000.0, name='Bill') 1801 | """ 1802 | (d, self), args = args[:2], args[2:] 1803 | if args[0] == 'all!': 1804 | assert len(args) == 1 1805 | for k in d: setattr(self, k, d[k]) 1806 | else: 1807 | if len(args) != len(set(args)) or set(kwargs) & set(args): 1808 | raise ValueError('Duplicate keys: %s' % 1809 | list(notUnique(args)) + list(set(kwargs) & set(args))) 1810 | for k in args: 1811 | if k in kwargs: raise ValueError('%s specified twice' % k) 1812 | setattr(self, k, d[k]) 1813 | for dk in kwargs: 1814 | setattr(self, dk, d.get(dk, kwargs[dk])) 1815 | 1816 | def ipairwise(fun, v): 1817 | for x0,x1 in window(v): yield fun(x0,x1) 1818 | 1819 | def pairwise(fun, v): 1820 | """ 1821 | >>> pairwise(operator.sub, [4,3,2,1,-10]) 1822 | [1, 1, 1, 11] 1823 | >>> import numpy 1824 | >>> pairwise(numpy.subtract, numpy.array([4,3,2,1,-10])) 1825 | array([ 1, 1, 1, 11]) 1826 | """ 1827 | if not hasattr(v, 'shape'): 1828 | return list(ipairwise(fun,v)) 1829 | else: 1830 | return fun(v[:-1],v[1:]) 1831 | 1832 | 1833 | 1834 | def ignoreErrors(func, *args, **kwargs): 1835 | """ 1836 | >>> ignoreErrors(int, '3') 1837 | 3 1838 | >>> ignoreErrors(int, 'three') 1839 | 1840 | """ 1841 | try: return func(*args, **kwargs) 1842 | except Exception: return None 1843 | 1844 | 1845 | 1846 | def argmax(iterable, key=None, both=False): 1847 | """ 1848 | >>> argmax([4,2,-5]) 1849 | 0 1850 | >>> argmax([4,2,-5], key=abs) 1851 | 2 1852 | >>> argmax([4,2,-5], key=abs, both=True) 1853 | (2, 5) 1854 | """ 1855 | if key is not None: 1856 | it = imap(key, iterable) 1857 | else: 1858 | it = iter(iterable) 1859 | score, argmax = reduce(max, izip(it, count())) 1860 | if both: 1861 | return argmax, score 1862 | return argmax 1863 | 1864 | def argmin(iterable, key=None, both=False): 1865 | """See `argmax`. 1866 | """ 1867 | if key is not None: 1868 | it = imap(key, iterable) 1869 | else: 1870 | it = iter(iterable) 1871 | score, argmin = reduce(min, izip(it, count())) 1872 | if both: 1873 | return argmin, score 1874 | return argmin 1875 | 1876 | 1877 | 1878 | 1879 | # FIXME this won't work for complexes but then you can't use them as ints 1880 | # anyway 1881 | def isInt(num): 1882 | """Returns true if `num` is (sort of) an integer. 1883 | >>> isInt(3) == isInt(3.0) == 1 1884 | True 1885 | >>> isInt(3.2) 1886 | False 1887 | >>> import numpy 1888 | >>> isInt(numpy.array(1)) 1889 | True 1890 | >>> isInt(numpy.array([1])) 1891 | False 1892 | """ 1893 | try: 1894 | len(num) # FIXME fails for Numeric but Numeric is obsolete 1895 | except: 1896 | try: 1897 | return (num - math.floor(num) == 0) == True 1898 | except: return False 1899 | else: return False 1900 | 1901 | def ordinalStr(num): 1902 | """ Convert `num` to english ordinal. 1903 | >>> map(ordinalStr, range(6)) 1904 | ['0th', '1st', '2nd', '3rd', '4th', '5th'] 1905 | """ 1906 | assert isInt(num) 1907 | return "%d%s" % (num, {1:'st', 2:'nd', 3:'rd'}.get(int(num), 'th')) 1908 | 1909 | 1910 | # The stuff here is work in progress and may be dropped or modified 1911 | # incompatibly without further notice 1912 | 1913 | 1914 | 1915 | 1916 | # rename to mapcan 1917 | def mapConcat(func, *iterables): 1918 | """Similar to `map` but the instead of collecting the return values of 1919 | `func` in a list, the items of each return value are instaed collected 1920 | (so `func` must return an iterable type). 1921 | 1922 | Examples: 1923 | 1924 | >>> mapConcat(lambda x:[x], [1,2,3]) 1925 | [1, 2, 3] 1926 | >>> mapConcat(lambda x: [x,str(x)], [1,2,3]) 1927 | [1, '1', 2, '2', 3, '3'] 1928 | """ 1929 | return [e for l in imap(func, *iterables) for e in l] 1930 | 1931 | 1932 | 1933 | 1934 | 1935 | def unfold(seed, by, last = __unique): 1936 | """ 1937 | >>> list(unfold(1234, lambda x: divmod(x,10)))[::-1] 1938 | [1, 2, 3, 4] 1939 | >>> sum(imap(operator.mul,unfold(1234, lambda x:divmod(x,10)), iterate(lambda x:x*10)(1))) 1940 | 1234 1941 | >>> g = unfold(1234, lambda x:divmod(x,10)) 1942 | >>> reduce((lambda (total,pow),digit:(total+pow*digit, 10*pow)), g, (0,1)) 1943 | (1234, 10000) 1944 | """ 1945 | 1946 | while True: 1947 | seed, val = by(seed); 1948 | if last == seed: return 1949 | last = seed; yield val 1950 | 1951 | def reduceR(f, sequence, initial=__unique): 1952 | """*R*ight reduce. 1953 | >>> reduceR(lambda x,y:x/y, [1.,2.,3.,4]) == 1./(2./(3./4.)) == (1./2.)*(3./4.) 1954 | True 1955 | >>> reduceR(lambda x,y:x-y, iter([1,2,3]),4) == 1-(2-(3-4)) == (1-2)+(3-4) 1956 | True 1957 | """ 1958 | try: rev = reversed(sequence) 1959 | except TypeError: rev = reversed(list(sequence)) 1960 | if initial is __unique: return reduce(lambda x,y:f(y,x), rev) 1961 | else: return reduce(lambda x,y:f(y,x), rev, initial) 1962 | 1963 | # FIXME add doc kwarg? 1964 | def compose(*funcs): 1965 | """Compose `funcs` to a single function. 1966 | 1967 | >>> compose(operator.abs, operator.add)(-2,-3) 1968 | 5 1969 | >>> compose()('nada') 1970 | 'nada' 1971 | >>> compose(sorted, set, partial(filter, None))(range(3)[::-1]*2) 1972 | [1, 2] 1973 | """ 1974 | # slightly optimized for most common cases and hence verbose 1975 | if len(funcs) == 2: f0,f1=funcs; return lambda *a,**kw: f0(f1(*a,**kw)) 1976 | elif len(funcs) == 3: f0,f1,f2=funcs; return lambda *a,**kw: f0(f1(f2(*a,**kw))) 1977 | elif len(funcs) == 0: return lambda x:x # XXX single kwarg 1978 | elif len(funcs) == 1: return funcs[0] 1979 | else: 1980 | def composed(*args,**kwargs): 1981 | y = funcs[-1](*args,**kwargs) 1982 | for f in funcs[:0:-1]: y = f(y) 1983 | return y 1984 | return composed 1985 | 1986 | 1987 | def romanNumeral(n): 1988 | """ 1989 | >>> romanNumeral(13) 1990 | 'XIII' 1991 | >>> romanNumeral(2944) 1992 | 'MMCMXLIV' 1993 | """ 1994 | if 0 > n > 4000: raise ValueError('``n`` must lie between 1 and 3999: %d' % n) 1995 | roman = 'I IV V IX X XL L XC C CD D CM M'.split() 1996 | arabic = [1, 4, 5, 9, 10, 40, 50, 90, 100, 400, 500, 900, 1000] 1997 | res = [] 1998 | while n>0: 1999 | pos = bisect.bisect_right(arabic, n)-1 2000 | fit = n//arabic[pos] 2001 | res.append(roman[pos]*fit); n -= fit * arabic[pos] 2002 | return "".join(res) 2003 | 2004 | 2005 | 2006 | 2007 | def _docTest(): 2008 | import doctest, awmstools 2009 | return doctest.testmod(awmstools) 2010 | 2011 | def first(n, it, constructor=list): 2012 | """ 2013 | >>> first(3,iter([1,2,3,4])) 2014 | [1, 2, 3] 2015 | >>> first(3,iter([1,2,3,4]), iter) #doctest: +ELLIPSIS 2016 | 2017 | >>> first(3,iter([1,2,3,4]), tuple) 2018 | (1, 2, 3) 2019 | """ 2020 | return constructor(itertools.islice(it,n)) 2021 | def drop(n, it, constructor=list): 2022 | """ 2023 | >>> first(10,drop(10,xrange(sys.maxint),iter)) 2024 | [10, 11, 12, 13, 14, 15, 16, 17, 18, 19] 2025 | """ 2026 | return constructor(itertools.islice(it,n,None)) 2027 | 2028 | if __name__ in ("__main__", "__IPYTHON_main__"): 2029 | _docTest() 2030 | import unittest 2031 | class ProcessTest(unittest.TestCase): 2032 | def testProcesses(self): 2033 | assert silentlyRunProcess('ls') == 0 2034 | assert silentlyRunProcess('ls', '/tmp') == 0 2035 | assert silentlyRunProcess('i-DoNOT___EXIST') 2036 | assert silentlyRunProcess('latex') 2037 | assert readProcess('ls', '-d', '/tmp') == ('/tmp\n', '', 0) 2038 | out = readProcess('ls', '-d', '/i-DoNOT___.EXIST') 2039 | assert out[0] == '' 2040 | assert re.match( 'ls:.*/i-DoNOT___.EXIST:.*No such file or directory.*', out[1]) 2041 | assert out[2] == 2, out[2] 2042 | suite = unittest.TestSuite(map(unittest.makeSuite, 2043 | (ProcessTest, 2044 | ))) 2045 | #suite.debug() 2046 | unittest.TextTestRunner(verbosity=2).run(suite) 2047 | 2048 | 2049 | # don't litter the namespace on ``from awmstools import *`` 2050 | __all__ = [] 2051 | me = sys.modules[__name__] 2052 | for n in dir(me): 2053 | if n != me and not isinstance(getattr(me,n), type(me)): 2054 | __all__.append(n) 2055 | del me, n 2056 | 2057 | --------------------------------------------------------------------------------