├── pymetabiosis ├── test │ ├── __init__.py │ ├── test_module.py │ ├── test_numpy_convert.py │ └── test_wrapper.py ├── __init__.py ├── utils.py ├── module.py ├── numpy_convert.py ├── bindings.py └── wrapper.py ├── requirements.txt ├── .gitignore ├── .travis.yml ├── setup.py ├── README.rst └── LICENSE /pymetabiosis/test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cffi 2 | pytest 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | __pycache__ 3 | *.egg-info 4 | .coverage 5 | htmlcov 6 | _cffi*.so 7 | build 8 | dist 9 | .cache 10 | tags 11 | -------------------------------------------------------------------------------- /pymetabiosis/__init__.py: -------------------------------------------------------------------------------- 1 | from pymetabiosis.module import import_module 2 | from pymetabiosis.wrapper import init_cpy_to_pypy_converters 3 | 4 | init_cpy_to_pypy_converters() 5 | -------------------------------------------------------------------------------- /pymetabiosis/utils.py: -------------------------------------------------------------------------------- 1 | from pymetabiosis import import_module 2 | 3 | builtin = import_module("__builtin__") 4 | 5 | def activate_virtualenv(path): 6 | builtin.execfile(path, {"__file__" : path}) 7 | -------------------------------------------------------------------------------- /pymetabiosis/module.py: -------------------------------------------------------------------------------- 1 | from pymetabiosis.bindings import ffi, lib 2 | from pymetabiosis.wrapper import MetabiosisWrapper 3 | 4 | def import_module(name, noconvert=False): 5 | module_object = lib.PyImport_ImportModule(name) 6 | return MetabiosisWrapper(ffi.gc(module_object, lib.Py_DECREF), noconvert) 7 | -------------------------------------------------------------------------------- /pymetabiosis/test/test_module.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pymetabiosis.module import import_module 3 | 4 | def test_import_sqlite(): 5 | module = import_module("sqlite3") 6 | module_str = str(module) 7 | assert module_str.startswith("") 15 | 16 | def test_setattr_on_module(): 17 | pickle = import_module("pickle") 18 | assert isinstance(pickle, MetabiosisWrapper) 19 | pickle.a = 42 20 | assert pickle.a == 42 21 | 22 | def test_call_function(): 23 | sqlite = import_module("sqlite3") 24 | connection = sqlite.connect(":memory:") 25 | assert repr(connection).startswith("" 56 | 57 | def test_convert_return_value(): 58 | builtin = import_module("__builtin__") 59 | operator = import_module("operator") 60 | 61 | assert builtin.int(32) == 32 62 | assert builtin.float(3.123) == 3.123 63 | 64 | for s in ['a string']: # TODO 'a string \00yep']: 65 | assert builtin.str(s) == s 66 | 67 | u = u"some буквы are странные" 68 | assert builtin.unicode(u) == u 69 | 70 | t = (1, (2.3,)) 71 | assert builtin.tuple(t) == t 72 | 73 | d = {'a': 'b', 1: 2} 74 | assert builtin.dict(d) == d 75 | 76 | lst = ['a', 1, [2]] 77 | assert builtin.list(lst) == lst 78 | 79 | assert builtin.bool(True) is True 80 | assert builtin.bool(False) is False 81 | 82 | assert builtin.bool(None) is False 83 | assert operator.eq(None, None) is True 84 | assert operator.eq(None, False) is False 85 | 86 | def test_getitem_setitem_delitem(): 87 | builtin = import_module("__builtin__", noconvert=True) 88 | 89 | d = builtin.dict({1: 'foo', (1, 'a'): 'zoo'}) 90 | with pytest.raises(KeyError): 91 | d[2] 92 | assert pypy_convert(d[1]._cpyobj) == 'foo' 93 | assert pypy_convert(d[(1, 'a')]._cpyobj) == 'zoo' 94 | 95 | key, lst = (1, 2), ['a', 'b'] 96 | d[key] = lst 97 | assert pypy_convert(d[key]._cpyobj) == lst 98 | 99 | with pytest.raises(TypeError): 100 | d[[1, 2]] = 0 101 | 102 | del d[1] 103 | with pytest.raises(KeyError): 104 | d[1] 105 | 106 | with pytest.raises(KeyError): 107 | del d[2] 108 | 109 | def test_getattr_convert(): 110 | builtin = import_module("__builtin__", noconvert=True) 111 | s = builtin.slice(10, 11) 112 | s.__dict__['noconvert'] = False 113 | assert s.start == 10 114 | 115 | def test_str_repr_dir(): 116 | builtin = import_module("__builtin__", noconvert=True) 117 | 118 | assert str(builtin.None) == 'None' 119 | assert str(builtin.str('a')) == 'a' 120 | assert repr(builtin.str('a')) == "'a'" 121 | 122 | assert set(['rjust', 'rpartition', 'rstrip', '__le__'])\ 123 | .issubset(dir(builtin.str('a'))) 124 | 125 | def test_len(): 126 | builtin = import_module("__builtin__", noconvert=True) 127 | lst = builtin.list([1, 'a']) 128 | assert len(lst) == 2 129 | assert len(builtin.list()) == 0 130 | assert len(builtin.str('abc')) == 3 131 | 132 | with pytest.raises(TypeError): 133 | len(builtin.iter([1])) 134 | 135 | def test_bool(): 136 | builtin = import_module("__builtin__", noconvert=True) 137 | true = builtin.bool(True) 138 | false = builtin.bool(False) 139 | assert bool(true) is True 140 | assert bool(false) is False 141 | 142 | def test_type(): 143 | builtin = import_module("__builtin__") 144 | assert builtin.type(10) is int 145 | for _type in [float, int, bool, str, unicode]: 146 | assert builtin.str(_type) == repr(_type) 147 | 148 | def test_slice(): 149 | builtin = import_module("__builtin__", noconvert=True) 150 | lst = builtin.list(list(xrange(10))) 151 | assert _pypy_convert_list(lst) == list(xrange(10)) 152 | assert _pypy_convert_list(lst[-1:]) == [9] 153 | assert _pypy_convert_list(lst[:2]) == [0, 1] 154 | assert _pypy_convert_list(lst[-9:3]) == [1, 2] 155 | 156 | def test_invert(): 157 | builtin = import_module("__builtin__", noconvert=True) 158 | n = builtin.int(10) 159 | assert isinstance(n, MetabiosisWrapper) 160 | assert pypy_convert((~n)._cpyobj) == ~10 161 | 162 | def test_iter(): 163 | builtin = import_module("__builtin__", noconvert=True) 164 | assert _pypy_convert_list(builtin.list([1, 'a'])) == [1, 'a'] 165 | assert _pypy_convert_list(builtin.iter(['a'])) == ['a'] 166 | with pytest.raises(TypeError): 167 | builtin.iter(1) 168 | 169 | def _pypy_convert_list(lst): 170 | return [pypy_convert(x._cpyobj) for x in lst] 171 | 172 | def test_exceptions(): 173 | builtin = import_module("__builtin__") 174 | 175 | with pytest.raises(AttributeError): 176 | builtin.foo 177 | 178 | with pytest.raises(ValueError): # TODO UnicodeDecodeError 179 | builtin.unicode('\124\323') 180 | 181 | def test_no_convert(): 182 | operator = import_module("operator") 183 | functools = import_module("functools") 184 | builtin = import_module("__builtin__", noconvert=True) 185 | 186 | lst = builtin.list() 187 | 188 | part = functools.partial(operator.iadd, lst) 189 | part([1, 2, 3]) 190 | 191 | assert pypy_convert(lst._cpyobj) == [1, 2, 3] 192 | 193 | def test_applevel(): 194 | fn = applevel(''' 195 | def f(): 196 | return 3 197 | return f 198 | ''', noconvert=False) 199 | assert fn() == 3 200 | 201 | class Point(object): 202 | _pymetabiosis_wrap = True 203 | def __init__(self, x, y): 204 | self.x = x 205 | self.y = y 206 | 207 | def norm(self): 208 | return math.sqrt(self.x ** 2 + self.y ** 2) 209 | 210 | class DictSubclass(dict): 211 | _pymetabiosis_wrap = True 212 | 213 | def test_opaque_objects(): 214 | builtin = import_module("__builtin__") 215 | builtin_noconvert = import_module("__builtin__", noconvert=True) 216 | p1, p2 = Point(1.0, 2.0), Point(3.0, -1.0) 217 | d = DictSubclass() 218 | 219 | lst = builtin.list([p1, p2, d]) 220 | assert lst == [p1, p2, d] 221 | 222 | lst_cpy = builtin_noconvert.list([p1, p2, d]) 223 | assert pypy_convert(lst_cpy[0]._cpyobj) == p1 224 | assert pypy_convert(lst_cpy[1]._cpyobj) == p2 225 | assert pypy_convert(lst_cpy[2]._cpyobj) == d 226 | lst_cpy.reverse() 227 | assert pypy_convert(lst_cpy[0]._cpyobj) == d 228 | assert pypy_convert(lst_cpy[1]._cpyobj) == p2 229 | assert pypy_convert(lst_cpy[2]._cpyobj) == p1 230 | 231 | def test_isinstance(): 232 | builtin = import_module("__builtin__", noconvert=True) 233 | assert not isinstance(builtin.int(10), int) 234 | assert isinstance(builtin.int(10), builtin.int) 235 | assert isinstance(builtin.int(10), MetabiosisWrapper) 236 | 237 | def test_issubclass(): 238 | builtin = import_module("__builtin__", noconvert=True) 239 | types = import_module("types", noconvert=True) 240 | assert issubclass(builtin.int, builtin.int) 241 | assert issubclass(builtin.int, types.IntType) 242 | 243 | def test_callbacks_simple(): 244 | builtin = import_module("__builtin__", noconvert=True) 245 | lst = builtin.list([1, 2, 3, 4, 5, 6]) 246 | lst.sort(key=lambda x: x % 3) 247 | assert _pypy_convert_list(lst) == [3, 6, 1, 4, 2, 5] 248 | 249 | def test_callbacks_on_wrappers(): 250 | builtin = import_module("__builtin__", noconvert=True) 251 | p1, p2, p3, p4 = points = [ 252 | Point(0, 0), 253 | Point(0, 1), 254 | Point(1, 2), 255 | Point(3, 4)] 256 | lst = builtin.list([p3, p2, p1, p4]) 257 | lst.sort(key=lambda x: x.norm()) 258 | assert _pypy_convert_list(lst) == points 259 | 260 | # method callbacks 261 | class Norm(object): 262 | def __init__(self, n): 263 | self.n = n 264 | def norm(self, point): 265 | return math.pow(point.norm()**2, 1.0 / self.n) 266 | norm = Norm(2) 267 | lst.reverse() 268 | lst.sort(key=norm.norm) 269 | assert _pypy_convert_list(lst) == points 270 | 271 | # dict.get as a callback 272 | d = dict((p, p.norm()) for p in points) 273 | lst.reverse() 274 | lst.sort(key=d.get) 275 | 276 | 277 | def test_callbacks_exceptions(): 278 | builtin = import_module("__builtin__") 279 | d = {1: 2} 280 | fn = lambda x: d[x] 281 | assert builtin.apply(fn, (1,)) == 2 282 | # exception in callback 283 | with pytest.raises(KeyError): 284 | builtin.apply(fn, (2,)) 285 | # exception in converting result 286 | try: 287 | builtin.apply(lambda : object()) 288 | except SystemError: 289 | assert False 290 | except Exception: 291 | pass 292 | 293 | 294 | @pytest.mark.parametrize('op,input', [ 295 | (operator.abs, -1), 296 | (operator.index, 2), 297 | (operator.invert, 2), 298 | (operator.neg, 2), 299 | (operator.not_, 0), 300 | (operator.pos, -2), 301 | (operator.truth, 0), 302 | ]) 303 | def test_unaryop(op, input): 304 | cinput = convert(input) 305 | wrapper = MetabiosisWrapper(cinput) 306 | result = op(wrapper) 307 | if isinstance(result, MetabiosisWrapper): 308 | result = pypy_convert(result) 309 | expected = op(input) 310 | assert result == expected 311 | 312 | 313 | 314 | @pytest.mark.parametrize('op,arg1,arg2', [ 315 | (operator.add, 1, 2), 316 | (operator.and_, 2, 3), 317 | (operator.div, 15, 3), 318 | (operator.eq, 2, 3), 319 | (operator.floordiv, 14, 3), 320 | (operator.ge, 3, 4), 321 | (operator.gt, 3, 4), 322 | (operator.le, 3, 4), 323 | (operator.lshift, 3, 4), 324 | (operator.lt, 3, 4), 325 | (operator.mod, 3, 4), 326 | (operator.ne, 3, 4), 327 | (operator.or_, 3, 4), 328 | (operator.pow, 3, 4), 329 | (operator.rshift, 3, 4), 330 | (operator.sub, 3, 4), 331 | (operator.truediv, 3, 4), 332 | (operator.xor, 3, 4), 333 | ]) 334 | def test_binaryop(op, arg1, arg2): 335 | carg1 = convert(arg1) 336 | wrapper1 = MetabiosisWrapper(carg1) 337 | carg2 = convert(arg2) 338 | wrapper2 = MetabiosisWrapper(carg2) 339 | result = op(wrapper1, wrapper2) 340 | if isinstance(result, MetabiosisWrapper): 341 | result = pypy_convert(result) 342 | expected = op(arg1, arg2) 343 | assert result == expected 344 | 345 | 346 | 347 | 348 | -------------------------------------------------------------------------------- /pymetabiosis/bindings.py: -------------------------------------------------------------------------------- 1 | import atexit 2 | from cffi import FFI 3 | import os 4 | from subprocess import check_output 5 | import sys 6 | 7 | 8 | def _get_python(): 9 | """Find a suitable Python to embed. 10 | """ 11 | python_embed_prefix = os.environ.get('PYTHON_EMBED', '') 12 | if not python_embed_prefix: 13 | python = check_output( 14 | ["python", "-c", "import sys;print sys.executable"] 15 | ).strip() 16 | else: 17 | if sys.platform == 'win32': 18 | python = os.path.join(python_embed_prefix, 'python.exe') 19 | else: 20 | python = os.path.join(python_embed_prefix, 'bin', 'python') 21 | advice = "Please set your PYTHON_EMBED env var to point to your Python "\ 22 | "installation." 23 | if not os.path.exists(python): 24 | raise RuntimeError("Can not find python at %s. %s" % (python, advice)) 25 | is_pypy = check_output( 26 | [python, "-c", "import sys;print hasattr(sys, 'pypy_version_info')"] 27 | ).strip() 28 | if is_pypy.lower() == "true": 29 | msg = "%s is a PyPy interpreter and not Python. %s" % (python, advice) 30 | raise RuntimeError(msg) 31 | return python 32 | 33 | PYTHON = _get_python() 34 | 35 | def _get_include_dirs(): 36 | return [check_output( 37 | [PYTHON, "-c", 38 | "from distutils import sysconfig;" 39 | "print sysconfig.get_python_inc()"] 40 | ).strip()] 41 | 42 | 43 | def _get_library_dirs(): 44 | return [check_output( 45 | [PYTHON, "-c", 46 | "from distutils import sysconfig;" 47 | "print sysconfig.get_config_var('LIBDIR')"] 48 | ).strip()] 49 | 50 | LIBDIRS = _get_library_dirs() 51 | 52 | def _get_extra_link_args(): 53 | args = [] 54 | if sys.platform == 'darwin': 55 | libdir = LIBDIRS[0] 56 | args.append("-Wl,-rpath,%s"%os.path.dirname(libdir)) 57 | elif sys.platform.startswith("linux"): 58 | libdir = LIBDIRS[0] 59 | args.append("-Wl,-rpath,%s"%libdir) 60 | 61 | return args 62 | 63 | 64 | ffi = FFI() 65 | 66 | ffi.cdef(""" 67 | typedef ... PyTypeObject; 68 | 69 | typedef size_t Py_ssize_t; 70 | 71 | typedef struct { 72 | PyTypeObject* ob_type; 73 | Py_ssize_t ob_refcnt; 74 | ...; 75 | } PyObject; 76 | 77 | void Py_Initialize(); 78 | void Py_Finalize(); 79 | 80 | void Py_SetProgramName(char *name); 81 | int PyRun_SimpleString(const char *command); 82 | 83 | void Py_INCREF(PyObject *o); 84 | void Py_XINCREF(PyObject *o); 85 | void Py_DECREF(PyObject *o); 86 | void Py_XDECREF(PyObject *o); 87 | 88 | const int Py_file_input; 89 | PyObject* Py_CompileString(const char *str, const char *filename, int start); 90 | PyObject* PyEval_GetBuiltins(); 91 | PyObject* PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals); 92 | 93 | // Importing: https://docs.python.org/2/c-api/import.html 94 | PyObject* PyImport_ImportModule(const char *name); 95 | 96 | // Exceptions: https://docs.python.org/2/c-api/exceptions.html 97 | PyObject* PyErr_Occurred(); 98 | void PyErr_Print(); 99 | void PyErr_Clear(); 100 | int PyErr_ExceptionMatches(PyObject *exc); 101 | void PyErr_SetString(PyObject *type, const char *message); 102 | PyObject* const PyExc_BaseException; 103 | PyObject* const PyExc_Exception; 104 | PyObject* const PyExc_StandardError; 105 | PyObject* const PyExc_ArithmeticError; 106 | PyObject* const PyExc_LookupError; 107 | PyObject* const PyExc_AssertionError; 108 | PyObject* const PyExc_AttributeError; 109 | PyObject* const PyExc_EOFError; 110 | PyObject* const PyExc_EnvironmentError; 111 | PyObject* const PyExc_FloatingPointError; 112 | PyObject* const PyExc_IOError; 113 | PyObject* const PyExc_ImportError; 114 | PyObject* const PyExc_IndexError; 115 | PyObject* const PyExc_KeyError; 116 | PyObject* const PyExc_KeyboardInterrupt; 117 | PyObject* const PyExc_MemoryError; 118 | PyObject* const PyExc_NameError; 119 | PyObject* const PyExc_NotImplementedError; 120 | PyObject* const PyExc_OSError; 121 | PyObject* const PyExc_OverflowError; 122 | PyObject* const PyExc_ReferenceError; 123 | PyObject* const PyExc_RuntimeError; 124 | PyObject* const PyExc_SyntaxError; 125 | PyObject* const PyExc_SystemError; 126 | PyObject* const PyExc_SystemExit; 127 | PyObject* const PyExc_TypeError; 128 | PyObject* const PyExc_ValueError; 129 | PyObject* const PyExc_ZeroDivisionError; 130 | 131 | // Object: https://docs.python.org/2/c-api/object.html 132 | PyObject* PyObject_Str(PyObject *o); 133 | PyObject* PyObject_Repr(PyObject *o); 134 | PyObject* PyObject_Dir(PyObject *o); 135 | PyObject* PyObject_Call(PyObject *callable_object, PyObject *args, PyObject *kw); 136 | PyObject* PyObject_GetAttrString(PyObject *o, const char *attr_name); 137 | PyObject* PyObject_SetAttr(PyObject *o, PyObject *attr_name, PyObject *v); 138 | PyObject* PyObject_GetItem(PyObject *o, PyObject *key); 139 | int PyObject_SetItem(PyObject *o, PyObject *key, PyObject *v); 140 | int PyObject_DelItem(PyObject *o, PyObject *key); 141 | Py_ssize_t PyObject_Size(PyObject *o); 142 | PyObject* PyObject_GetIter(PyObject *o); 143 | int PyObject_IsTrue(PyObject *o); 144 | 145 | // Creating CPython function that call cffi callbacks 146 | // https://docs.python.org/2/c-api/structures.html#c.PyCFunction 147 | // http://bugs.python.org/file32578/16776.txt 148 | int const METH_VARARGS; 149 | int const METH_KEYWORDS; 150 | typedef ... PyCFunction; 151 | typedef struct { 152 | char* ml_name; // name of the method 153 | void* ml_meth; // pointer to the C implementation 154 | int ml_flags; // flag bits indicating how the call should be constructed 155 | char* ml_doc; // points to the contents of the docstring 156 | ...; 157 | } PyMethodDef; 158 | PyObject* PyCFunction_New(PyMethodDef *ml, PyObject *self); 159 | 160 | // Iterator: https://docs.python.org/2/c-api/iter.html 161 | PyObject* PyIter_Next(PyObject *o); 162 | 163 | // String: https://docs.python.org/2/c-api/string.html 164 | char* PyString_AsString(PyObject *string); 165 | PyObject* PyString_FromString(const char *v); 166 | 167 | // Unicode: https://docs.python.org/2/c-api/unicode.html 168 | PyObject* PyUnicode_AsUTF8String(PyObject* obj); 169 | PyObject* PyUnicode_FromString(const char *u); 170 | 171 | // Tuple: https://docs.python.org/2/c-api/tuple.html 172 | PyObject* PyTuple_Pack(Py_ssize_t n, ...); 173 | PyObject* PyTuple_GetItem(PyObject* tuple, int index); 174 | Py_ssize_t PyTuple_Size(PyObject* obj); 175 | 176 | // List: https://docs.python.org/2/c-api/list.html 177 | PyObject* PyList_New(Py_ssize_t len); 178 | PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index); 179 | Py_ssize_t PyList_Size(PyObject *list); 180 | int PyList_SetItem(PyObject *list, Py_ssize_t index, PyObject *item); 181 | 182 | // Dict: https://docs.python.org/2/c-api/dict.html 183 | PyObject* PyDict_New(); 184 | int PyDict_SetItem(PyObject *p, PyObject *key, PyObject *val); 185 | int PyDict_SetItemString(PyObject *p, const char *key, PyObject *val); 186 | PyObject* PyDict_Items(PyObject *p); 187 | 188 | // Slice: https://docs.python.org/2/c-api/slice.html 189 | PyObject* PySlice_New(PyObject *start, PyObject *stop, PyObject *step); 190 | 191 | // Integer: http://docs.python.org/2/c-api/int.html 192 | PyObject* PyInt_FromLong(long ival); 193 | long PyLong_AsLong(PyObject *obj); 194 | 195 | // Boolean: https://docs.python.org/2/c-api/bool.html 196 | PyObject* const Py_False; 197 | PyObject* const Py_True; 198 | 199 | PyObject* const Py_None; 200 | 201 | // Float: https://docs.python.org/2/c-api/float.html 202 | PyObject* PyFloat_FromDouble(double dval); 203 | double PyFloat_AsDouble(PyObject *obj); 204 | 205 | """) 206 | 207 | lib = ffi.verify(""" 208 | #include 209 | #ifdef PyTuple_GetItem 210 | #error "Picking Python.h from pypy" 211 | #endif 212 | """, 213 | include_dirs=_get_include_dirs(), 214 | libraries=["python2.7"], 215 | library_dirs=LIBDIRS, 216 | extra_link_args=_get_extra_link_args(), 217 | flags=ffi.RTLD_GLOBAL) 218 | 219 | prog_name = ffi.new("char[]", PYTHON) 220 | lib.Py_SetProgramName(prog_name) 221 | lib.Py_Initialize() 222 | atexit.register(lib.Py_Finalize) 223 | 224 | 225 | def add_exception_handling(name, errcond=ffi.NULL): 226 | 227 | fn = getattr(lib, name) 228 | 229 | def wrapper(*args): 230 | res = fn(*args) 231 | if errcond == res: 232 | py_exc_type = lib.PyErr_Occurred() 233 | if py_exc_type == ffi.NULL: 234 | # Some functions return NULL without raising an exception, 235 | # but also can raise an exception 236 | # (and also return NULL in this case). 237 | return None 238 | if py_exc_type in exception_by_py_exc: 239 | lib.PyErr_Clear() 240 | # TODO - get value 241 | raise exception_by_py_exc[py_exc_type] 242 | # less generic types first 243 | for py_exc_type, exc_type in reversed(exceptions): 244 | if lib.PyErr_ExceptionMatches(py_exc_type): 245 | lib.PyErr_Clear() 246 | # TODO - get value 247 | raise exc_type 248 | lib.PyErr_Clear() 249 | raise Exception("Call of '%s' exploded" % name) 250 | return res 251 | 252 | setattr(lib, name, wrapper) 253 | 254 | 255 | _exceptions = [ 256 | ('PyExc_BaseException', BaseException), 257 | ('PyExc_Exception', Exception), 258 | ('PyExc_StandardError', StandardError), 259 | ('PyExc_ArithmeticError', ArithmeticError), 260 | ('PyExc_LookupError', LookupError), 261 | ('PyExc_AssertionError', AssertionError), 262 | ('PyExc_AttributeError', AttributeError), 263 | ('PyExc_EOFError', EOFError), 264 | ('PyExc_EnvironmentError', EnvironmentError), 265 | ('PyExc_FloatingPointError', FloatingPointError), 266 | ('PyExc_IOError', IOError), 267 | ('PyExc_ImportError', ImportError), 268 | ('PyExc_IndexError', IndexError), 269 | ('PyExc_KeyError', KeyError), 270 | ('PyExc_KeyboardInterrupt', KeyboardInterrupt), 271 | ('PyExc_MemoryError', MemoryError), 272 | ('PyExc_NameError', NameError), 273 | ('PyExc_NotImplementedError', NotImplementedError), 274 | ('PyExc_OSError', OSError), 275 | ('PyExc_OverflowError', OverflowError), 276 | ('PyExc_ReferenceError', ReferenceError), 277 | ('PyExc_RuntimeError', RuntimeError), 278 | ('PyExc_SyntaxError', SyntaxError), 279 | ('PyExc_SystemError', SystemError), 280 | ('PyExc_SystemExit', SystemExit), 281 | ('PyExc_TypeError', TypeError), 282 | ('PyExc_ValueError', ValueError), 283 | #('PyExc_WindowsError', WindowsError), 284 | ('PyExc_ZeroDivisionError', ZeroDivisionError), 285 | ] 286 | 287 | exceptions = [(getattr(lib, exc_name), exc) for exc_name, exc in _exceptions] 288 | exception_by_py_exc = dict(exceptions) 289 | 290 | 291 | # Add exception handling for all functions that can raise errors 292 | 293 | for args in [ 294 | 'PyImport_ImportModule', 295 | 'PyObject_Str', 296 | 'PyObject_Repr', 297 | 'PyObject_Call', 298 | 'PyObject_GetAttrString', 299 | 'PyObject_GetItem', 300 | ('PyObject_SetItem', -1), 301 | ('PyObject_DelItem', -1), 302 | ('PyObject_Size', int(ffi.cast('Py_ssize_t', -1))), 303 | 'PyObject_GetIter', 304 | ('PyObject_IsTrue', -1), 305 | 'PyIter_Next', 306 | 'PyString_AsString', 307 | 'PyString_FromString', 308 | 'PyUnicode_AsUTF8String', # ? docs say nothing about these two 309 | 'PyUnicode_FromString', 310 | 'PyTuple_Pack', 311 | 'PyTuple_GetItem', 312 | 'PyList_New', 313 | 'PyList_GetItem', 314 | ('PyList_SetItem', -1), 315 | 'PyDict_New', 316 | ('PyDict_SetItem', -1), 317 | ('PyDict_SetItemString', -1), 318 | ('PyLong_AsLong', -1), 319 | 'PyFloat_FromDouble', 320 | ('PyFloat_FromDouble', -1.0), 321 | ]: 322 | if not isinstance(args, tuple): 323 | args = (args,) 324 | add_exception_handling(*args) 325 | -------------------------------------------------------------------------------- /pymetabiosis/wrapper.py: -------------------------------------------------------------------------------- 1 | import operator 2 | import types 3 | from __pypy__ import identity_dict 4 | import pymetabiosis.module 5 | from pymetabiosis.bindings import lib, ffi, exceptions 6 | 7 | 8 | def convert(obj): 9 | _type = type(obj) 10 | if _type in pypy_to_cpy_converters: 11 | try: 12 | return pypy_to_cpy_converters[_type](obj) 13 | except NoConvertError: 14 | pass 15 | if getattr(obj, '_pymetabiosis_wrap', None): 16 | return convert_unknown(obj) 17 | raise NoConvertError(type(obj)) 18 | 19 | def convert_string(s): 20 | return ffi.gc(lib.PyString_FromString(ffi.new("char[]", s)), lib.Py_DECREF) 21 | 22 | def convert_unicode(u): 23 | return ffi.gc( 24 | lib.PyUnicode_FromString(ffi.new("char[]", u.encode('utf-8'))), 25 | lib.Py_DECREF) 26 | 27 | def convert_tuple(values, convert_items=True): 28 | if convert_items: 29 | values = [convert(value) for value in values] 30 | 31 | return ffi.gc(lib.PyTuple_Pack(len(values), *values), lib.Py_DECREF) 32 | 33 | def convert_int(obj): 34 | return ffi.gc(lib.PyInt_FromLong(obj), lib.Py_DECREF) 35 | 36 | def convert_bool(obj): 37 | py_obj = lib.Py_True if obj else lib.Py_False 38 | lib.Py_INCREF(py_obj) 39 | return ffi.gc(py_obj, lib.Py_DECREF) 40 | 41 | def convert_None(obj): 42 | lib.Py_INCREF(lib.Py_None) 43 | return ffi.gc(lib.Py_None, lib.Py_DECREF) 44 | 45 | def convert_float(obj): 46 | return ffi.gc(lib.PyFloat_FromDouble(obj), lib.Py_DECREF) 47 | 48 | def convert_dict(obj, convert_values=True): 49 | dict = ffi.gc(lib.PyDict_New(), lib.Py_DECREF) 50 | 51 | for key, value in obj.iteritems(): 52 | if convert_values: 53 | value = convert(value) 54 | lib.PyDict_SetItem(dict, convert(key), value) 55 | 56 | return dict 57 | 58 | def convert_list(obj): 59 | lst = ffi.gc(lib.PyList_New(len(obj)), lib.Py_DECREF) 60 | for i, x in enumerate(obj): 61 | lib.PyList_SetItem(lst, i, convert(x)) 62 | return lst 63 | 64 | def convert_slice(obj): 65 | return ffi.gc( 66 | lib.PySlice_New( 67 | convert(obj.start), convert(obj.stop), convert(obj.step)), 68 | lib.Py_DECREF) 69 | 70 | def convert_type(obj): 71 | try: 72 | return pypy_to_cpy_types[obj] 73 | except KeyError: 74 | raise NoConvertError 75 | 76 | 77 | @ffi.callback("PyObject*(PyObject*, PyObject*, PyObject*)") 78 | def callback(py_self, py_args, py_kwargs): 79 | args = () if py_args == ffi.NULL else pypy_convert(py_args) 80 | kwargs = {} if py_kwargs == ffi.NULL else pypy_convert(py_kwargs) 81 | fn = pypy_convert(py_self) 82 | try: 83 | result = fn(*args, **kwargs) 84 | return convert(result) 85 | except Exception as e: 86 | default_message = "Exception in pymetabiosis callback" 87 | for py_exc_type, exc_type in reversed(exceptions): 88 | if isinstance(e, exc_type): 89 | lib.PyErr_SetString( 90 | py_exc_type, ffi.new("char[]", default_message)) 91 | return ffi.NULL 92 | lib.PyErr_SetString(lib.PyExc_Exception, 93 | ffi.new("char[]", default_message)) 94 | return ffi.NULL 95 | 96 | 97 | py_method = ffi.new("PyMethodDef*", dict( 98 | ml_name=ffi.new("char[]", "pypy_callback"), 99 | ml_meth=callback, 100 | ml_flags=lib.METH_VARARGS | lib.METH_KEYWORDS, 101 | ml_doc=ffi.new("char[]", "") 102 | )) 103 | 104 | def convert_function(obj): 105 | return lib.PyCFunction_New(py_method, convert_unknown(obj)) 106 | 107 | 108 | class MetabiosisWrapper(object): 109 | def __init__(self, obj, noconvert=False): 110 | self.__dict__['_cpyobj'] = obj 111 | self.__dict__['_noconvert'] = noconvert 112 | 113 | def __abs__(self): 114 | return self._maybe_pypy_convert(cpy_operator.abs(self)) 115 | 116 | def __add__(self, other): 117 | return self._maybe_pypy_convert(cpy_operator.add(self, other)) 118 | 119 | def __and__(self, other): 120 | return self._maybe_pypy_convert(cpy_operator.and_(self, other)) 121 | 122 | def __contains__(self, item): 123 | return self._maybe_pypy_convert(cpy_operator.contains(self, item)) 124 | 125 | def __delslice__(self, a, b): 126 | return self._maybe_pypy_convert(cpy_operator.delslice(self, a, b)) 127 | 128 | def __div__(self, other): 129 | return self._maybe_pypy_convert(cpy_operator.div(self, other)) 130 | 131 | def __eq__(self, other): 132 | return self._maybe_pypy_convert(cpy_operator.eq(self, other)) 133 | 134 | def __floordiv__(self, other): 135 | return self._maybe_pypy_convert(cpy_operator.floordiv(self, other)) 136 | 137 | def __ge__(self, other): 138 | return self._maybe_pypy_convert(cpy_operator.ge(self, other)) 139 | 140 | def __getslice__(self, a, b): 141 | return self._maybe_pypy_convert(cpy_operator.getslice(self, a, b)) 142 | 143 | def __gt__(self, other): 144 | return self._maybe_pypy_convert(cpy_operator.gt(self, other)) 145 | 146 | def __iadd__(self, other): 147 | return self._maybe_pypy_convert(cpy_operator.iadd(self, other)) 148 | 149 | def __iand__(self, other): 150 | return self._maybe_pypy_convert(cpy_operator.iand(self, other)) 151 | 152 | def __idiv__(self, other): 153 | return self._maybe_pypy_convert(cpy_operator.idiv(self, other)) 154 | 155 | def __ifloordiv__(self, other): 156 | return self._maybe_pypy_convert(cpy_operator.ifloordiv(self, other)) 157 | 158 | def __ilshift__(self, other): 159 | return self._maybe_pypy_convert(cpy_operator.ilshift(self, other)) 160 | 161 | def __imod__(self, other): 162 | return self._maybe_pypy_convert(cpy_operator.imod(self, other)) 163 | 164 | def __imul__(self, other): 165 | return self._maybe_pypy_convert(cpy_operator.imul(self, other)) 166 | 167 | def __index__(self): 168 | """ __index__ must always return an int. """ 169 | return pypy_convert_int(cpy_operator.index(self)._cpyobj) 170 | 171 | def __invert__(self): 172 | return self._maybe_pypy_convert(cpy_operator.invert(self)) 173 | 174 | def __ior__(self, other): 175 | return self._maybe_pypy_convert(cpy_operator.ior(self, other)) 176 | 177 | def __ipow__(self, other): 178 | return self._maybe_pypy_convert(cpy_operator.ipow(self, other)) 179 | 180 | def __irshift__(self, other): 181 | return self._maybe_pypy_convert(cpy_operator.irshift(self, other)) 182 | 183 | def __isub__(self, other): 184 | return self._maybe_pypy_convert(cpy_operator.isub(self, other)) 185 | 186 | def __itruediv__(self, other): 187 | return self._maybe_pypy_convert(cpy_operator.itruediv(self, other)) 188 | 189 | def __ixor__(self, other): 190 | return self._maybe_pypy_convert(cpy_operator.ixor(self, other)) 191 | 192 | def __le__(self, other): 193 | return self._maybe_pypy_convert(cpy_operator.le(self, other)) 194 | 195 | def __lshift__(self, other): 196 | return self._maybe_pypy_convert(cpy_operator.lshift(self, other)) 197 | 198 | def __lt__(self, other): 199 | return self._maybe_pypy_convert(cpy_operator.lt(self, other)) 200 | 201 | def __mod__(self, other): 202 | return self._maybe_pypy_convert(cpy_operator.mod(self, other)) 203 | 204 | def __mul__(self, other): 205 | return self._maybe_pypy_convert(cpy_operator.mul(self, other)) 206 | 207 | def __ne__(self, other): 208 | return self._maybe_pypy_convert(cpy_operator.ne(self, other)) 209 | 210 | def __neg__(self): 211 | return self._maybe_pypy_convert(cpy_operator.neg(self)) 212 | 213 | def __or__(self, other): 214 | return self._maybe_pypy_convert(cpy_operator.or_(self, other)) 215 | 216 | def __pos__(self): 217 | return self._maybe_pypy_convert(cpy_operator.pos(self)) 218 | 219 | def __pow__(self, other): 220 | return self._maybe_pypy_convert(cpy_operator.pow(self, other)) 221 | 222 | def __radd__(self, other): 223 | return self._maybe_pypy_convert(cpy_operator.add(other, self)) 224 | 225 | def __rand__(self, other): 226 | return self._maybe_pypy_convert(cpy_operator.and_(other, self)) 227 | 228 | def __rdiv__(self, other): 229 | return self._maybe_pypy_convert(cpy_operator.div(other, self)) 230 | 231 | def __rdivmod__(self, other): 232 | return self._maybe_pypy_convert(cpy_operator.divmod(other, self)) 233 | 234 | def __rfloordiv__(self, other): 235 | return self._maybe_pypy_convert(cpy_operator.floordiv(other, self)) 236 | 237 | def __rlshift__(self, other): 238 | return self._maybe_pypy_convert(cpy_operator.lshift(other, self)) 239 | 240 | def __rmod__(self, other): 241 | return self._maybe_pypy_convert(cpy_operator.mod(other, self)) 242 | 243 | def __rmul__(self, other): 244 | return self._maybe_pypy_convert(cpy_operator.mul(other, self)) 245 | 246 | def __ror__(self, other): 247 | return self._maybe_pypy_convert(cpy_operator.or_(other, self)) 248 | 249 | def __rpow__(self, other): 250 | return self._maybe_pypy_convert(cpy_operator.pow(other, self)) 251 | 252 | def __rrshift__(self, other): 253 | return self._maybe_pypy_convert(cpy_operator.rshift(other, self)) 254 | 255 | def __rxor__(self, other): 256 | return self._maybe_pypy_convert(cpy_operator.xor(other, self)) 257 | 258 | def __rshift__(self, other): 259 | return self._maybe_pypy_convert(cpy_operator.rshift(self, other)) 260 | 261 | def __rsub__(self, other): 262 | return self._maybe_pypy_convert(cpy_operator.sub(other, self)) 263 | 264 | def __rtruediv__(self, other): 265 | return self._maybe_pypy_convert(cpy_operator.truediv(other, self)) 266 | 267 | def __setslice__(self, a, b, value): 268 | return self._maybe_pypy_convert(cpy_operator.setslice(self, a, b, value)) 269 | 270 | def __sub__(self, other): 271 | return self._maybe_pypy_convert(cpy_operator.sub(self, other)) 272 | 273 | def __truediv__(self, other): 274 | return self._maybe_pypy_convert(cpy_operator.truediv(self, other)) 275 | 276 | def __xor__(self, other): 277 | return self._maybe_pypy_convert(cpy_operator.xor(self, other)) 278 | 279 | 280 | 281 | def __repr__(self): 282 | py_str = ffi.gc(lib.PyObject_Repr(self._cpyobj), lib.Py_DECREF) 283 | return pypy_convert(py_str) 284 | 285 | def __str__(self): 286 | py_str = ffi.gc(lib.PyObject_Str(self._cpyobj), lib.Py_DECREF) 287 | return pypy_convert(py_str) 288 | 289 | def __dir__(self): 290 | py_lst = ffi.gc(lib.PyObject_Dir(self._cpyobj), lib.Py_DECREF) 291 | return pypy_convert(py_lst) 292 | 293 | def __getattr__(self, name): 294 | return self._getattr(name) 295 | 296 | def _getattr(self, name): 297 | c_name = ffi.new("char[]", name) 298 | py_attr = ffi.gc( 299 | lib.PyObject_GetAttrString(self._cpyobj, c_name), 300 | lib.Py_DECREF) 301 | return self._maybe_pypy_convert(py_attr) 302 | 303 | def __setattr__(self, key, value): 304 | lib.PyObject_SetAttr(self._cpyobj, convert(key), convert(value)) 305 | 306 | def __getitem__(self, key): 307 | py_res = ffi.gc( 308 | lib.PyObject_GetItem(self._cpyobj, convert(key)), 309 | lib.Py_DECREF) 310 | return self._maybe_pypy_convert(py_res) 311 | 312 | def __setitem__(self, key, value): 313 | lib.PyObject_SetItem(self._cpyobj, convert(key), convert(value)) 314 | 315 | def __delitem__(self, key): 316 | lib.PyObject_DelItem(self._cpyobj, convert(key)) 317 | 318 | def __len__(self): 319 | return lib.PyObject_Size(self._cpyobj) 320 | 321 | def __nonzero__(self): 322 | return lib.PyObject_IsTrue(self._cpyobj) == 1 323 | 324 | def __iter__(self): 325 | py_iter = ffi.gc(lib.PyObject_GetIter(self._cpyobj), lib.Py_DECREF) 326 | while True: 327 | py_next = lib.PyIter_Next(py_iter) 328 | if py_next is None: 329 | break 330 | yield self._maybe_pypy_convert(py_next) 331 | 332 | def __instancecheck__(self, instance): 333 | if type(instance) is MetabiosisWrapper: 334 | return self._getattr('__instancecheck__')(instance) 335 | else: 336 | return super(MetabiosisWrapper, self).__instancecheck__(instance) 337 | 338 | def __subclasscheck__(self, subclass): 339 | if type(subclass) is MetabiosisWrapper: 340 | return self._getattr('__subclasscheck__')(subclass) 341 | else: 342 | return super(MetabiosisWrapper, self).__subclasscheck__(subclass) 343 | 344 | def __call__(self, *args, **kwargs): 345 | return self._call(args, kwargs) 346 | 347 | def _call(self, args, kwargs=None, args_kwargs_converted=False): 348 | convert = not args_kwargs_converted 349 | arguments_tuple = convert_tuple(args, convert_items=convert) 350 | 351 | keywordargs = ffi.NULL 352 | if kwargs: 353 | keywordargs = convert_dict(kwargs, convert_values=convert) 354 | 355 | return_value = ffi.gc( 356 | lib.PyObject_Call(self._cpyobj, arguments_tuple, keywordargs), 357 | lib.Py_DECREF) 358 | 359 | return self._maybe_pypy_convert(return_value) 360 | 361 | def get_type(self): 362 | typeobject = ffi.cast("PyObject*", self._cpyobj.ob_type) 363 | 364 | lib.Py_INCREF(typeobject) 365 | 366 | return MetabiosisWrapper(ffi.gc(typeobject, lib.Py_DECREF)) 367 | 368 | def _maybe_pypy_convert(self, py_obj): 369 | if isinstance(py_obj, MetabiosisWrapper): 370 | return py_obj 371 | if self._noconvert: 372 | return MetabiosisWrapper(py_obj, self._noconvert) 373 | else: 374 | return pypy_convert(py_obj) 375 | 376 | 377 | def pypy_convert(obj): 378 | if isinstance(obj, MetabiosisWrapper): 379 | return obj 380 | type = MetabiosisWrapper(obj).get_type()._cpyobj 381 | if type in cpy_to_pypy_converters: 382 | try: 383 | return cpy_to_pypy_converters[type](obj) 384 | except NoConvertError: 385 | pass 386 | if type == ApplevelWrapped._cpyobj: 387 | return _obj_by_applevel[obj] 388 | else: 389 | return MetabiosisWrapper(obj) 390 | 391 | def pypy_convert_int(obj): 392 | return int(lib.PyLong_AsLong(obj)) 393 | 394 | def pypy_convert_bool(obj): 395 | return obj == lib.Py_True 396 | 397 | def pypy_convert_None(obj): 398 | return None 399 | 400 | def pypy_convert_float(obj): 401 | return float(lib.PyFloat_AsDouble(obj)) 402 | 403 | def pypy_convert_string(obj): 404 | return ffi.string(lib.PyString_AsString(obj)) 405 | 406 | def pypy_convert_unicode(obj): 407 | return pypy_convert_string(lib.PyUnicode_AsUTF8String(obj))\ 408 | .decode('utf-8') 409 | 410 | def pypy_convert_tuple(obj): 411 | return tuple( 412 | pypy_convert(lib.PyTuple_GetItem(obj, i)) 413 | for i in xrange(lib.PyTuple_Size(obj))) 414 | 415 | def pypy_convert_dict(obj): 416 | items = ffi.gc(lib.PyDict_Items(obj), lib.Py_DECREF) 417 | return dict(pypy_convert_list(items)) 418 | 419 | def pypy_convert_list(obj): 420 | return [pypy_convert(lib.PyList_GetItem(obj, i)) 421 | for i in xrange(lib.PyList_Size(obj))] 422 | 423 | def pypy_convert_type(obj): 424 | try: 425 | return cpy_to_pypy_types[obj] 426 | except KeyError: 427 | raise NoConvertError 428 | 429 | class NoConvertError(Exception): 430 | pass 431 | 432 | 433 | pypy_to_cpy_converters = { 434 | MetabiosisWrapper : operator.attrgetter("_cpyobj"), 435 | int : convert_int, 436 | float : convert_float, 437 | str : convert_string, 438 | unicode : convert_unicode, 439 | tuple : convert_tuple, 440 | dict : convert_dict, 441 | list : convert_list, 442 | slice : convert_slice, 443 | bool : convert_bool, 444 | types.NoneType: convert_None, 445 | type : convert_type, 446 | types.FunctionType: convert_function, 447 | types.MethodType: convert_function, 448 | } 449 | pypy_to_cpy_types = {} 450 | cpy_to_pypy_converters = {} 451 | cpy_to_pypy_types = {} 452 | 453 | 454 | def init_cpy_to_pypy_converters(): 455 | global cpy_to_pypy_converters 456 | 457 | import __builtin__ 458 | builtin = pymetabiosis.module.import_module("__builtin__", noconvert=True) 459 | types = pymetabiosis.module.import_module("types") 460 | 461 | global cpy_operator 462 | cpy_operator = pymetabiosis.import_module('operator', noconvert=True) 463 | 464 | cpy_to_pypy_converters = { 465 | builtin.int._cpyobj : pypy_convert_int, 466 | builtin.float._cpyobj : pypy_convert_float, 467 | builtin.str._cpyobj : pypy_convert_string, 468 | builtin.unicode._cpyobj : pypy_convert_unicode, 469 | builtin.tuple._cpyobj : pypy_convert_tuple, 470 | builtin.dict._cpyobj : pypy_convert_dict, 471 | builtin.list._cpyobj : pypy_convert_list, 472 | builtin.bool._cpyobj : pypy_convert_bool, 473 | builtin.type._cpyobj : pypy_convert_type, 474 | types.NoneType._cpyobj : pypy_convert_None, 475 | } 476 | 477 | converted_types = ['int', 'float', 'bool', 'str', 'unicode'] 478 | for _type in converted_types: 479 | cpy_type = getattr(builtin, _type)._cpyobj 480 | pypy_type = getattr(__builtin__, _type) 481 | cpy_to_pypy_types[cpy_type] = pypy_type 482 | pypy_to_cpy_types[pypy_type] = cpy_type 483 | 484 | 485 | def applevel(code, noconvert=False): 486 | code = '\n'.join([' ' + line for line in code.split('\n') if line]) 487 | code = 'def anonymous():\n' + code 488 | py_code = ffi.gc( 489 | lib.Py_CompileString(code, 'exec', lib.Py_file_input), 490 | lib.Py_DECREF) 491 | lib.Py_INCREF(py_code) 492 | py_elem = lib.PyObject_GetAttrString(py_code, 'co_consts') 493 | lib.Py_INCREF(py_elem) 494 | py_zero = ffi.gc(lib.PyInt_FromLong(0), lib.Py_DECREF) 495 | py_item = lib.PyObject_GetItem(py_elem, py_zero) 496 | py_locals = ffi.gc(lib.PyDict_New(), lib.Py_DECREF) 497 | py_globals = ffi.gc(lib.PyDict_New(), lib.Py_DECREF) 498 | py_bltns = lib.PyEval_GetBuiltins() 499 | lib.PyDict_SetItemString(py_globals, '__builtins__', py_bltns) 500 | py_res = lib.PyEval_EvalCode(py_item, py_globals, py_locals) 501 | return MetabiosisWrapper(py_res, noconvert=noconvert) 502 | 503 | ApplevelWrapped = applevel(''' 504 | class ApplevelWrapped(object): 505 | pass 506 | return ApplevelWrapped 507 | ''', noconvert=True) 508 | 509 | _applevel_by_obj = {} 510 | _applevel_by_unhashable_obj = identity_dict() 511 | _obj_by_applevel = {} 512 | 513 | def convert_unknown(obj): 514 | try: 515 | aw = _applevel_by_obj.get(obj) 516 | except TypeError: 517 | aw = _applevel_by_unhashable_obj.get(obj) 518 | if aw is None: 519 | aw = ApplevelWrapped()._cpyobj 520 | try: 521 | _applevel_by_obj[obj] = aw 522 | except TypeError: 523 | _applevel_by_unhashable_obj[obj] = aw 524 | _obj_by_applevel[aw] = obj 525 | lib.Py_INCREF(aw) 526 | return aw 527 | --------------------------------------------------------------------------------