├── .coveragerc ├── .gitignore ├── .treerc ├── AUTHORS ├── LICENSE ├── README ├── byterun ├── __init__.py ├── __main__.py ├── abstractvm.py ├── execfile.py ├── pycfg.py ├── pyobj.py └── pyvm2.py ├── requirements.txt ├── setup.py ├── tests ├── __init__.py ├── test_abstractvm.py ├── test_basic.py ├── test_exceptions.py ├── test_functions.py ├── test_pycfg.py ├── test_with.py └── vmtest.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = true 3 | source = . 4 | omit = .tox/* 5 | 6 | [report] 7 | partial_branches = 8 | (?i)# *pragma[: ]*no *branch 9 | elif PY3: 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .coverage 3 | htmlcov 4 | .tox 5 | MANIFEST 6 | *.egg-info 7 | -------------------------------------------------------------------------------- /.treerc: -------------------------------------------------------------------------------- 1 | [default] 2 | ignore = 3 | *.pyc 4 | __pycache__ 5 | .tox 6 | htmlcov 7 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Paul Swartz 2 | Ned Batchelder 3 | Allison Kaptur 4 | Laura Lindzey 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013, Ned Batchelder 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 7 | of the Software, and to permit persons to whom the Software is furnished to do 8 | so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | 21 | 22 | PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 23 | -------------------------------------------- 24 | 25 | 1. This LICENSE AGREEMENT is between the Python Software Foundation 26 | ("PSF"), and the Individual or Organization ("Licensee") accessing and 27 | otherwise using this software ("Python") in source or binary form and 28 | its associated documentation. 29 | 30 | 2. Subject to the terms and conditions of this License Agreement, PSF 31 | hereby grants Licensee a nonexclusive, royalty-free, world-wide 32 | license to reproduce, analyze, test, perform and/or display publicly, 33 | prepare derivative works, distribute, and otherwise use Python 34 | alone or in any derivative version, provided, however, that PSF's 35 | License Agreement and PSF's notice of copyright, i.e., "Copyright (c) 36 | 2001, 2002, 2003, 2004, 2005, 2006 Python Software Foundation; All Rights 37 | Reserved" are retained in Python alone or in any derivative version 38 | prepared by Licensee. 39 | 40 | 3. In the event Licensee prepares a derivative work that is based on 41 | or incorporates Python or any part thereof, and wants to make 42 | the derivative work available to others as provided herein, then 43 | Licensee hereby agrees to include in any such work a brief summary of 44 | the changes made to Python. 45 | 46 | 4. PSF is making Python available to Licensee on an "AS IS" 47 | basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR 48 | IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND 49 | DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS 50 | FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT 51 | INFRINGE ANY THIRD PARTY RIGHTS. 52 | 53 | 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 54 | FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS 55 | A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, 56 | OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. 57 | 58 | 6. This License Agreement will automatically terminate upon a material 59 | breach of its terms and conditions. 60 | 61 | 7. Nothing in this License Agreement shall be deemed to create any 62 | relationship of agency, partnership, or joint venture between PSF and 63 | Licensee. This License Agreement does not grant permission to use PSF 64 | trademarks or trade name in a trademark sense to endorse or promote 65 | products or services of Licensee, or any third party. 66 | 67 | 8. By copying, installing or otherwise using Python, Licensee 68 | agrees to be bound by the terms and conditions of this License 69 | Agreement. 70 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Byterun 2 | ------- 3 | 4 | This is a pure-Python implementation of a Python bytecode execution virtual 5 | machine. I started it to get a better understanding of bytecodes so I could 6 | fix branch coverage bugs in coverage.py. 7 | 8 | -------------------------------------------------------------------------------- /byterun/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/byterun/e3dc1349ed0d5f737708f274f714ac77b6f047cc/byterun/__init__.py -------------------------------------------------------------------------------- /byterun/__main__.py: -------------------------------------------------------------------------------- 1 | """A main program for Byterun.""" 2 | 3 | import argparse 4 | import logging 5 | 6 | from . import execfile 7 | 8 | parser = argparse.ArgumentParser( 9 | prog="byterun", 10 | description="Run Python programs with a Python bytecode interpreter.", 11 | ) 12 | parser.add_argument( 13 | '-m', dest='module', action='store_true', 14 | help="prog is a module name, not a file name.", 15 | ) 16 | parser.add_argument( 17 | '-v', '--versbose', dest='verbose', action='store_true', 18 | help="trace the execution of the bytecode.", 19 | ) 20 | parser.add_argument( 21 | 'prog', 22 | help="The program to run.", 23 | ) 24 | parser.add_argument( 25 | 'args', nargs=argparse.REMAINDER, 26 | help="Arguments to pass to the program.", 27 | ) 28 | args = parser.parse_args() 29 | 30 | if args.module: 31 | run_fn = execfile.run_python_module 32 | else: 33 | run_fn = execfile.run_python_file 34 | 35 | level = logging.DEBUG if args.verbose else logging.WARNING 36 | logging.basicConfig(level=level) 37 | 38 | argv = [args.prog] + args.args 39 | run_fn(args.prog, argv) 40 | -------------------------------------------------------------------------------- /byterun/abstractvm.py: -------------------------------------------------------------------------------- 1 | """Classes to ease the abstraction of pyvm2.VirtualMachine. 2 | 3 | This module provides 2 classes that provide different kinds of 4 | abstraction. AbstractVirtualMachine abstracts operators and other magic method 5 | uses. AncestorTraversalVirtualMachine changes the execution order of basic 6 | blocks so that each only executes once. 7 | """ 8 | 9 | import logging 10 | 11 | 12 | from byterun import pycfg 13 | from byterun import pyvm2 14 | import six 15 | 16 | log = logging.getLogger(__name__) 17 | 18 | 19 | class AbstractVirtualMachine(pyvm2.VirtualMachine): 20 | """A base class for abstract interpreters based on VirtualMachine. 21 | 22 | AbstractVirtualMachine replaces the default metacyclic implementation of 23 | operators and other operations that actually forward to a python magic 24 | method with a virtual machine level attribute get and a call to the 25 | returned method. 26 | """ 27 | 28 | def __init__(self): 29 | super(AbstractVirtualMachine, self).__init__() 30 | # The key is the instruction suffix and the value is the magic method 31 | # name. 32 | binary_operator_name_mapping = dict( 33 | ADD="__add__", 34 | AND="__and__", 35 | DIVIDE="__div__", 36 | FLOOR_DIVIDE="__floordiv__", 37 | LSHIFT="__lshift__", 38 | MODULO="__mod__", 39 | MULTIPLY="__mul__", 40 | OR="__or__", 41 | POWER="__pow__", 42 | RSHIFT="__rshift__", 43 | SUBSCR="__getitem__", 44 | SUBTRACT="__sub__", 45 | TRUE_DIVIDE="__truediv__", 46 | XOR="__xor__", 47 | ) 48 | # Use the above data to generate wrappers for each magic operators. This 49 | # replaces the original dict since any operator that is not listed here 50 | # will not work, so it is better to have it cause a KeyError. 51 | self.binary_operators = dict((op, self.magic_operator(magic)) 52 | for op, magic in 53 | binary_operator_name_mapping.iteritems()) 54 | # TODO(ampere): Add support for unary and comparison operators 55 | 56 | def magic_operator(self, name): 57 | # TODO(ampere): Implement support for r-operators 58 | def magic_operator_wrapper(x, y): 59 | return self.call_function(self.load_attr(x, name), 60 | [y], {}) 61 | return magic_operator_wrapper 62 | 63 | reversable_operators = set([ 64 | "__add__", "__sub__", "__mul__", 65 | "__div__", "__truediv__", "__floordiv__", 66 | "__mod__", "__divmod__", "__pow__", 67 | "__lshift__", "__rshift__", "__and__", "__or__", "__xor__" 68 | ]) 69 | 70 | @staticmethod 71 | def reverse_operator_name(name): 72 | if name in AbstractVirtualMachine.reversable_operators: 73 | return "__r" + name[2:] 74 | return None 75 | 76 | def build_slice(self, start, stop, step): 77 | return slice(start, stop, step) 78 | 79 | def byte_GET_ITER(self): 80 | self.push(self.load_attr(self.pop(), "__iter__")) 81 | self.call_function_from_stack(0, [], {}) 82 | 83 | def byte_FOR_ITER(self, jump): 84 | try: 85 | self.push(self.load_attr(self.top(), "next")) 86 | self.call_function_from_stack(0, [], {}) 87 | self.jump(self.frame.f_lasti) 88 | except StopIteration: 89 | self.pop() 90 | self.jump(jump) 91 | 92 | def byte_STORE_MAP(self): 93 | # pylint: disable=unbalanced-tuple-unpacking 94 | the_map, val, key = self.popn(3) 95 | self.store_subscr(the_map, key, val) 96 | self.push(the_map) 97 | 98 | def del_subscr(self, obj, key): 99 | self.call_function(self.load_attr(obj, "__delitem__"), 100 | [key], {}) 101 | 102 | def store_subscr(self, obj, key, val): 103 | self.call_function(self.load_attr(obj, "__setitem__"), 104 | [key, val], {}) 105 | 106 | def sliceOperator(self, op): # pylint: disable=invalid-name 107 | start = 0 108 | end = None # we will take this to mean end 109 | op, count = op[:-2], int(op[-1]) 110 | if count == 1: 111 | start = self.pop() 112 | elif count == 2: 113 | end = self.pop() 114 | elif count == 3: 115 | end = self.pop() 116 | start = self.pop() 117 | l = self.pop() 118 | if end is None: 119 | end = self.call_function(self.load_attr(l, "__len__"), [], {}) 120 | if op.startswith('STORE_'): 121 | self.call_function(self.load_attr(l, "__setitem__"), 122 | [self.build_slice(start, end, 1), self.pop()], 123 | {}) 124 | elif op.startswith('DELETE_'): 125 | self.call_function(self.load_attr(l, "__delitem__"), 126 | [self.build_slice(start, end, 1)], 127 | {}) 128 | else: 129 | self.push(self.call_function(self.load_attr(l, "__getitem__"), 130 | [self.build_slice(start, end, 1)], 131 | {})) 132 | 133 | def byte_UNPACK_SEQUENCE(self, count): 134 | seq = self.pop() 135 | itr = self.call_function(self.load_attr(seq, "__iter__"), [], {}) 136 | values = [] 137 | for _ in range(count): 138 | # TODO(ampere): Fix for python 3 139 | values.append(self.call_function(self.load_attr(itr, "next"), 140 | [], {})) 141 | for value in reversed(values): 142 | self.push(value) 143 | 144 | 145 | class AncestorTraversalVirtualMachine(AbstractVirtualMachine): 146 | """An abstract interpreter implementing a traversal of basic blocks. 147 | 148 | This class replaces run_frame with a traversal that executes all basic 149 | blocks in ancestor first order starting with the entry block. This uses 150 | pycfg.BlockTable.get_ancestors_first_traversal(); see it's documentation for 151 | more information about the order. 152 | 153 | As the traversal is done there is no attempt to rollback the state, so 154 | parallel paths in the CFG (even those that cannot be run in the same 155 | execution) will often see each other's side-effects. Effectively this means 156 | that the execution of each basic block needs to commute with the execution 157 | of other blocks it is not ordered with. 158 | """ 159 | 160 | def __init__(self): 161 | super(AncestorTraversalVirtualMachine, self).__init__() 162 | self.cfg = pycfg.CFG() 163 | 164 | def frame_traversal_setup(self, frame): 165 | """Initialize a frame to allow ancestors first traversal. 166 | 167 | Args: 168 | frame: The execution frame to update. 169 | """ 170 | frame.block_table = self.cfg.get_block_table(frame.f_code) 171 | frame.order = frame.block_table.get_ancestors_first_traversal() 172 | assert frame.f_lasti == 0 173 | 174 | def frame_traversal_next(self, frame): 175 | """Move the frame instruction pointer to the next instruction. 176 | 177 | This implements the next instruction operation on the ancestors first 178 | traversal order. 179 | 180 | Args: 181 | frame: The execution frame to update. 182 | 183 | Returns: 184 | False if the traversal is done (every instruction in the frames code 185 | has been executed. True otherwise. 186 | """ 187 | head = frame.order[0] 188 | if frame.f_lasti < head.begin or frame.f_lasti > head.end: 189 | frame.order.pop(0) 190 | if not frame.order: 191 | return False 192 | head = frame.order[0] 193 | if frame.f_lasti != head.begin: 194 | log.debug("natural next %d, order next %d", 195 | frame.f_lasti, head.begin) 196 | frame.f_lasti = head.begin 197 | return True 198 | 199 | def run_frame(self, frame): 200 | """Run a frame until it returns (somehow). 201 | 202 | Exceptions are raised, the return value is returned. 203 | 204 | This implementation executes in ancestors first order. See 205 | pycfg.BlockTable.get_ancestors_first_traversal(). 206 | 207 | Args: 208 | frame: The execution frame. 209 | 210 | Returns: 211 | The return value of the frame after execution. 212 | """ 213 | self.push_frame(frame) 214 | self.frame_traversal_setup(frame) 215 | while True: 216 | why = self.run_instruction() 217 | # TODO(ampere): Store various breaking "why"s so they can be handled 218 | if not self.frame_traversal_next(frame): 219 | break 220 | self.pop_frame() 221 | 222 | # TODO(ampere): We don't really support exceptions. 223 | if why == "exception": 224 | six.reraise(*self.last_exception) 225 | 226 | return self.return_value 227 | -------------------------------------------------------------------------------- /byterun/execfile.py: -------------------------------------------------------------------------------- 1 | """Execute files of Python code.""" 2 | 3 | import imp 4 | import os 5 | import sys 6 | import tokenize 7 | 8 | from .pyvm2 import VirtualMachine 9 | 10 | 11 | # This code is ripped off from coverage.py. Define things it expects. 12 | try: 13 | open_source = tokenize.open # pylint: disable=E1101 14 | except: 15 | def open_source(fname): 16 | """Open a source file the best way.""" 17 | return open(fname, "rU") 18 | 19 | NoSource = Exception 20 | 21 | 22 | def exec_code_object(code, env): 23 | vm = VirtualMachine() 24 | vm.run_code(code, f_globals=env) 25 | 26 | 27 | # from coverage.py: 28 | 29 | try: 30 | # In Py 2.x, the builtins were in __builtin__ 31 | BUILTINS = sys.modules['__builtin__'] 32 | except KeyError: 33 | # In Py 3.x, they're in builtins 34 | BUILTINS = sys.modules['builtins'] 35 | 36 | 37 | def rsplit1(s, sep): 38 | """The same as s.rsplit(sep, 1), but works in 2.3""" 39 | parts = s.split(sep) 40 | return sep.join(parts[:-1]), parts[-1] 41 | 42 | 43 | def run_python_module(modulename, args): 44 | """Run a python module, as though with ``python -m name args...``. 45 | 46 | `modulename` is the name of the module, possibly a dot-separated name. 47 | `args` is the argument array to present as sys.argv, including the first 48 | element naming the module being executed. 49 | 50 | """ 51 | openfile = None 52 | glo, loc = globals(), locals() 53 | try: 54 | try: 55 | # Search for the module - inside its parent package, if any - using 56 | # standard import mechanics. 57 | if '.' in modulename: 58 | packagename, name = rsplit1(modulename, '.') 59 | package = __import__(packagename, glo, loc, ['__path__']) 60 | searchpath = package.__path__ 61 | else: 62 | packagename, name = None, modulename 63 | searchpath = None # "top-level search" in imp.find_module() 64 | openfile, pathname, _ = imp.find_module(name, searchpath) 65 | 66 | # Complain if this is a magic non-file module. 67 | if openfile is None and pathname is None: 68 | raise NoSource( 69 | "module does not live in a file: %r" % modulename 70 | ) 71 | 72 | # If `modulename` is actually a package, not a mere module, then we 73 | # pretend to be Python 2.7 and try running its __main__.py script. 74 | if openfile is None: 75 | packagename = modulename 76 | name = '__main__' 77 | package = __import__(packagename, glo, loc, ['__path__']) 78 | searchpath = package.__path__ 79 | openfile, pathname, _ = imp.find_module(name, searchpath) 80 | except ImportError: 81 | _, err, _ = sys.exc_info() 82 | raise NoSource(str(err)) 83 | finally: 84 | if openfile: 85 | openfile.close() 86 | 87 | # Finally, hand the file off to run_python_file for execution. 88 | args[0] = pathname 89 | run_python_file(pathname, args, package=packagename) 90 | 91 | 92 | def run_python_file(filename, args, package=None): 93 | """Run a python file as if it were the main program on the command line. 94 | 95 | `filename` is the path to the file to execute, it need not be a .py file. 96 | `args` is the argument array to present as sys.argv, including the first 97 | element naming the file being executed. `package` is the name of the 98 | enclosing package, if any. 99 | 100 | """ 101 | # Create a module to serve as __main__ 102 | old_main_mod = sys.modules['__main__'] 103 | main_mod = imp.new_module('__main__') 104 | sys.modules['__main__'] = main_mod 105 | main_mod.__file__ = filename 106 | if package: 107 | main_mod.__package__ = package 108 | # TODO(ampere): This may be incorrect if we are overriding builtins 109 | main_mod.__builtins__ = BUILTINS 110 | 111 | # Set sys.argv and the first path element properly. 112 | old_argv = sys.argv 113 | old_path0 = sys.path[0] 114 | sys.argv = args 115 | if package: 116 | sys.path[0] = '' 117 | else: 118 | sys.path[0] = os.path.abspath(os.path.dirname(filename)) 119 | 120 | try: 121 | # Open the source file. 122 | try: 123 | source_file = open_source(filename) 124 | except IOError: 125 | raise NoSource("No file to run: %r" % filename) 126 | 127 | try: 128 | source = source_file.read() 129 | finally: 130 | source_file.close() 131 | 132 | # We have the source. `compile` still needs the last line to be clean, 133 | # so make sure it is, then compile a code object from it. 134 | if not source or source[-1] != '\n': 135 | source += '\n' 136 | code = compile(source, filename, "exec") 137 | 138 | # Execute the source file. 139 | exec_code_object(code, main_mod.__dict__) 140 | finally: 141 | # Restore the old __main__ 142 | sys.modules['__main__'] = old_main_mod 143 | 144 | # Restore the old argv and path 145 | sys.argv = old_argv 146 | sys.path[0] = old_path0 147 | -------------------------------------------------------------------------------- /byterun/pycfg.py: -------------------------------------------------------------------------------- 1 | """Build a Control Flow Graph (CFG) from CPython bytecode. 2 | 3 | A class that builds and provides access to a CFG built from CPython bytecode. 4 | 5 | For a basic introduction to CFGs see the wikipedia article: 6 | http://en.wikipedia.org/wiki/Control_flow_graph 7 | """ 8 | 9 | import bisect 10 | import dis 11 | import itertools 12 | import logging 13 | 14 | 15 | import six 16 | 17 | PY3, PY2 = six.PY3, not six.PY3 18 | 19 | if six.PY3: 20 | byteint = lambda b: b 21 | else: 22 | byteint = ord 23 | 24 | log = logging.getLogger(__name__) 25 | 26 | # The following sets contain instructions with specific branching properties. 27 | 28 | # Untargetted unconditional jumps always jump, but do so to some statically 29 | # unknown location. Examples include, raising exceptions and returning from 30 | # functions: in both cases you are jumping but you cannot statically determine 31 | # to where. 32 | _UNTARGETTED_UNCONDITIONAL_JUMPS = frozenset([ 33 | dis.opmap["BREAK_LOOP"], 34 | dis.opmap["RETURN_VALUE"], 35 | dis.opmap["RAISE_VARARGS"], 36 | ]) 37 | 38 | # Untargetted conditional jumps may jump to a statically unknown location, but 39 | # may allow control to continue to the next instruction. 40 | _UNTARGETTED_CONDITIONAL_JUMPS = frozenset([ 41 | dis.opmap["END_FINALLY"], 42 | dis.opmap["EXEC_STMT"], 43 | dis.opmap["WITH_CLEANUP"], 44 | dis.opmap["IMPORT_NAME"], 45 | dis.opmap["IMPORT_FROM"], 46 | dis.opmap["IMPORT_STAR"], 47 | dis.opmap["CALL_FUNCTION"], 48 | dis.opmap["CALL_FUNCTION_VAR"], 49 | dis.opmap["CALL_FUNCTION_KW"], 50 | dis.opmap["CALL_FUNCTION_VAR_KW"], 51 | dis.opmap["YIELD_VALUE"], # yield is treated as both branching somewhere 52 | # unknown and to the next instruction. 53 | ]) 54 | 55 | # Targetted unconditional jumps always jump to a statically known target 56 | # instruction. 57 | _TARGETTED_UNCONDITIONAL_JUMPS = frozenset([ 58 | dis.opmap["CONTINUE_LOOP"], 59 | dis.opmap["JUMP_FORWARD"], 60 | dis.opmap["JUMP_ABSOLUTE"], 61 | ]) 62 | 63 | # Targetted conditional jumps either jump to a statically known target or they 64 | # continue to the next instruction. 65 | _TARGETTED_CONDITIONAL_JUMPS = frozenset([ 66 | dis.opmap["POP_JUMP_IF_TRUE"], 67 | dis.opmap["POP_JUMP_IF_FALSE"], 68 | dis.opmap["JUMP_IF_TRUE_OR_POP"], 69 | dis.opmap["JUMP_IF_FALSE_OR_POP"], 70 | dis.opmap["FOR_ITER"], 71 | ]) 72 | 73 | _TARGETTED_JUMPS = (_TARGETTED_CONDITIONAL_JUMPS | 74 | _TARGETTED_UNCONDITIONAL_JUMPS) 75 | 76 | _CONDITIONAL_JUMPS = (_TARGETTED_CONDITIONAL_JUMPS | 77 | _UNTARGETTED_CONDITIONAL_JUMPS) 78 | 79 | _UNTARGETTED_JUMPS = (_UNTARGETTED_CONDITIONAL_JUMPS | 80 | _UNTARGETTED_UNCONDITIONAL_JUMPS) 81 | 82 | 83 | def _parse_instructions(code): 84 | """A generator yielding each instruction in code. 85 | 86 | Args: 87 | code: A bytecode string (not a code object). 88 | 89 | Yields: 90 | A triple (opcode, argument or None, offset) for each instruction in code. 91 | Where offset is the byte offset of the beginning of the instruction. 92 | 93 | This is derived from dis.findlabels in the Python standard library. 94 | """ 95 | n = len(code) 96 | i = 0 97 | while i < n: 98 | offset = i 99 | op = byteint(code[i]) 100 | i += 1 101 | oparg = None 102 | if op >= dis.HAVE_ARGUMENT: 103 | oparg = byteint(code[i]) + byteint(code[i+1])*256 104 | i += 2 105 | yield (op, oparg, offset) 106 | 107 | 108 | class InstructionsIndex(object): 109 | """An index of all the instructions in a code object. 110 | 111 | Attributes: 112 | instruction_offsets: A list of instruction offsets. 113 | """ 114 | 115 | def __init__(self, code): 116 | self.instruction_offsets = [i for _, _, i in _parse_instructions(code)] 117 | 118 | def prev(self, offset): 119 | """Return the offset of the previous instruction. 120 | 121 | Args: 122 | offset: The offset of an instruction in the code. 123 | 124 | Returns: 125 | The offset of the instruction immediately before the instruction specified 126 | by the offset argument. 127 | 128 | Raises: 129 | IndexError: If the offset is outside the range of valid instructions. 130 | """ 131 | if offset < 0: 132 | raise IndexError("Instruction offset cannot be less than 0") 133 | if offset > self.instruction_offsets[-1]: 134 | raise IndexError("Instruction offset cannot be greater than " 135 | "the offset of the last instruction") 136 | # Find the rightmost instruction offset that is less than the offset 137 | # argument, this will be the previous instruction because it is closest 138 | # instruction that is before the offset. 139 | return self.instruction_offsets[ 140 | bisect.bisect_left(self.instruction_offsets, offset) - 1] 141 | 142 | def next(self, offset): 143 | """Return the offset of the next instruction. 144 | 145 | Args: 146 | offset: The offset of an instruction in the code. 147 | 148 | Returns: 149 | The offset of the instruction immediately after the instruction specified 150 | by the offset argument. 151 | 152 | Raises: 153 | IndexError: If the offset is outside the range of valid instructions. 154 | """ 155 | if offset < 0: 156 | raise IndexError("Instruction offset cannot be less than 0") 157 | if offset > self.instruction_offsets[-1]: 158 | raise IndexError("Instruction offset cannot be greater than " 159 | "the offset of the last instruction") 160 | # Find the leftmost instruction offset that is greater than the offset 161 | # argument, this will be the next instruction because it is closest 162 | # instruction that is after the offset. 163 | return self.instruction_offsets[ 164 | bisect.bisect_right(self.instruction_offsets, offset)] 165 | 166 | 167 | def _find_jumps(code): 168 | """Detect all offsets in a byte code which are instructions that can jump. 169 | 170 | Args: 171 | code: A bytecode string (not a code object). 172 | 173 | Returns: 174 | A pair of a dict and set. The dict mapping the offsets of jump instructions 175 | to sets with the same semantics as outgoing in Block. The set of all the 176 | jump targets it found. 177 | """ 178 | all_targets = set() 179 | jumps = {} 180 | for op, oparg, i in _parse_instructions(code): 181 | targets = set() 182 | is_jump = False 183 | next_i = i + 1 if oparg is None else i + 3 184 | if oparg is not None: 185 | if op in _TARGETTED_JUMPS: 186 | # Add the known jump target 187 | is_jump = True 188 | if op in dis.hasjrel: 189 | targets.add(next_i+oparg) 190 | all_targets.add(next_i+oparg) 191 | elif op in dis.hasjabs: 192 | targets.add(oparg) 193 | all_targets.add(oparg) 194 | else: 195 | targets.add(None) 196 | 197 | if op in _CONDITIONAL_JUMPS: 198 | # The jump is conditional so add the next instruction as a target 199 | is_jump = True 200 | targets.add(next_i) 201 | all_targets.add(next_i) 202 | if op in _UNTARGETTED_JUMPS: 203 | # The jump is untargetted so add None to mean unknown target 204 | is_jump = True 205 | targets.add(None) 206 | 207 | if is_jump: 208 | jumps[i] = targets 209 | return jumps, all_targets 210 | 211 | 212 | class Block(object): 213 | """A Block instance represents a basic block in the CFG. 214 | 215 | Each basic block has at most one jump instruction which is always at the 216 | end. In this representation we will not add forward jumps to blocks that don't 217 | have them and instead just take a block that has no jump instruction as 218 | implicitly jumping to the next instruction when it reaches the end of the 219 | block. Control may only jump to the beginning of a basic block, so if any 220 | instruction in a basic block executes they all do and they do so in order. 221 | 222 | Attributes: 223 | 224 | begin, end: The beginning and ending (resp) offsets of the basic block in 225 | bytes. 226 | 227 | outgoing: A set of blocks that the last instruction of this basic block can 228 | branch to. A None in this set denotes that there are statically 229 | unknown branch targets (due to exceptions, for instance). 230 | 231 | incoming: A set of blocks that can branch to the beginning of this 232 | basic block. 233 | 234 | code: The code object that contains this basic block. 235 | 236 | This object uses the identity hash and equality. This is correct as there 237 | should never be more than one block object that represents the same actual 238 | basic block. 239 | """ 240 | 241 | def __init__(self, begin, end, code, block_table): 242 | self.outgoing = set() 243 | self.incoming = set() 244 | self._dominators = set() 245 | self._reachable_from = None 246 | self.begin = begin 247 | self.end = end 248 | self.code = code 249 | self.block_table = block_table 250 | 251 | def reachable_from(self, other): 252 | """Return true if self is reachable from other. 253 | 254 | Args: 255 | other: A block. 256 | 257 | Returns: 258 | A boolean 259 | """ 260 | return other in self._reachable_from 261 | 262 | def dominates(self, other): 263 | """Return true if self dominates other. 264 | 265 | Args: 266 | other: A block. 267 | 268 | Returns: 269 | A boolean 270 | """ 271 | # This is an instance of my own class and this inversion makes the interface 272 | # cleaner 273 | # pylint: disable=protected-access 274 | return self in other._dominators 275 | 276 | def get_name(self): 277 | return "{}:{}-{}".format(self.block_table.get_filename(), 278 | self.block_table.get_line(self.begin), 279 | self.block_table.get_line(self.end)) 280 | 281 | def __repr__(self): 282 | return "{}(outgoing={{{}}},incoming={{{}}})".format( 283 | self.get_name(), 284 | ", ".join(b.get_name() for b in self.outgoing), 285 | ", ".join(b.get_name() for b in self.incoming)) 286 | 287 | 288 | class BlockTable(object): 289 | """A table of basic blocks in a single bytecode object. 290 | 291 | A None in an outgoing list means that that block can branch to an unknown 292 | location (usually by returning or raising an exception). At the moment, 293 | continue and break are also treated this way, however it will be possible to 294 | remove them as the static target is known from the enclosing SETUP_LOOP 295 | instruction. 296 | 297 | The algorithm to build the Control Flow Graph (CFG) is the naive algorithm 298 | presented in many compilers classes and probably most compiler text books. We 299 | simply find all the instructions where CFGs end and begin, make sure they 300 | match up (there is a begin after every end), and then build a basic block for 301 | ever range between a beginning and an end. This may not produce the smallest 302 | possible CFG, but it will produce a correct one because every branch point 303 | becomes the end of a basic block and every instruction that is branched to 304 | becomes the beginning of a basic block. 305 | """ 306 | 307 | def __init__(self, code): 308 | """Construct a table with the blocks in the given code object. 309 | 310 | Args: 311 | code: a code object (such as function.func_code) to process. 312 | """ 313 | self.code = code 314 | self.line_offsets, self.lines = zip(*dis.findlinestarts(self.code)) 315 | 316 | instruction_index = InstructionsIndex(code.co_code) 317 | 318 | # Get a map from jump instructions to jump targets and a combined set of all 319 | # targets. 320 | jumps, all_targets = _find_jumps(code.co_code) 321 | 322 | # TODO(ampere): Using dis.findlabels may not be the right 323 | # thing. Specifically it is not clear when the targets of SETUP_* 324 | # instructions should be used to make basic blocks. 325 | 326 | # Make a list of all the directly obvious block begins from the jump targets 327 | # found above and the labels found by dis. 328 | direct_begins = all_targets.union(dis.findlabels(code.co_code)) 329 | 330 | # Any jump instruction must be the end of a basic block. 331 | direct_ends = jumps.viewkeys() 332 | 333 | # The actual sorted list of begins is build using the direct_begins along 334 | # with all instructions that follow a jump instruction. Also the beginning 335 | # of the code is a begin. 336 | begins = [0] + sorted(set(list(direct_begins) + 337 | [instruction_index.next(i) for i in direct_ends 338 | if i < len(code.co_code) - 1])) 339 | # The actual ends are every instruction that proceeds a real block begin and 340 | # the last instruction in the code. Since we included the instruction after 341 | # every jump above this will include every jump and every instruction that 342 | # comes before a target. 343 | ends = ([instruction_index.prev(i) for i in begins if i > 0] + 344 | [instruction_index.instruction_offsets[-1]]) 345 | 346 | # Add targets for the ends of basic blocks that don't have a real jump 347 | # instruction. 348 | for end in ends: 349 | if end not in jumps: 350 | jumps[end] = set([instruction_index.next(end)]) 351 | 352 | # Build a reverse mapping from jump targets to the instructions that jump to 353 | # them. 354 | reversemap = {0: set()} 355 | for (jump, targets) in jumps.items(): 356 | for target in targets: 357 | reversemap.setdefault(target, set()).add(jump) 358 | for begin in begins: 359 | if begin not in reversemap: 360 | reversemap[begin] = set() 361 | 362 | assert len(begins) == len(ends) 363 | 364 | # Build the actual basic blocks by pairing the begins and ends directly. 365 | self._blocks = [Block(begin, end, code=code, block_table=self) 366 | for begin, end in itertools.izip(begins, ends)] 367 | # Build a begins list for use with bisect 368 | self._block_begins = [b.begin for b in self._blocks] 369 | # Fill in incoming and outgoing 370 | for block in self._blocks: 371 | block.outgoing = frozenset(self.get_basic_block(o) if 372 | o is not None else None 373 | for o in jumps[block.end]) 374 | block.incoming = frozenset(self.get_basic_block(o) 375 | for o in reversemap[block.begin]) 376 | # TODO(ampere): Both _dominators and _reachable_from are O(n^2) where n is 377 | # the number of blocks. This could be corrected by using a tree and 378 | # searching down it for lookups. 379 | self._compute_dominators() 380 | # Compute all the reachability information by starting recursion from each 381 | # node. 382 | # TODO(ampere): This could be much more efficient, but graphs are small. 383 | for block in self._blocks: 384 | if not block.incoming: 385 | self._compute_reachable_from(block, frozenset()) 386 | 387 | def get_basic_block(self, index): 388 | """Get the basic block that contains the instruction at the given index.""" 389 | return self._blocks[bisect.bisect_right(self._block_begins, index) - 1] 390 | 391 | def get_line(self, index): 392 | """Get the line number for an instruction. 393 | 394 | Args: 395 | index: The offset of the instruction. 396 | 397 | Returns: 398 | The line number of the specified instruction. 399 | """ 400 | return self.lines[max(bisect.bisect_right(self.line_offsets, index)-1, 0)] 401 | 402 | def get_filename(self): 403 | """Get the filename of the code object used in this table. 404 | 405 | Returns: 406 | The string filename. 407 | """ 408 | return self.code.co_filename 409 | 410 | @staticmethod 411 | def _compute_reachable_from(current, history): 412 | """Compute reachability information starting from current. 413 | 414 | The performs a depth first traversal over the graph and adds information to 415 | Block._reachable_from about what paths reach each node. 416 | 417 | Args: 418 | current: The current node in the traversal. 419 | history: A set of nodes that are on the current path to this node. 420 | """ 421 | orig = current._reachable_from # pylint: disable=protected-access 422 | new = history | (orig or set()) 423 | # The base case is that there is no new information, about this node. This 424 | # comparison is why None is used above; we need to be able to distinguish 425 | # nodes we have never touched from nodes with an empty reachable_from set. 426 | if new != orig: 427 | current._reachable_from = new # pylint: disable=protected-access 428 | for child in current.outgoing: 429 | if child: 430 | # pylint: disable=protected-access 431 | BlockTable._compute_reachable_from(child, history | {current}) 432 | 433 | def reachable_from(self, a, b): 434 | """True if the instruction at a is reachable from the instruction at b.""" 435 | block_a = self.get_basic_block(a) 436 | block_b = self.get_basic_block(b) 437 | if block_a == block_b: 438 | return a >= b 439 | else: 440 | return block_a.reachable_from(block_b) 441 | 442 | def _compute_dominators(self): 443 | """Compute dominators for all nodes by iteration. 444 | """ 445 | # pylint: disable=protected-access 446 | # For accessing Block._dominators 447 | entry = self._blocks[0] 448 | # Initialize dominators for the entry node to itself 449 | entry._dominators = frozenset([entry]) 450 | # Initialize all other nodes to be dominated by all nodes 451 | all_blocks_set = frozenset(self._blocks) 452 | for block in self._blocks[1:]: # all but entry block 453 | block._dominators = all_blocks_set 454 | # Now we perform iteration to solve for the dominators. 455 | while True: 456 | # TODO(ampere): use a worklist here. But graphs are small. 457 | changed = False 458 | for block in self._blocks[1:]: # all but entry block 459 | # Compute new dominator information for block by taking the intersection 460 | # of the dominators of every incoming block and adding itself. 461 | new_dominators = all_blocks_set 462 | for pred in block.incoming: 463 | new_dominators &= pred._dominators 464 | new_dominators |= {block} 465 | # Update only if something changed. 466 | if new_dominators != block._dominators: 467 | block._dominators = new_dominators 468 | changed = True 469 | # If we did a pass without changing anything exit. 470 | if not changed: 471 | break 472 | 473 | def dominates(self, a, b): 474 | """True if the instruction at a dominates the instruction at b.""" 475 | block_a = self.get_basic_block(a) 476 | block_b = self.get_basic_block(b) 477 | if block_a == block_b: 478 | # if they are in the same block domination is the same as instruction 479 | # ordering 480 | return a <= b 481 | else: 482 | return block_a.dominates(block_b) 483 | 484 | def get_ancestors_first_traversal(self): 485 | """Build an ancestors first traversal of the blocks in this table. 486 | 487 | Back edges are detected and handled specially. Specifically, the back edge 488 | is ignored for blocks in the cycle (allowing the cycle to be processed), but 489 | we do not allow the blocks after the loop to come before any block in the 490 | loop. 491 | 492 | Returns: 493 | A list of blocks in the proper order. 494 | """ 495 | # TODO(ampere): This assumes all loops are natural. This may be false, but I 496 | # kinda doubt it. The python compiler is very well behaved. 497 | order = [self._blocks[0]] 498 | # A partially processed block has been added to the order, but is part of a 499 | # loop that has not been fully processed. 500 | partially_processed = set() 501 | worklist = list(self._blocks) 502 | while worklist: 503 | block = worklist.pop(0) 504 | # We can process a block if: 505 | # 1) All forward incoming blocks are in the order 506 | # 2) All partially processed blocks are reachable from this block 507 | forward_incoming = set(b for b in block.incoming 508 | if not block.dominates(b)) 509 | all_forward_incoming_ordered = forward_incoming.issubset(order) 510 | # TODO(ampere): Replace forward_incoming in order check with a counter 511 | # that counts the remaining blocks not in order. Similarly below for 512 | # incoming. 513 | all_partially_processed_reachable = all(b.reachable_from(block) 514 | for b in partially_processed) 515 | if (not all_forward_incoming_ordered or 516 | not all_partially_processed_reachable): 517 | continue 518 | # When a node is processed: 519 | # If all incoming blocks (forward and backward) are in the order add to 520 | # the order or remove from partially_processed as needed 521 | # Otherwise, there are backward incoming blocks that are not in the 522 | # order, and we add block to the order and to partially_processed 523 | # We add children to the work list if we either removed block from 524 | # partially_processed or added it to order. 525 | all_incoming_ordered = block.incoming.issubset(order) 526 | # When adding to the work list remove None outgoing edges since they 527 | # represent unknown targets that we cannot handle. 528 | children = filter(None, block.outgoing) 529 | if all_incoming_ordered: 530 | if block in partially_processed: 531 | # block was waiting on a cycle it is part of, but now the cycle is 532 | # processed. 533 | partially_processed.remove(block) 534 | worklist += children 535 | elif block not in order: 536 | # block is ready to add and is not in the order. 537 | order.append(block) 538 | worklist += children 539 | elif block not in order: 540 | # block is not in the order and is part of a cycle. 541 | partially_processed.add(block) 542 | order.append(block) 543 | worklist += children 544 | return order 545 | 546 | 547 | class CFG(object): 548 | """A Control Flow Graph object. 549 | 550 | The CFG may contain any number of code objects, but edges never go between 551 | code objects. 552 | """ 553 | 554 | def __init__(self): 555 | """Initialize a CFG object.""" 556 | self._block_tables = {} 557 | 558 | def get_block_table(self, code): 559 | """Get (building if needed) the BlockTable for a given code object.""" 560 | if code in self._block_tables: 561 | ret = self._block_tables[code] 562 | else: 563 | ret = BlockTable(code) 564 | self._block_tables[code] = ret 565 | return ret 566 | 567 | def get_basic_block(self, code, index): 568 | """Get a basic block by code object and index.""" 569 | blocktable = self.get_block_table(code) 570 | return blocktable.get_basic_block(index) 571 | 572 | 573 | def _bytecode_repr(code): 574 | """Generate a python expression that evaluates to the bytecode. 575 | 576 | Args: 577 | code: A python code string. 578 | Returns: 579 | A human readable and python parsable expression that gives the bytecode. 580 | """ 581 | ret = [] 582 | for op, oparg, i in _parse_instructions(code): 583 | sb = "dis.opmap['" + dis.opname[op] + "']" 584 | if oparg is not None: 585 | sb += ", " + str(oparg & 255) + ", " + str((oparg >> 8) & 255) 586 | sb += ", # " + str(i) 587 | if oparg is not None: 588 | if op in dis.hasjrel: 589 | sb += ", dest=" + str(i+3+oparg) 590 | elif op in dis.hasjabs: 591 | sb += ", dest=" + str(oparg) 592 | else: 593 | sb += ", arg=" + str(oparg) 594 | ret.append(sb) 595 | return "pycfg._list_to_string([\n " + "\n ".join(ret) + "\n ])" 596 | 597 | 598 | def _list_to_string(lst): 599 | return "".join(chr(c) for c in lst) 600 | -------------------------------------------------------------------------------- /byterun/pyobj.py: -------------------------------------------------------------------------------- 1 | """Implementations of Python fundamental objects for Byterun.""" 2 | 3 | 4 | # TODO(ampere): Add doc strings and remove this. 5 | # pylint: disable=missing-docstring 6 | 7 | import collections 8 | import inspect 9 | import types 10 | 11 | import six 12 | 13 | PY3, PY2 = six.PY3, not six.PY3 14 | 15 | 16 | def make_cell(value): 17 | # Thanks to Alex Gaynor for help with this bit of twistiness. 18 | # Construct an actual cell object by creating a closure right here, 19 | # and grabbing the cell object out of the function we create. 20 | fn = (lambda x: lambda: x)(value) 21 | if PY3: 22 | return fn.__closure__[0] 23 | else: 24 | return fn.func_closure[0] 25 | 26 | 27 | class Function(object): 28 | __slots__ = [ 29 | 'func_code', 'func_name', 'func_defaults', 'func_globals', 30 | 'func_locals', 'func_dict', 'func_closure', 31 | '__name__', '__dict__', '__doc__', 32 | '_vm', '_func', 33 | ] 34 | 35 | CO_OPTIMIZED = 0x0001 36 | CO_NEWLOCALS = 0x0002 37 | CO_VARARGS = 0x0004 38 | CO_VARKEYWORDS = 0x0008 39 | CO_NESTED = 0x0010 40 | CO_GENERATOR = 0x0020 41 | CO_NOFREE = 0x0040 42 | CO_FUTURE_DIVISION = 0x2000 43 | CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 44 | CO_FUTURE_WITH_STATEMENT = 0x8000 45 | CO_FUTURE_PRINT_FUNCTION = 0x10000 46 | CO_FUTURE_UNICODE_LITERALS = 0x20000 47 | 48 | def __init__(self, name, code, globs, defaults, closure, vm): 49 | self._vm = vm 50 | self.func_code = code 51 | self.func_name = self.__name__ = name or code.co_name 52 | self.func_defaults = tuple(defaults) 53 | self.func_globals = globs 54 | self.func_locals = self._vm.frame.f_locals 55 | self.__dict__ = {} 56 | self.func_closure = closure 57 | self.__doc__ = code.co_consts[0] if code.co_consts else None 58 | 59 | # Sometimes, we need a real Python function. This is for that. 60 | kw = { 61 | 'argdefs': self.func_defaults, 62 | } 63 | if closure: 64 | kw['closure'] = tuple(make_cell(0) for _ in closure) 65 | self._func = types.FunctionType(code, globs, **kw) 66 | 67 | def __repr__(self): # pragma: no cover 68 | return '' % ( 69 | self.func_name, id(self) 70 | ) 71 | 72 | def __get__(self, instance, owner): 73 | if instance is not None: 74 | return Method(instance, owner, self) 75 | if PY2: 76 | return Method(None, owner, self) 77 | else: 78 | return self 79 | 80 | def __call__(self, *args, **kwargs): 81 | if PY2 and self.func_name in ['', '', '']: 82 | # D'oh! http://bugs.python.org/issue19611 Py2 doesn't know how to 83 | # inspect set comprehensions, dict comprehensions, or generator 84 | # expressions properly. They are always functions of one argument, 85 | # so just do the right thing. 86 | assert len(args) == 1 and not kwargs, 'Surprising comprehension!' 87 | callargs = {'.0': args[0]} 88 | else: 89 | callargs = inspect.getcallargs(self._func, *args, **kwargs) 90 | frame = self._vm.make_frame( 91 | self.func_code, callargs, self.func_globals, self.func_locals 92 | ) 93 | if self.func_code.co_flags & self.CO_GENERATOR: 94 | gen = Generator(frame, self._vm) 95 | frame.generator = gen 96 | retval = gen 97 | else: 98 | retval = self._vm.run_frame(frame) 99 | return retval 100 | 101 | 102 | class Class(object): 103 | """ 104 | The VM level mirror of python class type objects. 105 | """ 106 | 107 | def __init__(self, name, bases, methods, vm): 108 | self._vm = vm 109 | self.__name__ = name 110 | self.__bases__ = bases 111 | self.__mro__ = self._compute_mro(self) 112 | self.locals = dict(methods) 113 | self.locals['__name__'] = self.__name__ 114 | self.locals['__mro__'] = self.__mro__ 115 | self.locals['__bases__'] = self.__bases__ 116 | 117 | @classmethod 118 | def mro_merge(cls, seqs): 119 | """ 120 | Merge a sequence of MROs into a single resulting MRO. 121 | This code is copied from the following URL with print statments removed. 122 | https://www.python.org/download/releases/2.3/mro/ 123 | """ 124 | res = [] 125 | while True: 126 | nonemptyseqs = [seq for seq in seqs if seq] 127 | if not nonemptyseqs: 128 | return res 129 | for seq in nonemptyseqs: # find merge candidates among seq heads 130 | cand = seq[0] 131 | nothead = [s for s in nonemptyseqs if cand in s[1:]] 132 | if nothead: 133 | cand = None # reject candidate 134 | else: 135 | break 136 | if not cand: 137 | raise TypeError("Illegal inheritance.") 138 | res.append(cand) 139 | for seq in nonemptyseqs: # remove candidate 140 | if seq[0] == cand: 141 | del seq[0] 142 | 143 | @classmethod 144 | def _compute_mro(cls, c): 145 | """ 146 | Compute the class precedence list (mro) according to C3. 147 | This code is copied from the following URL with print statments removed. 148 | https://www.python.org/download/releases/2.3/mro/ 149 | """ 150 | return tuple(cls.mro_merge([[c]] + 151 | [list(base.__mro__) for base in c.__bases__] 152 | + [list(c.__bases__)])) 153 | 154 | def __call__(self, *args, **kw): 155 | return self._vm.make_instance(self, args, kw) 156 | 157 | def __repr__(self): # pragma: no cover 158 | return '' % (self.__name__, id(self)) 159 | 160 | def resolve_attr(self, name): 161 | """ 162 | Find an attribute in self and return it raw. This does not handle 163 | properties or method wrapping. 164 | """ 165 | for base in self.__mro__: 166 | # The following code does a double lookup on the dict, however 167 | # measurements show that this is faster than either a special 168 | # sentinel value or catching KeyError. 169 | # Handle both VM classes and python host environment classes. 170 | if isinstance(base, Class): 171 | if name in base.locals: 172 | return base.locals[name] 173 | else: 174 | if name in base.__dict__: 175 | # Avoid using getattr so we can handle method wrapping 176 | return base.__dict__[name] 177 | raise AttributeError( 178 | "%r class has no attribute %r" % (self.__name__, name) 179 | ) 180 | 181 | def __getattr__(self, name): 182 | val = self.resolve_attr(name) 183 | # Check if we have a descriptor 184 | get = getattr(val, '__get__', None) 185 | if get: 186 | return get(None, self) 187 | # Not a descriptor, return the value. 188 | return val 189 | 190 | 191 | class Object(object): 192 | 193 | def __init__(self, _class, args, kw): 194 | # pylint: disable=protected-access 195 | self._vm = _class._vm 196 | self._class = _class 197 | self.locals = {} 198 | if '__init__' in _class.locals: 199 | _class.locals['__init__'](self, *args, **kw) 200 | 201 | def __repr__(self): # pragma: no cover 202 | return '<%s Instance at 0x%08x>' % (self._class.__name__, id(self)) 203 | 204 | def __getattr__(self, name): 205 | if name in self.locals: 206 | val = self.locals[name] 207 | else: 208 | try: 209 | val = self._class.resolve_attr(name) 210 | except AttributeError: 211 | raise AttributeError( 212 | "%r object has no attribute %r" % 213 | (self._class.__name__, name) 214 | ) 215 | # Check if we have a descriptor 216 | get = getattr(val, '__get__', None) 217 | if get: 218 | return get(self, self._class) 219 | # Not a descriptor, return the value. 220 | return val 221 | 222 | # TODO(ampere): Does this need a __setattr__ and __delattr__ implementation? 223 | 224 | 225 | class Method(object): 226 | 227 | def __init__(self, obj, _class, func): 228 | self.im_self = obj 229 | self.im_class = _class 230 | self.im_func = func 231 | 232 | def __repr__(self): # pragma: no cover 233 | name = "%s.%s" % (self.im_class.__name__, self.im_func.func_name) 234 | if self.im_self is not None: 235 | return '' % (name, self.im_self) 236 | else: 237 | return '' % (name,) 238 | 239 | def __call__(self, *args, **kwargs): 240 | if self.im_self is not None: 241 | return self.im_func(self.im_self, *args, **kwargs) 242 | else: 243 | return self.im_func(*args, **kwargs) 244 | 245 | 246 | class Cell(object): 247 | """A fake cell for closures. 248 | 249 | Closures keep names in scope by storing them not in a frame, but in a 250 | separate object called a cell. Frames share references to cells, and 251 | the LOAD_DEREF and STORE_DEREF opcodes get and set the value from cells. 252 | 253 | This class acts as a cell, though it has to jump through two hoops to make 254 | the simulation complete: 255 | 256 | 1. In order to create actual FunctionType functions, we have to have 257 | actual cell objects, which are difficult to make. See the twisty 258 | double-lambda in __init__. 259 | 260 | 2. Actual cell objects can't be modified, so to implement STORE_DEREF, 261 | we store a one-element list in our cell, and then use [0] as the 262 | actual value. 263 | 264 | """ 265 | 266 | def __init__(self, value): 267 | self.contents = value 268 | 269 | def get(self): 270 | return self.contents 271 | 272 | def set(self, value): 273 | self.contents = value 274 | 275 | 276 | Block = collections.namedtuple("Block", "type, handler, level") 277 | 278 | 279 | class Frame(object): 280 | """ 281 | An interpreter frame. This contains the local value and block 282 | stacks and the associated code and pointer. The most complex usage 283 | is with generators in which a frame is stored and then repeatedly 284 | reactivated. Other than that frames are created executed and then 285 | discarded. 286 | 287 | Attributes: 288 | f_code: The code object this frame is executing. 289 | f_globals: The globals dict used for global name resolution. 290 | f_locals: Similar for locals. 291 | f_builtins: Similar for builtins. 292 | f_back: The frame above self on the stack. 293 | f_lineno: The first line number of the code object. 294 | f_lasti: The instruction pointer. Despite its name (which matches actual 295 | python frames) this points to the next instruction that will be executed. 296 | block_stack: A stack of blocks used to manage exceptions, loops, and 297 | "with"s. 298 | data_stack: The value stack that is used for instruction operands. 299 | generator: None or a Generator object if this frame is a generator frame. 300 | """ 301 | 302 | def __init__(self, f_code, f_globals, f_locals, f_back): 303 | self.f_code = f_code 304 | self.f_globals = f_globals 305 | self.f_locals = f_locals 306 | self.f_back = f_back 307 | if f_back: 308 | self.f_builtins = f_back.f_builtins 309 | else: 310 | self.f_builtins = f_locals['__builtins__'] 311 | if hasattr(self.f_builtins, '__dict__'): 312 | self.f_builtins = self.f_builtins.__dict__ 313 | 314 | self.f_lineno = f_code.co_firstlineno 315 | self.f_lasti = 0 316 | 317 | self.cells = {} 318 | if f_code.co_cellvars: 319 | if not f_back.cells: 320 | f_back.cells = {} 321 | for var in f_code.co_cellvars: 322 | # Make a cell for the variable in our locals, or None. 323 | cell = Cell(self.f_locals.get(var)) 324 | f_back.cells[var] = self.cells[var] = cell 325 | 326 | if f_code.co_freevars: 327 | if not self.cells: 328 | self.cells = {} 329 | for var in f_code.co_freevars: 330 | assert self.cells is not None 331 | assert f_back.cells, "f_back.cells: %r" % (f_back.cells,) 332 | self.cells[var] = f_back.cells[var] 333 | 334 | # The stack holding exception and generator handling information 335 | self.block_stack = [] 336 | # The stack holding input and output of bytecode instructions 337 | self.data_stack = [] 338 | self.generator = None 339 | 340 | def push(self, *vals): 341 | """Push values onto the value stack.""" 342 | self.data_stack.extend(vals) 343 | 344 | def __repr__(self): # pragma: no cover 345 | return '' % ( 346 | id(self), self.f_code.co_filename, self.f_lineno 347 | ) 348 | 349 | def line_number(self): 350 | """Get the current line number the frame is executing.""" 351 | # We don't keep f_lineno up to date, so calculate it based on the 352 | # instruction address and the line number table. 353 | lnotab = self.f_code.co_lnotab 354 | byte_increments = six.iterbytes(lnotab[0::2]) 355 | line_increments = six.iterbytes(lnotab[1::2]) 356 | 357 | byte_num = 0 358 | line_num = self.f_code.co_firstlineno 359 | 360 | for byte_incr, line_incr in zip(byte_increments, line_increments): 361 | byte_num += byte_incr 362 | if byte_num > self.f_lasti: 363 | break 364 | line_num += line_incr 365 | 366 | return line_num 367 | 368 | 369 | class Generator(object): 370 | 371 | def __init__(self, g_frame, vm): 372 | self.gi_frame = g_frame 373 | self.vm = vm 374 | self.first = True 375 | self.finished = False 376 | 377 | def __iter__(self): 378 | return self 379 | 380 | def next(self): 381 | if self.finished: 382 | raise StopIteration 383 | 384 | # Ordinary iteration is like sending None into a generator. 385 | # Push the value onto the frame stack. 386 | if not self.first: 387 | self.gi_frame.push(None) 388 | self.first = False 389 | # To get the next value from an iterator, push its frame onto the 390 | # stack, and let it run. 391 | val = self.vm.resume_frame(self.gi_frame) 392 | if self.finished: 393 | raise StopIteration 394 | return val 395 | 396 | __next__ = next 397 | -------------------------------------------------------------------------------- /byterun/pyvm2.py: -------------------------------------------------------------------------------- 1 | """A pure-Python Python bytecode interpreter.""" 2 | # Based on: 3 | # pyvm2 by Paul Swartz (z3p), from http://www.twistedmatrix.com/users/z3p/ 4 | 5 | 6 | # Disable because there are enough false positives to make it useless 7 | # pylint: disable=unbalanced-tuple-unpacking 8 | # pylint: disable=unpacking-non-sequence 9 | 10 | # TODO(ampere): Add doc strings and remove this. 11 | # pylint: disable=missing-docstring 12 | 13 | from __future__ import print_function, division 14 | import dis 15 | import inspect 16 | import linecache 17 | import logging 18 | import operator 19 | import sys 20 | 21 | import six 22 | from six.moves import reprlib 23 | 24 | from .pyobj import Frame, Block, Method, Object, Function, Class, Generator 25 | 26 | log = logging.getLogger(__name__) 27 | 28 | PY3, PY2 = six.PY3, not six.PY3 29 | 30 | if six.PY3: 31 | byteint = lambda b: b 32 | else: 33 | byteint = ord 34 | 35 | # Create a repr that won't overflow. 36 | repr_obj = reprlib.Repr() 37 | repr_obj.maxother = 120 38 | repper = repr_obj.repr 39 | 40 | 41 | class VirtualMachineError(Exception): 42 | """For raising errors in the operation of the VM.""" 43 | pass 44 | 45 | 46 | class VirtualMachine(object): 47 | 48 | def __init__(self): 49 | # The call stack of frames. 50 | self.frames = [] 51 | # The current frame. 52 | self.frame = None 53 | self.return_value = None 54 | self.last_exception = None 55 | self.vmbuiltins = dict(__builtins__) 56 | self.vmbuiltins["isinstance"] = self.isinstance 57 | # Operator tables. These are overriden by subclasses to replace the 58 | # meta-cyclic implementations. 59 | self.unary_operators = { 60 | 'POSITIVE': operator.pos, 61 | 'NEGATIVE': operator.neg, 62 | 'NOT': operator.not_, 63 | 'CONVERT': repr, 64 | 'INVERT': operator.invert, 65 | } 66 | self.binary_operators = { 67 | 'POWER': pow, 68 | 'MULTIPLY': operator.mul, 69 | 'DIVIDE': getattr(operator, 'div', lambda x, y: None), 70 | 'FLOOR_DIVIDE': operator.floordiv, 71 | 'TRUE_DIVIDE': operator.truediv, 72 | 'MODULO': operator.mod, 73 | 'ADD': operator.add, 74 | 'SUBTRACT': operator.sub, 75 | 'SUBSCR': operator.getitem, 76 | 'LSHIFT': operator.lshift, 77 | 'RSHIFT': operator.rshift, 78 | 'AND': operator.and_, 79 | 'XOR': operator.xor, 80 | 'OR': operator.or_, 81 | } 82 | self.compare_operators = [ 83 | operator.lt, 84 | operator.le, 85 | operator.eq, 86 | operator.ne, 87 | operator.gt, 88 | operator.ge, 89 | lambda x, y: x in y, 90 | lambda x, y: x not in y, 91 | lambda x, y: x is y, 92 | lambda x, y: x is not y, 93 | lambda x, y: issubclass(x, Exception) and issubclass(x, y), 94 | ] 95 | 96 | def top(self): 97 | """Return the value at the top of the stack, with no changes.""" 98 | return self.frame.data_stack[-1] 99 | 100 | def pop(self, i=0): 101 | """Pop a value from the stack. 102 | 103 | Default to the top of the stack, but `i` can be a count from the top 104 | instead. 105 | 106 | """ 107 | return self.frame.data_stack.pop(-1-i) 108 | 109 | def push(self, *vals): 110 | """Push values onto the value stack.""" 111 | self.frame.push(*vals) 112 | 113 | def popn(self, n): 114 | """Pop a number of values from the value stack. 115 | 116 | A list of `n` values is returned, the deepest value first. 117 | 118 | """ 119 | if n: 120 | ret = self.frame.data_stack[-n:] 121 | self.frame.data_stack[-n:] = [] 122 | return ret 123 | else: 124 | return [] 125 | 126 | def peek(self, n): 127 | """ 128 | Get a value `n` entries down in the stack, without changing the stack. 129 | """ 130 | return self.frame.data_stack[-n] 131 | 132 | def jump(self, jump): 133 | """ 134 | Move the bytecode pointer to `jump`, so it will execute next. 135 | 136 | Jump may be the very next instruction and hence already the value of 137 | f_lasti. This is used to notify a subclass when a jump was not taken and 138 | instead we continue to the next instruction. 139 | """ 140 | self.frame.f_lasti = jump 141 | 142 | def push_block(self, type, handler=None, level=None): 143 | if level is None: 144 | level = len(self.frame.data_stack) 145 | self.frame.block_stack.append(Block(type, handler, level)) 146 | 147 | def pop_block(self): 148 | return self.frame.block_stack.pop() 149 | 150 | def make_frame(self, code, callargs={}, f_globals=None, f_locals=None): 151 | # The callargs default is safe because we never modify the dict. 152 | # pylint: disable=dangerous-default-value 153 | log.info("make_frame: code=%r, callargs=%s, f_globals=%r, f_locals=%r", 154 | code, repper(callargs), (type(f_globals), id(f_globals)), 155 | (type(f_locals), id(f_locals))) 156 | if f_globals is not None: 157 | f_globals = f_globals 158 | if f_locals is None: 159 | f_locals = f_globals 160 | elif self.frames: 161 | f_globals = self.frame.f_globals 162 | f_locals = {} 163 | else: 164 | # TODO(ampere): __name__, __doc__, __package__ below are not correct 165 | f_globals = f_locals = { 166 | '__builtins__': self.vmbuiltins, 167 | '__name__': '__main__', 168 | '__doc__': None, 169 | '__package__': None, 170 | } 171 | 172 | # Implement NEWLOCALS flag. See Objects/frameobject.c in CPython. 173 | if code.co_flags & Function.CO_NEWLOCALS: 174 | f_locals = {} 175 | 176 | f_locals.update(callargs) 177 | frame = self.make_frame_with_dicts(code, f_globals, f_locals) 178 | log.info("%r", frame) 179 | return frame 180 | 181 | def push_frame(self, frame): 182 | self.frames.append(frame) 183 | self.frame = frame 184 | 185 | def pop_frame(self): 186 | self.frames.pop() 187 | if self.frames: 188 | self.frame = self.frames[-1] 189 | else: 190 | self.frame = None 191 | 192 | def print_frames(self): 193 | """Print the call stack, for debugging.""" 194 | for f in self.frames: 195 | filename = f.f_code.co_filename 196 | lineno = f.line_number() 197 | print(' File "%s", line %d, in %s' % ( 198 | filename, lineno, f.f_code.co_name 199 | )) 200 | linecache.checkcache(filename) 201 | line = linecache.getline(filename, lineno, f.f_globals) 202 | if line: 203 | print(' ' + line.strip()) 204 | 205 | def resume_frame(self, frame): 206 | frame.f_back = self.frame 207 | log.info("resume_frame: %r", frame) 208 | val = self.run_frame(frame) 209 | frame.f_back = None 210 | return val 211 | 212 | def run_code(self, code, f_globals=None, f_locals=None): 213 | frame = self.make_frame(code, f_globals=f_globals, f_locals=f_locals) 214 | val = self.run_frame(frame) 215 | # Check some invariants 216 | if self.frames: # pragma: no cover 217 | raise VirtualMachineError("Frames left over!") 218 | if self.frame is not None and self.frame.data_stack: # pragma: no cover 219 | raise VirtualMachineError("Data left on stack! %r" % 220 | self.frame.data_stack) 221 | 222 | return val 223 | 224 | def unwind_block(self, block): 225 | if block.type == 'except-handler': 226 | offset = 3 227 | else: 228 | offset = 0 229 | 230 | while len(self.frame.data_stack) > block.level + offset: 231 | self.pop() 232 | 233 | if block.type == 'except-handler': 234 | tb, value, exctype = self.popn(3) 235 | self.last_exception = exctype, value, tb 236 | 237 | def parse_byte_and_args(self): 238 | f = self.frame 239 | opoffset = f.f_lasti 240 | try: 241 | byteCode = byteint(f.f_code.co_code[opoffset]) 242 | except IndexError: 243 | raise VirtualMachineError( 244 | "Bad bytecode offset %d in %s (len=%d)" % 245 | (opoffset, str(f.f_code), len(f.f_code.co_code)) 246 | ) 247 | f.f_lasti += 1 248 | byteName = dis.opname[byteCode] 249 | arg = None 250 | arguments = [] 251 | if byteCode >= dis.HAVE_ARGUMENT: 252 | arg = f.f_code.co_code[f.f_lasti:f.f_lasti+2] 253 | f.f_lasti += 2 254 | intArg = byteint(arg[0]) + (byteint(arg[1]) << 8) 255 | if byteCode in dis.hasconst: 256 | arg = f.f_code.co_consts[intArg] 257 | elif byteCode in dis.hasfree: 258 | if intArg < len(f.f_code.co_cellvars): 259 | arg = f.f_code.co_cellvars[intArg] 260 | else: 261 | var_idx = intArg - len(f.f_code.co_cellvars) 262 | arg = f.f_code.co_freevars[var_idx] 263 | elif byteCode in dis.hasname: 264 | arg = f.f_code.co_names[intArg] 265 | elif byteCode in dis.hasjrel: 266 | arg = f.f_lasti + intArg 267 | elif byteCode in dis.hasjabs: 268 | arg = intArg 269 | elif byteCode in dis.haslocal: 270 | arg = f.f_code.co_varnames[intArg] 271 | else: 272 | arg = intArg 273 | arguments = [arg] 274 | 275 | return byteName, arguments, opoffset 276 | 277 | def log(self, byteName, arguments, opoffset): 278 | # pylint: disable=logging-not-lazy 279 | op = "%d: %s" % (opoffset, byteName) 280 | if arguments: 281 | op += " %r" % (arguments[0],) 282 | indent = " "*(len(self.frames)-1) 283 | stack_rep = repper(self.frame.data_stack) 284 | block_stack_rep = repper(self.frame.block_stack) 285 | 286 | log.info(" %sdata: %s" % (indent, stack_rep)) 287 | log.info(" %sblks: %s" % (indent, block_stack_rep)) 288 | log.info("%s%s" % (indent, op)) 289 | 290 | def dispatch(self, byteName, arguments): 291 | why = None 292 | try: 293 | if byteName.startswith('UNARY_'): 294 | self.unaryOperator(byteName[6:]) 295 | elif byteName.startswith('BINARY_'): 296 | self.binaryOperator(byteName[7:]) 297 | elif byteName.startswith('INPLACE_'): 298 | self.inplaceOperator(byteName[8:]) 299 | elif 'SLICE+' in byteName: 300 | self.sliceOperator(byteName) 301 | else: 302 | # dispatch 303 | bytecode_fn = getattr(self, 'byte_%s' % byteName, None) 304 | if not bytecode_fn: # pragma: no cover 305 | raise VirtualMachineError( 306 | "unknown bytecode type: %s" % byteName 307 | ) 308 | why = bytecode_fn(*arguments) 309 | except: # pylint: disable=bare-except 310 | # deal with exceptions encountered while executing the op. 311 | self.last_exception = sys.exc_info()[:2] + (None,) 312 | log.exception("Caught exception during execution") 313 | why = 'exception' 314 | 315 | return why 316 | 317 | def manage_block_stack(self, why): 318 | assert why != 'yield' 319 | 320 | block = self.frame.block_stack[-1] 321 | if block.type == 'loop' and why == 'continue': 322 | self.jump(self.return_value) 323 | why = None 324 | return why 325 | 326 | self.pop_block() 327 | self.unwind_block(block) 328 | 329 | if block.type == 'loop' and why == 'break': 330 | why = None 331 | self.jump(block.handler) 332 | return why 333 | 334 | if PY2: 335 | if ( 336 | block.type == 'finally' or 337 | (block.type == 'setup-except' and why == 'exception') or 338 | block.type == 'with' 339 | ): 340 | if why == 'exception': 341 | exctype, value, tb = self.last_exception 342 | self.push(tb, value, exctype) 343 | else: 344 | if why in ('return', 'continue'): 345 | self.push(self.return_value) 346 | self.push(why) 347 | 348 | why = None 349 | self.jump(block.handler) 350 | return why 351 | 352 | elif PY3: 353 | if ( 354 | why == 'exception' and 355 | block.type in ['setup-except', 'finally'] 356 | ): 357 | self.push_block('except-handler') 358 | exctype, value, tb = self.last_exception 359 | self.push(tb, value, exctype) 360 | # PyErr_Normalize_Exception goes here 361 | self.push(tb, value, exctype) 362 | why = None 363 | self.jump(block.handler) 364 | return why 365 | 366 | elif block.type == 'finally': 367 | if why in ('return', 'continue'): 368 | self.push(self.return_value) 369 | self.push(why) 370 | 371 | why = None 372 | self.jump(block.handler) 373 | return why 374 | 375 | return why 376 | 377 | def run_instruction(self): 378 | """Run one instruction in the current frame. 379 | 380 | Return None if the frame should continue executing otherwise return the 381 | reason it should stop. 382 | """ 383 | frame = self.frame 384 | byteName, arguments, opoffset = self.parse_byte_and_args() 385 | if log.isEnabledFor(logging.INFO): 386 | self.log(byteName, arguments, opoffset) 387 | 388 | # When unwinding the block stack, we need to keep track of why we 389 | # are doing it. 390 | why = self.dispatch(byteName, arguments) 391 | if why == 'exception': 392 | # TODO: ceval calls PyTraceBack_Here, not sure what that does. 393 | pass 394 | 395 | if why == 'reraise': 396 | why = 'exception' 397 | 398 | if why != 'yield': 399 | while why and frame.block_stack: 400 | # Deal with any block management we need to do. 401 | why = self.manage_block_stack(why) 402 | 403 | return why 404 | 405 | def run_frame(self, frame): 406 | """Run a frame until it returns (somehow). 407 | 408 | Exceptions are raised, the return value is returned. 409 | """ 410 | self.push_frame(frame) 411 | while True: 412 | why = self.run_instruction() 413 | if why: 414 | break 415 | self.pop_frame() 416 | 417 | if why == 'exception': 418 | six.reraise(*self.last_exception) 419 | 420 | return self.return_value 421 | 422 | ## Builders for objects that subclasses may want to replace with subclasses 423 | 424 | def make_instance(self, cls, args, kw): 425 | """ 426 | Create an instance of the given class with the given constructor args. 427 | """ 428 | return Object(cls, args, kw) 429 | 430 | def make_class(self, name, bases, methods): 431 | """ 432 | Create a class with the name bases and methods given. 433 | """ 434 | return Class(name, bases, methods, self) 435 | 436 | def make_function(self, name, code, globs, defaults, closure): 437 | """ 438 | Create a function or closure given the arguments. 439 | """ 440 | return Function(name, code, globs, defaults, closure, self) 441 | 442 | def make_frame_with_dicts(self, code, f_globals, f_locals): 443 | """ 444 | Create a frame with the given code, globals, and locals. 445 | """ 446 | return Frame(code, f_globals, f_locals, self.frame) 447 | 448 | ## Built-in overrides 449 | 450 | def isinstance(self, obj, cls): 451 | if isinstance(obj, Object): 452 | # pylint: disable=protected-access 453 | return issubclass(obj._class, cls) 454 | elif isinstance(cls, Class): 455 | return False 456 | else: 457 | return isinstance(obj, cls) 458 | 459 | ## Abstraction hooks 460 | 461 | def load_constant(self, value): 462 | """ 463 | Called when the constant value is loaded onto the stack. 464 | The returned value is pushed onto the stack instead. 465 | """ 466 | return value 467 | 468 | def get_locals_dict(self): 469 | """Get a real python dict of the locals.""" 470 | return self.frame.f_locals 471 | 472 | def get_locals_dict_bytecode(self): 473 | """Get a possibly abstract bytecode level representation of the locals. 474 | """ 475 | return self.frame.f_locals 476 | 477 | def set_locals_dict_bytecode(self, lcls): 478 | """Set the locals from a possibly abstract bytecode level dict. 479 | """ 480 | self.frame.f_locals = lcls 481 | 482 | def get_globals_dict(self): 483 | """Get a real python dict of the globals.""" 484 | return self.frame.f_globals 485 | 486 | def load_local(self, name): 487 | """ 488 | Called when a local is loaded onto the stack. 489 | The returned value is pushed onto the stack instead of the actual loaded 490 | value. 491 | """ 492 | return self.frame.f_locals[name] 493 | 494 | def load_builtin(self, name): 495 | return self.frame.f_builtins[name] 496 | 497 | def load_global(self, name): 498 | """ 499 | Same as load_local except for globals. 500 | """ 501 | return self.frame.f_globals[name] 502 | 503 | def load_deref(self, name): 504 | """ 505 | Same as load_local except for closure cells. 506 | """ 507 | return self.frame.cells[name].get() 508 | 509 | def store_local(self, name, value): 510 | """ 511 | Called when a local is written. 512 | The returned value is stored instead of the value on the stack. 513 | """ 514 | self.frame.f_locals[name] = value 515 | 516 | def store_deref(self, name, value): 517 | """ 518 | Same as store_local except for closure cells. 519 | """ 520 | self.frame.cells[name].set(value) 521 | 522 | def del_local(self, name): 523 | """ 524 | Called when a local is deleted. 525 | """ 526 | del self.frame.f_locals[name] 527 | 528 | def load_attr(self, obj, attr): 529 | """ 530 | Perform the actual attribute load on an object. This must support all 531 | objects that may appear in the VM. This defaults to just get attr. 532 | """ 533 | return getattr(obj, attr) 534 | 535 | def store_attr(self, obj, attr, value): 536 | """ 537 | Same as load_attr except for setting attributes. Defaults to setattr. 538 | """ 539 | setattr(obj, attr, value) 540 | 541 | def del_attr(self, obj, attr): 542 | """ 543 | Same as load_attr except for deleting attributes. Defaults to delattr. 544 | """ 545 | delattr(obj, attr) 546 | 547 | def build_tuple(self, content): 548 | """ 549 | Create a VM tuple from the given sequence. 550 | The returned object must support the tuple interface. 551 | """ 552 | return tuple(content) 553 | 554 | def build_list(self, content): 555 | """ 556 | Create a VM list from the given sequence. 557 | The returned object must support the list interface. 558 | """ 559 | return list(content) 560 | 561 | def build_set(self, content): 562 | """ 563 | Create a VM set from the given sequence. 564 | The returned object must support the set interface. 565 | """ 566 | return set(content) 567 | 568 | def build_map(self): 569 | """ 570 | Create an empty VM dict. 571 | The returned object must support the dict interface. 572 | """ 573 | return dict() 574 | 575 | def store_subscr(self, obj, subscr, val): 576 | obj[subscr] = val 577 | 578 | def del_subscr(self, obj, subscr): 579 | del obj[subscr] 580 | 581 | ## Stack manipulation 582 | 583 | def byte_LOAD_CONST(self, const): 584 | self.push(self.load_constant(const)) 585 | 586 | def byte_POP_TOP(self): 587 | self.pop() 588 | 589 | def byte_DUP_TOP(self): 590 | self.push(self.top()) 591 | 592 | def byte_DUP_TOPX(self, count): 593 | items = self.popn(count) 594 | for i in [1, 2]: 595 | self.push(*items) 596 | 597 | def byte_DUP_TOP_TWO(self): 598 | # Py3 only 599 | a, b = self.popn(2) 600 | self.push(a, b, a, b) 601 | 602 | def byte_ROT_TWO(self): 603 | a, b = self.popn(2) 604 | self.push(b, a) 605 | 606 | def byte_ROT_THREE(self): 607 | a, b, c = self.popn(3) 608 | self.push(c, a, b) 609 | 610 | def byte_ROT_FOUR(self): 611 | a, b, c, d = self.popn(4) 612 | self.push(d, a, b, c) 613 | 614 | ## Names 615 | 616 | def byte_LOAD_NAME(self, name): 617 | try: 618 | val = self.load_local(name) 619 | except KeyError: 620 | try: 621 | val = self.load_global(name) 622 | except KeyError: 623 | try: 624 | val = self.load_builtin(name) 625 | except KeyError: 626 | raise NameError("name '%s' is not defined" % name) 627 | self.push(val) 628 | 629 | def byte_STORE_NAME(self, name): 630 | self.store_local(name, self.pop()) 631 | 632 | def byte_DELETE_NAME(self, name): 633 | self.del_local(name) 634 | 635 | def byte_LOAD_FAST(self, name): 636 | try: 637 | val = self.load_local(name) 638 | log.info("LOAD_FAST: %s from %r -> %r", name, self.frame, val) 639 | except KeyError: 640 | raise UnboundLocalError( 641 | "local variable '%s' referenced before assignment" % name 642 | ) 643 | self.push(val) 644 | 645 | def byte_STORE_FAST(self, name): 646 | self.byte_STORE_NAME(name) 647 | 648 | def byte_DELETE_FAST(self, name): 649 | self.byte_DELETE_NAME(name) 650 | 651 | def byte_LOAD_GLOBAL(self, name): 652 | try: 653 | val = self.load_global(name) 654 | except KeyError: 655 | try: 656 | val = self.load_builtin(name) 657 | except KeyError: 658 | raise NameError("global name '%s' is not defined" % name) 659 | self.push(val) 660 | 661 | def byte_LOAD_DEREF(self, name): 662 | self.push(self.load_deref(name)) 663 | 664 | def byte_STORE_DEREF(self, name): 665 | self.store_deref(name, self.pop()) 666 | 667 | def byte_LOAD_LOCALS(self): 668 | self.push(self.get_locals_dict_bytecode()) 669 | 670 | ## Operators 671 | 672 | def unaryOperator(self, op): 673 | x = self.pop() 674 | self.push(self.unary_operators[op](x)) 675 | 676 | def binaryOperator(self, op): 677 | x, y = self.popn(2) 678 | self.push(self.binary_operators[op](x, y)) 679 | 680 | def inplaceOperator(self, op): 681 | x, y = self.popn(2) 682 | if op == 'POWER': 683 | x **= y 684 | elif op == 'MULTIPLY': 685 | x *= y 686 | elif op in ['DIVIDE', 'FLOOR_DIVIDE']: 687 | x //= y 688 | elif op == 'TRUE_DIVIDE': 689 | x /= y 690 | elif op == 'MODULO': 691 | x %= y 692 | elif op == 'ADD': 693 | x += y 694 | elif op == 'SUBTRACT': 695 | x -= y 696 | elif op == 'LSHIFT': 697 | x <<= y 698 | elif op == 'RSHIFT': 699 | x >>= y 700 | elif op == 'AND': 701 | x &= y 702 | elif op == 'XOR': 703 | x ^= y 704 | elif op == 'OR': 705 | x |= y 706 | else: # pragma: no cover 707 | raise VirtualMachineError("Unknown in-place operator: %r" % op) 708 | self.push(x) 709 | 710 | def sliceOperator(self, op): 711 | start = 0 712 | end = None # we will take this to mean end 713 | op, count = op[:-2], int(op[-1]) 714 | if count == 1: 715 | start = self.pop() 716 | elif count == 2: 717 | end = self.pop() 718 | elif count == 3: 719 | end = self.pop() 720 | start = self.pop() 721 | l = self.pop() 722 | if end is None: 723 | end = len(l) 724 | if op.startswith('STORE_'): 725 | l[start:end] = self.pop() 726 | elif op.startswith('DELETE_'): 727 | del l[start:end] 728 | else: 729 | self.push(l[start:end]) 730 | 731 | def byte_COMPARE_OP(self, opnum): 732 | x, y = self.popn(2) 733 | self.push(self.compare_operators[opnum](x, y)) 734 | 735 | ## Attributes and indexing 736 | 737 | def byte_LOAD_ATTR(self, attr): 738 | obj = self.pop() 739 | log.info("LOAD_ATTR: %r %s", type(obj), attr) 740 | val = self.load_attr(obj, attr) 741 | self.push(val) 742 | 743 | def byte_STORE_ATTR(self, name): 744 | val, obj = self.popn(2) 745 | self.store_attr(obj, name, val) 746 | 747 | def byte_DELETE_ATTR(self, name): 748 | obj = self.pop() 749 | self.del_attr(obj, name) 750 | 751 | def byte_STORE_SUBSCR(self): 752 | val, obj, subscr = self.popn(3) 753 | self.store_subscr(obj, subscr, val) 754 | 755 | def byte_DELETE_SUBSCR(self): 756 | obj, subscr = self.popn(2) 757 | self.del_subscr(obj, subscr) 758 | 759 | ## Building 760 | 761 | def byte_BUILD_TUPLE(self, count): 762 | elts = self.popn(count) 763 | self.push(self.build_tuple(elts)) 764 | 765 | def byte_BUILD_LIST(self, count): 766 | elts = self.popn(count) 767 | self.push(self.build_list(elts)) 768 | 769 | def byte_BUILD_SET(self, count): 770 | # TODO: Not documented in Py2 docs. 771 | elts = self.popn(count) 772 | self.push(self.build_set(elts)) 773 | 774 | def byte_BUILD_MAP(self, size): 775 | # size is ignored. 776 | self.push(self.build_map()) 777 | 778 | def byte_STORE_MAP(self): 779 | the_map, val, key = self.popn(3) 780 | the_map[key] = val 781 | self.push(the_map) 782 | 783 | def byte_UNPACK_SEQUENCE(self, count): 784 | seq = self.pop() 785 | for x in reversed(seq): 786 | self.push(x) 787 | 788 | def byte_BUILD_SLICE(self, count): 789 | if count == 2: 790 | x, y = self.popn(2) 791 | self.push(slice(x, y)) 792 | elif count == 3: 793 | x, y, z = self.popn(3) 794 | self.push(slice(x, y, z)) 795 | else: # pragma: no cover 796 | raise VirtualMachineError("Strange BUILD_SLICE count: %r" % count) 797 | 798 | def byte_LIST_APPEND(self, count): 799 | val = self.pop() 800 | the_list = self.peek(count) 801 | the_list.append(val) 802 | 803 | def byte_SET_ADD(self, count): 804 | val = self.pop() 805 | the_set = self.peek(count) 806 | the_set.add(val) 807 | 808 | def byte_MAP_ADD(self, count): 809 | val, key = self.popn(2) 810 | the_map = self.peek(count) 811 | the_map[key] = val 812 | 813 | ## Printing 814 | 815 | if 0: # Only used in the interactive interpreter, not in modules. 816 | def byte_PRINT_EXPR(self): 817 | print(self.pop()) 818 | 819 | def byte_PRINT_ITEM(self): 820 | item = self.pop() 821 | self.print_item(item) 822 | 823 | def byte_PRINT_ITEM_TO(self): 824 | to = self.pop() 825 | item = self.pop() 826 | self.print_item(item, to) 827 | 828 | def byte_PRINT_NEWLINE(self): 829 | self.print_newline() 830 | 831 | def byte_PRINT_NEWLINE_TO(self): 832 | to = self.pop() 833 | self.print_newline(to) 834 | 835 | def print_item(self, item, to=None): 836 | if to is None: 837 | to = sys.stdout 838 | if to.softspace: 839 | print(" ", end="", file=to) 840 | to.softspace = 0 841 | print(item, end="", file=to) 842 | if isinstance(item, str): 843 | if (not item) or (not item[-1].isspace()) or (item[-1] == " "): 844 | to.softspace = 1 845 | else: 846 | to.softspace = 1 847 | 848 | def print_newline(self, to=None): 849 | if to is None: 850 | to = sys.stdout 851 | print("", file=to) 852 | to.softspace = 0 853 | 854 | ## Jumps 855 | 856 | def byte_JUMP_FORWARD(self, jump): 857 | self.jump(jump) 858 | 859 | def byte_JUMP_ABSOLUTE(self, jump): 860 | self.jump(jump) 861 | 862 | if 0: # Not in py2.7 863 | def byte_JUMP_IF_TRUE(self, jump): 864 | val = self.top() 865 | if val: 866 | self.jump(jump) 867 | else: 868 | self.jump(self.frame.f_lasti) 869 | 870 | def byte_JUMP_IF_FALSE(self, jump): 871 | val = self.top() 872 | if not val: 873 | self.jump(jump) 874 | else: 875 | self.jump(self.frame.f_lasti) 876 | 877 | def byte_POP_JUMP_IF_TRUE(self, jump): 878 | val = self.pop() 879 | if val: 880 | self.jump(jump) 881 | else: 882 | self.jump(self.frame.f_lasti) 883 | 884 | def byte_POP_JUMP_IF_FALSE(self, jump): 885 | val = self.pop() 886 | if not val: 887 | self.jump(jump) 888 | else: 889 | self.jump(self.frame.f_lasti) 890 | 891 | def byte_JUMP_IF_TRUE_OR_POP(self, jump): 892 | val = self.top() 893 | if val: 894 | self.jump(jump) 895 | else: 896 | self.pop() 897 | self.jump(self.frame.f_lasti) 898 | 899 | def byte_JUMP_IF_FALSE_OR_POP(self, jump): 900 | val = self.top() 901 | if not val: 902 | self.jump(jump) 903 | else: 904 | self.pop() 905 | self.jump(self.frame.f_lasti) 906 | 907 | ## Blocks 908 | 909 | def byte_SETUP_LOOP(self, dest): 910 | self.push_block('loop', dest) 911 | 912 | def byte_GET_ITER(self): 913 | self.push(iter(self.pop())) 914 | 915 | def byte_FOR_ITER(self, jump): 916 | iterobj = self.top() 917 | try: 918 | v = next(iterobj) 919 | self.push(v) 920 | self.jump(self.frame.f_lasti) 921 | except StopIteration: 922 | self.pop() 923 | self.jump(jump) 924 | 925 | def byte_BREAK_LOOP(self): 926 | return 'break' 927 | 928 | def byte_CONTINUE_LOOP(self, dest): 929 | # This is a trick with the return value. 930 | # While unrolling blocks, continue and return both have to preserve 931 | # state as the finally blocks are executed. For continue, it's 932 | # where to jump to, for return, it's the value to return. It gets 933 | # pushed on the stack for both, so continue puts the jump destination 934 | # into return_value. 935 | self.return_value = dest 936 | return 'continue' 937 | 938 | def byte_SETUP_EXCEPT(self, dest): 939 | self.push_block('setup-except', dest) 940 | 941 | def byte_SETUP_FINALLY(self, dest): 942 | self.push_block('finally', dest) 943 | 944 | def byte_END_FINALLY(self): 945 | v = self.pop() 946 | if isinstance(v, str): 947 | why = v 948 | if why in ('return', 'continue'): 949 | self.return_value = self.pop() 950 | if why == 'silenced': # PY3 951 | block = self.pop_block() 952 | assert block.type == 'except-handler' 953 | self.unwind_block(block) 954 | why = None 955 | elif v is None: 956 | why = None 957 | elif issubclass(v, BaseException): 958 | exctype = v 959 | val = self.pop() 960 | tb = self.pop() 961 | self.last_exception = (exctype, val, tb) 962 | why = 'reraise' 963 | else: # pragma: no cover 964 | raise VirtualMachineError("Confused END_FINALLY") 965 | return why 966 | 967 | def byte_POP_BLOCK(self): 968 | self.pop_block() 969 | 970 | if PY2: 971 | def byte_RAISE_VARARGS(self, argc): 972 | # NOTE: the dis docs are completely wrong about the order of the 973 | # operands on the stack! 974 | exctype = val = tb = None 975 | if argc == 0: 976 | exctype, val, tb = self.last_exception 977 | elif argc == 1: 978 | exctype = self.pop() 979 | elif argc == 2: 980 | val = self.pop() 981 | exctype = self.pop() 982 | elif argc == 3: 983 | tb = self.pop() 984 | val = self.pop() 985 | exctype = self.pop() 986 | 987 | # There are a number of forms of "raise", normalize them somewhat. 988 | if isinstance(exctype, BaseException): 989 | val = exctype 990 | exctype = type(val) 991 | 992 | self.last_exception = (exctype, val, tb) 993 | 994 | if tb: 995 | return 'reraise' 996 | else: 997 | return 'exception' 998 | 999 | elif PY3: 1000 | def byte_RAISE_VARARGS(self, argc): 1001 | cause = exc = None 1002 | if argc == 2: 1003 | cause = self.pop() 1004 | exc = self.pop() 1005 | elif argc == 1: 1006 | exc = self.pop() 1007 | return self.do_raise(exc, cause) 1008 | 1009 | def do_raise(self, exc, cause): 1010 | if exc is None: # reraise 1011 | exc_type, val, tb = self.last_exception 1012 | if exc_type is None: 1013 | return 'exception' # error 1014 | else: 1015 | return 'reraise' 1016 | 1017 | elif type(exc) == type: 1018 | # As in `raise ValueError` 1019 | exc_type = exc 1020 | val = exc() # Make an instance. 1021 | elif isinstance(exc, BaseException): 1022 | # As in `raise ValueError('foo')` 1023 | exc_type = type(exc) 1024 | val = exc 1025 | else: 1026 | return 'exception' # error 1027 | 1028 | # If you reach this point, you're guaranteed that 1029 | # val is a valid exception instance and exc_type is its class. 1030 | # Now do a similar thing for the cause, if present. 1031 | if cause: 1032 | if type(cause) == type: 1033 | cause = cause() 1034 | elif not isinstance(cause, BaseException): 1035 | return 'exception' # error 1036 | 1037 | val.__cause__ = cause 1038 | 1039 | self.last_exception = exc_type, val, val.__traceback__ 1040 | return 'exception' 1041 | 1042 | def byte_POP_EXCEPT(self): 1043 | block = self.pop_block() 1044 | if block.type != 'except-handler': 1045 | raise Exception("popped block is not an except handler") 1046 | self.unwind_block(block) 1047 | 1048 | def byte_SETUP_WITH(self, dest): 1049 | ctxmgr = self.pop() 1050 | self.push(ctxmgr.__exit__) 1051 | ctxmgr_obj = ctxmgr.__enter__() 1052 | if PY2: 1053 | self.push_block('with', dest) 1054 | elif PY3: 1055 | self.push_block('finally', dest) 1056 | self.push(ctxmgr_obj) 1057 | 1058 | def byte_WITH_CLEANUP(self): 1059 | # The code here does some weird stack manipulation: the exit function 1060 | # is buried in the stack, and where depends on what's on top of it. 1061 | # Pull out the exit function, and leave the rest in place. 1062 | v = w = None 1063 | u = self.top() 1064 | if u is None: 1065 | exit_func = self.pop(1) 1066 | elif isinstance(u, str): 1067 | if u in ('return', 'continue'): 1068 | exit_func = self.pop(2) 1069 | else: 1070 | exit_func = self.pop(1) 1071 | u = None 1072 | elif issubclass(u, BaseException): 1073 | if PY2: 1074 | w, v, u = self.popn(3) 1075 | exit_func = self.pop() 1076 | self.push(w, v, u) 1077 | elif PY3: 1078 | w, v, u = self.popn(3) 1079 | tp, exc, tb = self.popn(3) 1080 | exit_func = self.pop() 1081 | self.push(tp, exc, tb) 1082 | self.push(None) 1083 | self.push(w, v, u) 1084 | block = self.pop_block() 1085 | assert block.type == 'except-handler' 1086 | self.push_block(block.type, block.handler, block.level-1) 1087 | else: # pragma: no cover 1088 | raise VirtualMachineError("Confused WITH_CLEANUP") 1089 | exit_ret = exit_func(u, v, w) 1090 | err = (u is not None) and bool(exit_ret) 1091 | if err: 1092 | # An error occurred, and was suppressed 1093 | if PY2: 1094 | self.popn(3) 1095 | self.push(None) 1096 | elif PY3: 1097 | self.push('silenced') 1098 | 1099 | ## Functions 1100 | 1101 | def byte_MAKE_FUNCTION(self, argc): 1102 | if PY3: 1103 | name = self.pop() 1104 | else: 1105 | name = None 1106 | code = self.pop() 1107 | defaults = self.popn(argc) 1108 | globs = self.get_globals_dict() 1109 | fn = self.make_function(name, code, globs, defaults, None) 1110 | self.push(fn) 1111 | 1112 | def byte_LOAD_CLOSURE(self, name): 1113 | self.push(self.frame.cells[name]) 1114 | 1115 | def byte_MAKE_CLOSURE(self, argc): 1116 | if PY3: 1117 | # TODO: the py3 docs don't mention this change. 1118 | name = self.pop() 1119 | else: 1120 | name = None 1121 | closure, code = self.popn(2) 1122 | defaults = self.popn(argc) 1123 | globs = self.get_globals_dict() 1124 | fn = self.make_function(None, code, globs, defaults, closure) 1125 | self.push(fn) 1126 | 1127 | def byte_CALL_FUNCTION(self, arg): 1128 | return self.call_function_from_stack(arg, [], {}) 1129 | 1130 | def byte_CALL_FUNCTION_VAR(self, arg): 1131 | args = self.pop() 1132 | return self.call_function_from_stack(arg, args, {}) 1133 | 1134 | def byte_CALL_FUNCTION_KW(self, arg): 1135 | kwargs = self.pop() 1136 | return self.call_function_from_stack(arg, [], kwargs) 1137 | 1138 | def byte_CALL_FUNCTION_VAR_KW(self, arg): 1139 | args, kwargs = self.popn(2) 1140 | return self.call_function_from_stack(arg, args, kwargs) 1141 | 1142 | def call_function_from_stack(self, arg, args, kwargs): 1143 | lenKw, lenPos = divmod(arg, 256) 1144 | namedargs = {} 1145 | for i in range(lenKw): 1146 | key, val = self.popn(2) 1147 | namedargs[key] = val 1148 | namedargs.update(kwargs) 1149 | posargs = self.popn(lenPos) 1150 | posargs.extend(args) 1151 | func = self.pop() 1152 | self.push(self.call_function(func, posargs, namedargs)) 1153 | 1154 | def call_function(self, func, posargs, namedargs=None): 1155 | """Call a VM function with the given arguments and return the result. 1156 | 1157 | This is were subclass override should occur as well. 1158 | 1159 | Args: 1160 | func: The function to call. 1161 | posargs: The positional arguments. 1162 | namedargs: The keyword arguments (defaults to {}). 1163 | Returns: 1164 | The return value of the function. 1165 | """ 1166 | namedargs = namedargs or {} 1167 | frame = self.frame 1168 | if hasattr(func, 'im_func'): 1169 | # Methods get self as an implicit first parameter. 1170 | if func.im_self: 1171 | posargs.insert(0, func.im_self) 1172 | # The first parameter must be the correct type. 1173 | if not self.isinstance(posargs[0], func.im_class): 1174 | raise TypeError( 1175 | 'unbound method %s() must be called with %s instance ' 1176 | 'as first argument (got %s instance instead)' % ( 1177 | func.im_func.func_name, 1178 | func.im_class.__name__, 1179 | type(posargs[0]).__name__, 1180 | ) 1181 | ) 1182 | func = func.im_func 1183 | return func(*posargs, **namedargs) 1184 | 1185 | def byte_RETURN_VALUE(self): 1186 | self.return_value = self.pop() 1187 | if self.frame.generator: 1188 | self.frame.generator.finished = True 1189 | return "return" 1190 | 1191 | def byte_YIELD_VALUE(self): 1192 | self.return_value = self.pop() 1193 | return "yield" 1194 | 1195 | ## Importing 1196 | 1197 | def import_name(self, name, fromlist, level): 1198 | """Import the module and return the module object.""" 1199 | return __import__(name, self.get_globals_dict(), self.get_locals_dict(), 1200 | fromlist, level) 1201 | 1202 | def get_module_attributes(self, mod): 1203 | """Return the modules members as a dict.""" 1204 | return {name: getattr(mod, name) for name in dir(mod)} 1205 | 1206 | def byte_IMPORT_NAME(self, name): 1207 | level, fromlist = self.popn(2) 1208 | frame = self.frame 1209 | self.push(self.import_name(name, fromlist, level)) 1210 | 1211 | def byte_IMPORT_STAR(self): 1212 | # TODO: this doesn't use __all__ properly. 1213 | mod = self.pop() 1214 | attrs = self.get_module_attributes(mod) 1215 | for attr, val in attrs.iteritems(): 1216 | if attr[0] != '_': 1217 | self.store_local(attr, val) 1218 | 1219 | def byte_IMPORT_FROM(self, name): 1220 | mod = self.top() 1221 | attrs = self.get_module_attributes(mod) 1222 | self.push(attrs[name]) 1223 | 1224 | ## And the rest... 1225 | 1226 | def byte_EXEC_STMT(self): 1227 | stmt, globs, locs = self.popn(3) 1228 | six.exec_(stmt, globs, locs) 1229 | 1230 | def byte_BUILD_CLASS(self): 1231 | name, bases, methods = self.popn(3) 1232 | self.push(self.make_class(name, bases, methods)) 1233 | 1234 | def byte_LOAD_BUILD_CLASS(self): 1235 | # New in py3 1236 | self.push(__build_class__) 1237 | 1238 | def byte_STORE_LOCALS(self): 1239 | self.set_locals_dict_bytecode(self.pop()) 1240 | 1241 | if 0: # Not in py2.7 1242 | def byte_SET_LINENO(self, lineno): 1243 | self.frame.f_lineno = lineno 1244 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tox 2 | six>=1.4 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from distutils.core import setup 4 | 5 | setup( 6 | name='Byterun', 7 | version='1.0', 8 | description='Pure-Python Python bytecode execution', 9 | author='Ned Batchelder', 10 | author_email='ned@nedbatchelder.com', 11 | url='http://github.com/nedbat/byterun', 12 | packages=['byterun'], 13 | install_requires=['six'], 14 | ) 15 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/byterun/e3dc1349ed0d5f737708f274f714ac77b6f047cc/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_abstractvm.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | import dis 4 | import logging 5 | import sys 6 | import types 7 | import unittest 8 | 9 | 10 | from byterun import abstractvm 11 | from byterun import pycfg 12 | import mock 13 | 14 | # It does not accept any styling for several different members for some reason. 15 | # pylint: disable=invalid-name 16 | 17 | 18 | class MockVM(abstractvm.AbstractVirtualMachine): 19 | 20 | def __init__(self): 21 | super(MockVM, self).__init__() 22 | self.load_attr = mock.MagicMock(spec=self.load_attr) 23 | 24 | 25 | class AbstractVirtualMachineTest(unittest.TestCase): 26 | 27 | def _run_code(self, code, consts, nlocals): 28 | """Run the given raw byte code. 29 | 30 | Args: 31 | code: A raw bytecode string to execute. 32 | consts: The constants for the code. 33 | nlocals: the number of locals the code uses. 34 | """ 35 | names = tuple("v" + str(i) for i in xrange(nlocals)) 36 | code = types.CodeType(0, # argcount 37 | nlocals, # nlocals 38 | 16, # stacksize 39 | 0, # flags 40 | code, # codestring 41 | consts, # constants 42 | names, # names 43 | names, # varnames 44 | "<>", # filename 45 | "", # name 46 | 0, # firstlineno 47 | "") # lnotab 48 | self.vm.run_code(code) 49 | 50 | def setUp(self): 51 | self.vm = MockVM() 52 | 53 | def testMagicOperator(self): 54 | code = pycfg._list_to_string([ 55 | dis.opmap["LOAD_CONST"], 0, 0, # 0, arg=0 56 | dis.opmap["LOAD_CONST"], 1, 0, # 9, arg=1 57 | dis.opmap["BINARY_ADD"], # 12 58 | dis.opmap["STORE_NAME"], 0, 0, # 13, arg=1 59 | dis.opmap["LOAD_CONST"], 2, 0, # 16, arg=2 60 | dis.opmap["RETURN_VALUE"], # 19 61 | ]) 62 | 63 | method = mock.MagicMock(spec=(1).__add__) 64 | self.vm.load_attr.return_value = method 65 | self._run_code(code, (1, 2, None), 1) 66 | self.vm.load_attr.assert_called_once_with(1, "__add__") 67 | method.assert_called_once_with(2) 68 | 69 | def testIter(self): 70 | code = pycfg._list_to_string([ 71 | dis.opmap["LOAD_CONST"], 0, 0, # 3, arg=0 72 | dis.opmap["LOAD_CONST"], 1, 0, # 6, arg=1 73 | dis.opmap["LOAD_CONST"], 2, 0, # 9, arg=2 74 | dis.opmap["BUILD_LIST"], 3, 0, # 12, arg=3 75 | dis.opmap["GET_ITER"], # 15 76 | dis.opmap["LOAD_CONST"], 3, 0, # 26, arg=3 77 | dis.opmap["RETURN_VALUE"], # 29 78 | ]) 79 | 80 | method = mock.MagicMock(spec=[1, 2, 3].__iter__) 81 | self.vm.load_attr.return_value = method 82 | self._run_code(code, (1, 2, 3, None), 0) 83 | self.vm.load_attr.assert_called_once_with([1, 2, 3], "__iter__") 84 | 85 | 86 | class TraceVM(abstractvm.AncestorTraversalVirtualMachine): 87 | 88 | def __init__(self): 89 | super(TraceVM, self).__init__() 90 | self.instructions_executed = set() 91 | 92 | def run_instruction(self): 93 | self.instructions_executed.add(self.frame.f_lasti) 94 | return super(TraceVM, self).run_instruction() 95 | 96 | 97 | class AncestorTraversalVirtualMachineTest(unittest.TestCase): 98 | 99 | def setUp(self): 100 | self.vm = TraceVM() 101 | 102 | srcNestedLoops = """ 103 | y = [1,2,3] 104 | z = 0 105 | for x in y: 106 | for a in y: 107 | if x: 108 | z += x*a 109 | """ 110 | codeNestedLoops = compile(srcNestedLoops, "<>", "exec", 0, 1) 111 | 112 | codeNestedLoopsBytecode = pycfg._list_to_string([ 113 | dis.opmap["LOAD_CONST"], 0, 0, # 0, arg=0 114 | dis.opmap["LOAD_CONST"], 1, 0, # 3, arg=1 115 | dis.opmap["LOAD_CONST"], 2, 0, # 6, arg=2 116 | dis.opmap["BUILD_LIST"], 3, 0, # 9, arg=3 117 | dis.opmap["STORE_NAME"], 0, 0, # 12, arg=0 118 | dis.opmap["LOAD_CONST"], 3, 0, # 15, arg=3 119 | dis.opmap["STORE_NAME"], 1, 0, # 18, arg=1 120 | dis.opmap["SETUP_LOOP"], 54, 0, # 21, dest=78 121 | dis.opmap["LOAD_NAME"], 0, 0, # 24, arg=0 122 | dis.opmap["GET_ITER"], # 27 123 | dis.opmap["FOR_ITER"], 46, 0, # 28, dest=77 124 | dis.opmap["STORE_NAME"], 2, 0, # 31, arg=2 125 | dis.opmap["SETUP_LOOP"], 37, 0, # 34, dest=74 126 | dis.opmap["LOAD_NAME"], 0, 0, # 37, arg=0 127 | dis.opmap["GET_ITER"], # 40 128 | dis.opmap["FOR_ITER"], 29, 0, # 41, dest=73 129 | dis.opmap["STORE_NAME"], 3, 0, # 44, arg=3 130 | dis.opmap["LOAD_NAME"], 2, 0, # 47, arg=2 131 | dis.opmap["POP_JUMP_IF_FALSE"], 41, 0, # 50, dest=41 132 | dis.opmap["LOAD_NAME"], 1, 0, # 53, arg=1 133 | dis.opmap["LOAD_NAME"], 2, 0, # 56, arg=2 134 | dis.opmap["LOAD_NAME"], 3, 0, # 59, arg=3 135 | dis.opmap["BINARY_MULTIPLY"], # 62 136 | dis.opmap["INPLACE_ADD"], # 63 137 | dis.opmap["STORE_NAME"], 1, 0, # 64, arg=1 138 | dis.opmap["JUMP_ABSOLUTE"], 41, 0, # 67, dest=41 139 | dis.opmap["JUMP_ABSOLUTE"], 41, 0, # 70, dest=41 140 | dis.opmap["POP_BLOCK"], # 73 141 | dis.opmap["JUMP_ABSOLUTE"], 28, 0, # 74, dest=28 142 | dis.opmap["POP_BLOCK"], # 77 143 | dis.opmap["LOAD_CONST"], 4, 0, # 78, arg=4 144 | dis.opmap["RETURN_VALUE"], # 81 145 | ]) 146 | 147 | def testEachInstructionOnceLoops(self): 148 | self.assertEqual(self.codeNestedLoops.co_code, 149 | self.codeNestedLoopsBytecode) 150 | self.vm.run_code(self.codeNestedLoops) 151 | # The number below are the instruction offsets in the above bytecode. 152 | self.assertItemsEqual(self.vm.instructions_executed, 153 | [0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 28, 31, 34, 37, 154 | 40, 41, 44, 47, 50, 53, 56, 59, 62, 63, 64, 67, 70, 155 | 73, 74, 77, 78, 81]) 156 | 157 | srcDeadCode = """ 158 | if False: 159 | x = 2 160 | raise RuntimeError 161 | x = 42 162 | """ 163 | codeDeadCode = compile(srcDeadCode, "<>", "exec", 0, 1) 164 | 165 | codeDeadCodeBytecode = pycfg._list_to_string([ 166 | dis.opmap["LOAD_NAME"], 0, 0, # 0, arg=0 167 | dis.opmap["POP_JUMP_IF_FALSE"], 15, 0, # 3, dest=15 168 | dis.opmap["LOAD_CONST"], 0, 0, # 6, arg=0 169 | dis.opmap["STORE_NAME"], 1, 0, # 9, arg=1 170 | dis.opmap["JUMP_FORWARD"], 0, 0, # 12, dest=15 171 | dis.opmap["LOAD_NAME"], 2, 0, # 15, arg=2 172 | dis.opmap["RAISE_VARARGS"], 1, 0, # 18, arg=1 173 | dis.opmap["LOAD_CONST"], 1, 0, # 21, arg=1 174 | dis.opmap["STORE_NAME"], 1, 0, # 24, arg=1 175 | dis.opmap["LOAD_CONST"], 2, 0, # 27, arg=2 176 | dis.opmap["RETURN_VALUE"], # 30 177 | ]) 178 | 179 | def testEachInstructionOnceDeadCode(self): 180 | self.assertEqual(self.codeDeadCode.co_code, 181 | self.codeDeadCodeBytecode) 182 | try: 183 | self.vm.run_code(self.codeDeadCode) 184 | except RuntimeError: 185 | pass # Ignore the exception that gets out. 186 | self.assertItemsEqual(self.vm.instructions_executed, 187 | [0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30]) 188 | 189 | 190 | if __name__ == "__main__": 191 | # TODO(ampere): This is just a useful hack. Should be replaced with real 192 | # argument handling. 193 | if len(sys.argv) > 1: 194 | logging.basicConfig(level=logging.DEBUG) 195 | unittest.main() 196 | -------------------------------------------------------------------------------- /tests/test_basic.py: -------------------------------------------------------------------------------- 1 | """Basic tests for Byterun.""" 2 | 3 | 4 | from __future__ import print_function 5 | 6 | import unittest 7 | from tests import vmtest 8 | import six 9 | 10 | PY3, PY2 = six.PY3, not six.PY3 11 | 12 | 13 | class TestIt(vmtest.VmTestCase): 14 | 15 | def test_constant(self): 16 | self.assert_ok("17") 17 | 18 | def test_for_loop(self): 19 | self.assert_ok("""\ 20 | out = "" 21 | for i in range(5): 22 | out = out + str(i) 23 | print(out) 24 | """) 25 | 26 | def test_inplace_operators(self): 27 | self.assert_ok("""\ 28 | x, y = 2, 3 29 | x **= y 30 | assert x == 8 and y == 3 31 | x *= y 32 | assert x == 24 and y == 3 33 | x //= y 34 | assert x == 8 and y == 3 35 | x %= y 36 | assert x == 2 and y == 3 37 | x += y 38 | assert x == 5 and y == 3 39 | x -= y 40 | assert x == 2 and y == 3 41 | x <<= y 42 | assert x == 16 and y == 3 43 | x >>= y 44 | assert x == 2 and y == 3 45 | 46 | x = 0x8F 47 | x &= 0xA5 48 | assert x == 0x85 49 | x |= 0x10 50 | assert x == 0x95 51 | x ^= 0x33 52 | assert x == 0xA6 53 | """) 54 | 55 | if PY2: 56 | def test_inplace_division(self): 57 | self.assert_ok("""\ 58 | x, y = 24, 3 59 | x /= y 60 | assert x == 8 and y == 3 61 | assert isinstance(x, int) 62 | x /= y 63 | assert x == 2 and y == 3 64 | assert isinstance(x, int) 65 | """) 66 | elif PY3: 67 | def test_inplace_division(self): 68 | self.assert_ok("""\ 69 | x, y = 24, 3 70 | x /= y 71 | assert x == 8.0 and y == 3 72 | assert isinstance(x, float) 73 | x /= y 74 | assert x == (8.0/3.0) and y == 3 75 | assert isinstance(x, float) 76 | """) 77 | 78 | def test_slice(self): 79 | self.assert_ok("""\ 80 | print("hello, world"[3:8]) 81 | """) 82 | self.assert_ok("""\ 83 | print("hello, world"[:8]) 84 | """) 85 | self.assert_ok("""\ 86 | print("hello, world"[3:]) 87 | """) 88 | self.assert_ok("""\ 89 | print("hello, world"[:]) 90 | """) 91 | self.assert_ok("""\ 92 | print("hello, world"[::-1]) 93 | """) 94 | self.assert_ok("""\ 95 | print("hello, world"[3:8:2]) 96 | """) 97 | 98 | def test_slice_assignment(self): 99 | self.assert_ok("""\ 100 | l = list(range(10)) 101 | l[3:8] = ["x"] 102 | print(l) 103 | """) 104 | self.assert_ok("""\ 105 | l = list(range(10)) 106 | l[:8] = ["x"] 107 | print(l) 108 | """) 109 | self.assert_ok("""\ 110 | l = list(range(10)) 111 | l[3:] = ["x"] 112 | print(l) 113 | """) 114 | self.assert_ok("""\ 115 | l = list(range(10)) 116 | l[:] = ["x"] 117 | print(l) 118 | """) 119 | 120 | def test_slice_deletion(self): 121 | self.assert_ok("""\ 122 | l = list(range(10)) 123 | del l[3:8] 124 | print(l) 125 | """) 126 | self.assert_ok("""\ 127 | l = list(range(10)) 128 | del l[:8] 129 | print(l) 130 | """) 131 | self.assert_ok("""\ 132 | l = list(range(10)) 133 | del l[3:] 134 | print(l) 135 | """) 136 | self.assert_ok("""\ 137 | l = list(range(10)) 138 | del l[:] 139 | print(l) 140 | """) 141 | self.assert_ok("""\ 142 | l = list(range(10)) 143 | del l[::2] 144 | print(l) 145 | """) 146 | 147 | def test_building_stuff(self): 148 | self.assert_ok("""\ 149 | print((1+1, 2+2, 3+3)) 150 | """) 151 | self.assert_ok("""\ 152 | print([1+1, 2+2, 3+3]) 153 | """) 154 | self.assert_ok("""\ 155 | print({1:1+1, 2:2+2, 3:3+3}) 156 | """) 157 | 158 | def test_subscripting(self): 159 | self.assert_ok("""\ 160 | l = list(range(10)) 161 | print("%s %s %s" % (l[0], l[3], l[9])) 162 | """) 163 | self.assert_ok("""\ 164 | l = list(range(10)) 165 | l[5] = 17 166 | print(l) 167 | """) 168 | self.assert_ok("""\ 169 | l = list(range(10)) 170 | del l[5] 171 | print(l) 172 | """) 173 | 174 | def test_generator_expression(self): 175 | self.assert_ok("""\ 176 | x = "-".join(str(z) for z in range(5)) 177 | assert x == "0-1-2-3-4" 178 | """) 179 | # From test_regr.py 180 | # This failed a different way than the previous join when genexps were 181 | # broken: 182 | self.assert_ok("""\ 183 | from textwrap import fill 184 | x = set(['test_str']) 185 | width = 70 186 | indent = 4 187 | blanks = ' ' * indent 188 | res = fill(' '.join(str(elt) for elt in sorted(x)), width, 189 | initial_indent=blanks, subsequent_indent=blanks) 190 | print(res) 191 | """) 192 | 193 | def test_list_comprehension(self): 194 | self.assert_ok("""\ 195 | x = [z*z for z in range(5)] 196 | assert x == [0, 1, 4, 9, 16] 197 | """) 198 | 199 | def test_dict_comprehension(self): 200 | self.assert_ok("""\ 201 | x = {z:z*z for z in range(5)} 202 | assert x == {0:0, 1:1, 2:4, 3:9, 4:16} 203 | """) 204 | 205 | def test_set_comprehension(self): 206 | self.assert_ok("""\ 207 | x = {z*z for z in range(5)} 208 | assert x == {0, 1, 4, 9, 16} 209 | """) 210 | 211 | def test_strange_sequence_ops(self): 212 | # from stdlib: test/test_augassign.py 213 | self.assert_ok("""\ 214 | x = [1,2] 215 | x += [3,4] 216 | x *= 2 217 | 218 | assert x == [1, 2, 3, 4, 1, 2, 3, 4] 219 | 220 | x = [1, 2, 3] 221 | y = x 222 | x[1:2] *= 2 223 | y[1:2] += [1] 224 | 225 | assert x == [1, 2, 1, 2, 3] 226 | assert x is y 227 | """) 228 | 229 | def test_unary_operators(self): 230 | self.assert_ok("""\ 231 | x = 8 232 | print(-x, ~x, not x) 233 | """) 234 | 235 | def test_attributes(self): 236 | self.assert_ok("""\ 237 | l = lambda: 1 # Just to have an object... 238 | l.foo = 17 239 | print(hasattr(l, "foo"), l.foo) 240 | del l.foo 241 | print(hasattr(l, "foo")) 242 | """) 243 | 244 | def test_attribute_inplace_ops(self): 245 | self.assert_ok("""\ 246 | l = lambda: 1 # Just to have an object... 247 | l.foo = 17 248 | l.foo -= 3 249 | print(l.foo) 250 | """) 251 | 252 | def test_deleting_names(self): 253 | self.assert_ok("""\ 254 | g = 17 255 | assert g == 17 256 | del g 257 | g 258 | """, raises=NameError) 259 | 260 | def test_deleting_local_names(self): 261 | self.assert_ok("""\ 262 | def f(): 263 | l = 23 264 | assert l == 23 265 | del l 266 | l 267 | f() 268 | """, raises=NameError) 269 | 270 | def test_import(self): 271 | self.assert_ok("""\ 272 | import math 273 | print(math.pi, math.e) 274 | from math import sqrt 275 | print(sqrt(2)) 276 | from math import * 277 | print(sin(2)) 278 | """) 279 | 280 | def test_classes(self): 281 | self.assert_ok("""\ 282 | class Thing(object): 283 | def __init__(self, x): 284 | self.x = x 285 | def meth(self, y): 286 | return self.x * y 287 | thing1 = Thing(2) 288 | thing2 = Thing(3) 289 | print(thing1.x, thing2.x) 290 | print(thing1.meth(4), thing2.meth(5)) 291 | """) 292 | 293 | def test_class_mros(self): 294 | self.assert_ok("""\ 295 | class A(object): pass 296 | class B(A): pass 297 | class C(A): pass 298 | class D(B, C): pass 299 | class E(C, B): pass 300 | print([c.__name__ for c in D.__mro__]) 301 | print([c.__name__ for c in E.__mro__]) 302 | """) 303 | 304 | def test_class_mro_method_calls(self): 305 | self.assert_ok("""\ 306 | class A(object): 307 | def f(self): return 'A' 308 | class B(A): pass 309 | class C(A): 310 | def f(self): return 'C' 311 | class D(B, C): pass 312 | print(D().f()) 313 | """) 314 | 315 | def test_calling_methods_wrong(self): 316 | self.assert_ok("""\ 317 | class Thing(object): 318 | def __init__(self, x): 319 | self.x = x 320 | def meth(self, y): 321 | return self.x * y 322 | thing1 = Thing(2) 323 | print(Thing.meth(14)) 324 | """, raises=TypeError) 325 | 326 | def test_calling_subclass_methods(self): 327 | self.assert_ok("""\ 328 | class Thing(object): 329 | def foo(self): 330 | return 17 331 | 332 | class SubThing(Thing): 333 | pass 334 | 335 | st = SubThing() 336 | print(st.foo()) 337 | """) 338 | 339 | def test_other_class_methods(self): 340 | self.assert_ok("""\ 341 | class Thing(object): 342 | def foo(self): 343 | return 17 344 | 345 | class SubThing(object): 346 | def bar(self): 347 | return 9 348 | 349 | st = SubThing() 350 | print(st.foo()) 351 | """, raises=AttributeError) 352 | 353 | def test_attribute_access(self): 354 | self.assert_ok("""\ 355 | class Thing(object): 356 | z = 17 357 | def __init__(self): 358 | self.x = 23 359 | t = Thing() 360 | print(Thing.z) 361 | print(t.z) 362 | print(t.x) 363 | """) 364 | 365 | self.assert_ok("""\ 366 | class Thing(object): 367 | z = 17 368 | def __init__(self): 369 | self.x = 23 370 | t = Thing() 371 | print(t.xyzzy) 372 | """, raises=AttributeError) 373 | 374 | def test_staticmethods(self): 375 | self.assert_ok("""\ 376 | class Thing(object): 377 | @staticmethod 378 | def smeth(x): 379 | print(x) 380 | @classmethod 381 | def cmeth(cls, x): 382 | print(x) 383 | 384 | Thing.smeth(1492) 385 | Thing.cmeth(1776) 386 | """) 387 | 388 | def test_unbound_methods(self): 389 | self.assert_ok("""\ 390 | class Thing(object): 391 | def meth(self, x): 392 | print(x) 393 | m = Thing.meth 394 | m(Thing(), 1815) 395 | """) 396 | 397 | def test_callback(self): 398 | self.assert_ok("""\ 399 | def lcase(s): 400 | return s.lower() 401 | l = ["xyz", "ABC"] 402 | l.sort(key=lcase) 403 | print(l) 404 | assert l == ["ABC", "xyz"] 405 | """) 406 | 407 | def test_unpacking(self): 408 | self.assert_ok("""\ 409 | a, b, c = (1, 2, 3) 410 | assert a == 1 411 | assert b == 2 412 | assert c == 3 413 | """) 414 | 415 | if PY2: 416 | def test_exec_statement(self): 417 | self.assert_ok("""\ 418 | g = {} 419 | exec "a = 11" in g, g 420 | assert g['a'] == 11 421 | """) 422 | elif PY3: 423 | def test_exec_statement(self): 424 | self.assert_ok("""\ 425 | g = {} 426 | exec("a = 11", g, g) 427 | assert g['a'] == 11 428 | """) 429 | 430 | def test_jump_if_true_or_pop(self): 431 | self.assert_ok("""\ 432 | def f(a, b): 433 | return a or b 434 | assert f(17, 0) == 17 435 | assert f(0, 23) == 23 436 | assert f(0, "") == "" 437 | """) 438 | 439 | def test_jump_if_false_or_pop(self): 440 | self.assert_ok("""\ 441 | def f(a, b): 442 | return not(a and b) 443 | assert f(17, 0) is True 444 | assert f(0, 23) is True 445 | assert f(0, "") is True 446 | assert f(17, 23) is False 447 | """) 448 | 449 | def test_pop_jump_if_true(self): 450 | self.assert_ok("""\ 451 | def f(a): 452 | if not a: 453 | return 'foo' 454 | else: 455 | return 'bar' 456 | assert f(0) == 'foo' 457 | assert f(1) == 'bar' 458 | """) 459 | 460 | def test_decorator(self): 461 | self.assert_ok("""\ 462 | def verbose(func): 463 | def _wrapper(*args, **kwargs): 464 | return func(*args, **kwargs) 465 | return _wrapper 466 | 467 | @verbose 468 | def add(x, y): 469 | return x+y 470 | 471 | add(7, 3) 472 | """) 473 | 474 | def test_multiple_classes(self): 475 | # Making classes used to mix together all the class-scoped values 476 | # across classes. This test would fail because A.__init__ would be 477 | # over-written with B.__init__, and A(1, 2, 3) would complain about 478 | # too many arguments. 479 | self.assert_ok("""\ 480 | class A(object): 481 | def __init__(self, a, b, c): 482 | self.sum = a + b + c 483 | 484 | class B(object): 485 | def __init__(self, x): 486 | self.x = x 487 | 488 | a = A(1, 2, 3) 489 | b = B(7) 490 | print(a.sum) 491 | print(b.x) 492 | """) 493 | 494 | 495 | if PY2: 496 | class TestPrinting(vmtest.VmTestCase): 497 | 498 | def test_printing(self): 499 | self.assert_ok("print 'hello'") 500 | self.assert_ok("a = 3; print a+4") 501 | self.assert_ok(""" 502 | print 'hi', 17, u'bye', 23, 503 | print "", "\t", "the end" 504 | """) 505 | 506 | def test_printing_in_a_function(self): 507 | self.assert_ok("""\ 508 | def fn(): 509 | print "hello" 510 | fn() 511 | print "bye" 512 | """) 513 | 514 | def test_printing_to_a_file(self): 515 | self.assert_ok("""\ 516 | import sys 517 | print >>sys.stdout, 'hello', 'there' 518 | """) 519 | 520 | 521 | class TestLoops(vmtest.VmTestCase): 522 | 523 | def test_for(self): 524 | self.assert_ok("""\ 525 | for i in range(10): 526 | print(i) 527 | print("done") 528 | """) 529 | 530 | def test_break(self): 531 | self.assert_ok("""\ 532 | for i in range(10): 533 | print(i) 534 | if i == 7: 535 | break 536 | print("done") 537 | """) 538 | 539 | def test_continue(self): 540 | # fun fact: this doesn't use CONTINUE_LOOP 541 | self.assert_ok("""\ 542 | for i in range(10): 543 | if i % 3 == 0: 544 | continue 545 | print(i) 546 | print("done") 547 | """) 548 | 549 | def test_continue_in_try_except(self): 550 | self.assert_ok("""\ 551 | for i in range(10): 552 | try: 553 | if i % 3 == 0: 554 | continue 555 | print(i) 556 | except ValueError: 557 | pass 558 | print("done") 559 | """) 560 | 561 | def test_continue_in_try_finally(self): 562 | self.assert_ok("""\ 563 | for i in range(10): 564 | try: 565 | if i % 3 == 0: 566 | continue 567 | print(i) 568 | finally: 569 | print(".") 570 | print("done") 571 | """) 572 | 573 | 574 | class TestComparisons(vmtest.VmTestCase): 575 | 576 | def test_in(self): 577 | self.assert_ok("""\ 578 | assert "x" in "xyz" 579 | assert "x" not in "abc" 580 | assert "x" in ("x", "y", "z") 581 | assert "x" not in ("a", "b", "c") 582 | """) 583 | 584 | def test_less(self): 585 | self.assert_ok("""\ 586 | assert 1 < 3 587 | assert 1 <= 2 and 1 <= 1 588 | assert "a" < "b" 589 | assert "a" <= "b" and "a" <= "a" 590 | """) 591 | 592 | def test_greater(self): 593 | self.assert_ok("""\ 594 | assert 3 > 1 595 | assert 3 >= 1 and 3 >= 3 596 | assert "z" > "a" 597 | assert "z" >= "a" and "z" >= "z" 598 | """) 599 | 600 | if __name__ == "__main__": 601 | unittest.main() 602 | -------------------------------------------------------------------------------- /tests/test_exceptions.py: -------------------------------------------------------------------------------- 1 | """Test exceptions for Byterun.""" 2 | 3 | 4 | from __future__ import print_function 5 | 6 | import unittest 7 | from tests import vmtest 8 | import six 9 | 10 | PY3, PY2 = six.PY3, not six.PY3 11 | 12 | 13 | class TestExceptions(vmtest.VmTestCase): 14 | 15 | def test_catching_exceptions(self): 16 | # Catch the exception precisely 17 | self.assert_ok("""\ 18 | try: 19 | [][1] 20 | print("Shouldn't be here...") 21 | except IndexError: 22 | print("caught it!") 23 | """) 24 | # Catch the exception by a parent class 25 | self.assert_ok("""\ 26 | try: 27 | [][1] 28 | print("Shouldn't be here...") 29 | except Exception: 30 | print("caught it!") 31 | """) 32 | # Catch all exceptions 33 | self.assert_ok("""\ 34 | try: 35 | [][1] 36 | print("Shouldn't be here...") 37 | except: 38 | print("caught it!") 39 | """) 40 | 41 | def test_raise_exception(self): 42 | self.assert_ok("raise Exception('oops')", raises=Exception) 43 | 44 | def test_raise_exception_class(self): 45 | self.assert_ok("raise ValueError", raises=ValueError) 46 | 47 | if PY2: 48 | def test_raise_exception_2args(self): 49 | self.assert_ok("raise ValueError, 'bad'", raises=ValueError) 50 | 51 | def test_raise_exception_3args(self): 52 | self.assert_ok("""\ 53 | from sys import exc_info 54 | try: 55 | raise Exception 56 | except: 57 | _, _, tb = exc_info() 58 | raise ValueError, "message", tb 59 | """, raises=ValueError) 60 | 61 | def test_raise_and_catch_exception(self): 62 | self.assert_ok("""\ 63 | try: 64 | raise ValueError("oops") 65 | except ValueError as e: 66 | print("Caught: %s" % e) 67 | print("All done") 68 | """) 69 | 70 | if PY3: 71 | def test_raise_exception_from(self): 72 | self.assert_ok( 73 | "raise ValueError from NameError", 74 | raises=ValueError 75 | ) 76 | 77 | def test_raise_and_catch_exception_in_function(self): 78 | self.assert_ok("""\ 79 | def fn(): 80 | raise ValueError("oops") 81 | 82 | try: 83 | fn() 84 | except ValueError as e: 85 | print("Caught: %s" % e) 86 | print("done") 87 | """) 88 | 89 | def test_global_name_error(self): 90 | self.assert_ok("fooey", raises=NameError) 91 | self.assert_ok("""\ 92 | try: 93 | fooey 94 | print("Yes fooey?") 95 | except NameError: 96 | print("No fooey") 97 | """) 98 | 99 | def test_local_name_error(self): 100 | self.assert_ok("""\ 101 | def fn(): 102 | fooey 103 | fn() 104 | """, raises=NameError) 105 | 106 | def test_catch_local_name_error(self): 107 | self.assert_ok("""\ 108 | def fn(): 109 | try: 110 | fooey 111 | print("Yes fooey?") 112 | except NameError: 113 | print("No fooey") 114 | fn() 115 | """) 116 | 117 | def test_reraise(self): 118 | self.assert_ok("""\ 119 | def fn(): 120 | try: 121 | fooey 122 | print("Yes fooey?") 123 | except NameError: 124 | print("No fooey") 125 | raise 126 | fn() 127 | """, raises=NameError) 128 | 129 | def test_reraise_explicit_exception(self): 130 | self.assert_ok("""\ 131 | def fn(): 132 | try: 133 | raise ValueError("ouch") 134 | except ValueError as e: 135 | print("Caught %s" % e) 136 | raise 137 | fn() 138 | """, raises=ValueError) 139 | 140 | def test_finally_while_throwing(self): 141 | self.assert_ok("""\ 142 | def fn(): 143 | try: 144 | print("About to..") 145 | raise ValueError("ouch") 146 | finally: 147 | print("Finally") 148 | fn() 149 | print("Done") 150 | """, raises=ValueError) 151 | 152 | def test_coverage_issue_92(self): 153 | self.assert_ok("""\ 154 | l = [] 155 | for i in range(3): 156 | try: 157 | l.append(i) 158 | finally: 159 | l.append('f') 160 | l.append('e') 161 | l.append('r') 162 | print(l) 163 | assert l == [0, 'f', 'e', 1, 'f', 'e', 2, 'f', 'e', 'r'] 164 | """) 165 | 166 | if __name__ == "__main__": 167 | unittest.main() 168 | -------------------------------------------------------------------------------- /tests/test_functions.py: -------------------------------------------------------------------------------- 1 | """Test functions etc, for Byterun.""" 2 | 3 | 4 | from __future__ import print_function 5 | 6 | import unittest 7 | from tests import vmtest 8 | 9 | 10 | class TestFunctions(vmtest.VmTestCase): 11 | def test_functions(self): 12 | self.assert_ok("""\ 13 | def fn(a, b=17, c="Hello", d=[]): 14 | d.append(99) 15 | print(a, b, c, d) 16 | fn(1) 17 | fn(2, 3) 18 | fn(3, c="Bye") 19 | fn(4, d=["What?"]) 20 | fn(5, "b", "c") 21 | """) 22 | 23 | def test_function_locals(self): 24 | self.assert_ok("""\ 25 | def f(): 26 | x = "Spite" 27 | print(x) 28 | def g(): 29 | x = "Malice" 30 | print(x) 31 | x = "Humility" 32 | f() 33 | print(x) 34 | g() 35 | print(x) 36 | """) 37 | 38 | 39 | def test_recursion(self): 40 | self.assert_ok("""\ 41 | def fact(n): 42 | if n <= 1: 43 | return 1 44 | else: 45 | return n * fact(n-1) 46 | f6 = fact(6) 47 | print(f6) 48 | assert f6 == 720 49 | """) 50 | 51 | def test_calling_functions_with_args_kwargs(self): 52 | self.assert_ok("""\ 53 | def fn(a, b=17, c="Hello", d=[]): 54 | d.append(99) 55 | print(a, b, c, d) 56 | fn(6, *[77, 88]) 57 | fn(**{'c': 23, 'a': 7}) 58 | fn(6, *[77], **{'c': 23, 'd': [123]}) 59 | """) 60 | 61 | def test_defining_functions_with_args_kwargs(self): 62 | self.assert_ok("""\ 63 | def fn(*args): 64 | print("args is %r" % (args,)) 65 | fn(1, 2) 66 | """) 67 | self.assert_ok("""\ 68 | def fn(**kwargs): 69 | print("kwargs is %r" % (kwargs,)) 70 | fn(red=True, blue=False) 71 | """) 72 | self.assert_ok("""\ 73 | def fn(*args, **kwargs): 74 | print("args is %r" % (args,)) 75 | print("kwargs is %r" % (kwargs,)) 76 | fn(1, 2, red=True, blue=False) 77 | """) 78 | self.assert_ok("""\ 79 | def fn(x, y, *args, **kwargs): 80 | print("x is %r, y is %r" % (x, y)) 81 | print("args is %r" % (args,)) 82 | print("kwargs is %r" % (kwargs,)) 83 | fn('a', 'b', 1, 2, red=True, blue=False) 84 | """) 85 | 86 | def test_defining_functions_with_empty_args_kwargs(self): 87 | self.assert_ok("""\ 88 | def fn(*args): 89 | print("args is %r" % (args,)) 90 | fn() 91 | """) 92 | self.assert_ok("""\ 93 | def fn(**kwargs): 94 | print("kwargs is %r" % (kwargs,)) 95 | fn() 96 | """) 97 | self.assert_ok("""\ 98 | def fn(*args, **kwargs): 99 | print("args is %r, kwargs is %r" % (args, kwargs)) 100 | fn() 101 | """) 102 | 103 | def test_partial(self): 104 | self.assert_ok("""\ 105 | from _functools import partial 106 | 107 | def f(a,b): 108 | return a-b 109 | 110 | f7 = partial(f, 7) 111 | four = f7(3) 112 | assert four == 4 113 | """) 114 | 115 | def test_partial_with_kwargs(self): 116 | self.assert_ok("""\ 117 | from _functools import partial 118 | 119 | def f(a,b,c=0,d=0): 120 | return (a,b,c,d) 121 | 122 | f7 = partial(f, b=7, c=1) 123 | them = f7(10) 124 | assert them == (10,7,1,0) 125 | """) 126 | 127 | def test_wraps(self): 128 | self.assert_ok("""\ 129 | from functools import wraps 130 | def my_decorator(f): 131 | dec = wraps(f) 132 | def wrapper(*args, **kwds): 133 | print('Calling decorated function') 134 | return f(*args, **kwds) 135 | wrapper = dec(wrapper) 136 | return wrapper 137 | 138 | @my_decorator 139 | def example(): 140 | '''Docstring''' 141 | return 17 142 | 143 | assert example() == 17 144 | """) 145 | 146 | 147 | class TestClosures(vmtest.VmTestCase): 148 | def test_closures(self): 149 | self.assert_ok("""\ 150 | def make_adder(x): 151 | def add(y): 152 | return x+y 153 | return add 154 | a = make_adder(10) 155 | print(a(7)) 156 | assert a(7) == 17 157 | """) 158 | 159 | def test_closures_store_deref(self): 160 | self.assert_ok("""\ 161 | def make_adder(x): 162 | z = x+1 163 | def add(y): 164 | return x+y+z 165 | return add 166 | a = make_adder(10) 167 | print(a(7)) 168 | assert a(7) == 28 169 | """) 170 | 171 | def test_closures_in_loop(self): 172 | self.assert_ok("""\ 173 | def make_fns(x): 174 | fns = [] 175 | for i in range(x): 176 | fns.append(lambda i=i: i) 177 | return fns 178 | fns = make_fns(3) 179 | for f in fns: 180 | print(f()) 181 | assert (fns[0](), fns[1](), fns[2]()) == (0, 1, 2) 182 | """) 183 | 184 | def test_closures_with_defaults(self): 185 | self.assert_ok("""\ 186 | def make_adder(x, y=13, z=43): 187 | def add(q, r=11): 188 | return x+y+z+q+r 189 | return add 190 | a = make_adder(10, 17) 191 | print(a(7)) 192 | assert a(7) == 88 193 | """) 194 | 195 | def test_deep_closures(self): 196 | self.assert_ok("""\ 197 | def f1(a): 198 | b = 2*a 199 | def f2(c): 200 | d = 2*c 201 | def f3(e): 202 | f = 2*e 203 | def f4(g): 204 | h = 2*g 205 | return a+b+c+d+e+f+g+h 206 | return f4 207 | return f3 208 | return f2 209 | answer = f1(3)(4)(5)(6) 210 | print(answer) 211 | assert answer == 54 212 | """) 213 | 214 | 215 | class TestGenerators(vmtest.VmTestCase): 216 | def test_first(self): 217 | self.assert_ok("""\ 218 | def two(): 219 | yield 1 220 | yield 2 221 | for i in two(): 222 | print(i) 223 | """) 224 | 225 | def test_partial_generator(self): 226 | self.assert_ok("""\ 227 | from _functools import partial 228 | 229 | def f(a,b): 230 | num = a+b 231 | while num: 232 | yield num 233 | num -= 1 234 | 235 | f2 = partial(f, 2) 236 | three = f2(1) 237 | assert list(three) == [3,2,1] 238 | """) 239 | 240 | def test_yield_multiple_values(self): 241 | self.assert_ok("""\ 242 | def triples(): 243 | yield 1, 2, 3 244 | yield 4, 5, 6 245 | 246 | for a, b, c in triples(): 247 | print(a, b, c) 248 | """) 249 | 250 | def test_generator_reuse(self): 251 | self.assert_ok("""\ 252 | g = (x*x for x in range(5)) 253 | print(list(g)) 254 | print(list(g)) 255 | """) 256 | 257 | def test_generator_from_generator2(self): 258 | self.assert_ok("""\ 259 | g = (x*x for x in range(3)) 260 | print(list(g)) 261 | 262 | g = (x*x for x in range(5)) 263 | g = (y+1 for y in g) 264 | print(list(g)) 265 | """) 266 | 267 | def test_generator_from_generator(self): 268 | self.assert_ok("""\ 269 | class Thing(object): 270 | RESOURCES = ('abc', 'def') 271 | def get_abc(self): 272 | return "ABC" 273 | def get_def(self): 274 | return "DEF" 275 | def resource_info(self): 276 | for name in self.RESOURCES: 277 | get_name = 'get_' + name 278 | yield name, getattr(self, get_name) 279 | 280 | def boom(self): 281 | #d = list((name, get()) for name, get in self.resource_info()) 282 | d = [(name, get()) for name, get in self.resource_info()] 283 | return d 284 | 285 | print(Thing().boom()) 286 | """) 287 | 288 | if __name__ == "__main__": 289 | unittest.main() 290 | -------------------------------------------------------------------------------- /tests/test_pycfg.py: -------------------------------------------------------------------------------- 1 | """Tests for pycfg. 2 | """ 3 | 4 | import dis 5 | import inspect 6 | import logging 7 | import unittest 8 | 9 | 10 | from byterun import pycfg 11 | 12 | # Disable because pylint does not like any name for the nested test_code 13 | # functions used to get the needed bytecode. 14 | # pylint: disable=invalid-name 15 | 16 | # The bytecode constants used to check against the generated code are formatted 17 | # as follows. Each line is one instruction. Blank lines separate basic blocks. 18 | # 19 | # dis.opmap[""], , , # , 20 | # 21 | # The is a decoded version of the argument. This is more useful for 22 | # relative jumps. 23 | 24 | 25 | def line_number(): 26 | """Returns the line number of the call site.""" 27 | return inspect.currentframe().f_back.f_lineno 28 | 29 | 30 | class CFGTest(unittest.TestCase): 31 | 32 | def assertEndsWith(self, actual, expected): 33 | self.assertTrue(actual.endswith(expected), 34 | msg="'%s' does not end with '%s'" % (actual, expected)) 35 | 36 | # Copy this line into your test when developing it. It prints the formatted 37 | # bytecode to use as the expected. 38 | # print pycfg._bytecode_repr(test_code.func_code.co_code) 39 | 40 | def checkBlocks(self, table, expected): 41 | self.assertEqual(len(table._blocks), len(expected)) 42 | for block, (expected_begin, expected_end) in zip(table._blocks, expected): 43 | self.assertEqual(block.begin, expected_begin) 44 | self.assertEqual(block.end, expected_end) 45 | 46 | @staticmethod 47 | def codeOneBlock(): 48 | return x + 1 # pylint: disable=undefined-variable 49 | 50 | codeOneBlockBytecode = pycfg._list_to_string([ 51 | dis.opmap["LOAD_GLOBAL"], 0, 0, # 0 52 | dis.opmap["LOAD_CONST"], 1, 0, # 3 53 | dis.opmap["BINARY_ADD"], # 6 54 | dis.opmap["RETURN_VALUE"], # 7 55 | ]) 56 | 57 | def testOneBlock(self): 58 | # Check the code to make sure the test will fail if the compilation changes. 59 | self.assertEqual(self.codeOneBlock.func_code.co_code, 60 | self.codeOneBlockBytecode) 61 | 62 | cfg = pycfg.CFG() 63 | table = cfg.get_block_table(self.codeOneBlock.func_code) 64 | # Should all be one basic block. 65 | self.assertIs(table.get_basic_block(0), table.get_basic_block(3)) 66 | self.assertIs(table.get_basic_block(0), table.get_basic_block(6)) 67 | self.assertIs(table.get_basic_block(0), table.get_basic_block(7)) 68 | # No incoming 69 | self.assertItemsEqual(table.get_basic_block(0).incoming, []) 70 | # Outgoing is an unknown return location 71 | self.assertItemsEqual(table.get_basic_block(0).outgoing, [None]) 72 | 73 | @staticmethod 74 | def codeTriangle(y): 75 | x = y 76 | if y > 10: 77 | x -= 2 78 | return x 79 | codeTriangleLineNumber = line_number() - 4 80 | # codeTriangleLineNumber is used to compute the correct line numbers for code 81 | # in codeTriangle. This makes the tests less brittle if other tests in the 82 | # file are changed. However the "- 4" will need to be changed if codeTriangle 83 | # is changed or anything is inserted between the line_number() call and the 84 | # definition of codeTriangle. 85 | 86 | codeTriangleBytecode = pycfg._list_to_string([ 87 | dis.opmap["LOAD_FAST"], 0, 0, # 0, arg=0 88 | dis.opmap["STORE_FAST"], 1, 0, # 3, arg=1 89 | dis.opmap["LOAD_FAST"], 0, 0, # 6, arg=0 90 | dis.opmap["LOAD_CONST"], 1, 0, # 9, arg=1 91 | dis.opmap["COMPARE_OP"], 4, 0, # 12, arg=4 92 | dis.opmap["POP_JUMP_IF_FALSE"], 31, 0, # 15, dest=31 93 | 94 | dis.opmap["LOAD_FAST"], 1, 0, # 18, arg=1 95 | dis.opmap["LOAD_CONST"], 2, 0, # 21, arg=2 96 | dis.opmap["INPLACE_SUBTRACT"], # 24 97 | dis.opmap["STORE_FAST"], 1, 0, # 25, arg=1 98 | dis.opmap["JUMP_FORWARD"], 0, 0, # 28, dest=31 99 | 100 | dis.opmap["LOAD_FAST"], 1, 0, # 31, arg=1 101 | dis.opmap["RETURN_VALUE"], # 34 102 | ]) 103 | 104 | def testTriangle(self): 105 | self.assertEqual(self.codeTriangle.func_code.co_code, 106 | self.codeTriangleBytecode) 107 | 108 | cfg = pycfg.CFG() 109 | table = cfg.get_block_table(self.codeTriangle.func_code) 110 | expected = [(0, 15), 111 | (18, 28), 112 | (31, 34)] 113 | self.checkBlocks(table, expected) 114 | bb = table.get_basic_block 115 | # Check the POP_JUMP_IF_FALSE conditional jump 116 | self.assertItemsEqual(bb(0).outgoing, [bb(18), bb(31)]) 117 | # Check the return 118 | self.assertItemsEqual(bb(44).outgoing, [None]) 119 | # Check the incoming of the entry block 120 | self.assertItemsEqual(bb(0).incoming, []) 121 | # Check incoming of the merge block. 122 | self.assertItemsEqual(bb(44).incoming, [bb(28), bb(15)]) 123 | self.assertEndsWith( 124 | bb(21).get_name(), 125 | "tests/test_pycfg.py:{0}-{0}".format(self.codeTriangleLineNumber+2)) 126 | 127 | def testTriangleDominators(self): 128 | cfg = pycfg.CFG() 129 | table = cfg.get_block_table(self.codeTriangle.func_code) 130 | 131 | bb = table.get_basic_block 132 | self.assertEqual(bb(0)._dominators, {bb(0)}) 133 | self.assertEqual(bb(18)._dominators, {bb(0), bb(18)}) 134 | self.assertEqual(bb(31)._dominators, {bb(0), bb(31)}) 135 | self.assertEqual(bb(41)._dominators, {bb(0), bb(41)}) 136 | 137 | self.assertTrue(table.dominates(0, 37)) 138 | self.assertFalse(table.dominates(24, 41)) 139 | self.assertTrue(table.dominates(21, 28)) 140 | self.assertFalse(table.dominates(28, 21)) 141 | 142 | def testTriangleOrder(self): 143 | cfg = pycfg.CFG() 144 | table = cfg.get_block_table(self.codeTriangle.func_code) 145 | 146 | bb = table.get_basic_block 147 | self.assertEqual(table.get_ancestors_first_traversal(), 148 | [bb(o) for o in [0, 18, 31]]) 149 | 150 | @staticmethod 151 | def codeDiamond(y): 152 | x = y 153 | if y > 10: 154 | x -= 2 155 | else: 156 | x += 2 157 | return x 158 | 159 | codeDiamondBytecode = pycfg._list_to_string([ 160 | dis.opmap["LOAD_FAST"], 0, 0, # 0, arg=0 161 | dis.opmap["STORE_FAST"], 1, 0, # 3, arg=1 162 | dis.opmap["LOAD_FAST"], 0, 0, # 6, arg=0 163 | dis.opmap["LOAD_CONST"], 1, 0, # 9, arg=1 164 | dis.opmap["COMPARE_OP"], 4, 0, # 12, arg=4 165 | dis.opmap["POP_JUMP_IF_FALSE"], 31, 0, # 15, dest=31 166 | 167 | dis.opmap["LOAD_FAST"], 1, 0, # 18, arg=1 168 | dis.opmap["LOAD_CONST"], 2, 0, # 21, arg=2 169 | dis.opmap["INPLACE_SUBTRACT"], # 24 170 | dis.opmap["STORE_FAST"], 1, 0, # 25, arg=1 171 | dis.opmap["JUMP_FORWARD"], 10, 0, # 28, dest=41 172 | 173 | dis.opmap["LOAD_FAST"], 1, 0, # 31, arg=1 174 | dis.opmap["LOAD_CONST"], 2, 0, # 34, arg=2 175 | dis.opmap["INPLACE_ADD"], # 37 176 | dis.opmap["STORE_FAST"], 1, 0, # 38, arg=1 177 | 178 | dis.opmap["LOAD_FAST"], 1, 0, # 41, arg=1 179 | dis.opmap["RETURN_VALUE"], # 44 180 | ]) 181 | 182 | def testDiamond(self): 183 | self.assertEqual(self.codeDiamond.func_code.co_code, 184 | self.codeDiamondBytecode) 185 | 186 | cfg = pycfg.CFG() 187 | table = cfg.get_block_table(self.codeDiamond.func_code) 188 | expected = [(0, 15), 189 | (18, 28), 190 | (31, 38), 191 | (41, 44)] 192 | self.checkBlocks(table, expected) 193 | bb = table.get_basic_block 194 | # Check the POP_JUMP_IF_FALSE conditional jump 195 | self.assertItemsEqual(bb(0).outgoing, [bb(18), bb(31)]) 196 | # Check the jumps at the end of the 2 of branches 197 | self.assertItemsEqual(bb(18).outgoing, [bb(41)]) 198 | self.assertItemsEqual(bb(38).outgoing, [bb(41)]) 199 | # Check the return 200 | self.assertItemsEqual(bb(44).outgoing, [None]) 201 | # Check the incoming of the entry block 202 | self.assertItemsEqual(bb(0).incoming, []) 203 | # Check the incoming of the 2 if branches 204 | self.assertItemsEqual(bb(18).incoming, [bb(15)]) 205 | self.assertItemsEqual(bb(31).incoming, [bb(15)]) 206 | # Check incoming of the merge block. 207 | self.assertItemsEqual(bb(44).incoming, [bb(28), bb(38)]) 208 | 209 | def testDiamondDominators(self): 210 | cfg = pycfg.CFG() 211 | table = cfg.get_block_table(self.codeDiamond.func_code) 212 | 213 | bb = table.get_basic_block 214 | self.assertEqual(bb(0)._dominators, {bb(0)}) 215 | self.assertEqual(bb(18)._dominators, {bb(0), bb(18)}) 216 | self.assertEqual(bb(31)._dominators, {bb(0), bb(31)}) 217 | self.assertEqual(bb(41)._dominators, {bb(0), bb(41)}) 218 | 219 | self.assertTrue(table.dominates(0, 37)) 220 | self.assertFalse(table.dominates(24, 41)) 221 | self.assertTrue(table.dominates(21, 28)) 222 | self.assertFalse(table.dominates(28, 21)) 223 | 224 | def testDiamondOrder(self): 225 | cfg = pycfg.CFG() 226 | table = cfg.get_block_table(self.codeDiamond.func_code) 227 | 228 | bb = table.get_basic_block 229 | self.assertEqual(table.get_ancestors_first_traversal(), 230 | [bb(o) for o in [0, 18, 31, 41]]) 231 | 232 | @staticmethod 233 | def codeLoop(y): 234 | z = 0 235 | for x in y: 236 | z += x 237 | return z 238 | 239 | codeLoopBytecode = pycfg._list_to_string([ 240 | dis.opmap["LOAD_CONST"], 1, 0, # 0, arg=1 241 | dis.opmap["STORE_FAST"], 1, 0, # 3, arg=1 242 | dis.opmap["SETUP_LOOP"], 24, 0, # 6, dest=33 243 | dis.opmap["LOAD_FAST"], 0, 0, # 9, arg=0 244 | dis.opmap["GET_ITER"], # 12 245 | 246 | dis.opmap["FOR_ITER"], 16, 0, # 13, dest=32 247 | 248 | dis.opmap["STORE_FAST"], 2, 0, # 16, arg=2 249 | dis.opmap["LOAD_FAST"], 1, 0, # 19, arg=1 250 | dis.opmap["LOAD_FAST"], 2, 0, # 22, arg=2 251 | dis.opmap["INPLACE_ADD"], # 25 252 | dis.opmap["STORE_FAST"], 1, 0, # 26, arg=1 253 | dis.opmap["JUMP_ABSOLUTE"], 13, 0, # 29, dest=13 254 | 255 | dis.opmap["POP_BLOCK"], # 32 256 | 257 | dis.opmap["LOAD_FAST"], 1, 0, # 33, arg=1 258 | dis.opmap["RETURN_VALUE"], # 36 259 | ]) 260 | 261 | def testLoop(self): 262 | self.assertEqual(self.codeLoop.func_code.co_code, 263 | self.codeLoopBytecode) 264 | 265 | cfg = pycfg.CFG() 266 | table = cfg.get_block_table(self.codeLoop.func_code) 267 | expected = [(0, 12), 268 | (13, 13), 269 | (16, 29), 270 | (32, 32), 271 | (33, 36)] 272 | self.checkBlocks(table, expected) 273 | bb = table.get_basic_block 274 | # Check outgoing of the loop handler instruction. 275 | self.assertItemsEqual(bb(13).outgoing, [bb(16), bb(32)]) 276 | self.assertItemsEqual(bb(0).outgoing, [bb(13)]) 277 | 278 | def testLoopDominators(self): 279 | cfg = pycfg.CFG() 280 | table = cfg.get_block_table(self.codeLoop.func_code) 281 | 282 | bb = table.get_basic_block 283 | self.assertEqual(bb(0)._dominators, {bb(0)}) 284 | self.assertEqual(bb(13)._dominators, {bb(0), bb(13)}) 285 | self.assertEqual(bb(16)._dominators, {bb(0), bb(13), bb(16)}) 286 | self.assertEqual(bb(32)._dominators, {bb(0), bb(13), bb(32)}) 287 | self.assertEqual(bb(33)._dominators, 288 | {bb(0), bb(13), bb(32), bb(33)}) 289 | 290 | def testLoopOrder(self): 291 | cfg = pycfg.CFG() 292 | table = cfg.get_block_table(self.codeLoop.func_code) 293 | 294 | bb = table.get_basic_block 295 | self.assertEqual(table.get_ancestors_first_traversal(), 296 | [bb(o) for o in [0, 13, 16, 32, 33]]) 297 | 298 | @staticmethod 299 | def codeNestedLoops(y): 300 | z = 0 301 | for x in y: 302 | for x in y: 303 | z += x*x 304 | return z 305 | codeNestedLoopsLineNumber = line_number() - 5 306 | # See comment on codeTriangleLineNumber above. 307 | 308 | codeNestedLoopsBytecode = pycfg._list_to_string([ 309 | dis.opmap["LOAD_CONST"], 1, 0, # 0, arg=1 310 | dis.opmap["STORE_FAST"], 1, 0, # 3, arg=1 311 | dis.opmap["SETUP_LOOP"], 45, 0, # 6, dest=54 312 | dis.opmap["LOAD_FAST"], 0, 0, # 9, arg=0 313 | dis.opmap["GET_ITER"], # 12 314 | 315 | dis.opmap["FOR_ITER"], 37, 0, # 13, dest=53 316 | 317 | dis.opmap["STORE_FAST"], 2, 0, # 16, arg=2 318 | dis.opmap["SETUP_LOOP"], 28, 0, # 19, dest=50 319 | dis.opmap["LOAD_FAST"], 0, 0, # 22, arg=0 320 | dis.opmap["GET_ITER"], # 25 321 | 322 | dis.opmap["FOR_ITER"], 20, 0, # 26, dest=49 323 | 324 | dis.opmap["STORE_FAST"], 2, 0, # 29, arg=2 325 | dis.opmap["LOAD_FAST"], 1, 0, # 32, arg=1 326 | dis.opmap["LOAD_FAST"], 2, 0, # 35, arg=2 327 | dis.opmap["LOAD_FAST"], 2, 0, # 38, arg=2 328 | dis.opmap["BINARY_MULTIPLY"], # 41 329 | dis.opmap["INPLACE_ADD"], # 42 330 | dis.opmap["STORE_FAST"], 1, 0, # 43, arg=1 331 | dis.opmap["JUMP_ABSOLUTE"], 26, 0, # 46, dest=26 332 | 333 | dis.opmap["POP_BLOCK"], # 49 334 | 335 | dis.opmap["JUMP_ABSOLUTE"], 13, 0, # 50, dest=13 336 | 337 | dis.opmap["POP_BLOCK"], # 53 338 | 339 | dis.opmap["LOAD_FAST"], 1, 0, # 54, arg=1 340 | dis.opmap["RETURN_VALUE"], # 57 341 | ]) 342 | 343 | def testNestedLoops(self): 344 | self.assertEqual(self.codeNestedLoops.func_code.co_code, 345 | self.codeNestedLoopsBytecode) 346 | 347 | cfg = pycfg.CFG() 348 | table = cfg.get_block_table(self.codeNestedLoops.func_code) 349 | expected = [(0, 12), 350 | (13, 13), 351 | (16, 25), 352 | (26, 26), 353 | (29, 46), 354 | (49, 49), 355 | (50, 50), 356 | (53, 53), 357 | (54, 57)] 358 | self.checkBlocks(table, expected) 359 | bb = table.get_basic_block 360 | self.assertItemsEqual(bb(13).incoming, [bb(12), bb(50)]) 361 | self.assertItemsEqual(bb(13).outgoing, [bb(16), bb(53)]) 362 | self.assertItemsEqual(bb(26).incoming, [bb(25), bb(46)]) 363 | self.assertItemsEqual(bb(26).outgoing, [bb(29), bb(49)]) 364 | self.assertEndsWith( 365 | bb(43).get_name(), 366 | "tests/test_pycfg.py:{}-{}".format(self.codeNestedLoopsLineNumber + 2, 367 | self.codeNestedLoopsLineNumber + 3)) 368 | 369 | def testNestedLoopsDominators(self): 370 | cfg = pycfg.CFG() 371 | table = cfg.get_block_table(self.codeNestedLoops.func_code) 372 | bb = table.get_basic_block 373 | 374 | self.assertEqual(bb(0)._dominators, 375 | {bb(0)}) 376 | self.assertEqual(bb(13)._dominators, 377 | {bb(0), bb(13)}) 378 | self.assertEqual(bb(16)._dominators, 379 | {bb(0), bb(13), bb(16)}) 380 | self.assertEqual(bb(26)._dominators, 381 | {bb(0), bb(13), bb(16), bb(26)}) 382 | self.assertEqual(bb(29)._dominators, 383 | {bb(0), bb(13), bb(16), bb(26), bb(29)}) 384 | self.assertEqual(bb(49)._dominators, 385 | {bb(0), bb(13), bb(16), bb(26), bb(49)}) 386 | self.assertEqual(bb(50)._dominators, 387 | {bb(0), bb(13), bb(16), bb(26), bb(49), bb(50)}) 388 | self.assertEqual(bb(53)._dominators, 389 | {bb(0), bb(13), bb(53)}) 390 | self.assertEqual(bb(54)._dominators, 391 | {bb(0), bb(13), bb(53), bb(54)}) 392 | 393 | def testNestedLoopsReachable(self): 394 | cfg = pycfg.CFG() 395 | table = cfg.get_block_table(self.codeNestedLoops.func_code) 396 | bb = table.get_basic_block 397 | 398 | self.assertEqual(bb(26)._reachable_from, 399 | set([bb(0), bb(13), bb(16), bb(26), 400 | bb(29), bb(49), bb(50)])) 401 | 402 | self.assertTrue(table.reachable_from(41, 50)) 403 | self.assertTrue(table.reachable_from(50, 41)) 404 | self.assertFalse(table.reachable_from(41, 53)) 405 | 406 | def testNestedLoopsOrder(self): 407 | cfg = pycfg.CFG() 408 | table = cfg.get_block_table(self.codeNestedLoops.func_code) 409 | 410 | bb = table.get_basic_block 411 | self.assertEqual(table.get_ancestors_first_traversal(), 412 | [bb(o) for o in [0, 13, 16, 26, 29, 49, 50, 53, 54]]) 413 | 414 | @staticmethod 415 | def codeContinue(y): 416 | z = 0 417 | for x in y: 418 | if x == 1: 419 | continue 420 | z += x*x 421 | return z 422 | 423 | codeContinueBytecode = pycfg._list_to_string([ 424 | dis.opmap["LOAD_CONST"], 1, 0, # 0, arg=1 425 | dis.opmap["STORE_FAST"], 1, 0, # 3, arg=1 426 | dis.opmap["SETUP_LOOP"], 46, 0, # 6, dest=55 427 | dis.opmap["LOAD_FAST"], 0, 0, # 9, arg=0 428 | dis.opmap["GET_ITER"], # 12 429 | 430 | dis.opmap["FOR_ITER"], 38, 0, # 13, dest=54 431 | 432 | dis.opmap["STORE_FAST"], 2, 0, # 16, arg=2 433 | dis.opmap["LOAD_FAST"], 2, 0, # 19, arg=2 434 | dis.opmap["LOAD_CONST"], 2, 0, # 22, arg=2 435 | dis.opmap["COMPARE_OP"], 2, 0, # 25, arg=2 436 | dis.opmap["POP_JUMP_IF_FALSE"], 37, 0, # 28, dest=37 437 | 438 | dis.opmap["JUMP_ABSOLUTE"], 13, 0, # 31, dest=13 439 | 440 | dis.opmap["JUMP_FORWARD"], 0, 0, # 34, dest=37 441 | 442 | dis.opmap["LOAD_FAST"], 1, 0, # 37, arg=1 443 | dis.opmap["LOAD_FAST"], 2, 0, # 40, arg=2 444 | dis.opmap["LOAD_FAST"], 2, 0, # 43, arg=2 445 | dis.opmap["BINARY_MULTIPLY"], # 46 446 | dis.opmap["INPLACE_ADD"], # 47 447 | dis.opmap["STORE_FAST"], 1, 0, # 48, arg=1 448 | dis.opmap["JUMP_ABSOLUTE"], 13, 0, # 51, dest=13 449 | 450 | dis.opmap["POP_BLOCK"], # 54 451 | dis.opmap["LOAD_FAST"], 1, 0, # 55, arg=1 452 | dis.opmap["RETURN_VALUE"], # 58 453 | ]) 454 | 455 | def testContinue(self): 456 | self.assertEqual(self.codeContinue.func_code.co_code, 457 | self.codeContinueBytecode) 458 | 459 | cfg = pycfg.CFG() 460 | table = cfg.get_block_table(self.codeContinue.func_code) 461 | bb = table.get_basic_block 462 | self.assertItemsEqual(bb(31).outgoing, [bb(13)]) 463 | self.assertItemsEqual(bb(13).incoming, [bb(12), bb(51), bb(31)]) 464 | self.assertItemsEqual(bb(13).outgoing, [bb(16), bb(54)]) 465 | 466 | @staticmethod 467 | def codeBreak(y): 468 | z = 0 469 | for x in y: 470 | if x == 1: 471 | break 472 | z += x*x 473 | return z 474 | 475 | codeBreakBytecode = pycfg._list_to_string([ 476 | dis.opmap["LOAD_CONST"], 1, 0, # 0, arg=1 477 | dis.opmap["STORE_FAST"], 1, 0, # 3, arg=1 478 | dis.opmap["SETUP_LOOP"], 44, 0, # 6, dest=53 479 | dis.opmap["LOAD_FAST"], 0, 0, # 9, arg=0 480 | dis.opmap["GET_ITER"], # 12 481 | 482 | dis.opmap["FOR_ITER"], 36, 0, # 13, dest=52 483 | 484 | dis.opmap["STORE_FAST"], 2, 0, # 16, arg=2 485 | dis.opmap["LOAD_FAST"], 2, 0, # 19, arg=2 486 | dis.opmap["LOAD_CONST"], 2, 0, # 22, arg=2 487 | dis.opmap["COMPARE_OP"], 2, 0, # 25, arg=2 488 | dis.opmap["POP_JUMP_IF_FALSE"], 35, 0, # 28, dest=35 489 | 490 | dis.opmap["BREAK_LOOP"], # 31 491 | 492 | dis.opmap["JUMP_FORWARD"], 0, 0, # 32, dest=35 493 | 494 | dis.opmap["LOAD_FAST"], 1, 0, # 35, arg=1 495 | dis.opmap["LOAD_FAST"], 2, 0, # 38, arg=2 496 | dis.opmap["LOAD_FAST"], 2, 0, # 41, arg=2 497 | dis.opmap["BINARY_MULTIPLY"], # 44 498 | dis.opmap["INPLACE_ADD"], # 45 499 | dis.opmap["STORE_FAST"], 1, 0, # 46, arg=1 500 | dis.opmap["JUMP_ABSOLUTE"], 13, 0, # 49, dest=13 501 | 502 | dis.opmap["POP_BLOCK"], # 52 503 | 504 | dis.opmap["LOAD_FAST"], 1, 0, # 53, arg=1 505 | dis.opmap["RETURN_VALUE"], # 56 506 | ]) 507 | 508 | def testBreak(self): 509 | self.assertEqual(self.codeBreak.func_code.co_code, 510 | self.codeBreakBytecode) 511 | 512 | cfg = pycfg.CFG() 513 | table = cfg.get_block_table(self.codeBreak.func_code) 514 | bb = table.get_basic_block 515 | self.assertItemsEqual(bb(13).incoming, [bb(12), bb(49)]) 516 | self.assertItemsEqual(bb(31).incoming, [bb(28)]) 517 | self.assertItemsEqual(bb(31).outgoing, [None]) 518 | # TODO(ampere): This is correct, however more information would make the 519 | # following succeed. 520 | # self.assertItemsEqual(bb(31).incoming, [53]) 521 | 522 | @staticmethod 523 | def codeYield(): 524 | yield 1 525 | yield 2 526 | yield 3 527 | 528 | codeYieldBytecode = pycfg._list_to_string([ 529 | dis.opmap["LOAD_CONST"], 1, 0, # 0, arg=1 530 | dis.opmap["YIELD_VALUE"], # 3 531 | 532 | dis.opmap["POP_TOP"], # 4 533 | dis.opmap["LOAD_CONST"], 2, 0, # 5, arg=2 534 | dis.opmap["YIELD_VALUE"], # 8 535 | 536 | dis.opmap["POP_TOP"], # 9 537 | dis.opmap["LOAD_CONST"], 3, 0, # 10, arg=3 538 | dis.opmap["YIELD_VALUE"], # 13 539 | 540 | dis.opmap["POP_TOP"], # 14 541 | dis.opmap["LOAD_CONST"], 0, 0, # 15, arg=0 542 | dis.opmap["RETURN_VALUE"], # 18 543 | ]) 544 | 545 | def testYield(self): 546 | self.assertEqual(self.codeYield.func_code.co_code, 547 | self.codeYieldBytecode) 548 | 549 | cfg = pycfg.CFG() 550 | table = cfg.get_block_table(self.codeYield.func_code) 551 | expected = [(0, 3), 552 | (4, 8), 553 | (9, 13), 554 | (14, 18)] 555 | self.checkBlocks(table, expected) 556 | bb = table.get_basic_block 557 | # We both branch to unknown and to the best instruction for each yield. 558 | self.assertItemsEqual(bb(0).outgoing, [None, bb(4)]) 559 | self.assertItemsEqual(bb(4).outgoing, [None, bb(9)]) 560 | self.assertItemsEqual(bb(9).incoming, [bb(8)]) 561 | self.assertItemsEqual(bb(9).outgoing, [None, bb(14)]) 562 | 563 | @staticmethod 564 | def codeRaise(): 565 | raise ValueError() 566 | return 0 # pylint: disable=unreachable 567 | 568 | codeRaiseBytecode = pycfg._list_to_string([ 569 | dis.opmap["LOAD_GLOBAL"], 0, 0, # 0, arg=0 570 | dis.opmap["CALL_FUNCTION"], 0, 0, # 3, arg=0 571 | 572 | dis.opmap["RAISE_VARARGS"], 1, 0, # 6, arg=1 573 | 574 | dis.opmap["LOAD_CONST"], 1, 0, # 9, arg=1 575 | dis.opmap["RETURN_VALUE"], # 12 576 | ]) 577 | 578 | def testRaise(self): 579 | self.assertEqual(self.codeRaise.func_code.co_code, 580 | self.codeRaiseBytecode) 581 | 582 | cfg = pycfg.CFG() 583 | table = cfg.get_block_table(self.codeRaise.func_code) 584 | expected = [(0, 3), 585 | (6, 6), 586 | (9, 12)] 587 | self.checkBlocks(table, expected) 588 | bb = table.get_basic_block 589 | # CALL_FUNCTION could either continue or raise 590 | self.assertItemsEqual(bb(0).outgoing, [bb(6), None]) 591 | # RAISE_VARARGS always raises 592 | self.assertItemsEqual(bb(6).outgoing, [None]) 593 | # This basic block is unreachable 594 | self.assertItemsEqual(bb(9).incoming, []) 595 | # We return to an unknown location 596 | self.assertItemsEqual(bb(9).outgoing, [None]) 597 | 598 | def testRaiseOrder(self): 599 | cfg = pycfg.CFG() 600 | table = cfg.get_block_table(self.codeRaise.func_code) 601 | 602 | bb = table.get_basic_block 603 | self.assertEqual(table.get_ancestors_first_traversal(), 604 | [bb(o) for o in [0, 6, 9]]) 605 | 606 | 607 | class InstructionsIndexTest(unittest.TestCase): 608 | 609 | @staticmethod 610 | def simple_function(x): 611 | x += 1 612 | y = 4 613 | x **= y 614 | return x + y 615 | 616 | def setUp(self): 617 | self.index = pycfg.InstructionsIndex(self.simple_function.func_code.co_code) 618 | 619 | def testNext(self): 620 | self.assertEqual(self.index.next(0), 3) 621 | self.assertEqual(self.index.next(6), 7) 622 | self.assertEqual(self.index.next(23), 26) 623 | 624 | def testPrev(self): 625 | self.assertEqual(self.index.prev(3), 0) 626 | self.assertEqual(self.index.prev(7), 6) 627 | self.assertEqual(self.index.prev(26), 23) 628 | 629 | def testRoundTrip(self): 630 | offset = 3 631 | while offset < len(self.simple_function.func_code.co_code)-1: 632 | self.assertEqual(self.index.prev(self.index.next(offset)), offset) 633 | self.assertEqual(self.index.next(self.index.prev(offset)), offset) 634 | offset = self.index.next(offset) 635 | 636 | 637 | class BytecodeReprTest(unittest.TestCase): 638 | 639 | def checkRoundTrip(self, code): 640 | self.assertEqual(eval(pycfg._bytecode_repr(code)), code) 641 | 642 | def testOtherTestMethods(self): 643 | for method in CFGTest.__dict__: 644 | if hasattr(method, "func_code"): 645 | self.checkRoundTrip(method.func_code.co_code) 646 | 647 | def testThisTestMethods(self): 648 | for method in BytecodeReprTest.__dict__: 649 | if hasattr(method, "func_code"): 650 | self.checkRoundTrip(method.func_code.co_code) 651 | 652 | if __name__ == "__main__": 653 | logging.basicConfig(level=logging.DEBUG) 654 | unittest.main() 655 | -------------------------------------------------------------------------------- /tests/test_with.py: -------------------------------------------------------------------------------- 1 | """Test the with statement for Byterun.""" 2 | 3 | from __future__ import print_function 4 | 5 | import unittest 6 | from tests import vmtest 7 | 8 | 9 | class TestWithStatement(vmtest.VmTestCase): 10 | 11 | def test_simple_context_manager(self): 12 | self.assert_ok("""\ 13 | class NullContext(object): 14 | def __enter__(self): 15 | l.append('i') 16 | # __enter__ usually returns self, but doesn't have to. 17 | return 17 18 | 19 | def __exit__(self, exc_type, exc_val, exc_tb): 20 | l.append('o') 21 | return False 22 | 23 | l = [] 24 | for i in range(3): 25 | with NullContext() as val: 26 | assert val == 17 27 | l.append('w') 28 | l.append('e') 29 | l.append('r') 30 | s = ''.join(l) 31 | print("Look: %r" % s) 32 | assert s == "iwoeiwoeiwoer" 33 | """) 34 | 35 | def test_raise_in_context_manager(self): 36 | self.assert_ok("""\ 37 | class NullContext(object): 38 | def __enter__(self): 39 | l.append('i') 40 | return self 41 | 42 | def __exit__(self, exc_type, exc_val, exc_tb): 43 | assert exc_type is ValueError, \\ 44 | "Expected ValueError: %r" % exc_type 45 | l.append('o') 46 | return False 47 | 48 | l = [] 49 | try: 50 | with NullContext(): 51 | l.append('w') 52 | raise ValueError("Boo!") 53 | l.append('e') 54 | except ValueError: 55 | l.append('x') 56 | l.append('r') 57 | s = ''.join(l) 58 | print("Look: %r" % s) 59 | assert s == "iwoxr" 60 | """) 61 | 62 | def test_suppressed_raise_in_context_manager(self): 63 | self.assert_ok("""\ 64 | class SuppressingContext(object): 65 | def __enter__(self): 66 | l.append('i') 67 | return self 68 | 69 | def __exit__(self, exc_type, exc_val, exc_tb): 70 | assert exc_type is ValueError, \\ 71 | "Expected ValueError: %r" % exc_type 72 | l.append('o') 73 | return True 74 | 75 | l = [] 76 | try: 77 | with SuppressingContext(): 78 | l.append('w') 79 | raise ValueError("Boo!") 80 | l.append('e') 81 | except ValueError: 82 | l.append('x') 83 | l.append('r') 84 | s = ''.join(l) 85 | print("Look: %r" % s) 86 | assert s == "iwoer" 87 | """) 88 | 89 | def test_return_in_with(self): 90 | self.assert_ok("""\ 91 | class NullContext(object): 92 | def __enter__(self): 93 | l.append('i') 94 | return self 95 | 96 | def __exit__(self, exc_type, exc_val, exc_tb): 97 | l.append('o') 98 | return False 99 | 100 | l = [] 101 | def use_with(val): 102 | with NullContext(): 103 | l.append('w') 104 | return val 105 | l.append('e') 106 | 107 | assert use_with(23) == 23 108 | l.append('r') 109 | s = ''.join(l) 110 | print("Look: %r" % s) 111 | assert s == "iwor" 112 | """) 113 | 114 | def test_continue_in_with(self): 115 | self.assert_ok("""\ 116 | class NullContext(object): 117 | def __enter__(self): 118 | l.append('i') 119 | return self 120 | 121 | def __exit__(self, exc_type, exc_val, exc_tb): 122 | l.append('o') 123 | return False 124 | 125 | l = [] 126 | for i in range(3): 127 | with NullContext(): 128 | l.append('w') 129 | if i % 2: 130 | continue 131 | l.append('z') 132 | l.append('e') 133 | 134 | l.append('r') 135 | s = ''.join(l) 136 | print("Look: %r" % s) 137 | assert s == "iwzoeiwoiwzoer" 138 | """) 139 | 140 | def test_break_in_with(self): 141 | self.assert_ok("""\ 142 | class NullContext(object): 143 | def __enter__(self): 144 | l.append('i') 145 | return self 146 | 147 | def __exit__(self, exc_type, exc_val, exc_tb): 148 | l.append('o') 149 | return False 150 | 151 | l = [] 152 | for i in range(3): 153 | with NullContext(): 154 | l.append('w') 155 | if i % 2: 156 | break 157 | l.append('z') 158 | l.append('e') 159 | 160 | l.append('r') 161 | s = ''.join(l) 162 | print("Look: %r" % s) 163 | assert s == "iwzoeiwor" 164 | """) 165 | 166 | def test_raise_in_with(self): 167 | self.assert_ok("""\ 168 | class NullContext(object): 169 | def __enter__(self): 170 | l.append('i') 171 | return self 172 | 173 | def __exit__(self, exc_type, exc_val, exc_tb): 174 | l.append('o') 175 | return False 176 | 177 | l = [] 178 | try: 179 | with NullContext(): 180 | l.append('w') 181 | raise ValueError("oops") 182 | l.append('z') 183 | l.append('e') 184 | except ValueError as e: 185 | assert str(e) == "oops" 186 | l.append('x') 187 | l.append('r') 188 | s = ''.join(l) 189 | print("Look: %r" % s) 190 | assert s == "iwoxr", "What!?" 191 | """) 192 | 193 | def test_at_context_manager_simplified(self): 194 | self.assert_ok("""\ 195 | class GeneratorContextManager(object): 196 | def __init__(self, gen): 197 | self.gen = gen 198 | 199 | def __enter__(self): 200 | try: 201 | return next(self.gen) 202 | except StopIteration: 203 | raise RuntimeError("generator didn't yield") 204 | 205 | def __exit__(self, type, value, traceback): 206 | if type is None: 207 | try: 208 | next(self.gen) 209 | except StopIteration: 210 | return 211 | else: 212 | raise RuntimeError("generator didn't stop") 213 | else: 214 | if value is None: 215 | value = type() 216 | try: 217 | self.gen.throw(type, value, traceback) 218 | raise RuntimeError( 219 | "generator didn't stop after throw()" 220 | ) 221 | except StopIteration as exc: 222 | return exc is not value 223 | except: 224 | if sys.exc_info()[1] is not value: 225 | raise 226 | 227 | def contextmanager(func): 228 | def helper(*args, **kwds): 229 | return GeneratorContextManager(func(*args, **kwds)) 230 | return helper 231 | 232 | @contextmanager 233 | def my_context_manager(val): 234 | yield val 235 | 236 | with my_context_manager(17) as x: 237 | assert x == 17 238 | """) 239 | 240 | def test_at_context_manager_complete(self): 241 | # The complete code for an @contextmanager example, lifted from 242 | # the stdlib. 243 | self.assert_ok("""\ 244 | from _functools import partial 245 | 246 | WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__doc__') 247 | WRAPPER_UPDATES = ('__dict__',) 248 | 249 | def update_wrapper(wrapper, 250 | wrapped, 251 | assigned = WRAPPER_ASSIGNMENTS, 252 | updated = WRAPPER_UPDATES): 253 | for attr in assigned: 254 | setattr(wrapper, attr, getattr(wrapped, attr)) 255 | for attr in updated: 256 | getattr(wrapper, attr).update(getattr(wrapped, attr, {})) 257 | # Return the wrapper so this can be used as a decorator 258 | # via partial(). 259 | return wrapper 260 | 261 | def wraps(wrapped, 262 | assigned = WRAPPER_ASSIGNMENTS, 263 | updated = WRAPPER_UPDATES): 264 | return partial(update_wrapper, wrapped=wrapped, 265 | assigned=assigned, updated=updated) 266 | 267 | class GeneratorContextManager(object): 268 | def __init__(self, gen): 269 | self.gen = gen 270 | 271 | def __enter__(self): 272 | try: 273 | return next(self.gen) 274 | except StopIteration: 275 | raise RuntimeError("generator didn't yield") 276 | 277 | def __exit__(self, type, value, traceback): 278 | if type is None: 279 | try: 280 | next(self.gen) 281 | except StopIteration: 282 | return 283 | else: 284 | raise RuntimeError("generator didn't stop") 285 | else: 286 | if value is None: 287 | value = type() 288 | try: 289 | self.gen.throw(type, value, traceback) 290 | raise RuntimeError( 291 | "generator didn't stop after throw()" 292 | ) 293 | except StopIteration as exc: 294 | return exc is not value 295 | except: 296 | if sys.exc_info()[1] is not value: 297 | raise 298 | 299 | def contextmanager(func): 300 | @wraps(func) 301 | def helper(*args, **kwds): 302 | return GeneratorContextManager(func(*args, **kwds)) 303 | return helper 304 | 305 | @contextmanager 306 | def my_context_manager(val): 307 | yield val 308 | 309 | with my_context_manager(17) as x: 310 | assert x == 17 311 | """) 312 | 313 | if __name__ == "__main__": 314 | unittest.main() 315 | -------------------------------------------------------------------------------- /tests/vmtest.py: -------------------------------------------------------------------------------- 1 | """Testing tools for byterun.""" 2 | 3 | from __future__ import print_function 4 | 5 | import dis 6 | import sys 7 | import textwrap 8 | import types 9 | import unittest 10 | 11 | 12 | from byterun.abstractvm import AbstractVirtualMachine 13 | from byterun.pyvm2 import VirtualMachine, VirtualMachineError 14 | import six 15 | 16 | # Make this false if you need to run the debugger inside a test. 17 | CAPTURE_STDOUT = ("-s" not in sys.argv) 18 | # Make this false to see the traceback from a failure inside pyvm2. 19 | CAPTURE_EXCEPTION = True 20 | 21 | 22 | def dis_code(code): 23 | """Disassemble `code` and all the code it refers to.""" 24 | for const in code.co_consts: 25 | if isinstance(const, types.CodeType): 26 | dis_code(const) 27 | 28 | print("") 29 | print(code) 30 | dis.dis(code) 31 | sys.stdout.flush() 32 | 33 | 34 | def run_with_byterun(code, vmclass=VirtualMachine): 35 | real_stdout = sys.stdout 36 | try: 37 | # Run the code through our VM. 38 | vm_stdout = six.StringIO() 39 | if CAPTURE_STDOUT: # pragma: no branch 40 | sys.stdout = vm_stdout 41 | vm = vmclass() 42 | 43 | vm_value = vm_exc = None 44 | try: 45 | vm_value = vm.run_code(code) 46 | except VirtualMachineError: # pragma: no cover 47 | # If the VM code raises an error, show it. 48 | raise 49 | except AssertionError: # pragma: no cover 50 | # If test code fails an assert, show it. 51 | raise 52 | except Exception as e: 53 | # Otherwise, keep the exception for comparison later. 54 | if not CAPTURE_EXCEPTION: # pragma: no cover 55 | raise 56 | vm_exc = e 57 | finally: 58 | real_stdout.write("-- stdout ----------\n") 59 | real_stdout.write(vm_stdout.getvalue()) 60 | return vm_value, vm_stdout.getvalue(), vm_exc 61 | finally: 62 | sys.stdout = real_stdout 63 | 64 | 65 | def run_with_eval(code): 66 | real_stdout = sys.stdout 67 | try: 68 | # Run the code through the real Python interpreter, for comparison. 69 | py_stdout = six.StringIO() 70 | sys.stdout = py_stdout 71 | 72 | py_value = py_exc = None 73 | globs = {} 74 | try: 75 | py_value = eval(code, globs, globs) 76 | except AssertionError: # pragma: no cover 77 | raise 78 | except Exception as e: 79 | py_exc = e 80 | return py_value, py_stdout.getvalue(), py_exc 81 | finally: 82 | sys.stdout = real_stdout 83 | 84 | 85 | class VmTestCase(unittest.TestCase): 86 | 87 | def assert_ok(self, code, raises=None): 88 | """Run `code` in our VM and in real Python: they behave the same.""" 89 | 90 | code = textwrap.dedent(code) 91 | code = compile(code, "<%s>" % self.id(), "exec", 0, 1) 92 | 93 | # Print the disassembly so we'll see it if the test fails. 94 | dis_code(code) 95 | 96 | vm_value, vm_stdout_value, vm_exc = run_with_byterun(code) 97 | abstractvm_value, abstractvm_stdout_value, abstractvm_exc = ( 98 | run_with_byterun(code, AbstractVirtualMachine)) 99 | py_value, py_stdout_value, py_exc = run_with_eval(code) 100 | 101 | self.assert_same_exception(vm_exc, py_exc) 102 | self.assert_same_exception(abstractvm_exc, py_exc) 103 | self.assertEqual(vm_stdout_value, py_stdout_value) 104 | self.assertEqual(abstractvm_stdout_value, py_stdout_value) 105 | self.assertEqual(vm_value, py_value) 106 | self.assertEqual(abstractvm_value, py_value) 107 | if raises: 108 | self.assertIsInstance(vm_exc, raises) 109 | self.assertIsInstance(abstractvm_exc, raises) 110 | else: 111 | self.assertIsNone(vm_exc) 112 | self.assertIsNone(abstractvm_exc) 113 | 114 | def assert_same_exception(self, e1, e2): 115 | """Exceptions don't implement __eq__, check it ourselves.""" 116 | self.assertEqual(str(e1), str(e2)) 117 | self.assertIs(type(e1), type(e2)) 118 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (http://tox.testrun.org/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | 6 | [tox] 7 | envlist = py27, py33 8 | 9 | [testenv] 10 | commands = 11 | nosetests {posargs} 12 | 13 | deps = 14 | nose 15 | coverage 16 | --------------------------------------------------------------------------------