├── .coveragerc ├── .gitattributes ├── .gitignore ├── .travis.yml ├── LICENSE ├── MANIFEST.in ├── README.rst ├── codetransformer ├── __init__.py ├── _version.py ├── code.py ├── core.py ├── decompiler │ ├── _343.py │ └── __init__.py ├── instructions.py ├── patterns.py ├── tests │ ├── __init__.py │ ├── test_code.py │ ├── test_core.py │ ├── test_decompiler.py │ └── test_instructions.py ├── transformers │ ├── __init__.py │ ├── add2mul.py │ ├── constants.py │ ├── interpolated_strings.py │ ├── literals.py │ ├── pattern_matched_exceptions.py │ ├── precomputed_slices.py │ └── tests │ │ ├── __init__.py │ │ ├── test_add2mul.py │ │ ├── test_constants.py │ │ ├── test_exc_patterns.py │ │ ├── test_interpolated_strings.py │ │ ├── test_literals.py │ │ └── test_precomputed_slices.py └── utils │ ├── __init__.py │ ├── functional.py │ ├── immutable.py │ ├── instance.py │ ├── no_default.py │ ├── pretty.py │ └── tests │ ├── __init__.py │ ├── test_immutable.py │ └── test_pretty.py ├── docs ├── .dir-locals.el ├── Makefile └── source │ ├── add2mul.py │ ├── appendix.rst │ ├── code-objects.rst │ ├── conf.py │ ├── index.rst │ ├── magics.rst │ └── patterns.rst ├── requirements_doc.txt ├── setup.cfg ├── setup.py ├── tox.ini └── versioneer.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | codetransformer/_version.py 4 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | codetransformer/_version.py export-subst 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .bundle 2 | db/*.sqlite3 3 | log/*.log 4 | *.log 5 | tmp/**/* 6 | tmp/* 7 | *.swp 8 | *~ 9 | #mac autosaving file 10 | .DS_Store 11 | *.py[co] 12 | 13 | # Installer logs 14 | pip-log.txt 15 | 16 | # Unit test / coverage reports 17 | .coverage 18 | .tox 19 | test.log 20 | .noseids 21 | *.xlsx 22 | 23 | # Compiled python files 24 | *.py[co] 25 | 26 | # Packages 27 | *.egg 28 | *.egg-info 29 | dist 30 | build 31 | eggs 32 | cover 33 | parts 34 | bin 35 | var 36 | sdist 37 | develop-eggs 38 | .installed.cfg 39 | coverage.xml 40 | nosetests.xml 41 | 42 | # C Extensions 43 | *.o 44 | *.so 45 | *.out 46 | 47 | # Vim 48 | *.swp 49 | *.swo 50 | 51 | # Built documentation 52 | docs/_build/* 53 | 54 | # database of vbench 55 | benchmarks.db 56 | 57 | # Vagrant temp folder 58 | .vagrant 59 | 60 | # pypi 61 | MANIFEST 62 | 63 | # pytest 64 | .cache 65 | 66 | htmlcov 67 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | python: 4 | - 3.4.3 5 | - 3.4 6 | - 3.5 7 | - 3.6 8 | 9 | install: 10 | - pip install -e .[dev] 11 | 12 | script: 13 | - py.test codetransformer 14 | - flake8 codetransformer 15 | 16 | notifications: 17 | email: false 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include versioneer.py 2 | include codetransformer/_version.py 3 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ``codetransformer`` 2 | =================== 3 | 4 | |build status| |documentation| 5 | 6 | Bytecode transformers for CPython inspired by the ``ast`` module's 7 | ``NodeTransformer``. 8 | 9 | What is ``codetransformer``? 10 | ---------------------------- 11 | 12 | ``codetransformer`` is a library that allows us to work with CPython's bytecode 13 | representation at runtime. ``codetransformer`` provides a level of abstraction 14 | between the programmer and the raw bytes read by the eval loop so that we can 15 | more easily inspect and modify bytecode. 16 | 17 | ``codetransformer`` is motivated by the need to override parts of the python 18 | language that are not already hooked into through data model methods. For example: 19 | 20 | * Override the ``is`` and ``not`` operators. 21 | * Custom data structure literals. 22 | * Syntax features that cannot be represented with valid python AST or source. 23 | * Run without a modified CPython interpreter. 24 | 25 | ``codetransformer`` was originally developed as part of lazy_ to implement 26 | the transformations needed to override the code objects at runtime. 27 | 28 | Example Uses 29 | ------------ 30 | 31 | Overloading Literals 32 | ~~~~~~~~~~~~~~~~~~~~ 33 | 34 | While this can be done as an AST transformation, we will often need to execute 35 | the constructor for the literal multiple times. Also, we need to be sure that 36 | any additional names required to run our code are provided when we run. With 37 | ``codetransformer``, we can pre compute our new literals and emit code that is 38 | as fast as loading our unmodified literals without requiring any additional 39 | names be available implicitly. 40 | 41 | In the following block we demonstrate overloading dictionary syntax to result in 42 | ``collections.OrderedDict`` objects. ``OrderedDict`` is like a ``dict``; 43 | however, the order of the keys is preserved. 44 | 45 | .. code-block:: python 46 | 47 | >>> from codetransformer.transformers.literals import ordereddict_literals 48 | >>> @ordereddict_literals 49 | ... def f(): 50 | ... return {'a': 1, 'b': 2, 'c': 3} 51 | >>> f() 52 | OrderedDict([('a', 1), ('b', 2), ('c', 3)]) 53 | 54 | This also supports dictionary comprehensions: 55 | 56 | .. code-block:: python 57 | 58 | >>> @ordereddict_literals 59 | ... def f(): 60 | ... return {k: v for k, v in zip('abc', (1, 2, 3))} 61 | >>> f() 62 | OrderedDict([('a', 1), ('b', 2), ('c', 3)]) 63 | 64 | The next block overrides ``float`` literals with ``decimal.Decimal`` 65 | objects. These objects support arbitrary precision arithmetic. 66 | 67 | .. code-block:: python 68 | 69 | >>> from codetransformer.transformers.literals import decimal_literals 70 | >>> @decimal_literals 71 | ... def f(): 72 | ... return 1.5 73 | >>> f() 74 | Decimal('1.5') 75 | 76 | Pattern Matched Exceptions 77 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 78 | 79 | Pattern matched exceptions are a good example of a ``CodeTransformer`` that 80 | would be very complicated to implement at the AST level. This transformation 81 | extends the ``try/except`` syntax to accept instances of ``BaseException`` as 82 | well subclasses of ``BaseException``. When excepting an instance, the ``args`` 83 | of the exception will be compared for equality to determine which exception 84 | handler should be invoked. For example: 85 | 86 | .. code-block:: python 87 | 88 | >>> @pattern_matched_exceptions() 89 | ... def foo(): 90 | ... try: 91 | ... raise ValueError('bar') 92 | ... except ValueError('buzz'): 93 | ... return 'buzz' 94 | ... except ValueError('bar'): 95 | ... return 'bar' 96 | >>> foo() 97 | 'bar' 98 | 99 | This function raises an instance of ``ValueError`` and attempts to catch it. The 100 | first check looks for instances of ``ValueError`` that were constructed with an 101 | argument of ``'buzz'``. Because our custom exception is raised with ``'bar'``, 102 | these are not equal and we do not enter this handler. The next handler looks for 103 | ``ValueError('bar')`` which does match the exception we raised. We then enter 104 | this block and normal python rules take over. 105 | 106 | We may also pass their own exception matching function: 107 | 108 | .. code-block:: python 109 | 110 | >>> def match_greater(match_expr, exc_type, exc_value, exc_traceback): 111 | ... return math_expr > exc_value.args[0] 112 | 113 | >>> @pattern_matched_exceptions(match_greater) 114 | ... def foo(): 115 | ... try: 116 | ... raise ValueError(5) 117 | ... except 4: 118 | ... return 4 119 | ... except 5: 120 | ... return 5 121 | ... except 6: 122 | ... return 6 123 | >>> foo() 124 | 6 125 | 126 | This matches on when the match expression is greater in value than the first 127 | argument of any exception type that is raised. This particular behavior would be 128 | very hard to mimic through AST level transformations. 129 | 130 | Core Abstractions 131 | ----------------- 132 | 133 | The three core abstractions of ``codetransformer`` are: 134 | 135 | 1. The ``Instruction`` object which represents an opcode_ which may be paired 136 | with some argument. 137 | 2. The ``Code`` object which represents a collection of ``Instruction``\s. 138 | 3. The ``CodeTransformer`` object which represents a set of rules for 139 | manipulating ``Code`` objects. 140 | 141 | Instructions 142 | ~~~~~~~~~~~~ 143 | 144 | The ``Instruction`` object represents an atomic operation that can be performed 145 | by the CPython virtual machine. These are things like ``LOAD_NAME`` which loads 146 | a name onto the stack, or ``ROT_TWO`` which rotates the top two stack elements. 147 | 148 | Some instructions accept an argument, for example ``LOAD_NAME``, which modifies 149 | the behavior of the instruction. This is much like a function call where some 150 | functions accept arguments. Because the bytecode is always packed as raw bytes, 151 | the argument must be some integer (CPython stores all arguments two in bytes). 152 | This means that things that need a more rich argument system (like ``LOAD_NAME`` 153 | which needs the actual name to look up) must carry around the actual arguments 154 | in some table and use the integer as an offset into this array. One of the key 155 | abstractions of the ``Instruction`` object is that the argument is always some 156 | python object that represents the actual argument. Any lookup table management 157 | is handled for the user. This is helpful because some arguments share this table 158 | so we don't want to add extra entries or forget to add them at all. 159 | 160 | Another annoyance is that the instructions that handle control flow use their 161 | argument to say what bytecode offset to jump to. Some jumps use the absolute 162 | index, others use a relative index. This also makes it hard if you want to add 163 | or remove instructions because all of the offsets must be recomputed. In 164 | ``codetransformer``, the jump instructions all accept another ``Instruction`` as 165 | the argument so that the assembler can manage this for the user. We also provide 166 | an easy way for new instructions to "steal" jumps that targeted another 167 | instruction so that can manage altering the bytecode around jump targets. 168 | 169 | Code 170 | ~~~~ 171 | 172 | ``Code`` objects are a nice abstraction over python's 173 | ``types.CodeType``. Quoting the ``CodeType`` constructor docstring: 174 | 175 | :: 176 | 177 | code(argcount, kwonlyargcount, nlocals, stacksize, flags, codestring, 178 | constants, names, varnames, filename, name, firstlineno, 179 | lnotab[, freevars[, cellvars]]) 180 | 181 | Create a code object. Not for the faint of heart. 182 | 183 | The ``codetransformer`` abstraction is designed to make it easy to dynamically 184 | construct and inspect these objects. This allows us to easy set things like the 185 | argument names, and manipulate the line number mappings. 186 | 187 | The ``Code`` object provides methods for converting to and from Python's code 188 | representation: 189 | 190 | 1. ``from_pycode`` 191 | 2. ``to_pycode``. 192 | 193 | This allows us to take an existing function, parse the meaning from it, modify 194 | it, and then assemble this back into a new python code object. 195 | 196 | .. note:: 197 | 198 | ``Code`` objects are immutable. When we say "modify", we mean create a copy 199 | with different values. 200 | 201 | CodeTransformers 202 | ---------------- 203 | 204 | This is the set of rules that are used to actually modify the ``Code`` 205 | objects. These rules are defined as a set of ``patterns`` which are a DSL used 206 | to define a DFA for matching against sequences of ``Instruction`` objects. Once 207 | we have matched a segment, we yield new instructions to replace what we have 208 | matched. A simple codetransformer looks like: 209 | 210 | .. code-block:: python 211 | 212 | from codetransformer import CodeTransformer, instructions 213 | 214 | class FoldNames(CodeTransformer): 215 | @pattern( 216 | instructions.LOAD_GLOBAL, 217 | instructions.LOAD_GLOBAL, 218 | instructions.BINARY_ADD, 219 | ) 220 | def _load_fast(self, a, b, add): 221 | yield instructions.LOAD_FAST(a.arg + b.arg).steal(a) 222 | 223 | This ``CodeTransformer`` uses the ``+`` operator to implement something like 224 | ``CPP``\s token pasting for local variables. We read this pattern as a sequence 225 | of two ``LOAD_GLOBAL`` (global name lookups) followed by a ``BINARY_ADD`` 226 | instruction (``+`` operator call). This will then call the function with the 227 | three instructions passed positionally. This handler replaces this sequence with 228 | a single instruction that emits a ``LOAD_FAST`` (local name lookup) that is the 229 | result of adding the two names together. We then steal any jumps that used to 230 | target the first ``LOAD_GLOBAL``. 231 | 232 | We can execute this transformer by calling an instance of it on a 233 | function object, or using it like a decorator. For example: 234 | 235 | .. code-block:: python 236 | 237 | >>> @FoldNames() 238 | ... def f(): 239 | ... ab = 3 240 | ... return a + b 241 | >>> f() 242 | 3 243 | 244 | 245 | License 246 | ------- 247 | 248 | ``codetransformer`` is free software, licensed under the GNU General Public 249 | License, version 2. For more information see the ``LICENSE`` file. 250 | 251 | 252 | Source 253 | ------ 254 | 255 | Source code is hosted on github at 256 | https://github.com/llllllllll/codetransformer. 257 | 258 | 259 | .. _lazy: https://github.com/llllllllll/lazy_python 260 | .. _opcode: https://docs.python.org/3.5/library/dis.html#opcode-NOP 261 | .. |build status| image:: https://travis-ci.org/llllllllll/codetransformer.svg?branch=master 262 | :target: https://travis-ci.org/llllllllll/codetransformer 263 | .. |documentation| image:: https://readthedocs.org/projects/codetransformer/badge/?version=stable 264 | :target: http://codetransformer.readthedocs.io/en/stable/?badge=stable 265 | :alt: Documentation Status 266 | -------------------------------------------------------------------------------- /codetransformer/__init__.py: -------------------------------------------------------------------------------- 1 | from .code import Code, Flag 2 | from .core import CodeTransformer 3 | from . patterns import ( 4 | matchany, 5 | not_, 6 | option, 7 | or_, 8 | pattern, 9 | plus, 10 | seq, 11 | var, 12 | ) 13 | from . import instructions 14 | from . import transformers 15 | from .utils.pretty import a, d, display, pprint_ast, pformat_ast 16 | from ._version import get_versions 17 | 18 | 19 | __version__ = get_versions()['version'] 20 | del get_versions 21 | 22 | 23 | def load_ipython_extension(ipython): # pragma: no cover 24 | 25 | def dis_magic(line, cell=None): 26 | if cell is None: 27 | return d(line) 28 | return d(cell) 29 | ipython.register_magic_function(dis_magic, 'line_cell', 'dis') 30 | 31 | def ast_magic(line, cell=None): 32 | if cell is None: 33 | return a(line) 34 | return a(cell) 35 | ipython.register_magic_function(ast_magic, 'line_cell', 'ast') 36 | 37 | 38 | __all__ = [ 39 | 'a', 40 | 'd', 41 | 'display', 42 | 'Code', 43 | 'CodeTransformer', 44 | 'Flag', 45 | 'instructions', 46 | 'matchany', 47 | 'not_', 48 | 'option', 49 | 'or_', 50 | 'pattern', 51 | 'pattern', 52 | 'plus', 53 | 'pformat_ast', 54 | 'pprint_ast', 55 | 'seq', 56 | 'var', 57 | 'transformers', 58 | ] 59 | -------------------------------------------------------------------------------- /codetransformer/_version.py: -------------------------------------------------------------------------------- 1 | 2 | # This file helps to compute a version number in source trees obtained from 3 | # git-archive tarball (such as those provided by githubs download-from-tag 4 | # feature). Distribution tarballs (built by setup.py sdist) and build 5 | # directories (produced by setup.py build) will contain a much shorter file 6 | # that just contains the computed version number. 7 | 8 | # This file is released into the public domain. Generated by 9 | # versioneer-0.15 (https://github.com/warner/python-versioneer) 10 | 11 | import errno 12 | import os 13 | import re 14 | import subprocess 15 | import sys 16 | 17 | 18 | def get_keywords(): 19 | # these strings will be replaced by git during git-archive. 20 | # setup.py/versioneer.py will grep for the variable names, so they must 21 | # each be defined on a line of their own. _version.py will just call 22 | # get_keywords(). 23 | git_refnames = " (HEAD -> master)" 24 | git_full = "c5f551e915df45adc7da7e0b1b635f0cc6a1bb27" 25 | keywords = {"refnames": git_refnames, "full": git_full} 26 | return keywords 27 | 28 | 29 | class VersioneerConfig: 30 | pass 31 | 32 | 33 | def get_config(): 34 | # these strings are filled in when 'setup.py versioneer' creates 35 | # _version.py 36 | cfg = VersioneerConfig() 37 | cfg.VCS = "git" 38 | cfg.style = "pep440" 39 | cfg.tag_prefix = "" 40 | cfg.parentdir_prefix = "codetransformer-" 41 | cfg.versionfile_source = "codetransformer/_version.py" 42 | cfg.verbose = False 43 | return cfg 44 | 45 | 46 | class NotThisMethod(Exception): 47 | pass 48 | 49 | 50 | LONG_VERSION_PY = {} 51 | HANDLERS = {} 52 | 53 | 54 | def register_vcs_handler(vcs, method): # decorator 55 | def decorate(f): 56 | if vcs not in HANDLERS: 57 | HANDLERS[vcs] = {} 58 | HANDLERS[vcs][method] = f 59 | return f 60 | return decorate 61 | 62 | 63 | def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): 64 | assert isinstance(commands, list) 65 | p = None 66 | for c in commands: 67 | try: 68 | dispcmd = str([c] + args) 69 | # remember shell=False, so use git.cmd on windows, not just git 70 | p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, 71 | stderr=(subprocess.PIPE if hide_stderr 72 | else None)) 73 | break 74 | except EnvironmentError: 75 | e = sys.exc_info()[1] 76 | if e.errno == errno.ENOENT: 77 | continue 78 | if verbose: 79 | print("unable to run %s" % dispcmd) 80 | print(e) 81 | return None 82 | else: 83 | if verbose: 84 | print("unable to find command, tried %s" % (commands,)) 85 | return None 86 | stdout = p.communicate()[0].strip() 87 | if sys.version_info[0] >= 3: 88 | stdout = stdout.decode() 89 | if p.returncode != 0: 90 | if verbose: 91 | print("unable to run %s (error)" % dispcmd) 92 | return None 93 | return stdout 94 | 95 | 96 | def versions_from_parentdir(parentdir_prefix, root, verbose): 97 | # Source tarballs conventionally unpack into a directory that includes 98 | # both the project name and a version string. 99 | dirname = os.path.basename(root) 100 | if not dirname.startswith(parentdir_prefix): 101 | if verbose: 102 | print("guessing rootdir is '%s', but '%s' doesn't start with " 103 | "prefix '%s'" % (root, dirname, parentdir_prefix)) 104 | raise NotThisMethod("rootdir doesn't start with parentdir_prefix") 105 | return {"version": dirname[len(parentdir_prefix):], 106 | "full-revisionid": None, 107 | "dirty": False, "error": None} 108 | 109 | 110 | @register_vcs_handler("git", "get_keywords") 111 | def git_get_keywords(versionfile_abs): 112 | # the code embedded in _version.py can just fetch the value of these 113 | # keywords. When used from setup.py, we don't want to import _version.py, 114 | # so we do it with a regexp instead. This function is not used from 115 | # _version.py. 116 | keywords = {} 117 | try: 118 | f = open(versionfile_abs, "r") 119 | for line in f.readlines(): 120 | if line.strip().startswith("git_refnames ="): 121 | mo = re.search(r'=\s*"(.*)"', line) 122 | if mo: 123 | keywords["refnames"] = mo.group(1) 124 | if line.strip().startswith("git_full ="): 125 | mo = re.search(r'=\s*"(.*)"', line) 126 | if mo: 127 | keywords["full"] = mo.group(1) 128 | f.close() 129 | except EnvironmentError: 130 | pass 131 | return keywords 132 | 133 | 134 | @register_vcs_handler("git", "keywords") 135 | def git_versions_from_keywords(keywords, tag_prefix, verbose): 136 | if not keywords: 137 | raise NotThisMethod("no keywords at all, weird") 138 | refnames = keywords["refnames"].strip() 139 | if refnames.startswith("$Format"): 140 | if verbose: 141 | print("keywords are unexpanded, not using") 142 | raise NotThisMethod("unexpanded keywords, not a git-archive tarball") 143 | refs = set([r.strip() for r in refnames.strip("()").split(",")]) 144 | # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of 145 | # just "foo-1.0". If we see a "tag: " prefix, prefer those. 146 | TAG = "tag: " 147 | tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) 148 | if not tags: 149 | # Either we're using git < 1.8.3, or there really are no tags. We use 150 | # a heuristic: assume all version tags have a digit. The old git %d 151 | # expansion behaves like git log --decorate=short and strips out the 152 | # refs/heads/ and refs/tags/ prefixes that would let us distinguish 153 | # between branches and tags. By ignoring refnames without digits, we 154 | # filter out many common branch names like "release" and 155 | # "stabilization", as well as "HEAD" and "master". 156 | tags = set([r for r in refs if re.search(r'\d', r)]) 157 | if verbose: 158 | print("discarding '%s', no digits" % ",".join(refs-tags)) 159 | if verbose: 160 | print("likely tags: %s" % ",".join(sorted(tags))) 161 | for ref in sorted(tags): 162 | # sorting will prefer e.g. "2.0" over "2.0rc1" 163 | if ref.startswith(tag_prefix): 164 | r = ref[len(tag_prefix):] 165 | if verbose: 166 | print("picking %s" % r) 167 | return {"version": r, 168 | "full-revisionid": keywords["full"].strip(), 169 | "dirty": False, "error": None 170 | } 171 | # no suitable tags, so version is "0+unknown", but full hex is still there 172 | if verbose: 173 | print("no suitable tags, using unknown + full revision id") 174 | return {"version": "0+unknown", 175 | "full-revisionid": keywords["full"].strip(), 176 | "dirty": False, "error": "no suitable tags"} 177 | 178 | 179 | @register_vcs_handler("git", "pieces_from_vcs") 180 | def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): 181 | # this runs 'git' from the root of the source tree. This only gets called 182 | # if the git-archive 'subst' keywords were *not* expanded, and 183 | # _version.py hasn't already been rewritten with a short version string, 184 | # meaning we're inside a checked out source tree. 185 | 186 | if not os.path.exists(os.path.join(root, ".git")): 187 | if verbose: 188 | print("no .git in %s" % root) 189 | raise NotThisMethod("no .git directory") 190 | 191 | GITS = ["git"] 192 | if sys.platform == "win32": 193 | GITS = ["git.cmd", "git.exe"] 194 | # if there is a tag, this yields TAG-NUM-gHEX[-dirty] 195 | # if there are no tags, this yields HEX[-dirty] (no NUM) 196 | describe_out = run_command(GITS, ["describe", "--tags", "--dirty", 197 | "--always", "--long"], 198 | cwd=root) 199 | # --long was added in git-1.5.5 200 | if describe_out is None: 201 | raise NotThisMethod("'git describe' failed") 202 | describe_out = describe_out.strip() 203 | full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) 204 | if full_out is None: 205 | raise NotThisMethod("'git rev-parse' failed") 206 | full_out = full_out.strip() 207 | 208 | pieces = {} 209 | pieces["long"] = full_out 210 | pieces["short"] = full_out[:7] # maybe improved later 211 | pieces["error"] = None 212 | 213 | # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] 214 | # TAG might have hyphens. 215 | git_describe = describe_out 216 | 217 | # look for -dirty suffix 218 | dirty = git_describe.endswith("-dirty") 219 | pieces["dirty"] = dirty 220 | if dirty: 221 | git_describe = git_describe[:git_describe.rindex("-dirty")] 222 | 223 | # now we have TAG-NUM-gHEX or HEX 224 | 225 | if "-" in git_describe: 226 | # TAG-NUM-gHEX 227 | mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) 228 | if not mo: 229 | # unparseable. Maybe git-describe is misbehaving? 230 | pieces["error"] = ("unable to parse git-describe output: '%s'" 231 | % describe_out) 232 | return pieces 233 | 234 | # tag 235 | full_tag = mo.group(1) 236 | if not full_tag.startswith(tag_prefix): 237 | if verbose: 238 | fmt = "tag '%s' doesn't start with prefix '%s'" 239 | print(fmt % (full_tag, tag_prefix)) 240 | pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" 241 | % (full_tag, tag_prefix)) 242 | return pieces 243 | pieces["closest-tag"] = full_tag[len(tag_prefix):] 244 | 245 | # distance: number of commits since tag 246 | pieces["distance"] = int(mo.group(2)) 247 | 248 | # commit: short hex revision ID 249 | pieces["short"] = mo.group(3) 250 | 251 | else: 252 | # HEX: no tags 253 | pieces["closest-tag"] = None 254 | count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], 255 | cwd=root) 256 | pieces["distance"] = int(count_out) # total number of commits 257 | 258 | return pieces 259 | 260 | 261 | def plus_or_dot(pieces): 262 | if "+" in pieces.get("closest-tag", ""): 263 | return "." 264 | return "+" 265 | 266 | 267 | def render_pep440(pieces): 268 | # now build up version string, with post-release "local version 269 | # identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you 270 | # get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty 271 | 272 | # exceptions: 273 | # 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] 274 | 275 | if pieces["closest-tag"]: 276 | rendered = pieces["closest-tag"] 277 | if pieces["distance"] or pieces["dirty"]: 278 | rendered += plus_or_dot(pieces) 279 | rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) 280 | if pieces["dirty"]: 281 | rendered += ".dirty" 282 | else: 283 | # exception #1 284 | rendered = "0+untagged.%d.g%s" % (pieces["distance"], 285 | pieces["short"]) 286 | if pieces["dirty"]: 287 | rendered += ".dirty" 288 | return rendered 289 | 290 | 291 | def render_pep440_pre(pieces): 292 | # TAG[.post.devDISTANCE] . No -dirty 293 | 294 | # exceptions: 295 | # 1: no tags. 0.post.devDISTANCE 296 | 297 | if pieces["closest-tag"]: 298 | rendered = pieces["closest-tag"] 299 | if pieces["distance"]: 300 | rendered += ".post.dev%d" % pieces["distance"] 301 | else: 302 | # exception #1 303 | rendered = "0.post.dev%d" % pieces["distance"] 304 | return rendered 305 | 306 | 307 | def render_pep440_post(pieces): 308 | # TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that 309 | # .dev0 sorts backwards (a dirty tree will appear "older" than the 310 | # corresponding clean one), but you shouldn't be releasing software with 311 | # -dirty anyways. 312 | 313 | # exceptions: 314 | # 1: no tags. 0.postDISTANCE[.dev0] 315 | 316 | if pieces["closest-tag"]: 317 | rendered = pieces["closest-tag"] 318 | if pieces["distance"] or pieces["dirty"]: 319 | rendered += ".post%d" % pieces["distance"] 320 | if pieces["dirty"]: 321 | rendered += ".dev0" 322 | rendered += plus_or_dot(pieces) 323 | rendered += "g%s" % pieces["short"] 324 | else: 325 | # exception #1 326 | rendered = "0.post%d" % pieces["distance"] 327 | if pieces["dirty"]: 328 | rendered += ".dev0" 329 | rendered += "+g%s" % pieces["short"] 330 | return rendered 331 | 332 | 333 | def render_pep440_old(pieces): 334 | # TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. 335 | 336 | # exceptions: 337 | # 1: no tags. 0.postDISTANCE[.dev0] 338 | 339 | if pieces["closest-tag"]: 340 | rendered = pieces["closest-tag"] 341 | if pieces["distance"] or pieces["dirty"]: 342 | rendered += ".post%d" % pieces["distance"] 343 | if pieces["dirty"]: 344 | rendered += ".dev0" 345 | else: 346 | # exception #1 347 | rendered = "0.post%d" % pieces["distance"] 348 | if pieces["dirty"]: 349 | rendered += ".dev0" 350 | return rendered 351 | 352 | 353 | def render_git_describe(pieces): 354 | # TAG[-DISTANCE-gHEX][-dirty], like 'git describe --tags --dirty 355 | # --always' 356 | 357 | # exceptions: 358 | # 1: no tags. HEX[-dirty] (note: no 'g' prefix) 359 | 360 | if pieces["closest-tag"]: 361 | rendered = pieces["closest-tag"] 362 | if pieces["distance"]: 363 | rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) 364 | else: 365 | # exception #1 366 | rendered = pieces["short"] 367 | if pieces["dirty"]: 368 | rendered += "-dirty" 369 | return rendered 370 | 371 | 372 | def render_git_describe_long(pieces): 373 | # TAG-DISTANCE-gHEX[-dirty], like 'git describe --tags --dirty 374 | # --always -long'. The distance/hash is unconditional. 375 | 376 | # exceptions: 377 | # 1: no tags. HEX[-dirty] (note: no 'g' prefix) 378 | 379 | if pieces["closest-tag"]: 380 | rendered = pieces["closest-tag"] 381 | rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) 382 | else: 383 | # exception #1 384 | rendered = pieces["short"] 385 | if pieces["dirty"]: 386 | rendered += "-dirty" 387 | return rendered 388 | 389 | 390 | def render(pieces, style): 391 | if pieces["error"]: 392 | return {"version": "unknown", 393 | "full-revisionid": pieces.get("long"), 394 | "dirty": None, 395 | "error": pieces["error"]} 396 | 397 | if not style or style == "default": 398 | style = "pep440" # the default 399 | 400 | if style == "pep440": 401 | rendered = render_pep440(pieces) 402 | elif style == "pep440-pre": 403 | rendered = render_pep440_pre(pieces) 404 | elif style == "pep440-post": 405 | rendered = render_pep440_post(pieces) 406 | elif style == "pep440-old": 407 | rendered = render_pep440_old(pieces) 408 | elif style == "git-describe": 409 | rendered = render_git_describe(pieces) 410 | elif style == "git-describe-long": 411 | rendered = render_git_describe_long(pieces) 412 | else: 413 | raise ValueError("unknown style '%s'" % style) 414 | 415 | return {"version": rendered, "full-revisionid": pieces["long"], 416 | "dirty": pieces["dirty"], "error": None} 417 | 418 | 419 | def get_versions(): 420 | # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have 421 | # __file__, we can work backwards from there to the root. Some 422 | # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which 423 | # case we can only use expanded keywords. 424 | 425 | cfg = get_config() 426 | verbose = cfg.verbose 427 | 428 | try: 429 | return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, 430 | verbose) 431 | except NotThisMethod: 432 | pass 433 | 434 | try: 435 | root = os.path.realpath(__file__) 436 | # versionfile_source is the relative path from the top of the source 437 | # tree (where the .git directory might live) to this file. Invert 438 | # this to find the root from __file__. 439 | for i in cfg.versionfile_source.split('/'): 440 | root = os.path.dirname(root) 441 | except NameError: 442 | return {"version": "0+unknown", "full-revisionid": None, 443 | "dirty": None, 444 | "error": "unable to find root of source tree"} 445 | 446 | try: 447 | pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) 448 | return render(pieces, cfg.style) 449 | except NotThisMethod: 450 | pass 451 | 452 | try: 453 | if cfg.parentdir_prefix: 454 | return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) 455 | except NotThisMethod: 456 | pass 457 | 458 | return {"version": "0+unknown", "full-revisionid": None, 459 | "dirty": None, 460 | "error": "unable to compute version"} 461 | -------------------------------------------------------------------------------- /codetransformer/core.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | from contextlib import contextmanager 3 | from ctypes import py_object, pythonapi 4 | from itertools import chain 5 | from types import CodeType, FunctionType 6 | from weakref import WeakKeyDictionary 7 | 8 | try: 9 | import threading 10 | except ImportError: 11 | import dummy_threading as threading 12 | 13 | from .code import Code 14 | from .instructions import LOAD_CONST, STORE_FAST, LOAD_FAST 15 | from .patterns import ( 16 | boundpattern, 17 | patterndispatcher, 18 | DEFAULT_STARTCODE, 19 | ) 20 | from .utils.instance import instance 21 | 22 | 23 | _cell_new = pythonapi.PyCell_New 24 | _cell_new.argtypes = (py_object,) 25 | _cell_new.restype = py_object 26 | 27 | 28 | def _a_if_not_none(a, b): 29 | return a if a is not None else b 30 | 31 | 32 | def _new_lnotab(instrs, lnotab): 33 | """The updated lnotab after the instructions have been transformed. 34 | 35 | Parameters 36 | ---------- 37 | instrs : iterable[Instruction] 38 | The new instructions. 39 | lnotab : dict[Instruction -> int] 40 | The lnotab for the old code object. 41 | 42 | Returns 43 | ------- 44 | new_lnotab : dict[Instruction -> int] 45 | The post transform lnotab. 46 | """ 47 | return { 48 | lno: _a_if_not_none(instr._stolen_by, instr) 49 | for lno, instr in lnotab.items() 50 | } 51 | 52 | 53 | class NoContext(Exception): 54 | """Exception raised to indicate that the ``code` or ``startcode`` 55 | attribute was accessed outside of a code context. 56 | """ 57 | def __init__(self): 58 | return super().__init__('no active transformation context') 59 | 60 | 61 | class Context: 62 | """Empty object for holding the transformation context. 63 | """ 64 | def __init__(self, code): 65 | self.code = code 66 | self.startcode = DEFAULT_STARTCODE 67 | 68 | def __repr__(self): # pragma: no cover 69 | return '<%s: %r>' % (type(self).__name__, self.__dict__) 70 | 71 | 72 | class CodeTransformerMeta(type): 73 | """Meta class for CodeTransformer to collect all of the patterns 74 | and ensure the class dict is ordered. 75 | 76 | Patterns are created when a method is decorated with 77 | ``codetransformer.pattern.pattern`` 78 | """ 79 | def __new__(mcls, name, bases, dict_): 80 | dict_['patterndispatcher'] = patterndispatcher(*chain( 81 | (v for v in dict_.values() if isinstance(v, boundpattern)), 82 | *( 83 | d and d.patterns for d in ( 84 | getattr(b, 'patterndispatcher', ()) for b in bases 85 | ) 86 | ) 87 | )) 88 | return super().__new__(mcls, name, bases, dict_) 89 | 90 | def __prepare__(self, bases): 91 | return OrderedDict() 92 | 93 | 94 | class CodeTransformer(metaclass=CodeTransformerMeta): 95 | """A code object transformer, similar to the NodeTransformer 96 | from the ast module. 97 | 98 | Attributes 99 | ---------- 100 | code 101 | """ 102 | __slots__ = '__weakref__', 103 | 104 | def transform_consts(self, consts): 105 | """transformer for the co_consts field. 106 | 107 | Override this method to transform the `co_consts` of the code object. 108 | 109 | Parameters 110 | ---------- 111 | consts : tuple 112 | The co_consts 113 | 114 | Returns 115 | ------- 116 | new_consts : tuple 117 | The new constants. 118 | """ 119 | return tuple( 120 | self.transform(Code.from_pycode(const)).to_pycode() 121 | if isinstance(const, CodeType) else 122 | const 123 | for const in consts 124 | ) 125 | 126 | def _id(self, obj): 127 | """Identity function. 128 | 129 | Parameters 130 | ---------- 131 | obj : any 132 | The object to return 133 | 134 | Returns 135 | ------- 136 | obj : any 137 | The input unchanged 138 | """ 139 | return obj 140 | 141 | transform_name = _id 142 | transform_names = _id 143 | transform_varnames = _id 144 | transform_freevars = _id 145 | transform_cellvars = _id 146 | transform_defaults = _id 147 | 148 | del _id 149 | 150 | def transform(self, code, *, name=None, filename=None): 151 | """Transform a codetransformer.Code object applying the transforms. 152 | 153 | Parameters 154 | ---------- 155 | code : Code 156 | The code object to transform. 157 | name : str, optional 158 | The new name for this code object. 159 | filename : str, optional 160 | The new filename for this code object. 161 | 162 | Returns 163 | ------- 164 | new_code : Code 165 | The transformed code object. 166 | """ 167 | # reverse lookups from for constants and names. 168 | reversed_consts = {} 169 | reversed_names = {} 170 | reversed_varnames = {} 171 | for instr in code: 172 | if isinstance(instr, LOAD_CONST): 173 | reversed_consts[instr] = instr.arg 174 | if instr.uses_name: 175 | reversed_names[instr] = instr.arg 176 | if isinstance(instr, (STORE_FAST, LOAD_FAST)): 177 | reversed_varnames[instr] = instr.arg 178 | 179 | instrs, consts = tuple(zip(*reversed_consts.items())) or ((), ()) 180 | for instr, const in zip(instrs, self.transform_consts(consts)): 181 | instr.arg = const 182 | 183 | instrs, names = tuple(zip(*reversed_names.items())) or ((), ()) 184 | for instr, name_ in zip(instrs, self.transform_names(names)): 185 | instr.arg = name_ 186 | 187 | instrs, varnames = tuple(zip(*reversed_varnames.items())) or ((), ()) 188 | for instr, varname in zip(instrs, self.transform_varnames(varnames)): 189 | instr.arg = varname 190 | 191 | with self._new_context(code): 192 | post_transform = self.patterndispatcher(code) 193 | 194 | return Code( 195 | post_transform, 196 | code.argnames, 197 | cellvars=self.transform_cellvars(code.cellvars), 198 | freevars=self.transform_freevars(code.freevars), 199 | name=name if name is not None else code.name, 200 | filename=filename if filename is not None else code.filename, 201 | firstlineno=code.firstlineno, 202 | lnotab=_new_lnotab(post_transform, code.lnotab), 203 | flags=code.flags, 204 | ) 205 | 206 | def __call__(self, f, *, 207 | globals_=None, name=None, defaults=None, closure=None): 208 | # Callable so that we can use CodeTransformers as decorators. 209 | if closure is not None: 210 | closure = tuple(map(_cell_new, closure)) 211 | else: 212 | closure = f.__closure__ 213 | 214 | return FunctionType( 215 | self.transform(Code.from_pycode(f.__code__)).to_pycode(), 216 | _a_if_not_none(globals_, f.__globals__), 217 | _a_if_not_none(name, f.__name__), 218 | _a_if_not_none(defaults, f.__defaults__), 219 | closure, 220 | ) 221 | 222 | @instance 223 | class _context_stack(threading.local): 224 | """Thread safe transformation context stack. 225 | 226 | Each thread will get it's own ``WeakKeyDictionary`` that maps 227 | instances to a stack of ``Context`` objects. When this descriptor 228 | is looked up we first try to get the weakkeydict off of the thread 229 | local storage. If it doesn't exist we make a new map. Then we lookup 230 | our instance in this map. If it doesn't exist yet create a new stack 231 | (as an empty list). 232 | 233 | This allows a single instance of ``CodeTransformer`` to be used 234 | recursively to transform code objects in a thread safe way while 235 | still being able to use a stateful context. 236 | """ 237 | def __get__(self, instance, owner): 238 | try: 239 | stacks = self._context_stacks 240 | except AttributeError: 241 | stacks = self._context_stacks = WeakKeyDictionary() 242 | 243 | if instance is None: 244 | # when looked up off the class return the current threads 245 | # context stacks map 246 | return stacks 247 | 248 | return stacks.setdefault(instance, []) 249 | 250 | @contextmanager 251 | def _new_context(self, code): 252 | self._context_stack.append(Context(code)) 253 | try: 254 | yield 255 | finally: 256 | self._context_stack.pop() 257 | 258 | @property 259 | def context(self): 260 | """Lookup the current transformation context. 261 | 262 | Raises 263 | ------ 264 | NoContext 265 | Raised when there is no active transformation context. 266 | """ 267 | try: 268 | return self._context_stack[-1] 269 | except IndexError: 270 | raise NoContext() 271 | 272 | @property 273 | def code(self): 274 | """The code object we are currently manipulating. 275 | """ 276 | return self.context.code 277 | 278 | @property 279 | def startcode(self): 280 | """The startcode we are currently in. 281 | """ 282 | return self.context.startcode 283 | 284 | def begin(self, startcode): 285 | """Begin a new startcode. 286 | 287 | Parameters 288 | ---------- 289 | startcode : any 290 | The startcode to begin. 291 | """ 292 | self.context.startcode = startcode 293 | -------------------------------------------------------------------------------- /codetransformer/decompiler/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from ..code import Flag 4 | 5 | 6 | def paramnames(co): 7 | """ 8 | Get the parameter names from a pycode object. 9 | 10 | Returns a 4-tuple of (args, kwonlyargs, varargs, varkwargs). 11 | varargs and varkwargs will be None if the function doesn't take *args or 12 | **kwargs, respectively. 13 | """ 14 | flags = co.co_flags 15 | varnames = co.co_varnames 16 | 17 | argcount, kwonlyargcount = co.co_argcount, co.co_kwonlyargcount 18 | total = argcount + kwonlyargcount 19 | 20 | args = varnames[:argcount] 21 | kwonlyargs = varnames[argcount:total] 22 | varargs, varkwargs = None, None 23 | if flags & Flag.CO_VARARGS: 24 | varargs = varnames[total] 25 | total += 1 26 | if flags & Flag.CO_VARKEYWORDS: 27 | varkwargs = varnames[total] 28 | 29 | return args, kwonlyargs, varargs, varkwargs 30 | 31 | 32 | if sys.version_info[:3] == (3, 4, 3): 33 | from ._343 import * # noqa 34 | -------------------------------------------------------------------------------- /codetransformer/instructions.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta, abstractmethod 2 | from dis import opname, opmap, hasjabs, hasjrel, HAVE_ARGUMENT, stack_effect 3 | from enum import ( 4 | IntEnum, 5 | unique, 6 | ) 7 | from operator import attrgetter 8 | from re import escape 9 | 10 | from .patterns import matchable 11 | from .utils.immutable import immutableattr 12 | from .utils.no_default import no_default 13 | 14 | 15 | __all__ = ['Instruction'] + sorted(list(opmap)) 16 | 17 | # The instructions that use the co_names tuple. 18 | _uses_name = frozenset({ 19 | 'DELETE_ATTR', 20 | 'DELETE_GLOBAL', 21 | 'DELETE_NAME', 22 | 'IMPORT_FROM', 23 | 'IMPORT_NAME', 24 | 'LOAD_ATTR', 25 | 'LOAD_GLOBAL', 26 | 'LOAD_NAME', 27 | 'STORE_ATTR', 28 | 'STORE_GLOBAL', 29 | 'STORE_NAME', 30 | }) 31 | # The instructions that use the co_varnames tuple. 32 | _uses_varname = frozenset({ 33 | 'LOAD_FAST', 34 | 'STORE_FAST', 35 | 'DELETE_FAST', 36 | }) 37 | # The instructions that use the co_freevars tuple. 38 | _uses_free = frozenset({ 39 | 'DELETE_DEREF', 40 | 'LOAD_CLASSDEREF', 41 | 'LOAD_CLOSURE', 42 | 'LOAD_DEREF', 43 | 'STORE_DEREF', 44 | }) 45 | 46 | 47 | def _notimplemented(name): 48 | @property 49 | @abstractmethod 50 | def _(self): 51 | raise NotImplementedError(name) 52 | return _ 53 | 54 | 55 | @property 56 | def _vartype(self): 57 | try: 58 | return self._vartype 59 | except AttributeError: 60 | raise AttributeError( 61 | "vartype is not available on instructions " 62 | "constructed outside of a Code object." 63 | ) 64 | 65 | 66 | class InstructionMeta(ABCMeta, matchable): 67 | _marker = object() # sentinel 68 | _type_cache = {} 69 | 70 | def __init__(self, *args, opcode=None): 71 | return super().__init__(*args) 72 | 73 | def __new__(mcls, name, bases, dict_, *, opcode=None): 74 | try: 75 | return mcls._type_cache[opcode] 76 | except KeyError: 77 | pass 78 | 79 | if len(bases) != 1: 80 | raise TypeError( 81 | '{} does not support multiple inheritance'.format( 82 | mcls.__name__, 83 | ), 84 | ) 85 | 86 | if bases[0] is mcls._marker: 87 | dict_['_reprname'] = immutableattr(name) 88 | for attr in ('absjmp', 'have_arg', 'opcode', 'opname', 'reljmp'): 89 | dict_[attr] = _notimplemented(attr) 90 | return super().__new__(mcls, name, (object,), dict_) 91 | 92 | if opcode not in opmap.values(): 93 | raise TypeError('Invalid opcode: {}'.format(opcode)) 94 | 95 | opname_ = opname[opcode] 96 | dict_['opname'] = dict_['_reprname'] = immutableattr(opname_) 97 | dict_['opcode'] = immutableattr(opcode) 98 | 99 | absjmp = opcode in hasjabs 100 | reljmp = opcode in hasjrel 101 | dict_['absjmp'] = immutableattr(absjmp) 102 | dict_['reljmp'] = immutableattr(reljmp) 103 | dict_['is_jmp'] = immutableattr(absjmp or reljmp) 104 | 105 | dict_['uses_name'] = immutableattr(opname_ in _uses_name) 106 | dict_['uses_varname'] = immutableattr(opname_ in _uses_varname) 107 | dict_['uses_free'] = immutableattr(opname_ in _uses_free) 108 | if opname_ in _uses_free: 109 | dict_['vartype'] = _vartype 110 | 111 | dict_['have_arg'] = immutableattr(opcode >= HAVE_ARGUMENT) 112 | 113 | cls = mcls._type_cache[opcode] = super().__new__( 114 | mcls, opname[opcode], bases, dict_, 115 | ) 116 | return cls 117 | 118 | def mcompile(self): 119 | return escape(bytes((self.opcode,))) 120 | 121 | def __repr__(self): 122 | return self._reprname 123 | __str__ = __repr__ 124 | 125 | 126 | class Instruction(InstructionMeta._marker, metaclass=InstructionMeta): 127 | """ 128 | Base class for all instruction types. 129 | 130 | Parameters 131 | ---------- 132 | arg : any, optional 133 | 134 | The argument for the instruction. This should be the actual value of 135 | the argument, for example, if this is a 136 | :class:`~codetransformer.instructions.LOAD_CONST`, use the constant 137 | value, not the index that would appear in the bytecode. 138 | """ 139 | _no_arg = no_default 140 | 141 | def __init__(self, arg=_no_arg): 142 | if self.have_arg and arg is self._no_arg: 143 | raise TypeError( 144 | "{} missing 1 required argument: 'arg'".format(self.opname), 145 | ) 146 | self.arg = self._normalize_arg(arg) 147 | self._target_of = set() 148 | self._stolen_by = None # used for lnotab recalculation 149 | 150 | def __repr__(self): 151 | arg = self.arg 152 | return '{op}{arg}'.format( 153 | op=self.opname, 154 | arg='(%r)' % arg if self.arg is not self._no_arg else '', 155 | ) 156 | 157 | @staticmethod 158 | def _normalize_arg(arg): 159 | return arg 160 | 161 | def steal(self, instr): 162 | """Steal the jump index off of `instr`. 163 | 164 | This makes anything that would have jumped to `instr` jump to 165 | this Instruction instead. 166 | 167 | Parameters 168 | ---------- 169 | instr : Instruction 170 | The instruction to steal the jump sources from. 171 | 172 | Returns 173 | ------- 174 | self : Instruction 175 | The instruction that owns this method. 176 | 177 | Notes 178 | ----- 179 | This mutates self and ``instr`` inplace. 180 | """ 181 | instr._stolen_by = self 182 | for jmp in instr._target_of: 183 | jmp.arg = self 184 | self._target_of = instr._target_of 185 | instr._target_of = set() 186 | return self 187 | 188 | @classmethod 189 | def from_opcode(cls, opcode, arg=_no_arg): 190 | """ 191 | Create an instruction from an opcode and raw argument. 192 | 193 | Parameters 194 | ---------- 195 | opcode : int 196 | Opcode for the instruction to create. 197 | arg : int, optional 198 | The argument for the instruction. 199 | 200 | Returns 201 | ------- 202 | intsr : Instruction 203 | An instance of the instruction named by ``opcode``. 204 | """ 205 | return type(cls)(opname[opcode], (cls,), {}, opcode=opcode)(arg) 206 | 207 | @property 208 | def stack_effect(self): 209 | """ 210 | The net effect of executing this instruction on the interpreter stack. 211 | 212 | Instructions that pop values off the stack have negative stack effect 213 | equal to the number of popped values. 214 | 215 | Instructions that push values onto the stack have positive stack effect 216 | equal to the number of popped values. 217 | 218 | Examples 219 | -------- 220 | - LOAD_{FAST,NAME,GLOBAL,DEREF} push one value onto the stack. 221 | They have a stack_effect of 1. 222 | - POP_JUMP_IF_{TRUE,FALSE} always pop one value off the stack. 223 | They have a stack effect of -1. 224 | - BINARY_* instructions pop two instructions off the stack, apply a 225 | binary operator, and push the resulting value onto the stack. 226 | They have a stack effect of -1 (-2 values consumed + 1 value pushed). 227 | """ 228 | if self.opcode == NOP.opcode: # noqa 229 | # dis.stack_effect is broken here 230 | return 0 231 | 232 | return stack_effect( 233 | self.opcode, 234 | *((self.arg if isinstance(self.arg, int) else 0,) 235 | if self.have_arg else ()) 236 | ) 237 | 238 | def equiv(self, instr): 239 | """Check equivalence of instructions. This checks against the types 240 | and the arguments of the instructions 241 | 242 | Parameters 243 | ---------- 244 | instr : Instruction 245 | The instruction to check against. 246 | 247 | Returns 248 | ------- 249 | is_equiv : bool 250 | If the instructions are equivalent. 251 | 252 | Notes 253 | ----- 254 | This is a separate concept from instruction identity. Two separate 255 | instructions can be equivalent without being the same exact instance. 256 | This means that two equivalent instructions can be at different points 257 | in the bytecode or be targeted by different jumps. 258 | """ 259 | return type(self) == type(instr) and self.arg == instr.arg 260 | 261 | 262 | class _RawArg(int): 263 | """A class to hold arguments that are not yet initialized so that they 264 | don't break subclass's type checking code. 265 | 266 | This is used in the first pass of instruction creating in Code.from_pycode. 267 | """ 268 | 269 | 270 | def _mk_call_init(class_): 271 | """Create an __init__ function for a call type instruction. 272 | 273 | Parameters 274 | ---------- 275 | class_ : type 276 | The type to bind the function to. 277 | 278 | Returns 279 | ------- 280 | __init__ : callable 281 | The __init__ method for the class. 282 | """ 283 | def __init__(self, packed=no_default, *, positional=0, keyword=0): 284 | if packed is no_default: 285 | arg = int.from_bytes(bytes((positional, keyword)), 'little') 286 | elif not positional and not keyword: 287 | arg = packed 288 | else: 289 | raise TypeError('cannot specify packed and unpacked arguments') 290 | self.positional, self.keyword = arg.to_bytes(2, 'little') 291 | super(class_, self).__init__(arg) 292 | 293 | return __init__ 294 | 295 | 296 | def _call_repr(self): 297 | return '%s(positional=%d, keyword=%d)' % ( 298 | type(self).__name__, 299 | self.positional, 300 | self.keyword, 301 | ) 302 | 303 | 304 | def _check_jmp_arg(self, arg): 305 | if not isinstance(arg, (Instruction, _RawArg)): 306 | raise TypeError( 307 | 'argument to %s must be an instruction, got: %r' % ( 308 | type(self).__name__, arg, 309 | ), 310 | ) 311 | if isinstance(arg, Instruction): 312 | arg._target_of.add(self) 313 | return arg 314 | 315 | 316 | class CompareOpMeta(InstructionMeta): 317 | """ 318 | Special-case metaclass for the COMPARE_OP instruction type that provides 319 | default constructors for the various kinds of comparisons. 320 | 321 | These default constructors are implemented as descriptors so that we can 322 | write:: 323 | 324 | new_compare = COMPARE_OP.LT 325 | 326 | and have it be equivalent to:: 327 | 328 | new_compare = COMPARE_OP(COMPARE_OP.comparator.LT) 329 | """ 330 | 331 | @unique 332 | class comparator(IntEnum): 333 | LT = 0 334 | LE = 1 335 | EQ = 2 336 | NE = 3 337 | GT = 4 338 | GE = 5 339 | IN = 6 340 | NOT_IN = 7 341 | IS = 8 342 | IS_NOT = 9 343 | EXCEPTION_MATCH = 10 344 | 345 | def __repr__(self): 346 | return '' % ( 347 | self.__class__.__name__, self._name_, self._value_, 348 | ) 349 | 350 | class ComparatorDescr: 351 | """ 352 | A descriptor on the **metaclass** of COMPARE_OP that constructs new 353 | instances of COMPARE_OP on attribute access. 354 | 355 | Parameters 356 | ---------- 357 | op : comparator 358 | The element of the `comparator` enum that this descriptor will 359 | forward to the COMPARE_OP constructor. 360 | """ 361 | def __init__(self, op): 362 | self._op = op 363 | 364 | def __get__(self, instance, owner): 365 | # Since this descriptor is added to the current metaclass, 366 | # ``instance`` here is the COMPARE_OP **class**. 367 | 368 | if instance is None: 369 | # If someone does `CompareOpMeta.LT`, give them back the 370 | # descriptor object itself. 371 | return self 372 | 373 | # If someone does `COMPARE_OP.LT`, return a **new instance** of 374 | # COMPARE_OP. 375 | # We create new instances so that consumers can take ownership 376 | # without worrying about other jumps targeting the new instruction. 377 | return instance(self._op) 378 | 379 | # Dynamically add an instance of ComparatorDescr for each comparator 380 | # opcode. 381 | # This is equivalent to doing: 382 | # LT = ComparatorDescr(comparator.LT) 383 | # GT = ComparatorDescr(comparator.GT) 384 | # ... 385 | for c in comparator: 386 | locals()[c._name_] = ComparatorDescr(c) 387 | del c 388 | del ComparatorDescr 389 | 390 | 391 | metamap = { 392 | 'COMPARE_OP': CompareOpMeta, 393 | } 394 | 395 | 396 | globals_ = globals() 397 | for name, opcode in opmap.items(): 398 | globals_[name] = class_ = metamap.get(name, InstructionMeta)( 399 | opname[opcode], 400 | (Instruction,), { 401 | '__module__': __name__, 402 | '__qualname__': '.'.join((__name__, name)), 403 | }, 404 | opcode=opcode, 405 | ) 406 | if name.startswith('CALL_FUNCTION'): 407 | class_.__init__ = _mk_call_init(class_) 408 | class_.__repr__ = _call_repr 409 | 410 | if name == 'COMPARE_OP': 411 | class_._normalize_arg = staticmethod(class_.comparator) 412 | 413 | if class_.is_jmp: 414 | class_._normalize_arg = _check_jmp_arg 415 | 416 | class_.__doc__ = ( 417 | """ 418 | See Also 419 | -------- 420 | dis.{name} 421 | """.format(name=name), 422 | ) 423 | 424 | del class_ 425 | 426 | 427 | # Clean up the namespace 428 | del name 429 | del globals_ 430 | del metamap 431 | del _check_jmp_arg 432 | del _call_repr 433 | del _mk_call_init 434 | 435 | # The instructions that use the co_names tuple. 436 | uses_name = frozenset( 437 | filter(attrgetter('uses_name'), Instruction.__subclasses__()), 438 | ) 439 | # The instructions that use the co_varnames tuple. 440 | uses_varname = frozenset( 441 | filter(attrgetter('uses_varname'), Instruction.__subclasses__()), 442 | ) 443 | # The instructions that use the co_freevars tuple. 444 | uses_free = frozenset( 445 | filter(attrgetter('uses_free'), Instruction.__subclasses__()), 446 | ) 447 | -------------------------------------------------------------------------------- /codetransformer/patterns.py: -------------------------------------------------------------------------------- 1 | from operator import methodcaller, index, attrgetter 2 | import re 3 | from types import MethodType 4 | 5 | from .utils.instance import instance 6 | from .utils.immutable import immutable 7 | 8 | 9 | #: The default startcode for patterns. 10 | DEFAULT_STARTCODE = 0 11 | mcompile = methodcaller('mcompile') 12 | 13 | 14 | def _prepr(m): 15 | if isinstance(m, or_): 16 | return '(%r)' % m 17 | 18 | return repr(m) 19 | 20 | 21 | def coerce_ellipsis(p): 22 | """Convert ... into a matchany 23 | """ 24 | if p is ...: 25 | return matchany 26 | 27 | return p 28 | 29 | 30 | class matchable: 31 | """Mixin for defining the operators on patterns. 32 | """ 33 | def __or__(self, other): 34 | other = coerce_ellipsis(other) 35 | if self is other: 36 | return self 37 | 38 | if not isinstance(other, matchable): 39 | return NotImplemented 40 | 41 | patterns = [] 42 | if isinstance(self, or_): 43 | patterns.extend(self.matchables) 44 | else: 45 | patterns.append(self) 46 | if isinstance(other, or_): 47 | patterns.extend(other.matchables) 48 | else: 49 | patterns.append(other) 50 | 51 | return or_(*patterns) 52 | 53 | def __ror__(self, other): 54 | # Flip the order on the or method 55 | if not isinstance(other, matchable): 56 | return NotImplemented 57 | 58 | return type(self).__or__(coerce_ellipsis(other), self) 59 | 60 | def __invert__(self): 61 | return not_(self) 62 | 63 | def __getitem__(self, key): 64 | try: 65 | n = index(key) 66 | except TypeError: 67 | pass 68 | else: 69 | return matchrange(self, n) 70 | 71 | if isinstance(key, tuple) and len(key) in (1, 2): 72 | return matchrange(self, *key) 73 | 74 | if isinstance(key, modifier): 75 | return postfix_modifier(self, key) 76 | 77 | raise TypeError('invalid modifier: {0}'.format(key)) 78 | 79 | 80 | class postfix_modifier(immutable, matchable): 81 | """A pattern with a modifier paired with it. 82 | """ 83 | __slots__ = 'matchable', 'modifier' 84 | 85 | def mcompile(self): 86 | return self.matchable.mcompile() + self.modifier.mcompile() 87 | 88 | def __repr__(self): 89 | return '%r[%r]' % (self.matchable, self.modifier) 90 | __str__ = __repr__ 91 | 92 | 93 | class meta(matchable): 94 | """Class for meta patterns and pattern likes. for example: ``matchany``. 95 | """ 96 | def mcompile(self): 97 | return self._token 98 | 99 | def __repr__(self): 100 | return self._token.decode('utf-8') 101 | __str__ = __repr__ 102 | 103 | 104 | class modifier(meta): 105 | """Marker class for modifier types. 106 | """ 107 | pass 108 | 109 | 110 | @instance 111 | class var(modifier): 112 | """Modifier that matches zero or more of a pattern. 113 | """ 114 | _token = b'*' 115 | 116 | 117 | @instance 118 | class plus(modifier): 119 | """Modifier that matches one or more of a pattern. 120 | """ 121 | _token = b'+' 122 | 123 | 124 | @instance 125 | class option(modifier): 126 | """Modifier that matches zero or one of a pattern. 127 | """ 128 | _token = b'?' 129 | 130 | 131 | class matchrange(immutable, meta, defaults={'m': None}): 132 | __slots__ = 'matchable', 'n', 'm' 133 | 134 | def mcompile(self): 135 | m = self.m 136 | return ( 137 | self.matchable.mcompile() + 138 | b'{' + 139 | bytes(str(self.n), 'utf-8') + 140 | b',' + (b'' if m is None else (b', ' + bytes(str(m), 'utf-8'))) + 141 | b'}' 142 | ) 143 | 144 | def __repr__(self): 145 | return '{matchable}[{args}]'.format( 146 | matchable=_prepr(self.matchable), 147 | args=', '.join(map(str, filter(bool, (self.n, self.m)))), 148 | ) 149 | 150 | 151 | @instance 152 | class matchany(meta): 153 | """Matchable that matches any instruction. 154 | """ 155 | _token = b'.' 156 | 157 | def __repr__(self): 158 | return '...' 159 | 160 | 161 | class seq(immutable, matchable): 162 | """A sequence of matchables to match in order. 163 | 164 | Parameters 165 | ---------- 166 | \*matchables : iterable of matchable 167 | The matchables to match against. 168 | """ 169 | __slots__ = 'matchables', 170 | 171 | def __new__(cls, *matchables): 172 | if not matchables: 173 | raise TypeError('cannot create an empty sequence') 174 | 175 | if len(matchables) == 1: 176 | return coerce_ellipsis(matchables[0]) 177 | return super().__new__(cls) 178 | 179 | def __init__(self, *matchables): 180 | self.matchables = tuple(map(coerce_ellipsis, matchables)) 181 | 182 | def mcompile(self): 183 | return b''.join(map(mcompile, self.matchables)) 184 | 185 | def __repr__(self): 186 | return '{cls}({args})'.format( 187 | cls=type(self).__name__, 188 | args=', '.join(map(_prepr, self.matchables)) 189 | ) 190 | 191 | 192 | class or_(immutable, matchable): 193 | """Logical or of multiple matchables. 194 | 195 | Parameters 196 | ---------- 197 | *matchables : iterable of matchable 198 | The matchables to or together. 199 | """ 200 | __slots__ = '*matchables', 201 | 202 | def mcompile(self): 203 | return b'(' + b'|'.join(map(mcompile, self.matchables)) + b')' 204 | 205 | def __repr__(self): 206 | return ' | '.join(map(_prepr, self.matchables)) 207 | 208 | 209 | class not_(immutable, matchable): 210 | """Logical not of a matchable. 211 | """ 212 | __slots__ = 'matchable', 213 | 214 | def mcompile(self): 215 | matchable = self.matchable 216 | if isinstance(matchable, (seq, or_, not_)): 217 | return b'((?!(' + matchable.mcompile() + b')).)*' 218 | 219 | return b'[^' + matchable.mcompile() + b']' 220 | 221 | def __repr__(self): 222 | return '~' + _prepr(self.matchable) 223 | 224 | 225 | class pattern(immutable): 226 | """ 227 | A pattern of instructions that can be matched against. 228 | 229 | This class is intended to be used as a decorator on methods of 230 | CodeTransformer subclasses. It is used to mark that a given method should 231 | be called on sequences of instructions that match the pattern described by 232 | the inputs. 233 | 234 | Parameters 235 | ---------- 236 | \*matchables : iterable of matchable 237 | The type of instructions to match against. 238 | startcodes : container of any 239 | The startcodes where this pattern should be tried. 240 | 241 | Examples 242 | -------- 243 | Match a single BINARY_ADD instruction:: 244 | 245 | pattern(BINARY_ADD) 246 | 247 | Match a single BINARY_ADD followed by a RETURN_VALUE:: 248 | 249 | pattern(BINARY_ADD, RETURN_VALUE) 250 | 251 | Match a single BINARY_ADD followed by any other single instruction:: 252 | 253 | pattern(BINARY_ADD, matchany) 254 | 255 | Match a single BINARY_ADD followed by any number of instructions:: 256 | 257 | pattern(BINARY_ADD, matchany[var]) 258 | """ 259 | __slots__ = 'matchable', 'startcodes', '_compiled' 260 | 261 | def __init__(self, *matchables, startcodes=(DEFAULT_STARTCODE,)): 262 | if not matchables: 263 | raise TypeError('expected at least one matchable') 264 | self.matchable = matchable = seq(*matchables) 265 | self.startcodes = startcodes 266 | self._compiled = re.compile(matchable.mcompile()) 267 | 268 | def __call__(self, f): 269 | return boundpattern(self._compiled, self.startcodes, f) 270 | 271 | def __repr__(self): 272 | return '{cls}(matchable={m!r}, startcodes={s})'.format( 273 | cls=type(self).__name__, 274 | m=self.matchable, 275 | s=self.startcodes, 276 | ) 277 | 278 | 279 | class boundpattern(immutable): 280 | """A pattern bound to a function. 281 | """ 282 | __slots__ = '_compiled', '_startcodes', '_f' 283 | 284 | def __get__(self, instance, owner): 285 | if instance is None: 286 | return self 287 | 288 | return type(self)( 289 | self._compiled, 290 | self._startcodes, 291 | MethodType(self._f, instance) 292 | ) 293 | 294 | def __call__(self, compiled_instrs, instrs, startcode): 295 | if startcode not in self._startcodes: 296 | raise NoMatch(compiled_instrs, startcode) 297 | 298 | match = self._compiled.match(compiled_instrs) 299 | if match is None or match.end is 0: 300 | raise NoMatch(compiled_instrs, startcode) 301 | 302 | mend = match.end() 303 | return self._f(*instrs[:mend]), mend 304 | 305 | 306 | class NoMatch(Exception): 307 | """Indicates that there was no match found in this dispatcher. 308 | """ 309 | pass 310 | 311 | 312 | class patterndispatcher(immutable): 313 | """A set of patterns that can dispatch onto instrs. 314 | """ 315 | __slots__ = '*patterns', 316 | 317 | def __get__(self, instance, owner): 318 | if instance is None: 319 | return self 320 | 321 | return boundpatterndispatcher( 322 | instance, 323 | *map( 324 | methodcaller('__get__', instance, owner), 325 | self.patterns, 326 | ) 327 | ) 328 | 329 | 330 | class boundpatterndispatcher(immutable): 331 | """A set of patterns bound to a transformer. 332 | """ 333 | __slots__ = 'transformer', '*patterns' 334 | 335 | def _dispatch(self, compiled_instrs, instrs, startcode): 336 | for p in self.patterns: 337 | try: 338 | return p(compiled_instrs, instrs, startcode) 339 | except NoMatch: 340 | pass 341 | 342 | raise NoMatch(instrs, startcode) 343 | 344 | def __call__(self, instrs): 345 | opcodes = bytes(map(attrgetter('opcode'), instrs)) 346 | idx = 0 # The current index into the pre-transformed instrs. 347 | post_transform = [] # The instrs that have been transformed. 348 | transformer = self.transformer 349 | while idx < len(instrs): 350 | try: 351 | processed, nconsumed = self._dispatch( 352 | opcodes[idx:], 353 | instrs[idx:], 354 | # NOTE: do not remove this attribute access 355 | # self._dispatch can mutate the value of the startcode 356 | transformer.startcode, 357 | ) 358 | except NoMatch: 359 | post_transform.append(instrs[idx]) 360 | idx += 1 361 | else: 362 | post_transform.extend(processed) 363 | idx += nconsumed 364 | return tuple(post_transform) 365 | -------------------------------------------------------------------------------- /codetransformer/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llllllllll/codetransformer/c5f551e915df45adc7da7e0b1b635f0cc6a1bb27/codetransformer/tests/__init__.py -------------------------------------------------------------------------------- /codetransformer/tests/test_code.py: -------------------------------------------------------------------------------- 1 | from dis import dis 2 | from io import StringIO 3 | from itertools import product, chain 4 | import random 5 | import sys 6 | 7 | import pytest 8 | 9 | from codetransformer.code import Code, Flag, pycode 10 | from codetransformer.instructions import LOAD_CONST, LOAD_FAST, uses_free 11 | 12 | 13 | @pytest.fixture(scope='module') 14 | def sample_flags(request): 15 | random.seed(8025816322119661921) # ayy lmao 16 | nflags = len(Flag.__members__) 17 | return tuple( 18 | dict(zip(Flag.__members__.keys(), case)) for case in chain( 19 | random.sample(list(product((True, False), repeat=nflags)), 1000), 20 | [[True] * nflags], 21 | [[False] * nflags], 22 | ) 23 | ) 24 | 25 | 26 | def test_lnotab_roundtrip(): 27 | # DO NOT ADD EXTRA LINES HERE 28 | def f(): # pragma: no cover 29 | a = 1 30 | b = 2 31 | c = 3 32 | d = 4 33 | a, b, c, d 34 | 35 | start_line = test_lnotab_roundtrip.__code__.co_firstlineno + 3 36 | lines = [start_line + n for n in range(5)] 37 | code = Code.from_pycode(f.__code__) 38 | lnotab = code.lnotab 39 | assert lnotab.keys() == set(lines) 40 | assert isinstance(lnotab[lines[0]], LOAD_CONST) 41 | assert lnotab[lines[0]].arg == 1 42 | assert isinstance(lnotab[lines[1]], LOAD_CONST) 43 | assert lnotab[lines[1]].arg == 2 44 | assert isinstance(lnotab[lines[2]], LOAD_CONST) 45 | assert lnotab[lines[2]].arg == 3 46 | assert isinstance(lnotab[lines[3]], LOAD_CONST) 47 | assert lnotab[lines[3]].arg == 4 48 | assert isinstance(lnotab[lines[4]], LOAD_FAST) 49 | assert lnotab[lines[4]].arg == 'a' 50 | assert f.__code__.co_lnotab == code.py_lnotab == code.to_pycode().co_lnotab 51 | 52 | 53 | def test_lnotab_really_dumb_whitespace(): 54 | ns = {} 55 | exec('def f():\n lol = True' + '\n' * 1024 + ' wut = True', ns) 56 | f = ns['f'] 57 | code = Code.from_pycode(f.__code__) 58 | lines = [2, 1026] 59 | lnotab = code.lnotab 60 | assert lnotab.keys() == set(lines) 61 | assert isinstance(lnotab[lines[0]], LOAD_CONST) 62 | assert lnotab[lines[0]].arg 63 | assert isinstance(lnotab[lines[1]], LOAD_CONST) 64 | assert lnotab[lines[1]].arg 65 | assert f.__code__.co_lnotab == code.py_lnotab == code.to_pycode().co_lnotab 66 | 67 | 68 | def test_flag_packing(sample_flags): 69 | for flags in sample_flags: 70 | assert Flag.unpack(Flag.pack(**flags)) == flags 71 | 72 | 73 | def test_flag_unpack_too_big(): 74 | assert all(Flag.unpack(Flag.max).values()) 75 | with pytest.raises(ValueError): 76 | Flag.unpack(Flag.max + 1) 77 | 78 | 79 | def test_flag_max(): 80 | assert Flag.pack( 81 | CO_OPTIMIZED=True, 82 | CO_NEWLOCALS=True, 83 | CO_VARARGS=True, 84 | CO_VARKEYWORDS=True, 85 | CO_NESTED=True, 86 | CO_GENERATOR=True, 87 | CO_NOFREE=True, 88 | CO_COROUTINE=True, 89 | CO_ITERABLE_COROUTINE=True, 90 | CO_FUTURE_DIVISION=True, 91 | CO_FUTURE_ABSOLUTE_IMPORT=True, 92 | CO_FUTURE_WITH_STATEMENT=True, 93 | CO_FUTURE_PRINT_FUNCTION=True, 94 | CO_FUTURE_UNICODE_LITERALS=True, 95 | CO_FUTURE_BARRY_AS_BDFL=True, 96 | CO_FUTURE_GENERATOR_STOP=True, 97 | ) == Flag.max 98 | 99 | 100 | def test_flag_max_immutable(): 101 | with pytest.raises(AttributeError): 102 | Flag.CO_OPTIMIZED.max = None 103 | 104 | 105 | def test_code_multiple_varargs(): 106 | with pytest.raises(ValueError) as e: 107 | Code( 108 | (), ( 109 | '*args', 110 | '*other', 111 | ), 112 | ) 113 | 114 | assert str(e.value) == 'cannot specify *args more than once' 115 | 116 | 117 | def test_code_multiple_kwargs(): 118 | with pytest.raises(ValueError) as e: 119 | Code( 120 | (), ( 121 | '**kwargs', 122 | '**kwargs', 123 | ), 124 | ) 125 | 126 | assert str(e.value) == 'cannot specify **kwargs more than once' 127 | 128 | 129 | @pytest.mark.parametrize('cls', uses_free) 130 | def test_dangling_var(cls): 131 | instr = cls('dangling') 132 | with pytest.raises(ValueError) as e: 133 | Code((instr,)) 134 | 135 | assert ( 136 | str(e.value) == 137 | "Argument to %r is not in cellvars or freevars." % instr 138 | ) 139 | 140 | 141 | def test_code_flags(sample_flags): 142 | attr_map = { 143 | 'CO_NESTED': 'is_nested', 144 | 'CO_GENERATOR': 'is_generator', 145 | 'CO_COROUTINE': 'is_coroutine', 146 | 'CO_ITERABLE_COROUTINE': 'is_iterable_coroutine', 147 | 'CO_NEWLOCALS': 'constructs_new_locals', 148 | } 149 | for flags in sample_flags: 150 | if sys.version_info < (3, 6): 151 | codestring = b'd\x00\x00S' # return None 152 | else: 153 | codestring = b'd\x00S' # return None 154 | 155 | code = Code.from_pycode(pycode( 156 | argcount=0, 157 | kwonlyargcount=0, 158 | nlocals=2, 159 | stacksize=0, 160 | flags=Flag.pack(**flags), 161 | codestring=codestring, 162 | constants=(None,), 163 | names=(), 164 | varnames=('a', 'b'), 165 | filename='', 166 | name='', 167 | firstlineno=0, 168 | lnotab=b'', 169 | )) 170 | assert code.flags == flags 171 | for flag, attr in attr_map.items(): 172 | if flags[flag]: 173 | assert getattr(code, attr) 174 | 175 | 176 | @pytest.fixture 177 | def abc_code(): 178 | a = LOAD_CONST('a') 179 | b = LOAD_CONST('b') 180 | c = LOAD_CONST('c') # not in instrs 181 | code = Code((a, b), argnames=()) 182 | 183 | return (a, b, c), code 184 | 185 | 186 | def test_instr_index(abc_code): 187 | (a, b, c), code = abc_code 188 | 189 | assert code.index(a) == 0 190 | assert code.index(b) == 1 191 | 192 | with pytest.raises(ValueError): 193 | code.index(c) 194 | 195 | 196 | def test_code_contains(abc_code): 197 | (a, b, c), code = abc_code 198 | 199 | assert a in code 200 | assert b in code 201 | assert c not in code 202 | 203 | 204 | def test_code_dis(capsys): 205 | @Code.from_pyfunc 206 | def code(): # pragma: no cover 207 | a = 1 208 | b = 2 209 | return a, b 210 | 211 | buf = StringIO() 212 | dis(code.to_pycode(), file=buf) 213 | expected = buf.getvalue() 214 | 215 | code.dis() 216 | out, err = capsys.readouterr() 217 | assert not err 218 | assert out == expected 219 | 220 | buf = StringIO() 221 | code.dis(file=buf) 222 | assert buf.getvalue() == expected 223 | -------------------------------------------------------------------------------- /codetransformer/tests/test_core.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import toolz.curried.operator as op 3 | 4 | from codetransformer import CodeTransformer, Code, pattern 5 | from codetransformer.core import Context, NoContext 6 | from codetransformer.instructions import Instruction 7 | from codetransformer.patterns import DEFAULT_STARTCODE 8 | from codetransformer.utils.instance import instance 9 | 10 | 11 | def test_inherit_patterns(): 12 | class C(CodeTransformer): 13 | matched = False 14 | 15 | @pattern(...) 16 | def _(self, instr): 17 | self.matched = True 18 | yield instr 19 | 20 | class D(C): 21 | pass 22 | 23 | d = D() 24 | assert not d.matched 25 | 26 | @d 27 | def f(): 28 | pass 29 | 30 | assert d.matched 31 | 32 | 33 | def test_override_patterns(): 34 | class C(CodeTransformer): 35 | matched_super = False 36 | matched_sub = False 37 | 38 | @pattern(...) 39 | def _(self, instr): 40 | self.matched_super = True 41 | yield instr 42 | 43 | class D(C): 44 | @pattern(...) 45 | def _(self, instr): 46 | self.matched_sub = True 47 | yield instr 48 | 49 | d = D() 50 | assert not d.matched_super 51 | assert not d.matched_sub 52 | 53 | @d 54 | def f(): 55 | pass 56 | 57 | assert d.matched_sub 58 | assert not d.matched_super 59 | 60 | 61 | def test_updates_lnotab(): 62 | @instance 63 | class c(CodeTransformer): 64 | @pattern(...) 65 | def _(self, instr): 66 | yield type(instr)(instr.arg).steal(instr) 67 | 68 | def f(): # pragma: no cover 69 | # this function has irregular whitespace for testing the lnotab 70 | a = 1 71 | # intentional line 72 | b = 2 73 | # intentional line 74 | c = 3 75 | # intentional line 76 | return a, b, c 77 | 78 | original = Code.from_pyfunc(f) 79 | post_transform = c.transform(original) 80 | 81 | # check that something happened 82 | assert original.lnotab != post_transform.lnotab 83 | # check that we preserved the line numbers 84 | assert ( 85 | original.lnotab.keys() == 86 | post_transform.lnotab.keys() == 87 | set(map(op.add(original.firstlineno), (2, 4, 6, 8))) 88 | ) 89 | 90 | def sorted_instrs(lnotab): 91 | order = sorted(lnotab.keys()) 92 | for idx in order: 93 | yield lnotab[idx] 94 | 95 | # check that the instrs are correct 96 | assert all(map( 97 | Instruction.equiv, 98 | sorted_instrs(original.lnotab), 99 | sorted_instrs(post_transform.lnotab), 100 | )) 101 | 102 | # sanity check that the function is correct 103 | assert f() == c(f)() 104 | 105 | 106 | def test_context(): 107 | def f(): # pragma: no cover 108 | pass 109 | 110 | code = Code.from_pyfunc(f) 111 | c = Context(code) 112 | 113 | # check default attributes 114 | assert c.code is code 115 | assert c.startcode == DEFAULT_STARTCODE 116 | 117 | # check that the object acts like a namespace 118 | c.attr = 'test' 119 | assert c.attr == 'test' 120 | 121 | 122 | def test_no_context(): 123 | @instance 124 | class c(CodeTransformer): 125 | pass 126 | 127 | with pytest.raises(NoContext) as e: 128 | c.context 129 | 130 | assert str(e.value) == 'no active transformation context' 131 | -------------------------------------------------------------------------------- /codetransformer/tests/test_instructions.py: -------------------------------------------------------------------------------- 1 | from codetransformer.instructions import Instruction 2 | 3 | 4 | def test_repr_types(): 5 | assert repr(Instruction) == 'Instruction' 6 | for tp in Instruction.__subclasses__(): 7 | assert repr(tp) == tp.opname 8 | -------------------------------------------------------------------------------- /codetransformer/transformers/__init__.py: -------------------------------------------------------------------------------- 1 | from .constants import asconstants 2 | from .interpolated_strings import interpolated_strings 3 | from .pattern_matched_exceptions import pattern_matched_exceptions 4 | from .precomputed_slices import precomputed_slices 5 | from .literals import ( 6 | bytearray_literals, 7 | decimal_literals, 8 | haskell_strs, 9 | islice_literals, 10 | overloaded_complexes, 11 | overloaded_floats, 12 | overloaded_ints, 13 | overloaded_lists, 14 | overloaded_sets, 15 | overloaded_slices, 16 | overloaded_strs, 17 | overloaded_tuples, 18 | ) 19 | 20 | 21 | __all__ = [ 22 | 'asconstants', 23 | 'bytearray_literals', 24 | 'decimal_literals', 25 | 'haskell_strs', 26 | 'interpolated_strings', 27 | 'islice_literals', 28 | 'overloaded_complexes', 29 | 'overloaded_floats', 30 | 'overloaded_ints', 31 | 'overloaded_lists', 32 | 'overloaded_sets', 33 | 'overloaded_slices', 34 | 'overloaded_strs', 35 | 'overloaded_tuples', 36 | 'pattern_matched_exceptions', 37 | 'precomputed_slices', 38 | ] 39 | -------------------------------------------------------------------------------- /codetransformer/transformers/add2mul.py: -------------------------------------------------------------------------------- 1 | """ 2 | add2mul 3 | -------- 4 | 5 | A transformer that replaces BINARY_ADD instructions with BINARY_MULTIPLY 6 | instructions. 7 | 8 | This isn't useful, but it's good introductory example/tutorial material. 9 | """ 10 | from codetransformer import CodeTransformer, pattern 11 | from codetransformer.instructions import BINARY_ADD, BINARY_MULTIPLY 12 | 13 | 14 | class add2mul(CodeTransformer): 15 | @pattern(BINARY_ADD) 16 | def _add2mul(self, add_instr): 17 | yield BINARY_MULTIPLY().steal(add_instr) 18 | -------------------------------------------------------------------------------- /codetransformer/transformers/constants.py: -------------------------------------------------------------------------------- 1 | import builtins 2 | 3 | from ..core import CodeTransformer 4 | from ..instructions import ( 5 | DELETE_DEREF, 6 | DELETE_FAST, 7 | DELETE_GLOBAL, 8 | DELETE_NAME, 9 | LOAD_CLASSDEREF, 10 | LOAD_CONST, 11 | LOAD_DEREF, 12 | LOAD_GLOBAL, 13 | LOAD_NAME, 14 | STORE_DEREF, 15 | STORE_FAST, 16 | STORE_GLOBAL, 17 | STORE_NAME, 18 | ) 19 | from ..patterns import pattern 20 | 21 | 22 | def _assign_or_del(type_): 23 | assert type_ in ('assign to', 'delete') 24 | 25 | def handler(self, instr): 26 | name = instr.arg 27 | if name not in self._constnames: 28 | yield instr 29 | return 30 | 31 | code = self.code 32 | filename = code.filename 33 | lno = code.lno_of_instr[instr] 34 | try: 35 | with open(filename) as f: 36 | line = f.readlines()[lno - 1] 37 | except IOError: 38 | line = '???' 39 | 40 | raise SyntaxError( 41 | "can't %s constant name %r" % (type_, name), 42 | (filename, lno, len(line), line), 43 | ) 44 | 45 | return handler 46 | 47 | 48 | class asconstants(CodeTransformer): 49 | """ 50 | A code transformer that inlines names as constants. 51 | 52 | - Positional arguments are interpreted as names of builtins (e.g. ``len``, 53 | ``print``) to freeze as constants in the decorated function's namespace. 54 | 55 | - Keyword arguments provide additional custom names to freeze as constants. 56 | 57 | - If invoked with no positional or keyword arguments, ``asconstants`` 58 | inlines all names in ``builtins``. 59 | 60 | Parameters 61 | ---------- 62 | \*builtin_names 63 | Names of builtins to freeze as constants. 64 | \*\*kwargs 65 | Additional key-value pairs to bind as constants. 66 | 67 | Examples 68 | -------- 69 | Freezing Builtins: 70 | 71 | >>> from codetransformer.transformers import asconstants 72 | >>> 73 | >>> @asconstants('len') 74 | ... def with_asconstants(x): 75 | ... return len(x) * 2 76 | ... 77 | >>> def without_asconstants(x): 78 | ... return len(x) * 2 79 | ... 80 | >>> len = lambda x: 0 81 | >>> with_asconstants([1, 2, 3]) 82 | 6 83 | >>> without_asconstants([1, 2, 3]) 84 | 0 85 | 86 | Adding Custom Constants: 87 | 88 | >>> @asconstants(a=1) 89 | ... def f(): 90 | ... return a 91 | ... 92 | >>> f() 93 | 1 94 | >>> a = 5 95 | >>> f() 96 | 1 97 | """ 98 | def __init__(self, *builtin_names, **kwargs): 99 | super().__init__() 100 | bltins = vars(builtins) 101 | if not (builtin_names or kwargs): 102 | self._constnames = bltins.copy() 103 | else: 104 | self._constnames = constnames = {} 105 | for arg in builtin_names: 106 | constnames[arg] = bltins[arg] 107 | overlap = constnames.keys() & kwargs.keys() 108 | if overlap: 109 | raise TypeError('Duplicate keys: {!r}'.format(overlap)) 110 | constnames.update(kwargs) 111 | 112 | def transform(self, code, **kwargs): 113 | overlap = self._constnames.keys() & set(code.argnames) 114 | if overlap: 115 | raise SyntaxError( 116 | 'argument names overlap with constant names: %r' % overlap, 117 | ) 118 | return super().transform(code, **kwargs) 119 | 120 | @pattern(LOAD_NAME | LOAD_GLOBAL | LOAD_DEREF | LOAD_CLASSDEREF) 121 | def _load_name(self, instr): 122 | name = instr.arg 123 | if name not in self._constnames: 124 | yield instr 125 | return 126 | 127 | yield LOAD_CONST(self._constnames[name]).steal(instr) 128 | 129 | _store = pattern( 130 | STORE_NAME | STORE_GLOBAL | STORE_DEREF | STORE_FAST, 131 | )(_assign_or_del('assign to')) 132 | _delete = pattern( 133 | DELETE_NAME | DELETE_GLOBAL | DELETE_DEREF | DELETE_FAST, 134 | )(_assign_or_del('delete')) 135 | -------------------------------------------------------------------------------- /codetransformer/transformers/interpolated_strings.py: -------------------------------------------------------------------------------- 1 | """ 2 | A transformer implementing ruby-style interpolated strings. 3 | """ 4 | import sys 5 | 6 | from codetransformer import pattern, CodeTransformer 7 | from codetransformer.instructions import ( 8 | BUILD_TUPLE, 9 | LOAD_CONST, 10 | LOAD_ATTR, 11 | CALL_FUNCTION, 12 | CALL_FUNCTION_KW, 13 | ROT_TWO, 14 | ) 15 | from codetransformer.utils.functional import flatten, is_a 16 | 17 | 18 | class interpolated_strings(CodeTransformer): 19 | """ 20 | A transformer that interpolates local variables into string literals. 21 | 22 | Parameters 23 | ---------- 24 | transform_bytes : bool, optional 25 | Whether to transform bytes literals to interpolated unicode strings. 26 | Default is True. 27 | transform_str : bool, optional 28 | Whether to interpolate values into unicode strings. 29 | Default is False. 30 | 31 | Example 32 | ------- 33 | >>> @interpolated_strings() # doctest: +SKIP 34 | ... def foo(a, b): 35 | ... c = a + b 36 | ... return b"{a} + {b} = {c}" 37 | ... 38 | >>> foo(1, 2) # doctest: +SKIP 39 | '1 + 2 = 3' 40 | """ 41 | 42 | if sys.version_info >= (3, 6): 43 | def __init__(self, *, transform_bytes=True, transform_str=False): 44 | raise NotImplementedError( 45 | '%s is not supported on 3.6 or newer, just use f-strings' % 46 | type(self).__name__, 47 | ) 48 | else: 49 | def __init__(self, *, transform_bytes=True, transform_str=False): 50 | super().__init__() 51 | self._transform_bytes = transform_bytes 52 | self._transform_str = transform_str 53 | 54 | @property 55 | def types(self): 56 | """ 57 | Tuple containing types transformed by this transformer. 58 | """ 59 | out = [] 60 | if self._transform_bytes: 61 | out.append(bytes) 62 | if self._transform_str: 63 | out.append(str) 64 | return tuple(out) 65 | 66 | @pattern(LOAD_CONST) 67 | def _load_const(self, instr): 68 | const = instr.arg 69 | 70 | if isinstance(const, (tuple, frozenset)): 71 | yield from self._transform_constant_sequence(const) 72 | return 73 | 74 | if isinstance(const, bytes) and self._transform_bytes: 75 | yield from self.transform_stringlike(const) 76 | elif isinstance(const, str) and self._transform_str: 77 | yield from self.transform_stringlike(const) 78 | else: 79 | yield instr 80 | 81 | def _transform_constant_sequence(self, seq): 82 | """ 83 | Transform a frozenset or tuple. 84 | """ 85 | should_transform = is_a(self.types) 86 | 87 | if not any(filter(should_transform, flatten(seq))): 88 | # Tuple doesn't contain any transformable strings. Ignore. 89 | yield LOAD_CONST(seq) 90 | return 91 | 92 | for const in seq: 93 | if should_transform(const): 94 | yield from self.transform_stringlike(const) 95 | elif isinstance(const, (tuple, frozenset)): 96 | yield from self._transform_constant_sequence(const) 97 | else: 98 | yield LOAD_CONST(const) 99 | 100 | if isinstance(seq, tuple): 101 | yield BUILD_TUPLE(len(seq)) 102 | else: 103 | assert isinstance(seq, frozenset) 104 | yield BUILD_TUPLE(len(seq)) 105 | yield LOAD_CONST(frozenset) 106 | yield ROT_TWO() 107 | yield CALL_FUNCTION(1) 108 | 109 | def transform_stringlike(self, const): 110 | """ 111 | Yield instructions to process a str or bytes constant. 112 | """ 113 | yield LOAD_CONST(const) 114 | if isinstance(const, bytes): 115 | yield from self.bytes_instrs 116 | elif isinstance(const, str): 117 | yield from self.str_instrs 118 | 119 | @property 120 | def bytes_instrs(self): 121 | """ 122 | Yield instructions to call TOS.decode('utf-8').format(**locals()). 123 | """ 124 | yield LOAD_ATTR('decode') 125 | yield LOAD_CONST('utf-8') 126 | yield CALL_FUNCTION(1) 127 | yield from self.str_instrs 128 | 129 | @property 130 | def str_instrs(self): 131 | """ 132 | Yield instructions to call TOS.format(**locals()). 133 | """ 134 | yield LOAD_ATTR('format') 135 | yield LOAD_CONST(locals) 136 | yield CALL_FUNCTION(0) 137 | yield CALL_FUNCTION_KW() 138 | -------------------------------------------------------------------------------- /codetransformer/transformers/literals.py: -------------------------------------------------------------------------------- 1 | from collections import OrderedDict 2 | from decimal import Decimal 3 | from itertools import islice 4 | import sys 5 | from textwrap import dedent 6 | 7 | from .. import instructions 8 | from ..core import CodeTransformer 9 | from ..patterns import pattern, matchany, var 10 | from ..utils.instance import instance 11 | 12 | 13 | IN_COMPREHENSION = 'in_comprehension' 14 | 15 | 16 | class overloaded_dicts(CodeTransformer): 17 | """Transformer that allows us to overload dictionary literals. 18 | 19 | This acts by creating an empty map and then inserting every 20 | key value pair in order. 21 | 22 | The code that is generated will turn something like:: 23 | 24 | {k_0: v_0, k_1: v_1, ..., k_n: v_n} 25 | 26 | into:: 27 | 28 | _tmp = astype() 29 | _tmp[k_0] = v_0 30 | _tmp[k_1] = v_1 31 | ... 32 | _tmp[k_n] = v_n 33 | _tmp # leaves the map on the stack. 34 | 35 | Parameters 36 | ---------- 37 | astype : callable 38 | The constructor for the type to create. 39 | 40 | Examples 41 | -------- 42 | >>> from collections import OrderedDict 43 | >>> ordereddict_literals = overloaded_dicts(OrderedDict) 44 | >>> @ordereddict_literals 45 | ... def f(): 46 | ... return {'a': 1, 'b': 2, 'c': 3} 47 | ... 48 | >>> f() 49 | OrderedDict([('a', 1), ('b', 2), ('c', 3)]) 50 | """ 51 | def __init__(self, astype): 52 | super().__init__() 53 | self.astype = astype 54 | 55 | @pattern(instructions.BUILD_MAP, matchany[var], instructions.MAP_ADD) 56 | def _start_comprehension(self, instr, *instrs): 57 | yield instructions.LOAD_CONST(self.astype).steal(instr) 58 | # TOS = self.astype 59 | 60 | yield instructions.CALL_FUNCTION(0) 61 | # TOS = m = self.astype() 62 | 63 | yield instructions.STORE_FAST('__map__') 64 | 65 | *body, map_add = instrs 66 | yield from self.patterndispatcher(body) 67 | # TOS = k 68 | # TOS1 = v 69 | 70 | yield instructions.LOAD_FAST('__map__').steal(map_add) 71 | # TOS = __map__ 72 | # TOS1 = k 73 | # TOS2 = v 74 | 75 | yield instructions.ROT_TWO() 76 | # TOS = k 77 | # TOS1 = __map__ 78 | # TOS2 = v 79 | 80 | yield instructions.STORE_SUBSCR() 81 | self.begin(IN_COMPREHENSION) 82 | 83 | @pattern(instructions.RETURN_VALUE, startcodes=(IN_COMPREHENSION,)) 84 | def _return_value(self, instr): 85 | yield instructions.LOAD_FAST('__map__').steal(instr) 86 | # TOS = __map__ 87 | 88 | yield instr 89 | 90 | if sys.version_info[:2] <= (3, 4): 91 | # Python 3.4 92 | 93 | @pattern(instructions.BUILD_MAP) 94 | def _build_map(self, instr): 95 | yield instructions.LOAD_CONST(self.astype).steal(instr) 96 | # TOS = self.astype 97 | 98 | yield instructions.CALL_FUNCTION(0) 99 | # TOS = m = self.astype() 100 | 101 | yield from (instructions.DUP_TOP(),) * instr.arg 102 | # TOS = m 103 | # ... 104 | # TOS[instr.arg] = m 105 | 106 | @pattern(instructions.STORE_MAP) 107 | def _store_map(self, instr): 108 | # TOS = k 109 | # TOS1 = v 110 | # TOS2 = m 111 | # TOS3 = m 112 | 113 | yield instructions.ROT_THREE().steal(instr) 114 | # TOS = v 115 | # TOS1 = m 116 | # TOS2 = k 117 | # TOS3 = m 118 | 119 | yield instructions.ROT_THREE() 120 | # TOS = m 121 | # TOS1 = k 122 | # TOS2 = v 123 | # TOS3 = m 124 | 125 | yield instructions.ROT_TWO() 126 | # TOS = k 127 | # TOS1 = m 128 | # TOS2 = v 129 | # TOS3 = m 130 | 131 | yield instructions.STORE_SUBSCR() 132 | # TOS = m 133 | 134 | else: 135 | # Python 3.5 and beyond! 136 | 137 | def _construct_map(self, key_value_pairs): 138 | mapping = self.astype() 139 | for key, value in zip(key_value_pairs[::2], key_value_pairs[1::2]): 140 | mapping[key] = value 141 | return mapping 142 | 143 | @pattern(instructions.BUILD_MAP) 144 | def _build_map(self, instr): 145 | # TOS = vn 146 | # TOS1 = kn 147 | # ... 148 | # TOSN = v0 149 | # TOSN + 1 = k0 150 | # Construct a tuple of (k0, v0, k1, v1, ..., kn, vn) for 151 | # each of the key: value pairs in the dictionary. 152 | yield instructions.BUILD_TUPLE(instr.arg * 2).steal(instr) 153 | # TOS = (k0, v0, k1, v1, ..., kn, vn) 154 | 155 | yield instructions.LOAD_CONST(self._construct_map) 156 | # TOS = self._construct_map 157 | # TOS1 = (k0, v0, k1, v1, ..., kn, vn) 158 | 159 | yield instructions.ROT_TWO() 160 | # TOS = (k0, v0, k1, v1, ..., kn, vn) 161 | # TOS1 = self._construct_map 162 | 163 | yield instructions.CALL_FUNCTION(1) 164 | 165 | if sys.version_info >= (3, 6): 166 | def _construct_const_map(self, values, keys): 167 | mapping = self.astype() 168 | for key, value in zip(keys, values): 169 | mapping[key] = value 170 | return mapping 171 | 172 | @pattern(instructions.LOAD_CONST, instructions.BUILD_CONST_KEY_MAP) 173 | def _build_const_map(self, keys, instr): 174 | yield instructions.BUILD_TUPLE(len(keys.arg)).steal(keys) 175 | # TOS = (v0, v1, ..., vn) 176 | 177 | yield keys 178 | # TOS = (k0, k1, ..., kn) 179 | # TOS1 = (v0, v1, ..., vn) 180 | 181 | yield instructions.LOAD_CONST(self._construct_const_map) 182 | # TOS = self._construct_const_map 183 | # TOS1 = (k0, k1, ..., kn) 184 | # TOS2 = (v0, v1, ..., vn) 185 | 186 | yield instructions.ROT_THREE() 187 | # TOS = (k0, k1, ..., kn) 188 | # TOS1 = (v0, v1, ..., vn) 189 | # TOS2 = self._construct_const_map 190 | 191 | yield instructions.CALL_FUNCTION(2) 192 | 193 | 194 | ordereddict_literals = overloaded_dicts(OrderedDict) 195 | 196 | 197 | def _format_constant_docstring(type_): 198 | return dedent( 199 | """ 200 | Transformer that applies a callable to each {type_} constant in the 201 | transformed code object. 202 | 203 | Parameters 204 | ---------- 205 | xform : callable 206 | A callable to be applied to {type_} literals. 207 | 208 | See Also 209 | -------- 210 | codetransformer.transformers.literals.overloaded_strs 211 | """ 212 | ).format(type_=type_.__name__) 213 | 214 | 215 | class _ConstantTransformerBase(CodeTransformer): 216 | 217 | def __init__(self, xform): 218 | super().__init__() 219 | self.xform = xform 220 | 221 | def transform_consts(self, consts): 222 | # This is all one expression. 223 | return super().transform_consts( 224 | tuple( 225 | frozenset(self.transform_consts(tuple(const))) 226 | if isinstance(const, frozenset) 227 | else self.transform_consts(const) 228 | if isinstance(const, tuple) 229 | else self.xform(const) 230 | if isinstance(const, self._type) 231 | else const 232 | for const in consts 233 | ) 234 | ) 235 | 236 | 237 | def overloaded_constants(type_, __doc__=None): 238 | """A factory for transformers that apply functions to literals. 239 | 240 | Parameters 241 | ---------- 242 | type_ : type 243 | The type to overload. 244 | __doc__ : str, optional 245 | Docstring for the generated transformer. 246 | 247 | Returns 248 | ------- 249 | transformer : subclass of CodeTransformer 250 | A new code transformer class that will overload the provided 251 | literal types. 252 | """ 253 | typename = type_.__name__ 254 | if typename.endswith('x'): 255 | typename += 'es' 256 | elif not typename.endswith('s'): 257 | typename += 's' 258 | 259 | if __doc__ is None: 260 | __doc__ = _format_constant_docstring(type_) 261 | 262 | return type( 263 | "overloaded_" + typename, 264 | (_ConstantTransformerBase,), { 265 | '_type': type_, 266 | '__doc__': __doc__, 267 | }, 268 | ) 269 | 270 | 271 | overloaded_strs = overloaded_constants( 272 | str, 273 | __doc__=dedent( 274 | """ 275 | A transformer that overloads string literals. 276 | 277 | Rewrites all constants of the form:: 278 | 279 | "some string" 280 | 281 | as:: 282 | 283 | xform("some string") 284 | 285 | Parameters 286 | ---------- 287 | xform : callable 288 | Function to call on all string literals in the transformer target. 289 | 290 | Examples 291 | -------- 292 | >>> @overloaded_strs(lambda x: "ayy lmao ") 293 | ... def prepend_foo(s): 294 | ... return "foo" + s 295 | ... 296 | >>> prepend_foo("bar") 297 | 'ayy lmao bar' 298 | """ 299 | ) 300 | ) 301 | overloaded_bytes = overloaded_constants(bytes) 302 | overloaded_floats = overloaded_constants(float) 303 | overloaded_ints = overloaded_constants(int) 304 | overloaded_complexes = overloaded_constants(complex) 305 | 306 | haskell_strs = overloaded_strs(tuple) 307 | bytearray_literals = overloaded_bytes(bytearray) 308 | decimal_literals = overloaded_floats(Decimal) 309 | 310 | 311 | def _start_comprehension(self, *instrs): 312 | self.begin(IN_COMPREHENSION) 313 | yield from self.patterndispatcher(instrs) 314 | 315 | 316 | def _return_value(self, instr): 317 | # TOS = collection 318 | 319 | yield instructions.LOAD_CONST(self.xform).steal(instr) 320 | # TOS = self.xform 321 | # TOS1 = collection 322 | 323 | yield instructions.ROT_TWO() 324 | # TOS = collection 325 | # TOS1 = self.xform 326 | 327 | yield instructions.CALL_FUNCTION(1) 328 | # TOS = self.xform(collection) 329 | 330 | yield instr 331 | 332 | 333 | # Added as a method for overloaded_build 334 | def _build(self, instr): 335 | yield instr 336 | # TOS = new_list 337 | 338 | yield instructions.LOAD_CONST(self.xform) 339 | # TOS = astype 340 | # TOS1 = new_list 341 | 342 | yield instructions.ROT_TWO() 343 | # TOS = new_list 344 | # TOS1 = astype 345 | 346 | yield instructions.CALL_FUNCTION(1) 347 | # TOS = astype(new_list) 348 | 349 | 350 | def overloaded_build(type_, add_name=None): 351 | """Factory for constant transformers that apply to a given 352 | build instruction. 353 | 354 | Parameters 355 | ---------- 356 | type_ : type 357 | The object type to overload the construction of. This must be one of 358 | "buildable" types, or types with a "BUILD_*" instruction. 359 | add_name : str, optional 360 | The suffix of the instruction tha adds elements to the collection. 361 | For example: 'add' or 'append' 362 | 363 | Returns 364 | ------- 365 | transformer : subclass of CodeTransformer 366 | A new code transformer class that will overload the provided 367 | literal types. 368 | """ 369 | typename = type_.__name__ 370 | instrname = 'BUILD_' + typename.upper() 371 | dict_ = OrderedDict( 372 | __doc__=dedent( 373 | """ 374 | A CodeTransformer for overloading {name} instructions. 375 | """.format(name=instrname) 376 | ) 377 | ) 378 | 379 | try: 380 | build_instr = getattr(instructions, instrname) 381 | except AttributeError: 382 | raise TypeError("type %s is not buildable" % typename) 383 | 384 | if add_name is not None: 385 | try: 386 | add_instr = getattr( 387 | instructions, 388 | '_'.join((typename, add_name)).upper(), 389 | ) 390 | except AttributeError: 391 | TypeError("type %s is not addable" % typename) 392 | 393 | dict_['_start_comprehension'] = pattern( 394 | build_instr, matchany[var], add_instr, 395 | )(_start_comprehension) 396 | dict_['_return_value'] = pattern( 397 | instructions.RETURN_VALUE, startcodes=(IN_COMPREHENSION,), 398 | )(_return_value) 399 | else: 400 | add_instr = None 401 | 402 | dict_['_build'] = pattern(build_instr)(_build) 403 | 404 | if not typename.endswith('s'): 405 | typename = typename + 's' 406 | 407 | return type( 408 | 'overloaded_' + typename, 409 | (overloaded_constants(type_),), 410 | dict_, 411 | ) 412 | 413 | 414 | overloaded_slices = overloaded_build(slice) 415 | overloaded_lists = overloaded_build(list, 'append') 416 | overloaded_sets = overloaded_build(set, 'add') 417 | 418 | 419 | # Add a special method for set overloader. 420 | def transform_consts(self, consts): 421 | consts = super(overloaded_sets, self).transform_consts(consts) 422 | return tuple( 423 | # Always pass a thawed set so mutations can happen inplace. 424 | self.xform(set(const)) if isinstance(const, frozenset) else const 425 | for const in consts 426 | ) 427 | 428 | 429 | overloaded_sets.transform_consts = transform_consts 430 | del transform_consts 431 | frozenset_literals = overloaded_sets(frozenset) 432 | 433 | 434 | overloaded_tuples = overloaded_build(tuple) 435 | 436 | 437 | # Add a special method for the tuple overloader. 438 | def transform_consts(self, consts): 439 | consts = super(overloaded_tuples, self).transform_consts(consts) 440 | return tuple( 441 | self.xform(const) if isinstance(const, tuple) else const 442 | for const in consts 443 | ) 444 | 445 | 446 | overloaded_tuples.transform_consts = transform_consts 447 | del transform_consts 448 | 449 | 450 | @instance 451 | class islice_literals(CodeTransformer): 452 | """Transformer that turns slice indexing into an islice object. 453 | 454 | Examples 455 | -------- 456 | >>> from codetransformer.transformers.literals import islice_literals 457 | >>> @islice_literals 458 | ... def f(): 459 | ... return map(str, (1, 2, 3, 4))[:2] 460 | ... 461 | >>> f() 462 | 463 | >>> tuple(f()) 464 | ('1', '2') 465 | """ 466 | @pattern(instructions.BINARY_SUBSCR) 467 | def _binary_subscr(self, instr): 468 | yield instructions.LOAD_CONST(self._islicer).steal(instr) 469 | # TOS = self._islicer 470 | # TOS1 = k 471 | # TOS2 = m 472 | 473 | yield instructions.ROT_THREE() 474 | # TOS = k 475 | # TOS1 = m 476 | # TOS2 = self._islicer 477 | 478 | yield instructions.CALL_FUNCTION(2) 479 | # TOS = self._islicer(m, k) 480 | 481 | @staticmethod 482 | def _islicer(m, k): 483 | if isinstance(k, slice): 484 | return islice(m, k.start, k.stop, k.step) 485 | 486 | return m[k] 487 | -------------------------------------------------------------------------------- /codetransformer/transformers/pattern_matched_exceptions.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from ..core import CodeTransformer 4 | from ..instructions import ( 5 | BUILD_TUPLE, 6 | CALL_FUNCTION, 7 | COMPARE_OP, 8 | LOAD_CONST, 9 | POP_TOP, 10 | ROT_TWO, 11 | ) 12 | from ..patterns import pattern 13 | 14 | 15 | def match(match_expr, exc_type, exc_value, exc_traceback): 16 | """ 17 | Called to determine whether or not an except block should be matched. 18 | 19 | True -> enter except block 20 | False -> don't enter except block 21 | """ 22 | # Emulate standard behavior when match_expr is an exception subclass. 23 | if isinstance(match_expr, type) and issubclass(match_expr, BaseException): 24 | return issubclass(exc_type, match_expr) 25 | 26 | # Match on type and args when match_expr is an exception instance. 27 | return ( 28 | issubclass(exc_type, type(match_expr)) 29 | and 30 | match_expr.args == exc_value.args 31 | ) 32 | 33 | 34 | class pattern_matched_exceptions(CodeTransformer): 35 | """ 36 | Allows usage of arbitrary expressions and matching functions in 37 | `except` blocks. 38 | 39 | When an exception is raised in an except block in a function decorated with 40 | `pattern_matched_exceptions`, a matching function will be called with the 41 | block's expression and the three values returned by sys.exc_info(). If the 42 | matching function returns `True`, we enter the corresponding except-block, 43 | otherwise we continue to the next block, or re-raise if there are no more 44 | blocks to check 45 | 46 | Parameters 47 | ---------- 48 | matcher : function, optional 49 | A function accepting an expression and the values of sys.exc_info, 50 | returning True if the exception info "matches" the expression. 51 | 52 | The default behavior is to emulate standard python when the match 53 | expression is a *subtype* of Exception, and to compare exc.type and 54 | exc.args when the match expression is an *instance* of Exception. 55 | 56 | Example 57 | ------- 58 | >>> @pattern_matched_exceptions() 59 | ... def foo(): 60 | ... try: 61 | ... raise ValueError('bar') 62 | ... except ValueError('buzz'): 63 | ... return 'buzz' 64 | ... except ValueError('bar'): 65 | ... return 'bar' 66 | >>> foo() 67 | 'bar' 68 | """ 69 | def __init__(self, matcher=match): 70 | super().__init__() 71 | self._matcher = matcher 72 | 73 | if sys.version_info < (3, 6): 74 | from ..instructions import CALL_FUNCTION_VAR 75 | 76 | def _match(self, 77 | instr, 78 | CALL_FUNCTION_VAR=CALL_FUNCTION_VAR): 79 | yield ROT_TWO().steal(instr) 80 | yield POP_TOP() 81 | yield LOAD_CONST(self._matcher) 82 | yield ROT_TWO() 83 | yield LOAD_CONST(sys.exc_info) 84 | yield CALL_FUNCTION(0) 85 | yield CALL_FUNCTION_VAR(1) 86 | 87 | del CALL_FUNCTION_VAR 88 | else: 89 | from ..instructions import ( 90 | CALL_FUNCTION_EX, 91 | BUILD_TUPLE_UNPACK_WITH_CALL, 92 | ) 93 | 94 | def _match(self, 95 | instr, 96 | CALL_FUNCTION_EX=CALL_FUNCTION_EX, 97 | BUILD_TUPLE_UNPACK_WITH_CALL=BUILD_TUPLE_UNPACK_WITH_CALL): 98 | yield ROT_TWO().steal(instr) 99 | yield POP_TOP() 100 | yield LOAD_CONST(self._matcher) 101 | yield ROT_TWO() 102 | yield BUILD_TUPLE(1) 103 | yield LOAD_CONST(sys.exc_info) 104 | yield CALL_FUNCTION(0) 105 | yield BUILD_TUPLE_UNPACK_WITH_CALL(2) 106 | yield CALL_FUNCTION_EX(0) 107 | 108 | del CALL_FUNCTION_EX 109 | del BUILD_TUPLE_UNPACK_WITH_CALL 110 | 111 | @pattern(COMPARE_OP) 112 | def _compare_op(self, instr): 113 | if instr.equiv(COMPARE_OP.EXCEPTION_MATCH): 114 | yield from self._match(instr) 115 | else: 116 | yield instr 117 | -------------------------------------------------------------------------------- /codetransformer/transformers/precomputed_slices.py: -------------------------------------------------------------------------------- 1 | from codetransformer.core import CodeTransformer 2 | from codetransformer.instructions import LOAD_CONST, BUILD_SLICE 3 | from codetransformer.patterns import pattern, plus 4 | 5 | 6 | class precomputed_slices(CodeTransformer): 7 | """ 8 | An optimizing transformer that precomputes and inlines slice literals. 9 | 10 | Example 11 | ------- 12 | >>> from dis import dis 13 | >>> def first_five(l): 14 | ... return l[:5] 15 | ... 16 | >>> dis(first_five) # doctest: +SKIP 17 | 2 0 LOAD_FAST 0 (l) 18 | 3 LOAD_CONST 0 (None) 19 | 6 LOAD_CONST 1 (5) 20 | 9 BUILD_SLICE 2 21 | 12 BINARY_SUBSCR 22 | 13 RETURN_VALUE 23 | >>> dis(precomputed_slices()(first_five)) # doctest: +SKIP 24 | 2 0 LOAD_FAST 0 (l) 25 | 3 LOAD_CONST 0 (slice(None, 5, None)) 26 | 6 BINARY_SUBSCR 27 | 7 RETURN_VALUE 28 | """ 29 | @pattern(LOAD_CONST[plus], BUILD_SLICE) 30 | def make_constant_slice(self, *instrs): 31 | *loads, build = instrs 32 | if build.arg != len(loads): 33 | # There are non-constant loads before the consts: 34 | # e.g. x[:1:2] 35 | yield from instrs 36 | 37 | slice_ = slice(*(instr.arg for instr in loads)) 38 | yield LOAD_CONST(slice_).steal(loads[0]) 39 | -------------------------------------------------------------------------------- /codetransformer/transformers/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llllllllll/codetransformer/c5f551e915df45adc7da7e0b1b635f0cc6a1bb27/codetransformer/transformers/tests/__init__.py -------------------------------------------------------------------------------- /codetransformer/transformers/tests/test_add2mul.py: -------------------------------------------------------------------------------- 1 | from ..add2mul import add2mul 2 | 3 | 4 | def test_add2mul(): 5 | 6 | @add2mul() 7 | def foo(a, b): 8 | return (a + b + 2) - 1 9 | 10 | assert foo(1, 2) == 3 11 | assert foo(2, 2) == 7 12 | -------------------------------------------------------------------------------- /codetransformer/transformers/tests/test_constants.py: -------------------------------------------------------------------------------- 1 | import os 2 | from sys import _getframe 3 | from types import CodeType 4 | 5 | import pytest 6 | 7 | from codetransformer.code import Code 8 | from ..constants import asconstants 9 | 10 | 11 | basename = os.path.basename(__file__) 12 | 13 | 14 | def test_global(): 15 | 16 | @asconstants(a=1) 17 | def f(): 18 | return a # noqa 19 | 20 | assert f() == 1 21 | 22 | 23 | def test_name(): 24 | for const in compile( 25 | 'class C:\n b = a', '', 'exec').co_consts: 26 | 27 | if isinstance(const, CodeType): 28 | pre_transform = Code.from_pycode(const) 29 | code = asconstants(a=1).transform(pre_transform) 30 | break 31 | else: 32 | raise AssertionError('There should be a code object in there!') 33 | 34 | ns = {} 35 | exec(code.to_pycode(), ns) 36 | assert ns['b'] == 1 37 | 38 | 39 | def test_closure(): 40 | def f(): 41 | a = 2 42 | 43 | @asconstants(a=1) 44 | def g(): 45 | return a 46 | 47 | return g 48 | 49 | assert f()() == 1 50 | 51 | 52 | def test_store(): 53 | with pytest.raises(SyntaxError) as e: 54 | @asconstants(a=1) 55 | def f(): 56 | a = 1 # noqa 57 | 58 | line = _getframe().f_lineno - 2 59 | assert ( 60 | str(e.value) == 61 | "can't assign to constant name 'a' (%s, line %d)" % (basename, line) 62 | ) 63 | 64 | 65 | def test_delete(): 66 | with pytest.raises(SyntaxError) as e: 67 | @asconstants(a=1) 68 | def f(): 69 | del a # noqa 70 | 71 | line = _getframe().f_lineno - 2 72 | assert ( 73 | str(e.value) == 74 | "can't delete constant name 'a' (%s, line %d)" % (basename, line) 75 | ) 76 | 77 | 78 | def test_argname_overlap(): 79 | with pytest.raises(SyntaxError) as e: 80 | @asconstants(a=1) 81 | def f(a): 82 | pass 83 | 84 | assert str(e.value) == "argument names overlap with constant names: {'a'}" 85 | -------------------------------------------------------------------------------- /codetransformer/transformers/tests/test_exc_patterns.py: -------------------------------------------------------------------------------- 1 | from pytest import raises 2 | from ..pattern_matched_exceptions import pattern_matched_exceptions 3 | 4 | 5 | def test_patterns(): 6 | 7 | @pattern_matched_exceptions() 8 | def foo(): 9 | try: 10 | raise ValueError("bar") 11 | except TypeError: 12 | raise 13 | except ValueError("foo"): 14 | raise 15 | except ValueError("bar"): 16 | return "bar" 17 | except ValueError("buzz"): 18 | raise 19 | 20 | assert foo() == "bar" 21 | 22 | 23 | def test_patterns_bind_name(): 24 | 25 | @pattern_matched_exceptions() 26 | def foo(): 27 | try: 28 | raise ValueError("bar") 29 | except ValueError("foo") as e: 30 | return e.args[0] 31 | except ValueError("bar") as e: 32 | return e.args[0] 33 | except ValueError("buzz") as e: 34 | return e.args[0] 35 | 36 | assert foo() == "bar" 37 | 38 | 39 | def test_patterns_reraise(): 40 | 41 | @pattern_matched_exceptions() 42 | def foo(): 43 | try: 44 | raise ValueError("bar") 45 | except ValueError("bar"): 46 | raise 47 | 48 | with raises(ValueError) as err: 49 | foo() 50 | 51 | assert err.type == ValueError 52 | assert err.value.args == ('bar',) 53 | 54 | 55 | def test_normal_exc_match(): 56 | 57 | @pattern_matched_exceptions() 58 | def foo(): 59 | try: 60 | raise ValueError("bar") 61 | except ValueError: 62 | return "matched" 63 | except ValueError("bar"): 64 | raise 65 | 66 | assert foo() == "matched" 67 | 68 | 69 | def test_exc_match_custom_func(): 70 | 71 | def match_greater(expr, exc_type, exc_value, exc_traceback): 72 | return expr > exc_value.args[0] 73 | 74 | @pattern_matched_exceptions(match_greater) 75 | def foo(): 76 | try: 77 | raise ValueError(5) 78 | except 4: 79 | return 4 80 | except 5: 81 | return 5 82 | except 6: 83 | return 6 84 | 85 | assert foo() == 6 86 | -------------------------------------------------------------------------------- /codetransformer/transformers/tests/test_interpolated_strings.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import pytest 4 | 5 | from ..interpolated_strings import interpolated_strings 6 | 7 | 8 | pytestmark = pytest.mark.skipif( 9 | sys.version_info >= (3, 6), 10 | reason='interpolated_strings is deprecated, just use f-strings', 11 | ) 12 | 13 | 14 | def test_interpolated_bytes(): 15 | 16 | @interpolated_strings(transform_bytes=True) 17 | def enabled(a, b, c): 18 | return b"{a} {b!r} {c}" 19 | 20 | assert enabled(1, 2, 3) == "{a} {b!r} {c}".format(a=1, b=2, c=3) 21 | 22 | @interpolated_strings() 23 | def default(a, b, c): 24 | return b"{a} {b!r} {c}" 25 | 26 | assert default(1, 2, 3) == "{a} {b!r} {c}".format(a=1, b=2, c=3) 27 | 28 | @interpolated_strings(transform_bytes=False) 29 | def disabled(a, b, c): 30 | return b"{a} {b!r} {c}" 31 | 32 | assert disabled(1, 2, 3) == b"{a} {b!r} {c}" 33 | 34 | 35 | def test_interpolated_str(): 36 | 37 | @interpolated_strings(transform_str=True) 38 | def enabled(a, b, c): 39 | return "{a} {b!r} {c}" 40 | 41 | assert enabled(1, 2, 3) == "{a} {b!r} {c}".format(a=1, b=2, c=3) 42 | 43 | @interpolated_strings() 44 | def default(a, b, c): 45 | return "{a} {b!r} {c}" 46 | 47 | assert default(1, 2, 3) == "{a} {b!r} {c}" 48 | 49 | @interpolated_strings(transform_bytes=False) 50 | def disabled(a, b, c): 51 | return "{a} {b!r} {c}" 52 | 53 | assert disabled(1, 2, 3) == "{a} {b!r} {c}" 54 | 55 | 56 | def test_no_cross_pollination(): 57 | 58 | @interpolated_strings(transform_bytes=True) 59 | def ignore_str(a): 60 | u = "{a}" 61 | b = b"{a}" 62 | return u, b 63 | 64 | assert ignore_str(1) == ("{a}", "1") 65 | 66 | @interpolated_strings(transform_bytes=False, transform_str=True) 67 | def ignore_bytes(a): 68 | u = "{a}" 69 | b = b"{a}" 70 | return u, b 71 | 72 | assert ignore_bytes(1) == ("1", b"{a}") 73 | 74 | 75 | def test_string_in_nested_const(): 76 | 77 | @interpolated_strings(transform_str=True) 78 | def foo(a, b): 79 | return ("{a}", (("{b}",), "{a} {b}"), (1, 2)) 80 | 81 | assert foo(1, 2) == ("1", (("2",), "1 2"), (1, 2)) 82 | 83 | @interpolated_strings(transform_str=True) 84 | def bar(a): 85 | return "1" in {"{a}"} 86 | 87 | assert bar(1) 88 | assert not bar(2) 89 | -------------------------------------------------------------------------------- /codetransformer/transformers/tests/test_literals.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for literal transformers 3 | """ 4 | from collections import OrderedDict 5 | from decimal import Decimal 6 | from itertools import islice 7 | 8 | from ..literals import ( 9 | islice_literals, 10 | overloaded_dicts, 11 | overloaded_bytes, 12 | overloaded_floats, 13 | overloaded_lists, 14 | overloaded_sets, 15 | overloaded_slices, 16 | overloaded_strs, 17 | overloaded_tuples, 18 | ) 19 | 20 | 21 | def test_overload_thing_with_thing_is_noop(): 22 | test_vals = [('a', 1), ('b', 2), ('c', 3)] 23 | for t in dict, set, list, tuple: 24 | expected = t(test_vals) 25 | f = eval("lambda: %s" % (expected,)) 26 | overloaded = eval(t.__name__.join(['overloaded_', 's']))(t)(f) 27 | assert f() == overloaded() == expected 28 | 29 | 30 | def test_overloaded_dicts(): 31 | 32 | @overloaded_dicts(OrderedDict) 33 | def literal(): 34 | return {'a': 1, 'b': 2, 'c': 3} 35 | 36 | assert literal() == OrderedDict((('a', 1), ('b', 2), ('c', 3))) 37 | 38 | @overloaded_dicts(OrderedDict) 39 | def comprehension(): 40 | return {k: n for n, k in enumerate('abc', 1)} 41 | 42 | assert comprehension() == OrderedDict((('a', 1), ('b', 2), ('c', 3))) 43 | 44 | 45 | def test_overloaded_bytes(): 46 | 47 | @overloaded_bytes(list) 48 | def bytes_to_list(): 49 | return ["unicode", b"bytes", 1, 2, 3] 50 | 51 | assert bytes_to_list() == ["unicode", list(b"bytes"), 1, 2, 3] 52 | 53 | @overloaded_bytes(list) 54 | def bytes_to_list_tuple(): 55 | return "unicode", b"bytes", 1, 2, 3 56 | 57 | assert bytes_to_list_tuple() == ("unicode", list(b"bytes"), 1, 2, 3) 58 | 59 | @overloaded_bytes(int) 60 | def bytes_in_set(x): 61 | return x in {b'3'} 62 | 63 | assert not bytes_in_set(b'3') 64 | assert bytes_in_set(3) 65 | 66 | @overloaded_bytes(bytearray) 67 | def mutable_bytes(): 68 | return b'123' 69 | 70 | assert isinstance(mutable_bytes(), bytearray) 71 | 72 | 73 | def test_overloaded_floats(): 74 | 75 | @overloaded_floats(Decimal) 76 | def float_to_decimal(): 77 | return [2, 2.0, 3.5] 78 | 79 | assert float_to_decimal() == [2, Decimal(2.0), Decimal(3.5)] 80 | 81 | @overloaded_floats(Decimal) 82 | def float_to_decimal_tuple(): 83 | return (2, 2.0, 3.5) 84 | 85 | assert float_to_decimal_tuple() == (2, Decimal(2.0), Decimal(3.5)) 86 | 87 | @overloaded_floats(Decimal) 88 | def float_in_set(x): 89 | return x in {3.0} 90 | 91 | xformed_const = float_in_set.__code__.co_consts[0] 92 | assert isinstance(xformed_const, frozenset) 93 | assert len(xformed_const) == 1 94 | assert isinstance(tuple(xformed_const)[0], Decimal) 95 | assert tuple(xformed_const)[0] == Decimal(3.0) 96 | 97 | 98 | def test_overloaded_lists(): 99 | 100 | @overloaded_lists(tuple) 101 | def frozen_list(): 102 | return [1, 2, 3] 103 | 104 | assert frozen_list() == (1, 2, 3) 105 | 106 | @overloaded_lists(tuple) 107 | def frozen_in_tuple(): 108 | return [1, 2, 3], [4, 5, 6] 109 | 110 | assert frozen_in_tuple() == ((1, 2, 3), (4, 5, 6)) 111 | 112 | @overloaded_lists(tuple) 113 | def frozen_in_set(): 114 | # lists are not hashable but tuple are. 115 | return [1, 2, 3] in {[1, 2, 3]} 116 | 117 | assert frozen_in_set() 118 | 119 | @overloaded_lists(tuple) 120 | def frozen_comprehension(): 121 | return [a for a in (1, 2, 3)] 122 | 123 | assert frozen_comprehension() == (1, 2, 3) 124 | 125 | 126 | def test_overloaded_strs(): 127 | 128 | @overloaded_strs(tuple) 129 | def haskell_strs(): 130 | return 'abc' 131 | 132 | assert haskell_strs() == ('a', 'b', 'c') 133 | 134 | @overloaded_strs(tuple) 135 | def cs_in_tuple(): 136 | return 'abc', 'def' 137 | 138 | assert cs_in_tuple() == (('a', 'b', 'c'), ('d', 'e', 'f')) 139 | 140 | 141 | def test_overloaded_sets(): 142 | 143 | @overloaded_sets(frozenset) 144 | def f(): 145 | return {'a', 'b', 'c'} 146 | 147 | assert isinstance(f(), frozenset) 148 | assert f() == frozenset({'a', 'b', 'c'}) 149 | 150 | class invertedset(set): 151 | def __contains__(self, e): 152 | return not super().__contains__(e) 153 | 154 | @overloaded_sets(invertedset) 155 | def containment_with_consts(): 156 | # This will create a frozenset FIRST and then we should pull it 157 | # into an invertedset 158 | return 'd' in {'e'} 159 | 160 | assert containment_with_consts() 161 | 162 | def frozen_comprehension(): 163 | return {a for a in 'abc'} 164 | 165 | assert frozen_comprehension() == frozenset('abc') 166 | 167 | 168 | def test_overloaded_tuples(): 169 | 170 | @overloaded_tuples(list) 171 | def nonconst(): 172 | a = 1 173 | b = 2 174 | c = 3 175 | return (a, b, c) 176 | 177 | assert nonconst() == [1, 2, 3] 178 | 179 | @overloaded_tuples(list) 180 | def const(): 181 | return (1, 2, 3) 182 | 183 | assert const() == [1, 2, 3] 184 | 185 | 186 | def test_overloaded_slices(): 187 | 188 | def concrete_slice(slice_): 189 | return tuple(range(slice_.start, slice_.stop))[::slice_.step] 190 | 191 | class C: 192 | _idx = None 193 | 194 | def __getitem__(self, idx): 195 | self._idx = idx 196 | return idx 197 | 198 | c = C() 199 | 200 | @overloaded_slices(concrete_slice) 201 | def f(): 202 | return c[1:10:2] 203 | 204 | f() 205 | assert c._idx == (1, 3, 5, 7, 9) 206 | 207 | 208 | def test_islice_literals(): 209 | 210 | @islice_literals 211 | def islice_test(): 212 | return map(str, (1, 2, 3, 4))[:2] 213 | 214 | assert isinstance(islice_test(), islice) 215 | assert tuple(islice_test()) == ('1', '2') 216 | -------------------------------------------------------------------------------- /codetransformer/transformers/tests/test_precomputed_slices.py: -------------------------------------------------------------------------------- 1 | from codetransformer.code import Code 2 | from codetransformer.instructions import BUILD_SLICE, LOAD_CONST 3 | 4 | from ..precomputed_slices import precomputed_slices 5 | 6 | 7 | def test_precomputed_slices(): 8 | 9 | @precomputed_slices() 10 | def foo(a): 11 | return a[1:5] 12 | 13 | l = list(range(10)) 14 | assert foo(l) == l[1:5] 15 | assert slice(1, 5) in foo.__code__.co_consts 16 | 17 | instrs = Code.from_pyfunc(foo).instrs 18 | assert LOAD_CONST(slice(1, 5)).equiv(instrs[1]) 19 | assert BUILD_SLICE not in set(map(type, instrs)) 20 | 21 | 22 | def test_precomputed_slices_non_const(): 23 | 24 | transformer = precomputed_slices() 25 | 26 | def f(a, b): 27 | with_non_const = a[b] 28 | with_mixed = a[1, b] 29 | return with_non_const, with_mixed 30 | 31 | transformed = transformer(f) 32 | 33 | f_instrs = Code.from_pyfunc(f).instrs 34 | transformed_instrs = Code.from_pyfunc(transformed).instrs 35 | 36 | for orig, xformed in zip(f_instrs, transformed_instrs): 37 | assert orig.equiv(xformed) 38 | -------------------------------------------------------------------------------- /codetransformer/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llllllllll/codetransformer/c5f551e915df45adc7da7e0b1b635f0cc6a1bb27/codetransformer/utils/__init__.py -------------------------------------------------------------------------------- /codetransformer/utils/functional.py: -------------------------------------------------------------------------------- 1 | """ 2 | codetransformer.utils.functional 3 | -------------------------------- 4 | 5 | Utilities for functional programming. 6 | """ 7 | 8 | from toolz import complement, flip 9 | 10 | 11 | def is_a(type_): 12 | """More curryable version of isinstance.""" 13 | return flip(isinstance, type_) 14 | 15 | 16 | def not_a(type_): 17 | """More curryable version of not isinstance.""" 18 | return complement(is_a(type_)) 19 | 20 | 21 | def scanl(f, n, ns): 22 | """Reduce ns by f starting with n yielding each intermediate value. 23 | 24 | tuple(scanl(f, n, ns))[-1] == reduce(f, ns, n) 25 | 26 | Parameters 27 | ---------- 28 | f : callable 29 | A binary function. 30 | n : any 31 | The starting value. 32 | ns : iterable of any 33 | The iterable to scan over. 34 | 35 | Yields 36 | ------ 37 | p : any 38 | The value of reduce(f, ns[:idx]) where idx is the current index. 39 | 40 | Examples 41 | -------- 42 | >>> import operator as op 43 | >>> tuple(scanl(op.add, 0, (1, 2, 3, 4))) 44 | (0, 1, 3, 6, 10) 45 | """ 46 | yield n 47 | for m in ns: 48 | n = f(n, m) 49 | yield n 50 | 51 | 52 | def reverse_dict(d): 53 | """Reverse a dictionary, replacing the keys and values. 54 | 55 | Parameters 56 | ---------- 57 | d : dict 58 | The dict to reverse. 59 | 60 | Returns 61 | ------- 62 | rd : dict 63 | The dict with the keys and values flipped. 64 | 65 | Examples 66 | -------- 67 | >>> d = {'a': 1, 'b': 2, 'c': 3} 68 | >>> e = reverse_dict(d) 69 | >>> e == {1: 'a', 2: 'b', 3: 'c'} 70 | True 71 | """ 72 | return {v: k for k, v in d.items()} 73 | 74 | 75 | def ffill(iterable): 76 | """Forward fill non None values in some iterable. 77 | 78 | Parameters 79 | ---------- 80 | iterable : iterable 81 | The iterable to forward fill. 82 | 83 | Yields 84 | ------ 85 | e : any 86 | The last non None value or None if there has not been a non None value. 87 | """ 88 | it = iter(iterable) 89 | previous = next(it) 90 | yield previous 91 | for e in it: 92 | if e is None: 93 | yield previous 94 | else: 95 | previous = e 96 | yield e 97 | 98 | 99 | def flatten(seq, *, recurse_types=(tuple, list, set, frozenset)): 100 | """ 101 | Convert a (possibly nested) iterator into a flattened iterator. 102 | 103 | Parameters 104 | ---------- 105 | seq : iterable 106 | The sequence to flatten. 107 | recurse_types, optional 108 | Types to recursively flatten. 109 | Defaults to (tuple, list, set, frozenset). 110 | 111 | >>> list(flatten((1, (2, 3), ((4,), 5)))) 112 | [1, 2, 3, 4, 5] 113 | 114 | >>> list(flatten(["abc", "def"], recurse_types=(str,))) 115 | ['a', 'b', 'c', 'd', 'e', 'f'] 116 | """ 117 | for elem in seq: 118 | if isinstance(elem, recurse_types): 119 | yield from flatten(elem) 120 | else: 121 | yield elem 122 | -------------------------------------------------------------------------------- /codetransformer/utils/immutable.py: -------------------------------------------------------------------------------- 1 | """ 2 | codetransformer.utils.immutable 3 | ------------------------------- 4 | 5 | Utilities for creating and working with immutable objects. 6 | """ 7 | 8 | from collections import ChainMap 9 | from inspect import getfullargspec 10 | from itertools import starmap, repeat 11 | from textwrap import dedent 12 | from weakref import WeakKeyDictionary 13 | 14 | 15 | class immutableattr: 16 | """An immutable attribute of a class. 17 | 18 | Parameters 19 | ---------- 20 | attr : any 21 | The attribute. 22 | """ 23 | def __init__(self, attr): 24 | self._attr = attr 25 | 26 | def __get__(self, instance, owner): 27 | return self._attr 28 | 29 | 30 | class lazyval: 31 | """A memoizing property. 32 | 33 | Parameters 34 | ---------- 35 | func : callable 36 | The function used to compute the value of the descriptor. 37 | """ 38 | def __init__(self, func): 39 | self._cache = WeakKeyDictionary() 40 | self._func = func 41 | 42 | def __get__(self, instance, owner): 43 | if instance is None: 44 | return self 45 | 46 | cache = self._cache 47 | try: 48 | return cache[instance] 49 | except KeyError: 50 | cache[instance] = val = self._func(instance) 51 | return val 52 | 53 | 54 | def _no_arg_init(self): 55 | pass 56 | 57 | 58 | object_setattr = object.__setattr__ 59 | 60 | 61 | def initialize_slot(obj, name, value): 62 | """Initalize an unitialized slot to a value. 63 | 64 | If there is already a value for this slot, this is a nop. 65 | 66 | Parameters 67 | ---------- 68 | obj : immutable 69 | An immutable object. 70 | name : str 71 | The name of the slot to initialize. 72 | value : any 73 | The value to initialize the slot to. 74 | """ 75 | if not hasattr(obj, name): 76 | object_setattr(obj, name, value) 77 | 78 | 79 | def _create_init(name, slots, defaults): 80 | """Create the __init__ function for an immutable object. 81 | 82 | Parameters 83 | ---------- 84 | name : str 85 | The name of the immutable class. 86 | slots : iterable of str 87 | The __slots__ field from the class. 88 | defaults : dict or None 89 | The default values for the arguments to __init__. 90 | 91 | Returns 92 | ------- 93 | init : callable 94 | The __init__ function for the new immutable class. 95 | """ 96 | if any(s.startswith('__') for s in slots): 97 | raise TypeError( 98 | "immutable classes may not have slots that start with '__'", 99 | ) 100 | 101 | # If we have no defaults, ignore all of this. 102 | kwdefaults = None 103 | if defaults is not None: 104 | hit_default = False 105 | _defaults = [] # positional defaults 106 | kwdefaults = {} # kwonly defaults 107 | kwdefs = False 108 | for s in slots: 109 | if s not in defaults and hit_default: 110 | raise SyntaxError( 111 | 'non-default argument follows default argument' 112 | ) 113 | 114 | if not kwdefs: 115 | try: 116 | # Try to grab the next default. 117 | # Pop so that we know they were all consumed when we 118 | # are done. 119 | _defaults.append(defaults.pop(s)) 120 | except KeyError: 121 | # Not in the dict, we haven't hit any defaults yet. 122 | pass 123 | else: 124 | # We are now consuming default arguments. 125 | hit_default = True 126 | if s.startswith('*'): 127 | if s in defaults: 128 | raise TypeError( 129 | 'cannot set default for var args or var kwargs', 130 | ) 131 | if not s.startswith('**'): 132 | kwdefs = True 133 | else: 134 | kwdefaults[s] = defaults.pop(s) 135 | 136 | if defaults: 137 | # We didn't consume all of the defaults. 138 | raise TypeError( 139 | 'default value for non-existent argument%s: %s' % ( 140 | 's' if len(defaults) > 1 else '', 141 | ', '.join(starmap('{0}={1!r}'.format, defaults.items())), 142 | ) 143 | ) 144 | 145 | # cast back to tuples 146 | defaults = tuple(_defaults) 147 | 148 | if not slots: 149 | return _no_arg_init, () 150 | 151 | ns = {'__initialize_slot': initialize_slot} 152 | # filter out lone star 153 | slotnames = tuple(filter(None, (s.strip('*') for s in slots))) 154 | # We are using exec here so that we can later inspect the call signature 155 | # of the __init__. This makes the positional vs keywords work as intended. 156 | # This is totally reasonable, no h8 m8! 157 | exec( 158 | 'def __init__(_{name}__self, {args}): \n {assign}'.format( 159 | name=name, 160 | args=', '.join(slots), 161 | assign='\n '.join( 162 | map( 163 | '__initialize_slot(_{1}__self, "{0}", {0})'.format, 164 | slotnames, 165 | repeat(name), 166 | ), 167 | ), 168 | ), 169 | ns, 170 | ) 171 | init = ns['__init__'] 172 | init.__defaults__ = defaults 173 | init.__kwdefaults__ = kwdefaults 174 | return init, slotnames 175 | 176 | 177 | def _wrapinit(init): 178 | """Wrap an existing initialize function by thawing self for the duration 179 | of the init. 180 | 181 | Parameters 182 | ---------- 183 | init : callable 184 | The user-provided init. 185 | 186 | Returns 187 | ------- 188 | wrapped : callable 189 | The wrapped init method. 190 | """ 191 | try: 192 | spec = getfullargspec(init) 193 | except TypeError: 194 | # we cannot preserve the type signature. 195 | def __init__(*args, **kwargs): 196 | self = args[0] 197 | __setattr__._initializing.add(self) 198 | init(*args, **kwargs) 199 | __setattr__._initializing.remove(self) 200 | _check_missing_slots(self) 201 | 202 | return __init__ 203 | 204 | args = spec.args 205 | varargs = spec.varargs 206 | if not (args or varargs): 207 | raise TypeError( 208 | "%r must accept at least one positional argument for 'self'" % 209 | getattr(init, '__qualname__', getattr(init, '__name__', init)), 210 | ) 211 | 212 | if not args: 213 | self = '%s[0]' % varargs 214 | forward = argspec = '*' + varargs 215 | else: 216 | self = args[0] 217 | forward = argspec = ', '.join(args) 218 | 219 | if args and varargs: 220 | forward = '%s, *%s' % (forward, spec.varargs) 221 | argspec = '%s, *%s' % (argspec, spec.varargs) 222 | if spec.kwonlyargs: 223 | forward = '%s, %s' % ( 224 | forward, 225 | ', '.join(map('{0}={0}'.format, spec.kwonlyargs)) 226 | ) 227 | argspec = '%s,%s%s' % ( 228 | argspec, 229 | '*, ' if not spec.varargs else '', 230 | ', '.join(spec.kwonlyargs), 231 | ) 232 | if spec.varkw: 233 | forward = '%s, **%s' % (forward, spec.varkw) 234 | argspec = '%s, **%s' % (argspec, spec.varkw) 235 | 236 | ns = { 237 | '__init': init, 238 | '__initializing': __setattr__._initializing, 239 | '__check_missing_slots': _check_missing_slots, 240 | } 241 | exec( 242 | dedent( 243 | """\ 244 | def __init__({argspec}): 245 | __initializing.add({self}) 246 | __init({forward}) 247 | __initializing.remove({self}) 248 | __check_missing_slots({self}) 249 | """.format( 250 | argspec=argspec, 251 | self=self, 252 | forward=forward, 253 | ), 254 | ), 255 | ns, 256 | ) 257 | __init__ = ns['__init__'] 258 | __init__.__defaults__ = spec.defaults 259 | __init__.__kwdefaults__ = spec.kwonlydefaults 260 | __init__.__annotations__ = spec.annotations 261 | return __init__ 262 | 263 | 264 | def _check_missing_slots(ob): 265 | """Check that all slots have been initialized when a custom __init__ method 266 | is provided. 267 | 268 | Parameters 269 | ---------- 270 | ob : immutable 271 | The instance that was just initialized. 272 | 273 | Raises 274 | ------ 275 | TypeError 276 | Raised when the instance has not set values that are named in the 277 | __slots__. 278 | """ 279 | missing_slots = tuple( 280 | filter(lambda s: not hasattr(ob, s), ob.__slots__), 281 | ) 282 | if missing_slots: 283 | raise TypeError( 284 | 'not all slots initialized in __init__, missing: {0}'.format( 285 | missing_slots, 286 | ), 287 | ) 288 | 289 | 290 | def __setattr__(self, name, value): 291 | if self not in __setattr__._initializing: 292 | raise AttributeError('cannot mutate immutable object') 293 | object_setattr(self, name, value) 294 | 295 | 296 | __setattr__._initializing = set() 297 | 298 | 299 | def __repr__(self): 300 | return '{cls}({args})'.format( 301 | cls=type(self).__name__, 302 | args=', '.join(starmap( 303 | '{0}={1!r}'.format, 304 | ((s, getattr(self, s)) for s in self.__slots__), 305 | )), 306 | ) 307 | 308 | 309 | class ImmutableMeta(type): 310 | """A metaclass for creating immutable objects. 311 | """ 312 | def __new__(mcls, name, bases, dict_, *, defaults=None): 313 | if '__slots__' not in dict_: 314 | raise TypeError('immutable classes must have a __slots__') 315 | if '__setattr__' in dict_: 316 | raise TypeError('immutable classes cannot have a __setattr__') 317 | 318 | try: 319 | dict_['__init__'] = _wrapinit(dict_['__init__']) 320 | except KeyError: 321 | dict_['__init__'], dict_['__slots__'] = _create_init( 322 | name, 323 | dict_['__slots__'], 324 | defaults, 325 | ) 326 | 327 | dict_['__setattr__'] = __setattr__ 328 | cls = super().__new__(mcls, name, bases, dict_) 329 | 330 | if cls.__repr__ is object.__repr__: 331 | # Put a namedtuple-like repr on this class if there is no custom 332 | # repr on the class. 333 | cls.__repr__ = __repr__ 334 | 335 | return cls 336 | 337 | def __init__(self, *args, defaults=None): 338 | # ignore the defaults kwarg. 339 | return super().__init__(*args) 340 | 341 | 342 | class immutable(metaclass=ImmutableMeta): 343 | """A base class for immutable objects. 344 | """ 345 | __slots__ = () 346 | 347 | def to_dict(self): 348 | return {s: getattr(self, s) for s in self.__slots__} 349 | 350 | def update(self, **updates): 351 | return type(self)(**ChainMap(updates, self.to_dict())) 352 | -------------------------------------------------------------------------------- /codetransformer/utils/instance.py: -------------------------------------------------------------------------------- 1 | def instance(cls): 2 | """Decorator for creating one of instances. 3 | 4 | Parameters 5 | ---------- 6 | cls : type 7 | A class. 8 | 9 | Returns 10 | ------- 11 | instance : cls 12 | A new instance of ``cls``. 13 | """ 14 | return cls() 15 | -------------------------------------------------------------------------------- /codetransformer/utils/no_default.py: -------------------------------------------------------------------------------- 1 | @object.__new__ 2 | class no_default: 3 | def __new__(cls): 4 | return no_default 5 | 6 | def __repr__(self): 7 | return 'no_default' 8 | __str__ = __repr__ 9 | 10 | def __reduce__(self): 11 | return 'no_default' 12 | 13 | def __deepcopy__(self): 14 | return self 15 | __copy__ = __deepcopy__ 16 | -------------------------------------------------------------------------------- /codetransformer/utils/pretty.py: -------------------------------------------------------------------------------- 1 | """ 2 | codetransformer.utils.pretty 3 | ---------------------------- 4 | 5 | Utilities for pretty-printing ASTs and code objects. 6 | """ 7 | from ast import iter_fields, AST, Name, Num, parse 8 | import dis 9 | from functools import partial, singledispatch 10 | from io import StringIO 11 | from itertools import chain 12 | from operator import attrgetter 13 | import sys 14 | from types import CodeType 15 | 16 | from codetransformer.code import Flag 17 | 18 | 19 | INCLUDE_ATTRIBUTES_DEFAULT = False 20 | INDENT_DEFAULT = ' ' 21 | 22 | __all__ = [ 23 | 'a', 24 | 'd', 25 | 'display', 26 | 'pformat_ast', 27 | 'pprint_ast', 28 | ] 29 | 30 | 31 | def pformat_ast(node, 32 | include_attributes=INCLUDE_ATTRIBUTES_DEFAULT, 33 | indent=INDENT_DEFAULT): 34 | """ 35 | Pretty-format an AST tree element 36 | 37 | Parameters 38 | ---------- 39 | node : ast.AST 40 | Top-level node to render. 41 | include_attributes : bool, optional 42 | Whether to include node attributes. Default False. 43 | indent : str, optional. 44 | Indentation string for nested expressions. Default is two spaces. 45 | """ 46 | def _fmt(node, prefix, level): 47 | 48 | def with_indent(*strs): 49 | return ''.join(((indent * level,) + strs)) 50 | 51 | with_prefix = partial(with_indent, prefix) 52 | 53 | if isinstance(node, Name): 54 | # Special Case: 55 | # Render Name nodes on a single line. 56 | yield with_prefix( 57 | type(node).__name__, 58 | '(id=', 59 | repr(node.id), 60 | ', ctx=', 61 | type(node.ctx).__name__, 62 | '()),', 63 | ) 64 | 65 | elif isinstance(node, Num): 66 | # Special Case: 67 | # Render Num nodes on a single line without names. 68 | yield with_prefix( 69 | type(node).__name__, 70 | '(%r),' % node.n, 71 | ) 72 | 73 | elif isinstance(node, AST): 74 | fields_attrs = list( 75 | chain( 76 | iter_fields(node), 77 | iter_attributes(node) if include_attributes else (), 78 | ) 79 | ) 80 | if not fields_attrs: 81 | # Special Case: 82 | # Render the whole expression on one line if there are no 83 | # attributes. 84 | yield with_prefix(type(node).__name__, '(),') 85 | return 86 | 87 | yield with_prefix(type(node).__name__, '(') 88 | for name, value in fields_attrs: 89 | yield from _fmt(value, name + '=', level + 1) 90 | # Put a trailing comma if we're not at the top level. 91 | yield with_indent(')', ',' if level > 0 else '') 92 | 93 | elif isinstance(node, list): 94 | if not node: 95 | # Special Case: 96 | # Render empty lists on one line. 97 | yield with_prefix('[],') 98 | return 99 | 100 | yield with_prefix('[') 101 | yield from chain.from_iterable( 102 | map(partial(_fmt, prefix='', level=level + 1), node) 103 | ) 104 | yield with_indent('],') 105 | else: 106 | yield with_prefix(repr(node), ',') 107 | 108 | return '\n'.join(_fmt(node, prefix='', level=0)) 109 | 110 | 111 | def _extend_name(prev, parent_co): 112 | return prev + ( 113 | '..' if parent_co.co_flags & Flag.CO_NEWLOCALS else '.' 114 | ) 115 | 116 | 117 | def pprint_ast(node, 118 | include_attributes=INCLUDE_ATTRIBUTES_DEFAULT, 119 | indent=INDENT_DEFAULT, 120 | file=None): 121 | """ 122 | Pretty-print an AST tree. 123 | 124 | Parameters 125 | ---------- 126 | node : ast.AST 127 | Top-level node to render. 128 | include_attributes : bool, optional 129 | Whether to include node attributes. Default False. 130 | indent : str, optional. 131 | Indentation string for nested expressions. Default is two spaces. 132 | file : None or file-like object, optional 133 | File to use to print output. If the default of `None` is passed, we 134 | use sys.stdout. 135 | """ 136 | if file is None: 137 | file = sys.stdout 138 | 139 | print( 140 | pformat_ast( 141 | node, 142 | include_attributes=include_attributes, 143 | indent=indent 144 | ), 145 | file=file, 146 | ) 147 | 148 | 149 | def walk_code(co, _prefix=''): 150 | """ 151 | Traverse a code object, finding all consts which are also code objects. 152 | 153 | Yields pairs of (name, code object). 154 | """ 155 | name = _prefix + co.co_name 156 | yield name, co 157 | yield from chain.from_iterable( 158 | walk_code(c, _prefix=_extend_name(name, co)) 159 | for c in co.co_consts 160 | if isinstance(c, CodeType) 161 | ) 162 | 163 | 164 | def iter_attributes(node): 165 | attrs = node._attributes 166 | if not attrs: 167 | return 168 | 169 | yield from zip(attrs, attrgetter(*attrs)(node)) 170 | 171 | 172 | def a(text, mode='exec', indent=' ', file=None): 173 | """ 174 | Interactive convenience for displaying the AST of a code string. 175 | 176 | Writes a pretty-formatted AST-tree to `file`. 177 | 178 | Parameters 179 | ---------- 180 | text : str 181 | Text of Python code to render as AST. 182 | mode : {'exec', 'eval'}, optional 183 | Mode for `ast.parse`. Default is 'exec'. 184 | indent : str, optional 185 | String to use for indenting nested expressions. Default is two spaces. 186 | file : None or file-like object, optional 187 | File to use to print output. If the default of `None` is passed, we 188 | use sys.stdout. 189 | """ 190 | pprint_ast(parse(text, mode=mode), indent=indent, file=file) 191 | 192 | 193 | def d(obj, mode='exec', file=None): 194 | """ 195 | Interactive convenience for displaying the disassembly of a function, 196 | module, or code string. 197 | 198 | Compiles `text` and recursively traverses the result looking for `code` 199 | objects to render with `dis.dis`. 200 | 201 | Parameters 202 | ---------- 203 | obj : str, CodeType, or object with __code__ attribute 204 | Object to disassemble. 205 | If `obj` is an instance of CodeType, we use it unchanged. 206 | If `obj` is a string, we compile it with `mode` and then disassemble. 207 | Otherwise, we look for a `__code__` attribute on `obj`. 208 | mode : {'exec', 'eval'}, optional 209 | Mode for `compile`. Default is 'exec'. 210 | file : None or file-like object, optional 211 | File to use to print output. If the default of `None` is passed, we 212 | use sys.stdout. 213 | """ 214 | if file is None: 215 | file = sys.stdout 216 | 217 | for name, co in walk_code(extract_code(obj, compile_mode=mode)): 218 | print(name, file=file) 219 | print('-' * len(name), file=file) 220 | dis.dis(co, file=file) 221 | print('', file=file) 222 | 223 | 224 | @singledispatch 225 | def extract_code(obj, compile_mode): 226 | """ 227 | Generic function for converting objects into instances of `CodeType`. 228 | """ 229 | try: 230 | code = obj.__code__ 231 | if isinstance(code, CodeType): 232 | return code 233 | raise ValueError( 234 | "{obj} has a `__code__` attribute, " 235 | "but it's an instance of {notcode!r}, not CodeType.".format( 236 | obj=obj, 237 | notcode=type(code).__name__, 238 | ) 239 | ) 240 | except AttributeError: 241 | raise ValueError("Don't know how to extract code from %s." % obj) 242 | 243 | 244 | @extract_code.register(CodeType) 245 | def _(obj, compile_mode): 246 | return obj 247 | 248 | 249 | @extract_code.register(str) # noqa 250 | def _(obj, compile_mode): 251 | return compile(obj, '', compile_mode) 252 | 253 | 254 | _DISPLAY_TEMPLATE = """\ 255 | ==== 256 | Text 257 | ==== 258 | 259 | {text} 260 | 261 | ==================== 262 | Abstract Syntax Tree 263 | ==================== 264 | 265 | {ast} 266 | 267 | =========== 268 | Disassembly 269 | =========== 270 | 271 | {code} 272 | """ 273 | 274 | 275 | def display(text, mode='exec', file=None): 276 | """ 277 | Show `text`, rendered as AST and as Bytecode. 278 | 279 | Parameters 280 | ---------- 281 | text : str 282 | Text of Python code to render. 283 | mode : {'exec', 'eval'}, optional 284 | Mode for `ast.parse` and `compile`. Default is 'exec'. 285 | file : None or file-like object, optional 286 | File to use to print output. If the default of `None` is passed, we 287 | use sys.stdout. 288 | """ 289 | 290 | if file is None: 291 | file = sys.stdout 292 | 293 | ast_section = StringIO() 294 | a(text, mode=mode, file=ast_section) 295 | 296 | code_section = StringIO() 297 | d(text, mode=mode, file=code_section) 298 | 299 | rendered = _DISPLAY_TEMPLATE.format( 300 | text=text, 301 | ast=ast_section.getvalue(), 302 | code=code_section.getvalue(), 303 | ) 304 | print(rendered, file=file) 305 | -------------------------------------------------------------------------------- /codetransformer/utils/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llllllllll/codetransformer/c5f551e915df45adc7da7e0b1b635f0cc6a1bb27/codetransformer/utils/tests/__init__.py -------------------------------------------------------------------------------- /codetransformer/utils/tests/test_immutable.py: -------------------------------------------------------------------------------- 1 | from inspect import getfullargspec 2 | 3 | import pytest 4 | 5 | from codetransformer.utils.immutable import immutable 6 | 7 | 8 | class a(immutable): 9 | __slots__ = 'a', 10 | 11 | def spec(__self, a): 12 | pass 13 | 14 | 15 | class b(immutable): 16 | __slots__ = 'a', 'b' 17 | 18 | def spec(__self, a, b): 19 | pass 20 | 21 | 22 | class c(immutable): 23 | __slots__ = 'a', 'b', '*c' 24 | 25 | def spec(__self, a, b, *c): 26 | pass 27 | 28 | 29 | class d(immutable): 30 | __slots__ = 'a', 'b', '**c' 31 | 32 | def spec(__self, a, b, **c): 33 | pass 34 | 35 | 36 | class e(immutable): 37 | __slots__ = 'a', 'b', '*', 'c' 38 | 39 | def spec(__self, a, b, *, c): 40 | pass 41 | 42 | 43 | class f(immutable): 44 | __slots__ = 'a', 'b', '*c', 'd' 45 | 46 | def spec(__self, a, b, *c, d): 47 | pass 48 | 49 | 50 | class g(immutable, defaults={'a': 1}): 51 | __slots__ = 'a', 52 | 53 | def spec(__self, a=1): 54 | pass 55 | 56 | 57 | class h(immutable, defaults={'b': 2}): 58 | __slots__ = 'a', 'b' 59 | 60 | def spec(__self, a, b=2): 61 | pass 62 | 63 | 64 | class i(immutable, defaults={'a': 1, 'b': 2}): 65 | __slots__ = 'a', 'b' 66 | 67 | def spec(__self, a=1, b=2): 68 | pass 69 | 70 | 71 | class j(immutable, defaults={'c': 3}): 72 | __slots__ = 'a', 'b', '*', 'c' 73 | 74 | def spec(__self, a, b, *, c=3): 75 | pass 76 | 77 | 78 | @pytest.mark.parametrize('cls', (a, b, c, d, e, f, g, h, i, j)) 79 | def test_created_signature_single(cls): 80 | assert getfullargspec(cls) == getfullargspec(cls.spec) 81 | 82 | 83 | class k(immutable): 84 | __slots__ = 'a', 85 | 86 | def __init__(self, a): 87 | pass 88 | 89 | 90 | class l(immutable): 91 | __slots__ = 'a', 92 | 93 | def __init__(self, *a): 94 | pass 95 | 96 | 97 | class m(immutable): 98 | __slots__ = 'a', 99 | 100 | def __init__(self, **a): 101 | pass 102 | 103 | 104 | class n(immutable): 105 | __slots__ = 'a', 106 | 107 | def __init__(self, *, a): 108 | pass 109 | 110 | 111 | class o(immutable): 112 | __slots__ = 'a', 'b' 113 | 114 | def __init__(self, a, b=2): 115 | pass 116 | 117 | 118 | class p(immutable): 119 | __slots__ = 'a', 'b' 120 | 121 | def __init__(self, a=1, b=2): 122 | pass 123 | 124 | 125 | class q(immutable): 126 | __slots__ = 'a', 'b' 127 | 128 | def __init__(self, a, *b): 129 | pass 130 | 131 | 132 | class r(immutable): 133 | __slots__ = 'a', 'b' 134 | 135 | def __init__(self, a=1, *b): 136 | pass 137 | 138 | 139 | class s(immutable): 140 | __slots__ = 'a', 'b', 'c' 141 | 142 | def __init__(self, a=1, *b, c): 143 | pass 144 | 145 | 146 | class t(immutable): 147 | __slots__ = 'a', 'b', 'c' 148 | 149 | def __init__(self, a, *b, c=3): 150 | pass 151 | 152 | 153 | class u(immutable): 154 | __slots__ = 'a', 'b', 'c' 155 | 156 | def __init__(self, a=1, *b, c=3): 157 | pass 158 | 159 | 160 | class v(immutable): 161 | __slots__ = 'a', 'b', 'c' 162 | 163 | def __init__(self, a, **b): 164 | pass 165 | 166 | 167 | class w(immutable): 168 | __slots__ = 'a', 'b', 'c' 169 | 170 | def __init__(self, a, b, **c): 171 | pass 172 | 173 | 174 | class x(immutable): 175 | __slots__ = 'a', 'b', 'c' 176 | 177 | def __init__(self, a, *b, **c): 178 | pass 179 | 180 | 181 | class y(immutable): 182 | __slots__ = 'a', 'b', 'c', 'd' 183 | 184 | def __init__(self, a, *b, c, **d): 185 | pass 186 | 187 | 188 | class z(immutable): 189 | __slots__ = 'a', 'b', 'c', 'd' 190 | 191 | def __init__(self, a, *b, c=1, **d): 192 | pass 193 | 194 | 195 | @pytest.mark.parametrize('cls', ( 196 | k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z, 197 | )) 198 | def test_preserve_custom_init_signature(cls): 199 | assert getfullargspec(cls) == getfullargspec(cls.__init__) 200 | -------------------------------------------------------------------------------- /codetransformer/utils/tests/test_pretty.py: -------------------------------------------------------------------------------- 1 | from io import StringIO 2 | from textwrap import dedent 3 | from types import CodeType 4 | 5 | from ..pretty import a, walk_code 6 | 7 | 8 | def test_a(capsys): 9 | text = dedent( 10 | """ 11 | def inc(a): 12 | b = a + 1 13 | return b 14 | """ 15 | ) 16 | expected = dedent( 17 | """\ 18 | Module( 19 | body=[ 20 | FunctionDef( 21 | name='inc', 22 | args=arguments( 23 | args=[ 24 | arg( 25 | arg='a', 26 | annotation=None, 27 | ), 28 | ], 29 | vararg=None, 30 | kwonlyargs=[], 31 | kw_defaults=[], 32 | kwarg=None, 33 | defaults=[], 34 | ), 35 | body=[ 36 | Assign( 37 | targets=[ 38 | Name(id='b', ctx=Store()), 39 | ], 40 | value=BinOp( 41 | left=Name(id='a', ctx=Load()), 42 | op=Add(), 43 | right=Num(1), 44 | ), 45 | ), 46 | Return( 47 | value=Name(id='b', ctx=Load()), 48 | ), 49 | ], 50 | decorator_list=[], 51 | returns=None, 52 | ), 53 | ], 54 | ) 55 | """ 56 | ) 57 | 58 | a(text) 59 | stdout, stderr = capsys.readouterr() 60 | assert stdout == expected 61 | assert stderr == '' 62 | 63 | file_ = StringIO() 64 | a(text, file=file_) 65 | assert capsys.readouterr() == ('', '') 66 | 67 | result = file_.getvalue() 68 | assert result == expected 69 | 70 | 71 | def test_walk_code(): 72 | module = dedent( 73 | """\ 74 | class Foo: 75 | def bar(self): 76 | def buzz(): 77 | pass 78 | def bazz(): 79 | pass 80 | return buzz 81 | """ 82 | ) 83 | 84 | co = compile(module, '', 'exec') 85 | 86 | foo = [c for c in co.co_consts if isinstance(c, CodeType)][0] 87 | bar = [c for c in foo.co_consts if isinstance(c, CodeType)][0] 88 | buzz = [c for c in bar.co_consts 89 | if isinstance(c, CodeType) and c.co_name == 'buzz'][0] 90 | bazz = [c for c in bar.co_consts 91 | if isinstance(c, CodeType) and c.co_name == 'bazz'][0] 92 | 93 | result = list(walk_code(co)) 94 | expected = [ 95 | ('', co), 96 | ('.Foo', foo), 97 | ('.Foo.bar', bar), 98 | ('.Foo.bar..buzz', buzz), 99 | ('.Foo.bar..bazz', bazz), 100 | ] 101 | 102 | assert result == expected 103 | -------------------------------------------------------------------------------- /docs/.dir-locals.el: -------------------------------------------------------------------------------- 1 | ;; Set compile-commnd for everything in this directory to 2 | ;; "make -C html" 3 | 4 | ;; This is an association list mapping directory prefixes (in this case nil, 5 | ;; meaning "all files"), to another association list mapping dir-local variable 6 | ;; names to values. An equivalent Python structure would be something like: 7 | ;; {None: {'compile-command': "make -C .. html"}} 8 | ((nil . ((compile-command . (concat "make -C .. html"))))) 9 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " applehelp to make an Apple Help Book" 34 | @echo " devhelp to make HTML files and a Devhelp project" 35 | @echo " epub to make an epub" 36 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 37 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 38 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 39 | @echo " text to make text files" 40 | @echo " man to make manual pages" 41 | @echo " texinfo to make Texinfo files" 42 | @echo " info to make Texinfo files and run them through makeinfo" 43 | @echo " gettext to make PO message catalogs" 44 | @echo " changes to make an overview of all changed/added/deprecated items" 45 | @echo " xml to make Docutils-native XML files" 46 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 47 | @echo " linkcheck to check all external links for integrity" 48 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 49 | @echo " coverage to run coverage check of the documentation (if enabled)" 50 | 51 | clean: 52 | rm -rf $(BUILDDIR)/* 53 | 54 | html: 55 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 56 | @echo 57 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 58 | 59 | livehtml: 60 | sphinx-autobuild -p 9999 -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 61 | 62 | dirhtml: 63 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 64 | @echo 65 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 66 | 67 | singlehtml: 68 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 69 | @echo 70 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 71 | 72 | pickle: 73 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 74 | @echo 75 | @echo "Build finished; now you can process the pickle files." 76 | 77 | json: 78 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 79 | @echo 80 | @echo "Build finished; now you can process the JSON files." 81 | 82 | htmlhelp: 83 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 84 | @echo 85 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 86 | ".hhp project file in $(BUILDDIR)/htmlhelp." 87 | 88 | qthelp: 89 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 90 | @echo 91 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 92 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 93 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/codetransformer.qhcp" 94 | @echo "To view the help file:" 95 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/codetransformer.qhc" 96 | 97 | applehelp: 98 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 99 | @echo 100 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 101 | @echo "N.B. You won't be able to view it unless you put it in" \ 102 | "~/Library/Documentation/Help or install it in your application" \ 103 | "bundle." 104 | 105 | devhelp: 106 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 107 | @echo 108 | @echo "Build finished." 109 | @echo "To view the help file:" 110 | @echo "# mkdir -p $$HOME/.local/share/devhelp/codetransformer" 111 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/codetransformer" 112 | @echo "# devhelp" 113 | 114 | epub: 115 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 116 | @echo 117 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 118 | 119 | latex: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo 122 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 123 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 124 | "(use \`make latexpdf' here to do that automatically)." 125 | 126 | latexpdf: 127 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 128 | @echo "Running LaTeX files through pdflatex..." 129 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 130 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 131 | 132 | latexpdfja: 133 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 134 | @echo "Running LaTeX files through platex and dvipdfmx..." 135 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 136 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 137 | 138 | text: 139 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 140 | @echo 141 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 142 | 143 | man: 144 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 145 | @echo 146 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 147 | 148 | texinfo: 149 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 150 | @echo 151 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 152 | @echo "Run \`make' in that directory to run these through makeinfo" \ 153 | "(use \`make info' here to do that automatically)." 154 | 155 | info: 156 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 157 | @echo "Running Texinfo files through makeinfo..." 158 | make -C $(BUILDDIR)/texinfo info 159 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 160 | 161 | gettext: 162 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 163 | @echo 164 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 165 | 166 | changes: 167 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 168 | @echo 169 | @echo "The overview file is in $(BUILDDIR)/changes." 170 | 171 | linkcheck: 172 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 173 | @echo 174 | @echo "Link check complete; look for any errors in the above output " \ 175 | "or in $(BUILDDIR)/linkcheck/output.txt." 176 | 177 | doctest: 178 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 179 | @echo "Testing of doctests in the sources finished, look at the " \ 180 | "results in $(BUILDDIR)/doctest/output.txt." 181 | 182 | coverage: 183 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 184 | @echo "Testing of coverage in the sources finished, look at the " \ 185 | "results in $(BUILDDIR)/coverage/python.txt." 186 | 187 | xml: 188 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 189 | @echo 190 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 191 | 192 | pseudoxml: 193 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 194 | @echo 195 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 196 | -------------------------------------------------------------------------------- /docs/source/add2mul.py: -------------------------------------------------------------------------------- 1 | ../../codetransformer/transformers/add2mul.py -------------------------------------------------------------------------------- /docs/source/appendix.rst: -------------------------------------------------------------------------------- 1 | API Reference 2 | ============= 3 | 4 | ``codetransformer.transformers`` 5 | -------------------------------- 6 | 7 | .. automodule:: codetransformer.transformers 8 | :members: 9 | 10 | .. autodata:: islice_literals 11 | :annotation: 12 | 13 | .. data:: bytearray_literals 14 | 15 | A transformer that converts :class:`bytes` literals to :class:`bytearray`. 16 | 17 | .. data:: decimal_literals 18 | 19 | A transformer that converts :class:`float` literals to :class:`~decimal.Decimal`. 20 | 21 | ``codetransformer.code`` 22 | ------------------------ 23 | 24 | .. autoclass:: codetransformer.code.Code 25 | :members: 26 | 27 | .. autoclass:: codetransformer.code.Flag 28 | :members: 29 | :undoc-members: 30 | 31 | ``codetransformer.core`` 32 | ------------------------ 33 | 34 | .. autoclass:: codetransformer.core.CodeTransformer 35 | :members: 36 | 37 | ``codetransformer.instructions`` 38 | -------------------------------- 39 | 40 | For details on particular instructions, see `the dis stdlib module docs.`_ 41 | 42 | .. automodule:: codetransformer.instructions 43 | :members: 44 | :undoc-members: 45 | 46 | 47 | ``codetransformer.patterns`` 48 | ---------------------------- 49 | 50 | .. autoclass:: codetransformer.patterns.pattern 51 | 52 | .. autodata:: codetransformer.patterns.DEFAULT_STARTCODE 53 | 54 | DSL Objects 55 | ~~~~~~~~~~~ 56 | 57 | .. autodata:: codetransformer.patterns.matchany 58 | .. autoclass:: codetransformer.patterns.seq 59 | .. autodata:: codetransformer.patterns.var 60 | .. autodata:: codetransformer.patterns.plus 61 | .. autodata:: codetransformer.patterns.option 62 | 63 | ``codetransformer.utils`` 64 | ------------------------- 65 | 66 | .. automodule:: codetransformer.utils.pretty 67 | :members: 68 | 69 | .. automodule:: codetransformer.utils.immutable 70 | :members: immutable, lazyval, immutableattr 71 | 72 | .. automodule:: codetransformer.utils.functional 73 | :members: 74 | 75 | 76 | ``codetransformer.decompiler`` 77 | ------------------------------ 78 | 79 | .. automodule:: codetransformer.decompiler 80 | :members: decompile, pycode_to_body, DecompilationContext, DecompilationError 81 | 82 | .. _`the dis stdlib module docs.` : https://docs.python.org/3.4/library/dis.html#python-bytecode-instructions 83 | -------------------------------------------------------------------------------- /docs/source/code-objects.rst: -------------------------------------------------------------------------------- 1 | =========================== 2 | Working with Code Objects 3 | =========================== 4 | 5 | The :class:`~codetransformer.code.Code` type is the foundational abstraction in 6 | ``codetransformer``. It provides high-level APIs for working with 7 | logically-grouped sets of instructions and for converting to and from CPython's 8 | native :class:`code ` type. 9 | 10 | Constructing Code Objects 11 | ========================= 12 | 13 | The most common way constructing a Code object is to use the 14 | :meth:`~codetransformer.code.Code.from_pycode` classmethod, which accepts a 15 | CPython :class:`code ` object. 16 | 17 | There are two common ways of building raw code objects: 18 | 19 | - CPython functions have a ``__code__`` attribute, which contains the bytecode 20 | executed by the function. 21 | - The :func:`compile` builtin can compile a string of Python source code into a 22 | code object. 23 | 24 | Using :meth:`~codetransformer.code.Code.from_pycode`, we can build a Code 25 | object and inspect its contents:: 26 | 27 | >>> from codetransformer import Code 28 | >>> def add2(x): 29 | ... return x + 2 30 | ... 31 | >>> co = Code.from_pycode(add.__code__) 32 | >>> co.instrs 33 | (LOAD_FAST('x'), LOAD_CONST(2), BINARY_ADD, RETURN_VALUE) 34 | >>> co.argnames 35 | ('x',) 36 | >>> c.consts 37 | (2,) 38 | 39 | We can convert our Code object back into its raw form via the 40 | :meth:`~codetransformer.code.Code.to_pycode` method:: 41 | 42 | >>> co.to_pycode() 43 | ", line 1> 44 | 45 | Building Transformers 46 | ===================== 47 | 48 | Once we have the ability to convert to and from an abstract code 49 | representation, we gain the ability to perform transformations on that abtract 50 | representation. 51 | 52 | Let's say that we want to replace the addition operation in our ``add2`` 53 | function with a multiplication. We could try to mutate our 54 | :class:`~codetransformer.code.Code` object directly before converting back to 55 | Python bytecode, but there are many subtle invariants [#f1]_ between the 56 | instructions and the other pieces of metadata that must be maintained to ensure 57 | that the generated output can be executed correctly. 58 | 59 | Rather than encourage users to mutate Code objects in place, 60 | ``codetransformer`` provides the :class:`~codetransformer.core.CodeTransformer` 61 | class, which allows users to declaratively describe operations to perform on 62 | sequences of instructions. 63 | 64 | Implemented as a :class:`~codetransformer.core.CodeTransformer`, our "replace 65 | additions with multiplications" operation looks like this: 66 | 67 | .. literalinclude:: add2mul.py 68 | :language: python 69 | :lines: 10- 70 | 71 | The important piece here is the ``_add2mul`` method, which has been decorated 72 | with a :class:`~codetransformer.patterns.pattern`. Patterns provide an API for 73 | describing sequences of instructions to match against for replacement and/or 74 | modification. The :class:`~codetransformer.core.CodeTransformer` base class 75 | looks at methods with registered patterns and compares them against the 76 | instructions of the Code object under transformation. For each matching 77 | sequence of instructions, the decorated method is called with all matching 78 | instructions \*-unpacked into the method. The method's job is to take the 79 | input instructions and return an iterable of new instructions to serve as 80 | replacements. It is often convenient to implement transformer methods as 81 | `generator functions`_, as we've done here. 82 | 83 | In this example, we've supplied the simplest possible pattern: a single 84 | instruction type to match. [#f2]_ Our transformer method will be called on 85 | every ``BINARY_ADD`` instruction in the target code object, and it will yield a 86 | ``BINARY_MULTIPLY`` as replacement each time. 87 | 88 | Applying Transformers 89 | ===================== 90 | 91 | To apply a :class:`~codetransformer.core.CodeTransformer` to a function, we 92 | construct an instance of the transformer and call it on the function we want to 93 | modify. The result is a new function whose instructions have been rewritten 94 | applying our transformer's methods to matched sequences of the input function's 95 | instructions. The original function is not mutated in place. 96 | 97 | **Example:** 98 | 99 | .. code-block:: python 100 | 101 | >>> transformer = add2mul() 102 | >>> mul2 = transformer(add2) # mult2 is a brand-new function 103 | >>> mul2(5) 104 | 10 105 | 106 | When we don't care about having access to the pre-transformed version of a 107 | function, it's convenient and idiomatic to apply transformers as decorators:: 108 | 109 | >>> @add2mul() 110 | ... def mul2(x): 111 | ... return x + 2 112 | ... 113 | >>> mul2(5) 114 | 10 115 | 116 | .. [#f1] For example, if we add a new constant, we have to ensure that we 117 | correctly maintain the indices of existing constants in the generated 118 | code's ``co_consts``, and if we replace an instruction that was the 119 | target of a jump, we have to make sure that the jump instruction 120 | resolves correctly to our new instruction. 121 | 122 | .. [#f2] Many more complex patterns are possible. See the docs for 123 | :class:`codetransformer.patterns.pattern` for more examples. 124 | .. _`generator functions` : https://docs.python.org/2/tutorial/classes.html#generators 125 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # codetransformer documentation build configuration file, created by 5 | # sphinx-quickstart on Sat Sep 5 21:06:06 2015. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | import sys 17 | import os 18 | import shlex 19 | 20 | # If extensions (or modules to document with autodoc) are in another directory, 21 | # add these directories to sys.path here. If the directory is relative to the 22 | # documentation root, use os.path.abspath to make it absolute, like shown here. 23 | # sys.path.insert(0, os.path.abspath('..')) 24 | 25 | # -- General configuration ------------------------------------------------ 26 | 27 | # If your documentation needs a minimal Sphinx version, state it here. 28 | #needs_sphinx = '1.0' 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be 31 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 32 | # ones. 33 | extensions = [ 34 | 'sphinx.ext.autodoc', 35 | 'sphinx.ext.autosummary', 36 | 'sphinx.ext.doctest', 37 | 'sphinx.ext.intersphinx', 38 | 'sphinx.ext.todo', 39 | 'sphinx.ext.coverage', 40 | 'sphinx.ext.viewcode', 41 | 'sphinx.ext.napoleon', 42 | ] 43 | 44 | # Add any paths that contain templates here, relative to this directory. 45 | templates_path = ['_templates'] 46 | 47 | # The suffix(es) of source filenames. 48 | # You can specify multiple suffix as a list of string: 49 | # source_suffix = ['.rst', '.md'] 50 | source_suffix = '.rst' 51 | 52 | # The encoding of source files. 53 | #source_encoding = 'utf-8-sig' 54 | 55 | # The master toctree document. 56 | master_doc = 'index' 57 | 58 | # General information about the project. 59 | project = 'codetransformer' 60 | copyright = '2016, Joe Jevnik and Scott Sanderson' 61 | author = 'Joe Jevnik and Scott Sanderson' 62 | 63 | # The version info for the project you're documenting, acts as replacement for 64 | # |version| and |release|, also used in various other places throughout the 65 | # built documents. 66 | # 67 | # The short X.Y version. 68 | version = '0.6.0' 69 | # The full version, including alpha/beta/rc tags. 70 | release = '0.6.0' 71 | 72 | # The language for content autogenerated by Sphinx. Refer to documentation 73 | # for a list of supported languages. 74 | # 75 | # This is also used if you do content translation via gettext catalogs. 76 | # Usually you set "language" from the command line for these cases. 77 | language = None 78 | 79 | # There are two options for replacing |today|: either, you set today to some 80 | # non-false value, then it is used: 81 | #today = '' 82 | # Else, today_fmt is used as the format for a strftime call. 83 | #today_fmt = '%B %d, %Y' 84 | 85 | # List of patterns, relative to source directory, that match files and 86 | # directories to ignore when looking for source files. 87 | exclude_patterns = [] 88 | 89 | # The reST default role (used for this markup: `text`) to use for all 90 | # documents. 91 | #default_role = None 92 | 93 | # If true, '()' will be appended to :func: etc. cross-reference text. 94 | #add_function_parentheses = True 95 | 96 | # If true, the current module name will be prepended to all description 97 | # unit titles (such as .. function::). 98 | #add_module_names = True 99 | 100 | # If true, sectionauthor and moduleauthor directives will be shown in the 101 | # output. They are ignored by default. 102 | #show_authors = False 103 | 104 | # The name of the Pygments (syntax highlighting) style to use. 105 | pygments_style = 'sphinx' 106 | 107 | # A list of ignored prefixes for module index sorting. 108 | #modindex_common_prefix = [] 109 | 110 | # If true, keep warnings as "system message" paragraphs in the built documents. 111 | #keep_warnings = False 112 | 113 | # If true, `todo` and `todoList` produce output, else they produce nothing. 114 | todo_include_todos = True 115 | 116 | 117 | # -- Options for HTML output ---------------------------------------------- 118 | 119 | # The theme to use for HTML and HTML Help pages. See the documentation for 120 | # a list of builtin themes. 121 | html_theme = 'sphinx_rtd_theme' 122 | 123 | # Theme options are theme-specific and customize the look and feel of a theme 124 | # further. For a list of options available for each theme, see the 125 | # documentation. 126 | #html_theme_options = {} 127 | 128 | # Add any paths that contain custom themes here, relative to this directory. 129 | #html_theme_path = [] 130 | 131 | # The name for this set of Sphinx documents. If None, it defaults to 132 | # " v documentation". 133 | #html_title = None 134 | 135 | # A shorter title for the navigation bar. Default is the same as html_title. 136 | #html_short_title = None 137 | 138 | # The name of an image file (relative to this directory) to place at the top 139 | # of the sidebar. 140 | #html_logo = None 141 | 142 | # The name of an image file (within the static path) to use as favicon of the 143 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 144 | # pixels large. 145 | #html_favicon = None 146 | 147 | # Add any paths that contain custom static files (such as style sheets) here, 148 | # relative to this directory. They are copied after the builtin static files, 149 | # so a file named "default.css" will overwrite the builtin "default.css". 150 | html_static_path = ['_static'] 151 | 152 | # Add any extra paths that contain custom files (such as robots.txt or 153 | # .htaccess) here, relative to this directory. These files are copied 154 | # directly to the root of the documentation. 155 | #html_extra_path = [] 156 | 157 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 158 | # using the given strftime format. 159 | #html_last_updated_fmt = '%b %d, %Y' 160 | 161 | # If true, SmartyPants will be used to convert quotes and dashes to 162 | # typographically correct entities. 163 | #html_use_smartypants = True 164 | 165 | # Custom sidebar templates, maps document names to template names. 166 | #html_sidebars = {} 167 | 168 | # Additional templates that should be rendered to pages, maps page names to 169 | # template names. 170 | #html_additional_pages = {} 171 | 172 | # If false, no module index is generated. 173 | #html_domain_indices = True 174 | 175 | # If false, no index is generated. 176 | #html_use_index = True 177 | 178 | # If true, the index is split into individual pages for each letter. 179 | #html_split_index = False 180 | 181 | # If true, links to the reST sources are added to the pages. 182 | #html_show_sourcelink = True 183 | 184 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 185 | #html_show_sphinx = True 186 | 187 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 188 | #html_show_copyright = True 189 | 190 | # If true, an OpenSearch description file will be output, and all pages will 191 | # contain a tag referring to it. The value of this option must be the 192 | # base URL from which the finished HTML is served. 193 | #html_use_opensearch = '' 194 | 195 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 196 | #html_file_suffix = None 197 | 198 | # Language to be used for generating the HTML full-text search index. 199 | # Sphinx supports the following languages: 200 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' 201 | # 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr' 202 | #html_search_language = 'en' 203 | 204 | # A dictionary with options for the search language support, empty by default. 205 | # Now only 'ja' uses this config value 206 | #html_search_options = {'type': 'default'} 207 | 208 | # The name of a javascript file (relative to the configuration directory) that 209 | # implements a search results scorer. If empty, the default will be used. 210 | #html_search_scorer = 'scorer.js' 211 | 212 | # Output file base name for HTML help builder. 213 | htmlhelp_basename = 'codetransformerdoc' 214 | 215 | # -- Options for LaTeX output --------------------------------------------- 216 | 217 | latex_elements = { 218 | # The paper size ('letterpaper' or 'a4paper'). 219 | #'papersize': 'letterpaper', 220 | 221 | # The font size ('10pt', '11pt' or '12pt'). 222 | #'pointsize': '10pt', 223 | 224 | # Additional stuff for the LaTeX preamble. 225 | #'preamble': '', 226 | 227 | # Latex figure (float) alignment 228 | #'figure_align': 'htbp', 229 | } 230 | 231 | # Grouping the document tree into LaTeX files. List of tuples 232 | # (source start file, target name, title, 233 | # author, documentclass [howto, manual, or own class]). 234 | latex_documents = [ 235 | (master_doc, 'codetransformer.tex', 'codetransformer Documentation', 236 | 'Joe Jevnik and Scott Sanderson', 'manual'), 237 | ] 238 | 239 | # The name of an image file (relative to this directory) to place at the top of 240 | # the title page. 241 | #latex_logo = None 242 | 243 | # For "manual" documents, if this is true, then toplevel headings are parts, 244 | # not chapters. 245 | #latex_use_parts = False 246 | 247 | # If true, show page references after internal links. 248 | #latex_show_pagerefs = False 249 | 250 | # If true, show URL addresses after external links. 251 | #latex_show_urls = False 252 | 253 | # Documents to append as an appendix to all manuals. 254 | #latex_appendices = [] 255 | 256 | # If false, no module index is generated. 257 | #latex_domain_indices = True 258 | 259 | 260 | # -- Options for manual page output --------------------------------------- 261 | 262 | # One entry per manual page. List of tuples 263 | # (source start file, name, description, authors, manual section). 264 | man_pages = [ 265 | (master_doc, 'codetransformer', 'codetransformer Documentation', 266 | [author], 1) 267 | ] 268 | 269 | # If true, show URL addresses after external links. 270 | #man_show_urls = False 271 | 272 | 273 | # -- Options for Texinfo output ------------------------------------------- 274 | 275 | # Grouping the document tree into Texinfo files. List of tuples 276 | # (source start file, target name, title, author, 277 | # dir menu entry, description, category) 278 | texinfo_documents = [ 279 | (master_doc, 'codetransformer', 'codetransformer Documentation', 280 | author, 'codetransformer', 'One line description of project.', 281 | 'Miscellaneous'), 282 | ] 283 | 284 | # Documents to append as an appendix to all manuals. 285 | #texinfo_appendices = [] 286 | 287 | # If false, no module index is generated. 288 | #texinfo_domain_indices = True 289 | 290 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 291 | #texinfo_show_urls = 'footnote' 292 | 293 | # If true, do not generate a @detailmenu in the "Top" node's menu. 294 | #texinfo_no_detailmenu = False 295 | 296 | 297 | # Example configuration for intersphinx: refer to the Python standard library. 298 | intersphinx_mapping = {'https://docs.python.org/3/': None} 299 | 300 | # This makes a big difference for Code's many attributes. 301 | napoleon_use_ivar = True 302 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | codetransformer 2 | =============== 3 | 4 | Bytecode transformers for CPython inspired by the ``ast`` module's 5 | ``NodeTransformer``. 6 | 7 | ``codetransformer`` is a library that provides utilities for working with 8 | CPython bytecode at runtime. Among other things, it provides: 9 | 10 | - A :class:`~codetransformer.code.Code` type for representing and manipulating 11 | Python bytecode. 12 | - An :class:`~codetransformer.instructions.Instruction` type, with 13 | :class:`subclasses ` for each opcode 14 | used by the CPython interpreter. 15 | - A :class:`~codetransformer.core.CodeTransformer` type providing a 16 | pattern-based API for describing transformations on 17 | :class:`~codetransformer.code.Code` objects. Example transformers can be 18 | found in :mod:`codetransformer.transformers`. 19 | - An experimental :mod:`decompiler ` for 20 | determining the AST tree that would generate a code object. 21 | 22 | The existence of ``codetransformer`` is motivated by the desire to override 23 | parts of the python language that cannot be easily hooked via more standard 24 | means. Examples of program transformations made possible using code 25 | transformers include: 26 | 27 | * Overriding the ``is`` and ``not`` operators. 28 | * `Overloading Python's data structure literals`_. 29 | * `Optimizing functions by freezing globals as constants`_. 30 | * `Exception handlers that match on exception instances`_. 31 | 32 | Contents: 33 | 34 | .. toctree:: 35 | :maxdepth: 2 36 | 37 | code-objects.rst 38 | patterns.rst 39 | magics.rst 40 | appendix.rst 41 | 42 | 43 | Indices and tables 44 | ------------------ 45 | 46 | * :ref:`genindex` 47 | * :ref:`modindex` 48 | * :ref:`search` 49 | 50 | .. _lazy: https://github.com/llllllllll/lazy_python 51 | .. _Overloading Python's data structure literals: appendix.html\#codetransformer.transformers.literals.overloaded_dicts 52 | .. _Optimizing functions by freezing globals as constants: appendix.html#codetransformer.transformers.asconstants 53 | .. _Exception handlers that match on exception instances: appendix.html#codetransformer.transformers.exc_patterns.pattern_matched_exceptions 54 | -------------------------------------------------------------------------------- /docs/source/magics.rst: -------------------------------------------------------------------------------- 1 | Interactive Conveniences 2 | ======================== 3 | 4 | When developing projects using :mod:`codetransformer`, it's often helpful to be 5 | able to quickly and easily visualize the AST and/or disassembly generated by 6 | CPython for a given source text. 7 | 8 | The :mod:`codetransformer.utils.pretty` module provides utilities for viewing 9 | AST trees and the disassembly of nested code objects: 10 | 11 | .. autosummary:: 12 | 13 | ~codetransformer.utils.pretty.a 14 | ~codetransformer.utils.pretty.d 15 | ~codetransformer.utils.pretty.display 16 | ~codetransformer.utils.pretty.extract_code 17 | 18 | For users of `IPython`_, :mod:`codetransformer` provides an IPython extension 19 | that adds ``%%ast`` and ``%%dis`` magics. 20 | 21 | .. code-block:: python 22 | 23 | In [1]: %load_ext codetransformer 24 | In [2]: %%dis 25 | ...: def foo(a, b): 26 | ...: return a + b 27 | ...: 28 | 29 | -------- 30 | 1 0 LOAD_CONST 0 (", line 1>) 31 | 3 LOAD_CONST 1 ('foo') 32 | 6 MAKE_FUNCTION 0 33 | 9 STORE_NAME 0 (foo) 34 | 12 LOAD_CONST 2 (None) 35 | 15 RETURN_VAL 36 | 37 | .foo 38 | ------------ 39 | 2 0 LOAD_FAST 0 (a) 40 | 3 LOAD_FAST 1 (b) 41 | 6 BINARY_ADD 42 | 7 RETURN_VAL 43 | 44 | 45 | In [3]: %%ast 46 | ...: def foo(a, b): 47 | ...: return a + b 48 | ...: 49 | Module( 50 | body=[ 51 | FunctionDef( 52 | name='foo', 53 | args=arguments( 54 | args=[ 55 | arg( 56 | arg='a', 57 | annotation=None, 58 | ), 59 | arg( 60 | arg='b', 61 | annotation=None, 62 | ), 63 | ], 64 | vararg=None, 65 | kwonlyargs=[], 66 | kw_defaults=[], 67 | kwarg=None, 68 | defaults=[], 69 | ), 70 | body=[ 71 | Return( 72 | value=BinOp( 73 | left=Name(id='a', ctx=Load()), 74 | op=Add(), 75 | right=Name(id='b', ctx=Load()), 76 | ), 77 | ), 78 | ], 79 | decorator_list=[], 80 | returns=None, 81 | ), 82 | ], 83 | ) 84 | 85 | .. _`IPython` : https://ipython.readthedocs.org/en/stable/ 86 | -------------------------------------------------------------------------------- /docs/source/patterns.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Pattern API 3 | ============ 4 | 5 | Most bytecode transformations are best expressed by identifying a pattern in the 6 | bytecode and emitting some replacement. ``codetransformer`` makes it easy to 7 | express and work on these patterns by defining a small dsl for use in 8 | :class:`~codetransformer.core.CodeTransformer` classes. 9 | 10 | Matchables 11 | ========== 12 | 13 | A pattern is expressed by a sequence of matchables paired with the startcode. A 14 | matchable is anything that we can compare a sequence of bytecode to. 15 | 16 | Instructions 17 | ------------ 18 | 19 | The most atomic matchable is any 20 | :class:`~codetransformer.instructions.Instruction` class. These classes each can 21 | be used to define a pattern that matches instances of that instruction. For 22 | example, the pattern:: 23 | 24 | LOAD_CONST 25 | 26 | will match a single :class:`~codetransformer.instructions.LOAD_CONST` instance. 27 | 28 | All matchables support the following operations: 29 | 30 | ``or`` 31 | ------ 32 | 33 | Matchables can be or'd together to create a new matchable that matches either 34 | the lhs or the rhs. For example:: 35 | 36 | LOAD_CONST | LOAD_FAST 37 | 38 | will match a either a single :class:`~codetransformer.instructions.LOAD_CONST` 39 | or a :class:`~codetransformer.instructions.LOAD_FAST`. 40 | 41 | ``not`` 42 | ------- 43 | 44 | Matchables may be negated to create a new matchable that matches anything the 45 | original did not match. For example:: 46 | 47 | ~LOAD_CONST 48 | 49 | will match any instruction except an instance of 50 | :class:`~codetransformer.instructions.LOAD_CONST`. 51 | 52 | ``matchrange`` 53 | -------------- 54 | 55 | It is possible to create a matchable from another such that it matches the same 56 | pattern repeated multiple times. For example:: 57 | 58 | LOAD_CONST[3] 59 | 60 | will match exactly three :class:`~codetransformer.instructions.LOAD_CONST` 61 | instances in a row. This will not match on any less than three and will match on 62 | the first three if there are more than three 63 | :class:`~codetransformer.instructions.LOAD_CONST` instructions in a row. 64 | 65 | This can be specified with an upper bound also like:: 66 | 67 | LOAD_CONST[3, 5] 68 | 69 | This matches between three and five 70 | :class:`~codetransformer.instructions.LOAD_CONST` instructions. This is greedy 71 | meaning that if four or five :class:`~codetransformer.instructions.LOAD_CONST` 72 | instructions exist it will consume as many as possible up to five. 73 | 74 | ``var`` 75 | ------- 76 | 77 | :data:`~codetransformer.patterns.var` is a modifier that matches zero or more 78 | instances of another matchable. For example:: 79 | 80 | LOAD_CONST[var] 81 | 82 | will match as many :class:`~codetransformer.instructions.LOAD_CONST` 83 | instructions appear in a row or an empty instruction set. 84 | 85 | ``plus`` 86 | -------- 87 | 88 | :data:`~codetransformer.patterns.plus` is a modifier that matches one or more 89 | instances of another matchable. For example:: 90 | 91 | LOAD_CONST[plus] 92 | 93 | will match as many :class:`~codetransformer.instructions.LOAD_CONST` 94 | instructions appear in a row as long as there is at least one. 95 | 96 | ``option`` 97 | ---------- 98 | 99 | :data:`~codetransformer.patterns.option` is a modifier that matches zero or one 100 | instance of another matchable. For example:: 101 | 102 | LOAD_CONST[option] 103 | 104 | will match either an empty instruction set or exactly one 105 | :class:`~codetransformer.instructions.LOAD_CONST`. 106 | 107 | ``matchany`` 108 | ------------ 109 | 110 | :data:`~codetransformer.patterns.matchany` is a special matchable that matches 111 | any single instruction. ``...`` is an alias for 112 | :data:`~codetransformer.patterns.matchany`. 113 | 114 | ``seq`` 115 | ------- 116 | 117 | :class:`~codetransformer.patterns.seq` is a matchable that matches a sequence of 118 | other matchables. For example:: 119 | 120 | seq(LOAD_CONST, ..., ~LOAD_CONST) 121 | 122 | will match a single :class:`~codetransformer.instructions.LOAD_CONST` followed 123 | by any instruction followed by any instruction that is not a 124 | :class:`~codetransformer.instructions.LOAD_CONST`. This example show how we can 125 | compose all of our matchable together to build more complex matchables. 126 | 127 | ``pattern`` 128 | =========== 129 | 130 | In order to use our DSL we need a way to register transformations to these 131 | matchables. To do this we may decorate methods of a 132 | :class:`~codetransformer.core.CodeTransformer` with 133 | :class:`~codetransformer.patterns.pattern`. This registers the function to the 134 | pattern. For example:: 135 | 136 | class MyTransformer(CodeTransformer): 137 | @pattern(LOAD_CONST, ..., ~LOAD_CONST) 138 | def _f(self, load_const, any, not_load_const): 139 | ... 140 | 141 | The argument list of a :class:`~codetransformer.patterns.pattern` is implicitly 142 | made into a `seq`_. When using ``MyTransformer`` to transform some bytecode 143 | ``_f`` will be called only when we see a 144 | :class:`~codetransformer.instructions.LOAD_CONST` followed by any instruction 145 | followed by any instruction that is not a 146 | :class:`~codetransformer.instructions.LOAD_CONST`. This function will be passed 147 | these three instruction objects positionally and should yield the instructions 148 | to replace them with. 149 | 150 | Resolution Order 151 | ---------------- 152 | 153 | Patterns are checked in the order they are defined in the class body. This is 154 | because some patterns may overlap with eachother. For example, given the two 155 | classes:: 156 | 157 | class OrderOne(CodeTransformer): 158 | @pattern(LOAD_CONST) 159 | def _load_const(self, instr): 160 | print('LOAD_CONST') 161 | yield instr 162 | 163 | @pattern(...) 164 | def _any(self, instr): 165 | print('...') 166 | yield instr 167 | 168 | 169 | class OrderTwo(CodeTransformer): 170 | @pattern(...) 171 | def _any(self, instr): 172 | print('...') 173 | yield instr 174 | 175 | @pattern(LOAD_CONST) 176 | def _load_const(self, instr): 177 | print('LOAD_CONST') 178 | yield instr 179 | 180 | 181 | 182 | 183 | and the following bytecode sequence:: 184 | 185 | LOAD_CONST POP_TOP LOAD_CONST RETURN_VALUE 186 | 187 | When running with ``OrderOne`` we would see:: 188 | 189 | 190 | LOAD_CONST 191 | ... 192 | LOAD_CONST 193 | ... 194 | 195 | but when running with ``OrderTwo``:: 196 | 197 | ... 198 | ... 199 | ... 200 | ... 201 | 202 | This is because we will always match on the ``...`` pattern where ``OrderOne`` 203 | will check against :class:`~codetransformer.instructions.LOAD_CONST` before 204 | falling back to the :data:`~codetransformer.instructions.matchany`. 205 | 206 | Contextual Patterns 207 | ------------------- 208 | 209 | Sometimes a pattern should only be matched given that some condition has been 210 | met. An example of this is that you want to modify comprehensions. In order to 211 | be sure that you are only modifying the bodies of the comprehensions we must 212 | only match when we know we are in 213 | one. :class:`~codetransformer.patterns.pattern` accepts a keyword only argument 214 | ``startcodes`` which is a set of contexts where this pattern should apply. By 215 | default this is :data:`~codetransformer.patterns.DEFAULT_STARTCODE` which is the 216 | default state. A startcode may be anything hashable; however it is best to use 217 | strings or integer constants to make it easy to debug. 218 | 219 | The :meth:`~codetransformer.core.CodeTransformer.begin` method enters a new 220 | startcode. For example:: 221 | 222 | class FindDictComprehensions(CodeTransformer): 223 | @pattern(BUILD_MAP, matchany[var], MAP_ADD) 224 | def _start_comprehension(self, *instrs): 225 | print('starting dict comprehension') 226 | self.begin('in_comprehension') 227 | yield from instrs 228 | 229 | @pattern(RETURN_VALUE, startcodes=('in_comprehension',)) 230 | def _return_from_comprehension(self, instr): 231 | print('returning from comprehension') 232 | yield instr 233 | 234 | @pattern(RETURN_VALUE) 235 | def _return_default(self, instr): 236 | print('returning from non-comprehension') 237 | yield instr 238 | 239 | 240 | This transformer will find dictionary comprehensions and enter a new 241 | startcode. Inside this startcode we will handle 242 | :class:`~codetransformer.instructions.RETURN_VALUE` instructions differently. 243 | 244 | .. code-block:: python 245 | 246 | >>> @FindDictComprehensions() 247 | ... def f(): 248 | ... pass 249 | ... 250 | returning from non-comprehension 251 | 252 | >>> @FindDictComprehensions() 253 | ... def g(): 254 | ... {a: b for a, b in it} 255 | ... 256 | starting dict comprehension 257 | returning from comprehension 258 | returning from non-comprehension 259 | 260 | 261 | It is important to remember that when we recurse into a nested code object (like 262 | a comprehension) that we do not inherit the startcode from our parent. Instead 263 | it always starts at :data:`~codetransformer.patterns.DEFAULT_STARTCODE`. 264 | -------------------------------------------------------------------------------- /requirements_doc.txt: -------------------------------------------------------------------------------- 1 | Sphinx==1.3.5 2 | sphinx-rtd-theme==0.1.9 3 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | # See the docstring in versioneer.py for instructions. Note that you must 2 | # re-run 'versioneer.py setup' after changing this section, and commit the 3 | # resulting files. 4 | [versioneer] 5 | VCS=git 6 | style=pep440 7 | versionfile_source=codetransformer/_version.py 8 | versionfile_build=codetransformer/_version.py 9 | tag_prefix= 10 | parentdir_prefix=codetransformer- 11 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from setuptools import setup, find_packages 3 | import sys 4 | 5 | import versioneer 6 | 7 | long_description = '' 8 | 9 | if 'upload' in sys.argv: 10 | with open('README.rst') as f: 11 | long_description = f.read() 12 | 13 | 14 | setup( 15 | name='codetransformer', 16 | version=versioneer.get_version(), 17 | cmdclass=versioneer.get_cmdclass(), 18 | description='Python code object transformers', 19 | author='Joe Jevnik and Scott Sanderson', 20 | author_email='joejev@gmail.com', 21 | packages=find_packages(), 22 | long_description=long_description, 23 | license='GPL-2', 24 | classifiers=[ 25 | 'Development Status :: 3 - Alpha', 26 | 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)', 27 | 'Natural Language :: English', 28 | 'Programming Language :: Python :: 3.4', 29 | 'Programming Language :: Python :: 3.5', 30 | 'Programming Language :: Python :: 3 :: Only', 31 | 'Programming Language :: Python :: Implementation :: CPython', 32 | 'Operating System :: POSIX', 33 | 'Topic :: Software Development :: Pre-processors', 34 | ], 35 | url='https://github.com/llllllllll/codetransformer', 36 | install_requires=['toolz'], 37 | extras_require={ 38 | 'dev': [ 39 | 'flake8==3.3.0', 40 | 'pytest==2.8.4', 41 | 'pytest-cov==2.2.1', 42 | ], 43 | }, 44 | ) 45 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist=py{34,35,36} 3 | skip_missing_interpreters=True 4 | 5 | [testenv] 6 | commands= 7 | pip install -e .[dev] 8 | py.test 9 | 10 | [pytest] 11 | addopts = --doctest-modules --cov codetransformer --cov-report term-missing --ignore setup.py 12 | testpaths = codetransformer 13 | norecursedirs = decompiler 14 | --------------------------------------------------------------------------------