├── .gitignore ├── .travis.yml ├── COPYING ├── MANIFEST.in ├── README.rst ├── benchmarks ├── bench_copy_builtin_to_cst.py ├── bench_copy_global_to_cst.py ├── bench_guards.py ├── bench_len_abc.py ├── bench_list_append.py ├── bench_optimizer.py └── bench_posixpath.py ├── doc ├── Makefile ├── benchmarks.rst ├── changelog.rst ├── conf.py ├── fat.rst ├── fatoptimizer.rst ├── gsoc.rst ├── index.rst ├── make.bat ├── microbenchmarks.rst ├── misc.rst ├── optimizations.rst ├── semantics.rst └── todo.rst ├── fatoptimizer ├── __init__.py ├── base_optimizer.py ├── benchmark.py ├── bltin_const.py ├── builtins.py ├── call_method.py ├── call_pure.py ├── config.py ├── const_fold.py ├── const_propagate.py ├── convert_const.py ├── copy_bltin_to_const.py ├── dead_code.py ├── inline.py ├── iterable.py ├── methods.py ├── namespace.py ├── optimizer.py ├── pure.py ├── specialized.py ├── tools.py └── unroll.py ├── setup.py ├── test_fat_config.py ├── test_fat_site.py ├── test_fatoptimizer.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | *.swp 3 | MANIFEST 4 | build 5 | dist 6 | 7 | # generated by tox 8 | .tox/ 9 | fatoptimizer.egg-info/ 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | env: 3 | - TOXENV=py35 4 | - TOXENV=doc 5 | - TOXENV=pep8 6 | install: pip install -U tox 7 | script: tox 8 | notifications: 9 | email: 10 | - victor.stinner@gmail.com 11 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright (c) 2015 Red Hat. 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a 5 | copy of this software and associated documentation files (the 6 | "Software"), to deal in the Software without restriction, including 7 | without limitation the rights to use, copy, modify, merge, publish, 8 | distribute, sublicense, and/or sell copies of the Software, and to 9 | permit persons to whom the Software is furnished to do so, subject to 10 | the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included 13 | in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 16 | OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 18 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 19 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 20 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 21 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include COPYING 2 | include MANIFEST.in 3 | include README.rst 4 | 5 | include benchmarks/*.py 6 | 7 | include doc/conf.py doc/make.bat doc/Makefile 8 | include doc/*.rst 9 | 10 | include tox.ini 11 | include test_fat_config.py 12 | include test_fat_site.py 13 | include test_fatoptimizer.py 14 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ************ 2 | fatoptimizer 3 | ************ 4 | 5 | .. image:: https://travis-ci.org/vstinner/fatoptimizer.svg?branch=master 6 | :alt: Build status of fatoptimizer on Travis CI 7 | :target: https://travis-ci.org/vstinner/fatoptimizer 8 | 9 | .. image:: http://unmaintained.tech/badge.svg 10 | :target: http://unmaintained.tech/ 11 | :alt: No Maintenance Intended 12 | 13 | ``fatoptimizer`` is a static optimizer for Python 3.6 using function 14 | specialization with guards. It is implemented as an AST optimizer. 15 | 16 | Optimized code requires the ``fat`` module at runtime if at least one 17 | function was specialized. 18 | 19 | * `fatoptimizer documentation 20 | `_ 21 | * `fatoptimizer project at GitHub 22 | `_ (code, bug tracker) 23 | * `fat module `_ 24 | * `FAT Python 25 | `_ 26 | * `fatoptimizer tests running on the Travis-CI 27 | `_ 28 | -------------------------------------------------------------------------------- /benchmarks/bench_copy_builtin_to_cst.py: -------------------------------------------------------------------------------- 1 | """ 2 | Microbenchmark on "copy builtin function to constant". 3 | 4 | The benchmark doesn't use fatoptimize, but specialize explicitly the function. 5 | """ 6 | import fat 7 | import sys 8 | 9 | def func(obj): 10 | return len(obj) 11 | 12 | if fat.get_specialized(func): 13 | print("ERROR: func() was already specialized") 14 | sys.exit(1) 15 | 16 | def func_cst(obj): 17 | return 'LEN'(obj) 18 | func_cst.__code__ = fat.replace_consts(func_cst.__code__, {'LEN': len}) 19 | 20 | def run_benchmark(bench): 21 | bench.timeit(stmt='func("abc")', 22 | globals=globals(), 23 | name='original bytecode (LOAD_GLOBAL)') 24 | 25 | bench.timeit(stmt='func_cst("abc")', 26 | globals=globals(), 27 | name='LOAD_CONST') 28 | 29 | fat.specialize(func, func_cst, [fat.GuardBuiltins(('len',))]) 30 | assert fat.get_specialized(func) 31 | 32 | bench.timeit(stmt='func("abc")', 33 | globals=globals(), 34 | name='LOAD_CONST with guard on builtins and globals') 35 | 36 | -------------------------------------------------------------------------------- /benchmarks/bench_copy_global_to_cst.py: -------------------------------------------------------------------------------- 1 | """ 2 | Microbenchmark on "copy global function to constant". 3 | 4 | The benchmark doesn't use fatoptimize, but specialize explicitly the function. 5 | """ 6 | import fat 7 | import sys 8 | 9 | mylen = len 10 | 11 | def func(obj): 12 | return mylen(obj) 13 | 14 | if fat.get_specialized(func): 15 | print("ERROR: func() was already specialized") 16 | sys.exit(1) 17 | 18 | def func_cst(obj): 19 | return 'MYLEN'(obj) 20 | func_cst.__code__ = fat.replace_consts(func_cst.__code__, {'MYLEN': mylen}) 21 | 22 | def run_benchmark(bench): 23 | bench.timeit(stmt='func("abc")', 24 | globals=globals(), 25 | name='original bytecode (LOAD_GLOBAL)') 26 | 27 | bench.timeit(stmt='func_cst("abc")', 28 | globals=globals(), 29 | name='LOAD_CONST') 30 | 31 | fat.specialize(func, func_cst, [fat.GuardGlobals(('mylen',))]) 32 | assert fat.get_specialized(func) 33 | 34 | bench.timeit(stmt='func("abc")', 35 | globals=globals(), 36 | name='LOAD_CONST with guard on globals') 37 | 38 | -------------------------------------------------------------------------------- /benchmarks/bench_guards.py: -------------------------------------------------------------------------------- 1 | import fat 2 | import sys 3 | import timeit 4 | from fatoptimizer.benchmark import bench, format_dt 5 | 6 | def fast_func(): 7 | pass 8 | 9 | 10 | # use for a dummy guard below 11 | global_var = 2 12 | 13 | 14 | def bench_guards(nguard): 15 | def func(): 16 | pass 17 | 18 | no_guard = bench(func, number=100) 19 | print("no guard: %s" % format_dt(no_guard)) 20 | 21 | if fat.get_specialized(func): 22 | print("ERROR: func already specialized") 23 | sys.exit(1) 24 | 25 | guards = [fat.GuardDict(globals(), ('global_var',)) for i in range(nguard)] 26 | fat.specialize(func, fast_func, guards) 27 | 28 | with_guards = bench(func) 29 | print("with %s guards on globals: %s" 30 | % (nguard, format_dt(with_guards))) 31 | 32 | dt = with_guards - no_guard 33 | print("cost of %s guards: %s (%.1f%%)" 34 | % (nguard, format_dt(dt), dt * 100 / no_guard)) 35 | 36 | dt = dt / nguard 37 | print("average cost of 1 guard: %s (%.1f%%)" 38 | % (format_dt(dt), dt * 100 / no_guard)) 39 | print() 40 | 41 | 42 | def main(): 43 | for nguard in (1000, 100, 10, 1): 44 | bench_guards(nguard) 45 | 46 | 47 | if __name__ == "__main__": 48 | main() 49 | -------------------------------------------------------------------------------- /benchmarks/bench_len_abc.py: -------------------------------------------------------------------------------- 1 | """ 2 | Microbenchmark on the "call builtin functions [with a constant]". 3 | 4 | The benchmark doesn't use fatoptimize, but specialize explicitly the function. 5 | """ 6 | import fat 7 | import sys 8 | from fatoptimizer.benchmark import bench, format_dt, compared_dt 9 | 10 | def func(): 11 | return len("abc") 12 | 13 | if fat.get_specialized(func): 14 | print("ERROR: func() was already specialized") 15 | sys.exit(1) 16 | 17 | def func_cst(): 18 | return 3 19 | 20 | def run_benchmark(bench): 21 | bench.timeit('func()', 22 | globals=globals(), 23 | name="original bytecode (call len)") 24 | 25 | bench.timeit('func_cst()', 26 | globals=globals(), 27 | name="return 3") 28 | 29 | fat.specialize(func, func_cst, [fat.GuardBuiltins(('len',))]) 30 | assert fat.get_specialized(func) 31 | 32 | bench.timeit('func()', 33 | globals=globals(), 34 | name="return 3 with guard on builtins") 35 | -------------------------------------------------------------------------------- /benchmarks/bench_list_append.py: -------------------------------------------------------------------------------- 1 | """ 2 | Microbenchmark on "move invariant (list.append) out of loops" optimization. 3 | 4 | Specialize manually the function. 5 | """ 6 | 7 | import fat 8 | import sys 9 | import timeit 10 | from fatoptimizer.benchmark import bench, format_dt, compared_dt 11 | 12 | 13 | def func(obj, data): 14 | for item in data: 15 | obj.append(item) 16 | 17 | 18 | def func2(obj, data): 19 | for item in data: 20 | obj.append(item) 21 | 22 | 23 | def fast_func2(obj, data): 24 | append = obj.append 25 | for item in data: 26 | append(item) 27 | 28 | 29 | def bench_list(func_name, range_pow10): 30 | repeat = 10 ** max(5 - range_pow10, 1) 31 | number = 10 32 | timer = timeit.Timer('mylist = []; %s(mylist, data)' % func_name, 33 | setup='data=range(10 ** %s)' % range_pow10, 34 | globals=globals()) 35 | return min(timer.repeat(repeat=repeat, number=number)) / number 36 | 37 | 38 | def main(): 39 | if fat.get_specialized(func) or fat.get_specialized(func2): 40 | print("ERROR: functions already specialized!") 41 | sys.exit(1) 42 | 43 | fat.specialize(func2, fast_func2, [fat.GuardArgType(0, (list,))]) 44 | 45 | for range_pow10 in (0, 1, 3, 5): 46 | print("range(10 ** %s)" % range_pow10) 47 | 48 | dt = bench_list('func', range_pow10) 49 | print("- original bytecode: %s" % format_dt(dt)) 50 | 51 | dt2 = bench_list('func2', range_pow10) 52 | print("- append=obj.append with guards: %s" % compared_dt(dt2, dt)) 53 | 54 | dt2 = bench_list('fast_func2', range_pow10) 55 | print("- append=obj.append: %s" % compared_dt(dt2, dt)) 56 | 57 | 58 | if __name__ == "__main__": 59 | main() 60 | -------------------------------------------------------------------------------- /benchmarks/bench_optimizer.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import fatoptimizer 3 | import time 4 | from fatoptimizer.benchmark import format_dt 5 | 6 | config = fatoptimizer.Config() 7 | config.enable_all() 8 | filename = 'x.py' 9 | tree = ast.parse('x') 10 | 11 | print("Optimize AST tree:") 12 | print(ast.dump(tree)) 13 | 14 | loops = 1000 15 | 16 | best = None 17 | for run in range(5): 18 | start = time.perf_counter() 19 | for loop in range(loops): 20 | fatoptimizer.optimize(tree, filename, config) 21 | dt = (time.perf_counter() - start) / loops 22 | 23 | if best is not None: 24 | best = min(best, dt) 25 | else: 26 | best = dt 27 | 28 | print("%s / call" % format_dt(dt)) 29 | -------------------------------------------------------------------------------- /benchmarks/bench_posixpath.py: -------------------------------------------------------------------------------- 1 | """ 2 | Microbenchmark on function inlining. 3 | 4 | Inline manuall _get_sep() in isabs(). Both functions come from the posixpath 5 | module of the standard library. 6 | """ 7 | 8 | import fat 9 | from fatoptimizer.benchmark import bench, format_dt, compared_dt 10 | 11 | def _get_sep(path): 12 | if isinstance(path, bytes): 13 | return b'/' 14 | else: 15 | return '/' 16 | 17 | def isabs(s): 18 | """Test whether a path is absolute""" 19 | sep = _get_sep(s) 20 | return s.startswith(sep) 21 | 22 | def fast_isabs(s): 23 | """Test whether a path is absolute""" 24 | sep = _get_sep(s) 25 | return s.startswith(sep) 26 | 27 | # Manually inline _get_sep() in isabs() depending on the type of the s argument 28 | def isabs_str(s): 29 | return s.startswith('/') 30 | 31 | for func in (_get_sep, isabs, fast_isabs, isabs_str): 32 | if fat.get_specialized(func): 33 | print("ERROR: a function is already specialized!") 34 | sys.exit(1) 35 | 36 | fat.specialize(fast_isabs, isabs_str, 37 | [fat.GuardArgType(0, (str,)), 38 | fat.GuardGlobals(('_get_sep',)), 39 | fat.GuardBuiltins(('isinstance',)), 40 | fat.GuardFunc(_get_sep)]) 41 | 42 | dt = bench("isabs('/abc')") 43 | print("original isabs() bytecode: %s" % format_dt(dt)) 44 | 45 | dt2 = bench("fast_isabs('/abc')") 46 | print("_get_sep() inlined in isabs(): %s" % compared_dt(dt2, dt)) 47 | 48 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/fatoptimizer.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/fatoptimizer.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/fatoptimizer" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/fatoptimizer" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /doc/benchmarks.rst: -------------------------------------------------------------------------------- 1 | ++++++++++ 2 | Benchmarks 3 | ++++++++++ 4 | 5 | fatoptimizer is not ready for macro benchmarks. Important optimizations like 6 | function inlining are still missing. See the :ref:`fatoptimizer TODO list 7 | `. 8 | 9 | See :ref:`Microbenchmarks `. 10 | 11 | 12 | The Grand Unified Python Benchmark Suite 13 | ======================================== 14 | 15 | Project hosted at https://hg.python.org/benchmarks 16 | 17 | 2016-01-22, don't specialized nested functions anymore:: 18 | 19 | $ time python3 ../benchmarks/perf.py ../default/python ../fatpython/python 2>&1|tee log 20 | INFO:root:Automatically selected timer: perf_counter 21 | INFO:root:Running `../fatpython/python ../benchmarks/lib3/2to3/2to3 -f all ../benchmarks/lib/2to3` 22 | INFO:root:Running `../fatpython/python ../benchmarks/lib3/2to3/2to3 -f all ../benchmarks/lib/2to3` 1 time 23 | INFO:root:Running `../default/python ../benchmarks/lib3/2to3/2to3 -f all ../benchmarks/lib/2to3` 24 | INFO:root:Running `../default/python ../benchmarks/lib3/2to3/2to3 -f all ../benchmarks/lib/2to3` 1 time 25 | INFO:root:Running `../fatpython/python ../benchmarks/performance/bm_chameleon_v2.py -n 50 --timer perf_counter` 26 | INFO:root:Running `../default/python ../benchmarks/performance/bm_chameleon_v2.py -n 50 --timer perf_counter` 27 | INFO:root:Running `../fatpython/python ../benchmarks/performance/bm_django_v3.py -n 50 --timer perf_counter` 28 | INFO:root:Running `../default/python ../benchmarks/performance/bm_django_v3.py -n 50 --timer perf_counter` 29 | INFO:root:Running `../fatpython/python ../benchmarks/performance/bm_pickle.py -n 50 --timer perf_counter --use_cpickle pickle` 30 | INFO:root:Running `../default/python ../benchmarks/performance/bm_pickle.py -n 50 --timer perf_counter --use_cpickle pickle` 31 | INFO:root:Running `../fatpython/python ../benchmarks/performance/bm_pickle.py -n 50 --timer perf_counter --use_cpickle unpickle` 32 | INFO:root:Running `../default/python ../benchmarks/performance/bm_pickle.py -n 50 --timer perf_counter --use_cpickle unpickle` 33 | INFO:root:Running `../fatpython/python ../benchmarks/performance/bm_json_v2.py -n 50 --timer perf_counter` 34 | INFO:root:Running `../default/python ../benchmarks/performance/bm_json_v2.py -n 50 --timer perf_counter` 35 | INFO:root:Running `../fatpython/python ../benchmarks/performance/bm_json.py -n 50 --timer perf_counter json_load` 36 | INFO:root:Running `../default/python ../benchmarks/performance/bm_json.py -n 50 --timer perf_counter json_load` 37 | INFO:root:Running `../fatpython/python ../benchmarks/performance/bm_nbody.py -n 50 --timer perf_counter` 38 | INFO:root:Running `../default/python ../benchmarks/performance/bm_nbody.py -n 50 --timer perf_counter` 39 | INFO:root:Running `../fatpython/python ../benchmarks/performance/bm_regex_v8.py -n 50 --timer perf_counter` 40 | INFO:root:Running `../default/python ../benchmarks/performance/bm_regex_v8.py -n 50 --timer perf_counter` 41 | INFO:root:Running `../fatpython/python ../benchmarks/performance/bm_tornado_http.py -n 100 --timer perf_counter` 42 | INFO:root:Running `../default/python ../benchmarks/performance/bm_tornado_http.py -n 100 --timer perf_counter` 43 | [ 1/10] 2to3... 44 | [ 2/10] chameleon_v2... 45 | [ 3/10] django_v3... 46 | [ 4/10] fastpickle... 47 | [ 5/10] fastunpickle... 48 | [ 6/10] json_dump_v2... 49 | [ 7/10] json_load... 50 | [ 8/10] nbody... 51 | [ 9/10] regex_v8... 52 | [10/10] tornado_http... 53 | 54 | Report on Linux smithers 4.2.8-300.fc23.x86_64 #1 SMP Tue Dec 15 16:49:06 UTC 2015 x86_64 x86_64 55 | Total CPU cores: 8 56 | 57 | ### 2to3 ### 58 | 7.232935 -> 7.078553: 1.02x faster 59 | 60 | ### chameleon_v2 ### 61 | Min: 5.740738 -> 5.642322: 1.02x faster 62 | Avg: 5.805132 -> 5.669008: 1.02x faster 63 | Significant (t=5.61) 64 | Stddev: 0.17073 -> 0.01766: 9.6699x smaller 65 | 66 | ### fastpickle ### 67 | Min: 0.448408 -> 0.454956: 1.01x slower 68 | Avg: 0.450220 -> 0.469483: 1.04x slower 69 | Significant (t=-8.28) 70 | Stddev: 0.00364 -> 0.01605: 4.4102x larger 71 | 72 | ### fastunpickle ### 73 | Min: 0.546227 -> 0.582611: 1.07x slower 74 | Avg: 0.554405 -> 0.602790: 1.09x slower 75 | Significant (t=-6.05) 76 | Stddev: 0.01496 -> 0.05453: 3.6461x larger 77 | 78 | ### regex_v8 ### 79 | Min: 0.042338 -> 0.043736: 1.03x slower 80 | Avg: 0.042663 -> 0.044073: 1.03x slower 81 | Significant (t=-4.09) 82 | Stddev: 0.00173 -> 0.00172: 1.0021x smaller 83 | 84 | ### tornado_http ### 85 | Min: 0.260895 -> 0.274085: 1.05x slower 86 | Avg: 0.265663 -> 0.277511: 1.04x slower 87 | Significant (t=-11.26) 88 | Stddev: 0.00988 -> 0.00360: 2.7464x smaller 89 | 90 | The following not significant results are hidden, use -v to show them: 91 | django_v3, json_dump_v2, json_load, nbody. 92 | 93 | real 20m7.994s 94 | user 19m44.016s 95 | sys 0m22.894s 96 | 97 | 2016-01-21:: 98 | 99 | $ time python3 ../benchmarks/perf.py ../default/python ../fatpython/python 100 | INFO:root:Automatically selected timer: perf_counter 101 | [ 1/10] 2to3... 102 | INFO:root:Running `../fatpython/python ../benchmarks/lib3/2to3/2to3 -f all ../benchmarks/lib/2to3` 103 | INFO:root:Running `../fatpython/python ../benchmarks/lib3/2to3/2to3 -f all ../benchmarks/lib/2to3` 1 time 104 | INFO:root:Running `../default/python ../benchmarks/lib3/2to3/2to3 -f all ../benchmarks/lib/2to3` 105 | INFO:root:Running `../default/python ../benchmarks/lib3/2to3/2to3 -f all ../benchmarks/lib/2to3` 1 time 106 | [ 2/10] chameleon_v2... 107 | INFO:root:Running `../fatpython/python ../benchmarks/performance/bm_chameleon_v2.py -n 50 --timer perf_counter` 108 | INFO:root:Running `../default/python ../benchmarks/performance/bm_chameleon_v2.py -n 50 --timer perf_counter` 109 | [ 3/10] django_v3... 110 | INFO:root:Running `../fatpython/python ../benchmarks/performance/bm_django_v3.py -n 50 --timer perf_counter` 111 | INFO:root:Running `../default/python ../benchmarks/performance/bm_django_v3.py -n 50 --timer perf_counter` 112 | [ 4/10] fastpickle... 113 | INFO:root:Running `../fatpython/python ../benchmarks/performance/bm_pickle.py -n 50 --timer perf_counter --use_cpickle pickle` 114 | INFO:root:Running `../default/python ../benchmarks/performance/bm_pickle.py -n 50 --timer perf_counter --use_cpickle pickle` 115 | [ 5/10] fastunpickle... 116 | INFO:root:Running `../fatpython/python ../benchmarks/performance/bm_pickle.py -n 50 --timer perf_counter --use_cpickle unpickle` 117 | INFO:root:Running `../default/python ../benchmarks/performance/bm_pickle.py -n 50 --timer perf_counter --use_cpickle unpickle` 118 | [ 6/10] json_dump_v2... 119 | INFO:root:Running `../fatpython/python ../benchmarks/performance/bm_json_v2.py -n 50 --timer perf_counter` 120 | INFO:root:Running `../default/python ../benchmarks/performance/bm_json_v2.py -n 50 --timer perf_counter` 121 | [ 7/10] json_load... 122 | INFO:root:Running `../fatpython/python ../benchmarks/performance/bm_json.py -n 50 --timer perf_counter json_load` 123 | INFO:root:Running `../default/python ../benchmarks/performance/bm_json.py -n 50 --timer perf_counter json_load` 124 | [ 8/10] nbody... 125 | INFO:root:Running `../fatpython/python ../benchmarks/performance/bm_nbody.py -n 50 --timer perf_counter` 126 | INFO:root:Running `../default/python ../benchmarks/performance/bm_nbody.py -n 50 --timer perf_counter` 127 | [ 9/10] regex_v8... 128 | INFO:root:Running `../fatpython/python ../benchmarks/performance/bm_regex_v8.py -n 50 --timer perf_counter` 129 | INFO:root:Running `../default/python ../benchmarks/performance/bm_regex_v8.py -n 50 --timer perf_counter` 130 | [10/10] tornado_http... 131 | INFO:root:Running `../fatpython/python ../benchmarks/performance/bm_tornado_http.py -n 100 --timer perf_counter` 132 | INFO:root:Running `../default/python ../benchmarks/performance/bm_tornado_http.py -n 100 --timer perf_counter` 133 | 134 | Report on Linux smithers 4.2.8-300.fc23.x86_64 #1 SMP Tue Dec 15 16:49:06 UTC 2015 x86_64 x86_64 135 | Total CPU cores: 8 136 | 137 | ### 2to3 ### 138 | 6.969972 -> 7.362033: 1.06x slower 139 | 140 | ### chameleon_v2 ### 141 | Min: 5.686547 -> 5.945011: 1.05x slower 142 | Avg: 5.731851 -> 5.976754: 1.04x slower 143 | Significant (t=-21.46) 144 | Stddev: 0.06645 -> 0.04580: 1.4511x smaller 145 | 146 | ### fastpickle ### 147 | Min: 0.489443 -> 0.448850: 1.09x faster 148 | Avg: 0.518914 -> 0.458638: 1.13x faster 149 | Significant (t=6.48) 150 | Stddev: 0.05688 -> 0.03304: 1.7218x smaller 151 | 152 | ### fastunpickle ### 153 | Min: 0.598339 -> 0.559612: 1.07x faster 154 | Avg: 0.604129 -> 0.564821: 1.07x faster 155 | Significant (t=13.55) 156 | Stddev: 0.01493 -> 0.01408: 1.0601x smaller 157 | 158 | ### json_dump_v2 ### 159 | Min: 2.794058 -> 4.456882: 1.60x slower 160 | Avg: 2.806195 -> 4.467750: 1.59x slower 161 | Significant (t=-801.42) 162 | Stddev: 0.00722 -> 0.01276: 1.7678x larger 163 | 164 | ### regex_v8 ### 165 | Min: 0.041685 -> 0.050890: 1.22x slower 166 | Avg: 0.042082 -> 0.051579: 1.23x slower 167 | Significant (t=-26.94) 168 | Stddev: 0.00177 -> 0.00175: 1.0105x smaller 169 | 170 | ### tornado_http ### 171 | Min: 0.258212 -> 0.272552: 1.06x slower 172 | Avg: 0.263689 -> 0.280610: 1.06x slower 173 | Significant (t=-8.59) 174 | Stddev: 0.01614 -> 0.01130: 1.4282x smaller 175 | 176 | The following not significant results are hidden, use -v to show them: 177 | django_v3, json_load, nbody. 178 | 179 | real 21m53.511s 180 | user 21m29.279s 181 | sys 0m23.055s 182 | -------------------------------------------------------------------------------- /doc/changelog.rst: -------------------------------------------------------------------------------- 1 | +++++++++ 2 | Changelog 3 | +++++++++ 4 | 5 | fatoptimizer changelog 6 | ====================== 7 | 8 | * Version 0.3 9 | 10 | * Drop Python 3.4 support: Python 3.4 reached its end of life in 2019. 11 | * Experimental implementation of function inlining, implemented by David 12 | Malcolm. 13 | * New optimization: call pure methods of builtin types. For example, 14 | replace ``"abc".encode()`` with ``b'abc'``. 15 | * Update for fat API version 0.3, GuardBuiltins constructor changed. 16 | * Basic "loop unrolling" on list-comprehension, set-comprehension 17 | and dict-comprehension. Only if there is a single comprehension using a 18 | constant iterable without if. 19 | 20 | * 2016-01-23: Version 0.2 21 | 22 | * Fix the function optimizer: don't specialized nested function. The 23 | specialization is more expensive than the speedup of optimizations. 24 | * Fix Config.replace(): copy logger attribute 25 | * get_literal() now also returns tuple literals when items are not constants 26 | * Adjust usage of get_literal() 27 | * SimplifyIterable also replaces empty dict (created a runtime) with an empty 28 | tuple (constant) 29 | * Update benchmark scripts and benchmark results in the documentation 30 | 31 | * 2016-01-18: Version 0.1 32 | 33 | * Add ``fatoptimizer.pretty_dump()`` 34 | * Add Sphinx documentation: ``doc/`` directory 35 | * Add benchmark scripts: ``benchmarks/`` directory 36 | * Update ``fatoptimizer._register()`` for the new version of the PEP 511 37 | (``sys.set_code_transformers()``) 38 | 39 | * 2016-01-14: First public release, version 0.0. 40 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # fatoptimizer documentation build configuration file, created by 4 | # sphinx-quickstart on Thu Jan 14 01:51:34 2016. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | import sys 16 | import os 17 | 18 | # If extensions (or modules to document with autodoc) are in another directory, 19 | # add these directories to sys.path here. If the directory is relative to the 20 | # documentation root, use os.path.abspath to make it absolute, like shown here. 21 | #sys.path.insert(0, os.path.abspath('.')) 22 | 23 | # -- General configuration ------------------------------------------------ 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | #needs_sphinx = '1.0' 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be 29 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 30 | # ones. 31 | extensions = [] 32 | 33 | # Add any paths that contain templates here, relative to this directory. 34 | templates_path = ['_templates'] 35 | 36 | # The suffix of source filenames. 37 | source_suffix = '.rst' 38 | 39 | # The encoding of source files. 40 | #source_encoding = 'utf-8-sig' 41 | 42 | # The master toctree document. 43 | master_doc = 'index' 44 | 45 | # General information about the project. 46 | project = u'fatoptimizer' 47 | copyright = u'2016, Victor Stinner' 48 | 49 | # The version info for the project you're documenting, acts as replacement for 50 | # |version| and |release|, also used in various other places throughout the 51 | # built documents. 52 | # 53 | # The short X.Y version. 54 | version = release = '0.3' 55 | 56 | # The language for content autogenerated by Sphinx. Refer to documentation 57 | # for a list of supported languages. 58 | #language = None 59 | 60 | # There are two options for replacing |today|: either, you set today to some 61 | # non-false value, then it is used: 62 | #today = '' 63 | # Else, today_fmt is used as the format for a strftime call. 64 | #today_fmt = '%B %d, %Y' 65 | 66 | # List of patterns, relative to source directory, that match files and 67 | # directories to ignore when looking for source files. 68 | exclude_patterns = ['_build'] 69 | 70 | # The reST default role (used for this markup: `text`) to use for all 71 | # documents. 72 | #default_role = None 73 | 74 | # If true, '()' will be appended to :func: etc. cross-reference text. 75 | #add_function_parentheses = True 76 | 77 | # If true, the current module name will be prepended to all description 78 | # unit titles (such as .. function::). 79 | #add_module_names = True 80 | 81 | # If true, sectionauthor and moduleauthor directives will be shown in the 82 | # output. They are ignored by default. 83 | #show_authors = False 84 | 85 | # The name of the Pygments (syntax highlighting) style to use. 86 | pygments_style = 'sphinx' 87 | 88 | # A list of ignored prefixes for module index sorting. 89 | #modindex_common_prefix = [] 90 | 91 | # If true, keep warnings as "system message" paragraphs in the built documents. 92 | #keep_warnings = False 93 | 94 | 95 | # -- Options for HTML output ---------------------------------------------- 96 | 97 | # The theme to use for HTML and HTML Help pages. See the documentation for 98 | # a list of builtin themes. 99 | html_theme = 'default' 100 | 101 | 102 | # Theme options are theme-specific and customize the look and feel of a theme 103 | # further. For a list of options available for each theme, see the 104 | # documentation. 105 | #html_theme_options = {} 106 | 107 | # Add any paths that contain custom themes here, relative to this directory. 108 | #html_theme_path = [] 109 | 110 | # The name for this set of Sphinx documents. If None, it defaults to 111 | # " v documentation". 112 | #html_title = None 113 | 114 | # A shorter title for the navigation bar. Default is the same as html_title. 115 | #html_short_title = None 116 | 117 | # The name of an image file (relative to this directory) to place at the top 118 | # of the sidebar. 119 | #html_logo = None 120 | 121 | # The name of an image file (within the static path) to use as favicon of the 122 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 123 | # pixels large. 124 | #html_favicon = None 125 | 126 | # Add any paths that contain custom static files (such as style sheets) here, 127 | # relative to this directory. They are copied after the builtin static files, 128 | # so a file named "default.css" will overwrite the builtin "default.css". 129 | html_static_path = ['_static'] 130 | 131 | # Add any extra paths that contain custom files (such as robots.txt or 132 | # .htaccess) here, relative to this directory. These files are copied 133 | # directly to the root of the documentation. 134 | #html_extra_path = [] 135 | 136 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 137 | # using the given strftime format. 138 | #html_last_updated_fmt = '%b %d, %Y' 139 | 140 | # If true, SmartyPants will be used to convert quotes and dashes to 141 | # typographically correct entities. 142 | #html_use_smartypants = True 143 | 144 | # Custom sidebar templates, maps document names to template names. 145 | #html_sidebars = {} 146 | 147 | # Additional templates that should be rendered to pages, maps page names to 148 | # template names. 149 | #html_additional_pages = {} 150 | 151 | # If false, no module index is generated. 152 | #html_domain_indices = True 153 | 154 | # If false, no index is generated. 155 | #html_use_index = True 156 | 157 | # If true, the index is split into individual pages for each letter. 158 | #html_split_index = False 159 | 160 | # If true, links to the reST sources are added to the pages. 161 | #html_show_sourcelink = True 162 | 163 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 164 | #html_show_sphinx = True 165 | 166 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 167 | #html_show_copyright = True 168 | 169 | # If true, an OpenSearch description file will be output, and all pages will 170 | # contain a tag referring to it. The value of this option must be the 171 | # base URL from which the finished HTML is served. 172 | #html_use_opensearch = '' 173 | 174 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 175 | #html_file_suffix = None 176 | 177 | # Output file base name for HTML help builder. 178 | htmlhelp_basename = 'fatoptimizerdoc' 179 | 180 | 181 | # -- Options for LaTeX output --------------------------------------------- 182 | 183 | latex_elements = { 184 | # The paper size ('letterpaper' or 'a4paper'). 185 | #'papersize': 'letterpaper', 186 | 187 | # The font size ('10pt', '11pt' or '12pt'). 188 | #'pointsize': '10pt', 189 | 190 | # Additional stuff for the LaTeX preamble. 191 | #'preamble': '', 192 | } 193 | 194 | # Grouping the document tree into LaTeX files. List of tuples 195 | # (source start file, target name, title, 196 | # author, documentclass [howto, manual, or own class]). 197 | latex_documents = [ 198 | ('index', 'fatoptimizer.tex', u'fatoptimizer Documentation', 199 | u'Victor Stinner', 'manual'), 200 | ] 201 | 202 | # The name of an image file (relative to this directory) to place at the top of 203 | # the title page. 204 | #latex_logo = None 205 | 206 | # For "manual" documents, if this is true, then toplevel headings are parts, 207 | # not chapters. 208 | #latex_use_parts = False 209 | 210 | # If true, show page references after internal links. 211 | #latex_show_pagerefs = False 212 | 213 | # If true, show URL addresses after external links. 214 | #latex_show_urls = False 215 | 216 | # Documents to append as an appendix to all manuals. 217 | #latex_appendices = [] 218 | 219 | # If false, no module index is generated. 220 | #latex_domain_indices = True 221 | 222 | 223 | # -- Options for manual page output --------------------------------------- 224 | 225 | # One entry per manual page. List of tuples 226 | # (source start file, name, description, authors, manual section). 227 | man_pages = [ 228 | ('index', 'fatoptimizer', u'fatoptimizer Documentation', 229 | [u'Victor Stinner'], 1) 230 | ] 231 | 232 | # If true, show URL addresses after external links. 233 | #man_show_urls = False 234 | 235 | 236 | # -- Options for Texinfo output ------------------------------------------- 237 | 238 | # Grouping the document tree into Texinfo files. List of tuples 239 | # (source start file, target name, title, author, 240 | # dir menu entry, description, category) 241 | texinfo_documents = [ 242 | ('index', 'fatoptimizer', u'fatoptimizer Documentation', 243 | u'Victor Stinner', 'fatoptimizer', 'One line description of project.', 244 | 'Miscellaneous'), 245 | ] 246 | 247 | # Documents to append as an appendix to all manuals. 248 | #texinfo_appendices = [] 249 | 250 | # If false, no module index is generated. 251 | #texinfo_domain_indices = True 252 | 253 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 254 | #texinfo_show_urls = 'footnote' 255 | 256 | # If true, do not generate a @detailmenu in the "Top" node's menu. 257 | #texinfo_no_detailmenu = False 258 | -------------------------------------------------------------------------------- /doc/fat.rst: -------------------------------------------------------------------------------- 1 | .. _fat: 2 | 3 | ++++++++++ 4 | fat module 5 | ++++++++++ 6 | 7 | The ``fat`` module is a Python extension module (written in C) implementing 8 | fast guards. The :ref:`fatoptimizer optimizer ` uses ``fat`` 9 | guards to specialize functions. ``fat`` guards are used to verify assumptions 10 | used to specialize the code. If an assumption is no more true, the specialized 11 | code is not used. 12 | 13 | The ``fat`` module is required to run code optimized by ``fatoptimizer`` if at 14 | least one function is specialized. 15 | 16 | * `fat project at GitHub 17 | `_ 18 | 19 | The ``fat`` module requires a Python 3.6 patched with `PEP 509 "Add a private 20 | version to dict" `_ and `PEP 510 21 | "Specialize functions with guards" 22 | `_ patches. 23 | 24 | 25 | fat module API 26 | ============== 27 | 28 | .. warning:: 29 | The API is not stable yet. 30 | 31 | ``fat.__version__`` is the module version string (ex: ``'0.3'``). 32 | 33 | Functions 34 | --------- 35 | 36 | .. function:: replace_consts(code, mapping) 37 | 38 | Create a copy of the code object with replaced constants:: 39 | 40 | new_consts = tuple(mapping.get(const, const) for const in consts) 41 | 42 | 43 | .. function:: specialize(func, code, guards) 44 | 45 | Specialize a Python function: add a specialized code with guards. 46 | 47 | *code* must be a callable or code object, *guards* must be a non-empty 48 | sequence of guards. 49 | 50 | 51 | .. function:: get_specialized(func) 52 | 53 | Get the list of specialized codes with guards. Return a list of ``(code, 54 | guards)`` tuples. 55 | 56 | 57 | See the `PEP 510 "Specialize functions with guards" 58 | `_ for the API of ``specialize()`` 59 | and ``get_specialized()``. 60 | 61 | 62 | .. _guard: 63 | 64 | Guard types 65 | ----------- 66 | 67 | .. class:: GuardArgType(arg_index, arg_types) 68 | 69 | Check the type of the nth argument. *arg_types* must be a sequence of 70 | types. 71 | 72 | The guard check fails temporarily (returns ``1``) if the argument has a 73 | different type or if the guard is checked with less than *arg_index* 74 | parameters. 75 | 76 | The guard does not support keyword parameters yet. If the guard is checked 77 | with keyword parameters, it fails temporarily (returns ``1``). 78 | 79 | Attributes: 80 | 81 | .. attribute:: arg_index 82 | 83 | Index of the argument (``int``). Read-only attribute. 84 | 85 | .. attribute:: arg_types 86 | 87 | List of accepted types for the argument: list of types. 88 | Read-only property. 89 | 90 | Keep a strong reference to *arg_types* types. 91 | 92 | 93 | .. class:: GuardBuiltins(\*names) 94 | 95 | Subtype of :class:`GuardDict`. 96 | 97 | Watch for: 98 | 99 | * globals of the current frame (``frame.f_globals``) 100 | * ``globals()[name]`` for all *names*. 101 | * builtins of the current frame (``frame.f_builtins``) 102 | * ``builtins.__dict__[name]`` for all *names* 103 | 104 | The guard initialization fails if ``builtins.__dict__[name]`` was replaced 105 | after ``fat`` was imported, or if ``globals()[name]`` already exists. 106 | 107 | In addition to :class:`GuardDict` checks and 108 | :attr:`GuardBuiltins.guard_globals` checks, the guard check always fails 109 | (returns ``2``) if the frame builtins changed. 110 | 111 | Attributes: 112 | 113 | .. attribute:: guard_globals 114 | 115 | The :class:`GuardGlobals` used to watch for the global variables. 116 | Read-only attribute. 117 | 118 | Keep a strong references to the builtin namespace (``builtins.__dict__`` 119 | dictionary), to the global namespace (``globals()`` dictionary), to *names* 120 | and to existing builtin symbols called *names* (``builtins.__dict__[name]`` 121 | for all *names*). 122 | 123 | 124 | .. class:: GuardDict(dict, \*keys) 125 | 126 | Watch for ``dict[key]`` for all *keys*. 127 | 128 | The guard check always fails (returns ``2``) if at least one key of *keys* 129 | was modified. 130 | 131 | *keys* strings are interned: see `sys.intern`. 132 | 133 | Attributes: 134 | 135 | .. attribute:: dict 136 | 137 | Watched dictionary (``dict``). Read-only attribute. 138 | 139 | .. attribute:: keys 140 | 141 | List of watched dictionary keys: list of ``str``. Read-only property. 142 | 143 | Keep a strong references to *dict*, to *keys* and to existing dictionary 144 | values (``dict[key]`` for all keys). 145 | 146 | 147 | .. class:: GuardFunc(func) 148 | 149 | Watch for the code object (``func.__code__``) of a Python function. 150 | 151 | The guard check always fails (returns ``2``) if the function code was 152 | replaced. 153 | 154 | ``GuardFunc(func)`` must not be used to specialize ``func``. Replacing the 155 | code object of a function already removes its specialized code, no need to 156 | add a guard. 157 | 158 | Attributes: 159 | 160 | .. attribute:: code 161 | 162 | Watched code object. Read-only attribute. 163 | 164 | .. attribute:: func 165 | 166 | Watched function. Read-only attribute. 167 | 168 | Keep a strong references to *func* and to ``func.__code__``. 169 | 170 | 171 | .. class:: GuardGlobals(\*names) 172 | 173 | Subtype of :class:`GuardDict`. 174 | 175 | In addition to :class:`GuardDict` checks, the guard check always fails 176 | (returns ``2``) if the frame globals changed. 177 | 178 | Watch for: 179 | 180 | * globals of the current frame (``frame.f_globals``) 181 | * ``globals()[name]`` for all *names*. 182 | 183 | Keep a strong references to the global namespace (``globals()`` dictionary), 184 | to *names* and to existing global variables called *names* 185 | (``globals()[name]`` for all *names*). 186 | 187 | 188 | Guard helper functions 189 | ---------------------- 190 | 191 | .. function:: guard_type_dict(type, attrs) 192 | 193 | Create ``GuardDict(type.__dict__, attrs)`` but access the real type 194 | dictionary, not ``type.__dict`` which is a read-only proxy. 195 | 196 | Watch for ``type.attr`` (``type.__dict__[attr]``) for all *attrs*. 197 | 198 | 199 | Installation 200 | ============ 201 | 202 | The ``fat`` module requires a Python 3.6 patched with `PEP 509 "Add a private 203 | version to dict" `_ and `PEP 510 204 | "Specialize functions with guards" 205 | `_ patches. 206 | 207 | Type:: 208 | 209 | pip install fat 210 | 211 | Manual installation:: 212 | 213 | python3.6 setup.py install 214 | 215 | 216 | Run tests 217 | ========= 218 | 219 | Type:: 220 | 221 | ./runtests.sh 222 | 223 | 224 | Changelog 225 | ========= 226 | 227 | * Version 0.3 228 | 229 | * Change constructors: 230 | 231 | * ``GuardDict(dict, keys)`` becomes ``GuardDict(dict, *keys)`` 232 | * ``GuardBuiltins(name)`` becomes ``GuardBuiltins(*names)`` 233 | * ``GuardGlobals(name)`` becomes ``GuardGlobals(*names)`` 234 | 235 | * ``GuardFunc(func)`` init function now raises a :exc:`ValueError` if it is 236 | used to specialize ``func``. 237 | * ``GuardDict(keys)`` now interns *keys* strings. 238 | 239 | * 2016-01-22: Version 0.2 240 | 241 | * :class:`GuardBuiltins` now also checks the builtins and the globals of the 242 | current frame. In practice, the guard fails if it is created in a namespace 243 | and checked in a different namespace. 244 | * Add a new :class:`GuardGlobals` type which replaces the previous 245 | :func:`guard_globals()` helper function (removed). The guard check checks if 246 | the frame globals changed or not. 247 | * Guards are now tracked by the garbage collector to handle correctly a 248 | reference cycle with GuardGlobals which keeps a reference to the module 249 | namespace (``globals()``). 250 | * Fix type of dictionary version for 32-bit platforms: ``PY_UINT64_T``, not 251 | ``size_t``. 252 | * Fix :class:`GuardFunc` traverse method: visit also the ``code`` attribute. 253 | * Implement a traverse method to :class:`GuardBuiltins` to detect correctly 254 | reference cycles. 255 | 256 | * 2016-01-18: Version 0.1 257 | 258 | * GuardBuiltins check remembers if guard init failed 259 | * Rename :class:`GuardGlobals` to :func:`guard_globals` 260 | * Rename :class:`GuardTypeDict` to :func:`guard_dict_type` 261 | 262 | * 2016-01-13: First public release, version 0.0. 263 | -------------------------------------------------------------------------------- /doc/fatoptimizer.rst: -------------------------------------------------------------------------------- 1 | +++++++++++++++++++ 2 | fatoptimizer module 3 | +++++++++++++++++++ 4 | 5 | fatoptimizer API 6 | ================ 7 | 8 | .. warning:: 9 | The API is not stable yet. 10 | 11 | ``fatoptimizer.__version__`` is the module version string (ex: ``'0.3'``). 12 | 13 | .. function:: optimize(tree, filename, config) 14 | 15 | Optimize an AST tree. Return the optimized AST tree. 16 | 17 | 18 | .. function:: pretty_dump(node, annotate_fields=True, include_attributes=False, lineno=False, indent=' ') 19 | 20 | Return a formatted dump of the tree in *node*. This is mainly useful for 21 | debugging purposes. The returned string will show the names and the values 22 | for fields. This makes the code impossible to evaluate, so if evaluation is 23 | wanted *annotate_fields* must be set to False. Attributes such as line 24 | numbers and column offsets are not dumped by default. If this is wanted, 25 | *include_attributes* can be set to True. 26 | 27 | 28 | .. class:: Config 29 | 30 | Configuration of the optimizer. 31 | 32 | See :ref:`fatoptimizer configuration `. 33 | 34 | 35 | .. class:: FATOptimizer(config) 36 | 37 | Code transformers for ``sys.set_code_transformers()``. 38 | 39 | 40 | .. class:: OptimizerError 41 | 42 | Exception raised on bugs in the optimizer. 43 | 44 | 45 | Installation 46 | ============ 47 | 48 | The :ref:`fatoptimizer module ` requires a Python 3.6 patched 49 | with `PEP 510 "Specialize functions with guards" 50 | `_ and `PEP 511 "API for code 51 | transformers" `_ patches. 52 | 53 | Type:: 54 | 55 | pip install fatoptimizer 56 | 57 | Manual installation:: 58 | 59 | python3.6 setup.py install 60 | 61 | Optimized code requires the :ref:`fat module ` at runtime if at least one 62 | function is specialized. 63 | 64 | 65 | .. _config: 66 | 67 | Configuration 68 | ============= 69 | 70 | It is possible to configure the AST optimizer per module by setting 71 | the ``__fatoptimizer__`` variable. Configuration keys: 72 | 73 | * ``enabled`` (``bool``): set to ``False`` to disable all optimization (default: true) 74 | 75 | * ``constant_propagation`` (``bool``): enable :ref:`constant propagation ` 76 | optimization? (default: true) 77 | 78 | * ``constant_folding`` (``bool``): enable :ref:`constant folding 79 | ` optimization? (default: true) 80 | 81 | * ``copy_builtin_to_constant`` (``bool``): enable :ref:`copy builtin functions 82 | to constants ` optimization? (default: false) 83 | 84 | * ``inlining`` (``bool``): enable :ref:`function inlining 85 | ` optimization? (default: false) 86 | 87 | * ``remove_dead_code`` (``bool``): enable :ref:`dead code elimination 88 | ` optimization? (default: true) 89 | 90 | * maximum size of constants: 91 | 92 | - ``max_bytes_len``: Maximum number of bytes of a text string (default: 128) 93 | - ``max_int_bits``: Maximum number of bits of an integer (default: 256) 94 | - ``max_str_len``: Maximum number of characters of a text string (default: 128) 95 | - ``max_seq_len``: Maximum length in number of items of a sequence like 96 | tuples (default: 32). It is only a preliminary check: ``max_constant_size`` 97 | still applies for sequences. 98 | - ``max_constant_size``: Maximum size in bytes of other constants 99 | (default: 128 bytes), the size is computed with ``len(marshal.dumps(obj))`` 100 | 101 | * ``replace_builtin_constant`` (``bool``): enable :ref:`replace builtin 102 | constants ` optimization? (default: true) 103 | 104 | * ``simplify_iterable`` (``bool``): enable :ref:`simplify iterable optimization 105 | `? (default: true) 106 | 107 | * ``unroll_loops``: Maximum number of loop iteration for loop unrolling 108 | (default: ``16``). Set it to ``0`` to disable loop unrolling. See 109 | :ref:`loop unrolling ` and :ref:`simplify comprehension ` 110 | optimizations. 111 | 112 | Example to disable all optimizations in a module:: 113 | 114 | __fatoptimizer__ = {'enabled': False} 115 | 116 | Example to disable the constant folding optimization:: 117 | 118 | __fatoptimizer__ = {'constant_folding': False} 119 | 120 | See the :class:`Config` class. 121 | 122 | 123 | Run tests 124 | ========= 125 | 126 | Type:: 127 | 128 | tox 129 | 130 | You may need to install or update tox:: 131 | 132 | pip3 install -U tox 133 | 134 | Run manually tests:: 135 | 136 | python3 test_fatoptimizer.py 137 | 138 | There are also integration tests which requires a Python 3.6 with patches PEP 139 | 509, PEP 510 and PEP 511. Run integration tests:: 140 | 141 | python3.6 -X fat test_fat_config.py 142 | python3.6 -X fat test_fat_size.py 143 | -------------------------------------------------------------------------------- /doc/gsoc.rst: -------------------------------------------------------------------------------- 1 | +++++++++++++++++++++ 2 | Google Summer of Code 3 | +++++++++++++++++++++ 4 | 5 | Google Summer of Code and the PSF 6 | ================================= 7 | 8 | The `Google Summer of Code `_ (GSoC) "is 9 | a global program focused on bringing more student developers into open source 10 | software development. Students work with an open source organization on a 3 11 | month programming project during their break from school." 12 | 13 | The `Python Software Foundation `_ is part of 14 | the Google Summer of Code (GSoC) program in 2016, as previous years. See: 15 | 16 | * `PSF GSoC projects 2016 17 | `_ 18 | * `Python core projects of PSF GSoC 2016 19 | `_: this page 20 | mentions the GSoC project on FAT Python (but this page is more complete). 21 | 22 | 23 | FAT Python 24 | ========== 25 | 26 | FAT Python is a new static optimizer for Python 3.6, it specializes functions 27 | and use guards to decide if specialized code can be called or not. See `FAT 28 | Python homepage `_ and 29 | the `slides of my talk at FOSDEM 2016 30 | `_ for 31 | more information. 32 | 33 | The design is *inspired* by JIT compilers, but is simpler. FAT Python has been 34 | designed to be able to merge changes required to use FAT Python into CPython 35 | 3.6. The expected use case is to compile modules and applications 36 | ahead-of-time, so the performance of the optimizer itself don't matter much. 37 | 38 | FAT Python is made of different parts: 39 | 40 | * CPython 3.6 41 | * fat module: fast guards implemented in C 42 | * fatoptimizer module: the static optimizer implemented as an AST optimizer. 43 | It produces specialized functions of functions using guards. 44 | * PEP 509 (dict version): patch required by the fat module to implement fast 45 | guards on Python namespaces 46 | * PEP 510 (function specialization): private C API to add specialized code 47 | to Python functions 48 | * PEP 511 (API for AST optimizers): new Python API to register an AST optimizer 49 | 50 | Status at March 2016: 51 | 52 | * First patches to implement AST optimizers have already been merged in CPython 53 | 3.6 54 | * fat and fatoptimizer have been implemented, are fully functional and have 55 | unit tests 56 | * early benchmarks don't show major speedup on "The Grand Unified Python 57 | Benchmark Suite" 58 | * fatoptimizer is quite slow 59 | * PEP 509 need to be modified to make the dictionary versions globally unique. 60 | PEP 509 is required by the promising `Speedup method calls 1.2x 61 | `_ change written by Yury Selivanov. 62 | This change can help to get this PEP accepted. 63 | * PEP 511 is still a work-in-progress, it's even unclear if the whole PEP 64 | is required. It only makes the usage of FAT Python more practical. It avoids 65 | conflicts on .pyc files using the ``-o`` command line option proposed in the 66 | PEP. 67 | * PEP 509, 510 and 511 are basically blocked by an expectation on concrete 68 | speedup 69 | 70 | 71 | FAT Python GSoC Roadmap 72 | ======================= 73 | 74 | GSoC takes 4 months, the exact planning is not defined yet. 75 | 76 | Goal 77 | ---- 78 | 79 | The overall goal is to enhance FAT Python to get concrete speedup on the 80 | benchmark suite and on applications. 81 | 82 | 83 | Requirements 84 | ------------ 85 | 86 | * All requirements of the GSoC program! (like being available during 87 | the 4 months of the GSoC program) 88 | * Able to read and write technical english 89 | * Better if already the student worked remotely on a free software before 90 | * Good knowledge of the Python programming language 91 | * (Optional?) Basic knowledge of how compilers are implemented 92 | * (Optional?) Basic knowledge of static optimizations like constant folding 93 | 94 | 95 | Milestone 0 to select the student 96 | --------------------------------- 97 | 98 | fatoptimizer: 99 | 100 | * Download `fatoptimizer `_ and run tests:: 101 | 102 | git clone https://github.com/vstinner/fatoptimizer 103 | cd fatoptimizer 104 | python3 test_fatoptimizer.py 105 | 106 | * Read the `fatoptimizer documentation `_ 107 | * Pick a simple task in the :ref:`fatoptimizer TODO list ` and send a 108 | pull request 109 | * MANDATORY: Submit a final PDF proposal: see 110 | https://wiki.python.org/moin/SummerOfCode/2016 for a template 111 | 112 | Optional: 113 | 114 | * Download and compile FAT Python: 115 | https://faster-cpython.readthedocs.io/fat_python.html#getting-started 116 | * Run Python test suite of FAT Python 117 | 118 | 119 | Milestone 1 120 | ----------- 121 | 122 | Discover FAT Python. 123 | 124 | * Select a set of benchmarks 125 | * Run benchmarks to have a reference for performances (better: write a script 126 | for that) 127 | * Implement the most easy optimizations of the :ref:`TODO list ` 128 | (like remaining constant folding optimizations) 129 | * Run the full Python test suite with FAT Python 130 | * Run real applications like Django with FAT Python to identify bugs 131 | * Propose fixes or workaround for bugs 132 | 133 | Goal: have at least one new optimization merged into fatoptimizer. 134 | 135 | Milestone 2 136 | ----------- 137 | 138 | Function inlining. 139 | 140 | * Test the existing (basic) implementation of function inlining 141 | * Fix function inlining 142 | * Enhance function inlining to use it in more cases 143 | * Wider tests of the new features 144 | * Fix bugs 145 | 146 | Goal: make function inlining usable with the default config without breaking 147 | the Python test suite, even if it's only a subset of the feature. 148 | 149 | 150 | Milestone 3 151 | ----------- 152 | 153 | Remove useless variables. For example, remove ``x`` in 154 | ``def func(): x = 1; return 2``. 155 | 156 | * Add configuration option to enable this optimization 157 | * Write an unit test for the expected behaviour 158 | * Implement algorithm to compute where and when a variable is alive or not 159 | * Use this algorithm to find dead variables and then remove them 160 | * Wider tests of the new features 161 | * Fix bugs 162 | 163 | Goal: remove useless variables with the default config without breaking the 164 | Python test suite, even if it's only a subset of the feature. 165 | 166 | 167 | Milestone 4 (a) 168 | --------------- 169 | 170 | Detect pure function, first subpart: implement it manually. 171 | 172 | * Add an option to __fatoptimizer__ module configuration to explicitly declare 173 | constants 174 | * Write a patch to declare some constants in the Python standard library 175 | * Add an option to __fatoptimizer__ module configuration to explicitly declare 176 | pure functions 177 | * Write a patch to declare some pure functions in the Python standard library, 178 | ex: os.path._getsep(). 179 | 180 | Goal: annotate a few constants and pure functions in the Python standard 181 | library and ensure that they are optimized. 182 | 183 | Milestone 4 (b) 184 | --------------- 185 | 186 | Detect pure function, second and last subpart: implement automatic detection. 187 | 188 | * Write a safe heuristic to detect pure functions using a small whitelist of 189 | instructions which are known to be pure 190 | * Wider tests of the new features 191 | * Fix bugs 192 | * Extend the whitelist, add more and more instructions 193 | * Run tests 194 | * Fix bugs 195 | * Iterate until the whitelist is considered big enough? 196 | * Maybe design a better algorithm than a white list? 197 | 198 | See also pythran which already implemented this feature :-) 199 | 200 | Goal: detect that os.path._getsep() is pure. 201 | 202 | Goal 2, optional: inline os.path._getsep() in isabs(). 203 | 204 | 205 | More milestones? 206 | ---------------- 207 | 208 | The exact planning will be adapted depending on the speed of the student, 209 | the availability of mentors, etc. 210 | 211 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | .. _fatoptimizer: 2 | 3 | ++++++++++++ 4 | fatoptimizer 5 | ++++++++++++ 6 | 7 | .. image:: https://travis-ci.org/vstinner/fatoptimizer.svg?branch=master 8 | :alt: Build status of fatoptimizer on Travis CI 9 | :target: https://travis-ci.org/vstinner/fatoptimizer 10 | 11 | ``fatoptimizer`` is a static optimizer for Python 3.6 using function 12 | specialization with guards. It is implemented as an AST optimizer. 13 | 14 | Optimized code requires the :ref:`fat module ` at runtime if at least one 15 | function is specialized. 16 | 17 | Links: 18 | 19 | * `fatoptimizer documentation 20 | `_ (this documentation) 21 | * `fatoptimizer project at GitHub 22 | `_ (code, bug tracker) 23 | * `FAT Python 24 | `_ 25 | * `fatoptimizer tests running on the Travis-CI 26 | `_ 27 | 28 | The ``fatoptimizer`` module requires a Python 3.6 patched with `PEP 510 29 | "Specialize functions with guards" 30 | `_ and `PEP 511 "API for code 31 | transformers" `_ patches. 32 | 33 | 34 | Table Of Contents 35 | ================= 36 | 37 | .. toctree:: 38 | :maxdepth: 1 39 | 40 | fatoptimizer 41 | fat 42 | optimizations 43 | semantics 44 | benchmarks 45 | microbenchmarks 46 | changelog 47 | todo 48 | gsoc 49 | misc 50 | -------------------------------------------------------------------------------- /doc/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\fatoptimizer.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\fatoptimizer.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /doc/microbenchmarks.rst: -------------------------------------------------------------------------------- 1 | .. _microbench: 2 | 3 | +++++++++++++++ 4 | Microbenchmarks 5 | +++++++++++++++ 6 | 7 | REMINDER: on a microbenchmark, even a significant speedup doesn't mean that you 8 | will get a significant speedup on your application. 9 | 10 | The ``benchmarks/`` directory contains microbenchmarks used to test ``fat`` and 11 | ``fatoptimizer`` performances. 12 | 13 | 14 | Function inlining and specialization using the parameter type 15 | ============================================================= 16 | 17 | Optimize:: 18 | 19 | def _get_sep(path): 20 | if isinstance(path, bytes): 21 | return b'/' 22 | else: 23 | return '/' 24 | 25 | def isabs(s): 26 | """Test whether a path is absolute""" 27 | sep = _get_sep(s) 28 | return s.startswith(sep) 29 | 30 | to:: 31 | 32 | def isabs(s): 33 | return s.startswith('/') 34 | 35 | but only if *s* parameter is a string. 36 | 37 | 2016-01-21:: 38 | 39 | original isabs() bytecode: 488 ns 40 | _get_sep() inlined in isabs(): 268 ns (-220 ns, -45.0%, 1.8x faster :-)) 41 | 42 | 2015-10-21:: 43 | 44 | $ ./python -m timeit 'import posixpath; isabs = posixpath.isabs' 'isabs("/root")' 45 | 1000000 loops, best of 3: 0.939 usec per loop 46 | $ ./python -F -m timeit 'import posixpath; isabs = posixpath.isabs' 'isabs("/root")' 47 | 1000000 loops, best of 3: 0.755 usec per loop 48 | 49 | Script: ``benchmarks/bench_posixpath.py``. 50 | 51 | 52 | Move invariant out of loop (list.append) 53 | ======================================== 54 | 55 | Optimize:: 56 | 57 | def func(obj, data): 58 | for item in data: 59 | obj.append(item) 60 | 61 | to:: 62 | 63 | def func(obj, data): 64 | append = obj.append 65 | for item in data: 66 | append(item) 67 | 68 | 69 | 2016-01-21:: 70 | 71 | range(10 ** 0) 72 | - original bytecode: 297 ns 73 | - append=obj.append with guards: 310 ns (+13 ns, +4.4%, 1.0x slower :-() 74 | - append=obj.append: 306 ns (+9 ns, +3.1%, 1.0x slower :-() 75 | range(10 ** 1) 76 | - original bytecode: 972 ns 77 | - append=obj.append with guards: 703 ns (-268 ns, -27.6%, 1.4x faster :-)) 78 | - append=obj.append: 701 ns (-271 ns, -27.9%, 1.4x faster :-)) 79 | range(10 ** 3) 80 | - original bytecode: 72.2 us 81 | - append=obj.append with guards: 43.8 us (-28.4 us, -39.4%, 1.6x faster :-)) 82 | - append=obj.append: 43.7 us (-28.6 us, -39.5%, 1.7x faster :-)) 83 | range(10 ** 5) 84 | - original bytecode: 8131.5 us 85 | - append=obj.append with guards: 5289.5 us (-2842.0 us, -35.0%, 1.5x faster :-)) 86 | - append=obj.append: 5294.2 us (-2837.4 us, -34.9%, 1.5x faster :-)) 87 | 88 | 2015-10-21:: 89 | 90 | $ ./python bench.py 91 | regular python: range(1)-> 502 ns 92 | regular python: range(10)-> 1.7 us 93 | regular python: range(10**3)-> 122.0 us 94 | regular python: range(10**5)-> 8.5 ms 95 | 96 | $ ./python -F bench.py 97 | fat python: range(1)-> 479 ns (-5%) 98 | fat python: range(10)-> 1.1 us (-35%) 99 | fat python: range(10**3)-> 65.2 us (-47%) 100 | fat python: range(10**5)-> 5.3 ms (-38%) 101 | 102 | Script: ``benchmarks/bench_list_append.py``. 103 | 104 | 105 | Call builtin 106 | ============ 107 | 108 | Optimize:: 109 | 110 | def func(): 111 | return len("abc") 112 | 113 | to:: 114 | 115 | def func(): 116 | return 3 117 | 118 | 2015-01-21 (best timing of 5 runs): 119 | 120 | =============================== ====== 121 | Test Perf 122 | =============================== ====== 123 | Original bytecode (call len) 116 ns 124 | return 3 with guard on builtins 90 ns 125 | return 3 79 ns 126 | =============================== ====== 127 | 128 | GuardBuiltins has a cost of 11 ns. 129 | 130 | Script: ``benchmarks/bench_len_abc.py``. 131 | 132 | 133 | Copy builtin function to constant 134 | ================================= 135 | 136 | Optimize:: 137 | 138 | def func(obj): 139 | return len(obj) 140 | 141 | to:: 142 | 143 | def func(obj): 144 | return 'LEN'(obj) 145 | func.__code__ = fat.replace_consts(func.__code__, {'LEN': len}) 146 | 147 | 2015-01-21 (best timing of 5 runs): 148 | 149 | ================================= ====== 150 | Test Perf 151 | ================================= ====== 152 | Original bytecode (LOAD_GLOBAL) 121 ns 153 | LOAD_CONST with guard on builtins 116 ns 154 | LOAD_CONST 105 ns 155 | ================================= ====== 156 | 157 | GuardBuiltins has a cost of 11 ns. 158 | 159 | Script: ``benchmarks/bench_copy_builtin_to_cst.py``. 160 | 161 | 162 | Copy global function to constant 163 | ================================ 164 | 165 | Optimize:: 166 | 167 | mylen = len 168 | 169 | def func(obj): 170 | return len(obj) 171 | 172 | to:: 173 | 174 | mylen = len 175 | 176 | def func(obj): 177 | return 'MYLEN'(obj) 178 | func.__code__ = fat.replace_consts(func.__code__, {'MYLEN': len}) 179 | 180 | 2015-01-21 (best timing of 5 runs): 181 | 182 | ================================= ====== 183 | Test Perf 184 | ================================= ====== 185 | Original bytecode (LOAD_GLOBAL) 115 ns 186 | LOAD_CONST with guard on globals 112 ns 187 | LOAD_CONST 105 ns 188 | ================================= ====== 189 | 190 | GuardGlobals has a cost of 7 ns. 191 | 192 | Script: ``benchmarks/bench_copy_global_to_cst.py``. 193 | 194 | 195 | Cost of guards 196 | ============== 197 | 198 | Cost of GuardDict guard. 199 | 200 | 2016-01-21:: 201 | 202 | no guard: 81 ns 203 | with 1000 guards on globals: 3749 ns 204 | cost of 1000 guards: 3667 ns (4503.4%) 205 | average cost of 1 guard: 4 ns (4.5%) 206 | 207 | no guard: 82 ns 208 | with 100 guards on globals: 419 ns 209 | cost of 100 guards: 338 ns (414.6%) 210 | average cost of 1 guard: 3 ns (4.1%) 211 | 212 | no guard: 81 ns 213 | with 10 guards on globals: 117 ns 214 | cost of 10 guards: 36 ns (43.9%) 215 | average cost of 1 guard: 4 ns (4.4%) 216 | 217 | no guard: 82 ns 218 | with 1 guards on globals: 87 ns 219 | cost of 1 guards: 5 ns (6.5%) 220 | average cost of 1 guard: 5 ns (6.5%) 221 | 222 | 2016-01-06:: 223 | 224 | no guard: 431 ns 225 | with 1000 guards on globals: 7974 ns 226 | cost of 1000 guards: 7542 ns (1748.1%) 227 | average cost of 1 guard: 8 ns (1.7%) 228 | 229 | no guard: 429 ns 230 | with 100 guards on globals: 1197 ns 231 | cost of 100 guards: 768 ns (179.0%) 232 | average cost of 1 guard: 8 ns (1.8%) 233 | 234 | no guard: 426 ns 235 | with 10 guards on globals: 515 ns 236 | cost of 10 guards: 89 ns (20.8%) 237 | average cost of 1 guard: 9 ns (2.1%) 238 | 239 | no guard: 430 ns 240 | with 1 guards on globals: 449 ns 241 | cost of 1 guards: 19 ns (4.5%) 242 | average cost of 1 guard: 19 ns (4.5%) 243 | 244 | Script: ``benchmarks/bench_guards.py``. 245 | -------------------------------------------------------------------------------- /doc/misc.rst: -------------------------------------------------------------------------------- 1 | ++++ 2 | Misc 3 | ++++ 4 | 5 | Implementation 6 | ============== 7 | 8 | Steps and stages 9 | ---------------- 10 | 11 | The optimizer is splitted into multiple steps. Each optimization has its own 12 | step: fatoptimizer.const_fold.ConstantFolding implements for example constant 13 | folding. 14 | 15 | The function optimizer is splitted into two stages: 16 | 17 | * stage 1: run steps which don't require function specialization 18 | * stage 2: run steps which can add guard and specialize the function 19 | 20 | Main classes: 21 | 22 | * ModuleOptimizer: Optimizer for ast.Module nodes. It starts by looking for 23 | :ref:`__fatoptimizer__ configuration `. 24 | * FunctionOptimizer: Optimizer for ast.FunctionDef nodes. It starts by running 25 | FunctionOptimizerStage1. 26 | * Optimizer: Optimizer for other AST nodes. 27 | 28 | Steps used by ModuleOptimizer, Optimizer and FunctionOptimizerStage1: 29 | 30 | * NamespaceStep: populate a Namespace object which tracks the local variables, 31 | used by ConstantPropagation 32 | * ReplaceBuiltinConstant: replace builtin optimization 33 | * ConstantPropagation: constant propagation optimization 34 | * ConstantFolding: constant folding optimization 35 | * RemoveDeadCode: dead code elimitation optimization 36 | 37 | Steps used by FunctionOptimizer: 38 | 39 | * NamespaceStep: populate a Namespace object which tracks the local variables 40 | * UnrollStep: loop unrolling optimization 41 | * CallPureBuiltin: call builtin optimization 42 | * CopyBuiltinToConstantStep: copy builtins to constants optimization 43 | 44 | Some optimizations produce a new AST tree which must be optimized again. For 45 | example, loop unrolling produces new nodes like "i = 0" and duplicates the loop 46 | body which uses "i". We need to rerun the optimizer on this new AST tree to run 47 | optimizations like constant propagation or constant folding. 48 | 49 | 50 | Pure functions 51 | ============== 52 | 53 | A "pure" function is a function with no side effect. 54 | 55 | Example of pure operators: 56 | 57 | * x+y, x-y, x*y, x/y, x//y, x**y for types int, float, complex, bytes, str, 58 | and also tuple and list for x+y 59 | 60 | Example of instructions with side effect: 61 | 62 | * "global var" 63 | 64 | Example of pure function:: 65 | 66 | def mysum(x, y): 67 | return x + y 68 | 69 | Example of function with side effect:: 70 | 71 | global _last_sum 72 | 73 | def mysum(x, y): 74 | global _last_sum 75 | s = x + y 76 | _last_sum = s 77 | return s 78 | 79 | 80 | Constants 81 | ========= 82 | 83 | FAT Python introduced a new AST type: ``ast.Constant``. The optimizer starts by 84 | converting ``ast.NameConstant``, ``ast.Num``, ``ast.Str``, ``ast.Bytes`` and 85 | ``ast.Tuple`` to ``ast.Constant``. Later, it can create constant of other 86 | types. For example, ``frozenset('abc')`` creates a ``frozenset`` constant. 87 | 88 | Supported constants: 89 | 90 | * ``None`` singleton 91 | * ``bool``: ``True`` and ``False`` 92 | * numbers: ``int``, ``float``, ``complex`` 93 | * strings: ``bytes``, ``str`` 94 | * containers: ``tuple``, ``frozenset`` 95 | 96 | 97 | Literals 98 | ======== 99 | 100 | Literals are a superset of constants. 101 | 102 | Supported literal types: 103 | 104 | * (all constant types) 105 | * containers: ``list``, ``dict``, ``set`` 106 | 107 | 108 | FunctionOptimizer 109 | ================= 110 | 111 | ``FunctionOptimizer`` handles ``ast.FunctionDef`` and emits a specialized 112 | function if a call to a builtin function can be replaced with its result. 113 | 114 | For example, this simple function:: 115 | 116 | def func(): 117 | return chr(65) 118 | 119 | is optimized to:: 120 | 121 | def func(): 122 | return chr(65) 123 | 124 | _ast_optimized = func 125 | 126 | def func(): 127 | return "A" 128 | _ast_optimized.specialize(func, 129 | [{'guard_type': 'builtins', 'names': ('chr',)}]) 130 | 131 | func = _ast_optimized 132 | del _ast_optimized 133 | 134 | 135 | Detection of free variables 136 | =========================== 137 | 138 | VariableVisitor detects local and global variables of an ``ast.FunctionDef`` 139 | node. It is used by the ``FunctionOptimizer`` to detect free variables. 140 | 141 | 142 | Corner cases 143 | ============ 144 | 145 | Calling the ``super()`` function requires a cell variables. 146 | -------------------------------------------------------------------------------- /doc/semantics.rst: -------------------------------------------------------------------------------- 1 | ++++++++++++++++++++++++++++++++ 2 | Python semantics and Limitations 3 | ++++++++++++++++++++++++++++++++ 4 | 5 | fatoptimizer bets that the Python code is not modified when modules are loaded, 6 | but only later, when functions and classes are executed. If this assumption is 7 | wrong, fatoptimizer changes the semantics of Python. 8 | 9 | .. _semantics: 10 | 11 | Python semantics 12 | ================ 13 | 14 | It is very hard, to not say impossible, to implementation and keep the exact 15 | behaviour of regular CPython. CPython implementation is used as the Python 16 | "standard". Since CPython is the most popular implementation, a Python 17 | implementation must do its best to mimic CPython behaviour. We will call it the 18 | Python semantics. 19 | 20 | fatoptimizer should not change the Python semantics with the default 21 | configuration. Optimizations modifying the Python semantics must be disabled 22 | by default: opt-in options. 23 | 24 | As written above, it's really hard to mimic exactly CPython behaviour. For 25 | example, in CPython, it's technically possible to modify local variables of a 26 | function from anywhere, a function can modify its caller, or a thread B can 27 | modify a thread A (just for fun). See `Everything in Python is mutable 28 | `_ for more information. 29 | It's also hard to support all introspections features like ``locals()`` 30 | (``vars()``, ``dir()``), ``globals()`` and 31 | ``sys._getframe()``. 32 | 33 | Builtin functions replaced in the middle of a function 34 | ====================================================== 35 | 36 | fatoptimizer uses :ref:`guards ` to disable specialized function when 37 | assumptions made to optimize the function are no more true. The problem is that 38 | guard are only called at the entry of a function. For example, if a specialized 39 | function ensures that the builtin function ``chr()`` was not modified, but 40 | ``chr()`` is modified during the call of the function, the specialized function 41 | will continue to call the old ``chr()`` function. 42 | 43 | The :ref:`copy builtin functions to constants ` 44 | optimization changes the Python semantics. If a builtin function is replaced 45 | while the specialized function is optimized, the specialized function will 46 | continue to use the old builtin function. For this reason, the optimization 47 | is disabled by default. 48 | 49 | Example:: 50 | 51 | def func(arg): 52 | x = chr(arg) 53 | 54 | with unittest.mock.patch('builtins.chr', result='mock'): 55 | y = chr(arg) 56 | 57 | return (x == y) 58 | 59 | If the :ref:`copy builtin functions to constants 60 | ` optimization is used on this function, the 61 | specialized function returns ``True``, whereas the original function returns 62 | ``False``. 63 | 64 | It is possible to work around this limitation by adding the following 65 | :ref:`configuration ` at the top of the file:: 66 | 67 | __fatoptimizer__ = {'copy_builtin_to_constant': False} 68 | 69 | But the following use cases works as expected in FAT mode:: 70 | 71 | import unittest.mock 72 | 73 | def func(): 74 | return chr(65) 75 | 76 | def test(): 77 | print(func()) 78 | with unittest.mock.patch('builtins.chr', return_value="mock"): 79 | print(func()) 80 | 81 | Output:: 82 | 83 | A 84 | mock 85 | 86 | The ``test()`` function doesn't use the builtin ``chr()`` function. 87 | The ``func()`` function checks its guard on the builtin ``chr()`` function only 88 | when it's called, so it doesn't use the specialized function when ``chr()`` 89 | is mocked. 90 | 91 | 92 | Guards on builtin functions 93 | =========================== 94 | 95 | When a function is specialized, the specialization is ignored if a builtin 96 | function was replaced after the end of the Python initialization. Typically, 97 | the end of the Python initialization occurs just after the execution of the 98 | ``site`` module. It means that if a builtin is replaced during Python 99 | initialization, a function will be specialized even if the builtin is not the 100 | expected builtin function. 101 | 102 | Example:: 103 | 104 | import builtins 105 | 106 | builtins.chr = lambda: mock 107 | 108 | def func(): 109 | return len("abc") 110 | 111 | In this example, the ``func()`` is optimized, but the function is *not* 112 | specialize. The internal call to ``func.specialize()`` is ignored because the 113 | ``chr()`` function was replaced after the end of the Python initialization. 114 | 115 | 116 | Guards on type dictionary and global namespace 117 | =============================================== 118 | 119 | For other guards on dictionaries (type dictionary, global namespace), the guard 120 | uses the current value of the mapping. It doesn't check if the dictionary value 121 | was "modified". 122 | 123 | 124 | Tracing and profiling 125 | ===================== 126 | 127 | Tracing and profiling works in FAT mode, but the exact control flow and traces 128 | are different in regular and FAT mode. For example, :ref:`loop unrolling 129 | ` removes the call to ``range(n)``. 130 | 131 | See ``sys.settrace()`` and ``sys.setprofiling()`` functions. 132 | 133 | 134 | Expected limitations 135 | ==================== 136 | 137 | Function inlining optimization makes debugging more complex: 138 | 139 | * sys.getframe() 140 | * locals() 141 | * pdb 142 | * etc. 143 | * don't work as expected anymore 144 | 145 | Bugs, shit happens: 146 | 147 | * Missing guard: specialized function is called even if the "environment" 148 | was modified 149 | 150 | FAT python! Memory vs CPU, fight! 151 | 152 | * Memory footprint: loading two versions of a function is memory uses more 153 | memory 154 | * Disk usage: .pyc will be more larger 155 | 156 | Possible worse performance: 157 | 158 | * guards adds an overhead higher than the optimization of the specialized code 159 | * specialized code may be slower than the original bytecode 160 | -------------------------------------------------------------------------------- /doc/todo.rst: -------------------------------------------------------------------------------- 1 | .. _todo: 2 | 3 | ++++++++++++++++++++++ 4 | fatoptimizer TODO list 5 | ++++++++++++++++++++++ 6 | 7 | 8 | Easy issues, for new contributors 9 | ================================= 10 | 11 | * Complete fatoptimizer/methods.py to support more pure methods. 12 | * Complete fatoptimizer/builtins.py to support more pure builtin functions. 13 | 14 | 15 | Goal 16 | ==== 17 | 18 | To get visible performance gain, the following optimizations must be 19 | implemented: 20 | 21 | * Function inlining 22 | * Detect and call pure functions 23 | * Elimination of unused variables (set but never read): the constant 24 | propagation and loop unrolling create many of them. For example, 25 | replace "def f(): x=1; return x" with "def f(): return 1" 26 | * Copy constant global variable to function globals 27 | * Specialization for argument types: move invariant out of loops. 28 | Ex: create a bounded method "obj_append = obj.append" out of the loop. 29 | 30 | Even if many optimizations can be implemented with a static optimizers, it's 31 | still not a JIT compiler. A JIT compiler is required to implement even more 32 | optimizations. 33 | 34 | 35 | Known Bugs 36 | ========== 37 | 38 | * ``import *`` is ignored 39 | * Usage of locals() or vars() must disable optimization. Maybe only when the 40 | optimizer produces new variables? 41 | 42 | 43 | Search ideas of new optimizations 44 | ================================= 45 | 46 | * `python.org wiki: PythonSpeed/PerformanceTips 47 | `_ 48 | * `Open issues of type Performance 49 | `_ 50 | * `Closed issues of type Performance 51 | `_ 52 | * `Unladen Swallow ProjectPlan 53 | `_ 54 | * Ideas from PyPy, Pyston, Numba, etc. 55 | 56 | 57 | More optimizations 58 | ================== 59 | 60 | MUST HAVE 61 | --------- 62 | 63 | More complex to implement (without breaking Python semantics). 64 | 65 | * Remove useless temporary variables. Example: 66 | 67 | Code:: 68 | 69 | def func(): 70 | res = 1 71 | return res 72 | 73 | Constant propagation:: 74 | 75 | def func(): 76 | res = 1 77 | return 1 78 | 79 | Remove *res* local variable:: 80 | 81 | def func(): 82 | return 1 83 | 84 | Maybe only for simple types (int, str). It changes object lifetime: 85 | https://bugs.python.org/issue2181#msg63090 86 | 87 | * Function inlining: see `Issue #10399 `_, 88 | AST Optimization: inlining of function calls 89 | 90 | * Inline calls to all functions, short or not? Need guards on these functions 91 | and the global namespace. Example: posixpath._get_sep(). 92 | 93 | * Call pure functions of math, struct and string modules. 94 | Example: replace math.log(32) / math.log(2) with 5.0. 95 | 96 | 97 | Pure functions 98 | -------------- 99 | 100 | * Compute if a function is pure. See pythran.analysis.PureFunctions of pythran 101 | project, depend on ArgumentEffects and GlobalEffects analysys 102 | 103 | 104 | Random 105 | ------ 106 | 107 | Easy to implement. 108 | 109 | * [Python-ideas] (FAT Python) Convert keyword arguments to positional? 110 | https://mail.python.org/pipermail/python-ideas/2016-January/037874.html 111 | 112 | * Loop unrolling: support multiple targets:: 113 | 114 | for x, y in ((1, 2), (3, 4)): 115 | print(x, y) 116 | 117 | * Tests: 118 | 119 | - ``if a: if b: code`` => ``if a and b: code`` 120 | 121 | * Optimize ``str%args`` and ``bytes%args`` 122 | 123 | * Constant folding: 124 | 125 | * replace get_constant() with get_literal()? 126 | 127 | - list + list 128 | - frozenset | frozenset 129 | - set | set 130 | 131 | * 2.0j ** 3.0 132 | * 1 < 2 < 3 133 | * ``if x and True: pass`` => ``if x: pass`` 134 | http://bugs.python.org/issue7682 135 | * replace '(a and b) and c' (2 op) with 'a and b and c' (1 op), 136 | same for "or" operator 137 | 138 | * Specialize also AsyncFunctionDef (run stage 2, not only stage 1) 139 | 140 | 141 | Can be done later 142 | ----------------- 143 | 144 | Unknown speedup, easy to medium to implement. 145 | 146 | * Replace dict(...) with {...} (dict literal): 147 | https://doughellmann.com/blog/2012/11/12/the-performance-impact-of-using-dict-instead-of-in-cpython-2-7-2/ 148 | 149 | * Use SimplifyIterable on dict/frozenset argument 150 | 151 | * print(): convert arguments to strings 152 | 153 | * Remove dead code: remove "pass; pass" 154 | 155 | * Simplify iteratable: 156 | 157 | - for x in set("abc"): ... => for x in frozenset("abc"): ... 158 | Need a guard on set builtin 159 | 160 | - for x in "abc": ... => for x in ("a", "b", "c"): ... 161 | Is it faster? Does it use less memory? 162 | 163 | - at least, loop unrolling must work on "for x in 'abc': ..." 164 | 165 | 166 | Can be done later and are complex 167 | --------------------------------- 168 | 169 | Unknown speedup, complex to implement. 170 | 171 | * Remove "if 0: yield" but tag FunctionDef as a generator? 172 | 173 | * Implement CALL_METHOD bytecode, but execute the following code correctly 174 | (output must be 1, 2 and not 1, 1):: 175 | 176 | class C(object): 177 | def foo(self): 178 | return 1 179 | c = c() 180 | print c.foo() 181 | c.foo = lambda: 2 182 | print c.foo() 183 | 184 | Need a guard on C.foo? 185 | 186 | See https://bugs.python.org/issue6033#msg95707 187 | 188 | Is it really possible? FAT Python doesn't support guards on the instance 189 | dict, it's more designed to use guards on the type dict. 190 | 191 | * Optimize 'lambda: chr(65)'. Lambda are functions, but defined as expressions. 192 | It's not easy to inject the func.specialize() call, 193 | func.__code__.replace_consts() call, etc. Maybe only optimize in some 194 | specific cases? 195 | 196 | Specialization of nested function was disabled because the cost to 197 | specialize the function can be higher than the speedup if the function 198 | is called once and then destroyed. 199 | 200 | * Enable copy builtins to constants when we know that builtins and globals are 201 | not modified. Need to ensure that the function is pure and only calls pure 202 | functions. 203 | 204 | * Move invariant out of loops using guards on argument types: 205 | 206 | - Merge duplicate LOAD_ATTR, when we can make sure that the attribute will 207 | not be modified 208 | - list.append: only for list type 209 | 210 | * Loop unrolling: 211 | 212 | - support break and continue 213 | - support raise used outside try/except 214 | 215 | * Constant propagation, copy accross namespaces: 216 | 217 | - list-comprehension has its own separated namespace:: 218 | 219 | n = 100 220 | seq = [randrange(n) for i in range(n)] 221 | 222 | - copy globals to locals: need a guard on globals 223 | 224 | * Convert naive loop to list/dict/set comprehension. 225 | Replace "x=[]; for item in data: x.append(item.upper())" 226 | with "x=[item.upper() for item in data]". Same for x=set() and x={}. 227 | 228 | * Call more builtin functions: 229 | 230 | - all(), any() 231 | - enumerate(iterable), zip() 232 | - format() 233 | - filter(pred, iterable), map(pred, iterable), reversed() 234 | 235 | * operator module: 236 | 237 | - need to add an import, need to ensure that operator name is not used 238 | - lambda x: x[1] => operator.itemgetter(1) 239 | - lambda x: x.a => operator.attrgetter('a') 240 | - lambda x: x.f('a', b=1) => operator.methodcaller('f', 'a', b=1) 241 | 242 | * map, itertools.map, filter: 243 | 244 | - [f(x) for x in a] => map(f, a) / list(map(f, a)) 245 | - (f(x) for x in a) => itertools.map(f, a) / map(f, a) ? scope ? 246 | - (x for x in a if f(x)) => filter(f, a) 247 | - (x for x in a if not f(x)) => __builtin_filternot__(f, a) ? 248 | - (2 * x for x in a) => map((2).__mul__, a) 249 | - (x for x in a if x in 'abc') => filter('abc'.__contains__, a) 250 | 251 | 252 | 253 | Profiling 254 | ========= 255 | 256 | * implement code to detect the exact type of function parameters and function 257 | locals and save it into an annotation file 258 | * implement profiling directed optimization: benchmark guards at runtime 259 | to decide if it's worth to use a specialized function. Measure maybe also 260 | the memory footprint using tracemalloc? 261 | * implement basic stategy to decide if specialized function must be emitted 262 | or not using raw estimation, like the size of the bytecode in bytes 263 | 264 | 265 | 266 | Later 267 | ===== 268 | 269 | * efficient optimizations on objects, not only simple functions 270 | * handle python modules and python imports 271 | 272 | - checksum of the .py content? 273 | - how to handle C extensions? checksum of the .so file? 274 | - how to handle .pyc files? 275 | 276 | * find an efficient way to specialize nested functions 277 | * configuration to manually help the optimizer: 278 | 279 | - give a whitelist of "constants": app.DEBUG, app.enum.BLUE, ... 280 | - type hint with strict types: x is Python int in range [3; 10] 281 | - expect platform values to be constant: sys.version_info, sys.maxunicode, 282 | os.name, sys.platform, os.linesep, etc. 283 | - declare pure functions 284 | - see fatoptimizer for more ideas 285 | 286 | * Restrict the number of guards, number of specialized bytecode, number 287 | of arg_type types with fatoptimizer.Config 288 | * fatoptimizer.VariableVisitor: support complex assignments like 289 | 'type(mock)._mock_check_sig = checksig' 290 | * Support specialized CFunction_Type, not only specialized bytecode? 291 | * Add an opt-in option to skip some guards if the user knows that the 292 | application will never modify function __code__, override builtin methods, 293 | modify a constant, etc. 294 | * Optimize real objects, not only simple functions. For example, inline a 295 | method. 296 | * Function parameter: support more complex guard to complex types like 297 | list of integers? 298 | * handle default argument values for argument type guards? 299 | * Support locals()[key], vars()[key], globals()[key]? 300 | * Support decorators 301 | * Copy super() builtin to constants doesn't work. Calling the builtin super() 302 | function creates a free variable, whereas calling the constant doesn't 303 | create a free variable. 304 | * Tail-call recursion? 305 | 306 | def factorial(n): 307 | if n > 1: 308 | return n * factorial(n-1) 309 | else: 310 | return 1 311 | 312 | 313 | Support decorator 314 | ================= 315 | 316 | weakref.py:: 317 | 318 | @property 319 | def atexit(self): 320 | """Whether finalizer should be called at exit""" 321 | info = self._registry.get(self) 322 | return bool(info) and info.atexit 323 | 324 | @atexit.setter 325 | def atexit(self, value): 326 | info = self._registry.get(self) 327 | if info: 328 | info.atexit = bool(value) 329 | 330 | It's not possible to replace it with:: 331 | 332 | def atexit(self): 333 | """Whether finalizer should be called at exit""" 334 | info = self._registry.get(self) 335 | return bool(info) and info.atexit 336 | atexit = property(atexit) 337 | 338 | def atexit(self, value): 339 | info = self._registry.get(self) 340 | if info: 341 | info.atexit = bool(value) 342 | atexit = atexit.setter(atexit) 343 | 344 | The last line 'atexit = atexit.setter(atexit)' because 'atexit' is now 345 | the second function, not more the first decorated function (the property). 346 | 347 | Define the second atexit under a different name? No! It changes the code name, 348 | which is wrong. 349 | 350 | Maybe we can replace it with:: 351 | 352 | def atexit(self): 353 | """Whether finalizer should be called at exit""" 354 | info = self._registry.get(self) 355 | return bool(info) and info.atexit 356 | atexit = property(atexit) 357 | 358 | _old_atexit = atexit 359 | def atexit(self, value): 360 | info = self._registry.get(self) 361 | if info: 362 | info.atexit = bool(value) 363 | atexit = _old_atexit.setter(atexit) 364 | 365 | But for this, we need to track the namespace during the optimization. The 366 | VariableVisitor in run *before* the optimizer, it doesn't track the namespace 367 | at the same time. 368 | 369 | 370 | Possible optimizations 371 | ====================== 372 | 373 | Short term: 374 | 375 | * Function func2() calls func1() if func1() is pure: inline func1() 376 | into func2() 377 | * Call builtin pure functions during compilation. Example: replace len("abc") 378 | with 3 or range(3) with (0, 1, 2). 379 | * Constant folding: replace a variable with its value. We may do that for 380 | optimal parameters with default value if these parameters are not set. 381 | Example: replace app.DEBUG with False. 382 | 383 | Using types: 384 | 385 | * Detect the exact type of parameters and function local variables 386 | * Specialized code relying on the types. For example, move invariant out of 387 | loops (ex: obj.append for list). 388 | * x + 0 gives a TypeError for str, but can be replaced with x for int and 389 | float. Same optimization for x*0. 390 | * See astoptimizer for more ideas. 391 | 392 | Longer term: 393 | 394 | * Compile to machine code using Cython, Numba, PyPy, etc. Maybe only for 395 | numeric types at the beginning? Release the GIL if possible, but check 396 | "sometimes" if we got UNIX signals. 397 | -------------------------------------------------------------------------------- /fatoptimizer/__init__.py: -------------------------------------------------------------------------------- 1 | from .tools import pretty_dump, OptimizerError 2 | from .config import Config 3 | from .optimizer import ModuleOptimizer as _ModuleOptimizer 4 | import sys 5 | 6 | 7 | __version__ = '0.3' 8 | 9 | 10 | def optimize(tree, filename, config): 11 | optimizer = _ModuleOptimizer(config, filename) 12 | return optimizer.optimize(tree) 13 | 14 | 15 | class FATOptimizer: 16 | name = "fat" 17 | 18 | def __init__(self, config): 19 | self.config = config 20 | 21 | def ast_transformer(self, tree, context): 22 | filename = context.filename 23 | if sys.flags.verbose and not filename.startswith('<'): 24 | print("# run fatoptimizer on %s" % filename, file=sys.stderr) 25 | return optimize(tree, context.filename, self.config) 26 | 27 | 28 | def _register(): 29 | # First, import the fat module to create the copy of the builtins dict 30 | import fat 31 | 32 | import sys 33 | 34 | config = Config() 35 | config.enable_all() 36 | if sys.flags.verbose: 37 | config.logger = sys.stderr 38 | 39 | transformers = sys.get_code_transformers() 40 | # add the FAT optimizer before the peephole optimizer 41 | transformers.insert(0, FATOptimizer(config)) 42 | sys.set_code_transformers(transformers) 43 | -------------------------------------------------------------------------------- /fatoptimizer/base_optimizer.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | from .namespace import Namespace 4 | from .tools import NodeTransformer 5 | 6 | 7 | _COND_BLOCK = {ast.If, ast.For, ast.While, ast.Try} 8 | 9 | 10 | class BaseOptimizer(NodeTransformer): 11 | def __init__(self, filename): 12 | super().__init__(filename) 13 | self.namespace = Namespace() 14 | 15 | def _visit_attr(self, parent_node, attr_name, node): 16 | parent_type = type(parent_node) 17 | if (parent_type in _COND_BLOCK 18 | and attr_name != "finalbody" 19 | and not(attr_name == "test" and parent_type == ast.If)): 20 | with self.namespace.cond_block(): 21 | return self.visit(node) 22 | else: 23 | return self.visit(node) 24 | 25 | def _run_new_optimizer(self, node): 26 | optimizer = BaseOptimizer() 27 | return optimizer.visit(node) 28 | 29 | def fullvisit_FunctionDef(self, node): 30 | return self._run_new_optimizer(node) 31 | 32 | def fullvisit_AsyncFunctionDef(self, node): 33 | return self._run_new_optimizer(node) 34 | 35 | def fullvisit_ClassDef(self, node): 36 | return self._run_new_optimizer(node) 37 | 38 | def fullvisit_DictComp(self, node): 39 | return self._run_new_optimizer(node) 40 | 41 | def fullvisit_ListComp(self, node): 42 | return self._run_new_optimizer(node) 43 | 44 | def fullvisit_SetComp(self, node): 45 | return self._run_new_optimizer(node) 46 | 47 | def fullvisit_GeneratorExp(self, node): 48 | return self._run_new_optimizer(node) 49 | 50 | def fullvisit_Lambda(self, node): 51 | return self._run_new_optimizer(node) 52 | -------------------------------------------------------------------------------- /fatoptimizer/benchmark.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tools used by microbenchmarks. 3 | """ 4 | 5 | import sys 6 | import timeit 7 | 8 | 9 | def bench(stmt, *, setup='', repeat=10**5, number=10): 10 | caller_globals = sys._getframe(1).f_globals 11 | timer = timeit.Timer(stmt, setup=setup, globals=caller_globals) 12 | return min(timer.repeat(repeat=repeat, number=number)) / number 13 | 14 | 15 | def format_dt(dt, sign=False): 16 | if abs(dt) > 10e-3: 17 | if sign: 18 | return "%+.1f ms" % (dt*1e3) 19 | else: 20 | return "%.1f ms" % (dt*1e3) 21 | elif abs(dt) > 10e-6: 22 | if sign: 23 | return "%+.1f us" % (dt*1e6) 24 | else: 25 | return "%.1f us" % (dt*1e6) 26 | else: 27 | if sign: 28 | return "%+.0f ns" % (dt*1e9) 29 | else: 30 | return "%.0f ns" % (dt*1e9) 31 | 32 | 33 | def compared_dt(specialized_dt, original_dt): 34 | percent = (specialized_dt - original_dt) * 100 / original_dt 35 | ratio = original_dt / specialized_dt 36 | if ratio >= 1.0: 37 | what = 'faster :-)' 38 | else: 39 | what = 'slower :-(' 40 | return ('%s (%s, %+.1f%%, %.1fx %s)' 41 | % (format_dt(specialized_dt), 42 | format_dt(specialized_dt - original_dt, sign=True), 43 | percent, ratio, what)) 44 | -------------------------------------------------------------------------------- /fatoptimizer/bltin_const.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | from .tools import OptimizerStep 4 | 5 | 6 | CONSTANTS = { 7 | '__debug__': __debug__, 8 | } 9 | 10 | 11 | class ReplaceBuiltinConstant(OptimizerStep): 12 | def visit_Name(self, node): 13 | if not self.config.replace_builtin_constant: 14 | return 15 | if not isinstance(node, ast.Name) or not isinstance(node.ctx, ast.Load): 16 | return 17 | 18 | name = node.id 19 | if name not in CONSTANTS: 20 | return 21 | if not self.is_builtin_variable(name): 22 | # constant overriden in the local or in the global namespace 23 | return 24 | 25 | result = CONSTANTS[name] 26 | return self.new_constant(node, result) 27 | -------------------------------------------------------------------------------- /fatoptimizer/builtins.py: -------------------------------------------------------------------------------- 1 | import builtins 2 | import functools 3 | 4 | from .tools import (UNSET, 5 | FLOAT_TYPES, COMPLEX_TYPES, STR_TYPES, ITERABLE_TYPES) 6 | from .const_fold import check_pow 7 | from .pure import PureFunction 8 | 9 | 10 | def _chr_check_args(args): 11 | code_point = args[0] 12 | return 0 <= code_point <= 0x10ffff 13 | 14 | 15 | def _complex_check_args(args): 16 | code_point = args[0] 17 | return 0 <= code_point <= 0x10ffff 18 | 19 | 20 | def _divmod_check_args(args): 21 | # don't divide by zero 22 | return bool(args[1]) 23 | 24 | 25 | def _ord_check_args(args): 26 | string = args[0] 27 | return (len(string) == 1) 28 | 29 | 30 | def _bytes_check_args(args): 31 | arg = args[0] 32 | if not isinstance(arg, tuple): 33 | return True 34 | return all(0 <= item <= 255 for item in arg) 35 | 36 | 37 | def _pow_check_args(config, args): 38 | num = args[0] 39 | exp = args[1] 40 | if len(args) >= 3: 41 | mod = args[2] 42 | else: 43 | mod = None 44 | return check_pow(config, num, exp, mod) 45 | 46 | 47 | def add_pure_builtins(config): 48 | def add(name, *args, **kw): 49 | func = getattr(builtins, name) 50 | pure_func = PureFunction(func, name, *args, **kw) 51 | config._pure_builtins[name] = pure_func 52 | 53 | ANY_TYPE = None 54 | 55 | pow_check_args = functools.partial(_pow_check_args, config) 56 | add('abs', 1, COMPLEX_TYPES) 57 | add('ascii', 1, ANY_TYPE) 58 | add('bin', 1, int) 59 | add('bool', 1, COMPLEX_TYPES + STR_TYPES) 60 | add('bytes', 1, (bytes,) + (tuple,), check_args=_bytes_check_args) 61 | add('chr', 1, int, check_args=_chr_check_args) 62 | # FIXME: optimize also complex(int, int) 63 | add('complex', (1, 2), COMPLEX_TYPES + STR_TYPES, COMPLEX_TYPES, 64 | # catch ValueError for complex('xyz') 65 | # catch TypeError for complex('xyz', 1) 66 | exceptions=(ValueError, TypeError)) 67 | # catch TypeError for unhashable keys 68 | add('dict', (0, 1), ITERABLE_TYPES, exceptions=TypeError) 69 | add('divmod', 2, FLOAT_TYPES, FLOAT_TYPES, check_args=_divmod_check_args) 70 | add('float', 1, FLOAT_TYPES + STR_TYPES, 71 | # catch ValueError for float('xyz') 72 | exceptions=ValueError) 73 | # frozenset(([1, 2, 3],)) raises TypeError: unhashable type 74 | add('frozenset', (0, 1), ITERABLE_TYPES, exceptions=TypeError) 75 | add('hex', 1, int) 76 | add('int', 1, FLOAT_TYPES + STR_TYPES, 77 | # catch ValueError for int('xyz') 78 | exceptions=ValueError) 79 | add('len', 1, ITERABLE_TYPES) 80 | add('list', 1, ITERABLE_TYPES) 81 | add('oct', 1, int) 82 | add('ord', 1, STR_TYPES, check_args=_ord_check_args) 83 | add('max', (1, None), ANY_TYPE, 84 | # catch TypeError for non comparable values 85 | exceptions=TypeError) 86 | add('min', (1, None), ANY_TYPE, 87 | # catch TypeError for non comparable values 88 | exceptions=TypeError) 89 | add('pow', (2, 3), FLOAT_TYPES, FLOAT_TYPES, FLOAT_TYPES, 90 | check_args=pow_check_args, 91 | exceptions=(ValueError, TypeError, OverflowError)) 92 | add('repr', 1, COMPLEX_TYPES + STR_TYPES) 93 | add('round', (1, 2), FLOAT_TYPES, int) 94 | # set(([1, 2, 3],)) raises TypeError: unhashable type 95 | add('set', 1, ITERABLE_TYPES, exceptions=TypeError) 96 | add('str', 1, COMPLEX_TYPES + (str,)) 97 | # sum(): int+list raises TypeError 98 | add('sum', (1, 2), ANY_TYPE, ANY_TYPE, 99 | exceptions=TypeError) 100 | add('tuple', 1, ITERABLE_TYPES) 101 | -------------------------------------------------------------------------------- /fatoptimizer/call_method.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | from .tools import OptimizerStep, UNSET, get_literal 4 | from .specialized import BuiltinGuard 5 | 6 | 7 | class CallPureMethods(OptimizerStep): 8 | """Call methods of builtin types which have no side effect.""" 9 | 10 | def call_method(self, pure_func, obj, node): 11 | value = pure_func.call_method(obj, node) 12 | if value is UNSET: 13 | return 14 | 15 | new_node = self.new_constant(node, value) 16 | if new_node is None: 17 | return 18 | 19 | self.log(node, "call pure method: replace %s with %r", 20 | ast.dump(node), value, add_line=True) 21 | return new_node 22 | 23 | def visit_Call(self, node): 24 | attr = node.func 25 | if not isinstance(attr, ast.Attribute): 26 | return 27 | if not isinstance(attr.value, ast.Constant): 28 | return 29 | method_name = attr.attr 30 | 31 | obj = attr.value.value 32 | value_type = type(obj) 33 | if value_type not in self.config._pure_methods: 34 | return 35 | methods = self.config._pure_methods[value_type] 36 | if method_name not in methods: 37 | return 38 | pure_func = methods[method_name] 39 | 40 | return self.call_method(pure_func, obj, node) 41 | -------------------------------------------------------------------------------- /fatoptimizer/call_pure.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | from .tools import OptimizerStep, UNSET, get_literal 4 | from .specialized import BuiltinGuard 5 | 6 | 7 | class CallPureBuiltin(OptimizerStep): 8 | def call_builtin(self, node, pure_func): 9 | value = pure_func.call_func(node) 10 | if value is UNSET: 11 | return 12 | 13 | new_node = self.new_constant(node, value) 14 | if new_node is None: 15 | return 16 | 17 | self.log(node, "call pure builtin function: replace %s with %r", 18 | ast.dump(node), value, add_line=True) 19 | self.add_guard(BuiltinGuard(node.func.id, 'call builtin')) 20 | return new_node 21 | 22 | def visit_Call(self, node): 23 | func = node.func 24 | 25 | if (isinstance(func, ast.Name) 26 | and func.id in self.config._pure_builtins 27 | and self.is_builtin_variable(func.id)): 28 | pure_func = self.config._pure_builtins[func.id] 29 | new_node = self.call_builtin(node, pure_func) 30 | if new_node is not None: 31 | return new_node 32 | -------------------------------------------------------------------------------- /fatoptimizer/config.py: -------------------------------------------------------------------------------- 1 | import builtins 2 | 3 | from .tools import get_constant_size, ITERABLE_TYPES 4 | 5 | 6 | class Config: 7 | # FIXME: use dir()? 8 | _attributes = ''' 9 | _pure_builtins 10 | _pure_methods 11 | constant_folding 12 | constant_propagation 13 | copy_builtin_to_constant 14 | enabled 15 | inlining 16 | logger 17 | max_bytes_len 18 | max_constant_size 19 | max_int_bits 20 | max_str_len 21 | max_seq_len 22 | remove_dead_code 23 | replace_builtin_constant 24 | simplify_iterable 25 | unroll_loops 26 | '''.strip().split() 27 | 28 | def __init__(self, *, _optimize=True): 29 | # Is the AST optimizer enabled? 30 | self.enabled = True 31 | 32 | # File where logs are written to 33 | self.logger = None 34 | 35 | # Maximum size of a constant in bytes: the constant size is computed 36 | # using the size in bytes of marshal.dumps() output 37 | self.max_constant_size = 128 38 | 39 | # Maximum number of bits of a constant integer. Ignore the sign. 40 | self.max_int_bits = 256 41 | 42 | # Maximum length in bytes of a bytes string. 43 | self.max_bytes_len = self.max_constant_size 44 | 45 | # Maximum length in characters of a Unicode string. 46 | self.max_str_len = self.max_constant_size 47 | 48 | # Maximum length in number of items of a sequence. It is only a 49 | # preliminary check: max_constant_size still applies for sequences. 50 | self.max_seq_len = self.max_constant_size // 4 51 | 52 | # Methods of builtin types which have no side effect. 53 | # 54 | # Mapping: type => method_mapping 55 | # where method_mapping is a mapping: name => PureFunction 56 | self._pure_methods = {} 57 | 58 | # Builtin functions (PureFunction instances) which have no side effect 59 | # and so can be called during the compilation 60 | self._pure_builtins = {} 61 | 62 | # copy a global variable to a local variable, optimized used to load 63 | # builtin functions from slow builtins to fast local variables 64 | # 65 | # This optimizations breaks test_dynamic which explicitly modifies 66 | # builtins in the middle of a generator. 67 | self.copy_builtin_to_constant = False 68 | self._copy_builtin_to_constant = set(dir(builtins)) 69 | 70 | # Loop unrolling (disabled by default): maximum number of loop 71 | # iterations (ex: n in 'for index in range(n):') 72 | self.unroll_loops = 16 73 | 74 | # Constant propagation 75 | self.constant_propagation = True 76 | 77 | # Constant folding 78 | self.constant_folding = True 79 | 80 | # Replace builtin constants (__debug__) 81 | self.replace_builtin_constant = True 82 | 83 | # Simplify iterables? 84 | # Example: replace 'for x in {}: ...' with 'for x in (): ...' 85 | self.simplify_iterable = True 86 | 87 | # Remove dead code? 88 | # Example: "if 0: ..." => "pass" 89 | self.remove_dead_code = True 90 | 91 | # Function inlining 92 | # Disable by default: it's still experimental. 93 | self.inlining = False 94 | 95 | if _optimize: 96 | from .builtins import add_pure_builtins 97 | add_pure_builtins(self) 98 | 99 | from .methods import add_pure_methods 100 | add_pure_methods(self) 101 | 102 | def replace(self, config): 103 | new_config = Config(_optimize=False) 104 | for attr in self._attributes: 105 | if not attr.startswith('_') and attr in config: 106 | value = config[attr] 107 | else: 108 | value = getattr(self, attr) 109 | setattr(new_config, attr, value) 110 | return new_config 111 | 112 | def disable_all(self): 113 | self.max_constant_size = 128 114 | self.max_int_bits = 256 115 | self.max_bytes_len = self.max_constant_size 116 | self.max_str_len = self.max_constant_size 117 | self.max_seq_len = self.max_constant_size // 4 118 | self._pure_builtins = {} 119 | self._pure_methods = {} 120 | self.copy_builtin_to_constant = False 121 | self._copy_builtin_to_constant = set() 122 | self.unroll_loops = 0 123 | self.constant_propagation = False 124 | self.constant_folding = False 125 | self.replace_builtin_constant = False 126 | self.remove_dead_code = False 127 | self.simplify_iterable = False 128 | # inlining is disabled, too experimental and buggy 129 | 130 | def enable_all(self): 131 | self.max_constant_size = 1024 # 1 KB 132 | self.max_int_bits = self.max_constant_size 133 | self.max_bytes_len = self.max_constant_size 134 | self.max_str_len = self.max_constant_size 135 | self.max_seq_len = self.max_constant_size 136 | 137 | self.copy_builtin_to_constant = True 138 | self._copy_builtin_to_constant = set(dir(builtins)) 139 | self.unroll_loops = 256 140 | self.constant_propagation = True 141 | self.constant_folding = True 142 | self.replace_builtin_constant = True 143 | self.remove_dead_code = True 144 | self.simplify_iterable = True 145 | self.inlining = True 146 | 147 | from .builtins import add_pure_builtins 148 | add_pure_builtins(self) 149 | 150 | def check_result(self, value): 151 | if isinstance(value, int): 152 | return (value.bit_length() <= self.max_int_bits) 153 | 154 | if isinstance(value, bytes): 155 | return (len(value) <= self.max_bytes_len) 156 | 157 | if isinstance(value, str): 158 | return (len(value) <= self.max_str_len) 159 | 160 | if isinstance(value, ITERABLE_TYPES) and len(value) > self.max_seq_len: 161 | return False 162 | 163 | size = get_constant_size(value) 164 | return (size <= self.max_constant_size) 165 | -------------------------------------------------------------------------------- /fatoptimizer/const_fold.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import math 3 | import operator 4 | 5 | from .tools import (OptimizerStep, UNSET, 6 | FLOAT_TYPES, COMPLEX_TYPES, ITERABLE_TYPES, 7 | copy_lineno, get_constant, get_constant_size, copy_node, get_literal, 8 | compact_ascii) 9 | 10 | 11 | # set and frozenset don't support indexing 12 | SUBSCRIPT_INDEX_TYPES = tuple(set(ITERABLE_TYPES) - {set, frozenset}) 13 | # set, frozenset, dict are not subscriptable 14 | SUBSCRIPT_SLICE_TYPES = tuple(set(ITERABLE_TYPES) - {set, frozenset, dict}) 15 | SLICE_ARG_TYPES = (int, type(None)) 16 | 17 | 18 | DIVIDE_BINOPS = (ast.Div, ast.FloorDiv, ast.Mod) 19 | 20 | EVAL_BINOP = { 21 | # a + b, a - b, a * b 22 | ast.Add: operator.add, 23 | ast.Sub: operator.sub, 24 | ast.Mult: operator.mul, 25 | # see binop(): floordiv() may be used for int/int on Python 2 26 | ast.Div: operator.truediv, 27 | ast.FloorDiv: operator.floordiv, 28 | ast.Mod: operator.mod, 29 | # a ** b 30 | ast.Pow: operator.pow, 31 | # a << b, a >> b 32 | ast.LShift: operator.lshift, 33 | ast.RShift: operator.rshift, 34 | # a & b, a | b, a ^ b 35 | ast.BitAnd: operator.and_, 36 | ast.BitOr: operator.or_, 37 | ast.BitXor: operator.xor, 38 | } 39 | BINOP_STR = { 40 | ast.Add: '+', 41 | ast.Sub: '-', 42 | ast.Mult: '*', 43 | ast.Div: '/', 44 | ast.FloorDiv: '//', 45 | ast.Mod: '%', 46 | ast.Pow: '**', 47 | ast.LShift: '<<', 48 | ast.RShift: '>>', 49 | ast.BitAnd: '&', 50 | ast.BitOr: '|', 51 | ast.BitXor: '^', 52 | } 53 | 54 | # int: accept all keys of EVAL_BINOP 55 | FLOAT_BINOPS = ( 56 | ast.Add, ast.Sub, 57 | ast.Mult, ast.Div, ast.FloorDiv, ast.Mod, 58 | ast.Pow) 59 | COMPLEX_BINOPS = ( 60 | ast.Add, ast.Sub, 61 | ast.Mult, ast.Div, 62 | ) 63 | 64 | EVAL_UNARYOP = { 65 | # not a, ~a, +a, -a 66 | ast.Not: operator.not_, 67 | ast.Invert: operator.invert, 68 | ast.UAdd: operator.pos, 69 | ast.USub: operator.neg, 70 | } 71 | 72 | NOT_COMPARE = { 73 | ast.In: ast.NotIn, 74 | ast.NotIn: ast.In, 75 | 76 | ast.Is: ast.IsNot, 77 | ast.IsNot: ast.Is, 78 | 79 | # Don't replace 'not(x < y)' with 'x >= y' because both expressions 80 | # can be different. For example, 'not(math.nan < 1.0)' is true, 81 | # whereas 'math.nan >= 1.0' is false. 82 | # 83 | # Don't replace 'not(x == y)' with 'x != y' because 'not x.__eq__(y)' 84 | # can return a different result than 'x.__ne__(y)'. For example, 85 | # a class may implement __eq__() but not __ne__() and the default 86 | # implementation of __ne__() has a different behaviour than 87 | # the class implementation of __eq__(). 88 | } 89 | 90 | EVAL_COMPARE = { 91 | ast.In: lambda obj, seq: obj in seq, 92 | ast.NotIn: lambda obj, seq: obj not in seq, 93 | 94 | ast.Lt: operator.lt, 95 | ast.LtE: operator.le, 96 | ast.Gt: operator.gt, 97 | ast.GtE: operator.ge, 98 | 99 | ast.Eq: operator.eq, 100 | ast.NotEq: operator.ne, 101 | } 102 | 103 | 104 | def check_pow(config, num, exp, mod=None): 105 | if num == 0 and exp < 0: 106 | # 0 ** -1 raises a ZeroDivisionError 107 | return False 108 | 109 | if num < 0 and exp < 1.0 and exp != 0.0: 110 | # pow(-25, 0.5) raises a ValueError 111 | return False 112 | 113 | if mod is not None: 114 | # pow(a, b, m) only works if a and b are integers 115 | if not isinstance(num, int): 116 | return False 117 | if not isinstance(exp, int): 118 | return False 119 | 120 | if mod == 0: 121 | # pow(2, 1024, 0) raises a ValueError: 122 | # 'pow() 3rd argument cannot be 0' 123 | return False 124 | 125 | if (isinstance(num, int) 126 | and isinstance(exp, int) 127 | # don't call log2(0) (error) 128 | and num != 0 129 | # if exp < 0, the result is a float which has a fixed size 130 | and exp > 0): 131 | # bits(num ** exp) = log2(num) * exp 132 | if math.log2(abs(num)) * exp >= config.max_int_bits: 133 | # pow() result will be larger than max_constant_size. 134 | return False 135 | 136 | return True 137 | 138 | 139 | class ConstantFolding(OptimizerStep): 140 | def check_binop(self, op, left, right): 141 | if isinstance(left, COMPLEX_TYPES) and isinstance(right, COMPLEX_TYPES): 142 | if isinstance(op, DIVIDE_BINOPS) and not right: 143 | # x/0: ZeroDivisionError 144 | return False 145 | 146 | if isinstance(op, ast.Pow): 147 | if isinstance(left, complex) or isinstance(right, complex): 148 | return False 149 | 150 | return check_pow(self.config, left, right) 151 | 152 | if isinstance(op, (ast.LShift, ast.RShift)) and right < 0: 153 | # 1 << -3 and 1 >> -3 raise a ValueError 154 | return False 155 | 156 | if isinstance(left, int) and isinstance(right, int): 157 | return True 158 | 159 | if isinstance(left, FLOAT_TYPES) and isinstance(right, FLOAT_TYPES): 160 | return isinstance(op, FLOAT_BINOPS) 161 | 162 | if isinstance(left, COMPLEX_TYPES) and isinstance(right, COMPLEX_TYPES): 163 | return isinstance(op, COMPLEX_BINOPS) 164 | 165 | if isinstance(op, ast.Mult): 166 | if isinstance(right, int): 167 | # bytes * int 168 | if isinstance(left, bytes): 169 | return (len(left) * right <= self.config.max_bytes_len) 170 | # str * int 171 | if isinstance(left, str): 172 | return (len(left) * right <= self.config.max_str_len) 173 | # tuple * int 174 | if isinstance(left, tuple): 175 | size = get_constant_size(left) 176 | return (size * right <= self.config.max_seq_len) 177 | 178 | if isinstance(left, int): 179 | # int * bytes 180 | if isinstance(right, bytes): 181 | return (left * len(right) <= self.config.max_bytes_len) 182 | # int * str 183 | if isinstance(right, str): 184 | return (left * len(right) <= self.config.max_str_len) 185 | # int * tuple 186 | if isinstance(right, tuple): 187 | size = get_constant_size(right) 188 | return (left * size <= self.config.max_seq_len) 189 | 190 | if isinstance(op, ast.Add): 191 | if isinstance(left, str) and isinstance(right, str): 192 | return ((len(left) + len(right)) <= self.config.max_str_len) 193 | 194 | if isinstance(left, bytes) and isinstance(right, bytes): 195 | return ((len(left) + len(right)) <= self.config.max_bytes_len) 196 | 197 | if isinstance(left, tuple) and isinstance(right, tuple): 198 | return ((len(left) + len(right)) <= self.config.max_seq_len) 199 | 200 | return False 201 | 202 | def visit_BinOp(self, node): 203 | if not self.config.constant_folding: 204 | return 205 | 206 | eval_binop = EVAL_BINOP.get(node.op.__class__) 207 | if not eval_binop: 208 | return 209 | 210 | if isinstance(node.op, ast.Mod): 211 | # FIXME: optimize str%args and bytes%args 212 | left_types = COMPLEX_TYPES 213 | else: 214 | left_types = None 215 | 216 | left = get_constant(node.left, types=left_types) 217 | if left is UNSET: 218 | return 219 | 220 | right = get_constant(node.right) 221 | if right is UNSET: 222 | return 223 | 224 | ok = self.check_binop(node.op, left, right) 225 | if not ok: 226 | return 227 | 228 | result = eval_binop(left, right) 229 | new_node = self.new_constant(node, result) 230 | if new_node is None: 231 | return 232 | 233 | op_str = BINOP_STR[node.op.__class__] 234 | self.log(node, "constant folding: replace %s %s %s with %s", 235 | compact_ascii(left), op_str, compact_ascii(right), 236 | compact_ascii(result), add_line=True) 237 | return new_node 238 | 239 | def not_compare(self, node): 240 | compare = node.operand 241 | if len(compare.ops) != 1: 242 | # FIXME: optimize: 'not a <= b <= c' to 'a > b or b > c' 243 | return 244 | 245 | op = compare.ops[0] 246 | try: 247 | op = NOT_COMPARE[op.__class__]() 248 | except KeyError: 249 | return 250 | new_cmp = ast.Compare(left=compare.left, ops=[op], 251 | comparators=compare.comparators) 252 | copy_lineno(compare, new_cmp) 253 | return new_cmp 254 | 255 | def visit_UnaryOp(self, node): 256 | if not self.config.constant_folding: 257 | return 258 | 259 | eval_unaryop = EVAL_UNARYOP.get(node.op.__class__) 260 | if eval_unaryop is None: 261 | return 262 | 263 | if isinstance(node.op, ast.Invert): 264 | types = int 265 | else: 266 | types = COMPLEX_TYPES 267 | 268 | value = get_constant(node.operand, types=types) 269 | if value is not UNSET: 270 | result = eval_unaryop(value) 271 | return self.new_constant(node, result) 272 | 273 | if (isinstance(node.op, ast.Not) 274 | and isinstance(node.operand, ast.Compare)): 275 | new_node = self.not_compare(node) 276 | if new_node is not None: 277 | return new_node 278 | 279 | def subscript_slice(self, node): 280 | value = get_literal(node.value, types=SUBSCRIPT_SLICE_TYPES) 281 | if value is UNSET: 282 | return 283 | 284 | ast_start = node.slice.lower 285 | ast_stop = node.slice.upper 286 | ast_step = node.slice.step 287 | 288 | if ast_start is not None: 289 | start = get_constant(ast_start, types=SLICE_ARG_TYPES) 290 | if start is UNSET: 291 | return 292 | else: 293 | start = None 294 | if ast_stop is not None: 295 | stop = get_constant(ast_stop, types=SLICE_ARG_TYPES) 296 | if stop is UNSET: 297 | return 298 | else: 299 | stop = None 300 | if ast_step is not None: 301 | step = get_constant(ast_step, types=SLICE_ARG_TYPES) 302 | if step is UNSET: 303 | return 304 | else: 305 | step = None 306 | 307 | myslice = slice(start, stop, step) 308 | result = value[myslice] 309 | return self.new_constant(node, result) 310 | 311 | def subscript_index(self, node): 312 | value = get_literal(node.value, types=SUBSCRIPT_INDEX_TYPES) 313 | if value is UNSET: 314 | return 315 | 316 | if isinstance(value, dict): 317 | # dict[key] accepts any hashable key (all constants are hashable) 318 | index_types = None 319 | else: 320 | index_types = int 321 | index = get_constant(node.slice.value, types=index_types) 322 | if index is UNSET: 323 | return 324 | 325 | try: 326 | result = value[index] 327 | except (IndexError, KeyError): 328 | return 329 | 330 | return self.new_constant(node, result) 331 | 332 | def visit_Subscript(self, node): 333 | if not self.config.constant_folding: 334 | return 335 | 336 | if isinstance(node.slice, ast.Slice): 337 | new_node = self.subscript_slice(node) 338 | if new_node is not None: 339 | return new_node 340 | 341 | elif isinstance(node.slice, ast.Index): 342 | new_node = self.subscript_index(node) 343 | if new_node is not None: 344 | return new_node 345 | 346 | def compare_cst(self, node): 347 | node_op = node.ops[0].__class__ 348 | eval_op = EVAL_COMPARE.get(node_op) 349 | if eval_op is None: 350 | return 351 | 352 | if node_op in (ast.In, ast.NotIn): 353 | left_hashable = True 354 | right_types = ITERABLE_TYPES 355 | else: 356 | left_hashable = False 357 | right_types = None 358 | 359 | if left_hashable: 360 | left = get_constant(node.left) 361 | else: 362 | left = get_literal(node.left) 363 | if left is UNSET: 364 | return 365 | right = get_literal(node.comparators[0], types=right_types) 366 | if right is UNSET: 367 | return 368 | 369 | if (node_op in (ast.Eq, ast.NotEq) 370 | and ((isinstance(left, str) and isinstance(right, bytes)) 371 | or (isinstance(left, bytes) and isinstance(right, str)))): 372 | # comparison between bytes and str can raise BytesWarning depending 373 | # on runtime option 374 | return 375 | 376 | try: 377 | result = eval_op(left, right) 378 | except TypeError: 379 | return 380 | return self.new_constant(node, result) 381 | 382 | def compare_contains(self, node): 383 | seq_ast = node.comparators[0] 384 | if not isinstance(seq_ast, (ast.Set, ast.List)): 385 | return 386 | 387 | # elements must be hashable 388 | seq = get_literal(seq_ast, constant_items=True) 389 | if seq is UNSET: 390 | return 391 | 392 | if isinstance(seq_ast, ast.Set): 393 | seq = frozenset(seq) 394 | else: 395 | seq = tuple(seq) 396 | 397 | new_seq_ast = self.new_constant(seq_ast, seq) 398 | if new_seq_ast is None: 399 | return 400 | 401 | new_node = copy_node(node) 402 | new_node.comparators[0] = new_seq_ast 403 | return new_node 404 | 405 | def visit_Compare(self, node): 406 | if not self.config.constant_folding: 407 | return 408 | 409 | if len(node.ops) != 1: 410 | # FIXME: implement 1 < 2 < 3 411 | return 412 | if len(node.comparators) != 1: 413 | # FIXME: support this case? What's the syntax of this case? 414 | return 415 | 416 | new_node = self.compare_cst(node) 417 | if new_node is not None: 418 | return new_node 419 | 420 | # replace 'None is None' with True 421 | if (isinstance(node.ops[0], (ast.Is, ast.IsNot)) 422 | and isinstance(node.left, ast.Constant) 423 | and node.left.value is None 424 | and isinstance(node.comparators[0], ast.Constant) 425 | and node.comparators[0].value is None): 426 | result = isinstance(node.ops[0], ast.Is) 427 | return self.new_constant(node, result) 428 | 429 | # replace 'x in {1, 2}' with 'x in frozenset({1, 2})' 430 | # replace 'x in [1, 2]' with 'x in frozenset((1, 2))' 431 | if isinstance(node.ops[0], ast.In): 432 | new_node = self.compare_contains(node) 433 | if new_node is not None: 434 | return new_node 435 | -------------------------------------------------------------------------------- /fatoptimizer/const_propagate.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | from .tools import OptimizerStep, UNSET, compact_ascii 4 | 5 | 6 | class ConstantPropagation(OptimizerStep): 7 | """Propagate constant values to variables. 8 | 9 | This optimizer step requires the NamespaceStep step. 10 | """ 11 | def visit_Name(self, node): 12 | if not self.config.constant_propagation: 13 | return 14 | 15 | if not isinstance(node, ast.Name) or not isinstance(node.ctx, ast.Load): 16 | return 17 | name = node.id 18 | if name not in self.local_variables: 19 | # the Namespace object only tracks local variables 20 | return 21 | 22 | value = self.namespace.get(name) 23 | if value is UNSET: 24 | return 25 | 26 | new_node = self.new_constant(node, value) 27 | if new_node is None: 28 | return 29 | 30 | self.log(node, "constant propagation: replace %s with %s", 31 | node.id, compact_ascii(value), add_line=True) 32 | return new_node 33 | -------------------------------------------------------------------------------- /fatoptimizer/convert_const.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | from .tools import NodeTransformer, copy_lineno 4 | 5 | 6 | class ConvertConstant(NodeTransformer): 7 | # Note: update PRIMITIVE_TYPES, ITERABLE_TYPES 8 | # and _is_constant() of tools when new types are supported 9 | 10 | def convert(self, node, value): 11 | new_node = ast.Constant(value=value) 12 | copy_lineno(node, new_node) 13 | return new_node 14 | 15 | def visit_NameConstant(self, node): 16 | return self.convert(node, node.value) 17 | 18 | def visit_Num(self, node): 19 | return self.convert(node, node.n) 20 | 21 | def visit_Str(self, node): 22 | return self.convert(node, node.s) 23 | 24 | def visit_Bytes(self, node): 25 | return self.convert(node, node.s) 26 | 27 | def visit_Tuple(self, node): 28 | elts = [] 29 | for elt in node.elts: 30 | if not isinstance(elt, ast.Constant): 31 | return 32 | elts.append(elt.value) 33 | return self.convert(node, tuple(elts)) 34 | -------------------------------------------------------------------------------- /fatoptimizer/copy_bltin_to_const.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | from .tools import OptimizerStep 4 | 5 | 6 | class CopyBuiltinToConstant: 7 | def __init__(self, global_name, unique_constant): 8 | self.global_name = global_name 9 | self.unique_constant = unique_constant 10 | 11 | 12 | class CopyBuiltinToConstantStep(OptimizerStep): 13 | def visit_Call(self, node): 14 | if not self.config.copy_builtin_to_constant: 15 | return 16 | 17 | if not isinstance(node.func, ast.Name): 18 | return 19 | func = node.func.id 20 | 21 | if func not in self.config._copy_builtin_to_constant: 22 | return 23 | 24 | if func in self.copy_builtin_to_constants: 25 | # already replaced 26 | return 27 | 28 | if not self.is_builtin_variable(func): 29 | return 30 | 31 | # If super() is replace with a string, the required free variable 32 | # (reference to the current class) is not created by the compiler 33 | if func == 'super': 34 | return 35 | 36 | unique_constant = self.new_str_constant('LOAD_GLOBAL %s' % func) 37 | copy_global = CopyBuiltinToConstant(func, unique_constant) 38 | self.copy_builtin_to_constants[func] = copy_global 39 | self.log(node, "copy %s builtin to constants", func) 40 | -------------------------------------------------------------------------------- /fatoptimizer/dead_code.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | from .tools import (OptimizerStep, 4 | copy_lineno, ast_contains, copy_node, compact_dump) 5 | 6 | 7 | # AST types of nodes that cannot be removed 8 | _CANNOT_REMOVE_TYPES = (ast.Global, ast.Nonlocal, ast.Yield, ast.YieldFrom, 9 | # don't remove 'except' block if it contains continue 10 | # or break: see can_move_final() for the rationale 11 | ast.Continue) 12 | 13 | _CANNOT_MOVE_FINAL = (ast.Continue,) 14 | 15 | 16 | def is_empty_body(node_list): 17 | if not node_list: 18 | return True 19 | return all(isinstance(node, ast.Pass) for node in node_list) 20 | 21 | 22 | def can_remove(node_list): 23 | if not node_list: 24 | # None and [] can be removed 25 | return True 26 | if ast_contains(node_list, _CANNOT_REMOVE_TYPES): 27 | return False 28 | return True 29 | 30 | 31 | def can_move_final(node_list): 32 | """Check if continue is in node_list. 33 | 34 | Using continue in a final block (of try/finally) is illegal: these 35 | instructions must not be moved, they must raise a SyntaxError (see 36 | test_syntax). 37 | """ 38 | if not node_list: 39 | # None and [] can be moved 40 | return True 41 | return not ast_contains(node_list, _CANNOT_MOVE_FINAL) 42 | 43 | 44 | def log_node_removal(optimizer, message, node_list): 45 | for node in node_list: 46 | node_repr = compact_dump(node) 47 | optimizer.log(node, "%s: %s", message, node_repr) 48 | 49 | 50 | def remove_dead_code(optimizer, node_list): 51 | """Remove dead code. 52 | 53 | Modify node_list in-place. 54 | Example: replace "return 1; return 2" with "return 1". 55 | """ 56 | 57 | truncate = None 58 | stop = len(node_list) - 1 59 | for index, node in enumerate(node_list): 60 | if index == stop: 61 | break 62 | if not isinstance(node, (ast.Return, ast.Raise)): 63 | continue 64 | if not can_remove(node_list[index+1:]): 65 | continue 66 | truncate = index 67 | break 68 | # FIXME: use for/else: ? 69 | if truncate is None: 70 | return node_list 71 | optimizer.log_node_removal("Remove unreachable code", node_list[truncate+1:]) 72 | return node_list[:truncate+1] 73 | 74 | 75 | class RemoveDeadCode(OptimizerStep): 76 | def log_node_removal(self, message, node_list): 77 | log_node_removal(self, message, node_list) 78 | 79 | def _replace_node(self, node, node_list): 80 | if node_list: 81 | return node_list 82 | 83 | # FIXME: move this in NodeTransformer? 84 | new_node = ast.Pass() 85 | copy_lineno(node, new_node) 86 | return new_node 87 | 88 | def _visit_if_while(self, node): 89 | if not self.config.remove_dead_code: 90 | return 91 | 92 | if not isinstance(node.test, ast.Constant): 93 | return 94 | 95 | test_true = bool(node.test.value) 96 | if test_true: 97 | if isinstance(node, ast.While): 98 | # while of 'while 1: ...' must not be removed 99 | return 100 | new_nodes = node.body 101 | removed_nodes = node.orelse 102 | reason = "test always true" 103 | else: 104 | new_nodes = node.orelse 105 | removed_nodes = node.body 106 | reason = "test always false" 107 | 108 | if not can_remove(removed_nodes): 109 | return 110 | 111 | self.log_node_removal("Remove dead code (%s)" % reason, 112 | removed_nodes) 113 | return self._replace_node(node, new_nodes) 114 | 115 | def visit_If(self, node): 116 | new_node = self._visit_if_while(node) 117 | if new_node is not None: 118 | return new_node 119 | 120 | if node.orelse and is_empty_body(node.orelse): 121 | self.log_node_removal("Remove dead code (empty else block of if)", 122 | node.orelse) 123 | new_node = copy_node(node) 124 | del new_node.orelse[:] 125 | node = new_node 126 | 127 | if is_empty_body(node.body) and not is_empty_body(node.orelse): 128 | self.log_node_removal("Remove dead code (empty if block)", 129 | node.body) 130 | new_node = copy_node(node) 131 | not_test = ast.UnaryOp(op=ast.Not(), operand=node.test) 132 | copy_lineno(node.test, not_test) 133 | new_node = ast.If(test=not_test, body=new_node.orelse, orelse=[]) 134 | copy_lineno(node, new_node) 135 | return new_node 136 | 137 | return node 138 | 139 | def visit_While(self, node): 140 | new_node = self._visit_if_while(node) 141 | if new_node is not None: 142 | return new_node 143 | 144 | if node.orelse and is_empty_body(node.orelse): 145 | self.log_node_removal("Remove dead code " 146 | "(empty else block of while)", 147 | node.orelse) 148 | new_node = copy_node(node) 149 | del new_node.orelse[:] 150 | return new_node 151 | 152 | def _try_empty_body(self, node): 153 | if not can_remove(node.body): 154 | return 155 | if not can_remove(node.handlers): 156 | return 157 | # body block is empty, handlers can be removed 158 | 159 | self.log_node_removal("Remove dead code (empty try block)", 160 | node.body) 161 | self.log_node_removal("Remove dead code (empty try block)", 162 | node.handlers) 163 | 164 | if not node.orelse: 165 | # body and else blocks are empty 166 | # 167 | # try: pass (except: ...) finally: final_code 168 | # => final_code 169 | if not can_move_final(node.finalbody): 170 | return 171 | return self._replace_node(node, node.finalbody) 172 | 173 | if is_empty_body(node.finalbody): 174 | # body and finally blocks are empty, else block is non empty 175 | # 176 | # try: pass (except: ...) else: else_code (final: pass) 177 | # => else_code 178 | self.log_node_removal("Remove dead code (empty finally block)", 179 | node.finalbody) 180 | return self._replace_node(node, node.orelse) 181 | 182 | # body block is empty, else and final blocks are non empty 183 | # 184 | # try: pass (except: ...) else: code1 finally: code2 185 | # => try: code1 finally: code2 186 | if not can_move_final(node.finalbody): 187 | return 188 | 189 | new_node = ast.Try(body=node.orelse, finalbody=node.finalbody, 190 | handlers=[], orelse=[]) 191 | copy_lineno(node, new_node) 192 | return new_node 193 | 194 | def visit_Try(self, node): 195 | if not self.config.remove_dead_code: 196 | return 197 | 198 | if node.orelse and is_empty_body(node.orelse): 199 | # remove 'else: pass' 200 | self.log_node_removal("Remove dead code (empty else block in try/except)", 201 | node.orelse) 202 | 203 | node = copy_node(node) 204 | node.orelse.clear() 205 | 206 | if is_empty_body(node.body): 207 | new_node = self._try_empty_body(node) 208 | if new_node is not None: 209 | return new_node 210 | 211 | return node 212 | 213 | def _remove_for(self, node): 214 | if node.orelse: 215 | node_list = node.body 216 | else: 217 | node_list = (node,) 218 | self.log_node_removal("Read dead code (empty for iterator)", 219 | node_list) 220 | return self._replace_node(node, node.orelse) 221 | 222 | def visit_For(self, node): 223 | if not self.config.remove_dead_code: 224 | return 225 | 226 | if (isinstance(node.iter, ast.Constant) 227 | and isinstance(node.iter.value, tuple) 228 | and not node.iter.value 229 | and can_remove(node.body)): 230 | return self._remove_for(node) 231 | 232 | if node.orelse and is_empty_body(node.orelse): 233 | self.log_node_removal("Remove dead code (empty else block of for)", 234 | node.orelse) 235 | new_node = copy_node(node) 236 | del new_node.orelse[:] 237 | return new_node 238 | -------------------------------------------------------------------------------- /fatoptimizer/inline.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | from .tools import (OptimizerStep, NodeTransformer, NodeVisitor, 4 | pretty_dump, get_starargs, get_keywords, get_varkeywords) 5 | 6 | class Checker(NodeVisitor): 7 | '''Gather a list of problems that would prevent inlining a function.''' 8 | def __init__(self): 9 | self.problems = [] 10 | 11 | def visit_Call(self, node): 12 | # Reject explicit attempts to use locals() 13 | # FIXME: detect uses via other names 14 | if isinstance(node.func, ast.Name): 15 | if node.func.id == 'locals': 16 | self.problems.append('use of locals()') 17 | 18 | 19 | def locate_kwarg(funcdef, name): 20 | '''Get the index of an argument of funcdef by name.''' 21 | for idx, arg in enumerate(funcdef.args.args): 22 | if arg.arg == name: 23 | return idx 24 | raise ValueError('argument %r not found' % name) 25 | 26 | 27 | class RenameVisitor(NodeTransformer): 28 | # FIXME: Reuse tools.ReplaceVariable 29 | 30 | def __init__(self, callsite, inlinable, actual_pos_args): 31 | assert get_starargs(callsite) is None 32 | assert not get_varkeywords(callsite) is not None 33 | assert inlinable.args.vararg is None 34 | assert inlinable.args.kwonlyargs == [] 35 | assert inlinable.args.kw_defaults == [] 36 | assert inlinable.args.kwarg is None 37 | assert inlinable.args.defaults == [] 38 | 39 | # Mapping from name in callee to node in caller 40 | self.remapping = {} 41 | for formal, actual in zip(inlinable.args.args, actual_pos_args): 42 | self.remapping[formal.arg] = actual 43 | 44 | def visit_Name(self, node): 45 | if node.id in self.remapping: 46 | return self.remapping[node.id] 47 | return node 48 | 49 | 50 | class Expansion: 51 | '''Information about a callsite that's a candidate for inlining, giving 52 | the funcdef, and the actual positional arguments (having 53 | resolved any keyword arguments.''' 54 | def __init__(self, funcdef, actual_pos_args): 55 | self.funcdef = funcdef 56 | self.actual_pos_args = actual_pos_args 57 | 58 | 59 | class InlineSubstitution(OptimizerStep): 60 | """Function call inlining.""" 61 | 62 | def build_positional_args(self, candidate, callsite): 63 | """Attempt to convert the positional and keyword args supplied at 64 | the given callsite to the positional args expected by the candidate 65 | funcdef. 66 | 67 | Return a list of ast.Node instances, or raise ValueError if it 68 | can't be done. 69 | """ 70 | if len(callsite.args) > len(candidate.args.args): 71 | raise ValueError('too many positional arguments') 72 | slots = {} 73 | for idx, arg in enumerate(callsite.args): 74 | slots[idx] = arg 75 | for actual_kwarg in get_keywords(callsite): 76 | idx = locate_kwarg(candidate, actual_kwarg.arg) 77 | if idx in slots: 78 | raise ValueError('positional slot %i already filled' % idx) 79 | slots[idx] = actual_kwarg.value 80 | actual_pos_args = [] 81 | for idx in range(len(candidate.args.args)): 82 | if idx not in slots: 83 | raise ValueError('argument %i not filled' % idx) 84 | actual_pos_args.append(slots[idx]) 85 | return actual_pos_args 86 | 87 | def can_inline(self, callsite): 88 | '''Given a Call callsite, determine whether we should inline 89 | the callee. If so, return an Expansion instance, otherwise 90 | return None.''' 91 | # FIXME: size criteria? 92 | # FIXME: don't do it for recursive functions 93 | if not isinstance(callsite.func, ast.Name): 94 | return None 95 | from .namespace import _fndefs 96 | if callsite.func.id not in _fndefs: 97 | return None 98 | candidate = _fndefs[callsite.func.id] 99 | 100 | # For now, only support simple positional arguments 101 | # and keyword arguments 102 | if get_starargs(callsite) is not None: 103 | return False 104 | if get_varkeywords(callsite) is not None: 105 | return False 106 | if candidate.args.vararg: 107 | return False 108 | if candidate.args.kwonlyargs: 109 | return False 110 | if candidate.args.kw_defaults: 111 | return False 112 | if candidate.args.kwarg: 113 | return False 114 | if candidate.args.defaults: 115 | return False 116 | 117 | # Attempt to match up the calling convention at the callsite 118 | # with the candidate funcdef 119 | try: 120 | actual_pos_args = self.build_positional_args(candidate, callsite) 121 | except ValueError: 122 | return None 123 | # For now, only allow functions that simply return a value 124 | body = candidate.body 125 | if len(body) != 1: 126 | return None 127 | if not (isinstance(body[0], ast.Return) 128 | or isinstance(body[0], ast.Pass)): 129 | return None 130 | 131 | # Walk the candidate's nodes looking for potential problems 132 | c = Checker() 133 | c.visit(body[0]) 134 | if c.problems: 135 | return None 136 | 137 | # All checks passed 138 | return Expansion(candidate, actual_pos_args) 139 | 140 | def visit_Call(self, node): 141 | if not self.config.inlining: 142 | return 143 | 144 | # FIXME: renaming variables to avoid clashes 145 | # or do something like: 146 | # .saved_locals = locals() 147 | # set params to args 148 | # body of called function 149 | # locals() = .saved_locals 150 | # how to things that aren't just a return 151 | # how to handle early return 152 | # FIXME: what guards are needed? 153 | # etc 154 | expansion = self.can_inline(node) 155 | if not expansion: 156 | return node 157 | funcdef = expansion.funcdef 158 | # Substitute the Call with the expression of the single return stmt 159 | # within the callee. 160 | # This assumes a single Return or Pass stmt 161 | stmt = funcdef.body[0] 162 | if isinstance(stmt, ast.Return): 163 | returned_expr = funcdef.body[0].value 164 | # Rename params/args 165 | v = RenameVisitor(node, funcdef, expansion.actual_pos_args) 166 | new_expr = v.visit(returned_expr) 167 | else: 168 | assert isinstance(stmt, ast.Pass) 169 | new_expr = self.new_constant(stmt, None) 170 | return new_expr 171 | -------------------------------------------------------------------------------- /fatoptimizer/iterable.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | from .tools import (OptimizerStep, 4 | get_literal, copy_lineno, get_constant, get_keywords, 5 | UNSET, ITERABLE_TYPES) 6 | from .specialized import BuiltinGuard 7 | 8 | 9 | class BaseSimplifyIterable(OptimizerStep): 10 | """Simplify iterable expressions.""" 11 | 12 | def optimize_iterable(self, node): 13 | raise NotImplementedError 14 | 15 | def visit_For(self, node): 16 | if not self.config.simplify_iterable: 17 | return 18 | 19 | new_iter = self.optimize_iterable(node.iter) 20 | if new_iter is None: 21 | return 22 | 23 | new_node = ast.For(target=node.target, 24 | iter=new_iter, 25 | body=node.body, 26 | orelse=node.orelse) 27 | copy_lineno(node, new_node) 28 | return new_node 29 | 30 | 31 | class SimplifyIterable(BaseSimplifyIterable): 32 | def optimize_iterable(self, node): 33 | # it's already a constant, nothing to do 34 | if isinstance(node, ast.Constant): 35 | return 36 | 37 | # remplace empty dict (create at runtime) with an empty tuple 38 | # (constant) 39 | if isinstance(node, ast.Dict) and not node.keys: 40 | return self.new_constant(node, ()) 41 | 42 | # FIXME: optimize dict? 43 | value = get_literal(node, types=(list, set), constant_items=True) 44 | if value is UNSET: 45 | return 46 | 47 | if not value: 48 | # replace empty iterable with an empty tuple 49 | return self.new_constant(node, ()) 50 | 51 | if len(value) > self.config.max_seq_len: 52 | return 53 | 54 | if isinstance(value, list): 55 | return self.new_constant(node, tuple(value)) 56 | if isinstance(value, set): 57 | return self.new_constant(node, frozenset(value)) 58 | 59 | 60 | class SimplifyIterableSpecialize(BaseSimplifyIterable): 61 | def optimize_range(self, node): 62 | if not(1 <= len(node.args) <= 3): 63 | return 64 | if get_keywords(node): 65 | return 66 | args = [] 67 | for node_arg in node.args: 68 | arg = get_constant(node_arg, types=int) 69 | if arg is UNSET: 70 | return 71 | args.append(arg) 72 | 73 | seq = range(*args) 74 | if len(seq) > self.config.max_seq_len: 75 | return 76 | value = self.new_constant(node, tuple(seq)) 77 | if value is None: 78 | return 79 | 80 | self.add_guard(BuiltinGuard('range')) 81 | return value 82 | 83 | def optimize_iterable(self, node): 84 | if (isinstance(node, ast.Call) 85 | and isinstance(node.func, ast.Name) 86 | and node.func.id == 'range' 87 | and self.is_builtin_variable('range')): 88 | return self.optimize_range(node) 89 | -------------------------------------------------------------------------------- /fatoptimizer/methods.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | 3 | from .pure import PureFunction 4 | 5 | 6 | def check_encoding(args): 7 | if len(args) >= 1: 8 | encoding = args[0] 9 | try: 10 | encoding = codecs.lookup(encoding).name 11 | except LookupError: 12 | return False 13 | # FIXME: Allow more encodings? 14 | if encoding not in ('ascii', 'iso8859-1', 'utf-8'): 15 | return False 16 | 17 | if len(args) >= 2: 18 | errors = args[1] 19 | # FIXME: support more error handlers 20 | # 'backslashreplace' (only for encode), 'surrogateescape', etc. 21 | if errors not in ('strict', 'replace', 'ignore'): 22 | return False 23 | 24 | return True 25 | 26 | 27 | def check_bytetype(args): 28 | return all(isinstance(arg, bytes) for arg in args) 29 | 30 | 31 | def check_byte_or_int(*args): 32 | return all(isinstance(arg, bytes) or ((isinstance(arg, int) and 0 <= arg <= 255)) 33 | for arg in args) 34 | 35 | 36 | def add_pure_methods(config): 37 | def add(obj_type, name, *args, **kw): 38 | if obj_type not in config._pure_methods: 39 | config._pure_methods[obj_type] = {} 40 | func = getattr(obj_type, name) 41 | pure = PureFunction(func, name, *args, **kw) 42 | config._pure_methods[obj_type][name] = pure 43 | 44 | add(bytes, 'decode', (0, 2), str, str, 45 | check_args=check_encoding, 46 | exceptions=UnicodeDecodeError) 47 | add(bytes, 'count', (1, 3), object, int, int, 48 | check_args=check_byte_or_int, 49 | exceptions=TypeError) 50 | add(bytes, 'endswith', (1, 3), tuple, int, int) 51 | add(bytes, 'find', (1, 3), object, int, int, 52 | check_args= check_byte_or_int, 53 | exceptions=TypeError) 54 | add(bytes, 'index', (1, 3), object, int, int, 55 | check_args= check_byte_or_int, 56 | exceptions=TypeError) 57 | 58 | add(bytes, 'join', (0,1), object, 59 | check_args=check_bytetype, 60 | exceptions=TypeError) 61 | add(bytes, 'maketrans', 2, bytes, bytes) 62 | add(bytes, 'partition', 1, bytes) 63 | add(bytes, 'replace', (2,3), bytes, bytes, int) 64 | add(bytes, 'rfind', (1, 3), bytes, int, int) 65 | add(bytes, 'rindex', (1, 3), bytes, int, int) 66 | add(bytes, 'rpartition', 1, bytes) 67 | add(bytes, 'startswith', (1, 3), bytes, int, int) 68 | add(bytes, 'translate', (1, 2), bytes, bytes) 69 | add(bytes, 'center', (1, 2), int, str) 70 | add(bytes, 'ljust', (1, 2), int, str) 71 | add(bytes, 'lstrip', (1, 2), bytes) 72 | add(bytes, 'rjust', (1, 2), int, str) 73 | add(bytes, 'rstrip', (0, 1), bytes) 74 | add(bytes, 'rsplit', (0, 2), bytes, int) 75 | add(bytes, 'split', (0, 2), bytes, int) 76 | add(bytes, 'strip', (0, 1), bytes) 77 | add(bytes, 'capitalize', 0) 78 | add(bytes, 'expandtabs', (0, 1), int) 79 | add(bytes, 'isalnum', 0) 80 | add(bytes, 'isalpha', 0) 81 | add(bytes, 'isdigit', 0) 82 | add(bytes, 'islower', 0) 83 | add(bytes, 'isspace', 0) 84 | add(bytes, 'istitle', 0) 85 | add(bytes, 'isupper', 0) 86 | add(bytes, 'islower', 0) 87 | add(bytes, 'splitlines', (0, 1), bool) 88 | add(bytes, 'swapcase', 0) 89 | add(bytes, 'title', 0) 90 | add(bytes, 'upper', 0) 91 | add(bytes, 'zfill', 1, int) 92 | 93 | 94 | # FIXME: add config option since IEEE 754 can be funny on some corner 95 | # cases? 96 | add(float, 'as_integer_ratio', 0) 97 | add(float, 'is_integer', 0) 98 | add(float, 'hex', 0) 99 | 100 | # FIXME: frozenset: 101 | # 'copy', 'difference', 'intersection', 'union', 'symmetric_difference', 102 | # 'isdisjoint', 'issubset', 'issuperset', 103 | 104 | add(int, 'bit_length', 0) 105 | 106 | add(str, 'encode', (0, 2), str, str, 107 | check_args=check_encoding, 108 | exceptions=UnicodeEncodeError) 109 | add(str, 'lower', 0) 110 | add(str, 'upper', 0) 111 | add(str, 'capitalize', 0) 112 | add(str, 'swapcase', 0) 113 | add(str, 'casefold', 0) 114 | add(str, 'isalpha', 0) 115 | add(str, 'isalnum', 0) 116 | add(str, 'isdecimal', 0) 117 | add(str, 'isdigit', 0) 118 | add(str, 'islower', 0) 119 | add(str, 'isnumeric', 0) 120 | add(str, 'isupper', 0) 121 | add(str, 'isidentifier', 0) 122 | add(str, 'istitle', 0) 123 | add(str, 'isspace', 0) 124 | add(str, 'swapcase', 0) 125 | add(str, 'title', 0) 126 | add(str, 'center', (1, 2), int, str) 127 | add(str, 'count', (1, 3), str, int, int) 128 | add(str, 'endswith', (1, 3), str, int, int) 129 | add(str, 'expandtabs', (0, 1), int) 130 | add(str, 'find', (1, 3), str, int, int) 131 | add(str, 'index', (1, 3), str, int, int) 132 | add(str, 'isprintable', 0) 133 | add(str, 'isupper', 0) 134 | add(str, 'ljust', (1, 2), int, str) 135 | add(str, 'lstrip', (0, 1), str) 136 | add(str, 'maketrans', (1, 3), str, str, str) 137 | add(str, 'partition', 1, str) 138 | add(str, 'replace', (2, 3), str, str, int) 139 | add(str, 'rfind', (1, 3), str, int, int) 140 | add(str, 'rindex', (1, 3), str, int, int) 141 | add(str, 'rjust', (1, 2), int, str) 142 | add(str, 'rpartition', 1, str) 143 | add(str, 'rsplit', (0, 2), str, int) 144 | add(str, 'rstrip', (0, 1), str) 145 | add(str, 'split', (0, 2), str, int) 146 | add(str, 'splitlines', (0, 1), bool) 147 | add(str, 'startswith', (1, 3), str, int, int) 148 | add(str, 'strip', (0, 1), str) 149 | add(str, 'zfill', 1, int) 150 | 151 | # FIXME: tuple: count, index 152 | -------------------------------------------------------------------------------- /fatoptimizer/namespace.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import contextlib 3 | 4 | from .tools import (UNSET, get_constant, compact_dump, 5 | OptimizerError, NodeVisitor, RestrictToFunctionDefMixin, OptimizerStep) 6 | 7 | 8 | class ComplexAssignment(OptimizerError): 9 | def __init__(self, node): 10 | super().__init__("Complex assignment: %s" % compact_dump(node)) 11 | self.node = node 12 | 13 | 14 | def _get_ast_name_node(node): 15 | while True: 16 | # only accept '*var' 17 | if isinstance(node, ast.Starred): 18 | # '*var = value' => 'var' 19 | node = node.value 20 | elif isinstance(node, ast.Subscript): 21 | # 'obj[slice] = value' => 'obj' 22 | node = node.value 23 | elif isinstance(node, ast.Attribute): 24 | # 'obj.attr = value' => 'obj' 25 | node = node.value 26 | elif (isinstance(node, ast.Call) 27 | and isinstance(node.func, ast.Attribute)): 28 | # 'obj.method().attr = value' => 'obj.method' 29 | node = node.func 30 | else: 31 | return node 32 | 33 | def get_ast_names(node): 34 | node = _get_ast_name_node(node) 35 | 36 | if isinstance(node, ast.Name): 37 | return (node.id,) 38 | 39 | if isinstance(node, ast.Tuple): 40 | names = [] 41 | for item in node.elts: 42 | item_names = get_ast_names(item) 43 | if item_names is None: 44 | return None 45 | names.extend(item_names) 46 | return names 47 | 48 | # unknown node type: return None 49 | 50 | 51 | def _get_assign_names(targets, load_names, store_names): 52 | for target in targets: 53 | orig_target = target 54 | target = _get_ast_name_node(target) 55 | if (isinstance(target, ast.Name) 56 | and isinstance(target.ctx, ast.Store)): 57 | # 'x = value': store name 'x' 58 | store_names.add(target.id) 59 | elif (isinstance(target, ast.Name) 60 | and isinstance(target.ctx, ast.Load)): 61 | # 'obj.attr = value': load name 'obj' 62 | load_names.add(target.id) 63 | elif isinstance(target, ast.Tuple): 64 | # x, y = ... 65 | _get_assign_names(target.elts, load_names, store_names) 66 | elif isinstance(target, ast.Constant): 67 | # '(1).__class__ = MyInt': it raises a TypeError 68 | raise ComplexAssignment(orig_target) 69 | elif isinstance(target, (ast.Dict, ast.List)): 70 | # '{...}[key] = ...', '[...][index] = ...' 71 | pass 72 | elif isinstance(target, ast.Call): 73 | # 'globals()[key] = value' 74 | # 'type(mock)._mock_check_sig = checksig' 75 | raise ComplexAssignment(orig_target) 76 | else: 77 | raise Exception("unsupported assign target: %s" 78 | % ast.dump(target)) 79 | 80 | 81 | class GlobalVisitor(NodeVisitor, RestrictToFunctionDefMixin): 82 | """Search for 'global var' statements.""" 83 | 84 | def __init__(self, filename): 85 | super().__init__(filename) 86 | self.global_variables = set() 87 | 88 | def visit_Global(self, node): 89 | self.global_variables |= set(node.names) 90 | 91 | 92 | class NonlocalVisitor(NodeVisitor): 93 | """Search for 'nonlocal var' statements.""" 94 | 95 | def __init__(self, filename): 96 | super().__init__(filename) 97 | self.nonlocal_variables = set() 98 | 99 | def visit_Nonlocal(self, node): 100 | self.nonlocal_variables |= set(node.names) 101 | 102 | 103 | class VariableVisitor(NodeVisitor, RestrictToFunctionDefMixin): 104 | """Find local and global variables. 105 | 106 | Find local and global variables of a function, but exclude variables of 107 | nested functions (functions, list comprehensions, generator expressions, 108 | etc.). 109 | """ 110 | def __init__(self, filename): 111 | super().__init__(filename) 112 | # variable names 113 | self.global_variables = set() 114 | self.local_variables = set() 115 | self.nonlocal_variables = set() 116 | 117 | @classmethod 118 | def from_node_list(cls, filename, node_list): 119 | visitor = cls(filename) 120 | for node in node_list: 121 | visitor.find_variables(node) 122 | return visitor 123 | 124 | def find_variables(self, node): 125 | # search for "global var" 126 | visitor = GlobalVisitor(self.filename) 127 | visitor.generic_visit(node) 128 | self.global_variables |= visitor.global_variables 129 | 130 | # search for "nonlocal var" 131 | visitor = NonlocalVisitor(self.filename) 132 | visitor.generic_visit(node) 133 | self.nonlocal_variables |= visitor.nonlocal_variables 134 | 135 | # visit all nodes 136 | self.generic_visit(node) 137 | 138 | def visit_arg(self, node): 139 | self.local_variables.add(node.arg) 140 | 141 | def _assign(self, targets): 142 | # get variables 143 | load_names = set() 144 | store_names = set() 145 | _get_assign_names(targets, load_names, store_names) 146 | 147 | # Global and non local variables cannot be local variables 148 | store_names -= (self.global_variables | self.nonlocal_variables) 149 | self.local_variables |= store_names 150 | self.global_variables |= load_names 151 | 152 | def visit_For(self, node): 153 | self._assign([node.target]) 154 | 155 | def visit_Assign(self, node): 156 | self._assign(node.targets) 157 | 158 | def visit_AugAssign(self, node): 159 | # We don't really need to handle AugAssign, Assign is enough to 160 | # detect local variables 161 | self._assign([node.target]) 162 | 163 | def fullvisit_FunctionDef(self, node): 164 | self.local_variables.add(node.name) 165 | 166 | def fullvisit_AsyncFunctionDef(self, node): 167 | self.local_variables.add(node.name) 168 | 169 | def fullvisit_ClassDef(self, node): 170 | self.local_variables.add(node.name) 171 | 172 | def _visit_import_names(self, names): 173 | for name in names: 174 | if name.asname: 175 | self.local_variables.add(name.asname) 176 | else: 177 | self.local_variables.add(name.name) 178 | 179 | def visit_Import(self, node): 180 | self._visit_import_names(node.names) 181 | 182 | def visit_ImportFrom(self, node): 183 | self._visit_import_names(node.names) 184 | 185 | def visit_withitem(self, node): 186 | if node.optional_vars is not None: 187 | self._assign([node.optional_vars]) 188 | 189 | 190 | class Namespace: 191 | def __init__(self): 192 | # True if we are unable to follow the namespace, False otherwise 193 | self._unknown_state = False 194 | # mapping: variable name => value, 195 | # value can be UNSET for unknown value 196 | self._variables = {} 197 | # True if we are inside a conditional block (ast.If, ast.For body, etc.) 198 | self._inside_cond = False 199 | 200 | @contextlib.contextmanager 201 | def cond_block(self): 202 | """Enter a conditional block. 203 | 204 | Operations on local variables inside a condition block makes these 205 | variables as "unknown state". 206 | """ 207 | was_inside = self._inside_cond 208 | try: 209 | self._inside_cond = True 210 | yield 211 | finally: 212 | self._inside_cond = was_inside 213 | 214 | def enter_unknown_state(self): 215 | if self._unknown_state: 216 | return True 217 | self._variables.clear() 218 | self._unknown_state = True 219 | return False 220 | 221 | def set(self, name, value): 222 | if not isinstance(name, str): 223 | raise TypeError("expect str") 224 | if self._unknown_state: 225 | return 226 | if self._inside_cond: 227 | value = UNSET 228 | self._variables[name] = value 229 | 230 | def unset(self, name): 231 | if self._unknown_state: 232 | return 233 | if self._inside_cond: 234 | self._variables[name] = UNSET 235 | else: 236 | if name in self._variables: 237 | del self._variables[name] 238 | 239 | def get(self, name): 240 | """Get the current value of a variable. 241 | 242 | Return UNSET if its value is unknown, or if the variable is not set. 243 | """ 244 | if self._inside_cond: 245 | return UNSET 246 | return self._variables.get(name, UNSET) 247 | 248 | # HACK! 249 | # Nasty global state. Mapping from names to FunctionDef nodes. 250 | # I attempted to wire up discovery of this mapping into NamespaceStep, 251 | # but for some reason if I store it in self.namespace, the inliner 252 | # instance doesn't see it (perhaps is seeing a different Namespace 253 | # instance, or the order in which the visit hooks is run is wrong 254 | _fndefs = {} 255 | 256 | class NamespaceStep(OptimizerStep): 257 | def fullvisit_FunctionDef(self, node): 258 | self.namespace.set(node.name, UNSET) 259 | global _fndefs 260 | _fndefs[node.name] = node 261 | 262 | def fullvisit_AsyncFunctionDef(self, node): 263 | self.namespace.set(node.name, UNSET) 264 | 265 | def fullvisit_ClassDef(self, node): 266 | self.namespace.set(node.name, UNSET) 267 | 268 | def _namespace_set(self, node, value, unset=False): 269 | if value is not UNSET: 270 | if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store): 271 | names = (node.id,) 272 | else: 273 | names = get_ast_names(node) 274 | value = UNSET 275 | else: 276 | names = get_ast_names(node) 277 | 278 | if names is None: 279 | if self.namespace.enter_unknown_state(): 280 | self.log(node, 281 | "enter unknown namespace state: " 282 | "don't support assignment %s", 283 | compact_dump(node)) 284 | return False 285 | 286 | for name in names: 287 | if unset: 288 | self.namespace.unset(name) 289 | else: 290 | self.namespace.set(name, value) 291 | return True 292 | 293 | def visit_Assign(self, node): 294 | value = get_constant(node.value) 295 | for target in node.targets: 296 | if not self._namespace_set(target, value): 297 | break 298 | 299 | def visit_AugAssign(self, node): 300 | self._namespace_set(node.target, UNSET) 301 | 302 | def visit_For(self, node): 303 | self._namespace_set(node.target, UNSET) 304 | 305 | def _visit_Import(self, node): 306 | for modname in node.names: 307 | if modname.asname: 308 | name = modname.asname 309 | else: 310 | name = modname.name 311 | # replace 'os.path' with 'os' 312 | name = name.split('.', 1)[0] 313 | self.namespace.set(name, UNSET) 314 | 315 | def visit_Import(self, node): 316 | self._visit_Import(node) 317 | 318 | def visit_ImportFrom(self, node): 319 | self._visit_Import(node) 320 | 321 | def visit_withitem(self, node): 322 | if node.optional_vars is not None: 323 | self._namespace_set(node.optional_vars, UNSET) 324 | 325 | def visit_Delete(self, node): 326 | for target in node.targets: 327 | if not self._namespace_set(target, UNSET, unset=True): 328 | break 329 | -------------------------------------------------------------------------------- /fatoptimizer/optimizer.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import linecache 3 | 4 | from .namespace import (VariableVisitor, ComplexAssignment, 5 | NamespaceStep) 6 | from .tools import (copy_lineno, _new_constant, pretty_dump, 7 | ReplaceVariable, FindStrVisitor, get_literal, 8 | RestrictToFunctionDefMixin, UNSET) 9 | from .specialized import BuiltinGuard, SpecializedFunction 10 | from .base_optimizer import BaseOptimizer 11 | from .const_propagate import ConstantPropagation 12 | from .const_fold import ConstantFolding 13 | from .call_pure import CallPureBuiltin 14 | from .unroll import UnrollStep, UnrollListComp 15 | from .copy_bltin_to_const import CopyBuiltinToConstantStep 16 | from .bltin_const import ReplaceBuiltinConstant 17 | from .convert_const import ConvertConstant 18 | from .dead_code import RemoveDeadCode, remove_dead_code 19 | from .iterable import SimplifyIterable, SimplifyIterableSpecialize 20 | from .call_method import CallPureMethods 21 | from .inline import InlineSubstitution 22 | 23 | def add_import(tree, name, asname): 24 | # import fat as __fat__ 25 | import_node = ast.Import(names=[ast.alias(name=name, asname=asname)], 26 | lineno=1, col_offset=1) 27 | for index, node in enumerate(tree.body): 28 | if (index == 0 and isinstance(node, ast.Expr) 29 | and isinstance(node.value, ast.Constant) 30 | and isinstance(node.value.value, str)): 31 | # docstring 32 | continue 33 | if (isinstance(node, ast.ImportFrom) and node.module == '__future__'): 34 | # from __future__ import ... 35 | continue 36 | tree.body.insert(index, import_node) 37 | break 38 | else: 39 | # body is empty or only contains __future__ imports 40 | tree.body.append(import_node) 41 | 42 | 43 | class NakedOptimizer(BaseOptimizer): 44 | """Optimizer without any optimization.""" 45 | 46 | def __init__(self, config, filename, parent=None): 47 | BaseOptimizer.__init__(self, filename) 48 | self.config = config 49 | if parent is not None: 50 | self.parent = parent 51 | # module is a ModuleOptimizer instance 52 | self.module = parent.module 53 | self.funcdef_depth = parent.funcdef_depth 54 | else: 55 | self.parent = None 56 | self.module = self 57 | self.funcdef_depth = 0 58 | # attributes set in optimize() 59 | self.root = None 60 | self._global_variables = set() 61 | self.nonlocal_variables = set() 62 | self.local_variables = set() 63 | # used by FunctionOptimizer.new_str_constant() 64 | self._new_str_constants = set() 65 | 66 | def optimize_node_list(self, node_list): 67 | if not self.config.remove_dead_code: 68 | return node_list 69 | return remove_dead_code(self, node_list) 70 | 71 | @classmethod 72 | def from_parent(cls, parent): 73 | return cls(parent.config, parent.filename, parent=parent) 74 | 75 | def new_constant(self, node, value): 76 | if not self.config.check_result(value): 77 | return 78 | return _new_constant(node, value) 79 | 80 | def log(self, node, message, *args, add_line=False): 81 | logger = self.config.logger 82 | if not logger: 83 | return 84 | message = message % args 85 | message = "%s: fatoptimizer: %s" % (self.error_where(node), message) 86 | print(message, file=logger) 87 | 88 | if add_line: 89 | line = linecache.getline(self.filename, node.lineno) 90 | if line: 91 | line = line.strip() 92 | if line: 93 | print(" %s" % line, file=logger) 94 | 95 | logger.flush() 96 | 97 | def _is_global_variable(self, name): 98 | if name in self._global_variables: 99 | return True 100 | module = self.module 101 | if module is not self: 102 | if name in module.local_variables: 103 | return True 104 | if name in module._global_variables: 105 | return True 106 | return False 107 | 108 | def is_builtin_variable(self, name): 109 | # local variable? 110 | if name in self.local_variables: 111 | return False 112 | 113 | # global variable? 114 | if self._is_global_variable(name): 115 | return False 116 | 117 | # non local variable? 118 | if name in self.nonlocal_variables: 119 | return False 120 | 121 | # free variable? (local variable of a parent function) 122 | parent = self.parent 123 | while parent is not None: 124 | if name in parent.local_variables: 125 | return False 126 | parent = parent.parent 127 | 128 | # variable not defined anywhere: it is likely 129 | # the expected builtin function 130 | return True 131 | 132 | def new_local_variable(self, name): 133 | if name in self.local_variables: 134 | index = 2 135 | while True: 136 | name2 = "%s%s" % (name, index) 137 | if name2 not in self.local_variables: 138 | break 139 | index += 1 140 | name = name2 141 | return name 142 | 143 | def _run_new_optimizer(self, node): 144 | optimizer = Optimizer.from_parent(self) 145 | return optimizer.optimize(node) 146 | 147 | def _run_sub_optimizer(self, optimizer, node): 148 | new_node = optimizer.optimize(node) 149 | if isinstance(new_node, list): 150 | # The function was optimized 151 | 152 | # find new local variables 153 | visitor = VariableVisitor.from_node_list(self.filename, new_node) 154 | self.local_variables |= visitor.local_variables 155 | return new_node 156 | 157 | def fullvisit_FunctionDef(self, node): 158 | optimizer = FunctionOptimizer.from_parent(self) 159 | return self._run_sub_optimizer(optimizer, node) 160 | 161 | def fullvisit_ListComp(self, node): 162 | optimizer = ComprehensionOptimizer.from_parent(self) 163 | return self._run_sub_optimizer(optimizer, node) 164 | 165 | def fullvisit_SetComp(self, node): 166 | optimizer = ComprehensionOptimizer.from_parent(self) 167 | return self._run_sub_optimizer(optimizer, node) 168 | 169 | def fullvisit_DictComp(self, node): 170 | optimizer = ComprehensionOptimizer.from_parent(self) 171 | return self._run_sub_optimizer(optimizer, node) 172 | 173 | def _optimize(self, tree): 174 | return self.generic_visit(tree) 175 | 176 | def optimize(self, tree): 177 | self.root = tree 178 | 179 | # Find variables 180 | visitor = VariableVisitor(self.filename) 181 | try: 182 | visitor.find_variables(tree) 183 | except ComplexAssignment as exc: 184 | # globals() is used to store a variable: 185 | # give up, don't optimize the function 186 | self.log(exc.node, "skip optimisation: %s", exc) 187 | return tree 188 | self._global_variables |= visitor.global_variables 189 | self.nonlocal_variables |= visitor.nonlocal_variables 190 | self.local_variables |= visitor.local_variables 191 | 192 | # Optimize nodes 193 | return self._optimize(tree) 194 | 195 | 196 | class Optimizer(NakedOptimizer, 197 | NamespaceStep, 198 | ReplaceBuiltinConstant, 199 | CallPureMethods, 200 | UnrollStep, 201 | ConstantPropagation, 202 | SimplifyIterable, 203 | ConstantFolding, 204 | RemoveDeadCode): 205 | """Optimizer for AST nodes other than Module and FunctionDef.""" 206 | 207 | 208 | class FunctionOptimizerStage1(RestrictToFunctionDefMixin, Optimizer): 209 | """Stage 1 optimizer for ast.FunctionDef nodes.""" 210 | 211 | 212 | 213 | class ComprehensionOptimizer(RestrictToFunctionDefMixin, 214 | UnrollListComp, 215 | Optimizer): 216 | """Optimizer for ast.ListComp and ast.SetComp nodes.""" 217 | 218 | def _optimize(self, tree): 219 | tree = self.generic_visit(tree) 220 | new_tree = self.unroll_comprehension(tree) 221 | if new_tree is not None: 222 | # run again stage1 223 | tree = self.generic_visit(new_tree) 224 | return tree 225 | 226 | 227 | class FunctionOptimizer(NakedOptimizer, 228 | CallPureBuiltin, 229 | SimplifyIterableSpecialize, 230 | InlineSubstitution, 231 | CopyBuiltinToConstantStep): 232 | """Optimizer for ast.FunctionDef nodes. 233 | 234 | First, run FunctionOptimizerStage1 and then run optimizations which may 235 | create a specialized function. 236 | """ 237 | 238 | def __init__(self, *args, **kw): 239 | super().__init__(*args, **kw) 240 | if self.parent is None: 241 | raise ValueError("parent is not set") 242 | self.funcdef_depth += 1 243 | self._guards = [] 244 | # FIXME: move this to the optimizer step? 245 | # global name => CopyBuiltinToConstant 246 | self.copy_builtin_to_constants = {} 247 | 248 | def add_guard(self, new_guard): 249 | if not isinstance(new_guard, BuiltinGuard): 250 | raise ValueError("unsupported guard") 251 | if self._guards: 252 | guard = self._guards[0] 253 | guard.add(new_guard) 254 | else: 255 | self._guards.append(new_guard) 256 | 257 | def new_str_constant(self, value): 258 | str_constants = self._new_str_constants 259 | str_constants |= self.parent._new_str_constants 260 | 261 | # FIXME: self.root is an old version of the tree, the new tree can 262 | # contain new strings 263 | visitor = FindStrVisitor.from_node(self.filename, self.root) 264 | str_constants |= visitor.str_constants 265 | 266 | visitor = FindStrVisitor.from_node(self.filename, self.parent.root) 267 | str_constants |= visitor.str_constants 268 | 269 | if value in str_constants: 270 | index = 2 271 | while True: 272 | new_value = "%s#%s" % (value, index) 273 | if new_value not in str_constants: 274 | break 275 | index += 1 276 | value = new_value 277 | 278 | self._new_str_constants.add(value) 279 | self.parent._new_str_constants.add(value) 280 | return value 281 | 282 | def _patch_constants(self, node): 283 | copy_builtin_to_constants = self.copy_builtin_to_constants.values() 284 | patch_constants = {} 285 | for copy_global in copy_builtin_to_constants: 286 | builtin_name = copy_global.global_name 287 | value = ast.Name(id=builtin_name, ctx=ast.Load()) 288 | patch_constants[copy_global.unique_constant] = value 289 | self.add_guard(BuiltinGuard(builtin_name, reason='patch constant')) 290 | 291 | names = dict((copy_global.global_name, copy_global.unique_constant) 292 | for copy_global in copy_builtin_to_constants) 293 | replace = ReplaceVariable(self.filename, names) 294 | new_node = replace.replace_func_def(node) 295 | return (new_node, patch_constants) 296 | 297 | def _specialize(self, func_node, new_node): 298 | if self.copy_builtin_to_constants: 299 | new_node, patch_constants = self._patch_constants(new_node) 300 | else: 301 | patch_constants = None 302 | 303 | self.log(func_node, "specialize function %s, guards: %s", 304 | func_node.name, self._guards) 305 | 306 | new_body = [func_node] 307 | 308 | tmp_name = self.parent.new_local_variable('_ast_optimized') 309 | func = SpecializedFunction(new_node.body, self._guards, patch_constants) 310 | 311 | modname = self.module.get_fat_module_name() 312 | for node in func.to_ast(modname, func_node, tmp_name): 313 | copy_lineno(func_node, node) 314 | new_body.append(node) 315 | 316 | return new_body 317 | 318 | def _stage1(self, tree): 319 | optimizer = FunctionOptimizerStage1.from_parent(self) 320 | return optimizer.optimize(tree) 321 | 322 | def optimize(self, func_node): 323 | func_node = self._stage1(func_node) 324 | 325 | if func_node.decorator_list: 326 | # FIXME: support decorators 327 | self.log(func_node, "skip optimisation: don't support decorators") 328 | return func_node 329 | 330 | # FIXME: specialize somehow nested functions? 331 | if self.funcdef_depth > 1: 332 | self.log(func_node, 333 | "skip optimisation requiring specialization " 334 | "on nested function") 335 | return func_node 336 | 337 | new_node = super().optimize(func_node) 338 | 339 | if self._guards: 340 | # calling pure functions, replacing range(n) with a tuple, etc. 341 | # can allow new optimizations with the stage 1 342 | new_node = self._stage1(new_node) 343 | 344 | if self.copy_builtin_to_constants or self._guards: 345 | new_node = self._specialize(func_node, new_node) 346 | 347 | return new_node 348 | 349 | 350 | class ModuleOptimizer(Optimizer): 351 | """Optimizer for ast.Module nodes.""" 352 | 353 | def __init__(self, *args, **kw): 354 | super().__init__(*args, **kw) 355 | self._fat_module = None 356 | 357 | def get_fat_module_name(self): 358 | if not self._fat_module: 359 | # FIXME: ensure that the name is unique... 360 | self._fat_module = '__fat__' 361 | return self._fat_module 362 | 363 | def _replace_config(self, node): 364 | config = get_literal(node, types=dict, constant_items=True) 365 | if config is UNSET: 366 | # ignore invalid config 367 | return True 368 | 369 | # Replace the configuration 370 | # Note: unknown configuration options are ignored 371 | self.config = self.config.replace(config) 372 | 373 | def _find_config(self, body): 374 | # FIXME: only search in the N first statements? 375 | # Example: skip docstring, but stop at the first import? 376 | for node in body: 377 | if (isinstance(node, ast.Assign) 378 | and len(node.targets) == 1 379 | and isinstance(node.targets[0], ast.Name) 380 | and node.targets[0].id == '__fatoptimizer__'): 381 | self._replace_config(node.value) 382 | 383 | def optimize(self, tree): 384 | orig_tree = tree 385 | 386 | tree = ConvertConstant(self.filename).visit(tree) 387 | 388 | if isinstance(tree, ast.Module): 389 | self._find_config(tree.body) 390 | if not self.config.enabled: 391 | self.log(tree, 392 | "skip optimisation: disabled in __fatoptimizer__") 393 | return orig_tree 394 | 395 | tree = super().optimize(tree) 396 | 397 | if self._fat_module: 398 | add_import(tree, 'fat', self._fat_module) 399 | 400 | return tree 401 | -------------------------------------------------------------------------------- /fatoptimizer/pure.py: -------------------------------------------------------------------------------- 1 | from .tools import UNSET, get_literal, get_keywords 2 | 3 | 4 | class PureFunction: 5 | def __init__(self, func, name, narg, *arg_types, check_args=None, exceptions=None): 6 | self.func = func 7 | self.name = name 8 | if isinstance(narg, tuple): 9 | self.min_narg, self.max_narg = narg 10 | if self.min_narg is None: 11 | raise ValueError("minimum number of parameters is None") 12 | elif isinstance(narg, int): 13 | self.min_narg = narg 14 | self.max_narg = narg 15 | else: 16 | raise TypeError("narg must be tuple or int, got %s" 17 | % type(narg).__name__) 18 | self.arg_types = arg_types 19 | if len(self.arg_types) < self.min_narg: 20 | raise ValueError("not enough argument types") 21 | if self.max_narg is not None and len(self.arg_types) > self.max_narg: 22 | raise ValueError("too many argument types") 23 | self._check_args_cb = check_args 24 | self.exceptions = exceptions 25 | 26 | def check_nargs(self, nargs): 27 | if self.min_narg is not None and self.min_narg > nargs: 28 | return False 29 | if self.max_narg is not None and self.max_narg < nargs: 30 | return False 31 | return True 32 | 33 | def _check_args(self, args): 34 | if not self.check_nargs(len(args)): 35 | return False 36 | if self._check_args_cb is not None: 37 | if not self._check_args_cb(args): 38 | return False 39 | return True 40 | 41 | def get_args(self, node): 42 | if get_keywords(node): 43 | # FIXME: support keywords 44 | return 45 | 46 | if not self.check_nargs(len(node.args)): 47 | return 48 | 49 | values = [] 50 | for index, node_arg in enumerate(node.args): 51 | try: 52 | arg_type = self.arg_types[index] 53 | except IndexError: 54 | arg_type = None 55 | value = get_literal(node_arg, types=arg_type) 56 | if value is UNSET: 57 | return 58 | values.append(value) 59 | return values 60 | 61 | def _call(self, obj, node): 62 | args = self.get_args(node) 63 | if args is None: 64 | return UNSET 65 | 66 | if not self._check_args(args): 67 | return UNSET 68 | 69 | try: 70 | if obj is not UNSET: 71 | result = self.func(obj, *args) 72 | else: 73 | result = self.func(*args) 74 | except Exception as exc: 75 | if (self.exceptions is not None 76 | and isinstance(exc, self.exceptions)): 77 | result = UNSET 78 | else: 79 | raise 80 | 81 | return result 82 | 83 | def call_func(self, node): 84 | return self._call(UNSET, node) 85 | 86 | def call_method(self, obj, node): 87 | return self._call(obj, node) 88 | -------------------------------------------------------------------------------- /fatoptimizer/specialized.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | from .tools import copy_lineno, _new_constant, Call 4 | 5 | 6 | class BuiltinGuard: 7 | def __init__(self, name, reason=None): 8 | self.names = {name} 9 | self.reason = reason 10 | 11 | def add(self, guard): 12 | self.names |= guard.names 13 | 14 | def as_ast(self, node, modname): 15 | name = ast.Name(id=modname, ctx=ast.Load()) 16 | copy_lineno(node, name) 17 | 18 | func = ast.Attribute(value=name, attr='GuardBuiltins', ctx=ast.Load()) 19 | copy_lineno(node, func) 20 | 21 | names = [_new_constant(node, name) for name in sorted(self.names)] 22 | call = Call(func=func, args=names, keywords=[]) 23 | copy_lineno(node, call) 24 | return call 25 | 26 | def __repr__(self): 27 | info = ['names=%r' % self.names] 28 | if self.reason: 29 | info.append('reason=%r' % self.reason) 30 | return '<%s %s>' % (self.__class__.__name__, ' '.join(info)) 31 | 32 | 33 | class SpecializedFunction: 34 | def __init__(self, body, guards, patch_constants=None): 35 | self.body = body 36 | self.guards = guards 37 | self.patch_constants = patch_constants 38 | 39 | def to_ast(self, modname, func, tmp_name): 40 | # tmp_name = func 41 | yield ast.Assign(targets=[ast.Name(id=tmp_name, ctx=ast.Store())], 42 | value=ast.Name(id=func.name, ctx=ast.Load())) 43 | 44 | # def func2(...): ... 45 | for node in self.body: 46 | copy_lineno(func, node) 47 | func2 = ast.FunctionDef(name=func.name, args=func.args, body=self.body, 48 | # explicitly drops decorator for the 49 | # specialized function 50 | decorator_list=[], 51 | returns=None) 52 | yield func2 53 | 54 | if self.patch_constants: 55 | # func.__code__ = func.__code__.replace_consts({...}) 56 | dict_keys = [] 57 | dict_values = [] 58 | for key, value in self.patch_constants.items(): 59 | # FIXME: use optimizer.new_constant()? 60 | key = _new_constant(func, key) 61 | value = _new_constant(func, value) 62 | dict_keys.append(key) 63 | dict_values.append(value) 64 | mapping = ast.Dict(keys=dict_keys, values=dict_values) 65 | copy_lineno(func, mapping) 66 | 67 | mod = ast.Name(id=modname, ctx=ast.Load()) 68 | name_func = ast.Name(id=func2.name, ctx=ast.Load()) 69 | attr = ast.Attribute(value=name_func, attr='__code__', ctx=ast.Load()) 70 | call = Call(func=ast.Attribute(value=mod, 71 | attr='replace_consts', ctx=ast.Load()), 72 | args=[attr, mapping], 73 | keywords=[]) 74 | copy_lineno(func, call) 75 | 76 | target = ast.Attribute(value=name_func, attr='__code__', ctx=ast.Store()) 77 | yield ast.Assign(targets=[target], 78 | value=call) 79 | 80 | # encode guards 81 | guards = [guard.as_ast(func, modname) for guard in self.guards] 82 | guards = ast.List(elts=guards, ctx=ast.Load()) 83 | copy_lineno(func, guards) 84 | 85 | # fat.specialize(tmp_name, func2, guards) 86 | specialize = ast.Attribute(value=ast.Name(id=modname, ctx=ast.Load()), 87 | attr='specialize', ctx=ast.Load()) 88 | name_func = ast.Name(id=tmp_name, ctx=ast.Load()) 89 | code = ast.Attribute(value=ast.Name(id=func2.name, ctx=ast.Load()), 90 | attr='__code__', ctx=ast.Load()) 91 | call = Call(func=specialize, args=[name_func, code, guards], 92 | keywords=[]) 93 | yield ast.Expr(value=call) 94 | 95 | # func = tmp_name 96 | yield ast.Assign(targets=[ast.Name(id=func.name, ctx=ast.Store())], 97 | value=ast.Name(id=tmp_name, ctx=ast.Load())) 98 | 99 | # del tmp_name 100 | yield ast.Delete(targets=[ast.Name(id=tmp_name, ctx=ast.Del())]) 101 | -------------------------------------------------------------------------------- /fatoptimizer/unroll.py: -------------------------------------------------------------------------------- 1 | import ast 2 | 3 | from .tools import (OptimizerStep, ReplaceVariable, FindNodes, 4 | compact_dump, copy_lineno, 5 | ITERABLE_TYPES) 6 | 7 | 8 | CANNOT_UNROLL = (ast.Break, ast.Continue, ast.Raise) 9 | 10 | 11 | class UnrollStep(OptimizerStep): 12 | def _visit_For(self, node): 13 | if not isinstance(node.target, ast.Name): 14 | return 15 | 16 | # for i in (1, 2, 3): ... 17 | if not isinstance(node.iter, ast.Constant): 18 | return 19 | iter_value = node.iter.value 20 | if not isinstance(iter_value, tuple): 21 | return 22 | if not(1 <= len(iter_value) <= self.config.unroll_loops): 23 | return 24 | 25 | # don't optimize if 'break' or 'continue' is found in the loop body 26 | found = None 27 | def find_callback(node): 28 | nonlocal found 29 | found = node 30 | return False 31 | 32 | # FIXME: restrict this the current scope 33 | # (don't enter class/function def/list comprehension/...) 34 | visitor = FindNodes(CANNOT_UNROLL, find_callback) 35 | visitor.visit(node) 36 | if found is not None: 37 | self.log(node, 38 | "cannot unroll loop: %s is used at line %s", 39 | compact_dump(found), 40 | found.lineno) 41 | return 42 | 43 | name = node.target.id 44 | body = node.body 45 | 46 | # replace 'for i in (1, 2, 3): body' with... 47 | new_node = [] 48 | for value in node.iter.value: 49 | value_ast = self.new_constant(node.iter, value) 50 | if value_ast is None: 51 | return 52 | 53 | # 'i = 1' 54 | name_ast = ast.Name(id=name, ctx=ast.Store()) 55 | copy_lineno(node, name_ast) 56 | assign = ast.Assign(targets=[name_ast], 57 | value=value_ast) 58 | copy_lineno(node, assign) 59 | new_node.append(assign) 60 | 61 | # duplicate 'body' 62 | new_node.extend(body) 63 | 64 | if node.orelse: 65 | new_node.extend(node.orelse) 66 | 67 | self.log(node, "unroll loop (%s iterations)", len(node.iter.value)) 68 | 69 | return new_node 70 | 71 | def visit_For(self, node): 72 | if not self.config.unroll_loops: 73 | return 74 | 75 | new_node = self._visit_For(node) 76 | if new_node is None: 77 | return 78 | 79 | # loop was unrolled: run again the optimize on the new nodes 80 | return self.visit_node_list(new_node) 81 | 82 | 83 | class UnrollListComp: 84 | def unroll_comprehension(self, node): 85 | if not self.config.unroll_loops: 86 | return 87 | 88 | # FIXME: support multiple generators 89 | # [i for i in range(3) for y in range(3)] 90 | if len(node.generators) > 1: 91 | return 92 | 93 | generator = node.generators[0] 94 | if not isinstance(generator, ast.comprehension): 95 | return 96 | # FIXME: support if 97 | if generator.ifs: 98 | return 99 | 100 | if not isinstance(generator.target, ast.Name): 101 | return 102 | target = generator.target.id 103 | 104 | if not isinstance(generator.iter, ast.Constant): 105 | return 106 | iter_value = generator.iter.value 107 | if not isinstance(iter_value, ITERABLE_TYPES): 108 | return 109 | if not(1 <= len(iter_value) <= self.config.unroll_loops): 110 | return 111 | 112 | if isinstance(node, ast.DictComp): 113 | keys = [] 114 | values = [] 115 | for value in iter_value: 116 | ast_value = self.new_constant(node, value) 117 | if ast_value is None: 118 | return 119 | replace = ReplaceVariable(self.filename, {target: ast_value}) 120 | 121 | key = replace.visit(node.key) 122 | keys.append(key) 123 | 124 | value = replace.visit(node.value) 125 | values.append(value) 126 | 127 | new_node = ast.Dict(keys=keys, values=values, ctx=ast.Load()) 128 | else: 129 | items = [] 130 | for value in iter_value: 131 | ast_value = self.new_constant(node, value) 132 | if ast_value is None: 133 | return 134 | replace = ReplaceVariable(self.filename, {target: ast_value}) 135 | item = replace.visit(node.elt) 136 | items.append(item) 137 | 138 | # FIXME: move below? 139 | if isinstance(node, ast.SetComp): 140 | new_node = ast.Set(elts=items, ctx=ast.Load()) 141 | else: 142 | assert isinstance(node, ast.ListComp) 143 | new_node = ast.List(elts=items, ctx=ast.Load()) 144 | 145 | copy_lineno(node, new_node) 146 | return new_node 147 | 148 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Prepare a release: 4 | # 5 | # - git pull --rebase 6 | # - run tests: tox 7 | # - update VERSION in setup.py, fatoptimizer/__init__.py and doc/conf.py 8 | # - set release date in the changelog in doc/misc.rst 9 | # - git commit -a 10 | # - git push 11 | # 12 | # Release a new version: 13 | # 14 | # - git tag VERSION 15 | # - git push --tags 16 | # - python3 setup.py register sdist bdist_wheel upload 17 | # 18 | # After the release: 19 | # 20 | # - set version to n+1 21 | # - git commit 22 | # - git push 23 | 24 | VERSION = '0.3' 25 | 26 | DESCRIPTION = ('Static optimizer for Python 3.6 using function ' 27 | 'specialization with guards') 28 | CLASSIFIERS = [ 29 | 'Development Status :: 4 - Beta', 30 | 'Intended Audience :: Developers', 31 | 'License :: OSI Approved :: MIT License', 32 | 'Natural Language :: English', 33 | 'Operating System :: OS Independent', 34 | 'Programming Language :: C', 35 | 'Programming Language :: Python', 36 | 'Topic :: Software Development :: Libraries :: Python Modules', 37 | ] 38 | 39 | 40 | # put most of the code inside main() to be able to import setup.py in 41 | # test_fatoptimizer.py, to ensure that VERSION is the same than 42 | # fatoptimizer.__version__. 43 | def main(): 44 | try: 45 | from setuptools import setup 46 | except ImportError: 47 | from distutils.core import setup 48 | 49 | with open('README.rst') as fp: 50 | long_description = fp.read().strip() 51 | 52 | options = { 53 | 'name': 'fatoptimizer', 54 | 'version': VERSION, 55 | 'license': 'MIT license', 56 | 'description': DESCRIPTION, 57 | 'long_description': long_description, 58 | 'url': 'https://fatoptimizer.readthedocs.io/en/latest/', 59 | 'author': 'Victor Stinner', 60 | 'author_email': 'victor.stinner@gmail.com', 61 | 'classifiers': CLASSIFIERS, 62 | 'packages': ['fatoptimizer'], 63 | } 64 | setup(**options) 65 | 66 | 67 | if __name__ == '__main__': 68 | main() 69 | -------------------------------------------------------------------------------- /test_fat_config.py: -------------------------------------------------------------------------------- 1 | """ 2 | Integration test: test FAT mode with the fatoptimizer configured in this file. 3 | """ 4 | import fatoptimizer 5 | import dis 6 | import fat 7 | import io 8 | import sys 9 | import textwrap 10 | import unittest 11 | 12 | # Disable the AST optimizer on this module 13 | __fatoptimizer__ = {'enabled': False} 14 | 15 | def create_optimizer(): 16 | config = fatoptimizer.Config() 17 | config.strict = False 18 | config.copy_builtin_to_constant = {'chr'} 19 | 20 | return fatoptimizer.FATOptimizer(config) 21 | 22 | # Replace existing AST transformers with our optimizer 23 | sys.set_code_transformers([create_optimizer()]) 24 | 25 | 26 | def disassemble(obj): 27 | output = io.StringIO() 28 | dis.dis(obj, file=output) 29 | return output.getvalue() 30 | 31 | 32 | class CopyGlobalToLocal(unittest.TestCase): 33 | def test_builtin(self): 34 | ns = {} 35 | exec(textwrap.dedent(""" 36 | def func(x): 37 | return chr(x) 38 | """), ns, ns) 39 | func = ns['func'] 40 | 41 | self.assertIn('LOAD_GLOBAL', disassemble(func)) 42 | self.assertEqual(func.__code__.co_consts, (None,)) 43 | 44 | # the specialized bytecode must not use LOAD_GLOBAL, but have 45 | # chr in its constants 46 | self.assertEqual(len(fat.get_specialized(func)), 1) 47 | new_code = fat.get_specialized(func)[0][0] 48 | self.assertNotIn('LOAD_GLOBAL', disassemble(new_code)) 49 | self.assertEqual(new_code.co_consts, (None, chr)) 50 | 51 | # call the specialized function 52 | self.assertNotIn('chr', globals()) 53 | self.assertEqual(func(65), 'A') 54 | 55 | # chr() is modified in globals(): call the original function 56 | # and remove the specialized bytecode 57 | ns['chr'] = str 58 | self.assertEqual(func(65), '65') 59 | self.assertEqual(len(fat.get_specialized(func)), 0) 60 | 61 | 62 | if __name__ == "__main__": 63 | unittest.main() 64 | -------------------------------------------------------------------------------- /test_fat_site.py: -------------------------------------------------------------------------------- 1 | """ 2 | Integration test: Test FAT mode with the fatoptimizer configured by the site 3 | module. 4 | """ 5 | import dis 6 | import fat 7 | import io 8 | import sys 9 | import textwrap 10 | import unittest 11 | 12 | 13 | #if not any(transformer.name == 'fat' for transformer in sys.get_code_transformers()): 14 | # raise Exception("test must be run with python3 -X fat") 15 | 16 | 17 | def disassemble(obj): 18 | output = io.StringIO() 19 | dis.dis(obj, file=output) 20 | return output.getvalue() 21 | 22 | 23 | def call_builtin(): 24 | return len("abc") 25 | 26 | 27 | 28 | class CallPureBuiltins(unittest.TestCase): 29 | def test_code(self): 30 | self.assertIn('LOAD_GLOBAL', disassemble(call_builtin)) 31 | 32 | self.assertEqual(len(fat.get_specialized(call_builtin)), 1) 33 | 34 | code = fat.get_specialized(call_builtin)[0][0] 35 | self.assertEqual(code.co_name, call_builtin.__name__) 36 | self.assertNotIn('LOAD_GLOBAL', disassemble(code)) 37 | 38 | def test_import(self): 39 | ns = {} 40 | code = textwrap.dedent(""" 41 | from builtins import str as chr 42 | 43 | def func(): 44 | # chr() is not the expected builtin function, 45 | # it must not be optimized 46 | return chr(65) 47 | """) 48 | exec(code, ns, ns) 49 | func = ns['func'] 50 | 51 | self.assertEqual(fat.get_specialized(func), []) 52 | 53 | 54 | def copy_builtin(x): 55 | len(x) 56 | def nested(): 57 | pass 58 | return nested.__qualname__ 59 | 60 | 61 | class CopyBuiltinToConstant(unittest.TestCase): 62 | def test_qualname(self): 63 | self.assertEqual(len(fat.get_specialized(copy_builtin)), 1) 64 | 65 | # optimizations must not modify the function name 66 | qualname = copy_builtin("abc") 67 | self.assertEqual(qualname, 'copy_builtin..nested') 68 | 69 | 70 | if __name__ == "__main__": 71 | unittest.main() 72 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py35, doc, pep8 3 | 4 | [testenv] 5 | commands= 6 | python test_fatoptimizer.py 7 | 8 | [testenv:pep8] 9 | basepython = python3 10 | deps = flake8 11 | commands = 12 | flake8 perf setup.py doc/examples/ 13 | 14 | [flake8] 15 | # E501 line too long (88 > 79 characters) 16 | ignore = E501 17 | 18 | [testenv:doc] 19 | basepython = python3 20 | deps= 21 | sphinx 22 | whitelist_externals = make 23 | commands= 24 | make -C doc html 25 | --------------------------------------------------------------------------------