├── .bzrignore ├── .gitignore ├── .travis.yml ├── Changelog ├── LICENSE ├── MANIFEST.in ├── README.md ├── bytecode_tracer ├── __init__.py ├── bytecode_tracer.py ├── code_rewriting_importer.py ├── imputil.py └── py_frame_object.py ├── doc ├── FAQ └── basic-tutorial.txt ├── lib2to3 ├── Grammar.txt ├── PatternGrammar.txt ├── __init__.py ├── patcomp.py ├── pgen2 │ ├── __init__.py │ ├── conv.py │ ├── driver.py │ ├── grammar.py │ ├── literals.py │ ├── parse.py │ ├── pgen.py │ ├── token.py │ └── tokenize.py ├── pygram.py └── pytree.py ├── misc └── pythoscope.el ├── pythoscope ├── __init__.py ├── _util.c ├── astbuilder.py ├── astvisitor.py ├── cmdline.py ├── code_trees_manager.py ├── compat.py ├── debug.py ├── event.py ├── execution.py ├── generator │ ├── __init__.py │ ├── adder.py │ ├── assertions.py │ ├── builder.py │ ├── case_namer.py │ ├── cleaner.py │ ├── code_string.py │ ├── constructor.py │ ├── dependencies.py │ ├── lines.py │ ├── method_call_context.py │ ├── objects_namer.py │ ├── optimizer.py │ └── selector.py ├── inspector │ ├── __init__.py │ ├── dynamic.py │ ├── file_system.py │ └── static.py ├── localizable.py ├── logger.py ├── point_of_entry.py ├── py_wrapper_object.py ├── serializer.py ├── side_effect.py ├── snippet.py ├── store.py ├── tracer.py └── util.py ├── scripts └── pythoscope ├── setup.cfg ├── setup.py ├── test ├── __init__.py ├── assertions.py ├── data │ ├── appending_test_cases_added_method_output_expected.py │ ├── appending_test_cases_module_added_method.py │ ├── appending_test_cases_module_initial.py │ ├── appending_test_cases_module_modified.py │ ├── appending_test_cases_output_expected.py │ ├── appending_test_cases_output_initial.py │ ├── attributes_rebind_module.py │ ├── attributes_rebind_output.py │ ├── generic_acceptance_poe.py │ ├── global_variables_module.py │ ├── global_variables_output.py │ ├── objects_identity_module.py │ ├── objects_identity_output.py │ ├── side_effects_on_lists_module.py │ ├── side_effects_on_lists_output.py │ ├── static_analysis_module.py │ └── static_analysis_output.py ├── factories.py ├── generator_helper.py ├── helper.py ├── inspector_assertions.py ├── inspector_helper.py ├── test_acceptance.py ├── test_astbuilder.py ├── test_astvisitor.py ├── test_bytecode_tracer.py ├── test_code_trees_manager.py ├── test_documentation.py ├── test_dynamic_inspector.py ├── test_generator.py ├── test_generator_adder.py ├── test_generator_generate_test_case.py ├── test_inspector.py ├── test_logger.py ├── test_point_of_entry.py ├── test_project.py ├── test_serializer.py ├── test_static_inspector.py ├── test_store.py ├── test_tracing_side_effects.py └── testing_project.py └── tools ├── gather-metrics.py ├── memory_benchmark.py ├── projects ├── Reverend-r17924.tar.gz ├── Reverend_poe_from_homepage.py ├── Reverend_poe_from_readme.py ├── freshwall-1.1.2-lib.tar.gz ├── freshwall_bin_with_snippet.py ├── http-parser-0.2.0.tar.gz ├── http-parser-example-1.py ├── http-parser-example-2.py ├── isodate-0.4.4-src.tar.gz ├── isodate_poe.py ├── pyatom-1.2.tar.gz └── pyatom_poe_from_readme.py ├── rst2wikidot.py ├── run-tests.py └── speed_benchmark.py /.bzrignore: -------------------------------------------------------------------------------- 1 | pythoscope.egg-info 2 | lib2to3/Grammar*.pickle 3 | lib2to3/PatternGrammar*.pickle 4 | build/ 5 | dist/ 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | pythoscope.egg-info 2 | lib2to3/Grammar*.pickle 3 | lib2to3/PatternGrammar*.pickle 4 | build/ 5 | dist/ 6 | *.pyc -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.6" 4 | - "2.7" 5 | install: "pip install nose pinocchio" 6 | script: nosetests 7 | -------------------------------------------------------------------------------- /Changelog: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | 0.4.3 5 | ----- 6 | 7 | * Pythoscope now ignores application and test modules it could not inspect (`#487138 `_ `#462849 `_). 8 | * Improved generation of assertions for functions with varargs and kwargs (`#475414 `_ `#475409 `_). 9 | * Pythoscope will no longer generate two test cases with the same name (`#475504 `_). 10 | * Installation package now contains all essential files (`#481238 `_). 11 | * Pythoscope now properly handles string exceptions thrown inside entry points (`#522364 `_). 12 | * Entry points are now run with the current directory being the project root (`#524352 `_). 13 | 14 | 0.4.2 15 | ----- 16 | 17 | * Removed `dependency on fixture module `_. 18 | * Fixed test generation bug related to function definitions with varargs (`#440773 `_). 19 | * Dynamic inspector can now `handle all types of exceptions `_ (including string exceptions). 20 | * Frequently Asked Questions document has been created, also `available online `_. 21 | * Added `support for Pythons 2.3 `_ `through 2.6 `_. 22 | * New imports added by Pythoscope are now placed after existing ones, so they don't cause a syntax error when __future__ imports were used (`#373978 `_). 23 | * Fixed inspection bug related to classes deriving from namedtuple instances (`#460715 `_). 24 | * `Setuptools is no longer required for installation `_. 25 | 26 | 0.4.1 27 | ----- 28 | 29 | * Greatly improved `information storage performance `_. 30 | * Added Pythoscope module for Emacs to the misc/ directory in the source tree. 31 | * Unittests can now be used as points of entry (`#275059 `_). 32 | * Pythoscope now ignores source control files (`#284568 `_). 33 | * Static inspection happens on --init, as it should be (`#325928 `_). 34 | * Ported itertive pattern matcher for lib2to3 from Python trunk (`#304541 `_). 35 | * Fixed test generation bug related to nested function arguments (`#344220 `_). 36 | * Fixed point of entry cleanup bug (`#324522 `_). 37 | * Fixed some more Windows-specific bugs (`#348136 `_). 38 | 39 | 0.4 40 | --- 41 | 42 | * Stopped `using pickle for object serialization `_ and implemented our own mechanism that carefully captures changing state of objects during dynamic inspection. 43 | * Implemented `preserve objects identity `_ blueprint. 44 | * Fixed bug related to multiple generator calls (`#295340 `_). 45 | * Made handling of special method names (like __init__ or __eq__) consistent with handling of normal method names. 46 | * Made `test stubs more useful `_. 47 | 48 | 0.3.2 49 | ----- 50 | 51 | * Made Pythoscope `more verbose `_. 52 | * Added `support for user-defined exceptions `_. 53 | * Fixed unicode handling bug (`#284585 `_). 54 | * Improved performance of the internal information storage. 55 | 56 | 0.3.1 57 | ----- 58 | 59 | * Added implementation of samefile function for Windows (`#271882 `_). 60 | * Fixed wrong indentation bug (`#271892 `_). 61 | * Made lib2to3.pgen2.parse.ParseError pickable (`#271904 `_). 62 | * Added `support for Python generators `_. 63 | * Fixed static inspection of functions having attributes with default values (`#275459 `_). 64 | 65 | 0.3 66 | --- 67 | 68 | * Fixed generate bug for test modules (`#264449 `_). 69 | * .pythoscope became a directory. 70 | * Introduced --init option for initializing .pythoscope/ directory. 71 | * Added a notion of points of entry introducing dynamic analysis. 72 | * Pythoscope can now generate assert_equal and assert_raises type of assertions. 73 | * Implemented `no more inspect command blueprint `_. 74 | * Changed the default test directory from pythoscope-tests/ to tests/. 75 | * Added a tutorial to the README file. 76 | 77 | 0.2.2 78 | ----- 79 | 80 | * Fixed the inner classes bug (`#260924 `_). 81 | * Collector appends new data to .pythoscope file instead of overwriting it. 82 | * Test modules are being analyzed as well. 83 | * Using lib2to3 for static code analysis instead of stdlib's compiler module. 84 | * Generator can append test cases to existing test modules. Preserves comments and original whitespace. 85 | * Cheetah is no longer a dependency. 86 | * Renamed 'collect' command to 'inspect'. 87 | 88 | 0.2.1 89 | ----- 90 | 91 | Contains a packaging bug fix, which prevented users from using the tests 92 | cases generator and running internal pythoscope tests. 93 | 94 | 0.2 95 | --- 96 | 97 | First release, featuring static code analysis and generation of test 98 | stubs. 99 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2008-2010 Michal Kwiatkowski 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include Changelog 2 | include doc/* 3 | include test/assertions.py 4 | include test/factories.py 5 | include test/helper.py 6 | include test/testing_project.py 7 | include test/__init__.py 8 | include test/data/*.py 9 | include lib2to3/Grammar.txt 10 | include lib2to3/PatternGrammar.txt 11 | include scripts/pythoscope 12 | include pythoscope/_util.c 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Installation 2 | 3 | Easiest way to get Pythoscope is via setuptools: 4 | 5 | $ easy_install pythoscope 6 | 7 | You can also download a source package from 8 | [pythoscope.org](http://pythoscope.org/local--files/download/pythoscope-0.4.3.tar.gz) 9 | or get a copy of a development branch through github: 10 | 11 | $ git clone git@github.com:mkwiatkowski/pythoscope.git 12 | 13 | To install the package from the source directory do: 14 | 15 | $ python setup.py install 16 | 17 | You don't need setuptools for this to work, a bare Python will do just fine. 18 | 19 | However, if you *do* have setuptools installed, you may also consider running 20 | the whole test suite of Pythoscope: 21 | 22 | $ python setup.py test 23 | 24 | ## Usage 25 | 26 | You can use the tool through a single `pythoscope` command. To prepare 27 | your project for use with Pythoscope, type: 28 | 29 | $ pythoscope --init path/to/your/project/ 30 | 31 | It's only doing static analysis, and doesn't import your modules or 32 | execute your code in any way, so you're perfectly safe to run it on 33 | anything you want. After that, a directory named `.pythoscope` will be 34 | created in the current directory. To generate test stubs based on your 35 | project, select files you want to generate tests for: 36 | 37 | $ pythoscope path/to/your/project/specific/file.py path/to/your/project/other/*.py 38 | 39 | Test files will be saved to your test directory, if you have one, or 40 | into a new `tests/` directory otherwise. Test cases are aggregated 41 | into `TestCase` classes. Currently each production class and each 42 | production function gets its own `TestCase` class. 43 | 44 | Some of the classes and functions are ignored by the generator - all 45 | which name begins with an underscore, exception classes, and some 46 | others. 47 | 48 | Generator itself is configurable to some extent, see: 49 | 50 | $ pythoscope --help 51 | 52 | for more information on available options. 53 | 54 | ## Editor Integration 55 | 56 | ### Emacs 57 | 58 | We put out an elisp script that integrates Pythoscope into Emacs. The file is in the the `misc/` directory of the source distribution. You can also [look at the file on github](https://github.com/mkwiatkowski/pythoscope/blob/master/misc/pythoscope.el). Usage and installation instructions are in the comments at the top of the file. 59 | 60 | ### Vim 61 | 62 | There is interest in Vim integration and someone is working on it but we have nothing for you right now. 63 | 64 | ## License 65 | 66 | All Pythoscope source code is licensed under an MIT license (see LICENSE file). 67 | All files under lib2to3/ are licensed under PSF license. 68 | File named imputil.py under bytecode_tracer/ is also licensed under PSF license. 69 | -------------------------------------------------------------------------------- /bytecode_tracer/__init__.py: -------------------------------------------------------------------------------- 1 | from bytecode_tracer import BytecodeTracer, rewrite_function,\ 2 | has_been_rewritten, rewrite_lnotab 3 | -------------------------------------------------------------------------------- /bytecode_tracer/code_rewriting_importer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Custom importer that additionally rewrites code of all imported modules. 3 | 4 | Based on Demo/imputil/importers.py file distributed with Python 2.x. 5 | """ 6 | 7 | import imp 8 | import imputil 9 | import marshal 10 | import os 11 | import struct 12 | import sys 13 | 14 | from types import CodeType 15 | 16 | # byte-compiled file suffic character 17 | _suffix_char = __debug__ and 'c' or 'o' 18 | 19 | # byte-compiled file suffix 20 | _suffix = '.py' + _suffix_char 21 | 22 | # the C_EXTENSION suffixes 23 | _c_suffixes = filter(lambda x: x[2] == imp.C_EXTENSION, imp.get_suffixes()) 24 | 25 | 26 | def _timestamp(pathname): 27 | "Return the file modification time as a Long." 28 | try: 29 | s = os.stat(pathname) 30 | except OSError: 31 | return None 32 | return long(s[8]) 33 | 34 | def _compile(path): 35 | "Read and compile Python source code from file." 36 | f = open(path) 37 | c = f.read() 38 | f.close() 39 | return compile(c, path, 'exec') 40 | 41 | def _fs_import(dir, modname, fqname): 42 | "Fetch a module from the filesystem." 43 | 44 | pathname = os.path.join(dir, modname) 45 | if os.path.isdir(pathname): 46 | values = { '__pkgdir__' : pathname, '__path__' : [ pathname ] } 47 | ispkg = 1 48 | pathname = os.path.join(pathname, '__init__') 49 | else: 50 | values = { } 51 | ispkg = 0 52 | 53 | # look for dynload modules 54 | for desc in _c_suffixes: 55 | file = pathname + desc[0] 56 | try: 57 | fp = open(file, desc[1]) 58 | except IOError: 59 | pass 60 | else: 61 | module = imp.load_module(fqname, fp, file, desc) 62 | values['__file__'] = file 63 | return 0, module, values 64 | 65 | t_py = _timestamp(pathname + '.py') 66 | t_pyc = _timestamp(pathname + _suffix) 67 | if t_py is None and t_pyc is None: 68 | return None 69 | code = None 70 | if t_py is None or (t_pyc is not None and t_pyc >= t_py): 71 | file = pathname + _suffix 72 | f = open(file, 'rb') 73 | if f.read(4) == imp.get_magic(): 74 | t = struct.unpack('`_). If you have a system written in Python and value testing, Pythoscope can help you achieve a high test coverage. 11 | 12 | Why should I use Pythoscope? 13 | ---------------------------- 14 | 15 | If you want to introduce testing to your project, Pythoscope will be helpful to you, regardless if you're just starting out, or your project is in maintenance mode. 16 | 17 | Having a comprehensive set of automated unit tests allows to make changes to the software quicker, by: 18 | 19 | * reducing the number of bugs introduced during modification of a system, 20 | * localizing defects much quicker (and at the same time reducing the need for debugging), 21 | * helping developers understand the system, 22 | * making the development process more predictable, 23 | * and more. 24 | 25 | For more comprehensive discussion of unit test benefits see: 26 | 27 | * `The benefits of automated unit testing `_ 28 | * `Cost benefits of Unit Testing `_ 29 | * `Goals of Test Automation `_ 30 | 31 | Does Pythoscope support iterative development? 32 | ---------------------------------------------- 33 | 34 | Absolutely! 35 | 36 | As you modify your system you can run Pythoscope over and over again to complement the test suite with the newly added behavior. At the same time you are free to remove and/or modify those tests - Pythoscope will leave your changes intact. 37 | 38 | How stable Pythoscope is? 39 | ------------------------- 40 | 41 | Pythoscope is in an experimentation stage of development, so stability is not our main priority right now. Having said that, it shouldn't overwrite or delete any of your files, so you're free to experiment. The worst thing that can happen is that you'll get a Python stack trace. In that case, please `report a bug `_. :-) 42 | 43 | If you want to interface with Pythoscope (for example, for editor integration), please let us know, so we could take future API changes into consideration. 44 | 45 | What license does Pythoscope use? 46 | --------------------------------- 47 | 48 | All Pythoscope code is licensed under `MIT license `_. Pythoscope releases include a lib2to3 library, which is licensed under `PSF license `_. 49 | 50 | 51 | Installation and usage 52 | ====================== 53 | 54 | Where can I get Pythoscope? 55 | --------------------------- 56 | 57 | After each release we publish packages in the following places: 58 | 59 | * `download page at wikidot `_ 60 | * `PyPI `_ 61 | * `Launchpad `_ 62 | 63 | If you want the latest development version use `bazaar `_:: 64 | 65 | $ bzr branch lp:pythoscope 66 | 67 | What versions of Python are supported? 68 | -------------------------------------- 69 | 70 | Since release 0.4.2 Pythoscope supports Pythons 2.3 through 2.6. Python 3.0 is not supported yet. `Let us know `_ if you need Python 3.0 support. 71 | 72 | What OSes are supported? 73 | ------------------------ 74 | 75 | Pythoscope works on Linux, Windows and MacOS. 76 | 77 | What about Jython, IronPython or PyPy? 78 | -------------------------------------- 79 | 80 | Currently Pythoscope is developed on CPython exclusively and we have never tested it on alterive implementations. Eventually we'd like to support all Python implementations, so if you're adventurous please do try Pythoscope on the implementation of your choice and let us know of your findings. 81 | 82 | Can I use an existing unit test as a point of entry? 83 | ---------------------------------------------------- 84 | 85 | Yes, you can. Because Pythoscope doesn't execute points of entry in 86 | separate modules, you won't be able to use **unittest.main()** to 87 | run them though. Use the following snippet to run a test class 88 | (see `unittest module documentation `_ for details):: 89 | 90 | suite = unittest.TestLoader().loadTestsFromTestCase(YourTestClass) 91 | unittest.TextTestRunner(verbosity=2).run(suite) 92 | 93 | Communication 94 | ============= 95 | 96 | Where do I go for help? 97 | ----------------------- 98 | 99 | The best place to get help is `the pythoscope mailing list `_. Both Pythoscope users and developers hang out there, so you're free to ask there all kinds of questions related to Pythoscope. If you'd rather use a private channel, just drop us a line at developers@pythoscope.org. 100 | -------------------------------------------------------------------------------- /doc/basic-tutorial.txt: -------------------------------------------------------------------------------- 1 | Let's say you're working on this old Python project. It's ugly and 2 | unpredictable, but you have no choice but to keep it alive. Luckily 3 | you've heard about this new tool called Pythoscope, which can help 4 | you cure the old guy. 5 | 6 | You start by descending to the project directory:: 7 | 8 | $ cd wild_pythons/ 9 | 10 | and initializing Pythoscope internal structures:: 11 | 12 | $ pythoscope --init 13 | 14 | This command creates **.pythoscope/** subdirectory, which will hold all 15 | information related to Pythoscope. You look at the poor snake:: 16 | 17 | # old_python.py 18 | class OldPython(object): 19 | def __init__(self, age): 20 | pass # more code... 21 | 22 | def hiss(self): 23 | pass # even more code... 24 | 25 | and decide that it requires immediate attention. So you run Pythoscope 26 | on it:: 27 | 28 | $ pythoscope old_python.py 29 | 30 | and see a test module generated in the **tests/** directory:: 31 | 32 | # tests/test_old_python.py 33 | import unittest 34 | 35 | class TestOldPython(unittest.TestCase): 36 | def test___init__(self): 37 | # old_python = OldPython(age) 38 | assert False # TODO: implement your test here 39 | 40 | def test_hiss(self): 41 | # old_python = OldPython(age) 42 | # self.assertEqual(expected, old_python.hiss()) 43 | assert False # TODO: implement your test here 44 | 45 | if __name__ == '__main__': 46 | unittest.main() 47 | 48 | That's a starting point for your testing struggle, but there's much more 49 | Pythoscope can help you with. All you have to do is give it some more 50 | information about your project. 51 | 52 | Since Python is a very dynamic language it's no surprise that most 53 | information about the application can be gathered during runtime. But legacy 54 | applications can be tricky and dangerous, so Pythoscope won't run any code 55 | at all unless you explicitly tell it to do so. You can specify which code 56 | is safe to run through, so called, points of entry. 57 | 58 | Point of entry is a plain Python module that executes some parts of your code. 59 | You should keep each point of entry in a separate file in the 60 | **.pythoscope/points-of-entry/** directory. Let's look closer at our old friend:: 61 | 62 | # old_python.py 63 | class OldPython(object): 64 | def __init__(self, age): 65 | if age < 50: 66 | raise ValueError("%d isn't old" % age) 67 | self.age = age 68 | 69 | def hiss(self): 70 | if self.age < 60: 71 | return "sss sss" 72 | elif self.age < 70: 73 | return "SSss SSss" 74 | else: 75 | return "sss... *cough* *cough*" 76 | 77 | Based on that definition we come up with the following point of entry:: 78 | 79 | # .pythoscope/points-of-entry/123_years_old_python.py 80 | from old_python import OldPython 81 | OldPython(123).hiss() 82 | 83 | Once we have that we may try to generate new test cases. Simply call pythoscope 84 | on the old python again:: 85 | 86 | $ pythoscope old_python.py 87 | 88 | Pythoscope will execute our new point of entry, gathering as much dynamic 89 | information as possible. If you look at your test module now you'll notice 90 | that a new test case has been added:: 91 | 92 | # tests/test_old_python.py 93 | import unittest 94 | from old_python import OldPython 95 | 96 | class TestOldPython(unittest.TestCase): 97 | def test___init__(self): 98 | # old_python = OldPython(age) 99 | assert False # TODO: implement your test here 100 | 101 | def test_hiss(self): 102 | # old_python = OldPython(age) 103 | # self.assertEqual(expected, old_python.hiss()) 104 | assert False # TODO: implement your test here 105 | 106 | def test_hiss_returns_sss_cough_cough_after_creation_with_123(self): 107 | old_python = OldPython(123) 108 | self.assertEqual('sss... *cough* *cough*', old_python.hiss()) 109 | 110 | if __name__ == '__main__': 111 | unittest.main() 112 | 113 | Pythoscope correctly captured creation of the **OldPython** object and call to 114 | its **hiss()** method. Congratulations, you have a first working test case without 115 | doing much work! But Pythoscope can generate more than just invdividual test 116 | cases. It all depends on the points of entry you define. More high-level they 117 | are, the more information Pythoscope will be able to gather, which directly 118 | translates to the number of generated test cases. 119 | 120 | So let's try writing another point of entry. But first look at another module 121 | we have in our project:: 122 | 123 | # old_nest.py 124 | from old_python import OldPython 125 | 126 | class OldNest(object): 127 | def __init__(self, ages): 128 | self.pythons = [] 129 | for age in ages: 130 | try: 131 | self.pythons.append(OldPython(age)) 132 | except ValueError: 133 | pass # Ignore the youngsters. 134 | def put_hand(self): 135 | return '\n'.join([python.hiss() for python in self.pythons]) 136 | 137 | This module seems a bit higher-level than **old_python.py**. Yet, writing a point 138 | of entry for it is also straightforward:: 139 | 140 | # .pythoscope/points-of-entry/old_nest_with_four_pythons.py 141 | from old_nest import OldNest 142 | OldNest([45, 55, 65, 75]).put_hand() 143 | 144 | Don't hesitate and run Pythoscope right away. Note that you can provide many 145 | modules as arguments - all of them will be handled at once:: 146 | 147 | $ pythoscope old_python.py old_nest.py 148 | 149 | This new point of entry not only allowed to create a test case for **OldNest**:: 150 | 151 | # tests/test_old_nest.py 152 | import unittest 153 | from old_nest import OldNest 154 | 155 | class TestOldNest(unittest.TestCase): 156 | def test_put_hand_returns_sss_sss_SSss_SSss_sss_cough_cough_after_creation_with_list(self): 157 | old_nest = OldNest([45, 55, 65, 75]) 158 | self.assertEqual('sss sss\nSSss SSss\nsss... *cough* *cough*', old_nest.put_hand()) 159 | 160 | if __name__ == '__main__': 161 | unittest.main() 162 | 163 | but also added 4 new test cases for **OldPython**:: 164 | 165 | def test_creation_with_45_raises_value_error(self): 166 | self.assertRaises(ValueError, lambda: OldPython(45)) 167 | 168 | def test_hiss_returns_SSss_SSss_after_creation_with_65(self): 169 | old_python = OldPython(65) 170 | self.assertEqual('SSss SSss', old_python.hiss()) 171 | 172 | def test_hiss_returns_sss_cough_cough_after_creation_with_75(self): 173 | old_python = OldPython(75) 174 | self.assertEqual('sss... *cough* *cough*', old_python.hiss()) 175 | 176 | def test_hiss_returns_sss_sss_after_creation_with_55(self): 177 | old_python = OldPython(55) 178 | self.assertEqual('sss sss', old_python.hiss()) 179 | 180 | You got all of that for mere 2 additional lines of code. What's even better 181 | is the fact that you can safely modify and extend test cases generated by 182 | Pythoscope. Once you write another point of entry or add new behavior to your 183 | system you can run Pythoscope again and it will only append new test cases 184 | to existing test modules, preserving any modifications you could have made 185 | to them. 186 | 187 | That sums up this basic tutorial. If you have any questions, feel free to 188 | ask them on the `pythoscope google group `_. 189 | -------------------------------------------------------------------------------- /lib2to3/Grammar.txt: -------------------------------------------------------------------------------- 1 | # Grammar for Python 2 | 3 | # Note: Changing the grammar specified in this file will most likely 4 | # require corresponding changes in the parser module 5 | # (../Modules/parsermodule.c). If you can't make the changes to 6 | # that module yourself, please co-ordinate the required changes 7 | # with someone who can; ask around on python-dev for help. Fred 8 | # Drake will probably be listening there. 9 | 10 | # NOTE WELL: You should also follow all the steps listed in PEP 306, 11 | # "How to Change Python's Grammar" 12 | 13 | # Commands for Kees Blom's railroad program 14 | #diagram:token NAME 15 | #diagram:token NUMBER 16 | #diagram:token STRING 17 | #diagram:token NEWLINE 18 | #diagram:token ENDMARKER 19 | #diagram:token INDENT 20 | #diagram:output\input python.bla 21 | #diagram:token DEDENT 22 | #diagram:output\textwidth 20.04cm\oddsidemargin 0.0cm\evensidemargin 0.0cm 23 | #diagram:rules 24 | 25 | # Start symbols for the grammar: 26 | # file_input is a module or sequence of commands read from an input file; 27 | # single_input is a single interactive statement; 28 | # eval_input is the input for the eval() and input() functions. 29 | # NB: compound_stmt in single_input is followed by extra NEWLINE! 30 | file_input: (NEWLINE | stmt)* ENDMARKER 31 | single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE 32 | eval_input: testlist NEWLINE* ENDMARKER 33 | 34 | decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE 35 | decorators: decorator+ 36 | decorated: decorators (classdef | funcdef) 37 | funcdef: 'def' NAME parameters ['->' test] ':' suite 38 | parameters: '(' [typedargslist] ')' 39 | typedargslist: ((tfpdef ['=' test] ',')* 40 | ('*' [tname] (',' tname ['=' test])* [',' '**' tname] | '**' tname) 41 | | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) 42 | tname: NAME [':' test] 43 | tfpdef: tname | '(' tfplist ')' 44 | tfplist: tfpdef (',' tfpdef)* [','] 45 | varargslist: ((vfpdef ['=' test] ',')* 46 | ('*' [vname] (',' vname ['=' test])* [',' '**' vname] | '**' vname) 47 | | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) 48 | vname: NAME 49 | vfpdef: vname | '(' vfplist ')' 50 | vfplist: vfpdef (',' vfpdef)* [','] 51 | 52 | stmt: simple_stmt | compound_stmt 53 | simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE 54 | small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | 55 | import_stmt | global_stmt | exec_stmt | assert_stmt) 56 | expr_stmt: testlist (augassign (yield_expr|testlist) | 57 | ('=' (yield_expr|testlist))*) 58 | augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | 59 | '<<=' | '>>=' | '**=' | '//=') 60 | # For normal assignments, additional restrictions enforced by the interpreter 61 | print_stmt: 'print' ( [ test (',' test)* [','] ] | 62 | '>>' test [ (',' test)+ [','] ] ) 63 | del_stmt: 'del' exprlist 64 | pass_stmt: 'pass' 65 | flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt 66 | break_stmt: 'break' 67 | continue_stmt: 'continue' 68 | return_stmt: 'return' [testlist] 69 | yield_stmt: yield_expr 70 | raise_stmt: 'raise' [test ['from' test | ',' test [',' test]]] 71 | import_stmt: import_name | import_from 72 | import_name: 'import' dotted_as_names 73 | import_from: ('from' ('.'* dotted_name | '.'+) 74 | 'import' ('*' | '(' import_as_names ')' | import_as_names)) 75 | import_as_name: NAME ['as' NAME] 76 | dotted_as_name: dotted_name ['as' NAME] 77 | import_as_names: import_as_name (',' import_as_name)* [','] 78 | dotted_as_names: dotted_as_name (',' dotted_as_name)* 79 | dotted_name: NAME ('.' NAME)* 80 | global_stmt: ('global' | 'nonlocal') NAME (',' NAME)* 81 | exec_stmt: 'exec' expr ['in' test [',' test]] 82 | assert_stmt: 'assert' test [',' test] 83 | 84 | compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated 85 | if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] 86 | while_stmt: 'while' test ':' suite ['else' ':' suite] 87 | for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] 88 | try_stmt: ('try' ':' suite 89 | ((except_clause ':' suite)+ 90 | ['else' ':' suite] 91 | ['finally' ':' suite] | 92 | 'finally' ':' suite)) 93 | with_stmt: 'with' test [ with_var ] ':' suite 94 | with_var: 'as' expr 95 | # NB compile.c makes sure that the default except clause is last 96 | except_clause: 'except' [test [(',' | 'as') test]] 97 | suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT 98 | 99 | # Backward compatibility cruft to support: 100 | # [ x for x in lambda: True, lambda: False if x() ] 101 | # even while also allowing: 102 | # lambda x: 5 if x else 2 103 | # (But not a mix of the two) 104 | testlist_safe: old_test [(',' old_test)+ [',']] 105 | old_test: or_test | old_lambdef 106 | old_lambdef: 'lambda' [varargslist] ':' old_test 107 | 108 | test: or_test ['if' or_test 'else' test] | lambdef 109 | or_test: and_test ('or' and_test)* 110 | and_test: not_test ('and' not_test)* 111 | not_test: 'not' not_test | comparison 112 | comparison: expr (comp_op expr)* 113 | comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' 114 | expr: xor_expr ('|' xor_expr)* 115 | xor_expr: and_expr ('^' and_expr)* 116 | and_expr: shift_expr ('&' shift_expr)* 117 | shift_expr: arith_expr (('<<'|'>>') arith_expr)* 118 | arith_expr: term (('+'|'-') term)* 119 | term: factor (('*'|'/'|'%'|'//') factor)* 120 | factor: ('+'|'-'|'~') factor | power 121 | power: atom trailer* ['**' factor] 122 | atom: ('(' [yield_expr|testlist_gexp] ')' | 123 | '[' [listmaker] ']' | 124 | '{' [dictsetmaker] '}' | 125 | '`' testlist1 '`' | 126 | NAME | NUMBER | STRING+ | '.' '.' '.') 127 | listmaker: test ( comp_for | (',' test)* [','] ) 128 | testlist_gexp: test ( comp_for | (',' test)* [','] ) 129 | lambdef: 'lambda' [varargslist] ':' test 130 | trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME 131 | subscriptlist: subscript (',' subscript)* [','] 132 | subscript: test | [test] ':' [test] [sliceop] 133 | sliceop: ':' [test] 134 | exprlist: expr (',' expr)* [','] 135 | testlist: test (',' test)* [','] 136 | dictsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | 137 | (test (comp_for | (',' test)* [','])) ) 138 | 139 | classdef: 'class' NAME ['(' [arglist] ')'] ':' suite 140 | 141 | arglist: (argument ',')* (argument [','] 142 | |'*' test (',' argument)* [',' '**' test] 143 | |'**' test) 144 | argument: test [comp_for] | test '=' test # Really [keyword '='] test 145 | 146 | comp_iter: comp_for | comp_if 147 | comp_for: 'for' exprlist 'in' testlist_safe [comp_iter] 148 | comp_if: 'if' old_test [comp_iter] 149 | 150 | testlist1: test (',' test)* 151 | 152 | # not used in grammar, but may appear in "node" passed from Parser to Compiler 153 | encoding_decl: NAME 154 | 155 | yield_expr: 'yield' [testlist] 156 | -------------------------------------------------------------------------------- /lib2to3/PatternGrammar.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2006 Google, Inc. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | 4 | # A grammar to describe tree matching patterns. 5 | # Not shown here: 6 | # - 'TOKEN' stands for any token (leaf node) 7 | # - 'any' stands for any node (leaf or interior) 8 | # With 'any' we can still specify the sub-structure. 9 | 10 | # The start symbol is 'Matcher'. 11 | 12 | Matcher: Alternatives ENDMARKER 13 | 14 | Alternatives: Alternative ('|' Alternative)* 15 | 16 | Alternative: (Unit | NegatedUnit)+ 17 | 18 | Unit: [NAME '='] ( STRING [Repeater] 19 | | NAME [Details] [Repeater] 20 | | '(' Alternatives ')' [Repeater] 21 | | '[' Alternatives ']' 22 | ) 23 | 24 | NegatedUnit: 'not' (STRING | NAME [Details] | '(' Alternatives ')') 25 | 26 | Repeater: '*' | '+' | '{' NUMBER [',' NUMBER] '}' 27 | 28 | Details: '<' Alternatives '>' 29 | -------------------------------------------------------------------------------- /lib2to3/__init__.py: -------------------------------------------------------------------------------- 1 | #empty 2 | -------------------------------------------------------------------------------- /lib2to3/patcomp.py: -------------------------------------------------------------------------------- 1 | # Copyright 2006 Google, Inc. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | 4 | """Pattern compiler. 5 | 6 | The grammer is taken from PatternGrammar.txt. 7 | 8 | The compiler compiles a pattern to a pytree.*Pattern instance. 9 | """ 10 | 11 | __author__ = "Guido van Rossum " 12 | 13 | # Python imports 14 | import os 15 | 16 | # Fairly local imports 17 | from pgen2 import driver 18 | from pgen2 import literals 19 | from pgen2 import token 20 | from pgen2 import tokenize 21 | 22 | # Really local imports 23 | import pytree 24 | import pygram 25 | 26 | # The pattern grammar file 27 | _PATTERN_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), 28 | "PatternGrammar.txt") 29 | 30 | 31 | def tokenize_wrapper(input): 32 | """Tokenizes a string suppressing significant whitespace.""" 33 | skip = (token.NEWLINE, token.INDENT, token.DEDENT) 34 | tokens = tokenize.generate_tokens(driver.generate_lines(input).next) 35 | for quintuple in tokens: 36 | type, value, start, end, line_text = quintuple 37 | if type not in skip: 38 | yield quintuple 39 | 40 | 41 | class PatternCompiler(object): 42 | 43 | def __init__(self, grammar_file=_PATTERN_GRAMMAR_FILE): 44 | """Initializer. 45 | 46 | Takes an optional alternative filename for the pattern grammar. 47 | """ 48 | self.grammar = driver.load_grammar(grammar_file) 49 | self.syms = pygram.Symbols(self.grammar) 50 | self.pygrammar = pygram.python_grammar 51 | self.pysyms = pygram.python_symbols 52 | self.driver = driver.Driver(self.grammar, convert=pattern_convert) 53 | 54 | def compile_pattern(self, input, debug=False): 55 | """Compiles a pattern string to a nested pytree.*Pattern object.""" 56 | tokens = tokenize_wrapper(input) 57 | root = self.driver.parse_tokens(tokens, debug=debug) 58 | return self.compile_node(root) 59 | 60 | def compile_node(self, node): 61 | """Compiles a node, recursively. 62 | 63 | This is one big switch on the node type. 64 | """ 65 | # XXX Optimize certain Wildcard-containing-Wildcard patterns 66 | # that can be merged 67 | if node.type == self.syms.Matcher: 68 | node = node.children[0] # Avoid unneeded recursion 69 | 70 | if node.type == self.syms.Alternatives: 71 | # Skip the odd children since they are just '|' tokens 72 | alts = [self.compile_node(ch) for ch in node.children[::2]] 73 | if len(alts) == 1: 74 | return alts[0] 75 | p = pytree.WildcardPattern([[a] for a in alts], min=1, max=1) 76 | return p.optimize() 77 | 78 | if node.type == self.syms.Alternative: 79 | units = [self.compile_node(ch) for ch in node.children] 80 | if len(units) == 1: 81 | return units[0] 82 | p = pytree.WildcardPattern([units], min=1, max=1) 83 | return p.optimize() 84 | 85 | if node.type == self.syms.NegatedUnit: 86 | pattern = self.compile_basic(node.children[1:]) 87 | p = pytree.NegatedPattern(pattern) 88 | return p.optimize() 89 | 90 | assert node.type == self.syms.Unit 91 | 92 | name = None 93 | nodes = node.children 94 | if len(nodes) >= 3 and nodes[1].type == token.EQUAL: 95 | name = nodes[0].value 96 | nodes = nodes[2:] 97 | repeat = None 98 | if len(nodes) >= 2 and nodes[-1].type == self.syms.Repeater: 99 | repeat = nodes[-1] 100 | nodes = nodes[:-1] 101 | 102 | # Now we've reduced it to: STRING | NAME [Details] | (...) | [...] 103 | pattern = self.compile_basic(nodes, repeat) 104 | 105 | if repeat is not None: 106 | assert repeat.type == self.syms.Repeater 107 | children = repeat.children 108 | child = children[0] 109 | if child.type == token.STAR: 110 | min = 0 111 | max = pytree.HUGE 112 | elif child.type == token.PLUS: 113 | min = 1 114 | max = pytree.HUGE 115 | elif child.type == token.LBRACE: 116 | assert children[-1].type == token.RBRACE 117 | assert len(children) in (3, 5) 118 | min = max = self.get_int(children[1]) 119 | if len(children) == 5: 120 | max = self.get_int(children[3]) 121 | else: 122 | assert False 123 | if min != 1 or max != 1: 124 | pattern = pattern.optimize() 125 | pattern = pytree.WildcardPattern([[pattern]], min=min, max=max) 126 | 127 | if name is not None: 128 | pattern.name = name 129 | return pattern.optimize() 130 | 131 | def compile_basic(self, nodes, repeat=None): 132 | # Compile STRING | NAME [Details] | (...) | [...] 133 | assert len(nodes) >= 1 134 | node = nodes[0] 135 | if node.type == token.STRING: 136 | value = literals.evalString(node.value) 137 | return pytree.LeafPattern(content=value) 138 | elif node.type == token.NAME: 139 | value = node.value 140 | if value.isupper(): 141 | if value not in TOKEN_MAP: 142 | raise SyntaxError("Invalid token: %r" % value) 143 | return pytree.LeafPattern(TOKEN_MAP[value]) 144 | else: 145 | if value == "any": 146 | type = None 147 | elif not value.startswith("_"): 148 | type = getattr(self.pysyms, value, None) 149 | if type is None: 150 | raise SyntaxError("Invalid symbol: %r" % value) 151 | if nodes[1:]: # Details present 152 | content = [self.compile_node(nodes[1].children[1])] 153 | else: 154 | content = None 155 | return pytree.NodePattern(type, content) 156 | elif node.value == "(": 157 | return self.compile_node(nodes[1]) 158 | elif node.value == "[": 159 | assert repeat is None 160 | subpattern = self.compile_node(nodes[1]) 161 | return pytree.WildcardPattern([[subpattern]], min=0, max=1) 162 | assert False, node 163 | 164 | def get_int(self, node): 165 | assert node.type == token.NUMBER 166 | return int(node.value) 167 | 168 | 169 | # Map named tokens to the type value for a LeafPattern 170 | TOKEN_MAP = {"NAME": token.NAME, 171 | "STRING": token.STRING, 172 | "NUMBER": token.NUMBER, 173 | "TOKEN": None} 174 | 175 | 176 | def pattern_convert(grammar, raw_node_info): 177 | """Converts raw node information to a Node or Leaf instance.""" 178 | type, value, context, children = raw_node_info 179 | if children or type in grammar.number2symbol: 180 | return pytree.Node(type, children, context=context) 181 | else: 182 | return pytree.Leaf(type, value, context=context) 183 | 184 | 185 | def compile_pattern(pattern): 186 | return PatternCompiler().compile_pattern(pattern) 187 | -------------------------------------------------------------------------------- /lib2to3/pgen2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | 4 | """The pgen2 package.""" 5 | -------------------------------------------------------------------------------- /lib2to3/pgen2/driver.py: -------------------------------------------------------------------------------- 1 | # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | 4 | # Modifications: 5 | # Copyright 2006 Google, Inc. All Rights Reserved. 6 | # Licensed to PSF under a Contributor Agreement. 7 | 8 | """Parser driver. 9 | 10 | This provides a high-level interface to parse a file into a syntax tree. 11 | 12 | """ 13 | 14 | __author__ = "Guido van Rossum " 15 | 16 | __all__ = ["Driver", "load_grammar"] 17 | 18 | # Python imports 19 | import os 20 | import logging 21 | import sys 22 | 23 | # Pgen imports 24 | import grammar, parse, token, tokenize, pgen 25 | 26 | 27 | class Driver(object): 28 | 29 | def __init__(self, grammar, convert=None, logger=None): 30 | self.grammar = grammar 31 | if logger is None: 32 | logger = logging.getLogger() 33 | self.logger = logger 34 | self.convert = convert 35 | 36 | def parse_tokens(self, tokens, debug=False): 37 | """Parse a series of tokens and return the syntax tree.""" 38 | # XXX Move the prefix computation into a wrapper around tokenize. 39 | p = parse.Parser(self.grammar, self.convert) 40 | p.setup() 41 | lineno = 1 42 | column = 0 43 | type = value = start = end = line_text = None 44 | prefix = "" 45 | for quintuple in tokens: 46 | type, value, start, end, line_text = quintuple 47 | if start != (lineno, column): 48 | assert (lineno, column) <= start, ((lineno, column), start) 49 | s_lineno, s_column = start 50 | if lineno < s_lineno: 51 | prefix += "\n" * (s_lineno - lineno) 52 | lineno = s_lineno 53 | column = 0 54 | if column < s_column: 55 | prefix += line_text[column:s_column] 56 | column = s_column 57 | if type in (tokenize.COMMENT, tokenize.NL): 58 | prefix += value 59 | lineno, column = end 60 | if value.endswith("\n"): 61 | lineno += 1 62 | column = 0 63 | continue 64 | if type == token.OP: 65 | type = grammar.opmap[value] 66 | if debug: 67 | self.logger.debug("%s %r (prefix=%r)", 68 | token.tok_name[type], value, prefix) 69 | if p.addtoken(type, value, (prefix, start)): 70 | if debug: 71 | self.logger.debug("Stop.") 72 | break 73 | prefix = "" 74 | lineno, column = end 75 | if value.endswith("\n"): 76 | lineno += 1 77 | column = 0 78 | else: 79 | # We never broke out -- EOF is too soon (how can this happen???) 80 | raise parse.ParseError("incomplete input", 81 | type, value, (prefix, start)) 82 | return p.rootnode 83 | 84 | def parse_stream_raw(self, stream, debug=False): 85 | """Parse a stream and return the syntax tree.""" 86 | tokens = tokenize.generate_tokens(stream.readline) 87 | return self.parse_tokens(tokens, debug) 88 | 89 | def parse_stream(self, stream, debug=False): 90 | """Parse a stream and return the syntax tree.""" 91 | return self.parse_stream_raw(stream, debug) 92 | 93 | def parse_file(self, filename, debug=False): 94 | """Parse a file and return the syntax tree.""" 95 | stream = open(filename) 96 | try: 97 | return self.parse_stream(stream, debug) 98 | finally: 99 | stream.close() 100 | 101 | def parse_string(self, text, debug=False): 102 | """Parse a string and return the syntax tree.""" 103 | tokens = tokenize.generate_tokens(generate_lines(text).next) 104 | return self.parse_tokens(tokens, debug) 105 | 106 | 107 | def generate_lines(text): 108 | """Generator that behaves like readline without using StringIO.""" 109 | for line in text.splitlines(True): 110 | yield line 111 | while True: 112 | yield "" 113 | 114 | 115 | def load_grammar(gt="Grammar.txt", gp=None, 116 | save=True, force=False, logger=None): 117 | """Load the grammar (maybe from a pickle).""" 118 | if logger is None: 119 | logger = logging.getLogger() 120 | if gp is None: 121 | head, tail = os.path.splitext(gt) 122 | if tail == ".txt": 123 | tail = "" 124 | gp = head + tail + ".".join(map(str, sys.version_info)) + ".pickle" 125 | if force or not _newer(gp, gt): 126 | logger.info("Generating grammar tables from %s", gt) 127 | g = pgen.generate_grammar(gt) 128 | if save: 129 | logger.info("Writing grammar tables to %s", gp) 130 | try: 131 | g.dump(gp) 132 | except IOError, e: 133 | logger.info("Writing failed:"+str(e)) 134 | else: 135 | g = grammar.Grammar() 136 | g.load(gp) 137 | return g 138 | 139 | 140 | def _newer(a, b): 141 | """Inquire whether file a was written since file b.""" 142 | if not os.path.exists(a): 143 | return False 144 | if not os.path.exists(b): 145 | return True 146 | return os.path.getmtime(a) >= os.path.getmtime(b) 147 | -------------------------------------------------------------------------------- /lib2to3/pgen2/grammar.py: -------------------------------------------------------------------------------- 1 | # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | 4 | """This module defines the data structures used to represent a grammar. 5 | 6 | These are a bit arcane because they are derived from the data 7 | structures used by Python's 'pgen' parser generator. 8 | 9 | There's also a table here mapping operators to their names in the 10 | token module; the Python tokenize module reports all operators as the 11 | fallback token code OP, but the parser needs the actual token code. 12 | 13 | """ 14 | 15 | # Python imports 16 | import pickle 17 | 18 | # Local imports 19 | import token, tokenize 20 | 21 | 22 | class Grammar(object): 23 | """Pgen parsing tables tables conversion class. 24 | 25 | Once initialized, this class supplies the grammar tables for the 26 | parsing engine implemented by parse.py. The parsing engine 27 | accesses the instance variables directly. The class here does not 28 | provide initialization of the tables; several subclasses exist to 29 | do this (see the conv and pgen modules). 30 | 31 | The load() method reads the tables from a pickle file, which is 32 | much faster than the other ways offered by subclasses. The pickle 33 | file is written by calling dump() (after loading the grammar 34 | tables using a subclass). The report() method prints a readable 35 | representation of the tables to stdout, for debugging. 36 | 37 | The instance variables are as follows: 38 | 39 | symbol2number -- a dict mapping symbol names to numbers. Symbol 40 | numbers are always 256 or higher, to distinguish 41 | them from token numbers, which are between 0 and 42 | 255 (inclusive). 43 | 44 | number2symbol -- a dict mapping numbers to symbol names; 45 | these two are each other's inverse. 46 | 47 | states -- a list of DFAs, where each DFA is a list of 48 | states, each state is is a list of arcs, and each 49 | arc is a (i, j) pair where i is a label and j is 50 | a state number. The DFA number is the index into 51 | this list. (This name is slightly confusing.) 52 | Final states are represented by a special arc of 53 | the form (0, j) where j is its own state number. 54 | 55 | dfas -- a dict mapping symbol numbers to (DFA, first) 56 | pairs, where DFA is an item from the states list 57 | above, and first is a set of tokens that can 58 | begin this grammar rule (represented by a dict 59 | whose values are always 1). 60 | 61 | labels -- a list of (x, y) pairs where x is either a token 62 | number or a symbol number, and y is either None 63 | or a string; the strings are keywords. The label 64 | number is the index in this list; label numbers 65 | are used to mark state transitions (arcs) in the 66 | DFAs. 67 | 68 | start -- the number of the grammar's start symbol. 69 | 70 | keywords -- a dict mapping keyword strings to arc labels. 71 | 72 | tokens -- a dict mapping token numbers to arc labels. 73 | 74 | """ 75 | 76 | def __init__(self): 77 | self.symbol2number = {} 78 | self.number2symbol = {} 79 | self.states = [] 80 | self.dfas = {} 81 | self.labels = [(0, "EMPTY")] 82 | self.keywords = {} 83 | self.tokens = {} 84 | self.symbol2label = {} 85 | self.start = 256 86 | 87 | def dump(self, filename): 88 | """Dump the grammar tables to a pickle file.""" 89 | f = open(filename, "wb") 90 | pickle.dump(self.__dict__, f, 2) 91 | f.close() 92 | 93 | def load(self, filename): 94 | """Load the grammar tables from a pickle file.""" 95 | f = open(filename, "rb") 96 | d = pickle.load(f) 97 | f.close() 98 | self.__dict__.update(d) 99 | 100 | def report(self): 101 | """Dump the grammar tables to standard output, for debugging.""" 102 | from pprint import pprint 103 | print "s2n" 104 | pprint(self.symbol2number) 105 | print "n2s" 106 | pprint(self.number2symbol) 107 | print "states" 108 | pprint(self.states) 109 | print "dfas" 110 | pprint(self.dfas) 111 | print "labels" 112 | pprint(self.labels) 113 | print "start", self.start 114 | 115 | 116 | # Map from operator to number (since tokenize doesn't do this) 117 | 118 | opmap_raw = """ 119 | ( LPAR 120 | ) RPAR 121 | [ LSQB 122 | ] RSQB 123 | : COLON 124 | , COMMA 125 | ; SEMI 126 | + PLUS 127 | - MINUS 128 | * STAR 129 | / SLASH 130 | | VBAR 131 | & AMPER 132 | < LESS 133 | > GREATER 134 | = EQUAL 135 | . DOT 136 | % PERCENT 137 | ` BACKQUOTE 138 | { LBRACE 139 | } RBRACE 140 | @ AT 141 | == EQEQUAL 142 | != NOTEQUAL 143 | <> NOTEQUAL 144 | <= LESSEQUAL 145 | >= GREATEREQUAL 146 | ~ TILDE 147 | ^ CIRCUMFLEX 148 | << LEFTSHIFT 149 | >> RIGHTSHIFT 150 | ** DOUBLESTAR 151 | += PLUSEQUAL 152 | -= MINEQUAL 153 | *= STAREQUAL 154 | /= SLASHEQUAL 155 | %= PERCENTEQUAL 156 | &= AMPEREQUAL 157 | |= VBAREQUAL 158 | ^= CIRCUMFLEXEQUAL 159 | <<= LEFTSHIFTEQUAL 160 | >>= RIGHTSHIFTEQUAL 161 | **= DOUBLESTAREQUAL 162 | // DOUBLESLASH 163 | //= DOUBLESLASHEQUAL 164 | -> RARROW 165 | """ 166 | 167 | opmap = {} 168 | for line in opmap_raw.splitlines(): 169 | if line: 170 | op, name = line.split() 171 | opmap[op] = getattr(token, name) 172 | -------------------------------------------------------------------------------- /lib2to3/pgen2/literals.py: -------------------------------------------------------------------------------- 1 | # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | 4 | """Safely evaluate Python string literals without using eval().""" 5 | 6 | import re 7 | 8 | simple_escapes = {"a": "\a", 9 | "b": "\b", 10 | "f": "\f", 11 | "n": "\n", 12 | "r": "\r", 13 | "t": "\t", 14 | "v": "\v", 15 | "'": "'", 16 | '"': '"', 17 | "\\": "\\"} 18 | 19 | def escape(m): 20 | all, tail = m.group(0, 1) 21 | assert all.startswith("\\") 22 | esc = simple_escapes.get(tail) 23 | if esc is not None: 24 | return esc 25 | if tail.startswith("x"): 26 | hexes = tail[1:] 27 | if len(hexes) < 2: 28 | raise ValueError("invalid hex string escape ('\\%s')" % tail) 29 | try: 30 | i = int(hexes, 16) 31 | except ValueError: 32 | raise ValueError("invalid hex string escape ('\\%s')" % tail) 33 | else: 34 | try: 35 | i = int(tail, 8) 36 | except ValueError: 37 | raise ValueError("invalid octal string escape ('\\%s')" % tail) 38 | return chr(i) 39 | 40 | def evalString(s): 41 | assert s.startswith("'") or s.startswith('"'), repr(s[:1]) 42 | q = s[0] 43 | if s[:3] == q*3: 44 | q = q*3 45 | assert s.endswith(q), repr(s[-len(q):]) 46 | assert len(s) >= 2*len(q) 47 | s = s[len(q):-len(q)] 48 | return re.sub(r"\\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3})", escape, s) 49 | 50 | def test(): 51 | for i in range(256): 52 | c = chr(i) 53 | s = repr(c) 54 | e = evalString(s) 55 | if e != c: 56 | print i, c, s, e 57 | 58 | 59 | if __name__ == "__main__": 60 | test() 61 | -------------------------------------------------------------------------------- /lib2to3/pgen2/token.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | """Token constants (from "token.h").""" 4 | 5 | # Taken from Python (r53757) and modified to include some tokens 6 | # originally monkeypatched in by pgen2.tokenize 7 | 8 | #--start constants-- 9 | ENDMARKER = 0 10 | NAME = 1 11 | NUMBER = 2 12 | STRING = 3 13 | NEWLINE = 4 14 | INDENT = 5 15 | DEDENT = 6 16 | LPAR = 7 17 | RPAR = 8 18 | LSQB = 9 19 | RSQB = 10 20 | COLON = 11 21 | COMMA = 12 22 | SEMI = 13 23 | PLUS = 14 24 | MINUS = 15 25 | STAR = 16 26 | SLASH = 17 27 | VBAR = 18 28 | AMPER = 19 29 | LESS = 20 30 | GREATER = 21 31 | EQUAL = 22 32 | DOT = 23 33 | PERCENT = 24 34 | BACKQUOTE = 25 35 | LBRACE = 26 36 | RBRACE = 27 37 | EQEQUAL = 28 38 | NOTEQUAL = 29 39 | LESSEQUAL = 30 40 | GREATEREQUAL = 31 41 | TILDE = 32 42 | CIRCUMFLEX = 33 43 | LEFTSHIFT = 34 44 | RIGHTSHIFT = 35 45 | DOUBLESTAR = 36 46 | PLUSEQUAL = 37 47 | MINEQUAL = 38 48 | STAREQUAL = 39 49 | SLASHEQUAL = 40 50 | PERCENTEQUAL = 41 51 | AMPEREQUAL = 42 52 | VBAREQUAL = 43 53 | CIRCUMFLEXEQUAL = 44 54 | LEFTSHIFTEQUAL = 45 55 | RIGHTSHIFTEQUAL = 46 56 | DOUBLESTAREQUAL = 47 57 | DOUBLESLASH = 48 58 | DOUBLESLASHEQUAL = 49 59 | AT = 50 60 | OP = 51 61 | COMMENT = 52 62 | NL = 53 63 | RARROW = 54 64 | ERRORTOKEN = 55 65 | N_TOKENS = 56 66 | NT_OFFSET = 256 67 | #--end constants-- 68 | 69 | tok_name = {} 70 | for _name, _value in globals().items(): 71 | if type(_value) is type(0): 72 | tok_name[_value] = _name 73 | 74 | 75 | def ISTERMINAL(x): 76 | return x < NT_OFFSET 77 | 78 | def ISNONTERMINAL(x): 79 | return x >= NT_OFFSET 80 | 81 | def ISEOF(x): 82 | return x == ENDMARKER 83 | -------------------------------------------------------------------------------- /lib2to3/pygram.py: -------------------------------------------------------------------------------- 1 | # Copyright 2006 Google, Inc. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | 4 | """Export the Python grammar and symbols.""" 5 | 6 | # Python imports 7 | import os 8 | 9 | # Local imports 10 | from pgen2 import token 11 | from pgen2 import driver 12 | import pytree 13 | 14 | # The grammar file 15 | _GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "Grammar.txt") 16 | 17 | 18 | class Symbols(object): 19 | 20 | def __init__(self, grammar): 21 | """Initializer. 22 | 23 | Creates an attribute for each grammar symbol (nonterminal), 24 | whose value is the symbol's type (an int >= 256). 25 | """ 26 | for name, symbol in grammar.symbol2number.iteritems(): 27 | setattr(self, name, symbol) 28 | 29 | 30 | python_grammar = driver.load_grammar(_GRAMMAR_FILE) 31 | python_symbols = Symbols(python_grammar) 32 | -------------------------------------------------------------------------------- /misc/pythoscope.el: -------------------------------------------------------------------------------- 1 | ;;;;;;;;;;;;;;;;;;;;; 2 | ;;; Generate Python unit tests 3 | ;;;;;;;;;;;;;;;;;;;;; 4 | ;;; pythoscope.el --- generate Python unit tests for file 5 | ;;; 6 | ;;; This file is part of the pythoscope distribution and can be found at 7 | ;;; http://pythoscope.org 8 | ;;; 9 | ;;; This file is licensed as part of pythoscope. 10 | ;;; http://pythoscope.org/documentation#License 11 | ;;; 12 | 13 | ;;; Usage 14 | ;;; M-x pythoscope-run-on-current-file will generate unit test for the 15 | ;;; current file. 16 | 17 | 18 | ;;; Installation: 19 | 20 | ;; Put this file somewhere where Emacs can find it (i.e. in one of the 21 | ;; directories in your `load-path' such as `site-lisp'), optionally 22 | ;; byte-compile it, and put this in your .emacs: 23 | ;; 24 | ;; (require 'pythoscope) 25 | 26 | 27 | (require 'cl) 28 | 29 | (defun string-join (separator strings) 30 | (mapconcat 'identity strings separator)) 31 | 32 | (defun pluralize (word count) 33 | "Return word with a counter in singular or plural form, depending on count." 34 | (if (= count 1) 35 | (format "one %s" word) 36 | (format "%d %ss" count word))) 37 | 38 | (defvar *pythoscope-process-output* "") 39 | 40 | (defun pythoscope-generated-tests () 41 | "Based on output collected in *pythoscope-process-output* return hash 42 | mapping modules to number of tests that where added to them." 43 | (let ((tests (make-hash-table :test #'equal))) 44 | (loop for start = 0 then (match-end 0) 45 | while (string-match "Adding generated \\w+ to \\(.*?\\)\\.$" 46 | *pythoscope-process-output* start) 47 | do 48 | (let ((modname (match-string 1 *pythoscope-process-output*))) 49 | (puthash modname (1+ (gethash modname tests 0)) tests))) 50 | tests)) 51 | 52 | (defun pythoscope-tests-hash-to-descriptions (tests) 53 | (loop for modname being the hash-keys in tests 54 | using (hash-value test-count) 55 | collect (format "%s for %s" (pluralize "test" test-count) modname))) 56 | 57 | (defun pythoscope-generated-tests-summary () 58 | (let ((tests (pythoscope-generated-tests))) 59 | (if (zerop (hash-table-count tests)) 60 | "No tests were generated." 61 | (concat "Generated " 62 | (string-join ", " (pythoscope-tests-hash-to-descriptions tests)) 63 | ".")))) 64 | 65 | (defun pythoscope-process-sentinel (process event) 66 | (when (memq (process-status process) '(signal exit)) 67 | (let ((exit-status (process-exit-status process))) 68 | (if (zerop exit-status) 69 | (message (pythoscope-generated-tests-summary)) 70 | (message "pythoscope[%d] exited with code %d" 71 | (process-id process) exit-status))))) 72 | 73 | (defun pythoscope-process-filter (process output) 74 | "Save all pythoscope output to *pythoscope-process-output* for later 75 | inspection." 76 | (setq *pythoscope-process-output* 77 | (concat *pythoscope-process-output* output))) 78 | 79 | (defun pythoscope-run-on-file (filename) 80 | "Generate tests for given file using pythoscope." 81 | (interactive "f") 82 | (setq *pythoscope-process-output* "") 83 | (let ((process (start-process "pythoscope-process" 84 | (current-buffer) 85 | "pythoscope" 86 | filename))) 87 | (set-process-sentinel process 'pythoscope-process-sentinel) 88 | (set-process-filter process 'pythoscope-process-filter)) 89 | (message "Generating tests...")) 90 | 91 | (defun pythoscope-run-on-current-file () 92 | "Generate tests for file open in the current buffer." 93 | (interactive) 94 | (let ((filename (buffer-file-name))) 95 | (when filename 96 | (pythoscope-run-on-file filename)))) 97 | 98 | 99 | (provide 'pythoscope) 100 | -------------------------------------------------------------------------------- /pythoscope/__init__.py: -------------------------------------------------------------------------------- 1 | from cmdline import main, __version__ 2 | from snippet import start, stop 3 | -------------------------------------------------------------------------------- /pythoscope/_util.c: -------------------------------------------------------------------------------- 1 | /* Implementation of utility functions that couldn't be done in pure Python. */ 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | /* Python 2.3 headers don't include genobject (defined in Include/genobject.h 8 | in later versions). We only need to grab the gi_frame, so this definition 9 | will do. */ 10 | typedef struct { 11 | PyObject_HEAD 12 | PyFrameObject *gi_frame; 13 | } genobject; 14 | 15 | static PyObject * 16 | _generator_has_ended(PyObject *self, PyObject *args) 17 | { 18 | genobject *gen; 19 | PyFrameObject *frame; 20 | 21 | if (!PyArg_ParseTuple(args, "O", &gen)) 22 | return NULL; 23 | /* Check if gen is a generator done on the Python level. */ 24 | 25 | frame = gen->gi_frame; 26 | 27 | /* Python 2.5 releases gi_frame once the generator is done, so it has to be 28 | checked first. 29 | Earlier Pythons leave gi_frame intact, so the f_stacktop pointer 30 | determines whether the generator is still running. */ 31 | return PyBool_FromLong(frame == NULL || frame->f_stacktop == NULL); 32 | } 33 | 34 | static PyMethodDef UtilMethods[] = { 35 | {"_generator_has_ended", _generator_has_ended, METH_VARARGS, NULL}, 36 | {NULL, NULL, 0, NULL} 37 | }; 38 | 39 | PyMODINIT_FUNC 40 | init_util(void) 41 | { 42 | (void) Py_InitModule("_util", UtilMethods); 43 | } 44 | -------------------------------------------------------------------------------- /pythoscope/astbuilder.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module containing functions for parsing and modification of an AST. 3 | """ 4 | 5 | from pythoscope.logger import log 6 | from pythoscope.util import quoted_block 7 | from pythoscope.astvisitor import is_leaf_of_type, is_node_of_type 8 | 9 | from lib2to3 import pygram, pytree 10 | from lib2to3.pgen2 import driver, token 11 | from lib2to3.pgen2.parse import ParseError 12 | from lib2to3.pygram import python_symbols as syms 13 | from lib2to3.pytree import Node, Leaf 14 | 15 | __all__ = ["EmptyCode", "Newline", "ParseError", "clone", "create_import", 16 | "insert_after", "insert_before", "parse", "parse_fragment", "regenerate"] 17 | 18 | EmptyCode = lambda: Node(syms.file_input, []) 19 | Newline = lambda: Leaf(token.NEWLINE, "\n") 20 | 21 | def clone(tree): 22 | """Clone the tree, preserving its add_newline attribute. 23 | """ 24 | if tree is None: 25 | return None 26 | 27 | new_tree = tree.clone() 28 | if hasattr(tree, 'added_newline') and tree.added_newline: 29 | new_tree.added_newline = True 30 | return new_tree 31 | 32 | def create_import(import_desc): 33 | """Create an AST representing import statement from given description. 34 | 35 | >>> regenerate(create_import("unittest")) 36 | 'import unittest\\n' 37 | >>> regenerate(create_import(("nose", "SkipTest"))) 38 | 'from nose import SkipTest\\n' 39 | """ 40 | if isinstance(import_desc, tuple): 41 | package, name = import_desc 42 | return Node(syms.import_from, 43 | [Leaf(token.NAME, 'from'), 44 | Leaf(token.NAME, package, prefix=" "), 45 | Leaf(token.NAME, 'import', prefix=" "), 46 | Leaf(token.NAME, name, prefix=" "), 47 | Newline()]) 48 | else: 49 | return Node(syms.import_name, 50 | [Leaf(token.NAME, 'import'), 51 | Leaf(token.NAME, import_desc, prefix=" "), 52 | Newline()]) 53 | 54 | def index(node): 55 | """Return index this node is at in parent's children list. 56 | """ 57 | return node.parent.children.index(node) 58 | 59 | def insert_after(node, code): 60 | if not node.parent: 61 | raise TypeError("Can't insert after node that doesn't have a parent.") 62 | node.parent.insert_child(index(node)+1, code) 63 | 64 | def insert_before(node, code): 65 | if not node.parent: 66 | raise TypeError("Can't insert before node that doesn't have a parent.") 67 | node.parent.insert_child(index(node), code) 68 | 69 | def parse(code): 70 | """String -> AST 71 | 72 | Parse the string and return its AST representation. May raise 73 | a ParseError exception. 74 | """ 75 | added_newline = False 76 | if not code.endswith("\n"): 77 | code += "\n" 78 | added_newline = True 79 | 80 | try: 81 | drv = driver.Driver(pygram.python_grammar, pytree.convert) 82 | result = drv.parse_string(code, True) 83 | except ParseError: 84 | log.debug("Had problems parsing:\n%s\n" % quoted_block(code)) 85 | raise 86 | 87 | # Always return a Node, not a Leaf. 88 | if isinstance(result, Leaf): 89 | result = Node(syms.file_input, [result]) 90 | 91 | result.added_newline = added_newline 92 | 93 | return result 94 | 95 | def parse_fragment(code): 96 | """Works like parse() but returns an object stripped of the file_input 97 | wrapper. This eases merging this piece of code into other ones. 98 | """ 99 | parsed_code = parse(code) 100 | 101 | if is_node_of_type(parsed_code, 'file_input') and \ 102 | len(parsed_code.children) == 2 and \ 103 | is_leaf_of_type(parsed_code.children[-1], token.ENDMARKER): 104 | return parsed_code.children[0] 105 | return parsed_code 106 | 107 | def regenerate(tree): 108 | """AST -> String 109 | 110 | Regenerate the source code from the AST tree. 111 | """ 112 | if hasattr(tree, 'added_newline') and tree.added_newline: 113 | return str(tree)[:-1] 114 | else: 115 | return str(tree) 116 | -------------------------------------------------------------------------------- /pythoscope/code_trees_manager.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from pythoscope.logger import log 4 | from pythoscope.util import string2filename, load_pickle_from 5 | 6 | 7 | class CodeTreeNotFound(Exception): 8 | def __init__(self, module_subpath): 9 | Exception.__init__(self, "Couldn't find code tree for module %r." % module_subpath) 10 | self.module_subpath = module_subpath 11 | 12 | class CodeTreesManager(object): 13 | def __init__(self, code_trees_path): 14 | raise NotImplementedError 15 | 16 | # :: (CodeTree, str) -> None 17 | def remember_code_tree(self, code_tree, module_subpath): 18 | raise NotImplementedError 19 | 20 | # :: str -> CodeTree 21 | def recall_code_tree(self, module_subpath): 22 | """Return code tree corresponding to a module located under given subpath. 23 | 24 | May raise CodeTreeNotFound exception. 25 | """ 26 | raise NotImplementedError 27 | 28 | # :: str -> None 29 | def forget_code_tree(self, module_subpath): 30 | """Get rid of the CodeTree for a module located under given subpath. 31 | Do nothing if the module doesn't exist. 32 | """ 33 | raise NotImplementedError 34 | 35 | def clear_cache(self): 36 | pass 37 | 38 | class FilesystemCodeTreesManager(CodeTreesManager): 39 | """Manager of CodeTree instances that keeps at most one CodeTree instance 40 | in a memory, storing the rest in files. 41 | """ 42 | def __init__(self, code_trees_path): 43 | self.code_trees_path = code_trees_path 44 | self._cached_code_tree = None 45 | 46 | def remember_code_tree(self, code_tree, module_subpath): 47 | log.debug("Saving code tree for module %r to a file and caching..." % \ 48 | module_subpath) 49 | code_tree.save(self._code_tree_path(module_subpath)) 50 | self._cache(code_tree, module_subpath) 51 | 52 | def recall_code_tree(self, module_subpath): 53 | if self._is_cached(module_subpath): 54 | return self._cached_code_tree[1] 55 | try: 56 | log.debug("Loading code tree for module %r from a file and caching..." % \ 57 | module_subpath) 58 | code_tree = load_pickle_from(self._code_tree_path(module_subpath)) 59 | self._cache(code_tree, module_subpath) 60 | return code_tree 61 | except IOError: 62 | raise CodeTreeNotFound(module_subpath) 63 | 64 | def forget_code_tree(self, module_subpath): 65 | try: 66 | os.remove(self._code_tree_path(module_subpath)) 67 | except OSError: 68 | pass 69 | self._remove_from_cache(module_subpath) 70 | 71 | def clear_cache(self): 72 | if self._cached_code_tree: 73 | old_module_subpath, old_code_tree = self._cached_code_tree 74 | log.debug("Code tree for module %r gets out of cache, "\ 75 | "saving to a file..." % old_module_subpath) 76 | old_code_tree.save(self._code_tree_path(old_module_subpath)) 77 | self._cached_code_tree = None 78 | 79 | def _cache(self, code_tree, module_subpath): 80 | self.clear_cache() 81 | self._cached_code_tree = (module_subpath, code_tree) 82 | 83 | def _is_cached(self, module_subpath): 84 | return self._cached_code_tree and self._cached_code_tree[0] == module_subpath 85 | 86 | def _remove_from_cache(self, module_subpath): 87 | if self._is_cached(module_subpath): 88 | self._cached_code_tree = None 89 | 90 | def _code_tree_path(self, module_subpath): 91 | code_tree_filename = string2filename(module_subpath) + '.pickle' 92 | return os.path.join(self.code_trees_path, code_tree_filename) 93 | -------------------------------------------------------------------------------- /pythoscope/compat.py: -------------------------------------------------------------------------------- 1 | """Portability code for different Python versions and platforms. 2 | """ 3 | 4 | import os 5 | import warnings 6 | 7 | 8 | try: 9 | all = all 10 | except NameError: 11 | def all(iterable): 12 | for element in iterable: 13 | if not element: 14 | return False 15 | return True 16 | 17 | try: 18 | any = any 19 | except NameError: 20 | def any(iterable): 21 | for element in iterable: 22 | if element: 23 | return True 24 | return False 25 | 26 | try: 27 | set = set 28 | except NameError: 29 | from sets import Set as set 30 | 31 | try: 32 | frozenset = frozenset 33 | except NameError: 34 | from sets import ImmutableSet as frozenset 35 | 36 | try: 37 | sorted = sorted 38 | except NameError: 39 | def sorted(iterable, compare=cmp, key=None): 40 | if key: 41 | compare = lambda x,y: cmp(key(x), key(y)) 42 | alist = list(iterable) 43 | alist.sort(compare) 44 | return alist 45 | 46 | try: 47 | from itertools import groupby 48 | except ImportError: 49 | # Code taken from http://docs.python.org/lib/itertools-functions.html . 50 | class groupby(object): 51 | def __init__(self, iterable, key=None): 52 | if key is None: 53 | key = lambda x: x 54 | self.keyfunc = key 55 | self.it = iter(iterable) 56 | self.tgtkey = self.currkey = self.currvalue = xrange(0) 57 | def __iter__(self): 58 | return self 59 | def next(self): 60 | while self.currkey == self.tgtkey: 61 | self.currvalue = self.it.next() # Exit on StopIteration 62 | self.currkey = self.keyfunc(self.currvalue) 63 | self.tgtkey = self.currkey 64 | return (self.currkey, self._grouper(self.tgtkey)) 65 | def _grouper(self, tgtkey): 66 | while self.currkey == tgtkey: 67 | yield self.currvalue 68 | self.currvalue = self.it.next() # Exit on StopIteration 69 | self.currkey = self.keyfunc(self.currvalue) 70 | 71 | try: 72 | from os.path import samefile 73 | except ImportError: 74 | def samefile(file1, file2): 75 | return os.path.realpath(file1) == os.path.realpath(file2) 76 | 77 | # Ignore Python's 2.6 deprecation warning for sets module. 78 | warnings.simplefilter('ignore') 79 | import sets 80 | warnings.resetwarnings() 81 | -------------------------------------------------------------------------------- /pythoscope/debug.py: -------------------------------------------------------------------------------- 1 | from util import class_of 2 | 3 | 4 | def print_assigned_names(assigned_names): 5 | print "ASSIGNED NAMES:" 6 | for obj, name in assigned_names.iteritems(): 7 | print " %s: %s(id=%s)" % (name, class_of(obj).__name__, id(obj)) 8 | 9 | def print_timeline(timeline): 10 | print "TIMELINE:" 11 | for event in timeline: 12 | print " %5.2f: %r" % (event.timestamp, event) 13 | 14 | -------------------------------------------------------------------------------- /pythoscope/event.py: -------------------------------------------------------------------------------- 1 | class Event(object): 2 | _last_timestamp = 0 3 | 4 | def __init__(self): 5 | self.timestamp = Event.next_timestamp() 6 | 7 | def next_timestamp(cls): 8 | cls._last_timestamp += 1 9 | return cls._last_timestamp 10 | next_timestamp = classmethod(next_timestamp) 11 | 12 | def __eq__(self, other): 13 | return isinstance(other, Event) and \ 14 | self.timestamp == other.timestamp 15 | 16 | def __hash__(self): 17 | return hash(self.timestamp) 18 | -------------------------------------------------------------------------------- /pythoscope/generator/cleaner.py: -------------------------------------------------------------------------------- 1 | from pythoscope.generator.dependencies import objects_affected_by_side_effects,\ 2 | resolve_dependencies 3 | from pythoscope.generator.lines import * 4 | from pythoscope.generator.method_call_context import MethodCallContext 5 | from pythoscope.side_effect import SideEffect 6 | from pythoscope.serializer import ImmutableObject 7 | from pythoscope.util import all_of_type, compact, counted 8 | 9 | 10 | # :: [Event] -> [Event] 11 | def remove_objects_unworthy_of_naming(events): 12 | new_events = list(events) 13 | side_effects = all_of_type(events, SideEffect) 14 | affected_objects = objects_affected_by_side_effects(side_effects) 15 | invoked_objects = objects_with_method_calls(events) + objects_with_attribute_references(events) 16 | for obj, usage_count in object_usage_counts(events): 17 | # ImmutableObjects don't need to be named, as their identity is 18 | # always unambiguous. 19 | if not isinstance(obj, ImmutableObject): 20 | # Anything mentioned more than once have to be named. 21 | if usage_count > 1: 22 | continue 23 | # Anything affected by side effects is also worth naming. 24 | if obj in affected_objects: 25 | continue 26 | # All user objects with method calls should also get names for 27 | # readability. 28 | if obj in invoked_objects: 29 | continue 30 | try: 31 | while True: 32 | new_events.remove(obj) 33 | except ValueError: 34 | pass # If the element wasn't on the timeline, even better. 35 | return new_events 36 | 37 | # :: [Event] -> [SerializedObject] 38 | def objects_with_method_calls(events): 39 | def objects_from_methods(event): 40 | if isinstance(event, MethodCallContext): 41 | return event.user_object 42 | elif isinstance(event, EqualAssertionLine): 43 | return objects_from_methods(event.actual) 44 | elif isinstance(event, RaisesAssertionLine): 45 | return objects_from_methods(event.call) 46 | elif isinstance(event, GeneratorAssertionLine): 47 | return objects_from_methods(event.generator_call) 48 | else: 49 | return None 50 | return compact(map(objects_from_methods, events)) 51 | 52 | # :: [Event] -> [SerializedObject] 53 | def objects_with_attribute_references(events): 54 | def objects_from_references(event): 55 | if isinstance(event, ObjectAttributeReference): 56 | return event.obj 57 | elif isinstance(event, EqualAssertionLine): 58 | return objects_from_references(event.actual) 59 | elif isinstance(event, RaisesAssertionLine): 60 | return objects_from_references(event.call) 61 | elif isinstance(event, GeneratorAssertionLine): 62 | return objects_from_references(event.generator_call) 63 | else: 64 | return None 65 | return compact(map(objects_from_references, events)) 66 | 67 | # :: [Event] -> {SerializedObject: int} 68 | def object_usage_counts(timeline): 69 | return counted(resolve_dependencies(timeline)) 70 | -------------------------------------------------------------------------------- /pythoscope/generator/code_string.py: -------------------------------------------------------------------------------- 1 | from pythoscope.compat import any, set 2 | from pythoscope.util import union 3 | 4 | 5 | class CodeString(str): 6 | """A string that holds information on the piece of code (like a function 7 | or method call) it represents. 8 | 9 | `uncomplete` attribute denotes whether it is a complete, runnable code 10 | or just a template. 11 | 12 | `imports` is a set of imports that this piece of code requires. 13 | """ 14 | def __new__(cls, string, uncomplete=False, imports=None): 15 | if imports is None: 16 | imports = set() 17 | code_string = str.__new__(cls, string) 18 | code_string.uncomplete = uncomplete 19 | code_string.imports = imports 20 | return code_string 21 | 22 | def combine_two_code_strings(template, cs1, cs2): 23 | return CodeString(template % (cs1, cs2), 24 | cs1.uncomplete or cs2.uncomplete, 25 | union(cs1.imports, cs2.imports)) 26 | 27 | def combine_string_and_code_string(template, s, cs): 28 | return CodeString(template % (s, cs), cs.uncomplete, cs.imports) 29 | 30 | def combine_code_string_and_string(template, cs, s): 31 | return CodeString(template % (cs, s), cs.uncomplete, cs.imports) 32 | 33 | def combine(cs1, cs2, template="%s%s"): 34 | """Concatenate two CodeStrings, or a string and a CodeString, preserving 35 | information on `uncomplete` and `imports`. 36 | """ 37 | if isinstance(cs1, CodeString) and isinstance(cs2, CodeString): 38 | return combine_two_code_strings(template, cs1, cs2) 39 | elif type(cs1) is str: 40 | return combine_string_and_code_string(template, cs1, cs2) 41 | else: 42 | return combine_code_string_and_string(template, cs1, cs2) 43 | 44 | def join(char, code_strings): 45 | return CodeString(char.join(code_strings), 46 | any([cs.uncomplete for cs in code_strings]), 47 | union(*[cs.imports for cs in code_strings])) 48 | 49 | def putinto(cs, template, imports=set()): 50 | """Put the CodeString into a template, adding additional imports. 51 | """ 52 | return CodeString(template % cs, cs.uncomplete, union(cs.imports, imports)) 53 | 54 | def addimport(cs, imp): 55 | return putinto(cs, "%s", set([imp])) 56 | -------------------------------------------------------------------------------- /pythoscope/generator/dependencies.py: -------------------------------------------------------------------------------- 1 | from pythoscope.generator.lines import * 2 | from pythoscope.generator.method_call_context import MethodCallContext 3 | from pythoscope.serializer import BuiltinException, ImmutableObject, MapObject,\ 4 | UnknownObject, SequenceObject, SerializedObject, LibraryObject 5 | from pythoscope.store import FunctionCall, UserObject, MethodCall,\ 6 | GeneratorObject, GeneratorObjectInvocation, CallToC 7 | from pythoscope.side_effect import SideEffect 8 | from pythoscope.util import all_of_type, flatten 9 | 10 | 11 | # :: [SerializedObject|Call] -> [SerializedObject|Call] 12 | def sorted_by_timestamp(objects): 13 | return sorted(objects, key=lambda o: o.timestamp) 14 | 15 | # :: ([Event], int) -> [Event] 16 | def older_than(events, reference_timestamp): 17 | return filter(lambda e: e.timestamp < reference_timestamp, events) 18 | 19 | # :: ([Event], int) -> [Event] 20 | def newer_than(events, reference_timestamp): 21 | return filter(lambda e: e.timestamp > reference_timestamp, events) 22 | 23 | # :: Call -> Call 24 | def top_caller(call): 25 | if call.caller is None: 26 | return call 27 | return top_caller(call.caller) 28 | 29 | # :: (Call, int) -> [Call] 30 | def subcalls_before_timestamp(call, reference_timestamp): 31 | for c in older_than(call.subcalls, reference_timestamp): 32 | yield c 33 | for sc in subcalls_before_timestamp(c, reference_timestamp): 34 | yield sc 35 | 36 | # :: Call -> [Call] 37 | def calls_before(call): 38 | """Go up the call graph and return all calls that happened before 39 | the given one. 40 | 41 | >>> class Call(object): 42 | ... def __init__(self, caller, timestamp): 43 | ... self.subcalls = [] 44 | ... self.caller = caller 45 | ... self.timestamp = timestamp 46 | ... if caller: 47 | ... caller.subcalls.append(self) 48 | >>> top = Call(None, 1) 49 | >>> branch1 = Call(top, 2) 50 | >>> leaf1 = Call(branch1, 3) 51 | >>> branch2 = Call(top, 4) 52 | >>> leaf2 = Call(branch2, 5) 53 | >>> leaf3 = Call(branch2, 6) 54 | >>> leaf4 = Call(branch2, 7) 55 | >>> branch3 = Call(top, 8) 56 | >>> calls_before(branch3) == [top, branch1, leaf1, branch2, leaf2, leaf3, leaf4] 57 | True 58 | >>> calls_before(leaf3) == [top, branch1, leaf1, branch2, leaf2] 59 | True 60 | >>> calls_before(branch2) == [top, branch1, leaf1] 61 | True 62 | >>> calls_before(branch1) == [top] 63 | True 64 | """ 65 | top = top_caller(call) 66 | return [top] + list(subcalls_before_timestamp(top, call.timestamp)) 67 | 68 | # :: [Call] -> [SideEffect] 69 | def side_effects_of(calls): 70 | return flatten(map(lambda c: c.side_effects, calls)) 71 | 72 | # :: Call -> [SideEffect] 73 | def side_effects_before(call): 74 | return older_than(side_effects_of(calls_before(call)), call.timestamp) 75 | 76 | # :: [SideEffect] -> [SerializedObject] 77 | def objects_affected_by_side_effects(side_effects): 78 | return flatten(map(lambda se: se.affected_objects, side_effects)) 79 | 80 | # :: [Event] -> [SerializedObject] 81 | def resolve_dependencies(events): 82 | events_so_far = set() 83 | def get_those_and_contained_objects(objs): 84 | return all_of_type(objs, SerializedObject) + get_contained_objects(objs) 85 | def get_contained_objects(obj): 86 | if isinstance(obj, list): 87 | return flatten(map(get_contained_objects, obj)) 88 | if obj in events_so_far: 89 | return [] 90 | else: 91 | events_so_far.add(obj) 92 | if isinstance(obj, SequenceObject): 93 | return get_those_and_contained_objects(obj.contained_objects) 94 | elif isinstance(obj, MapObject): 95 | return get_those_and_contained_objects(flatten(obj.mapping)) 96 | elif isinstance(obj, LibraryObject): 97 | return get_those_and_contained_objects(obj.arguments) 98 | elif isinstance(obj, BuiltinException): 99 | return get_those_and_contained_objects(obj.args) 100 | elif isinstance(obj, UserObject): 101 | return get_contained_objects(obj.get_init_call() or []) 102 | elif isinstance(obj, (FunctionCall, MethodCall, GeneratorObjectInvocation)): 103 | return get_those_and_contained_objects(obj.input.values()) 104 | elif isinstance(obj, GeneratorObject): 105 | if obj.is_activated(): 106 | return get_those_and_contained_objects(obj.args.values() + obj.calls) 107 | return [] 108 | elif isinstance(obj, SideEffect): 109 | return get_those_and_contained_objects(list(obj.affected_objects)) 110 | elif isinstance(obj, MethodCallContext): 111 | return get_those_and_contained_objects([obj.call, obj.user_object]) 112 | elif isinstance(obj, EqualAssertionLine): 113 | # Actual may be just a variable name, so just skip that. 114 | if isinstance(obj.actual, str): 115 | return get_those_and_contained_objects([obj.expected]) 116 | return get_those_and_contained_objects([obj.expected, obj.actual]) 117 | elif isinstance(obj, GeneratorAssertionLine): 118 | return get_contained_objects(obj.generator_call) 119 | elif isinstance(obj, RaisesAssertionLine): 120 | return get_those_and_contained_objects([obj.call, obj.expected_exception]) 121 | elif isinstance(obj, Assign): 122 | if isinstance(obj.obj, SerializedObject): 123 | return get_those_and_contained_objects([obj.obj]) 124 | return [] 125 | elif isinstance(obj, ObjectAttributeReference): 126 | return get_those_and_contained_objects([obj.obj]) 127 | elif isinstance(obj, BindingChange): 128 | return get_those_and_contained_objects([obj.obj, obj.name]) 129 | elif isinstance(obj, (ImmutableObject, UnknownObject, CallToC, 130 | CommentLine, SkipTestLine, EqualAssertionStubLine, 131 | ModuleVariableReference)): 132 | return [] 133 | else: 134 | raise TypeError("Wrong argument to get_contained_objects: %s." % repr(obj)) 135 | return get_contained_objects(events) 136 | -------------------------------------------------------------------------------- /pythoscope/generator/lines.py: -------------------------------------------------------------------------------- 1 | from pythoscope.event import Event 2 | 3 | 4 | __all__ = ['EqualAssertionLine', 'EqualAssertionStubLine', 5 | 'GeneratorAssertionLine', 'RaisesAssertionLine', 6 | 'CommentLine', 'SkipTestLine', 7 | 'ModuleVariableReference', 'ObjectAttributeReference', 8 | 'BindingChange', 'Assign'] 9 | 10 | class Line(Event): 11 | def __init__(self, timestamp): 12 | # We don't call Event.__init__ on purpose, we set our own timestamp. 13 | self.timestamp = timestamp 14 | 15 | class EqualAssertionLine(Line): 16 | def __init__(self, expected, actual, timestamp): 17 | Line.__init__(self, timestamp) 18 | self.expected = expected 19 | self.actual = actual 20 | 21 | def __repr__(self): 22 | return "EqualAssertionLine(expected=%r, actual=%r)" % (self.expected, self.actual) 23 | 24 | class EqualAssertionStubLine(Line): 25 | def __init__(self, actual, timestamp): 26 | Line.__init__(self, timestamp) 27 | self.actual = actual 28 | 29 | class GeneratorAssertionLine(Line): 30 | def __init__(self, generator_call, timestamp): 31 | Line.__init__(self, timestamp) 32 | self.generator_call = generator_call 33 | 34 | class RaisesAssertionLine(Line): 35 | def __init__(self, expected_exception, call, timestamp): 36 | Line.__init__(self, timestamp) 37 | self.expected_exception = expected_exception 38 | self.call = call 39 | 40 | class CommentLine(Line): 41 | def __init__(self, comment, timestamp): 42 | Line.__init__(self, timestamp) 43 | self.comment = comment 44 | 45 | class SkipTestLine(Line): 46 | def __init__(self, timestamp): 47 | Line.__init__(self, timestamp) 48 | 49 | class ModuleVariableReference(Line): 50 | def __init__(self, module, name, timestamp): 51 | Line.__init__(self, timestamp) 52 | self.module = module 53 | self.name = name 54 | 55 | class ObjectAttributeReference(Line): 56 | def __init__(self, obj, name, timestamp): 57 | Line.__init__(self, timestamp) 58 | self.obj = obj 59 | self.name = name 60 | 61 | def __repr__(self): 62 | return "ObjectAttributeReference(obj=%r, name=%r)" % (self.obj, self.name) 63 | 64 | # This is a virtual line, not necessarily appearing in a test case. It is used 65 | # to notify builder of a change in name bidning that happened under-the-hood, 66 | # e.g. during a function call. 67 | class BindingChange(Line): 68 | def __init__(self, name, obj, timestamp): 69 | Line.__init__(self, timestamp) 70 | self.name = name 71 | self.obj = obj 72 | 73 | def __repr__(self): 74 | return "%s(name=%r, obj=%r)" % (self.__class__.__name__, self.name, self.obj) 75 | 76 | # Assignment statement. 77 | class Assign(BindingChange): 78 | pass 79 | -------------------------------------------------------------------------------- /pythoscope/generator/method_call_context.py: -------------------------------------------------------------------------------- 1 | class MethodCallContext(object): 2 | def __init__(self, call, user_object): 3 | self.call = call 4 | self.user_object = user_object 5 | 6 | def __getattr__(self, name): 7 | if name in ['input', 'definition', 'calls', 'args', 'output', 'timestamp']: 8 | return getattr(self.call, name) 9 | 10 | def __repr__(self): 11 | return "MethodCallContext(obj=%r, call=%r)" % (self.user_object, self.call) 12 | -------------------------------------------------------------------------------- /pythoscope/generator/objects_namer.py: -------------------------------------------------------------------------------- 1 | from pythoscope.generator.dependencies import sorted_by_timestamp 2 | from pythoscope.generator.lines import Assign 3 | from pythoscope.serializer import SerializedObject 4 | from pythoscope.util import all_of_type, key_for_value, underscore 5 | 6 | 7 | # :: SerializedObject -> str 8 | def get_name_base_for_object(obj): 9 | common_names = {'list': 'alist', 10 | 'dict': 'adict', 11 | 'array.array': 'array', 12 | 'datetime': 'dt', # we can't name it 'datetime', because that is module's name 13 | 'types.FunctionType': 'function', 14 | 'types.GeneratorType': 'generator'} 15 | return common_names.get(obj.type_name, underscore(obj.type_name)) 16 | 17 | # :: [str], str -> str 18 | def get_next_name(names, base): 19 | """Figure out a new name starting with base that doesn't appear in given 20 | list of names. 21 | 22 | >>> get_next_name(["alist", "adict1", "adict2"], "adict") 23 | 'adict3' 24 | """ 25 | base_length = len(base) 26 | def has_right_base(name): 27 | return name.startswith(base) 28 | def get_index(name): 29 | return int(name[base_length:]) 30 | return base + str(max(map(get_index, filter(has_right_base, names))) + 1) 31 | 32 | # :: SerializedObject, {SerializedObject: str}, bool -> None 33 | def assign_name_to_object(obj, assigned_names, rename=True): 34 | """Assign a right name for given object. 35 | 36 | May reassign an existing name for an object as a side effect, unless 37 | `rename` is False. 38 | """ 39 | if assigned_names.has_key(obj): 40 | return 41 | base = get_name_base_for_object(obj) 42 | other_obj = key_for_value(assigned_names, base) 43 | 44 | if other_obj: 45 | # Avoid overlapping names by numbering objects with the same base. 46 | if rename: 47 | assigned_names[other_obj] = base+"1" 48 | assigned_names[obj] = base+"2" 49 | elif base+"1" in assigned_names.values(): 50 | # We have some objects already numbered, insert a name with a new index. 51 | assigned_names[obj] = get_next_name(assigned_names.values(), base) 52 | else: 53 | # It's the first object with that base. 54 | assigned_names[obj] = base 55 | 56 | # :: ([SerializedObject], {SerializedObject: str}), bool -> None 57 | def assign_names_to_objects(objects, names, rename=True): 58 | """Modifies names dictionary as a side effect. 59 | """ 60 | for obj in sorted_by_timestamp(objects): 61 | assign_name_to_object(obj, names, rename) 62 | 63 | # :: [Event] -> [SerializedObject] 64 | def objects_only(events): 65 | return all_of_type(events, SerializedObject) 66 | 67 | # :: [Event] -> [Event] 68 | def name_objects_on_timeline(events): 69 | names = {} 70 | assign_names_to_objects(objects_only(events), names) 71 | def map_object_to_assign(event): 72 | if isinstance(event, SerializedObject): 73 | return Assign(names[event], event, event.timestamp) 74 | return event 75 | return map(map_object_to_assign, events) 76 | -------------------------------------------------------------------------------- /pythoscope/generator/optimizer.py: -------------------------------------------------------------------------------- 1 | from pythoscope.serializer import SequenceObject 2 | from pythoscope.side_effect import SideEffect, ListAppend 3 | 4 | 5 | class NonSerializingSequenceObject(SequenceObject): 6 | def __init__(self, contained_objects): 7 | super(NonSerializingSequenceObject, self).__init__([], lambda x:x) 8 | self.contained_objects = contained_objects 9 | 10 | # :: ([Event], Event, SideEffect, Event) -> [Event] 11 | def replace_pair_with_event(timeline, event1, event2, new_event): 12 | """Replaces pair of events with a single event. The second event 13 | must be a SideEffect. 14 | 15 | Optimizer only works on values with names, which means we don't really 16 | have to traverse the whole Project tree and replace all occurences 17 | of an object. It is sufficient to replace it on the dependencies 18 | timeline, which will be used as a base for naming objects and their 19 | later usage. 20 | """ 21 | if not isinstance(event2, SideEffect): 22 | raise TypeError("Second argument to replace_pair_with_object has to be a SideEffect, was %r instead." % event2) 23 | new_event.timestamp = event1.timestamp 24 | timeline[timeline.index(event1)] = new_event 25 | timeline.remove(event2) 26 | 27 | def optimize(timeline): 28 | """Shorten a chain of events, by replacing pairs with single events. 29 | 30 | For example, a creation of an empty list and appending to it a number: 31 | 32 | >>> x = [] 33 | >>> x.append(1) 34 | 35 | can be shortened to a single creation: 36 | 37 | >>> x = [1] 38 | 39 | and that's exactly what this optimizer does. 40 | """ 41 | i = 0 42 | while i+1 < len(timeline): 43 | e1 = timeline[i] 44 | e2 = timeline[i+1] 45 | # "x = [y..]" and "x.append(z)" is "x = [y..z]" 46 | if isinstance(e1, SequenceObject) and isinstance(e2, ListAppend) and e2.obj == e1: 47 | replace_pair_with_event(timeline, e1, e2, NonSerializingSequenceObject(e1.contained_objects + list(e2.args))) 48 | continue 49 | i += 1 50 | return timeline 51 | -------------------------------------------------------------------------------- /pythoscope/generator/selector.py: -------------------------------------------------------------------------------- 1 | from pythoscope.store import Class, Definition, GeneratorObject, TestClass 2 | 3 | 4 | def testable_objects(module): 5 | return [o for o in module.objects if is_testable_object(o)] 6 | 7 | def is_testable_object(obj): 8 | if isinstance(obj, TestClass): 9 | return False 10 | elif isinstance(obj, Class): 11 | ignored_superclasses = ['Exception', 'unittest.TestCase'] 12 | for klass in ignored_superclasses: 13 | if klass in obj.bases: 14 | return False 15 | return True 16 | elif isinstance(obj, Definition): 17 | return not obj.name.startswith('_') 18 | 19 | def testable_calls(calls): 20 | return [c for c in calls if is_testable_call(c)] 21 | 22 | def is_testable_call(call): 23 | if isinstance(call, GeneratorObject): 24 | return call.is_activated() and len(call.calls) > 0 25 | return True 26 | -------------------------------------------------------------------------------- /pythoscope/inspector/__init__.py: -------------------------------------------------------------------------------- 1 | from pythoscope.inspector import static, dynamic 2 | from pythoscope.inspector.file_system import python_modules_below 3 | from pythoscope.logger import log 4 | from pythoscope.store import ModuleNotFound 5 | from pythoscope.point_of_entry import PointOfEntry 6 | from pythoscope.util import generator_has_ended, last_traceback, \ 7 | last_exception_as_string 8 | 9 | 10 | def inspect_project(project): 11 | remove_deleted_modules(project) 12 | remove_deleted_points_of_entry(project) 13 | 14 | updates = inspect_project_statically(project) 15 | 16 | # If nothing new was discovered statically and there are no new points of 17 | # entry, don't run dynamic inspection. 18 | if updates: 19 | inspect_project_dynamically(project) 20 | else: 21 | log.info("No changes discovered in the source code, skipping dynamic inspection.") 22 | 23 | def remove_deleted_modules(project): 24 | subpaths = [mod.subpath for mod in project.iter_modules() if not mod.exists()] 25 | for subpath in subpaths: 26 | project.remove_module(subpath) 27 | 28 | def add_and_update_modules(project): 29 | count = 0 30 | for modpath in python_modules_below(project.path): 31 | try: 32 | module = project.find_module_by_full_path(modpath) 33 | if module.is_up_to_date(): 34 | log.debug("%s hasn't changed since last inspection, skipping." % module.subpath) 35 | continue 36 | except ModuleNotFound: 37 | pass 38 | log.info("Inspecting module %s." % project._extract_subpath(modpath)) 39 | static.inspect_module(project, modpath) 40 | count += 1 41 | return count 42 | 43 | def remove_deleted_points_of_entry(project): 44 | names = [poe.name for poe in project.points_of_entry.values() if not poe.exists()] 45 | for name in names: 46 | project.remove_point_of_entry(name) 47 | 48 | def ensure_point_of_entry(project, path): 49 | name = project._extract_point_of_entry_subpath(path) 50 | if not project.contains_point_of_entry(name): 51 | poe = PointOfEntry(project=project, name=name) 52 | project.add_point_of_entry(poe) 53 | return project.get_point_of_entry(name) 54 | 55 | def add_and_update_points_of_entry(project): 56 | count = 0 57 | for path in python_modules_below(project.get_points_of_entry_path()): 58 | poe = ensure_point_of_entry(project, path) 59 | if poe.is_out_of_sync(): 60 | count += 1 61 | return count 62 | 63 | def inspect_project_statically(project): 64 | return add_and_update_modules(project) + \ 65 | add_and_update_points_of_entry(project) 66 | 67 | def inspect_project_dynamically(project): 68 | if project.points_of_entry and hasattr(generator_has_ended, 'unreliable'): 69 | log.warning("Pure Python implementation of util.generator_has_ended is " 70 | "not reliable on Python 2.4 and lower. Please compile the " 71 | "_util module or use Python 2.5 or higher.") 72 | 73 | for poe in project.points_of_entry.values(): 74 | try: 75 | log.info("Inspecting point of entry %s." % poe.name) 76 | dynamic.inspect_point_of_entry(poe) 77 | except SyntaxError, err: 78 | log.warning("Point of entry contains a syntax error: %s" % err) 79 | except: 80 | log.warning("Point of entry exited with error: %s" % last_exception_as_string()) 81 | log.debug("Full traceback:\n" + last_traceback()) 82 | -------------------------------------------------------------------------------- /pythoscope/inspector/dynamic.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from pythoscope.side_effect import recognize_side_effect, MissingSideEffectType,\ 5 | GlobalRebind, GlobalRead, AttributeRebind 6 | from pythoscope.store import CallToC, UnknownCall 7 | from pythoscope.tracer import ICallback, Tracer 8 | from pythoscope.util import get_names 9 | 10 | 11 | class CallStack(object): 12 | def __init__(self): 13 | self.last_traceback = None 14 | self.stack = [] 15 | self.top_level_calls = [] 16 | self.top_level_side_effects = [] # TODO use this list for creating global setup & teardown methods 17 | 18 | def called(self, call): 19 | if self.stack: 20 | self.stack[-1].add_subcall(call) 21 | else: 22 | self.top_level_calls.append(call) 23 | self.stack.append(call) 24 | 25 | def returned(self, output): 26 | if self.stack: 27 | caller = self.stack.pop() 28 | caller.set_output(output) 29 | 30 | # If the last exception is reported by sys.exc_info() it means 31 | # it was handled inside the returning call. 32 | handled_traceback = sys.exc_info()[2] 33 | if handled_traceback is self.last_traceback: 34 | caller.clear_exception() 35 | 36 | # Register a side effect when applicable. 37 | if isinstance(caller, CallToC) and caller.side_effect: 38 | if not caller.raised_exception(): 39 | self.side_effect(caller.side_effect) 40 | caller.clear_side_effect() 41 | 42 | def raised(self, exception, traceback): 43 | if self.stack: 44 | caller = self.stack[-1] 45 | caller.set_exception(exception) 46 | self.last_traceback = traceback 47 | 48 | def unwind(self, value): 49 | while self.stack: 50 | self.returned(value) 51 | 52 | def assert_last_call_was_c_call(self): 53 | assert isinstance(self._last_call(), CallToC) 54 | 55 | def assert_last_call_was_python_call(self): 56 | assert not isinstance(self._last_call(), CallToC) 57 | 58 | def _last_call(self): 59 | if self.stack: 60 | return self.stack[-1] 61 | 62 | def side_effect(self, side_effect): 63 | if self.stack: 64 | self.stack[-1].add_side_effect(side_effect) 65 | else: 66 | self.top_level_side_effects.append(side_effect) 67 | 68 | # :: (Module, str) -> bool 69 | def has_defined_name(module, name): 70 | # TODO: also look at the list of imports 71 | return name in get_names(module.objects) 72 | 73 | class Inspector(ICallback): 74 | """Controller of the dynamic inspection process. It receives information 75 | from the tracer and propagates it to Execution and CallStack objects. 76 | """ 77 | def __init__(self, execution): 78 | self.execution = execution 79 | self.call_stack = CallStack() 80 | 81 | def finalize(self): 82 | # TODO: There are ways for the application to terminate (easiest 83 | # being os._exit) without unwinding the stack. This means Pythoscope 84 | # will be left with some calls registered on the stack without a return. 85 | # We remedy the situation by injecting None as the return value for 86 | # those calls. In the future we should also associate some kind of 87 | # an "exit" side effect with those calls. 88 | self.call_stack.unwind(self.execution.serialize(None)) 89 | 90 | # Copy the call graph structure to the Execution instance. 91 | self.execution.call_graph = self.call_stack.top_level_calls 92 | self.execution.finalize() 93 | 94 | def method_called(self, name, obj, args, code, frame): 95 | call = self.execution.create_method_call(name, obj, args, code, frame) 96 | return self.called(call) 97 | 98 | def function_called(self, name, args, code, frame): 99 | call = self.execution.create_function_call(name, args, code, frame) 100 | return self.called(call) 101 | 102 | def c_method_called(self, obj, klass, name, pargs): 103 | try: 104 | se_type = recognize_side_effect(klass, name) 105 | se = self.execution.create_side_effect(se_type, obj, *pargs) 106 | call = CallToC(name, se) 107 | except MissingSideEffectType: 108 | call = CallToC(name) 109 | self.call_stack.called(call) 110 | 111 | def c_function_called(self, name, pargs): 112 | self.call_stack.called(CallToC(name)) 113 | 114 | def returned(self, output): 115 | self.call_stack.assert_last_call_was_python_call() 116 | self.call_stack.returned(self.execution.serialize(output)) 117 | 118 | def c_returned(self, output): 119 | self.call_stack.assert_last_call_was_c_call() 120 | self.call_stack.returned(self.execution.serialize(output)) 121 | 122 | def raised(self, exception, traceback): 123 | self.call_stack.raised(self.execution.serialize(exception), traceback) 124 | 125 | def called(self, call): 126 | if call: 127 | self.call_stack.called(call) 128 | else: 129 | self.call_stack.called(UnknownCall()) 130 | return True 131 | 132 | def attribute_rebound(self, obj, name, value): 133 | se = AttributeRebind(self.execution.serialize(obj), name, self.execution.serialize(value)) 134 | self.call_stack.side_effect(se) 135 | 136 | def global_read(self, module_name, name, value): 137 | try: 138 | if has_defined_name(self.execution.project[module_name], name): 139 | return 140 | except: 141 | pass 142 | se = GlobalRead(module_name, name, self.execution.serialize(value)) 143 | self.call_stack.side_effect(se) 144 | 145 | def global_rebound(self, module, name, value): 146 | se = GlobalRebind(module, name, self.execution.serialize(value)) 147 | self.call_stack.side_effect(se) 148 | 149 | def inspect_point_of_entry(point_of_entry): 150 | projects_root = point_of_entry.project.path 151 | point_of_entry.clear_previous_run() 152 | 153 | # Put project's path into PYTHONPATH, so point of entry's imports work. 154 | sys.path.insert(0, projects_root) 155 | # Change current directory to the project's root, so the POE code can use 156 | # relative paths for reading project data files. 157 | old_cwd = os.getcwd() 158 | os.chdir(projects_root) 159 | 160 | try: 161 | inspect_code_in_context(point_of_entry.get_content(), 162 | point_of_entry.execution) 163 | finally: 164 | sys.path.remove(projects_root) 165 | os.chdir(old_cwd) 166 | 167 | # :: (str, Execution) -> None 168 | def inspect_code_in_context(code, execution): 169 | """Inspect given piece of code in the context of given Execution instance. 170 | 171 | May raise exceptions. 172 | """ 173 | inspector = Inspector(execution) 174 | tracer = Tracer(inspector) 175 | try: 176 | tracer.trace(code) 177 | finally: 178 | inspector.finalize() 179 | -------------------------------------------------------------------------------- /pythoscope/inspector/file_system.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from pythoscope.compat import set 4 | 5 | 6 | def python_modules_below(path): 7 | VCS_PATHS = set([".bzr", "CVS", "_darcs", ".git", ".hg", ".svn"]) 8 | def is_python_module(path): 9 | return path.endswith(".py") 10 | def not_vcs_file(path): 11 | return not set(path.split(os.path.sep)).intersection(VCS_PATHS) 12 | return filter(not_vcs_file, filter(is_python_module, rlistdir(path))) 13 | 14 | def rlistdir(path): 15 | """Resursive directory listing. Yield all files below given path, 16 | ignoring those which names begin with a dot. 17 | """ 18 | if os.path.basename(path).startswith('.'): 19 | return 20 | 21 | if os.path.isdir(path): 22 | for entry in os.listdir(path): 23 | for subpath in rlistdir(os.path.join(path, entry)): 24 | yield subpath 25 | else: 26 | yield path 27 | -------------------------------------------------------------------------------- /pythoscope/inspector/static.py: -------------------------------------------------------------------------------- 1 | import re 2 | import types 3 | 4 | from pythoscope.astvisitor import descend, ASTVisitor 5 | from pythoscope.astbuilder import parse, ParseError 6 | from pythoscope.logger import log 7 | from pythoscope.store import Class, Function, Method, TestClass,TestMethod 8 | from pythoscope.util import all_of_type, is_generator_code, \ 9 | read_file_contents, compile_without_warnings 10 | 11 | 12 | def is_test_class(name, bases): 13 | """Look at the name and bases of a class to determine whether it's a test 14 | class or not. 15 | 16 | >>> is_test_class("TestSomething", []) 17 | True 18 | >>> is_test_class("SomethingElse", []) 19 | False 20 | >>> is_test_class("ItDoesntLookLikeOne", ["unittest.TestCase"]) 21 | True 22 | """ 23 | return name.startswith("Test") or name.endswith("Test") \ 24 | or "unittest.TestCase" in bases 25 | 26 | def unindent(string): 27 | """Remove the initial part of whitespace from string. 28 | 29 | >>> unindent("1 + 2 + 3\\n") 30 | '1 + 2 + 3' 31 | >>> unindent(" def fun():\\n return 42\\n") 32 | 'def fun():\\n return 42' 33 | >>> unindent("\\n def fun():\\n return 42\\n") 34 | 'def fun():\\n return 42' 35 | >>> unindent(" def fun():\\n return 42\\n\\n") 36 | 'def fun():\\n return 42' 37 | """ 38 | string = re.sub(r'^\n*', '', string.rstrip()) # ignore leading and trailing newlines 39 | match = re.match(r'^([\t ]+)', string) 40 | if not match: 41 | return string 42 | whitespace = match.group(1) 43 | 44 | lines = [] 45 | for line in string.splitlines(True): 46 | if line.startswith(whitespace): 47 | lines.append(line[len(whitespace):]) 48 | else: 49 | return string 50 | return ''.join(lines) 51 | 52 | def function_code_from_definition(definition): 53 | """Return a code object of a given function definition. 54 | 55 | Can raise SyntaxError if the definition is not valid. 56 | """ 57 | consts = compile_without_warnings(unindent(str(definition))).co_consts 58 | return all_of_type(consts, types.CodeType)[0] 59 | 60 | def is_generator_definition(definition): 61 | """Return True if given piece of code is a generator definition. 62 | 63 | >>> is_generator_definition("def f():\\n return 1\\n") 64 | False 65 | >>> is_generator_definition("def g():\\n yield 2\\n") 66 | True 67 | >>> is_generator_definition(" def indented_gen():\\n yield 3\\n") 68 | True 69 | >>> is_generator_definition("\\n def indented_gen():\\n yield 3\\n") 70 | True 71 | """ 72 | try: 73 | return is_generator_code(function_code_from_definition(definition)) 74 | except SyntaxError: 75 | # This most likely means given code used "return" with argument 76 | # inside generator. 77 | return False 78 | 79 | def create_definition(name, args, code, definition_type): 80 | return definition_type(name, args=args, code=code, 81 | is_generator=is_generator_definition(code)) 82 | 83 | class ModuleVisitor(ASTVisitor): 84 | def __init__(self): 85 | ASTVisitor.__init__(self) 86 | self.imports = [] 87 | self.objects = [] 88 | self.main_snippet = None 89 | self.last_import = None 90 | self.past_imports = False 91 | 92 | def visit_class(self, name, bases, body): 93 | visitor = descend(body.children, ClassVisitor) 94 | if is_test_class(name, bases): 95 | methods = [TestMethod(n, c) for (n, a, c) in visitor.methods] 96 | klass = TestClass(name=name, test_cases=methods, code=body) 97 | else: 98 | methods = [create_definition(n, a, c, Method) for (n, a, c) in visitor.methods] 99 | klass = Class(name=name, methods=methods, bases=bases) 100 | self.objects.append(klass) 101 | self.past_imports = True 102 | 103 | def visit_function(self, name, args, body): 104 | self.objects.append(create_definition(name, args, body, Function)) 105 | self.past_imports = True 106 | 107 | def visit_lambda_assign(self, name, args): 108 | self.objects.append(Function(name, args=args)) 109 | self.past_imports = True 110 | 111 | def visit_import(self, names, import_from, body): 112 | if import_from: 113 | for name in names: 114 | self.imports.append((import_from, name)) 115 | else: 116 | self.imports.extend(names) 117 | if not self.past_imports: 118 | self.last_import = body 119 | 120 | def visit_main_snippet(self, body): 121 | self.main_snippet = body 122 | self.past_imports = True 123 | 124 | class ClassVisitor(ASTVisitor): 125 | def __init__(self): 126 | ASTVisitor.__init__(self) 127 | self.methods = [] 128 | 129 | def visit_class(self, name, bases, body): 130 | # Ignore definitions of subclasses. 131 | pass 132 | 133 | def visit_function(self, name, args, body): 134 | self.methods.append((name, args, body)) 135 | 136 | def inspect_module(project, path): 137 | return inspect_code(project, path, read_file_contents(path)) 138 | 139 | # :: (Project, string, string) -> Module 140 | def inspect_code(project, path, code): 141 | try: 142 | tree = parse(code) 143 | except ParseError, e: 144 | log.warning("Inspection of module %s failed." % path) 145 | return project.create_module(path, errors=[e]) 146 | visitor = descend(tree, ModuleVisitor) 147 | 148 | # We assume that all test classes in this module has dependencies on 149 | # all imports the module contains. 150 | for test_class in [o for o in visitor.objects if isinstance(o, TestClass)]: 151 | # We gathered all imports in a single list, but import lists of those 152 | # classes may diverge in time, so we don't want to share their 153 | # structure. 154 | test_class.imports = visitor.imports[:] 155 | 156 | return project.create_module(path, code=tree, objects=visitor.objects, 157 | imports=visitor.imports, main_snippet=visitor.main_snippet, 158 | last_import=visitor.last_import) 159 | -------------------------------------------------------------------------------- /pythoscope/localizable.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | 4 | from pythoscope.util import ensure_directory, get_last_modification_time, \ 5 | module_path_to_name, write_content_to_file 6 | 7 | 8 | class Localizable(object): 9 | """An object which has a corresponding file belonging to some Project. 10 | 11 | Each Localizable has a 'path' attribute and an information when it was 12 | created, to be in sync with its file system counterpart. Path is always 13 | relative to the project this localizable belongs to. 14 | """ 15 | def __init__(self, project, subpath, created=None): 16 | self.project = project 17 | self.subpath = subpath 18 | if created is None: 19 | created = time.time() 20 | self.created = created 21 | 22 | def _get_locator(self): 23 | return module_path_to_name(self.subpath, newsep=".") 24 | locator = property(_get_locator) 25 | 26 | def is_out_of_sync(self): 27 | """Is the object out of sync with its file. 28 | """ 29 | return get_last_modification_time(self.get_path()) > self.created 30 | 31 | def is_up_to_date(self): 32 | return not self.is_out_of_sync() 33 | 34 | def get_path(self): 35 | """Return the full path to the file. 36 | """ 37 | return os.path.join(self.project.path, self.subpath) 38 | 39 | def write(self, new_content): 40 | """Overwrite the file with new contents and update its created time. 41 | 42 | Creates the containing directories if needed. 43 | """ 44 | ensure_directory(os.path.dirname(self.get_path())) 45 | write_content_to_file(new_content, self.get_path()) 46 | self.created = time.time() 47 | 48 | def exists(self): 49 | return os.path.isfile(self.get_path()) 50 | -------------------------------------------------------------------------------- /pythoscope/logger.py: -------------------------------------------------------------------------------- 1 | """This module defines the logging system. 2 | 3 | To change the logging level, assign DEBUG, ERROR, INFO or WARNING to log.level. 4 | Default is INFO. 5 | 6 | To change the output stream, call the set_output() function. Default is 7 | sys.stderr. 8 | """ 9 | 10 | import logging 11 | import os.path 12 | import re 13 | 14 | from time import strftime, localtime 15 | 16 | from pythoscope.util import module_path_to_name 17 | 18 | 19 | DEBUG = logging.DEBUG 20 | ERROR = logging.ERROR 21 | INFO = logging.INFO 22 | WARNING = logging.WARNING 23 | 24 | def path2modname(path, default=""): 25 | """Take a path to a pythoscope module and return a module name in dot-style 26 | notation. Return default if path doesn't point to a pythoscope module. 27 | """ 28 | match = re.search(r'.*pythoscope%s(.*)$' % re.escape(os.path.sep), path) 29 | if match: 30 | return module_path_to_name(match.group(1), newsep=".") 31 | else: 32 | return default 33 | 34 | class LogFormatter(logging.Formatter): 35 | def format(self, record): 36 | """Show a message with a loglevel in normal verbosity mode and much more 37 | in debug mode. 38 | """ 39 | message = "%s: %s" % (record.levelname, record.getMessage()) 40 | if log.level == DEBUG: 41 | return "%s.%d %s:%d %s" % \ 42 | (strftime("%H%M%S", localtime(record.created)), 43 | record.msecs, 44 | path2modname(record.pathname, default=record.module), 45 | record.lineno, 46 | message) 47 | return message 48 | 49 | # Don't call this "setup" or nose will assume this is the fixture setup 50 | # function for this module. 51 | def setup_logger(): 52 | handler = logging.StreamHandler() 53 | handler.setFormatter(LogFormatter()) 54 | log.addHandler(handler) 55 | log.level = INFO 56 | 57 | def get_output(): 58 | return log.handlers[0].stream 59 | 60 | def set_output(stream): 61 | "Change the output of all the logging calls to go to given stream." 62 | log.handlers[0].stream = stream 63 | 64 | log = logging.getLogger('pythoscope') 65 | setup_logger() 66 | -------------------------------------------------------------------------------- /pythoscope/point_of_entry.py: -------------------------------------------------------------------------------- 1 | from pythoscope.execution import Execution 2 | from pythoscope.store import Localizable 3 | from pythoscope.util import read_file_contents 4 | 5 | 6 | class PointOfEntry(Localizable): 7 | """Piece of code provided by the user that allows dynamic analysis. 8 | 9 | Each point of entry keeps a reference to its last run in the `execution` 10 | attribute. 11 | """ 12 | def __init__(self, project, name): 13 | Localizable.__init__(self, project, project.subpath_for_point_of_entry(name)) 14 | 15 | self.project = project 16 | self.name = name 17 | self.execution = Execution(project) 18 | 19 | def _get_created(self): 20 | "Points of entry are not up-to-date until they're run." 21 | return self.execution.ended or 0 22 | def _set_created(self, value): 23 | pass 24 | created = property(_get_created, _set_created) 25 | 26 | def get_path(self): 27 | return self.project.path_for_point_of_entry(self.name) 28 | 29 | def get_content(self): 30 | return read_file_contents(self.get_path()) 31 | 32 | def clear_previous_run(self): 33 | self.execution.destroy() 34 | self.execution = Execution(self.project) 35 | -------------------------------------------------------------------------------- /pythoscope/py_wrapper_object.py: -------------------------------------------------------------------------------- 1 | """Definition of wrapperobject CPython's structure in ctypes. With this you can 2 | get into wrapperobject internals without going to the C level. 3 | 4 | See descrobject.c for reference: 5 | http://svn.python.org/view/python/trunk/Objects/descrobject.c?view=markup 6 | 7 | Note that not all fields are defined, only those that I needed. 8 | """ 9 | 10 | from ctypes import c_long, py_object, cast, Structure, POINTER 11 | 12 | 13 | ssize_t = c_long 14 | 15 | class PyWrapperObject(Structure): 16 | _fields_ = [("ob_refcnt", ssize_t), 17 | ("ob_type", py_object), 18 | ("descr", py_object), 19 | ("self", py_object)] 20 | 21 | def _wrapper_internals(wrapper): 22 | return cast(id(wrapper), POINTER(PyWrapperObject)).contents 23 | 24 | def get_wrapper_self(wrapper): 25 | return _wrapper_internals(wrapper).self 26 | -------------------------------------------------------------------------------- /pythoscope/side_effect.py: -------------------------------------------------------------------------------- 1 | from pythoscope.store import Function 2 | from pythoscope.event import Event 3 | 4 | 5 | class MissingSideEffectType(Exception): 6 | def __repr__(self): 7 | return "" % self.args 8 | 9 | known_side_effects = {} 10 | def register_side_effect_type(trigger, klass): 11 | if known_side_effects.has_key(trigger): 12 | raise ValueError("Side effect for trigger %r already registered by %r." %\ 13 | (trigger, known_side_effects[trigger])) 14 | known_side_effects[trigger] = klass 15 | 16 | def recognize_side_effect(klass, func_name): 17 | try: 18 | return known_side_effects[(klass, func_name)] 19 | except KeyError: 20 | raise MissingSideEffectType(klass, func_name) 21 | 22 | class MetaSideEffect(type): 23 | """This metaclass will register a side effect when a class is created. 24 | """ 25 | def __init__(cls, *args, **kwds): 26 | super(MetaSideEffect, cls).__init__(*args, **kwds) 27 | if hasattr(cls, 'trigger'): 28 | register_side_effect_type(cls.trigger, cls) 29 | 30 | class SideEffect(Event): 31 | __metaclass__ = MetaSideEffect 32 | def __init__(self, affected_objects, only_referenced_objects): 33 | super(SideEffect, self).__init__() 34 | self.affected_objects = affected_objects 35 | self.referenced_objects = affected_objects + only_referenced_objects 36 | 37 | class GlobalVariableSideEffect(SideEffect): 38 | def get_full_name(self): 39 | return "%s.%s" % (self.module, self.name) 40 | 41 | def __repr__(self): 42 | return "%s(%r, %r, %r)" % (self.__class__.__name__, self.module, self.name, self.value) 43 | 44 | class GlobalRead(GlobalVariableSideEffect): 45 | def __init__(self, module, name, value): 46 | super(GlobalRead, self).__init__([], []) 47 | self.module = module 48 | self.name = name 49 | self.value = value 50 | 51 | class GlobalRebind(GlobalVariableSideEffect): 52 | def __init__(self, module, name, value): 53 | super(GlobalRebind, self).__init__([], []) # TODO: module's __dict__ is affected 54 | self.module = module 55 | self.name = name 56 | self.value = value 57 | 58 | class AttributeRebind(SideEffect): 59 | def __init__(self, obj, name, value): 60 | super(AttributeRebind, self).__init__([obj], [value]) 61 | self.obj = obj 62 | self.name = name 63 | self.value = value 64 | def __repr__(self): 65 | return "%s(id=%r, %r, %s, %r)" % (self.__class__.__name__, id(self), self.obj, self.name, self.value) 66 | 67 | class BuiltinMethodWithPositionArgsSideEffect(SideEffect): 68 | definition = None # set in a subclass 69 | 70 | def __init__(self, obj, *args): 71 | super(BuiltinMethodWithPositionArgsSideEffect, self).__init__([obj], list(args)) 72 | self.obj = obj 73 | self.args = args 74 | 75 | def args_mapping(self): 76 | return dict(zip(self.definition.args, self.args)) 77 | 78 | class ListAppend(BuiltinMethodWithPositionArgsSideEffect): 79 | trigger = (list, 'append') 80 | definition = Function('append', ['object']) 81 | 82 | class ListExtend(BuiltinMethodWithPositionArgsSideEffect): 83 | trigger = (list, 'extend') 84 | definition = Function('extend', ['iterable']) 85 | 86 | class ListInsert(BuiltinMethodWithPositionArgsSideEffect): 87 | trigger = (list, 'insert') 88 | definition = Function('insert', ['index', 'object']) 89 | 90 | class ListPop(BuiltinMethodWithPositionArgsSideEffect): 91 | trigger = (list, 'pop') 92 | definition = Function('pop', ['index']) 93 | 94 | class ListRemove(BuiltinMethodWithPositionArgsSideEffect): 95 | trigger = (list, 'remove') 96 | definition = Function('remove', ['value']) 97 | 98 | class ListReverse(BuiltinMethodWithPositionArgsSideEffect): 99 | trigger = (list, 'reverse') 100 | definition = Function('reverse', []) 101 | 102 | class ListSort(BuiltinMethodWithPositionArgsSideEffect): 103 | trigger = (list, 'sort') 104 | definition = Function('sort', []) 105 | -------------------------------------------------------------------------------- /pythoscope/snippet.py: -------------------------------------------------------------------------------- 1 | """ 2 | The idea is to allow the user to place the following pair of snippets in 3 | the application code. One at the top of the first module to be imported: 4 | 5 | import pythoscope 6 | pythoscope.start() 7 | 8 | and the second before the application exists: 9 | 10 | pythoscope.stop() 11 | """ 12 | 13 | import os 14 | import sys 15 | 16 | from cmdline import find_project_directory, PythoscopeDirectoryMissing 17 | from execution import Execution 18 | from store import Project 19 | from tracer import Tracer 20 | from inspector.dynamic import Inspector 21 | 22 | 23 | project = None 24 | tracer = None 25 | inspector = None 26 | 27 | def start(): 28 | global project, tracer, inspector 29 | try: 30 | project = Project.from_directory(find_project_directory(os.getcwd())) 31 | execution = Execution(project) 32 | inspector = Inspector(execution) 33 | tracer = Tracer(inspector) 34 | tracer.btracer.setup() 35 | sys.settrace(tracer.tracer) 36 | except PythoscopeDirectoryMissing: 37 | print "Can't find .pythoscope/ directory for this project. " \ 38 | "Initialize the project with the '--init' option first. " \ 39 | "Pythoscope tracing disabled for this run." 40 | 41 | def stop(): 42 | global project, tracer, inspector 43 | if project is None or tracer is None or inspector is None: 44 | return 45 | sys.settrace(None) 46 | tracer.btracer.teardown() 47 | inspector.finalize() 48 | project.remember_execution_from_snippet(inspector.execution) 49 | project.save() 50 | project, tracer, inspector = None, None, None 51 | -------------------------------------------------------------------------------- /scripts/pythoscope: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from pythoscope import main 4 | 5 | main() 6 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [nosetests] 2 | with-doctest=1 3 | doctest-tests=1 4 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | try: 4 | from setuptools import setup 5 | # ctypes library is part of standard library since Python 2.5. 6 | if sys.version_info < (2, 5): 7 | install_requires = ['ctypes'] 8 | else: 9 | install_requires = [] 10 | args = dict( 11 | entry_points = {'console_scripts': ['pythoscope = pythoscope:main']}, 12 | install_requires = install_requires, 13 | test_suite = 'nose.collector', 14 | tests_require = ['nose', 'mock', 'docutils']) 15 | except ImportError: 16 | from distutils.core import setup 17 | args = dict(scripts = ['scripts/pythoscope']) 18 | 19 | # The C module doesn't need to be built for Python 2.5 and higher. 20 | if sys.version_info < (2, 5): 21 | from distutils.core import Extension 22 | ext_modules = [Extension('pythoscope._util', sources=['pythoscope/_util.c'])] 23 | else: 24 | ext_modules = [] 25 | 26 | 27 | from pythoscope import __version__ as VERSION 28 | 29 | setup( 30 | name='pythoscope', 31 | version=VERSION, 32 | 33 | author = 'Michal Kwiatkowski', 34 | author_email = 'constant.beta@gmail.com', 35 | description = 'unit test generator for Python', 36 | long_description = open("README.md").read() + "\n" + open("Changelog").read(), 37 | license = 'MIT', 38 | url = 'http://pythoscope.org', 39 | 40 | ext_modules = ext_modules, 41 | 42 | packages = ['pythoscope', 'pythoscope.inspector', 'pythoscope.generator', 43 | 'bytecode_tracer', 44 | 'lib2to3', 'lib2to3.pgen2'], 45 | package_data = {'pythoscope': [], 46 | 'bytecode_tracer': [], 47 | 'lib2to3': ['*.txt']}, 48 | 49 | classifiers = [ 50 | 'Development Status :: 3 - Alpha', 51 | 'Environment :: Console', 52 | 'Intended Audience :: Developers', 53 | 'Programming Language :: Python', 54 | 'Topic :: Software Development :: Code Generators', 55 | 'Topic :: Software Development :: Testing', 56 | ], 57 | 58 | **args 59 | ) 60 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- 1 | """Module containing code that has to be executed before any of the tests. 2 | """ 3 | 4 | # Make pythoscope importable directly from the test modules. 5 | import os, sys 6 | pythoscope_path = os.path.join(os.path.dirname(__file__), os.pardir) 7 | sys.path.insert(0, os.path.abspath(pythoscope_path)) 8 | 9 | # Make sys.stdout the logger's output stream, so nose capture 10 | # plugin can get hold of it. 11 | # We can't set_output to sys.stdout directly, because capture 12 | # plugin changes that before each test. 13 | class AlwaysCurrentStdout: 14 | def __getattr__(self, name): 15 | return getattr(sys.stdout, name) 16 | from pythoscope.logger import DEBUG, log, set_output 17 | set_output(AlwaysCurrentStdout()) 18 | log.level = DEBUG 19 | 20 | # Make sure all those suspiciously looking classes and functions aren't treated 21 | # as tests by nose. 22 | from pythoscope.store import TestClass, TestMethod 23 | from pythoscope.generator import add_tests_to_project, TestGenerator 24 | from pythoscope.generator.adder import add_test_case, add_test_case_to_project, \ 25 | find_test_module, module_path_to_test_path, replace_test_case 26 | from pythoscope.generator import generate_test_case 27 | from pythoscope.generator.builder import generate_test_contents 28 | from pythoscope.generator.assertions import test_timeline_for_call 29 | from pythoscope.generator.selector import testable_calls 30 | 31 | TestClass.__test__ = False 32 | TestMethod.__test__ = False 33 | 34 | add_tests_to_project.__test__ = False 35 | TestGenerator.__test__ = False 36 | 37 | add_test_case.__test__ = False 38 | add_test_case_to_project.__test__ = False 39 | find_test_module.__test__ = False 40 | module_path_to_test_path.__test__ = False 41 | replace_test_case.__test__ = False 42 | 43 | generate_test_contents.__test__ = False 44 | generate_test_case.__test__ = False 45 | test_timeline_for_call.__test__ = False 46 | testable_calls.__test__ = False 47 | -------------------------------------------------------------------------------- /test/assertions.py: -------------------------------------------------------------------------------- 1 | import difflib 2 | import re 3 | 4 | from nose.tools import assert_equal, assert_not_equal, assert_raises 5 | 6 | from pythoscope.compat import set 7 | from pythoscope.util import quoted_block 8 | 9 | __all__ = [ 10 | # Nose assertions. 11 | "assert_equal", 12 | "assert_not_equal", 13 | "assert_raises", 14 | 15 | # Our assertions. 16 | "assert_contains", 17 | "assert_contains_once", 18 | "assert_contains_one_after_another", 19 | "assert_doesnt_contain", 20 | "assert_equal_sets", 21 | "assert_equal_strings", 22 | "assert_function", 23 | "assert_instance", 24 | "assert_length", 25 | "assert_matches", 26 | "assert_not_raises", 27 | "assert_single_class", 28 | "assert_single_function", 29 | "assert_one_element_and_return"] 30 | 31 | 32 | def assert_contains(haystack, needle): 33 | assert needle in haystack,\ 34 | "Expected\n%s\nto contain %r, but it didn't." % (quoted_block(haystack), needle) 35 | 36 | def assert_contains_once(haystack, needle): 37 | repeated = len(re.findall(re.escape(needle), haystack)) 38 | assert repeated == 1, "Expected\n%s\nto contain %r once, but it contained it %d times instead." %\ 39 | (quoted_block(haystack), needle, repeated) 40 | 41 | def assert_contains_one_after_another(haystack, needle1, needle2): 42 | assert re.search(''.join([needle1, '.*', needle2]), haystack, re.DOTALL), \ 43 | "Expected\n%s\nto contain %r and then %r, but it didn't." %\ 44 | (quoted_block(haystack), needle1, needle2) 45 | 46 | def assert_doesnt_contain(haystack, needle): 47 | assert needle not in haystack,\ 48 | "Expected\n%s\nto NOT contain %r, but it did." % (quoted_block(haystack), needle) 49 | 50 | def assert_equal_sets(collection1, collection2): 51 | """Assert that both collections have the same number and set of elements. 52 | """ 53 | # Checking length of both collections first, so we catch duplicates that 54 | # appear in one collection and not the other. 55 | assert_length(collection2, len(collection1)) 56 | assert_equal(set(collection1), set(collection2)) 57 | 58 | def assert_equal_strings(s1, s2): 59 | if not isinstance(s1, str): 60 | raise AssertionError("Expected s1=%r to be a string" % s1) 61 | if not isinstance(s2, str): 62 | raise AssertionError("Expected s2=%r to be a string" % s2) 63 | assert_equal(s1, s2, "Strings not equal. Diff:\n\n%s" % ''.join(difflib.ndiff(s1.splitlines(True), s2.splitlines(True)))) 64 | 65 | def assert_function(function, name, args): 66 | assert_equal(name, function.name) 67 | assert_equal(args, function.args) 68 | 69 | def assert_instance(obj, objtype): 70 | assert isinstance(obj, objtype), \ 71 | "Expected object %r to be of type %r, it was of type %r instead." % \ 72 | (obj, objtype, type(obj)) 73 | 74 | def assert_length(collection, expected_length): 75 | actual_length = len(collection) 76 | assert expected_length == actual_length,\ 77 | "Expected collection to have %d elements, it had %d instead." %\ 78 | (expected_length, actual_length) 79 | 80 | def assert_matches(regexp, string, anywhere=False): 81 | if anywhere: 82 | match = re.search 83 | else: 84 | match = re.match 85 | assert match(regexp, string, re.DOTALL), \ 86 | "Expected\n%s\nto match r'%s', but it didn't." % (quoted_block(string), regexp) 87 | 88 | def assert_not_raises(exception, function): 89 | try: 90 | function() 91 | except exception: 92 | assert False, "Exception %s has been raised." % exception 93 | 94 | def assert_single_class(info, name): 95 | assert_length(info.classes, 1) 96 | assert_equal(name, info.classes[0].name) 97 | 98 | def assert_single_function(info, name, args=None): 99 | assert_length(info.functions, 1) 100 | assert_equal(name, info.functions[0].name) 101 | if args is not None: 102 | assert_equal(args, info.functions[0].args) 103 | 104 | def assert_one_element_and_return(collection): 105 | """Assert that the collection has exactly one element and return this 106 | element. 107 | """ 108 | assert_length(collection, 1) 109 | return collection[0] 110 | -------------------------------------------------------------------------------- /test/data/appending_test_cases_added_method_output_expected.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | class TestSomeClass(unittest.TestCase): 4 | def test___init__(self): 5 | assert True # implemented test case 6 | 7 | def test_some_method(self): 8 | assert True # implemented test case 9 | 10 | def test_new_method(self): 11 | # some_class = SomeClass() 12 | # self.assertEqual(expected, some_class.new_method()) 13 | assert False # TODO: implement your test here 14 | 15 | if __name__ == '__main__': 16 | unittest.main() 17 | -------------------------------------------------------------------------------- /test/data/appending_test_cases_module_added_method.py: -------------------------------------------------------------------------------- 1 | class SomeClass(object): 2 | def __init__(self): 3 | pass 4 | 5 | def some_method(self): 6 | pass 7 | 8 | def new_method(self): 9 | pass 10 | -------------------------------------------------------------------------------- /test/data/appending_test_cases_module_initial.py: -------------------------------------------------------------------------------- 1 | class SomeClass(object): 2 | def __init__(self): 3 | pass 4 | 5 | def some_method(self): 6 | pass 7 | -------------------------------------------------------------------------------- /test/data/appending_test_cases_module_modified.py: -------------------------------------------------------------------------------- 1 | class SomeClass(object): 2 | def __init__(self): 3 | pass 4 | 5 | def some_method(self): 6 | pass 7 | 8 | class NewClass(object): 9 | def new_method(self): 10 | pass 11 | -------------------------------------------------------------------------------- /test/data/appending_test_cases_output_expected.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | class TestSomeClass(unittest.TestCase): 4 | def test___init__(self): 5 | assert True # implemented test case 6 | 7 | def test_some_method(self): 8 | assert True # implemented test case 9 | 10 | class TestNewClass(unittest.TestCase): 11 | def test_new_method(self): 12 | # new_class = NewClass() 13 | # self.assertEqual(expected, new_class.new_method()) 14 | assert False # TODO: implement your test here 15 | 16 | if __name__ == '__main__': 17 | unittest.main() 18 | -------------------------------------------------------------------------------- /test/data/appending_test_cases_output_initial.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | class TestSomeClass(unittest.TestCase): 4 | def test___init__(self): 5 | assert True # implemented test case 6 | 7 | def test_some_method(self): 8 | assert True # implemented test case 9 | 10 | if __name__ == '__main__': 11 | unittest.main() 12 | -------------------------------------------------------------------------------- /test/data/attributes_rebind_module.py: -------------------------------------------------------------------------------- 1 | class OldStyle: 2 | def setx(self, x): 3 | self.x = x 4 | 5 | class NewStyle(object): 6 | def __init__(self, x): 7 | self.x = x 8 | def incrx(self): 9 | self.x += 1 10 | 11 | class UsingOther(object): 12 | def create(self): 13 | other = NewStyle(13) 14 | other.incrx() 15 | return other 16 | def process(self, ns): 17 | ns.incrx() 18 | 19 | class UsingOtherInternally(object): 20 | def __init__(self): 21 | self.internal = NewStyle(100) 22 | def use(self): 23 | self.internal.x += 111 24 | self.internal._y = 'private' 25 | 26 | def main(): 27 | o = OldStyle() 28 | o.setx(42) 29 | 30 | n = NewStyle(3) 31 | n.incrx() 32 | 33 | uo = UsingOther() 34 | uo.process(uo.create()) 35 | 36 | uoi = UsingOtherInternally() 37 | uoi.use() 38 | 39 | -------------------------------------------------------------------------------- /test/data/attributes_rebind_output.py: -------------------------------------------------------------------------------- 1 | from module import OldStyle 2 | import unittest 3 | from module import NewStyle 4 | from module import UsingOther 5 | from module import UsingOtherInternally 6 | from module import main 7 | 8 | 9 | class TestOldStyle(unittest.TestCase): 10 | def test_setx_returns_None_for_42(self): 11 | old_style = OldStyle() 12 | self.assertEqual(None, old_style.setx(42)) 13 | self.assertEqual(42, old_style.x) 14 | 15 | class TestNewStyle(unittest.TestCase): 16 | def test_creation_with_100(self): 17 | new_style = NewStyle(100) 18 | # Make sure it doesn't raise any exceptions. 19 | 20 | def test_incrx_2_times_after_creation_with_13(self): 21 | new_style = NewStyle(13) 22 | self.assertEqual(None, new_style.incrx()) 23 | self.assertEqual(14, new_style.x) 24 | self.assertEqual(None, new_style.incrx()) 25 | self.assertEqual(15, new_style.x) 26 | 27 | def test_incrx_returns_None_after_creation_with_3(self): 28 | new_style = NewStyle(3) 29 | self.assertEqual(None, new_style.incrx()) 30 | self.assertEqual(4, new_style.x) 31 | 32 | class TestUsingOther(unittest.TestCase): 33 | def test_create_and_process(self): 34 | using_other = UsingOther() 35 | result = using_other.create() 36 | new_style = NewStyle(13) 37 | result.x = 13 38 | result.x = 14 39 | self.assertEqual(new_style, result) 40 | self.assertEqual(None, using_other.process(result)) 41 | 42 | class TestUsingOtherInternally(unittest.TestCase): 43 | def test_use_returns_None(self): 44 | using_other_internally = UsingOtherInternally() 45 | self.assertEqual(None, using_other_internally.use()) 46 | self.assertEqual(211, using_other_internally.internal.x) 47 | self.assertEqual('private', using_other_internally.internal._y) 48 | 49 | class TestMain(unittest.TestCase): 50 | def test_main_returns_None(self): 51 | self.assertEqual(None, main()) 52 | 53 | if __name__ == '__main__': 54 | unittest.main() 55 | -------------------------------------------------------------------------------- /test/data/generic_acceptance_poe.py: -------------------------------------------------------------------------------- 1 | from module import main 2 | main() 3 | -------------------------------------------------------------------------------- /test/data/global_variables_module.py: -------------------------------------------------------------------------------- 1 | var = 1 2 | 3 | def main(): 4 | global var 5 | old = var 6 | var = 2 7 | return old 8 | -------------------------------------------------------------------------------- /test/data/global_variables_output.py: -------------------------------------------------------------------------------- 1 | from module import main 2 | import unittest 3 | import module 4 | 5 | 6 | class TestMain(unittest.TestCase): 7 | def test_main_returns_1(self): 8 | old_module_var = module.var 9 | module.var = 1 10 | self.assertEqual(module.var, main()) 11 | self.assertEqual(2, module.var) 12 | module.var = old_module_var 13 | 14 | if __name__ == '__main__': 15 | unittest.main() 16 | -------------------------------------------------------------------------------- /test/data/objects_identity_module.py: -------------------------------------------------------------------------------- 1 | class Facade(object): 2 | def __init__(self, system): 3 | self.system = system 4 | 5 | def just_do_it(self): 6 | self.system.do_this() 7 | self.system.do_that() 8 | 9 | class System(object): 10 | def __init__(self, composite): 11 | self.composite = composite 12 | 13 | def do_this(self): 14 | self.composite.this() 15 | 16 | def do_that(self): 17 | self.composite.that() 18 | 19 | class Composite(object): 20 | def __init__(self, objects): 21 | self.objects = objects 22 | 23 | def this(self): 24 | for obj in self.objects: 25 | obj.this() 26 | 27 | def that(self): 28 | for obj in self.objects: 29 | obj.that() 30 | 31 | class Object(object): 32 | def __init__(self, x): 33 | self.x = x 34 | 35 | def this(self): 36 | pass 37 | 38 | def that(self): 39 | pass 40 | 41 | def do_something_simple_with_system(system): 42 | facade = Facade(system) 43 | facade.just_do_it() 44 | 45 | def main(): 46 | objects = [] 47 | for key in ["one", "two", "three"]: 48 | objects.append(Object(key)) 49 | 50 | composite = Composite(objects) 51 | system = System(composite) 52 | 53 | do_something_simple_with_system(system) 54 | 55 | if __name__ == '__main__': 56 | main() 57 | -------------------------------------------------------------------------------- /test/data/objects_identity_output.py: -------------------------------------------------------------------------------- 1 | from module import Facade 2 | import unittest 3 | from module import Object 4 | from module import Composite 5 | from module import System 6 | from module import do_something_simple_with_system 7 | from module import main 8 | 9 | 10 | class TestFacade(unittest.TestCase): 11 | def test_just_do_it_returns_None_after_creation_with_system_instance(self): 12 | alist = [Object('one'), Object('two'), Object('three')] 13 | composite = Composite(alist) 14 | system = System(composite) 15 | facade = Facade(system) 16 | self.assertEqual(None, facade.just_do_it()) 17 | 18 | class TestSystem(unittest.TestCase): 19 | def test_do_that_and_do_this_after_creation_with_composite_instance(self): 20 | alist = [Object('one'), Object('two'), Object('three')] 21 | composite = Composite(alist) 22 | system = System(composite) 23 | self.assertEqual(None, system.do_this()) 24 | self.assertEqual(None, system.do_that()) 25 | 26 | class TestComposite(unittest.TestCase): 27 | def test_that_and_this_after_creation_with_list(self): 28 | alist = [Object('one'), Object('two'), Object('three')] 29 | composite = Composite(alist) 30 | self.assertEqual(None, composite.this()) 31 | self.assertEqual(None, composite.that()) 32 | 33 | class TestObject(unittest.TestCase): 34 | def test_that_and_this_after_creation_with_one(self): 35 | object = Object('one') 36 | self.assertEqual(None, object.this()) 37 | self.assertEqual(None, object.that()) 38 | 39 | def test_that_and_this_after_creation_with_three(self): 40 | object = Object('three') 41 | self.assertEqual(None, object.this()) 42 | self.assertEqual(None, object.that()) 43 | 44 | def test_that_and_this_after_creation_with_two(self): 45 | object = Object('two') 46 | self.assertEqual(None, object.this()) 47 | self.assertEqual(None, object.that()) 48 | 49 | class TestDoSomethingSimpleWithSystem(unittest.TestCase): 50 | def test_do_something_simple_with_system_returns_None_for_system_instance(self): 51 | alist = [Object('one'), Object('two'), Object('three')] 52 | composite = Composite(alist) 53 | self.assertEqual(None, do_something_simple_with_system(System(composite))) 54 | 55 | class TestMain(unittest.TestCase): 56 | def test_main_returns_None(self): 57 | self.assertEqual(None, main()) 58 | 59 | if __name__ == '__main__': 60 | unittest.main() 61 | -------------------------------------------------------------------------------- /test/data/side_effects_on_lists_module.py: -------------------------------------------------------------------------------- 1 | def before(): 2 | alist = [] 3 | alist.append(1) 4 | alist.extend([3, 2]) 5 | alist.insert(0, 4) 6 | alist.pop() 7 | alist.remove(3) 8 | alist.sort() 9 | return alist 10 | 11 | def after(alist): 12 | alist.reverse() 13 | return alist 14 | 15 | def main(): 16 | after(before()) 17 | -------------------------------------------------------------------------------- /test/data/side_effects_on_lists_output.py: -------------------------------------------------------------------------------- 1 | from module import before 2 | import unittest 3 | from module import after 4 | from module import main 5 | 6 | 7 | class TestBefore(unittest.TestCase): 8 | def test_before_returns_list(self): 9 | alist = [1] 10 | alist.extend([3, 2]) 11 | alist.insert(0, 4) 12 | alist.pop() 13 | alist.remove(3) 14 | alist.sort() 15 | self.assertEqual(alist, before()) 16 | 17 | class TestAfter(unittest.TestCase): 18 | def test_after_returns_alist_for_alist_equal_list(self): 19 | alist1 = [] 20 | alist2 = [] 21 | alist1.append(1) 22 | alist2.append(1) 23 | alist1.extend([3, 2]) 24 | alist2.extend([3, 2]) 25 | alist1.insert(0, 4) 26 | alist2.insert(0, 4) 27 | alist1.pop() 28 | alist2.pop() 29 | alist1.remove(3) 30 | alist2.remove(3) 31 | alist1.sort() 32 | alist2.sort() 33 | alist2.reverse() 34 | self.assertEqual(alist1, after(alist1)) 35 | self.assertEqual(alist2, alist1) 36 | 37 | class TestMain(unittest.TestCase): 38 | def test_main_returns_None(self): 39 | self.assertEqual(None, main()) 40 | 41 | if __name__ == '__main__': 42 | unittest.main() 43 | -------------------------------------------------------------------------------- /test/data/static_analysis_module.py: -------------------------------------------------------------------------------- 1 | class SimpleClass(object): 2 | def simple_method(self): 3 | pass 4 | 5 | def method_with_one_arg(self, argument): 6 | pass 7 | 8 | class ClassWithInit(object): 9 | def __init__(self): 10 | self.attr = 42 11 | 12 | def method(self, arg): 13 | self.attr += arg 14 | 15 | class OldStyleClass: 16 | def m(self): 17 | pass 18 | 19 | class EmptyClass(object): 20 | class_attr = 13 21 | 22 | class SubclassOfEmpty(EmptyClass): 23 | def new_method(self): 24 | pass 25 | 26 | def stand_alone_function(arg1, arg2): 27 | def inner_function(arg): 28 | pass 29 | class InnerClass(object): 30 | pass 31 | pass 32 | 33 | class TopLevelClass(object): 34 | class AnotherInnerClass(object): 35 | pass 36 | def method(self): 37 | pass 38 | -------------------------------------------------------------------------------- /test/data/static_analysis_output.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | class TestSimpleClass(unittest.TestCase): 5 | def test_method_with_one_arg(self): 6 | # simple_class = SimpleClass() 7 | # self.assertEqual(expected, simple_class.method_with_one_arg(argument)) 8 | assert False # TODO: implement your test here 9 | 10 | def test_simple_method(self): 11 | # simple_class = SimpleClass() 12 | # self.assertEqual(expected, simple_class.simple_method()) 13 | assert False # TODO: implement your test here 14 | 15 | class TestClassWithInit(unittest.TestCase): 16 | def test___init__(self): 17 | # class_with_init = ClassWithInit() 18 | assert False # TODO: implement your test here 19 | 20 | def test_method(self): 21 | # class_with_init = ClassWithInit() 22 | # self.assertEqual(expected, class_with_init.method(arg)) 23 | assert False # TODO: implement your test here 24 | 25 | class TestOldStyleClass(unittest.TestCase): 26 | def test_m(self): 27 | # old_style_class = OldStyleClass() 28 | # self.assertEqual(expected, old_style_class.m()) 29 | assert False # TODO: implement your test here 30 | 31 | class TestSubclassOfEmpty(unittest.TestCase): 32 | def test_new_method(self): 33 | # subclass_of_empty = SubclassOfEmpty() 34 | # self.assertEqual(expected, subclass_of_empty.new_method()) 35 | assert False # TODO: implement your test here 36 | 37 | class TestStandAloneFunction(unittest.TestCase): 38 | def test_stand_alone_function(self): 39 | # self.assertEqual(expected, stand_alone_function(arg1, arg2)) 40 | assert False # TODO: implement your test here 41 | 42 | class TestTopLevelClass(unittest.TestCase): 43 | def test_method(self): 44 | # top_level_class = TopLevelClass() 45 | # self.assertEqual(expected, top_level_class.method()) 46 | assert False # TODO: implement your test here 47 | 48 | if __name__ == '__main__': 49 | unittest.main() 50 | -------------------------------------------------------------------------------- /test/factories.py: -------------------------------------------------------------------------------- 1 | """An easy way to create domain objects with required parameters, which aren't 2 | always important during testing. 3 | 4 | This loosely follows Creation Method pattern (see 5 | for details). 6 | 7 | First, register a factory for some domain object. 8 | >>> class Struct: 9 | ... def __init__(self, name): 10 | ... self.name = name 11 | >>> register_factory(Struct, name="nice_structure") #doctest: +ELLIPSIS 12 | 13 | 14 | Now, use it in tests: 15 | >>> struct = create(Struct) 16 | >>> struct.name 17 | 'nice_structure' 18 | 19 | You can also overload defaults if you want: 20 | >>> struct = create(Struct, name="my_struct") 21 | >>> struct.name 22 | 'my_struct' 23 | 24 | Sometimes you want to generate an attribute each time the object is created. 25 | In that cases use register_dynamic_factory: 26 | >>> class Tree: 27 | ... def __init__(self, leaves): 28 | ... self.leaves = leaves 29 | >>> register_dynamic_factory(Tree, lambda:dict(leaves=[])) #doctest: +ELLIPSIS 30 | 31 | >>> klass1 = create(Tree) 32 | >>> klass2 = create(Tree) 33 | >>> klass1.leaves is not klass2.leaves 34 | True 35 | """ 36 | 37 | FACTORIES = {} 38 | 39 | def create(klass, **kwds): 40 | return FACTORIES[klass].invoke(klass, kwds) 41 | 42 | def register_factory(klass, **kwds): 43 | factory = Factory(kwds.copy) 44 | FACTORIES[klass] = factory 45 | return factory 46 | 47 | def register_dynamic_factory(klass, function): 48 | factory = Factory(function) 49 | FACTORIES[klass] = factory 50 | return factory 51 | 52 | class Factory(object): 53 | def __init__(self, callback): 54 | self.args_callback = callback 55 | self.after_callback = None 56 | 57 | def after(self, callback): 58 | self.after_callback = callback 59 | return self 60 | 61 | def invoke(self, klass, kwargs): 62 | args = self.args_callback() 63 | args.update(kwargs) 64 | obj = klass(**args) 65 | if self.after_callback: 66 | self.after_callback(obj) 67 | return obj 68 | 69 | ######################################################################## 70 | ## A few handy factories for Pythoscope. 71 | ## 72 | from pythoscope.astbuilder import parse 73 | from pythoscope.serializer import UnknownObject, ImmutableObject, SequenceObject 74 | from pythoscope.store import Function, FunctionCall, Definition, TestClass,\ 75 | TestMethod, Module, Project 76 | 77 | register_factory(Project, 78 | path="/tmp/") 79 | register_factory(Module, 80 | project=create(Project), subpath="module") 81 | register_factory(Definition, 82 | name="definition") 83 | register_factory(Function, 84 | name="function", module=create(Module)) 85 | register_factory(UnknownObject, 86 | obj=None) 87 | register_factory(ImmutableObject, 88 | obj=1) 89 | register_factory(SequenceObject, 90 | obj=[], serialize=lambda x: create(UnknownObject, obj=x)) 91 | register_dynamic_factory(FunctionCall, 92 | lambda:dict(definition=create(Function), args={}, output=create(ImmutableObject))).\ 93 | after(lambda fc: fc.definition.add_call(fc)) 94 | register_dynamic_factory(TestMethod, 95 | lambda:dict(name="test_method", code=parse("# a test method"))) 96 | register_dynamic_factory(TestClass, 97 | lambda:dict(name="TestClass", code=parse("# a test class"))) 98 | -------------------------------------------------------------------------------- /test/generator_helper.py: -------------------------------------------------------------------------------- 1 | from pythoscope.store import FunctionCall 2 | 3 | from factories import create 4 | 5 | 6 | def put_on_timeline(*objects): 7 | timestamp = 1 8 | for obj in objects: 9 | obj.timestamp = timestamp 10 | timestamp += 1 11 | 12 | def create_parent_call_with_side_effects(call, side_effects): 13 | parent_call = create(FunctionCall) 14 | parent_call.add_subcall(call) 15 | map(parent_call.add_side_effect, side_effects) 16 | -------------------------------------------------------------------------------- /test/inspector_assertions.py: -------------------------------------------------------------------------------- 1 | from pythoscope.serializer import MapObject, UnknownObject, SequenceObject,\ 2 | BuiltinException 3 | 4 | from assertions import assert_equal 5 | from helper import EmptyProjectExecution 6 | 7 | 8 | __all__ = ["assert_serialized", "assert_collection_of_serialized", 9 | "assert_call_arguments", "serialize_value"] 10 | 11 | 12 | def assert_serialized(expected_unserialized, actual_serialized): 13 | assert_equal_serialized(serialize_value(expected_unserialized), actual_serialized) 14 | def assert_collection_of_serialized(expected_collection, actual_collection): 15 | assert_equal_serialized(serialize_collection(expected_collection), actual_collection) 16 | def assert_call_arguments(expected_args, actual_args): 17 | assert_equal_serialized(serialize_arguments(expected_args), actual_args) 18 | 19 | def serialize_value(value): 20 | return EmptyProjectExecution().serialize(value) 21 | def serialize_collection(collection): 22 | return map(serialize_value, collection) 23 | def serialize_arguments(args): 24 | return EmptyProjectExecution().serialize_call_arguments(args) 25 | 26 | def assert_equal_serialized(obj1, obj2): 27 | """Equal assertion that ignores UnknownObjects, SequenceObjects and 28 | MapObjects identity. For testing purposes only. 29 | """ 30 | def unknown_object_eq(o1, o2): 31 | if not isinstance(o2, UnknownObject): 32 | return False 33 | return o1.partial_reconstructor == o2.partial_reconstructor 34 | def sequence_object_eq(o1, o2): 35 | if not isinstance(o2, SequenceObject): 36 | return False 37 | return o1.constructor_format == o2.constructor_format \ 38 | and o1.contained_objects == o2.contained_objects 39 | def map_object_eq(o1, o2): 40 | if not isinstance(o2, MapObject): 41 | return False 42 | return o1.mapping == o2.mapping 43 | def builtin_exception_eq(o1, o2): 44 | if not isinstance(o2, BuiltinException): 45 | return False 46 | return o1.args == o2.args 47 | try: 48 | UnknownObject.__eq__ = unknown_object_eq 49 | SequenceObject.__eq__ = sequence_object_eq 50 | MapObject.__eq__ = map_object_eq 51 | BuiltinException.__eq__ = builtin_exception_eq 52 | assert_equal(obj1, obj2) 53 | finally: 54 | del UnknownObject.__eq__ 55 | del SequenceObject.__eq__ 56 | del MapObject.__eq__ 57 | del BuiltinException.__eq__ 58 | -------------------------------------------------------------------------------- /test/inspector_helper.py: -------------------------------------------------------------------------------- 1 | from pythoscope.inspector.dynamic import inspect_code_in_context 2 | from pythoscope.execution import Execution 3 | from pythoscope.store import Class, Function, Method, Project 4 | from pythoscope.util import last_exception_as_string, last_traceback 5 | 6 | from assertions import * 7 | 8 | __all__ = ["inspect_returning_callables", "inspect_returning_execution", 9 | "inspect_returning_callables_and_execution", 10 | "inspect_returning_single_callable", "inspect_returning_single_call"] 11 | 12 | 13 | class ClassMock(Class): 14 | """Class that has all the methods you try to find inside it via 15 | find_method_by_name(). 16 | """ 17 | def __init__(self, name): 18 | Class.__init__(self, name) 19 | self._methods = {} 20 | 21 | def find_method_by_name(self, name): 22 | if not self._methods.has_key(name): 23 | self._methods[name] = Method(name) 24 | return self._methods[name] 25 | 26 | class ProjectMock(Project): 27 | """Project that has all the classes, functions and generators you try to 28 | find inside it via find_object(). 29 | """ 30 | ignored_modules = ["__builtin__", "exceptions"] 31 | 32 | def __init__(self, ignored_functions=[]): 33 | self.ignored_functions = ignored_functions 34 | self.path = "." 35 | self._classes = {} 36 | self._functions = {} 37 | 38 | def find_object(self, type, name, modulepath): 39 | if modulepath in self.ignored_modules: 40 | return None 41 | if type is Function and name in self.ignored_functions: 42 | return None 43 | 44 | object_id = (name, modulepath) 45 | container = self._get_container_for(type) 46 | 47 | if not container.has_key(object_id): 48 | container[object_id] = self._create_object(type, name) 49 | return container[object_id] 50 | 51 | def iter_callables(self): 52 | for klass in self._classes.values(): 53 | for user_object in klass.user_objects: 54 | yield user_object 55 | for function in self._functions.values(): 56 | yield function 57 | 58 | def get_callables(self): 59 | return list(self.iter_callables()) 60 | 61 | def _get_container_for(self, type): 62 | if type is Class: 63 | return self._classes 64 | elif type is Function: 65 | return self._functions 66 | else: 67 | raise TypeError("Cannot store %r inside a module." % type) 68 | 69 | def _create_object(self, type, name): 70 | if type is Class: 71 | return ClassMock(name) 72 | else: 73 | return type(name) 74 | 75 | def inspect_returning_callables_and_execution(fun, ignored_functions=None): 76 | project = ProjectMock(ignored_functions or []) 77 | execution = Execution(project=project) 78 | 79 | try: 80 | inspect_code_in_context(fun, execution) 81 | # Don't allow any POEs exceptions to propagate to the testing code. 82 | # Catch both string and normal exceptions. 83 | except: 84 | print "Caught exception inside point of entry:", last_exception_as_string() 85 | print last_traceback() 86 | 87 | return project.get_callables(), execution 88 | 89 | def inspect_returning_callables(fun, ignored_functions=None): 90 | return inspect_returning_callables_and_execution(fun, ignored_functions)[0] 91 | 92 | def inspect_returning_execution(fun): 93 | return inspect_returning_callables_and_execution(fun, None)[1] 94 | 95 | def inspect_returning_single_callable(fun): 96 | callables = inspect_returning_callables(fun) 97 | return assert_one_element_and_return(callables) 98 | 99 | def inspect_returning_single_call(fun): 100 | callables = inspect_returning_callables(fun) 101 | callable = assert_one_element_and_return(callables) 102 | return assert_one_element_and_return(callable.calls) 103 | 104 | -------------------------------------------------------------------------------- /test/test_acceptance.py: -------------------------------------------------------------------------------- 1 | from pythoscope.inspector import inspect_project 2 | from pythoscope.generator import add_tests_to_project 3 | from pythoscope.util import read_file_contents, write_content_to_file 4 | 5 | from nose import SkipTest 6 | 7 | from assertions import * 8 | from helper import get_test_module_contents, CapturedLogger, \ 9 | ProjectInDirectory, putfile, TempDirectory, read_data 10 | 11 | 12 | class TestStaticAnalysis(CapturedLogger, TempDirectory): 13 | def test_generates_test_stubs(self): 14 | expected_result = read_data("static_analysis_output.py") 15 | project = ProjectInDirectory(self.tmpdir) 16 | module_path = putfile(project.path, "module.py", read_data("static_analysis_module.py")) 17 | 18 | inspect_project(project) 19 | add_tests_to_project(project, [module_path], 'unittest') 20 | result = get_test_module_contents(project) 21 | 22 | assert_equal_strings(expected_result, result) 23 | 24 | class TestAppendingTestClasses(CapturedLogger, TempDirectory): 25 | def test_appends_test_classes_to_existing_test_modules(self): 26 | self._test_appending("appending_test_cases_module_modified.py", 27 | "appending_test_cases_output_expected.py") 28 | 29 | def test_appends_test_methods_to_existing_test_classes(self): 30 | self._test_appending("appending_test_cases_module_added_method.py", 31 | "appending_test_cases_added_method_output_expected.py") 32 | 33 | def _test_appending(self, modified_input, expected_output): 34 | project = ProjectInDirectory(self.tmpdir) 35 | 36 | module_path = putfile(project.path, "module.py", read_data("appending_test_cases_module_initial.py")) 37 | test_module_path = putfile(project.path, "test_module.py", read_data("appending_test_cases_output_initial.py")) 38 | 39 | # Analyze the project with an existing test module. 40 | inspect_project(project) 41 | 42 | # Filesystem stat has resolution of 1 second, and we don't want to 43 | # sleep in a test, so we just fake the original files creation time. 44 | project["module"].created = 0 45 | project["test_module"].created = 0 46 | 47 | # Modify the application module and analyze it again. 48 | putfile(project.path, "module.py", read_data(modified_input)) 49 | inspect_project(project) 50 | 51 | # Regenerate the tests. 52 | add_tests_to_project(project, [module_path], 'unittest') 53 | project.save() 54 | 55 | assert_length(project.get_modules(), 2) 56 | result = read_file_contents(test_module_path) 57 | expected_result = read_data(expected_output) 58 | assert_equal_strings(expected_result, result) 59 | 60 | class TestAcceptanceWithPointOfEntry(CapturedLogger, TempDirectory): 61 | def execute_with_point_of_entry_and_assert(self, id): 62 | expected_result = read_data("%s_output.py" % id) 63 | project = ProjectInDirectory(self.tmpdir).with_points_of_entry(["poe.py"]) 64 | module_path = putfile(project.path, "module.py", read_data("%s_module.py" % id)) 65 | write_content_to_file(read_data("generic_acceptance_poe.py"), project.path_for_point_of_entry("poe.py")) 66 | 67 | inspect_project(project) 68 | add_tests_to_project(project, [module_path], 'unittest') 69 | result = get_test_module_contents(project) 70 | 71 | assert_equal_strings(expected_result, result) 72 | 73 | class TestObjectsIdentityPreservation(TestAcceptanceWithPointOfEntry): 74 | def test_preserves_identity_of_objects(self): 75 | self.execute_with_point_of_entry_and_assert("objects_identity") 76 | 77 | class TestSideEffectsCaptureAndGeneration(TestAcceptanceWithPointOfEntry): 78 | def test_captures_and_generates_tests_for_code_with_side_effects_on_lists(self): 79 | self.execute_with_point_of_entry_and_assert("side_effects_on_lists") 80 | 81 | class TestGlobalVariables(TestAcceptanceWithPointOfEntry): 82 | def test_handles_global_variables(self): 83 | self.execute_with_point_of_entry_and_assert("global_variables") 84 | 85 | class TestAttributesRebind(TestAcceptanceWithPointOfEntry): 86 | def test_handles_attribute_rebind(self): 87 | self.execute_with_point_of_entry_and_assert("attributes_rebind") 88 | -------------------------------------------------------------------------------- /test/test_astbuilder.py: -------------------------------------------------------------------------------- 1 | from pythoscope.astbuilder import parse, regenerate 2 | 3 | from assertions import * 4 | 5 | 6 | class TestParser: 7 | def test_handles_inputs_without_newline(self): 8 | tree = parse("42 # answer") 9 | assert_equal("42 # answer", regenerate(tree)) 10 | 11 | -------------------------------------------------------------------------------- /test/test_astvisitor.py: -------------------------------------------------------------------------------- 1 | from pythoscope.astvisitor import ASTVisitor 2 | from pythoscope.astbuilder import parse, regenerate 3 | 4 | from assertions import * 5 | 6 | 7 | class TestASTVisitorImports: 8 | def test_handles_simple_imports(self): 9 | code = "import unittest" 10 | def assertions(names, import_from): 11 | assert_equal(["unittest"], names) 12 | assert_equal(None, import_from) 13 | 14 | self._test_import(code, assertions) 15 | 16 | def test_handles_multiple_imports(self): 17 | code = "import unittest, nose" 18 | def assertions(names, import_from): 19 | assert_equal(["unittest", "nose"], names) 20 | assert_equal(None, import_from) 21 | 22 | self._test_import(code, assertions) 23 | 24 | def test_handles_deep_imports(self): 25 | code = "import abc.xyz.FBR" 26 | def assertions(names, import_from): 27 | assert_equal(["abc.xyz.FBR"], names) 28 | assert_equal(None, import_from) 29 | 30 | self._test_import(code, assertions) 31 | 32 | def test_handles_multiple_deep_imports(self): 33 | code = "import abc.xyz, abc.zyx" 34 | def assertions(names, import_from): 35 | assert_equal(["abc.xyz", "abc.zyx"], names) 36 | assert_equal(None, import_from) 37 | 38 | self._test_import(code, assertions) 39 | 40 | def test_handles_from_imports(self): 41 | code = "from nose import SkipTest" 42 | def assertions(names, import_from): 43 | assert_equal(["SkipTest"], names) 44 | assert_equal("nose", import_from) 45 | 46 | self._test_import(code, assertions) 47 | 48 | def test_handles_multiple_from_imports(self): 49 | code = "from nose import SkipTest, DeprecatedTest" 50 | def assertions(names, import_from): 51 | assert_equal(["SkipTest", "DeprecatedTest"], names) 52 | assert_equal("nose", import_from) 53 | 54 | self._test_import(code, assertions) 55 | 56 | def test_handles_deep_from_imports(self): 57 | code = "from nose.tools import assert_equal" 58 | def assertions(names, import_from): 59 | assert_equal(["assert_equal"], names) 60 | assert_equal("nose.tools", import_from) 61 | 62 | self._test_import(code, assertions) 63 | 64 | def test_handles_imports_with_as(self): 65 | code = "import unittest as test" 66 | def assertions(names, import_from): 67 | assert_equal([("unittest", "test")], names) 68 | assert_equal(None, import_from) 69 | 70 | self._test_import(code, assertions) 71 | 72 | def test_handles_multiple_imports_with_as(self): 73 | code = "import X as Y, A as B" 74 | def assertions(names, import_from): 75 | assert_equal([("X", "Y"), ("A", "B")], names) 76 | assert_equal(None, import_from) 77 | 78 | self._test_import(code, assertions) 79 | 80 | def _test_import(self, code, method): 81 | method_called = [False] 82 | class TestVisitor(ASTVisitor): 83 | def visit_import(self, names, import_from, body): 84 | method(names, import_from) 85 | method_called[0] = True 86 | 87 | TestVisitor().visit(parse(code)) 88 | assert method_called[0], "visit_import wasn't called at all" 89 | 90 | class TestASTVisitorMainSnippet: 91 | def test_detects_the_main_snippet(self): 92 | code = "import unittest\n\nif __name__ == '__main__':\n unittest.main()\n" 93 | def assertions(body): 94 | assert_equal("\nif __name__ == '__main__':\n unittest.main()\n", body) 95 | 96 | self._test_main_snippet(code, assertions) 97 | 98 | def test_detects_main_snippet_with_different_quotes(self): 99 | code = 'import unittest\n\nif __name__ == "__main__":\n unittest.main()\n' 100 | def assertions(body): 101 | assert_equal('\nif __name__ == "__main__":\n unittest.main()\n', body) 102 | 103 | self._test_main_snippet(code, assertions) 104 | 105 | def _test_main_snippet(self, code, method): 106 | method_called = [False] 107 | class TestVisitor(ASTVisitor): 108 | def visit_main_snippet(self, body): 109 | method(regenerate(body)) 110 | method_called[0] = True 111 | 112 | TestVisitor().visit(parse(code)) 113 | assert method_called[0], "visit_main_snippet wasn't called at all" 114 | -------------------------------------------------------------------------------- /test/test_code_trees_manager.py: -------------------------------------------------------------------------------- 1 | import gc 2 | import os.path 3 | 4 | from mock import Mock 5 | 6 | from pythoscope.code_trees_manager import CodeTreeNotFound, \ 7 | FilesystemCodeTreesManager 8 | from pythoscope.store import CodeTree, Module 9 | 10 | from assertions import * 11 | from helper import TempDirectory 12 | 13 | 14 | class TestFilesystemCodeTreesManager(TempDirectory): 15 | def setUp(self): 16 | super(TestFilesystemCodeTreesManager, self).setUp() 17 | self.manager = FilesystemCodeTreesManager(self.tmpdir) 18 | 19 | def assert_empty_cache(self): 20 | assert_equal(None, self.manager._cached_code_tree) 21 | 22 | def assert_cache(self, module_subpath): 23 | assert_equal(module_subpath, self.manager._cached_code_tree[0]) 24 | 25 | def assert_recalled_tree(self, module_subpath, code): 26 | assert_equal(code, self.manager.recall_code_tree(module_subpath).code) 27 | 28 | def assert_code_tree_saved(self, module_subpath, saved=True): 29 | path = self.manager._code_tree_path(module_subpath) 30 | assert_equal(saved, os.path.exists(path)) 31 | 32 | def assert_code_tree_not_saved(self, module_subpath): 33 | self.assert_code_tree_saved(module_subpath, saved=False) 34 | 35 | def assert_calls_once(self, mock, callback): 36 | """Assert that given callback calls given Mock object exactly once. 37 | """ 38 | before_count = mock.call_count 39 | callback() 40 | assert_equal(before_count + 1, mock.call_count) 41 | 42 | def test_remembered_code_trees_can_be_recalled(self): 43 | code_tree = CodeTree(None) 44 | self.manager.remember_code_tree(code_tree, "module.py") 45 | 46 | assert_equal(code_tree, self.manager.recall_code_tree("module.py")) 47 | 48 | def test_remembered_and_forgotten_code_trees_cannot_be_recalled(self): 49 | code_tree = CodeTree(None) 50 | self.manager.remember_code_tree(code_tree, "module.py") 51 | self.manager.forget_code_tree("module.py") 52 | 53 | assert_raises(CodeTreeNotFound, lambda: self.manager.recall_code_tree("module.py")) 54 | 55 | def test_cache_is_empty_right_after_initialization(self): 56 | self.assert_empty_cache() 57 | 58 | def test_cache_is_empty_after_clearing(self): 59 | code_tree = CodeTree(None) 60 | self.manager.remember_code_tree(code_tree, "module.py") 61 | self.manager.clear_cache() 62 | 63 | self.assert_empty_cache() 64 | 65 | def test_cache_contains_the_last_recalled_or_remembered_code_tree(self): 66 | # We use numbers to identify CodeTrees. We cannot use their id, because 67 | # pickling doesn't preserve those. 68 | cts = map(CodeTree, [0, 1, 2]) 69 | for i, ct in enumerate(cts): 70 | self.manager.remember_code_tree(ct, "module%d.py" % i) 71 | 72 | # Checking all combinations of recall/remember calls. 73 | self.assert_recalled_tree("module0.py", 0) 74 | self.assert_cache("module0.py") 75 | self.assert_recalled_tree("module1.py", 1) 76 | self.assert_cache("module1.py") 77 | self.manager.remember_code_tree(CodeTree(3), "module3.py") 78 | self.assert_cache("module3.py") 79 | self.manager.remember_code_tree(CodeTree(4), "module4.py") 80 | self.assert_cache("module4.py") 81 | self.assert_recalled_tree("module2.py", 2) 82 | self.assert_cache("module2.py") 83 | 84 | def test_remembering_code_tree_saves_it_to_the_filesystem(self): 85 | code_tree = CodeTree(None) 86 | self.manager.remember_code_tree(code_tree, "module.py") 87 | self.assert_code_tree_saved("module.py") 88 | 89 | def test_forgetting_code_tree_removes_its_file_from_the_filesystem(self): 90 | code_tree = CodeTree(None) 91 | self.manager.remember_code_tree(code_tree, "module.py") 92 | 93 | self.manager.forget_code_tree("module.py") 94 | self.assert_code_tree_not_saved("module.py") 95 | 96 | def test_when_clearing_cache_code_tree_currently_in_cache_is_saved_to_the_filesystem(self): 97 | code_tree = CodeTree(None) 98 | code_tree.save = Mock() 99 | self.manager.remember_code_tree(code_tree, "module.py") 100 | self.assert_cache("module.py") 101 | 102 | self.assert_calls_once(code_tree.save, self.manager.clear_cache) 103 | 104 | def test_code_tree_not_in_cache_can_be_garbage_collected(self): 105 | code_tree = CodeTree(None) 106 | self.manager.remember_code_tree(code_tree, "module.py") 107 | # Referred from the test and from the CodeTreesManager. 108 | assert_length(gc.get_referrers(code_tree), 2) 109 | 110 | self.manager.clear_cache() 111 | 112 | # No longer referred from the CodeTreesManager. 113 | assert_length(gc.get_referrers(code_tree), 1) 114 | -------------------------------------------------------------------------------- /test/test_documentation.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from nose import SkipTest 4 | 5 | from pythoscope.util import read_file_contents 6 | 7 | 8 | def test_documentation_syntax(): 9 | # May not be present in all distributions (added to stdlib in Python 2.5). 10 | try: 11 | import docutils.parsers.rst 12 | import docutils.utils 13 | except ImportError: 14 | raise SkipTest 15 | 16 | def test(doc): 17 | path = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, doc)) 18 | 19 | parser = docutils.parsers.rst.Parser() 20 | contents = read_file_contents(path) 21 | 22 | document = docutils.utils.new_document(path) 23 | document.settings.tab_width = 4 24 | document.settings.pep_references = 1 25 | document.settings.rfc_references = 1 26 | 27 | # Will raise exception on a mere warning from the parser. 28 | document.reporter.halt_level = 0 29 | 30 | parser.parse(contents, document) 31 | 32 | for doc in ["README", "Changelog", "doc/FAQ", "doc/basic-tutorial.txt"]: 33 | yield test, doc 34 | -------------------------------------------------------------------------------- /test/test_inspector.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from nose import SkipTest 4 | 5 | from pythoscope.inspector import inspect_project 6 | from pythoscope.util import generator_has_ended 7 | 8 | from assertions import * 9 | from helper import CapturedLogger, CapturedDebugLogger, P, ProjectInDirectory,\ 10 | TempDirectory 11 | 12 | 13 | class TestInspector(CapturedLogger, TempDirectory): 14 | def test_skips_dynamic_inspection_when_no_changes_were_made_to_the_project(self): 15 | project = ProjectInDirectory(self.tmpdir) 16 | inspect_project(project) 17 | assert_equal_strings("INFO: No changes discovered in the source code, skipping dynamic inspection.\n", 18 | self._get_log_output()) 19 | 20 | def test_reports_each_inspected_module(self): 21 | paths = ["module.py", "something_else.py", P("module/in/directory.py")] 22 | project = ProjectInDirectory(self.tmpdir).with_modules(paths) 23 | # Force the inspection by faking files creation time. 24 | project["module"].created = 0 25 | project["something_else"].created = 0 26 | project["module.in.directory"].created = 0 27 | 28 | inspect_project(project) 29 | 30 | for path in paths: 31 | assert_contains_once(self._get_log_output(), 32 | "INFO: Inspecting module %s." % path) 33 | 34 | def test_reports_each_inspected_point_of_entry(self): 35 | paths = ["one.py", "two.py"] 36 | project = ProjectInDirectory(self.tmpdir).with_points_of_entry(paths) 37 | 38 | inspect_project(project) 39 | 40 | for path in paths: 41 | assert_contains_once(self._get_log_output(), 42 | "INFO: Inspecting point of entry %s." % path) 43 | 44 | def test_warns_about_unreliable_implementation_of_util_generator_has_ended(self): 45 | if not hasattr(generator_has_ended, 'unreliable'): 46 | raise SkipTest 47 | 48 | paths = ["edgar.py", "allan.py"] 49 | project = ProjectInDirectory(self.tmpdir).with_points_of_entry(paths) 50 | 51 | inspect_project(project) 52 | 53 | assert_contains_once(self._get_log_output(), 54 | "WARNING: Pure Python implementation of " 55 | "util.generator_has_ended is not reliable on " 56 | "Python 2.4 and lower. Please compile the _util " 57 | "module or use Python 2.5 or higher.") 58 | 59 | def test_catches_exceptions_raised_by_entry_points(self): 60 | project = ProjectInDirectory(self.tmpdir).with_point_of_entry("exc.py", "raise Exception") 61 | inspect_project(project) 62 | if sys.version_info < (2, 5): 63 | assert_contains_once(self._get_log_output(), 64 | "WARNING: Point of entry exited with error: (2, 6, 4): # Message changed a bit from 2.6.4 to 2.6.5 76 | assert_contains_once(self._get_log_output(), 77 | "WARNING: Point of entry exited with error: " 78 | "TypeError('exceptions must be old-style classes or derived from BaseException, not str',)") 79 | else: 80 | assert_contains_once(self._get_log_output(), 81 | "WARNING: Point of entry exited with error: " 82 | "TypeError('exceptions must be classes or instances, not str',)") 83 | 84 | class TestInspectorWithDebugOutput(CapturedDebugLogger, TempDirectory): 85 | def test_skips_inspection_of_up_to_date_modules(self): 86 | paths = ["module.py", "something_else.py", P("module/in/directory.py")] 87 | project = ProjectInDirectory(self.tmpdir).with_modules(paths) 88 | 89 | inspect_project(project) 90 | 91 | for path in paths: 92 | assert_contains_once(self._get_log_output(), 93 | "DEBUG: %s hasn't changed since last inspection, skipping." % path) 94 | -------------------------------------------------------------------------------- /test/test_logger.py: -------------------------------------------------------------------------------- 1 | from pythoscope.logger import log, path2modname 2 | 3 | from assertions import * 4 | from helper import CapturedLogger, CapturedDebugLogger, P 5 | 6 | 7 | class TestLogger(CapturedLogger): 8 | def test_info_message_in_normal_mode(self): 9 | log.info("Log this") 10 | assert_equal_strings("INFO: Log this\n", self.captured.getvalue()) 11 | 12 | class TestDebugLogger(CapturedDebugLogger): 13 | def test_info_message_in_debug_mode(self): 14 | log.info("Log that") 15 | assert_matches(r"\d+\.\d+ .*test_logger:\d+ INFO: Log that\n", 16 | self._get_log_output()) 17 | 18 | class TestPath2Modname: 19 | def test_path2modname(self): 20 | assert_equal('astvisitor', path2modname(P("sth/pythoscope/astvisitor.py"))) 21 | assert_equal('generator', path2modname(P("sth/pythoscope/generator/__init__.py"))) 22 | assert_equal('generator.adder', path2modname(P("sth/pythoscope/generator/adder.py"))) 23 | -------------------------------------------------------------------------------- /test/test_point_of_entry.py: -------------------------------------------------------------------------------- 1 | from pythoscope.astbuilder import EmptyCode 2 | from pythoscope.point_of_entry import PointOfEntry 3 | from pythoscope.serializer import ImmutableObject, UnknownObject,\ 4 | SequenceObject, MapObject 5 | from pythoscope.store import Class, Function, FunctionCall, GeneratorObject,\ 6 | Method, UserObject 7 | 8 | from assertions import * 9 | from helper import EmptyProject 10 | 11 | 12 | def inject_user_object(poe, obj, klass): 13 | def create_user_object(_): 14 | return UserObject(obj, klass) 15 | user_object = poe.execution._retrieve_or_capture(obj, create_user_object) 16 | klass.add_user_object(user_object) 17 | return user_object 18 | 19 | def inject_function_call(poe, function, args={}): 20 | call = FunctionCall(function, args) 21 | poe.execution.captured_calls.append(call) 22 | for arg in args.values(): 23 | poe.execution.captured_objects[id(arg)] = arg 24 | function.add_call(call) 25 | return call 26 | 27 | def inject_generator_object(poe, obj, *args): 28 | return poe.execution._retrieve_or_capture(obj, 29 | lambda _:GeneratorObject(obj, *args)) 30 | 31 | class TestPointOfEntry: 32 | def _create_project_with_two_points_of_entry(self, *objs): 33 | project = EmptyProject() 34 | project.create_module("module.py", code=EmptyCode(), objects=list(objs)) 35 | self.first = PointOfEntry(project, 'first') 36 | self.second = PointOfEntry(project, 'second') 37 | 38 | def test_clear_previous_run_removes_user_objects_from_classes(self): 39 | klass = Class('SomeClass') 40 | self._create_project_with_two_points_of_entry(klass) 41 | 42 | obj1 = inject_user_object(self.first, 1, klass) 43 | obj2 = inject_user_object(self.first, 2, klass) 44 | obj3 = inject_user_object(self.second, 1, klass) 45 | 46 | self.first.clear_previous_run() 47 | 48 | # Only the UserObject from the second POE remains. 49 | assert_equal_sets([obj3], klass.user_objects) 50 | 51 | def test_clear_previous_run_removes_function_calls_from_functions(self): 52 | function = Function('some_function') 53 | self._create_project_with_two_points_of_entry(function) 54 | 55 | call1 = inject_function_call(self.first, function) 56 | call2 = inject_function_call(self.first, function) 57 | call3 = inject_function_call(self.second, function) 58 | 59 | self.first.clear_previous_run() 60 | 61 | # Only the FunctionCall from the second POE remains. 62 | assert_equal_sets([call3], function.calls) 63 | 64 | def test_clear_previous_run_removes_generator_objects_from_functions(self): 65 | function = Function('generator', is_generator=True) 66 | method = Method('generator_method', is_generator=True) 67 | klass = Class('ClassWithGenerators', methods=[method]) 68 | self._create_project_with_two_points_of_entry(function, klass) 69 | 70 | user_object = inject_user_object(self.first, 1, klass) 71 | inject_generator_object(self.first, 2, function, {}, function) 72 | inject_generator_object(self.first, 3, method, {}, user_object) 73 | 74 | self.first.clear_previous_run() 75 | 76 | assert_equal([], klass.user_objects) 77 | assert_equal([], function.calls) 78 | 79 | def test_clear_previous_run_ignores_not_referenced_objects(self): 80 | function = Function('some_function') 81 | self._create_project_with_two_points_of_entry(function) 82 | 83 | args = {'i': ImmutableObject(123), 'u': UnknownObject(None), 84 | 's': SequenceObject([], None), 'm': MapObject({}, None)} 85 | inject_function_call(self.first, function, args) 86 | 87 | self.first.clear_previous_run() 88 | # Make sure it doesn't raise any exceptions. 89 | -------------------------------------------------------------------------------- /test/test_project.py: -------------------------------------------------------------------------------- 1 | import os 2 | from cPickle import PicklingError 3 | 4 | from pythoscope.store import Project, Class, Function, Method, TestClass, \ 5 | TestMethod, ModuleNotFound 6 | from pythoscope.inspector import remove_deleted_modules 7 | from pythoscope.generator.adder import add_test_case 8 | from pythoscope.util import get_names, read_file_contents 9 | 10 | from assertions import * 11 | from factories import create 12 | from helper import EmptyProject, P, ProjectInDirectory, ProjectWithModules, \ 13 | UNPICKABLE_OBJECT, TempDirectory, putdir 14 | 15 | 16 | class TestProject: 17 | def test_can_be_queried_for_modules_by_their_path(self): 18 | paths = ["module.py", P("sub/dir/module.py"), P("package/__init__.py")] 19 | project = ProjectWithModules(paths) 20 | 21 | for path in paths: 22 | assert_equal(path, project[path].subpath) 23 | 24 | def test_raises_module_not_found_exception_when_no_module_like_that_is_present(self): 25 | project = EmptyProject() 26 | assert_raises(ModuleNotFound, lambda: project["whatever"]) 27 | 28 | def test_can_be_queried_for_modules_by_their_locator(self): 29 | paths = ["module.py", P("sub/dir/module.py"), P("package/__init__.py")] 30 | locators = ["module", "sub.dir.module", "package"] 31 | project = ProjectWithModules(paths) 32 | 33 | for path, locator in zip(paths, locators): 34 | assert_equal(path, project[locator].subpath) 35 | 36 | def test_replaces_old_module_objects_with_new_ones_during_create_module(self): 37 | paths = ["module.py", P("sub/dir/module.py"), P("other/module.py")] 38 | project = ProjectWithModules(paths) 39 | 40 | new_module = project.create_module(P("other/module.py")) 41 | 42 | assert_length(project.get_modules(), 3) 43 | assert project[P("other/module.py")] is new_module 44 | 45 | def test_replaces_module_instance_in_test_cases_associated_modules_during_module_replacement(self): 46 | paths = ["module.py", P("sub/dir/module.py"), P("other/module.py")] 47 | project = ProjectWithModules(paths) 48 | test_class = create(TestClass, name='TestAnything', associated_modules=[project[P("other/module.py")]]) 49 | add_test_case(project[P("other/module.py")], test_class) 50 | 51 | new_module = project.create_module(P("other/module.py")) 52 | 53 | assert_length(test_class.associated_modules, 1) 54 | assert test_class.associated_modules[0] is new_module 55 | 56 | 57 | class TestProjectOnTheFilesystem(TempDirectory): 58 | def test_can_be_saved_and_restored_from_file(self): 59 | project = ProjectInDirectory(self.tmpdir).with_modules(["good_module.py", "bad_module.py"]) 60 | project['good_module'].add_objects([Class("AClass", [Method("amethod")]), 61 | Function("afunction")]) 62 | project['bad_module'].errors = ["Syntax error"] 63 | project.save() 64 | 65 | project = Project.from_directory(project.path) 66 | 67 | assert_equal(2, len(project.get_modules())) 68 | assert_equal(2, len(project['good_module'].objects)) 69 | assert_equal(["AClass"], get_names(project['good_module'].classes)) 70 | assert_equal(["amethod"], get_names(project['good_module'].classes[0].methods)) 71 | assert_equal(["afunction"], get_names(project['good_module'].functions)) 72 | assert_equal(["Syntax error"], project['bad_module'].errors) 73 | 74 | def _test_finds_new_test_directory(self, test_module_dir): 75 | putdir(self.tmpdir, ".pythoscope") 76 | putdir(self.tmpdir, test_module_dir) 77 | project = Project(self.tmpdir) 78 | assert_equal(test_module_dir, project.new_tests_directory) 79 | 80 | def test_finds_new_tests_directory(self): 81 | test_module_dirs = ["test", "functional_test", "unit_test", 82 | "tests", "functional_tests", "unit_tests", 83 | "pythoscope-tests", "unit-tests"] 84 | for test_module_dir in test_module_dirs: 85 | yield '_test_finds_new_test_directory', test_module_dir 86 | 87 | def test_removes_definitions_of_modules_that_dont_exist_anymore(self): 88 | project = ProjectInDirectory(self.tmpdir).with_modules(["module.py", "other_module.py", "test_module.py"]) 89 | test_class = create(TestClass, associated_modules=[project["module"]]) 90 | add_test_case(project["test_module.py"], test_class) 91 | project.save() 92 | 93 | os.remove(os.path.join(project.path, "other_module.py")) 94 | 95 | remove_deleted_modules(project) 96 | 97 | assert_not_raises(ModuleNotFound, lambda: project["module"]) 98 | assert_raises(ModuleNotFound, lambda: project["other_module"]) 99 | assert_not_raises(ModuleNotFound, lambda: project["test_module"]) 100 | 101 | def test_doesnt_save_uncomplete_pickle_files(self): 102 | project = ProjectInDirectory(self.tmpdir) 103 | project.save() 104 | original_pickle = read_file_contents(project._get_pickle_path()) 105 | 106 | # Inject unpickable object into project. 107 | project._injected_attr = UNPICKABLE_OBJECT 108 | try: 109 | project.save() 110 | except PicklingError: 111 | pass 112 | 113 | # Make sure that the original file wasn't overwritten. 114 | assert_equal_strings(original_pickle, 115 | read_file_contents(project._get_pickle_path())) 116 | -------------------------------------------------------------------------------- /test/test_serializer.py: -------------------------------------------------------------------------------- 1 | from pythoscope.serializer import get_partial_reconstructor 2 | 3 | from assertions import * 4 | 5 | 6 | class TestGetPartialReconstructor: 7 | def test_uses_name_of_the_class_for_instances_of_new_style_classes(self): 8 | class SomeClass(object): 9 | pass 10 | assert_equal("test.test_serializer.SomeClass", 11 | get_partial_reconstructor(SomeClass())) 12 | 13 | def test_uses_name_of_the_class_for_instances_of_old_style_classes(self): 14 | class SomeClass: 15 | pass 16 | assert_equal("test.test_serializer.SomeClass", 17 | get_partial_reconstructor(SomeClass())) 18 | -------------------------------------------------------------------------------- /test/test_store.py: -------------------------------------------------------------------------------- 1 | from pythoscope.astbuilder import parse 2 | from pythoscope.code_trees_manager import CodeTreeNotFound 3 | from pythoscope.store import Class, Function, Method, Module, CodeTree,\ 4 | TestClass, TestMethod, code_of, module_of 5 | from pythoscope.generator.adder import add_test_case 6 | 7 | from assertions import * 8 | from helper import CustomSeparator, EmptyProject 9 | 10 | 11 | class TestModule: 12 | def setUp(self): 13 | self.project = EmptyProject() 14 | self.module = self.project.create_module("module.py", code=parse("# only comments")) 15 | self.test_class = TestClass(name="TestSomething", code=parse("# some test code")) 16 | 17 | def test_can_add_test_cases_to_empty_modules(self): 18 | add_test_case(self.module, self.test_class) 19 | # Make sure it doesn't raise any exceptions. 20 | 21 | def test_adding_a_test_case_adds_it_to_list_of_objects(self): 22 | add_test_case(self.module, self.test_class) 23 | 24 | assert_equal([self.test_class], self.module.objects) 25 | 26 | def test_test_cases_can_be_added_using_add_objects_method(self): 27 | test_class_1 = TestClass(name="TestSomethingElse") 28 | test_class_2 = TestClass(name="TestSomethingCompletelyDifferent") 29 | self.module.add_objects([test_class_1, test_class_2]) 30 | 31 | assert_equal([test_class_1, test_class_2], self.module.objects) 32 | assert_equal([test_class_1, test_class_2], self.module.test_cases) 33 | 34 | def test_module_with_errors_doesnt_get_a_code_tree(self): 35 | module = self.project.create_module("module_with_errors.py", errors=[Exception()]) 36 | assert_raises(CodeTreeNotFound, lambda: CodeTree.of(module)) 37 | 38 | class TestStoreWithCustomSeparator(CustomSeparator): 39 | def test_uses_system_specific_path_separator(self): 40 | module = Module(subpath="some#path.py", project=EmptyProject()) 41 | assert_equal("some.path", module.locator) 42 | 43 | class TestModuleOf: 44 | def setUp(self): 45 | project = EmptyProject() 46 | self.module = Module(project=project, subpath='module.py') 47 | self.klass = Class('Klass', module=self.module) 48 | self.tclass = TestClass('TClass', parent=self.module) 49 | 50 | def test_module_of_for_module(self): 51 | assert_equal(self.module, module_of(self.module)) 52 | 53 | def test_module_of_for_function(self): 54 | fun = Function('fun', module=self.module) 55 | assert_equal(self.module, module_of(fun)) 56 | 57 | def test_module_of_for_class(self): 58 | assert_equal(self.module, module_of(self.klass)) 59 | 60 | def test_module_of_for_method(self): 61 | meth = Method('meth', klass=self.klass) 62 | assert_equal(self.module, module_of(meth)) 63 | 64 | def test_module_of_for_test_classes(self): 65 | assert_equal(self.module, module_of(self.tclass)) 66 | 67 | def test_module_of_for_test_methods(self): 68 | tmeth = TestMethod('tmeth', parent=self.tclass) 69 | assert_equal(self.module, module_of(tmeth)) 70 | 71 | class TestCodeOf: 72 | def setUp(self): 73 | project = EmptyProject() 74 | self.code = object() # A unique fake object. 75 | self.module = Module(project=project, subpath='module.py') 76 | self.code_tree = CodeTree(self.code) 77 | project.remember_code_tree(self.code_tree, self.module) 78 | 79 | def test_code_of_module(self): 80 | assert_equal(self.code, code_of(self.module)) 81 | 82 | def test_code_of_function(self): 83 | function = Function('fun', module=self.module) 84 | function_code = object() 85 | self.code_tree.add_object(function, function_code) 86 | 87 | assert_equal(function_code, code_of(function)) 88 | 89 | def test_code_of_class(self): 90 | klass = Class('Class', module=self.module) 91 | class_code = object() 92 | self.code_tree.add_object(klass, class_code) 93 | 94 | assert_equal(class_code, code_of(klass)) 95 | 96 | def test_code_of_method(self): 97 | klass = Class('Class', module=self.module) 98 | method = Method('method', klass=klass) 99 | method_code = object() 100 | self.code_tree.add_object(method, method_code) 101 | 102 | assert_equal(method_code, code_of(method)) 103 | 104 | def test_code_of_test_class(self): 105 | test_class = TestClass('TestClass', parent=self.module) 106 | test_class_code = object() 107 | self.code_tree.add_object(test_class, test_class_code) 108 | 109 | assert_equal(test_class_code, code_of(test_class)) 110 | 111 | def test_code_of_test_method(self): 112 | test_class = TestClass('TestClass', parent=self.module) 113 | test_method = TestMethod('test_method', parent=test_class) 114 | test_method_code = object() 115 | self.code_tree.add_object(test_method, test_method_code) 116 | 117 | assert_equal(test_method_code, code_of(test_method)) 118 | 119 | class TestCodeTree: 120 | def test_instance_is_accesible_from_the_moment_it_is_created(self): 121 | project = EmptyProject() 122 | mod = Module(project=project, subpath='module.py') 123 | ct = CodeTree(None) 124 | project.remember_code_tree(ct, mod) 125 | 126 | assert_equal(ct, CodeTree.of(mod)) 127 | 128 | def test_removal_of_a_module_removes_its_code_tree(self): 129 | project = EmptyProject() 130 | mod = project.create_module('module.py') 131 | ct = CodeTree(None) 132 | project.remember_code_tree(ct, mod) 133 | 134 | project.remove_module(mod.subpath) 135 | 136 | assert_raises(CodeTreeNotFound, lambda: CodeTree.of(mod)) 137 | -------------------------------------------------------------------------------- /test/test_tracing_side_effects.py: -------------------------------------------------------------------------------- 1 | from pythoscope.side_effect import ListAppend, ListExtend, ListInsert, ListPop,\ 2 | GlobalRebind, GlobalRead 3 | 4 | from assertions import * 5 | from inspector_assertions import * 6 | from inspector_helper import * 7 | 8 | 9 | def function_doing_to_list(action, *args, **kwds): 10 | alist = kwds.get('alist', []) 11 | def fun(): 12 | def foo(x): 13 | getattr(x, action)(*args) 14 | foo(alist) 15 | return fun 16 | 17 | def assert_builtin_method_side_effects(se, klass, obj, *args): 18 | assert_instance(se, klass) 19 | assert_serialized(obj, se.obj) 20 | assert_collection_of_serialized(list(args), list(se.args)) 21 | 22 | class TestMutation: 23 | def test_handles_list_append(self): 24 | fun = function_doing_to_list('append', 1) 25 | call = inspect_returning_single_call(fun) 26 | se = assert_one_element_and_return(call.side_effects) 27 | assert_builtin_method_side_effects(se, ListAppend, [], 1) 28 | 29 | def test_handles_list_extend(self): 30 | fun = function_doing_to_list('extend', [2]) 31 | call = inspect_returning_single_call(fun) 32 | se = assert_one_element_and_return(call.side_effects) 33 | assert_builtin_method_side_effects(se, ListExtend, [], [2]) 34 | 35 | def test_handles_list_insert(self): 36 | fun = function_doing_to_list('insert', 0, 3) 37 | call = inspect_returning_single_call(fun) 38 | se = assert_one_element_and_return(call.side_effects) 39 | assert_builtin_method_side_effects(se, ListInsert, [], 0, 3) 40 | 41 | def test_handles_list_pop_without_arguments(self): 42 | fun = function_doing_to_list('pop', alist=[1, 2, 3]) 43 | call = inspect_returning_single_call(fun) 44 | se = assert_one_element_and_return(call.side_effects) 45 | assert_builtin_method_side_effects(se, ListPop, [1, 2, 3]) 46 | 47 | def test_handles_list_pop_with_an_argument(self): 48 | fun = function_doing_to_list('pop', 1, alist=[1, 2, 3]) 49 | call = inspect_returning_single_call(fun) 50 | se = assert_one_element_and_return(call.side_effects) 51 | assert_builtin_method_side_effects(se, ListPop, [1, 2, 3], 1) 52 | 53 | def test_handles_list_pop_without_arguments_on_empty_list(self): 54 | def fun(): 55 | def foo(x): 56 | try: 57 | x.pop() 58 | except IndexError: 59 | pass 60 | foo([]) 61 | call = inspect_returning_single_call(fun) 62 | assert_equal([], call.side_effects) 63 | 64 | class TestGlobalVariables: 65 | def test_handles_reading(self): 66 | global was_run 67 | was_run = False 68 | def function_reading_global_variable(): 69 | def function(): 70 | return was_run 71 | function() 72 | call = inspect_returning_single_call(function_reading_global_variable) 73 | se = assert_one_element_and_return(call.side_effects) 74 | assert_instance(se, GlobalRead) 75 | assert_equal('test.test_tracing_side_effects', se.module) 76 | assert_equal('was_run', se.name) 77 | assert_serialized(False, se.value) 78 | 79 | def test_handles_rebinding(self): 80 | def function_rebinding_global_variable(): 81 | def function(): 82 | global was_run 83 | was_run = 0 84 | function() 85 | call = inspect_returning_single_call(function_rebinding_global_variable) 86 | se = assert_one_element_and_return(call.side_effects) 87 | assert_instance(se, GlobalRebind) 88 | assert_equal('test.test_tracing_side_effects', se.module) 89 | assert_equal('was_run', se.name) 90 | assert_serialized(0, se.value) 91 | -------------------------------------------------------------------------------- /test/testing_project.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | 3 | from pythoscope.astbuilder import EmptyCode 4 | from pythoscope.execution import Execution 5 | from pythoscope.store import Project 6 | 7 | from helper import MemoryCodeTreesManager 8 | 9 | 10 | class TestingProject(Project): 11 | """Project subclass useful during testing. 12 | 13 | It contains handy creation methods, which can all be nested. 14 | """ 15 | __test__ = False 16 | 17 | def __init__(self, path=os.path.realpath(".")): 18 | Project.__init__(self, path=path, 19 | code_trees_manager_class=MemoryCodeTreesManager) 20 | self._last_module = None 21 | self._all_catch_module = None 22 | 23 | def with_module(self, path="module.py"): 24 | modpath = os.path.join(self.path, path) 25 | self._last_module = self.create_module(modpath, code=EmptyCode()) 26 | return self 27 | 28 | def with_all_catch_module(self): 29 | """All object lookups will go through this single module. 30 | """ 31 | if self._all_catch_module is not None: 32 | raise ValueError("Already specified an all-catch module.") 33 | self.with_module() 34 | self._all_catch_module = self._last_module 35 | return self 36 | 37 | def with_object(self, obj): 38 | if self._last_module is None: 39 | raise ValueError("Tried to use with_object() without a module.") 40 | self._last_module.add_object(obj) 41 | return self 42 | 43 | def make_new_execution(self): 44 | return Execution(project=self) 45 | 46 | def find_object(self, type, name, modulename=None): 47 | if self._all_catch_module is not None: 48 | return self._all_catch_module.find_object(type, name) 49 | return Project.find_object(self, type, name, modulename) 50 | 51 | -------------------------------------------------------------------------------- /tools/memory_benchmark.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import sys 3 | 4 | pythoscope_path = os.path.join(os.path.dirname(__file__), os.pardir) 5 | sys.path.insert(0, os.path.abspath(pythoscope_path)) 6 | 7 | import pythoscope 8 | from pythoscope.inspector import inspect_project_statically 9 | from pythoscope.store import Module, Class, Function, Method, CodeTree 10 | from pympler import heapmonitor 11 | 12 | if len(sys.argv) != 2: 13 | print "usage:\n %s application_path\n" % sys.argv[0] 14 | print "application_path should point to a directory containing\n"\ 15 | "the project you wish to test pythoscope memory usage on.\n"\ 16 | "It should *not* be initialized (as in pythoscope --init)." 17 | sys.exit(1) 18 | 19 | def setup_tracking(project): 20 | heapmonitor.track_object(project) 21 | heapmonitor.track_class(Module) 22 | heapmonitor.track_class(Class) 23 | heapmonitor.track_class(Function) 24 | heapmonitor.track_class(Method) 25 | heapmonitor.track_class(CodeTree) 26 | heapmonitor.create_snapshot() 27 | 28 | # Finally call the real inspector. 29 | inspect_project_statically(project) 30 | 31 | def benchmark_project_memory_usage(): 32 | # Take the argument to this script and inject it as an argument to 33 | # pythoscope's --init. 34 | application_path = sys.argv[1] 35 | sys.argv = ["pythoscope", "--init", application_path] 36 | 37 | # Inject a setup function before performing an inspection. 38 | pythoscope.inspect_project_statically = setup_tracking 39 | 40 | # Invoke pythoscope --init. 41 | pythoscope.main() 42 | 43 | # Show statistics. 44 | heapmonitor.create_snapshot() 45 | heapmonitor.print_stats(detailed=False) 46 | 47 | if __name__ == "__main__": 48 | benchmark_project_memory_usage() 49 | -------------------------------------------------------------------------------- /tools/projects/Reverend-r17924.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mkwiatkowski/pythoscope/b4b89b77b5184b25992893e320d58de32ed987f1/tools/projects/Reverend-r17924.tar.gz -------------------------------------------------------------------------------- /tools/projects/Reverend_poe_from_homepage.py: -------------------------------------------------------------------------------- 1 | from reverend.thomas import Bayes 2 | guesser = Bayes() 3 | guesser.train('french', 'le la les du un une je il elle de en') 4 | guesser.train('german', 'der die das ein eine') 5 | guesser.train('spanish', 'el uno una las de la en') 6 | guesser.train('english', 'the it she he they them are were to') 7 | guesser.guess('they went to el cantina') 8 | guesser.guess('they were flying planes') 9 | guesser.train('english', 'the rain in spain falls mainly on the plain') 10 | guesser.save('my_guesser.bay') 11 | -------------------------------------------------------------------------------- /tools/projects/Reverend_poe_from_readme.py: -------------------------------------------------------------------------------- 1 | from reverend.thomas import Bayes 2 | 3 | guesser = Bayes() 4 | guesser.train('fish', 'salmon trout cod carp') 5 | guesser.train('fowl', 'hen chicken duck goose') 6 | 7 | guesser.guess('chicken tikka marsala') 8 | 9 | guesser.untrain('fish','salmon carp') 10 | -------------------------------------------------------------------------------- /tools/projects/freshwall-1.1.2-lib.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mkwiatkowski/pythoscope/b4b89b77b5184b25992893e320d58de32ed987f1/tools/projects/freshwall-1.1.2-lib.tar.gz -------------------------------------------------------------------------------- /tools/projects/freshwall_bin_with_snippet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright 2009 Chad Daelhousen. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 6 | # use this file except in compliance with the License. You may obtain a copy of 7 | # the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations under 15 | # the License. 16 | # 17 | 18 | import pythoscope 19 | pythoscope.start() 20 | 21 | import freshwall.core as core 22 | from optparse import OptionValueError 23 | import os.path 24 | import sys 25 | 26 | 27 | def parse_args (argv, parser=None): 28 | if parser is None: 29 | parser = core.get_parser(program=argv[0]) 30 | 31 | # Prefs which change the overall operating mode 32 | parser.add_option("-d", "--daemon", action="store_true", 33 | dest="daemon", default=False, 34 | help="run in daemon mode") 35 | parser.add_option("-p", "--preferences", action="store_true", 36 | dest="prefs", default=False, 37 | help="open preferences window") 38 | parser.add_option("-x", "--exit-daemon", action="store_true", 39 | dest="exit_daemon", default=False, 40 | help="signal the daemon to shut down") 41 | 42 | # Daemon mode options 43 | parser.add_option("-D", "--no-detach", action="store_false", 44 | dest="detach", default=True, 45 | help="do not detach when running as a daemon") 46 | parser.add_option("-s", "--spread", 47 | dest="spread", default="0", metavar="PERCENT", 48 | help="amount of randomness of the period (0-100)") 49 | parser.add_option("-T", "--period", 50 | dest="period", default="", metavar="TIME[d|h|m|s]", 51 | help="average time between wallpaper changes") 52 | 53 | return parser.parse_args(argv[1:]) 54 | 55 | 56 | def run_prefs_gui (prefs): 57 | from freshwall import gui 58 | return gui.run_prefs_dialog(prefs) 59 | 60 | def run_daemon (prefs, period, spread, detach): 61 | from freshwall import daemon 62 | return daemon.run(prefs, period, spread, detach) 63 | 64 | def exit_daemon (): 65 | from freshwall import daemon 66 | return daemon.exit() 67 | 68 | 69 | def main (argv=None): 70 | rv = 0 71 | if argv is None: 72 | argv = ['???'] 73 | 74 | prefs = core.load_prefs() 75 | options, args = parse_args(argv) 76 | 77 | if options.prefs: 78 | rv = run_prefs_gui(prefs) 79 | elif options.exit_daemon: 80 | rv = exit_daemon() 81 | elif options.daemon: 82 | rv = run_daemon(prefs, options.period, options.spread, options.detach) 83 | else: 84 | core.change_wallpaper(prefs) 85 | 86 | # clean up 87 | prefs.nuke() 88 | pythoscope.stop() 89 | return rv 90 | 91 | 92 | if __name__ == '__main__': 93 | sys.exit(main(sys.argv[:])) 94 | 95 | -------------------------------------------------------------------------------- /tools/projects/http-parser-0.2.0.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mkwiatkowski/pythoscope/b4b89b77b5184b25992893e320d58de32ed987f1/tools/projects/http-parser-0.2.0.tar.gz -------------------------------------------------------------------------------- /tools/projects/http-parser-example-1.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import pythoscope 3 | pythoscope.start() 4 | import socket 5 | 6 | from http_parser.parser import HttpParser 7 | 8 | 9 | def main(): 10 | 11 | p = HttpParser() 12 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 13 | body = [] 14 | header_done = False 15 | try: 16 | s.connect(('gunicorn.org', 80)) 17 | s.send("GET / HTTP/1.1\r\nHost: gunicorn.org\r\n\r\n") 18 | 19 | while True: 20 | data = s.recv(1024) 21 | if not data: 22 | break 23 | 24 | recved = len(data) 25 | nparsed = p.execute(data, recved) 26 | assert nparsed == recved 27 | 28 | if p.is_headers_complete() and not header_done: 29 | print p.get_headers() 30 | header_done = True 31 | 32 | if p.is_partial_body(): 33 | body.append(p.recv_body()) 34 | 35 | if p.is_message_complete(): 36 | break 37 | 38 | print "".join(body) 39 | 40 | finally: 41 | s.close() 42 | 43 | if __name__ == "__main__": 44 | main() 45 | pythoscope.stop() 46 | 47 | 48 | -------------------------------------------------------------------------------- /tools/projects/http-parser-example-2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import pythoscope 3 | pythoscope.start() 4 | import socket 5 | 6 | from http_parser.http import HttpStream 7 | from http_parser.reader import SocketReader 8 | 9 | def main(): 10 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 11 | try: 12 | s.connect(('gunicorn.org', 80)) 13 | s.send("GET / HTTP/1.1\r\nHost: gunicorn.org\r\n\r\n") 14 | p = HttpStream(SocketReader(s)) 15 | print p.headers() 16 | print p.body_file().read() 17 | finally: 18 | s.close() 19 | 20 | if __name__ == "__main__": 21 | main() 22 | pythoscope.stop() 23 | 24 | 25 | -------------------------------------------------------------------------------- /tools/projects/isodate-0.4.4-src.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mkwiatkowski/pythoscope/b4b89b77b5184b25992893e320d58de32ed987f1/tools/projects/isodate-0.4.4-src.tar.gz -------------------------------------------------------------------------------- /tools/projects/isodate_poe.py: -------------------------------------------------------------------------------- 1 | from isodate import * 2 | 3 | parse_time("2011-04-16 11:51Z") 4 | parse_tzinfo("2011-04-16 11:51Z") 5 | parse_datetime("2011-04-16T11:51Z") 6 | parse_tzinfo("2011-04-16T11:51Z") 7 | 8 | for s in ["2011-04-16", "2011-W15-6", "2011-106"]: 9 | parse_date(s) 10 | -------------------------------------------------------------------------------- /tools/projects/pyatom-1.2.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mkwiatkowski/pythoscope/b4b89b77b5184b25992893e320d58de32ed987f1/tools/projects/pyatom-1.2.tar.gz -------------------------------------------------------------------------------- /tools/projects/pyatom_poe_from_readme.py: -------------------------------------------------------------------------------- 1 | from pyatom import AtomFeed 2 | import datetime 3 | 4 | feed = AtomFeed(title="My Blog", 5 | subtitle="My example blog for a feed test.", 6 | feed_url="http://example.org/feed", 7 | url="http://example.org", 8 | author="Me") 9 | 10 | # Do this for each feed entry 11 | feed.add(title="My Post", 12 | content="Body of my post", 13 | content_type="html", 14 | author="Me", 15 | url="http://example.org/entry1", 16 | updated=datetime.datetime.utcnow()) 17 | 18 | print feed.to_string() 19 | -------------------------------------------------------------------------------- /tools/rst2wikidot.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from docutils.core import publish_cmdline, default_description 4 | from docutils.nodes import NodeVisitor 5 | from docutils.writers import Writer 6 | 7 | 8 | class WikidotTranslator(NodeVisitor): 9 | """Write output in Wikidot format. 10 | 11 | Based on http://www.wikidot.com/doc:wiki-syntax 12 | """ 13 | def __init__(self, document): 14 | NodeVisitor.__init__(self, document) 15 | 16 | self.section_level = 1 17 | self.first_paragraph = True 18 | self.inside_literal_block = False 19 | self.lists = [] 20 | self.block_input = False 21 | self._content = [] 22 | 23 | def get_text(self): 24 | return ''.join(self._content) 25 | 26 | def _add(self, string): 27 | if not self.block_input: 28 | self._content.append(string) 29 | 30 | def _nop(self, node): 31 | pass 32 | 33 | def _newline_if_not_first(self): 34 | if not self.first_paragraph: 35 | self._add("\n") 36 | 37 | visit_document = _nop 38 | depart_document = _nop 39 | 40 | def visit_section(self, node): 41 | self.section_level += 1 42 | def depart_section(self, node): 43 | self.section_level -= 1 44 | 45 | def visit_title(self, node): 46 | self._newline_if_not_first() 47 | self._add("+" * self.section_level + " ") 48 | def depart_title(self, node): 49 | self._add("\n") 50 | self.first_paragraph = False 51 | 52 | def visit_Text(self, node): 53 | string = node.astext() 54 | if not self.inside_literal_block: 55 | string = string.replace('\n', ' ') 56 | self._add(string) 57 | depart_Text = _nop 58 | 59 | def visit_paragraph(self, node): 60 | self._newline_if_not_first() 61 | def depart_paragraph(self, node): 62 | self._add("\n") 63 | self.first_paragraph = False 64 | 65 | def visit_strong(self, node): 66 | self._add("**") 67 | depart_strong = visit_strong 68 | 69 | def visit_reference(self, node): 70 | if node.has_key('name'): 71 | self._add("[%s " % node['refuri']) 72 | def depart_reference(self, node): 73 | if node.has_key('name'): 74 | self._add("]") 75 | 76 | visit_target = _nop 77 | depart_target = _nop 78 | 79 | def visit_literal_block(self, node): 80 | if re.search(r'(class )|(def )|(import )', node.astext()): 81 | self._add("\n[[code type=\"Python\"]]\n") 82 | else: 83 | self._add("\n[[code]]\n") 84 | self.inside_literal_block = True 85 | def depart_literal_block(self, node): 86 | self._add("\n[[/code]]\n") 87 | self.inside_literal_block = False 88 | 89 | def visit_topic(self, node): 90 | if 'contents' in node['classes']: 91 | self._add("[[toc]]\n") 92 | self.block_input = True 93 | def depart_topic(self, node): 94 | self.block_input = False 95 | 96 | def visit_bullet_list(self, node): 97 | self.lists.append('bullet') 98 | def depart_bullet_list(self, node): 99 | self.lists.pop() 100 | 101 | def visit_enumerated_list(self, node): 102 | self.lists.append('enumerated') 103 | def depart_enumerated_list(self, node): 104 | self.lists.pop() 105 | 106 | def visit_list_item(self, node): 107 | self._add(" " * (len(self.lists) - 1) * 2) 108 | if self.lists[-1] is 'enumerated': 109 | self._add("# ") 110 | else: 111 | self._add("* ") 112 | self.first_paragraph = True 113 | depart_list_item = _nop 114 | 115 | class WikidotWriter(Writer): 116 | def translate(self): 117 | visitor = WikidotTranslator(self.document) 118 | self.document.walkabout(visitor) 119 | self.output = visitor.get_text() 120 | 121 | 122 | if __name__ == '__main__': 123 | description = ('Generates documents in Wikidot format from standalone ' 124 | 'reStructuredText sources. ' + default_description) 125 | publish_cmdline(writer=WikidotWriter(), description=description) 126 | 127 | -------------------------------------------------------------------------------- /tools/run-tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import glob 4 | import shutil 5 | import sys 6 | 7 | from os import system as run 8 | from os import remove as rm 9 | 10 | def cp(src, dst): 11 | shutil.copy(glob.glob(src)[0], dst) 12 | 13 | def main(): 14 | VERSIONS = [('2.3', ['tests', 'build']), 15 | ('2.4', ['tests', 'build']), 16 | ('2.5', ['tests']), 17 | ('2.6', ['tests'])] 18 | results = {} 19 | 20 | for ver, types in VERSIONS: 21 | if 'tests' in types: 22 | version = "%s-tests" % ver 23 | print "*** Running tests on Python %s without binary modules." % ver 24 | if run("nosetests-%s" % ver) == 0: 25 | results[version] = 'OK' 26 | else: 27 | results[version] = 'FAIL (tests)' 28 | 29 | if 'build' in types: 30 | version = "%s-build" % ver 31 | res1 = res2 = None 32 | print "*** Running tests on Python %s with binary modules." % ver 33 | res1 = run("python%s setup.py build -f" % ver) 34 | if res1 == 0: 35 | cp("build/lib.*-%s/pythoscope/_util.so" % ver, "pythoscope/") 36 | res2 = run("nosetests-%s" % ver) 37 | rm("pythoscope/_util.so") 38 | if res1 == 0 and res2 == 0: 39 | results[version] = 'OK' 40 | else: 41 | if res1 != 0: 42 | results[version] = 'FAIL (compilation)' 43 | else: 44 | results[version] = 'FAIL (tests)' 45 | 46 | print 47 | for ver, result in sorted(results.iteritems()): 48 | print "%s: %s" % (ver, result) 49 | 50 | if [v for v in results.values() if v != 'OK']: 51 | return 1 52 | return 0 53 | 54 | if __name__ == '__main__': 55 | sys.exit(main()) 56 | -------------------------------------------------------------------------------- /tools/speed_benchmark.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | 3 | import gc 4 | import os.path 5 | import sys 6 | import timeit 7 | 8 | pythoscope_path = os.path.join(os.path.dirname(__file__), os.pardir) 9 | sys.path.insert(0, os.path.abspath(pythoscope_path)) 10 | 11 | from pythoscope.cmdline import init_project 12 | from pythoscope.store import get_pickle_path 13 | from test.helper import putfile, rmtree, tmpdir 14 | 15 | 16 | def make_class(name, methods_count=20): 17 | code = ["class %s(object):\n" % name] 18 | for i in range(methods_count): 19 | code.append(" def method_%d(self):\n pass\n" % i) 20 | return ''.join(code) 21 | 22 | def make_function(name): 23 | return "def %s():\n pass\n" % name 24 | 25 | def make_module(classes_count=10, functions_count=10): 26 | code = [] 27 | for i in range(classes_count): 28 | code.append(make_class("Class%d" % i)) 29 | for i in range(functions_count): 30 | code.append(make_function("function_%d" % i)) 31 | return ''.join(code) 32 | 33 | # Run the setup once, stmt n times and report the minimum running time. 34 | # 35 | # Based on timeit module. I had to modify it, because: 36 | # - timer.timeit(n) returns time of running stmt n times (the sum, not the minimum), 37 | # - min(timer.repeat(n, 1)) runs the setup n times. 38 | timer_template = """ 39 | def inner(_n, _timer): 40 | _results = [] 41 | %(setup)s 42 | for _i in range(_n): 43 | _t0 = _timer() 44 | %(stmt)s 45 | _t1 = _timer() 46 | _results.append(_t1 - _t0) 47 | return min(_results) 48 | """ 49 | 50 | def run_timer(stmt, setup, n=3, timer=timeit.default_timer): 51 | src = timer_template % {'stmt': stmt, 'setup': setup} 52 | code = compile(src, '', "exec") 53 | ns = {} 54 | exec code in globals(), ns 55 | inner = ns["inner"] 56 | 57 | gcold = gc.isenabled() 58 | gc.disable() 59 | timing = inner(n, timer) 60 | if gcold: 61 | gc.enable() 62 | return timing 63 | 64 | def human_size(bytes, prefixes=['', 'K', 'M', 'G']): 65 | if bytes > 1024: 66 | return human_size(bytes/1024, prefixes[1:]) 67 | return "%.2f%sb" % (bytes, prefixes[0]) 68 | 69 | def benchmark_project_load_performance(modules_count=25): 70 | print "==> Creating project with %d modules..." % modules_count 71 | project_path = tmpdir() 72 | module = make_module() 73 | for i in range(modules_count): 74 | putfile(project_path, "module%s.py" % i, module) 75 | init_project(project_path, skip_inspection=True) 76 | 77 | print "==> Inspecting project.." 78 | elapsed = run_timer("inspect_project(Project('%s'))" % project_path, 79 | "from pythoscope.inspector import inspect_project; from pythoscope.store import Project") 80 | print "It took %f seconds to inspect." % elapsed 81 | 82 | print "==> Saving project information" 83 | elapsed = run_timer("project.save()", 84 | """from pythoscope.inspector import inspect_project ;\ 85 | from pythoscope.store import Project ;\ 86 | project = Project('%s') ;\ 87 | inspect_project(project)""" % project_path) 88 | print "It took %f seconds to save the project information." % elapsed 89 | 90 | print "==> Reading project information" 91 | elapsed = run_timer("Project.from_directory('%s')" % project_path, 92 | "from pythoscope.store import Project") 93 | print "It took %f seconds to read project information from %s pickle." % \ 94 | (elapsed, human_size(os.path.getsize(get_pickle_path(project_path)))) 95 | 96 | rmtree(project_path) 97 | 98 | if __name__ == "__main__": 99 | benchmark_project_load_performance() 100 | --------------------------------------------------------------------------------