├── .gitmodules ├── slicker ├── __init__.py ├── unicode_util.py ├── cleanup.py ├── moves.py ├── util.py ├── inputs.py ├── removal.py ├── replacement.py ├── model.py └── khodemod.py ├── .flake8 ├── requirements.dev.txt ├── .gitignore ├── testdata ├── syntax_error_in.py ├── same_alias_in.py ├── source_file_in.py ├── destination_file_out.py ├── same_alias_unused_in.py ├── same_alias_unused_out.py ├── same_alias_out.py ├── symbol_alias_none_out.py ├── symbol_out.py ├── symbol_in.py ├── simple_out.py ├── symbol_alias_none_in.py ├── simple_in.py ├── moving_to_from_in.py ├── conflict_in.py ├── destination_file_in.py ├── somepackage │ ├── relative_out.py │ ├── relative_in.py │ ├── relative_same_package_in.py │ └── relative_same_package_out.py ├── moving_to_from_out.py ├── conflict_2_in.py ├── source_file_2_in.py ├── unused_conflict_in.py ├── unused_conflict_out.py ├── destination_file_2_out.py ├── linebreaks_out.py ├── whole_file_alias_in.py ├── source_file_2_out.py ├── whole_file_alias_out.py ├── destination_file_2_in.py ├── repeated_name_in.py ├── same_prefix_in.py ├── same_prefix_out.py ├── repeated_name_out.py ├── source_file_out.py ├── mock_out.py ├── many_imports_out.py ├── third_party_sorting_in.py ├── many_imports_in.py ├── whole_file_in.py ├── whole_file_out.py ├── third_party_sorting_out.py ├── double_implicit_out.py ├── double_implicit_in.py ├── moving_implicit_in.py ├── moving_implicit_out.py ├── unicode_out.py ├── unicode_in.py ├── linebreaks_in.py ├── unused_in.py ├── unused_out.py ├── implicit_in.py ├── implicit_out.py ├── imported_twice_out.py ├── mock_in.py ├── imported_twice_in.py ├── implicit_and_alias_out.py ├── comments_out.py ├── implicit_and_alias_in.py ├── comments_in.py ├── comments_whole_file_in.py ├── comments_whole_file_out.py ├── late_import_in.py ├── late_import_out.py ├── comments_top_level_in.py ├── comments_top_level_out.py ├── slicker_in.py └── slicker_out.py ├── .travis.yml ├── Makefile ├── setup.py ├── tests ├── test_khodemod.py ├── base.py ├── test_util.py ├── test_replacement.py ├── test_cleanup.py ├── test_inputs.py └── test_moves.py └── README.md /.gitmodules: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /slicker/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = .git,testdata 3 | -------------------------------------------------------------------------------- /requirements.dev.txt: -------------------------------------------------------------------------------- 1 | mock 2 | flake8 3 | twine 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | dist 3 | build 4 | slicker.egg-info 5 | -------------------------------------------------------------------------------- /testdata/syntax_error_in.py: -------------------------------------------------------------------------------- 1 | import foo.some_function 2 | 3 | def func(): 4 | pass 5 | -------------------------------------------------------------------------------- /testdata/same_alias_in.py: -------------------------------------------------------------------------------- 1 | import foo 2 | 3 | 4 | def f(): 5 | foo.some_function() 6 | -------------------------------------------------------------------------------- /testdata/source_file_in.py: -------------------------------------------------------------------------------- 1 | """Some docstring.""" 2 | 3 | 4 | def f(): 5 | myfunc() 6 | -------------------------------------------------------------------------------- /testdata/destination_file_out.py: -------------------------------------------------------------------------------- 1 | """Some docstring.""" 2 | 3 | 4 | def f(): 5 | myfunc() 6 | -------------------------------------------------------------------------------- /testdata/same_alias_unused_in.py: -------------------------------------------------------------------------------- 1 | import foo 2 | 3 | 4 | def f(): 5 | foo.unrelated() 6 | -------------------------------------------------------------------------------- /testdata/same_alias_unused_out.py: -------------------------------------------------------------------------------- 1 | import foo 2 | 3 | 4 | def f(): 5 | foo.unrelated() 6 | -------------------------------------------------------------------------------- /testdata/same_alias_out.py: -------------------------------------------------------------------------------- 1 | import bar as foo 2 | 3 | 4 | def f(): 5 | foo.some_function() 6 | -------------------------------------------------------------------------------- /testdata/symbol_alias_none_out.py: -------------------------------------------------------------------------------- 1 | import bar 2 | 3 | 4 | def do_thing(): 5 | bar.new_name(1) 6 | -------------------------------------------------------------------------------- /testdata/symbol_out.py: -------------------------------------------------------------------------------- 1 | from bar import new_name 2 | 3 | 4 | def do_thing(): 5 | new_name(1) 6 | -------------------------------------------------------------------------------- /testdata/symbol_in.py: -------------------------------------------------------------------------------- 1 | from foo import some_function 2 | 3 | 4 | def do_thing(): 5 | some_function(1) 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | install: make dev_deps install 5 | script: make check 6 | -------------------------------------------------------------------------------- /testdata/simple_out.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import bar # an import 4 | 5 | bar.new_name() 6 | -------------------------------------------------------------------------------- /testdata/symbol_alias_none_in.py: -------------------------------------------------------------------------------- 1 | from foo import some_function 2 | 3 | 4 | def do_thing(): 5 | some_function(1) 6 | -------------------------------------------------------------------------------- /testdata/simple_in.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import foo # an import 4 | 5 | foo.some_function() 6 | -------------------------------------------------------------------------------- /testdata/moving_to_from_in.py: -------------------------------------------------------------------------------- 1 | import foo 2 | 3 | 4 | def f(): 5 | foo.boring_function() 6 | foo.interesting_function() 7 | -------------------------------------------------------------------------------- /testdata/conflict_in.py: -------------------------------------------------------------------------------- 1 | import foo.bar 2 | 3 | 4 | def f(): 5 | foo.bar.boring_function() 6 | foo.bar.interesting_function() 7 | -------------------------------------------------------------------------------- /testdata/destination_file_in.py: -------------------------------------------------------------------------------- 1 | """Some docstring.""" 2 | import somewhere_else 3 | 4 | 5 | def f(): 6 | somewhere_else.myfunc() 7 | -------------------------------------------------------------------------------- /testdata/somepackage/relative_out.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import bar # an import 4 | 5 | bar.new_name() 6 | -------------------------------------------------------------------------------- /testdata/moving_to_from_out.py: -------------------------------------------------------------------------------- 1 | from bar import foo 2 | 3 | 4 | def f(): 5 | foo.boring_function() 6 | foo.interesting_function() 7 | -------------------------------------------------------------------------------- /testdata/somepackage/relative_in.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from . import foo # an import 4 | 5 | foo.some_function() 6 | -------------------------------------------------------------------------------- /testdata/conflict_2_in.py: -------------------------------------------------------------------------------- 1 | import quux as foo 2 | import bar 3 | 4 | 5 | def f(): 6 | foo.boring_function() 7 | bar.interesting_function() 8 | -------------------------------------------------------------------------------- /testdata/source_file_2_in.py: -------------------------------------------------------------------------------- 1 | """Some docstring.""" 2 | import somewhere_else 3 | 4 | 5 | def f(): 6 | myfunc() 7 | somewhere_else.otherfunc() 8 | -------------------------------------------------------------------------------- /testdata/unused_conflict_in.py: -------------------------------------------------------------------------------- 1 | import foo.bar 2 | 3 | 4 | def f(): 5 | """This is not interesting_function().""" 6 | foo.bar.boring_function() 7 | -------------------------------------------------------------------------------- /testdata/unused_conflict_out.py: -------------------------------------------------------------------------------- 1 | import foo.bar 2 | 3 | 4 | def f(): 5 | """This is not interesting_function().""" 6 | foo.bar.boring_function() 7 | -------------------------------------------------------------------------------- /testdata/destination_file_2_out.py: -------------------------------------------------------------------------------- 1 | """Some docstring.""" 2 | import somewhere_else 3 | 4 | 5 | def f(): 6 | myfunc() 7 | somewhere_else.otherfunc() 8 | -------------------------------------------------------------------------------- /testdata/somepackage/relative_same_package_in.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from . import foo # an import 4 | 5 | foo.some_function() 6 | -------------------------------------------------------------------------------- /testdata/somepackage/relative_same_package_out.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from . import bar # an import 4 | 5 | bar.some_function() 6 | -------------------------------------------------------------------------------- /testdata/linebreaks_out.py: -------------------------------------------------------------------------------- 1 | import quux 2 | 3 | 4 | def f(): 5 | x = (quux.new_name 6 | ()) 7 | return (quux.new_name( 8 | x)) 9 | -------------------------------------------------------------------------------- /testdata/whole_file_alias_in.py: -------------------------------------------------------------------------------- 1 | import foo # an import 2 | 3 | foo.some_function() 4 | 5 | 6 | def f(): 7 | """Docstring.""" 8 | foo.other_function() 9 | -------------------------------------------------------------------------------- /testdata/source_file_2_out.py: -------------------------------------------------------------------------------- 1 | """Some docstring.""" 2 | import somewhere_else 3 | 4 | 5 | def f(): 6 | somewhere_else.myfunc() 7 | somewhere_else.otherfunc() 8 | -------------------------------------------------------------------------------- /testdata/whole_file_alias_out.py: -------------------------------------------------------------------------------- 1 | import bar as baz # an import 2 | 3 | baz.some_function() 4 | 5 | 6 | def f(): 7 | """Docstring.""" 8 | baz.other_function() 9 | -------------------------------------------------------------------------------- /testdata/destination_file_2_in.py: -------------------------------------------------------------------------------- 1 | """Some docstring.""" 2 | import somewhere_else 3 | 4 | 5 | def f(): 6 | somewhere_else.myfunc() 7 | somewhere_else.otherfunc() 8 | -------------------------------------------------------------------------------- /testdata/repeated_name_in.py: -------------------------------------------------------------------------------- 1 | from foo import foo 2 | 3 | def test(): 4 | with mock.patch('foo.foo.myfunc', lambda: None): 5 | pass 6 | foo.otherfunc(foo) 7 | -------------------------------------------------------------------------------- /testdata/same_prefix_in.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import foo.bar 4 | 5 | 6 | def f(): 7 | foo.bar.some_function() # call the thing 8 | -------------------------------------------------------------------------------- /testdata/same_prefix_out.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import foo.baz 4 | 5 | 6 | def f(): 7 | foo.baz.some_function() # call the thing 8 | -------------------------------------------------------------------------------- /testdata/repeated_name_out.py: -------------------------------------------------------------------------------- 1 | import bar.foo.foo 2 | 3 | def test(): 4 | with mock.patch('bar.foo.foo.myfunc', lambda: None): 5 | pass 6 | bar.foo.foo.otherfunc(bar.foo.foo) 7 | -------------------------------------------------------------------------------- /testdata/source_file_out.py: -------------------------------------------------------------------------------- 1 | """Some docstring.""" 2 | 3 | 4 | from __future__ import absolute_import 5 | 6 | import somewhere_else 7 | 8 | 9 | def f(): 10 | somewhere_else.myfunc() 11 | -------------------------------------------------------------------------------- /testdata/mock_out.py: -------------------------------------------------------------------------------- 1 | def test(): 2 | mock_function('quux.some_function') 3 | mock_function("quux.some_function") 4 | mock_function('quux.some_function') 5 | mock_function("quux.some_function") 6 | -------------------------------------------------------------------------------- /testdata/many_imports_out.py: -------------------------------------------------------------------------------- 1 | import baz 2 | import foo.bar 3 | import foo.baz 4 | import foo.foobar 5 | 6 | 7 | def f(): 8 | foo.bar.asdf() 9 | foo.baz.something(baz.replaced, foo.foobar.the_foobar) 10 | -------------------------------------------------------------------------------- /testdata/third_party_sorting_in.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from __future__ import absolute_import 3 | import os 4 | import mycode2 5 | import third_party.slicker 6 | import mycode1 7 | 8 | def foo(): 9 | return 5 10 | -------------------------------------------------------------------------------- /testdata/many_imports_in.py: -------------------------------------------------------------------------------- 1 | import foo.bar 2 | import foo.baz 3 | import foo.foobar 4 | from foo import quux 5 | 6 | 7 | def f(): 8 | foo.bar.asdf() 9 | foo.baz.something(quux.replaceme, foo.foobar.the_foobar) 10 | -------------------------------------------------------------------------------- /testdata/whole_file_in.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, print_function 2 | 3 | import foo # an import 4 | 5 | foo.some_function() 6 | 7 | 8 | def f(): 9 | """Docstring.""" 10 | foo.other_function() 11 | -------------------------------------------------------------------------------- /testdata/whole_file_out.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, print_function 2 | 3 | import bar # an import 4 | 5 | bar.some_function() 6 | 7 | 8 | def f(): 9 | """Docstring.""" 10 | bar.other_function() 11 | -------------------------------------------------------------------------------- /testdata/third_party_sorting_out.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import os 4 | import sys 5 | 6 | import third_party.slicker 7 | 8 | import mycode1 9 | import mycode2 10 | 11 | def foo(): 12 | return 5 13 | -------------------------------------------------------------------------------- /testdata/double_implicit_out.py: -------------------------------------------------------------------------------- 1 | # does stuff 2 | import foo.public 3 | import quux 4 | 5 | 6 | def f(): 7 | """Calls some stuff in foo, mwahaha!""" 8 | quux.new_name() 9 | foo.public.function() 10 | foo.secrets.lulz() 11 | -------------------------------------------------------------------------------- /testdata/double_implicit_in.py: -------------------------------------------------------------------------------- 1 | # does stuff 2 | import foo.bar.baz 3 | import foo.public 4 | 5 | 6 | def f(): 7 | """Calls some stuff in foo, mwahaha!""" 8 | foo.bar.baz.some_function() 9 | foo.public.function() 10 | foo.secrets.lulz() 11 | -------------------------------------------------------------------------------- /testdata/moving_implicit_in.py: -------------------------------------------------------------------------------- 1 | # does stuff 2 | import foo.bar.baz 3 | import foo.public 4 | 5 | 6 | def f(): 7 | """Calls some stuff in foo, mwahaha!""" 8 | foo.bar.baz.some_function() 9 | foo.public.function() 10 | foo.secrets.lulz() 11 | -------------------------------------------------------------------------------- /testdata/moving_implicit_out.py: -------------------------------------------------------------------------------- 1 | # does stuff 2 | import foo.bar.baz 3 | import foo.public 4 | import quux 5 | 6 | 7 | def f(): 8 | """Calls some stuff in foo, mwahaha!""" 9 | foo.bar.baz.some_function() 10 | foo.public.function() 11 | quux.new_name() 12 | -------------------------------------------------------------------------------- /testdata/unicode_out.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """A file with some uͮ͑̔̉ͮ͝n̽ͦ̋̃ͦ̌̇͢͜ì̐̏̅̔͛́͜c̊͐o̓͊̽́ͩ͠dͬͧe̢̍ͩ̂̊̄͘ text.""" 3 | from __future__ import absolute_import 4 | 5 | import bar # an i͋͆̑́̚͏̕mͮͦ̂́͛ͬͯ͢p̓̋̌̓̈́̄o̴̧ͩ̆̆ͩ҉r͆̆ͬ͊̒͊̚̕͢t̡̧ͮ̐̏ͨ̒ͣ͊̋ 😁 6 | 7 | bar.new_name() 8 | -------------------------------------------------------------------------------- /testdata/unicode_in.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """A file with some uͮ͑̔̉ͮ͝n̽ͦ̋̃ͦ̌̇͢͜ì̐̏̅̔͛́͜c̊͐o̓͊̽́ͩ͠dͬͧe̢̍ͩ̂̊̄͘ text.""" 3 | from __future__ import absolute_import 4 | 5 | import foo # an i͋͆̑́̚͏̕mͮͦ̂́͛ͬͯ͢p̓̋̌̓̈́̄o̴̧ͩ̆̆ͩ҉r͆̆ͬ͊̒͊̚̕͢t̡̧ͮ̐̏ͨ̒ͣ͊̋ 😁 6 | 7 | foo.some_function() 8 | -------------------------------------------------------------------------------- /testdata/linebreaks_in.py: -------------------------------------------------------------------------------- 1 | import foo.bar.baz 2 | 3 | 4 | def f(): 5 | x = (foo 6 | .bar 7 | .baz 8 | .some_function 9 | ()) 10 | return (foo. 11 | bar. 12 | baz. 13 | some_function( 14 | x)) 15 | -------------------------------------------------------------------------------- /testdata/unused_in.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | # this is a special import block (and this comment ends up 4 | # in arguably the wrong place!) 5 | import foo.bar # @UnusedImport 6 | 7 | 8 | def f(): 9 | foo.bar.some_function() # needs foo.bar to be imported 10 | -------------------------------------------------------------------------------- /testdata/unused_out.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import foo.bar # @UnusedImport 4 | # this is a special import block (and this comment ends up 5 | # in arguably the wrong place!) 6 | import quux # @UnusedImport 7 | 8 | 9 | def f(): 10 | quux.some_function() # needs foo.bar to be imported 11 | -------------------------------------------------------------------------------- /testdata/implicit_in.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | # this is a special import block (and this comment ends up 4 | # in arguably the wrong place!) 5 | import foo.bar.baz 6 | 7 | 8 | def f(): 9 | """Calls some stuff in foo, mwahaha!""" 10 | foo.bar.baz.some_function() 11 | foo.secrets.lulz() 12 | -------------------------------------------------------------------------------- /testdata/implicit_out.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import foo.bar.baz 4 | # this is a special import block (and this comment ends up 5 | # in arguably the wrong place!) 6 | import quux 7 | 8 | 9 | def f(): 10 | """Calls some stuff in foo, mwahaha!""" 11 | quux.new_name() 12 | foo.secrets.lulz() 13 | -------------------------------------------------------------------------------- /testdata/imported_twice_out.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | # TODO(benkraft): In the case where these two imports are rewritten to be 4 | # identical, maybe we should remove the now-exact duplicate? 5 | import quux 6 | import quux 7 | 8 | 9 | def f(): 10 | # These are secretly the same! 11 | quux.some_function() 12 | quux.some_function() 13 | -------------------------------------------------------------------------------- /testdata/mock_in.py: -------------------------------------------------------------------------------- 1 | def test(): 2 | mock_function('foo.bar.some_function') 3 | mock_function("foo." 4 | "bar.some_function") 5 | mock_function('foo.' 6 | """bar.""" 7 | 'some_function') 8 | mock_function('foo.' 9 | 'bar.' 10 | 'some_' 11 | "function") 12 | -------------------------------------------------------------------------------- /testdata/imported_twice_in.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | # TODO(benkraft): In the case where these two imports are rewritten to be 4 | # identical, maybe we should remove the now-exact duplicate? 5 | import foo.bar 6 | from foo import bar 7 | 8 | 9 | def f(): 10 | # These are secretly the same! 11 | foo.bar.some_function() 12 | bar.some_function() 13 | -------------------------------------------------------------------------------- /testdata/implicit_and_alias_out.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | # this is a special import block (and this comment ends up 4 | # in arguably the wrong place!) 5 | # TODO(benkraft): In the case where these two imports are rewritten to be 6 | # identical, maybe we should remove the now-exact duplicate? 7 | import quux 8 | 9 | 10 | def f(): 11 | """Calls some stuff in foo, mwahaha!""" 12 | quux.new_name() 13 | quux.new_name() 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | dev_deps: 2 | pip install -r requirements.dev.txt 3 | 4 | install: 5 | pip install -e . 6 | 7 | test: 8 | python -m unittest discover tests 9 | 10 | lint: 11 | flake8 12 | 13 | check: lint test ; 14 | 15 | clean: 16 | git clean -xffd 17 | 18 | build: dev_deps 19 | python setup.py sdist # builds source distribution 20 | python setup.py bdist_wheel # builds wheel 21 | 22 | release: clean build 23 | twine upload dist/* 24 | 25 | .PHONY: deps test lint build release 26 | -------------------------------------------------------------------------------- /testdata/comments_out.py: -------------------------------------------------------------------------------- 1 | """File docstring mentioning that we depend on quux.mod.some_function. 2 | 3 | Also mentions quux.mod.some_function in the body, for good measure. 4 | """ 5 | import quux.mod as al 6 | 7 | 8 | def f(): 9 | # References quux.mod.some_function, here called al.some_function. 10 | al.some_function("""al.some_function, 11 | other_function""") 12 | al.some_function("super wacky " """al.some_function, 13 | other_function""") 14 | -------------------------------------------------------------------------------- /testdata/implicit_and_alias_in.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | # this is a special import block (and this comment ends up 4 | # in arguably the wrong place!) 5 | # TODO(benkraft): In the case where these two imports are rewritten to be 6 | # identical, maybe we should remove the now-exact duplicate? 7 | import foo.other 8 | from foo.bar import baz 9 | 10 | 11 | def f(): 12 | """Calls some stuff in foo, mwahaha!""" 13 | baz.some_function() 14 | foo.bar.baz.some_function() 15 | -------------------------------------------------------------------------------- /testdata/comments_in.py: -------------------------------------------------------------------------------- 1 | """File docstring mentioning that we depend on foo.bar.some_function. 2 | 3 | Also mentions foo.bar.some_function in the body, for good measure. 4 | """ 5 | import foo.bar as baz 6 | 7 | 8 | def f(): 9 | # References foo.bar.some_function, here called baz.some_function. 10 | baz.some_function('baz.some_' 11 | """function, 12 | other_function""") 13 | baz.some_function("super wacky baz." 'some_' 14 | """function, 15 | other_function""") 16 | -------------------------------------------------------------------------------- /testdata/comments_whole_file_in.py: -------------------------------------------------------------------------------- 1 | """File docstring mentioning that we depend on foo.bar.some_function. 2 | 3 | Also mentions foo.bar.some_function in the body, for good measure. FYI: the 4 | function comes from foo/bar.py. 5 | """ 6 | import foo.bar as baz 7 | 8 | 9 | def f(): 10 | # References foo.bar.some_function from foo/bar.py, here called 11 | # baz.some_function. 12 | baz.some_function('baz.some_' 13 | """function, 14 | other_function""") 15 | baz.some_function("super wacky baz." 'some_' 16 | """function, 17 | other_function""") 18 | -------------------------------------------------------------------------------- /testdata/comments_whole_file_out.py: -------------------------------------------------------------------------------- 1 | """File docstring mentioning that we depend on quux.mod.some_function. 2 | 3 | Also mentions quux.mod.some_function in the body, for good measure. FYI: the 4 | function comes from quux/mod.py. 5 | """ 6 | import quux.mod as al 7 | 8 | 9 | def f(): 10 | # References quux.mod.some_function from quux/mod.py, here called 11 | # al.some_function. 12 | al.some_function('al.some_' 13 | """function, 14 | other_function""") 15 | al.some_function("super wacky al." 'some_' 16 | """function, 17 | other_function""") 18 | -------------------------------------------------------------------------------- /testdata/late_import_in.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import foo.baz # should update this, for the call in g(), but we don't 4 | 5 | 6 | def f(): 7 | # NOTE(benkraft): Here and below, we don't order things right due to 8 | # limitations of fix_python_imports (it doesn't deal with late imports). 9 | # Additionally, here, we don't notice that we can remove foo.bar, because 10 | # we don't chase scopes. 11 | import foo.bar 12 | foo.bar.some_function() 13 | 14 | 15 | def g(): 16 | foo.bar.some_function() 17 | foo.baz.other_function() 18 | 19 | 20 | def h(): 21 | import foo.bar 22 | foo.bar.some_function() 23 | foo.bar.other_function() 24 | -------------------------------------------------------------------------------- /testdata/late_import_out.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import foo.baz # should update this, for the call in g(), but we don't 4 | 5 | 6 | def f(): 7 | # NOTE(benkraft): Here and below, we don't order things right due to 8 | # limitations of fix_python_imports (it doesn't deal with late imports). 9 | # Additionally, here, we don't notice that we can remove foo.bar, because 10 | # we don't chase scopes. 11 | import quux 12 | import foo.bar 13 | quux.some_function() 14 | 15 | 16 | def g(): 17 | quux.some_function() 18 | foo.baz.other_function() 19 | 20 | 21 | def h(): 22 | import quux 23 | import foo.bar 24 | quux.some_function() 25 | foo.bar.other_function() 26 | -------------------------------------------------------------------------------- /testdata/comments_top_level_in.py: -------------------------------------------------------------------------------- 1 | """File docstring mentioning that we depend on foo.some_function. 2 | 3 | Also mentions foo.some_function in the body, for good measure. FYI: the 4 | function comes from foo.py. 5 | 6 | Here are some things that should not be changed when we rewrite: 7 | foo_bar bar_foo bar_foo_baz foobar barfoo barfoobaz 8 | bar_foo.py barfoo.py foo.python bar/foo.py 9 | """ 10 | import foo as baz 11 | 12 | 13 | def f(): 14 | # References foo.some_function from foo.py, here called 15 | # baz.some_function. 16 | baz.some_function('baz.some_' 17 | """function, 18 | other_function""") 19 | baz.some_function("super wacky baz." 'some_' 20 | """function, 21 | other_function""") 22 | -------------------------------------------------------------------------------- /testdata/comments_top_level_out.py: -------------------------------------------------------------------------------- 1 | """File docstring mentioning that we depend on quux.mod.some_function. 2 | 3 | Also mentions quux.mod.some_function in the body, for good measure. FYI: the 4 | function comes from quux/mod.py. 5 | 6 | Here are some things that should not be changed when we rewrite: 7 | foo_bar bar_foo bar_foo_baz foobar barfoo barfoobaz 8 | bar_foo.py barfoo.py foo.python bar/foo.py 9 | """ 10 | import quux.mod as al 11 | 12 | 13 | def f(): 14 | # References quux.mod.some_function from quux/mod.py, here called 15 | # al.some_function. 16 | al.some_function('al.some_' 17 | """function, 18 | other_function""") 19 | al.some_function("super wacky al." 'some_' 20 | """function, 21 | other_function""") 22 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup 4 | 5 | with open("README.md") as f: 6 | long_description = f.read() 7 | 8 | setup( 9 | name='slicker', 10 | version='0.9.3', 11 | description='A tool for moving python files.', 12 | long_description=long_description, 13 | long_description_content_type='text/markdown', 14 | author='Khan Academy', 15 | author_email='opensource+pypi@khanacademy.org', 16 | url='https://github.com/Khan/slicker', 17 | keywords=['codemod', 'refactor', 'refactoring'], 18 | packages=['slicker'], 19 | install_requires=['asttokens==1.1.8', 'tqdm==4.19.5', 'fix-includes==0.2'], 20 | entry_points={ 21 | # setuptools magic to make a `slicker` binary 22 | 'console_scripts': ['slicker = slicker.slicker:main'], 23 | }, 24 | classifiers=[ 25 | 'License :: OSI Approved :: MIT License', 26 | 'Programming Language :: Python :: 2', 27 | 'Programming Language :: Python :: 2.7', 28 | ], 29 | ) 30 | -------------------------------------------------------------------------------- /tests/test_khodemod.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from slicker import khodemod 4 | 5 | import base 6 | 7 | 8 | class PathFilterTest(base.TestBase): 9 | def test_resolve_paths(self): 10 | self.write_file('foo.py', '') 11 | self.write_file('bar/baz.py', '') 12 | self.write_file('.dotfile.py', '') 13 | self.write_file('.dotdir/something.py', '') 14 | self.write_file('foo_extensionless_py', '') 15 | self.write_file('foo.js', '') 16 | self.write_file('foo.css', '') 17 | self.write_file('genfiles/qux.py', '') 18 | self.write_file('build/qux.py', '') 19 | 20 | self.assertItemsEqual( 21 | khodemod.resolve_paths( 22 | khodemod.default_path_filter(), 23 | root=self.tmpdir), 24 | ['foo.py', 'bar/baz.py', 'build/qux.py']) 25 | 26 | self.assertItemsEqual( 27 | khodemod.resolve_paths( 28 | khodemod.default_path_filter( 29 | exclude_paths=('genfiles', 'build')), 30 | root=self.tmpdir), 31 | ['foo.py', 'bar/baz.py']) 32 | 33 | self.assertItemsEqual( 34 | khodemod.resolve_paths( 35 | khodemod.default_path_filter( 36 | extensions=('js', 'css'), include_extensionless=True), 37 | root=self.tmpdir), 38 | ['foo_extensionless_py', 'foo.js', 'foo.css']) 39 | -------------------------------------------------------------------------------- /tests/base.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import os 4 | import shutil 5 | import tempfile 6 | import unittest 7 | 8 | from slicker import khodemod 9 | 10 | 11 | class TestBase(unittest.TestCase): 12 | maxDiff = None 13 | 14 | def setUp(self): 15 | self.tmpdir = os.path.realpath( 16 | tempfile.mkdtemp(prefix=(self.__class__.__name__ + '.'))) 17 | self.error_output = [] 18 | # Poor-man's mock. 19 | _old_emit = khodemod.emit 20 | 21 | def restore_emit(): 22 | khodemod.emit = _old_emit 23 | self.addCleanup(restore_emit) 24 | khodemod.emit = lambda txt: self.error_output.append(txt) 25 | 26 | def tearDown(self): 27 | shutil.rmtree(self.tmpdir) 28 | 29 | def join(self, *args): 30 | return os.path.join(self.tmpdir, *args) 31 | 32 | def copy_file(self, filename): 33 | """Copy a file from testdata to tmpdir.""" 34 | shutil.copyfile(os.path.join('testdata', filename), 35 | os.path.join(self.tmpdir, filename)) 36 | 37 | def write_file(self, filename, contents): 38 | if not os.path.exists(self.join(os.path.dirname(filename))): 39 | os.makedirs(os.path.dirname(self.join(filename))) 40 | with open(self.join(filename), 'w') as f: 41 | f.write(contents) 42 | # We may have a cached path-resolution; if we made a new file, it's now 43 | # wrong. (We could instead call khodemod.write_file which does this 44 | # more precisely, but this is more convenient.) 45 | khodemod._RESOLVE_PATHS_CACHE.clear() 46 | 47 | def assertFileIs(self, filename, expected): 48 | with open(self.join(filename)) as f: 49 | actual = f.read() 50 | self.assertMultiLineEqual(expected, actual) 51 | 52 | def assertFileIsNot(self, filename): 53 | self.assertFalse(os.path.exists(self.join(filename))) 54 | -------------------------------------------------------------------------------- /slicker/unicode_util.py: -------------------------------------------------------------------------------- 1 | """Utils for handling unicode in source. 2 | 3 | Slicker doesn't currently handle unicode identifiers properly -- they aren't 4 | allowed in python 2 anyway -- but it does need to be able to work around 5 | unicode in comments and docstrings. But doing so is a bit weird, because ast 6 | likes to operate on bytes (it will accept unicode, but effectively converts it 7 | back to bytes, and complains if it has a coding comment), whereas asttokens 8 | (not unreasonably) does everything in unicode. (In the future, if we support 9 | python 3, we'll also need to do everything in unicode; right now we'd be okay 10 | with either.) So we have to convert back and forth a bit; this file has utils 11 | we use to do that. 12 | 13 | TODO(benkraft): This breaks the rule that khodemod shouldn't do anything 14 | file-format-specific. Figure out a better way. 15 | TODO(benkraft): If we move a symbol whose definition contains unicode, we don't 16 | currently move/copy the magic coding comment correctly; fix that. 17 | """ 18 | from __future__ import absolute_import 19 | 20 | import re 21 | 22 | from . import khodemod 23 | 24 | 25 | # From PEP 263: https://www.python.org/dev/peps/pep-0263/ 26 | _PYTHON_ENCODING_RE = re.compile( 27 | r'^[ \t\v]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)') 28 | 29 | 30 | def _get_encoding(filename, text): 31 | """Determine the encoding of a python file, per PEP 263. 32 | 33 | Note that text may be either string or unicode, depending on whether we're 34 | reading/decoding or writing/encoding. 35 | """ 36 | if not filename.endswith('.py'): 37 | # Not implemented yet! 38 | return 'ascii' 39 | 40 | for line in text.splitlines()[:2]: 41 | match = _PYTHON_ENCODING_RE.search(line) 42 | if match: 43 | return match.group(1) 44 | return 'ascii' 45 | 46 | 47 | def encode(filename, text): 48 | encoding = _get_encoding(filename, text) 49 | try: 50 | return text.encode(encoding) 51 | except UnicodeEncodeError as e: 52 | # This one is unlikely, if we decoded successfully, but it's possible 53 | # we wrote some bad data. 54 | raise khodemod.FatalError( 55 | filename, 1, 56 | "Invalid %s data in file: %s" % (encoding, e)) 57 | 58 | 59 | def decode(filename, text): 60 | encoding = _get_encoding(filename, text) 61 | try: 62 | return text.decode(encoding) 63 | except UnicodeDecodeError as e: 64 | raise khodemod.FatalError( 65 | filename, 1, 66 | "Invalid %s data in file: %s" % (encoding, e)) 67 | -------------------------------------------------------------------------------- /tests/test_util.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import ast 4 | import unittest 5 | 6 | from slicker import util 7 | 8 | 9 | class DottedPrefixTest(unittest.TestCase): 10 | def test_dotted_starts_with(self): 11 | self.assertTrue(util.dotted_starts_with('abc', 'abc')) 12 | self.assertTrue(util.dotted_starts_with('abc.de', 'abc')) 13 | self.assertTrue(util.dotted_starts_with('abc.de', 'abc.de')) 14 | self.assertTrue(util.dotted_starts_with('abc.de.fg', 'abc')) 15 | self.assertTrue(util.dotted_starts_with('abc.de.fg', 'abc.de')) 16 | self.assertTrue(util.dotted_starts_with('abc.de.fg', 'abc.de.fg')) 17 | self.assertFalse(util.dotted_starts_with('abc', 'd')) 18 | self.assertFalse(util.dotted_starts_with('abc', 'ab')) 19 | self.assertFalse(util.dotted_starts_with('abc', 'abc.de')) 20 | self.assertFalse(util.dotted_starts_with('abc.de', 'ab')) 21 | self.assertFalse(util.dotted_starts_with('abc.de', 'abc.d')) 22 | self.assertFalse(util.dotted_starts_with('abc.de', 'abc.h')) 23 | 24 | def test_dotted_prefixes(self): 25 | self.assertItemsEqual( 26 | util.dotted_prefixes('abc'), 27 | ['abc']) 28 | self.assertItemsEqual( 29 | util.dotted_prefixes('abc.def'), 30 | ['abc', 'abc.def']) 31 | self.assertItemsEqual( 32 | util.dotted_prefixes('abc.def.ghi'), 33 | ['abc', 'abc.def', 'abc.def.ghi']) 34 | 35 | 36 | class NamesStartingWithTest(unittest.TestCase): 37 | def test_simple(self): 38 | self.assertEqual( 39 | set(util.names_starting_with('a', ast.parse('a\n'))), 40 | {'a'}) 41 | self.assertEqual( 42 | set(util.names_starting_with( 43 | 'a', ast.parse('a.b.c\n'))), 44 | {'a.b.c'}) 45 | self.assertEqual( 46 | set(util.names_starting_with( 47 | 'a', ast.parse('d.e.f\n'))), 48 | set()) 49 | 50 | self.assertEqual( 51 | set(util.names_starting_with( 52 | 'abc', ast.parse('abc.de\n'))), 53 | {'abc.de'}) 54 | self.assertEqual( 55 | set(util.names_starting_with( 56 | 'ab', ast.parse('abc.de\n'))), 57 | set()) 58 | 59 | self.assertEqual( 60 | set(util.names_starting_with( 61 | 'a', ast.parse('"a.b.c"\n'))), 62 | set()) 63 | self.assertEqual( 64 | set(util.names_starting_with( 65 | 'a', ast.parse('import a.b.c\n'))), 66 | set()) 67 | self.assertEqual( 68 | set(util.names_starting_with( 69 | 'a', ast.parse('b.c.a.b.c\n'))), 70 | set()) 71 | 72 | def test_in_context(self): 73 | self.assertEqual( 74 | set(util.names_starting_with('a', ast.parse( 75 | 'def abc():\n' 76 | ' if a.b == a.c:\n' 77 | ' return a.d(a.e + a.f)\n' 78 | 'abc(a.g)\n'))), 79 | {'a.b', 'a.c', 'a.d', 'a.e', 'a.f', 'a.g'}) 80 | -------------------------------------------------------------------------------- /slicker/cleanup.py: -------------------------------------------------------------------------------- 1 | """Suggestors relating to cleaning up after the contentful changes we've made. 2 | 3 | This file contains the suggestors (see khodemod.py) which pertain to cleaning 4 | up any non-contentful problems potentially introduced by the changes we've made 5 | -- things like fixing whitespace and sorting imports. These are used by 6 | slicker.slicker.make_fixes after it does all its other work. 7 | """ 8 | from __future__ import absolute_import 9 | 10 | import ast 11 | import difflib 12 | import os 13 | import sys 14 | 15 | from fix_includes import fix_python_imports 16 | 17 | from . import util 18 | from . import khodemod 19 | 20 | 21 | def remove_empty_files_suggestor(filename, body): 22 | """Suggestor to remove any empty files we leave behind. 23 | 24 | We also remove the file if it has only __future__ imports. If all that's 25 | left is docstrings, comments, and non-__future__ imports, we warn but don't 26 | remove it. (We ignore __init__.py files since those are often 27 | intentionally empty or kept only for some imports.) 28 | """ 29 | if os.path.basename(filename) == '__init__.py': 30 | # Ignore __init__.py files. 31 | return 32 | 33 | file_info = util.File(filename, body) 34 | 35 | has_docstrings_comments_or_imports = '#' in body 36 | for stmt in file_info.tree.body: 37 | if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Str): 38 | # A docstring. 39 | has_docstrings_comments_or_imports = True 40 | elif isinstance(stmt, ast.ImportFrom) and stmt.module == '__future__': 41 | # A __future__ import, which won't force us to keep the file. 42 | pass 43 | elif isinstance(stmt, (ast.Import, ast.ImportFrom)): 44 | # A non-__future__ import. 45 | has_docstrings_comments_or_imports = True 46 | else: 47 | # Some real code; we don't want to do anything. 48 | return 49 | 50 | # If we've gotten here, there's no "real code". 51 | if has_docstrings_comments_or_imports: 52 | yield khodemod.WarningInfo( 53 | filename, 0, "This file looks mostly empty; consider removing it.") 54 | else: 55 | # It's actually empty, so we can just go ahead and remove. 56 | yield khodemod.Patch(filename, body, None, 0, len(body)) 57 | 58 | 59 | def remove_leading_whitespace_suggestor(filename, body): 60 | """Suggestor to remove any leading whitespace we leave behind.""" 61 | lstripped_body = body.lstrip() 62 | if lstripped_body != body: 63 | whitespace_len = len(body) - len(lstripped_body) 64 | yield khodemod.Patch(filename, body[:whitespace_len], '', 65 | 0, whitespace_len) 66 | 67 | 68 | class _FakeOptions(object): 69 | """A fake `options` object to pass in to fix_python_imports.""" 70 | def __init__(self, project_root): 71 | self.safe_headers = True 72 | self.root = project_root 73 | 74 | 75 | def import_sort_suggestor(project_root): 76 | """Suggestor to fix up imports in a file.""" 77 | fix_imports_flags = _FakeOptions(project_root) 78 | 79 | def suggestor(filename, body): 80 | """`filename` relative to project_root.""" 81 | # TODO(benkraft): merge this with the import-adding, so we just show 82 | # one diff to add in the right place, unless there is additional 83 | # sorting to do. 84 | # Now call out to fix_python_imports to do the import-sorting 85 | change_record = fix_python_imports.ChangeRecord('fake_file.py') 86 | 87 | # A modified version of fix_python_imports.GetFixedFile 88 | # NOTE: fix_python_imports needs the rootdir to be on the 89 | # path so it can figure out third-party deps correctly. 90 | # (That's in addition to having it be in FakeOptions, sigh.) 91 | try: 92 | sys.path.insert(0, os.path.abspath(project_root)) 93 | file_line_infos = fix_python_imports.ParseOneFile( 94 | body, change_record) 95 | fixed_lines = fix_python_imports.FixFileLines( 96 | change_record, file_line_infos, fix_imports_flags) 97 | finally: 98 | del sys.path[0] 99 | 100 | if fixed_lines is None: 101 | return 102 | fixed_body = ''.join(['%s\n' % line for line in fixed_lines 103 | if line is not None]) 104 | if fixed_body == body: 105 | return 106 | 107 | diffs = difflib.SequenceMatcher(None, body, fixed_body).get_opcodes() 108 | for op, i1, i2, j1, j2 in diffs: 109 | if op != 'equal': 110 | yield khodemod.Patch(filename, 111 | body[i1:i2], fixed_body[j1:j2], i1, i2) 112 | 113 | return suggestor 114 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Slicker: A Tool for Moving Things in Python 2 | ------------------------------------------- 3 | 4 | [![Build Status](https://travis-ci.org/Khan/slicker.svg?branch=master)](https://travis-ci.org/Khan/slicker) 5 | 6 | If you've ever tried to move a function or class in python, you'll find it's 7 | kind of a pain: you have to not only move the definition (and its imports, 8 | etc.) but also update references across the codebase. Slicker is a tool for 9 | doing just that! 10 | 11 | **Note:** At this time Slicker is [Python 2 only](https://github.com/Khan/slicker/issues/21). 12 | 13 | ## Installation 14 | 15 | `pip2 install slicker` 16 | 17 | ## Usage 18 | 19 | To move a function `myfunc` defined in `foo/bar.py` to `foo/baz.py`: 20 | ``` 21 | slicker foo.bar.myfunc foo.baz.myfunc 22 | ``` 23 | 24 | The same syntax works if `myfunc` is instead a constant or class (although I 25 | sure hope you didn't name a class `myfunc`!). It also works if you want to 26 | change the name of `myfunc`: 27 | ``` 28 | slicker foo.bar.myfunc foo.bar.new_name_for_myfunc 29 | ``` 30 | (And you can also make both changes at once, in the natural way.) 31 | 32 | To move an entire module `foo/bar.py` to `foo/baz.py` you can do similarly: 33 | ``` 34 | slicker foo.bar foo.baz 35 | ``` 36 | or use filenames like: 37 | ``` 38 | slicker foo/bar.py foo/baz.py 39 | ``` 40 | 41 | You can also move a symbol into an existing module, or a module into an 42 | existing directory, just like `mv`. So this is equivalent to the first 43 | example: 44 | ``` 45 | slicker foo.bar.myfunc foo.baz 46 | ``` 47 | And to move `foo/bar.py` to a new file `newfoo/bar.py` in an existing directory 48 | `newfoo/`, you could do 49 | ``` 50 | slicker foo.bar newfoo # (or slicker foo/bar.py newfoo/) 51 | ``` 52 | Using this syntax, you can also specify multiple things to move, so you could 53 | move both `foo/bar.py` and `foo/baz.py` to `newfoo/` with 54 | ``` 55 | slicker foo/bar.py foo/baz.py newfoo/ 56 | ``` 57 | 58 | You can tell slicker to use an alias when adding imports using `-a`/`--alias`: 59 | ``` 60 | slicker foo.bar.myfunc foo.baz.myfunc --alias baz 61 | ``` 62 | in which case slicker will add `from foo import baz` everywhere instead of 63 | `import foo.baz`. (You could also have used `--alias foobaz` in which case 64 | we would have done `import foo.baz as foobaz`.) 65 | 66 | If you prefer to move the actual definition yourself, and just have slicker 67 | update the references, you can pass `--no-automove`. It's probably best to run 68 | `slicker` after doing said move. 69 | 70 | For a full list of options, run `slicker --help`. 71 | 72 | 73 | ## Frequently and Infrequently Asked Questions 74 | 75 | ### What does slicker mean if it says "This import may be used implicitly."? 76 | 77 | If you do `import foo.bar`, and some other file (perhaps another one you 78 | import) does `import foo.baz`, then your `foo` now also has a `foo.baz`, and so 79 | you can do `foo.baz.func()` with impunity, even though no import in your file 80 | directly mentions that module. (This is because `foo` in both files refers to 81 | the same object -- a.k.a. `sys.modules['foo']` -- and so when the other file 82 | does `import foo.baz` it attaches `baz` to that shared object.) So if you've 83 | asked slicker to move `foo.bar` to `newfoo.bar`, when updating this file, it 84 | would like to replace the `import foo.bar` with `import newfoo.bar`, but it 85 | can't -- you're actually still using the import. So it will warn you of this 86 | case, and let you sort things out by hand. 87 | 88 | ### Slicker left me with a bunch of misindented or long lines! 89 | 90 | Yep, we don't fix these correctly (yet). Your linter should tell you what to 91 | fix, though. 92 | 93 | ### Why is it called slicker? 94 | 95 | Because pythons slither to move around, but this way is, uh, slicker. Which is 96 | to say: it seemed like a good idea at the time and as far as I could tell the 97 | name wasn't already taken. 98 | 99 | ### How does it work? 100 | 101 | See the [blog post](http://engineering.khanacademy.org/posts/slicker.htm) for 102 | an overview. If that's not enough, bug the authors or read the source! 103 | 104 | ### Why don't you just use [PyCharm](https://www.jetbrains.com/pycharm/) or [rope](https://github.com/python-rope/rope)? 105 | 106 | Good question -- we tried! Both are great projects and do a lot of things 107 | slicker doesn't; if they work for you then definitely use them. But for us, 108 | they were a little buggy and didn't fit our workflow. For more details, see 109 | the [blog post](http://engineering.khanacademy.org/posts/slicker.htm). 110 | 111 | ### Why don't you just use `codemod` or `sed`/`perl`? 112 | 113 | Good question -- we tried! But it takes a lot of gluing things together to 114 | figure out all the right references to fix up in each file. And there's 115 | basically no hope of doing the right thing when fixing up string-references. 116 | We needed something that knew what python imports mean and could handle their 117 | special cases. 118 | 119 | ## Changelog 120 | 121 | ### 0.9.3 122 | 123 | - Fix description on PyPI, again 124 | 125 | ### 0.9.2 126 | 127 | - Fix description on PyPI 128 | 129 | ### 0.9.1 130 | 131 | - Handle relative imports correctly 132 | - Lots of internal refactoring 133 | 134 | ### 0.9 135 | 136 | - Initial release to PyPI 137 | -------------------------------------------------------------------------------- /tests/test_replacement.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import os 4 | 5 | from slicker import model 6 | from slicker import slicker 7 | 8 | import base 9 | 10 | 11 | class ReplaceInStringTest(base.TestBase): 12 | def assert_(self, old_module, new_module, old_string, new_string, 13 | alias=None): 14 | """Assert that a file that imports old_module rewrites its strings too. 15 | 16 | We create a temp file that imports old_module as alias, and then 17 | defines a docstring with the contents old_string. We then rename 18 | old_module to new_module, and make sure that our temp file not 19 | only has the import renamed, it has the string renamed as well. 20 | """ 21 | self.write_file(old_module.replace('.', os.sep) + '.py', '# A file') 22 | self.write_file('in.py', '"""%s"""\n%s\n\n_ = %s.myfunc()\n' 23 | % (old_string, 24 | model.Import( 25 | old_module, alias or old_module, 26 | 'absolute', None, None).import_stmt(), 27 | alias or old_module)) 28 | 29 | slicker.make_fixes([old_module], new_module, 30 | project_root=self.tmpdir, automove=False) 31 | self.assertFalse(self.error_output) 32 | 33 | expected = ('"""%s"""\nimport %s\n\n_ = %s.myfunc()\n' 34 | % (new_string, new_module, new_module)) 35 | with open(self.join('in.py')) as f: 36 | actual = f.read() 37 | self.assertMultiLineEqual(expected, actual) 38 | 39 | def test_simple(self): 40 | self.assert_('foo', 'bar.baz', "foo.myfunc", "bar.baz.myfunc") 41 | 42 | def test_word(self): 43 | self.assert_('exercise', 'foo.bar', 44 | ("I will exercise `exercise.myfunc()` in exercise.py. " 45 | "It will not rename 'exercise' and exercises " 46 | "not-renaming content_exercise or exercise_util but " 47 | "does rename `exercise`."), 48 | ("I will exercise `foo.bar.myfunc()` in foo/bar.py. " 49 | "It will not rename 'exercise' and exercises " 50 | "not-renaming content_exercise or exercise_util but " 51 | "does rename `foo.bar`.")) 52 | 53 | def test_word_via_as(self): 54 | self.assert_('qux', 'foo.bar', 55 | ("I will exercise `exercise.myfunc()` in exercise.py. " 56 | "It will not rename 'exercise' and exercises " 57 | "not-renaming content_exercise or exercise_util but " 58 | "does rename `exercise`. And what about " 59 | "qux.myfunc()? Or just 'qux'? `qux`?"), 60 | ("I will exercise `foo.bar.myfunc()` in exercise.py. " 61 | "It will not rename 'exercise' and exercises " 62 | "not-renaming content_exercise or exercise_util but " 63 | "does rename `foo.bar`. And what about " 64 | "foo.bar.myfunc()? Or just 'qux'? `foo.bar`?"), 65 | alias='exercise') # file reads 'import qux as exercise' 66 | 67 | def test_word_via_from(self): 68 | self.assert_('qux.exercise', 'foo.bar', 69 | ("I will exercise `exercise.myfunc()` in exercise.py. " 70 | "It will not rename 'exercise' and exercises " 71 | "not-renaming content_exercise or exercise_util but " 72 | "does rename `exercise`. And what about " 73 | "qux.exercise.myfunc()? Or just 'qux.exercise'? " 74 | "`qux.exercise`?"), 75 | ("I will exercise `foo.bar.myfunc()` in exercise.py. " 76 | "It will not rename 'exercise' and exercises " 77 | "not-renaming content_exercise or exercise_util but " 78 | "does rename `foo.bar`. And what about " 79 | "foo.bar.myfunc()? Or just 'foo.bar'? " 80 | "`foo.bar`?"), 81 | alias='exercise') # file reads 'from qux import exercise' 82 | 83 | def test_module_and_alias_the_same(self): 84 | self.assert_('exercise.exercise', 'foo.bar', 85 | ("I will exercise `exercise.myfunc()` in exercise.py. " 86 | "It will not rename 'exercise' and exercises " 87 | "not-renaming content_exercise or exercise_util or " 88 | "`exercise`. But what about exercise.exercise.myfunc()?" 89 | "Or just 'exercise.exercise'? `exercise.exercise`?"), 90 | ("I will exercise `exercise.myfunc()` in exercise.py. " 91 | "It will not rename 'exercise' and exercises " 92 | "not-renaming content_exercise or exercise_util or " 93 | "`exercise`. But what about foo.bar.myfunc()?" 94 | "Or just 'foo.bar'? `foo.bar`?"), 95 | alias='exercise') # 'from exercise import exercise' 96 | 97 | def test_does_not_rename_files_in_other_dirs(self): 98 | self.assert_('exercise', 'foo.bar', 99 | "otherdir/exercise.py", "otherdir/exercise.py") 100 | 101 | def test_does_not_rename_html_files(self): 102 | # Regular-english-word case. 103 | self.assert_('exercise', 'foo.bar', 104 | "otherdir/exercise.html", "otherdir/exercise.html") 105 | # Obviously-a-symbol case. 106 | self.assert_('exercise_util', 'foo.bar', 107 | "dir/exercise_util.html", "dir/exercise_util.html") 108 | 109 | def test_renames_complex_strings_but_not_simple_ones(self): 110 | self.assert_('exercise', 'foo.bar', 111 | "I like 'exercise'", "I like 'exercise'") 112 | self.assert_('exercise_util', 'foo.bar', 113 | "I like 'exercise_util'", "I like 'foo.bar'") 114 | 115 | def test_renames_simple_strings_when_it_is_the_whole_string(self): 116 | self.assert_('exercise', 'foo.bar', 117 | "exercise", "foo.bar") 118 | 119 | def test_word_at_the_end_of_a_sentence(self): 120 | # Regular-english-word case. 121 | self.assert_('exercise', 'foo.bar', 122 | "I need some exercise. Yes, exercise.", 123 | "I need some exercise. Yes, exercise.") 124 | # Obviously-a-symbol case. 125 | self.assert_('exercise_util', 'foo.bar', 126 | "I need to look at exercise_util. Yes, exercise_util.", 127 | "I need to look at foo.bar. Yes, foo.bar.") 128 | -------------------------------------------------------------------------------- /tests/test_cleanup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import os 4 | 5 | from slicker import slicker 6 | 7 | import base 8 | 9 | 10 | class RemoveEmptyFilesSuggestorTest(base.TestBase): 11 | def test_removes_remaining_whitespace(self): 12 | self.write_file('foo.py', 13 | ('\n\n\n \n\n \n' 14 | 'import bar\n\n\n' 15 | 'def myfunc():\n' 16 | ' return bar.unrelated_function()\n')) 17 | slicker.make_fixes(['foo.myfunc'], 'newfoo.myfunc', 18 | project_root=self.tmpdir) 19 | self.assertFileIsNot('foo.py') 20 | self.assertFileIs('newfoo.py', 21 | ('from __future__ import absolute_import\n\n' 22 | 'import bar\n\n\n' 23 | 'def myfunc():\n' 24 | ' return bar.unrelated_function()\n')) 25 | self.assertFalse(self.error_output) 26 | 27 | def test_removes_remaining_future_import(self): 28 | self.write_file('foo.py', 29 | ('from __future__ import absolute_import\n\n' 30 | 'import bar\n\n\n' 31 | 'def myfunc():\n' 32 | ' return bar.unrelated_function()\n')) 33 | slicker.make_fixes(['foo.myfunc'], 'newfoo.myfunc', 34 | project_root=self.tmpdir) 35 | self.assertFileIsNot('foo.py') 36 | self.assertFileIs('newfoo.py', 37 | ('from __future__ import absolute_import\n\n' 38 | 'import bar\n\n\n' 39 | 'def myfunc():\n' 40 | ' return bar.unrelated_function()\n')) 41 | self.assertFalse(self.error_output) 42 | 43 | def test_warns_remaining_import(self): 44 | self.write_file('foo.py', 45 | ('from __future__ import absolute_import\n\n' 46 | 'import asdf # @UnusedImport\n' 47 | 'import bar\n\n\n' 48 | 'def myfunc():\n' 49 | ' return bar.unrelated_function()\n')) 50 | slicker.make_fixes(['foo.myfunc'], 'newfoo.myfunc', 51 | project_root=self.tmpdir) 52 | self.assertFileIs('foo.py', 53 | ('from __future__ import absolute_import\n\n' 54 | 'import asdf # @UnusedImport\n')) 55 | self.assertFileIs('newfoo.py', 56 | ('from __future__ import absolute_import\n\n' 57 | 'import bar\n\n\n' 58 | 'def myfunc():\n' 59 | ' return bar.unrelated_function()\n')) 60 | self.assertEqual( 61 | self.error_output, 62 | [('WARNING:Not removing import with @Nolint.' 63 | '\n on foo.py:3 --> import asdf # @UnusedImport'), 64 | ('WARNING:This file looks mostly empty; consider removing it.' 65 | '\n on foo.py:1 --> from __future__ import absolute_import')]) 66 | 67 | def test_warns_remaining_comment(self): 68 | self.write_file('foo.py', 69 | ('# this comment is very important!!!!!111\n' 70 | 'from __future__ import absolute_import\n\n' 71 | 'import bar\n\n\n' 72 | 'def myfunc():\n' 73 | ' return bar.unrelated_function()\n')) 74 | slicker.make_fixes(['foo.myfunc'], 'newfoo.myfunc', 75 | project_root=self.tmpdir) 76 | self.assertFileIs('foo.py', 77 | ('# this comment is very important!!!!!111\n' 78 | 'from __future__ import absolute_import\n\n')) 79 | self.assertFileIs('newfoo.py', 80 | ('from __future__ import absolute_import\n\n' 81 | 'import bar\n\n\n' 82 | 'def myfunc():\n' 83 | ' return bar.unrelated_function()\n')) 84 | self.assertEqual( 85 | self.error_output, 86 | ['WARNING:This file looks mostly empty; consider removing it.' 87 | '\n on foo.py:1 --> # this comment is very important!!!!!111']) 88 | 89 | def test_warns_remaining_docstring(self): 90 | self.write_file('foo.py', 91 | ('"""This file frobnicates the doodad."""\n' 92 | 'from __future__ import absolute_import\n\n' 93 | 'import bar\n\n\n' 94 | 'def myfunc():\n' 95 | ' return bar.unrelated_function()\n')) 96 | slicker.make_fixes(['foo.myfunc'], 'newfoo.myfunc', 97 | project_root=self.tmpdir) 98 | self.assertFileIs('foo.py', 99 | ('"""This file frobnicates the doodad."""\n' 100 | 'from __future__ import absolute_import\n\n')) 101 | self.assertFileIs('newfoo.py', 102 | ('from __future__ import absolute_import\n\n' 103 | 'import bar\n\n\n' 104 | 'def myfunc():\n' 105 | ' return bar.unrelated_function()\n')) 106 | self.assertEqual( 107 | self.error_output, 108 | ['WARNING:This file looks mostly empty; consider removing it.' 109 | '\n on foo.py:1 --> """This file frobnicates the doodad."""']) 110 | 111 | def test_warns_remaining_code(self): 112 | self.write_file('foo.py', 113 | ('from __future__ import absolute_import\n\n' 114 | 'baz = 1\n\n' 115 | 'import bar\n\n\n' 116 | 'def myfunc():\n' 117 | ' return bar.unrelated_function()\n')) 118 | slicker.make_fixes(['foo.myfunc'], 'newfoo.myfunc', 119 | project_root=self.tmpdir) 120 | self.assertFileIs('foo.py', 121 | ('from __future__ import absolute_import\n\n' 122 | 'baz = 1\n\n')) 123 | self.assertFileIs('newfoo.py', 124 | ('from __future__ import absolute_import\n\n' 125 | 'import bar\n\n\n' 126 | 'def myfunc():\n' 127 | ' return bar.unrelated_function()\n')) 128 | self.assertFalse(self.error_output) 129 | 130 | 131 | class ImportSortTest(base.TestBase): 132 | def test_third_party_sorting(self): 133 | self.copy_file('third_party_sorting_in.py') 134 | 135 | os.mkdir(self.join('third_party')) 136 | for f in ('mycode1.py', 'mycode2.py', 137 | 'third_party/__init__.py', 'third_party/slicker.py'): 138 | with open(self.join(f), 'w') as f: 139 | print >>f, '# A file' 140 | 141 | slicker.make_fixes(['third_party_sorting_in'], 'out', 142 | project_root=self.tmpdir) 143 | 144 | with open(self.join('out.py')) as f: 145 | actual = f.read() 146 | with open('testdata/third_party_sorting_out.py') as f: 147 | expected = f.read() 148 | self.assertMultiLineEqual(expected, actual) 149 | self.assertFalse(self.error_output) 150 | -------------------------------------------------------------------------------- /slicker/moves.py: -------------------------------------------------------------------------------- 1 | """The suggestors for moving things around.""" 2 | from __future__ import absolute_import 3 | 4 | import ast 5 | import os 6 | import stat 7 | 8 | from . import khodemod 9 | from . import util 10 | 11 | 12 | def _add_init_py(filename): 13 | """Make sure __init__.py exists in every dir from dir(filename)..root.""" 14 | dirname = os.path.dirname(filename) 15 | # We do not put an __init__.py in the rootdir itself; it doesn't need one. 16 | while dirname: 17 | yield khodemod.Patch(os.path.join(dirname, '__init__.py'), 18 | None, '', 0, 0) 19 | dirname = os.path.dirname(dirname) 20 | 21 | 22 | def move_module_suggestor(project_root, old_fullname, new_fullname): 23 | """Move a module from old_fullname to new_fullname. 24 | 25 | old_fullname and new_fullname should be dotted names. Their paths 26 | are taken to be relative to project_root. The destination must 27 | not already exist. 28 | """ 29 | def filename_for(mod): 30 | return os.path.join(project_root, util.filename_for_module_name(mod)) 31 | 32 | def suggestor(filename, body): 33 | new_filename = util.filename_for_module_name(new_fullname) 34 | old_pathname = filename_for(old_fullname) 35 | new_pathname = filename_for(new_fullname) 36 | # We only need to operate on the old file (although we'll generate a 37 | # patch for the new one as well). Caller should ensure this but we 38 | # check to be safe. 39 | if (os.path.normpath(os.path.join(project_root, filename)) != 40 | os.path.normpath(old_pathname)): 41 | return 42 | # We don't expect new_pathname to exist, but we allow it if 43 | # it's an empty file. This can happen with __init__.py 44 | # files, which we create sometimes. 45 | assert (not os.path.exists(new_pathname) or 46 | os.stat(new_pathname).st_size == 0), new_pathname 47 | 48 | # Make sure new_filename has the same permissions as the old. 49 | # This is most important to keep the executable bit. 50 | # TODO(csilvers): don't have low-level file ops here. 51 | # TODO(csilvers): also change uid/gid? 52 | file_permissions = stat.S_IMODE(os.stat(old_pathname).st_mode) 53 | 54 | yield khodemod.Patch(filename, body, None, 0, len(body)) 55 | yield khodemod.Patch(new_filename, None, body, 0, 0, 56 | file_permissions=file_permissions) 57 | 58 | for patch in _add_init_py(new_filename): 59 | yield patch 60 | 61 | return suggestor 62 | 63 | 64 | def move_symbol_suggestor(project_root, old_fullname, new_fullname): 65 | """Move a symbol from old_fullname to new_fullname. 66 | 67 | old_fullname and new_fullname should both be dotted names of 68 | the form module.symbol. The destination fullname should not 69 | already exist (though the destination module may). 70 | """ 71 | def suggestor(filename, body): 72 | (old_module, old_symbol) = old_fullname.rsplit('.', 1) 73 | (new_module, new_symbol) = new_fullname.rsplit('.', 1) 74 | 75 | # We only need to operate on the old file (although we'll generate a 76 | # patch for the new one as well). Caller should ensure this but we 77 | # check to be safe. 78 | if filename != util.filename_for_module_name(old_module): 79 | return 80 | 81 | file_info = util.File(filename, body) 82 | 83 | # Find where old_fullname is defined in old_module. 84 | # TODO(csilvers): traverse try/except, for, etc, and complain 85 | # if we see the symbol defined inside there. 86 | # TODO(csilvers): look for ast.AugAssign and complain if our 87 | # symbol is in there. 88 | old_module_toplevel = util.toplevel_names(file_info) 89 | if old_symbol not in old_module_toplevel: 90 | raise khodemod.FatalError(filename, 0, 91 | "Could not find symbol '%s' in '%s': " 92 | "maybe it's in a try/finally or if?" 93 | % (old_symbol, old_module)) 94 | 95 | # Now get the startpos and endpos of this symbol's definition. 96 | node_to_move = old_module_toplevel[old_symbol] 97 | start, end = util.get_area_for_ast_node( 98 | node_to_move, file_info, include_previous_comments=True) 99 | definition_region = body[start:end] 100 | 101 | # Decide what text to add, which may require a rename. 102 | if old_symbol == new_symbol: 103 | new_definition_region = definition_region 104 | else: 105 | # Find the token with the name of the symbol, and update it. 106 | if isinstance(node_to_move, (ast.FunctionDef, ast.ClassDef)): 107 | for token in file_info.tokens.get_tokens(node_to_move): 108 | if token.string in ('def', 'class'): 109 | break 110 | else: 111 | raise khodemod.FatalError( 112 | filename, 0, 113 | "Could not find symbol '%s' in " 114 | "'%s': maybe it's defined weirdly?" 115 | % (old_symbol, old_module)) 116 | # We want the token after the def. 117 | name_token = file_info.tokens.next_token(token) 118 | else: # isinstance(node_to_move, ast.Assign) 119 | # The name should be a single token, if we get here. 120 | name_token, = list(file_info.tokens.get_tokens( 121 | node_to_move.targets[0])) 122 | 123 | if name_token.string != old_symbol: 124 | raise khodemod.FatalError(filename, 0, 125 | "Could not find symbol '%s' in " 126 | "'%s': maybe it's defined weirdly?" 127 | % (old_symbol, old_module)) 128 | new_definition_region = ( 129 | body[start:name_token.startpos] + new_symbol 130 | + body[name_token.endpos:end]) 131 | 132 | if old_module == new_module: 133 | # Just patch the module in place. 134 | yield khodemod.Patch( 135 | filename, definition_region, new_definition_region, start, end) 136 | else: 137 | # Remove the region from the old file. 138 | # (If we've removed the remainder of the file, 139 | # _remove_empty_files_suggestor will clean up.) 140 | yield khodemod.Patch(filename, definition_region, '', start, end) 141 | 142 | # Add the region to the new file. 143 | new_filename = util.filename_for_module_name(new_module) 144 | new_file_body = khodemod.read_file( 145 | project_root, new_filename) or '' 146 | 147 | # Mess about with leading newlines. First, we strip any existing 148 | # ones. Then, if we are adding to an existing file, we add enough 149 | # to satisfy pep8. 150 | new_definition_region = new_definition_region.lstrip('\r\n') 151 | if new_file_body: 152 | current_newlines = ( 153 | len(new_file_body) - len(new_file_body.rstrip('\r\n')) 154 | + len(new_definition_region) 155 | - len(new_definition_region.lstrip('\r\n'))) 156 | if current_newlines < 3: 157 | new_definition_region = ('\n' * (3 - current_newlines) 158 | + new_definition_region) 159 | 160 | # Now we need to add the new symbol to new_module. 161 | # TODO(benkraft): Allow, as an option, adding it after a specific 162 | # other symbol in new_module. 163 | yield khodemod.Patch(new_filename, '', new_definition_region, 164 | len(new_file_body), len(new_file_body)) 165 | 166 | # TODO(benkraft): Fix up imports in the new and old modules. 167 | 168 | new_filename = util.filename_for_module_name(new_module) 169 | for patch in _add_init_py(new_filename): 170 | yield patch 171 | 172 | return suggestor 173 | -------------------------------------------------------------------------------- /slicker/util.py: -------------------------------------------------------------------------------- 1 | """Utilities for interacting with the AST, files, and names.""" 2 | from __future__ import absolute_import 3 | 4 | import ast 5 | import os 6 | import tokenize 7 | 8 | import asttokens 9 | 10 | from . import khodemod 11 | from . import unicode_util 12 | 13 | 14 | def filename_for_module_name(module_name): 15 | """filename is relative to a sys.path entry, such as your project-root.""" 16 | return '%s.py' % module_name.replace('.', os.sep) 17 | 18 | 19 | def module_name_for_filename(filename): 20 | """filename is relative to a sys.path entry, such as your project-root.""" 21 | return os.path.splitext(filename)[0].replace(os.sep, '.') 22 | 23 | 24 | class File(object): 25 | """Represents information about a file. 26 | 27 | TODO(benkraft): Also cache things like _compute_all_imports. 28 | """ 29 | def __init__(self, filename, body): 30 | """filename is relative to the value of --root.""" 31 | self.filename = filename 32 | self.body = body 33 | self._tree = None # computed lazily 34 | self._tokens = None # computed lazily 35 | 36 | @property 37 | def tree(self): 38 | """The AST for the file. Computed lazily on first use.""" 39 | if self._tree is None: 40 | try: 41 | # ast.parse would really prefer to run on bytes. 42 | # Luckily we ignore all of the (useless) line/col 43 | # information in the AST nodes -- we get it via 44 | # asttokens instead -- so we don't have to worry 45 | # about the fact that these will be byte offsets. 46 | self._tree = ast.parse( 47 | unicode_util.encode(self.filename, self.body)) 48 | except SyntaxError as e: 49 | raise khodemod.FatalError(self.filename, 0, 50 | "Couldn't parse this file: %s" % e) 51 | return self._tree 52 | 53 | @property 54 | def tokens(self): 55 | """The asttokens.ASTTokens mapping for the file. 56 | 57 | This is computed lazily on first use, and is somewhat slow to compute, 58 | so we try to only use it when we need to (i.e. on files we are 59 | editing). 60 | """ 61 | if self._tokens is None: 62 | self._tokens = asttokens.ASTTokens(self.body, tree=self.tree) 63 | return self._tokens 64 | 65 | def __repr__(self): 66 | return "File(filename=%r)" % self.filename 67 | 68 | 69 | def is_newline(token): 70 | # I think this is equivalent to doing 71 | # token.type in (tokenize.NEWLINE, tokenize.NL) 72 | # TODO(benkraft): We don't really handle files with windows newlines 73 | # correctly -- any newlines we add will be wrong. Do the right thing. 74 | return token.string in ('\n', '\r\n') 75 | 76 | 77 | def get_area_for_ast_node(node, file_info, include_previous_comments): 78 | """Return the start/end character offsets of the input ast-node + friends. 79 | 80 | We include every line that node spans, as well as their ending newlines, 81 | though if the last line has a semicolon we end at the semicolon. 82 | 83 | If include_previous_comments is True, we also include all comments 84 | and newlines that directly precede the given node. 85 | """ 86 | toks = list(file_info.tokens.get_tokens(node, include_extra=True)) 87 | first_tok = toks[0] 88 | last_tok = toks[-1] 89 | 90 | if include_previous_comments: 91 | for istart in xrange(first_tok.index - 1, -1, -1): 92 | tok = file_info.tokens.tokens[istart] 93 | if (tok.string and not tok.type == tokenize.COMMENT 94 | and not tok.string.isspace()): 95 | break 96 | else: 97 | istart = -1 98 | else: 99 | for istart in xrange(first_tok.index - 1, -1, -1): 100 | tok = file_info.tokens.tokens[istart] 101 | if tok.string and (is_newline(tok) or not tok.string.isspace()): 102 | break 103 | else: 104 | istart = -1 105 | 106 | # We don't want the *very* earliest newline before us to be 107 | # part of our context: it's ending the previous statement. 108 | if istart >= 0 and is_newline(file_info.tokens.tokens[istart + 1]): 109 | istart += 1 110 | 111 | prev_tok_endpos = (file_info.tokens.tokens[istart].endpos 112 | if istart >= 0 else 0) 113 | 114 | # Figure out how much of the last line to keep. 115 | for tok in file_info.tokens.tokens[last_tok.index + 1:]: 116 | if tok.type == tokenize.COMMENT: 117 | last_tok = tok 118 | elif is_newline(tok): 119 | last_tok = tok 120 | break 121 | else: 122 | break 123 | 124 | return (prev_tok_endpos, last_tok.endpos) 125 | 126 | 127 | def toplevel_names(file_info): 128 | """Return a dict of name -> AST node with toplevel definitions in the file. 129 | 130 | This includes function definitions, class definitions, and constants. 131 | """ 132 | # TODO(csilvers): traverse try/except, for, etc, and complain 133 | # if we see the symbol defined inside there. 134 | # TODO(benkraft): Figure out how to handle ast.AugAssign (+=) and multiple 135 | # assignments like `a, b = x, y`. 136 | retval = {} 137 | for top_level_stmt in file_info.tree.body: 138 | if isinstance(top_level_stmt, (ast.FunctionDef, ast.ClassDef)): 139 | retval[top_level_stmt.name] = top_level_stmt 140 | elif isinstance(top_level_stmt, ast.Assign): 141 | # Ignore assignments like 'a, b = x, y', and 'x.y = 5' 142 | if (len(top_level_stmt.targets) == 1 and 143 | isinstance(top_level_stmt.targets[0], ast.Name)): 144 | retval[top_level_stmt.targets[0].id] = top_level_stmt 145 | return retval 146 | 147 | 148 | def dotted_starts_with(string, prefix): 149 | """Like string.startswith(prefix), but in the dotted sense. 150 | 151 | That is, abc is a prefix of abc.de but not abcde.ghi. 152 | """ 153 | return prefix == string or string.startswith('%s.' % prefix) 154 | 155 | 156 | def dotted_prefixes(string, proper_only=False): 157 | """All prefixes of string, in the dotted sense. 158 | 159 | That is, all strings p such that dotted_starts_with(string, p), in order 160 | from shortest to longest. 161 | 162 | If proper_prefixes is True, do not include string itself. 163 | """ 164 | string_parts = string.split('.') 165 | for i in xrange(len(string_parts) - (1 if proper_only else 0)): 166 | yield '.'.join(string_parts[:i + 1]) 167 | 168 | 169 | def name_for_node(node): 170 | """Return the dotted name of an AST node, if there's a reasonable one. 171 | 172 | A 'name' is just a dotted-symbol, e.g. `myvar` or `myvar.mystruct.myprop`. 173 | 174 | This only does anything interesting for Name and Attribute, and for 175 | Attribute only if it's like a.b.c, not (a + b).c. 176 | """ 177 | if isinstance(node, ast.Name): 178 | return node.id 179 | elif isinstance(node, ast.Attribute): 180 | value = name_for_node(node.value) 181 | if value: 182 | return '%s.%s' % (value, node.attr) 183 | 184 | 185 | def all_names(root): 186 | """All names in the file. 187 | 188 | A 'name' is just a dotted-symbol, e.g. `myvar` or `myvar.mystruct.myprop`. 189 | 190 | Does not include imports or string references or anything else funky like 191 | that, and only returns the "biggest" possible name -- if you reference 192 | a.b.c we won't include a.b. 193 | 194 | Returns pairs (name, node) 195 | """ 196 | name = name_for_node(root) 197 | if name: 198 | return {(name, root)} 199 | else: 200 | return {(name, node) 201 | for child in ast.iter_child_nodes(root) 202 | for name, node in all_names(child)} 203 | 204 | 205 | def names_starting_with(prefix, ast_node): 206 | """Returns all dotted names in the given file beginning with 'prefix'. 207 | 208 | Does not include imports or string references or anything else funky like 209 | that. "Beginning with prefix" in the dotted sense (see 210 | dotted_starts_with). 211 | 212 | Returns a dict of name -> list of AST nodes. 213 | """ 214 | retval = {} 215 | for name, node in all_names(ast_node): 216 | if dotted_starts_with(name, prefix): 217 | retval.setdefault(name, []).append(node) 218 | return retval 219 | -------------------------------------------------------------------------------- /tests/test_inputs.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import unittest 4 | 5 | from slicker import inputs 6 | 7 | import base 8 | 9 | 10 | class OneInputTest(base.TestBase): 11 | def setUp(self): 12 | super(OneInputTest, self).setUp() 13 | self.write_file('foo.py', 'def myfunc(): return 4\n') 14 | self.write_file('dir/__init__.py', '') 15 | self.write_file('dir/subdir/__init__.py', '') 16 | 17 | def _assert(self, old_fullname, new_fullname, expected): 18 | actual = inputs.expand_and_normalize(self.tmpdir, 19 | [old_fullname], new_fullname) 20 | self.assertItemsEqual(expected, actual) 21 | 22 | def assert_fails(self, old_fullname, new_fullname, error_text): 23 | with self.assertRaises(ValueError) as e: 24 | inputs.expand_and_normalize(self.tmpdir, 25 | [old_fullname], new_fullname) 26 | self.assertEqual(error_text, str(e.exception)) 27 | 28 | def test_self_move(self): 29 | error = 'Cannot move an object (%s) to itself' 30 | self.assert_fails('foo', 'foo', error % 'foo') 31 | self.assert_fails('foo.myfunc', 'foo.myfunc', error % 'foo.myfunc') 32 | self.assert_fails('foo', self.join('foo.py'), error % 'foo') 33 | self.assert_fails(self.join('dir/subdir/'), 'dir.subdir', 34 | error % 'dir.subdir') 35 | 36 | def test_two_modules(self): 37 | self._assert('foo', 'bar', 38 | [('foo', 'bar', False)]) 39 | 40 | def test_module_and_file(self): 41 | self._assert('foo', self.join('bar.py'), 42 | [('foo', 'bar', False)]) 43 | 44 | def test_module_and_file_in_directory(self): 45 | self._assert('foo', self.join('bar/baz.py'), 46 | [('foo', 'bar.baz', False)]) 47 | 48 | def test_two_modules_both_existing(self): 49 | error = 'Cannot use slicker to merge modules (bar already exists)' 50 | self.write_file('bar.py', 'def myfunc(): return 4\n') 51 | self.assert_fails('foo', 'bar', error) 52 | 53 | def test_non_existing_source_module(self): 54 | error = "Cannot figure out what 'baz' is: module or package not found" 55 | self.assert_fails('baz', 'bar', error) 56 | 57 | def test_non_existing_source_file(self): 58 | self.assert_fails(self.join('baz.py'), 'bar', 59 | ("Cannot move baz: %s not found" 60 | % self.join("baz.py"))) 61 | self.assert_fails(self.join('dir/baz.py'), 'bar', 62 | ("Cannot move dir.baz: %s not found" 63 | % self.join("dir", "baz.py"))) 64 | 65 | def test_module_to_symbol(self): 66 | self.write_file('baz.py', '') 67 | error = "Cannot move a module 'foo' to a symbol (baz.newfunc)" 68 | self.assert_fails('foo', 'baz.newfunc', error) 69 | 70 | def test_module_to_existing_package(self): 71 | self._assert('foo', 'dir.subdir', 72 | [('foo', 'dir.subdir.foo', False)]) 73 | 74 | def test_module_to_directory(self): 75 | self._assert('foo', self.join('dir/subdir'), 76 | [('foo', 'dir.subdir.foo', False)]) 77 | self._assert('foo', self.join('dir/subdir/'), 78 | [('foo', 'dir.subdir.foo', False)]) 79 | 80 | def test_file_to_directory(self): 81 | self._assert(self.join('foo.py'), self.join('dir/subdir'), 82 | [('foo', 'dir.subdir.foo', False)]) 83 | 84 | def test_package_to_symbol(self): 85 | error = "Cannot move a package 'dir' into a symbol (foo.newfunc)" 86 | self.assert_fails('dir', 'foo.newfunc', error) 87 | 88 | def test_package_to_module(self): 89 | self.assert_fails('dir', 'foo', 90 | "Cannot move a package 'dir' into a module (foo)") 91 | 92 | def test_package_to_new_package(self): 93 | self._assert('dir', 'newdir', 94 | [('dir.__init__', 'newdir.__init__', False), 95 | ('dir.subdir.__init__', 'newdir.subdir.__init__', False), 96 | ]) 97 | 98 | def test_package_to_existing_package(self): 99 | self.write_file('newdir/__init__.py', '') 100 | self._assert('dir', 'newdir', 101 | [('dir.__init__', 'newdir.dir.__init__', False), 102 | ('dir.subdir.__init__', 'newdir.dir.subdir.__init__', 103 | False), 104 | ]) 105 | 106 | def test_package_to_its_own_subdirectory(self): 107 | error = "Cannot move a package 'dir' to its own subdir (dir.subdir)" 108 | self.assert_fails('dir', 'dir.subdir', error) 109 | self.assert_fails('dir', self.join('dir/subdir'), error) 110 | 111 | def test_symbol_to_new_symbol_in_same_file(self): 112 | self._assert('foo.myfunc', 'foo.newfunc', 113 | [('foo.myfunc', 'foo.newfunc', True)]) 114 | 115 | def test_symbol_to_new_symbol_in_new_file(self): 116 | self._assert('foo.myfunc', 'bar.newfunc', 117 | [('foo.myfunc', 'bar.newfunc', True)]) 118 | 119 | def test_symbol_to_existing_module(self): 120 | self.write_file('bar.py', '') 121 | self._assert('foo.myfunc', 'bar', 122 | [('foo.myfunc', 'bar.myfunc', True)]) 123 | 124 | def test_symbol_to_new_module(self): 125 | self._assert('foo.myfunc', 'bar', 126 | [('foo.myfunc', 'bar.myfunc', True)]) 127 | 128 | def test_symbol_to_new_filename(self): 129 | self._assert('foo.myfunc', self.join('dir/bar.py'), 130 | [('foo.myfunc', 'dir.bar.myfunc', True)]) 131 | 132 | def test_symbol_to_new_module_in_subdir(self): 133 | self._assert('foo.myfunc', 'dir.bar', 134 | [('foo.myfunc', 'dir.bar.myfunc', True)]) 135 | 136 | def test_symbol_to_new_symbol_in_new_module_in_subdir(self): 137 | self._assert('foo.myfunc', 'dir.bar.newfunc', 138 | [('foo.myfunc', 'dir.bar.newfunc', True)]) 139 | 140 | def test_symbol_to_package(self): 141 | self.assert_fails('foo.myfunc', 'dir', 142 | "Cannot move symbol 'foo.myfunc' to a package (dir)") 143 | 144 | @unittest.skip("We don't yet validate this case.") 145 | def test_symbol_to_existing_symbol(self): 146 | self.write_file('bar.py', 'def myfunc(): return 4\n') 147 | error = ("Cannot move symbol 'foo.myfunc' to 'bar': " 148 | "'bar' already defines a symbol named 'myfunc'.") 149 | self.assert_fails('foo.myfunc', 'bar', error) 150 | 151 | 152 | class ManyInputsTest(base.TestBase): 153 | def setUp(self): 154 | super(ManyInputsTest, self).setUp() 155 | self.write_file('foo.py', ('def myfunc(): return 4\n\n' 156 | 'def myfunc2(): return 42\n\n')) 157 | self.write_file('bar.py', 'def otherfunc(): return 5\n') 158 | self.write_file('dir/__init__.py', '') 159 | self.write_file('dir/subdir/__init__.py', '') 160 | self.write_file('dir2/__init__.py', '') 161 | 162 | def _assert(self, old_fullnames, new_fullname, expected): 163 | actual = inputs.expand_and_normalize(self.tmpdir, 164 | old_fullnames, new_fullname) 165 | self.assertItemsEqual(expected, actual) 166 | 167 | def assert_fails(self, old_fullnames, new_fullname, error_text): 168 | with self.assertRaises(ValueError) as e: 169 | inputs.expand_and_normalize(self.tmpdir, 170 | old_fullnames, new_fullname) 171 | self.assertEqual(error_text, str(e.exception)) 172 | 173 | def test_symbol_to_new_module(self): 174 | self._assert(['foo.myfunc', 'foo.myfunc2'], 'baz', 175 | [('foo.myfunc', 'baz.myfunc', True), 176 | ('foo.myfunc2', 'baz.myfunc2', True)]) 177 | self._assert(['foo.myfunc', 'bar.otherfunc'], 'baz', 178 | [('foo.myfunc', 'baz.myfunc', True), 179 | ('bar.otherfunc', 'baz.otherfunc', True)]) 180 | 181 | def test_symbol_to_existing_module(self): 182 | self._assert(['foo.myfunc', 'foo.myfunc2'], 'bar', 183 | [('foo.myfunc', 'bar.myfunc', True), 184 | ('foo.myfunc2', 'bar.myfunc2', True)]) 185 | 186 | def test_symbol_to_symbol(self): 187 | self.assert_fails(['foo.myfunc', 'foo.myfunc2'], 'bar.badfunc', 188 | "You asked to rename both 'foo.myfunc2' and " 189 | "'foo.myfunc' to 'bar.badfunc'. Impossible!") 190 | 191 | def test_symbol_to_package(self): 192 | self.assert_fails(['foo.myfunc', 'foo.myfunc2'], 'dir', 193 | "Cannot move symbol 'foo.myfunc' to a package (dir)") 194 | 195 | def test_module_to_package(self): 196 | self._assert(['foo', 'bar'], 'dir.subdir', 197 | [('foo', 'dir.subdir.foo', False), 198 | ('bar', 'dir.subdir.bar', False)]) 199 | 200 | def test_module_to_directory(self): 201 | self._assert(['foo', 'bar'], self.join('dir/subdir'), 202 | [('foo', 'dir.subdir.foo', False), 203 | ('bar', 'dir.subdir.bar', False)]) 204 | 205 | def test_module_to_new_directory(self): 206 | self.assert_fails(['foo', 'bar'], 'newdir', 207 | "You asked to rename both 'bar' and 'foo' " 208 | "to 'newdir'. Impossible!") 209 | 210 | def test_file_to_directory(self): 211 | self._assert([self.join('foo.py'), self.join('bar.py')], 212 | self.join('dir/subdir'), 213 | [('foo', 'dir.subdir.foo', False), 214 | ('bar', 'dir.subdir.bar', False)]) 215 | 216 | def test_module_to_module(self): 217 | self.assert_fails(['foo', 'bar'], 'baz', 218 | "You asked to rename both 'bar' and 'foo' " 219 | "to 'baz'. Impossible!") 220 | 221 | def test_package_to_new_directory(self): 222 | self.assert_fails(['dir', 'dir2'], 'newdir', 223 | "You asked to rename both 'dir2.__init__' and " 224 | "'dir.__init__' to 'newdir.__init__'. Impossible!") 225 | 226 | def test_package_to_existing_package(self): 227 | self.write_file('dir3/__init__.py', '') 228 | self._assert(['dir', 'dir2'], 'dir3', 229 | [('dir.__init__', 'dir3.dir.__init__', False), 230 | ('dir.subdir.__init__', 'dir3.dir.subdir.__init__', 231 | False), 232 | ('dir2.__init__', 'dir3.dir2.__init__', False)]) 233 | -------------------------------------------------------------------------------- /slicker/inputs.py: -------------------------------------------------------------------------------- 1 | """Function for converting the input fullnames to a canonical form. 2 | 3 | We support many types of inputs: 4 | . renaming a symbol in a module to another symbol in a different module 5 | . renaming a module to another name 6 | . moving a module to another package 7 | . moving multiple modules to other packages 8 | . renaming a package to another name 9 | . moving a package to be a sub-package of some existing package 10 | . renaming a file to another file 11 | . renaming a directory to another directory 12 | . moving a file from one directory to another 13 | . moving a directory from one directory to another 14 | . and more! 15 | 16 | This is the function that takes the inputs of all these different 17 | forms and converts them to one of only two types of moves: 18 | . Move a module from one name to another 19 | . Move a symbol from one name to another 20 | 21 | It also does sanity-checking of the inputs, to make sure they refer 22 | to real objects. 23 | """ 24 | from __future__ import absolute_import 25 | 26 | import os 27 | 28 | from . import khodemod 29 | from . import util 30 | 31 | 32 | def _expand_and_normalize_one(project_root, old_fullname, new_fullname, 33 | path_filter=khodemod.default_path_filter()): 34 | """See expand_and_normalize.__doc__.""" 35 | def filename_for(mod): 36 | return os.path.join(project_root, util.filename_for_module_name(mod)) 37 | 38 | def _assert_exists(module, error_prefix): 39 | if not os.path.exists(filename_for(module)): 40 | raise ValueError("%s: %s not found" 41 | % (error_prefix, filename_for(module))) 42 | 43 | def _normalize_fullname_and_get_type(fullname): 44 | # Check the cases that fullname is a file or a directory. 45 | # We convert it to a module if so. 46 | if fullname.endswith('.py'): 47 | relpath = os.path.relpath(fullname, project_root) 48 | return (util.module_name_for_filename(relpath), "module") 49 | if os.sep in fullname: 50 | relpath = os.path.relpath(fullname, project_root) 51 | return (util.module_name_for_filename(relpath), "package") 52 | 53 | if os.path.exists(filename_for(fullname)): 54 | return (fullname, "module") 55 | if os.path.exists(filename_for(fullname + '.__init__')): 56 | return (fullname, "package") 57 | 58 | # If we're foo.bar, we could be a symbol named bar in foo.py 59 | # or we could be a file foo/bar.py. To distinguish, we check 60 | # if foo/__init__.py exists. 61 | if '.' in fullname: 62 | (parent, symbol) = fullname.rsplit('.', 1) 63 | if os.path.exists(filename_for(parent + '.__init__')): 64 | return (fullname, "module") 65 | if os.path.exists(filename_for(parent)): 66 | return (fullname, "symbol") 67 | 68 | return (fullname, "unknown") 69 | 70 | def _modules_under(package_name): 71 | """Yield module-names relative to package_name-root.""" 72 | package_dir = os.path.dirname(filename_for(package_name + '.__init__')) 73 | for path in khodemod.resolve_paths(path_filter, root=package_dir): 74 | yield util.module_name_for_filename(path) 75 | 76 | (old_fullname, old_type) = _normalize_fullname_and_get_type(old_fullname) 77 | (new_fullname, new_type) = _normalize_fullname_and_get_type(new_fullname) 78 | 79 | if old_fullname == new_fullname: 80 | raise ValueError("Cannot move an object (%s) to itself" % old_fullname) 81 | 82 | # Below, we follow the following rule: if we don't know what 83 | # the type of new_type is (because it doesn't exist yet), we 84 | # assume the user wanted it to be the same type as old_type. 85 | 86 | if old_type == "symbol": 87 | (module, symbol) = old_fullname.rsplit('.', 1) 88 | _assert_exists(module, "Cannot move %s" % old_fullname) 89 | 90 | # TODO(csilvers): check that the 2nd element of the return-value 91 | # doesn't refer to a symbol that already exists. 92 | if new_type == "symbol": 93 | yield (old_fullname, new_fullname, True) 94 | elif new_type == "module": 95 | yield (old_fullname, '%s.%s' % (new_fullname, symbol), True) 96 | elif new_type == "package": 97 | raise ValueError("Cannot move symbol '%s' to a package (%s)" 98 | % (old_fullname, new_fullname)) 99 | elif new_type == "unknown": 100 | # According to the rule above, we should treat new_fullname 101 | # as a symbol. But if it doesn't have a dot, it *can't* be 102 | # a symbol; symbols must look like "module.symbol". So we 103 | # assume it's a module instead. 104 | if "." in new_fullname: 105 | yield (old_fullname, new_fullname, True) 106 | else: 107 | yield (old_fullname, '%s.%s' % (new_fullname, symbol), True) 108 | 109 | elif old_type == "module": 110 | _assert_exists(old_fullname, "Cannot move %s" % old_fullname) 111 | if new_type == "symbol": 112 | raise ValueError("Cannot move a module '%s' to a symbol (%s)" 113 | % (old_fullname, new_fullname)) 114 | elif new_type == "module": 115 | if os.path.exists(filename_for(new_fullname)): 116 | raise ValueError("Cannot use slicker to merge modules " 117 | "(%s already exists)" % new_fullname) 118 | yield (old_fullname, new_fullname, False) 119 | elif new_type == "package": 120 | module_basename = old_fullname.rsplit('.', 1)[-1] 121 | if os.path.exists(filename_for(new_fullname)): 122 | raise ValueError("Cannot move module '%s' into '%s': " 123 | "'%s.%s' already exists" 124 | % (old_fullname, new_fullname, 125 | new_fullname, module_basename)) 126 | yield (old_fullname, '%s.%s' % (new_fullname, module_basename), 127 | False) 128 | elif new_type == "unknown": 129 | yield (old_fullname, new_fullname, False) 130 | 131 | elif old_type == "package": 132 | _assert_exists(old_fullname + '.__init__', 133 | "Cannot move %s" % old_fullname) 134 | if new_type in ("symbol", "module"): 135 | raise ValueError("Cannot move a package '%s' into a %s (%s)" 136 | % (old_fullname, new_type, new_fullname)) 137 | elif new_type == "package": 138 | if new_fullname.startswith(old_fullname + '.'): 139 | raise ValueError("Cannot move a package '%s' to its own " 140 | "subdir (%s)" % (old_fullname, new_fullname)) 141 | if os.path.exists(filename_for(new_fullname + '.__init__')): 142 | # mv semantics, same as if we did 'mv /var/log /etc' 143 | package_basename = old_fullname.rsplit('.', 1)[-1] 144 | new_fullname = '%s.%s' % (new_fullname, package_basename) 145 | if os.path.exists(filename_for(new_fullname)): 146 | raise ValueError("Cannot move package '%s': " 147 | "'%s' already exists" 148 | % (old_fullname, new_fullname)) 149 | for module in _modules_under(old_fullname): 150 | yield ('%s.%s' % (old_fullname, module), 151 | '%s.%s' % (new_fullname, module), 152 | False) 153 | elif new_type == "unknown": 154 | for module in _modules_under(old_fullname): 155 | yield ('%s.%s' % (old_fullname, module), 156 | '%s.%s' % (new_fullname, module), 157 | False) 158 | 159 | elif old_type == "unknown": 160 | raise ValueError("Cannot figure out what '%s' is: " 161 | "module or package not found" % old_fullname) 162 | 163 | 164 | def expand_and_normalize(project_root, old_fullnames, new_fullname, 165 | path_filter=khodemod.default_path_filter()): 166 | """Return a list of old-new-info triples that effect the requested rename. 167 | 168 | In the simple case old_fullname is a module and new_fullname is a 169 | module, this will just return the input: [(old_fullname, 170 | new_fullname)]. Likewise if old_fullname is a symbol and 171 | new_fullname is a symbol. 172 | 173 | But if old_fullname is a package (dir) and so is new_fullname, 174 | then we will return a list of (old_module, new_module, is_symbol) 175 | triples for every module under the dir. 176 | 177 | We also handle cases where old_fullname and new_fullname are not 178 | of the same "type", and where new_fullname already exists. For 179 | instance, when moving a module into a package, we convert 180 | new_fullname to have the right name in the destination package. 181 | And when moving a package into a directory that already exists, 182 | we'll make it a subdir of the target-dir. 183 | 184 | We *also* handle cases where old_fullname and new_fullname are 185 | files instead of dotted module names. In that case we convert 186 | them to module-names first. 187 | 188 | When moving a symbol to a module, a module to a package, or a 189 | package into a new directory, you can have several inputs, and 190 | each will be moved. 191 | 192 | Some types of input are illegal: it's probably a mistake if 193 | old_fullname is a symbol and new_fullname is a package. Or 194 | if old_fullname doesn't actually exist. We raise in those cases. 195 | 196 | Returns: 197 | (old_fullname, new_fullname, is_symbol) triples. 198 | is_symbol is true if old_fullname is a symbol in a module, or 199 | false if it's a module. 200 | """ 201 | retval = [] 202 | for old_fullname in old_fullnames: 203 | retval.extend(_expand_and_normalize_one(project_root, old_fullname, 204 | new_fullname, path_filter)) 205 | 206 | # Sanity-check. If two different things are being moved to the 207 | # same new-fullname, that's a problem. It probably means we tried 208 | # to move two modules into a module instead of into a package, or 209 | # some such. 210 | seen_newnames = {} 211 | for (old_fullname, new_fullname, _) in retval: 212 | if new_fullname in seen_newnames: 213 | raise ValueError( 214 | "You asked to rename both '%s' and '%s' to '%s'. Impossible!" 215 | % (old_fullname, seen_newnames[new_fullname], new_fullname)) 216 | seen_newnames[new_fullname] = old_fullname 217 | 218 | return retval 219 | -------------------------------------------------------------------------------- /slicker/removal.py: -------------------------------------------------------------------------------- 1 | """Suggestors relating to removing no longer needed imports. 2 | 3 | This file contains the suggestors (in the sense of khodemod.py) which pertain 4 | to removing imports that are no longer used. After making other changes, 5 | slicker often needs to move imports; we think of this as adding a new import, 6 | and removing an old one. (In some cases we only do one or the other -- if the 7 | old import is still used, or the new one already exists.) This file is 8 | responsible for the latter. (The former is generally handled by the suggestor 9 | that moved the reference to the import, since that is more tied to the other 10 | changes being made.) 11 | """ 12 | from __future__ import absolute_import 13 | 14 | import tokenize 15 | 16 | from . import khodemod 17 | from . import model 18 | from . import util 19 | 20 | 21 | def _unused_imports(imports, old_fullname, file_info, within_node=None): 22 | """Decide what imports we can remove. 23 | 24 | Note that this should be run after the patches to references in the file 25 | have been applied, i.e. in a separate suggestor. 26 | 27 | Arguments: 28 | imports: set of imports to consider removing. These should likely be 29 | the imports that got us the symbol whose references you're 30 | updating. 31 | old_fullname: the fullname we deleted. If it's of a module, then 32 | imports of that module are definitely unused (as that module 33 | no longer exists). If it's of a symbol, this is ignored 34 | unless old_fullname was an import of just that symbol. 35 | file_info: the util.File object. 36 | within_node: if set, only consider imports within this AST node. 37 | (Useful for deciding whether to remove imports in that node.) 38 | 39 | Returns (set of imports we can remove, 40 | set of imports that may be used implicitly). 41 | 42 | "set of imports that may be used implicitly" is when we do 43 | "import foo.bar" and access "foo.baz.myfunc()" -- see 44 | special case (1) in the module docstring. 45 | """ 46 | if within_node is None: 47 | within_node = file_info.tree 48 | # Decide whether to keep the old import if we changed references to it. 49 | unused_imports = set() 50 | implicitly_used_imports = set() 51 | for imp in imports: 52 | # This includes all names that we might be *implicitly* 53 | # accessing via this import (special case (1) of the 54 | # module docstring, e.g. 'import foo.bar; foo.baz.myfunc()'. 55 | implicitly_used_names = util.names_starting_with( 56 | imp.alias.split('.', 1)[0], within_node) 57 | # This is only those names that we are explicitly accessing 58 | # via this import, i.e. not via such an "implicit import". 59 | explicitly_referenced_names = [ 60 | name for name in implicitly_used_names 61 | if util.dotted_starts_with(name, imp.alias)] 62 | 63 | if imp.name == old_fullname: 64 | unused_imports.add(imp) 65 | elif explicitly_referenced_names: 66 | pass # import is used 67 | elif implicitly_used_names: 68 | implicitly_used_imports.add(imp) 69 | else: 70 | unused_imports.add(imp) 71 | 72 | # Now, if there was an import (say 'import foo.baz') we were considering 73 | # removing but which might be used implicitly, and we are keeping a 74 | # different import (say 'import foo.bar') that gets us the same things, we 75 | # can remove the former. 76 | # We need to compute the full list of imports to do this, because we want 77 | # to count even imports we weren't asked to look at -- if we were asked to 78 | # look at 'import foo.baz', an unrelated 'foo.bar' counts too. 79 | if within_node is file_info.tree: 80 | # Additionally, if we are not looking at a particular node, we should 81 | # only consider toplevel imports, since a late 'import foo.bar' doesn't 82 | # necessarily mean we can remove a toplevel 'import foo.baz'. 83 | # TODO(benkraft): We can remove the conditional by making 84 | # model.compute_all_imports support passing both within_node and 85 | # toplevel_only. 86 | all_imports = model.compute_all_imports(file_info, toplevel_only=True) 87 | else: 88 | all_imports = model.compute_all_imports( 89 | file_info, within_node=within_node) 90 | 91 | kept_imports = all_imports - unused_imports - implicitly_used_imports 92 | for maybe_removable_imp in list(implicitly_used_imports): 93 | prefix = maybe_removable_imp.alias.split('.')[0] 94 | for kept_imp in kept_imports: 95 | if util.dotted_starts_with(kept_imp.alias, prefix): 96 | implicitly_used_imports.remove(maybe_removable_imp) 97 | unused_imports.add(maybe_removable_imp) 98 | break 99 | 100 | return (unused_imports, implicitly_used_imports) 101 | 102 | 103 | def _remove_import_patch(imp, file_info): 104 | """Remove the given import from the given file. 105 | 106 | Returns a khodemod.Patch, or a khodemod.WarningInfo if we can't/won't 107 | remove the import. 108 | """ 109 | toks = list(file_info.tokens.get_tokens(imp.node, include_extra=False)) 110 | next_tok = file_info.tokens.next_token(toks[-1], include_extra=True) 111 | if next_tok.type == tokenize.COMMENT and ( 112 | '@nolint' in next_tok.string.lower() or 113 | '@unusedimport' in next_tok.string.lower()): 114 | # Don't touch nolinted imports; they may be there for a reason. 115 | # TODO(benkraft): Handle this case for implicit imports as well 116 | return khodemod.WarningInfo( 117 | file_info.filename, imp.start, 118 | "Not removing import with @Nolint.") 119 | elif ',' in file_info.body[imp.start:imp.end]: 120 | # TODO(benkraft): better would be to check for `,` in each 121 | # token so we don't match commas in internal comments. 122 | # TODO(benkraft): learn to handle this case. 123 | return khodemod.WarningInfo( 124 | file_info.filename, imp.start, 125 | "I don't know how to edit this import.") 126 | else: 127 | # TODO(benkraft): Should we look at preceding comments? 128 | # We end up fighting with fix_python_imports if we do. 129 | start, end = util.get_area_for_ast_node( 130 | imp.node, file_info, include_previous_comments=False) 131 | return khodemod.Patch(file_info.filename, 132 | file_info.body[start:end], '', start, end) 133 | 134 | 135 | def remove_imports_suggestor(old_fullname): 136 | """The suggestor to remove imports for now-changed references. 137 | 138 | Note that this should run after _fix_uses_suggestor. 139 | 140 | Arguments: 141 | old_fullname: the pre-move fullname (module when moving a module, 142 | module.symbol when moving a symbol) that we're moving. (We 143 | only remove imports that could have gotten us that symbol.) 144 | """ 145 | def suggestor(filename, body): 146 | file_info = util.File(filename, body) 147 | 148 | # First, set things up, and do some checks. 149 | # TODO(benkraft): Don't recompute these; _fix_uses_suggestor has 150 | # already done so. 151 | old_localnames = model.localnames_from_fullnames( 152 | file_info, {old_fullname}) 153 | old_imports = {ln.imp for ln in old_localnames if ln.imp is not None} 154 | 155 | # Next, remove imports, if any are now unused. 156 | unused_imports, implicitly_used_imports = _unused_imports( 157 | old_imports, old_fullname, file_info) 158 | 159 | for imp in implicitly_used_imports: 160 | yield khodemod.WarningInfo( 161 | filename, imp.start, "This import may be used implicitly.") 162 | for imp in unused_imports: 163 | yield _remove_import_patch(imp, file_info) 164 | 165 | return suggestor 166 | 167 | 168 | def remove_old_file_imports_suggestor(project_root, old_fullname): 169 | """Suggestor to remove unused imports from old-file after moving a region. 170 | 171 | When we move the definition of a symbol, it may have been the only user of 172 | some imports in its file. We need to remove those now-unused imports. 173 | This runs after _fix_moved_region_suggestor, which probably added some of 174 | the imports we will remove to the new location of the symbol. 175 | 176 | Arguments: 177 | project_root: as elsewhere 178 | old_fullname: the pre-move fullname of the symbol we are moving 179 | """ 180 | # TODO(benkraft): Instead of having three suggestors for removing imports 181 | # that do slightly different things, have options for a single suggestor. 182 | old_module, old_symbol = old_fullname.rsplit('.', 1) 183 | 184 | def suggestor(filename, body): 185 | """filename is relative to the value of --root.""" 186 | # We only need to operate on the old file. Caller should ensure this 187 | # but we check to be safe. 188 | if util.module_name_for_filename(filename) != old_module: 189 | return 190 | 191 | file_info = util.File(filename, body) 192 | 193 | # Remove toplevel imports in the old file that are no longer used. 194 | # Sadly, it's difficult to determine which ones might be at all related 195 | # to the moved code, so we just remove anything that looks unused. 196 | # TODO(benkraft): Be more precise so we don't touch unrelated things. 197 | unused_imports, implicitly_used_imports = _unused_imports( 198 | model.compute_all_imports(file_info, toplevel_only=True), 199 | old_fullname, file_info) 200 | for imp in implicitly_used_imports: 201 | yield khodemod.WarningInfo( 202 | filename, imp.start, "This import may be used implicitly.") 203 | for imp in unused_imports: 204 | yield _remove_import_patch(imp, file_info) 205 | 206 | return suggestor 207 | 208 | 209 | def remove_moved_region_late_imports_suggestor(project_root, new_fullname): 210 | """Suggestor to remove unused imports after moving a region. 211 | 212 | When we move the definition of a symbol, it may have imported its new 213 | module as a "late-import"; this suggestor removes any such import. 214 | It runs after _fix_moved_region_suggestor and 215 | remove_old_file_imports_suggestor, and only operates on the new file. 216 | TODO(benkraft): We should also remove late imports if the new file also 217 | imported the same module at the toplevel. 218 | 219 | Arguments: 220 | project_root: as elsewhere 221 | new_fullname: the post-move fullname of the symbol we are moving 222 | """ 223 | new_module, new_symbol = new_fullname.rsplit('.', 1) 224 | 225 | def suggestor(filename, body): 226 | """filename is relative to the value of --root.""" 227 | # We only need to operate on the new file; that's where the moved 228 | # region will be by now. Caller should ensure this but we check to be 229 | # safe. 230 | if util.module_name_for_filename(filename) != new_module: 231 | return 232 | 233 | file_info = util.File(filename, body) 234 | 235 | # Find the region we moved. 236 | toplevel_names_in_new_file = util.toplevel_names(file_info) 237 | if new_symbol not in toplevel_names_in_new_file: 238 | raise khodemod.FatalError(filename, 0, 239 | "Could not find symbol '%s' in " 240 | "'%s': maybe it's defined weirdly?" 241 | % (new_symbol, new_module)) 242 | moved_node = toplevel_names_in_new_file[new_symbol] 243 | 244 | # Remove imports in the moved region itself that are no longer used. 245 | # This should probably just be imports of new_module, or things that 246 | # got us it, so we only look at those. 247 | unused_imports, implicitly_used_imports = _unused_imports( 248 | {imp for imp in model.compute_all_imports( 249 | file_info, within_node=moved_node) 250 | if model._import_provides_module(imp, new_module)}, 251 | None, file_info, within_node=moved_node) 252 | for imp in implicitly_used_imports: 253 | yield khodemod.WarningInfo( 254 | filename, imp.start, "This import may be used implicitly.") 255 | for imp in unused_imports: 256 | yield _remove_import_patch(imp, file_info) 257 | 258 | return suggestor 259 | -------------------------------------------------------------------------------- /slicker/replacement.py: -------------------------------------------------------------------------------- 1 | """Logic to replace references to a name in a file. 2 | 3 | This file doesn't deal with imports -- it's just about replacing the actual 4 | references in the file, for some given set of localnames. See the main 5 | entrypoint, replace_in_file, for more. 6 | """ 7 | from __future__ import absolute_import 8 | 9 | import ast 10 | import re 11 | import string 12 | import tokenize 13 | 14 | from . import khodemod 15 | from . import util 16 | 17 | 18 | _FILENAME_EXTENSIONS = ('.py', '.js', '.jsx', '.png', '.jpg', '.svg', '.html', 19 | '.less', '.handlebars', '.json', '.txt', '.css') 20 | _FILENAME_EXTENSIONS_RE_STRING = '|'.join(re.escape(e) 21 | for e in _FILENAME_EXTENSIONS) 22 | 23 | 24 | def _re_for_name(name): 25 | """Find a dotted-name (a.b.c) given that Python allows whitespace. 26 | 27 | This is actually pretty tricky. Here are some issues: 28 | 1) We don't want a name `a.b.c` to match `d.a.b.c`. 29 | 2) We don't want a name `foo` to match `foo.py` -- that's a filename, 30 | not a module-name (and is handled separately). We also don't 31 | want it to match other common filename extensions. 32 | 3) We don't want a name `browser` to match text like 33 | "# Open a new browser window" 34 | 35 | The first two issues are easy to handle. For the third, we add a 36 | special case for "English-seeming" names: those that have only 37 | alphabetic chars. For those words, we only rename them if they're 38 | followed by a dot (which could be module.function) or match 39 | the entire string (which could be a mock or some other literal 40 | use). We also allow surrounded-by-backticks, since that's 41 | markup-language for "code". 42 | """ 43 | # TODO(csilvers): replace '\s*' by '\s*#\s*' below, and then we 44 | # can use this to match line-broken dotted-names inside comments too! 45 | name_with_spaces = re.escape(name).replace(r'\.', r'\s*\.\s*') 46 | if not name.strip(string.ascii_letters): 47 | # Name is entirely alphabetic. 48 | return re.compile(r'(?)` 288 | corresponding to them. (So for each input localname the corresponding 289 | output tuple(s) will have that localname as tuple.localname.) 290 | 291 | If passed, we use the imports from 'imports', which should be a set 292 | of imports; otherwise we use all the imports from the file_info. 293 | 294 | Returns an iterable of LocalName namedtuples. 295 | 296 | See also model.localnames_from_fullnames, which returns more or less 297 | the same data, but starts from fullnames instead of localnames. 298 | 299 | If the unqualified name of a symbol defined in this file 300 | appears in localnames, the corresponding LocalName will be 301 | LocalName(fullname, unqualified_name, None). 302 | 303 | Note that 'import foo.baz' also makes 'foo.bar.myfunc' available 304 | (see module docstring, "implicit imports"), so so we have to include 305 | that as well. If you also did 'import foo.bar', we don't bother -- 306 | we only include the "best" name when we can. (We make this choice 307 | per-localname, so if you did 'import foo.baz' and 308 | 'from foo import bar', and localnames is {'foo.bar.myfunc', 309 | 'bar.myfunc'}, we'll return the quirky LocalName for 310 | 'foo.bar.myfunc' as well as the more normal one for 'bar.myfunc'. 311 | 312 | If a fullname is not made available by any import in this file, 313 | we won't return any corresponding LocalNames. It might seem 314 | like this set should always have at most one LocalName for 315 | each fullname, but there are several cases it might have more: 316 | 1) In the "quirk of python" case mentioned above. 317 | 2) If you import a module two ways or from itself (see special 318 | cases (3) and (4) in the module docstring). 319 | 4) If you do several "late imports" (see module docstring), 320 | you'll get one return-value per late-import that you do. 321 | 322 | If a localname is not made available by any import in this file, 323 | we won't return any corresponding LocalNames -- perhaps it's 324 | actually a local variable. It might seem like this set 325 | should always have at most one LocalName for each localname, 326 | but there are several cases it might have more: 327 | 1) If there are multiple "implicit imports" as mentioned above. 328 | 2) If you do several "late imports" (see module docstring), 329 | you'll get one return-value per late-import that you do. 330 | 3) If the localname is defined in this file, and the file also 331 | imports itself (special case (4) in the module docstring). 332 | """ 333 | # TODO(benkraft): Share code with model.localnames_from_fullnames, they do 334 | # similar things. 335 | if imports is None: 336 | imports = compute_all_imports(file_info) 337 | current_module_name = util.module_name_for_filename(file_info.filename) 338 | toplevel_names = util.toplevel_names(file_info) 339 | 340 | imports_by_alias = {} 341 | imports_by_alias_prefix = {} 342 | for imp in imports: 343 | alias_prefix = imp.alias.split('.', 1)[0] 344 | imports_by_alias.setdefault(imp.alias, []).append(imp) 345 | imports_by_alias_prefix.setdefault(alias_prefix, []).append(imp) 346 | 347 | for localname in localnames: 348 | found_explicit_import = False 349 | for localname_prefix in util.dotted_prefixes(localname): 350 | if localname_prefix in imports_by_alias: 351 | for imp in imports_by_alias[localname_prefix]: 352 | yield LocalName(imp.name + localname[len(imp.alias):], 353 | localname, imp) 354 | found_explicit_import = True 355 | 356 | if not found_explicit_import: 357 | # This deals with the case where you did 'import foo.bar' and then 358 | # used 'foo.baz' -- an "implicit import". 359 | implicit_imports = imports_by_alias_prefix.get( 360 | localname.split('.', 1)[0], []) 361 | for imp in implicit_imports: 362 | yield LocalName(localname, localname, imp) 363 | 364 | # If the name is a specific symbol defined in the file on which we are 365 | # operating, we also treat the unqualified reference as a localname, 366 | # with null import. 367 | for toplevel_name in toplevel_names: 368 | if util.dotted_starts_with(localname, toplevel_name): 369 | yield LocalName('%s.%s' % (current_module_name, toplevel_name), 370 | toplevel_name, None) 371 | break 372 | -------------------------------------------------------------------------------- /slicker/khodemod.py: -------------------------------------------------------------------------------- 1 | """Utility for modifying code. 2 | 3 | Heavily inspired by https://github.com/facebook/codemod and with a similar API, 4 | although it's written from scratch. The user-facing functionality will 5 | eventually be pretty similar, but khodemod is designed for use as a library -- 6 | so each component is pluggable -- as well as for Khan Academy's use cases. 7 | 8 | TERMS: 9 | "suggestor": These are how one implements code-changes to be applied using 10 | khodemod. They're just functions (often curried -- that is, the actual 11 | suggestor is the function returned by calling some_suggestor(...)) 12 | accepting a filename (string) and body (string, the text of that file) and 13 | yielding a series of khodemod.Patch objects, representing the changes to be 14 | made. They may also yield khodemod.WarningInfo objects, which will be 15 | displayed to the user as warnings, or raise khodemod.FatalError exceptions, 16 | to refuse to process the given file. Note that these changes will not be 17 | applied until the suggestor completes operation. For an example, see 18 | regex_suggestor() below, which implements a simple find-and-replace. 19 | "frontend": These are responsible for applying the changes given by a 20 | suggestor, perhaps displaying output to the user (or even prompting for 21 | input) as they go. Currently, only one is implemented, 22 | khodemod.AcceptingFrontend, which simply applies the changes, perhaps 23 | displaying a progress bar. 24 | TODO(benkraft): Implement other frontends. 25 | "root": The directory in which we should operate, often the current working 26 | directory. 27 | "path_filter": These are how one decides what code to operate on: one passes a 28 | path filter, which is just a function which takes a filename relative to 29 | "root" and returns True if we should operate on it. (It may also be passed 30 | a directory, which will end with a slash; if it returns False we skip the 31 | entire directory.) These are useful for ignoring generated files and the 32 | like. 33 | 34 | TODO(benkraft): Implement a commandline interface for the regex suggestors. 35 | """ 36 | from __future__ import absolute_import 37 | 38 | import collections 39 | import os 40 | 41 | import tqdm 42 | 43 | 44 | DEFAULT_EXCLUDE_PATHS = ('genfiles', 'third_party') 45 | DEFAULT_EXTENSIONS = ('py',) 46 | 47 | 48 | # Dict from (path-filter function, root) to the actual list of paths. 49 | _RESOLVE_PATHS_CACHE = {} 50 | 51 | 52 | def regex_suggestor(regex, replacement): 53 | """Replaces regex (object) with replacement. 54 | 55 | Replacment may use backreferences and such. 56 | TODO(benkraft): Support passing a function as a replacement. 57 | TODO(benkraft): This won't necessarily work right for overlapping matches; 58 | the patches may fail to apply. 59 | """ 60 | def suggestor(filename, body): 61 | for match in regex.finditer(body): 62 | yield Patch(filename, match.group(0), match.expand(replacement), 63 | match.start(), match.end()) 64 | return suggestor 65 | 66 | 67 | # old/new are unicode; 68 | # start/end are (unicode) character offsets for the old text. 69 | # TODO(benkraft): Include context for patching? 70 | _Patch = collections.namedtuple('Patch', 71 | ['filename', 'old', 'new', 'start', 'end']) 72 | # pos is a (unicode) character offset for the warning. 73 | WarningInfo = collections.namedtuple('WarningInfo', 74 | ['filename', 'pos', 'message']) 75 | 76 | 77 | class Patch(object): 78 | def __init__(self, filename, old, new, start, end, file_permissions=None): 79 | """An object representing a change to make to a filename. 80 | 81 | Arguments: 82 | filename: the filename to modify, relative to project-root. 83 | old: the contents to remove (should be filename_body[start:end]) 84 | new: the contents to replace `old` with 85 | start: (unicode) character offset for the old text 86 | end: (unicode) character offset for the old text 87 | file_permissions: if set, change the file-mode of filename to this 88 | """ 89 | self.filename = filename 90 | self.old = old 91 | self.new = new 92 | self.start = start 93 | self.end = end 94 | self.permissions = file_permissions 95 | 96 | def __repr__(self): 97 | return (' %s (%s:%s-%s)>' 98 | % (self.old, self.new, self.filename, self.start, self.end)) 99 | 100 | def apply_to(self, body): 101 | if body[self.start:self.end] != (self.old or ''): 102 | raise FatalError(self.filename, self.start, 103 | "patch didn't apply: %s" % (self,)) 104 | if self.new is None: # means we want to delete the new file 105 | assert self.start == 0 and self.end == len(body), self 106 | return None 107 | else: 108 | return body[:self.start] + self.new + body[self.end:] 109 | 110 | 111 | class FatalError(RuntimeError): 112 | """Something went horribly wrong; we should give up patching this file.""" 113 | def __init__(self, filename, pos, message): 114 | self.filename = filename 115 | self.pos = pos 116 | self.message = message 117 | 118 | def __repr__(self): 119 | return "FatalError(%r, %r, %r)" % (self.filename, self.pos, 120 | self.message) 121 | 122 | def __unicode__(self): 123 | return "Fatal Error:%s:%s:%s" % (self.filename, self.pos, self.message) 124 | 125 | def __eq__(self, other): 126 | return (isinstance(other, FatalError) and 127 | self.filename == other.filename and self.pos == other.pos and 128 | self.message == other.message) 129 | 130 | 131 | def emit(txt): 132 | """This is a function so tests can override it.""" 133 | print txt 134 | 135 | 136 | def extensions_path_filter(extensions, include_extensionless=False): 137 | if extensions == '*': 138 | return lambda path: True 139 | 140 | def filter_path(path): 141 | if path.endswith(os.sep): 142 | # Always include directories. 143 | return True 144 | _, ext = os.path.splitext(path) 145 | if not ext and include_extensionless: 146 | return True 147 | if ext and ext.lstrip(os.path.extsep) in extensions: 148 | return True 149 | return False 150 | 151 | return filter_path 152 | 153 | 154 | def dotfiles_path_filter(): 155 | return lambda path: not any(len(part) > 1 and path.startswith('.') 156 | for part in os.path.split(path)) 157 | 158 | 159 | def exclude_paths_filter(exclude_paths): 160 | return lambda path: not any(part in exclude_paths 161 | for part in path.split(os.path.sep)) 162 | 163 | 164 | def and_filters(filters): 165 | return lambda item: all(f(item) for f in filters) 166 | 167 | 168 | def default_path_filter(extensions=DEFAULT_EXTENSIONS, 169 | include_extensionless=False, 170 | exclude_paths=DEFAULT_EXCLUDE_PATHS): 171 | return and_filters([ 172 | extensions_path_filter(extensions, include_extensionless), 173 | dotfiles_path_filter(), 174 | exclude_paths_filter(exclude_paths), 175 | ]) 176 | 177 | 178 | def read_file(root, filename): 179 | """Return file contents, or None if the file is not found. 180 | 181 | filename is taken relative to root. 182 | """ 183 | # TODO(benkraft): Cache contents. 184 | try: 185 | with open(os.path.join(root, filename)) as f: 186 | from . import unicode_util 187 | return unicode_util.decode(filename, f.read()) 188 | except IOError as e: 189 | if e.errno == 2: # No such file 190 | return None # empty file 191 | raise 192 | 193 | 194 | def _resolve_paths(path_filter, root='.'): 195 | """Actually resolve the paths, and update the cache. 196 | 197 | Returns a generator; we need to return a generator when path computation is 198 | nontrivial, so the caller can, if desired, show a progress bar for that 199 | operation. 200 | TODO(benkraft): There's probably a cleaner way, e.g. we could own our own 201 | progress bar, or accept a progress-bar fn. 202 | """ 203 | paths = [] 204 | for dirpath, dirnames, filenames in os.walk(root): 205 | # Prune directories to traverse according to the path filter. 206 | # Go in reverse order to keep indexes the same as we delete things. 207 | for i, name in reversed(list(enumerate(dirnames))): 208 | relname = os.path.relpath(os.path.join(dirpath, name), root) 209 | relname = os.path.join(relname, '') # add trailing slash 210 | if not path_filter(relname): 211 | del dirnames[i] 212 | 213 | # Filter filenames and yield according to the path filter. 214 | for name in filenames: 215 | relname = os.path.relpath(os.path.join(dirpath, name), root) 216 | if path_filter(relname): 217 | paths.append(relname) 218 | yield relname 219 | 220 | # We're done; we can cache the result now. 221 | _RESOLVE_PATHS_CACHE[(path_filter, root)] = paths 222 | 223 | 224 | def resolve_paths(path_filter, root='.'): 225 | """All files under root (relative to root), ignoring filtered files. 226 | 227 | This is cached across runs over the same path_filter function, 228 | although note that if you iterate only partway through the 229 | returned iterable the cache may not get populated. 230 | """ 231 | cached_value = _RESOLVE_PATHS_CACHE.get((path_filter, root)) 232 | if cached_value is not None: 233 | return cached_value 234 | else: 235 | # This is a generator; it will update the cache when exhausted. 236 | return _resolve_paths(path_filter, root) 237 | 238 | 239 | def pos_to_line_col(text, pos): 240 | """Accept a character position in text, return (lineno, colno). 241 | 242 | lineno and colno are, as usual, 1-indexed. 243 | """ 244 | lines = text.splitlines(True) 245 | for i, line in enumerate(lines): 246 | if pos < len(line): 247 | return (i + 1, pos + 1) 248 | else: 249 | pos -= len(line) 250 | raise RuntimeError("Invalid position %s!" % pos) 251 | 252 | 253 | def line_col_to_pos(text, line, col): 254 | """Accept a line/column in text, return character position. 255 | 256 | lineno and colno are, as usual, 1-indexed. 257 | """ 258 | lines = text.splitlines(True) 259 | try: 260 | return sum(len(line) for line in lines[:line - 1]) + col - 1 261 | except IndexError: 262 | raise RuntimeError("Invalid line number %s!" % line) 263 | 264 | 265 | class Frontend(object): 266 | def __init__(self): 267 | # (root, filename) of files we've modified. 268 | # filename is relative to root. 269 | self._modified_files = set() 270 | 271 | def handle_patches(self, root, filename, patches): 272 | """Accept a list of patches for a file, and apply them. 273 | 274 | This may prompt the user for confirmation, inform them of the patches, 275 | or simply apply them without input. It is the responsibility of this 276 | method to do any merging necessary to accomplish that. 277 | 278 | The patches will be ordered by start position. 279 | """ 280 | raise NotImplementedError("Subclasses must override.") 281 | 282 | def handle_warnings(self, root, filename, warnings): 283 | """Accept a list of warnings for a file, and tell the user. 284 | 285 | Or don't! It's up to the subclasses. The warnings will be ordered by 286 | start position. This will be called before any patching, and may raise 287 | FatalError if we want to not proceed. 288 | """ 289 | raise NotImplementedError("Subclasses must override.") 290 | 291 | def handle_error(self, root, error): 292 | """Accept a fatal error, and tell the user we'll skip this file.""" 293 | raise NotImplementedError("Subclasses must override.") 294 | 295 | def write_file(self, root, filename, text, file_permissions=None): 296 | """filename is taken to be relative to root. 297 | 298 | Note you need a Frontend to write files (so we can update the 299 | list of modified files), but not to read them. 300 | 301 | If file_permissions is not None, set the perms of filename. 302 | """ 303 | abspath = os.path.abspath(os.path.join(root, filename)) 304 | if text is None: # it means we want to delete filename 305 | try: 306 | os.unlink(abspath) 307 | # We changed what files exist: clear the cache. 308 | _RESOLVE_PATHS_CACHE.clear() 309 | except OSError as e: 310 | if e.errno == 2: # No such file: already deleted 311 | pass 312 | raise 313 | # TODO(csilvers): delete our parent dirs if they're empty? 314 | else: 315 | try: 316 | os.makedirs(os.path.dirname(abspath)) 317 | except (IOError, OSError): # hopefully "directory already exists" 318 | pass 319 | if not os.path.exists(abspath): 320 | # We changed what files exist: clear the cache. 321 | _RESOLVE_PATHS_CACHE.clear() 322 | with open(abspath, 'w') as f: 323 | from . import unicode_util 324 | f.write(unicode_util.encode(filename, text)) 325 | self._modified_files.add((root, filename)) 326 | if file_permissions: 327 | os.chmod(abspath, file_permissions) 328 | 329 | def progress_bar(self, paths): 330 | """Return the passed iterable of paths, and perhaps update progress. 331 | 332 | Subclasses may override. 333 | """ 334 | return paths 335 | 336 | def _run_suggestor_on_file(self, suggestor, filename, root): 337 | """filename is relative to root.""" 338 | try: 339 | # Ensure the entire suggestor runs before we start patching. 340 | vals = list( 341 | suggestor(filename, read_file(root, filename) or '')) 342 | patches = [p for p in vals if isinstance(p, Patch) 343 | and p.old != p.new] 344 | # HACK: consider addition-ish before deletion-ish. 345 | patches.sort(key=lambda p: (p.start, 346 | len(p.old or '') - len(p.new or ''))) 347 | warnings = [w for w in vals if isinstance(w, WarningInfo)] 348 | warnings.sort(key=lambda w: w.pos) 349 | 350 | # Typically when you run a suggestor on a file, all the 351 | # patches it suggests will be for that file as well, but 352 | # it's possible for a suggestor to suggest changes to 353 | # another file (e.g. when moving code from one file to 354 | # another). So we group by file-to-change here. 355 | patches_by_file = {} 356 | warnings_by_file = {} 357 | for patch in patches: 358 | patches_by_file.setdefault(patch.filename, []).append(patch) 359 | for warning in warnings: 360 | warnings_by_file.setdefault(warning.filename, []).append( 361 | warning) 362 | 363 | seen_filenames = list(set(patches_by_file) | set(warnings_by_file)) 364 | seen_filenames.sort(key=lambda f: (0 if f == filename else 1, f)) 365 | for filename in seen_filenames: 366 | if filename in warnings_by_file: 367 | self.handle_warnings(root, filename, 368 | warnings_by_file[filename]) 369 | if filename in patches_by_file: 370 | self.handle_patches(root, filename, 371 | patches_by_file[filename]) 372 | except FatalError as e: 373 | self.handle_error(root, e) 374 | 375 | def run_suggestor_on_files(self, suggestor, filenames, root='.'): 376 | """Like run_suggestor, but on exactly the given files.""" 377 | for filename in self.progress_bar(filenames): 378 | self._run_suggestor_on_file(suggestor, filename, root) 379 | 380 | def run_suggestor(self, suggestor, 381 | path_filter=default_path_filter(), root='.'): 382 | """Run the suggestor on all files matching the path_filter.""" 383 | self.run_suggestor_on_files( 384 | suggestor, resolve_paths(path_filter, root), root) 385 | 386 | def run_suggestor_on_modified_files(self, suggestor): 387 | """Like run_suggestor, but only on files we've modified. 388 | 389 | Useful for fixups after the fact that we don't want to apply to the 390 | whole codebase, only the files we touched. 391 | 392 | Note that this doesn't take a root, because we use the one from 393 | when we first modified the file. 394 | """ 395 | for (root, filename) in self.progress_bar(self._modified_files): 396 | # If we modified a file by deleting it, no more 397 | # suggestions for you! 398 | if os.path.exists(os.path.join(root, filename)): 399 | self._run_suggestor_on_file(suggestor, filename, root) 400 | 401 | 402 | class AcceptingFrontend(Frontend): 403 | """A frontend where we apply all patches without question.""" 404 | def __init__(self, verbose=False, **kwargs): 405 | super(AcceptingFrontend, self).__init__(**kwargs) 406 | self.verbose = verbose 407 | 408 | def progress_bar(self, paths): 409 | if self.verbose: 410 | if not isinstance(paths, (list, tuple, set, frozenset, dict)): 411 | paths = list( 412 | tqdm.tqdm(paths, desc='Computing paths', unit=' files')) 413 | if len(paths) > 1: 414 | return tqdm.tqdm(paths, desc='Applying changes', unit=' files') 415 | return paths 416 | 417 | def handle_patches(self, root, filename, patches): 418 | body = read_file(root, filename) 419 | # We operate in reverse order to avoid having to keep track of changing 420 | # offsets. 421 | new_body = body or '' 422 | new_file_perms = None 423 | for patch in reversed(patches): 424 | assert filename == patch.filename, patch 425 | new_body = patch.apply_to(new_body) 426 | # The last-specified permission (due to reversed()) wins. 427 | new_file_perms = new_file_perms or patch.permissions 428 | if body != new_body: 429 | self.write_file(root, filename, new_body, new_file_perms) 430 | 431 | def handle_warnings(self, root, filename, warnings): 432 | body = read_file(root, filename) or '' 433 | for warning in warnings: 434 | assert filename == warning.filename, warning 435 | lineno, _ = pos_to_line_col(body, warning.pos) 436 | line = body.splitlines()[lineno - 1] 437 | emit("WARNING:%s\n on %s:%s --> %s" 438 | % (warning.message, filename, lineno, line)) 439 | 440 | def handle_error(self, root, error): 441 | body = read_file(root, error.filename) 442 | if body: 443 | try: 444 | lineno, _ = pos_to_line_col(body, error.pos) 445 | line = body.splitlines()[lineno - 1] 446 | line_info = ":%s --> %s" % (lineno, line) 447 | except Exception: 448 | # Error error! Make sure not to crash so we still log it. 449 | line_info = " at invalid position %s" % error.pos 450 | else: 451 | line_info = " (empty)" 452 | emit("ERROR:%s\n on %s%s" 453 | % (error.message, error.filename, line_info)) 454 | --------------------------------------------------------------------------------