├── .appveyor.yml ├── .gitattributes ├── .github ├── ISSUE_TEMPLATE.md └── workflows │ ├── python-app.yml │ └── python-publish.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CHANGELOG.md ├── LICENSE.txt ├── MANIFEST.in ├── README.rst ├── gmock_test.cpp ├── massedit.bat ├── massedit.py ├── output.txt ├── pyproject.toml └── tests.py /.appveyor.yml: -------------------------------------------------------------------------------- 1 | # appveyor.yml - https://www.appveyor.com/docs/lang/python 2 | --- 3 | image: 4 | - Visual Studio 2019 5 | 6 | environment: 7 | # https://devguide.python.org/versions 8 | matrix: 9 | - TOXENV: py38 10 | - TOXENV: py39 11 | - TOXENV: py310 12 | - TOXENV: py311 13 | 14 | build: false 15 | 16 | install: 17 | # - py --list 18 | # - py -m pip install --upgrade pip 19 | - py -m pip install -e .[test] 20 | 21 | test_script: 22 | - py -m tox 23 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Set default behaviour, in case users don't have core.autocrlf set. 2 | * text=auto 3 | 4 | # Explicitly declare text files we want to always be normalized and converted 5 | # to native line endings on checkout. 6 | *.c text 7 | *.h text 8 | *.py text 9 | *.rst text 10 | .gitignore text 11 | 12 | # Declare files that will always have CRLF line endings on checkout. 13 | *.sln text eol=crlf 14 | *.bat text eol=crlf 15 | 16 | # Denote all files that are truly binary and should not be modified. 17 | *.png binary 18 | *.jpg binary 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | * codemetrics version: 2 | * Python version: 3 | * Operating System: 4 | 5 | ### Description 6 | 7 | Describe what you were trying to get done. 8 | Tell us what happened, what went wrong, and what you expected to happen. 9 | 10 | ### What I Did 11 | 12 | ``` 13 | Paste the command(s) you ran and the output. 14 | If there was a crash, please include the traceback here. 15 | ``` 16 | -------------------------------------------------------------------------------- /.github/workflows/python-app.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python 3 | 4 | name: Python application 5 | 6 | on: 7 | push: 8 | branches: [ "main" ] 9 | pull_request: 10 | branches: [ "main" ] 11 | 12 | permissions: 13 | contents: read 14 | 15 | jobs: 16 | build: 17 | 18 | runs-on: ubuntu-latest 19 | strategy: 20 | matrix: 21 | python-version: ["3.7","3.8","3.9","3.10","3.11"] 22 | 23 | steps: 24 | - uses: actions/checkout@v3 25 | - name: Set up Python ${{matrix.python-version}} 26 | uses: actions/setup-python@v3 27 | with: 28 | python-version: ${{matrix.python-version}} 29 | - name: Install dependencies 30 | run: | 31 | python -m pip install --upgrade pip 32 | python -m pip install -e .[test] 33 | - name: Lint with flake8 34 | run: | 35 | # stop the build if there are Python syntax errors or undefined names 36 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 37 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 38 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 39 | - name: Test with pytest 40 | run: | 41 | python tests.py 42 | -------------------------------------------------------------------------------- /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will upload a Python Package using Twine when a release is created 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries 3 | 4 | # This workflow uses actions that are not certified by GitHub. 5 | # They are provided by a third-party and are governed by 6 | # separate terms of service, privacy policy, and support 7 | # documentation. 8 | 9 | name: Upload Python Package 10 | 11 | on: 12 | push: 13 | branches: 14 | - main 15 | tags: 16 | - v* 17 | 18 | permissions: 19 | contents: read 20 | 21 | jobs: 22 | deploy: 23 | runs-on: ubuntu-latest 24 | steps: 25 | - uses: actions/checkout@v3 26 | - name: Set up Python 27 | uses: actions/setup-python@v3 28 | with: 29 | python-version: '3.x' 30 | - name: Install dependencies 31 | run: | 32 | python -m pip install --upgrade pip 33 | pip install build 34 | - name: Build package 35 | run: python -m build 36 | - name: Publish package 37 | uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 38 | with: 39 | user: __token__ 40 | password: ${{ secrets.PYPI_API_TOKEN }} 41 | skip_existing: true 42 | verbose: true 43 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.swp 3 | *.tmp 4 | .coverage 5 | .eggs 6 | .idea 7 | .unison/ar* 8 | .unison/fp* 9 | .project 10 | .pydevproject 11 | .travis-solo/ 12 | .tox/ 13 | build/ 14 | dist/ 15 | sdist/ 16 | tags 17 | vimfiles/.* 18 | venv 19 | *.egg-info 20 | *.egg/ 21 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.4.0 4 | hooks: 5 | - id: check-yaml 6 | - id: end-of-file-fixer 7 | - id: trailing-whitespace 8 | - id: check-toml 9 | - repo: https://github.com/psf/black 10 | rev: 23.9.1 11 | hooks: 12 | - id: black 13 | language_version: python 14 | - repo: https://github.com/PyCQA/flake8 15 | rev: 6.1.0 16 | hooks: 17 | - id: flake8 18 | language: python 19 | additional_dependencies: [flake8-comprehensions] 20 | args: [--max-line-length=999] 21 | - repo: https://github.com/commitizen-tools/commitizen 22 | rev: 3.8.2 23 | hooks: 24 | - id: commitizen 25 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## v0.70.0 (2023-08-31) 2 | 3 | ### Fix 4 | 5 | - fix commitizen configuration 6 | - fix .github/workflows/python-app.yml (#18) 7 | 8 | ## v0.69.1 (2020-12-22) 9 | 10 | ## v0.69.0 (2020-12-22) 11 | 12 | ## v0.68.6 (2019-12-02) 13 | 14 | ## v0.68.5 (2019-04-13) 15 | 16 | ## v0.68.4 (2017-10-24) 17 | 18 | ## v0.68.1 (2016-06-04) 19 | 20 | ## v0.67.1 (2015-06-28) 21 | 22 | ## v0.67 (2015-06-23) 23 | 24 | ## v0.66 (2013-07-14) 25 | 26 | ## v0.65 (2013-07-12) 27 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2012-2019 elmotec 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE.txt 2 | include README.rst 3 | include tests.py 4 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://img.shields.io/pypi/v/massedit.svg 2 | :target: https://pypi.python.org/pypi/massedit/ 3 | :alt: PyPi version 4 | 5 | .. image:: https://img.shields.io/pypi/pyversions/massedit.svg 6 | :target: https://pypi.python.org/pypi/massedit/ 7 | :alt: Python compatibility 8 | 9 | .. image:: https://img.shields.io/github/workflow/status/elmotec/massedit/Python%20application 10 | :target: https://github.com/elmotec/massedit/actions?query=workflow%3A%22Python+application%22 11 | :alt: GitHub Workflow Python application 12 | 13 | .. image:: https://img.shields.io/appveyor/ci/elmotec/massedit.svg?label=AppVeyor 14 | :target: https://ci.appveyor.com/project/elmotec/massedit 15 | :alt: AppVeyor status 16 | 17 | .. image:: https://img.shields.io/pypi/dm/massedit.svg 18 | :alt: PyPi 19 | :target: https://pypi.python.org/pypi/massedit 20 | 21 | .. image:: https://img.shields.io/librariesio/release/pypi/massedit.svg?label=libraries.io 22 | :alt: Libraries.io dependency status for latest release 23 | :target: https://libraries.io/pypi/massedit 24 | 25 | .. image:: https://coveralls.io/repos/elmotec/massedit/badge.svg 26 | :target: https://coveralls.io/r/elmotec/massedit 27 | :alt: Coverage 28 | 29 | .. image:: https://img.shields.io/codacy/grade/474b0af6853a4c5f8f9214d3220571f9.svg 30 | :target: https://www.codacy.com/app/elmotec/massedit/dashboard 31 | :alt: Codacy 32 | 33 | 34 | ======== 35 | massedit 36 | ======== 37 | 38 | *formerly known as Python Mass Editor* 39 | 40 | Implements a python mass editor to process text files using Python 41 | code. The modification(s) is (are) shown on stdout as a diff output. One 42 | can then modify the target file(s) in place with the -w/--write option. 43 | This is very similar to 2to3 tool that ships with Python 3. 44 | 45 | 46 | +--------------------------------------------------------------------------+ 47 | | **WARNING**: A word of caution about the usage of ``eval()`` | 48 | +--------------------------------------------------------------------------+ 49 | | This tool is useful as far as it goes but it does rely on the python | 50 | | ``eval()`` function and does not check the code being executed. | 51 | | **It is a major security risk** and one should not use this tool in a | 52 | | production environment. | 53 | | | 54 | | See `Ned Batchelder's article`_ for a thorough discussion of the dangers | 55 | | linked to ``eval()`` and ways to circumvent them. Note that None of the | 56 | | counter-measure suggested in the article are implemented at this time. | 57 | +--------------------------------------------------------------------------+ 58 | 59 | Usage 60 | ----- 61 | 62 | You probably will need to know the basics of the `Python re module`_ (regular 63 | expressions). 64 | 65 | :: 66 | 67 | usage: massedit.py [-h] [-V] [-w] [-v] [-e EXPRESSIONS] [-f FUNCTIONS] 68 | [-x EXECUTABLES] [-s START_DIRS] [-m MAX_DEPTH] [-o FILE] 69 | [-g FILE] [--encoding ENCODING] [--newline NEWLINE] 70 | [file pattern [file pattern ...]] 71 | 72 | Python mass editor 73 | 74 | positional arguments: 75 | file pattern shell-like file name patterns to process or - to read 76 | from stdin. 77 | 78 | optional arguments: 79 | -h, --help show this help message and exit 80 | -V, --version show program's version number and exit 81 | -w, --write modify target file(s) in place. Shows diff otherwise. 82 | -v, --verbose increases log verbosity (can be specified multiple 83 | times) 84 | -e EXPRESSIONS, --expression EXPRESSIONS 85 | Python expressions applied to target files. Use the 86 | line variable to reference the current line. 87 | -f FUNCTIONS, --function FUNCTIONS 88 | Python function to apply to target file. Takes file 89 | content as input and yield lines. Specify function as 90 | [module]:?. 91 | -x EXECUTABLES, --executable EXECUTABLES 92 | Python executable to apply to target file. 93 | -s START_DIRS, --start START_DIRS 94 | Directory(ies) from which to look for targets. 95 | -m MAX_DEPTH, --max-depth-level MAX_DEPTH 96 | Maximum depth when walking subdirectories. 97 | -o FILE, --output FILE 98 | redirect output to a file 99 | -g FILE, --generate FILE 100 | generate stub file suitable for -f option 101 | --encoding ENCODING Encoding of input and output files 102 | --newline NEWLINE Newline character for output files 103 | 104 | Examples: 105 | # Simple string substitution (-e). Will show a diff. No changes applied. 106 | massedit.py -e "re.sub('failIf', 'assertFalse', line)" *.py 107 | 108 | # File level modifications (-f). Overwrites the files in place (-w). 109 | massedit.py -w -f fixer:fixit *.py 110 | 111 | # Will change all test*.py in subdirectories of tests. 112 | massedit.py -e "re.sub('failIf', 'assertFalse', line)" -s tests test*.py 113 | 114 | # Will transform virtual methods (almost) to MOCK_METHOD suitable for gmock (see https://github.com/google/googletest). 115 | massedit.py -e "re.sub(r'\s*virtual\s+([\w:<>,\s&*]+)\s+(\w+)(\([^\)]*\))\s*((\w+)*)(=\s*0)?;', 'MOCK_METHOD(\g<1>, \g<2>, \g<3>, (\g<4>, override));', line)" gmock_test.cpp 116 | 117 | 118 | If massedit is installed as a package (from pypi for instance), one can interact with it as a command line tool: 119 | 120 | :: 121 | 122 | python -m massedit -e "re.sub('assertEquals', 'assertEqual', line)" test.py 123 | 124 | 125 | Or as a library (command line option above to be passed as kewyord arguments): 126 | 127 | :: 128 | 129 | >>> import massedit 130 | >>> filenames = ['massedit.py'] 131 | >>> massedit.edit_files(filenames, ["re.sub('Jerome', 'J.', line)"]) 132 | 133 | 134 | Lastly, there is a convenient ``massedit.bat`` wrapper for Windows included in 135 | the distribution. 136 | 137 | 138 | Installation 139 | ------------ 140 | 141 | Download ``massedit.py`` from ``http://github.com/elmotec/massedit`` or : 142 | 143 | :: 144 | 145 | python -m pip install massedit 146 | 147 | 148 | Poor man source-to-source manipulation 149 | -------------------------------------- 150 | 151 | I find myself using massedit mostly for source to source modification of 152 | large code bases like this: 153 | 154 | First create a ``fixer.py`` python module with the function that will 155 | process your source code. For instance, to add a header: 156 | 157 | :: 158 | 159 | def add_header(lines, file_name): 160 | yield '// This is my header' # will be the first line of the file. 161 | for line in lines: 162 | yield line 163 | 164 | 165 | Adds the location of ``fixer.py`` to your ``$PYTHONPATH``, then simply 166 | call ``massedit.py`` like this: 167 | 168 | :: 169 | 170 | massedit.py -f fixer:add_header *.h 171 | 172 | 173 | You can add the ``-s .`` option to process all the ``.h`` files reccursively. 174 | 175 | 176 | Plans 177 | ----- 178 | 179 | - Add support for 3rd party tool (e.g. `autopep8`_) to process the files. 180 | - Add support for a file of expressions as an argument to allow multiple 181 | modification at once. 182 | - Find a satisfactory way (ie. easy to use) to handle multiline regex as the 183 | current version works on a line by line basis. 184 | 185 | 186 | Rationale 187 | --------- 188 | 189 | - I have a hard time practicing more than a few dialects of regular 190 | expressions. 191 | - I need something portable to Windows without being bothered by eol. 192 | - I believe Python is the ideal tool to build something more powerful than 193 | simple regex based substitutions. 194 | 195 | 196 | Background 197 | ---------- 198 | 199 | I have been using runsed and checksed (from Unix Power Tools) for years and 200 | did not find a good substitute under Windows until I came across Graham 201 | Fawcett python recipe 437932_ on ActiveState. It inspired me to write the 202 | massedit. 203 | 204 | The core was fleshed up a little, and here we are. If you find it useful and 205 | enhance it please, do not forget to submit patches. Thanks! 206 | 207 | If you are more interested in awk-like tool, you probably will find pyp_ a 208 | better alternative. 209 | 210 | 211 | Contributing 212 | ------------ 213 | 214 | To set things up for development, the easiest is to pip-install the develop 215 | extra configuration: 216 | 217 | :: 218 | 219 | python -m venv venv 220 | . venv/bin/activate 221 | python -m pip install -e .[develop] 222 | 223 | 224 | The best is to use commitizen_ when performing commits. 225 | 226 | License 227 | ------- 228 | 229 | Licensed under the term of `MIT License`_. See attached file LICENSE.txt. 230 | 231 | 232 | Changes 233 | ------- 234 | 235 | See CHANGELOG.md for changes later than 0.69.0 236 | 237 | 0.69.1 (2023-09-10) 238 | Updated infrastructure files to setup.cfg/pyproject.toml instead of 239 | setup.py. Thanks @isidroas. 240 | 241 | 0.69.0 (2020-12-22) 242 | Also moved CI to github workflows from travis and added 243 | regression tests for Python 2.7. 244 | 245 | 0.68.6 (2019-12-02) 246 | Added support for Python 3.8, stdin input via - argument. Documented 247 | regex to turn base classes into googlemock MOCK_METHOD. 248 | 249 | 0.68.5 (2019-04-13) 250 | Added --newline option to force newline output. Thanks @ALFNeT! 251 | 252 | 0.68.4 (2017-10-24) 253 | Fixed bug that would cause changes to be missed when the -w option is 254 | ommited. Thanks @tgoodlet! 255 | 256 | 0.68.3 (2017-09-20) 257 | Added --generate option to quickly generate a fixer.py template file 258 | to be modified to be used with -f fixer.fixit option. Added official 259 | support for Python 3.6 260 | 261 | 0.68.1 (2016-06-04) 262 | Fixed encoding issues when processing non-ascii files. 263 | Added --encoding option to force the value of the encoding if need be. 264 | Listed support for Python 3.5 265 | 266 | 0.67.1 (2015-06-28) 267 | Documentation fixes. 268 | 269 | 0.67 (2015-06-23) 270 | Added file_name argument to processing functions. 271 | Fixed incorrect closing of sys.stdout/stderr. 272 | Improved diagnostic when the processing function does not take 2 arguments. 273 | Swapped -v and -V option to be consistent with Python. 274 | Pylint fixes. 275 | Added support for Python 3.4. 276 | Dropped support for Python 3.2. 277 | 278 | 0.66 (2013-07-14) 279 | Fixed lost executable bit with -f option (thanks myint). 280 | 281 | 0.65 (2013-07-12) 282 | Added -f option to execute code in a separate file/module. Added Travis continuous integration (thanks myint). Fixed python 2.7 support (thanks myint). 283 | 284 | 0.64 (2013-06-01) 285 | Fixed setup.py so that massedit installs as a script. Fixed eol issues (thanks myint). 286 | 287 | 0.63 (2013-05-27) 288 | Renamed to massedit. Previous version are still known as Python-Mass-Editor. 289 | 290 | 0.62 (2013-04-11) 291 | Fixed bug that caused an EditorError to be raised when the result of the 292 | expression is an empty string. 293 | 294 | 0.61 (2012-07-06) 295 | Added massedit.edit_files function to ease usage as library instead of as 296 | a command line tool (suggested by Maxim Veksler). 297 | 298 | 0.60 (2012-07-04) 299 | Treats arguments as patterns rather than files to ease processing of 300 | multiple files in multiple subdirectories. Added -s (start directory) 301 | and -m (max depth) options. 302 | 303 | 0.52 (2012-06-05) 304 | Upgraded for python 3. Still compatible with python 2.7. 305 | 306 | 0.51 (2012-05) 307 | Initial release (Beta). 308 | 309 | 310 | Contributor acknowledgement 311 | --------------------------- 312 | 313 | https://github.com/myint 314 | https://github.com/tgoodlet 315 | https://github.com/ALFNeT 316 | https://github.com/isidroas 317 | 318 | 319 | 320 | .. _437932: http://code.activestate.com/recipes/437932-pyline-a-grep-like-sed-like-command-line-tool/ 321 | .. _Python re module: http://docs.python.org/library/re.html 322 | .. _Pyp: http://code.google.com/p/pyp/ 323 | .. _MIT License: http://en.wikipedia.org/wiki/MIT_License 324 | .. _autopep8: http://pypi.python.org/pypi/autopep8 325 | .. _Ned Batchelder's article: http://nedbatchelder.com/blog/201206/eval_really_is_dangerous.html 326 | .. _commitizen: https://commitizen-tools.github.io/commitizen/ 327 | -------------------------------------------------------------------------------- /gmock_test.cpp: -------------------------------------------------------------------------------- 1 | #ifndef INC_TEST_GMOCK_REGEX_H 2 | #define INC_TEST_GMOCK_REGEX_H 3 | 4 | /// Simple test class to run against the regex in the documentation. 5 | class Test 6 | { 7 | public: 8 | virtual int simple_method(); 9 | virtual int simple_method_args(int, int); 10 | virtual int simple_const_method_args(int, int) const; 11 | virtual int simple_const_method_vals(int x, int y) const; 12 | virtual std::pair get_pair(); 13 | virtual bool check_map(std::map, bool); 14 | virtual bool transform(Gadget * g) = 0; 15 | virtual Bar & GetBar(); 16 | virtual const Bar & GetBar() const; 17 | }; 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /massedit.bat: -------------------------------------------------------------------------------- 1 | python %~dp0\massedit.py %* 2 | -------------------------------------------------------------------------------- /massedit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | 4 | """A python bulk editor class to apply the same code to many files.""" 5 | 6 | # Copyright (c) 2012-21 Elmotec 7 | # 8 | # Permission is hereby granted, free of charge, to any person obtaining a copy 9 | # of this software and associated documentation files (the "Software"), to deal 10 | # in the Software without restriction, including without limitation the rights 11 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | # copies of the Software, and to permit persons to whom the Software is 13 | # furnished to do so, subject to the following conditions: 14 | # 15 | # The above copyright notice and this permission notice shall be included in 16 | # all copies or substantial portions of the Software. 17 | # 18 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 24 | # THE SOFTWARE. 25 | 26 | from __future__ import unicode_literals 27 | 28 | import argparse 29 | import difflib 30 | import fnmatch 31 | import io 32 | import logging 33 | import os 34 | import re # noqa: F401 pylint: disable=W0611 35 | import shutil 36 | import subprocess 37 | import sys 38 | 39 | __version__ = "0.69.1" # UPDATE setup.cfg when changing version. 40 | __author__ = "Elmotec" 41 | __license__ = "MIT" 42 | 43 | 44 | log = logging.getLogger(__name__) 45 | 46 | 47 | try: 48 | unicode 49 | except NameError: 50 | unicode = str # pylint: disable=invalid-name, redefined-builtin 51 | 52 | 53 | def is_list(arg): 54 | """Factor determination if arg is a list. 55 | 56 | Small utility for a better diagnostic because str/unicode are also 57 | iterable. 58 | 59 | """ 60 | return iter(arg) and not isinstance(arg, unicode) 61 | 62 | 63 | def get_function(fn_name): 64 | """Retrieve the function defined by the function_name. 65 | 66 | Arguments: 67 | fn_name: specification of the type module:function_name. 68 | 69 | """ 70 | module_name, callable_name = fn_name.split(":") 71 | current = globals() 72 | if not callable_name: 73 | callable_name = module_name 74 | else: 75 | import importlib 76 | 77 | try: 78 | module = importlib.import_module(module_name) 79 | except ImportError: 80 | log.error("failed to import %s", module_name) 81 | raise 82 | current = module 83 | for level in callable_name.split("."): 84 | current = getattr(current, level) 85 | code = current.__code__ 86 | if code.co_argcount != 2: 87 | raise ValueError("function should take 2 arguments: lines, file_name") 88 | return current 89 | 90 | 91 | def readlines(input_): 92 | """Return lines from input.""" 93 | try: 94 | return input_.readlines() 95 | except UnicodeDecodeError as err: 96 | log.error("encoding error (see --encoding): %s", err) 97 | raise 98 | 99 | 100 | class MassEdit(object): 101 | 102 | """Mass edit lines of files.""" 103 | 104 | def __init__(self, **kwds): 105 | """Initialize MassEdit object. 106 | 107 | Args: 108 | - code (byte code object): code to execute on input file. 109 | - function (str or callable): function to call on input file. 110 | - module (str): module name where to find the function. 111 | - executable (str): executable file name to execute on input file. 112 | - dry_run (bool): skip actual modification of input file if True. 113 | 114 | """ 115 | self.code_objs = {} 116 | self._codes = [] 117 | self._functions = [] 118 | self._executables = [] 119 | self.dry_run = None 120 | self.encoding = "utf-8" 121 | self.newline = None 122 | if "module" in kwds: 123 | self.import_module(kwds["module"]) 124 | if "code" in kwds: 125 | self.append_code_expr(kwds["code"]) 126 | if "function" in kwds: 127 | self.append_function(kwds["function"]) 128 | if "executable" in kwds: 129 | self.append_executable(kwds["executable"]) 130 | if "dry_run" in kwds: 131 | self.dry_run = kwds["dry_run"] 132 | if "encoding" in kwds: 133 | self.encoding = kwds["encoding"] 134 | if "newline" in kwds: 135 | self.newline = kwds["newline"] 136 | 137 | @staticmethod 138 | def import_module(module): # pylint: disable=R0201 139 | """Import module that are needed for the code expr to compile. 140 | 141 | Argument: 142 | module (str or list): module(s) to import. 143 | 144 | """ 145 | if isinstance(module, list): 146 | all_modules = module 147 | else: 148 | all_modules = [module] 149 | for mod in all_modules: 150 | globals()[mod] = __import__(mod.strip()) 151 | 152 | @staticmethod 153 | def __edit_line(line, code, code_obj): # pylint: disable=R0201 154 | """Edit a line with one code object built in the ctor.""" 155 | try: 156 | # pylint: disable=eval-used 157 | result = eval(code_obj, globals(), locals()) 158 | except TypeError as ex: 159 | log.error("failed to execute %s: %s", code, ex) 160 | raise 161 | if result is None: 162 | log.error("cannot process line '%s' with %s", line, code) 163 | raise RuntimeError("failed to process line") 164 | elif isinstance(result, list) or isinstance(result, tuple): 165 | line = unicode(" ".join([unicode(res_element) for res_element in result])) 166 | else: 167 | line = unicode(result) 168 | return line 169 | 170 | def edit_line(self, line): 171 | """Edit a single line using the code expression.""" 172 | for code, code_obj in self.code_objs.items(): 173 | line = self.__edit_line(line, code, code_obj) 174 | return line 175 | 176 | def edit_content(self, original_lines, file_name): 177 | """Processes a file contents. 178 | 179 | First processes the contents line by line applying the registered 180 | expressions, then process the resulting contents using the 181 | registered functions. 182 | 183 | Arguments: 184 | original_lines (list of str): file content. 185 | file_name (str): name of the file. 186 | 187 | """ 188 | lines = [self.edit_line(line) for line in original_lines] 189 | for function in self._functions: 190 | try: 191 | lines = list(function(lines, file_name)) 192 | except UnicodeDecodeError as err: 193 | log.error("failed to process %s: %s", file_name, err) 194 | return lines 195 | except Exception as err: 196 | log.error( 197 | "failed to process %s with code %s: %s", file_name, function, err 198 | ) 199 | raise # Let the exception be handled at a higher level. 200 | return lines 201 | 202 | def write_to(self, file_name, to_lines): 203 | """Writes output lines to file.""" 204 | bak_file_name = file_name + ".bak" 205 | if os.path.exists(bak_file_name): 206 | msg = "{} already exists".format(bak_file_name) 207 | if sys.version_info < (3, 3): 208 | raise OSError(msg) 209 | else: 210 | # noinspection PyCompatibility 211 | # pylint: disable=undefined-variable 212 | raise FileExistsError(msg) 213 | try: 214 | os.rename(file_name, bak_file_name) 215 | with io.open( 216 | file_name, "w", encoding=self.encoding, newline=self.newline 217 | ) as new: 218 | new.writelines(to_lines) 219 | # Keeps mode of original file. 220 | shutil.copymode(bak_file_name, file_name) 221 | except Exception as err: 222 | log.error("failed to write output to %s: %s", file_name, err) 223 | # Try to recover... 224 | try: 225 | os.rename(bak_file_name, file_name) 226 | except OSError as err: 227 | log.error( 228 | "failed to restore %s from %s: %s", 229 | file_name, 230 | bak_file_name, 231 | err, 232 | ) 233 | raise 234 | try: 235 | os.unlink(bak_file_name) 236 | except OSError as err: 237 | log.warning("failed to remove backup %s: %s", bak_file_name, err) 238 | 239 | def edit_file(self, file_name): 240 | """Edit file in place, returns a list of modifications (unified diff). 241 | 242 | Arguments: 243 | file_name (str, unicode): The name of the file. 244 | 245 | """ 246 | if file_name == "-": 247 | from_lines = readlines(sys.stdin) 248 | else: 249 | with io.open(file_name, "r", encoding=self.encoding) as from_file: 250 | from_lines = readlines(from_file) 251 | 252 | if self._executables: 253 | nb_execs = len(self._executables) 254 | if nb_execs > 1: 255 | log.warning("found %d executables. Will use first one", nb_execs) 256 | exec_list = self._executables[0].split() 257 | exec_list.append(file_name) 258 | try: 259 | log.info("running %s...", " ".join(exec_list)) 260 | output = subprocess.check_output(exec_list, universal_newlines=True) 261 | except Exception as err: 262 | log.error("failed to execute %s: %s", " ".join(exec_list), err) 263 | raise # Let the exception be handled at a higher level. 264 | to_lines = output.split(unicode("\n")) 265 | else: 266 | to_lines = from_lines 267 | 268 | # unified_diff wants structure of known length. Convert to a list. 269 | to_lines = list(self.edit_content(to_lines, file_name)) 270 | diffs = difflib.unified_diff( 271 | from_lines, to_lines, fromfile=file_name, tofile="" 272 | ) 273 | if not self.dry_run: 274 | if file_name == "-": 275 | sys.stdout.writelines(to_lines) 276 | else: 277 | self.write_to(file_name, to_lines) 278 | return list(diffs) 279 | 280 | def append_code_expr(self, code): 281 | """Compile argument and adds it to the list of code objects.""" 282 | # expects a string. 283 | if isinstance(code, str) and not isinstance(code, unicode): 284 | code = unicode(code) 285 | if not isinstance(code, unicode): 286 | raise TypeError("string expected") 287 | log.debug("compiling code %s...", code) 288 | try: 289 | code_obj = compile(code, "", "eval") 290 | self.code_objs[code] = code_obj 291 | except SyntaxError as syntax_err: 292 | log.error("cannot compile %s: %s", code, syntax_err) 293 | raise 294 | log.debug("compiled code %s", code) 295 | 296 | def append_function(self, function): 297 | """Append the function to the list of functions to be called. 298 | 299 | If the function is already a callable, use it. If it's a type str 300 | try to interpret it as [module]:?, load the module 301 | if there is one and retrieve the callable. 302 | 303 | Argument: 304 | function (str or callable): function to call on input. 305 | 306 | """ 307 | if not hasattr(function, "__call__"): 308 | function = get_function(function) 309 | if not hasattr(function, "__call__"): 310 | raise ValueError("function is expected to be callable") 311 | self._functions.append(function) 312 | log.debug("registered %s", function.__name__) 313 | 314 | def append_executable(self, executable): 315 | """Append san executable os command to the list to be called. 316 | 317 | Argument: 318 | executable (str): os callable executable. 319 | 320 | """ 321 | if isinstance(executable, str) and not isinstance(executable, unicode): 322 | executable = unicode(executable) 323 | if not isinstance(executable, unicode): 324 | raise TypeError( 325 | "expected executable name as str, not {}".format( 326 | executable.__class__.__name__ 327 | ) 328 | ) 329 | self._executables.append(executable) 330 | 331 | def set_code_exprs(self, codes): 332 | """Convenience: sets all the code expressions at once.""" 333 | self.code_objs = {} 334 | self._codes = [] 335 | for code in codes: 336 | self.append_code_expr(code) 337 | 338 | def set_functions(self, functions): 339 | """Check functions passed as argument and set them to be used.""" 340 | for func in functions: 341 | try: 342 | self.append_function(func) 343 | except (ValueError, AttributeError) as ex: 344 | log.error("'%s' is not a callable function: %s", func, ex) 345 | raise 346 | 347 | def set_executables(self, executables): 348 | """Check and set the executables to be used.""" 349 | for exc in executables: 350 | self.append_executable(exc) 351 | 352 | 353 | def parse_command_line(argv): 354 | """Parse command line argument. See -h option. 355 | 356 | Arguments: 357 | argv: arguments on the command line must include caller file name. 358 | 359 | """ 360 | import textwrap 361 | 362 | example = textwrap.dedent( 363 | r""" 364 | Examples: 365 | # Simple string substitution (-e). Will show a diff. No changes applied. 366 | {0} -e "re.sub('failIf', 'assertFalse', line)" *.py 367 | 368 | # File level modifications (-f). Overwrites the files in place (-w). 369 | {0} -w -f fixer:fixit *.py 370 | 371 | # Will change all test*.py in subdirectories of tests. 372 | {0} -e "re.sub('failIf', 'assertFalse', line)" -s tests test*.py 373 | 374 | # Will transform virtual methods (almost) to MOCK_METHOD suitable for gmock (see https://github.com/google/googletest). 375 | {0} -e "re.sub(r'\s*virtual\s+([\w:<>,\s&*]+)\s+(\w+)(\([^\)]*\))\s*((\w+)*)(=\s*0)?;', 'MOCK_METHOD(\g<1>, \g<2>, \g<3>, (\g<4>, override));', line)" test.cpp 376 | """ 377 | ).format(os.path.basename(argv[0])) 378 | formatter_class = argparse.RawDescriptionHelpFormatter 379 | parser = argparse.ArgumentParser( 380 | description="Python mass editor", 381 | epilog=example, 382 | formatter_class=formatter_class, 383 | ) 384 | parser.add_argument( 385 | "-V", "--version", action="version", version="%(prog)s {}".format(__version__) 386 | ) 387 | parser.add_argument( 388 | "-w", 389 | "--write", 390 | dest="dry_run", 391 | action="store_false", 392 | default=True, 393 | help="modify target file(s) in place. " "Shows diff otherwise.", 394 | ) 395 | parser.add_argument( 396 | "-v", 397 | "--verbose", 398 | dest="verbose_count", 399 | action="count", 400 | default=0, 401 | help="increases log verbosity (can be specified " "multiple times)", 402 | ) 403 | parser.add_argument( 404 | "-e", 405 | "--expression", 406 | dest="expressions", 407 | nargs=1, 408 | help="Python expressions applied to target files. " 409 | "Use the line variable to reference the current line.", 410 | ) 411 | parser.add_argument( 412 | "-f", 413 | "--function", 414 | dest="functions", 415 | nargs=1, 416 | help="Python function to apply to target file. " 417 | "Takes file content as input and yield lines. " 418 | "Specify function as [module]:?.", 419 | ) 420 | parser.add_argument( 421 | "-x", 422 | "--executable", 423 | dest="executables", 424 | nargs=1, 425 | help="Python executable to apply to target file.", 426 | ) 427 | parser.add_argument( 428 | "-s", 429 | "--start", 430 | dest="start_dirs", 431 | help="Directory(ies) from which to look for targets.", 432 | ) 433 | parser.add_argument( 434 | "-m", 435 | "--max-depth-level", 436 | type=int, 437 | dest="max_depth", 438 | help="Maximum depth when walking subdirectories.", 439 | ) 440 | parser.add_argument( 441 | "-o", 442 | "--output", 443 | metavar="FILE", 444 | type=argparse.FileType("w"), 445 | default=sys.stdout, 446 | help="redirect output to a file", 447 | ) 448 | parser.add_argument( 449 | "-g", 450 | "--generate", 451 | metavar="FILE", 452 | type=str, 453 | help="generate stub file suitable for -f option", 454 | ) 455 | parser.add_argument( 456 | "--encoding", dest="encoding", help="Encoding of input and output files" 457 | ) 458 | parser.add_argument( 459 | "--newline", dest="newline", help="Newline character for output files" 460 | ) 461 | parser.add_argument( 462 | "patterns", 463 | metavar="file pattern", 464 | nargs="*", # argparse.REMAINDER, 465 | help="shell-like file name patterns to process or - to read from stdin.", 466 | ) 467 | arguments = parser.parse_args(argv[1:]) 468 | 469 | if not ( 470 | arguments.expressions 471 | or arguments.functions 472 | or arguments.generate 473 | or arguments.executables 474 | ): 475 | parser.error("--expression, --function, --generate or --executable missing") 476 | 477 | # Sets log level to WARN going more verbose for each new -V. 478 | log.setLevel(max(3 - arguments.verbose_count, 0) * 10) 479 | return arguments 480 | 481 | 482 | def get_paths(patterns, start_dirs=None, max_depth=1): 483 | """Retrieve files that match any of the patterns.""" 484 | # Shortcut: if there is only one pattern, make sure we process just that. 485 | if len(patterns) == 1 and not start_dirs: 486 | pattern = patterns[0] 487 | if pattern == "-": 488 | yield pattern 489 | return 490 | directory = os.path.dirname(pattern) 491 | if directory: 492 | patterns = [os.path.basename(pattern)] 493 | start_dirs = directory 494 | max_depth = 1 495 | 496 | if not start_dirs or start_dirs == ".": 497 | start_dirs = os.getcwd() 498 | for start_dir in start_dirs.split(","): 499 | for root, dirs, files in os.walk(start_dir): # pylint: disable=W0612 500 | if max_depth is not None: 501 | relpath = os.path.relpath(root, start=start_dir) 502 | depth = len(relpath.split(os.sep)) 503 | if depth > max_depth: 504 | continue 505 | names = [] 506 | for pattern in patterns: 507 | names += fnmatch.filter(files, pattern) 508 | for name in names: 509 | path = os.path.join(root, name) 510 | yield path 511 | 512 | 513 | fixer_template = """\ 514 | #!/usr/bin/env python 515 | 516 | def fixit(lines, file_name): 517 | '''Edit files passed to massedit 518 | 519 | :param list(str) lines: list of lines contained in the input file 520 | :param str file_name: name of the file the lines were read from 521 | 522 | :return: modified lines 523 | :rtype: list(str) 524 | 525 | Please modify the logic below (it does not change anything right now) 526 | and apply your logic to the in your directory like this: 527 | 528 | massedit -f :fixit files_to_modify/* 529 | 530 | See massedit -h for help and other options. 531 | 532 | ''' 533 | changed_lines = [] 534 | for lineno, line in enumerate(lines): 535 | # See https://regex101.com/?filterFlavors=python 536 | changed_line = line.sub(pat, repl, line) 537 | changed_lines.append(changed_line) 538 | return changed_lines 539 | 540 | 541 | """ 542 | 543 | 544 | def generate_fixer_file(output): 545 | """Generate a template fixer file to be used with --function option.""" 546 | with open(output, "w+") as fh: 547 | fh.write(fixer_template) 548 | return 549 | 550 | 551 | # pylint: disable=too-many-arguments, too-many-locals 552 | def edit_files( 553 | patterns, 554 | expressions=None, 555 | functions=None, 556 | executables=None, 557 | start_dirs=None, 558 | max_depth=1, 559 | dry_run=True, 560 | output=sys.stdout, 561 | encoding=None, 562 | newline=None, 563 | ): 564 | """Process patterns with MassEdit. 565 | 566 | Arguments: 567 | patterns: file pattern to identify the files to be processed. 568 | expressions: single python expression to be applied line by line. 569 | functions: functions to process files contents. 570 | executables: os executables to execute on the argument files. 571 | max_depth: maximum recursion level when looking for file matches. 572 | start_dirs: workspace(ies) where to start the file search. 573 | dry_run: only display differences if True. Save modified file otherwise. 574 | output: handle where the output should be redirected. 575 | 576 | Return: 577 | list of files processed. 578 | 579 | """ 580 | if not is_list(patterns): 581 | raise TypeError("patterns should be a list") 582 | if expressions and not is_list(expressions): 583 | raise TypeError("expressions should be a list of exec expressions") 584 | if functions and not is_list(functions): 585 | raise TypeError("functions should be a list of functions") 586 | if executables and not is_list(executables): 587 | raise TypeError("executables should be a list of program names") 588 | 589 | editor = MassEdit(dry_run=dry_run, encoding=encoding, newline=newline) 590 | if expressions: 591 | editor.set_code_exprs(expressions) 592 | if functions: 593 | editor.set_functions(functions) 594 | if executables: 595 | editor.set_executables(executables) 596 | 597 | processed_paths = [] 598 | for path in get_paths(patterns, start_dirs=start_dirs, max_depth=max_depth): 599 | try: 600 | diffs = list(editor.edit_file(path)) 601 | if dry_run: 602 | # At this point, encoding is the input encoding. 603 | diff = "".join(diffs) 604 | if not diff: 605 | continue 606 | # The encoding of the target output may not match the input 607 | # encoding. If it's defined, we round trip the diff text 608 | # to bytes and back to silence any conversion errors. 609 | encoding = output.encoding 610 | if encoding: 611 | bytes_diff = diff.encode(encoding=encoding, errors="ignore") 612 | diff = bytes_diff.decode(encoding=output.encoding) 613 | output.write(diff) 614 | except UnicodeDecodeError as err: 615 | log.error("failed to process %s: %s", path, err) 616 | continue 617 | processed_paths.append(os.path.abspath(path)) 618 | return processed_paths 619 | 620 | 621 | def command_line(argv): 622 | """Instantiate an editor and process arguments. 623 | 624 | Optional argument: 625 | - processed_paths: paths processed are appended to the list. 626 | 627 | """ 628 | arguments = parse_command_line(argv) 629 | if arguments.generate: 630 | generate_fixer_file(arguments.generate) 631 | paths = edit_files( 632 | arguments.patterns, 633 | expressions=arguments.expressions, 634 | functions=arguments.functions, 635 | executables=arguments.executables, 636 | start_dirs=arguments.start_dirs, 637 | max_depth=arguments.max_depth, 638 | dry_run=arguments.dry_run, 639 | output=arguments.output, 640 | encoding=arguments.encoding, 641 | newline=arguments.newline, 642 | ) 643 | # If the output is not sys.stdout, we need to close it because 644 | # argparse.FileType does not do it for us. 645 | is_sys = arguments.output in [sys.stdout, sys.stderr] 646 | if not is_sys and isinstance(arguments.output, io.IOBase): 647 | arguments.output.close() 648 | return paths 649 | 650 | 651 | def main(): 652 | """Main function.""" 653 | logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) 654 | try: 655 | command_line(sys.argv) 656 | finally: 657 | logging.shutdown() 658 | 659 | 660 | if __name__ == "__main__": 661 | sys.exit(main()) 662 | -------------------------------------------------------------------------------- /output.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elmotec/massedit/a6f8c91aebc2870654968b3857cf5deb1d8c0e1a/output.txt -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "massedit" 3 | description = "Edit multiple files using Python text processing modules" 4 | version = "0.70.0" 5 | authors = [{name = "elmotec", email = "elmotec@gmx.com"}] 6 | license = {text = "MIT"} 7 | keywords = [ 8 | "sed", 9 | "editor", 10 | "stream", 11 | "python", 12 | "edit", 13 | "mass" 14 | ] 15 | classifiers = [ 16 | "Development Status :: 5 - Production/Stable", 17 | "License :: OSI Approved :: MIT License", 18 | "Environment :: Console", 19 | "Natural Language :: English", 20 | "Programming Language :: Python :: 3.7", 21 | "Programming Language :: Python :: 3.8", 22 | "Programming Language :: Python :: 3.9", 23 | "Programming Language :: Python :: 3.10", 24 | "Programming Language :: Python :: 3.11", 25 | "Topic :: Software Development", 26 | "Topic :: Software Development :: Code Generators", 27 | "Topic :: Text Editors :: Text Processing", 28 | "Topic :: Text Processing :: Filters", 29 | "Topic :: Utilities", 30 | "Intended Audience :: Developers", 31 | ] 32 | urls = {Homepage = "http://github.com/elmotec/massedit"} 33 | requires-python = ">=2.7" # not supported before min version in classifier. 34 | 35 | [project.readme] 36 | file = "README.rst" 37 | content-type = "text/x-rst" 38 | 39 | [project.scripts] 40 | massedit='massedit:main' 41 | 42 | [project.optional-dependencies] 43 | test = [ 44 | "flake8", 45 | "tox", 46 | ] 47 | 48 | develop = [ 49 | "black", 50 | "commitizen", 51 | "flake8", 52 | "pre-commit", 53 | "pylint", 54 | "tox", 55 | ] 56 | 57 | [build-system] 58 | build-backend = "setuptools.build_meta" 59 | requires = ["setuptools>=61.2"] 60 | 61 | [tool.commitizen] 62 | name = "cz_conventional_commits" 63 | tag_format = "v$version" 64 | version_scheme = "semver" 65 | version_provider = "pep621" 66 | update_changelog_on_bump = true 67 | major_version_zero = true 68 | 69 | [tool.tox] 70 | legacy_tox_ini = """ 71 | [tox] 72 | envlist = py3{7,8,9,10,11} 73 | isolated_build = True 74 | 75 | [testenv] 76 | description = Unit tests 77 | commands = python tests.py 78 | """ 79 | -------------------------------------------------------------------------------- /tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | """Test module to test massedit.""" 5 | 6 | # Copyright (c) 2012-17 Jérôme Lecomte 7 | # 8 | # Permission is hereby granted, free of charge, to any person obtaining a copy 9 | # of this software and associated documentation files (the "Software"), to deal 10 | # in the Software without restriction, including without limitation the rights 11 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | # copies of the Software, and to permit persons to whom the Software is 13 | # furnished to do so, subject to the following conditions: 14 | # 15 | # The above copyright notice and this permission notice shall be included in 16 | # all copies or substantial portions of the Software. 17 | # 18 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 24 | # THE SOFTWARE. 25 | 26 | from __future__ import unicode_literals 27 | 28 | import io 29 | import logging 30 | import os 31 | import platform 32 | import shutil 33 | import sys 34 | import tempfile 35 | import textwrap 36 | import unittest 37 | 38 | import massedit 39 | 40 | if sys.version_info < (3, 3): 41 | import mock # pylint: disable=import-error 42 | 43 | builtins = "__builtin__" 44 | else: 45 | from unittest import mock # pylint: disable=import-error, no-name-in-module 46 | 47 | builtins = "builtins" 48 | 49 | 50 | try: 51 | unicode 52 | except NameError: 53 | unicode = str # pylint: disable=invalid-name, redefined-builtin 54 | 55 | 56 | zen = unicode( 57 | """The Zen of Python, by Tim Peters 58 | 59 | Beautiful is better than ugly. 60 | Explicit is better than implicit. 61 | Simple is better than complex. 62 | Complex is better than complicated. 63 | Flat is better than nested. 64 | Sparse is better than dense. 65 | Readability counts. 66 | Special cases aren't special enough to break the rules. 67 | Although practicality beats purity. 68 | Errors should never pass silently. 69 | Unless explicitly silenced. 70 | In the face of ambiguity, refuse the temptation to guess. 71 | There should be one-- and preferably only one --obvious way to do it. 72 | Although that way may not be obvious at first unless you're Dutch. 73 | Now is better than never. 74 | Although never is often better than *right* now. 75 | If the implementation is hard to explain, it's a bad idea. 76 | If the implementation is easy to explain, it may be a good idea. 77 | Namespaces are one honking great idea -- let's do more of those! 78 | """ 79 | ) 80 | 81 | 82 | class Workspace: 83 | """Wraps creation of files/workspace. 84 | 85 | For some reason tempfile.mkdtemp() causes problems with Python 2.7: 86 | File "C:\\Python27\\lib\\random.py", line 113, in seed 87 | a = long(_hexlify(_urandom(2500)), 16) 88 | TypeError: 'NoneType' object is not callable 89 | 90 | """ 91 | 92 | def __init__(self, parent_dir=None): 93 | """Initialize Temp class. 94 | 95 | Arguments: 96 | parent_dir (str): workspace where to create temporary files/dirs. 97 | 98 | If parent_dir is not given, uses tempfile pacakge to try to get a 99 | temporary directory. If that fails, use the current working directory. 100 | 101 | """ 102 | 103 | if parent_dir is None: 104 | parent_dir = tempfile.tempdir 105 | if parent_dir is None: 106 | parent_dir = os.path.abspath(os.path.curdir) 107 | self.top_dir = self.get_directory(parent_dir=parent_dir) 108 | 109 | # Cannot use __del__ here because self.top_dir does not always exist 110 | # when _del__ is called: http://stackoverflow.com/questions/865115/ 111 | def cleanup(self): 112 | """Delete temporary directories/files.""" 113 | shutil.rmtree(self.top_dir) 114 | 115 | def get_base_name(self): 116 | """Create a base""" 117 | import binascii 118 | 119 | suffix = binascii.hexlify(os.urandom(4)).decode("ascii") 120 | return "massedit" + suffix 121 | 122 | def get_directory(self, parent_dir=None): 123 | """Create a temporary directory in parent_dir.""" 124 | 125 | if parent_dir is None: 126 | parent_dir = self.top_dir 127 | dir_name = os.path.join(parent_dir, self.get_base_name()) 128 | os.mkdir(dir_name) 129 | return dir_name 130 | 131 | def get_file(self, parent_dir=None, extension=None): 132 | """Get a new temporary file name.""" 133 | 134 | if not parent_dir: 135 | parent_dir = self.top_dir 136 | file_name = os.path.join(parent_dir, self.get_base_name()) 137 | if extension: 138 | file_name += extension 139 | return file_name 140 | 141 | 142 | class LogInterceptor: # pylint: disable=too-few-public-methods 143 | """Replaces all log handlers and redirect log to the stream.""" 144 | 145 | def __init__(self, logger): 146 | """Sets up log handler for logger and remove all existing handlers. 147 | 148 | Arguments: 149 | logger (logging.Logger): logger to be modified. 150 | 151 | Sets up variables: 152 | self.__content (io.StringIO): stores the log. 153 | self.handler (logging.StreamHandler): handler for self.__content. 154 | self.logger (logging.Logger): the logger to intercept. 155 | 156 | """ 157 | # Stores original values. 158 | self.__handlers = [] 159 | self.__propagate = logger.propagate 160 | self.__content = io.StringIO() 161 | self.logger = logger 162 | self.logger.propagate = False 163 | self.handler = logging.StreamHandler(self.__content) 164 | for handler in logger.handlers: 165 | self.__handlers.append(handler) 166 | logger.removeHandler(handler) 167 | logger.addHandler(self.handler) 168 | 169 | @property 170 | def log(self): 171 | """Flush the handler and return the content of self.__content.""" 172 | self.handler.flush() 173 | return self.__content.getvalue() 174 | 175 | def __del__(self): 176 | """Reset the handlers the way they were.""" 177 | self.logger.removeHandler(self.handler) 178 | for hdlr in self.__handlers: 179 | self.logger.addHandler(hdlr) 180 | self.logger.propagate = self.__propagate 181 | 182 | 183 | def dutch_is_guido(lines, _): 184 | """Helper function that substitute Dutch with Guido.""" 185 | import re 186 | 187 | for line in lines: 188 | yield re.sub("Dutch", "Guido", line) 189 | 190 | 191 | def remove_module(module_name): 192 | """Remove the module from memory.""" 193 | if module_name in sys.modules: 194 | del sys.modules[module_name] 195 | 196 | 197 | class TestGetFunction(unittest.TestCase): # pylint: disable=R0904 198 | 199 | """Test the functon get_function.""" 200 | 201 | def test_simple_retrieval(self): 202 | """test retrieval of function in argument string.""" 203 | function = massedit.get_function("tests:dutch_is_guido") 204 | # Functions are not the same but the code is. 205 | self.assertEqual(dutch_is_guido.__code__, function.__code__) 206 | 207 | 208 | class TestMassEdit(unittest.TestCase): # pylint: disable=R0904 209 | 210 | """Test the massedit module.""" 211 | 212 | def setUp(self): 213 | self.editor = massedit.MassEdit() 214 | 215 | def tearDown(self): 216 | del self.editor 217 | 218 | def test_no_change(self): 219 | """Test the editor does nothing when not told to do anything.""" 220 | input_line = "some info" 221 | output_line = self.editor.edit_line(input_line) 222 | self.assertEqual(output_line, input_line) 223 | 224 | def test_simple_replace(self): 225 | """Simple replacement check.""" 226 | original_line = "What a nice cat!" 227 | self.editor.append_code_expr("re.sub('cat','horse',line)") 228 | new_line = self.editor.edit_line(original_line) 229 | self.assertEqual(new_line, "What a nice horse!") 230 | self.assertEqual(original_line, "What a nice cat!") 231 | 232 | def test_replace_all(self): 233 | """Test replacement of an entire line.""" 234 | original_line = "all of it" 235 | self.editor.append_code_expr("re.sub('all of it', '', line)") 236 | new_line = self.editor.edit_line(original_line) 237 | self.assertEqual(new_line, "") 238 | 239 | def test_syntax_error(self): 240 | """Check we get a SyntaxError if the code is not valid.""" 241 | with mock.patch("massedit.log", autospec=True): 242 | with self.assertRaises(SyntaxError): 243 | self.editor.append_code_expr("invalid expression") 244 | self.assertIsNone(self.editor.code_objs) 245 | 246 | def test_invalid_code_expr2(self): 247 | """Check we get a SyntaxError if the code is missing an argument.""" 248 | self.editor.append_code_expr("re.sub('def test', 'def toast')") 249 | massedit.log.disabled = True 250 | with self.assertRaises(TypeError): 251 | self.editor.edit_line("some line") 252 | massedit.log.disabled = False 253 | 254 | def test_missing_module(self): 255 | """Check that missing module generates an exception.""" 256 | self.editor.append_code_expr("random.randint(0,10)") 257 | with self.assertRaises(NameError): 258 | self.editor.edit_line("need to edit a line to execute the code") 259 | 260 | def test_module_import(self): 261 | """Check the module import functinality.""" 262 | remove_module("random") 263 | self.editor.import_module("random") 264 | self.editor.append_code_expr("random.randint(0,9)") 265 | random_number = self.editor.edit_line("to be replaced") 266 | self.assertIn(random_number, [str(x) for x in range(10)]) 267 | 268 | def test_file_edit(self): 269 | """Simple replacement check.""" 270 | original_file = zen.split("\n") 271 | self.editor.append_function(dutch_is_guido) 272 | actual_file = list(self.editor.edit_content(original_file, "filename")) 273 | expected_file = original_file 274 | expected_file[15] = ( 275 | "Although that way may not be obvious " "at first unless you're Guido." 276 | ) 277 | self.editor.max_diff = None 278 | self.assertEqual(actual_file, expected_file) 279 | 280 | 281 | class TestMassEditWithFile(unittest.TestCase): 282 | 283 | """Test massedit with an actual file.""" 284 | 285 | def setUp(self): 286 | self.editor = massedit.MassEdit() 287 | self.workspace = Workspace() 288 | self.file_name = os.path.join(self.workspace.top_dir, unicode("somefile.txt")) 289 | 290 | def tearDown(self): 291 | """Remove the temporary file.""" 292 | self.workspace.cleanup() 293 | 294 | def write_input_file(self, text, encoding=None): 295 | """Write text in input file. 296 | 297 | :param encoding: defaults to utf-8 298 | 299 | """ 300 | if not encoding: 301 | encoding = "utf-8" 302 | with io.open(self.file_name, "w+", encoding=encoding) as fh: 303 | fh.write(text) 304 | 305 | def test_non_utf8_with_utf8_setting(self): 306 | """Check files with non-utf8 characters are skipped with a warning.""" 307 | log_sink = LogInterceptor(massedit.log) 308 | content = unicode("This is ok\nThis \u00F1ot") 309 | self.write_input_file(content, encoding="cp1252") 310 | 311 | def identity(lines, _): 312 | """Return the line itself.""" 313 | for line in lines: 314 | yield line 315 | 316 | self.editor.append_function(identity) 317 | with self.assertRaises(UnicodeDecodeError): 318 | _ = self.editor.edit_file(self.file_name) 319 | self.assertIn("encoding error", log_sink.log) 320 | 321 | def test_handling_of_cp1252(self): 322 | """Check files with non-utf8 characters are skipped with a warning.""" 323 | encoding = "cp1252" 324 | self.editor.encoding = encoding 325 | content = unicode("This is ok\nThis \u00F1ot") 326 | self.write_input_file(content, encoding=encoding) 327 | 328 | def identity(lines, _): 329 | """Return the line itself.""" 330 | for line in lines: 331 | yield line 332 | 333 | self.editor.append_function(identity) 334 | diffs = self.editor.edit_file(self.file_name) 335 | self.assertEqual(diffs, []) 336 | 337 | def test_forcing_end_of_line_for_output_files(self): 338 | """Check files with CRLF are created with LF when using newline setting""" 339 | self.editor.newline = "\n" 340 | 341 | content = "This is a line finishing with CRLF\r\n" 342 | 343 | self.write_input_file(content) 344 | 345 | def identity(lines, _): 346 | """Return the line itself.""" 347 | for line in lines: 348 | yield line 349 | 350 | self.editor.append_function(identity) 351 | diffs = self.editor.edit_file(self.file_name) 352 | 353 | self.assertEqual(diffs, []) 354 | 355 | with io.open(self.file_name) as f: 356 | f.readline() 357 | output_newline = f.newlines 358 | 359 | expected_eol = self.editor.newline 360 | if expected_eol is None: 361 | # If not specified use the string to terminate lines on the current platform 362 | expected_eol = os.linesep 363 | 364 | self.assertEqual(expected_eol, output_newline) 365 | 366 | 367 | class TestMassEditWithZenFile(TestMassEditWithFile): # pylint: disable=R0904 368 | 369 | """Test the command line interface of massedit.py with actual file.""" 370 | 371 | def setUp(self): 372 | """Use zen of Python as content.""" 373 | TestMassEditWithFile.setUp(self) 374 | self.write_input_file(zen) 375 | 376 | def test_setup(self): 377 | """Check that we have a temporary file to work with.""" 378 | self.assertTrue(os.path.exists(self.file_name)) 379 | 380 | def test_replace_in_file(self): 381 | """Check editing of an entire file.""" 382 | self.editor.append_code_expr("re.sub('Dutch', 'Guido', line)") 383 | diffs = self.editor.edit_file(self.file_name) 384 | self.assertEqual(len(diffs), 11) 385 | expected_diffs = textwrap.dedent( 386 | """ 387 | There should be one-- and preferably only one --obvious way to do it. 388 | -Although that way may not be obvious at first unless you're Dutch. 389 | +Although that way may not be obvious at first unless you're Guido. 390 | Now is better than never.\n""" 391 | ) 392 | self.assertEqual("".join(diffs[5:9]), "".join(expected_diffs[1:])) 393 | 394 | def test_replace_cannot_backup(self): 395 | """Check replacement fails if backup fails.""" 396 | self.editor.append_code_expr("re.sub('Dutch', 'Guido', line)") 397 | backup = self.file_name + ".bak" 398 | try: 399 | shutil.copy(self.file_name, backup) 400 | # FileExistsError in more recent version of Python. 401 | with self.assertRaises(OSError): 402 | self.editor.edit_file(self.file_name) 403 | finally: 404 | os.unlink(backup) 405 | 406 | def test_command_line_replace(self): 407 | """Check simple replacement via command line.""" 408 | file_base_name = os.path.basename(self.file_name) 409 | massedit.command_line( 410 | [ 411 | "massedit.py", 412 | "-w", 413 | "-e", 414 | "re.sub('Dutch', 'Guido', line)", 415 | "-w", 416 | "-s", 417 | self.workspace.top_dir, 418 | file_base_name, 419 | ] 420 | ) 421 | with io.open(self.file_name, "r") as new_file: 422 | new_lines = new_file.readlines() 423 | original_lines = zen.splitlines(True) 424 | self.assertEqual(len(new_lines), len(original_lines)) 425 | n_lines = len(new_lines) 426 | for line in range(n_lines): 427 | if line != 16: 428 | self.assertEqual(new_lines[line - 1], original_lines[line - 1]) 429 | else: 430 | expected_line_16 = ( 431 | "Although that way may not be obvious " 432 | + "at first unless you're Guido.\n" 433 | ) 434 | self.assertEqual(new_lines[line - 1], expected_line_16) 435 | 436 | def test_command_line_check(self): 437 | """Check dry run via command line with start workspace option.""" 438 | out_file_name = self.workspace.get_file() 439 | basename = os.path.basename(self.file_name) 440 | arguments = [ 441 | "test", 442 | "-e", 443 | "re.sub('Dutch', 'Guido', line)", 444 | "-o", 445 | out_file_name, 446 | "-s", 447 | self.workspace.top_dir, 448 | basename, 449 | ] 450 | processed = massedit.command_line(arguments) 451 | self.assertEqual(processed, [os.path.abspath(self.file_name)]) 452 | with io.open(self.file_name, "r") as updated_file: 453 | new_lines = updated_file.readlines() 454 | original_lines = zen.splitlines(True) 455 | self.assertEqual(original_lines, new_lines) 456 | self.assertTrue(os.path.exists(out_file_name)) 457 | os.unlink(out_file_name) 458 | 459 | def test_absolute_path_arg(self): 460 | """Check dry run via command line with single file name argument.""" 461 | out_file_name = self.workspace.get_file() 462 | arguments = [ 463 | "massedit.py", 464 | "-e", 465 | "re.sub('Dutch', 'Guido', line)", 466 | "-o", 467 | out_file_name, 468 | self.file_name, 469 | ] 470 | processed = massedit.command_line(arguments) 471 | self.assertEqual(processed, [os.path.abspath(self.file_name)]) 472 | with io.open(self.file_name, "r") as updated_file: 473 | new_lines = updated_file.readlines() 474 | original_lines = zen.splitlines(True) 475 | self.assertEqual(original_lines, new_lines) 476 | self.assertTrue(os.path.exists(out_file_name)) 477 | os.unlink(out_file_name) 478 | 479 | def test_api(self): 480 | """Check simple replacement via api.""" 481 | file_base_name = os.path.basename(self.file_name) 482 | processed = massedit.edit_files( 483 | [file_base_name], 484 | ["re.sub('Dutch', 'Guido', line)"], 485 | [], 486 | start_dirs=self.workspace.top_dir, 487 | dry_run=False, 488 | ) 489 | self.assertEqual(processed, [self.file_name]) 490 | with io.open(self.file_name, "r") as new_file: 491 | new_lines = new_file.readlines() 492 | original_lines = zen.splitlines(True) 493 | self.assertEqual(len(new_lines), len(original_lines)) 494 | n_lines = len(new_lines) 495 | for line in range(n_lines): 496 | if line != 16: 497 | self.assertEqual(new_lines[line - 1], original_lines[line - 1]) 498 | else: 499 | expected_line_16 = ( 500 | "Although that way may not be obvious " 501 | + "at first unless you're Guido.\n" 502 | ) 503 | self.assertEqual(new_lines[line - 1], expected_line_16) 504 | 505 | @unittest.skipIf( 506 | platform.system() == "Windows", "No exec bit for Python on windows" 507 | ) 508 | def test_preserve_permissions(self): 509 | """Test that the exec bit is preserved when processing file.""" 510 | import stat 511 | 512 | def is_executable(file_name): 513 | """Check if the file has the exec bit set.""" 514 | return stat.S_IXUSR & os.stat(file_name)[stat.ST_MODE] > 0 515 | 516 | self.assertFalse(is_executable(self.file_name)) 517 | mode = os.stat(self.file_name)[stat.ST_MODE] | stat.S_IEXEC 518 | # Windows supports READ and WRITE, but not EXEC bit. 519 | os.chmod(self.file_name, mode) 520 | self.assertTrue(is_executable(self.file_name)) 521 | file_base_name = os.path.basename(self.file_name) 522 | massedit.command_line( 523 | [ 524 | "massedit.py", 525 | "-w", 526 | "-e", 527 | "re.sub('Dutch', 'Guido', line)", 528 | "-w", 529 | "-s", 530 | self.workspace.top_dir, 531 | file_base_name, 532 | ] 533 | ) 534 | statinfo = os.stat(self.file_name) 535 | self.assertEqual(statinfo.st_mode, mode) 536 | 537 | 538 | class TestMassEditWalk(unittest.TestCase): # pylint: disable=R0904 539 | 540 | """Test recursion when processing files.""" 541 | 542 | def setUp(self): 543 | self.workspace = Workspace() 544 | self.subdirectory = self.workspace.get_directory() 545 | self.file_names = [] 546 | for ii in range(3): 547 | file_name = self.workspace.get_file( 548 | parent_dir=self.subdirectory, extension=".txt" 549 | ) 550 | with io.open(file_name, "w+") as fh: 551 | fh.write(unicode("some text ") + unicode(ii)) 552 | self.file_names.append(file_name) 553 | 554 | def tearDown(self): 555 | self.workspace.cleanup() 556 | 557 | def test_feature(self): 558 | """Trivial test to make sure setUp and tearDown work.""" 559 | pass 560 | 561 | def test_process_subdirectory_dry_run(self): 562 | """Check that ommiting -w option does not change the files.""" 563 | output = io.StringIO() 564 | processed_files = massedit.edit_files( 565 | ["*.txt"], 566 | expressions=["re.sub('text', 'blah blah', line)"], 567 | start_dirs=self.workspace.top_dir, 568 | output=output, 569 | ) 570 | self.assertEqual(sorted(processed_files), sorted(self.file_names)) 571 | index = {} 572 | for ii, file_name in enumerate(self.file_names): 573 | with io.open(file_name) as fh: 574 | new_lines = fh.readlines() 575 | self.assertEqual(new_lines, ["some text " + unicode(ii)]) 576 | index[file_name] = ii 577 | actual = output.getvalue() 578 | expected = "".join( 579 | [ 580 | textwrap.dedent( 581 | """\ 582 | --- {} 583 | +++ 584 | @@ -1 +1 @@ 585 | -some text {}+some blah blah {}""" 586 | ).format(file_name, index[file_name], index[file_name]) 587 | for file_name in processed_files 588 | ] 589 | ) 590 | self.assertEqual(actual, expected) 591 | 592 | def test_process_subdirectory_dry_run_with_one_change(self): 593 | """Check that ommiting -w option does not change the files.""" 594 | output = io.StringIO() 595 | processed_files = massedit.edit_files( 596 | ["*.txt"], 597 | expressions=["re.sub('text 1', 'blah blah 1', line)"], 598 | start_dirs=self.workspace.top_dir, 599 | output=output, 600 | ) 601 | self.assertEqual(processed_files, self.file_names[1:2]) 602 | index = {} 603 | for ii, file_name in enumerate(self.file_names): 604 | with io.open(file_name) as fh: 605 | new_lines = fh.readlines() 606 | self.assertEqual(new_lines, ["some text " + unicode(ii)]) 607 | index[file_name] = ii 608 | actual = output.getvalue() 609 | expected = "".join( 610 | [ 611 | textwrap.dedent( 612 | """\ 613 | --- {} 614 | +++ 615 | @@ -1 +1 @@ 616 | -some text {}+some blah blah {}""" 617 | ).format(file_name, index[file_name], index[file_name]) 618 | for file_name in processed_files 619 | ] 620 | ) 621 | self.assertEqual(actual, expected) 622 | 623 | def test_process_subdirectory(self): 624 | """Check that the editor works correctly in subdirectories.""" 625 | arguments = [ 626 | "-r", 627 | "-s", 628 | self.workspace.top_dir, 629 | "-w", 630 | "-e", 631 | "re.sub('text', 'blah blah', line)", 632 | "*.txt", 633 | ] 634 | processed_files = massedit.command_line(arguments) 635 | self.assertEqual(sorted(processed_files), sorted(self.file_names)) 636 | for ii, file_name in enumerate(self.file_names): 637 | with io.open(file_name) as fh: 638 | new_lines = fh.readlines() 639 | self.assertEqual(new_lines, ["some blah blah " + unicode(ii)]) 640 | 641 | def test_maxdepth_one(self): 642 | """Check that specifying -m 1 prevents modifiction to subdir.""" 643 | arguments = [ 644 | "-r", 645 | "-s", 646 | self.workspace.top_dir, 647 | "-w", 648 | "-e", 649 | "re.sub('text', 'blah blah', line)", 650 | "-m", 651 | "0", 652 | "*.txt", 653 | ] 654 | processed_files = massedit.command_line(arguments) 655 | self.assertEqual(processed_files, []) 656 | for ii, file_name in enumerate(self.file_names): 657 | with io.open(file_name) as fh: 658 | new_lines = fh.readlines() 659 | self.assertEqual(new_lines, ["some text " + unicode(ii)]) 660 | 661 | 662 | class TestIsList(unittest.TestCase): 663 | 664 | """Test the is_list function.""" 665 | 666 | def test_single_element_list(self): 667 | """Base case.""" 668 | self.assertTrue(massedit.is_list(["test"])) 669 | 670 | def test_empty_list(self): 671 | """Empty lists should work too.""" 672 | self.assertTrue(massedit.is_list([])) 673 | 674 | def test_string_not_ok(self): 675 | """String should not be confused with lists""" 676 | self.assertFalse(massedit.is_list("test")) 677 | 678 | def test_unicode_string_not_ok(self): 679 | """String should not be confused with lists""" 680 | self.assertFalse(massedit.is_list(unicode("test"))) 681 | 682 | 683 | class TestCommandLine(unittest.TestCase): # pylint: disable=R0904 684 | 685 | """Test handing of command line arguments.""" 686 | 687 | def test_parse_expression(self): 688 | """Simple test to show expression is handled by parser.""" 689 | expr_name = "re.subst('Dutch', 'Guido', line)" 690 | argv = ["massedit.py", "--expression", expr_name, "tests.py"] 691 | arguments = massedit.parse_command_line(argv) 692 | self.assertEqual(arguments.expressions, [expr_name]) 693 | 694 | def test_parse_function(self): 695 | """Simple test to show function is handled by parser.""" 696 | function_name = "tests:dutch_is_guido" 697 | argv = ["massedit.py", "--function", function_name, "tests.py"] 698 | arguments = massedit.parse_command_line(argv) 699 | self.assertEqual(arguments.functions, [function_name]) 700 | 701 | def test_exception_on_bad_patterns(self): 702 | """Check edit_files raises an error string instead of a list.""" 703 | with self.assertRaises(TypeError): 704 | massedit.edit_files("test", [], []) 705 | 706 | def test_file_option(self): 707 | """Test processing of a file.""" 708 | 709 | def add_header(data, _): 710 | """Add header on top of the file.""" 711 | yield "header on top\n" 712 | for line in data: 713 | yield line 714 | 715 | output = io.StringIO() 716 | massedit.edit_files(["tests.py"], [], [add_header], output=output) 717 | # third line shows the added header. 718 | actual = output.getvalue().split("\n")[3] 719 | expected = "+header on top" 720 | self.assertEqual(actual, expected) 721 | 722 | def test_bad_module(self): 723 | """Test error when a bad module is passed to the command.""" 724 | log_sink = LogInterceptor(massedit.log) 725 | with self.assertRaises(ImportError): 726 | massedit.edit_files(["tests.py"], functions=["bong:modify"]) 727 | expected = "failed to import bong\n" 728 | self.assertEqual(log_sink.log, expected) 729 | 730 | def test_empty_function(self): 731 | """Test empty argument.""" 732 | log_sink = LogInterceptor(massedit.log) 733 | with self.assertRaises(AttributeError): 734 | massedit.edit_files(["tests.py"], functions=[":"]) 735 | expected = ( 736 | "':' is not a callable function: " + "'dict' object has no attribute ''\n" 737 | ) 738 | self.assertEqual(log_sink.log, expected) 739 | 740 | def test_bad_function_name(self): 741 | """Check error when the function name is not valid.""" 742 | log_sink = LogInterceptor(massedit.log) 743 | with self.assertRaises(AttributeError): 744 | massedit.edit_files(["tests.py"], functions=["massedit:bad_fun"]) 745 | expected = "has no attribute 'bad_fun'\n" 746 | self.assertIn(expected, log_sink.log) 747 | 748 | def test_missing_function_name(self): 749 | """Check error when the function is empty but not the module.""" 750 | log_sink = LogInterceptor(massedit.log) 751 | with self.assertRaises(AttributeError): 752 | massedit.edit_files(["tests.py"], functions=["massedit:"]) 753 | expected = ( 754 | "'massedit:' is not a callable function: " 755 | + "'dict' object has no attribute 'massedit'\n" 756 | ) 757 | self.assertEqual(log_sink.log, expected) 758 | 759 | def test_wrong_number_of_argument(self): 760 | """Test passing function that has the wrong number of arguments.""" 761 | log_sink = LogInterceptor(massedit.log) 762 | with self.assertRaises(ValueError): 763 | massedit.edit_files(["tests.py"], functions=["massedit:get_function"]) 764 | expected = ( 765 | "'massedit:get_function' is not a callable function: " 766 | + "function should take 2 arguments: lines, file_name\n" 767 | ) 768 | self.assertEqual(log_sink.log, expected) 769 | 770 | def test_error_in_function(self): 771 | """Check error when the function triggers an exception.""" 772 | 773 | def divide_by_zero(*_): 774 | """Simulates division by zero.""" 775 | raise ZeroDivisionError() 776 | 777 | output = io.StringIO() 778 | massedit.log.disabled = True 779 | with self.assertRaises(ZeroDivisionError): 780 | massedit.edit_files(["tests.py"], [], [divide_by_zero], output=output) 781 | massedit.log.disabled = False 782 | 783 | def test_exec_option(self): 784 | """Check trivial call using executable.""" 785 | output = io.StringIO() 786 | execname = "head -1" 787 | if platform.system() == "Windows": 788 | execname = "powershell -NoProfile -c gc -head 1" 789 | next(massedit.get_paths(["tests.py"])) 790 | massedit.edit_files(["tests.py"], executables=[execname], output=output) 791 | actual = output.getvalue().split("\n") 792 | self.assertEqual(actual[3], "-#!/usr/bin/env python") 793 | self.assertEqual(actual[-1], "+#!/usr/bin/env python+") 794 | 795 | def test_write_to_cp437_output(self): 796 | """Check writing to a cp437 output (e.g. Windows console).""" 797 | raw = io.BytesIO() 798 | output = io.TextIOWrapper( 799 | io.BufferedWriter(raw), encoding="cp437" 800 | ) # Windows console. 801 | massedit.edit_files(["tests.py"], expressions=["line[:10]"], output=output) 802 | actual = raw.getvalue() 803 | self.assertIsNotNone(actual) 804 | 805 | @mock.patch(builtins + ".open", new_callable=mock.mock_open) 806 | def test_generate_fixer(self, mock_open): 807 | """Generate a fixer template file with --generate option.""" 808 | cmd = "massedit.py --generate fixer.py" 809 | massedit.command_line(cmd.split()) 810 | mock_open.assert_called_with("fixer.py", "w+") 811 | mock_open().write.assert_called_with(massedit.fixer_template) 812 | 813 | @mock.patch("massedit.readlines", return_value=["some example text"]) 814 | @mock.patch("sys.stdout", new_callable=io.StringIO) 815 | def test_from_stdin(self, stdout_, _): 816 | """A simple dash reads input test from stdin.""" 817 | # Note that double quotes will be interpreted by Python below. 818 | cmd = """massedit.py -e line.replace("text","test") -w -""" 819 | massedit.command_line(cmd.split()) 820 | self.assertEqual("some example test", stdout_.getvalue()) 821 | 822 | 823 | if __name__ == "__main__": 824 | logging.basicConfig(stream=sys.stderr, level=logging.ERROR) 825 | try: 826 | unittest.main(argv=sys.argv) 827 | finally: 828 | logging.shutdown() 829 | --------------------------------------------------------------------------------