├── .gitignore ├── requirements.txt ├── MANIFEST ├── setup.cfg ├── pyle ├── LICENSE ├── setup.py ├── pyle_test.py ├── README.md └── pyle.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | build/ 3 | dist/ -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | future==0.17.1 2 | sh==1.12.14 3 | -------------------------------------------------------------------------------- /MANIFEST: -------------------------------------------------------------------------------- 1 | # file GENERATED by distutils, do NOT edit 2 | pyle 3 | pyle.py 4 | pyle_test.py 5 | setup.py 6 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | license_files = LICENSE 3 | description-file = README.md 4 | 5 | [bdist_wheel] 6 | universal = 1 7 | -------------------------------------------------------------------------------- /pyle: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pyle import pyle 5 | 6 | if __name__ == '__main__': 7 | pyle() 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2019 Alexander Ljungberg. 2 | 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. Redistributions in binary 10 | form must reproduce the above copyright notice, this list of conditions and 11 | the following disclaimer in the documentation and/or other materials provided 12 | with the distribution. Neither the name of WireLoad Inc. nor the names 13 | of its contributors may be used to endorse or promote products derived from 14 | this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 | POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import division 3 | from __future__ import print_function 4 | from __future__ import unicode_literals 5 | 6 | from setuptools import setup 7 | 8 | setup( 9 | name='pyle', 10 | version='0.4.1', 11 | description='Use Python for shell one-liners.', 12 | author='Alexander Ljungberg', 13 | author_email='aljungberg@slevenbits.com', 14 | url='https://github.com/aljungberg/pyle', 15 | license='BSD', 16 | py_modules=['pyle'], 17 | python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', 18 | scripts=['pyle'], 19 | test_suite='pyle_test', 20 | keywords=["shell"], 21 | classifiers=[ 22 | "Programming Language :: Python :: 2", 23 | "Programming Language :: Python :: 2.7", 24 | "Programming Language :: Python :: 3", 25 | "Programming Language :: Python :: 3.4", 26 | "Programming Language :: Python :: 3.5", 27 | "Programming Language :: Python :: 3.6", 28 | "Programming Language :: Python :: 3.7", 29 | "Development Status :: 4 - Beta", 30 | "Environment :: Console", 31 | "Intended Audience :: System Administrators", 32 | "Intended Audience :: Developers", 33 | "License :: OSI Approved :: BSD License", 34 | "Operating System :: POSIX", 35 | "Topic :: System :: Shells", 36 | "Topic :: System :: Systems Administration" 37 | ], 38 | long_description="""\ 39 | Use Python for shell one-liners 40 | ------------------------------- 41 | 42 | Pyle makes it easy to use Python as a replacement for command line tools 43 | such as ``sed`` or ``perl``. 44 | 45 | See https://github.com/aljungberg/pyle for more information. 46 | """, 47 | long_description_content_type='text/markdown', 48 | install_requires=[ 49 | 'sh >= 1.12.14', 50 | 'future >= 0.17.1', 51 | ], 52 | ) 53 | -------------------------------------------------------------------------------- /pyle_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from __future__ import absolute_import 5 | from __future__ import division 6 | from __future__ import print_function 7 | from __future__ import unicode_literals 8 | 9 | 10 | from functools import reduce 11 | from subprocess import Popen, PIPE, STDOUT 12 | import operator 13 | import os 14 | import sys 15 | import tempfile 16 | import unittest 17 | 18 | test_input_a = """A few characters 19 | dance on the 20 | short little lines""" 21 | 22 | test_input_b = """ 23 | This line protected by cowboys. 24 | An alien? This box is FILLED with aliens!" 25 | """ 26 | 27 | 28 | class TestPyle(unittest.TestCase): 29 | @staticmethod 30 | def std_run(code, input_string, more_code=None, modules=None, print_traceback=False): 31 | cmd = [sys.executable, 'pyle.py', '-e', code] 32 | if more_code: 33 | cmd.extend(reduce(operator.add, [['-e', c] for c in more_code])) 34 | if modules: 35 | cmd += ['-m'] + [modules] 36 | if print_traceback: 37 | cmd += ['--traceback'] 38 | 39 | p = Popen(cmd, stdout=PIPE, stdin=PIPE, stderr=STDOUT) 40 | return p.communicate(input=input_string.encode('utf-8'))[0].decode('utf-8') 41 | 42 | def test_first_five_lines(self): 43 | output = self.std_run('line[:5]', test_input_a) 44 | 45 | self.assertEquals(output, """A few 46 | dance 47 | short""") 48 | 49 | def test_first_five_line_from_file(self): 50 | tmp_file = tempfile.NamedTemporaryFile(delete=False) 51 | try: 52 | tmp_file.write(test_input_a.encode('utf-8')) 53 | tmp_file.close() 54 | 55 | p = Popen([sys.executable, 'pyle.py', '-e', 'line[:5]', tmp_file.name], stdout=PIPE, stdin=PIPE, stderr=STDOUT) 56 | output = p.communicate()[0].decode('utf-8') 57 | 58 | self.assertEquals(output, """A few 59 | dance 60 | short""") 61 | finally: 62 | os.unlink(tmp_file.name) 63 | 64 | def in_place_run(self, code, input_string): 65 | tmp_file = tempfile.NamedTemporaryFile(delete=False) 66 | 67 | try: 68 | tmp_file.write(input_string.encode('utf-8')) 69 | tmp_file.close() 70 | 71 | p = Popen([sys.executable, 'pyle.py', '-ie', code, tmp_file.name], stdout=PIPE, stdin=PIPE, stderr=STDOUT) 72 | output = p.communicate()[0].decode('utf-8') 73 | 74 | self.maxDiff = 10000 75 | self.assertEquals(output, '') 76 | 77 | with open(tmp_file.name, 'rb') as tmp_file_2: 78 | output = tmp_file_2.read().decode('utf-8') 79 | finally: 80 | os.unlink(tmp_file.name) 81 | 82 | return output 83 | 84 | def test_first_five_in_place(self): 85 | output = self.in_place_run('line[:5]', test_input_a) 86 | self.assertEquals(output, """A few 87 | dance 88 | short""") 89 | 90 | def test_aliens_substitution(self): 91 | output = self.in_place_run(r"re.sub(r'alien(s|)?', r'angel\1', line)", test_input_b) 92 | self.assertEquals(output, """ 93 | This line protected by cowboys. 94 | An angel? This box is FILLED with angels!" 95 | """) 96 | 97 | def test_unicode_input(self): 98 | test_str = 'Segla f\xf6rutan vind\n' 99 | output = self.std_run('line', test_str) 100 | self.assertEquals(output, test_str) 101 | 102 | def test_unicode_output(self): 103 | output = self.std_run('u"test"', "\n") 104 | self.assertEquals(output, "test\n") 105 | 106 | def test_binary_input(self): 107 | test_str = '\x00\x01\x02' 108 | output = self.std_run('line', test_str) 109 | self.assertEquals(output, test_str) 110 | 111 | def test_error_message(self): 112 | output = self.std_run('int(line)', "1\nPylo\n3\n") 113 | self.assertEquals(output, "1\nAt :1 ('Pylo'): `int(line)`: invalid literal for int() with base 10: 'Pylo'\n3\n") 114 | 115 | def test_traceback(self): 116 | output = self.std_run('int(line)', "1\nPylo\n3\n", print_traceback=True) 117 | self.assertTrue("invalid literal for int() with base 10" in output) 118 | self.assertTrue("Traceback (most recent call last)" in output) 119 | 120 | def test_multiple_expressions(self): 121 | output = self.std_run('re.sub("a", "B", line)', 'aaa', 122 | more_code=['re.sub("B", "c", line)', 'line[:2]']) 123 | self.assertEquals(output, 'cc') 124 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Pyle 2 | ==== 3 | 4 | Use Python for sed or perl-like shell scripts 5 | --------------------------------------------- 6 | 7 | Pyle makes it easy to use Python as a replacement for command line tools such as `sed` or `perl`. 8 | 9 | Pyle reads its standard input and evaluates each line with the expression specified, outputting the results on standard out. 10 | 11 | To print the first 20 characters of each line of a file: 12 | 13 | cat README.md | pyle -e "line[:20]" 14 | 15 | or: 16 | 17 | pyle -e "line[:20]" README.md 18 | 19 | List all `/tmp/` files with a filename with an even length: 20 | 21 | ls /tmp/ | pyle -e "sh.ls('-l', line) if len(line) % 2 == 0 else None" 22 | 23 | Perform an in-place string substitution, overwriting the original file with the updated file: 24 | 25 | pyle -ie "re.sub(r'alien(s|)?', r'ghost\1', line)" TextAboutAliens.md 26 | 27 | The special variable`line` is the current line (each line of input is evaluated through the given expression(s)). The variable `words` is the current line split by whitespace. To print just the URLs in an Apache access log (the seventh "word" in the line): 28 | 29 | tail access_log | pyle -e "words[6]" 30 | 31 | Print the SHA 256 sum of each `*.py` file in the current directory: 32 | 33 | ls *.py | pyle -m hashlib -e "'%s %s' % (hashlib.sha256(line).hexdigest(), line)" 34 | 348e4a65e24bab4eed8e2bbe6f4c8176ddec60051d1918eea38b34b1103a8af6 pyle.py 35 | b28c7f73e6df990a96cfb724be1d673c2d3c43f68d4b6c06d8e5a9b29e5d12cb pyle_test.py 36 | 37 | If your expression returns a list or a tuple, the items will be printed joined by spaces. With that in mind we can simplify the above example: 38 | 39 | ls *.py | pyle -m hashlib -e "(hashlib.sha256(line).hexdigest(), line)" 40 | 348e4a65e24bab4eed8e2bbe6f4c8176ddec60051d1918eea38b34b1103a8af6 pyle.py 41 | b28c7f73e6df990a96cfb724be1d673c2d3c43f68d4b6c06d8e5a9b29e5d12cb pyle_test.py 42 | 43 | Print the first five lines of each file with file names and line numbers: 44 | 45 | pyle -e "'%-15s:%04d %s' % (filename, 1 + num, line) if num < 5 else None" *.py 46 | 47 | You can also specify multiple expressions by repeating the `-e` option. Just 48 | like in `sed` each expression acts on `line` (and `words`, etc.) as modified by 49 | the previous expression. E.g. replace a letter in the first five words of a file: 50 | 51 | pyle -e "words[:5]" -e "re.sub('A', 'B', line)" README.md 52 | 53 | The idea for Pyle is based on Graham Fawcett's [PyLine](http://code.activestate.com/recipes/437932-pyle-a-grep-like-sed-like-command-line-tool/). Pyle is generally compatible with PyLine but requires a `-e` before the evaluation statement. 54 | 55 | Pyle imports the [sh](https://github.com/amoffat/sh) module by default, which enables easy shell command execution. 56 | 57 | ## Installation ## 58 | 59 | Pyle is compatible with Python 2.7 and Python 3.4 and above. 60 | 61 | pip install pyle 62 | 63 | ## Documentation ## 64 | 65 | This file and `pyle --help`. 66 | 67 | The following variables are available in the global scope: 68 | 69 | * `line`: the current input line being processed 70 | * `words`: line split by whitespace 71 | * `num`: line number 72 | * `filename`: the name of the current file 73 | 74 | The following modules are imported by default: 75 | 76 | * `re`: Python regular expressions 77 | * `sh`: the [`sh` module](https://github.com/amoffat/sh) 78 | 79 | The sh module makes it easy to run additional commands from within the expression. 80 | 81 | Pyle can operate on a list of filenames in which case each file is read in order and evaluated line by line. 82 | 83 | ## Why Pyle? ## 84 | 85 | Some of us are just simply awful at remembering the `sed`, `perl` or even `bash` syntax but feel right at home with Python. Python code is often a little more verbose but what good is saving characters if you can't remember what they do? 86 | 87 | Here's an example of `sed` vs `pyle`. This isn't a very good `sed` expression, admittedly, but the people who will find Pyle useful are not `sed` experts. 88 | 89 | To change home directories from `/var/X` to `/home/X` with sed: 90 | 91 | sed 's/^\(\([^:]*:\)\{5\}\)\/var\/\(.*\)/\1\/home\/\3/g' /etc/passwd 92 | 93 | With Pyle: 94 | 95 | pyle -e "re.sub(r'^(([^:]*:){5})/var/(.*)', r'\1/home/\3', line)" /etc/passwd 96 | 97 | If you find the Python code more readable, Pyle is for you. 98 | 99 | ## Tests ## 100 | 101 | Tests need to be run both in Python 2 and Python 3. Best way to do that is to have one virtual environment for each. If you use virtualenv wrapper, something like this: 102 | 103 | workon pyle2 104 | python2 -3 -Werror -m unittest discover -p pyle_test.py 105 | 106 | workon pyle3 107 | python3 -m unittest discover -p pyle_test.py 108 | 109 | ## License ## 110 | 111 | Free to use and modify under the terms of the BSD open source license. 112 | -------------------------------------------------------------------------------- /pyle.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Pyle makes it easy to use Python as a replacement for command line tools such as `sed` or `perl`. 5 | 6 | """ 7 | from __future__ import absolute_import 8 | from __future__ import division 9 | from __future__ import print_function 10 | from __future__ import unicode_literals 11 | 12 | from future import standard_library 13 | standard_library.install_aliases() 14 | from future.utils import string_types 15 | 16 | import argparse 17 | import io 18 | import re 19 | import sh 20 | import sys 21 | import traceback 22 | 23 | __version__ = "0.4.1" 24 | 25 | 26 | STANDARD_MODULES = { 27 | 're': re, 28 | 'sh': sh 29 | } 30 | 31 | 32 | def truncate_ellipsis(line, length=30): 33 | """Truncate a line to the specified length followed by ``...`` unless its shorter than length already.""" 34 | 35 | return line if len(line) < length else line[:length - 3] + "..." 36 | 37 | 38 | def pyle_evaluate(expressions=None, modules=(), inplace=False, files=None, print_traceback=False): 39 | """The main method of pyle.""" 40 | 41 | eval_globals = {} 42 | 43 | eval_globals.update(STANDARD_MODULES) 44 | 45 | for module_arg in modules or (): 46 | for module in module_arg.strip().split(","): 47 | module = module.strip() 48 | if module: 49 | eval_globals[module] = __import__(module) 50 | 51 | if not expressions: 52 | # Default 'do nothing' program 53 | expressions = ['line'] 54 | 55 | encoding = sys.getdefaultencoding() 56 | 57 | files = files or ['-'] 58 | eval_locals = {} 59 | for file in files: 60 | if file == '-': 61 | file = sys.stdin 62 | 63 | out_buf = sys.stdout if not inplace else io.StringIO() 64 | 65 | out_line = None 66 | with (io.open(file, 'r', encoding=encoding) if not hasattr(file, 'read') else file) as in_file: 67 | for num, line in enumerate(in_file.readlines()): 68 | 69 | was_whole_line = False 70 | if line[-1] == '\n': 71 | was_whole_line = True 72 | line = line[:-1] 73 | 74 | expr = "" 75 | try: 76 | for expr in expressions: 77 | words = [word.strip() 78 | for word in re.split(r'\s+', line) 79 | if word] 80 | eval_locals.update({ 81 | 'line': line, 82 | 'words': words, 83 | 'filename': in_file.name, 84 | 'num': num 85 | }) 86 | 87 | out_line = eval(expr, eval_globals, eval_locals) 88 | 89 | if out_line is None: 90 | continue 91 | 92 | # If the result is something list-like or iterable, 93 | # output each item space separated. 94 | if not isinstance(out_line, string_types): 95 | try: 96 | out_line = u' '.join(str(part) for part in out_line) 97 | except: 98 | # Guess it wasn't a list after all. 99 | out_line = str(out_line) 100 | 101 | line = out_line 102 | except Exception as e: 103 | sys.stdout.flush() 104 | sys.stderr.write("At %s:%d ('%s'): `%s`: %s\n" % ( 105 | in_file.name, num, truncate_ellipsis(line), expr, e)) 106 | if print_traceback: 107 | traceback.print_exc(None, sys.stderr) 108 | else: 109 | if out_line is None: 110 | continue 111 | 112 | out_line = out_line or u'' 113 | out_buf.write(out_line) 114 | if was_whole_line: 115 | out_buf.write('\n') 116 | 117 | if inplace: 118 | with io.open(file, 'w', encoding=encoding) as out_file: 119 | out_file.write(out_buf.getvalue()) 120 | out_buf.close() 121 | 122 | 123 | def pyle(argv=None): 124 | """Execute pyle with the specified arguments, or sys.argv if no arguments specified.""" 125 | 126 | parser = argparse.ArgumentParser(description=__doc__) 127 | 128 | parser.add_argument("-m", "--modules", dest="modules", action='append', 129 | help="import MODULE before evaluation. May be specified more than once.") 130 | parser.add_argument("-i", "--inplace", dest="inplace", action='store_true', default=False, 131 | help="edit files in place. When used with file name arguments, the files will be replaced by the output of the evaluation") 132 | parser.add_argument("-e", "--expression", action="append", 133 | dest="expressions", help="an expression to evaluate for each line") 134 | parser.add_argument('files', nargs='*', 135 | help="files to read as input. If used with --inplace, the files will be replaced with the output") 136 | parser.add_argument("--traceback", action="store_true", default=False, 137 | help="print a traceback on stderr when an expression fails for a line") 138 | 139 | args = parser.parse_args() if not argv else parser.parse_args(argv) 140 | 141 | pyle_evaluate(args.expressions, args.modules, args.inplace, args.files, args.traceback) 142 | 143 | 144 | if __name__ == '__main__': 145 | pyle() 146 | --------------------------------------------------------------------------------