├── .github └── workflows │ └── python.yml ├── .gitignore ├── LICENSE ├── README.md ├── example ├── allinone │ ├── common │ │ ├── __init__.spy │ │ └── common.spy │ ├── hello.py │ ├── test.spy │ ├── test3.spy │ ├── test3_interactive.spy │ ├── test_flags.spy │ └── test_interactive.spy ├── curl.spy ├── git.spy └── import_from_python │ ├── import.py │ └── shared.spy ├── setup.py ├── shellpy ├── shellpy3 └── shellpython ├── __init__.py ├── config.py ├── constants.py ├── core.py ├── header.tpl ├── header_root.tpl ├── helpers.py ├── importer.py ├── locator.py ├── preprocessor.py ├── shellpy.py └── tests ├── __init__.py ├── data ├── locator │ ├── testfilepy.py │ ├── testfilespy.spy │ ├── testmodulepy │ │ └── __init__.py │ └── testmodulespy │ │ └── __init__.spy └── preprocessor │ ├── meta.py │ ├── test.imdt │ └── test.spy ├── test_helpers.py ├── test_locator.py ├── test_preprocessor.py ├── test_preprocessor_intermediate.py └── test_unix_core.py /.github/workflows/python.yml: -------------------------------------------------------------------------------- 1 | name: Python package 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ubuntu-latest 9 | strategy: 10 | matrix: 11 | python-version: [2.7, 3.6, 3.7, 3.8, 3.9, pypy-2.7, pypy-3.6] 12 | os: [ubuntu-latest, macos-latest, windows-latest] 13 | 14 | steps: 15 | - uses: actions/checkout@v2 16 | - name: Set up Python ${{ matrix.python-version }} 17 | uses: actions/setup-python@v2 18 | with: 19 | python-version: ${{ matrix.python-version }} 20 | 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | pip install . 25 | pip install mock 26 | python ./setup.py install 27 | - name: Test with pytest 28 | run: | 29 | python -m unittest discover 30 | 31 | - name: Run test scripts 32 | run: | 33 | example/import_from_python/import.py 34 | 35 | - name: Run test scripts python2 36 | if: matrix.python-version == '2.7' || matrix.python-version == 'pypy-2.7' 37 | run: | 38 | shellpy example/allinone/test.spy 39 | 40 | - name: Run test scripts python3 41 | if: matrix.python-version != '2.7' && matrix.python-version != 'pypy-2.7' 42 | run: | 43 | shellpy3 example/allinone/test3.spy 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | 55 | # Sphinx documentation 56 | docs/_build/ 57 | 58 | # PyBuilder 59 | target/ 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, lamerman 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of shellpy nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # shellpy 2 | A tool for convenient shell scripting in Python. It allows you to write all your shell scripts in Python in a convenient way and in many cases replace Bash/Sh. 3 | 4 | ## Preface - Why do we need shell python? 5 | 6 | For many people bash/sh seem to be pretty complicated. An example would be regular expressions, working with json/yaml/xml, named arguments parsing and so on. There are many things that are much easier in python to understand and work with. 7 | 8 | ## Introduction 9 | 10 | Shell python has no differences from python except for one. Grave accent symbol (`) does not mean eval, it means execution of shell commands. So 11 | 12 | `ls -l` 13 | 14 | will execute `ls -l` in shell. You can also skip one ` in the end of line 15 | 16 | `ls -l 17 | 18 | and it will also be a correct syntax. It is also possible to write multiline expressions 19 | 20 | ` 21 | echo test > test.txt 22 | cat test.txt 23 | ` 24 | 25 | and long lines 26 | 27 | `echo This is \ 28 | a very long \ 29 | line 30 | 31 | Every shellpy expression returns a Result 32 | 33 | result = `ls -l 34 | 35 | or normally raises an error in case of non zero output of a command 36 | 37 | try: 38 | result = `ls -l non_existent_file 39 | except NonZeroReturnCodeError as e: 40 | result = e.result 41 | 42 | The result can be either [Result](https://github.com/lamerman/shellpy/wiki/Simple-mode#result) or [InteractiveResult](https://github.com/lamerman/shellpy/wiki/Interactive-mode#interactive-result). Let's start with a simple Result. You can check returncode of a command 43 | 44 | result = `ls -l 45 | print result.returncode 46 | 47 | You can also get text from stdout or stderr 48 | 49 | result = `ls -l 50 | result_text = result.stdout 51 | result_error = result.stderr 52 | 53 | You can iterate over lines of result stdout 54 | 55 | result = `ls -l 56 | for line in result: 57 | print line.upper() 58 | 59 | and so on. 60 | 61 | ## Integration with python and code reuse 62 | 63 | As it was said before shellpython does not differ a lot from ordinary python. You can import python modules and use them as usual 64 | 65 | import os.path 66 | 67 | `mkdir /tmp/mydir 68 | os.path.exists('/tmp/mydir') # True 69 | 70 | And you can do the same with shellpython modules. Suppose you have shellpy module `common` as in examples directory. So this is how it looks 71 | 72 | ls common/ 73 | common.spy __init__.spy 74 | 75 | So you have directory `common` and two files inside: `__init__.spy` and `common.spy`. Looks like a python module right? Exactly. The only difference is file extension. For `__init__.spy` and other files it must be `.spy`. Let's look inside `common.spy` 76 | 77 | def common_func(): 78 | return `echo 5 79 | 80 | A simple function that returns [Result](https://github.com/lamerman/shellpy/wiki/Simple-mode#result) of `echo 5` execution. How is it used how in code? As same as in python 81 | 82 | from common.common import common_func 83 | 84 | print('Result of imported function is ' + str(common_func())) 85 | 86 | Note that the `common` directory must be in pythonpath to be imported. 87 | 88 | ### How does import work? 89 | 90 | It uses import hooks described in [PEP 0302 -- New Import Hooks](https://www.python.org/dev/peps/pep-0302/). So, whenever importer finds a shellpy module or a file with .spy extension and with the name that you import, it will try to first preprocess it from shellpy to python and then import it using standard python import. Once preprocessed, the file is cached in your system temp directory and the second time it will be just imported directly. 91 | 92 | ### Important note about import 93 | 94 | Import of shellpython modules requires import hook to be installed. There are two way how to do it: 95 | - run shellpython scripts with the `shellpy` tool as described below in the section [Running](https://github.com/lamerman/shellpy#running) 96 | - run your python scripts as usual with `python` but initialize shellpython before importing any module with `shellpython.init()` as in the [Example](https://github.com/lamerman/shellpy/blob/master/example/import_from_python/import.py) 97 | 98 | ### Example 99 | 100 | This script clones shellpython to temporary directory and finds the commit hash where README was created 101 | 102 | ```python 103 | 104 | import tempfile 105 | import os.path 106 | from shellpython.helpers import Dir 107 | 108 | # We will make everything in temp directory. Dir helper allows you to change current directory 109 | # withing 'with' block 110 | with Dir(tempfile.gettempdir()): 111 | if not os.path.exists('shellpy'): 112 | # just executes shell command 113 | `git clone https://github.com/lamerman/shellpy.git 114 | 115 | # switch to newly created tempdirectory/shellpy 116 | with Dir('shellpy'): 117 | # here we capture result of shell execution. log here is an instance of Result class 118 | log = `git log --pretty=oneline --grep='Create' 119 | 120 | # shellpy allows you to iterate over lines in stdout with this syntactic sugar 121 | for line in log: 122 | if line.find('README.md'): 123 | hashcode = log.stdout.split(' ')[0] 124 | print hashcode 125 | exit(0) 126 | 127 | print 'The commit where the readme was created was not found' 128 | 129 | exit(1) 130 | ``` 131 | 132 | Two lines here are executed in shell ```git clone https://github.com/lamerman/shellpy.git``` and ```git log --pretty=oneline --grep='Create'```. The result of the second line is assigned to variable ```log``` and then we iterate over the result line by line in the for cycle 133 | 134 | ### Installation 135 | 136 | You can install it either with ```pip install shellpy``` or by cloning this repository and execution of ```setup.py install```. After that you will have ```shellpy``` command installed. 137 | 138 | ### Running 139 | 140 | You can try shellpython by running examples after installation. Download this repository and run the following command in the root folder of the cloned repository: 141 | 142 | ```shellpy example/curl.spy``` 143 | 144 | ```shellpy example/git.spy``` 145 | 146 | There is also so called allinone example which you can have a look at and execute like this: 147 | 148 | ```shellpy example/allinone/test.spy``` 149 | 150 | It is called all in one because it demonstrates all features available in shellpy. If you have python3 run instead: 151 | 152 | ```shellpy example/allinone/test3.spy``` 153 | 154 | ### Documentation 155 | 156 | [Wiki](https://github.com/lamerman/shellpy/wiki) 157 | 158 | ### Compatibility 159 | 160 | It works on Linux and Mac for both Python 2.x and 3.x. It should also work on Windows. 161 | -------------------------------------------------------------------------------- /example/allinone/common/__init__.spy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lamerman/shellpy/544cb811d5fb0e8cb6d9934cef71f9e66e645cf8/example/allinone/common/__init__.spy -------------------------------------------------------------------------------- /example/allinone/common/common.spy: -------------------------------------------------------------------------------- 1 | def common_func(): 2 | return `echo 5 3 | 4 | if __name__ == '__main__': 5 | print('Not reachable') 6 | -------------------------------------------------------------------------------- /example/allinone/hello.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import time 4 | from sys import stdin, stdout 5 | 6 | print('Enter your name') 7 | stdout.flush() 8 | userinput = stdin.readline() 9 | 10 | time.sleep(1) 11 | 12 | print('Hello ' + userinput.rstrip(os.linesep)) 13 | print('End') 14 | -------------------------------------------------------------------------------- /example/allinone/test.spy: -------------------------------------------------------------------------------- 1 | #import and use of usual python modules 2 | import os 3 | from shellpython import core 4 | print os.name 5 | 6 | from shellpython import config as config 7 | config.PRINT_ALL_COMMANDS = True 8 | config.COLORAMA_ENABLED = True 9 | 10 | # import from shellpy module 11 | from common.common import common_func 12 | 13 | # iteration over command output 14 | for line in `ls -l`: 15 | # print of output with local variable capture 16 | print `echo 'LINE IS: {line}' 17 | 18 | # return code capture 19 | try: 20 | s = `ls -l | grep non_existent_string 21 | print 'string found' 22 | except core.NonZeroReturnCodeError: 23 | print 'string not found' 24 | 25 | # execution of function from imported module 26 | print 'Result of imported function is ' + str(common_func()) 27 | 28 | # several commands on one line 29 | print 'It should be False: ' + str(`echo 1` == `echo 2`) 30 | print 'It should be True: ' + str(`echo 1` == `echo 1`) 31 | 32 | # multiline execution 33 | print ` 34 | echo Line one of multiline 35 | echo Line two of multiline 36 | ` 37 | 38 | # multiline execution with result capture 39 | f = ` 40 | echo Line one of multiline with capture 41 | echo Line two of multiline with capture 42 | ` 43 | print f 44 | 45 | # split one long command to several lines 46 | print `echo \ 47 | This is a \ 48 | long command \ 49 | on several lines 50 | 51 | # all commands with 'p' param which means print 52 | x = p`echo 1 53 | p`echo {x}` 54 | 55 | p`echo \ 56 | This is a \ 57 | long command \ 58 | on several lines 59 | 60 | p` 61 | echo Line one of multiline 62 | echo Line two of multiline 63 | ` 64 | 65 | # std error stream test 66 | 67 | # this line will just print stderr ('e' flag) without throwing an exception because of the 'n' (no throw) flag 68 | en`ls -l /non_existent_directory 69 | 70 | try: 71 | x = `ls -l /non_existent_directory2 72 | except core.NonZeroReturnCodeError as e: 73 | print e.result.stderr 74 | 75 | # escaped 76 | 77 | p`echo 'escaped' 78 | -------------------------------------------------------------------------------- /example/allinone/test3.spy: -------------------------------------------------------------------------------- 1 | #import and use of usual python modules 2 | import os 3 | from shellpython import core 4 | print (os.name) 5 | 6 | from shellpython import config as config 7 | config.PRINT_ALL_COMMANDS = True 8 | config.COLORAMA_ENABLED = True 9 | 10 | # import from shellpy module 11 | from common.common import common_func 12 | 13 | # iteration over command output 14 | for line in `ls -l`: 15 | # print of output with local variable capture 16 | print(`echo 'LINE IS: {line}'`) 17 | 18 | # return code capture 19 | try: 20 | s = `ls -l | grep non_existent_string 21 | print('string found') 22 | except core.NonZeroReturnCodeError: 23 | print('string not found') 24 | 25 | # execution of function from imported module 26 | print('Result of imported function is ' + str(common_func())) 27 | 28 | # several commands on one line 29 | print('It should be False: ' + str(`echo 1` == `echo 2`)) 30 | print('It should be True: ' + str(`echo 1` == `echo 1`)) 31 | 32 | # multiline execution with result capture 33 | f = ` 34 | echo Line one of multiline with capture 35 | echo Line two of multiline with capture 36 | ` 37 | print(f) 38 | 39 | # split one long command to several lines 40 | a = `echo \ 41 | This is a \ 42 | long command \ 43 | on several lines 44 | 45 | print(a) 46 | 47 | # all commands with 'p' param which means print 48 | x = p`echo 1 49 | p`echo {x}` 50 | 51 | p`echo \ 52 | This is a \ 53 | long command \ 54 | on several lines 55 | 56 | p` 57 | echo Line one of multiline 58 | echo Line two of multiline 59 | ` 60 | 61 | # std error stream test 62 | 63 | # this line will just print stderr ('e' flag) without throwing an exception because of the 'n' (no throw) flag 64 | en`ls -l /non_existent_directory 65 | 66 | try: 67 | x = `ls -l /non_existent_directory2 68 | except core.NonZeroReturnCodeError as e: 69 | print(e.result.stderr) 70 | 71 | # escaped 72 | 73 | p`echo 'escaped' -------------------------------------------------------------------------------- /example/allinone/test3_interactive.spy: -------------------------------------------------------------------------------- 1 | #import and use of usual python modules 2 | import os 3 | 4 | result = i`ping -c 3 8.8.8.8 5 | 6 | for line in result.stdout: 7 | print(line) 8 | 9 | result = ip`example/allinone/hello.py 10 | if result.sreadline() == 'Enter your name': 11 | result.swriteline('Alexander') 12 | else: 13 | raise RuntimeError('Unexpected output of script') 14 | 15 | welcome_text = result.sreadline() 16 | 17 | print(result.returncode) 18 | 19 | result = ie`ls -l /non_existent_directory 20 | result.stderr.sreadline() 21 | -------------------------------------------------------------------------------- /example/allinone/test_flags.spy: -------------------------------------------------------------------------------- 1 | import shellpython.config as config 2 | 3 | def test_print_stdout_always(): 4 | `echo it should NOT be printed 5 | 6 | config.PRINT_STDOUT_ALWAYS = True 7 | 8 | `echo it should be printed 9 | 10 | config.PRINT_STDOUT_ALWAYS = False 11 | 12 | def test_print_all_commands(): 13 | `echo the echo command should NOT be printed 14 | 15 | config.PRINT_ALL_COMMANDS = True 16 | 17 | `echo the echo command should be printed 18 | 19 | config.PRINT_ALL_COMMANDS = False 20 | 21 | test_print_stdout_always() 22 | test_print_all_commands() -------------------------------------------------------------------------------- /example/allinone/test_interactive.spy: -------------------------------------------------------------------------------- 1 | #import and use of usual python modules 2 | import os 3 | 4 | from shellpython import config as config 5 | config.PRINT_ALL_COMMANDS = True 6 | config.COLORAMA_ENABLED = True 7 | 8 | result = i`ping -c 3 8.8.8.8 9 | 10 | for line in result.stdout: 11 | print line 12 | 13 | result = ip`example/allinone/hello.py 14 | if result.sreadline() == 'Enter your name': 15 | result.swriteline('Alexander') 16 | else: 17 | raise RuntimeError('Unexpected output of script') 18 | 19 | welcome_text = result.sreadline() 20 | 21 | print result.returncode 22 | 23 | result = ie`ls -l /non_existent_directory 24 | result.stderr.sreadline() -------------------------------------------------------------------------------- /example/curl.spy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env shellpy 2 | """ 3 | This script downloads avatar of github user 'python' with curl 4 | """ 5 | 6 | import json 7 | import os 8 | import tempfile 9 | 10 | # get the api answer with curl 11 | answer = `curl https://api.github.com/users/python 12 | 13 | answer_json = json.loads(answer.stdout) 14 | avatar_url = answer_json['avatar_url'] 15 | 16 | destination = os.path.join(tempfile.gettempdir(), 'python.png') 17 | 18 | # execute curl once again, this time to get the image 19 | result = `curl {avatar_url} > {destination} 20 | p`ls -l {destination} 21 | -------------------------------------------------------------------------------- /example/git.spy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env shellpy 2 | """ 3 | This script clones shellpython to temporary directory and finds the commit hash where README was created 4 | """ 5 | 6 | import tempfile 7 | import os.path 8 | from shellpython.helpers import Dir 9 | 10 | # We will make everything in temp directory. Dir helper allows you to change current directory 11 | # withing 'with' block 12 | with Dir(tempfile.gettempdir()): 13 | if not os.path.exists('shellpython'): 14 | # just executes shell command 15 | `git clone https://github.com/lamerman/shellpy.git shellpython 16 | 17 | # switch to newly created tempdirectory/shellpy 18 | with Dir('shellpython'): 19 | # here we capture result of shell execution. log here is an instance of Result class 20 | log = `git log --pretty=oneline --grep='Create' 21 | 22 | # shellpy allows you to iterate over lines in stdout with this syntactic sugar 23 | for line in log: 24 | if line.find('README.md'): 25 | hashcode = log.stdout.split(' ')[0] 26 | print(hashcode) 27 | exit(0) 28 | 29 | print('The commit where the readme was created was not found') 30 | 31 | exit(1) 32 | -------------------------------------------------------------------------------- /example/import_from_python/import.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | This script shows how to use shellpython directly from python without using the shellpy command 4 | """ 5 | 6 | import shellpython 7 | shellpython.init() # installs the shellpython import hook. Call uninit() to remove the hook if no longer needed 8 | 9 | import shared # imports shellpy module 10 | 11 | 12 | def main(): 13 | print(shared.shared_func()) # call to a function from shellpy module 14 | 15 | if __name__ == '__main__': 16 | main() 17 | -------------------------------------------------------------------------------- /example/import_from_python/shared.spy: -------------------------------------------------------------------------------- 1 | def shared_func(): 2 | return `echo 'Hello from imported shellpython function' -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | try: 4 | from setuptools import setup 5 | 6 | args_for_setup = {'entry_points': { 7 | 'console_scripts': { 8 | 'shellpy = shellpython.shellpy:main2', 9 | 'shellpy2 = shellpython.shellpy:main2', 10 | 'shellpy3 = shellpython.shellpy:main3' 11 | } 12 | }} 13 | 14 | except ImportError: 15 | from distutils.core import setup 16 | 17 | args_for_setup = {'scripts': ['shellpy']} 18 | 19 | 20 | setup(name='shellpy', 21 | version='0.5.1', 22 | description='A convenient tool for shell scripting in python', 23 | author='Alexander Ponomarev', 24 | author_email='alexander996@yandex.ru', 25 | url='https://github.com/lamerman/shellpy/', 26 | download_url='https://github.com/lamerman/shellpy/tarball/0.5.1', 27 | keywords=['shell', 'bash', 'sh'], 28 | packages=['shellpython'], 29 | package_data={'shellpython': ['*.tpl']}, 30 | install_requires=['colorama'], 31 | tests_require=['mock'], 32 | **args_for_setup 33 | ) 34 | -------------------------------------------------------------------------------- /shellpy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from shellpython import shellpy 3 | 4 | shellpy.main2() 5 | -------------------------------------------------------------------------------- /shellpy3: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from shellpython import shellpy 3 | 4 | shellpy.main3() 5 | -------------------------------------------------------------------------------- /shellpython/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from shellpython.importer import PreprocessorImporter 3 | 4 | _importer = PreprocessorImporter() 5 | 6 | 7 | def init(): 8 | """Initialize shellpython by installing the import hook 9 | """ 10 | if _importer not in sys.meta_path: 11 | sys.meta_path.insert(0, _importer) 12 | 13 | 14 | def uninit(): 15 | """Uninitialize shellpython by removing the import hook 16 | """ 17 | sys.meta_path.remove(_importer) 18 | -------------------------------------------------------------------------------- /shellpython/config.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import base64 3 | import sys 4 | 5 | # TODO: subject for further refactoring. Config should be serialized in nicer way. Now we cannot do it unless we break 6 | # compatibility 7 | 8 | # prints all commands being executed 9 | PRINT_ALL_COMMANDS = False 10 | 11 | # prints stdout of every command executed 12 | PRINT_STDOUT_ALWAYS = False 13 | 14 | # prints stderr of every command executed 15 | PRINT_STDERR_ALWAYS = False 16 | 17 | # colorama is a plugin that makes output colored, this flag controls whether it is enabled 18 | COLORAMA_ENABLED = True 19 | 20 | 21 | def dumps(): 22 | config_tuple = (PRINT_ALL_COMMANDS, PRINT_STDOUT_ALWAYS, PRINT_STDERR_ALWAYS, COLORAMA_ENABLED) 23 | serialized_config = pickle.dumps(config_tuple) 24 | 25 | if sys.version_info[0] == 2: 26 | return base64.b64encode(serialized_config) 27 | else: 28 | return str(base64.b64encode(serialized_config), 'utf-8') 29 | 30 | 31 | def loads(data): 32 | global PRINT_ALL_COMMANDS, PRINT_STDOUT_ALWAYS, PRINT_STDERR_ALWAYS, COLORAMA_ENABLED 33 | 34 | if sys.version_info[0] == 2: 35 | serialized_config = base64.b64decode(data) 36 | else: 37 | serialized_config = base64.b64decode(bytes(data, 'utf-8')) 38 | 39 | config_tuple = pickle.loads(serialized_config) 40 | 41 | PRINT_ALL_COMMANDS, PRINT_STDOUT_ALWAYS, PRINT_STDERR_ALWAYS, COLORAMA_ENABLED = config_tuple 42 | -------------------------------------------------------------------------------- /shellpython/constants.py: -------------------------------------------------------------------------------- 1 | SHELLPY_PARAMS = 'SHELLPY_PARAMS' 2 | -------------------------------------------------------------------------------- /shellpython/core.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import os 3 | import sys 4 | import subprocess 5 | from os import environ as env 6 | from shellpython import config 7 | 8 | _colorama_intialized = False 9 | _colorama_available = True 10 | try: 11 | import colorama 12 | from colorama import Fore, Style 13 | except ImportError: 14 | _colorama_available = False 15 | 16 | 17 | def _is_colorama_enabled(): 18 | return _colorama_available and config.COLORAMA_ENABLED 19 | 20 | 21 | def _print_stdout(text): 22 | print(text) 23 | 24 | 25 | def _print_stderr(text): 26 | print(text, file=sys.stderr) 27 | 28 | # print all stdout of executed command 29 | _PARAM_PRINT_STDOUT = 'p' 30 | 31 | # print all stderr of executed command 32 | _PARAM_PRINT_STDERR = 'e' 33 | 34 | # runs command in interactive mode when user can read output line by line and send to stdin 35 | _PARAM_INTERACTIVE = 'i' 36 | 37 | # no throw mode. With this parameter user explicitly says that NonZeroReturnCodeError must not be thrown for this 38 | # specific command. It may be useful if for some reason this command does not return 0 even for successful run 39 | _PARAM_NO_THROW = 'n' 40 | 41 | 42 | def exe(cmd, params): 43 | """This function runs after preprocessing of code. It actually executes commands with subprocess 44 | 45 | :param cmd: command to be executed with subprocess 46 | :param params: parameters passed before ` character, i.e. p`echo 1 which means print result of execution 47 | :return: result of execution. It may be either Result or InteractiveResult 48 | """ 49 | 50 | global _colorama_intialized 51 | if _is_colorama_enabled() and not _colorama_intialized: 52 | _colorama_intialized = True 53 | colorama.init() 54 | 55 | if config.PRINT_ALL_COMMANDS: 56 | if _is_colorama_enabled(): 57 | _print_stdout(Fore.GREEN + '>>> ' + cmd + Style.RESET_ALL) 58 | else: 59 | _print_stdout('>>> ' + cmd) 60 | 61 | if _is_param_set(params, _PARAM_INTERACTIVE): 62 | return _create_interactive_result(cmd, params) 63 | else: 64 | return _create_result(cmd, params) 65 | 66 | 67 | def _is_param_set(params, param): 68 | return True if params.find(param) != -1 else False 69 | 70 | 71 | class ShellpyError(Exception): 72 | """Base error for shell python 73 | """ 74 | pass 75 | 76 | 77 | class NonZeroReturnCodeError(ShellpyError): 78 | """This is thrown when the executed command does not return 0 79 | """ 80 | def __init__(self, cmd, result): 81 | self.cmd = cmd 82 | self.result = result 83 | 84 | def __str__(self): 85 | if _is_colorama_enabled(): 86 | return 'Command {red}\'{cmd}\'{end} failed with error code {code}, stderr output is {red}{stderr}{end}'\ 87 | .format(red=Fore.RED, end=Style.RESET_ALL, cmd=self.cmd, code=self.result.returncode, 88 | stderr=self.result.stderr) 89 | else: 90 | return 'Command \'{cmd}\' failed with error code {code}, stderr output is {stderr}'.format( 91 | cmd=self.cmd, code=self.result.returncode, stderr=self.result.stderr) 92 | 93 | 94 | class Stream: 95 | def __init__(self, file, encoding, print_out_stream=False, color=None): 96 | self._file = file 97 | self._encoding = encoding 98 | self._print_out_stream = print_out_stream 99 | self._color = color 100 | 101 | def __iter__(self): 102 | return self 103 | 104 | def next(self): 105 | return self.sreadline() 106 | 107 | __next__ = next 108 | 109 | def sreadline(self): 110 | line = self._file.readline() 111 | if sys.version_info[0] == 3: 112 | line = line.decode(self._encoding) 113 | 114 | if line == '': 115 | raise StopIteration 116 | else: 117 | line = line.rstrip(os.linesep) 118 | if self._print_out_stream: 119 | if self._color is None: 120 | _print_stdout(line) 121 | else: 122 | _print_stdout(self._color + line + Style.RESET_ALL) 123 | 124 | return line 125 | 126 | def swriteline(self, text): 127 | text_with_linesep = text + os.linesep 128 | if sys.version_info[0] == 3: 129 | text_with_linesep = text_with_linesep.encode(self._encoding) 130 | 131 | self._file.write(text_with_linesep) 132 | self._file.flush() 133 | 134 | 135 | class InteractiveResult: 136 | """Result of a shell command execution. 137 | 138 | To get the result as string use str(Result) 139 | To get lines use the Result.lines field 140 | You can also iterate over lines of result like this: for line in Result: 141 | You can compare two results that will mean compare of result strings 142 | """ 143 | def __init__(self, process, params): 144 | self._process = process 145 | self._params = params 146 | self.stdin = Stream(process.stdin, sys.stdin.encoding) 147 | 148 | print_stdout = _is_param_set(params, _PARAM_PRINT_STDOUT) or config.PRINT_STDOUT_ALWAYS 149 | self.stdout = Stream(process.stdout, sys.stdout.encoding, print_stdout) 150 | 151 | print_stderr = _is_param_set(params, _PARAM_PRINT_STDERR) or config.PRINT_STDERR_ALWAYS 152 | color = None if not _is_colorama_enabled() else Fore.RED 153 | self.stderr = Stream(process.stderr, sys.stderr.encoding, print_stderr, color) 154 | 155 | def sreadline(self): 156 | return self.stdout.sreadline() 157 | 158 | def swriteline(self, text): 159 | self.stdin.swriteline(text) 160 | 161 | @property 162 | def returncode(self): 163 | self._process.wait() 164 | return self._process.returncode 165 | 166 | def __iter__(self): 167 | return iter(self.stdout) 168 | 169 | def __bool__(self): 170 | return self.returncode == 0 171 | 172 | __nonzero__ = __bool__ 173 | 174 | 175 | class Result: 176 | """Result of a shell command execution. 177 | 178 | To get the result stdout as string use str(Result) or Result.stdout or print Result 179 | To get output of stderr use Result.stderr() 180 | 181 | You can also iterate over lines of stdout like this: for line in Result: 182 | 183 | You can access underlying lines of result streams as Result.stdout_lines Result.stderr_lines. 184 | E.g. line_two = Result.stdout_lines[2] 185 | 186 | You can also compare two results that will mean compare of result stdouts 187 | """ 188 | def __init__(self): 189 | self._stdout_lines = [] 190 | self._stderr_lines = [] 191 | self.returncode = None 192 | 193 | @property 194 | def stdout(self): 195 | """Stdout of Result as text 196 | """ 197 | return os.linesep.join(self._stdout_lines) 198 | 199 | @property 200 | def stderr(self): 201 | """Stderr of Result as text 202 | """ 203 | return os.linesep.join(self._stderr_lines) 204 | 205 | @property 206 | def stdout_lines(self): 207 | """List of all lines from stdout 208 | """ 209 | return self._stdout_lines 210 | 211 | @property 212 | def stderr_lines(self): 213 | """List of all lines from stderr 214 | """ 215 | return self._stderr_lines 216 | 217 | def _add_stdout_line(self, line): 218 | line = line.rstrip(os.linesep) 219 | self._stdout_lines.append(line) 220 | 221 | def _add_stderr_line(self, line): 222 | line = line.rstrip(os.linesep) 223 | self._stderr_lines.append(line) 224 | 225 | def __str__(self): 226 | return self.stdout 227 | 228 | def __iter__(self): 229 | return iter(self._stdout_lines) 230 | 231 | def __eq__(self, other): 232 | return self.__str__() == other.__str__() 233 | 234 | def __bool__(self): 235 | return self.returncode == 0 236 | 237 | __nonzero__ = __bool__ 238 | 239 | 240 | def _create_result(cmd, params): 241 | p = subprocess.Popen(cmd, 242 | shell=True, 243 | stdout=subprocess.PIPE, 244 | stderr=subprocess.PIPE, 245 | env=os.environ) 246 | 247 | result = Result() 248 | 249 | for line in p.stdout.readlines(): 250 | if sys.version_info[0] == 3: 251 | line = line.decode(sys.stdout.encoding) 252 | 253 | result._add_stdout_line(line) 254 | 255 | for line in p.stderr.readlines(): 256 | if sys.version_info[0] == 3: 257 | line = line.decode(sys.stderr.encoding) 258 | 259 | result._add_stderr_line(line) 260 | 261 | p.wait() 262 | 263 | if (_is_param_set(params, _PARAM_PRINT_STDOUT) or config.PRINT_STDOUT_ALWAYS) and len(result.stdout) > 0: 264 | _print_stdout(result.stdout) 265 | 266 | if (_is_param_set(params, _PARAM_PRINT_STDERR) or config.PRINT_STDERR_ALWAYS) and len(result.stderr) > 0: 267 | if _is_colorama_enabled(): 268 | _print_stderr(Fore.RED + result.stderr + Style.RESET_ALL) 269 | else: 270 | _print_stderr(result.stderr) 271 | 272 | result.returncode = p.returncode 273 | 274 | if p.returncode != 0 and not _is_param_set(params, _PARAM_NO_THROW): 275 | raise NonZeroReturnCodeError(cmd, result) 276 | 277 | return result 278 | 279 | 280 | def _create_interactive_result(cmd, params): 281 | p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) 282 | 283 | result = InteractiveResult(p, params) 284 | 285 | return result 286 | -------------------------------------------------------------------------------- /shellpython/header.tpl: -------------------------------------------------------------------------------- 1 | #shellpy-encoding 2 | #shellpy-meta:{meta} 3 | from shellpython.core import exe, NonZeroReturnCodeError 4 | 5 | -------------------------------------------------------------------------------- /shellpython/header_root.tpl: -------------------------------------------------------------------------------- 1 | #shellpy-python-executable 2 | #shellpy-encoding 3 | #shellpy-meta:{meta} 4 | 5 | import os 6 | import shellpython 7 | import shellpython.config 8 | from shellpython.constants import * 9 | from shellpython.core import exe, NonZeroReturnCodeError, _print_stderr as shellpy_print_stderr 10 | 11 | if __name__ == '__main__': 12 | shellpython.init() 13 | 14 | if SHELLPY_PARAMS in os.environ: 15 | try: 16 | shellpython.config.loads(os.environ[SHELLPY_PARAMS]) 17 | except Exception as e: 18 | shellpy_print_stderr('Could not load shellpy config: ' + str(e)) 19 | -------------------------------------------------------------------------------- /shellpython/helpers.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | class Dir: 5 | """This class allows to execute commands in directories different from current like this 6 | 7 | with Dir('tmp'): 8 | `ls -l 9 | """ 10 | 11 | def __init__(self, directory): 12 | self.current_dir = directory 13 | 14 | def __enter__(self): 15 | self.previous_dir = os.getcwd() 16 | os.chdir(self.current_dir) 17 | 18 | def __exit__(self, type, value, traceback): 19 | os.chdir(self.previous_dir) 20 | -------------------------------------------------------------------------------- /shellpython/importer.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from importlib import import_module 4 | from shellpython import locator 5 | from shellpython import preprocessor 6 | 7 | 8 | class PreprocessorImporter(object): 9 | """ 10 | Every import of shellpy code requires first preprocessing of shellpy script into a usual python script 11 | and then import of it. To make it we need a hook for every import with standard python hooks. 12 | See PEP-0302 https://www.python.org/dev/peps/pep-0302/ for more details 13 | 14 | When an import occurs the find module function will be called first. It tries to find a shell python module or 15 | file using the Locator class. If the search is successful and there is actually shellpy module or file with 16 | the name specified, the find_module function will return self as Loader. If nothing is found, None will be 17 | returned and the import mechanism of python will not be affected in any other way. 18 | So if something was found then loader will preprocess file or module and import it with standard python 19 | import 20 | """ 21 | 22 | def find_module(self, module_name, package_path): 23 | """This function is part of interface defined in import hooks PEP-0302 24 | Given the name of the module its goal is to locate it. If shellpy module with the name was found, 25 | self is returned as Loader for this module. Otherwise None is returned and standard python import 26 | works as expected 27 | 28 | :param module_name: the module to locate 29 | :param package_path: part of interface, not used, see PEP-0302 30 | :return: self if shellpy module was found, None if not 31 | """ 32 | if module_name.find('.') == -1: 33 | root_module_name = module_name 34 | else: 35 | root_module_name = module_name.split('.')[0] 36 | 37 | spy_module_path = locator.locate_spy_module(root_module_name) 38 | spy_file_path = locator.locate_spy_file(root_module_name) 39 | 40 | if spy_module_path is not None or spy_file_path is not None: 41 | return self 42 | else: 43 | return None 44 | 45 | def load_module(self, module_name): 46 | """If the module was located it then is loaded by this function. It is also a part of PEP-0302 interface 47 | Loading means first preprocessing of shell python code if it is not processed already and the addition 48 | to system path and import. 49 | 50 | :param module_name: the name of the module to load 51 | :return: the module imported. This function assumes that it will import a module anyway since find_module 52 | already found the module 53 | """ 54 | sys.meta_path.remove(self) 55 | 56 | if module_name.find('.') == -1: 57 | spy_module_path = locator.locate_spy_module(module_name) 58 | spy_file_path = locator.locate_spy_file(module_name) 59 | 60 | if spy_module_path is not None: 61 | new_module_path = preprocessor.preprocess_module(spy_module_path) 62 | new_module_pythonpath = os.path.split(new_module_path)[0] 63 | 64 | if new_module_pythonpath not in sys.path: 65 | sys.path.append(new_module_pythonpath) 66 | 67 | module = import_module(module_name) 68 | 69 | elif spy_file_path is not None: 70 | new_module_path = preprocessor.preprocess_file(spy_file_path, is_root_script=False) 71 | new_module_pythonpath = os.path.split(new_module_path)[0] 72 | 73 | if new_module_pythonpath not in sys.path: 74 | sys.path.append(new_module_pythonpath) 75 | 76 | module = import_module(module_name) 77 | 78 | else: 79 | raise ImportError("Unexpected error occured in importer. Neither shellpy module not file was found") 80 | 81 | else: 82 | module = import_module(module_name) 83 | 84 | sys.meta_path.insert(0, self) 85 | 86 | return module 87 | -------------------------------------------------------------------------------- /shellpython/locator.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os.path 3 | 4 | 5 | def locate_spy_module(module_name): 6 | """Tries to find shellpy module on filesystem. Given a module name it tries to locate it in pythonpath. It looks 7 | for a module with the same name and __init__.spy inside of it 8 | 9 | :param module_name: Filename without extension 10 | :return: Path to shellpy file or None if not found 11 | """ 12 | for python_path in sys.path: 13 | 14 | possible_module_path = os.path.join(python_path, module_name) 15 | if os.path.exists(possible_module_path): 16 | if os.path.exists(os.path.join(possible_module_path, '__init__.spy')): 17 | return possible_module_path 18 | 19 | return None 20 | 21 | 22 | def locate_spy_file(file_name): 23 | """Tries to find shellpy file on filesystem. Given a filename without extension it tries to locate it with .spy 24 | extension in pythonpath 25 | 26 | :param file_name: Filename without extension 27 | :return: Path to shellpy file or None if not found 28 | """ 29 | for python_path in sys.path: 30 | 31 | possible_file_path = os.path.join(python_path, file_name + '.spy') 32 | if os.path.exists(possible_file_path): 33 | return possible_file_path 34 | 35 | return None 36 | -------------------------------------------------------------------------------- /shellpython/preprocessor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import stat 4 | import tempfile 5 | import re 6 | import getpass 7 | import json 8 | 9 | spy_file_pattern = re.compile(r'(.*)\.spy$') 10 | shellpy_meta_pattern = re.compile(r'#shellpy-meta:(.*)') 11 | shellpy_encoding_pattern = '#shellpy-encoding' 12 | 13 | 14 | def preprocess_module(module_path): 15 | """The function compiles a module in shellpy to a python module, walking through all the shellpy files inside of 16 | the module and compiling all of them to python 17 | 18 | :param module_path: The path of module 19 | :return: The path of processed module 20 | """ 21 | for item in os.walk(module_path): 22 | path, dirs, files = item 23 | for file in files: 24 | if spy_file_pattern.match(file): 25 | filepath = os.path.join(path, file) 26 | preprocess_file(filepath, is_root_script=False) 27 | 28 | return _translate_to_temp_path(module_path) 29 | 30 | 31 | def preprocess_file(in_filepath, is_root_script, python_version=None): 32 | """Coverts a single shellpy file to python 33 | 34 | :param in_filepath: The path of shellpy file to be processed 35 | :param is_root_script: Shows whether the file being processed is a root file, which means the one 36 | that user executed 37 | :param python_version: version of python, needed to set correct header for root scripts 38 | :return: The path of python file that was created of shellpy script 39 | """ 40 | 41 | new_filepath = spy_file_pattern.sub(r"\1.py", in_filepath) 42 | out_filename = _translate_to_temp_path(new_filepath) 43 | out_folder_path = os.path.dirname(out_filename) 44 | 45 | if not is_root_script and not _is_compilation_needed(in_filepath, out_filename): 46 | # TODO: cache root also 47 | # TODO: if you don't compile but it's root, you need to change to exec 48 | return out_filename 49 | 50 | if not os.path.exists(out_folder_path): 51 | os.makedirs(out_folder_path, mode=0o700) 52 | 53 | header_data = _get_header(in_filepath, is_root_script, python_version) 54 | 55 | with open(in_filepath, 'r') as f: 56 | code = f.read() 57 | 58 | out_file_data = _add_encoding_to_header(header_data, code) 59 | 60 | intermediate = _preprocess_code_to_intermediate(code) 61 | processed_code = _intermediate_to_final(intermediate) 62 | 63 | out_file_data += processed_code 64 | 65 | with open(out_filename, 'w') as f: 66 | f.write(out_file_data) 67 | 68 | in_file_stat = os.stat(in_filepath) 69 | os.chmod(out_filename, in_file_stat.st_mode) 70 | 71 | if is_root_script: 72 | os.chmod(out_filename, in_file_stat.st_mode | stat.S_IEXEC) 73 | 74 | return out_filename 75 | 76 | 77 | def _get_username(): 78 | """Returns the name of current user. The function is used in construction of the path for processed shellpy files on 79 | temp file system 80 | 81 | :return: The name of current user 82 | """ 83 | try: 84 | n = getpass.getuser() 85 | return n 86 | except: 87 | return 'no_username_found' 88 | 89 | 90 | def _translate_to_temp_path(path): 91 | """Compiled shellpy files are stored on temp filesystem on path like this /{tmp}/{user}/{real_path_of_file_on_fs} 92 | Every user will have its own copy of compiled shellpy files. Since we store them somewhere else relative to 93 | the place where they actually are, we need a translation function that would allow us to easily get path 94 | of compiled file 95 | 96 | :param path: The path to be translated 97 | :return: The translated path 98 | """ 99 | absolute_path = os.path.abspath(path) 100 | relative_path = os.path.relpath(absolute_path, os.path.abspath(os.sep)) 101 | # TODO: this will not work in win where root is C:\ and absolute_in_path 102 | # is on D:\ 103 | translated_path = os.path.join(tempfile.gettempdir(), 'shellpy_' + _get_username(), relative_path) 104 | return translated_path 105 | 106 | 107 | def _is_compilation_needed(in_filepath, out_filepath): 108 | """Shows whether compilation of input file is required. It may be not required if the output file did not change 109 | 110 | :param in_filepath: The path of shellpy file to be processed 111 | :param out_filepath: The path of the processed python file. It may exist or not. 112 | :return: True if compilation is needed, False otherwise 113 | """ 114 | if not os.path.exists(out_filepath): 115 | return True 116 | 117 | in_mtime = os.path.getmtime(in_filepath) 118 | 119 | with open(out_filepath, 'r') as f: 120 | for i in range(0, 3): # scan only for three first lines 121 | line = f.readline() 122 | line_result = shellpy_meta_pattern.search(line) 123 | if line_result: 124 | meta = line_result.group(1) 125 | meta = json.loads(meta) 126 | if str(in_mtime) == meta['mtime']: 127 | return False 128 | 129 | return True 130 | 131 | 132 | def _get_header(filepath, is_root_script, python_version): 133 | """To execute converted shellpy file we need to add a header to it. The header contains needed imports and 134 | required code 135 | 136 | :param filepath: A shellpy file that is being converted. It is needed to get modification time of it and save it 137 | to the created python file. Then this modification time will be used to find out whether recompilation is needed 138 | :param is_root_script: Shows whether the file being processed is a root file, which means the one 139 | that user executed 140 | :param python_version: version of python, needed to set correct header for root scripts 141 | :return: data of the header 142 | """ 143 | header_name = 'header_root.tpl' if is_root_script else 'header.tpl' 144 | header_filename = os.path.join(os.path.dirname(__file__), header_name) 145 | 146 | with open(header_filename, 'r') as f: 147 | header_data = f.read() 148 | mod_time = os.path.getmtime(filepath) 149 | meta = {'mtime': str(mod_time)} 150 | 151 | header_data = header_data.replace('{meta}', json.dumps(meta)) 152 | 153 | if is_root_script: 154 | executables = { 155 | 2: '#!/usr/bin/env python', 156 | 3: '#!/usr/bin/env python3' 157 | } 158 | header_data = header_data.replace('#shellpy-python-executable', executables[python_version]) 159 | 160 | return header_data 161 | 162 | 163 | def _preprocess_code_to_intermediate(code): 164 | """Before compiling to actual python code all expressions are converted to universal intermediate form 165 | It is very convenient as it is possible to perform common operations for all expressions 166 | The intermediate form looks like this: 167 | longline_shexe(echo 1)shexe(p)shexe 168 | 169 | :param code: code to convert to intermediate form 170 | :return: converted code 171 | """ 172 | processed_code = _process_multilines(code) 173 | processed_code = _process_long_lines(processed_code) 174 | processed_code = _process_code_both(processed_code) 175 | processed_code = _process_code_start(processed_code) 176 | 177 | return _escape(processed_code) 178 | 179 | 180 | def _process_multilines(script_data): 181 | """Converts a pyshell multiline expression to one line pyshell expression, each line of which is separated 182 | by semicolon. An example would be: 183 | f = ` 184 | echo 1 > test.txt 185 | ls -l 186 | ` 187 | 188 | :param script_data: the string of the whole script 189 | :return: the shellpy script with multiline expressions converted to intermediate form 190 | """ 191 | code_multiline_pattern = re.compile(r'^([^`\n\r]*?)([a-z]*)`\s*?$[\n\r]{1,2}(.*?)`\s*?$', re.MULTILINE | re.DOTALL) 192 | 193 | script_data = code_multiline_pattern.sub(r'\1multiline_shexe(\3)shexe(\2)shexe', script_data) 194 | 195 | pattern = re.compile(r'multiline_shexe.*?shexe', re.DOTALL) 196 | 197 | new_script_data = script_data 198 | for match in pattern.finditer(script_data): 199 | original_str = script_data[match.start():match.end()] 200 | processed_str = re.sub(r'([\r\n]{1,2})', r'; \\\1', original_str) 201 | 202 | new_script_data = new_script_data.replace( 203 | original_str, processed_str) 204 | 205 | return new_script_data 206 | 207 | 208 | def _process_long_lines(script_data): 209 | """Converts to python a pyshell expression that takes more than one line. An example would be: 210 | f = `echo The string \ 211 | on several \ 212 | lines 213 | 214 | :param script_data: the string of the whole script 215 | :return: the shellpy script converted to intermediate form 216 | """ 217 | code_long_line_pattern = re.compile(r'([a-z]*)`(((.*?\\\s*?$)[\n\r]{1,2})+(.*$))', re.MULTILINE) 218 | new_script_data = code_long_line_pattern.sub(r'longline_shexe(\2)shexe(\1)shexe', script_data) 219 | return new_script_data 220 | 221 | 222 | def _process_code_both(script_data): 223 | """Converts to python a pyshell script that has ` symbol both in the beginning of expression and in the end. 224 | An example would be: 225 | f = `echo 1` 226 | 227 | :param script_data: the string of the whole script 228 | :return: the shellpy script converted to intermediate form 229 | """ 230 | code_both_pattern = re.compile(r'([a-z]*)`(.*?)`') 231 | new_script_data = code_both_pattern.sub(r'both_shexe(\2)shexe(\1)shexe', script_data) 232 | return new_script_data 233 | 234 | 235 | def _process_code_start(script_data): 236 | """Converts to python a pyshell script that has ` symbol only in the beginning. An example would be: 237 | f = `echo 1 238 | 239 | :param script_data: the string of the whole script 240 | :return: the shellpy script converted to intermediate form 241 | """ 242 | code_start_pattern = re.compile(r'^([^\n\r`]*?)([a-z]*)`([^`\n\r]+)$', re.MULTILINE) 243 | new_script_data = code_start_pattern.sub(r'\1start_shexe(\3)shexe(\2)shexe', script_data) 244 | return new_script_data 245 | 246 | 247 | def _escape(script_data): 248 | """Escapes shell commands 249 | 250 | :param script_data: the string of the whole script 251 | :return: escaped script 252 | """ 253 | pattern = re.compile(r'[a-z]*_shexe.*?shexe', re.DOTALL) 254 | 255 | new_script_data = script_data 256 | for match in pattern.finditer(script_data): 257 | original_str = script_data[match.start():match.end()] 258 | if original_str.find('\'') != -1: 259 | processed_str = original_str.replace('\'', '\\\'') 260 | 261 | new_script_data = new_script_data.replace( 262 | original_str, processed_str) 263 | 264 | return new_script_data 265 | 266 | 267 | def _intermediate_to_final(script_data): 268 | """All shell blocks are first compiled to intermediate form. This part of code converts the intermediate 269 | to final python code 270 | 271 | :param script_data: the string of the whole script 272 | :return: python script ready to be executed 273 | """ 274 | intermediate_pattern = re.compile(r'[a-z]*_shexe\((.*?)\)shexe\((.*?)\)shexe', re.MULTILINE | re.DOTALL) 275 | final_script = intermediate_pattern.sub(r"exe('\1'.format(**dict(locals(), **globals())),'\2')", script_data) 276 | return final_script 277 | 278 | 279 | def _add_encoding_to_header(header_data, script_data): 280 | """PEP-0263 defines a way to specify python file encoding. If this encoding is present in first 281 | two lines of a shellpy script it will then be moved to the top generated output file 282 | 283 | :param script_data: the string of the whole script 284 | :return: the script with the encoding moved to top, if it's present 285 | """ 286 | encoding_pattern = re.compile(r'^(#[-*\s]*coding[:=]\s*([-\w.]+)[-*\s]*)$') 287 | 288 | # we use \n here instead of os.linesep since \n is universal as it is present in all OSes 289 | # when \r\n returned by os.linesep may not work if you run against unix files from win 290 | first_two_lines = script_data.split('\n')[:2] 291 | for line in first_two_lines: 292 | encoding = encoding_pattern.search(line) 293 | if encoding is not None: 294 | break 295 | 296 | if not encoding: 297 | return header_data 298 | else: 299 | new_header_data = header_data.replace(shellpy_encoding_pattern, encoding.group(1)) 300 | return new_header_data 301 | -------------------------------------------------------------------------------- /shellpython/shellpy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import os 4 | import re 5 | import subprocess 6 | import shellpython.config as config 7 | from shellpython.preprocessor import preprocess_file 8 | from argparse import ArgumentParser 9 | from shellpython.constants import * 10 | 11 | 12 | def main2(): 13 | main(python_version=2) 14 | 15 | 16 | def main3(): 17 | main(python_version=3) 18 | 19 | 20 | def main(python_version): 21 | custom_usage = '''%(prog)s [SHELLPY ARGS] file [SCRIPT ARGS] 22 | 23 | For arguments help use: 24 | %(prog)s --help 25 | ''' 26 | custom_epilog = '''github : github.com/lamerman/shellpy''' 27 | 28 | try: 29 | spy_file_index = next(index for index, arg in enumerate(sys.argv) if re.match('.+\.spy$', arg)) 30 | shellpy_args = sys.argv[1:spy_file_index] 31 | script_args = sys.argv[spy_file_index + 1:] 32 | except StopIteration: 33 | shellpy_args = sys.argv[1:] 34 | spy_file_index = None 35 | 36 | parser = ArgumentParser(description='A tool for convenient shell scripting in python', 37 | usage=custom_usage, epilog=custom_epilog) 38 | parser.add_argument('-v', '--verbose', help='increase output verbosity. Always print the command being executed', 39 | action="store_true") 40 | parser.add_argument('-vv', help='even bigger output verbosity. All stdout and stderr of executed commands is ' 41 | 'printed', action="store_true") 42 | 43 | shellpy_args, _ = parser.parse_known_args(shellpy_args) 44 | 45 | if spy_file_index is None: 46 | exit('No *.spy file was specified. Only *.spy files are supported by the tool.') 47 | 48 | if shellpy_args.verbose or shellpy_args.vv: 49 | config.PRINT_ALL_COMMANDS = True 50 | 51 | if shellpy_args.vv: 52 | config.PRINT_STDOUT_ALWAYS = True 53 | config.PRINT_STDERR_ALWAYS = True 54 | 55 | filename = sys.argv[spy_file_index] 56 | 57 | processed_file = preprocess_file(filename, is_root_script=True, python_version=python_version) 58 | 59 | # include directory of the script to pythonpath 60 | new_env = os.environ.copy() 61 | new_env['PYTHONPATH'] = new_env.get("PYTHONPATH", '') + os.pathsep + os.path.dirname(filename) 62 | new_env[SHELLPY_PARAMS] = config.dumps() 63 | 64 | root_command = processed_file 65 | if sys.platform == "win32": 66 | root_command = "python " + root_command 67 | retcode = subprocess.call(root_command + ' ' + ' '.join(script_args), shell=True, env=new_env) 68 | exit(retcode) 69 | 70 | 71 | if __name__ == '__main__': 72 | main() 73 | -------------------------------------------------------------------------------- /shellpython/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lamerman/shellpy/544cb811d5fb0e8cb6d9934cef71f9e66e645cf8/shellpython/tests/__init__.py -------------------------------------------------------------------------------- /shellpython/tests/data/locator/testfilepy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lamerman/shellpy/544cb811d5fb0e8cb6d9934cef71f9e66e645cf8/shellpython/tests/data/locator/testfilepy.py -------------------------------------------------------------------------------- /shellpython/tests/data/locator/testfilespy.spy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lamerman/shellpy/544cb811d5fb0e8cb6d9934cef71f9e66e645cf8/shellpython/tests/data/locator/testfilespy.spy -------------------------------------------------------------------------------- /shellpython/tests/data/locator/testmodulepy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lamerman/shellpy/544cb811d5fb0e8cb6d9934cef71f9e66e645cf8/shellpython/tests/data/locator/testmodulepy/__init__.py -------------------------------------------------------------------------------- /shellpython/tests/data/locator/testmodulespy/__init__.spy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lamerman/shellpy/544cb811d5fb0e8cb6d9934cef71f9e66e645cf8/shellpython/tests/data/locator/testmodulespy/__init__.spy -------------------------------------------------------------------------------- /shellpython/tests/data/preprocessor/meta.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | #shellpy-meta:{"mtime": "1111111111.11"} 4 | -------------------------------------------------------------------------------- /shellpython/tests/data/preprocessor/test.imdt: -------------------------------------------------------------------------------- 1 | # simple 2 | 3 | start_shexe(echo 1)shexe()shexe 4 | 5 | both_shexe(echo 2)shexe()shexe 6 | 7 | longline_shexe(echo Long \ 8 | line)shexe()shexe 9 | 10 | multiline_shexe(echo line 1; \ 11 | echo line 2; \ 12 | )shexe()shexe 13 | 14 | # with argument 15 | 16 | start_shexe(echo 1)shexe(p)shexe 17 | 18 | both_shexe(echo 2)shexe(p)shexe 19 | 20 | longline_shexe(echo Long \ 21 | line)shexe(p)shexe 22 | 23 | multiline_shexe(echo line 1; \ 24 | echo line 2; \ 25 | )shexe(p)shexe 26 | 27 | # assignment 28 | 29 | p = start_shexe(echo 1)shexe()shexe 30 | 31 | p = both_shexe(echo 2)shexe()shexe 32 | 33 | p = longline_shexe(echo Long \ 34 | line)shexe()shexe 35 | 36 | p = multiline_shexe(echo line 1; \ 37 | echo line 2; \ 38 | )shexe()shexe 39 | 40 | # complex 41 | 42 | both_shexe(echo 1)shexe()shexe == start_shexe(echo 1)shexe()shexe 43 | 44 | both_shexe(echo 1)shexe()shexe == both_shexe(echo 1)shexe()shexe 45 | -------------------------------------------------------------------------------- /shellpython/tests/data/preprocessor/test.spy: -------------------------------------------------------------------------------- 1 | # simple 2 | 3 | `echo 1 4 | 5 | `echo 2` 6 | 7 | `echo Long \ 8 | line 9 | 10 | ` 11 | echo line 1 12 | echo line 2 13 | ` 14 | 15 | # with argument 16 | 17 | p`echo 1 18 | 19 | p`echo 2` 20 | 21 | p`echo Long \ 22 | line 23 | 24 | p` 25 | echo line 1 26 | echo line 2 27 | ` 28 | 29 | # assignment 30 | 31 | p = `echo 1 32 | 33 | p = `echo 2` 34 | 35 | p = `echo Long \ 36 | line 37 | 38 | p = ` 39 | echo line 1 40 | echo line 2 41 | ` 42 | 43 | # complex 44 | 45 | `echo 1` == `echo 1 46 | 47 | `echo 1` == `echo 1` 48 | -------------------------------------------------------------------------------- /shellpython/tests/test_helpers.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import tempfile 3 | import os 4 | from os import path 5 | from shellpython.helpers import Dir 6 | 7 | 8 | class TestDirectory(unittest.TestCase): 9 | 10 | def test_relative_dirs(self): 11 | cur_dir = path.dirname(path.abspath(__file__)) 12 | 13 | with Dir(path.join(cur_dir, 'data')): 14 | self.assertEqual(path.join(cur_dir, 'data'), os.getcwd()) 15 | with Dir(path.join('locator')): 16 | self.assertEqual(path.join(cur_dir, 'data', 'locator'), os.getcwd()) 17 | 18 | def test_absolute_dirs(self): 19 | with Dir(tempfile.gettempdir()): 20 | self.assertEqual(tempfile.gettempdir(), os.getcwd()) 21 | -------------------------------------------------------------------------------- /shellpython/tests/test_locator.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import unittest 4 | from shellpython import locator 5 | 6 | 7 | class TestLocateModule(unittest.TestCase): 8 | def setUp(self): 9 | cur_dir = os.path.split(__file__)[0] 10 | self.test_dir = os.path.join(cur_dir, 'data', 'locator') 11 | 12 | sys.path.append(self.test_dir) 13 | 14 | def tearDown(self): 15 | sys.path.remove(self.test_dir) 16 | 17 | def test_files(self): 18 | self.assertIsNotNone(locator.locate_spy_file('testfilespy')) 19 | self.assertIsNone(locator.locate_spy_file('testfilepy')) 20 | self.assertIsNone(locator.locate_spy_file('non_existent_file')) 21 | self.assertIsNone(locator.locate_spy_file('testmodulespy')) 22 | 23 | def test_modules(self): 24 | self.assertIsNotNone(locator.locate_spy_module('testmodulespy')) 25 | self.assertIsNone(locator.locate_spy_module('testmodulepy')) 26 | self.assertIsNone(locator.locate_spy_module('testmoduleempty')) 27 | self.assertIsNone(locator.locate_spy_module('testfilepy')) 28 | self.assertIsNone(locator.locate_spy_module('non_existent_file')) 29 | -------------------------------------------------------------------------------- /shellpython/tests/test_preprocessor.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import getpass 3 | import tempfile 4 | import os.path 5 | import mock 6 | from shellpython import preprocessor 7 | 8 | 9 | class TestCodeStart(unittest.TestCase): 10 | def test_simple(self): 11 | cmd = 'x = `echo 1' 12 | intermediate = preprocessor._process_code_start(cmd) 13 | self.assertEqual(intermediate, 'x = start_shexe(echo 1)shexe()shexe') 14 | 15 | def test_very_simple(self): 16 | cmd = '`echo 1' 17 | intermediate = preprocessor._process_code_start(cmd) 18 | self.assertEqual(intermediate, 'start_shexe(echo 1)shexe()shexe') 19 | 20 | def test_with_param(self): 21 | cmd = 'x = p`echo 1' 22 | intermediate = preprocessor._process_code_start(cmd) 23 | self.assertEqual(intermediate, 'x = start_shexe(echo 1)shexe(p)shexe') 24 | 25 | def test_multiline(self): 26 | cmd = 'i = 1\nx = `echo 1\ni = 2' 27 | intermediate = preprocessor._process_code_start(cmd) 28 | self.assertEqual(intermediate, 'i = 1\nx = start_shexe(echo 1)shexe()shexe\ni = 2') 29 | 30 | def test_no_false_positive_both(self): 31 | cmd = 'x = `echo 1`' 32 | intermediate = preprocessor._process_code_start(cmd) 33 | self.assertEqual(intermediate, 'x = `echo 1`') 34 | 35 | 36 | class TestCodeBoth(unittest.TestCase): 37 | def test_simple(self): 38 | cmd = 'x = `echo 1`' 39 | intermediate = preprocessor._process_code_both(cmd) 40 | self.assertEqual(intermediate, 'x = both_shexe(echo 1)shexe()shexe') 41 | 42 | def test_very_simple(self): 43 | cmd = '`echo 1`' 44 | intermediate = preprocessor._process_code_both(cmd) 45 | self.assertEqual(intermediate, 'both_shexe(echo 1)shexe()shexe') 46 | 47 | def test_with_param(self): 48 | cmd = 'x = p`echo 1`' 49 | intermediate = preprocessor._process_code_both(cmd) 50 | self.assertEqual(intermediate, 'x = both_shexe(echo 1)shexe(p)shexe') 51 | 52 | def test_multiline(self): 53 | cmd = 'i = 1\nx = `echo 1`\ni = 2' 54 | intermediate = preprocessor._process_code_both(cmd) 55 | self.assertEqual(intermediate, 'i = 1\nx = both_shexe(echo 1)shexe()shexe\ni = 2') 56 | 57 | def test_no_false_positive_start(self): 58 | cmd = 'x = `echo 1' 59 | intermediate = preprocessor._process_code_both(cmd) 60 | self.assertEqual(intermediate, 'x = `echo 1') 61 | 62 | 63 | class TestLongLines(unittest.TestCase): 64 | def test_simple_two_lines(self): 65 | cmd = 'x = `echo Very \\\n long line' 66 | intermediate = preprocessor._process_long_lines(cmd) 67 | self.assertEqual(intermediate, 'x = longline_shexe(echo Very \\\n long line)shexe()shexe') 68 | 69 | def test_very_simple_two_lines(self): 70 | cmd = '`echo Very \\\n long line' 71 | intermediate = preprocessor._process_long_lines(cmd) 72 | self.assertEqual(intermediate, 'longline_shexe(echo Very \\\n long line)shexe()shexe') 73 | 74 | def test_simple_three_lines(self): 75 | cmd = 'x = `echo Very \\\n long \\\n line' 76 | intermediate = preprocessor._process_long_lines(cmd) 77 | self.assertEqual(intermediate, 'x = longline_shexe(echo Very \\\n long \\\n line)shexe()shexe') 78 | 79 | def test_simple_two_lines_with_param(self): 80 | cmd = 'x = p`echo Very \\\n long line' 81 | intermediate = preprocessor._process_long_lines(cmd) 82 | self.assertEqual(intermediate, 'x = longline_shexe(echo Very \\\n long line)shexe(p)shexe') 83 | 84 | def test_no_false_positive_start(self): 85 | cmd = 'x = `echo 1' 86 | intermediate = preprocessor._process_long_lines(cmd) 87 | self.assertEqual(intermediate, 'x = `echo 1') 88 | 89 | 90 | class TestMultiline(unittest.TestCase): 91 | def test_simple_one_line(self): 92 | cmd = 'x = `\necho 1\n`' 93 | intermediate = preprocessor._process_multilines(cmd) 94 | self.assertEqual(intermediate, 'x = multiline_shexe(echo 1; \\\n)shexe()shexe') 95 | 96 | def test_simple_one_line_with_param(self): 97 | cmd = 'x = p`\necho 1\n`' 98 | intermediate = preprocessor._process_multilines(cmd) 99 | self.assertEqual(intermediate, 'x = multiline_shexe(echo 1; \\\n)shexe(p)shexe') 100 | 101 | def test_very_simple_one_line(self): 102 | cmd = '`\necho 1\n`' 103 | intermediate = preprocessor._process_multilines(cmd) 104 | self.assertEqual(intermediate, 'multiline_shexe(echo 1; \\\n)shexe()shexe') 105 | 106 | def test_simple_two_lines(self): 107 | cmd = 'x = `\necho 1\necho 2\n`' 108 | intermediate = preprocessor._process_multilines(cmd) 109 | self.assertEqual(intermediate, 'x = multiline_shexe(echo 1; \\\necho 2; \\\n)shexe()shexe') 110 | 111 | def test_no_false_positive_start(self): 112 | cmd = 'x = `echo 1' 113 | intermediate = preprocessor._process_multilines(cmd) 114 | self.assertEqual(intermediate, 'x = `echo 1') 115 | 116 | def test_no_false_positive_both(self): 117 | cmd = 'x = `echo 1`' 118 | intermediate = preprocessor._process_multilines(cmd) 119 | self.assertEqual(intermediate, 'x = `echo 1`') 120 | 121 | 122 | class TestEscape(unittest.TestCase): 123 | def test_escape_nothing(self): 124 | intermediate = 'x = start_shexe(echo 1)shexe()shexe' 125 | escaped = preprocessor._escape(intermediate) 126 | self.assertEqual(escaped, intermediate) 127 | 128 | def test_escape_quote(self): 129 | intermediate = 'x = start_shexe(echo \'1\')shexe()shexe' 130 | escaped = preprocessor._escape(intermediate) 131 | self.assertEqual(escaped, 'x = start_shexe(echo \\\'1\\\')shexe()shexe') 132 | 133 | 134 | class TestIntermediateToFinal(unittest.TestCase): 135 | def test_common(self): 136 | intermediate = 'x = int_shexe(expr)shexe(param)shexe' 137 | final = preprocessor._intermediate_to_final(intermediate) 138 | self.assertEqual(final, "x = exe('expr'.format(**dict(locals(), **globals())),'param')") 139 | 140 | 141 | class TestFileOperations(unittest.TestCase): 142 | def test_translate_to_temp(self): 143 | username = getpass.getuser() 144 | translated_path = preprocessor._translate_to_temp_path('/code/test') 145 | 146 | expected_path = os.path.join(tempfile.gettempdir(), 'shellpy_' + username, 'code/test') 147 | 148 | self.assertEqual(translated_path, expected_path) 149 | 150 | 151 | class TestMeta(unittest.TestCase): 152 | def setUp(self): 153 | cur_dir = os.path.split(__file__)[0] 154 | self.test_dir = os.path.join(cur_dir, 'data', 'preprocessor') 155 | 156 | @mock.patch('os.path.getmtime', return_value=1111111111.11) 157 | def test_is_compilation_needed_time_match(self, getmtime_mock_arg): 158 | compilation_needed = preprocessor._is_compilation_needed('mocked_spy_file', 159 | os.path.join(self.test_dir, 'meta.py')) 160 | self.assertFalse(compilation_needed) 161 | 162 | @mock.patch('os.path.getmtime', return_value=0) 163 | def test_is_compilation_needed_time_not_match(self, getmtime_mock_arg): 164 | compilation_needed = preprocessor._is_compilation_needed('mocked_spy_file', 165 | os.path.join(self.test_dir, 'meta.py')) 166 | self.assertTrue(compilation_needed) 167 | 168 | def test_is_compilation_needed_out_does_not_exist(self): 169 | compilation_needed = preprocessor._is_compilation_needed('mocked_spy_file', 170 | 'non_existing_py_file') 171 | self.assertTrue(compilation_needed) 172 | 173 | 174 | class TestEncoding(unittest.TestCase): 175 | 176 | def setUp(self): 177 | self.header_text = '''#!/usr/bin/env python 178 | #shellpy-encoding 179 | #shellpy-meta:{meta} 180 | 181 | import sys 182 | import os 183 | ''' 184 | 185 | self.header_text_with_encoding = '''#!/usr/bin/env python 186 | # -*- coding: utf-8 -*- 187 | #shellpy-meta:{meta} 188 | 189 | import sys 190 | import os 191 | ''' 192 | 193 | def test_no_encoding(self): 194 | script_text = '''print 1 195 | ''' 196 | header_text_with_encoding = preprocessor._add_encoding_to_header(self.header_text, script_text) 197 | self.assertEqual(header_text_with_encoding, self.header_text) 198 | 199 | def test_encoding_on_first_line(self): 200 | script_text = '''# -*- coding: utf-8 -*- 201 | print 1 202 | ''' 203 | header_text_with_encoding = preprocessor._add_encoding_to_header(self.header_text, script_text) 204 | self.assertEqual(header_text_with_encoding, self.header_text_with_encoding) 205 | 206 | def test_encoding_on_second_line(self): 207 | script_text = '''#!/usr/bin/env python 208 | # -*- coding: utf-8 -*- 209 | print 1 210 | ''' 211 | header_text_with_encoding = preprocessor._add_encoding_to_header(self.header_text, script_text) 212 | self.assertEqual(header_text_with_encoding, self.header_text_with_encoding) 213 | 214 | def test_encoding_on_third_line(self): 215 | script_text = '''#!/usr/bin/env python 216 | # some comment 217 | # -*- coding: utf-8 -*- 218 | print 1 219 | ''' 220 | header_text_with_encoding = preprocessor._add_encoding_to_header(self.header_text, script_text) 221 | self.assertEqual(header_text_with_encoding, self.header_text) 222 | 223 | 224 | -------------------------------------------------------------------------------- /shellpython/tests/test_preprocessor_intermediate.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | import re 4 | from shellpython import preprocessor 5 | 6 | 7 | class TestPreprocessorIntermediate(unittest.TestCase): 8 | def setUp(self): 9 | cur_dir = os.path.split(__file__)[0] 10 | self.test_dir = os.path.join(cur_dir, 'data', 'preprocessor') 11 | 12 | def test_process_and_compare_file(self): 13 | with open(os.path.join(self.test_dir, 'test.spy')) as code_f: 14 | imdt = preprocessor._preprocess_code_to_intermediate(code_f.read()) 15 | 16 | with open(os.path.join(self.test_dir, 'test.imdt')) as imdt_etalon_f: 17 | imdt_etalon = imdt_etalon_f.read() 18 | 19 | statement_regex = re.compile(r'((.+\n)+)') 20 | 21 | imdt_matches = [[match[0]] for match in statement_regex.findall(imdt)] 22 | imdt_etalon_matches = [[match[0]] for match in statement_regex.findall(imdt_etalon)] 23 | 24 | self.assertEqual(len(imdt_matches), len(imdt_etalon_matches)) 25 | 26 | zipped_matches = map(list.__add__, imdt_matches, imdt_etalon_matches) 27 | for matches in zipped_matches: 28 | self.assertEqual(matches[0], matches[1]) 29 | 30 | -------------------------------------------------------------------------------- /shellpython/tests/test_unix_core.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import mock 3 | 4 | from shellpython import core, config 5 | 6 | 7 | class StreamMock: 8 | 9 | def __init__(self): 10 | self.lines = [] 11 | 12 | def mocked_print(self, text): 13 | self.lines.append(text) 14 | 15 | 16 | class TestExecute(unittest.TestCase): 17 | 18 | def setUp(self): 19 | self.stdout_mock = StreamMock() 20 | self.stderr_mock = StreamMock() 21 | core._print_stdout = self.stdout_mock.mocked_print 22 | core._print_stderr = self.stderr_mock.mocked_print 23 | 24 | def test_simple_echo_success(self): 25 | result = core._create_result('echo 1', '') 26 | self.assertEqual(result.returncode, 0) 27 | 28 | self.assertEqual(result.stdout, '1') 29 | self.assertEqual(len(result.stdout_lines), 1) 30 | self.assertEqual(result.stdout_lines[0], '1') 31 | 32 | self.assertEqual(result.stderr, '') 33 | self.assertEqual(len(result.stderr_lines), 0) 34 | 35 | self.assertEqual(len(self.stdout_mock.lines), 0) 36 | self.assertEqual(len(self.stderr_mock.lines), 0) 37 | 38 | def test_simple_echo_failure(self): 39 | with self.assertRaises(core.NonZeroReturnCodeError) as cm: 40 | core._create_result('cat non_existent_file', '') 41 | 42 | result = cm.exception.result 43 | 44 | self.assertNotEqual(result.returncode, 0) 45 | 46 | self.assertEqual(result.stdout, '') 47 | self.assertEqual(len(result.stdout_lines), 0) 48 | 49 | self.assertGreater(len(result.stderr), 0) 50 | self.assertGreater(len(result.stderr_lines), 0) 51 | 52 | self.assertEqual(len(self.stdout_mock.lines), 0) 53 | self.assertEqual(len(self.stderr_mock.lines), 0) 54 | 55 | def test_param_p(self): 56 | core._create_result('echo 1', 'p') 57 | 58 | self.assertEqual(len(self.stdout_mock.lines), 1) 59 | self.assertEqual(self.stdout_mock.lines[0], '1') 60 | 61 | self.assertEqual(len(self.stderr_mock.lines), 0) 62 | 63 | def test_param_e(self): 64 | with self.assertRaises(core.NonZeroReturnCodeError) as cm: 65 | core._create_result('cat non_existent_file', 'e') 66 | 67 | result = cm.exception.result 68 | 69 | self.assertEqual(len(self.stdout_mock.lines), 0) 70 | 71 | self.assertGreater(len(self.stderr_mock.lines), 0) 72 | self.assertEqual(len(self.stderr_mock.lines), len(result.stderr_lines)) 73 | 74 | def test_param_e_no_stderr(self): 75 | core._create_result('echo 1', 'e') 76 | 77 | self.assertEqual(len(self.stderr_mock.lines), 0) 78 | 79 | def test_param_p_no_stdout(self): 80 | with self.assertRaises(core.NonZeroReturnCodeError): 81 | core._create_result('cat non_existent_file', 'p') 82 | 83 | self.assertEqual(len(self.stdout_mock.lines), 0) 84 | 85 | def test_exe(self): 86 | result = core.exe('echo 1', '') 87 | 88 | self.assertEqual(result.stdout, '1') 89 | self.assertEqual(str(result), '1') 90 | self.assertTrue(result == '1') 91 | self.assertEqual(result.__bool__(), True) 92 | 93 | self.assertEqual(len(self.stdout_mock.lines), 0) 94 | self.assertEqual(len(self.stderr_mock.lines), 0) 95 | 96 | def test_interactive(self): 97 | result = core.exe('echo 1', 'i') 98 | 99 | self.assertEqual(result.sreadline(), '1') 100 | self.assertRaises(StopIteration, result.sreadline) 101 | 102 | self.assertEqual(len(self.stdout_mock.lines), 0) 103 | self.assertEqual(len(self.stderr_mock.lines), 0) 104 | 105 | def test_interactive_param_p(self): 106 | result = core.exe('echo 1', 'ip') 107 | 108 | self.assertEqual(result.sreadline(), '1') 109 | self.assertRaises(StopIteration, result.sreadline) 110 | 111 | self.assertEqual(len(self.stdout_mock.lines), 1) 112 | self.assertEqual(self.stdout_mock.lines[0], '1') 113 | 114 | self.assertEqual(len(self.stderr_mock.lines), 0) 115 | 116 | @mock.patch('shellpython.config.PRINT_ALL_COMMANDS', True) 117 | @mock.patch('shellpython.config.COLORAMA_ENABLED', False) 118 | def test_config_print_all(self): 119 | core.exe('echo 1', '') 120 | 121 | self.assertEqual(len(self.stdout_mock.lines), 1) 122 | self.assertEqual(self.stdout_mock.lines[0], '>>> echo 1') 123 | 124 | --------------------------------------------------------------------------------