├── .gitignore ├── README.md ├── doc ├── short.png └── source.png ├── poetry.sh ├── pyproject.toml ├── src └── pysh │ ├── __init__.py │ ├── pype.py │ ├── serializer.py │ └── utils.py ├── tests ├── demo.py ├── features.py ├── myscript.sh ├── regtext.py └── stdout.py └── todo.md /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | dist/ 3 | *.egg-info/ 4 | .pypirc 5 | 6 | scratch.md 7 | blocks.dat -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Pysh/Pype 2 | 3 | Python source file preprocessor/interpreter and subprocess pipe manager to enable running in-line bash/shell code during python runtime. 4 | 5 | ```Python 6 | #!/usr/bin/env python 7 | from pysh import pysh 8 | pysh() 9 | 10 | #$@ echo "Pysh activated" 11 | stdout = ""#$ echo "This is standard out" 12 | print(stdout) 13 | 14 | ##@!python print("Python through a pysh pype") 15 | ``` 16 | 17 | Use #$ flagged directives to signify shell code to be piped through a subprocess. 18 | 19 | When you run pysh() execution stops, and the source file will be processed with regex to extract and replace the code blocks with a subprocess wrapper, and then the source file itself is run through it. 20 | 21 | ##### Real usage 22 | ```Python 23 | # Script your system in parallel with your python code execution 24 | # Do anything you can in bash e.g. 25 | 26 | build_service() 27 | #$ cd ~/hosted/myservice && docker compose up 28 | 29 | aggregate_assets() 30 | crf = "23" 31 | in_file = "/path/in.mp4" 32 | out_file = "/path/out.mp4" 33 | fmpg_result = ""#$ ffmpeg -i {$in_file$} \ 34 | #$ -crf {$crf$} -o {$out_file$} \ 35 | #$ && rm {$in_file$} 36 | process_assets(process_fmpg_stdout(fmpg_result)) 37 | 38 | print("Process complete") 39 | ``` 40 | 41 | ### Installation 42 | From PyPI: 43 | 44 | `pip3 install pyshpype` 45 | 46 | Git to local folder: 47 | 48 | `pip3 install -e "git+https://github.com/blipk/pysh.git#egg=pysh"` 49 | 50 | 51 | 52 | ###### General syntax 53 | ```Python 54 | #!/usr/bin/python 55 | from pysh import pysh 56 | pysh() 57 | #$ echo "pysh activated" >> .pysh 58 | 59 | # Use the @ flag to always print(stdout) to main sys.stdout 60 | #$@ echo "hello from bash" 61 | 62 | # This is a python comment 63 | #$@ ls . # shell eol comment 64 | ##$ sudo rm -rf / # disable pysh line with pysh comment 65 | 66 | # Capture stdout from the shell subprocess 67 | stout = ""#$ echo "I'm actually a bytes string" 68 | print(stdout.decode("UTF-8")) 69 | 70 | # Pass any python variable thats in scope to the pysh script 71 | my_var = "hello" 72 | stdout = ""#$ echo "{$myvar$}" 73 | 74 | # run external script with double $ 75 | #$$ my_script.sh 76 | 77 | # optionally pass arguments to it 78 | #$$ argumentative_script.sh arg1 arg2 79 | 80 | # Use the ! flag to change the shell that interprets the script 81 | # must support -c command_strings or filepath if external $$ 82 | #$!sh echo "simple" 83 | #$!perl oysters.pl 84 | #$$@!bash printscript.sh 85 | 86 | # Multiple flags/features 87 | stdout = ""#$@!python import time 88 | #$ print("The time and date", time.asctime()) 89 | 90 | # Use the % flag to catch errors, 91 | # otherwise they will be printed but not raised 92 | try: 93 | result = ""#$$% tests/dinger/notfoundscript.sh "argone" 94 | except SystemExit as e: 95 | print("Error", e) 96 | 97 | if __name__ == "main": 98 | print("Before the above script blocks are run") 99 | ``` 100 | 101 | 102 | ###### Multiple inline pysh 103 | ```Python 104 | # Pysh runs code in blocks that are executed in-place 105 | 106 | # Block 0 107 | #$ cd $HOME 108 | 109 | stdout_block1 = ""#$ echo "first block is multiline" 110 | #$ echo "line1" 111 | #$ echo "line2" 112 | 113 | # The last script block won't be run 114 | sys.exit(1) 115 | stdout_block2 == ""#$ echo "Second" 116 | #$ echo "Block" 117 | ``` 118 | 119 | 120 | ##### Advanced usage 121 | ```Python 122 | # run pysh manually 123 | from pysh import Pysh 124 | source_file = __file__ 125 | pysher = Pysh(source_file) 126 | blocks = pysher.findblocks() 127 | 128 | # Run a a single block 129 | blocks[0].run() # Not run in-place, no stdout. Silent. 130 | 131 | # Run script block again, and print stdout with label for block 132 | blocks[0].runp() 133 | 134 | # Run all wanted blocks sequentially at this point, 135 | # and print their stdout with labels 136 | run_blocks = [block.runp() for block in blocks 137 | if "/root" in block.srcs] 138 | 139 | # Start the python interpreter with pysh on source_file 140 | # This is the same as running pysh(__file__) 141 | pysher.shyp() 142 | #$ echo "pysh enabled" 143 | 144 | # Switch to another source file and run it through pysh 145 | pysher.pysh(__file__) 146 | 147 | # Equivalent to above 148 | pysher.updatesrc(__file__) 149 | pysher.pysh() 150 | 151 | # Get information about the script blocks at runtime 152 | t_block = pysher.blocks[0] 153 | print(t_block.hasrun, t_block.returncode) 154 | print(t_block.srcs, "\n--\n", t_block.stdout) 155 | ``` -------------------------------------------------------------------------------- /doc/short.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blipk/pysh/0963a2ae0f52b8b3361720d7d76b1c3e61ca995c/doc/short.png -------------------------------------------------------------------------------- /doc/source.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blipk/pysh/0963a2ae0f52b8b3361720d7d76b1c3e61ca995c/doc/source.png -------------------------------------------------------------------------------- /poetry.sh: -------------------------------------------------------------------------------- 1 | # poetry init # create config 2 | # poetry shell # create venv 3 | # poetry add # add packagers 4 | poetry build 5 | poetry -v build --format=wheel 6 | # poetry publish 7 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "pyshpype" 3 | version = "0.1.2" 4 | description = "Run shell in python" 5 | authors = [ 6 | "Blipk A.D." 7 | ] 8 | homepage = "https://github.com/blipk/pysh" 9 | repository = "https://github.com/blipk/pysh" 10 | documentation = "https://github.com/blipk/pysh" 11 | license = "MIT" 12 | 13 | # README file(s) are used as the package description 14 | readme = "README.md" #, "LICENSE" 15 | keywords = ["pysh", "shell", "bash"] # Keywords (translated to tags on the package index) 16 | classifiers = [ 17 | "Topic :: Software Development :: Libraries :: Python Modules" 18 | ] 19 | 20 | packages = [ 21 | { include = "pysh", from = "src" } 22 | ] 23 | # include = [ 24 | # { path = "tests", format = "sdist" } 25 | # ] 26 | 27 | 28 | [tool.poetry.dependencies] 29 | # Compatible Python versions 30 | python = ">=3.10" 31 | 32 | # # Dependency groups are supported for organizing your dependencies 33 | # [tool.poetry.group.dev.dependencies] 34 | # pytest = "^7.1.2" 35 | # pytest-cov = "^3.0" 36 | 37 | # # ...and can be installed only when explicitly requested 38 | # [tool.poetry.group.docs] 39 | # optional = true 40 | # [tool.poetry.group.docs.dependencies] 41 | # Sphinx = "^5.1.1" 42 | 43 | # # Python-style entrypoints and scripts are easily expressed 44 | # [tool.poetry.scripts] 45 | # my-script = "my_package:main" -------------------------------------------------------------------------------- /src/pysh/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import os 3 | import re 4 | import sys 5 | import uuid 6 | import shlex 7 | import msgpack 8 | from pathlib import Path 9 | from inspect import stack 10 | from ast import literal_eval 11 | 12 | from pysh.pype import pype, ScriptException, ScriptRun, default_pipe 13 | from pysh.serializer import encode, decode 14 | from pysh.utils import repr_, timeit 15 | 16 | PYSH_LINE = "#$" 17 | 18 | injector_imports = """import os 19 | import msgpack 20 | from pathlib import Path 21 | from pysh import Pysh, pysher, pysh, encode, decode, BashBlock, timeit 22 | from pysh.pype import pype, ScriptException 23 | """ 24 | 25 | block_injector = injector_imports + """ 26 | class BlockInjector(): 27 | def __init__(self, pysher, pipe=None): 28 | self.pysh = pysher 29 | env = {"PYSH_RUNNING": "1"} 30 | # keep single pipe context here 31 | self.pipe = pipe or pype(extra_env=env) 32 | with open("blocks.dat", "rb") as f: 33 | blocks, srcsw = msgpack.unpackb(f.read(), object_hook=decode) 34 | self.blocks = blocks 35 | assert self.pysh.srcf == __file__ or self.pysh.srcf == Path( 36 | __file__), f"Injector not run from where it was injected {self.pysh.srcf} X{__file__}" 37 | self.pysh.blocks = self.blocks 38 | self.pysh.srcsw = srcsw 39 | 40 | # Reserialize references 41 | for i, block in enumerate(self.blocks): 42 | assert self.blocks[i].serialized == True, "Block was saved without serialized flag" 43 | self.blocks[i].pysh = self.pysh 44 | blocks[i].pipe = self.pipe 45 | # Optionally reset 46 | 47 | def runblock(self, blockid, argvarvals): 48 | try: 49 | block = self.getblock(blockid) 50 | if not block: 51 | raise IndexError("Couldn't find block ID in block injector") 52 | block.run(pipe=self.pipe, argvarvals=argvarvals) 53 | except ScriptException as e: 54 | if block.raise_errors: 55 | from traceback import format_exc 56 | from tempfile import NamedTemporaryFile 57 | with NamedTemporaryFile(mode="w", delete=False) as f: 58 | trace_string = format_exc() 59 | message = trace_string.replace("", f.name) 60 | f.write(self.pysh.srcsw) 61 | raise SystemExit(message) from e 62 | else: 63 | print("!!!ScriptException", e) 64 | if "@" in block.flags: 65 | print(block.stdout.decode("UTF-8").strip()) 66 | return block.stdout 67 | 68 | def getblock(self, blockid): 69 | block = next((b for b in self.blocks if b.blockid == blockid), None) 70 | return block 71 | 72 | def getblockidx(self, blockid): 73 | block = self.getblock(blockid) 74 | index = self.blocks.index(block) 75 | return index 76 | block_injector = BlockInjector(pysher=Pysh(__file__)) 77 | """ 78 | 79 | class BlockInjector(): 80 | def __init__(self, pysher, pipe=None): 81 | self.pysh = pysher 82 | env = {"PYSH_RUNNING": "1"} 83 | # keep single pipe context here 84 | self.pipe = pipe or pype(extra_env=env) 85 | with open("blocks.dat", "rb") as f: 86 | blocks, srcsw = msgpack.unpackb(f.read(), object_hook=decode) 87 | self.blocks = blocks 88 | assert self.pysh.srcf == __file__ or self.pysh.srcf == Path( 89 | __file__), f"Injector not run from where it was injected {self.pysh.srcf} X{__file__}" 90 | self.pysh.blocks = self.blocks 91 | self.pysh.srcsw = srcsw 92 | 93 | # Reserialize references 94 | for i, block in enumerate(self.blocks): 95 | assert self.blocks[i].serialized == True, "Block was saved without serialized flag" 96 | self.blocks[i].pysh = self.pysh 97 | blocks[i].pipe = self.pipe 98 | # Optionally reset 99 | 100 | def runblock(self, blockid, argvarvals): 101 | try: 102 | block = self.getblock(blockid) 103 | if not block: 104 | raise IndexError("Couldn't find block ID in block injector") 105 | block.run(pipe=self.pipe, argvarvals=argvarvals) 106 | except ScriptException as e: 107 | if block.raise_errors: 108 | from traceback import format_exc 109 | from tempfile import NamedTemporaryFile 110 | with NamedTemporaryFile(mode="w", delete=False) as f: 111 | trace_string = format_exc() 112 | message = trace_string.replace("", f.name) 113 | f.write(self.pysh.srcsw) 114 | raise SystemExit(message) from e 115 | else: 116 | print("!!!ScriptException", e) 117 | if "@" in block.flags: 118 | print(block.stdout.decode("UTF-8").strip()) 119 | return block.stdout 120 | 121 | def getblock(self, blockid): 122 | block = next((b for b in self.blocks if b.blockid == blockid), None) 123 | return block 124 | 125 | def getblockidx(self, blockid): 126 | block = self.getblock(blockid) 127 | index = self.blocks.index(block) 128 | return index 129 | 130 | 131 | class BashBlock(ScriptRun): 132 | def __init__(self, srcs: str, 133 | blockindex: int, 134 | position: tuple, 135 | srcf: str | Path, 136 | shell="bash", 137 | pysh=None, 138 | pipe=default_pipe, 139 | serialized=False, 140 | blockid=None, 141 | env=None, 142 | matches=None, 143 | argvarmatches=None, 144 | flags=None, 145 | **scriptrun_args): 146 | self.blockid = blockid or str(uuid.uuid4()) 147 | self.pysh = pysh 148 | self.srcf = srcf 149 | self.blockindex = blockindex 150 | self.position = position 151 | self.serialized = serialized 152 | self.flags = flags 153 | env = env or {} 154 | env = env | {"PYSH_BLOCK": self.blockid} 155 | self.matches = matches 156 | self.argvarmatches = argvarmatches 157 | super().__init__(srcs=srcs, shell=shell, pipe=pipe, env=env, **scriptrun_args) 158 | 159 | @property 160 | def linecount(self): 161 | return len(self.srcs.split("\n")) 162 | 163 | def run(self, *args, argvarvals=[], **kwargs): 164 | srcsw = self.srcs 165 | # Interpolate the {$$} vars into the source 166 | if argvarvals or self.argvarmatches: 167 | assert len(argvarvals) == len( 168 | self.argvarmatches), "Provided runvars don't match block.argvars" 169 | for i, v in enumerate(self.argvarmatches): 170 | srcsw = srcsw.replace(v["total"], argvarvals[i]) 171 | # print("Running block", self.blockid, self.srcf) 172 | super().run(srcs=srcsw, *args, **kwargs) 173 | # print("stdout = ", self.stdout.decode("UTF-8")) 174 | 175 | def runs(self, *args, **kwargs): 176 | self.run(*args, **kwargs) 177 | return self.stdout 178 | 179 | def runp(self, *args, **kwargs): 180 | self.run(*args, **kwargs) 181 | sname = os.path.basename(self.pysh.srcf) 182 | stdout = self.stdout.decode("UTF-8").strip() 183 | print(f"[root@pysh {sname} {self.blockindex}]$ {stdout}") 184 | return self 185 | 186 | def serialize(self): 187 | return msgpack.packb(self, default=encode) 188 | 189 | def __repr__(self) -> str: 190 | classname = self.__class__.__name__ 191 | args = ", ".join( 192 | [f"{k}={repr(v)}" for (k, v) in self.__dict__.items()]) 193 | return f"{classname}({args})" 194 | 195 | 196 | class Pysh(): 197 | def __init__(self, srcf, init=True, test_mode=False) -> None: 198 | """ 199 | """ 200 | self.blocks = [] 201 | self.test_mode = test_mode 202 | self.srcsw = None 203 | self.pyshed_block = None 204 | if init is not False: 205 | self.updatesrc(srcf) 206 | 207 | def readsrc(self): 208 | srcs = None 209 | with open(self.srcf, "r") as f: 210 | srcs = f.read() 211 | return srcs 212 | 213 | def updatesrc(self, srcf): 214 | self.srcf = os.path.realpath(srcf) 215 | self.srcs = self.readsrc() 216 | self.srclines = self.srcs.split("\n") 217 | self.blocks = self.findblocks() 218 | 219 | def pysh(self, srcf=None, exits=True, test_mode=None): 220 | if test_mode: 221 | self.test_mode = test_mode 222 | if os.environ.get("PYSH_ROOT", None): 223 | if self.test_mode: 224 | print("#####PYSH_ROOT can't run Pysh.pysh() while pysh is active") 225 | return 226 | srcf = srcf or self.srcf 227 | self.updatesrc(srcf) 228 | ret = self.shyp(exits) 229 | return self 230 | 231 | def shyp(self, exits=True): 232 | if os.environ.get("PYSH_ROOT", None): 233 | if self.test_mode: 234 | print("#####PYSH_ROOT can't run Pysh.shyp() while pysh is active") 235 | return 236 | # Run this script with the wrapped pysh calls and then exit 237 | root_script, block_file = self.wrapped() 238 | env = {"PYSH_ROOT": "1"} 239 | pyshed = BashBlock(root_script, 240 | 0, 241 | (0, -1), 242 | self.srcf, 243 | shell="python", 244 | pysh=self, 245 | stdin_pipe=sys.stdin, 246 | stderr_pipe=sys.stderr, 247 | stdout_pipe=sys.stdout, 248 | raise_errors=False, 249 | env=env) 250 | self.pyshed_block = pyshed 251 | if self.test_mode: 252 | print("#####PYSH_SHIPPED") 253 | pyshed.run() 254 | # print("Pyshed stdout:", pyshed.stdout) # Should be none as piped to sys.stdout 255 | Path(block_file).unlink() 256 | if exits: 257 | sys.exit(pyshed.returncode) 258 | 259 | def wrapped(self, srcs=None, blocks=None): 260 | srcs = srcs or self.srcs 261 | blocks = blocks or self.blocks 262 | srcsw, block_file = self.wrap_imports(srcs, blocks) 263 | self.srcsw = srcsw 264 | return (srcsw, block_file) 265 | 266 | def wrap_imports(self, srcs=None, blocks=None): 267 | srcs = srcs or self.srcs 268 | blocks = blocks or self.blocks 269 | 270 | # TODO: remove anything before the shyp() call as it will be rerun ?? 271 | 272 | # Inject pysh header 273 | srclines = srcs.split("\n") 274 | file_injector = f"__file__ = '{self.srcf}'\n" 275 | header = f"{file_injector}{block_injector}\n" 276 | header_length = len(header) 277 | srcsw = header + '\n'.join(srclines) 278 | 279 | # TODO additional modes, assignment returns besides stdout 280 | next_start = None 281 | difference = 0 282 | for i, block in enumerate(blocks): 283 | _, spanend = block.position 284 | start, end = spanend 285 | slength = (end - start) 286 | block_start_m = next( 287 | (m for m in block.matches if "linecontents" in m and m["linecontents"])) 288 | if len(block_start_m["mode"]) > 1: 289 | extern_block = True 290 | length = sum( 291 | [len(m.get("line", "") or "") 292 | for m in block.matches 293 | if "linecontents" in m]) 294 | first_match = next((m for m in block.matches if not m["pyvar"])) 295 | sindex = srcsw.index(first_match["block"]) 296 | assert slength == length, f"mismatch {slength} {length}" 297 | args = ",".join([match['argname'] 298 | for match in block.argvarmatches]) 299 | runblock = f"block_injector.runblock('{block.blockid}', argvarvals=[{args}])" 300 | next_start = sindex # header_length + start_accum + start - difference 301 | eend = next_start+length-1 302 | # difference = (eend - next_start) 303 | # assert difference == length - 1, f"diff mismatch {difference} {length}" 304 | srcsw = srcsw[:next_start] + runblock + srcsw[eend:] 305 | # print("S2", sindex, next_start, difference, header_length, length, start_accum) 306 | 307 | # replace assignment quotes 308 | srcsw = srcsw.replace('""block_injector', "block_injector") 309 | 310 | # Save blocks to serial file 311 | block_file = "blocks.dat" 312 | with open(block_file, "wb") as f: 313 | f.write(msgpack.packb(([block 314 | for block in self.blocks], srcsw), default=encode)) 315 | 316 | # print("#----#") 317 | # print(srcsw) 318 | # print("#----#") 319 | 320 | return (srcsw, block_file) 321 | 322 | def findblocks(self, srcs=None): 323 | if os.environ.get("PYSH_ROOT", None): 324 | if self.test_mode: 325 | print("#####PYSH_ROOT can't run Pysh.findblocks() while pysh is active") 326 | return 327 | if srcs: 328 | self.blocks = [] 329 | srcs = srcs or self.srcs 330 | if self.blocks: 331 | return self.blocks 332 | 333 | # Extract the pysh syntax lines with regex 334 | pattern = r"(?P(?P.*)(?P\=\s\"{2})|(?P(?P(?P(?\$+)(?P(%{0,1})(@{0,1})(!{0,1}))(?P\w*)?)(?P[\s]+)(?P.*)(?P\n)))" 335 | matches = re.finditer(pattern, srcs) 336 | assert matches, "Root source file doesn't contain any Pysh" 337 | matches = list(matches) 338 | 339 | # Group the matches into blocks 340 | accum = -1 341 | mblock_i = 0 342 | match_blocks = [] 343 | for i, match in enumerate(matches): 344 | if not match["mode"]: 345 | assert match["pyvar"], "Pyvar line can't have mode" 346 | if not match["pyvar"]: 347 | assert match["mode"], "Mode line can't have pyvar" 348 | 349 | lstart, lend = match.span() 350 | if mblock_i not in match_blocks: 351 | match_blocks.append({"matches": []}) 352 | match_blocks[mblock_i]["matches"].append(match) 353 | next_match = matches[i+1] if not i > len(matches)-2 else None 354 | if next_match: 355 | nlstart, nlend = next_match.span() 356 | next_match_groups = next_match.groupdict() 357 | if (lend != nlstart # Doesn't immediately end 358 | # Next line is a pyvar, this shouldnt really happen 359 | or not next_match_groups["mode"] 360 | # Pyvar should always be in next group 361 | or (next_match_groups["pyvar"]) 362 | or (len(next_match_groups["mode"]) > 1 and not match["pyvar"])): # Next line is extern, and this line is not assignment 363 | mblock_i += 1 364 | continue 365 | accum += 1 366 | match_blocks = match_blocks[0:-accum] 367 | 368 | # Create BashBlock objects from the block-grouped matches 369 | def reduce_mblocks_t(mblock): 370 | return "".join([str((match.groupdict()["linecontents"])) + str((match.groupdict()["eol"])) 371 | for match in mblock["matches"] 372 | if match.groupdict()["linecontents"] is not None]) # [:-1] # Ignore pre-pysh assignment, trim last \n 373 | new_blocks = [] 374 | for i, mblock in enumerate(match_blocks): 375 | matches = mblock["matches"] 376 | sstart, send = matches[0].span() 377 | estart, eend = matches[-1].span() 378 | whole = (sstart, eend) 379 | matches_groups = sorted( 380 | [m.groupdict() | {"_span": m.span()} for m in matches], key=lambda x: x["_span"][0]) 381 | assignment = next( 382 | (mgroup for mgroup in matches_groups if mgroup["assign"]), None) 383 | block_start_m = next( 384 | (m for m in matches_groups if "linecontents" in m and m["linecontents"])) 385 | shell = next((mgroup for mgroup in matches_groups if mgroup["shell"] and not mgroup["assign"]), { 386 | "shell": None})["shell"] 387 | spanend = (assignment["_span"][1], eend) if assignment else whole 388 | position = (whole, spanend) 389 | block_srcs = reduce_mblocks_t(mblock) 390 | pattern2 = r"(?P({\$(?P[\w_]+)\$}))" 391 | argvarmatches = re.finditer(pattern2, block_srcs) 392 | argvarmatches = list(argvarmatches) 393 | argvarmatches = [ 394 | m.groupdict() | {"_span": m.span()} for m in argvarmatches] 395 | for m in matches_groups: 396 | assert m["block"] == m["line"] if m[ 397 | "line"] is not None else True, f"line doesnt match block \n{m['block']}\n{m['line']}" 398 | command_args = [] 399 | raw_command = False 400 | if len(block_start_m["mode"] or "") > 1: 401 | assert len(matches_groups) == 1 or (matches_groups[0]["pyvar"] and len(matches_groups) == 2), \ 402 | f"Extern blocklinegroup mixed with others \n {matches_groups}" 403 | command_args = shlex.split(block_start_m["linecontents"]) 404 | raw_command = True 405 | raise_errors = False 406 | if "%" in block_start_m["flags"]: 407 | raise_errors = True 408 | script_block = BashBlock(srcs=block_srcs, 409 | shell=shell, 410 | blockindex=i, 411 | position=position, 412 | srcf="#internal", 413 | pysh=self, 414 | matches=matches_groups, 415 | argvarmatches=argvarmatches, 416 | command_args=command_args, 417 | raise_errors=raise_errors, 418 | raw_command=raw_command, 419 | flags=block_start_m["flags"]) 420 | new_blocks.append(script_block) 421 | if self.test_mode: 422 | for block in new_blocks: 423 | print(repr_(block, incl=["position", "srcs", "raise_errors"])) 424 | for match in block.matches: 425 | print(repr_(match, incl=["_span", "block", "line"])) 426 | # sys.exit(0) 427 | 428 | # Assign to self if not running on an arbitrary script source 429 | if not srcs: 430 | self.blocks = new_blocks 431 | 432 | return new_blocks 433 | 434 | def __repr__(self) -> str: 435 | return repr_(self, ["blocks"]) 436 | 437 | 438 | if os.environ.get("PYSH_ROOT", None) != "1": 439 | # print("#####PYSH_ROOT no creating importable instance") 440 | 441 | # Function wrapper to run on call source 442 | def x(): 443 | return os.path.realpath(stack()[-1].filename) 444 | pysher = Pysh(x(), init=False) 445 | pysh = pysher.pysh 446 | else: 447 | pysher = None 448 | pysh = None 449 | -------------------------------------------------------------------------------- /src/pysh/pype.py: -------------------------------------------------------------------------------- 1 | # py process executor 2 | import os 3 | import subprocess 4 | from pathlib import Path 5 | from tempfile import NamedTemporaryFile 6 | from pysh.utils import repr_, timeit 7 | 8 | shells = ("bash", "python") 9 | 10 | 11 | class pype(): 12 | def __init__(self, pipe_kwargs=None, base_env=None, extra_env=None): 13 | self.pipe_kwargs = pipe_kwargs 14 | self.default_shell = shells[0] 15 | self.base_env = base_env or os.environ.copy() 16 | self.extra_env = extra_env or {} 17 | 18 | def run_script(self, script: str | Path, 19 | shell=None, 20 | pipe_kwargs: dict = None, 21 | stdout=None, 22 | stderr=None, 23 | stdin=None, 24 | env=None, 25 | raise_errors=True, 26 | timeout=None, 27 | raw_command=False, 28 | command_args=[]): 29 | """ Run string source script in bash or python by passing a command_string string with the -c flag or calling a script file 30 | 31 | :param script: as str to pass a command string to the shell, 32 | or as Path object to call a script file 33 | 34 | :returns: (stdout, stderr) 35 | :raises: error on non-zero process exit code 36 | """ 37 | env = env or {} 38 | env = self.base_env | env | self.extra_env 39 | shell = shell or self.default_shell 40 | 41 | if type(script) == type(Path()): 42 | command = [shell, str(Path)] 43 | else: 44 | command = [shell, "-c", script] 45 | if raw_command == True: 46 | command = [shell] 47 | default_args = ("script", "shell", "pipe_kwargs", 48 | "stdin", "raise_errors") 49 | command_args = [ 50 | a for a in command_args if a not in default_args] # set -ex 51 | pipe_kwargs = pipe_kwargs or self.pipe_kwargs or {} 52 | stdout = stdout or subprocess.PIPE 53 | stderr = stderr or subprocess.PIPE 54 | pipe_args = dict(stdout=stdout, 55 | stderr=stderr, 56 | stdin=stdin or subprocess.PIPE, 57 | bufsize=8192, 58 | shell=False, 59 | env=env, 60 | close_fds=True) | pipe_kwargs 61 | #cwd, text 62 | proc = subprocess.Popen(command + command_args, **pipe_args) 63 | try: 64 | stdout, stderr = proc.communicate(input=stdin, timeout=timeout) 65 | except subprocess.TimeoutExpired: 66 | proc.kill() 67 | stdout, stderr = proc.communicate() 68 | if raise_errors is None: 69 | raise_errors = True 70 | # print("ProcDebug:", raise_errors, proc.returncode) 71 | if raise_errors and proc.returncode: 72 | exc_kwargs = dict(stdout=stdout, stderr=stderr, 73 | shell=shell, srcs=script) 74 | raise ScriptException(proc, **exc_kwargs) 75 | return stdout, stderr, proc.returncode, proc.pid 76 | 77 | 78 | default_pipe = pype() 79 | 80 | 81 | class ScriptRun(): 82 | def __init__(self, srcs: str, 83 | shell: str = shells[0], 84 | returncode: int | None = None, 85 | stdout: str | None = None, 86 | stderr: str | None = None, 87 | hasrun=False, 88 | pipe=None, 89 | env=None, 90 | timeout=None, 91 | raise_errors=None, 92 | command_args=[], 93 | raw_command=False, 94 | **kwargs): 95 | self.pipe = pipe or default_pipe 96 | self.shell = shell 97 | self.srcs = srcs 98 | self.returncode = returncode 99 | self.stdout = stdout 100 | self.stderr = stderr 101 | self.hasrun = hasrun 102 | self.exectime = None # TODO 103 | self.env = env 104 | self.timeout = timeout 105 | self.raise_errors = raise_errors 106 | self.command_args = command_args 107 | self.raw_command = raw_command 108 | self.stdout_pipe = kwargs.get("stdout_pipe", None) 109 | self.stderr_pipe = kwargs.get("stderr_pipe", None) 110 | self.stdin_pipe = kwargs.get("stdin_pipe", None) 111 | 112 | def run(self, srcs=None, pipe=None, env=None): 113 | pipe = pipe or self.pipe 114 | srcs = srcs or self.srcs 115 | env = env or self.env 116 | stdout, stderr, returncode, pid = \ 117 | pipe.run_script(srcs, 118 | shell=self.shell, 119 | stdin=self.stdin_pipe, 120 | stdout=self.stdout_pipe, 121 | stderr=self.stderr_pipe, 122 | raise_errors=self.raise_errors, 123 | command_args=self.command_args, 124 | raw_command=self.raw_command, 125 | env=env, 126 | timeout=self.timeout) 127 | self.returncode = returncode 128 | self.stdout = stdout 129 | self.stderr = stderr 130 | self.pid = pid 131 | self.hasrun = True 132 | return returncode 133 | 134 | 135 | class ScriptException(ScriptRun, Exception): 136 | def __init__(self, proc, **ScriptRunI): 137 | self.proc = proc 138 | super().__init__(**ScriptRunI) 139 | # For debugging 140 | with NamedTemporaryFile(mode="w", delete=False) as f: 141 | f.write(ScriptRunI["srcs"]) 142 | trace = ScriptRunI["stderr"].decode("UTF-8") 143 | trace = trace.replace("", f.name) 144 | message = f"Error running {ScriptRunI['shell']} script {f.name} Trace:\n\t{trace}".strip() 145 | self.message = message 146 | super(Exception, self).__init__(message) 147 | 148 | def __repr__(self): 149 | return repr_(self) 150 | -------------------------------------------------------------------------------- /src/pysh/serializer.py: -------------------------------------------------------------------------------- 1 | import lzma 2 | from streamlit.runtime.uploaded_file_manager import UploadedFile, UploadedFileRec 3 | from typing import Any, Optional 4 | import pysh 5 | 6 | 7 | def decode(obj: Any, force_type: Optional[str] = None) -> Any: 8 | try: 9 | obj = lzma.decompress(obj) 10 | except: 11 | pass 12 | 13 | if "__serialized_type__" in obj: 14 | type = obj["__serialized_type__"] 15 | 16 | if type == "BashBlock" or force_type == "BashBlock": 17 | obj = pysh.BashBlock(**obj["class_dict"]) 18 | return obj 19 | 20 | 21 | def type_check(obj, class_ref, class_str, force_type=None): 22 | """ 23 | isinstance and is class are failing across threads when they shouldn't. 24 | Possibly to do with PYTHONHASHSEED or import hierarchy. 25 | 26 | obj.__class__.__name__ seems to work best 27 | """ 28 | # print(obj.__dict__) 29 | # print(vars(obj)) 30 | # print(obj.__class__) 31 | # print(obj.__class__.__name__) 32 | # print( obj.__name__,) 33 | 34 | return isinstance(obj, class_ref) \ 35 | or type(obj) is type(class_ref) \ 36 | or type(obj) is class_ref \ 37 | or (hasattr(obj, "__class__") and obj.__class__ is class_ref) \ 38 | or (hasattr(obj, "__class__") and hasattr(obj.__class__, "__name__") and obj.__class__.__name__ == class_str) \ 39 | or force_type == class_str 40 | 41 | 42 | def encode(obj: Any, force_type: Optional[str] = None) -> Any: 43 | serialized = None 44 | 45 | if type_check(obj, pysh.BashBlock, "BashBlock", force_type): 46 | attr_dict = obj.__dict__.copy() 47 | del attr_dict["pysh"] 48 | del attr_dict["pipe"] 49 | attr_dict["serialized"] = True 50 | serialized = { 51 | "__serialized_type__": "BashBlock", 52 | "class_dict": attr_dict 53 | } 54 | return serialized 55 | 56 | return serialized if serialized else obj 57 | -------------------------------------------------------------------------------- /src/pysh/utils.py: -------------------------------------------------------------------------------- 1 | def repr_(cls, incl=True) -> str: 2 | classname = cls.__class__.__name__ 3 | dct = cls.__dict__ if hasattr(cls, "__dict__") else cls 4 | args = ", ".join( 5 | [f"{k}={repr(v)}" for (k, v) in dct.items() 6 | if incl is True or (k in incl)]) 7 | return f"{classname}({args})" 8 | 9 | from functools import wraps 10 | from time import time 11 | 12 | def timeit(f): 13 | @wraps(f) 14 | def wrap(*args, **kw): 15 | ts = time() 16 | result = f(*args, **kw) 17 | te = time() 18 | print("func:%r took: %2.4f sec \n args:[%r, %r]" % \ 19 | (f.__name__, args[0:1], kw, te-ts)) 20 | return result 21 | return wrap -------------------------------------------------------------------------------- /tests/demo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import sys 3 | 4 | from pysh import pysher, pysh 5 | 6 | # run pysh whenever this file is interpreted 7 | pysh(__file__) 8 | test = ""#$ echo "Pysh activated" 9 | 10 | # Example usage 11 | #$ ls . # comment 12 | ##$ xdg-open . # double comment to disable in pysh 13 | sys.exit(0) 14 | #$$ my_script.sh arg1 arg2 # run a bash script and pass arguments to it with double $ 15 | 16 | # Script your system in paralel with your python code execution 17 | ##$ firefox https://news.ycombinator.com && echo "opened link in firefox" 18 | ##$ cd ~/hosted/myservice && docker compose up 19 | ##$ ffmpeg -i -crf 20 | 21 | 22 | # Save an inline or external scripts stdout to a variable in your python - computed at python runtime 23 | stdout = ""#$ echo "non-stdout" 24 | print(stdout) 25 | #>non-stdout 26 | 27 | stdout = ""#$ echo "multiline" 28 | #$ echo "l1" 29 | #$ echo "l2" 30 | #$ echo "l3" 31 | print(stdout) 32 | #> multiline 33 | #>l1 34 | #>l2 35 | #>l3 36 | 37 | ####################### 38 | 39 | # run pysh manually 40 | from pysh import Pysh 41 | source_file = __file__ 42 | p = Pysh(source_file) 43 | blocks = p.findblocks() 44 | 45 | # Run all blocks and print their stdout 46 | run_blocks = [block.runp() for block in blocks] 47 | # Run a a single block and silently 48 | blocks[0].run() 49 | 50 | # Start the python interpreter with pysh on source_file 51 | p.shyp() -------------------------------------------------------------------------------- /tests/features.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import os 3 | print("####BEGIN TEST", os.environ.get("PYSH_ROOT", None), os.environ.get("PYSH_RUNNING", None), os.environ.get("PYSH_BLOCK", None)) 4 | 5 | from pysh import Pysh, ScriptException 6 | source_file = __file__ 7 | pysher = Pysh(source_file, test_mode=False) 8 | pysher.shyp() 9 | #$@ echo "pysh activated" 10 | 11 | # Alternate shells 12 | text = "The time and date" 13 | import time 14 | ttime = time.asctime() 15 | py_stdout = ""#$@!python import time 16 | #$ print("{$text$}", "{$ttime$}") 17 | print("py_stdout = ", py_stdout) 18 | should_be = bytes((text + " " + ttime + "\n").encode("UTF-8")) 19 | assert py_stdout == should_be, f"py_stdout is wrong: {py_stdout}" 20 | 21 | # print(pysher.blocks[0]) 22 | 23 | # Current directory 24 | stdout = ""#$ echo "$PWD" 25 | assert stdout.decode("UTF-8").strip() == os.getcwd(), f"$PWD from script is wrong {stdout}" 26 | print("CWD:", stdout) 27 | 28 | # External scripts 29 | #$ chmod +x tests/myscript.sh 30 | extern_stdout = ""#$$ tests/myscript.sh "argone" "argtwo" 31 | assert extern_stdout == b"external script. args: argone argtwo\n", "external script output is wrong" 32 | print("Extern:", extern_stdout) 33 | 34 | # Missing script 35 | try: 36 | extern_stdout = ""#$$% tests/notfoundscript.sh "argone" "argtwo" 37 | print(extern_stdout) 38 | except SystemExit as e: 39 | print("Caught expection ScriptException") 40 | # print("Error:", e) 41 | 42 | 43 | print("Test Passed") 44 | -------------------------------------------------------------------------------- /tests/myscript.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "external script. args: $1 $2" 3 | -------------------------------------------------------------------------------- /tests/regtext.py: -------------------------------------------------------------------------------- 1 | from pysh import pysh 2 | pysh(__file__) 3 | 4 | #$@ echo "simple" 5 | 6 | #$@!sh echo "simple" 7 | #$%@!sh echo "simple" 8 | 9 | #$!@sh echo 10 | #$@%!sh echo 11 | #$@% echo 12 | 13 | #$!sh echo "simple" 14 | 15 | #$ echo "Pysh activated" 16 | stdout = ""#$ echo "This is standard out" 17 | 18 | 19 | 20 | # Python comments 21 | #$ ls . # pysh eol comment 22 | ##$ sudo rm -rf # disable pysh line 23 | 24 | #$ cd $HOME 25 | 26 | #$$ my_script.sh 27 | #$$ argumentative_script.sh arg1 arg2 28 | 29 | #$@sh echo "simple" 30 | 31 | #$!sh echo "simple" 32 | #$!perl oysters.pl 33 | #$$!bash script.sh 34 | stdout = ""#$!python import time; print("The time and date", time.asctime()) 35 | 36 | if __name__ == "main": 37 | print("Before the above script blocks are run") 38 | 39 | 40 | stdout_block1 = ""#$ echo "first block is multiline" 41 | #$ echo "line1" 42 | #$ echo "line2" 43 | 44 | # The last script block won't be run 45 | sys.exit(1) 46 | stdout_block2 == ""#$ echo "Second" 47 | #$ echo "Block" 48 | 49 | 50 | 51 | build_service() 52 | #$ cd ~/hosted/myservice && docker compose up 53 | 54 | aggregate_assets() 55 | fmpg_result = ""#$ ffmpeg -i raw_video.mkv -crf {{crf}} -o 56 | process_assets(process_fmpg_stdout(fmpg_result)) 57 | 58 | pysher.shyp() 59 | #$ echo "pysh enabled" 60 | 61 | -------------------------------------------------------------------------------- /tests/stdout.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | import os 3 | print("####BEGIN TEST", os.environ.get("PYSH_ROOT", None), os.environ.get("PYSH_RUNNING", None), os.environ.get("PYSH_BLOCK", None)) 4 | from pysh import Pysh 5 | source_file = __file__ 6 | pysher = Pysh(source_file) 7 | pysher.shyp() 8 | 9 | stdout = ""#$@ echo "pysh activated" 10 | assert stdout == b"pysh activated\n", f"stdout wrong: {stdout}" 11 | 12 | print("WAIT for user dialog") 13 | #$ /usr/bin/zenity --info # pause here 14 | 15 | print("STDOUT BLOCKED IN BETWEEN BLOCKS") 16 | 17 | text = "pysh test" 18 | multiline_stdout = ""#$ echo "{$text$}" 19 | #$ echo "pysh test line 2" 20 | assert multiline_stdout == b"pysh test\npysh test line 2\n", f"multiline_stdout wrong: {multiline_stdout}" 21 | 22 | more_lines_stdout = ""#$ echo "one" 23 | #$ echo "two" 24 | #$ echo "three" 25 | 26 | assert more_lines_stdout == b"""one 27 | two 28 | three 29 | """, f"more lines failure {more_lines_stdout}" 30 | 31 | print("Test Passed") -------------------------------------------------------------------------------- /todo.md: -------------------------------------------------------------------------------- 1 | # TODO 2 | - flag to capture more than just stdout 3 | - run in remote shell --------------------------------------------------------------------------------