├── README.md └── pyp /README.md: -------------------------------------------------------------------------------- 1 | pyp - The Pyed Piper 2 | ==================== 3 | 4 | This is the __unofficial__ git mirror for the pyp project. The original project page can be found at http://code.google.com/p/pyp/. 5 | 6 | Installation 7 | ------------ 8 | 9 | - Pyp is now available from PyPI (http://pypi.python.org/pypi/pyp) and can be installed via `pip install pyp` or `easy_install install pyp`. 10 | - Otherwise, the easiest way would be to pull the raw pyp script (https://raw.github.com/yuvadm/pyp/master/pyp) and link it to your favorite `bin` location. 11 | -------------------------------------------------------------------------------- /pyp: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #version 2.12 3 | #author tobyrosen@gmail.com 4 | """ 5 | Copyright (c) 2011, Sony Pictures Imageworks 6 | All rights reserved. 7 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 8 | Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 9 | Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation 10 | and/or other materials provided with the distribution. 11 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 12 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS 13 | BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 14 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 15 | IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 16 | OF THE POSSIBILITY OF SUCH DAMAGE. 17 | """ 18 | 19 | 20 | 21 | import optparse 22 | import sys 23 | import os 24 | import time 25 | import json 26 | import glob 27 | import tempfile 28 | import datetime 29 | import getpass 30 | import re 31 | 32 | 33 | #try to import user customized classes if they exist. default is null class. 34 | try: 35 | from PypCustom import PypCustom 36 | except ImportError: 37 | class PypCustom(): 38 | pass 39 | 40 | try: 41 | from PypCustom import PowerPipeListCustom 42 | except ImportError : 43 | class PowerPipeListCustom(): 44 | pass 45 | 46 | try: 47 | from PypCustom import PypStrCustom 48 | except ImportError : 49 | class PypStrCustom(): 50 | pass 51 | 52 | try : 53 | from PypCustom import PypListCustom 54 | except ImportError: 55 | class PypListCustom(): 56 | pass 57 | 58 | try: 59 | from PypCustom import PypFunctionCustom 60 | except ImportError: 61 | class PypFunctionCustom(): 62 | pass 63 | 64 | 65 | class Colors(object): 66 | '''defines basic color scheme''' 67 | OFF = chr(27) + '[0m' 68 | RED = chr(27) + '[31m' 69 | GREEN = chr(27) + '[32m' 70 | YELLOW = chr(27) + '[33m' 71 | MAGENTA = chr(27) + '[35m' 72 | CYAN = chr(27) + '[36m' 73 | WHITE = chr(27) + '[37m' 74 | BLUE = chr(27) + '[34m' 75 | BOLD = chr(27) + '[1m' 76 | COLORS = [OFF, RED, GREEN, YELLOW, MAGENTA, CYAN, WHITE, BLUE, BOLD] 77 | 78 | class NoColors(object): 79 | '''defines basic null color scheme''' 80 | OFF = '' 81 | RED = '' 82 | GREEN ='' 83 | YELLOW = '' 84 | MAGENTA = '' 85 | CYAN = '' 86 | WHITE ='' 87 | BLUE = '' 88 | BOLD = '' 89 | COLORS = [OFF, RED, GREEN, YELLOW, MAGENTA, CYAN, WHITE, BLUE, BOLD] 90 | 91 | class PowerPipeList(list,PowerPipeListCustom): 92 | ''' 93 | defines pp object, allows manipulation of entire input using python list methods 94 | ''' 95 | def __init__(self, *args): 96 | super(PowerPipeList, self).__init__(*args) 97 | try: 98 | PowerPipeListCustom.__init__(self) 99 | except AttributeError: 100 | pass 101 | self.pyp = Pyp() 102 | 103 | 104 | def divide(self, n_split): 105 | ''' 106 | splits list into subarrays with n_split members 107 | @param n_split: number of members produced by split 108 | @type n_split: int 109 | @return : new array split up by n_split 110 | @rtype : list 111 | ''' 112 | sub_out = [] 113 | out = [] 114 | n = 0 115 | pyp = Pyp() 116 | inputs = self.pyp.flatten_list(self) 117 | 118 | while inputs: 119 | input = inputs.pop(0) 120 | n = n + 1 121 | sub_out.append(input) 122 | if not n % n_split or not inputs: 123 | out.append([sub_out]) 124 | sub_out = [] 125 | 126 | return out 127 | 128 | def delimit(self, delimiter): 129 | ''' 130 | splits up array based on delimited instead of newlines 131 | @param delimiter: delimiter used for split 132 | @type delimiter: str 133 | @return: new string split by delimiter and joined by ' ' 134 | @rtype: list 135 | ''' 136 | return ' '.join(self.pyp.flatten_list(self)).split(delimiter) 137 | 138 | def oneline(self,delimiter = ' '): 139 | ''' 140 | combines list to one line with optional delimeter 141 | @param delimiter: delimiter used for joining to one line 142 | @type delimiter: str 143 | @return: one line output joined by delimiter 144 | @rtype: list 145 | ''' 146 | 147 | flat_list = self.flatten_list(self) 148 | return delimiter.join(flat_list) 149 | 150 | def uniq(self): 151 | ''' 152 | returns only unique elements from list 153 | @return: unique items 154 | @rtype: list 155 | ''' 156 | strings= self.pyp.flatten_list(self) 157 | 158 | return list(set(strings)) 159 | 160 | def flatten_list(self, iterables): 161 | ''' 162 | returns a list of strings from nested lists 163 | @param iterables: nested lists containing strs or PypStrs 164 | @type iterables: list 165 | @return: unnested list of strings 166 | @rtype: list 167 | ''' 168 | return self.pyp.flatten_list(iterables) 169 | 170 | def unlist(self): 171 | ''' 172 | splits a list into one element per line 173 | @param self: nested list 174 | @type self: list 175 | @return: unnested list 176 | @rtype: list 177 | ''' 178 | return self.pyp.flatten_list(self) 179 | 180 | def after(self, target, after_n=1): 181 | ''' 182 | consolidates after_n lines after matching target text to 1 line 183 | @param target: target string to find 184 | @type target: str 185 | @param after_n: number of lines to consolidate 186 | @type after_n: int 187 | @return: list of after_n members 188 | @rtype: list 189 | ''' 190 | out = [] 191 | n = 0 192 | inputs = self.pyp.flatten_list(self) 193 | 194 | for input in inputs: 195 | n = n + 1 196 | if target in input: 197 | out.append([ [input] + inputs[n:n + after_n] ]) 198 | return out 199 | 200 | def before(self, target, before_n=1): 201 | ''' 202 | consolidates before_n lines before matching target text to 1 line 203 | @param target: target string to find 204 | @type target: str 205 | @param before_n: number of lines to consolidate 206 | @type before_n: int 207 | @return: list of before_n members 208 | @rtype: list 209 | ''' 210 | out = [] 211 | n = 0 212 | inputs = self.pyp.flatten_list(self) 213 | 214 | for input in inputs: 215 | n = n + 1 216 | if target in input: 217 | out.append([ [input] + inputs[n - before_n - 1:n - 1] ]) 218 | 219 | return out 220 | 221 | def matrix(self, target, matrix_n=1): 222 | ''' 223 | consolidates matrix_n lines surrounding matching target text to 1 line 224 | @param target: target string to find 225 | @type target: str 226 | @param matrix_n: number of lines to consolidate 227 | @type matrix_n: int 228 | @return: list of matrix_n members 229 | @rtype: list 230 | ''' 231 | out = [] 232 | n = 0 233 | inputs = self.pyp.flatten_list(self) 234 | 235 | for input in inputs: 236 | n = n + 1 237 | if target in input: 238 | out.append([ inputs[n - matrix_n - 1:n - 1] + [input] + inputs[n:n + matrix_n] ]) 239 | 240 | return out 241 | 242 | 243 | class PypStr(str,PypStrCustom): 244 | ''' 245 | defines p string object, allows manipulation of input line by line using python 246 | string methods 247 | 248 | ''' 249 | def __init__(self, *args): 250 | super(PypStr, self).__init__() 251 | try: 252 | PypStrCustom.__init__(self) 253 | except AttributeError: 254 | pass 255 | 256 | 257 | try: 258 | self.dir = os.path.split(self.rstrip('/'))[0] 259 | self.file = os.path.split(self)[1] 260 | self.ext = self.split('.')[-1] 261 | except: 262 | pass 263 | 264 | def trim(self,delim='/'): 265 | ''' 266 | returns everything but the last directory/file 267 | @param self: directory path 268 | @type self: str 269 | @return: directory path missing without last directory/file 270 | @rtype: PypStr 271 | ''' 272 | return PypStr(delim.join(self.split(delim)[0:-1])) 273 | 274 | 275 | def kill(self, *args): 276 | ''' 277 | replaces to_kill with '' in string 278 | @param args: strings to remove 279 | @type args : strs 280 | @return: string without to_kill 281 | @rtype: PypStr 282 | ''' 283 | for arg in args: 284 | self = self.replace(arg, '') 285 | 286 | return PypStr(self) 287 | 288 | def letters(self): 289 | ''' 290 | returns only letters 291 | @return: list of strings with only letters 292 | @rtype: PypList 293 | ''' 294 | 295 | new_string='' 296 | for letter in list(self): 297 | if letter.isalpha(): 298 | new_string = new_string + letter 299 | else: 300 | new_string = new_string + ' ' 301 | return [PypStr(x) for x in new_string.split() if x] 302 | 303 | def punctuation(self): 304 | ''' 305 | returns only punctuation 306 | @return: list of strings with only punctuation 307 | @rtype: PypList 308 | ''' 309 | 310 | new_string='' 311 | for letter in list(self): 312 | if letter in """!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~""": 313 | new_string = new_string + letter 314 | else: 315 | new_string = new_string + ' ' 316 | return [PypStr(x) for x in new_string.split() if x] 317 | 318 | def digits(self): 319 | ''' 320 | returns only digits 321 | @return: list of string with only digits 322 | @rtype: PypList 323 | ''' 324 | new_string='' 325 | for letter in list(self): 326 | if letter.isdigit(): 327 | new_string = new_string + letter 328 | else: 329 | new_string = new_string + ' ' 330 | return [PypStr(x) for x in new_string.split() if x] 331 | 332 | 333 | 334 | def clean(self,delim = '_'): 335 | ''' 336 | returns a metacharater sanitized version of input. ignores underscores and slashes and dots. 337 | @return: string with delim (default '_') replacing bad metacharacters 338 | @rtype: PypStr 339 | @param delim: delimeter to rejoin cleaned string with. default is "_" 340 | @type delime: str 341 | ''' 342 | 343 | for char in self: 344 | if not char.isalnum() and char not in ['/','.',delim]: 345 | self = self.replace(char, ' ') 346 | return PypStr(delim.join([x for x in self.split() if x.strip()])) 347 | 348 | def re(self,to_match): 349 | ''' 350 | returns characters that match a regex using to_match 351 | @return: portion of string that matches regex 352 | @rtype: PypStr 353 | @param to_match: regex used for matching 354 | @type to_match: str 355 | ''' 356 | 357 | match = re.search(to_match,self) 358 | if match: 359 | return PypStr(match.group(0)) 360 | else: 361 | return '' 362 | 363 | 364 | class PypList(list,PypListCustom): 365 | ''' 366 | defines p list object, allows manipulation of input line by line using python 367 | list methods 368 | ''' 369 | 370 | def __init__(self, *args): 371 | super(PypList, self).__init__(*args) 372 | try: 373 | PypListCustom.__init__(self) 374 | except AttributeError: 375 | pass 376 | class Pyp(object): 377 | ''' 378 | pyp engine. manipulates input stream using python methods 379 | @ivar history: master record of all manipulations 380 | @type history: dict 381 | @ivar pwd: current directory 382 | @type pwd: str 383 | @ivar p: current input line being manipulated 384 | @type p: str or list 385 | @ivar n: current input line number 386 | @type n: int 387 | ''' 388 | 389 | def __init__(self): 390 | self.history = {} #dictionary of all data organized input line by input line 391 | try: #occasionally, python loses pwd info 392 | self.pwd = os.getcwd() 393 | except: 394 | self.pwd ='' 395 | 396 | def get_custom_execute(self): 397 | '''returns customized paths to macro files if they are setup''' 398 | custom_ob = PypCustom() 399 | custom_attrs = dir(custom_ob) 400 | 401 | if 'custom_execute' in custom_attrs and custom_ob.custom_execute: 402 | final_execute = custom_ob.custom_execute 403 | else: 404 | final_execute = self.default_final_execute 405 | 406 | return final_execute 407 | 408 | def default_final_execute(self,cmds): 409 | for cmd in cmds: 410 | os.system(cmd) 411 | 412 | 413 | def get_custom_macro_paths(self): 414 | '''returns customized paths to macro files if they are setup''' 415 | home = os.path.expanduser('~') 416 | custom_ob = PypCustom() 417 | custom_attrs = dir(custom_ob) 418 | 419 | if 'user_macro_path' in custom_attrs: 420 | user_macro_path = custom_ob.user_macro_path 421 | else: 422 | user_macro_path = home + '/pyp_user_macros.json' 423 | 424 | 425 | if 'group_macro_path' in custom_attrs: 426 | group_macro_path = custom_ob.group_macro_path 427 | else: 428 | group_macro_path = home + '/pyp_group_macros.json' 429 | 430 | return user_macro_path,group_macro_path 431 | 432 | def cmds_split(self, cmds, macros): 433 | ''' 434 | splits total commmand array based on pipes taking into account quotes, 435 | parantheses and escapes. returns array of commands that will be processed procedurally. 436 | Substitutes macros without executable commands. 437 | 438 | @param cmds: user supplied command set 439 | @type cmds: list 440 | @param macros: user defined marcros 441 | @type macros: dict 442 | @return: list of commands to be evaluated 443 | @rtype: list 444 | ''' 445 | 446 | cmd_array = [] 447 | cmd = '' 448 | open_single = False 449 | open_double = False 450 | open_parenth = 0 451 | escape = False 452 | letters = list(cmds) 453 | while letters: 454 | letter = letters.pop(0) 455 | if cmd and cmd[-1] == '\\': escape = True 456 | 457 | #COUNTS QUOTES 458 | if letter == "'": 459 | if open_single and not escape: 460 | open_single = not open_single 461 | else: 462 | open_single = True 463 | if letter == '"': 464 | if open_double and not escape: 465 | open_double = not open_double 466 | else: 467 | open_double = True 468 | 469 | #COUNTS REAL PARANTHESES 470 | if not open_single and not open_double: 471 | if letter == '(' : 472 | open_parenth = open_parenth + 1 473 | if letter == ')': 474 | open_parenth = open_parenth - 1 475 | 476 | #MONEY MOVE--substitutes command for macro or starts building new command after adding command to cmd_array 477 | if cmd.strip() in macros and letter in ['|', '[', '%', ',', '+', ' ']: 478 | cmd = cmd.strip() 479 | letters = list('|'.join(macros[cmd]['command']) + letter + ''.join(letters)) 480 | cmd = '' 481 | elif letter == '|' and not open_single and not open_double and not open_parenth:# 482 | cmd_array.append(cmd) 483 | cmd = '' 484 | else: 485 | cmd = cmd + letter 486 | escape = False 487 | 488 | #for last command, either recursively run cmd_split or add last command to array 489 | if cmd.strip() in macros and not options.macro_save_name: #allows macro be split and also to be correctly overwritten 490 | return self.cmds_split('|'.join(cmd_array + macros[cmd]['command']), macros) #this is by definition the last cmd. 491 | else: 492 | cmd_array.append(cmd) #gets last cmd 493 | 494 | 495 | return [x for x in cmd_array if x] 496 | 497 | def load_macros(self,macro_path): 498 | ''' 499 | loads macro file; returns macros dict 500 | @param macro_path: file path to macro file 501 | @type macro_path: str 502 | @return: dictionary of user defined macros 503 | @rtype: dict 504 | ''' 505 | #macro_path = self.macro_path 506 | if os.path.exists(macro_path): 507 | macro_ob = open(macro_path) 508 | macros = json.load(macro_ob) 509 | macro_ob.close() 510 | else: 511 | macros = {} 512 | return macros 513 | 514 | def write_macros(self, macros,macro_path, cmds): 515 | ''' 516 | writes macro file 517 | @param macros: dictionary of user defined macros 518 | @type macros: dict 519 | @param macro_path: file path to macro file 520 | @type macro_path: str 521 | @param cmds: commands to be saved as a macro 522 | @type cmds: list 523 | ''' 524 | 525 | if options.macro_save_name: 526 | macro = options.macro_save_name 527 | macro_name = macro.split('#')[0].strip() 528 | macros[macro_name] = {} 529 | macros[macro_name]['command'] = cmds 530 | macros[macro_name]['user'] = getpass.getuser() 531 | macros[macro_name]['date'] =str(datetime.datetime.now()).split('.')[0] 532 | 533 | if '#' in macro: #deals with comments 534 | macros[macro_name]['comments'] = '#' + macro.split('#')[1].strip() 535 | else: 536 | macros[macro_name]['comments'] = '' 537 | macro_ob = open(macro_path, 'w') 538 | json.dump(macros, macro_ob) 539 | macro_ob.close() 540 | self.load_macros(macro_path) 541 | if macro_name in macros: 542 | print Colors.YELLOW + macro_name , "successfully saved!" + Colors.OFF 543 | sys.exit() 544 | else: 545 | print Colors.RED + macro_name, 'was not saved...unknown error!' + Colors.OFF 546 | sys.exit(1) 547 | 548 | def delete_macros(self, macros,macro_path): 549 | ''' 550 | deletes macro from file 551 | @param macros: dictionary of user defined macros 552 | @type macros: dict 553 | @param macro_path: file path to macro file 554 | @type macro_path: str 555 | ''' 556 | if options.macro_delete_name: 557 | if options.macro_delete_name in macros: 558 | del macros[options.macro_delete_name] 559 | json_ob = open(macro_path, 'w') 560 | json.dump(macros, json_ob) 561 | json_ob.close() 562 | print Colors.MAGENTA + options.macro_delete_name + " macro has been successfully obliterated" + Colors.OFF 563 | sys.exit() 564 | else: 565 | print Colors.RED + options.macro_delete_name + " does not exist" + Colors.OFF 566 | sys.exit(1) 567 | 568 | def list_macros(self, macros): 569 | ''' 570 | prints out formated macros, takes dictionary macros as input 571 | @param macros: dictionary of user defined macros 572 | @type macros: dict 573 | ''' 574 | if options.macro_list or options.macro_find_name: 575 | macros_sorted = [x for x in macros] 576 | macros_sorted.sort() 577 | for macro_name in macros_sorted: 578 | if options.macro_list or options.macro_find_name in macro_name or options.macro_find_name in macros[macro_name]['user']: 579 | print Colors.MAGENTA + macro_name + '\n\t ' + Colors.YELLOW+macros[macro_name]['user'] \ 580 | + '\t' + macros[macro_name]['date']\ 581 | +'\n\t\t' + Colors.OFF + '"'\ 582 | + '|'.join(macros[macro_name]['command']) + '"' + Colors.GREEN + '\n\t\t'\ 583 | + macros[macro_name].get('comments', '') + Colors.OFF + '\n' 584 | sys.exit() 585 | 586 | def load_file(self): 587 | ''' 588 | loads file for pyp processing 589 | @return: file data 590 | @rtype: list 591 | ''' 592 | if options.text_file: 593 | if not os.path.exists(options.text_file): 594 | print Colors.RED + options.text_file + " does not exist" + Colors.OFF 595 | sys.exit() 596 | else: 597 | f = [x.rstrip() for x in open(options.text_file) ] 598 | return f 599 | else: 600 | return [] 601 | 602 | 603 | def shell(self, command): 604 | ''' 605 | executes a shell commands, returns output in array sh 606 | @param command: shell command to be evaluated 607 | @type command: str 608 | @return: output of shell command 609 | @rtype: list 610 | ''' 611 | sh = [x.strip() for x in os.popen(command).readlines()] 612 | return sh 613 | 614 | def shelld(self, command, *args): 615 | ''' 616 | executes a shell commands, returns output in dictionary based on args 617 | @param command: shell command to be evaluated 618 | @type command: str 619 | @param args: optional delimiter. default is ":". 620 | @type args: list 621 | @return: hashed output of shell command based on delimiter 622 | @rtype: dict 623 | 624 | ''' 625 | if not args: 626 | ofs = ':' 627 | else: 628 | ofs = args[0] 629 | shd = {} 630 | for line in [x.strip() for x in os.popen(command).readlines()]: 631 | try: 632 | key = line.split(ofs)[0] 633 | value = ofs.join(line.split(ofs)[1:]) 634 | shd[key] = value 635 | except IndexError: 636 | pass 637 | 638 | return shd 639 | 640 | 641 | def rekeep(self,to_match): 642 | ''' 643 | keeps lines based on regex string matches 644 | @param to_match: regex 645 | @type to_match: str 646 | @return: True if any of the strings match regex else False 647 | @rtype: bool 648 | ''' 649 | 650 | match = [] 651 | flat_p = self.flatten_list(self.p) 652 | for item in flat_p: 653 | if re.search(to_match,item): 654 | match.append(item) 655 | if match: 656 | return True 657 | else: 658 | return False 659 | 660 | def relose(self,to_match): 661 | ''' 662 | loses lines based on regex string matches 663 | @param to_match: regex 664 | @type to_match: str 665 | @return: False if any of the strings match regex else True 666 | @rtype: bool 667 | ''' 668 | 669 | return not self.rekeep(to_match) 670 | 671 | 672 | def keep(self,*args): 673 | ''' 674 | keeps lines based on string matches 675 | @param args: strings to search for 676 | @type args: list 677 | @return: True if any of the strings are found else False 678 | @rtype: bool 679 | ''' 680 | 681 | kept = [] 682 | for arg in args: 683 | flat_p = self.flatten_list(self.p) 684 | for item in flat_p: 685 | if arg in item: 686 | kept.append(arg) 687 | 688 | if kept: 689 | return True 690 | else: 691 | return False 692 | 693 | def lose(self,*args): 694 | ''' 695 | removes lines based on string matches 696 | @param args: strings to search for 697 | @type args: list 698 | @return: True if any of the strings are not found else False 699 | @rtype: bool 700 | ''' 701 | return not self.keep(*args) 702 | 703 | def array_tracer(self, input,power_pipe=''): 704 | ''' 705 | generates colored, numbered output for lists and dictionaries and other types 706 | @param input: one line of input from evaluted pyp command 707 | @type input: any 708 | @param power_pipe: Output from powerpipe (pp) evaluation 709 | @type power_pipe: bool 710 | @return: colored output based on input contents 711 | @rtype: str 712 | ''' 713 | if not input and input is not 0: #TRANSLATE FALSES TO EMPTY STRINGS OR ARRAYS. SUPPLIES DUMMY INPUT TO KEEP LINES IF NEEDED. 714 | if options.keep_false or power_pipe: 715 | input = ' ' 716 | else: 717 | return '' 718 | 719 | #BASIC VARIABLES 720 | nf = 0 721 | output = '' 722 | 723 | if power_pipe: 724 | n_index = Colors.MAGENTA + '[%s]' % (self.n) + Colors.GREEN 725 | final_color = Colors.OFF 726 | else: 727 | n_index = '' 728 | final_color='' 729 | 730 | #DEALS WITH DIFFERENT TYPES OF INPUTS 731 | if type(input) in [ list, PypList, PowerPipeList] :#deals with lists 732 | 733 | for field in input: 734 | if not nf == len(input): 735 | if type(field) in [str, PypStr]: 736 | COLOR = Colors.GREEN 737 | else: 738 | COLOR = Colors.MAGENTA 739 | 740 | output = str(output) + Colors.BOLD + Colors.BLUE + "[%s]" % nf + Colors.OFF + COLOR + str(field) + Colors.GREEN 741 | nf = nf + 1 742 | return n_index + Colors.GREEN + Colors.BOLD + '[' + Colors.OFF + output + Colors.GREEN + Colors.BOLD + ']' + Colors.OFF 743 | 744 | elif type(input) in [str, PypStr] : 745 | return n_index + str(input) + final_color 746 | 747 | elif type(input) in [int, float] : 748 | return n_index + Colors.YELLOW + str(input) + Colors.OFF 749 | 750 | elif type(input) is dict: #deals with dictionaries 751 | for field in sorted(input,key=lambda x : x.lower()): 752 | output = output + Colors.OFF + Colors.BOLD + Colors.BLUE + field + Colors.GREEN + ": " + Colors.OFF + Colors.GREEN + str(input[field]) + Colors.BOLD + Colors.GREEN + ',\n ' 753 | return n_index + Colors.GREEN + Colors.BOLD + '{' + output.strip().strip(' ,') + Colors.GREEN + Colors.BOLD + '}' + Colors.OFF 754 | 755 | else: #catches every else 756 | return n_index + Colors.MAGENTA + str(input) + Colors.OFF 757 | 758 | def cmd_split(self, cmds): 759 | ''' 760 | takes a command (as previously split up by pipes and input as array cmds), 761 | and returns individual terms (cmd_array) that will be evaluated individually. 762 | Also returns a string_format string that will be used to stitch together 763 | the output with the proper spacing based on the presence of "+" and "," 764 | @param cmds: individual commands separated by pipes 765 | @type cmds: list 766 | @return: individual commands with corresponding string format 767 | @rtype: list 768 | ''' 769 | 770 | string_format = '%s' 771 | cmd_array = [] 772 | cmd = '' 773 | open_quote = False 774 | open_parenth = 0 775 | open_bracket = 0 776 | 777 | for letter in cmds: 778 | 779 | if letter in [ "'" , '"']: 780 | if cmd and cmd[-1] == '\\': 781 | open_quote = True 782 | else: 783 | open_quote = not open_quote 784 | 785 | if not open_quote: #this all ignores text in () or [] from being split by by , or + 786 | if letter == '(' : 787 | open_parenth = open_parenth + 1 788 | elif letter == ')': 789 | open_parenth = open_parenth - 1 790 | elif letter == '[' : 791 | open_bracket = open_bracket + 1 792 | elif letter == ']': 793 | open_bracket = open_bracket - 1 794 | 795 | if not open_parenth and not open_bracket and letter in [',', '+']: #these are actual formatting characters 796 | cmd_array.append(cmd) 797 | cmd = '' 798 | string_format = string_format + letter.replace('+', '%s').replace(',', ' %s') 799 | continue 800 | 801 | cmd = cmd + letter 802 | 803 | 804 | cmd_array.append(cmd) 805 | output = [(cmd_array, string_format)] 806 | return output 807 | 808 | def all_meta_split(self, input_str): 809 | ''' 810 | splits a string on any metacharacter 811 | @param input_str: input string 812 | @type input_str: str 813 | @return: list with no metacharacters 814 | @rtype: list 815 | ''' 816 | 817 | for char in input_str: 818 | if not char.isalnum(): 819 | input_str = input_str.replace(char, ' ') 820 | return [x for x in input_str.split() if x.strip()] 821 | 822 | def string_splitter(self): 823 | ''' 824 | splits self.p based on common metacharacters. returns a 825 | dictionary of this information. 826 | @return: input split up by common metacharacters 827 | @rtype: dict> 828 | ''' 829 | 830 | whitespace =self.p.split(None) 831 | slash =self.p.split('/') 832 | underscore =self.p.split('_') 833 | colon =self.p.split(':') 834 | dot =self.p.split('.') 835 | minus =self.p.split('-') 836 | all= self.all_meta_split(self.p) 837 | comma = self.p.split(',') 838 | 839 | split_variables_raw = { 840 | 841 | 'whitespace' :whitespace, 842 | 'slash' :slash, 843 | 'underscore' :underscore, 844 | 'colon' :colon, 845 | 'dot' :dot, 846 | 'minus' :minus, 847 | 'all' : all, 848 | 'comma' : comma, 849 | 850 | 'w' :whitespace, 851 | 's' :slash, 852 | 'u' :underscore, 853 | 'c' :colon, 854 | 'd' :dot, 855 | 'm' :minus, 856 | 'a' : all, 857 | 'mm' : comma, 858 | } 859 | #gets rid of empty fields 860 | split_variables = dict((x, PypList([PypStr(y) for y in split_variables_raw[x]])) for x in split_variables_raw) 861 | return split_variables 862 | 863 | def join_and_format(self, join_type): 864 | ''' 865 | joins self.p arrays with a specified metacharacter 866 | @param join_type: metacharacter to join array 867 | @type join_type: str 868 | @return: string joined by metacharacter 869 | @rtype: str 870 | ''' 871 | 872 | temp_joins = [] 873 | 874 | derived_string_format = self.history[self.n]['string_format'][-1] 875 | len_derived_str_format = len(derived_string_format.strip('%').split('%')) 876 | if len(self.p) == len_derived_str_format: 877 | string_format = derived_string_format #normal output 878 | for sub_p in self.p: 879 | if type(sub_p) in [list, PypList]: 880 | temp_joins.append(join_type.join(sub_p)) 881 | else: #deals with printing lists and strings 882 | temp_joins.append(sub_p) 883 | 884 | return PypStr(string_format % tuple(temp_joins)) 885 | 886 | else: #deals with piping pure arrays to p 887 | return PypStr(join_type.join(PypStr(x)for x in self.p)) 888 | 889 | def array_joiner(self): 890 | ''' 891 | generates a dict of self.p arrays joined with various common metacharacters 892 | @return: input joined by common metacharacters 893 | @rtype: dict 894 | ''' 895 | whitespace = self.join_and_format(' ') 896 | slash = self.join_and_format(os.sep) 897 | underscore = self.join_and_format('_') 898 | colon = self.join_and_format(':') 899 | dot = self.join_and_format('.') 900 | minus = self.join_and_format('-') 901 | all = self.join_and_format(' ') 902 | comma = self.join_and_format(',') 903 | 904 | join_variables = { 905 | 'w' : whitespace, 906 | 's' : slash, 907 | 'u' : underscore, 908 | 'c' : colon, 909 | 'd' : dot, 910 | 'm' : minus, 911 | 'a' : all, 912 | 'mm' : comma, 913 | 914 | 'whitespace' : whitespace, 915 | 'slash' : slash, 916 | 'underscore' : underscore, 917 | 'colon' : colon, 918 | 'dot' : dot, 919 | 'minus' : minus, 920 | 'all' : all, 921 | 'comma' : comma 922 | } 923 | 924 | return join_variables 925 | 926 | def translate_preset_variables(self, translate_preset_variables,file_input, second_stream_input): 927 | ''' 928 | translates variables to protected namespace dictionary for feeding into eval command. 929 | @param file_input: data from file 930 | @type file_input: list 931 | @param second_stream_input: input from second stream 932 | @type second_stream_input: list 933 | @return: values of preset variable for direct use by users 934 | @rtype: dict 935 | ''' 936 | 937 | 938 | #generic variables 939 | presets = { 940 | 'n' : self.kept_n, 941 | 'on' : self.n, 942 | 'fpp' : file_input, 943 | 'spp' : second_stream_input, 944 | 'nk': 1000 + self.kept_n, 945 | 'shell': self.shell, 946 | 'shelld' : self.shelld, 947 | 'keep': self.keep, 948 | 'lose': self.lose, 949 | 'k': self.keep, 950 | 'l':self.lose, 951 | 'rekeep':self.rekeep, 952 | 'relose':self.relose, 953 | 'rek':self.rekeep, 954 | 'rel':self.relose, 955 | 'quote': '"', 956 | 'apost':"'", 957 | 'qu':'"', 958 | 'dollar': '$', 959 | 'pwd': self.pwd, 960 | 'date': datetime.datetime.now(), 961 | 'env': os.environ.get, 962 | 'glob' : glob.glob, 963 | 'letters': 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ', 964 | 'digits': '0123456789', 965 | 'punctuation': """!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~""", 966 | 'str':(PypStr), 967 | } 968 | 969 | #removes nested entries from history 970 | history = [] 971 | for hist in self.history[self.n]['history']: 972 | if type(hist) in (list,PypList): 973 | hist = self.unlist_p(hist) 974 | history.append(hist) 975 | presets['history'] = presets['h'] = history 976 | 977 | # file 978 | if options.text_file: 979 | try: 980 | fp = file_input[self.n] 981 | except IndexError: 982 | fp = '' 983 | presets['fp'] = fp 984 | 985 | # second stream 986 | try: 987 | sp = second_stream_input[self.n] 988 | except IndexError: 989 | sp = '' 990 | 991 | presets['sp'] = sp 992 | 993 | #original input 994 | if self.history[self.n]['output']: 995 | presets['o'] = self.history[self.n]['history'][0] 996 | 997 | else: 998 | presets['o'] = '' 999 | presets['original'] = presets['o'] 1000 | 1001 | # p cleanup 1002 | p = self.p 1003 | if type(p) in [str]: 1004 | presets['p'] = PypStr(p) 1005 | elif type(p) in [list]: 1006 | presets['p'] = PypList(p) 1007 | else: 1008 | presets['p'] = p 1009 | 1010 | #custom functions 1011 | presets.update(PypFunctionCustom.__dict__) #adds user defined functions 1012 | return presets 1013 | 1014 | def initialize_n(self): 1015 | ''' 1016 | initializes history dict for a particular n, 1017 | where n is the line of the input being processed 1018 | ''' 1019 | self.history[self.n] = {} #creates dict 1020 | self.history[self.n]['error'] = '' # error data 1021 | self.history[self.n]['history'] = [] # 1022 | self.history[self.n]['history'].append(self.p) # records current p 1023 | self.history[self.n]['string_format'] = [] #used for formating output 1024 | self.history[self.n]['original_splits'] = {}#dict of original splits 1025 | self.history[self.n]['output'] = True 1026 | 1027 | def safe_eval(self, cmd, variables): 1028 | ''' 1029 | evaluates a str command (cmd) safely. takes a dictionary of protected 1030 | namespace variables as input.returns output of python call, which can 1031 | be any type of python data type (typically str or array though). 1032 | @param cmd: command to be evaluated using python 1033 | @type cmd: str 1034 | @param variables: preset variables used for evaluation 1035 | @type variables: dictionary 1036 | @return: output from python evaluation 1037 | @rtype: list 1038 | ''' 1039 | 1040 | if not self.history[self.n]['error'] and self.history[self.n]['output']: #if no errors, go forward 1041 | total_output = [] 1042 | for cm_tuple in self.cmd_split(cmd):#cm_tuple consists of commands and string format.cmd_split splits each command into terms. 1043 | string_format = cm_tuple[1] #how the results will eventually be formatted. 1044 | for cm in cm_tuple[0]:#cm is the expression seperated by a + or a , 1045 | #evaluate cm and add to dictionary catching any error. 1046 | try: 1047 | output = eval(cm, variables) #500 lines of code wrap this line!!! 1048 | except KeyboardInterrupt: 1049 | print Colors.RED + "killed by user" + Colors.OFF 1050 | sys.exit() 1051 | except Exception, err: 1052 | self.history[self.n]['error'] = Colors.RED + 'error: ' + str(err) + Colors.OFF, Colors.RED + cmd + Colors.OFF 1053 | break 1054 | #totals output for each cm 1055 | try: 1056 | if output is True : #allows truth tests 1057 | output = self.p 1058 | except: 1059 | pass 1060 | 1061 | total_output.append(output) 1062 | 1063 | #updates self.history dictionary 1064 | self.history[self.n]['string_format'].append(string_format) 1065 | #generates printable output 1066 | return total_output 1067 | 1068 | 1069 | def get_user_input(self, total_output,second_stream_input,file_input, power_pipe): 1070 | ''' 1071 | figures out what to show user in terms of powerpipe output. does NOT update history dictionary. 1072 | 1073 | @param total_output: line output from eval 1074 | @type total_output: list 1075 | @param second_stream_input: entire input from second string 1076 | @type second_stream_input: list 1077 | @param file_input: entire input from file 1078 | @type file_input: list 1079 | @param power_pipe: kind of power pipe 1080 | @type power_pipe: string 1081 | @return: output for display 1082 | @rtype: list 1083 | ''' 1084 | 1085 | try: #who knows what could happen with user input 1086 | n = self.n 1087 | if power_pipe == 'pp' and total_output or not power_pipe: #standard input 1088 | user_output = total_output 1089 | elif power_pipe == 'spp' and second_stream_input: 1090 | user_output = [second_stream_input[n]] 1091 | elif power_pipe == 'fpp' and file_input: 1092 | user_output = [file_input[n]] 1093 | elif power_pipe: # power pipe variable is referenced, but does not exist. 1094 | print Colors.RED + "YOU'RE LIST VARIABLE DOES NOT EXIST: " + Colors.GREEN + power_pipe + Colors.OFF 1095 | sys.exit() 1096 | except: #default output is null per line 1097 | user_output =[' '] 1098 | 1099 | return user_output 1100 | 1101 | def update_history(self, total_output,second_stream_input,file_input, power_pipe): 1102 | ''' 1103 | updates history dictionary with output from python evaluation 1104 | @param total_output: line output from eval 1105 | @type total_output: list 1106 | @param second_stream_input: entire input from second string 1107 | @type second_stream_input: list 1108 | @param file_input: entire input from file 1109 | @type file_input: list 1110 | @param power_pipe: kind of power pipe 1111 | @type power_pipe: string 1112 | ''' 1113 | 1114 | #marks null output, as '' except when output is zero, poerpope, or we are printing out null lines 1115 | if (not total_output or not [x for x in total_output if x]or self.history[self.n]['error'])\ 1116 | and total_output != [0]\ 1117 | and not power_pipe: #kill irrelevant output 1118 | 1119 | self.history[self.n]['history'].append(False) 1120 | self.history[self.n]['output']=False 1121 | else: # good output 1122 | string_format = self.history[self.n]['string_format'][-1] 1123 | output_array = [] 1124 | history_array = [] 1125 | contains_list = False 1126 | #actual output is p or pp unless spp is invoked. 1127 | user_input = self.get_user_input(total_output, second_stream_input, file_input, power_pipe) 1128 | self.kept_n = self.kept_n + 1 #only update if output is kept 1129 | 1130 | #update history array 1131 | for out in total_output: # forms an array called_output array of strings or array_traced strings 1132 | history_array.append(out) # for feeding back to pipe 1133 | contains_list = True if type(out) not in [str, PypStr] else False 1134 | 1135 | #update actual output 1136 | for out in user_input: 1137 | output_array.append(self.array_tracer(out, power_pipe)) # for output 1138 | 1139 | self.history[self.n]['output'] = string_format % (tuple(output_array)) 1140 | 1141 | if contains_list: #this section prevents buildup of recursive lists. 1142 | self.history[self.n]['history'].append(total_output) # just adds list to total output if list 1143 | else: 1144 | self.history[self.n]['history'].append(string_format % (tuple(history_array))) # adds properly formatted string if string. 1145 | 1146 | def flatten_list(self, iterables): 1147 | ''' 1148 | returns a list of strings from nested lists 1149 | @param iterables: nested list to flatten 1150 | @type iterables: list 1151 | ''' 1152 | out = [] 1153 | try: 1154 | if [x for x in iterables if type(x) in [str, PypStr]]: 1155 | out = out + iterables 1156 | else: 1157 | for x in iterables: 1158 | out = out + self.flatten_list(x) 1159 | except: #catches non iterables 1160 | out = [iterables] 1161 | 1162 | return out 1163 | 1164 | def power_pipe_eval(self, cmd, inputs, second_stream_input, file_input, power_pipe_type): 1165 | ''' 1166 | evaluates pp statement. returns sanitized result. 1167 | @param cmd: power pipe command 1168 | @type cmd: str 1169 | @param inputs: inputs from std-in or previous python eval 1170 | @type inputs: list 1171 | @param power_pipe_type: kind of powerpipe 1172 | @type power_pipe_type: str 1173 | @return: 'p' and output of python evaluation 1174 | @rtype: list 1175 | ''' 1176 | variables = {} 1177 | self.history = {} 1178 | padded_output = [] 1179 | 1180 | 1181 | variables['str'] = PypStr #useful for list comps 1182 | variables['n'] = self.kept_n 1183 | variables['on'] = self.n 1184 | 1185 | inputs = self.flatten_list(inputs) 1186 | 1187 | inputs = [x for x in inputs if self.unlist_p(x) is not False] #keeps unfiltered output 1188 | 1189 | variables['pp'] = PowerPipeList(inputs) 1190 | variables['spp'] = PowerPipeList(second_stream_input) 1191 | variables['fpp'] = PowerPipeList(file_input) 1192 | 1193 | 1194 | try: 1195 | output = eval(cmd, variables) #1000 lines of code wrap this line!!! 1196 | except KeyboardInterrupt: 1197 | print Colors.RED + "killed by user" + Colors.OFF 1198 | sys.exit() 1199 | except Exception, err: 1200 | print Colors.RED + 'error: ' + str(err) + Colors.OFF, Colors.RED + cmd + Colors.OFF 1201 | sys.exit() 1202 | 1203 | if output is None: #allows use of inplace methods like sort 1204 | output = variables[power_pipe_type] 1205 | 1206 | if type(output) in [int, float]: #keeps output in array 1207 | output = [output] 1208 | 1209 | if type(output) in [str, PypStr, tuple]: #makes sure output is in list of lists 1210 | output = [[output]] 1211 | 1212 | if [x for x in output if type(x) in [tuple]]:#changes tuples to lists 1213 | output = [PypList(x) for x in output if type(x) in [tuple]] 1214 | 1215 | if len(output) == 1: #turn off powerpipe if output is single item 1216 | power_pipe_type = '' 1217 | 1218 | return output, power_pipe_type 1219 | 1220 | def detect_power_pipe(self, command, power_pipe_type): 1221 | ''' 1222 | detects presense of powerpipe 1223 | @param command: command to be evaluated 1224 | @type command: str 1225 | @param power_pipe_type: kind of powerpipe (future use) 1226 | @type power_pipe_type: str 1227 | @return: True if powerpipe else False 1228 | @rtype: bool 1229 | ''' 1230 | open_quote = False 1231 | cmd_raw = list(command) 1232 | cmd = [] 1233 | 1234 | for letter in cmd_raw: 1235 | if letter not in ['"', "'"] and not letter.isalnum(): 1236 | letter = ' ' 1237 | cmd.append(letter) 1238 | 1239 | 1240 | cmds = ''.join(cmd).split() 1241 | for cmd in cmds: 1242 | cmd = list(cmd) 1243 | test_cmd = '' 1244 | while cmd: 1245 | letter = cmd.pop(0) 1246 | test_cmd = test_cmd + letter 1247 | if not open_quote: 1248 | if power_pipe_type == test_cmd and not cmd: 1249 | return True 1250 | 1251 | if letter in [ "'" , '"']: 1252 | if cmd and cmd[0] == '\\': 1253 | open_quote = True 1254 | else: 1255 | open_quote = not open_quote 1256 | return False 1257 | 1258 | def format_input(self, cmd, input_set, second_stream_input, file_input): 1259 | ''' 1260 | checks for powerpipe presence, evaluates powerpipe pp and returns 1261 | formatted output if detected 1262 | @param cmd: user command 1263 | @type cmd: str 1264 | @param input_set: input from std-in or previous python evaluation 1265 | @type input_set: list 1266 | @return: command, input set, presence of powerpipe 1267 | @rtype: list 1268 | ''' 1269 | #POWER PIPES 1270 | power_pipe = '' #power pipe is off by default 1271 | if self.detect_power_pipe(cmd, 'pp') : 1272 | input_set,power_pipe = self.power_pipe_eval( cmd, input_set, second_stream_input, file_input,'pp') 1273 | cmd = 'p' 1274 | elif self.detect_power_pipe(cmd, 'spp') : 1275 | second_stream_input, power_pipe = self.power_pipe_eval( cmd, input_set, second_stream_input, file_input,'spp') 1276 | cmd = 'p' 1277 | elif self.detect_power_pipe(cmd, 'fpp'): 1278 | file_input, power_pipe = self.power_pipe_eval( cmd, input_set, second_stream_input, file_input,'fpp') 1279 | cmd = 'p' 1280 | 1281 | return cmd, input_set,second_stream_input, file_input, power_pipe 1282 | 1283 | def unlist_p(self, p): 1284 | ''' 1285 | changes one item arrays to strings for input cleanup 1286 | @param p: input from std-in or previous pyp evaluation 1287 | @type p: list 1288 | @return: will return a string if input is a list and has one member 1289 | @rtype: list,str 1290 | ''' 1291 | if type(p) in [list, PypList] and len(p) == 1: 1292 | p = p[0] 1293 | 1294 | return p 1295 | 1296 | 1297 | def process(self, inputs, file_input, cmds, second_stream_input): 1298 | ''' 1299 | takes primary data from input stream (can be string, array or dictionary), applies user commands to it, 1300 | captures this output. Also, generates several variables such as line counter and 1301 | various string split and join variables. Outputs string, array, or dictionary in 1302 | a format deliniated by the string formating stamp 1303 | @param inputs: inputs from std-in or previous pyp eval 1304 | @type inputs: str,list 1305 | @param file_input: inputs from file 1306 | @type file_input: list 1307 | @param cmds: python commands to be evaluated 1308 | @type cmds: list 1309 | @param second_stream_input: second stream input 1310 | @type second_stream_input: list 1311 | ''' 1312 | 1313 | while cmds: #cmds are commands that will be executed on the input stream 1314 | self.n = -1 # overall line counter. will change to 0 asap. 1315 | self.kept_n = 0 # counter of kept lines. needs to be avail for eval, so starts as 0 1316 | cmd = cmds.pop(0) 1317 | cmd, input_set,second_stream_input, file_input, power_pipe = self.format_input(cmd, inputs,second_stream_input, file_input) 1318 | original_input_set = input_set[:] 1319 | #MAIN LOOP 1320 | while input_set: 1321 | 1322 | self.p = self.unlist_p(input_set.pop(0)) # p is main line variable being manipulated 1323 | self.n = self.n + 1 # raises counters 1324 | variables = {} 1325 | 1326 | if not self.n in self.history: # initializes self.history dict for line 1327 | self.initialize_n() 1328 | else: 1329 | if self.p is False: #skip false output but n is updated 1330 | continue 1331 | 1332 | if type(self.p) in [ str, PypStr]: # p is string 1333 | variables = self.string_splitter() 1334 | if not self.history[self.n]['original_splits']: #makes variables dealing with original input 1335 | self.history[self.n]['original_splits'] = dict(('o' + x, variables[x]) for x in variables) 1336 | if not self.history[self.n]['output']: #kills o variables if there is no output 1337 | self.history[self.n]['original_splits'] = dict(('o' + x, '') for x in variables) 1338 | 1339 | elif type(self.p) in [list, PypList] and not power_pipe: # p is list of lists, constructs various joins 1340 | try: #for going from list to powerpipe 1341 | variables = self.array_joiner() 1342 | except: 1343 | pass 1344 | 1345 | 1346 | variables.update(self.translate_preset_variables(original_input_set,file_input, second_stream_input)) #add incrementals 1347 | variables.update(self.history[self.n]['original_splits']) # updates with original splits 1348 | 1349 | total_output = self.safe_eval(cmd, variables) 1350 | self.update_history(total_output,second_stream_input,file_input ,power_pipe) 1351 | #takes output, feeds back into input 1352 | 1353 | new_input = [self.history[x]['history'][-1] for x in self.history ] # takes last output as new input 1354 | self.process(new_input, file_input, cmds, second_stream_input) #process new input 1355 | 1356 | def output(self, total_cmds): 1357 | ''' 1358 | generates final output. 1359 | @param total_cmds: all commands executed 1360 | @type total_cmds: list 1361 | ''' 1362 | execute_cmds = [] 1363 | for self.history_index in self.history: 1364 | error = self.history[self.history_index]['error'] 1365 | 1366 | if not error or "list index out of range" in error[0] or 'string index out of range' in error[0] : #no error 1367 | 1368 | cmd = self.history[self.history_index]['output'] #color formated output 1369 | 1370 | if cmd: #kept commands 1371 | if options.execute: #executes command 1372 | execute_cmds.append(cmd) 1373 | else: 1374 | print cmd # normal output 1375 | elif options.keep_false: #prints blank lines for lost False commands 1376 | print 1377 | else: #error 1378 | print Colors.RED + self.history[self.history_index]['error'][0] + Colors.RED + ' : ' + self.history[self.history_index]['error'][1] + Colors.OFF 1379 | 1380 | if execute_cmds: 1381 | self.final_execute(execute_cmds) 1382 | 1383 | 1384 | def initilize_input(self): 1385 | ''' 1386 | decides what type of input to use (all arrays. can be from rerun file, yaml, or st-in. 1387 | also does some basic processing, for using different delimeters or looking for jts numbers 1388 | @return: starting input for pyp processing 1389 | @rtype: list 1390 | ''' 1391 | 1392 | if options.manual: 1393 | print Docs.manual 1394 | sys.exit() 1395 | 1396 | if options.unmodified_config: 1397 | print Docs.unmodified_config 1398 | sys.exit() 1399 | 1400 | rerun_path = '/%s/pyp_rerun_%d.txt' %(tempfile.gettempdir(),os.getppid()) 1401 | 1402 | if options.rerun: #deals with rerunning script with -r flag 1403 | if not os.path.exists(rerun_path): 1404 | gpid = int(os.popen("ps -p %d -oppid=" % os.getppid()).read().strip()) 1405 | rerun_gpid_path = '/%s/pyp_rerun_%d.txt' %(tempfile.gettempdir() ,gpid) 1406 | if os.path.exists(rerun_gpid_path): 1407 | rerun_path = rerun_gpid_path 1408 | else: 1409 | print Colors.RED + rerun_path + " does not exist" + Colors.OFF 1410 | sys.exit() 1411 | pipe_input = [x.strip() for x in open(rerun_path) if x.strip()] 1412 | #print pipe_input 1413 | 1414 | elif options.blank_inputs: 1415 | pipe_input = [] 1416 | end_n = int(options.blank_inputs) 1417 | for n in range(0,end_n): 1418 | pipe_input.append('') 1419 | 1420 | 1421 | elif options.no_input: 1422 | pipe_input = [''] 1423 | 1424 | else: 1425 | pipe_input = [x.rstrip() for x in sys.stdin.readlines() if x.strip()] 1426 | if not pipe_input: 1427 | pipe_input = [''] #for using control d to activate comands with no input 1428 | 1429 | rerun_file = open(rerun_path, 'w') 1430 | rerun_file.write('\n'.join([str(x) for x in pipe_input])) 1431 | rerun_file.close() 1432 | 1433 | 1434 | inputs = pipe_input 1435 | 1436 | return [[PypStr(x)] for x in inputs] 1437 | 1438 | def main(self): 1439 | '''generates input stream based on file, std-in options, rerun, starts process loop, generates output''' 1440 | second_stream_input = [PypStr(x) for x in args[1:]] #2nd stream input 1441 | file_input = [PypStr(x) for x in self.load_file() ]# file input 1442 | 1443 | #load custom executer if possible. 1444 | self.final_execute=self.get_custom_execute() 1445 | 1446 | #load user and group macros. 1447 | user_macro_path,group_macro_path=self.get_custom_macro_paths() 1448 | user_macros = self.load_macros(user_macro_path) 1449 | group_macros = self.load_macros(group_macro_path) 1450 | group_macros.update(user_macros) #merges group and user macros 1451 | macros = group_macros 1452 | 1453 | #macros for action ie saving and deleting 1454 | action_macros = group_macros if options.macro_group else user_macros 1455 | action_macros_path = group_macro_path if options.macro_group else user_macro_path 1456 | 1457 | self.list_macros(macros) 1458 | self.delete_macros(action_macros, action_macros_path) 1459 | 1460 | if not args: # default command is just to print. 1461 | cmds = ['p'] 1462 | else: 1463 | cmds = self.cmds_split(args[0], macros) 1464 | 1465 | self.write_macros(action_macros, action_macros_path, cmds) #needs cmds before we write macros 1466 | 1467 | inputs = self.initilize_input() #figure out our input stream 1468 | 1469 | self.process(inputs, file_input, cmds, second_stream_input,) #recursive processing to generate history dict 1470 | 1471 | self.output(cmds) #output text or execute commands from history dict 1472 | 1473 | 1474 | 1475 | 1476 | 1477 | 1478 | class Docs(): 1479 | manual = ''' 1480 | =================================================================================== 1481 | PYED PIPER MANUAL 1482 | 1483 | pyp is a command line utility for parsing text output and generating complex 1484 | unix commands using standard python methods. pyp is powered by python, so any 1485 | standard python string or list operation is available. 1486 | 1487 | The variable "p" represents EACH line of the input as a python string, so for 1488 | example, you can replace all "FOO" with "GOO" using "p.replace('FOO','GOO')". 1489 | Likewise, the variable "pp" represents the ENTIRE input as a python array, so 1490 | to sort the input alphabetically line-by-line, use "pp.sort()" 1491 | 1492 | Standard python relies on whitespace formating such as indentions. Since this 1493 | is not convenient with command line operations, pyp employs an internal piping 1494 | structure ("|") similar to unix pipes. This allows passing of the output of 1495 | one command to the input of the next command without nested "(())" structures. 1496 | It also allows easy spliting and joining of text using single, commonsense 1497 | variables (see below). An added bonus is that any subresult between pipes 1498 | is available, making it easy to refer to the original input if needed. 1499 | 1500 | Filtering output is straightforward using python Logic operations. Any output 1501 | that is "True" is kept while anything "False" is eliminated. So "p.isdigit()" 1502 | will keep all lines that are completely numbers. 1503 | 1504 | The output of pyp has been optimized for typical command line scenarios. For 1505 | example, if text is broken up into an array using the "split()" method, the 1506 | output will be conveniently numbered by field because a field selection is 1507 | anticipated. If the variable "pp" is employed, the output will be numbered 1508 | line-by-line to facilitate picking any particular line or range of lines. In 1509 | both cases, standard python methods (list[start:end]) can be used to select 1510 | fields or lines of interest. Also, the standard python string and list objects 1511 | have been overloaded with commonly used methods and attributes. For example, 1512 | "pp.uniq()" returns all unique members in an array, and p.kill('foo') will 1513 | eliminate all "foo" in the input. 1514 | 1515 | pyp commands can be easily saved to disk and recalled using user-defined macros, 1516 | so a complicated parsing operation requiring 20 or more steps can be recalled 1517 | easily, providing an alternative to quick and dirty scripts. For more advanced 1518 | users, these macros can be saved to a central location, allowing other users to 1519 | execute them. Also, an additional text file (PypCustom.py) can be set up that 1520 | allows additional methods to be added to the pyp str and list methods, allowing 1521 | tight integration with larger facilities data structures or custom tool sets. 1522 | 1523 | ----------------------------------------------------------------------------------- 1524 | PIPING IN THE PIPER 1525 | ----------------------------------------------------------------------------------- 1526 | You can pipe data WITHIN a pyp statement using standard unix style pipes ("|"), 1527 | where "p" now represents the evaluation of the python statement before the "|". 1528 | You can also refer back to the ORIGINAL, unadulterated input using the variable 1529 | "o" or "original" at any time...and the variable "h" or "history" allows you 1530 | to refer back to ANY subresult generated between pipes ("|"). 1531 | 1532 | All pyp statements should be enclosed in double quotes, with single quotes being 1533 | used to enclose any strings.''' + Colors.YELLOW + ''' 1534 | 1535 | echo 'FOO IS AN ' | pyp "p.replace('FOO','THIS') | p + 'EXAMPLE'" 1536 | ==> THIS IS AN EXAMPLE''' + Colors.GREEN + ''' 1537 | 1538 | ----------------------------------------------------------------------------------- 1539 | THE TYPE OF COLOR IS THE TYPE 1540 | ----------------------------------------------------------------------------------- 1541 | pyp uses a simple color and numerical indexing scheme to help you identify what 1542 | kind of objects you are working with. Don't worry about the specifics right now, 1543 | just keep in mind that different types can be readily identified: 1544 | 1545 | strings: hello world 1546 | 1547 | integers or floats:''' + Colors.YELLOW + ''' 1984''' + Colors.GREEN + ''' 1548 | 1549 | split-up line: ''' + Colors.BOLD + ''' [''' + Colors.BLUE + '[0]' + Colors.OFF + Colors.GREEN \ 1550 | + 'hello' + Colors.BOLD + Colors.BLUE + '[1]' + Colors.OFF + Colors.GREEN + '''world''' +\ 1551 | Colors.BOLD + '] ' + Colors.OFF + Colors.GREEN + ''' 1552 | 1553 | entire input list: ''' +\ 1554 | Colors.MAGENTA + '[0]' + Colors.GREEN + 'first line\n' + Colors.OFF +\ 1555 | Colors.MAGENTA + ' [1]' + Colors.GREEN + 'second line' + Colors.OFF + Colors.GREEN + ''' 1556 | 1557 | dictionaries:''' + Colors.BOLD + ''' {''' + Colors.BLUE + 'hello world' + Colors.BOLD +\ 1558 | Colors.GREEN + ': ' + Colors.OFF + Colors.GREEN + '1984' + Colors.BOLD + '}' + Colors.OFF + Colors.GREEN + ''' 1559 | 1560 | other objects:''' + Colors.MAGENTA + ' RANDOM_OBJECT' + Colors.GREEN + ''' 1561 | 1562 | The examples below will use a yellow/blue color scheme to seperate them 1563 | from the main text however. Also, all colors can be removed using the 1564 | --turn_off_color flag. 1565 | 1566 | ----------------------------------------------------------------------------------- 1567 | STRING OPERATIONS 1568 | ----------------------------------------------------------------------------------- 1569 | Here is a simple example for splitting the output of "ls" (unix file list) on '.':''' + Colors.YELLOW + ''' 1570 | 1571 | ls random_frame.jpg | pyp "p.split('.')" 1572 | ==> [''' + Colors.BLUE + '[0]' + Colors.YELLOW + 'random_frame' + Colors.BLUE + '[1]' + Colors.YELLOW + 'jpg] ''' + Colors.GREEN + ''' 1573 | 1574 | The variable "p" represents each line piped in from "ls". Notice the output has 1575 | index numbers, so it's trivial to pick a particular field or range of fields, 1576 | i.e. pyp "p.split('.')[0]" is the FIRST field. There are some pyp generated 1577 | variables that make this simpler, for example the variable "d" or "dot" is the 1578 | same as p.split('.'):''' + Colors.YELLOW + ''' 1579 | 1580 | ls random_frame.jpg | pyp "dot" 1581 | ==> [''' + Colors.BLUE + '[0]' + Colors.YELLOW + 'random_frame' + Colors.BLUE + '[1]' + Colors.YELLOW + '''jpg] 1582 | 1583 | ls random_frame.jpg | pyp "dot[0]" 1584 | ==> random_frame''' + Colors.GREEN + ''' 1585 | 1586 | To Join lists back together, just pipe them to the same or another built-in 1587 | variable(in this case "u", or "underscore"):''' + Colors.YELLOW + ''' 1588 | 1589 | ls random_frame.jpg | pyp "dot" 1590 | ==> [''' + Colors.BLUE + '[0]' + Colors.YELLOW + 'random_frame' + Colors.BLUE + '[1]' + Colors.YELLOW + '''jpg] 1591 | 1592 | ls random_frame.jpg | pyp "dot|underscore" 1593 | ==> random_frame_jpg ''' + Colors.GREEN + ''' 1594 | 1595 | To add text, just enclose it in quotes, and use "+" or "," just like python: ''' + Colors.YELLOW + ''' 1596 | 1597 | ls random_frame.jpg | pyp "'mkdir seq.tp_' , d[0]+ '_v1/misc_vd8'" 1598 | ==> mkdir seq.tp_random_frame_v1/misc_vd8'" ''' + Colors.GREEN + ''' 1599 | 1600 | A fundamental difference between pyp and standard python is that pyp allows you 1601 | to print out strings and lists on the same line using the standard "+" and "," 1602 | notation that is used for string construction. This allows you to have a string 1603 | and then print out the results of a particular split so it's easy to pick out 1604 | your field of interest: ''' + Colors.YELLOW + ''' 1605 | 1606 | ls random_frame.jpg | pyp "'mkdir', dot" 1607 | ==> mkdir [''' + Colors.BLUE + '[0]' + Colors.YELLOW + 'random_frame' + Colors.BLUE + '[1]' + Colors.YELLOW + '''jpg] '''+ Colors.GREEN + ''' 1608 | 1609 | In the same way, two lists can be displayed on the same line using "+" or ",". 1610 | If you are trying to actually combine two lists, enclose them in parentheses:''' + Colors.YELLOW + ''' 1611 | 1612 | ls random_frame.jpg | pyp "(underscore + dot)" 1613 | ==> [''' + Colors.BLUE + '[0]' + Colors.YELLOW + 'random' + Colors.BLUE + '[1]' + Colors.YELLOW +'frame.jpg'\ 1614 | + Colors.BLUE + '[2]' + Colors.YELLOW + 'random_frame'+ Colors.BLUE + '[3]' + Colors.YELLOW + '''jpg] ''' + Colors.GREEN + ''' 1615 | 1616 | This behaviour with '+' and ',' holds true in fact for ANY object, making 1617 | it easy to build statements without having to worry about whether they 1618 | are strings or not. 1619 | 1620 | ----------------------------------------------------------------------------------- 1621 | ENTIRE INPUT LIST OPERATIONS 1622 | ----------------------------------------------------------------------------------- 1623 | To perform operations that operate on the ENTIRE array of std-in, Use the variable 1624 | "pp", which you can manipulate using any standard python list methods. For example, 1625 | to sort the input, use:''' + Colors.YELLOW + ''' 1626 | 1627 | pp.sort()''' + Colors.GREEN + ''' 1628 | 1629 | When in array context, each line will be numbered with it's index in the array, 1630 | so it's easy to, for example select the 6th line of input by using "pp[5]". 1631 | You can pipe this back to p to continue modifying this input on a 1632 | line-by-line basis: ''' + Colors.YELLOW + ''' 1633 | 1634 | pp.sort() | p ''' + Colors.GREEN + ''' 1635 | 1636 | You can add arbitrary entries to your std-in stream at this point using 1637 | list addition. For example, to add an entry to the start and end:''' + Colors.YELLOW + ''' 1638 | 1639 | ['first entry'] + pp + ['last entry'] ''' + Colors.GREEN + ''' 1640 | 1641 | The new pp will reflect these changes for all future operations. 1642 | 1643 | There are several methods that have been added to python's normal list methods 1644 | to facilitate common operations. For example, keeping unique members or 1645 | consolidating all input to a single line can be accomplished with: '''+ Colors.YELLOW + ''' 1646 | 1647 | pp.uniq() 1648 | pp.oneline()'''+ Colors.GREEN + ''' 1649 | 1650 | Also, there are a few useful python math functions that work on lists of 1651 | integers or floats like sum, min, and max. For example, to add up 1652 | all of the integers in the last column of input: '''+ Colors.YELLOW + ''' 1653 | 1654 | whitespace[-1] | int(p) | sum(pp) '''+ Colors.GREEN + ''' 1655 | 1656 | 1657 | ----------------------------------------------------------------------------------- 1658 | MATH OPERATIONS 1659 | ----------------------------------------------------------------------------------- 1660 | To perform simple math, use the integer or float functions (int() or float()) 1661 | AND put the math in "()" + ''' + Colors.YELLOW + ''' 1662 | 1663 | echo 665 | pyp "(int(p) + 1)" 1664 | ==> 666 ''' + Colors.GREEN + ''' 1665 | ----------------------------------------------------------------------------------- 1666 | LOGIC FILTERS 1667 | ----------------------------------------------------------------------------------- 1668 | To filter output based on a python function that returns a Booleon (True or False), 1669 | just pipe the input to this function, and all lines that return True will keep 1670 | their current value, while all lines that return False will be eliminated. ''' + Colors.YELLOW + ''' 1671 | 1672 | echo 666 | pyp "p.isdigit()" 1673 | ==> 666''' + Colors.GREEN + ''' 1674 | 1675 | Keep in mind, that if the Boolean is True, the entire value of p is returned. 1676 | This comes in handy when you want to test on one field, but use something else. 1677 | For example, a[2].isdigit() will return p, not a[2] if a[2] is a digit. 1678 | 1679 | Standard python logic operators such as "and","or","not", and 'in' work as well. 1680 | 1681 | For example to filter output based on the presence of "GOO" in the line, use this:''' + Colors.YELLOW + ''' 1682 | 1683 | echo GOO | pyp "'G' in p" 1684 | ==> GOO'''+ Colors.GREEN + ''' 1685 | 1686 | The pyp functions "keep(STR)" and "lose(STR)", and their respective shortcuts, 1687 | "k(STR)" and "i(STR)", are very useful for simple OR style string 1688 | filtering. See below. 1689 | 1690 | Also note, all lines that test False ('', {}, [], False, 0) are eliminated from 1691 | the output completely. You can instead print out a blank line if something tests 1692 | false using --keep_false. This is useful if you need placeholders to keep lists 1693 | in sync, for example. 1694 | ----------------------------------------------------------------------------------- 1695 | SECOND STREAM, TEXT FILE, AND BLANK INPUT 1696 | ----------------------------------------------------------------------------------- 1697 | Normally, pyp receives its input by piping into it like a standard unix shell 1698 | command, but sometimes it's necessary to combine two streams of inputs, such as 1699 | consolidating the output of two shell commands line by line. pyp provides 1700 | for this with the second stream input. Essentially anything after the pyp 1701 | command that is not associated with an option flag is brought into pyp as 1702 | the second stream, and can be accessed seperately from the primary stream 1703 | by using the variable 'sp' 1704 | 1705 | To input a second stream of data, just tack on strings or execute (use backticks) 1706 | a command to the end of the pyp command, and then access this array using the 1707 | variable 'sp' ''' + Colors.YELLOW + ''' 1708 | 1709 | echo random_frame.jpg | pyp "p, sp" `echo "random_string"` 1710 | ===> random_frame.jpg random_string''' + Colors.GREEN + ''' 1711 | 1712 | In a similar way, text input can be read in from a text file using the 1713 | --text_file flag. You can access the entire file as a list using the variable 1714 | 'fpp', while the variable 'fp' reads in one line at a time. This text file 1715 | capability is very useful for lining up std-in data piped into pyp with 1716 | data in a text file like this:''' + Colors.YELLOW + ''' 1717 | 1718 | echo normal_input | pyp -text_file example.txt "p, fp" ''' + Colors.GREEN + ''' 1719 | 1720 | This setup is geared mostly towards combining data from std-in with that in 1721 | a text file. If the text file is your only data, you should cat it, and pipe 1722 | this into pyp. 1723 | 1724 | If you need to generate output from pyp with no input, use --blank_inputs. 1725 | This is useful for generating text based on line numbers using the 'n' 1726 | variable. 1727 | 1728 | ----------------------------------------------------------------------------------- 1729 | TEXT FILE AND SECOND STREAM LIST OPERATIONS 1730 | ----------------------------------------------------------------------------------- 1731 | List operations can be performed on file inputs and second stream 1732 | inputs using the variables spp and fpp, respectively. For example to sort 1733 | a file input, use: ''' + Colors.YELLOW + ''' 1734 | 1735 | fpp.sort() ''' + Colors.GREEN + ''' 1736 | 1737 | Once this operation takes place, the sorted fpp will be used for all future 1738 | operations, such as referring to the file input line-by-line using fp. 1739 | 1740 | You can add these inputs to the std-in stream using simple list 1741 | additions like this: ''' + Colors.YELLOW + ''' 1742 | 1743 | pp + fpp ''' + Colors.GREEN + ''' 1744 | 1745 | If pp is 10 lines, and fpp is 10 line, this will result in a new pp stream 1746 | of 20 lines. fpp will remain untouched, only pp will change with this 1747 | operation. 1748 | 1749 | Of course, you can trim these to your needs using standard 1750 | python list selection techniques: ''' + Colors.YELLOW + ''' 1751 | 1752 | pp[0:5] + fpp[0:5] ''' + Colors.GREEN + ''' 1753 | 1754 | This will result in a new composite input stream of 10 lines. 1755 | 1756 | Keep in mind that the length of fpp and spp is trimmed to reflect 1757 | that of std-in. If you need to see more of your file or second 1758 | stream input, you can extend your std-in stream simply:''' + Colors.YELLOW + ''' 1759 | 1760 | pp + ['']*10 ''' + Colors.GREEN + ''' 1761 | 1762 | will add 10 blank lines to std-in, and thus reveal another 10 1763 | lines of fpp if available. 1764 | 1765 | 1766 | ----------------------------------------------------------------------------------- 1767 | MACRO USAGE 1768 | ----------------------------------------------------------------------------------- 1769 | Macros are a way to permently store useful commands for future recall. They are 1770 | stored in your home directory by default. Facilites are provided to store public 1771 | macros as well, which is useful for sharing complex commands within your work group. 1772 | Paths to these text files can be reset to anything you choose my modifying the 1773 | PypCustom.py config file. Macros can become quite complex, and provide 1774 | a useful intermediate between shell commands and scripts, especially for solving 1775 | one-off problems. Macro listing, saving, deleting, and searching capabilities are 1776 | accessible using --macrolist, --macro_save, --macro_delete, --macro_find flags. 1777 | Run pyp --help for more details. 1778 | 1779 | you can pyp to and from macros just like any normal pyp command. ''' + Colors.YELLOW + ''' 1780 | pyp "a[0]| my_favorite_macros | 'ls', p" ''' + Colors.GREEN + ''' 1781 | 1782 | Note, if the macro returns a list, you can access individual elements using 1783 | [n] syntax:''' + Colors.YELLOW + ''' 1784 | pyp "my_list_macro[2]" ''' + Colors.GREEN + ''' 1785 | 1786 | Also, if the macro uses %s, you can append a %(string,..) to then end to string 1787 | substitute: ''' + Colors.YELLOW + ''' 1788 | pyp "my_string_substitution_macro%('test','case')" ''' + Colors.GREEN + ''' 1789 | 1790 | By default, macros are saved in your home directory. This can be modifed to any 1791 | directory by modifying the user_macro_path attribute in your PypCustom.py. If 1792 | you work in a group, you can also save macros for use by others in a specific 1793 | location by modifying group_macro_path. See the section below on custom 1794 | methods about how to set up this file. 1795 | ----------------------------------------------------------------------------------- 1796 | CUSTOM METHODS 1797 | ----------------------------------------------------------------------------------- 1798 | pyed pyper relies on overloading the standard python string and list objects 1799 | with its own custom methods. If you'd like to try writing your own methods 1800 | either to simplify a common task or integrate custom functions using a 1801 | proprietary API, it's straightforward to do. You'll have to setup a config 1802 | file first: 1803 | 1804 | pyp --unmodified_config > PypCustom.py 1805 | sudo chmod 666 PypCustom.py 1806 | 1807 | There are example functions for string, list, powerpipe, and generic methods. 1808 | to get you started. When pyp runs, it looks for this text file and automatically 1809 | loads any found functions, overloading them into the appropriate objects. You 1810 | can then use your custom methods just like any other pyp function. 1811 | ----------------------------------------------------------------------------------- 1812 | TIPS AND TRICKS 1813 | ----------------------------------------------------------------------------------- 1814 | If you have to cut and paste data (from an email for example), execute pyp, paste 1815 | in your data, then hit CTRL-D. This will put the data into the disk buffer. Then, 1816 | just rerun pyp with --rerun, and you'll be able to access this data for further 1817 | pyp manipulations! 1818 | 1819 | If you have split up a line into a list, and want to process this list line by 1820 | line, simply pipe the list to pp and then back to p: pyp "w | pp |p" 1821 | 1822 | Using --rerun is also great way to buffer data into pyp from long-running scripts 1823 | 1824 | pyp is an easy way to generate commands before executing them...iteratively keep 1825 | adding commands until you are confident, then use the --execute flag or pipe them 1826 | to sh. You can use ";" to set up dependencies between these commands...which is 1827 | an easy way to work out command sequences that would typically be executed in a 1828 | "foreach" loop. 1829 | 1830 | Break out complex intermediate steps into macros. Macros can be run at any point in a 1831 | pyp command. 1832 | 1833 | If you find yourself shelling out constantly to particular commands, it might 1834 | be worth adding python methods to the PypCustom.py config, especially if you 1835 | are at a large facility. 1836 | 1837 | Many command line tools (like stat) use a KEY:VALUE format. The shelld function 1838 | will turn this into a python dictionary, so you can access specific data using 1839 | their respective keys by using something like this: shelld(COMMAND)[KEY] 1840 | 1841 | =================================================================================== 1842 | HERE ARE THE BUILT IN VARIABLES: 1843 | 1844 | STD-IN (PRIMARY INPUT) 1845 | ------------- 1846 | p line-by-line std-in variable. p represents whatever was 1847 | evaluated to before the previous pipe (|). 1848 | 1849 | pp python list of ALL std-in input. In-place methods like 1850 | sort() will work as well as list methods like sorted(LIST) 1851 | 1852 | SECOND STREAM 1853 | -------------- 1854 | sp line-by-line input second stream input, like p, but from all 1855 | non-flag arguments AFTER pyp command: pyp "p, sp" SP1 SP2 SP3 ... 1856 | 1857 | spp python list of ALL second stream list input. Modifications of 1858 | this list will be picked up with future references to sp 1859 | 1860 | FILE INPUT 1861 | -------------- 1862 | fp line-by-line file input using --text_file TEXT_FILE. fp on 1863 | the first line of output is the first line of the text file 1864 | 1865 | fpp python list of ALL text file input. Modifications of 1866 | this list will be picked up with future references to fp 1867 | 1868 | 1869 | COMMON VARIABLES 1870 | ---------------- 1871 | original original line by line input to pyp 1872 | o same as original 1873 | 1874 | quote a literal " (double quotes can't be used in a pyp expression) 1875 | paran a literal ' 1876 | dollar a literal $ 1877 | 1878 | n line counter (1st line is 0, 2nd line is 1,...use the form "(n+3)" 1879 | to modify this value. n changes to reflect filtering and list ops. 1880 | nk n + 1000 1881 | 1882 | date date and time. Returns the current datetime.datetime.now() object. 1883 | pwd present working directory 1884 | 1885 | history history array of all previous results: 1886 | so pyp "a|u|s|i|h[-3]" shows eval of s 1887 | h same as history 1888 | 1889 | digits all numbers [0-9] 1890 | letters all upper and lowercase letters (useful when combined with variable n). 1891 | letters[n] will print out "a" on the first line, "b" on the second... 1892 | punctuation all punctuation [!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~] 1893 | 1894 | 1895 | =================================================================================== 1896 | THE FOLLOWING ARE SPLIT OR JOINED BASED ON p BEING A STRING OR AN ARRAY: 1897 | 1898 | s OR slash p split/joined on "/" 1899 | d OR dot p split/joined on "." 1900 | w OR whitespace p split/joined on whitespace (on spaces,tabs,etc) 1901 | u OR underscore p split/joined on '_' 1902 | c OR colon p split/joined on ':' 1903 | mm OR comma p split/joined on ',' 1904 | m OR minus p split/joined on '-' 1905 | a OR all p split on [' '-_=$...] (on "All" metacharacters) 1906 | 1907 | Also, the ORIGINAL INPUT (history[0]) lines are split on delimiters as above, but 1908 | stored in os, od, ow, ou, oc, omm, om and oa as well as oslash, odot, owhitepace, 1909 | ocomma, ominus, and oall''' + Colors.GREEN + ''' 1910 | 1911 | =================================================================================== 1912 | HERE ARE THE BUILT IN FUNCTIONS AND ATTRIBUTES: 1913 | 1914 | Function Notes 1915 | -------------------------------------------------------------------------------- 1916 | STRING (all python STRING methods like p.replace(STRING1,STRING2) work) 1917 | -------------------------------------------------------------------------------- 1918 | p.digits() returns a list of contiguous numbers present in p 1919 | p.letters() returns a list of contiguous letters present in p 1920 | p.punctuation() returns a list of contiguous punctuation present in p 1921 | 1922 | p.trim(delimiter) removes last field from string based on delimiter 1923 | with the default being "/" 1924 | p.kill(STR1,STR2...) removes specified strings 1925 | p.clean(delimeter) removes all metacharacters except for slashes, dots and 1926 | the joining delimeter (default is "_") 1927 | p.re(REGEX) returns portion of string that matches REGEX regular 1928 | expression. works great with p.replace(p.re(REGEX),STR) 1929 | 1930 | p.dir directory of path 1931 | p.file file name of path 1932 | p.ext file extension (jpg, tif, hip, etc) of path 1933 | 1934 | These fuctions will work with all pyp strings eg: p, o, dot[0], p.trim(), etc. 1935 | Strings returned by native python functions (like split()) won't have these 1936 | available, but you can still access them using str(STRING). Basically, 1937 | manually recasting anything using as a str(STRING) will endow them with 1938 | the custom pyp methods and attributes. 1939 | 1940 | -------------------------------------------------------------------------------- 1941 | LIST (all LIST methods like pp.sort(), pp[-1], and pp.reverse() work) 1942 | -------------------------------------------------------------------------------- 1943 | pp.delimit(DELIM) split input on delimiter instead of newlines 1944 | pp.divide(N) consolidates N consecutive lines to 1 line. 1945 | pp.before(STRING, N) searches for STRING, colsolidates N lines BEFORE it to 1946 | the same line. Default N is 1. 1947 | pp.after(STRING, N) searches for STRING, colsolidates N lines AFTER it to 1948 | same line. Default N is 1. 1949 | pp.matrix(STRING, N) returns pp.before(STRING, N) and pp.after(STRING, N). 1950 | Default N is 1. 1951 | pp.oneline(DELIM) combines all list elements to one line with delimiter. 1952 | Default delimeter is space. 1953 | pp.uniq() returns only unique elements 1954 | pp.unlist() breaks up ALL lists up into seperate single lines 1955 | 1956 | pp + [STRING] normal python list addition extends list 1957 | pp + spp + fpp normal python list addition combines several inputs. 1958 | new input will be pp; spp and fpp are unaffected. 1959 | sum(pp), max(pp),... normal python list math works if pp is properly cast 1960 | i.e. all members of pp should be integers or floats. 1961 | 1962 | These functions will also work on file and second stream lists: fpp and spp 1963 | 1964 | 1965 | -------------------------------------------------------------------------------- 1966 | NATIVE PYP FUNCTIONS 1967 | -------------------------------------------------------------------------------- 1968 | keep(STR1,STR2,...) keep all lines that have at least one STRING in them 1969 | k(STR1,STR2,...) shortcut for keep(STR1,STR2,...) 1970 | lose(STR1,STR2,...) lose all lines that have at least one STRING in them 1971 | l(STR1,STR2,...) shortcut for lose(STR1,STR2,...) 1972 | 1973 | rekeep(REGEX) keep all lines that match REGEX regular expression 1974 | rek(REGEX) shortcut for rekeep(REGEX) 1975 | relose(REGEX) lose all lines that match REGEX regular expression 1976 | rel(REGEX) shortcut for relose(REGEX) 1977 | 1978 | shell(SCRIPT) returns output of SCRIPT in a list. 1979 | shelld(SCRIPT,DELIM) returns output of SCRIPT in dictionary key/value seperated 1980 | on ':' (default) or supplied delimeter 1981 | env(ENVIROMENT_VAR) returns value of evironment variable using os.environ.get() 1982 | glob(PATH) returns globed files/directories at PATH. Make sure to use 1983 | '*' wildcard 1984 | str(STR) turns any object into an PypStr object, allowing use 1985 | of custom pyp methods as well as normal string methods. 1986 | 1987 | SIMPLE EXAMPLES: 1988 | =================================================================================== 1989 | pyp "'foo ' + p" ==> "foo" + current line 1990 | pyp "p.replace('x','y') | p + o" ==> current line w/replacement + original line 1991 | pyp "p.split(':')[0]" ==> first field of string split on ':' 1992 | pyp "slash[1:3]" ==> array of fields 1 and 2 of string split on '/' 1993 | pyp "s[1:3]|s" ==> string of above joined with '/' 1994 | ''' + Colors.OFF 1995 | 1996 | 1997 | 1998 | unmodified_config = ''' 1999 | #!/usr/bin/env python 2000 | # This must be saved in same directory as pyp (or be in the python path) 2001 | # make sure to name this PypCustom.py and change permission to 666 2002 | 2003 | import sys 2004 | import os 2005 | 2006 | 2007 | class Colors(object): 2008 | OFF = chr(27) + '[0m' 2009 | RED = chr(27) + '[31m' 2010 | GREEN = chr(27) + '[32m' 2011 | YELLOW = chr(27) + '[33m' 2012 | MAGENTA = chr(27) + '[35m' 2013 | CYAN = chr(27) + '[36m' 2014 | WHITE = chr(27) + '[37m' 2015 | BLUE = chr(27) + '[34m' 2016 | BOLD = chr(27) + '[1m' 2017 | COLORS = [OFF, RED, GREEN, YELLOW, MAGENTA, CYAN, WHITE, BLUE, BOLD] 2018 | 2019 | 2020 | class NoColors(object): 2021 | OFF = '' 2022 | RED = '' 2023 | GREEN ='' 2024 | YELLOW = '' 2025 | MAGENTA = '' 2026 | CYAN = '' 2027 | WHITE ='' 2028 | BLUE = '' 2029 | BOLD = '' 2030 | COLORS = [OFF, RED, GREEN, YELLOW, MAGENTA, CYAN, WHITE, BLUE, BOLD] 2031 | 2032 | 2033 | class PypCustom(object): 2034 | 'modify below paths to set macro paths' 2035 | def __init__(self): 2036 | self.user_macro_path = os.path.expanduser('~')+ '/pyp_user_macros.json' 2037 | self.group_macro_path = os.path.expanduser('~')+ '/pyp_user_macros.json' 2038 | self.custom_execute = False 2039 | 2040 | 2041 | class PowerPipeListCustom(): 2042 | 'this is used for pp functions (list fuctions like sort) that operate on all inputs at once.' 2043 | def __init__(self, *args): 2044 | pass 2045 | 2046 | def test(self): 2047 | print 'test' #pp.test() will print "test" 2048 | 2049 | 2050 | class PypStrCustom(): 2051 | 'this is used for string functions using p and other pyp string variables' 2052 | def __init__(self, *args): 2053 | self.test_attr = 'test attr' 2054 | 2055 | def test(self): 2056 | print 'test' #p.test() will print "test" is p is a str 2057 | 2058 | 2059 | class PypListCustom(): 2060 | def __init__(self, *args): 2061 | pass 2062 | 2063 | def test(self): 2064 | print 'test' #p.test() will print "test" is p is a list broken up from a str 2065 | 2066 | 2067 | class PypFunctionCustom(object): 2068 | 'this is used for custom functions and variables (non-instance)' 2069 | test_var = 'works' 2070 | 2071 | def __init__(self, *args): 2072 | pass 2073 | 2074 | def test(self): 2075 | print 'working func ' + self 2076 | ''' 2077 | 2078 | 2079 | usage = """ 2080 | pyp is a python-centric command line text manipulation tool. It allows you to format, replace, augment 2081 | and otherwise mangle text using standard python syntax with a few golden-oldie tricks from unix commands 2082 | of the past. You can pipe data into pyp or cut and paste text, and then hit ctrl-D to get your input into pyp. 2083 | 2084 | After it's in, you can use the standard repertoire of python commands to modify the text. The key variables 2085 | are "p", which represents EACH LINE of the input as a PYTHON STRING. and "pp", which represents ALL of the 2086 | inputs as a PYTHON ARRAY. 2087 | 2088 | You can pipe data WITHIN a pyp statement using standard unix style pipes ("|"), where "p" now represents the 2089 | evaluation of the python statement before the "|". You can also refer back to the ORIGINAL, unadulterated 2090 | input using the variable "o" or "original" at any time...and the variable "h" or "history" allows you 2091 | to refer back to ANY subresult generated between pipes ("|"). 2092 | 2093 | All pyp statements should be enclosed in double quotes, with single quotes being used to enclose any strings. 2094 | 2095 | echo 'FOO IS AN ' | pyp "p.replace('FOO','THIS') | p + 'EXAMPLE'" 2096 | ==> THIS IS AN EXAMPLE 2097 | 2098 | Splitting texton metacharacters is often critical for picking out particular fields of interest, 2099 | so common SPLITS and JOINS have been assigned variables. For example, "underscore" or "u" will split a string 2100 | to an array based on undercores ("_"), while "underscore" or "u" will ALSO join an array with underscores ("_") 2101 | back to a string. 2102 | 2103 | Here are a few key split/join variables; run with --manual for all variable and see examples below in the string section. 2104 | 2105 | s OR slash splits AND joins on "/" 2106 | u OR underscore splits AND joins on "_" 2107 | w OR whitespace splits on whitespace (spaces,tabs,etc) AND joins with spaces 2108 | a OR all splits on ALL metacharacters [!@#$%^&*()...] AND joins with spaces 2109 | 2110 | EXAMPLES: 2111 | ------------------------------------------------------------------------------ 2112 | List Operations # all python list methods work 2113 | ------------------------------------------------------------------------------ 2114 | print all lines ==> pyp "pp" 2115 | sort all input lines ==> pyp "pp.sort()" 2116 | eliminate duplicates ==> pyp "pp.uniq()" 2117 | combine all lines to one line ==> pyp "pp.oneline()" 2118 | print line after FOO ==> pyp "pp.after('FOO')" 2119 | list comprehenision ==> pyp "[x for x in pp]" 2120 | return to string context after sort ==> pyp "pp.sort() | p" 2121 | 2122 | ------------------------------------------------------------------------------- 2123 | String Operations # all python str methods work 2124 | ------------------------------------------------------------------------------- 2125 | print line ==> pyp "p" 2126 | combine line with FOO ==> pyp "p +'FOO'" 2127 | above, but combine with original input ==> pyp "p +'FOO'| p + o" 2128 | 2129 | replace FOO with GOO ==> pyp "p.replace('FOO','GOO')" 2130 | remove all GOO and FOO ==> pyp "p.kill('GOO','FOO')" 2131 | 2132 | string substitution ==> pyp "'%s FOO %s %s GOO'%(p,p,5)" 2133 | 2134 | split up line by FOO ==> pyp "p.split('FOO')" 2135 | split up line by '/' ==> pyp "slash" 2136 | select 1st field split up by '/' ==> pyp "slash[0]" 2137 | select fields 3 through 5 split up by '/' ==> pyp "s[2:6]" 2138 | above joined together with '/' ==> pyp "s[2:6] | s" 2139 | 2140 | ------------------------------------------------------------------------------- 2141 | Logic Filters # all python Booleon methods work 2142 | ------------------------------------------------------------------------------- 2143 | keep all lines with GOO and FOO ==> pyp "'GOO' in p and 'FOO' in p" 2144 | keep all lines with GOO or FOO ==> pyp "keep('GOO','FOO')" 2145 | keep all lines that are numbers ==> pyp "p.isdigit()" 2146 | 2147 | lose all lines with GOO and FOO ==> pyp "'GOO' not in p and 'FOO' not in p" 2148 | lose all lines with GOO or FOO ==> pyp "lose('GOO','FOO')" 2149 | lose all lines that are numbers ==> pyp "not p.isdigit()" 2150 | 2151 | ------------------------------------------------------------------------------- 2152 | TO SEE EXTENDED HELP, use --manual 2153 | 2154 | """ 2155 | 2156 | if __name__ == '__main__': 2157 | parser = optparse.OptionParser(Docs.usage) 2158 | 2159 | parser.add_option("-m", "--manual", action='store_true', help="prints out extended help") 2160 | parser.add_option("-l", "--macro_list", action='store_true', help="lists all available macros") 2161 | parser.add_option("-s", "--macro_save", dest='macro_save_name', type='string', help='saves current command as macro. use "#" for adding comments EXAMPLE: pyp -s "great_macro # prints first letter" "p[1]"') 2162 | parser.add_option("-f", "--macro_find", dest='macro_find_name', type='string', help='searches for macros with keyword or user name') 2163 | parser.add_option("-d", "--macro_delete", dest='macro_delete_name', type='string', help='deletes specified public macro') 2164 | parser.add_option("-g", "--macro_group", action='store_true', help="specify group macros for save and delete; default is user") 2165 | parser.add_option("-t", "--text_file", type='string', help="specify text file to load. for advanced users, you should typically cat a file into pyp") 2166 | parser.add_option("-x", "--execute", action='store_true', help="execute all commands.") 2167 | parser.add_option("-c", "--turn_off_color", action='store_true', help="prints raw, uncolored output") 2168 | parser.add_option("-u", "--unmodified_config", action='store_true', help="prints out generic PypCustom.py config file") 2169 | parser.add_option("-b", "--blank_inputs", action='store', type='string', help="generate this number of blank input lines; useful for generating numbered lists with variable 'n'") 2170 | parser.add_option("-n", "--no_input", action='store_true', help="use with command that generates output with no input; same as --dummy_input 1") 2171 | parser.add_option("-k", "--keep_false", action='store_true', help="print blank lines for lines that test as False. default is to filter out False lines from the output") 2172 | parser.add_option("-r", "--rerun", action="store_true", help="rerun based on automatically cached data from the last run. use this after executing \"pyp\", pasting input into the shell, and hitting CTRL-D") 2173 | 2174 | (options, args) = parser.parse_args() 2175 | 2176 | if options.turn_off_color or options.execute: # overall color switch asap. 2177 | Colors = NoColors 2178 | 2179 | try: 2180 | pyp = Pyp().main() 2181 | except Exception, err: 2182 | print Colors.RED + str(err) + Colors.OFF 2183 | 2184 | 2185 | 2186 | --------------------------------------------------------------------------------