├── .gitignore ├── .pc ├── .quilt_patches ├── .quilt_series ├── .version ├── applied-patches ├── ayrton_customization.diff │ ├── .timestamp │ └── ayrton │ │ └── parser │ │ └── astcompiler │ │ └── astbuilder.py └── pypy_py35-python36.diff │ ├── .timestamp │ └── ayrton │ └── parser │ ├── astcompiler │ ├── assemble.py │ ├── astbuilder.py │ ├── fstring.py │ ├── misc.py │ └── tools │ │ └── asdl_py.py │ ├── error.py │ ├── pyparser │ ├── automata.py │ ├── dfa_generated.py │ ├── future.py │ ├── gendfa.py │ ├── metaparser.py │ ├── parsestring.py │ ├── pygram.py │ ├── pylexer.py │ ├── pyparse.py │ ├── pytokenize.py │ └── pytokenizer.py │ └── unicodehelper.py ├── ChangeLog.rst ├── LICENSE.txt ├── MANIFEST.in ├── Makefile ├── NEWS.rst ├── README.md ├── TODO.rst ├── ayrton ├── __init__.py ├── ast_pprinter.py ├── castt.py ├── execute.py ├── expansion.py ├── file_test.py ├── functions.py ├── importer.py ├── logger.py ├── parser │ ├── __init__.py │ ├── astcompiler │ │ ├── __init__.py │ │ ├── astbuilder.py │ │ ├── consts.py │ │ ├── fstring.py │ │ ├── misc.py │ │ └── tools │ │ │ ├── Python.asdl │ │ │ ├── asdl.py │ │ │ ├── asdl_py.py │ │ │ └── spark.py │ ├── debug.py │ ├── error.py │ └── pyparser │ │ ├── __init__.py │ │ ├── automata.py │ │ ├── data │ │ ├── Grammar2.5 │ │ ├── Grammar2.7 │ │ ├── Grammar3.2 │ │ ├── Grammar3.3 │ │ ├── Grammar3.5 │ │ └── Grammar3.6 │ │ ├── dfa_generated.py │ │ ├── error.py │ │ ├── future.py │ │ ├── gendfa.py │ │ ├── metaparser.py │ │ ├── parser.py │ │ ├── pygram.py │ │ ├── pylexer.py │ │ ├── pyparse.py │ │ ├── pytoken.py │ │ ├── pytokenize.py │ │ └── pytokenizer.py ├── remote.py ├── tests │ ├── __init__.py │ ├── data │ │ ├── string_stdin.txt │ │ ├── test.me.py │ │ └── test_environ.sh │ ├── scripts │ │ ├── package │ │ │ ├── __init__.ay │ │ │ ├── ay_module.ay │ │ │ └── py_module.py │ │ ├── stderr.sh │ │ ├── testAbsoluteExecutables.ay │ │ ├── testAssign.ay │ │ ├── testBg.ay │ │ ├── testCallFun.ay │ │ ├── testCommandIsPresent.ay │ │ ├── testComposing.ay │ │ ├── testCwdPwdRename.ay │ │ ├── testDashInExecutables.ay │ │ ├── testDefFun1.ay │ │ ├── testDefFun2.ay │ │ ├── testDefFunFails1.ay │ │ ├── testDefine.ay │ │ ├── testDefineAgain.ay │ │ ├── testDefineValue.ay │ │ ├── testDel.ay │ │ ├── testDotInExecutables.ay │ │ ├── testEnvVarAsGlobalVar.ay │ │ ├── testEnviron.ay │ │ ├── testExit0.ay │ │ ├── testExit1.ay │ │ ├── testExitCodeNOK.ay │ │ ├── testExitCodeOK.ay │ │ ├── testExportSetsGlobalVar.ay │ │ ├── testFails.ay │ │ ├── testForDefinesTarget.ay │ │ ├── testForDefinesTargets.ay │ │ ├── testFromImport.ay │ │ ├── testFromImportAs.ay │ │ ├── testFromImportAsFails.ay │ │ ├── testFromImportFails.ay │ │ ├── testGt.ay │ │ ├── testGtToUnwritableFile.ay │ │ ├── testImport.ay │ │ ├── testImportCallFromFunc.ay │ │ ├── testImportFrom.ay │ │ ├── testImportFromCallFromFunc.ay │ │ ├── testImportLocalAy.ay │ │ ├── testImportLocalAyModule.ay │ │ ├── testImportLocalAyPackage.ay │ │ ├── testImportLocalAyPackageAyModule.ay │ │ ├── testImportLocalAyPackagePyModule.ay │ │ ├── testImportLocalPy.ay │ │ ├── testImportLocalPyModule.py │ │ ├── testInstantiateClass.ay │ │ ├── testLocalVarToRealRemoteToLocal.ay │ │ ├── testLongIter.ay │ │ ├── testLongOutput.ay │ │ ├── testLongPipe.ay │ │ ├── testLt.ay │ │ ├── testLtGt.ay │ │ ├── testNotExecuteLiteralMethods.ay │ │ ├── testO.ay │ │ ├── testOptionETrue.ay │ │ ├── testOptionErrexit.ay │ │ ├── testOptionMinus_e.ay │ │ ├── testOptionPlus_e.ay │ │ ├── testPipe.ay │ │ ├── testRShift.ay │ │ ├── testRelativeExecutables.ay │ │ ├── testRemoteCommandStderr.ay │ │ ├── testRemoteCommandStdin.ay │ │ ├── testRemoteCommandStdout.ay │ │ ├── testShift.ay │ │ ├── testShiftArray.ay │ │ ├── testShifts.ay │ │ ├── testShortIter.ay │ │ ├── testSimpleCase.ay │ │ ├── testSimpleCaseFails.ay │ │ ├── testStdout.ay │ │ ├── testStdoutEqCapture.ay │ │ ├── testStdoutEqNone.ay │ │ ├── testUnset.ay │ │ ├── testWithCd.ay │ │ ├── testZEmptyString.ay │ │ ├── testZEnvVar.ay │ │ ├── testZInt.ay │ │ ├── testZNone.ay │ │ ├── testZNotDefined.ay │ │ └── testZString.ay │ ├── test_ayrton.py │ ├── test_castt.py │ ├── test_execute.py │ ├── test_expansion.py │ ├── test_file_tests.py │ ├── test_pyparser.py │ └── test_remote.py └── utils.py ├── bash-to-ayrton.md ├── bin └── ayrton ├── contrib └── kate │ └── ayrton.xml ├── doc ├── Makefile ├── examples │ ├── arc.ay │ ├── argv.ay │ ├── capture.ay │ ├── envvars.ay │ ├── hello_world.ay │ ├── with_cd.ay │ └── with_remote.ay ├── idea.txt └── source │ ├── conf.py │ ├── index.rst │ └── reference.rst ├── examples └── backup.ay ├── import_pypy_parser.sh ├── patches ├── ayrton_customization.diff ├── changes.sed ├── pypy_py35-python36.diff └── series ├── release.ay ├── requirements.txt ├── run_test.sh ├── setup.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | build 3 | Attic 4 | dist 5 | MANIFEST 6 | trash & toys 7 | .merge_message 8 | *.pyc 9 | *.pyo 10 | .tox 11 | rsa_server_key* 12 | -------------------------------------------------------------------------------- /.pc/.quilt_patches: -------------------------------------------------------------------------------- 1 | patches 2 | -------------------------------------------------------------------------------- /.pc/.quilt_series: -------------------------------------------------------------------------------- 1 | series 2 | -------------------------------------------------------------------------------- /.pc/.version: -------------------------------------------------------------------------------- 1 | 2 2 | -------------------------------------------------------------------------------- /.pc/applied-patches: -------------------------------------------------------------------------------- 1 | pypy_py35-python36.diff 2 | ayrton_customization.diff 3 | -------------------------------------------------------------------------------- /.pc/ayrton_customization.diff/.timestamp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StyXman/ayrton/97cc1cfcbc1bda2a7fd2f09f6f368444b08a0627/.pc/ayrton_customization.diff/.timestamp -------------------------------------------------------------------------------- /.pc/pypy_py35-python36.diff/.timestamp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StyXman/ayrton/97cc1cfcbc1bda2a7fd2f09f6f368444b08a0627/.pc/pypy_py35-python36.diff/.timestamp -------------------------------------------------------------------------------- /.pc/pypy_py35-python36.diff/ayrton/parser/astcompiler/misc.py: -------------------------------------------------------------------------------- 1 | from pypy.interpreter import gateway 2 | from rpython.rlib.objectmodel import we_are_translated 3 | from rpython.rlib.unroll import unrolling_iterable 4 | 5 | 6 | app = gateway.applevel(""" 7 | def syntax_warning(msg, fn, lineno, offset): 8 | import warnings 9 | try: 10 | warnings.warn_explicit(msg, SyntaxWarning, fn, lineno) 11 | except SyntaxWarning: 12 | raise SyntaxError(msg, (fn, lineno, offset, msg)) 13 | """, filename=__file__) 14 | _emit_syntax_warning = app.interphook("syntax_warning") 15 | del app 16 | 17 | def syntax_warning(space, msg, fn, lineno, offset): 18 | """Raise an applevel SyntaxWarning. 19 | 20 | If the user has set this warning to raise an error, a SyntaxError will be 21 | raised.""" 22 | w_msg = space.newtext(msg) 23 | w_filename = space.newfilename(fn) 24 | w_lineno = space.newint(lineno) 25 | w_offset = space.newint(offset) 26 | _emit_syntax_warning(space, w_msg, w_filename, w_lineno, w_offset) 27 | 28 | 29 | def parse_future(tree, feature_flags): 30 | from pypy.interpreter.astcompiler import ast 31 | future_lineno = 0 32 | future_column = 0 33 | flags = 0 34 | have_docstring = False 35 | body = None 36 | if isinstance(tree, ast.Module): 37 | body = tree.body 38 | elif isinstance(tree, ast.Interactive): 39 | body = tree.body 40 | if body is None: 41 | return 0, 0, 0 42 | for stmt in body: 43 | if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Str): 44 | if have_docstring: 45 | break 46 | else: 47 | have_docstring = True 48 | elif isinstance(stmt, ast.ImportFrom): 49 | if stmt.module == "__future__": 50 | future_lineno = stmt.lineno 51 | future_column = stmt.col_offset 52 | for alias in stmt.names: 53 | assert isinstance(alias, ast.alias) 54 | # If this is an invalid flag, it will be caught later in 55 | # codegen.py. 56 | flags |= feature_flags.get(alias.name, 0) 57 | else: 58 | break 59 | else: 60 | break 61 | return flags, future_lineno, future_column 62 | 63 | 64 | class ForbiddenNameAssignment(Exception): 65 | 66 | def __init__(self, name, node): 67 | self.name = name 68 | self.node = node 69 | 70 | 71 | def check_forbidden_name(name, node=None): 72 | """Raise an error if the name cannot be assigned to.""" 73 | if name in ("None", "__debug__"): 74 | raise ForbiddenNameAssignment(name, node) 75 | # XXX Warn about using True and False 76 | 77 | 78 | def dict_to_switch(d): 79 | """Convert of dictionary with integer keys to a switch statement.""" 80 | def lookup(query): 81 | if we_are_translated(): 82 | for key, value in unrolling_iteritems: 83 | if key == query: 84 | return value 85 | else: 86 | raise KeyError 87 | else: 88 | return d[query] 89 | lookup._always_inline_ = True 90 | unrolling_iteritems = unrolling_iterable(d.iteritems()) 91 | return lookup 92 | 93 | 94 | def mangle(name, klass): 95 | if not name.startswith('__'): 96 | return name 97 | # Don't mangle __id__ or names with dots. The only time a name with a dot 98 | # can occur is when we are compiling an import statement that has a package 99 | # name. 100 | if name.endswith('__') or '.' in name: 101 | return name 102 | try: 103 | i = 0 104 | while klass[i] == '_': 105 | i = i + 1 106 | except IndexError: 107 | return name 108 | return "_%s%s" % (klass[i:], name) 109 | 110 | 111 | def intern_if_common_string(space, w_const): 112 | # only intern identifier-like strings 113 | from pypy.objspace.std.unicodeobject import _isidentifier 114 | if (space.is_w(space.type(w_const), space.w_unicode) and 115 | _isidentifier(space.unicode_w(w_const))): 116 | return space.new_interned_w_str(w_const) 117 | return w_const 118 | 119 | 120 | def new_identifier(space, name): 121 | # Check whether there are non-ASCII characters in the identifier; if 122 | # so, normalize to NFKC 123 | for c in name: 124 | if ord(c) > 0x80: 125 | break 126 | else: 127 | return name 128 | 129 | from pypy.module.unicodedata.interp_ucd import ucd 130 | w_name = space.newtext(name) 131 | w_id = space.call_method(ucd, 'normalize', space.newtext('NFKC'), w_name) 132 | return space.text_w(w_id) 133 | -------------------------------------------------------------------------------- /.pc/pypy_py35-python36.diff/ayrton/parser/pyparser/automata.py: -------------------------------------------------------------------------------- 1 | # ______________________________________________________________________ 2 | """Module automata 3 | 4 | THIS FILE WAS COPIED FROM pypy/module/parser/pytokenize.py AND ADAPTED 5 | TO BE ANNOTABLE (Mainly made the DFA's __init__ accept two lists 6 | instead of a unique nested one) 7 | 8 | $Id: automata.py,v 1.2 2003/10/02 17:37:17 jriehl Exp $ 9 | """ 10 | # ______________________________________________________________________ 11 | # Module level definitions 12 | 13 | # PYPY Modification: removed the EMPTY class as it's not needed here 14 | 15 | 16 | # PYPY Modification: DEFAULT is a singleton, used only in the pre-RPython 17 | # dicts (see pytokenize.py). Then DFA.__init__() turns these dicts into 18 | # more compact strings. 19 | DEFAULT = object() 20 | 21 | # PYPY Modification : removed all automata functions (any, maybe, 22 | # newArcPair, etc.) 23 | 24 | ERROR_STATE = chr(255) 25 | 26 | class DFA: 27 | # ____________________________________________________________ 28 | def __init__(self, states, accepts, start = 0): 29 | """ NOT_RPYTHON """ 30 | assert len(states) < 255 # no support for huge amounts of states 31 | # construct string for looking up state transitions 32 | string_states = [] * len(states) 33 | # compute maximum 34 | maximum = 0 35 | for state in states: 36 | for key in state: 37 | if key == DEFAULT: 38 | continue 39 | maximum = max(ord(key), maximum) 40 | self.max_char = maximum + 1 41 | 42 | defaults = [] 43 | for i, state in enumerate(states): 44 | default = ERROR_STATE 45 | if DEFAULT in state: 46 | default = chr(state[DEFAULT]) 47 | defaults.append(default) 48 | string_state = [default] * self.max_char 49 | for key, value in state.iteritems(): 50 | if key == DEFAULT: 51 | continue 52 | assert len(key) == 1 53 | assert ord(key) < self.max_char 54 | string_state[ord(key)] = chr(value) 55 | string_states.extend(string_state) 56 | self.states = "".join(string_states) 57 | self.defaults = "".join(defaults) 58 | self.accepts = accepts 59 | self.start = start 60 | 61 | # ____________________________________________________________ 62 | 63 | def _next_state(self, item, crntState): 64 | if ord(item) >= self.max_char: 65 | return self.defaults[crntState] 66 | else: 67 | return self.states[crntState * self.max_char + ord(item)] 68 | 69 | def recognize(self, inVec, pos = 0): 70 | crntState = self.start 71 | lastAccept = False 72 | i = pos 73 | for i in range(pos, len(inVec)): 74 | item = inVec[i] 75 | if ord(item) > 0x80: 76 | item = "\x80" # NON_ASCII 77 | accept = self.accepts[crntState] 78 | crntState = self._next_state(item, crntState) 79 | if crntState != ERROR_STATE: 80 | pass 81 | elif accept: 82 | return i 83 | elif lastAccept: 84 | # This is now needed b/c of exception cases where there are 85 | # transitions to dead states 86 | return i - 1 87 | else: 88 | return -1 89 | crntState = ord(crntState) 90 | lastAccept = accept 91 | # if self.states[crntState][1]: 92 | if self.accepts[crntState]: 93 | return i + 1 94 | elif lastAccept: 95 | return i 96 | else: 97 | return -1 98 | 99 | # ______________________________________________________________________ 100 | 101 | class NonGreedyDFA (DFA): 102 | 103 | def recognize(self, inVec, pos = 0): 104 | crntState = self.start 105 | i = pos 106 | for i in range(pos, len(inVec)): 107 | item = inVec[i] 108 | accept = self.accepts[crntState] 109 | if accept: 110 | return i 111 | crntState = self._next_state(item, crntState) 112 | if crntState == ERROR_STATE: 113 | return -1 114 | crntState = ord(crntState) 115 | i += 1 116 | if self.accepts[crntState]: 117 | return i 118 | else: 119 | return -1 120 | 121 | # ______________________________________________________________________ 122 | # End of automata.py 123 | -------------------------------------------------------------------------------- /.pc/pypy_py35-python36.diff/ayrton/parser/pyparser/future.py: -------------------------------------------------------------------------------- 1 | from pypy.tool import stdlib___future__ as future 2 | 3 | class FutureFlags(object): 4 | 5 | def __init__(self, version): 6 | compiler_flags = 0 7 | self.compiler_features = {} 8 | self.mandatory_flags = 0 9 | for fname in future.all_feature_names: 10 | feature = getattr(future, fname) 11 | if version >= feature.getOptionalRelease(): 12 | flag = feature.compiler_flag 13 | compiler_flags |= flag 14 | self.compiler_features[fname] = flag 15 | if version >= feature.getMandatoryRelease(): 16 | self.mandatory_flags |= feature.compiler_flag 17 | self.allowed_flags = compiler_flags 18 | 19 | def get_flag_names(self, space, flags): 20 | flag_names = [] 21 | for name, value in self.compiler_features.items(): 22 | if flags & value: 23 | flag_names.append(name) 24 | return flag_names 25 | 26 | def get_compiler_feature(self, name): 27 | return self.compiler_features.get(name, 0) 28 | 29 | futureFlags_2_4 = FutureFlags((2, 4, 4, 'final', 0)) 30 | futureFlags_2_5 = FutureFlags((2, 5, 0, 'final', 0)) 31 | futureFlags_2_7 = FutureFlags((2, 7, 0, 'final', 0)) 32 | futureFlags_3_2 = FutureFlags((3, 2, 0, 'final', 0)) 33 | futureFlags_3_5 = FutureFlags((3, 5, 0, 'final', 0)) 34 | 35 | 36 | class TokenIterator: 37 | def __init__(self, tokens): 38 | self.tokens = tokens 39 | self.index = 0 40 | self.next() 41 | 42 | def next(self): 43 | index = self.index 44 | self.index = index + 1 45 | self.tok = self.tokens[index] 46 | 47 | def skip(self, n): 48 | if self.tok[0] == n: 49 | self.next() 50 | return True 51 | else: 52 | return False 53 | 54 | def skip_name(self, name): 55 | from pypy.interpreter.pyparser import pygram 56 | if self.tok[0] == pygram.tokens.NAME and self.tok[1] == name: 57 | self.next() 58 | return True 59 | else: 60 | return False 61 | 62 | def next_feature_name(self): 63 | from pypy.interpreter.pyparser import pygram 64 | if self.tok[0] == pygram.tokens.NAME: 65 | name = self.tok[1] 66 | self.next() 67 | if self.skip_name("as"): 68 | self.skip(pygram.tokens.NAME) 69 | return name 70 | else: 71 | return '' 72 | 73 | def skip_newlines(self): 74 | from pypy.interpreter.pyparser import pygram 75 | while self.skip(pygram.tokens.NEWLINE): 76 | pass 77 | 78 | 79 | def add_future_flags(future_flags, tokens): 80 | from pypy.interpreter.pyparser import pygram 81 | it = TokenIterator(tokens) 82 | result = 0 83 | last_position = (0, 0) 84 | # 85 | # The only things that can precede a future statement are another 86 | # future statement and a doc string (only one). This is a very 87 | # permissive parsing of the given list of tokens; it relies on 88 | # the real parsing done afterwards to give errors. 89 | it.skip_newlines() 90 | it.skip_name("r") or it.skip_name("u") or it.skip_name("ru") 91 | if it.skip(pygram.tokens.STRING): 92 | it.skip_newlines() 93 | 94 | while (it.skip_name("from") and 95 | it.skip_name("__future__") and 96 | it.skip_name("import")): 97 | it.skip(pygram.tokens.LPAR) # optionally 98 | # return in 'last_position' any line-column pair that points 99 | # somewhere inside the last __future__ import statement 100 | # (at the start would be fine too, but it's easier to grab a 101 | # random position inside) 102 | last_position = (it.tok[2], it.tok[3]) 103 | result |= future_flags.get_compiler_feature(it.next_feature_name()) 104 | while it.skip(pygram.tokens.COMMA): 105 | result |= future_flags.get_compiler_feature(it.next_feature_name()) 106 | it.skip(pygram.tokens.RPAR) # optionally 107 | it.skip(pygram.tokens.SEMI) # optionally 108 | it.skip_newlines() 109 | 110 | # remove the flags that were specified but are anyway mandatory 111 | result &= ~future_flags.mandatory_flags 112 | 113 | return result, last_position 114 | -------------------------------------------------------------------------------- /.pc/pypy_py35-python36.diff/ayrton/parser/pyparser/pygram.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pypy.interpreter.pyparser import parser, pytoken, metaparser 3 | 4 | class PythonGrammar(parser.Grammar): 5 | 6 | KEYWORD_TOKEN = pytoken.python_tokens["NAME"] 7 | TOKENS = pytoken.python_tokens 8 | OPERATOR_MAP = pytoken.python_opmap 9 | 10 | def _get_python_grammar(): 11 | here = os.path.dirname(__file__) 12 | fp = open(os.path.join(here, "data", "Grammar3.5")) 13 | try: 14 | gram_source = fp.read() 15 | finally: 16 | fp.close() 17 | pgen = metaparser.ParserGenerator(gram_source) 18 | return pgen.build_grammar(PythonGrammar) 19 | 20 | 21 | python_grammar = _get_python_grammar() 22 | 23 | class _Tokens(object): 24 | pass 25 | for tok_name, idx in pytoken.python_tokens.iteritems(): 26 | setattr(_Tokens, tok_name, idx) 27 | tokens = _Tokens() 28 | 29 | class _Symbols(object): 30 | pass 31 | rev_lookup = {} 32 | for sym_name, idx in python_grammar.symbol_ids.iteritems(): 33 | setattr(_Symbols, sym_name, idx) 34 | rev_lookup[idx] = sym_name 35 | syms = _Symbols() 36 | syms._rev_lookup = rev_lookup # for debugging 37 | 38 | del _get_python_grammar, _Tokens, tok_name, sym_name, idx 39 | -------------------------------------------------------------------------------- /.pc/pypy_py35-python36.diff/ayrton/parser/pyparser/pytokenize.py: -------------------------------------------------------------------------------- 1 | # ______________________________________________________________________ 2 | """Module pytokenize 3 | 4 | THIS FILE WAS COPIED FROM pypy/module/parser/pytokenize.py AND ADAPTED 5 | TO BE ANNOTABLE (Mainly made lists homogeneous) 6 | 7 | This is a modified version of Ka-Ping Yee's tokenize module found in the 8 | Python standard library. 9 | 10 | The primary modification is the removal of the tokenizer's dependence on the 11 | standard Python regular expression module, which is written in C. The regular 12 | expressions have been replaced with hand built DFA's using the 13 | basil.util.automata module. 14 | 15 | $Id: pytokenize.py,v 1.3 2003/10/03 16:31:53 jriehl Exp $ 16 | """ 17 | # ______________________________________________________________________ 18 | 19 | from pypy.interpreter.pyparser import automata 20 | from pypy.interpreter.pyparser.dfa_generated import * 21 | 22 | __all__ = [ "tokenize" ] 23 | 24 | endDFAs = {"'" : singleDFA, 25 | '"' : doubleDFA, 26 | 'r' : None, 27 | 'R' : None, 28 | "u" : None, 29 | "U" : None, 30 | 'f' : None, 31 | 'F' : None, 32 | 'b' : None, 33 | 'B' : None} 34 | 35 | for uniPrefix in ("", "b", "B", "f", "F"): 36 | for rawPrefix in ("", "r", "R"): 37 | prefix_1 = uniPrefix + rawPrefix 38 | prefix_2 = rawPrefix + uniPrefix 39 | 40 | endDFAs[prefix_1 + "'''"] = single3DFA 41 | endDFAs[prefix_1 + '"""'] = double3DFA 42 | endDFAs[prefix_2 + "'''"] = single3DFA 43 | endDFAs[prefix_2 + '"""'] = double3DFA 44 | 45 | for uniPrefix in ("u", "U"): 46 | endDFAs[uniPrefix + "'''"] = single3DFA 47 | endDFAs[uniPrefix + '"""'] = double3DFA 48 | 49 | whiteSpaceStatesAccepts = [True] 50 | whiteSpaceStates = [{'\t': 0, ' ': 0, '\x0c': 0}] 51 | whiteSpaceDFA = automata.DFA(whiteSpaceStates, whiteSpaceStatesAccepts) 52 | 53 | # ______________________________________________________________________ 54 | # COPIED: 55 | 56 | triple_quoted = {} 57 | for t in ("'''", '"""', 58 | "r'''", 'r"""', "R'''", 'R"""', 59 | "u'''", 'u"""', "U'''", 'U"""', 60 | "f'''", 'f"""', "F'''", 'F"""', 61 | "fr'''", 'fr"""', "Fr'''", 'Fr"""', 62 | "fR'''", 'fR"""', "FR'''", 'FR"""', 63 | "rf'''", 'rf"""', "rF'''", 'rF"""', 64 | "Rf'''", 'Rf"""', "RF'''", 'RF"""', 65 | "b'''", 'b"""', "B'''", 'B"""', 66 | "br'''", 'br"""', "Br'''", 'Br"""', 67 | "bR'''", 'bR"""', "BR'''", 'BR"""', 68 | "rb'''", 'rb"""', "rB'''", 'rB"""', 69 | "Rb'''", 'Rb"""', "RB'''", 'RB"""'): 70 | triple_quoted[t] = t 71 | single_quoted = {} 72 | for t in ("'", '"', 73 | "r'", 'r"', "R'", 'R"', 74 | "u'", 'u"', "U'", 'U"', 75 | "f'", 'f"', "F'", 'F"', 76 | "fr'", 'fr"', "Fr'", 'Fr"', 77 | "fR'", 'fR"', "FR'", 'FR"', 78 | "rf'", 'rf"', "rF'", 'rF"', 79 | "Rf'", 'Rf"', "RF'", 'RF"', 80 | "b'", 'b"', "B'", 'B"', 81 | "br'", 'br"', "Br'", 'Br"', 82 | "bR'", 'bR"', "BR'", 'BR"', 83 | "rb'", 'rb"', "rB'", 'rB"', 84 | "Rb'", 'Rb"', "RB'", 'RB"'): 85 | 86 | single_quoted[t] = t 87 | 88 | tabsize = 8 89 | alttabsize = 1 90 | 91 | # PYPY MODIFICATION: removed TokenError class as it's not needed here 92 | 93 | # PYPY MODIFICATION: removed StopTokenizing class as it's not needed here 94 | 95 | # PYPY MODIFICATION: removed printtoken() as it's not needed here 96 | 97 | # PYPY MODIFICATION: removed tokenize() as it's not needed here 98 | 99 | # PYPY MODIFICATION: removed tokenize_loop() as it's not needed here 100 | 101 | # PYPY MODIFICATION: removed generate_tokens() as it was copied / modified 102 | # in pythonlexer.py 103 | 104 | # PYPY MODIFICATION: removed main() as it's not needed here 105 | 106 | # ______________________________________________________________________ 107 | # End of pytokenize.py 108 | 109 | -------------------------------------------------------------------------------- /.pc/pypy_py35-python36.diff/ayrton/parser/unicodehelper.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pypy.interpreter.error import OperationError, oefmt 3 | from rpython.rlib.objectmodel import specialize 4 | from rpython.rlib import runicode 5 | from pypy.module._codecs import interp_codecs 6 | _WIN32 = sys.platform == 'win32' 7 | _MACOSX = sys.platform == 'darwin' 8 | if _WIN32: 9 | from rpython.rlib.runicode import str_decode_mbcs, unicode_encode_mbcs 10 | else: 11 | # Workaround translator's confusion 12 | str_decode_mbcs = unicode_encode_mbcs = lambda *args, **kwargs: None 13 | 14 | @specialize.memo() 15 | def decode_error_handler(space): 16 | # Fast version of the "strict" errors handler. 17 | def raise_unicode_exception_decode(errors, encoding, msg, s, 18 | startingpos, endingpos): 19 | raise OperationError(space.w_UnicodeDecodeError, 20 | space.newtuple([space.wrap(encoding), 21 | space.newbytes(s), 22 | space.wrap(startingpos), 23 | space.wrap(endingpos), 24 | space.wrap(msg)])) 25 | return raise_unicode_exception_decode 26 | 27 | @specialize.memo() 28 | def encode_error_handler(space): 29 | # Fast version of the "strict" errors handler. 30 | def raise_unicode_exception_encode(errors, encoding, msg, u, 31 | startingpos, endingpos): 32 | raise OperationError(space.w_UnicodeEncodeError, 33 | space.newtuple([space.wrap(encoding), 34 | space.wrap(u), 35 | space.wrap(startingpos), 36 | space.wrap(endingpos), 37 | space.wrap(msg)])) 38 | return raise_unicode_exception_encode 39 | 40 | class RUnicodeEncodeError(Exception): 41 | def __init__(self, encoding, object, start, end, reason): 42 | self.encoding = encoding 43 | self.object = object 44 | self.start = start 45 | self.end = end 46 | self.reason = reason 47 | 48 | @specialize.memo() 49 | def rpy_encode_error_handler(): 50 | # A RPython version of the "strict" error handler. 51 | def raise_unicode_exception_encode(errors, encoding, msg, u, 52 | startingpos, endingpos): 53 | raise RUnicodeEncodeError(encoding, u, startingpos, endingpos, msg) 54 | return raise_unicode_exception_encode 55 | 56 | # ____________________________________________________________ 57 | 58 | def fsdecode(space, w_string): 59 | state = space.fromcache(interp_codecs.CodecState) 60 | if _WIN32: 61 | bytes = space.bytes_w(w_string) 62 | uni = str_decode_mbcs(bytes, len(bytes), 'strict', 63 | errorhandler=decode_error_handler(space), 64 | force_ignore=False)[0] 65 | elif _MACOSX: 66 | bytes = space.bytes_w(w_string) 67 | uni = runicode.str_decode_utf_8( 68 | bytes, len(bytes), 'surrogateescape', 69 | errorhandler=state.decode_error_handler)[0] 70 | elif space.sys.filesystemencoding is None or state.codec_need_encodings: 71 | # bootstrap check: if the filesystemencoding isn't initialized 72 | # or the filesystem codec is implemented in Python we cannot 73 | # use it before the codecs are ready. use the locale codec 74 | # instead 75 | from pypy.module._codecs.locale import ( 76 | str_decode_locale_surrogateescape) 77 | bytes = space.bytes_w(w_string) 78 | uni = str_decode_locale_surrogateescape( 79 | bytes, errorhandler=decode_error_handler(space)) 80 | else: 81 | from pypy.module.sys.interp_encoding import getfilesystemencoding 82 | return space.call_method(w_string, 'decode', 83 | getfilesystemencoding(space), 84 | space.wrap('surrogateescape')) 85 | return space.wrap(uni) 86 | 87 | def fsencode(space, w_uni): 88 | state = space.fromcache(interp_codecs.CodecState) 89 | if _WIN32: 90 | uni = space.unicode_w(w_uni) 91 | bytes = unicode_encode_mbcs(uni, len(uni), 'strict', 92 | errorhandler=encode_error_handler(space), 93 | force_replace=False) 94 | elif _MACOSX: 95 | uni = space.unicode_w(w_uni) 96 | bytes = runicode.unicode_encode_utf_8( 97 | uni, len(uni), 'surrogateescape', 98 | errorhandler=state.encode_error_handler) 99 | elif space.sys.filesystemencoding is None or state.codec_need_encodings: 100 | # bootstrap check: if the filesystemencoding isn't initialized 101 | # or the filesystem codec is implemented in Python we cannot 102 | # use it before the codecs are ready. use the locale codec 103 | # instead 104 | from pypy.module._codecs.locale import ( 105 | unicode_encode_locale_surrogateescape) 106 | uni = space.unicode_w(w_uni) 107 | if u'\x00' in uni: 108 | raise oefmt(space.w_ValueError, "embedded null character") 109 | bytes = unicode_encode_locale_surrogateescape( 110 | uni, errorhandler=encode_error_handler(space)) 111 | else: 112 | from pypy.module.sys.interp_encoding import getfilesystemencoding 113 | return space.call_method(w_uni, 'encode', 114 | getfilesystemencoding(space), 115 | space.wrap('surrogateescape')) 116 | return space.newbytes(bytes) 117 | 118 | def encode(space, w_data, encoding=None, errors='strict'): 119 | from pypy.objspace.std.unicodeobject import encode_object 120 | return encode_object(space, w_data, encoding, errors) 121 | 122 | # These functions take and return unwrapped rpython strings and unicodes 123 | def decode_unicode_escape(space, string): 124 | state = space.fromcache(interp_codecs.CodecState) 125 | unicodedata_handler = state.get_unicodedata_handler(space) 126 | result, consumed = runicode.str_decode_unicode_escape( 127 | string, len(string), "strict", 128 | final=True, errorhandler=decode_error_handler(space), 129 | unicodedata_handler=unicodedata_handler) 130 | return result 131 | 132 | def decode_raw_unicode_escape(space, string): 133 | result, consumed = runicode.str_decode_raw_unicode_escape( 134 | string, len(string), "strict", 135 | final=True, errorhandler=decode_error_handler(space)) 136 | return result 137 | 138 | def decode_utf8(space, string, allow_surrogates=False): 139 | # Note that Python3 tends to forbid *all* surrogates in utf-8. 140 | # If allow_surrogates=True, then revert to the Python 2 behavior, 141 | # i.e. surrogates are accepted and not treated specially at all. 142 | # If there happen to be two 3-bytes encoding a pair of surrogates, 143 | # you still get two surrogate unicode characters in the result. 144 | result, consumed = runicode.str_decode_utf_8( 145 | string, len(string), "strict", 146 | final=True, errorhandler=decode_error_handler(space), 147 | allow_surrogates=allow_surrogates) 148 | return result 149 | 150 | def encode_utf8(space, uni, allow_surrogates=False): 151 | # Note that Python3 tends to forbid *all* surrogates in utf-8. 152 | # If allow_surrogates=True, then revert to the Python 2 behavior 153 | # which never raises UnicodeEncodeError. Surrogate pairs are then 154 | # allowed, either paired or lone. A paired surrogate is considered 155 | # like the non-BMP character it stands for. See also unicode_utf8sp(). 156 | return runicode.unicode_encode_utf_8( 157 | uni, len(uni), "strict", 158 | errorhandler=encode_error_handler(space), 159 | allow_surrogates=allow_surrogates) 160 | 161 | def encode_utf8sp(space, uni): 162 | # Surrogate-preserving utf-8 encoding. Any surrogate character 163 | # turns into its 3-bytes encoding, whether it is paired or not. 164 | # This should always be reversible, and the reverse is 165 | # decode_utf8sp(). 166 | return runicode.unicode_encode_utf8sp(uni, len(uni)) 167 | 168 | def decode_utf8sp(space, string): 169 | # Surrogate-preserving utf-8 decoding. Assuming there is no 170 | # encoding error, it should always be reversible, and the reverse is 171 | # encode_utf8sp(). 172 | return decode_utf8(space, string, allow_surrogates=True) 173 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include Makefile 2 | include README.md 3 | include TODO.rst 4 | include LICENSE.txt 5 | include MANIFEST.in 6 | 7 | recursive-include contrib * 8 | recursive-include doc *.txt Makefile 9 | recursive-include doc/source * 10 | recursive-include doc/examples *.ay 11 | 12 | exclude doc/examples/with_ssh.ay 13 | exclude doc/examples/pipe.ay 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | DEBUG_MULTI=strace -tt -T -ff -o debug/runner -s 128 2 | DEBUG_SIMPLE=strace -tt -T -o debug/runner -s 128 3 | PYTHON=python3.6 4 | RUNNER=$(PYTHON) 5 | # no python3.6-coverage yet :( 6 | # RUNNER=$(PYTHON)-coverage run 7 | # can't use --buffer because: 8 | # File "/home/mdione/src/projects/ayrton/ayrton/__init__.py", line 191, in polute 9 | # self[std]= getattr (sys, std).buffer 10 | # AttributeError: '_io.StringIO' object has no attribute 'buffer' 11 | UNITTEST_OPTS=--verbose 12 | 13 | all: docs 14 | 15 | INSTALL_DIR=$(HOME)/local 16 | 17 | tests: 18 | LC_ALL=C $(RUNNER) -m unittest discover $(UNITTEST_OPTS) ayrton 19 | 20 | slowtest: debug 21 | # LC_ALL=C $(DEBUG_SIMPLE) $(RUNNER) -m unittest discover --failfast \ 22 | # $(UNITTEST_OPTS) ayrton 23 | LC_ALL=C $(DEBUG_MULTI) $(RUNNER) -m unittest discover --failfast \ 24 | $(UNITTEST_OPTS) ayrton 25 | 26 | quicktest: 27 | LC_ALL=C $(RUNNER) -m unittest discover --failfast $(UNITTEST_OPTS) ayrton 28 | 29 | docs: 30 | RUNNERPATH=${PWD} make -C doc html 31 | 32 | install: tests 33 | $(PYTHON) setup.py install --prefix=$(INSTALL_DIR) 34 | 35 | unsafe-install: 36 | @echo "unsafe install, are you sure?" 37 | @read foo 38 | $(PYTHON) setup.py install --prefix=$(INSTALL_DIR) 39 | 40 | upload: tests upload-docs 41 | $(PYTHON) setup.py sdist upload 42 | 43 | upload-docs: docs 44 | rsync --archive --verbose --compress --rsh ssh doc/build/html/ www.grulic.org.ar:www/projects/ayrton/ 45 | 46 | push: tests 47 | git push 48 | 49 | check: 50 | flake8 --ignore E201,E211,E225,E221,E226,E202 --show-source --statistics --max-line-length 130 ayrton/*.py 51 | 52 | testclean: 53 | rm -f ayrton.*log debug/runner* debug/remote* *.ayrtmp 54 | 55 | debug: 56 | mkdir -pv debug 57 | 58 | rsa_server_key: 59 | # generate a rsa server key 60 | ssh-keygen -f rsa_server_key -N '' -t rsa; \ 61 | 62 | debugserver: rsa_server_key 63 | # TODO: discover sshd's path? 64 | # sshd re-exec requires execution with an absolute path 65 | /usr/sbin/sshd -dd -e -h $(shell pwd)/rsa_server_key -p 2244 66 | 67 | covreport: 68 | $(PYTHON)-coverage report -m | grep ayrton | egrep -v '/(parser|tests)/' 69 | -------------------------------------------------------------------------------- /NEWS.rst: -------------------------------------------------------------------------------- 1 | ayrton (0.4) UNRELEASED; urgency=low 2 | 3 | * bash() now returns strings if the result would be a list with only 4 | one string. 5 | 6 | -- Marcos Dione Mon, 28 Oct 2013 17:40:49 -0300 7 | 8 | ayrton (0.2) unstable; urgency=low 9 | 10 | * The `ssh()` context manager was renamed to `remote()` so the `ssh` 11 | executable is stills reachable from code. This was due to the fact 12 | that `ssh` is too complex to mimic. 13 | 14 | -- Marcos Dione Sat, 14 Sep 2013 17:59:27 +0200 15 | -------------------------------------------------------------------------------- /TODO.rst: -------------------------------------------------------------------------------- 1 | Even when most of these are in the github issues, I leave it here so I can 2 | attack it offline (when I code best). 3 | 4 | Really do: 5 | ---------- 6 | 7 | * better error reporting, including remotes 8 | 9 | * interface so local and remote can easily setup more communication channels 10 | 11 | * we're weeding out imports, we could gather a list and reimport them in the 12 | remote 13 | 14 | * from foo import bar is gonna break 15 | 16 | * imported ayrton scripts should be parsed with the ayrton parser. 17 | 18 | * becareful with if cat () | grep (); error codes must be carried too 19 | 20 | * exit code of last Command should be used as return code of functions 21 | 22 | * process substitution 23 | 24 | * https://github.com/amoffat/sh/issues/66 25 | 26 | * becareful with buitins, might eclipse valid usages: bash() (exp) blocks /bin/bash 27 | * rename bash() to expand() 28 | * add option _exec=True for actually executing the binary. 29 | 30 | * check ``bash``'s manpage and see what's missing. 31 | * subprocess 32 | 33 | * with ayrton (): ... 34 | 35 | * a setting for making references to unkown envvars as in bash. 36 | * trap? 37 | * executable path caching à la bash. 38 | * with parallel(2): 39 | .... 40 | 41 | Think deeply about: 42 | ------------------- 43 | 44 | * what to do about relative/absolute command paths? 45 | * git (commit) vs git.commit() vs git ('commit') 46 | * function names are expressions too: 47 | * / as unary op? => /path/to/excecutable and relative/path 48 | * foo_bar vs foo__bar vs foo-bar 49 | * -f vs (-)f vs _f 50 | * commands in keywords should also be _out=Capture 51 | * which is the sanest default, bash (..., single=True) or otherwise 52 | 53 | * exec &> ./create.log 54 | 55 | * implement -m so we can run ayrton -x -m foo 56 | 57 | * check whether we can replace a lot of code with Popen/subprocess 58 | 59 | * convert tests to function calls 60 | 61 | * -P=True, ok; -P=False? 62 | * format strings -> bash expansion 63 | * s/bash/expand/ 64 | 65 | * case/matching? 66 | 67 | If we have the time: 68 | -------------------- 69 | 70 | * ast.Name('x', ast.Load(), lineno=1, col_offset=6) 71 | -------------------------------------------------------------------------------- /ayrton/file_test.py: -------------------------------------------------------------------------------- 1 | # (c) 2013 Marcos Dione 2 | 3 | # This file is part of ayrton. 4 | # 5 | # ayrton is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # ayrton is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with ayrton. If not, see . 17 | 18 | import os 19 | import stat 20 | import os.path 21 | 22 | # these functions imitate the -* tests from [ (as per bash's man page) 23 | 24 | def simple_stat (fname): 25 | try: 26 | return os.stat (fname) 27 | except (IOError, OSError): 28 | return None 29 | 30 | 31 | class FalseBool: 32 | """This class is needed so file test X() can be called -X() and not break 33 | the semantics. The problem is that -True==-1 and -False==0. Also: 34 | TypeError: type 'bool' is not an acceptable base type.""" 35 | 36 | def __init__ (self, value): 37 | if not isinstance (value, bool): 38 | raise ValueError 39 | 40 | self.value= value 41 | 42 | def __bool__ (self): 43 | return self.value 44 | 45 | def __neg__ (self): 46 | return self.value 47 | 48 | 49 | def a (fname): 50 | return FalseBool (simple_stat (fname) is not None) 51 | 52 | def b (fname): 53 | s= simple_stat (fname) 54 | return FalseBool (s is not None and stat.S_ISBLK (s.st_mode)) 55 | 56 | def c (fname): 57 | s= simple_stat (fname) 58 | return FalseBool (s is not None and stat.S_ISCHR (s.st_mode)) 59 | 60 | def d (fname): 61 | s= simple_stat (fname) 62 | return FalseBool (s is not None and stat.S_ISDIR (s.st_mode)) 63 | 64 | # both return the same thing! 65 | e= a 66 | 67 | def f (fname): 68 | s= simple_stat (fname) 69 | return FalseBool (s is not None and stat.S_ISREG (s.st_mode)) 70 | 71 | def g (fname): 72 | s= simple_stat (fname) 73 | return FalseBool (s is not None and (stat.S_IMODE (s.st_mode) & stat.S_ISGID)!=0) 74 | 75 | def h (fname): 76 | return FalseBool (os.path.islink (fname)) 77 | 78 | def k (fname): 79 | s= simple_stat (fname) 80 | # VTX?!? 81 | return FalseBool (s is not None and (stat.S_IMODE (s.st_mode) & stat.S_ISVTX)!=0) 82 | 83 | def p (fname): 84 | s= simple_stat (fname) 85 | return FalseBool (s is not None and stat.S_ISFIFO (s.st_mode)) 86 | 87 | def r (fname): 88 | s= simple_stat (fname) 89 | return FalseBool (s is not None and (stat.S_IMODE (s.st_mode) & 90 | (stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH))!=0) 91 | 92 | def s (fname): 93 | s= simple_stat (fname) 94 | return FalseBool (s is not None and s.st_size>0) 95 | 96 | # TODO: t 97 | # os.isatty(fd) 98 | 99 | def u (fname): 100 | s= simple_stat (fname) 101 | return FalseBool (s is not None and (stat.S_IMODE (s.st_mode) & stat.S_ISUID)!=0) 102 | 103 | def w (fname): 104 | s= simple_stat (fname) 105 | return FalseBool (s is not None and (stat.S_IMODE (s.st_mode) & 106 | (stat.S_IWUSR|stat.S_IWGRP|stat.S_IWOTH))!=0) 107 | 108 | def x (fname): 109 | s= simple_stat (fname) 110 | return FalseBool (s is not None and (stat.S_IMODE (s.st_mode) & 111 | (stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH))!=0) 112 | 113 | # TODO: G, O 114 | 115 | L= h 116 | 117 | def N (fname): 118 | s= simple_stat (fname) 119 | return FalseBool (s is not None and s.st_mtime_ns > s.st_atime_ns) 120 | 121 | def S (fname): 122 | s= simple_stat (fname) 123 | return FalseBool (s is not None and stat.S_ISSOCK (s.st_mode)) 124 | 125 | # TODO: ef, 126 | 127 | def nt (a, b): 128 | # file1 is newer (according to modification date) than file2, or if file1 exists and file2 does not. 129 | s1= simple_stat (a) 130 | s2= simple_stat (b) 131 | return FalseBool ( (s1 is not None and s2 is None) 132 | or (s1.st_mtime_ns > s2.st_mtime_ns)) 133 | 134 | def ot (a, b): 135 | # file1 is older than file2, or if file2 exists and file1 does not. 136 | s1= simple_stat (a) 137 | s2= simple_stat (b) 138 | return FalseBool ( (s2 is not None and s1 is None) 139 | or (s2.st_mtime_ns > s1.st_mtime_ns)) 140 | 141 | def z(value): 142 | ans = value is None or value == '' 143 | 144 | return FalseBool(ans) 145 | -------------------------------------------------------------------------------- /ayrton/functions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # (c) 2013 Marcos Dione 4 | 5 | # This file is part of ayrton. 6 | # 7 | # ayrton is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # ayrton is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with ayrton. If not, see . 19 | 20 | import ayrton 21 | import ayrton.execute 22 | import os 23 | import signal 24 | from collections.abc import Iterable 25 | 26 | import logging 27 | logger= logging.getLogger ('ayrton.functions') 28 | 29 | # NOTE: all this code is executed in the script's environment 30 | 31 | class cd (object): 32 | def __init__ (self, dir): 33 | self.old_dir= os.getcwd () 34 | os.chdir (dir) 35 | 36 | def __enter__ (self): 37 | pass 38 | 39 | def __exit__ (self, *args): 40 | os.chdir (self.old_dir) 41 | 42 | 43 | def define(*varnames, **defaults): 44 | # defaults have priority over simple names (because they provide a value) 45 | for varname in varnames: 46 | if varname not in defaults: 47 | defaults[varname] = None 48 | 49 | for varname, value in defaults.items(): 50 | if not isinstance(varname, str): 51 | raise ValueError('variable name cannot be non-string: %r' % varname) 52 | 53 | if varname not in ayrton.runner.globals: 54 | ayrton.runner.globals[varname] = value 55 | 56 | 57 | class Exit (Exception): 58 | # exit() has to be implemented with an exception 59 | # because sys.exit() makes the whole interpreter go down 60 | # that is, the interpreter interpreting ayrton :) 61 | # in fact sys.exit() is *also* implemented with an exception :) 62 | def __init__ (self, exit_value): 63 | self.exit_value= exit_value 64 | 65 | def exit (value): 66 | raise Exit (value) 67 | 68 | 69 | def export (**kwargs): 70 | for k, v in kwargs.items (): 71 | ayrton.runner.globals[k]= str (v) 72 | os.environ[k]= str (v) 73 | 74 | 75 | option_map= dict ( 76 | e= 'errexit', 77 | ) 78 | 79 | 80 | def option (option, value=True): 81 | if len (option)==2: 82 | if option[0]=='-': 83 | value= True 84 | elif option[0]=='+': 85 | value= False 86 | else: 87 | raise ValueError ("Malformed option %r" % option) 88 | 89 | try: 90 | option= option_map[option[1]] 91 | except KeyError: 92 | raise KeyError ("Unrecognized option %r" % option) 93 | 94 | if option not in option_map.values (): 95 | raise KeyError ("Unrecognized option %r" % option) 96 | 97 | ayrton.runner.options[option]= value 98 | 99 | 100 | def run (path, *args, **kwargs): 101 | c= ayrton.execute.Command (path) 102 | return c (*args, **kwargs) 103 | 104 | 105 | def shift (*args): 106 | """`shift()` returns the leftmost element of `argv`. 107 | `shitf(integer)` return the `integer` leftmost elements of `argv` as a list. 108 | `shift(iterable)` and `shift(iterable, integer)` operate over `iterable`.""" 109 | if len(args) > 2: 110 | raise ValueError("shift() takes 0, 1 or 2 arguments.") 111 | 112 | n = 1 113 | l = ayrton.runner.globals['argv'] 114 | 115 | logger.debug2("%s(%d)", args, len(args)) 116 | if len(args) == 1: 117 | value = args[0] 118 | logger.debug2(type(value)) 119 | if isinstance(value, int): 120 | n = value 121 | elif isinstance(value, Iterable): 122 | l = value 123 | else: 124 | raise ValueError("First parameter must be Iterable or int().") 125 | elif len(args) == 2: 126 | l, n = args 127 | 128 | logger.debug2("%s(%d)", args, len(args)) 129 | logger.debug("%s[%d]", l, n) 130 | 131 | if n == 1: 132 | ans= l.pop(0) 133 | elif n > 1: 134 | ans= [ l.pop(0) for i in range(n) ] 135 | else: 136 | raise ValueError("Integer parameter must be >= 0.") 137 | 138 | return ans 139 | 140 | 141 | def trap(handler, *signals): 142 | for signal in signals: 143 | signal.signal(signal, handler) 144 | 145 | 146 | def unset (*args): 147 | for k in args: 148 | if k in ayrton.runner.globals: 149 | # found, remove it 150 | del ayrton.runner.globals[k] 151 | del os.environ[k] 152 | -------------------------------------------------------------------------------- /ayrton/importer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # (c) 2016 Marcos Dione 4 | 5 | # This file is part of ayrton. 6 | # 7 | # ayrton is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # ayrton is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with ayrton. If not, see . 19 | 20 | from importlib.abc import MetaPathFinder, Loader 21 | try: 22 | # py3.4 23 | from importlib.machinery import ModuleSpec 24 | except ImportError: # pragma: no cover 25 | # py3.3 26 | pass # sorry, no support 27 | 28 | import sys 29 | import os 30 | import os.path 31 | 32 | import logging 33 | logger= logging.getLogger ('ayrton.importer') 34 | 35 | from ayrton.file_test import a, d 36 | from ayrton import Ayrton 37 | import ayrton.utils 38 | 39 | 40 | class AyrtonLoader (Loader): 41 | 42 | @classmethod 43 | def exec_module (klass, module): 44 | # module is a freshly created, empty module 45 | # «the loader should execute the module’s code 46 | # in the module’s global name space (module.__dict__).» 47 | load_path= module.__spec__.origin 48 | logger.debug2 ('loading %s [%s]', module, load_path) 49 | # I *need* to polute the globals, so modules can use any of ayrton's builtins 50 | loader= Ayrton (g=module.__dict__) 51 | loader.run_file (load_path) 52 | 53 | # set the __path__ 54 | # TODO: read PEP 420 55 | init_file_name= '__init__.ay' 56 | if load_path.endswith (init_file_name): 57 | # also remove the '/' 58 | module.__path__= [ load_path[:-len (init_file_name)-1] ] 59 | 60 | logger.debug3 ('module.__dict__: %s ', ayrton.utils.dump_dict (module.__dict__)) 61 | 62 | loader= AyrtonLoader () 63 | 64 | 65 | class AyrtonFinder (MetaPathFinder): 66 | 67 | @classmethod 68 | def find_spec (klass, full_name, paths=None, target=None): 69 | # full_name is the full python path (as in grandparent.parent.child) 70 | # and path is the path of the parent (in a list, see PEP 420); 71 | # if None, then we're loading a root module 72 | # let's start with a single file 73 | # TODO: read PEP 420 :) 74 | logger.debug2 ('searching for %s under %s for %s', full_name, paths, target) 75 | last_mile= full_name.split ('.')[-1] 76 | 77 | if paths is not None: 78 | python_path= paths # search only there 79 | else: 80 | python_path= sys.path 81 | 82 | logger.debug2 (python_path) 83 | for path in python_path: 84 | full_path= os.path.join (path, last_mile) 85 | init_full_path= os.path.join (full_path, '__init__.ay') 86 | module_full_path= full_path+'.ay' 87 | 88 | logger.debug2 ('trying %s', init_full_path) 89 | if -d (full_path) and -a (init_full_path): 90 | logger.debug2 ('found package %s', full_path) 91 | return ModuleSpec (full_name, loader, origin=init_full_path) 92 | 93 | else: 94 | logger.debug2 ('trying %s', module_full_path) 95 | if -a (module_full_path): 96 | logger.debug2 ('found module %s', module_full_path) 97 | return ModuleSpec (full_name, loader, origin=module_full_path) 98 | 99 | logger.debug2 ('404 Not Found') 100 | return None 101 | 102 | finder= AyrtonFinder () 103 | 104 | 105 | # I must insert it at the beginning so it goes before FileFinder 106 | sys.meta_path.insert (0, finder) 107 | -------------------------------------------------------------------------------- /ayrton/logger.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # (c) 2017 Marcos Dione 4 | 5 | # This file is part of ayrton. 6 | # 7 | # ayrton is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # ayrton is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with ayrton. If not, see . 19 | 20 | import os.path 21 | import logging 22 | import io 23 | 24 | # NOTE: you can't log in this module! 25 | # of course you can? 26 | logger= logging.getLogger(__file__) 27 | 28 | 29 | # shamelessly copied from py3.6's logging/__init__.py 30 | # NOTE: update as needed 31 | 32 | def foo(): 33 | pass 34 | 35 | _srcfile = os.path.normcase(foo.__code__.co_filename) 36 | 37 | class Logger(logging.Logger): 38 | def __init__(self, *args, **kwargs): 39 | super().__init__(*args, **kwargs) 40 | 41 | 42 | # ditto 43 | def findCaller(self, stack_info=False, callers=0): 44 | """ 45 | Find the stack frame of the caller so that we can note the source 46 | file name, line number and function name. Not only ignore this class's 47 | and logger's source, but also as many callers as requested. 48 | """ 49 | f = logging.currentframe() 50 | #On some versions of IronPython, currentframe() returns None if 51 | #IronPython isn't run with -X:Frames. 52 | if f is not None: 53 | if callers > 0: 54 | # yes we can! 55 | co = f.f_code 56 | logger.debug2("%s:%s", co.co_filename, co.co_name) 57 | f = f.f_back 58 | rv = "(unknown file)", 0, "(unknown function)", None 59 | countdown = callers 60 | while hasattr(f, "f_code"): 61 | co = f.f_code 62 | filename = os.path.normcase(co.co_filename) 63 | if callers > 0: 64 | # yes we can! 65 | logger.debug2("%s:%s", co.co_filename, co.co_name) 66 | if filename in(_srcfile, logging._srcfile): 67 | f = f.f_back 68 | continue 69 | if countdown > 0: 70 | f = f.f_back 71 | countdown -= 1 72 | continue 73 | sinfo = None 74 | if stack_info: 75 | sio = io.StringIO() 76 | sio.write('Stack (most recent call last):\n') 77 | traceback.print_stack(f, file=sio) 78 | sinfo = sio.getvalue() 79 | if sinfo[-1] == '\n': 80 | sinfo = sinfo[:-1] 81 | sio.close() 82 | rv = (co.co_filename, f.f_lineno, co.co_name, sinfo) 83 | break 84 | return rv 85 | 86 | 87 | # ditto 88 | def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False, 89 | callers=0): 90 | """ 91 | Low-level logging routine which creates a LogRecord and then calls 92 | all the handlers of this logger to handle the record. 93 | """ 94 | sinfo = None 95 | if _srcfile: 96 | #IronPython doesn't track Python frames, so findCaller raises an 97 | #exception on some versions of IronPython. We trap it here so that 98 | #IronPython can use logging. 99 | try: 100 | fn, lno, func, sinfo = self.findCaller(stack_info, callers) 101 | except ValueError: # pragma: no cover 102 | fn, lno, func = "(unknown file)", 0, "(unknown function)" 103 | else: # pragma: no cover 104 | fn, lno, func = "(unknown file)", 0, "(unknown function)" 105 | if exc_info: 106 | if isinstance(exc_info, BaseException): 107 | exc_info = (type(exc_info), exc_info, exc_info.__traceback__) 108 | elif not isinstance(exc_info, tuple): 109 | exc_info = sys.exc_info() 110 | record = self.makeRecord(self.name, level, fn, lno, msg, args, 111 | exc_info, func, extra, sinfo) 112 | self.handle(record) 113 | 114 | 115 | # insert this code in logging's infra 116 | logging.setLoggerClass(Logger) 117 | -------------------------------------------------------------------------------- /ayrton/parser/__init__.py: -------------------------------------------------------------------------------- 1 | # empty 2 | -------------------------------------------------------------------------------- /ayrton/parser/astcompiler/__init__.py: -------------------------------------------------------------------------------- 1 | # empty 2 | -------------------------------------------------------------------------------- /ayrton/parser/astcompiler/consts.py: -------------------------------------------------------------------------------- 1 | """ 2 | Various flags used during the compilation process. 3 | """ 4 | 5 | CO_OPTIMIZED = 0x0001 6 | CO_NEWLOCALS = 0x0002 7 | CO_VARARGS = 0x0004 8 | CO_VARKEYWORDS = 0x0008 9 | CO_NESTED = 0x0010 10 | CO_GENERATOR = 0x0020 11 | CO_NOFREE = 0x0040 12 | CO_COROUTINE = 0x0080 13 | CO_ITERABLE_COROUTINE = 0x0100 # set by @types.coroutine 14 | CO_GENERATOR_ALLOWED = 0x1000 15 | CO_FUTURE_DIVISION = 0x2000 16 | CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 17 | CO_FUTURE_WITH_STATEMENT = 0x8000 18 | CO_FUTURE_PRINT_FUNCTION = 0x10000 19 | CO_FUTURE_UNICODE_LITERALS = 0x20000 20 | CO_FUTURE_BARRY_AS_BDFL = 0x40000 21 | CO_FUTURE_GENERATOR_STOP = 0x80000 22 | #pypy specific: 23 | CO_KILL_DOCSTRING = 0x100000 24 | CO_YIELD_INSIDE_TRY = 0x200000 25 | 26 | PyCF_MASK = (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | 27 | CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | 28 | CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | 29 | CO_FUTURE_GENERATOR_STOP) 30 | PyCF_SOURCE_IS_UTF8 = 0x0100 31 | PyCF_DONT_IMPLY_DEDENT = 0x0200 32 | PyCF_ONLY_AST = 0x0400 33 | PyCF_IGNORE_COOKIE = 0x0800 34 | PyCF_ACCEPT_NULL_BYTES = 0x10000000 # PyPy only, for compile() 35 | PyCF_FOUND_ENCODING = 0x20000000 # PyPy only, for pytokenizer 36 | 37 | # Masks and values used by FORMAT_VALUE opcode 38 | FVC_MASK = 0x3 39 | FVC_NONE = 0x0 40 | FVC_STR = 0x1 41 | FVC_REPR = 0x2 42 | FVC_ASCII = 0x3 43 | FVS_MASK = 0x4 44 | FVS_HAVE_SPEC = 0x4 45 | -------------------------------------------------------------------------------- /ayrton/parser/astcompiler/misc.py: -------------------------------------------------------------------------------- 1 | def parse_future(tree, feature_flags): 2 | from ayrton.parser.astcompiler import ast 3 | future_lineno = 0 4 | future_column = 0 5 | flags = 0 6 | have_docstring = False 7 | body = None 8 | if isinstance(tree, ast.Module): 9 | body = tree.body 10 | elif isinstance(tree, ast.Interactive): 11 | body = tree.body 12 | if body is None: 13 | return 0, 0, 0 14 | for stmt in body: 15 | if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Str): 16 | if have_docstring: 17 | break 18 | else: 19 | have_docstring = True 20 | elif isinstance(stmt, ast.ImportFrom): 21 | if stmt.module == "__future__": 22 | future_lineno = stmt.lineno 23 | future_column = stmt.col_offset 24 | for alias in stmt.names: 25 | assert isinstance(alias, ast.alias) 26 | # If this is an invalid flag, it will be caught later in 27 | # codegen.py. 28 | flags |= feature_flags.get(alias.name, 0) 29 | else: 30 | break 31 | else: 32 | break 33 | return flags, future_lineno, future_column 34 | 35 | 36 | class ForbiddenNameAssignment(Exception): 37 | 38 | def __init__(self, name, node): 39 | self.name = name 40 | self.node = node 41 | 42 | 43 | def check_forbidden_name(name, node=None): 44 | """Raise an error if the name cannot be assigned to.""" 45 | if name in ("None", "__debug__"): 46 | raise ForbiddenNameAssignment(name, node) 47 | # XXX Warn about using True and False 48 | 49 | 50 | # shamelessly lifted off rpython 51 | class unrolling_iterable: 52 | 53 | def __init__(self, iterable): 54 | self._items = list(iterable) 55 | self._head = _unroller(self._items) 56 | 57 | def __iter__(self): 58 | return iter(self._items) 59 | 60 | def get_unroller(self): 61 | return self._head 62 | 63 | 64 | # ditto 65 | class _unroller: 66 | 67 | def __init__(self, items, i=0): 68 | self._items = items 69 | self._i = i 70 | self._next = None 71 | 72 | def step(self): 73 | v = self._items[self._i] 74 | if self._next is None: 75 | self._next = _unroller(self._items, self._i+1) 76 | return v, self._next 77 | 78 | 79 | def dict_to_switch(d): 80 | """Convert of dictionary with integer keys to a switch statement.""" 81 | def lookup(query): 82 | return d[query] 83 | lookup._always_inline_ = True 84 | unrolling_items = unrolling_iterable(d.items()) 85 | return lookup 86 | 87 | 88 | def mangle(name, klass): 89 | if not name.startswith('__'): 90 | return name 91 | # Don't mangle __id__ or names with dots. The only time a name with a dot 92 | # can occur is when we are compiling an import statement that has a package 93 | # name. 94 | if name.endswith('__') or '.' in name: 95 | return name 96 | try: 97 | i = 0 98 | while klass[i] == '_': 99 | i = i + 1 100 | except IndexError: 101 | return name 102 | return "_%s%s" % (klass[i:], name) 103 | 104 | 105 | def intern_if_common_string(space, w_const): 106 | # only intern identifier-like strings 107 | from pypy.objspace.std.unicodeobject import _isidentifier 108 | if (space.is_w(space.type(w_const), space.w_unicode) and 109 | _isidentifier(space.unicode_w(w_const))): 110 | return space.new_interned_w_str(w_const) 111 | return w_const 112 | 113 | 114 | def new_identifier(space, name): 115 | # Check whether there are non-ASCII characters in the identifier; if 116 | # so, normalize to NFKC 117 | for c in name: 118 | if ord(c) > 0x80: 119 | break 120 | else: 121 | return name 122 | 123 | from pypy.module.unicodedata.interp_ucd import ucd 124 | w_name = space.newtext(name) 125 | w_id = space.call_method(ucd, 'normalize', space.newtext('NFKC'), w_name) 126 | return space.text_w(w_id) 127 | -------------------------------------------------------------------------------- /ayrton/parser/astcompiler/tools/Python.asdl: -------------------------------------------------------------------------------- 1 | -- ASDL's 7 builtin types are: 2 | -- identifier, int, string, bytes, object, singleton, constant 3 | -- 4 | -- singleton: None, True or False 5 | -- constant can be None, whereas None means "no value" for object. 6 | 7 | module Python 8 | { 9 | mod = Module(stmt* body) 10 | | Interactive(stmt* body) 11 | | Expression(expr body) 12 | 13 | -- not really an actual node but useful in Jython's typesystem. 14 | | Suite(stmt* body) 15 | 16 | stmt = FunctionDef(identifier name, arguments args, 17 | stmt* body, expr* decorator_list, expr? returns) 18 | | AsyncFunctionDef(identifier name, arguments args, 19 | stmt* body, expr* decorator_list, expr? returns) 20 | 21 | | ClassDef(identifier name, 22 | expr* bases, 23 | keyword* keywords, 24 | stmt* body, 25 | expr* decorator_list) 26 | | Return(expr? value) 27 | 28 | | Delete(expr* targets) 29 | | Assign(expr* targets, expr value) 30 | | AugAssign(expr target, operator op, expr value) 31 | -- 'simple' indicates that we annotate simple name without parens 32 | | AnnAssign(expr target, expr annotation, expr? value, int simple) 33 | 34 | -- use 'orelse' because else is a keyword in target languages 35 | | For(expr target, expr iter, stmt* body, stmt* orelse) 36 | | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse) 37 | | While(expr test, stmt* body, stmt* orelse) 38 | | If(expr test, stmt* body, stmt* orelse) 39 | | With(withitem* items, stmt* body) 40 | | AsyncWith(withitem* items, stmt* body) 41 | 42 | | Raise(expr? exc, expr? cause) 43 | | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody) 44 | | Assert(expr test, expr? msg) 45 | 46 | | Import(alias* names) 47 | | ImportFrom(identifier? module, alias* names, int? level) 48 | 49 | | Global(identifier* names) 50 | | Nonlocal(identifier* names) 51 | | Expr(expr value) 52 | | Pass | Break | Continue 53 | 54 | -- XXX Jython will be different 55 | -- col_offset is the byte offset in the utf8 string the parser uses 56 | attributes (int lineno, int col_offset) 57 | 58 | -- BoolOp() can use left & right? 59 | expr = BoolOp(boolop op, expr* values) 60 | | BinOp(expr left, operator op, expr right) 61 | | UnaryOp(unaryop op, expr operand) 62 | | Lambda(arguments args, expr body) 63 | | IfExp(expr test, expr body, expr orelse) 64 | | Dict(expr* keys, expr* values) 65 | | Set(expr* elts) 66 | | ListComp(expr elt, comprehension* generators) 67 | | SetComp(expr elt, comprehension* generators) 68 | | DictComp(expr key, expr value, comprehension* generators) 69 | | GeneratorExp(expr elt, comprehension* generators) 70 | -- the grammar constrains where yield expressions can occur 71 | | Await(expr value) 72 | | Yield(expr? value) 73 | | YieldFrom(expr value) 74 | -- need sequences for compare to distinguish between 75 | -- x < 4 < 3 and (x < 4) < 3 76 | | Compare(expr left, cmpop* ops, expr* comparators) 77 | | Call(expr func, expr* args, keyword* keywords) 78 | | Num(object n) -- a number as a PyObject. 79 | | Str(string s) -- need to specify raw, unicode, etc? 80 | | FormattedValue(expr value, int? conversion, expr? format_spec) 81 | | JoinedStr(expr* values) 82 | | Bytes(bytes s) 83 | | NameConstant(singleton value) 84 | | Ellipsis 85 | | Constant(constant value) 86 | 87 | -- the following expression can appear in assignment context 88 | | Attribute(expr value, identifier attr, expr_context ctx) 89 | | Subscript(expr value, slice slice, expr_context ctx) 90 | | Starred(expr value, expr_context ctx) 91 | | Name(identifier id, expr_context ctx) 92 | | List(expr* elts, expr_context ctx) 93 | | Tuple(expr* elts, expr_context ctx) 94 | 95 | -- col_offset is the byte offset in the utf8 string the parser uses 96 | attributes (int lineno, int col_offset) 97 | 98 | expr_context = Load | Store | Del | AugLoad | AugStore | Param 99 | 100 | slice = Slice(expr? lower, expr? upper, expr? step) 101 | | ExtSlice(slice* dims) 102 | | Index(expr value) 103 | 104 | boolop = And | Or 105 | 106 | operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift 107 | | RShift | BitOr | BitXor | BitAnd | FloorDiv 108 | 109 | unaryop = Invert | Not | UAdd | USub 110 | 111 | cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn 112 | 113 | comprehension = (expr target, expr iter, expr* ifs, int is_async) 114 | 115 | excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body) 116 | attributes (int lineno, int col_offset) 117 | 118 | arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, 119 | arg? kwarg, expr* defaults) 120 | 121 | arg = (identifier arg, expr? annotation) 122 | attributes (int lineno, int col_offset) 123 | 124 | -- keyword arguments supplied to call (NULL identifier for **kwargs) 125 | keyword = (identifier? arg, expr value) 126 | 127 | -- import name with optional 'as' alias. 128 | alias = (identifier name, identifier? asname) 129 | 130 | withitem = (expr context_expr, expr? optional_vars) 131 | } 132 | 133 | -------------------------------------------------------------------------------- /ayrton/parser/debug.py: -------------------------------------------------------------------------------- 1 | """ 2 | PyPy-oriented interface to pdb. 3 | """ 4 | 5 | import pdb 6 | 7 | def fire(operationerr): 8 | if not operationerr.debug_excs: 9 | return 10 | exc, val, tb = operationerr.debug_excs[-1] 11 | pdb.post_mortem(tb) 12 | -------------------------------------------------------------------------------- /ayrton/parser/pyparser/__init__.py: -------------------------------------------------------------------------------- 1 | # empty 2 | -------------------------------------------------------------------------------- /ayrton/parser/pyparser/automata.py: -------------------------------------------------------------------------------- 1 | # ______________________________________________________________________ 2 | """Module automata 3 | 4 | THIS FILE WAS COPIED FROM pypy/module/parser/pytokenize.py AND ADAPTED 5 | TO BE ANNOTABLE (Mainly made the DFA's __init__ accept two lists 6 | instead of a unique nested one) 7 | 8 | $Id: automata.py,v 1.2 2003/10/02 17:37:17 jriehl Exp $ 9 | """ 10 | # ______________________________________________________________________ 11 | # Module level definitions 12 | 13 | # PYPY Modification: removed the EMPTY class as it's not needed here 14 | 15 | 16 | # PYPY Modification: we don't need a particuliar DEFAULT class here 17 | # a simple None works fine. 18 | # (Having a DefaultClass inheriting from str makes 19 | # the annotator crash) 20 | DEFAULT = "\00default" # XXX hack, the rtyper does not support dict of with str|None keys 21 | # anyway using dicts doesn't seem the best final way to store these char indexed tables 22 | # PYPY Modification : removed all automata functions (any, maybe, 23 | # newArcPair, etc.) 24 | 25 | class DFA: 26 | # ____________________________________________________________ 27 | def __init__(self, states, accepts, start = 0): 28 | self.states = states 29 | self.accepts = accepts 30 | self.start = start 31 | 32 | # ____________________________________________________________ 33 | def recognize (self, inVec, pos = 0): # greedy = True 34 | crntState = self.start 35 | lastAccept = False 36 | i = pos 37 | for i in range(pos, len(inVec)): 38 | item = inVec[i] 39 | if ord(item) > 0x80: 40 | item = "\x80" # NON_ASCII 41 | # arcMap, accept = self.states[crntState] 42 | arcMap = self.states[crntState] 43 | accept = self.accepts[crntState] 44 | if item in arcMap: 45 | crntState = arcMap[item] 46 | elif DEFAULT in arcMap: 47 | crntState = arcMap[DEFAULT] 48 | elif accept: 49 | return i 50 | elif lastAccept: 51 | # This is now needed b/c of exception cases where there are 52 | # transitions to dead states 53 | return i - 1 54 | else: 55 | return -1 56 | lastAccept = accept 57 | # if self.states[crntState][1]: 58 | if self.accepts[crntState]: 59 | return i + 1 60 | elif lastAccept: 61 | return i 62 | else: 63 | return -1 64 | 65 | # ______________________________________________________________________ 66 | 67 | class NonGreedyDFA (DFA): 68 | def recognize (self, inVec, pos = 0): 69 | crntState = self.start 70 | i = pos 71 | for item in inVec[pos:]: 72 | # arcMap, accept = self.states[crntState] 73 | arcMap = self.states[crntState] 74 | accept = self.accepts[crntState] 75 | if accept: 76 | return i 77 | elif item in arcMap: 78 | crntState = arcMap[item] 79 | elif DEFAULT in arcMap: 80 | crntState = arcMap[DEFAULT] 81 | else: 82 | return -1 83 | i += 1 84 | # if self.states[crntState][1]: 85 | if self.accepts[crntState]: 86 | return i 87 | else: 88 | return -1 89 | 90 | # ______________________________________________________________________ 91 | # End of automata.py 92 | -------------------------------------------------------------------------------- /ayrton/parser/pyparser/data/Grammar2.5: -------------------------------------------------------------------------------- 1 | # Grammar for Python 2 | 3 | # Note: Changing the grammar specified in this file will most likely 4 | # require corresponding changes in the parser module 5 | # (../Modules/parsermodule.c). If you can't make the changes to 6 | # that module yourself, please co-ordinate the required changes 7 | # with someone who can; ask around on python-dev for help. Fred 8 | # Drake will probably be listening there. 9 | 10 | # NOTE WELL: You should also follow all the steps listed in PEP 306, 11 | # "How to Change Python's Grammar" 12 | 13 | # Commands for Kees Blom's railroad program 14 | #diagram:token NAME 15 | #diagram:token NUMBER 16 | #diagram:token STRING 17 | #diagram:token NEWLINE 18 | #diagram:token ENDMARKER 19 | #diagram:token INDENT 20 | #diagram:output\input python.bla 21 | #diagram:token DEDENT 22 | #diagram:output\textwidth 20.04cm\oddsidemargin 0.0cm\evensidemargin 0.0cm 23 | #diagram:rules 24 | 25 | # Start symbols for the grammar: 26 | # single_input is a single interactive statement; 27 | # file_input is a module or sequence of commands read from an input file; 28 | # eval_input is the input for the eval() and input() functions. 29 | # NB: compound_stmt in single_input is followed by extra NEWLINE! 30 | single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE 31 | file_input: (NEWLINE | stmt)* ENDMARKER 32 | eval_input: testlist NEWLINE* ENDMARKER 33 | 34 | decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE 35 | decorators: decorator+ 36 | funcdef: [decorators] 'def' NAME parameters ':' suite 37 | parameters: '(' [varargslist] ')' 38 | varargslist: ((fpdef ['=' test] ',')* 39 | ('*' NAME [',' '**' NAME] | '**' NAME) | 40 | fpdef ['=' test] (',' fpdef ['=' test])* [',']) 41 | fpdef: NAME | '(' fplist ')' 42 | fplist: fpdef (',' fpdef)* [','] 43 | 44 | stmt: simple_stmt | compound_stmt 45 | simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE 46 | small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | 47 | import_stmt | global_stmt | exec_stmt | assert_stmt) 48 | expr_stmt: testlist (augassign (yield_expr|testlist) | 49 | ('=' (yield_expr|testlist))*) 50 | augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | 51 | '<<=' | '>>=' | '**=' | '//=') 52 | # For normal assignments, additional restrictions enforced by the interpreter 53 | print_stmt: 'print' ( [ test (',' test)* [','] ] | 54 | '>>' test [ (',' test)+ [','] ] ) 55 | del_stmt: 'del' exprlist 56 | pass_stmt: 'pass' 57 | flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt 58 | break_stmt: 'break' 59 | continue_stmt: 'continue' 60 | return_stmt: 'return' [testlist] 61 | yield_stmt: yield_expr 62 | raise_stmt: 'raise' [test [',' test [',' test]]] 63 | import_stmt: import_name | import_from 64 | import_name: 'import' dotted_as_names 65 | import_from: ('from' ('.'* dotted_name | '.'+) 66 | 'import' ('*' | '(' import_as_names ')' | import_as_names)) 67 | import_as_name: NAME [('as' | NAME) NAME] 68 | dotted_as_name: dotted_name [('as' | NAME) NAME] 69 | import_as_names: import_as_name (',' import_as_name)* [','] 70 | dotted_as_names: dotted_as_name (',' dotted_as_name)* 71 | dotted_name: NAME ('.' NAME)* 72 | global_stmt: 'global' NAME (',' NAME)* 73 | exec_stmt: 'exec' expr ['in' test [',' test]] 74 | assert_stmt: 'assert' test [',' test] 75 | 76 | compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef 77 | if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] 78 | while_stmt: 'while' test ':' suite ['else' ':' suite] 79 | for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] 80 | try_stmt: ('try' ':' suite 81 | ((except_clause ':' suite)+ 82 | ['else' ':' suite] 83 | ['finally' ':' suite] | 84 | 'finally' ':' suite)) 85 | with_stmt: 'with' test [ with_var ] ':' suite 86 | with_var: ('as' | NAME) expr 87 | # NB compile.c makes sure that the default except clause is last 88 | except_clause: 'except' [test [',' test]] 89 | suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT 90 | 91 | # Backward compatibility cruft to support: 92 | # [ x for x in lambda: True, lambda: False if x() ] 93 | # even while also allowing: 94 | # lambda x: 5 if x else 2 95 | # (But not a mix of the two) 96 | testlist_safe: old_test [(',' old_test)+ [',']] 97 | old_test: or_test | old_lambdef 98 | old_lambdef: 'lambda' [varargslist] ':' old_test 99 | 100 | test: or_test ['if' or_test 'else' test] | lambdef 101 | or_test: and_test ('or' and_test)* 102 | and_test: not_test ('and' not_test)* 103 | not_test: 'not' not_test | comparison 104 | comparison: expr (comp_op expr)* 105 | comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' 106 | expr: xor_expr ('|' xor_expr)* 107 | xor_expr: and_expr ('^' and_expr)* 108 | and_expr: shift_expr ('&' shift_expr)* 109 | shift_expr: arith_expr (('<<'|'>>') arith_expr)* 110 | arith_expr: term (('+'|'-') term)* 111 | term: factor (('*'|'/'|'%'|'//') factor)* 112 | factor: ('+'|'-'|'~') factor | power 113 | power: atom trailer* ['**' factor] 114 | atom: ('(' [yield_expr|testlist_gexp] ')' | 115 | '[' [listmaker] ']' | 116 | '{' [dictmaker] '}' | 117 | '`' testlist1 '`' | 118 | NAME | NUMBER | STRING+) 119 | listmaker: test ( list_for | (',' test)* [','] ) 120 | testlist_gexp: test ( gen_for | (',' test)* [','] ) 121 | lambdef: 'lambda' [varargslist] ':' test 122 | trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME 123 | subscriptlist: subscript (',' subscript)* [','] 124 | subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop] 125 | sliceop: ':' [test] 126 | exprlist: expr (',' expr)* [','] 127 | testlist: test (',' test)* [','] 128 | dictmaker: test ':' test (',' test ':' test)* [','] 129 | 130 | classdef: 'class' NAME ['(' [testlist] ')'] ':' suite 131 | 132 | arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test) 133 | argument: test [gen_for] | test '=' test # Really [keyword '='] test 134 | 135 | list_iter: list_for | list_if 136 | list_for: 'for' exprlist 'in' testlist_safe [list_iter] 137 | list_if: 'if' old_test [list_iter] 138 | 139 | gen_iter: gen_for | gen_if 140 | gen_for: 'for' exprlist 'in' or_test [gen_iter] 141 | gen_if: 'if' old_test [gen_iter] 142 | 143 | testlist1: test (',' test)* 144 | 145 | # not used in grammar, but may appear in "node" passed from Parser to Compiler 146 | encoding_decl: NAME 147 | 148 | yield_expr: 'yield' [testlist] 149 | -------------------------------------------------------------------------------- /ayrton/parser/pyparser/data/Grammar2.7: -------------------------------------------------------------------------------- 1 | # Grammar for Python 2 | 3 | # Note: Changing the grammar specified in this file will most likely 4 | # require corresponding changes in the parser module 5 | # (../Modules/parsermodule.c). If you can't make the changes to 6 | # that module yourself, please co-ordinate the required changes 7 | # with someone who can; ask around on python-dev for help. Fred 8 | # Drake will probably be listening there. 9 | 10 | # NOTE WELL: You should also follow all the steps listed in PEP 306, 11 | # "How to Change Python's Grammar" 12 | 13 | # Start symbols for the grammar: 14 | # single_input is a single interactive statement; 15 | # file_input is a module or sequence of commands read from an input file; 16 | # eval_input is the input for the eval() and input() functions. 17 | # NB: compound_stmt in single_input is followed by extra NEWLINE! 18 | single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE 19 | file_input: (NEWLINE | stmt)* ENDMARKER 20 | eval_input: testlist NEWLINE* ENDMARKER 21 | 22 | decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE 23 | decorators: decorator+ 24 | decorated: decorators (classdef | funcdef) 25 | funcdef: 'def' NAME parameters ':' suite 26 | parameters: '(' [varargslist] ')' 27 | varargslist: ((fpdef ['=' test] ',')* 28 | ('*' NAME [',' '**' NAME] | '**' NAME) | 29 | fpdef ['=' test] (',' fpdef ['=' test])* [',']) 30 | fpdef: NAME | '(' fplist ')' 31 | fplist: fpdef (',' fpdef)* [','] 32 | 33 | stmt: simple_stmt | compound_stmt 34 | simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE 35 | small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | 36 | import_stmt | global_stmt | exec_stmt | assert_stmt) 37 | expr_stmt: testlist (augassign (yield_expr|testlist) | 38 | ('=' (yield_expr|testlist))*) 39 | augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | 40 | '<<=' | '>>=' | '**=' | '//=') 41 | # For normal assignments, additional restrictions enforced by the interpreter 42 | print_stmt: 'print' ( [ test (',' test)* [','] ] | 43 | '>>' test [ (',' test)+ [','] ] ) 44 | del_stmt: 'del' exprlist 45 | pass_stmt: 'pass' 46 | flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt 47 | break_stmt: 'break' 48 | continue_stmt: 'continue' 49 | return_stmt: 'return' [testlist] 50 | yield_stmt: yield_expr 51 | raise_stmt: 'raise' [test [',' test [',' test]]] 52 | import_stmt: import_name | import_from 53 | import_name: 'import' dotted_as_names 54 | import_from: ('from' ('.'* dotted_name | '.'+) 55 | 'import' ('*' | '(' import_as_names ')' | import_as_names)) 56 | import_as_name: NAME ['as' NAME] 57 | dotted_as_name: dotted_name ['as' NAME] 58 | import_as_names: import_as_name (',' import_as_name)* [','] 59 | dotted_as_names: dotted_as_name (',' dotted_as_name)* 60 | dotted_name: NAME ('.' NAME)* 61 | global_stmt: 'global' NAME (',' NAME)* 62 | exec_stmt: 'exec' expr ['in' test [',' test]] 63 | assert_stmt: 'assert' test [',' test] 64 | 65 | compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated 66 | if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] 67 | while_stmt: 'while' test ':' suite ['else' ':' suite] 68 | for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] 69 | try_stmt: ('try' ':' suite 70 | ((except_clause ':' suite)+ 71 | ['else' ':' suite] 72 | ['finally' ':' suite] | 73 | 'finally' ':' suite)) 74 | with_stmt: 'with' with_item (',' with_item)* ':' suite 75 | with_item: test ['as' expr] 76 | # NB compile.c makes sure that the default except clause is last 77 | except_clause: 'except' [test [('as' | ',') test]] 78 | suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT 79 | 80 | # Backward compatibility cruft to support: 81 | # [ x for x in lambda: True, lambda: False if x() ] 82 | # even while also allowing: 83 | # lambda x: 5 if x else 2 84 | # (But not a mix of the two) 85 | testlist_safe: old_test [(',' old_test)+ [',']] 86 | old_test: or_test | old_lambdef 87 | old_lambdef: 'lambda' [varargslist] ':' old_test 88 | 89 | test: or_test ['if' or_test 'else' test] | lambdef 90 | or_test: and_test ('or' and_test)* 91 | and_test: not_test ('and' not_test)* 92 | not_test: 'not' not_test | comparison 93 | comparison: expr (comp_op expr)* 94 | comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' 95 | expr: xor_expr ('|' xor_expr)* 96 | xor_expr: and_expr ('^' and_expr)* 97 | and_expr: shift_expr ('&' shift_expr)* 98 | shift_expr: arith_expr (('<<'|'>>') arith_expr)* 99 | arith_expr: term (('+'|'-') term)* 100 | term: factor (('*'|'/'|'%'|'//') factor)* 101 | factor: ('+'|'-'|'~') factor | power 102 | power: atom trailer* ['**' factor] 103 | atom: ('(' [yield_expr|testlist_comp] ')' | 104 | '[' [listmaker] ']' | 105 | '{' [dictorsetmaker] '}' | 106 | '`' testlist1 '`' | 107 | NAME | NUMBER | STRING+) 108 | listmaker: test ( list_for | (',' test)* [','] ) 109 | testlist_comp: test ( comp_for | (',' test)* [','] ) 110 | lambdef: 'lambda' [varargslist] ':' test 111 | trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME 112 | subscriptlist: subscript (',' subscript)* [','] 113 | subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop] 114 | sliceop: ':' [test] 115 | exprlist: expr (',' expr)* [','] 116 | testlist: test (',' test)* [','] 117 | dictmaker: test ':' test (',' test ':' test)* [','] 118 | dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | 119 | (test (comp_for | (',' test)* [','])) ) 120 | 121 | classdef: 'class' NAME ['(' [testlist] ')'] ':' suite 122 | 123 | arglist: (argument ',')* (argument [','] 124 | |'*' test (',' argument)* [',' '**' test] 125 | |'**' test) 126 | # The reason that keywords are test nodes instead of NAME is that using NAME 127 | # results in an ambiguity. ast.c makes sure it's a NAME. 128 | argument: test [comp_for] | test '=' test 129 | 130 | list_iter: list_for | list_if 131 | list_for: 'for' exprlist 'in' testlist_safe [list_iter] 132 | list_if: 'if' old_test [list_iter] 133 | 134 | comp_iter: comp_for | comp_if 135 | comp_for: 'for' exprlist 'in' or_test [comp_iter] 136 | comp_if: 'if' old_test [comp_iter] 137 | 138 | testlist1: test (',' test)* 139 | 140 | # not used in grammar, but may appear in "node" passed from Parser to Compiler 141 | encoding_decl: NAME 142 | 143 | yield_expr: 'yield' [testlist] 144 | -------------------------------------------------------------------------------- /ayrton/parser/pyparser/data/Grammar3.2: -------------------------------------------------------------------------------- 1 | # Grammar for Python 2 | 3 | # Note: Changing the grammar specified in this file will most likely 4 | # require corresponding changes in the parser module 5 | # (../Modules/parsermodule.c). If you can't make the changes to 6 | # that module yourself, please co-ordinate the required changes 7 | # with someone who can; ask around on python-dev for help. Fred 8 | # Drake will probably be listening there. 9 | 10 | # NOTE WELL: You should also follow all the steps listed in PEP 306, 11 | # "How to Change Python's Grammar" 12 | 13 | # Start symbols for the grammar: 14 | # single_input is a single interactive statement; 15 | # file_input is a module or sequence of commands read from an input file; 16 | # eval_input is the input for the eval() and input() functions. 17 | # NB: compound_stmt in single_input is followed by extra NEWLINE! 18 | single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE 19 | file_input: (NEWLINE | stmt)* ENDMARKER 20 | eval_input: testlist NEWLINE* ENDMARKER 21 | 22 | decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE 23 | decorators: decorator+ 24 | decorated: decorators (classdef | funcdef) 25 | funcdef: 'def' NAME parameters ['->' test] ':' suite 26 | parameters: '(' [typedargslist] ')' 27 | typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' 28 | ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]] 29 | | '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef) 30 | tfpdef: NAME [':' test] 31 | varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' 32 | ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]] 33 | | '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef) 34 | vfpdef: NAME 35 | 36 | stmt: simple_stmt | compound_stmt 37 | simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE 38 | small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt | 39 | import_stmt | global_stmt | nonlocal_stmt | assert_stmt) 40 | expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | 41 | ('=' (yield_expr|testlist_star_expr))*) 42 | testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] 43 | augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | 44 | '<<=' | '>>=' | '**=' | '//=') 45 | # For normal assignments, additional restrictions enforced by the interpreter 46 | del_stmt: 'del' exprlist 47 | pass_stmt: 'pass' 48 | flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt 49 | break_stmt: 'break' 50 | continue_stmt: 'continue' 51 | return_stmt: 'return' [testlist] 52 | yield_stmt: yield_expr 53 | raise_stmt: 'raise' [test ['from' test]] 54 | import_stmt: import_name | import_from 55 | import_name: 'import' dotted_as_names 56 | # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS 57 | import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+) 58 | 'import' ('*' | '(' import_as_names ')' | import_as_names)) 59 | import_as_name: NAME ['as' NAME] 60 | dotted_as_name: dotted_name ['as' NAME] 61 | import_as_names: import_as_name (',' import_as_name)* [','] 62 | dotted_as_names: dotted_as_name (',' dotted_as_name)* 63 | dotted_name: NAME ('.' NAME)* 64 | global_stmt: 'global' NAME (',' NAME)* 65 | nonlocal_stmt: 'nonlocal' NAME (',' NAME)* 66 | assert_stmt: 'assert' test [',' test] 67 | 68 | compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated 69 | if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] 70 | while_stmt: 'while' test ':' suite ['else' ':' suite] 71 | for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] 72 | try_stmt: ('try' ':' suite 73 | ((except_clause ':' suite)+ 74 | ['else' ':' suite] 75 | ['finally' ':' suite] | 76 | 'finally' ':' suite)) 77 | with_stmt: 'with' with_item (',' with_item)* ':' suite 78 | with_item: test ['as' expr] 79 | # NB compile.c makes sure that the default except clause is last 80 | except_clause: 'except' [test ['as' NAME]] 81 | suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT 82 | 83 | test: or_test ['if' or_test 'else' test] | lambdef 84 | test_nocond: or_test | lambdef_nocond 85 | lambdef: 'lambda' [varargslist] ':' test 86 | lambdef_nocond: 'lambda' [varargslist] ':' test_nocond 87 | or_test: and_test ('or' and_test)* 88 | and_test: not_test ('and' not_test)* 89 | not_test: 'not' not_test | comparison 90 | comparison: expr (comp_op expr)* 91 | # <> isn't actually a valid comparison operator in Python. It's here for the 92 | # sake of a __future__ import described in PEP 401 93 | comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' 94 | star_expr: '*' expr 95 | expr: xor_expr ('|' xor_expr)* 96 | xor_expr: and_expr ('^' and_expr)* 97 | and_expr: shift_expr ('&' shift_expr)* 98 | shift_expr: arith_expr (('<<'|'>>') arith_expr)* 99 | arith_expr: term (('+'|'-') term)* 100 | term: factor (('*'|'/'|'%'|'//') factor)* 101 | factor: ('+'|'-'|'~') factor | power 102 | power: atom trailer* ['**' factor] 103 | atom: ('(' [yield_expr|testlist_comp] ')' | 104 | '[' [testlist_comp] ']' | 105 | '{' [dictorsetmaker] '}' | 106 | NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False') 107 | testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) 108 | trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME 109 | subscriptlist: subscript (',' subscript)* [','] 110 | subscript: test | [test] ':' [test] [sliceop] 111 | sliceop: ':' [test] 112 | exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] 113 | testlist: test (',' test)* [','] 114 | dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | 115 | (test (comp_for | (',' test)* [','])) ) 116 | 117 | classdef: 'class' NAME ['(' [arglist] ')'] ':' suite 118 | 119 | arglist: (argument ',')* (argument [','] 120 | |'*' test (',' argument)* [',' '**' test] 121 | |'**' test) 122 | # The reason that keywords are test nodes instead of NAME is that using NAME 123 | # results in an ambiguity. ast.c makes sure it's a NAME. 124 | argument: test [comp_for] | test '=' test # Really [keyword '='] test 125 | comp_iter: comp_for | comp_if 126 | comp_for: 'for' exprlist 'in' or_test [comp_iter] 127 | comp_if: 'if' test_nocond [comp_iter] 128 | 129 | # not used in grammar, but may appear in "node" passed from Parser to Compiler 130 | encoding_decl: NAME 131 | 132 | yield_expr: 'yield' [testlist] 133 | -------------------------------------------------------------------------------- /ayrton/parser/pyparser/data/Grammar3.3: -------------------------------------------------------------------------------- 1 | # Grammar for Python 2 | 3 | # Note: Changing the grammar specified in this file will most likely 4 | # require corresponding changes in the parser module 5 | # (../Modules/parsermodule.c). If you can't make the changes to 6 | # that module yourself, please co-ordinate the required changes 7 | # with someone who can; ask around on python-dev for help. Fred 8 | # Drake will probably be listening there. 9 | 10 | # NOTE WELL: You should also follow all the steps listed in PEP 306, 11 | # "How to Change Python's Grammar" 12 | 13 | # Start symbols for the grammar: 14 | # single_input is a single interactive statement; 15 | # file_input is a module or sequence of commands read from an input file; 16 | # eval_input is the input for the eval() functions. 17 | # NB: compound_stmt in single_input is followed by extra NEWLINE! 18 | single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE 19 | file_input: (NEWLINE | stmt)* ENDMARKER 20 | eval_input: testlist NEWLINE* ENDMARKER 21 | 22 | decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE 23 | decorators: decorator+ 24 | decorated: decorators (classdef | funcdef) 25 | funcdef: 'def' NAME parameters ['->' test] ':' suite 26 | parameters: '(' [typedargslist] ')' 27 | typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' 28 | ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]] 29 | | '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef) 30 | tfpdef: NAME [':' test] 31 | varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' 32 | ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]] 33 | | '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef) 34 | vfpdef: NAME 35 | 36 | stmt: simple_stmt | compound_stmt 37 | simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE 38 | small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt | 39 | import_stmt | global_stmt | nonlocal_stmt | assert_stmt) 40 | expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | 41 | ('=' (yield_expr|testlist_star_expr))*) 42 | testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] 43 | augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | 44 | '<<=' | '>>=' | '**=' | '//=') 45 | # For normal assignments, additional restrictions enforced by the interpreter 46 | del_stmt: 'del' exprlist 47 | pass_stmt: 'pass' 48 | flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt 49 | break_stmt: 'break' 50 | continue_stmt: 'continue' 51 | return_stmt: 'return' [testlist] 52 | yield_stmt: yield_expr 53 | raise_stmt: 'raise' [test ['from' test]] 54 | import_stmt: import_name | import_from 55 | import_name: 'import' dotted_as_names 56 | # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS 57 | import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+) 58 | 'import' ('*' | '(' import_as_names ')' | import_as_names)) 59 | import_as_name: NAME ['as' NAME] 60 | dotted_as_name: dotted_name ['as' NAME] 61 | import_as_names: import_as_name (',' import_as_name)* [','] 62 | dotted_as_names: dotted_as_name (',' dotted_as_name)* 63 | dotted_name: NAME ('.' NAME)* 64 | global_stmt: 'global' NAME (',' NAME)* 65 | nonlocal_stmt: 'nonlocal' NAME (',' NAME)* 66 | assert_stmt: 'assert' test [',' test] 67 | 68 | compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated 69 | if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] 70 | while_stmt: 'while' test ':' suite ['else' ':' suite] 71 | for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] 72 | try_stmt: ('try' ':' suite 73 | ((except_clause ':' suite)+ 74 | ['else' ':' suite] 75 | ['finally' ':' suite] | 76 | 'finally' ':' suite)) 77 | with_stmt: 'with' with_item (',' with_item)* ':' suite 78 | with_item: test ['as' expr] 79 | # NB compile.c makes sure that the default except clause is last 80 | except_clause: 'except' [test ['as' NAME]] 81 | suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT 82 | 83 | test: or_test ['if' or_test 'else' test] | lambdef 84 | test_nocond: or_test | lambdef_nocond 85 | lambdef: 'lambda' [varargslist] ':' test 86 | lambdef_nocond: 'lambda' [varargslist] ':' test_nocond 87 | or_test: and_test ('or' and_test)* 88 | and_test: not_test ('and' not_test)* 89 | not_test: 'not' not_test | comparison 90 | comparison: expr (comp_op expr)* 91 | # <> isn't actually a valid comparison operator in Python. It's here for the 92 | # sake of a __future__ import described in PEP 401 93 | comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' 94 | star_expr: '*' expr 95 | expr: xor_expr ('|' xor_expr)* 96 | xor_expr: and_expr ('^' and_expr)* 97 | and_expr: shift_expr ('&' shift_expr)* 98 | shift_expr: arith_expr (('<<'|'>>') arith_expr)* 99 | arith_expr: term (('+'|'-') term)* 100 | term: factor (('*'|'/'|'%'|'//') factor)* 101 | factor: ('+'|'-'|'~') factor | power 102 | power: atom trailer* ['**' factor] 103 | atom: ('(' [yield_expr|testlist_comp] ')' | 104 | '[' [testlist_comp] ']' | 105 | '{' [dictorsetmaker] '}' | 106 | NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False') 107 | testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) 108 | trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME 109 | subscriptlist: subscript (',' subscript)* [','] 110 | subscript: test | [test] ':' [test] [sliceop] 111 | sliceop: ':' [test] 112 | exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] 113 | testlist: test (',' test)* [','] 114 | dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | 115 | (test (comp_for | (',' test)* [','])) ) 116 | 117 | classdef: 'class' NAME ['(' [arglist] ')'] ':' suite 118 | 119 | arglist: (argument ',')* (argument [','] 120 | |'*' test (',' argument)* [',' '**' test] 121 | |'**' test) 122 | # The reason that keywords are test nodes instead of NAME is that using NAME 123 | # results in an ambiguity. ast.c makes sure it's a NAME. 124 | argument: test [comp_for] | test '=' test # Really [keyword '='] test 125 | comp_iter: comp_for | comp_if 126 | comp_for: 'for' exprlist 'in' or_test [comp_iter] 127 | comp_if: 'if' test_nocond [comp_iter] 128 | 129 | # not used in grammar, but may appear in "node" passed from Parser to Compiler 130 | encoding_decl: NAME 131 | 132 | yield_expr: 'yield' [yield_arg] 133 | yield_arg: 'from' test | testlist 134 | -------------------------------------------------------------------------------- /ayrton/parser/pyparser/data/Grammar3.5: -------------------------------------------------------------------------------- 1 | # Grammar for Python 2 | 3 | # Note: Changing the grammar specified in this file will most likely 4 | # require corresponding changes in the parser module 5 | # (../Modules/parsermodule.c). If you can't make the changes to 6 | # that module yourself, please co-ordinate the required changes 7 | # with someone who can; ask around on python-dev for help. Fred 8 | # Drake will probably be listening there. 9 | 10 | # NOTE WELL: You should also follow all the steps listed at 11 | # https://docs.python.org/devguide/grammar.html 12 | 13 | # Start symbols for the grammar: 14 | # single_input is a single interactive statement; 15 | # file_input is a module or sequence of commands read from an input file; 16 | # eval_input is the input for the eval() functions. 17 | # NB: compound_stmt in single_input is followed by extra NEWLINE! 18 | single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE 19 | file_input: (NEWLINE | stmt)* ENDMARKER 20 | eval_input: testlist NEWLINE* ENDMARKER 21 | 22 | decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE 23 | decorators: decorator+ 24 | decorated: decorators (classdef | funcdef | async_funcdef) 25 | 26 | async_funcdef: ASYNC funcdef 27 | funcdef: 'def' NAME parameters ['->' test] ':' suite 28 | 29 | parameters: '(' [typedargslist] ')' 30 | typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' 31 | ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]] 32 | | '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef) 33 | tfpdef: NAME [':' test] 34 | varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' 35 | ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]] 36 | | '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef) 37 | vfpdef: NAME 38 | 39 | stmt: simple_stmt | compound_stmt 40 | simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE 41 | small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt | 42 | import_stmt | global_stmt | nonlocal_stmt | assert_stmt) 43 | expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | 44 | ('=' (yield_expr|testlist_star_expr))*) 45 | testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] 46 | augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | 47 | '<<=' | '>>=' | '**=' | '//=') 48 | # For normal assignments, additional restrictions enforced by the interpreter 49 | del_stmt: 'del' exprlist 50 | pass_stmt: 'pass' 51 | flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt 52 | break_stmt: 'break' 53 | continue_stmt: 'continue' 54 | return_stmt: 'return' [testlist] 55 | yield_stmt: yield_expr 56 | raise_stmt: 'raise' [test ['from' test]] 57 | import_stmt: import_name | import_from 58 | import_name: 'import' dotted_as_names 59 | # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS 60 | import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+) 61 | 'import' ('*' | '(' import_as_names ')' | import_as_names)) 62 | import_as_name: NAME ['as' NAME] 63 | dotted_as_name: dotted_name ['as' NAME] 64 | import_as_names: import_as_name (',' import_as_name)* [','] 65 | dotted_as_names: dotted_as_name (',' dotted_as_name)* 66 | dotted_name: NAME ('.' NAME)* 67 | global_stmt: 'global' NAME (',' NAME)* 68 | nonlocal_stmt: 'nonlocal' NAME (',' NAME)* 69 | assert_stmt: 'assert' test [',' test] 70 | 71 | compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt 72 | async_stmt: ASYNC (funcdef | with_stmt | for_stmt) 73 | if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] 74 | while_stmt: 'while' test ':' suite ['else' ':' suite] 75 | for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] 76 | try_stmt: ('try' ':' suite 77 | ((except_clause ':' suite)+ 78 | ['else' ':' suite] 79 | ['finally' ':' suite] | 80 | 'finally' ':' suite)) 81 | with_stmt: 'with' with_item (',' with_item)* ':' suite 82 | with_item: test ['as' expr] 83 | # NB compile.c makes sure that the default except clause is last 84 | except_clause: 'except' [test ['as' NAME]] 85 | suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT 86 | 87 | test: or_test ['if' or_test 'else' test] | lambdef 88 | test_nocond: or_test | lambdef_nocond 89 | lambdef: 'lambda' [varargslist] ':' test 90 | lambdef_nocond: 'lambda' [varargslist] ':' test_nocond 91 | or_test: and_test ('or' and_test)* 92 | and_test: not_test ('and' not_test)* 93 | not_test: 'not' not_test | comparison 94 | comparison: expr (comp_op expr)* 95 | # <> isn't actually a valid comparison operator in Python. It's here for the 96 | # sake of a __future__ import described in PEP 401 (which really works :-) 97 | comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' 98 | star_expr: '*' expr 99 | expr: xor_expr ('|' xor_expr)* 100 | xor_expr: and_expr ('^' and_expr)* 101 | and_expr: shift_expr ('&' shift_expr)* 102 | shift_expr: arith_expr (('<<'|'>>') arith_expr)* 103 | arith_expr: term (('+'|'-') term)* 104 | term: factor (('*'|'@'|'/'|'%'|'//') factor)* 105 | factor: ('+'|'-'|'~') factor | power 106 | power: atom_expr ['**' factor] 107 | atom_expr: [AWAIT] atom trailer* 108 | atom: ('(' [yield_expr|testlist_comp] ')' | 109 | '[' [testlist_comp] ']' | 110 | '{' [dictorsetmaker] '}' | 111 | NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False') 112 | testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) 113 | trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME 114 | subscriptlist: subscript (',' subscript)* [','] 115 | subscript: test | [test] ':' [test] [sliceop] 116 | sliceop: ':' [test] 117 | exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] 118 | testlist: test (',' test)* [','] 119 | dictorsetmaker: ( ((test ':' test | '**' expr) 120 | (comp_for | (',' (test ':' test | '**' expr))* [','])) | 121 | ((test | star_expr) 122 | (comp_for | (',' (test | star_expr))* [','])) ) 123 | 124 | classdef: 'class' NAME ['(' [arglist] ')'] ':' suite 125 | 126 | arglist: argument (',' argument)* [','] 127 | 128 | # The reason that keywords are test nodes instead of NAME is that using NAME 129 | # results in an ambiguity. ast.c makes sure it's a NAME. 130 | # "test '=' test" is really "keyword '=' test", but we have no such token. 131 | # These need to be in a single rule to avoid grammar that is ambiguous 132 | # to our LL(1) parser. Even though 'test' includes '*expr' in star_expr, 133 | # we explicitly match '*' here, too, to give it proper precedence. 134 | # Illegal combinations and orderings are blocked in ast.c: 135 | # multiple (test comp_for) arguements are blocked; keyword unpackings 136 | # that precede iterable unpackings are blocked; etc. 137 | argument: ( test [comp_for] | 138 | test '=' test | 139 | '**' test | 140 | '*' test ) 141 | 142 | comp_iter: comp_for | comp_if 143 | comp_for: 'for' exprlist 'in' or_test [comp_iter] 144 | comp_if: 'if' test_nocond [comp_iter] 145 | 146 | # not used in grammar, but may appear in "node" passed from Parser to Compiler 147 | encoding_decl: NAME 148 | 149 | yield_expr: 'yield' [yield_arg] 150 | yield_arg: 'from' test | testlist 151 | -------------------------------------------------------------------------------- /ayrton/parser/pyparser/data/Grammar3.6: -------------------------------------------------------------------------------- 1 | # Grammar for Python 2 | 3 | # Note: Changing the grammar specified in this file will most likely 4 | # require corresponding changes in the parser module 5 | # (../Modules/parsermodule.c). If you can't make the changes to 6 | # that module yourself, please co-ordinate the required changes 7 | # with someone who can; ask around on python-dev for help. Fred 8 | # Drake will probably be listening there. 9 | 10 | # NOTE WELL: You should also follow all the steps listed at 11 | # https://docs.python.org/devguide/grammar.html 12 | 13 | # Start symbols for the grammar: 14 | # single_input is a single interactive statement; 15 | # file_input is a module or sequence of commands read from an input file; 16 | # eval_input is the input for the eval() functions. 17 | # NB: compound_stmt in single_input is followed by extra NEWLINE! 18 | single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE 19 | file_input: (NEWLINE | stmt)* ENDMARKER 20 | eval_input: testlist NEWLINE* ENDMARKER 21 | 22 | decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE 23 | decorators: decorator+ 24 | decorated: decorators (classdef | funcdef | async_funcdef) 25 | 26 | async_funcdef: ASYNC funcdef 27 | funcdef: 'def' NAME parameters ['->' test] ':' suite 28 | 29 | parameters: '(' [typedargslist] ')' 30 | typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' 31 | ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]] 32 | | '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef) 33 | tfpdef: NAME [':' test] 34 | varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' 35 | ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]] 36 | | '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef) 37 | vfpdef: NAME 38 | 39 | stmt: simple_stmt | compound_stmt 40 | simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE 41 | small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt | 42 | import_stmt | global_stmt | nonlocal_stmt | assert_stmt) 43 | expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | 44 | ('=' (yield_expr|testlist_star_expr))*) 45 | testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] 46 | augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | 47 | '<<=' | '>>=' | '**=' | '//=') 48 | # For normal assignments, additional restrictions enforced by the interpreter 49 | del_stmt: 'del' exprlist 50 | pass_stmt: 'pass' 51 | flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt 52 | break_stmt: 'break' 53 | continue_stmt: 'continue' 54 | return_stmt: 'return' [testlist] 55 | yield_stmt: yield_expr 56 | raise_stmt: 'raise' [test ['from' test]] 57 | import_stmt: import_name | import_from 58 | import_name: 'import' dotted_as_names 59 | # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS 60 | import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+) 61 | 'import' ('*' | '(' import_as_names ')' | import_as_names)) 62 | import_as_name: NAME ['as' NAME] 63 | dotted_as_name: dotted_name ['as' NAME] 64 | import_as_names: import_as_name (',' import_as_name)* [','] 65 | dotted_as_names: dotted_as_name (',' dotted_as_name)* 66 | dotted_name: NAME ('.' NAME)* 67 | global_stmt: 'global' NAME (',' NAME)* 68 | nonlocal_stmt: 'nonlocal' NAME (',' NAME)* 69 | assert_stmt: 'assert' test [',' test] 70 | 71 | compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt 72 | async_stmt: ASYNC (funcdef | with_stmt | for_stmt) 73 | if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] 74 | while_stmt: 'while' test ':' suite ['else' ':' suite] 75 | for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] 76 | try_stmt: ('try' ':' suite 77 | ((except_clause ':' suite)+ 78 | ['else' ':' suite] 79 | ['finally' ':' suite] | 80 | 'finally' ':' suite)) 81 | with_stmt: 'with' with_item (',' with_item)* ':' suite 82 | with_item: test ['as' expr] 83 | # NB compile.c makes sure that the default except clause is last 84 | except_clause: 'except' [test ['as' NAME]] 85 | suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT 86 | 87 | test: or_test ['if' or_test 'else' test] | lambdef 88 | test_nocond: or_test | lambdef_nocond 89 | lambdef: 'lambda' [varargslist] ':' test 90 | lambdef_nocond: 'lambda' [varargslist] ':' test_nocond 91 | or_test: and_test ('or' and_test)* 92 | and_test: not_test ('and' not_test)* 93 | not_test: 'not' not_test | comparison 94 | comparison: expr (comp_op expr)* 95 | # <> isn't actually a valid comparison operator in Python. It's here for the 96 | # sake of a __future__ import described in PEP 401 (which really works :-) 97 | comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' 98 | star_expr: '*' expr 99 | expr: xor_expr ('|' xor_expr)* 100 | xor_expr: and_expr ('^' and_expr)* 101 | and_expr: shift_expr ('&' shift_expr)* 102 | shift_expr: arith_expr (('<<'|'>>') arith_expr)* 103 | arith_expr: term (('+'|'-') term)* 104 | term: factor (('*'|'@'|'/'|'%'|'//') factor)* 105 | factor: ('+'|'-'|'~') factor | power 106 | power: atom_expr ['**' factor] 107 | atom_expr: [AWAIT] atom trailer* 108 | atom: ('(' [yield_expr|testlist_comp] ')' | 109 | '[' [testlist_comp] ']' | 110 | '{' [dictorsetmaker] '}' | 111 | NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False') 112 | testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) 113 | trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME 114 | subscriptlist: subscript (',' subscript)* [','] 115 | subscript: test | [test] ':' [test] [sliceop] 116 | sliceop: ':' [test] 117 | exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] 118 | testlist: test (',' test)* [','] 119 | dictorsetmaker: ( ((test ':' test | '**' expr) 120 | (comp_for | (',' (test ':' test | '**' expr))* [','])) | 121 | ((test | star_expr) 122 | (comp_for | (',' (test | star_expr))* [','])) ) 123 | 124 | classdef: 'class' NAME ['(' [arglist] ')'] ':' suite 125 | 126 | arglist: argument (',' argument)* [','] 127 | 128 | # The reason that keywords are test nodes instead of NAME is that using NAME 129 | # results in an ambiguity. ast.c makes sure it's a NAME. 130 | # "test '=' test" is really "keyword '=' test", but we have no such token. 131 | # These need to be in a single rule to avoid grammar that is ambiguous 132 | # to our LL(1) parser. Even though 'test' includes '*expr' in star_expr, 133 | # we explicitly match '*' here, too, to give it proper precedence. 134 | # Illegal combinations and orderings are blocked in ast.c: 135 | # multiple (test comp_for) arguements are blocked; keyword unpackings 136 | # that precede iterable unpackings are blocked; etc. 137 | argument: ( test [comp_for] | 138 | test '=' test | 139 | '**' test | 140 | '*' test ) 141 | 142 | comp_iter: comp_for | comp_if 143 | comp_for: 'for' exprlist 'in' or_test [comp_iter] 144 | comp_if: 'if' test_nocond [comp_iter] 145 | 146 | # not used in grammar, but may appear in "node" passed from Parser to Compiler 147 | encoding_decl: NAME 148 | 149 | yield_expr: 'yield' [yield_arg] 150 | yield_arg: 'from' test | testlist 151 | -------------------------------------------------------------------------------- /ayrton/parser/pyparser/error.py: -------------------------------------------------------------------------------- 1 | 2 | class SyntaxError(Exception): 3 | """Base class for exceptions raised by the parser.""" 4 | 5 | def __init__(self, msg, lineno=0, offset=0, text=None, filename=None, 6 | lastlineno=0): 7 | self.msg = msg 8 | self.lineno = lineno 9 | self.offset = offset 10 | self.text = text 11 | self.filename = filename 12 | self.lastlineno = lastlineno 13 | 14 | def wrap_info(self, space): 15 | w_text = w_filename = space.w_None 16 | offset = self.offset 17 | if self.text is not None: 18 | from rpython.rlib.runicode import str_decode_utf_8 19 | # self.text may not be UTF-8 in case of decoding errors. 20 | # adjust the encoded text offset to a decoded offset 21 | # XXX do the right thing about continuation lines, which 22 | # XXX are their own fun, sometimes giving offset > 23 | # XXX len(self.text) for example (right now, avoid crashing) 24 | if offset > len(self.text): 25 | offset = len(self.text) 26 | text, _ = str_decode_utf_8(self.text, offset, 'replace') 27 | offset = len(text) 28 | if len(self.text) != offset: 29 | text, _ = str_decode_utf_8(self.text, len(self.text), 30 | 'replace') 31 | w_text = space.newunicode(text) 32 | if self.filename is not None: 33 | w_filename = space.newfilename(self.filename) 34 | return space.newtuple([space.newtext(self.msg), 35 | space.newtuple([w_filename, 36 | space.newint(self.lineno), 37 | space.newint(offset), 38 | w_text, 39 | space.newint(self.lastlineno)])]) 40 | 41 | def __str__(self): 42 | return "%s at pos (%d, %d) in %r" % (self.__class__.__name__, 43 | self.lineno, 44 | self.offset, 45 | self.text) 46 | 47 | class IndentationError(SyntaxError): 48 | pass 49 | 50 | class TabError(IndentationError): 51 | def __init__(self, lineno=0, offset=0, text=None, filename=None, 52 | lastlineno=0): 53 | msg = "inconsistent use of tabs and spaces in indentation" 54 | IndentationError.__init__(self, msg, lineno, offset, text, filename, lastlineno) 55 | 56 | class ASTError(Exception): 57 | def __init__(self, msg, ast_node ): 58 | self.msg = msg 59 | self.ast_node = ast_node 60 | 61 | 62 | class TokenError(SyntaxError): 63 | 64 | def __init__(self, msg, line, lineno, column, tokens, lastlineno=0): 65 | SyntaxError.__init__(self, msg, lineno, column, line, 66 | lastlineno=lastlineno) 67 | self.tokens = tokens 68 | 69 | class TokenIndentationError(IndentationError): 70 | 71 | def __init__(self, msg, line, lineno, column, tokens): 72 | SyntaxError.__init__(self, msg, lineno, column, line) 73 | self.tokens = tokens 74 | -------------------------------------------------------------------------------- /ayrton/parser/pyparser/future.py: -------------------------------------------------------------------------------- 1 | import __future__ as future 2 | 3 | class FutureFlags(object): 4 | 5 | def __init__(self, version): 6 | compiler_flags = 0 7 | self.compiler_features = {} 8 | self.mandatory_flags = 0 9 | for fname in future.all_feature_names: 10 | feature = getattr(future, fname) 11 | if version >= feature.getOptionalRelease(): 12 | flag = feature.compiler_flag 13 | compiler_flags |= flag 14 | self.compiler_features[fname] = flag 15 | if version >= feature.getMandatoryRelease(): 16 | self.mandatory_flags |= feature.compiler_flag 17 | self.allowed_flags = compiler_flags 18 | 19 | def get_flag_names(self, space, flags): 20 | flag_names = [] 21 | for name, value in self.compiler_features.items(): 22 | if flags & value: 23 | flag_names.append(name) 24 | return flag_names 25 | 26 | def get_compiler_feature(self, name): 27 | return self.compiler_features.get(name, 0) 28 | 29 | futureFlags_2_4 = FutureFlags((2, 4, 4, 'final', 0)) 30 | futureFlags_2_5 = FutureFlags((2, 5, 0, 'final', 0)) 31 | futureFlags_2_7 = FutureFlags((2, 7, 0, 'final', 0)) 32 | futureFlags_3_2 = FutureFlags((3, 2, 0, 'final', 0)) 33 | futureFlags_3_5 = FutureFlags((3, 5, 0, 'final', 0)) 34 | 35 | 36 | class TokenIterator: 37 | def __init__(self, tokens): 38 | self.tokens = tokens 39 | self.index = 0 40 | self.next() 41 | 42 | def next(self): 43 | index = self.index 44 | self.index = index + 1 45 | self.tok = self.tokens[index] 46 | 47 | def skip(self, n): 48 | if self.tok[0] == n: 49 | self.next() 50 | return True 51 | else: 52 | return False 53 | 54 | def skip_name(self, name): 55 | from ayrton.parser.pyparser import pygram 56 | if self.tok[0] == pygram.tokens.NAME and self.tok[1] == name: 57 | self.next() 58 | return True 59 | else: 60 | return False 61 | 62 | def next_feature_name(self): 63 | from ayrton.parser.pyparser import pygram 64 | if self.tok[0] == pygram.tokens.NAME: 65 | name = self.tok[1] 66 | self.next() 67 | if self.skip_name("as"): 68 | self.skip(pygram.tokens.NAME) 69 | return name 70 | else: 71 | return '' 72 | 73 | def skip_newlines(self): 74 | from ayrton.parser.pyparser import pygram 75 | while self.skip(pygram.tokens.NEWLINE): 76 | pass 77 | 78 | 79 | def add_future_flags(future_flags, tokens): 80 | from ayrton.parser.pyparser import pygram 81 | it = TokenIterator(tokens) 82 | result = 0 83 | last_position = (0, 0) 84 | # 85 | # The only things that can precede a future statement are another 86 | # future statement and a doc string (only one). This is a very 87 | # permissive parsing of the given list of tokens; it relies on 88 | # the real parsing done afterwards to give errors. 89 | it.skip_newlines() 90 | it.skip_name("r") or it.skip_name("u") or it.skip_name("ru") 91 | if it.skip(pygram.tokens.STRING): 92 | it.skip_newlines() 93 | 94 | while (it.skip_name("from") and 95 | it.skip_name("__future__") and 96 | it.skip_name("import")): 97 | it.skip(pygram.tokens.LPAR) # optionally 98 | # return in 'last_position' any line-column pair that points 99 | # somewhere inside the last __future__ import statement 100 | # (at the start would be fine too, but it's easier to grab a 101 | # random position inside) 102 | last_position = (it.tok[2], it.tok[3]) 103 | result |= future_flags.get_compiler_feature(it.next_feature_name()) 104 | while it.skip(pygram.tokens.COMMA): 105 | result |= future_flags.get_compiler_feature(it.next_feature_name()) 106 | it.skip(pygram.tokens.RPAR) # optionally 107 | it.skip(pygram.tokens.SEMI) # optionally 108 | it.skip_newlines() 109 | 110 | # remove the flags that were specified but are anyway mandatory 111 | result &= ~future_flags.mandatory_flags 112 | 113 | return result, last_position 114 | -------------------------------------------------------------------------------- /ayrton/parser/pyparser/parser.py: -------------------------------------------------------------------------------- 1 | """ 2 | A CPython inspired RPython parser. 3 | """ 4 | 5 | 6 | class Grammar(object): 7 | """ 8 | Base Grammar object. 9 | 10 | Pass this to ParserGenerator.build_grammar to fill it with useful values for 11 | the Parser. 12 | """ 13 | 14 | def __init__(self): 15 | self.symbol_ids = {} 16 | self.symbol_names = {} 17 | self.symbol_to_label = {} 18 | self.keyword_ids = {} 19 | self.dfas = [] 20 | self.labels = [0] 21 | self.token_ids = {} 22 | self.start = -1 23 | 24 | def shared_copy(self): 25 | new = self.__class__() 26 | new.symbol_ids = self.symbol_ids 27 | new.symbols_names = self.symbol_names 28 | new.keyword_ids = self.keyword_ids 29 | new.dfas = self.dfas 30 | new.labels = self.labels 31 | new.token_ids = self.token_ids 32 | return new 33 | 34 | def _freeze_(self): 35 | # Remove some attributes not used in parsing. 36 | try: 37 | del self.symbol_to_label 38 | del self.symbol_names 39 | del self.symbol_ids 40 | except AttributeError: 41 | pass 42 | return True 43 | 44 | 45 | class Node(object): 46 | 47 | __slots__ = "type value children lineno col_offset".split() 48 | 49 | def __init__(self, type, value, children, lineno, column): 50 | self.type = type 51 | self.value = value 52 | self.children = children 53 | self.lineno = lineno 54 | self.col_offset = column 55 | 56 | def __eq__(self, other): 57 | # For tests. 58 | return (self.type == other.type and 59 | self.value == other.value and 60 | self.children == other.children) 61 | 62 | def __repr__(self): 63 | if self.value is None: 64 | return "Node(type=%s, children=%r)" % (self.type, self.children) 65 | else: 66 | return "Node(type=%s, value=%r)" % (self.type, self.value) 67 | 68 | 69 | class ParseError(Exception): 70 | 71 | def __init__(self, msg, token_type, value, lineno, column, line, 72 | expected=-1): 73 | self.msg = msg 74 | self.token_type = token_type 75 | self.value = value 76 | self.lineno = lineno 77 | self.col_offset = column 78 | self.line = line 79 | self.expected = expected 80 | 81 | def __str__(self): 82 | return "ParserError(%s, %r)" % (self.token_type, self.value) 83 | 84 | 85 | class Parser(object): 86 | 87 | def __init__(self, grammar): 88 | self.grammar = grammar 89 | self.root = None 90 | self.stack = None 91 | 92 | def prepare(self, start=-1): 93 | """Setup the parser for parsing. 94 | 95 | Takes the starting symbol as an argument. 96 | """ 97 | if start == -1: 98 | start = self.grammar.start 99 | self.root = None 100 | current_node = Node(start, None, [], 0, 0) 101 | self.stack = [] 102 | self.stack.append((self.grammar.dfas[start - 256], 0, current_node)) 103 | 104 | def add_token(self, token_type, value, lineno, column, line): 105 | label_index = self.classify(token_type, value, lineno, column, line) 106 | sym_id = 0 # for the annotator 107 | while True: 108 | dfa, state_index, node = self.stack[-1] 109 | states, first = dfa 110 | arcs, is_accepting = states[state_index] 111 | for i, next_state in arcs: 112 | sym_id = self.grammar.labels[i] 113 | if label_index == i: 114 | # We matched a non-terminal. 115 | self.shift(next_state, token_type, value, lineno, column) 116 | state = states[next_state] 117 | # While the only possible action is to accept, pop nodes off 118 | # the stack. 119 | while state[1] and not state[0]: 120 | self.pop() 121 | if not self.stack: 122 | # Parsing is done. 123 | return True 124 | dfa, state_index, node = self.stack[-1] 125 | state = dfa[0][state_index] 126 | return False 127 | elif sym_id >= 256: 128 | sub_node_dfa = self.grammar.dfas[sym_id - 256] 129 | # Check if this token can start a child node. 130 | if label_index in sub_node_dfa[1]: 131 | self.push(sub_node_dfa, next_state, sym_id, lineno, 132 | column) 133 | break 134 | else: 135 | # We failed to find any arcs to another state, so unless this 136 | # state is accepting, it's invalid input. 137 | if is_accepting: 138 | self.pop() 139 | if not self.stack: 140 | raise ParseError("too much input", token_type, value, 141 | lineno, column, line) 142 | else: 143 | # If only one possible input would satisfy, attach it to the 144 | # error. 145 | if len(arcs) == 1: 146 | expected = sym_id 147 | else: 148 | expected = -1 149 | raise ParseError("bad input", token_type, value, lineno, 150 | column, line, expected) 151 | 152 | def classify(self, token_type, value, lineno, column, line): 153 | """Find the label for a token.""" 154 | if token_type == self.grammar.KEYWORD_TOKEN: 155 | label_index = self.grammar.keyword_ids.get(value, -1) 156 | if label_index != -1: 157 | return label_index 158 | label_index = self.grammar.token_ids.get(token_type, -1) 159 | if label_index == -1: 160 | raise ParseError("invalid token", token_type, value, lineno, column, 161 | line) 162 | return label_index 163 | 164 | def shift(self, next_state, token_type, value, lineno, column): 165 | """Shift a non-terminal and prepare for the next state.""" 166 | dfa, state, node = self.stack[-1] 167 | new_node = Node(token_type, value, None, lineno, column) 168 | node.children.append(new_node) 169 | self.stack[-1] = (dfa, next_state, node) 170 | 171 | def push(self, next_dfa, next_state, node_type, lineno, column): 172 | """Push a terminal and adjust the current state.""" 173 | dfa, state, node = self.stack[-1] 174 | new_node = Node(node_type, None, [], lineno, column) 175 | self.stack[-1] = (dfa, next_state, node) 176 | self.stack.append((next_dfa, 0, new_node)) 177 | 178 | def pop(self): 179 | """Pop an entry off the stack and make its node a child of the last.""" 180 | dfa, state, node = self.stack.pop() 181 | if self.stack: 182 | self.stack[-1][2].children.append(node) 183 | else: 184 | self.root = node 185 | -------------------------------------------------------------------------------- /ayrton/parser/pyparser/pygram.py: -------------------------------------------------------------------------------- 1 | import os 2 | from ayrton.parser.pyparser import parser, pytoken, metaparser 3 | 4 | class PythonGrammar(parser.Grammar): 5 | 6 | KEYWORD_TOKEN = pytoken.python_tokens["NAME"] 7 | TOKENS = pytoken.python_tokens 8 | OPERATOR_MAP = pytoken.python_opmap 9 | 10 | def _get_python_grammar(): 11 | here = os.path.dirname(__file__) 12 | fp = open(os.path.join(here, "data", "Grammar3.6")) 13 | try: 14 | gram_source = fp.read() 15 | finally: 16 | fp.close() 17 | pgen = metaparser.ParserGenerator(gram_source) 18 | return pgen.build_grammar(PythonGrammar) 19 | 20 | 21 | python_grammar = _get_python_grammar() 22 | 23 | class _Tokens(object): 24 | pass 25 | for tok_name, idx in pytoken.python_tokens.items(): 26 | setattr(_Tokens, tok_name, idx) 27 | tokens = _Tokens() 28 | 29 | class _Symbols(object): 30 | pass 31 | rev_lookup = {} 32 | for sym_name, idx in python_grammar.symbol_ids.items(): 33 | setattr(_Symbols, sym_name, idx) 34 | rev_lookup[idx] = sym_name 35 | syms = _Symbols() 36 | syms._rev_lookup = rev_lookup # for debugging 37 | 38 | del _get_python_grammar, _Tokens, tok_name, sym_name, idx 39 | -------------------------------------------------------------------------------- /ayrton/parser/pyparser/pytoken.py: -------------------------------------------------------------------------------- 1 | """Python token definitions.""" 2 | 3 | python_tokens = {} 4 | python_opmap = {} 5 | 6 | def _add_tok(name, *values): 7 | index = len(python_tokens) 8 | assert index < 256 9 | python_tokens[name] = index 10 | for value in values: 11 | python_opmap[value] = index 12 | 13 | _add_tok('ENDMARKER') 14 | _add_tok('NAME') 15 | _add_tok('NUMBER') 16 | _add_tok('STRING') 17 | _add_tok('NEWLINE') 18 | _add_tok('INDENT') 19 | _add_tok('DEDENT') 20 | _add_tok('LPAR', "(") 21 | _add_tok('RPAR', ")") 22 | _add_tok('LSQB', "[") 23 | _add_tok('RSQB', "]") 24 | _add_tok('COLON', ":") 25 | _add_tok('COMMA', "," ) 26 | _add_tok('SEMI', ";" ) 27 | _add_tok('PLUS', "+" ) 28 | _add_tok('MINUS', "-" ) 29 | _add_tok('STAR', "*" ) 30 | _add_tok('SLASH', "/" ) 31 | _add_tok('VBAR', "|" ) 32 | _add_tok('AMPER', "&" ) 33 | _add_tok('LESS', "<" ) 34 | _add_tok('GREATER', ">" ) 35 | _add_tok('EQUAL', "=" ) 36 | _add_tok('DOT', "." ) 37 | _add_tok('PERCENT', "%" ) 38 | _add_tok('BACKQUOTE', "`" ) 39 | _add_tok('LBRACE', "{" ) 40 | _add_tok('RBRACE', "}" ) 41 | _add_tok('EQEQUAL', "==" ) 42 | _add_tok('NOTEQUAL', "!=", "<>" ) 43 | _add_tok('LESSEQUAL', "<=" ) 44 | _add_tok('GREATEREQUAL', ">=" ) 45 | _add_tok('TILDE', "~" ) 46 | _add_tok('CIRCUMFLEX', "^" ) 47 | _add_tok('LEFTSHIFT', "<<" ) 48 | _add_tok('RIGHTSHIFT', ">>" ) 49 | _add_tok('DOUBLESTAR', "**" ) 50 | _add_tok('PLUSEQUAL', "+=" ) 51 | _add_tok('MINEQUAL', "-=" ) 52 | _add_tok('STAREQUAL', "*=" ) 53 | _add_tok('SLASHEQUAL', "/=" ) 54 | _add_tok('PERCENTEQUAL', "%=" ) 55 | _add_tok('AMPEREQUAL', "&=" ) 56 | _add_tok('VBAREQUAL', "|=" ) 57 | _add_tok('CIRCUMFLEXEQUAL', "^=" ) 58 | _add_tok('LEFTSHIFTEQUAL', "<<=" ) 59 | _add_tok('RIGHTSHIFTEQUAL', ">>=" ) 60 | _add_tok('DOUBLESTAREQUAL', "**=" ) 61 | _add_tok('DOUBLESLASH', "//" ) 62 | _add_tok('DOUBLESLASHEQUAL',"//=" ) 63 | _add_tok('AT', "@" ) 64 | _add_tok('ATEQUAL', "@=" ) 65 | _add_tok('RARROW', "->") 66 | _add_tok('ELLIPSIS', "...") 67 | _add_tok('OP') 68 | _add_tok('ASYNC') 69 | _add_tok('AWAIT') 70 | _add_tok('ERRORTOKEN') 71 | 72 | # extra PyPy-specific tokens 73 | _add_tok("COMMENT") 74 | _add_tok("NL") 75 | 76 | del _add_tok 77 | -------------------------------------------------------------------------------- /ayrton/parser/pyparser/pytokenize.py: -------------------------------------------------------------------------------- 1 | # ______________________________________________________________________ 2 | """Module pytokenize 3 | 4 | THIS FILE WAS COPIED FROM pypy/module/parser/pytokenize.py AND ADAPTED 5 | TO BE ANNOTABLE (Mainly made lists homogeneous) 6 | 7 | This is a modified version of Ka-Ping Yee's tokenize module found in the 8 | Python standard library. 9 | 10 | The primary modification is the removal of the tokenizer's dependence on the 11 | standard Python regular expression module, which is written in C. The regular 12 | expressions have been replaced with hand built DFA's using the 13 | basil.util.automata module. 14 | 15 | $Id: pytokenize.py,v 1.3 2003/10/03 16:31:53 jriehl Exp $ 16 | """ 17 | # ______________________________________________________________________ 18 | 19 | from ayrton.parser.pyparser import automata 20 | from ayrton.parser.pyparser.dfa_generated import * 21 | 22 | __all__ = [ "tokenize" ] 23 | 24 | endDFAs = {"'" : singleDFA, 25 | '"' : doubleDFA, 26 | 'r' : None, 27 | 'R' : None, 28 | "u" : None, 29 | "U" : None, 30 | 'f' : None, 31 | 'F' : None, 32 | 'b' : None, 33 | 'B' : None} 34 | 35 | for uniPrefix in ("", "b", "B", "f", "F"): 36 | for rawPrefix in ("", "r", "R"): 37 | prefix_1 = uniPrefix + rawPrefix 38 | prefix_2 = rawPrefix + uniPrefix 39 | 40 | endDFAs[prefix_1 + "'''"] = single3DFA 41 | endDFAs[prefix_1 + '"""'] = double3DFA 42 | endDFAs[prefix_2 + "'''"] = single3DFA 43 | endDFAs[prefix_2 + '"""'] = double3DFA 44 | 45 | for uniPrefix in ("u", "U"): 46 | endDFAs[uniPrefix + "'''"] = single3DFA 47 | endDFAs[uniPrefix + '"""'] = double3DFA 48 | 49 | whiteSpaceStatesAccepts = [True] 50 | whiteSpaceStates = [{'\t': 0, ' ': 0, '\x0c': 0}] 51 | whiteSpaceDFA = automata.DFA(whiteSpaceStates, whiteSpaceStatesAccepts) 52 | 53 | # ______________________________________________________________________ 54 | # COPIED: 55 | 56 | triple_quoted = {} 57 | for t in ("'''", '"""', 58 | "r'''", 'r"""', "R'''", 'R"""', 59 | "u'''", 'u"""', "U'''", 'U"""', 60 | "f'''", 'f"""', "F'''", 'F"""', 61 | "fr'''", 'fr"""', "Fr'''", 'Fr"""', 62 | "fR'''", 'fR"""', "FR'''", 'FR"""', 63 | "rf'''", 'rf"""', "rF'''", 'rF"""', 64 | "Rf'''", 'Rf"""', "RF'''", 'RF"""', 65 | "b'''", 'b"""', "B'''", 'B"""', 66 | "br'''", 'br"""', "Br'''", 'Br"""', 67 | "bR'''", 'bR"""', "BR'''", 'BR"""', 68 | "rb'''", 'rb"""', "rB'''", 'rB"""', 69 | "Rb'''", 'Rb"""', "RB'''", 'RB"""'): 70 | triple_quoted[t] = t 71 | single_quoted = {} 72 | for t in ("'", '"', 73 | "r'", 'r"', "R'", 'R"', 74 | "u'", 'u"', "U'", 'U"', 75 | "f'", 'f"', "F'", 'F"', 76 | "fr'", 'fr"', "Fr'", 'Fr"', 77 | "fR'", 'fR"', "FR'", 'FR"', 78 | "rf'", 'rf"', "rF'", 'rF"', 79 | "Rf'", 'Rf"', "RF'", 'RF"', 80 | "b'", 'b"', "B'", 'B"', 81 | "br'", 'br"', "Br'", 'Br"', 82 | "bR'", 'bR"', "BR'", 'BR"', 83 | "rb'", 'rb"', "rB'", 'rB"', 84 | "Rb'", 'Rb"', "RB'", 'RB"'): 85 | 86 | single_quoted[t] = t 87 | 88 | tabsize = 8 89 | alttabsize = 1 90 | 91 | # PYPY MODIFICATION: removed TokenError class as it's not needed here 92 | 93 | # PYPY MODIFICATION: removed StopTokenizing class as it's not needed here 94 | 95 | # PYPY MODIFICATION: removed printtoken() as it's not needed here 96 | 97 | # PYPY MODIFICATION: removed tokenize() as it's not needed here 98 | 99 | # PYPY MODIFICATION: removed tokenize_loop() as it's not needed here 100 | 101 | # PYPY MODIFICATION: removed generate_tokens() as it was copied / modified 102 | # in pythonlexer.py 103 | 104 | # PYPY MODIFICATION: removed main() as it's not needed here 105 | 106 | # ______________________________________________________________________ 107 | # End of pytokenize.py 108 | 109 | -------------------------------------------------------------------------------- /ayrton/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # (c) 2013 Marcos Dione 2 | 3 | # This file is part of ayrton. 4 | # 5 | # ayrton is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # ayrton is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with ayrton. If not, see . 17 | 18 | # nothing 19 | -------------------------------------------------------------------------------- /ayrton/tests/data/string_stdin.txt: -------------------------------------------------------------------------------- 1 | stdin_from_file! 2 | -------------------------------------------------------------------------------- /ayrton/tests/data/test.me.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3.6 2 | 3 | # -*- coding: utf-8 -*- 4 | 5 | # (c) 2013 Marcos Dione 6 | 7 | # This file is part of ayrton. 8 | # 9 | # ayrton is free software: you can redistribute it and/or modify 10 | # it under the terms of the GNU General Public License as published by 11 | # the Free Software Foundation, either version 3 of the License, or 12 | # (at your option) any later version. 13 | # 14 | # ayrton is distributed in the hope that it will be useful, 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | # GNU General Public License for more details. 18 | # 19 | # You should have received a copy of the GNU General Public License 20 | # along with ayrton. If not, see . 21 | 22 | # this is not an ayrton script, but a normal python script to ytest a couple of things 23 | # equivalent to true 24 | import sys 25 | 26 | sys.exit (0) 27 | -------------------------------------------------------------------------------- /ayrton/tests/data/test_environ.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | echo "$testEnviron" 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/package/__init__.ay: -------------------------------------------------------------------------------- 1 | # this module is intentionally left almost blank 2 | 3 | bar= 24 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/package/ay_module.ay: -------------------------------------------------------------------------------- 1 | # this module is intentionally left blank 2 | 3 | ay= True 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/package/py_module.py: -------------------------------------------------------------------------------- 1 | # this module is intentionally left blank 2 | 3 | py= True 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/stderr.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | if [ $# -eq 2 ]; then 4 | out="$1" 5 | err="$2" 6 | else 7 | err="$1" 8 | fi 9 | 10 | if [ -n "$out" ]; then 11 | echo "$out" 12 | fi 13 | 14 | echo "$err" >&2 15 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testAbsoluteExecutables.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # not yet working 4 | /bin/true () 5 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testAssign.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | a= lambda x: x 4 | a (1) 5 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testBg.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | a= find ("/usr", _bg=True, _out=None, _fails=True) 4 | ayrton_return_value= "yes!" 5 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testCallFun.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | def func (): 4 | true () 5 | 6 | func () 7 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testCommandIsPresent.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | c = Command('ls') 4 | c() 5 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testComposing.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | ayrton_return_value= str (grep (ls (), "setup", _out=Capture)) 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testCwdPwdRename.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | import os.path 4 | 5 | # tests that cwd() is properly renamed to pwd() 6 | ayrton_return_value= os.path.split (pwd ())[-1] 7 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testDashInExecutables.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # not yet working 4 | test-me.py () 5 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testDefFun1.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # local definitions do not eclipse global ones ofter the scope finished 4 | def foo (): 5 | true= 40 6 | 7 | true () 8 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testDefFun2.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | option ('errexit') 4 | 5 | def foo (): 6 | false= 41 7 | 8 | false () 9 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testDefFunFails1.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | option ('errexit') 4 | 5 | def foo (): 6 | # local definitions work 7 | false= lambda: True 8 | false () 9 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testDefine.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | define(this_variable_did_not_exist) 4 | 5 | ayrton_return_value= -z(this_variable_did_not_exist) 6 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testDefineAgain.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | define(HOME) 4 | 5 | ayrton_return_value= -z(HOME) 6 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testDefineValue.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | define(this_variable_did_not_exist=6) 4 | 5 | ayrton_return_value = this_variable_did_not_exist 6 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testDel.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | option ('errexit') 4 | 5 | # defining a local... 6 | false= lambda: x 7 | # ... and deleting it... 8 | del false 9 | # still allows us to call executables 10 | false () 11 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testDotInExecutables.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # names with dots work 4 | test.me.py () 5 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testEnvVarAsGlobalVar.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # see ayrton/tests/test_ayrton.py:MiscTests:testEnvVarAsGlobalVar() 4 | ayrton_return_value= testEnvVarAsGlobalVar 5 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testEnviron.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # exporting variables... 4 | export (testEnviron=44); 5 | 6 | # ... are seen in subprocesses 7 | ayrton_return_value= str (run ("./ayrton/tests/data/test_environ.sh", _out=Capture)) 8 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testExit0.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | exit (0) 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testExit1.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | exit (1) 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testExitCodeNOK.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # failing executables can be tested as booleans 4 | if not false(_fails=True): 5 | ayrton_return_value = "yes!" 6 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testExitCodeOK.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # successful executables can be tested as booleans 4 | if true (): 5 | ayrton_return_value= "yes!" 6 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testExportSetsGlobalVar.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # exported variables are seen as global variables 4 | export (foo=47) 5 | 6 | ayrton_return_value= foo 7 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testFails.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | false (_fails=True) 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testForDefinesTarget.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | for line in [ '' ]: 4 | line.strip () 5 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testForDefinesTargets.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | for i, line in enumerate ([ '0' ]): 4 | line.strip () 5 | line[i] 6 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testFromImport.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # I can call functions imported 4 | from random import seed 5 | 6 | seed () 7 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testFromImportAs.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # we can call aliases for imported functions 4 | from random import seed as foo 5 | 6 | foo () 7 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testFromImportAsFails.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | from random import seed as foo 4 | 5 | bar () # CommandNotFound 6 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testFromImportFails.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | from random import seed 4 | 5 | foo () # CommandNotFound 6 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testGt.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | echo ("yes") > argv[1] 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testGtToUnwritableFile.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | option('+e') 4 | 5 | echo ("yes") > "/foo/bar/baz/quux/moo" 6 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testImport.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # import still works 4 | import math 5 | 6 | math.floor (1.1) 7 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testImportCallFromFunc.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # import works and is global 4 | import math 5 | 6 | def foo (): 7 | math.floor (1.1) 8 | 9 | foo () 10 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testImportFrom.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # from ... import works 4 | from math import floor 5 | 6 | floor (1.1) 7 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testImportFromCallFromFunc.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # we didn't break from ... import! 4 | from math import floor 5 | 6 | def foo (): 7 | floor (1.1) 8 | 9 | foo () 10 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testImportLocalAy.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # importing local ayrton modules works 4 | import testImportLocalAyModule 5 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testImportLocalAyModule.ay: -------------------------------------------------------------------------------- 1 | # this module is intentionally left almost blank 2 | 3 | foo= 42 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testImportLocalAyPackage.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # importing ayrton packages works 4 | import package 5 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testImportLocalAyPackageAyModule.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # importing ayrton modules in ayrton packages works 4 | import package.ay_module 5 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testImportLocalAyPackagePyModule.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # importing python modules in ayrton packages works 4 | import package.py_module 5 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testImportLocalPy.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # importing local modules still works 4 | import testImportLocalPyModule 5 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testImportLocalPyModule.py: -------------------------------------------------------------------------------- 1 | # this module is intentionally left blank 2 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testInstantiateClass.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # we can define classes 4 | class Foo (): 5 | pass 6 | 7 | foo= Foo () 8 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testLocalVarToRealRemoteToLocal.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # we can define a local variable... 4 | testLocalVarToRealRemoteToLocal= False 5 | 6 | with remote ('127.0.0.1', _test=True): 7 | # ... modify it in the remote... 8 | testLocalVarToRealRemoteToLocal= True 9 | 10 | # .. and see the modification locally 11 | assert (testLocalVarToRealRemoteToLocal) 12 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testLongIter.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | for line in find ("/usr", _fails=True): 4 | pass 5 | 6 | ayrton_return_value= "yes!" 7 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testLongOutput.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | lines= find ("/usr", _out=Capture, _fails=True) 4 | 5 | ayrton_return_value= "yes!" 6 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testLongPipe.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # long piping works, and so do -o(ptions) 4 | ayrton_return_value= str (ls () | grep ("setup") | wc (-l=True, _out=Capture)) 5 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testLt.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | cat () < argv[1] 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testLtGt.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # double redirection 4 | cat () < argv[1] > argv[2] 5 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testNotExecuteLiteralMethods.ay: -------------------------------------------------------------------------------- 1 | ".".join ([]) 2 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testO.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # this was an intermediate way to specify named arguments in any position 4 | # which is not even properly demonstrated here 5 | ls (o (full_time=True), _fails=True) 6 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testOptionETrue.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # equivalent to -e 4 | option (e=True) 5 | 6 | false () # CommandFailed 7 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testOptionErrexit.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # equivalent to -e 4 | option ('errexit') 5 | 6 | false () # CommandFailed 7 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testOptionMinus_e.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | option ('-e') 4 | 5 | false () # CommandFailed 6 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testOptionPlus_e.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | option ('+e') 4 | 5 | false () # doesn't fail 6 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testPipe.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | ayrton_return_value= str (ls () | grep ("setup", _out=Capture)) 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testRShift.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | echo ('yes') > argv[1] 4 | echo ('yes!') >> argv[1] 5 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testRelativeExecutables.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # does not work yet 4 | more/test.me.py () 5 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testRemoteCommandStderr.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | # _test=True is for internal testing 4 | with remote ('127.0.0.1', _test=True): 5 | ls('foobarbaz', _fails=True) 6 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testRemoteCommandStdin.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | with remote ('127.0.0.1', _test=True): 4 | input ('type here: ') 5 | input ('type here too: ') 6 | run ('bash') 7 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testRemoteCommandStdout.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | with remote ('127.0.0.1', _test=True): 4 | # stdout from the remote is seen in the local 5 | ls(-l=True) 6 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testShift.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | ayrton_return_value= shift () 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testShiftArray.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | l = [ 1, 2, 3 ] 4 | 5 | ayrton_return_value = shift (l, 2) 6 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testShifts.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | ayrton_return_value= shift (2) 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testShortIter.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | for line in echo ('yes!'): 4 | ayrton_return_value= line.strip () 5 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testSimpleCase.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | true () 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testSimpleCaseFails.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | foo () # CommandNotfound 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testStdout.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | echo ('foo') 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testStdoutEqCapture.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | ayrton_return_value= str (echo ("foo", _out=Capture)) 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testStdoutEqNone.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | echo ('foo', _out=None) 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testUnset.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | ayrton_return_value= [] 4 | 5 | # values are converted to str... 6 | export (testUnset=45) 7 | # ... so this appends '45' 8 | ayrton_return_value.append (testUnset) 9 | unset (testUnset) 10 | 11 | try: 12 | testUnset 13 | except NameError: 14 | ayrton_return_value.append ("yes") 15 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testWithCd.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | import os.path 4 | 5 | # this change of directory does not affect the code outside the with 6 | with cd ("bin"): 7 | pass 8 | 9 | ayrton_return_value= os.path.split (pwd ())[-1] 10 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testZEmptyString.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | ayrton_return_value = -z('') 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testZEnvVar.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | ayrton_return_value= -z(HOME) 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testZInt.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | ayrton_return_value= -z(0) 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testZNone.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | ayrton_return_value= -z(None) 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testZNotDefined.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | ayrton_return_value= -z(this_variable_does_not_exist) 4 | -------------------------------------------------------------------------------- /ayrton/tests/scripts/testZString.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | ayrton_return_value= -z('a string') 4 | -------------------------------------------------------------------------------- /ayrton/tests/test_expansion.py: -------------------------------------------------------------------------------- 1 | # (c) 2016 Marcos Dione 2 | 3 | # This file is part of ayrton. 4 | # 5 | # ayrton is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # ayrton is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with ayrton. If not, see . 17 | 18 | import unittest 19 | import os 20 | 21 | from ayrton.expansion import * 22 | import ayrton 23 | from ayrton.execute import Command, Capture 24 | 25 | import logging 26 | 27 | logger= logging.getLogger ('ayton.tests.expansion') 28 | 29 | # create one of these 30 | ayrton.runner= ayrton.Ayrton () 31 | real_bash= Command ('bash') 32 | 33 | class ExpansionTest (unittest.TestCase): 34 | 35 | def doTest (function, *args, single=False): 36 | bash_answer= real_bash ({ '-c': "echo %s" % string_to_expand }) 37 | bash_answer= bash_answer.readline ().split () 38 | if single: 39 | bash_answer= bash_answer[0] 40 | 41 | self.assertEqual (function (*args), bash_answer) 42 | 43 | class TildeExpansion (unittest.TestCase): 44 | 45 | def test_tilde (self): 46 | self.assertEqual (tilde_expand ('~'), os.environ['HOME']) 47 | 48 | def test_tilde_user (self): 49 | self.assertEqual (tilde_expand ('~root'), '/root') 50 | 51 | def test_tilde_user_more (self): 52 | self.assertEqual (tilde_expand ('~root/.'), '/root/.') 53 | 54 | 55 | class ParameterExpansion (unittest.TestCase): 56 | # used by some tests 57 | value= '0123456789' 58 | 59 | def test_default_undefined (self): 60 | self.assertRaises (NameError, default, 'foo', 'bar') 61 | 62 | def test_default_empty (self): 63 | ayrton.runner.globals['foo']= '' 64 | 65 | self.assertEqual (default ('foo', 'bar'), 'bar') 66 | 67 | del ayrton.runner.globals['foo'] 68 | 69 | def test_default_non_empty (self): 70 | ayrton.runner.globals['foo']= 'baz' 71 | 72 | self.assertEqual (default ('foo', 'bar'), 'baz') 73 | 74 | del ayrton.runner.globals['foo'] 75 | 76 | 77 | # according to bash's manpage, default's second parameter should be expanded 78 | # but tests have shown that it is not so 79 | # def test_default_expanded (self): 80 | # self.assertEqual (default ('foo', '~root'), '/root') 81 | 82 | def test_replace_if_set_undef (self): 83 | self.assertRaises (NameError, replace_if_set, 'foo', 'bar') 84 | 85 | def test_replace_if_set_empty (self): 86 | ayrton.runner.globals['foo']= '' 87 | 88 | self.assertEqual (replace_if_set ('foo', 'bar'), '') 89 | 90 | del ayrton.runner.globals['foo'] 91 | 92 | 93 | def do_substr_test (self, offset, length, value): 94 | ayrton.runner.globals['foo']= self.value 95 | 96 | self.assertEqual (substr ('foo', offset, length), value) 97 | 98 | del ayrton.runner.globals['foo'] 99 | 100 | def test_substr_offset (self): 101 | self.do_substr_test (4, None, self.value[4:]) 102 | 103 | def test_substr_offset_length (self): 104 | self.do_substr_test (4, 2, self.value[4:6]) 105 | 106 | def test_substr_offset_too_big (self): 107 | self.do_substr_test (10, None, '') 108 | 109 | def test_substr_offset_length_too_big (self): 110 | self.do_substr_test (4, 10, self.value[4:]) 111 | 112 | def test_substr_offset_too_big_length (self): 113 | self.do_substr_test (10, 2, '') 114 | 115 | def test_substr_negative_offset (self): 116 | self.do_substr_test (4, None, self.value[4:]) 117 | 118 | 119 | class BraceExpansion(unittest.TestCase): 120 | 121 | def test_simple1_brace (self): 122 | self.assertEqual (bash ('{acde,b}'), [ 'acde', 'b' ]) 123 | 124 | def test_simple2_brace (self): 125 | self.assertEqual (bash ('a{b,ce}d'), [ 'abd', 'aced' ]) 126 | 127 | def test_simple3_brace (self): 128 | self.assertEqual (bash ('{a}'), [ '{a}' ]) 129 | 130 | def test_simple3_brace_single (self): 131 | self.assertEqual (bash ('{a}', single=True), '{a}') 132 | 133 | def test_simple4_brace (self): 134 | self.assertEqual (bash ('a}'), [ 'a}' ]) 135 | 136 | def test_simple4_brace_single (self): 137 | self.assertEqual (bash ('a}', single=True), 'a}') 138 | 139 | def test_simple5_brace (self): 140 | self.assertEqual (bash ('a{bfgh,{ci,djkl}e'), [ 'a{bfgh,cie', 'a{bfgh,djkle' ]) 141 | 142 | def test_simple6_brace (self): 143 | self.assertEqual (bash ('{a,{b,c}d}'), [ 'a', 'bd', 'cd' ]) 144 | 145 | def test_simple7_brace (self): 146 | self.assertEqual (bash ('foo{,bar}'), [ 'foo', 'foobar' ]) 147 | 148 | def test_nested1_brace (self): 149 | # note how this is equivalent to a{b,c,d}e! 150 | self.assertEqual (bash ('a{b,{c,d}}e'), [ 'abe', 'ace', 'ade' ]) 151 | 152 | def test_nested2_brace (self): 153 | self.assertEqual (bash ('{c{a,b}d,e{f,g}h}'), [ 'cad', 'cbd', 'efh', 'egh' ]) 154 | 155 | def test_escaped_brace (self): 156 | self.assertEqual (bash ('\{a,b}'), [ '{a,b}' ]) 157 | 158 | def test_escaped_brace_single (self): 159 | self.assertEqual (bash ('\{a,b}', single=True), '{a,b}') 160 | 161 | def test_escaped_brace_inside (self): 162 | self.assertEqual (bash ('{\{a,b}'), [ '{a', 'b' ]) 163 | 164 | def test_escaped_comma (self): 165 | self.assertEqual (bash ('{a\,b}'), [ '{a,b}' ]) 166 | 167 | def test_bam (self): 168 | self.assertEqual (bash (','), [ ',' ]) 169 | 170 | def test_real_example1 (self): 171 | # tiles/{legend*,Elevation.dgml,preview.png,Makefile} 172 | pass 173 | 174 | 175 | # class SequenceExpressionExpansion(unittest.TestCase): 176 | class SequenceExpressionExpansion(): 177 | 178 | def test_bam (self): 179 | self.assertEqual (bash ('{.2}'), [ '{.2}' ]) 180 | 181 | def test_not_really (self): 182 | self.assertEqual (bash ('{1.2}'), [ '{1.2}' ]) 183 | 184 | def test_simple (self): 185 | self.assertEqual (bash ('{1..2}'), [ '1', '2' ]) 186 | 187 | def do_not_test_more (self): 188 | self.assertEqual (bash ('{1..3}'), [ '1', '2', '3' ]) 189 | 190 | def do_not_test_escaped_dot_dot (self): 191 | self.assertEqual (bash ('{1\..2}'), [ '{1..2}' ]) 192 | 193 | def do_not_test_dot_escaped_dot (self): 194 | self.assertEqual (bash ('{1.\.2}'), [ '{1..2}' ]) 195 | 196 | 197 | class Bash(unittest.TestCase): 198 | def test_simple_string (self): 199 | self.assertEqual (bash ('s'), [ 's' ]) 200 | 201 | def test_simple_string_single (self): 202 | self.assertEqual (bash ('s', single=True), 's') 203 | 204 | def test_glob1 (self): 205 | self.assertEqual (bash ('*.py'), [ 'setup.py' ]) 206 | 207 | def test_glob1_single (self): 208 | self.assertEqual (bash ('*.py', single=True), 'setup.py') 209 | 210 | def test_glob2 (self): 211 | self.assertEqual (sorted (bash ([ '*.py', '*.txt' ])), [ 'LICENSE.txt', 'requirements.txt', 'setup.py', ]) 212 | 213 | def test_glob_brace1 (self): 214 | self.assertEqual (sorted (bash ('s{a,*.py}')), [ 'sa', 'setup.py' ]) 215 | 216 | def test_glob_brace2 (self): 217 | self.assertEqual (sorted (bash ('ayrton/tests/data/{a,*.py}')), [ 'ayrton/tests/data/a', 'ayrton/tests/data/test.me.py' ]) 218 | 219 | def test_tilde (self): 220 | self.assertEqual (bash ('~'), [ os.environ['HOME'] ]) 221 | 222 | def test_tilde_single (self): 223 | self.assertEqual (bash ('~', single=True), os.environ['HOME']) 224 | -------------------------------------------------------------------------------- /ayrton/tests/test_file_tests.py: -------------------------------------------------------------------------------- 1 | # (c) 2016 Marcos Dione 2 | 3 | # This file is part of ayrton. 4 | # 5 | # ayrton is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # ayrton is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with ayrton. If not, see . 17 | 18 | import unittest 19 | import os 20 | import tempfile 21 | import time 22 | 23 | from ayrton.file_test import * 24 | import ayrton 25 | from ayrton.execute import CommandNotFound 26 | 27 | import logging 28 | 29 | logger= logging.getLogger ('ayrton.tests.file_tests') 30 | 31 | # create one of these 32 | # ayrton.runner= ayrton.Ayrton () 33 | 34 | class TestFalseBool (unittest.TestCase): 35 | 36 | def test_true (self): 37 | self.assertTrue (FalseBool (True)) 38 | 39 | def test_false (self): 40 | self.assertFalse (FalseBool (False)) 41 | 42 | def test_neg_true (self): 43 | self.assertTrue (-FalseBool (True)) 44 | 45 | def test_neg_false (self): 46 | self.assertFalse (-FalseBool (False)) 47 | 48 | class FileTests (unittest.TestCase): 49 | 50 | def touch_file (self, text=None): 51 | fd, name= tempfile.mkstemp (suffix='.ayrtmp') 52 | self.addCleanup (os.unlink, name) 53 | 54 | if text is not None: 55 | os.write (fd, text) 56 | 57 | os.close (fd) 58 | 59 | return name 60 | 61 | 62 | def touch_dir (self): 63 | name= tempfile.mkdtemp (suffix='.ayrtmp') 64 | self.addCleanup (os.rmdir, name) 65 | 66 | return name 67 | 68 | 69 | def touch_symlink (self, dir=False): 70 | if dir: 71 | name= self.touch_dir () 72 | else: 73 | name= self.touch_file () 74 | 75 | # yes, mktemp() is deprecated, but this is a bloody test 76 | # and there's no mkltemp() 77 | link= tempfile.mktemp (suffix='.ayrtmp') 78 | os.symlink (name, link) 79 | self.addCleanup (os.unlink, link) 80 | 81 | return link 82 | 83 | 84 | def test_a_file (self): 85 | name= self.touch_file () 86 | self.assertTrue (-a (name)) 87 | 88 | def test_a_symlink (self): 89 | name= self.touch_symlink () 90 | self.assertTrue (-a (name)) 91 | 92 | def test_a_dir (self): 93 | name= self.touch_dir () 94 | self.assertTrue (-a (name)) 95 | 96 | # TODO: -b 97 | 98 | # TODO: -c 99 | 100 | 101 | def test_d_file (self): 102 | name= self.touch_file () 103 | self.assertFalse (-d (name)) 104 | 105 | def test_d_symfile (self): 106 | name= self.touch_symlink () 107 | self.assertFalse (-d (name)) 108 | 109 | def test_d_dir (self): 110 | name= self.touch_dir () 111 | self.assertTrue (-d (name)) 112 | 113 | def test_d_symdir (self): 114 | name= self.touch_symlink (dir=True) 115 | self.assertTrue (-d (name)) 116 | 117 | 118 | def test_e (self): 119 | name= self.touch_file () 120 | self.assertTrue (-e (name)) 121 | 122 | 123 | def test_f_file (self): 124 | name= self.touch_file () 125 | self.assertTrue (-f (name)) 126 | 127 | def test_f_symfile (self): 128 | name= self.touch_symlink () 129 | self.assertTrue (-f (name)) 130 | 131 | def test_f_dir (self): 132 | name= self.touch_dir () 133 | self.assertFalse (-f (name)) 134 | 135 | def test_f_symdir (self): 136 | name= self.touch_symlink (dir=True) 137 | self.assertFalse (-f (name)) 138 | 139 | 140 | # TODO: -g 141 | 142 | 143 | def test_h_file (self): 144 | name= self.touch_file () 145 | self.assertFalse (-h (name)) 146 | 147 | def test_h_symfile (self): 148 | name= self.touch_symlink () 149 | self.assertTrue (-h (name)) 150 | 151 | def test_h_dir (self): 152 | name= self.touch_dir () 153 | self.assertFalse (-h (name)) 154 | 155 | def test_h_symdir (self): 156 | name= self.touch_symlink (dir=True) 157 | self.assertTrue (-h (name)) 158 | 159 | 160 | # TODO: -k 161 | 162 | # TODO: -p 163 | 164 | 165 | def test_r_readable (self): 166 | name= self.touch_file () 167 | self.assertTrue (-r (name)) 168 | 169 | def test_r_unreadable (self): 170 | name= self.touch_file () 171 | os.chmod (name, 0) 172 | self.assertFalse (-r (name)) 173 | 174 | 175 | def test_s_empty (self): 176 | name= self.touch_file () 177 | self.assertFalse (-s (name)) 178 | 179 | def test_s_something (self): 180 | name= self.touch_file (text=b'foo') 181 | self.assertTrue (-s (name)) 182 | 183 | 184 | # TODO: -u 185 | 186 | 187 | def test_w_writable (self): 188 | name= self.touch_file () 189 | self.assertTrue (-w (name)) 190 | 191 | def test_w_nonwritable (self): 192 | name= self.touch_file () 193 | os.chmod (name, 0) 194 | self.assertFalse (-w (name)) 195 | 196 | def test_x_file (self): 197 | name= self.touch_file () 198 | # file are not executable by default 199 | self.assertFalse (-x (name)) 200 | 201 | def test_x_dir (self): 202 | name= self.touch_dir () 203 | # but dirs are 204 | self.assertTrue (-x (name)) 205 | 206 | 207 | # TODO: -N 208 | 209 | # TODO: -S 210 | 211 | 212 | def test_nt_no_file2 (self): 213 | name1= self.touch_file () 214 | name2= tempfile.mktemp (suffix='.ayrtmp') 215 | self.assertTrue (-nt (name1, name2)) 216 | 217 | def test_nt_true (self): 218 | name2= self.touch_file (b'foo') 219 | # I need a noticeable diff in times 220 | time.sleep (0.1) 221 | # name1 must be newer than name2 222 | name1= self.touch_file (b'bar') 223 | self.assertTrue (-nt (name1, name2)) 224 | 225 | def test_nt_false (self): 226 | # name1 must be older than name2 227 | name1= self.touch_file (b'foo') 228 | time.sleep (0.1) 229 | name2= self.touch_file (b'bar') 230 | self.assertFalse (-nt (name1, name2)) 231 | 232 | 233 | def test_ot_no_file1 (self): 234 | name1= tempfile.mktemp (suffix='.ayrtmp') 235 | name2= self.touch_file () 236 | self.assertTrue (-ot (name1, name2)) 237 | 238 | def test_ot_true (self): 239 | # name1 must be older than name2 240 | name1= self.touch_file (b'foo') 241 | time.sleep (0.1) 242 | name2= self.touch_file (b'bar') 243 | self.assertTrue (-ot (name1, name2)) 244 | 245 | def test_ot_false (self): 246 | name2= self.touch_file (b'foo') 247 | time.sleep (0.1) 248 | # name1 must be newer than name2 249 | name1= self.touch_file (b'bar') 250 | self.assertFalse (-ot (name1, name2)) 251 | -------------------------------------------------------------------------------- /ayrton/tests/test_pyparser.py: -------------------------------------------------------------------------------- 1 | # (c) 2015 Marcos Dione 2 | 3 | # This file is part of ayrton. 4 | # 5 | # ayrton is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # ayrton is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with ayrton. If not, see . 17 | 18 | import unittest 19 | from ayrton.parser.pyparser.pyparse import PythonParser, CompileInfo 20 | from ayrton.parser.astcompiler.astbuilder import ast_from_node 21 | import ast 22 | from functools import reduce 23 | import operator 24 | 25 | class Parser(unittest.TestCase): 26 | def setUp (self): 27 | self.parser= PythonParser (None) 28 | self.info= CompileInfo('test_parser.py', 'exec') 29 | 30 | def equal (self, a, b): 31 | # ast.AST does not has a good definitoion of __eq__ 32 | # so I have to make my own 33 | eq= True 34 | # print (a, b) 35 | 36 | try: 37 | items= a.__dict__.items () 38 | except AttributeError: 39 | # no __dict__ means simple value 40 | # just compare them directly 41 | # print (a, b, a==b) 42 | eq= a==b 43 | else: 44 | for k, v1 in items: 45 | try: 46 | v2= b.__dict__[k] 47 | except KeyError: 48 | # print ('here3') 49 | eq= False 50 | break 51 | else: 52 | # print (k, v1, v2) 53 | # skip special stuff 54 | if k.startswith ('__'): 55 | continue 56 | # methods too 57 | if callable (v1): 58 | continue 59 | 60 | if type (v1) in (list, tuple): 61 | for e1, e2 in zip (v1, v2): 62 | if not self.equal (e1, e2): 63 | # print ('here1') 64 | eq= False 65 | break 66 | else: 67 | eq= eq and self.equal (v1, v2) 68 | if not eq: 69 | # print ('here2') 70 | break 71 | 72 | # print (eq) 73 | return eq 74 | 75 | def parse (self, source): 76 | t= self.parser.parse_source (source, self.info) 77 | ast1= ast_from_node (None, t, self.info) 78 | ast2= ast.parse (source) 79 | self.assertTrue (self.equal (ast1, ast2), 80 | "\n%s != \n%s" % (ast.dump (ast1), ast.dump (ast2))) 81 | 82 | def test_comp (self): 83 | self.parse (b'[ x for x in foo() ]') 84 | 85 | def test_comp_if (self): 86 | self.parse (b'[ x for x in foo() if x ]') 87 | 88 | 89 | def test_elif (self): 90 | self.parse (b'''if True: 91 | pass 92 | elif False: 93 | pass 94 | else: 95 | pass''') 96 | -------------------------------------------------------------------------------- /ayrton/tests/test_remote.py: -------------------------------------------------------------------------------- 1 | # (c) 2015 Marcos Dione 2 | 3 | # This file is part of ayrton. 4 | # 5 | # ayrton is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # ayrton is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with ayrton. If not, see . 17 | 18 | import unittest 19 | import unittest.case 20 | import sys 21 | import io 22 | import os 23 | import tempfile 24 | import os.path 25 | import time 26 | import signal 27 | from socket import socket, AF_INET, SOCK_STREAM, SO_REUSEADDR, SOL_SOCKET 28 | from tempfile import mkstemp 29 | import traceback 30 | import paramiko.ssh_exception 31 | 32 | from ayrton.expansion import bash 33 | import ayrton 34 | from ayrton.execute import CommandNotFound 35 | from ayrton.utils import copy_loop, close 36 | 37 | import logging 38 | 39 | logger= logging.getLogger ('ayrton.tests.remote') 40 | 41 | 42 | class OtherFunctions (unittest.TestCase): 43 | 44 | def test_copy_loop_pipe (self): 45 | data= 'yabadabadoo' 46 | 47 | pipe= os.pipe () 48 | self.addCleanup (close, pipe[0]) 49 | self.addCleanup (close, pipe[1]) 50 | 51 | with open (pipe[1], 'w') as w: 52 | w.write (data) 53 | 54 | r= pipe[0] 55 | w, dst= mkstemp (suffix='.ayrtmp', dir='.') 56 | self.addCleanup (os.unlink, dst) 57 | 58 | copy_loop ({ r: w }, buf_len=4) 59 | 60 | with open (dst) as r: 61 | self.assertEqual (r.read (), data) 62 | 63 | 64 | class RemoteTests (unittest.TestCase): 65 | 66 | def setUp (self): 67 | # create one of these 68 | self.runner= ayrton.Ayrton () 69 | 70 | 71 | class DebugRemoteTests (RemoteTests): 72 | 73 | def setUp (self): 74 | super ().setUp () 75 | 76 | # fork and execute nc 77 | pid= os.fork () 78 | if pid != 0: 79 | logger.debug ('main parent') 80 | # parent 81 | self.addCleanup (os.waitpid, pid, 0) 82 | 83 | # give nc time to come up 84 | time.sleep (0.2) 85 | else: 86 | try: 87 | # child 88 | logger.debug('nc') 89 | 90 | server = socket(AF_INET, SOCK_STREAM) 91 | logger.debug(server.fileno()) 92 | server.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) 93 | server.settimeout(1) 94 | server.bind(('127.0.0.1', 2233)) 95 | server.listen() 96 | 97 | client, _ = server.accept() 98 | logger.debug(client.fileno()) 99 | # bash resets the O_ONONBLOCK flag 100 | # see below 101 | # client.settimeout (3) 102 | close (server) 103 | 104 | logger.debug ('bash') 105 | os.dup2 (client.fileno (), 0) 106 | os.dup2 (client.fileno (), 1) 107 | close (client) 108 | 109 | pid= os.fork () 110 | if pid==0: 111 | # child vvvv-- don't forget argv[0] 112 | os.execlp ('bash', 'bash') 113 | # NOTE: does not return 114 | # but if there's a bug, it might 115 | else: 116 | # implement timeout like this 117 | time.sleep (2) 118 | try: 119 | os.kill (pid, signal.SIGKILL) 120 | except ChildProcessError: 121 | pass 122 | os.waitpid (pid, 0) 123 | 124 | # pass through finally 125 | except Exception as e: 126 | logger.debug ('*BOOM*') 127 | traceback.print_exc () 128 | logger.debug (traceback.format_exc ()) 129 | finally: 130 | # connect with the unittest which can be waiting forever 131 | with socket (AF_INET, SOCK_STREAM) as client: 132 | try: 133 | client.connect (('127.0.0.1', 4227)) 134 | except ConnectionRefusedError: 135 | pass 136 | 137 | self.commit_suicide () 138 | 139 | def commit_suicide (self): 140 | # suicide blonde 141 | logger.debug ('Goodbye, cruel wrold!') 142 | os.kill (os.getpid (), signal.SIGKILL) 143 | 144 | 145 | def testRemoteEnv (self): 146 | self.runner.run_script ('''with remote ('127.0.0.1', _test=True, _ncserver=True): 147 | user= USER''', 'testRemoteEnv.py') 148 | 149 | self.assertEqual (self.runner.locals['user'], os.environ['USER']) 150 | 151 | 152 | def testRemoteVar (self): 153 | self.runner.run_script ('''with remote ('127.0.0.1', _test=True, _ncserver=True): 154 | testRemoteVar= 56''', 'testRemoteVar.py') 155 | 156 | self.assertEqual (self.runner.locals['testRemoteVar'], 56) 157 | 158 | 159 | def testRaisesInternal (self): 160 | self.runner.run_script ('''raised= False 161 | 162 | try: 163 | with remote ('127.0.0.1', _test=True, _ncserver=True): 164 | raise SystemError() 165 | except SystemError: 166 | raised= True''', 'testRaisesInternal.py') 167 | 168 | self.assertTrue (self.runner.locals['raised']) 169 | 170 | 171 | def testRaisesExternal (self): 172 | self.assertRaises (SystemError, self.runner.run_script, 173 | '''with remote ('127.0.0.1', _test=True, _ncserver=True): 174 | raise SystemError()''', 'testRaisesExternal.py') 175 | 176 | 177 | def testLocalVarToRemote (self): 178 | self.runner.run_script ('''testLocalVarToRemote= True 179 | 180 | with remote ('127.0.0.1', _test=True, _ncserver=True): 181 | assert (testLocalVarToRemote)''', 'testLocalVarToRemote.py') 182 | 183 | 184 | def __testLocalFunToRemote (self): 185 | self.runner.run_script ('''def testLocalFunToRemote(): pass 186 | 187 | with remote ('127.0.0.1', _test=True, _ncserver=True): 188 | testLocalFunToRemote''', 'testLocalFunToRemote.py') 189 | 190 | 191 | def __testLocalClassToRemote (self): 192 | self.runner.run_script ('''class TestLocalClassToRemote: pass 193 | 194 | with remote ('127.0.0.1', _test=True, _ncserver=True): 195 | TestLocalClassToRemote''', 'testLocalClassToRemote.py') 196 | 197 | 198 | def testRemoteVarToLocal (self): 199 | self.runner.run_script ('''with remote ('127.0.0.1', _test=True, _ncserver=True): 200 | testRemoteVarToLocal= True''', 'testRemoteVarToLocal.py') 201 | 202 | self.assertTrue (self.runner.locals['testRemoteVarToLocal']) 203 | 204 | 205 | def testLocalVarToRemoteToLocal (self): 206 | self.runner.run_script ('''testLocalVarToRemoteToLocal= False 207 | 208 | with remote ('127.0.0.1', _test=True, _ncserver=True): 209 | testLocalVarToRemoteToLocal= True''', 'testLocalVarToRemoteToLocal.py') 210 | 211 | self.assertTrue (self.runner.locals['testLocalVarToRemoteToLocal']) 212 | 213 | 214 | def testRemoteCommandStdout (self): 215 | self.runner.run_script ('''with remote ('127.0.0.1', _test=True, _ncserver=True): 216 | ls(-l=True)''', 'testRemoteCommand.py') 217 | 218 | 219 | def testRemoteCommandStderr (self): 220 | self.runner.run_script ('''with remote ('127.0.0.1', _test=True, _ncserver=True): 221 | ls('foobarbaz', _fails=True)''', 'testRemoteCommand.py') 222 | 223 | 224 | def skip_if_AuthException (test): 225 | def wrapper (self): 226 | try: 227 | test (self) 228 | except paramiko.ssh_exception.AuthenticationException: 229 | self.skipTest ("""This test only succeeds if you you have password/passphrase-less access to localhost""") 230 | 231 | return wrapper 232 | 233 | class RealRemoteTests (RemoteTests): 234 | 235 | @skip_if_AuthException 236 | def testLocalVarToRemoteToLocal (self): 237 | self.runner.run_file ('ayrton/tests/scripts/testLocalVarToRealRemoteToLocal.ay') 238 | 239 | self.assertTrue (self.runner.locals['testLocalVarToRealRemoteToLocal']) 240 | 241 | 242 | @skip_if_AuthException 243 | def testRemoteCommandStdout (self): 244 | self.runner.run_file ('ayrton/tests/scripts/testRemoteCommandStdout.ay') 245 | 246 | 247 | @skip_if_AuthException 248 | def testRemoteCommandStderr (self): 249 | self.runner.run_file ('ayrton/tests/scripts/testRemoteCommandStderr.ay') 250 | -------------------------------------------------------------------------------- /ayrton/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # (c) 2015 Marcos Dione 4 | 5 | # This file is part of ayrton. 6 | # 7 | # ayrton is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # ayrton is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with ayrton. If not, see . 19 | 20 | import functools 21 | from selectors import DefaultSelector, EVENT_READ 22 | import os 23 | from socket import socket 24 | import itertools 25 | import errno 26 | import paramiko.channel 27 | import traceback 28 | 29 | import logging 30 | # I hope this is not toot late... 31 | import ayrton.logger 32 | logger= logging.getLogger ('ayrton.utils') 33 | 34 | 35 | # TODO: there's probably a better way to do this 36 | # TODO: move to logger 37 | def debug2 (self, msg, *args, **kwargs): # pragma: no cover 38 | if self.manager.disable>=logging.DEBUG2: 39 | return 40 | 41 | if logging.DEBUG2>=self.getEffectiveLevel (): 42 | self._log (logging.DEBUG2, msg, args, **kwargs) 43 | 44 | 45 | def debug3 (self, msg, *args, **kwargs): # pragma: no cover 46 | if self.manager.disable>=logging.DEBUG3: 47 | return 48 | 49 | if logging.DEBUG3>=self.getEffectiveLevel (): 50 | self._log (logging.DEBUG3, msg, args, **kwargs) 51 | 52 | 53 | def patch_logging (): 54 | # based on https://mail.python.org/pipermail/tutor/2007-August/056243.html 55 | logging.DEBUG2= 9 56 | logging.DEBUG3= 8 57 | 58 | logging.addLevelName (logging.DEBUG2, 'DEBUG2') 59 | logging.addLevelName (logging.DEBUG3, 'DEBUG3') 60 | 61 | logging.Logger.debug2= debug2 62 | logging.Logger.debug3= debug3 63 | 64 | 65 | patch_logging () 66 | 67 | 68 | def any_comparator (a, b): # pragma: no cover 69 | try: 70 | if a==b: 71 | return 0 72 | elif a0: 176 | logger.debug (copy_to) 177 | 178 | events= selector.select () 179 | for key, _ in events: 180 | logger.debug ("%s is ready to read", key) 181 | i= key.fileobj 182 | 183 | # for error in e: 184 | # logger.debug ('%s error') 185 | # TODO: what? 186 | 187 | o= copy_to[i] 188 | 189 | try: 190 | data= read (i, buf_len) 191 | logger.debug2 ('%s -> %s: %s', i, o, data) 192 | 193 | # ValueError: read of closed file 194 | except (OSError, ValueError) as e: 195 | logger.debug ('stopping copying for %s', i) 196 | logger.debug (traceback.format_exc ()) 197 | close_file (i) 198 | break 199 | else: 200 | if len (data)==0: 201 | logger.debug ('stopping copying for %s, no more data', i) 202 | close_file (i) 203 | 204 | if finished is not None and i==finished: 205 | logger.debug ('finishing') 206 | # quite a hack :) 207 | copy_to= {} 208 | 209 | break 210 | else: 211 | write (o, data) 212 | 213 | selector.close () 214 | logger.debug ('over and out') 215 | -------------------------------------------------------------------------------- /bin/ayrton: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3.6 2 | # -*- coding: utf-8 -*- 3 | 4 | # (c) 2013 Marcos Dione 5 | 6 | # This file is part of ayrton. 7 | # 8 | # ayrton is free software: you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License as published by 10 | # the Free Software Foundation, either version 3 of the License, or 11 | # (at your option) any later version. 12 | # 13 | # ayrton is distributed in the hope that it will be useful, 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | # GNU General Public License for more details. 17 | # 18 | # You should have received a copy of the GNU General Public License 19 | # along with ayrton. If not, see . 20 | 21 | # cannot be simpler :) 22 | import ayrton 23 | 24 | # well, actually, it could :) 25 | import sys 26 | import os.path 27 | import traceback 28 | import logging 29 | 30 | usage="""ayrton - A Python-based shell-like scripting language. 31 | 32 | Usage: 33 | 34 | ayrton -h|--help 35 | ayrton -v|--version 36 | ayrton [DEBUG_OPTS] -c|--script SCRIPT [argv ...] 37 | ayrton [DEBUG_OPTS] [-f|--file] file [argv ...] 38 | 39 | Options: 40 | -h, --help Show this help message and exit. 41 | -c SCRIPT, --script SCRIPT 42 | If this option is present, the script is read from its 43 | argument. 44 | -v, --version Show the version number and exit. 45 | [-f, --file] FILE Name of the file from which the script is read. 46 | 47 | Debug Options: 48 | -d, --debug Enable debugging, which writes lots of ayrton's execution 49 | information in a file called 'ayrton.log' in the current 50 | directory. 51 | -dd, --debug2 More execution info. 52 | -ddd, --debug3 Even more execution info. 53 | -p, --pdb Launch pdb on any unhandled exception. 54 | -x, --trace Trace script execution. It does not trace other modules 55 | used. If debug is enabled, trace goes to log. 56 | -xx, --trace-with-lineno 57 | Trace and print the line number. 58 | -xxx, --trace-all Trace all executed lines, including in used modules. 59 | File name and line number will be printed. 60 | WARNING: Too much info will be in the output. 61 | 62 | Arguments: 63 | argv Arguments to be passed to the script.""" 64 | 65 | # argument parsing has to be done by hand, so arguments can be given to the script 66 | # it *is* kinda idiotic, but I can't tell any of the parsing modules to just parse those options 67 | # and the options are not many and are simple to handle, anyways 68 | 69 | file_name= None 70 | script= None 71 | file= None 72 | params= ayrton.ExecParams () 73 | # sys.argv[0] is 'ayrton', always defined 74 | args= sys.argv[1:] 75 | reason= None 76 | 77 | # special case so I can boot fast 78 | if len (args)==0: 79 | print (usage) 80 | sys.exit (0) 81 | 82 | try: 83 | for index, arg in enumerate (args): 84 | if arg in ('-h', '--help'): 85 | print (usage) 86 | sys.exit (0) 87 | 88 | if arg in ('-v', '--version'): 89 | print (ayrton.__version__) 90 | sys.exit (0) 91 | 92 | if arg in ('-c', '--script'): 93 | reason= 'Missing argument to option %s' % arg 94 | script= args[index+1] 95 | file_name= '' 96 | # fake argv[0] 97 | script_args= [ file_name ]+args[index+2:] 98 | # stop parsing, anything else is passed to the script 99 | break 100 | 101 | if arg in ('-x', '--trace'): 102 | params.trace= True 103 | continue 104 | 105 | if arg in ('-xx', '--trace-with-linenos'): 106 | params.trace= True 107 | params.linenos= True 108 | continue 109 | 110 | if arg in ('-xxx', '--trace_all'): 111 | params.trace= True 112 | params.linenos= True 113 | params.trace_all= True 114 | continue 115 | 116 | if arg in ('-d', '--debug'): 117 | params.debug= True 118 | # set this ASAP 119 | ayrton.set_debug () 120 | continue 121 | 122 | if arg in ('-dd', '--debug2'): 123 | params.debug= True 124 | # set this ASAP 125 | ayrton.set_debug (logging.DEBUG2) 126 | continue 127 | 128 | if arg in ('-ddd', '--debug3'): 129 | params.debug= True 130 | # set this ASAP 131 | ayrton.set_debug (logging.DEBUG3) 132 | continue 133 | 134 | if arg in ('-p', '--pdb'): 135 | params.pdb= True 136 | continue 137 | 138 | if arg in ('-f', '--file'): 139 | reason= 'Missing argument to option %s' % arg 140 | # -f|--file is optional (because idiot /usr/bin/env ayrton -f does not work) 141 | # we could let it fallback to the last case, 142 | # but this way we can handle the actual argument to the option 143 | file_name= args[index+1] 144 | # script_args[0] is the script's file name 145 | script_args= args[index+1:] 146 | # stop parsing, anything else is passed to the script 147 | break 148 | 149 | # as every branch either exit()s, break's or continue's, 150 | # here we're in the situation where the next argument is the script name 151 | file_name= arg 152 | # script_args[0] is the script's file name 153 | script_args= args[index:] 154 | # stop parsing, anything else is passed to the script 155 | break 156 | 157 | except IndexError: 158 | print (reason) 159 | print (usage) 160 | sys.exit (1) 161 | 162 | if file_name is None and script is None: 163 | print (usage) 164 | sys.exit (1) 165 | 166 | try: 167 | v= ayrton.run_file_or_script (file_name=file_name, script=script, 168 | argv=script_args, params=params) 169 | 170 | if type(v)==bool: 171 | if v: 172 | v= 0 # success in shellspeak 173 | else: 174 | v= 1 # 'some' error 175 | else: 176 | try: 177 | v= int (v) 178 | except (ValueError, TypeError): 179 | # I can only assume it's ok 180 | v= 0 181 | 182 | sys.exit (v) 183 | 184 | except Exception: 185 | t, e, tb= sys.exc_info () 186 | 187 | if False: 188 | # skip ayrton's stack 189 | # TODO: if the script has a syntax error, the stack is shorter 190 | if script is not None: 191 | for i in range (5): 192 | tb= tb.tb_next 193 | else: 194 | for i in range (6): 195 | tb= tb.tb_next 196 | 197 | traceback.print_exception (t, e, tb) 198 | sys.exit (1) 199 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = python3.6 $(shell which sphinx-build) 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 16 | 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | help: 20 | @echo "Please use \`make ' where is one of" 21 | @echo " html to make standalone HTML files" 22 | @echo " dirhtml to make HTML files named index.html in directories" 23 | @echo " singlehtml to make a single large HTML file" 24 | @echo " pickle to make pickle files" 25 | @echo " json to make JSON files" 26 | @echo " htmlhelp to make HTML files and a HTML help project" 27 | @echo " qthelp to make HTML files and a qthelp project" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 31 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 32 | @echo " text to make text files" 33 | @echo " man to make manual pages" 34 | @echo " texinfo to make Texinfo files" 35 | @echo " info to make Texinfo files and run them through makeinfo" 36 | @echo " gettext to make PO message catalogs" 37 | @echo " changes to make an overview of all changed/added/deprecated items" 38 | @echo " linkcheck to check all external links for integrity" 39 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 40 | 41 | clean: 42 | -rm -rf $(BUILDDIR)/* 43 | 44 | html: 45 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 48 | 49 | dirhtml: 50 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 51 | @echo 52 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 53 | 54 | singlehtml: 55 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 56 | @echo 57 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 58 | 59 | pickle: 60 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 61 | @echo 62 | @echo "Build finished; now you can process the pickle files." 63 | 64 | json: 65 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 66 | @echo 67 | @echo "Build finished; now you can process the JSON files." 68 | 69 | htmlhelp: 70 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 71 | @echo 72 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 73 | ".hhp project file in $(BUILDDIR)/htmlhelp." 74 | 75 | qthelp: 76 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 77 | @echo 78 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 79 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 80 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/ayrton.qhcp" 81 | @echo "To view the help file:" 82 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/ayrton.qhc" 83 | 84 | devhelp: 85 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 86 | @echo 87 | @echo "Build finished." 88 | @echo "To view the help file:" 89 | @echo "# mkdir -p $$HOME/.local/share/devhelp/ayrton" 90 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/ayrton" 91 | @echo "# devhelp" 92 | 93 | epub: 94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 95 | @echo 96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 97 | 98 | latex: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo 101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 103 | "(use \`make latexpdf' here to do that automatically)." 104 | 105 | latexpdf: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo "Running LaTeX files through pdflatex..." 108 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 110 | 111 | text: 112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 113 | @echo 114 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 115 | 116 | man: 117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 118 | @echo 119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 120 | 121 | texinfo: 122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 123 | @echo 124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 125 | @echo "Run \`make' in that directory to run these through makeinfo" \ 126 | "(use \`make info' here to do that automatically)." 127 | 128 | info: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo "Running Texinfo files through makeinfo..." 131 | make -C $(BUILDDIR)/texinfo info 132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 133 | 134 | gettext: 135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 136 | @echo 137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 138 | 139 | changes: 140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 141 | @echo 142 | @echo "The overview file is in $(BUILDDIR)/changes." 143 | 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | doctest: 151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 152 | @echo "Testing of doctests in the sources finished, look at the " \ 153 | "results in $(BUILDDIR)/doctest/output.txt." 154 | -------------------------------------------------------------------------------- /doc/examples/arc.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | # this example is a simple script to control an audacious instance 3 | # running in another machine 4 | 5 | # you run it as: 6 | # arc.ay 7 | 8 | # for the moment it assumes you have password/passphrase-less ssh access into 9 | # , the remote machine 10 | 11 | # can be Play, Stop, Next, etc 12 | 13 | where= argv[2] 14 | 15 | with remote (where, allow_agent=False): 16 | # notice how we can reference to the local's argv 17 | what= argv[1] 18 | # we assume Audacious is running on display :0 19 | export (DISPLAY=':0') 20 | # use qdbus to send the command 21 | qdbus ('org.mpris.MediaPlayer2.audacious', '/org/mpris/MediaPlayer2', 22 | 'org.mpris.MediaPlayer2.Player.%s' % what) 23 | -------------------------------------------------------------------------------- /doc/examples/argv.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | print (argv) 4 | -------------------------------------------------------------------------------- /doc/examples/capture.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | a= echo ("foo", _out=Capture) 4 | print ("echo: %s" % a) 5 | -------------------------------------------------------------------------------- /doc/examples/envvars.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | print (PATH) 4 | -------------------------------------------------------------------------------- /doc/examples/hello_world.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | echo ("Hello, world!") 4 | -------------------------------------------------------------------------------- /doc/examples/with_cd.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | with cd ('bin'): 4 | print (pwd ()) 5 | 6 | print (pwd ()) 7 | -------------------------------------------------------------------------------- /doc/examples/with_remote.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | import random 4 | 5 | a= 42 6 | 7 | with remote ('localhost', allow_agent=False) as s: 8 | import random 9 | 10 | b= input ('gimme a number, bro!\n') 11 | print (a+int (b)+random.randint (10, 100)) 12 | 13 | (i, o, e)= s 14 | i.write (bytes ('%d\n' % random.randint (10, 100), 'ascii')) 15 | 16 | print (o.readlines ()) 17 | print (e.readlines ()) 18 | 19 | for x in i, o, e: 20 | x.close () 21 | -------------------------------------------------------------------------------- /doc/source/index.rst: -------------------------------------------------------------------------------- 1 | .. ayrton documentation master file, created by 2 | sphinx-quickstart on Sun Jul 21 22:47:04 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to ayrton's documentation! 7 | ================================== 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | reference 15 | 16 | 17 | Indices and tables 18 | ================== 19 | 20 | * :ref:`genindex` 21 | * :ref:`search` 22 | 23 | Motivation 24 | ========== 25 | 26 | `ayrton` is an extension of the Python language that tries to make it look more 27 | like a shell programming language. The first thing to note is that it's not 28 | intended to be an interactive shell, and I think that anyone used to them for their 29 | day to day computer usage (most notably, fellow SysAdmins) would find such a 30 | shell too cumbersome. 31 | 32 | One of the ideas that triggered the development of `ayrton` is that shell 33 | languages have very rudimentary data handling mechanisms: most data are strings; 34 | these strings can be treated as integers in some circumstances; there is no concept 35 | of floats; and only a few of them have arrays, even associative ones. For any other 36 | data manipulation, developers have to use tools like `awk`, `sed`, `grep`, `cut`, even 37 | Perl from time to time. In some cases this means that scripts, once they start 38 | to grow a little, become cumbersome and/or unmaintainable. 39 | 40 | From the other side, Python is very good at data manipulation, but some things 41 | that are really simple and common in shell languages, like program execution, 42 | are a little more complex in Python. A very good alternative for this is the `sh` 43 | module, but in some aspects it still leaves an alien flavor to a shell 44 | programmer. Here is where `ayrton` comes in and tries to fill in that gap. 45 | 46 | ``ayrton`` for Pythonistas using ``sh`` 47 | ======================================= 48 | 49 | As mentioned, `ayrton` was originally heavily based on the `sh` module, but currently 50 | implements its own Command class, which introduces 51 | some changes to make it look more like a shell language. This section lists all 52 | the places where it is different from that module and from Python vanilla: 53 | 54 | Executables as globals 55 | The first thing to notice is that you don't need to 'import' the executables 56 | like with ``sh`` (``from sh import ls``) or use them as functions in a module 57 | (``sh.ls()``). Instead, you simply invoke the executable as it were a 58 | global: ``ls()``. **This introduces an unexpected behavior**. This means 59 | that typos in names will be treated as commands to be run. *This is intentional*. 60 | This also means that you will get nasty backtraces when the command is not 61 | found. 62 | 63 | *stdout* and *sterr* as default 64 | In normal scripting languages, commands' output goes directly to the terminal; 65 | in ``sh`` it goes to the return value of the function. By default, ``ayrton`` 66 | imitates shells scripting. If you want the output to go to the return of the 67 | function, you have to pass the option ``_out=Capture``. 68 | 69 | Commands do not raise Exceptions 70 | When you run a command with ``sh`` and it either exited with a return code 71 | different from 0 or it was stopped/finished by a signal, it would raise an 72 | exception. You then can recover the exit code and the output from the 73 | exception. In ``ayrton`` not only exceptions are not raised by default, you can ask the 74 | exit code in the ``code`` attribute and the output in the ``stdout`` and 75 | ``stderr`` attributes. You can also use commands as booleans in conditions 76 | to ``if``, ``while``, etc: ``if ls(): then echo ("yes!")``. Exceptions are raised when 77 | the command is not found; or when the `errexit` :py:func:`option` is set, a 78 | command returns an exit code not 0, and ``_fails`` was not specified. See also 79 | :py:func:`foo`. 80 | 81 | Commands with dots are supported 82 | You can call `foo.py()`, no extra work needed. 83 | 84 | ``ayrton`` for Shell Scripters 85 | ============================== 86 | 87 | I will not lie to you. This proto-language *will* feel somewhat alien to you. 88 | A lot of things will look like too verbose or explicit, paths now need to be 89 | closed in quotes (``"`` or ``'``), arguments are separated by commas or 90 | sometimes have to be given within a string. We will try to minimize them as much 91 | as possible, but as long as ``ayrton`` stays closer to Python that to ``bash``, some will be 92 | impossible to fix. Having said that, we think that ``ayrton`` will be powerful 93 | enough that the benefits will overweight this. 94 | 95 | * You can't reference undefined variables, which makes impossible to use 96 | environment variables that might be defined or not. Use `define()` to declare 97 | them. 98 | * You don't call ``[``, see the reference. 99 | * Redirection is better accomplished with _out, _err, etc. 100 | * Currently, absolute and relative paths do not work directly, you have to use ``run()``. 101 | * Expansions are not done automatically; variables can be expanded with %; 102 | brace, tilde and glob expansions can be done with ``bash()``, 103 | command substitution is yet to come. Expansions return lists. 104 | * If you name a variable with the same name as an executable, you can't execute it until 105 | the variable is out of scope. This is exactly the same thing that happens when you 106 | eclipse a Python variable from an outer scope, and similar to when you define a function 107 | in `bash` with the same names as the executable (but that you can go around by giving 108 | the full path if you know it, which you can't do in `ayrton` yet). 109 | * You can't use Python keywords except where they are valid Python code. For instance, 110 | you can't use an ``--continue`` option). 111 | * Special parameters are replaced as following: 112 | 113 | * ``$*`` and ``$@`` are mostly replaced with :py:data:`argv`; ``$0`` is `argv[0]`. 114 | * ``$#`` is replaced by ``len(argv)``. 115 | * ``$?`` and ``$!`` are implemented as methods of the `Command` objects. 116 | * ``$-`` and ``$_`` do not have an equivalent yet, sorry. 117 | * ``$$`` can be replaced with ``os.getpid()``. 118 | 119 | * ``ayrton`` does not make 120 | a difference whether you're double quoting or not; everything remains intact 121 | and no further tokenizing is applied to positional parameters or program 122 | output (except for line cutting). If you rely on that, you will have to do it 123 | yourself. 124 | 125 | -------------------------------------------------------------------------------- /import_pypy_parser.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | set -eu 4 | 5 | cpython_src="$1" 6 | pypy_src="$2" 7 | 8 | fun copy() { 9 | src="$1" 10 | dst="$2" 11 | 12 | mkdir --parents --verbose "$(dirname $dst)" 13 | cp --verbose "$src" "$dst" 14 | } 15 | 16 | # NOTE: not every Python version changes the syntax 17 | for file in astcompiler/__init__.py \ 18 | astcompiler/astbuilder.py \ 19 | astcompiler/consts.py \ 20 | astcompiler/misc.py \ 21 | pyparser/__init__.py \ 22 | pyparser/automata.py \ 23 | pyparser/data/Grammar2.5 \ 24 | pyparser/data/Grammar2.7 \ 25 | pyparser/data/Grammar3.2 \ 26 | pyparser/data/Grammar3.3 \ 27 | pyparser/data/Grammar3.5 \ 28 | pyparser/data/Grammar3.6 \ 29 | pyparser/error.py \ 30 | pyparser/future.py \ 31 | pyparser/gendfa.py \ 32 | pyparser/metaparser.py \ 33 | pyparser/parser.py \ 34 | pyparser/pygram.py \ 35 | pyparser/pyparse.py \ 36 | pyparser/pytoken.py \ 37 | pyparser/pytokenize.py \ 38 | pyparser/pytokenizer.py \ 39 | __init__.py \ 40 | debug.py \ 41 | error.py; do 42 | src="$pypy_src/pypy/interpreter/$file" 43 | dst="ayrton/parser/$file" 44 | 45 | copy("$src", "$dst") 46 | done 47 | 48 | # run generators 49 | # ayrton/parser/pyparser/dfa_generated.py 50 | PYTHONPATH=$(pwd) \ 51 | ./ayrton/parser/pyparser/gendfa.py | \ 52 | python3.6 -c 'import sys; data = sys.stdin.read(); open(sys.argv[1], "w+").write(data)' \ 53 | ./ayrton/parser/pyparser/dfa_generated.py 54 | -------------------------------------------------------------------------------- /patches/ayrton_customization.diff: -------------------------------------------------------------------------------- 1 | --- a/ayrton/parser/astcompiler/astbuilder.py 2 | +++ b/ayrton/parser/astcompiler/astbuilder.py 3 | @@ -564,26 +564,6 @@ 4 | new_node.col_offset = try_node.col_offset 5 | return new_node 6 | 7 | - def handle_with_stmt(self, with_node): 8 | - body = self.handle_suite(with_node.children[-1]) 9 | - i = len(with_node.children) - 1 10 | - while True: 11 | - i -= 2 12 | - item = with_node.children[i] 13 | - test = self.handle_expr(item.children[0]) 14 | - if len(item.children) == 3: 15 | - target = self.handle_expr(item.children[2]) 16 | - self.set_context(target, ast.Store()) 17 | - else: 18 | - target = None 19 | - wi = ast.With (test, target, body) 20 | - wi.lineno = with_node.lineno 21 | - wi.col_offset = with_node.col_offset 22 | - if i == 1: 23 | - break 24 | - body = [wi] 25 | - return wi 26 | - 27 | def handle_with_item(self, item_node): 28 | test = self.handle_expr(item_node.children[0]) 29 | if len(item_node.children) == 3: 30 | @@ -1336,15 +1316,20 @@ 31 | expr_node = argument.children[0] 32 | if len(argument.children) == 1: 33 | # a positional argument 34 | - if keywords: 35 | - if doublestars_count: 36 | - self.error("positional argument follows " 37 | - "keyword argument unpacking", 38 | - expr_node) 39 | - else: 40 | - self.error("positional argument follows " 41 | - "keyword argument", 42 | - expr_node) 43 | + 44 | + # we disable these checks so we can get 45 | + # grep(quiet=True, **user_args, '/etc/passwd') 46 | + # they will be converted to o()'s later 47 | + 48 | + # if keywords: 49 | + # if doublestars_count: 50 | + # self.error("positional argument follows " 51 | + # "keyword argument unpacking", 52 | + # expr_node) 53 | + # else: 54 | + # self.error("positional argument follows " 55 | + # "keyword argument", 56 | + # expr_node) 57 | args.append(self.handle_expr(expr_node)) 58 | elif expr_node.type == tokens.STAR: 59 | # an iterable argument unpacking 60 | @@ -1373,13 +1358,28 @@ 61 | self.error("lambda cannot contain assignment", 62 | expr_node) 63 | keyword = keyword_expr 64 | + # TODO: if we disable this, we can allow 65 | + # f(a=1, a=2) 66 | if keyword in used_keywords: 67 | self.error("keyword argument repeated", expr_node) 68 | used_keywords[keyword] = None # why not use a set for this? 69 | if isinstance (keyword, ast.Name): 70 | - self.check_forbidden_name(keyword, expr_node) 71 | + # NOTE: we could disable this too :) 72 | + self.check_forbidden_name(keyword.id, expr_node) 73 | keyword_value = self.handle_expr(argument.children[2]) 74 | - keywords.append(ast.keyword(keyword, keyword_value)) 75 | + if isinstance (keyword, ast.Name) and keyword.id in Command.supported_options: 76 | + keywords.append(ast.keyword(keyword.id, keyword_value)) 77 | + else: 78 | + kw = ast.keyword(keyword, keyword_value) 79 | + kw.lineno = expr_node.lineno 80 | + kw.col_offset = expr_node.col_offset 81 | + name = ast.Name('o', ast.Load()) 82 | + name.lineno = expr_node.lineno 83 | + name.col_offset = expr_node.col_offset 84 | + arg = ast.Call(name, [], [ kw ]) 85 | + arg.lineno = expr_node.lineno 86 | + arg.col_offset = expr_node.col_offset 87 | + args.append(arg) 88 | i += 1 89 | if not args: 90 | args = [] 91 | @@ -1479,7 +1479,7 @@ 92 | if (n_maker_children == 1 or 93 | (n_maker_children > 1 and 94 | maker.children[1].type == tokens.COMMA)): 95 | - # a set display 96 | + # a set display, {1, 2, ...} 97 | return self.handle_setdisplay(maker, atom_node) 98 | elif n_maker_children > 1 and maker.children[1].type == syms.comp_for: 99 | # a set comprehension 100 | @@ -1493,7 +1493,7 @@ 101 | 102 | return self.handle_dictcomp(maker, atom_node) 103 | else: 104 | - # a dictionary display 105 | + # a dictionary display, {1: 2, ...} 106 | return self.handle_dictdisplay(maker, atom_node) 107 | else: 108 | raise AssertionError("unknown atom") 109 | -------------------------------------------------------------------------------- /patches/changes.sed: -------------------------------------------------------------------------------- 1 | ( +)return ast\.(.*)\((.*), (.*)\.get_lineno\(\), (.*)\.get_col_offset\(\)\) 2 | \1new_node = ast.\2(\3)\n\1new_node.lineno = \4.lineno\n\1new_node.col_offset = \5.col_offset\n\1return new_node 3 | 4 | ( +)return ast\.(.*)\((.*)\.get_lineno\(\), (.*)\.get_col_offset\(\)\) 5 | \1new_node = ast.\2()\n\1new_node.lineno = \3.lineno\n\1new_node.col_offset = \4.col_offset\n\1return new_node 6 | 7 | ast.Load, 8 | ast.Load(), 9 | 10 | ([a-z_]+)\.num_children\(\) 11 | len(\1.children) 12 | 13 | ([a-z_]+)\.get_child\((.+)\) 14 | \1.children[\2] 15 | 16 | ([a-z_]+)\.get_value\(\) 17 | \1.value 18 | -------------------------------------------------------------------------------- /patches/series: -------------------------------------------------------------------------------- 1 | pypy_py35-python36.diff 2 | ayrton_customization.diff 3 | -------------------------------------------------------------------------------- /release.ay: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env ayrton 2 | 3 | import logging 4 | 5 | import ayrton 6 | 7 | option ('-e') 8 | 9 | if make ('install'): # this includes tests 10 | # with remote ('mustang'): 11 | # cd (bash ('~/src/projects/ayrton')) 12 | # git ('pull', 'devel') 13 | # # if this fails, hopefully the whole script stops :) 14 | # make ('tests') 15 | 16 | # this command might fail if it wants to 17 | uncommited= git ('status', --short=True) | grep ('^ M', _out=Capture, _fails=True) 18 | if uncommited: 19 | print ("The following files are not commited, I can't release like this") 20 | print (str (uncommited)) 21 | exit (1) 22 | 23 | # git ('checkout', -b="release-%s" % ayrton.__version__) 24 | 25 | if 0 < logging.root.level < 30: 26 | print ("logging is at %d, I can't release like this!" % logging.root.level) 27 | exit (1) 28 | 29 | dch (--changelog='ChangeLog.rst', --newversion=ayrton.__version__) 30 | dch (--changelog='ChangeLog.rst', --release=True) 31 | commited= git ('commit', 'ChangeLog.rst', 32 | --message="[*] new entry for %s." % ayrton.__version__, 33 | _out=Capture) 34 | if not commited and 'nothing added to commit' not in commited: 35 | exit (1) 36 | 37 | # docs 38 | make ('docs') 39 | rsync (--archive=True, --verbose=True, --compress=True, --rsh='ssh', 40 | 'doc/build/html/', 'www.grulic.org.ar:www/projects/ayrton/') 41 | 42 | # release 43 | python3.6 ('setup.py', 'sdist', 'upload') 44 | nvie ('release', 'release-'+ayrton.__version__) 45 | git ('push') 46 | # TODO: does github have an API for making a release? 47 | # no they don't yet 48 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | paramiko>=1.15 # first one with py3 support 2 | coverage>=4.2 3 | Sphinx>=1.4.6 4 | -------------------------------------------------------------------------------- /run_test.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | case "$1" in 4 | -n|--no-clean) 5 | shift 6 | break;; 7 | *) 8 | make testclean 9 | break;; 10 | esac 11 | 12 | exec python3.6 -m unittest -v ayrton.tests.$1 13 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3.6 2 | # -*- coding: utf-8 -*- 3 | 4 | # (c) 2013 Marcos Dione 5 | # for licensing details see the file LICENSE.txt 6 | 7 | from distutils.core import setup 8 | import ayrton 9 | 10 | setup ( 11 | name='ayrton', 12 | version= ayrton.__version__, 13 | description= 'a shell-like scripting language based on Python3.', 14 | author= 'Marcos Dione', 15 | author_email= 'mdione@grulic.org.ar', 16 | url= 'https://github.com/StyXman/ayrton', 17 | packages= [ 'ayrton', 'ayrton.parser', 'ayrton.parser.pyparser', 18 | 'ayrton.parser.astcompiler' ], 19 | package_data= { 20 | 'ayrton.parser.pyparser': [ 'data/Grammar3.3' ], 21 | }, 22 | scripts= [ 'bin/ayrton' ], 23 | license= 'GPLv3', 24 | classifiers= [ 25 | 'Development Status :: 3 - Alpha', 26 | 'Environment :: Console', 27 | 'Intended Audience :: Developers', 28 | 'Intended Audience :: System Administrators', 29 | 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)', 30 | 'Operating System :: POSIX', 31 | 'Programming Language :: Python :: 3', 32 | 'Topic :: System', 33 | 'Topic :: System :: Systems Administration', 34 | ], 35 | ) 36 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | # because of deps (1.6.1) 3 | # and passenv (2.0) 4 | minversion= 2.0 5 | # py33 disabled because it requires a new importer 6 | envlist= py34, py35, py36 7 | 8 | [testenv] 9 | whitelist_externals= make 10 | commands= make testclean tests 11 | deps= -rrequirements.txt 12 | # space separated, really? 13 | passenv= HOME SSH_AUTH_SOCK TERM USER 14 | --------------------------------------------------------------------------------