├── meta.php ├── .gitattributes ├── tests ├── fixtures │ ├── scope │ │ ├── run.php │ │ └── macros.php │ ├── parsers.php │ ├── expression │ │ ├── bad.php │ │ └── good.php │ └── expanders.php ├── phpt │ ├── errors │ │ ├── bad_token_type.phpt │ │ ├── macro_empty_pattern.phpt │ │ ├── capture_without_type.phpt │ │ ├── macro_without_expansion.phpt │ │ ├── bad_dominant_macro_mark_end.phpt │ │ ├── bad_dominant_macro_mark_start.phpt │ │ ├── capture_redefinition.phpt │ │ ├── missing_expander_argument.phpt │ │ ├── expansion_undeclared.phpt │ │ └── error_line_number.phpt │ ├── macro │ │ ├── operator_if_defined_then_expand_I.phpt │ │ ├── operator_if_defined_then_expand_II.phpt │ │ ├── block_indirect_macro_recursion.phpt │ │ ├── operator_if_not_defined_then_expand_on_error_II.phpt │ │ ├── macro_ast_unpacking_with_optional_flag.phpt │ │ ├── macro_keyword.phpt │ │ ├── empty_expansion_003.phpt │ │ ├── layer_matcher_error_unbalanced_end.phpt │ │ ├── layer_matcher_error_unbalanced.phpt │ │ ├── macro_pattern_layer_matcher.phpt │ │ ├── operator_if_not_defined_then_expand_on_error_I.phpt │ │ ├── operator_if_defined_then_expand_on_error_I.phpt │ │ ├── layer_matcher.phpt │ │ ├── parser_combinator_001.phpt │ │ ├── macro_ast_unpacking_with_delimiter_trailing.phpt │ │ ├── macro_resonance.phpt │ │ ├── macro_ast_unpacking_with_delimiter_non_trailing.phpt │ │ ├── macro_pattern_whitespace_01.phpt │ │ ├── macro_expansion_whitespace.phpt │ │ ├── macro_pattern_whitespace_02.phpt │ │ ├── block_macro_recursion.phpt │ │ ├── cloaking.phpt │ │ ├── macro_expansion_comments.phpt │ │ ├── empty_expansion_002.phpt │ │ ├── dominant_macro.phpt │ │ ├── macro_ast_unpacking_without_optional_flag.phpt │ │ ├── operator_null_coalesce_I.phpt │ │ ├── operator_null_coalesce_II.phpt │ │ ├── macro_pattern_edge_cases.phpt │ │ ├── empty_expansion_001.phpt │ │ ├── macro_pattern_layer_matcher_deep.phpt │ │ ├── macro_buffer_matcher.phpt │ │ ├── macro_deep_ast_access_error.phpt │ │ ├── macro_pattern_block_matcher.phpt │ │ ├── macro_pattern_matched_contiguously.phpt │ │ ├── hygiene.phpt │ │ ├── macro_deep_ast_access.phpt │ │ └── compiler_pass.phpt │ ├── expanders │ │ ├── concat.phpt │ │ ├── error_when_undefined.phpt │ │ ├── stringify.phpt │ │ ├── fully_qualified_expander.phpt │ │ ├── composition_tokenstream_expander.phpt │ │ └── lazy_evaluation_of_expanders.phpt │ ├── parsers │ │ ├── fully_qualified_parser.phpt │ │ ├── token.phpt │ │ ├── ls.phpt │ │ ├── expression.phpt │ │ └── array_arg.phpt │ ├── unless.phpt │ ├── swap.phpt │ ├── expansion_key.phpt │ ├── midrule.phpt │ ├── unreachable.phpt │ ├── union_catch.phpt │ ├── opaque_types.phpt │ ├── group_use_grammar.phpt │ ├── test_dsl.phpt │ ├── this_shorthand.phpt │ ├── json.phpt │ ├── group_use.phpt │ ├── guard.phpt │ ├── defer.phpt │ ├── retry.phpt │ ├── issues │ │ ├── ircmaxell-php-compiler#29.phpt │ │ └── issue#30.phpt │ ├── primitve_generics.phpt │ ├── short_functions_with_no_braces.phpt │ ├── primitive_union_return_types.phpt │ ├── short_functions_with_lexical_scope.phpt │ ├── property_accessors.phpt │ └── enums.phpt ├── ParserOptimizationTest.php ├── MacroScopeTest.php ├── TokenTest.php ├── SpecsTest.php ├── AstTest.php └── TokenStreamTest.php ├── src ├── Index.php ├── YayPreprocessorError.php ├── Result.php ├── NodeStart.php ├── NodeEnd.php ├── PatternInterface.php ├── ContextLookupError.php ├── ParserTracer │ ├── ParserTracer.php │ ├── NullParserTracer.php │ └── CliParserTracer.php ├── Directive.php ├── Cycle.php ├── Stack.php ├── Node.php ├── CompilerPass.php ├── BlueContext.php ├── AnonymousFunction.php ├── MacroMember.php ├── Map.php ├── Expected.php ├── Error.php ├── Macro.php ├── expanders.php ├── Token.php ├── Parser.php ├── Ast.php ├── parsers_internal.php ├── Engine.php ├── TokenStream.php ├── GrammarPattern.php └── Pattern.php ├── .gitignore ├── phpbench.json ├── .travis.yml ├── LICENSE ├── phpunit.xml ├── bin ├── yay-pretty └── yay ├── composer.json ├── benchmarks └── EngineBenchmark.php └── README.md /meta.php: -------------------------------------------------------------------------------- 1 | '0.7' 3 | ]; 4 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.phpt linguist-language=php 2 | *.phppt linguist-language=php 3 | -------------------------------------------------------------------------------- /tests/fixtures/scope/run.php: -------------------------------------------------------------------------------- 1 | > { true } 4 | 5 | $(macro) { 'LOCAL_MACRO' } >> { true } 6 | 7 | return ['GLOBAL_MACRO', 'LOCAL_MACRO']; 8 | -------------------------------------------------------------------------------- /src/Result.php: -------------------------------------------------------------------------------- 1 | > { }; 7 | 8 | ?> 9 | --EXPECTF-- 10 | Undefined token type 'T_HAKUNAMATATA', in %s.phpt on line 3. 11 | -------------------------------------------------------------------------------- /src/NodeStart.php: -------------------------------------------------------------------------------- 1 | > { 9 | x 10 | } 11 | 12 | ?> 13 | --EXPECTF-- 14 | Empty macro pattern, in %s.phpt on line 3. 15 | -------------------------------------------------------------------------------- /src/NodeEnd.php: -------------------------------------------------------------------------------- 1 | > { $(foo) } 7 | 8 | ?> 9 | --EXPECTF-- 10 | Bad macro capture identifier '$(foo)', in %s.phpt on line 3. 11 | -------------------------------------------------------------------------------- /src/PatternInterface.php: -------------------------------------------------------------------------------- 1 | 9 | --EXPECTF-- 10 | Unexpected T_CLOSE_TAG(?>), in %s.phpt on line 5, expected T_SR(). 11 | -------------------------------------------------------------------------------- /tests/phpt/errors/bad_dominant_macro_mark_end.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Bad dominant macro offset 3 | --FILE-- 4 | > { 9 | _ 10 | } 11 | 12 | ?> 13 | --EXPECTF-- 14 | Bad dominant macro marker '$!' offset 2, in %s.phpt on line 4. 15 | -------------------------------------------------------------------------------- /tests/phpt/errors/bad_dominant_macro_mark_start.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Bad dominant macro offset 3 | --FILE-- 4 | > { 9 | _ 10 | } 11 | 12 | ?> 13 | --EXPECTF-- 14 | Bad dominant macro marker '$!' offset 0, in %s.phpt on line 4. 15 | -------------------------------------------------------------------------------- /tests/phpt/macro/operator_if_defined_then_expand_I.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Test for ? operator 3 | --FILE-- 4 | >{ 9 | $(foo ? {pass($(foo))}); 10 | } 11 | 12 | test; 13 | 14 | ?> 15 | --EXPECTF-- 16 | 21 | -------------------------------------------------------------------------------- /tests/phpt/macro/operator_if_defined_then_expand_II.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Test for ? operator 3 | --FILE-- 4 | >{ 9 | $(undefined ? { pass($(foo)) }); 10 | } 11 | 12 | test; 13 | 14 | ?> 15 | --EXPECTF-- 16 | 21 | -------------------------------------------------------------------------------- /src/ContextLookupError.php: -------------------------------------------------------------------------------- 1 | symbols = $symbols; 8 | } 9 | 10 | function symbols() : array { 11 | return $this->symbols; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /tests/phpt/expanders/concat.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Test concat expander 3 | --FILE-- 4 | > { 9 | $$(concat(foo_ $(word) _baz)) 10 | } 11 | 12 | yay\concat(bar); 13 | 14 | ?> 15 | --EXPECTF-- 16 | 21 | -------------------------------------------------------------------------------- /tests/phpt/macro/block_indirect_macro_recursion.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Detect indirect infine macro recursion 3 | --FILE-- 4 | > { B A }; 7 | $(macro) { B } >> { B C }; 8 | $(macro) { C } >> { C A }; 9 | 10 | A; 11 | 12 | ?> 13 | --EXPECTF-- 14 | 19 | -------------------------------------------------------------------------------- /tests/phpt/errors/capture_redefinition.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Compile time capture redefinition 3 | --FILE-- 4 | > { 9 | foo 10 | } 11 | 12 | ?> 13 | --EXPECTF-- 14 | Redefinition of macro capture identifier 'foo', in %s.phpt on line 4. 15 | -------------------------------------------------------------------------------- /tests/phpt/macro/operator_if_not_defined_then_expand_on_error_II.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Test for ! operator 3 | --FILE-- 4 | > { 9 | $(bar ! {pass($(foo))}); 10 | } 11 | 12 | test; 13 | 14 | ?> 15 | --EXPECTF-- 16 | 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | composer.lock 2 | tmp/ 3 | build/ 4 | vendor/ 5 | tests/phpt/*.diff 6 | tests/phpt/*.out 7 | tests/phpt/*.php 8 | tests/phpt/*.exp 9 | tests/phpt/*.log 10 | tests/phpt/*.sh 11 | tests/phpt/*/*.diff 12 | tests/phpt/*/*.out 13 | tests/phpt/*/*.php 14 | tests/phpt/*/*.exp 15 | tests/phpt/*/*.log 16 | tests/phpt/*/*.sh 17 | -------------------------------------------------------------------------------- /src/ParserTracer/ParserTracer.php: -------------------------------------------------------------------------------- 1 | > { 9 | $(undefined ?... { ($(item)) }); 10 | } 11 | 12 | foo; 13 | 14 | ?> 15 | --EXPECTF-- 16 | 21 | -------------------------------------------------------------------------------- /tests/phpt/macro/macro_keyword.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Macro should not become a reserved keyword 3 | --FILE-- 4 | > { y } 7 | 8 | function macro(){} 9 | 10 | macro(); 11 | 12 | x(); 13 | 14 | ?> 15 | --EXPECTF-- 16 | 25 | -------------------------------------------------------------------------------- /tests/phpt/expanders/error_when_undefined.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Test non existant expander 3 | --FILE-- 4 | > { 9 | $$(undefined($(args))) 10 | } 11 | 12 | yay\undefined(...); 13 | 14 | ?> 15 | --EXPECTF-- 16 | Bad macro expander 'undefined', in %s.phpt on line 6. 17 | -------------------------------------------------------------------------------- /tests/phpt/macro/empty_expansion_003.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Empty expansion and comments 3 | --FILE-- 4 | > { }; 7 | 8 | @foo; 9 | 10 | @ /**/ bar /**/; 11 | 12 | @ 13 | baz 14 | ; 15 | 16 | ?> 17 | --EXPECTF-- 18 | 27 | -------------------------------------------------------------------------------- /src/Directive.php: -------------------------------------------------------------------------------- 1 | > { 9 | MATCH 10 | } 11 | 12 | µ((() // pairs don't match 13 | 14 | ?> 15 | --EXPECTF-- 16 | Unexpected end at T_CLOSE_TAG(?>), in %s.phpt on line 11, expected ')'. 17 | -------------------------------------------------------------------------------- /tests/phpt/macro/layer_matcher_error_unbalanced.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Layer matcher error with unbalanced token pairs 3 | --FILE-- 4 | > { 9 | MATCH 10 | } 11 | 12 | µ(foo, {bar, [baz}]); // pairs don't match 13 | 14 | ?> 15 | --EXPECTF-- 16 | Unexpected '}', in %s.phpt on line 9, expected ']'. 17 | -------------------------------------------------------------------------------- /tests/phpt/macro/macro_pattern_layer_matcher.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Non delimited layer matching 3 | --FILE-- 4 | > { 9 | $(A)($(rest)) 10 | } 11 | 12 | (sum 1 (multiply 2 3)) 13 | 14 | ?> 15 | --EXPECTF-- 16 | 21 | -------------------------------------------------------------------------------- /tests/phpt/macro/operator_if_not_defined_then_expand_on_error_I.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Test for ! operator 3 | --FILE-- 4 | > { 9 | $(bar ! {$(bar)}); 10 | } 11 | 12 | test; 13 | 14 | ?> 15 | --EXPECTF-- 16 | Undefined macro expansion 'bar', in %s.phpt on line 6 with context: [ 17 | "foo", 18 | 0 19 | ] 20 | -------------------------------------------------------------------------------- /src/ParserTracer/NullParserTracer.php: -------------------------------------------------------------------------------- 1 | > { 9 | ok($$(stringify($(matched)))) 10 | } 11 | 12 | test(foo); 13 | 14 | ?> 15 | --EXPECTF-- 16 | 21 | -------------------------------------------------------------------------------- /tests/phpt/macro/operator_if_defined_then_expand_on_error_I.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Test for ? operator 3 | --FILE-- 4 | >{ 9 | $(foo ? { $(undefined) }); 10 | } 11 | 12 | test; 13 | 14 | ?> 15 | --EXPECTF-- 16 | Undefined macro expansion 'undefined', in %s.phpt on line 6 with context: [ 17 | "foo", 18 | 0 19 | ] 20 | -------------------------------------------------------------------------------- /tests/phpt/macro/layer_matcher.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Layer matcher simple test --pretty-print 3 | --FILE-- 4 | > { 9 | $$(stringify($(bar))) 10 | } 11 | 12 | match {this is inside a layer { and this is too } } 13 | 14 | ?> 15 | --EXPECTF-- 16 | 21 | -------------------------------------------------------------------------------- /tests/phpt/macro/parser_combinator_001.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Parser combinator with type and alias. Ex.: "token(T_STRING) as name" 3 | --FILE-- 4 | > { 9 | [$(names ...(, ) {$(name)})] 10 | } 11 | 12 | { a, b, c } 13 | 14 | ?> 15 | --EXPECTF-- 16 | 21 | -------------------------------------------------------------------------------- /src/Cycle.php: -------------------------------------------------------------------------------- 1 | id++; } 12 | 13 | /** 14 | * Not security related, just making scope id not humanely predictable. 15 | */ 16 | function id() : string { return md5((string) $this->id); } 17 | } 18 | -------------------------------------------------------------------------------- /phpbench.json: -------------------------------------------------------------------------------- 1 | { 2 | "bootstrap": "vendor/autoload.php", 3 | "reports": { 4 | "aggregate": { 5 | "generator": "table", 6 | "cols": [ 7 | "params", 8 | "its", 9 | "mem_peak", 10 | "best", 11 | "mean", 12 | "mode", 13 | "worst", 14 | "stdev", 15 | "rstdev", 16 | "diff" 17 | ] 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /tests/phpt/macro/macro_ast_unpacking_with_delimiter_trailing.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Test ast unpacking with trailing delimiter 3 | --FILE-- 4 | > { 9 | [ $(list ...(, ){$(label)}), ] 10 | } 11 | 12 | { A: B: C: D }; 13 | 14 | ?> 15 | --EXPECTF-- 16 | 21 | -------------------------------------------------------------------------------- /tests/phpt/macro/macro_resonance.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Macros should resonate according to declaration order 3 | --FILE-- 4 | > { y() } 7 | $(macro) { y ( ) } >> { z() } 8 | 9 | x(); y(); z(); 10 | 11 | $(macro) { z ( ) } >> { a() } 12 | 13 | x(); y(); z(); 14 | 15 | ?> 16 | --EXPECTF-- 17 | 24 | -------------------------------------------------------------------------------- /src/Stack.php: -------------------------------------------------------------------------------- 1 | stack[] = $value; 12 | } 13 | 14 | function pop() { 15 | array_pop($this->stack); 16 | } 17 | 18 | function current() { 19 | return end($this->stack); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /tests/phpt/macro/macro_ast_unpacking_with_delimiter_non_trailing.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Test ast unpacking with non trailing delimiter 3 | --FILE-- 4 | > { 9 | [ $(list ...(, ){$(label)}) ] 10 | } 11 | 12 | { A: B: C: D }; 13 | 14 | ?> 15 | --EXPECTF-- 16 | 21 | -------------------------------------------------------------------------------- /tests/phpt/macro/macro_pattern_whitespace_01.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Macro patterns should be whitespace insensitive by default 3 | --FILE-- 4 | > { y() } 7 | 8 | x(); 9 | 10 | x() && y(); 11 | 12 | x ( 13 | ); 14 | 15 | x(foo); // no match 16 | 17 | ?> 18 | --EXPECTF-- 19 | 30 | -------------------------------------------------------------------------------- /tests/phpt/macro/macro_expansion_whitespace.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Macro expansions should be whitespace sensitive by default 3 | --FILE-- 4 | > { y ( ) } 7 | 8 | x(); 9 | 10 | x() && y(); 11 | 12 | x ( 13 | ); 14 | 15 | x(foo); // no match 16 | 17 | ?> 18 | --EXPECTF-- 19 | 30 | -------------------------------------------------------------------------------- /tests/phpt/parsers/token.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Extra test for `token()` and `$(T_TOKEN_NAME as label)` 3 | --FILE-- 4 | > { 9 | $$(stringify($(foo)_$(bar)_$(ast ... {$(baz)_$(buz)}))) 10 | } 11 | 12 | a b c d; 13 | 14 | ?> 15 | --EXPECTF-- 16 | 21 | -------------------------------------------------------------------------------- /tests/phpt/expanders/stringify.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Test stringify expander 3 | --FILE-- 4 | > { 9 | $$(stringify($(args))) 10 | } 11 | 12 | $source = yay\stringify(function($a, $b $c){ echo 'the sum is: ' . $a + $b + $c; }); 13 | 14 | ?> 15 | --EXPECTF-- 16 | 21 | -------------------------------------------------------------------------------- /src/Node.php: -------------------------------------------------------------------------------- 1 | token = $token; 12 | $this->skippable = $token->isSkippable(); // cache skipability 13 | } 14 | 15 | function __debugInfo() { return [$this->token]; } 16 | } 17 | -------------------------------------------------------------------------------- /tests/phpt/expanders/fully_qualified_expander.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Uses a custom fully qualified expansion function --pretty-print 3 | --FILE-- 4 | > { 9 | $$(\Yay\tests\fixtures\expanders\my_hello_tokenstream_expander($(matched))); 10 | } 11 | 12 | hello(Chris); 13 | 14 | ?> 15 | --EXPECTF-- 16 | 21 | -------------------------------------------------------------------------------- /tests/phpt/macro/macro_pattern_whitespace_02.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Macro patterns should be whitespace insensitive by default 3 | --FILE-- 4 | > { y() } 7 | 8 | x(); 9 | 10 | x() && y(); 11 | 12 | x ( ); 13 | 14 | x ( 15 | ); 16 | 17 | x(foo); // no match 18 | 19 | ?> 20 | --EXPECTF-- 21 | 34 | -------------------------------------------------------------------------------- /tests/phpt/errors/missing_expander_argument.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Test expander call with missing argument 3 | --FILE-- 4 | > { 9 | $$(stringify(/* forgotten $(args) /o\ */)) 10 | } 11 | 12 | match(foo); 13 | 14 | ?> 15 | --EXPECTF-- 16 | 17 | TokenStream expander call without tokens `$$(stringify())` as function Yay\DSL\Expanders\stringify(Yay\TokenStream $ts), in %s.phpt on line 6 18 | -------------------------------------------------------------------------------- /tests/phpt/parsers/ls.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Extra test for ls() 3 | --FILE-- 4 | > { 15 | match($(vars ...(, ){$(var)})) 16 | } 17 | 18 | $a : $b : $c AND $x : $y : $z; 19 | 20 | ?> 21 | --EXPECTF-- 22 | 27 | -------------------------------------------------------------------------------- /src/CompilerPass.php: -------------------------------------------------------------------------------- 1 | closure) return $this->apply(...$args); 8 | } 9 | 10 | function apply(Ast $ast, TokenStream $ts, Index $startNode, Index $endNode, Engine $engine) { 11 | if ($this->closure) return ($this->closure)($ast, $ts, $startNode, $endNode, $engine); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /tests/phpt/unless.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Proof of concept "unless" implementation 3 | --FILE-- 4 | > { 9 | if (! ($(expression))) { 10 | $(body) 11 | } 12 | } 13 | 14 | unless ($x === 1) { 15 | echo "\$x is not 1"; 16 | } 17 | 18 | ?> 19 | --EXPECTF-- 20 | 28 | -------------------------------------------------------------------------------- /tests/ParserOptimizationTest.php: -------------------------------------------------------------------------------- 1 | optimize(), $expected); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /tests/phpt/errors/expansion_undeclared.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Expansion tokens should always have a valid reference 3 | --FILE-- 4 | > $(T_VARIABLE as bar) 9 | } >> { 10 | $(foo) $(bar) $(baz) 11 | // ^ undefined expansion!!! 12 | } 13 | 14 | 15 | $a >> $b; 16 | 17 | ?> 18 | --EXPECTF-- 19 | Undefined macro expansion 'baz', in %s.phpt on line 7 with context: [ 20 | "foo", 21 | 0, 22 | "bar" 23 | ] 24 | -------------------------------------------------------------------------------- /tests/phpt/macro/block_macro_recursion.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Disallow simple infinite macro recursion (constant macros) 3 | --FILE-- 4 | > { FOO(FOO) : FOO() }; 7 | 8 | FOO; 9 | 10 | $(macro) { BAR } >> { BAR BAR(BAR) }; 11 | 12 | BAR; 13 | 14 | $(macro) { C } >> { A } 15 | $(macro) { A } >> { B } 16 | $(macro) { B } >> { C } 17 | 18 | B; 19 | 20 | ?> 21 | --EXPECTF-- 22 | 31 | -------------------------------------------------------------------------------- /tests/phpt/macro/cloaking.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Cloaking (necessary for certain kinds of second order macros) 3 | --FILE-- 4 | > { \\$$(notAnExpander(first)) } 7 | 8 | $(macro) { bar } >> { \\$(notAnExpansion) } 9 | 10 | foo; 11 | 12 | bar; 13 | 14 | $(macro) { $ ( baz ) } >> { \\$$(notAnExpander(second)) } 15 | 16 | baz; 17 | 18 | ?> 19 | --EXPECTF-- 20 | 29 | -------------------------------------------------------------------------------- /tests/phpt/macro/macro_expansion_comments.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Macro expansions should be comment insensitive by default 3 | --FILE-- 4 | > { 14 | y() 15 | } 16 | 17 | x(/** */); 18 | 19 | x() && y(); 20 | 21 | x ( 22 | // 23 | ); 24 | 25 | x(foo); // no match 26 | 27 | ?> 28 | --EXPECTF-- 29 | 40 | -------------------------------------------------------------------------------- /tests/phpt/macro/empty_expansion_002.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Empty expansions 3 | --FILE-- 4 | bar->baz; } >> { }; 7 | 8 | $(macro) { DEBUG { $(layer() as body) } } >> { }; 9 | 10 | $foo->bar; 11 | 12 | $foo->bar->baz; // match 13 | 14 | $foo->/**/bar->/**/baz; // match 15 | 16 | DEBUG { 17 | log('debug!'); 18 | } 19 | 20 | DEBUG(); 21 | 22 | ?> 23 | --EXPECTF-- 24 | bar; 27 | 28 | // match 29 | 30 | // match 31 | 32 | 33 | 34 | DEBUG(); 35 | 36 | ?> 37 | -------------------------------------------------------------------------------- /tests/phpt/macro/dominant_macro.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Dominant macro 3 | --FILE-- 4 | $(T_STRING as B) } } >> { "works" } 7 | 8 | entry_point_a { A => B => C } // backtracks at second "=>" and ignores 9 | 10 | $(macro) { entry_point_b $! { $(T_STRING as A) => $(T_STRING as B) } } >> { "works" } 11 | 12 | entry_point_b { X => Y -> Z } // fails at "->" 13 | 14 | ?> 15 | --EXPECTF-- 16 | 17 | Unexpected T_OBJECT_OPERATOR(->), in %s.phpt on line 9, expected '}'. 18 | -------------------------------------------------------------------------------- /tests/fixtures/parsers.php: -------------------------------------------------------------------------------- 1 | > { 11 | $(foo)($(list ... { ($(item))})); 12 | } 13 | 14 | foo(); 15 | 16 | bar(a, b, c); 17 | 18 | baz(a); 19 | 20 | bus(); 21 | 22 | ?> 23 | --EXPECTF-- 24 | 35 | -------------------------------------------------------------------------------- /tests/phpt/expanders/composition_tokenstream_expander.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Uses a custom fully qualified expansion function --pretty-print 3 | --FILE-- 4 | > { 9 | $$(\Yay\tests\fixtures\expanders\my_cheers_tokenstream_expander( 10 | $$(\Yay\tests\fixtures\expanders\my_hello_tokenstream_expander( 11 | $(matched) 12 | )) 13 | )); 14 | } 15 | 16 | hello(Chris); 17 | 18 | ?> 19 | --EXPECTF-- 20 | 25 | -------------------------------------------------------------------------------- /tests/phpt/macro/operator_null_coalesce_I.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Test for ?! operator --pretty-print 3 | --FILE-- 4 | > { 10 | class $(handler) $(extended ?! {extends \StandardType}) 11 | } 12 | 13 | type Foo 14 | { 15 | } 16 | type Bar extends Foo 17 | { 18 | } 19 | 20 | ?> 21 | --EXPECTF-- 22 | 32 | 33 | -------------------------------------------------------------------------------- /tests/phpt/errors/error_line_number.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Ensures preprocessor syntax errors occurs in the right line number 3 | --FILE-- 4 | > { 11 | 12 | function expansion() 13 | { 14 | $(captured) $(captured); 15 | } 16 | 17 | } 18 | 19 | $(macro) { 20 | 21 | $(token(T_STRING, 'foo')) $! expected 22 | 23 | } >> { 24 | 25 | END; 26 | } 27 | 28 | foo; // L:25 29 | 30 | ?> 31 | --EXPECTF-- 32 | Unexpected T_STRING(foo), in %s.phpt on line 25, expected T_STRING(expected). 33 | -------------------------------------------------------------------------------- /tests/phpt/macro/operator_null_coalesce_II.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Test for ?! operator --pretty-print 3 | --FILE-- 4 | > { 10 | class $(handler) $(undefined ?! {extends \StandardType}) 11 | } 12 | 13 | type Foo 14 | { 15 | } 16 | type Bar extends Foo 17 | { 18 | } 19 | 20 | ?> 21 | --EXPECTF-- 22 | 32 | 33 | -------------------------------------------------------------------------------- /tests/phpt/swap.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Swap 3 | --FILE-- 4 | > { 9 | (list($(A), $(B)) = [$(B), $(A)]) 10 | } 11 | 12 | $x = 1; 13 | $y = 0; 14 | 15 | swap($x, $y); 16 | 17 | var_dump($x, $y); 18 | 19 | swap 20 | ( 21 | $x, 22 | $y 23 | ); 24 | 25 | var_dump($x, $y); 26 | 27 | ?> 28 | --EXPECTF-- 29 | 43 | -------------------------------------------------------------------------------- /tests/phpt/macro/macro_pattern_edge_cases.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Edge cases with '{', T_CURLY_OPEN, T_DOLLAR_OPEN_CURLY_BRACES, T_STRING_VARNAME 3 | --FILE-- 4 | > { ok } 7 | $(macro) { "Foo {$x->$(T_STRING as member)}" } >> { ok $(member) } 8 | $(macro) { "Foo {$x->$(T_STRING as member)()}" } >> { ok $(member) } 9 | $(macro) { "Foo ${$(T_STRING_VARNAME as name)}" } >> { ok $(name) } 10 | 11 | "Foo { literal string }"; 12 | "Foo {$x->y}"; 13 | "Foo {$x->y()}"; 14 | "Foo ${x}"; 15 | 16 | ?> 17 | --EXPECTF-- 18 | 26 | -------------------------------------------------------------------------------- /tests/phpt/expansion_key.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Expansion --pretty-print 3 | --FILE-- 4 | > { 9 | const 10 | $(properties ...(,) i { 11 | Sort_$(property) = $(i) 12 | }) 13 | ; 14 | } 15 | 16 | class Collection 17 | { 18 | protected enum Sort { 19 | Normal, 20 | Key, 21 | Assoc, 22 | } 23 | } 24 | 25 | ?> 26 | --EXPECTF-- 27 | -------------------------------------------------------------------------------- /tests/phpt/macro/empty_expansion_001.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Empty expansions 3 | --FILE-- 4 | HTML here, this should be preserved: @ "debug" public 5 | 6 | > { } 9 | $(macro) { @ } >> { } 10 | $(macro) { public } >> { } 11 | $(macro) { "debug" } >> { } // this comment should be preserved 12 | 13 | @test("debug"); 14 | 15 | class X { 16 | public function test(){} 17 | } 18 | 19 | ?> 20 | --EXPECTF-- 21 | HTML here, this should be preserved: @ "debug" public 22 | 23 | 34 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: php 2 | 3 | php: 4 | - 7.0 5 | - 7.1 6 | - 7.2 7 | - 7.3 8 | - 7.4snapshot 9 | - nightly 10 | 11 | matrix: 12 | allow_failures: 13 | - php: nightly 14 | - php: 7.4snapshot 15 | 16 | sudo: false 17 | 18 | before_script: 19 | - phpenv config-rm xdebug.ini || true 20 | - if [[ $TRAVIS_PHP_VERSION = nightly ]]; then export COMPOSER_FLAGS=" --ignore-platform-reqs"; fi 21 | - composer require satooshi/php-coveralls:~2.0.0 --no-update --dev $COMPOSER_FLAGS 22 | - composer install --prefer-source $COMPOSER_FLAGS 23 | 24 | script: 25 | - php vendor/bin/phpunit 26 | 27 | after_script: 28 | - php vendor/bin/coveralls 29 | -------------------------------------------------------------------------------- /tests/phpt/macro/macro_pattern_layer_matcher_deep.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Non delimited layer matching with nesting 3 | --FILE-- 4 | > { 9 | ($(B) $(rest)) 10 | } 11 | 12 | (level_a (level_b [level_c, 1, 2, 3])) 13 | 14 | // done 15 | 16 | $(macro) { 17 | ($(T_STRING as A) ($(T_STRING as B) ($(T_STRING as C) $(... as rest)))) 18 | } >> { 19 | ($(C) $(rest)) 20 | } 21 | 22 | (level_a (level_b (level_c [level_d, 1, 2, 3, { level_e : (4) }]))) 23 | 24 | ?> 25 | --EXPECTF-- 26 | 35 | -------------------------------------------------------------------------------- /tests/MacroScopeTest.php: -------------------------------------------------------------------------------- 1 | expand(file_get_contents(self::ABSOLUTE_FIXTURES_DIR . '/macros.php'))); 18 | $this->assertSame([true, true], eval($source)); 19 | 20 | $source = str_replace('expand(file_get_contents(self::ABSOLUTE_FIXTURES_DIR . '/run.php'))); 21 | $this->assertSame([true, 'LOCAL_MACRO'], eval($source)); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/BlueContext.php: -------------------------------------------------------------------------------- 1 | $_) $this->map[$token->id()][$id] = true; 13 | } 14 | 15 | function getDisabledMacrosFromToken($token) { 16 | assert($token instanceof Token); 17 | 18 | if (isset($this->map[$token->id()])) return $this->map[$token->id()]; 19 | 20 | return []; 21 | } 22 | 23 | function getDisabledMacrosFromTokens($tokens) { 24 | assert(\is_array($tokens)); 25 | 26 | return array_reduce($tokens, function($macros, $token) { 27 | return $macros += $this->getDisabledMacrosFromToken($token); 28 | }, []); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /tests/phpt/midrule.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Check that midrule works 3 | --FILE-- 4 | index(); 12 | 13 | if (in_array(strtolower($stream->current()), ["true", "false", "null"])) { 14 | return new \Yay\Error(null, null, $stream->last()); 15 | } 16 | 17 | return new \Yay\Ast; 18 | }), 19 | ns() as ns 20 | ) 21 | ) 22 | } >> { 23 | mateched $(ns) 24 | } 25 | 26 | 27 | attempt true 28 | attempt false 29 | attempt null 30 | 31 | attempt strtoupper 32 | attempt ucwords 33 | 34 | ?> 35 | --EXPECTF-- 36 | 46 | -------------------------------------------------------------------------------- /src/AnonymousFunction.php: -------------------------------------------------------------------------------- 1 | isEmpty()) $this->closure = $this->compileAnonymousFunctionArg($ast); 12 | } 13 | 14 | function __invoke(...$args) { 15 | if ($this->closure) return ($this->closure)(...$args); 16 | } 17 | 18 | private function compileAnonymousFunctionArg(Ast $ast) : \Closure { 19 | $arglist = $ast->{'* args'}->implode(); 20 | $body = $ast->{'* body'}->implode(); 21 | $source = "> { 9 | throw new \FlowError('Unreachable point reached.') 10 | } 11 | 12 | $(macro) { UNREACHABLE ($(string() as message)) } >> { 13 | throw new \FlowError($(message)) 14 | } 15 | 16 | $var = '?'; 17 | 18 | switch ($var) { 19 | case '@' : 20 | return true; 21 | case '!' : 22 | return false; 23 | default: 24 | UNREACHABLE(); 25 | } 26 | 27 | UNREACHABLE('Error message.'); 28 | 29 | ?> 30 | --EXPECTF-- 31 | 49 | -------------------------------------------------------------------------------- /tests/phpt/union_catch.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Union exception catch --pretty-print 3 | 4 | Reference https://wiki.php.net/rfc/multiple-catch 5 | 6 | --FILE-- 7 | > { 12 | $(types ... { 13 | catch($(type) $(exception_var)) { 14 | $(body) 15 | } 16 | }) 17 | } 18 | 19 | try { 20 | throw new FooException(); 21 | } catch (\BarException | FooException $e) { 22 | doSomething(); 23 | throw $e; 24 | } catch (\Exception $e) { 25 | doSomethingElse(); 26 | } 27 | 28 | ?> 29 | --EXPECTF-- 30 | 45 | -------------------------------------------------------------------------------- /tests/phpt/expanders/lazy_evaluation_of_expanders.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Only evaluates custom expanders when required --pretty-print 3 | --FILE-- 4 | > { 17 | $(matches ... { 18 | $(fooMatch ? { 19 | $$(\Yay\tests\fixtures\expanders\my_foo_expander($(fooMatch))); 20 | }) 21 | 22 | $(barMatch ? { 23 | $$(\Yay\tests\fixtures\expanders\my_bar_expander($(barMatch))); 24 | }) 25 | }) 26 | } 27 | 28 | foo bar foo 29 | 30 | ?> 31 | --EXPECTF-- 32 | 39 | -------------------------------------------------------------------------------- /tests/phpt/macro/macro_buffer_matcher.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Test buffer() 3 | --FILE-- 4 | ')) $(T_VARIABLE as Y) 8 | } >> { 9 | ($(X) ."hug". $(Y)) 10 | } 11 | 12 | $foo <(o.o)> $bar; 13 | // ^ this is not a hug! 14 | 15 | $a <(o . o)> $b; 16 | 17 | $b<(o . o)>$c; 18 | 19 | $c<(o . o)> $d; 20 | 21 | $d<(o . o)> $e; 22 | 23 | $e 24 | <(o . o)> 25 | $f; 26 | 27 | $f 28 | <(o . o)> 29 | $g; 30 | 31 | $foo < (o.o) > $bar; 32 | // ^ this is not a hug! 33 | 34 | $e 35 | <(o . o)> /**/ 36 | $f; 37 | 38 | ?> 39 | --EXPECTF-- 40 | $bar; 43 | // ^ this is not a hug! 44 | 45 | ($a ."hug". $b); 46 | 47 | ($b ."hug". $c); 48 | 49 | ($c ."hug". $d); 50 | 51 | ($d ."hug". $e); 52 | 53 | ($e ."hug". $f); 54 | 55 | ($f ."hug". $g); 56 | 57 | $foo < (o.o) > $bar; 58 | // ^ this is not a hug! 59 | 60 | ($e ."hug". $f); 61 | 62 | ?> 63 | -------------------------------------------------------------------------------- /tests/phpt/parsers/expression.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Extra test for expression() 3 | --FILE-- 4 | > { 9 | ($$(stringify($(someExpression)))); // expression 10 | } 11 | 12 | null; 13 | 14 | 1; 15 | 16 | 1 + 1; 17 | 18 | SomeConstant; 19 | 20 | []; 21 | 22 | [1, 2, 3]; 23 | 24 | function(){}; 25 | 26 | (function(){}); 27 | 28 | new class {}; 29 | 30 | (new class { function foo(){ } })->foo(); 31 | 32 | ?> 33 | --EXPECTF-- 34 | foo()'); // expression 64 | 65 | 66 | ?> 67 | -------------------------------------------------------------------------------- /tests/phpt/opaque_types.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Opaque types with macros that generate other macros :> 3 | --FILE-- 4 | > { 11 | \\$(macro) { 12 | \\$(either(instanceof, token(','), token('(')) as prec) \\$(optional(indentation()) as whitespace) $(basetype) 13 | } >> { 14 | \\$(prec)\\$(whitespace)$(newtype) 15 | } 16 | } 17 | 18 | type Username = string; 19 | type Password = string; 20 | 21 | function register_user(Username $nick, Password $password ) : User {} 22 | function register_user(\Username $nick, \Password $password ) : User {} 23 | 24 | ?> 25 | --EXPECTF-- 26 | 32 | -------------------------------------------------------------------------------- /tests/phpt/group_use_grammar.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | A proof of concept polyfill for group use --pretty-print 3 | --FILE-- 4 | > { 17 | $(group_use ... { 18 | $(entries ... { 19 | $(entry ... { 20 | use $(type) $(base)\$(name) $(alias ...{as $(label)}); 21 | }) 22 | }) 23 | }) 24 | } 25 | 26 | use A\B\C\{ 27 | Foo, 28 | Foo\Bar, 29 | Baz as Boo, 30 | const X as Y, 31 | function d\e as f 32 | } 33 | 34 | ?> 35 | --EXPECTF-- 36 | 45 | -------------------------------------------------------------------------------- /src/MacroMember.php: -------------------------------------------------------------------------------- 1 | implode(); 21 | 22 | if (0 !== strpos($function, '\\')) $function = $namespace . $function; 23 | 24 | if (! function_exists($function)) { 25 | $tokens = $type->tokens(); 26 | $this->fail( 27 | $error, 28 | $name, 29 | $tokens[0] != '\\' ? $tokens[0]->line() : $tokens[1]->line() 30 | ); 31 | } 32 | 33 | return $function; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /tests/phpt/macro/macro_deep_ast_access_error.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Test macro $(deep[ast][access]) syntax --pretty-print 3 | --FILE-- 4 | > { 30 | matched($(level_a[level_b][level_c][leaf_level_x])); 31 | } 32 | 33 | match { 34 | leaf_level_a 35 | leaf_level_b 36 | leaf_level_c 37 | } 38 | 39 | ?> 40 | --EXPECTF-- 41 | Undefined macro expansion 'level_a[level_b][level_c][leaf_level_x]', in %s.phpt on line 27 with context: [ 42 | "level_a" 43 | ] 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) Márcio Almada 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /phpunit.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 11 | 12 | 13 | 14 | tests/ 15 | 16 | 17 | 18 | 19 | 20 | src/ 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /tests/phpt/test_dsl.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Example DSL for testing --pretty-print 3 | --FILE-- 4 | > { 9 | $(procedure)($(description), function() {$(body)}); 10 | } 11 | 12 | $(macro) { 13 | $(identifier() as procedure) $({...} as body) 14 | } >> { 15 | $(procedure)(function() {$(body)}); 16 | } 17 | 18 | describe 'ArrayObject' { 19 | beforeEach { 20 | $this->arrayObject = new ArrayObject([1, 2, 3]); 21 | } 22 | describe '->count()' { 23 | it 'should return the number of items' { 24 | assert($this->arrayObject->count() === 3, 'expected 3'); 25 | } 26 | } 27 | } 28 | 29 | ?> 30 | --EXPECTF-- 31 | arrayObject = new ArrayObject([1, 2, 3]); 36 | }); 37 | describe('->count()', function () { 38 | it('should return the number of items', function () { 39 | assert($this->arrayObject->count() === 3, 'expected 3'); 40 | }); 41 | }); 42 | }); 43 | 44 | ?> 45 | -------------------------------------------------------------------------------- /bin/yay-pretty: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env php 2 | create(ParserFactory::PREFER_PHP7); 32 | $prettyPrinter = new PrettyPrinter\Standard; 33 | $stmts = $parser->parse($source); 34 | $output = $prettyPrinter->prettyPrintFile($stmts); 35 | 36 | file_put_contents('php://stdout', $output); 37 | } 38 | catch (Exception $e) { 39 | file_put_contents('php://stderr', $e . PHP_EOL); 40 | } 41 | -------------------------------------------------------------------------------- /tests/phpt/macro/macro_pattern_block_matcher.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Some macros 3 | --FILE-- 4 | > { 11 | 12 | test { 13 | $(body) 14 | } 15 | 16 | } 17 | 18 | block { 19 | // random block of code 20 | $foo->reset(); 21 | while ($current = $foo->current()) { 22 | if ($meta = $current->meta()) { 23 | if (! $meta[2]) { 24 | throw new Exception("Foo {$x->y()}"); 25 | } 26 | $current->__construct([$meta[1], null, null]); 27 | $current->tag('crossover', $meta[0]); 28 | } 29 | 30 | $foo->next(); 31 | } 32 | $foo->reset(); 33 | } 34 | 35 | ?> 36 | --EXPECTF-- 37 | reset(); 41 | while ($current = $foo->current()) { 42 | if ($meta = $current->meta()) { 43 | if (! $meta[2]) { 44 | throw new Exception("Foo {$x->y()}"); 45 | } 46 | $current->__construct([$meta[1], null, null]); 47 | $current->tag('crossover', $meta[0]); 48 | } 49 | 50 | $foo->next(); 51 | } 52 | $foo->reset(); 53 | 54 | } 55 | 56 | ?> 57 | -------------------------------------------------------------------------------- /src/Map.php: -------------------------------------------------------------------------------- 1 | map[$key] ?? null; 10 | } 11 | 12 | function add($key, $value = true) { 13 | return $this->map[$key] = $value; 14 | } 15 | 16 | function remove($key) { 17 | unset($this->map[$key]); 18 | } 19 | 20 | function contains($key) : bool { 21 | return isset($this->map[$key]); 22 | } 23 | 24 | function symbols() : array { 25 | return array_keys($this->map); 26 | } 27 | 28 | function count() : int { 29 | return count($this->map); 30 | } 31 | 32 | static function fromValues(array $values = []) : self { 33 | $m = self::fromEmpty(); 34 | foreach($values as $value) $m->add($value); 35 | 36 | return $m; 37 | } 38 | 39 | static function fromKeysAndValues(array $values = []) : self { 40 | $m = self::fromEmpty(); 41 | foreach($values as $key => $value) $m->add($key, $value); 42 | 43 | return $m; 44 | } 45 | 46 | static function fromEmpty() : self { 47 | return new self; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /tests/phpt/this_shorthand.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | This shorthand $-> 3 | --FILE-- 4 | > { $this } 7 | 8 | class Foo 9 | { 10 | private $bar = 'bar'; 11 | 12 | function bar() : string { 13 | return $->bar; 14 | } 15 | 16 | function baz() : string { 17 | return $->bar(); 18 | } 19 | 20 | function this() : self { 21 | return $; 22 | } 23 | 24 | function __toString() { 25 | return "{$->bar}"; 26 | } 27 | } 28 | 29 | // the following should not be matched by the macro 30 | 31 | ${'var'}; 32 | 33 | ${"${var}"}; 34 | 35 | $$var; 36 | 37 | ?> 38 | --EXPECTF-- 39 | bar; 47 | } 48 | 49 | function baz() : string { 50 | return $this->bar(); 51 | } 52 | 53 | function this() : self { 54 | return $this; 55 | } 56 | 57 | function __toString() { 58 | return "{$this->bar}"; 59 | } 60 | } 61 | 62 | // the following should not be matched by the macro 63 | 64 | ${'var'}; 65 | 66 | ${"${var}"}; 67 | 68 | $$var; 69 | 70 | ?> 71 | -------------------------------------------------------------------------------- /tests/phpt/json.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Proof of concept native json support with PEG macro --pretty-print 3 | --FILE-- 4 | > { 28 | JSON_MATCH 29 | } 30 | 31 | json : { 32 | 'a' : true, 33 | 'b' : false, 34 | 'c' : null, 35 | 'd' : 'string', 36 | 'e' : { 37 | 'a' : true, 38 | 'b' : false, 39 | 'c' : null, 40 | 'd' : 'string', 41 | 'e' : { 42 | 'f': {} 43 | }, 44 | 'f' : ['', {'g': {'h': {}}}, null, true, false, [], [1]] 45 | } 46 | }; 47 | 48 | 49 | ?> 50 | --EXPECTF-- 51 | 56 | -------------------------------------------------------------------------------- /tests/phpt/macro/macro_pattern_matched_contiguously.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Macro pattern matched contiguously --pretty-print 3 | --FILE-- 4 | > { 11 | function $(method) ($(args)) { 12 | $(body) 13 | } 14 | } 15 | 16 | /** 17 | * @group small 18 | */ 19 | class FooTest 20 | { 21 | barProvider() 22 | { 23 | return [['a', 'b', true], ['a', 'b', false]]; 24 | } 25 | testBar($a, $b, bool $assertion) 26 | { 27 | 'method body'; 28 | } 29 | testFoo() 30 | { 31 | 'method body'; 32 | } 33 | testBaz() 34 | { 35 | 'method body'; 36 | } 37 | } 38 | ?> 39 | --EXPECTF-- 40 | 67 | -------------------------------------------------------------------------------- /tests/phpt/macro/hygiene.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Macro hygiene 3 | --FILE-- 4 | > { 7 | 8 | $x += $(A); 9 | $y += $(B); 10 | 11 | } 12 | 13 | test($a, $b); 14 | 15 | test($b, $c); 16 | 17 | $(macro) { $(T_VARIABLE as A) += $(T_VARIABLE as B); } >> { $z = ($(A) += $(B)); } 18 | 19 | test($a, $b); 20 | 21 | $(macro) { unsafe_test($(T_VARIABLE as A)) } >> { $unsafe = $$(unsafe($code)) = $(A); } 22 | 23 | unsafe_test($dirty); 24 | 25 | $(macro) { 26 | 27 | retry ( $(layer() as times) ) { $(layer() as body) } 28 | 29 | } >> { 30 | 31 | $times = (int)($(times)); 32 | 33 | retry: { 34 | if ($times > 0) { 35 | $(body); 36 | $times--; 37 | goto retry; 38 | } 39 | } 40 | } 41 | 42 | retry(3) { echo "Attempt..."; } 43 | 44 | ?> 45 | --EXPECTF-- 46 | 0) { 63 | echo "Attempt..."; ; 64 | $times___6--; 65 | goto retry___6; 66 | } 67 | } 68 | 69 | ?> 70 | -------------------------------------------------------------------------------- /tests/phpt/group_use.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | A proof of concept polyfill for group use --pretty-print 3 | --FILE-- 4 | > { 32 | 33 | $(entries ... { 34 | $(entry ... { 35 | use $(type) $(base)\$(name) $(alias ... {as $(label)}); 36 | }) 37 | }) 38 | } 39 | 40 | use A\B\C\{ 41 | Foo, 42 | Foo\Bar, 43 | Baz as Boo, 44 | const X as Y, 45 | function d\e as f 46 | } 47 | 48 | ?> 49 | --EXPECTF-- 50 | 59 | -------------------------------------------------------------------------------- /tests/phpt/guard.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Guards 3 | --FILE-- 4 | > { 9 | guard ($(condition)) { 10 | throw new \GuardError($(message)); 11 | } 12 | } 13 | 14 | $(macro) { guard $((...) as condition) $({...} as body) } >> { 15 | if (! ($(condition))) { 16 | $(body) 17 | throw new \GuardError("Guard error."); 18 | } 19 | } 20 | 21 | /// 22 | 23 | function repeat(int $times, callable $action) { 24 | guard ($times > 1) : '$times must be larger than 1'; 25 | 26 | guard ($callable instanceof Action::class) { 27 | throw new \InvalidArgumentException('$callable must be instance of Action.'); 28 | } 29 | } 30 | 31 | ?> 32 | --EXPECTF-- 33 | 1)) { 41 | throw new \GuardError('$times must be larger than 1'); 42 | 43 | throw new \GuardError("Guard error."); 44 | } 45 | 46 | if (! ($callable instanceof Action::class)) { 47 | throw new \InvalidArgumentException('$callable must be instance of Action.'); 48 | 49 | throw new \GuardError("Guard error."); 50 | } 51 | } 52 | 53 | ?> 54 | -------------------------------------------------------------------------------- /tests/fixtures/expression/bad.php: -------------------------------------------------------------------------------- 1 | '++', 5 | __LINE__ => '&$foo->bar["baz"]', 6 | __LINE__ => '&$var', 7 | __LINE__ => '1 1 1', 8 | __LINE__ => '1 -- 1', 9 | __LINE__ => '(', 10 | __LINE__ => '() ', 11 | __LINE__ => '(() ', 12 | __LINE__ => '))', 13 | __LINE__ => ')', 14 | __LINE__ => '(((([))))', 15 | __LINE__ => '1 + 1 + (2 + )', 16 | __LINE__ => '1 + 1 + (2 + (3 + 4)', 17 | __LINE__ => '(function', 18 | __LINE__ => '${var}', // because var is reserved, because PHP 19 | __LINE__ => '${const}', // because const is reserved, because PHP 20 | __LINE__ => '($foo->bar(->baz)', 21 | __LINE__ => '($foo)->bar(->baz)', 22 | __LINE__ => '($foo->bar()->baz', 23 | __LINE__ => '($foo->bar) = ', 24 | __LINE__ => '@->baz', 25 | __LINE__ => '"foo $foo->bar->baz->bar $foo->bar(1, 2, 3)->baz->bar()->biz $foo', 26 | __LINE__ => '"foo ${bar {$baz} $boo {$foo->bar->baz} {$foo->bar->baz()} {$foo->bar()->baz()}"', 27 | __LINE__ => '"foo ${bar $baz} $boo {$foo->bar->baz} {$foo->bar->baz()} {$foo->bar()->baz()}"', 28 | __LINE__ => '[1,, 2,, (3)]', 29 | __LINE__ => '[1, 2, (3)],', 30 | __LINE__ => '[1, 2, 3,,] + [1, 2, 3, 4, 5,]', 31 | __LINE__ => '[1, 2, 3,] + [1,, 2, 3, 4, 5,]', 32 | __LINE__ => '$foo->bar()->(baz)', 33 | ]; 34 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "yay/yay", 3 | "description": "A high level PHP Pre-Processor", 4 | "license": "MIT", 5 | "keywords": [ 6 | "pre-processor", 7 | "language", 8 | "syntax" 9 | ], 10 | "authors": [ 11 | { 12 | "name": "Márcio Almada", 13 | "email": "marcio3w@gmail.com", 14 | "homepage": "https://github.com/marcioAlmada" 15 | } 16 | ], 17 | "require": { 18 | "php": "7.*", 19 | "ext-mbstring": "*", 20 | "ext-tokenizer": "*", 21 | "nikic/php-parser": "^2.1|^3.0|^4.0", 22 | "docopt/docopt": "^1.0" 23 | }, 24 | "autoload": { 25 | "files": [ 26 | "src/parsers.php", 27 | "src/parsers_internal.php", 28 | "src/expanders.php" 29 | ], 30 | "psr-4": { 31 | "Yay\\": "src/" 32 | } 33 | }, 34 | "autoload-dev": { 35 | "psr-4": { 36 | "Yay\\": "tests/" 37 | }, 38 | "files": [ 39 | "tests/fixtures/parsers.php", 40 | "tests/fixtures/expanders.php" 41 | ] 42 | }, 43 | "bin": [ 44 | "bin/yay", 45 | "bin/yay-pretty" 46 | ], 47 | "require-dev": { 48 | "phpunit/phpunit": "~6.5", 49 | "phpbench/phpbench": "@dev" 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /tests/phpt/defer.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Defer 3 | --FILE-- 4 | > { 9 | $deferred = new class($(deferred)) { 10 | $$(unsafe { 11 | private $deferred = null; 12 | function __construct(callable $deferred){ $this->deferred = $deferred; } 13 | function __destruct(){ ($this->deferred)(); } 14 | }) 15 | }; 16 | } 17 | 18 | function app($input){ 19 | defer function(){ echo 'Bye!', PHP_EOL; }; 20 | defer function() use ($input) { echo "Handling {$input}\n"; }; 21 | echo 'Hello world!', PHP_EOL; 22 | } 23 | 24 | app("request"); 25 | 26 | ?> 27 | --EXPECTF-- 28 | deferred = $deferred; } 34 | function __destruct(){ ($this->deferred)(); } 35 | 36 | }; 37 | $deferred___1 = new class(function()use($input){echo "Handling {$input}\n"; }) { 38 | private $deferred = null; 39 | function __construct(callable $deferred){ $this->deferred = $deferred; } 40 | function __destruct(){ ($this->deferred)(); } 41 | 42 | }; 43 | echo 'Hello world!', PHP_EOL; 44 | } 45 | 46 | app("request"); 47 | 48 | ?> 49 | -------------------------------------------------------------------------------- /tests/phpt/macro/macro_deep_ast_access.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Test macro $(deep[ast][access]) syntax --pretty-print 3 | --FILE-- 4 | > { 30 | matched($(level_a[leaf_level_a])); 31 | matched($(level_a[level_b][leaf_level_b])); 32 | matched($(level_a[level_b][level_c][leaf_level_c])); 33 | // equivalent to: 34 | matched($(level_a ... { $(leaf_level_a) })); 35 | matched($(level_a ... { $(level_b ... { $(leaf_level_b) }) })); 36 | matched($(level_a ... { $(level_b ... { $(level_c ... { $(leaf_level_c) })}) })); 37 | } 38 | 39 | match { 40 | leaf_level_a 41 | leaf_level_b 42 | leaf_level_c 43 | } 44 | 45 | ?> 46 | --EXPECTF-- 47 | 58 | -------------------------------------------------------------------------------- /src/Expected.php: -------------------------------------------------------------------------------- 1 | tokens = $tokens; 11 | } 12 | 13 | function append(self $expected) : self { 14 | foreach ($expected->tokens as $token) $this->tokens[] = $token; 15 | 16 | return $this; 17 | } 18 | 19 | function all() : array { 20 | return $this->tokens; 21 | } 22 | 23 | function negate() : self { 24 | $expected = clone $this; 25 | $expected->negation = true; 26 | 27 | return $expected; 28 | } 29 | 30 | function __toString() : string { 31 | return 32 | ($this->negation ? 'not ' : '') . 33 | implode( 34 | ' or ' . ($this->negation ? 'not ' : ''), 35 | array_unique( 36 | array_map( 37 | function(Token $t) { 38 | return $t->dump(); 39 | }, 40 | $this->all() 41 | ) 42 | ) 43 | ) 44 | ; 45 | } 46 | 47 | function raytrace() : string { 48 | return 49 | ($this->negation ? 'not ' : '') . 50 | implode( 51 | ' | ', 52 | array_map( 53 | function(Token $t){ return $t->dump(); }, 54 | $this->tokens 55 | ) 56 | ) 57 | ; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /tests/phpt/retry.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Proof of concept "retry" implementation --pretty-print 3 | --FILE-- 4 | > { goto retry; } 7 | 8 | $(macro) { 9 | try { 10 | $(layer() as try_body) 11 | } 12 | catch($(ns() as type) $(T_VARIABLE as exception)) { 13 | $(layer() as catch_body) 14 | } 15 | } >> { 16 | /* 17 | This implementation is full of issues: 18 | 19 | - no recursion support (try catch inside another try catch) 20 | - macro hygienization can fail under weird circuntances 21 | - a more procedural macro api would be necessary 22 | */ 23 | try { 24 | retry: 25 | $(try_body) 26 | } 27 | catch($(type) $(exception)) { 28 | $$(expand($(catch_body))) 29 | } 30 | } 31 | 32 | function request_something() { 33 | static $count = 0; 34 | 35 | if ($count < 3) { 36 | $count++; 37 | throw new \Exception("Tried {$count}", 1); 38 | } 39 | } 40 | 41 | try { 42 | request_something(); 43 | } 44 | catch (Exception $e) { 45 | echo $e->getMessage() . PHP_EOL; 46 | retry; 47 | } 48 | 49 | echo 'END'; 50 | ?> 51 | --EXPECTF-- 52 | getMessage() . PHP_EOL; 67 | goto retry___0; 68 | } 69 | echo 'END'; 70 | 71 | ?> 72 | -------------------------------------------------------------------------------- /tests/phpt/issues/ircmaxell-php-compiler#29.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Test for bug found at https://github.com/ircmaxell/php-compiler/pull/29 --pretty-print 3 | --FILE-- 4 | > { 28 | $(stmts ... { 29 | $(assignop ? ... { 30 | $(binary ? ... { 31 | $(binary_op ... { 32 | $(binary_xor ? ... { 33 | $(result) = $this->context->builder->bitwiseXor($(binary_left), $__right); 34 | }) 35 | }) 36 | }) 37 | }) 38 | }) 39 | } 40 | 41 | compile { 42 | $result = $value ^ 1; 43 | } 44 | 45 | ?> 46 | --EXPECTF-- 47 | Error unpacking a non unpackable Ast node on `$(binary_xor?... {` at line 29 with context: [ 48 | "^" 49 | ] 50 | 51 | Hint: use a non ellipsis expansion as in `$(binary_xor ? {` 52 | -------------------------------------------------------------------------------- /tests/fixtures/expanders.php: -------------------------------------------------------------------------------- 1 | first()->line() 12 | ) 13 | ) 14 | ; 15 | } 16 | 17 | function my_cheers_tokenstream_expander(TokenStream $ts) : TokenStream { 18 | $str = str_replace("'", "", (string) $ts); 19 | 20 | return 21 | TokenStream::fromSequence( 22 | new Token( 23 | T_CONSTANT_ENCAPSED_STRING, "'{$str} Cheers!'", $ts->first()->line() 24 | ) 25 | ) 26 | ; 27 | } 28 | 29 | function my_hello_ast_expander(\Yay\Ast $ast) : \Yay\Ast { 30 | return new \Yay\Ast($ast->label(), new Token( 31 | T_CONSTANT_ENCAPSED_STRING, "'Hello, {$ast->token()}. From Ast.'", $ast->token()->line() 32 | )); 33 | } 34 | 35 | function my_cheers_ast_expander(\Yay\Ast $ast) : \Yay\Ast { 36 | $str = str_replace("'", "", (string) $ast->token()); 37 | 38 | return new \Yay\Ast($ast->label(), new Token( 39 | T_CONSTANT_ENCAPSED_STRING, "'{$str} Cheers!'", $ast->token()->line() 40 | )); 41 | } 42 | 43 | function my_foo_expander(\Yay\Ast $ast) { 44 | return TokenStream::fromSourceWithoutOpenTag(sprintf("'called %s(%s)'", __FUNCTION__, $ast->implode())); 45 | } 46 | 47 | function my_bar_expander(\Yay\Ast $ast) { 48 | return TokenStream::fromSourceWithoutOpenTag(sprintf("'called %s(%s)'", __FUNCTION__, $ast->implode())); 49 | } 50 | -------------------------------------------------------------------------------- /tests/phpt/primitve_generics.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Proof of concept inlined generics 3 | --FILE-- 4 | 10 | 11 | } >> { 12 | 13 | class { 14 | private $stack = []; 15 | 16 | function push($(type) $item) { 17 | $this->stack[] = $item; 18 | } 19 | 20 | function pop() : $(type) { 21 | return end($this->stack); 22 | } 23 | } 24 | } 25 | 26 | new Stack 27 | < 28 | stdclass 29 | >; 30 | 31 | $stack = new Stack; 32 | 33 | $stack->push(new stdclass); 34 | $stack->push(new ArrayObject); 35 | 36 | $stack = new Stack<\Some\Full\Qualified\ClassName>; 37 | 38 | ?> 39 | --EXPECTF-- 40 | stack[] = $item; 47 | } 48 | 49 | function pop() : stdclass { 50 | return end($this->stack); 51 | } 52 | }; 53 | 54 | $stack = new class { 55 | private $stack = []; 56 | 57 | function push(stdclass $item) { 58 | $this->stack[] = $item; 59 | } 60 | 61 | function pop() : stdclass { 62 | return end($this->stack); 63 | } 64 | }; 65 | 66 | $stack->push(new stdclass); 67 | $stack->push(new ArrayObject); 68 | 69 | $stack = new class { 70 | private $stack = []; 71 | 72 | function push(\Some\Full\Qualified\ClassName $item) { 73 | $this->stack[] = $item; 74 | } 75 | 76 | function pop() : \Some\Full\Qualified\ClassName { 77 | return end($this->stack); 78 | } 79 | }; 80 | 81 | ?> 82 | -------------------------------------------------------------------------------- /tests/phpt/macro/compiler_pass.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Test compiler pass arguments --pretty-print 3 | --FILE-- 4 | > function(\Yay\Ast $ast, \Yay\TokenStream $ts, \Yay\Index $start, \Yay\Index $end, \Yay\Engine $engine){ 11 | ob_start(); 12 | var_dump($ast, $ts, $start, $end, get_class($engine)); 13 | $result = PHP_EOL . ob_get_clean(); 14 | 15 | $ast->append(new \Yay\Ast('debug', new \Yay\Token(T_CONSTANT_ENCAPSED_STRING, $result))); 16 | } 17 | >> { 18 | $$(stringify($(debug))) 19 | } 20 | 21 | $x($y); 22 | 23 | ?> 24 | --EXPECTF-- 25 | 30 | string(0) "" 31 | ["ast":protected]=> 32 | array(%d) { 33 | ["A"]=> 34 | object(Yay\\Token)#%d (%d) { 35 | [0]=> 36 | string(14) "T_VARIABLE($x)" 37 | } 38 | [0]=> 39 | object(Yay\\Token)#%d (%d) { 40 | [0]=> 41 | string(%d) "\'(\'" 42 | } 43 | ["B"]=> 44 | object(Yay\\Token)#%d (%d) { 45 | [0]=> 46 | string(14) "T_VARIABLE($y)" 47 | } 48 | [1]=> 49 | object(Yay\\Token)#%d (%d) { 50 | [0]=> 51 | string(%d) "\')\'" 52 | } 53 | [2]=> 54 | array(%d) { 55 | } 56 | } 57 | ["meta":"Yay\\Ast":private]=> 58 | NULL 59 | } 60 | object(Yay\\TokenStream)#%d (%d) { 61 | ["first":protected]=> 62 | object(Yay\\NodeStart)#%d (%d) { 63 | } 64 | ["current":protected]=> 65 | object(Yay\\Node)#%d (%d) { 66 | [0]=> 67 | object(Yay\\Token)#%d (%d) { 68 | [0]=> 69 | string(%d) "\';\'" 70 | } 71 | } 72 | ["last":protected]=> 73 | object(Yay\\NodeEnd)#%d (%d) { 74 | } 75 | } 76 | object(Yay\\Node)#%d (%d) { 77 | [0]=> 78 | object(Yay\\Token)#%d (%d) { 79 | [0]=> 80 | string(14) "T_VARIABLE($x)" 81 | } 82 | } 83 | object(Yay\\Node)#%d (%d) { 84 | [0]=> 85 | object(Yay\\Token)#%d (%d) { 86 | [0]=> 87 | string(%d) "\')\'" 88 | } 89 | } 90 | string(10) "Yay\\Engine" 91 | '; 92 | 93 | ?> 94 | -------------------------------------------------------------------------------- /tests/TokenTest.php: -------------------------------------------------------------------------------- 1 | assertEquals( 50 | $assertion 51 | , 52 | $token_a->equals($token_b) 53 | , 54 | "{$token_a->dump()} !== {$token_b->dump()}" 55 | ); 56 | } 57 | 58 | function testIs() { 59 | $token = new Token('$'); 60 | $this->assertTrue($token->is('$')); 61 | $this->assertFalse($token->is('!')); 62 | 63 | $token = new Token(T_STRING); 64 | $this->assertFalse($token->is(T_OPEN_TAG)); 65 | $this->assertTrue($token->is(T_STRING)); 66 | 67 | $token = new Token(T_STRING, '"value"', null); 68 | $this->assertFalse($token->is(T_OPEN_TAG)); 69 | $this->assertTrue($token->is(T_STRING)); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /bin/yay: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env php 2 | 33 | yay --macros= 34 | yay --macros= 35 | yay -h | --help 36 | yay --version 37 | 38 | Options: 39 | -h --help Show this screen. 40 | --version Show version. 41 | --macros= PHP glob pattern for macros to be loaded. Ex: my/macros/*.yay [default: ''] 42 | 43 | Examples: 44 | 45 | ``` 46 | # Process file: 47 | yay input.php > output.php 48 | 49 | # Process file, preload project macros: 50 | > yay --macros="./project-macros/*.yay" input.php > output.php 51 | 52 | # Process stdin: 53 | > cat input.php | yay > output.php 54 | 55 | # Process stdin, preload project macros: 56 | > cat input.php | yay --macros="./project-macros/*.yay" > output.php 57 | ``` 58 | DOC; 59 | 60 | $argv = Docopt::handle($doc, include __DIR__ . '/../meta.php'); 61 | 62 | $file = $argv[''] ?? 'php://stdin'; 63 | 64 | $source = file_get_contents($file); 65 | 66 | $engine = new Engine; 67 | 68 | gc_disable(); 69 | 70 | foreach(glob((string) $argv['--macros']) as $f) $engine->expand(file_get_contents($f), $f); 71 | 72 | $expansion = $engine->expand($source, $file); 73 | 74 | gc_enable(); 75 | 76 | file_put_contents('php://stdout', $expansion); 77 | } 78 | catch (YayPreprocessorError $e) { 79 | file_put_contents('php://stderr', $e . PHP_EOL); 80 | } 81 | catch (Exception $e) { 82 | file_put_contents('php://stderr', $e . PHP_EOL); 83 | } 84 | -------------------------------------------------------------------------------- /src/Error.php: -------------------------------------------------------------------------------- 1 | expected = $expected; 36 | $this->unexpected = $unexpected; 37 | $this->last = $last; 38 | } 39 | 40 | function as(string $label = null) : Result { 41 | return $this; 42 | } 43 | 44 | function withMeta(Map $meta) : Result { 45 | return $this; 46 | } 47 | 48 | function meta() : Map { 49 | return Map::fromEmpty(); 50 | } 51 | 52 | function with(self $e) { 53 | $this->and = $e; 54 | } 55 | 56 | function message() : string { 57 | $errors = []; 58 | $error = $this; 59 | while($error) { 60 | $unexpected = ($error->unexpected ?: $error->last); 61 | $prefix = sprintf( 62 | $error->unexpected ? self::UNEXPECTED : self::UNEXPECTED_END, 63 | $unexpected->dump(), 64 | $unexpected->line() 65 | ); 66 | if (isset($errors[$prefix])) 67 | $errors[$prefix]->append($error->expected); 68 | else 69 | $errors[$prefix] = $error->expected; 70 | 71 | $error = $error->and; 72 | } 73 | 74 | $messages = []; 75 | foreach ($errors as $prefix => $expected) { 76 | $messages[] = $prefix . sprintf(self::EXPECTED, (string) $expected); 77 | } 78 | 79 | return implode(PHP_EOL, $messages); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /tests/phpt/short_functions_with_no_braces.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | legit arrow functions with lexical scoping 3 | 4 | use --pretty-print 5 | 6 | --FILE-- 7 | '))// the swimming arrow operator 20 | 21 | $(expression() as single_expression_body) // the short closure's body 22 | 23 | $(_() as scope) // dummy label signaling that $(scope) exists, it's added dynamically through the compiler pass 24 | 25 | } >> function($ast) { 26 | $defined = []; 27 | foreach ($ast->{'args'} as $node) $defined[(string) $node['arg']['arg_name']] = true; 28 | 29 | $scoped = []; 30 | $scope = new \Yay\Ast('scope'); 31 | foreach ($ast->{'* single_expression_body'}->tokens() as $token) { 32 | if ( 33 | $token->is(T_VARIABLE) && 34 | ('$this' !== (string) $token) && 35 | false === isset($defined[(string) $token]) && 36 | false === isset($scoped[(string) $token]) 37 | ){ 38 | $scope->push(new \Yay\Ast('var', $token)); 39 | $scoped[(string) $token] = true; 40 | } 41 | } 42 | 43 | $ast->append($scope); 44 | } >> { 45 | $(scope ? { 46 | [ 47 | $(scope ...(, ) { $(var) = $(var) ?? null}), 48 | 'short_closure' => function ($(args ...(, ){ $(arg ...{$(type) $(arg_name)}) })) use($(scope ...(, ) { $(var) })) $(return_type) { 49 | return $(single_expression_body); 50 | } 51 | ]['short_closure'] 52 | }) 53 | $(scope ! { 54 | function ($(args ...(, ){ $(arg ...{$(type) $(arg_name)}) })) $(return_type) { 55 | return $(single_expression_body); 56 | } 57 | }) 58 | } 59 | 60 | $y = 100; 61 | // 62 | $result = array_map((int $x):int ==> $x * 2 * ++$y , range(1, 10)); 63 | // 64 | assert($y === 100); 65 | var_dump($result); 66 | 67 | ?> 68 | --EXPECTF-- 69 | function (int $x) use($y) : int { 74 | return $x * 2 * ++$y; 75 | }]['short_closure'], range(1, 10)); 76 | // 77 | assert($y === 100); 78 | var_dump($result); 79 | 80 | ?> 81 | 82 | -------------------------------------------------------------------------------- /tests/phpt/primitive_union_return_types.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Primitive union return types --pretty-print 3 | --FILE-- 4 | |<\B>|" 11 | $( 12 | optional 13 | ( 14 | chain 15 | ( 16 | token(':'), 17 | ls 18 | ( 19 | ns() as type, 20 | token('|') 21 | ) 22 | as union 23 | ) 24 | as return_type 25 | ) 26 | ) 27 | $({...} as body) 28 | } >> { 29 | function $(name) ($(args)) 30 | { 31 | $fn = (function($(args)){ 32 | $(body) 33 | }); 34 | 35 | $ret = isset($this) 36 | ? $fn->call($this, ...function_get_args()) 37 | : $fn(...function_get_args()); 38 | 39 | if ( 40 | $(return_type ... { 41 | $(union ... ( && ) { 42 | ! $ret instanceof $(type) 43 | }) 44 | }) 45 | ) { 46 | throw new TypeError("Some fancy type Error"); 47 | } 48 | 49 | return $ret; 50 | } 51 | } 52 | 53 | class Foo { 54 | function bar(bool $x) : A|Foo\B|\Foo\Bar\C { 55 | if ($x) { 56 | return new Z; 57 | } else { 58 | return new A; 59 | } 60 | } 61 | } 62 | 63 | $fn = function() : Foo|Bar { 64 | return null; 65 | }; 66 | 67 | ?> 68 | --EXPECTF-- 69 | call($this, ...function_get_args()) : $fn(...function_get_args()); 83 | if (!$ret instanceof A && !$ret instanceof Foo\B && !$ret instanceof \Foo\Bar\C) { 84 | throw new TypeError("Some fancy type Error"); 85 | } 86 | return $ret; 87 | } 88 | } 89 | $fn = function () { 90 | $fn = function () { 91 | return null; 92 | }; 93 | $ret = isset($this) ? $fn->call($this, ...function_get_args()) : $fn(...function_get_args()); 94 | if (!$ret instanceof Foo && !$ret instanceof Bar) { 95 | throw new TypeError("Some fancy type Error"); 96 | } 97 | return $ret; 98 | }; 99 | 100 | ?> 101 | -------------------------------------------------------------------------------- /tests/phpt/parsers/array_arg.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Test for constant array as parsec arguments 3 | --FILE-- 4 | 2, 3 => '4', '5' => 6, 'foo' => 'bar', 'baz' => [ 19 | 1 => 2, 3 => '4', '5' => 6, 'foo' => 'bar', 'baz' => 7 20 | ] 21 | ]) as nested_assoc_array, 22 | \Yay\tests\fixtures\parsers\var_dump_args_parser([[], ['foo' => []], 2 => ['bar' => [1, 2, 3]]]) as random_typing 23 | ) as var_dump) 24 | 25 | ; 26 | 27 | } >> { 28 | $(var_dump) 29 | } 30 | 31 | dummy; 32 | 33 | ?> 34 | --EXPECTF-- 35 | 43 | int(1) 44 | [1]=> 45 | string(1) "2" 46 | [2]=> 47 | string(3) "foo" 48 | [3]=> 49 | string(3) "bar" 50 | [4]=> 51 | int(5) 52 | [5]=> 53 | array(0) { 54 | } 55 | } 56 | *//* 57 | nested_non_assoc_array: array(6) { 58 | [0]=> 59 | int(1) 60 | [1]=> 61 | string(1) "2" 62 | [2]=> 63 | string(3) "foo" 64 | [3]=> 65 | string(3) "bar" 66 | [4]=> 67 | int(5) 68 | [5]=> 69 | array(6) { 70 | [0]=> 71 | int(1) 72 | [1]=> 73 | string(1) "2" 74 | [2]=> 75 | string(3) "foo" 76 | [3]=> 77 | string(3) "bar" 78 | [4]=> 79 | int(5) 80 | [5]=> 81 | array(0) { 82 | } 83 | } 84 | } 85 | *//* 86 | nested_assoc_array: array(5) { 87 | [1]=> 88 | int(2) 89 | [3]=> 90 | string(1) "4" 91 | [5]=> 92 | int(6) 93 | ["foo"]=> 94 | string(3) "bar" 95 | ["baz"]=> 96 | array(5) { 97 | [1]=> 98 | int(2) 99 | [3]=> 100 | string(1) "4" 101 | [5]=> 102 | int(6) 103 | ["foo"]=> 104 | string(3) "bar" 105 | ["baz"]=> 106 | int(7) 107 | } 108 | } 109 | *//* 110 | random_typing: array(3) { 111 | [0]=> 112 | array(0) { 113 | } 114 | [1]=> 115 | array(1) { 116 | ["foo"]=> 117 | array(0) { 118 | } 119 | } 120 | [2]=> 121 | array(1) { 122 | ["bar"]=> 123 | array(3) { 124 | [0]=> 125 | int(1) 126 | [1]=> 127 | int(2) 128 | [2]=> 129 | int(3) 130 | } 131 | } 132 | } 133 | */ 134 | 135 | ?> 136 | -------------------------------------------------------------------------------- /src/Macro.php: -------------------------------------------------------------------------------- 1 | id = (__CLASS__)::$_id++; 24 | $this->tags = $tags; 25 | $this->pattern = $pattern; 26 | $this->compilerPass = $compilerPass; 27 | $this->expansion = $expansion; 28 | 29 | $this->enableParserTracer = $this->tags->contains('enable_parser_tracer'); 30 | $this->isTerminal = !$this->expansion->isRecursive(); 31 | } 32 | 33 | function id() : int { 34 | return $this->id; 35 | } 36 | 37 | function tags() : Map { 38 | return $this->tags; 39 | } 40 | 41 | function pattern() : Pattern { 42 | return $this->pattern; 43 | } 44 | 45 | function expansion() : Expansion { 46 | return $this->expansion; 47 | } 48 | 49 | function apply(TokenStream $ts, Engine $engine) { 50 | 51 | $from = $ts->index(); 52 | 53 | try { 54 | if($this->enableParserTracer) Parser::setTracer(new ParserTracer\CliParserTracer); 55 | 56 | $crossover = $this->pattern->match($ts); 57 | } 58 | finally { 59 | if($this->enableParserTracer) Parser::setTracer(new ParserTracer\NullParserTracer); 60 | } 61 | 62 | if ($crossover instanceof Ast ) { 63 | $ts->unskip(); 64 | $to = $ts->index(); 65 | 66 | ($this->compilerPass)($crossover, $ts, $from, $to->previous, $engine); 67 | 68 | $blueContext = $engine->blueContext(); 69 | $blueMacros = $blueContext->getDisabledMacrosFromTokens($crossover->tokens()); 70 | 71 | if ($this->isTerminal && isset($blueMacros[$this->id])) { // already expanded 72 | $ts->jump($from); 73 | 74 | return; 75 | } 76 | 77 | $ts->extract($from, $to); 78 | 79 | $expansion = $this->expansion->expand($crossover, $engine); 80 | 81 | $blueMacros[$this->id] = true; 82 | 83 | $node = $expansion->index(); 84 | while ($node instanceof Node) { 85 | // paint blue context with tokens from expansion and disabled macros 86 | $blueContext->addDisabledMacros($node->token, $blueMacros); 87 | $node = $node->next; 88 | } 89 | 90 | $ts->inject($expansion); 91 | 92 | $engine->cycle()->next(); 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /tests/phpt/issues/issue#30.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Issue #30 --pretty-print 3 | --FILE-- 4 | > { 30 | $(annotations ... { 31 | new class(new \ReflectionClass($(class_name)::class)) extends $(class) 32 | { 33 | public function __construct(\ReflectionClass $context) 34 | { 35 | $fields = []; 36 | $(annotation_arguments ... { 37 | $this->$(field) = $fields[$$(stringify($(field)))] = $(value); 38 | }) 39 | parent::__construct($fields, $context); 40 | } 41 | }; 42 | }) 43 | class $(class_name) 44 | } 45 | 46 | @EmptyAparameters(); 47 | @Any(foo = 1); 48 | @Some(foo = 2, bar = 3); 49 | @ParametersWithTrailingDelimiter(foo = 4, bar = 5,); 50 | class Test 51 | { 52 | } 53 | 54 | ?> 55 | --EXPECTF-- 56 | foo = $fields['foo'] = 1; 72 | parent::__construct($fields, $context); 73 | } 74 | }; 75 | new class(new \ReflectionClass(Test::class)) extends Some 76 | { 77 | public function __construct(\ReflectionClass $context) 78 | { 79 | $fields = []; 80 | $this->foo = $fields['foo'] = 2; 81 | $this->bar = $fields['bar'] = 3; 82 | parent::__construct($fields, $context); 83 | } 84 | }; 85 | new class(new \ReflectionClass(Test::class)) extends ParametersWithTrailingDelimiter 86 | { 87 | public function __construct(\ReflectionClass $context) 88 | { 89 | $fields = []; 90 | $this->foo = $fields['foo'] = 4; 91 | $this->bar = $fields['bar'] = 5; 92 | parent::__construct($fields, $context); 93 | } 94 | }; 95 | class Test 96 | { 97 | } 98 | 99 | ?> 100 | -------------------------------------------------------------------------------- /tests/phpt/short_functions_with_lexical_scope.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Shorthand function with support for: 3 | [x] speed 4 | [x] lexical scoping through generated explicit use() directive 5 | [x] return types 6 | [x] argument types 7 | [ ] default argument values // requires constantExpression() parser 8 | 9 | use --pretty-print 10 | 11 | --FILE-- 12 | // the swimming arrow operator 25 | 26 | $({...} as body) // the short closure's body 27 | 28 | $(_() as scope) // dummy label signaling that $(scope) exists, it's added dynamically through the compiler pass 29 | 30 | } >> function($ast) { 31 | $defined = []; 32 | foreach ($ast->{'args'} as $node) $defined[(string) $node['arg']['arg_name']] = true; 33 | 34 | $scoped = []; 35 | $scope = new \Yay\Ast('scope'); 36 | foreach ($ast->{'body'} as $token) { 37 | if ( 38 | $token->is(T_VARIABLE) && 39 | ('$this' !== (string) $token) && 40 | false === isset($defined[(string) $token]) && 41 | false === isset($scoped[(string) $token]) 42 | ){ 43 | $scope->push(new \Yay\Ast('var', $token)); 44 | $scoped[(string) $token] = true; 45 | } 46 | } 47 | 48 | $ast->append($scope); 49 | } >> { 50 | $(scope ? { 51 | [ 52 | $(scope ...(, ) { $(var) = $(var) ?? null}), 53 | 'short_closure' => function ($(args ...(, ){ $(arg ...{$(type) $(arg_name)}) })) use($(scope ...(, ) { $(var) })) $(return_type) { 54 | return $(body); 55 | } 56 | ]['short_closure'] 57 | }) 58 | $(scope ! { 59 | function ($(args ...(, ){ $(arg ...{ $(type) $(arg_name) }) })) $(return_type) { 60 | return $(body); 61 | } 62 | }) 63 | } 64 | 65 | $y = 100; 66 | // 67 | $result = array_map((int $x):int ~> { $x * 2 * ++$y }, range(1, 10)); 68 | // 69 | assert($y === 100); 70 | var_dump($result); 71 | // 72 | // 73 | $y = 100; 74 | // 75 | $result = array_map((int $x):int ~> { $x * 2 }, range(1, 10)); 76 | // 77 | var_dump($result); 78 | 79 | ?> 80 | --EXPECTF-- 81 | function (int $x) use($y) : int { 86 | return $x * 2 * ++$y; 87 | }]['short_closure'], range(1, 10)); 88 | // 89 | assert($y === 100); 90 | var_dump($result); 91 | // 92 | // 93 | $y = 100; 94 | // 95 | $result = array_map(function (int $x) : int { 96 | return $x * 2; 97 | }, range(1, 10)); 98 | // 99 | var_dump($result); 100 | 101 | ?> 102 | 103 | -------------------------------------------------------------------------------- /benchmarks/EngineBenchmark.php: -------------------------------------------------------------------------------- 1 | fixtures = new class { 11 | function create(string $source) : array { 12 | $file = sys_get_temp_dir() . '/' . substr(md5($source), 0, 6); 13 | file_put_contents($file, $source); 14 | return [ 15 | 'file' => $file, 16 | 'length' => strlen($source), 17 | ]; 18 | } 19 | 20 | function load(string $file) : string { 21 | return file_get_contents($file); 22 | } 23 | }; 24 | } 25 | 26 | public function sourceProvider() 27 | { 28 | yield $this->literalEntryPointMacrosFixture(1000, 2000); 29 | yield $this->literalEntryPointMacrosFixture(2500, 5000); 30 | yield $this->literalEntryPointMacrosFixture(5000, 10000); 31 | yield $this->literalEntryPointMacrosFixture(7500, 15000); 32 | yield $this->literalEntryPointMacrosFixture(15000, 30000); 33 | yield $this->nonLiteralEntryPointMacrosFixture(5000); 34 | yield $this->nonLiteralEntryPointMacrosFixture(10000); 35 | } 36 | 37 | /** 38 | * @OutputTimeUnit("milliseconds") 39 | * @Warmup(2) 40 | * @Iterations(5) 41 | * @ParamProviders({"sourceProvider"}) 42 | */ 43 | public function benchMacroExpansion(array $params) 44 | { 45 | $expansion = (new Engine)->expand($this->fixtures->load($params['file']), 'bench.php'); 46 | } 47 | 48 | private function literalEntryPointMacrosFixture(int $min, int $max) : array { 49 | $source = <<> { __function } 53 | $(macro) { extends } >> { __extends } 54 | $(macro) { Parser } >> { __Parser } 55 | $(macro) { new } >> { __new } 56 | $(macro) { instanceof $(T_STRING as s) } >> { __instanceof $(s) } 57 | SRC; 58 | 59 | $fixture = str_replace( 60 | '> { __new } // redundant macro 68 | SRC; 69 | 70 | while(substr_count($source, PHP_EOL) < $max) $source .= $fixture; 71 | 72 | return $this->fixtures->create($source); 73 | } 74 | 75 | private function nonLiteralEntryPointMacrosFixture(int $max) : array { 76 | $source = <<> { __$(s) } // slow macro 80 | SRC; 81 | 82 | $fixture = str_replace( 83 | 'fixtures->create($source); 90 | } 91 | } 92 | 93 | -------------------------------------------------------------------------------- /src/ParserTracer/CliParserTracer.php: -------------------------------------------------------------------------------- 1 | 0, 21 | self::OPTION_PRETTY_TRACE => true, 22 | self::OPTION_TRUNCATE_TRACE => 120, 23 | self::OPTION_COLOR_TRACE => "\033[0;30m", 24 | self::OPTION_COLOR_ATTEMPT => "\033[0;0m", 25 | self::OPTION_COLOR_ERROR => "\033[0;31m", 26 | self::OPTION_COLOR_PRODUCTION => "\033[0;32m", 27 | ]; 28 | 29 | private 30 | $lastDepth = 0, 31 | $stack = [], 32 | $options = [] 33 | ; 34 | 35 | function __construct(array $options = []) 36 | { 37 | $this->options = 38 | $options 39 | + [self::OPTION_TRUNCATE_TRACE => ((int) exec('tput cols')) ?: self::OPTIONS_DEFAULT[self::OPTION_TRUNCATE_TRACE]] 40 | + self::OPTIONS_DEFAULT 41 | ; 42 | } 43 | 44 | function push(Parser $parser) 45 | { 46 | $this->stack[] = $parser; 47 | } 48 | 49 | function pop(Parser $parser) 50 | { 51 | $popped = array_pop($this->stack); 52 | 53 | assert($parser === $popped); 54 | } 55 | 56 | function trace(Index $index, string $event = 'trace', string $message = '') 57 | { 58 | $parser = end($this->stack); 59 | 60 | $output = sprintf( 61 | "%s%s %s%s at %s from Parser<%s>", 62 | $this->stackmap(), 63 | $this->options['color.' . $event] ?? $this->options[self::OPTION_COLOR_TRACE], 64 | $message ? $event . ' ' : $event, 65 | ($message ? "\033[0;7m {$message} " . $this->options['color.' . $event] ?? $this->options[self::OPTION_COLOR_TRACE] : ''), 66 | $index->token ? $index->token->dump() : 'EOF', 67 | $parser->__debugInfo()['label'] ?: $this->autolabel($parser) 68 | ); 69 | 70 | if (($offset = mb_strlen($output)) > $this->options[self::OPTION_TRUNCATE_TRACE] + ($message ? 14 : 0)) { 71 | $output = mb_substr($output, 0, $this->options[self::OPTION_TRUNCATE_TRACE] + ($message ? 14 : 0)); 72 | $output .= '...)>'; 73 | } 74 | 75 | echo $output, "\033[0m", PHP_EOL; 76 | } 77 | 78 | function autolabel(Parser $parser) : string 79 | { 80 | $expected = implode(' ', array_unique(array_map(function($t){ return str_replace('()', '', $t->dump()); }, $parser->expected()->all()))); 81 | 82 | return str_replace('Yay\\', '', $parser->__debugInfo()['type']) . '(' . $expected . ')'; 83 | } 84 | 85 | function stackmap() : string 86 | { 87 | $depth = count($this->stack); 88 | $output = str_pad((string) $depth, 3); 89 | 90 | $color = ((int) $depth % 6) + 200; 91 | 92 | if ($this->options[self::OPTION_PRETTY_TRACE]) $output .= str_repeat('│', $depth); 93 | 94 | $this->lastDepth = $depth; 95 | 96 | return $output; 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /tests/phpt/property_accessors.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | General property --pretty-print 3 | --FILE-- 4 | > { 9 | $(class) { 10 | use \Pre\AccessorsTrait; 11 | 12 | $(body) 13 | } 14 | } 15 | 16 | $(macro) { 17 | private $(T_VARIABLE as variable) { 18 | $( 19 | repeat 20 | ( 21 | either 22 | ( 23 | chain 24 | ( 25 | get, 26 | optional(chain(token(':'), ns())) as getter_return_type, 27 | between(token('{'), layer(), token('}')) as getter_body 28 | ) 29 | as getter 30 | , 31 | chain 32 | ( 33 | set, 34 | token('('), 35 | layer() as setter_args, 36 | token(')'), 37 | optional(chain(token(':'), ns())) as setter_return_type, 38 | between(token('{'), layer(), token('}')) as setter_body 39 | ) 40 | as setter 41 | , 42 | chain 43 | ( 44 | unset, 45 | optional(chain(token(':'), ns())) as unsetter_return_type, 46 | between(token('{'), layer(), token('}')) as unsetter_body 47 | ) 48 | as unsetter 49 | ) 50 | ) as accessors 51 | ) 52 | }; 53 | } >> { 54 | private $(variable); 55 | 56 | $(accessors ... { 57 | $(setter ?... { 58 | private function $$(concat(__set_ $$(unvar($(variable)))))($(setter_args)) $(setter_return_type) { 59 | $(setter_body) 60 | } 61 | 62 | }) 63 | 64 | $(getter ?... { 65 | private function $$(concat(__get_ $$(unvar($(variable)))))() $(getter_return_type) { 66 | $(getter_body) 67 | } 68 | }) 69 | 70 | $(unsetter ?... { 71 | private function $$(concat(__unset_ $$(unvar($(variable)))))() $(unsetter_return_type) { 72 | $(unsetter_body) 73 | } 74 | }) 75 | }) 76 | } 77 | 78 | namespace App; 79 | 80 | class Sprocket 81 | { 82 | private $type { 83 | set(string $value) { 84 | $this->type = $value; 85 | } 86 | 87 | get :string { 88 | return $this->type; 89 | } 90 | 91 | unset { 92 | $this->type = ''; 93 | } 94 | }; 95 | 96 | private $name { 97 | get :string { 98 | return $this->name; 99 | } 100 | }; 101 | } 102 | 103 | ?> 104 | --EXPECTF-- 105 | type = $value; 116 | } 117 | private function __get_type() : string 118 | { 119 | return $this->type; 120 | } 121 | private function __unset_type() 122 | { 123 | $this->type = ''; 124 | } 125 | private $name; 126 | private function __get_name() : string 127 | { 128 | return $this->name; 129 | } 130 | } 131 | 132 | ?> 133 | -------------------------------------------------------------------------------- /src/expanders.php: -------------------------------------------------------------------------------- 1 | first()->line() 17 | ) 18 | ) 19 | ; 20 | } 21 | 22 | function unvar(TokenStream $ts) : TokenStream { 23 | $str = preg_replace('/^\$+/', '', (string) $ts); 24 | 25 | return 26 | TokenStream::fromSequence( 27 | new Token( 28 | T_CONSTANT_ENCAPSED_STRING, $str 29 | ) 30 | ) 31 | ; 32 | } 33 | 34 | function concat(TokenStream $ts) : TokenStream { 35 | $ts->reset(); 36 | $buffer = ''; 37 | $line = $ts->current()->line(); 38 | while($t = $ts->current()) { 39 | $str = (string) $t; 40 | if (! preg_match('/^\w+$/', $str)) 41 | throw new YayPreprocessorError( 42 | "Only valid identifiers are mergeable, '{$t->dump()}' given."); 43 | 44 | $buffer .= $str; 45 | $ts->next(); 46 | } 47 | 48 | return TokenStream::fromSequence(new Token(T_STRING, $buffer, $line)); 49 | } 50 | 51 | function hygienize(TokenStream $ts, Engine $engine) : TokenStream { 52 | $ts->reset(); 53 | 54 | $cg = (object)[ 55 | 'node' => null, 56 | 'scope' => $engine->cycle()->id(), 57 | 'ts' => $ts 58 | ]; 59 | 60 | $saveNode = function(Parser $parser) use($cg) { 61 | return midrule(function($ts) use ($cg, $parser) { 62 | $cg->node = $ts->index(); 63 | 64 | return $parser->parse($ts); 65 | }); 66 | }; 67 | 68 | traverse 69 | ( 70 | // hygiene must skip whatever is passed through the $$(unsafe()) expander 71 | chain(buffer('$$'), token('('), token(T_STRING, 'unsafe'), either(parentheses(), braces()), token(')')) 72 | , 73 | either 74 | ( 75 | $saveNode(token(T_VARIABLE)) 76 | , 77 | chain($saveNode(identifier()), token(':')) 78 | , 79 | chain(token(T_GOTO), $saveNode(identifier())) 80 | ) 81 | ->onCommit(function(Ast $result) use ($cg) { 82 | if (($t = $cg->node->token) && (($value = (string) $t) !== '$this')) 83 | $cg->node->token = new Token($t->type(), "{$value}___{$cg->scope}", $t->line()); 84 | 85 | $cg->node = null; 86 | }) 87 | ) 88 | ->parse($ts); 89 | 90 | $ts->reset(); 91 | 92 | return $ts; 93 | } 94 | 95 | function unsafe(TokenStream $ts) : TokenStream { return $ts; } 96 | 97 | function whitespace(TokenStream $ts) : TokenStream { 98 | return 99 | TokenStream::fromSequence( 100 | new Token( 101 | T_WHITESPACE, str_repeat(PHP_EOL, substr_count((string) $ts, PHP_EOL) + 1), $ts->first()->line() 102 | ) 103 | ) 104 | ; 105 | } 106 | 107 | function expand(TokenStream $ts, Engine $engine) : TokenStream { 108 | 109 | $ts = TokenStream::fromSource($engine->expand((string) $ts, $engine->currentFileName(), Engine::GC_ENGINE_DISABLED)); 110 | 111 | return $ts; 112 | } 113 | -------------------------------------------------------------------------------- /src/Token.php: -------------------------------------------------------------------------------- 1 | true, 9 | T_COMMENT => true, 10 | T_DOC_COMMENT => true, 11 | ]; 12 | 13 | /** 14 | * pseudo token types 15 | */ 16 | const 17 | ANY = 1021, 18 | NONE = 1032, 19 | BUFFER = 1054, 20 | ESCAPED = 1065 21 | ; 22 | 23 | /** 24 | * lookup table used to dump pseudo token types 25 | */ 26 | const TOKENS = [ 27 | self::ANY => 'ANY', 28 | self::NONE => 'NONE', 29 | self::BUFFER => 'BUFFER', 30 | self::ESCAPED => 'ESCAPED' 31 | ]; 32 | 33 | protected 34 | $type, 35 | $value, 36 | $line, 37 | $skippable = false 38 | ; 39 | 40 | private 41 | $id 42 | ; 43 | 44 | protected static $_id = 0; 45 | 46 | function __construct($type, $value = null, $line = null) { 47 | 48 | assert(null === $this->type, "Attempt to modify immutable token."); 49 | 50 | $this->id = (__CLASS__)::$_id++; 51 | 52 | if (\is_string($type)) { 53 | $this->value = $type; 54 | } 55 | else { 56 | $this->skippable = isset((__CLASS__)::SKIPPABLE[$type]); 57 | $this->value = $value; 58 | } 59 | 60 | $this->type = $type; 61 | $this->line = $line; 62 | 63 | assert($this->check()); 64 | } 65 | 66 | function __toString() { 67 | return (string) $this->value; 68 | } 69 | 70 | function __debugInfo() { 71 | return [$this->dump()]; 72 | } 73 | 74 | function dump(): string { 75 | $name = $this->name(); 76 | 77 | return $this->type === $this->value ? "'{$name}'" : "{$name}({$this->value})"; 78 | } 79 | 80 | function is($type) { 81 | return $this->type === $type; 82 | } 83 | 84 | function equals(self $token) { 85 | return 86 | ($this->type === $token->type && 87 | ($this->value === $token->value ?: 88 | ($token->value === null ?: $this->value === null))); 89 | } 90 | 91 | function isSkippable() { 92 | return $this->skippable; 93 | } 94 | 95 | function name(): string { 96 | return 97 | ($this->type === $this->value) 98 | ? $this->type 99 | : (__CLASS__)::TOKENS[$this->type] ?? \token_name($this->type) 100 | ; 101 | } 102 | 103 | function type() /* : string|int */ { 104 | return $this->type; 105 | } 106 | 107 | function value() { 108 | return $this->value; 109 | } 110 | 111 | function line() { 112 | return $this->line; 113 | } 114 | 115 | function id() { 116 | return $this->id; 117 | } 118 | 119 | function jsonSerialize() { 120 | return (string) $this; 121 | } 122 | 123 | private function check() { 124 | assert(\is_int($this->id)); 125 | assert(\is_bool($this->skippable)); 126 | assert(\is_int($this->type) || (\is_string($this->type) && \strlen($this->type) === 1), "Token type must be int or string[0]."); 127 | assert(\is_string($this->value) || (\is_null($this->value)), "Token value must be string or null."); 128 | assert(\is_int($this->line) || (\is_null($this->line)), "Token line must be int or null."); 129 | 130 | return true; 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /src/Parser.php: -------------------------------------------------------------------------------- 1 | type = $type; 34 | $this->stack = $stack; 35 | $this->withErrorLevel($this->errorLevel); 36 | } 37 | 38 | final function __toString() : string 39 | { 40 | return $this->type . ($this->label !== '' ? " as {$this->label}" : ''); 41 | } 42 | 43 | final function __debugInfo() 44 | { 45 | return [ 46 | 'type' => $this->type, 47 | 'label' => $this->label, 48 | 'stack' => $this->stack, 49 | ]; 50 | } 51 | 52 | final function __clone() 53 | { 54 | $this->onCommit = null; 55 | } 56 | 57 | function parse(TokenStream $ts) /*: Result|null*/ 58 | { 59 | try { 60 | self::$tracer->push($this); 61 | 62 | $index = $ts->index(); 63 | 64 | self::$tracer->trace($index, 'attempt'); 65 | 66 | $result = $this->parser($ts, ...$this->stack); 67 | 68 | if ($result instanceof Ast) { 69 | self::$tracer->trace($index, 'production', $result->implode()); 70 | 71 | if (null !== $this->onCommit) ($this->onCommit)($result); 72 | } 73 | else { 74 | $ts->jump($index); 75 | self::$tracer->trace($index, 'error'); 76 | } 77 | } 78 | catch(YayPreprocessorError $e) { 79 | $ts->jump($index); 80 | self::$tracer->trace($index, 'error'); 81 | 82 | throw $e; 83 | } 84 | finally { 85 | self::$tracer->pop($this); 86 | } 87 | 88 | return $result; 89 | } 90 | 91 | function optimize() : self 92 | { 93 | if (false === $this->optimized) { 94 | $this->type = '*' . $this->type; 95 | $this->optimized = true; 96 | array_walk_recursive($this->stack, function(&$parser) { 97 | if ($parser instanceof self) $parser = $parser->optimize(); 98 | }); 99 | } 100 | 101 | return $this; 102 | } 103 | 104 | function as(string $label) : self 105 | { 106 | if ('' !== (string) $label) { 107 | if(false !== strpos($label, ' ')) 108 | throw new YayPreprocessorError( 109 | "Parser label cannot contain spaces, '{$label}' given."); 110 | 111 | $this->label = $label; 112 | } 113 | 114 | return $this; 115 | } 116 | 117 | final function onCommit(callable $fn) : self 118 | { 119 | $this->onCommit = $fn; 120 | 121 | return $this; 122 | } 123 | 124 | final function withErrorLevel(bool $errorLevel) : self 125 | { 126 | if ($this->errorLevel !== $errorLevel) { 127 | $this->errorLevel = $errorLevel; 128 | 129 | if ($this->stack) { 130 | array_walk_recursive($this->stack, function($substack){ 131 | if ($substack instanceof self) $substack->withErrorLevel($this->errorLevel); 132 | }); 133 | } 134 | } 135 | 136 | return $this; 137 | } 138 | 139 | final function error(TokenStream $ts, Expected $expected = null) /*: Error|null*/ 140 | { 141 | if ($this->errorLevel === Error::ENABLED) 142 | return new Error($expected ?: $this->expected(), $ts->current(), $ts->last()); 143 | } 144 | 145 | final static function setTracer(ParserTracer $tracer) 146 | { 147 | self::$tracer = $tracer; 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /tests/phpt/enums.phpt: -------------------------------------------------------------------------------- 1 | --TEST-- 2 | Proof of concept backport of enums from future --pretty-print 3 | --FILE-- 4 | > { 31 | class $(name) implements Enum { 32 | private static $store; 33 | 34 | private function __construct() {} 35 | 36 | static function __callStatic(string $field, array $args) : self { 37 | if(! self::$store) { 38 | self::$store = new \stdclass; 39 | $(fields ... { 40 | self::$store->$(field) = new class extends $(name) {}; 41 | }) 42 | } 43 | 44 | if (isset(self::$store->$field)) return self::$store->$field; 45 | 46 | throw new \Exception('Undefined enum field ' . __CLASS__ . "->{$field}."); 47 | } 48 | } 49 | } 50 | 51 | $(macro) { 52 | $( 53 | // sequence that matches the enum field access syntax: 54 | chain( 55 | ns() as class, // matches a namespace 56 | token(T_DOUBLE_COLON), // matches T_DOUBLE_COLON used for static access 57 | not(class), // avoids matching ::class resolution syntax 58 | label() as field, // matches the enum field name 59 | not(token('(')) // avoids matching static method calls 60 | ) 61 | ) 62 | } >> { 63 | \enum_field_or_class_constant($(class)::class, $$(stringify($(field)))) 64 | } 65 | 66 | // 67 | 68 | enum Fruits { Apple, Orange } 69 | 70 | // macro should work with Enums only 71 | 72 | var_dump(Fruits::Orange instanceof Fruits); 73 | var_dump(Fruits::Orange <=> Fruits::Apple); 74 | var_dump(Fruits::Apple); 75 | 76 | // macro skips class constants access 77 | 78 | class NotEnum { 79 | const Orange = 1; 80 | static function method() {} 81 | } 82 | 83 | var_dump(NotEnum::Orange); 84 | 85 | // macro skips ::class resolution 86 | 87 | var_dump(NotEnum::class); 88 | 89 | // macro skips static method calls 90 | 91 | var_dump(NotEnum::method()); 92 | 93 | ?> 94 | --EXPECTF-- 95 | Apple = new class extends Fruits 117 | { 118 | }; 119 | self::$store->Orange = new class extends Fruits 120 | { 121 | }; 122 | } 123 | if (isset(self::$store->{$field})) { 124 | return self::$store->{$field}; 125 | } 126 | throw new \Exception('Undefined enum field ' . __CLASS__ . "->{$field}."); 127 | } 128 | } 129 | // macro should work with Enums only 130 | var_dump(\enum_field_or_class_constant(Fruits::class, 'Orange') instanceof Fruits); 131 | var_dump(\enum_field_or_class_constant(Fruits::class, 'Orange') <=> \enum_field_or_class_constant(Fruits::class, 'Apple')); 132 | var_dump(\enum_field_or_class_constant(Fruits::class, 'Apple')); 133 | // macro skips class constants access 134 | class NotEnum 135 | { 136 | const Orange = 1; 137 | static function method() 138 | { 139 | } 140 | } 141 | var_dump(\enum_field_or_class_constant(NotEnum::class, 'Orange')); 142 | // macro skips ::class resolution 143 | var_dump(NotEnum::class); 144 | // macro skips static method calls 145 | var_dump(NotEnum::method()); 146 | 147 | ?> 148 | -------------------------------------------------------------------------------- /tests/SpecsTest.php: -------------------------------------------------------------------------------- 1 | id() predictable during tests only! 22 | */ 23 | function md5($foo) { return $foo; } 24 | } 25 | 26 | function specProvider() : array { 27 | $files = new RegexIterator( 28 | new RecursiveIteratorIterator( 29 | new RecursiveDirectoryIterator(__DIR__ . '/phpt/') 30 | ) 31 | , 32 | '/\.phpt$/', RegexIterator::MATCH 33 | ); 34 | 35 | $tests = []; 36 | 37 | foreach ($files as $i => $test) 38 | $tests[$i] = [new Test($test->getRealPath())]; 39 | 40 | return $tests; 41 | } 42 | 43 | /** 44 | * @dataProvider specProvider 45 | */ 46 | function testSpecs(Test $test) { 47 | try { 48 | $test->run(); 49 | $test->clean(); 50 | } catch(\Exception $e) { 51 | $test->dump(); 52 | 53 | throw $e; 54 | } 55 | 56 | $this->assertNotEquals(Test::BORKED, $test->status(), 'Test borked!'); 57 | } 58 | } 59 | 60 | class Test { 61 | 62 | const 63 | BORKED = 'BORKED', 64 | FAILED = 'FAILED', 65 | PASSED = 'PASSED', 66 | DIFFCM = '\ 67 | diff --strip-trailing-cr \ 68 | --label "%s" \ 69 | --label "%s" \ 70 | --unified "%s" "%s"' 71 | ; 72 | 73 | 74 | protected 75 | $engine, 76 | $status, 77 | $name, 78 | $source, 79 | $expected, 80 | $out, 81 | $file, 82 | $file_expect, 83 | $file_diff, 84 | $file_out 85 | ; 86 | 87 | function __construct(string $file) { 88 | $this->file = $file; 89 | $this->file_expect = preg_replace('/\.phpt$/', '.exp', $this->file); 90 | $this->file_diff = preg_replace('/\.phpt$/', '.diff', $this->file); 91 | $this->file_out = preg_replace('/\.phpt$/', '.out', $this->file); 92 | 93 | $this->engine = new Engine; 94 | } 95 | 96 | function run() { 97 | $raw = is_readable($this->file) ? file_get_contents($this->file) : ''; 98 | $sections = array_values( 99 | array_filter( 100 | array_map( 101 | 'trim', preg_split('/^--(TEST|FILE|EXPECTF)--$/m', $raw)))); 102 | 103 | if(3 === count($sections)) { 104 | list($this->name, $this->source, $this->expected) = $sections; 105 | 106 | try { 107 | $this->out = $this->engine->expand($this->source, $this->file); 108 | 109 | if (false !== strpos($this->name, '--pretty-print')) { 110 | $parser = (new ParserFactory)->create(ParserFactory::PREFER_PHP7); 111 | $prettyPrinter = new PrettyPrinter\Standard; 112 | $stmts = $parser->parse($this->out); 113 | $this->out = $prettyPrinter->prettyPrintFile($stmts) . PHP_EOL . PHP_EOL . '?>'; 114 | } 115 | } catch(YayPreprocessorError $e) { 116 | $this->out = $e->getMessage(); 117 | // $this->out = (string) $e; 118 | } catch(YayRuntimeException $e) { 119 | $this->out = $e->getMessage(); 120 | } catch(\PhpParser\Error $e){ 121 | $this->out = 'PHP ' . $e->getMessage(); 122 | // $this->out = (string) $e; 123 | } catch(Exception $e) { 124 | $this->out = (string) $e; 125 | } 126 | 127 | try{ 128 | \PHPUnit\Framework\Assert::assertStringMatchesFormat($this->expected, $this->out); 129 | $this->status = self::PASSED; 130 | } 131 | catch(Exception $e) { 132 | $this->status = self::FAILED; 133 | 134 | throw $e; 135 | } 136 | } 137 | else 138 | $this->status = self::BORKED; 139 | } 140 | 141 | function status() : string { 142 | return $this->status; 143 | } 144 | 145 | function diff() : string { 146 | $diff = ''; 147 | if($this->status === self::FAILED) { 148 | exec( 149 | sprintf( 150 | self::DIFFCM, "expect", "out", $this->file_expect, $this->file_out), $out); 151 | $diff = implode(PHP_EOL, (array) $out); 152 | } 153 | 154 | return $diff; 155 | } 156 | 157 | function dump() { 158 | @file_put_contents($this->file_expect, $this->expected . PHP_EOL); 159 | @file_put_contents($this->file_out, $this->out . PHP_EOL); 160 | @file_put_contents($this->file_diff, $this->diff()); 161 | // @file_put_contents($this->file_php, $this->source); 162 | } 163 | 164 | function clean() { 165 | @unlink($this->file_expect); 166 | @unlink($this->file_out); 167 | @unlink($this->file_diff); 168 | // @unlink($this->file_php); 169 | } 170 | 171 | function __toString() { 172 | $relative_file = str_replace(__DIR__ . '/', '', $this->file); 173 | 174 | return "[{$this->status}] {$this->name} [{$relative_file}]"; 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /tests/AstTest.php: -------------------------------------------------------------------------------- 1 | ['baz' => true, 'buz' => false]]); 12 | 13 | $this->assertSame('foo', $ast->label()); 14 | $this->assertSame(['bar'], $ast->symbols()); 15 | 16 | $childAst = $ast->{'* foo bar'}; 17 | $this->assertInstanceOf(Ast::class, $childAst); 18 | $this->assertSame('bar', $childAst->label()); 19 | $this->assertSame([], $childAst->symbols()); 20 | $this->assertSame(null, $childAst->unwrap()); 21 | $this->assertNull($childAst->{'baz'}); 22 | $this->assertNull($childAst->{'buz'}); 23 | $this->assertNull($childAst->{'undefined'}); 24 | 25 | $childAst = $ast->{'* bar'}; 26 | $this->assertInstanceOf(Ast::class, $childAst); 27 | $this->assertSame('bar', $childAst->label()); 28 | $this->assertSame(['baz', 'buz'], $childAst->symbols()); 29 | $this->assertSame(['baz' => true, 'buz' => false], $childAst->unwrap()); 30 | $this->assertTrue($childAst->{'baz'}); 31 | $this->assertFalse($childAst->{'buz'}); 32 | $this->assertNull($childAst->{'undefined'}); 33 | } 34 | 35 | function providerForTestMapAstCastOnFailure() { 36 | return [ 37 | ['* defined', 'null', 'null'], 38 | ['* undefined', 'bool', 'boolean'], 39 | ['* undefined', 'array', 'array'], 40 | ['* undefined', 'token', preg_quote(Token::class)], 41 | ]; 42 | } 43 | 44 | /** 45 | * @dataProvider providerForTestMapAstCastOnFailure 46 | */ 47 | function testMapAstCastOnFailure(string $path, string $castMethod, string $typeName) { 48 | $this->expectException(YayPreprocessorError::class); 49 | $this->expectExceptionMessageRegExp("/^Ast cannot be casted to '{$typeName}'$/"); 50 | $ast = new Ast('', ['defined' => true]); 51 | var_dump($ast->{$path}->$castMethod()); 52 | } 53 | 54 | function providerForTestAstCast() { 55 | return [ 56 | ['* some null', 'null', null], 57 | ['* some boolean', 'bool', true], 58 | ['* some string', 'string', 'foo'], 59 | ['* some array', 'array', ['foo', 'bar']], 60 | ['* some token', 'token', new Token(';')], 61 | ['* some tokens', 'tokens', []], 62 | ]; 63 | } 64 | 65 | /** 66 | * @dataProvider providerForTestAstCast 67 | */ 68 | function testAstCast(string $path, string $castMethod, $expected) { 69 | $ast = new Ast('', [ 70 | 'some' => [ 71 | 'null' => null, 72 | 'boolean' => true, 73 | 'string' => 'foo', 74 | 'array' => ['foo', 'bar'], 75 | 'token' => new Token(';'), 76 | 'tokens' => ['deep' => ['inside' => []]], 77 | ] 78 | ]); 79 | 80 | if ($expected instanceof Token) 81 | $this->assertEquals((string) $expected, (string) $ast->{$path}->$castMethod()); 82 | else 83 | $this->assertEquals($expected, $ast->{$path}->$castMethod()); 84 | } 85 | 86 | function testAstFlattenning() { 87 | $ast = new Ast('', [ 88 | 'deep' => [ 89 | 'token' => $token1 = new Token(T_STRING, 'foo'), 90 | 'deeper' => [ 91 | 'token' => $token2 = new Token(T_STRING, 'bar'), 92 | ], 93 | ] 94 | ]); 95 | 96 | $this->assertEquals([$token1, $token2], $ast->tokens()); 97 | 98 | $flattened = $ast->flatten(); 99 | 100 | $this->assertInstanceOf(Ast::class, $flattened); 101 | 102 | $this->assertEquals([$token1, $token2], $flattened->tokens()); 103 | 104 | $this->assertEquals([$token1, $token2], $flattened->unwrap()); 105 | 106 | $this->assertEquals([$token1, $token2], $flattened->array()); 107 | } 108 | 109 | function testAstSet() { 110 | $ast = new Ast('label', [ 111 | 'deep' => [ 112 | 'token' => new Token(T_STRING, 'foo'), 113 | 'deeper' => [ 114 | 'tokens' => [0 => new Token(T_STRING, 'bar'), 1 => new Token(T_STRING, 'baz')], 115 | ], 116 | ], 117 | ]); 118 | 119 | $ast->set('deep token', $patchedFooToken = new Token(T_STRING, 'patched_foo')); 120 | 121 | $ast->{'deep deeper tokens 0'} = $patchedBarToken = new Token(T_STRING, 'patched_bar'); 122 | $ast->{'deep deeper tokens 1'} = $patchedBazToken = new Token(T_STRING, 'patched_baz'); 123 | 124 | $expected = [ 125 | 'deep' => [ 126 | 'token' => $patchedFooToken, 127 | 'deeper' => [ 128 | 'tokens' => [0 => $patchedBarToken, 1 => $patchedBazToken], 129 | ], 130 | ], 131 | ]; 132 | 133 | $this->assertSame($expected, $ast->unwrap()); 134 | $this->assertSame($expected, $ast->array()); 135 | } 136 | 137 | function testAstHiddenNodes() { 138 | $exposed = new Token(T_STRING, 'exposed'); 139 | $hidden = new Token(T_STRING, '_hidden'); 140 | $ast = new Ast('', [ 141 | 'exposed' => $exposed, 142 | '_hidden' => $hidden, 143 | 'deep' => [ 144 | 'exposed' => $exposed, 145 | '_hidden' => $hidden, 146 | ] 147 | ]); 148 | $this->assertSame([$exposed, $exposed], $ast->tokens()); 149 | $this->assertSame('exposed,exposed', $ast->implode(',')); 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /src/Ast.php: -------------------------------------------------------------------------------- 1 | ast = $ast; 25 | $this->label = $label; 26 | } 27 | 28 | function __set($path, $value) { 29 | return $this->set($path, $value); 30 | } 31 | 32 | function set($strPath, $value) { 33 | $keys = preg_split('/\s+/', $strPath); 34 | 35 | if ([] === $keys) return; 36 | 37 | $current = &$this->ast; 38 | foreach ($keys as $key) { 39 | if (!is_array($current)) $current = []; 40 | $current = &$current[$key]; 41 | } 42 | $current = $value; 43 | } 44 | 45 | function __get($path) { 46 | return $this->get($path); 47 | } 48 | 49 | function get($strPath) { 50 | $ret = null; 51 | $path = preg_split('/\s+/', $strPath); 52 | 53 | if ($wrap = ('*' === $path[0])) { 54 | array_shift($path); 55 | } 56 | 57 | $ret = $this->getIn((array) $this->ast, $path); 58 | 59 | if ($wrap) { 60 | $label = end($path) ?: ''; 61 | $ret = new self($label, $ret instanceof Ast ? $ret->unwrap() : $ret); 62 | } 63 | 64 | return $ret; 65 | } 66 | 67 | function unwrap() { 68 | return $this->ast; 69 | } 70 | 71 | function token() { 72 | if ($this->ast instanceof Token) return $this->ast; 73 | 74 | $this->failCasting(Token::class); 75 | } 76 | 77 | function tokens() { 78 | $tokens = []; 79 | $exposed = []; 80 | 81 | if (\is_array($this->ast)) $exposed = $this->ast; 82 | else $exposed = [$this->ast]; 83 | 84 | array_walk_recursive( 85 | $exposed, 86 | function($i, $key) use(&$tokens){ 87 | if (0 === strpos((string) $key, self::NULL_LABEL)) return; 88 | if($i instanceof Token) $tokens[] = $i; 89 | elseif ($i instanceof self) $tokens = array_merge($tokens, $i->tokens()); 90 | } 91 | ); 92 | 93 | return $tokens; 94 | } 95 | 96 | function null() { 97 | if (\is_null($this->ast)) return $this->ast; 98 | 99 | $this->failCasting('null'); 100 | } 101 | 102 | function bool() { 103 | if (\is_bool($this->ast)) return $this->ast; 104 | 105 | $this->failCasting('boolean'); 106 | } 107 | 108 | function string() { 109 | if (\is_string($this->ast)) return $this->ast; 110 | 111 | $this->failCasting('string'); 112 | } 113 | 114 | 115 | function array() { 116 | if (\is_array($this->ast)) return $this->ast; 117 | 118 | $this->failCasting('array'); 119 | } 120 | 121 | function list() { 122 | foreach (array_keys($this->array()) as $i) if($i !== self::NULL_LABEL) yield $i => $this->{"* {$i}"}; 123 | } 124 | 125 | function flatten() : self { 126 | return new self($this->label, $this->tokens()); 127 | } 128 | 129 | function append(self $ast) : self { 130 | if ('' !== $ast->label) { 131 | if (isset($this->ast[$ast->label])) 132 | throw new YayPreprocessorError("Duplicated AST label '{$ast->label}'."); 133 | 134 | $this->ast[$ast->label] = $ast->ast; 135 | } 136 | else $this->ast[] = $ast->ast; 137 | 138 | return $this; 139 | } 140 | 141 | function push(self $ast) : self { 142 | $this->ast[] = $ast->label ? [$ast->label => $ast->ast] : $ast->ast; 143 | 144 | return $this; 145 | } 146 | 147 | function isEmpty() : bool { 148 | return null === $this->ast || [] === $this->ast; 149 | } 150 | 151 | function as(string $label = '') : Result { 152 | if ('' !== $label) $this->label = $label; 153 | 154 | return $this; 155 | } 156 | 157 | function label() { 158 | return $this->label; 159 | } 160 | 161 | function withMeta(Map $meta) : Result { 162 | $this->meta = $meta; 163 | 164 | return $this; 165 | } 166 | 167 | function meta() : Map { 168 | return $this->meta ?: $this->meta = Map::fromEmpty(); 169 | } 170 | 171 | 172 | function symbols() : array { 173 | return \is_array($this->ast) ? \array_keys($this->ast) : []; 174 | } 175 | 176 | function implode(string $glue = '') : string { 177 | return implode($glue, $this->tokens()); 178 | } 179 | 180 | /** 181 | * Stolen from igorw/get-in because YAY can't have a lot of dependencies 182 | */ 183 | private function getIn(array $array, array $keys, $default = null) 184 | { 185 | if (!$keys) { 186 | return $array; 187 | } 188 | 189 | // This is a micro-optimization, it is fast for non-nested keys, but fails for null values 190 | if (\count($keys) === 1 && isset($array[$keys[0]])) { 191 | return $array[$keys[0]]; 192 | } 193 | 194 | $current = $array; 195 | foreach ($keys as $key) { 196 | if (!\is_array($current) || !\array_key_exists($key, $current)) { 197 | return $default; 198 | } 199 | 200 | $current = $current[$key]; 201 | } 202 | 203 | return $current; 204 | } 205 | 206 | private function failCasting(string $type) { 207 | throw new YayPreprocessorError(sprintf("Ast cannot be casted to '%s'", $type)); 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /src/parsers_internal.php: -------------------------------------------------------------------------------- 1 | as('sigil'), ...$parsers); 44 | } 45 | 46 | function expander_sigil_prefix() : Parser { 47 | return buffer(YAY_SIGIL_FOR_EXPANDER); 48 | } 49 | 50 | function escaped_expander_sigil_prefix() : Parser { 51 | return buffer(YAY_ESCAPED_ESIGIL_FOR_EXPANDER); 52 | } 53 | 54 | /** 55 | * Defines the preprocessor expander sigil started by `$$(` and ended by `)` 56 | */ 57 | function expander_sigil(Parser ...$parsers) : Parser { 58 | return seal(expander_sigil_prefix()->as('expander_sigil'), ...$parsers); 59 | } 60 | 61 | /** 62 | * Defines the preprocessor aliased capture syntax as in `as foo` used like `$(T_STRING as foo)` 63 | */ 64 | function alias() : Parser { 65 | return 66 | chain( 67 | token(T_AS), 68 | label()->as('name') 69 | ) 70 | ->as('alias') 71 | ; 72 | } 73 | 74 | function token_constant() : Parser { 75 | return rtoken('/^T_\w+$/')->as('token_constant'); 76 | } 77 | 78 | function array_arg(): Parser { 79 | $string = string()->as('string'); 80 | $int = token(T_LNUMBER)->as('int'); 81 | return $array = 82 | chain( 83 | token('[') 84 | , 85 | commit( 86 | optional( 87 | lst( 88 | either( 89 | chain( 90 | either($int, $string)->as('key'), 91 | token(T_DOUBLE_ARROW), 92 | either( 93 | chain($int)->as('value'), 94 | chain($string)->as('value'), 95 | chain(pointer($array))->as('value') 96 | ) 97 | ) 98 | ->as('key_value_pair') 99 | , 100 | chain($int)->as('value'), 101 | chain($string)->as('value'), 102 | chain(pointer($array))->as('value') 103 | ), 104 | token(',') 105 | ) 106 | ->as('values') 107 | ) 108 | ) 109 | , 110 | token(']') 111 | ) 112 | ->as('array'); 113 | } 114 | 115 | function parsec() : Parser { 116 | return 117 | $parser = 118 | chain 119 | ( 120 | ns()->as('type') 121 | , 122 | token('(') 123 | , 124 | optional 125 | ( 126 | ls 127 | ( 128 | either 129 | ( 130 | pointer 131 | ( 132 | $parser // recursion !!! 133 | ) 134 | , 135 | chain 136 | ( 137 | token(T_FUNCTION) 138 | , 139 | parentheses()->as('args') 140 | , 141 | braces()->as('body') 142 | ) 143 | ->as('function') 144 | , 145 | string()->as('string') 146 | , 147 | chain(token_constant(), alias())->as('named_token_constant') 148 | , 149 | token_constant() 150 | , 151 | sigil(token(T_STRING, 'this'))->as('this') 152 | , 153 | label()->as('literal') 154 | , 155 | array_arg()->as('array') 156 | ) 157 | , 158 | token(',') 159 | ) 160 | ) 161 | ->as('args') 162 | , 163 | token(')') 164 | , 165 | optional(alias()) 166 | ) 167 | ->as('parsec') 168 | ; 169 | } 170 | 171 | function pattern_commit() : Parser { 172 | return buffer(YAY_PATTERN_COMMIT); 173 | } 174 | 175 | function label_or_array_access() : Parser { 176 | return 177 | /** 178 | * Matches `foo` or `foo[bar]` or `foo[bar][baz]` and so on... 179 | */ 180 | chain 181 | ( 182 | label() 183 | , 184 | optional(repeat(chain(token('['), label(), token(']'))))->as('complex') 185 | ) 186 | ->as('label') 187 | ->onCommit(function(Ast $ast){ 188 | // modifying the Ast so `T_STRING(foo) T_STRING(bar) T_STRING(baz)` 189 | // becomes a single Ast path string like `T_STRING(foo bar baz)` 190 | $ast->__construct( 191 | $ast->label(), 192 | $ast->unwrap() + [ 193 | '_name' => new Token(T_STRING, str_replace(['[', ']'], [' ', ''], $ast->implode()), $ast->tokens()[0]->line()), 194 | '_complex_name' => new Token(T_STRING, $ast->implode(), $ast->tokens()[0]->line()), 195 | '_complex' => (bool) $ast->complex, 196 | ] 197 | ); 198 | }) 199 | ; 200 | } 201 | -------------------------------------------------------------------------------- /tests/TokenStreamTest.php: -------------------------------------------------------------------------------- 1 | assertEquals('assertEquals('', (string) $ts); 17 | } 18 | 19 | function testIsEmpty() { 20 | $ts = TokenStream::fromSource(''); 21 | $this->assertTrue($ts->isEmpty()); 22 | 23 | $ts = TokenStream::fromSource('assertFalse($ts->isEmpty()); 25 | } 26 | 27 | function testStep() { 28 | $ts = TokenStream::fromSource("assertSame('current()); 31 | $this->assertSame('A', (string) $ts->step()); 32 | $this->assertSame(' ', (string) $ts->step()); 33 | $this->assertSame('B', (string) $ts->step()); 34 | $this->assertSame(' ', (string) $ts->step()); 35 | $this->assertSame('C', (string) $ts->step()); 36 | $this->assertSame(' ', (string) $ts->step()); 37 | $this->assertSame('D', (string) $ts->step()); 38 | $this->assertSame(" \n ", (string) $ts->step()); 39 | $this->assertSame('E', (string) $ts->step()); 40 | $this->assertSame(" \n\n ", (string) $ts->step()); 41 | $this->assertSame('F', (string) $ts->step()); 42 | $this->assertSame(null, $ts->step()); 43 | $this->assertSame(null, $ts->current()); 44 | } 45 | 46 | function testBack() { 47 | $ts = TokenStream::fromSource("assertSame('current()); 50 | $this->assertSame('A', (string) $ts->step()); 51 | $this->assertSame(' ', (string) $ts->step()); 52 | $this->assertSame('B', (string) $ts->step()); 53 | $this->assertSame(' ', (string) $ts->back()); 54 | $this->assertSame('A', (string) $ts->back()); 55 | $this->assertSame('back()); 56 | $this->assertSame(null, $ts->back()); 57 | $this->assertSame(null, $ts->current()); 58 | } 59 | 60 | 61 | function testNext() { 62 | $ts = TokenStream::fromSource("assertSame('current()); 65 | $this->assertSame('1', (string) $ts->next()); 66 | $this->assertSame('2', (string) $ts->next()); 67 | $this->assertSame('3', (string) $ts->next()); 68 | $this->assertSame('4', (string) $ts->next()); 69 | $this->assertSame('5', (string) $ts->next()); 70 | $this->assertSame('6', (string) $ts->next()); 71 | $this->assertSame(null, $ts->next()); 72 | $this->assertSame(null, $ts->current()); 73 | } 74 | 75 | function testReset() { 76 | $ts = TokenStream::fromSource("next()); 79 | 80 | $this->assertNull($ts->current()); 81 | 82 | $ts->reset(); 83 | 84 | $this->assertEquals('current()); 85 | } 86 | 87 | function providerForTestTrim() { 88 | return [ 89 | ['{A B C}', '{A B C}'], 90 | [' {A B C}', '{A B C}'], 91 | ['{A B C} ', '{A B C}'], 92 | [' {A B C} ', '{A B C}'], 93 | ["\t\n\t{A B C}\t\n\t", '{A B C}'], 94 | [' ', ''], 95 | ]; 96 | } 97 | 98 | /** 99 | * @dataProvider providerForTestTrim 100 | */ 101 | function testTrim(string $src, string $expected) { 102 | $ts = TokenStream::fromSource('shift(); 104 | $ts->trim(); 105 | $this->assertEquals($expected, (string) $ts); 106 | } 107 | 108 | function providerForTestLoop() { 109 | return [ 110 | [' HTML next()); 122 | $this->assertNull($ts->current(), 'EOF was not reach.'); 123 | $this->assertEquals($src, (string) $ts); 124 | $this->assertNull($ts->current(), 'Index was not preserved after string conversion.'); 125 | } 126 | 127 | function testClone() { 128 | $tsa = TokenStream::fromSource('assertNotSame($tsa->index(), $tsb->index()); 131 | } 132 | 133 | function testExtract() { 134 | $ts = TokenStream::fromSequence( 135 | new Token(T_STRING, 'A', 0), new Token(T_WHITESPACE, ' ', 0), new Token(T_STRING, 'B', 0)); 136 | 137 | $ts->extract($ts->index(), $ts->index()->next); 138 | 139 | $this->assertEquals(' B', (string) $ts); 140 | } 141 | 142 | function testInject() { 143 | $ts = TokenStream::fromSource('next(); 145 | $ts->step(); 146 | $ts->inject(TokenStream::fromSequence( 147 | new Token(T_STRING, 'MIDDLE_B', 0), new Token(T_WHITESPACE, ' ', 0))); 148 | $ts->inject(TokenStream::fromSequence( 149 | new Token(T_STRING, 'MIDDLE_A', 0),new Token( T_WHITESPACE, ' ', 0))); 150 | $this->assertEquals('inject(TokenStream::fromSequence( 154 | new Token(T_WHITESPACE, ' ', 0), new Token(T_STRING, 'BAR', 0), new Token(T_WHITESPACE, ' ', 0))); 155 | $this->assertEquals(' BAR ', (string) $ts); 156 | 157 | $ts->inject(TokenStream::fromSequence( 158 | new Token(T_WHITESPACE, ' ', 0), new Token(T_STRING, 'FOO', 0), new Token(T_WHITESPACE, ' ', 0))); 159 | $this->assertEquals(' FOO BAR ', (string) $ts); 160 | 161 | $ts = TokenStream::fromSource('next(); 163 | $ts->next(); 164 | $ts->next(); 165 | $node = $ts->index(); 166 | $partial = TokenStream::fromSequence( 167 | new Token(T_WHITESPACE, ' ', 0), new Token(T_STRING, 'C', 0), new Token(T_WHITESPACE, ' ', 0)); 168 | $index = $partial->index(); 169 | $ts->inject($partial); 170 | $this->assertEquals('assertSame($index, $ts->index()); 172 | } 173 | 174 | function testPush() { 175 | $ts = TokenStream::fromSource("push(new Token(T_LNUMBER, '4')); 178 | $ts->push(new Token(T_WHITESPACE, ' ')); 179 | $ts->push(new Token(T_LNUMBER, '5')); 180 | $ts->push(new Token(T_WHITESPACE, ' ')); 181 | $this->assertEquals('push(new Token(T_STRING, 'A')); 185 | $ts->push(new Token(T_WHITESPACE, ' ')); 186 | $ts->push(new Token(T_STRING, 'B')); 187 | $ts->push(new Token(T_WHITESPACE, ' ')); 188 | $this->assertEquals('cycle = new Cycle; 27 | $this->blueContext = new BlueContext; 28 | 29 | $macroParser = 30 | consume 31 | ( 32 | chain 33 | ( 34 | sigil( 35 | token(T_STRING, 'macro') 36 | , 37 | optional 38 | ( 39 | repeat 40 | ( 41 | chain(token(':'), label()->as('tag')) 42 | ) 43 | ) 44 | ->as('tags') 45 | ) 46 | ->as('declaration') 47 | , 48 | commit 49 | ( 50 | chain 51 | ( 52 | lookahead 53 | ( 54 | token('{') 55 | ) 56 | , 57 | commit 58 | ( 59 | chain 60 | ( 61 | braces()->as('pattern') 62 | , 63 | token(T_SR) 64 | , 65 | optional 66 | ( 67 | chain 68 | ( 69 | token(T_FUNCTION)->as('declaration') 70 | , 71 | parentheses()->as('args') 72 | , 73 | braces()->as('body') 74 | , 75 | token(T_SR) 76 | ) 77 | ) 78 | ->as('compiler_pass') 79 | , 80 | braces()->as('expansion') 81 | ) 82 | ) 83 | ->as('body') 84 | , 85 | optional 86 | ( 87 | token(';') 88 | ) 89 | ) 90 | ->as('macro') 91 | ) 92 | ) 93 | , 94 | CONSUME_DO_TRIM 95 | ) 96 | ->onCommit(function(Ast $macroAst) { 97 | 98 | $tags = Map::fromValues(array_map( 99 | function(Ast $node) :string { return (string) $node->{'* tag'}->token(); }, 100 | iterator_to_array($macroAst->{'* declaration tags'}->list()) 101 | )); 102 | 103 | if ($tags->contains('grammar')) 104 | $pattern = new GrammarPattern($macroAst->{'declaration'}[0]->line(), $macroAst->{'macro body pattern'}, $tags, Map::fromEmpty()); 105 | else 106 | $pattern = new Pattern($macroAst->{'declaration'}[0]->line(), $macroAst->{'macro body pattern'}, $tags, Map::fromEmpty()); 107 | 108 | $compilerPass = new CompilerPass($macroAst->{'* macro body compiler_pass'}); 109 | $expansion = new Expansion($macroAst->{'macro body expansion'}, $tags); 110 | $macro = new Macro($tags, $pattern, $compilerPass, $expansion); 111 | 112 | $this->registerDirective($macro); 113 | }) 114 | ; 115 | 116 | $this->expander = function(TokenStream $ts) use($macroParser) { 117 | $token = $ts->current(); 118 | while ($token instanceof Token) { 119 | $tstring = $token->value(); 120 | 121 | // here we attempt to parse, compile and allocate new macros 122 | if (YAY_DOLLAR === $tstring) $macroParser->parse($ts); 123 | 124 | // here attempt to match and expand userland macros 125 | // but just in case at least one macro passes the entry point heuristics 126 | if (isset($this->literalHitMap[$tstring])) { 127 | foreach ($this->literalHitMap[$tstring] as $directives) { 128 | foreach ($directives as $directive) { 129 | $directive->apply($ts, $this); 130 | } 131 | } 132 | } 133 | else if (isset($this->typeHitMap[$token->type()])) { 134 | foreach ($this->typeHitMap[$token->type()] as $directives) { 135 | foreach ($directives as $directive) { 136 | $directive->apply($ts, $this); 137 | } 138 | } 139 | } 140 | 141 | $token = $ts->next(); 142 | } 143 | }; 144 | } 145 | 146 | function registerDirective(Directive $directive) { 147 | $specificity = $directive->pattern()->specificity(); 148 | $identity = $directive->id(); 149 | $expectations = $directive->pattern()->expected()->all(); 150 | 151 | foreach ($expectations as $expected) { 152 | if ($key = (string) $expected) { 153 | $this->literalHitMap[$key][$specificity][$identity] = $directive; 154 | krsort($this->literalHitMap[$key]); 155 | } 156 | else { 157 | $this->typeHitMap[$expected->type()][$specificity][$identity] = $directive; 158 | krsort($this->typeHitMap[$expected->type()]); 159 | } 160 | } 161 | 162 | if ($directive->tags()->contains('global')) $this->globalDirectives[] = $directive; 163 | } 164 | 165 | function blueContext() : BlueContext { 166 | return $this->blueContext; 167 | } 168 | 169 | function cycle() : Cycle { 170 | return $this->cycle; 171 | } 172 | 173 | function currentFileName() : string { 174 | return $this->filename; 175 | } 176 | 177 | function expand(string $source, string $filename = '', int $gc = self::GC_ENGINE_ENABLED) : string { 178 | $this->filename = $filename; 179 | 180 | foreach ($this->globalDirectives as $d) $this->registerDirective($d); 181 | 182 | $ts = TokenStream::{$filename && self::GC_ENGINE_ENABLED === $gc ? 'fromSource' : 'FromSourceWithoutOpenTag'}($source); 183 | 184 | try { 185 | ($this->expander)($ts); 186 | 187 | return (string) $ts; 188 | } 189 | catch(YayPreprocessorError $error) { 190 | throw new class( 191 | str_replace(' on line', sprintf(', in %s on line', $this->filename), $error->getMessage()), 192 | $error->getCode(), 193 | $error, 194 | $this->filename ?: '-', 195 | 1 196 | ) extends YayPreprocessorError { 197 | function __construct($message = '', $code = 0, \Throwable $error = null, $file = '', $line = 0) { 198 | parent::__construct($message, $code, $error); 199 | $this->file = $file; 200 | $this->line = $line; 201 | } 202 | }; 203 | } 204 | finally { 205 | if (self::GC_ENGINE_ENABLED === $gc) { 206 | // almost everything is local per file so state must be destroyed after expansion 207 | // unless the flag ::GC_ENGINE_ENABLED forces a recycle during nested expansions 208 | // global directives are allocated again later to give impression of persistence 209 | // ::GC_ENGINE_DISABLED indicates the current pass is an internal Engine recursion 210 | $this->cycle = new Cycle; 211 | $this->literalHitMap= $this->typeHitMap = []; 212 | $this->blueContext = new BlueContext; 213 | } 214 | $this->filename = ''; 215 | } 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /tests/fixtures/expression/good.php: -------------------------------------------------------------------------------- 1 | '1', 5 | __LINE__ => '-1', 6 | __LINE__ => '+1', 7 | __LINE__ => '1 ** 1', 8 | __LINE__ => '1 - - 1', // not the same as 1 -- 1 9 | __LINE__ => '$var', 10 | __LINE__ => '$$varvar', 11 | __LINE__ => '$$$$varvar', 12 | __LINE__ => '${1}', 13 | __LINE__ => '${varname}', 14 | __LINE__ => '${$foo}', 15 | __LINE__ => '${$$varvar}', 16 | __LINE__ => '(1)', 17 | __LINE__ => '($var)', 18 | __LINE__ => '($$$$varvar)', 19 | __LINE__ => '(((1)))', 20 | __LINE__ => '((($var)))', 21 | __LINE__ => '((((($$$$varvar)))))', 22 | __LINE__ => '1 + 1', 23 | __LINE__ => '1 + 1 + 1', 24 | __LINE__ => '1 + +1 + -1', 25 | __LINE__ => '(1 + 1)', 26 | __LINE__ => '(1 + (1))', 27 | __LINE__ => '(array) $var', 28 | __LINE__ => '(array) (array) $var', 29 | __LINE__ => '(array) (1 + $a + $b + ($c + ($d)))', 30 | __LINE__ => '1 * 1 + $var + 1 + (1 + 1 + ((($var))))', 31 | __LINE__ => '--$var', 32 | __LINE__ => '++$var', 33 | __LINE__ => '1 + ++$var', 34 | __LINE__ => '1 - ++$var', 35 | __LINE__ => '--1 + --$var', 36 | __LINE__ => '--1 - --$var - --1 + ++1', 37 | __LINE__ => '@$e', 38 | __LINE__ => '--$e', 39 | __LINE__ => '++$e', 40 | __LINE__ => '$e++', 41 | __LINE__ => '$e--', 42 | __LINE__ => '$a-- + --$e', 43 | __LINE__ => '--$a + --$e', 44 | __LINE__ => '$a++ + $e--', 45 | __LINE__ => '- ++$var', 46 | __LINE__ => '[] + []', 47 | __LINE__ => '+ --$var', 48 | __LINE__ => '(int) $e', 49 | __LINE__ => '(int) ($e)', 50 | __LINE__ => 'true && false && !true', 51 | __LINE__ => '@$var', 52 | __LINE__ => '@$var--', 53 | __LINE__ => '@$var++', 54 | // constant 55 | __LINE__ => 'Foo', 56 | __LINE__ => '\Foo', 57 | __LINE__ => '\Foo\Bar\Baz', 58 | __LINE__ => '\Foo\Bar\Baz::test', 59 | __LINE__ => '\Foo\Bar\Baz::test', 60 | __LINE__ => 'self::test', 61 | __LINE__ => 'static::test', 62 | __LINE__ => 'namespace\Foo', 63 | __LINE__ => 'namespace\Foo::BAR["baz"]', 64 | __LINE__ => 'test::class', 65 | // static member 66 | __LINE__ => 'Foo::$var', 67 | __LINE__ => '\Foo::$bar', 68 | __LINE__ => '\Foo\Bar\Baz::$baz', 69 | __LINE__ => '\Foo\Bar\Baz::$x', 70 | __LINE__ => '\Foo\Bar\Baz::$y', 71 | __LINE__ => 'self::$test', 72 | __LINE__ => 'static::$test', 73 | __LINE__ => 'namespace\Foo::$test', 74 | __LINE__ => 'test::$class["bar"]::baz', 75 | __LINE__ => 'test::$class::baz', 76 | __LINE__ => 'test::$class()::baz', 77 | __LINE__ => 'test::$class::baz()', 78 | // __LINE__ => 'test::$class(new class{})::baz(1, 2, 3)', 79 | // yield 80 | __LINE__ => 'yield 1 + 1', 81 | __LINE__ => 'yield', 82 | __LINE__ => 'yield yield', 83 | __LINE__ => 'yield yield yield', 84 | __LINE__ => 'yield yield yield yield 1', 85 | __LINE__ => 'yield yield from yield yield', 86 | __LINE__ => 'yield yield from yield yield 1 + 1', 87 | __LINE__ => 'yield yield from yield yield 1 + 1 + 1', 88 | __LINE__ => 'yield 1', 89 | __LINE__ => 'yield 1 * 1 + $var', 90 | __LINE__ => 'yield ${$var}', 91 | __LINE__ => 'yield constant', 92 | __LINE__ => 'yield 1 => (2)', 93 | __LINE__ => 'yield 1 => 1 * 1 + $var', 94 | __LINE__ => 'yield $var', 95 | __LINE__ => 'yield from $var', 96 | __LINE__ => 'yield from (1 * 1 + $var)', 97 | __LINE__ => 'yield from 1 * 1 + $var', 98 | __LINE__ => 'yield 1 + yield 2 + 1 + 1 <=> $var', 99 | __LINE__ => 'yield 1 + yield 2 /* comment */ + 1 /* comment */ + 1 /* comment */ <=> $var', 100 | // coallesce 101 | __LINE__ => '1 ?? 2', 102 | __LINE__ => '["0", "1",][3] ?? ["0", "1",][true][false] ?? ["0", "1",][false]', 103 | // ternary 104 | __LINE__ => '$var ?: 1', 105 | __LINE__ => '$var++ ?: $var--', 106 | __LINE__ => '--$var ?: --$var', 107 | __LINE__ => '$var ?: $var ?: 1', 108 | __LINE__ => '$var ? $var + 1 : 1', 109 | __LINE__ => '$var ? $var : $var ?: 1', 110 | __LINE__ => '--$var ? $var++ + 1 : 1', 111 | __LINE__ => '$var++ ? $var++ + 1 : 1', 112 | // 113 | __LINE__ => '"foo $bar $baz"', 114 | __LINE__ => '"foo ${bar} {$baz} $boo {$foo->bar->baz} {$foo->bar->baz()} {$foo->bar()->baz()}"', 115 | __LINE__ => '"foo $foo->bar->baz->bar $foo->bar(1, 2, 3)->baz->bar()->biz $foo"', 116 | __LINE__ => "'foo' . 'bar'", 117 | __LINE__ => '"foo {$bar} " . "bar"', 118 | __LINE__ => '$foo instanceof Bar', 119 | __LINE__ => '$foo instanceof \Bar', 120 | __LINE__ => '$foo instanceof Foo\Bar\Baz', 121 | __LINE__ => '$foo instanceof \Foo\Baz\Baz', 122 | __LINE__ => '$foo instanceof $bar', 123 | __LINE__ => '$foo->bar', 124 | __LINE__ => '@$foo->bar->baz', 125 | __LINE__ => '--$foo->bar->baz++', 126 | __LINE__ => '$foo->bar->baz', 127 | __LINE__ => '$foo->bar->baz->biz->boz', 128 | __LINE__ => '$foo->bar()', 129 | __LINE__ => '$foo->$bar->baz()', 130 | __LINE__ => '$foo->bar->baz()', 131 | __LINE__ => '$foo->bar()->baz', 132 | __LINE__ => '$foo->bar(1)', 133 | __LINE__ => '$foo->bar(1)->baz()->biz(1, 2)->boz', 134 | __LINE__ => '$foo->bar(1)->baz()->biz(1, 2, ...[$foo, $bar])->boz', 135 | __LINE__ => '($foo->bar->baz)()', 136 | __LINE__ => '(((($foo->bar)->baz)()))', 137 | __LINE__ => '(($foo->bar)->baz)(1, 2, 3)', 138 | __LINE__ => '($foo->bar)->baz', 139 | __LINE__ => '(string) $foo->bar->baz(1+1)', 140 | __LINE__ => '(string) $foo->bar->baz((1+1))', 141 | __LINE__ => '(string) $foo->bar->baz(((1+1)))', 142 | __LINE__ => '(string) $foo->bar->baz($foo, 1, 1+1)', 143 | __LINE__ => '($foo->bar->baz)', 144 | __LINE__ => '($foo->bar()->baz)', 145 | __LINE__ => '$foo->bar(++$foo)->baz', 146 | __LINE__ => '$foo->bar(new Foo)->baz', 147 | __LINE__ => '($foo->bar(new class{})->baz)', 148 | __LINE__ => '($foo->bar()->baz())', 149 | __LINE__ => '$foo->bar()->baz() = 1', 150 | __LINE__ => 'foo() = 1', 151 | __LINE__ => 'bar()->baz', 152 | __LINE__ => 'bar()->baz()', 153 | __LINE__ => '($foo)()', 154 | __LINE__ => '(object) $foo()', 155 | __LINE__ => 'bar()', 156 | __LINE__ => '$this()', 157 | __LINE__ => 'function(){}', 158 | __LINE__ => '(function(){})', 159 | __LINE__ => '(function(){})()', 160 | __LINE__ => '(function(){})()()', 161 | __LINE__ => '(static function() use($x) :x {})()', 162 | __LINE__ => '1 + print 1', 163 | // array 164 | __LINE__ => '[]', 165 | __LINE__ => '(((([]))))', 166 | __LINE__ => '[1, 2, 3, 4, 5]', 167 | __LINE__ => '[[], 1, [1, []]]', 168 | __LINE__ => '[[], (((1))), [((1)), []]]', 169 | __LINE__ => '[function(){}]', 170 | __LINE__ => '[(function(){})(), foo]', 171 | __LINE__ => '[[] => []]', 172 | __LINE__ => '["foo" => 1, 2, 5 => 3, 4, (((10 * 100))) => (string) 5]', 173 | __LINE__ => '["foo"]', 174 | __LINE__ => '[1 + 2]', 175 | __LINE__ => '["foo" => 1]', 176 | __LINE__ => '["foo" => 1 + 2]', 177 | __LINE__ => '["foo" => 1, "bar" => 3]', 178 | __LINE__ => '((([(function(){})() => [], 1, [function(){}, \'closure\' => function(){ return [];}]])))', 179 | __LINE__ => 'array()', 180 | __LINE__ => '((((array([array()])))))', 181 | __LINE__ => 'array(1, 2, 3, 4, 5)', 182 | __LINE__ => 'array([], 1, [1, array()])', 183 | __LINE__ => 'array([], (((1))), array(((1)), array()))', 184 | __LINE__ => 'array(function(){})', 185 | __LINE__ => 'array((function(){})(), foo)', 186 | __LINE__ => 'array(array() => array())', 187 | __LINE__ => 'array("foo" => 1, 2, 5 => 3, 4, (((10 * 100))) => (string) 5)', 188 | __LINE__ => '(((array((function(){})() => [], 1, [function(){}, \'closure\' => function(){ return [];}]))))', 189 | __LINE__ => '$foo->bar["baz"]', 190 | __LINE__ => '$foo->bar["baz"]->bar->baz(1, 2, 3)', 191 | __LINE__ => '$foo->bar()->baz->biz()->bar()["baz"]->foo', 192 | __LINE__ => '$foo = &$foo->bar["baz"]', 193 | __LINE__ => '"foo ${($bar)[1]}" . "bar"', 194 | 195 | // new expression 196 | __LINE__ => 'new Bar()', 197 | __LINE__ => 'new \Foo()', 198 | __LINE__ => 'new Foo\Bar()', 199 | __LINE__ => 'new \Foo\Bar\Baz()', 200 | __LINE__ => 'new $var()', 201 | __LINE__ => 'new class {}', 202 | __LINE__ => 'new class extends Bar {}', 203 | __LINE__ => 'new class extends \Foo {}', 204 | __LINE__ => 'new class extends Foo\Bar {}', 205 | __LINE__ => 'new class extends \Foo\Bar\Baz {}', 206 | __LINE__ => 'new class implements Bar, \Foo, Foo\Bar, \Foo\Bar\Baz {}', 207 | 208 | // should not pass but YOLO! 209 | __LINE__ => 'new ($var)()', 210 | __LINE__ => '($foo->bar) = 1', 211 | __LINE__ => '($foo->bar) = &1', 212 | __LINE__ => '1 = &1', 213 | __LINE__ => '1 = 1', 214 | __LINE__ => '1 ** 3 > 3 > 1', 215 | ]; 216 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # YAY! 2 | 3 | [![Build Status](https://travis-ci.org/marcioAlmada/yay.svg?branch=master)](https://travis-ci.org/marcioAlmada/yay) 4 | [![Coverage Status](https://coveralls.io/repos/github/marcioAlmada/yay/badge.svg?branch=travis)](https://coveralls.io/github/marcioAlmada/yay?branch=travis) 5 | [![Latest Stable Version](https://poser.pugx.org/yay/yay/v/stable.png)](https://packagist.org/packages/yay/yay) 6 | [![Join the chat at https://gitter.im/marcioAlmada/yay](https://badges.gitter.im/marcioAlmada/yay.svg)](https://gitter.im/marcioAlmada/yay?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 7 | [![License](https://poser.pugx.org/yay/yay/license.png)](https://packagist.org/packages/yay/yay) 8 | 9 | **YAY!** is a high level parser combinator based PHP preprocessor that allows anyone to augment PHP with PHP :boom: 10 | 11 | This means that language features could be distributed as composer packages (as long as the macro based implementations 12 | can be expressed in pure PHP code, and the implementation is fast enough). 13 | 14 | [Roadmap](https://github.com/marcioAlmada/yay/issues/3). 15 | 16 | ## Set Up 17 | 18 | ```bash 19 | composer require yay/yay:dev-master 20 | ``` 21 | 22 | ## Usage 23 | 24 | ### Command Line 25 | 26 | ``` 27 | yay some/file/with/macros.php >> target/file.php 28 | ``` 29 | 30 | ### Runtime Mode 31 | 32 | The "runtime" mode is W.I.P and will use stream wrappers along with composer integration in order 33 | to preprocess every file that gets included. It may have some opcache/cache support, so files will be 34 | only preprocessed/expanded once and when needed. 35 | 36 | See feature progress at issue [#11](https://github.com/marcioAlmada/yay/issues/11). 37 | 38 | ## How it works 39 | 40 | ### Very Simple Example 41 | 42 | Every macro consist of a matcher and an expander that when executed allows you to augment PHP. 43 | Consider the simplest example possible: 44 | 45 | ```php 46 | $(macro :unsafe) { $ } >> { $this } // this shorthand 47 | ``` 48 | 49 | The macro is basically expanding a literal `$` token to `$this`. The following code would expand to: 50 | 51 | ```php 52 | // source | // expansion 53 | class Foo { | class Foo { 54 | protected $a = 1, $b = 2, $c = 3; | protected $a = 1, $b = 2, $c = 3; 55 | | 56 | function getProduct(): int { | function getProduct(): int { 57 | return $->a * $->b * $->c; | return $this->a * $this->b *$this->c; 58 | } | } 59 | } | } 60 | ``` 61 | 62 | > Notice that the `:unsafe` tag is necessary to avoid macro hygiene on `$this` expansion. 63 | 64 | This macro is actually very naive, a more producion ready version would be: 65 | 66 | 67 | ```php 68 | $(macro :unsafe){ 69 | $ // litterally matches '$' 70 | // but not followed by: 71 | $(not(token(T_VARIABLE))) // avoids var var false positives such as '$$foo' 72 | $(not(token('{'))) // avoids false positives such as '${foo}' 73 | } >> { 74 | $this 75 | } 76 | ``` 77 | 78 | ### Simple Example 79 | 80 | Apart from literal characher sequences, it's also possible to match specific token types using the token matcher in 81 | the form of `$(TOKEN_TYPE as label)`. 82 | 83 | The following macro matches token sequences like `__swap($x, $y)` or `__swap($foo, $bar)`: 84 | 85 | ```php 86 | $(macro) { 87 | __swap ( $(T_VARIABLE as A) , $(T_VARIABLE as B) ) 88 | } >> { 89 | (list($(A), $(B)) = [$(B), $(A)]) 90 | } 91 | ``` 92 | 93 | The expansion should be pretty obvious: 94 | ```php 95 | // source | // expansion 96 | __swap($foo, $bar); | (list($foo, $bar) = [$bar, $foo]); 97 | ``` 98 | 99 | ### Another Simple Example 100 | 101 | To implement `unless` we need to match the literal `unless` keyword followed by a layer of tokens between parentheses 102 | `(...)` and a block of code `{...}`. Fortunately, the macro DSL has a very straightforward layer matching construct: 103 | 104 | ```php 105 | $(macro) { 106 | unless ($(layer() as expression)) { $(layer() as body) } 107 | } >> { 108 | if (! ($(expression))) { 109 | $(body) 110 | } 111 | } 112 | ``` 113 | 114 | The macro in action: 115 | 116 | ```php 117 | // source | // expansion 118 | unless ($x === 1) { | if (! ($x === 1)) { 119 | echo "\$x is not 1"; | echo "\$x is not 1"; 120 | } | } 121 | ``` 122 | 123 | > PS: Please don't implement "unless". This is here just for didactic reasons. 124 | 125 | ### Advanced Example 126 | 127 | A more complex example could be porting enums from the future to PHP with a syntax like: 128 | 129 | ```php 130 | enum Fruits { 131 | Apple, 132 | Orange 133 | } 134 | 135 | var_dump(\Fruits::Orange <=> \Fruits::Apple); 136 | ``` 137 | So, syntactically, enums are declared with the literal `enum` word followed by a `T_STRING` and a comma 138 | separated list of identifiers withing braces such as `{A, B, C}`. 139 | 140 | YAY uses parser combinators internally for everything and these more high level parsers are fully 141 | exposed on macro declarations. Our enum macro will need high level matchers like `ls()` and `label()` 142 | combined to match the desired syntax, like so: 143 | 144 | ```php 145 | $(macro) { 146 | enum $(T_STRING as name) { 147 | $( 148 | // ls() matches a delimited list 149 | // in this case a list of label() delimited by ',' such as `foo, bar, baz` 150 | ls 151 | ( 152 | label() as field 153 | , 154 | token(',') 155 | ) 156 | as fields 157 | ) 158 | } 159 | } >> { 160 | "it works"; 161 | } 162 | ``` 163 | 164 | The macro is already capable to match the enum syntax: 165 | 166 | ```php 167 | // source // expansion 168 | enum Order {ASC, DESC}; | "it works"; 169 | ``` 170 | 171 | I won't explain how enums are implemented, you can read the [RFC](https://wiki.php.net/rfc/enum) if you wish 172 | and then see how the expansion below works: 173 | 174 | ```php 175 | // things here would normally be under a namespace, but since we want a concise example... 176 | 177 | interface Enum 178 | { 179 | } 180 | 181 | function enum_field_or_class_constant(string $class, string $field) 182 | { 183 | return (\in_array(\Enum::class, \class_implements($class)) ? $class::$field() : \constant("{$class}::{$field}")); 184 | } 185 | 186 | $(macro :unsafe) { 187 | // the enum declaration 188 | enum $(T_STRING as name) { 189 | $( 190 | ls 191 | ( 192 | label() as field 193 | , 194 | token(',') 195 | ) 196 | as fields 197 | ) 198 | } 199 | } >> { 200 | class $(name) implements Enum { 201 | private static $registry; 202 | 203 | private function __construct() {} 204 | 205 | static function __callStatic(string $type, array $args) : self { 206 | if(! self::$registry) { 207 | self::$registry = new \stdclass; 208 | $(fields ... { 209 | self::$registry->$(field) = new class extends $(name) {}; 210 | }) 211 | } 212 | 213 | if (isset(self::$registry->$type)) return self::$registry->$type; 214 | 215 | throw new \Exception(sprintf('Undefined enum type %s->%s', __CLASS__, $type)); 216 | } 217 | } 218 | } 219 | 220 | $(macro) { 221 | $( 222 | // sequence that matches the enum field access syntax: 223 | chain( 224 | ns() as class, // matches a namespace 225 | token(T_DOUBLE_COLON), // matches T_DOUBLE_COLON used for static access 226 | not(class), // avoids matching `Foo::class`, class resolution syntax 227 | label() as field, // matches the enum field name 228 | not(token('(')) // avoids matching static method calls such as `Foo::bar()` 229 | ) 230 | ) 231 | } >> { 232 | \enum_field_or_class_constant($(class)::class, $$(stringify($(field)))) 233 | } 234 | ``` 235 | 236 | > More examples within the phpt tests folder https://github.com/marcioAlmada/yay/tree/master/tests/phpt 237 | 238 | # FAQ 239 | 240 | > Why "YAY!"? 241 | 242 | \- PHP with feature "x": yay or nay? :wink: 243 | 244 | > Where is the documentation? 245 | 246 | A cookbook is on the making 247 | 248 | > Why are you working on this? 249 | 250 | Because it's being fun. It may become useful. [Because we can™](https://github.com/haskellcamargo/because-we-can). 251 | 252 | # Conclusion 253 | 254 | For now this is an experiment about how to build a high level preprocessor DSL using parser combinators 255 | on a languages like PHP. Why? 256 | 257 | PHP is very far from being [homoiconic](https://en.wikipedia.org/wiki/Homoiconicity) and therefore requires 258 | complex deterministic parsing and a big AST implementation with a node visitor API to modify source code - and 259 | in the end, you're not even able to easily process unknown syntax `¯\_(⊙_ʖ⊙)_/¯`. 260 | 261 | That's why this project was born. It was also part of the challenge: 262 | 263 | 0. Create a minimalistic architecture that exposes a subset of the internal components, that power the preprocessor itself, to the user DSL. 264 | 0. Create parser combinators with decent error reporting and grammar invalidation, because of 1 265 | 266 | ## Copyright 267 | 268 | Copyright (c) 2015-* Márcio Almada. Distributed under the terms of an MIT-style license. 269 | See LICENSE for details. 270 | -------------------------------------------------------------------------------- /src/TokenStream.php: -------------------------------------------------------------------------------- 1 | next; 22 | 23 | while ($node instanceof Node) { 24 | $str .= $node->token->value; 25 | $node = $node->next; 26 | } 27 | 28 | return $str; 29 | } 30 | })->toSource($this->first); 31 | } 32 | 33 | function debug() : string { 34 | return (new class extends Token { 35 | function __construct() {} 36 | function toSource(NodeStart $node, Index $current) : string { 37 | $str = ''; 38 | $node = $node->next; 39 | 40 | while ($node instanceof Node) { 41 | if ($node === $current) $str .= "\033[1;37;40m"; 42 | $str .= $node->token->value; 43 | if ($node === $current) $str .= "\033[0m"; 44 | $node = $node->next; 45 | } 46 | 47 | return $str; 48 | } 49 | })->toSource($this->first, $this->current); 50 | } 51 | 52 | function __clone() { 53 | $first = new NodeStart; 54 | $last = new NodeEnd; 55 | 56 | $first->next = $last; 57 | $last->previous = $first; 58 | 59 | $current = $first; 60 | $old = $this->first->next; 61 | while ($old instanceof Node) { 62 | $node = new Node($old->token); 63 | $current->next = $node; 64 | $node->previous = $current; 65 | $current = $node; 66 | $old = $old->next; 67 | } 68 | 69 | $current->next = $last; 70 | $last->previous = $current; 71 | 72 | $this->first = $first; 73 | $this->last = $last; 74 | $this->current = $this->first->next; 75 | } 76 | 77 | function index() /* : Node|null */ { return $this->current; } 78 | 79 | function jump($index) /* : void */ { 80 | assert($index instanceof Index); 81 | 82 | if ($index instanceof NodeStart) 83 | $this->current = $this->first->next; 84 | else 85 | $this->current = $index; 86 | } 87 | 88 | function reset() /* : void */ { 89 | $this->current = $this->first->next; 90 | } 91 | 92 | function current() /* : Token|null */ { 93 | return $this->current->token; 94 | } 95 | 96 | function step() /* : Token|null */ { 97 | $this->current = $this->current->next; 98 | 99 | return $this->current->token; 100 | } 101 | 102 | function back() /* : Token|null */ { 103 | $this->current = $this->current->previous; 104 | 105 | return $this->current->token; 106 | } 107 | 108 | function skip() /* : Token|null */ { 109 | while ($this->current->skippable) { 110 | $this->current = $this->current->next; 111 | } 112 | 113 | return $this->current->token; 114 | } 115 | 116 | function unskip() /* : Token|null */ { 117 | $this->current = $this->current->previous; 118 | 119 | while ($this->current->skippable) { 120 | $this->current = $this->current->previous; 121 | } 122 | 123 | $this->current = $this->current->next; 124 | 125 | return $this->current->token; 126 | } 127 | 128 | function next() /* : Token|null */ { 129 | $this->current = $this->current->next; 130 | 131 | while ($this->current->skippable) { 132 | $this->current = $this->current->next; 133 | } 134 | 135 | return $this->current->token; 136 | } 137 | 138 | function previous() /* : Token|null */ { 139 | $this->current = $this->current->previous; 140 | 141 | while ($this->current->skippable) { 142 | $this->current = $this->current->previous; 143 | } 144 | 145 | return $this->current->token; 146 | } 147 | 148 | function last() : Token { 149 | return $this->last->previous->token; 150 | } 151 | 152 | function first() : Token { 153 | return $this->first->next->token; 154 | } 155 | 156 | function trim() { 157 | while (null !== ($t = $this->first->next->token) && $t->is(T_WHITESPACE)) { 158 | $this->first->next = $this->first->next->next; 159 | $this->first->next->previous = $this->first; 160 | } 161 | while (null !== ($t = $this->last->previous->token) && $t->is(T_WHITESPACE)) { 162 | $this->last->previous = $this->last->previous->previous; 163 | $this->last->previous->next = $this->last; 164 | } 165 | $this->current = $this->first->next; 166 | } 167 | 168 | function shift() { 169 | $this->first->next = $this->first->next->next; 170 | $this->first->next->previous = $this->first; 171 | } 172 | 173 | function pop() { 174 | $this->last->previous = $this->last->previous->previous; 175 | $this->last->previous->next = $this->last; 176 | } 177 | 178 | function extract($from, $to) { 179 | assert($from instanceof Index); 180 | assert($to instanceof Index); 181 | 182 | assert($from !== $to); 183 | assert(! $this->isEmpty()); 184 | 185 | $from = $from->previous; 186 | $from->next = $to; 187 | $to->previous = $from; 188 | 189 | $this->current = $from; 190 | } 191 | 192 | function inject($ts) { 193 | assert($ts instanceof self); 194 | 195 | if (($ts->first->next instanceof NodeEnd) 196 | && ($ts->last->previous instanceof NodeStart)) return; 197 | 198 | if ($this->current instanceof NodeEnd) $this->current = $this->last->previous; 199 | 200 | $a = $this->current; 201 | $b = $ts->first->next; 202 | $e = $ts->last->previous; 203 | $f = $this->current->next; 204 | 205 | $a->next = $b; 206 | $b->previous = $a; 207 | 208 | $e->next = $f; 209 | $f->previous = $e; 210 | } 211 | 212 | function push($token) { 213 | assert($token instanceof Token); 214 | 215 | $a = $this->last->previous; 216 | $b = new Node($token); 217 | $c = $this->last; 218 | 219 | $a->next = $b; 220 | $b->previous = $a; 221 | 222 | $b->next = $c; 223 | $c->previous = $b; 224 | } 225 | 226 | function isEmpty() : bool { 227 | return 228 | ($this->first->next instanceof NodeEnd) 229 | && ($this->last->previous instanceof NodeStart) 230 | ; 231 | } 232 | 233 | static function fromSourceWithoutOpenTag(string $source) : self { 234 | $ts = self::fromSource('first->next = $ts->first->next->next; 236 | $ts->first->next->previous = $ts->first; 237 | $ts->reset(); 238 | 239 | return $ts; 240 | } 241 | 242 | static function fromSource(string $source) : self { 243 | $tokens = \token_get_all($source); 244 | 245 | $ts = new self; 246 | $first = new NodeStart; 247 | $last = new NodeEnd; 248 | 249 | $first->next = $last; 250 | $last->previous = $first; 251 | 252 | $line = 0; 253 | $current = $first; 254 | $realign = []; // tokenizer omits line number sometimes so we borrow from next non whitespace 255 | foreach ($tokens as $t){ 256 | if (\is_array($t)) { 257 | $line = $t[2]; 258 | $token = new Token(...$t); 259 | 260 | if (T_WHITESPACE !== $token->type()) { 261 | foreach ($realign as $node) 262 | $node->token = new Token($node->token->type(), $node->token->value(), $line); 263 | 264 | $realign = []; 265 | } 266 | } 267 | else { 268 | $token = new Token($t, $t, $line); 269 | } 270 | 271 | $node = new Node($token); 272 | $current->next = $node; 273 | $node->previous = $current; 274 | 275 | $current = $node; 276 | 277 | if (is_string($current->token->type())) $realign[] = $current; 278 | } 279 | 280 | $current->next = $last; 281 | $last->previous = $current; 282 | 283 | $ts->first = $first; 284 | $ts->last = $last; 285 | 286 | $ts->current = $ts->first->next; 287 | 288 | return $ts; 289 | } 290 | 291 | static function fromSlice(array $tokens) : self { 292 | $ts = new self; 293 | $first = new NodeStart; 294 | $last = new NodeEnd; 295 | 296 | $first->next = $last; 297 | $last->previous = $first; 298 | 299 | $current = $first; 300 | foreach ($tokens as $token){ 301 | $node = new Node($token); 302 | $current->next = $node; 303 | $node->previous = $current; 304 | 305 | $current = $node; 306 | } 307 | 308 | $current->next = $last; 309 | $last->previous = $current; 310 | 311 | $ts->first = $first; 312 | $ts->last = $last; 313 | 314 | $ts->current = $ts->first->next; 315 | 316 | return $ts; 317 | } 318 | 319 | static function fromSequence(Token ...$tokens) : self { 320 | return self::fromSlice($tokens); 321 | } 322 | } 323 | -------------------------------------------------------------------------------- /src/GrammarPattern.php: -------------------------------------------------------------------------------- 1 | fail(self::E_EMPTY_PATTERN, $line); 28 | 29 | $this->scope = $scope; 30 | $this->unreached = new Map; 31 | $this->staged = new Map; 32 | $this->collected = new Map; 33 | $this->pattern = $this->compile($line, $pattern); 34 | } 35 | 36 | private function compile(int $line, array $tokens) { 37 | 38 | $label = label()->as('label'); 39 | 40 | $doubleQuotes = token(T_CONSTANT_ENCAPSED_STRING, "''"); 41 | 42 | $commit = chain(buffer('$!'))->as('commit'); 43 | 44 | $literal = between($doubleQuotes, any(), $doubleQuotes)->as('literal'); 45 | 46 | $constant = rtoken('/^T_\w+$/')->as('constant'); 47 | 48 | $optionalModifier = optional(token('?'), false)->as('optional?'); 49 | 50 | $productionModifier = optional(token(T_SL), false)->as('production?'); 51 | 52 | $parsec = sigil(parsec())->as('parser'); 53 | 54 | $labelReference = 55 | sigil( 56 | $label 57 | , 58 | optional 59 | ( 60 | alias() 61 | ) 62 | ) 63 | ->as('reference') 64 | ; 65 | 66 | $list = 67 | chain 68 | ( 69 | $optionalModifier 70 | , 71 | token(T_LIST) 72 | , 73 | token('(') 74 | , 75 | pointer($sequence)->as('member') 76 | , 77 | token(',') 78 | , 79 | (clone $literal)->as('delimiter') 80 | , 81 | token(')') 82 | ) 83 | ->as('list') 84 | ; 85 | 86 | $sequence = 87 | repeat 88 | ( 89 | either 90 | ( 91 | $list 92 | , 93 | $parsec 94 | , 95 | $labelReference 96 | , 97 | $constant 98 | , 99 | $literal 100 | , 101 | $commit 102 | ) 103 | ) 104 | ->as('sequence') 105 | ; 106 | 107 | $disjunction = ls($sequence, token('|'))->as('disjunction'); 108 | 109 | $rule = 110 | commit 111 | ( 112 | chain 113 | ( 114 | $productionModifier 115 | , 116 | sigil($label)->as('rule_name') 117 | , 118 | $optionalModifier 119 | , 120 | either 121 | ( 122 | between 123 | ( 124 | token('{') 125 | , 126 | $sequence 127 | , 128 | token('}') 129 | ) 130 | , 131 | between 132 | ( 133 | token('{') 134 | , 135 | $disjunction 136 | , 137 | token('}') 138 | ) 139 | ) 140 | ) 141 | ) 142 | ->as('rule') 143 | ; 144 | 145 | $grammar = 146 | commit 147 | ( 148 | chain 149 | ( 150 | optional 151 | ( 152 | repeat($rule) 153 | ) 154 | ->as('rules') 155 | ) 156 | ) 157 | ; 158 | 159 | $grammarAst = $grammar->parse(TokenStream::fromSlice($tokens)); 160 | 161 | $productions = new Map; 162 | 163 | foreach ($grammarAst->{'* rules'}->list() as $ast) { 164 | 165 | $ruleAst = $ast->{'* rule'}; 166 | $labelAst = $ast->{'* rule rule_name label'}; 167 | 168 | $label = (string) $labelAst->token(); 169 | 170 | if ($ruleAst->{'production?'}) { 171 | 172 | $productions->add($label, $ruleAst); 173 | 174 | if ($productions->count() > 1) { 175 | $this->fail( 176 | self::E_GRAMMAR_MULTIPLE_PRODUCTIONS, 177 | $line, 178 | json_encode($productions->symbols(), self::PRETTY_PRINT) 179 | ); 180 | } 181 | 182 | continue; 183 | } 184 | 185 | if ($this->unreached->contains($label)) 186 | $this->fail(self::E_GRAMMAR_DUPLICATED_RULE_LABEL, $label, $labelAst->token()->line()); 187 | 188 | $this->unreached->add($label, $ruleAst); 189 | } 190 | 191 | if ($productions->count() === 0) 192 | $this->fail(self::E_GRAMMAR_MISSING_PRODUCTION, $line); 193 | 194 | $productionLabel = $productions->symbols()[0]; 195 | 196 | $this->scope->add($productionLabel); 197 | 198 | $pattern = $this->compilePattern($productions->get($productionLabel)); 199 | 200 | if ($this->unreached->count() > 0) { 201 | $this->fail( 202 | self::E_GRAMMAR_UNREACHABLE_NONTERMINAL, 203 | $productionLabel, 204 | $line, 205 | json_encode($this->unreached->symbols(), self::PRETTY_PRINT) 206 | ); 207 | } 208 | 209 | if ($this->staged->count() > 0) { 210 | $this->fail( 211 | self::E_GRAMMAR_UNREACHABLE_NONTERMINAL, 212 | $productionLabel, 213 | $line, 214 | json_encode($this->staged->symbols(), self::PRETTY_PRINT) 215 | ); 216 | } 217 | 218 | $this->specificity = $this->collected->count(); 219 | 220 | if (! $pattern->isFallible()) 221 | $this->fail(self::E_GRAMMAR_NON_FALLIBLE, $productionLabel, $line); 222 | 223 | return $pattern; 224 | } 225 | 226 | private function compilePattern(Ast $rule) : Parser { 227 | 228 | $label = (string) $rule->{'* rule_name label'}->token(); 229 | 230 | if (! ($sequence = $rule->{'* sequence'})->isEmpty()) 231 | $pattern = $this->compileSequence($sequence, $label); 232 | else if(! ($disjunction = $rule->{'* disjunction'})->isEmpty()) 233 | $pattern = $this->compileDisjunction($disjunction, $label); 234 | else 235 | assert(false, 'Unknown pattern definition.'); 236 | 237 | if ($rule->{'optional?'}) $pattern = optional($pattern); 238 | 239 | $pattern->as($label); 240 | 241 | return $pattern; 242 | } 243 | 244 | private function compileSequence(Ast $sequence, string $label) : Parser { 245 | $commit = false; 246 | $this->staged->add($label); 247 | $chain = []; 248 | foreach ($sequence->list() as $ast) { 249 | $type = key($ast->array()); 250 | $ast = $ast->{"* {$type}"}; 251 | switch ($type) { 252 | case 'literal': // matches double quoted like: '','' or ''use'' 253 | $chain[] = token($ast->token()); 254 | break; 255 | case 'constant': // T_* 256 | $chain[] = token(parent::compileTokenConstant($ast)); 257 | break; 258 | case 'parser': 259 | $chain[] = parent::compileParser($ast->{'* parsec'}); 260 | break; 261 | case 'reference': 262 | $refLabel = (string) $ast->{'label'}; 263 | $link = $this->collected->get($refLabel); 264 | if ($link === null) { 265 | if ($this->staged->contains($refLabel)) { 266 | $link = pointer($this->references[$refLabel]); 267 | } 268 | else { 269 | $link = $this->compilePattern($this->unreached->get($refLabel)); 270 | $this->references[$refLabel] = $link; 271 | $this->collected->add($refLabel, $link); 272 | $this->unreached->remove($refLabel); 273 | } 274 | } 275 | 276 | $link = (clone $link)->as((string) $ast->{'alias name'} ?: ''); 277 | 278 | $chain[] = $link; 279 | break; 280 | case 'list': 281 | $link = $this->compileSequence($ast->{'* member'}, $label); 282 | $chain[] = optional(ls($link, token($ast->{'* delimiter'}->token()))); 283 | break; 284 | case 'commit': 285 | $commit = true; 286 | break; 287 | default: 288 | assert(false, "Unknown sequence step {$type}."); 289 | break; 290 | } 291 | 292 | if ($commit && ($length = count($chain)) > 0) $chain[$length-1] = commit(end($chain)); 293 | } 294 | 295 | if (count($chain) > 1) { 296 | $pattern = chain(...$chain); 297 | $pattern->as($label); 298 | } 299 | else { 300 | $pattern = $chain[0]; 301 | } 302 | 303 | $this->staged->remove($label); 304 | 305 | 306 | return $pattern; 307 | } 308 | 309 | private function compileDisjunction(Ast $disjunctions, string $label) : Parser { 310 | 311 | $this->staged->add($label); 312 | 313 | $chain = []; 314 | foreach ($disjunctions->list() as $disjunction) { 315 | foreach ($disjunction->list() as $sequence) { 316 | $link = $this->compileSequence($sequence, $label); 317 | $this->collected->add($label, $link); 318 | $this->unreached->remove($label); 319 | $chain[] = $link; 320 | } 321 | } 322 | 323 | $pattern = either(...$chain)->as($label); 324 | 325 | $this->staged->remove($label); 326 | 327 | return $pattern; 328 | } 329 | } 330 | -------------------------------------------------------------------------------- /src/Pattern.php: -------------------------------------------------------------------------------- 1 | fail(self::E_EMPTY_PATTERN, $line); 30 | 31 | $this->scope = $scope; 32 | $this->pattern = $this->compile($pattern); 33 | 34 | if ($tags->contains('optimize')) $this->pattern = $this->pattern->optimize(); 35 | } 36 | 37 | function match(TokenStream $ts) { 38 | return $this->pattern->parse($ts); 39 | } 40 | 41 | function specificity() : int { 42 | return $this->specificity; 43 | } 44 | 45 | function expected() : Expected { 46 | return $this->pattern->expected(); 47 | } 48 | 49 | private function compile(array $tokens) { 50 | 51 | // cg is the compiler globals 52 | $cg = (object)[ 53 | 'ts' => TokenStream::fromSlice($tokens), 54 | 'parsers' => [], 55 | ]; 56 | 57 | /* 58 | Here we traverse the macro declaration token stream and look for 59 | declared ast node matchers under the preprocessor sigil `$(...)` 60 | */ 61 | traverse 62 | ( 63 | /* 64 | Matches: 65 | $(T_STRING) 66 | Compiles to: 67 | token(T_STRING) 68 | 69 | Matches: 70 | $(T_STRING as foo) 71 | Compiles to: 72 | token(T_STRING)->as('foo') 73 | */ 74 | sigil(token_constant(), optional(alias())) 75 | ->onCommit(function(Ast $result) use($cg) { 76 | $token = $this->compileTokenConstant($result->{'* token_constant'}); 77 | $alias = $this->compileAlias($result->{'* alias'}); 78 | $cg->parsers[] = token($token)->as($alias); 79 | }) 80 | , 81 | // Matches complex parser combinator declarations 82 | sigil(parsec()) 83 | ->onCommit(function(Ast $result) use($cg) { 84 | $cg->parsers[] = $this->compileParser($result->{'* parsec'}); 85 | }) 86 | , 87 | /* 88 | Matches: 89 | $({...}) 90 | Compiles to: 91 | braces() 92 | 93 | Matches: 94 | $({...} as foo) 95 | Compiles to: 96 | braces()->as('foo') 97 | */ 98 | sigil($this->layer('{', '}', braces(), $cg)) 99 | , 100 | /* 101 | Matches: 102 | $([...]) 103 | Compiles to: 104 | brackets() 105 | 106 | Matches: 107 | $([...] as foo) 108 | Compiles to: 109 | brackets()->as('foo') 110 | */ 111 | sigil($this->layer('[', ']', brackets(), $cg)) 112 | , 113 | /* 114 | Matches: 115 | $((...)) 116 | Compiles to: 117 | parentheses() 118 | 119 | Matches: 120 | $((...) as foo) 121 | Compiles to: 122 | parentheses()->as('foo') 123 | */ 124 | sigil($this->layer('(', ')', parentheses(), $cg)) 125 | , 126 | /* 127 | Matches: 128 | $(...) 129 | Compiles to: 130 | layer() 131 | 132 | Matches: 133 | $(... as foo) 134 | Compiles to: 135 | layer()->as('foo') 136 | */ 137 | sigil(token(T_ELLIPSIS), optional(alias())) 138 | ->onCommit(function(Ast $result) use($cg) { 139 | $alias = $this->compileAlias($result->{'* alias'}); 140 | $cg->parsers[] = layer()->as($alias); 141 | }) 142 | , 143 | /* 144 | Matches: 145 | $! 146 | Compiles to: 147 | commit() 148 | 149 | > Causes the pattern after $ to throw a preprocessor error in case the pattern is 150 | not fully matched. The normal behavior is to silent failure and backtrack. This is 151 | useful to introduce first class language features with elegant syntax errors within 152 | DSLs 153 | */ 154 | pattern_commit() 155 | ->onCommit(function(Ast $result) use ($cg) { 156 | $offset = \count($cg->parsers); 157 | if (0 !== $this->dominance || 0 === $offset) { 158 | $this->fail(self::E_BAD_DOMINANCE, $offset, $result->tokens()[0]->line()); 159 | } 160 | $this->dominance = $offset; 161 | }) 162 | , 163 | /* 164 | Matches: 165 | Possible orphaned $() 166 | 167 | > Causes a preprocessor error pointing a macro syntax error 168 | */ 169 | sigil(layer()) 170 | ->onCommit(function(Ast $result) use ($cg) { 171 | $this->fail(self::E_BAD_CAPTURE, $result->implode(), $result->{'sigil'}[0]->line()); 172 | }) 173 | , 174 | /* 175 | Matches: 176 | Anything the preprocessor is not aware of 177 | Compiles to: 178 | A literal pattern of whatever was matched 179 | */ 180 | any() 181 | ->onCommit(function(Ast $result) use($cg) { 182 | $cg->parsers[] = token($result->token()); 183 | }) 184 | ) 185 | ->parse($cg->ts); 186 | 187 | $this->specificity = \count($cg->parsers); 188 | 189 | // check if macro dominance '$' is last token 190 | if ($this->dominance === $this->specificity) 191 | $this->fail(self::E_BAD_DOMINANCE, $this->dominance, $cg->ts->last()->line()); 192 | 193 | if ($this->specificity > 1) { 194 | if (0 === $this->dominance) { 195 | $pattern = chain(...$cg->parsers); 196 | } 197 | else { 198 | /* 199 | dominat macros are partially wrapped in commit()s and dominance 200 | is the offset used as the 'event horizon' point... once the entry 201 | point is matched, there is no way back and a parser error arises 202 | */ 203 | $prefix = array_slice($cg->parsers, 0, $this->dominance); 204 | $suffix = array_slice($cg->parsers, $this->dominance); 205 | $pattern = chain(...array_merge($prefix, array_map(commit::class, $suffix))); 206 | } 207 | } 208 | else { 209 | /* 210 | micro optimization to save one function call for every token on the subject 211 | token stream whenever the macro pattern consists of a single parser 212 | */ 213 | $pattern = $cg->parsers[0]; 214 | } 215 | 216 | return $pattern; 217 | } 218 | 219 | private function layer(string $start, string $end, Parser $parser, $cg) : Parser { 220 | return 221 | chain 222 | ( 223 | token($start) 224 | , 225 | token(T_ELLIPSIS) 226 | , 227 | commit(token($end)) 228 | , 229 | alias() 230 | ) 231 | ->onCommit(function(Ast $result) use($parser, $cg) { 232 | $identifier = $this->compileAlias($result->{'* alias'}); 233 | $cg->parsers[] = (clone $parser)->as($identifier); 234 | }); 235 | } 236 | 237 | protected function compileTokenConstant(Ast $constant) : int { 238 | $type = (string) $constant->token(); 239 | if (! defined($type)) 240 | $this->fail(self::E_BAD_TOKEN_TYPE, $type, $constant->token()->line()); 241 | 242 | return constant($type); 243 | } 244 | 245 | private function compileAlias(Ast $alias) : string { 246 | $identifier = $alias->{'name'} ? (string) $alias->{'* name'}->token() : '_'; 247 | 248 | if ($identifier === self::NULL_LABEL) return ''; 249 | 250 | if ($this->scope->contains($identifier)) 251 | $this->fail(self::E_IDENTIFIER_REDEFINITION, $identifier, $alias->{'* name'}->token()->line()); 252 | 253 | $this->scope->add($identifier); 254 | 255 | return $identifier; 256 | } 257 | 258 | protected function compileArray(Ast $array) : array { 259 | $compiled = []; 260 | foreach ($array->{'* values'}->list() as $valueNode) { 261 | $key = count($compiled); 262 | $value = $valueNode->{'* value'}; 263 | 264 | if ($valueNode->{'key_value_pair'}) { 265 | $value = $valueNode->{'* key_value_pair value'}; 266 | $type = key($valueNode->{'* key_value_pair key'}->array()); 267 | $key = $valueNode->{"* key_value_pair key {$type}"}; 268 | switch ($type) { 269 | case 'string': 270 | $key = trim((string) $key->token(), '"\''); 271 | break; 272 | case 'int': 273 | $key = (int) $key->token()->value(); 274 | } 275 | } 276 | 277 | $type = key($value->array()); 278 | $value = $value->{"* {$type}"}; 279 | switch ($type) { 280 | case 'string': 281 | $value = trim((string) $value->token(), '"\''); 282 | break; 283 | case 'int': 284 | $value = (int) $value->token()->value(); 285 | break; 286 | case 'array': 287 | $value = $this->compileArray($value); 288 | } 289 | 290 | $compiled[$key] = $value; 291 | } 292 | 293 | return $compiled; 294 | } 295 | 296 | protected function compileParser(Ast $ast) : Parser { 297 | $parser = $this->compileCallable('\Yay\\', $ast->{'* type'}, self::E_BAD_PARSER_NAME); 298 | $args = $ast->{'* args'}->isEmpty() ? [] : $this->compileParserArgs($ast->{'* args'}); 299 | $parser = $parser(...$args); 300 | $alias = $this->compileAlias($ast->{'* alias'}); 301 | $parser->as((string) $alias); 302 | 303 | return $parser; 304 | } 305 | 306 | protected function compileParserArgs(Ast $args) : array { 307 | $compiled = []; 308 | foreach ($args->list() as $ast) { 309 | $type = key($ast->array()); 310 | $arg = $ast->{"* {$type}"}; 311 | switch ($type) { 312 | case 'this': 313 | $compiled[] = pointer($this->pattern); 314 | break; 315 | case 'named_token_constant': 316 | $token = $this->compileTokenConstant($arg->{'* token_constant'}); 317 | $alias = $this->compileAlias($arg->{'* alias'}); 318 | $compiled[] = token($token)->as($alias); 319 | break; 320 | case 'token_constant': 321 | $compiled[] = $this->compileTokenConstant($arg); 322 | break; 323 | case 'literal': 324 | $compiled[] = token($arg->token()); 325 | break; 326 | case 'parsec': 327 | $compiled[] = $this->compileParser($arg); 328 | break; 329 | case 'string': 330 | $compiled[] = trim((string) $arg->token(), '"\''); 331 | break; 332 | case 'array': 333 | $compiled[] = $this->compileArray($arg); 334 | break; 335 | case 'function': // function(...){...} 336 | $compiled[] = new AnonymousFunction($arg); 337 | break; 338 | default: 339 | assert(false, "Unknown parser argument type '{$type}'"); 340 | } 341 | } 342 | 343 | return $compiled; 344 | } 345 | } 346 | --------------------------------------------------------------------------------