├── .gitignore ├── LICENSE ├── README.md ├── examples └── calc │ └── calc.go ├── impl.go ├── parser.go └── private.go /.gitignore: -------------------------------------------------------------------------------- 1 | # 2 | # https://github.com/github/gitignore (November 09, 2010) 3 | # 4 | 5 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 6 | *.o 7 | *.a 8 | *.so 9 | 10 | # Folders 11 | _obj 12 | _test 13 | 14 | # Architecture specific extensions/prefixes 15 | *.[568vq] 16 | [568vq].out 17 | 18 | *.cgo1.go 19 | *.cgo2.c 20 | _cgo_defun.c 21 | _cgo_gotypes.go 22 | _cgo_export.* 23 | 24 | _testmain.go 25 | 26 | *.exe 27 | 28 | # 29 | # Project-Specific (not part of gitignore project) 30 | # 31 | 32 | .DS_Store 33 | build.out 34 | examples/calc/calc 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 David Farrell. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of iNamik LLC. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | go_parser 2 | ========= 3 | 4 | **Parser API in Go** 5 | 6 | 7 | ABOUT 8 | ----- 9 | 10 | The 'go_parser' package is an API to help you create hand-written lexers and parsers. 11 | 12 | The package was inspired by Rob Pikes' video [Lexical Scanning In Go](http://youtu.be/HxaD_trXwRE) and golang's 'template' package. 13 | 14 | PARSER INTERFACE 15 | --------------- 16 | 17 | Below is the interface for the main Parser type: 18 | 19 | // Parser helps you process lexer tokens 20 | type Parser interface { 21 | 22 | // PeekTokenType allows you to look ahead at tokens without consuming them 23 | PeekTokenType(int) lexer.TokenType 24 | 25 | // PeekToken allows you to look ahead at tokens without consuming them 26 | PeekToken(int) *lexer.Token 27 | 28 | // NextToken consumes and returns the next token 29 | NextToken() *lexer.Token 30 | 31 | // SkipToken consumes the next token without returning it 32 | SkipToken() 33 | 34 | // SkipTokens consumes the next n tokens without returning them 35 | SkipTokens(int) 36 | 37 | // BackupToken un-consumes the last token 38 | BackupToken() 39 | 40 | // BackupTokens un-consumes the last n tokens 41 | BackupTokens(int) 42 | 43 | // ClearTokens clears all consumed tokens 44 | ClearTokens() 45 | 46 | // Emit emits an object, consuming matched tokens 47 | Emit(interface{}) 48 | 49 | EOF() bool 50 | 51 | // Next retrieves the next emitted item 52 | Next() interface{} 53 | 54 | // Marker returns a marker that you can use to reset the parser state later 55 | Marker() *Marker 56 | 57 | // Reset resets the lexer state to the specified marker 58 | Reset(*Marker) 59 | } 60 | 61 | 62 | EXAMPLE 63 | ------- 64 | 65 | Below is a sample calculator program that uses the parser (and lexer) API: 66 | 67 | // 68 | // calc.go implements a simple calculator using the iNamik lexer and parser api. 69 | // 70 | // Input is read from STDIN 71 | // 72 | // The input expression is matched against the following pattern: 73 | // 74 | // input_exp: 75 | // ( id '=' )? general_exp 76 | // general_exp: 77 | // operand ( operator operand )? 78 | // operand: 79 | // number | id | '(' general_exp ')' 80 | // operator: 81 | // '+' | '-' | '*' | '/' 82 | // number: 83 | // digit+ ( '.' digit+ )? 84 | // digit: 85 | // ['0'..'9'] 86 | // id: 87 | // alpha ( alpha | digit )* 88 | // alpha: 89 | // ['a'..'z'] | ['A'..'Z'] 90 | // 91 | // Precedence is as expected, with '*' and '/' have higher precedence 92 | // than '+' and '-', as follows: 93 | // 94 | // 1 + 2 * 3 - 4 / 5 == 1 + (2 * 3) - (4 / 5) 95 | // 96 | 97 | package main 98 | 99 | import ( 100 | "bufio" 101 | "bytes" 102 | "fmt" 103 | "os" 104 | "strconv" 105 | "strings" 106 | ) 107 | import ( 108 | "github.com/iNamik/go_lexer" 109 | "github.com/iNamik/go_lexer/rangeutil" 110 | "github.com/iNamik/go_parser" 111 | ) 112 | 113 | // We define our lexer tokens starting from the pre-defined EOF token 114 | const ( 115 | T_EOF lexer.TokenType = lexer.TokenTypeEOF 116 | T_NIL = lexer.TokenTypeEOF + iota 117 | T_ID 118 | T_NUMBER 119 | T_PLUS 120 | T_MINUS 121 | T_MULTIPLY 122 | T_DIVIDE 123 | T_EQUALS 124 | T_OPEN_PAREN 125 | T_CLOSE_PAREN 126 | ) 127 | 128 | // To store variables 129 | var vars = map[string]float64{} 130 | 131 | // Single-character tokens 132 | var singleChars = []byte{'+', '-', '*', '/', '=', '(', ')'} 133 | 134 | var singleTokens = []lexer.TokenType{T_PLUS, T_MINUS, T_MULTIPLY, T_DIVIDE, T_EQUALS, T_OPEN_PAREN, T_CLOSE_PAREN} 135 | 136 | // Multi-character tokens 137 | var bytesWhitespace = []byte{' ', '\t'} 138 | 139 | var bytesDigits = rangeutil.RangeToBytes("0-9") 140 | 141 | var bytesAlpha = rangeutil.RangeToBytes("a-zA-Z") 142 | 143 | var bytesAlphaNum = rangeutil.RangeToBytes("0-9a-zA-Z") 144 | 145 | // main 146 | func main() { 147 | // Create a buffered reader from STDIN 148 | stdin := bufio.NewReader(os.Stdin) 149 | 150 | for { 151 | // Read a line of input 152 | input, _, err := stdin.ReadLine() 153 | 154 | // Error? we're done 155 | if nil != err { 156 | break 157 | } 158 | 159 | // Anything to process? 160 | if len(input) > 0 { 161 | // Create a new lexer to turn the input text into tokens 162 | l := lexer.New(lex, strings.NewReader(string(input)), len(input), 2) 163 | 164 | // Create a new parser that feeds off the lexer and generates expression values 165 | p := parser.New(parse, l, 2) 166 | 167 | // Loop over parser emits 168 | for i := p.Next(); nil != i; i = p.Next() { 169 | fmt.Printf("%v\n", i) 170 | } 171 | } 172 | } 173 | } 174 | 175 | // lex is the starting (and only) StateFn for lexing the input into tokens 176 | func lex(l lexer.Lexer) lexer.StateFn { 177 | 178 | // EOF 179 | if l.MatchEOF() { 180 | l.EmitEOF() 181 | return nil // We're done here 182 | } 183 | 184 | // Single-char token? 185 | if i := bytes.IndexRune(singleChars, l.PeekRune(0)); i >= 0 { 186 | l.NextRune() 187 | l.EmitToken(singleTokens[i]) 188 | return lex 189 | } 190 | 191 | switch { 192 | 193 | // Skip whitespace 194 | case l.MatchOneOrMoreBytes(bytesWhitespace): 195 | l.IgnoreToken() 196 | 197 | // Number 198 | case l.MatchOneOrMoreBytes(bytesDigits): 199 | if l.PeekRune(0) == '.' { 200 | l.NextRune() // skip '.' 201 | if !l.MatchOneOrMoreBytes(bytesDigits) { 202 | printError(l.Column(), "Illegal number format - Missing digits after '.'") 203 | l.IgnoreToken() 204 | break 205 | } 206 | } 207 | l.EmitTokenWithBytes(T_NUMBER) 208 | 209 | // ID 210 | case l.MatchOneBytes(bytesAlpha) && l.MatchZeroOrMoreBytes(bytesAlphaNum): 211 | l.EmitTokenWithBytes(T_ID) 212 | 213 | // Unknown 214 | default: 215 | l.NextRune() 216 | printError(l.Column(), "Unknown Character") 217 | l.IgnoreToken() 218 | } 219 | 220 | // See you again soon! 221 | return lex 222 | } 223 | 224 | // parse tries to execute a general expression from the lexed tokens. 225 | // Returns nil - We only take one pass at the input string 226 | func parse(p parser.Parser) parser.StateFn { 227 | 228 | if p.PeekTokenType(0) != T_EOF { 229 | // Assignment ( id = general_expression ) 230 | if p.PeekTokenType(0) == T_ID && p.PeekTokenType(1) == T_EQUALS { 231 | tId := p.NextToken() 232 | 233 | p.SkipToken() // skip '=' 234 | 235 | val, ok := pGeneralExpression(p) 236 | 237 | if ok { 238 | t := p.NextToken() 239 | if t.Type() != T_EOF { 240 | printError(t.Column(), "Expecting operator") 241 | } else { 242 | id := string(tId.Bytes()) 243 | vars[id] = val 244 | } 245 | } 246 | // General expression 247 | } else { 248 | val, ok := pGeneralExpression(p) 249 | 250 | if ok { 251 | t := p.NextToken() 252 | if t.Type() != T_EOF { 253 | printError(t.Column(), "Expecting operator") 254 | } else { 255 | p.Emit(val) 256 | } 257 | } 258 | } 259 | } 260 | 261 | p.ClearTokens() 262 | p.Emit(nil) // We're done - One pass only 263 | 264 | return nil 265 | } 266 | 267 | // pGeneralExpression is the starting point for parsing a General Expression. 268 | // It is basically a pass-through to pAdditiveExpression, but it feels cleaner 269 | func pGeneralExpression(p parser.Parser) (f float64, ok bool) { 270 | return pAdditiveExpression(p) 271 | } 272 | 273 | // pAdditiveExpression parses [ expression ( ( '+' | '-' ) expression )? ] 274 | func pAdditiveExpression(p parser.Parser) (f float64, ok bool) { 275 | 276 | f, ok = pMultiplicitiveExpression(p) 277 | 278 | if ok { 279 | t := p.NextToken() 280 | switch t.Type() { 281 | 282 | // Add (+) 283 | case T_PLUS: 284 | r, ok := pAdditiveExpression(p) 285 | if ok { 286 | f += r 287 | } 288 | 289 | // Subtract (-) 290 | case T_MINUS: 291 | r, ok := pAdditiveExpression(p) 292 | if ok { 293 | f -= r 294 | } 295 | 296 | // Unknown - Send it back upstream 297 | default: 298 | p.BackupToken() 299 | ok = true 300 | } 301 | } 302 | 303 | return 304 | } 305 | 306 | // pMultiplicitiveExpression parses [ expression ( ( '*' | '/' ) expression )? ] 307 | func pMultiplicitiveExpression(p parser.Parser) (f float64, ok bool) { 308 | f, ok = pOperand(p) 309 | 310 | if ok { 311 | t := p.NextToken() 312 | switch t.Type() { 313 | 314 | // Multiply (*) 315 | case T_MULTIPLY: 316 | r, ok := pMultiplicitiveExpression(p) 317 | if ok { 318 | f *= r 319 | } 320 | 321 | // Divide (/) 322 | case T_DIVIDE: 323 | r, ok := pMultiplicitiveExpression(p) 324 | if ok { 325 | f /= r 326 | } 327 | 328 | // Unknown - Send it back upstream 329 | default: 330 | p.BackupToken() 331 | ok = true 332 | } 333 | } 334 | 335 | return 336 | } 337 | 338 | // pOperand parses [ id | number | '(' expression ')' ] 339 | func pOperand(p parser.Parser) (f float64, ok bool) { 340 | var err error 341 | 342 | m := p.Marker() 343 | t := p.NextToken() 344 | 345 | switch t.Type() { 346 | 347 | // ID 348 | case T_ID: 349 | var id = string(t.Bytes()) 350 | f, ok = vars[id] 351 | if !ok { 352 | printError(t.Column(), fmt.Sprint("id '", id, "' not defined")) 353 | f = 0.0 354 | } 355 | 356 | // Number 357 | case T_NUMBER: 358 | f, err = strconv.ParseFloat(string(t.Bytes()), 64) 359 | ok = nil == err 360 | if !ok { 361 | printError(t.Column(), fmt.Sprint("Error reading number: ", err.Error())) 362 | f = 0.0 363 | } 364 | 365 | // '(' Expresson ')' 366 | case T_OPEN_PAREN: 367 | f, ok = pGeneralExpression(p) 368 | if ok { 369 | t2 := p.NextToken() 370 | if t2.Type() != T_CLOSE_PAREN { 371 | printError(t.Column(), "Unbalanced Paren") 372 | ok = false 373 | f = 0.0 374 | } 375 | } 376 | 377 | // EOF 378 | case T_EOF: 379 | printError(t.Column(), "Unexpected EOF - Expecting operand") 380 | ok = false 381 | f = 0.0 382 | 383 | // Unknown 384 | default: 385 | printError(t.Column(), "Expecting operand") 386 | ok = false 387 | f = 0.0 388 | } 389 | 390 | if !ok { 391 | p.Reset(m) 392 | } 393 | 394 | return 395 | } 396 | 397 | // printError prints an error msg pointing to the specified column of the input. 398 | func printError(col int, msg string) { 399 | fmt.Print(strings.Repeat(" ", col-1), "^ ", msg, "\n") 400 | } 401 | 402 | 403 | INSTALL 404 | ------- 405 | 406 | The package is built using the Go tool. Assuming you have correctly set the 407 | $GOPATH variable, you can run the folloing command: 408 | 409 | go get github.com/iNamik/go_parser 410 | 411 | 412 | DEPENDENCIES 413 | ------------ 414 | 415 | go_parser depends on the iNamik go_container queue package and the iNamik go_lexer package: 416 | 417 | * https://github.com/iNamik/go_lexer 418 | * https://github.com/iNamik/go_container 419 | 420 | 421 | AUTHORS 422 | ------- 423 | 424 | * David Farrell 425 | 426 | -------------------------------------------------------------------------------- /examples/calc/calc.go: -------------------------------------------------------------------------------- 1 | // 2 | // calc.go implements a simple calculator using the iNamik lexer and parser api. 3 | // 4 | // Input is read from STDIN 5 | // 6 | // The input expression is matched against the following pattern: 7 | // 8 | // input_exp: 9 | // ( id '=' )? general_exp 10 | // general_exp: 11 | // operand ( operator operand )? 12 | // operand: 13 | // number | id | '(' general_exp ')' 14 | // operator: 15 | // '+' | '-' | '*' | '/' 16 | // number: 17 | // digit+ ( '.' digit+ )? 18 | // digit: 19 | // ['0'..'9'] 20 | // id: 21 | // alpha ( alpha | digit )* 22 | // alpha: 23 | // ['a'..'z'] | ['A'..'Z'] 24 | // 25 | // Precedence is as expected, with '*' and '/' have higher precedence 26 | // than '+' and '-', as follows: 27 | // 28 | // 1 + 2 * 3 - 4 / 5 == 1 + (2 * 3) - (4 / 5) 29 | // 30 | 31 | package main 32 | 33 | import ( 34 | "bufio" 35 | "bytes" 36 | "fmt" 37 | "os" 38 | "strconv" 39 | "strings" 40 | ) 41 | import ( 42 | "github.com/iNamik/go_lexer" 43 | "github.com/iNamik/go_lexer/rangeutil" 44 | "github.com/iNamik/go_parser" 45 | ) 46 | 47 | // We define our lexer tokens starting from the pre-defined EOF token 48 | const ( 49 | T_EOF lexer.TokenType = lexer.T_EOF 50 | T_NIL = lexer.T_EOF + iota 51 | T_ID 52 | T_NUMBER 53 | T_PLUS 54 | T_MINUS 55 | T_MULTIPLY 56 | T_DIVIDE 57 | T_EQUALS 58 | T_OPEN_PAREN 59 | T_CLOSE_PAREN 60 | ) 61 | 62 | // To store variables 63 | var vars = map[string]float64{} 64 | 65 | // Single-character tokens 66 | var singleChars = []byte{'+', '-', '*', '/', '=', '(', ')'} 67 | 68 | var singleTokens = []lexer.TokenType{T_PLUS, T_MINUS, T_MULTIPLY, T_DIVIDE, T_EQUALS, T_OPEN_PAREN, T_CLOSE_PAREN} 69 | 70 | // Multi-character tokens 71 | var bytesWhitespace = []byte{' ', '\t'} 72 | 73 | var bytesDigits = rangeutil.RangeToBytes("0-9") 74 | 75 | var bytesAlpha = rangeutil.RangeToBytes("a-zA-Z") 76 | 77 | var bytesAlphaNum = rangeutil.RangeToBytes("0-9a-zA-Z") 78 | 79 | // main 80 | func main() { 81 | // Create a buffered reader from STDIN 82 | stdin := bufio.NewReader(os.Stdin) 83 | 84 | for { 85 | // Read a line of input 86 | input, _, err := stdin.ReadLine() 87 | 88 | // Error? we're done 89 | if nil != err { 90 | break 91 | } 92 | 93 | // Anything to process? 94 | if len(input) > 0 { 95 | // Create a new lexer to turn the input text into tokens 96 | l := lexer.NewFromBytes(lex, input, 2) 97 | 98 | // Create a new parser that feeds off the lexer and generates expression values 99 | p := parser.New(parse, l, 2) 100 | 101 | // Loop over parser emits 102 | for i := p.Next(); nil != i; i = p.Next() { 103 | fmt.Printf("%v\n", i) 104 | } 105 | } 106 | } 107 | } 108 | 109 | // lex is the starting (and only) StateFn for lexing the input into tokens 110 | func lex(l lexer.Lexer) lexer.StateFn { 111 | 112 | // EOF 113 | if l.MatchEOF() { 114 | l.EmitEOF() 115 | return nil // We're done here 116 | } 117 | 118 | // Single-char token? 119 | if i := bytes.IndexRune(singleChars, l.PeekRune(0)); i >= 0 { 120 | l.NextRune() 121 | l.EmitToken(singleTokens[i]) 122 | return lex 123 | } 124 | 125 | switch { 126 | 127 | // Skip whitespace 128 | case l.MatchOneOrMoreBytes(bytesWhitespace): 129 | l.IgnoreToken() 130 | 131 | // Number 132 | case l.MatchOneOrMoreBytes(bytesDigits): 133 | if l.PeekRune(0) == '.' { 134 | l.NextRune() // skip '.' 135 | if !l.MatchOneOrMoreBytes(bytesDigits) { 136 | printError(l.Column(), "Illegal number format - Missing digits after '.'") 137 | l.IgnoreToken() 138 | break 139 | } 140 | } 141 | l.EmitTokenWithBytes(T_NUMBER) 142 | 143 | // ID 144 | case l.MatchOneBytes(bytesAlpha) && l.MatchZeroOrMoreBytes(bytesAlphaNum): 145 | l.EmitTokenWithBytes(T_ID) 146 | 147 | // Unknown 148 | default: 149 | l.NextRune() 150 | printError(l.Column(), "Unknown Character") 151 | l.IgnoreToken() 152 | } 153 | 154 | // See you again soon! 155 | return lex 156 | } 157 | 158 | // parse tries to execute a general expression from the lexed tokens. 159 | // Returns nil - We only take one pass at the input string 160 | func parse(p parser.Parser) parser.StateFn { 161 | 162 | if p.PeekTokenType(0) != T_EOF { 163 | // Assignment ( id = general_expression ) 164 | if p.PeekTokenType(0) == T_ID && p.PeekTokenType(1) == T_EQUALS { 165 | tId := p.NextToken() 166 | 167 | p.SkipToken() // skip '=' 168 | 169 | val, ok := pGeneralExpression(p) 170 | 171 | if ok { 172 | t := p.NextToken() 173 | if t.Type() != T_EOF { 174 | printError(t.Column(), "Expecting operator") 175 | } else { 176 | id := string(tId.Bytes()) 177 | vars[id] = val 178 | } 179 | } 180 | // General expression 181 | } else { 182 | val, ok := pGeneralExpression(p) 183 | 184 | if ok { 185 | t := p.NextToken() 186 | if t.Type() != T_EOF { 187 | printError(t.Column(), "Expecting operator") 188 | } else { 189 | p.Emit(val) 190 | } 191 | } 192 | } 193 | } 194 | 195 | p.ClearTokens() 196 | p.Emit(nil) // We're done - One pass only 197 | 198 | return nil 199 | } 200 | 201 | // pGeneralExpression is the starting point for parsing a General Expression. 202 | // It is basically a pass-through to pAdditiveExpression, but it feels cleaner 203 | func pGeneralExpression(p parser.Parser) (f float64, ok bool) { 204 | return pAdditiveExpression(p) 205 | } 206 | 207 | // pAdditiveExpression parses [ expression ( ( '+' | '-' ) expression )? ] 208 | func pAdditiveExpression(p parser.Parser) (f float64, ok bool) { 209 | 210 | f, ok = pMultiplicitiveExpression(p) 211 | 212 | if ok { 213 | t := p.NextToken() 214 | switch t.Type() { 215 | 216 | // Add (+) 217 | case T_PLUS: 218 | r, ok := pAdditiveExpression(p) 219 | if ok { 220 | f += r 221 | } 222 | 223 | // Subtract (-) 224 | case T_MINUS: 225 | r, ok := pAdditiveExpression(p) 226 | if ok { 227 | f -= r 228 | } 229 | 230 | // Unknown - Send it back upstream 231 | default: 232 | p.BackupToken() 233 | ok = true 234 | } 235 | } 236 | 237 | return 238 | } 239 | 240 | // pMultiplicitiveExpression parses [ expression ( ( '*' | '/' ) expression )? ] 241 | func pMultiplicitiveExpression(p parser.Parser) (f float64, ok bool) { 242 | f, ok = pOperand(p) 243 | 244 | if ok { 245 | t := p.NextToken() 246 | switch t.Type() { 247 | 248 | // Multiply (*) 249 | case T_MULTIPLY: 250 | r, ok := pMultiplicitiveExpression(p) 251 | if ok { 252 | f *= r 253 | } 254 | 255 | // Divide (/) 256 | case T_DIVIDE: 257 | r, ok := pMultiplicitiveExpression(p) 258 | if ok { 259 | f /= r 260 | } 261 | 262 | // Unknown - Send it back upstream 263 | default: 264 | p.BackupToken() 265 | ok = true 266 | } 267 | } 268 | 269 | return 270 | } 271 | 272 | // pOperand parses [ id | number | '(' expression ')' ] 273 | func pOperand(p parser.Parser) (f float64, ok bool) { 274 | var err error 275 | 276 | m := p.Marker() 277 | t := p.NextToken() 278 | 279 | switch t.Type() { 280 | 281 | // ID 282 | case T_ID: 283 | var id = string(t.Bytes()) 284 | f, ok = vars[id] 285 | if !ok { 286 | printError(t.Column(), fmt.Sprint("id '", id, "' not defined")) 287 | f = 0.0 288 | } 289 | 290 | // Number 291 | case T_NUMBER: 292 | f, err = strconv.ParseFloat(string(t.Bytes()), 64) 293 | ok = nil == err 294 | if !ok { 295 | printError(t.Column(), fmt.Sprint("Error reading number: ", err.Error())) 296 | f = 0.0 297 | } 298 | 299 | // '(' Expresson ')' 300 | case T_OPEN_PAREN: 301 | f, ok = pGeneralExpression(p) 302 | if ok { 303 | t2 := p.NextToken() 304 | if t2.Type() != T_CLOSE_PAREN { 305 | printError(t.Column(), "Unbalanced Paren") 306 | ok = false 307 | f = 0.0 308 | } 309 | } 310 | 311 | // EOF 312 | case T_EOF: 313 | printError(t.Column(), "Unexpected EOF - Expecting operand") 314 | ok = false 315 | f = 0.0 316 | 317 | // Unknown 318 | default: 319 | printError(t.Column(), "Expecting operand") 320 | ok = false 321 | f = 0.0 322 | } 323 | 324 | if !ok { 325 | p.Reset(m) 326 | } 327 | 328 | return 329 | } 330 | 331 | // printError prints an error msg pointing to the specified column of the input. 332 | func printError(col int, msg string) { 333 | fmt.Print(strings.Repeat(" ", col-1), "^ ", msg, "\n") 334 | } 335 | -------------------------------------------------------------------------------- /impl.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import "github.com/iNamik/go_lexer" 4 | 5 | // Parser::Line 6 | func (p *parser) Line() int { 7 | return p.PeekToken(0).Line() 8 | } 9 | 10 | // Parser::Column 11 | func (p *parser) Column() int { 12 | return p.PeekToken(0).Column() 13 | } 14 | 15 | // Parser::Next - Returns the next emit from the parser. 16 | func (p *parser) Next() interface{} { 17 | for { 18 | select { 19 | case i := <-p.chn: 20 | return i 21 | default: 22 | p.state = p.state(p) 23 | } 24 | } 25 | panic("not reached") 26 | } 27 | 28 | // Parser::PeekTokenType 29 | func (p *parser) PeekTokenType(n int) lexer.TokenType { 30 | return p.PeekToken(n).Type() 31 | } 32 | 33 | // Parser::PeekToken 34 | func (p *parser) PeekToken(n int) *lexer.Token { 35 | ok := p.ensureTokenLen(p.pos + n + 1) // Correct for 0-based 'n' 36 | 37 | if !ok { 38 | if nil == p.eofToken { 39 | panic("illegal state: eofToken is nil") 40 | } 41 | return p.eofToken 42 | } 43 | 44 | i := p.tokens.Peek(p.pos + n) 45 | 46 | return i.(*lexer.Token) 47 | } 48 | 49 | // Parser::NextToken 50 | func (p *parser) NextToken() *lexer.Token { 51 | ok := p.ensureTokenLen(p.pos + 1) 52 | 53 | if !ok { 54 | if nil == p.eofToken { 55 | panic("illegal state: eofToken is nil") 56 | } 57 | return p.eofToken 58 | } 59 | 60 | i := p.tokens.Peek(p.pos) // 0-based 61 | 62 | p.pos++ 63 | 64 | return i.(*lexer.Token) 65 | } 66 | 67 | // Parser::SkipToken 68 | func (p *parser) SkipToken() { 69 | ok := p.ensureTokenLen(p.pos + 1) 70 | 71 | if ok { 72 | p.pos++ 73 | } 74 | } 75 | 76 | // Parser::SkipTokens 77 | func (p *parser) SkipTokens(n int) { 78 | ok := p.ensureTokenLen(p.pos + n + 1) 79 | 80 | if ok { 81 | p.pos += n 82 | } 83 | } 84 | 85 | // Parser::BackupToken 86 | func (p *parser) BackupToken() { 87 | p.BackupTokens(1) 88 | } 89 | 90 | // Parser::BackupTokens 91 | func (p *parser) BackupTokens(n int) { 92 | if n > p.pos { 93 | panic("Underflow Exception") 94 | } 95 | p.pos -= n 96 | } 97 | 98 | // Parser::ClearTokens 99 | func (p *parser) ClearTokens() { 100 | for ; p.pos > 0; p.pos-- { 101 | p.tokens.Remove() 102 | } 103 | } 104 | 105 | // Parser::Emit 106 | func (p *parser) Emit(i interface{}) { 107 | p.chn <- i 108 | } 109 | 110 | // Parser::EOF 111 | func (p *parser) EOF() bool { 112 | return p.eof 113 | } 114 | 115 | // Parser::Marker 116 | func (p *parser) Marker() *Marker { 117 | return &Marker{sequence: p.sequence, pos: p.pos} 118 | } 119 | 120 | // Parser::Reset 121 | func (p *parser) Reset(m *Marker) { 122 | if m.sequence != p.sequence || m.pos < 0 || m.pos >= p.tokens.Len() { 123 | panic("Invalid marker") 124 | } 125 | p.pos = m.pos 126 | } 127 | -------------------------------------------------------------------------------- /parser.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import "github.com/iNamik/go_lexer" 4 | import "github.com/iNamik/go_container/queue" 5 | 6 | // StateFn represents the state of the parser as a function that returns the next state. 7 | type StateFn func(Parser) StateFn 8 | 9 | // Marker stores the state of the parser to allow rewinding 10 | type Marker struct { 11 | sequence int 12 | pos int 13 | } 14 | 15 | // Parser helps you process lexer tokens 16 | type Parser interface { 17 | 18 | // Line returns the line number of the next token (aka PeekTotenType(0).Line() ) 19 | Line() int 20 | 21 | // Column returns the column number of the next token (aka PeekTokenType(0).column() ) 22 | Column() int 23 | 24 | // PeekTokenType allows you to look ahead at tokens without consuming them 25 | PeekTokenType(int) lexer.TokenType 26 | 27 | // PeekToken allows you to look ahead at tokens without consuming them 28 | PeekToken(int) *lexer.Token 29 | 30 | // NextToken consumes and returns the next token 31 | NextToken() *lexer.Token 32 | 33 | // SkipToken consumes the next token without returning it 34 | SkipToken() 35 | 36 | // SkipTokens consumes the next n tokens without returning them 37 | SkipTokens(int) 38 | 39 | // BackupToken un-consumes the last token 40 | BackupToken() 41 | 42 | // BackupTokens un-consumes the last n tokens 43 | BackupTokens(int) 44 | 45 | // ClearTokens clears all consumed tokens 46 | ClearTokens() 47 | 48 | // Emit emits an object, consuming matched tokens 49 | Emit(interface{}) 50 | 51 | EOF() bool 52 | 53 | // Next retrieves the next emitted item 54 | Next() interface{} 55 | 56 | // Marker returns a marker that you can use to reset the parser state later 57 | Marker() *Marker 58 | 59 | // Reset resets the lexer state to the specified marker 60 | Reset(*Marker) 61 | } 62 | 63 | // New returns a new Parser object 64 | func New(startState StateFn, lex lexer.Lexer, channelCap int) Parser { 65 | p := &parser{ 66 | lex: lex, 67 | tokens: queue.New(4), // 4 is just a nice number that seems appropriate 68 | pos: 0, 69 | sequence: 0, 70 | eofToken: nil, 71 | eof: false, 72 | state: startState, 73 | chn: make(chan interface{}, channelCap), 74 | } 75 | return p 76 | } 77 | -------------------------------------------------------------------------------- /private.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/iNamik/go_container/queue" 5 | "github.com/iNamik/go_lexer" 6 | ) 7 | 8 | type parser struct { 9 | lex lexer.Lexer 10 | tokens queue.Interface 11 | pos int 12 | sequence int 13 | eof bool 14 | eofToken *lexer.Token 15 | state StateFn 16 | chn chan interface{} // channel of objects 17 | } 18 | 19 | func (p *parser) ensureTokenLen(n int) bool { 20 | for !p.eof && p.tokens.Len() < n { 21 | token := p.lex.NextToken() 22 | if token.EOF() { 23 | p.eofToken = token 24 | p.eof = true 25 | } 26 | p.tokens.Add(token) 27 | } 28 | return p.tokens.Len() >= n 29 | } 30 | --------------------------------------------------------------------------------