├── .gitignore ├── examples ├── add.scm ├── minus.scm ├── example.scm ├── bad_parse_missing_closing_paren.scm ├── func.scm └── fib.scm ├── go.mod ├── README.md ├── main.go ├── go.sum ├── parse.go ├── lex_test.go ├── parse_test.go ├── ast_walker.go └── lex.go /.gitignore: -------------------------------------------------------------------------------- 1 | livescheme -------------------------------------------------------------------------------- /examples/add.scm: -------------------------------------------------------------------------------- 1 | (+ 12 1) 2 | -------------------------------------------------------------------------------- /examples/minus.scm: -------------------------------------------------------------------------------- 1 | (- 3 2) 2 | -------------------------------------------------------------------------------- /examples/example.scm: -------------------------------------------------------------------------------- 1 | ( + 13 ( - 12 1) ) 2 | -------------------------------------------------------------------------------- /examples/bad_parse_missing_closing_paren.scm: -------------------------------------------------------------------------------- 1 | 2 | ( 3 | -------------------------------------------------------------------------------- /examples/func.scm: -------------------------------------------------------------------------------- 1 | (func plus (a b) (+ a b)) 2 | 3 | (plus 2 3) 4 | -------------------------------------------------------------------------------- /examples/fib.scm: -------------------------------------------------------------------------------- 1 | (func fib (a) 2 | (if (< a 2) 3 | a 4 | (+ (fib (- a 1)) (fib (- a 2))))) 5 | 6 | (fib 11) 7 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module livescheme 2 | 3 | go 1.19 4 | 5 | require github.com/stretchr/testify v1.8.1 6 | 7 | require ( 8 | github.com/davecgh/go-spew v1.1.1 // indirect 9 | github.com/pmezard/go-difflib v1.0.0 // indirect 10 | gopkg.in/yaml.v3 v3.0.1 // indirect 11 | ) 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Building a Scheme-like language in Golang live on Twitch 2 | 3 | Requirements: 4 | * Go 1.18+ 5 | 6 | To run: 7 | 8 | ```shell 9 | $ go mod tidy 10 | $ go test 11 | $ go build 12 | $ cat examples/fib.scm 13 | (func fib (a) 14 | (if (< a 2) 15 | a 16 | (+ (fib (- a 1)) (fib (- a 2))))) 17 | 18 | (fib 11) 19 | $ ./livescheme examples/fib.scm 20 | 89 21 | ``` 22 | 23 | Archives: 24 | * [Part 1: A lexer](https://www.youtube.com/watch?v=lZNhZI-dN9k) 25 | * [Part 2: Parsing](https://www.youtube.com/watch?v=5ttFEPQopXc) 26 | * [Part 3: AST walking interpreter](https://www.youtube.com/watch?v=YwmGcverSHI) 27 | * [Part 4: Cleanup and Fibonacci](https://www.youtube.com/watch?v=skDhTWILH8I) 28 | 29 | 30 | Stream (Paused): 31 | * Sundays at 5pm NY time 32 | * [twitch.tv/eatonphil](https://twitch.tv/eatonphil) 33 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "os" 4 | import "fmt" 5 | 6 | func main() { 7 | // accept program 8 | lc := newLexingContext(os.Args[1]) 9 | 10 | tokens := lc.lex() 11 | debug := false 12 | if debug { 13 | for _, token := range tokens { 14 | fmt.Println(token.value) 15 | } 16 | } 17 | 18 | var parseIndex int 19 | var a = ast{ 20 | value{ 21 | kind: literalValue, 22 | literal: &token{ 23 | value: "begin", 24 | kind: identifierToken, 25 | }, 26 | }, 27 | } 28 | 29 | // Need to keep parsing until end of ALL tokens 30 | for parseIndex < len(tokens) { 31 | childAst, nextIndex := parse(tokens, parseIndex) 32 | a = append(a, value{ 33 | kind: listValue, 34 | list: &childAst, 35 | }) 36 | parseIndex = nextIndex 37 | } 38 | 39 | if parseIndex < len(tokens) { 40 | panic("Incomplete parse") 41 | } 42 | 43 | if debug { 44 | fmt.Println(a.pretty()) 45 | } 46 | 47 | // Other potential steps: 48 | // 1. static type checking? 49 | // not in our language 50 | 51 | // 2. other optimization steps: constant propagation? (+ 5 2) => 7 52 | // not for now 53 | 54 | initializeBuiltins() 55 | ctx := map[string]any{} 56 | value := astWalk(a, ctx) 57 | fmt.Println(value) 58 | 59 | // TODO: compile the AST to JavaScript? Go? C? Assembly? LLVM? 60 | //compile(ast) 61 | } 62 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 5 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 6 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 7 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 8 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 9 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 10 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 11 | github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= 12 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 13 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 14 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 15 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 16 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 17 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 18 | -------------------------------------------------------------------------------- /parse.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "os" 4 | 5 | type valueKind uint 6 | 7 | const ( 8 | literalValue valueKind = iota 9 | listValue 10 | ) 11 | 12 | type value struct { 13 | kind valueKind 14 | literal *token 15 | list *ast 16 | } 17 | 18 | func (v value) pretty() string { 19 | if v.kind == literalValue { 20 | return v.literal.value 21 | } 22 | 23 | return v.list.pretty() 24 | } 25 | 26 | type ast []value 27 | 28 | func (a ast) pretty() string { 29 | p := "(" 30 | for _, value := range a { 31 | p += value.pretty() 32 | p += " " 33 | } 34 | 35 | return p + ")" 36 | } 37 | 38 | // for example: "(+ 13 (- 12 1)" 39 | // parse(["(", "+", "13", "(", "-", "12", "1", ")", ")"]): 40 | // 41 | // should produce: ast{ 42 | // value{ 43 | // kind: literal, 44 | // literal: "+", 45 | // }, 46 | // value{ 47 | // kind: literal, 48 | // literal: "13", 49 | // }, 50 | // value{ 51 | // kind: list, 52 | // list: ast { 53 | // value { 54 | // kind: literal, 55 | // literal: "-", 56 | // }, 57 | // value { 58 | // kind: literal, 59 | // literal: "12", 60 | // }, 61 | // value { 62 | // kind: literal, 63 | // literal: "1", 64 | // }, 65 | // } 66 | // } 67 | // } 68 | func parse(tokens []token, index int) (ast, int) { 69 | var a ast 70 | 71 | token := tokens[index] 72 | if !(token.kind == syntaxToken && 73 | token.value == "(") { 74 | panic("Should be an open parenthesis") 75 | } 76 | index++ 77 | 78 | for index < len(tokens) { 79 | token := tokens[index] 80 | if token.kind == syntaxToken && 81 | token.value == "(" { 82 | // Maybe should have error handling here? 83 | child, nextIndex := parse(tokens, index) 84 | a = append(a, value{ 85 | kind: listValue, 86 | list: &child, 87 | }) 88 | index = nextIndex 89 | continue 90 | } 91 | 92 | if token.kind == syntaxToken && 93 | token.value == ")" { 94 | // TBD if the index we're returning is correct 95 | return a, index + 1 96 | } 97 | 98 | a = append(a, value{ 99 | kind: literalValue, 100 | literal: &token, 101 | }) 102 | index++ 103 | } 104 | 105 | if tokens[index-1].kind == syntaxToken && 106 | tokens[index-1].value != ")" { 107 | tokens[index-1].debug("Expected closing paren") 108 | os.Exit(1) 109 | } 110 | 111 | return a, index 112 | } 113 | -------------------------------------------------------------------------------- /lex_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | // lexIntegerToken("foo 123", 4) => "123" 9 | // lexIntegerToken("foo 12 3", 4) => "12" 10 | // lexIntegerToken("foo 12a 3", 4) => "12" <-- Ignoring this situation 11 | func Test_lexIntegerToken(t *testing.T) { 12 | tests := []struct { 13 | source string 14 | cursor int 15 | expectedValue string 16 | expectedCursor int 17 | }{ 18 | { 19 | "foo 123", 20 | 4, 21 | "123", 22 | 7, 23 | }, 24 | { 25 | "foo 12 3", 26 | 4, 27 | "12", 28 | 6, 29 | }, 30 | { 31 | "foo 12a 3", 32 | 4, 33 | "12", 34 | 6, 35 | }, 36 | } 37 | for _, test := range tests { 38 | lc := lexingContext{ 39 | source: []rune(test.source), 40 | sourceFileName: "", 41 | } 42 | cursor, token := lc.lexIntegerToken(test.cursor) 43 | assert.Equal(t, cursor, test.expectedCursor) 44 | assert.Equal(t, token.value, test.expectedValue) 45 | assert.Equal(t, token.kind, integerToken) 46 | } 47 | } 48 | 49 | // lexIdentifierToken("123 ab + ", 4) => "ab" 50 | // lexIdentifierToken("123 ab123 + ", 4) => "ab123" 51 | func Test_lexIdentifierToken(t *testing.T) { 52 | tests := []struct { 53 | source string 54 | cursor int 55 | expectedValue string 56 | expectedCursor int 57 | }{ 58 | { 59 | "123 ab + ", 60 | 4, 61 | "ab", 62 | 6, 63 | }, 64 | { 65 | "123 ab123 + ", 66 | 4, 67 | "ab123", 68 | 9, 69 | }, 70 | } 71 | for _, test := range tests { 72 | lc := lexingContext{ 73 | source: []rune(test.source), 74 | sourceFileName: "", 75 | } 76 | cursor, token := lc.lexIdentifierToken(test.cursor) 77 | assert.Equal(t, cursor, test.expectedCursor) 78 | assert.Equal(t, token.value, test.expectedValue) 79 | assert.Equal(t, token.kind, identifierToken) 80 | } 81 | } 82 | 83 | // lex(" ( + 13 2 )") should produce: ["(", "+", "13", "2", ")"] 84 | func Test_lex(t *testing.T) { 85 | tests := []struct { 86 | source string 87 | tokens []token 88 | }{ 89 | { 90 | " ( + 13 2 )", 91 | []token{ 92 | { 93 | value: "(", 94 | kind: syntaxToken, 95 | location: 1, 96 | }, 97 | { 98 | value: "+", 99 | kind: identifierToken, 100 | location: 3, 101 | }, 102 | { 103 | value: "13", 104 | kind: integerToken, 105 | location: 5, 106 | }, 107 | { 108 | value: "2", 109 | kind: integerToken, 110 | location: 8, 111 | }, 112 | { 113 | value: ")", 114 | kind: syntaxToken, 115 | location: 11, 116 | }, 117 | }, 118 | }, 119 | } 120 | 121 | for _, test := range tests { 122 | lc := lexingContext{ 123 | source: []rune(test.source), 124 | sourceFileName: "", 125 | } 126 | tokens := lc.lex() 127 | for i, token := range tokens { 128 | // Cheating by setting the received token's 129 | // lexingContext to the expected lexingContext 130 | token.lc = test.tokens[i].lc 131 | assert.Equal(t, token, test.tokens[i]) 132 | } 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /parse_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | func compareValue(a value, b value) bool { 10 | if a.kind != b.kind { 11 | fmt.Println("Value kinds not equal", a.kind, b.kind) 12 | return false 13 | } 14 | 15 | if a.kind == literalValue { 16 | if a.literal.value != b.literal.value { 17 | fmt.Println("Literals not equal", a.literal, b.literal) 18 | return false 19 | } 20 | 21 | return true 22 | } 23 | 24 | return compareAst(*a.list, *b.list) 25 | } 26 | 27 | func compareAst(a ast, b ast) bool { 28 | if len(a) != len(b) { 29 | fmt.Println("AST lengths not equal", len(a), len(b)) 30 | return false 31 | } 32 | 33 | for i := range a { 34 | aI := a[i] 35 | bI := b[i] 36 | 37 | if !compareValue(aI, bI) { 38 | return false 39 | } 40 | } 41 | 42 | return true 43 | } 44 | 45 | func Test_parse(t *testing.T) { 46 | tests := []struct { 47 | input string 48 | prettyOutput string 49 | output ast 50 | }{ 51 | { 52 | "(+ 1 2)", 53 | "(+ 1 2 )", 54 | ast{ 55 | value{ 56 | kind: literalValue, 57 | literal: &token{value: "+"}, 58 | }, 59 | value{ 60 | kind: literalValue, 61 | literal: &token{value: "1"}, 62 | }, 63 | value{ 64 | kind: literalValue, 65 | literal: &token{value: "2"}, 66 | }, 67 | }, 68 | }, 69 | { 70 | "(+ 1 (- 12 9))", 71 | "(+ 1 (- 12 9 ) )", 72 | ast{ 73 | value{ 74 | kind: literalValue, 75 | literal: &token{value: "+"}, 76 | }, 77 | value{ 78 | kind: literalValue, 79 | literal: &token{value: "1"}, 80 | }, 81 | value{ 82 | kind: listValue, 83 | list: &ast{ 84 | value{ 85 | kind: literalValue, 86 | literal: &token{value: "-"}, 87 | }, 88 | value{ 89 | kind: literalValue, 90 | literal: &token{value: "12"}, 91 | }, 92 | value{ 93 | kind: literalValue, 94 | literal: &token{value: "9"}, 95 | }, 96 | }, 97 | }, 98 | }, 99 | }, 100 | { 101 | "(+ 1 (- 12 9) 12)", 102 | "(+ 1 (- 12 9 ) 12 )", 103 | ast{ 104 | value{ 105 | kind: literalValue, 106 | literal: &token{value: "+"}, 107 | }, 108 | value{ 109 | kind: literalValue, 110 | literal: &token{value: "1"}, 111 | }, 112 | value{ 113 | kind: listValue, 114 | list: &ast{ 115 | value{ 116 | kind: literalValue, 117 | literal: &token{value: "-"}, 118 | }, 119 | value{ 120 | kind: literalValue, 121 | literal: &token{value: "12"}, 122 | }, 123 | value{ 124 | kind: literalValue, 125 | literal: &token{value: "9"}, 126 | }, 127 | }, 128 | }, 129 | value{ 130 | kind: literalValue, 131 | literal: &token{value: "12"}, 132 | }, 133 | }, 134 | }, 135 | { 136 | "((+ 1 2) 1 (- 12 9) 12)", 137 | "((+ 1 2 ) 1 (- 12 9 ) 12 )", 138 | ast{ 139 | value{ 140 | kind: listValue, 141 | list: &ast{ 142 | value{ 143 | kind: literalValue, 144 | literal: &token{value: "+"}, 145 | }, 146 | value{ 147 | kind: literalValue, 148 | literal: &token{value: "1"}, 149 | }, 150 | value{ 151 | kind: literalValue, 152 | literal: &token{value: "2"}, 153 | }, 154 | }, 155 | }, 156 | value{ 157 | kind: literalValue, 158 | literal: &token{value: "1"}, 159 | }, 160 | value{ 161 | kind: listValue, 162 | list: &ast{ 163 | value{ 164 | kind: literalValue, 165 | literal: &token{value: "-"}, 166 | }, 167 | value{ 168 | kind: literalValue, 169 | literal: &token{value: "12"}, 170 | }, 171 | value{ 172 | kind: literalValue, 173 | literal: &token{value: "9"}, 174 | }, 175 | }, 176 | }, 177 | value{ 178 | kind: literalValue, 179 | literal: &token{value: "12"}, 180 | }, 181 | }, 182 | }, 183 | } 184 | 185 | for _, test := range tests { 186 | lc := lexingContext{ 187 | source: []rune(test.input), 188 | sourceFileName: "", 189 | } 190 | tokens := lc.lex() 191 | ast, _ := parse(tokens, 0) 192 | assert.Equal(t, test.prettyOutput, ast.pretty()) 193 | assert.True(t, compareAst(test.output, ast)) 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /ast_walker.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strconv" 7 | ) 8 | 9 | var builtins = map[string]func([]value, map[string]any) any{} 10 | 11 | func copyContext(in map[string]any) map[string]any { 12 | out := map[string]any{} 13 | for key, val := range in { 14 | out[key] = val 15 | } 16 | 17 | return out 18 | } 19 | 20 | // Eventually with all operations implemented we can do stuff like fib: 21 | // (func fib (a) 22 | // (if (a < 1) 23 | // a 24 | // (fib (- a 1) (- a 2)) 25 | 26 | func initializeBuiltins() { 27 | // (if (< a 2) 28 | // then 29 | // elsecase) 30 | builtins["if"] = func(args []value, ctx map[string]any) any { 31 | condition := astWalk2(args[0], ctx) 32 | then := args[1] 33 | _else := args[2] 34 | 35 | if condition.(bool) == true { 36 | return astWalk2(then, ctx) 37 | } 38 | 39 | return astWalk2(_else, ctx) 40 | } 41 | 42 | builtins["<"] = func(args []value, ctx map[string]any) any { 43 | return astWalk2(args[0], ctx).(int64) < astWalk2(args[1], ctx).(int64) 44 | } 45 | 46 | builtins["+"] = func(args []value, ctx map[string]any) any { 47 | var i int64 48 | for _, arg := range args { 49 | i += astWalk2(arg, ctx).(int64) 50 | } 51 | return i 52 | } 53 | 54 | builtins["-"] = func(args []value, ctx map[string]any) any { 55 | i := astWalk2(args[0], ctx).(int64) 56 | for _, arg := range args[1:] { 57 | i -= astWalk2(arg, ctx).(int64) 58 | } 59 | return i 60 | } 61 | 62 | builtins["begin"] = func(args []value, ctx map[string]any) any { 63 | var last any 64 | for _, arg := range args { 65 | last = astWalk2(arg, ctx) 66 | } 67 | 68 | return last 69 | } 70 | 71 | // (var a 2) 72 | // (func plus (a b) (+ a b)) 73 | builtins["func"] = func(args []value, ctx map[string]any) any { 74 | // e.g. `plus` 75 | functionName := (*args[0].literal).value 76 | 77 | // e.g. `(a b)` 78 | params := *args[1].list 79 | 80 | // e.g. (+ a b) 81 | body := *args[2].list 82 | 83 | // e.g. `(plus 1 2)` 84 | ctx[functionName] = func(args []any, ctx map[string]any) any { 85 | childCtx := copyContext(ctx) 86 | if len(params) != len(args) { 87 | // TODO: instead of accepting args 88 | // already evaluated, we should 89 | // evaluate args inside of here so we 90 | // can give a nice error message 91 | // instead of panic-ing. To do that we 92 | // need the original tokens, so we can 93 | // debug the token. 94 | panic(fmt.Sprintf("Expected %d args to `%s`, got %d", len(params), functionName, len(args))) 95 | } 96 | for i, param := range params { 97 | childCtx[(*param.literal).value] = args[i] 98 | } 99 | 100 | return astWalk(body, childCtx) 101 | } 102 | 103 | return ctx[functionName] 104 | } 105 | } 106 | 107 | // Later: user defined functions 108 | // (func plus (a b) (+ a b)) 109 | // And also user defined variables 110 | // (var a 12) 111 | 112 | // Example of evaluation 113 | // ( + 13 ( - 12 1) ) 114 | // + 13 11 115 | // 24 116 | 117 | // Example file that is ok: 118 | // (+ 12 1) 119 | // (- 134 9) 120 | // All expression get evaluated. Only the last one is returned. 121 | // That file is transformed into: 122 | // (begin 123 | // 124 | // (+ 12 1) 125 | // (- 134 9)) 126 | // 127 | // Before the astWalk stage 128 | func astWalk(ast []value, ctx map[string]any) any { 129 | // Default case: we've got a list 130 | // Example: (+ 1 2) 131 | // Example: `if`, `+` 132 | functionName := (*ast[0].literal).value 133 | 134 | if builtinFunction, ok := builtins[functionName]; ok { 135 | return builtinFunction(ast[1:], ctx) 136 | } 137 | 138 | // Case: calling a function that is not built in 139 | maybeFunction, ok := ctx[functionName] 140 | if !ok { 141 | (*ast[0].literal).debug(fmt.Sprintf("Expected function, got %s", functionName)) 142 | os.Exit(1) 143 | } 144 | userDefinedFunction := maybeFunction.(func([]any, map[string]any) any) 145 | 146 | // Do we evaluate args here? 147 | // If so, special functions like `if` must be handled separately 148 | var args []any 149 | for _, unevaluatedArg := range ast[1:] { 150 | args = append(args, astWalk2(unevaluatedArg, ctx)) 151 | } 152 | 153 | return userDefinedFunction(args, ctx) 154 | } 155 | 156 | func astWalk2(v value, ctx map[string]any) any { 157 | if v.kind == literalValue { 158 | t := *v.literal 159 | switch t.kind { 160 | // `12`, `1` 161 | case integerToken: 162 | // parseInt here 163 | i, err := strconv.ParseInt(t.value, 10, 64) 164 | if err != nil { 165 | fmt.Println("Expected an integer, got: ", t.value) 166 | panic(err) 167 | } 168 | 169 | return i 170 | // (var a 3) 171 | // (+ a 1) => result should be `4` 172 | // `+`, or `if` 173 | case identifierToken: 174 | return ctx[t.value] 175 | } 176 | } 177 | 178 | return astWalk(*v.list, ctx) 179 | } 180 | -------------------------------------------------------------------------------- /lex.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "unicode" 7 | ) 8 | 9 | type lexingContext struct { 10 | source []rune 11 | sourceFileName string 12 | } 13 | 14 | type tokenKind uint 15 | 16 | const ( 17 | // e.g. "(", ")" 18 | syntaxToken tokenKind = iota 19 | // e.g. "1", "12" 20 | integerToken 21 | // e.g. "+", "define" 22 | identifierToken 23 | ) 24 | 25 | type token struct { 26 | value string 27 | kind tokenKind 28 | location int 29 | lc lexingContext 30 | } 31 | 32 | func (t token) debug(description string) { 33 | // 1. Grab the entire line from the source code where the token is at 34 | // 2. Print the entire line 35 | // 3. Print a marker to the column where the token is at 36 | // 4. Print the error/debug description 37 | 38 | var tokenLine []rune 39 | var tokenLineNumber int 40 | var tokenColumn int 41 | var inTokenLine bool 42 | var i int 43 | 44 | for i < len(t.lc.source) { 45 | r := t.lc.source[i] 46 | 47 | if i < t.location { 48 | tokenColumn++ 49 | } 50 | 51 | tokenLine = append(tokenLine, r) 52 | 53 | if r == '\n' { 54 | tokenLineNumber++ 55 | // Got to the end of the line that the token is in. 56 | if inTokenLine { 57 | // Now outside the loop, `tokenLine` 58 | // will contain the entire source code 59 | // line where the token was. And 60 | // `tokenColumn` will be the column 61 | // number of the token. 62 | break 63 | } 64 | 65 | tokenColumn = 1 66 | tokenLine = nil 67 | } 68 | 69 | if i == t.location { 70 | inTokenLine = true 71 | } 72 | 73 | i++ 74 | } 75 | 76 | fmt.Printf("%s [at line %d, column %d in file %s]\n", 77 | description, tokenLineNumber, tokenColumn, t.lc.sourceFileName) 78 | fmt.Println(string(tokenLine)) 79 | 80 | // WILL NOT IF THERE ARE TABS OR OTHER WEIRD CHARACTERS 81 | for tokenColumn >= 1 { 82 | fmt.Printf(" ") 83 | tokenColumn-- 84 | } 85 | fmt.Println("^ near here") 86 | } 87 | 88 | func eatWhitespace(source []rune, cursor int) int { 89 | for cursor < len(source) { 90 | if unicode.IsSpace(source[cursor]) { 91 | cursor++ 92 | continue 93 | } 94 | 95 | break 96 | } 97 | 98 | return cursor 99 | } 100 | 101 | func (lc lexingContext) lexSyntaxToken(cursor int) (int, *token) { 102 | if lc.source[cursor] == '(' || lc.source[cursor] == ')' { 103 | return cursor + 1, &token{ 104 | value: string([]rune{lc.source[cursor]}), 105 | kind: syntaxToken, 106 | location: cursor, 107 | lc: lc, 108 | } 109 | } 110 | 111 | return cursor, nil 112 | } 113 | 114 | // lexIntegerToken("foo 123", 4) => "123" 115 | // lexIntegerToken("foo 12 3", 4) => "12" 116 | // lexIntegerToken("foo 12a 3", 4) => "12" <-- Ignoring this situation 117 | func (lc lexingContext) lexIntegerToken(cursor int) (int, *token) { 118 | originalCursor := cursor 119 | 120 | var value []rune 121 | for cursor < len(lc.source) { 122 | r := lc.source[cursor] 123 | if r >= '0' && r <= '9' { 124 | value = append(value, r) 125 | cursor++ 126 | continue 127 | } 128 | 129 | break 130 | } 131 | 132 | if len(value) == 0 { 133 | return originalCursor, nil 134 | } 135 | 136 | return cursor, &token{ 137 | value: string(value), 138 | kind: integerToken, 139 | location: originalCursor, 140 | lc: lc, 141 | } 142 | } 143 | 144 | // lexIdentifierToken("123 ab + ", 4) => "ab" 145 | // lexIdentifierToken("123 ab123 + ", 4) => "ab123" 146 | func (lc lexingContext) lexIdentifierToken(cursor int) (int, *token) { 147 | originalCursor := cursor 148 | var value []rune 149 | 150 | for cursor < len(lc.source) { 151 | r := lc.source[cursor] 152 | if !(unicode.IsSpace(r) || r == ')') { 153 | value = append(value, r) 154 | cursor++ 155 | continue 156 | } 157 | 158 | break 159 | } 160 | 161 | if len(value) == 0 { 162 | return originalCursor, nil 163 | } 164 | 165 | return cursor, &token{ 166 | value: string(value), 167 | kind: identifierToken, 168 | location: originalCursor, 169 | lc: lc, 170 | } 171 | } 172 | 173 | // for example: "(+ 13 2)" 174 | // lex(" ( + 13 2 )") should produce: ["(", "+", "13", "2", ")"] 175 | func (lc lexingContext) lex() []token { 176 | var tokens []token 177 | var t *token 178 | 179 | cursor := 0 180 | for cursor < len(lc.source) { 181 | // eat whitespace 182 | cursor = eatWhitespace(lc.source, cursor) 183 | if cursor == len(lc.source) { 184 | break 185 | } 186 | 187 | cursor, t = lc.lexSyntaxToken(cursor) 188 | if t != nil { 189 | tokens = append(tokens, *t) 190 | continue 191 | } 192 | 193 | cursor, t = lc.lexIntegerToken(cursor) 194 | if t != nil { 195 | tokens = append(tokens, *t) 196 | continue 197 | } 198 | 199 | cursor, t = lc.lexIdentifierToken(cursor) 200 | if t != nil { 201 | tokens = append(tokens, *t) 202 | continue 203 | } 204 | 205 | // Lexed nothing, not good! 206 | // fmt.Println(tokens[len(tokens)-1].debug()) // line of code 207 | panic("Could not lex") 208 | } 209 | 210 | return tokens 211 | } 212 | 213 | func newLexingContext(file string) lexingContext { 214 | program, err := os.ReadFile(os.Args[1]) 215 | if err != nil { 216 | panic(err) 217 | } 218 | 219 | return lexingContext{ 220 | sourceFileName: file, 221 | source: []rune(string(program)), 222 | } 223 | } 224 | --------------------------------------------------------------------------------