├── testdata ├── .gitignore ├── 4.proto ├── 1.proto ├── 3.proto ├── imp.proto ├── 2.proto ├── imp2.proto ├── 4.expected ├── 1.expected ├── 3.expected ├── 2.expected ├── mini.proto ├── run.sh └── protocmp.go ├── .gitignore ├── README.md ├── Makefile ├── LICENSE ├── main.go ├── parser ├── resolver.go ├── parser_test.go └── parser.go ├── ast └── ast.go └── gendesc └── gendesc.go /testdata/.gitignore: -------------------------------------------------------------------------------- 1 | *.baseline 2 | _baseline.raw 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | .DS_Store 3 | _mini* 4 | _testmain.go 5 | gotoc 6 | testdata/*.actual 7 | testdata/mini*.pb.go 8 | testdata/protocmp 9 | -------------------------------------------------------------------------------- /testdata/4.proto: -------------------------------------------------------------------------------- 1 | // Group. 2 | 3 | syntax = "proto2"; 4 | 5 | message A { 6 | optional group B = 1 { 7 | required string x = 2; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /testdata/1.proto: -------------------------------------------------------------------------------- 1 | // Just the basics 2 | 3 | syntax = "proto2"; 4 | 5 | message A { 6 | required int32 a = 1; 7 | optional string b = 2; 8 | // skip field 3 9 | repeated sint64 c = 4; 10 | } 11 | -------------------------------------------------------------------------------- /testdata/3.proto: -------------------------------------------------------------------------------- 1 | // Default values. 2 | 3 | syntax = "proto2"; 4 | 5 | message A { 6 | optional int32 int = 1 [default=7]; 7 | optional bool boolean = 2 [default=true]; 8 | optional string str = 3 [default="boo"]; 9 | } 10 | -------------------------------------------------------------------------------- /testdata/imp.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | // This is an imported proto. 4 | 5 | message Foreign { 6 | optional string country = 1; 7 | 8 | enum Currency { 9 | AUD = 1; 10 | USD = 2; 11 | GBP = 3; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /testdata/2.proto: -------------------------------------------------------------------------------- 1 | // Nested message, plus some message references. 2 | 3 | syntax = "proto2"; 4 | 5 | message A { 6 | message B { 7 | } 8 | optional B b = 1; 9 | } 10 | 11 | message C { 12 | optional A a = 1; 13 | optional A.B b = 2; 14 | } 15 | -------------------------------------------------------------------------------- /testdata/imp2.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | // This is an imported proto, but this time it has a package. 4 | package imp2; 5 | 6 | import "testdata/imp.proto"; 7 | 8 | message Document { 9 | optional fixed64 docid = 1; 10 | 11 | optional Foreign.Currency currency = 2; 12 | } 13 | -------------------------------------------------------------------------------- /testdata/4.expected: -------------------------------------------------------------------------------- 1 | file: < 2 | name: "4.proto" 3 | message_type: < 4 | name: "A" 5 | field: < 6 | name: "b" 7 | number: 1 8 | label: LABEL_OPTIONAL 9 | type: TYPE_GROUP 10 | type_name: ".A.B" 11 | > 12 | nested_type: < 13 | name: "B" 14 | field { 15 | name: "x" 16 | number: 2 17 | label: LABEL_REQUIRED 18 | type: TYPE_STRING 19 | } 20 | > 21 | > 22 | > 23 | -------------------------------------------------------------------------------- /testdata/1.expected: -------------------------------------------------------------------------------- 1 | file: < 2 | name: "1.proto" 3 | message_type: < 4 | name: "A" 5 | field: < 6 | name: "a" 7 | number: 1 8 | label: LABEL_REQUIRED 9 | type: TYPE_INT32 10 | > 11 | field: < 12 | name: "b" 13 | number: 2 14 | label: LABEL_OPTIONAL 15 | type: TYPE_STRING 16 | > 17 | field: < 18 | name: "c" 19 | number: 4 20 | label: LABEL_REPEATED 21 | type: TYPE_SINT64 22 | > 23 | > 24 | > 25 | -------------------------------------------------------------------------------- /testdata/3.expected: -------------------------------------------------------------------------------- 1 | file: < 2 | name: "3.proto" 3 | message_type: < 4 | name: "A" 5 | field: < 6 | name: "int" 7 | number: 1 8 | label: LABEL_OPTIONAL 9 | type: TYPE_INT32 10 | default_value: "7" 11 | > 12 | field: < 13 | name: "boolean" 14 | number: 2 15 | label: LABEL_OPTIONAL 16 | type: TYPE_BOOL 17 | default_value: "true" 18 | > 19 | field: < 20 | name: "str" 21 | number: 3 22 | label: LABEL_OPTIONAL 23 | type: TYPE_STRING 24 | default_value: "boo" 25 | > 26 | > 27 | > 28 | -------------------------------------------------------------------------------- /testdata/2.expected: -------------------------------------------------------------------------------- 1 | file: < 2 | name: "2.proto" 3 | message_type: < 4 | name: "A" 5 | field: < 6 | name: "b" 7 | number: 1 8 | label: LABEL_OPTIONAL 9 | type: TYPE_MESSAGE 10 | type_name: ".A.B" 11 | > 12 | nested_type: < 13 | name: "B" 14 | > 15 | > 16 | message_type: < 17 | name: "C" 18 | field: < 19 | name: "a" 20 | number: 1 21 | label: LABEL_OPTIONAL 22 | type: TYPE_MESSAGE 23 | type_name: ".A" 24 | > 25 | field: < 26 | name: "b" 27 | number: 2 28 | label: LABEL_OPTIONAL 29 | type: TYPE_MESSAGE 30 | type_name: ".A.B" 31 | > 32 | > 33 | > 34 | -------------------------------------------------------------------------------- /testdata/mini.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | // This is a test package. 4 | // As you can see, comments will be ignored. 5 | package foo.bar.baz; 6 | 7 | import "testdata/imp.proto"; 8 | import "testdata/imp2.proto"; 9 | 10 | message Bar { 11 | required int32 height = 1; 12 | optional string name = 2; 13 | optional Other other = 3; 14 | 15 | optional group Groupie = 7 { 16 | required string x = 1; 17 | } 18 | 19 | message Nested {} 20 | optional Nested nested = 4; 21 | 22 | optional Foreign alien = 5; 23 | optional imp2.Document doc = 6; 24 | 25 | map a_map_field = 8; 26 | } 27 | 28 | message Other { 29 | required int64 count = 1; 30 | } 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | gotoc 2 | ===== 3 | This is gotoc, a protocol buffer compiler written in Go. 4 | 5 | This is **only** the parser side; you will need a plugin to generate code. 6 | 7 | Quick Start 8 | ----------- 9 | ```shell 10 | go get github.com/dsymonds/gotoc 11 | go get github.com/golang/protobuf/protoc-gen-go 12 | gotoc foo.proto 13 | ``` 14 | 15 | To enable [gRPC](http://www.grpc.io/) output, use the `-params` flag: 16 | ```shell 17 | gotoc -params plugins=grpc foo.proto 18 | ``` 19 | 20 | License 21 | ------- 22 | This is licensed under the [BSD 3-Clause Licence](http://opensource.org/licenses/BSD-3-Clause). 23 | See the LICENSE file for more details. 24 | 25 | Read more 26 | --------- 27 | * [The protocol buffers open source project](https://developers.google.com/protocol-buffers/) 28 | * [The Go protocol buffer code generator plugin and support library](https://github.com/golang/protobuf) 29 | -------------------------------------------------------------------------------- /testdata/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | cd $(dirname $0) 4 | 5 | MAX=100 6 | GOTOC=../gotoc 7 | PROTOCMP=./protocmp 8 | PROTOBUF=$HOME/src/protobuf 9 | 10 | go build -o protocmp protocmp.go 11 | 12 | failures=0 13 | for ((i=1; $i <= $MAX; i=$((i+1)))); do 14 | if [ ! -f $i.proto ]; then continue; fi 15 | echo "---[ Test $i ]---" 1>&2 16 | cat $i.proto | sed 's,^, ,' 1>&2 17 | 18 | $GOTOC --descriptor_only $i.proto > $i.actual 19 | $PROTOCMP $i.expected $i.actual || { 20 | echo "==> FAILED expected vs. actual" 1>&2 21 | failures=$(($failures + 1)) 22 | } 23 | 24 | protoc --descriptor_set_out=_baseline.raw $i.proto 25 | protoc --decode=google.protobuf.FileDescriptorSet -I $PROTOBUF $PROTOBUF/src/google/protobuf/descriptor.proto \ 26 | < _baseline.raw > $i.baseline 27 | $PROTOCMP $i.expected $i.baseline || { 28 | echo "==> BASELINE MISMATCH" 1>&2 29 | failures=$(($failures + 1)) 30 | } 31 | done 32 | 33 | echo "----------" 1>&2 34 | if [ $failures -eq 0 ]; then 35 | echo "All OK" 1>&2 36 | else 37 | echo "$failures test failure(s)" 1>&2 38 | fi 39 | echo "----------" 1>&2 40 | exit $failures 41 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | minitest: 2 | go build 3 | ./gotoc testdata/mini.proto 4 | 5 | minidiff: 6 | go build 7 | ./gotoc testdata/mini.proto 8 | mv testdata/mini.pb.go testdata/mini-gotoc.pb.go 9 | protoc --go_out=. testdata/mini.proto 10 | mv testdata/mini.pb.go testdata/mini-protoc.pb.go 11 | sed -i '' '/^var fileDescriptor/,/^}$$/d' testdata/mini-{gotoc,protoc}.pb.go 12 | diff -ud testdata/mini-{gotoc,protoc}.pb.go || true 13 | 14 | regtest: 15 | go build 16 | testdata/run.sh 17 | 18 | PROTOBUF=$(HOME)/src/protobuf 19 | MINI_TMP=_mini.pb 20 | baseline: 21 | @protoc --descriptor_set_out=$(MINI_TMP) --include_imports testdata/mini.proto 22 | @protoc --decode=google.protobuf.FileDescriptorSet -I $(PROTOBUF) $(PROTOBUF)/src/google/protobuf/descriptor.proto < $(MINI_TMP) 23 | @rm $(MINI_TMP) 24 | 25 | MINI_FSET_GOTOC=_mini_fset_gotoc.txt 26 | MINI_FSET_PROTOC=_mini_fset_protoc.txt 27 | baselinediff: 28 | go build 29 | # First, gotoc 30 | @./gotoc --descriptor_only testdata/mini.proto > $(MINI_FSET_GOTOC) 31 | @sed -i '' 's/: $$/}/g' $(MINI_FSET_GOTOC) 33 | # Next, protoc 34 | @protoc --descriptor_set_out=$(MINI_TMP) --include_imports testdata/mini.proto 35 | @protoc --decode=google.protobuf.FileDescriptorSet -I $(PROTOBUF) $(PROTOBUF)/src/google/protobuf/descriptor.proto < $(MINI_TMP) > $(MINI_FSET_PROTOC) 36 | # Compare 37 | diff -ud $(MINI_FSET_PROTOC) $(MINI_FSET_GOTOC) 38 | @rm $(MINI_TMP) $(MINI_FSET_GOTOC) $(MINI_FSET_PROTOC) 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | gotoc - Go protocol buffer compiler 2 | https://github.com/dsymonds/gotoc 3 | 4 | Copyright (c) 2010, David Symonds. 5 | All rights reserved. 6 | 7 | Redistribution and use in source and binary forms, with or without modification, 8 | are permitted provided that the following conditions are met: 9 | 10 | * Redistributions of source code must retain the above copyright notice, 11 | this list of conditions and the following disclaimer. 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | * Neither the name of the author nor the names of its contributors may be used 16 | to endorse or promote products derived from this software 17 | without specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 20 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 23 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 24 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 26 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 28 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | gotoc is a protocol buffer compiler. It reads and parses .proto files, 3 | and produces output that can be consumed by a protoc-compatible plugin 4 | (such as protoc-gen-go) to produce generated code. 5 | */ 6 | package main 7 | 8 | import ( 9 | "bytes" 10 | "flag" 11 | "fmt" 12 | "io/ioutil" 13 | "os" 14 | "os/exec" 15 | "path" 16 | "strings" 17 | 18 | "github.com/golang/protobuf/proto" 19 | plugin "github.com/golang/protobuf/protoc-gen-go/plugin" 20 | 21 | "github.com/dsymonds/gotoc/gendesc" 22 | "github.com/dsymonds/gotoc/parser" 23 | ) 24 | 25 | var ( 26 | // Flags 27 | helpShort = flag.Bool("h", false, "Show usage text (same as --help).") 28 | helpLong = flag.Bool("help", false, "Show usage text (same as -h).") 29 | 30 | importPath = flag.String("import_path", ".", "Comma-separated list of paths to search for imports.") 31 | pluginBinary = flag.String("plugin", "protoc-gen-go", "The code generator plugin to use.") 32 | descriptorOnly = flag.Bool("descriptor_only", false, "Whether to print out only the FileDescriptorSet.") 33 | params = flag.String("params", "", "Parameters to pass to the code generator plugin (plugin-specific format).") 34 | ) 35 | 36 | func fullPath(binary string, paths []string) string { 37 | if strings.Index(binary, "/") >= 0 { 38 | // path with path component 39 | return binary 40 | } 41 | for _, p := range paths { 42 | full := path.Join(p, binary) 43 | fi, err := os.Stat(full) 44 | if err == nil && !fi.IsDir() { 45 | return full 46 | } 47 | } 48 | return "" 49 | } 50 | 51 | func main() { 52 | flag.Usage = usage 53 | flag.Parse() 54 | if *helpShort || *helpLong || flag.NArg() == 0 { 55 | flag.Usage() 56 | os.Exit(1) 57 | } 58 | 59 | fs, err := parser.ParseFiles(flag.Args(), strings.Split(*importPath, ",")) 60 | if err != nil { 61 | fatalf("%v", err) 62 | } 63 | fds, err := gendesc.Generate(fs) 64 | if err != nil { 65 | fatalf("Failed generating descriptors: %v", err) 66 | } 67 | 68 | if *descriptorOnly { 69 | proto.MarshalText(os.Stdout, fds) 70 | os.Exit(0) 71 | } 72 | 73 | //fmt.Println("-----") 74 | //proto.MarshalText(os.Stdout, fds) 75 | //fmt.Println("-----") 76 | 77 | // Prepare request. 78 | cgRequest := &plugin.CodeGeneratorRequest{ 79 | FileToGenerate: flag.Args(), 80 | ProtoFile: fds.File, 81 | } 82 | if *params != "" { 83 | cgRequest.Parameter = params 84 | } 85 | buf, err := proto.Marshal(cgRequest) 86 | if err != nil { 87 | fatalf("Failed marshaling CG request: %v", err) 88 | } 89 | 90 | // Find plugin. 91 | pluginPath := fullPath(*pluginBinary, strings.Split(os.Getenv("PATH"), ":")) 92 | if pluginPath == "" { 93 | fatalf("Failed finding plugin binary %q", *pluginBinary) 94 | } 95 | 96 | // Run the plugin subprocess. 97 | cmd := &exec.Cmd{ 98 | Path: pluginPath, 99 | Stdin: bytes.NewBuffer(buf), 100 | Stderr: os.Stderr, 101 | } 102 | buf, err = cmd.Output() 103 | if err != nil { 104 | fatalf("Failed running plugin: %v", err) 105 | } 106 | 107 | // Parse the response. 108 | cgResponse := new(plugin.CodeGeneratorResponse) 109 | if err = proto.Unmarshal(buf, cgResponse); err != nil { 110 | fatalf("Failed unmarshaling CG response: %v", err) 111 | } 112 | 113 | // TODO: check cgResponse.Error 114 | 115 | for _, f := range cgResponse.File { 116 | // TODO: If f.Name is nil, the content should be appended to the previous file. 117 | if f.Name == nil || f.Content == nil { 118 | fatalf("Malformed CG response") 119 | } 120 | if err := ioutil.WriteFile(*f.Name, []byte(*f.Content), 0644); err != nil { 121 | fatalf("Failed writing output file: %v", err) 122 | } 123 | } 124 | } 125 | 126 | func usage() { 127 | fmt.Fprintf(os.Stderr, "Usage: %s [options] ...\n", os.Args[0]) 128 | flag.PrintDefaults() 129 | } 130 | 131 | func fatalf(format string, args ...interface{}) { 132 | fmt.Fprintf(os.Stderr, format+"\n", args...) 133 | os.Exit(1) 134 | } 135 | -------------------------------------------------------------------------------- /testdata/protocmp.go: -------------------------------------------------------------------------------- 1 | // A small tool to compare two text-format FileDescriptorSet protocol buffers. 2 | 3 | package main 4 | 5 | import ( 6 | "flag" 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | 11 | "github.com/golang/protobuf/proto" 12 | . "github.com/golang/protobuf/protoc-gen-go/descriptor" 13 | ) 14 | 15 | func main() { 16 | flag.Parse() 17 | if flag.NArg() != 2 { 18 | log.Fatalf("usage: %v ", os.Args[0]) 19 | } 20 | 21 | a, b := mustLoad(flag.Arg(0)), mustLoad(flag.Arg(1)) 22 | cmpSets(a, b) 23 | } 24 | 25 | func mustLoad(filename string) *FileDescriptorSet { 26 | buf, err := ioutil.ReadFile(filename) 27 | if err != nil { 28 | log.Fatalf("Failed reading %v: %v", filename, err) 29 | } 30 | fds := new(FileDescriptorSet) 31 | if err := proto.UnmarshalText(string(buf), fds); err != nil { 32 | log.Fatalf("Failed parsing %v: %v", filename, err) 33 | } 34 | return fds 35 | } 36 | 37 | func cmpSets(a, b *FileDescriptorSet) { 38 | // Index each set by filename. 39 | indexA, indexB := make(map[string]int), make(map[string]int) 40 | for i, fd := range a.File { 41 | indexA[*fd.Name] = i 42 | } 43 | for i, fd := range b.File { 44 | indexB[*fd.Name] = i 45 | } 46 | 47 | // Check that the filename sets match. 48 | match := true 49 | if len(indexA) != len(indexB) { 50 | match = false 51 | } 52 | for filename, _ := range indexA { 53 | if _, ok := indexB[filename]; !ok { 54 | match = false 55 | break 56 | } 57 | } 58 | for filename, _ := range indexB { 59 | if _, ok := indexA[filename]; !ok { 60 | match = false 61 | break 62 | } 63 | } 64 | if !match { 65 | log.Printf("Sets of filenames do not match.") 66 | log.Printf("A: %+v", indexA) 67 | log.Printf("B: %+v", indexB) 68 | os.Exit(1) 69 | } 70 | 71 | // TODO: could also verify that the file ordering is topological? 72 | 73 | for _, fdA := range a.File { 74 | fdB := b.File[indexB[*fdA.Name]] 75 | cmpFiles(fdA, fdB) 76 | } 77 | } 78 | 79 | func cmpFiles(a, b *FileDescriptorProto) { 80 | if ap, bp := a.GetPackage(), b.GetPackage(); ap != bp { 81 | log.Fatalf("Package name mismatch in %v: %q vs. %q", *a.Name, ap, bp) 82 | } 83 | 84 | match := true 85 | if len(a.Dependency) != len(b.Dependency) { 86 | match = false 87 | } else { 88 | for i, depA := range a.Dependency { 89 | if depA != b.Dependency[i] { 90 | match = false 91 | break 92 | } 93 | } 94 | } 95 | if !match { 96 | log.Fatalf("Different dependency list in %v", *a.Name) 97 | } 98 | 99 | // TODO: this should be order-independent. 100 | if len(a.MessageType) != len(b.MessageType) { 101 | log.Fatalf("Different number of messages in %v", *a.Name) 102 | } 103 | for i, msgA := range a.MessageType { 104 | cmpMessages(msgA, b.MessageType[i]) 105 | } 106 | 107 | // TODO: enum_type 108 | } 109 | 110 | func cmpMessages(a, b *DescriptorProto) { 111 | // TODO: this check shouldn't be necessary from here. 112 | if *a.Name != *b.Name { 113 | log.Fatalf("Different message names: %q vs. %q", *a.Name, *b.Name) 114 | } 115 | 116 | // TODO: this should be order-independent. 117 | if len(a.Field) != len(b.Field) { 118 | log.Fatalf("Different number of fields in message %v: %d vs. %d", *a.Name, len(a.Field), len(b.Field)) 119 | } 120 | for i, fA := range a.Field { 121 | cmpFields(fA, b.Field[i]) 122 | } 123 | 124 | // TODO: this should be order-independent too. 125 | if len(a.NestedType) != len(b.NestedType) { 126 | log.Fatalf("Different number of nested messages in message %v: %d vs. %d", 127 | *a.Name, len(a.NestedType), len(b.NestedType)) 128 | } 129 | for i, msgA := range a.NestedType { 130 | cmpMessages(msgA, b.NestedType[i]) 131 | } 132 | 133 | // TODO: nested_type, enum_type 134 | } 135 | 136 | func cmpFields(a, b *FieldDescriptorProto) { 137 | // TODO: this check shouldn't be necessary from here. 138 | if *a.Name != *b.Name { 139 | log.Fatalf("Different field names: %q vs. %q", *a.Name, *b.Name) 140 | } 141 | if *a.Number != *b.Number { 142 | log.Fatalf("Different field number for %v: %d vs. %d", *a.Name, *a.Number, *b.Number) 143 | } 144 | if *a.Label != *b.Label { 145 | log.Fatalf("Different field labels for %v: %v vs. %v", *a.Name, 146 | FieldDescriptorProto_Label_name[int32(*a.Label)], 147 | FieldDescriptorProto_Label_name[int32(*b.Label)]) 148 | } 149 | if *a.Type != *b.Type { 150 | log.Fatalf("Different field types for %v: %v vs. %v", *a.Name, 151 | FieldDescriptorProto_Type_name[int32(*a.Type)], 152 | FieldDescriptorProto_Type_name[int32(*b.Type)]) 153 | } 154 | if aTN, bTN := a.GetTypeName(), b.GetTypeName(); aTN != bTN { 155 | log.Fatalf("Different field type_name for %v: %q vs. %q", *a.Name, aTN, bTN) 156 | } 157 | if ad, bd := a.GetDefaultValue(), b.GetDefaultValue(); ad != bd { 158 | log.Fatalf("Different field default_value for %v: %q vs. %q", *a.Name, ad, bd) 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /parser/resolver.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | // This file implements the symbol resolution stage of parsing. 4 | // TODO: make this more efficient if needed. 5 | 6 | import ( 7 | "fmt" 8 | "strings" 9 | 10 | "github.com/dsymonds/gotoc/ast" 11 | ) 12 | 13 | func resolveSymbols(fset *ast.FileSet) error { 14 | r := &resolver{fset: fset} 15 | s := new(scope) 16 | s.push(fset) 17 | for _, f := range fset.Files { 18 | if err := r.resolveFile(s, f); err != nil { 19 | return err 20 | } 21 | } 22 | return nil 23 | } 24 | 25 | // A scope represents the context of the traversal. 26 | type scope struct { 27 | // Valid types: FileSet, File, Message, Enum 28 | objects []interface{} 29 | } 30 | 31 | func (s *scope) global() bool { return len(s.objects) == 0 } 32 | func (s *scope) push(o interface{}) { s.objects = append(s.objects, o) } 33 | func (s *scope) pop() { s.objects = s.objects[:len(s.objects)-1] } 34 | 35 | func (s *scope) dup() *scope { 36 | sc := &scope{ 37 | objects: make([]interface{}, len(s.objects)), 38 | } 39 | copy(sc.objects, s.objects) 40 | return sc 41 | } 42 | 43 | func (s *scope) last() interface{} { 44 | if s.global() { 45 | return nil 46 | } 47 | return s.objects[len(s.objects)-1] 48 | } 49 | 50 | // findName attemps to find the given name in the scope. 51 | // Only immediate names are found; it does not recurse. 52 | func (s *scope) findName(name string) []interface{} { 53 | o := s.last() 54 | if o == nil { 55 | return nil 56 | } 57 | switch ov := o.(type) { 58 | case *ast.FileSet: 59 | ret := []interface{}{} 60 | for _, f := range ov.Files { 61 | if len(f.Package) == 0 { 62 | // No package; match on message/enum names 63 | fs := s.dup() 64 | fs.push(f) 65 | ret = append(ret, fs.findName(name)...) 66 | } else { 67 | // Match on package name 68 | // TODO: fix this for dotted package names 69 | if f.Package[0] == name { 70 | return []interface{}{f} 71 | } 72 | } 73 | } 74 | return ret 75 | case *ast.File: 76 | for _, msg := range ov.Messages { 77 | if msg.Name == name { 78 | return []interface{}{msg} 79 | } 80 | } 81 | for _, enum := range ov.Enums { 82 | if enum.Name == name { 83 | return []interface{}{enum} 84 | } 85 | } 86 | case *ast.Message: 87 | for _, msg := range ov.Messages { 88 | if msg.Name == name { 89 | return []interface{}{msg} 90 | } 91 | } 92 | for _, enum := range ov.Enums { 93 | if enum.Name == name { 94 | return []interface{}{enum} 95 | } 96 | } 97 | // can't be *EnumDescriptorProto 98 | } 99 | return nil 100 | } 101 | 102 | func (s *scope) fullName() string { 103 | n := make([]string, 0, len(s.objects)) 104 | for _, o := range s.objects { 105 | switch ov := o.(type) { 106 | case *ast.File: 107 | n = append(n, ov.Package...) 108 | case *ast.Message: 109 | n = append(n, ov.Name) 110 | case *ast.Enum: 111 | n = append(n, ov.Name) 112 | } 113 | } 114 | return "." + strings.Join(n, ".") 115 | } 116 | 117 | type resolver struct { 118 | fset *ast.FileSet 119 | } 120 | 121 | func (r *resolver) resolveFile(s *scope, f *ast.File) error { 122 | fs := s.dup() 123 | fs.push(f) 124 | 125 | // Resolve messages. 126 | for _, msg := range f.Messages { 127 | if err := r.resolveMessage(fs, msg); err != nil { 128 | return fmt.Errorf("(%v): %v", msg.Name, err) 129 | } 130 | } 131 | // Resolve messages in services. 132 | for _, srv := range f.Services { 133 | for _, mth := range srv.Methods { 134 | if err := r.resolveMethod(fs, mth); err != nil { 135 | return fmt.Errorf("(%s.%s): %v", srv.Name, mth.Name, err) 136 | } 137 | } 138 | } 139 | // Resolve types in extensions. 140 | for _, ext := range f.Extensions { 141 | if err := r.resolveExtension(fs, ext); err != nil { 142 | return fmt.Errorf("(ext %s): %v", ext.Extendee, err) 143 | } 144 | } 145 | 146 | // TODO: resolve other types. 147 | 148 | return nil 149 | } 150 | 151 | var fieldTypeInverseMap = make(map[string]ast.FieldType) 152 | 153 | func init() { 154 | for ft, name := range ast.FieldTypeMap { 155 | fieldTypeInverseMap[name] = ft 156 | } 157 | } 158 | 159 | var validMapKeyTypes = map[string]bool{ 160 | "int64": true, 161 | "uint64": true, 162 | "int32": true, 163 | "fixed64": true, 164 | "fixed32": true, 165 | "bool": true, 166 | "string": true, 167 | "uint32": true, 168 | "sfixed32": true, 169 | "sfixed64": true, 170 | "sint32": true, 171 | "sint64": true, 172 | } 173 | 174 | func (r *resolver) resolveMessage(s *scope, msg *ast.Message) error { 175 | ms := s.dup() 176 | ms.push(msg) 177 | 178 | // Resolve fields. 179 | for _, field := range msg.Fields { 180 | ft, ok := r.resolveFieldTypeName(ms, field.TypeName) 181 | if !ok { 182 | return fmt.Errorf("failed to resolve name %q", field.TypeName) 183 | } 184 | field.Type = ft 185 | 186 | if ktn := field.KeyTypeName; ktn != "" { 187 | if !validMapKeyTypes[ktn] { 188 | return fmt.Errorf("invalid map key type %q", ktn) 189 | } 190 | field.KeyType = fieldTypeInverseMap[ktn] 191 | } 192 | } 193 | // Resolve types in extensions. 194 | for _, ext := range msg.Extensions { 195 | if err := r.resolveExtension(ms, ext); err != nil { 196 | return err 197 | } 198 | } 199 | // Resolve nested types. 200 | for _, nmsg := range msg.Messages { 201 | if err := r.resolveMessage(ms, nmsg); err != nil { 202 | return err 203 | } 204 | } 205 | return nil 206 | } 207 | 208 | func (r *resolver) resolveFieldTypeName(s *scope, name string) (interface{}, bool) { 209 | if ft, ok := fieldTypeInverseMap[name]; ok { 210 | // field is a primitive type 211 | return ft, true 212 | } 213 | // field must be a named type, message or enum 214 | o := r.resolveName(s, name) 215 | if o != nil { 216 | //log.Printf("(resolved %q to %q)", name, o.fullName()) 217 | return o.last(), true 218 | } 219 | return nil, false 220 | } 221 | 222 | func (r *resolver) resolveMethod(s *scope, mth *ast.Method) error { 223 | o := r.resolveName(s, mth.InTypeName) 224 | if o == nil { 225 | return fmt.Errorf("failed to resolve name %q", mth.InTypeName) 226 | } 227 | mth.InType = o.last() 228 | 229 | o = r.resolveName(s, mth.OutTypeName) 230 | if o == nil { 231 | return fmt.Errorf("failed to resolve name %q", mth.OutTypeName) 232 | } 233 | mth.OutType = o.last() 234 | 235 | return nil 236 | } 237 | 238 | func (r *resolver) resolveExtension(s *scope, ext *ast.Extension) error { 239 | o := r.resolveName(s, ext.Extendee) 240 | if o == nil { 241 | return fmt.Errorf("failed to resolve name %q", ext.Extendee) 242 | } 243 | m, ok := o.last().(*ast.Message) 244 | if !ok { 245 | return fmt.Errorf("extendee %q resolved to non-message %T", ext.Extendee, o.last()) 246 | } 247 | ext.ExtendeeType = m 248 | // Resolve fields. 249 | for _, field := range ext.Fields { 250 | ft, ok := r.resolveFieldTypeName(s, field.TypeName) 251 | if !ok { 252 | return fmt.Errorf("failed to resolve name %q", field.TypeName) 253 | } 254 | field.Type = ft 255 | 256 | // TODO: Map fields should be forbidden? 257 | } 258 | return nil 259 | } 260 | 261 | func (r *resolver) resolveName(s *scope, name string) *scope { 262 | parts := strings.Split(name, ".") 263 | 264 | // Move up the scope, finding a place where the name makes sense. 265 | for ws := s.dup(); !ws.global(); ws.pop() { 266 | //log.Printf("Trying to resolve %q in %q", name, ws.fullName()) 267 | if os := matchNameComponents(ws, parts); os != nil { 268 | return os 269 | } 270 | } 271 | 272 | return nil // failed 273 | } 274 | 275 | func matchNameComponents(s *scope, parts []string) *scope { 276 | first, rem := parts[0], parts[1:] 277 | for _, o := range s.findName(first) { 278 | os := s.dup() 279 | os.push(o) 280 | if len(rem) == 0 { 281 | return os 282 | } 283 | // TODO: catch ambiguous names here 284 | if is := matchNameComponents(os, rem); is != nil { 285 | return is 286 | } 287 | } 288 | return nil 289 | } 290 | -------------------------------------------------------------------------------- /ast/ast.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package ast defines the AST data structures used by gotoc. 3 | */ 4 | package ast 5 | 6 | import ( 7 | "fmt" 8 | "log" 9 | "sort" 10 | ) 11 | 12 | // Node is implemented by concrete types that represent things appearing in a proto file. 13 | type Node interface { 14 | Pos() Position 15 | File() *File 16 | } 17 | 18 | // FileSet describes a set of proto files. 19 | type FileSet struct { 20 | // Files is sorted in topological order, bottom up. 21 | // That means that a file X will only import a file Y 22 | // if Y occurs in this slice before X. 23 | Files []*File 24 | } 25 | 26 | // Sort sorts fs.Files topologically. 27 | func (fs *FileSet) Sort() { 28 | in := fs.Files // old version of fs.Files; shrinks each loop 29 | out := make([]*File, 0, len(in)) // new version of fs.Files; grows each loop 30 | done := make(map[string]bool) // filenames that we've seen and that don't have un-done imports 31 | for len(in) > 0 { 32 | // Find a file that doesn't have an un-done import. 33 | var next *File 34 | for i, f := range in { 35 | ok := true 36 | for _, imp := range f.Imports { 37 | if !done[imp] { 38 | ok = false 39 | break 40 | } 41 | } 42 | if !ok { 43 | continue 44 | } 45 | next = f 46 | copy(in[i:], in[i+1:]) 47 | in = in[:len(in)-1] 48 | break 49 | } 50 | if next == nil { 51 | panic("import loop!") // shouldn't happen 52 | } 53 | out = append(out, next) 54 | done[next.Name] = true 55 | } 56 | fs.Files = out 57 | } 58 | 59 | // File represents a single proto file. 60 | type File struct { 61 | Name string // filename 62 | Syntax string // "proto2" or "proto3" 63 | Package []string 64 | Options [][2]string // slice of key/value pairs 65 | 66 | Imports []string 67 | PublicImports []int // list of indexes in the Imports slice 68 | 69 | Messages []*Message // top-level messages 70 | Enums []*Enum // top-level enums 71 | Services []*Service // services 72 | Extensions []*Extension // top-level extensions 73 | 74 | Comments []*Comment // all the comments for this file, sorted by position 75 | } 76 | 77 | // Message represents a proto message. 78 | type Message struct { 79 | Position Position // position of the "message" token 80 | Name string 81 | Group bool 82 | Fields []*Field 83 | Extensions []*Extension 84 | Oneofs []*Oneof 85 | 86 | Messages []*Message // includes groups 87 | Enums []*Enum 88 | 89 | ExtensionRanges [][2]int // extension ranges (inclusive at both ends) 90 | 91 | Up interface{} // either *File or *Message 92 | } 93 | 94 | func (m *Message) Pos() Position { return m.Position } 95 | func (m *Message) File() *File { 96 | for x := m.Up; ; { 97 | switch up := x.(type) { 98 | case *File: 99 | return up 100 | case *Message: 101 | x = up.Up 102 | default: 103 | log.Panicf("internal error: Message.Up is a %T", up) 104 | } 105 | } 106 | } 107 | 108 | // Oneof represents a oneof bracketing a set of fields in a message. 109 | type Oneof struct { 110 | Position Position // position of "oneof" token 111 | Name string 112 | 113 | Up *Message 114 | } 115 | 116 | // Field represents a field in a message. 117 | type Field struct { 118 | Position Position // position of "required"/"optional"/"repeated"/type 119 | 120 | // TypeName is the raw name parsed from the input. 121 | // Type is set during resolution; it will be a FieldType, *Message or *Enum. 122 | TypeName string 123 | Type interface{} 124 | 125 | // For a map field, the TypeName/Type fields are the value type, 126 | // and KeyTypeName/KeyType will be set. 127 | KeyTypeName string 128 | KeyType FieldType 129 | 130 | // At most one of {required,repeated} is set. 131 | Required bool 132 | Repeated bool 133 | Name string 134 | Tag int 135 | 136 | HasDefault bool 137 | Default string // e.g. "foo", 7, true 138 | 139 | HasPacked bool 140 | Packed bool 141 | 142 | Oneof *Oneof 143 | 144 | Up Node // either *Message or *Extension 145 | } 146 | 147 | func (f *Field) Pos() Position { return f.Position } 148 | func (f *Field) File() *File { return f.Up.File() } 149 | 150 | type FieldType int8 151 | 152 | const ( 153 | min FieldType = iota 154 | Double 155 | Float 156 | Int64 157 | Uint64 158 | Int32 159 | Fixed64 160 | Fixed32 161 | Bool 162 | String 163 | Bytes 164 | Uint32 165 | Sfixed32 166 | Sfixed64 167 | Sint32 168 | Sint64 169 | max 170 | ) 171 | 172 | func (ft FieldType) IsValid() bool { return min < ft && ft < max } 173 | 174 | var FieldTypeMap = map[FieldType]string{ 175 | Double: "double", 176 | Float: "float", 177 | Int64: "int64", 178 | Uint64: "uint64", 179 | Int32: "int32", 180 | Fixed64: "fixed64", 181 | Fixed32: "fixed32", 182 | Bool: "bool", 183 | String: "string", 184 | Bytes: "bytes", 185 | Uint32: "uint32", 186 | Sfixed32: "sfixed32", 187 | Sfixed64: "sfixed64", 188 | Sint32: "sint32", 189 | Sint64: "sint64", 190 | } 191 | 192 | func (ft FieldType) String() string { 193 | if s, ok := FieldTypeMap[ft]; ok { 194 | return s 195 | } 196 | return "UNKNOWN" 197 | } 198 | 199 | type Enum struct { 200 | Position Position // position of "enum" token 201 | Name string 202 | Values []*EnumValue 203 | 204 | Up interface{} // either *File or *Message 205 | } 206 | 207 | func (enum *Enum) Pos() Position { return enum.Position } 208 | func (enum *Enum) File() *File { 209 | for x := enum.Up; ; { 210 | switch up := x.(type) { 211 | case *File: 212 | return up 213 | case *Message: 214 | x = up.Up 215 | default: 216 | log.Panicf("internal error: Enum.Up is a %T", up) 217 | } 218 | } 219 | } 220 | 221 | type EnumValue struct { 222 | Position Position // position of Name 223 | Name string 224 | Number int32 225 | 226 | Up *Enum 227 | } 228 | 229 | func (ev *EnumValue) Pos() Position { return ev.Position } 230 | func (ev *EnumValue) File() *File { return ev.Up.File() } 231 | 232 | // Service represents an RPC service. 233 | type Service struct { 234 | Position Position // position of the "service" token 235 | Name string 236 | 237 | Methods []*Method 238 | 239 | Up *File 240 | } 241 | 242 | func (s *Service) Pos() Position { return s.Position } 243 | func (s *Service) File() *File { return s.Up } 244 | 245 | // Method represents an RPC method. 246 | type Method struct { 247 | Position Position // position of the "rpc" token 248 | Name string 249 | 250 | // InTypeName/OutTypeName are the raw names parsed from the input. 251 | // InType/OutType is set during resolution; it will be a *Message. 252 | InTypeName, OutTypeName string 253 | InType, OutType interface{} 254 | 255 | // ClientStreaming and ServerStreaming indicate whether the argument and 256 | // return value to the rpc are streams. 257 | ClientStreaming, ServerStreaming bool 258 | 259 | Up *Service 260 | } 261 | 262 | func (m *Method) Pos() Position { return m.Position } 263 | func (m *Method) File() *File { return m.Up.Up } 264 | 265 | // Extension represents an extension definition. 266 | type Extension struct { 267 | Position Position // position of the "extend" token 268 | 269 | Extendee string // the thing being extended 270 | ExtendeeType *Message // set during resolution 271 | 272 | Fields []*Field 273 | 274 | Up interface{} // either *File or *Message or ... 275 | } 276 | 277 | func (e *Extension) Pos() Position { return e.Position } 278 | func (e *Extension) File() *File { 279 | switch up := e.Up.(type) { 280 | case *File: 281 | return up 282 | case *Message: 283 | return up.File() 284 | default: 285 | log.Panicf("internal error: Extension.Up is a %T", up) 286 | } 287 | panic("unreachable") 288 | } 289 | 290 | // Comment represents a comment. 291 | type Comment struct { 292 | Start, End Position // position of first and last "//" 293 | Text []string 294 | } 295 | 296 | func (c *Comment) Pos() Position { return c.Start } 297 | 298 | // LeadingComment returns the comment that immediately precedes a node, 299 | // or nil if there's no such comment. 300 | func LeadingComment(n Node) *Comment { 301 | f := n.File() 302 | // Get the comment whose End position is on the previous line. 303 | lineEnd := n.Pos().Line - 1 304 | ci := sort.Search(len(f.Comments), func(i int) bool { 305 | return f.Comments[i].End.Line >= lineEnd 306 | }) 307 | if ci >= len(f.Comments) || f.Comments[ci].End.Line != lineEnd { 308 | return nil 309 | } 310 | return f.Comments[ci] 311 | } 312 | 313 | // InlineComment returns the comment on the same line as a node, 314 | // or nil if there's no inline comment. 315 | // The returned comment is guaranteed to be a single line. 316 | func InlineComment(n Node) *Comment { 317 | // TODO: Do we care about comments line this? 318 | // string name = 1; /* foo 319 | // bar */ 320 | 321 | f := n.File() 322 | pos := n.Pos() 323 | ci := sort.Search(len(f.Comments), func(i int) bool { 324 | return f.Comments[i].Start.Line >= pos.Line 325 | }) 326 | if ci >= len(f.Comments) || f.Comments[ci].Start.Line != pos.Line { 327 | return nil 328 | } 329 | c := f.Comments[ci] 330 | // Sanity check; it should only be one line. 331 | if c.Start != c.End || len(c.Text) != 1 { 332 | log.Panicf("internal error: bad inline comment: %+v", c) 333 | } 334 | return c 335 | } 336 | 337 | // Position describes a source position in an input file. 338 | // It is only valid if the line number is positive. 339 | type Position struct { 340 | Line int // 1-based line number 341 | Offset int // 0-based byte offset 342 | } 343 | 344 | func (pos Position) IsValid() bool { return pos.Line > 0 } 345 | func (pos Position) Before(other Position) bool { return pos.Offset < other.Offset } 346 | func (pos Position) String() string { 347 | if pos.Line == 0 { 348 | return ":" 349 | } 350 | return fmt.Sprintf(":%d", pos.Line) 351 | } 352 | -------------------------------------------------------------------------------- /gendesc/gendesc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package gendesc generates descriptor protos from an AST. 3 | */ 4 | package gendesc 5 | 6 | import ( 7 | "fmt" 8 | "sort" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/dsymonds/gotoc/ast" 13 | "github.com/golang/protobuf/proto" 14 | pb "github.com/golang/protobuf/protoc-gen-go/descriptor" 15 | ) 16 | 17 | func Generate(fs *ast.FileSet) (*pb.FileDescriptorSet, error) { 18 | fds := new(pb.FileDescriptorSet) 19 | for _, f := range fs.Files { 20 | fdp, err := genFile(f) 21 | if err != nil { 22 | return nil, err 23 | } 24 | fds.File = append(fds.File, fdp) 25 | } 26 | return fds, nil 27 | } 28 | 29 | func genFile(f *ast.File) (*pb.FileDescriptorProto, error) { 30 | fdp := &pb.FileDescriptorProto{ 31 | Name: maybeString(f.Name), 32 | Package: maybeString(strings.Join(f.Package, ".")), 33 | } 34 | for _, imp := range f.Imports { 35 | fdp.Dependency = append(fdp.Dependency, imp) 36 | } 37 | for _, i := range f.PublicImports { 38 | fdp.PublicDependency = append(fdp.PublicDependency, int32(i)) 39 | } 40 | sort.Sort(int32Slice(fdp.PublicDependency)) 41 | for _, m := range f.Messages { 42 | dp, err := genMessage(m) 43 | if err != nil { 44 | return nil, err 45 | } 46 | fdp.MessageType = append(fdp.MessageType, dp) 47 | } 48 | for _, enum := range f.Enums { 49 | edp, err := genEnum(enum) 50 | if err != nil { 51 | return nil, err 52 | } 53 | fdp.EnumType = append(fdp.EnumType, edp) 54 | } 55 | for _, srv := range f.Services { 56 | sdp, err := genService(srv) 57 | if err != nil { 58 | return nil, err 59 | } 60 | fdp.Service = append(fdp.Service, sdp) 61 | } 62 | for _, ext := range f.Extensions { 63 | fdps, err := genExtension(ext) 64 | if err != nil { 65 | return nil, err 66 | } 67 | fdp.Extension = append(fdp.Extension, fdps...) 68 | } 69 | for _, opt := range f.Options { 70 | if fdp.Options == nil { 71 | fdp.Options = new(pb.FileOptions) 72 | } 73 | // TODO: interpret common options 74 | uo := new(pb.UninterpretedOption) 75 | for _, part := range strings.Split(opt[0], ".") { 76 | // TODO: support IsExtension 77 | uo.Name = append(uo.Name, &pb.UninterpretedOption_NamePart{ 78 | NamePart: proto.String(part), 79 | IsExtension: proto.Bool(false), 80 | }) 81 | // TODO: need to handle more types 82 | if strings.HasPrefix(opt[1], `"`) { 83 | // TODO: doesn't handle single quote strings, etc. 84 | unq, err := strconv.Unquote(opt[1]) 85 | if err != nil { 86 | return nil, err 87 | } 88 | uo.StringValue = []byte(unq) 89 | } else { 90 | uo.IdentifierValue = proto.String(opt[1]) 91 | } 92 | } 93 | fdp.Options.UninterpretedOption = append(fdp.Options.UninterpretedOption, uo) 94 | } 95 | // TODO: SourceCodeInfo 96 | switch f.Syntax { 97 | case "proto2", "": 98 | // "proto2" is considered the default; don't set anything. 99 | default: 100 | fdp.Syntax = proto.String(f.Syntax) 101 | } 102 | 103 | return fdp, nil 104 | } 105 | 106 | func genMessage(m *ast.Message) (*pb.DescriptorProto, error) { 107 | dp := &pb.DescriptorProto{ 108 | Name: proto.String(m.Name), 109 | } 110 | var extraNested []*pb.DescriptorProto 111 | for _, f := range m.Fields { 112 | fdp, xdp, err := genField(f) 113 | if err != nil { 114 | return nil, err 115 | } 116 | dp.Field = append(dp.Field, fdp) 117 | if xdp != nil { 118 | extraNested = append(extraNested, xdp) 119 | } 120 | } 121 | for _, ext := range m.Extensions { 122 | fdps, err := genExtension(ext) 123 | if err != nil { 124 | return nil, err 125 | } 126 | dp.Extension = append(dp.Extension, fdps...) 127 | } 128 | for _, nm := range m.Messages { 129 | ndp, err := genMessage(nm) 130 | if err != nil { 131 | return nil, err 132 | } 133 | dp.NestedType = append(dp.NestedType, ndp) 134 | } 135 | // Put extra nested DescriptorProtos (e.g. from a map field) 136 | // at the end so they don't disrupt message indexes. 137 | dp.NestedType = append(dp.NestedType, extraNested...) 138 | for _, ne := range m.Enums { 139 | edp, err := genEnum(ne) 140 | if err != nil { 141 | return nil, err 142 | } 143 | dp.EnumType = append(dp.EnumType, edp) 144 | } 145 | for _, r := range m.ExtensionRanges { 146 | // DescriptorProto.ExtensionRange uses a half-open interval. 147 | dp.ExtensionRange = append(dp.ExtensionRange, &pb.DescriptorProto_ExtensionRange{ 148 | Start: proto.Int32(int32(r[0])), 149 | End: proto.Int32(int32(r[1] + 1)), 150 | }) 151 | } 152 | for _, oo := range m.Oneofs { 153 | dp.OneofDecl = append(dp.OneofDecl, &pb.OneofDescriptorProto{ 154 | Name: proto.String(oo.Name), 155 | }) 156 | } 157 | return dp, nil 158 | } 159 | 160 | func genField(f *ast.Field) (*pb.FieldDescriptorProto, *pb.DescriptorProto, error) { 161 | fdp := &pb.FieldDescriptorProto{ 162 | Name: proto.String(f.Name), 163 | Number: proto.Int32(int32(f.Tag)), 164 | } 165 | switch { 166 | case f.Required: 167 | fdp.Label = pb.FieldDescriptorProto_LABEL_REQUIRED.Enum() 168 | case f.Repeated: 169 | fdp.Label = pb.FieldDescriptorProto_LABEL_REPEATED.Enum() 170 | default: 171 | // default is optional 172 | fdp.Label = pb.FieldDescriptorProto_LABEL_OPTIONAL.Enum() 173 | } 174 | if f.KeyTypeName != "" { 175 | mname := camelCase(f.Name) + "Entry" 176 | vmsg := &ast.Message{ 177 | Name: mname, 178 | Fields: []*ast.Field{ 179 | { 180 | TypeName: f.KeyTypeName, 181 | Type: f.KeyType, 182 | Name: "key", 183 | Tag: 1, 184 | }, 185 | { 186 | TypeName: f.TypeName, 187 | Type: f.Type, 188 | Name: "value", 189 | Tag: 2, 190 | }, 191 | }, 192 | Up: f.Up, 193 | } 194 | vmsg.Fields[0].Up = vmsg 195 | vmsg.Fields[1].Up = vmsg 196 | xdp, err := genMessage(vmsg) 197 | if err != nil { 198 | return nil, nil, fmt.Errorf("internal error: %v", err) 199 | } 200 | xdp.Options = &pb.MessageOptions{ 201 | MapEntry: proto.Bool(true), 202 | } 203 | fdp.Type = pb.FieldDescriptorProto_TYPE_MESSAGE.Enum() 204 | fdp.TypeName = proto.String(qualifiedName(vmsg)) 205 | return fdp, xdp, nil 206 | } 207 | switch t := f.Type.(type) { 208 | case ast.FieldType: 209 | pt, ok := fieldTypeMap[t] 210 | if !ok { 211 | return nil, nil, fmt.Errorf("internal error: no mapping from ast.FieldType %v", t) 212 | } 213 | fdp.Type = pt.Enum() 214 | case *ast.Message: 215 | if !t.Group { 216 | fdp.Type = pb.FieldDescriptorProto_TYPE_MESSAGE.Enum() 217 | } else { 218 | fdp.Type = pb.FieldDescriptorProto_TYPE_GROUP.Enum() 219 | // The field name is lowercased by protoc. 220 | *fdp.Name = strings.ToLower(*fdp.Name) 221 | } 222 | fdp.TypeName = proto.String(qualifiedName(t)) 223 | case *ast.Enum: 224 | fdp.Type = pb.FieldDescriptorProto_TYPE_ENUM.Enum() 225 | fdp.TypeName = proto.String(qualifiedName(t)) 226 | default: 227 | return nil, nil, fmt.Errorf("internal error: bad ast.Field.Type type %T", f.Type) 228 | } 229 | if ext, ok := f.Up.(*ast.Extension); ok { 230 | fdp.Extendee = proto.String(qualifiedName(ext.ExtendeeType)) 231 | } 232 | if f.HasDefault { 233 | fdp.DefaultValue = proto.String(f.Default) 234 | } 235 | if f.Oneof != nil { 236 | n := 0 237 | for _, oo := range f.Oneof.Up.Oneofs { 238 | if oo == f.Oneof { 239 | break 240 | } 241 | n++ 242 | } 243 | fdp.OneofIndex = proto.Int(n) 244 | } 245 | 246 | return fdp, nil, nil 247 | } 248 | 249 | func genEnum(enum *ast.Enum) (*pb.EnumDescriptorProto, error) { 250 | edp := &pb.EnumDescriptorProto{ 251 | Name: proto.String(enum.Name), 252 | } 253 | for _, ev := range enum.Values { 254 | edp.Value = append(edp.Value, &pb.EnumValueDescriptorProto{ 255 | Name: proto.String(ev.Name), 256 | Number: proto.Int32(ev.Number), 257 | }) 258 | } 259 | return edp, nil 260 | } 261 | 262 | func genService(srv *ast.Service) (*pb.ServiceDescriptorProto, error) { 263 | sdp := &pb.ServiceDescriptorProto{ 264 | Name: proto.String(srv.Name), 265 | } 266 | for _, mth := range srv.Methods { 267 | mdp, err := genMethod(mth) 268 | if err != nil { 269 | return nil, err 270 | } 271 | sdp.Method = append(sdp.Method, mdp) 272 | } 273 | return sdp, nil 274 | } 275 | 276 | func genMethod(mth *ast.Method) (*pb.MethodDescriptorProto, error) { 277 | mdp := &pb.MethodDescriptorProto{ 278 | Name: proto.String(mth.Name), 279 | InputType: proto.String(qualifiedName(mth.InType)), 280 | OutputType: proto.String(qualifiedName(mth.OutType)), 281 | } 282 | if mth.ClientStreaming { 283 | mdp.ClientStreaming = proto.Bool(true) 284 | } 285 | if mth.ServerStreaming { 286 | mdp.ServerStreaming = proto.Bool(true) 287 | } 288 | return mdp, nil 289 | } 290 | 291 | func genExtension(ext *ast.Extension) ([]*pb.FieldDescriptorProto, error) { 292 | var fdps []*pb.FieldDescriptorProto 293 | for _, f := range ext.Fields { 294 | // TODO: It should be impossible to get a map field? 295 | fdp, _, err := genField(f) 296 | if err != nil { 297 | return nil, err 298 | } 299 | fdps = append(fdps, fdp) 300 | } 301 | return fdps, nil 302 | } 303 | 304 | // qualifiedName returns the fully-qualified name of x, 305 | // which must be either *ast.Message or *ast.Enum. 306 | func qualifiedName(x interface{}) string { 307 | var parts []string 308 | for { 309 | switch v := x.(type) { 310 | case *ast.Message: 311 | parts = append(parts, v.Name) 312 | x = v.Up 313 | continue 314 | case *ast.Enum: 315 | parts = append(parts, v.Name) 316 | x = v.Up 317 | continue 318 | } 319 | break // *ast.File 320 | } 321 | if f := x.(*ast.File); true { 322 | // Add package components in reverse order. 323 | for i := len(f.Package) - 1; i >= 0; i-- { 324 | parts = append(parts, f.Package[i]) 325 | } 326 | } 327 | // Reverse parts, then join with dots. 328 | for i, j := 0, len(parts)-1; i < j; { 329 | parts[i], parts[j] = parts[j], parts[i] 330 | i++ 331 | j-- 332 | } 333 | return "." + strings.Join(parts, ".") 334 | } 335 | 336 | // A mapping of ast.FieldType to the proto type. 337 | // Does not include TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP. 338 | var fieldTypeMap = map[ast.FieldType]pb.FieldDescriptorProto_Type{ 339 | ast.Double: pb.FieldDescriptorProto_TYPE_DOUBLE, 340 | ast.Float: pb.FieldDescriptorProto_TYPE_FLOAT, 341 | ast.Int64: pb.FieldDescriptorProto_TYPE_INT64, 342 | ast.Uint64: pb.FieldDescriptorProto_TYPE_UINT64, 343 | ast.Int32: pb.FieldDescriptorProto_TYPE_INT32, 344 | ast.Fixed64: pb.FieldDescriptorProto_TYPE_FIXED64, 345 | ast.Fixed32: pb.FieldDescriptorProto_TYPE_FIXED32, 346 | ast.Bool: pb.FieldDescriptorProto_TYPE_BOOL, 347 | ast.String: pb.FieldDescriptorProto_TYPE_STRING, 348 | ast.Bytes: pb.FieldDescriptorProto_TYPE_BYTES, 349 | ast.Uint32: pb.FieldDescriptorProto_TYPE_UINT32, 350 | ast.Sfixed32: pb.FieldDescriptorProto_TYPE_SFIXED32, 351 | ast.Sfixed64: pb.FieldDescriptorProto_TYPE_SFIXED64, 352 | ast.Sint32: pb.FieldDescriptorProto_TYPE_SINT32, 353 | ast.Sint64: pb.FieldDescriptorProto_TYPE_SINT64, 354 | } 355 | 356 | func maybeString(s string) *string { 357 | if s != "" { 358 | return &s 359 | } 360 | return nil 361 | } 362 | 363 | type int32Slice []int32 364 | 365 | func (s int32Slice) Len() int { return len(s) } 366 | func (s int32Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 367 | func (s int32Slice) Less(i, j int) bool { return s[i] < s[j] } 368 | 369 | // camelCase turns foo_bar into FooBar. 370 | func camelCase(s string) string { 371 | words := strings.Split(s, "_") 372 | for i, word := range words { 373 | words[i] = strings.Title(word) 374 | } 375 | return strings.Join(words, "") 376 | } 377 | -------------------------------------------------------------------------------- /parser/parser_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/dsymonds/gotoc/ast" 7 | "github.com/dsymonds/gotoc/gendesc" 8 | "github.com/golang/protobuf/proto" 9 | pb "github.com/golang/protobuf/protoc-gen-go/descriptor" 10 | ) 11 | 12 | // tryParse attempts to parse the input, and verifies that it matches 13 | // the FileDescriptorProto represented in text format. 14 | func tryParse(t *testing.T, input, output string) { 15 | want := new(pb.FileDescriptorProto) 16 | if err := proto.UnmarshalText(output, want); err != nil { 17 | t.Fatalf("Test failure parsing a wanted proto: %v", err) 18 | } 19 | 20 | p := newParser("-", input) 21 | f := new(ast.File) 22 | if pe := p.readFile(f); pe != nil { 23 | t.Errorf("Failed parsing input: %v", pe) 24 | return 25 | } 26 | fset := &ast.FileSet{Files: []*ast.File{f}} 27 | if err := resolveSymbols(fset); err != nil { 28 | t.Errorf("Resolving symbols: %v", err) 29 | return 30 | } 31 | 32 | fds, err := gendesc.Generate(fset) 33 | if err != nil { 34 | t.Errorf("Generating FileDescriptorSet: %v", err) 35 | return 36 | } 37 | if n := len(fds.File); n != 1 { 38 | t.Errorf("Generated %d FileDescriptorProtos, want 1", n) 39 | return 40 | } 41 | got := fds.File[0] 42 | 43 | if !proto.Equal(got, want) { 44 | t.Errorf("Mismatch!\nGot:\n%v\nWant:\n%v", got, want) 45 | } 46 | } 47 | 48 | type parseTest struct { 49 | name string 50 | input, expected string 51 | } 52 | 53 | // used to shorten the FieldDefaults expected output. 54 | const fieldDefaultsEtc = `name:"foo" label:LABEL_REQUIRED number:1` 55 | 56 | var parseTests = []parseTest{ 57 | { 58 | "SimpleMessage", 59 | "message TestMessage {\n required int32 foo = 1;\n}\n", 60 | `message_type { name: "TestMessage" field { name:"foo" label:LABEL_REQUIRED type:TYPE_INT32 number:1 } }`, 61 | }, 62 | { 63 | "ImplicitSyntaxIdentifier", 64 | "message TestMessage {\n required int32 foo = 1;\n}\n", 65 | `message_type { name: "TestMessage" field { name:"foo" label:LABEL_REQUIRED type:TYPE_INT32 number:1 } }`, 66 | }, 67 | { 68 | "ExplicitSyntaxIdentifier", 69 | "syntax = \"proto2\";\nmessage TestMessage {\n required int32 foo = 1;\n}\n", 70 | `message_type { name: "TestMessage" field { name:"foo" label:LABEL_REQUIRED type:TYPE_INT32 number:1 } }`, 71 | }, 72 | { 73 | "SimpleFields", 74 | "message TestMessage {\n required int32 foo = 15;\n optional int32 bar = 34;\n repeated int32 baz = 3;\n}\n", 75 | `message_type { 76 | name: "TestMessage" 77 | field { name:"foo" label:LABEL_REQUIRED type:TYPE_INT32 number:15 } 78 | field { name:"bar" label:LABEL_OPTIONAL type:TYPE_INT32 number:34 } 79 | field { name:"baz" label:LABEL_REPEATED type:TYPE_INT32 number:3 } 80 | }`, 81 | }, 82 | { 83 | "PrimitiveFieldTypes", 84 | `message TestMessage { 85 | required int32 foo = 1; 86 | required int64 foo = 1; 87 | required uint32 foo = 1; 88 | required uint64 foo = 1; 89 | required sint32 foo = 1; 90 | required sint64 foo = 1; 91 | required fixed32 foo = 1; 92 | required fixed64 foo = 1; 93 | required sfixed32 foo = 1; 94 | required sfixed64 foo = 1; 95 | required float foo = 1; 96 | required double foo = 1; 97 | required string foo = 1; 98 | required bytes foo = 1; 99 | required bool foo = 1; 100 | }`, 101 | `message_type { 102 | name: "TestMessage" 103 | field { name:"foo" label:LABEL_REQUIRED type:TYPE_INT32 number:1 } 104 | field { name:"foo" label:LABEL_REQUIRED type:TYPE_INT64 number:1 } 105 | field { name:"foo" label:LABEL_REQUIRED type:TYPE_UINT32 number:1 } 106 | field { name:"foo" label:LABEL_REQUIRED type:TYPE_UINT64 number:1 } 107 | field { name:"foo" label:LABEL_REQUIRED type:TYPE_SINT32 number:1 } 108 | field { name:"foo" label:LABEL_REQUIRED type:TYPE_SINT64 number:1 } 109 | field { name:"foo" label:LABEL_REQUIRED type:TYPE_FIXED32 number:1 } 110 | field { name:"foo" label:LABEL_REQUIRED type:TYPE_FIXED64 number:1 } 111 | field { name:"foo" label:LABEL_REQUIRED type:TYPE_SFIXED32 number:1 } 112 | field { name:"foo" label:LABEL_REQUIRED type:TYPE_SFIXED64 number:1 } 113 | field { name:"foo" label:LABEL_REQUIRED type:TYPE_FLOAT number:1 } 114 | field { name:"foo" label:LABEL_REQUIRED type:TYPE_DOUBLE number:1 } 115 | field { name:"foo" label:LABEL_REQUIRED type:TYPE_STRING number:1 } 116 | field { name:"foo" label:LABEL_REQUIRED type:TYPE_BYTES number:1 } 117 | field { name:"foo" label:LABEL_REQUIRED type:TYPE_BOOL number:1 } 118 | }`, 119 | }, 120 | { 121 | "FieldDefaults", 122 | `message TestMessage { 123 | required int32 foo = 1 [default= 1 ]; 124 | required int32 foo = 1 [default= -2 ]; 125 | required int64 foo = 1 [default= 3 ]; 126 | required int64 foo = 1 [default= -4 ]; 127 | required uint32 foo = 1 [default= 5 ]; 128 | required uint64 foo = 1 [default= 6 ]; 129 | required float foo = 1 [default= 7.5]; 130 | required float foo = 1 [default= -8.5]; 131 | required float foo = 1 [default= 9 ]; 132 | required double foo = 1 [default= 10.5]; 133 | required double foo = 1 [default=-11.5]; 134 | required double foo = 1 [default= 12 ]; 135 | required double foo = 1 [default= inf ]; 136 | required double foo = 1 [default=-inf ]; 137 | required double foo = 1 [default= nan ]; 138 | // TODO: uncomment these when the string parser handles them. 139 | //required string foo = 1 [default='13\\001']; 140 | //required string foo = 1 [default='a' "b" 141 | //"c"]; 142 | //required bytes foo = 1 [default='14\\002']; 143 | //required bytes foo = 1 [default='a' "b" 144 | //'c']; 145 | required bool foo = 1 [default=true ]; 146 | required Foo foo = 1 [default=FOO ]; 147 | required int32 foo = 1 [default= 0x7FFFFFFF]; 148 | required int32 foo = 1 [default=-0x80000000]; 149 | required uint32 foo = 1 [default= 0xFFFFFFFF]; 150 | required int64 foo = 1 [default= 0x7FFFFFFFFFFFFFFF]; 151 | required int64 foo = 1 [default=-0x8000000000000000]; 152 | required uint64 foo = 1 [default= 0xFFFFFFFFFFFFFFFF]; 153 | } 154 | enum Foo { UNKNOWN=0; FOO=1; } 155 | `, 156 | `message_type { 157 | name: "TestMessage" 158 | field { type:TYPE_INT32 default_value:"1" ` + fieldDefaultsEtc + ` } 159 | field { type:TYPE_INT32 default_value:"-2" ` + fieldDefaultsEtc + ` } 160 | field { type:TYPE_INT64 default_value:"3" ` + fieldDefaultsEtc + ` } 161 | field { type:TYPE_INT64 default_value:"-4" ` + fieldDefaultsEtc + ` } 162 | field { type:TYPE_UINT32 default_value:"5" ` + fieldDefaultsEtc + ` } 163 | field { type:TYPE_UINT64 default_value:"6" ` + fieldDefaultsEtc + ` } 164 | field { type:TYPE_FLOAT default_value:"7.5" ` + fieldDefaultsEtc + ` } 165 | field { type:TYPE_FLOAT default_value:"-8.5" ` + fieldDefaultsEtc + ` } 166 | field { type:TYPE_FLOAT default_value:"9" ` + fieldDefaultsEtc + ` } 167 | field { type:TYPE_DOUBLE default_value:"10.5" ` + fieldDefaultsEtc + ` } 168 | field { type:TYPE_DOUBLE default_value:"-11.5" ` + fieldDefaultsEtc + ` } 169 | field { type:TYPE_DOUBLE default_value:"12" ` + fieldDefaultsEtc + ` } 170 | field { type:TYPE_DOUBLE default_value:"inf" ` + fieldDefaultsEtc + ` } 171 | field { type:TYPE_DOUBLE default_value:"-inf" ` + fieldDefaultsEtc + ` } 172 | field { type:TYPE_DOUBLE default_value:"nan" ` + fieldDefaultsEtc + ` } 173 | ` + 174 | /* 175 | field { type:TYPE_STRING default_value:"13\\001" ` + fieldDefaultsEtc + ` } 176 | field { type:TYPE_STRING default_value:"abc" ` + fieldDefaultsEtc + ` } 177 | field { type:TYPE_BYTES default_value:"14\\\\002" ` + fieldDefaultsEtc + ` } 178 | */ 179 | ` 180 | field { type:TYPE_BOOL default_value:"true" ` + fieldDefaultsEtc + ` } 181 | field { type:TYPE_ENUM type_name:".Foo" default_value:"FOO"` + fieldDefaultsEtc + ` } 182 | 183 | ` + 184 | /* 185 | descriptor.proto says "For numeric types, contains the original text representation of the value."; 186 | we match that, and thus diverge from protoc. 187 | */ 188 | ` 189 | field { type:TYPE_INT32 default_value:"0x7FFFFFFF" ` + fieldDefaultsEtc + ` } 190 | field { type:TYPE_INT32 default_value:"-0x80000000" ` + fieldDefaultsEtc + ` } 191 | field { type:TYPE_UINT32 default_value:"0xFFFFFFFF" ` + fieldDefaultsEtc + ` } 192 | field { type:TYPE_INT64 default_value:"0x7FFFFFFFFFFFFFFF" ` + fieldDefaultsEtc + ` } 193 | field { type:TYPE_INT64 default_value:"-0x8000000000000000"` + fieldDefaultsEtc + ` } 194 | field { type:TYPE_UINT64 default_value:"0xFFFFFFFFFFFFFFFF" ` + fieldDefaultsEtc + ` } 195 | } 196 | enum_type { 197 | name:"Foo" 198 | value { name:"UNKNOWN" number:0 } 199 | value { name:"FOO" number:1 } 200 | } 201 | `, 202 | }, 203 | // TODO: FieldOptions 204 | { 205 | "Oneof", 206 | "message TestMessage {\n oneof foo {\n int32 a = 1;\n string b = 2;\n TestMessage c = 3;\n group D = 4 { optional int32 i = 5; }\n }\n}\n", 207 | `message_type { 208 | name: "TestMessage" 209 | field { name:"a" label:LABEL_OPTIONAL type:TYPE_INT32 number:1 oneof_index:0 } 210 | field { name:"b" label:LABEL_OPTIONAL type:TYPE_STRING number:2 oneof_index:0 } 211 | field { name:"c" label:LABEL_OPTIONAL type:TYPE_MESSAGE type_name:".TestMessage" number:3 oneof_index:0 } 212 | field { name:"d" label:LABEL_OPTIONAL type:TYPE_GROUP type_name:".TestMessage.D" number:4 oneof_index:0 } 213 | oneof_decl { 214 | name: "foo" 215 | } 216 | nested_type { 217 | name: "D" 218 | field { name:"i" label:LABEL_OPTIONAL type:TYPE_INT32 number:5 } 219 | } 220 | }`, 221 | }, 222 | { 223 | "MultipleOneofs", 224 | "message TestMessage {\n oneof foo {\n int32 a = 1;\n string b = 2;\n }\n oneof bar {\n int32 c = 3;\n string d = 4;\n }\n}\n", 225 | `message_type { 226 | name: "TestMessage" 227 | field { name:"a" label:LABEL_OPTIONAL type:TYPE_INT32 number:1 oneof_index:0 } 228 | field { name:"b" label:LABEL_OPTIONAL type:TYPE_STRING number:2 oneof_index:0 } 229 | field { name:"c" label:LABEL_OPTIONAL type:TYPE_INT32 number:3 oneof_index:1 } 230 | field { name:"d" label:LABEL_OPTIONAL type:TYPE_STRING number:4 oneof_index:1 } 231 | oneof_decl { 232 | name: "foo" 233 | } 234 | oneof_decl { 235 | name: "bar" 236 | } 237 | }`, 238 | }, 239 | { 240 | "Maps", 241 | "message TestMessage {\n map primitive_type_map = 1;\n}\n", 242 | `message_type { 243 | name: "TestMessage" 244 | nested_type { 245 | name: "PrimitiveTypeMapEntry" 246 | field { name: "key" number: 1 label:LABEL_OPTIONAL type:TYPE_INT32 } 247 | field { name: "value" number: 2 label:LABEL_OPTIONAL type:TYPE_STRING } 248 | options { map_entry: true } 249 | } 250 | field { name: "primitive_type_map" label: LABEL_REPEATED type:TYPE_MESSAGE type_name: ".TestMessage.PrimitiveTypeMapEntry" number: 1 } 251 | }`, 252 | }, 253 | { 254 | "Group", 255 | "message TestMessage {\n optional group TestGroup = 1 {};\n}\n", 256 | `message_type { 257 | name: "TestMessage" 258 | nested_type { name: "TestGroup" } 259 | field { name:"testgroup" label:LABEL_OPTIONAL number:1 type:TYPE_GROUP type_name: ".TestMessage.TestGroup" } 260 | }`, 261 | }, 262 | { 263 | "NestedMessage", 264 | "message TestMessage {\n message Nested {}\n optional Nested test_nested = 1;\n }\n", 265 | `message_type { name: "TestMessage" nested_type { name: "Nested" } field { name:"test_nested" label:LABEL_OPTIONAL number:1 type:TYPE_MESSAGE type_name:".TestMessage.Nested" } }`, 266 | }, 267 | { 268 | "NestedEnum", 269 | "message TestMessage {\n enum NestedEnum {}\n optional NestedEnum test_enum = 1;\n }\n", 270 | `message_type { name: "TestMessage" enum_type { name: "NestedEnum" } field { name:"test_enum" label:LABEL_OPTIONAL number:1 type:TYPE_ENUM type_name:".TestMessage.NestedEnum" } }`, 271 | }, 272 | { 273 | "ExtensionRange", 274 | "message TestMessage {\n extensions 10 to 19;\n extensions 30 to max;\n}\n", 275 | `message_type { name: "TestMessage" extension_range { start:10 end:20 } extension_range { start:30 end:536870912 } }`, 276 | }, 277 | { 278 | "CompoundExtensionRange", 279 | "message TestMessage {\n extensions 2, 15, 9 to 11, 100 to max, 3;\n}\n", 280 | `message_type { name: "TestMessage" ` + 281 | ` extension_range { start:2 end:3 }` + 282 | ` extension_range { start:15 end:16 }` + 283 | ` extension_range { start:9 end:12 }` + 284 | ` extension_range { start:100 end:536870912 }` + 285 | ` extension_range { start:3 end:4 }` + 286 | `}`, 287 | }, 288 | { 289 | "Extensions", 290 | "extend Extendee1 { optional int32 foo = 12; }\nextend Extendee2 { repeated TestMessage bar = 22; }\n" + 291 | "message Extendee1 { extensions 12; } message Extendee2 { extensions 20 to 24; } message TestMessage{}", 292 | `extension { name:"foo" label:LABEL_OPTIONAL type:TYPE_INT32 number:12 extendee: ".Extendee1" } ` + 293 | `extension { name:"bar" label:LABEL_REPEATED number:22 type:TYPE_MESSAGE type_name:".TestMessage" extendee: ".Extendee2" }` + 294 | `message_type{name:"Extendee1" extension_range{start:12 end:13} } ` + 295 | `message_type{name:"Extendee2" extension_range{start:20 end:25} } ` + 296 | `message_type{name:"TestMessage"}`, 297 | }, 298 | { 299 | "ExtensionsInMessageScope", 300 | "message TestMessage {\n extend Extendee1 { optional int32 foo = 12; }\n extend Extendee2 { repeated TestMessage bar = 22; }\n}\n" + 301 | "message Extendee1 { extensions 12; } message Extendee2 { extensions 20 to 24; }", 302 | `message_type { name: "TestMessage"` + 303 | ` extension { name:"foo" label:LABEL_OPTIONAL type:TYPE_INT32 number:12 extendee: ".Extendee1" }` + 304 | ` extension { name:"bar" label:LABEL_REPEATED number:22 type:TYPE_MESSAGE type_name:".TestMessage" extendee: ".Extendee2" }` + 305 | `}` + 306 | `message_type{name:"Extendee1" extension_range{start:12 end:13} } ` + 307 | `message_type{name:"Extendee2" extension_range{start:20 end:25} } `, 308 | }, 309 | { 310 | "MultipleExtensionsOneExtendee", 311 | "extend Extendee1 {\n optional int32 foo = 12;\n repeated TestMessage bar = 22;\n}\n" + 312 | "message Extendee1 { extensions 12 to 24; } message TestMessage{}", 313 | `extension { name:"foo" label:LABEL_OPTIONAL type:TYPE_INT32 number:12 extendee: ".Extendee1" } ` + 314 | `extension { name:"bar" label:LABEL_REPEATED number:22 type:TYPE_MESSAGE type_name:".TestMessage" extendee: ".Extendee1" }` + 315 | `message_type{name:"Extendee1" extension_range{start:12 end:25} } ` + 316 | `message_type{name:"TestMessage"}`, 317 | }, 318 | { 319 | "OptionalOptionalLabelProto3", 320 | "syntax = \"proto3\";\nmessage TestMessage {\n int32 foo = 1;\n optional int32 bar = 2;\n}\n", 321 | `syntax: "proto3" message_type { name: "TestMessage" ` + 322 | ` field { name:"foo" label:LABEL_OPTIONAL type:TYPE_INT32 number:1 }` + 323 | ` field { name:"bar" label:LABEL_OPTIONAL type:TYPE_INT32 number:2 }` + 324 | `}`, 325 | }, 326 | { 327 | "EnumValues", 328 | "enum TestEnum {\n FOO = 13;\n BAR = -10;\n BAZ = 500;\n}\n", 329 | `enum_type { name: "TestEnum" value { name:"FOO" number:13 } value { name:"BAR" number:-10 } value { name:"BAZ" number:500 } }`, 330 | }, 331 | { 332 | "SimpleService", 333 | "service TestService {\n rpc Foo(In) returns (Out);\n}\n message In{} message Out{}", 334 | `service { name: "TestService" method { name:"Foo" input_type:".In" output_type:".Out" } }` + 335 | `message_type:{name:"In"} message_type:{name:"Out"}`, 336 | }, 337 | { 338 | "SimpleServiceWithMessageCalledStream", 339 | "service TestService {\n rpc Foo(stream) returns (stream);\n}\n message stream {}", 340 | `service { name: "TestService" method { name:"Foo" input_type:".stream" output_type:".stream" } }` + 341 | `message_type:{name:"stream"}`, 342 | }, 343 | { 344 | "StreamingService", 345 | "service TestService {\n rpc Foo(stream In) returns (stream Out);\n}\n message In{} message Out{}", 346 | `service { name: "TestService" method { name:"Foo" input_type:".In" output_type:".Out" client_streaming: true server_streaming: true } }` + 347 | `message_type:{name:"In"} message_type:{name:"Out"}`, 348 | }, 349 | { 350 | "ParseImport", 351 | "import \"foo/bar/baz.proto\";\n", 352 | `dependency: "foo/bar/baz.proto"`, 353 | }, 354 | { 355 | "ParsePackage", 356 | "package foo.bar.baz;\n", 357 | `package: "foo.bar.baz"`, 358 | }, 359 | { 360 | "ParsePackageWithSpaces", 361 | "package foo . bar. \n baz;\n", 362 | `package: "foo.bar.baz"`, 363 | }, 364 | { 365 | "ParseFileOptions", 366 | "option java_package = \"com.google.foo\";\noption optimize_for = CODE_SIZE;", 367 | `options { uninterpreted_option { name { name_part: "java_package" is_extension: false } string_value: "com.google.foo"} uninterpreted_option { name { name_part: "optimize_for" is_extension: false } identifier_value: "CODE_SIZE" } }`, 368 | }, 369 | { 370 | "ParsePublicImports", 371 | "import \"foo.proto\";\nimport public \"bar.proto\";\nimport \"baz.proto\";\nimport public \"qux.proto\";\n", 372 | `dependency: "foo.proto" dependency: "bar.proto" dependency: "baz.proto" dependency: "qux.proto" public_dependency: 1 public_dependency: 3`, 373 | }, 374 | } 375 | 376 | func TestParsing(t *testing.T) { 377 | for _, pt := range parseTests { 378 | t.Logf("[ %v ]", pt.name) 379 | tryParse(t, pt.input, pt.expected) 380 | } 381 | } 382 | -------------------------------------------------------------------------------- /parser/parser.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package parser parses proto files into gotoc's AST representation. 3 | */ 4 | package parser 5 | 6 | import ( 7 | "fmt" 8 | "io/ioutil" 9 | "log" 10 | "os" 11 | "path/filepath" 12 | "strconv" 13 | "strings" 14 | "unicode" 15 | 16 | "github.com/dsymonds/gotoc/ast" 17 | ) 18 | 19 | const debugging = false 20 | 21 | func debugf(format string, args ...interface{}) { 22 | if debugging { 23 | log.Printf(format, args...) 24 | } 25 | } 26 | 27 | // ParseFiles parses one or more files. 28 | // The files should generally consist of one package. 29 | // Any .proto files that these files import should be discoverable 30 | // relative to an element of importPaths; if importPaths is empty 31 | // then the current directory is searched. 32 | func ParseFiles(filenames []string, importPaths []string) (*ast.FileSet, error) { 33 | // Force importPaths to have at least one element. 34 | if len(importPaths) == 0 { 35 | importPaths = []string{"."} 36 | } 37 | 38 | fset := new(ast.FileSet) 39 | 40 | index := make(map[string]int) // filename => index in fset.Files 41 | 42 | for len(filenames) > 0 { 43 | filename := filenames[0] 44 | filenames = filenames[1:] 45 | if _, ok := index[filename]; ok { 46 | continue // already parsed this one 47 | } 48 | 49 | f := &ast.File{Name: filename} 50 | index[filename] = len(fset.Files) 51 | fset.Files = append(fset.Files, f) 52 | 53 | // Read the first existing file relative to an element of importPaths. 54 | var buf []byte 55 | for _, impPath := range importPaths { 56 | b, err := ioutil.ReadFile(filepath.Join(impPath, filename)) 57 | if err != nil { 58 | if os.IsNotExist(err) { 59 | continue 60 | } 61 | return nil, err 62 | } 63 | buf = b 64 | break 65 | } 66 | if buf == nil { 67 | return nil, fmt.Errorf("file not found: %s", filename) 68 | } 69 | 70 | p := newParser(filename, string(buf)) 71 | if pe := p.readFile(f); pe != nil { 72 | return nil, pe 73 | } 74 | if p.s != "" { 75 | return nil, p.errorf("input was not all consumed") 76 | } 77 | 78 | // enqueue unparsed imports 79 | for _, imp := range f.Imports { 80 | if _, ok := index[imp]; !ok { 81 | filenames = append(filenames, imp) 82 | } 83 | } 84 | } 85 | 86 | if err := resolveSymbols(fset); err != nil { 87 | return nil, err 88 | } 89 | fset.Sort() 90 | return fset, nil 91 | } 92 | 93 | type parseError struct { 94 | message string 95 | filename string 96 | line int // 1-based line number 97 | offset int // 0-based byte offset from start of input 98 | } 99 | 100 | func (pe *parseError) Error() string { 101 | if pe == nil { 102 | return "" 103 | } 104 | if pe.line == 1 { 105 | return fmt.Sprintf("%s:1.%d: %v", pe.filename, pe.offset, pe.message) 106 | } 107 | return fmt.Sprintf("%s:%d: %v", pe.filename, pe.line, pe.message) 108 | } 109 | 110 | var eof = &parseError{message: "EOF"} 111 | 112 | type token struct { 113 | value string 114 | err *parseError 115 | line, offset int 116 | unquoted string // unquoted version of value 117 | } 118 | 119 | func (t *token) astPosition() ast.Position { 120 | return ast.Position{ 121 | Line: t.line, 122 | Offset: t.offset, 123 | } 124 | } 125 | 126 | type parser struct { 127 | filename string 128 | s string // remaining input 129 | done bool 130 | backed bool // whether back() was called 131 | offset, line int 132 | cur token 133 | 134 | comments []comment // accumulated during parse 135 | } 136 | 137 | type comment struct { 138 | text string 139 | line, offset int 140 | } 141 | 142 | func newParser(filename, s string) *parser { 143 | return &parser{ 144 | filename: filename, 145 | s: s, 146 | line: 1, 147 | cur: token{line: 1}, 148 | } 149 | } 150 | 151 | func (p *parser) readFile(f *ast.File) *parseError { 152 | // Parse top-level things. 153 | for !p.done { 154 | tok := p.next() 155 | if tok.err == eof { 156 | break 157 | } else if tok.err != nil { 158 | return tok.err 159 | } 160 | // TODO: enforce ordering? package, imports, remainder 161 | switch tok.value { 162 | case "package": 163 | if f.Package != nil { 164 | return p.errorf("duplicate package statement") 165 | } 166 | var pkg string 167 | for { 168 | tok := p.next() 169 | if tok.err != nil { 170 | return tok.err 171 | } 172 | if tok.value == ";" { 173 | break 174 | } 175 | if tok.value == "." { 176 | // okay if we already have at least one package component, 177 | // and didn't just read a dot. 178 | if pkg == "" || strings.HasSuffix(pkg, ".") { 179 | return p.errorf(`got ".", want package name`) 180 | } 181 | } else { 182 | // okay if we don't have a package component, 183 | // or just read a dot. 184 | if pkg != "" && !strings.HasSuffix(pkg, ".") { 185 | return p.errorf(`got %q, want "." or ";"`, tok.value) 186 | } 187 | // TODO: validate more 188 | } 189 | pkg += tok.value 190 | } 191 | f.Package = strings.Split(pkg, ".") 192 | case "option": 193 | tok := p.next() 194 | if tok.err != nil { 195 | return tok.err 196 | } 197 | key := tok.value 198 | if err := p.readToken("="); err != nil { 199 | return err 200 | } 201 | tok = p.next() 202 | if tok.err != nil { 203 | return tok.err 204 | } 205 | value := tok.value 206 | if err := p.readToken(";"); err != nil { 207 | return err 208 | } 209 | f.Options = append(f.Options, [2]string{key, value}) 210 | case "syntax": 211 | if f.Syntax != "" { 212 | return p.errorf("duplicate syntax statement") 213 | } 214 | if err := p.readToken("="); err != nil { 215 | return err 216 | } 217 | tok, err := p.readString() 218 | if err != nil { 219 | return err 220 | } 221 | switch s := tok.unquoted; s { 222 | case "proto2", "proto3": 223 | f.Syntax = s 224 | default: 225 | return p.errorf("invalid syntax value %q", s) 226 | } 227 | if err := p.readToken(";"); err != nil { 228 | return err 229 | } 230 | case "import": 231 | if err := p.readToken("public"); err == nil { 232 | f.PublicImports = append(f.PublicImports, len(f.Imports)) 233 | } else { 234 | p.back() 235 | } 236 | tok, err := p.readString() 237 | if err != nil { 238 | return err 239 | } 240 | f.Imports = append(f.Imports, tok.unquoted) 241 | if err := p.readToken(";"); err != nil { 242 | return err 243 | } 244 | case "message": 245 | p.back() 246 | msg := new(ast.Message) 247 | f.Messages = append(f.Messages, msg) 248 | if err := p.readMessage(msg); err != nil { 249 | return err 250 | } 251 | msg.Up = f 252 | case "enum": 253 | p.back() 254 | enum := new(ast.Enum) 255 | f.Enums = append(f.Enums, enum) 256 | if err := p.readEnum(enum); err != nil { 257 | return err 258 | } 259 | enum.Up = f 260 | case "service": 261 | p.back() 262 | srv := new(ast.Service) 263 | f.Services = append(f.Services, srv) 264 | if err := p.readService(srv); err != nil { 265 | return err 266 | } 267 | srv.Up = f 268 | case "extend": 269 | p.back() 270 | ext := new(ast.Extension) 271 | f.Extensions = append(f.Extensions, ext) 272 | if err := p.readExtension(ext); err != nil { 273 | return err 274 | } 275 | ext.Up = f 276 | default: 277 | return p.errorf("unknown top-level thing %q", tok.value) 278 | } 279 | } 280 | 281 | // Handle comments. 282 | for len(p.comments) > 0 { 283 | n := 1 284 | for ; n < len(p.comments); n++ { 285 | if p.comments[n].line != p.comments[n-1].line+1 { 286 | break 287 | } 288 | } 289 | c := &ast.Comment{ 290 | Start: ast.Position{ 291 | Line: p.comments[0].line, 292 | Offset: p.comments[0].offset, 293 | }, 294 | End: ast.Position{ 295 | Line: p.comments[n-1].line, 296 | Offset: p.comments[n-1].offset, 297 | }, 298 | } 299 | for _, comm := range p.comments[:n] { 300 | c.Text = append(c.Text, comm.text) 301 | } 302 | p.comments = p.comments[n:] 303 | 304 | // Strip common whitespace prefix and any whitespace suffix. 305 | // TODO: this is a bodgy implementation of Longest Common Prefix, 306 | // and also doesn't do tabs vs. spaces well. 307 | var prefix string 308 | for i, line := range c.Text { 309 | line = strings.TrimRightFunc(line, unicode.IsSpace) 310 | c.Text[i] = line 311 | trim := len(line) - len(strings.TrimLeftFunc(line, unicode.IsSpace)) 312 | if i == 0 { 313 | prefix = line[:trim] 314 | } else { 315 | // Check how much of prefix is in common. 316 | for !strings.HasPrefix(line, prefix) { 317 | prefix = prefix[:len(prefix)-1] 318 | } 319 | } 320 | if prefix == "" { 321 | break 322 | } 323 | } 324 | if prefix != "" { 325 | for i, line := range c.Text { 326 | c.Text[i] = strings.TrimPrefix(line, prefix) 327 | } 328 | } 329 | 330 | f.Comments = append(f.Comments, c) 331 | } 332 | // No need to sort comments; they are already in source order. 333 | 334 | return nil 335 | } 336 | 337 | func (p *parser) readMessage(msg *ast.Message) *parseError { 338 | if err := p.readToken("message"); err != nil { 339 | return err 340 | } 341 | msg.Position = p.cur.astPosition() 342 | 343 | tok := p.next() 344 | if tok.err != nil { 345 | return tok.err 346 | } 347 | msg.Name = tok.value // TODO: validate 348 | 349 | if err := p.readToken("{"); err != nil { 350 | return err 351 | } 352 | 353 | if err := p.readMessageContents(msg); err != nil { 354 | return err 355 | } 356 | 357 | return p.readToken("}") 358 | } 359 | 360 | func (p *parser) readMessageContents(msg *ast.Message) *parseError { 361 | // Parse message fields and other things inside a message. 362 | var oneof *ast.Oneof // set while inside a oneof 363 | for !p.done { 364 | tok := p.next() 365 | if tok.err != nil { 366 | return tok.err 367 | } 368 | switch tok.value { 369 | case "extend": 370 | // extension 371 | p.back() 372 | ext := new(ast.Extension) 373 | msg.Extensions = append(msg.Extensions, ext) 374 | if err := p.readExtension(ext); err != nil { 375 | return err 376 | } 377 | ext.Up = msg 378 | case "oneof": 379 | // oneof 380 | if oneof != nil { 381 | return p.errorf("nested oneof not permitted") 382 | } 383 | oneof = new(ast.Oneof) 384 | msg.Oneofs = append(msg.Oneofs, oneof) 385 | oneof.Position = p.cur.astPosition() 386 | 387 | tok := p.next() 388 | if tok.err != nil { 389 | return tok.err 390 | } 391 | oneof.Name = tok.value // TODO: validate 392 | oneof.Up = msg 393 | 394 | if err := p.readToken("{"); err != nil { 395 | return err 396 | } 397 | case "message": 398 | // nested message 399 | p.back() 400 | nmsg := new(ast.Message) 401 | msg.Messages = append(msg.Messages, nmsg) 402 | if err := p.readMessage(nmsg); err != nil { 403 | return err 404 | } 405 | nmsg.Up = msg 406 | case "enum": 407 | // nested enum 408 | p.back() 409 | ne := new(ast.Enum) 410 | msg.Enums = append(msg.Enums, ne) 411 | if err := p.readEnum(ne); err != nil { 412 | return err 413 | } 414 | ne.Up = msg 415 | case "extensions": 416 | // extension range 417 | p.back() 418 | r, err := p.readExtensionRange() 419 | if err != nil { 420 | return err 421 | } 422 | msg.ExtensionRanges = append(msg.ExtensionRanges, r...) 423 | default: 424 | // field; this token is required/optional/repeated, 425 | // a primitive type, or a named type. 426 | p.back() 427 | field := new(ast.Field) 428 | msg.Fields = append(msg.Fields, field) 429 | field.Oneof = oneof 430 | field.Up = msg // p.readField uses this 431 | if err := p.readField(field); err != nil { 432 | return err 433 | } 434 | case "}": 435 | if oneof != nil { 436 | // end of oneof 437 | oneof = nil 438 | continue 439 | } 440 | // end of message 441 | p.back() 442 | return nil 443 | } 444 | } 445 | return p.errorf("unexpected EOF while parsing message") 446 | } 447 | 448 | func (p *parser) readField(f *ast.Field) *parseError { 449 | _, inMsg := f.Up.(*ast.Message) 450 | 451 | // TODO: enforce type limitations if f.Oneof != nil 452 | 453 | // look for required/optional/repeated 454 | tok := p.next() 455 | if tok.err != nil { 456 | return tok.err 457 | } 458 | f.Position = p.cur.astPosition() 459 | switch tok.value { 460 | case "required": 461 | f.Required = true 462 | case "optional": 463 | // nothing to do 464 | case "repeated": 465 | f.Repeated = true 466 | case "map": 467 | // map < Key , Value > 468 | if err := p.readToken("<"); err != nil { 469 | return err 470 | } 471 | tok = p.next() 472 | if tok.err != nil { 473 | return tok.err 474 | } 475 | f.KeyTypeName = tok.value // checked during resolution 476 | if err := p.readToken(","); err != nil { 477 | return err 478 | } 479 | tok = p.next() 480 | if tok.err != nil { 481 | return tok.err 482 | } 483 | f.TypeName = tok.value // checked during resolution 484 | if err := p.readToken(">"); err != nil { 485 | return err 486 | } 487 | f.Repeated = true // maps are repeated 488 | goto parseFromFieldName 489 | default: 490 | // assume this is a type name 491 | p.back() 492 | } 493 | 494 | tok = p.next() 495 | if tok.err != nil { 496 | return tok.err 497 | } 498 | f.TypeName = tok.value // checked during resolution 499 | 500 | parseFromFieldName: 501 | tok = p.next() 502 | if tok.err != nil { 503 | return tok.err 504 | } 505 | f.Name = tok.value // TODO: validate 506 | 507 | if err := p.readToken("="); err != nil { 508 | return err 509 | } 510 | 511 | tag, err := p.readTagNumber(false) 512 | if err != nil { 513 | return err 514 | } 515 | f.Tag = tag 516 | 517 | if f.TypeName == "group" && inMsg { 518 | if err := p.readToken("{"); err != nil { 519 | return err 520 | } 521 | 522 | group := &ast.Message{ 523 | // the current parse position is probably good enough 524 | Position: p.cur.astPosition(), 525 | Name: f.Name, 526 | Group: true, 527 | Up: f.Up, 528 | } 529 | if err := p.readMessageContents(group); err != nil { 530 | return err 531 | } 532 | f.TypeName = f.Name 533 | msg := f.Up.(*ast.Message) 534 | msg.Messages = append(msg.Messages, group) // ugh 535 | if err := p.readToken("}"); err != nil { 536 | return err 537 | } 538 | // A semicolon after a group is optional. 539 | if err := p.readToken(";"); err != nil { 540 | p.back() 541 | } 542 | return nil 543 | } 544 | 545 | if err := p.readToken("["); err == nil { 546 | p.back() 547 | if err := p.readFieldOptions(f); err != nil { 548 | return err 549 | } 550 | } else { 551 | p.back() 552 | } 553 | 554 | if err := p.readToken(";"); err != nil { 555 | return err 556 | } 557 | return nil 558 | } 559 | 560 | func (p *parser) readFieldOptions(f *ast.Field) *parseError { 561 | if err := p.readToken("["); err != nil { 562 | return err 563 | } 564 | for !p.done { 565 | tok := p.next() 566 | if tok.err != nil { 567 | return tok.err 568 | } 569 | // TODO: support more options than just default and packed 570 | switch tok.value { 571 | case "default": 572 | f.HasDefault = true 573 | if err := p.readToken("="); err != nil { 574 | return err 575 | } 576 | tok := p.next() 577 | if tok.err != nil { 578 | return tok.err 579 | } 580 | // TODO: check type 581 | switch f.TypeName { 582 | case "string": 583 | f.Default = tok.unquoted 584 | default: 585 | f.Default = tok.value 586 | } 587 | case "packed": 588 | f.HasPacked = true 589 | if err := p.readToken("="); err != nil { 590 | return err 591 | } 592 | packed, err := p.readBool() 593 | if err != nil { 594 | return err 595 | } 596 | f.Packed = packed 597 | default: 598 | return p.errorf(`got %q, want "default" or "packed"`, tok.value) 599 | } 600 | // next should be a comma or ] 601 | tok = p.next() 602 | if tok.err != nil { 603 | return tok.err 604 | } 605 | if tok.value == "," { 606 | continue 607 | } 608 | if tok.value == "]" { 609 | return nil 610 | } 611 | return p.errorf(`got %q, want "," or "]"`, tok.value) 612 | } 613 | return p.errorf("unexpected EOF while parsing field options") 614 | } 615 | 616 | func (p *parser) readExtensionRange() ([][2]int, *parseError) { 617 | if err := p.readToken("extensions"); err != nil { 618 | return nil, err 619 | } 620 | 621 | var rs [][2]int 622 | for { 623 | // next token must be a number, 624 | // followed by a comma, semicolon or "to". 625 | start, err := p.readTagNumber(false) 626 | if err != nil { 627 | return nil, err 628 | } 629 | end := start 630 | tok := p.next() 631 | if tok.err != nil { 632 | return nil, err 633 | } 634 | if tok.value == "to" { 635 | end, err = p.readTagNumber(true) // allow "max" 636 | if err != nil { 637 | return nil, err 638 | } 639 | if start > end { 640 | return nil, p.errorf("bad extension range order: %d > %d", start, end) 641 | } 642 | tok = p.next() 643 | if tok.err != nil { 644 | return nil, err 645 | } 646 | } 647 | rs = append(rs, [2]int{start, end}) 648 | if tok.value != "," && tok.value != ";" { 649 | return nil, p.errorf(`got %q, want ",", ";" or "to"`, tok.value) 650 | } 651 | if tok.value == ";" { 652 | break 653 | } 654 | } 655 | return rs, nil 656 | } 657 | 658 | func (p *parser) readTagNumber(allowMax bool) (int, *parseError) { 659 | tok := p.next() 660 | if tok.err != nil { 661 | return 0, tok.err 662 | } 663 | if allowMax && tok.value == "max" { 664 | return 1<<29 - 1, nil 665 | } 666 | n, err := strconv.ParseInt(tok.value, 10, 32) 667 | if err != nil { 668 | return 0, p.errorf("bad field number %q: %v", tok.value, err) 669 | } 670 | if n < 1 || n >= 1<<29 { 671 | return 0, p.errorf("field number %v out of range", n) 672 | } 673 | if 19000 <= n && n <= 19999 { // TODO: still relevant? 674 | return 0, p.errorf("field number %v in reserved range [19000, 19999]", n) 675 | } 676 | return int(n), nil 677 | } 678 | 679 | func (p *parser) readEnum(enum *ast.Enum) *parseError { 680 | if err := p.readToken("enum"); err != nil { 681 | return err 682 | } 683 | enum.Position = p.cur.astPosition() 684 | 685 | tok := p.next() 686 | if tok.err != nil { 687 | return tok.err 688 | } 689 | enum.Name = tok.value // TODO: validate 690 | 691 | if err := p.readToken("{"); err != nil { 692 | return err 693 | } 694 | 695 | // Parse enum values 696 | for !p.done { 697 | tok := p.next() 698 | if tok.err != nil { 699 | return tok.err 700 | } 701 | if tok.value == "}" { 702 | // end of enum 703 | // A semicolon after an enum is optional. 704 | if err := p.readToken(";"); err != nil { 705 | p.back() 706 | } 707 | return nil 708 | } 709 | // TODO: verify tok.value is a valid enum value name. 710 | ev := new(ast.EnumValue) 711 | enum.Values = append(enum.Values, ev) 712 | ev.Position = tok.astPosition() 713 | ev.Name = tok.value // TODO: validate 714 | ev.Up = enum 715 | 716 | if err := p.readToken("="); err != nil { 717 | return err 718 | } 719 | 720 | tok = p.next() 721 | if tok.err != nil { 722 | return tok.err 723 | } 724 | // TODO: check that tok.value is a valid enum value number. 725 | num, err := strconv.ParseInt(tok.value, 10, 32) 726 | if err != nil { 727 | return p.errorf("bad enum number %q: %v", tok.value, err) 728 | } 729 | ev.Number = int32(num) // TODO: validate 730 | 731 | if err := p.readToken(";"); err != nil { 732 | return err 733 | } 734 | } 735 | 736 | return p.errorf("unexpected EOF while parsing enum") 737 | } 738 | 739 | func (p *parser) readService(srv *ast.Service) *parseError { 740 | if err := p.readToken("service"); err != nil { 741 | return err 742 | } 743 | srv.Position = p.cur.astPosition() 744 | 745 | tok := p.next() 746 | if tok.err != nil { 747 | return tok.err 748 | } 749 | srv.Name = tok.value // TODO: validate 750 | 751 | if err := p.readToken("{"); err != nil { 752 | return err 753 | } 754 | 755 | // Parse methods 756 | for !p.done { 757 | tok := p.next() 758 | if tok.err != nil { 759 | return tok.err 760 | } 761 | switch tok.value { 762 | case "}": 763 | // end of service 764 | return nil 765 | case "rpc": 766 | // handled below 767 | default: 768 | return p.errorf(`got %q, want "rpc" or "}"`, tok.value) 769 | } 770 | 771 | tok = p.next() 772 | if tok.err != nil { 773 | return tok.err 774 | } 775 | mth := new(ast.Method) 776 | srv.Methods = append(srv.Methods, mth) 777 | mth.Position = tok.astPosition() 778 | mth.Name = tok.value // TODO: validate 779 | mth.Up = srv 780 | 781 | if err := p.readToken("("); err != nil { 782 | return err 783 | } 784 | 785 | tok = p.next() 786 | if tok.err != nil { 787 | return tok.err 788 | } 789 | mth.InTypeName = tok.value // TODO: validate 790 | if tok.value == "stream" { 791 | // If the next token isn't ")", this is a stream. 792 | tok = p.next() 793 | if tok.err != nil { 794 | return tok.err 795 | } 796 | if tok.value != ")" { 797 | mth.InTypeName = tok.value 798 | mth.ClientStreaming = true 799 | } else { 800 | p.back() 801 | } 802 | } 803 | if err := p.readToken(")"); err != nil { 804 | return err 805 | } 806 | if err := p.readToken("returns"); err != nil { 807 | return err 808 | } 809 | if err := p.readToken("("); err != nil { 810 | return err 811 | } 812 | tok = p.next() 813 | if tok.err != nil { 814 | return tok.err 815 | } 816 | mth.OutTypeName = tok.value // TODO: validate 817 | 818 | if tok.value == "stream" { 819 | // If the next token isn't ")", this is a stream. 820 | tok = p.next() 821 | if tok.err != nil { 822 | return tok.err 823 | } 824 | if tok.value != ")" { 825 | mth.OutTypeName = tok.value 826 | mth.ServerStreaming = true 827 | } else { 828 | p.back() 829 | } 830 | } 831 | if err := p.readToken(")"); err != nil { 832 | return err 833 | } 834 | if err := p.readToken(";"); err != nil { 835 | return err 836 | } 837 | } 838 | 839 | return p.errorf("unexpected EOF while parsing service") 840 | } 841 | 842 | func (p *parser) readExtension(ext *ast.Extension) *parseError { 843 | if err := p.readToken("extend"); err != nil { 844 | return err 845 | } 846 | ext.Position = p.cur.astPosition() 847 | 848 | tok := p.next() 849 | if tok.err != nil { 850 | return tok.err 851 | } 852 | ext.Extendee = tok.value // checked during resolution 853 | 854 | if err := p.readToken("{"); err != nil { 855 | return err 856 | } 857 | 858 | for !p.done { 859 | tok := p.next() 860 | if tok.err != nil { 861 | return tok.err 862 | } 863 | if tok.value == "}" { 864 | // end of extension 865 | return nil 866 | } 867 | p.back() 868 | field := new(ast.Field) 869 | ext.Fields = append(ext.Fields, field) 870 | field.Up = ext // p.readFile uses this 871 | if err := p.readField(field); err != nil { 872 | return err 873 | } 874 | } 875 | return p.errorf("unexpected EOF while parsing extension") 876 | } 877 | 878 | func (p *parser) readString() (*token, *parseError) { 879 | tok := p.next() 880 | if tok.err != nil { 881 | return nil, tok.err 882 | } 883 | if tok.value[0] != '"' { 884 | return nil, p.errorf("got %q, want string", tok.value) 885 | } 886 | return tok, nil 887 | } 888 | 889 | func (p *parser) readBool() (bool, *parseError) { 890 | tok := p.next() 891 | if tok.err != nil { 892 | return false, tok.err 893 | } 894 | // TODO: check which values for bools are valid. 895 | switch tok.value { 896 | case "true": 897 | return true, nil 898 | case "false": 899 | return false, nil 900 | default: 901 | return false, p.errorf(`got %q, want "true" or "false"`, tok.value) 902 | } 903 | } 904 | 905 | func (p *parser) readToken(want string) *parseError { 906 | tok := p.next() 907 | if tok.err != nil { 908 | return tok.err 909 | } 910 | if tok.value != want { 911 | return p.errorf("got %q, want %q", tok.value, want) 912 | } 913 | return nil 914 | } 915 | 916 | // Back off the parser by one token; may only be done between calls to p.next(). 917 | func (p *parser) back() { 918 | debugf("parser·back(): backed %q [err: %v]", p.cur.value, p.cur.err) 919 | p.done = false // in case this was the last token 920 | p.backed = true 921 | // In case an error was being recovered, ignore any error. 922 | // Don't do this for EOF, though, since we know that's what 923 | // we'll return next. 924 | if p.cur.err != eof { 925 | p.cur.err = nil // in case an error was being recovered 926 | } 927 | } 928 | 929 | // Advances the parser and returns the new current token. 930 | func (p *parser) next() *token { 931 | if p.backed || p.done { 932 | p.backed = false 933 | } else { 934 | p.advance() 935 | debugf("parser·next(): advanced to %q [err: %v]", p.cur.value, p.cur.err) 936 | if p.done && p.cur.err == nil { 937 | p.cur.value = "" 938 | p.cur.err = eof 939 | } 940 | } 941 | debugf("parser·next(): returning %q [err: %v]", p.cur.value, p.cur.err) 942 | return &p.cur 943 | } 944 | 945 | func (p *parser) advance() { 946 | // Skip whitespace 947 | p.skipWhitespaceAndComments() 948 | if p.done { 949 | return 950 | } 951 | 952 | // Start of non-whitespace 953 | p.cur.err = nil 954 | p.cur.offset, p.cur.line = p.offset, p.line 955 | switch p.s[0] { 956 | // TODO: more cases, like punctuation. 957 | case ';', '{', '}', '=', '[', ']', ',', '<', '>', '(', ')': 958 | // Single symbol 959 | p.cur.value, p.s = p.s[:1], p.s[1:] 960 | case '"', '\'': 961 | // Quoted string 962 | i := 1 963 | for i < len(p.s) && p.s[i] != p.s[0] { 964 | if p.s[i] == '\\' && i+1 < len(p.s) { 965 | // skip escaped character 966 | i++ 967 | } 968 | i++ 969 | } 970 | if i >= len(p.s) { 971 | p.errorf("encountered EOF inside string") 972 | return 973 | } 974 | i++ 975 | p.cur.value, p.s = p.s[:i], p.s[i:] 976 | // TODO: This doesn't work for single quote strings; 977 | // quotes will be mangled. 978 | unq, err := strconv.Unquote(p.cur.value) 979 | if err != nil { 980 | p.errorf("invalid quoted string [%s]: %v", p.cur.value, err) 981 | } 982 | p.cur.unquoted = unq 983 | default: 984 | i := 0 985 | for i < len(p.s) && isIdentOrNumberChar(p.s[i]) { 986 | i++ 987 | } 988 | if i == 0 { 989 | p.errorf("unexpected byte 0x%02x (%q)", p.s[0], string(p.s[:1])) 990 | return 991 | } 992 | p.cur.value, p.s = p.s[:i], p.s[i:] 993 | } 994 | p.offset += len(p.cur.value) 995 | } 996 | 997 | func (p *parser) skipWhitespaceAndComments() { 998 | i := 0 999 | for i < len(p.s) { 1000 | if isWhitespace(p.s[i]) { 1001 | if p.s[i] == '\n' { 1002 | p.line++ 1003 | } 1004 | i++ 1005 | continue 1006 | } 1007 | if i+1 < len(p.s) && p.s[i] == '/' && p.s[i+1] == '/' { 1008 | si := i + 2 1009 | c := comment{line: p.line, offset: p.offset + i} 1010 | // XXX: set c.text 1011 | // comment; skip to end of line or input 1012 | for i < len(p.s) && p.s[i] != '\n' { 1013 | i++ 1014 | } 1015 | c.text = p.s[si:i] 1016 | p.comments = append(p.comments, c) 1017 | if i < len(p.s) { 1018 | // end of line; keep going 1019 | p.line++ 1020 | i++ 1021 | continue 1022 | } 1023 | // end of input; fall out of loop 1024 | } 1025 | break 1026 | } 1027 | p.offset += i 1028 | p.s = p.s[i:] 1029 | if len(p.s) == 0 { 1030 | p.done = true 1031 | } 1032 | } 1033 | 1034 | func (p *parser) errorf(format string, a ...interface{}) *parseError { 1035 | pe := &parseError{ 1036 | message: fmt.Sprintf(format, a...), 1037 | filename: p.filename, 1038 | line: p.cur.line, 1039 | offset: p.cur.offset, 1040 | } 1041 | p.cur.err = pe 1042 | p.done = true 1043 | return pe 1044 | } 1045 | 1046 | func isWhitespace(c byte) bool { 1047 | // TODO: do more accurately 1048 | return unicode.IsSpace(rune(c)) 1049 | } 1050 | 1051 | // Numbers and identifiers are matched by [-+._A-Za-z0-9] 1052 | func isIdentOrNumberChar(c byte) bool { 1053 | switch { 1054 | case 'A' <= c && c <= 'Z', 'a' <= c && c <= 'z': 1055 | return true 1056 | case '0' <= c && c <= '9': 1057 | return true 1058 | } 1059 | switch c { 1060 | case '-', '+', '.', '_': 1061 | return true 1062 | } 1063 | return false 1064 | } 1065 | --------------------------------------------------------------------------------