├── .github ├── dependabot.yml └── workflows │ ├── install_dep.sh │ └── lint-and-test.yml ├── .gitignore ├── .idea ├── encodings.xml ├── go-protoparser.iml ├── misc.xml ├── modules.xml ├── vcs.xml └── watcherTasks.xml ├── LICENSE.md ├── Makefile ├── README.md ├── _example └── dump │ └── main.go ├── _testdata ├── bom.proto ├── cloudEndpoints.proto ├── double_semicolon_issue84 │ └── main.go ├── extend.proto ├── extension_declaration.proto ├── goProtoValidators.proto ├── grpc-gateway_a_bit_of_everything.proto ├── simple.proto ├── simpleWithComments.proto └── simplev2.proto ├── go.mod ├── internal └── util_test │ └── pretty.go ├── interpret └── unordered │ ├── enum.go │ ├── enum_test.go │ ├── extend.go │ ├── message.go │ ├── message_test.go │ ├── proto.go │ ├── proto_test.go │ ├── service.go │ └── service_test.go ├── lexer ├── constant.go ├── constant_test.go ├── emptyStatement.go ├── emptyStatement_test.go ├── enumType.go ├── enumType_test.go ├── error.go ├── fullIdent.go ├── fullIdent_test.go ├── lexer.go ├── messageType.go ├── messageType_test.go └── scanner │ ├── boolLit.go │ ├── comment.go │ ├── eof.go │ ├── error.go │ ├── floatLit.go │ ├── ident.go │ ├── lettersdigits.go │ ├── mode.go │ ├── numberLit.go │ ├── position.go │ ├── position_test.go │ ├── quote.go │ ├── scanner.go │ ├── scanner_test.go │ ├── strLit.go │ └── token.go ├── parser ├── comment.go ├── comment_test.go ├── declaration.go ├── declaration_test.go ├── edition.go ├── edition_test.go ├── emptyStatement.go ├── enum.go ├── enum_test.go ├── error.go ├── extend.go ├── extend_test.go ├── extensions.go ├── extensions_test.go ├── field.go ├── field_test.go ├── groupField.go ├── groupField_test.go ├── import.go ├── import_test.go ├── inlineComment.go ├── inlineComment_test.go ├── mapField.go ├── mapField_test.go ├── message.go ├── message_test.go ├── meta │ ├── error.go │ ├── meta.go │ ├── position.go │ └── position_test.go ├── oneof.go ├── oneof_test.go ├── option.go ├── option_test.go ├── package.go ├── package_test.go ├── parser.go ├── proto.go ├── proto_accept_test.go ├── proto_test.go ├── reserved.go ├── reserved_test.go ├── service.go ├── service_test.go ├── syntax.go ├── syntax_test.go ├── visitee.go └── visitor.go └── protoparser.go /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gomod" 4 | directory: "/" 5 | # Check for updates once a day 6 | schedule: 7 | interval: "daily" 8 | - package-ecosystem: "github-actions" 9 | directory: "/" 10 | # Check for updates once a week 11 | schedule: 12 | interval: "weekly" 13 | -------------------------------------------------------------------------------- /.github/workflows/install_dep.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euxo pipefail 4 | 5 | go install golang.org/x/tools/cmd/goimports@latest 6 | go install golang.org/x/lint/golint@latest 7 | go install github.com/kisielk/errcheck@latest 8 | go install github.com/gordonklaus/ineffassign@latest 9 | # I got Error: ../../../go/pkg/mod/golang.org/x/tools@v0.20.0/go/types/objectpath/objectpath.go:397:10: meth.Origin undefined (type *types.Func has no field or method Origin) 10 | # go install github.com/opennota/check/cmd/varcheck@latest 11 | # go install github.com/opennota/check/cmd/aligncheck@latest 12 | # Comment out because of the error: internal error: package "fmt" without types was imported from 13 | # go install github.com/mdempsky/unconvert@latest 14 | -------------------------------------------------------------------------------- /.github/workflows/lint-and-test.yml: -------------------------------------------------------------------------------- 1 | # Runs lint and tests on master and feature branches. 2 | name: Lint and test 3 | 4 | on: push 5 | 6 | jobs: 7 | lint: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Checkout 11 | uses: actions/checkout@v4 12 | - uses: actions/setup-go@v5 13 | with: 14 | go-version: '1.21' 15 | - name: Install dev deps 16 | run: make dev/install/dep 17 | - name: Lint 18 | run: make test/lint 19 | test: 20 | runs-on: ubuntu-latest 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@v4 24 | - uses: actions/setup-go@v5 25 | with: 26 | go-version: '1.21' 27 | - name: Run tests 28 | run: make test 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # user-specific .idea files 2 | .idea 3 | 4 | # CMake 5 | cmake-build-debug/ 6 | 7 | ## File-based project format: 8 | *.iws 9 | 10 | ## Plugin-specific files: 11 | 12 | # IntelliJ 13 | out/ 14 | 15 | # mpeltonen/sbt-idea plugin 16 | .idea_modules/ 17 | 18 | # JIRA plugin 19 | atlassian-ide-plugin.xml 20 | 21 | .config/ 22 | tags/ 23 | -------------------------------------------------------------------------------- /.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.idea/go-protoparser.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/watcherTasks.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 16 | 28 | 29 | 40 | 52 | 53 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 YOSHIMUTA YOHEI 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ## test/all runs all related tests. 2 | test/all: test/lint test 3 | 4 | ## test runs `go test` 5 | test: 6 | go test -v -p 2 -count 1 -timeout 240s -race ./... 7 | 8 | ## test runs `go test -run $(RUN)` 9 | test/run: 10 | go test -v -p 2 -count 1 -timeout 240s -race ./... -run $(RUN) 11 | 12 | ## test/lint runs linter 13 | test/lint: 14 | # checks the coding style. 15 | (! gofmt -s -d `find . -name vendor -prune -type f -o -name '*.go'` | grep '^') 16 | golint -set_exit_status `go list ./...` 17 | # checks the import format. 18 | (! goimports -l `find . -name vendor -prune -type f -o -name '*.go'` | grep 'go') 19 | # checks the error the compiler can't find. 20 | go vet ./... 21 | # checks shadowed variables. 22 | go vet -vettool=$(which shadow) ./... 23 | # checks not to ignore the error. 24 | errcheck ./... 25 | # checks unused global variables and constants. 26 | # varcheck ./... 27 | # checks no used assigned value. 28 | ineffassign . 29 | # checks dispensable type conversions. 30 | ## Comment out because of the error: internal error: package "fmt" without types was imported from 31 | # unconvert -v ./... 32 | 33 | ## dev/install/dep installs depenencies required for development. 34 | dev/install/dep: 35 | ./.github/workflows/install_dep.sh 36 | 37 | ## RUN_EXAMPLE_DEBUG is a debug flag argument for run/example. 38 | RUN_EXAMPLE_DEBUG=false 39 | 40 | ## RUN_EXAMPLE_PERMISSIVE is a permissive flag argument for run/example. 41 | RUN_EXAMPLE_PERMISSIVE=true 42 | 43 | ## RUN_EXAMPLE_UNORDERED is an unordered flag argument for run/example. 44 | RUN_EXAMPLE_UNORDERED=false 45 | 46 | ## run/dump/example runs `go run _example/dump/main.go` 47 | run/dump/example: 48 | go run _example/dump/main.go -debug=$(RUN_EXAMPLE_DEBUG) -permissive=$(RUN_EXAMPLE_PERMISSIVE) -unordered=${RUN_EXAMPLE_UNORDERED} 49 | -------------------------------------------------------------------------------- /_example/dump/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "fmt" 7 | "os" 8 | 9 | "path/filepath" 10 | 11 | protoparser "github.com/yoheimuta/go-protoparser/v4" 12 | ) 13 | 14 | var ( 15 | proto = flag.String("proto", "_testdata/simple.proto", "path to the Protocol Buffer file") 16 | debug = flag.Bool("debug", false, "debug flag to output more parsing process detail") 17 | permissive = flag.Bool("permissive", true, "permissive flag to allow the permissive parsing rather than the just documented spec") 18 | unordered = flag.Bool("unordered", false, "unordered flag to output another one without interface{}") 19 | ) 20 | 21 | func run() int { 22 | flag.Parse() 23 | 24 | reader, err := os.Open(*proto) 25 | if err != nil { 26 | fmt.Fprintf(os.Stderr, "failed to open %s, err %v\n", *proto, err) 27 | return 1 28 | } 29 | defer func() { 30 | if err := reader.Close(); err != nil { 31 | fmt.Printf("Error closing file: %s\n", err) 32 | } 33 | }() 34 | 35 | got, err := protoparser.Parse( 36 | reader, 37 | protoparser.WithDebug(*debug), 38 | protoparser.WithPermissive(*permissive), 39 | protoparser.WithFilename(filepath.Base(*proto)), 40 | ) 41 | if err != nil { 42 | fmt.Fprintf(os.Stderr, "failed to parse, err %v\n", err) 43 | return 1 44 | } 45 | 46 | var v interface{} 47 | v = got 48 | if *unordered { 49 | v, err = protoparser.UnorderedInterpret(got) 50 | if err != nil { 51 | fmt.Fprintf(os.Stderr, "failed to interpret, err %v\n", err) 52 | return 1 53 | } 54 | } 55 | 56 | gotJSON, err := json.MarshalIndent(v, "", " ") 57 | if err != nil { 58 | fmt.Fprintf(os.Stderr, "failed to marshal, err %v\n", err) 59 | } 60 | fmt.Print(string(gotJSON)) 61 | return 0 62 | } 63 | 64 | func main() { 65 | os.Exit(run()) 66 | } 67 | -------------------------------------------------------------------------------- /_testdata/bom.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | -------------------------------------------------------------------------------- /_testdata/cloudEndpoints.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Google LLC All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package stable.agones.dev.sdk; 18 | option go_package = "sdk"; 19 | 20 | import "google/api/annotations.proto"; 21 | 22 | // SDK service to be used in the GameServer SDK to the Pod Sidecar 23 | service SDK { 24 | // Call when the GameServer is ready 25 | rpc Ready (Empty) returns (Empty) { 26 | option (google.api.http) = { 27 | post: "/ready" 28 | body: "*" 29 | }; 30 | } 31 | 32 | // Call to self Allocation the GameServer 33 | rpc Allocate(Empty) returns (Empty) { 34 | option (google.api.http) = { 35 | post: "/allocate" 36 | body: "*" 37 | }; 38 | } 39 | 40 | // Call when the GameServer is shutting down 41 | rpc Shutdown (Empty) returns (Empty) { 42 | option (google.api.http) = { 43 | post: "/shutdown" 44 | body: "*" 45 | }; 46 | } 47 | // Send a Empty every d Duration to declare that this GameSever is healthy 48 | rpc Health (stream Empty) returns (Empty) { 49 | option (google.api.http) = { 50 | post: "/health" 51 | body: "*" 52 | }; 53 | } 54 | // Retrieve the current GameServer data 55 | rpc GetGameServer (Empty) returns (GameServer) { 56 | option (google.api.http) = { 57 | get: "/gameserver" 58 | }; 59 | } 60 | // Send GameServer details whenever the GameServer is updated 61 | rpc WatchGameServer (Empty) returns (stream GameServer) { 62 | option (google.api.http) = { 63 | get: "/watch/gameserver" 64 | }; 65 | } 66 | 67 | // Apply a Label to the backing GameServer metadata 68 | rpc SetLabel(KeyValue) returns (Empty) { 69 | option (google.api.http) = { 70 | put: "/metadata/label" 71 | body: "*" 72 | }; 73 | } 74 | 75 | // Apply a Annotation to the backing GameServer metadata 76 | rpc SetAnnotation(KeyValue) returns (Empty) { 77 | option (google.api.http) = { 78 | put: "/metadata/annotation" 79 | body: "*" 80 | }; 81 | } 82 | } 83 | 84 | // I am Empty 85 | message Empty { 86 | } 87 | 88 | // Key, Value entry 89 | message KeyValue { 90 | string key = 1; 91 | string value = 2; 92 | } 93 | 94 | // A GameServer Custom Resource Definition object 95 | // We will only export those resources that make the most 96 | // sense. Can always expand to more as needed. 97 | message GameServer { 98 | ObjectMeta object_meta = 1; 99 | Spec spec = 2; 100 | Status status = 3; 101 | 102 | // representation of the K8s ObjectMeta resource 103 | message ObjectMeta { 104 | string name = 1; 105 | string namespace = 2; 106 | string uid = 3; 107 | string resource_version = 4; 108 | int64 generation = 5; 109 | // timestamp is in Epoch format, unit: seconds 110 | int64 creation_timestamp = 6; 111 | // optional deletion timestamp in Epoch format, unit: seconds 112 | int64 deletion_timestamp = 7; 113 | map annotations = 8; 114 | map labels = 9; 115 | } 116 | 117 | message Spec { 118 | Health health = 1; 119 | 120 | message Health { 121 | bool Disabled = 1; 122 | int32 PeriodSeconds = 2; 123 | int32 FailureThreshold = 3; 124 | int32 InitialDelaySeconds = 4; 125 | } 126 | } 127 | 128 | message Status { 129 | message Port { 130 | string name = 1; 131 | int32 port = 2; 132 | } 133 | 134 | string state = 1; 135 | string address = 2; 136 | repeated Port ports = 3; 137 | } 138 | } -------------------------------------------------------------------------------- /_testdata/double_semicolon_issue84/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | 8 | "github.com/yoheimuta/go-protoparser/v4/lexer" 9 | "github.com/yoheimuta/go-protoparser/v4/parser" 10 | ) 11 | 12 | // SimpleVisitor is a simple implementation of the parser.Visitor interface. 13 | type SimpleVisitor struct{} 14 | 15 | func (v *SimpleVisitor) VisitComment(*parser.Comment) {} 16 | func (v *SimpleVisitor) VisitDeclaration(*parser.Declaration) bool { 17 | return true 18 | } 19 | func (v *SimpleVisitor) VisitEdition(*parser.Edition) bool { 20 | return true 21 | } 22 | func (v *SimpleVisitor) VisitEmptyStatement(*parser.EmptyStatement) bool { 23 | return true 24 | } 25 | func (v *SimpleVisitor) VisitEnum(*parser.Enum) bool { 26 | return true 27 | } 28 | func (v *SimpleVisitor) VisitEnumField(*parser.EnumField) bool { 29 | return true 30 | } 31 | func (v *SimpleVisitor) VisitExtend(*parser.Extend) bool { 32 | return true 33 | } 34 | func (v *SimpleVisitor) VisitExtensions(*parser.Extensions) bool { 35 | return true 36 | } 37 | func (v *SimpleVisitor) VisitField(*parser.Field) bool { 38 | return true 39 | } 40 | func (v *SimpleVisitor) VisitGroupField(*parser.GroupField) bool { 41 | return true 42 | } 43 | func (v *SimpleVisitor) VisitImport(*parser.Import) bool { 44 | return true 45 | } 46 | func (v *SimpleVisitor) VisitMapField(*parser.MapField) bool { 47 | return true 48 | } 49 | func (v *SimpleVisitor) VisitMessage(*parser.Message) bool { 50 | return true 51 | } 52 | func (v *SimpleVisitor) VisitOneof(*parser.Oneof) bool { 53 | return true 54 | } 55 | func (v *SimpleVisitor) VisitOneofField(*parser.OneofField) bool { 56 | return true 57 | } 58 | func (v *SimpleVisitor) VisitOption(*parser.Option) bool { 59 | return true 60 | } 61 | func (v *SimpleVisitor) VisitPackage(*parser.Package) bool { 62 | return true 63 | } 64 | func (v *SimpleVisitor) VisitReserved(*parser.Reserved) bool { 65 | return true 66 | } 67 | func (v *SimpleVisitor) VisitRPC(*parser.RPC) bool { 68 | return true 69 | } 70 | func (v *SimpleVisitor) VisitService(*parser.Service) bool { 71 | return true 72 | } 73 | func (v *SimpleVisitor) VisitSyntax(*parser.Syntax) bool { 74 | return true 75 | } 76 | 77 | func main() { 78 | // Proto file with double semicolon in service option 79 | input := ` 80 | syntax = "proto3"; 81 | import "google/protobuf/descriptor.proto"; 82 | 83 | extend google.protobuf.ServiceOptions { 84 | string service_description = 51000; 85 | } 86 | 87 | service MyService{ 88 | option(service_description) = "description";; 89 | } 90 | ` 91 | 92 | // Parse the proto file 93 | p := parser.NewParser(lexer.NewLexer(strings.NewReader(input))) 94 | proto, err := p.ParseProto() 95 | if err != nil { 96 | fmt.Fprintf(os.Stderr, "Failed to parse proto: %v\n", err) 97 | os.Exit(1) 98 | } 99 | 100 | // Use the visitor pattern to process the parsed proto 101 | visitor := &SimpleVisitor{} 102 | proto.Accept(visitor) 103 | 104 | fmt.Println("Successfully processed proto with double semicolon") 105 | } 106 | -------------------------------------------------------------------------------- /_testdata/extend.proto: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015, Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package google.api; 18 | 19 | import "google/api/http.proto"; 20 | import "google/protobuf/descriptor.proto"; 21 | 22 | option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; 23 | option java_multiple_files = true; 24 | option java_outer_classname = "AnnotationsProto"; 25 | option java_package = "com.google.api"; 26 | option objc_class_prefix = "GAPI"; 27 | 28 | extend google.protobuf.MethodOptions { 29 | // See `HttpRule`. 30 | HttpRule http = 72295728; 31 | } -------------------------------------------------------------------------------- /_testdata/extension_declaration.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | message Foo { 4 | extensions 4 to 1000 [ 5 | declaration = { 6 | number: 4, 7 | full_name: ".my.package.event_annotations", 8 | type: ".logs.proto.ValidationAnnotations", 9 | repeated: true }, 10 | declaration = { 11 | number: 999, 12 | full_name: ".foo.package.bar", 13 | type: "int32"}]; 14 | } 15 | 16 | message Bar { 17 | extensions 1000 to 2000 [ 18 | declaration = { 19 | number: 1000, 20 | full_name: ".foo.package", 21 | type: ".foo.type" 22 | } 23 | ]; 24 | } 25 | -------------------------------------------------------------------------------- /_testdata/goProtoValidators.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package parserpb; 4 | option go_package = "github.com/yoheimuta/apis/v1/parser/parserpb"; 5 | 6 | import "google/protobuf/empty.proto"; 7 | import "github.com/mwitkow/go-proto-validators/validator.proto"; 8 | import "google/protobuf/timestamp.proto"; 9 | 10 | import "github.com/yoheimuta/apis/v1/entities/entities.proto"; 11 | import "github.com/yoheimuta/apis/v1/entities/aggregates.proto"; 12 | import "github.com/yoheimuta/apis/v1/values/itemContentCondition.proto"; 13 | import "github.com/yoheimuta/apis/v1/values/itemNoContentCondition.proto"; 14 | 15 | // ItemService is a service to manage items. 16 | service ItemService { 17 | // CreateUserItem is a method to create a user's item. 18 | rpc CreateUserItem(CreateUserItemRequest) returns (aggregatespb.UserItemAggregate) {} 19 | 20 | // UpdateUserItem is a method to update a user's item. 21 | rpc UpdateUserItem(UpdateUserItemRequest) returns (entitiespb.UserItem) {} 22 | } 23 | 24 | // CreateUserItemRequest is a request message for CreateUserItem. 25 | message CreateUserItemRequest { 26 | // Image is an item's image information for create 27 | message Image { 28 | // display_order is an order of position. Starts 1 at left and increment by one. Required. 29 | int64 display_order = 1 [(validator.field) = {int_gt: 0}]; 30 | // binary is an image binary. Required. 31 | bytes binary = 2 [(validator.field) = {length_gt: 0}]; 32 | } 33 | // Mapping is 34 | // an information of an item mapping. 35 | message Mapping { 36 | // product is an item master information. 37 | entitiespb.UserItemMappingProduct product = 1; 38 | } 39 | 40 | // item is an item entity. Required. 41 | entitiespb.UserItem item = 1 [(validator.field) = {msg_exists : true}]; 42 | // images are item's images. Max count is 10. Optional. 43 | repeated Image images = 2 [(validator.field) = {repeated_count_max: 10}]; 44 | // mapping is a item's mapping information. Required. 45 | Mapping mapping = 3 [(validator.field) = {msg_exists : true}]; 46 | // condition_oneof is an item's condition. Required. 47 | oneof condition_oneof { 48 | // content_type_id is a condition ID of an item with content. 49 | itemContentConditionpb.Type content_type_id = 4; 50 | // no_content_type_id is a condition ID of an item without content. 51 | itemNoContentConditionpb.Type no_content_type_id = 5; 52 | } 53 | } 54 | 55 | -------------------------------------------------------------------------------- /_testdata/simple.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | // An example of the official reference 3 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#proto_file 4 | package examplepb; 5 | import public "other.proto"; 6 | option java_package = "com.example.foo"; 7 | enum EnumAllowingAlias { 8 | option allow_alias = true; 9 | UNKNOWN = 0; 10 | STARTED = 1; 11 | RUNNING = 2 [(custom_option) = "this is a " 12 | "string on two lines" 13 | ]; 14 | } 15 | message outer { 16 | option (my_option).a = true; 17 | message inner { // Level 2 18 | int64 ival = 1; 19 | } 20 | repeated inner inner_message = 2; 21 | EnumAllowingAlias enum_field =3; 22 | map my_map = 4; 23 | } 24 | service HelloService { 25 | rpc SayHello (HelloRequest) returns (HelloResponse) {}; 26 | } -------------------------------------------------------------------------------- /_testdata/simpleWithComments.proto: -------------------------------------------------------------------------------- 1 | // An example of the official reference with additional comments. 2 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#proto_file 3 | syntax = "proto3"; 4 | // Imports other.proto 5 | import public "other.proto"; 6 | // java_package option 7 | option java_package = "com.example.foo"; 8 | // EnumAllowingAlias is an alias 9 | enum EnumAllowingAlias { 10 | // allow_alias option is true 11 | option allow_alias = true; 12 | UNKNOWN = 0; 13 | STARTED = 1; 14 | RUNNING = 2 [(custom_option) = "hello world"]; 15 | } 16 | // outer message 17 | message outer { 18 | option (my_option).a = true; 19 | // inner message 20 | message inner { 21 | int64 ival = 1; 22 | } 23 | repeated inner inner_message = 2; 24 | EnumAllowingAlias enum_field =3; 25 | map my_map = 4; 26 | } 27 | -------------------------------------------------------------------------------- /_testdata/simplev2.proto: -------------------------------------------------------------------------------- 1 | // An example of the official reference 2 | // See https://developers.google.com/protocol-buffers/docs/reference/proto2-spec#proto_file 3 | syntax = "proto2"; 4 | import public "other.proto"; 5 | option java_package = "com.example.foo"; 6 | enum EnumAllowingAlias { 7 | option allow_alias = true; 8 | UNKNOWN = 0; 9 | STARTED = 1; 10 | RUNNING = 2 [(custom_option) = "hello world"]; 11 | } 12 | message outer { 13 | option (my_option).a = true; 14 | message inner { // Level 2 15 | required int64 ival = 1; 16 | } 17 | repeated inner inner_message = 2; 18 | optional EnumAllowingAlias enum_field = 3; 19 | map my_map = 4; 20 | extensions 20 to 30; 21 | } 22 | message foo { 23 | optional group GroupMessage = 1 { 24 | optional int64 a = 1; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/yoheimuta/go-protoparser/v4 2 | 3 | go 1.13 4 | -------------------------------------------------------------------------------- /internal/util_test/pretty.go: -------------------------------------------------------------------------------- 1 | package util_test 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | ) 7 | 8 | // PrettyFormat formats any objects for test debug use.. 9 | func PrettyFormat(v interface{}) string { 10 | b, err := json.MarshalIndent(v, "", " ") 11 | if err != nil { 12 | return fmt.Sprintf("orig=%v, err=%v", v, err) 13 | } 14 | return string(b) 15 | } 16 | -------------------------------------------------------------------------------- /interpret/unordered/enum.go: -------------------------------------------------------------------------------- 1 | package unordered 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/yoheimuta/go-protoparser/v4/parser" 7 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 8 | ) 9 | 10 | // EnumBody is unordered in nature, but each slice field preserves the original order. 11 | type EnumBody struct { 12 | Options []*parser.Option 13 | EnumFields []*parser.EnumField 14 | Reserveds []*parser.Reserved 15 | EmptyStatements []*parser.EmptyStatement 16 | } 17 | 18 | // Enum consists of a name and an enum body. 19 | type Enum struct { 20 | EnumName string 21 | EnumBody *EnumBody 22 | 23 | // Comments are the optional ones placed at the beginning. 24 | Comments []*parser.Comment 25 | // InlineComment is the optional one placed at the ending. 26 | InlineComment *parser.Comment 27 | // InlineCommentBehindLeftCurly is the optional one placed behind a left curly. 28 | InlineCommentBehindLeftCurly *parser.Comment 29 | // Meta is the meta information. 30 | Meta meta.Meta 31 | } 32 | 33 | // InterpretEnum interprets *parser.Enum to *Enum. 34 | func InterpretEnum(src *parser.Enum) (*Enum, error) { 35 | if src == nil { 36 | return nil, nil 37 | } 38 | 39 | enumBody, err := interpretEnumBody(src.EnumBody) 40 | if err != nil { 41 | return nil, fmt.Errorf("invalid Enum %s: %w", src.EnumName, err) 42 | } 43 | return &Enum{ 44 | EnumName: src.EnumName, 45 | EnumBody: enumBody, 46 | Comments: src.Comments, 47 | InlineComment: src.InlineComment, 48 | InlineCommentBehindLeftCurly: src.InlineCommentBehindLeftCurly, 49 | Meta: src.Meta, 50 | }, nil 51 | } 52 | 53 | func interpretEnumBody(src []parser.Visitee) ( 54 | *EnumBody, 55 | error, 56 | ) { 57 | var options []*parser.Option 58 | var enumFields []*parser.EnumField 59 | var reserveds []*parser.Reserved 60 | var emptyStatements []*parser.EmptyStatement 61 | for _, s := range src { 62 | switch t := s.(type) { 63 | case *parser.Option: 64 | options = append(options, t) 65 | case *parser.EnumField: 66 | enumFields = append(enumFields, t) 67 | case *parser.Reserved: 68 | reserveds = append(reserveds, t) 69 | case *parser.EmptyStatement: 70 | emptyStatements = append(emptyStatements, t) 71 | default: 72 | return nil, fmt.Errorf("invalid EnumBody type %T of %v", t, t) 73 | } 74 | } 75 | return &EnumBody{ 76 | Options: options, 77 | EnumFields: enumFields, 78 | Reserveds: reserveds, 79 | EmptyStatements: emptyStatements, 80 | }, nil 81 | } 82 | -------------------------------------------------------------------------------- /interpret/unordered/enum_test.go: -------------------------------------------------------------------------------- 1 | package unordered_test 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/yoheimuta/go-protoparser/v4/interpret/unordered" 8 | "github.com/yoheimuta/go-protoparser/v4/parser" 9 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 10 | ) 11 | 12 | func TestInterpretEnum(t *testing.T) { 13 | tests := []struct { 14 | name string 15 | inputEnum *parser.Enum 16 | wantEnum *unordered.Enum 17 | wantErr bool 18 | }{ 19 | { 20 | name: "interpreting a nil", 21 | }, 22 | { 23 | name: "interpreting an excerpt from the official reference with comments and reserved", 24 | inputEnum: &parser.Enum{ 25 | EnumName: "EnumAllowingAlias", 26 | EnumBody: []parser.Visitee{ 27 | &parser.Option{ 28 | OptionName: "allow_alias", 29 | Constant: "true", 30 | }, 31 | &parser.EnumField{ 32 | Ident: "UNKNOWN", 33 | Number: "0", 34 | }, 35 | &parser.EnumField{ 36 | Ident: "STARTED", 37 | Number: "1", 38 | }, 39 | &parser.EnumField{ 40 | Ident: "RUNNING", 41 | Number: "2", 42 | EnumValueOptions: []*parser.EnumValueOption{ 43 | { 44 | OptionName: "(custom_option)", 45 | Constant: `"hello world"`, 46 | }, 47 | }, 48 | }, 49 | &parser.Reserved{ 50 | FieldNames: []string{ 51 | `"FOO"`, 52 | `"BAR"`, 53 | }, 54 | }, 55 | }, 56 | Comments: []*parser.Comment{ 57 | { 58 | Raw: "// enum", 59 | }, 60 | }, 61 | InlineComment: &parser.Comment{ 62 | Raw: "// TODO: implementation", 63 | Meta: meta.Meta{ 64 | Pos: meta.Position{ 65 | Offset: 25, 66 | Line: 2, 67 | Column: 26, 68 | }, 69 | }, 70 | }, 71 | InlineCommentBehindLeftCurly: &parser.Comment{ 72 | Raw: "// TODO: implementation2", 73 | Meta: meta.Meta{ 74 | Pos: meta.Position{ 75 | Offset: 25, 76 | Line: 1, 77 | Column: 26, 78 | }, 79 | }, 80 | }, 81 | Meta: meta.Meta{ 82 | Pos: meta.Position{ 83 | Offset: 21, 84 | Line: 3, 85 | Column: 1, 86 | }, 87 | }, 88 | }, 89 | wantEnum: &unordered.Enum{ 90 | EnumName: "EnumAllowingAlias", 91 | EnumBody: &unordered.EnumBody{ 92 | Options: []*parser.Option{ 93 | { 94 | OptionName: "allow_alias", 95 | Constant: "true", 96 | }, 97 | }, 98 | EnumFields: []*parser.EnumField{ 99 | { 100 | Ident: "UNKNOWN", 101 | Number: "0", 102 | }, 103 | { 104 | Ident: "STARTED", 105 | Number: "1", 106 | }, 107 | { 108 | Ident: "RUNNING", 109 | Number: "2", 110 | EnumValueOptions: []*parser.EnumValueOption{ 111 | { 112 | OptionName: "(custom_option)", 113 | Constant: `"hello world"`, 114 | }, 115 | }, 116 | }, 117 | }, 118 | Reserveds: []*parser.Reserved{ 119 | { 120 | FieldNames: []string{ 121 | `"FOO"`, 122 | `"BAR"`, 123 | }, 124 | }, 125 | }, 126 | }, 127 | Comments: []*parser.Comment{ 128 | { 129 | Raw: "// enum", 130 | }, 131 | }, 132 | InlineComment: &parser.Comment{ 133 | Raw: "// TODO: implementation", 134 | Meta: meta.Meta{ 135 | Pos: meta.Position{ 136 | Offset: 25, 137 | Line: 2, 138 | Column: 26, 139 | }, 140 | }, 141 | }, 142 | InlineCommentBehindLeftCurly: &parser.Comment{ 143 | Raw: "// TODO: implementation2", 144 | Meta: meta.Meta{ 145 | Pos: meta.Position{ 146 | Offset: 25, 147 | Line: 1, 148 | Column: 26, 149 | }, 150 | }, 151 | }, 152 | Meta: meta.Meta{ 153 | Pos: meta.Position{ 154 | Offset: 21, 155 | Line: 3, 156 | Column: 1, 157 | }, 158 | }, 159 | }, 160 | }, 161 | } 162 | 163 | for _, test := range tests { 164 | test := test 165 | t.Run(test.name, func(t *testing.T) { 166 | got, err := unordered.InterpretEnum(test.inputEnum) 167 | switch { 168 | case test.wantErr: 169 | if err == nil { 170 | t.Errorf("got err nil, but want err, parsed=%v", got) 171 | } 172 | return 173 | case !test.wantErr && err != nil: 174 | t.Errorf("got err %v, but want nil", err) 175 | return 176 | } 177 | 178 | if !reflect.DeepEqual(got, test.wantEnum) { 179 | t.Errorf("got %v, but want %v", got, test.wantEnum) 180 | } 181 | }) 182 | } 183 | 184 | } 185 | -------------------------------------------------------------------------------- /interpret/unordered/extend.go: -------------------------------------------------------------------------------- 1 | package unordered 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/yoheimuta/go-protoparser/v4/parser" 7 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 8 | ) 9 | 10 | // ExtendBody is unordered in nature, but each slice field preserves the original order. 11 | type ExtendBody struct { 12 | Fields []*parser.Field 13 | EmptyStatements []*parser.EmptyStatement 14 | } 15 | 16 | // Extend consists of a messageType and a extend body. 17 | type Extend struct { 18 | MessageType string 19 | ExtendBody *ExtendBody 20 | 21 | // Comments are the optional ones placed at the beginning. 22 | Comments []*parser.Comment 23 | // InlineComment is the optional one placed at the ending. 24 | InlineComment *parser.Comment 25 | // InlineCommentBehindLeftCurly is the optional one placed behind a left curly. 26 | InlineCommentBehindLeftCurly *parser.Comment 27 | // Meta is the meta information. 28 | Meta meta.Meta 29 | } 30 | 31 | // InterpretExtend interprets *parser.Extend to *Extend. 32 | func InterpretExtend(src *parser.Extend) (*Extend, error) { 33 | if src == nil { 34 | return nil, nil 35 | } 36 | 37 | extendBody, err := interpretExtendBody(src.ExtendBody) 38 | if err != nil { 39 | return nil, err 40 | } 41 | return &Extend{ 42 | MessageType: src.MessageType, 43 | ExtendBody: extendBody, 44 | Comments: src.Comments, 45 | InlineComment: src.InlineComment, 46 | InlineCommentBehindLeftCurly: src.InlineCommentBehindLeftCurly, 47 | Meta: src.Meta, 48 | }, nil 49 | } 50 | 51 | func interpretExtendBody(src []parser.Visitee) ( 52 | *ExtendBody, 53 | error, 54 | ) { 55 | var fields []*parser.Field 56 | var emptyStatements []*parser.EmptyStatement 57 | for _, s := range src { 58 | switch t := s.(type) { 59 | case *parser.Field: 60 | fields = append(fields, t) 61 | case *parser.EmptyStatement: 62 | emptyStatements = append(emptyStatements, t) 63 | default: 64 | return nil, fmt.Errorf("invalid ExtendBody type %T of %v", t, t) 65 | } 66 | } 67 | return &ExtendBody{ 68 | Fields: fields, 69 | EmptyStatements: emptyStatements, 70 | }, nil 71 | } 72 | -------------------------------------------------------------------------------- /interpret/unordered/message.go: -------------------------------------------------------------------------------- 1 | package unordered 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/yoheimuta/go-protoparser/v4/parser" 7 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 8 | ) 9 | 10 | // MessageBody is unordered in nature, but each slice field preserves the original order. 11 | type MessageBody struct { 12 | Fields []*parser.Field 13 | Enums []*Enum 14 | Messages []*Message 15 | Options []*parser.Option 16 | Oneofs []*parser.Oneof 17 | Maps []*parser.MapField 18 | Groups []*parser.GroupField 19 | Reserves []*parser.Reserved 20 | Extends []*parser.Extend 21 | EmptyStatements []*parser.EmptyStatement 22 | Extensions []*parser.Extensions 23 | } 24 | 25 | // Message consists of a message name and a message body. 26 | type Message struct { 27 | MessageName string 28 | MessageBody *MessageBody 29 | 30 | // Comments are the optional ones placed at the beginning. 31 | Comments []*parser.Comment 32 | // InlineComment is the optional one placed at the ending. 33 | InlineComment *parser.Comment 34 | // InlineCommentBehindLeftCurly is the optional one placed behind a left curly. 35 | InlineCommentBehindLeftCurly *parser.Comment 36 | // Meta is the meta information. 37 | Meta meta.Meta 38 | } 39 | 40 | // InterpretMessage interprets *parser.Message to *Message. 41 | func InterpretMessage(src *parser.Message) (*Message, error) { 42 | if src == nil { 43 | return nil, nil 44 | } 45 | 46 | messageBody, err := interpretMessageBody(src.MessageBody) 47 | if err != nil { 48 | return nil, fmt.Errorf("invalid Message %s: %w", src.MessageName, err) 49 | } 50 | return &Message{ 51 | MessageName: src.MessageName, 52 | MessageBody: messageBody, 53 | Comments: src.Comments, 54 | InlineComment: src.InlineComment, 55 | InlineCommentBehindLeftCurly: src.InlineCommentBehindLeftCurly, 56 | Meta: src.Meta, 57 | }, nil 58 | } 59 | 60 | func interpretMessageBody(src []parser.Visitee) ( 61 | *MessageBody, 62 | error, 63 | ) { 64 | var fields []*parser.Field 65 | var enums []*Enum 66 | var messages []*Message 67 | var options []*parser.Option 68 | var oneofs []*parser.Oneof 69 | var maps []*parser.MapField 70 | var groups []*parser.GroupField 71 | var reserves []*parser.Reserved 72 | var extends []*parser.Extend 73 | var emptyStatements []*parser.EmptyStatement 74 | var extensions []*parser.Extensions 75 | for _, s := range src { 76 | switch t := s.(type) { 77 | case *parser.Field: 78 | fields = append(fields, t) 79 | case *parser.Enum: 80 | enum, err := InterpretEnum(t) 81 | if err != nil { 82 | return nil, err 83 | } 84 | enums = append(enums, enum) 85 | case *parser.Message: 86 | message, err := InterpretMessage(t) 87 | if err != nil { 88 | return nil, err 89 | } 90 | messages = append(messages, message) 91 | case *parser.Option: 92 | options = append(options, t) 93 | case *parser.Oneof: 94 | oneofs = append(oneofs, t) 95 | case *parser.MapField: 96 | maps = append(maps, t) 97 | case *parser.GroupField: 98 | groups = append(groups, t) 99 | case *parser.Reserved: 100 | reserves = append(reserves, t) 101 | case *parser.Extend: 102 | extends = append(extends, t) 103 | case *parser.EmptyStatement: 104 | emptyStatements = append(emptyStatements, t) 105 | case *parser.Extensions: 106 | extensions = append(extensions, t) 107 | default: 108 | return nil, fmt.Errorf("invalid MessageBody type %T of %v", t, t) 109 | } 110 | } 111 | return &MessageBody{ 112 | Fields: fields, 113 | Enums: enums, 114 | Messages: messages, 115 | Options: options, 116 | Oneofs: oneofs, 117 | Maps: maps, 118 | Groups: groups, 119 | Reserves: reserves, 120 | Extends: extends, 121 | EmptyStatements: emptyStatements, 122 | Extensions: extensions, 123 | }, nil 124 | } 125 | -------------------------------------------------------------------------------- /interpret/unordered/message_test.go: -------------------------------------------------------------------------------- 1 | package unordered_test 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/yoheimuta/go-protoparser/v4/interpret/unordered" 8 | "github.com/yoheimuta/go-protoparser/v4/parser" 9 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 10 | ) 11 | 12 | func TestInterpretMessage(t *testing.T) { 13 | tests := []struct { 14 | name string 15 | inputMessage *parser.Message 16 | wantMessage *unordered.Message 17 | wantErr bool 18 | }{ 19 | { 20 | name: "interpreting a nil", 21 | }, 22 | { 23 | name: "interpreting an excerpt from the official reference with comments", 24 | inputMessage: &parser.Message{ 25 | MessageName: "Outer", 26 | MessageBody: []parser.Visitee{ 27 | &parser.Option{ 28 | OptionName: "(my_option).a", 29 | Constant: "true", 30 | }, 31 | &parser.Message{ 32 | MessageName: "Inner", 33 | MessageBody: []parser.Visitee{ 34 | &parser.Field{ 35 | Type: "int64", 36 | FieldName: "ival", 37 | FieldNumber: "1", 38 | }, 39 | }, 40 | }, 41 | &parser.MapField{ 42 | KeyType: "int32", 43 | Type: "string", 44 | MapName: "my_map", 45 | FieldNumber: "2", 46 | }, 47 | }, 48 | Comments: []*parser.Comment{ 49 | { 50 | Raw: "// message", 51 | }, 52 | }, 53 | InlineComment: &parser.Comment{ 54 | Raw: "// TODO: implementation", 55 | Meta: meta.Meta{ 56 | Pos: meta.Position{ 57 | Offset: 25, 58 | Line: 2, 59 | Column: 26, 60 | }, 61 | }, 62 | }, 63 | InlineCommentBehindLeftCurly: &parser.Comment{ 64 | Raw: "// TODO: implementation2", 65 | Meta: meta.Meta{ 66 | Pos: meta.Position{ 67 | Offset: 25, 68 | Line: 1, 69 | Column: 26, 70 | }, 71 | }, 72 | }, 73 | Meta: meta.Meta{ 74 | Pos: meta.Position{ 75 | Offset: 21, 76 | Line: 3, 77 | Column: 1, 78 | }, 79 | }, 80 | }, 81 | wantMessage: &unordered.Message{ 82 | MessageName: "Outer", 83 | MessageBody: &unordered.MessageBody{ 84 | Options: []*parser.Option{ 85 | { 86 | OptionName: "(my_option).a", 87 | Constant: "true", 88 | }, 89 | }, 90 | Messages: []*unordered.Message{ 91 | { 92 | MessageName: "Inner", 93 | MessageBody: &unordered.MessageBody{ 94 | Fields: []*parser.Field{ 95 | { 96 | Type: "int64", 97 | FieldName: "ival", 98 | FieldNumber: "1", 99 | }, 100 | }, 101 | }, 102 | }, 103 | }, 104 | Maps: []*parser.MapField{ 105 | { 106 | KeyType: "int32", 107 | Type: "string", 108 | MapName: "my_map", 109 | FieldNumber: "2", 110 | }, 111 | }, 112 | }, 113 | Comments: []*parser.Comment{ 114 | { 115 | Raw: "// message", 116 | }, 117 | }, 118 | InlineComment: &parser.Comment{ 119 | Raw: "// TODO: implementation", 120 | Meta: meta.Meta{ 121 | Pos: meta.Position{ 122 | Offset: 25, 123 | Line: 2, 124 | Column: 26, 125 | }, 126 | }, 127 | }, 128 | InlineCommentBehindLeftCurly: &parser.Comment{ 129 | Raw: "// TODO: implementation2", 130 | Meta: meta.Meta{ 131 | Pos: meta.Position{ 132 | Offset: 25, 133 | Line: 1, 134 | Column: 26, 135 | }, 136 | }, 137 | }, 138 | Meta: meta.Meta{ 139 | Pos: meta.Position{ 140 | Offset: 21, 141 | Line: 3, 142 | Column: 1, 143 | }, 144 | }, 145 | }, 146 | }, 147 | } 148 | 149 | for _, test := range tests { 150 | test := test 151 | t.Run(test.name, func(t *testing.T) { 152 | got, err := unordered.InterpretMessage(test.inputMessage) 153 | switch { 154 | case test.wantErr: 155 | if err == nil { 156 | t.Errorf("got err nil, but want err, parsed=%v", got) 157 | } 158 | return 159 | case !test.wantErr && err != nil: 160 | t.Errorf("got err %v, but want nil", err) 161 | return 162 | } 163 | 164 | if !reflect.DeepEqual(got, test.wantMessage) { 165 | t.Errorf("got %v, but want %v", got, test.wantMessage) 166 | } 167 | }) 168 | } 169 | 170 | } 171 | -------------------------------------------------------------------------------- /interpret/unordered/proto.go: -------------------------------------------------------------------------------- 1 | package unordered 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/yoheimuta/go-protoparser/v4/parser" 7 | ) 8 | 9 | // ProtoBody is unordered in nature, but each slice field preserves the original order. 10 | type ProtoBody struct { 11 | Imports []*parser.Import 12 | Packages []*parser.Package 13 | Options []*parser.Option 14 | Messages []*Message 15 | Extends []*Extend 16 | Enums []*Enum 17 | Services []*Service 18 | EmptyStatements []*parser.EmptyStatement 19 | } 20 | 21 | // Proto represents a protocol buffer definition. 22 | type Proto struct { 23 | Syntax *parser.Syntax 24 | ProtoBody *ProtoBody 25 | } 26 | 27 | // InterpretProto interprets *parser.Proto to *Proto. 28 | func InterpretProto(src *parser.Proto) (*Proto, error) { 29 | if src == nil { 30 | return nil, nil 31 | } 32 | 33 | enumBody, err := interpretProtoBody(src.ProtoBody) 34 | if err != nil { 35 | return nil, err 36 | } 37 | return &Proto{ 38 | Syntax: src.Syntax, 39 | ProtoBody: enumBody, 40 | }, nil 41 | } 42 | 43 | func interpretProtoBody(src []parser.Visitee) ( 44 | *ProtoBody, 45 | error, 46 | ) { 47 | var imports []*parser.Import 48 | var packages []*parser.Package 49 | var options []*parser.Option 50 | var messages []*Message 51 | var extends []*Extend 52 | var enums []*Enum 53 | var services []*Service 54 | var emptyStatements []*parser.EmptyStatement 55 | for _, s := range src { 56 | switch t := s.(type) { 57 | case *parser.Import: 58 | imports = append(imports, t) 59 | case *parser.Package: 60 | packages = append(packages, t) 61 | case *parser.Option: 62 | options = append(options, t) 63 | case *parser.Message: 64 | message, err := InterpretMessage(t) 65 | if err != nil { 66 | return nil, err 67 | } 68 | messages = append(messages, message) 69 | case *parser.Extend: 70 | extend, err := InterpretExtend(t) 71 | if err != nil { 72 | return nil, err 73 | } 74 | extends = append(extends, extend) 75 | case *parser.Enum: 76 | enum, err := InterpretEnum(t) 77 | if err != nil { 78 | return nil, err 79 | } 80 | enums = append(enums, enum) 81 | case *parser.Service: 82 | service, err := InterpretService(t) 83 | if err != nil { 84 | return nil, err 85 | } 86 | services = append(services, service) 87 | case *parser.EmptyStatement: 88 | emptyStatements = append(emptyStatements, t) 89 | default: 90 | return nil, fmt.Errorf("invalid ProtoBody type %T of %v", t, t) 91 | } 92 | } 93 | return &ProtoBody{ 94 | Imports: imports, 95 | Packages: packages, 96 | Options: options, 97 | Messages: messages, 98 | Extends: extends, 99 | Enums: enums, 100 | Services: services, 101 | EmptyStatements: emptyStatements, 102 | }, nil 103 | } 104 | -------------------------------------------------------------------------------- /interpret/unordered/proto_test.go: -------------------------------------------------------------------------------- 1 | package unordered_test 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/yoheimuta/go-protoparser/v4/interpret/unordered" 8 | "github.com/yoheimuta/go-protoparser/v4/parser" 9 | ) 10 | 11 | func TestInterpretProto(t *testing.T) { 12 | tests := []struct { 13 | name string 14 | inputProto *parser.Proto 15 | wantProto *unordered.Proto 16 | wantErr bool 17 | }{ 18 | { 19 | name: "interpreting a nil", 20 | }, 21 | { 22 | name: "interpreting an excerpt from the official reference", 23 | inputProto: &parser.Proto{ 24 | Syntax: &parser.Syntax{ 25 | ProtobufVersion: "proto3", 26 | }, 27 | ProtoBody: []parser.Visitee{ 28 | &parser.Import{ 29 | Modifier: parser.ImportModifierPublic, 30 | Location: `"other.proto"`, 31 | }, 32 | &parser.Option{ 33 | OptionName: "java_package", 34 | Constant: `"com.example.foo"`, 35 | }, 36 | &parser.Enum{ 37 | EnumName: "EnumAllowingAlias", 38 | EnumBody: []parser.Visitee{ 39 | &parser.Option{ 40 | OptionName: "allow_alias", 41 | Constant: "true", 42 | }, 43 | &parser.EnumField{ 44 | Ident: "UNKNOWN", 45 | Number: "0", 46 | }, 47 | &parser.EnumField{ 48 | Ident: "STARTED", 49 | Number: "1", 50 | }, 51 | &parser.EnumField{ 52 | Ident: "RUNNING", 53 | Number: "2", 54 | EnumValueOptions: []*parser.EnumValueOption{ 55 | { 56 | OptionName: "(custom_option)", 57 | Constant: `"hello world"`, 58 | }, 59 | }, 60 | }, 61 | }, 62 | }, 63 | &parser.Message{ 64 | MessageName: "outer", 65 | MessageBody: []parser.Visitee{ 66 | &parser.Option{ 67 | OptionName: "(my_option).a", 68 | Constant: "true", 69 | }, 70 | &parser.Message{ 71 | MessageName: "inner", 72 | MessageBody: []parser.Visitee{ 73 | &parser.Field{ 74 | Type: "int64", 75 | FieldName: "ival", 76 | FieldNumber: "1", 77 | }, 78 | }, 79 | }, 80 | &parser.Field{ 81 | IsRepeated: true, 82 | Type: "inner", 83 | FieldName: "inner_message", 84 | FieldNumber: "2", 85 | }, 86 | &parser.Field{ 87 | Type: "EnumAllowingAlias", 88 | FieldName: "enum_field", 89 | FieldNumber: "3", 90 | }, 91 | &parser.MapField{ 92 | KeyType: "int32", 93 | Type: "string", 94 | MapName: "my_map", 95 | FieldNumber: "4", 96 | }, 97 | }, 98 | }, 99 | }, 100 | }, 101 | wantProto: &unordered.Proto{ 102 | Syntax: &parser.Syntax{ 103 | ProtobufVersion: "proto3", 104 | }, 105 | ProtoBody: &unordered.ProtoBody{ 106 | Imports: []*parser.Import{ 107 | { 108 | Modifier: parser.ImportModifierPublic, 109 | Location: `"other.proto"`, 110 | }, 111 | }, 112 | Options: []*parser.Option{ 113 | { 114 | OptionName: "java_package", 115 | Constant: `"com.example.foo"`, 116 | }, 117 | }, 118 | Enums: []*unordered.Enum{ 119 | { 120 | 121 | EnumName: "EnumAllowingAlias", 122 | EnumBody: &unordered.EnumBody{ 123 | Options: []*parser.Option{ 124 | { 125 | OptionName: "allow_alias", 126 | Constant: "true", 127 | }, 128 | }, 129 | EnumFields: []*parser.EnumField{ 130 | { 131 | Ident: "UNKNOWN", 132 | Number: "0", 133 | }, 134 | { 135 | Ident: "STARTED", 136 | Number: "1", 137 | }, 138 | { 139 | Ident: "RUNNING", 140 | Number: "2", 141 | EnumValueOptions: []*parser.EnumValueOption{ 142 | { 143 | OptionName: "(custom_option)", 144 | Constant: `"hello world"`, 145 | }, 146 | }, 147 | }, 148 | }, 149 | }, 150 | }, 151 | }, 152 | Messages: []*unordered.Message{ 153 | { 154 | MessageName: "outer", 155 | MessageBody: &unordered.MessageBody{ 156 | Options: []*parser.Option{ 157 | { 158 | OptionName: "(my_option).a", 159 | Constant: "true", 160 | }, 161 | }, 162 | Messages: []*unordered.Message{ 163 | { 164 | MessageName: "inner", 165 | MessageBody: &unordered.MessageBody{ 166 | Fields: []*parser.Field{ 167 | { 168 | Type: "int64", 169 | FieldName: "ival", 170 | FieldNumber: "1", 171 | }, 172 | }, 173 | }, 174 | }, 175 | }, 176 | Fields: []*parser.Field{ 177 | { 178 | IsRepeated: true, 179 | Type: "inner", 180 | FieldName: "inner_message", 181 | FieldNumber: "2", 182 | }, 183 | { 184 | Type: "EnumAllowingAlias", 185 | FieldName: "enum_field", 186 | FieldNumber: "3", 187 | }, 188 | }, 189 | Maps: []*parser.MapField{ 190 | { 191 | KeyType: "int32", 192 | Type: "string", 193 | MapName: "my_map", 194 | FieldNumber: "4", 195 | }, 196 | }, 197 | }, 198 | }, 199 | }, 200 | }, 201 | }, 202 | }, 203 | } 204 | 205 | for _, test := range tests { 206 | test := test 207 | t.Run(test.name, func(t *testing.T) { 208 | got, err := unordered.InterpretProto(test.inputProto) 209 | switch { 210 | case test.wantErr: 211 | if err == nil { 212 | t.Errorf("got err nil, but want err, parsed=%v", got) 213 | } 214 | return 215 | case !test.wantErr && err != nil: 216 | t.Errorf("got err %v, but want nil", err) 217 | return 218 | } 219 | 220 | if !reflect.DeepEqual(got, test.wantProto) { 221 | t.Errorf("got %v, but want %v", got, test.wantProto) 222 | } 223 | }) 224 | } 225 | 226 | } 227 | -------------------------------------------------------------------------------- /interpret/unordered/service.go: -------------------------------------------------------------------------------- 1 | package unordered 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/yoheimuta/go-protoparser/v4/parser" 7 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 8 | ) 9 | 10 | // ServiceBody is unordered in nature, but each slice field preserves the original order. 11 | type ServiceBody struct { 12 | Options []*parser.Option 13 | RPCs []*parser.RPC 14 | } 15 | 16 | // Service consists of RPCs. 17 | type Service struct { 18 | ServiceName string 19 | ServiceBody *ServiceBody 20 | 21 | // Comments are the optional ones placed at the beginning. 22 | Comments []*parser.Comment 23 | // InlineComment is the optional one placed at the ending. 24 | InlineComment *parser.Comment 25 | // InlineCommentBehindLeftCurly is the optional one placed behind a left curly. 26 | InlineCommentBehindLeftCurly *parser.Comment 27 | // Meta is the meta information. 28 | Meta meta.Meta 29 | } 30 | 31 | // InterpretService interprets *parser.Service to *Service. 32 | func InterpretService(src *parser.Service) (*Service, error) { 33 | if src == nil { 34 | return nil, nil 35 | } 36 | 37 | serviceBody, err := interpretServiceBody(src.ServiceBody) 38 | if err != nil { 39 | return nil, fmt.Errorf("invalid Service %s: %w", src.ServiceName, err) 40 | } 41 | return &Service{ 42 | ServiceName: src.ServiceName, 43 | ServiceBody: serviceBody, 44 | Comments: src.Comments, 45 | InlineComment: src.InlineComment, 46 | InlineCommentBehindLeftCurly: src.InlineCommentBehindLeftCurly, 47 | Meta: src.Meta, 48 | }, nil 49 | } 50 | 51 | func interpretServiceBody(src []parser.Visitee) ( 52 | *ServiceBody, 53 | error, 54 | ) { 55 | var options []*parser.Option 56 | var rpcs []*parser.RPC 57 | for _, s := range src { 58 | switch t := s.(type) { 59 | case *parser.Option: 60 | options = append(options, t) 61 | case *parser.RPC: 62 | rpcs = append(rpcs, t) 63 | default: 64 | return nil, fmt.Errorf("invalid ServiceBody type %T of %v", t, t) 65 | } 66 | } 67 | return &ServiceBody{ 68 | Options: options, 69 | RPCs: rpcs, 70 | }, nil 71 | } 72 | -------------------------------------------------------------------------------- /interpret/unordered/service_test.go: -------------------------------------------------------------------------------- 1 | package unordered_test 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/yoheimuta/go-protoparser/v4/interpret/unordered" 8 | "github.com/yoheimuta/go-protoparser/v4/parser" 9 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 10 | ) 11 | 12 | func TestInterpretService(t *testing.T) { 13 | tests := []struct { 14 | name string 15 | inputService *parser.Service 16 | wantService *unordered.Service 17 | wantErr bool 18 | }{ 19 | { 20 | name: "interpreting a nil", 21 | }, 22 | { 23 | name: "interpreting an excerpt from the official reference with a option and comments", 24 | inputService: &parser.Service{ 25 | ServiceName: "SearchService", 26 | ServiceBody: []parser.Visitee{ 27 | &parser.Option{ 28 | OptionName: "case-sensitive", 29 | Constant: "true", 30 | }, 31 | &parser.RPC{ 32 | RPCName: "Search", 33 | RPCRequest: &parser.RPCRequest{ 34 | MessageType: "SearchRequest", 35 | }, 36 | RPCResponse: &parser.RPCResponse{ 37 | MessageType: "SearchResponse", 38 | }, 39 | }, 40 | }, 41 | Comments: []*parser.Comment{ 42 | { 43 | Raw: "// service", 44 | }, 45 | }, 46 | InlineComment: &parser.Comment{ 47 | Raw: "// TODO: implementation", 48 | Meta: meta.Meta{ 49 | Pos: meta.Position{ 50 | Offset: 25, 51 | Line: 2, 52 | Column: 26, 53 | }, 54 | }, 55 | }, 56 | InlineCommentBehindLeftCurly: &parser.Comment{ 57 | Raw: "// TODO: implementation2", 58 | Meta: meta.Meta{ 59 | Pos: meta.Position{ 60 | Offset: 25, 61 | Line: 1, 62 | Column: 26, 63 | }, 64 | }, 65 | }, 66 | Meta: meta.Meta{ 67 | Pos: meta.Position{ 68 | Offset: 21, 69 | Line: 3, 70 | Column: 1, 71 | }, 72 | }, 73 | }, 74 | wantService: &unordered.Service{ 75 | ServiceName: "SearchService", 76 | ServiceBody: &unordered.ServiceBody{ 77 | Options: []*parser.Option{ 78 | { 79 | OptionName: "case-sensitive", 80 | Constant: "true", 81 | }, 82 | }, 83 | RPCs: []*parser.RPC{ 84 | { 85 | RPCName: "Search", 86 | RPCRequest: &parser.RPCRequest{ 87 | MessageType: "SearchRequest", 88 | }, 89 | RPCResponse: &parser.RPCResponse{ 90 | MessageType: "SearchResponse", 91 | }, 92 | }, 93 | }, 94 | }, 95 | Comments: []*parser.Comment{ 96 | { 97 | Raw: "// service", 98 | }, 99 | }, 100 | InlineComment: &parser.Comment{ 101 | Raw: "// TODO: implementation", 102 | Meta: meta.Meta{ 103 | Pos: meta.Position{ 104 | Offset: 25, 105 | Line: 2, 106 | Column: 26, 107 | }, 108 | }, 109 | }, 110 | InlineCommentBehindLeftCurly: &parser.Comment{ 111 | Raw: "// TODO: implementation2", 112 | Meta: meta.Meta{ 113 | Pos: meta.Position{ 114 | Offset: 25, 115 | Line: 1, 116 | Column: 26, 117 | }, 118 | }, 119 | }, 120 | Meta: meta.Meta{ 121 | Pos: meta.Position{ 122 | Offset: 21, 123 | Line: 3, 124 | Column: 1, 125 | }, 126 | }, 127 | }, 128 | }, 129 | } 130 | 131 | for _, test := range tests { 132 | test := test 133 | t.Run(test.name, func(t *testing.T) { 134 | got, err := unordered.InterpretService(test.inputService) 135 | switch { 136 | case test.wantErr: 137 | if err == nil { 138 | t.Errorf("got err nil, but want err, parsed=%v", got) 139 | } 140 | return 141 | case !test.wantErr && err != nil: 142 | t.Errorf("got err %v, but want nil", err) 143 | return 144 | } 145 | 146 | if !reflect.DeepEqual(got, test.wantService) { 147 | t.Errorf("got %v, but want %v", got, test.wantService) 148 | } 149 | }) 150 | } 151 | 152 | } 153 | -------------------------------------------------------------------------------- /lexer/constant.go: -------------------------------------------------------------------------------- 1 | package lexer 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 7 | ) 8 | 9 | // ReadConstant reads a constant. If permissive is true, accepts multiline string literals. 10 | // constant = fullIdent | ( [ "-" | "+" ] intLit ) | ( [ "-" | "+" ] floatLit ) | strLit | boolLit 11 | func (lex *Lexer) ReadConstant(permissive bool) (string, scanner.Position, error) { 12 | lex.NextLit() 13 | 14 | startPos := lex.Pos 15 | cons := lex.Text 16 | 17 | switch { 18 | case lex.Token == scanner.TSTRLIT: 19 | if permissive { 20 | return lex.mergeMultilineStrLit(), startPos, nil 21 | } 22 | return cons, startPos, nil 23 | case lex.Token == scanner.TBOOLLIT: 24 | return cons, startPos, nil 25 | case lex.Token == scanner.TIDENT: 26 | lex.UnNext() 27 | fullIdent, pos, err := lex.ReadFullIdent() 28 | if err != nil { 29 | return "", scanner.Position{}, err 30 | } 31 | return fullIdent, pos, nil 32 | case lex.Token == scanner.TINTLIT, lex.Token == scanner.TFLOATLIT: 33 | return cons, startPos, nil 34 | case lex.Text == "-" || lex.Text == "+": 35 | lex.NextLit() 36 | 37 | switch lex.Token { 38 | case scanner.TINTLIT, scanner.TFLOATLIT: 39 | cons += lex.Text 40 | return cons, startPos, nil 41 | default: 42 | return "", scanner.Position{}, lex.unexpected(lex.Text, "TINTLIT or TFLOATLIT") 43 | } 44 | default: 45 | return "", scanner.Position{}, lex.unexpected(lex.Text, "constant") 46 | } 47 | } 48 | 49 | // Merges a multiline string literal into a single string. 50 | func (lex *Lexer) mergeMultilineStrLit() string { 51 | q := "'" 52 | if strings.HasPrefix(lex.Text, "\"") { 53 | q = "\"" 54 | } 55 | var b strings.Builder 56 | b.WriteString(q) 57 | for lex.Token == scanner.TSTRLIT { 58 | strippedString := strings.Trim(lex.Text, q) 59 | b.WriteString(strippedString) 60 | lex.NextLit() 61 | } 62 | lex.UnNext() 63 | b.WriteString(q) 64 | return b.String() 65 | } 66 | -------------------------------------------------------------------------------- /lexer/constant_test.go: -------------------------------------------------------------------------------- 1 | package lexer_test 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/yoheimuta/go-protoparser/v4/lexer" 8 | ) 9 | 10 | func TestLexer2_ReadConstant(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | input string 14 | wantText string 15 | wantIsEOF bool 16 | wantErr bool 17 | }{ 18 | { 19 | name: "fullIdent", 20 | input: "foo.bar", 21 | wantText: "foo.bar", 22 | wantIsEOF: true, 23 | }, 24 | { 25 | name: "intLit", 26 | input: "1928", 27 | wantText: "1928", 28 | wantIsEOF: true, 29 | }, 30 | { 31 | name: "+intLit", 32 | input: "+1928", 33 | wantText: "+1928", 34 | wantIsEOF: true, 35 | }, 36 | { 37 | name: "-intLit", 38 | input: "-1928", 39 | wantText: "-1928", 40 | wantIsEOF: true, 41 | }, 42 | { 43 | name: "floatLit", 44 | input: "1928.123", 45 | wantText: "1928.123", 46 | wantIsEOF: true, 47 | }, 48 | { 49 | name: "+floatLit", 50 | input: "+1928e10", 51 | wantText: "+1928e10", 52 | wantIsEOF: true, 53 | }, 54 | { 55 | name: "-floatLit", 56 | input: "-1928E-3", 57 | wantText: "-1928E-3", 58 | wantIsEOF: true, 59 | }, 60 | { 61 | name: "single line strLit", 62 | input: `"あいうえお''"`, 63 | wantText: `"あいうえお''"`, 64 | wantIsEOF: true, 65 | }, 66 | { 67 | name: "multiline strLit with double quotes", 68 | input: "\"line1 \"\n\"line2 \" \n\"line3\" ", 69 | wantText: `"line1 line2 line3"`, 70 | wantIsEOF: true, 71 | }, 72 | { 73 | name: "multiline strLit with single quotes", 74 | input: "'line1 '\n'line2 ' \n'line3' ", 75 | wantText: `'line1 line2 line3'`, 76 | wantIsEOF: true, 77 | }, 78 | { 79 | name: "boolLit", 80 | input: "true", 81 | wantText: "true", 82 | wantIsEOF: true, 83 | }, 84 | { 85 | name: "boolLit.", 86 | input: "false.", 87 | wantText: "false", 88 | }, 89 | { 90 | name: "ident.", 91 | input: "rpc.", 92 | wantErr: true, 93 | }, 94 | { 95 | name: `left quote`, 96 | input: `"`, 97 | wantErr: true, 98 | }, 99 | } 100 | for _, test := range tests { 101 | test := test 102 | t.Run(test.name, func(t *testing.T) { 103 | lex := lexer.NewLexer(strings.NewReader(test.input)) 104 | got, pos, err := lex.ReadConstant(true) 105 | 106 | switch { 107 | case test.wantErr: 108 | if err == nil { 109 | t.Errorf("got err nil, but want err") 110 | } 111 | return 112 | case !test.wantErr && err != nil: 113 | t.Errorf("got err %v, but want nil", err) 114 | return 115 | } 116 | 117 | if got != test.wantText { 118 | t.Errorf("got %s, but want %s", got, test.wantText) 119 | } 120 | 121 | if pos.Offset != 0 { 122 | t.Errorf("got %d, but want 0", pos.Offset) 123 | } 124 | if pos.Line != 1 { 125 | t.Errorf("got %d, but want 1", pos.Line) 126 | } 127 | if pos.Column != 1 { 128 | t.Errorf("got %d, but want 1", pos.Column) 129 | } 130 | 131 | lex.Next() 132 | if lex.IsEOF() != test.wantIsEOF { 133 | t.Errorf("got %v, but want %v", lex.IsEOF(), test.wantIsEOF) 134 | } 135 | }) 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /lexer/emptyStatement.go: -------------------------------------------------------------------------------- 1 | package lexer 2 | 3 | import ( 4 | "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 5 | ) 6 | 7 | // ReadEmptyStatement reads an emptyStatement. 8 | // 9 | // emptyStatement = ";" 10 | // 11 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#emptystatement 12 | func (lex *Lexer) ReadEmptyStatement() error { 13 | lex.Next() 14 | 15 | if lex.Token == scanner.TSEMICOLON { 16 | return nil 17 | } 18 | lex.UnNext() 19 | return lex.unexpected(lex.Text, ";") 20 | } 21 | -------------------------------------------------------------------------------- /lexer/emptyStatement_test.go: -------------------------------------------------------------------------------- 1 | package lexer_test 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/yoheimuta/go-protoparser/v4/lexer" 8 | ) 9 | 10 | func TestLexer2_ReadEmptyStatement(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | input string 14 | wantErr bool 15 | }{ 16 | { 17 | name: "read ;", 18 | input: ";", 19 | }, 20 | { 21 | name: "not found ;", 22 | input: ":", 23 | wantErr: true, 24 | }, 25 | } 26 | for _, test := range tests { 27 | test := test 28 | t.Run(test.name, func(t *testing.T) { 29 | lex := lexer.NewLexer(strings.NewReader(test.input)) 30 | err := lex.ReadEmptyStatement() 31 | 32 | switch { 33 | case test.wantErr: 34 | if err == nil { 35 | t.Errorf("got err nil, but want err") 36 | } 37 | return 38 | case !test.wantErr && err != nil: 39 | t.Errorf("got err %v, but want nil", err) 40 | return 41 | } 42 | 43 | lex.Next() 44 | if !lex.IsEOF() { 45 | t.Errorf("got not eof, but want eof") 46 | } 47 | }) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /lexer/enumType.go: -------------------------------------------------------------------------------- 1 | package lexer 2 | 3 | import "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 4 | 5 | // ReadEnumType reads a messageType. 6 | // enumType = [ "." ] { ident "." } enumName 7 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#identifiers 8 | func (lex *Lexer) ReadEnumType() (string, scanner.Position, error) { 9 | return lex.ReadMessageType() 10 | } 11 | -------------------------------------------------------------------------------- /lexer/enumType_test.go: -------------------------------------------------------------------------------- 1 | package lexer_test 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/yoheimuta/go-protoparser/v4/lexer" 8 | ) 9 | 10 | func TestLexer2_ReadEnumType(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | input string 14 | wantText string 15 | wantIsEOF bool 16 | wantErr bool 17 | }{ 18 | { 19 | name: "ident", 20 | input: "EnumAllowingAlias", 21 | wantText: "EnumAllowingAlias", 22 | wantIsEOF: true, 23 | }, 24 | { 25 | name: ".ident", 26 | input: ".EnumAllowingAlias", 27 | wantText: ".EnumAllowingAlias", 28 | wantIsEOF: true, 29 | }, 30 | { 31 | name: ".ident.ident", 32 | input: ".search.EnumAllowingAlias", 33 | wantText: ".search.EnumAllowingAlias", 34 | wantIsEOF: true, 35 | }, 36 | } 37 | for _, test := range tests { 38 | test := test 39 | t.Run(test.name, func(t *testing.T) { 40 | lex := lexer.NewLexer(strings.NewReader(test.input)) 41 | got, pos, err := lex.ReadEnumType() 42 | 43 | switch { 44 | case test.wantErr: 45 | if err == nil { 46 | t.Errorf("got err nil, but want err") 47 | } 48 | return 49 | case !test.wantErr && err != nil: 50 | t.Errorf("got err %v, but want nil", err) 51 | return 52 | } 53 | 54 | if got != test.wantText { 55 | t.Errorf("got %s, but want %s", got, test.wantText) 56 | } 57 | 58 | if pos.Offset != 0 { 59 | t.Errorf("got %d, but want 0", pos.Offset) 60 | } 61 | if pos.Line != 1 { 62 | t.Errorf("got %d, but want 1", pos.Line) 63 | } 64 | if pos.Column != 1 { 65 | t.Errorf("got %d, but want 1", pos.Column) 66 | } 67 | 68 | lex.Next() 69 | if lex.IsEOF() != test.wantIsEOF { 70 | t.Errorf("got %v, but want %v", lex.IsEOF(), test.wantIsEOF) 71 | } 72 | }) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /lexer/error.go: -------------------------------------------------------------------------------- 1 | package lexer 2 | 3 | import ( 4 | "runtime" 5 | 6 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 7 | ) 8 | 9 | func (lex *Lexer) unexpected(found, expected string) error { 10 | err := &meta.Error{ 11 | Pos: lex.Pos.Position, 12 | Expected: expected, 13 | Found: lex.Text, 14 | } 15 | if lex.debug { 16 | _, file, line, _ := runtime.Caller(1) 17 | err.SetOccured(file, line) 18 | } 19 | return err 20 | } 21 | -------------------------------------------------------------------------------- /lexer/fullIdent.go: -------------------------------------------------------------------------------- 1 | package lexer 2 | 3 | import "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 4 | 5 | // ReadFullIdent reads a fullIdent. 6 | // fullIdent = ident { "." ident } 7 | func (lex *Lexer) ReadFullIdent() (string, scanner.Position, error) { 8 | lex.Next() 9 | if lex.Token != scanner.TIDENT { 10 | return "", scanner.Position{}, lex.unexpected(lex.Text, "TIDENT") 11 | } 12 | startPos := lex.Pos 13 | 14 | fullIdent := lex.Text 15 | lex.Next() 16 | 17 | for !lex.IsEOF() { 18 | if lex.Token != scanner.TDOT { 19 | lex.UnNext() 20 | break 21 | } 22 | 23 | lex.Next() 24 | if lex.Token != scanner.TIDENT { 25 | return "", scanner.Position{}, lex.unexpected(lex.Text, "TIDENT") 26 | } 27 | fullIdent += "." + lex.Text 28 | lex.Next() 29 | } 30 | return fullIdent, startPos, nil 31 | } 32 | -------------------------------------------------------------------------------- /lexer/fullIdent_test.go: -------------------------------------------------------------------------------- 1 | package lexer_test 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/yoheimuta/go-protoparser/v4/lexer" 8 | ) 9 | 10 | func TestLexer2_ReadFullIdent(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | input string 14 | wantText string 15 | wantIsEOF bool 16 | wantErr bool 17 | }{ 18 | { 19 | name: "ident", 20 | input: "foo", 21 | wantText: "foo", 22 | wantIsEOF: true, 23 | }, 24 | { 25 | name: "ident;", 26 | input: "foo;", 27 | wantText: "foo", 28 | }, 29 | { 30 | name: "ident.ident", 31 | input: "foo.true", 32 | wantText: "foo.true", 33 | wantIsEOF: true, 34 | }, 35 | { 36 | name: "ident.ident.ident.ident", 37 | input: "foo.bar.rpc.fuga", 38 | wantText: "foo.bar.rpc.fuga", 39 | wantIsEOF: true, 40 | }, 41 | { 42 | name: "read invalid {.", 43 | input: "{int_gt: 0}", 44 | wantText: "{int_gt:0}", 45 | wantErr: true, 46 | }, 47 | { 48 | name: "empty string", 49 | input: "", 50 | wantErr: true, 51 | }, 52 | { 53 | name: "ident.", 54 | input: "foo.", 55 | wantErr: true, 56 | }, 57 | } 58 | for _, test := range tests { 59 | test := test 60 | t.Run(test.name, func(t *testing.T) { 61 | lex := lexer.NewLexer(strings.NewReader(test.input)) 62 | got, pos, err := lex.ReadFullIdent() 63 | 64 | switch { 65 | case test.wantErr: 66 | if err == nil { 67 | t.Errorf("got err nil, but want err") 68 | } 69 | return 70 | case !test.wantErr && err != nil: 71 | t.Errorf("got err %v, but want nil", err) 72 | return 73 | } 74 | 75 | if got != test.wantText { 76 | t.Errorf("got %s, but want %s", got, test.wantText) 77 | } 78 | 79 | if pos.Offset != 0 { 80 | t.Errorf("got %d, but want 0", pos.Offset) 81 | } 82 | if pos.Line != 1 { 83 | t.Errorf("got %d, but want 1", pos.Line) 84 | } 85 | if pos.Column != 1 { 86 | t.Errorf("got %d, but want 1", pos.Column) 87 | } 88 | 89 | lex.Next() 90 | if lex.IsEOF() != test.wantIsEOF { 91 | t.Errorf("got %v, but want %v", lex.IsEOF(), test.wantIsEOF) 92 | } 93 | }) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /lexer/lexer.go: -------------------------------------------------------------------------------- 1 | package lexer 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "path/filepath" 7 | "runtime" 8 | 9 | "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 10 | ) 11 | 12 | // Lexer is a lexer. 13 | type Lexer struct { 14 | // Token is the lexical token. 15 | Token scanner.Token 16 | 17 | // Text is the lexical value. 18 | Text string 19 | 20 | // RawText is the scanned raw text. 21 | RawText []rune 22 | 23 | // Pos is the source position. 24 | Pos scanner.Position 25 | 26 | // Error is called for each error encountered. If no Error 27 | // function is set, the error is reported to os.Stderr. 28 | Error func(lexer *Lexer, err error) 29 | 30 | scanner *scanner.Scanner 31 | scannerOpts []scanner.Option 32 | scanErr error 33 | debug bool 34 | } 35 | 36 | // Option is an option for lexer.NewLexer. 37 | type Option func(*Lexer) 38 | 39 | // WithDebug is an option to enable the debug mode. 40 | func WithDebug(debug bool) Option { 41 | return func(l *Lexer) { 42 | l.debug = debug 43 | } 44 | } 45 | 46 | // WithFilename is an option for scanner.Option. 47 | func WithFilename(filename string) Option { 48 | return func(l *Lexer) { 49 | l.scannerOpts = append(l.scannerOpts, scanner.WithFilename(filename)) 50 | } 51 | } 52 | 53 | // NewLexer creates a new lexer. 54 | func NewLexer(input io.Reader, opts ...Option) *Lexer { 55 | lex := new(Lexer) 56 | for _, opt := range opts { 57 | opt(lex) 58 | } 59 | 60 | lex.Error = func(_ *Lexer, err error) { 61 | log.Printf(`Lexer encountered the error "%v"`, err) 62 | } 63 | lex.scanner = scanner.NewScanner(input, lex.scannerOpts...) 64 | return lex 65 | } 66 | 67 | // Next scans the read buffer. 68 | func (lex *Lexer) Next() { 69 | defer func() { 70 | if lex.debug { 71 | _, file, line, ok := runtime.Caller(2) 72 | if ok { 73 | log.Printf( 74 | "[DEBUG] Text=[%s], Token=[%v], Pos=[%s] called from %s:%d\n", 75 | lex.Text, 76 | lex.Token, 77 | lex.Pos, 78 | filepath.Base(file), 79 | line, 80 | ) 81 | } 82 | } 83 | }() 84 | 85 | var err error 86 | lex.Token, lex.Text, lex.Pos, err = lex.scanner.Scan() 87 | lex.RawText = lex.scanner.LastScanRaw() 88 | if err != nil { 89 | lex.scanErr = err 90 | lex.Error(lex, err) 91 | } 92 | } 93 | 94 | // NextN scans the read buffer nth times. 95 | func (lex *Lexer) NextN(n int) { 96 | for 0 < n { 97 | lex.Next() 98 | n-- 99 | } 100 | } 101 | 102 | // NextKeywordOrStrLit scans the read buffer with ScanKeyword or ScanStrLit modes. 103 | func (lex *Lexer) NextKeywordOrStrLit() { 104 | lex.nextWithSpecificMode(scanner.ScanKeyword | scanner.ScanStrLit) 105 | } 106 | 107 | // NextKeyword scans the read buffer with ScanKeyword mode. 108 | func (lex *Lexer) NextKeyword() { 109 | lex.nextWithSpecificMode(scanner.ScanKeyword) 110 | } 111 | 112 | // NextStrLit scans the read buffer with ScanStrLit mode. 113 | func (lex *Lexer) NextStrLit() { 114 | lex.nextWithSpecificMode(scanner.ScanStrLit) 115 | } 116 | 117 | // NextLit scans the read buffer with ScanLit mode. 118 | func (lex *Lexer) NextLit() { 119 | lex.nextWithSpecificMode(scanner.ScanLit) 120 | } 121 | 122 | // NextNumberLit scans the read buffer with ScanNumberLit mode. 123 | func (lex *Lexer) NextNumberLit() { 124 | lex.nextWithSpecificMode(scanner.ScanNumberLit) 125 | } 126 | 127 | // NextComment scans the read buffer with ScanComment mode. 128 | func (lex *Lexer) NextComment() { 129 | lex.nextWithSpecificMode(scanner.ScanComment) 130 | } 131 | 132 | func (lex *Lexer) nextWithSpecificMode(nextMode scanner.Mode) { 133 | mode := lex.scanner.Mode 134 | defer func() { 135 | lex.scanner.Mode = mode 136 | }() 137 | 138 | lex.scanner.Mode = nextMode 139 | lex.Next() 140 | } 141 | 142 | // IsEOF checks whether read buffer is empty. 143 | func (lex *Lexer) IsEOF() bool { 144 | return lex.Token == scanner.TEOF 145 | } 146 | 147 | // LatestErr returns the latest non-EOF error that was encountered by the Lexer.Next(). 148 | func (lex *Lexer) LatestErr() error { 149 | return lex.scanErr 150 | } 151 | 152 | // Peek returns the next token with keeping the read buffer unchanged. 153 | func (lex *Lexer) Peek() scanner.Token { 154 | lex.Next() 155 | defer lex.UnNext() 156 | return lex.Token 157 | } 158 | 159 | // PeekN returns the nth next token with keeping the read buffer unchanged. 160 | func (lex *Lexer) PeekN(n int) scanner.Token { 161 | var lasts [][]rune 162 | for 0 < n { 163 | lex.Next() 164 | lasts = append(lasts, lex.RawText) 165 | n-- 166 | } 167 | token := lex.Token 168 | for i := len(lasts) - 1; 0 <= i; i-- { 169 | lex.UnNextTo(lasts[i]) 170 | } 171 | return token 172 | } 173 | 174 | // UnNext put the latest text back to the read buffer. 175 | func (lex *Lexer) UnNext() { 176 | lex.Pos = lex.scanner.UnScan() 177 | lex.Token = scanner.TILLEGAL 178 | } 179 | 180 | // UnNextTo put the given latest text back to the read buffer. 181 | func (lex *Lexer) UnNextTo(lastScan []rune) { 182 | lex.scanner.SetLastScanRaw(lastScan) 183 | lex.UnNext() 184 | } 185 | 186 | // ConsumeToken consumes a given token if it exists. Otherwise, it consumes no token. 187 | func (lex *Lexer) ConsumeToken(t scanner.Token) { 188 | lex.Next() 189 | if lex.Token == t { 190 | return 191 | } 192 | lex.UnNext() 193 | } 194 | 195 | // FindMidComments finds comments between from and to. 196 | func (lex *Lexer) FindMidComments(from scanner.Position, to scanner.Position) []scanner.Text { 197 | comments := lex.scanner.GetScannedComments() 198 | var mid []scanner.Text 199 | for _, c := range comments { 200 | if from.Offset < c.Pos.Offset && c.Pos.Offset < to.Offset { 201 | mid = append(mid, c) 202 | } 203 | } 204 | return mid 205 | } 206 | -------------------------------------------------------------------------------- /lexer/messageType.go: -------------------------------------------------------------------------------- 1 | package lexer 2 | 3 | import "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 4 | 5 | // ReadMessageType reads a messageType. 6 | // messageType = [ "." ] { ident "." } messageName 7 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#identifiers 8 | func (lex *Lexer) ReadMessageType() (string, scanner.Position, error) { 9 | lex.Next() 10 | startPos := lex.Pos 11 | 12 | var messageType string 13 | if lex.Token == scanner.TDOT { 14 | messageType = lex.Text 15 | } else { 16 | lex.UnNext() 17 | } 18 | 19 | lex.Next() 20 | for !lex.IsEOF() { 21 | if lex.Token != scanner.TIDENT { 22 | return "", scanner.Position{}, lex.unexpected(lex.Text, "ident") 23 | } 24 | messageType += lex.Text 25 | 26 | lex.Next() 27 | if lex.Token != scanner.TDOT { 28 | lex.UnNext() 29 | break 30 | } 31 | messageType += lex.Text 32 | 33 | lex.Next() 34 | } 35 | 36 | return messageType, startPos, nil 37 | } 38 | -------------------------------------------------------------------------------- /lexer/messageType_test.go: -------------------------------------------------------------------------------- 1 | package lexer_test 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/yoheimuta/go-protoparser/v4/lexer" 8 | ) 9 | 10 | func TestLexer2_ReadMessageType(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | input string 14 | wantText string 15 | wantIsEOF bool 16 | wantErr bool 17 | }{ 18 | { 19 | name: "ident", 20 | input: "SearchRequest", 21 | wantText: "SearchRequest", 22 | wantIsEOF: true, 23 | }, 24 | { 25 | name: ".ident", 26 | input: ".SearchRequest", 27 | wantText: ".SearchRequest", 28 | wantIsEOF: true, 29 | }, 30 | { 31 | name: ".ident.ident", 32 | input: ".search.SearchRequest", 33 | wantText: ".search.SearchRequest", 34 | wantIsEOF: true, 35 | }, 36 | { 37 | name: "ident.ident", 38 | input: "aggregatespb.UserItemAggregate", 39 | wantText: "aggregatespb.UserItemAggregate", 40 | wantIsEOF: true, 41 | }, 42 | } 43 | for _, test := range tests { 44 | test := test 45 | t.Run(test.name, func(t *testing.T) { 46 | lex := lexer.NewLexer(strings.NewReader(test.input)) 47 | got, pos, err := lex.ReadMessageType() 48 | 49 | switch { 50 | case test.wantErr: 51 | if err == nil { 52 | t.Errorf("got err nil, but want err") 53 | } 54 | return 55 | case !test.wantErr && err != nil: 56 | t.Errorf("got err %v, but want nil", err) 57 | return 58 | } 59 | 60 | if got != test.wantText { 61 | t.Errorf("got %s, but want %s", got, test.wantText) 62 | } 63 | 64 | if pos.Offset != 0 { 65 | t.Errorf("got %d, but want 0", pos.Offset) 66 | } 67 | if pos.Line != 1 { 68 | t.Errorf("got %d, but want 1", pos.Line) 69 | } 70 | if pos.Column != 1 { 71 | t.Errorf("got %d, but want 1", pos.Column) 72 | } 73 | 74 | lex.Next() 75 | if lex.IsEOF() != test.wantIsEOF { 76 | t.Errorf("got %v(%v)(%v), but want %v", lex.IsEOF(), lex.Token, lex.Text, test.wantIsEOF) 77 | } 78 | }) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /lexer/scanner/boolLit.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | // boolLit = "true" | "false" 4 | func isBoolLit(ident string) bool { 5 | switch ident { 6 | case "true", "false": 7 | return true 8 | default: 9 | return false 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /lexer/scanner/comment.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | // comment = ( "//" { [^\n] } "\n" ) | ( "/*" { any } "*/" ) 4 | func (s *Scanner) scanComment() (string, error) { 5 | lit := string(s.read()) 6 | 7 | ch := s.read() 8 | switch ch { 9 | case '/': 10 | for ch != '\n' { 11 | lit += string(ch) 12 | 13 | if s.isEOF() { 14 | return lit, nil 15 | } 16 | ch = s.read() 17 | } 18 | case '*': 19 | for { 20 | if s.isEOF() { 21 | return lit, s.unexpected(eof, "\n") 22 | } 23 | lit += string(ch) 24 | 25 | ch = s.read() 26 | chn := s.peek() 27 | if ch == '*' && chn == '/' { 28 | lit += string(ch) 29 | lit += string(s.read()) 30 | break 31 | } 32 | } 33 | default: 34 | return "", s.unexpected(ch, "/ or *") 35 | } 36 | 37 | return lit, nil 38 | } 39 | -------------------------------------------------------------------------------- /lexer/scanner/eof.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | func (s *Scanner) isEOF() bool { 4 | ch := s.peek() 5 | return ch == eof 6 | } 7 | -------------------------------------------------------------------------------- /lexer/scanner/error.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | import ( 4 | "runtime" 5 | 6 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 7 | ) 8 | 9 | func (s *Scanner) unexpected(found rune, expected string) error { 10 | _, file, line, _ := runtime.Caller(1) 11 | err := &meta.Error{ 12 | Pos: s.pos.Position, 13 | Expected: expected, 14 | Found: string(found), 15 | } 16 | err.SetOccured(file, line) 17 | return err 18 | } 19 | -------------------------------------------------------------------------------- /lexer/scanner/floatLit.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | // floatLit = ( decimals "." [ decimals ] [ exponent ] | decimals exponent | "."decimals [ exponent ] ) | "inf" | "nan" 4 | func isFloatLitKeyword(ident string) bool { 5 | switch ident { 6 | case "inf", "nan": 7 | return true 8 | default: 9 | return false 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /lexer/scanner/ident.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | // ident = letter { letter | decimalDigit | "_" } 4 | func (s *Scanner) scanIdent() string { 5 | ident := string(s.read()) 6 | 7 | for { 8 | next := s.peek() 9 | switch { 10 | case isLetter(next), isDecimalDigit(next), next == '_': 11 | ident += string(s.read()) 12 | default: 13 | return ident 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /lexer/scanner/lettersdigits.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | // See 4 | // https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#letters_and_digits 5 | // https://ascii.cl/ 6 | 7 | // letter = "A" … "Z" | "a" … "z" 8 | func isLetter(r rune) bool { 9 | if r < 'A' { 10 | return false 11 | } 12 | 13 | if r > 'z' { 14 | return false 15 | } 16 | 17 | if r > 'Z' && r < 'a' { 18 | return false 19 | } 20 | 21 | return true 22 | } 23 | 24 | // decimalDigit = "0" … "9" 25 | func isDecimalDigit(r rune) bool { 26 | return '0' <= r && r <= '9' 27 | } 28 | 29 | // octalDigit = "0" … "7" 30 | func isOctalDigit(r rune) bool { 31 | return '0' <= r && r <= '7' 32 | } 33 | 34 | // hexDigit = "0" … "9" | "A" … "F" | "a" … "f" 35 | func isHexDigit(r rune) bool { 36 | if '0' <= r && r <= '9' { 37 | return true 38 | } 39 | if 'A' <= r && r <= 'F' { 40 | return true 41 | } 42 | if 'a' <= r && r <= 'f' { 43 | return true 44 | } 45 | return false 46 | } 47 | -------------------------------------------------------------------------------- /lexer/scanner/mode.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | // Mode is an enum type to control recognition of tokens. 4 | type Mode uint 5 | 6 | // Predefined mode bits to control recognition of tokens. 7 | const ( 8 | ScanIdent Mode = 1 << iota 9 | ScanNumberLit 10 | ScanStrLit 11 | ScanBoolLit 12 | ScanKeyword 13 | ScanComment 14 | ScanLit = ScanNumberLit | ScanStrLit | ScanBoolLit 15 | ) 16 | -------------------------------------------------------------------------------- /lexer/scanner/numberLit.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | // intLit = decimalLit | octalLit | hexLit 4 | // decimalLit = ( "1" … "9" ) { decimalDigit } 5 | // octalLit = "0" { octalDigit } 6 | // hexLit = "0" ( "x" | "X" ) hexDigit { hexDigit } 7 | // 8 | // floatLit = ( decimals "." [ decimals ] [ exponent ] | decimals exponent | "."decimals [ exponent ] ) | "inf" | "nan" 9 | func (s *Scanner) scanNumberLit() (Token, string, error) { 10 | lit := string(s.read()) 11 | ch := s.peek() 12 | 13 | switch { 14 | case lit == "0" && (ch == 'x' || ch == 'X'): 15 | // hexLit 16 | lit += string(s.read()) 17 | if !isHexDigit(s.peek()) { 18 | return TILLEGAL, "", s.unexpected(s.peek(), "hexDigit") 19 | } 20 | lit += string(s.read()) 21 | 22 | for !s.isEOF() { 23 | if !isHexDigit(s.peek()) { 24 | break 25 | } 26 | lit += string(s.read()) 27 | } 28 | return TINTLIT, lit, nil 29 | case lit == ".": 30 | // floatLit 31 | fractional, err := s.scanFractionPartNoOmit() 32 | if err != nil { 33 | return TILLEGAL, "", err 34 | } 35 | return TFLOATLIT, lit + fractional, nil 36 | case ch == '.': 37 | // floatLit 38 | lit += string(s.read()) 39 | fractional, err := s.scanFractionPart() 40 | if err != nil { 41 | return TILLEGAL, "", err 42 | } 43 | return TFLOATLIT, lit + fractional, nil 44 | case ch == 'e' || ch == 'E': 45 | // floatLit 46 | exp, err := s.scanExponent() 47 | if err != nil { 48 | return TILLEGAL, "", err 49 | } 50 | return TFLOATLIT, lit + exp, nil 51 | case lit == "0": 52 | // octalLit 53 | for !s.isEOF() { 54 | if !isOctalDigit(s.peek()) { 55 | break 56 | } 57 | lit += string(s.read()) 58 | } 59 | return TINTLIT, lit, nil 60 | default: 61 | // decimalLit or floatLit 62 | for !s.isEOF() { 63 | if !isDecimalDigit(s.peek()) { 64 | break 65 | } 66 | lit += string(s.read()) 67 | } 68 | 69 | switch s.peek() { 70 | case '.': 71 | // floatLit 72 | lit += string(s.read()) 73 | fractional, err := s.scanFractionPart() 74 | if err != nil { 75 | return TILLEGAL, "", err 76 | } 77 | return TFLOATLIT, lit + fractional, nil 78 | case 'e', 'E': 79 | // floatLit 80 | exp, err := s.scanExponent() 81 | if err != nil { 82 | return TILLEGAL, "", err 83 | } 84 | return TFLOATLIT, lit + exp, nil 85 | default: 86 | // decimalLit 87 | return TINTLIT, lit, nil 88 | } 89 | } 90 | } 91 | 92 | // [ decimals ] [ exponent ] 93 | func (s *Scanner) scanFractionPart() (string, error) { 94 | lit := "" 95 | 96 | ch := s.peek() 97 | switch { 98 | case isDecimalDigit(ch): 99 | decimals, err := s.scanDecimals() 100 | if err != nil { 101 | return "", err 102 | } 103 | lit += decimals 104 | } 105 | 106 | switch s.peek() { 107 | case 'e', 'E': 108 | exp, err := s.scanExponent() 109 | if err != nil { 110 | return "", err 111 | } 112 | lit += exp 113 | } 114 | return lit, nil 115 | } 116 | 117 | // decimals [ exponent ] 118 | func (s *Scanner) scanFractionPartNoOmit() (string, error) { 119 | decimals, err := s.scanDecimals() 120 | if err != nil { 121 | return "", err 122 | } 123 | 124 | switch s.peek() { 125 | case 'e', 'E': 126 | exp, err := s.scanExponent() 127 | if err != nil { 128 | return "", err 129 | } 130 | return decimals + exp, nil 131 | default: 132 | return decimals, nil 133 | } 134 | } 135 | 136 | // exponent = ( "e" | "E" ) [ "+" | "-" ] decimals 137 | func (s *Scanner) scanExponent() (string, error) { 138 | ch := s.peek() 139 | switch ch { 140 | case 'e', 'E': 141 | lit := string(s.read()) 142 | 143 | switch s.peek() { 144 | case '+', '-': 145 | lit += string(s.read()) 146 | } 147 | decimals, err := s.scanDecimals() 148 | if err != nil { 149 | return "", err 150 | } 151 | return lit + decimals, nil 152 | default: 153 | return "", s.unexpected(ch, "e or E") 154 | } 155 | } 156 | 157 | // decimals = decimalDigit { decimalDigit } 158 | func (s *Scanner) scanDecimals() (string, error) { 159 | ch := s.peek() 160 | if !isDecimalDigit(ch) { 161 | return "", s.unexpected(ch, "decimalDigit") 162 | } 163 | lit := string(s.read()) 164 | 165 | for !s.isEOF() { 166 | if !isDecimalDigit(s.peek()) { 167 | break 168 | } 169 | lit += string(s.read()) 170 | } 171 | return lit, nil 172 | } 173 | -------------------------------------------------------------------------------- /lexer/scanner/position.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | import ( 4 | "unicode/utf8" 5 | 6 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 7 | ) 8 | 9 | // Position represents a source position. 10 | type Position struct { 11 | meta.Position 12 | 13 | // columns is a map which the key is a line number and the value is a column number. 14 | columns map[int]int 15 | } 16 | 17 | // NewPosition creates a new Position. 18 | func NewPosition() *Position { 19 | return &Position{ 20 | Position: meta.Position{ 21 | Offset: 0, 22 | Line: 1, 23 | Column: 1, 24 | }, 25 | columns: make(map[int]int), 26 | } 27 | } 28 | 29 | // String stringify the position. 30 | func (pos Position) String() string { 31 | return pos.Position.String() 32 | } 33 | 34 | // Advance advances the position value. 35 | func (pos *Position) Advance(r rune) { 36 | length := utf8.RuneLen(r) 37 | pos.Offset += length 38 | 39 | if r == '\n' { 40 | pos.columns[pos.Line] = pos.Column 41 | 42 | pos.Line++ 43 | pos.Column = 1 44 | } else { 45 | pos.Column++ 46 | } 47 | } 48 | 49 | // AdvancedBulk returns a new position that advances the position value in a row. 50 | func (pos Position) AdvancedBulk(s string) Position { 51 | for _, r := range s { 52 | pos.Advance(r) 53 | } 54 | last, _ := utf8.DecodeLastRuneInString(s) 55 | pos.Revert(last) 56 | return pos 57 | } 58 | 59 | // Revert reverts the position value. 60 | func (pos *Position) Revert(r rune) { 61 | length := utf8.RuneLen(r) 62 | pos.Offset -= length 63 | 64 | if r == '\n' { 65 | pos.Line-- 66 | pos.Column = pos.columns[pos.Line] 67 | } else { 68 | pos.Column-- 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /lexer/scanner/position_test.go: -------------------------------------------------------------------------------- 1 | package scanner_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 7 | ) 8 | 9 | func TestPosition_Advance(t *testing.T) { 10 | tests := []struct { 11 | name string 12 | inputRunes []rune 13 | wantOffset int 14 | wantLine int 15 | wantColumn int 16 | }{ 17 | { 18 | name: "advance an ascii character", 19 | inputRunes: []rune{ 20 | 'a', 21 | }, 22 | wantOffset: 1, 23 | wantLine: 1, 24 | wantColumn: 2, 25 | }, 26 | { 27 | name: "advance ascii characters", 28 | inputRunes: []rune{ 29 | 'a', 30 | 'b', 31 | }, 32 | wantOffset: 2, 33 | wantLine: 1, 34 | wantColumn: 3, 35 | }, 36 | { 37 | name: "advance an ascii character and a new line", 38 | inputRunes: []rune{ 39 | 'a', 40 | '\n', 41 | }, 42 | wantOffset: 2, 43 | wantLine: 2, 44 | wantColumn: 1, 45 | }, 46 | { 47 | name: "advance utf8 characters and a new line", 48 | inputRunes: []rune{ 49 | 'あ', 50 | '\n', 51 | 'い', 52 | }, 53 | wantOffset: 7, 54 | wantLine: 2, 55 | wantColumn: 2, 56 | }, 57 | } 58 | 59 | for _, test := range tests { 60 | test := test 61 | t.Run(test.name, func(t *testing.T) { 62 | pos := scanner.NewPosition() 63 | for _, r := range test.inputRunes { 64 | pos.Advance(r) 65 | } 66 | 67 | if pos.Offset != test.wantOffset { 68 | t.Errorf("got %d, but want %d", pos.Offset, test.wantOffset) 69 | } 70 | if pos.Line != test.wantLine { 71 | t.Errorf("got %d, but want %d", pos.Line, test.wantLine) 72 | } 73 | if pos.Column != test.wantColumn { 74 | t.Errorf("got %d, but want %d", pos.Column, test.wantColumn) 75 | } 76 | }) 77 | } 78 | } 79 | 80 | func TestPosition_Revert(t *testing.T) { 81 | tests := []struct { 82 | name string 83 | inputAdvancingRunes []rune 84 | inputRevertingRunes []rune 85 | wantOffset int 86 | wantLine int 87 | wantColumn int 88 | }{ 89 | { 90 | name: "advance and revert an ascii character", 91 | inputAdvancingRunes: []rune{ 92 | 'a', 93 | }, 94 | inputRevertingRunes: []rune{ 95 | 'a', 96 | }, 97 | wantOffset: 0, 98 | wantLine: 1, 99 | wantColumn: 1, 100 | }, 101 | { 102 | name: "advance and revert ascii characters", 103 | inputAdvancingRunes: []rune{ 104 | 'a', 105 | 'b', 106 | }, 107 | inputRevertingRunes: []rune{ 108 | 'b', 109 | 'a', 110 | }, 111 | wantOffset: 0, 112 | wantLine: 1, 113 | wantColumn: 1, 114 | }, 115 | { 116 | name: "advance and revert an ascii character and a new line", 117 | inputAdvancingRunes: []rune{ 118 | 'a', 119 | '\n', 120 | }, 121 | inputRevertingRunes: []rune{ 122 | '\n', 123 | 'a', 124 | }, 125 | wantOffset: 0, 126 | wantLine: 1, 127 | wantColumn: 1, 128 | }, 129 | { 130 | name: "advance and revert utf8 characters and a new line", 131 | inputAdvancingRunes: []rune{ 132 | 'あ', 133 | '\n', 134 | 'い', 135 | }, 136 | inputRevertingRunes: []rune{ 137 | 'い', 138 | '\n', 139 | 'あ', 140 | }, 141 | wantOffset: 0, 142 | wantLine: 1, 143 | wantColumn: 1, 144 | }, 145 | } 146 | 147 | for _, test := range tests { 148 | test := test 149 | t.Run(test.name, func(t *testing.T) { 150 | pos := scanner.NewPosition() 151 | for _, r := range test.inputAdvancingRunes { 152 | pos.Advance(r) 153 | } 154 | for _, r := range test.inputRevertingRunes { 155 | pos.Revert(r) 156 | } 157 | 158 | if pos.Offset != test.wantOffset { 159 | t.Errorf("got %d, but want %d", pos.Offset, test.wantOffset) 160 | } 161 | if pos.Line != test.wantLine { 162 | t.Errorf("got %d, but want %d", pos.Line, test.wantLine) 163 | } 164 | if pos.Column != test.wantColumn { 165 | t.Errorf("got %d, but want %d", pos.Column, test.wantColumn) 166 | } 167 | }) 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /lexer/scanner/quote.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | // isQuote checks ch is the quote. 4 | // quote = "'" | '"' 5 | func isQuote(ch rune) bool { 6 | switch ch { 7 | case '\'', '"': 8 | return true 9 | default: 10 | return false 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /lexer/scanner/scanner.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "unicode" 7 | ) 8 | 9 | var eof = rune(0) 10 | 11 | // Text is a literal with a position. 12 | type Text struct { 13 | Literal string 14 | Pos Position 15 | } 16 | 17 | // Scanner represents a lexical scanner. 18 | type Scanner struct { 19 | r *bufio.Reader 20 | lastReadBuffer []rune 21 | lastScanRaw []rune 22 | 23 | // pos is a current source position. 24 | pos *Position 25 | 26 | // The Mode field controls which tokens are recognized. 27 | Mode Mode 28 | 29 | // comments are all the scanned comments. 30 | // These can be duplicated. 31 | comments []Text 32 | } 33 | 34 | // Option is an option for scanner.NewScanner. 35 | type Option func(*Scanner) 36 | 37 | // WithFilename is an option to set filename to the pos. 38 | func WithFilename(filename string) Option { 39 | return func(l *Scanner) { 40 | l.pos.Filename = filename 41 | } 42 | } 43 | 44 | // NewScanner returns a new instance of Scanner. 45 | func NewScanner(r io.Reader, opts ...Option) *Scanner { 46 | s := &Scanner{ 47 | r: bufio.NewReader(r), 48 | pos: NewPosition(), 49 | } 50 | for _, opt := range opts { 51 | opt(s) 52 | } 53 | return s 54 | } 55 | 56 | func (s *Scanner) read() (r rune) { 57 | defer func() { 58 | if r == eof { 59 | return 60 | } 61 | s.lastScanRaw = append(s.lastScanRaw, r) 62 | 63 | s.pos.Advance(r) 64 | }() 65 | 66 | if 0 < len(s.lastReadBuffer) { 67 | var ch rune 68 | ch, s.lastReadBuffer = s.lastReadBuffer[len(s.lastReadBuffer)-1], s.lastReadBuffer[:len(s.lastReadBuffer)-1] 69 | return ch 70 | } 71 | ch, _, err := s.r.ReadRune() 72 | if err != nil { 73 | return eof 74 | } 75 | return ch 76 | } 77 | 78 | func (s *Scanner) unread(ch rune) { 79 | s.lastReadBuffer = append(s.lastReadBuffer, ch) 80 | 81 | s.pos.Revert(ch) 82 | } 83 | 84 | func (s *Scanner) peek() rune { 85 | ch := s.read() 86 | if ch != eof { 87 | s.lastScanRaw = s.lastScanRaw[0 : len(s.lastScanRaw)-1] 88 | s.unread(ch) 89 | } 90 | return ch 91 | } 92 | 93 | // UnScan put the last scanned text back to the read buffer. 94 | func (s *Scanner) UnScan() Position { 95 | var reversedRunes []rune 96 | for _, ch := range s.lastScanRaw { 97 | reversedRunes = append([]rune{ch}, reversedRunes...) 98 | } 99 | for _, ch := range reversedRunes { 100 | s.unread(ch) 101 | } 102 | return *s.pos 103 | } 104 | 105 | // Scan returns the next token and text value. 106 | func (s *Scanner) Scan() (Token, string, Position, error) { 107 | s.lastScanRaw = s.lastScanRaw[:0] 108 | return s.scan() 109 | } 110 | 111 | // LastScanRaw returns the deep-copied lastScanRaw. 112 | func (s *Scanner) LastScanRaw() []rune { 113 | r := make([]rune, len(s.lastScanRaw)) 114 | copy(r, s.lastScanRaw) 115 | return r 116 | } 117 | 118 | // SetLastScanRaw sets lastScanRaw to the given raw. 119 | func (s *Scanner) SetLastScanRaw(raw []rune) { 120 | s.lastScanRaw = raw 121 | } 122 | 123 | // GetScannedComments returns all the uniquely scanned comments. 124 | func (s *Scanner) GetScannedComments() []Text { 125 | var uniqueComments []Text 126 | find := func(c Text) bool { 127 | for _, uc := range uniqueComments { 128 | if c.Pos.Offset == uc.Pos.Offset && c.Literal == uc.Literal { 129 | return true 130 | } 131 | } 132 | return false 133 | } 134 | 135 | for _, c := range s.comments { 136 | if !find(c) { 137 | uniqueComments = append(uniqueComments, c) 138 | } 139 | } 140 | return uniqueComments 141 | } 142 | 143 | func (s *Scanner) scan() (Token, string, Position, error) { 144 | ch := s.peek() 145 | 146 | startPos := *s.pos 147 | 148 | switch { 149 | case unicode.IsSpace(ch): 150 | s.read() 151 | return s.scan() 152 | case s.isEOF(): 153 | return TEOF, "", startPos, nil 154 | case isLetter(ch), ch == '_': 155 | ident := s.scanIdent() 156 | if s.Mode&ScanBoolLit != 0 && isBoolLit(ident) { 157 | return TBOOLLIT, ident, startPos, nil 158 | } 159 | if s.Mode&ScanNumberLit != 0 && isFloatLitKeyword(ident) { 160 | return TFLOATLIT, ident, startPos, nil 161 | } 162 | if s.Mode&ScanKeyword != 0 && asKeywordToken(ident) != TILLEGAL { 163 | return asKeywordToken(ident), ident, startPos, nil 164 | } 165 | return TIDENT, ident, startPos, nil 166 | case ch == '/': 167 | lit, err := s.scanComment() 168 | if err != nil { 169 | return TILLEGAL, "", startPos, err 170 | } 171 | s.comments = append(s.comments, Text{Literal: lit, Pos: startPos}) 172 | if s.Mode&ScanComment != 0 { 173 | return TCOMMENT, lit, startPos, nil 174 | } 175 | return s.scan() 176 | case isQuote(ch) && s.Mode&ScanStrLit != 0: 177 | lit, err := s.scanStrLit() 178 | if err != nil { 179 | return TILLEGAL, "", startPos, err 180 | } 181 | return TSTRLIT, lit, startPos, nil 182 | case (isDecimalDigit(ch) || ch == '.') && s.Mode&ScanNumberLit != 0: 183 | tok, lit, err := s.scanNumberLit() 184 | if err != nil { 185 | return TILLEGAL, "", startPos, err 186 | } 187 | return tok, lit, startPos, nil 188 | default: 189 | return asMiscToken(ch), string(s.read()), startPos, nil 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /lexer/scanner/strLit.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | // strLit = ( "'" { charValue } "'" ) | ( '"' { charValue } '"' ) 4 | func (s *Scanner) scanStrLit() (string, error) { 5 | quote := s.read() 6 | lit := string(quote) 7 | 8 | ch := s.peek() 9 | for ch != quote { 10 | cv, err := s.scanCharValue() 11 | if err != nil { 12 | return "", err 13 | } 14 | lit += cv 15 | ch = s.peek() 16 | } 17 | 18 | // consume quote 19 | lit += string(s.read()) 20 | return lit, nil 21 | } 22 | 23 | // charValue = hexEscape | octEscape | charEscape | /[^\0\n\\]/ 24 | func (s *Scanner) scanCharValue() (string, error) { 25 | ch := s.peek() 26 | 27 | switch ch { 28 | case eof, '\n': 29 | return "", s.unexpected(ch, `/[^\0\n\\]`) 30 | case '\\': 31 | return s.tryScanEscape(), nil 32 | default: 33 | return string(s.read()), nil 34 | } 35 | } 36 | 37 | // hexEscape = '\' ( "x" | "X" ) hexDigit hexDigit 38 | // octEscape = '\' octalDigit octalDigit octalDigit 39 | // charEscape = '\' ( "a" | "b" | "f" | "n" | "r" | "t" | "v" | '\' | "'" | '"' ) 40 | func (s *Scanner) tryScanEscape() string { 41 | lit := string(s.read()) 42 | 43 | isCharEscape := func(r rune) bool { 44 | cs := []rune{'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', '\'', '"'} 45 | for _, c := range cs { 46 | if r == c { 47 | return true 48 | } 49 | } 50 | return false 51 | } 52 | 53 | ch := s.peek() 54 | switch { 55 | case ch == 'x' || ch == 'X': 56 | lit += string(s.read()) 57 | 58 | for i := 0; i < 2; i++ { 59 | if !isHexDigit(s.peek()) { 60 | return lit 61 | } 62 | lit += string(s.read()) 63 | } 64 | case isOctalDigit(ch): 65 | for i := 0; i < 3; i++ { 66 | if !isOctalDigit(s.peek()) { 67 | return lit 68 | } 69 | lit += string(s.read()) 70 | } 71 | case isCharEscape(ch): 72 | lit += string(s.read()) 73 | return lit 74 | } 75 | return lit 76 | } 77 | -------------------------------------------------------------------------------- /lexer/scanner/token.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | // Token represents a lexical token. 4 | type Token int 5 | 6 | // The result of Scan is one of these tokens. 7 | const ( 8 | // Special tokens 9 | TILLEGAL Token = iota 10 | TEOF 11 | 12 | // Identifiers 13 | TIDENT 14 | 15 | // Literals 16 | TINTLIT 17 | TFLOATLIT 18 | TBOOLLIT 19 | TSTRLIT 20 | 21 | // Comment 22 | TCOMMENT 23 | 24 | // Misc characters 25 | TSEMICOLON // ; 26 | TCOLON // : 27 | TEQUALS // = 28 | TQUOTE // " or ' 29 | TLEFTPAREN // ( 30 | TRIGHTPAREN // ) 31 | TLEFTCURLY // { 32 | TRIGHTCURLY // } 33 | TLEFTSQUARE // [ 34 | TRIGHTSQUARE // ] 35 | TLESS // < 36 | TGREATER // > 37 | TCOMMA // , 38 | TDOT // . 39 | TMINUS // - 40 | TBOM // Byte Order Mark 41 | 42 | // Keywords 43 | TSYNTAX 44 | TEDITION 45 | TSERVICE 46 | TRPC 47 | TRETURNS 48 | TMESSAGE 49 | TEXTEND 50 | TIMPORT 51 | TPACKAGE 52 | TOPTION 53 | TREPEATED 54 | TREQUIRED 55 | TOPTIONAL 56 | TWEAK 57 | TPUBLIC 58 | TONEOF 59 | TMAP 60 | TRESERVED 61 | TEXTENSIONS 62 | TDECLARATION 63 | TNUMBER 64 | TFULLNAME 65 | TTYPE 66 | TENUM 67 | TSTREAM 68 | TGROUP 69 | ) 70 | 71 | func asMiscToken(ch rune) Token { 72 | m := map[rune]Token{ 73 | ';': TSEMICOLON, 74 | ':': TCOLON, 75 | '=': TEQUALS, 76 | '"': TQUOTE, 77 | '\'': TQUOTE, 78 | '(': TLEFTPAREN, 79 | ')': TRIGHTPAREN, 80 | '{': TLEFTCURLY, 81 | '}': TRIGHTCURLY, 82 | '[': TLEFTSQUARE, 83 | ']': TRIGHTSQUARE, 84 | '<': TLESS, 85 | '>': TGREATER, 86 | ',': TCOMMA, 87 | '.': TDOT, 88 | '-': TMINUS, 89 | '\uFEFF': TBOM, 90 | } 91 | if t, ok := m[ch]; ok { 92 | return t 93 | } 94 | return TILLEGAL 95 | } 96 | 97 | func asKeywordToken(st string) Token { 98 | m := map[string]Token{ 99 | "syntax": TSYNTAX, 100 | "edition": TEDITION, 101 | "service": TSERVICE, 102 | "rpc": TRPC, 103 | "returns": TRETURNS, 104 | "message": TMESSAGE, 105 | "extend": TEXTEND, 106 | "import": TIMPORT, 107 | "package": TPACKAGE, 108 | "option": TOPTION, 109 | "repeated": TREPEATED, 110 | "required": TREQUIRED, 111 | "optional": TOPTIONAL, 112 | "weak": TWEAK, 113 | "public": TPUBLIC, 114 | "oneof": TONEOF, 115 | "map": TMAP, 116 | "reserved": TRESERVED, 117 | "extensions": TEXTENSIONS, 118 | "number": TNUMBER, 119 | "full_name": TFULLNAME, 120 | "type": TTYPE, 121 | "declaration": TDECLARATION, 122 | "enum": TENUM, 123 | "stream": TSTREAM, 124 | "group": TGROUP, 125 | } 126 | 127 | if t, ok := m[st]; ok { 128 | return t 129 | } 130 | return TILLEGAL 131 | } 132 | -------------------------------------------------------------------------------- /parser/comment.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 7 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 8 | ) 9 | 10 | const ( 11 | cStyleCommentPrefix = "/*" 12 | cStyleCommentSuffix = "*/" 13 | cPlusStyleCommentPrefix = "//" 14 | ) 15 | 16 | // Comment is a comment in either C/C++-style // and /* ... */ syntax. 17 | type Comment struct { 18 | // Raw includes a comment syntax like // and /* */. 19 | Raw string 20 | // Meta is the meta information. 21 | Meta meta.Meta 22 | } 23 | 24 | // IsCStyle refers to /* ... */. 25 | func (c *Comment) IsCStyle() bool { 26 | return strings.HasPrefix(c.Raw, cStyleCommentPrefix) 27 | } 28 | 29 | // Lines formats comment text lines without prefixes //, /* or suffix */. 30 | func (c *Comment) Lines() []string { 31 | raw := c.Raw 32 | if c.IsCStyle() { 33 | raw = strings.TrimPrefix(raw, cStyleCommentPrefix) 34 | raw = strings.TrimSuffix(raw, cStyleCommentSuffix) 35 | } else { 36 | raw = strings.TrimPrefix(raw, cPlusStyleCommentPrefix) 37 | } 38 | return strings.Split(raw, "\n") 39 | } 40 | 41 | // Accept dispatches the call to the visitor. 42 | func (c *Comment) Accept(v Visitor) { 43 | v.VisitComment(c) 44 | } 45 | 46 | // ParseComments parsers a sequence of comments. 47 | // 48 | // comments = { comment } 49 | // 50 | // See https://developers.google.com/protocol-buffers/docs/proto3#adding-comments 51 | func (p *Parser) ParseComments() []*Comment { 52 | var comments []*Comment 53 | for { 54 | comment, err := p.parseComment() 55 | if err != nil { 56 | // ignores the err because the comment is optional. 57 | return comments 58 | } 59 | comments = append(comments, comment) 60 | } 61 | } 62 | 63 | // See https://developers.google.com/protocol-buffers/docs/proto3#adding-comments 64 | func (p *Parser) parseComment() (*Comment, error) { 65 | p.lex.NextComment() 66 | if p.lex.Token == scanner.TCOMMENT { 67 | return &Comment{ 68 | Raw: p.lex.Text, 69 | Meta: meta.Meta{ 70 | Pos: p.lex.Pos.Position, 71 | LastPos: p.lex.Pos.AdvancedBulk(p.lex.Text).Position, 72 | }, 73 | }, nil 74 | } 75 | defer p.lex.UnNext() 76 | return nil, p.unexpected("comment") 77 | } 78 | 79 | func (p *Parser) findEmbeddedComments(from scanner.Position, to scanner.Position) []*Comment { 80 | var comments []*Comment 81 | for _, c := range p.lex.FindMidComments(from, to) { 82 | comments = append(comments, &Comment{ 83 | Raw: c.Literal, 84 | Meta: meta.Meta{ 85 | Pos: c.Pos.Position, 86 | LastPos: c.Pos.AdvancedBulk(c.Literal).Position, 87 | }, 88 | }) 89 | } 90 | return comments 91 | } 92 | -------------------------------------------------------------------------------- /parser/comment_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/yoheimuta/go-protoparser/v4/internal/util_test" 9 | "github.com/yoheimuta/go-protoparser/v4/lexer" 10 | "github.com/yoheimuta/go-protoparser/v4/parser" 11 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 12 | ) 13 | 14 | func TestComment_IsCStyle(t *testing.T) { 15 | tests := []struct { 16 | name string 17 | inputComment *parser.Comment 18 | wantIsCStyle bool 19 | }{ 20 | { 21 | name: "parsing a C-style comment", 22 | inputComment: &parser.Comment{ 23 | Raw: `/* 24 | comment 25 | */ 26 | `, 27 | }, 28 | wantIsCStyle: true, 29 | }, 30 | { 31 | name: "parsing a C++-style comment", 32 | inputComment: &parser.Comment{ 33 | Raw: "// comment", 34 | }, 35 | }, 36 | } 37 | 38 | for _, test := range tests { 39 | test := test 40 | t.Run(test.name, func(t *testing.T) { 41 | got := test.inputComment.IsCStyle() 42 | if got != test.wantIsCStyle { 43 | t.Errorf("got %v, but want %v", got, test.wantIsCStyle) 44 | } 45 | }) 46 | } 47 | } 48 | 49 | func TestComment_Lines(t *testing.T) { 50 | tests := []struct { 51 | name string 52 | inputComment *parser.Comment 53 | wantLines []string 54 | }{ 55 | { 56 | name: "parsing a C-style comment", 57 | inputComment: &parser.Comment{ 58 | Raw: `/*comment*/`, 59 | }, 60 | wantLines: []string{ 61 | "comment", 62 | }, 63 | }, 64 | { 65 | name: "parsing C-style comments", 66 | inputComment: &parser.Comment{ 67 | Raw: `/* comment1 68 | comment2 69 | */`, 70 | }, 71 | wantLines: []string{ 72 | " comment1", 73 | "comment2", 74 | "", 75 | }, 76 | }, 77 | { 78 | name: "parsing a C++-style comment", 79 | inputComment: &parser.Comment{ 80 | Raw: "// comment", 81 | }, 82 | wantLines: []string{ 83 | " comment", 84 | }, 85 | }, 86 | } 87 | 88 | for _, test := range tests { 89 | test := test 90 | t.Run(test.name, func(t *testing.T) { 91 | got := test.inputComment.Lines() 92 | if !reflect.DeepEqual(got, test.wantLines) { 93 | t.Errorf("got %v, but want %v", util_test.PrettyFormat(got), util_test.PrettyFormat(test.wantLines)) 94 | } 95 | }) 96 | } 97 | } 98 | 99 | func TestParser_ParseComments(t *testing.T) { 100 | tests := []struct { 101 | name string 102 | input string 103 | wantComments []*parser.Comment 104 | }{ 105 | { 106 | name: "parsing an empty", 107 | }, 108 | { 109 | name: "parsing a C++-style comment", 110 | input: `// comment 111 | `, 112 | wantComments: []*parser.Comment{ 113 | { 114 | Raw: `// comment`, 115 | Meta: meta.Meta{ 116 | Pos: meta.Position{ 117 | Offset: 0, 118 | Line: 1, 119 | Column: 1, 120 | }, 121 | LastPos: meta.Position{ 122 | Offset: 9, 123 | Line: 1, 124 | Column: 10, 125 | }, 126 | }, 127 | }, 128 | }, 129 | }, 130 | { 131 | name: "parsing C++-style comments", 132 | input: `// comment 133 | // comment2 134 | `, 135 | wantComments: []*parser.Comment{ 136 | { 137 | Raw: `// comment`, 138 | Meta: meta.Meta{ 139 | Pos: meta.Position{ 140 | Offset: 0, 141 | Line: 1, 142 | Column: 1, 143 | }, 144 | LastPos: meta.Position{ 145 | Offset: 9, 146 | Line: 1, 147 | Column: 10, 148 | }, 149 | }, 150 | }, 151 | { 152 | Raw: `// comment2`, 153 | Meta: meta.Meta{ 154 | Pos: meta.Position{ 155 | Offset: 11, 156 | Line: 2, 157 | Column: 1, 158 | }, 159 | LastPos: meta.Position{ 160 | Offset: 21, 161 | Line: 2, 162 | Column: 11, 163 | }, 164 | }, 165 | }, 166 | }, 167 | }, 168 | { 169 | name: "parsing a C-style comment", 170 | input: `/* 171 | comment 172 | */`, 173 | wantComments: []*parser.Comment{ 174 | { 175 | Raw: `/* 176 | comment 177 | */`, 178 | Meta: meta.Meta{ 179 | Pos: meta.Position{ 180 | Offset: 0, 181 | Line: 1, 182 | Column: 1, 183 | }, 184 | LastPos: meta.Position{ 185 | Offset: 12, 186 | Line: 3, 187 | Column: 2, 188 | }, 189 | }, 190 | }, 191 | }, 192 | }, 193 | { 194 | name: "parsing C-style comments", 195 | input: `/* 196 | comment 197 | */ 198 | /* 199 | comment2 200 | */`, 201 | wantComments: []*parser.Comment{ 202 | { 203 | Raw: `/* 204 | comment 205 | */`, 206 | Meta: meta.Meta{ 207 | Pos: meta.Position{ 208 | Offset: 0, 209 | Line: 1, 210 | Column: 1, 211 | }, 212 | LastPos: meta.Position{ 213 | Offset: 12, 214 | Line: 3, 215 | Column: 2, 216 | }, 217 | }, 218 | }, 219 | { 220 | Raw: `/* 221 | comment2 222 | */`, 223 | Meta: meta.Meta{ 224 | Pos: meta.Position{ 225 | Offset: 14, 226 | Line: 4, 227 | Column: 1, 228 | }, 229 | LastPos: meta.Position{ 230 | Offset: 27, 231 | Line: 6, 232 | Column: 2, 233 | }, 234 | }, 235 | }, 236 | }, 237 | }, 238 | { 239 | name: "parsing a C-style comment and a C++-style comment", 240 | input: `/* 241 | comment 242 | */ 243 | 244 | // comment2 245 | `, 246 | wantComments: []*parser.Comment{ 247 | { 248 | Raw: `/* 249 | comment 250 | */`, 251 | Meta: meta.Meta{ 252 | Pos: meta.Position{ 253 | Offset: 0, 254 | Line: 1, 255 | Column: 1, 256 | }, 257 | LastPos: meta.Position{ 258 | Offset: 12, 259 | Line: 3, 260 | Column: 2, 261 | }, 262 | }, 263 | }, 264 | { 265 | Raw: `// comment2`, 266 | Meta: meta.Meta{ 267 | Pos: meta.Position{ 268 | Offset: 15, 269 | Line: 5, 270 | Column: 1, 271 | }, 272 | LastPos: meta.Position{ 273 | Offset: 25, 274 | Line: 5, 275 | Column: 11, 276 | }, 277 | }, 278 | }, 279 | }, 280 | }, 281 | } 282 | 283 | for _, test := range tests { 284 | test := test 285 | t.Run(test.name, func(t *testing.T) { 286 | p := parser.NewParser(lexer.NewLexer(strings.NewReader(test.input))) 287 | got := p.ParseComments() 288 | 289 | if !reflect.DeepEqual(got, test.wantComments) { 290 | t.Errorf("got %v, but want %v", util_test.PrettyFormat(got), util_test.PrettyFormat(test.wantComments)) 291 | } 292 | 293 | if !p.IsEOF() { 294 | t.Errorf("got not eof, but want eof") 295 | } 296 | }) 297 | } 298 | } 299 | -------------------------------------------------------------------------------- /parser/declaration.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 5 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 6 | ) 7 | 8 | // Declaration is an option of extension ranges. 9 | type Declaration struct { 10 | Number string 11 | FullName string 12 | Type string 13 | Reserved bool 14 | Repeated bool 15 | 16 | // Comments are the optional ones placed at the beginning. 17 | Comments []*Comment 18 | // InlineComment is the optional one placed at the ending. 19 | InlineComment *Comment 20 | // InlineCommentBehindLeftCurly is the optional one placed behind a left curly. 21 | InlineCommentBehindLeftCurly *Comment 22 | // Meta is the meta information. 23 | Meta meta.Meta 24 | } 25 | 26 | // SetInlineComment implements the HasInlineCommentSetter interface. 27 | func (d *Declaration) SetInlineComment(comment *Comment) { 28 | d.InlineComment = comment 29 | } 30 | 31 | // Accept dispatches the call to the visitor. 32 | func (d *Declaration) Accept(v Visitor) { 33 | if !v.VisitDeclaration(d) { 34 | return 35 | } 36 | 37 | for _, comment := range d.Comments { 38 | comment.Accept(v) 39 | } 40 | if d.InlineComment != nil { 41 | d.InlineComment.Accept(v) 42 | } 43 | if d.InlineCommentBehindLeftCurly != nil { 44 | d.InlineCommentBehindLeftCurly.Accept(v) 45 | } 46 | } 47 | 48 | // ParseDeclaration parses a declaration. 49 | // 50 | // declaration = "declaration" "=" "{" 51 | // "number" ":" number "," 52 | // "full_name" ":" string "," 53 | // "type" ":" string "," 54 | // "repeated" ":" bool "," 55 | // "reserved" ":" bool 56 | // "}" 57 | // 58 | // See https://protobuf.dev/programming-guides/extension_declarations/ 59 | func (p *Parser) ParseDeclaration() (*Declaration, error) { 60 | p.lex.NextKeyword() 61 | if p.lex.Token != scanner.TDECLARATION { 62 | return nil, p.unexpected("declaration") 63 | } 64 | startPos := p.lex.Pos 65 | 66 | p.lex.Next() 67 | if p.lex.Token != scanner.TEQUALS { 68 | return nil, p.unexpected("=") 69 | } 70 | 71 | p.lex.Next() 72 | if p.lex.Token != scanner.TLEFTCURLY { 73 | return nil, p.unexpected("{") 74 | } 75 | 76 | inlineLeftCurly := p.parseInlineComment() 77 | 78 | var number string 79 | var fullName string 80 | var typeStr string 81 | var repeated bool 82 | var reserved bool 83 | 84 | for { 85 | p.lex.Next() 86 | if p.lex.Token == scanner.TRIGHTCURLY { 87 | break 88 | } 89 | if p.lex.Token != scanner.TCOMMA { 90 | p.lex.UnNext() 91 | } 92 | 93 | p.lex.NextKeyword() 94 | if p.lex.Token == scanner.TNUMBER { 95 | p.lex.Next() 96 | if p.lex.Token != scanner.TCOLON { 97 | return nil, p.unexpected(":") 98 | } 99 | p.lex.NextNumberLit() 100 | if p.lex.Token != scanner.TINTLIT { 101 | return nil, p.unexpected("number") 102 | } 103 | number = p.lex.Text 104 | } else if p.lex.Token == scanner.TFULLNAME { 105 | p.lex.Next() 106 | if p.lex.Token != scanner.TCOLON { 107 | return nil, p.unexpected(":") 108 | } 109 | p.lex.NextStrLit() 110 | if p.lex.Token != scanner.TSTRLIT { 111 | return nil, p.unexpected("full_name string") 112 | } 113 | fullName = p.lex.Text 114 | } else if p.lex.Token == scanner.TTYPE { 115 | p.lex.Next() 116 | if p.lex.Token != scanner.TCOLON { 117 | return nil, p.unexpected(":") 118 | } 119 | p.lex.NextStrLit() 120 | if p.lex.Token != scanner.TSTRLIT { 121 | return nil, p.unexpected("type string") 122 | } 123 | typeStr = p.lex.Text 124 | } else if p.lex.Token == scanner.TREPEATED { 125 | p.lex.Next() 126 | if p.lex.Token != scanner.TCOLON { 127 | return nil, p.unexpected(":") 128 | } 129 | p.lex.Next() 130 | if p.lex.Token != scanner.TIDENT { 131 | return nil, p.unexpected("repeated bool") 132 | } 133 | repeated = p.lex.Text == "true" 134 | } else if p.lex.Token == scanner.TRESERVED { 135 | p.lex.Next() 136 | if p.lex.Token != scanner.TCOLON { 137 | return nil, p.unexpected(":") 138 | } 139 | p.lex.Next() 140 | if p.lex.Token != scanner.TIDENT { 141 | return nil, p.unexpected("reserved bool") 142 | } 143 | reserved = p.lex.Text == "true" 144 | } else { 145 | return nil, p.unexpected("number, full_name, type, repeated, reserved, or }") 146 | } 147 | } 148 | 149 | return &Declaration{ 150 | Number: number, 151 | FullName: fullName, 152 | Type: typeStr, 153 | Reserved: reserved, 154 | Repeated: repeated, 155 | InlineCommentBehindLeftCurly: inlineLeftCurly, 156 | Meta: meta.Meta{Pos: startPos.Position, LastPos: p.lex.Pos.Position}, 157 | }, nil 158 | } 159 | -------------------------------------------------------------------------------- /parser/declaration_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/yoheimuta/go-protoparser/v4/internal/util_test" 9 | "github.com/yoheimuta/go-protoparser/v4/lexer" 10 | "github.com/yoheimuta/go-protoparser/v4/parser" 11 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 12 | ) 13 | 14 | func TestParser_ParseDeclaration(t *testing.T) { 15 | tests := []struct { 16 | name string 17 | input string 18 | wantDeclaration *parser.Declaration 19 | wantErr bool 20 | }{ 21 | { 22 | name: "parsing an empty", 23 | wantErr: true, 24 | }, 25 | { 26 | name: "parsing an excerpt from the official reference", 27 | input: `declaration = { 28 | number: 4, 29 | full_name: ".my.package.event_annotations", 30 | type: ".logs.proto.ValidationAnnotations", 31 | repeated: true }`, 32 | wantDeclaration: &parser.Declaration{ 33 | Number: "4", 34 | FullName: `".my.package.event_annotations"`, 35 | Type: `".logs.proto.ValidationAnnotations"`, 36 | Repeated: true, 37 | Meta: meta.Meta{ 38 | Pos: meta.Position{ 39 | Offset: 0, 40 | Line: 1, 41 | Column: 1, 42 | }, 43 | LastPos: meta.Position{ 44 | Offset: 153, 45 | Line: 5, 46 | Column: 22, 47 | }, 48 | }, 49 | }, 50 | }, 51 | { 52 | name: "parsing another excerpt from the official reference", 53 | input: `declaration = { 54 | number: 500, 55 | full_name: ".my.package.event_annotations", 56 | type: ".logs.proto.ValidationAnnotations", 57 | reserved: true }`, 58 | wantDeclaration: &parser.Declaration{ 59 | Number: "500", 60 | FullName: `".my.package.event_annotations"`, 61 | Type: `".logs.proto.ValidationAnnotations"`, 62 | Reserved: true, 63 | Meta: meta.Meta{ 64 | Pos: meta.Position{ 65 | Offset: 0, 66 | Line: 1, 67 | Column: 1, 68 | }, 69 | LastPos: meta.Position{ 70 | Offset: 155, 71 | Line: 5, 72 | Column: 22, 73 | }, 74 | }, 75 | }, 76 | }, 77 | } 78 | 79 | for _, test := range tests { 80 | test := test 81 | t.Run(test.name, func(t *testing.T) { 82 | p := parser.NewParser(lexer.NewLexer(strings.NewReader(test.input))) 83 | got, err := p.ParseDeclaration() 84 | switch { 85 | case test.wantErr: 86 | if err == nil { 87 | t.Errorf("got err nil, but want err") 88 | } 89 | return 90 | case !test.wantErr && err != nil: 91 | t.Errorf("got err %v, but want nil", err) 92 | return 93 | } 94 | 95 | if !reflect.DeepEqual(got, test.wantDeclaration) { 96 | t.Errorf("got %v, but want %v", util_test.PrettyFormat(got), util_test.PrettyFormat(test.wantDeclaration)) 97 | } 98 | 99 | if !p.IsEOF() { 100 | t.Errorf("got not eof, but want eof") 101 | } 102 | }) 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /parser/edition.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 5 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 6 | ) 7 | 8 | // Edition is used to define the protobuf version. 9 | type Edition struct { 10 | Edition string 11 | 12 | // EditionQuote includes quotes 13 | EditionQuote string 14 | 15 | // Comments are the optional ones placed at the beginning. 16 | Comments []*Comment 17 | // InlineComment is the optional one placed at the ending. 18 | InlineComment *Comment 19 | // Meta is the meta information. 20 | Meta meta.Meta 21 | } 22 | 23 | // SetInlineComment implements the HasInlineCommentSetter interface. 24 | func (s *Edition) SetInlineComment(comment *Comment) { 25 | s.InlineComment = comment 26 | } 27 | 28 | // Accept dispatches the call to the visitor. 29 | func (s *Edition) Accept(v Visitor) { 30 | if !v.VisitEdition(s) { 31 | return 32 | } 33 | 34 | for _, comment := range s.Comments { 35 | comment.Accept(v) 36 | } 37 | if s.InlineComment != nil { 38 | s.InlineComment.Accept(v) 39 | } 40 | } 41 | 42 | // ParseEdition parses the Edition. 43 | // 44 | // edition = "edition" "=" [ ( "'" decimalLit "'" ) | ( '"' decimalLit '"' ) ] ";" 45 | // 46 | // See https://protobuf.dev/reference/protobuf/edition-2023-spec/#edition 47 | func (p *Parser) ParseEdition() (*Edition, error) { 48 | p.lex.NextKeyword() 49 | if p.lex.Token != scanner.TEDITION { 50 | p.lex.UnNext() 51 | return nil, nil 52 | } 53 | startPos := p.lex.Pos 54 | 55 | p.lex.Next() 56 | if p.lex.Token != scanner.TEQUALS { 57 | return nil, p.unexpected("=") 58 | } 59 | 60 | p.lex.Next() 61 | if p.lex.Token != scanner.TQUOTE { 62 | return nil, p.unexpected("quote") 63 | } 64 | lq := p.lex.Text 65 | 66 | p.lex.NextNumberLit() 67 | if p.lex.Token != scanner.TINTLIT { 68 | return nil, p.unexpected("intLit") 69 | } 70 | edition := p.lex.Text 71 | 72 | p.lex.Next() 73 | if p.lex.Token != scanner.TQUOTE { 74 | return nil, p.unexpected("quote") 75 | } 76 | tq := p.lex.Text 77 | 78 | p.lex.Next() 79 | if p.lex.Token != scanner.TSEMICOLON { 80 | return nil, p.unexpected(";") 81 | } 82 | 83 | return &Edition{ 84 | Edition: edition, 85 | EditionQuote: lq + edition + tq, 86 | Meta: meta.Meta{ 87 | Pos: startPos.Position, 88 | LastPos: p.lex.Pos.Position, 89 | }, 90 | }, nil 91 | } 92 | -------------------------------------------------------------------------------- /parser/edition_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/yoheimuta/go-protoparser/v4/lexer" 9 | "github.com/yoheimuta/go-protoparser/v4/parser" 10 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 11 | ) 12 | 13 | func TestParser_ParseEdition(t *testing.T) { 14 | tests := []struct { 15 | name string 16 | input string 17 | wantEdition *parser.Edition 18 | wantErr bool 19 | }{ 20 | { 21 | name: "parsing an empty", 22 | }, 23 | { 24 | name: "parsing an excerpt from the official reference", 25 | input: `edition = "2023";`, 26 | wantEdition: &parser.Edition{ 27 | Edition: "2023", 28 | EditionQuote: `"2023"`, 29 | Meta: meta.Meta{ 30 | Pos: meta.Position{ 31 | Offset: 0, 32 | Line: 1, 33 | Column: 1, 34 | }, 35 | LastPos: meta.Position{ 36 | Offset: 16, 37 | Line: 1, 38 | Column: 17, 39 | }, 40 | }, 41 | }, 42 | }, 43 | } 44 | 45 | for _, test := range tests { 46 | test := test 47 | t.Run(test.name, func(t *testing.T) { 48 | p := parser.NewParser(lexer.NewLexer(strings.NewReader(test.input))) 49 | got, err := p.ParseEdition() 50 | switch { 51 | case test.wantErr: 52 | if err == nil { 53 | t.Errorf("got err nil, but want err") 54 | } 55 | return 56 | case !test.wantErr && err != nil: 57 | t.Errorf("got err %v, but want nil", err) 58 | return 59 | } 60 | 61 | if !reflect.DeepEqual(got, test.wantEdition) { 62 | t.Errorf("got %v, but want %v", got, test.wantEdition) 63 | } 64 | 65 | if !p.IsEOF() { 66 | t.Errorf("got not eof, but want eof") 67 | } 68 | }) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /parser/emptyStatement.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | // EmptyStatement represents ";". 4 | type EmptyStatement struct { 5 | // InlineComment is the optional one placed at the ending. 6 | InlineComment *Comment 7 | } 8 | 9 | // SetInlineComment implements the HasInlineCommentSetter interface. 10 | func (e *EmptyStatement) SetInlineComment(comment *Comment) { 11 | e.InlineComment = comment 12 | } 13 | 14 | // Accept dispatches the call to the visitor. 15 | func (e *EmptyStatement) Accept(v Visitor) { 16 | if !v.VisitEmptyStatement(e) { 17 | return 18 | } 19 | 20 | if e.InlineComment != nil { 21 | e.InlineComment.Accept(v) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /parser/error.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | 7 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 8 | ) 9 | 10 | func (p *Parser) unexpected(expected string) error { 11 | _, file, line, _ := runtime.Caller(1) 12 | err := &meta.Error{ 13 | Pos: p.lex.Pos.Position, 14 | Expected: expected, 15 | Found: fmt.Sprintf("%q(Token=%v, Pos=%s)", p.lex.Text, p.lex.Token, p.lex.Pos), 16 | } 17 | err.SetOccured(file, line) 18 | return err 19 | } 20 | 21 | func (p *Parser) unexpectedf( 22 | format string, 23 | a ...interface{}, 24 | ) error { 25 | return p.unexpected(fmt.Sprintf(format, a...)) 26 | } 27 | -------------------------------------------------------------------------------- /parser/extend.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 7 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 8 | ) 9 | 10 | type parseExtendBodyStatementErr struct { 11 | parseFieldErr error 12 | parseEmptyStatementErr error 13 | } 14 | 15 | func (e *parseExtendBodyStatementErr) Error() string { 16 | return fmt.Sprintf( 17 | "%v:%v", 18 | e.parseFieldErr, 19 | e.parseEmptyStatementErr, 20 | ) 21 | } 22 | 23 | // Extend consists of a messageType and an extend body. 24 | type Extend struct { 25 | MessageType string 26 | // ExtendBody can have fields and emptyStatements 27 | ExtendBody []Visitee 28 | 29 | // Comments are the optional ones placed at the beginning. 30 | Comments []*Comment 31 | // InlineComment is the optional one placed at the ending. 32 | InlineComment *Comment 33 | // InlineCommentBehindLeftCurly is the optional one placed behind a left curly. 34 | InlineCommentBehindLeftCurly *Comment 35 | // Meta is the meta information. 36 | Meta meta.Meta 37 | } 38 | 39 | // SetInlineComment implements the HasInlineCommentSetter interface. 40 | func (m *Extend) SetInlineComment(comment *Comment) { 41 | m.InlineComment = comment 42 | } 43 | 44 | // Accept dispatches the call to the visitor. 45 | func (m *Extend) Accept(v Visitor) { 46 | if !v.VisitExtend(m) { 47 | return 48 | } 49 | 50 | for _, body := range m.ExtendBody { 51 | body.Accept(v) 52 | } 53 | for _, comment := range m.Comments { 54 | comment.Accept(v) 55 | } 56 | if m.InlineComment != nil { 57 | m.InlineComment.Accept(v) 58 | } 59 | if m.InlineCommentBehindLeftCurly != nil { 60 | m.InlineCommentBehindLeftCurly.Accept(v) 61 | } 62 | } 63 | 64 | // ParseExtend parses the extend. 65 | // Note that group is not supported. 66 | // 67 | // extend = "extend" messageType "{" {field | group | emptyStatement} "}" 68 | // 69 | // See https://developers.google.com/protocol-buffers/docs/reference/proto2-spec#extend 70 | func (p *Parser) ParseExtend() (*Extend, error) { 71 | p.lex.NextKeyword() 72 | if p.lex.Token != scanner.TEXTEND { 73 | return nil, p.unexpected("extend") 74 | } 75 | startPos := p.lex.Pos 76 | 77 | messageType, _, err := p.lex.ReadMessageType() 78 | if err != nil { 79 | return nil, err 80 | } 81 | 82 | extendBody, inlineLeftCurly, lastPos, err := p.parseExtendBody() 83 | if err != nil { 84 | return nil, err 85 | } 86 | 87 | return &Extend{ 88 | MessageType: messageType, 89 | ExtendBody: extendBody, 90 | InlineCommentBehindLeftCurly: inlineLeftCurly, 91 | Meta: meta.Meta{ 92 | Pos: startPos.Position, 93 | LastPos: lastPos.Position, 94 | }, 95 | }, nil 96 | } 97 | 98 | // extendBody = "{" {field | group | emptyStatement} "}" 99 | func (p *Parser) parseExtendBody() ( 100 | []Visitee, 101 | *Comment, 102 | scanner.Position, 103 | error, 104 | ) { 105 | p.lex.Next() 106 | if p.lex.Token != scanner.TLEFTCURLY { 107 | return nil, nil, scanner.Position{}, p.unexpected("{") 108 | } 109 | 110 | inlineLeftCurly := p.parseInlineComment() 111 | 112 | // Parses emptyBody. This spec is not documented, but allowed in general. { 113 | p.lex.Next() 114 | if p.lex.Token == scanner.TRIGHTCURLY { 115 | lastPos := p.lex.Pos 116 | if p.permissive { 117 | // accept a block followed by semicolon. See https://github.com/yoheimuta/go-protoparser/v4/issues/30. 118 | p.lex.ConsumeToken(scanner.TSEMICOLON) 119 | if p.lex.Token == scanner.TSEMICOLON { 120 | lastPos = p.lex.Pos 121 | } 122 | } 123 | 124 | return nil, nil, lastPos, nil 125 | } 126 | p.lex.UnNext() 127 | // } 128 | 129 | var stmts []Visitee 130 | 131 | for { 132 | comments := p.ParseComments() 133 | 134 | p.lex.NextKeyword() 135 | token := p.lex.Token 136 | p.lex.UnNext() 137 | 138 | var stmt interface { 139 | HasInlineCommentSetter 140 | Visitee 141 | } 142 | 143 | switch token { 144 | case scanner.TRIGHTCURLY: 145 | if p.bodyIncludingComments { 146 | for _, comment := range comments { 147 | stmts = append(stmts, Visitee(comment)) 148 | } 149 | } 150 | p.lex.Next() 151 | 152 | lastPos := p.lex.Pos 153 | if p.permissive { 154 | // accept a block followed by semicolon. See https://github.com/yoheimuta/go-protoparser/v4/issues/30. 155 | p.lex.ConsumeToken(scanner.TSEMICOLON) 156 | if p.lex.Token == scanner.TSEMICOLON { 157 | lastPos = p.lex.Pos 158 | } 159 | } 160 | return stmts, inlineLeftCurly, lastPos, nil 161 | default: 162 | field, fieldErr := p.ParseField() 163 | if fieldErr == nil { 164 | field.Comments = comments 165 | stmt = field 166 | break 167 | } 168 | p.lex.UnNext() 169 | 170 | emptyErr := p.lex.ReadEmptyStatement() 171 | if emptyErr == nil { 172 | stmt = &EmptyStatement{} 173 | break 174 | } 175 | 176 | return nil, nil, scanner.Position{}, &parseExtendBodyStatementErr{ 177 | parseFieldErr: fieldErr, 178 | parseEmptyStatementErr: emptyErr, 179 | } 180 | } 181 | 182 | p.MaybeScanInlineComment(stmt) 183 | stmts = append(stmts, stmt) 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /parser/extend_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 9 | 10 | "github.com/yoheimuta/go-protoparser/v4/internal/util_test" 11 | "github.com/yoheimuta/go-protoparser/v4/lexer" 12 | "github.com/yoheimuta/go-protoparser/v4/parser" 13 | ) 14 | 15 | func TestParser_ParseExtend(t *testing.T) { 16 | tests := []struct { 17 | name string 18 | input string 19 | inputBodyIncludingComments bool 20 | permissive bool 21 | wantExtend *parser.Extend 22 | wantErr bool 23 | }{ 24 | { 25 | name: "parsing an empty", 26 | wantErr: true, 27 | }, 28 | { 29 | name: "parsing an excerpt from the official reference", 30 | input: ` 31 | extend Foo { 32 | int32 bar = 126; 33 | } 34 | `, 35 | wantExtend: &parser.Extend{ 36 | MessageType: "Foo", 37 | ExtendBody: []parser.Visitee{ 38 | &parser.Field{ 39 | Type: "int32", 40 | FieldName: "bar", 41 | FieldNumber: "126", 42 | Meta: meta.Meta{ 43 | Pos: meta.Position{ 44 | Offset: 16, 45 | Line: 3, 46 | Column: 3, 47 | }, 48 | LastPos: meta.Position{ 49 | Offset: 31, 50 | Line: 3, 51 | Column: 18, 52 | }, 53 | }, 54 | }, 55 | }, 56 | Meta: meta.Meta{ 57 | Pos: meta.Position{ 58 | Offset: 1, 59 | Line: 2, 60 | Column: 1, 61 | }, 62 | LastPos: meta.Position{ 63 | Offset: 33, 64 | Line: 4, 65 | Column: 1, 66 | }, 67 | }, 68 | }, 69 | }, 70 | { 71 | name: "parsing an excerpt from the google/api/annotations.proto", 72 | input: ` 73 | extend google.protobuf.MethodOptions { 74 | // See HttpRule. 75 | HttpRule http = 72295728; 76 | }`, 77 | wantExtend: &parser.Extend{ 78 | MessageType: "google.protobuf.MethodOptions", 79 | ExtendBody: []parser.Visitee{ 80 | &parser.Field{ 81 | Type: "HttpRule", 82 | FieldName: "http", 83 | FieldNumber: "72295728", 84 | Comments: []*parser.Comment{ 85 | { 86 | Raw: "// See HttpRule.", 87 | Meta: meta.Meta{ 88 | Pos: meta.Position{ 89 | Offset: 42, 90 | Line: 3, 91 | Column: 3, 92 | }, 93 | LastPos: meta.Position{ 94 | Offset: 57, 95 | Line: 3, 96 | Column: 18, 97 | }, 98 | }, 99 | }, 100 | }, 101 | Meta: meta.Meta{ 102 | Pos: meta.Position{ 103 | Offset: 61, 104 | Line: 4, 105 | Column: 3, 106 | }, 107 | LastPos: meta.Position{ 108 | Offset: 85, 109 | Line: 4, 110 | Column: 27, 111 | }, 112 | }, 113 | }, 114 | }, 115 | Meta: meta.Meta{ 116 | Pos: meta.Position{ 117 | Offset: 1, 118 | Line: 2, 119 | Column: 1, 120 | }, 121 | LastPos: meta.Position{ 122 | Offset: 87, 123 | Line: 5, 124 | Column: 1, 125 | }, 126 | }, 127 | }, 128 | }, 129 | { 130 | name: "parsing a block followed by semicolon", 131 | input: ` 132 | extend Foo { 133 | }; 134 | `, 135 | permissive: true, 136 | wantExtend: &parser.Extend{ 137 | MessageType: "Foo", 138 | Meta: meta.Meta{ 139 | Pos: meta.Position{ 140 | Offset: 1, 141 | Line: 2, 142 | Column: 1, 143 | }, 144 | LastPos: meta.Position{ 145 | Offset: 15, 146 | Line: 3, 147 | Column: 2, 148 | }, 149 | }, 150 | }, 151 | }, 152 | { 153 | name: "set LastPos to the correct position when a semicolon doesn't follow the last block", 154 | input: ` 155 | extend Foo { 156 | } 157 | `, 158 | permissive: true, 159 | wantExtend: &parser.Extend{ 160 | MessageType: "Foo", 161 | Meta: meta.Meta{ 162 | Pos: meta.Position{ 163 | Offset: 1, 164 | Line: 2, 165 | Column: 1, 166 | }, 167 | LastPos: meta.Position{ 168 | Offset: 14, 169 | Line: 3, 170 | Column: 1, 171 | }, 172 | }, 173 | }, 174 | }, 175 | } 176 | 177 | for _, test := range tests { 178 | test := test 179 | t.Run(test.name, func(t *testing.T) { 180 | p := parser.NewParser( 181 | lexer.NewLexer(strings.NewReader(test.input)), 182 | parser.WithBodyIncludingComments(test.inputBodyIncludingComments), 183 | parser.WithPermissive(test.permissive), 184 | ) 185 | got, err := p.ParseExtend() 186 | switch { 187 | case test.wantErr: 188 | if err == nil { 189 | t.Errorf("got err nil, but want err, parsed=%v", got) 190 | } 191 | return 192 | case !test.wantErr && err != nil: 193 | t.Errorf("got err %v, but want nil", err) 194 | return 195 | } 196 | 197 | if !reflect.DeepEqual(got, test.wantExtend) { 198 | t.Errorf("got %v, but want %v", util_test.PrettyFormat(got), util_test.PrettyFormat(test.wantExtend)) 199 | } 200 | 201 | if !p.IsEOF() { 202 | t.Errorf("got not eof, but want eof") 203 | } 204 | }) 205 | } 206 | 207 | } 208 | -------------------------------------------------------------------------------- /parser/extensions.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 5 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 6 | ) 7 | 8 | // Extensions declare that a range of field numbers in a message are available for third-party extensions. 9 | type Extensions struct { 10 | Ranges []*Range 11 | Declarations []*Declaration 12 | 13 | // Comments are the optional ones placed at the beginning. 14 | Comments []*Comment 15 | // InlineComment is the optional one placed at the ending. 16 | InlineComment *Comment 17 | // InlineCommentBehindLeftSquare is the optional one placed behind a left square. 18 | InlineCommentBehindLeftSquare *Comment 19 | // Meta is the meta information. 20 | Meta meta.Meta 21 | } 22 | 23 | // SetInlineComment implements the HasInlineCommentSetter interface. 24 | func (e *Extensions) SetInlineComment(comment *Comment) { 25 | e.InlineComment = comment 26 | } 27 | 28 | // Accept dispatches the call to the visitor. 29 | func (e *Extensions) Accept(v Visitor) { 30 | if !v.VisitExtensions(e) { 31 | return 32 | } 33 | 34 | for _, declaration := range e.Declarations { 35 | declaration.Accept(v) 36 | } 37 | for _, comment := range e.Comments { 38 | comment.Accept(v) 39 | } 40 | if e.InlineComment != nil { 41 | e.InlineComment.Accept(v) 42 | } 43 | } 44 | 45 | // ParseExtensions parses the extensions. 46 | // 47 | // extensions = "extensions" ranges ";" 48 | // 49 | // See https://developers.google.com/protocol-buffers/docs/reference/proto2-spec#extensions 50 | func (p *Parser) ParseExtensions() (*Extensions, error) { 51 | p.lex.NextKeyword() 52 | if p.lex.Token != scanner.TEXTENSIONS { 53 | return nil, p.unexpected("extensions") 54 | } 55 | startPos := p.lex.Pos 56 | 57 | ranges, err := p.parseRanges() 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | declarations, inlineLeftSquare, err := p.parseDeclarations() 63 | if err != nil { 64 | return nil, err 65 | } 66 | 67 | p.lex.Next() 68 | if p.lex.Token != scanner.TSEMICOLON { 69 | return nil, p.unexpected(";") 70 | } 71 | 72 | return &Extensions{ 73 | Ranges: ranges, 74 | Declarations: declarations, 75 | InlineCommentBehindLeftSquare: inlineLeftSquare, 76 | Meta: meta.Meta{Pos: startPos.Position, LastPos: p.lex.Pos.Position}, 77 | }, nil 78 | } 79 | 80 | // parseDeclarations parses the declarations. 81 | // 82 | // declarations = "[" declaration { "," declaration } "]" 83 | // 84 | // See https://protobuf.dev/programming-guides/extension_declarations/ 85 | func (p *Parser) parseDeclarations() ([]*Declaration, *Comment, error) { 86 | declarations := []*Declaration{} 87 | p.lex.Next() 88 | if p.lex.Token != scanner.TLEFTSQUARE { 89 | p.lex.UnNext() 90 | return nil, nil, nil 91 | } 92 | inlineLeftSquare := p.parseInlineComment() 93 | 94 | for { 95 | comments := p.ParseComments() 96 | 97 | declaration, err := p.ParseDeclaration() 98 | if err != nil { 99 | return nil, nil, err 100 | } 101 | declaration.Comments = comments 102 | declarations = append(declarations, declaration) 103 | 104 | p.lex.Next() 105 | token := p.lex.Token 106 | inlineComment1 := p.parseInlineComment() 107 | if token == scanner.TRIGHTSQUARE { 108 | p.assignInlineComments(declaration, inlineComment1, p.parseInlineComment()) 109 | break 110 | } 111 | if token != scanner.TCOMMA { 112 | return nil, nil, p.unexpected(", or ]") 113 | } 114 | p.assignInlineComments(declaration, inlineComment1, p.parseInlineComment()) 115 | } 116 | return declarations, inlineLeftSquare, nil 117 | } 118 | 119 | // assignInlineComments assigns inline comments to a declaration, ensuring proper order. 120 | func (p *Parser) assignInlineComments(declaration *Declaration, comments ...*Comment) { 121 | for _, comment := range comments { 122 | if comment != nil { 123 | declaration.SetInlineComment(comment) 124 | } 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /parser/extensions_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/yoheimuta/go-protoparser/v4/internal/util_test" 9 | "github.com/yoheimuta/go-protoparser/v4/lexer" 10 | "github.com/yoheimuta/go-protoparser/v4/parser" 11 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 12 | ) 13 | 14 | func TestParser_ParseExtensions(t *testing.T) { 15 | tests := []struct { 16 | name string 17 | input string 18 | wantExtensions *parser.Extensions 19 | wantErr bool 20 | }{ 21 | { 22 | name: "parsing an empty", 23 | wantErr: true, 24 | }, 25 | { 26 | name: "parsing an invalid; without to", 27 | input: "extensions 2, 15, 9 11;", 28 | wantErr: true, 29 | }, 30 | { 31 | name: "parsing an invalid; including both ranges and fieldNames", 32 | input: `extensions 2, "foo", 9 to 11;`, 33 | wantErr: true, 34 | }, 35 | { 36 | name: "parsing an excerpt from the official reference", 37 | input: `extensions 100 to 199;`, 38 | wantExtensions: &parser.Extensions{ 39 | Ranges: []*parser.Range{ 40 | { 41 | Begin: "100", 42 | End: "199", 43 | }, 44 | }, 45 | Meta: meta.Meta{ 46 | Pos: meta.Position{ 47 | Offset: 0, 48 | Line: 1, 49 | Column: 1, 50 | }, 51 | LastPos: meta.Position{ 52 | Offset: 21, 53 | Line: 1, 54 | Column: 22, 55 | }, 56 | }, 57 | }, 58 | }, 59 | { 60 | name: "parsing another excerpt from the official reference", 61 | input: `extensions 4, 20 to max;`, 62 | wantExtensions: &parser.Extensions{ 63 | Ranges: []*parser.Range{ 64 | { 65 | Begin: "4", 66 | }, 67 | { 68 | Begin: "20", 69 | End: "max", 70 | }, 71 | }, 72 | Meta: meta.Meta{ 73 | Pos: meta.Position{ 74 | Offset: 0, 75 | Line: 1, 76 | Column: 1, 77 | }, 78 | LastPos: meta.Position{ 79 | Offset: 23, 80 | Line: 1, 81 | Column: 24, 82 | }, 83 | }, 84 | }, 85 | }, 86 | { 87 | name: "parsing an excerpt with extension declarations from the official reference", 88 | input: `extensions 4 to 1000 [ 89 | declaration = { 90 | number: 4, 91 | full_name: ".my.package.event_annotations", 92 | type: ".logs.proto.ValidationAnnotations", 93 | repeated: true }, 94 | declaration = { 95 | number: 999, 96 | full_name: ".foo.package.bar", 97 | type: "int32"}];`, 98 | wantExtensions: &parser.Extensions{ 99 | Ranges: []*parser.Range{ 100 | { 101 | Begin: "4", 102 | End: "1000", 103 | }, 104 | }, 105 | Declarations: []*parser.Declaration{ 106 | { 107 | Number: "4", 108 | FullName: `".my.package.event_annotations"`, 109 | Type: `".logs.proto.ValidationAnnotations"`, 110 | Repeated: true, 111 | Meta: meta.Meta{ 112 | Pos: meta.Position{ 113 | Offset: 27, 114 | Line: 2, 115 | Column: 5, 116 | }, 117 | LastPos: meta.Position{ 118 | Offset: 180, 119 | Line: 6, 120 | Column: 22, 121 | }, 122 | }, 123 | }, 124 | { 125 | Number: "999", 126 | FullName: `".foo.package.bar"`, 127 | Type: `"int32"`, 128 | Meta: meta.Meta{ 129 | Pos: meta.Position{ 130 | Offset: 187, 131 | Line: 7, 132 | Column: 5, 133 | }, 134 | LastPos: meta.Position{ 135 | Offset: 278, 136 | Line: 10, 137 | Column: 20, 138 | }, 139 | }, 140 | }, 141 | }, 142 | Meta: meta.Meta{ 143 | Pos: meta.Position{ 144 | Offset: 0, 145 | Line: 1, 146 | Column: 1, 147 | }, 148 | LastPos: meta.Position{ 149 | Offset: 280, 150 | Line: 10, 151 | Column: 22, 152 | }, 153 | }, 154 | }, 155 | }, 156 | } 157 | 158 | for _, test := range tests { 159 | test := test 160 | t.Run(test.name, func(t *testing.T) { 161 | p := parser.NewParser(lexer.NewLexer(strings.NewReader(test.input))) 162 | got, err := p.ParseExtensions() 163 | switch { 164 | case test.wantErr: 165 | if err == nil { 166 | t.Errorf("got err nil, but want err") 167 | } 168 | return 169 | case !test.wantErr && err != nil: 170 | t.Errorf("got err %v, but want nil", err) 171 | return 172 | } 173 | 174 | if !reflect.DeepEqual(got, test.wantExtensions) { 175 | t.Errorf("got %v, but want %v", util_test.PrettyFormat(got), util_test.PrettyFormat(test.wantExtensions)) 176 | } 177 | 178 | if !p.IsEOF() { 179 | t.Errorf("got not eof, but want eof") 180 | } 181 | }) 182 | } 183 | 184 | } 185 | -------------------------------------------------------------------------------- /parser/field.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 5 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 6 | ) 7 | 8 | // FieldOption is an option for the field. 9 | type FieldOption struct { 10 | OptionName string 11 | Constant string 12 | } 13 | 14 | // Field is a normal field that is the basic element of a protocol buffer message. 15 | type Field struct { 16 | IsRepeated bool 17 | IsRequired bool // proto2 only 18 | IsOptional bool // proto2 only 19 | Type string 20 | FieldName string 21 | FieldNumber string 22 | FieldOptions []*FieldOption 23 | 24 | // Comments are the optional ones placed at the beginning. 25 | Comments []*Comment 26 | // InlineComment is the optional one placed at the ending. 27 | InlineComment *Comment 28 | // Meta is the meta information. 29 | Meta meta.Meta 30 | } 31 | 32 | // SetInlineComment implements the HasInlineCommentSetter interface. 33 | func (f *Field) SetInlineComment(comment *Comment) { 34 | f.InlineComment = comment 35 | } 36 | 37 | // Accept dispatches the call to the visitor. 38 | func (f *Field) Accept(v Visitor) { 39 | if !v.VisitField(f) { 40 | return 41 | } 42 | 43 | for _, comment := range f.Comments { 44 | comment.Accept(v) 45 | } 46 | if f.InlineComment != nil { 47 | f.InlineComment.Accept(v) 48 | } 49 | } 50 | 51 | // ParseField parses the field. 52 | // 53 | // field = [ "repeated" ] type fieldName "=" fieldNumber [ "[" fieldOptions "]" ] ";" 54 | // field = [ "required" | "optional" | "repeated" ] type fieldName "=" fieldNumber [ "[" fieldOptions "]" ] ";" 55 | // 56 | // See 57 | // 58 | // https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#normal_field 59 | // https://developers.google.com/protocol-buffers/docs/reference/proto2-spec#normal_field 60 | func (p *Parser) ParseField() (*Field, error) { 61 | var isRepeated bool 62 | var isRequired bool 63 | var isOptional bool 64 | p.lex.NextKeyword() 65 | startPos := p.lex.Pos 66 | 67 | if p.lex.Token == scanner.TREPEATED { 68 | isRepeated = true 69 | } else if p.lex.Token == scanner.TREQUIRED { 70 | isRequired = true 71 | } else if p.lex.Token == scanner.TOPTIONAL { 72 | isOptional = true 73 | } else { 74 | p.lex.UnNext() 75 | } 76 | 77 | typeValue, _, err := p.parseType() 78 | if err != nil { 79 | return nil, p.unexpected("type") 80 | } 81 | 82 | p.lex.Next() 83 | if p.lex.Token != scanner.TIDENT { 84 | return nil, p.unexpected("fieldName") 85 | } 86 | fieldName := p.lex.Text 87 | 88 | p.lex.Next() 89 | if p.lex.Token != scanner.TEQUALS { 90 | return nil, p.unexpected("=") 91 | } 92 | 93 | fieldNumber, err := p.parseFieldNumber() 94 | if err != nil { 95 | return nil, p.unexpected("fieldNumber") 96 | } 97 | 98 | fieldOptions, err := p.parseFieldOptionsOption() 99 | if err != nil { 100 | return nil, err 101 | } 102 | 103 | p.lex.Next() 104 | if p.lex.Token != scanner.TSEMICOLON { 105 | return nil, p.unexpected(";") 106 | } 107 | 108 | return &Field{ 109 | IsRepeated: isRepeated, 110 | IsRequired: isRequired, 111 | IsOptional: isOptional, 112 | Type: typeValue, 113 | FieldName: fieldName, 114 | FieldNumber: fieldNumber, 115 | FieldOptions: fieldOptions, 116 | Meta: meta.Meta{Pos: startPos.Position, LastPos: p.lex.Pos.Position}, 117 | }, nil 118 | } 119 | 120 | // [ "[" fieldOptions "]" ] 121 | func (p *Parser) parseFieldOptionsOption() ([]*FieldOption, error) { 122 | p.lex.Next() 123 | if p.lex.Token == scanner.TLEFTSQUARE { 124 | fieldOptions, err := p.parseFieldOptions() 125 | if err != nil { 126 | return nil, err 127 | } 128 | 129 | p.lex.Next() 130 | if p.lex.Token != scanner.TRIGHTSQUARE { 131 | return nil, p.unexpected("]") 132 | } 133 | return fieldOptions, nil 134 | } 135 | p.lex.UnNext() 136 | return nil, nil 137 | } 138 | 139 | // fieldOptions = fieldOption { "," fieldOption } 140 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#field 141 | func (p *Parser) parseFieldOptions() ([]*FieldOption, error) { 142 | opt, err := p.parseFieldOption() 143 | if err != nil { 144 | return nil, err 145 | } 146 | 147 | var opts []*FieldOption 148 | opts = append(opts, opt) 149 | 150 | for { 151 | p.lex.Next() 152 | if p.lex.Token != scanner.TCOMMA { 153 | p.lex.UnNext() 154 | break 155 | } 156 | 157 | opt, err = p.parseFieldOption() 158 | if err != nil { 159 | return nil, p.unexpected("fieldOption") 160 | } 161 | opts = append(opts, opt) 162 | } 163 | return opts, nil 164 | } 165 | 166 | // fieldOption = optionName "=" constant 167 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#field 168 | func (p *Parser) parseFieldOption() (*FieldOption, error) { 169 | optionName, err := p.parseOptionName() 170 | if err != nil { 171 | return nil, err 172 | } 173 | 174 | p.lex.Next() 175 | if p.lex.Token != scanner.TEQUALS { 176 | return nil, p.unexpected("=") 177 | } 178 | 179 | constant, err := p.parseOptionConstant() 180 | if err != nil { 181 | return nil, err 182 | } 183 | 184 | return &FieldOption{ 185 | OptionName: optionName, 186 | Constant: constant, 187 | }, nil 188 | } 189 | 190 | var typeConstants = map[string]struct{}{ 191 | "double": {}, 192 | "float": {}, 193 | "int32": {}, 194 | "int64": {}, 195 | "uint32": {}, 196 | "uint64": {}, 197 | "sint32": {}, 198 | "sint64": {}, 199 | "fixed32": {}, 200 | "fixed64": {}, 201 | "sfixed32": {}, 202 | "sfixed64": {}, 203 | "bool": {}, 204 | "string": {}, 205 | "bytes": {}, 206 | } 207 | 208 | // type = "double" | "float" | "int32" | "int64" | "uint32" | "uint64" 209 | // 210 | // | "sint32" | "sint64" | "fixed32" | "fixed64" | "sfixed32" | "sfixed64" 211 | // | "bool" | "string" | "bytes" | messageType | enumType 212 | // 213 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#fields 214 | func (p *Parser) parseType() (string, scanner.Position, error) { 215 | p.lex.Next() 216 | if _, ok := typeConstants[p.lex.Text]; ok { 217 | return p.lex.Text, p.lex.Pos, nil 218 | } 219 | p.lex.UnNext() 220 | 221 | messageOrEnumType, startPos, err := p.lex.ReadMessageType() 222 | if err != nil { 223 | return "", scanner.Position{}, err 224 | } 225 | return messageOrEnumType, startPos, nil 226 | } 227 | 228 | // fieldNumber = intLit; 229 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#fields 230 | func (p *Parser) parseFieldNumber() (string, error) { 231 | p.lex.NextNumberLit() 232 | if p.lex.Token != scanner.TINTLIT { 233 | return "", p.unexpected("intLit") 234 | } 235 | return p.lex.Text, nil 236 | } 237 | -------------------------------------------------------------------------------- /parser/groupField.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "unicode/utf8" 5 | 6 | "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 7 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 8 | ) 9 | 10 | // GroupField is one way to nest information in message definitions. 11 | // proto2 only. 12 | type GroupField struct { 13 | IsRepeated bool 14 | IsRequired bool 15 | IsOptional bool 16 | // GroupName must begin with capital letter. 17 | GroupName string 18 | // MessageBody can have fields, nested enum definitions, nested message definitions, 19 | // options, oneofs, map fields, extends, reserved, and extensions statements. 20 | MessageBody []Visitee 21 | FieldNumber string 22 | 23 | // Comments are the optional ones placed at the beginning. 24 | Comments []*Comment 25 | // InlineComment is the optional one placed at the ending. 26 | InlineComment *Comment 27 | // InlineCommentBehindLeftCurly is the optional one placed behind a left curly. 28 | InlineCommentBehindLeftCurly *Comment 29 | // Meta is the meta information. 30 | Meta meta.Meta 31 | } 32 | 33 | // SetInlineComment implements the HasInlineCommentSetter interface. 34 | func (f *GroupField) SetInlineComment(comment *Comment) { 35 | f.InlineComment = comment 36 | } 37 | 38 | // Accept dispatches the call to the visitor. 39 | func (f *GroupField) Accept(v Visitor) { 40 | if !v.VisitGroupField(f) { 41 | return 42 | } 43 | 44 | for _, body := range f.MessageBody { 45 | body.Accept(v) 46 | } 47 | for _, comment := range f.Comments { 48 | comment.Accept(v) 49 | } 50 | if f.InlineComment != nil { 51 | f.InlineComment.Accept(v) 52 | } 53 | if f.InlineCommentBehindLeftCurly != nil { 54 | f.InlineCommentBehindLeftCurly.Accept(v) 55 | } 56 | } 57 | 58 | // ParseGroupField parses the group. 59 | // 60 | // group = label "group" groupName "=" fieldNumber messageBody 61 | // 62 | // See https://developers.google.com/protocol-buffers/docs/reference/proto2-spec#group_field 63 | func (p *Parser) ParseGroupField() (*GroupField, error) { 64 | var isRepeated bool 65 | var isRequired bool 66 | var isOptional bool 67 | p.lex.NextKeyword() 68 | startPos := p.lex.Pos 69 | 70 | if p.lex.Token == scanner.TREPEATED { 71 | isRepeated = true 72 | } else if p.lex.Token == scanner.TREQUIRED { 73 | isRequired = true 74 | } else if p.lex.Token == scanner.TOPTIONAL { 75 | isOptional = true 76 | } else { 77 | p.lex.UnNext() 78 | } 79 | 80 | p.lex.NextKeyword() 81 | if p.lex.Token != scanner.TGROUP { 82 | return nil, p.unexpected("group") 83 | } 84 | 85 | p.lex.Next() 86 | if p.lex.Token != scanner.TIDENT { 87 | return nil, p.unexpected("groupName") 88 | } 89 | if !isCapitalized(p.lex.Text) { 90 | return nil, p.unexpectedf("groupName %q must begin with capital letter.", p.lex.Text) 91 | } 92 | groupName := p.lex.Text 93 | 94 | p.lex.Next() 95 | if p.lex.Token != scanner.TEQUALS { 96 | return nil, p.unexpected("=") 97 | } 98 | 99 | fieldNumber, err := p.parseFieldNumber() 100 | if err != nil { 101 | return nil, p.unexpected("fieldNumber") 102 | } 103 | 104 | messageBody, inlineLeftCurly, lastPos, err := p.parseMessageBody() 105 | if err != nil { 106 | return nil, err 107 | } 108 | 109 | return &GroupField{ 110 | IsRepeated: isRepeated, 111 | IsRequired: isRequired, 112 | IsOptional: isOptional, 113 | GroupName: groupName, 114 | FieldNumber: fieldNumber, 115 | MessageBody: messageBody, 116 | 117 | InlineCommentBehindLeftCurly: inlineLeftCurly, 118 | Meta: meta.Meta{ 119 | Pos: startPos.Position, 120 | LastPos: lastPos.Position, 121 | }, 122 | }, nil 123 | } 124 | 125 | func (p *Parser) peekIsGroup() bool { 126 | p.lex.NextKeyword() 127 | switch p.lex.Token { 128 | case scanner.TREPEATED, 129 | scanner.TREQUIRED, 130 | scanner.TOPTIONAL: 131 | defer p.lex.UnNextTo(p.lex.RawText) 132 | default: 133 | p.lex.UnNext() 134 | } 135 | 136 | p.lex.NextKeyword() 137 | defer p.lex.UnNextTo(p.lex.RawText) 138 | if p.lex.Token != scanner.TGROUP { 139 | return false 140 | } 141 | 142 | p.lex.Next() 143 | defer p.lex.UnNextTo(p.lex.RawText) 144 | if p.lex.Token != scanner.TIDENT { 145 | return false 146 | } 147 | if !isCapitalized(p.lex.Text) { 148 | return false 149 | } 150 | 151 | p.lex.Next() 152 | defer p.lex.UnNextTo(p.lex.RawText) 153 | if p.lex.Token != scanner.TEQUALS { 154 | return false 155 | } 156 | 157 | _, err := p.parseFieldNumber() 158 | defer p.lex.UnNextTo(p.lex.RawText) 159 | if err != nil { 160 | return false 161 | } 162 | 163 | p.lex.Next() 164 | defer p.lex.UnNextTo(p.lex.RawText) 165 | if p.lex.Token != scanner.TLEFTCURLY { 166 | return false 167 | } 168 | return true 169 | } 170 | 171 | // isCapitalized returns true if is not empty and the first letter is 172 | // an uppercase character. 173 | func isCapitalized(s string) bool { 174 | if s == "" { 175 | return false 176 | } 177 | r, _ := utf8.DecodeRuneInString(s) 178 | return isUpper(r) 179 | } 180 | 181 | func isUpper(r rune) bool { 182 | return 'A' <= r && r <= 'Z' 183 | } 184 | -------------------------------------------------------------------------------- /parser/groupField_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 9 | 10 | "github.com/yoheimuta/go-protoparser/v4/internal/util_test" 11 | "github.com/yoheimuta/go-protoparser/v4/lexer" 12 | "github.com/yoheimuta/go-protoparser/v4/parser" 13 | ) 14 | 15 | func TestParser_ParseGroupField(t *testing.T) { 16 | tests := []struct { 17 | name string 18 | input string 19 | permissive bool 20 | wantGroupField *parser.GroupField 21 | wantErr bool 22 | }{ 23 | { 24 | name: "parsing an empty", 25 | wantErr: true, 26 | }, 27 | { 28 | name: "parsing an invalid: groupName is not capitalized.", 29 | input: ` 30 | repeated group result = 1 { 31 | required string url = 2; 32 | optional string title = 3; 33 | repeated string snippets = 4; 34 | } 35 | `, 36 | wantErr: true, 37 | }, 38 | { 39 | name: "parsing an excerpt from the official reference", 40 | input: ` 41 | repeated group Result = 1 { 42 | required string url = 2; 43 | optional string title = 3; 44 | repeated string snippets = 4; 45 | } 46 | `, 47 | wantGroupField: &parser.GroupField{ 48 | IsRepeated: true, 49 | GroupName: "Result", 50 | FieldNumber: "1", 51 | MessageBody: []parser.Visitee{ 52 | &parser.Field{ 53 | IsRequired: true, 54 | Type: "string", 55 | FieldName: "url", 56 | FieldNumber: "2", 57 | Meta: meta.Meta{ 58 | Pos: meta.Position{ 59 | Offset: 33, 60 | Line: 3, 61 | Column: 5, 62 | }, 63 | LastPos: meta.Position{ 64 | Offset: 56, 65 | Line: 3, 66 | Column: 28, 67 | }, 68 | }, 69 | }, 70 | &parser.Field{ 71 | IsOptional: true, 72 | Type: "string", 73 | FieldName: "title", 74 | FieldNumber: "3", 75 | Meta: meta.Meta{ 76 | Pos: meta.Position{ 77 | Offset: 62, 78 | Line: 4, 79 | Column: 5, 80 | }, 81 | LastPos: meta.Position{ 82 | Offset: 87, 83 | Line: 4, 84 | Column: 30, 85 | }, 86 | }, 87 | }, 88 | &parser.Field{ 89 | IsRepeated: true, 90 | Type: "string", 91 | FieldName: "snippets", 92 | FieldNumber: "4", 93 | Meta: meta.Meta{ 94 | Pos: meta.Position{ 95 | Offset: 93, 96 | Line: 5, 97 | Column: 5, 98 | }, 99 | LastPos: meta.Position{ 100 | Offset: 121, 101 | Line: 5, 102 | Column: 33, 103 | }, 104 | }, 105 | }, 106 | }, 107 | Meta: meta.Meta{ 108 | Pos: meta.Position{ 109 | Offset: 1, 110 | Line: 2, 111 | Column: 1, 112 | }, 113 | LastPos: meta.Position{ 114 | Offset: 123, 115 | Line: 6, 116 | Column: 1, 117 | }, 118 | }, 119 | }, 120 | }, 121 | { 122 | name: "parsing a block followed by semicolon", 123 | input: ` 124 | group Result = 1 { 125 | }; 126 | `, 127 | permissive: true, 128 | wantGroupField: &parser.GroupField{ 129 | GroupName: "Result", 130 | FieldNumber: "1", 131 | Meta: meta.Meta{ 132 | Pos: meta.Position{ 133 | Offset: 1, 134 | Line: 2, 135 | Column: 1, 136 | }, 137 | LastPos: meta.Position{ 138 | Offset: 21, 139 | Line: 3, 140 | Column: 2, 141 | }, 142 | }, 143 | }, 144 | }, 145 | { 146 | name: "set LastPos to the correct position when a semicolon doesn't follow the last block", 147 | input: ` 148 | group Result = 1 { 149 | } 150 | `, 151 | permissive: true, 152 | wantGroupField: &parser.GroupField{ 153 | GroupName: "Result", 154 | FieldNumber: "1", 155 | Meta: meta.Meta{ 156 | Pos: meta.Position{ 157 | Offset: 1, 158 | Line: 2, 159 | Column: 1, 160 | }, 161 | LastPos: meta.Position{ 162 | Offset: 20, 163 | Line: 3, 164 | Column: 1, 165 | }, 166 | }, 167 | }, 168 | }, 169 | } 170 | 171 | for _, test := range tests { 172 | test := test 173 | t.Run(test.name, func(t *testing.T) { 174 | p := parser.NewParser(lexer.NewLexer(strings.NewReader(test.input)), parser.WithPermissive(test.permissive)) 175 | got, err := p.ParseGroupField() 176 | switch { 177 | case test.wantErr: 178 | if err == nil { 179 | t.Errorf("got err nil, but want err") 180 | } 181 | return 182 | case !test.wantErr && err != nil: 183 | t.Errorf("got err %v, but want nil", err) 184 | return 185 | } 186 | 187 | if !reflect.DeepEqual(got, test.wantGroupField) { 188 | t.Errorf("got %v, but want %v", util_test.PrettyFormat(got), util_test.PrettyFormat(test.wantGroupField)) 189 | 190 | } 191 | 192 | if !p.IsEOF() { 193 | t.Errorf("got not eof, but want eof") 194 | } 195 | }) 196 | } 197 | 198 | } 199 | -------------------------------------------------------------------------------- /parser/import.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 5 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 6 | ) 7 | 8 | // ImportModifier is a modifier enum type for import behavior. 9 | type ImportModifier uint 10 | 11 | // Optional import modifier value to change a default behavior. 12 | const ( 13 | ImportModifierNone ImportModifier = iota 14 | ImportModifierPublic 15 | ImportModifierWeak 16 | ) 17 | 18 | // Import is used to import another .proto's definitions. 19 | type Import struct { 20 | Modifier ImportModifier 21 | Location string 22 | 23 | // Comments are the optional ones placed at the beginning. 24 | Comments []*Comment 25 | // InlineComment is the optional one placed at the ending. 26 | InlineComment *Comment 27 | // Meta is the meta information. 28 | Meta meta.Meta 29 | } 30 | 31 | // SetInlineComment implements the HasInlineCommentSetter interface. 32 | func (i *Import) SetInlineComment(comment *Comment) { 33 | i.InlineComment = comment 34 | } 35 | 36 | // Accept dispatches the call to the visitor. 37 | func (i *Import) Accept(v Visitor) { 38 | if !v.VisitImport(i) { 39 | return 40 | } 41 | 42 | for _, comment := range i.Comments { 43 | comment.Accept(v) 44 | } 45 | if i.InlineComment != nil { 46 | i.InlineComment.Accept(v) 47 | } 48 | } 49 | 50 | // ParseImport parses the import. 51 | // 52 | // import = "import" [ "weak" | "public" ] strLit ";" 53 | // 54 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#import_statement 55 | func (p *Parser) ParseImport() (*Import, error) { 56 | p.lex.NextKeyword() 57 | if p.lex.Token != scanner.TIMPORT { 58 | return nil, p.unexpected(`"import"`) 59 | } 60 | startPos := p.lex.Pos 61 | 62 | var modifier ImportModifier 63 | p.lex.NextKeywordOrStrLit() 64 | switch p.lex.Token { 65 | case scanner.TPUBLIC: 66 | modifier = ImportModifierPublic 67 | case scanner.TWEAK: 68 | modifier = ImportModifierWeak 69 | case scanner.TSTRLIT: 70 | modifier = ImportModifierNone 71 | p.lex.UnNext() 72 | } 73 | 74 | p.lex.NextStrLit() 75 | if p.lex.Token != scanner.TSTRLIT { 76 | return nil, p.unexpected("strLit") 77 | } 78 | location := p.lex.Text 79 | 80 | p.lex.Next() 81 | if p.lex.Token != scanner.TSEMICOLON { 82 | return nil, p.unexpected(";") 83 | } 84 | 85 | return &Import{ 86 | Modifier: modifier, 87 | Location: location, 88 | Meta: meta.Meta{ 89 | Pos: startPos.Position, 90 | LastPos: p.lex.Pos.Position, 91 | }, 92 | }, nil 93 | } 94 | -------------------------------------------------------------------------------- /parser/import_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/yoheimuta/go-protoparser/v4/lexer" 9 | "github.com/yoheimuta/go-protoparser/v4/parser" 10 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 11 | ) 12 | 13 | func TestParser_ParseImport(t *testing.T) { 14 | tests := []struct { 15 | name string 16 | input string 17 | wantImport *parser.Import 18 | wantErr bool 19 | }{ 20 | { 21 | name: "parsing an empty", 22 | wantErr: true, 23 | }, 24 | { 25 | name: "parsing the invalid statement without import", 26 | input: `"other.proto";`, 27 | wantErr: true, 28 | }, 29 | { 30 | name: "parsing the invalid statement without strLit", 31 | input: `import 'other.proto";`, 32 | wantErr: true, 33 | }, 34 | { 35 | name: "parsing the statement without a modifier", 36 | input: `import "google/protobuf/timestamp.proto";`, 37 | wantImport: &parser.Import{ 38 | Modifier: parser.ImportModifierNone, 39 | Location: `"google/protobuf/timestamp.proto"`, 40 | Meta: meta.Meta{ 41 | Pos: meta.Position{ 42 | Offset: 0, 43 | Line: 1, 44 | Column: 1, 45 | }, 46 | LastPos: meta.Position{ 47 | Offset: 40, 48 | Line: 1, 49 | Column: 41, 50 | }, 51 | }, 52 | }, 53 | }, 54 | { 55 | name: "parsing an excerpt from the official reference", 56 | input: `import public "other.proto";`, 57 | wantImport: &parser.Import{ 58 | Modifier: parser.ImportModifierPublic, 59 | Location: `"other.proto"`, 60 | Meta: meta.Meta{ 61 | Pos: meta.Position{ 62 | Offset: 0, 63 | Line: 1, 64 | Column: 1, 65 | }, 66 | LastPos: meta.Position{ 67 | Offset: 27, 68 | Line: 1, 69 | Column: 28, 70 | }, 71 | }, 72 | }, 73 | }, 74 | { 75 | name: "parsing the statement with weak", 76 | input: `import weak "other.proto";`, 77 | wantImport: &parser.Import{ 78 | Modifier: parser.ImportModifierWeak, 79 | Location: `"other.proto"`, 80 | Meta: meta.Meta{ 81 | Pos: meta.Position{ 82 | Offset: 0, 83 | Line: 1, 84 | Column: 1, 85 | }, 86 | LastPos: meta.Position{ 87 | Offset: 25, 88 | Line: 1, 89 | Column: 26, 90 | }, 91 | }, 92 | }, 93 | }, 94 | } 95 | 96 | for _, test := range tests { 97 | test := test 98 | t.Run(test.name, func(t *testing.T) { 99 | p := parser.NewParser(lexer.NewLexer(strings.NewReader(test.input))) 100 | got, err := p.ParseImport() 101 | switch { 102 | case test.wantErr: 103 | if err == nil { 104 | t.Errorf("got err nil, but want err") 105 | } 106 | return 107 | case !test.wantErr && err != nil: 108 | t.Errorf("got err %v, but want nil", err) 109 | return 110 | } 111 | 112 | if !reflect.DeepEqual(got, test.wantImport) { 113 | t.Errorf("got %v, but want %v", got, test.wantImport) 114 | } 115 | 116 | if !p.IsEOF() { 117 | t.Errorf("got not eof, but want eof") 118 | } 119 | }) 120 | } 121 | 122 | } 123 | -------------------------------------------------------------------------------- /parser/inlineComment.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | // HasInlineCommentSetter requires to have a setter for an InlineComment field. 4 | type HasInlineCommentSetter interface { 5 | SetInlineComment(comment *Comment) 6 | } 7 | 8 | // MaybeScanInlineComment tries to scan a comment on the current line. If present then set it with setter. 9 | func (p *Parser) MaybeScanInlineComment( 10 | hasSetter HasInlineCommentSetter, 11 | ) { 12 | inlineComment := p.parseInlineComment() 13 | if inlineComment == nil { 14 | return 15 | } 16 | hasSetter.SetInlineComment(inlineComment) 17 | } 18 | 19 | func (p *Parser) parseInlineComment() *Comment { 20 | currentPos := p.lex.Pos 21 | 22 | comment, err := p.parseComment() 23 | if err != nil { 24 | return nil 25 | } 26 | 27 | if currentPos.Line != comment.Meta.Pos.Line { 28 | p.lex.UnNext() 29 | return nil 30 | } 31 | 32 | return comment 33 | } 34 | -------------------------------------------------------------------------------- /parser/inlineComment_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/yoheimuta/go-protoparser/v4/internal/util_test" 9 | "github.com/yoheimuta/go-protoparser/v4/lexer" 10 | "github.com/yoheimuta/go-protoparser/v4/parser" 11 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 12 | ) 13 | 14 | type mockHasInlineCommentSetter struct { 15 | inlineComment *parser.Comment 16 | } 17 | 18 | func (m *mockHasInlineCommentSetter) SetInlineComment(comment *parser.Comment) { 19 | m.inlineComment = comment 20 | } 21 | 22 | func TestParser_MaybeScanInlineComment(t *testing.T) { 23 | tests := []struct { 24 | name string 25 | input string 26 | wantInlineComment *parser.Comment 27 | }{ 28 | { 29 | name: "parsing an empty", 30 | }, 31 | { 32 | name: "parsing a C++-style comment on the current line", 33 | input: `int32 page_number = 2; // Which page number do we want? 34 | `, 35 | wantInlineComment: &parser.Comment{ 36 | Raw: "// Which page number do we want?", 37 | Meta: meta.Meta{ 38 | Pos: meta.Position{ 39 | Offset: 24, 40 | Line: 1, 41 | Column: 25, 42 | }, 43 | LastPos: meta.Position{ 44 | Offset: 55, 45 | Line: 1, 46 | Column: 56, 47 | }, 48 | }, 49 | }, 50 | }, 51 | { 52 | name: "parsing a C-style comment on the current line", 53 | input: `int32 page_number = 2; /* Which page number do we want? 54 | */ 55 | `, 56 | wantInlineComment: &parser.Comment{ 57 | Raw: `/* Which page number do we want? 58 | */`, 59 | Meta: meta.Meta{ 60 | Pos: meta.Position{ 61 | Offset: 24, 62 | Line: 1, 63 | Column: 25, 64 | }, 65 | LastPos: meta.Position{ 66 | Offset: 58, 67 | Line: 2, 68 | Column: 2, 69 | }, 70 | }, 71 | }, 72 | }, 73 | { 74 | name: "parsing a C++-style comment on the next line", 75 | input: `int32 page_number = 2; 76 | // Which page number do we want? 77 | `, 78 | }, 79 | { 80 | name: "parsing a C-style comment on the next line", 81 | input: `int32 page_number = 2; 82 | /* Which page number do we want? 83 | */ 84 | `, 85 | }, 86 | { 87 | name: "parsing C++-style comments on the current and next line", 88 | input: `int32 page_number = 2; // Which page number do we want? 89 | // Number of results to return per page. 90 | `, 91 | wantInlineComment: &parser.Comment{ 92 | Raw: "// Which page number do we want?", 93 | Meta: meta.Meta{ 94 | Pos: meta.Position{ 95 | Offset: 24, 96 | Line: 1, 97 | Column: 25, 98 | }, 99 | LastPos: meta.Position{ 100 | Offset: 55, 101 | Line: 1, 102 | Column: 56, 103 | }, 104 | }, 105 | }, 106 | }, 107 | { 108 | name: "parsing C-style comments on the current and next line", 109 | input: `int32 page_number = 2; /* Which page number do we want? 110 | */ 111 | /* Number of results to return per page. 112 | */ 113 | `, 114 | wantInlineComment: &parser.Comment{ 115 | Raw: `/* Which page number do we want? 116 | */`, 117 | Meta: meta.Meta{ 118 | Pos: meta.Position{ 119 | Offset: 24, 120 | Line: 1, 121 | Column: 25, 122 | }, 123 | LastPos: meta.Position{ 124 | Offset: 58, 125 | Line: 2, 126 | Column: 2, 127 | }, 128 | }, 129 | }, 130 | }, 131 | } 132 | 133 | for _, test := range tests { 134 | test := test 135 | t.Run(test.name, func(t *testing.T) { 136 | p := parser.NewParser(lexer.NewLexer(strings.NewReader(test.input))) 137 | _, _ = p.ParseField() 138 | 139 | hasSetter := &mockHasInlineCommentSetter{} 140 | p.MaybeScanInlineComment(hasSetter) 141 | got := hasSetter.inlineComment 142 | 143 | if !reflect.DeepEqual(got, test.wantInlineComment) { 144 | t.Errorf("got %v, but want %v", util_test.PrettyFormat(got), util_test.PrettyFormat(test.wantInlineComment)) 145 | } 146 | 147 | if !p.IsEOF() { 148 | t.Errorf("got not eof, but want eof") 149 | } 150 | }) 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /parser/mapField.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 5 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 6 | ) 7 | 8 | // MapField is an associative map. 9 | type MapField struct { 10 | KeyType string 11 | Type string 12 | MapName string 13 | FieldNumber string 14 | FieldOptions []*FieldOption 15 | 16 | // Comments are the optional ones placed at the beginning. 17 | Comments []*Comment 18 | // InlineComment is the optional one placed at the ending. 19 | InlineComment *Comment 20 | // Meta is the meta information. 21 | Meta meta.Meta 22 | } 23 | 24 | // SetInlineComment implements the HasInlineCommentSetter interface. 25 | func (m *MapField) SetInlineComment(comment *Comment) { 26 | m.InlineComment = comment 27 | } 28 | 29 | // Accept dispatches the call to the visitor. 30 | func (m *MapField) Accept(v Visitor) { 31 | if !v.VisitMapField(m) { 32 | return 33 | } 34 | 35 | for _, comment := range m.Comments { 36 | comment.Accept(v) 37 | } 38 | if m.InlineComment != nil { 39 | m.InlineComment.Accept(v) 40 | } 41 | } 42 | 43 | // ParseMapField parses the mapField. 44 | // 45 | // mapField = "map" "<" keyType "," type ">" mapName "=" fieldNumber [ "[" fieldOptions "]" ] ";" 46 | // 47 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#map_field 48 | func (p *Parser) ParseMapField() (*MapField, error) { 49 | p.lex.NextKeyword() 50 | if p.lex.Token != scanner.TMAP { 51 | return nil, p.unexpected("map") 52 | } 53 | startPos := p.lex.Pos 54 | 55 | p.lex.Next() 56 | if p.lex.Token != scanner.TLESS { 57 | return nil, p.unexpected("<") 58 | } 59 | 60 | keyType, err := p.parseKeyType() 61 | if err != nil { 62 | return nil, err 63 | } 64 | 65 | p.lex.Next() 66 | if p.lex.Token != scanner.TCOMMA { 67 | return nil, p.unexpected(",") 68 | } 69 | 70 | typeValue, _, err := p.parseType() 71 | if err != nil { 72 | return nil, p.unexpected("type") 73 | } 74 | 75 | p.lex.Next() 76 | if p.lex.Token != scanner.TGREATER { 77 | return nil, p.unexpected(">") 78 | } 79 | 80 | p.lex.Next() 81 | if p.lex.Token != scanner.TIDENT { 82 | return nil, p.unexpected("mapName") 83 | } 84 | mapName := p.lex.Text 85 | 86 | p.lex.Next() 87 | if p.lex.Token != scanner.TEQUALS { 88 | return nil, p.unexpected("=") 89 | } 90 | 91 | fieldNumber, err := p.parseFieldNumber() 92 | if err != nil { 93 | return nil, p.unexpected("fieldNumber") 94 | } 95 | 96 | fieldOptions, err := p.parseFieldOptionsOption() 97 | if err != nil { 98 | return nil, err 99 | } 100 | 101 | p.lex.Next() 102 | if p.lex.Token != scanner.TSEMICOLON { 103 | return nil, p.unexpected(";") 104 | } 105 | 106 | return &MapField{ 107 | KeyType: keyType, 108 | Type: typeValue, 109 | MapName: mapName, 110 | FieldNumber: fieldNumber, 111 | FieldOptions: fieldOptions, 112 | Meta: meta.Meta{Pos: startPos.Position, LastPos: p.lex.Pos.Position}, 113 | }, nil 114 | } 115 | 116 | var keyTypeConstants = map[string]struct{}{ 117 | "int32": {}, 118 | "int64": {}, 119 | "uint32": {}, 120 | "uint64": {}, 121 | "sint32": {}, 122 | "sint64": {}, 123 | "fixed32": {}, 124 | "fixed64": {}, 125 | "sfixed32": {}, 126 | "sfixed64": {}, 127 | "bool": {}, 128 | "string": {}, 129 | } 130 | 131 | // keyType = "int32" | "int64" | "uint32" | "uint64" | "sint32" | "sint64" | 132 | // 133 | // "fixed32" | "fixed64" | "sfixed32" | "sfixed64" | "bool" | "string" 134 | // 135 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#map_field 136 | func (p *Parser) parseKeyType() (string, error) { 137 | p.lex.Next() 138 | if _, ok := keyTypeConstants[p.lex.Text]; ok { 139 | return p.lex.Text, nil 140 | } 141 | return "", p.unexpected("keyType constant") 142 | } 143 | -------------------------------------------------------------------------------- /parser/mapField_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/yoheimuta/go-protoparser/v4/internal/util_test" 9 | "github.com/yoheimuta/go-protoparser/v4/lexer" 10 | "github.com/yoheimuta/go-protoparser/v4/parser" 11 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 12 | ) 13 | 14 | func TestParser_ParseMapField(t *testing.T) { 15 | tests := []struct { 16 | name string 17 | input string 18 | wantMapField *parser.MapField 19 | wantErr bool 20 | }{ 21 | { 22 | name: "parsing an empty", 23 | wantErr: true, 24 | }, 25 | { 26 | name: "parsing an invalid; without map", 27 | input: " projects = 3;", 28 | wantErr: true, 29 | }, 30 | { 31 | name: "parsing an invalid; not keyType constant", 32 | input: "map projects = 3;", 33 | wantErr: true, 34 | }, 35 | { 36 | name: "parsing an excerpt from the official reference", 37 | input: "map projects = 3;", 38 | wantMapField: &parser.MapField{ 39 | KeyType: "string", 40 | Type: "Project", 41 | MapName: "projects", 42 | FieldNumber: "3", 43 | Meta: meta.Meta{ 44 | Pos: meta.Position{ 45 | Offset: 0, 46 | Line: 1, 47 | Column: 1, 48 | }, 49 | LastPos: meta.Position{ 50 | Offset: 33, 51 | Line: 1, 52 | Column: 34, 53 | }, 54 | }, 55 | }, 56 | }, 57 | } 58 | 59 | for _, test := range tests { 60 | test := test 61 | t.Run(test.name, func(t *testing.T) { 62 | p := parser.NewParser(lexer.NewLexer(strings.NewReader(test.input))) 63 | got, err := p.ParseMapField() 64 | switch { 65 | case test.wantErr: 66 | if err == nil { 67 | t.Errorf("got err nil, but want err") 68 | } 69 | return 70 | case !test.wantErr && err != nil: 71 | t.Errorf("got err %v, but want nil", err) 72 | return 73 | } 74 | 75 | if !reflect.DeepEqual(got, test.wantMapField) { 76 | t.Errorf("got %v, but want %v", util_test.PrettyFormat(got), util_test.PrettyFormat(test.wantMapField)) 77 | } 78 | 79 | if !p.IsEOF() { 80 | t.Errorf("got not eof, but want eof") 81 | } 82 | }) 83 | } 84 | 85 | } 86 | -------------------------------------------------------------------------------- /parser/meta/error.go: -------------------------------------------------------------------------------- 1 | package meta 2 | 3 | import "fmt" 4 | 5 | // Error is the error type returned for all scanning/lexing/parsing related errors. 6 | type Error struct { 7 | Pos Position 8 | Expected string 9 | Found string 10 | 11 | occuredIn string 12 | occuredAt int 13 | } 14 | 15 | func (e *Error) Error() string { 16 | if e.occuredAt == 0 && e.occuredIn == "" { 17 | return fmt.Sprintf("found %q but expected [%s]", e.Found, e.Expected) 18 | } 19 | return fmt.Sprintf("found %q but expected [%s] at %s:%d", e.Found, e.Expected, e.occuredIn, e.occuredAt) 20 | } 21 | 22 | // SetOccured sets the file and the line number at which the error was raised (through runtime.Caller). 23 | func (e *Error) SetOccured(occuredIn string, occuredAt int) { 24 | e.occuredIn = occuredIn 25 | e.occuredAt = occuredAt 26 | } 27 | -------------------------------------------------------------------------------- /parser/meta/meta.go: -------------------------------------------------------------------------------- 1 | package meta 2 | 3 | // Meta represents a meta information about the parsed element. 4 | type Meta struct { 5 | // Pos is the source position. 6 | Pos Position 7 | // LastPos is the last source position. 8 | // Currently it is set when the parsed element type is 9 | // syntax, package, comment, import, option, message, enum, oneof, rpc or service. 10 | LastPos Position 11 | } 12 | -------------------------------------------------------------------------------- /parser/meta/position.go: -------------------------------------------------------------------------------- 1 | package meta 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // Position represents a source position. 8 | type Position struct { 9 | // Filename is a name of file, if any 10 | Filename string 11 | // Offset is a byte offset, starting at 0 12 | Offset int 13 | // Line is a line number, starting at 1 14 | Line int 15 | // Column is a column number, starting at 1 (character count per line) 16 | Column int 17 | } 18 | 19 | // String stringify the position. 20 | func (pos Position) String() string { 21 | s := pos.Filename 22 | if s == "" { 23 | s = "" 24 | } 25 | s += fmt.Sprintf(":%d:%d", pos.Line, pos.Column) 26 | return s 27 | } 28 | -------------------------------------------------------------------------------- /parser/meta/position_test.go: -------------------------------------------------------------------------------- 1 | package meta_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 7 | ) 8 | 9 | func TestPosition_String(t *testing.T) { 10 | tests := []struct { 11 | name string 12 | inputPos meta.Position 13 | wantString string 14 | }{ 15 | { 16 | name: "pos without Filename", 17 | inputPos: meta.Position{ 18 | Offset: 0, 19 | Line: 1, 20 | Column: 1, 21 | }, 22 | wantString: `:1:1`, 23 | }, 24 | { 25 | name: "pos with Filename", 26 | inputPos: meta.Position{ 27 | Filename: "test.proto", 28 | Offset: 0, 29 | Line: 1, 30 | Column: 1, 31 | }, 32 | wantString: `test.proto:1:1`, 33 | }, 34 | } 35 | 36 | for _, test := range tests { 37 | test := test 38 | t.Run(test.name, func(t *testing.T) { 39 | got := test.inputPos.String() 40 | if got != test.wantString { 41 | t.Errorf("got %s, but want %s", got, test.wantString) 42 | } 43 | }) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /parser/oneof.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 5 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 6 | ) 7 | 8 | // OneofField is a constituent field of oneof. 9 | type OneofField struct { 10 | Type string 11 | FieldName string 12 | FieldNumber string 13 | FieldOptions []*FieldOption 14 | 15 | // Comments are the optional ones placed at the beginning. 16 | Comments []*Comment 17 | // InlineComment is the optional one placed at the ending. 18 | InlineComment *Comment 19 | // Meta is the meta information. 20 | Meta meta.Meta 21 | } 22 | 23 | // SetInlineComment implements the HasInlineCommentSetter interface. 24 | func (f *OneofField) SetInlineComment(comment *Comment) { 25 | f.InlineComment = comment 26 | } 27 | 28 | // Accept dispatches the call to the visitor. 29 | func (f *OneofField) Accept(v Visitor) { 30 | if !v.VisitOneofField(f) { 31 | return 32 | } 33 | 34 | for _, comment := range f.Comments { 35 | comment.Accept(v) 36 | } 37 | if f.InlineComment != nil { 38 | f.InlineComment.Accept(v) 39 | } 40 | } 41 | 42 | // Oneof consists of oneof fields and a oneof name. 43 | type Oneof struct { 44 | OneofFields []*OneofField 45 | OneofName string 46 | 47 | Options []*Option 48 | 49 | // Comments are the optional ones placed at the beginning. 50 | Comments []*Comment 51 | // InlineComment is the optional one placed at the ending. 52 | InlineComment *Comment 53 | // InlineCommentBehindLeftCurly is the optional one placed behind a left curly. 54 | InlineCommentBehindLeftCurly *Comment 55 | // Meta is the meta information. 56 | Meta meta.Meta 57 | } 58 | 59 | // SetInlineComment implements the HasInlineCommentSetter interface. 60 | func (o *Oneof) SetInlineComment(comment *Comment) { 61 | o.InlineComment = comment 62 | } 63 | 64 | // Accept dispatches the call to the visitor. 65 | func (o *Oneof) Accept(v Visitor) { 66 | if !v.VisitOneof(o) { 67 | return 68 | } 69 | 70 | for _, field := range o.OneofFields { 71 | field.Accept(v) 72 | } 73 | for _, option := range o.Options { 74 | option.Accept(v) 75 | } 76 | for _, comment := range o.Comments { 77 | comment.Accept(v) 78 | } 79 | if o.InlineComment != nil { 80 | o.InlineComment.Accept(v) 81 | } 82 | } 83 | 84 | // ParseOneof parses the oneof. 85 | // 86 | // oneof = "oneof" oneofName "{" { option | oneofField | emptyStatement } "}" 87 | // 88 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#oneof_and_oneof_field 89 | func (p *Parser) ParseOneof() (*Oneof, error) { 90 | p.lex.NextKeyword() 91 | if p.lex.Token != scanner.TONEOF { 92 | return nil, p.unexpected("oneof") 93 | } 94 | startPos := p.lex.Pos 95 | 96 | p.lex.Next() 97 | if p.lex.Token != scanner.TIDENT { 98 | return nil, p.unexpected("oneofName") 99 | } 100 | oneofName := p.lex.Text 101 | 102 | p.lex.Next() 103 | if p.lex.Token != scanner.TLEFTCURLY { 104 | return nil, p.unexpected("{") 105 | } 106 | 107 | inlineLeftCurly := p.parseInlineComment() 108 | 109 | var oneofFields []*OneofField 110 | var options []*Option 111 | for { 112 | comments := p.ParseComments() 113 | 114 | err := p.lex.ReadEmptyStatement() 115 | if err == nil { 116 | continue 117 | } 118 | 119 | p.lex.NextKeyword() 120 | token := p.lex.Token 121 | p.lex.UnNext() 122 | if token == scanner.TOPTION { 123 | // See https://github.com/yoheimuta/go-protoparser/issues/57 124 | option, err := p.ParseOption() 125 | if err != nil { 126 | return nil, err 127 | } 128 | option.Comments = comments 129 | p.MaybeScanInlineComment(option) 130 | options = append(options, option) 131 | } else { 132 | oneofField, err := p.parseOneofField() 133 | if err != nil { 134 | return nil, err 135 | } 136 | oneofField.Comments = comments 137 | p.MaybeScanInlineComment(oneofField) 138 | oneofFields = append(oneofFields, oneofField) 139 | } 140 | 141 | p.lex.Next() 142 | if p.lex.Token == scanner.TRIGHTCURLY { 143 | break 144 | } else { 145 | p.lex.UnNext() 146 | } 147 | } 148 | 149 | lastPos := p.lex.Pos 150 | if p.permissive { 151 | // accept a block followed by semicolon. See https://github.com/yoheimuta/go-protoparser/v4/issues/30. 152 | p.lex.ConsumeToken(scanner.TSEMICOLON) 153 | if p.lex.Token == scanner.TSEMICOLON { 154 | lastPos = p.lex.Pos 155 | } 156 | } 157 | 158 | return &Oneof{ 159 | OneofFields: oneofFields, 160 | OneofName: oneofName, 161 | Options: options, 162 | InlineCommentBehindLeftCurly: inlineLeftCurly, 163 | Meta: meta.Meta{ 164 | Pos: startPos.Position, 165 | LastPos: lastPos.Position, 166 | }, 167 | }, nil 168 | } 169 | 170 | // oneofField = type fieldName "=" fieldNumber [ "[" fieldOptions "]" ] ";" 171 | // https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#oneof_and_oneof_field 172 | func (p *Parser) parseOneofField() (*OneofField, error) { 173 | typeValue, startPos, err := p.parseType() 174 | if err != nil { 175 | return nil, p.unexpected("type") 176 | } 177 | 178 | p.lex.Next() 179 | if p.lex.Token != scanner.TIDENT { 180 | return nil, p.unexpected("fieldName") 181 | } 182 | fieldName := p.lex.Text 183 | 184 | p.lex.Next() 185 | if p.lex.Token != scanner.TEQUALS { 186 | return nil, p.unexpected("=") 187 | } 188 | 189 | fieldNumber, err := p.parseFieldNumber() 190 | if err != nil { 191 | return nil, p.unexpected("fieldNumber") 192 | } 193 | 194 | fieldOptions, err := p.parseFieldOptionsOption() 195 | if err != nil { 196 | return nil, err 197 | } 198 | 199 | p.lex.Next() 200 | if p.lex.Token != scanner.TSEMICOLON { 201 | return nil, p.unexpected(";") 202 | } 203 | 204 | return &OneofField{ 205 | Type: typeValue, 206 | FieldName: fieldName, 207 | FieldNumber: fieldNumber, 208 | FieldOptions: fieldOptions, 209 | Meta: meta.Meta{Pos: startPos.Position, LastPos: p.lex.Pos.Position}, 210 | }, nil 211 | } 212 | -------------------------------------------------------------------------------- /parser/package.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 5 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 6 | ) 7 | 8 | // Package can be used to prevent name clashes between protocol message types. 9 | type Package struct { 10 | Name string 11 | 12 | // Comments are the optional ones placed at the beginning. 13 | Comments []*Comment 14 | // InlineComment is the optional one placed at the ending. 15 | InlineComment *Comment 16 | // Meta is the meta information. 17 | Meta meta.Meta 18 | } 19 | 20 | // SetInlineComment implements the HasInlineCommentSetter interface. 21 | func (p *Package) SetInlineComment(comment *Comment) { 22 | p.InlineComment = comment 23 | } 24 | 25 | // Accept dispatches the call to the visitor. 26 | func (p *Package) Accept(v Visitor) { 27 | if !v.VisitPackage(p) { 28 | return 29 | } 30 | 31 | for _, comment := range p.Comments { 32 | comment.Accept(v) 33 | } 34 | if p.InlineComment != nil { 35 | p.InlineComment.Accept(v) 36 | } 37 | } 38 | 39 | // ParsePackage parses the package. 40 | // 41 | // package = "package" fullIdent ";" 42 | // 43 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#package 44 | func (p *Parser) ParsePackage() (*Package, error) { 45 | p.lex.NextKeyword() 46 | if p.lex.Token != scanner.TPACKAGE { 47 | return nil, p.unexpected("package") 48 | } 49 | startPos := p.lex.Pos 50 | 51 | ident, _, err := p.lex.ReadFullIdent() 52 | if err != nil { 53 | return nil, p.unexpected("fullIdent") 54 | } 55 | 56 | p.lex.Next() 57 | if p.lex.Token != scanner.TSEMICOLON { 58 | return nil, p.unexpected(";") 59 | } 60 | 61 | return &Package{ 62 | Name: ident, 63 | Meta: meta.Meta{ 64 | Pos: startPos.Position, 65 | LastPos: p.lex.Pos.Position, 66 | }, 67 | }, nil 68 | } 69 | -------------------------------------------------------------------------------- /parser/package_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/yoheimuta/go-protoparser/v4/lexer" 9 | "github.com/yoheimuta/go-protoparser/v4/parser" 10 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 11 | ) 12 | 13 | func TestParser_ParsePackage(t *testing.T) { 14 | tests := []struct { 15 | name string 16 | input string 17 | wantPackage *parser.Package 18 | wantErr bool 19 | }{ 20 | { 21 | name: "parsing an empty", 22 | wantErr: true, 23 | }, 24 | { 25 | name: "parsing an excerpt from the official reference", 26 | input: `package foo.bar;`, 27 | wantPackage: &parser.Package{ 28 | Name: "foo.bar", 29 | Meta: meta.Meta{ 30 | Pos: meta.Position{ 31 | Offset: 0, 32 | Line: 1, 33 | Column: 1, 34 | }, 35 | LastPos: meta.Position{ 36 | Offset: 15, 37 | Line: 1, 38 | Column: 16, 39 | }, 40 | }, 41 | }, 42 | }, 43 | } 44 | 45 | for _, test := range tests { 46 | test := test 47 | t.Run(test.name, func(t *testing.T) { 48 | p := parser.NewParser(lexer.NewLexer(strings.NewReader(test.input))) 49 | got, err := p.ParsePackage() 50 | switch { 51 | case test.wantErr: 52 | if err == nil { 53 | t.Errorf("got err nil, but want err") 54 | } 55 | return 56 | case !test.wantErr && err != nil: 57 | t.Errorf("got err %v, but want nil", err) 58 | return 59 | } 60 | 61 | if !reflect.DeepEqual(got, test.wantPackage) { 62 | t.Errorf("got %v, but want %v", got, test.wantPackage) 63 | } 64 | 65 | if !p.IsEOF() { 66 | t.Errorf("got not eof, but want eof") 67 | } 68 | }) 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /parser/parser.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import "github.com/yoheimuta/go-protoparser/v4/lexer" 4 | 5 | // Parser is a parser. 6 | type Parser struct { 7 | lex *lexer.Lexer 8 | 9 | permissive bool 10 | bodyIncludingComments bool 11 | } 12 | 13 | // ConfigOption is an option for Parser. 14 | type ConfigOption func(*Parser) 15 | 16 | // WithPermissive is an option to allow the permissive parsing rather than the just documented spec. 17 | func WithPermissive(permissive bool) ConfigOption { 18 | return func(p *Parser) { 19 | p.permissive = permissive 20 | } 21 | } 22 | 23 | // WithBodyIncludingComments is an option to allow to include comments into each element's body. 24 | // The comments are remaining of other elements'Comments and InlineComment. 25 | func WithBodyIncludingComments(bodyIncludingComments bool) ConfigOption { 26 | return func(p *Parser) { 27 | p.bodyIncludingComments = bodyIncludingComments 28 | } 29 | } 30 | 31 | // NewParser creates a new Parser. 32 | func NewParser(lex *lexer.Lexer, opts ...ConfigOption) *Parser { 33 | p := &Parser{ 34 | lex: lex, 35 | } 36 | for _, opt := range opts { 37 | opt(p) 38 | } 39 | return p 40 | } 41 | 42 | // IsEOF checks whether the lex's read buffer is empty. 43 | func (p *Parser) IsEOF() bool { 44 | p.lex.Next() 45 | defer p.lex.UnNext() 46 | return p.lex.IsEOF() 47 | } 48 | -------------------------------------------------------------------------------- /parser/proto.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 4 | 5 | // ProtoMeta represents a meta information about the Proto. 6 | type ProtoMeta struct { 7 | // Filename is a name of file, if any. 8 | Filename string 9 | } 10 | 11 | // Proto represents a protocol buffer definition. 12 | type Proto struct { 13 | Syntax *Syntax 14 | Edition *Edition 15 | // ProtoBody is a slice of sum type consisted of *Import, *Package, *Option, *Message, *Enum, *Service, *Extend and *EmptyStatement. 16 | ProtoBody []Visitee 17 | Meta *ProtoMeta 18 | } 19 | 20 | // Accept dispatches the call to the visitor. 21 | func (p *Proto) Accept(v Visitor) { 22 | if p.Syntax != nil { 23 | p.Syntax.Accept(v) 24 | } 25 | if p.Edition != nil { 26 | p.Edition.Accept(v) 27 | } 28 | 29 | for _, body := range p.ProtoBody { 30 | body.Accept(v) 31 | } 32 | } 33 | 34 | // ParseProto parses the proto. 35 | // 36 | // proto = [syntax] [edition] { import | package | option | topLevelDef | emptyStatement } 37 | // 38 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#proto_file 39 | // See https://protobuf.dev/reference/protobuf/edition-2023-spec/#proto_file 40 | func (p *Parser) ParseProto() (*Proto, error) { 41 | p.parseBOM() 42 | 43 | comments := p.ParseComments() 44 | syntax, err := p.ParseSyntax() 45 | if err != nil { 46 | return nil, err 47 | } 48 | if syntax != nil { 49 | syntax.Comments = comments 50 | p.MaybeScanInlineComment(syntax) 51 | } 52 | 53 | edition, err := p.ParseEdition() 54 | if err != nil { 55 | return nil, err 56 | } 57 | if edition != nil { 58 | edition.Comments = comments 59 | p.MaybeScanInlineComment(edition) 60 | } 61 | 62 | protoBody, err := p.parseProtoBody() 63 | if err != nil { 64 | return nil, err 65 | } 66 | 67 | return &Proto{ 68 | Syntax: syntax, 69 | Edition: edition, 70 | ProtoBody: protoBody, 71 | Meta: &ProtoMeta{ 72 | Filename: p.lex.Pos.Filename, 73 | }, 74 | }, nil 75 | } 76 | 77 | // See https://protobuf.com/docs/language-spec#source-code-representation 78 | func (p *Parser) parseBOM() { 79 | p.lex.Next() 80 | if p.lex.Token == scanner.TBOM { 81 | return 82 | } 83 | defer p.lex.UnNext() 84 | } 85 | 86 | // protoBody = { import | package | option | topLevelDef | emptyStatement } 87 | // topLevelDef = message | enum | service | extend 88 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#proto_file 89 | func (p *Parser) parseProtoBody() ([]Visitee, error) { 90 | var protoBody []Visitee 91 | 92 | for { 93 | comments := p.ParseComments() 94 | 95 | if p.IsEOF() { 96 | if p.bodyIncludingComments { 97 | for _, comment := range comments { 98 | protoBody = append(protoBody, Visitee(comment)) 99 | } 100 | } 101 | return protoBody, nil 102 | } 103 | 104 | p.lex.NextKeyword() 105 | token := p.lex.Token 106 | p.lex.UnNext() 107 | 108 | var stmt interface { 109 | HasInlineCommentSetter 110 | Visitee 111 | } 112 | 113 | switch token { 114 | case scanner.TIMPORT: 115 | importValue, err := p.ParseImport() 116 | if err != nil { 117 | return nil, err 118 | } 119 | importValue.Comments = comments 120 | stmt = importValue 121 | case scanner.TPACKAGE: 122 | packageValue, err := p.ParsePackage() 123 | if err != nil { 124 | return nil, err 125 | } 126 | packageValue.Comments = comments 127 | stmt = packageValue 128 | case scanner.TOPTION: 129 | option, err := p.ParseOption() 130 | if err != nil { 131 | return nil, err 132 | } 133 | option.Comments = comments 134 | stmt = option 135 | case scanner.TMESSAGE: 136 | message, err := p.ParseMessage() 137 | if err != nil { 138 | return nil, err 139 | } 140 | message.Comments = comments 141 | stmt = message 142 | case scanner.TENUM: 143 | enum, err := p.ParseEnum() 144 | if err != nil { 145 | return nil, err 146 | } 147 | enum.Comments = comments 148 | stmt = enum 149 | case scanner.TSERVICE: 150 | service, err := p.ParseService() 151 | if err != nil { 152 | return nil, err 153 | } 154 | service.Comments = comments 155 | stmt = service 156 | case scanner.TEXTEND: 157 | extend, err := p.ParseExtend() 158 | if err != nil { 159 | return nil, err 160 | } 161 | extend.Comments = comments 162 | stmt = extend 163 | default: 164 | err := p.lex.ReadEmptyStatement() 165 | if err != nil { 166 | return nil, err 167 | } 168 | protoBody = append(protoBody, &EmptyStatement{}) 169 | } 170 | 171 | p.MaybeScanInlineComment(stmt) 172 | protoBody = append(protoBody, stmt) 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /parser/reserved.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 7 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 8 | ) 9 | 10 | type parseReservedErr struct { 11 | parseRangesErr error 12 | parseFieldNamesErr error 13 | } 14 | 15 | func (e *parseReservedErr) Error() string { 16 | return fmt.Sprintf("%v:%v", e.parseRangesErr, e.parseFieldNamesErr) 17 | } 18 | 19 | // Range is a range of field numbers. End is an optional value. 20 | type Range struct { 21 | Begin string 22 | End string 23 | } 24 | 25 | // Reserved declares a range of field numbers or field names that cannot be used in this message. 26 | // These component Ranges and FieldNames are mutually exclusive. 27 | type Reserved struct { 28 | Ranges []*Range 29 | FieldNames []string 30 | 31 | // Comments are the optional ones placed at the beginning. 32 | Comments []*Comment 33 | // InlineComment is the optional one placed at the ending. 34 | InlineComment *Comment 35 | // Meta is the meta information. 36 | Meta meta.Meta 37 | } 38 | 39 | // SetInlineComment implements the HasInlineCommentSetter interface. 40 | func (r *Reserved) SetInlineComment(comment *Comment) { 41 | r.InlineComment = comment 42 | } 43 | 44 | // Accept dispatches the call to the visitor. 45 | func (r *Reserved) Accept(v Visitor) { 46 | if !v.VisitReserved(r) { 47 | return 48 | } 49 | 50 | for _, comment := range r.Comments { 51 | comment.Accept(v) 52 | } 53 | if r.InlineComment != nil { 54 | r.InlineComment.Accept(v) 55 | } 56 | } 57 | 58 | // ParseReserved parses the reserved. 59 | // 60 | // reserved = "reserved" ( ranges | fieldNames ) ";" 61 | // reserved = "reserved" ( ranges | reservedIdent ) ";" 62 | // 63 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#reserved 64 | // See https://protobuf.dev/reference/protobuf/edition-2023-spec/#reserved 65 | func (p *Parser) ParseReserved() (*Reserved, error) { 66 | p.lex.NextKeyword() 67 | if p.lex.Token != scanner.TRESERVED { 68 | return nil, p.unexpected("reserved") 69 | } 70 | startPos := p.lex.Pos 71 | 72 | parse := func() ([]*Range, []string, error) { 73 | ranges, err := p.parseRanges() 74 | if err == nil { 75 | return ranges, nil, nil 76 | } 77 | 78 | fieldNames, ferr := p.parseFieldNames() 79 | if ferr == nil { 80 | return nil, fieldNames, nil 81 | } 82 | 83 | return nil, nil, &parseReservedErr{ 84 | parseRangesErr: err, 85 | parseFieldNamesErr: ferr, 86 | } 87 | } 88 | 89 | ranges, fieldNames, err := parse() 90 | if err != nil { 91 | return nil, err 92 | } 93 | 94 | p.lex.Next() 95 | if p.lex.Token != scanner.TSEMICOLON { 96 | return nil, p.unexpected(";") 97 | } 98 | 99 | return &Reserved{ 100 | Ranges: ranges, 101 | FieldNames: fieldNames, 102 | Meta: meta.Meta{Pos: startPos.Position, LastPos: p.lex.Pos.Position}, 103 | }, nil 104 | } 105 | 106 | // ranges = range { "," range } 107 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#reserved 108 | func (p *Parser) parseRanges() ([]*Range, error) { 109 | var ranges []*Range 110 | rangeValue, err := p.parseRange() 111 | if err != nil { 112 | return nil, err 113 | } 114 | ranges = append(ranges, rangeValue) 115 | 116 | for { 117 | p.lex.Next() 118 | if p.lex.Token != scanner.TCOMMA { 119 | p.lex.UnNext() 120 | break 121 | } 122 | 123 | rangeValue, err := p.parseRange() 124 | if err != nil { 125 | return nil, err 126 | } 127 | ranges = append(ranges, rangeValue) 128 | } 129 | return ranges, nil 130 | } 131 | 132 | // range = intLit [ "to" ( intLit | "max" ) ] 133 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#reserved 134 | func (p *Parser) parseRange() (*Range, error) { 135 | p.lex.NextNumberLit() 136 | if p.lex.Token != scanner.TINTLIT { 137 | p.lex.UnNext() 138 | return nil, p.unexpected("intLit") 139 | } 140 | begin := p.lex.Text 141 | 142 | p.lex.Next() 143 | if p.lex.Text != "to" { 144 | p.lex.UnNext() 145 | return &Range{ 146 | Begin: begin, 147 | }, nil 148 | } 149 | 150 | p.lex.NextNumberLit() 151 | switch { 152 | case p.lex.Token == scanner.TINTLIT, 153 | p.lex.Text == "max": 154 | return &Range{ 155 | Begin: begin, 156 | End: p.lex.Text, 157 | }, nil 158 | default: 159 | break 160 | } 161 | return nil, p.unexpected(`"intLit | "max"`) 162 | } 163 | 164 | // fieldNames = fieldName { "," fieldName } 165 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#reserved 166 | // Note: While the spec requires commas between field names, this parser also supports 167 | // field names separated by whitespace without commas, which is not mentioned in the spec 168 | // but is supported by protoc and other parsers. 169 | func (p *Parser) parseFieldNames() ([]string, error) { 170 | var fieldNames []string 171 | 172 | fieldName, err := p.parseFieldName() 173 | if err != nil { 174 | return nil, err 175 | } 176 | fieldNames = append(fieldNames, fieldName) 177 | 178 | for { 179 | // Check if next token is a comma 180 | p.lex.Next() 181 | if p.lex.Token == scanner.TCOMMA { 182 | // If it's a comma, parse the next field name 183 | fieldName, err = p.parseFieldName() 184 | if err != nil { 185 | return nil, err 186 | } 187 | fieldNames = append(fieldNames, fieldName) 188 | } else { 189 | // If it's not a comma, put it back and try to parse another field name 190 | p.lex.UnNext() 191 | 192 | // Try to parse another field name 193 | nextFieldName, err := p.parseFieldName() 194 | if err != nil { 195 | // If parsing fails, we're done with field names 196 | break 197 | } 198 | 199 | // Successfully parsed another field name 200 | fieldNames = append(fieldNames, nextFieldName) 201 | } 202 | } 203 | return fieldNames, nil 204 | } 205 | 206 | // fieldName = quote + fieldName + quote 207 | func (p *Parser) parseFieldName() (string, error) { 208 | quoted, err := p.parseQuotedFieldName() 209 | if err == nil { 210 | return quoted, nil 211 | } 212 | 213 | // If it is not a quotedFieldName, it should be a fieldName in Editions. 214 | p.lex.Next() 215 | if p.lex.Token != scanner.TIDENT { 216 | p.lex.UnNext() 217 | return "", p.unexpected("fieldName or quotedFieldName") 218 | } 219 | return p.lex.Text, nil 220 | } 221 | 222 | // quotedFieldName = quote + fieldName + quote 223 | // TODO: Fixed according to defined documentation. Currently(2018.10.16) the reference lacks the spec. 224 | // See https://github.com/protocolbuffers/protobuf/issues/4558 225 | func (p *Parser) parseQuotedFieldName() (string, error) { 226 | p.lex.NextStrLit() 227 | if p.lex.Token != scanner.TSTRLIT { 228 | p.lex.UnNext() 229 | return "", p.unexpected("quotedFieldName") 230 | } 231 | return p.lex.Text, nil 232 | } 233 | -------------------------------------------------------------------------------- /parser/reserved_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/yoheimuta/go-protoparser/v4/internal/util_test" 9 | "github.com/yoheimuta/go-protoparser/v4/lexer" 10 | "github.com/yoheimuta/go-protoparser/v4/parser" 11 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 12 | ) 13 | 14 | func TestParser_ParseReserved(t *testing.T) { 15 | tests := []struct { 16 | name string 17 | input string 18 | wantReserved *parser.Reserved 19 | wantErr bool 20 | }{ 21 | { 22 | name: "parsing an empty", 23 | wantErr: true, 24 | }, 25 | { 26 | name: "parsing an invalid; without to", 27 | input: "reserved 2, 15, 9 11;", 28 | wantErr: true, 29 | }, 30 | { 31 | name: "parsing an invalid; including both ranges and fieldNames", 32 | input: `reserved 2, "foo", 9 to 11;`, 33 | wantErr: true, 34 | }, 35 | { 36 | name: "parsing an excerpt from the official reference", 37 | input: "reserved 2, 15, 9 to 11;", 38 | wantReserved: &parser.Reserved{ 39 | Ranges: []*parser.Range{ 40 | { 41 | Begin: "2", 42 | }, 43 | { 44 | Begin: "15", 45 | }, 46 | { 47 | Begin: "9", 48 | End: "11", 49 | }, 50 | }, 51 | Meta: meta.Meta{ 52 | Pos: meta.Position{ 53 | Offset: 0, 54 | Line: 1, 55 | Column: 1, 56 | }, 57 | LastPos: meta.Position{ 58 | Offset: 23, 59 | Line: 1, 60 | Column: 24, 61 | }, 62 | }, 63 | }, 64 | }, 65 | { 66 | name: "parsing another excerpt from the official reference", 67 | input: `reserved "foo", "bar";`, 68 | wantReserved: &parser.Reserved{ 69 | FieldNames: []string{ 70 | `"foo"`, 71 | `"bar"`, 72 | }, 73 | Meta: meta.Meta{ 74 | Pos: meta.Position{ 75 | Offset: 0, 76 | Line: 1, 77 | Column: 1, 78 | }, 79 | LastPos: meta.Position{ 80 | Offset: 21, 81 | Line: 1, 82 | Column: 22, 83 | }, 84 | }, 85 | }, 86 | }, 87 | { 88 | name: "parsing an input with max", 89 | input: "reserved 9 to max;", 90 | wantReserved: &parser.Reserved{ 91 | Ranges: []*parser.Range{ 92 | { 93 | Begin: "9", 94 | End: "max", 95 | }, 96 | }, 97 | Meta: meta.Meta{ 98 | Pos: meta.Position{ 99 | Offset: 0, 100 | Line: 1, 101 | Column: 1, 102 | }, 103 | LastPos: meta.Position{ 104 | Offset: 17, 105 | Line: 1, 106 | Column: 18, 107 | }, 108 | }, 109 | }, 110 | }, 111 | { 112 | name: "parsing reservedIdent from editions", 113 | input: `reserved foo, bar;`, 114 | wantReserved: &parser.Reserved{ 115 | FieldNames: []string{ 116 | `foo`, 117 | `bar`, 118 | }, 119 | Meta: meta.Meta{ 120 | Pos: meta.Position{ 121 | Offset: 0, 122 | Line: 1, 123 | Column: 1, 124 | }, 125 | LastPos: meta.Position{ 126 | Offset: 17, 127 | Line: 1, 128 | Column: 18, 129 | }, 130 | }, 131 | }, 132 | }, 133 | { 134 | name: "parsing field names on separate lines without commas. See #96", 135 | input: `reserved 136 | "skipped" 137 | "ignore_empty" 138 | ;`, 139 | wantReserved: &parser.Reserved{ 140 | FieldNames: []string{ 141 | `"skipped"`, 142 | `"ignore_empty"`, 143 | }, 144 | Meta: meta.Meta{ 145 | Pos: meta.Position{ 146 | Offset: 0, 147 | Line: 1, 148 | Column: 1, 149 | }, 150 | LastPos: meta.Position{ 151 | Offset: 36, 152 | Line: 4, 153 | Column: 1, 154 | }, 155 | }, 156 | }, 157 | }, 158 | { 159 | name: "parsing a mix of comma-separated and whitespace-separated field names", 160 | input: `reserved "field1", 161 | "field2" 162 | "field3", "field4";`, 163 | wantReserved: &parser.Reserved{ 164 | FieldNames: []string{ 165 | `"field1"`, 166 | `"field2"`, 167 | `"field3"`, 168 | `"field4"`, 169 | }, 170 | Meta: meta.Meta{ 171 | Pos: meta.Position{ 172 | Offset: 0, 173 | Line: 1, 174 | Column: 1, 175 | }, 176 | LastPos: meta.Position{ 177 | Offset: 49, 178 | Line: 3, 179 | Column: 20, 180 | }, 181 | }, 182 | }, 183 | }, 184 | } 185 | 186 | for _, test := range tests { 187 | test := test 188 | t.Run(test.name, func(t *testing.T) { 189 | p := parser.NewParser(lexer.NewLexer(strings.NewReader(test.input))) 190 | got, err := p.ParseReserved() 191 | switch { 192 | case test.wantErr: 193 | if err == nil { 194 | t.Errorf("got err nil, but want err") 195 | } 196 | return 197 | case !test.wantErr && err != nil: 198 | t.Errorf("got err %v, but want nil", err) 199 | return 200 | } 201 | 202 | if !reflect.DeepEqual(got, test.wantReserved) { 203 | t.Errorf("got %v, but want %v", util_test.PrettyFormat(got), util_test.PrettyFormat(test.wantReserved)) 204 | 205 | } 206 | 207 | if !p.IsEOF() { 208 | t.Errorf("got not eof, but want eof") 209 | } 210 | }) 211 | } 212 | 213 | } 214 | -------------------------------------------------------------------------------- /parser/syntax.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "github.com/yoheimuta/go-protoparser/v4/lexer/scanner" 5 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 6 | ) 7 | 8 | // Syntax is used to define the protobuf version. 9 | type Syntax struct { 10 | ProtobufVersion string 11 | 12 | // ProtobufVersionQuote includes quotes 13 | ProtobufVersionQuote string 14 | 15 | // Comments are the optional ones placed at the beginning. 16 | Comments []*Comment 17 | // InlineComment is the optional one placed at the ending. 18 | InlineComment *Comment 19 | // Meta is the meta information. 20 | Meta meta.Meta 21 | } 22 | 23 | // SetInlineComment implements the HasInlineCommentSetter interface. 24 | func (s *Syntax) SetInlineComment(comment *Comment) { 25 | s.InlineComment = comment 26 | } 27 | 28 | // Accept dispatches the call to the visitor. 29 | func (s *Syntax) Accept(v Visitor) { 30 | if !v.VisitSyntax(s) { 31 | return 32 | } 33 | 34 | for _, comment := range s.Comments { 35 | comment.Accept(v) 36 | } 37 | if s.InlineComment != nil { 38 | s.InlineComment.Accept(v) 39 | } 40 | } 41 | 42 | // Version returns the version number. 43 | func (s *Syntax) Version() int { 44 | switch s.ProtobufVersion { 45 | case "proto3": 46 | return 3 47 | case "proto2": 48 | return 2 49 | default: 50 | return 0 51 | } 52 | } 53 | 54 | // ParseSyntax parses the syntax. 55 | // 56 | // syntax = "syntax" "=" quote "proto3" quote ";" 57 | // syntax = "syntax" "=" quote "proto2" quote ";" 58 | // 59 | // See https://developers.google.com/protocol-buffers/docs/reference/proto3-spec#syntax 60 | func (p *Parser) ParseSyntax() (*Syntax, error) { 61 | p.lex.NextKeyword() 62 | if p.lex.Token != scanner.TSYNTAX { 63 | p.lex.UnNext() 64 | return nil, nil 65 | } 66 | startPos := p.lex.Pos 67 | 68 | p.lex.Next() 69 | if p.lex.Token != scanner.TEQUALS { 70 | return nil, p.unexpected("=") 71 | } 72 | 73 | p.lex.Next() 74 | if p.lex.Token != scanner.TQUOTE { 75 | return nil, p.unexpected("quote") 76 | } 77 | lq := p.lex.Text 78 | 79 | p.lex.Next() 80 | if p.lex.Text != "proto3" && p.lex.Text != "proto2" { 81 | return nil, p.unexpected("proto3 or proto2") 82 | } 83 | version := p.lex.Text 84 | 85 | p.lex.Next() 86 | if p.lex.Token != scanner.TQUOTE { 87 | return nil, p.unexpected("quote") 88 | } 89 | tq := p.lex.Text 90 | 91 | p.lex.Next() 92 | if p.lex.Token != scanner.TSEMICOLON { 93 | return nil, p.unexpected(";") 94 | } 95 | 96 | return &Syntax{ 97 | ProtobufVersion: version, 98 | ProtobufVersionQuote: lq + version + tq, 99 | Meta: meta.Meta{ 100 | Pos: startPos.Position, 101 | LastPos: p.lex.Pos.Position, 102 | }, 103 | }, nil 104 | } 105 | -------------------------------------------------------------------------------- /parser/syntax_test.go: -------------------------------------------------------------------------------- 1 | package parser_test 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/yoheimuta/go-protoparser/v4/lexer" 9 | "github.com/yoheimuta/go-protoparser/v4/parser" 10 | "github.com/yoheimuta/go-protoparser/v4/parser/meta" 11 | ) 12 | 13 | func TestParser_ParseSyntax(t *testing.T) { 14 | tests := []struct { 15 | name string 16 | input string 17 | wantSyntax *parser.Syntax 18 | wantErr bool 19 | }{ 20 | { 21 | name: "parsing an empty", 22 | }, 23 | { 24 | name: "parsing an excerpt from the official reference", 25 | input: `syntax = "proto3";`, 26 | wantSyntax: &parser.Syntax{ 27 | ProtobufVersion: "proto3", 28 | ProtobufVersionQuote: `"proto3"`, 29 | Meta: meta.Meta{ 30 | Pos: meta.Position{ 31 | Offset: 0, 32 | Line: 1, 33 | Column: 1, 34 | }, 35 | LastPos: meta.Position{ 36 | Offset: 17, 37 | Line: 1, 38 | Column: 18, 39 | }, 40 | }, 41 | }, 42 | }, 43 | { 44 | name: "parsing a single-quote string", 45 | input: `syntax = 'proto3';`, 46 | wantSyntax: &parser.Syntax{ 47 | ProtobufVersion: "proto3", 48 | ProtobufVersionQuote: `'proto3'`, 49 | Meta: meta.Meta{ 50 | Pos: meta.Position{ 51 | Offset: 0, 52 | Line: 1, 53 | Column: 1, 54 | }, 55 | LastPos: meta.Position{ 56 | Offset: 17, 57 | Line: 1, 58 | Column: 18, 59 | }, 60 | }, 61 | }, 62 | }, 63 | { 64 | name: "parsing an excerpt from the official reference(proto2)", 65 | input: `syntax = "proto2";`, 66 | wantSyntax: &parser.Syntax{ 67 | ProtobufVersion: "proto2", 68 | ProtobufVersionQuote: `"proto2"`, 69 | Meta: meta.Meta{ 70 | Pos: meta.Position{ 71 | Offset: 0, 72 | Line: 1, 73 | Column: 1, 74 | }, 75 | LastPos: meta.Position{ 76 | Offset: 17, 77 | Line: 1, 78 | Column: 18, 79 | }, 80 | }, 81 | }, 82 | }, 83 | } 84 | 85 | for _, test := range tests { 86 | test := test 87 | t.Run(test.name, func(t *testing.T) { 88 | p := parser.NewParser(lexer.NewLexer(strings.NewReader(test.input))) 89 | got, err := p.ParseSyntax() 90 | switch { 91 | case test.wantErr: 92 | if err == nil { 93 | t.Errorf("got err nil, but want err") 94 | } 95 | return 96 | case !test.wantErr && err != nil: 97 | t.Errorf("got err %v, but want nil", err) 98 | return 99 | } 100 | 101 | if !reflect.DeepEqual(got, test.wantSyntax) { 102 | t.Errorf("got %v, but want %v", got, test.wantSyntax) 103 | } 104 | 105 | if !p.IsEOF() { 106 | t.Errorf("got not eof, but want eof") 107 | } 108 | }) 109 | } 110 | 111 | } 112 | -------------------------------------------------------------------------------- /parser/visitee.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | // Visitee is implemented by all Protocol Buffer elements. 4 | type Visitee interface { 5 | Accept(v Visitor) 6 | } 7 | -------------------------------------------------------------------------------- /parser/visitor.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | // Visitor is for dispatching Protocol Buffer elements. 4 | type Visitor interface { 5 | VisitComment(*Comment) 6 | VisitDeclaration(*Declaration) (next bool) 7 | VisitEdition(*Edition) (next bool) 8 | VisitEmptyStatement(*EmptyStatement) (next bool) 9 | VisitEnum(*Enum) (next bool) 10 | VisitEnumField(*EnumField) (next bool) 11 | VisitExtend(*Extend) (next bool) 12 | VisitExtensions(*Extensions) (next bool) 13 | VisitField(*Field) (next bool) 14 | VisitGroupField(*GroupField) (next bool) 15 | VisitImport(*Import) (next bool) 16 | VisitMapField(*MapField) (next bool) 17 | VisitMessage(*Message) (next bool) 18 | VisitOneof(*Oneof) (next bool) 19 | VisitOneofField(*OneofField) (next bool) 20 | VisitOption(*Option) (next bool) 21 | VisitPackage(*Package) (next bool) 22 | VisitReserved(*Reserved) (next bool) 23 | VisitRPC(*RPC) (next bool) 24 | VisitService(*Service) (next bool) 25 | VisitSyntax(*Syntax) (next bool) 26 | } 27 | -------------------------------------------------------------------------------- /protoparser.go: -------------------------------------------------------------------------------- 1 | package protoparser 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/yoheimuta/go-protoparser/v4/interpret/unordered" 7 | "github.com/yoheimuta/go-protoparser/v4/lexer" 8 | "github.com/yoheimuta/go-protoparser/v4/parser" 9 | ) 10 | 11 | // ParseConfig is a config for parser. 12 | type ParseConfig struct { 13 | debug bool 14 | permissive bool 15 | bodyIncludingComments bool 16 | filename string 17 | } 18 | 19 | // Option is an option for ParseConfig. 20 | type Option func(*ParseConfig) 21 | 22 | // WithDebug is an option to enable the debug mode. 23 | func WithDebug(debug bool) Option { 24 | return func(c *ParseConfig) { 25 | c.debug = debug 26 | } 27 | } 28 | 29 | // WithPermissive is an option to allow the permissive parsing rather than the just documented spec. 30 | func WithPermissive(permissive bool) Option { 31 | return func(c *ParseConfig) { 32 | c.permissive = permissive 33 | } 34 | } 35 | 36 | // WithBodyIncludingComments is an option to allow to include comments into each element's body. 37 | // The comments are remaining of other elements'Comments and InlineComment. 38 | func WithBodyIncludingComments(bodyIncludingComments bool) Option { 39 | return func(c *ParseConfig) { 40 | c.bodyIncludingComments = bodyIncludingComments 41 | } 42 | } 43 | 44 | // WithFilename is an option to set filename to the Position. 45 | func WithFilename(filename string) Option { 46 | return func(c *ParseConfig) { 47 | c.filename = filename 48 | } 49 | } 50 | 51 | // Parse parses a Protocol Buffer file. 52 | func Parse(input io.Reader, options ...Option) (*parser.Proto, error) { 53 | config := &ParseConfig{ 54 | permissive: true, 55 | } 56 | for _, opt := range options { 57 | opt(config) 58 | } 59 | 60 | p := parser.NewParser( 61 | lexer.NewLexer( 62 | input, 63 | lexer.WithDebug(config.debug), 64 | lexer.WithFilename(config.filename), 65 | ), 66 | parser.WithPermissive(config.permissive), 67 | parser.WithBodyIncludingComments(config.bodyIncludingComments), 68 | ) 69 | return p.ParseProto() 70 | } 71 | 72 | // UnorderedInterpret interprets a Proto to an unordered one without interface{}. 73 | func UnorderedInterpret(proto *parser.Proto) (*unordered.Proto, error) { 74 | return unordered.InterpretProto(proto) 75 | } 76 | --------------------------------------------------------------------------------